From efb84df3ee5e7a6cabee3b057e3addcb575e4542 Mon Sep 17 00:00:00 2001 From: Luke Powell Date: Wed, 8 Apr 2020 13:25:06 -0600 Subject: [PATCH 001/449] tlogbe: Add tlogbe implementation. --- politeiad/backend/backend.go | 12 +- politeiad/backend/tlogbe/blog.go | 96 ++++ politeiad/backend/tlogbe/log.go | 25 + politeiad/backend/tlogbe/tlogbe.go | 872 +++++++++++++++++++++++++++++ tlog/README.md | 2 +- tlog/tclient/tclient.go | 2 +- 6 files changed, 1001 insertions(+), 8 deletions(-) create mode 100644 politeiad/backend/tlogbe/blog.go create mode 100644 politeiad/backend/tlogbe/log.go create mode 100644 politeiad/backend/tlogbe/tlogbe.go diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index 43cfef007..f043ac1c2 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -58,10 +58,10 @@ func (c ContentVerificationError) Error() string { } type File struct { - Name string // Basename of the file - MIME string // MIME type - Digest string // SHA256 of decoded Payload - Payload string // base64 encoded file + Name string `json:"name"` // Basename of the file + MIME string `json:"mime"` // MIME type + Digest string `json:"digest"` // SHA256 of decoded Payload + Payload string `json:"payload"` // base64 encoded file } type MDStatusT int @@ -114,8 +114,8 @@ type RecordMetadata struct { // MetadataStream describes a single metada stream. The ID determines how and // where it is stored. type MetadataStream struct { - ID uint64 // Stream identity - Payload string // String encoded metadata + ID uint64 `json:"id"` // Stream identity + Payload string `json:"payload"` // String encoded metadata } // Record is a permanent Record that includes the submitted files, metadata and diff --git a/politeiad/backend/tlogbe/blog.go b/politeiad/backend/tlogbe/blog.go new file mode 100644 index 000000000..314daa77f --- /dev/null +++ b/politeiad/backend/tlogbe/blog.go @@ -0,0 +1,96 @@ +package tlobe + +import ( + "bytes" + "compress/gzip" + "crypto/rand" + "encoding/gob" + "errors" + "fmt" + "io" + + "github.com/decred/politeia/tlog/api/v1" + "golang.org/x/crypto/nacl/secretbox" +) + +func blobify(re v1.RecordEntry) ([]byte, error) { + var b bytes.Buffer + zw := gzip.NewWriter(&b) + enc := gob.NewEncoder(zw) + err := enc.Encode(re) + if err != nil { + return nil, err + } + err = zw.Close() // we must flush gzip buffers + if err != nil { + return nil, err + } + return b.Bytes(), nil +} + +func deblob(blob []byte) (*v1.RecordEntry, error) { + zr, err := gzip.NewReader(bytes.NewReader(blob)) + if err != nil { + return nil, err + } + r := gob.NewDecoder(zr) + var re v1.RecordEntry + err = r.Decode(&re) + if err != nil { + return nil, err + } + return &re, nil +} + +func NewKey() (*[32]byte, error) { + var k [32]byte + + _, err := io.ReadFull(rand.Reader, k[:]) + if err != nil { + return nil, err + } + + return &k, nil +} + +func encryptAndPack(data []byte, key *[32]byte) ([]byte, error) { + var nonce [24]byte + + // random nonce + _, err := io.ReadFull(rand.Reader, nonce[:]) + if err != nil { + return nil, err + } + + // encrypt data + blob := secretbox.Seal(nil, data, &nonce, key) + + // pack all the things + packed := make([]byte, len(nonce)+len(blob)) + copy(packed[0:], nonce[:]) + copy(packed[24:], blob) + + return packed, nil +} + +func unpackAndDecrypt(key *[32]byte, packed []byte) ([]byte, error) { + if len(packed) < 24 { + return nil, errors.New("not an sbox file") + } + + var nonce [24]byte + copy(nonce[:], packed[0:24]) + + decrypted, ok := secretbox.Open(nil, packed[24:], &nonce, key) + if !ok { + return nil, fmt.Errorf("could not decrypt") + } + return decrypted, nil +} + +type Blob interface { + Put([]byte) ([]byte, error) // Store blob and return identifier + Get([]byte) ([]byte, error) // Get blob by identifier + Del([]byte) error // Attempt to delete object + Enum(func([]byte, []byte) error) error // Enumerate over all objects +} diff --git a/politeiad/backend/tlogbe/log.go b/politeiad/backend/tlogbe/log.go new file mode 100644 index 000000000..8a615c0a2 --- /dev/null +++ b/politeiad/backend/tlogbe/log.go @@ -0,0 +1,25 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlogbe + +import "github.com/decred/slog" + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log = slog.Disabled + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = slog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using slog. +func UseLogger(logger slog.Logger) { + log = logger +} diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go new file mode 100644 index 000000000..29efbf475 --- /dev/null +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -0,0 +1,872 @@ +package tlogbe + +import ( + "bytes" + "crypto" + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "strings" + "sync" + "time" + + "github.com/decred/dcrtime/merkle" + pd "github.com/decred/politeia/politeiad/api/v1" + "github.com/decred/politeia/politeiad/api/v1/identity" + "github.com/decred/politeia/politeiad/backend" + tlog "github.com/decred/politeia/tlog/api/v1" + tlogutil "github.com/decred/politeia/tlog/util" + "github.com/decred/politeia/util" + "github.com/google/trillian" + "github.com/google/trillian/client" + tcrypto "github.com/google/trillian/crypto" +) + +// TODO do we need to be able to recreate the indexes from scratch? + +// tlogbe implements the Backend interface. +type tlogbe struct { + sync.RWMutex + + dataDir string + trillianHost string + dcrtimeHost string + dcrtimeCert string + testnet bool + + publicKeyFilename string // Remote server identity filename + publicKey crypto.PublicKey // Remote server signing key + myID *identity.FullIdentity // tlogbe identity + + client trillian.TrillianLogClient + admin trillian.TrillianAdminClient + + client *http.Client + + // TODO these need to be persistent + unvetted map[string]map[uint]record // [token][version]record + vetted map[string]map[uint]record // [token][version]record +} + +const ( + dataDescriptorFile = "file" + dataDescriptorRecordMetadata = "recordmetadata" + dataDescriptorMetadataStream = "metadatastream" +) + +// record represents a tlog index for a backend Record. +type record struct { + metadata string // RecordMetadata merkle hash + files map[string]string // [filename]merkleHash + mdstreams map[uint64]string // [mdstreamID]merkleHash +} + +func tokenFromTreeID(treeID int64) string { + b := make([]byte, binary.MaxVarintLen64) + binary.LittleEndian.PutUint64(b, uint64(treeID)) + return hex.EncodeToString(b) +} + +func treeIDFromToken(token string) (int64, error) { + b, err := hex.DecodeString(token) + if err != nil { + return 0, err + } + return int64(binary.LittleEndian.Uint64(b)), nil +} + +func recordCopy(rv record) record { + files := make(map[string]string, len(rv.files)) + mdstreams := make(map[uint64]string, len(rv.mdstreams)) + for k, v := range rv.files { + files[k] = v + } + for k, v := range rv.mdstreams { + mdstreams[k] = v + } + return record{ + metadata: rv.metadata, + files: files, + mdstreams: mdstreams, + } +} + +func (t *tlogbe) unvettedExists(token string) bool { + t.RLock() + defer t.RUnlock() + + _, ok := t.unvetted[token] + return ok +} + +// unvettedGetLatest returns the lastest version of the record index for the +// provided token. +// +// This function must be called WITHOUT the read lock held. +func (t *tlogbe) unvettedGetLatest(token string) (*record, error) { + t.RLock() + defer t.RUnlock() + + latest := uint(len(t.unvetted[token])) + r, ok := t.unvetted[token][latest] + if !ok { + return nil, backend.ErrRecordNotFound + } + return &r, nil +} + +// unvettedAdd adds the provided record to the unvetted index as a new version. +// +// This function must be called WITHOUT the lock held. +func (t *tlogbe) unvettedAdd(token string, r record) { + t.Lock() + defer t.Unlock() + + versions, ok := t.unvetted[token] + if !ok { + t.unvetted[token] = make(map[uint]record) + } + + t.unvetted[token][uint(len(versions)+1)] = r +} + +func (t *tlogbe) vettedExists(token string) bool { + t.RLock() + defer t.Unlock() + + _, ok := t.vetted[token] + return ok +} + +// errorFromResponse extracts a user-readable string from the response from +// tlog, which will contain a JSON error. +func errorFromResponse(r *http.Response) (string, error) { + var errMsg string + decoder := json.NewDecoder(r.Body) + if r.StatusCode == http.StatusInternalServerError { + var e tlog.ErrorReply + if err := decoder.Decode(&e); err != nil { + return "", err + } + errMsg = fmt.Sprintf("%v", e.ErrorCode) + } else { + var e tlog.UserError + if err := decoder.Decode(&e); err != nil { + return "", err + } + errMsg = tlog.ErrorStatus[e.ErrorCode] + " " + if e.ErrorContext != nil && len(e.ErrorContext) > 0 { + errMsg += strings.Join(e.ErrorContext, ", ") + } + } + + return errMsg, nil +} + +// makeRequests sends an http request to the tlog rpc host using the given +// request parameters. +func (t *tlogbe) makeRequest(method string, route string, body interface{}) ([]byte, error) { + var ( + reqBody []byte + err error + ) + if body != nil { + reqBody, err = json.Marshal(body) + if err != nil { + return nil, err + } + } + + fullRoute := t.rpcHost + route + req, err := http.NewRequest(method, fullRoute, bytes.NewReader(reqBody)) + if err != nil { + return nil, err + } + req.SetBasicAuth(t.rpcUser, t.rpcPass) + r, err := t.client.Do(req) + if err != nil { + return nil, err + } + defer r.Body.Close() + + if r.StatusCode != http.StatusOK { + e, err := errorFromResponse(r) + if err != nil { + return nil, fmt.Errorf("%v", r.Status) + } + return nil, fmt.Errorf("%v: %v", r.Status, e) + } + + return util.ConvertBodyToByteArray(r.Body, false), nil +} + +func (t *tlogbe) recordNew(entries []tlog.RecordEntry) (*tlog.RecordNewReply, error) { + log.Tracef("recordNew") + + // Send request + rn := tlog.RecordNew{ + RecordEntries: entries, + } + respBody, err := t.makeRequest(http.MethodPost, tlog.RouteRecordNew, rn) + if err != nil { + return nil, err + } + + // Decode and verify response + var rnr tlog.RecordNewReply + err = json.Unmarshal(respBody, &rnr) + if err != nil { + return nil, fmt.Errorf("unmarshal RecordNewReply: %v", err) + } + verifier, err := client.NewLogVerifierFromTree(&rnr.Tree) + if err != nil { + return nil, err + } + _, err = tcrypto.VerifySignedLogRoot(verifier.PubKey, + crypto.SHA256, &rnr.InitialRoot) + if err != nil { + return nil, fmt.Errorf("invalid InitialRoot %v: %v", + rnr.Tree.TreeId, err) + } + lrSTH, err := tcrypto.VerifySignedLogRoot(verifier.PubKey, + crypto.SHA256, &rnr.STH) + if err != nil { + return nil, fmt.Errorf("invalid STH %v: %v", + rnr.Tree.TreeId, err) + } + for _, v := range rnr.Proofs { + err := tlogutil.QueuedLeafProofVerify(t.publicKey, lrSTH, v) + if err != nil { + return nil, fmt.Errorf("invalid QueuedLeafProof %v %x: %v", + rnr.Tree.TreeId, v.QueuedLeaf.Leaf.MerkleLeafHash, err) + } + } + + return &rnr, err +} + +func (t *tlogbe) recordAppend(treeID int64, entries []tlog.RecordEntry) (*tlog.RecordAppendReply, error) { + log.Tracef("recordAppend: %v", treeID) + + // Send request + ra := tlog.RecordAppend{ + Id: treeID, + RecordEntries: entries, + } + respBody, err := t.makeRequest(http.MethodPost, tlog.RouteRecordAppend, ra) + if err != nil { + return nil, err + } + + // Decode and verify response + var rar tlog.RecordAppendReply + err = json.Unmarshal(respBody, &rar) + if err != nil { + return nil, fmt.Errorf("unmarshal RecordAppendReply: %v", err) + } + lrv1, err := tcrypto.VerifySignedLogRoot(t.publicKey, + crypto.SHA256, &rar.STH) + if err != nil { + return nil, fmt.Errorf("invalid STH %v: %v", + treeID, err) + } + for _, v := range rar.Proofs { + err := tlogutil.QueuedLeafProofVerify(t.publicKey, lrv1, v) + if err != nil { + return nil, fmt.Errorf("invalid QueuedLeafProof %v %x: %v", + treeID, v.QueuedLeaf.Leaf.MerkleLeafHash, err) + } + } + + return &rar, nil +} + +func (t *tlogbe) recordEntryProofs(treeID int64, merkleHashes []string) ([]tlog.RecordEntryProof, error) { + log.Tracef("recordEntryProofs: %v %v", treeID, merkleHashes) + + // Prepare request + entries := make([]tlog.RecordEntryIdentifier, 0, len(merkleHashes)) + for _, v := range merkleHashes { + entries = append(entries, tlog.RecordEntryIdentifier{ + Id: treeID, + MerkleHash: v, + }) + } + reg := tlog.RecordEntriesGet{ + Entries: entries, + } + + // Send request + respBody, err := t.makeRequest(http.MethodGet, + tlog.RouteRecordEntriesGet, reg) + if err != nil { + return nil, err + } + + // Decode and verify response + var reply tlog.RecordEntriesGetReply + err = json.Unmarshal(respBody, &reply) + if err != nil { + return nil, fmt.Errorf("unmarshal RecordEntriesGetReply: %v", err) + } + for _, v := range reply.Proofs { + err := tlogutil.RecordEntryProofVerify(t.publicKey, v) + if err != nil { + return nil, fmt.Errorf("invalid RecordEntryProof %v %x: %v", + treeID, v.Leaf.MerkleLeafHash, err) + } + } + + return reply.Proofs, nil +} + +func (t *tlogbe) convertRecordEntryFromFile(f backend.File) (*tlog.RecordEntry, error) { + data, err := json.Marshal(f) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + tlog.DataDescriptor{ + Type: tlog.DataTypeStructure, + Descriptor: dataDescriptorFile, + }) + if err != nil { + return nil, err + } + re := tlogutil.RecordEntryNew(t.myID, hint, data) + return &re, nil +} + +func (t *tlogbe) convertRecordEntryFromMetadataStream(ms backend.MetadataStream) (*tlog.RecordEntry, error) { + data, err := json.Marshal(ms) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + tlog.DataDescriptor{ + Type: tlog.DataTypeStructure, + Descriptor: dataDescriptorMetadataStream, + }) + if err != nil { + return nil, err + } + re := tlogutil.RecordEntryNew(t.myID, hint, data) + return &re, nil +} + +func (t *tlogbe) convertRecordEntryFromRecordMetadata(rm backend.RecordMetadata) (*tlog.RecordEntry, error) { + data, err := json.Marshal(rm) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + tlog.DataDescriptor{ + Type: tlog.DataTypeStructure, + Descriptor: dataDescriptorRecordMetadata, + }) + if err != nil { + return nil, err + } + re := tlogutil.RecordEntryNew(t.myID, hint, data) + return &re, nil +} + +func (t *tlogbe) convertRecordEntriesFromFiles(files []backend.File) ([]tlog.RecordEntry, error) { + entries := make([]tlog.RecordEntry, 0, len(files)) + for _, v := range files { + re, err := t.convertRecordEntryFromFile(v) + if err != nil { + return nil, err + } + entries = append(entries, *re) + } + return entries, nil +} + +func (t *tlogbe) convertRecordEntriesFromMetadataStreams(streams []backend.MetadataStream) ([]tlog.RecordEntry, error) { + entries := make([]tlog.RecordEntry, 0, len(streams)) + for _, v := range streams { + re, err := t.convertRecordEntryFromMetadataStream(v) + if err != nil { + return nil, err + } + entries = append(entries, *re) + } + return entries, nil +} + +func recordMetadataNew(files []backend.File, token string, status backend.MDStatusT, iteration uint64) (*backend.RecordMetadata, error) { + hashes := make([]*[sha256.Size]byte, 0, len(files)) + for _, v := range files { + var d [sha256.Size]byte + copy(d[:], v.Digest) + hashes = append(hashes, &d) + } + + m := *merkle.Root(hashes) + return &backend.RecordMetadata{ + Version: backend.VersionRecordMD, + Iteration: iteration, + Status: status, + Merkle: hex.EncodeToString(m[:]), + Timestamp: time.Now().Unix(), + Token: token, + }, nil +} + +// New satisfies the Backend interface. +func (t *tlogbe) New(mdstreams []backend.MetadataStream, files []backend.File) (*backend.RecordMetadata, error) { + log.Tracef("New") + + // TODO Validate files + + // Generate token + // TODO use treeID as token + // TODO handle token prefix collisions + tokenb, err := util.Random(pd.TokenSize) + if err != nil { + return nil, err + } + token := hex.EncodeToString(tokenb) + + // Prepare tlog record entries + reMDStreams, err := t.convertRecordEntriesFromMetadataStreams(mdstreams) + if err != nil { + return nil, err + } + reFiles, err := t.convertRecordEntriesFromFiles(files) + if err != nil { + return nil, err + } + entries := append(reMDStreams, reFiles...) + + rm, err := recordMetadataNew(files, token, backend.MDStatusUnvetted, 1) + if err != nil { + return nil, err + } + reMetadata, err := t.convertRecordEntryFromRecordMetadata(*rm) + if err != nil { + return nil, err + } + entries = append(entries, *reMetadata) + + // Create new tlog record + rnr, err := t.recordNew(entries) + if err != nil { + return nil, err + } + + // Create a new record index + r := record{ + files: make(map[string]string, len(files)), + mdstreams: make(map[uint64]string, len(mdstreams)), + } + for _, v := range rnr.Proofs { + hash := hex.EncodeToString(v.QueuedLeaf.Leaf.LeafValue) + merkle := hex.EncodeToString(v.QueuedLeaf.Leaf.MerkleLeafHash) + + // Check if proof is for the RecordMetadata + if reMetadata.Hash == hash { + r.metadata = merkle + continue + } + + // Check if proof is for any of the files + for i, re := range reFiles { + if re.Hash == hash { + // files slice shares the same ordering as reFiles + r.files[files[i].Name] = merkle + continue + } + } + + // Check if proof is for any of the mdstreams + for i, re := range reMDStreams { + if re.Hash == hash { + // mdstreams slice shares the same ordering as reMDStreams + r.mdstreams[mdstreams[i].ID] = merkle + continue + } + } + + return nil, fmt.Errorf("unknown proof hash %v %v %v", + rnr.Tree.TreeId, hash, merkle) + } + + // Save record index + t.unvettedAdd(token, r) + + log.Debugf("New %v %v", token, rnr.Tree.TreeId) + + return rm, nil +} + +func convertMetadataStreamFromRecordEntry(re tlog.RecordEntry) (*backend.MetadataStream, error) { + // TODO this should be in the calling function + // Decode and validate the DataHint + b, err := base64.StdEncoding.DecodeString(re.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd tlog.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorMetadataStream { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorMetadataStream) + } + + // Decode the MetadataStream + b, err = base64.StdEncoding.DecodeString(re.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + var ms backend.MetadataStream + err = json.Unmarshal(b, &ms) + if err != nil { + return nil, fmt.Errorf("unmarshal MetadataStream: %v", err) + } + + return &ms, nil +} + +func convertMetadataStreamsFromRecordEntryProofs(proofs []tlog.RecordEntryProof) ([]backend.MetadataStream, error) { + mdstreams := make([]backend.MetadataStream, 0, len(proofs)) + for _, v := range proofs { + md, err := convertMetadataStreamFromRecordEntry(*v.RecordEntry) + if err != nil { + return nil, fmt.Errorf("convertMetadataStreamFromRecordEntry %x: %v", + v.Leaf.MerkleLeafHash, err) + } + mdstreams = append(mdstreams, *md) + } + return mdstreams, nil +} + +// recordUpdate... +// This function assumes that the contents have already been validated. +// +// This function must be called with the lock held. +func (t *tlogbe) recordUpdate(token string, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*record, error) { + versions, ok := t.unvetted[token] + if !ok { + return nil, backend.ErrRecordNotFound + } + + treeID, err := treeIDFromToken(token) + if err != nil { + return nil, err + } + + // Make a copy of the most recent record. This will serve as + // the basis for the new version of the record. + r := recordCopy(versions[uint(len(versions))]) + + // entries will be be used to aggregate all new and updated + // files that need to be appended to tlog. + entries := make([]tlog.RecordEntry, 0, 64) + + // Fetch mdstreams for the mdstream append work + merkles := make([]string, 0, len(mdAppend)) + for _, v := range mdAppend { + m, ok := r.mdstreams[v.ID] + if !ok { + return nil, fmt.Errorf("append mdstream %v not found", v.ID) + } + merkles = append(merkles, m) + } + proofs, err := t.recordEntryProofs(treeID, merkles) + if err != nil { + return nil, fmt.Errorf("recordEntryProofs: %v", err) + } + ms, err := convertMetadataStreamsFromRecordEntryProofs(proofs) + if err != nil { + return nil, err + } + mdstreams := make(map[uint64]backend.MetadataStream, len(ms)) + for _, v := range ms { + mdstreams[v.ID] = v + } + + // This map is used to correlate the MetadataStream to the + // tlog returned LogLeaf when we create the index. + hashesMDStreams := make(map[string]backend.MetadataStream, + len(mdAppend)+len(mdOverwrite)) + + // Prepare work for mdstream appends + for i, v := range mdAppend { + m, ok := mdstreams[v.ID] + if !ok { + return nil, fmt.Errorf("tlog entry not found for mdstream %v", v.ID) + } + m.Payload += v.Payload + re, err := t.convertRecordEntryFromMetadataStream(m) + if err != nil { + return nil, err + } + hashesMDStreams[re.Hash] = mdAppend[i] + entries = append(entries, *re) + } + + // Prepare work for mdstream overwrites + for _, v := range mdOverwrite { + re, err := t.convertRecordEntryFromMetadataStream(v) + if err != nil { + return nil, err + } + entries = append(entries, *re) + } + + // This map is used to correlate the File to the tlog returned + // LogLeaf when we create the index. + hashesFiles := make(map[string]backend.File, len(filesAdd)) + + // Prepare work for file adds + for i, v := range filesAdd { + re, err := t.convertRecordEntryFromFile(v) + if err != nil { + return nil, err + } + hashesFiles[re.Hash] = filesAdd[i] + entries = append(entries, *re) + } + + // Apply file deletes + del := make(map[string]struct{}, len(filesDel)) + for _, fn := range filesDel { + del[fn] = struct{}{} + } + for fn := range r.files { + if _, ok := del[fn]; ok { + delete(r.files, fn) + } + } + + // Append new and updated files to tlog + rar, err := t.recordAppend(treeID, entries) + if err != nil { + return nil, fmt.Errorf("recordAppend: %v", err) + } + + // Update record index + for _, v := range rar.Proofs { + hash := hex.EncodeToString(v.QueuedLeaf.Leaf.LeafValue) + merkle := hex.EncodeToString(v.QueuedLeaf.Leaf.MerkleLeafHash) + + m, ok := hashesMDStreams[hash] + if ok { + r.mdstreams[m.ID] = merkle + continue + } + + f, ok := hashesFiles[hash] + if ok { + r.files[f.Name] = merkle + continue + } + + // Proof doesn't correspond to any of the record + // entries we appended. + return nil, fmt.Errorf("unknown LogLeaf %v", hash) + } + + return &r, nil +} + +func (t *tlogbe) updateUnvettedRecord(token string, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) error { + // TODO copy verifyContents() from gitbe + + t.Lock() + defer t.Unlock() + + r, err := t.recordUpdate(token, mdAppend, mdOverwrite, + filesAdd, filesDel) + if err != nil { + return err + } + + // Save the index of the new record version + version := len(t.unvetted[token]) + t.unvetted[token][uint(version+1)] = *r + + return nil +} + +func (t *tlogbe) record(token string, r record) (*backend.Record, error) { + // Aggregate merkle hashes + merkles := make([]string, 0, len(r.files)+len(r.mdstreams)+1) + merkles = append(merkles, r.metadata) + for _, v := range r.files { + merkles = append(merkles, v) + } + for _, v := range r.mdstreams { + merkles = append(merkles, v) + } + + treeID, err := treeIDFromToken(token) + if err != nil { + return nil, err + } + + // Fetch record entry proofs + proofs, err := t.recordEntryProofs(treeID, merkles) + if err != nil { + return nil, err + } + + // Decode the record entries into their appropriate types + var rm backend.RecordMetadata + files := make([]backend.File, 0, len(proofs)) + mdstreams := make([]backend.MetadataStream, 0, len(proofs)) + for _, v := range proofs { + // Decode and unmarshal the data hint + b, err := base64.StdEncoding.DecodeString(v.RecordEntry.DataHint) + if err != nil { + return nil, err + } + var dd tlog.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint %x: %v", + v.Leaf.MerkleLeafHash, err) + } + + // Decode and unmarshal the data + datab, err := base64.StdEncoding.DecodeString(v.RecordEntry.Data) + if err != nil { + return nil, err + } + switch dd.Descriptor { + case dataDescriptorFile: + var f backend.File + err = json.Unmarshal(datab, &f) + if err != nil { + return nil, fmt.Errorf("unmarshal File %x: %v", + v.Leaf.MerkleLeafHash, err) + } + files = append(files, f) + case dataDescriptorRecordMetadata: + err = json.Unmarshal(datab, &rm) + if err != nil { + return nil, fmt.Errorf("unmarshal RecordMetadata %x: %v", + v.Leaf.MerkleLeafHash, err) + } + case dataDescriptorMetadataStream: + var ms backend.MetadataStream + err = json.Unmarshal(datab, &ms) + if err != nil { + return nil, fmt.Errorf("unmarshal MetadataStream %x: %v", + v.Leaf.MerkleLeafHash, err) + } + mdstreams = append(mdstreams, ms) + default: + return nil, fmt.Errorf("unknown data descriptor %x %v", + v.Leaf.MerkleLeafHash, dd.Descriptor) + } + } + + return &backend.Record{ + RecordMetadata: rm, + Files: files, + Metadata: mdstreams, + }, nil +} + +func (t *tlogbe) UpdateUnvettedRecord(tokenb []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { + log.Tracef("UpdateUnvettedRecord: %x", tokenb) + + token := hex.EncodeToString(tokenb) + if !t.unvettedExists(token) { + return nil, backend.ErrRecordNotFound + } + + err := t.updateUnvettedRecord(token, mdAppend, mdOverwrite, + filesAdd, filesDel) + if err != nil { + return nil, err + } + + r, err := t.unvettedGetLatest(token) + if err != nil { + return nil, err + } + + return t.record(token) +} + +func (t *tlogbe) UpdateVettedRecord([]byte, []backend.MetadataStream, []backend.MetadataStream, []backend.File, []string) (*backend.Record, error) { + return nil, nil +} + +func (t *tlogbe) UpdateVettedMetadata([]byte, []backend.MetadataStream, []backend.MetadataStream) error { + return nil +} + +func (t *tlogbe) UpdateReadme(string) error { + return nil +} + +func (t *tlogbe) UnvettedExists(tokenb []byte) bool { + log.Tracef("UnvettedExists %x", tokenb) + + return t.unvettedExists(hex.EncodeToString(tokenb)) +} + +func (t *tlogbe) VettedExists(tokenb []byte) bool { + log.Tracef("VettedExists %x", tokenb) + + return t.vettedExists(hex.EncodeToString(tokenb)) +} + +func (t *tlogbe) GetUnvetted([]byte) (*backend.Record, error) { + return nil, nil +} + +func (t *tlogbe) GetVetted([]byte, string) (*backend.Record, error) { + return nil, nil +} + +func (t *tlogbe) SetUnvettedStatus([]byte, backend.MDStatusT, []backend.MetadataStream, []backend.MetadataStream) (*backend.Record, error) { + return nil, nil +} + +func (t *tlogbe) SetVettedStatus([]byte, backend.MDStatusT, []backend.MetadataStream, []backend.MetadataStream) (*backend.Record, error) { + return nil, nil +} + +func (t *tlogbe) Inventory(uint, uint, bool, bool) ([]backend.Record, []backend.Record, error) { + return nil, nil, nil +} + +func (t *tlogbe) GetPlugins() ([]backend.Plugin, error) { + return nil, nil +} + +func (t *tlogbe) Plugin(string, string) (string, string, error) { + return "", "", nil +} + +func (t *tlogbe) Close() {} + +func tlogbeNew(rpcUser, rpcPass, rpcHost, rpcCert string, testnet bool) (*tlogbe, error) { + client, err := util.NewClient(false, rpcCert) + if err != nil { + return nil, err + } + return &tlogbe{ + rpcUser: rpcUser, + rpcPass: rpcPass, + rpcHost: rpcHost, + rpcCert: rpcCert, + client: client, + testnet: testnet, + unvetted: make(map[string]map[uint]record), + vetted: make(map[string]map[uint]record), + }, nil +} diff --git a/tlog/README.md b/tlog/README.md index 525a822f6..1935c431e 100644 --- a/tlog/README.md +++ b/tlog/README.md @@ -61,5 +61,5 @@ tclient --testnet publickey Submit a new record. ``` -tclient --testnet recrodnew README.md +tclient --testnet recordnew README.md ``` diff --git a/tlog/tclient/tclient.go b/tlog/tclient/tclient.go index b6b039e51..157f3a2fe 100644 --- a/tlog/tclient/tclient.go +++ b/tlog/tclient/tclient.go @@ -53,7 +53,7 @@ var ( ) // getErrorFromResponse extracts a user-readable string from the response from -// politeiad, which will contain a JSON error. +// tlog, which will contain a JSON error. func getErrorFromResponse(r *http.Response) (string, error) { var errMsg string decoder := json.NewDecoder(r.Body) From 1bb98c7caff57788ae611d132ab0a716fccf78d0 Mon Sep 17 00:00:00 2001 From: Luke Powell Date: Tue, 12 May 2020 19:03:54 -0600 Subject: [PATCH 002/449] update --- politeiad/backend/gitbe/gitbe.go | 1 + politeiad/backend/tlogbe/anchor.go | 5 + politeiad/backend/tlogbe/blob/blob.go | 19 + .../tlogbe/blob/filesystem/filesystem.go | 75 + politeiad/backend/tlogbe/blobentry.go | 124 ++ politeiad/backend/tlogbe/blobentry_test.go | 7 + politeiad/backend/tlogbe/blog.go | 96 -- politeiad/backend/tlogbe/encrypt.go | 60 + politeiad/backend/tlogbe/index.go | 159 ++ politeiad/backend/tlogbe/tlogbe.go | 1308 +++++++++-------- politeiad/backend/tlogbe/trillian.go | 260 ++++ politeiad/backend/util.go | 168 +++ tlog/tserver/tserver.go | 5 +- 13 files changed, 1606 insertions(+), 681 deletions(-) create mode 100644 politeiad/backend/tlogbe/anchor.go create mode 100644 politeiad/backend/tlogbe/blob/blob.go create mode 100644 politeiad/backend/tlogbe/blob/filesystem/filesystem.go create mode 100644 politeiad/backend/tlogbe/blobentry.go create mode 100644 politeiad/backend/tlogbe/blobentry_test.go delete mode 100644 politeiad/backend/tlogbe/blog.go create mode 100644 politeiad/backend/tlogbe/encrypt.go create mode 100644 politeiad/backend/tlogbe/index.go create mode 100644 politeiad/backend/tlogbe/trillian.go create mode 100644 politeiad/backend/util.go diff --git a/politeiad/backend/gitbe/gitbe.go b/politeiad/backend/gitbe/gitbe.go index e54243685..db87a3f74 100644 --- a/politeiad/backend/gitbe/gitbe.go +++ b/politeiad/backend/gitbe/gitbe.go @@ -256,6 +256,7 @@ func extendSHA1FromString(s string) (string, error) { return hex.EncodeToString(d), nil } +// TODO this should use the backend.VerifyContent // verifyContent verifies that all provided backend.MetadataStream and // backend.File are sane and returns a cooked array of the files. func verifyContent(metadata []backend.MetadataStream, files []backend.File, filesDel []string) ([]file, error) { diff --git a/politeiad/backend/tlogbe/anchor.go b/politeiad/backend/tlogbe/anchor.go new file mode 100644 index 000000000..200109a52 --- /dev/null +++ b/politeiad/backend/tlogbe/anchor.go @@ -0,0 +1,5 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlogbe diff --git a/politeiad/backend/tlogbe/blob/blob.go b/politeiad/backend/tlogbe/blob/blob.go new file mode 100644 index 000000000..a6f225a96 --- /dev/null +++ b/politeiad/backend/tlogbe/blob/blob.go @@ -0,0 +1,19 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package blob + +import "errors" + +var ( + ErrNotFound = errors.New("not found") +) + +type Blob interface { + Put(key string, blob []byte) error // Store blob + PutMulti(blobs map[string][]byte) error // Store multiple blobs atomically + Get(key string) ([]byte, error) // Get blob by identifier + Del(key string) error // Attempt to delete object + Enum(func(key string, blob []byte) error) error // Enumerate over all objects +} diff --git a/politeiad/backend/tlogbe/blob/filesystem/filesystem.go b/politeiad/backend/tlogbe/blob/filesystem/filesystem.go new file mode 100644 index 000000000..4b6c3cf67 --- /dev/null +++ b/politeiad/backend/tlogbe/blob/filesystem/filesystem.go @@ -0,0 +1,75 @@ +package filesystem + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "github.com/decred/politeia/politeiad/backend/tlogbe/blob" +) + +var ( + _ blob.Blob = (*blobFilesystem)(nil) +) + +// blobFilesystem implements the Blob interface using the filesystem. +type blobFilesystem struct { + path string // Location of files +} + +func (b *blobFilesystem) Put(key string, value []byte) error { + return ioutil.WriteFile(filepath.Join(b.path, key), value, 0600) +} + +func (b *blobFilesystem) PutMulti(blobs map[string][]byte) error { + // TODO implement this + return fmt.Errorf("not implemenated") +} + +func (b *blobFilesystem) Get(key string) ([]byte, error) { + blob, err := ioutil.ReadFile(filepath.Join(b.path, key)) + if err != nil { + return nil, err + } + return blob, nil +} + +func (b *blobFilesystem) Del(key string) error { + err := os.Remove(filepath.Join(b.path, key)) + if err != nil { + // Always return not found + return blob.ErrNotFound + } + return nil +} + +func (b *blobFilesystem) Enum(f func(key string, value []byte) error) error { + files, err := ioutil.ReadDir(b.path) + if err != nil { + return err + } + + for _, file := range files { + if file.Name() == ".." { + continue + } + // TODO should we return filesystem errors or wrap them? + blob, err := b.Get(file.Name()) + if err != nil { + return err + } + err = f(file.Name(), blob) + if err != nil { + return err + } + } + + return nil +} + +func BlobFilesystemNew(path string) *blobFilesystem { + return &blobFilesystem{ + path: path, + } +} diff --git a/politeiad/backend/tlogbe/blobentry.go b/politeiad/backend/tlogbe/blobentry.go new file mode 100644 index 000000000..169607c57 --- /dev/null +++ b/politeiad/backend/tlogbe/blobentry.go @@ -0,0 +1,124 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlogbe + +import ( + "bytes" + "compress/gzip" + "encoding/base64" + "encoding/gob" + "encoding/hex" + + "github.com/decred/politeia/util" +) + +const ( + // dataDescriptor.Type values. These may be freely edited since + // they are solely hints to the application. + dataTypeKeyValue = "kv" // Descriptor is empty but data is key/value + dataTypeMime = "mime" // Descriptor contains a mime type + dataTypeStructure = "struct" // Descriptor contains a structure + + dataDescriptorFile = "file" + dataDescriptorRecordMetadata = "recordmetadata" + dataDescriptorMetadataStream = "metadatastream" + dataDescriptorRecordHistory = "recordhistory" + dataDescriptorAnchor = "anchor" +) + +// dataKeyValue is an encoded key/value pair. +type dataKeyValue struct { + Key string `json:"key"` + Value string `json:"value"` +} + +// dataDescriptor provides hints about a data blob. In practise we JSON encode +// this struture and stuff it into blobEntry.DataHint. +type dataDescriptor struct { + Type string `json:"type,omitempty"` // Type of data that is stored + Descriptor string `json:"descriptor,omitempty"` // Description of the data + ExtraData string `json:"extradata,omitempty"` // Value to be freely used by caller +} + +// blobEntry is the structure used to store data in the Blob key-value store. +type blobEntry struct { + Hash string `json:"hash"` // SHA256 hash of the data payload, hex encoded + DataHint string `json:"datahint"` // Hint that describes the data, base64 encoded + Data string `json:"data"` // Data payload, base64 encoded +} + +func blobify(be blobEntry) ([]byte, error) { + var b bytes.Buffer + zw := gzip.NewWriter(&b) + enc := gob.NewEncoder(zw) + err := enc.Encode(be) + if err != nil { + return nil, err + } + err = zw.Close() // we must flush gzip buffers + if err != nil { + return nil, err + } + return b.Bytes(), nil +} + +func deblob(blob []byte) (*blobEntry, error) { + zr, err := gzip.NewReader(bytes.NewReader(blob)) + if err != nil { + return nil, err + } + r := gob.NewDecoder(zr) + var be blobEntry + err = r.Decode(&be) + if err != nil { + return nil, err + } + return &be, nil +} + +func blobifyEncrypted(be blobEntry, key *[32]byte) ([]byte, error) { + var b bytes.Buffer + zw := gzip.NewWriter(&b) + enc := gob.NewEncoder(zw) + err := enc.Encode(be) + if err != nil { + return nil, err + } + err = zw.Close() // we must flush gzip buffers + if err != nil { + return nil, err + } + blob, err := encryptAndPack(b.Bytes(), key) + if err != nil { + return nil, err + } + return blob, nil +} + +func deblobEncrypted(blob []byte, key *[32]byte) (*blobEntry, error) { + b, err := unpackAndDecrypt(key, blob) + if err != nil { + return nil, err + } + zr, err := gzip.NewReader(bytes.NewReader(b)) + if err != nil { + return nil, err + } + r := gob.NewDecoder(zr) + var be blobEntry + err = r.Decode(&be) + if err != nil { + return nil, err + } + return &be, nil +} + +func blobEntryNew(dataHint, data []byte) blobEntry { + return blobEntry{ + Hash: hex.EncodeToString(util.Digest(data)), + DataHint: base64.StdEncoding.EncodeToString(dataHint), + Data: base64.StdEncoding.EncodeToString(data), + } +} diff --git a/politeiad/backend/tlogbe/blobentry_test.go b/politeiad/backend/tlogbe/blobentry_test.go new file mode 100644 index 000000000..7a588c6eb --- /dev/null +++ b/politeiad/backend/tlogbe/blobentry_test.go @@ -0,0 +1,7 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlogbe + +// TODO implement blobEntry tests diff --git a/politeiad/backend/tlogbe/blog.go b/politeiad/backend/tlogbe/blog.go deleted file mode 100644 index 314daa77f..000000000 --- a/politeiad/backend/tlogbe/blog.go +++ /dev/null @@ -1,96 +0,0 @@ -package tlobe - -import ( - "bytes" - "compress/gzip" - "crypto/rand" - "encoding/gob" - "errors" - "fmt" - "io" - - "github.com/decred/politeia/tlog/api/v1" - "golang.org/x/crypto/nacl/secretbox" -) - -func blobify(re v1.RecordEntry) ([]byte, error) { - var b bytes.Buffer - zw := gzip.NewWriter(&b) - enc := gob.NewEncoder(zw) - err := enc.Encode(re) - if err != nil { - return nil, err - } - err = zw.Close() // we must flush gzip buffers - if err != nil { - return nil, err - } - return b.Bytes(), nil -} - -func deblob(blob []byte) (*v1.RecordEntry, error) { - zr, err := gzip.NewReader(bytes.NewReader(blob)) - if err != nil { - return nil, err - } - r := gob.NewDecoder(zr) - var re v1.RecordEntry - err = r.Decode(&re) - if err != nil { - return nil, err - } - return &re, nil -} - -func NewKey() (*[32]byte, error) { - var k [32]byte - - _, err := io.ReadFull(rand.Reader, k[:]) - if err != nil { - return nil, err - } - - return &k, nil -} - -func encryptAndPack(data []byte, key *[32]byte) ([]byte, error) { - var nonce [24]byte - - // random nonce - _, err := io.ReadFull(rand.Reader, nonce[:]) - if err != nil { - return nil, err - } - - // encrypt data - blob := secretbox.Seal(nil, data, &nonce, key) - - // pack all the things - packed := make([]byte, len(nonce)+len(blob)) - copy(packed[0:], nonce[:]) - copy(packed[24:], blob) - - return packed, nil -} - -func unpackAndDecrypt(key *[32]byte, packed []byte) ([]byte, error) { - if len(packed) < 24 { - return nil, errors.New("not an sbox file") - } - - var nonce [24]byte - copy(nonce[:], packed[0:24]) - - decrypted, ok := secretbox.Open(nil, packed[24:], &nonce, key) - if !ok { - return nil, fmt.Errorf("could not decrypt") - } - return decrypted, nil -} - -type Blob interface { - Put([]byte) ([]byte, error) // Store blob and return identifier - Get([]byte) ([]byte, error) // Get blob by identifier - Del([]byte) error // Attempt to delete object - Enum(func([]byte, []byte) error) error // Enumerate over all objects -} diff --git a/politeiad/backend/tlogbe/encrypt.go b/politeiad/backend/tlogbe/encrypt.go new file mode 100644 index 000000000..6848ca4bf --- /dev/null +++ b/politeiad/backend/tlogbe/encrypt.go @@ -0,0 +1,60 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlogbe + +import ( + "crypto/rand" + "errors" + "fmt" + "io" + + "golang.org/x/crypto/nacl/secretbox" +) + +func newKey() (*[32]byte, error) { + var k [32]byte + + _, err := io.ReadFull(rand.Reader, k[:]) + if err != nil { + return nil, err + } + + return &k, nil +} + +func encryptAndPack(data []byte, key *[32]byte) ([]byte, error) { + var nonce [24]byte + + // Random nonce + _, err := io.ReadFull(rand.Reader, nonce[:]) + if err != nil { + return nil, err + } + + // Encrypt data + blob := secretbox.Seal(nil, data, &nonce, key) + + // Pack all the things + packed := make([]byte, len(nonce)+len(blob)) + copy(packed[0:], nonce[:]) + copy(packed[24:], blob) + + return packed, nil +} + +func unpackAndDecrypt(key *[32]byte, packed []byte) ([]byte, error) { + if len(packed) < 24 { + return nil, errors.New("not an sbox file") + } + + var nonce [24]byte + copy(nonce[:], packed[0:24]) + + decrypted, ok := secretbox.Open(nil, packed[24:], &nonce, key) + if !ok { + return nil, fmt.Errorf("could not decrypt") + } + return decrypted, nil +} diff --git a/politeiad/backend/tlogbe/index.go b/politeiad/backend/tlogbe/index.go new file mode 100644 index 000000000..65f74c01a --- /dev/null +++ b/politeiad/backend/tlogbe/index.go @@ -0,0 +1,159 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlogbe + +import ( + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + + "github.com/decred/politeia/politeiad/backend" +) + +const ( + stateUnvetted = "unvetted" + stateVetted = "vetted" +) + +// recordIndex represents an index for a backend Record. The merkleLeafHash +// refers to a trillian LogLeaf.MerkleLeafHash. This value can be used to +// lookup the inclusion proof from a trillian tree as well as the actual data +// from the blob storage layer. All merkleLeafHashes are hex encoded. +type recordIndex struct { + RecordMetadata string `json:"recordmetadata"` // RecordMetadata merkleLeafHash + Metadata map[uint64]string `json:"metadata"` // [mdstreamID]merkleLeafHash + Files map[string]string `json:"files"` // [filename]merkleLeafHash +} + +// recordHistory provides the record index for all versions of a record. The +// recordHistory is stored to the blob storage layer with the token as the key. +type recordHistory struct { + Token string `json:"token"` + State string `json:"state"` // Unvetted or vetted + Versions map[uint32]recordIndex `json:"versions"` // [version]recordIndex +} + +func latestVersion(rh recordHistory) uint32 { + return uint32(len(rh.Versions)) - 1 +} + +func (t *tlogbe) recordHistory(token string) (*recordHistory, error) { + b, err := t.blob.Get(token) + if err != nil { + return nil, fmt.Errorf("blob get: %v", err) + } + var rh recordHistory + err = json.Unmarshal(b, &rh) + if err != nil { + return nil, err + } + return &rh, nil +} + +// recordHistoryAdd adds the provided recordIndex as a new version to the +// recordHistory. +func (t *tlogbe) recordHistoryAdd(token string, ri recordIndex) (*recordHistory, error) { + // TODO implement + // A new version can only be added to vetted records + return nil, nil +} + +// recordHistoryUpdate updates the existing version of the recordHistory with +// the provided state and recordIndex. +func (t *tlogbe) recordHistoryUpdate(token, state string, ri recordIndex) (*recordHistory, error) { + // TODO implement + // This will be needed for unvetted updates and metadata updates + return nil, nil +} + +func (t *tlogbe) recordIndexLatest(token string) (*recordIndex, error) { + rh, err := t.recordHistory(token) + if err != nil { + return nil, err + } + latest := rh.Versions[latestVersion(*rh)] + return &latest, nil +} + +// recordIndexUpdate updates the provided recordIndex with the provided +// blobEntries then returns the updated recordIndex. +func recordIndexUpdate(r recordIndex, entries []blobEntry, proofs []queuedLeafProof) (*recordIndex, error) { + // TODO implement + return nil, nil +} + +func recordIndexNew(entries []blobEntry, proofs []queuedLeafProof) (*recordIndex, error) { + merkleHashes := make(map[string]string, len(entries)) // [leafValue]merkleLeafHash + for _, v := range proofs { + leafValue := hex.EncodeToString(v.QueuedLeaf.Leaf.LeafValue) + merkleHash := hex.EncodeToString(v.QueuedLeaf.Leaf.MerkleLeafHash) + merkleHashes[leafValue] = merkleHash + } + + // Find the merkleLeafHash for each of the record components. The + // blobEntry.Hash is the value that is saved to trillian as the + // LogLeaf.LeafValue. + var ( + recordMD string + metadata = make(map[uint64]string, len(entries)) // [mdstreamID]merkleLeafHash + files = make(map[string]string, len(entries)) // [filename]merkleLeafHash + ) + for _, v := range entries { + b, err := base64.StdEncoding.DecodeString(v.DataHint) + if err != nil { + return nil, err + } + var dd dataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, err + } + switch dd.Descriptor { + case dataDescriptorRecordMetadata: + merkleHash, ok := merkleHashes[v.Hash] + if !ok { + return nil, fmt.Errorf("merkle not found for record metadata") + } + recordMD = merkleHash + case dataDescriptorMetadataStream: + b, err := base64.StdEncoding.DecodeString(v.Data) + if err != nil { + return nil, err + } + var ms backend.MetadataStream + err = json.Unmarshal(b, &ms) + if err != nil { + return nil, err + } + merkleHash, ok := merkleHashes[v.Hash] + if !ok { + return nil, fmt.Errorf("merkle not found for mdstream %v", ms.ID) + } + metadata[ms.ID] = merkleHash + case dataDescriptorFile: + b, err := base64.StdEncoding.DecodeString(v.Data) + if err != nil { + return nil, err + } + var f backend.File + err = json.Unmarshal(b, &f) + if err != nil { + return nil, err + } + merkleHash, ok := merkleHashes[v.Hash] + if !ok { + return nil, fmt.Errorf("merkle not found for file %v", f.Name) + } + files[f.Name] = merkleHash + } + } + + return &recordIndex{ + RecordMetadata: recordMD, + Metadata: metadata, + Files: files, + }, nil +} diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 29efbf475..08fc30c8d 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -1,7 +1,12 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + package tlogbe import ( "bytes" + "context" "crypto" "crypto/sha256" "encoding/base64" @@ -9,64 +14,69 @@ import ( "encoding/hex" "encoding/json" "fmt" - "net/http" - "strings" + "io/ioutil" + "os" + "path/filepath" + "strconv" "sync" "time" "github.com/decred/dcrtime/merkle" pd "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" - tlog "github.com/decred/politeia/tlog/api/v1" - tlogutil "github.com/decred/politeia/tlog/util" + "github.com/decred/politeia/politeiad/backend/tlogbe/blob" + "github.com/decred/politeia/politeiad/backend/tlogbe/blob/filesystem" "github.com/decred/politeia/util" "github.com/google/trillian" - "github.com/google/trillian/client" - tcrypto "github.com/google/trillian/crypto" + "github.com/google/trillian/crypto/keys" + "github.com/google/trillian/crypto/keys/der" + "github.com/google/trillian/crypto/keyspb" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" +) + +const ( + blobsDirname = "blobs" ) -// TODO do we need to be able to recreate the indexes from scratch? +var ( + _ backend.Backend = (*tlogbe)(nil) +) // tlogbe implements the Backend interface. type tlogbe struct { sync.RWMutex - - dataDir string - trillianHost string - dcrtimeHost string - dcrtimeCert string - testnet bool - - publicKeyFilename string // Remote server identity filename - publicKey crypto.PublicKey // Remote server signing key - myID *identity.FullIdentity // tlogbe identity - - client trillian.TrillianLogClient - admin trillian.TrillianAdminClient - - client *http.Client - - // TODO these need to be persistent - unvetted map[string]map[uint]record // [token][version]record - vetted map[string]map[uint]record // [token][version]record -} - -const ( - dataDescriptorFile = "file" - dataDescriptorRecordMetadata = "recordmetadata" - dataDescriptorMetadataStream = "metadatastream" -) - -// record represents a tlog index for a backend Record. -type record struct { - metadata string // RecordMetadata merkle hash - files map[string]string // [filename]merkleHash - mdstreams map[uint64]string // [mdstreamID]merkleHash + shutdown bool + root string // Root directory + dcrtimeHost string // Dcrtimed host + encryptionKey *[32]byte // Private key for encrypting data + exit chan struct{} // Close channel + blob blob.Blob // Blob key-value store + plugins []backend.Plugin // Plugins + + // Trillian setup + client trillian.TrillianLogClient // Trillian log client + admin trillian.TrillianAdminClient // Trillian admin client + ctx context.Context // Context used for trillian calls + privateKey *keyspb.PrivateKey // Trillian signing key + publicKey crypto.PublicKey // Trillian public key + + // dirty keeps track of which tree is dirty at what height. Dirty + // means that the tree has leaves that have not been timestamped, + // i.e. anchored, onto the decred blockchain. At start-of-day we + // scan all records and look for STH that have not been anchored. + // Note that we only anchor the latest STH and we do so + // opportunistically. If the application is closed and restarted + // it simply will drop a new anchor at the next interval; it will + // not try to finish a prior outstanding anchor drop. + dirty map[int64]int64 // [treeid]height + droppingAnchor bool // anchor dropping is in progress } func tokenFromTreeID(treeID int64) string { b := make([]byte, binary.MaxVarintLen64) + // Converting between int64 and uint64 doesn't change the sign bit, + // only the way it's interpreted. binary.LittleEndian.PutUint64(b, uint64(treeID)) return hex.EncodeToString(b) } @@ -79,306 +89,100 @@ func treeIDFromToken(token string) (int64, error) { return int64(binary.LittleEndian.Uint64(b)), nil } -func recordCopy(rv record) record { - files := make(map[string]string, len(rv.files)) - mdstreams := make(map[uint64]string, len(rv.mdstreams)) - for k, v := range rv.files { - files[k] = v - } - for k, v := range rv.mdstreams { - mdstreams[k] = v - } - return record{ - metadata: rv.metadata, - files: files, - mdstreams: mdstreams, - } -} - -func (t *tlogbe) unvettedExists(token string) bool { - t.RLock() - defer t.RUnlock() - - _, ok := t.unvetted[token] - return ok -} - -// unvettedGetLatest returns the lastest version of the record index for the -// provided token. -// -// This function must be called WITHOUT the read lock held. -func (t *tlogbe) unvettedGetLatest(token string) (*record, error) { - t.RLock() - defer t.RUnlock() - - latest := uint(len(t.unvetted[token])) - r, ok := t.unvetted[token][latest] - if !ok { - return nil, backend.ErrRecordNotFound - } - return &r, nil -} - -// unvettedAdd adds the provided record to the unvetted index as a new version. -// -// This function must be called WITHOUT the lock held. -func (t *tlogbe) unvettedAdd(token string, r record) { - t.Lock() - defer t.Unlock() - - versions, ok := t.unvetted[token] - if !ok { - t.unvetted[token] = make(map[uint]record) - } - - t.unvetted[token][uint(len(versions)+1)] = r -} - -func (t *tlogbe) vettedExists(token string) bool { - t.RLock() - defer t.Unlock() - - _, ok := t.vetted[token] - return ok -} - -// errorFromResponse extracts a user-readable string from the response from -// tlog, which will contain a JSON error. -func errorFromResponse(r *http.Response) (string, error) { - var errMsg string - decoder := json.NewDecoder(r.Body) - if r.StatusCode == http.StatusInternalServerError { - var e tlog.ErrorReply - if err := decoder.Decode(&e); err != nil { - return "", err - } - errMsg = fmt.Sprintf("%v", e.ErrorCode) - } else { - var e tlog.UserError - if err := decoder.Decode(&e); err != nil { - return "", err - } - errMsg = tlog.ErrorStatus[e.ErrorCode] + " " - if e.ErrorContext != nil && len(e.ErrorContext) > 0 { - errMsg += strings.Join(e.ErrorContext, ", ") - } - } - - return errMsg, nil -} - -// makeRequests sends an http request to the tlog rpc host using the given -// request parameters. -func (t *tlogbe) makeRequest(method string, route string, body interface{}) ([]byte, error) { - var ( - reqBody []byte - err error - ) - if body != nil { - reqBody, err = json.Marshal(body) - if err != nil { - return nil, err - } - } - - fullRoute := t.rpcHost + route - req, err := http.NewRequest(method, fullRoute, bytes.NewReader(reqBody)) - if err != nil { - return nil, err - } - req.SetBasicAuth(t.rpcUser, t.rpcPass) - r, err := t.client.Do(req) - if err != nil { - return nil, err - } - defer r.Body.Close() - - if r.StatusCode != http.StatusOK { - e, err := errorFromResponse(r) - if err != nil { - return nil, fmt.Errorf("%v", r.Status) - } - return nil, fmt.Errorf("%v: %v", r.Status, e) +func merkleRoot(files []backend.File) [sha256.Size]byte { + hashes := make([]*[sha256.Size]byte, 0, len(files)) + for _, v := range files { + var d [sha256.Size]byte + copy(d[:], v.Digest) + hashes = append(hashes, &d) } - - return util.ConvertBodyToByteArray(r.Body, false), nil + return *merkle.Root(hashes) } -func (t *tlogbe) recordNew(entries []tlog.RecordEntry) (*tlog.RecordNewReply, error) { - log.Tracef("recordNew") - - // Send request - rn := tlog.RecordNew{ - RecordEntries: entries, - } - respBody, err := t.makeRequest(http.MethodPost, tlog.RouteRecordNew, rn) - if err != nil { - return nil, err - } - - // Decode and verify response - var rnr tlog.RecordNewReply - err = json.Unmarshal(respBody, &rnr) - if err != nil { - return nil, fmt.Errorf("unmarshal RecordNewReply: %v", err) - } - verifier, err := client.NewLogVerifierFromTree(&rnr.Tree) - if err != nil { - return nil, err - } - _, err = tcrypto.VerifySignedLogRoot(verifier.PubKey, - crypto.SHA256, &rnr.InitialRoot) - if err != nil { - return nil, fmt.Errorf("invalid InitialRoot %v: %v", - rnr.Tree.TreeId, err) - } - lrSTH, err := tcrypto.VerifySignedLogRoot(verifier.PubKey, - crypto.SHA256, &rnr.STH) - if err != nil { - return nil, fmt.Errorf("invalid STH %v: %v", - rnr.Tree.TreeId, err) - } - for _, v := range rnr.Proofs { - err := tlogutil.QueuedLeafProofVerify(t.publicKey, lrSTH, v) - if err != nil { - return nil, fmt.Errorf("invalid QueuedLeafProof %v %x: %v", - rnr.Tree.TreeId, v.QueuedLeaf.Leaf.MerkleLeafHash, err) - } +func recordMetadataNew(token string, files []backend.File, status backend.MDStatusT, iteration uint64) backend.RecordMetadata { + m := merkleRoot(files) + return backend.RecordMetadata{ + Version: backend.VersionRecordMD, + Iteration: iteration, + Status: status, + Merkle: hex.EncodeToString(m[:]), + Timestamp: time.Now().Unix(), + Token: token, } - - return &rnr, err } -func (t *tlogbe) recordAppend(treeID int64, entries []tlog.RecordEntry) (*tlog.RecordAppendReply, error) { - log.Tracef("recordAppend: %v", treeID) - - // Send request - ra := tlog.RecordAppend{ - Id: treeID, - RecordEntries: entries, - } - respBody, err := t.makeRequest(http.MethodPost, tlog.RouteRecordAppend, ra) +func convertBlobEntryFromFile(f backend.File) (*blobEntry, error) { + data, err := json.Marshal(f) if err != nil { return nil, err } - - // Decode and verify response - var rar tlog.RecordAppendReply - err = json.Unmarshal(respBody, &rar) - if err != nil { - return nil, fmt.Errorf("unmarshal RecordAppendReply: %v", err) - } - lrv1, err := tcrypto.VerifySignedLogRoot(t.publicKey, - crypto.SHA256, &rar.STH) - if err != nil { - return nil, fmt.Errorf("invalid STH %v: %v", - treeID, err) - } - for _, v := range rar.Proofs { - err := tlogutil.QueuedLeafProofVerify(t.publicKey, lrv1, v) - if err != nil { - return nil, fmt.Errorf("invalid QueuedLeafProof %v %x: %v", - treeID, v.QueuedLeaf.Leaf.MerkleLeafHash, err) - } - } - - return &rar, nil -} - -func (t *tlogbe) recordEntryProofs(treeID int64, merkleHashes []string) ([]tlog.RecordEntryProof, error) { - log.Tracef("recordEntryProofs: %v %v", treeID, merkleHashes) - - // Prepare request - entries := make([]tlog.RecordEntryIdentifier, 0, len(merkleHashes)) - for _, v := range merkleHashes { - entries = append(entries, tlog.RecordEntryIdentifier{ - Id: treeID, - MerkleHash: v, + hint, err := json.Marshal( + dataDescriptor{ + Type: dataTypeStructure, + Descriptor: dataDescriptorFile, }) - } - reg := tlog.RecordEntriesGet{ - Entries: entries, - } - - // Send request - respBody, err := t.makeRequest(http.MethodGet, - tlog.RouteRecordEntriesGet, reg) if err != nil { return nil, err } - - // Decode and verify response - var reply tlog.RecordEntriesGetReply - err = json.Unmarshal(respBody, &reply) - if err != nil { - return nil, fmt.Errorf("unmarshal RecordEntriesGetReply: %v", err) - } - for _, v := range reply.Proofs { - err := tlogutil.RecordEntryProofVerify(t.publicKey, v) - if err != nil { - return nil, fmt.Errorf("invalid RecordEntryProof %v %x: %v", - treeID, v.Leaf.MerkleLeafHash, err) - } - } - - return reply.Proofs, nil + be := blobEntryNew(hint, data) + return &be, nil } -func (t *tlogbe) convertRecordEntryFromFile(f backend.File) (*tlog.RecordEntry, error) { - data, err := json.Marshal(f) +func convertBlobEntryFromMetadataStream(ms backend.MetadataStream) (*blobEntry, error) { + data, err := json.Marshal(ms) if err != nil { return nil, err } hint, err := json.Marshal( - tlog.DataDescriptor{ - Type: tlog.DataTypeStructure, - Descriptor: dataDescriptorFile, + dataDescriptor{ + Type: dataTypeStructure, + Descriptor: dataDescriptorMetadataStream, }) if err != nil { return nil, err } - re := tlogutil.RecordEntryNew(t.myID, hint, data) - return &re, nil + be := blobEntryNew(hint, data) + return &be, nil } -func (t *tlogbe) convertRecordEntryFromMetadataStream(ms backend.MetadataStream) (*tlog.RecordEntry, error) { - data, err := json.Marshal(ms) +func convertBlobEntryFromRecordMetadata(rm backend.RecordMetadata) (*blobEntry, error) { + data, err := json.Marshal(rm) if err != nil { return nil, err } hint, err := json.Marshal( - tlog.DataDescriptor{ - Type: tlog.DataTypeStructure, - Descriptor: dataDescriptorMetadataStream, + dataDescriptor{ + Type: dataTypeStructure, + Descriptor: dataDescriptorRecordMetadata, }) if err != nil { return nil, err } - re := tlogutil.RecordEntryNew(t.myID, hint, data) - return &re, nil + be := blobEntryNew(hint, data) + return &be, nil } -func (t *tlogbe) convertRecordEntryFromRecordMetadata(rm backend.RecordMetadata) (*tlog.RecordEntry, error) { - data, err := json.Marshal(rm) +func convertBlobEntryFromRecordHistory(rh recordHistory) (*blobEntry, error) { + data, err := json.Marshal(rh) if err != nil { return nil, err } hint, err := json.Marshal( - tlog.DataDescriptor{ - Type: tlog.DataTypeStructure, - Descriptor: dataDescriptorRecordMetadata, + dataDescriptor{ + Type: dataTypeStructure, + Descriptor: dataDescriptorRecordHistory, }) if err != nil { return nil, err } - re := tlogutil.RecordEntryNew(t.myID, hint, data) - return &re, nil + be := blobEntryNew(hint, data) + return &be, nil } -func (t *tlogbe) convertRecordEntriesFromFiles(files []backend.File) ([]tlog.RecordEntry, error) { - entries := make([]tlog.RecordEntry, 0, len(files)) +func convertBlobEntriesFromFiles(files []backend.File) ([]blobEntry, error) { + entries := make([]blobEntry, 0, len(files)) for _, v := range files { - re, err := t.convertRecordEntryFromFile(v) + re, err := convertBlobEntryFromFile(v) if err != nil { return nil, err } @@ -387,10 +191,10 @@ func (t *tlogbe) convertRecordEntriesFromFiles(files []backend.File) ([]tlog.Rec return entries, nil } -func (t *tlogbe) convertRecordEntriesFromMetadataStreams(streams []backend.MetadataStream) ([]tlog.RecordEntry, error) { - entries := make([]tlog.RecordEntry, 0, len(streams)) +func convertBlobEntriesFromMetadataStreams(streams []backend.MetadataStream) ([]blobEntry, error) { + entries := make([]blobEntry, 0, len(streams)) for _, v := range streams { - re, err := t.convertRecordEntryFromMetadataStream(v) + re, err := convertBlobEntryFromMetadataStream(v) if err != nil { return nil, err } @@ -399,474 +203,810 @@ func (t *tlogbe) convertRecordEntriesFromMetadataStreams(streams []backend.Metad return entries, nil } -func recordMetadataNew(files []backend.File, token string, status backend.MDStatusT, iteration uint64) (*backend.RecordMetadata, error) { - hashes := make([]*[sha256.Size]byte, 0, len(files)) - for _, v := range files { - var d [sha256.Size]byte - copy(d[:], v.Digest) - hashes = append(hashes, &d) +func convertRecordMetadataFromBlobEntry(be blobEntry) (*backend.RecordMetadata, error) { + // Decode and validate the DataHint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) } - - m := *merkle.Root(hashes) - return &backend.RecordMetadata{ - Version: backend.VersionRecordMD, - Iteration: iteration, - Status: status, - Merkle: hex.EncodeToString(m[:]), - Timestamp: time.Now().Unix(), - Token: token, - }, nil -} - -// New satisfies the Backend interface. -func (t *tlogbe) New(mdstreams []backend.MetadataStream, files []backend.File) (*backend.RecordMetadata, error) { - log.Tracef("New") - - // TODO Validate files - - // Generate token - // TODO use treeID as token - // TODO handle token prefix collisions - tokenb, err := util.Random(pd.TokenSize) + var dd dataDescriptor + err = json.Unmarshal(b, &dd) if err != nil { - return nil, err + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorRecordMetadata { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorRecordMetadata) } - token := hex.EncodeToString(tokenb) - // Prepare tlog record entries - reMDStreams, err := t.convertRecordEntriesFromMetadataStreams(mdstreams) + // Decode the MetadataStream + b, err = base64.StdEncoding.DecodeString(be.Data) if err != nil { - return nil, err + return nil, fmt.Errorf("decode Data: %v", err) } - reFiles, err := t.convertRecordEntriesFromFiles(files) + var rm backend.RecordMetadata + err = json.Unmarshal(b, &rm) if err != nil { - return nil, err + return nil, fmt.Errorf("unmarshal RecordMetadata: %v", err) } - entries := append(reMDStreams, reFiles...) - rm, err := recordMetadataNew(files, token, backend.MDStatusUnvetted, 1) + return &rm, nil +} + +func convertMetadataStreamFromBlobEntry(be blobEntry) (*backend.MetadataStream, error) { + // Decode and validate the DataHint + b, err := base64.StdEncoding.DecodeString(be.DataHint) if err != nil { - return nil, err + return nil, fmt.Errorf("decode DataHint: %v", err) } - reMetadata, err := t.convertRecordEntryFromRecordMetadata(*rm) + var dd dataDescriptor + err = json.Unmarshal(b, &dd) if err != nil { - return nil, err + return nil, fmt.Errorf("unmarshal DataHint: %v", err) } - entries = append(entries, *reMetadata) - - // Create new tlog record - rnr, err := t.recordNew(entries) - if err != nil { - return nil, err + if dd.Descriptor != dataDescriptorMetadataStream { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorMetadataStream) } - // Create a new record index - r := record{ - files: make(map[string]string, len(files)), - mdstreams: make(map[uint64]string, len(mdstreams)), + // Decode the MetadataStream + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) } - for _, v := range rnr.Proofs { - hash := hex.EncodeToString(v.QueuedLeaf.Leaf.LeafValue) - merkle := hex.EncodeToString(v.QueuedLeaf.Leaf.MerkleLeafHash) - - // Check if proof is for the RecordMetadata - if reMetadata.Hash == hash { - r.metadata = merkle - continue - } - - // Check if proof is for any of the files - for i, re := range reFiles { - if re.Hash == hash { - // files slice shares the same ordering as reFiles - r.files[files[i].Name] = merkle - continue - } - } - - // Check if proof is for any of the mdstreams - for i, re := range reMDStreams { - if re.Hash == hash { - // mdstreams slice shares the same ordering as reMDStreams - r.mdstreams[mdstreams[i].ID] = merkle - continue - } - } - - return nil, fmt.Errorf("unknown proof hash %v %v %v", - rnr.Tree.TreeId, hash, merkle) + var ms backend.MetadataStream + err = json.Unmarshal(b, &ms) + if err != nil { + return nil, fmt.Errorf("unmarshal MetadataStream: %v", err) } - // Save record index - t.unvettedAdd(token, r) - - log.Debugf("New %v %v", token, rnr.Tree.TreeId) - - return rm, nil + return &ms, nil } -func convertMetadataStreamFromRecordEntry(re tlog.RecordEntry) (*backend.MetadataStream, error) { - // TODO this should be in the calling function +func convertFileFromBlobEntry(be blobEntry) (*backend.File, error) { // Decode and validate the DataHint - b, err := base64.StdEncoding.DecodeString(re.DataHint) + b, err := base64.StdEncoding.DecodeString(be.DataHint) if err != nil { return nil, fmt.Errorf("decode DataHint: %v", err) } - var dd tlog.DataDescriptor + var dd dataDescriptor err = json.Unmarshal(b, &dd) if err != nil { return nil, fmt.Errorf("unmarshal DataHint: %v", err) } - if dd.Descriptor != dataDescriptorMetadataStream { + if dd.Descriptor != dataDescriptorFile { return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorMetadataStream) + dd.Descriptor, dataDescriptorFile) } // Decode the MetadataStream - b, err = base64.StdEncoding.DecodeString(re.Data) + b, err = base64.StdEncoding.DecodeString(be.Data) if err != nil { return nil, fmt.Errorf("decode Data: %v", err) } - var ms backend.MetadataStream - err = json.Unmarshal(b, &ms) + var f backend.File + err = json.Unmarshal(b, &f) if err != nil { - return nil, fmt.Errorf("unmarshal MetadataStream: %v", err) + return nil, fmt.Errorf("unmarshal File: %v", err) } - return &ms, nil + return &f, nil } -func convertMetadataStreamsFromRecordEntryProofs(proofs []tlog.RecordEntryProof) ([]backend.MetadataStream, error) { - mdstreams := make([]backend.MetadataStream, 0, len(proofs)) - for _, v := range proofs { - md, err := convertMetadataStreamFromRecordEntry(*v.RecordEntry) +func convertLeavesFromBlobEntries(entries []blobEntry) ([]*trillian.LogLeaf, error) { + leaves := make([]*trillian.LogLeaf, 0, len(entries)) + for _, v := range entries { + b, err := hex.DecodeString(v.Hash) if err != nil { - return nil, fmt.Errorf("convertMetadataStreamFromRecordEntry %x: %v", - v.Leaf.MerkleLeafHash, err) + return nil, err } - mdstreams = append(mdstreams, *md) + leaves = append(leaves, logLeafNew(b)) } - return mdstreams, nil + return leaves, nil } -// recordUpdate... -// This function assumes that the contents have already been validated. -// -// This function must be called with the lock held. -func (t *tlogbe) recordUpdate(token string, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*record, error) { - versions, ok := t.unvetted[token] - if !ok { - return nil, backend.ErrRecordNotFound +func convertBlobEntries(mdstreams []backend.MetadataStream, files []backend.File, recordMD backend.RecordMetadata) ([]blobEntry, error) { + blobStreams, err := convertBlobEntriesFromMetadataStreams(mdstreams) + if err != nil { + return nil, err } - - treeID, err := treeIDFromToken(token) + blobFiles, err := convertBlobEntriesFromFiles(files) + if err != nil { + return nil, err + } + blobRecordMD, err := convertBlobEntryFromRecordMetadata(recordMD) if err != nil { return nil, err } - // Make a copy of the most recent record. This will serve as - // the basis for the new version of the record. - r := recordCopy(versions[uint(len(versions))]) + // The recordMD is intentionally put first. In the event that the + // record indexes ever have to be recovered by walking the trillian + // trees then a recordMD will mark the start of a new version of + // the record. This should never be required, but just in case. + entries := make([]blobEntry, 0, len(blobStreams)+len(blobFiles)+1) + entries = append(entries, *blobRecordMD) + entries = append(entries, blobStreams...) + entries = append(entries, blobFiles...) - // entries will be be used to aggregate all new and updated - // files that need to be appended to tlog. - entries := make([]tlog.RecordEntry, 0, 64) + return entries, nil +} - // Fetch mdstreams for the mdstream append work - merkles := make([]string, 0, len(mdAppend)) - for _, v := range mdAppend { - m, ok := r.mdstreams[v.ID] - if !ok { - return nil, fmt.Errorf("append mdstream %v not found", v.ID) - } - merkles = append(merkles, m) +// blobEntriesAppend appends the provided blob entries onto the trillain tree +// that corresponds to the provided token. An error is returned if any leaves +// are not successfully added. The only exception to this is if a leaf is not +// added because it is a duplicate. +func (t *tlogbe) blobEntriesAppend(token string, entries []blobEntry) ([]queuedLeafProof, *trillian.SignedLogRoot, error) { + // Setup request + treeID, err := treeIDFromToken(token) + if err != nil { + return nil, nil, err } - proofs, err := t.recordEntryProofs(treeID, merkles) + leaves, err := convertLeavesFromBlobEntries(entries) if err != nil { - return nil, fmt.Errorf("recordEntryProofs: %v", err) + return nil, nil, err } - ms, err := convertMetadataStreamsFromRecordEntryProofs(proofs) + + // Append leaves + proofs, slr, err := t.leavesAppend(treeID, leaves) if err != nil { - return nil, err + return nil, nil, fmt.Errorf("leavesAppend: %v", err) } - mdstreams := make(map[uint64]backend.MetadataStream, len(ms)) - for _, v := range ms { - mdstreams[v.ID] = v + if len(proofs) != len(leaves) { + // Sanity check. Even if a leaf fails to be added there should + // still be a QueuedLogLeaf with the error code. + return nil, nil, fmt.Errorf("proofs do not match leaves: got %v, want %v", + len(proofs), len(leaves)) + } + missing := make([]string, 0, len(proofs)) + for _, v := range proofs { + c := codes.Code(v.QueuedLeaf.GetStatus().GetCode()) + // Its ok if the error is because of a duplicate since it still + // allows us to retreive an inclusion proof. + if c != codes.OK && c != codes.AlreadyExists { + missing = append(missing, fmt.Sprint("%v", c)) + } + } + if len(missing) > 0 { + return nil, nil, fmt.Errorf("leaves failed with error codes %v", missing) } - // This map is used to correlate the MetadataStream to the - // tlog returned LogLeaf when we create the index. - hashesMDStreams := make(map[string]backend.MetadataStream, - len(mdAppend)+len(mdOverwrite)) + return proofs, slr, nil +} - // Prepare work for mdstream appends - for i, v := range mdAppend { - m, ok := mdstreams[v.ID] - if !ok { - return nil, fmt.Errorf("tlog entry not found for mdstream %v", v.ID) +func (t *tlogbe) recordMetadata(key, state string) (*backend.RecordMetadata, error) { + b, err := t.blob.Get(key) + if err != nil { + return nil, fmt.Errorf("blob Get: %v", err) + } + var be *blobEntry + switch state { + case stateUnvetted: + // Unvetted blobs will be encrypted + be, err = deblobEncrypted(b, t.encryptionKey) + if err != nil { + return nil, err } - m.Payload += v.Payload - re, err := t.convertRecordEntryFromMetadataStream(m) + case stateVetted: + // Vetted blobs will not be encrypted + be, err = deblob(b) if err != nil { return nil, err } - hashesMDStreams[re.Hash] = mdAppend[i] - entries = append(entries, *re) + default: + return nil, fmt.Errorf("unkown record history state: %v", state) + } + rm, err := convertRecordMetadataFromBlobEntry(*be) + if err != nil { + return nil, err } + return rm, nil +} - // Prepare work for mdstream overwrites - for _, v := range mdOverwrite { - re, err := t.convertRecordEntryFromMetadataStream(v) +func (t *tlogbe) metadataStream(key, state string) (*backend.MetadataStream, error) { + b, err := t.blob.Get(key) + if err != nil { + return nil, fmt.Errorf("blob Get: %v", err) + } + var be *blobEntry + switch state { + case stateUnvetted: + // Unvetted blobs will be encrypted + be, err = deblobEncrypted(b, t.encryptionKey) if err != nil { return nil, err } - entries = append(entries, *re) + case stateVetted: + // Vetted blobs will not be encrypted + be, err = deblob(b) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unkown record history state: %v", state) } + ms, err := convertMetadataStreamFromBlobEntry(*be) + if err != nil { + return nil, err + } + return ms, nil +} - // This map is used to correlate the File to the tlog returned - // LogLeaf when we create the index. - hashesFiles := make(map[string]backend.File, len(filesAdd)) - - // Prepare work for file adds - for i, v := range filesAdd { - re, err := t.convertRecordEntryFromFile(v) +func (t *tlogbe) file(key, state string) (*backend.File, error) { + b, err := t.blob.Get(key) + if err != nil { + return nil, fmt.Errorf("blob Get: %v", err) + } + var be *blobEntry + switch state { + case stateUnvetted: + // Unvetted blobs will be encrypted + be, err = deblobEncrypted(b, t.encryptionKey) if err != nil { return nil, err } - hashesFiles[re.Hash] = filesAdd[i] - entries = append(entries, *re) + case stateVetted: + // Vetted blobs will not be encrypted + be, err = deblob(b) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unkown record history state: %v", state) + } + f, err := convertFileFromBlobEntry(*be) + if err != nil { + return nil, err } + return f, nil +} - // Apply file deletes - del := make(map[string]struct{}, len(filesDel)) - for _, fn := range filesDel { - del[fn] = struct{}{} +// record returns the backend record given the record index. +func (t *tlogbe) record(ri recordIndex, version uint32, state string) (*backend.Record, error) { + recordMD, err := t.recordMetadata(ri.RecordMetadata, state) + if err != nil { + return nil, fmt.Errorf("recordMetadata: %v", err) } - for fn := range r.files { - if _, ok := del[fn]; ok { - delete(r.files, fn) + metadata := make([]backend.MetadataStream, 0, len(ri.Metadata)) + for _, v := range ri.Metadata { + ms, err := t.metadataStream(v, state) + if err != nil { + return nil, fmt.Errorf("metadataStream %v: %v", v, err) + } + metadata = append(metadata, *ms) + } + files := make([]backend.File, 0, len(ri.Files)) + for _, v := range ri.Files { + f, err := t.file(v, state) + if err != nil { + return nil, fmt.Errorf("file %v: %v", v, err) } + files = append(files, *f) } + return &backend.Record{ + Version: strconv.FormatUint(uint64(version), 10), + RecordMetadata: *recordMD, + Metadata: metadata, + Files: files, + }, nil +} + +// recordSave saves the provided record as a new version. This includes +// appending the hashes of the record contents onto the associated trillian +// tree, saving the record contents as blobs in the storage layer, and saving +// a record index. This function assumes the record contents have already been +// validated. The blobs for unvetted record files (just files, not metadata) +// are encrypted before being saved to the storage layer. +func (t *tlogbe) recordSave(token string, metadata []backend.MetadataStream, files []backend.File, recordMD backend.RecordMetadata) error { + // TODO implement this and remove the recordSaveUnvetted and + // recordSaveVetted functions. + return nil +} + +// recordUpdate updates the current version of the provided record. This +// includes appending the hashes of the record contents onto the associated +// trillian tree, saving the record contents as blobs in the storage layer, and +// updating the existing record index. This function assumes the record +// contents have already been validated. The blobs for unvetted record files +// (just files, not metadata) are encrypted before being saved to the storage +// layer. +// +// It is the responsibility of the caller to remove any blobs that were +// orphaned by this update from the storage layer. +func (t *tlogbe) recordUpdate(token string, metadata []backend.MetadataStream, files []backend.File, + recordMD backend.RecordMetadata) error { + // TODO implement this and remove the recordSaveUnvetted and + // recordSaveVetted functions. + return nil +} - // Append new and updated files to tlog - rar, err := t.recordAppend(treeID, entries) +// recordSaveUnvetted saves the hashes of the record contents to the provided +// trillian tree, saves the record contents as blobs in the storage layer, and +// saves an index of the record that can be used to lookup both the trillian +// inclusion proofs and the blobs required to reconstruct the record. This +// function assumes the record contents have already been validated. Unvetted +// record blobs are encrypted before being saved to the storage layer. +func (t *tlogbe) recordSaveUnvetted(token string, mdstreams []backend.MetadataStream, files []backend.File, recordMD backend.RecordMetadata) error { + // Prepare blob entries + entries, err := convertBlobEntries(mdstreams, files, recordMD) if err != nil { - return nil, fmt.Errorf("recordAppend: %v", err) + return err } - // Update record index - for _, v := range rar.Proofs { - hash := hex.EncodeToString(v.QueuedLeaf.Leaf.LeafValue) - merkle := hex.EncodeToString(v.QueuedLeaf.Leaf.MerkleLeafHash) + // Append leaves onto trillian tree for all record contents + proofs, _, err := t.blobEntriesAppend(token, entries) + if err != nil { + return err + } - m, ok := hashesMDStreams[hash] - if ok { - r.mdstreams[m.ID] = merkle - continue + // The merkleLeafHash is used as the key for the blob in the + // storage layer. + merkleHashes := make(map[string]string, len(entries)) // [leafValue]merkleLeafHash + for _, v := range proofs { + leafValue := hex.EncodeToString(v.QueuedLeaf.Leaf.LeafValue) + merkleHash := hex.EncodeToString(v.QueuedLeaf.Leaf.MerkleLeafHash) + merkleHashes[leafValue] = merkleHash + } + + // Prepare blobs for the storage layer. Unvetted record blobs are + // encrypted. + blobs := make(map[string][]byte, len(entries)) // [merkleLeafHash]blob + for _, v := range entries { + // Get the merkle leaf hash that corresponds to this blob entry + merkle, ok := merkleHashes[v.Hash] + if !ok { + return fmt.Errorf("no merkle leaf hash found for %v", v.Hash) } - f, ok := hashesFiles[hash] - if ok { - r.files[f.Name] = merkle + // Check if this blob already exists in the storage layer. This + // can happen when a new record version is submitted but some of + // the the files and mdstreams remains the same. + _, err = t.blob.Get(merkle) + if err == nil { + // Blob aleady exists. Skip it. continue } - // Proof doesn't correspond to any of the record - // entries we appended. - return nil, fmt.Errorf("unknown LogLeaf %v", hash) + // Prepare encrypted blob + b, err := blobifyEncrypted(v, t.encryptionKey) + if err != nil { + return err + } + blobs[merkle] = b + } + + // Update the record history and blobify it + ri, err := recordIndexNew(entries, proofs) + if err != nil { + return err + } + rh, err := t.recordHistoryUpdate(token, stateUnvetted, *ri) + if err != nil { + return err + } + be, err := convertBlobEntryFromRecordHistory(*rh) + if err != nil { + return err + } + b, err := blobify(*be) + if err != nil { + return err + } + blobs[token] = b + + // Save all blobs + err = t.blob.PutMulti(blobs) + if err != nil { + return fmt.Errorf("blob PutMulti: %v", err) } - return &r, nil + return nil } -func (t *tlogbe) updateUnvettedRecord(token string, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) error { - // TODO copy verifyContents() from gitbe +// recordSaveVetted saves the hashes of the record contents to the provided +// trillian tree, saves the record contents as blobs in the storage layer, and +// saves a record index that can be used to lookup both the trillian inclusion +// proofs and the blobs required to reconstruct the record. This function +// assumes the record contents have already been validated. Vetted record blobs +// are NOT encrypted before being saved to the storage layer. +func (t *tlogbe) recordSaveVetted(token string, version uint32, mdstreams []backend.MetadataStream, files []backend.File, recordMD backend.RecordMetadata) error { + // Prepare blob entries + entries, err := convertBlobEntries(mdstreams, files, recordMD) + if err != nil { + return err + } - t.Lock() - defer t.Unlock() + // Append leaves onto trillian tree for all record contents + proofs, _, err := t.blobEntriesAppend(token, entries) + if err != nil { + return err + } + + // The merkleLeafHash is used as the key for the blob in the + // storage layer. + merkleHashes := make(map[string]string, len(entries)) // [leafValue]merkleLeafHash + for _, v := range proofs { + leafValue := hex.EncodeToString(v.QueuedLeaf.Leaf.LeafValue) + merkleHash := hex.EncodeToString(v.QueuedLeaf.Leaf.MerkleLeafHash) + merkleHashes[leafValue] = merkleHash + } + + // Prepare blobs for the storage layer. Vetted record blobs are not + // encrypted. + blobs := make(map[string][]byte, len(entries)) // [merkleLeafHash]blob + for _, v := range entries { + // Get the merkle leaf hash that corresponds to this blob entry + merkle, ok := merkleHashes[v.Hash] + if !ok { + return fmt.Errorf("no merkle leaf hash found for %v", v.Hash) + } + + // Check if this blob already exists in the storage layer. This + // can happen when a new record version is submitted but some of + // the the files and mdstreams remains the same. + _, err = t.blob.Get(merkle) + if err == nil { + // Blob aleady exists. Skip it. + continue + } + + // Prepare blob + b, err := blobify(v) + if err != nil { + return err + } + blobs[merkle] = b + } - r, err := t.recordUpdate(token, mdAppend, mdOverwrite, - filesAdd, filesDel) + // Add a record index to the record history and blobify it + ri, err := recordIndexNew(entries, proofs) + if err != nil { + return err + } + rh, err := t.recordHistoryAdd(token, *ri) + if err != nil { + return err + } + be, err := convertBlobEntryFromRecordHistory(*rh) if err != nil { return err } + b, err := blobify(*be) + if err != nil { + return err + } + blobs[token] = b - // Save the index of the new record version - version := len(t.unvetted[token]) - t.unvetted[token][uint(version+1)] = *r + // Save all blobs + err = t.blob.PutMulti(blobs) + if err != nil { + return fmt.Errorf("blob PutMulti: %v", err) + } return nil } -func (t *tlogbe) record(token string, r record) (*backend.Record, error) { - // Aggregate merkle hashes - merkles := make([]string, 0, len(r.files)+len(r.mdstreams)+1) - merkles = append(merkles, r.metadata) - for _, v := range r.files { - merkles = append(merkles, v) +// New satisfies the Backend interface. +func (t *tlogbe) New(metadata []backend.MetadataStream, files []backend.File) (*backend.RecordMetadata, error) { + log.Tracef("New") + + // Validate record contents + err := backend.VerifyContent(metadata, files, []string{}) + if err != nil { + return nil, err } - for _, v := range r.mdstreams { - merkles = append(merkles, v) + + // Create a new trillian tree. The treeID is used as the record + // token. + // TODO handle token prefix collisions + tree, _, err := t.treeNew() + if err != nil { + return nil, fmt.Errorf("treeNew: %v", err) } + token := tokenFromTreeID(tree.TreeId) - treeID, err := treeIDFromToken(token) + // Save record + rm := recordMetadataNew(token, files, backend.MDStatusUnvetted, 1) + err = t.recordSaveUnvetted(token, metadata, files, rm) if err != nil { - return nil, err + return nil, fmt.Errorf("recordSaveUnvetted %v: %v", token, err) } - // Fetch record entry proofs - proofs, err := t.recordEntryProofs(treeID, merkles) + log.Infof("New record %v %v", tree.TreeId, token) + + return &rm, nil +} + +func (t *tlogbe) UpdateUnvettedRecord(tokenb []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { + log.Tracef("UpdateUnvettedRecord: %x", tokenb) + + // Send in a single metadata array to verify there are no dups + allMD := append(mdAppend, mdOverwrite...) + err := backend.VerifyContent(allMD, filesAdd, filesDel) if err != nil { - return nil, err + e, ok := err.(backend.ContentVerificationError) + if !ok { + return nil, err + } + // Allow ErrorStatusEmpty which indicates no new files are + // being added. This can happen when metadata only is being + // updated. + if e.ErrorCode != pd.ErrorStatusEmpty { + return nil, err + } } - // Decode the record entries into their appropriate types - var rm backend.RecordMetadata - files := make([]backend.File, 0, len(proofs)) - mdstreams := make([]backend.MetadataStream, 0, len(proofs)) - for _, v := range proofs { - // Decode and unmarshal the data hint - b, err := base64.StdEncoding.DecodeString(v.RecordEntry.DataHint) + token := hex.EncodeToString(tokenb) + + t.Lock() + defer t.Unlock() + + // Ensure record is not vetted + rh, err := t.recordHistory(token) + if err != nil { + return nil, fmt.Errorf("recordHistory: %v", err) + } + if rh.State != stateUnvetted { + // This error doesn't really make sense in the context of tlogbe, + // but its what the gitbe returns in this situation so we return + // it to keep it consistent. For gitbe, it means a record was + // found in a directory that should only contain vetted records. + return nil, backend.ErrRecordFound + } + + l := len(mdAppend) + len(mdOverwrite) + len(filesAdd) + entries := make([]blobEntry, 0, l) + + // Retrive existing record index + version := latestVersion(*rh) + idx := rh.Versions[version] + + // Overwrite metadata. The recordIndex uses the metadata ID as the + // key so any existing metadata will get overwritten when the new + // recordIndex is created. + for _, v := range mdOverwrite { + be, err := convertBlobEntryFromMetadataStream(v) if err != nil { return nil, err } - var dd tlog.DataDescriptor - err = json.Unmarshal(b, &dd) + entries = append(entries, *be) + } + + // Append metadata + for _, v := range mdAppend { + m, ok := idx.Metadata[v.ID] + if !ok { + return nil, fmt.Errorf("append metadata not found: %v", v.ID) + } + ms, err := t.metadataStream(m, rh.State) if err != nil { - return nil, fmt.Errorf("unmarshal DataHint %x: %v", - v.Leaf.MerkleLeafHash, err) + return nil, fmt.Errorf("metadataStream %v: %v", m, err) + } + buf := bytes.NewBuffer([]byte(ms.Payload)) + buf.WriteString(v.Payload) + ms.Payload = buf.String() + be, err := convertBlobEntryFromMetadataStream(*ms) + if err != nil { + return nil, err } + entries = append(entries, *be) + } - // Decode and unmarshal the data - datab, err := base64.StdEncoding.DecodeString(v.RecordEntry.Data) + // Add files + for _, v := range filesAdd { + be, err := convertBlobEntryFromFile(v) if err != nil { return nil, err } - switch dd.Descriptor { - case dataDescriptorFile: - var f backend.File - err = json.Unmarshal(datab, &f) - if err != nil { - return nil, fmt.Errorf("unmarshal File %x: %v", - v.Leaf.MerkleLeafHash, err) - } - files = append(files, f) - case dataDescriptorRecordMetadata: - err = json.Unmarshal(datab, &rm) - if err != nil { - return nil, fmt.Errorf("unmarshal RecordMetadata %x: %v", - v.Leaf.MerkleLeafHash, err) - } - case dataDescriptorMetadataStream: - var ms backend.MetadataStream - err = json.Unmarshal(datab, &ms) - if err != nil { - return nil, fmt.Errorf("unmarshal MetadataStream %x: %v", - v.Leaf.MerkleLeafHash, err) - } - mdstreams = append(mdstreams, ms) - default: - return nil, fmt.Errorf("unknown data descriptor %x %v", - v.Leaf.MerkleLeafHash, dd.Descriptor) + entries = append(entries, *be) + } + + // Delete files + for _, fn := range filesDel { + _, ok := idx.Files[fn] + if !ok { + return nil, fmt.Errorf("file to delete not found: %v", fn) } + delete(idx.Files, fn) } - return &backend.Record{ - RecordMetadata: rm, - Files: files, - Metadata: mdstreams, - }, nil -} + // TODO Ensure changes were actually made -func (t *tlogbe) UpdateUnvettedRecord(tokenb []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { - log.Tracef("UpdateUnvettedRecord: %x", tokenb) + // Aggregate all files. We need this for the merkle root calc. + files := make([]backend.File, 0, len(idx.Files)+len(filesAdd)) + for _, v := range idx.Files { + f, err := t.file(v, rh.State) + if err != nil { + return nil, fmt.Errorf("file %v: %v", v, err) + } + files = append(files, *f) + } + for _, v := range filesAdd { + files = append(files, v) + } - token := hex.EncodeToString(tokenb) - if !t.unvettedExists(token) { - return nil, backend.ErrRecordNotFound + // Update the record metadata + rm, err := t.recordMetadata(idx.RecordMetadata, rh.State) + if err != nil { + return nil, fmt.Errorf("recordMetadata %v: %v", + idx.RecordMetadata, err) } + rmNew := recordMetadataNew(token, files, rm.Status, rm.Iteration+1) + be, err := convertBlobEntryFromRecordMetadata(rmNew) + if err != nil { + return nil, err + } + entries = append(entries, *be) - err := t.updateUnvettedRecord(token, mdAppend, mdOverwrite, - filesAdd, filesDel) + // Append new content to the trillian tree + proofs, _, err := t.blobEntriesAppend(token, entries) if err != nil { return nil, err } - r, err := t.unvettedGetLatest(token) + // Update the record history + idxp, err := recordIndexUpdate(idx, entries, proofs) + if err != nil { + return nil, err + } + rh, err = t.recordHistoryUpdate(token, rh.State, *idxp) if err != nil { return nil, err } - return t.record(token) + // Save blobs + + // TODO Delete orphaned blobs + + // TODO call plugin hooks + + return nil, nil } -func (t *tlogbe) UpdateVettedRecord([]byte, []backend.MetadataStream, []backend.MetadataStream, []backend.File, []string) (*backend.Record, error) { +func (t *tlogbe) UpdateVettedRecord(tokenb []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { + log.Tracef("UpdateVettedRecord: %x", tokenb) + return nil, nil } -func (t *tlogbe) UpdateVettedMetadata([]byte, []backend.MetadataStream, []backend.MetadataStream) error { +func (t *tlogbe) UpdateVettedMetadata(tokenb []byte, mdAppend, mdOverwrite []backend.MetadataStream) error { + log.Tracef("UpdateVettedMetadata: %x", tokenb) + return nil } -func (t *tlogbe) UpdateReadme(string) error { - return nil +func (t *tlogbe) UpdateReadme(content string) error { + return fmt.Errorf("not implemented") } func (t *tlogbe) UnvettedExists(tokenb []byte) bool { log.Tracef("UnvettedExists %x", tokenb) - return t.unvettedExists(hex.EncodeToString(tokenb)) + return false } func (t *tlogbe) VettedExists(tokenb []byte) bool { log.Tracef("VettedExists %x", tokenb) - return t.vettedExists(hex.EncodeToString(tokenb)) + return false } -func (t *tlogbe) GetUnvetted([]byte) (*backend.Record, error) { +func (t *tlogbe) GetUnvetted(tokenb []byte) (*backend.Record, error) { + log.Tracef("GetUnvetted: %x", tokenb) + return nil, nil } -func (t *tlogbe) GetVetted([]byte, string) (*backend.Record, error) { +func (t *tlogbe) GetVetted(tokenb []byte, version string) (*backend.Record, error) { + log.Tracef("GetVetted: %x", tokenb) + return nil, nil } -func (t *tlogbe) SetUnvettedStatus([]byte, backend.MDStatusT, []backend.MetadataStream, []backend.MetadataStream) (*backend.Record, error) { +func (t *tlogbe) SetUnvettedStatus(tokenb []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { + log.Tracef("SetUnvettedStatus: %x", tokenb) + return nil, nil } -func (t *tlogbe) SetVettedStatus([]byte, backend.MDStatusT, []backend.MetadataStream, []backend.MetadataStream) (*backend.Record, error) { +func (t *tlogbe) SetVettedStatus(tokenb []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { + log.Tracef("SetVettedStatus: %x", tokenb) + return nil, nil } -func (t *tlogbe) Inventory(uint, uint, bool, bool) ([]backend.Record, []backend.Record, error) { +func (t *tlogbe) Inventory(vettedCount uint, branchCount uint, includeFiles, allVersions bool) ([]backend.Record, []backend.Record, error) { return nil, nil, nil } func (t *tlogbe) GetPlugins() ([]backend.Plugin, error) { - return nil, nil + log.Tracef("GetPlugins") + + return t.plugins, nil } -func (t *tlogbe) Plugin(string, string) (string, string, error) { +func (t *tlogbe) Plugin(command, payload string) (string, string, error) { + log.Tracef("Plugin: %v", command) + return "", "", nil } -func (t *tlogbe) Close() {} +func (t *tlogbe) Close() { + log.Tracef("Close") + + t.Lock() + defer t.Unlock() + + t.shutdown = true + close(t.exit) + + // Zero out encryption key + util.Zero(t.encryptionKey[:]) + t.encryptionKey = nil +} + +func tlogbeNew(root, trillianHost, privateKeyFile string) (*tlogbe, error) { + // Create a new signing key + if !util.FileExists(privateKeyFile) { + log.Infof("Generating signing key...") + privateKey, err := keys.NewFromSpec(&keyspb.Specification{ + // TODO Params: &keyspb.Specification_Ed25519Params{}, + Params: &keyspb.Specification_EcdsaParams{}, + }) + if err != nil { + return nil, err + } + b, err := der.MarshalPrivateKey(privateKey) + if err != nil { + return nil, err + } + err = ioutil.WriteFile(privateKeyFile, b, 0400) + if err != nil { + return nil, err + } + + log.Infof("Signing Key created...") + } + + // Load signing key + var err error + var privateKey = &keyspb.PrivateKey{} + privateKey.Der, err = ioutil.ReadFile(privateKeyFile) + if err != nil { + return nil, err + } + signer, err := der.UnmarshalPrivateKey(privateKey.Der) + if err != nil { + return nil, err + } + + // Connect to trillian + log.Infof("Trillian log server: %v", trillianHost) + g, err := grpc.Dial(trillianHost, grpc.WithInsecure()) + if err != nil { + return nil, err + } + defer g.Close() -func tlogbeNew(rpcUser, rpcPass, rpcHost, rpcCert string, testnet bool) (*tlogbe, error) { - client, err := util.NewClient(false, rpcCert) + // Setup blob directory + blobsPath := filepath.Join(root, blobsDirname) + err = os.MkdirAll(blobsPath, 0700) if err != nil { return nil, err } + + // TODO setup encryption key + + // TODO we need a fsck that ensures there are no orphaned blobs in + // the storage layer and that record indexes don't have any missing + // blobs. + return &tlogbe{ - rpcUser: rpcUser, - rpcPass: rpcPass, - rpcHost: rpcHost, - rpcCert: rpcCert, - client: client, - testnet: testnet, - unvetted: make(map[string]map[uint]record), - vetted: make(map[string]map[uint]record), + root: root, + blob: filesystem.BlobFilesystemNew(blobsPath), + client: trillian.NewTrillianLogClient(g), + admin: trillian.NewTrillianAdminClient(g), + ctx: context.Background(), + privateKey: privateKey, + publicKey: signer.Public(), }, nil } diff --git a/politeiad/backend/tlogbe/trillian.go b/politeiad/backend/tlogbe/trillian.go new file mode 100644 index 000000000..cc5ad34a5 --- /dev/null +++ b/politeiad/backend/tlogbe/trillian.go @@ -0,0 +1,260 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlogbe + +import ( + "crypto" + "crypto/sha256" + "fmt" + + "github.com/golang/protobuf/ptypes" + "github.com/google/trillian" + "github.com/google/trillian/client" + tcrypto "github.com/google/trillian/crypto" + "github.com/google/trillian/crypto/sigpb" + "github.com/google/trillian/merkle/hashers" + "github.com/google/trillian/merkle/rfc6962" + "github.com/google/trillian/types" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type queuedLeafProof struct { + QueuedLeaf *trillian.QueuedLogLeaf + Proof *trillian.Proof +} + +// merkleLeafHash returns the LogLeaf.MerkleLeafHash for the provided +// LogLeaf.LeafData. +func mekleLeafHash(leafValue []byte) []byte { + h := sha256.New() + h.Write([]byte{rfc6962.RFC6962LeafHashPrefix}) + h.Write(leafValue) + return h.Sum(nil) +} + +func logLeafNew(value []byte) *trillian.LogLeaf { + return &trillian.LogLeaf{ + LeafValue: value, + } +} + +func (t *tlogbe) signedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoot, *types.LogRootV1, error) { + // Get latest signed root + resp, err := t.client.GetLatestSignedLogRoot(t.ctx, + &trillian.GetLatestSignedLogRootRequest{LogId: tree.TreeId}) + if err != nil { + return nil, nil, err + } + + // Verify root + verifier, err := client.NewLogVerifierFromTree(tree) + if err != nil { + return nil, nil, err + } + lrv1, err := tcrypto.VerifySignedLogRoot(verifier.PubKey, + crypto.SHA256, resp.SignedLogRoot) + if err != nil { + return nil, nil, err + } + + return resp.SignedLogRoot, lrv1, nil +} + +// waitForRootUpdate waits until the trillian root is updated. +func (t *tlogbe) waitForRootUpdate(tree *trillian.Tree, root *trillian.SignedLogRoot) error { + // Wait for update + var logRoot types.LogRootV1 + err := logRoot.UnmarshalBinary(root.LogRoot) + if err != nil { + return err + } + c, err := client.NewFromTree(t.client, tree, logRoot) + if err != nil { + return err + } + _, err = c.WaitForRootUpdate(t.ctx) + if err != nil { + return err + } + return nil +} + +func (t *tlogbe) inclusionProof(treeID int64, leafHash []byte, lrv1 *types.LogRootV1) (*trillian.Proof, error) { + resp, err := t.client.GetInclusionProofByHash(t.ctx, + &trillian.GetInclusionProofByHashRequest{ + LogId: treeID, + LeafHash: leafHash, + TreeSize: int64(lrv1.TreeSize), + }) + if err != nil { + return nil, fmt.Errorf("GetInclusionProof: %v", err) + } + if len(resp.Proof) != 1 { + return nil, fmt.Errorf("invalid number of proofs: got %v, want 1", + len(resp.Proof)) + } + proof := resp.Proof[0] + + // Verify inclusion proof + lh, err := hashers.NewLogHasher(trillian.HashStrategy_RFC6962_SHA256) + if err != nil { + return nil, err + } + verifier := client.NewLogVerifier(lh, t.publicKey, crypto.SHA256) + err = verifier.VerifyInclusionByHash(lrv1, leafHash, proof) + if err != nil { + return nil, fmt.Errorf("VerifyInclusionByHash: %v", err) + } + + return proof, nil +} + +func (t *tlogbe) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, *trillian.SignedLogRoot, error) { + // Get the tree + tree, err := t.admin.GetTree(t.ctx, &trillian.GetTreeRequest{ + TreeId: treeID, + }) + if err != nil { + return nil, nil, fmt.Errorf("GetTree: %v", err) + } + if tree.TreeId != treeID { + // Sanity check + return nil, nil, fmt.Errorf("invalid treeID; got %v, want %v", + tree.TreeId, treeID) + } + + // Get the latest signed log root + slr, _, err := t.signedLogRoot(tree) + if err != nil { + return nil, nil, fmt.Errorf("signedLogRoot pre update: %v", err) + } + + // Append leaves to log + qlr, err := t.client.QueueLeaves(t.ctx, &trillian.QueueLeavesRequest{ + LogId: treeID, + Leaves: leaves, + }) + if err != nil { + return nil, nil, fmt.Errorf("QueuedLeaves: %v", err) + } + + // Only wait if we actually updated the tree + var n int + for k := range qlr.QueuedLeaves { + c := codes.Code(qlr.QueuedLeaves[k].GetStatus().GetCode()) + if c != codes.OK { + n++ + } + } + if len(leaves)-n != 0 { + // Wait for update + log.Debugf("Waiting for update: %v", treeID) + err = t.waitForRootUpdate(tree, slr) + if err != nil { + return nil, nil, fmt.Errorf("waitForRootUpdate: %v", err) + } + } + + log.Debugf("Stored/Ignored leaves: %v/%v %v", len(leaves)-n, n, treeID) + + // TODO Mark tree as dirty + + // Get the latest signed log root + slr, lrv1, err := t.signedLogRoot(tree) + if err != nil { + return nil, nil, fmt.Errorf("signedLogRoot post update: %v", err) + } + + // Get inclusion proofs + proofs := make([]queuedLeafProof, 0, len(qlr.QueuedLeaves)) + for _, v := range qlr.QueuedLeaves { + qlp := queuedLeafProof{ + QueuedLeaf: v, + } + // Only retrieve the inclusion proof if the leaf was successfully + // added. A leaf might not have been added if it was a duplicate. + // This is ok. All other errors are not ok. + c := codes.Code(v.GetStatus().GetCode()) + if c == codes.OK || c == codes.AlreadyExists { + // The LeafIndex of a QueuedLogLeaf will not be set yet. Get the + // inclusion proof by MerkleLeafHash. + qlp.Proof, err = t.inclusionProof(treeID, v.Leaf.MerkleLeafHash, lrv1) + if err != nil { + return nil, nil, fmt.Errorf("inclusionProof %v %x: %v", + treeID, v.Leaf.MerkleLeafHash, err) + } + } + proofs = append(proofs, qlp) + } + + return proofs, slr, nil +} + +// treeNew returns a new trillian tree and verifies that the signatures are +// correct. It returns the tree and the signed log root which can be externally +// verified. +func (t *tlogbe) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { + pk, err := ptypes.MarshalAny(t.privateKey) + if err != nil { + return nil, nil, err + } + + // Create new trillian tree + tree, err := t.admin.CreateTree(t.ctx, &trillian.CreateTreeRequest{ + Tree: &trillian.Tree{ + TreeState: trillian.TreeState_ACTIVE, + TreeType: trillian.TreeType_LOG, + HashStrategy: trillian.HashStrategy_RFC6962_SHA256, + HashAlgorithm: sigpb.DigitallySigned_SHA256, + SignatureAlgorithm: sigpb.DigitallySigned_ECDSA, + // TODO SignatureAlgorithm: sigpb.DigitallySigned_ED25519, + DisplayName: "", + Description: "", + MaxRootDuration: ptypes.DurationProto(0), + PrivateKey: pk, + }, + }) + if err != nil { + return nil, nil, err + } + + // Init tree or signer goes bananas + ilr, err := t.client.InitLog(t.ctx, &trillian.InitLogRequest{ + LogId: tree.TreeId, + }) + if err != nil { + return nil, nil, err + } + + // Check trillian errors + switch code := status.Code(err); code { + case codes.Unavailable: + err = fmt.Errorf("log server unavailable: %v", err) + case codes.AlreadyExists: + err = fmt.Errorf("just-created Log (%v) is already initialised: %v", + tree.TreeId, err) + case codes.OK: + log.Debugf("Initialised Log: %v", tree.TreeId) + default: + err = fmt.Errorf("failed to InitLog (unknown error)") + } + if err != nil { + return nil, nil, err + } + + // Verify root signature + verifier, err := client.NewLogVerifierFromTree(tree) + if err != nil { + return nil, nil, err + } + _, err = tcrypto.VerifySignedLogRoot(verifier.PubKey, + crypto.SHA256, ilr.Created) + if err != nil { + return nil, nil, err + } + + return tree, ilr.Created, nil +} diff --git a/politeiad/backend/util.go b/politeiad/backend/util.go new file mode 100644 index 000000000..1ac1aa15f --- /dev/null +++ b/politeiad/backend/util.go @@ -0,0 +1,168 @@ +package backend + +import ( + "bytes" + "encoding/base64" + "path/filepath" + "strconv" + + pd "github.com/decred/politeia/politeiad/api/v1" + "github.com/decred/politeia/politeiad/api/v1/mime" + "github.com/decred/politeia/util" + "github.com/subosito/gozaru" +) + +// VerifyContent verifies that all provided MetadataStream and File are sane. +func VerifyContent(metadata []MetadataStream, files []File, filesDel []string) error { + // Make sure all metadata is within maxima. + for _, v := range metadata { + if v.ID > pd.MetadataStreamsMax-1 { + return ContentVerificationError{ + ErrorCode: pd.ErrorStatusInvalidMDID, + ErrorContext: []string{ + strconv.FormatUint(v.ID, 10), + }, + } + } + } + for i := range metadata { + for j := range metadata { + // Skip self and non duplicates. + if i == j || metadata[i].ID != metadata[j].ID { + continue + } + return ContentVerificationError{ + ErrorCode: pd.ErrorStatusDuplicateMDID, + ErrorContext: []string{ + strconv.FormatUint(metadata[i].ID, 10), + }, + } + } + } + + // Prevent paths + for i := range files { + if filepath.Base(files[i].Name) != files[i].Name { + return ContentVerificationError{ + ErrorCode: pd.ErrorStatusInvalidFilename, + ErrorContext: []string{ + files[i].Name, + }, + } + } + } + for _, v := range filesDel { + if filepath.Base(v) != v { + return ContentVerificationError{ + ErrorCode: pd.ErrorStatusInvalidFilename, + ErrorContext: []string{ + v, + }, + } + } + } + + // Now check files + if len(files) == 0 { + return ContentVerificationError{ + ErrorCode: pd.ErrorStatusEmpty, + } + } + + // Prevent bad filenames and duplicate filenames + for i := range files { + for j := range files { + if i == j { + continue + } + if files[i].Name == files[j].Name { + return ContentVerificationError{ + ErrorCode: pd.ErrorStatusDuplicateFilename, + ErrorContext: []string{ + files[i].Name, + }, + } + } + } + // Check against filesDel + for _, v := range filesDel { + if files[i].Name == v { + return ContentVerificationError{ + ErrorCode: pd.ErrorStatusDuplicateFilename, + ErrorContext: []string{ + files[i].Name, + }, + } + } + } + } + + for i := range files { + if gozaru.Sanitize(files[i].Name) != files[i].Name { + return ContentVerificationError{ + ErrorCode: pd.ErrorStatusInvalidFilename, + ErrorContext: []string{ + files[i].Name, + }, + } + } + + // Validate digest + d, ok := util.ConvertDigest(files[i].Digest) + if !ok { + return ContentVerificationError{ + ErrorCode: pd.ErrorStatusInvalidFileDigest, + ErrorContext: []string{ + files[i].Name, + }, + } + } + + // Decode base64 payload + var err error + payload, err := base64.StdEncoding.DecodeString(files[i].Payload) + if err != nil { + return ContentVerificationError{ + ErrorCode: pd.ErrorStatusInvalidBase64, + ErrorContext: []string{ + files[i].Name, + }, + } + } + + // Calculate payload digest + dp := util.Digest(payload) + if !bytes.Equal(d[:], dp) { + return ContentVerificationError{ + ErrorCode: pd.ErrorStatusInvalidFileDigest, + ErrorContext: []string{ + files[i].Name, + }, + } + } + + // Verify MIME + detectedMIMEType := mime.DetectMimeType(payload) + if detectedMIMEType != files[i].MIME { + return ContentVerificationError{ + ErrorCode: pd.ErrorStatusInvalidMIMEType, + ErrorContext: []string{ + files[i].Name, + detectedMIMEType, + }, + } + } + + if !mime.MimeValid(files[i].MIME) { + return ContentVerificationError{ + ErrorCode: pd.ErrorStatusUnsupportedMIMEType, + ErrorContext: []string{ + files[i].Name, + files[i].MIME, + }, + } + } + } + + return nil +} diff --git a/tlog/tserver/tserver.go b/tlog/tserver/tserver.go index 4fec06476..d27b45acb 100644 --- a/tlog/tserver/tserver.go +++ b/tlog/tserver/tserver.go @@ -32,7 +32,6 @@ import ( "github.com/google/trillian/crypto/keys/der" "github.com/google/trillian/crypto/keyspb" "github.com/google/trillian/crypto/sigpb" - _ "github.com/google/trillian/merkle/rfc6962" "github.com/google/trillian/types" "github.com/gorilla/mux" "github.com/robfig/cron" @@ -503,6 +502,10 @@ func (t *tserver) appendRecord(tree *trillian.Tree, root *trillian.SignedLogRoot // Get inclusion proofs proofs := make([]v1.QueuedLeafProof, 0, len(qlr.QueuedLeaves)) for _, v := range qlr.QueuedLeaves { + fmt.Printf("%x\n", v.Leaf.LeafValue) + fmt.Printf("%x\n", v.Leaf.MerkleLeafHash) + fmt.Printf("%x\n", util.Digest(v.Leaf.LeafValue)) + qllp := v1.QueuedLeafProof{ QueuedLeaf: *v, } From 6f71b3f47d3c95fc01c4aa39ffc400050ba510ee Mon Sep 17 00:00:00 2001 From: Luke Powell Date: Tue, 2 Jun 2020 09:15:39 -0600 Subject: [PATCH 003/449] finish tlogbe --- politeiad/backend/tlogbe/anchor.go | 346 ++++ .../tlogbe/blob/filesystem/filesystem.go | 75 - politeiad/backend/tlogbe/blobentry.go | 46 +- politeiad/backend/tlogbe/blobentry_test.go | 7 - politeiad/backend/tlogbe/encrypt.go | 12 +- politeiad/backend/tlogbe/index.go | 194 +- .../tlogbe/store/filesystem/filesystem.go | 216 +++ .../backend/tlogbe/store/filesystem/log.go | 25 + .../tlogbe/{blob/blob.go => store/store.go} | 11 +- politeiad/backend/tlogbe/tlogbe.go | 1638 ++++++++++++----- politeiad/backend/tlogbe/trillian.go | 290 +-- politeiad/cmd/politeia/README.md | 68 +- politeiad/cmd/politeia/politeia.go | 18 +- politeiad/config.go | 30 +- politeiad/log.go | 24 +- politeiad/politeiad.go | 29 +- util/convert.go | 9 +- 17 files changed, 2279 insertions(+), 759 deletions(-) delete mode 100644 politeiad/backend/tlogbe/blob/filesystem/filesystem.go delete mode 100644 politeiad/backend/tlogbe/blobentry_test.go create mode 100644 politeiad/backend/tlogbe/store/filesystem/filesystem.go create mode 100644 politeiad/backend/tlogbe/store/filesystem/log.go rename politeiad/backend/tlogbe/{blob/blob.go => store/store.go} (64%) diff --git a/politeiad/backend/tlogbe/anchor.go b/politeiad/backend/tlogbe/anchor.go index 200109a52..0077374e5 100644 --- a/politeiad/backend/tlogbe/anchor.go +++ b/politeiad/backend/tlogbe/anchor.go @@ -3,3 +3,349 @@ // license that can be found in the LICENSE file. package tlogbe + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "time" + + "github.com/davecgh/go-spew/spew" + dcrtime "github.com/decred/dcrtime/api/v1" + "github.com/decred/politeia/util" + "github.com/google/trillian/types" +) + +// TODO handle reorgs. A anchor record may become invalid in the case +// of a reorg. + +const ( + // anchorSchedule determines how often we anchor records. dcrtime + // drops an anchor on the hour mark so we submit new anchors a few + // minutes prior. + // Seconds Minutes Hours Days Months DayOfWeek + anchorSchedule = "0 56 * * * *" // At 56 minutes every hour + + // TODO does this need to be unique? + anchorID = "tlogbe" +) + +// anchor represents an anchor, i.e. timestamp, of a trillian tree at a +// specific tree size. The LogRoot is hashed and anchored using dcrtime. Once +// dcrtime drops an anchor, the anchor structure is updated and saved to the +// key-value store. +type anchor struct { + TreeID int64 `json:"treeid"` + LogRoot *types.LogRootV1 `json:"logroot"` + VerifyDigest *dcrtime.VerifyDigest `json:"verifydigest"` +} + +// anchorSave saves the anchor to the key-value store and updates the record +// history of the record that corresponds to the anchor TreeID. This function +// should be called once the dcrtime anchor has been dropped. +// +// This function must be called WITHOUT the read/write lock held. +func (t *tlogbe) anchorSave(a anchor) error { + log.Debugf("Saving anchor for tree %v at height %v", + a.TreeID, a.LogRoot.TreeSize) + + // Sanity checks + switch { + case a.TreeID == 0: + return fmt.Errorf("invalid tree id of 0") + case a.LogRoot == nil: + return fmt.Errorf("log root not found") + case a.VerifyDigest == nil: + return fmt.Errorf("verify digest not found") + } + + // Compute the log root hash. This will be used as the key for the + // anchor in the key-value store. + b, err := a.LogRoot.MarshalBinary() + if err != nil { + return fmt.Errorf("MarshalBinary %v %x: %v", + a.TreeID, a.LogRoot.RootHash, err) + } + logRootHash := util.Hash(b) + + // Get the record history for this tree and update the appropriate + // record content with the anchor. The lock must be held during + // this update. + t.Lock() + defer t.Unlock() + + token := tokenFromTreeID(a.TreeID) + rh, err := t.recordHistory(token) + if err != nil { + return fmt.Errorf("recordHistory: %v", err) + } + + // Aggregate all record content that does not currently have an + // anchor. + noAnchor := make([][]byte, 0, 256) + for _, v := range rh.Versions { + _, ok := rh.Anchors[hex.EncodeToString(v.RecordMetadata)] + if !ok { + noAnchor = append(noAnchor, v.RecordMetadata) + } + for _, merkle := range v.Metadata { + _, ok := rh.Anchors[hex.EncodeToString(merkle)] + if !ok { + noAnchor = append(noAnchor, merkle) + } + } + for _, merkle := range v.Files { + _, ok := rh.Anchors[hex.EncodeToString(merkle)] + if !ok { + noAnchor = append(noAnchor, merkle) + } + } + } + if len(noAnchor) == 0 { + // All record content has already been anchored. This should not + // happen. A tree is only anchored when it has at least one + // unanchored leaf. + return fmt.Errorf("all record content is already anchored") + } + + // Get the leaves for the record content that has not been anchored + // yet. We'll use these to check if the leaf was included in the + // current anchor. + leaves, err := t.leavesByHash(a.TreeID, noAnchor) + if err != nil { + return fmt.Errorf("leavesByHash: %v", err) + } + + var anchorCount int + for _, v := range leaves { + // Check leaf height + if int64(a.LogRoot.TreeSize) < v.LeafIndex { + // Leaf was not included in anchor + continue + } + + // Leaf was included in the anchor + anchorCount++ + + // Sanity check. Get the inclusion proof. This function will + // throw an error if the leaf is not part of the log root. + _, err = t.inclusionProof(a.TreeID, v.MerkleLeafHash, a.LogRoot) + if err != nil { + return fmt.Errorf("inclusionProof %x: %v", v.MerkleLeafHash, err) + } + + // Update record history + rh.Anchors[hex.EncodeToString(v.MerkleLeafHash)] = logRootHash[:] + + log.Debugf("Anchor added to leaf: %x", v.MerkleLeafHash) + } + if anchorCount == 0 { + // This should not happen. If a tree was anchored then it should + // have at least one leaf that was included in the anchor. + return fmt.Errorf("no record content was included in the anchor") + } + + // Save the updated record history + be, err := convertBlobEntryFromRecordHistory(*rh) + if err != nil { + return err + } + b, err = blobify(*be) + if err != nil { + return err + } + err = t.store.Put(keyRecordHistory(rh.Token), b) + if err != nil { + return fmt.Errorf("store Put: %v", err) + } + + log.Debugf("Record history updated") + + // Mark tree as clean if no additional leaves have been added while + // we've been waiting for the anchor to drop. + height, ok := t.dirtyHeight(a.TreeID) + if !ok { + return fmt.Errorf("dirty tree height not found") + } + + log.Debugf("Tree anchored at height %v, current height %v", + a.LogRoot.TreeSize, height) + + if height == a.LogRoot.TreeSize { + // All tree leaves have been anchored. Remove tree from dirty + // list. + t.dirtyDel(a.TreeID) + log.Debugf("Tree removed from dirty list") + } + + return nil +} + +// anchorWait waits for dcrtime to drop an anchor for the provided hashes. +func (t *tlogbe) anchorWait(anchors []anchor, hashes []string) { + // Ensure we are not reentrant + t.Lock() + if t.droppingAnchor { + log.Errorf("waitForAchor: called reentrantly") + return + } + t.droppingAnchor = true + t.Unlock() + + // Whatever happens in this function we must clear droppingAnchor. + defer func() { + t.Lock() + t.droppingAnchor = false + t.Unlock() + }() + + // Wait for anchor to drop + log.Infof("Waiting for anchor to drop") + + // Continually check with dcrtime if the anchor has been dropped. + var ( + period = time.Duration(1) * time.Minute // check every 1 minute + retries = 30 / int(period) // for up to 30 minutes + ticker = time.NewTicker(period) + ) + defer ticker.Stop() + for try := 0; try < retries; try++ { + restart: + <-ticker.C + + log.Debugf("Verify anchor attempt %v/%v", try+1, retries) + + vr, err := util.Verify(anchorID, t.dcrtimeHost, hashes) + if err != nil { + if _, ok := err.(util.ErrNotAnchored); ok { + // Anchor not dropped, try again + continue + } + log.Errorf("anchorWait exiting: %v", err) + return + } + + // Make sure we are actually anchored. + for _, v := range vr.Digests { + if v.ChainInformation.ChainTimestamp == 0 { + log.Debugf("anchorRecords ChainTimestamp 0: %v", v.Digest) + goto restart + } + } + + log.Debugf("%T %v", vr, spew.Sdump(vr)) + + log.Infof("Anchor dropped") + + // Save anchor records + for k, v := range anchors { + // Sanity check + verifyDigest := vr.Digests[k] + b, err := v.LogRoot.MarshalBinary() + if err != nil { + log.Errorf("anchorWait: MarshalBinary %v %x: %v", + v.TreeID, v.LogRoot.RootHash, err) + continue + } + h := util.Hash(b) + if hex.EncodeToString(h[:]) != verifyDigest.Digest { + log.Errorf("anchorWait: digest mismatch: got %x, want %v", + h[:], verifyDigest.Digest) + continue + } + + err = t.anchorSave(anchor{ + TreeID: v.TreeID, + LogRoot: v.LogRoot, + VerifyDigest: v.VerifyDigest, + }) + if err != nil { + log.Errorf("anchorWait: anchorSave %v: %v", v.TreeID, err) + continue + } + } + + log.Info("Anchored records updated") + return + } + + log.Errorf("Anchor drop timeout, waited for: %v", period*time.Minute) +} + +// anchor is a function that is periodically called to anchor dirty trees. +func (t *tlogbe) anchor() { + log.Debugf("Start anchoring process") + + var exitError error // Set on exit if there is an error + defer func() { + if exitError != nil { + log.Errorf("anchorRecords: %v", exitError) + } + }() + + // Get dirty trees + dirty := t.dirtyCopy() + anchors := make([]anchor, 0, len(dirty)) + for treeID := range dirty { + anchors = append(anchors, anchor{ + TreeID: treeID, + }) + } + if len(anchors) == 0 { + log.Infof("Nothing to anchor") + return + } + + // Aggregate the log root for each tree. A hash of the log root is + // what we anchor. + hashes := make([]*[sha256.Size]byte, len(anchors)) + for k, v := range anchors { + log.Debugf("Obtaining anchoring data: %v", v.TreeID) + + tree, err := t.tree(v.TreeID) + if err != nil { + exitError = fmt.Errorf("tree %v: %v", v.TreeID, err) + return + } + _, lr, err := t.signedLogRoot(tree) + if err != nil { + exitError = fmt.Errorf("signedLogRoot %v: %v", v.TreeID, err) + return + } + + anchors[k].LogRoot = lr + lrb, err := lr.MarshalBinary() + if err != nil { + exitError = fmt.Errorf("MarshalBinary %v: %v", v.TreeID, err) + return + } + hashes[k] = util.Hash(lrb) + } + + // Ensure we are not reentrant + t.Lock() + if t.droppingAnchor { + // This shouldn't happen so let's warn the user of something + // misbehaving. + t.Unlock() + log.Errorf("Dropping anchor already in progress") + return + } + t.Unlock() + + // Submit dcrtime anchor request + log.Infof("Anchoring records: %v", len(anchors)) + + err := util.Timestamp(anchorID, t.dcrtimeHost, hashes) + if err != nil { + exitError = err + return + } + + h := make([]string, 0, len(hashes)) + for _, v := range hashes { + h = append(h, hex.EncodeToString(v[:])) + } + + go t.anchorWait(anchors, h) +} diff --git a/politeiad/backend/tlogbe/blob/filesystem/filesystem.go b/politeiad/backend/tlogbe/blob/filesystem/filesystem.go deleted file mode 100644 index 4b6c3cf67..000000000 --- a/politeiad/backend/tlogbe/blob/filesystem/filesystem.go +++ /dev/null @@ -1,75 +0,0 @@ -package filesystem - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - - "github.com/decred/politeia/politeiad/backend/tlogbe/blob" -) - -var ( - _ blob.Blob = (*blobFilesystem)(nil) -) - -// blobFilesystem implements the Blob interface using the filesystem. -type blobFilesystem struct { - path string // Location of files -} - -func (b *blobFilesystem) Put(key string, value []byte) error { - return ioutil.WriteFile(filepath.Join(b.path, key), value, 0600) -} - -func (b *blobFilesystem) PutMulti(blobs map[string][]byte) error { - // TODO implement this - return fmt.Errorf("not implemenated") -} - -func (b *blobFilesystem) Get(key string) ([]byte, error) { - blob, err := ioutil.ReadFile(filepath.Join(b.path, key)) - if err != nil { - return nil, err - } - return blob, nil -} - -func (b *blobFilesystem) Del(key string) error { - err := os.Remove(filepath.Join(b.path, key)) - if err != nil { - // Always return not found - return blob.ErrNotFound - } - return nil -} - -func (b *blobFilesystem) Enum(f func(key string, value []byte) error) error { - files, err := ioutil.ReadDir(b.path) - if err != nil { - return err - } - - for _, file := range files { - if file.Name() == ".." { - continue - } - // TODO should we return filesystem errors or wrap them? - blob, err := b.Get(file.Name()) - if err != nil { - return err - } - err = f(file.Name(), blob) - if err != nil { - return err - } - } - - return nil -} - -func BlobFilesystemNew(path string) *blobFilesystem { - return &blobFilesystem{ - path: path, - } -} diff --git a/politeiad/backend/tlogbe/blobentry.go b/politeiad/backend/tlogbe/blobentry.go index 169607c57..1320dfaeb 100644 --- a/politeiad/backend/tlogbe/blobentry.go +++ b/politeiad/backend/tlogbe/blobentry.go @@ -15,31 +15,28 @@ import ( ) const ( - // dataDescriptor.Type values. These may be freely edited since - // they are solely hints to the application. - dataTypeKeyValue = "kv" // Descriptor is empty but data is key/value - dataTypeMime = "mime" // Descriptor contains a mime type + // Data descriptor types. These may be freely edited since they are + // solely hints to the application. dataTypeStructure = "struct" // Descriptor contains a structure + // Data descriptors dataDescriptorFile = "file" dataDescriptorRecordMetadata = "recordmetadata" dataDescriptorMetadataStream = "metadatastream" dataDescriptorRecordHistory = "recordhistory" - dataDescriptorAnchor = "anchor" -) -// dataKeyValue is an encoded key/value pair. -type dataKeyValue struct { - Key string `json:"key"` - Value string `json:"value"` -} + // Blob entry key prefixes for the key-value store + keyPrefixRecordHistory = "index" + keyPrefixRecordContent = "record" + keyPrefixAnchor = "anchor" +) // dataDescriptor provides hints about a data blob. In practise we JSON encode // this struture and stuff it into blobEntry.DataHint. type dataDescriptor struct { - Type string `json:"type,omitempty"` // Type of data that is stored - Descriptor string `json:"descriptor,omitempty"` // Description of the data - ExtraData string `json:"extradata,omitempty"` // Value to be freely used by caller + Type string `json:"type"` // Type of data that is stored + Descriptor string `json:"descriptor"` // Description of the data + ExtraData string `json:"extradata,omitempty"` // Value to be freely used by caller } // blobEntry is the structure used to store data in the Blob key-value store. @@ -49,6 +46,27 @@ type blobEntry struct { Data string `json:"data"` // Data payload, base64 encoded } +// keyRecordHistory returns the key for the blob key-value store for a record +// history. +func keyRecordHistory(token []byte) string { + return keyPrefixRecordHistory + hex.EncodeToString(token) +} + +// keyRecordContent returns the key for the blob key-value store for any type +// of record content (files, metadata streams, record metadata). Its possible +// for two different records to submit the same file resulting in identical +// merkle leaf hashes. The token is included in the key to ensure that a +// situation like this does not lead to unwanted behavior. +func keyRecordContent(token, merkleLeafHash []byte) string { + return keyPrefixRecordContent + hex.EncodeToString(token) + + hex.EncodeToString(merkleLeafHash) +} + +// keyAnchor returns the key for the blob key-value store for a anchor record. +func keyAnchor(logRootHash []byte) string { + return keyPrefixRecordHistory + hex.EncodeToString(logRootHash) +} + func blobify(be blobEntry) ([]byte, error) { var b bytes.Buffer zw := gzip.NewWriter(&b) diff --git a/politeiad/backend/tlogbe/blobentry_test.go b/politeiad/backend/tlogbe/blobentry_test.go deleted file mode 100644 index 7a588c6eb..000000000 --- a/politeiad/backend/tlogbe/blobentry_test.go +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright (c) 2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package tlogbe - -// TODO implement blobEntry tests diff --git a/politeiad/backend/tlogbe/encrypt.go b/politeiad/backend/tlogbe/encrypt.go index 6848ca4bf..ab5716552 100644 --- a/politeiad/backend/tlogbe/encrypt.go +++ b/politeiad/backend/tlogbe/encrypt.go @@ -13,17 +13,7 @@ import ( "golang.org/x/crypto/nacl/secretbox" ) -func newKey() (*[32]byte, error) { - var k [32]byte - - _, err := io.ReadFull(rand.Reader, k[:]) - if err != nil { - return nil, err - } - - return &k, nil -} - +// TODO use secretbox functions instead func encryptAndPack(data []byte, key *[32]byte) ([]byte, error) { var nonce [24]byte diff --git a/politeiad/backend/tlogbe/index.go b/politeiad/backend/tlogbe/index.go index 65f74c01a..9c5f936ad 100644 --- a/politeiad/backend/tlogbe/index.go +++ b/politeiad/backend/tlogbe/index.go @@ -14,92 +14,119 @@ import ( ) const ( + // Record states stateUnvetted = "unvetted" stateVetted = "vetted" ) -// recordIndex represents an index for a backend Record. The merkleLeafHash -// refers to a trillian LogLeaf.MerkleLeafHash. This value can be used to -// lookup the inclusion proof from a trillian tree as well as the actual data -// from the blob storage layer. All merkleLeafHashes are hex encoded. +var ( + // recordStateFromStatus maps the backend record statuses to one + // of the record states. + recordStateFromStatus = map[backend.MDStatusT]string{ + backend.MDStatusUnvetted: stateUnvetted, + backend.MDStatusIterationUnvetted: stateUnvetted, + backend.MDStatusCensored: stateUnvetted, + backend.MDStatusVetted: stateVetted, + backend.MDStatusArchived: stateVetted, + } +) + +// recordIndex represents an index for a backend Record that contains the +// merkle leaf hash for each piece of record content. The merkle leaf hash +// refers to the trillian LogLeaf.MerkleLeafHash that was returned when the +// record content was appened onto to the trillian tree. Each piece of record +// content is appended as a seperate leaf onto the trillian tree and thus has +// a unique merkle leaf hash. This value can be used to lookup the inclusion +// proof from the trillian tree as well as the actual content from the blob +// key-value store. type recordIndex struct { - RecordMetadata string `json:"recordmetadata"` // RecordMetadata merkleLeafHash - Metadata map[uint64]string `json:"metadata"` // [mdstreamID]merkleLeafHash - Files map[string]string `json:"files"` // [filename]merkleLeafHash + RecordMetadata []byte `json:"recordmetadata"` + Metadata map[uint64][]byte `json:"metadata"` // [metadataID]merkle + Files map[string][]byte `json:"files"` // [filename]merkle } -// recordHistory provides the record index for all versions of a record. The -// recordHistory is stored to the blob storage layer with the token as the key. +// recordHistory contains the record index for all versions of a record and +// the anchor data for all record content. type recordHistory struct { - Token string `json:"token"` + Token []byte `json:"token"` // Record token State string `json:"state"` // Unvetted or vetted Versions map[uint32]recordIndex `json:"versions"` // [version]recordIndex -} -func latestVersion(rh recordHistory) uint32 { - return uint32(len(rh.Versions)) - 1 + // TODO remove anchor when deleting an orphaned blob + // Anchors contains the anchored log root hash for each piece of + // record content. It aggregates the merkle leaf hashes from all + // record index versions. The log root hash can be used to lookup + // the anchor structure from the key-value store, which contains + // the dcrtime inclusion proof, or can be used to obtain the + // inclusion proof from dcrtime itself if needed. The merkle leaf + // hash is hex encoded. The log root hash is a SHA256 digest of the + // encoded LogRootV1. + Anchors map[string][]byte `json:"anchors"` // [merkleLeafHash]logRootHash } -func (t *tlogbe) recordHistory(token string) (*recordHistory, error) { - b, err := t.blob.Get(token) - if err != nil { - return nil, fmt.Errorf("blob get: %v", err) - } - var rh recordHistory - err = json.Unmarshal(b, &rh) - if err != nil { - return nil, err +// String returns the recordHistory printed in human readable format. +func (r *recordHistory) String() string { + s := fmt.Sprintf("Token: %x\n", r.Token) + s += fmt.Sprintf("State: %v\n", r.State) + for k, v := range r.Versions { + s += fmt.Sprintf("Version %v\n", k) + s += fmt.Sprintf(" RecordMD : %x\n", v.RecordMetadata) + for id, merkle := range v.Metadata { + s += fmt.Sprintf(" Metadata %2v: %x\n", id, merkle) + } + for fn, merkle := range v.Files { + s += fmt.Sprintf(" %-11v: %x\n", fn, merkle) + } } - return &rh, nil + return s } -// recordHistoryAdd adds the provided recordIndex as a new version to the -// recordHistory. -func (t *tlogbe) recordHistoryAdd(token string, ri recordIndex) (*recordHistory, error) { - // TODO implement - // A new version can only be added to vetted records - return nil, nil -} - -// recordHistoryUpdate updates the existing version of the recordHistory with -// the provided state and recordIndex. -func (t *tlogbe) recordHistoryUpdate(token, state string, ri recordIndex) (*recordHistory, error) { - // TODO implement - // This will be needed for unvetted updates and metadata updates - return nil, nil -} - -func (t *tlogbe) recordIndexLatest(token string) (*recordIndex, error) { - rh, err := t.recordHistory(token) +// recordIndexUpdate updates the provided recordIndex with the provided +// blobEntries and orphaned blobs then returns the updated recordIndex. +func recordIndexUpdate(idx recordIndex, entries []blobEntry, merkles map[string][]byte, orphaned [][]byte) (*recordIndex, error) { + // Create a record index using the new blob entires + idxNew, err := recordIndexNew(entries, merkles) if err != nil { return nil, err } - latest := rh.Versions[latestVersion(*rh)] - return &latest, nil -} -// recordIndexUpdate updates the provided recordIndex with the provided -// blobEntries then returns the updated recordIndex. -func recordIndexUpdate(r recordIndex, entries []blobEntry, proofs []queuedLeafProof) (*recordIndex, error) { - // TODO implement - return nil, nil -} - -func recordIndexNew(entries []blobEntry, proofs []queuedLeafProof) (*recordIndex, error) { - merkleHashes := make(map[string]string, len(entries)) // [leafValue]merkleLeafHash - for _, v := range proofs { - leafValue := hex.EncodeToString(v.QueuedLeaf.Leaf.LeafValue) - merkleHash := hex.EncodeToString(v.QueuedLeaf.Leaf.MerkleLeafHash) - merkleHashes[leafValue] = merkleHash + // Add existing record index content to the newly created index. + // If a merkle leaf hash is included in the orphaned list it means + // that it is no longer part of the record and should not be + // included. Orphaned merkle leaf hashes are put in a map for + // linear time lookups. + skip := make(map[string]struct{}, len(orphaned)) + for _, v := range orphaned { + skip[hex.EncodeToString(v)] = struct{}{} + } + if _, ok := skip[hex.EncodeToString(idx.RecordMetadata)]; !ok { + idxNew.RecordMetadata = idx.RecordMetadata } + for k, v := range idx.Metadata { + if _, ok := skip[hex.EncodeToString(v)]; ok { + continue + } + idxNew.Metadata[k] = v + } + for k, v := range idx.Files { + if _, ok := skip[hex.EncodeToString(v)]; ok { + continue + } + idxNew.Files[k] = v + } + + return idxNew, nil +} - // Find the merkleLeafHash for each of the record components. The - // blobEntry.Hash is the value that is saved to trillian as the - // LogLeaf.LeafValue. +func recordIndexNew(entries []blobEntry, merkles map[string][]byte) (*recordIndex, error) { + // The merkle leaf hash is used to lookup a blob entry in both the + // trillian tree and the blob key-value store. Create a record + // index by associating each piece of record content with its + // merkle root hash. var ( - recordMD string - metadata = make(map[uint64]string, len(entries)) // [mdstreamID]merkleLeafHash - files = make(map[string]string, len(entries)) // [filename]merkleLeafHash + recordMD []byte + metadata = make(map[uint64][]byte, len(entries)) // [mdstreamID]merkleLeafHash + files = make(map[string][]byte, len(entries)) // [filename]merkleLeafHash ) for _, v := range entries { b, err := base64.StdEncoding.DecodeString(v.DataHint) @@ -113,11 +140,12 @@ func recordIndexNew(entries []blobEntry, proofs []queuedLeafProof) (*recordIndex } switch dd.Descriptor { case dataDescriptorRecordMetadata: - merkleHash, ok := merkleHashes[v.Hash] + merkle, ok := merkles[v.Hash] if !ok { return nil, fmt.Errorf("merkle not found for record metadata") } - recordMD = merkleHash + recordMD = merkle + log.Debugf("Record metadata: %x", merkle) case dataDescriptorMetadataStream: b, err := base64.StdEncoding.DecodeString(v.Data) if err != nil { @@ -128,11 +156,12 @@ func recordIndexNew(entries []blobEntry, proofs []queuedLeafProof) (*recordIndex if err != nil { return nil, err } - merkleHash, ok := merkleHashes[v.Hash] + merkle, ok := merkles[v.Hash] if !ok { return nil, fmt.Errorf("merkle not found for mdstream %v", ms.ID) } - metadata[ms.ID] = merkleHash + metadata[ms.ID] = merkle + log.Debugf("Metadata %v: %x", ms.ID, merkle) case dataDescriptorFile: b, err := base64.StdEncoding.DecodeString(v.Data) if err != nil { @@ -143,11 +172,12 @@ func recordIndexNew(entries []blobEntry, proofs []queuedLeafProof) (*recordIndex if err != nil { return nil, err } - merkleHash, ok := merkleHashes[v.Hash] + merkle, ok := merkles[v.Hash] if !ok { return nil, fmt.Errorf("merkle not found for file %v", f.Name) } - files[f.Name] = merkleHash + files[f.Name] = merkle + log.Debugf("%v: %x", f.Name, merkle) } } @@ -157,3 +187,29 @@ func recordIndexNew(entries []blobEntry, proofs []queuedLeafProof) (*recordIndex Files: files, }, nil } + +// latestVersion returns the most recent version that exists. The versions +// start at 1 so the latest version is the same as the length. +func latestVersion(rh recordHistory) uint32 { + return uint32(len(rh.Versions)) +} + +func (t *tlogbe) recordHistory(token []byte) (*recordHistory, error) { + b, err := t.store.Get(keyRecordHistory(token)) + if err != nil { + return nil, err + } + be, err := deblob(b) + if err != nil { + return nil, err + } + return convertRecordHistoryFromBlobEntry(*be) +} + +func recordHistoryNew(token []byte) recordHistory { + return recordHistory{ + Token: token, + State: stateUnvetted, + Versions: make(map[uint32]recordIndex), + } +} diff --git a/politeiad/backend/tlogbe/store/filesystem/filesystem.go b/politeiad/backend/tlogbe/store/filesystem/filesystem.go new file mode 100644 index 000000000..c5f3604bb --- /dev/null +++ b/politeiad/backend/tlogbe/store/filesystem/filesystem.go @@ -0,0 +1,216 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package filesystem + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sync" + + "github.com/decred/politeia/politeiad/backend/tlogbe/store" +) + +var ( + _ store.Blob = (*fileSystem)(nil) +) + +// fileSystem implements the Blob interface using the file system. +type fileSystem struct { + sync.RWMutex + root string // Location of files +} + +func (f *fileSystem) put(key string, value []byte) error { + return ioutil.WriteFile(filepath.Join(f.root, key), value, 0600) +} + +func (f *fileSystem) Put(key string, value []byte) error { + log.Tracef("Put: %v", key) + + f.Lock() + defer f.Unlock() + + return f.put(key, value) +} + +func (f *fileSystem) get(key string) ([]byte, error) { + b, err := ioutil.ReadFile(filepath.Join(f.root, key)) + if err != nil { + if os.IsNotExist(err) { + return nil, store.ErrNotFound + } + return nil, err + } + return b, nil +} + +func (f *fileSystem) Get(key string) ([]byte, error) { + log.Tracef("Get: %v", key) + + f.RLock() + defer f.RUnlock() + + return f.get(key) +} + +func (f *fileSystem) del(key string) error { + err := os.Remove(filepath.Join(f.root, key)) + if err != nil { + // Always return not found + return store.ErrNotFound + } + return nil +} + +func (f *fileSystem) Del(key string) error { + log.Tracef("Del: %v", key) + + f.Lock() + defer f.Unlock() + + return f.del(key) +} + +func (f *fileSystem) Enum(cb func(key string, value []byte) error) error { + log.Tracef("Enum") + + f.RLock() + defer f.RUnlock() + + files, err := ioutil.ReadDir(f.root) + if err != nil { + return err + } + + for _, file := range files { + if file.Name() == ".." { + continue + } + blob, err := f.Get(file.Name()) + if err != nil { + return err + } + err = cb(file.Name(), blob) + if err != nil { + return err + } + } + + return nil +} + +func (f *fileSystem) multi(ops store.Ops) error { + for _, fn := range ops.Del { + log.Tracef("del: %v", fn) + err := f.del(fn) + if err != nil { + return fmt.Errorf("del %v: %v", fn, err) + } + } + for fn, b := range ops.Put { + log.Tracef("put: %v", fn) + err := f.put(fn, b) + if err != nil { + return fmt.Errorf("put %v: %v", fn, err) + } + } + return nil +} + +func (f *fileSystem) Multi(ops store.Ops) error { + log.Tracef("Multi") + + f.Lock() + defer f.Unlock() + + // Temporarily store del files in case we need to unwind + dels := make(map[string][]byte, len(ops.Del)) + for _, fn := range ops.Del { + b, err := f.get(fn) + if err != nil { + return fmt.Errorf("get %v: %v", fn, err) + } + dels[fn] = b + } + + // Temporarily store existing put files in case we need to unwind. + // An existing put file may or may not exist. + puts := make(map[string][]byte, len(ops.Put)) + for fn := range ops.Put { + b, err := f.get(fn) + if err != nil { + if err == store.ErrNotFound { + // File doesn't exist. This is ok. + continue + } + return fmt.Errorf("get %v: %v", fn, err) + } + puts[fn] = b + } + + err := f.multi(ops) + if err != nil { + // Unwind puts + for fn := range ops.Put { + err2 := f.del(fn) + if err2 != nil { + // This is ok. It just means the file was never saved before + // the multi function exited with an error. + log.Debugf("unwind multi: del %v: %v", fn, err2) + continue + } + } + + // Replace existing puts + var unwindFailed bool + for fn, b := range puts { + err2 := f.put(fn, b) + if err2 != nil { + // We're in trouble! + log.Criticalf("multi unwind: unable to put original file back %v: %v", + fn, err2) + unwindFailed = true + continue + } + } + + // Unwind deletes + for fn, b := range dels { + _, err2 := f.get(fn) + if err2 == nil { + // File was never deleted. Nothing to do. + continue + } + // File was deleted. Put it back. + err2 = f.put(fn, b) + if err2 != nil { + // We're in trouble! + log.Criticalf("multi unwind: unable to put deleted file back %v: %v", + fn, err2) + unwindFailed = true + continue + } + } + + if unwindFailed { + // Print orignal error that caused the unwind then panic + // because the unwind failed. + log.Errorf("multi: %v", err) + panic("multi unwind failed") + } + + return err + } + + return nil +} + +func New(root string) *fileSystem { + return &fileSystem{ + root: root, + } +} diff --git a/politeiad/backend/tlogbe/store/filesystem/log.go b/politeiad/backend/tlogbe/store/filesystem/log.go new file mode 100644 index 000000000..7234a6148 --- /dev/null +++ b/politeiad/backend/tlogbe/store/filesystem/log.go @@ -0,0 +1,25 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package filesystem + +import "github.com/decred/slog" + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log = slog.Disabled + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = slog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using slog. +func UseLogger(logger slog.Logger) { + log = logger +} diff --git a/politeiad/backend/tlogbe/blob/blob.go b/politeiad/backend/tlogbe/store/store.go similarity index 64% rename from politeiad/backend/tlogbe/blob/blob.go rename to politeiad/backend/tlogbe/store/store.go index a6f225a96..2a4525396 100644 --- a/politeiad/backend/tlogbe/blob/blob.go +++ b/politeiad/backend/tlogbe/store/store.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package blob +package store import "errors" @@ -10,10 +10,17 @@ var ( ErrNotFound = errors.New("not found") ) +// Ops specifies multiple Blob operations that should be performed atomically. +type Ops struct { + Put map[string][]byte + Del []string +} + +// Blob represents a blob key-value store. type Blob interface { Put(key string, blob []byte) error // Store blob - PutMulti(blobs map[string][]byte) error // Store multiple blobs atomically Get(key string) ([]byte, error) // Get blob by identifier Del(key string) error // Attempt to delete object Enum(func(key string, blob []byte) error) error // Enumerate over all objects + Multi(Ops) error // Perform multiple operations atomically } diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 08fc30c8d..5f9b8e302 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -15,100 +15,190 @@ import ( "encoding/json" "fmt" "io/ioutil" + "net/url" "os" "path/filepath" "strconv" + "strings" "sync" "time" "github.com/decred/dcrtime/merkle" pd "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/blob" - "github.com/decred/politeia/politeiad/backend/tlogbe/blob/filesystem" + "github.com/decred/politeia/politeiad/backend/tlogbe/store" + "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" "github.com/decred/politeia/util" "github.com/google/trillian" "github.com/google/trillian/crypto/keys" "github.com/google/trillian/crypto/keys/der" "github.com/google/trillian/crypto/keyspb" + "github.com/google/trillian/types" + "github.com/marcopeereboom/sbox" "google.golang.org/grpc" "google.golang.org/grpc/codes" ) +// TODO make unvetted record metdata behavior the same as vetted +// and document why gitbe handles them differently +// TODO if we want to get rid of the cache we will need to make the +// locking more granular and lock on the individual token level + const ( + defaultTrillianKeyFilename = "trillian.key" + defaultEncryptionKeyFilename = "tlogbe.key" + blobsDirname = "blobs" ) var ( _ backend.Backend = (*tlogbe)(nil) + + // statusChanges contains the allowed record status changes + statusChanges = map[backend.MDStatusT]map[backend.MDStatusT]struct{}{ + // Unvetted status changes + backend.MDStatusUnvetted: map[backend.MDStatusT]struct{}{ + backend.MDStatusIterationUnvetted: struct{}{}, + backend.MDStatusVetted: struct{}{}, + backend.MDStatusCensored: struct{}{}, + }, + backend.MDStatusIterationUnvetted: map[backend.MDStatusT]struct{}{ + backend.MDStatusVetted: struct{}{}, + backend.MDStatusCensored: struct{}{}, + }, + + // Vetted status changes + backend.MDStatusVetted: map[backend.MDStatusT]struct{}{ + backend.MDStatusArchived: struct{}{}, + }, + } ) // tlogbe implements the Backend interface. type tlogbe struct { sync.RWMutex shutdown bool - root string // Root directory - dcrtimeHost string // Dcrtimed host - encryptionKey *[32]byte // Private key for encrypting data - exit chan struct{} // Close channel - blob blob.Blob // Blob key-value store - plugins []backend.Plugin // Plugins - - // Trillian setup + homeDir string + dataDir string + dcrtimeHost string + encryptionKey *[32]byte + store store.Blob + plugins []backend.Plugin + + // Trillian + grpc *grpc.ClientConn // Trillian grpc connection client trillian.TrillianLogClient // Trillian log client admin trillian.TrillianAdminClient // Trillian admin client ctx context.Context // Context used for trillian calls privateKey *keyspb.PrivateKey // Trillian signing key publicKey crypto.PublicKey // Trillian public key - // dirty keeps track of which tree is dirty at what height. Dirty - // means that the tree has leaves that have not been timestamped, - // i.e. anchored, onto the decred blockchain. At start-of-day we - // scan all records and look for STH that have not been anchored. - // Note that we only anchor the latest STH and we do so - // opportunistically. If the application is closed and restarted - // it simply will drop a new anchor at the next interval; it will - // not try to finish a prior outstanding anchor drop. - dirty map[int64]int64 // [treeid]height - droppingAnchor bool // anchor dropping is in progress + // dirty keeps track of which trillian trees are dirty and at what + // height. Dirty means that the tree has leaves that have not been + // anchored yet. + dirtyMtx sync.RWMutex + dirty map[int64]uint64 // [treeid]height + droppingAnchor bool // Anchor dropping is in progress } -func tokenFromTreeID(treeID int64) string { +func tokenFromTreeID(treeID int64) []byte { b := make([]byte, binary.MaxVarintLen64) // Converting between int64 and uint64 doesn't change the sign bit, // only the way it's interpreted. binary.LittleEndian.PutUint64(b, uint64(treeID)) - return hex.EncodeToString(b) + return b } -func treeIDFromToken(token string) (int64, error) { - b, err := hex.DecodeString(token) - if err != nil { - return 0, err +func treeIDFromToken(token []byte) int64 { + return int64(binary.LittleEndian.Uint64(token)) +} + +func tokenString(token []byte) string { + return hex.EncodeToString(token) +} + +// statusChangeIsAllowed returns whether the provided status change is allowed +// by tlogbe. An invalid 'from' status will panic since the 'from' status +// represents the existing status of a record and should never be invalid. +func statusChangeIsAllowed(from, to backend.MDStatusT) bool { + allowed, ok := statusChanges[from] + if !ok { + e := fmt.Sprintf("status invalid: %v", from) + panic(e) + } + _, ok = allowed[to] + return ok +} + +func (t *tlogbe) dirtyAdd(treeID int64, treeSize uint64) { + log.Tracef("dirtyAdd treeID:%v treeSize:%v", treeID, treeSize) + + t.dirtyMtx.Lock() + defer t.dirtyMtx.Unlock() + + t.dirty[treeID] = treeSize +} + +func (t *tlogbe) dirtyDel(treeID int64) { + log.Tracef("dirtyDel: %v", treeID) + + t.dirtyMtx.Lock() + defer t.dirtyMtx.Unlock() + + delete(t.dirty, treeID) +} + +func (t *tlogbe) dirtyHeight(treeID int64) (uint64, bool) { + log.Tracef("dirtyHeight: %v", treeID) + + t.dirtyMtx.RLock() + defer t.dirtyMtx.RUnlock() + + h, ok := t.dirty[treeID] + return h, ok +} + +func (t *tlogbe) dirtyCopy() map[int64]uint64 { + log.Tracef("dirtyCopy") + + t.dirtyMtx.RLock() + defer t.dirtyMtx.RUnlock() + + dirtyCopy := make(map[int64]uint64, len(t.dirty)) + for k, v := range t.dirty { + dirtyCopy[k] = v } - return int64(binary.LittleEndian.Uint64(b)), nil + + return dirtyCopy } -func merkleRoot(files []backend.File) [sha256.Size]byte { +func merkleRoot(files []backend.File) (*[sha256.Size]byte, error) { hashes := make([]*[sha256.Size]byte, 0, len(files)) for _, v := range files { + b, err := hex.DecodeString(v.Digest) + if err != nil { + return nil, err + } var d [sha256.Size]byte - copy(d[:], v.Digest) + copy(d[:], b) hashes = append(hashes, &d) } - return *merkle.Root(hashes) + return merkle.Root(hashes), nil } -func recordMetadataNew(token string, files []backend.File, status backend.MDStatusT, iteration uint64) backend.RecordMetadata { - m := merkleRoot(files) - return backend.RecordMetadata{ +func recordMetadataNew(token []byte, files []backend.File, status backend.MDStatusT, iteration uint64) (*backend.RecordMetadata, error) { + m, err := merkleRoot(files) + if err != nil { + return nil, err + } + return &backend.RecordMetadata{ Version: backend.VersionRecordMD, Iteration: iteration, Status: status, Merkle: hex.EncodeToString(m[:]), Timestamp: time.Now().Unix(), - Token: token, - } + Token: tokenString(token), + }, nil } func convertBlobEntryFromFile(f backend.File) (*blobEntry, error) { @@ -305,147 +395,231 @@ func convertLeavesFromBlobEntries(entries []blobEntry) ([]*trillian.LogLeaf, err return leaves, nil } -func convertBlobEntries(mdstreams []backend.MetadataStream, files []backend.File, recordMD backend.RecordMetadata) ([]blobEntry, error) { - blobStreams, err := convertBlobEntriesFromMetadataStreams(mdstreams) +func convertRecordHistoryFromBlobEntry(be blobEntry) (*recordHistory, error) { + // Decode and validate the DataHint + b, err := base64.StdEncoding.DecodeString(be.DataHint) if err != nil { - return nil, err + return nil, fmt.Errorf("decode DataHint: %v", err) } - blobFiles, err := convertBlobEntriesFromFiles(files) + var dd dataDescriptor + err = json.Unmarshal(b, &dd) if err != nil { - return nil, err + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorRecordHistory { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorRecordHistory) } - blobRecordMD, err := convertBlobEntryFromRecordMetadata(recordMD) + + // Decode the MetadataStream + b, err = base64.StdEncoding.DecodeString(be.Data) if err != nil { - return nil, err + return nil, fmt.Errorf("decode Data: %v", err) + } + var rh recordHistory + err = json.Unmarshal(b, &rh) + if err != nil { + return nil, fmt.Errorf("unmarshal recordHistory: %v", err) } - // The recordMD is intentionally put first. In the event that the - // record indexes ever have to be recovered by walking the trillian - // trees then a recordMD will mark the start of a new version of - // the record. This should never be required, but just in case. - entries := make([]blobEntry, 0, len(blobStreams)+len(blobFiles)+1) - entries = append(entries, *blobRecordMD) - entries = append(entries, blobStreams...) - entries = append(entries, blobFiles...) + return &rh, nil +} - return entries, nil +func filesApplyChanges(files, filesAdd []backend.File, filesDel []string) []backend.File { + del := make(map[string]struct{}, len(filesDel)) + for _, fn := range filesDel { + del[fn] = struct{}{} + } + f := make([]backend.File, 0, len(files)+len(filesAdd)) + for _, v := range files { + if _, ok := del[v.Name]; ok { + continue + } + f = append(f, v) + } + for _, v := range filesAdd { + f = append(f, v) + } + return f +} + +func metadataStreamsApplyChanges(md, mdAppend, mdOverwrite []backend.MetadataStream) []backend.MetadataStream { + // Include all overwrites + metadata := make([]backend.MetadataStream, 0, len(md)+len(mdOverwrite)) + for _, v := range mdOverwrite { + metadata = append(metadata, v) + } + + // Add in existing metadata that wasn't overwritten + overwrite := make(map[uint64]struct{}, len(mdOverwrite)) + for _, v := range mdOverwrite { + overwrite[v.ID] = struct{}{} + } + for _, v := range md { + if _, ok := overwrite[v.ID]; ok { + // Metadata has already been overwritten + continue + } + metadata = append(metadata, v) + } + + // Apply appends + appends := make(map[uint64]backend.MetadataStream, len(mdAppend)) + for _, v := range mdAppend { + appends[v.ID] = v + } + for i, v := range metadata { + ms, ok := appends[v.ID] + if !ok { + continue + } + buf := bytes.NewBuffer([]byte(v.Payload)) + buf.WriteString(ms.Payload) + metadata[i].Payload = buf.String() + } + + return metadata } // blobEntriesAppend appends the provided blob entries onto the trillain tree // that corresponds to the provided token. An error is returned if any leaves // are not successfully added. The only exception to this is if a leaf is not -// added because it is a duplicate. -func (t *tlogbe) blobEntriesAppend(token string, entries []blobEntry) ([]queuedLeafProof, *trillian.SignedLogRoot, error) { +// appended because it is a duplicate. This can happen in certain situations +// such as when a file is deleted from a record then added back to the record +// without being altered. The order of the returned leaf proofs is not +// guaranteed. +func (t *tlogbe) blobEntriesAppend(token []byte, entries []blobEntry) ([]leafProof, *types.LogRootV1, error) { // Setup request - treeID, err := treeIDFromToken(token) - if err != nil { - return nil, nil, err - } + treeID := treeIDFromToken(token) leaves, err := convertLeavesFromBlobEntries(entries) if err != nil { return nil, nil, err } // Append leaves - proofs, slr, err := t.leavesAppend(treeID, leaves) + queued, lr, err := t.leavesAppend(treeID, leaves) if err != nil { return nil, nil, fmt.Errorf("leavesAppend: %v", err) } - if len(proofs) != len(leaves) { - // Sanity check. Even if a leaf fails to be added there should + if len(queued) != len(leaves) { + // Sanity check. Even if a leaf fails to be appended there should // still be a QueuedLogLeaf with the error code. - return nil, nil, fmt.Errorf("proofs do not match leaves: got %v, want %v", - len(proofs), len(leaves)) + return nil, nil, fmt.Errorf("wrong number of leaves: got %v, want %v", + len(queued), len(leaves)) } - missing := make([]string, 0, len(proofs)) - for _, v := range proofs { + + // Convert queuedLeafProofs to leafProofs. Fail if any of the + // leaves were not appended successfully. The exception to this is + // if the leaf was not appended because it was a duplicate. + proofs := make([]leafProof, 0, len(queued)) + dups := make([][]byte, 0, len(queued)) + failed := make([]string, 0, len(queued)) + for _, v := range queued { c := codes.Code(v.QueuedLeaf.GetStatus().GetCode()) - // Its ok if the error is because of a duplicate since it still - // allows us to retreive an inclusion proof. - if c != codes.OK && c != codes.AlreadyExists { - missing = append(missing, fmt.Sprint("%v", c)) + switch c { + case codes.OK: + // Leaf successfully appended to tree + proofs = append(proofs, leafProof{ + Leaf: v.QueuedLeaf.Leaf, + Proof: v.Proof, + }) + + case codes.AlreadyExists: + // We need to retrieve the leaf proof manually for this leaf + // because it was a duplicate. This can happen in certain + // situations such as when a record file is deleted then added + // back at a later date without being altered. A duplicate in + // trillian is ok. A duplicate in the storage layer is not ok + // so check the storage layer first to ensure this is not a + // real duplicate. + m := merkleLeafHash(v.QueuedLeaf.Leaf.LeafValue) + _, err := t.store.Get(keyRecordContent(token, m)) + if err == nil { + return nil, nil, fmt.Errorf("duplicate found in store: %x", m) + } + dups = append(dups, m) + + log.Debugf("Duplicate leaf %x, retreiving proof manually", m) + + default: + // All other errors. This is not ok. + failed = append(failed, fmt.Sprint("%v", c)) } } - if len(missing) > 0 { - return nil, nil, fmt.Errorf("leaves failed with error codes %v", missing) + if len(failed) > 0 { + return nil, nil, fmt.Errorf("append leaves failed: %v", failed) + } + + // Retrieve leaf proofs for duplicates + if len(dups) > 0 { + p, err := t.leafProofs(treeID, dups, lr) + if err != nil { + return nil, nil, fmt.Errorf("leafProofs: %v", err) + } + proofs = append(proofs, p...) } - return proofs, slr, nil + return proofs, lr, nil } -func (t *tlogbe) recordMetadata(key, state string) (*backend.RecordMetadata, error) { - b, err := t.blob.Get(key) +func (t *tlogbe) recordMetadata(token, merkleLeafHash []byte) (*backend.RecordMetadata, error) { + log.Tracef("recordMetadata: %x", merkleLeafHash) + + key := keyRecordContent(token, merkleLeafHash) + b, err := t.store.Get(key) if err != nil { - return nil, fmt.Errorf("blob Get: %v", err) + return nil, err } - var be *blobEntry - switch state { - case stateUnvetted: - // Unvetted blobs will be encrypted - be, err = deblobEncrypted(b, t.encryptionKey) - if err != nil { - return nil, err - } - case stateVetted: - // Vetted blobs will not be encrypted - be, err = deblob(b) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("unkown record history state: %v", state) + be, err := deblob(b) + if err != nil { + return nil, err } rm, err := convertRecordMetadataFromBlobEntry(*be) if err != nil { return nil, err } + return rm, nil } -func (t *tlogbe) metadataStream(key, state string) (*backend.MetadataStream, error) { - b, err := t.blob.Get(key) +func (t *tlogbe) metadataStream(token, merkleLeafHash []byte) (*backend.MetadataStream, error) { + log.Tracef("metadataStream: %x", merkleLeafHash) + + key := keyRecordContent(token, merkleLeafHash) + b, err := t.store.Get(key) if err != nil { - return nil, fmt.Errorf("blob Get: %v", err) + return nil, err } - var be *blobEntry - switch state { - case stateUnvetted: - // Unvetted blobs will be encrypted - be, err = deblobEncrypted(b, t.encryptionKey) - if err != nil { - return nil, err - } - case stateVetted: - // Vetted blobs will not be encrypted - be, err = deblob(b) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("unkown record history state: %v", state) + be, err := deblob(b) + if err != nil { + return nil, err } ms, err := convertMetadataStreamFromBlobEntry(*be) if err != nil { return nil, err } + return ms, nil } -func (t *tlogbe) file(key, state string) (*backend.File, error) { - b, err := t.blob.Get(key) +func (t *tlogbe) file(token, merkleLeafHash []byte, state string) (*backend.File, error) { + log.Tracef("file: %v %x", state, merkleLeafHash) + + key := keyRecordContent(token, merkleLeafHash) + b, err := t.store.Get(key) if err != nil { - return nil, fmt.Errorf("blob Get: %v", err) + return nil, fmt.Errorf("store Get: %v", err) } var be *blobEntry switch state { case stateUnvetted: - // Unvetted blobs will be encrypted + // Unvetted backend File blobs will be encrypted be, err = deblobEncrypted(b, t.encryptionKey) if err != nil { return nil, err } case stateVetted: - // Vetted blobs will not be encrypted + // Vetted backend File blobs will not be encrypted be, err = deblob(b) if err != nil { return nil, err @@ -457,31 +631,37 @@ func (t *tlogbe) file(key, state string) (*backend.File, error) { if err != nil { return nil, err } + return f, nil } // record returns the backend record given the record index. -func (t *tlogbe) record(ri recordIndex, version uint32, state string) (*backend.Record, error) { - recordMD, err := t.recordMetadata(ri.RecordMetadata, state) +func (t *tlogbe) record(token []byte, ri recordIndex, version uint32, state string) (*backend.Record, error) { + log.Tracef("record: %x", token) + + recordMD, err := t.recordMetadata(token, ri.RecordMetadata) if err != nil { - return nil, fmt.Errorf("recordMetadata: %v", err) + return nil, fmt.Errorf("recordMetadata %x: %v", ri.RecordMetadata, err) } + metadata := make([]backend.MetadataStream, 0, len(ri.Metadata)) - for _, v := range ri.Metadata { - ms, err := t.metadataStream(v, state) + for id, merkle := range ri.Metadata { + ms, err := t.metadataStream(token, merkle) if err != nil { - return nil, fmt.Errorf("metadataStream %v: %v", v, err) + return nil, fmt.Errorf("metadataStream %v %x: %v", id, merkle, err) } metadata = append(metadata, *ms) } + files := make([]backend.File, 0, len(ri.Files)) - for _, v := range ri.Files { - f, err := t.file(v, state) + for fn, merkle := range ri.Files { + f, err := t.file(token, merkle, state) if err != nil { - return nil, fmt.Errorf("file %v: %v", v, err) + return nil, fmt.Errorf("file %v %x: %v", fn, merkle, err) } files = append(files, *f) } + return &backend.Record{ Version: strconv.FormatUint(uint64(version), 10), RecordMetadata: *recordMD, @@ -494,232 +674,505 @@ func (t *tlogbe) record(ri recordIndex, version uint32, state string) (*backend. // appending the hashes of the record contents onto the associated trillian // tree, saving the record contents as blobs in the storage layer, and saving // a record index. This function assumes the record contents have already been -// validated. The blobs for unvetted record files (just files, not metadata) -// are encrypted before being saved to the storage layer. -func (t *tlogbe) recordSave(token string, metadata []backend.MetadataStream, files []backend.File, recordMD backend.RecordMetadata) error { - // TODO implement this and remove the recordSaveUnvetted and - // recordSaveVetted functions. - return nil -} - -// recordUpdate updates the current version of the provided record. This -// includes appending the hashes of the record contents onto the associated -// trillian tree, saving the record contents as blobs in the storage layer, and -// updating the existing record index. This function assumes the record -// contents have already been validated. The blobs for unvetted record files -// (just files, not metadata) are encrypted before being saved to the storage -// layer. -// -// It is the responsibility of the caller to remove any blobs that were -// orphaned by this update from the storage layer. -func (t *tlogbe) recordUpdate(token string, metadata []backend.MetadataStream, files []backend.File, - recordMD backend.RecordMetadata) error { - // TODO implement this and remove the recordSaveUnvetted and - // recordSaveVetted functions. - return nil -} - -// recordSaveUnvetted saves the hashes of the record contents to the provided -// trillian tree, saves the record contents as blobs in the storage layer, and -// saves an index of the record that can be used to lookup both the trillian -// inclusion proofs and the blobs required to reconstruct the record. This -// function assumes the record contents have already been validated. Unvetted -// record blobs are encrypted before being saved to the storage layer. -func (t *tlogbe) recordSaveUnvetted(token string, mdstreams []backend.MetadataStream, files []backend.File, recordMD backend.RecordMetadata) error { +// validated. +func (t *tlogbe) recordSave(token []byte, metadata []backend.MetadataStream, files []backend.File, recordMD backend.RecordMetadata, rh recordHistory) (*backend.Record, error) { // Prepare blob entries - entries, err := convertBlobEntries(mdstreams, files, recordMD) + recordMDEntry, err := convertBlobEntryFromRecordMetadata(recordMD) if err != nil { - return err + return nil, err + } + metadataEntries, err := convertBlobEntriesFromMetadataStreams(metadata) + if err != nil { + return nil, err + } + fileEntries, err := convertBlobEntriesFromFiles(files) + if err != nil { + return nil, err } + // The RecordMetadata is intentionally put first so that it is + // added to the trillian tree first. If we ever need to walk the + // tree a RecordMetadata will signify the start of a new record. + entries := make([]blobEntry, 0, len(metadata)+len(files)+1) + entries = append(entries, *recordMDEntry) + entries = append(entries, metadataEntries...) + entries = append(entries, fileEntries...) // Append leaves onto trillian tree for all record contents proofs, _, err := t.blobEntriesAppend(token, entries) if err != nil { - return err + return nil, fmt.Errorf("blobEntriesAppend %x: %v", token, err) } - // The merkleLeafHash is used as the key for the blob in the - // storage layer. - merkleHashes := make(map[string]string, len(entries)) // [leafValue]merkleLeafHash + // Aggregate the merkle leaf hashes. These are used as the keys + // when saving blobs to the key-value store. + merkles := make(map[string][]byte, len(entries)) // [leafValue]merkleLeafHash for _, v := range proofs { - leafValue := hex.EncodeToString(v.QueuedLeaf.Leaf.LeafValue) - merkleHash := hex.EncodeToString(v.QueuedLeaf.Leaf.MerkleLeafHash) - merkleHashes[leafValue] = merkleHash + k := hex.EncodeToString(v.Leaf.LeafValue) + merkles[k] = v.Leaf.MerkleLeafHash } - // Prepare blobs for the storage layer. Unvetted record blobs are - // encrypted. - blobs := make(map[string][]byte, len(entries)) // [merkleLeafHash]blob + // Aggregate the blob entry hashes of all the file blobs. These + // are used to determine if the blob entry should be encrypted. + // Unvetted files are stored as encrypted blobs. All other record + // content is stored as unencrypted blobs. + fileHashes := make(map[string]struct{}, len(fileEntries)) + for _, v := range fileEntries { + fileHashes[v.Hash] = struct{}{} + } + + // Prepare blobs for the storage layer. Unvetted files are stored + // as encrypted blobs. The merkle leaf hash is used as the key in + // the blob key-value store for all record content. + blobs := make(map[string][]byte, len(entries)) // [key]blob for _, v := range entries { - // Get the merkle leaf hash that corresponds to this blob entry - merkle, ok := merkleHashes[v.Hash] + merkle, ok := merkles[v.Hash] if !ok { - return fmt.Errorf("no merkle leaf hash found for %v", v.Hash) + return nil, fmt.Errorf("no merkle leaf hash for %v", v.Hash) } - // Check if this blob already exists in the storage layer. This - // can happen when a new record version is submitted but some of - // the the files and mdstreams remains the same. - _, err = t.blob.Get(merkle) - if err == nil { - // Blob aleady exists. Skip it. - continue + var b []byte + _, ok = fileHashes[v.Hash] + if ok && rh.State == stateUnvetted { + // This is an unvetted file. Store it as an encrypted blob. + b, err = blobifyEncrypted(v, t.encryptionKey) + if err != nil { + return nil, err + } + } else { + // All other record content is store as unencrypted blobs. + b, err = blobify(v) + if err != nil { + return nil, err + } } - // Prepare encrypted blob - b, err := blobifyEncrypted(v, t.encryptionKey) - if err != nil { - return err - } - blobs[merkle] = b + blobs[keyRecordContent(token, merkle)] = b } - // Update the record history and blobify it - ri, err := recordIndexNew(entries, proofs) - if err != nil { - return err - } - rh, err := t.recordHistoryUpdate(token, stateUnvetted, *ri) + // Retrieve the record history and add a new record index version + // to it. This updated record history will be saved with the rest + // of the blobs. The token is used as the record history key. + ri, err := recordIndexNew(entries, merkles) if err != nil { - return err + return nil, err } - be, err := convertBlobEntryFromRecordHistory(*rh) + rh.Versions[latestVersion(rh)+1] = *ri + be, err := convertBlobEntryFromRecordHistory(rh) if err != nil { - return err + return nil, err } b, err := blobify(*be) if err != nil { - return err + return nil, err } - blobs[token] = b + blobs[keyRecordHistory(token)] = b // Save all blobs - err = t.blob.PutMulti(blobs) + log.Debugf("Saving %v blobs to kv store", len(blobs)) + + err = t.store.Multi(store.Ops{ + Put: blobs, + }) if err != nil { - return fmt.Errorf("blob PutMulti: %v", err) + return nil, fmt.Errorf("store Multi: %v", err) } - return nil -} + // Lookup new version of the record + log.Debugf("Record index:\n%v", rh.String()) -// recordSaveVetted saves the hashes of the record contents to the provided -// trillian tree, saves the record contents as blobs in the storage layer, and -// saves a record index that can be used to lookup both the trillian inclusion -// proofs and the blobs required to reconstruct the record. This function -// assumes the record contents have already been validated. Vetted record blobs -// are NOT encrypted before being saved to the storage layer. -func (t *tlogbe) recordSaveVetted(token string, version uint32, mdstreams []backend.MetadataStream, files []backend.File, recordMD backend.RecordMetadata) error { - // Prepare blob entries - entries, err := convertBlobEntries(mdstreams, files, recordMD) + version := latestVersion(rh) + r, err := t.record(token, *ri, version, rh.State) if err != nil { - return err + return nil, fmt.Errorf("record: %v", err) } - // Append leaves onto trillian tree for all record contents - proofs, _, err := t.blobEntriesAppend(token, entries) - if err != nil { - return err - } + return r, nil +} + +// recordUpdate updates the current version of the provided record. This +// includes appending the hashes of the record contents onto the associated +// trillian tree, saving the record contents as blobs in the storage layer, and +// updating the existing record index. This function assumes the record +// contents have already been validated. The blobs for unvetted record files +// (just files, not metadata) are encrypted before being saved to the storage +// layer. +// +// This function must be called WITH the lock held. +func (t *tlogbe) recordUpdate(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string, rm backend.RecordMetadata, rh recordHistory) (*backend.Record, error) { + log.Tracef("recordUpdate: %x", token) - // The merkleLeafHash is used as the key for the blob in the + // Get the record index + idx := rh.Versions[latestVersion(rh)] + + // entries is used to keep track of the new record content that + // needs to be added to the trillian tree and saved to the blob + // storage layer. The blobEntry.Hash is all that is appened onto + // the trillian tree. The full blobEntry is saved to the blob // storage layer. - merkleHashes := make(map[string]string, len(entries)) // [leafValue]merkleLeafHash - for _, v := range proofs { - leafValue := hex.EncodeToString(v.QueuedLeaf.Leaf.LeafValue) - merkleHash := hex.EncodeToString(v.QueuedLeaf.Leaf.MerkleLeafHash) - merkleHashes[leafValue] = merkleHash - } + l := len(mdAppend) + len(mdOverwrite) + len(filesAdd) + len(filesDel) + entries := make([]blobEntry, 0, l) - // Prepare blobs for the storage layer. Vetted record blobs are not - // encrypted. - blobs := make(map[string][]byte, len(entries)) // [merkleLeafHash]blob - for _, v := range entries { - // Get the merkle leaf hash that corresponds to this blob entry - merkle, ok := merkleHashes[v.Hash] - if !ok { - return fmt.Errorf("no merkle leaf hash found for %v", v.Hash) - } + // encrypt tracks the blob entries that will be saved as encrypted + // blobs. + encrypt := make(map[string]struct{}, l) // [hash]struct{} - // Check if this blob already exists in the storage layer. This - // can happen when a new record version is submitted but some of - // the the files and mdstreams remains the same. - _, err = t.blob.Get(merkle) - if err == nil { - // Blob aleady exists. Skip it. - continue + // orphaned tracks the merkle leaf hashes of the blobs that have + // been orphaned by this update. An orphaned blob is one that does + // not correspond to a specific record version. These blobs are + // deleted from the record index and the blob storage layer. + orphaned := make([][]byte, 0, l) + + // Append metadata + for _, v := range mdAppend { + // Lookup existing metadata stream. It's ok if one does not + // already exist. A new one will be created. + ms := backend.MetadataStream{ + ID: v.ID, + } + merkle, ok := idx.Metadata[v.ID] + if ok { + // Metadata stream already exists. Retrieve it. + m, err := t.metadataStream(token, merkle) + if err != nil { + return nil, fmt.Errorf("metadataStream %v: %v", v.ID, err) + } + ms.Payload = m.Payload + + // This metadata stream blob will be orphaned by the metadata + // stream blob with the newly appended data. + orphaned = append(orphaned, merkle) } - // Prepare blob - b, err := blobify(v) + // Append new data + buf := bytes.NewBuffer([]byte(ms.Payload)) + buf.WriteString(v.Payload) + ms.Payload = buf.String() + be, err := convertBlobEntryFromMetadataStream(ms) if err != nil { - return err + return nil, err } - blobs[merkle] = b - } - // Add a record index to the record history and blobify it - ri, err := recordIndexNew(entries, proofs) - if err != nil { - return err - } - rh, err := t.recordHistoryAdd(token, *ri) - if err != nil { - return err - } - be, err := convertBlobEntryFromRecordHistory(*rh) - if err != nil { - return err - } - b, err := blobify(*be) - if err != nil { - return err - } - blobs[token] = b + // Save updated metadata stream + entries = append(entries, *be) - // Save all blobs - err = t.blob.PutMulti(blobs) - if err != nil { - return fmt.Errorf("blob PutMulti: %v", err) + if ok { + log.Debugf("Append MD %v, orphaned blob %x", v.ID, merkle) + } else { + log.Debugf("Append MD %v, no orphaned blob", v.ID) + } } - return nil -} + // Overwrite metdata + for _, v := range mdOverwrite { + be, err := convertBlobEntryFromMetadataStream(v) + if err != nil { + return nil, err + } -// New satisfies the Backend interface. -func (t *tlogbe) New(metadata []backend.MetadataStream, files []backend.File) (*backend.RecordMetadata, error) { - log.Tracef("New") + // Check if this metadata stream is a duplicate + merkle, ok := idx.Metadata[v.ID] + if ok { + // Metadata stream already exists. Check if there are any + // changes being made to it. + b, err := hex.DecodeString(be.Hash) + if err != nil { + return nil, err + } + m := merkleLeafHash(b) + if bytes.Equal(merkle, m) { + // Existing metadata stream is the same as the new metadata + // stream. No need to save it again. + continue + } + } - // Validate record contents - err := backend.VerifyContent(metadata, files, []string{}) - if err != nil { - return nil, err - } + // Metdata stream is not a duplicate of the existing metadata + // stream. Save it. The existing metadata stream blob will be + // orphaned. + entries = append(entries, *be) + orphaned = append(orphaned, merkle) - // Create a new trillian tree. The treeID is used as the record - // token. - // TODO handle token prefix collisions - tree, _, err := t.treeNew() - if err != nil { - return nil, fmt.Errorf("treeNew: %v", err) + log.Debugf("Overwrite MD %v, orphaned blob %x", v.ID, merkle) } - token := tokenFromTreeID(tree.TreeId) - // Save record - rm := recordMetadataNew(token, files, backend.MDStatusUnvetted, 1) - err = t.recordSaveUnvetted(token, metadata, files, rm) - if err != nil { - return nil, fmt.Errorf("recordSaveUnvetted %v: %v", token, err) - } + // Add files + for _, v := range filesAdd { + be, err := convertBlobEntryFromFile(v) + if err != nil { + return nil, err + } - log.Infof("New record %v %v", tree.TreeId, token) + // Check if this file is a duplicate + merkle, ok := idx.Files[v.Name] + if ok { + // File name already exists. Check if there are any changes + // being made to it. + b, err := hex.DecodeString(be.Hash) + if err != nil { + return nil, err + } + m := merkleLeafHash(b) + if bytes.Equal(merkle, m) { + // Existing file is the same as the new file. No need to save + // it again. + continue + } + + // New file is different. The existing file will be orphaned. + orphaned = append(orphaned, merkle) + } - return &rm, nil + // Save new file + entries = append(entries, *be) + + log.Debugf("Add file %v, orphaned blob %x", v.Name, merkle) + + // Unvetted files are stored as encryted blobs + if rh.State == stateUnvetted { + encrypt[be.Hash] = struct{}{} + } + } + + // Delete files + for _, fn := range filesDel { + // Ensure file exists + merkle, ok := idx.Files[fn] + if !ok { + return nil, backend.ContentVerificationError{ + ErrorCode: pd.ErrorStatusFileNotFound, + ErrorContext: []string{fn}, + } + } + + // This file will be orphaned + orphaned = append(orphaned, merkle) + log.Debugf("Del file %v, orphaned blob %x", fn, merkle) + } + + // Check if the record metadata status is being updated + var statusUpdate bool + var decryptFiles bool + currRM, err := t.recordMetadata(token, idx.RecordMetadata) + if err != nil { + return nil, fmt.Errorf("recordMetadata %v: %v", + idx.RecordMetadata, err) + } + if currRM.Status != rm.Status { + // The record status is being updated + statusUpdate = true + + log.Debugf("Record status is being updated from %v to %v", + currRM.Status, rm.Status) + + // Check if the status is being updated from an unvetted status + // to a vetted status. If so, we will need to decrypt the record + // files and save them as unencrypted blobs. + from := recordStateFromStatus[currRM.Status] + to := recordStateFromStatus[rm.Status] + if from == stateUnvetted && to == stateVetted { + decryptFiles = true + } + } + + // Ensure changes were actually made. The only time we allow an + // update with no changes to the files or metadata streams is when + // the record status is being updated. + if len(entries) == 0 && len(orphaned) == 0 && !statusUpdate { + return nil, backend.ErrNoChanges + } + + // Handle record metadata. The record metadata will have changed + // if the record files are being updated or the record status is + // being updated. It will remain unchanged if this is a metadata + // stream only update. + // + // tlogbe manually updates the record status the first time that + // an unvetted record has its files changed. The status gets + // flipped from Unvetted to UnvettedIteration. All other status + // changes are initiated by the client. + filesUpdated := len(filesAdd) != 0 || len(filesDel) != 0 + if filesUpdated && rm.Status == backend.MDStatusUnvetted { + rm.Status = backend.MDStatusIterationUnvetted + } + be, err := convertBlobEntryFromRecordMetadata(rm) + if err != nil { + return nil, err + } + b, err := hex.DecodeString(be.Hash) + if err != nil { + return nil, err + } + m := merkleLeafHash(b) + if !bytes.Equal(idx.RecordMetadata, m) { + // Record metadata is not the same. Save the new one. The + // existing record metadata is going to be orphaned. + entries = append(entries, *be) + orphaned = append(orphaned, idx.RecordMetadata) + + log.Debugf("Record metadata updated") + } + + // Append leaves onto the trillian tree + proofs, _, err := t.blobEntriesAppend(token, entries) + if err != nil { + return nil, fmt.Errorf("blobEntriesAppend %x: %v", token, err) + } + + // Aggregate the merkle leaf hashes. These are used as the keys + // when saving a blob to the key-value store. + merkles := make(map[string][]byte, len(entries)) // [leafValue]merkleLeafHash + for _, v := range proofs { + k := hex.EncodeToString(v.Leaf.LeafValue) + merkles[k] = v.Leaf.MerkleLeafHash + } + + // Update the record index and record history + ri, err := recordIndexUpdate(idx, entries, merkles, orphaned) + if err != nil { + return nil, err + } + rh.Versions[latestVersion(rh)] = *ri + state, ok := recordStateFromStatus[rm.Status] + if !ok { + return nil, fmt.Errorf("status %v does not map to a state", rm.Status) + } + rh.State = state + + // Blobify all the things. The merkle leaf hash is used as the key + // for record content in the key-value store. + blobs := make(map[string][]byte, len(entries)+1) // [key][]byte + for _, v := range entries { + // Get the merkle leaf hash for this blob entry + merkle, ok := merkles[v.Hash] + if !ok { + return nil, fmt.Errorf("no merkle leaf hash for %v", v.Hash) + } + + // Sanity check. Blob should not already exist. + _, err = t.store.Get(keyRecordContent(token, merkle)) + if err == nil { + return nil, fmt.Errorf("unexpected blob found %v %v", v.Hash, merkle) + } + + // Prepare blob + var b []byte + if _, ok := encrypt[v.Hash]; ok { + b, err = blobifyEncrypted(v, t.encryptionKey) + } else { + b, err = blobify(v) + } + if err != nil { + return nil, err + } + blobs[keyRecordContent(token, merkle)] = b + } + + // Blobify the record history. The record token is used as the key + // for record histories in the key-value store. + be, err = convertBlobEntryFromRecordHistory(rh) + if err != nil { + return nil, err + } + b, err = blobify(*be) + if err != nil { + return nil, err + } + blobs[keyRecordHistory(token)] = b + + // Check if the existing file blobs in the store need to be + // converted from encrypted blobs to unencrypted blobs. + if decryptFiles { + log.Debugf("Converting encrypted blobs to unecrypted blobs") + for _, merkle := range ri.Files { + f, err := t.file(token, merkle, stateUnvetted) + if err != nil { + return nil, fmt.Errorf("file %x: %v", merkle, err) + } + be, err := convertBlobEntryFromFile(*f) + if err != nil { + return nil, err + } + b, err := blobify(*be) + if err != nil { + return nil, err + } + blobs[keyRecordContent(token, merkle)] = b + } + } + + // Convert the orphaned merkle root hashes to blob keys + del := make([]string, 0, len(orphaned)) + for _, merkle := range orphaned { + del = append(del, keyRecordContent(token, merkle)) + } + + // Save all the blob changes + err = t.store.Multi(store.Ops{ + Put: blobs, + Del: del, + }) + if err != nil { + return nil, fmt.Errorf("store Multi: %v", err) + } + + // Retrieve and return the updated record + rhp, err := t.recordHistory(token) + if err != nil { + return nil, fmt.Errorf("recordHistory: %v", err) + } + + log.Debugf("Record index:\n%v", rh.String()) + + version := latestVersion(*rhp) + idx = rhp.Versions[version] + r, err := t.record(token, idx, version, rhp.State) + if err != nil { + return nil, fmt.Errorf("record: %v", err) + } + + return r, nil +} + +/// New satisfies the Backend interface. +func (t *tlogbe) New(metadata []backend.MetadataStream, files []backend.File) (*backend.RecordMetadata, error) { + log.Tracef("New") + + // Validate record contents + err := backend.VerifyContent(metadata, files, []string{}) + if err != nil { + return nil, err + } + + // Create a new trillian tree. The treeID is used as the record + // token. + // TODO handle token prefix collisions + tree, _, err := t.treeNew() + if err != nil { + return nil, fmt.Errorf("treeNew: %v", err) + } + token := tokenFromTreeID(tree.TreeId) + + // Save record + rh := recordHistoryNew(token) + rm, err := recordMetadataNew(token, files, backend.MDStatusUnvetted, 1) + if err != nil { + return nil, err + } + _, err = t.recordSave(token, metadata, files, *rm, rh) + if err != nil { + return nil, fmt.Errorf("recordSave %x: %v", token, err) + } + + log.Infof("New record tree:%v token:%x", tree.TreeId, token) + + return rm, nil } -func (t *tlogbe) UpdateUnvettedRecord(tokenb []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { - log.Tracef("UpdateUnvettedRecord: %x", tokenb) +func (t *tlogbe) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { + log.Tracef("UpdateUnvettedRecord: %x", token) - // Send in a single metadata array to verify there are no dups + // Validate record contents. Send in a single metadata array to + // verify there are no dups. allMD := append(mdAppend, mdOverwrite...) err := backend.VerifyContent(allMD, filesAdd, filesDel) if err != nil { @@ -727,149 +1180,198 @@ func (t *tlogbe) UpdateUnvettedRecord(tokenb []byte, mdAppend, mdOverwrite []bac if !ok { return nil, err } - // Allow ErrorStatusEmpty which indicates no new files are - // being added. This can happen when metadata only is being - // updated. + // Allow ErrorStatusEmpty which indicates no new files are being + // added. This can happen when metadata only is being updated or + // when files are being deleted without any files being added. if e.ErrorCode != pd.ErrorStatusEmpty { return nil, err } } - token := hex.EncodeToString(tokenb) - t.Lock() defer t.Unlock() + if t.shutdown { + return nil, backend.ErrShutdown + } - // Ensure record is not vetted + // Ensure unvetted record exists rh, err := t.recordHistory(token) if err != nil { - return nil, fmt.Errorf("recordHistory: %v", err) + if err == store.ErrNotFound { + return nil, backend.ErrRecordNotFound + } + return nil, fmt.Errorf("recordHistory %x: %v", token, err) } if rh.State != stateUnvetted { - // This error doesn't really make sense in the context of tlogbe, - // but its what the gitbe returns in this situation so we return - // it to keep it consistent. For gitbe, it means a record was - // found in a directory that should only contain vetted records. - return nil, backend.ErrRecordFound + return nil, backend.ErrRecordNotFound } - l := len(mdAppend) + len(mdOverwrite) + len(filesAdd) - entries := make([]blobEntry, 0, l) - - // Retrive existing record index + // Get existing record version := latestVersion(*rh) - idx := rh.Versions[version] + if version != 1 { + // Unvetted records should only ever have a single version + return nil, fmt.Errorf("invalid unvetted record version: %v", version) + } + ri := rh.Versions[version] + r, err := t.record(token, ri, version, rh.State) + if err != nil { + return nil, fmt.Errorf("record %x %v: %v", token, version, err) + } - // Overwrite metadata. The recordIndex uses the metadata ID as the - // key so any existing metadata will get overwritten when the new - // recordIndex is created. - for _, v := range mdOverwrite { - be, err := convertBlobEntryFromMetadataStream(v) - if err != nil { - return nil, err - } - entries = append(entries, *be) + // Update the record metadata + f := filesApplyChanges(r.Files, filesAdd, filesDel) + rm, err := recordMetadataNew(token, f, r.RecordMetadata.Status, + r.RecordMetadata.Iteration+1) + if err != nil { + return nil, err } - // Append metadata - for _, v := range mdAppend { - m, ok := idx.Metadata[v.ID] + // Update record + r, err = t.recordUpdate(token, mdAppend, mdOverwrite, + filesAdd, filesDel, *rm, *rh) + if err != nil { + return nil, err + } + + // TODO Call plugin hooks + + return r, nil +} + +func (t *tlogbe) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { + log.Tracef("UpdateVettedRecord: %x", token) + + // Validate record contents. Send in a single metadata array to + // verify there are no dups. + allMD := append(mdAppend, mdOverwrite...) + err := backend.VerifyContent(allMD, filesAdd, filesDel) + if err != nil { + e, ok := err.(backend.ContentVerificationError) if !ok { - return nil, fmt.Errorf("append metadata not found: %v", v.ID) - } - ms, err := t.metadataStream(m, rh.State) - if err != nil { - return nil, fmt.Errorf("metadataStream %v: %v", m, err) - } - buf := bytes.NewBuffer([]byte(ms.Payload)) - buf.WriteString(v.Payload) - ms.Payload = buf.String() - be, err := convertBlobEntryFromMetadataStream(*ms) - if err != nil { return nil, err } - entries = append(entries, *be) - } - - // Add files - for _, v := range filesAdd { - be, err := convertBlobEntryFromFile(v) - if err != nil { + // Allow ErrorStatusEmpty which indicates no new files are being + // being added. This can happen when files are being deleted + // without any new files being added. + if e.ErrorCode != pd.ErrorStatusEmpty { return nil, err } - entries = append(entries, *be) } - // Delete files - for _, fn := range filesDel { - _, ok := idx.Files[fn] - if !ok { - return nil, fmt.Errorf("file to delete not found: %v", fn) - } - delete(idx.Files, fn) + t.Lock() + defer t.Unlock() + if t.shutdown { + return nil, backend.ErrShutdown } - // TODO Ensure changes were actually made - - // Aggregate all files. We need this for the merkle root calc. - files := make([]backend.File, 0, len(idx.Files)+len(filesAdd)) - for _, v := range idx.Files { - f, err := t.file(v, rh.State) - if err != nil { - return nil, fmt.Errorf("file %v: %v", v, err) + // Ensure record is vetted + rh, err := t.recordHistory(token) + if err != nil { + if err == store.ErrNotFound { + return nil, backend.ErrRecordNotFound } - files = append(files, *f) + return nil, fmt.Errorf("recordHistory %x: %v", token, err) } - for _, v := range filesAdd { - files = append(files, v) + if rh.State != stateVetted { + return nil, backend.ErrRecordNotFound } - // Update the record metadata - rm, err := t.recordMetadata(idx.RecordMetadata, rh.State) + // Get existing record + version := latestVersion(*rh) + ri := rh.Versions[version] + r, err := t.record(token, ri, version, rh.State) if err != nil { - return nil, fmt.Errorf("recordMetadata %v: %v", - idx.RecordMetadata, err) + return nil, fmt.Errorf("record %x %v: %v", token, version, err) } - rmNew := recordMetadataNew(token, files, rm.Status, rm.Iteration+1) - be, err := convertBlobEntryFromRecordMetadata(rmNew) + + // Apply changes + files := filesApplyChanges(r.Files, filesAdd, filesDel) + metadata := metadataStreamsApplyChanges(r.Metadata, mdAppend, mdOverwrite) + + // Ensure changes were actually made + m1, err := merkleRoot(r.Files) if err != nil { return nil, err } - entries = append(entries, *be) - - // Append new content to the trillian tree - proofs, _, err := t.blobEntriesAppend(token, entries) + m2, err := merkleRoot(files) if err != nil { return nil, err } + if bytes.Equal(m1[:], m2[:]) { + return nil, backend.ErrNoChanges + } - // Update the record history - idxp, err := recordIndexUpdate(idx, entries, proofs) + // Create an updated record metadata + rm, err := recordMetadataNew(token, files, r.RecordMetadata.Status, + r.RecordMetadata.Iteration+1) if err != nil { return nil, err } - rh, err = t.recordHistoryUpdate(token, rh.State, *idxp) + + // Save a new version of the record + r, err = t.recordSave(token, metadata, files, *rm, *rh) if err != nil { - return nil, err + return nil, fmt.Errorf("recordSave %v: %v", err) } - // Save blobs + // TODO Call plugin hooks - // TODO Delete orphaned blobs + return r, nil +} - // TODO call plugin hooks +func (t *tlogbe) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite []backend.MetadataStream) error { + log.Tracef("UpdateVettedMetadata: %x", token) - return nil, nil -} + // Validate record contents. Send in a single metadata array to + // verify there are no dups. + allMD := append(mdAppend, mdOverwrite...) + err := backend.VerifyContent(allMD, []backend.File{}, []string{}) + if err != nil { + e, ok := err.(backend.ContentVerificationError) + if !ok { + return err + } + // Allow ErrorStatusEmpty which indicates no new files are being + // being added. This is expected. + if e.ErrorCode != pd.ErrorStatusEmpty { + return err + } + } -func (t *tlogbe) UpdateVettedRecord(tokenb []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { - log.Tracef("UpdateVettedRecord: %x", tokenb) + t.Lock() + defer t.Unlock() + if t.shutdown { + return backend.ErrShutdown + } - return nil, nil -} + // Ensure record is vetted + rh, err := t.recordHistory(token) + if err != nil { + if err == store.ErrNotFound { + return backend.ErrRecordNotFound + } + return fmt.Errorf("recordHistory %x: %v", token, err) + } + if rh.State != stateVetted { + return backend.ErrRecordNotFound + } + + // Get current record metadata. This will remain unchanged, but we + // need it for the recordUpdate() call. + version := latestVersion(*rh) + ri := rh.Versions[version] -func (t *tlogbe) UpdateVettedMetadata(tokenb []byte, mdAppend, mdOverwrite []backend.MetadataStream) error { - log.Tracef("UpdateVettedMetadata: %x", tokenb) + rm, err := t.recordMetadata(token, ri.RecordMetadata) + if err != nil { + return fmt.Errorf("recordMetadata %v: %v", ri.RecordMetadata, err) + } + + // Update record + _, err = t.recordUpdate(token, mdAppend, mdOverwrite, + []backend.File{}, []string{}, *rm, *rh) + if err != nil { + return err + } return nil } @@ -878,55 +1380,276 @@ func (t *tlogbe) UpdateReadme(content string) error { return fmt.Errorf("not implemented") } -func (t *tlogbe) UnvettedExists(tokenb []byte) bool { - log.Tracef("UnvettedExists %x", tokenb) +func (t *tlogbe) UnvettedExists(token []byte) bool { + log.Tracef("UnvettedExists %x", token) + + rh, err := t.recordHistory(token) + if err != nil { + if err != store.ErrNotFound { + log.Errorf("UnvettedExists: recordHistory %x: %v", token, err) + } + return false + } + if rh.State == stateUnvetted { + return true + } return false } -func (t *tlogbe) VettedExists(tokenb []byte) bool { - log.Tracef("VettedExists %x", tokenb) +func (t *tlogbe) VettedExists(token []byte) bool { + log.Tracef("VettedExists %x", token) + + rh, err := t.recordHistory(token) + if err != nil { + if err != store.ErrNotFound { + log.Errorf("VettedExists: recordHistory %x: %v", token, err) + } + return false + } + if rh.State == stateVetted { + return true + } return false } -func (t *tlogbe) GetUnvetted(tokenb []byte) (*backend.Record, error) { - log.Tracef("GetUnvetted: %x", tokenb) +func (t *tlogbe) GetUnvetted(token []byte) (*backend.Record, error) { + log.Tracef("GetUnvetted: %x", token) - return nil, nil + rh, err := t.recordHistory(token) + if err != nil { + if err == store.ErrNotFound { + return nil, backend.ErrRecordNotFound + } + return nil, fmt.Errorf("recordHistory %x: %v", token, err) + } + if rh.State != stateUnvetted { + return nil, backend.ErrRecordNotFound + } + version := latestVersion(*rh) + ri := rh.Versions[version] + + r, err := t.record(token, ri, version, rh.State) + if err != nil { + return nil, fmt.Errorf("record %x %v: %v", token, version, err) + } + + return r, nil } -func (t *tlogbe) GetVetted(tokenb []byte, version string) (*backend.Record, error) { - log.Tracef("GetVetted: %x", tokenb) +func (t *tlogbe) GetVetted(token []byte, version string) (*backend.Record, error) { + log.Tracef("GetVetted: %x", token) + + // Ensure record is vetted + rh, err := t.recordHistory(token) + if err != nil { + if err == store.ErrNotFound { + return nil, backend.ErrRecordNotFound + } + return nil, fmt.Errorf("recordHistory %x: %v", token, err) + } + if rh.State != stateVetted { + return nil, backend.ErrRecordNotFound + } - return nil, nil + // Lookup record. If no version was specified, return the latest + // version. + var v uint32 + if version == "" { + v = latestVersion(*rh) + } else { + vr, err := strconv.ParseUint(version, 10, 32) + if err != nil { + return nil, backend.ErrRecordNotFound + } + v = uint32(vr) + } + ri, ok := rh.Versions[v] + if !ok { + return nil, backend.ErrRecordNotFound + } + r, err := t.record(token, ri, uint32(v), rh.State) + if err != nil { + return nil, fmt.Errorf("record %x %v: %v", token, version, err) + } + + return r, nil } -func (t *tlogbe) SetUnvettedStatus(tokenb []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { - log.Tracef("SetUnvettedStatus: %x", tokenb) +func (t *tlogbe) SetUnvettedStatus(token []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { + log.Tracef("SetUnvettedStatus: %x %v (%v)", + token, status, backend.MDStatus[status]) + + t.Lock() + defer t.Unlock() + if t.shutdown { + return nil, backend.ErrShutdown + } + + // Ensure record is unvetted + rh, err := t.recordHistory(token) + if err != nil { + if err == store.ErrNotFound { + return nil, backend.ErrRecordNotFound + } + return nil, fmt.Errorf("recordHistory %x: %v", token, err) + } + if rh.State != stateUnvetted { + return nil, backend.ErrRecordNotFound + } + + // Get the current record metadata + idx := rh.Versions[latestVersion(*rh)] + rm, err := t.recordMetadata(token, idx.RecordMetadata) + if err != nil { + return nil, fmt.Errorf("recordMetadata %v: %v", + idx.RecordMetadata, err) + } + + // Validate status change + if !statusChangeIsAllowed(rm.Status, status) { + return nil, backend.StateTransitionError{ + From: rm.Status, + To: status, + } + } + + log.Debugf("Status change %x from %v (%v) to %v (%v)", token, + backend.MDStatus[rm.Status], rm.Status, backend.MDStatus[status], status) - return nil, nil + // Apply status change + rm.Status = status + rm.Iteration += 1 + rm.Timestamp = time.Now().Unix() + + // Update record + return t.recordUpdate(token, mdAppend, mdOverwrite, + []backend.File{}, []string{}, *rm, *rh) } -func (t *tlogbe) SetVettedStatus(tokenb []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { - log.Tracef("SetVettedStatus: %x", tokenb) +func (t *tlogbe) SetVettedStatus(token []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { + log.Tracef("SetVettedStatus: %x %v (%v)", + token, status, backend.MDStatus[status]) - return nil, nil + t.Lock() + defer t.Unlock() + if t.shutdown { + return nil, backend.ErrShutdown + } + + // Ensure record is vetted + rh, err := t.recordHistory(token) + if err != nil { + if err == store.ErrNotFound { + return nil, backend.ErrRecordNotFound + } + return nil, fmt.Errorf("recordHistory %x: %v", token, err) + } + if rh.State != stateVetted { + return nil, backend.ErrRecordNotFound + } + + // Get the current record metadata + idx := rh.Versions[latestVersion(*rh)] + rm, err := t.recordMetadata(token, idx.RecordMetadata) + if err != nil { + return nil, fmt.Errorf("recordMetadata %v: %v", + idx.RecordMetadata, err) + } + + // Validate status change + if !statusChangeIsAllowed(rm.Status, status) { + return nil, backend.StateTransitionError{ + From: rm.Status, + To: status, + } + } + + log.Debugf("Status change %x from %v (%v) to %v (%v)", token, + backend.MDStatus[rm.Status], rm.Status, backend.MDStatus[status], status) + + // Apply status change + rm.Status = status + rm.Iteration += 1 + rm.Timestamp = time.Now().Unix() + + // Update record + return t.recordUpdate(token, mdAppend, mdOverwrite, + []backend.File{}, []string{}, *rm, *rh) } func (t *tlogbe) Inventory(vettedCount uint, branchCount uint, includeFiles, allVersions bool) ([]backend.Record, []backend.Record, error) { - return nil, nil, nil + log.Tracef("Inventory: %v %v", includeFiles, allVersions) + + // vettedCount specifies the last N vetted records that should be + // returned. branchCount specifies the last N branches, which in + // gitbe correspond to unvetted records. Neither of these are + // implemented in gitbe so they will not be implemented here + // either. They can be added in the future if they are needed. + + // Get all record histories from the store + hists := make([]recordHistory, 0, 1024) + err := t.store.Enum(func(key string, blob []byte) error { + if strings.HasPrefix(key, keyPrefixRecordHistory) { + // This is a record history blob. Decode and save it. + var rh recordHistory + err := json.Unmarshal(blob, &rh) + if err != nil { + return fmt.Errorf("unmarshal recordHistory %v: %v", key, err) + } + hists = append(hists, rh) + } + return nil + }) + if err != nil { + return nil, nil, fmt.Errorf("store Enum: %v", err) + } + + // Retreive the records + unvetted := make([]backend.Record, 0, len(hists)) + vetted := make([]backend.Record, 0, len(hists)) + for _, rh := range hists { + for version, idx := range rh.Versions { + if !allVersions && version != latestVersion(rh) { + continue + } + r, err := t.record(rh.Token, idx, version, rh.State) + if err != nil { + return nil, nil, fmt.Errorf("record %v %v: %v", + rh.Token, version, err) + } + if !includeFiles { + r.Files = []backend.File{} + } + switch rh.State { + case stateUnvetted: + unvetted = append(unvetted, *r) + case stateVetted: + vetted = append(vetted, *r) + default: + return nil, nil, fmt.Errorf("unknown record history state %v: %v", + rh.Token, rh.State) + } + } + } + + return vetted, unvetted, nil } func (t *tlogbe) GetPlugins() ([]backend.Plugin, error) { log.Tracef("GetPlugins") + // TODO implement plugins + return t.plugins, nil } func (t *tlogbe) Plugin(command, payload string) (string, string, error) { log.Tracef("Plugin: %v", command) + // TODO implement plugins + return "", "", nil } @@ -936,41 +1659,84 @@ func (t *tlogbe) Close() { t.Lock() defer t.Unlock() + // Shutdown backend t.shutdown = true - close(t.exit) + + // Close trillian connection + t.grpc.Close() // Zero out encryption key util.Zero(t.encryptionKey[:]) t.encryptionKey = nil } -func tlogbeNew(root, trillianHost, privateKeyFile string) (*tlogbe, error) { - // Create a new signing key - if !util.FileExists(privateKeyFile) { - log.Infof("Generating signing key...") - privateKey, err := keys.NewFromSpec(&keyspb.Specification{ +func New(homeDir, dataDir, trillianHost, trillianKeyFile, dcrtimeHost, encryptionKeyFile string) (*tlogbe, error) { + // Setup trillian key file + if trillianKeyFile == "" { + // No file path was given. Use the default path. + trillianKeyFile = filepath.Join(homeDir, defaultTrillianKeyFilename) + } + if !util.FileExists(trillianKeyFile) { + // Trillian key file does not exist. Create one. + log.Infof("Generating trillian private key") + if trillianKeyFile == "" { + } + key, err := keys.NewFromSpec(&keyspb.Specification{ // TODO Params: &keyspb.Specification_Ed25519Params{}, Params: &keyspb.Specification_EcdsaParams{}, }) if err != nil { return nil, err } - b, err := der.MarshalPrivateKey(privateKey) + b, err := der.MarshalPrivateKey(key) if err != nil { return nil, err } - err = ioutil.WriteFile(privateKeyFile, b, 0400) + err = ioutil.WriteFile(trillianKeyFile, b, 0400) if err != nil { return nil, err } + log.Infof("Trillian private key created: %v", trillianKeyFile) + } - log.Infof("Signing Key created...") + // Setup encryption key file + if encryptionKeyFile == "" { + // No file path was given. Use the default path. + encryptionKeyFile = filepath.Join(homeDir, defaultEncryptionKeyFilename) + } + if !util.FileExists(encryptionKeyFile) { + // Encryption key file does not exist. Create one. + log.Infof("Generating encryption key") + key, err := sbox.NewKey() + if err != nil { + return nil, err + } + err = ioutil.WriteFile(encryptionKeyFile, key[:], 0400) + if err != nil { + return nil, err + } + util.Zero(key[:]) + log.Infof("Encryption key created: %v", encryptionKeyFile) } - // Load signing key - var err error + // Connect to trillian + log.Infof("Trillian log server: %v", trillianHost) + g, err := grpc.Dial(trillianHost, grpc.WithInsecure()) + if err != nil { + return nil, fmt.Errorf("grpc dial: %v", err) + } + + // Setup blob key-value store + blobsPath := filepath.Join(dataDir, blobsDirname) + err = os.MkdirAll(blobsPath, 0700) + if err != nil { + return nil, err + } + store := filesystem.New(blobsPath) + + // Load trillian key pair var privateKey = &keyspb.PrivateKey{} - privateKey.Der, err = ioutil.ReadFile(privateKeyFile) + privateKey.Der, err = ioutil.ReadFile(trillianKeyFile) if err != nil { return nil, err } @@ -978,35 +1744,47 @@ func tlogbeNew(root, trillianHost, privateKeyFile string) (*tlogbe, error) { if err != nil { return nil, err } + log.Infof("Trillian key loaded") - // Connect to trillian - log.Infof("Trillian log server: %v", trillianHost) - g, err := grpc.Dial(trillianHost, grpc.WithInsecure()) + // Load encryption key + f, err := os.Open(encryptionKeyFile) if err != nil { return nil, err } - defer g.Close() - - // Setup blob directory - blobsPath := filepath.Join(root, blobsDirname) - err = os.MkdirAll(blobsPath, 0700) + var encryptionKey [32]byte + n, err := f.Read(encryptionKey[:]) + if n != len(encryptionKey) { + return nil, fmt.Errorf("invalid encryption key length") + } if err != nil { return nil, err } + f.Close() + log.Infof("Encryption key loaded") - // TODO setup encryption key + // Dcrtime host + _, err = url.Parse(dcrtimeHost) + if err != nil { + return nil, fmt.Errorf("parse dcrtime host '%v': %v", dcrtimeHost, err) + } + log.Infof("Anchor host: %v", dcrtimeHost) - // TODO we need a fsck that ensures there are no orphaned blobs in - // the storage layer and that record indexes don't have any missing - // blobs. + // TODO Launch cron + // TODO fsck + // TODO test trillian connection return &tlogbe{ - root: root, - blob: filesystem.BlobFilesystemNew(blobsPath), - client: trillian.NewTrillianLogClient(g), - admin: trillian.NewTrillianAdminClient(g), - ctx: context.Background(), - privateKey: privateKey, - publicKey: signer.Public(), + homeDir: homeDir, + dataDir: dataDir, + encryptionKey: &encryptionKey, + dcrtimeHost: dcrtimeHost, + store: store, + grpc: g, + client: trillian.NewTrillianLogClient(g), + admin: trillian.NewTrillianAdminClient(g), + ctx: context.Background(), + privateKey: privateKey, + publicKey: signer.Public(), + dirty: make(map[int64]uint64), }, nil } diff --git a/politeiad/backend/tlogbe/trillian.go b/politeiad/backend/tlogbe/trillian.go index cc5ad34a5..995b4d9dc 100644 --- a/politeiad/backend/tlogbe/trillian.go +++ b/politeiad/backend/tlogbe/trillian.go @@ -5,6 +5,7 @@ package tlogbe import ( + "bytes" "crypto" "crypto/sha256" "fmt" @@ -21,14 +22,24 @@ import ( "google.golang.org/grpc/status" ) +// leafProof contains a log leaf and the inclusion proof for the log leaf. +type leafProof struct { + Leaf *trillian.LogLeaf + Proof *trillian.Proof +} + +// queuedLeafProof contains the results of a leaf append command, i.e. the +// QueuedLeaf, and the inclusion proof for that leaf. The inclusion proof will +// not be present if the leaf append command failed and the QueuedLeaf will +// contain the error code from the failure. type queuedLeafProof struct { QueuedLeaf *trillian.QueuedLogLeaf Proof *trillian.Proof } -// merkleLeafHash returns the LogLeaf.MerkleLeafHash for the provided -// LogLeaf.LeafData. -func mekleLeafHash(leafValue []byte) []byte { +// merkleLeafHash returns the merkle leaf hash for the provided leaf value. +// This is the same merkle leaf hash that is calculated by trillian. +func merkleLeafHash(leafValue []byte) []byte { h := sha256.New() h.Write([]byte{rfc6962.RFC6962LeafHashPrefix}) h.Write(leafValue) @@ -41,7 +52,95 @@ func logLeafNew(value []byte) *trillian.LogLeaf { } } +func (t *tlogbe) tree(treeID int64) (*trillian.Tree, error) { + log.Tracef("tree: %v", treeID) + + tree, err := t.admin.GetTree(t.ctx, &trillian.GetTreeRequest{ + TreeId: treeID, + }) + if err != nil { + return nil, fmt.Errorf("GetTree: %v", err) + } + if tree.TreeId != treeID { + // Sanity check + return nil, fmt.Errorf("wrong tree returned; got %v, want %v", + tree.TreeId, treeID) + } + + return tree, nil +} + +// treeNew returns a new trillian tree and verifies that the signatures are +// correct. It returns the tree and the signed log root which can be externally +// verified. +func (t *tlogbe) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { + log.Tracef("treeNew") + + pk, err := ptypes.MarshalAny(t.privateKey) + if err != nil { + return nil, nil, err + } + + // Create new trillian tree + tree, err := t.admin.CreateTree(t.ctx, &trillian.CreateTreeRequest{ + Tree: &trillian.Tree{ + TreeState: trillian.TreeState_ACTIVE, + TreeType: trillian.TreeType_LOG, + HashStrategy: trillian.HashStrategy_RFC6962_SHA256, + HashAlgorithm: sigpb.DigitallySigned_SHA256, + SignatureAlgorithm: sigpb.DigitallySigned_ECDSA, + // TODO SignatureAlgorithm: sigpb.DigitallySigned_ED25519, + DisplayName: "", + Description: "", + MaxRootDuration: ptypes.DurationProto(0), + PrivateKey: pk, + }, + }) + if err != nil { + return nil, nil, err + } + + // Init tree or signer goes bananas + ilr, err := t.client.InitLog(t.ctx, &trillian.InitLogRequest{ + LogId: tree.TreeId, + }) + if err != nil { + return nil, nil, err + } + + // Check trillian errors + switch code := status.Code(err); code { + case codes.Unavailable: + err = fmt.Errorf("log server unavailable: %v", err) + case codes.AlreadyExists: + err = fmt.Errorf("just-created Log (%v) is already initialised: %v", + tree.TreeId, err) + case codes.OK: + log.Debugf("Initialised Log: %v", tree.TreeId) + default: + err = fmt.Errorf("failed to InitLog (unknown error)") + } + if err != nil { + return nil, nil, err + } + + // Verify root signature + verifier, err := client.NewLogVerifierFromTree(tree) + if err != nil { + return nil, nil, err + } + _, err = tcrypto.VerifySignedLogRoot(verifier.PubKey, + crypto.SHA256, ilr.Created) + if err != nil { + return nil, nil, err + } + + return tree, ilr.Created, nil +} + func (t *tlogbe) signedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoot, *types.LogRootV1, error) { + log.Tracef("signedLogRoot: %v", tree.TreeId) + // Get latest signed root resp, err := t.client.GetLatestSignedLogRoot(t.ctx, &trillian.GetLatestSignedLogRootRequest{LogId: tree.TreeId}) @@ -63,30 +162,13 @@ func (t *tlogbe) signedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoot, *t return resp.SignedLogRoot, lrv1, nil } -// waitForRootUpdate waits until the trillian root is updated. -func (t *tlogbe) waitForRootUpdate(tree *trillian.Tree, root *trillian.SignedLogRoot) error { - // Wait for update - var logRoot types.LogRootV1 - err := logRoot.UnmarshalBinary(root.LogRoot) - if err != nil { - return err - } - c, err := client.NewFromTree(t.client, tree, logRoot) - if err != nil { - return err - } - _, err = c.WaitForRootUpdate(t.ctx) - if err != nil { - return err - } - return nil -} +func (t *tlogbe) inclusionProof(treeID int64, merkleLeafHash []byte, lrv1 *types.LogRootV1) (*trillian.Proof, error) { + log.Tracef("inclusionProof: %v %x", treeID, merkleLeafHash) -func (t *tlogbe) inclusionProof(treeID int64, leafHash []byte, lrv1 *types.LogRootV1) (*trillian.Proof, error) { resp, err := t.client.GetInclusionProofByHash(t.ctx, &trillian.GetInclusionProofByHashRequest{ LogId: treeID, - LeafHash: leafHash, + LeafHash: merkleLeafHash, TreeSize: int64(lrv1.TreeSize), }) if err != nil { @@ -104,7 +186,7 @@ func (t *tlogbe) inclusionProof(treeID int64, leafHash []byte, lrv1 *types.LogRo return nil, err } verifier := client.NewLogVerifier(lh, t.publicKey, crypto.SHA256) - err = verifier.VerifyInclusionByHash(lrv1, leafHash, proof) + err = verifier.VerifyInclusionByHash(lrv1, merkleLeafHash, proof) if err != nil { return nil, fmt.Errorf("VerifyInclusionByHash: %v", err) } @@ -112,26 +194,42 @@ func (t *tlogbe) inclusionProof(treeID int64, leafHash []byte, lrv1 *types.LogRo return proof, nil } -func (t *tlogbe) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, *trillian.SignedLogRoot, error) { - // Get the tree - tree, err := t.admin.GetTree(t.ctx, &trillian.GetTreeRequest{ - TreeId: treeID, - }) +func (t *tlogbe) leavesByHash(treeID int64, merkleLeafHashes [][]byte) ([]*trillian.LogLeaf, error) { + log.Tracef("leavesByHash: %v %x", treeID, merkleLeafHashes) + + res, err := t.client.GetLeavesByHash(t.ctx, + &trillian.GetLeavesByHashRequest{ + LogId: treeID, + LeafHash: merkleLeafHashes, + }) if err != nil { - return nil, nil, fmt.Errorf("GetTree: %v", err) - } - if tree.TreeId != treeID { - // Sanity check - return nil, nil, fmt.Errorf("invalid treeID; got %v, want %v", - tree.TreeId, treeID) + return nil, fmt.Errorf("GetLeavesByHashRequest: %v", err) } + return res.Leaves, nil +} + +// leavesAppend appends the provided leaves onto the provided tree. The leaf +// and the inclusion proof for the leaf are returned. If a leaf was not +// successfully appended, the leaf will be returned without an inclusion proof. +// The error status code can be found in the returned leaf. Note leaves that +// are duplicates will fail and it is the callers responsibility to determine +// how they should be handled. +func (t *tlogbe) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, *types.LogRootV1, error) { + log.Tracef("leavesAppend: %v", treeID) + // Get the latest signed log root + tree, err := t.tree(treeID) + if err != nil { + return nil, nil, err + } slr, _, err := t.signedLogRoot(tree) if err != nil { return nil, nil, fmt.Errorf("signedLogRoot pre update: %v", err) } + log.Debugf("Appending %v leaves to tree id %v", len(leaves), treeID) + // Append leaves to log qlr, err := t.client.QueueLeaves(t.ctx, &trillian.QueueLeavesRequest{ LogId: treeID, @@ -150,17 +248,25 @@ func (t *tlogbe) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queue } } if len(leaves)-n != 0 { - // Wait for update - log.Debugf("Waiting for update: %v", treeID) - err = t.waitForRootUpdate(tree, slr) + // Wait for root update + log.Debugf("Waiting for root update") + + var logRoot types.LogRootV1 + err := logRoot.UnmarshalBinary(slr.LogRoot) if err != nil { - return nil, nil, fmt.Errorf("waitForRootUpdate: %v", err) + return nil, nil, err + } + c, err := client.NewFromTree(t.client, tree, logRoot) + if err != nil { + return nil, nil, err + } + _, err = c.WaitForRootUpdate(t.ctx) + if err != nil { + return nil, nil, fmt.Errorf("WaitForRootUpdate: %v", err) } } - log.Debugf("Stored/Ignored leaves: %v/%v %v", len(leaves)-n, n, treeID) - - // TODO Mark tree as dirty + log.Debugf("Stored/Ignored leaves: %v/%v", len(leaves)-n, n) // Get the latest signed log root slr, lrv1, err := t.signedLogRoot(tree) @@ -168,17 +274,33 @@ func (t *tlogbe) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queue return nil, nil, fmt.Errorf("signedLogRoot post update: %v", err) } + // The tree is now dirty + t.dirtyAdd(treeID, lrv1.TreeSize) + // Get inclusion proofs proofs := make([]queuedLeafProof, 0, len(qlr.QueuedLeaves)) for _, v := range qlr.QueuedLeaves { qlp := queuedLeafProof{ QueuedLeaf: v, } + // Only retrieve the inclusion proof if the leaf was successfully - // added. A leaf might not have been added if it was a duplicate. - // This is ok. All other errors are not ok. + // appended. Leaves that were not successfully appended will be + // returned without an inclusion proof and the caller can decide + // what to do. Note this includes leaves that were not appended + // becuase they were a duplicate. c := codes.Code(v.GetStatus().GetCode()) - if c == codes.OK || c == codes.AlreadyExists { + if c == codes.OK { + // Validate merkle leaf hash. We compute the merkle leaf hash in + // other parts of tlogbe manually so we need to ensure that the + // returned merkle leaf hashes are what we expect. + m := merkleLeafHash(v.Leaf.LeafValue) + if !bytes.Equal(m, v.Leaf.MerkleLeafHash) { + e := fmt.Sprintf("unknown merkle leaf hash: got %x, want %x", + m, v.Leaf.MerkleLeafHash) + panic(e) + } + // The LeafIndex of a QueuedLogLeaf will not be set yet. Get the // inclusion proof by MerkleLeafHash. qlp.Proof, err = t.inclusionProof(treeID, v.Leaf.MerkleLeafHash, lrv1) @@ -187,74 +309,32 @@ func (t *tlogbe) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queue treeID, v.Leaf.MerkleLeafHash, err) } } + proofs = append(proofs, qlp) } - return proofs, slr, nil + return proofs, lrv1, nil } -// treeNew returns a new trillian tree and verifies that the signatures are -// correct. It returns the tree and the signed log root which can be externally -// verified. -func (t *tlogbe) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { - pk, err := ptypes.MarshalAny(t.privateKey) - if err != nil { - return nil, nil, err - } - - // Create new trillian tree - tree, err := t.admin.CreateTree(t.ctx, &trillian.CreateTreeRequest{ - Tree: &trillian.Tree{ - TreeState: trillian.TreeState_ACTIVE, - TreeType: trillian.TreeType_LOG, - HashStrategy: trillian.HashStrategy_RFC6962_SHA256, - HashAlgorithm: sigpb.DigitallySigned_SHA256, - SignatureAlgorithm: sigpb.DigitallySigned_ECDSA, - // TODO SignatureAlgorithm: sigpb.DigitallySigned_ED25519, - DisplayName: "", - Description: "", - MaxRootDuration: ptypes.DurationProto(0), - PrivateKey: pk, - }, - }) - if err != nil { - return nil, nil, err - } - - // Init tree or signer goes bananas - ilr, err := t.client.InitLog(t.ctx, &trillian.InitLogRequest{ - LogId: tree.TreeId, - }) +func (t *tlogbe) leafProofs(treeID int64, merkleLeafHashes [][]byte, lr *types.LogRootV1) ([]leafProof, error) { + // Retrieve leaves + leaves, err := t.leavesByHash(treeID, merkleLeafHashes) if err != nil { - return nil, nil, err - } - - // Check trillian errors - switch code := status.Code(err); code { - case codes.Unavailable: - err = fmt.Errorf("log server unavailable: %v", err) - case codes.AlreadyExists: - err = fmt.Errorf("just-created Log (%v) is already initialised: %v", - tree.TreeId, err) - case codes.OK: - log.Debugf("Initialised Log: %v", tree.TreeId) - default: - err = fmt.Errorf("failed to InitLog (unknown error)") - } - if err != nil { - return nil, nil, err + return nil, err } - // Verify root signature - verifier, err := client.NewLogVerifierFromTree(tree) - if err != nil { - return nil, nil, err - } - _, err = tcrypto.VerifySignedLogRoot(verifier.PubKey, - crypto.SHA256, ilr.Created) - if err != nil { - return nil, nil, err + // Retrieve proofs + proofs := make([]leafProof, 0, len(leaves)) + for _, v := range leaves { + p, err := t.inclusionProof(treeID, v.MerkleLeafHash, lr) + if err != nil { + return nil, fmt.Errorf("inclusionProof %x: %v", v.MerkleLeafHash, err) + } + proofs = append(proofs, leafProof{ + Leaf: v, + Proof: p, + }) } - return tree, ilr.Created, nil + return proofs, nil } diff --git a/politeiad/cmd/politeia/README.md b/politeiad/cmd/politeia/README.md index 5bf1699a6..f16737d6e 100644 --- a/politeiad/cmd/politeia/README.md +++ b/politeiad/cmd/politeia/README.md @@ -1,6 +1,9 @@ # politeia refclient examples -Obtain politeiad identity: +## Obtain politeiad identity + +The retrieved identity is used to verify replies from politeiad. + ``` $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass identity Key : 8f627e9da14322626d7e81d789f7fcafd25f62235a95377f39cbc7293c4944ad @@ -10,9 +13,16 @@ Save to /home/marco/.politeia/identity.json or ctrl-c to abort Identity saved to: /home/marco/.politeia/identity.json ``` -Add a new record: +## Add a new record + +At least one file must be included. This is the `filepath` argument in the +example below. The provided file must already exist. Arguments are matched +against the regex `^metadata[\d]{1,2}:` to determine if the string is record +metadata. Arguments that are not classified as metadata are assumed to be file +paths. + ``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass new 'metadata12:{"moo":"lala"}' 'metadata2:{"foo":"bar"}' a +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass new 'metadata12:{"moo":"lala"}' 'metadata2:{"foo":"bar"}' filepath 00: 22e88c7d6da9b73fbb515ed6a8f6d133c680527a799e3069ca7ce346d90649b2 a text/plain; charset=utf-8 Record submitted Censorship record: @@ -21,7 +31,8 @@ Record submitted Signature: 28c75019fb15af4e81ee1607deff58a8a82896d6bb1af4e813c5c996069ad7872505e4f25e067e8f310af82981aca1b02050ee23029f6d1e87b8ea8f0b3bcd08 ``` -Get unvetted record: +## Get unvetted record + ``` $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass getunvetted 43c2d4a2c846c188ab0b49012ed17e5f2c16bd6e276cfbb42e30352dffb1743f Unvetted record: @@ -30,7 +41,7 @@ Unvetted record: Censorship record: Merkle : 22e88c7d6da9b73fbb515ed6a8f6d133c680527a799e3069ca7ce346d90649b2 Token : 43c2d4a2c846c188ab0b49012ed17e5f2c16bd6e276cfbb42e30352dffb1743f - Signature: 5c28d2a93ff9cfe35e8a6b465ae06fa596b08bfe7b980ff9dbe68877e7d860010ec3c4fd8c8b739dc4ceeda3a2381899c7741896323856f0f267abf9a40b8003 + Signature: 5c28d2a93ff9cfe35e8a6b465ae06fa596b08bfe7b980ff9dbe68877e7d860010ec3c4fd8c8b739dc4ceeda3a2381899c7741896323856f0f267abf9a40b8003 Metadata : [{2 {"foo":"bar"}} {12 "zap"}] File (00) : Name : a @@ -38,9 +49,21 @@ Unvetted record: Digest : 22e88c7d6da9b73fbb515ed6a8f6d133c680527a799e3069ca7ce346d90649b2 ``` -Update an unvetted record: +## Update an unvetted record + +Files can be updated using the arguments: +- `add:[filepath]` +- `del:[filename]` + +Metadata can be updated using the arguments: +- `'appendmetadata[ID]:[metadataJSON]'` +- `'overwritemetadata[ID]:[metadataJSON]'` + +Metadata provided using the `overwritemetadata` argument does not have to +already exist. The token argument should be prefixed with `token:`. + ``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass updateunvetted 'appendmetadata12:{"foo":"bar"}' 'overwritemetadata2:{"12foo":"12bar"}' del:a add:b token:72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass updateunvetted 'appendmetadata12:{"foo":"bar"}' 'overwritemetadata2:{"12foo":"12bar"}' del:filename add filepath token:72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 Update record: 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 Files add : 00: 12a31b5e662dfa0a572e9fc523eb703f9708de5e2d53aba74f8ebcebbdb706f7 b text/plain; charset=utf-8 Files delete : a @@ -48,21 +71,20 @@ Update record: 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 Metadata append : 12 ``` -Censor a record (and zap metadata stream 12): -``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass setunvettedstatus censor 43c2d4a2c846c188ab0b49012ed17e5f2c16bd6e276cfbb42e30352dffb1743f 'overwritemetadata12:"zap"' -Set record status: - Status : censored -``` +## Set unvetted status + +You can update the status of an unvetted record using `publish` or `censor` +arguments. `publish` makes the record a public, vetted record. `censor` keeps +the record private. Note the token argument is not prefixed in this command. -Publish a record: ``` $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass setunvettedstatus publish 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 Set record status: Status : public ``` -Get vetted record: +## Get vetted record + ``` $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass getvetted 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 Vetted record: @@ -79,7 +101,8 @@ Vetted record: Digest : 12a31b5e662dfa0a572e9fc523eb703f9708de5e2d53aba74f8ebcebbdb706f7 ``` -Update a vetted record: +## Update a vetted record + ``` $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass updatevetted 'appendmetadata12:{"foo":"bar"}' 'overwritemetadata2:{"12foo":"12bar"}' del:a add:b token:72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 Update record: 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 @@ -89,7 +112,18 @@ Update record: 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 Metadata append : 12 ``` -Inventory all records: +## Set vetted status + +Censor a record (and zap metadata stream 12): + +``` +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass setunvettedstatus censor 43c2d4a2c846c188ab0b49012ed17e5f2c16bd6e276cfbb42e30352dffb1743f 'overwritemetadata12:"zap"' +Set record status: + Status : censored +``` + +## Inventory all records + ``` politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass inventory 1 1 Vetted record: diff --git a/politeiad/cmd/politeia/politeia.go b/politeiad/cmd/politeia/politeia.go index cb6cc26dc..019e6c4e4 100644 --- a/politeiad/cmd/politeia/politeia.go +++ b/politeiad/cmd/politeia/politeia.go @@ -430,6 +430,7 @@ func inventory() error { func getFile(filename string) (*v1.File, *[sha256.Size]byte, error) { var err error + filename = util.CleanAndExpandPath(filename) file := &v1.File{ Name: filepath.Base(filename), } @@ -511,10 +512,15 @@ func newRecord() error { n.Files = append(n.Files, *file) hashes = append(hashes, digest) - fmt.Printf("%02v: %v %v %v\n", - i, file.Digest, file.Name, file.MIME) + if !*printJson { + fmt.Printf("%02v: %v %v %v\n", + i, file.Digest, file.Name, file.MIME) + } + } + + if !*printJson { + fmt.Printf("Record submitted\n") } - fmt.Printf("Record submitted\n") // Convert Verify to JSON b, err := json.Marshal(n) @@ -572,8 +578,10 @@ func newRecord() error { copy(signature[:], sig) // Verify merkle root. - if !bytes.Equal(merkle.Root(hashes)[:], root) { - return fmt.Errorf("invalid merkle root") + m := merkle.Root(hashes) + if !bytes.Equal(m[:], root) { + return fmt.Errorf("invalid merkle root; got %x, want %x", + root, m[:]) } // Verify record token signature. diff --git a/politeiad/config.go b/politeiad/config.go index 16b9a8f57..f9f527330 100644 --- a/politeiad/config.go +++ b/politeiad/config.go @@ -41,6 +41,13 @@ const ( defaultMainnetDcrdata = "dcrdata.decred.org:443" defaultTestnetDcrdata = "testnet.decred.org:443" + + // Backend options + backendGit = "git" + backendTlog = "tlog" + defaultBackend = backendGit + + defaultTrillianHost = "localhost:8090" ) var ( @@ -89,6 +96,11 @@ type config struct { Identity string `long:"identity" description:"File containing the politeiad identity file"` GitTrace bool `long:"gittrace" description:"Enable git tracing in logs"` DcrdataHost string `long:"dcrdatahost" description:"Dcrdata ip:port"` + + Backend string `long:"backend"` + TrillianHost string `long:"trillianhost"` + TrillianKey string `long:"trilliankey"` + EncryptionKey string `long:"encryptionkey"` } // serviceOptions defines the configuration options for the daemon as a service @@ -283,14 +295,16 @@ func newConfigParser(cfg *config, so *serviceOptions, options flags.Options) *fl func loadConfig() (*config, []string, error) { // Default config. cfg := config{ - HomeDir: defaultHomeDir, - ConfigFile: defaultConfigFile, - DebugLevel: defaultLogLevel, - DataDir: defaultDataDir, - LogDir: defaultLogDir, - HTTPSKey: defaultHTTPSKeyFile, - HTTPSCert: defaultHTTPSCertFile, - Version: version.String(), + HomeDir: defaultHomeDir, + ConfigFile: defaultConfigFile, + DebugLevel: defaultLogLevel, + DataDir: defaultDataDir, + LogDir: defaultLogDir, + HTTPSKey: defaultHTTPSKeyFile, + HTTPSCert: defaultHTTPSCertFile, + Version: version.String(), + Backend: defaultBackend, + TrillianHost: defaultTrillianHost, } // Service options which are only added on Windows. diff --git a/politeiad/log.go b/politeiad/log.go index 01570160d..c6f0c152f 100644 --- a/politeiad/log.go +++ b/politeiad/log.go @@ -9,6 +9,10 @@ import ( "os" "path/filepath" + "github.com/decred/politeia/politeiad/backend/gitbe" + "github.com/decred/politeia/politeiad/backend/tlogbe" + "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" + "github.com/decred/politeia/politeiad/cache/cockroachdb" "github.com/decred/slog" "github.com/jrick/logrotate/rotator" ) @@ -40,16 +44,28 @@ var ( // application shutdown. logRotator *rotator.Rotator - log = backendLog.Logger("POLI") - gitbeLog = backendLog.Logger("GITB") - cockroachdbLog = backendLog.Logger("CODB") + log = backendLog.Logger("POLI") + gitbeLog = backendLog.Logger("GITB") + tlogbeLog = backendLog.Logger("TLOG") + blobLog = backendLog.Logger("BLOB") + cacheLog = backendLog.Logger("CACH") ) +// Initialize package-global logger variables. +func init() { + cockroachdb.UseLogger(cacheLog) + gitbe.UseLogger(gitbeLog) + tlogbe.UseLogger(tlogbeLog) + filesystem.UseLogger(blobLog) +} + // subsystemLoggers maps each subsystem identifier to its associated logger. var subsystemLoggers = map[string]slog.Logger{ "POLI": log, "GITB": gitbeLog, - "CODB": cockroachdbLog, + "TLOG": tlogbeLog, + "BLOB": blobLog, + "CACH": cacheLog, } // initLogRotator initializes the logging rotater to write logs to logFile and diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index b9e72de8b..45b3b52c2 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -25,6 +25,7 @@ import ( "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/gitbe" + "github.com/decred/politeia/politeiad/backend/tlogbe" "github.com/decred/politeia/politeiad/cache" "github.com/decred/politeia/politeiad/cache/cachestub" "github.com/decred/politeia/politeiad/cache/cockroachdb" @@ -1145,19 +1146,31 @@ func _main() error { } // Setup backend. - gitbe.UseLogger(gitbeLog) - b, err := gitbe.New(activeNetParams.Params, loadedCfg.DataDir, - loadedCfg.DcrtimeHost, "", p.identity, loadedCfg.GitTrace, - loadedCfg.DcrdataHost) - if err != nil { - return err + log.Infof("Backend: %v", loadedCfg.Backend) + switch loadedCfg.Backend { + case backendGit: + b, err := gitbe.New(activeNetParams.Params, loadedCfg.DataDir, + loadedCfg.DcrtimeHost, "", p.identity, loadedCfg.GitTrace, + loadedCfg.DcrdataHost) + if err != nil { + return fmt.Errorf("new gitbe: %v", err) + } + p.backend = b + case backendTlog: + b, err := tlogbe.New(loadedCfg.HomeDir, loadedCfg.DataDir, + loadedCfg.TrillianHost, loadedCfg.TrillianKey, loadedCfg.DcrtimeHost, + loadedCfg.EncryptionKey) + if err != nil { + return fmt.Errorf("new tlogbe: %v", err) + } + p.backend = b + default: + return fmt.Errorf("invalid backend selected: %v", loadedCfg.Backend) } - p.backend = b // Setup cache if p.cfg.EnableCache { // Create a new cache context - cockroachdb.UseLogger(cockroachdbLog) net := filepath.Base(p.cfg.DataDir) db, err := cockroachdb.New(cockroachdb.UserPoliteiad, p.cfg.CacheHost, net, p.cfg.CacheRootCert, p.cfg.CacheCert, p.cfg.CacheKey) diff --git a/util/convert.go b/util/convert.go index d3e2bb563..135739d8a 100644 --- a/util/convert.go +++ b/util/convert.go @@ -10,7 +10,6 @@ import ( "fmt" "github.com/decred/dcrtime/api/v1" - pd "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/identity" ) @@ -33,9 +32,11 @@ func ConvertSignature(s string) ([identity.SignatureSize]byte, error) { // ConvertStringToken verifies and converts a string token to a proper sized // []byte. func ConvertStringToken(token string) ([]byte, error) { - if len(token) != pd.TokenSize*2 { - return nil, fmt.Errorf("invalid censorship token size") - } + /* + if len(token) != pd.TokenSize*2 { + return nil, fmt.Errorf("invalid censorship token size") + } + */ blob, err := hex.DecodeString(token) if err != nil { return nil, err From 66e36889c69aa25fef04a986659dd24a709dc0f0 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 30 Jun 2020 11:40:30 -0600 Subject: [PATCH 004/449] start comments plugin and tlogbe unvetted changes --- go.mod | 2 + go.sum | 3 + plugins/comments/comments.go | 155 +++++++ politeiad/api/v1/v1.go | 5 + politeiad/backend/backend.go | 2 +- politeiad/backend/tlogbe/anchor.go | 402 ++++++++++-------- politeiad/backend/tlogbe/blobentry.go | 36 +- politeiad/backend/tlogbe/dcrtime.go | 168 ++++++++ politeiad/backend/tlogbe/encrypt.go | 50 --- politeiad/backend/tlogbe/encryptionkey.go | 58 +++ politeiad/backend/tlogbe/index.go | 20 +- .../tlogbe/plugin/comments/comments.go | 255 +++++++++++ .../backend/tlogbe/plugin/comments/journal.go | 104 +++++ .../backend/tlogbe/plugin/comments/log.go | 25 ++ politeiad/backend/tlogbe/plugin/plugin.go | 14 + .../tlogbe/store/filesystem/filesystem.go | 23 +- politeiad/backend/tlogbe/store/store.go | 38 +- politeiad/backend/tlogbe/tlog.go | 187 ++++++++ politeiad/backend/tlogbe/tlogbe.go | 280 +++++------- politeiad/backend/tlogbe/trillian.go | 249 +++++++++-- politeiad/cmd/politeia/README.md | 9 +- politeiad/log.go | 8 +- tlog/tserver/tserver.go | 5 +- util/convert.go | 3 +- util/dcrtime.go | 8 +- 25 files changed, 1601 insertions(+), 508 deletions(-) create mode 100644 plugins/comments/comments.go create mode 100644 politeiad/backend/tlogbe/dcrtime.go delete mode 100644 politeiad/backend/tlogbe/encrypt.go create mode 100644 politeiad/backend/tlogbe/encryptionkey.go create mode 100644 politeiad/backend/tlogbe/plugin/comments/comments.go create mode 100644 politeiad/backend/tlogbe/plugin/comments/journal.go create mode 100644 politeiad/backend/tlogbe/plugin/comments/log.go create mode 100644 politeiad/backend/tlogbe/plugin/plugin.go create mode 100644 politeiad/backend/tlogbe/tlog.go diff --git a/go.mod b/go.mod index 2b679172d..e855760c4 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/decred/dcrdata/pubsub/v4 v4.0.3-0.20191219212733-19f656d6d679 github.com/decred/dcrdata/semver v1.0.0 github.com/decred/dcrtime v0.0.0-20191018193024-8d8b4ef0458e + github.com/decred/dcrtime/api/v2 v2.0.0-20200618212201-c181ffffd11c github.com/decred/dcrwallet v1.2.3-0.20190128160919-849f7c01c12d github.com/decred/dcrwallet/rpc/walletrpc v0.2.0 github.com/decred/go-socks v1.1.0 @@ -73,6 +74,7 @@ require ( golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271 golang.org/x/sync v0.0.0-20190423024810-112230192c58 + google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb google.golang.org/grpc v1.24.0 sigs.k8s.io/yaml v1.1.0 // indirect ) diff --git a/go.sum b/go.sum index f7a19d84c..9357c5031 100644 --- a/go.sum +++ b/go.sum @@ -233,6 +233,9 @@ github.com/decred/dcrdata/txhelpers/v4 v4.0.1 h1:jNPPSP5HzE4cfddj5zIJhrIEus/Tvd2 github.com/decred/dcrdata/txhelpers/v4 v4.0.1/go.mod h1:cUJbgsIzzI42llHDS0nkPlG49vPJ0cW6IZGbfu5sFrA= github.com/decred/dcrtime v0.0.0-20191018193024-8d8b4ef0458e h1:sNDR7vx6gaA3WD+WoEofTvtdjfwHAiogtjB3kt8iFco= github.com/decred/dcrtime v0.0.0-20191018193024-8d8b4ef0458e/go.mod h1:IyZnyBE3E6RBFsEjwEs21FrO/UsrLrL15hUnpZZQxpU= +github.com/decred/dcrtime v0.0.0-20200618212201-c181ffffd11c h1:YfctVlvdhxKbDZIsEkWv1I2I9J3jDlPgnSX66SW9xhg= +github.com/decred/dcrtime/api/v2 v2.0.0-20200618212201-c181ffffd11c h1:EVzHNtdbWw+MdmJk1DOuH8jBQpx5oSUzKDd8RjvOFGc= +github.com/decred/dcrtime/api/v2 v2.0.0-20200618212201-c181ffffd11c/go.mod h1:JdIX208vnNj4TdU6hDRaN+ccxmxp1I1R6sWGZNK1BAQ= github.com/decred/dcrwallet v1.2.2/go.mod h1:BrSus0F+Rx8UhvPNBfuRMIjRJBNrW2sLspN9iQR5hm8= github.com/decred/dcrwallet v1.2.3-0.20190128160919-849f7c01c12d h1:29CvbLfATHA5XDxr6w4Z3hYx/tqM+tgnj63LJhUVCCU= github.com/decred/dcrwallet v1.2.3-0.20190128160919-849f7c01c12d/go.mod h1:jtNUWAO6lg2t2UHOlT9fmjuxD8os9JMljZNEXaw95Tg= diff --git a/plugins/comments/comments.go b/plugins/comments/comments.go new file mode 100644 index 000000000..e12c441aa --- /dev/null +++ b/plugins/comments/comments.go @@ -0,0 +1,155 @@ +package comments + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "strconv" + + "github.com/decred/politeia/politeiad/api/v1/identity" + "github.com/decred/politeia/util" +) + +type ErrorStatusT int + +const ( + Version = "1" + ID = "comments" + + // Plugin commands + CmdNew = "new" // Create a new comment + CmdUnvetted = "unvetted" // Get unvetted comments + CmdVetted = "vetted" // Get vetted comments + CmdGet = "get" // Get a comment + CmdEdit = "edit" // Edit a comment + CmdDel = "del" // Delete a comment + CmdVote = "vote" // Vote on a comment + CmdCensor = "censor" // Censor a comment + CmdCount = "count" // Get comments count + CmdProofs = "proofs" // Get comment proofs + + // Error status codes + ErrorStatusInvalid ErrorStatusT = 0 + ErrorStatusPublicKeyInvalid ErrorStatusT = 1 + ErrorStatusSignatureInvalid ErrorStatusT = 2 + ErrorStatusRecordNotFound ErrorStatusT = 3 + ErrorStatusCommentNotFound ErrorStatusT = 4 +) + +var ( + // Human readable error messages + ErrorStatus = map[ErrorStatusT]string{ + ErrorStatusPublicKeyInvalid: "invalid public key", + ErrorStatusSignatureInvalid: "invalid signature", + ErrorStatusRecordNotFound: "record not found", + ErrorStatusCommentNotFound: "comment not found", + } +) + +// PluginError is emitted when input provided to a plugin command is invalid. +type PluginError struct { + ErrorCode ErrorStatusT + ErrorContext []string +} + +// Error satisfies the error interface. +func (e PluginError) Error() string { + return fmt.Sprintf("plugin error code: %v", e.ErrorCode) +} + +// Comment represent a user submitted comment and includes both the user +// generated comment data and the server generated metadata. +type Comment struct { + // Data generated by client + Token string `json:"token"` // Record token + ParentID uint32 `json:"parentid"` // Parent comment ID + Comment string `json:"comment"` // Comment + PublicKey string `json:"publickey"` // Pubkey used for Signature + Signature string `json:"signature"` // Signature of Token+ParentID+Comment + + // Metadata generated by server + CommentID uint32 `json:"commentid"` // Comment ID + Version uint32 `json:"version"` // Comment version + Receipt string `json:"receipt"` // Server signature of client signature + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + Score int32 `json:"score"` // Vote score + Deleted bool `json:"deleted"` // Comment has been deleted by author + Censored bool `json:"censored"` // Comment has been censored by admin +} + +func VerifyCommentSignature(signature, pubkey, token string, parentID uint32, comment string) error { + sig, err := util.ConvertSignature(signature) + if err != nil { + return PluginError{ + ErrorCode: ErrorStatusSignatureInvalid, + ErrorContext: []string{err.Error()}, + } + } + b, err := hex.DecodeString(pubkey) + if err != nil { + return PluginError{ + ErrorCode: ErrorStatusPublicKeyInvalid, + ErrorContext: []string{"key is not hex"}, + } + } + pk, err := identity.PublicIdentityFromBytes(b) + if err != nil { + return PluginError{ + ErrorCode: ErrorStatusPublicKeyInvalid, + ErrorContext: []string{err.Error()}, + } + } + msg := token + strconv.FormatUint(uint64(parentID), 10) + comment + if !pk.VerifyMessage([]byte(msg), sig) { + return PluginError{ + ErrorCode: ErrorStatusSignatureInvalid, + ErrorContext: []string{err.Error()}, + } + } + return nil +} + +type New struct { + Token string `json:"token"` // Record token + ParentID uint32 `json:"parentid"` // Parent comment ID + Comment string `json:"comment"` // Comment + PublicKey string `json:"publickey"` // Pubkey used for Signature + Signature string `json:"signature"` // Signature of Token+ParentID+Comment +} + +// EncodeNew encodes a New into a JSON byte slice. +func EncodeNew(n New) ([]byte, error) { + return json.Marshal(n) +} + +// DecodeNew decodes a JSON byte slice into a New. +func DecodeNew(b []byte) (*New, error) { + var n New + err := json.Unmarshal(b, &n) + if err != nil { + return nil, err + } + return &n, nil +} + +// NewReply is the reply to the New command. +type NewReply struct { + CommentID string `json:"commentid"` // Comment ID + Receipt string `json:"receipt"` // Server signature of comment signature + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp +} + +// EncodeNew encodes a NewReply into a JSON byte slice. +func EncodeNewReply(r NewReply) ([]byte, error) { + return json.Marshal(r) +} + +// DecodeNew decodes a JSON byte slice into a NewReply. +func DecodeNewReply(b []byte) (*NewReply, error) { + var r NewReply + err := json.Unmarshal(b, &r) + if err != nil { + return nil, err + } + return &r, nil +} diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index 34f4e1548..3517485e7 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -119,6 +119,11 @@ var ( ErrInvalidBase64 = errors.New("corrupt base64") ErrInvalidMerkle = errors.New("merkle roots do not match") ErrCorrupt = errors.New("signature verification failed") + + // Length of prefix of token used for lookups. The length 7 was selected to + // match github's abbreviated hash length. This is a var so that it can be + // updated during testing. + TokenPrefixLength = 7 ) // Verify ensures that a CensorshipRecord properly describes the array of diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index f043ac1c2..752804032 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -9,7 +9,7 @@ import ( "fmt" "regexp" - "github.com/decred/politeia/politeiad/api/v1" + v1 "github.com/decred/politeia/politeiad/api/v1" ) var ( diff --git a/politeiad/backend/tlogbe/anchor.go b/politeiad/backend/tlogbe/anchor.go index 0077374e5..2c7d275c1 100644 --- a/politeiad/backend/tlogbe/anchor.go +++ b/politeiad/backend/tlogbe/anchor.go @@ -7,22 +7,24 @@ package tlogbe import ( "crypto/sha256" "encoding/hex" + "encoding/json" "fmt" "time" - "github.com/davecgh/go-spew/spew" - dcrtime "github.com/decred/dcrtime/api/v1" + dcrtime "github.com/decred/dcrtime/api/v2" "github.com/decred/politeia/util" "github.com/google/trillian/types" ) -// TODO handle reorgs. A anchor record may become invalid in the case -// of a reorg. +// TODO handle reorgs. A anchor record may become invalid in the case of a +// reorg. We don't create the anchor record until the anchor tx has 6 +// confirmations so the probability of this occuring on mainnet is very low, +// but it should still be handled. const ( // anchorSchedule determines how often we anchor records. dcrtime - // drops an anchor on the hour mark so we submit new anchors a few - // minutes prior. + // currently drops an anchor on the hour mark so we submit new + // anchors a few minutes prior to that. // Seconds Minutes Hours Days Months DayOfWeek anchorSchedule = "0 56 * * * *" // At 56 minutes every hour @@ -32,22 +34,36 @@ const ( // anchor represents an anchor, i.e. timestamp, of a trillian tree at a // specific tree size. The LogRoot is hashed and anchored using dcrtime. Once -// dcrtime drops an anchor, the anchor structure is updated and saved to the -// key-value store. +// dcrtime timestamp is verified the anchor structure is updated and saved to +// the key-value store. type anchor struct { TreeID int64 `json:"treeid"` LogRoot *types.LogRootV1 `json:"logroot"` VerifyDigest *dcrtime.VerifyDigest `json:"verifydigest"` } -// anchorSave saves the anchor to the key-value store and updates the record -// history of the record that corresponds to the anchor TreeID. This function -// should be called once the dcrtime anchor has been dropped. +func convertBlobEntryFromAnchor(a anchor) (*blobEntry, error) { + data, err := json.Marshal(a) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + dataDescriptor{ + Type: dataTypeStructure, + Descriptor: dataDescriptorAnchor, + }) + if err != nil { + return nil, err + } + be := blobEntryNew(hint, data) + return &be, nil +} + +// anchorSave saves an anchor to the key-value store and updates the record +// history of the record that corresponds to tree that was anchored. // // This function must be called WITHOUT the read/write lock held. func (t *tlogbe) anchorSave(a anchor) error { - log.Debugf("Saving anchor for tree %v at height %v", - a.TreeID, a.LogRoot.TreeSize) // Sanity checks switch { @@ -59,18 +75,30 @@ func (t *tlogbe) anchorSave(a anchor) error { return fmt.Errorf("verify digest not found") } - // Compute the log root hash. This will be used as the key for the - // anchor in the key-value store. - b, err := a.LogRoot.MarshalBinary() + // Save the anchor record + be, err := convertBlobEntryFromAnchor(a) + if err != nil { + return err + } + b, err := blobify(*be) + if err != nil { + return err + } + lrb, err := a.LogRoot.MarshalBinary() + if err != nil { + return err + } + logRootHash := util.Hash(lrb)[:] + err = t.store.Put(keyAnchor(logRootHash), b) if err != nil { - return fmt.Errorf("MarshalBinary %v %x: %v", - a.TreeID, a.LogRoot.RootHash, err) + return fmt.Errorf("Put: %v", err) } - logRootHash := util.Hash(b) - // Get the record history for this tree and update the appropriate - // record content with the anchor. The lock must be held during - // this update. + log.Debugf("Anchor saved for tree %v at height %v", + a.TreeID, a.LogRoot.TreeSize) + + // Update the record history with the anchor. The lock must be held + // during this update. t.Lock() defer t.Unlock() @@ -80,73 +108,9 @@ func (t *tlogbe) anchorSave(a anchor) error { return fmt.Errorf("recordHistory: %v", err) } - // Aggregate all record content that does not currently have an - // anchor. - noAnchor := make([][]byte, 0, 256) - for _, v := range rh.Versions { - _, ok := rh.Anchors[hex.EncodeToString(v.RecordMetadata)] - if !ok { - noAnchor = append(noAnchor, v.RecordMetadata) - } - for _, merkle := range v.Metadata { - _, ok := rh.Anchors[hex.EncodeToString(merkle)] - if !ok { - noAnchor = append(noAnchor, merkle) - } - } - for _, merkle := range v.Files { - _, ok := rh.Anchors[hex.EncodeToString(merkle)] - if !ok { - noAnchor = append(noAnchor, merkle) - } - } - } - if len(noAnchor) == 0 { - // All record content has already been anchored. This should not - // happen. A tree is only anchored when it has at least one - // unanchored leaf. - return fmt.Errorf("all record content is already anchored") - } - - // Get the leaves for the record content that has not been anchored - // yet. We'll use these to check if the leaf was included in the - // current anchor. - leaves, err := t.leavesByHash(a.TreeID, noAnchor) - if err != nil { - return fmt.Errorf("leavesByHash: %v", err) - } - - var anchorCount int - for _, v := range leaves { - // Check leaf height - if int64(a.LogRoot.TreeSize) < v.LeafIndex { - // Leaf was not included in anchor - continue - } - - // Leaf was included in the anchor - anchorCount++ - - // Sanity check. Get the inclusion proof. This function will - // throw an error if the leaf is not part of the log root. - _, err = t.inclusionProof(a.TreeID, v.MerkleLeafHash, a.LogRoot) - if err != nil { - return fmt.Errorf("inclusionProof %x: %v", v.MerkleLeafHash, err) - } - - // Update record history - rh.Anchors[hex.EncodeToString(v.MerkleLeafHash)] = logRootHash[:] - - log.Debugf("Anchor added to leaf: %x", v.MerkleLeafHash) - } - if anchorCount == 0 { - // This should not happen. If a tree was anchored then it should - // have at least one leaf that was included in the anchor. - return fmt.Errorf("no record content was included in the anchor") - } + rh.Anchors[a.LogRoot.TreeSize] = logRootHash - // Save the updated record history - be, err := convertBlobEntryFromRecordHistory(*rh) + be, err = convertBlobEntryFromRecordHistory(*rh) if err != nil { return err } @@ -156,33 +120,21 @@ func (t *tlogbe) anchorSave(a anchor) error { } err = t.store.Put(keyRecordHistory(rh.Token), b) if err != nil { - return fmt.Errorf("store Put: %v", err) + return fmt.Errorf("Put: %v", err) } - log.Debugf("Record history updated") - - // Mark tree as clean if no additional leaves have been added while - // we've been waiting for the anchor to drop. - height, ok := t.dirtyHeight(a.TreeID) - if !ok { - return fmt.Errorf("dirty tree height not found") - } - - log.Debugf("Tree anchored at height %v, current height %v", - a.LogRoot.TreeSize, height) - - if height == a.LogRoot.TreeSize { - // All tree leaves have been anchored. Remove tree from dirty - // list. - t.dirtyDel(a.TreeID) - log.Debugf("Tree removed from dirty list") - } + log.Debugf("Anchor added to record history %x", token) return nil } -// anchorWait waits for dcrtime to drop an anchor for the provided hashes. -func (t *tlogbe) anchorWait(anchors []anchor, hashes []string) { +// waitForAnchor waits for the anchor to drop. The anchor is not considered +// dropped until dcrtime returns the ChainTimestamp in the reply. dcrtime does +// not return the ChainTimestamp until the timestamp transaction has 6 +// confirmations. Once the timestamp has been dropped, the anchor record is +// saved to the key-value store and the record histories of the corresponding +// timestamped trees are updated. +func (t *tlogbe) waitForAnchor(anchors []anchor, hashes []string) { // Ensure we are not reentrant t.Lock() if t.droppingAnchor { @@ -192,160 +144,244 @@ func (t *tlogbe) anchorWait(anchors []anchor, hashes []string) { t.droppingAnchor = true t.Unlock() - // Whatever happens in this function we must clear droppingAnchor. + // Whatever happens in this function we must clear droppingAnchor + var exitErr error defer func() { t.Lock() t.droppingAnchor = false t.Unlock() + + if exitErr != nil { + log.Errorf("waitForAnchor: %v", exitErr) + } }() // Wait for anchor to drop log.Infof("Waiting for anchor to drop") // Continually check with dcrtime if the anchor has been dropped. + // The anchor is not considered dropped until the ChainTimestamp + // field of the dcrtime reply has been populated. dcrtime only + // populates the ChainTimestamp field once the dcr transaction has + // 6 confirmations. var ( - period = time.Duration(1) * time.Minute // check every 1 minute - retries = 30 / int(period) // for up to 30 minutes + // The max retry period is set to 180 minutes to ensure that + // enough time is given for the anchor transaction to recieve 6 + // confirmations. This is based on the fact that each block has + // a 99.75% chance of being mined within 30 minutes. + // + // TODO change period to 5 minutes when done testing + period = 1 * time.Minute // check every 5 minute + retries = 180 / int(period.Minutes()) // for up to 180 minutes ticker = time.NewTicker(period) ) defer ticker.Stop() for try := 0; try < retries; try++ { - restart: <-ticker.C log.Debugf("Verify anchor attempt %v/%v", try+1, retries) - vr, err := util.Verify(anchorID, t.dcrtimeHost, hashes) + vbr, err := verifyBatch(t.dcrtimeHost, anchorID, hashes) if err != nil { - if _, ok := err.(util.ErrNotAnchored); ok { - // Anchor not dropped, try again - continue - } - log.Errorf("anchorWait exiting: %v", err) + exitErr = fmt.Errorf("verifyBatch: %v", err) return } - // Make sure we are actually anchored. - for _, v := range vr.Digests { + // Make sure we're actually anchored + var retry bool + for _, v := range vbr.Digests { + if v.Result != dcrtime.ResultOK { + // Something is wrong. Log the error and retry. + log.Errorf("Digest %v: %v (%v)", + v.Digest, dcrtime.Result[v.Result], v.Result) + retry = true + break + } + // Transaction will be populated once the tx has been sent, + // otherwise is will be a zeroed out SHA256 digest. + b := make([]byte, sha256.Size) + if v.ChainInformation.Transaction == hex.EncodeToString(b) { + log.Debugf("Anchor tx not sent yet; retry in %v", period) + retry = true + break + } + // ChainTimestamp will be populated once the tx has 6 + // confirmations. if v.ChainInformation.ChainTimestamp == 0 { - log.Debugf("anchorRecords ChainTimestamp 0: %v", v.Digest) - goto restart + log.Debugf("Anchor tx %v not enough confirmations; retry in %v", + v.ChainInformation.Transaction, period) + retry = true + break } } - - log.Debugf("%T %v", vr, spew.Sdump(vr)) - - log.Infof("Anchor dropped") + if retry { + continue + } // Save anchor records for k, v := range anchors { - // Sanity check - verifyDigest := vr.Digests[k] + // Sanity checks. Anchor log root digest should match digest + // that was anchored. b, err := v.LogRoot.MarshalBinary() if err != nil { - log.Errorf("anchorWait: MarshalBinary %v %x: %v", + log.Errorf("waitForAnchor: MarshalBinary %v %x: %v", v.TreeID, v.LogRoot.RootHash, err) continue } - h := util.Hash(b) - if hex.EncodeToString(h[:]) != verifyDigest.Digest { - log.Errorf("anchorWait: digest mismatch: got %x, want %v", - h[:], verifyDigest.Digest) + anchorDigest := hex.EncodeToString(util.Hash(b)[:]) + dcrtimeDigest := vbr.Digests[k].Digest + if anchorDigest != dcrtimeDigest { + log.Errorf("waitForAnchor: digest mismatch: got %x, want %v", + dcrtimeDigest, anchorDigest) continue } - err = t.anchorSave(anchor{ - TreeID: v.TreeID, - LogRoot: v.LogRoot, - VerifyDigest: v.VerifyDigest, - }) + // Add VerifyDigest to anchor before saving it + v.VerifyDigest = &vbr.Digests[k] + + // Save anchor + err = t.anchorSave(v) if err != nil { - log.Errorf("anchorWait: anchorSave %v: %v", v.TreeID, err) + log.Errorf("waitForAnchor: anchorSave %v: %v", v.TreeID, err) continue } } - log.Info("Anchored records updated") + log.Infof("Anchor dropped for %v records", len(vbr.Digests)) return } - log.Errorf("Anchor drop timeout, waited for: %v", period*time.Minute) + log.Errorf("Anchor drop timeout, waited for: %v", + int(period.Minutes())*retries) } -// anchor is a function that is periodically called to anchor dirty trees. -func (t *tlogbe) anchor() { - log.Debugf("Start anchoring process") +// anchor drops an anchor for any trees that have unanchored leaves at the +// time of function invocation. A digest of the tree's log root at its current +// height is timestamped onto the decred blockchain using the dcrtime service. +// The anchor data is saved to the key-value store and the record history that +// corresponds to the anchored tree is updated with the anchor data. +func (t *tlogbe) anchorTrees() { + log.Debugf("Start anchor process") - var exitError error // Set on exit if there is an error + var exitErr error // Set on exit if there is an error defer func() { - if exitError != nil { - log.Errorf("anchorRecords: %v", exitError) + if exitErr != nil { + log.Errorf("anchorTrees: %v", exitErr) } }() - // Get dirty trees - dirty := t.dirtyCopy() - anchors := make([]anchor, 0, len(dirty)) - for treeID := range dirty { - anchors = append(anchors, anchor{ - TreeID: treeID, - }) - } - if len(anchors) == 0 { - log.Infof("Nothing to anchor") + trees, err := t.tlog.treesAll() + if err != nil { + exitErr = fmt.Errorf("treesAll: %v", err) return } - // Aggregate the log root for each tree. A hash of the log root is - // what we anchor. - hashes := make([]*[sha256.Size]byte, len(anchors)) - for k, v := range anchors { - log.Debugf("Obtaining anchoring data: %v", v.TreeID) - - tree, err := t.tree(v.TreeID) + // digests contains the SHA256 digests of the log roots of the + // trees that need to be anchored. These will be submitted to + // dcrtime to be included in a dcrtime timestamp. + digests := make([]string, 0, len(trees)) + + // anchors contains an anchor structure for each tree that is being + // anchored. Once the dcrtime timestamp is successful, these + // anchors will be updated with the timestamp data and saved to the + // key-value store. + anchors := make([]anchor, 0, len(trees)) + + // Find the trees that need to be anchored + for _, v := range trees { + // Check if this tree has unanchored leaves + _, lr, err := t.tlog.signedLogRoot(v) if err != nil { - exitError = fmt.Errorf("tree %v: %v", v.TreeID, err) + exitErr = fmt.Errorf("signedLogRoot %v: %v", v.TreeId, err) return } - _, lr, err := t.signedLogRoot(tree) + token := tokenFromTreeID(v.TreeId) + rh, err := t.recordHistory(token) if err != nil { - exitError = fmt.Errorf("signedLogRoot %v: %v", v.TreeID, err) - return + exitErr = fmt.Errorf("recordHistory %x: %v", token, err) } + _, ok := rh.Anchors[lr.TreeSize] + if ok { + // Tree has already been anchored at the current height. Check + // the next one. + continue + } + + // Tree has not been anchored at current height. Anchor it. + log.Debugf("Tree %v (%x) anchoring at height %v", + v.TreeId, token, lr.TreeSize) - anchors[k].LogRoot = lr + // Setup anchor record + anchors = append(anchors, anchor{ + TreeID: v.TreeId, + LogRoot: lr, + }) + + // Collate the log root digest lrb, err := lr.MarshalBinary() if err != nil { - exitError = fmt.Errorf("MarshalBinary %v: %v", v.TreeID, err) + exitErr = fmt.Errorf("MarshalBinary %v: %v", v.TreeId, err) return } - hashes[k] = util.Hash(lrb) + d := hex.EncodeToString(util.Hash(lrb)[:]) + digests = append(digests, d) + } + if len(anchors) == 0 { + log.Infof("Nothing to anchor") + return } // Ensure we are not reentrant t.Lock() if t.droppingAnchor { - // This shouldn't happen so let's warn the user of something - // misbehaving. + // An anchor is not considered dropped until dcrtime returns the + // ChainTimestamp in the VerifyReply. dcrtime does not do this + // until the anchor tx has 6 confirmations, therefor, this code + // path can be hit if 6 blocks are not mined within the period + // specified by the anchor schedule. Though rare, the probability + // of this happening is not zero and should not be considered an + // error. We simply exit and will drop a new anchor at the next + // anchor period. t.Unlock() - log.Errorf("Dropping anchor already in progress") + log.Infof("Attempting to drop an anchor while previous anchor " + + "has not finished dropping; skipping current anchor period") return } t.Unlock() // Submit dcrtime anchor request - log.Infof("Anchoring records: %v", len(anchors)) + log.Infof("Anchoring %v trees", len(anchors)) - err := util.Timestamp(anchorID, t.dcrtimeHost, hashes) + tbr, err := timestampBatch(t.dcrtimeHost, anchorID, digests) if err != nil { - exitError = err + exitErr = fmt.Errorf("timestampBatch: %v", err) return } - - h := make([]string, 0, len(hashes)) - for _, v := range hashes { - h = append(h, hex.EncodeToString(v[:])) + var failed bool + for i, v := range tbr.Results { + switch v { + case dcrtime.ResultOK: + // We're good; continue + case dcrtime.ResultExistsError: + // I can't think of any situations where this would happen, but + // it's ok if it does since we'll still be able to retrieve the + // VerifyDigest from dcrtime for this digest. + // + // Log this as a warning to bring it to our attention. Do not + // exit. + log.Warnf("Digest failed %v: %v (%v)", + tbr.Digests[i], dcrtime.Result[v], v) + default: + // Something went wrong; exit + log.Errorf("Digest failed %v: %v (%v)", + tbr.Digests[i], dcrtime.Result[v], v) + failed = true + } + } + if failed { + exitErr = fmt.Errorf("dcrtime failed to timestamp digests") + return } - go t.anchorWait(anchors, h) + go t.waitForAnchor(anchors, digests) } diff --git a/politeiad/backend/tlogbe/blobentry.go b/politeiad/backend/tlogbe/blobentry.go index 1320dfaeb..bdb4f25c0 100644 --- a/politeiad/backend/tlogbe/blobentry.go +++ b/politeiad/backend/tlogbe/blobentry.go @@ -15,6 +15,8 @@ import ( ) const ( + blobEntryVersion uint32 = 1 + // Data descriptor types. These may be freely edited since they are // solely hints to the application. dataTypeStructure = "struct" // Descriptor contains a structure @@ -24,8 +26,11 @@ const ( dataDescriptorRecordMetadata = "recordmetadata" dataDescriptorMetadataStream = "metadatastream" dataDescriptorRecordHistory = "recordhistory" + dataDescriptorAnchor = "anchor" + + dataDescriptorRecordIndex = "recordindex" - // Blob entry key prefixes for the key-value store + // Key prefixes for blob entries saved to the key-value store keyPrefixRecordHistory = "index" keyPrefixRecordContent = "record" keyPrefixAnchor = "anchor" @@ -34,15 +39,16 @@ const ( // dataDescriptor provides hints about a data blob. In practise we JSON encode // this struture and stuff it into blobEntry.DataHint. type dataDescriptor struct { - Type string `json:"type"` // Type of data that is stored + Type string `json:"type"` // Type of data Descriptor string `json:"descriptor"` // Description of the data - ExtraData string `json:"extradata,omitempty"` // Value to be freely used by caller + ExtraData string `json:"extradata,omitempty"` // Value to be freely used } // blobEntry is the structure used to store data in the Blob key-value store. +// All data in the Blob key-value store will be encoded as a blobEntry. type blobEntry struct { - Hash string `json:"hash"` // SHA256 hash of the data payload, hex encoded - DataHint string `json:"datahint"` // Hint that describes the data, base64 encoded + Hash string `json:"hash"` // SHA256 hash of data payload, hex encoded + DataHint string `json:"datahint"` // Hint that describes data, base64 encoded Data string `json:"data"` // Data payload, base64 encoded } @@ -52,11 +58,11 @@ func keyRecordHistory(token []byte) string { return keyPrefixRecordHistory + hex.EncodeToString(token) } -// keyRecordContent returns the key for the blob key-value store for any type -// of record content (files, metadata streams, record metadata). Its possible -// for two different records to submit the same file resulting in identical -// merkle leaf hashes. The token is included in the key to ensure that a -// situation like this does not lead to unwanted behavior. +// keyRecordContent returns the key-value store key for any type of record +// content (files, metadata streams, record metadata). Its possible for two +// different records to submit the same file resulting in identical merkle leaf +// hashes. The token is included in the key to ensure that a situation like +// this does not lead to unwanted behavior. func keyRecordContent(token, merkleLeafHash []byte) string { return keyPrefixRecordContent + hex.EncodeToString(token) + hex.EncodeToString(merkleLeafHash) @@ -64,7 +70,7 @@ func keyRecordContent(token, merkleLeafHash []byte) string { // keyAnchor returns the key for the blob key-value store for a anchor record. func keyAnchor(logRootHash []byte) string { - return keyPrefixRecordHistory + hex.EncodeToString(logRootHash) + return keyPrefixAnchor + hex.EncodeToString(logRootHash) } func blobify(be blobEntry) ([]byte, error) { @@ -96,7 +102,7 @@ func deblob(blob []byte) (*blobEntry, error) { return &be, nil } -func blobifyEncrypted(be blobEntry, key *[32]byte) ([]byte, error) { +func blobifyEncrypted(be blobEntry, key *EncryptionKey) ([]byte, error) { var b bytes.Buffer zw := gzip.NewWriter(&b) enc := gob.NewEncoder(zw) @@ -108,15 +114,15 @@ func blobifyEncrypted(be blobEntry, key *[32]byte) ([]byte, error) { if err != nil { return nil, err } - blob, err := encryptAndPack(b.Bytes(), key) + blob, err := key.Encrypt(blobEntryVersion, b.Bytes()) if err != nil { return nil, err } return blob, nil } -func deblobEncrypted(blob []byte, key *[32]byte) (*blobEntry, error) { - b, err := unpackAndDecrypt(key, blob) +func deblobEncrypted(blob []byte, key *EncryptionKey) (*blobEntry, error) { + b, _, err := key.Decrypt(blob) if err != nil { return nil, err } diff --git a/politeiad/backend/tlogbe/dcrtime.go b/politeiad/backend/tlogbe/dcrtime.go new file mode 100644 index 000000000..2a8168bea --- /dev/null +++ b/politeiad/backend/tlogbe/dcrtime.go @@ -0,0 +1,168 @@ +package tlogbe + +import ( + "bytes" + "crypto/tls" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "time" + + dcrtime "github.com/decred/dcrtime/api/v2" + "github.com/decred/dcrtime/merkle" + "github.com/decred/politeia/util" +) + +var ( + // TODO flip skipVerify to false when done testing + skipVerify = true + httpClient = &http.Client{ + Timeout: 1 * time.Minute, + Transport: &http.Transport{ + IdleConnTimeout: 1 * time.Minute, + ResponseHeaderTimeout: 1 * time.Minute, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: skipVerify, + }, + }, + } +) + +// isDigest returns whether the provided digest is a valid SHA256 digest. +func isDigest(digest string) bool { + return dcrtime.RegexpSHA256.MatchString(digest) +} + +// timestampBatch posts the provided digests to the dcrtime v2 batch timestamp +// route. +func timestampBatch(host, id string, digests []string) (*dcrtime.TimestampBatchReply, error) { + log.Tracef("timestampBatch: %v %v %v", host, id, digests) + + // Validate digests + for _, v := range digests { + if !isDigest(v) { + return nil, fmt.Errorf("invalid digest: %v", v) + } + } + + // Setup request + tb := dcrtime.TimestampBatch{ + ID: id, + Digests: digests, + } + b, err := json.Marshal(tb) + if err != nil { + return nil, err + } + + // Send request + route := host + dcrtime.TimestampBatchRoute + r, err := httpClient.Post(route, "application/json", bytes.NewReader(b)) + if err != nil { + return nil, err + } + defer r.Body.Close() + + // Handle response + if r.StatusCode != http.StatusOK { + e, err := util.GetErrorFromJSON(r.Body) + if err != nil { + return nil, fmt.Errorf("%v", r.Status) + } + return nil, fmt.Errorf("%v: %v", r.Status, e) + } + var tbr dcrtime.TimestampBatchReply + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&tbr); err != nil { + return nil, fmt.Errorf("decode TimestampBatchReply: %v", err) + } + + return &tbr, nil +} + +// verifyBatch returns the data to verify that a digest was included in a +// dcrtime timestamp. This function verifies the merkle path and merkle root of +// all successful timestamps. The caller is responsible for check the result +// code and handling digests that failed to be timestamped. +// +// Note the Result in the reply will be set to OK as soon as the digest is +// waiting to be anchored. The ChainInformation will be populated once the +// digest has been included in a dcr transaction, except for the ChainTimestamp +// field. The ChainTimestamp field is only populated once the dcr transaction +// has 6 confirmations. +func verifyBatch(host, id string, digests []string) (*dcrtime.VerifyBatchReply, error) { + log.Tracef("verifyBatch: %v %v %v", host, id, digests) + + // Validate digests + for _, v := range digests { + if !isDigest(v) { + return nil, fmt.Errorf("invalid digest: %v", v) + } + } + + // Setup request + vb := dcrtime.VerifyBatch{ + ID: id, + Digests: digests, + } + b, err := json.Marshal(vb) + if err != nil { + return nil, err + } + + // Send request + route := host + dcrtime.VerifyBatchRoute + r, err := httpClient.Post(route, "application/json", bytes.NewReader(b)) + if err != nil { + return nil, err + } + defer r.Body.Close() + + // Handle response + if r.StatusCode != http.StatusOK { + e, err := util.GetErrorFromJSON(r.Body) + if err != nil { + return nil, fmt.Errorf("%v", r.Status) + } + return nil, fmt.Errorf("%v: %v", r.Status, e) + } + var vbr dcrtime.VerifyBatchReply + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&vbr); err != nil { + return nil, fmt.Errorf("decode VerifyBatchReply: %v", err) + } + + // Verify the merkle path and the merkle root of the timestamps + // that were successful. The caller is responsible for handling + // the digests that failed be timestamped. + for _, v := range vbr.Digests { + if v.Result != dcrtime.ResultOK { + // Nothing to verify + continue + } + + // Verify merkle path + root, err := merkle.VerifyAuthPath(&v.ChainInformation.MerklePath) + if err != nil { + if err == merkle.ErrEmpty { + // A dcr transaction has not been sent yet so there is + // nothing to verify. + continue + } + return nil, fmt.Errorf("VerifyAuthPath %v: %v", v.Digest, err) + } + + // Verify merkle root + merkleRoot, err := hex.DecodeString(v.ChainInformation.MerkleRoot) + if err != nil { + return nil, fmt.Errorf("invalid merkle root: %v", err) + } + if !bytes.Equal(merkleRoot, root[:]) { + return nil, fmt.Errorf("invalid merkle root %v: got %x, want %x", + v.Digest, merkleRoot, root[:]) + } + } + + return &vbr, nil +} diff --git a/politeiad/backend/tlogbe/encrypt.go b/politeiad/backend/tlogbe/encrypt.go deleted file mode 100644 index ab5716552..000000000 --- a/politeiad/backend/tlogbe/encrypt.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) 2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package tlogbe - -import ( - "crypto/rand" - "errors" - "fmt" - "io" - - "golang.org/x/crypto/nacl/secretbox" -) - -// TODO use secretbox functions instead -func encryptAndPack(data []byte, key *[32]byte) ([]byte, error) { - var nonce [24]byte - - // Random nonce - _, err := io.ReadFull(rand.Reader, nonce[:]) - if err != nil { - return nil, err - } - - // Encrypt data - blob := secretbox.Seal(nil, data, &nonce, key) - - // Pack all the things - packed := make([]byte, len(nonce)+len(blob)) - copy(packed[0:], nonce[:]) - copy(packed[24:], blob) - - return packed, nil -} - -func unpackAndDecrypt(key *[32]byte, packed []byte) ([]byte, error) { - if len(packed) < 24 { - return nil, errors.New("not an sbox file") - } - - var nonce [24]byte - copy(nonce[:], packed[0:24]) - - decrypted, ok := secretbox.Open(nil, packed[24:], &nonce, key) - if !ok { - return nil, fmt.Errorf("could not decrypt") - } - return decrypted, nil -} diff --git a/politeiad/backend/tlogbe/encryptionkey.go b/politeiad/backend/tlogbe/encryptionkey.go new file mode 100644 index 000000000..d439ec288 --- /dev/null +++ b/politeiad/backend/tlogbe/encryptionkey.go @@ -0,0 +1,58 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlogbe + +import ( + "sync" + + "github.com/decred/politeia/util" + "github.com/marcopeereboom/sbox" +) + +// EncryptionKey provides an API for encrypting and decrypting data using a +// structure that can be passed to plugins and accessed concurrently. +type EncryptionKey struct { + sync.RWMutex + key *[32]byte +} + +// Encrypt encrypts the provided data. It prefixes the encrypted blob with an +// sbox header which encodes the provided version. The version is user provided +// and can be used as a hint to identify or version the packed blob. Version is +// not inspected or used by Encrypt and Decrypt. The read lock is held to +// prevent the golang race detector from complaining when the encryption key is +// zeroed out on application exit. +func (e *EncryptionKey) Encrypt(version uint32, blob []byte) ([]byte, error) { + e.RLock() + defer e.RUnlock() + + return sbox.Encrypt(version, e.key, blob) +} + +// decrypt decrypts the provided packed blob. The decrypted blob and the +// version that was used to encrypt the blob are returned. The read lock is +// held to prevent the golang race detector from complaining when the +// encryption key is zeroed out on application exit. +func (e *EncryptionKey) Decrypt(blob []byte) ([]byte, uint32, error) { + e.RLock() + defer e.RUnlock() + + return sbox.Decrypt(e.key, blob) +} + +// Zero zeroes out the encryption key. +func (e *EncryptionKey) Zero() { + e.Lock() + defer e.Unlock() + + util.Zero(e.key[:]) + e.key = nil +} + +func encryptionKeyNew(key *[32]byte) *EncryptionKey { + return &EncryptionKey{ + key: key, + } +} diff --git a/politeiad/backend/tlogbe/index.go b/politeiad/backend/tlogbe/index.go index 9c5f936ad..25bb1d06b 100644 --- a/politeiad/backend/tlogbe/index.go +++ b/politeiad/backend/tlogbe/index.go @@ -52,19 +52,16 @@ type recordHistory struct { State string `json:"state"` // Unvetted or vetted Versions map[uint32]recordIndex `json:"versions"` // [version]recordIndex - // TODO remove anchor when deleting an orphaned blob - // Anchors contains the anchored log root hash for each piece of - // record content. It aggregates the merkle leaf hashes from all - // record index versions. The log root hash can be used to lookup - // the anchor structure from the key-value store, which contains - // the dcrtime inclusion proof, or can be used to obtain the - // inclusion proof from dcrtime itself if needed. The merkle leaf - // hash is hex encoded. The log root hash is a SHA256 digest of the - // encoded LogRootV1. - Anchors map[string][]byte `json:"anchors"` // [merkleLeafHash]logRootHash + // Anchors contains the digests of all log roots that have been + // anchored for this record. The leaf height of any individual + // piece of record content can be used to look up the earliest + // anchor that the leaf was included in. The log root digest can + // be used to lookup the anchor record. The log root digest is a + // SHA256 digest of the encoded LogRootV1 structure. + Anchors map[uint64][]byte `json:"anchors"` // [treeHeight]logRootDigest } -// String returns the recordHistory printed in human readable format. +// String returns the recordHistory printed in a human readable format. func (r *recordHistory) String() string { s := fmt.Sprintf("Token: %x\n", r.Token) s += fmt.Sprintf("State: %v\n", r.State) @@ -211,5 +208,6 @@ func recordHistoryNew(token []byte) recordHistory { Token: token, State: stateUnvetted, Versions: make(map[uint32]recordIndex), + Anchors: make(map[uint64][]byte), } } diff --git a/politeiad/backend/tlogbe/plugin/comments/comments.go b/politeiad/backend/tlogbe/plugin/comments/comments.go new file mode 100644 index 000000000..9247e3463 --- /dev/null +++ b/politeiad/backend/tlogbe/plugin/comments/comments.go @@ -0,0 +1,255 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package comments + +import ( + "encoding/hex" + "os" + "path/filepath" + "sync" + + "github.com/decred/politeia/plugins/comments" + "github.com/decred/politeia/politeiad/api/v1/identity" + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugin" + "github.com/decred/politeia/politeiad/backend/tlogbe/store" + "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" +) + +const ( + commentsDirname = "comments" + + // Data descriptors + dataDescriptorRecordComments = "recordcomments" + dataDescriptorComment = "comment" + + // Key-value store key prefixes + keyPrefixRecordComments = "record" + keyPrefixComment = "comment" +) + +var ( + _ plugin.Plugin = (*commentsPlugin)(nil) +) + +// TODO unvetted comments should be encrypted +// TODO journal should be encrypted + +type commentsPlugin struct { + sync.RWMutex + id *identity.FullIdentity + encyrptionKey *tlogbe.EncryptionKey + tlog *tlogbe.TrillianClient + backend backend.Backend + store store.Blob + + // Mutexes contains a mutex for each record. The mutexes are lazy + // loaded. + mutexes map[string]*sync.RWMutex // [token]mutex +} + +type commentIndex struct { + CommentID uint32 `json:"commentid"` + Versions map[uint32][]byte `json:"versions"` // [version]merkleLeafHash +} + +// recordComments contains the comment index for all comments made on a record. +type recordComments struct { + Token string `json:"token"` + + // LastID contains the last comment ID that has been assigned for + // this record. Comment IDs are sequential starting with 1. The + // state of the record, unvetted or vetted, does not impact the + // comment ID that is assinged. + LastID uint32 `json:"lastid"` + + // Unvetted contains comments that were made on the record when it + // was in an unvetted state. Unvetted comments are encrypted before + // being saved to the key-value store. They remain encrypted for + // the duration of their lifetime, even after the record itself + // becomes vetted. + // + // map[commentID]commentIndex + Unvetted map[uint32]commentIndex `json:"unvetted"` + + // Vetted contains comments that were made on a vetted record. + // Vetted comments are stored in the key-value store unencrypted. + // + // map[commentID]commentIndex + Vetted map[uint32]commentIndex `json:"vetted"` +} + +func keyRecordComments(token string) string { + return keyPrefixRecordComments + token +} + +// keyComment returns the key for a comment in the key-value store. +func keyComment(token string, merkleLeafHash []byte) string { + return keyPrefixComment + hex.EncodeToString(merkleLeafHash) +} + +/* +func convertBlobEntryFromRecordComments(rc recordComments) (*blobEntry, error) { + data, err := json.Marshal(rc) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + dataDescriptor{ + Type: dataTypeStructure, + Descriptor: dataDescriptorRecordComments, + }) + if err != nil { + return nil, err + } + be := blobEntryNew(hint, data) + return &be, nil +} + +// mutex returns the mutex for the provided token. This function assumes that +// the provided token has already been validated and corresponds to a record in +// the Backend. +func (p *commentsPlugin) mutex(token string) (*sync.RWMutex, error) { + p.Lock() + defer p.Unlock() + + m, ok := p.mutexes[token] + if !ok { + // Mutexes is lazy loaded + m = &sync.RWMutex{} + p.mutexes[token] = m + } + + return m, nil +} + +// recordExists returns whether the provided record exists in the backend. +// This function does not differentiate between unvetted and vetted records. +func (p *commentsPlugin) recordExists(token string) bool { + t, err := hex.DecodeString(token) + if err != nil { + return false + } + if p.backend.UnvettedExists(t) { + return true + } + if p.backend.VettedExists(t) { + return true + } + return false +} + +/* +func (p *commentsPlugin) recordComments(token string) (*recordComments, error) { + be, err := p.store.Get(keyRecordComments(token)) + if err != nil { + return nil, err + } + rc, err := convertBlobEntryFromRecordComments(be) + if err != nil { + return nil, err + } + return &rc, nil +} + +func (p *commentsPlugin) commentExists(token string, commentID uint32) bool { + ri, err := p.recordComments(token) + if err != nil { + return false + } + _, ok := ri.Unvetted[commentID] + if ok { + return true + } + _, ok = ri.Vetted[commentID] + if ok { + return true + } + return false +} + +func (p *commentsPlugin) cmdNew(payload string) (string, error) { + n, err := comments.DecodeNew([]byte(payload)) + if err != nil { + return "", err + } + + // Verify signature + err = comments.VerifyCommentSignature(n.Signature, n.PublicKey, + n.Token, n.ParentID, n.Comment) + if err != nil { + return "", err + } + + // Ensure record exists + if !p.recordExists(n.Token) { + return "", fmt.Errorf("record not found %v", n.Token) + } + + // Ensure parent comment exists if set. A parent ID of 0 means that + // this is a base level comment, not a reply comment. + if n.ParentID > 0 && !p.commentExists(n.Token, n.ParentID) { + e := fmt.Sprintf("parent ID %v comment", n.ParentID) + return "", comments.PluginError{ + ErrorCode: comments.ErrorStatusCommentNotFound, + ErrorContext: []string{e}, + } + } + + // Setup the comment + c := comments.Comment{ + Token: n.Token, + ParentID: n.ParentID, + Comment: n.Comment, + PublicKey: n.PublicKey, + Signature: n.Signature, + // CommentID: , + // Version: , + // Receipt: "", + Timestamp: time.Now().Unix(), + // Score: 0, + Deleted: false, + Censored: false, + } + _ = c + + // Append to trillian tree + + // Save to key-value store + + // Prepare reply + + return "", nil +} +*/ + +func (p *commentsPlugin) Cmd(id, payload string) (string, error) { + switch id { + case comments.CmdNew: + // return p.cmdNew(payload) + } + return "", plugin.ErrInvalidPluginCmd +} + +func (p *commentsPlugin) Setup() error { + return nil +} + +func New(dataDir string, tlog *tlogbe.TrillianClient, backend backend.Backend) (*commentsPlugin, error) { + // Setup key-value store + fp := filepath.Join(dataDir, commentsDirname) + err := os.MkdirAll(fp, 0700) + if err != nil { + return nil, err + } + store := filesystem.New(fp) + + return &commentsPlugin{ + tlog: tlog, + store: store, + backend: backend, + }, nil +} diff --git a/politeiad/backend/tlogbe/plugin/comments/journal.go b/politeiad/backend/tlogbe/plugin/comments/journal.go new file mode 100644 index 000000000..8ddfc1299 --- /dev/null +++ b/politeiad/backend/tlogbe/plugin/comments/journal.go @@ -0,0 +1,104 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package comments + +import "encoding/json" + +const ( + // JournalVersion is the current version of the comments journal. + journalVersion = "1" + + // keyPrefixJournal is the prefix to the key-value store key for a + // journal record. + keyPrefixJournal = "journal" + + // Journal actions + journalActionAdd = "add" // Add entry + journalActionEdit = "edit" // Edit entry + journalActionDel = "del" // Delete entry + journalActionCensor = "censor" // Censor entry + journalActionVote = "vote" // Vote on entry +) + +var ( + // Pregenerated journal actions + journalAdd []byte + journalEdit []byte + journalDel []byte + journalCensor []byte + journalVote []byte +) + +// journalAction prefixes and determines what the next structure is in the JSON +// journal. +type journalAction struct { + Version string `json:"version"` + Action string `json:"action"` +} + +type actionAdd struct { + ParentID uint32 `json:"parentid"` + Comment string `json:"comment"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` + CommentID uint32 `json:"commentid"` + Receipt string `json:"receipt"` + Timestamp int64 `json:"timestamp"` +} + +type actionEdit struct{} + +type actionDel struct{} + +type actionCensor struct{} + +type actionVote struct{} + +// keyJournal returns the key-value store key for a journal record. The token +// is not included in any journal actions since it is a static value and is +// already part of the key. +func keyJournal(token string) string { + return keyPrefixJournal + token +} + +// init is used to pregenerate the JSON journal actions. +func init() { + var err error + journalAdd, err = json.Marshal(journalAction{ + Version: journalVersion, + Action: journalActionAdd, + }) + if err != nil { + panic(err.Error()) + } + journalEdit, err = json.Marshal(journalAction{ + Version: journalVersion, + Action: journalActionEdit, + }) + if err != nil { + panic(err.Error()) + } + journalDel, err = json.Marshal(journalAction{ + Version: journalVersion, + Action: journalActionDel, + }) + if err != nil { + panic(err.Error()) + } + journalCensor, err = json.Marshal(journalAction{ + Version: journalVersion, + Action: journalActionCensor, + }) + if err != nil { + panic(err.Error()) + } + journalVote, err = json.Marshal(journalAction{ + Version: journalVersion, + Action: journalActionVote, + }) + if err != nil { + panic(err.Error()) + } +} diff --git a/politeiad/backend/tlogbe/plugin/comments/log.go b/politeiad/backend/tlogbe/plugin/comments/log.go new file mode 100644 index 000000000..f1d18be7c --- /dev/null +++ b/politeiad/backend/tlogbe/plugin/comments/log.go @@ -0,0 +1,25 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package comments + +import "github.com/decred/slog" + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log = slog.Disabled + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = slog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using slog. +func UseLogger(logger slog.Logger) { + log = logger +} diff --git a/politeiad/backend/tlogbe/plugin/plugin.go b/politeiad/backend/tlogbe/plugin/plugin.go new file mode 100644 index 000000000..25936a9bd --- /dev/null +++ b/politeiad/backend/tlogbe/plugin/plugin.go @@ -0,0 +1,14 @@ +package plugin + +import "errors" + +var ( + // ErrInvalidPluginCmd is emitted when an invalid plugin command is + // used. + ErrInvalidPluginCmd = errors.New("invalid plugin command") +) + +type Plugin interface { + Setup() error + Cmd(id, payload string) (string, error) +} diff --git a/politeiad/backend/tlogbe/store/filesystem/filesystem.go b/politeiad/backend/tlogbe/store/filesystem/filesystem.go index c5f3604bb..c2a06bc6c 100644 --- a/politeiad/backend/tlogbe/store/filesystem/filesystem.go +++ b/politeiad/backend/tlogbe/store/filesystem/filesystem.go @@ -15,7 +15,8 @@ import ( ) var ( - _ store.Blob = (*fileSystem)(nil) +// TODO put back +// _ store.Blob = (*fileSystem)(nil) ) // fileSystem implements the Blob interface using the file system. @@ -103,7 +104,7 @@ func (f *fileSystem) Enum(cb func(key string, value []byte) error) error { return nil } -func (f *fileSystem) multi(ops store.Ops) error { +func (f *fileSystem) batch(ops store.Ops) error { for _, fn := range ops.Del { log.Tracef("del: %v", fn) err := f.del(fn) @@ -121,8 +122,8 @@ func (f *fileSystem) multi(ops store.Ops) error { return nil } -func (f *fileSystem) Multi(ops store.Ops) error { - log.Tracef("Multi") +func (f *fileSystem) Batch(ops store.Ops) error { + log.Tracef("Batch") f.Lock() defer f.Unlock() @@ -152,15 +153,15 @@ func (f *fileSystem) Multi(ops store.Ops) error { puts[fn] = b } - err := f.multi(ops) + err := f.batch(ops) if err != nil { // Unwind puts for fn := range ops.Put { err2 := f.del(fn) if err2 != nil { // This is ok. It just means the file was never saved before - // the multi function exited with an error. - log.Debugf("unwind multi: del %v: %v", fn, err2) + // the batch function exited with an error. + log.Debugf("batch unwind: del %v: %v", fn, err2) continue } } @@ -171,7 +172,7 @@ func (f *fileSystem) Multi(ops store.Ops) error { err2 := f.put(fn, b) if err2 != nil { // We're in trouble! - log.Criticalf("multi unwind: unable to put original file back %v: %v", + log.Criticalf("batch unwind: unable to put original file back %v: %v", fn, err2) unwindFailed = true continue @@ -189,7 +190,7 @@ func (f *fileSystem) Multi(ops store.Ops) error { err2 = f.put(fn, b) if err2 != nil { // We're in trouble! - log.Criticalf("multi unwind: unable to put deleted file back %v: %v", + log.Criticalf("batch unwind: unable to put deleted file back %v: %v", fn, err2) unwindFailed = true continue @@ -199,8 +200,8 @@ func (f *fileSystem) Multi(ops store.Ops) error { if unwindFailed { // Print orignal error that caused the unwind then panic // because the unwind failed. - log.Errorf("multi: %v", err) - panic("multi unwind failed") + log.Errorf("batch: %v", err) + panic("batch unwind failed") } return err diff --git a/politeiad/backend/tlogbe/store/store.go b/politeiad/backend/tlogbe/store/store.go index 2a4525396..ced8e02e8 100644 --- a/politeiad/backend/tlogbe/store/store.go +++ b/politeiad/backend/tlogbe/store/store.go @@ -7,20 +7,44 @@ package store import "errors" var ( + // ErrNotFound is emitted when a blob is not found. ErrNotFound = errors.New("not found") ) -// Ops specifies multiple Blob operations that should be performed atomically. type Ops struct { Put map[string][]byte Del []string } -// Blob represents a blob key-value store. type Blob interface { - Put(key string, blob []byte) error // Store blob - Get(key string) ([]byte, error) // Get blob by identifier - Del(key string) error // Attempt to delete object - Enum(func(key string, blob []byte) error) error // Enumerate over all objects - Multi(Ops) error // Perform multiple operations atomically + Get(string) ([]byte, error) + Put(string, []byte) error + Del(key string) error + Enum(func(key string, blob []byte) error) error + Batch(Ops) error +} + +// Blob represents a blob key-value store. +type Blob_ interface { + // Get returns blobs from the store. An entry will not exist in the + // returned map if for any blobs that are not found. It is the + // responsibility of the caller to ensure a blob was returned for + // all provided keys. + Get(keys []string) (map[string][]byte, error) + + // Put saves the provided blobs to the store. The keys for the + // blobs are returned using the same odering that the blobs were + // provided in. This operation is performed atomically. + Put(blobs [][]byte) ([]string, error) + + // Del deletes the provided blobs from the store. This operation + // is performed atomically. + Del(keys []string) error + + // Enum enumerates over all blobs in the store, invoking the + // provided function for each blob. + Enum(func(key string, blob []byte) error) error + + // Closes closes the blob store connection. + Close() } diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go new file mode 100644 index 000000000..28aecaffd --- /dev/null +++ b/politeiad/backend/tlogbe/tlog.go @@ -0,0 +1,187 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlogbe + +import ( + "bytes" + "encoding/base64" + "encoding/binary" + "encoding/hex" + "encoding/json" + "fmt" + + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/store" +) + +var ( + prefixRecordIndex = []byte("index:") +) + +// We do not unwind. +type tlog struct { + dataDir string + client *TrillianClient + encryptionKey *EncryptionKey + store store.Blob_ +} + +type versionIndex struct { + RecordMetadata []byte `json:"recordmetadata"` + Metadata map[uint64][]byte `json:"metadata"` // [metadataID]leafHash + Files map[string][]byte `json:"files"` // [filename]leafHash +} + +type recordIndex_ struct { + Versions map[uint32]versionIndex `json:"versions"` +} + +func tokenFromTreeID(treeID int64) []byte { + b := make([]byte, binary.MaxVarintLen64) + // Converting between int64 and uint64 doesn't change the sign bit, + // only the way it's interpreted. + binary.LittleEndian.PutUint64(b, uint64(treeID)) + return b +} + +func treeIDFromToken(token []byte) int64 { + return int64(binary.LittleEndian.Uint64(token)) +} + +func tokenString(token []byte) string { + return hex.EncodeToString(token) +} + +func convertRecordIndexFromBlobEntry(be blobEntry) (*recordIndex, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd dataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorRecordIndex { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorRecordIndex) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + var ri recordIndex_ + err = json.Unmarshal(b, &ri) + if err != nil { + return nil, fmt.Errorf("unmarshal recordIndex: %v", err) + } + + return &ri, nil +} + +func (t *tlog) tokenNew() ([]byte, error) { + tree, _, err := t.client.treeNew() + if err != nil { + return nil, fmt.Errorf("treeNew: %v", err) + } + return tokenFromTreeID(tree.TreeId), nil +} + +func (t *tlog) recordExists() {} + +func (t *tlog) recordGetVersion(token []byte, version uint64) (*backend.Record, error) { + // Get tree leaves + treeID := treeIDFromToken(token) + leaves, err := t.client.LeavesAll(treeID) + if err != nil { + return nil, fmt.Errorf("LeavesAll %v: %v", treeID, err) + } + + // Walk the leaves backwards to find the most recent record index. + var indexKey string + prefix := []byte(prefixRecordIndex) + for i := len(leaves) - 1; i >= 0; i-- { + l := leaves[i] + if bytes.HasPrefix(l.ExtraData, prefix) { + // Record index found. Extract key. + s := bytes.SplitAfter(l.ExtraData, prefix) + if len(s) != 2 { + return nil, fmt.Errorf("invalid leaf extra data: %x %s", + l.MerkleLeafHash, l.ExtraData) + } + indexKey = string(s[1]) + } + } + if indexKey == "" { + return nil, fmt.Errorf("record index not found") + } + + // Get the record index from the store + blobs, err := t.store.Get([]string{indexKey}) + if err != nil { + return nil, fmt.Errorf("store Get %x: %v", err) + } + b, ok := blobs[indexKey] + if !ok { + return nil, fmt.Errorf("record index not found: %v", indexKey) + } + be, err := deblob(b) + if err != nil { + return nil, err + } + r, err := convertRecordIndexFromBlobEntry(*be) + if err != nil { + return nil, err + } + if len(r.Versions) == 0 { + return nil, fmt.Errorf("version indexes not found") + } + + // Get the record content from the store + metadata := make([]backend.MetadataStream, 0, len(idx.Metadata)) + files := make([]backend.File, 0, len(idx.Files)) + + return nil, nil +} + +func (t *tlog) recordGet(token []byte) (*backend.Record, error) { + return nil, nil +} + +func (t *tlog) recordSave(token []byte, metadata []backend.MetadataStream, files []backend.File, rm backend.RecordMetadata) (*backend.Record, error) { + // Validate changes + switch { + case rm.Iteration == 1 && len(files) == 0: + // A new record must contain files + return nil, fmt.Errorf("no files") + case rm.Iteration > 1: + // Get the existing record and ensure that files changes are + // being made. + } + + // Save content to key-value store + + // Append content to trillian tree + + // Save record index + + return nil, nil +} + +func (t *tlog) recordStatusUpdate() {} + +func (t *tlog) recordMetdataUpdate() {} + +func (t *tlog) recordProof() {} + +func (t *tlog) recordFreeze(token []byte, pointer int64) {} + +func (t *tlog) fsck() { + // TODO soft delete trees that don't have leaves and are more than + // a week old. This can happen if a record new call fails. +} diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 5f9b8e302..774dbe3cf 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -6,11 +6,8 @@ package tlogbe import ( "bytes" - "context" - "crypto" "crypto/sha256" "encoding/base64" - "encoding/binary" "encoding/hex" "encoding/json" "fmt" @@ -30,31 +27,28 @@ import ( "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" "github.com/decred/politeia/util" "github.com/google/trillian" - "github.com/google/trillian/crypto/keys" - "github.com/google/trillian/crypto/keys/der" - "github.com/google/trillian/crypto/keyspb" "github.com/google/trillian/types" "github.com/marcopeereboom/sbox" - "google.golang.org/grpc" + "github.com/robfig/cron" "google.golang.org/grpc/codes" ) -// TODO make unvetted record metdata behavior the same as vetted -// and document why gitbe handles them differently -// TODO if we want to get rid of the cache we will need to make the -// locking more granular and lock on the individual token level +// TODO we need to seperate testnet and mainnet trillian trees. This will need +// to be done through setup scripts and config options. Use different ports for +// testnet. +// TODO lock on the token level const ( defaultTrillianKeyFilename = "trillian.key" defaultEncryptionKeyFilename = "tlogbe.key" - blobsDirname = "blobs" + recordsDirname = "records" ) var ( _ backend.Backend = (*tlogbe)(nil) - // statusChanges contains the allowed record status changes + // statusChanges contains the allowed record status changes. statusChanges = map[backend.MDStatusT]map[backend.MDStatusT]struct{}{ // Unvetted status changes backend.MDStatusUnvetted: map[backend.MDStatusT]struct{}{ @@ -81,40 +75,25 @@ type tlogbe struct { homeDir string dataDir string dcrtimeHost string - encryptionKey *[32]byte + encryptionKey *EncryptionKey store store.Blob + tlog *TrillianClient + cron *cron.Cron plugins []backend.Plugin - // Trillian - grpc *grpc.ClientConn // Trillian grpc connection - client trillian.TrillianLogClient // Trillian log client - admin trillian.TrillianAdminClient // Trillian admin client - ctx context.Context // Context used for trillian calls - privateKey *keyspb.PrivateKey // Trillian signing key - publicKey crypto.PublicKey // Trillian public key - - // dirty keeps track of which trillian trees are dirty and at what - // height. Dirty means that the tree has leaves that have not been - // anchored yet. - dirtyMtx sync.RWMutex - dirty map[int64]uint64 // [treeid]height - droppingAnchor bool // Anchor dropping is in progress -} + unvetted *tlog + vetted *tlog -func tokenFromTreeID(treeID int64) []byte { - b := make([]byte, binary.MaxVarintLen64) - // Converting between int64 and uint64 doesn't change the sign bit, - // only the way it's interpreted. - binary.LittleEndian.PutUint64(b, uint64(treeID)) - return b -} + // prefixes contains the first n characters of each record token, + // where n is defined by the TokenPrefixLength from the politeiad + // API. Lookups by token prefix are allowed. This cache is used to + // prevent prefix collisions when creating new tokens. + prefixes map[string]struct{} -func treeIDFromToken(token []byte) int64 { - return int64(binary.LittleEndian.Uint64(token)) -} - -func tokenString(token []byte) string { - return hex.EncodeToString(token) + // droppingAnchor indicates whether tlogbe is in the process of + // dropping an anchor, i.e. timestamping unanchored trillian trees + // using dcrtime. An anchor is dropped periodically using cron. + droppingAnchor bool } // statusChangeIsAllowed returns whether the provided status change is allowed @@ -130,48 +109,6 @@ func statusChangeIsAllowed(from, to backend.MDStatusT) bool { return ok } -func (t *tlogbe) dirtyAdd(treeID int64, treeSize uint64) { - log.Tracef("dirtyAdd treeID:%v treeSize:%v", treeID, treeSize) - - t.dirtyMtx.Lock() - defer t.dirtyMtx.Unlock() - - t.dirty[treeID] = treeSize -} - -func (t *tlogbe) dirtyDel(treeID int64) { - log.Tracef("dirtyDel: %v", treeID) - - t.dirtyMtx.Lock() - defer t.dirtyMtx.Unlock() - - delete(t.dirty, treeID) -} - -func (t *tlogbe) dirtyHeight(treeID int64) (uint64, bool) { - log.Tracef("dirtyHeight: %v", treeID) - - t.dirtyMtx.RLock() - defer t.dirtyMtx.RUnlock() - - h, ok := t.dirty[treeID] - return h, ok -} - -func (t *tlogbe) dirtyCopy() map[int64]uint64 { - log.Tracef("dirtyCopy") - - t.dirtyMtx.RLock() - defer t.dirtyMtx.RUnlock() - - dirtyCopy := make(map[int64]uint64, len(t.dirty)) - for k, v := range t.dirty { - dirtyCopy[k] = v - } - - return dirtyCopy -} - func merkleRoot(files []backend.File) (*[sha256.Size]byte, error) { hashes := make([]*[sha256.Size]byte, 0, len(files)) for _, v := range files { @@ -488,7 +425,7 @@ func metadataStreamsApplyChanges(md, mdAppend, mdOverwrite []backend.MetadataStr // such as when a file is deleted from a record then added back to the record // without being altered. The order of the returned leaf proofs is not // guaranteed. -func (t *tlogbe) blobEntriesAppend(token []byte, entries []blobEntry) ([]leafProof, *types.LogRootV1, error) { +func (t *tlogbe) blobEntriesAppend(token []byte, entries []blobEntry) ([]LeafProof, *types.LogRootV1, error) { // Setup request treeID := treeIDFromToken(token) leaves, err := convertLeavesFromBlobEntries(entries) @@ -497,7 +434,7 @@ func (t *tlogbe) blobEntriesAppend(token []byte, entries []blobEntry) ([]leafPro } // Append leaves - queued, lr, err := t.leavesAppend(treeID, leaves) + queued, lr, err := t.tlog.LeavesAppend(treeID, leaves) if err != nil { return nil, nil, fmt.Errorf("leavesAppend: %v", err) } @@ -511,7 +448,7 @@ func (t *tlogbe) blobEntriesAppend(token []byte, entries []blobEntry) ([]leafPro // Convert queuedLeafProofs to leafProofs. Fail if any of the // leaves were not appended successfully. The exception to this is // if the leaf was not appended because it was a duplicate. - proofs := make([]leafProof, 0, len(queued)) + proofs := make([]LeafProof, 0, len(queued)) dups := make([][]byte, 0, len(queued)) failed := make([]string, 0, len(queued)) for _, v := range queued { @@ -519,7 +456,7 @@ func (t *tlogbe) blobEntriesAppend(token []byte, entries []blobEntry) ([]leafPro switch c { case codes.OK: // Leaf successfully appended to tree - proofs = append(proofs, leafProof{ + proofs = append(proofs, LeafProof{ Leaf: v.QueuedLeaf.Leaf, Proof: v.Proof, }) @@ -552,7 +489,7 @@ func (t *tlogbe) blobEntriesAppend(token []byte, entries []blobEntry) ([]leafPro // Retrieve leaf proofs for duplicates if len(dups) > 0 { - p, err := t.leafProofs(treeID, dups, lr) + p, err := t.tlog.LeafProofs(treeID, dups, lr) if err != nil { return nil, nil, fmt.Errorf("leafProofs: %v", err) } @@ -689,6 +626,7 @@ func (t *tlogbe) recordSave(token []byte, metadata []backend.MetadataStream, fil if err != nil { return nil, err } + // The RecordMetadata is intentionally put first so that it is // added to the trillian tree first. If we ever need to walk the // tree a RecordMetadata will signify the start of a new record. @@ -770,11 +708,11 @@ func (t *tlogbe) recordSave(token []byte, metadata []backend.MetadataStream, fil // Save all blobs log.Debugf("Saving %v blobs to kv store", len(blobs)) - err = t.store.Multi(store.Ops{ + err = t.store.Batch(store.Ops{ Put: blobs, }) if err != nil { - return nil, fmt.Errorf("store Multi: %v", err) + return nil, fmt.Errorf("store Batch: %v", err) } // Lookup new version of the record @@ -1107,12 +1045,12 @@ func (t *tlogbe) recordUpdate(token []byte, mdAppend, mdOverwrite []backend.Meta } // Save all the blob changes - err = t.store.Multi(store.Ops{ + err = t.store.Batch(store.Ops{ Put: blobs, Del: del, }) if err != nil { - return nil, fmt.Errorf("store Multi: %v", err) + return nil, fmt.Errorf("store Batch: %v", err) } // Retrieve and return the updated record @@ -1133,7 +1071,7 @@ func (t *tlogbe) recordUpdate(token []byte, mdAppend, mdOverwrite []backend.Meta return r, nil } -/// New satisfies the Backend interface. +// New satisfies the Backend interface. func (t *tlogbe) New(metadata []backend.MetadataStream, files []backend.File) (*backend.RecordMetadata, error) { log.Tracef("New") @@ -1143,29 +1081,52 @@ func (t *tlogbe) New(metadata []backend.MetadataStream, files []backend.File) (* return nil, err } - // Create a new trillian tree. The treeID is used as the record - // token. - // TODO handle token prefix collisions - tree, _, err := t.treeNew() + // Create token + token, err := t.unvetted.tokenNew() if err != nil { - return nil, fmt.Errorf("treeNew: %v", err) + return nil, err } - token := tokenFromTreeID(tree.TreeId) - // Save record - rh := recordHistoryNew(token) + // TODO handle token prefix collisions + + // Create record metadata rm, err := recordMetadataNew(token, files, backend.MDStatusUnvetted, 1) if err != nil { return nil, err } - _, err = t.recordSave(token, metadata, files, *rm, rh) + + // Save record + r, err := t.unvetted.recordSave(token, metadata, files, *rm) if err != nil { - return nil, fmt.Errorf("recordSave %x: %v", token, err) + return nil, fmt.Errorf("unvetted save %x: %v", token, err) } - log.Infof("New record tree:%v token:%x", tree.TreeId, token) + log.Infof("New record %x", token) - return rm, nil + return &r.RecordMetadata, nil + /* + // Create a tree + tree, _, err := t.client.treeNew() + if err != nil { + return nil, fmt.Errorf("treeNew: %v", err) + } + token := tokenFromTreeID(tree.TreeId) + + // Save record + rh := recordHistoryNew(token) + rm, err := recordMetadataNew(token, files, backend.MDStatusUnvetted, 1) + if err != nil { + return nil, err + } + _, err = t.recordSave(token, metadata, files, *rm, rh) + if err != nil { + return nil, fmt.Errorf("recordSave %x: %v", token, err) + } + + log.Infof("New record tree:%v token:%x", tree.TreeId, token) + + return rm, nil + */ } func (t *tlogbe) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { @@ -1587,6 +1548,12 @@ func (t *tlogbe) Inventory(vettedCount uint, branchCount uint, includeFiles, all // gitbe correspond to unvetted records. Neither of these are // implemented in gitbe so they will not be implemented here // either. They can be added in the future if they are needed. + switch { + case vettedCount != 0: + return nil, nil, fmt.Errorf("vetted count is not implemented") + case branchCount != 0: + return nil, nil, fmt.Errorf("branch count is not implemented") + } // Get all record histories from the store hists := make([]recordHistory, 0, 1024) @@ -1663,42 +1630,13 @@ func (t *tlogbe) Close() { t.shutdown = true // Close trillian connection - t.grpc.Close() + t.tlog.Close() // Zero out encryption key - util.Zero(t.encryptionKey[:]) - t.encryptionKey = nil + t.encryptionKey.Zero() } func New(homeDir, dataDir, trillianHost, trillianKeyFile, dcrtimeHost, encryptionKeyFile string) (*tlogbe, error) { - // Setup trillian key file - if trillianKeyFile == "" { - // No file path was given. Use the default path. - trillianKeyFile = filepath.Join(homeDir, defaultTrillianKeyFilename) - } - if !util.FileExists(trillianKeyFile) { - // Trillian key file does not exist. Create one. - log.Infof("Generating trillian private key") - if trillianKeyFile == "" { - } - key, err := keys.NewFromSpec(&keyspb.Specification{ - // TODO Params: &keyspb.Specification_Ed25519Params{}, - Params: &keyspb.Specification_EcdsaParams{}, - }) - if err != nil { - return nil, err - } - b, err := der.MarshalPrivateKey(key) - if err != nil { - return nil, err - } - err = ioutil.WriteFile(trillianKeyFile, b, 0400) - if err != nil { - return nil, err - } - log.Infof("Trillian private key created: %v", trillianKeyFile) - } - // Setup encryption key file if encryptionKeyFile == "" { // No file path was given. Use the default path. @@ -1719,72 +1657,66 @@ func New(homeDir, dataDir, trillianHost, trillianKeyFile, dcrtimeHost, encryptio log.Infof("Encryption key created: %v", encryptionKeyFile) } - // Connect to trillian - log.Infof("Trillian log server: %v", trillianHost) - g, err := grpc.Dial(trillianHost, grpc.WithInsecure()) - if err != nil { - return nil, fmt.Errorf("grpc dial: %v", err) - } - - // Setup blob key-value store - blobsPath := filepath.Join(dataDir, blobsDirname) - err = os.MkdirAll(blobsPath, 0700) + // Setup trillian client + tlog, err := trillianClientNew(homeDir, trillianHost, trillianKeyFile) if err != nil { return nil, err } - store := filesystem.New(blobsPath) - // Load trillian key pair - var privateKey = &keyspb.PrivateKey{} - privateKey.Der, err = ioutil.ReadFile(trillianKeyFile) - if err != nil { - return nil, err - } - signer, err := der.UnmarshalPrivateKey(privateKey.Der) + // Setup key-value store + fp := filepath.Join(dataDir, recordsDirname) + err = os.MkdirAll(fp, 0700) if err != nil { return nil, err } - log.Infof("Trillian key loaded") + store := filesystem.New(fp) // Load encryption key f, err := os.Open(encryptionKeyFile) if err != nil { return nil, err } - var encryptionKey [32]byte - n, err := f.Read(encryptionKey[:]) - if n != len(encryptionKey) { + var key [32]byte + n, err := f.Read(key[:]) + if n != len(key) { return nil, fmt.Errorf("invalid encryption key length") } if err != nil { return nil, err } f.Close() + encryptionKey := encryptionKeyNew(&key) + log.Infof("Encryption key loaded") - // Dcrtime host + // Setup dcrtime host _, err = url.Parse(dcrtimeHost) if err != nil { return nil, fmt.Errorf("parse dcrtime host '%v': %v", dcrtimeHost, err) } log.Infof("Anchor host: %v", dcrtimeHost) - // TODO Launch cron - // TODO fsck - // TODO test trillian connection - - return &tlogbe{ + t := tlogbe{ homeDir: homeDir, dataDir: dataDir, - encryptionKey: &encryptionKey, + encryptionKey: encryptionKey, dcrtimeHost: dcrtimeHost, store: store, - grpc: g, - client: trillian.NewTrillianLogClient(g), - admin: trillian.NewTrillianAdminClient(g), - ctx: context.Background(), - privateKey: privateKey, - publicKey: signer.Public(), - dirty: make(map[int64]uint64), - }, nil + tlog: tlog, + cron: cron.New(), + } + + // Launch cron + log.Infof("Launch cron anchor job") + err = t.cron.AddFunc(anchorSchedule, func() { + t.anchorTrees() + }) + if err != nil { + return nil, err + } + t.cron.Start() + + // TODO fsck + + return &t, nil } diff --git a/politeiad/backend/tlogbe/trillian.go b/politeiad/backend/tlogbe/trillian.go index 995b4d9dc..80c7c20b3 100644 --- a/politeiad/backend/tlogbe/trillian.go +++ b/politeiad/backend/tlogbe/trillian.go @@ -6,33 +6,57 @@ package tlogbe import ( "bytes" + "context" "crypto" "crypto/sha256" "fmt" + "io/ioutil" + "path/filepath" + "time" + "github.com/decred/politeia/util" "github.com/golang/protobuf/ptypes" "github.com/google/trillian" "github.com/google/trillian/client" tcrypto "github.com/google/trillian/crypto" + "github.com/google/trillian/crypto/keys" + "github.com/google/trillian/crypto/keys/der" + "github.com/google/trillian/crypto/keyspb" "github.com/google/trillian/crypto/sigpb" "github.com/google/trillian/merkle/hashers" "github.com/google/trillian/merkle/rfc6962" "github.com/google/trillian/types" + "google.golang.org/genproto/protobuf/field_mask" + "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" "google.golang.org/grpc/status" ) -// leafProof contains a log leaf and the inclusion proof for the log leaf. -type leafProof struct { +// TrillianClient provides a trillian client that abstracts over the existing +// TrillianLogClient and TrillianAdminClient. This is done to ensure proper +// verification of all trillian responses is performed and to restrict plugin +// access to only certain trillian functionality. +type TrillianClient struct { + grpc *grpc.ClientConn + client trillian.TrillianLogClient + admin trillian.TrillianAdminClient + ctx context.Context + privateKey *keyspb.PrivateKey // Trillian signing key + publicKey crypto.PublicKey // Trillian public key +} + +// LeafProof contains a log leaf and the inclusion proof for the log leaf. +type LeafProof struct { Leaf *trillian.LogLeaf Proof *trillian.Proof } -// queuedLeafProof contains the results of a leaf append command, i.e. the +// QueuedLeafProof contains the results of a leaf append command, i.e. the // QueuedLeaf, and the inclusion proof for that leaf. The inclusion proof will -// not be present if the leaf append command failed and the QueuedLeaf will +// not be present if the leaf append command failed. The QueuedLeaf will // contain the error code from the failure. -type queuedLeafProof struct { +type QueuedLeafProof struct { QueuedLeaf *trillian.QueuedLogLeaf Proof *trillian.Proof } @@ -52,7 +76,7 @@ func logLeafNew(value []byte) *trillian.LogLeaf { } } -func (t *tlogbe) tree(treeID int64) (*trillian.Tree, error) { +func (t *TrillianClient) tree(treeID int64) (*trillian.Tree, error) { log.Tracef("tree: %v", treeID) tree, err := t.admin.GetTree(t.ctx, &trillian.GetTreeRequest{ @@ -73,7 +97,7 @@ func (t *tlogbe) tree(treeID int64) (*trillian.Tree, error) { // treeNew returns a new trillian tree and verifies that the signatures are // correct. It returns the tree and the signed log root which can be externally // verified. -func (t *tlogbe) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { +func (t *TrillianClient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { log.Tracef("treeNew") pk, err := ptypes.MarshalAny(t.privateKey) @@ -138,31 +162,42 @@ func (t *tlogbe) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { return tree, ilr.Created, nil } -func (t *tlogbe) signedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoot, *types.LogRootV1, error) { - log.Tracef("signedLogRoot: %v", tree.TreeId) - - // Get latest signed root - resp, err := t.client.GetLatestSignedLogRoot(t.ctx, - &trillian.GetLatestSignedLogRootRequest{LogId: tree.TreeId}) +func (t *TrillianClient) treeFreeze(treeID int64) (*trillian.Tree, error) { + // Get the current tree + tree, err := t.tree(treeID) if err != nil { - return nil, nil, err + return nil, fmt.Errorf("tree: %v", err) } - // Verify root - verifier, err := client.NewLogVerifierFromTree(tree) + // Update the tree state + tree.TreeState = trillian.TreeState_FROZEN + + // Apply update + updated, err := t.admin.UpdateTree(t.ctx, &trillian.UpdateTreeRequest{ + Tree: tree, + UpdateMask: &field_mask.FieldMask{ + Paths: []string{"tree_state"}, + }, + }) if err != nil { - return nil, nil, err + return nil, fmt.Errorf("UpdateTree: %v", err) } - lrv1, err := tcrypto.VerifySignedLogRoot(verifier.PubKey, - crypto.SHA256, resp.SignedLogRoot) + + return updated, nil +} + +func (t *TrillianClient) treesAll() ([]*trillian.Tree, error) { + log.Tracef("treesAll") + + ltr, err := t.admin.ListTrees(t.ctx, &trillian.ListTreesRequest{}) if err != nil { - return nil, nil, err + return nil, err } - return resp.SignedLogRoot, lrv1, nil + return ltr.Tree, nil } -func (t *tlogbe) inclusionProof(treeID int64, merkleLeafHash []byte, lrv1 *types.LogRootV1) (*trillian.Proof, error) { +func (t *TrillianClient) inclusionProof(treeID int64, merkleLeafHash []byte, lrv1 *types.LogRootV1) (*trillian.Proof, error) { log.Tracef("inclusionProof: %v %x", treeID, merkleLeafHash) resp, err := t.client.GetInclusionProofByHash(t.ctx, @@ -194,29 +229,53 @@ func (t *tlogbe) inclusionProof(treeID int64, merkleLeafHash []byte, lrv1 *types return proof, nil } -func (t *tlogbe) leavesByHash(treeID int64, merkleLeafHashes [][]byte) ([]*trillian.LogLeaf, error) { - log.Tracef("leavesByHash: %v %x", treeID, merkleLeafHashes) +func (t *TrillianClient) signedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoot, *types.LogRootV1, error) { + // Get the signed log root for the current tree height + resp, err := t.client.GetLatestSignedLogRoot(t.ctx, + &trillian.GetLatestSignedLogRootRequest{LogId: tree.TreeId}) + if err != nil { + return nil, nil, err + } - res, err := t.client.GetLeavesByHash(t.ctx, - &trillian.GetLeavesByHashRequest{ - LogId: treeID, - LeafHash: merkleLeafHashes, - }) + // Verify the log root + verifier, err := client.NewLogVerifierFromTree(tree) if err != nil { - return nil, fmt.Errorf("GetLeavesByHashRequest: %v", err) + return nil, nil, err + } + lrv1, err := tcrypto.VerifySignedLogRoot(verifier.PubKey, + crypto.SHA256, resp.SignedLogRoot) + if err != nil { + return nil, nil, err } - return res.Leaves, nil + return resp.SignedLogRoot, lrv1, nil } -// leavesAppend appends the provided leaves onto the provided tree. The leaf +// SignedLogRoot returns the signed log root for the provided tree ID at its +// current height. The log root is structure is decoded an returned as well. +func (t *TrillianClient) SignedLogRoot(treeID int64) (*trillian.SignedLogRoot, *types.LogRootV1, error) { + log.Tracef("SignedLogRoot: %v", treeID) + + tree, err := t.tree(treeID) + if err != nil { + return nil, nil, fmt.Errorf("tree: %v", err) + } + slr, lr, err := t.signedLogRoot(tree) + if err != nil { + return nil, nil, fmt.Errorf("signedLogRoot: %v", err) + } + + return slr, lr, nil +} + +// LeavesAppend appends the provided leaves onto the provided tree. The leaf // and the inclusion proof for the leaf are returned. If a leaf was not // successfully appended, the leaf will be returned without an inclusion proof. // The error status code can be found in the returned leaf. Note leaves that // are duplicates will fail and it is the callers responsibility to determine // how they should be handled. -func (t *tlogbe) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, *types.LogRootV1, error) { - log.Tracef("leavesAppend: %v", treeID) +func (t *TrillianClient) LeavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]QueuedLeafProof, *types.LogRootV1, error) { + log.Tracef("LeavesAppend: %v", treeID) // Get the latest signed log root tree, err := t.tree(treeID) @@ -274,13 +333,10 @@ func (t *tlogbe) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queue return nil, nil, fmt.Errorf("signedLogRoot post update: %v", err) } - // The tree is now dirty - t.dirtyAdd(treeID, lrv1.TreeSize) - // Get inclusion proofs - proofs := make([]queuedLeafProof, 0, len(qlr.QueuedLeaves)) + proofs := make([]QueuedLeafProof, 0, len(qlr.QueuedLeaves)) for _, v := range qlr.QueuedLeaves { - qlp := queuedLeafProof{ + qlp := QueuedLeafProof{ QueuedLeaf: v, } @@ -316,21 +372,52 @@ func (t *tlogbe) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queue return proofs, lrv1, nil } -func (t *tlogbe) leafProofs(treeID int64, merkleLeafHashes [][]byte, lr *types.LogRootV1) ([]leafProof, error) { +// LeavesAll returns all of the leaves for the provided treeID. +func (t *TrillianClient) LeavesAll(treeID int64) ([]*trillian.LogLeaf, error) { + // Get log root + _, lr, err := t.SignedLogRoot(treeID) + if err != nil { + return nil, fmt.Errorf("SignedLogRoot: %v", err) + } + + // Get all leaves + glbrr, err := t.client.GetLeavesByRange(t.ctx, + &trillian.GetLeavesByRangeRequest{ + LogId: treeID, + StartIndex: 0, + Count: int64(lr.TreeSize), + }) + if err != nil { + return nil, fmt.Errorf("GetLeavesByRangeRequest: %v", err) + } + + return glbrr.Leaves, nil +} + +// LeafProofs returns the LeafProofs for the provided treeID and merkle leaf +// hashes. The inclusion proof returned in the LeafProof is for the tree height +// specified by the provided LogRootV1. +func (t *TrillianClient) LeafProofs(treeID int64, merkleLeafHashes [][]byte, lr *types.LogRootV1) ([]LeafProof, error) { + log.Tracef("LeafProofs: %v %v %x", treeID, lr.TreeSize, merkleLeafHashes) + // Retrieve leaves - leaves, err := t.leavesByHash(treeID, merkleLeafHashes) + r, err := t.client.GetLeavesByHash(t.ctx, + &trillian.GetLeavesByHashRequest{ + LogId: treeID, + LeafHash: merkleLeafHashes, + }) if err != nil { - return nil, err + return nil, fmt.Errorf("GetLeavesByHashRequest: %v", err) } // Retrieve proofs - proofs := make([]leafProof, 0, len(leaves)) - for _, v := range leaves { + proofs := make([]LeafProof, 0, len(r.Leaves)) + for _, v := range r.Leaves { p, err := t.inclusionProof(treeID, v.MerkleLeafHash, lr) if err != nil { return nil, fmt.Errorf("inclusionProof %x: %v", v.MerkleLeafHash, err) } - proofs = append(proofs, leafProof{ + proofs = append(proofs, LeafProof{ Leaf: v, Proof: p, }) @@ -338,3 +425,75 @@ func (t *tlogbe) leafProofs(treeID int64, merkleLeafHashes [][]byte, lr *types.L return proofs, nil } + +// Close closes the trillian grpc connection. +func (t *TrillianClient) Close() { + t.grpc.Close() +} + +func trillianClientNew(homeDir, host, keyFile string) (*TrillianClient, error) { + // Setup trillian key file + if keyFile == "" { + // No file path was given. Use the default path. + keyFile = filepath.Join(homeDir, defaultTrillianKeyFilename) + } + if !util.FileExists(keyFile) { + // Trillian key file does not exist. Create one. + log.Infof("Generating trillian private key") + if keyFile == "" { + } + key, err := keys.NewFromSpec(&keyspb.Specification{ + // TODO Params: &keyspb.Specification_Ed25519Params{}, + Params: &keyspb.Specification_EcdsaParams{}, + }) + if err != nil { + return nil, err + } + b, err := der.MarshalPrivateKey(key) + if err != nil { + return nil, err + } + err = ioutil.WriteFile(keyFile, b, 0400) + if err != nil { + return nil, err + } + log.Infof("Trillian private key created: %v", keyFile) + } + + // Setup trillian connection + log.Infof("Trillian host: %v", host) + g, err := grpc.Dial(host, grpc.WithInsecure()) + if err != nil { + return nil, fmt.Errorf("grpc dial: %v", err) + } + + // Load trillian key pair + var privateKey = &keyspb.PrivateKey{} + privateKey.Der, err = ioutil.ReadFile(keyFile) + if err != nil { + return nil, err + } + signer, err := der.UnmarshalPrivateKey(privateKey.Der) + if err != nil { + return nil, err + } + log.Infof("Trillian key loaded") + + t := TrillianClient{ + grpc: g, + client: trillian.NewTrillianLogClient(g), + admin: trillian.NewTrillianAdminClient(g), + ctx: context.Background(), + privateKey: privateKey, + publicKey: signer.Public(), + } + + // Ensure trillian is up and running + for t.grpc.GetState() != connectivity.Ready { + wait := 15 * time.Second + log.Infof("Cannot connect to trillian; retry in %v ", wait) + time.Sleep(wait) + } + + return &t, nil +} diff --git a/politeiad/cmd/politeia/README.md b/politeiad/cmd/politeia/README.md index f16737d6e..6fc972ac1 100644 --- a/politeiad/cmd/politeia/README.md +++ b/politeiad/cmd/politeia/README.md @@ -73,9 +73,12 @@ Update record: 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 ## Set unvetted status -You can update the status of an unvetted record using `publish` or `censor` -arguments. `publish` makes the record a public, vetted record. `censor` keeps -the record private. Note the token argument is not prefixed in this command. +You can update the status of an unvetted record using one of the following +statuses: +- `publish` - make the record a public, vetted record. +- `censor` - keep the record unvetted and mark as censored. + +Note `token:` is not prefixed to the token in this command. ``` $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass setunvettedstatus publish 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 diff --git a/politeiad/log.go b/politeiad/log.go index c6f0c152f..65d6f668d 100644 --- a/politeiad/log.go +++ b/politeiad/log.go @@ -11,6 +11,7 @@ import ( "github.com/decred/politeia/politeiad/backend/gitbe" "github.com/decred/politeia/politeiad/backend/tlogbe" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugin/comments" "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" "github.com/decred/politeia/politeiad/cache/cockroachdb" "github.com/decred/slog" @@ -47,8 +48,9 @@ var ( log = backendLog.Logger("POLI") gitbeLog = backendLog.Logger("GITB") tlogbeLog = backendLog.Logger("TLOG") - blobLog = backendLog.Logger("BLOB") cacheLog = backendLog.Logger("CACH") + blobLog = backendLog.Logger("BLOB") + pluginLog = backendLog.Logger("PLGN") ) // Initialize package-global logger variables. @@ -57,6 +59,7 @@ func init() { gitbe.UseLogger(gitbeLog) tlogbe.UseLogger(tlogbeLog) filesystem.UseLogger(blobLog) + comments.UseLogger(pluginLog) } // subsystemLoggers maps each subsystem identifier to its associated logger. @@ -64,8 +67,9 @@ var subsystemLoggers = map[string]slog.Logger{ "POLI": log, "GITB": gitbeLog, "TLOG": tlogbeLog, - "BLOB": blobLog, "CACH": cacheLog, + "BLOB": blobLog, + "PLGN": pluginLog, } // initLogRotator initializes the logging rotater to write logs to logFile and diff --git a/tlog/tserver/tserver.go b/tlog/tserver/tserver.go index d27b45acb..4fec06476 100644 --- a/tlog/tserver/tserver.go +++ b/tlog/tserver/tserver.go @@ -32,6 +32,7 @@ import ( "github.com/google/trillian/crypto/keys/der" "github.com/google/trillian/crypto/keyspb" "github.com/google/trillian/crypto/sigpb" + _ "github.com/google/trillian/merkle/rfc6962" "github.com/google/trillian/types" "github.com/gorilla/mux" "github.com/robfig/cron" @@ -502,10 +503,6 @@ func (t *tserver) appendRecord(tree *trillian.Tree, root *trillian.SignedLogRoot // Get inclusion proofs proofs := make([]v1.QueuedLeafProof, 0, len(qlr.QueuedLeaves)) for _, v := range qlr.QueuedLeaves { - fmt.Printf("%x\n", v.Leaf.LeafValue) - fmt.Printf("%x\n", v.Leaf.MerkleLeafHash) - fmt.Printf("%x\n", util.Digest(v.Leaf.LeafValue)) - qllp := v1.QueuedLeafProof{ QueuedLeaf: *v, } diff --git a/util/convert.go b/util/convert.go index 135739d8a..caab7e50f 100644 --- a/util/convert.go +++ b/util/convert.go @@ -9,7 +9,7 @@ import ( "encoding/hex" "fmt" - "github.com/decred/dcrtime/api/v1" + v1 "github.com/decred/dcrtime/api/v1" "github.com/decred/politeia/politeiad/api/v1/identity" ) @@ -32,6 +32,7 @@ func ConvertSignature(s string) ([identity.SignatureSize]byte, error) { // ConvertStringToken verifies and converts a string token to a proper sized // []byte. func ConvertStringToken(token string) ([]byte, error) { + // TODO add new token size /* if len(token) != pd.TokenSize*2 { return nil, fmt.Errorf("invalid censorship token size") diff --git a/util/dcrtime.go b/util/dcrtime.go index d07f10489..96966a90a 100644 --- a/util/dcrtime.go +++ b/util/dcrtime.go @@ -41,7 +41,7 @@ func (e ErrNotAnchored) Error() string { return e.err.Error() } -// isTimestamp determines if a string is a valid SHA256 digest. +// isDigest determines if a string is a valid SHA256 digest. func isDigest(digest string) bool { return v1.RegexpSHA256.MatchString(digest) } @@ -141,6 +141,12 @@ func Timestamp(id, host string, digests []*[sha256.Size]byte) error { // error is returned. If the reply is valid it is returned to the caller for // further processing. This means that the caller can be assured that all // checks have been done and the data is readily usable. +// +// Note the Result in the reply will be set to OK as soon as the digest is +// waiting to be anchored. The ChainInformation will be populated once the +// digest has been included in a dcr transaction, except for the ChainTimestamp +// field. The ChainTimestamp field is only populated once the dcr transaction +// has 6 confirmations. func Verify(id, host string, digests []string) (*v1.VerifyReply, error) { ver := v1.Verify{ ID: id, From bb8548f3724975c82fcddb472c6f4e23c5b7e29b Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 30 Jun 2020 17:46:10 -0600 Subject: [PATCH 005/449] add dual trillian setup --- plugins/comments/comments.go | 19 +- politeiad/api/v1/v1.go | 3 +- politeiad/backend/backend.go | 164 ++- politeiad/backend/tlogbe/anchor.go | 99 +- politeiad/backend/tlogbe/blobentry.go | 74 +- politeiad/backend/tlogbe/dcrtime.go | 8 +- politeiad/backend/tlogbe/index.go | 213 --- politeiad/backend/tlogbe/store/store.go | 9 +- politeiad/backend/tlogbe/tlog.go | 1216 +++++++++++++++- politeiad/backend/tlogbe/tlogbe.go | 1691 ++++++----------------- politeiad/backend/tlogbe/trillian.go | 45 +- politeiad/backend/util.go | 168 --- politeiad/politeiad.go | 2 +- util/convert.go | 1 + 14 files changed, 1804 insertions(+), 1908 deletions(-) delete mode 100644 politeiad/backend/tlogbe/index.go delete mode 100644 politeiad/backend/util.go diff --git a/plugins/comments/comments.go b/plugins/comments/comments.go index e12c441aa..fc4c1e48d 100644 --- a/plugins/comments/comments.go +++ b/plugins/comments/comments.go @@ -17,16 +17,15 @@ const ( ID = "comments" // Plugin commands - CmdNew = "new" // Create a new comment - CmdUnvetted = "unvetted" // Get unvetted comments - CmdVetted = "vetted" // Get vetted comments - CmdGet = "get" // Get a comment - CmdEdit = "edit" // Edit a comment - CmdDel = "del" // Delete a comment - CmdVote = "vote" // Vote on a comment - CmdCensor = "censor" // Censor a comment - CmdCount = "count" // Get comments count - CmdProofs = "proofs" // Get comment proofs + CmdNew = "new" // Create a new comment + CmdGet = "get" // Get comments + CmdEdit = "edit" // Edit a comment + CmdDel = "del" // Delete a comment + CmdExists = "exists" // Does a comment exist + CmdVote = "vote" // Vote on a comment + CmdCensor = "censor" // Censor a comment + CmdCount = "count" // Get comments count + CmdProofs = "proofs" // Get comment proofs // Error status codes ErrorStatusInvalid ErrorStatusT = 0 diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index 3517485e7..6675609a4 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -301,7 +301,8 @@ type SetVettedStatusReply struct { Response string `json:"response"` // Challenge response } -// UpdateRecord update an unvetted record. +// UpdateRecord updates a record. This is used for both unvetted and vetted +// records. type UpdateRecord struct { Challenge string `json:"challenge"` // Random challenge Token string `json:"token"` // Censorship token diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index 752804032..f6cff5b81 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -5,11 +5,18 @@ package backend import ( + "bytes" + "encoding/base64" "errors" "fmt" + "path/filepath" "regexp" + "strconv" v1 "github.com/decred/politeia/politeiad/api/v1" + "github.com/decred/politeia/politeiad/api/v1/mime" + "github.com/decred/politeia/util" + "github.com/subosito/gozaru" ) var ( @@ -108,7 +115,7 @@ type RecordMetadata struct { Status MDStatusT `json:"status"` // Current status of the record Merkle string `json:"merkle"` // Merkle root of all files in record Timestamp int64 `json:"timestamp"` // Last updated - Token string `json:"token"` // Record authentication token + Token string `json:"token"` // Record authentication token, hex encoded } // MetadataStream describes a single metada stream. The ID determines how and @@ -127,6 +134,161 @@ type Record struct { Files []File // User provided files } +// VerifyContent verifies that all provided MetadataStream and File are sane. +func VerifyContent(metadata []MetadataStream, files []File, filesDel []string) error { + // Make sure all metadata is within maxima. + for _, v := range metadata { + if v.ID > v1.MetadataStreamsMax-1 { + return ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidMDID, + ErrorContext: []string{ + strconv.FormatUint(v.ID, 10), + }, + } + } + } + for i := range metadata { + for j := range metadata { + // Skip self and non duplicates. + if i == j || metadata[i].ID != metadata[j].ID { + continue + } + return ContentVerificationError{ + ErrorCode: v1.ErrorStatusDuplicateMDID, + ErrorContext: []string{ + strconv.FormatUint(metadata[i].ID, 10), + }, + } + } + } + + // Prevent paths + for i := range files { + if filepath.Base(files[i].Name) != files[i].Name { + return ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidFilename, + ErrorContext: []string{ + files[i].Name, + }, + } + } + } + for _, v := range filesDel { + if filepath.Base(v) != v { + return ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidFilename, + ErrorContext: []string{ + v, + }, + } + } + } + + // Now check files + if len(files) == 0 { + return ContentVerificationError{ + ErrorCode: v1.ErrorStatusEmpty, + } + } + + // Prevent bad filenames and duplicate filenames + for i := range files { + for j := range files { + if i == j { + continue + } + if files[i].Name == files[j].Name { + return ContentVerificationError{ + ErrorCode: v1.ErrorStatusDuplicateFilename, + ErrorContext: []string{ + files[i].Name, + }, + } + } + } + // Check against filesDel + for _, v := range filesDel { + if files[i].Name == v { + return ContentVerificationError{ + ErrorCode: v1.ErrorStatusDuplicateFilename, + ErrorContext: []string{ + files[i].Name, + }, + } + } + } + } + + for i := range files { + if gozaru.Sanitize(files[i].Name) != files[i].Name { + return ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidFilename, + ErrorContext: []string{ + files[i].Name, + }, + } + } + + // Validate digest + d, ok := util.ConvertDigest(files[i].Digest) + if !ok { + return ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidFileDigest, + ErrorContext: []string{ + files[i].Name, + }, + } + } + + // Decode base64 payload + var err error + payload, err := base64.StdEncoding.DecodeString(files[i].Payload) + if err != nil { + return ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidBase64, + ErrorContext: []string{ + files[i].Name, + }, + } + } + + // Calculate payload digest + dp := util.Digest(payload) + if !bytes.Equal(d[:], dp) { + return ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidFileDigest, + ErrorContext: []string{ + files[i].Name, + }, + } + } + + // Verify MIME + detectedMIMEType := mime.DetectMimeType(payload) + if detectedMIMEType != files[i].MIME { + return ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidMIMEType, + ErrorContext: []string{ + files[i].Name, + detectedMIMEType, + }, + } + } + + if !mime.MimeValid(files[i].MIME) { + return ContentVerificationError{ + ErrorCode: v1.ErrorStatusUnsupportedMIMEType, + ErrorContext: []string{ + files[i].Name, + files[i].MIME, + }, + } + } + } + + return nil +} + // PluginSettings type PluginSetting struct { Key string // Name of setting diff --git a/politeiad/backend/tlogbe/anchor.go b/politeiad/backend/tlogbe/anchor.go index 2c7d275c1..9c7d3e12d 100644 --- a/politeiad/backend/tlogbe/anchor.go +++ b/politeiad/backend/tlogbe/anchor.go @@ -5,21 +5,14 @@ package tlogbe import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" - "time" - dcrtime "github.com/decred/dcrtime/api/v2" - "github.com/decred/politeia/util" "github.com/google/trillian/types" ) // TODO handle reorgs. A anchor record may become invalid in the case of a // reorg. We don't create the anchor record until the anchor tx has 6 -// confirmations so the probability of this occuring on mainnet is very low, -// but it should still be handled. +// confirmations so the probability of this occuring on mainnet is low, but it +// still needs to be handled. const ( // anchorSchedule determines how often we anchor records. dcrtime @@ -42,6 +35,7 @@ type anchor struct { VerifyDigest *dcrtime.VerifyDigest `json:"verifydigest"` } +/* func convertBlobEntryFromAnchor(a anchor) (*blobEntry, error) { data, err := json.Marshal(a) if err != nil { @@ -75,55 +69,55 @@ func (t *tlogbe) anchorSave(a anchor) error { return fmt.Errorf("verify digest not found") } - // Save the anchor record - be, err := convertBlobEntryFromAnchor(a) - if err != nil { - return err - } - b, err := blobify(*be) - if err != nil { - return err - } - lrb, err := a.LogRoot.MarshalBinary() - if err != nil { - return err - } - logRootHash := util.Hash(lrb)[:] - err = t.store.Put(keyAnchor(logRootHash), b) - if err != nil { - return fmt.Errorf("Put: %v", err) - } + // Save the anchor record + be, err := convertBlobEntryFromAnchor(a) + if err != nil { + return err + } + b, err := blobify(*be) + if err != nil { + return err + } + lrb, err := a.LogRoot.MarshalBinary() + if err != nil { + return err + } + logRootHash := util.Hash(lrb)[:] + err = t.store.Put(keyAnchor(logRootHash), b) + if err != nil { + return fmt.Errorf("Put: %v", err) + } - log.Debugf("Anchor saved for tree %v at height %v", - a.TreeID, a.LogRoot.TreeSize) + log.Debugf("Anchor saved for tree %v at height %v", + a.TreeID, a.LogRoot.TreeSize) - // Update the record history with the anchor. The lock must be held - // during this update. - t.Lock() - defer t.Unlock() + // Update the record history with the anchor. The lock must be held + // during this update. + t.Lock() + defer t.Unlock() - token := tokenFromTreeID(a.TreeID) - rh, err := t.recordHistory(token) - if err != nil { - return fmt.Errorf("recordHistory: %v", err) - } + token := tokenFromTreeID(a.TreeID) + rh, err := t.recordHistory(token) + if err != nil { + return fmt.Errorf("recordHistory: %v", err) + } - rh.Anchors[a.LogRoot.TreeSize] = logRootHash + rh.Anchors[a.LogRoot.TreeSize] = logRootHash - be, err = convertBlobEntryFromRecordHistory(*rh) - if err != nil { - return err - } - b, err = blobify(*be) - if err != nil { - return err - } - err = t.store.Put(keyRecordHistory(rh.Token), b) - if err != nil { - return fmt.Errorf("Put: %v", err) - } + be, err = convertBlobEntryFromRecordHistory(*rh) + if err != nil { + return err + } + b, err = blobify(*be) + if err != nil { + return err + } + err = t.store.Put(keyRecordHistory(rh.Token), b) + if err != nil { + return fmt.Errorf("Put: %v", err) + } - log.Debugf("Anchor added to record history %x", token) + log.Debugf("Anchor added to record history %x", token) return nil } @@ -385,3 +379,4 @@ func (t *tlogbe) anchorTrees() { go t.waitForAnchor(anchors, digests) } +*/ diff --git a/politeiad/backend/tlogbe/blobentry.go b/politeiad/backend/tlogbe/blobentry.go index bdb4f25c0..7fabc8699 100644 --- a/politeiad/backend/tlogbe/blobentry.go +++ b/politeiad/backend/tlogbe/blobentry.go @@ -15,25 +15,15 @@ import ( ) const ( + // blobEntryVersion is encoded in the sbox header of encrypted + // blobs. blobEntryVersion uint32 = 1 // Data descriptor types. These may be freely edited since they are // solely hints to the application. dataTypeStructure = "struct" // Descriptor contains a structure - // Data descriptors - dataDescriptorFile = "file" - dataDescriptorRecordMetadata = "recordmetadata" - dataDescriptorMetadataStream = "metadatastream" - dataDescriptorRecordHistory = "recordhistory" - dataDescriptorAnchor = "anchor" - - dataDescriptorRecordIndex = "recordindex" - - // Key prefixes for blob entries saved to the key-value store - keyPrefixRecordHistory = "index" - keyPrefixRecordContent = "record" - keyPrefixAnchor = "anchor" + dataDescriptorAnchor = "anchor" ) // dataDescriptor provides hints about a data blob. In practise we JSON encode @@ -52,27 +42,6 @@ type blobEntry struct { Data string `json:"data"` // Data payload, base64 encoded } -// keyRecordHistory returns the key for the blob key-value store for a record -// history. -func keyRecordHistory(token []byte) string { - return keyPrefixRecordHistory + hex.EncodeToString(token) -} - -// keyRecordContent returns the key-value store key for any type of record -// content (files, metadata streams, record metadata). Its possible for two -// different records to submit the same file resulting in identical merkle leaf -// hashes. The token is included in the key to ensure that a situation like -// this does not lead to unwanted behavior. -func keyRecordContent(token, merkleLeafHash []byte) string { - return keyPrefixRecordContent + hex.EncodeToString(token) + - hex.EncodeToString(merkleLeafHash) -} - -// keyAnchor returns the key for the blob key-value store for a anchor record. -func keyAnchor(logRootHash []byte) string { - return keyPrefixAnchor + hex.EncodeToString(logRootHash) -} - func blobify(be blobEntry) ([]byte, error) { var b bytes.Buffer zw := gzip.NewWriter(&b) @@ -102,43 +71,6 @@ func deblob(blob []byte) (*blobEntry, error) { return &be, nil } -func blobifyEncrypted(be blobEntry, key *EncryptionKey) ([]byte, error) { - var b bytes.Buffer - zw := gzip.NewWriter(&b) - enc := gob.NewEncoder(zw) - err := enc.Encode(be) - if err != nil { - return nil, err - } - err = zw.Close() // we must flush gzip buffers - if err != nil { - return nil, err - } - blob, err := key.Encrypt(blobEntryVersion, b.Bytes()) - if err != nil { - return nil, err - } - return blob, nil -} - -func deblobEncrypted(blob []byte, key *EncryptionKey) (*blobEntry, error) { - b, _, err := key.Decrypt(blob) - if err != nil { - return nil, err - } - zr, err := gzip.NewReader(bytes.NewReader(b)) - if err != nil { - return nil, err - } - r := gob.NewDecoder(zr) - var be blobEntry - err = r.Decode(&be) - if err != nil { - return nil, err - } - return &be, nil -} - func blobEntryNew(dataHint, data []byte) blobEntry { return blobEntry{ Hash: hex.EncodeToString(util.Digest(data)), diff --git a/politeiad/backend/tlogbe/dcrtime.go b/politeiad/backend/tlogbe/dcrtime.go index 2a8168bea..b13b902ea 100644 --- a/politeiad/backend/tlogbe/dcrtime.go +++ b/politeiad/backend/tlogbe/dcrtime.go @@ -87,10 +87,10 @@ func timestampBatch(host, id string, digests []string) (*dcrtime.TimestampBatchR // code and handling digests that failed to be timestamped. // // Note the Result in the reply will be set to OK as soon as the digest is -// waiting to be anchored. The ChainInformation will be populated once the -// digest has been included in a dcr transaction, except for the ChainTimestamp -// field. The ChainTimestamp field is only populated once the dcr transaction -// has 6 confirmations. +// waiting to be anchored. All the ChainInformation fields will be populated +// once the digest has been included in a dcr transaction, except for the +// ChainTimestamp field. The ChainTimestamp field is only populated once the +// dcr transaction has 6 confirmations. func verifyBatch(host, id string, digests []string) (*dcrtime.VerifyBatchReply, error) { log.Tracef("verifyBatch: %v %v %v", host, id, digests) diff --git a/politeiad/backend/tlogbe/index.go b/politeiad/backend/tlogbe/index.go deleted file mode 100644 index 25bb1d06b..000000000 --- a/politeiad/backend/tlogbe/index.go +++ /dev/null @@ -1,213 +0,0 @@ -// Copyright (c) 2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package tlogbe - -import ( - "encoding/base64" - "encoding/hex" - "encoding/json" - "fmt" - - "github.com/decred/politeia/politeiad/backend" -) - -const ( - // Record states - stateUnvetted = "unvetted" - stateVetted = "vetted" -) - -var ( - // recordStateFromStatus maps the backend record statuses to one - // of the record states. - recordStateFromStatus = map[backend.MDStatusT]string{ - backend.MDStatusUnvetted: stateUnvetted, - backend.MDStatusIterationUnvetted: stateUnvetted, - backend.MDStatusCensored: stateUnvetted, - backend.MDStatusVetted: stateVetted, - backend.MDStatusArchived: stateVetted, - } -) - -// recordIndex represents an index for a backend Record that contains the -// merkle leaf hash for each piece of record content. The merkle leaf hash -// refers to the trillian LogLeaf.MerkleLeafHash that was returned when the -// record content was appened onto to the trillian tree. Each piece of record -// content is appended as a seperate leaf onto the trillian tree and thus has -// a unique merkle leaf hash. This value can be used to lookup the inclusion -// proof from the trillian tree as well as the actual content from the blob -// key-value store. -type recordIndex struct { - RecordMetadata []byte `json:"recordmetadata"` - Metadata map[uint64][]byte `json:"metadata"` // [metadataID]merkle - Files map[string][]byte `json:"files"` // [filename]merkle -} - -// recordHistory contains the record index for all versions of a record and -// the anchor data for all record content. -type recordHistory struct { - Token []byte `json:"token"` // Record token - State string `json:"state"` // Unvetted or vetted - Versions map[uint32]recordIndex `json:"versions"` // [version]recordIndex - - // Anchors contains the digests of all log roots that have been - // anchored for this record. The leaf height of any individual - // piece of record content can be used to look up the earliest - // anchor that the leaf was included in. The log root digest can - // be used to lookup the anchor record. The log root digest is a - // SHA256 digest of the encoded LogRootV1 structure. - Anchors map[uint64][]byte `json:"anchors"` // [treeHeight]logRootDigest -} - -// String returns the recordHistory printed in a human readable format. -func (r *recordHistory) String() string { - s := fmt.Sprintf("Token: %x\n", r.Token) - s += fmt.Sprintf("State: %v\n", r.State) - for k, v := range r.Versions { - s += fmt.Sprintf("Version %v\n", k) - s += fmt.Sprintf(" RecordMD : %x\n", v.RecordMetadata) - for id, merkle := range v.Metadata { - s += fmt.Sprintf(" Metadata %2v: %x\n", id, merkle) - } - for fn, merkle := range v.Files { - s += fmt.Sprintf(" %-11v: %x\n", fn, merkle) - } - } - return s -} - -// recordIndexUpdate updates the provided recordIndex with the provided -// blobEntries and orphaned blobs then returns the updated recordIndex. -func recordIndexUpdate(idx recordIndex, entries []blobEntry, merkles map[string][]byte, orphaned [][]byte) (*recordIndex, error) { - // Create a record index using the new blob entires - idxNew, err := recordIndexNew(entries, merkles) - if err != nil { - return nil, err - } - - // Add existing record index content to the newly created index. - // If a merkle leaf hash is included in the orphaned list it means - // that it is no longer part of the record and should not be - // included. Orphaned merkle leaf hashes are put in a map for - // linear time lookups. - skip := make(map[string]struct{}, len(orphaned)) - for _, v := range orphaned { - skip[hex.EncodeToString(v)] = struct{}{} - } - if _, ok := skip[hex.EncodeToString(idx.RecordMetadata)]; !ok { - idxNew.RecordMetadata = idx.RecordMetadata - } - for k, v := range idx.Metadata { - if _, ok := skip[hex.EncodeToString(v)]; ok { - continue - } - idxNew.Metadata[k] = v - } - for k, v := range idx.Files { - if _, ok := skip[hex.EncodeToString(v)]; ok { - continue - } - idxNew.Files[k] = v - } - - return idxNew, nil -} - -func recordIndexNew(entries []blobEntry, merkles map[string][]byte) (*recordIndex, error) { - // The merkle leaf hash is used to lookup a blob entry in both the - // trillian tree and the blob key-value store. Create a record - // index by associating each piece of record content with its - // merkle root hash. - var ( - recordMD []byte - metadata = make(map[uint64][]byte, len(entries)) // [mdstreamID]merkleLeafHash - files = make(map[string][]byte, len(entries)) // [filename]merkleLeafHash - ) - for _, v := range entries { - b, err := base64.StdEncoding.DecodeString(v.DataHint) - if err != nil { - return nil, err - } - var dd dataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, err - } - switch dd.Descriptor { - case dataDescriptorRecordMetadata: - merkle, ok := merkles[v.Hash] - if !ok { - return nil, fmt.Errorf("merkle not found for record metadata") - } - recordMD = merkle - log.Debugf("Record metadata: %x", merkle) - case dataDescriptorMetadataStream: - b, err := base64.StdEncoding.DecodeString(v.Data) - if err != nil { - return nil, err - } - var ms backend.MetadataStream - err = json.Unmarshal(b, &ms) - if err != nil { - return nil, err - } - merkle, ok := merkles[v.Hash] - if !ok { - return nil, fmt.Errorf("merkle not found for mdstream %v", ms.ID) - } - metadata[ms.ID] = merkle - log.Debugf("Metadata %v: %x", ms.ID, merkle) - case dataDescriptorFile: - b, err := base64.StdEncoding.DecodeString(v.Data) - if err != nil { - return nil, err - } - var f backend.File - err = json.Unmarshal(b, &f) - if err != nil { - return nil, err - } - merkle, ok := merkles[v.Hash] - if !ok { - return nil, fmt.Errorf("merkle not found for file %v", f.Name) - } - files[f.Name] = merkle - log.Debugf("%v: %x", f.Name, merkle) - } - } - - return &recordIndex{ - RecordMetadata: recordMD, - Metadata: metadata, - Files: files, - }, nil -} - -// latestVersion returns the most recent version that exists. The versions -// start at 1 so the latest version is the same as the length. -func latestVersion(rh recordHistory) uint32 { - return uint32(len(rh.Versions)) -} - -func (t *tlogbe) recordHistory(token []byte) (*recordHistory, error) { - b, err := t.store.Get(keyRecordHistory(token)) - if err != nil { - return nil, err - } - be, err := deblob(b) - if err != nil { - return nil, err - } - return convertRecordHistoryFromBlobEntry(*be) -} - -func recordHistoryNew(token []byte) recordHistory { - return recordHistory{ - Token: token, - State: stateUnvetted, - Versions: make(map[uint32]recordIndex), - Anchors: make(map[uint64][]byte), - } -} diff --git a/politeiad/backend/tlogbe/store/store.go b/politeiad/backend/tlogbe/store/store.go index ced8e02e8..18ad95128 100644 --- a/politeiad/backend/tlogbe/store/store.go +++ b/politeiad/backend/tlogbe/store/store.go @@ -16,6 +16,7 @@ type Ops struct { Del []string } +// TODO get rid of this Blob implementation type Blob interface { Get(string) ([]byte, error) Put(string, []byte) error @@ -26,10 +27,10 @@ type Blob interface { // Blob represents a blob key-value store. type Blob_ interface { - // Get returns blobs from the store. An entry will not exist in the - // returned map if for any blobs that are not found. It is the - // responsibility of the caller to ensure a blob was returned for - // all provided keys. + // Get returns blobs from the store for the provided keys. An entry + // will not exist in the returned map if for any blobs that are not + // found. It is the responsibility of the caller to ensure a blob + // was returned for all provided keys. Get(keys []string) (map[string][]byte, error) // Put saves the provided blobs to the store. The keys for the diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index 28aecaffd..c00c09141 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -10,32 +10,100 @@ import ( "encoding/binary" "encoding/hex" "encoding/json" + "errors" "fmt" + "strconv" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/store" + "github.com/decred/politeia/util" + "github.com/google/trillian" + "github.com/robfig/cron" + "google.golang.org/grpc/codes" +) + +const ( + // Blob entry data descriptors + dataDescriptorFile = "file" + dataDescriptorRecordMetadata = "recordmetadata" + dataDescriptorMetadataStream = "metadatastream" + dataDescriptorRecordIndex = "recordindex" + dataDescriptorFreezeRecord = "freezerecord" + + // The keys for kv store blobs are saved by stuffing them into the + // ExtraData field of their corresponding trillian log leaf. The + // keys are prefixed with one of the follwing identifiers before + // being added to the log leaf so that we can correlate the leaf + // to the type of data it represents without having to pull the + // data out of the store, which can become an issue in situations + // such as searching for a record index that has been buried by + // thousands of leaves from plugin data. + keyPrefixRecordIndex = "index:" + keyPrefixRecordContent = "record:" + keyPrefixFreezeRecord = "freeze:" ) var ( - prefixRecordIndex = []byte("index:") + // errRecordNotFound is emitted when a record is not found. + errRecordNotFound = errors.New("record not found") + + errFreezeRecordNotFound = errors.New("freeze record not found") + + // errNoFileChanges is emitted when a new version of a record is + // attemtepd to be saved with no changes to the files. + errNoFileChanges = errors.New("no file changes") + + // errNoMetadataChanges is emitted when there are no metadata + // changes being made on a metadata update. + errNoMetadataChanges = errors.New("no metadata changes") ) // We do not unwind. type tlog struct { dataDir string - client *TrillianClient encryptionKey *EncryptionKey + client *TrillianClient store store.Blob_ + + // TODO implement anchoring + dcrtimeHost string + cron *cron.Cron + + // droppingAnchor indicates whether tlogbe is in the process of + // dropping an anchor, i.e. timestamping unanchored trillian trees + // using dcrtime. An anchor is dropped periodically using cron. + droppingAnchor bool } -type versionIndex struct { +type recordIndex struct { + // Version represents the version of the record. The version is + // only incremented when the record files are updated. + Version uint32 `json:"version"` + + // Iteration represents the iteration of the record. The iteration + // is incremented anytime any record content changes. This includes + // file changes that bump the version as well metadata stream and + // record metadata changes that don't bump the version. + // + // Note this is not the same as the RecordMetadata iteration, which + // does not get incremented on metadata stream updates. + Iteration uint32 `json:"iteration"` + + // The following fields contain the merkle leaf hashes of the + // trillian log leaves for the record content. The merkle leaf hash + // can be used to lookup the log leaf. The log leaf ExtraData field + // contains the key for the record content in the key-value store. RecordMetadata []byte `json:"recordmetadata"` - Metadata map[uint64][]byte `json:"metadata"` // [metadataID]leafHash - Files map[string][]byte `json:"files"` // [filename]leafHash + Metadata map[uint64][]byte `json:"metadata"` // [metadataID]merkle + Files map[string][]byte `json:"files"` // [filename]merkle } -type recordIndex_ struct { - Versions map[uint32]versionIndex `json:"versions"` +type freezeRecord struct { + TreeID int64 `json:"treeid,omitempty"` +} + +func treeIDFromToken(token []byte) int64 { + return int64(binary.LittleEndian.Uint64(token)) } func tokenFromTreeID(treeID int64) []byte { @@ -46,12 +114,169 @@ func tokenFromTreeID(treeID int64) []byte { return b } -func treeIDFromToken(token []byte) int64 { - return int64(binary.LittleEndian.Uint64(token)) +// isEncrypted returns whether the provided blob has been prefixed with an +// sbox header, indicating that it is an encrypted blob. +func isEncrypted(b []byte) bool { + return bytes.HasPrefix(b, []byte("sbox")) +} + +func isRecordIndexLeaf(l *trillian.LogLeaf) bool { + return bytes.HasPrefix(l.ExtraData, []byte(keyPrefixRecordIndex)) +} + +func isRecordContentLeaf(l *trillian.LogLeaf) bool { + return bytes.HasPrefix(l.ExtraData, []byte(keyPrefixRecordContent)) +} + +func isFreezeRecordLeaf(l *trillian.LogLeaf) bool { + return bytes.HasPrefix(l.ExtraData, []byte(keyPrefixFreezeRecord)) +} + +func extractKeyForRecordIndex(l *trillian.LogLeaf) (string, error) { + s := bytes.SplitAfter(l.ExtraData, []byte(keyPrefixRecordIndex)) + if len(s) != 2 { + return "", fmt.Errorf("invalid key %s", l.ExtraData) + } + return string(s[1]), nil +} + +func extractKeyForRecordContent(l *trillian.LogLeaf) (string, error) { + s := bytes.SplitAfter(l.ExtraData, []byte(keyPrefixRecordContent)) + if len(s) != 2 { + return "", fmt.Errorf("invalid key %s", l.ExtraData) + } + return string(s[1]), nil +} + +func extractKeyForFreezeRecord(l *trillian.LogLeaf) (string, error) { + s := bytes.SplitAfter(l.ExtraData, []byte(keyPrefixFreezeRecord)) + if len(s) != 2 { + return "", fmt.Errorf("invalid key %s", l.ExtraData) + } + return string(s[1]), nil +} + +func convertBlobEntryFromFile(f backend.File) (*blobEntry, error) { + data, err := json.Marshal(f) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + dataDescriptor{ + Type: dataTypeStructure, + Descriptor: dataDescriptorFile, + }) + if err != nil { + return nil, err + } + be := blobEntryNew(hint, data) + return &be, nil +} + +func convertBlobEntryFromMetadataStream(ms backend.MetadataStream) (*blobEntry, error) { + data, err := json.Marshal(ms) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + dataDescriptor{ + Type: dataTypeStructure, + Descriptor: dataDescriptorMetadataStream, + }) + if err != nil { + return nil, err + } + be := blobEntryNew(hint, data) + return &be, nil } -func tokenString(token []byte) string { - return hex.EncodeToString(token) +func convertBlobEntryFromRecordMetadata(rm backend.RecordMetadata) (*blobEntry, error) { + data, err := json.Marshal(rm) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + dataDescriptor{ + Type: dataTypeStructure, + Descriptor: dataDescriptorRecordMetadata, + }) + if err != nil { + return nil, err + } + be := blobEntryNew(hint, data) + return &be, nil +} + +func convertBlobEntryFromRecordIndex(ri recordIndex) (*blobEntry, error) { + data, err := json.Marshal(ri) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + dataDescriptor{ + Type: dataTypeStructure, + Descriptor: dataDescriptorRecordIndex, + }) + if err != nil { + return nil, err + } + be := blobEntryNew(hint, data) + return &be, nil +} + +func convertBlobEntryFromFreezeRecord(fr freezeRecord) (*blobEntry, error) { + data, err := json.Marshal(fr) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + dataDescriptor{ + Type: dataTypeStructure, + Descriptor: dataDescriptorFreezeRecord, + }) + if err != nil { + return nil, err + } + be := blobEntryNew(hint, data) + return &be, nil +} + +func convertFreezeRecordFromBlobEntry(be blobEntry) (*freezeRecord, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd dataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorFreezeRecord { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorFreezeRecord) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + hash, err := hex.DecodeString(be.Hash) + if err != nil { + return nil, fmt.Errorf("decode hash: %v", err) + } + if !bytes.Equal(util.Digest(b), hash) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), hash) + } + var fr freezeRecord + err = json.Unmarshal(b, &fr) + if err != nil { + return nil, fmt.Errorf("unmarshal freezeRecord: %v", err) + } + + return &fr, nil } func convertRecordIndexFromBlobEntry(be blobEntry) (*recordIndex, error) { @@ -75,7 +300,15 @@ func convertRecordIndexFromBlobEntry(be blobEntry) (*recordIndex, error) { if err != nil { return nil, fmt.Errorf("decode Data: %v", err) } - var ri recordIndex_ + hash, err := hex.DecodeString(be.Hash) + if err != nil { + return nil, fmt.Errorf("decode hash: %v", err) + } + if !bytes.Equal(util.Digest(b), hash) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), hash) + } + var ri recordIndex err = json.Unmarshal(b, &ri) if err != nil { return nil, fmt.Errorf("unmarshal recordIndex: %v", err) @@ -84,104 +317,949 @@ func convertRecordIndexFromBlobEntry(be blobEntry) (*recordIndex, error) { return &ri, nil } -func (t *tlog) tokenNew() ([]byte, error) { - tree, _, err := t.client.treeNew() +func (t *tlog) leafLatest(treeID int64) (*trillian.LogLeaf, error) { + + return nil, nil +} + +func (t *tlog) freezeRecord(treeID int64) (*freezeRecord, error) { + // Get the last leaf of the tree + tree, err := t.client.tree(treeID) if err != nil { - return nil, fmt.Errorf("treeNew: %v", err) + return nil, errRecordNotFound + } + _, lr, err := t.client.signedLogRoot(tree) + if err != nil { + return nil, fmt.Errorf("signedLogRoot: %v", err) + } + leaves, err := t.client.leavesByRange(treeID, int64(lr.TreeSize)-1, 1) + if err != nil { + return nil, fmt.Errorf("leavesByRange: %v", err) + } + if len(leaves) != 1 { + return nil, fmt.Errorf("unexpected leaves count: got %v, want 1", + len(leaves)) + } + l := leaves[0] + if !isFreezeRecordLeaf(l) { + // Leaf is not a freeze record + return nil, errFreezeRecordNotFound } - return tokenFromTreeID(tree.TreeId), nil + + // The leaf is a freeze record. Get it from the store. + k, err := extractKeyForFreezeRecord(l) + if err != nil { + return nil, err + } + blobs, err := t.store.Get([]string{k}) + if err != nil { + return nil, fmt.Errorf("store Get: %v", err) + } + if len(blobs) != 1 { + return nil, fmt.Errorf("unexpected blobs count: got %v, want 1", + len(blobs), 1) + } + + // Decode freeze record + b, ok := blobs[k] + if !ok { + return nil, fmt.Errorf("blob not found %v", k) + } + be, err := deblob(b) + if err != nil { + return nil, err + } + fr, err := convertFreezeRecordFromBlobEntry(*be) + if err != nil { + return nil, err + } + + return fr, nil } -func (t *tlog) recordExists() {} +func (t *tlog) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]*trillian.LogLeaf, error) { + // TODO Ensure the tree is not frozen + return nil, nil +} -func (t *tlog) recordGetVersion(token []byte, version uint64) (*backend.Record, error) { - // Get tree leaves - treeID := treeIDFromToken(token) - leaves, err := t.client.LeavesAll(treeID) +func (t *tlog) recordIndex(leaves []*trillian.LogLeaf, version uint32) (*recordIndex, error) { + // Walk the leaves and compile the keys of all the record indexes. + // Appending the record index leaf to the trillian tree is the last + // operation that occurs when updating a record, so if an index + // leaf exists then you can be sure that the index blob exists in + // the store as well as all of the record content blobs. It is + // possible for multiple indexes to exist for the same record + // version (they will have different iterations due to metadata + // only updates) so we have to pull the index blobs from the store + // in order to find the latest index for the specified version. + keys := make([]string, 0, 64) + for _, v := range leaves { + if isRecordIndexLeaf(v) { + // This is a record index leaf. Extract they kv store key. + k, err := extractKeyForRecordIndex(v) + if err != nil { + return nil, err + } + keys = append(keys, k) + } + } + + if len(keys) == 0 { + // No records have been added to this tree yet + return nil, errRecordNotFound + } + + // Get record indexes from store + blobs, err := t.store.Get(keys) if err != nil { - return nil, fmt.Errorf("LeavesAll %v: %v", treeID, err) + return nil, fmt.Errorf("store Get: %v", err) + } + missing := make([]string, 0, len(keys)) + for _, v := range keys { + if _, ok := blobs[v]; !ok { + missing = append(missing, v) + } + } + if len(missing) > 0 { + return nil, fmt.Errorf("record index not found: %v", missing) + } + + indexes := make([]*recordIndex, 0, len(blobs)) + for _, v := range blobs { + be, err := deblob(v) + if err != nil { + return nil, err + } + ri, err := convertRecordIndexFromBlobEntry(*be) + if err != nil { + return nil, err + } + indexes = append(indexes, ri) } - // Walk the leaves backwards to find the most recent record index. - var indexKey string - prefix := []byte(prefixRecordIndex) - for i := len(leaves) - 1; i >= 0; i-- { - l := leaves[i] - if bytes.HasPrefix(l.ExtraData, prefix) { - // Record index found. Extract key. - s := bytes.SplitAfter(l.ExtraData, prefix) - if len(s) != 2 { - return nil, fmt.Errorf("invalid leaf extra data: %x %s", - l.MerkleLeafHash, l.ExtraData) + // Sanity check. Index iterations should start with 1 and be + // sequential. Index versions should start with 1 and also be + // sequential, but duplicate versions can exist as long as the + // iteration has been incremented. + var versionPrev uint32 + var i uint32 = 1 + for _, v := range indexes { + if v.Iteration != i { + return nil, fmt.Errorf("invalid record index iteration: "+ + "got %v, want %v", v.Iteration, i) + } + diff := v.Version - versionPrev + if diff != 0 && diff != 1 { + return nil, fmt.Errorf("invalid record index version: "+ + "curr version %v, prev version %v", v.Version, versionPrev) + } + + i++ + versionPrev = v.Version + } + + // Return the record index for the specified version. A version of + // 0 indicates that the most recent version should be returned. + var ri *recordIndex + if version == 0 { + ri = indexes[len(indexes)-1] + } else { + // Walk the indexes backwards so the most recent iteration of the + // specified version is selected. + for i := len(indexes) - 1; i >= 0; i-- { + r := indexes[i] + if r.Version == version { + ri = r + break } - indexKey = string(s[1]) } } - if indexKey == "" { - return nil, fmt.Errorf("record index not found") + if ri == nil { + // The specified version does not exist + return nil, errRecordNotFound } - // Get the record index from the store - blobs, err := t.store.Get([]string{indexKey}) + return ri, nil +} + +func (t *tlog) recordIndexLatest(leaves []*trillian.LogLeaf) (*recordIndex, error) { + return t.recordIndex(leaves, 0) +} + +func (t *tlog) recordIndexSave(treeID int64, ri recordIndex) error { + // Save record index to the store + be, err := convertBlobEntryFromRecordIndex(ri) if err != nil { - return nil, fmt.Errorf("store Get %x: %v", err) + return err } - b, ok := blobs[indexKey] - if !ok { - return nil, fmt.Errorf("record index not found: %v", indexKey) + b, err := blobify(*be) + if err != nil { + return err } - be, err := deblob(b) + keys, err := t.store.Put([][]byte{b}) + if err != nil { + return fmt.Errorf("store Put: %v", err) + } + if len(keys) != 1 { + return fmt.Errorf("wrong number of keys: got %v, want 1", + len(keys)) + } + + // Append record index leaf to trillian tree + h, err := hex.DecodeString(be.Hash) + if err != nil { + return err + } + prefixedKey := []byte(keyPrefixRecordIndex + keys[0]) + queued, _, err := t.client.LeavesAppend(treeID, []*trillian.LogLeaf{ + logLeafNew(h, prefixedKey), + }) + if len(queued) != 1 { + return fmt.Errorf("wrong number of queud leaves: got %v, want 1", + len(queued)) + } + failed := make([]string, 0, len(queued)) + for _, v := range queued { + c := codes.Code(v.QueuedLeaf.GetStatus().GetCode()) + if c != codes.OK { + failed = append(failed, fmt.Sprintf("%v", c)) + } + } + if len(failed) > 0 { + return fmt.Errorf("append leaves failed: %v", failed) + } + + return nil +} + +type recordHashes struct { + recordMetadata string // Record metadata hash + metadata map[string]uint64 // [hash]metadataID + files map[string]string // [hash]filename +} + +type blobsPrepareArgs struct { + encryptionKey *EncryptionKey + leaves []*trillian.LogLeaf + recordMD backend.RecordMetadata + metadata []backend.MetadataStream + files []backend.File +} + +type blobsPrepareReply struct { + recordIndex recordIndex + recordHashes recordHashes + + // blobs and hashes MUST share the same ordering + blobs [][]byte + hashes [][]byte +} + +// TODO test this function +// TODO if we find a freeze record we need to fail +func blobsPrepare(args blobsPrepareArgs) (*blobsPrepareReply, error) { + // Check if any of the content already exists. Different record + // versions that reference the same data is fine, but this data + // should not be saved to the store again. We can find duplicates + // by walking the trillian tree and comparing the hash of the + // provided record content to the log leaf data, which will be the + // same for duplicates. + + // Compute record content hashes + rhashes := recordHashes{ + metadata: make(map[string]uint64, len(args.metadata)), + files: make(map[string]string, len(args.files)), + } + b, err := json.Marshal(args.recordMD) + if err != nil { + return nil, err + } + rhashes.recordMetadata = hex.EncodeToString(util.Digest(b)) + for _, v := range args.metadata { + b, err := json.Marshal(v) + if err != nil { + return nil, err + } + h := hex.EncodeToString(util.Digest(b)) + rhashes.metadata[h] = v.ID + } + for _, v := range args.files { + b, err := json.Marshal(v) + if err != nil { + return nil, err + } + h := hex.EncodeToString(util.Digest(b)) + rhashes.files[h] = v.Name + } + + // Compare leaf data to record content hashes to find duplicates + var ( + // Dups tracks duplicates so we know which blobs should be + // skipped when saving the blob to the store. + dups = make(map[string]struct{}, 64) + + // Any duplicates that are found are added to the record index + // since we already have the leaf data for them. + index = recordIndex{ + Metadata: make(map[uint64][]byte, len(args.metadata)), + Files: make(map[string][]byte, len(args.files)), + } + ) + for _, v := range args.leaves { + h := hex.EncodeToString(v.LeafValue) + + // Check record metadata + if h == rhashes.recordMetadata { + dups[h] = struct{}{} + index.RecordMetadata = v.MerkleLeafHash + continue + } + + // Check metadata streams + id, ok := rhashes.metadata[h] + if ok { + dups[h] = struct{}{} + index.Metadata[id] = v.MerkleLeafHash + continue + } + + // Check files + fn, ok := rhashes.files[h] + if ok { + dups[h] = struct{}{} + index.Files[fn] = v.MerkleLeafHash + continue + } + } + + // Prepare kv store blobs. The hashes of the record content are + // also aggregated and will be used to create the log leaves that + // are appended to the trillian tree. + l := len(args.metadata) + len(args.files) + 1 + hashes := make([][]byte, 0, l) + blobs := make([][]byte, 0, l) + be, err := convertBlobEntryFromRecordMetadata(args.recordMD) if err != nil { return nil, err } - r, err := convertRecordIndexFromBlobEntry(*be) + h, err := hex.DecodeString(be.Hash) if err != nil { return nil, err } - if len(r.Versions) == 0 { - return nil, fmt.Errorf("version indexes not found") + b, err = blobify(*be) + if err != nil { + return nil, err + } + _, ok := dups[be.Hash] + if !ok { + // Not a duplicate. Save blob to the store. + hashes = append(hashes, h) + blobs = append(blobs, b) } - // Get the record content from the store - metadata := make([]backend.MetadataStream, 0, len(idx.Metadata)) - files := make([]backend.File, 0, len(idx.Files)) + for _, v := range args.metadata { + be, err := convertBlobEntryFromMetadataStream(v) + if err != nil { + return nil, err + } + h, err := hex.DecodeString(be.Hash) + if err != nil { + return nil, err + } + b, err := blobify(*be) + if err != nil { + return nil, err + } + _, ok := dups[be.Hash] + if !ok { + // Not a duplicate. Save blob to the store. + hashes = append(hashes, h) + blobs = append(blobs, b) + } + } - return nil, nil + for _, v := range args.files { + be, err := convertBlobEntryFromFile(v) + if err != nil { + return nil, err + } + h, err := hex.DecodeString(be.Hash) + if err != nil { + return nil, err + } + b, err := blobify(*be) + if err != nil { + return nil, err + } + // Encypt file blobs if encryption key has been set + if args.encryptionKey != nil { + b, err = args.encryptionKey.Encrypt(blobEntryVersion, b) + if err != nil { + return nil, err + } + } + _, ok := dups[be.Hash] + if !ok { + // Not a duplicate. Save blob to the store. + hashes = append(hashes, h) + blobs = append(blobs, b) + } + } + + return &blobsPrepareReply{ + recordIndex: index, + recordHashes: rhashes, + blobs: blobs, + hashes: hashes, + }, nil } -func (t *tlog) recordGet(token []byte) (*backend.Record, error) { - return nil, nil +func (t *tlog) blobsSave(treeID int64, bpr blobsPrepareReply) (*recordIndex, error) { + var ( + index = bpr.recordIndex + rhashes = bpr.recordHashes + blobs = bpr.blobs + hashes = bpr.hashes + ) + + // Save blobs to store + keys, err := t.store.Put(blobs) + if err != nil { + return nil, fmt.Errorf("store Put: %v", err) + } + if len(keys) != len(blobs) { + return nil, fmt.Errorf("wrong number of keys: got %v, want %v", + len(keys), len(blobs)) + } + + // Prepare log leaves. hashes and keys share the same ordering. + leaves := make([]*trillian.LogLeaf, 0, len(blobs)) + for k := range blobs { + pk := []byte(keyPrefixRecordContent + keys[k]) + leaves = append(leaves, logLeafNew(hashes[k], pk)) + } + + // Append leaves to trillian tree + queued, _, err := t.client.LeavesAppend(treeID, leaves) + if err != nil { + return nil, fmt.Errorf("LeavesAppend: %v", err) + } + if len(queued) != len(leaves) { + return nil, fmt.Errorf("wrong number of queued leaves: got %v, want %v", + len(queued), len(leaves)) + } + failed := make([]string, 0, len(queued)) + for _, v := range queued { + c := codes.Code(v.QueuedLeaf.GetStatus().GetCode()) + if c != codes.OK { + failed = append(failed, fmt.Sprintf("%v", c)) + } + } + if len(failed) > 0 { + return nil, fmt.Errorf("append leaves failed: %v", failed) + } + + // Update the new record index with the log leaves + for _, v := range queued { + // Figure out what piece of record content this leaf represents + h := hex.EncodeToString(v.QueuedLeaf.Leaf.LeafValue) + + // Check record metadata + if h == rhashes.recordMetadata { + index.RecordMetadata = v.QueuedLeaf.Leaf.MerkleLeafHash + continue + } + + // Check metadata streams + id, ok := rhashes.metadata[h] + if ok { + index.Metadata[id] = v.QueuedLeaf.Leaf.MerkleLeafHash + continue + } + + // Check files + fn, ok := rhashes.files[h] + if ok { + index.Files[fn] = v.QueuedLeaf.Leaf.MerkleLeafHash + continue + } + + // Something went wrong. None of the record content matches the + // leaf. + return nil, fmt.Errorf("record content does not match leaf: %x", + v.QueuedLeaf.Leaf.MerkleLeafHash) + } + + return &index, nil +} + +func (t *tlog) treeNew() (int64, error) { + tree, _, err := t.client.treeNew() + if err != nil { + return 0, err + } + return tree.TreeId, nil } -func (t *tlog) recordSave(token []byte, metadata []backend.MetadataStream, files []backend.File, rm backend.RecordMetadata) (*backend.Record, error) { - // Validate changes +func (t *tlog) treeExists(treeID int64) bool { + _, err := t.client.tree(treeID) + if err == nil { + return true + } + return false +} + +func (t *tlog) recordIsFrozen(treeID int64) (bool, error) { + tree, err := t.client.tree(treeID) + if err != nil { + return false, fmt.Errorf("tree: %v", err) + } + if tree.TreeState == trillian.TreeState_FROZEN { + return true, nil + } + + // Its possible that a freeze record has been added to the tree but + // the call to flip the tree state to frozen has failed. In this + // case the tree is still considered frozen. We need to manually + // check the last leaf of the tree to see if it is a free record. + _, lr, err := t.client.signedLogRoot(tree) + if err != nil { + return false, fmt.Errorf("signedLogRoot: %v", err) + } + leaves, err := t.client.leavesByRange(treeID, int64(lr.TreeSize)-1, 1) + if err != nil { + return false, fmt.Errorf("leavesByRange: %v", err) + } + if len(leaves) != 1 { + return false, fmt.Errorf("unexpected leaves count: got %v, want 1", + len(leaves)) + } + if !isFreezeRecordLeaf(leaves[0]) { + // Not a freeze record leaf. Tree is not frozen. + return false, nil + } + + // Tree has a freeze record but the tree state is not frozen. This + // is bad. Fix it before returning. + _, err = t.client.treeFreeze(treeID) + if err != nil { + return true, fmt.Errorf("treeFreeze: %v", err) + } + + return true, nil +} + +func (t *tlog) recordVersion(treeID int64, version uint32) (*backend.Record, error) { + // Ensure tree exists + if !t.treeExists(treeID) { + return nil, errRecordNotFound + } + + // Get tree leaves + leaves, err := t.client.LeavesAll(treeID) + if err != nil { + return nil, fmt.Errorf("LeavesAll %v: %v", treeID, err) + } + + // Get the record index for the specified version + index, err := t.recordIndex(leaves, version) + if err != nil { + return nil, err + } + + // Use the record index to pull the record content from the store. + // The keys for the record content first need to be extracted from + // their associated log leaf. + + // Compile merkle root hashes of record content + merkles := make(map[string]struct{}, 64) + merkles[hex.EncodeToString(index.RecordMetadata)] = struct{}{} + for _, v := range index.Metadata { + merkles[hex.EncodeToString(v)] = struct{}{} + } + for _, v := range index.Files { + merkles[hex.EncodeToString(v)] = struct{}{} + } + + // Walk the tree and extract the record content keys + keys := make([]string, 0, len(index.Metadata)+len(index.Files)+1) + for _, v := range leaves { + _, ok := merkles[hex.EncodeToString(v.MerkleLeafHash)] + if !ok { + // Not part of the record content + continue + } + + // Leaf is part of record content. Extract the kv store key. + key, err := extractKeyForRecordContent(v) + if err != nil { + return nil, fmt.Errorf("extractKeyForRecordContent %x", + v.MerkleLeafHash) + } + + keys = append(keys, key) + } + + // Get record content from store + blobs, err := t.store.Get(keys) + if err != nil { + return nil, fmt.Errorf("store Get: %v", err) + } + if len(blobs) != len(keys) { + // One or more blobs were not found + missing := make([]string, 0, len(keys)) + for _, v := range keys { + _, ok := blobs[v] + if !ok { + missing = append(missing, v) + } + } + return nil, fmt.Errorf("blobs not found: %v", missing) + } + + // Decode blobs + entries := make([]blobEntry, 0, len(keys)) + for _, v := range blobs { + var be *blobEntry + if t.encryptionKey != nil && isEncrypted(v) { + v, _, err = t.encryptionKey.Decrypt(v) + if err != nil { + return nil, err + } + } + be, err := deblob(v) + if err != nil { + // Check if this is an encrypted blob that was not decrypted + if t.encryptionKey == nil && isEncrypted(v) { + return nil, fmt.Errorf("blob is encrypted but no encryption " + + "key found to decrypt blob") + } + return nil, err + } + entries = append(entries, *be) + } + + // Decode blob entries + var ( + recordMD *backend.RecordMetadata + metadata = make([]backend.MetadataStream, 0, len(index.Metadata)) + files = make([]backend.File, 0, len(index.Files)) + ) + for _, v := range entries { + // Decode the data hint + b, err := base64.StdEncoding.DecodeString(v.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd dataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Type != dataTypeStructure { + return nil, fmt.Errorf("invalid data type; got %v, want %v", + dd.Type, dataTypeStructure) + } + + // Decode the data + b, err = base64.StdEncoding.DecodeString(v.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + hash, err := hex.DecodeString(v.Hash) + if err != nil { + return nil, fmt.Errorf("decode Hash: %v", err) + } + if !bytes.Equal(util.Digest(b), hash) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), hash) + } + switch dd.Descriptor { + case dataDescriptorRecordMetadata: + var rm backend.RecordMetadata + err = json.Unmarshal(b, &rm) + if err != nil { + return nil, fmt.Errorf("unmarshal RecordMetadata: %v", err) + } + recordMD = &rm + case dataDescriptorMetadataStream: + var ms backend.MetadataStream + err = json.Unmarshal(b, &ms) + if err != nil { + return nil, fmt.Errorf("unmarshal MetadataStream: %v", err) + } + metadata = append(metadata, ms) + case dataDescriptorFile: + var f backend.File + err = json.Unmarshal(b, &f) + if err != nil { + return nil, fmt.Errorf("unmarshal File: %v", err) + } + files = append(files, f) + default: + return nil, fmt.Errorf("invalid descriptor %v", dd.Descriptor) + } + } + + // Sanity checks switch { - case rm.Iteration == 1 && len(files) == 0: - // A new record must contain files - return nil, fmt.Errorf("no files") - case rm.Iteration > 1: - // Get the existing record and ensure that files changes are - // being made. + case recordMD == nil: + return nil, fmt.Errorf("record metadata not found") + case len(metadata) != len(index.Metadata): + return nil, fmt.Errorf("invalid number of metadata; got %v, want %v", + len(metadata), len(index.Metadata)) + case len(files) != len(index.Files): + return nil, fmt.Errorf("invalid number of files; got %v, want %v", + len(files), len(index.Files)) } - // Save content to key-value store + return &backend.Record{ + Version: strconv.FormatUint(uint64(version), 10), + RecordMetadata: *recordMD, + Metadata: metadata, + Files: files, + }, nil +} + +func (t *tlog) recordLatest(treeID int64) (*backend.Record, error) { + return t.recordVersion(treeID, 0) +} - // Append content to trillian tree +// We do not unwind. +func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { + // Ensure tree exists + if !t.treeExists(treeID) { + return errRecordNotFound + } + + // Get tree leaves + leavesAll, err := t.client.LeavesAll(treeID) + if err != nil { + return fmt.Errorf("LeavesAll %v: %v", treeID, err) + } + + // Prepare kv store blobs + args := blobsPrepareArgs{ + encryptionKey: t.encryptionKey, + leaves: leavesAll, + recordMD: rm, + metadata: metadata, + files: files, + } + bpr, err := blobsPrepare(args) + if err != nil { + return err + } + + // Ensure file changes are being made + if len(bpr.recordIndex.Files) == len(files) { + return errNoFileChanges + } + + // Save blobs + idx, err := t.blobsSave(treeID, *bpr) + if err != nil { + return fmt.Errorf("blobsSave: %v", err) + } + + // Get the existing record index and use it to bump the version and + // iteration of the new record index. + oldIdx, err := t.recordIndexLatest(leavesAll) + if err == errRecordNotFound { + // No record versions exist yet. This is fine. The version and + // iteration will be incremented to 1. + } else if err != nil { + return fmt.Errorf("recordIndexLatest: %v", err) + } + idx.Version = oldIdx.Version + 1 + idx.Iteration = oldIdx.Iteration + 1 + + // Sanity check. The record index should be fully populated at this + // point. + switch { + case idx.Version != oldIdx.Version+1: + return fmt.Errorf("invalid index version: got %v, want %v", + idx.Version, oldIdx.Version+1) + case idx.Iteration != oldIdx.Iteration+1: + return fmt.Errorf("invalid index iteration: got %v, want %v", + idx.Iteration, oldIdx.Iteration+1) + case idx.RecordMetadata == nil: + return fmt.Errorf("invalid index record metadata") + case len(idx.Metadata) != len(metadata): + return fmt.Errorf("invalid index metadata: got %v, want %v", + len(idx.Metadata), len(metadata)) + case len(idx.Files) != len(files): + return fmt.Errorf("invalid index files: got %v, want %v", + len(idx.Files), len(files)) + } // Save record index + err = t.recordIndexSave(treeID, *idx) + if err != nil { + return fmt.Errorf("recordIndexSave: %v", err) + } - return nil, nil + return nil } -func (t *tlog) recordStatusUpdate() {} +func (t *tlog) recordMetadataUpdate(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { + // Ensure tree exists + if !t.treeExists(treeID) { + return errRecordNotFound + } -func (t *tlog) recordMetdataUpdate() {} + // Get tree leaves + leavesAll, err := t.client.LeavesAll(treeID) + if err != nil { + return fmt.Errorf("LeavesAll: %v", err) + } + + // Prepare kv store blobs + args := blobsPrepareArgs{ + encryptionKey: t.encryptionKey, + leaves: leavesAll, + recordMD: rm, + metadata: metadata, + } + bpr, err := blobsPrepare(args) + if err != nil { + return err + } + + // Ensure metadata has been changed + if len(bpr.blobs) == 0 { + return errNoMetadataChanges + } + + // Save the blobs + idx, err := t.blobsSave(treeID, *bpr) + if err != nil { + return fmt.Errorf("blobsSave: %v", err) + } + + // Get the existing record index and add the unchanged fields to + // the new record index. The version and files will remain the + // same. + oldIdx, err := t.recordIndexLatest(leavesAll) + if err != nil { + return fmt.Errorf("recordIndexLatest: %v", err) + } + idx.Version = oldIdx.Version + idx.Files = oldIdx.Files + + // Increment the iteration + idx.Iteration = oldIdx.Iteration + 1 + + // Sanity check. The record index should be fully populated at this + // point. + switch { + case idx.Version != oldIdx.Version: + return fmt.Errorf("invalid index version: got %v, want %v", + idx.Version, oldIdx.Version) + case idx.Version != oldIdx.Iteration+1: + return fmt.Errorf("invalid index iteration: got %v, want %v", + idx.Iteration, oldIdx.Iteration+1) + case idx.RecordMetadata == nil: + return fmt.Errorf("invalid index record metadata") + case len(idx.Metadata) != len(metadata): + return fmt.Errorf("invalid index metadata: got %v, want %v", + len(idx.Metadata), len(metadata)) + case len(idx.Files) != len(oldIdx.Files): + return fmt.Errorf("invalid index files: got %v, want %v", + len(idx.Files), len(oldIdx.Files)) + } + + // Save record index + err = t.recordIndexSave(treeID, *idx) + if err != nil { + return fmt.Errorf("recordIndexSave: %v", err) + } + + return nil +} + +// treeFreeze freezes the trillian tree for the provided token. Once a tree +// has been frozen it is no longer able to be appended to. The last leaf in a +// frozen tree will correspond to a freeze record in the key-value store. A +// tree is frozen when the status of the corresponding record is updated to a +// status that locks the record, such as when a record is censored. The status +// change and the freeze record are appended to the tree using a single call. +// +// It's possible for this function to fail in between the append leaves call +// and the call that updates the tree status to frozen. If this happens the +// freeze record will be the last leaf on the tree but the tree state will not +// be frozen. The tree is still considered frozen and no new leaves should be +// appended to it. The tree state will be updated in the next fsck. +func (t *tlog) treeFreeze(treeID int64, fr freezeRecord) error { + // Save freeze record to store + be, err := convertBlobEntryFromFreezeRecord(fr) + if err != nil { + return err + } + b, err := blobify(*be) + if err != nil { + return err + } + keys, err := t.store.Put([][]byte{b}) + if err != nil { + return fmt.Errorf("store Put: %v", err) + } + if len(keys) != 1 { + return fmt.Errorf("wrong number of keys: got %v, want 1", + len(keys)) + } + + // Append freeze record leaf to trillian tree + h, err := hex.DecodeString(be.Hash) + if err != nil { + return err + } + prefixedKey := []byte(keyPrefixFreezeRecord + keys[0]) + queued, _, err := t.client.LeavesAppend(treeID, []*trillian.LogLeaf{ + logLeafNew(h, prefixedKey), + }) + if len(queued) != 1 { + return fmt.Errorf("wrong number of queud leaves: got %v, want 1", + len(queued)) + } + failed := make([]string, 0, len(queued)) + for _, v := range queued { + c := codes.Code(v.QueuedLeaf.GetStatus().GetCode()) + if c != codes.OK { + failed = append(failed, fmt.Sprintf("%v", c)) + } + } + if len(failed) > 0 { + return fmt.Errorf("append leaves failed: %v", failed) + } + + // Freeze the tree + _, err = t.client.treeFreeze(treeID) + if err != nil { + return fmt.Errorf("treeFreeze: %v", err) + } -func (t *tlog) recordProof() {} + return nil +} -func (t *tlog) recordFreeze(token []byte, pointer int64) {} +func (t *tlog) recordProof(treeID int64, version uint32) {} func (t *tlog) fsck() { - // TODO soft delete trees that don't have leaves and are more than - // a week old. This can happen if a record new call fails. + // Failed freeze + // Failed censor +} + +func (t *tlog) close() { + // Close connections + t.store.Close() + t.client.Close() + + // Zero out encryption key + if t.encryptionKey != nil { + t.encryptionKey.Zero() + } +} + +func tlogNew() (*tlog, error) { + return &tlog{}, nil } diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 774dbe3cf..6502117de 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -7,36 +7,29 @@ package tlogbe import ( "bytes" "crypto/sha256" - "encoding/base64" "encoding/hex" - "encoding/json" "fmt" "io/ioutil" "net/url" "os" "path/filepath" - "strconv" - "strings" "sync" "time" "github.com/decred/dcrtime/merkle" pd "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/store" "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" "github.com/decred/politeia/util" - "github.com/google/trillian" - "github.com/google/trillian/types" "github.com/marcopeereboom/sbox" - "github.com/robfig/cron" - "google.golang.org/grpc/codes" ) -// TODO we need to seperate testnet and mainnet trillian trees. This will need -// to be done through setup scripts and config options. Use different ports for -// testnet. +// TODO Add UpdateUnvettedMetadata to backend interface +// TODO populate token prefixes cache on startup +// TODO testnet vs mainnet trillian databases // TODO lock on the token level +// TODO fsck +// TODO allow token prefix lookups const ( defaultTrillianKeyFilename = "trillian.key" @@ -65,1013 +58,181 @@ var ( backend.MDStatusVetted: map[backend.MDStatusT]struct{}{ backend.MDStatusArchived: struct{}{}, }, + + // Statuses that do not allow any further transitions + backend.MDStatusCensored: map[backend.MDStatusT]struct{}{}, + backend.MDStatusArchived: map[backend.MDStatusT]struct{}{}, } ) // tlogbe implements the Backend interface. type tlogbe struct { sync.RWMutex - shutdown bool - homeDir string - dataDir string - dcrtimeHost string - encryptionKey *EncryptionKey - store store.Blob - tlog *TrillianClient - cron *cron.Cron - plugins []backend.Plugin - + shutdown bool + homeDir string + dataDir string unvetted *tlog vetted *tlog - - // prefixes contains the first n characters of each record token, - // where n is defined by the TokenPrefixLength from the politeiad - // API. Lookups by token prefix are allowed. This cache is used to - // prevent prefix collisions when creating new tokens. - prefixes map[string]struct{} - - // droppingAnchor indicates whether tlogbe is in the process of - // dropping an anchor, i.e. timestamping unanchored trillian trees - // using dcrtime. An anchor is dropped periodically using cron. - droppingAnchor bool -} - -// statusChangeIsAllowed returns whether the provided status change is allowed -// by tlogbe. An invalid 'from' status will panic since the 'from' status -// represents the existing status of a record and should never be invalid. -func statusChangeIsAllowed(from, to backend.MDStatusT) bool { - allowed, ok := statusChanges[from] - if !ok { - e := fmt.Sprintf("status invalid: %v", from) - panic(e) - } - _, ok = allowed[to] + plugins []backend.Plugin + + // prefixes contains the token prefix to full token mapping for all + // records. The prefix is the first n characters of the hex encoded + // record token, where n is defined by the TokenPrefixLength from + // the politeiad API. Record lookups by token prefix are allowed. + // This cache is used to prevent prefix collisions when creating + // new tokens and to facilitate lookups by token prefix. This cache + // is loaded on tlogbe startup. + prefixes map[string][]byte // [tokenPrefix]fullToken + + // vettedTrees contains the token to tree ID mapping for vetted + // records. The token corresponds to the unvetted tree ID so + // unvetted lookups can be done directly, but vetted lookups + // required pulling the freeze record from the unvetted tree to + // get the vetted tree ID. This cache memoizes these results. + vettedTrees map[string]int64 // [token]treeID +} + +func (t *tlogbe) prefixExists(fullToken []byte) bool { + t.RLock() + defer t.RUnlock() + + prefix := hex.EncodeToString(fullToken)[:pd.TokenPrefixLength] + _, ok := t.prefixes[prefix] return ok } -func merkleRoot(files []backend.File) (*[sha256.Size]byte, error) { - hashes := make([]*[sha256.Size]byte, 0, len(files)) - for _, v := range files { - b, err := hex.DecodeString(v.Digest) - if err != nil { - return nil, err - } - var d [sha256.Size]byte - copy(d[:], b) - hashes = append(hashes, &d) - } - return merkle.Root(hashes), nil -} - -func recordMetadataNew(token []byte, files []backend.File, status backend.MDStatusT, iteration uint64) (*backend.RecordMetadata, error) { - m, err := merkleRoot(files) - if err != nil { - return nil, err - } - return &backend.RecordMetadata{ - Version: backend.VersionRecordMD, - Iteration: iteration, - Status: status, - Merkle: hex.EncodeToString(m[:]), - Timestamp: time.Now().Unix(), - Token: tokenString(token), - }, nil -} - -func convertBlobEntryFromFile(f backend.File) (*blobEntry, error) { - data, err := json.Marshal(f) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - dataDescriptor{ - Type: dataTypeStructure, - Descriptor: dataDescriptorFile, - }) - if err != nil { - return nil, err - } - be := blobEntryNew(hint, data) - return &be, nil -} - -func convertBlobEntryFromMetadataStream(ms backend.MetadataStream) (*blobEntry, error) { - data, err := json.Marshal(ms) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - dataDescriptor{ - Type: dataTypeStructure, - Descriptor: dataDescriptorMetadataStream, - }) - if err != nil { - return nil, err - } - be := blobEntryNew(hint, data) - return &be, nil -} - -func convertBlobEntryFromRecordMetadata(rm backend.RecordMetadata) (*blobEntry, error) { - data, err := json.Marshal(rm) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - dataDescriptor{ - Type: dataTypeStructure, - Descriptor: dataDescriptorRecordMetadata, - }) - if err != nil { - return nil, err - } - be := blobEntryNew(hint, data) - return &be, nil -} - -func convertBlobEntryFromRecordHistory(rh recordHistory) (*blobEntry, error) { - data, err := json.Marshal(rh) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - dataDescriptor{ - Type: dataTypeStructure, - Descriptor: dataDescriptorRecordHistory, - }) - if err != nil { - return nil, err - } - be := blobEntryNew(hint, data) - return &be, nil -} - -func convertBlobEntriesFromFiles(files []backend.File) ([]blobEntry, error) { - entries := make([]blobEntry, 0, len(files)) - for _, v := range files { - re, err := convertBlobEntryFromFile(v) - if err != nil { - return nil, err - } - entries = append(entries, *re) - } - return entries, nil -} - -func convertBlobEntriesFromMetadataStreams(streams []backend.MetadataStream) ([]blobEntry, error) { - entries := make([]blobEntry, 0, len(streams)) - for _, v := range streams { - re, err := convertBlobEntryFromMetadataStream(v) - if err != nil { - return nil, err - } - entries = append(entries, *re) - } - return entries, nil -} - -func convertRecordMetadataFromBlobEntry(be blobEntry) (*backend.RecordMetadata, error) { - // Decode and validate the DataHint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd dataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorRecordMetadata { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorRecordMetadata) - } - - // Decode the MetadataStream - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - var rm backend.RecordMetadata - err = json.Unmarshal(b, &rm) - if err != nil { - return nil, fmt.Errorf("unmarshal RecordMetadata: %v", err) - } - - return &rm, nil -} - -func convertMetadataStreamFromBlobEntry(be blobEntry) (*backend.MetadataStream, error) { - // Decode and validate the DataHint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd dataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorMetadataStream { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorMetadataStream) - } - - // Decode the MetadataStream - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - var ms backend.MetadataStream - err = json.Unmarshal(b, &ms) - if err != nil { - return nil, fmt.Errorf("unmarshal MetadataStream: %v", err) - } - - return &ms, nil -} - -func convertFileFromBlobEntry(be blobEntry) (*backend.File, error) { - // Decode and validate the DataHint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd dataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorFile { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorFile) - } - - // Decode the MetadataStream - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - var f backend.File - err = json.Unmarshal(b, &f) - if err != nil { - return nil, fmt.Errorf("unmarshal File: %v", err) - } - - return &f, nil -} - -func convertLeavesFromBlobEntries(entries []blobEntry) ([]*trillian.LogLeaf, error) { - leaves := make([]*trillian.LogLeaf, 0, len(entries)) - for _, v := range entries { - b, err := hex.DecodeString(v.Hash) - if err != nil { - return nil, err - } - leaves = append(leaves, logLeafNew(b)) - } - return leaves, nil -} - -func convertRecordHistoryFromBlobEntry(be blobEntry) (*recordHistory, error) { - // Decode and validate the DataHint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd dataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorRecordHistory { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorRecordHistory) - } - - // Decode the MetadataStream - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - var rh recordHistory - err = json.Unmarshal(b, &rh) - if err != nil { - return nil, fmt.Errorf("unmarshal recordHistory: %v", err) - } - - return &rh, nil -} - -func filesApplyChanges(files, filesAdd []backend.File, filesDel []string) []backend.File { - del := make(map[string]struct{}, len(filesDel)) - for _, fn := range filesDel { - del[fn] = struct{}{} - } - f := make([]backend.File, 0, len(files)+len(filesAdd)) - for _, v := range files { - if _, ok := del[v.Name]; ok { - continue - } - f = append(f, v) - } - for _, v := range filesAdd { - f = append(f, v) - } - return f -} - -func metadataStreamsApplyChanges(md, mdAppend, mdOverwrite []backend.MetadataStream) []backend.MetadataStream { - // Include all overwrites - metadata := make([]backend.MetadataStream, 0, len(md)+len(mdOverwrite)) - for _, v := range mdOverwrite { - metadata = append(metadata, v) - } - - // Add in existing metadata that wasn't overwritten - overwrite := make(map[uint64]struct{}, len(mdOverwrite)) - for _, v := range mdOverwrite { - overwrite[v.ID] = struct{}{} - } - for _, v := range md { - if _, ok := overwrite[v.ID]; ok { - // Metadata has already been overwritten - continue - } - metadata = append(metadata, v) - } - - // Apply appends - appends := make(map[uint64]backend.MetadataStream, len(mdAppend)) - for _, v := range mdAppend { - appends[v.ID] = v - } - for i, v := range metadata { - ms, ok := appends[v.ID] - if !ok { - continue - } - buf := bytes.NewBuffer([]byte(v.Payload)) - buf.WriteString(ms.Payload) - metadata[i].Payload = buf.String() - } - - return metadata -} - -// blobEntriesAppend appends the provided blob entries onto the trillain tree -// that corresponds to the provided token. An error is returned if any leaves -// are not successfully added. The only exception to this is if a leaf is not -// appended because it is a duplicate. This can happen in certain situations -// such as when a file is deleted from a record then added back to the record -// without being altered. The order of the returned leaf proofs is not -// guaranteed. -func (t *tlogbe) blobEntriesAppend(token []byte, entries []blobEntry) ([]LeafProof, *types.LogRootV1, error) { - // Setup request - treeID := treeIDFromToken(token) - leaves, err := convertLeavesFromBlobEntries(entries) - if err != nil { - return nil, nil, err - } - - // Append leaves - queued, lr, err := t.tlog.LeavesAppend(treeID, leaves) - if err != nil { - return nil, nil, fmt.Errorf("leavesAppend: %v", err) - } - if len(queued) != len(leaves) { - // Sanity check. Even if a leaf fails to be appended there should - // still be a QueuedLogLeaf with the error code. - return nil, nil, fmt.Errorf("wrong number of leaves: got %v, want %v", - len(queued), len(leaves)) - } - - // Convert queuedLeafProofs to leafProofs. Fail if any of the - // leaves were not appended successfully. The exception to this is - // if the leaf was not appended because it was a duplicate. - proofs := make([]LeafProof, 0, len(queued)) - dups := make([][]byte, 0, len(queued)) - failed := make([]string, 0, len(queued)) - for _, v := range queued { - c := codes.Code(v.QueuedLeaf.GetStatus().GetCode()) - switch c { - case codes.OK: - // Leaf successfully appended to tree - proofs = append(proofs, LeafProof{ - Leaf: v.QueuedLeaf.Leaf, - Proof: v.Proof, - }) - - case codes.AlreadyExists: - // We need to retrieve the leaf proof manually for this leaf - // because it was a duplicate. This can happen in certain - // situations such as when a record file is deleted then added - // back at a later date without being altered. A duplicate in - // trillian is ok. A duplicate in the storage layer is not ok - // so check the storage layer first to ensure this is not a - // real duplicate. - m := merkleLeafHash(v.QueuedLeaf.Leaf.LeafValue) - _, err := t.store.Get(keyRecordContent(token, m)) - if err == nil { - return nil, nil, fmt.Errorf("duplicate found in store: %x", m) - } - dups = append(dups, m) - - log.Debugf("Duplicate leaf %x, retreiving proof manually", m) - - default: - // All other errors. This is not ok. - failed = append(failed, fmt.Sprint("%v", c)) - } - } - if len(failed) > 0 { - return nil, nil, fmt.Errorf("append leaves failed: %v", failed) - } - - // Retrieve leaf proofs for duplicates - if len(dups) > 0 { - p, err := t.tlog.LeafProofs(treeID, dups, lr) - if err != nil { - return nil, nil, fmt.Errorf("leafProofs: %v", err) - } - proofs = append(proofs, p...) - } - - return proofs, lr, nil -} - -func (t *tlogbe) recordMetadata(token, merkleLeafHash []byte) (*backend.RecordMetadata, error) { - log.Tracef("recordMetadata: %x", merkleLeafHash) - - key := keyRecordContent(token, merkleLeafHash) - b, err := t.store.Get(key) - if err != nil { - return nil, err - } - be, err := deblob(b) - if err != nil { - return nil, err - } - rm, err := convertRecordMetadataFromBlobEntry(*be) - if err != nil { - return nil, err - } - - return rm, nil -} - -func (t *tlogbe) metadataStream(token, merkleLeafHash []byte) (*backend.MetadataStream, error) { - log.Tracef("metadataStream: %x", merkleLeafHash) - - key := keyRecordContent(token, merkleLeafHash) - b, err := t.store.Get(key) - if err != nil { - return nil, err - } - be, err := deblob(b) - if err != nil { - return nil, err - } - ms, err := convertMetadataStreamFromBlobEntry(*be) - if err != nil { - return nil, err - } - - return ms, nil -} - -func (t *tlogbe) file(token, merkleLeafHash []byte, state string) (*backend.File, error) { - log.Tracef("file: %v %x", state, merkleLeafHash) - - key := keyRecordContent(token, merkleLeafHash) - b, err := t.store.Get(key) - if err != nil { - return nil, fmt.Errorf("store Get: %v", err) - } - var be *blobEntry - switch state { - case stateUnvetted: - // Unvetted backend File blobs will be encrypted - be, err = deblobEncrypted(b, t.encryptionKey) - if err != nil { - return nil, err - } - case stateVetted: - // Vetted backend File blobs will not be encrypted - be, err = deblob(b) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("unkown record history state: %v", state) - } - f, err := convertFileFromBlobEntry(*be) - if err != nil { - return nil, err - } +func (t *tlogbe) prefixSet(fullToken []byte) { + t.Lock() + defer t.Unlock() - return f, nil + prefix := hex.EncodeToString(fullToken)[:pd.TokenPrefixLength] + t.prefixes[prefix] = fullToken } -// record returns the backend record given the record index. -func (t *tlogbe) record(token []byte, ri recordIndex, version uint32, state string) (*backend.Record, error) { - log.Tracef("record: %x", token) +func (t *tlogbe) vettedTreesGet(token string) (int64, bool) { + t.RLock() + defer t.RUnlock() - recordMD, err := t.recordMetadata(token, ri.RecordMetadata) - if err != nil { - return nil, fmt.Errorf("recordMetadata %x: %v", ri.RecordMetadata, err) - } - - metadata := make([]backend.MetadataStream, 0, len(ri.Metadata)) - for id, merkle := range ri.Metadata { - ms, err := t.metadataStream(token, merkle) - if err != nil { - return nil, fmt.Errorf("metadataStream %v %x: %v", id, merkle, err) - } - metadata = append(metadata, *ms) - } - - files := make([]backend.File, 0, len(ri.Files)) - for fn, merkle := range ri.Files { - f, err := t.file(token, merkle, state) - if err != nil { - return nil, fmt.Errorf("file %v %x: %v", fn, merkle, err) - } - files = append(files, *f) - } - - return &backend.Record{ - Version: strconv.FormatUint(uint64(version), 10), - RecordMetadata: *recordMD, - Metadata: metadata, - Files: files, - }, nil + treeID, ok := t.vettedTrees[token] + return treeID, ok } -// recordSave saves the provided record as a new version. This includes -// appending the hashes of the record contents onto the associated trillian -// tree, saving the record contents as blobs in the storage layer, and saving -// a record index. This function assumes the record contents have already been -// validated. -func (t *tlogbe) recordSave(token []byte, metadata []backend.MetadataStream, files []backend.File, recordMD backend.RecordMetadata, rh recordHistory) (*backend.Record, error) { - // Prepare blob entries - recordMDEntry, err := convertBlobEntryFromRecordMetadata(recordMD) - if err != nil { - return nil, err - } - metadataEntries, err := convertBlobEntriesFromMetadataStreams(metadata) - if err != nil { - return nil, err - } - fileEntries, err := convertBlobEntriesFromFiles(files) - if err != nil { - return nil, err - } - - // The RecordMetadata is intentionally put first so that it is - // added to the trillian tree first. If we ever need to walk the - // tree a RecordMetadata will signify the start of a new record. - entries := make([]blobEntry, 0, len(metadata)+len(files)+1) - entries = append(entries, *recordMDEntry) - entries = append(entries, metadataEntries...) - entries = append(entries, fileEntries...) - - // Append leaves onto trillian tree for all record contents - proofs, _, err := t.blobEntriesAppend(token, entries) - if err != nil { - return nil, fmt.Errorf("blobEntriesAppend %x: %v", token, err) - } - - // Aggregate the merkle leaf hashes. These are used as the keys - // when saving blobs to the key-value store. - merkles := make(map[string][]byte, len(entries)) // [leafValue]merkleLeafHash - for _, v := range proofs { - k := hex.EncodeToString(v.Leaf.LeafValue) - merkles[k] = v.Leaf.MerkleLeafHash - } - - // Aggregate the blob entry hashes of all the file blobs. These - // are used to determine if the blob entry should be encrypted. - // Unvetted files are stored as encrypted blobs. All other record - // content is stored as unencrypted blobs. - fileHashes := make(map[string]struct{}, len(fileEntries)) - for _, v := range fileEntries { - fileHashes[v.Hash] = struct{}{} - } - - // Prepare blobs for the storage layer. Unvetted files are stored - // as encrypted blobs. The merkle leaf hash is used as the key in - // the blob key-value store for all record content. - blobs := make(map[string][]byte, len(entries)) // [key]blob - for _, v := range entries { - merkle, ok := merkles[v.Hash] - if !ok { - return nil, fmt.Errorf("no merkle leaf hash for %v", v.Hash) - } - - var b []byte - _, ok = fileHashes[v.Hash] - if ok && rh.State == stateUnvetted { - // This is an unvetted file. Store it as an encrypted blob. - b, err = blobifyEncrypted(v, t.encryptionKey) - if err != nil { - return nil, err - } - } else { - // All other record content is store as unencrypted blobs. - b, err = blobify(v) - if err != nil { - return nil, err - } - } - - blobs[keyRecordContent(token, merkle)] = b - } - - // Retrieve the record history and add a new record index version - // to it. This updated record history will be saved with the rest - // of the blobs. The token is used as the record history key. - ri, err := recordIndexNew(entries, merkles) - if err != nil { - return nil, err - } - rh.Versions[latestVersion(rh)+1] = *ri - be, err := convertBlobEntryFromRecordHistory(rh) - if err != nil { - return nil, err - } - b, err := blobify(*be) - if err != nil { - return nil, err - } - blobs[keyRecordHistory(token)] = b - - // Save all blobs - log.Debugf("Saving %v blobs to kv store", len(blobs)) - - err = t.store.Batch(store.Ops{ - Put: blobs, - }) - if err != nil { - return nil, fmt.Errorf("store Batch: %v", err) - } - - // Lookup new version of the record - log.Debugf("Record index:\n%v", rh.String()) +func (t *tlogbe) vettedTreesSet(token string, treeID int64) { + t.Lock() + defer t.Unlock() - version := latestVersion(rh) - r, err := t.record(token, *ri, version, rh.State) - if err != nil { - return nil, fmt.Errorf("record: %v", err) - } + t.vettedTrees[token] = treeID - return r, nil + log.Debugf("vettedTreesSet: %v %v", token, treeID) } -// recordUpdate updates the current version of the provided record. This -// includes appending the hashes of the record contents onto the associated -// trillian tree, saving the record contents as blobs in the storage layer, and -// updating the existing record index. This function assumes the record -// contents have already been validated. The blobs for unvetted record files -// (just files, not metadata) are encrypted before being saved to the storage -// layer. -// -// This function must be called WITH the lock held. -func (t *tlogbe) recordUpdate(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string, rm backend.RecordMetadata, rh recordHistory) (*backend.Record, error) { - log.Tracef("recordUpdate: %x", token) - - // Get the record index - idx := rh.Versions[latestVersion(rh)] - - // entries is used to keep track of the new record content that - // needs to be added to the trillian tree and saved to the blob - // storage layer. The blobEntry.Hash is all that is appened onto - // the trillian tree. The full blobEntry is saved to the blob - // storage layer. - l := len(mdAppend) + len(mdOverwrite) + len(filesAdd) + len(filesDel) - entries := make([]blobEntry, 0, l) - - // encrypt tracks the blob entries that will be saved as encrypted - // blobs. - encrypt := make(map[string]struct{}, l) // [hash]struct{} - - // orphaned tracks the merkle leaf hashes of the blobs that have - // been orphaned by this update. An orphaned blob is one that does - // not correspond to a specific record version. These blobs are - // deleted from the record index and the blob storage layer. - orphaned := make([][]byte, 0, l) - - // Append metadata - for _, v := range mdAppend { - // Lookup existing metadata stream. It's ok if one does not - // already exist. A new one will be created. - ms := backend.MetadataStream{ - ID: v.ID, - } - merkle, ok := idx.Metadata[v.ID] - if ok { - // Metadata stream already exists. Retrieve it. - m, err := t.metadataStream(token, merkle) - if err != nil { - return nil, fmt.Errorf("metadataStream %v: %v", v.ID, err) - } - ms.Payload = m.Payload - - // This metadata stream blob will be orphaned by the metadata - // stream blob with the newly appended data. - orphaned = append(orphaned, merkle) - } - - // Append new data - buf := bytes.NewBuffer([]byte(ms.Payload)) - buf.WriteString(v.Payload) - ms.Payload = buf.String() - be, err := convertBlobEntryFromMetadataStream(ms) - if err != nil { - return nil, err - } - - // Save updated metadata stream - entries = append(entries, *be) - - if ok { - log.Debugf("Append MD %v, orphaned blob %x", v.ID, merkle) - } else { - log.Debugf("Append MD %v, no orphaned blob", v.ID) - } - } - - // Overwrite metdata - for _, v := range mdOverwrite { - be, err := convertBlobEntryFromMetadataStream(v) - if err != nil { - return nil, err - } - - // Check if this metadata stream is a duplicate - merkle, ok := idx.Metadata[v.ID] - if ok { - // Metadata stream already exists. Check if there are any - // changes being made to it. - b, err := hex.DecodeString(be.Hash) - if err != nil { - return nil, err - } - m := merkleLeafHash(b) - if bytes.Equal(merkle, m) { - // Existing metadata stream is the same as the new metadata - // stream. No need to save it again. - continue - } - } - - // Metdata stream is not a duplicate of the existing metadata - // stream. Save it. The existing metadata stream blob will be - // orphaned. - entries = append(entries, *be) - orphaned = append(orphaned, merkle) - - log.Debugf("Overwrite MD %v, orphaned blob %x", v.ID, merkle) - } - - // Add files - for _, v := range filesAdd { - be, err := convertBlobEntryFromFile(v) - if err != nil { - return nil, err - } - - // Check if this file is a duplicate - merkle, ok := idx.Files[v.Name] - if ok { - // File name already exists. Check if there are any changes - // being made to it. - b, err := hex.DecodeString(be.Hash) - if err != nil { - return nil, err - } - m := merkleLeafHash(b) - if bytes.Equal(merkle, m) { - // Existing file is the same as the new file. No need to save - // it again. - continue - } - - // New file is different. The existing file will be orphaned. - orphaned = append(orphaned, merkle) - } - - // Save new file - entries = append(entries, *be) - - log.Debugf("Add file %v, orphaned blob %x", v.Name, merkle) - - // Unvetted files are stored as encryted blobs - if rh.State == stateUnvetted { - encrypt[be.Hash] = struct{}{} - } - } - - // Delete files - for _, fn := range filesDel { - // Ensure file exists - merkle, ok := idx.Files[fn] - if !ok { - return nil, backend.ContentVerificationError{ - ErrorCode: pd.ErrorStatusFileNotFound, - ErrorContext: []string{fn}, - } - } - - // This file will be orphaned - orphaned = append(orphaned, merkle) - log.Debugf("Del file %v, orphaned blob %x", fn, merkle) - } - - // Check if the record metadata status is being updated - var statusUpdate bool - var decryptFiles bool - currRM, err := t.recordMetadata(token, idx.RecordMetadata) - if err != nil { - return nil, fmt.Errorf("recordMetadata %v: %v", - idx.RecordMetadata, err) - } - if currRM.Status != rm.Status { - // The record status is being updated - statusUpdate = true - - log.Debugf("Record status is being updated from %v to %v", - currRM.Status, rm.Status) - - // Check if the status is being updated from an unvetted status - // to a vetted status. If so, we will need to decrypt the record - // files and save them as unencrypted blobs. - from := recordStateFromStatus[currRM.Status] - to := recordStateFromStatus[rm.Status] - if from == stateUnvetted && to == stateVetted { - decryptFiles = true - } - } - - // Ensure changes were actually made. The only time we allow an - // update with no changes to the files or metadata streams is when - // the record status is being updated. - if len(entries) == 0 && len(orphaned) == 0 && !statusUpdate { - return nil, backend.ErrNoChanges - } - - // Handle record metadata. The record metadata will have changed - // if the record files are being updated or the record status is - // being updated. It will remain unchanged if this is a metadata - // stream only update. - // - // tlogbe manually updates the record status the first time that - // an unvetted record has its files changed. The status gets - // flipped from Unvetted to UnvettedIteration. All other status - // changes are initiated by the client. - filesUpdated := len(filesAdd) != 0 || len(filesDel) != 0 - if filesUpdated && rm.Status == backend.MDStatusUnvetted { - rm.Status = backend.MDStatusIterationUnvetted - } - be, err := convertBlobEntryFromRecordMetadata(rm) - if err != nil { - return nil, err - } - b, err := hex.DecodeString(be.Hash) - if err != nil { - return nil, err - } - m := merkleLeafHash(b) - if !bytes.Equal(idx.RecordMetadata, m) { - // Record metadata is not the same. Save the new one. The - // existing record metadata is going to be orphaned. - entries = append(entries, *be) - orphaned = append(orphaned, idx.RecordMetadata) - - log.Debugf("Record metadata updated") - } - - // Append leaves onto the trillian tree - proofs, _, err := t.blobEntriesAppend(token, entries) - if err != nil { - return nil, fmt.Errorf("blobEntriesAppend %x: %v", token, err) - } - - // Aggregate the merkle leaf hashes. These are used as the keys - // when saving a blob to the key-value store. - merkles := make(map[string][]byte, len(entries)) // [leafValue]merkleLeafHash - for _, v := range proofs { - k := hex.EncodeToString(v.Leaf.LeafValue) - merkles[k] = v.Leaf.MerkleLeafHash - } - - // Update the record index and record history - ri, err := recordIndexUpdate(idx, entries, merkles, orphaned) - if err != nil { - return nil, err - } - rh.Versions[latestVersion(rh)] = *ri - state, ok := recordStateFromStatus[rm.Status] +// statusChangeIsAllowed returns whether the provided status change is allowed +// by tlogbe. An invalid 'from' status will panic since the 'from' status +// represents the existing status of a record and should never be invalid. +func statusChangeIsAllowed(from, to backend.MDStatusT) bool { + allowed, ok := statusChanges[from] if !ok { - return nil, fmt.Errorf("status %v does not map to a state", rm.Status) + e := fmt.Sprintf("status invalid: %v", from) + panic(e) } - rh.State = state - - // Blobify all the things. The merkle leaf hash is used as the key - // for record content in the key-value store. - blobs := make(map[string][]byte, len(entries)+1) // [key][]byte - for _, v := range entries { - // Get the merkle leaf hash for this blob entry - merkle, ok := merkles[v.Hash] - if !ok { - return nil, fmt.Errorf("no merkle leaf hash for %v", v.Hash) - } - - // Sanity check. Blob should not already exist. - _, err = t.store.Get(keyRecordContent(token, merkle)) - if err == nil { - return nil, fmt.Errorf("unexpected blob found %v %v", v.Hash, merkle) - } + _, ok = allowed[to] + return ok +} - // Prepare blob - var b []byte - if _, ok := encrypt[v.Hash]; ok { - b, err = blobifyEncrypted(v, t.encryptionKey) - } else { - b, err = blobify(v) - } +func merkleRoot(files []backend.File) (*[sha256.Size]byte, error) { + hashes := make([]*[sha256.Size]byte, 0, len(files)) + for _, v := range files { + b, err := hex.DecodeString(v.Digest) if err != nil { return nil, err } - blobs[keyRecordContent(token, merkle)] = b + var d [sha256.Size]byte + copy(d[:], b) + hashes = append(hashes, &d) } + return merkle.Root(hashes), nil +} - // Blobify the record history. The record token is used as the key - // for record histories in the key-value store. - be, err = convertBlobEntryFromRecordHistory(rh) +func recordMetadataNew(token []byte, files []backend.File, status backend.MDStatusT, iteration uint64) (*backend.RecordMetadata, error) { + m, err := merkleRoot(files) if err != nil { return nil, err } - b, err = blobify(*be) - if err != nil { - return nil, err + return &backend.RecordMetadata{ + Version: backend.VersionRecordMD, + Iteration: iteration, + Status: status, + Merkle: hex.EncodeToString(m[:]), + Timestamp: time.Now().Unix(), + Token: hex.EncodeToString(token), + }, nil +} + +// TODO test this function +func filesUpdate(filesCurr, filesAdd []backend.File, filesDel []string) []backend.File { + // Apply deletes + del := make(map[string]struct{}, len(filesDel)) + for _, fn := range filesDel { + del[fn] = struct{}{} } - blobs[keyRecordHistory(token)] = b - - // Check if the existing file blobs in the store need to be - // converted from encrypted blobs to unencrypted blobs. - if decryptFiles { - log.Debugf("Converting encrypted blobs to unecrypted blobs") - for _, merkle := range ri.Files { - f, err := t.file(token, merkle, stateUnvetted) - if err != nil { - return nil, fmt.Errorf("file %x: %v", merkle, err) - } - be, err := convertBlobEntryFromFile(*f) - if err != nil { - return nil, err - } - b, err := blobify(*be) - if err != nil { - return nil, err - } - blobs[keyRecordContent(token, merkle)] = b + f := make([]backend.File, 0, len(filesCurr)+len(filesAdd)) + for _, v := range filesCurr { + if _, ok := del[v.Name]; ok { + continue } + f = append(f, v) } - // Convert the orphaned merkle root hashes to blob keys - del := make([]string, 0, len(orphaned)) - for _, merkle := range orphaned { - del = append(del, keyRecordContent(token, merkle)) + // Apply adds + for _, v := range filesAdd { + f = append(f, v) } - // Save all the blob changes - err = t.store.Batch(store.Ops{ - Put: blobs, - Del: del, - }) - if err != nil { - return nil, fmt.Errorf("store Batch: %v", err) + return f +} + +// TODO test this function +func metadataStreamsUpdate(mdCurr, mdAppend, mdOverwrite []backend.MetadataStream) []backend.MetadataStream { + // Convert existing metadata to map + md := make(map[uint64]backend.MetadataStream, len(mdCurr)+len(mdAppend)) + for _, v := range mdCurr { + md[v.ID] = v } - // Retrieve and return the updated record - rhp, err := t.recordHistory(token) - if err != nil { - return nil, fmt.Errorf("recordHistory: %v", err) + // Apply overwrites + for _, v := range mdOverwrite { + md[v.ID] = v } - log.Debugf("Record index:\n%v", rh.String()) + // Apply appends. Its ok if an append is specified but there is no + // existing metadata for that metadata stream. In this case the + // append data will become the full metadata stream. + for _, v := range mdAppend { + m, ok := md[v.ID] + if !ok { + // No existing metadata. Use append data as full metadata + // stream. + md[v.ID] = v + } - version := latestVersion(*rhp) - idx = rhp.Versions[version] - r, err := t.record(token, idx, version, rhp.State) - if err != nil { - return nil, fmt.Errorf("record: %v", err) + // Metadata exists. Append to it. + buf := bytes.NewBuffer([]byte(m.Payload)) + buf.WriteString(v.Payload) + m.Payload = buf.String() + md[v.ID] = m } - return r, nil + // Convert metadata back to a slice + metadata := make([]backend.MetadataStream, len(md)) + for _, v := range md { + metadata = append(metadata, v) + } + + return metadata } // New satisfies the Backend interface. +// TODO when does the signature get checked func (t *tlogbe) New(metadata []backend.MetadataStream, files []backend.File) (*backend.RecordMetadata, error) { log.Tracef("New") @@ -1081,13 +242,24 @@ func (t *tlogbe) New(metadata []backend.MetadataStream, files []backend.File) (* return nil, err } - // Create token - token, err := t.unvetted.tokenNew() - if err != nil { - return nil, err - } + // Create a new token + var token []byte + var treeID int64 + for retries := 0; retries < 10; retries++ { + treeID, err = t.unvetted.treeNew() + if err != nil { + return nil, err + } + token = tokenFromTreeID(treeID) - // TODO handle token prefix collisions + // Check for token prefix collisions + if !t.prefixExists(token) { + // Not a collision + break + } + + log.Infof("Token prefix collision %x, creating new token", token) + } // Create record metadata rm, err := recordMetadataNew(token, files, backend.MDStatusUnvetted, 1) @@ -1095,40 +267,19 @@ func (t *tlogbe) New(metadata []backend.MetadataStream, files []backend.File) (* return nil, err } - // Save record - r, err := t.unvetted.recordSave(token, metadata, files, *rm) + // Save the new version of the record + err = t.unvetted.recordSave(treeID, *rm, metadata, files) if err != nil { - return nil, fmt.Errorf("unvetted save %x: %v", token, err) + return nil, fmt.Errorf("recordSave %x: %v", token, err) } log.Infof("New record %x", token) - return &r.RecordMetadata, nil - /* - // Create a tree - tree, _, err := t.client.treeNew() - if err != nil { - return nil, fmt.Errorf("treeNew: %v", err) - } - token := tokenFromTreeID(tree.TreeId) - - // Save record - rh := recordHistoryNew(token) - rm, err := recordMetadataNew(token, files, backend.MDStatusUnvetted, 1) - if err != nil { - return nil, err - } - _, err = t.recordSave(token, metadata, files, *rm, rh) - if err != nil { - return nil, fmt.Errorf("recordSave %x: %v", token, err) - } - - log.Infof("New record tree:%v token:%x", tree.TreeId, token) - - return rm, nil - */ + return rm, nil } +// TODO Add UpdateUnvettedMetadata + func (t *tlogbe) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { log.Tracef("UpdateUnvettedRecord: %x", token) @@ -1142,8 +293,8 @@ func (t *tlogbe) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite []back return nil, err } // Allow ErrorStatusEmpty which indicates no new files are being - // added. This can happen when metadata only is being updated or - // when files are being deleted without any files being added. + // added. This can happen when files are being deleted without + // any new files being added. if e.ErrorCode != pd.ErrorStatusEmpty { return nil, err } @@ -1155,47 +306,44 @@ func (t *tlogbe) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite []back return nil, backend.ErrShutdown } - // Ensure unvetted record exists - rh, err := t.recordHistory(token) + // Get existing record + treeID := treeIDFromToken(token) + r, err := t.unvetted.recordLatest(treeID) if err != nil { - if err == store.ErrNotFound { + if err == errRecordNotFound { return nil, backend.ErrRecordNotFound } - return nil, fmt.Errorf("recordHistory %x: %v", token, err) - } - if rh.State != stateUnvetted { - return nil, backend.ErrRecordNotFound + return nil, fmt.Errorf("recordLatest: %v", err) } - // Get existing record - version := latestVersion(*rh) - if version != 1 { - // Unvetted records should only ever have a single version - return nil, fmt.Errorf("invalid unvetted record version: %v", version) - } - ri := rh.Versions[version] - r, err := t.record(token, ri, version, rh.State) - if err != nil { - return nil, fmt.Errorf("record %x %v: %v", token, version, err) - } + // Apply changes + metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) + files := filesUpdate(r.Files, filesAdd, filesDel) - // Update the record metadata - f := filesApplyChanges(r.Files, filesAdd, filesDel) - rm, err := recordMetadataNew(token, f, r.RecordMetadata.Status, - r.RecordMetadata.Iteration+1) + // Create record metadata + recordMD, err := recordMetadataNew(token, files, + backend.MDStatusIterationUnvetted, r.RecordMetadata.Iteration+1) if err != nil { return nil, err } - // Update record - r, err = t.recordUpdate(token, mdAppend, mdOverwrite, - filesAdd, filesDel, *rm, *rh) + // Save record + err = t.unvetted.recordSave(treeID, *recordMD, metadata, files) if err != nil { - return nil, err + if err == errNoFileChanges { + return nil, backend.ErrNoChanges + } + return nil, fmt.Errorf("recordSave: %v", err) } // TODO Call plugin hooks + // Get updated record + r, err = t.unvetted.recordLatest(treeID) + if err != nil { + return nil, fmt.Errorf("recordLatest: %v", err) + } + return r, nil } @@ -1212,8 +360,8 @@ func (t *tlogbe) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []backen return nil, err } // Allow ErrorStatusEmpty which indicates no new files are being - // being added. This can happen when files are being deleted - // without any new files being added. + // added. This can happen when files are being deleted without + // any new files being added. if e.ErrorCode != pd.ErrorStatusEmpty { return nil, err } @@ -1225,58 +373,44 @@ func (t *tlogbe) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []backen return nil, backend.ErrShutdown } - // Ensure record is vetted - rh, err := t.recordHistory(token) + // Get existing record + treeID := treeIDFromToken(token) + r, err := t.vetted.recordLatest(treeID) if err != nil { - if err == store.ErrNotFound { + if err == errRecordNotFound { return nil, backend.ErrRecordNotFound } - return nil, fmt.Errorf("recordHistory %x: %v", token, err) - } - if rh.State != stateVetted { - return nil, backend.ErrRecordNotFound - } - - // Get existing record - version := latestVersion(*rh) - ri := rh.Versions[version] - r, err := t.record(token, ri, version, rh.State) - if err != nil { - return nil, fmt.Errorf("record %x %v: %v", token, version, err) + return nil, fmt.Errorf("recordLatest: %v", err) } // Apply changes - files := filesApplyChanges(r.Files, filesAdd, filesDel) - metadata := metadataStreamsApplyChanges(r.Metadata, mdAppend, mdOverwrite) + metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) + files := filesUpdate(r.Files, filesAdd, filesDel) - // Ensure changes were actually made - m1, err := merkleRoot(r.Files) - if err != nil { - return nil, err - } - m2, err := merkleRoot(files) + // Create record metadata + recordMD, err := recordMetadataNew(token, files, + r.RecordMetadata.Status, r.RecordMetadata.Iteration+1) if err != nil { return nil, err } - if bytes.Equal(m1[:], m2[:]) { - return nil, backend.ErrNoChanges - } - // Create an updated record metadata - rm, err := recordMetadataNew(token, files, r.RecordMetadata.Status, - r.RecordMetadata.Iteration+1) + // Save record + err = t.unvetted.recordSave(treeID, *recordMD, metadata, files) if err != nil { - return nil, err + if err == errNoFileChanges { + return nil, backend.ErrNoChanges + } + return nil, fmt.Errorf("recordSave: %v", err) } - // Save a new version of the record - r, err = t.recordSave(token, metadata, files, *rm, *rh) + // TODO Call plugin hooks + + // Get updated record + r, err = t.unvetted.recordLatest(treeID) if err != nil { - return nil, fmt.Errorf("recordSave %v: %v", err) + return nil, fmt.Errorf("recordLatest: %v", err) } - // TODO Call plugin hooks - return r, nil } @@ -1293,11 +427,17 @@ func (t *tlogbe) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite []back return err } // Allow ErrorStatusEmpty which indicates no new files are being - // being added. This is expected. + // being added. This is expected since this is a metadata only + // update. if e.ErrorCode != pd.ErrorStatusEmpty { return err } } + if len(mdAppend) == 0 && len(mdOverwrite) == 0 { + return backend.ContentVerificationError{ + ErrorCode: pd.ErrorStatusNoChanges, + } + } t.Lock() defer t.Unlock() @@ -1305,137 +445,143 @@ func (t *tlogbe) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite []back return backend.ErrShutdown } - // Ensure record is vetted - rh, err := t.recordHistory(token) + // Get existing record + treeID := treeIDFromToken(token) + r, err := t.vetted.recordLatest(treeID) if err != nil { - if err == store.ErrNotFound { + if err == errRecordNotFound { return backend.ErrRecordNotFound } - return fmt.Errorf("recordHistory %x: %v", token, err) - } - if rh.State != stateVetted { - return backend.ErrRecordNotFound - } - - // Get current record metadata. This will remain unchanged, but we - // need it for the recordUpdate() call. - version := latestVersion(*rh) - ri := rh.Versions[version] - - rm, err := t.recordMetadata(token, ri.RecordMetadata) - if err != nil { - return fmt.Errorf("recordMetadata %v: %v", ri.RecordMetadata, err) + return fmt.Errorf("recordLatest: %v", err) } - // Update record - _, err = t.recordUpdate(token, mdAppend, mdOverwrite, - []backend.File{}, []string{}, *rm, *rh) - if err != nil { - return err - } + // Apply changes + metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) - return nil + // Update metadata + return t.vetted.recordMetadataUpdate(treeID, r.RecordMetadata, metadata) } func (t *tlogbe) UpdateReadme(content string) error { return fmt.Errorf("not implemented") } +// unvettedRecordExists returns whether the provided token corresponds to an +// unvetted record. +func (t *tlogbe) unvettedExists(token []byte) bool { + // TODO + return false +} + func (t *tlogbe) UnvettedExists(token []byte) bool { log.Tracef("UnvettedExists %x", token) + // TODO must check that it exists and it is not a vetted record too + // Checking if its frozen is not enough. A unvetted tree will be + // frozen when it is censored or abandoned. - rh, err := t.recordHistory(token) + return false +} + +func (t *tlogbe) vettedExists(token []byte) bool { + tk := hex.EncodeToString(token) + if _, ok := t.vettedTreesGet(tk); ok { + return true + } + + // Just because the token is not in the vetted trees cache does not + // mean a vetted tree does not exist. The cache is lazy loaded. + // Check if there is an unvetted tree for the token and if the last + // leaf of the unvetted tree is a freeze record. + treeID := treeIDFromToken(token) + fr, err := t.unvetted.freezeRecord(treeID) if err != nil { - if err != store.ErrNotFound { - log.Errorf("UnvettedExists: recordHistory %x: %v", token, err) + if err == errFreezeRecordNotFound { + // Unvetted tree is not frozen. Record is still unvetted. + return false } + + // Unexpected error + e := fmt.Sprintf("vettedExists %x: freezeRecord: %v", token, err) + panic(e) + } + + // Unvetted tree is frozen. Check if the freeze record points to a + // vetted tree ID or if it is blank. A vetted tree ID indicates the + // record was made public. A blank tree ID indicates the tree was + // frozen for some other reason such as if the record was censored + // or abandoned. + if fr.TreeID == 0 { + // No vetted tree pointer found return false } - if rh.State == stateUnvetted { - return true + + // Ensure vetted record exists + if !t.vetted.treeExists(fr.TreeID) { + // Uh oh. A freeze record points to this tree ID but the tree + // does not exist. Not good. + e := fmt.Sprintf("freeze record points to invalid tree %v", fr.TreeID) + panic(e) } - return false + // Cache the vetted tree ID + t.vettedTreesSet(hex.EncodeToString(token), fr.TreeID) + + return true } func (t *tlogbe) VettedExists(token []byte) bool { log.Tracef("VettedExists %x", token) - rh, err := t.recordHistory(token) - if err != nil { - if err != store.ErrNotFound { - log.Errorf("VettedExists: recordHistory %x: %v", token, err) - } - return false - } - if rh.State == stateVetted { - return true - } - return false } func (t *tlogbe) GetUnvetted(token []byte) (*backend.Record, error) { log.Tracef("GetUnvetted: %x", token) - rh, err := t.recordHistory(token) - if err != nil { - if err == store.ErrNotFound { - return nil, backend.ErrRecordNotFound - } - return nil, fmt.Errorf("recordHistory %x: %v", token, err) - } - if rh.State != stateUnvetted { - return nil, backend.ErrRecordNotFound - } - version := latestVersion(*rh) - ri := rh.Versions[version] - - r, err := t.record(token, ri, version, rh.State) - if err != nil { - return nil, fmt.Errorf("record %x %v: %v", token, version, err) - } - - return r, nil + return nil, nil } func (t *tlogbe) GetVetted(token []byte, version string) (*backend.Record, error) { log.Tracef("GetVetted: %x", token) - // Ensure record is vetted - rh, err := t.recordHistory(token) + return nil, nil +} + +// This function must be called with the read/write lock held. +func (t *tlogbe) unvettedPublish(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { + // Create a vetted record + // TODO check for collisions + treeID, err := t.vetted.treeNew() if err != nil { - if err == store.ErrNotFound { - return nil, backend.ErrRecordNotFound - } - return nil, fmt.Errorf("recordHistory %x: %v", token, err) - } - if rh.State != stateVetted { - return nil, backend.ErrRecordNotFound + return fmt.Errorf("vetted recordNew: %v", err) } - // Lookup record. If no version was specified, return the latest - // version. - var v uint32 - if version == "" { - v = latestVersion(*rh) - } else { - vr, err := strconv.ParseUint(version, 10, 32) - if err != nil { - return nil, backend.ErrRecordNotFound - } - v = uint32(vr) - } - ri, ok := rh.Versions[v] - if !ok { - return nil, backend.ErrRecordNotFound - } - r, err := t.record(token, ri, uint32(v), rh.State) + // Save the record as vetted + err = t.vetted.recordSave(treeID, rm, metadata, files) if err != nil { - return nil, fmt.Errorf("record %x %v: %v", token, version, err) + return fmt.Errorf("vetted recordSave: %v", err) } - return r, nil + // Freeze the unvetted tree + fr := freezeRecord{ + TreeID: treeID, + } + _ = fr + + return nil +} + +// This function must be called with the read/write lock held. +func (t *tlogbe) unvettedCensor() error { + // Freeze tree + // Delete the censored blobs + return nil +} + +// This function must be called with the read/write lock held. +func (t *tlogbe) unvettedArchive() error { + // Freeze tree + return nil } func (t *tlogbe) SetUnvettedStatus(token []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { @@ -1448,25 +594,13 @@ func (t *tlogbe) SetUnvettedStatus(token []byte, status backend.MDStatusT, mdApp return nil, backend.ErrShutdown } - // Ensure record is unvetted - rh, err := t.recordHistory(token) - if err != nil { - if err == store.ErrNotFound { - return nil, backend.ErrRecordNotFound - } - return nil, fmt.Errorf("recordHistory %x: %v", token, err) - } - if rh.State != stateUnvetted { - return nil, backend.ErrRecordNotFound - } - - // Get the current record metadata - idx := rh.Versions[latestVersion(*rh)] - rm, err := t.recordMetadata(token, idx.RecordMetadata) + // Get existing record + treeID := treeIDFromToken(token) + r, err := t.unvetted.recordLatest(treeID) if err != nil { - return nil, fmt.Errorf("recordMetadata %v: %v", - idx.RecordMetadata, err) + return nil, fmt.Errorf("recordLatest: %v", err) } + rm := r.RecordMetadata // Validate status change if !statusChangeIsAllowed(rm.Status, status) { @@ -1476,17 +610,59 @@ func (t *tlogbe) SetUnvettedStatus(token []byte, status backend.MDStatusT, mdApp } } - log.Debugf("Status change %x from %v (%v) to %v (%v)", token, - backend.MDStatus[rm.Status], rm.Status, backend.MDStatus[status], status) + log.Debugf("Status change %x from %v (%v) to %v (%v)", + token, backend.MDStatus[rm.Status], rm.Status, + backend.MDStatus[status], status) // Apply status change rm.Status = status rm.Iteration += 1 rm.Timestamp = time.Now().Unix() - // Update record - return t.recordUpdate(token, mdAppend, mdOverwrite, - []backend.File{}, []string{}, *rm, *rh) + // Apply metdata changes + metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) + + switch status { + case backend.MDStatusVetted: + err := t.unvettedPublish(token, rm, metadata, r.Files) + if err != nil { + return nil, fmt.Errorf("publish: %v", err) + } + case backend.MDStatusCensored: + err := t.unvettedCensor() + if err != nil { + return nil, fmt.Errorf("censor: %v", err) + } + case backend.MDStatusArchived: + err := t.unvettedArchive() + if err != nil { + return nil, fmt.Errorf("archive: %v", err) + } + default: + return nil, fmt.Errorf("unknown status: %v (%v)", + backend.MDStatus[status], status) + } + + // Return the record + r, err = t.unvetted.recordLatest(treeID) + if err != nil { + return nil, fmt.Errorf("recordLatest: %v", err) + } + + return r, nil +} + +// This function must be called with the read/write lock held. +func (t *tlogbe) vettedCensor() error { + // Freeze tree + // Delete the censored blobs + return nil +} + +// This function must be called with the read/write lock held. +func (t *tlogbe) vettedArchive() error { + // Freeze tree + return nil } func (t *tlogbe) SetVettedStatus(token []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { @@ -1499,109 +675,34 @@ func (t *tlogbe) SetVettedStatus(token []byte, status backend.MDStatusT, mdAppen return nil, backend.ErrShutdown } - // Ensure record is vetted - rh, err := t.recordHistory(token) - if err != nil { - if err == store.ErrNotFound { - return nil, backend.ErrRecordNotFound + /* + // Validate status change + if !statusChangeIsAllowed(rm.Status, status) { + return nil, backend.StateTransitionError{ + From: rm.Status, + To: status, + } } - return nil, fmt.Errorf("recordHistory %x: %v", token, err) - } - if rh.State != stateVetted { - return nil, backend.ErrRecordNotFound - } - // Get the current record metadata - idx := rh.Versions[latestVersion(*rh)] - rm, err := t.recordMetadata(token, idx.RecordMetadata) - if err != nil { - return nil, fmt.Errorf("recordMetadata %v: %v", - idx.RecordMetadata, err) - } - - // Validate status change - if !statusChangeIsAllowed(rm.Status, status) { - return nil, backend.StateTransitionError{ - From: rm.Status, - To: status, - } - } + log.Debugf("Status change %x from %v (%v) to %v (%v)", token, + backend.MDStatus[rm.Status], rm.Status, backend.MDStatus[status], status) - log.Debugf("Status change %x from %v (%v) to %v (%v)", token, - backend.MDStatus[rm.Status], rm.Status, backend.MDStatus[status], status) + // Apply status change + rm.Status = status + rm.Iteration += 1 + rm.Timestamp = time.Now().Unix() - // Apply status change - rm.Status = status - rm.Iteration += 1 - rm.Timestamp = time.Now().Unix() + // Update metadata + */ - // Update record - return t.recordUpdate(token, mdAppend, mdOverwrite, - []backend.File{}, []string{}, *rm, *rh) + return nil, nil } func (t *tlogbe) Inventory(vettedCount uint, branchCount uint, includeFiles, allVersions bool) ([]backend.Record, []backend.Record, error) { log.Tracef("Inventory: %v %v", includeFiles, allVersions) - // vettedCount specifies the last N vetted records that should be - // returned. branchCount specifies the last N branches, which in - // gitbe correspond to unvetted records. Neither of these are - // implemented in gitbe so they will not be implemented here - // either. They can be added in the future if they are needed. - switch { - case vettedCount != 0: - return nil, nil, fmt.Errorf("vetted count is not implemented") - case branchCount != 0: - return nil, nil, fmt.Errorf("branch count is not implemented") - } - - // Get all record histories from the store - hists := make([]recordHistory, 0, 1024) - err := t.store.Enum(func(key string, blob []byte) error { - if strings.HasPrefix(key, keyPrefixRecordHistory) { - // This is a record history blob. Decode and save it. - var rh recordHistory - err := json.Unmarshal(blob, &rh) - if err != nil { - return fmt.Errorf("unmarshal recordHistory %v: %v", key, err) - } - hists = append(hists, rh) - } - return nil - }) - if err != nil { - return nil, nil, fmt.Errorf("store Enum: %v", err) - } - - // Retreive the records - unvetted := make([]backend.Record, 0, len(hists)) - vetted := make([]backend.Record, 0, len(hists)) - for _, rh := range hists { - for version, idx := range rh.Versions { - if !allVersions && version != latestVersion(rh) { - continue - } - r, err := t.record(rh.Token, idx, version, rh.State) - if err != nil { - return nil, nil, fmt.Errorf("record %v %v: %v", - rh.Token, version, err) - } - if !includeFiles { - r.Files = []backend.File{} - } - switch rh.State { - case stateUnvetted: - unvetted = append(unvetted, *r) - case stateVetted: - vetted = append(vetted, *r) - default: - return nil, nil, fmt.Errorf("unknown record history state %v: %v", - rh.Token, rh.State) - } - } - } - - return vetted, unvetted, nil + // return vetted, unvetted, nil + return nil, nil, nil } func (t *tlogbe) GetPlugins() ([]backend.Plugin, error) { @@ -1629,11 +730,9 @@ func (t *tlogbe) Close() { // Shutdown backend t.shutdown = true - // Close trillian connection - t.tlog.Close() - - // Zero out encryption key - t.encryptionKey.Zero() + // Close out tlog connections + t.unvetted.close() + t.vetted.close() } func New(homeDir, dataDir, trillianHost, trillianKeyFile, dcrtimeHost, encryptionKeyFile string) (*tlogbe, error) { @@ -1696,27 +795,27 @@ func New(homeDir, dataDir, trillianHost, trillianKeyFile, dcrtimeHost, encryptio } log.Infof("Anchor host: %v", dcrtimeHost) + _ = encryptionKey + _ = dcrtimeHost + _ = store + _ = tlog t := tlogbe{ - homeDir: homeDir, - dataDir: dataDir, - encryptionKey: encryptionKey, - dcrtimeHost: dcrtimeHost, - store: store, - tlog: tlog, - cron: cron.New(), - } - - // Launch cron - log.Infof("Launch cron anchor job") - err = t.cron.AddFunc(anchorSchedule, func() { - t.anchorTrees() - }) - if err != nil { - return nil, err + homeDir: homeDir, + dataDir: dataDir, + // cron: cron.New(), } - t.cron.Start() - // TODO fsck + /* + // Launch cron + log.Infof("Launch cron anchor job") + err = t.cron.AddFunc(anchorSchedule, func() { + // t.anchorTrees() + }) + if err != nil { + return nil, err + } + t.cron.Start() + */ return &t, nil } diff --git a/politeiad/backend/tlogbe/trillian.go b/politeiad/backend/tlogbe/trillian.go index 80c7c20b3..a0429f9af 100644 --- a/politeiad/backend/tlogbe/trillian.go +++ b/politeiad/backend/tlogbe/trillian.go @@ -70,9 +70,10 @@ func merkleLeafHash(leafValue []byte) []byte { return h.Sum(nil) } -func logLeafNew(value []byte) *trillian.LogLeaf { +func logLeafNew(value []byte, extraData []byte) *trillian.LogLeaf { return &trillian.LogLeaf{ LeafValue: value, + ExtraData: extraData, } } @@ -268,12 +269,12 @@ func (t *TrillianClient) SignedLogRoot(treeID int64) (*trillian.SignedLogRoot, * return slr, lr, nil } -// LeavesAppend appends the provided leaves onto the provided tree. The leaf -// and the inclusion proof for the leaf are returned. If a leaf was not -// successfully appended, the leaf will be returned without an inclusion proof. -// The error status code can be found in the returned leaf. Note leaves that -// are duplicates will fail and it is the callers responsibility to determine -// how they should be handled. +// LeavesAppend appends the provided leaves onto the provided tree. The queued +// leaf and the leaf inclusion proof are returned. If a leaf was not +// successfully appended, the queued leaf will still be returned and the error +// will be in the queued leaf. Inclusion proofs will not exist for leaves that +// fail to be appended. Note leaves that are duplicates will fail and it is the +// callers responsibility to determine how they should be handled. func (t *TrillianClient) LeavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]QueuedLeafProof, *types.LogRootV1, error) { log.Tracef("LeavesAppend: %v", treeID) @@ -287,6 +288,11 @@ func (t *TrillianClient) LeavesAppend(treeID int64, leaves []*trillian.LogLeaf) return nil, nil, fmt.Errorf("signedLogRoot pre update: %v", err) } + // Ensure the tree is not frozen + if tree.TreeState == trillian.TreeState_FROZEN { + return nil, nil, fmt.Errorf("tree is frozen") + } + log.Debugf("Appending %v leaves to tree id %v", len(leaves), treeID) // Append leaves to log @@ -372,6 +378,19 @@ func (t *TrillianClient) LeavesAppend(treeID int64, leaves []*trillian.LogLeaf) return proofs, lrv1, nil } +func (t *TrillianClient) leavesByRange(treeID int64, startIndex, count int64) ([]*trillian.LogLeaf, error) { + glbrr, err := t.client.GetLeavesByRange(t.ctx, + &trillian.GetLeavesByRangeRequest{ + LogId: treeID, + StartIndex: startIndex, + Count: count, + }) + if err != nil { + return nil, err + } + return glbrr.Leaves, nil +} + // LeavesAll returns all of the leaves for the provided treeID. func (t *TrillianClient) LeavesAll(treeID int64) ([]*trillian.LogLeaf, error) { // Get log root @@ -381,17 +400,7 @@ func (t *TrillianClient) LeavesAll(treeID int64) ([]*trillian.LogLeaf, error) { } // Get all leaves - glbrr, err := t.client.GetLeavesByRange(t.ctx, - &trillian.GetLeavesByRangeRequest{ - LogId: treeID, - StartIndex: 0, - Count: int64(lr.TreeSize), - }) - if err != nil { - return nil, fmt.Errorf("GetLeavesByRangeRequest: %v", err) - } - - return glbrr.Leaves, nil + return t.leavesByRange(treeID, 0, int64(lr.TreeSize)) } // LeafProofs returns the LeafProofs for the provided treeID and merkle leaf diff --git a/politeiad/backend/util.go b/politeiad/backend/util.go deleted file mode 100644 index 1ac1aa15f..000000000 --- a/politeiad/backend/util.go +++ /dev/null @@ -1,168 +0,0 @@ -package backend - -import ( - "bytes" - "encoding/base64" - "path/filepath" - "strconv" - - pd "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/politeiad/api/v1/mime" - "github.com/decred/politeia/util" - "github.com/subosito/gozaru" -) - -// VerifyContent verifies that all provided MetadataStream and File are sane. -func VerifyContent(metadata []MetadataStream, files []File, filesDel []string) error { - // Make sure all metadata is within maxima. - for _, v := range metadata { - if v.ID > pd.MetadataStreamsMax-1 { - return ContentVerificationError{ - ErrorCode: pd.ErrorStatusInvalidMDID, - ErrorContext: []string{ - strconv.FormatUint(v.ID, 10), - }, - } - } - } - for i := range metadata { - for j := range metadata { - // Skip self and non duplicates. - if i == j || metadata[i].ID != metadata[j].ID { - continue - } - return ContentVerificationError{ - ErrorCode: pd.ErrorStatusDuplicateMDID, - ErrorContext: []string{ - strconv.FormatUint(metadata[i].ID, 10), - }, - } - } - } - - // Prevent paths - for i := range files { - if filepath.Base(files[i].Name) != files[i].Name { - return ContentVerificationError{ - ErrorCode: pd.ErrorStatusInvalidFilename, - ErrorContext: []string{ - files[i].Name, - }, - } - } - } - for _, v := range filesDel { - if filepath.Base(v) != v { - return ContentVerificationError{ - ErrorCode: pd.ErrorStatusInvalidFilename, - ErrorContext: []string{ - v, - }, - } - } - } - - // Now check files - if len(files) == 0 { - return ContentVerificationError{ - ErrorCode: pd.ErrorStatusEmpty, - } - } - - // Prevent bad filenames and duplicate filenames - for i := range files { - for j := range files { - if i == j { - continue - } - if files[i].Name == files[j].Name { - return ContentVerificationError{ - ErrorCode: pd.ErrorStatusDuplicateFilename, - ErrorContext: []string{ - files[i].Name, - }, - } - } - } - // Check against filesDel - for _, v := range filesDel { - if files[i].Name == v { - return ContentVerificationError{ - ErrorCode: pd.ErrorStatusDuplicateFilename, - ErrorContext: []string{ - files[i].Name, - }, - } - } - } - } - - for i := range files { - if gozaru.Sanitize(files[i].Name) != files[i].Name { - return ContentVerificationError{ - ErrorCode: pd.ErrorStatusInvalidFilename, - ErrorContext: []string{ - files[i].Name, - }, - } - } - - // Validate digest - d, ok := util.ConvertDigest(files[i].Digest) - if !ok { - return ContentVerificationError{ - ErrorCode: pd.ErrorStatusInvalidFileDigest, - ErrorContext: []string{ - files[i].Name, - }, - } - } - - // Decode base64 payload - var err error - payload, err := base64.StdEncoding.DecodeString(files[i].Payload) - if err != nil { - return ContentVerificationError{ - ErrorCode: pd.ErrorStatusInvalidBase64, - ErrorContext: []string{ - files[i].Name, - }, - } - } - - // Calculate payload digest - dp := util.Digest(payload) - if !bytes.Equal(d[:], dp) { - return ContentVerificationError{ - ErrorCode: pd.ErrorStatusInvalidFileDigest, - ErrorContext: []string{ - files[i].Name, - }, - } - } - - // Verify MIME - detectedMIMEType := mime.DetectMimeType(payload) - if detectedMIMEType != files[i].MIME { - return ContentVerificationError{ - ErrorCode: pd.ErrorStatusInvalidMIMEType, - ErrorContext: []string{ - files[i].Name, - detectedMIMEType, - }, - } - } - - if !mime.MimeValid(files[i].MIME) { - return ContentVerificationError{ - ErrorCode: pd.ErrorStatusUnsupportedMIMEType, - ErrorContext: []string{ - files[i].Name, - files[i].MIME, - }, - } - } - } - - return nil -} diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 7539da261..2f7a8f28b 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -1156,7 +1156,7 @@ func _main() error { case backendGit: b, err := gitbe.New(activeNetParams.Params, loadedCfg.DataDir, loadedCfg.DcrtimeHost, "", p.identity, loadedCfg.GitTrace, - loadedCfg.DcrdataHost) + loadedCfg.DcrdataHost, loadedCfg.Mode) if err != nil { return fmt.Errorf("new gitbe: %v", err) } diff --git a/util/convert.go b/util/convert.go index eaa11ed1d..b69fce20a 100644 --- a/util/convert.go +++ b/util/convert.go @@ -10,6 +10,7 @@ import ( "fmt" v1 "github.com/decred/dcrtime/api/v1" + pd "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/identity" ) From 9177c657a930203ffffe59bfae84d6486c35d5e5 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 27 Jul 2020 06:57:30 -0600 Subject: [PATCH 006/449] implement comments plugin --- plugins/comments/comments.go | 435 ++++- politeiad/backend/backend.go | 8 +- politeiad/backend/tlogbe/anchor.go | 193 +-- politeiad/backend/tlogbe/blobentry.go | 80 - politeiad/backend/tlogbe/encryptionkey.go | 28 +- .../tlogbe/plugin/comments/comments.go | 1441 +++++++++++++++-- .../backend/tlogbe/plugin/comments/journal.go | 134 +- politeiad/backend/tlogbe/plugin/plugin.go | 34 +- politeiad/backend/tlogbe/pluginclient.go | 289 ++++ politeiad/backend/tlogbe/store/store.go | 70 +- politeiad/backend/tlogbe/tlog.go | 1059 ++++++------ politeiad/backend/tlogbe/tlogbe.go | 501 ++++-- politeiad/backend/tlogbe/trillian.go | 117 +- politeiad/log.go | 6 +- 14 files changed, 3217 insertions(+), 1178 deletions(-) delete mode 100644 politeiad/backend/tlogbe/blobentry.go create mode 100644 politeiad/backend/tlogbe/pluginclient.go diff --git a/plugins/comments/comments.go b/plugins/comments/comments.go index fc4c1e48d..0c5ccd592 100644 --- a/plugins/comments/comments.go +++ b/plugins/comments/comments.go @@ -1,15 +1,15 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + package comments import ( - "encoding/hex" "encoding/json" "fmt" - "strconv" - - "github.com/decred/politeia/politeiad/api/v1/identity" - "github.com/decred/politeia/util" ) +type VoteT int type ErrorStatusT int const ( @@ -17,101 +17,87 @@ const ( ID = "comments" // Plugin commands - CmdNew = "new" // Create a new comment - CmdGet = "get" // Get comments - CmdEdit = "edit" // Edit a comment - CmdDel = "del" // Delete a comment - CmdExists = "exists" // Does a comment exist - CmdVote = "vote" // Vote on a comment - CmdCensor = "censor" // Censor a comment - CmdCount = "count" // Get comments count - CmdProofs = "proofs" // Get comment proofs + CmdNew = "new" // Create a new comment + CmdEdit = "edit" // Edit a comment + CmdDel = "del" // Del a comment + CmdGet = "get" // Get specified comments + CmdGetAll = "getall" // Get all comments for a record + CmdGetVersion = "getversion" // Get specified version of a comment + CmdCount = "count" // Get comments count for a record + CmdVote = "vote" // Vote on a comment + CmdProofs = "proofs" // Get inclusion proofs + + // Comment vote types + VoteInvalid VoteT = 0 + VoteDownvote VoteT = -1 + VoteUpvote VoteT = 1 + + // PolicayMaxVoteChanges is the maximum number times a user can + // change their vote on a comment. This prevents a malicious user + // from being able to spam comment votes. + PolicyMaxVoteChanges = 5 // Error status codes ErrorStatusInvalid ErrorStatusT = 0 - ErrorStatusPublicKeyInvalid ErrorStatusT = 1 - ErrorStatusSignatureInvalid ErrorStatusT = 2 - ErrorStatusRecordNotFound ErrorStatusT = 3 - ErrorStatusCommentNotFound ErrorStatusT = 4 + ErrorStatusTokenInvalid ErrorStatusT = 1 + ErrorStatusPublicKeyInvalid ErrorStatusT = 2 + ErrorStatusSignatureInvalid ErrorStatusT = 3 + ErrorStatusRecordNotFound ErrorStatusT = 4 + ErrorStatusCommentNotFound ErrorStatusT = 5 + ErrorStatusParentIDInvalid ErrorStatusT = 6 + ErrorStatusNoCommentChanges ErrorStatusT = 7 + ErrorStatusVoteInvalid ErrorStatusT = 8 + ErrorStatusMaxVoteChanges ErrorStatusT = 9 ) var ( // Human readable error messages ErrorStatus = map[ErrorStatusT]string{ + ErrorStatusInvalid: "invalid error status", + ErrorStatusTokenInvalid: "invalid token", ErrorStatusPublicKeyInvalid: "invalid public key", ErrorStatusSignatureInvalid: "invalid signature", ErrorStatusRecordNotFound: "record not found", ErrorStatusCommentNotFound: "comment not found", + ErrorStatusParentIDInvalid: "parent id invalid", + ErrorStatusNoCommentChanges: "comment did not change", + ErrorStatusVoteInvalid: "invalid vote", + ErrorStatusMaxVoteChanges: "user has changed their vote too many times", } ) -// PluginError is emitted when input provided to a plugin command is invalid. -type PluginError struct { +// UserError represents an error that is cause by something that the user did. +type UserError struct { ErrorCode ErrorStatusT ErrorContext []string } // Error satisfies the error interface. -func (e PluginError) Error() string { +func (e UserError) Error() string { return fmt.Sprintf("plugin error code: %v", e.ErrorCode) } -// Comment represent a user submitted comment and includes both the user -// generated comment data and the server generated metadata. +// Comment represent a record comment. type Comment struct { - // Data generated by client Token string `json:"token"` // Record token - ParentID uint32 `json:"parentid"` // Parent comment ID - Comment string `json:"comment"` // Comment - PublicKey string `json:"publickey"` // Pubkey used for Signature + ParentID uint32 `json:"parentid"` // Parent comment ID if reply + Comment string `json:"comment"` // Comment text + PublicKey string `json:"publickey"` // Public key used for Signature Signature string `json:"signature"` // Signature of Token+ParentID+Comment - - // Metadata generated by server CommentID uint32 `json:"commentid"` // Comment ID Version uint32 `json:"version"` // Comment version Receipt string `json:"receipt"` // Server signature of client signature - Timestamp int64 `json:"timestamp"` // Received UNIX timestamp - Score int32 `json:"score"` // Vote score - Deleted bool `json:"deleted"` // Comment has been deleted by author - Censored bool `json:"censored"` // Comment has been censored by admin -} - -func VerifyCommentSignature(signature, pubkey, token string, parentID uint32, comment string) error { - sig, err := util.ConvertSignature(signature) - if err != nil { - return PluginError{ - ErrorCode: ErrorStatusSignatureInvalid, - ErrorContext: []string{err.Error()}, - } - } - b, err := hex.DecodeString(pubkey) - if err != nil { - return PluginError{ - ErrorCode: ErrorStatusPublicKeyInvalid, - ErrorContext: []string{"key is not hex"}, - } - } - pk, err := identity.PublicIdentityFromBytes(b) - if err != nil { - return PluginError{ - ErrorCode: ErrorStatusPublicKeyInvalid, - ErrorContext: []string{err.Error()}, - } - } - msg := token + strconv.FormatUint(uint64(parentID), 10) + comment - if !pk.VerifyMessage([]byte(msg), sig) { - return PluginError{ - ErrorCode: ErrorStatusSignatureInvalid, - ErrorContext: []string{err.Error()}, - } - } - return nil + Timestamp int64 `json:"timestamp"` // UNIX timestamp of last edit + Score int64 `json:"score"` // Vote score + Deleted bool `json:"deleted"` // Comment has been deleted } +// New creates a new comment. A parent ID of 0 indicates that the comment is +// a base level comment and not a reply commment. type New struct { Token string `json:"token"` // Record token ParentID uint32 `json:"parentid"` // Parent comment ID - Comment string `json:"comment"` // Comment + Comment string `json:"comment"` // Comment text PublicKey string `json:"publickey"` // Pubkey used for Signature Signature string `json:"signature"` // Signature of Token+ParentID+Comment } @@ -133,8 +119,8 @@ func DecodeNew(b []byte) (*New, error) { // NewReply is the reply to the New command. type NewReply struct { - CommentID string `json:"commentid"` // Comment ID - Receipt string `json:"receipt"` // Server signature of comment signature + CommentID uint32 `json:"commentid"` // Comment ID + Receipt string `json:"receipt"` // Server sig of client sig Timestamp int64 `json:"timestamp"` // Received UNIX timestamp } @@ -152,3 +138,316 @@ func DecodeNewReply(b []byte) (*NewReply, error) { } return &r, nil } + +// Edit edits an existing comment. +type Edit struct { + Token string `json:"token"` // Record token + ParentID uint32 `json:"parentid"` // Parent comment ID + CommentID uint32 `json:"commentid"` // Comment ID + Comment string `json:"comment"` // Comment text + PublicKey string `json:"publickey"` // Pubkey used for Signature + Signature string `json:"signature"` // Signature of Token+ParentID+Comment +} + +// EncodeEdit encodes a Edit into a JSON byte slice. +func EncodeEdit(e Edit) ([]byte, error) { + return json.Marshal(e) +} + +// DecodeEdit decodes a JSON byte slice into a Edit. +func DecodeEdit(b []byte) (*Edit, error) { + var e Edit + err := json.Unmarshal(b, &e) + if err != nil { + return nil, err + } + return &e, nil +} + +// EditReply is the reply to the Edit command. +type EditReply struct { + Version uint32 `json:"version"` // Comment version + Receipt string `json:"receipt"` // Server signature of client signature + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp +} + +// EncodeEdit encodes a EditReply into a JSON byte slice. +func EncodeEditReply(r EditReply) ([]byte, error) { + return json.Marshal(r) +} + +// DecodeEdit decodes a JSON byte slice into a EditReply. +func DecodeEditReply(b []byte) (*EditReply, error) { + var r EditReply + err := json.Unmarshal(b, &r) + if err != nil { + return nil, err + } + return &r, nil +} + +// Del permanently deletes all versions of the provided comment. +type Del struct { + Token string `json:"token"` // Record token + CommentID uint32 `json:"commentid"` // Comment ID + Reason string `json:"reason"` // Reason for deletion + PublicKey string `json:"publickey"` // Public key used for signature + Signature string `json:"signature"` // Signature of Token+CommentID+Reason +} + +// EncodeDel encodes a Del into a JSON byte slice. +func EncodeDel(d Del) ([]byte, error) { + return json.Marshal(d) +} + +// DecodeDel decodes a JSON byte slice into a Del. +func DecodeDel(b []byte) (*Del, error) { + var d Del + err := json.Unmarshal(b, &d) + if err != nil { + return nil, err + } + return &d, nil +} + +// DelReply is the reply to the Del command. +type DelReply struct { + Receipt string `json:"receipt"` // Server signature of client signature + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp +} + +// EncodeDelReply encodes a DelReply into a JSON byte slice. +func EncodeDelReply(d DelReply) ([]byte, error) { + return json.Marshal(d) +} + +// DecodeDelReply decodes a JSON byte slice into a DelReply. +func DecodeDelReply(b []byte) (*DelReply, error) { + var d DelReply + err := json.Unmarshal(b, &d) + if err != nil { + return nil, err + } + return &d, nil +} + +// Get returns the latest version of the comments for the provided comment IDs. +// An error is not returned if a comment is not found for one or more of the +// comment IDs. Those entries will simply not be included in the reply. +type Get struct { + Token string `json:"token"` + CommentIDs []uint32 `json:"commentids"` +} + +// EncodeGet encodes a Get into a JSON byte slice. +func EncodeGet(g Get) ([]byte, error) { + return json.Marshal(g) +} + +// DecodeGet decodes a JSON byte slice into a Get. +func DecodeGet(b []byte) (*Get, error) { + var g Get + err := json.Unmarshal(b, &g) + if err != nil { + return nil, err + } + return &g, nil +} + +// GetReply is the reply to the Get command. The returned map will not include +// an entry for any comment IDs that did not correspond to an actual comment. +// It is the responsibility of the caller to ensure that a comment was returned +// for all of the provided comment IDs. +type GetReply struct { + Comments map[uint32]Comment `json:"comments"` // [commentID]Comment +} + +// EncodeGetReply encodes a GetReply into a JSON byte slice. +func EncodeGetReply(g GetReply) ([]byte, error) { + return json.Marshal(g) +} + +// DecodeGetReply decodes a JSON byte slice into a GetReply. +func DecodeGetReply(b []byte) (*GetReply, error) { + var g GetReply + err := json.Unmarshal(b, &g) + if err != nil { + return nil, err + } + return &g, nil +} + +// GetAll returns the latest version off all comments for the provided record. +type GetAll struct { + Token string `json:"token"` +} + +// EncodeGetAll encodes a GetAll into a JSON byte slice. +func EncodeGetAll(g GetAll) ([]byte, error) { + return json.Marshal(g) +} + +// DecodeGetAll decodes a JSON byte slice into a GetAll. +func DecodeGetAll(b []byte) (*GetAll, error) { + var g GetAll + err := json.Unmarshal(b, &g) + if err != nil { + return nil, err + } + return &g, nil +} + +// GetAllReply is the reply to the GetAll command. +type GetAllReply struct { + Comments []Comment `json:"comments"` +} + +// EncodeGetAllReply encodes a GetAllReply into a JSON byte slice. +func EncodeGetAllReply(g GetAllReply) ([]byte, error) { + return json.Marshal(g) +} + +// DecodeGetAllReply decodes a JSON byte slice into a GetAllReply. +func DecodeGetAllReply(b []byte) (*GetAllReply, error) { + var g GetAllReply + err := json.Unmarshal(b, &g) + if err != nil { + return nil, err + } + return &g, nil +} + +// GetVersion returns a specific version of a comment. +type GetVersion struct { + Token string `json:"token"` + CommentID uint32 `json:"commentid"` + Version uint32 `json:"version"` +} + +// EncodeGetVersion encodes a GetVersion into a JSON byte slice. +func EncodeGetVersion(g GetVersion) ([]byte, error) { + return json.Marshal(g) +} + +// DecodeGetVersion decodes a JSON byte slice into a GetVersion. +func DecodeGetVersion(b []byte) (*GetVersion, error) { + var g GetVersion + err := json.Unmarshal(b, &g) + if err != nil { + return nil, err + } + return &g, nil +} + +// GetVersionReply is the reply to the GetVersion command. +type GetVersionReply struct { + Comment Comment `json:"comment"` +} + +// EncodeGetVersionReply encodes a GetVersionReply into a JSON byte slice. +func EncodeGetVersionReply(g GetVersionReply) ([]byte, error) { + return json.Marshal(g) +} + +// DecodeGetVersionReply decodes a JSON byte slice into a GetVersionReply. +func DecodeGetVersionReply(b []byte) (*GetVersionReply, error) { + var g GetVersionReply + err := json.Unmarshal(b, &g) + if err != nil { + return nil, err + } + return &g, nil +} + +// Count returns the comments count for the provided record. +type Count struct { + Token string `json:"token"` +} + +// EncodeCount encodes a Count into a JSON byte slice. +func EncodeCount(c Count) ([]byte, error) { + return json.Marshal(c) +} + +// DecodeCount decodes a JSON byte slice into a Count. +func DecodeCount(b []byte) (*Count, error) { + var c Count + err := json.Unmarshal(b, &c) + if err != nil { + return nil, err + } + return &c, nil +} + +// CountReply is the reply to the Count command. +type CountReply struct { + Count uint64 `json:"count"` +} + +// EncodeCountReply encodes a CountReply into a JSON byte slice. +func EncodeCountReply(c CountReply) ([]byte, error) { + return json.Marshal(c) +} + +// DecodeCountReply decodes a JSON byte slice into a CountReply. +func DecodeCountReply(b []byte) (*CountReply, error) { + var c CountReply + err := json.Unmarshal(b, &c) + if err != nil { + return nil, err + } + return &c, nil +} + +// Vote casts a comment vote (upvote or downvote). +// +// The uuid is required because the effect of a new vote on a comment score +// depends on the previous vote from that uuid. Example, a user casts an upvote +// on a comment that they have already upvoted, the resulting vote score is 0 +// due to the second upvote removing the original upvote. The public key cannot +// be relied on to remain the same for each user so a uuid must be included. +type Vote struct { + UUID string `json:"uuid"` // Unique user ID + Token string `json:"token"` // Record token + CommentID uint32 `json:"commentid"` // Comment ID + Vote VoteT `json:"vote"` // Upvote or downvote + PublicKey string `json:"publickey"` // Public key used for signature + Signature string `json:"signature"` // Signature of Token+CommentID+Vote +} + +// EncodeVote encodes a Vote into a JSON byte slice. +func EncodeVote(v Vote) ([]byte, error) { + return json.Marshal(v) +} + +// DecodeVote decodes a JSON byte slice into a Vote. +func DecodeVote(b []byte) (*Vote, error) { + var v Vote + err := json.Unmarshal(b, &v) + if err != nil { + return nil, err + } + return &v, nil +} + +// VoteReply is the reply to the Vote command. +type VoteReply struct { + Score int64 `json:"score"` // Overall comment vote score + Receipt string `json:"receipt"` // Server signature of client signature + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp +} + +// EncodeVoteReply encodes a VoteReply into a JSON byte slice. +func EncodeVoteReply(v VoteReply) ([]byte, error) { + return json.Marshal(v) +} + +// DecodeVoteReply decodes a JSON byte slice into a VoteReply. +func DecodeVoteReply(b []byte) (*VoteReply, error) { + var v VoteReply + err := json.Unmarshal(b, &v) + if err != nil { + return nil, err + } + return &v, nil +} diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index f6cff5b81..4e55886ce 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -314,6 +314,10 @@ type Backend interface { UpdateVettedRecord([]byte, []MetadataStream, []MetadataStream, []File, []string) (*Record, error) + // Update unvetted metadata (token, mdAppend, mdOverwrite) + UpdateUnvettedMetadata([]byte, []MetadataStream, + []MetadataStream) error + // Update vetted metadata (token, mdAppend, mdOverwrite) UpdateVettedMetadata([]byte, []MetadataStream, []MetadataStream) error @@ -328,7 +332,7 @@ type Backend interface { VettedExists([]byte) bool // Get unvetted record - GetUnvetted([]byte) (*Record, error) + GetUnvetted([]byte, string) (*Record, error) // Get vetted record GetVetted([]byte, string) (*Record, error) @@ -348,7 +352,7 @@ type Backend interface { GetPlugins() ([]Plugin, error) // Plugin pass-through command - Plugin(string, string) (string, string, error) // command type, payload, error + Plugin(string, string, string) (string, string, error) // command type, payload, error // Close performs cleanup of the backend. Close() diff --git a/politeiad/backend/tlogbe/anchor.go b/politeiad/backend/tlogbe/anchor.go index 9c7d3e12d..b90019fc4 100644 --- a/politeiad/backend/tlogbe/anchor.go +++ b/politeiad/backend/tlogbe/anchor.go @@ -5,7 +5,14 @@ package tlogbe import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "time" + dcrtime "github.com/decred/dcrtime/api/v2" + "github.com/decred/politeia/politeiad/backend/tlogbe/store" + "github.com/decred/politeia/util" "github.com/google/trillian/types" ) @@ -21,44 +28,23 @@ const ( // Seconds Minutes Hours Days Months DayOfWeek anchorSchedule = "0 56 * * * *" // At 56 minutes every hour - // TODO does this need to be unique? + // anchorID is included in the timestamp and verify requests as a + // unique identifier. anchorID = "tlogbe" ) // anchor represents an anchor, i.e. timestamp, of a trillian tree at a -// specific tree size. The LogRoot is hashed and anchored using dcrtime. Once -// dcrtime timestamp is verified the anchor structure is updated and saved to -// the key-value store. +// specific tree size. A SHA256 digest of the LogRoot is timestamped using +// dcrtime. type anchor struct { TreeID int64 `json:"treeid"` LogRoot *types.LogRootV1 `json:"logroot"` VerifyDigest *dcrtime.VerifyDigest `json:"verifydigest"` } -/* -func convertBlobEntryFromAnchor(a anchor) (*blobEntry, error) { - data, err := json.Marshal(a) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - dataDescriptor{ - Type: dataTypeStructure, - Descriptor: dataDescriptorAnchor, - }) - if err != nil { - return nil, err - } - be := blobEntryNew(hint, data) - return &be, nil -} - -// anchorSave saves an anchor to the key-value store and updates the record -// history of the record that corresponds to tree that was anchored. -// -// This function must be called WITHOUT the read/write lock held. -func (t *tlogbe) anchorSave(a anchor) error { - +// anchorSave saves an anchor to the key-value store and appends a log leaf +// to the trillian tree for the anchor. +func (t *tlog) anchorSave(a anchor) error { // Sanity checks switch { case a.TreeID == 0: @@ -69,66 +55,39 @@ func (t *tlogbe) anchorSave(a anchor) error { return fmt.Errorf("verify digest not found") } - // Save the anchor record - be, err := convertBlobEntryFromAnchor(a) - if err != nil { - return err - } - b, err := blobify(*be) - if err != nil { - return err - } - lrb, err := a.LogRoot.MarshalBinary() - if err != nil { - return err - } - logRootHash := util.Hash(lrb)[:] - err = t.store.Put(keyAnchor(logRootHash), b) - if err != nil { - return fmt.Errorf("Put: %v", err) - } - - log.Debugf("Anchor saved for tree %v at height %v", - a.TreeID, a.LogRoot.TreeSize) - - // Update the record history with the anchor. The lock must be held - // during this update. - t.Lock() - defer t.Unlock() - - token := tokenFromTreeID(a.TreeID) - rh, err := t.recordHistory(token) - if err != nil { - return fmt.Errorf("recordHistory: %v", err) - } - - rh.Anchors[a.LogRoot.TreeSize] = logRootHash + // TODO + // Save the anchor record to store + be, err := convertBlobEntryFromAnchor(a) + if err != nil { + return err + } + b, err := store.Blobify(*be) + if err != nil { + return err + } + lrb, err := a.LogRoot.MarshalBinary() + if err != nil { + return err + } + logRootHash := util.Hash(lrb)[:] + _ = logRootHash + _ = b - be, err = convertBlobEntryFromRecordHistory(*rh) - if err != nil { - return err - } - b, err = blobify(*be) - if err != nil { - return err - } - err = t.store.Put(keyRecordHistory(rh.Token), b) - if err != nil { - return fmt.Errorf("Put: %v", err) - } + // Append anchor leaf to trillian tree - log.Debugf("Anchor added to record history %x", token) + log.Debugf("Saved %v anchor for tree %v at height %v", + t.id, a.TreeID, a.LogRoot.TreeSize) return nil } -// waitForAnchor waits for the anchor to drop. The anchor is not considered +// anchorWait waits for the anchor to drop. The anchor is not considered // dropped until dcrtime returns the ChainTimestamp in the reply. dcrtime does // not return the ChainTimestamp until the timestamp transaction has 6 // confirmations. Once the timestamp has been dropped, the anchor record is // saved to the key-value store and the record histories of the corresponding // timestamped trees are updated. -func (t *tlogbe) waitForAnchor(anchors []anchor, hashes []string) { +func (t *tlog) anchorWait(anchors []anchor, hashes []string) { // Ensure we are not reentrant t.Lock() if t.droppingAnchor { @@ -146,12 +105,12 @@ func (t *tlogbe) waitForAnchor(anchors []anchor, hashes []string) { t.Unlock() if exitErr != nil { - log.Errorf("waitForAnchor: %v", exitErr) + log.Errorf("anchorWait: %v", exitErr) } }() // Wait for anchor to drop - log.Infof("Waiting for anchor to drop") + log.Infof("Waiting for %v anchor to drop", t.id) // Continually check with dcrtime if the anchor has been dropped. // The anchor is not considered dropped until the ChainTimestamp @@ -173,7 +132,7 @@ func (t *tlogbe) waitForAnchor(anchors []anchor, hashes []string) { for try := 0; try < retries; try++ { <-ticker.C - log.Debugf("Verify anchor attempt %v/%v", try+1, retries) + log.Debugf("Verify %v anchor attempt %v/%v", t.id, try+1, retries) vbr, err := verifyBatch(t.dcrtimeHost, anchorID, hashes) if err != nil { @@ -218,14 +177,14 @@ func (t *tlogbe) waitForAnchor(anchors []anchor, hashes []string) { // that was anchored. b, err := v.LogRoot.MarshalBinary() if err != nil { - log.Errorf("waitForAnchor: MarshalBinary %v %x: %v", + log.Errorf("anchorWait: MarshalBinary %v %x: %v", v.TreeID, v.LogRoot.RootHash, err) continue } anchorDigest := hex.EncodeToString(util.Hash(b)[:]) dcrtimeDigest := vbr.Digests[k].Digest if anchorDigest != dcrtimeDigest { - log.Errorf("waitForAnchor: digest mismatch: got %x, want %v", + log.Errorf("anchorWait: digest mismatch: got %x, want %v", dcrtimeDigest, anchorDigest) continue } @@ -236,12 +195,12 @@ func (t *tlogbe) waitForAnchor(anchors []anchor, hashes []string) { // Save anchor err = t.anchorSave(v) if err != nil { - log.Errorf("waitForAnchor: anchorSave %v: %v", v.TreeID, err) + log.Errorf("anchorWait: anchorSave %v: %v", v.TreeID, err) continue } } - log.Infof("Anchor dropped for %v records", len(vbr.Digests)) + log.Infof("Anchor dropped for %v %v records", len(vbr.Digests), t.id) return } @@ -249,13 +208,12 @@ func (t *tlogbe) waitForAnchor(anchors []anchor, hashes []string) { int(period.Minutes())*retries) } -// anchor drops an anchor for any trees that have unanchored leaves at the -// time of function invocation. A digest of the tree's log root at its current -// height is timestamped onto the decred blockchain using the dcrtime service. -// The anchor data is saved to the key-value store and the record history that -// corresponds to the anchored tree is updated with the anchor data. -func (t *tlogbe) anchorTrees() { - log.Debugf("Start anchor process") +// anchor drops an anchor for any trees that have unanchored leaves at the time +// of function invocation. A SHA256 digest of the tree's log root at its +// current height is timestamped onto the decred blockchain using the dcrtime +// service. The anchor data is saved to the key-value store. +func (t *tlog) anchor() { + log.Debugf("Start %v anchor process", t.id) var exitErr error // Set on exit if there is an error defer func() { @@ -264,7 +222,7 @@ func (t *tlogbe) anchorTrees() { } }() - trees, err := t.tlog.treesAll() + trees, err := t.trillian.treesAll() if err != nil { exitErr = fmt.Errorf("treesAll: %v", err) return @@ -283,35 +241,35 @@ func (t *tlogbe) anchorTrees() { // Find the trees that need to be anchored for _, v := range trees { - // Check if this tree has unanchored leaves - _, lr, err := t.tlog.signedLogRoot(v) + // TODO this needs to pull the anchor record from the store and + // check the anchor tree height against the current tree height + + // Check if the last leaf is an anchor record + l, err := t.lastLeaf(v.TreeId) if err != nil { - exitErr = fmt.Errorf("signedLogRoot %v: %v", v.TreeId, err) + exitErr = fmt.Errorf("lastLeaf %v: %v", v.TreeId, err) return } - token := tokenFromTreeID(v.TreeId) - rh, err := t.recordHistory(token) - if err != nil { - exitErr = fmt.Errorf("recordHistory %x: %v", token, err) - } - _, ok := rh.Anchors[lr.TreeSize] - if ok { + if leafIsAnchorRecord(l) { // Tree has already been anchored at the current height. Check // the next one. continue } - // Tree has not been anchored at current height. Anchor it. - log.Debugf("Tree %v (%x) anchoring at height %v", - v.TreeId, token, lr.TreeSize) - - // Setup anchor record + // Tree has not been anchored at current height. Add it to the + // list of anchors. + _, lr, err := t.trillian.signedLogRootForTree(v) + if err != nil { + exitErr = fmt.Errorf("signedLogRoot %v: %v", v.TreeId, err) + return + } anchors = append(anchors, anchor{ TreeID: v.TreeId, LogRoot: lr, }) - // Collate the log root digest + // Collate the log root digest. This is what gets submitted to + // dcrtime. lrb, err := lr.MarshalBinary() if err != nil { exitErr = fmt.Errorf("MarshalBinary %v: %v", v.TreeId, err) @@ -319,6 +277,9 @@ func (t *tlogbe) anchorTrees() { } d := hex.EncodeToString(util.Hash(lrb)[:]) digests = append(digests, d) + + log.Debugf("Anchoring %v tree %v at height %v", + t.id, v.TreeId, lr.TreeSize) } if len(anchors) == 0 { log.Infof("Nothing to anchor") @@ -344,7 +305,7 @@ func (t *tlogbe) anchorTrees() { t.Unlock() // Submit dcrtime anchor request - log.Infof("Anchoring %v trees", len(anchors)) + log.Infof("Anchoring %v %v trees", len(anchors), t.id) tbr, err := timestampBatch(t.dcrtimeHost, anchorID, digests) if err != nil { @@ -357,13 +318,12 @@ func (t *tlogbe) anchorTrees() { case dcrtime.ResultOK: // We're good; continue case dcrtime.ResultExistsError: - // I can't think of any situations where this would happen, but - // it's ok if it does since we'll still be able to retrieve the - // VerifyDigest from dcrtime for this digest. + // I don't think this will ever happen, but it's ok if it does + // since we'll still be able to retrieve the VerifyDigest from + // dcrtime for this digest. // - // Log this as a warning to bring it to our attention. Do not - // exit. - log.Warnf("Digest failed %v: %v (%v)", + // Log a warning to bring it to our attention. Do not exit. + log.Warnf("Digest already exists %v: %v (%v)", tbr.Digests[i], dcrtime.Result[v], v) default: // Something went wrong; exit @@ -377,6 +337,5 @@ func (t *tlogbe) anchorTrees() { return } - go t.waitForAnchor(anchors, digests) + go t.anchorWait(anchors, digests) } -*/ diff --git a/politeiad/backend/tlogbe/blobentry.go b/politeiad/backend/tlogbe/blobentry.go deleted file mode 100644 index 7fabc8699..000000000 --- a/politeiad/backend/tlogbe/blobentry.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright (c) 2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package tlogbe - -import ( - "bytes" - "compress/gzip" - "encoding/base64" - "encoding/gob" - "encoding/hex" - - "github.com/decred/politeia/util" -) - -const ( - // blobEntryVersion is encoded in the sbox header of encrypted - // blobs. - blobEntryVersion uint32 = 1 - - // Data descriptor types. These may be freely edited since they are - // solely hints to the application. - dataTypeStructure = "struct" // Descriptor contains a structure - - dataDescriptorAnchor = "anchor" -) - -// dataDescriptor provides hints about a data blob. In practise we JSON encode -// this struture and stuff it into blobEntry.DataHint. -type dataDescriptor struct { - Type string `json:"type"` // Type of data - Descriptor string `json:"descriptor"` // Description of the data - ExtraData string `json:"extradata,omitempty"` // Value to be freely used -} - -// blobEntry is the structure used to store data in the Blob key-value store. -// All data in the Blob key-value store will be encoded as a blobEntry. -type blobEntry struct { - Hash string `json:"hash"` // SHA256 hash of data payload, hex encoded - DataHint string `json:"datahint"` // Hint that describes data, base64 encoded - Data string `json:"data"` // Data payload, base64 encoded -} - -func blobify(be blobEntry) ([]byte, error) { - var b bytes.Buffer - zw := gzip.NewWriter(&b) - enc := gob.NewEncoder(zw) - err := enc.Encode(be) - if err != nil { - return nil, err - } - err = zw.Close() // we must flush gzip buffers - if err != nil { - return nil, err - } - return b.Bytes(), nil -} - -func deblob(blob []byte) (*blobEntry, error) { - zr, err := gzip.NewReader(bytes.NewReader(blob)) - if err != nil { - return nil, err - } - r := gob.NewDecoder(zr) - var be blobEntry - err = r.Decode(&be) - if err != nil { - return nil, err - } - return &be, nil -} - -func blobEntryNew(dataHint, data []byte) blobEntry { - return blobEntry{ - Hash: hex.EncodeToString(util.Digest(data)), - DataHint: base64.StdEncoding.EncodeToString(dataHint), - Data: base64.StdEncoding.EncodeToString(data), - } -} diff --git a/politeiad/backend/tlogbe/encryptionkey.go b/politeiad/backend/tlogbe/encryptionkey.go index d439ec288..8717cafa2 100644 --- a/politeiad/backend/tlogbe/encryptionkey.go +++ b/politeiad/backend/tlogbe/encryptionkey.go @@ -11,20 +11,20 @@ import ( "github.com/marcopeereboom/sbox" ) -// EncryptionKey provides an API for encrypting and decrypting data using a -// structure that can be passed to plugins and accessed concurrently. -type EncryptionKey struct { +// encryptionKey provides an API for encrypting and decrypting data. The +// encryption key is zero'd out on application exit so the lock must be held +// anytime the key is accessed in order to prevent the goland race detector +// from complaining. +type encryptionKey struct { sync.RWMutex key *[32]byte } -// Encrypt encrypts the provided data. It prefixes the encrypted blob with an +// encrypt encrypts the provided data. It prefixes the encrypted blob with an // sbox header which encodes the provided version. The version is user provided // and can be used as a hint to identify or version the packed blob. Version is -// not inspected or used by Encrypt and Decrypt. The read lock is held to -// prevent the golang race detector from complaining when the encryption key is -// zeroed out on application exit. -func (e *EncryptionKey) Encrypt(version uint32, blob []byte) ([]byte, error) { +// not inspected or used by Encrypt and Decrypt. +func (e *encryptionKey) encrypt(version uint32, blob []byte) ([]byte, error) { e.RLock() defer e.RUnlock() @@ -32,10 +32,8 @@ func (e *EncryptionKey) Encrypt(version uint32, blob []byte) ([]byte, error) { } // decrypt decrypts the provided packed blob. The decrypted blob and the -// version that was used to encrypt the blob are returned. The read lock is -// held to prevent the golang race detector from complaining when the -// encryption key is zeroed out on application exit. -func (e *EncryptionKey) Decrypt(blob []byte) ([]byte, uint32, error) { +// version that was used to encrypt the blob are returned. +func (e *encryptionKey) decrypt(blob []byte) ([]byte, uint32, error) { e.RLock() defer e.RUnlock() @@ -43,7 +41,7 @@ func (e *EncryptionKey) Decrypt(blob []byte) ([]byte, uint32, error) { } // Zero zeroes out the encryption key. -func (e *EncryptionKey) Zero() { +func (e *encryptionKey) Zero() { e.Lock() defer e.Unlock() @@ -51,8 +49,8 @@ func (e *EncryptionKey) Zero() { e.key = nil } -func encryptionKeyNew(key *[32]byte) *EncryptionKey { - return &EncryptionKey{ +func encryptionKeyNew(key *[32]byte) *encryptionKey { + return &encryptionKey{ key: key, } } diff --git a/politeiad/backend/tlogbe/plugin/comments/comments.go b/politeiad/backend/tlogbe/plugin/comments/comments.go index 9247e3463..0cb54d063 100644 --- a/politeiad/backend/tlogbe/plugin/comments/comments.go +++ b/politeiad/backend/tlogbe/plugin/comments/comments.go @@ -5,10 +5,15 @@ package comments import ( + "bytes" + "encoding/base64" "encoding/hex" - "os" - "path/filepath" + "encoding/json" + "fmt" + "sort" + "strconv" "sync" + "time" "github.com/decred/politeia/plugins/comments" "github.com/decred/politeia/politeiad/api/v1/identity" @@ -16,240 +21,1400 @@ import ( "github.com/decred/politeia/politeiad/backend/tlogbe" "github.com/decred/politeia/politeiad/backend/tlogbe/plugin" "github.com/decred/politeia/politeiad/backend/tlogbe/store" - "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" + "github.com/decred/politeia/util" ) const ( - commentsDirname = "comments" + // Blob entry data descriptors + dataDescriptorCommentAdd = "commentadd" + dataDescriptorCommentDel = "commentdel" + dataDescriptorCommentVote = "commentvote" + dataDescriptorCommentsIndex = "commentsindex" - // Data descriptors - dataDescriptorRecordComments = "recordcomments" - dataDescriptorComment = "comment" - - // Key-value store key prefixes - keyPrefixRecordComments = "record" - keyPrefixComment = "comment" + // Prefixes that are appended to key-value store keys before + // storing them in the log leaf ExtraData field. + keyPrefixCommentAdd = "commentadd:" + keyPrefixCommentDel = "commentdel:" + keyPrefixCommentVote = "commentvote:" + keyPrefixCommentsIndex = "commentsindex:" ) var ( _ plugin.Plugin = (*commentsPlugin)(nil) ) -// TODO unvetted comments should be encrypted -// TODO journal should be encrypted - type commentsPlugin struct { - sync.RWMutex - id *identity.FullIdentity - encyrptionKey *tlogbe.EncryptionKey - tlog *tlogbe.TrillianClient - backend backend.Backend - store store.Blob + sync.Mutex + id *identity.FullIdentity + backend *tlogbe.Tlogbe // Mutexes contains a mutex for each record. The mutexes are lazy // loaded. - mutexes map[string]*sync.RWMutex // [token]mutex + mutexes map[string]*sync.Mutex // [string]mutex +} + +type voteIndex struct { + Vote comments.VoteT `json:"vote"` + MerkleHash []byte `json:"merklehash"` } type commentIndex struct { - CommentID uint32 `json:"commentid"` - Versions map[uint32][]byte `json:"versions"` // [version]merkleLeafHash + Adds map[uint32][]byte `json:"adds"` // [version]merkleHash + Del []byte `json:"del"` // Merkle hash of delete record + Score int64 `json:"score"` // Vote score + + // Votes contains the vote history for each uuid that voted on the + // comment. This data is memoized because the effect of a new vote + // on a comment depends on the previous vote from that uuid. + // Example, a user upvotes a comment that they have already + // upvoted, the resulting vote score is 0 due to the second upvote + // removing the original upvote. + Votes map[string][]voteIndex `json:"votes"` // [uuid]votes +} + +// TODO this is not very efficient and probably needs to be improved. +// It duplicates a lot of data and requires fetching all the tree +// leaves to get. This may be problematic when there are 20,000 vote +// leaves and you just want to get the comments count for a record. +type index struct { + Comments map[uint32]commentIndex `json:"comments"` // [commentID]comment } -// recordComments contains the comment index for all comments made on a record. -type recordComments struct { - Token string `json:"token"` +// TODO this needs to go in util +func verifySignature(signature, pubkey, msg string) error { + sig, err := util.ConvertSignature(signature) + if err != nil { + return comments.UserError{ + ErrorCode: comments.ErrorStatusSignatureInvalid, + ErrorContext: []string{err.Error()}, + } + } + b, err := hex.DecodeString(pubkey) + if err != nil { + return comments.UserError{ + ErrorCode: comments.ErrorStatusPublicKeyInvalid, + ErrorContext: []string{"key is not hex"}, + } + } + pk, err := identity.PublicIdentityFromBytes(b) + if err != nil { + return comments.UserError{ + ErrorCode: comments.ErrorStatusPublicKeyInvalid, + ErrorContext: []string{err.Error()}, + } + } + if !pk.VerifyMessage([]byte(msg), sig) { + return comments.UserError{ + ErrorCode: comments.ErrorStatusSignatureInvalid, + ErrorContext: []string{err.Error()}, + } + } + return nil +} - // LastID contains the last comment ID that has been assigned for - // this record. Comment IDs are sequential starting with 1. The - // state of the record, unvetted or vetted, does not impact the - // comment ID that is assinged. - LastID uint32 `json:"lastid"` +// mutex returns the mutex for the specified record. +func (p *commentsPlugin) mutex(token string) *sync.Mutex { + p.Lock() + defer p.Unlock() - // Unvetted contains comments that were made on the record when it - // was in an unvetted state. Unvetted comments are encrypted before - // being saved to the key-value store. They remain encrypted for - // the duration of their lifetime, even after the record itself - // becomes vetted. - // - // map[commentID]commentIndex - Unvetted map[uint32]commentIndex `json:"unvetted"` + m, ok := p.mutexes[token] + if !ok { + // Mutexes is lazy loaded + m = &sync.Mutex{} + p.mutexes[token] = m + } - // Vetted contains comments that were made on a vetted record. - // Vetted comments are stored in the key-value store unencrypted. - // - // map[commentID]commentIndex - Vetted map[uint32]commentIndex `json:"vetted"` + return m } -func keyRecordComments(token string) string { - return keyPrefixRecordComments + token +func convertBlobEntryFromCommentAdd(c commentAdd) (*store.BlobEntry, error) { + data, err := json.Marshal(c) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorCommentAdd, + }) + if err != nil { + return nil, err + } + be := store.BlobEntryNew(hint, data) + return &be, nil } -// keyComment returns the key for a comment in the key-value store. -func keyComment(token string, merkleLeafHash []byte) string { - return keyPrefixComment + hex.EncodeToString(merkleLeafHash) +func convertBlobEntryFromCommentDel(c commentDel) (*store.BlobEntry, error) { + data, err := json.Marshal(c) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorCommentDel, + }) + if err != nil { + return nil, err + } + be := store.BlobEntryNew(hint, data) + return &be, nil } -/* -func convertBlobEntryFromRecordComments(rc recordComments) (*blobEntry, error) { - data, err := json.Marshal(rc) +func convertBlobEntryFromCommentVote(c commentVote) (*store.BlobEntry, error) { + data, err := json.Marshal(c) if err != nil { return nil, err } hint, err := json.Marshal( - dataDescriptor{ - Type: dataTypeStructure, - Descriptor: dataDescriptorRecordComments, + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorCommentVote, }) if err != nil { return nil, err } - be := blobEntryNew(hint, data) + be := store.BlobEntryNew(hint, data) return &be, nil } -// mutex returns the mutex for the provided token. This function assumes that -// the provided token has already been validated and corresponds to a record in -// the Backend. -func (p *commentsPlugin) mutex(token string) (*sync.RWMutex, error) { - p.Lock() - defer p.Unlock() +func convertBlobEntryFromIndex(idx index) (*store.BlobEntry, error) { + data, err := json.Marshal(idx) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorCommentsIndex, + }) + if err != nil { + return nil, err + } + be := store.BlobEntryNew(hint, data) + return &be, nil +} - m, ok := p.mutexes[token] - if !ok { - // Mutexes is lazy loaded - m = &sync.RWMutex{} - p.mutexes[token] = m +func convertCommentAddFromBlobEntry(be store.BlobEntry) (*commentAdd, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorCommentAdd { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorCommentAdd) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + hash, err := hex.DecodeString(be.Hash) + if err != nil { + return nil, fmt.Errorf("decode hash: %v", err) + } + if !bytes.Equal(util.Digest(b), hash) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), hash) + } + var c commentAdd + err = json.Unmarshal(b, &c) + if err != nil { + return nil, fmt.Errorf("unmarshal index: %v", err) } - return m, nil + return &c, nil } -// recordExists returns whether the provided record exists in the backend. -// This function does not differentiate between unvetted and vetted records. -func (p *commentsPlugin) recordExists(token string) bool { - t, err := hex.DecodeString(token) +func convertIndexFromBlobEntry(be store.BlobEntry) (*index, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorCommentsIndex { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorCommentsIndex) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + hash, err := hex.DecodeString(be.Hash) if err != nil { - return false + return nil, fmt.Errorf("decode hash: %v", err) + } + if !bytes.Equal(util.Digest(b), hash) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), hash) } - if p.backend.UnvettedExists(t) { - return true + var idx index + err = json.Unmarshal(b, &idx) + if err != nil { + return nil, fmt.Errorf("unmarshal index: %v", err) } - if p.backend.VettedExists(t) { - return true + + return &idx, nil +} + +func convertCommentFromCommentAdd(ca commentAdd) comments.Comment { + // Score needs to be filled in seperately + return comments.Comment{ + Token: ca.Token, + ParentID: ca.ParentID, + Comment: ca.Comment, + PublicKey: ca.PublicKey, + Signature: ca.Signature, + CommentID: ca.CommentID, + Version: ca.Version, + Receipt: ca.Receipt, + Timestamp: ca.Timestamp, + Score: 0, + Deleted: false, } - return false } -/* -func (p *commentsPlugin) recordComments(token string) (*recordComments, error) { - be, err := p.store.Get(keyRecordComments(token)) +func commentAddSave(client *tlogbe.PluginClient, c commentAdd, encrypt bool) ([]byte, error) { + // Prepare blob + be, err := convertBlobEntryFromCommentAdd(c) + if err != nil { + return nil, err + } + h, err := hex.DecodeString(be.Hash) + if err != nil { + return nil, err + } + b, err := store.Blobify(*be) if err != nil { return nil, err } - rc, err := convertBlobEntryFromRecordComments(be) + + // Save blob + merkles, err := client.BlobsSave(keyPrefixCommentAdd, + [][]byte{b}, [][]byte{h}, encrypt) + if err != nil { + return nil, fmt.Errorf("BlobsSave: %v", err) + } + if len(merkles) != 1 { + return nil, fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", + len(merkles)) + } + + return merkles[0], nil +} + +func commentAdds(client *tlogbe.PluginClient, merkleHashes [][]byte) ([]commentAdd, error) { + // Retrieve blobs + blobs, err := client.BlobsByMerkleHash(merkleHashes) if err != nil { return nil, err } - return &rc, nil + if len(blobs) != len(merkleHashes) { + notFound := make([]string, 0, len(blobs)) + for _, v := range merkleHashes { + m := hex.EncodeToString(v) + _, ok := blobs[m] + if !ok { + notFound = append(notFound, m) + } + } + return nil, fmt.Errorf("blobs not found: %v", notFound) + } + + // Decode blobs + adds := make([]commentAdd, 0, len(blobs)) + for _, v := range blobs { + be, err := store.Deblob(v) + if err != nil { + return nil, err + } + c, err := convertCommentAddFromBlobEntry(*be) + if err != nil { + return nil, err + } + adds = append(adds, *c) + } + + return adds, nil } -func (p *commentsPlugin) commentExists(token string, commentID uint32) bool { - ri, err := p.recordComments(token) +func commentDelSave(client *tlogbe.PluginClient, c commentDel) ([]byte, error) { + // Prepare blob + be, err := convertBlobEntryFromCommentDel(c) + if err != nil { + return nil, err + } + h, err := hex.DecodeString(be.Hash) + if err != nil { + return nil, err + } + b, err := store.Blobify(*be) if err != nil { - return false + return nil, err } - _, ok := ri.Unvetted[commentID] - if ok { - return true + + // Save blob + merkles, err := client.BlobsSave(keyPrefixCommentDel, + [][]byte{b}, [][]byte{h}, false) + if err != nil { + return nil, fmt.Errorf("BlobsSave: %v", err) } - _, ok = ri.Vetted[commentID] - if ok { - return true + if len(merkles) != 1 { + return nil, fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", + len(merkles)) } - return false + + return merkles[0], nil } -func (p *commentsPlugin) cmdNew(payload string) (string, error) { - n, err := comments.DecodeNew([]byte(payload)) +func commentVoteSave(client *tlogbe.PluginClient, c commentVote) ([]byte, error) { + // Prepare blob + be, err := convertBlobEntryFromCommentVote(c) if err != nil { - return "", err + return nil, err + } + h, err := hex.DecodeString(be.Hash) + if err != nil { + return nil, err + } + b, err := store.Blobify(*be) + if err != nil { + return nil, err } - // Verify signature - err = comments.VerifyCommentSignature(n.Signature, n.PublicKey, - n.Token, n.ParentID, n.Comment) + // Save blob + merkles, err := client.BlobsSave(keyPrefixCommentAdd, + [][]byte{b}, [][]byte{h}, false) if err != nil { - return "", err + return nil, fmt.Errorf("BlobsSave: %v", err) } + if len(merkles) != 1 { + return nil, fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", + len(merkles)) + } + + return merkles[0], nil +} + +func indexSave(client *tlogbe.PluginClient, idx index) error { + // Prepare blob + be, err := convertBlobEntryFromIndex(idx) + if err != nil { + return err + } + h, err := hex.DecodeString(be.Hash) + if err != nil { + return err + } + b, err := store.Blobify(*be) + if err != nil { + return err + } + + // Save blob + merkles, err := client.BlobsSave(keyPrefixCommentsIndex, + [][]byte{b}, [][]byte{h}, false) + if err != nil { + return fmt.Errorf("BlobsSave: %v", err) + } + if len(merkles) != 1 { + return fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", + len(merkles)) + } + + return nil +} + +func indexLatest(client *tlogbe.PluginClient) (*index, error) { + // Get all comment indexes + blobs, err := client.BlobsByKeyPrefix(keyPrefixCommentsIndex) + if err != nil { + return nil, err + } + if len(blobs) == 0 { + // A comments index does not exist. This can happen when no + // comments have been made on the record yet. Return a new one. + return &index{ + Comments: make(map[uint32]commentIndex), + }, nil + } + + // Decode the most recent index + b := blobs[len(blobs)-1] + be, err := store.Deblob(b) + if err != nil { + return nil, err + } + return convertIndexFromBlobEntry(*be) +} + +func commentIDLatest(idx index) uint32 { + var maxID uint32 + for id := range idx.Comments { + if id > maxID { + maxID = id + } + } + return maxID +} + +func commentVersionLatest(cidx commentIndex) uint32 { + var maxVersion uint32 + for version := range cidx.Adds { + if version > maxVersion { + maxVersion = version + } + } + return maxVersion +} + +func commentExists(idx index, commentID uint32) bool { + _, ok := idx.Comments[commentID] + return ok +} + +// commentsLatest returns the most recent version of the specified comments. +// Deleted comments are returned with limited data. Comment IDs that do not +// correspond to an actual comment are not included in the returned map. It is +// the responsibility of the caller to ensure a comment is returned for each of +// the provided comment IDs. The comments index that was looked up during this +// process is also returned. +func commentsLatest(client *tlogbe.PluginClient, commentIDs []uint32) (map[uint32]comments.Comment, *index, error) { + // Get comments index + idx, err := indexLatest(client) + if err != nil { + return nil, nil, fmt.Errorf("indexLatest: %v", err) + } + + // Aggregate merkle hashes of the comment add records that need to + // be looked up. If the comment has been deleted then there is + // nothing to look up. + var ( + merkles = make([][]byte, 0, len(commentIDs)) + dels = make([]uint32, 0, len(commentIDs)) + ) + for _, v := range commentIDs { + cidx, ok := idx.Comments[v] + if !ok { + // Comment does not exist + continue + } + if cidx.Del != nil { + // Comment has been deleted + dels = append(dels, v) + continue + } + + // Save the merkle hash for the latest version + version := commentVersionLatest(cidx) + merkles = append(merkles, cidx.Adds[version]) + } + + // Get comment add records + adds, err := commentAdds(client, merkles) + if err != nil { + return nil, nil, fmt.Errorf("commentAdds: %v", err) + } + if len(adds) != len(merkles) { + return nil, nil, fmt.Errorf("wrong comment adds count; got %v, want %v", + len(adds), len(merkles)) + } + + // Prepare comments + cs := make(map[uint32]comments.Comment, len(commentIDs)) + for _, v := range adds { + c := convertCommentFromCommentAdd(v) + c.Score = idx.Comments[c.CommentID].Score + cs[v.CommentID] = c + } + for _, commentID := range dels { + score := idx.Comments[commentID].Score + cs[commentID] = comments.Comment{ + Token: hex.EncodeToString(client.Token), + CommentID: commentID, + Score: score, + Deleted: true, + } + } + + return cs, idx, nil +} - // Ensure record exists - if !p.recordExists(n.Token) { - return "", fmt.Errorf("record not found %v", n.Token) +// This function must be called WITH the record lock held. +func (p *commentsPlugin) new(client *tlogbe.PluginClient, n comments.New, encrypt bool) (*comments.NewReply, error) { + // Pull comments index + idx, err := indexLatest(client) + if err != nil { + return nil, err } // Ensure parent comment exists if set. A parent ID of 0 means that - // this is a base level comment, not a reply comment. - if n.ParentID > 0 && !p.commentExists(n.Token, n.ParentID) { - e := fmt.Sprintf("parent ID %v comment", n.ParentID) - return "", comments.PluginError{ - ErrorCode: comments.ErrorStatusCommentNotFound, - ErrorContext: []string{e}, + // this is a base level comment, not a reply to another comment. + if n.ParentID > 0 && !commentExists(*idx, n.ParentID) { + return nil, comments.UserError{ + ErrorCode: comments.ErrorStatusParentIDInvalid, + ErrorContext: []string{"comment not found"}, } } - // Setup the comment - c := comments.Comment{ + // Setup comment + receipt := p.id.SignMessage([]byte(n.Signature)) + c := commentAdd{ Token: n.Token, ParentID: n.ParentID, Comment: n.Comment, PublicKey: n.PublicKey, Signature: n.Signature, - // CommentID: , - // Version: , - // Receipt: "", + CommentID: commentIDLatest(*idx) + 1, + Version: 1, + Receipt: hex.EncodeToString(receipt[:]), Timestamp: time.Now().Unix(), - // Score: 0, - Deleted: false, - Censored: false, } - _ = c - // Append to trillian tree + // Save comment + merkleHash, err := commentAddSave(client, c, encrypt) + if err != nil { + return nil, fmt.Errorf("commentSave: %v", err) + } - // Save to key-value store + // Update index + idx.Comments[c.CommentID] = commentIndex{ + Adds: map[uint32][]byte{ + 1: merkleHash, + }, + Del: nil, + Votes: make(map[string][]voteIndex), + } - // Prepare reply + // Save index + err = indexSave(client, *idx) + if err != nil { + return nil, fmt.Errorf("indexSave: %v", err) + } - return "", nil + log.Debugf("Comment saved to record %v comment ID %v", + c.Token, c.CommentID) + + return &comments.NewReply{ + CommentID: c.CommentID, + Receipt: c.Receipt, + Timestamp: c.Timestamp, + }, nil } -*/ -func (p *commentsPlugin) Cmd(id, payload string) (string, error) { - switch id { - case comments.CmdNew: - // return p.cmdNew(payload) +func (p *commentsPlugin) cmdNew(payload string) (string, error) { + log.Tracef("comments cmdNew: %v", payload) + + // Decode payload + n, err := comments.DecodeNew([]byte(payload)) + if err != nil { + return "", err } - return "", plugin.ErrInvalidPluginCmd + + // Verify signature + msg := n.Token + strconv.FormatUint(uint64(n.ParentID), 10) + n.Comment + err = verifySignature(n.Signature, n.PublicKey, msg) + if err != nil { + return "", err + } + + // Get plugin client + token, err := hex.DecodeString(n.Token) + if err != nil { + return "", comments.UserError{ + ErrorCode: comments.ErrorStatusTokenInvalid, + } + } + client, err := p.backend.PluginClient(token) + if err != nil { + if err == backend.ErrRecordNotFound { + return "", comments.UserError{ + ErrorCode: comments.ErrorStatusRecordNotFound, + } + } + return "", err + } + + // The comments index must be pulled and updated. The record lock + // must be held for the remainder of this function. + m := p.mutex(n.Token) + m.Lock() + defer m.Unlock() + + // Save new comment + var nr *comments.NewReply + switch client.State { + case tlogbe.RecordStateUnvetted: + nr, err = p.new(client, *n, true) + if err != nil { + return "", err + } + case tlogbe.RecordStateVetted: + nr, err = p.new(client, *n, false) + if err != nil { + return "", err + } + default: + return "", fmt.Errorf("invalid record state %v", client.State) + } + + // Prepare reply + reply, err := comments.EncodeNewReply(*nr) + if err != nil { + return "", err + } + + return string(reply), nil } -func (p *commentsPlugin) Setup() error { - return nil +// This function must be called WITH the record lock held. +func (p *commentsPlugin) edit(client *tlogbe.PluginClient, e comments.Edit, encrypt bool) (*comments.EditReply, error) { + // Get the existing comment + cs, idx, err := commentsLatest(client, []uint32{e.CommentID}) + if err != nil { + return nil, fmt.Errorf("commentsLatest %v: %v", e.CommentID, err) + } + existing, ok := cs[e.CommentID] + if !ok { + return nil, comments.UserError{ + ErrorCode: comments.ErrorStatusCommentNotFound, + } + } + + // Validate the comment edit. The parent ID must remain the same. + // The comment text must be different. + if e.ParentID != existing.ParentID { + e := fmt.Sprintf("parent id cannot change; got %v, want %v", + e.ParentID, existing.ParentID) + return nil, comments.UserError{ + ErrorCode: comments.ErrorStatusParentIDInvalid, + ErrorContext: []string{e}, + } + } + if e.Comment == existing.Comment { + return nil, comments.UserError{ + ErrorCode: comments.ErrorStatusNoCommentChanges, + } + } + + // Create a new comment version + receipt := p.id.SignMessage([]byte(e.Signature)) + c := commentAdd{ + Token: e.Token, + ParentID: e.ParentID, + Comment: e.Comment, + PublicKey: e.PublicKey, + Signature: e.Signature, + CommentID: e.CommentID, + Version: existing.Version + 1, + Receipt: hex.EncodeToString(receipt[:]), + Timestamp: time.Now().Unix(), + } + + // Save comment + merkleHash, err := commentAddSave(client, c, encrypt) + if err != nil { + return nil, fmt.Errorf("commentSave: %v", err) + } + + // Update index + idx.Comments[c.CommentID].Adds[c.Version] = merkleHash + + // Save index + err = indexSave(client, *idx) + if err != nil { + return nil, fmt.Errorf("indexSave: %v", err) + } + + log.Debugf("Comment edited on record %v comment ID %v", + c.Token, c.CommentID) + + return &comments.EditReply{ + Version: c.Version, + Receipt: c.Receipt, + Timestamp: c.Timestamp, + }, nil } -func New(dataDir string, tlog *tlogbe.TrillianClient, backend backend.Backend) (*commentsPlugin, error) { - // Setup key-value store - fp := filepath.Join(dataDir, commentsDirname) - err := os.MkdirAll(fp, 0700) +func (p *commentsPlugin) cmdEdit(payload string) (string, error) { + log.Tracef("comments cmdEdit: %v", payload) + + // Decode payload + e, err := comments.DecodeEdit([]byte(payload)) if err != nil { - return nil, err + return "", err } - store := filesystem.New(fp) + // Verify signature + msg := e.Token + strconv.FormatUint(uint64(e.ParentID), 10) + e.Comment + err = verifySignature(e.Signature, e.PublicKey, msg) + if err != nil { + return "", err + } + + // Get plugin client + token, err := hex.DecodeString(e.Token) + if err != nil { + return "", comments.UserError{ + ErrorCode: comments.ErrorStatusTokenInvalid, + } + } + client, err := p.backend.PluginClient(token) + if err != nil { + if err == backend.ErrRecordNotFound { + return "", comments.UserError{ + ErrorCode: comments.ErrorStatusRecordNotFound, + } + } + return "", err + } + + // The existing comment must be pulled to validate the edit. The + // record lock must be held for the remainder of this function. + m := p.mutex(e.Token) + m.Lock() + defer m.Unlock() + + // Edit comment + var er *comments.EditReply + switch client.State { + case tlogbe.RecordStateUnvetted: + er, err = p.edit(client, *e, true) + if err != nil { + return "", err + } + case tlogbe.RecordStateVetted: + er, err = p.edit(client, *e, false) + if err != nil { + return "", err + } + default: + return "", fmt.Errorf("invalid record state %v", client.State) + } + + // Prepare reply + reply, err := comments.EncodeEditReply(*er) + if err != nil { + return "", err + } + + return string(reply), nil +} + +// This function must be called WITH the record lock held. +func (p *commentsPlugin) del(client *tlogbe.PluginClient, d comments.Del) (*comments.DelReply, error) { + // Get comments index + idx, err := indexLatest(client) + if err != nil { + return nil, err + } + + // Ensure comment exists + cidx, ok := idx.Comments[d.CommentID] + if !ok { + return nil, comments.UserError{ + ErrorCode: comments.ErrorStatusCommentNotFound, + } + } + + // Save delete record + receipt := p.id.SignMessage([]byte(d.Signature)) + cd := commentDel{ + Token: d.Token, + CommentID: d.CommentID, + Reason: d.Reason, + PublicKey: d.PublicKey, + Signature: d.Signature, + Receipt: hex.EncodeToString(receipt[:]), + Timestamp: time.Now().Unix(), + } + merkleHash, err := commentDelSave(client, cd) + if err != nil { + return nil, fmt.Errorf("commentDelSave: %v", err) + } + + // Update index + cidx.Del = merkleHash + idx.Comments[d.CommentID] = cidx + + // Save index + err = indexSave(client, *idx) + if err != nil { + return nil, fmt.Errorf("indexSave: %v", err) + } + + // Delete all comment versions + merkles := make([][]byte, 0, len(cidx.Adds)) + for _, v := range cidx.Adds { + merkles = append(merkles, v) + } + err = client.BlobsDel(merkles) + if err != nil { + return nil, fmt.Errorf("BlobsDel: %v", err) + } + + return &comments.DelReply{ + Receipt: cd.Receipt, + Timestamp: cd.Timestamp, + }, nil +} + +func (p *commentsPlugin) cmdDel(payload string) (string, error) { + log.Tracef("comments cmdDel: %v", payload) + + // Decode payload + d, err := comments.DecodeDel([]byte(payload)) + if err != nil { + return "", err + } + + // Verify signature + msg := d.Token + strconv.FormatUint(uint64(d.CommentID), 10) + d.Reason + err = verifySignature(d.Signature, d.PublicKey, msg) + if err != nil { + return "", err + } + + // Get plugin client + token, err := hex.DecodeString(d.Token) + if err != nil { + return "", comments.UserError{ + ErrorCode: comments.ErrorStatusTokenInvalid, + } + } + client, err := p.backend.PluginClient(token) + if err != nil { + if err == backend.ErrRecordNotFound { + return "", comments.UserError{ + ErrorCode: comments.ErrorStatusRecordNotFound, + } + } + return "", err + } + + // The comments index must be pulled and updated. The record lock + // must be held for the remainder of this function. + m := p.mutex(d.Token) + m.Lock() + defer m.Unlock() + + // Delete comment + cr, err := p.del(client, *d) + if err != nil { + return "", err + } + + // Prepare reply + reply, err := comments.EncodeDelReply(*cr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *commentsPlugin) cmdGet(payload string) (string, error) { + log.Tracef("comments cmdGet: %v", payload) + + // Decode payload + g, err := comments.DecodeGet([]byte(payload)) + if err != nil { + return "", err + } + + // Get plugin client + token, err := hex.DecodeString(g.Token) + if err != nil { + return "", comments.UserError{ + ErrorCode: comments.ErrorStatusTokenInvalid, + } + } + client, err := p.backend.PluginClient(token) + if err != nil { + if err == backend.ErrRecordNotFound { + return "", comments.UserError{ + ErrorCode: comments.ErrorStatusRecordNotFound, + } + } + return "", err + } + + // Get comments + cs, _, err := commentsLatest(client, g.CommentIDs) + if err != nil { + return "", fmt.Errorf("commentsLatest: %v", err) + } + + // Prepare reply + gr := comments.GetReply{ + Comments: cs, + } + reply, err := comments.EncodeGetReply(gr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *commentsPlugin) cmdGetAll(payload string) (string, error) { + log.Tracef("comments cmdGetAll: %v", payload) + + // Decode payload + ga, err := comments.DecodeGetAll([]byte(payload)) + if err != nil { + return "", err + } + + // Get plugin client + token, err := hex.DecodeString(ga.Token) + if err != nil { + return "", comments.UserError{ + ErrorCode: comments.ErrorStatusTokenInvalid, + } + } + client, err := p.backend.PluginClient(token) + if err != nil { + if err == backend.ErrRecordNotFound { + return "", comments.UserError{ + ErrorCode: comments.ErrorStatusRecordNotFound, + } + } + return "", err + } + + // Get comments index + idx, err := indexLatest(client) + if err != nil { + return "", fmt.Errorf("indexLatest: %v", err) + } + + // Aggregate merkle hashes of the comment add records that need to + // be looked up. If the comment has been deleted then there is + // nothing to look up. + var ( + merkles = make([][]byte, 0, len(idx.Comments)) + dels = make([]uint32, 0, len(idx.Comments)) + ) + for k, v := range idx.Comments { + if v.Del != nil { + // Comment has been deleted + dels = append(dels, k) + continue + } + + // Save the merkle hash for the latest version + version := commentVersionLatest(v) + merkles = append(merkles, v.Adds[version]) + } + + // Get comment add records + adds, err := commentAdds(client, merkles) + if err != nil { + return "", fmt.Errorf("commentAdds: %v", err) + } + if len(adds) != len(merkles) { + return "", fmt.Errorf("wrong comment adds count; got %v, want %v", + len(adds), len(merkles)) + } + + // Prepare comments + cs := make([]comments.Comment, 0, len(idx.Comments)) + for _, v := range adds { + c := convertCommentFromCommentAdd(v) + c.Score = idx.Comments[c.CommentID].Score + cs = append(cs, c) + } + for _, commentID := range dels { + score := idx.Comments[commentID].Score + cs = append(cs, comments.Comment{ + Token: hex.EncodeToString(client.Token), + CommentID: commentID, + Score: score, + Deleted: true, + }) + } + + // Order comments by comment ID + sort.SliceStable(cs, func(i, j int) bool { + return cs[i].CommentID < cs[j].CommentID + }) + + // Prepare reply + gar := comments.GetAllReply{ + Comments: cs, + } + reply, err := comments.EncodeGetAllReply(gar) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *commentsPlugin) cmdGetVersion(payload string) (string, error) { + log.Tracef("comments cmdGetVersion: %v", payload) + + // Decode payload + gv, err := comments.DecodeGetVersion([]byte(payload)) + if err != nil { + return "", err + } + + // Get plugin client + token, err := hex.DecodeString(gv.Token) + if err != nil { + return "", comments.UserError{ + ErrorCode: comments.ErrorStatusTokenInvalid, + } + } + client, err := p.backend.PluginClient(token) + if err != nil { + if err == backend.ErrRecordNotFound { + return "", comments.UserError{ + ErrorCode: comments.ErrorStatusRecordNotFound, + } + } + return "", err + } + + // Get comments index + idx, err := indexLatest(client) + if err != nil { + return "", fmt.Errorf("indexLatest: %v", err) + } + + // Ensure comment exists + cidx, ok := idx.Comments[gv.CommentID] + if !ok { + return "", comments.UserError{ + ErrorCode: comments.ErrorStatusCommentNotFound, + } + } + merkleHash, ok := cidx.Adds[gv.Version] + if !ok { + e := fmt.Sprintf("comment %v does not have version %v", + gv.CommentID, gv.Version) + return "", comments.UserError{ + ErrorCode: comments.ErrorStatusCommentNotFound, + ErrorContext: []string{e}, + } + } + + // Get comment add record + adds, err := commentAdds(client, [][]byte{merkleHash}) + if err != nil { + return "", fmt.Errorf("commentAdds: %v", err) + } + if len(adds) != 1 { + return "", fmt.Errorf("wrong comment adds count; got %v, want 1", + len(adds)) + } + + // Convert to a comment + c := convertCommentFromCommentAdd(adds[0]) + c.Score = cidx.Score + + // Prepare reply + gvr := comments.GetVersionReply{ + Comment: c, + } + reply, err := comments.EncodeGetVersionReply(gvr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *commentsPlugin) cmdCount(payload string) (string, error) { + log.Tracef("comments cmdCount: %v", payload) + + // Decode payload + c, err := comments.DecodeCount([]byte(payload)) + if err != nil { + return "", err + } + + // Get plugin client + token, err := hex.DecodeString(c.Token) + if err != nil { + return "", comments.UserError{ + ErrorCode: comments.ErrorStatusTokenInvalid, + } + } + client, err := p.backend.PluginClient(token) + if err != nil { + if err == backend.ErrRecordNotFound { + return "", comments.UserError{ + ErrorCode: comments.ErrorStatusRecordNotFound, + } + } + return "", err + } + + // Get comments index + idx, err := indexLatest(client) + if err != nil { + return "", fmt.Errorf("indexLatest: %v", err) + } + + // Prepare reply + cr := comments.CountReply{ + Count: uint64(len(idx.Comments)), + } + reply, err := comments.EncodeCountReply(cr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +// indexAddCommentVote adds the provided comment vote to the index and +// calculates the new vote score. The updated index is returned. The effect of +// a new vote on a comment depends on the previous vote from that uuid. +// Example, a user upvotes a comment that they have already upvoted, the +// resulting vote score is 0 due to the second upvote removing the original +// upvote. +func indexAddCommentVote(idx index, cv commentVote, merkleHash []byte) index { + // Get the existing votes for this uuid + cidx := idx.Comments[cv.CommentID] + votes, ok := cidx.Votes[cv.UUID] + if !ok { + // This uuid has not cast any votes + votes = make([]voteIndex, 0, 1) + } + + // Get the previous vote that this uuid made + var votePrev comments.VoteT + if len(votes) != 0 { + prev = votes[len(votes)-1].Vote + } + + // Update index vote score + voteNew := comments.VoteT(cv.Vote) + switch { + case votePrev == 0: + // No previous vote. Add the new vote to the score. + cidx.Score += int64(voteNew) + + case voteNew == votePrev: + // New vote is the same as the previous vote. Remove the previous + // vote from the score. + cidx.Score -= int64(votePrev) + + case voteNew != votePrev: + // New vote is different than the previous vote. Remove the + // previous vote from the score and add the new vote to the + // score. + cidx.Score -= int64(votePrev) + cidx.Score += int64(voteNew) + } + + // Update the index + votes = append(votes, voteIndex{ + Vote: comments.VoteT(cv.Vote), + MerkleHash: merkleHash, + }) + cidx.Votes[cv.UUID] = votes + idx.Comments[cv.CommentID] = cidx + + return idx +} + +func (p *commentsPlugin) cmdVote(payload string) (string, error) { + log.Tracef("comments cmdVote: %v", payload) + + // Decode payload + v, err := comments.DecodeVote([]byte(payload)) + if err != nil { + return "", err + } + + // Validate vote + switch v.Vote { + case comments.VoteDownvote: + // This is allowed + case comments.VoteUpvote: + // This is allowed + default: + return "", comments.UserError{ + ErrorCode: comments.ErrorStatusVoteInvalid, + } + } + + // Validate signature + msg := v.Token + strconv.FormatUint(uint64(v.CommentID), 10) + + strconv.FormatInt(int64(v.Vote), 10) + err = verifySignature(v.Signature, v.PublicKey, msg) + if err != nil { + return "", err + } + + // Get plugin client + token, err := hex.DecodeString(v.Token) + if err != nil { + return "", comments.UserError{ + ErrorCode: comments.ErrorStatusTokenInvalid, + } + } + client, err := p.backend.PluginClient(token) + if err != nil { + if err == backend.ErrRecordNotFound { + return "", comments.UserError{ + ErrorCode: comments.ErrorStatusRecordNotFound, + } + } + return "", err + } + + // The comments index must be pulled and updated. The record lock + // must be held for the remainder of this function. + m := p.mutex(v.Token) + m.Lock() + defer m.Unlock() + + // Get comments index + idx, err := indexLatest(client) + if err != nil { + return "", fmt.Errorf("indexLatest: %v", err) + } + + // Ensure comment exists + cidx, ok := idx.Comments[v.CommentID] + if !ok { + return "", comments.UserError{ + ErrorCode: comments.ErrorStatusCommentNotFound, + } + } + + // Ensure user has not exceeded max allowed vote changes + uvotes, ok := cidx.Votes[v.UUID] + if !ok { + uvotes = make([]voteIndex, 0) + } + if len(uvotes) > comments.PolicyMaxVoteChanges { + return "", comments.UserError{ + ErrorCode: comments.ErrorStatusMaxVoteChanges, + } + } + + // Prepare comment vote + receipt := p.id.SignMessage([]byte(v.Signature)) + cv := commentVote{ + UUID: v.UUID, + Token: v.Token, + CommentID: v.CommentID, + Vote: int64(v.Vote), + PublicKey: v.PublicKey, + Signature: v.Signature, + Receipt: hex.EncodeToString(receipt[:]), + Timestamp: time.Now().Unix(), + } + + // Save comment vote + merkleHash, err := commentVoteSave(client, cv) + if err != nil { + return "", fmt.Errorf("commentVoteSave: %v", err) + } + + // Update index + updatedIdx := indexAddCommentVote(*idx, cv, merkleHash) + + // Save index + err = indexSave(client, updatedIdx) + if err != nil { + return "", fmt.Errorf("indexSave: %v", err) + } + + // Prepare reply + vr := comments.VoteReply{ + Receipt: cv.Receipt, + Timestamp: cv.Timestamp, + Score: updatedIdx.Comments[cv.CommentID].Score, + } + reply, err := comments.EncodeVoteReply(vr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *commentsPlugin) cmdProofs(payload string) (string, error) { + log.Tracef("comments cmdProof: %v", payload) + return "", nil +} + +// Cmd executes a plugin command. +func (p *commentsPlugin) Cmd(cmd, payload string) (string, error) { + log.Tracef("comments Cmd: %v", cmd) + + switch cmd { + case comments.CmdNew: + return p.cmdNew(payload) + case comments.CmdEdit: + return p.cmdEdit(payload) + case comments.CmdDel: + return p.cmdDel(payload) + case comments.CmdGet: + return p.cmdGet(payload) + case comments.CmdGetAll: + return p.cmdGetAll(payload) + case comments.CmdGetVersion: + return p.cmdGetVersion(payload) + case comments.CmdCount: + return p.cmdCount(payload) + case comments.CmdVote: + return p.cmdVote(payload) + case comments.CmdProofs: + return p.cmdProofs(payload) + } + + return "", plugin.ErrInvalidPluginCmd +} + +// Hook executes a plugin hook. +func (p *commentsPlugin) Hook(h plugin.HookT, payload string) error { + log.Tracef("comments: Hook: %v", plugin.Hook[h]) + return nil +} + +// Fsck performs a plugin filesystem check. +func (p *commentsPlugin) Fsck() error { + log.Tracef("comments: Fsck") + + // Make sure commentDel blobs were actually deleted + + return nil +} + +// Setup performs any plugin setup work that needs to be done. +func (p *commentsPlugin) Setup() error { + log.Tracef("comments: Setup") + return nil +} + +// New returns a new comments plugin. +func New(id *identity.FullIdentity, backend *tlogbe.Tlogbe) (*commentsPlugin, error) { return &commentsPlugin{ - tlog: tlog, - store: store, + id: id, backend: backend, + mutexes: make(map[string]*sync.Mutex), }, nil } diff --git a/politeiad/backend/tlogbe/plugin/comments/journal.go b/politeiad/backend/tlogbe/plugin/comments/journal.go index 8ddfc1299..2c0d4f561 100644 --- a/politeiad/backend/tlogbe/plugin/comments/journal.go +++ b/politeiad/backend/tlogbe/plugin/comments/journal.go @@ -4,101 +4,49 @@ package comments -import "encoding/json" - -const ( - // JournalVersion is the current version of the comments journal. - journalVersion = "1" - - // keyPrefixJournal is the prefix to the key-value store key for a - // journal record. - keyPrefixJournal = "journal" - - // Journal actions - journalActionAdd = "add" // Add entry - journalActionEdit = "edit" // Edit entry - journalActionDel = "del" // Delete entry - journalActionCensor = "censor" // Censor entry - journalActionVote = "vote" // Vote on entry -) - -var ( - // Pregenerated journal actions - journalAdd []byte - journalEdit []byte - journalDel []byte - journalCensor []byte - journalVote []byte -) - -// journalAction prefixes and determines what the next structure is in the JSON -// journal. -type journalAction struct { - Version string `json:"version"` - Action string `json:"action"` +// The comments plugin treats the trillian tree as a journal. The following +// types are the journal actions that are saved to disk. + +// commentAdd is the structure that is saved to disk when a comment is created +// or edited. +type commentAdd struct { + // Data generated by client + Token string `json:"token"` // Record token + ParentID uint32 `json:"parentid"` // Parent comment ID + Comment string `json:"comment"` // Comment + PublicKey string `json:"publickey"` // Pubkey used for Signature + Signature string `json:"signature"` // Signature of Token+ParentID+Comment + + // Metadata generated by server + CommentID uint32 `json:"commentid"` // Comment ID + Version uint32 `json:"version"` // Comment version + Receipt string `json:"receipt"` // Server signature of client signature + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp } -type actionAdd struct { - ParentID uint32 `json:"parentid"` - Comment string `json:"comment"` - PublicKey string `json:"publickey"` - Signature string `json:"signature"` - CommentID uint32 `json:"commentid"` - Receipt string `json:"receipt"` - Timestamp int64 `json:"timestamp"` -} - -type actionEdit struct{} - -type actionDel struct{} - -type actionCensor struct{} - -type actionVote struct{} - -// keyJournal returns the key-value store key for a journal record. The token -// is not included in any journal actions since it is a static value and is -// already part of the key. -func keyJournal(token string) string { - return keyPrefixJournal + token +// commentDel is the structure that is saved to disk when a comment is deleted. +type commentDel struct { + // Data generated by client + Token string `json:"token"` // Record token + CommentID uint32 `json:"commentid"` // Comment ID + Reason string `json:"reason"` // Reason for deleting + PublicKey string `json:"publickey"` // Pubkey used for Signature + Signature string `json:"signature"` // Signature of Token+CommentID+Reason + + // Metadata generated by server + Receipt string `json:"receipt"` // Server signature of client signature + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp } -// init is used to pregenerate the JSON journal actions. -func init() { - var err error - journalAdd, err = json.Marshal(journalAction{ - Version: journalVersion, - Action: journalActionAdd, - }) - if err != nil { - panic(err.Error()) - } - journalEdit, err = json.Marshal(journalAction{ - Version: journalVersion, - Action: journalActionEdit, - }) - if err != nil { - panic(err.Error()) - } - journalDel, err = json.Marshal(journalAction{ - Version: journalVersion, - Action: journalActionDel, - }) - if err != nil { - panic(err.Error()) - } - journalCensor, err = json.Marshal(journalAction{ - Version: journalVersion, - Action: journalActionCensor, - }) - if err != nil { - panic(err.Error()) - } - journalVote, err = json.Marshal(journalAction{ - Version: journalVersion, - Action: journalActionVote, - }) - if err != nil { - panic(err.Error()) - } +// commentVote is the structure that is saved to disk when a comment is voted +// on. +type commentVote struct { + UUID string `json:"uuid"` // Unique user ID + Token string `json:"token"` // Record token + CommentID uint32 `json:"commentid"` // Comment ID + Vote int64 `json:"vote"` // Upvote or downvote + PublicKey string `json:"publickey"` // Public key used for signature + Signature string `json:"signature"` // Signature of Token+CommentID+Vote + Receipt string `json:"receipt"` // Server signature of client signature + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp } diff --git a/politeiad/backend/tlogbe/plugin/plugin.go b/politeiad/backend/tlogbe/plugin/plugin.go index 25936a9bd..6b02c3bca 100644 --- a/politeiad/backend/tlogbe/plugin/plugin.go +++ b/politeiad/backend/tlogbe/plugin/plugin.go @@ -1,14 +1,46 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + package plugin import "errors" +type HookT int + +const ( + // Plugin hooks + HookInvalid HookT = 0 + HookPostNewRecord HookT = 1 + HookPostEditRecord HookT = 2 + HookPostEditMetadata HookT = 3 + HookPostSetRecordStatus HookT = 4 +) + var ( + // Human readable plugin hooks + Hook = map[HookT]string{ + HookPostNewRecord: "post new record", + HookPostEditRecord: "post edit record", + HookPostEditMetadata: "post edit metadata", + HookPostSetRecordStatus: "post set record status", + } + // ErrInvalidPluginCmd is emitted when an invalid plugin command is // used. ErrInvalidPluginCmd = errors.New("invalid plugin command") ) type Plugin interface { + // Perform plugin setup Setup() error - Cmd(id, payload string) (string, error) + + // Execute a plugin command + Cmd(cmd, payload string) (string, error) + + // Execute a plugin hook + Hook(h HookT, payload string) error + + // Perform a plugin file system check + Fsck() error } diff --git a/politeiad/backend/tlogbe/pluginclient.go b/politeiad/backend/tlogbe/pluginclient.go new file mode 100644 index 000000000..04a812698 --- /dev/null +++ b/politeiad/backend/tlogbe/pluginclient.go @@ -0,0 +1,289 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlogbe + +import ( + "bytes" + "encoding/hex" + "fmt" + + "github.com/google/trillian" + "google.golang.org/grpc/codes" +) + +type RecordStateT int + +const ( + // Record types + RecordStateInvalid RecordStateT = 0 + RecordStateUnvetted RecordStateT = 1 + RecordStateVetted RecordStateT = 2 +) + +// PluginClient provides an API for plugins to save, retrieve, and delete +// plugin data for a specific record. Editing data is not allowed. +type PluginClient struct { + Token []byte + State RecordStateT + treeID int64 + tlog *tlog +} + +// hashes and keys must share the same ordering. +func (c *PluginClient) BlobsSave(keyPrefix string, blobs, hashes [][]byte, encrypt bool) ([][]byte, error) { + log.Tracef("BlobsSave: %v %v %v %x", c.treeID, keyPrefix, encrypt, hashes) + + // Ensure tree exists and is not frozen + if !c.tlog.treeExists(c.treeID) { + return nil, errRecordNotFound + } + _, err := c.tlog.freezeRecord(c.treeID) + if err != errFreezeRecordNotFound { + return nil, errTreeIsFrozen + } + + // Encrypt blobs if specified + if encrypt { + for k, v := range blobs { + e, err := c.tlog.encryptionKey.encrypt(0, v) + if err != nil { + return nil, err + } + blobs[k] = e + } + } + + // Save blobs to store + keys, err := c.tlog.store.Put(blobs) + if err != nil { + return nil, fmt.Errorf("store Put: %v", err) + } + if len(keys) != len(blobs) { + return nil, fmt.Errorf("wrong number of keys: got %v, want %v", + len(keys), len(blobs)) + } + + // Prepare log leaves. hashes and keys share the same ordering. + leaves := make([]*trillian.LogLeaf, 0, len(blobs)) + for k := range blobs { + pk := []byte(keyPrefix + keys[k]) + leaves = append(leaves, logLeafNew(hashes[k], pk)) + } + + // Append leaves to trillian tree + queued, _, err := c.tlog.trillian.leavesAppend(c.treeID, leaves) + if err != nil { + return nil, fmt.Errorf("leavesAppend: %v", err) + } + if len(queued) != len(leaves) { + return nil, fmt.Errorf("wrong number of queued leaves: got %v, want %v", + len(queued), len(leaves)) + } + failed := make([]string, 0, len(queued)) + for _, v := range queued { + c := codes.Code(v.QueuedLeaf.GetStatus().GetCode()) + if c != codes.OK { + failed = append(failed, fmt.Sprintf("%v", c)) + } + } + if len(failed) > 0 { + return nil, fmt.Errorf("append leaves failed: %v", failed) + } + + merkles := make([][]byte, 0, len(blobs)) + for _, v := range queued { + merkles = append(merkles, v.QueuedLeaf.Leaf.MerkleLeafHash) + } + + return merkles, nil +} + +func (c *PluginClient) BlobsDel(merkleHashes [][]byte) error { + log.Tracef("BlobsDel: %v %x", c.treeID, merkleHashes) + + // Ensure tree exists. We allow blobs to be deleted from both + // frozen and non frozen trees. + if !c.tlog.treeExists(c.treeID) { + return errRecordNotFound + } + + // Get all tree leaves + leaves, err := c.tlog.trillian.leavesAll(c.treeID) + if err != nil { + return err + } + + // Aggregate the key-value store keys for the provided merkle + // hashes. + merkles := make(map[string]struct{}, len(leaves)) + for _, v := range merkleHashes { + merkles[hex.EncodeToString(v)] = struct{}{} + } + keys := make([]string, 0, len(merkles)) + for _, v := range leaves { + _, ok := merkles[hex.EncodeToString(v.MerkleLeafHash)] + if ok { + key, err := extractKeyFromLeaf(v) + if err != nil { + return err + } + keys = append(keys, key) + } + } + + // Delete file blobs from the store + err = c.tlog.store.Del(keys) + if err != nil { + return fmt.Errorf("store Del: %v", err) + } + + return nil +} + +// If a blob does not exist it will not be included in the returned map. It is +// the responsibility of the caller to check that a blob is returned for each +// of the provided merkle hashes. +func (c *PluginClient) BlobsByMerkleHash(merkleHashes [][]byte) (map[string][]byte, error) { + log.Tracef("BlobsByMerkleHash: %v %x", merkleHashes) + + // Get leaves + leavesAll, err := c.tlog.trillian.leavesAll(c.treeID) + if err != nil { + return nil, fmt.Errorf("leavesAll: %v", err) + } + + // Aggregate the leaves that correspond to the provided merkle + // hashes. + // map[merkleHash]*trillian.LogLeaf + leaves := make(map[string]*trillian.LogLeaf, len(merkleHashes)) + for _, v := range merkleHashes { + leaves[hex.EncodeToString(v)] = nil + } + for _, v := range leavesAll { + m := hex.EncodeToString(v.MerkleLeafHash) + if _, ok := leaves[m]; ok { + leaves[m] = v + } + } + + // Ensure a leaf was found for all provided merkle hashes + for k, v := range leaves { + if v == nil { + return nil, fmt.Errorf("leaf not found for merkle hash: %v", k) + } + } + + // Extract the key-value store keys. These keys MUST be put in the + // same order that the merkle hashes were provided in. + keys := make([]string, 0, len(leaves)) + for _, v := range merkleHashes { + l, ok := leaves[hex.EncodeToString(v)] + if !ok { + return nil, fmt.Errorf("leaf not found for merkle hash: %x", v) + } + k, err := extractKeyFromLeaf(l) + if err != nil { + return nil, err + } + keys = append(keys, k) + } + + // Pull the blobs from the store. If is ok if one or more blobs is + // not found. It is the responsibility of the caller to decide how + // this should be handled. + blobs, err := c.tlog.store.Get(keys) + if err != nil { + return nil, fmt.Errorf("store Get: %v", err) + } + + // Decrypt any encrypted blobs + for k, v := range blobs { + if blobIsEncrypted(v) { + b, _, err := c.tlog.encryptionKey.decrypt(v) + if err != nil { + return nil, err + } + blobs[k] = b + } + } + + // Put blobs in a map so the caller can determine if any of the + // provided merkle hashes did not correspond to a blob in the + // store. + b := make(map[string][]byte, len(blobs)) // [merkleHash]blob + for k, v := range keys { + // The merkle hashes slice and keys slice share the same order + merkleHash := hex.EncodeToString(merkleHashes[k]) + blob, ok := blobs[v] + if !ok { + return nil, fmt.Errorf("blob not found for key %v", v) + } + b[merkleHash] = blob + } + + return b, nil +} + +func (c *PluginClient) BlobsByKeyPrefix(keyPrefix string) ([][]byte, error) { + // Get leaves + leaves, err := c.tlog.trillian.leavesAll(c.treeID) + if err != nil { + return nil, fmt.Errorf("leavesAll: %v", err) + } + + // Walk leaves and aggregate the key-value store keys for all + // leaves with a matching key prefix. + keys := make([]string, 0, len(leaves)) + for _, v := range leaves { + if bytes.HasPrefix(v.ExtraData, []byte(keyPrefix)) { + k, err := extractKeyFromLeaf(v) + if err != nil { + return nil, err + } + keys = append(keys, k) + } + } + + // Pull the blobs from the store + blobs, err := c.tlog.store.Get(keys) + if err != nil { + return nil, fmt.Errorf("store Get: %v", err) + } + if len(blobs) != len(keys) { + // One or more blobs were not found + missing := make([]string, 0, len(keys)) + for _, v := range keys { + _, ok := blobs[v] + if !ok { + missing = append(missing, v) + } + } + return nil, fmt.Errorf("blobs not found: %v", missing) + } + + // Decrypt any encrypted blobs + for k, v := range blobs { + if blobIsEncrypted(v) { + b, _, err := c.tlog.encryptionKey.decrypt(v) + if err != nil { + return nil, err + } + blobs[k] = b + } + } + + // Covert blobs from map to slice + b := make([][]byte, 0, len(blobs)) + for _, v := range blobs { + b = append(b, v) + } + + return b, nil +} + +// TODO +func (t *Tlogbe) PluginClient(token []byte) (*PluginClient, error) { + return nil, nil +} diff --git a/politeiad/backend/tlogbe/store/store.go b/politeiad/backend/tlogbe/store/store.go index 18ad95128..b3a3c10ec 100644 --- a/politeiad/backend/tlogbe/store/store.go +++ b/politeiad/backend/tlogbe/store/store.go @@ -4,7 +4,22 @@ package store -import "errors" +import ( + "bytes" + "compress/gzip" + "encoding/base64" + "encoding/gob" + "encoding/hex" + "errors" + + "github.com/decred/politeia/util" +) + +const ( + // Data descriptor types. These may be freely edited since they are + // solely hints to the application. + DataTypeStructure = "struct" // Descriptor contains a structure +) var ( // ErrNotFound is emitted when a blob is not found. @@ -25,6 +40,59 @@ type Blob interface { Batch(Ops) error } +// DataDescriptor provides hints about a data blob. In practise we JSON encode +// this struture and stuff it into BlobEntry.DataHint. +type DataDescriptor struct { + Type string `json:"type"` // Type of data + Descriptor string `json:"descriptor"` // Description of the data + ExtraData string `json:"extradata,omitempty"` // Value to be freely used +} + +// BlobEntry is the structure used to store data in the Blob key-value store. +// All data in the Blob key-value store will be encoded as a BlobEntry. +type BlobEntry struct { + Hash string `json:"hash"` // SHA256 hash of data payload, hex encoded + DataHint string `json:"datahint"` // Hint that describes data, base64 encoded + Data string `json:"data"` // Data payload, base64 encoded +} + +func Blobify(be BlobEntry) ([]byte, error) { + var b bytes.Buffer + zw := gzip.NewWriter(&b) + enc := gob.NewEncoder(zw) + err := enc.Encode(be) + if err != nil { + return nil, err + } + err = zw.Close() // we must flush gzip buffers + if err != nil { + return nil, err + } + return b.Bytes(), nil +} + +func Deblob(blob []byte) (*BlobEntry, error) { + zr, err := gzip.NewReader(bytes.NewReader(blob)) + if err != nil { + return nil, err + } + r := gob.NewDecoder(zr) + var be BlobEntry + err = r.Decode(&be) + if err != nil { + return nil, err + } + return &be, nil +} + +func BlobEntryNew(dataHint, data []byte) BlobEntry { + return BlobEntry{ + Hash: hex.EncodeToString(util.Digest(data)), + DataHint: base64.StdEncoding.EncodeToString(dataHint), + Data: base64.StdEncoding.EncodeToString(data), + } +} + // Blob represents a blob key-value store. type Blob_ interface { // Get returns blobs from the store for the provided keys. An entry diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index c00c09141..1e17d492d 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -13,6 +13,7 @@ import ( "errors" "fmt" "strconv" + "sync" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/store" @@ -29,6 +30,7 @@ const ( dataDescriptorMetadataStream = "metadatastream" dataDescriptorRecordIndex = "recordindex" dataDescriptorFreezeRecord = "freezerecord" + dataDescriptorAnchor = "anchor" // The keys for kv store blobs are saved by stuffing them into the // ExtraData field of their corresponding trillian log leaf. The @@ -38,43 +40,64 @@ const ( // data out of the store, which can become an issue in situations // such as searching for a record index that has been buried by // thousands of leaves from plugin data. - keyPrefixRecordIndex = "index:" + keyPrefixRecordIndex = "recordindex:" keyPrefixRecordContent = "record:" keyPrefixFreezeRecord = "freeze:" + keyPrefixAnchorRecord = "anchor:" ) var ( - // errRecordNotFound is emitted when a record is not found. + // errRecordNotFound is emitted when a record is not found. This + // can be because a tree does not exists for the provided tree id + // or when a tree does exist but the specified record version does + // not exist. errRecordNotFound = errors.New("record not found") - errFreezeRecordNotFound = errors.New("freeze record not found") - - // errNoFileChanges is emitted when a new version of a record is - // attemtepd to be saved with no changes to the files. + // errNoFileChanges is emitted when there are no files being + // changed. errNoFileChanges = errors.New("no file changes") // errNoMetadataChanges is emitted when there are no metadata - // changes being made on a metadata update. + // changes being made. errNoMetadataChanges = errors.New("no metadata changes") + + // errFreezeRecordNotFound is emitted when a freeze record does not + // exist for a tree. + errFreezeRecordNotFound = errors.New("freeze record not found") + + // errTreeIsFrozen is emitted when a frozen tree is attempted to be + // altered. + errTreeIsFrozen = errors.New("tree is frozen") ) // We do not unwind. type tlog struct { + sync.Mutex + // TODO shutdown bool + id string dataDir string - encryptionKey *EncryptionKey - client *TrillianClient + encryptionKey *encryptionKey + trillian *trillianClient store store.Blob_ + dcrtimeHost string + cron *cron.Cron - // TODO implement anchoring - dcrtimeHost string - cron *cron.Cron - - // droppingAnchor indicates whether tlogbe is in the process of + // droppingAnchor indicates whether tlog is in the process of // dropping an anchor, i.e. timestamping unanchored trillian trees // using dcrtime. An anchor is dropped periodically using cron. droppingAnchor bool } +// recordIndex contains the merkle leaf hashes of all the record content for a +// specific record version and iteration. The record index can be used to +// lookup the trillian log leaves for the record content. The ExtraData field +// of each log leaf contains the key-value store key for the record content +// data blob. +// +// Appending the record index leaf to the trillian tree is the last operation +// that occurs when updating a record, so if a record index leaf exists then +// the record update is considered valid and you can be sure that the data +// blobs were successfully saved to the key-value store. type recordIndex struct { // Version represents the version of the record. The version is // only incremented when the record files are updated. @@ -85,8 +108,8 @@ type recordIndex struct { // file changes that bump the version as well metadata stream and // record metadata changes that don't bump the version. // - // Note this is not the same as the RecordMetadata iteration, which - // does not get incremented on metadata stream updates. + // Note this is not the same as the RecordMetadata iteration field, + // which does not get incremented on metadata updates. Iteration uint32 `json:"iteration"` // The following fields contain the merkle leaf hashes of the @@ -98,10 +121,6 @@ type recordIndex struct { Files map[string][]byte `json:"files"` // [filename]merkle } -type freezeRecord struct { - TreeID int64 `json:"treeid,omitempty"` -} - func treeIDFromToken(token []byte) int64 { return int64(binary.LittleEndian.Uint64(token)) } @@ -114,140 +133,152 @@ func tokenFromTreeID(treeID int64) []byte { return b } -// isEncrypted returns whether the provided blob has been prefixed with an +// blobIsEncrypted returns whether the provided blob has been prefixed with an // sbox header, indicating that it is an encrypted blob. -func isEncrypted(b []byte) bool { +func blobIsEncrypted(b []byte) bool { return bytes.HasPrefix(b, []byte("sbox")) } -func isRecordIndexLeaf(l *trillian.LogLeaf) bool { +// treeIsFrozen deterimes if the tree is frozen given the full list of the +// tree's leaves. A tree is considered frozen if the last leaf on the tree +// corresponds to a freeze record. +func treeIsFrozen(leaves []*trillian.LogLeaf) bool { + return leafIsFreezeRecord(leaves[len(leaves)-1]) +} + +func leafIsRecordIndex(l *trillian.LogLeaf) bool { return bytes.HasPrefix(l.ExtraData, []byte(keyPrefixRecordIndex)) } -func isRecordContentLeaf(l *trillian.LogLeaf) bool { +func leafIsRecordContent(l *trillian.LogLeaf) bool { return bytes.HasPrefix(l.ExtraData, []byte(keyPrefixRecordContent)) } -func isFreezeRecordLeaf(l *trillian.LogLeaf) bool { +func leafIsFreezeRecord(l *trillian.LogLeaf) bool { return bytes.HasPrefix(l.ExtraData, []byte(keyPrefixFreezeRecord)) } -func extractKeyForRecordIndex(l *trillian.LogLeaf) (string, error) { - s := bytes.SplitAfter(l.ExtraData, []byte(keyPrefixRecordIndex)) - if len(s) != 2 { - return "", fmt.Errorf("invalid key %s", l.ExtraData) - } - return string(s[1]), nil -} - -func extractKeyForRecordContent(l *trillian.LogLeaf) (string, error) { - s := bytes.SplitAfter(l.ExtraData, []byte(keyPrefixRecordContent)) - if len(s) != 2 { - return "", fmt.Errorf("invalid key %s", l.ExtraData) - } - return string(s[1]), nil +func leafIsAnchorRecord(l *trillian.LogLeaf) bool { + return bytes.HasPrefix(l.ExtraData, []byte(keyPrefixAnchorRecord)) } -func extractKeyForFreezeRecord(l *trillian.LogLeaf) (string, error) { - s := bytes.SplitAfter(l.ExtraData, []byte(keyPrefixFreezeRecord)) +func extractKeyFromLeaf(l *trillian.LogLeaf) (string, error) { + s := bytes.SplitAfter(l.ExtraData, []byte(":")) if len(s) != 2 { return "", fmt.Errorf("invalid key %s", l.ExtraData) } return string(s[1]), nil } -func convertBlobEntryFromFile(f backend.File) (*blobEntry, error) { +func convertBlobEntryFromFile(f backend.File) (*store.BlobEntry, error) { data, err := json.Marshal(f) if err != nil { return nil, err } hint, err := json.Marshal( - dataDescriptor{ - Type: dataTypeStructure, + store.DataDescriptor{ + Type: store.DataTypeStructure, Descriptor: dataDescriptorFile, }) if err != nil { return nil, err } - be := blobEntryNew(hint, data) + be := store.BlobEntryNew(hint, data) return &be, nil } -func convertBlobEntryFromMetadataStream(ms backend.MetadataStream) (*blobEntry, error) { +func convertBlobEntryFromMetadataStream(ms backend.MetadataStream) (*store.BlobEntry, error) { data, err := json.Marshal(ms) if err != nil { return nil, err } hint, err := json.Marshal( - dataDescriptor{ - Type: dataTypeStructure, + store.DataDescriptor{ + Type: store.DataTypeStructure, Descriptor: dataDescriptorMetadataStream, }) if err != nil { return nil, err } - be := blobEntryNew(hint, data) + be := store.BlobEntryNew(hint, data) return &be, nil } -func convertBlobEntryFromRecordMetadata(rm backend.RecordMetadata) (*blobEntry, error) { +func convertBlobEntryFromRecordMetadata(rm backend.RecordMetadata) (*store.BlobEntry, error) { data, err := json.Marshal(rm) if err != nil { return nil, err } hint, err := json.Marshal( - dataDescriptor{ - Type: dataTypeStructure, + store.DataDescriptor{ + Type: store.DataTypeStructure, Descriptor: dataDescriptorRecordMetadata, }) if err != nil { return nil, err } - be := blobEntryNew(hint, data) + be := store.BlobEntryNew(hint, data) return &be, nil } -func convertBlobEntryFromRecordIndex(ri recordIndex) (*blobEntry, error) { +func convertBlobEntryFromRecordIndex(ri recordIndex) (*store.BlobEntry, error) { data, err := json.Marshal(ri) if err != nil { return nil, err } hint, err := json.Marshal( - dataDescriptor{ - Type: dataTypeStructure, + store.DataDescriptor{ + Type: store.DataTypeStructure, Descriptor: dataDescriptorRecordIndex, }) if err != nil { return nil, err } - be := blobEntryNew(hint, data) + be := store.BlobEntryNew(hint, data) return &be, nil } -func convertBlobEntryFromFreezeRecord(fr freezeRecord) (*blobEntry, error) { +func convertBlobEntryFromFreezeRecord(fr freezeRecord) (*store.BlobEntry, error) { data, err := json.Marshal(fr) if err != nil { return nil, err } hint, err := json.Marshal( - dataDescriptor{ - Type: dataTypeStructure, + store.DataDescriptor{ + Type: store.DataTypeStructure, Descriptor: dataDescriptorFreezeRecord, }) if err != nil { return nil, err } - be := blobEntryNew(hint, data) + be := store.BlobEntryNew(hint, data) + return &be, nil +} + +func convertBlobEntryFromAnchor(a anchor) (*store.BlobEntry, error) { + data, err := json.Marshal(a) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorAnchor, + }) + if err != nil { + return nil, err + } + be := store.BlobEntryNew(hint, data) return &be, nil } -func convertFreezeRecordFromBlobEntry(be blobEntry) (*freezeRecord, error) { +func convertFreezeRecordFromBlobEntry(be store.BlobEntry) (*freezeRecord, error) { // Decode and validate data hint b, err := base64.StdEncoding.DecodeString(be.DataHint) if err != nil { return nil, fmt.Errorf("decode DataHint: %v", err) } - var dd dataDescriptor + var dd store.DataDescriptor err = json.Unmarshal(b, &dd) if err != nil { return nil, fmt.Errorf("unmarshal DataHint: %v", err) @@ -279,13 +310,13 @@ func convertFreezeRecordFromBlobEntry(be blobEntry) (*freezeRecord, error) { return &fr, nil } -func convertRecordIndexFromBlobEntry(be blobEntry) (*recordIndex, error) { +func convertRecordIndexFromBlobEntry(be store.BlobEntry) (*recordIndex, error) { // Decode and validate data hint b, err := base64.StdEncoding.DecodeString(be.DataHint) if err != nil { return nil, fmt.Errorf("decode DataHint: %v", err) } - var dd dataDescriptor + var dd store.DataDescriptor err = json.Unmarshal(b, &dd) if err != nil { return nil, fmt.Errorf("unmarshal DataHint: %v", err) @@ -317,22 +348,182 @@ func convertRecordIndexFromBlobEntry(be blobEntry) (*recordIndex, error) { return &ri, nil } -func (t *tlog) leafLatest(treeID int64) (*trillian.LogLeaf, error) { +func (t *tlog) treeNew() (int64, error) { + tree, _, err := t.trillian.treeNew() + if err != nil { + return 0, err + } + return tree.TreeId, nil +} - return nil, nil +func (t *tlog) treeExists(treeID int64) bool { + _, err := t.trillian.tree(treeID) + if err == nil { + return true + } + return false } -func (t *tlog) freezeRecord(treeID int64) (*freezeRecord, error) { - // Get the last leaf of the tree - tree, err := t.client.tree(treeID) +type freezeRecord struct { + TreeID int64 `json:"treeid,omitempty"` +} + +// treeFreeze freezes the trillian tree for the provided token. Once a tree +// has been frozen it is no longer able to be appended to. The last leaf in a +// frozen tree will correspond to a freeze record in the key-value store. A +// tree is frozen when the status of the corresponding record is updated to a +// status that locks the record, such as when a record is censored. +// +// It's possible for this function to fail in between the append leaves call +// and the call that updates the tree status to frozen. If this happens the +// freeze record will still be the last leaf on the tree but the tree state +// will not be frozen. The tree is still considered frozen and no new leaves +// should be appended to it. The tree state will be updated in the next fsck. +// Functions that append new leaves onto the tree MUST ensure that the last +// leaf does not contain a freeze record. +func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, fr freezeRecord) error { + // Get tree leaves + leavesAll, err := t.trillian.leavesAll(treeID) + if err != nil { + return fmt.Errorf("leavesAll: %v", err) + } + + // Prepare kv store blobs for metadata + args := recordBlobsPrepareArgs{ + encryptionKey: t.encryptionKey, + leaves: leavesAll, + recordMD: rm, + metadata: metadata, + } + rbpr, err := recordBlobsPrepare(args) + if err != nil { + return err + } + + // Ensure metadata has been changed + if len(rbpr.blobs) == 0 { + return errNoMetadataChanges + } + + // Save metadata blobs + idx, err := t.recordBlobsSave(treeID, *rbpr) + if err != nil { + return fmt.Errorf("blobsSave: %v", err) + } + + // Get the existing record index and add the unchanged fields to + // the new record index. The version and files will remain the + // same. + oldIdx, err := t.recordIndexLatest(leavesAll) + if err != nil { + return fmt.Errorf("recordIndexLatest: %v", err) + } + idx.Version = oldIdx.Version + idx.Files = oldIdx.Files + + // Increment the iteration + idx.Iteration = oldIdx.Iteration + 1 + + // Sanity check. The record index should be fully populated at this + // point. + switch { + case idx.Version != oldIdx.Version: + return fmt.Errorf("invalid index version: got %v, want %v", + idx.Version, oldIdx.Version) + case idx.Version != oldIdx.Iteration+1: + return fmt.Errorf("invalid index iteration: got %v, want %v", + idx.Iteration, oldIdx.Iteration+1) + case idx.RecordMetadata == nil: + return fmt.Errorf("invalid index record metadata") + case len(idx.Metadata) != len(metadata): + return fmt.Errorf("invalid index metadata: got %v, want %v", + len(idx.Metadata), len(metadata)) + case len(idx.Files) != len(oldIdx.Files): + return fmt.Errorf("invalid index files: got %v, want %v", + len(idx.Files), len(oldIdx.Files)) + } + + // Blobify the record index and freeze record + blobs := make([][]byte, 0, 2) + be, err := convertBlobEntryFromRecordIndex(*idx) + if err != nil { + return err + } + idxHash, err := hex.DecodeString(be.Hash) + if err != nil { + return err + } + b, err := store.Blobify(*be) + if err != nil { + return err + } + blobs = append(blobs, b) + + be, err = convertBlobEntryFromFreezeRecord(fr) + if err != nil { + return err + } + freezeRecordHash, err := hex.DecodeString(be.Hash) + if err != nil { + return err + } + b, err = store.Blobify(*be) + if err != nil { + return err + } + blobs = append(blobs, b) + + // Save record index and freeze record blobs to the store + keys, err := t.store.Put(blobs) + if err != nil { + return fmt.Errorf("store Put: %v", err) + } + if len(keys) != 2 { + return fmt.Errorf("wrong number of keys: got %v, want 1", + len(keys)) + } + + // Append record index and freeze record leaves to trillian tree + leaves := []*trillian.LogLeaf{ + logLeafNew(idxHash, []byte(keyPrefixRecordIndex+keys[0])), + logLeafNew(freezeRecordHash, []byte(keyPrefixFreezeRecord+keys[1])), + } + queued, _, err := t.trillian.leavesAppend(treeID, leaves) + if len(queued) != 2 { + return fmt.Errorf("wrong number of queud leaves: got %v, want 2", + len(queued)) + } + failed := make([]string, 0, len(queued)) + for _, v := range queued { + c := codes.Code(v.QueuedLeaf.GetStatus().GetCode()) + if c != codes.OK { + failed = append(failed, fmt.Sprintf("%v", c)) + } + } + if len(failed) > 0 { + return fmt.Errorf("append leaves failed: %v", failed) + } + + // Update tree status to frozen + _, err = t.trillian.treeFreeze(treeID) + if err != nil { + return fmt.Errorf("treeFreeze: %v", err) + } + + return nil +} + +// lastLeaf returns the last leaf of the given trillian tree. +func (t *tlog) lastLeaf(treeID int64) (*trillian.LogLeaf, error) { + tree, err := t.trillian.tree(treeID) if err != nil { return nil, errRecordNotFound } - _, lr, err := t.client.signedLogRoot(tree) + _, lr, err := t.trillian.signedLogRootForTree(tree) if err != nil { - return nil, fmt.Errorf("signedLogRoot: %v", err) + return nil, fmt.Errorf("signedLogRootForTree: %v", err) } - leaves, err := t.client.leavesByRange(treeID, int64(lr.TreeSize)-1, 1) + leaves, err := t.trillian.leavesByRange(treeID, int64(lr.TreeSize)-1, 1) if err != nil { return nil, fmt.Errorf("leavesByRange: %v", err) } @@ -340,14 +531,24 @@ func (t *tlog) freezeRecord(treeID int64) (*freezeRecord, error) { return nil, fmt.Errorf("unexpected leaves count: got %v, want 1", len(leaves)) } - l := leaves[0] - if !isFreezeRecordLeaf(l) { + return leaves[0], nil +} + +// freeze record returns the freeze record of the provided tree if one exists. +// If one does not exists a errFreezeRecordNotFound error is returned. +func (t *tlog) freezeRecord(treeID int64) (*freezeRecord, error) { + // Check if the last leaf of the tree is a freeze record + l, err := t.lastLeaf(treeID) + if err != nil { + return nil, fmt.Errorf("lastLeaf %v: %v", treeID, err) + } + if !leafIsFreezeRecord(l) { // Leaf is not a freeze record return nil, errFreezeRecordNotFound } // The leaf is a freeze record. Get it from the store. - k, err := extractKeyForFreezeRecord(l) + k, err := extractKeyFromLeaf(l) if err != nil { return nil, err } @@ -357,7 +558,7 @@ func (t *tlog) freezeRecord(treeID int64) (*freezeRecord, error) { } if len(blobs) != 1 { return nil, fmt.Errorf("unexpected blobs count: got %v, want 1", - len(blobs), 1) + len(blobs)) } // Decode freeze record @@ -365,7 +566,7 @@ func (t *tlog) freezeRecord(treeID int64) (*freezeRecord, error) { if !ok { return nil, fmt.Errorf("blob not found %v", k) } - be, err := deblob(b) + be, err := store.Deblob(b) if err != nil { return nil, err } @@ -377,26 +578,99 @@ func (t *tlog) freezeRecord(treeID int64) (*freezeRecord, error) { return fr, nil } -func (t *tlog) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]*trillian.LogLeaf, error) { - // TODO Ensure the tree is not frozen - return nil, nil +func (t *tlog) recordIndexSave(treeID int64, ri recordIndex) error { + // Save record index to the store + be, err := convertBlobEntryFromRecordIndex(ri) + if err != nil { + return err + } + b, err := store.Blobify(*be) + if err != nil { + return err + } + keys, err := t.store.Put([][]byte{b}) + if err != nil { + return fmt.Errorf("store Put: %v", err) + } + if len(keys) != 1 { + return fmt.Errorf("wrong number of keys: got %v, want 1", + len(keys)) + } + + // Append record index leaf to trillian tree + h, err := hex.DecodeString(be.Hash) + if err != nil { + return err + } + prefixedKey := []byte(keyPrefixRecordIndex + keys[0]) + queued, _, err := t.trillian.leavesAppend(treeID, []*trillian.LogLeaf{ + logLeafNew(h, prefixedKey), + }) + if len(queued) != 1 { + return fmt.Errorf("wrong number of queud leaves: got %v, want 1", + len(queued)) + } + failed := make([]string, 0, len(queued)) + for _, v := range queued { + c := codes.Code(v.QueuedLeaf.GetStatus().GetCode()) + if c != codes.OK { + failed = append(failed, fmt.Sprintf("%v", c)) + } + } + if len(failed) > 0 { + return fmt.Errorf("append leaves failed: %v", failed) + } + + return nil +} + +func (t *tlog) recordIndexVersion(leaves []*trillian.LogLeaf, version uint32) (*recordIndex, error) { + indexes, err := t.recordIndexes(leaves) + if err != nil { + return nil, err + } + + // Return the record index for the specified version + var ri *recordIndex + if version == 0 { + // A version of 0 indicates that the most recent version should + // be returned. + ri = &indexes[len(indexes)-1] + } else { + // Walk the indexes backwards so the most recent iteration of the + // specified version is selected. + for i := len(indexes) - 1; i >= 0; i-- { + r := indexes[i] + if r.Version == version { + ri = &r + break + } + } + } + if ri == nil { + // The specified version does not exist + return nil, errRecordNotFound + } + + return ri, nil +} + +func (t *tlog) recordIndexLatest(leaves []*trillian.LogLeaf) (*recordIndex, error) { + return t.recordIndexVersion(leaves, 0) } -func (t *tlog) recordIndex(leaves []*trillian.LogLeaf, version uint32) (*recordIndex, error) { - // Walk the leaves and compile the keys of all the record indexes. - // Appending the record index leaf to the trillian tree is the last - // operation that occurs when updating a record, so if an index - // leaf exists then you can be sure that the index blob exists in - // the store as well as all of the record content blobs. It is - // possible for multiple indexes to exist for the same record +func (t *tlog) recordIndexes(leaves []*trillian.LogLeaf) ([]recordIndex, error) { + // Walk the leaves and compile the keys for all record indexes. It + // is possible for multiple indexes to exist for the same record // version (they will have different iterations due to metadata // only updates) so we have to pull the index blobs from the store - // in order to find the latest index for the specified version. + // in order to find the most recent iteration for the specified + // version. keys := make([]string, 0, 64) for _, v := range leaves { - if isRecordIndexLeaf(v) { + if leafIsRecordIndex(v) { // This is a record index leaf. Extract they kv store key. - k, err := extractKeyForRecordIndex(v) + k, err := extractKeyFromLeaf(v) if err != nil { return nil, err } @@ -424,9 +698,9 @@ func (t *tlog) recordIndex(leaves []*trillian.LogLeaf, version uint32) (*recordI return nil, fmt.Errorf("record index not found: %v", missing) } - indexes := make([]*recordIndex, 0, len(blobs)) + indexes := make([]recordIndex, 0, len(blobs)) for _, v := range blobs { - be, err := deblob(v) + be, err := store.Deblob(v) if err != nil { return nil, err } @@ -434,7 +708,7 @@ func (t *tlog) recordIndex(leaves []*trillian.LogLeaf, version uint32) (*recordI if err != nil { return nil, err } - indexes = append(indexes, ri) + indexes = append(indexes, *ri) } // Sanity check. Index iterations should start with 1 and be @@ -458,95 +732,24 @@ func (t *tlog) recordIndex(leaves []*trillian.LogLeaf, version uint32) (*recordI versionPrev = v.Version } - // Return the record index for the specified version. A version of - // 0 indicates that the most recent version should be returned. - var ri *recordIndex - if version == 0 { - ri = indexes[len(indexes)-1] - } else { - // Walk the indexes backwards so the most recent iteration of the - // specified version is selected. - for i := len(indexes) - 1; i >= 0; i-- { - r := indexes[i] - if r.Version == version { - ri = r - break - } - } - } - if ri == nil { - // The specified version does not exist - return nil, errRecordNotFound - } - - return ri, nil + return indexes, nil } -func (t *tlog) recordIndexLatest(leaves []*trillian.LogLeaf) (*recordIndex, error) { - return t.recordIndex(leaves, 0) +type recordHashes struct { + recordMetadata string // Record metadata hash + metadata map[string]uint64 // [hash]metadataID + files map[string]string // [hash]filename } -func (t *tlog) recordIndexSave(treeID int64, ri recordIndex) error { - // Save record index to the store - be, err := convertBlobEntryFromRecordIndex(ri) - if err != nil { - return err - } - b, err := blobify(*be) - if err != nil { - return err - } - keys, err := t.store.Put([][]byte{b}) - if err != nil { - return fmt.Errorf("store Put: %v", err) - } - if len(keys) != 1 { - return fmt.Errorf("wrong number of keys: got %v, want 1", - len(keys)) - } - - // Append record index leaf to trillian tree - h, err := hex.DecodeString(be.Hash) - if err != nil { - return err - } - prefixedKey := []byte(keyPrefixRecordIndex + keys[0]) - queued, _, err := t.client.LeavesAppend(treeID, []*trillian.LogLeaf{ - logLeafNew(h, prefixedKey), - }) - if len(queued) != 1 { - return fmt.Errorf("wrong number of queud leaves: got %v, want 1", - len(queued)) - } - failed := make([]string, 0, len(queued)) - for _, v := range queued { - c := codes.Code(v.QueuedLeaf.GetStatus().GetCode()) - if c != codes.OK { - failed = append(failed, fmt.Sprintf("%v", c)) - } - } - if len(failed) > 0 { - return fmt.Errorf("append leaves failed: %v", failed) - } - - return nil -} - -type recordHashes struct { - recordMetadata string // Record metadata hash - metadata map[string]uint64 // [hash]metadataID - files map[string]string // [hash]filename -} - -type blobsPrepareArgs struct { - encryptionKey *EncryptionKey +type recordBlobsPrepareArgs struct { + encryptionKey *encryptionKey leaves []*trillian.LogLeaf recordMD backend.RecordMetadata metadata []backend.MetadataStream files []backend.File } -type blobsPrepareReply struct { +type recordBlobsPrepareReply struct { recordIndex recordIndex recordHashes recordHashes @@ -556,8 +759,14 @@ type blobsPrepareReply struct { } // TODO test this function -// TODO if we find a freeze record we need to fail -func blobsPrepare(args blobsPrepareArgs) (*blobsPrepareReply, error) { +func recordBlobsPrepare(args recordBlobsPrepareArgs) (*recordBlobsPrepareReply, error) { + // Ensure tree is not frozen. A tree is considered frozen if the + // last leaf on the tree is a freeze record. + lastLeaf := args.leaves[len(args.leaves)-1] + if leafIsFreezeRecord(lastLeaf) { + return nil, errTreeIsFrozen + } + // Check if any of the content already exists. Different record // versions that reference the same data is fine, but this data // should not be saved to the store again. We can find duplicates @@ -646,7 +855,7 @@ func blobsPrepare(args blobsPrepareArgs) (*blobsPrepareReply, error) { if err != nil { return nil, err } - b, err = blobify(*be) + b, err = store.Blobify(*be) if err != nil { return nil, err } @@ -666,7 +875,7 @@ func blobsPrepare(args blobsPrepareArgs) (*blobsPrepareReply, error) { if err != nil { return nil, err } - b, err := blobify(*be) + b, err := store.Blobify(*be) if err != nil { return nil, err } @@ -687,13 +896,13 @@ func blobsPrepare(args blobsPrepareArgs) (*blobsPrepareReply, error) { if err != nil { return nil, err } - b, err := blobify(*be) + b, err := store.Blobify(*be) if err != nil { return nil, err } // Encypt file blobs if encryption key has been set if args.encryptionKey != nil { - b, err = args.encryptionKey.Encrypt(blobEntryVersion, b) + b, err = args.encryptionKey.encrypt(0, b) if err != nil { return nil, err } @@ -706,7 +915,7 @@ func blobsPrepare(args blobsPrepareArgs) (*blobsPrepareReply, error) { } } - return &blobsPrepareReply{ + return &recordBlobsPrepareReply{ recordIndex: index, recordHashes: rhashes, blobs: blobs, @@ -714,12 +923,12 @@ func blobsPrepare(args blobsPrepareArgs) (*blobsPrepareReply, error) { }, nil } -func (t *tlog) blobsSave(treeID int64, bpr blobsPrepareReply) (*recordIndex, error) { +func (t *tlog) recordBlobsSave(treeID int64, rbpr recordBlobsPrepareReply) (*recordIndex, error) { var ( - index = bpr.recordIndex - rhashes = bpr.recordHashes - blobs = bpr.blobs - hashes = bpr.hashes + index = rbpr.recordIndex + rhashes = rbpr.recordHashes + blobs = rbpr.blobs + hashes = rbpr.hashes ) // Save blobs to store @@ -740,9 +949,9 @@ func (t *tlog) blobsSave(treeID int64, bpr blobsPrepareReply) (*recordIndex, err } // Append leaves to trillian tree - queued, _, err := t.client.LeavesAppend(treeID, leaves) + queued, _, err := t.trillian.leavesAppend(treeID, leaves) if err != nil { - return nil, fmt.Errorf("LeavesAppend: %v", err) + return nil, fmt.Errorf("leavesAppend: %v", err) } if len(queued) != len(leaves) { return nil, fmt.Errorf("wrong number of queued leaves: got %v, want %v", @@ -793,60 +1002,223 @@ func (t *tlog) blobsSave(treeID int64, bpr blobsPrepareReply) (*recordIndex, err return &index, nil } -func (t *tlog) treeNew() (int64, error) { - tree, _, err := t.client.treeNew() +// We do not unwind. +func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { + // Ensure tree exists + if !t.treeExists(treeID) { + return errRecordNotFound + } + + // Get tree leaves + leavesAll, err := t.trillian.leavesAll(treeID) if err != nil { - return 0, err + return fmt.Errorf("leavesAll %v: %v", treeID, err) } - return tree.TreeId, nil -} -func (t *tlog) treeExists(treeID int64) bool { - _, err := t.client.tree(treeID) - if err == nil { - return true + // Ensure tree is not frozen + if treeIsFrozen(leavesAll) { + return errTreeIsFrozen } - return false + + // Prepare kv store blobs + args := recordBlobsPrepareArgs{ + encryptionKey: t.encryptionKey, + leaves: leavesAll, + recordMD: rm, + metadata: metadata, + files: files, + } + rbpr, err := recordBlobsPrepare(args) + if err != nil { + return err + } + + // Ensure file changes are being made + if len(rbpr.recordIndex.Files) == len(files) { + return errNoFileChanges + } + + // Save blobs + idx, err := t.recordBlobsSave(treeID, *rbpr) + if err != nil { + return fmt.Errorf("blobsSave: %v", err) + } + + // Get the existing record index and use it to bump the version and + // iteration of the new record index. + oldIdx, err := t.recordIndexLatest(leavesAll) + if err == errRecordNotFound { + // No record versions exist yet. This is fine. The version and + // iteration will be incremented to 1. + } else if err != nil { + return fmt.Errorf("recordIndexLatest: %v", err) + } + idx.Version = oldIdx.Version + 1 + idx.Iteration = oldIdx.Iteration + 1 + + // Sanity check. The record index should be fully populated at this + // point. + switch { + case idx.Version != oldIdx.Version+1: + return fmt.Errorf("invalid index version: got %v, want %v", + idx.Version, oldIdx.Version+1) + case idx.Iteration != oldIdx.Iteration+1: + return fmt.Errorf("invalid index iteration: got %v, want %v", + idx.Iteration, oldIdx.Iteration+1) + case idx.RecordMetadata == nil: + return fmt.Errorf("invalid index record metadata") + case len(idx.Metadata) != len(metadata): + return fmt.Errorf("invalid index metadata: got %v, want %v", + len(idx.Metadata), len(metadata)) + case len(idx.Files) != len(files): + return fmt.Errorf("invalid index files: got %v, want %v", + len(idx.Files), len(files)) + } + + // Save record index + err = t.recordIndexSave(treeID, *idx) + if err != nil { + return fmt.Errorf("recordIndexSave: %v", err) + } + + return nil } -func (t *tlog) recordIsFrozen(treeID int64) (bool, error) { - tree, err := t.client.tree(treeID) +func (t *tlog) recordMetadataUpdate(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { + // Ensure tree exists + if !t.treeExists(treeID) { + return errRecordNotFound + } + + // Get tree leaves + leavesAll, err := t.trillian.leavesAll(treeID) if err != nil { - return false, fmt.Errorf("tree: %v", err) + return fmt.Errorf("leavesAll: %v", err) } - if tree.TreeState == trillian.TreeState_FROZEN { - return true, nil + + // Ensure tree is not frozen + if treeIsFrozen(leavesAll) { + return errTreeIsFrozen } - // Its possible that a freeze record has been added to the tree but - // the call to flip the tree state to frozen has failed. In this - // case the tree is still considered frozen. We need to manually - // check the last leaf of the tree to see if it is a free record. - _, lr, err := t.client.signedLogRoot(tree) + // Prepare kv store blobs + args := recordBlobsPrepareArgs{ + encryptionKey: t.encryptionKey, + leaves: leavesAll, + recordMD: rm, + metadata: metadata, + } + bpr, err := recordBlobsPrepare(args) if err != nil { - return false, fmt.Errorf("signedLogRoot: %v", err) + return err } - leaves, err := t.client.leavesByRange(treeID, int64(lr.TreeSize)-1, 1) + + // Ensure metadata has been changed + if len(bpr.blobs) == 0 { + return errNoMetadataChanges + } + + // Save the blobs + idx, err := t.recordBlobsSave(treeID, *bpr) if err != nil { - return false, fmt.Errorf("leavesByRange: %v", err) + return fmt.Errorf("blobsSave: %v", err) } - if len(leaves) != 1 { - return false, fmt.Errorf("unexpected leaves count: got %v, want 1", - len(leaves)) + + // Get the existing record index and add the unchanged fields to + // the new record index. The version and files will remain the + // same. + oldIdx, err := t.recordIndexLatest(leavesAll) + if err != nil { + return fmt.Errorf("recordIndexLatest: %v", err) } - if !isFreezeRecordLeaf(leaves[0]) { - // Not a freeze record leaf. Tree is not frozen. - return false, nil + idx.Version = oldIdx.Version + idx.Files = oldIdx.Files + + // Increment the iteration + idx.Iteration = oldIdx.Iteration + 1 + + // Sanity check. The record index should be fully populated at this + // point. + switch { + case idx.Version != oldIdx.Version: + return fmt.Errorf("invalid index version: got %v, want %v", + idx.Version, oldIdx.Version) + case idx.Version != oldIdx.Iteration+1: + return fmt.Errorf("invalid index iteration: got %v, want %v", + idx.Iteration, oldIdx.Iteration+1) + case idx.RecordMetadata == nil: + return fmt.Errorf("invalid index record metadata") + case len(idx.Metadata) != len(metadata): + return fmt.Errorf("invalid index metadata: got %v, want %v", + len(idx.Metadata), len(metadata)) + case len(idx.Files) != len(oldIdx.Files): + return fmt.Errorf("invalid index files: got %v, want %v", + len(idx.Files), len(oldIdx.Files)) } - // Tree has a freeze record but the tree state is not frozen. This - // is bad. Fix it before returning. - _, err = t.client.treeFreeze(treeID) + // Save record index + err = t.recordIndexSave(treeID, *idx) if err != nil { - return true, fmt.Errorf("treeFreeze: %v", err) + return fmt.Errorf("recordIndexSave: %v", err) } - return true, nil + return nil +} + +// recordDel walks the provided tree and deletes all blobs in the store that +// correspond to record files. This is done for all versions of the record. +func (t *tlog) recordDel(treeID int64) error { + // Ensure tree exists + if !t.treeExists(treeID) { + return errRecordNotFound + } + + // Get all tree leaves + leaves, err := t.trillian.leavesAll(treeID) + if err != nil { + return err + } + + // Ensure tree is frozen. Deleting files from the store is only + // allowed on frozen trees. + if treeIsFrozen(leaves) { + return errTreeIsFrozen + } + + // Retrieve all the record indexes + indexes, err := t.recordIndexes(leaves) + if err != nil { + return fmt.Errorf("recordIndexes: %v", err) + } + + // Aggregate the keys for all file blobs of all versions. The + // record index points to the log leaf merkle leaf hash. The log + // leaf contains the kv store key. + merkles := make(map[string]struct{}, len(leaves)) + for _, v := range indexes { + for _, merkle := range v.Files { + merkles[hex.EncodeToString(merkle)] = struct{}{} + } + } + keys := make([]string, 0, len(merkles)) + for _, v := range leaves { + _, ok := merkles[hex.EncodeToString(v.MerkleLeafHash)] + if ok { + key, err := extractKeyFromLeaf(v) + if err != nil { + return err + } + keys = append(keys, key) + } + } + + // Delete file blobs from the store + err = t.store.Del(keys) + if err != nil { + return fmt.Errorf("store Del: %v", err) + } + + return nil } func (t *tlog) recordVersion(treeID int64, version uint32) (*backend.Record, error) { @@ -856,13 +1228,13 @@ func (t *tlog) recordVersion(treeID int64, version uint32) (*backend.Record, err } // Get tree leaves - leaves, err := t.client.LeavesAll(treeID) + leaves, err := t.trillian.leavesAll(treeID) if err != nil { - return nil, fmt.Errorf("LeavesAll %v: %v", treeID, err) + return nil, fmt.Errorf("leavesAll %v: %v", treeID, err) } // Get the record index for the specified version - index, err := t.recordIndex(leaves, version) + index, err := t.recordIndexVersion(leaves, version) if err != nil { return nil, err } @@ -891,7 +1263,7 @@ func (t *tlog) recordVersion(treeID int64, version uint32) (*backend.Record, err } // Leaf is part of record content. Extract the kv store key. - key, err := extractKeyForRecordContent(v) + key, err := extractKeyFromLeaf(v) if err != nil { return nil, fmt.Errorf("extractKeyForRecordContent %x", v.MerkleLeafHash) @@ -918,19 +1290,19 @@ func (t *tlog) recordVersion(treeID int64, version uint32) (*backend.Record, err } // Decode blobs - entries := make([]blobEntry, 0, len(keys)) + entries := make([]store.BlobEntry, 0, len(keys)) for _, v := range blobs { - var be *blobEntry - if t.encryptionKey != nil && isEncrypted(v) { - v, _, err = t.encryptionKey.Decrypt(v) + var be *store.BlobEntry + if t.encryptionKey != nil && blobIsEncrypted(v) { + v, _, err = t.encryptionKey.decrypt(v) if err != nil { return nil, err } } - be, err := deblob(v) + be, err := store.Deblob(v) if err != nil { // Check if this is an encrypted blob that was not decrypted - if t.encryptionKey == nil && isEncrypted(v) { + if t.encryptionKey == nil && blobIsEncrypted(v) { return nil, fmt.Errorf("blob is encrypted but no encryption " + "key found to decrypt blob") } @@ -951,14 +1323,14 @@ func (t *tlog) recordVersion(treeID int64, version uint32) (*backend.Record, err if err != nil { return nil, fmt.Errorf("decode DataHint: %v", err) } - var dd dataDescriptor + var dd store.DataDescriptor err = json.Unmarshal(b, &dd) if err != nil { return nil, fmt.Errorf("unmarshal DataHint: %v", err) } - if dd.Type != dataTypeStructure { + if dd.Type != store.DataTypeStructure { return nil, fmt.Errorf("invalid data type; got %v, want %v", - dd.Type, dataTypeStructure) + dd.Type, store.DataTypeStructure) } // Decode the data @@ -1025,223 +1397,6 @@ func (t *tlog) recordLatest(treeID int64) (*backend.Record, error) { return t.recordVersion(treeID, 0) } -// We do not unwind. -func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { - // Ensure tree exists - if !t.treeExists(treeID) { - return errRecordNotFound - } - - // Get tree leaves - leavesAll, err := t.client.LeavesAll(treeID) - if err != nil { - return fmt.Errorf("LeavesAll %v: %v", treeID, err) - } - - // Prepare kv store blobs - args := blobsPrepareArgs{ - encryptionKey: t.encryptionKey, - leaves: leavesAll, - recordMD: rm, - metadata: metadata, - files: files, - } - bpr, err := blobsPrepare(args) - if err != nil { - return err - } - - // Ensure file changes are being made - if len(bpr.recordIndex.Files) == len(files) { - return errNoFileChanges - } - - // Save blobs - idx, err := t.blobsSave(treeID, *bpr) - if err != nil { - return fmt.Errorf("blobsSave: %v", err) - } - - // Get the existing record index and use it to bump the version and - // iteration of the new record index. - oldIdx, err := t.recordIndexLatest(leavesAll) - if err == errRecordNotFound { - // No record versions exist yet. This is fine. The version and - // iteration will be incremented to 1. - } else if err != nil { - return fmt.Errorf("recordIndexLatest: %v", err) - } - idx.Version = oldIdx.Version + 1 - idx.Iteration = oldIdx.Iteration + 1 - - // Sanity check. The record index should be fully populated at this - // point. - switch { - case idx.Version != oldIdx.Version+1: - return fmt.Errorf("invalid index version: got %v, want %v", - idx.Version, oldIdx.Version+1) - case idx.Iteration != oldIdx.Iteration+1: - return fmt.Errorf("invalid index iteration: got %v, want %v", - idx.Iteration, oldIdx.Iteration+1) - case idx.RecordMetadata == nil: - return fmt.Errorf("invalid index record metadata") - case len(idx.Metadata) != len(metadata): - return fmt.Errorf("invalid index metadata: got %v, want %v", - len(idx.Metadata), len(metadata)) - case len(idx.Files) != len(files): - return fmt.Errorf("invalid index files: got %v, want %v", - len(idx.Files), len(files)) - } - - // Save record index - err = t.recordIndexSave(treeID, *idx) - if err != nil { - return fmt.Errorf("recordIndexSave: %v", err) - } - - return nil -} - -func (t *tlog) recordMetadataUpdate(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { - // Ensure tree exists - if !t.treeExists(treeID) { - return errRecordNotFound - } - - // Get tree leaves - leavesAll, err := t.client.LeavesAll(treeID) - if err != nil { - return fmt.Errorf("LeavesAll: %v", err) - } - - // Prepare kv store blobs - args := blobsPrepareArgs{ - encryptionKey: t.encryptionKey, - leaves: leavesAll, - recordMD: rm, - metadata: metadata, - } - bpr, err := blobsPrepare(args) - if err != nil { - return err - } - - // Ensure metadata has been changed - if len(bpr.blobs) == 0 { - return errNoMetadataChanges - } - - // Save the blobs - idx, err := t.blobsSave(treeID, *bpr) - if err != nil { - return fmt.Errorf("blobsSave: %v", err) - } - - // Get the existing record index and add the unchanged fields to - // the new record index. The version and files will remain the - // same. - oldIdx, err := t.recordIndexLatest(leavesAll) - if err != nil { - return fmt.Errorf("recordIndexLatest: %v", err) - } - idx.Version = oldIdx.Version - idx.Files = oldIdx.Files - - // Increment the iteration - idx.Iteration = oldIdx.Iteration + 1 - - // Sanity check. The record index should be fully populated at this - // point. - switch { - case idx.Version != oldIdx.Version: - return fmt.Errorf("invalid index version: got %v, want %v", - idx.Version, oldIdx.Version) - case idx.Version != oldIdx.Iteration+1: - return fmt.Errorf("invalid index iteration: got %v, want %v", - idx.Iteration, oldIdx.Iteration+1) - case idx.RecordMetadata == nil: - return fmt.Errorf("invalid index record metadata") - case len(idx.Metadata) != len(metadata): - return fmt.Errorf("invalid index metadata: got %v, want %v", - len(idx.Metadata), len(metadata)) - case len(idx.Files) != len(oldIdx.Files): - return fmt.Errorf("invalid index files: got %v, want %v", - len(idx.Files), len(oldIdx.Files)) - } - - // Save record index - err = t.recordIndexSave(treeID, *idx) - if err != nil { - return fmt.Errorf("recordIndexSave: %v", err) - } - - return nil -} - -// treeFreeze freezes the trillian tree for the provided token. Once a tree -// has been frozen it is no longer able to be appended to. The last leaf in a -// frozen tree will correspond to a freeze record in the key-value store. A -// tree is frozen when the status of the corresponding record is updated to a -// status that locks the record, such as when a record is censored. The status -// change and the freeze record are appended to the tree using a single call. -// -// It's possible for this function to fail in between the append leaves call -// and the call that updates the tree status to frozen. If this happens the -// freeze record will be the last leaf on the tree but the tree state will not -// be frozen. The tree is still considered frozen and no new leaves should be -// appended to it. The tree state will be updated in the next fsck. -func (t *tlog) treeFreeze(treeID int64, fr freezeRecord) error { - // Save freeze record to store - be, err := convertBlobEntryFromFreezeRecord(fr) - if err != nil { - return err - } - b, err := blobify(*be) - if err != nil { - return err - } - keys, err := t.store.Put([][]byte{b}) - if err != nil { - return fmt.Errorf("store Put: %v", err) - } - if len(keys) != 1 { - return fmt.Errorf("wrong number of keys: got %v, want 1", - len(keys)) - } - - // Append freeze record leaf to trillian tree - h, err := hex.DecodeString(be.Hash) - if err != nil { - return err - } - prefixedKey := []byte(keyPrefixFreezeRecord + keys[0]) - queued, _, err := t.client.LeavesAppend(treeID, []*trillian.LogLeaf{ - logLeafNew(h, prefixedKey), - }) - if len(queued) != 1 { - return fmt.Errorf("wrong number of queud leaves: got %v, want 1", - len(queued)) - } - failed := make([]string, 0, len(queued)) - for _, v := range queued { - c := codes.Code(v.QueuedLeaf.GetStatus().GetCode()) - if c != codes.OK { - failed = append(failed, fmt.Sprintf("%v", c)) - } - } - if len(failed) > 0 { - return fmt.Errorf("append leaves failed: %v", failed) - } - - // Freeze the tree - _, err = t.client.treeFreeze(treeID) - if err != nil { - return fmt.Errorf("treeFreeze: %v", err) - } - - return nil -} - func (t *tlog) recordProof(treeID int64, version uint32) {} func (t *tlog) fsck() { @@ -1252,7 +1407,7 @@ func (t *tlog) fsck() { func (t *tlog) close() { // Close connections t.store.Close() - t.client.Close() + t.trillian.close() // Zero out encryption key if t.encryptionKey != nil { diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 6502117de..105f8acdf 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -13,6 +13,7 @@ import ( "net/url" "os" "path/filepath" + "strconv" "sync" "time" @@ -24,10 +25,8 @@ import ( "github.com/marcopeereboom/sbox" ) -// TODO Add UpdateUnvettedMetadata to backend interface // TODO populate token prefixes cache on startup // TODO testnet vs mainnet trillian databases -// TODO lock on the token level // TODO fsck // TODO allow token prefix lookups @@ -39,7 +38,7 @@ const ( ) var ( - _ backend.Backend = (*tlogbe)(nil) + _ backend.Backend = (*Tlogbe)(nil) // statusChanges contains the allowed record status changes. statusChanges = map[backend.MDStatusT]map[backend.MDStatusT]struct{}{ @@ -57,6 +56,7 @@ var ( // Vetted status changes backend.MDStatusVetted: map[backend.MDStatusT]struct{}{ backend.MDStatusArchived: struct{}{}, + backend.MDStatusCensored: struct{}{}, }, // Statuses that do not allow any further transitions @@ -65,8 +65,8 @@ var ( } ) -// tlogbe implements the Backend interface. -type tlogbe struct { +// Tlogbe implements the Backend interface. +type Tlogbe struct { sync.RWMutex shutdown bool homeDir string @@ -79,51 +79,56 @@ type tlogbe struct { // records. The prefix is the first n characters of the hex encoded // record token, where n is defined by the TokenPrefixLength from // the politeiad API. Record lookups by token prefix are allowed. - // This cache is used to prevent prefix collisions when creating - // new tokens and to facilitate lookups by token prefix. This cache - // is loaded on tlogbe startup. + // This cache is loaded on tlogbe startup and is used to prevent + // prefix collisions when creating new tokens and to facilitate + // lookups by token prefix. prefixes map[string][]byte // [tokenPrefix]fullToken - // vettedTrees contains the token to tree ID mapping for vetted + // vettedTreeIDs contains the token to tree ID mapping for vetted // records. The token corresponds to the unvetted tree ID so // unvetted lookups can be done directly, but vetted lookups // required pulling the freeze record from the unvetted tree to // get the vetted tree ID. This cache memoizes these results. - vettedTrees map[string]int64 // [token]treeID + vettedTreeIDs map[string]int64 // [token]treeID +} + +func tokenPrefix(token []byte) string { + return hex.EncodeToString(token)[:pd.TokenPrefixLength] } -func (t *tlogbe) prefixExists(fullToken []byte) bool { +func (t *Tlogbe) prefixExists(fullToken []byte) bool { t.RLock() defer t.RUnlock() - prefix := hex.EncodeToString(fullToken)[:pd.TokenPrefixLength] - _, ok := t.prefixes[prefix] + _, ok := t.prefixes[tokenPrefix(fullToken)] return ok } -func (t *tlogbe) prefixSet(fullToken []byte) { +func (t *Tlogbe) prefixSet(fullToken []byte) { t.Lock() defer t.Unlock() - prefix := hex.EncodeToString(fullToken)[:pd.TokenPrefixLength] - t.prefixes[prefix] = fullToken + prefix := tokenPrefix(fullToken) + t.prefixes[tokenPrefix(fullToken)] = fullToken + + log.Debugf("Token prefix cached: %v", prefix) } -func (t *tlogbe) vettedTreesGet(token string) (int64, bool) { +func (t *Tlogbe) vettedTreeIDGet(token string) (int64, bool) { t.RLock() defer t.RUnlock() - treeID, ok := t.vettedTrees[token] + treeID, ok := t.vettedTreeIDs[token] return treeID, ok } -func (t *tlogbe) vettedTreesSet(token string, treeID int64) { +func (t *Tlogbe) vettedTreeIDSet(token string, treeID int64) { t.Lock() defer t.Unlock() - t.vettedTrees[token] = treeID + t.vettedTreeIDs[token] = treeID - log.Debugf("vettedTreesSet: %v %v", token, treeID) + log.Debugf("Vetted tree ID cached: %v %v", token, treeID) } // statusChangeIsAllowed returns whether the provided status change is allowed @@ -232,8 +237,9 @@ func metadataStreamsUpdate(mdCurr, mdAppend, mdOverwrite []backend.MetadataStrea } // New satisfies the Backend interface. -// TODO when does the signature get checked -func (t *tlogbe) New(metadata []backend.MetadataStream, files []backend.File) (*backend.RecordMetadata, error) { +// +// This function satisfies the Backend interface. +func (t *Tlogbe) New(metadata []backend.MetadataStream, files []backend.File) (*backend.RecordMetadata, error) { log.Tracef("New") // Validate record contents @@ -254,11 +260,13 @@ func (t *tlogbe) New(metadata []backend.MetadataStream, files []backend.File) (* // Check for token prefix collisions if !t.prefixExists(token) { - // Not a collision + // Not a collision. Update token prefixes cache. + t.prefixSet(token) break } - log.Infof("Token prefix collision %x, creating new token", token) + log.Infof("Token prefix collision %v, creating new token", + tokenPrefix(token)) } // Create record metadata @@ -267,7 +275,7 @@ func (t *tlogbe) New(metadata []backend.MetadataStream, files []backend.File) (* return nil, err } - // Save the new version of the record + // Save the record err = t.unvetted.recordSave(treeID, *rm, metadata, files) if err != nil { return nil, fmt.Errorf("recordSave %x: %v", token, err) @@ -278,9 +286,8 @@ func (t *tlogbe) New(metadata []backend.MetadataStream, files []backend.File) (* return rm, nil } -// TODO Add UpdateUnvettedMetadata - -func (t *tlogbe) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { +// This function satisfies the Backend interface. +func (t *Tlogbe) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { log.Tracef("UpdateUnvettedRecord: %x", token) // Validate record contents. Send in a single metadata array to @@ -300,8 +307,10 @@ func (t *tlogbe) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite []back } } - t.Lock() - defer t.Unlock() + // Apply the record changes and save the new version. The lock + // needs to be held for the remainder of the function. + t.unvetted.Lock() + defer t.unvetted.Unlock() if t.shutdown { return nil, backend.ErrShutdown } @@ -338,7 +347,7 @@ func (t *tlogbe) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite []back // TODO Call plugin hooks - // Get updated record + // Return updated record r, err = t.unvetted.recordLatest(treeID) if err != nil { return nil, fmt.Errorf("recordLatest: %v", err) @@ -347,7 +356,8 @@ func (t *tlogbe) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite []back return r, nil } -func (t *tlogbe) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { +// This function satisfies the Backend interface. +func (t *Tlogbe) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { log.Tracef("UpdateVettedRecord: %x", token) // Validate record contents. Send in a single metadata array to @@ -367,8 +377,10 @@ func (t *tlogbe) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []backen } } - t.Lock() - defer t.Unlock() + // Apply the record changes and save the new version. The lock + // needs to be held for the remainder of the function. + t.vetted.Lock() + defer t.vetted.Unlock() if t.shutdown { return nil, backend.ErrShutdown } @@ -395,7 +407,7 @@ func (t *tlogbe) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []backen } // Save record - err = t.unvetted.recordSave(treeID, *recordMD, metadata, files) + err = t.vetted.recordSave(treeID, *recordMD, metadata, files) if err != nil { if err == errNoFileChanges { return nil, backend.ErrNoChanges @@ -405,8 +417,8 @@ func (t *tlogbe) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []backen // TODO Call plugin hooks - // Get updated record - r, err = t.unvetted.recordLatest(treeID) + // Return updated record + r, err = t.vetted.recordLatest(treeID) if err != nil { return nil, fmt.Errorf("recordLatest: %v", err) } @@ -414,7 +426,57 @@ func (t *tlogbe) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []backen return r, nil } -func (t *tlogbe) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite []backend.MetadataStream) error { +// This function satisfies the Backend interface. +func (t *Tlogbe) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite []backend.MetadataStream) error { + // Validate record contents. Send in a single metadata array to + // verify there are no dups. + allMD := append(mdAppend, mdOverwrite...) + err := backend.VerifyContent(allMD, []backend.File{}, []string{}) + if err != nil { + e, ok := err.(backend.ContentVerificationError) + if !ok { + return err + } + // Allow ErrorStatusEmpty which indicates no new files are being + // being added. This is expected since this is a metadata only + // update. + if e.ErrorCode != pd.ErrorStatusEmpty { + return err + } + } + if len(mdAppend) == 0 && len(mdOverwrite) == 0 { + return backend.ContentVerificationError{ + ErrorCode: pd.ErrorStatusNoChanges, + } + } + + // Pull the existing record and apply the metadata updates. The + // unvetted lock must be held for the remainder of this function. + t.unvetted.Lock() + defer t.unvetted.Unlock() + if t.shutdown { + return backend.ErrShutdown + } + + // Get existing record + treeID := treeIDFromToken(token) + r, err := t.unvetted.recordLatest(treeID) + if err != nil { + if err == errRecordNotFound { + return backend.ErrRecordNotFound + } + return fmt.Errorf("recordLatest: %v", err) + } + + // Apply changes + metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) + + // Update metadata + return t.unvetted.recordMetadataUpdate(treeID, r.RecordMetadata, metadata) +} + +// This function satisfies the Backend interface. +func (t *Tlogbe) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite []backend.MetadataStream) error { log.Tracef("UpdateVettedMetadata: %x", token) // Validate record contents. Send in a single metadata array to @@ -439,8 +501,10 @@ func (t *tlogbe) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite []back } } - t.Lock() - defer t.Unlock() + // Pull the existing record and apply the metadata updates. The + // vetted lock must be held for the remainder of this function. + t.vetted.Lock() + defer t.vetted.Unlock() if t.shutdown { return backend.ErrShutdown } @@ -462,134 +526,221 @@ func (t *tlogbe) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite []back return t.vetted.recordMetadataUpdate(treeID, r.RecordMetadata, metadata) } -func (t *tlogbe) UpdateReadme(content string) error { +// TODO remove +func (t *Tlogbe) UpdateReadme(content string) error { return fmt.Errorf("not implemented") } -// unvettedRecordExists returns whether the provided token corresponds to an -// unvetted record. -func (t *tlogbe) unvettedExists(token []byte) bool { - // TODO - return false -} - -func (t *tlogbe) UnvettedExists(token []byte) bool { +// UnvettedExists returns whether the provided token corresponds to an unvetted +// record. +// +// This function satisfies the Backend interface. +func (t *Tlogbe) UnvettedExists(token []byte) bool { log.Tracef("UnvettedExists %x", token) - // TODO must check that it exists and it is not a vetted record too - // Checking if its frozen is not enough. A unvetted tree will be - // frozen when it is censored or abandoned. - return false + // If the token is in the vetted cache then we know this is not an + // unvetted record without having to make any network requests. + _, ok := t.vettedTreeIDGet(hex.EncodeToString(token)) + if ok { + return false + } + + // Check if unvetted tree exists + treeID := treeIDFromToken(token) + if !t.unvetted.treeExists(treeID) { + // Unvetted tree does not exists. No tree, no record. + return false + } + + // An unvetted tree exists. Check if a vetted tree also exists. If + // one does then it means this record has been made public and is + // no longer unvetted. + if t.VettedExists(token) { + return false + } + + // Vetted record does not exist. This is an unvetted record. + return true } -func (t *tlogbe) vettedExists(token []byte) bool { - tk := hex.EncodeToString(token) - if _, ok := t.vettedTreesGet(tk); ok { +// This function satisfies the Backend interface. +func (t *Tlogbe) VettedExists(token []byte) bool { + log.Tracef("VettedExists %x", token) + + // Check if the token is in the vetted cache. The vetted cache is + // lazy loaded if the token is not present then we need to check + // manually. + _, ok := t.vettedTreeIDGet(hex.EncodeToString(token)) + if ok { return true } - // Just because the token is not in the vetted trees cache does not - // mean a vetted tree does not exist. The cache is lazy loaded. - // Check if there is an unvetted tree for the token and if the last - // leaf of the unvetted tree is a freeze record. + // The token is derived from the unvetted tree ID. Check if the + // token corresponds to an unvetted tree. treeID := treeIDFromToken(token) + if !t.unvetted.treeExists(treeID) { + // Unvetted tree does not exists. This token does not correspond + // to any record. + return false + } + + // Unvetted tree exists. Get the freeze record to see if it + // contains a pointer to a vetted tree. fr, err := t.unvetted.freezeRecord(treeID) if err != nil { if err == errFreezeRecordNotFound { - // Unvetted tree is not frozen. Record is still unvetted. + // Unvetted tree exists and is not frozen. This is an unvetted + // record. return false } - - // Unexpected error - e := fmt.Sprintf("vettedExists %x: freezeRecord: %v", token, err) - panic(e) + log.Errorf("unvetted freezeRecord %v: %v", treeID, err) + return false } - - // Unvetted tree is frozen. Check if the freeze record points to a - // vetted tree ID or if it is blank. A vetted tree ID indicates the - // record was made public. A blank tree ID indicates the tree was - // frozen for some other reason such as if the record was censored - // or abandoned. if fr.TreeID == 0 { - // No vetted tree pointer found + // Unvetted tree has been frozen but does not contain a pointer + // to another tree. This means it was frozen for some other + // reason (ex. censored). This is not a vetted record. return false } - // Ensure vetted record exists + // Ensure the freeze record tree ID points to a valid vetted tree. + // This should not fail. if !t.vetted.treeExists(fr.TreeID) { - // Uh oh. A freeze record points to this tree ID but the tree - // does not exist. Not good. - e := fmt.Sprintf("freeze record points to invalid tree %v", fr.TreeID) + // We're in trouble! + e := fmt.Sprintf("freeze record of unvetted tree %v points to "+ + "an invalid vetted tree %v", treeID, fr.TreeID) panic(e) } - // Cache the vetted tree ID - t.vettedTreesSet(hex.EncodeToString(token), fr.TreeID) + // Update the vetted cache + t.vettedTreeIDSet(hex.EncodeToString(token), fr.TreeID) return true } -func (t *tlogbe) VettedExists(token []byte) bool { - log.Tracef("VettedExists %x", token) - - return false -} - -func (t *tlogbe) GetUnvetted(token []byte) (*backend.Record, error) { +// This function satisfies the Backend interface. +func (t *Tlogbe) GetUnvetted(token []byte, version string) (*backend.Record, error) { log.Tracef("GetUnvetted: %x", token) - return nil, nil + treeID := treeIDFromToken(token) + v, err := strconv.ParseUint(version, 10, 64) + if err != nil { + return nil, backend.ErrRecordNotFound + } + + return t.unvetted.recordVersion(treeID, uint32(v)) } -func (t *tlogbe) GetVetted(token []byte, version string) (*backend.Record, error) { +// This function satisfies the Backend interface. +func (t *Tlogbe) GetVetted(token []byte, version string) (*backend.Record, error) { log.Tracef("GetVetted: %x", token) - return nil, nil + treeID := treeIDFromToken(token) + v, err := strconv.ParseUint(version, 10, 64) + if err != nil { + return nil, backend.ErrRecordNotFound + } + + return t.vetted.recordVersion(treeID, uint32(v)) } -// This function must be called with the read/write lock held. -func (t *tlogbe) unvettedPublish(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { - // Create a vetted record - // TODO check for collisions - treeID, err := t.vetted.treeNew() - if err != nil { - return fmt.Errorf("vetted recordNew: %v", err) +// This function must be called WITH the unvetted lock held. +func (t *Tlogbe) unvettedPublish(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { + // Create a vetted tree + var ( + vettedToken []byte + vettedTreeID int64 + err error + ) + for retries := 0; retries < 10; retries++ { + vettedTreeID, err = t.vetted.treeNew() + if err != nil { + return err + } + vettedToken = tokenFromTreeID(vettedTreeID) + + // Check for token prefix collisions + if !t.prefixExists(vettedToken) { + // Not a collision. Update prefixes cache. + t.prefixSet(vettedToken) + break + } + + log.Infof("Token prefix collision %v, creating new token", + tokenPrefix(vettedToken)) } - // Save the record as vetted - err = t.vetted.recordSave(treeID, rm, metadata, files) + // Save the record to the vetted tlog + err = t.vetted.recordSave(vettedTreeID, rm, metadata, files) if err != nil { return fmt.Errorf("vetted recordSave: %v", err) } + log.Debugf("Unvetted record %x copied to vetted", token) + // Freeze the unvetted tree fr := freezeRecord{ - TreeID: treeID, + TreeID: vettedTreeID, + } + treeID := treeIDFromToken(token) + err = t.unvetted.treeFreeze(treeID, rm, metadata, fr) + if err != nil { + return fmt.Errorf("treeFreeze %v: %v", treeID, err) } - _ = fr + + log.Debugf("Unvetted record %x frozen", token) + + // Update the vetted cache + t.vettedTreeIDSet(hex.EncodeToString(token), vettedTreeID) return nil } -// This function must be called with the read/write lock held. -func (t *tlogbe) unvettedCensor() error { - // Freeze tree - // Delete the censored blobs +// This function must be called WITH the unvetted lock held. +func (t *Tlogbe) unvettedCensor(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { + // Freeze the tree + treeID := treeIDFromToken(token) + err := t.unvetted.treeFreeze(treeID, rm, metadata, freezeRecord{}) + if err != nil { + return fmt.Errorf("treeFreeze %v: %v", treeID, err) + } + + log.Debugf("Unvetted record %x frozen", token) + + // Delete all record files + err = t.unvetted.recordDel(treeID) + if err != nil { + return fmt.Errorf("recordDel %v: %v", treeID, err) + } + + log.Debug("Unvetted record %x files deleted", token) + return nil } -// This function must be called with the read/write lock held. -func (t *tlogbe) unvettedArchive() error { - // Freeze tree +// This function must be called WITH the unvetted lock held. +func (t *Tlogbe) unvettedArchive(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { + // Freeze the tree. Nothing else needs to be done for an archived + // record. + treeID := treeIDFromToken(token) + err := t.unvetted.treeFreeze(treeID, rm, metadata, freezeRecord{}) + if err != nil { + return err + } + + log.Debugf("Unvetted record %x frozen", token) + return nil } -func (t *tlogbe) SetUnvettedStatus(token []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { +func (t *Tlogbe) SetUnvettedStatus(token []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { log.Tracef("SetUnvettedStatus: %x %v (%v)", token, status, backend.MDStatus[status]) - t.Lock() - defer t.Unlock() + // The existing record must be pulled and updated. The unvetted + // lock must be held for the rest of this function. + t.unvetted.Lock() + defer t.unvetted.Unlock() if t.shutdown { return nil, backend.ErrShutdown } @@ -622,28 +773,29 @@ func (t *tlogbe) SetUnvettedStatus(token []byte, status backend.MDStatusT, mdApp // Apply metdata changes metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) + // Update record switch status { case backend.MDStatusVetted: err := t.unvettedPublish(token, rm, metadata, r.Files) if err != nil { - return nil, fmt.Errorf("publish: %v", err) + return nil, fmt.Errorf("unvettedPublish: %v", err) } case backend.MDStatusCensored: - err := t.unvettedCensor() + err := t.unvettedCensor(token, rm, metadata) if err != nil { - return nil, fmt.Errorf("censor: %v", err) + return nil, fmt.Errorf("unvettedCensor: %v", err) } case backend.MDStatusArchived: - err := t.unvettedArchive() + err := t.unvettedArchive(token, rm, metadata) if err != nil { - return nil, fmt.Errorf("archive: %v", err) + return nil, fmt.Errorf("unvettedArchive: %v", err) } default: return nil, fmt.Errorf("unknown status: %v (%v)", backend.MDStatus[status], status) } - // Return the record + // Return the updated record r, err = t.unvetted.recordLatest(treeID) if err != nil { return nil, fmt.Errorf("recordLatest: %v", err) @@ -652,60 +804,110 @@ func (t *tlogbe) SetUnvettedStatus(token []byte, status backend.MDStatusT, mdApp return r, nil } -// This function must be called with the read/write lock held. -func (t *tlogbe) vettedCensor() error { - // Freeze tree - // Delete the censored blobs +// This function must be called WITH the vetted lock held. +func (t *Tlogbe) vettedCensor(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { + // Freeze the tree + treeID := treeIDFromToken(token) + err := t.vetted.treeFreeze(treeID, rm, metadata, freezeRecord{}) + if err != nil { + return fmt.Errorf("treeFreeze %v: %v", treeID, err) + } + + // Delete all record files + err = t.vetted.recordDel(treeID) + if err != nil { + return fmt.Errorf("recordDel %v: %v", treeID, err) + } + return nil } -// This function must be called with the read/write lock held. -func (t *tlogbe) vettedArchive() error { - // Freeze tree - return nil +// This function must be called WITH the vetted lock held. +func (t *Tlogbe) vettedArchive(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { + // Freeze the tree. Nothing else needs to be done for an archived + // record. + treeID := treeIDFromToken(token) + return t.vetted.treeFreeze(treeID, rm, metadata, freezeRecord{}) } -func (t *tlogbe) SetVettedStatus(token []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { +// This function satisfies the Backend interface. +func (t *Tlogbe) SetVettedStatus(token []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { log.Tracef("SetVettedStatus: %x %v (%v)", token, status, backend.MDStatus[status]) - t.Lock() - defer t.Unlock() + // The existing record must be pulled and updated. The vetted lock + // must be held for the rest of this function. + t.vetted.Lock() + defer t.vetted.Unlock() if t.shutdown { return nil, backend.ErrShutdown } - /* - // Validate status change - if !statusChangeIsAllowed(rm.Status, status) { - return nil, backend.StateTransitionError{ - From: rm.Status, - To: status, - } + // Get existing record + treeID := treeIDFromToken(token) + r, err := t.vetted.recordLatest(treeID) + if err != nil { + return nil, fmt.Errorf("recordLatest: %v", err) + } + rm := r.RecordMetadata + + // Validate status change + if !statusChangeIsAllowed(rm.Status, status) { + return nil, backend.StateTransitionError{ + From: rm.Status, + To: status, } + } - log.Debugf("Status change %x from %v (%v) to %v (%v)", token, - backend.MDStatus[rm.Status], rm.Status, backend.MDStatus[status], status) + log.Debugf("Status change %x from %v (%v) to %v (%v)", + token, backend.MDStatus[rm.Status], rm.Status, + backend.MDStatus[status], status) - // Apply status change - rm.Status = status - rm.Iteration += 1 - rm.Timestamp = time.Now().Unix() + // Apply status change + rm.Status = status + rm.Iteration += 1 + rm.Timestamp = time.Now().Unix() - // Update metadata - */ + // Apply metdata changes + metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) - return nil, nil + // Update record + switch status { + case backend.MDStatusCensored: + err := t.vettedCensor(token, rm, metadata) + if err != nil { + return nil, fmt.Errorf("vettedCensor: %v", err) + } + case backend.MDStatusArchived: + err := t.vettedArchive(token, rm, metadata) + if err != nil { + return nil, fmt.Errorf("vettedArchive: %v", err) + } + default: + return nil, fmt.Errorf("unknown status: %v (%v)", + backend.MDStatus[status], status) + } + + // Return the updated record + r, err = t.vetted.recordLatest(treeID) + if err != nil { + return nil, fmt.Errorf("recordLatest: %v", err) + } + + return r, nil } -func (t *tlogbe) Inventory(vettedCount uint, branchCount uint, includeFiles, allVersions bool) ([]backend.Record, []backend.Record, error) { +// This function satisfies the Backend interface. +func (t *Tlogbe) Inventory(vettedCount uint, unvettedCount uint, includeFiles, allVersions bool) ([]backend.Record, []backend.Record, error) { log.Tracef("Inventory: %v %v", includeFiles, allVersions) + // TODO implement inventory + // return vetted, unvetted, nil return nil, nil, nil } -func (t *tlogbe) GetPlugins() ([]backend.Plugin, error) { +func (t *Tlogbe) GetPlugins() ([]backend.Plugin, error) { log.Tracef("GetPlugins") // TODO implement plugins @@ -713,7 +915,8 @@ func (t *tlogbe) GetPlugins() ([]backend.Plugin, error) { return t.plugins, nil } -func (t *tlogbe) Plugin(command, payload string) (string, string, error) { +// Add commandID to Plugin +func (t *Tlogbe) Plugin(pluginID, command, payload string) (string, string, error) { log.Tracef("Plugin: %v", command) // TODO implement plugins @@ -721,7 +924,7 @@ func (t *tlogbe) Plugin(command, payload string) (string, string, error) { return "", "", nil } -func (t *tlogbe) Close() { +func (t *Tlogbe) Close() { log.Tracef("Close") t.Lock() @@ -735,7 +938,7 @@ func (t *tlogbe) Close() { t.vetted.close() } -func New(homeDir, dataDir, trillianHost, trillianKeyFile, dcrtimeHost, encryptionKeyFile string) (*tlogbe, error) { +func New(homeDir, dataDir, trillianHost, trillianKeyFile, dcrtimeHost, encryptionKeyFile string) (*Tlogbe, error) { // Setup encryption key file if encryptionKeyFile == "" { // No file path was given. Use the default path. @@ -799,7 +1002,7 @@ func New(homeDir, dataDir, trillianHost, trillianKeyFile, dcrtimeHost, encryptio _ = dcrtimeHost _ = store _ = tlog - t := tlogbe{ + t := Tlogbe{ homeDir: homeDir, dataDir: dataDir, // cron: cron.New(), diff --git a/politeiad/backend/tlogbe/trillian.go b/politeiad/backend/tlogbe/trillian.go index a0429f9af..d4f83b004 100644 --- a/politeiad/backend/tlogbe/trillian.go +++ b/politeiad/backend/tlogbe/trillian.go @@ -33,11 +33,11 @@ import ( "google.golang.org/grpc/status" ) -// TrillianClient provides a trillian client that abstracts over the existing -// TrillianLogClient and TrillianAdminClient. This is done to ensure proper -// verification of all trillian responses is performed and to restrict plugin -// access to only certain trillian functionality. -type TrillianClient struct { +// trillianClient provides a client that abstracts over the existing +// TrillianLogClient and TrillianAdminClient. This provides a simplified API +// for the backend to use and ensures that proper verification of all trillian +// responses is performed. +type trillianClient struct { grpc *grpc.ClientConn client trillian.TrillianLogClient admin trillian.TrillianAdminClient @@ -46,17 +46,17 @@ type TrillianClient struct { publicKey crypto.PublicKey // Trillian public key } -// LeafProof contains a log leaf and the inclusion proof for the log leaf. -type LeafProof struct { +// leafProof contains a log leaf and the inclusion proof for the log leaf. +type leafProof struct { Leaf *trillian.LogLeaf Proof *trillian.Proof } -// QueuedLeafProof contains the results of a leaf append command, i.e. the +// queuedLeafProof contains the results of a leaf append command, i.e. the // QueuedLeaf, and the inclusion proof for that leaf. The inclusion proof will // not be present if the leaf append command failed. The QueuedLeaf will // contain the error code from the failure. -type QueuedLeafProof struct { +type queuedLeafProof struct { QueuedLeaf *trillian.QueuedLogLeaf Proof *trillian.Proof } @@ -77,28 +77,10 @@ func logLeafNew(value []byte, extraData []byte) *trillian.LogLeaf { } } -func (t *TrillianClient) tree(treeID int64) (*trillian.Tree, error) { - log.Tracef("tree: %v", treeID) - - tree, err := t.admin.GetTree(t.ctx, &trillian.GetTreeRequest{ - TreeId: treeID, - }) - if err != nil { - return nil, fmt.Errorf("GetTree: %v", err) - } - if tree.TreeId != treeID { - // Sanity check - return nil, fmt.Errorf("wrong tree returned; got %v, want %v", - tree.TreeId, treeID) - } - - return tree, nil -} - // treeNew returns a new trillian tree and verifies that the signatures are // correct. It returns the tree and the signed log root which can be externally // verified. -func (t *TrillianClient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { +func (t *trillianClient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { log.Tracef("treeNew") pk, err := ptypes.MarshalAny(t.privateKey) @@ -163,7 +145,7 @@ func (t *TrillianClient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, err return tree, ilr.Created, nil } -func (t *TrillianClient) treeFreeze(treeID int64) (*trillian.Tree, error) { +func (t *trillianClient) treeFreeze(treeID int64) (*trillian.Tree, error) { // Get the current tree tree, err := t.tree(treeID) if err != nil { @@ -187,7 +169,25 @@ func (t *TrillianClient) treeFreeze(treeID int64) (*trillian.Tree, error) { return updated, nil } -func (t *TrillianClient) treesAll() ([]*trillian.Tree, error) { +func (t *trillianClient) tree(treeID int64) (*trillian.Tree, error) { + log.Tracef("tree: %v", treeID) + + tree, err := t.admin.GetTree(t.ctx, &trillian.GetTreeRequest{ + TreeId: treeID, + }) + if err != nil { + return nil, fmt.Errorf("GetTree: %v", err) + } + if tree.TreeId != treeID { + // Sanity check + return nil, fmt.Errorf("wrong tree returned; got %v, want %v", + tree.TreeId, treeID) + } + + return tree, nil +} + +func (t *trillianClient) treesAll() ([]*trillian.Tree, error) { log.Tracef("treesAll") ltr, err := t.admin.ListTrees(t.ctx, &trillian.ListTreesRequest{}) @@ -198,7 +198,7 @@ func (t *TrillianClient) treesAll() ([]*trillian.Tree, error) { return ltr.Tree, nil } -func (t *TrillianClient) inclusionProof(treeID int64, merkleLeafHash []byte, lrv1 *types.LogRootV1) (*trillian.Proof, error) { +func (t *trillianClient) inclusionProof(treeID int64, merkleLeafHash []byte, lrv1 *types.LogRootV1) (*trillian.Proof, error) { log.Tracef("inclusionProof: %v %x", treeID, merkleLeafHash) resp, err := t.client.GetInclusionProofByHash(t.ctx, @@ -230,7 +230,7 @@ func (t *TrillianClient) inclusionProof(treeID int64, merkleLeafHash []byte, lrv return proof, nil } -func (t *TrillianClient) signedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoot, *types.LogRootV1, error) { +func (t *trillianClient) signedLogRootForTree(tree *trillian.Tree) (*trillian.SignedLogRoot, *types.LogRootV1, error) { // Get the signed log root for the current tree height resp, err := t.client.GetLatestSignedLogRoot(t.ctx, &trillian.GetLatestSignedLogRootRequest{LogId: tree.TreeId}) @@ -252,16 +252,16 @@ func (t *TrillianClient) signedLogRoot(tree *trillian.Tree) (*trillian.SignedLog return resp.SignedLogRoot, lrv1, nil } -// SignedLogRoot returns the signed log root for the provided tree ID at its +// signedLogRoot returns the signed log root for the provided tree ID at its // current height. The log root is structure is decoded an returned as well. -func (t *TrillianClient) SignedLogRoot(treeID int64) (*trillian.SignedLogRoot, *types.LogRootV1, error) { +func (t *trillianClient) signedLogRoot(treeID int64) (*trillian.SignedLogRoot, *types.LogRootV1, error) { log.Tracef("SignedLogRoot: %v", treeID) tree, err := t.tree(treeID) if err != nil { return nil, nil, fmt.Errorf("tree: %v", err) } - slr, lr, err := t.signedLogRoot(tree) + slr, lr, err := t.signedLogRootForTree(tree) if err != nil { return nil, nil, fmt.Errorf("signedLogRoot: %v", err) } @@ -269,13 +269,13 @@ func (t *TrillianClient) SignedLogRoot(treeID int64) (*trillian.SignedLogRoot, * return slr, lr, nil } -// LeavesAppend appends the provided leaves onto the provided tree. The queued +// leavesAppend appends the provided leaves onto the provided tree. The queued // leaf and the leaf inclusion proof are returned. If a leaf was not // successfully appended, the queued leaf will still be returned and the error // will be in the queued leaf. Inclusion proofs will not exist for leaves that // fail to be appended. Note leaves that are duplicates will fail and it is the // callers responsibility to determine how they should be handled. -func (t *TrillianClient) LeavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]QueuedLeafProof, *types.LogRootV1, error) { +func (t *trillianClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, *types.LogRootV1, error) { log.Tracef("LeavesAppend: %v", treeID) // Get the latest signed log root @@ -283,9 +283,9 @@ func (t *TrillianClient) LeavesAppend(treeID int64, leaves []*trillian.LogLeaf) if err != nil { return nil, nil, err } - slr, _, err := t.signedLogRoot(tree) + slr, _, err := t.signedLogRootForTree(tree) if err != nil { - return nil, nil, fmt.Errorf("signedLogRoot pre update: %v", err) + return nil, nil, fmt.Errorf("signedLogRootForTree pre update: %v", err) } // Ensure the tree is not frozen @@ -334,15 +334,15 @@ func (t *TrillianClient) LeavesAppend(treeID int64, leaves []*trillian.LogLeaf) log.Debugf("Stored/Ignored leaves: %v/%v", len(leaves)-n, n) // Get the latest signed log root - slr, lrv1, err := t.signedLogRoot(tree) + slr, lrv1, err := t.signedLogRootForTree(tree) if err != nil { - return nil, nil, fmt.Errorf("signedLogRoot post update: %v", err) + return nil, nil, fmt.Errorf("signedLogRootForTree post update: %v", err) } // Get inclusion proofs - proofs := make([]QueuedLeafProof, 0, len(qlr.QueuedLeaves)) + proofs := make([]queuedLeafProof, 0, len(qlr.QueuedLeaves)) for _, v := range qlr.QueuedLeaves { - qlp := QueuedLeafProof{ + qlp := queuedLeafProof{ QueuedLeaf: v, } @@ -353,9 +353,8 @@ func (t *TrillianClient) LeavesAppend(treeID int64, leaves []*trillian.LogLeaf) // becuase they were a duplicate. c := codes.Code(v.GetStatus().GetCode()) if c == codes.OK { - // Validate merkle leaf hash. We compute the merkle leaf hash in - // other parts of tlogbe manually so we need to ensure that the - // returned merkle leaf hashes are what we expect. + // Verify that the merkle leaf hash is using the expected + // hashing algorithm. m := merkleLeafHash(v.Leaf.LeafValue) if !bytes.Equal(m, v.Leaf.MerkleLeafHash) { e := fmt.Sprintf("unknown merkle leaf hash: got %x, want %x", @@ -378,7 +377,7 @@ func (t *TrillianClient) LeavesAppend(treeID int64, leaves []*trillian.LogLeaf) return proofs, lrv1, nil } -func (t *TrillianClient) leavesByRange(treeID int64, startIndex, count int64) ([]*trillian.LogLeaf, error) { +func (t *trillianClient) leavesByRange(treeID int64, startIndex, count int64) ([]*trillian.LogLeaf, error) { glbrr, err := t.client.GetLeavesByRange(t.ctx, &trillian.GetLeavesByRangeRequest{ LogId: treeID, @@ -391,10 +390,10 @@ func (t *TrillianClient) leavesByRange(treeID int64, startIndex, count int64) ([ return glbrr.Leaves, nil } -// LeavesAll returns all of the leaves for the provided treeID. -func (t *TrillianClient) LeavesAll(treeID int64) ([]*trillian.LogLeaf, error) { +// leavesAll returns all of the leaves for the provided treeID. +func (t *trillianClient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { // Get log root - _, lr, err := t.SignedLogRoot(treeID) + _, lr, err := t.signedLogRoot(treeID) if err != nil { return nil, fmt.Errorf("SignedLogRoot: %v", err) } @@ -403,11 +402,11 @@ func (t *TrillianClient) LeavesAll(treeID int64) ([]*trillian.LogLeaf, error) { return t.leavesByRange(treeID, 0, int64(lr.TreeSize)) } -// LeafProofs returns the LeafProofs for the provided treeID and merkle leaf -// hashes. The inclusion proof returned in the LeafProof is for the tree height +// leafProofs returns the leafProofs for the provided treeID and merkle leaf +// hashes. The inclusion proof returned in the leafProof is for the tree height // specified by the provided LogRootV1. -func (t *TrillianClient) LeafProofs(treeID int64, merkleLeafHashes [][]byte, lr *types.LogRootV1) ([]LeafProof, error) { - log.Tracef("LeafProofs: %v %v %x", treeID, lr.TreeSize, merkleLeafHashes) +func (t *trillianClient) leafProofs(treeID int64, merkleLeafHashes [][]byte, lr *types.LogRootV1) ([]leafProof, error) { + log.Tracef("leafProofs: %v %v %x", treeID, lr.TreeSize, merkleLeafHashes) // Retrieve leaves r, err := t.client.GetLeavesByHash(t.ctx, @@ -420,13 +419,13 @@ func (t *TrillianClient) LeafProofs(treeID int64, merkleLeafHashes [][]byte, lr } // Retrieve proofs - proofs := make([]LeafProof, 0, len(r.Leaves)) + proofs := make([]leafProof, 0, len(r.Leaves)) for _, v := range r.Leaves { p, err := t.inclusionProof(treeID, v.MerkleLeafHash, lr) if err != nil { return nil, fmt.Errorf("inclusionProof %x: %v", v.MerkleLeafHash, err) } - proofs = append(proofs, LeafProof{ + proofs = append(proofs, leafProof{ Leaf: v, Proof: p, }) @@ -436,11 +435,11 @@ func (t *TrillianClient) LeafProofs(treeID int64, merkleLeafHashes [][]byte, lr } // Close closes the trillian grpc connection. -func (t *TrillianClient) Close() { +func (t *trillianClient) close() { t.grpc.Close() } -func trillianClientNew(homeDir, host, keyFile string) (*TrillianClient, error) { +func trillianClientNew(homeDir, host, keyFile string) (*trillianClient, error) { // Setup trillian key file if keyFile == "" { // No file path was given. Use the default path. @@ -488,7 +487,7 @@ func trillianClientNew(homeDir, host, keyFile string) (*TrillianClient, error) { } log.Infof("Trillian key loaded") - t := TrillianClient{ + t := trillianClient{ grpc: g, client: trillian.NewTrillianLogClient(g), admin: trillian.NewTrillianAdminClient(g), diff --git a/politeiad/log.go b/politeiad/log.go index 65d6f668d..9f9abc45f 100644 --- a/politeiad/log.go +++ b/politeiad/log.go @@ -49,7 +49,7 @@ var ( gitbeLog = backendLog.Logger("GITB") tlogbeLog = backendLog.Logger("TLOG") cacheLog = backendLog.Logger("CACH") - blobLog = backendLog.Logger("BLOB") + storeLog = backendLog.Logger("STOR") pluginLog = backendLog.Logger("PLGN") ) @@ -58,7 +58,7 @@ func init() { cockroachdb.UseLogger(cacheLog) gitbe.UseLogger(gitbeLog) tlogbe.UseLogger(tlogbeLog) - filesystem.UseLogger(blobLog) + filesystem.UseLogger(storeLog) comments.UseLogger(pluginLog) } @@ -68,7 +68,7 @@ var subsystemLoggers = map[string]slog.Logger{ "GITB": gitbeLog, "TLOG": tlogbeLog, "CACH": cacheLog, - "BLOB": blobLog, + "STOR": storeLog, "PLGN": pluginLog, } From 972ad6a0863d37bc663d4d12f73e29ec5915e30c Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 28 Jul 2020 09:01:40 -0600 Subject: [PATCH 007/449] vote plugin start --- plugins/comments/comments.go | 80 +-- plugins/ticketvote/ticketvote.go | 527 ++++++++++++++++++ politeiad/backend/backend.go | 3 + .../backend/tlogbe/plugin/comments/journal.go | 52 -- .../{plugin/comments => plugins}/comments.go | 171 +++--- .../{plugin/comments => plugins}/log.go | 2 +- .../{plugin/plugin.go => plugins/plugins.go} | 4 +- .../backend/tlogbe/plugins/ticketvote.go | 123 ++++ politeiad/log.go | 18 +- 9 files changed, 815 insertions(+), 165 deletions(-) create mode 100644 plugins/ticketvote/ticketvote.go delete mode 100644 politeiad/backend/tlogbe/plugin/comments/journal.go rename politeiad/backend/tlogbe/{plugin/comments => plugins}/comments.go (93%) rename politeiad/backend/tlogbe/{plugin/comments => plugins}/log.go (97%) rename politeiad/backend/tlogbe/{plugin/plugin.go => plugins/plugins.go} (90%) create mode 100644 politeiad/backend/tlogbe/plugins/ticketvote.go diff --git a/plugins/comments/comments.go b/plugins/comments/comments.go index 0c5ccd592..3f80a1061 100644 --- a/plugins/comments/comments.go +++ b/plugins/comments/comments.go @@ -13,8 +13,8 @@ type VoteT int type ErrorStatusT int const ( - Version = "1" - ID = "comments" + Version uint32 = 1 + ID = "comments" // Plugin commands CmdNew = "new" // Create a new comment @@ -51,7 +51,7 @@ const ( ) var ( - // Human readable error messages + // ErrorStatus contains human readable error statuses. ErrorStatus = map[ErrorStatusT]string{ ErrorStatusInvalid: "invalid error status", ErrorStatusTokenInvalid: "invalid token", @@ -62,7 +62,7 @@ var ( ErrorStatusParentIDInvalid: "parent id invalid", ErrorStatusNoCommentChanges: "comment did not change", ErrorStatusVoteInvalid: "invalid vote", - ErrorStatusMaxVoteChanges: "user has changed their vote too many times", + ErrorStatusMaxVoteChanges: "max vote changes exceeded", } ) @@ -108,9 +108,9 @@ func EncodeNew(n New) ([]byte, error) { } // DecodeNew decodes a JSON byte slice into a New. -func DecodeNew(b []byte) (*New, error) { +func DecodeNew(payload []byte) (*New, error) { var n New - err := json.Unmarshal(b, &n) + err := json.Unmarshal(payload, &n) if err != nil { return nil, err } @@ -130,9 +130,9 @@ func EncodeNewReply(r NewReply) ([]byte, error) { } // DecodeNew decodes a JSON byte slice into a NewReply. -func DecodeNewReply(b []byte) (*NewReply, error) { +func DecodeNewReply(payload []byte) (*NewReply, error) { var r NewReply - err := json.Unmarshal(b, &r) + err := json.Unmarshal(payload, &r) if err != nil { return nil, err } @@ -155,9 +155,9 @@ func EncodeEdit(e Edit) ([]byte, error) { } // DecodeEdit decodes a JSON byte slice into a Edit. -func DecodeEdit(b []byte) (*Edit, error) { +func DecodeEdit(payload []byte) (*Edit, error) { var e Edit - err := json.Unmarshal(b, &e) + err := json.Unmarshal(payload, &e) if err != nil { return nil, err } @@ -177,9 +177,9 @@ func EncodeEditReply(r EditReply) ([]byte, error) { } // DecodeEdit decodes a JSON byte slice into a EditReply. -func DecodeEditReply(b []byte) (*EditReply, error) { +func DecodeEditReply(payload []byte) (*EditReply, error) { var r EditReply - err := json.Unmarshal(b, &r) + err := json.Unmarshal(payload, &r) if err != nil { return nil, err } @@ -201,9 +201,9 @@ func EncodeDel(d Del) ([]byte, error) { } // DecodeDel decodes a JSON byte slice into a Del. -func DecodeDel(b []byte) (*Del, error) { +func DecodeDel(payload []byte) (*Del, error) { var d Del - err := json.Unmarshal(b, &d) + err := json.Unmarshal(payload, &d) if err != nil { return nil, err } @@ -222,9 +222,9 @@ func EncodeDelReply(d DelReply) ([]byte, error) { } // DecodeDelReply decodes a JSON byte slice into a DelReply. -func DecodeDelReply(b []byte) (*DelReply, error) { +func DecodeDelReply(payload []byte) (*DelReply, error) { var d DelReply - err := json.Unmarshal(b, &d) + err := json.Unmarshal(payload, &d) if err != nil { return nil, err } @@ -245,9 +245,9 @@ func EncodeGet(g Get) ([]byte, error) { } // DecodeGet decodes a JSON byte slice into a Get. -func DecodeGet(b []byte) (*Get, error) { +func DecodeGet(payload []byte) (*Get, error) { var g Get - err := json.Unmarshal(b, &g) + err := json.Unmarshal(payload, &g) if err != nil { return nil, err } @@ -268,9 +268,9 @@ func EncodeGetReply(g GetReply) ([]byte, error) { } // DecodeGetReply decodes a JSON byte slice into a GetReply. -func DecodeGetReply(b []byte) (*GetReply, error) { +func DecodeGetReply(payload []byte) (*GetReply, error) { var g GetReply - err := json.Unmarshal(b, &g) + err := json.Unmarshal(payload, &g) if err != nil { return nil, err } @@ -288,9 +288,9 @@ func EncodeGetAll(g GetAll) ([]byte, error) { } // DecodeGetAll decodes a JSON byte slice into a GetAll. -func DecodeGetAll(b []byte) (*GetAll, error) { +func DecodeGetAll(payload []byte) (*GetAll, error) { var g GetAll - err := json.Unmarshal(b, &g) + err := json.Unmarshal(payload, &g) if err != nil { return nil, err } @@ -308,9 +308,9 @@ func EncodeGetAllReply(g GetAllReply) ([]byte, error) { } // DecodeGetAllReply decodes a JSON byte slice into a GetAllReply. -func DecodeGetAllReply(b []byte) (*GetAllReply, error) { +func DecodeGetAllReply(payload []byte) (*GetAllReply, error) { var g GetAllReply - err := json.Unmarshal(b, &g) + err := json.Unmarshal(payload, &g) if err != nil { return nil, err } @@ -330,9 +330,9 @@ func EncodeGetVersion(g GetVersion) ([]byte, error) { } // DecodeGetVersion decodes a JSON byte slice into a GetVersion. -func DecodeGetVersion(b []byte) (*GetVersion, error) { +func DecodeGetVersion(payload []byte) (*GetVersion, error) { var g GetVersion - err := json.Unmarshal(b, &g) + err := json.Unmarshal(payload, &g) if err != nil { return nil, err } @@ -350,9 +350,9 @@ func EncodeGetVersionReply(g GetVersionReply) ([]byte, error) { } // DecodeGetVersionReply decodes a JSON byte slice into a GetVersionReply. -func DecodeGetVersionReply(b []byte) (*GetVersionReply, error) { +func DecodeGetVersionReply(payload []byte) (*GetVersionReply, error) { var g GetVersionReply - err := json.Unmarshal(b, &g) + err := json.Unmarshal(payload, &g) if err != nil { return nil, err } @@ -370,9 +370,9 @@ func EncodeCount(c Count) ([]byte, error) { } // DecodeCount decodes a JSON byte slice into a Count. -func DecodeCount(b []byte) (*Count, error) { +func DecodeCount(payload []byte) (*Count, error) { var c Count - err := json.Unmarshal(b, &c) + err := json.Unmarshal(payload, &c) if err != nil { return nil, err } @@ -390,9 +390,9 @@ func EncodeCountReply(c CountReply) ([]byte, error) { } // DecodeCountReply decodes a JSON byte slice into a CountReply. -func DecodeCountReply(b []byte) (*CountReply, error) { +func DecodeCountReply(payload []byte) (*CountReply, error) { var c CountReply - err := json.Unmarshal(b, &c) + err := json.Unmarshal(payload, &c) if err != nil { return nil, err } @@ -402,10 +402,10 @@ func DecodeCountReply(b []byte) (*CountReply, error) { // Vote casts a comment vote (upvote or downvote). // // The uuid is required because the effect of a new vote on a comment score -// depends on the previous vote from that uuid. Example, a user casts an upvote -// on a comment that they have already upvoted, the resulting vote score is 0 -// due to the second upvote removing the original upvote. The public key cannot -// be relied on to remain the same for each user so a uuid must be included. +// depends on the previous vote from that uuid. Example, a user upvotes a +// comment that they have already upvoted, the resulting vote score is 0 due to +// the second upvote removing the original upvote. The public key cannot be +// relied on to remain the same for each user so a uuid must be included. type Vote struct { UUID string `json:"uuid"` // Unique user ID Token string `json:"token"` // Record token @@ -421,9 +421,9 @@ func EncodeVote(v Vote) ([]byte, error) { } // DecodeVote decodes a JSON byte slice into a Vote. -func DecodeVote(b []byte) (*Vote, error) { +func DecodeVote(payload []byte) (*Vote, error) { var v Vote - err := json.Unmarshal(b, &v) + err := json.Unmarshal(payload, &v) if err != nil { return nil, err } @@ -443,9 +443,9 @@ func EncodeVoteReply(v VoteReply) ([]byte, error) { } // DecodeVoteReply decodes a JSON byte slice into a VoteReply. -func DecodeVoteReply(b []byte) (*VoteReply, error) { +func DecodeVoteReply(payload []byte) (*VoteReply, error) { var v VoteReply - err := json.Unmarshal(b, &v) + err := json.Unmarshal(payload, &v) if err != nil { return nil, err } diff --git a/plugins/ticketvote/ticketvote.go b/plugins/ticketvote/ticketvote.go new file mode 100644 index 000000000..3e9f0df91 --- /dev/null +++ b/plugins/ticketvote/ticketvote.go @@ -0,0 +1,527 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ticketvote + +import ( + "encoding/json" + "fmt" +) + +type ActionT string +type VoteT int +type VoteStatusT int +type ErrorStatusT int + +const ( + Version uint32 = 1 + ID = "ticketvotes" + + // Plugin commands + CmdAuthorize = "authorize" // Authorize a vote + CmdStart = "start" // Start a vote + CmdStartRunoff = "startrunoff" // Start a runoff vote + CmdBallot = "ballot" // Cast a ballot of votes + CmdDetails = "details" // Get details of a vote + CmdCastVotes = "castvotes" // Get cast votes + CmdSummaries = "summaries" // Get vote summaries + CmdInventory = "inventory" // Get inventory grouped by vote status + CmdProofs = "proofs" // Get inclusion proofs + + // Authorize vote actions + ActionAuthorize ActionT = "authorize" + ActionRevoke ActionT = "revoke" + + // Vote statuses + VoteStatusInvalid VoteStatusT = 0 // Invalid status + VoteStatusUnauthorized VoteStatusT = 1 // Vote cannot be started + VoteStatusAuthorized VoteStatusT = 2 // Vote can be started + VoteStatusStarted VoteStatusT = 3 // Vote has been started + VoteStatusFinished VoteStatusT = 4 // Vote has finished + + // Vote types + VoteTypeInvalid VoteT = 0 + + // VoteTypeStandard is used to indicate a simple approve or reject + // vote where the winner is the voting option that has met the + // specified quorum and pass requirements. + VoteTypeStandard VoteT = 1 + + // VoteTypeRunoff specifies a runoff vote that multiple records + // compete in. All records are voted on like normal, but there can + // only be one winner in a runoff vote. The winner is the record + // that meets the quorum requirement, meets the pass requirement, + // and that has the most net yes votes. The winning record is + // considered approved and all other records are considered to be + // rejected. If no records meet the quorum and pass requirements + // then all records are considered rejected. Note, in a runoff vote + // it's possible for a proposal to meet both the quorum and pass + // requirements but still be rejected if it does not have the most + // net yes votes. + VoteTypeRunoff VoteT = 2 + + // Vote duration requirements in blocks + VoteDurationMinMainnet = 2016 + VoteDurationMaxMainnet = 4032 + VoteDurationMinTestnet = 0 + VoteDurationMaxTestnet = 4032 + + // Vote option IDs + VoteOptionIDApprove = "yes" + VoteOptionIDReject = "no" + + // Error status codes + // TODO change politeiavoter to use these error codes + ErrorStatusInvalid ErrorStatusT = 0 +) + +var ( + // ErrorStatus contains human readable error statuses. + ErrorStatus = map[ErrorStatusT]string{ + ErrorStatusInvalid: "invalid error status", + } +) + +// UserError represents an error that is cause by something that the user did. +type UserError struct { + ErrorCode ErrorStatusT + ErrorContext []string +} + +// Error satisfies the error interface. +func (e UserError) Error() string { + return fmt.Sprintf("plugin error code: %v", e.ErrorCode) +} + +// Authorize authorizes a ticket vote or revokes a previous authorization. +type Authorize struct { + Token string `json:"token"` // Record token + Version uint32 `json:"version"` // Record version + Action ActionT `json:"action"` // Authorize or revoke + PublicKey string `json:"publickey"` // Public key used for signature + Signature string `json:"signature"` // Signature of token+version+action +} + +// EncodeAuthorize encodes an Authorize into a JSON byte slice. +func EncodeAuthorize(a Authorize) ([]byte, error) { + return json.Marshal(a) +} + +// DecodeAuthorize decodes a JSON byte slice into a Authorize. +func DecodeAuthorize(payload []byte) (*Authorize, error) { + var a Authorize + err := json.Unmarshal(payload, &a) + if err != nil { + return nil, err + } + return &a, nil +} + +// AuthorizeReply is the reply to the Authorize command. +type AuthorizeReply struct { + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + Receipt string `json:"receipt"` // Server signature of client signature +} + +// VoteOption describes a single vote option. +type VoteOption struct { + ID string `json:"id"` // Single, unique word (e.g. yes) + Description string `json:"description"` // Longer description of the vote + Bits uint64 `json:"bits"` // Bits used for this option +} + +// Vote describes the options and parameters of a ticket vote. +type Vote struct { + Token string `json:"token"` // Record token + Version uint32 `json:"version"` // Record version + Type VoteT `json:"type"` // Vote type + Mask uint64 `json:"mask"` // Valid vote bits + Duration uint32 `json:"duration"` // Duration in blocks + + // QuorumPercentage is the percent of elligible votes required for + // the vote to meet a quorum. + QuorumPercentage uint32 `json:"quorumpercentage"` + + // PassPercentage is the percent of total votes that are required + // to consider a vote option as passed. + PassPercentage uint32 `json:"passpercentage"` + + Options []VoteOption `json:"options"` +} + +// Start starts a ticket vote. +type Start struct { + Vote Vote `json:"vote"` // Vote options and params + PublicKey string `json:"publickey"` // Public key used for signature + + // Signature is the signature of a SHA256 digest of the JSON + // encoded Vote structure. + Signature string `json:"signature"` +} + +// EncodeStart encodes a Start into a JSON byte slice. +func EncodeStart(s Start) ([]byte, error) { + return json.Marshal(s) +} + +// DecodeStart decodes a JSON byte slice into a Start. +func DecodeStartVote(payload []byte) (*Start, error) { + var s Start + err := json.Unmarshal(payload, &s) + if err != nil { + return nil, err + } + return &s, nil +} + +// StartReply is the reply to the Start command. +type StartReply struct { + StartBlockHeight uint32 `json:"startblockheight"` // Block height + StartBlockHash string `json:"startblockhash"` // Block hash + EndBlockHeight uint32 `json:"endblockheight"` // Height of vote end + EligibleTickets []string `json:"eligibletickets"` // Valid voting tickets +} + +// EncodeStartReply encodes a StartReply into a JSON byte slice. +func EncodeStartReply(sr StartReply) ([]byte, error) { + return json.Marshal(sr) +} + +// DecodeStartReply decodes a JSON byte slice into a StartReply. +func DecodeStartReplyVote(payload []byte) (*StartReply, error) { + var sr StartReply + err := json.Unmarshal(payload, &sr) + if err != nil { + return nil, err + } + return &sr, nil +} + +// StartRunoff starts a runoff vote between the provided submissions. Each +// submission is required to have its own Authorize and Start. +type StartRunoff struct { + Authorizations []Authorize `json:"authorizations"` + Votes []Start `json:"votes"` +} + +// EncodeStartRunoff encodes a StartRunoff into a JSON byte slice. +func EncodeStartRunoff(sr StartRunoff) ([]byte, error) { + return json.Marshal(sr) +} + +// DecodeStartRunoff decodes a JSON byte slice into a StartRunoff. +func DecodeStartRunoff(payload []byte) (*StartRunoff, error) { + var sr StartRunoff + err := json.Unmarshal(payload, &sr) + if err != nil { + return nil, err + } + return &sr, nil +} + +// StartRunoffReply is the reply to the StartRunoff command. +type StartRunoffReply struct { + StartBlockHeight uint32 `json:"startblockheight"` + StartBlockHash string `json:"startblockhash"` + EndBlockHeight uint32 `json:"endblockheight"` + EligibleTickets []string `json:"eligibletickets"` +} + +// EncodeStartRunoffReply encodes a StartRunoffReply into a JSON byte slice. +func EncodeStartRunoffReply(srr StartRunoffReply) ([]byte, error) { + return json.Marshal(srr) +} + +// DecodeStartRunoffReply decodes a JSON byte slice into a StartRunoffReply. +func DecodeStartRunoffReply(payload []byte) (*StartRunoffReply, error) { + var srr StartRunoffReply + err := json.Unmarshal(payload, &srr) + if err != nil { + return nil, err + } + return &srr, nil +} + +// CastVote is a signed ticket vote. +type CastVote struct { + Token string `json:"token"` // Record token + Ticket string `json:"ticket"` // Ticket ID + VoteBit string `json:"votebit"` // Selected vote bit, hex encoded + Signature string `json:"signature"` // Signature of Token+Ticket+VoteBit +} + +// CatVoteReply contains the receipt for the cast vote. If an error occured +// while casting the vote the receipt will be empty and a error code will be +// present. +type CastVoteReply struct { + Ticket string `json:"ticket"` // Ticket ID + Receipt string `json:"receipt"` // Server sig of client sig + ErrorCode ErrorStatusT `json:"errorcode,omitempty"` +} + +// Ballot is a batch of votes that are sent to the server. +type Ballot struct { + Votes []CastVote `json:"votes"` +} + +// EncodeBallot encodes a Ballot into a JSON byte slice. +func EncodeBallot(b Ballot) ([]byte, error) { + return json.Marshal(b) +} + +// DecodeBallot decodes a JSON byte slice into a Ballot. +func DecodeBallotVote(payload []byte) (*Ballot, error) { + var b Ballot + err := json.Unmarshal(payload, &b) + if err != nil { + return nil, err + } + return &b, nil +} + +// BallotReply is a reply to a batched list of votes. +type BallotReply struct { + Receipts []CastVoteReply `json:"receipts"` +} + +// EncodeBallot encodes a Ballot into a JSON byte slice. +func EncodeBallotReply(b BallotReply) ([]byte, error) { + return json.Marshal(b) +} + +// DecodeBallotReply decodes a JSON byte slice into a BallotReply. +func DecodeBallotReplyVote(payload []byte) (*BallotReply, error) { + var b BallotReply + err := json.Unmarshal(payload, &b) + if err != nil { + return nil, err + } + return &b, nil +} + +// Details requests the vote details for the specified record token. +type Details struct { + Token string `json:"token"` +} + +// EncodeDetails encodes a Details into a JSON byte slice. +func EncodeDetails(d Details) ([]byte, error) { + return json.Marshal(d) +} + +// DecodeDetails decodes a JSON byte slice into a Details. +func DecodeDetailsVote(payload []byte) (*Details, error) { + var d Details + err := json.Unmarshal(payload, &d) + if err != nil { + return nil, err + } + return &d, nil +} + +// AuthorizeDetails describes the details of a vote authorization. +type AuthorizeDetails struct { + Token string `json:"token"` // Record token + Version uint32 `json:"version"` // Record version + Action string `json:"action"` // Authorize or revoke + PublicKey string `json:"publickey"` // Public key used for signature + Signature string `json:"signature"` // Signature of token+version+action + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + Receipt string `json:"receipt"` // Server signature of client signature +} + +// DetailsReply is the reply to the Details command. +type DetailsReply struct { + // Auths contains all authorizations and revokes that were made for + // this ticket vote. + Auths []AuthorizeDetails `json:"auths"` + + // Vote details + Vote Vote `json:"vote"` // Vote params + PublicKey string `json:"publickey"` // Key used for sig + Signature string `json:"signature"` // Sig of Vote hash + StartBlockHeight uint32 `json:"startblockheight"` // Start block height + StartBlockHash string `json:"startblockhash"` // Start block hash + EndBlockHeight uint32 `json:"endblockheight"` // End block height + EligibleTickets []string `json:"eligibletickets"` // Valid voting tickets +} + +// EncodeDetailsReply encodes a DetailsReply into a JSON byte slice. +func EncodeDetailsReply(dr DetailsReply) ([]byte, error) { + return json.Marshal(dr) +} + +// DecodeDetailsReply decodes a JSON byte slice into a DetailsReply. +func DecodeDetailsReplyVote(payload []byte) (*DetailsReply, error) { + var dr DetailsReply + err := json.Unmarshal(payload, &dr) + if err != nil { + return nil, err + } + return &dr, nil +} + +// CastVotes requests the cast votes for the provided record token. +type CastVotes struct { + Token string `json:"token"` +} + +// EncodeCastVotes encodes a CastVotes into a JSON byte slice. +func EncodeCastVotes(cv CastVotes) ([]byte, error) { + return json.Marshal(cv) +} + +// DecodeCastVotes decodes a JSON byte slice into a CastVotes. +func DecodeCastVotesVote(payload []byte) (*CastVotes, error) { + var cv CastVotes + err := json.Unmarshal(payload, &cv) + if err != nil { + return nil, err + } + return &cv, nil +} + +// CastVotesReply is the rely to the CastVotes command. +type CastVotesReply struct { + CastVotes []CastVote `json:"castvotes"` +} + +// EncodeCastVotesReply encodes a CastVotesReply into a JSON byte slice. +func EncodeCastVotesReply(cvr CastVotesReply) ([]byte, error) { + return json.Marshal(cvr) +} + +// DecodeCastVotesReply decodes a JSON byte slice into a CastVotesReply. +func DecodeCastVotesReplyVote(payload []byte) (*CastVotesReply, error) { + var cvr CastVotesReply + err := json.Unmarshal(payload, &cvr) + if err != nil { + return nil, err + } + return &cvr, nil +} + +// Summaries requests the vote summaries for the provided record tokens. +type Summaries struct { + Tokens []string `json:"tokens"` +} + +// EncodeSummaries encodes a Summaries into a JSON byte slice. +func EncodeSummaries(s Summaries) ([]byte, error) { + return json.Marshal(s) +} + +// DecodeSummaries decodes a JSON byte slice into a Summaries. +func DecodeSummariesVote(payload []byte) (*Summaries, error) { + var s Summaries + err := json.Unmarshal(payload, &s) + if err != nil { + return nil, err + } + return &s, nil +} + +// Result describes a vote option and the total number of votes that have been +// cast for this option. +type Result struct { + ID string `json:"id"` // Single unique word (e.g. yes) + Description string `json:"description"` // Longer description of the vote + Bits uint64 `json:"bits"` // Bits used for this option + Votes uint64 `json:"votes"` // Votes cast for this option +} + +// Summary summarizes the vote params and results for a ticket vote. +type Summary struct { + Type VoteT `json:"type"` + Status VoteStatusT `json:"status"` + Duration uint32 `json:"duration"` + StartBlockHeight uint32 `json:"startblockheight"` + EndBlockHeight string `json:"endblockheight"` + EligibleTickets uint32 `json:"eligibletickets"` + QuorumPercentage uint32 `json:"quorumpercentage"` + PassPercentage uint32 `json:"passpercentage"` + Results []Result `json:"results"` + + // Approved describes whether the vote has been approved. This will + // only be present when the vote type is VoteTypeStandard or + // VoteTypeRunoff, both of which only allow for approve/reject + // voting options. + Approved bool `json:"approved,omitempty"` +} + +// SummariesReply is the reply to the Summaries command. +type SummariesReply struct { + // Summaries contains a vote summary for each of the provided + // tokens. The map will not contain an entry for any tokens that + // did not correspond to an actual record. It is the callers + // responsibility to ensure that a summary is returned for all of + // the provided tokens. + Summaries map[string]Summary `json:"summaries"` // [token]Summary + + // BestBlock is the best block value that was used to prepare the + // the summaries. + BestBlock uint64 `json:"bestblock"` +} + +// EncodeSummariesReply encodes a SummariesReply into a JSON byte slice. +func EncodeSummariesReply(sr SummariesReply) ([]byte, error) { + return json.Marshal(sr) +} + +// DecodeSummariesReply decodes a JSON byte slice into a SummariesReply. +func DecodeSummariesReplyVote(payload []byte) (*SummariesReply, error) { + var sr SummariesReply + err := json.Unmarshal(payload, &sr) + if err != nil { + return nil, err + } + return &sr, nil +} + +// Inventory requests the tokens of all public, non-abandoned records +// catagorized by vote status. +type Inventory struct{} + +// EncodeInventory encodes a Inventory into a JSON byte slice. +func EncodeInventory(i Inventory) ([]byte, error) { + return json.Marshal(i) +} + +// DecodeInventory decodes a JSON byte slice into a Inventory. +func DecodeInventoryVote(payload []byte) (*Inventory, error) { + var i Inventory + err := json.Unmarshal(payload, &i) + if err != nil { + return nil, err + } + return &i, nil +} + +// InventoryReply is the reply to the Inventory command. It contains the tokens +// of all public, non-abandoned records catagorized by vote status. +type InventoryReply struct { + Unauthorized []string `json:"unauthorized"` + Authorized []string `json:"authorized"` + Started []string `json:"started"` + Finished []string `json:"finished"` + + // BestBlock is the best block value that was used to prepare + // the inventory. + BestBlock uint64 `json:"bestblock"` +} + +// EncodeInventoryReply encodes a InventoryReply into a JSON byte slice. +func EncodeInventoryReply(ir InventoryReply) ([]byte, error) { + return json.Marshal(ir) +} + +// DecodeInventoryReply decodes a JSON byte slice into a InventoryReply. +func DecodeInventoryReplyVote(payload []byte) (*InventoryReply, error) { + var ir InventoryReply + err := json.Unmarshal(payload, &ir) + if err != nil { + return nil, err + } + return &ir, nil +} diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index 4e55886ce..aac385f3f 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -348,6 +348,9 @@ type Backend interface { // Inventory retrieves various record records. Inventory(uint, uint, bool, bool) ([]Record, []Record, error) + // TODO Inventory needs to return the token inventory grouped by + // record status. + // Obtain plugin settings GetPlugins() ([]Plugin, error) diff --git a/politeiad/backend/tlogbe/plugin/comments/journal.go b/politeiad/backend/tlogbe/plugin/comments/journal.go deleted file mode 100644 index 2c0d4f561..000000000 --- a/politeiad/backend/tlogbe/plugin/comments/journal.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright (c) 2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package comments - -// The comments plugin treats the trillian tree as a journal. The following -// types are the journal actions that are saved to disk. - -// commentAdd is the structure that is saved to disk when a comment is created -// or edited. -type commentAdd struct { - // Data generated by client - Token string `json:"token"` // Record token - ParentID uint32 `json:"parentid"` // Parent comment ID - Comment string `json:"comment"` // Comment - PublicKey string `json:"publickey"` // Pubkey used for Signature - Signature string `json:"signature"` // Signature of Token+ParentID+Comment - - // Metadata generated by server - CommentID uint32 `json:"commentid"` // Comment ID - Version uint32 `json:"version"` // Comment version - Receipt string `json:"receipt"` // Server signature of client signature - Timestamp int64 `json:"timestamp"` // Received UNIX timestamp -} - -// commentDel is the structure that is saved to disk when a comment is deleted. -type commentDel struct { - // Data generated by client - Token string `json:"token"` // Record token - CommentID uint32 `json:"commentid"` // Comment ID - Reason string `json:"reason"` // Reason for deleting - PublicKey string `json:"publickey"` // Pubkey used for Signature - Signature string `json:"signature"` // Signature of Token+CommentID+Reason - - // Metadata generated by server - Receipt string `json:"receipt"` // Server signature of client signature - Timestamp int64 `json:"timestamp"` // Received UNIX timestamp -} - -// commentVote is the structure that is saved to disk when a comment is voted -// on. -type commentVote struct { - UUID string `json:"uuid"` // Unique user ID - Token string `json:"token"` // Record token - CommentID uint32 `json:"commentid"` // Comment ID - Vote int64 `json:"vote"` // Upvote or downvote - PublicKey string `json:"publickey"` // Public key used for signature - Signature string `json:"signature"` // Signature of Token+CommentID+Vote - Receipt string `json:"receipt"` // Server signature of client signature - Timestamp int64 `json:"timestamp"` // Received UNIX timestamp -} diff --git a/politeiad/backend/tlogbe/plugin/comments/comments.go b/politeiad/backend/tlogbe/plugins/comments.go similarity index 93% rename from politeiad/backend/tlogbe/plugin/comments/comments.go rename to politeiad/backend/tlogbe/plugins/comments.go index 0cb54d063..db484ea7a 100644 --- a/politeiad/backend/tlogbe/plugin/comments/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package comments +package plugins import ( "bytes" @@ -19,7 +19,6 @@ import ( "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugin" "github.com/decred/politeia/politeiad/backend/tlogbe/store" "github.com/decred/politeia/util" ) @@ -40,9 +39,10 @@ const ( ) var ( - _ plugin.Plugin = (*commentsPlugin)(nil) + _ Plugin = (*commentsPlugin)(nil) ) +// commentsPlugin satisfies the Plugin interface. type commentsPlugin struct { sync.Mutex id *identity.FullIdentity @@ -53,6 +53,53 @@ type commentsPlugin struct { mutexes map[string]*sync.Mutex // [string]mutex } +// commentAdd is the structure that is saved to disk when a comment is created +// or edited. +type commentAdd struct { + // Data generated by client + Token string `json:"token"` // Record token + ParentID uint32 `json:"parentid"` // Parent comment ID + Comment string `json:"comment"` // Comment + PublicKey string `json:"publickey"` // Pubkey used for Signature + Signature string `json:"signature"` // Signature of Token+ParentID+Comment + + // Metadata generated by server + CommentID uint32 `json:"commentid"` // Comment ID + Version uint32 `json:"version"` // Comment version + Receipt string `json:"receipt"` // Server signature of client signature + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp +} + +// commentDel is the structure that is saved to disk when a comment is deleted. +type commentDel struct { + // Data generated by client + Token string `json:"token"` // Record token + CommentID uint32 `json:"commentid"` // Comment ID + Reason string `json:"reason"` // Reason for deleting + PublicKey string `json:"publickey"` // Pubkey used for Signature + Signature string `json:"signature"` // Signature of Token+CommentID+Reason + + // Metadata generated by server + Receipt string `json:"receipt"` // Server signature of client signature + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp +} + +// commentVote is the structure that is saved to disk when a comment is voted +// on. +type commentVote struct { + // Data generated by client + UUID string `json:"uuid"` // Unique user ID + Token string `json:"token"` // Record token + CommentID uint32 `json:"commentid"` // Comment ID + Vote int64 `json:"vote"` // Upvote or downvote + PublicKey string `json:"publickey"` // Public key used for signature + Signature string `json:"signature"` // Signature of Token+CommentID+Vote + + // Metadata generated by server + Receipt string `json:"receipt"` // Server signature of client signature + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp +} + type voteIndex struct { Vote comments.VoteT `json:"vote"` MerkleHash []byte `json:"merklehash"` @@ -439,6 +486,58 @@ func indexSave(client *tlogbe.PluginClient, idx index) error { return nil } +// indexAddCommentVote adds the provided comment vote to the index and +// calculates the new vote score. The updated index is returned. The effect of +// a new vote on a comment depends on the previous vote from that uuid. +// Example, a user upvotes a comment that they have already upvoted, the +// resulting vote score is 0 due to the second upvote removing the original +// upvote. +func indexAddCommentVote(idx index, cv commentVote, merkleHash []byte) index { + // Get the existing votes for this uuid + cidx := idx.Comments[cv.CommentID] + votes, ok := cidx.Votes[cv.UUID] + if !ok { + // This uuid has not cast any votes + votes = make([]voteIndex, 0, 1) + } + + // Get the previous vote that this uuid made + var votePrev comments.VoteT + if len(votes) != 0 { + votePrev = votes[len(votes)-1].Vote + } + + // Update index vote score + voteNew := comments.VoteT(cv.Vote) + switch { + case votePrev == 0: + // No previous vote. Add the new vote to the score. + cidx.Score += int64(voteNew) + + case voteNew == votePrev: + // New vote is the same as the previous vote. Remove the previous + // vote from the score. + cidx.Score -= int64(votePrev) + + case voteNew != votePrev: + // New vote is different than the previous vote. Remove the + // previous vote from the score and add the new vote to the + // score. + cidx.Score -= int64(votePrev) + cidx.Score += int64(voteNew) + } + + // Update the index + votes = append(votes, voteIndex{ + Vote: comments.VoteT(cv.Vote), + MerkleHash: merkleHash, + }) + cidx.Votes[cv.UUID] = votes + idx.Comments[cv.CommentID] = cidx + + return idx +} + func indexLatest(client *tlogbe.PluginClient) (*index, error) { // Get all comment indexes blobs, err := client.BlobsByKeyPrefix(keyPrefixCommentsIndex) @@ -1185,58 +1284,6 @@ func (p *commentsPlugin) cmdCount(payload string) (string, error) { return string(reply), nil } -// indexAddCommentVote adds the provided comment vote to the index and -// calculates the new vote score. The updated index is returned. The effect of -// a new vote on a comment depends on the previous vote from that uuid. -// Example, a user upvotes a comment that they have already upvoted, the -// resulting vote score is 0 due to the second upvote removing the original -// upvote. -func indexAddCommentVote(idx index, cv commentVote, merkleHash []byte) index { - // Get the existing votes for this uuid - cidx := idx.Comments[cv.CommentID] - votes, ok := cidx.Votes[cv.UUID] - if !ok { - // This uuid has not cast any votes - votes = make([]voteIndex, 0, 1) - } - - // Get the previous vote that this uuid made - var votePrev comments.VoteT - if len(votes) != 0 { - prev = votes[len(votes)-1].Vote - } - - // Update index vote score - voteNew := comments.VoteT(cv.Vote) - switch { - case votePrev == 0: - // No previous vote. Add the new vote to the score. - cidx.Score += int64(voteNew) - - case voteNew == votePrev: - // New vote is the same as the previous vote. Remove the previous - // vote from the score. - cidx.Score -= int64(votePrev) - - case voteNew != votePrev: - // New vote is different than the previous vote. Remove the - // previous vote from the score and add the new vote to the - // score. - cidx.Score -= int64(votePrev) - cidx.Score += int64(voteNew) - } - - // Update the index - votes = append(votes, voteIndex{ - Vote: comments.VoteT(cv.Vote), - MerkleHash: merkleHash, - }) - cidx.Votes[cv.UUID] = votes - idx.Comments[cv.CommentID] = cidx - - return idx -} - func (p *commentsPlugin) cmdVote(payload string) (string, error) { log.Tracef("comments cmdVote: %v", payload) @@ -1386,18 +1433,18 @@ func (p *commentsPlugin) Cmd(cmd, payload string) (string, error) { return p.cmdProofs(payload) } - return "", plugin.ErrInvalidPluginCmd + return "", ErrInvalidPluginCmd } // Hook executes a plugin hook. -func (p *commentsPlugin) Hook(h plugin.HookT, payload string) error { - log.Tracef("comments: Hook: %v", plugin.Hook[h]) +func (p *commentsPlugin) Hook(h HookT, payload string) error { + log.Tracef("comments Hook: %v", Hook[h]) return nil } // Fsck performs a plugin filesystem check. func (p *commentsPlugin) Fsck() error { - log.Tracef("comments: Fsck") + log.Tracef("comments Fsck") // Make sure commentDel blobs were actually deleted @@ -1406,12 +1453,12 @@ func (p *commentsPlugin) Fsck() error { // Setup performs any plugin setup work that needs to be done. func (p *commentsPlugin) Setup() error { - log.Tracef("comments: Setup") + log.Tracef("comments Setup") return nil } -// New returns a new comments plugin. -func New(id *identity.FullIdentity, backend *tlogbe.Tlogbe) (*commentsPlugin, error) { +// CommentsPluginNew returns a new comments plugin. +func CommentsPluginNew(id *identity.FullIdentity, backend *tlogbe.Tlogbe) (*commentsPlugin, error) { return &commentsPlugin{ id: id, backend: backend, diff --git a/politeiad/backend/tlogbe/plugin/comments/log.go b/politeiad/backend/tlogbe/plugins/log.go similarity index 97% rename from politeiad/backend/tlogbe/plugin/comments/log.go rename to politeiad/backend/tlogbe/plugins/log.go index f1d18be7c..18e5ec634 100644 --- a/politeiad/backend/tlogbe/plugin/comments/log.go +++ b/politeiad/backend/tlogbe/plugins/log.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package comments +package plugins import "github.com/decred/slog" diff --git a/politeiad/backend/tlogbe/plugin/plugin.go b/politeiad/backend/tlogbe/plugins/plugins.go similarity index 90% rename from politeiad/backend/tlogbe/plugin/plugin.go rename to politeiad/backend/tlogbe/plugins/plugins.go index 6b02c3bca..deaa2a87f 100644 --- a/politeiad/backend/tlogbe/plugin/plugin.go +++ b/politeiad/backend/tlogbe/plugins/plugins.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package plugin +package plugins import "errors" @@ -31,6 +31,8 @@ var ( ErrInvalidPluginCmd = errors.New("invalid plugin command") ) +// Plugin provides an interface for the backend to use when interacting with +// plugins. type Plugin interface { // Perform plugin setup Setup() error diff --git a/politeiad/backend/tlogbe/plugins/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote.go new file mode 100644 index 000000000..fddd6b630 --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/ticketvote.go @@ -0,0 +1,123 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package plugins + +import ( + "github.com/decred/politeia/plugins/ticketvote" + "github.com/decred/politeia/politeiad/api/v1/identity" + "github.com/decred/politeia/politeiad/backend/tlogbe" +) + +var ( + _ Plugin = (*ticketVotePlugin)(nil) +) + +// ticketVotePlugin satsifies the Plugin interface. +type ticketVotePlugin struct { + id *identity.FullIdentity + backend *tlogbe.Tlogbe +} + +func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { + log.Tracef("ticketvote cmdAuthorize: %v", payload) + + return "", nil +} + +func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { + log.Tracef("ticketvote cmdStart: %v", payload) + + return "", nil +} + +func (p *ticketVotePlugin) cmdStartRunoff(payload string) (string, error) { + log.Tracef("ticketvote cmdStartRunoff: %v", payload) + + return "", nil +} + +func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { + log.Tracef("ticketvote cmdBallot: %v", payload) + + return "", nil +} + +func (p *ticketVotePlugin) cmdDetails(payload string) (string, error) { + log.Tracef("ticketvote cmdDetails: %v", payload) + + return "", nil +} + +func (p *ticketVotePlugin) cmdCastVotes(payload string) (string, error) { + log.Tracef("ticketvote cmdCastVotes: %v", payload) + + return "", nil +} + +func (p *ticketVotePlugin) cmdSummaries(payload string) (string, error) { + log.Tracef("ticketvote cmdSummaries: %v", payload) + + return "", nil +} + +func (p *ticketVotePlugin) cmdInventory(payload string) (string, error) { + log.Tracef("ticketvote cmdInventory: %v", payload) + + return "", nil +} + +// Cmd executes a plugin command. +func (p *ticketVotePlugin) Cmd(cmd, payload string) (string, error) { + log.Tracef("ticketvote Cmd: %v", cmd) + + switch cmd { + case ticketvote.CmdAuthorize: + return p.cmdAuthorize(payload) + case ticketvote.CmdStart: + return p.cmdStart(payload) + case ticketvote.CmdStartRunoff: + return p.cmdStartRunoff(payload) + case ticketvote.CmdBallot: + return p.cmdBallot(payload) + case ticketvote.CmdDetails: + return p.cmdDetails(payload) + case ticketvote.CmdCastVotes: + return p.cmdCastVotes(payload) + case ticketvote.CmdSummaries: + return p.cmdSummaries(payload) + case ticketvote.CmdInventory: + return p.cmdInventory(payload) + } + + return "", ErrInvalidPluginCmd +} + +// Hook executes a plugin hook. +func (p *ticketVotePlugin) Hook(h HookT, payload string) error { + log.Tracef("ticketvote Hook: %v %v", h, payload) + + return nil +} + +// Fsck performs a plugin filesystem check. +func (p *ticketVotePlugin) Fsck() error { + log.Tracef("ticketvote Fsck") + + return nil +} + +// Setup performs any plugin setup work that needs to be done. +func (p *ticketVotePlugin) Setup() error { + log.Tracef("ticketvote Setup") + + return nil +} + +func TicketVotePluginNew(id *identity.FullIdentity, backend *tlogbe.Tlogbe) (*ticketVotePlugin, error) { + return &ticketVotePlugin{ + id: id, + backend: backend, + }, nil +} diff --git a/politeiad/log.go b/politeiad/log.go index 9f9abc45f..14bf69cd5 100644 --- a/politeiad/log.go +++ b/politeiad/log.go @@ -11,7 +11,7 @@ import ( "github.com/decred/politeia/politeiad/backend/gitbe" "github.com/decred/politeia/politeiad/backend/tlogbe" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugin/comments" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" "github.com/decred/politeia/politeiad/cache/cockroachdb" "github.com/decred/slog" @@ -45,12 +45,12 @@ var ( // application shutdown. logRotator *rotator.Rotator - log = backendLog.Logger("POLI") - gitbeLog = backendLog.Logger("GITB") - tlogbeLog = backendLog.Logger("TLOG") - cacheLog = backendLog.Logger("CACH") - storeLog = backendLog.Logger("STOR") - pluginLog = backendLog.Logger("PLGN") + log = backendLog.Logger("POLI") + gitbeLog = backendLog.Logger("GITB") + tlogbeLog = backendLog.Logger("TLOG") + cacheLog = backendLog.Logger("CACH") + storeLog = backendLog.Logger("STOR") + pluginsLog = backendLog.Logger("PLGN") ) // Initialize package-global logger variables. @@ -59,7 +59,7 @@ func init() { gitbe.UseLogger(gitbeLog) tlogbe.UseLogger(tlogbeLog) filesystem.UseLogger(storeLog) - comments.UseLogger(pluginLog) + plugins.UseLogger(pluginsLog) } // subsystemLoggers maps each subsystem identifier to its associated logger. @@ -69,7 +69,7 @@ var subsystemLoggers = map[string]slog.Logger{ "TLOG": tlogbeLog, "CACH": cacheLog, "STOR": storeLog, - "PLGN": pluginLog, + "PLGN": pluginsLog, } // initLogRotator initializes the logging rotater to write logs to logFile and From d6d9f2246f39d70bd3d5fa2487af172dabbb1c21 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 30 Jul 2020 11:30:59 -0600 Subject: [PATCH 008/449] setup dcrdata plugin --- plugins/dcrdata/dcrdata.go | 53 +++++++ politeiad/backend/tlogbe/plugins/comments.go | 12 +- politeiad/backend/tlogbe/plugins/dcrdata.go | 135 ++++++++++++++++++ .../backend/tlogbe/plugins/ticketvote.go | 8 ++ 4 files changed, 206 insertions(+), 2 deletions(-) create mode 100644 plugins/dcrdata/dcrdata.go create mode 100644 politeiad/backend/tlogbe/plugins/dcrdata.go diff --git a/plugins/dcrdata/dcrdata.go b/plugins/dcrdata/dcrdata.go new file mode 100644 index 000000000..75e7c0d02 --- /dev/null +++ b/plugins/dcrdata/dcrdata.go @@ -0,0 +1,53 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package dcrdata + +import "encoding/json" + +const ( + Version uint32 = 1 + ID = "dcrdata" + + // Plugin commands + CmdBestBlock = "bestblock" +) + +// BestBlock requests the best block. +type BestBlock struct{} + +// EncodeBestBlock encodes an BestBlock into a JSON byte slice. +func EncodeBestBlock(bb BestBlock) ([]byte, error) { + return json.Marshal(bb) +} + +// DecodeBestBlock decodes a JSON byte slice into a BestBlock. +func DecodeBestBlock(payload []byte) (*BestBlock, error) { + var bb BestBlock + err := json.Unmarshal(payload, &bb) + if err != nil { + return nil, err + } + return &bb, nil +} + +// BestBlockReply is the reply to the BestBlock command. +type BestBlockReply struct { + BestBlock uint64 `json:"bestblock"` +} + +// EncodeBestBlockReply encodes an BestBlockReply into a JSON byte slice. +func EncodeBestBlockReply(bbr BestBlockReply) ([]byte, error) { + return json.Marshal(bbr) +} + +// DecodeBestBlockReply decodes a JSON byte slice into a BestBlockReply. +func DecodeBestBlockReply(payload []byte) (*BestBlockReply, error) { + var bbr BestBlockReply + err := json.Unmarshal(payload, &bbr) + if err != nil { + return nil, err + } + return &bbr, nil +} diff --git a/politeiad/backend/tlogbe/plugins/comments.go b/politeiad/backend/tlogbe/plugins/comments.go index db484ea7a..e19ec4a80 100644 --- a/politeiad/backend/tlogbe/plugins/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments.go @@ -1409,6 +1409,8 @@ func (p *commentsPlugin) cmdProofs(payload string) (string, error) { } // Cmd executes a plugin command. +// +// This function satisfies the Plugin interface. func (p *commentsPlugin) Cmd(cmd, payload string) (string, error) { log.Tracef("comments Cmd: %v", cmd) @@ -1437,12 +1439,16 @@ func (p *commentsPlugin) Cmd(cmd, payload string) (string, error) { } // Hook executes a plugin hook. +// +// This function satisfies the Plugin interface. func (p *commentsPlugin) Hook(h HookT, payload string) error { log.Tracef("comments Hook: %v", Hook[h]) return nil } // Fsck performs a plugin filesystem check. +// +// This function satisfies the Plugin interface. func (p *commentsPlugin) Fsck() error { log.Tracef("comments Fsck") @@ -1452,16 +1458,18 @@ func (p *commentsPlugin) Fsck() error { } // Setup performs any plugin setup work that needs to be done. +// +// This function satisfies the Plugin interface. func (p *commentsPlugin) Setup() error { log.Tracef("comments Setup") return nil } // CommentsPluginNew returns a new comments plugin. -func CommentsPluginNew(id *identity.FullIdentity, backend *tlogbe.Tlogbe) (*commentsPlugin, error) { +func CommentsPluginNew(id *identity.FullIdentity, backend *tlogbe.Tlogbe) *commentsPlugin { return &commentsPlugin{ id: id, backend: backend, mutexes: make(map[string]*sync.Mutex), - }, nil + } } diff --git a/politeiad/backend/tlogbe/plugins/dcrdata.go b/politeiad/backend/tlogbe/plugins/dcrdata.go new file mode 100644 index 000000000..2d526efc3 --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/dcrdata.go @@ -0,0 +1,135 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package plugins + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "sync" + "time" + + v4 "github.com/decred/dcrdata/api/types/v4" + "github.com/decred/politeia/plugins/dcrdata" +) + +const ( + // Dcrdata routes + routeBestBlock = "/api/block/best" +) + +var ( + _ Plugin = (*dcrdataPlugin)(nil) +) + +// dcrdataplugin satisfies the Plugin interface. +type dcrdataPlugin struct { + sync.Mutex + host string + client *http.Client + bestBlock uint64 +} + +func (p *dcrdataPlugin) bestBlockGet() uint64 { + p.Lock() + defer p.Unlock() + + return p.bestBlock +} + +// TODO move this to util +// bestBlockHTTP fetches the best block from the dcrdata API. +func (p *dcrdataPlugin) bestBlockHTTP() (*v4.BlockDataBasic, error) { + url := p.host + routeBestBlock + + log.Tracef("dcrdata bestBlock: %v", url) + + r, err := p.client.Get(url) + log.Debugf("http connecting to %v", url) + if err != nil { + return nil, err + } + defer r.Body.Close() + + if r.StatusCode != http.StatusOK { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + return nil, fmt.Errorf("dcrdata error: %v %v %v", + r.StatusCode, url, err) + } + return nil, fmt.Errorf("dcrdata error: %v %v %s", + r.StatusCode, url, body) + } + + var bdb v4.BlockDataBasic + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&bdb); err != nil { + return nil, err + } + + return &bdb, nil +} + +// Cmd executes a plugin command. +// +// This function satisfies the Plugin interface. +func (p *dcrdataPlugin) Cmd(cmd, payload string) (string, error) { + log.Tracef("dcrdata Cmd: %v", cmd) + + switch cmd { + case dcrdata.CmdBestBlock: + return p.cmdBestBlock(payload) + } + + return "", ErrInvalidPluginCmd +} + +// Hook executes a plugin hook. +// +// This function satisfies the Plugin interface. +func (p *dcrdataPlugin) Hook(h HookT, payload string) error { + log.Tracef("dcrdata Hook: %v %v", h, payload) + + return nil +} + +// Fsck performs a plugin filesystem check. +// +// This function satisfies the Plugin interface. +func (p *dcrdataPlugin) Fsck() error { + log.Tracef("dcrdata Fsck") + + return nil +} + +// Setup performs any plugin setup work that needs to be done. +// +// This function satisfies the Plugin interface. +func (p *dcrdataPlugin) Setup() error { + log.Tracef("dcrdata Setup") + + return nil +} + +func DcrdataPluginNew(dcrdataHost string) *dcrdataPlugin { + // Setup http client + client := &http.Client{ + Timeout: 1 * time.Minute, + Transport: &http.Transport{ + IdleConnTimeout: 1 * time.Minute, + ResponseHeaderTimeout: 1 * time.Minute, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: false, + }, + }, + } + + return &dcrdataPlugin{ + host: dcrdataHost, + client: client, + } +} diff --git a/politeiad/backend/tlogbe/plugins/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote.go index fddd6b630..6c4d2e093 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote.go @@ -69,6 +69,8 @@ func (p *ticketVotePlugin) cmdInventory(payload string) (string, error) { } // Cmd executes a plugin command. +// +// This function satisfies the Plugin interface. func (p *ticketVotePlugin) Cmd(cmd, payload string) (string, error) { log.Tracef("ticketvote Cmd: %v", cmd) @@ -95,6 +97,8 @@ func (p *ticketVotePlugin) Cmd(cmd, payload string) (string, error) { } // Hook executes a plugin hook. +// +// This function satisfies the Plugin interface. func (p *ticketVotePlugin) Hook(h HookT, payload string) error { log.Tracef("ticketvote Hook: %v %v", h, payload) @@ -102,6 +106,8 @@ func (p *ticketVotePlugin) Hook(h HookT, payload string) error { } // Fsck performs a plugin filesystem check. +// +// This function satisfies the Plugin interface. func (p *ticketVotePlugin) Fsck() error { log.Tracef("ticketvote Fsck") @@ -109,6 +115,8 @@ func (p *ticketVotePlugin) Fsck() error { } // Setup performs any plugin setup work that needs to be done. +// +// This function satisfies the Plugin interface. func (p *ticketVotePlugin) Setup() error { log.Tracef("ticketvote Setup") From 518749d06255063cc52f86ad1b49d50671852f00 Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 31 Jul 2020 14:39:57 -0600 Subject: [PATCH 009/449] dcrdata and ticketvote plugin work --- plugins/dcrdata/dcrdata.go | 2 +- politeiad/backend/backend.go | 6 - politeiad/backend/tlogbe/plugins/dcrdata.go | 118 +++++++++++++++++- .../backend/tlogbe/plugins/ticketvote.go | 2 + wsdcrdata/wsdcrdata.go | 2 +- 5 files changed, 118 insertions(+), 12 deletions(-) diff --git a/plugins/dcrdata/dcrdata.go b/plugins/dcrdata/dcrdata.go index 75e7c0d02..1f16c68a4 100644 --- a/plugins/dcrdata/dcrdata.go +++ b/plugins/dcrdata/dcrdata.go @@ -34,7 +34,7 @@ func DecodeBestBlock(payload []byte) (*BestBlock, error) { // BestBlockReply is the reply to the BestBlock command. type BestBlockReply struct { - BestBlock uint64 `json:"bestblock"` + BestBlock uint32 `json:"bestblock"` } // EncodeBestBlockReply encodes an BestBlockReply into a JSON byte slice. diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index 78294a1c7..aac385f3f 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -331,12 +331,6 @@ type Backend interface { // Check if a vetted record exists VettedExists([]byte) bool - // Get all unvetted record tokens - UnvettedTokens() ([][]byte, error) - - // Get all vetted record tokens - VettedTokens() ([][]byte, error) - // Get unvetted record GetUnvetted([]byte, string) (*Record, error) diff --git a/politeiad/backend/tlogbe/plugins/dcrdata.go b/politeiad/backend/tlogbe/plugins/dcrdata.go index 2d526efc3..7b1b4e7e9 100644 --- a/politeiad/backend/tlogbe/plugins/dcrdata.go +++ b/politeiad/backend/tlogbe/plugins/dcrdata.go @@ -14,7 +14,10 @@ import ( "time" v4 "github.com/decred/dcrdata/api/types/v4" + exptypes "github.com/decred/dcrdata/explorer/types/v2" + pstypes "github.com/decred/dcrdata/pubsub/types/v3" "github.com/decred/politeia/plugins/dcrdata" + "github.com/decred/politeia/wsdcrdata" ) const ( @@ -30,18 +33,25 @@ var ( type dcrdataPlugin struct { sync.Mutex host string - client *http.Client - bestBlock uint64 + client *http.Client // HTTP client + ws *wsdcrdata.Client // Websocket client + bestBlock uint32 } -func (p *dcrdataPlugin) bestBlockGet() uint64 { +func (p *dcrdataPlugin) bestBlockGet() uint32 { p.Lock() defer p.Unlock() return p.bestBlock } -// TODO move this to util +func (p *dcrdataPlugin) bestBlockSet(bb uint32) { + p.Lock() + defer p.Unlock() + + p.bestBlock = bb +} + // bestBlockHTTP fetches the best block from the dcrdata API. func (p *dcrdataPlugin) bestBlockHTTP() (*v4.BlockDataBasic, error) { url := p.host + routeBestBlock @@ -74,6 +84,90 @@ func (p *dcrdataPlugin) bestBlockHTTP() (*v4.BlockDataBasic, error) { return &bdb, nil } +func (p *dcrdataPlugin) monitorWebsocket() { + defer func() { + log.Infof("Dcrdata websocket closed") + }() + + // Setup messages channel + receiver, err := p.ws.Receive() + if err == wsdcrdata.ErrShutdown { + return + } else if err != nil { + log.Errorf("dcrdata Receive: %v", err) + } + + for { + // Monitor for a new message + msg, ok := <-receiver + if !ok { + log.Infof("Dcrdata websocket channel closed. Will reconnect.") + goto reconnect + } + + // Handle new message + switch m := msg.Message.(type) { + case *exptypes.WebsocketBlock: + log.Debugf("Dcrdata websocket new block %v", m.Block.Height) + p.bestBlockSet(uint32(m.Block.Height)) + + case *pstypes.HangUp: + log.Infof("Dcrdata websocket has hung up. Will reconnect.") + goto reconnect + + case int: + // Ping messages are of type int + + default: + log.Errorf("Dcrdata websocket unhandled msg %v", msg) + } + + // Check for next message + continue + + reconnect: + // Connection was closed for some reason. Set the best block + // to 0 to indicate that its stale then reconnect to dcrdata. + p.bestBlockSet(0) + err = p.ws.Reconnect() + if err == wsdcrdata.ErrShutdown { + return + } else if err != nil { + log.Errorf("dcrdata Reconnect: %v", err) + continue + } + + // Setup a new messages channel using the new connection. + receiver, err = p.ws.Receive() + if err == wsdcrdata.ErrShutdown { + return + } else if err != nil { + log.Errorf("dcrdata Receive: %v", err) + continue + } + + log.Infof("Successfully reconnected dcrdata websocket") + } +} + +func (p *dcrdataPlugin) cmdBestBlock(payload string) (string, error) { + log.Tracef("dcrdata cmdBestBlock") + + // Payload is empty. No need to decode it. + + bb := p.bestBlockGet() + if bb == 0 { + // No cached best block means the websocket connection is down. + // Get the best block from the dcrdata API. + block, err := p.bestBlockHTTP() + if err != nil { + return "", err + } + } + + return "", nil +} + // Cmd executes a plugin command. // // This function satisfies the Plugin interface. @@ -112,6 +206,15 @@ func (p *dcrdataPlugin) Fsck() error { func (p *dcrdataPlugin) Setup() error { log.Tracef("dcrdata Setup") + // Setup websocket subscriptions + err := p.ws.NewBlockSubscribe() + if err != nil { + return err + } + + // Monitor websocket connection in a new go routine + go p.monitorWebsocket() + return nil } @@ -128,8 +231,15 @@ func DcrdataPluginNew(dcrdataHost string) *dcrdataPlugin { }, } + // Setup websocket client + ws, err := wsdcrdata.New(dcrdataHost) + if err != nil { + // TODO reconnect logic + } + return &dcrdataPlugin{ host: dcrdataHost, client: client, + ws: ws, } } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote.go index 6c4d2e093..306392aff 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote.go @@ -120,6 +120,8 @@ func (p *ticketVotePlugin) Fsck() error { func (p *ticketVotePlugin) Setup() error { log.Tracef("ticketvote Setup") + // Ensure dcrdata plugin has been registered + return nil } diff --git a/wsdcrdata/wsdcrdata.go b/wsdcrdata/wsdcrdata.go index 2c16c9f6a..d1ed54564 100644 --- a/wsdcrdata/wsdcrdata.go +++ b/wsdcrdata/wsdcrdata.go @@ -317,7 +317,7 @@ func (c *Client) Close() error { return c.client.Close() } -// New return a new Client. +// New returns a new Client. func New(dcrdataURL string) (*Client, error) { client, err := newDcrdataWSClient(dcrdataURL) if err != nil { From c1ad9d5c9d2d1b16dde1f1892067da715faf3ec5 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 2 Aug 2020 19:53:52 -0600 Subject: [PATCH 010/449] return del comment reason --- plugins/comments/comments.go | 3 +- plugins/ticketvote/ticketvote.go | 9 +- politeiad/backend/tlogbe/plugins/comments.go | 266 ++++++++++++------ politeiad/backend/tlogbe/plugins/dcrdata.go | 1 + .../backend/tlogbe/plugins/ticketvote.go | 37 +++ 5 files changed, 228 insertions(+), 88 deletions(-) diff --git a/plugins/comments/comments.go b/plugins/comments/comments.go index 3f80a1061..b45bcc277 100644 --- a/plugins/comments/comments.go +++ b/plugins/comments/comments.go @@ -90,6 +90,7 @@ type Comment struct { Timestamp int64 `json:"timestamp"` // UNIX timestamp of last edit Score int64 `json:"score"` // Vote score Deleted bool `json:"deleted"` // Comment has been deleted + Reason string `json:"reason"` // Reason for deletion } // New creates a new comment. A parent ID of 0 indicates that the comment is @@ -233,7 +234,7 @@ func DecodeDelReply(payload []byte) (*DelReply, error) { // Get returns the latest version of the comments for the provided comment IDs. // An error is not returned if a comment is not found for one or more of the -// comment IDs. Those entries will simply not be included in the reply. +// comment IDs. Those entries will simply not be included in the reply. type Get struct { Token string `json:"token"` CommentIDs []uint32 `json:"commentids"` diff --git a/plugins/ticketvote/ticketvote.go b/plugins/ticketvote/ticketvote.go index 3e9f0df91..75bbe7972 100644 --- a/plugins/ticketvote/ticketvote.go +++ b/plugins/ticketvote/ticketvote.go @@ -177,10 +177,10 @@ func DecodeStartVote(payload []byte) (*Start, error) { // StartReply is the reply to the Start command. type StartReply struct { - StartBlockHeight uint32 `json:"startblockheight"` // Block height - StartBlockHash string `json:"startblockhash"` // Block hash - EndBlockHeight uint32 `json:"endblockheight"` // Height of vote end - EligibleTickets []string `json:"eligibletickets"` // Valid voting tickets + StartBlockHeight uint32 `json:"startblockheight"` + StartBlockHash string `json:"startblockhash"` + EndBlockHeight uint32 `json:"endblockheight"` + EligibleTickets []string `json:"eligibletickets"` } // EncodeStartReply encodes a StartReply into a JSON byte slice. @@ -201,6 +201,7 @@ func DecodeStartReplyVote(payload []byte) (*StartReply, error) { // StartRunoff starts a runoff vote between the provided submissions. Each // submission is required to have its own Authorize and Start. type StartRunoff struct { + Token string `json:"token"` // RFP token Authorizations []Authorize `json:"authorizations"` Votes []Start `json:"votes"` } diff --git a/politeiad/backend/tlogbe/plugins/comments.go b/politeiad/backend/tlogbe/plugins/comments.go index e19ec4a80..aac98db25 100644 --- a/politeiad/backend/tlogbe/plugins/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments.go @@ -71,6 +71,10 @@ type commentAdd struct { } // commentDel is the structure that is saved to disk when a comment is deleted. +// Some additional fields like ParentID and AuthorPublicKey are required to be +// saved by the server because all comment add records will be deleted and +// the client will still require those fields to properly display the comment +// hierarchy. type commentDel struct { // Data generated by client Token string `json:"token"` // Record token @@ -80,8 +84,10 @@ type commentDel struct { Signature string `json:"signature"` // Signature of Token+CommentID+Reason // Metadata generated by server - Receipt string `json:"receipt"` // Server signature of client signature - Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + ParentID uint32 `json:"parentid"` // Parent comment ID + AuthorPublicKey string `json:"authorpublickey"` // Author public key + Receipt string `json:"receipt"` // Server sig of client sig + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp } // commentVote is the structure that is saved to disk when a comment is voted @@ -280,6 +286,44 @@ func convertCommentAddFromBlobEntry(be store.BlobEntry) (*commentAdd, error) { return &c, nil } +func convertCommentDelFromBlobEntry(be store.BlobEntry) (*commentDel, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorCommentDel { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorCommentDel) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + hash, err := hex.DecodeString(be.Hash) + if err != nil { + return nil, fmt.Errorf("decode hash: %v", err) + } + if !bytes.Equal(util.Digest(b), hash) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), hash) + } + var c commentDel + err = json.Unmarshal(b, &c) + if err != nil { + return nil, fmt.Errorf("unmarshal index: %v", err) + } + + return &c, nil +} + func convertIndexFromBlobEntry(be store.BlobEntry) (*index, error) { // Decode and validate data hint b, err := base64.StdEncoding.DecodeString(be.DataHint) @@ -332,6 +376,25 @@ func convertCommentFromCommentAdd(ca commentAdd) comments.Comment { Timestamp: ca.Timestamp, Score: 0, Deleted: false, + Reason: "", + } +} + +func convertCommentFromCommentDel(cd commentDel) comments.Comment { + // Score needs to be filled in seperately + return comments.Comment{ + Token: cd.Token, + ParentID: cd.ParentID, + Comment: "", + PublicKey: cd.AuthorPublicKey, + Signature: "", + CommentID: cd.CommentID, + Version: 0, + Receipt: "", + Timestamp: cd.Timestamp, + Score: 0, + Deleted: true, + Reason: cd.Reason, } } @@ -428,6 +491,41 @@ func commentDelSave(client *tlogbe.PluginClient, c commentDel) ([]byte, error) { return merkles[0], nil } +func commentDels(client *tlogbe.PluginClient, merkleHashes [][]byte) ([]commentDel, error) { + // Retrieve blobs + blobs, err := client.BlobsByMerkleHash(merkleHashes) + if err != nil { + return nil, err + } + if len(blobs) != len(merkleHashes) { + notFound := make([]string, 0, len(blobs)) + for _, v := range merkleHashes { + m := hex.EncodeToString(v) + _, ok := blobs[m] + if !ok { + notFound = append(notFound, m) + } + } + return nil, fmt.Errorf("blobs not found: %v", notFound) + } + + // Decode blobs + dels := make([]commentDel, 0, len(blobs)) + for _, v := range blobs { + be, err := store.Deblob(v) + if err != nil { + return nil, err + } + c, err := convertCommentDelFromBlobEntry(*be) + if err != nil { + return nil, err + } + dels = append(dels, *c) + } + + return dels, nil +} + func commentVoteSave(client *tlogbe.PluginClient, c commentVote) ([]byte, error) { // Prepare blob be, err := convertBlobEntryFromCommentVote(c) @@ -592,19 +690,15 @@ func commentExists(idx index, commentID uint32) bool { // the responsibility of the caller to ensure a comment is returned for each of // the provided comment IDs. The comments index that was looked up during this // process is also returned. -func commentsLatest(client *tlogbe.PluginClient, commentIDs []uint32) (map[uint32]comments.Comment, *index, error) { - // Get comments index - idx, err := indexLatest(client) - if err != nil { - return nil, nil, fmt.Errorf("indexLatest: %v", err) - } - - // Aggregate merkle hashes of the comment add records that need to - // be looked up. If the comment has been deleted then there is - // nothing to look up. +func commentsLatest(client *tlogbe.PluginClient, idx index, commentIDs []uint32) (map[uint32]comments.Comment, error) { + // Aggregate the merkle hashes for all records that need to be + // looked up. If a comment has been deleted then the only record + // that will still exist is the comment del record. If the comment + // has not been deleted then the comment add record will need to be + // retrieved for the latest version of the comment. var ( - merkles = make([][]byte, 0, len(commentIDs)) - dels = make([]uint32, 0, len(commentIDs)) + merklesAdd = make([][]byte, 0, len(commentIDs)) + merklesDel = make([][]byte, 0, len(commentIDs)) ) for _, v := range commentIDs { cidx, ok := idx.Comments[v] @@ -612,25 +706,36 @@ func commentsLatest(client *tlogbe.PluginClient, commentIDs []uint32) (map[uint3 // Comment does not exist continue } + + // Comment del record if cidx.Del != nil { - // Comment has been deleted - dels = append(dels, v) + merklesDel = append(merklesDel, cidx.Del) continue } - // Save the merkle hash for the latest version + // Comment add record version := commentVersionLatest(cidx) - merkles = append(merkles, cidx.Adds[version]) + merklesAdd = append(merklesAdd, cidx.Adds[version]) } // Get comment add records - adds, err := commentAdds(client, merkles) + adds, err := commentAdds(client, merklesAdd) if err != nil { - return nil, nil, fmt.Errorf("commentAdds: %v", err) + return nil, fmt.Errorf("commentAdds: %v", err) } - if len(adds) != len(merkles) { - return nil, nil, fmt.Errorf("wrong comment adds count; got %v, want %v", - len(adds), len(merkles)) + if len(adds) != len(merklesAdd) { + return nil, fmt.Errorf("wrong comment adds count; got %v, want %v", + len(adds), len(merklesAdd)) + } + + // Get comment del records + dels, err := commentDels(client, merklesDel) + if err != nil { + return nil, fmt.Errorf("commentDels: %v", err) + } + if len(dels) != len(merklesDel) { + return nil, fmt.Errorf("wrong comment dels count; got %v, want %v", + len(dels), len(merklesDel)) } // Prepare comments @@ -640,17 +745,12 @@ func commentsLatest(client *tlogbe.PluginClient, commentIDs []uint32) (map[uint3 c.Score = idx.Comments[c.CommentID].Score cs[v.CommentID] = c } - for _, commentID := range dels { - score := idx.Comments[commentID].Score - cs[commentID] = comments.Comment{ - Token: hex.EncodeToString(client.Token), - CommentID: commentID, - Score: score, - Deleted: true, - } + for _, v := range dels { + c := convertCommentFromCommentDel(v) + cs[v.CommentID] = c } - return cs, idx, nil + return cs, nil } // This function must be called WITH the record lock held. @@ -782,8 +882,14 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { // This function must be called WITH the record lock held. func (p *commentsPlugin) edit(client *tlogbe.PluginClient, e comments.Edit, encrypt bool) (*comments.EditReply, error) { + // Get comments index + idx, err := indexLatest(client) + if err != nil { + return nil, fmt.Errorf("indexLatest: %v", err) + } + // Get the existing comment - cs, idx, err := commentsLatest(client, []uint32{e.CommentID}) + cs, err := commentsLatest(client, *idx, []uint32{e.CommentID}) if err != nil { return nil, fmt.Errorf("commentsLatest %v: %v", e.CommentID, err) } @@ -919,11 +1025,15 @@ func (p *commentsPlugin) del(client *tlogbe.PluginClient, d comments.Del) (*comm // Get comments index idx, err := indexLatest(client) if err != nil { - return nil, err + return nil, fmt.Errorf("indexLatest: %v", err) } - // Ensure comment exists - cidx, ok := idx.Comments[d.CommentID] + // Get comment + cs, err := commentsLatest(client, *idx, []uint32{d.CommentID}) + if err != nil { + return nil, fmt.Errorf("commentsLatest %v: %v", d.CommentID, err) + } + comment, ok := cs[d.CommentID] if !ok { return nil, comments.UserError{ ErrorCode: comments.ErrorStatusCommentNotFound, @@ -933,13 +1043,15 @@ func (p *commentsPlugin) del(client *tlogbe.PluginClient, d comments.Del) (*comm // Save delete record receipt := p.id.SignMessage([]byte(d.Signature)) cd := commentDel{ - Token: d.Token, - CommentID: d.CommentID, - Reason: d.Reason, - PublicKey: d.PublicKey, - Signature: d.Signature, - Receipt: hex.EncodeToString(receipt[:]), - Timestamp: time.Now().Unix(), + Token: d.Token, + CommentID: d.CommentID, + Reason: d.Reason, + PublicKey: d.PublicKey, + Signature: d.Signature, + ParentID: comment.ParentID, + AuthorPublicKey: comment.PublicKey, + Receipt: hex.EncodeToString(receipt[:]), + Timestamp: time.Now().Unix(), } merkleHash, err := commentDelSave(client, cd) if err != nil { @@ -947,6 +1059,10 @@ func (p *commentsPlugin) del(client *tlogbe.PluginClient, d comments.Del) (*comm } // Update index + cidx, ok := idx.Comments[d.CommentID] + if !ok { + return nil, fmt.Errorf("comment not found in index: %v", d.CommentID) + } cidx.Del = merkleHash idx.Comments[d.CommentID] = cidx @@ -1052,8 +1168,14 @@ func (p *commentsPlugin) cmdGet(payload string) (string, error) { return "", err } + // Get comments index + idx, err := indexLatest(client) + if err != nil { + return "", fmt.Errorf("indexLatest: %v", err) + } + // Get comments - cs, _, err := commentsLatest(client, g.CommentIDs) + cs, err := commentsLatest(client, *idx, g.CommentIDs) if err != nil { return "", fmt.Errorf("commentsLatest: %v", err) } @@ -1102,50 +1224,22 @@ func (p *commentsPlugin) cmdGetAll(payload string) (string, error) { return "", fmt.Errorf("indexLatest: %v", err) } - // Aggregate merkle hashes of the comment add records that need to - // be looked up. If the comment has been deleted then there is - // nothing to look up. - var ( - merkles = make([][]byte, 0, len(idx.Comments)) - dels = make([]uint32, 0, len(idx.Comments)) - ) - for k, v := range idx.Comments { - if v.Del != nil { - // Comment has been deleted - dels = append(dels, k) - continue - } - - // Save the merkle hash for the latest version - version := commentVersionLatest(v) - merkles = append(merkles, v.Adds[version]) + // Compile comment IDs + commentIDs := make([]uint32, 0, len(idx.Comments)) + for k := range idx.Comments { + commentIDs = append(commentIDs, k) } - // Get comment add records - adds, err := commentAdds(client, merkles) + // Get comments + c, err := commentsLatest(client, *idx, commentIDs) if err != nil { - return "", fmt.Errorf("commentAdds: %v", err) - } - if len(adds) != len(merkles) { - return "", fmt.Errorf("wrong comment adds count; got %v, want %v", - len(adds), len(merkles)) + return "", fmt.Errorf("commentsLatest: %v", err) } - // Prepare comments - cs := make([]comments.Comment, 0, len(idx.Comments)) - for _, v := range adds { - c := convertCommentFromCommentAdd(v) - c.Score = idx.Comments[c.CommentID].Score - cs = append(cs, c) - } - for _, commentID := range dels { - score := idx.Comments[commentID].Score - cs = append(cs, comments.Comment{ - Token: hex.EncodeToString(client.Token), - CommentID: commentID, - Score: score, - Deleted: true, - }) + // Convert comments from a map to a slice + cs := make([]comments.Comment, 0, len(c)) + for _, v := range c { + cs = append(cs, v) } // Order comments by comment ID @@ -1204,6 +1298,12 @@ func (p *commentsPlugin) cmdGetVersion(payload string) (string, error) { ErrorCode: comments.ErrorStatusCommentNotFound, } } + if cidx.Del != nil { + return "", comments.UserError{ + ErrorCode: comments.ErrorStatusCommentNotFound, + ErrorContext: []string{"comment has been deleted"}, + } + } merkleHash, ok := cidx.Adds[gv.Version] if !ok { e := fmt.Sprintf("comment %v does not have version %v", diff --git a/politeiad/backend/tlogbe/plugins/dcrdata.go b/politeiad/backend/tlogbe/plugins/dcrdata.go index 7b1b4e7e9..3e13063ef 100644 --- a/politeiad/backend/tlogbe/plugins/dcrdata.go +++ b/politeiad/backend/tlogbe/plugins/dcrdata.go @@ -163,6 +163,7 @@ func (p *dcrdataPlugin) cmdBestBlock(payload string) (string, error) { if err != nil { return "", err } + _ = block } return "", nil diff --git a/politeiad/backend/tlogbe/plugins/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote.go index 306392aff..0b0801241 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote.go @@ -20,9 +20,46 @@ type ticketVotePlugin struct { backend *tlogbe.Tlogbe } +// authorize is the structure that is saved to disk when a vote is authorized +// or a previous authorizatio is revoked. +type authorize struct { + Token string `json:"token"` // Record token + Version uint32 `json:"version"` // Record version + Action string `json:"action"` // Authorize or revoke + PublicKey string `json:"publickey"` // Public key used for signature + Signature string `json:"signature"` // Signature of token+version+action + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + Receipt string `json:"receipt"` // Server signature of client signature +} + +// start is the structure that is saved to disk when a vote is started. +type start struct { + Vote ticketvote.Vote `json:"vote"` + PublicKey string `json:"publickey"` + StartBlockHeight uint32 `json:"startblockheight"` + StartBlockHash string `json:"startblockhash"` + EndBlockHeight uint32 `json:"endblockheight"` + EligibleTickets []string `json:"eligibletickets"` // Ticket hashes + + // Signature is a signature of the SHA256 digest of the JSON + // encoded Vote struct. + Signature string `json:"signature"` +} + func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { log.Tracef("ticketvote cmdAuthorize: %v", payload) + a, err := ticketvote.DecodeAuthorize([]byte(payload)) + if err != nil { + return "", err + } + _ = a + + // Ensure record exists + // Verify action + // Verify record state + // Prepare + return "", nil } From cd7d0cedf6e9eb105abb7d2708abad696f30e77b Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 3 Aug 2020 12:02:21 -0600 Subject: [PATCH 011/449] finish dcrdata plugin --- plugins/dcrdata/dcrdata.go | 20 ++- politeiad/backend/tlogbe/plugins/dcrdata.go | 186 ++++++++++++++------ 2 files changed, 151 insertions(+), 55 deletions(-) diff --git a/plugins/dcrdata/dcrdata.go b/plugins/dcrdata/dcrdata.go index 1f16c68a4..180e09861 100644 --- a/plugins/dcrdata/dcrdata.go +++ b/plugins/dcrdata/dcrdata.go @@ -6,15 +6,30 @@ package dcrdata import "encoding/json" +type StatusT int + const ( Version uint32 = 1 ID = "dcrdata" // Plugin commands CmdBestBlock = "bestblock" + + // Dcrdata connection statuses. + // + // Some commands will return cached results with the connection + // status when dcrdata cannot be reached. It is the callers + // responsibilty to determine the correct course of action when + // dcrdata cannot be reached. + StatusInvalid StatusT = 0 + StatusConnected StatusT = 1 + StatusDisconnected StatusT = 2 ) -// BestBlock requests the best block. +// BestBlock requests the best block. If dcrdata cannot be reached then the +// most recent cached best block will be returned along with a status of +// StatusDisconnected. It is the callers responsibility to determine if the +// stale best block should be used. type BestBlock struct{} // EncodeBestBlock encodes an BestBlock into a JSON byte slice. @@ -34,7 +49,8 @@ func DecodeBestBlock(payload []byte) (*BestBlock, error) { // BestBlockReply is the reply to the BestBlock command. type BestBlockReply struct { - BestBlock uint32 `json:"bestblock"` + Status StatusT `json:"status"` + BestBlock uint32 `json:"bestblock"` } // EncodeBestBlockReply encodes an BestBlockReply into a JSON byte slice. diff --git a/politeiad/backend/tlogbe/plugins/dcrdata.go b/politeiad/backend/tlogbe/plugins/dcrdata.go index 3e13063ef..9a236b32e 100644 --- a/politeiad/backend/tlogbe/plugins/dcrdata.go +++ b/politeiad/backend/tlogbe/plugins/dcrdata.go @@ -32,10 +32,17 @@ var ( // dcrdataplugin satisfies the Plugin interface. type dcrdataPlugin struct { sync.Mutex - host string - client *http.Client // HTTP client - ws *wsdcrdata.Client // Websocket client - bestBlock uint32 + host string + client *http.Client // HTTP client + ws *wsdcrdata.Client // Websocket client + + // bestBlock is the cached best block. This field is kept up to + // date by the websocket connection. If the websocket connection + // drops, the best block is marked as stale and is not marked as + // current again until the connection has been re-established and + // a new best block message is received. + bestBlock uint32 + bestBlockStale bool } func (p *dcrdataPlugin) bestBlockGet() uint32 { @@ -50,6 +57,21 @@ func (p *dcrdataPlugin) bestBlockSet(bb uint32) { defer p.Unlock() p.bestBlock = bb + p.bestBlockStale = false +} + +func (p *dcrdataPlugin) bestBlockSetStale() { + p.Lock() + defer p.Unlock() + + p.bestBlockStale = true +} + +func (p *dcrdataPlugin) bestBlockIsStale() bool { + p.Lock() + defer p.Unlock() + + return p.bestBlockStale } // bestBlockHTTP fetches the best block from the dcrdata API. @@ -84,31 +106,96 @@ func (p *dcrdataPlugin) bestBlockHTTP() (*v4.BlockDataBasic, error) { return &bdb, nil } -func (p *dcrdataPlugin) monitorWebsocket() { +// cmdBestBlock returns the best block. If the dcrdata websocket has been +// disconnected the best block will be fetched from the dcrdata API. If dcrdata +// cannot be reached then the most recent cached best block will be returned +// along with a status of StatusDisconnected. It is the callers responsibility +// to determine if the stale best block should be used. +func (p *dcrdataPlugin) cmdBestBlock(payload string) (string, error) { + log.Tracef("dcrdata cmdBestBlock") + + // Payload is empty. Nothing to decode. + + // Get the cached best block + bb := p.bestBlockGet() + var ( + fetch bool + stale uint32 + status = dcrdata.StatusConnected + ) + switch { + case bb == 0: + // No cached best block means that the best block has not been + // populated by the websocket yet. Fetch is manually. + fetch = true + case p.bestBlockIsStale(): + // The cached best block has been populated by the websocket, but + // the websocket is currently disconnected and the cached value + // is stale. Try to fetch the best block manually and only use + // the stale value if manually fetching it fails. + fetch = true + stale = bb + } + + // Fetch the best block manually if required + if fetch { + block, err := p.bestBlockHTTP() + switch { + case err == nil: + // We got the best block. Use it. + bb = block.Height + case stale != 0: + // Unable to fetch the best block manually. Use the stale + // value and mark the connection status as disconnected. + bb = stale + status = dcrdata.StatusDisconnected + default: + // Unable to fetch the best block manually and there is no + // stale cached value to return. + return "", err + } + } + + // Prepare reply + bbr := dcrdata.BestBlockReply{ + Status: status, + BestBlock: bb, + } + reply, err := dcrdata.EncodeBestBlockReply(bbr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *dcrdataPlugin) websocketMonitor() { defer func() { log.Infof("Dcrdata websocket closed") }() // Setup messages channel - receiver, err := p.ws.Receive() - if err == wsdcrdata.ErrShutdown { - return - } else if err != nil { - log.Errorf("dcrdata Receive: %v", err) - } + receiver := p.ws.Receive() for { // Monitor for a new message msg, ok := <-receiver if !ok { - log.Infof("Dcrdata websocket channel closed. Will reconnect.") + // Check if the websocket was shut down intentionally or was + // dropped unexpectedly. + if p.ws.Status() == wsdcrdata.StatusShutdown { + return + } + log.Infof("Dcrdata websocket connection unexpectedly dropped") goto reconnect } // Handle new message switch m := msg.Message.(type) { case *exptypes.WebsocketBlock: - log.Debugf("Dcrdata websocket new block %v", m.Block.Height) + log.Debugf("dcrdata WebsocketBlock: %v", m.Block.Height) + + // Update cached best block p.bestBlockSet(uint32(m.Block.Height)) case *pstypes.HangUp: @@ -119,54 +206,48 @@ func (p *dcrdataPlugin) monitorWebsocket() { // Ping messages are of type int default: - log.Errorf("Dcrdata websocket unhandled msg %v", msg) + log.Errorf("ws message of type %v unhandled: %v", + msg.EventId, m) } // Check for next message continue reconnect: - // Connection was closed for some reason. Set the best block - // to 0 to indicate that its stale then reconnect to dcrdata. - p.bestBlockSet(0) - err = p.ws.Reconnect() - if err == wsdcrdata.ErrShutdown { - return - } else if err != nil { - log.Errorf("dcrdata Reconnect: %v", err) - continue - } + // Mark cached best block as stale + p.bestBlockSetStale() + + // Reconnect + p.ws.Reconnect() // Setup a new messages channel using the new connection. - receiver, err = p.ws.Receive() - if err == wsdcrdata.ErrShutdown { - return - } else if err != nil { - log.Errorf("dcrdata Receive: %v", err) - continue - } + receiver = p.ws.Receive() log.Infof("Successfully reconnected dcrdata websocket") } } -func (p *dcrdataPlugin) cmdBestBlock(payload string) (string, error) { - log.Tracef("dcrdata cmdBestBlock") +func (p *dcrdataPlugin) websocketSetup() { + // Setup websocket subscriptions + var done bool + for !done { + // Best block + err := p.ws.NewBlockSubscribe() + if err != nil && err != wsdcrdata.ErrDuplicateSub { + log.Errorf("NewBlockSubscribe: %v", err) + goto reconnect + } - // Payload is empty. No need to decode it. + // All subscriptions setup + done = true + continue - bb := p.bestBlockGet() - if bb == 0 { - // No cached best block means the websocket connection is down. - // Get the best block from the dcrdata API. - block, err := p.bestBlockHTTP() - if err != nil { - return "", err - } - _ = block + reconnect: + p.ws.Reconnect() } - return "", nil + // Monitor websocket connection + go p.websocketMonitor() } // Cmd executes a plugin command. @@ -207,14 +288,11 @@ func (p *dcrdataPlugin) Fsck() error { func (p *dcrdataPlugin) Setup() error { log.Tracef("dcrdata Setup") - // Setup websocket subscriptions - err := p.ws.NewBlockSubscribe() - if err != nil { - return err - } - - // Monitor websocket connection in a new go routine - go p.monitorWebsocket() + // Setup dcrdata websocket subscriptions and monitoring. This is + // done in a go routine so setup will continue in the event that + // a dcrdata websocket connection was not able to be made during + // client initialization and reconnection attempts are required. + go p.websocketSetup() return nil } @@ -235,7 +313,9 @@ func DcrdataPluginNew(dcrdataHost string) *dcrdataPlugin { // Setup websocket client ws, err := wsdcrdata.New(dcrdataHost) if err != nil { - // TODO reconnect logic + // Continue even if a websocket connection was not able to be + // made. Reconnection attempts will be made in the plugin setup. + log.Errorf("wsdcrdata New: %v", err) } return &dcrdataPlugin{ From 7f8ba5de8b41504136e473c96e6e2e65600288a2 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 4 Aug 2020 07:54:08 -0600 Subject: [PATCH 012/449] move VerifySignature to util --- plugins/comments/comments.go | 2 +- plugins/ticketvote/ticketvote.go | 21 ++++-- politeiad/backend/tlogbe/plugins/comments.go | 66 ++++++++----------- .../backend/tlogbe/plugins/ticketvote.go | 58 ++++++++++++++-- util/signature.go | 65 ++++++++++++++++++ 5 files changed, 159 insertions(+), 53 deletions(-) create mode 100644 util/signature.go diff --git a/plugins/comments/comments.go b/plugins/comments/comments.go index b45bcc277..80c8aa766 100644 --- a/plugins/comments/comments.go +++ b/plugins/comments/comments.go @@ -74,7 +74,7 @@ type UserError struct { // Error satisfies the error interface. func (e UserError) Error() string { - return fmt.Sprintf("plugin error code: %v", e.ErrorCode) + return fmt.Sprintf("comments plugin error code: %v", e.ErrorCode) } // Comment represent a record comment. diff --git a/plugins/ticketvote/ticketvote.go b/plugins/ticketvote/ticketvote.go index 75bbe7972..a183f62e9 100644 --- a/plugins/ticketvote/ticketvote.go +++ b/plugins/ticketvote/ticketvote.go @@ -73,13 +73,20 @@ const ( // Error status codes // TODO change politeiavoter to use these error codes - ErrorStatusInvalid ErrorStatusT = 0 + ErrorStatusInvalid ErrorStatusT = 0 + ErrorStatusTokenInvalid ErrorStatusT = 1 + ErrorStatusPublicKeyInvalid ErrorStatusT = 2 + ErrorStatusSignatureInvalid ErrorStatusT = 3 + ErrorStatusRecordNotFound ErrorStatusT = 4 ) var ( // ErrorStatus contains human readable error statuses. ErrorStatus = map[ErrorStatusT]string{ - ErrorStatusInvalid: "invalid error status", + ErrorStatusInvalid: "invalid error status", + ErrorStatusTokenInvalid: "invalid token", + ErrorStatusPublicKeyInvalid: "invalid public key", + ErrorStatusSignatureInvalid: "invalid signature", } ) @@ -91,7 +98,7 @@ type UserError struct { // Error satisfies the error interface. func (e UserError) Error() string { - return fmt.Sprintf("plugin error code: %v", e.ErrorCode) + return fmt.Sprintf("ticketvote plugin error code: %v", e.ErrorCode) } // Authorize authorizes a ticket vote or revokes a previous authorization. @@ -226,7 +233,7 @@ type StartRunoffReply struct { StartBlockHeight uint32 `json:"startblockheight"` StartBlockHash string `json:"startblockhash"` EndBlockHeight uint32 `json:"endblockheight"` - EligibleTickets []string `json:"eligibletickets"` + EligibleTickets []string `json:"eligibletickets"` // Ticket hashes } // EncodeStartRunoffReply encodes a StartRunoffReply into a JSON byte slice. @@ -461,7 +468,7 @@ type SummariesReply struct { Summaries map[string]Summary `json:"summaries"` // [token]Summary // BestBlock is the best block value that was used to prepare the - // the summaries. + // summaries. BestBlock uint64 `json:"bestblock"` } @@ -507,8 +514,8 @@ type InventoryReply struct { Started []string `json:"started"` Finished []string `json:"finished"` - // BestBlock is the best block value that was used to prepare - // the inventory. + // BestBlock is the best block value that was used to prepare the + // inventory. BestBlock uint64 `json:"bestblock"` } diff --git a/politeiad/backend/tlogbe/plugins/comments.go b/politeiad/backend/tlogbe/plugins/comments.go index aac98db25..b7fb4b2df 100644 --- a/politeiad/backend/tlogbe/plugins/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments.go @@ -9,6 +9,7 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" + "errors" "fmt" "sort" "strconv" @@ -133,38 +134,6 @@ type index struct { Comments map[uint32]commentIndex `json:"comments"` // [commentID]comment } -// TODO this needs to go in util -func verifySignature(signature, pubkey, msg string) error { - sig, err := util.ConvertSignature(signature) - if err != nil { - return comments.UserError{ - ErrorCode: comments.ErrorStatusSignatureInvalid, - ErrorContext: []string{err.Error()}, - } - } - b, err := hex.DecodeString(pubkey) - if err != nil { - return comments.UserError{ - ErrorCode: comments.ErrorStatusPublicKeyInvalid, - ErrorContext: []string{"key is not hex"}, - } - } - pk, err := identity.PublicIdentityFromBytes(b) - if err != nil { - return comments.UserError{ - ErrorCode: comments.ErrorStatusPublicKeyInvalid, - ErrorContext: []string{err.Error()}, - } - } - if !pk.VerifyMessage([]byte(msg), sig) { - return comments.UserError{ - ErrorCode: comments.ErrorStatusSignatureInvalid, - ErrorContext: []string{err.Error()}, - } - } - return nil -} - // mutex returns the mutex for the specified record. func (p *commentsPlugin) mutex(token string) *sync.Mutex { p.Lock() @@ -180,6 +149,23 @@ func (p *commentsPlugin) mutex(token string) *sync.Mutex { return m } +func convertCommentsErrFromSignatureErr(err error) comments.UserError { + var e util.SignatureError + var s comments.ErrorStatusT + if errors.As(err, &e) { + switch e.ErrorCode { + case util.ErrorStatusPublicKeyInvalid: + s = comments.ErrorStatusPublicKeyInvalid + case util.ErrorStatusSignatureInvalid: + s = comments.ErrorStatusSignatureInvalid + } + } + return comments.UserError{ + ErrorCode: s, + ErrorContext: e.ErrorContext, + } +} + func convertBlobEntryFromCommentAdd(c commentAdd) (*store.BlobEntry, error) { data, err := json.Marshal(c) if err != nil { @@ -826,9 +812,9 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { // Verify signature msg := n.Token + strconv.FormatUint(uint64(n.ParentID), 10) + n.Comment - err = verifySignature(n.Signature, n.PublicKey, msg) + err = util.VerifySignature(n.Signature, n.PublicKey, msg) if err != nil { - return "", err + return "", convertCommentsErrFromSignatureErr(err) } // Get plugin client @@ -966,9 +952,9 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { // Verify signature msg := e.Token + strconv.FormatUint(uint64(e.ParentID), 10) + e.Comment - err = verifySignature(e.Signature, e.PublicKey, msg) + err = util.VerifySignature(e.Signature, e.PublicKey, msg) if err != nil { - return "", err + return "", convertCommentsErrFromSignatureErr(err) } // Get plugin client @@ -1099,9 +1085,9 @@ func (p *commentsPlugin) cmdDel(payload string) (string, error) { // Verify signature msg := d.Token + strconv.FormatUint(uint64(d.CommentID), 10) + d.Reason - err = verifySignature(d.Signature, d.PublicKey, msg) + err = util.VerifySignature(d.Signature, d.PublicKey, msg) if err != nil { - return "", err + return "", convertCommentsErrFromSignatureErr(err) } // Get plugin client @@ -1408,9 +1394,9 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { // Validate signature msg := v.Token + strconv.FormatUint(uint64(v.CommentID), 10) + strconv.FormatInt(int64(v.Vote), 10) - err = verifySignature(v.Signature, v.PublicKey, msg) + err = util.VerifySignature(v.Signature, v.PublicKey, msg) if err != nil { - return "", err + return "", convertCommentsErrFromSignatureErr(err) } // Get plugin client diff --git a/politeiad/backend/tlogbe/plugins/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote.go index 0b0801241..6e0f4687f 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote.go @@ -5,9 +5,15 @@ package plugins import ( + "encoding/hex" + "errors" + "strconv" + "github.com/decred/politeia/plugins/ticketvote" "github.com/decred/politeia/politeiad/api/v1/identity" + "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe" + "github.com/decred/politeia/util" ) var ( @@ -33,17 +39,34 @@ type authorize struct { } // start is the structure that is saved to disk when a vote is started. +// +// Signature is a signature of the SHA256 digest of the JSON encoded Vote +// struct. type start struct { Vote ticketvote.Vote `json:"vote"` PublicKey string `json:"publickey"` + Signature string `json:"signature"` StartBlockHeight uint32 `json:"startblockheight"` StartBlockHash string `json:"startblockhash"` EndBlockHeight uint32 `json:"endblockheight"` EligibleTickets []string `json:"eligibletickets"` // Ticket hashes +} - // Signature is a signature of the SHA256 digest of the JSON - // encoded Vote struct. - Signature string `json:"signature"` +func convertTicketVoteErrFromSignatureErr(err error) ticketvote.UserError { + var e util.SignatureError + var s ticketvote.ErrorStatusT + if errors.As(err, &e) { + switch e.ErrorCode { + case util.ErrorStatusPublicKeyInvalid: + s = ticketvote.ErrorStatusPublicKeyInvalid + case util.ErrorStatusSignatureInvalid: + s = ticketvote.ErrorStatusSignatureInvalid + } + } + return ticketvote.UserError{ + ErrorCode: s, + ErrorContext: e.ErrorContext, + } } func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { @@ -53,9 +76,34 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { if err != nil { return "", err } - _ = a - // Ensure record exists + // Verify signature + msg := a.Token + strconv.FormatUint(uint64(a.Version), 10) + string(a.Action) + err = util.VerifySignature(a.Signature, a.PublicKey, msg) + if err != nil { + return "", convertTicketVoteErrFromSignatureErr(err) + } + + // Get plugin client + token, err := hex.DecodeString(a.Token) + if err != nil { + return "", ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusTokenInvalid, + } + } + client, err := p.backend.PluginClient(token) + if err != nil { + if err == backend.ErrRecordNotFound { + return "", ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusRecordNotFound, + } + } + return "", err + } + + _ = client + + // Verify version // Verify action // Verify record state // Prepare diff --git a/util/signature.go b/util/signature.go new file mode 100644 index 000000000..b10ccddcb --- /dev/null +++ b/util/signature.go @@ -0,0 +1,65 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package util + +import ( + "encoding/hex" + "fmt" + + "github.com/decred/politeia/politeiad/api/v1/identity" +) + +type ErrorStatusT int + +const ( + // Error codes + ErrorStatusInvalid ErrorStatusT = 0 + ErrorStatusPublicKeyInvalid ErrorStatusT = 1 + ErrorStatusSignatureInvalid ErrorStatusT = 2 +) + +// SignatureError represents an error that was caused while verifying a +// signature. +type SignatureError struct { + ErrorCode ErrorStatusT + ErrorContext []string +} + +// Error satisfies the error interface. +func (e SignatureError) Error() string { + return fmt.Sprintf("signature error code: %v", e.ErrorCode) +} + +// VerifySignature verifies a Ed25519 signature. +func VerifySignature(signature, pubkey, msg string) error { + sig, err := ConvertSignature(signature) + if err != nil { + return SignatureError{ + ErrorCode: ErrorStatusSignatureInvalid, + ErrorContext: []string{err.Error()}, + } + } + b, err := hex.DecodeString(pubkey) + if err != nil { + return SignatureError{ + ErrorCode: ErrorStatusPublicKeyInvalid, + ErrorContext: []string{"key is not hex"}, + } + } + pk, err := identity.PublicIdentityFromBytes(b) + if err != nil { + return SignatureError{ + ErrorCode: ErrorStatusPublicKeyInvalid, + ErrorContext: []string{err.Error()}, + } + } + if !pk.VerifyMessage([]byte(msg), sig) { + return SignatureError{ + ErrorCode: ErrorStatusSignatureInvalid, + ErrorContext: []string{err.Error()}, + } + } + return nil +} From 824b7799821f15bd3e3be65d1afd85881ada188b Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 4 Aug 2020 10:56:49 -0600 Subject: [PATCH 013/449] ticketvote plugin --- plugins/comments/comments.go | 12 +- plugins/ticketvote/ticketvote.go | 40 ++- politeiad/backend/backend.go | 3 - politeiad/backend/tlogbe/plugins/comments.go | 26 +- .../backend/tlogbe/plugins/ticketvote.go | 228 +++++++++++++++++- politeiad/backend/tlogbe/tlogbe.go | 5 - 6 files changed, 268 insertions(+), 46 deletions(-) diff --git a/plugins/comments/comments.go b/plugins/comments/comments.go index 80c8aa766..57b913d04 100644 --- a/plugins/comments/comments.go +++ b/plugins/comments/comments.go @@ -74,7 +74,7 @@ type UserError struct { // Error satisfies the error interface. func (e UserError) Error() string { - return fmt.Sprintf("comments plugin error code: %v", e.ErrorCode) + return fmt.Sprintf("comments error code: %v", e.ErrorCode) } // Comment represent a record comment. @@ -86,8 +86,8 @@ type Comment struct { Signature string `json:"signature"` // Signature of Token+ParentID+Comment CommentID uint32 `json:"commentid"` // Comment ID Version uint32 `json:"version"` // Comment version - Receipt string `json:"receipt"` // Server signature of client signature Timestamp int64 `json:"timestamp"` // UNIX timestamp of last edit + Receipt string `json:"receipt"` // Server signature of client signature Score int64 `json:"score"` // Vote score Deleted bool `json:"deleted"` // Comment has been deleted Reason string `json:"reason"` // Reason for deletion @@ -121,8 +121,8 @@ func DecodeNew(payload []byte) (*New, error) { // NewReply is the reply to the New command. type NewReply struct { CommentID uint32 `json:"commentid"` // Comment ID - Receipt string `json:"receipt"` // Server sig of client sig Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + Receipt string `json:"receipt"` // Server sig of client sig } // EncodeNew encodes a NewReply into a JSON byte slice. @@ -168,8 +168,8 @@ func DecodeEdit(payload []byte) (*Edit, error) { // EditReply is the reply to the Edit command. type EditReply struct { Version uint32 `json:"version"` // Comment version - Receipt string `json:"receipt"` // Server signature of client signature Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + Receipt string `json:"receipt"` // Server signature of client signature } // EncodeEdit encodes a EditReply into a JSON byte slice. @@ -213,8 +213,8 @@ func DecodeDel(payload []byte) (*Del, error) { // DelReply is the reply to the Del command. type DelReply struct { - Receipt string `json:"receipt"` // Server signature of client signature Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + Receipt string `json:"receipt"` // Server signature of client signature } // EncodeDelReply encodes a DelReply into a JSON byte slice. @@ -434,8 +434,8 @@ func DecodeVote(payload []byte) (*Vote, error) { // VoteReply is the reply to the Vote command. type VoteReply struct { Score int64 `json:"score"` // Overall comment vote score - Receipt string `json:"receipt"` // Server signature of client signature Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + Receipt string `json:"receipt"` // Server signature of client signature } // EncodeVoteReply encodes a VoteReply into a JSON byte slice. diff --git a/plugins/ticketvote/ticketvote.go b/plugins/ticketvote/ticketvote.go index a183f62e9..f8b99ce23 100644 --- a/plugins/ticketvote/ticketvote.go +++ b/plugins/ticketvote/ticketvote.go @@ -73,20 +73,25 @@ const ( // Error status codes // TODO change politeiavoter to use these error codes - ErrorStatusInvalid ErrorStatusT = 0 - ErrorStatusTokenInvalid ErrorStatusT = 1 - ErrorStatusPublicKeyInvalid ErrorStatusT = 2 - ErrorStatusSignatureInvalid ErrorStatusT = 3 - ErrorStatusRecordNotFound ErrorStatusT = 4 + ErrorStatusInvalid ErrorStatusT = 0 + ErrorStatusTokenInvalid ErrorStatusT = 1 + ErrorStatusPublicKeyInvalid ErrorStatusT = 2 + ErrorStatusSignatureInvalid ErrorStatusT = 3 + ErrorStatusRecordNotFound ErrorStatusT = 4 + ErrorStatusRecordStateInvalid ErrorStatusT = 5 + ErrorStatusAuthorizeActionInvalid ErrorStatusT = 6 ) var ( // ErrorStatus contains human readable error statuses. ErrorStatus = map[ErrorStatusT]string{ - ErrorStatusInvalid: "invalid error status", - ErrorStatusTokenInvalid: "invalid token", - ErrorStatusPublicKeyInvalid: "invalid public key", - ErrorStatusSignatureInvalid: "invalid signature", + ErrorStatusInvalid: "invalid error status", + ErrorStatusTokenInvalid: "invalid token", + ErrorStatusPublicKeyInvalid: "invalid public key", + ErrorStatusSignatureInvalid: "invalid signature", + ErrorStatusRecordNotFound: "record not found", + ErrorStatusRecordStateInvalid: "record state invalid", + ErrorStatusAuthorizeActionInvalid: "authorize action invalid", } ) @@ -98,7 +103,7 @@ type UserError struct { // Error satisfies the error interface. func (e UserError) Error() string { - return fmt.Sprintf("ticketvote plugin error code: %v", e.ErrorCode) + return fmt.Sprintf("ticketvote error code: %v", e.ErrorCode) } // Authorize authorizes a ticket vote or revokes a previous authorization. @@ -131,6 +136,21 @@ type AuthorizeReply struct { Receipt string `json:"receipt"` // Server signature of client signature } +// EncodeAuthorizeReply encodes an AuthorizeReply into a JSON byte slice. +func EncodeAuthorizeReply(ar AuthorizeReply) ([]byte, error) { + return json.Marshal(ar) +} + +// DecodeAuthorizeReply decodes a JSON byte slice into a AuthorizeReply. +func DecodeAuthorizeReply(payload []byte) (*AuthorizeReply, error) { + var ar AuthorizeReply + err := json.Unmarshal(payload, &ar) + if err != nil { + return nil, err + } + return &ar, nil +} + // VoteOption describes a single vote option. type VoteOption struct { ID string `json:"id"` // Single, unique word (e.g. yes) diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index aac385f3f..e0e72f7b8 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -322,9 +322,6 @@ type Backend interface { UpdateVettedMetadata([]byte, []MetadataStream, []MetadataStream) error - // Update README.md file at the root of git repo - UpdateReadme(string) error - // Check if an unvetted record exists UnvettedExists([]byte) bool diff --git a/politeiad/backend/tlogbe/plugins/comments.go b/politeiad/backend/tlogbe/plugins/comments.go index b7fb4b2df..6799ac553 100644 --- a/politeiad/backend/tlogbe/plugins/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments.go @@ -67,8 +67,8 @@ type commentAdd struct { // Metadata generated by server CommentID uint32 `json:"commentid"` // Comment ID Version uint32 `json:"version"` // Comment version - Receipt string `json:"receipt"` // Server signature of client signature Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + Receipt string `json:"receipt"` // Server signature of client signature } // commentDel is the structure that is saved to disk when a comment is deleted. @@ -87,8 +87,8 @@ type commentDel struct { // Metadata generated by server ParentID uint32 `json:"parentid"` // Parent comment ID AuthorPublicKey string `json:"authorpublickey"` // Author public key - Receipt string `json:"receipt"` // Server sig of client sig Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + Receipt string `json:"receipt"` // Server sig of client sig } // commentVote is the structure that is saved to disk when a comment is voted @@ -103,8 +103,8 @@ type commentVote struct { Signature string `json:"signature"` // Signature of Token+CommentID+Vote // Metadata generated by server - Receipt string `json:"receipt"` // Server signature of client signature Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + Receipt string `json:"receipt"` // Server signature of client signature } type voteIndex struct { @@ -358,8 +358,8 @@ func convertCommentFromCommentAdd(ca commentAdd) comments.Comment { Signature: ca.Signature, CommentID: ca.CommentID, Version: ca.Version, - Receipt: ca.Receipt, Timestamp: ca.Timestamp, + Receipt: ca.Receipt, Score: 0, Deleted: false, Reason: "", @@ -376,8 +376,8 @@ func convertCommentFromCommentDel(cd commentDel) comments.Comment { Signature: "", CommentID: cd.CommentID, Version: 0, - Receipt: "", Timestamp: cd.Timestamp, + Receipt: "", Score: 0, Deleted: true, Reason: cd.Reason, @@ -766,8 +766,8 @@ func (p *commentsPlugin) new(client *tlogbe.PluginClient, n comments.New, encryp Signature: n.Signature, CommentID: commentIDLatest(*idx) + 1, Version: 1, - Receipt: hex.EncodeToString(receipt[:]), Timestamp: time.Now().Unix(), + Receipt: hex.EncodeToString(receipt[:]), } // Save comment @@ -796,8 +796,8 @@ func (p *commentsPlugin) new(client *tlogbe.PluginClient, n comments.New, encryp return &comments.NewReply{ CommentID: c.CommentID, - Receipt: c.Receipt, Timestamp: c.Timestamp, + Receipt: c.Receipt, }, nil } @@ -912,8 +912,8 @@ func (p *commentsPlugin) edit(client *tlogbe.PluginClient, e comments.Edit, encr Signature: e.Signature, CommentID: e.CommentID, Version: existing.Version + 1, - Receipt: hex.EncodeToString(receipt[:]), Timestamp: time.Now().Unix(), + Receipt: hex.EncodeToString(receipt[:]), } // Save comment @@ -936,8 +936,8 @@ func (p *commentsPlugin) edit(client *tlogbe.PluginClient, e comments.Edit, encr return &comments.EditReply{ Version: c.Version, - Receipt: c.Receipt, Timestamp: c.Timestamp, + Receipt: c.Receipt, }, nil } @@ -1036,8 +1036,8 @@ func (p *commentsPlugin) del(client *tlogbe.PluginClient, d comments.Del) (*comm Signature: d.Signature, ParentID: comment.ParentID, AuthorPublicKey: comment.PublicKey, - Receipt: hex.EncodeToString(receipt[:]), Timestamp: time.Now().Unix(), + Receipt: hex.EncodeToString(receipt[:]), } merkleHash, err := commentDelSave(client, cd) if err != nil { @@ -1069,8 +1069,8 @@ func (p *commentsPlugin) del(client *tlogbe.PluginClient, d comments.Del) (*comm } return &comments.DelReply{ - Receipt: cd.Receipt, Timestamp: cd.Timestamp, + Receipt: cd.Receipt, }, nil } @@ -1456,8 +1456,8 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { Vote: int64(v.Vote), PublicKey: v.PublicKey, Signature: v.Signature, - Receipt: hex.EncodeToString(receipt[:]), Timestamp: time.Now().Unix(), + Receipt: hex.EncodeToString(receipt[:]), } // Save comment vote @@ -1477,8 +1477,8 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { // Prepare reply vr := comments.VoteReply{ - Receipt: cv.Receipt, Timestamp: cv.Timestamp, + Receipt: cv.Receipt, Score: updatedIdx.Comments[cv.CommentID].Score, } reply, err := comments.EncodeVoteReply(vr) diff --git a/politeiad/backend/tlogbe/plugins/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote.go index 6e0f4687f..563978aff 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote.go @@ -5,17 +5,34 @@ package plugins import ( + "bytes" + "encoding/base64" "encoding/hex" + "encoding/json" "errors" + "fmt" "strconv" + "time" "github.com/decred/politeia/plugins/ticketvote" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe" + "github.com/decred/politeia/politeiad/backend/tlogbe/store" "github.com/decred/politeia/util" ) +const ( + // Blob entry data descriptors + dataDescriptorAuthorize = "ticketvoteauthorize" + dataDescriptorStart = "ticketvotestart" + + // Prefixes that are appended to key-value store keys before + // storing them in the log leaf ExtraData field. + keyPrefixAuthorize = "ticketvoteauthorize" + keyPrefixStart = "ticketvotestart" +) + var ( _ Plugin = (*ticketVotePlugin)(nil) ) @@ -27,7 +44,7 @@ type ticketVotePlugin struct { } // authorize is the structure that is saved to disk when a vote is authorized -// or a previous authorizatio is revoked. +// or a previous authorization is revoked. type authorize struct { Token string `json:"token"` // Record token Version uint32 `json:"version"` // Record version @@ -69,29 +86,138 @@ func convertTicketVoteErrFromSignatureErr(err error) ticketvote.UserError { } } +func convertAuthorizeFromBlobEntry(be store.BlobEntry) (*authorize, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorAuthorize { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorAuthorize) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + hash, err := hex.DecodeString(be.Hash) + if err != nil { + return nil, fmt.Errorf("decode hash: %v", err) + } + if !bytes.Equal(util.Digest(b), hash) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), hash) + } + var a authorize + err = json.Unmarshal(b, &a) + if err != nil { + return nil, fmt.Errorf("unmarshal index: %v", err) + } + + return &a, nil +} + +func convertBlobEntryFromAuthorize(a authorize) (*store.BlobEntry, error) { + data, err := json.Marshal(a) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorAuthorize, + }) + if err != nil { + return nil, err + } + be := store.BlobEntryNew(hint, data) + return &be, nil +} + +func authorizeSave(client *tlogbe.PluginClient, a authorize) error { + // Prepare blob + be, err := convertBlobEntryFromAuthorize(a) + if err != nil { + return err + } + h, err := hex.DecodeString(be.Hash) + if err != nil { + return err + } + b, err := store.Blobify(*be) + if err != nil { + return err + } + + // Save blob + merkles, err := client.BlobsSave(keyPrefixAuthorize, + [][]byte{b}, [][]byte{h}, false) + if err != nil { + return fmt.Errorf("BlobsSave: %v", err) + } + if len(merkles) != 1 { + return fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", + len(merkles)) + } + + return nil +} + +func authorizations(client *tlogbe.PluginClient) ([]authorize, error) { + // Retrieve blobs + blobs, err := client.BlobsByKeyPrefix(keyPrefixAuthorize) + if err != nil { + return nil, err + } + + // Decode blobs + auths := make([]authorize, 0, len(blobs)) + for _, v := range blobs { + be, err := store.Deblob(v) + if err != nil { + return nil, err + } + a, err := convertAuthorizeFromBlobEntry(*be) + if err != nil { + return nil, err + } + auths = append(auths, *a) + } + + return auths, nil +} func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { log.Tracef("ticketvote cmdAuthorize: %v", payload) + // Decode payload a, err := ticketvote.DecodeAuthorize([]byte(payload)) if err != nil { return "", err } // Verify signature - msg := a.Token + strconv.FormatUint(uint64(a.Version), 10) + string(a.Action) + version := strconv.FormatUint(uint64(a.Version), 10) + msg := a.Token + version + string(a.Action) err = util.VerifySignature(a.Signature, a.PublicKey, msg) if err != nil { return "", convertTicketVoteErrFromSignatureErr(err) } // Get plugin client - token, err := hex.DecodeString(a.Token) + tokenb, err := hex.DecodeString(a.Token) if err != nil { return "", ticketvote.UserError{ ErrorCode: ticketvote.ErrorStatusTokenInvalid, } } - client, err := p.backend.PluginClient(token) + client, err := p.backend.PluginClient(tokenb) if err != nil { if err == backend.ErrRecordNotFound { return "", ticketvote.UserError{ @@ -100,15 +226,99 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { } return "", err } + if client.State != tlogbe.RecordStateVetted { + return "", ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusRecordStateInvalid, + ErrorContext: []string{"record not vetted"}, + } + } - _ = client + // Verify record version + _, err = p.backend.GetVetted(tokenb, version) + if err != nil { + if err == backend.ErrRecordNotFound { + e := fmt.Sprintf("version %v not found", version) + return "", ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusRecordNotFound, + ErrorContext: []string{e}, + } + } + } - // Verify version // Verify action - // Verify record state - // Prepare + switch a.Action { + case ticketvote.ActionAuthorize: + // This is allowed + case ticketvote.ActionRevoke: + // This is allowed + default: + return "", ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusAuthorizeActionInvalid, + } + } - return "", nil + // Get any previous authorizations to verify that the new action + // is allowed based on the previous action. + auths, err := authorizations(client) + if err != nil { + return "", err + } + var prevAction ticketvote.ActionT + if len(auths) > 0 { + prevAction = ticketvote.ActionT(auths[len(auths)-1].Action) + } + switch { + case len(auths) == 0: + // No previous actions. New action must be an authorize. + if a.Action != ticketvote.ActionAuthorize { + return "", ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusAuthorizeActionInvalid, + ErrorContext: []string{"no prev action; action must be authorize"}, + } + } + case prevAction == ticketvote.ActionAuthorize: + // Previous action was a authorize. This action must be revoke. + return "", ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusAuthorizeActionInvalid, + ErrorContext: []string{"prev action was authorize"}, + } + case prevAction == ticketvote.ActionRevoke: + // Previous action was a revoke. This action must be authorize. + return "", ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusAuthorizeActionInvalid, + ErrorContext: []string{"prev action was revoke"}, + } + } + + // Prepare authorization + receipt := p.id.SignMessage([]byte(a.Signature)) + auth := authorize{ + Token: a.Token, + Version: a.Version, + Action: string(a.Action), + PublicKey: a.PublicKey, + Signature: a.Signature, + Timestamp: time.Now().Unix(), + Receipt: hex.EncodeToString(receipt[:]), + } + + // Save authorization + err = authorizeSave(client, auth) + if err != nil { + return "", err + } + + // Prepare reply + ar := ticketvote.AuthorizeReply{ + Timestamp: auth.Timestamp, + Receipt: auth.Receipt, + } + reply, err := ticketvote.EncodeAuthorizeReply(ar) + if err != nil { + return "", err + } + + return string(reply), nil } func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 105f8acdf..1e2a89151 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -526,11 +526,6 @@ func (t *Tlogbe) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite []back return t.vetted.recordMetadataUpdate(treeID, r.RecordMetadata, metadata) } -// TODO remove -func (t *Tlogbe) UpdateReadme(content string) error { - return fmt.Errorf("not implemented") -} - // UnvettedExists returns whether the provided token corresponds to an unvetted // record. // From c7f56b9978f1c7b085f58ef78155c2af850d178b Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 4 Aug 2020 12:35:11 -0600 Subject: [PATCH 014/449] ticketvote plugin cont --- plugins/dcrdata/dcrdata.go | 145 +- plugins/ticketvote/ticketvote.go | 213 +- politeiad/backend/tlogbe/plugins/comments.go | 11 +- politeiad/backend/tlogbe/plugins/dcrdata.go | 286 ++- .../backend/tlogbe/plugins/ticketvote.go | 1716 +++++++++++++++-- politeiawww/proposals.go | 3 + util/signature.go | 6 +- 7 files changed, 2061 insertions(+), 319 deletions(-) diff --git a/plugins/dcrdata/dcrdata.go b/plugins/dcrdata/dcrdata.go index 180e09861..83cdb8001 100644 --- a/plugins/dcrdata/dcrdata.go +++ b/plugins/dcrdata/dcrdata.go @@ -4,7 +4,11 @@ package dcrdata -import "encoding/json" +import ( + "encoding/json" + + v4 "github.com/decred/dcrdata/api/types/v4" +) type StatusT int @@ -13,7 +17,10 @@ const ( ID = "dcrdata" // Plugin commands - CmdBestBlock = "bestblock" + CmdBestBlock = "bestblock" // Get best block + CmdBlockDetails = "blockdetails" // Get details of a block + CmdTicketPool = "ticketpool" // Get ticket pool + CmdTxsTrimmed = "txstrimmed" // Get trimmed transactions // Dcrdata connection statuses. // @@ -26,10 +33,10 @@ const ( StatusDisconnected StatusT = 2 ) -// BestBlock requests the best block. If dcrdata cannot be reached then the -// most recent cached best block will be returned along with a status of -// StatusDisconnected. It is the callers responsibility to determine if the -// stale best block should be used. +// BestBlock requests best block data. If dcrdata cannot be reached then the +// data from the most recent cached best block will be returned along with a +// status of StatusDisconnected. It is the callers responsibility to determine +// if the stale best block height should be used. type BestBlock struct{} // EncodeBestBlock encodes an BestBlock into a JSON byte slice. @@ -49,8 +56,8 @@ func DecodeBestBlock(payload []byte) (*BestBlock, error) { // BestBlockReply is the reply to the BestBlock command. type BestBlockReply struct { - Status StatusT `json:"status"` - BestBlock uint32 `json:"bestblock"` + Status StatusT `json:"status"` + Height uint32 `json:"height"` } // EncodeBestBlockReply encodes an BestBlockReply into a JSON byte slice. @@ -67,3 +74,125 @@ func DecodeBestBlockReply(payload []byte) (*BestBlockReply, error) { } return &bbr, nil } + +// BlockDetails fetched the block details for the provided block height. +type BlockDetails struct { + Height uint32 `json:"height"` +} + +// EncodeBlockDetails encodes an BlockDetails into a JSON byte slice. +func EncodeBlockDetails(bd BlockDetails) ([]byte, error) { + return json.Marshal(bd) +} + +// DecodeBlockDetails decodes a JSON byte slice into a BlockDetails. +func DecodeBlockDetails(payload []byte) (*BlockDetails, error) { + var bd BlockDetails + err := json.Unmarshal(payload, &bd) + if err != nil { + return nil, err + } + return &bd, nil +} + +// BlockDetailsReply is the reply to the block details command. +type BlockDetailsReply struct { + Block v4.BlockDataBasic `json:"block"` +} + +// EncodeBlockDetailsReply encodes an BlockDetailsReply into a JSON byte slice. +func EncodeBlockDetailsReply(bdr BlockDetailsReply) ([]byte, error) { + return json.Marshal(bdr) +} + +// DecodeBlockDetailsReply decodes a JSON byte slice into a BlockDetailsReply. +func DecodeBlockDetailsReply(payload []byte) (*BlockDetailsReply, error) { + var bdr BlockDetailsReply + err := json.Unmarshal(payload, &bdr) + if err != nil { + return nil, err + } + return &bdr, nil +} + +// TicketPool requests the lists of tickets in the ticket for at the provided +// block hash. +type TicketPool struct { + BlockHash string `json:"blockhash"` +} + +// EncodeTicketPool encodes an TicketPool into a JSON byte slice. +func EncodeTicketPool(tp TicketPool) ([]byte, error) { + return json.Marshal(tp) +} + +// DecodeTicketPool decodes a JSON byte slice into a TicketPool. +func DecodeTicketPool(payload []byte) (*TicketPool, error) { + var tp TicketPool + err := json.Unmarshal(payload, &tp) + if err != nil { + return nil, err + } + return &tp, nil +} + +// TicketPoolReply is the reply to the TicketPool command. +type TicketPoolReply struct { + Tickets []string `json:"tickets"` // Ticket hashes +} + +// EncodeTicketPoolReply encodes an TicketPoolReply into a JSON byte slice. +func EncodeTicketPoolReply(tpr TicketPoolReply) ([]byte, error) { + return json.Marshal(tpr) +} + +// DecodeTicketPoolReply decodes a JSON byte slice into a TicketPoolReply. +func DecodeTicketPoolReply(payload []byte) (*TicketPoolReply, error) { + var tpr TicketPoolReply + err := json.Unmarshal(payload, &tpr) + if err != nil { + return nil, err + } + return &tpr, nil +} + +// TxsTrimmed requests the trimmed transaction information for the provided +// transaction IDs. +type TxsTrimmed struct { + TxIDs []string `json:"txids"` +} + +// EncodeTxsTrimmed encodes an TxsTrimmed into a JSON byte slice. +func EncodeTxsTrimmed(tt TxsTrimmed) ([]byte, error) { + return json.Marshal(tt) +} + +// DecodeTxsTrimmed decodes a JSON byte slice into a TxsTrimmed. +func DecodeTxsTrimmed(payload []byte) (*TxsTrimmed, error) { + var tt TxsTrimmed + err := json.Unmarshal(payload, &tt) + if err != nil { + return nil, err + } + return &tt, nil +} + +// TxsTrimmedReply is the reply to the TxsTrimmed command. +type TxsTrimmedReply struct { + Txs []v4.TrimmedTx `json:"txs"` +} + +// EncodeTxsTrimmedReply encodes an TxsTrimmedReply into a JSON byte slice. +func EncodeTxsTrimmedReply(ttr TxsTrimmedReply) ([]byte, error) { + return json.Marshal(ttr) +} + +// DecodeTxsTrimmedReply decodes a JSON byte slice into a TxsTrimmedReply. +func DecodeTxsTrimmedReply(payload []byte) (*TxsTrimmedReply, error) { + var ttr TxsTrimmedReply + err := json.Unmarshal(payload, &ttr) + if err != nil { + return nil, err + } + return &ttr, nil +} diff --git a/plugins/ticketvote/ticketvote.go b/plugins/ticketvote/ticketvote.go index f8b99ce23..f6c36ec85 100644 --- a/plugins/ticketvote/ticketvote.go +++ b/plugins/ticketvote/ticketvote.go @@ -12,11 +12,12 @@ import ( type ActionT string type VoteT int type VoteStatusT int +type VoteErrorT int type ErrorStatusT int const ( Version uint32 = 1 - ID = "ticketvotes" + ID = "ticketvote" // Plugin commands CmdAuthorize = "authorize" // Authorize a vote @@ -71,27 +72,60 @@ const ( VoteOptionIDApprove = "yes" VoteOptionIDReject = "no" - // Error status codes + // Vote error status codes. Vote errors are errors that occur while + // attempting to cast a vote. These errors are returned with the + // individual failed vote. // TODO change politeiavoter to use these error codes - ErrorStatusInvalid ErrorStatusT = 0 - ErrorStatusTokenInvalid ErrorStatusT = 1 - ErrorStatusPublicKeyInvalid ErrorStatusT = 2 - ErrorStatusSignatureInvalid ErrorStatusT = 3 - ErrorStatusRecordNotFound ErrorStatusT = 4 - ErrorStatusRecordStateInvalid ErrorStatusT = 5 - ErrorStatusAuthorizeActionInvalid ErrorStatusT = 6 + VoteErrorInvalid VoteErrorT = 0 + VoteErrorInternalError VoteErrorT = 1 + VoteErrorTokenInvalid VoteErrorT = 2 + VoteErrorRecordNotFound VoteErrorT = 3 + VoteErrorMultipleRecordVotes VoteErrorT = 4 + VoteErrorVoteStatusInvalid VoteErrorT = 5 + VoteErrorVoteBitInvalid VoteErrorT = 6 + VoteErrorSignatureInvalid VoteErrorT = 7 + VoteErrorTicketNotEligible VoteErrorT = 8 + VoteErrorTicketAlreadyVoted VoteErrorT = 9 + + // User error status codes + ErrorStatusInvalid ErrorStatusT = 0 + ErrorStatusTokenInvalid ErrorStatusT = 1 + ErrorStatusPublicKeyInvalid ErrorStatusT = 2 + ErrorStatusSignatureInvalid ErrorStatusT = 3 + ErrorStatusRecordNotFound ErrorStatusT = 4 + ErrorStatusRecordStateInvalid ErrorStatusT = 5 + ErrorStatusAuthorizationInvalid ErrorStatusT = 6 + ErrorStatusVoteInvalid ErrorStatusT = 7 + ErrorStatusVoteStatusInvalid ErrorStatusT = 8 + ErrorStatusBallotInvalid ErrorStatusT = 9 ) var ( - // ErrorStatus contains human readable error statuses. + VoteError = map[VoteErrorT]string{ + VoteErrorInvalid: "vote error invalid", + VoteErrorInternalError: "internal server error", + VoteErrorTokenInvalid: "token invalid", + VoteErrorRecordNotFound: "record not found", + VoteErrorMultipleRecordVotes: "attempting to vote on multiple records", + VoteErrorVoteStatusInvalid: "record vote status invalid", + VoteErrorVoteBitInvalid: "vote bit invalid", + VoteErrorSignatureInvalid: "signature invalid", + VoteErrorTicketNotEligible: "ticket not eligible", + VoteErrorTicketAlreadyVoted: "ticket already voted", + } + + // ErrorStatus contains human readable user error statuses. ErrorStatus = map[ErrorStatusT]string{ - ErrorStatusInvalid: "invalid error status", - ErrorStatusTokenInvalid: "invalid token", - ErrorStatusPublicKeyInvalid: "invalid public key", - ErrorStatusSignatureInvalid: "invalid signature", - ErrorStatusRecordNotFound: "record not found", - ErrorStatusRecordStateInvalid: "record state invalid", - ErrorStatusAuthorizeActionInvalid: "authorize action invalid", + ErrorStatusInvalid: "error status invalid", + ErrorStatusTokenInvalid: "token invalid", + ErrorStatusPublicKeyInvalid: "public key invalid", + ErrorStatusSignatureInvalid: "signature invalid", + ErrorStatusRecordNotFound: "record not found", + ErrorStatusRecordStateInvalid: "record state invalid", + ErrorStatusAuthorizationInvalid: "authorization invalid", + ErrorStatusVoteInvalid: "vote invalid", + ErrorStatusVoteStatusInvalid: "vote status invalid", + ErrorStatusBallotInvalid: "ballot invalid", } ) @@ -106,6 +140,53 @@ func (e UserError) Error() string { return fmt.Sprintf("ticketvote error code: %v", e.ErrorCode) } +// AuthorizeVote is the structure that is saved to disk when a vote is +// authorized or a previous authorization is revoked. +type AuthorizeVote struct { + // Data generated by client + Token string `json:"token"` // Record token + Version uint32 `json:"version"` // Record version + Action string `json:"action"` // Authorize or revoke + PublicKey string `json:"publickey"` // Public key used for signature + Signature string `json:"signature"` // Signature of token+version+action + + // Metadata generated by server + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + Receipt string `json:"receipt"` // Server signature of client signature +} + +// StartVote is the structure that is saved to disk when a vote is started. +// TODO does this need a receipt? +type StartVote struct { + // Data generated by client + Vote VoteDetails `json:"vote"` + PublicKey string `json:"publickey"` + + // Signature is the client signature of the SHA256 digest of the + // JSON encoded Vote struct. + Signature string `json:"signature"` + + // Metadata generated by server + StartBlockHeight uint32 `json:"startblockheight"` + StartBlockHash string `json:"startblockhash"` + EndBlockHeight uint32 `json:"endblockheight"` + EligibleTickets []string `json:"eligibletickets"` // Ticket hashes +} + +// CastVote is the structure that is saved to disk when a vote is cast. +// TODO vote option vote bit is a uint64, but this vote bit is a string. Is +// that on purpose? +type CastVote struct { + // Data generated by client + Token string `json:"token"` // Record token + Ticket string `json:"ticket"` // Ticket hash + VoteBit string `json:"votebits"` // Selected vote bit, hex encoded + Signature string `json:"signature"` // Signature of Token+Ticket+VoteBit + + // Metdata generated by server + Receipt string `json:"receipt"` // Server signature of client signature +} + // Authorize authorizes a ticket vote or revokes a previous authorization. type Authorize struct { Token string `json:"token"` // Record token @@ -155,11 +236,11 @@ func DecodeAuthorizeReply(payload []byte) (*AuthorizeReply, error) { type VoteOption struct { ID string `json:"id"` // Single, unique word (e.g. yes) Description string `json:"description"` // Longer description of the vote - Bits uint64 `json:"bits"` // Bits used for this option + Bit uint64 `json:"bit"` // Bit used for this option } -// Vote describes the options and parameters of a ticket vote. -type Vote struct { +// VoteDetails describes the options and parameters of a ticket vote. +type VoteDetails struct { Token string `json:"token"` // Record token Version uint32 `json:"version"` // Record version Type VoteT `json:"type"` // Vote type @@ -179,8 +260,8 @@ type Vote struct { // Start starts a ticket vote. type Start struct { - Vote Vote `json:"vote"` // Vote options and params - PublicKey string `json:"publickey"` // Public key used for signature + Vote VoteDetails `json:"vote"` // Vote options and params + PublicKey string `json:"publickey"` // Public key used for signature // Signature is the signature of a SHA256 digest of the JSON // encoded Vote structure. @@ -193,7 +274,7 @@ func EncodeStart(s Start) ([]byte, error) { } // DecodeStart decodes a JSON byte slice into a Start. -func DecodeStartVote(payload []byte) (*Start, error) { +func DecodeStart(payload []byte) (*Start, error) { var s Start err := json.Unmarshal(payload, &s) if err != nil { @@ -271,26 +352,30 @@ func DecodeStartRunoffReply(payload []byte) (*StartRunoffReply, error) { return &srr, nil } -// CastVote is a signed ticket vote. -type CastVote struct { +// Vote is a signed ticket vote. This structure gets saved to disk when +// a vote is cast. +type Vote struct { Token string `json:"token"` // Record token Ticket string `json:"ticket"` // Ticket ID - VoteBit string `json:"votebit"` // Selected vote bit, hex encoded + VoteBit string `json:"votebits"` // Selected vote bit, hex encoded Signature string `json:"signature"` // Signature of Token+Ticket+VoteBit } -// CatVoteReply contains the receipt for the cast vote. If an error occured -// while casting the vote the receipt will be empty and a error code will be -// present. -type CastVoteReply struct { - Ticket string `json:"ticket"` // Ticket ID - Receipt string `json:"receipt"` // Server sig of client sig - ErrorCode ErrorStatusT `json:"errorcode,omitempty"` +// VoteReply contains the receipt for the cast vote. +type VoteReply struct { + Ticket string `json:"ticket"` // Ticket ID + Receipt string `json:"receipt"` // Server signature of client signature + + // The follwing fields will only be present if an error occured + // while attempting to cast the vote. + ErrorCode VoteErrorT `json:"errorcode,omitempty"` + ErrorContext string `json:"errorcontext,omitempty"` } -// Ballot is a batch of votes that are sent to the server. +// Ballot is a batch of votes that are sent to the server. A ballot can only +// contain the votes for a single record. type Ballot struct { - Votes []CastVote `json:"votes"` + Votes []Vote `json:"votes"` } // EncodeBallot encodes a Ballot into a JSON byte slice. @@ -299,7 +384,7 @@ func EncodeBallot(b Ballot) ([]byte, error) { } // DecodeBallot decodes a JSON byte slice into a Ballot. -func DecodeBallotVote(payload []byte) (*Ballot, error) { +func DecodeBallot(payload []byte) (*Ballot, error) { var b Ballot err := json.Unmarshal(payload, &b) if err != nil { @@ -310,7 +395,7 @@ func DecodeBallotVote(payload []byte) (*Ballot, error) { // BallotReply is a reply to a batched list of votes. type BallotReply struct { - Receipts []CastVoteReply `json:"receipts"` + Receipts []VoteReply `json:"receipts"` } // EncodeBallot encodes a Ballot into a JSON byte slice. @@ -319,7 +404,7 @@ func EncodeBallotReply(b BallotReply) ([]byte, error) { } // DecodeBallotReply decodes a JSON byte slice into a BallotReply. -func DecodeBallotReplyVote(payload []byte) (*BallotReply, error) { +func DecodeBallotReply(payload []byte) (*BallotReply, error) { var b BallotReply err := json.Unmarshal(payload, &b) if err != nil { @@ -339,7 +424,7 @@ func EncodeDetails(d Details) ([]byte, error) { } // DecodeDetails decodes a JSON byte slice into a Details. -func DecodeDetailsVote(payload []byte) (*Details, error) { +func DecodeDetails(payload []byte) (*Details, error) { var d Details err := json.Unmarshal(payload, &d) if err != nil { @@ -348,31 +433,10 @@ func DecodeDetailsVote(payload []byte) (*Details, error) { return &d, nil } -// AuthorizeDetails describes the details of a vote authorization. -type AuthorizeDetails struct { - Token string `json:"token"` // Record token - Version uint32 `json:"version"` // Record version - Action string `json:"action"` // Authorize or revoke - PublicKey string `json:"publickey"` // Public key used for signature - Signature string `json:"signature"` // Signature of token+version+action - Timestamp int64 `json:"timestamp"` // Received UNIX timestamp - Receipt string `json:"receipt"` // Server signature of client signature -} - // DetailsReply is the reply to the Details command. type DetailsReply struct { - // Auths contains all authorizations and revokes that were made for - // this ticket vote. - Auths []AuthorizeDetails `json:"auths"` - - // Vote details - Vote Vote `json:"vote"` // Vote params - PublicKey string `json:"publickey"` // Key used for sig - Signature string `json:"signature"` // Sig of Vote hash - StartBlockHeight uint32 `json:"startblockheight"` // Start block height - StartBlockHash string `json:"startblockhash"` // Start block hash - EndBlockHeight uint32 `json:"endblockheight"` // End block height - EligibleTickets []string `json:"eligibletickets"` // Valid voting tickets + Auths []AuthorizeVote `json:"auths"` + Vote *StartVote `json:"vote"` } // EncodeDetailsReply encodes a DetailsReply into a JSON byte slice. @@ -381,7 +445,7 @@ func EncodeDetailsReply(dr DetailsReply) ([]byte, error) { } // DecodeDetailsReply decodes a JSON byte slice into a DetailsReply. -func DecodeDetailsReplyVote(payload []byte) (*DetailsReply, error) { +func DecodeDetailsReply(payload []byte) (*DetailsReply, error) { var dr DetailsReply err := json.Unmarshal(payload, &dr) if err != nil { @@ -401,7 +465,7 @@ func EncodeCastVotes(cv CastVotes) ([]byte, error) { } // DecodeCastVotes decodes a JSON byte slice into a CastVotes. -func DecodeCastVotesVote(payload []byte) (*CastVotes, error) { +func DecodeCastVotes(payload []byte) (*CastVotes, error) { var cv CastVotes err := json.Unmarshal(payload, &cv) if err != nil { @@ -412,7 +476,7 @@ func DecodeCastVotesVote(payload []byte) (*CastVotes, error) { // CastVotesReply is the rely to the CastVotes command. type CastVotesReply struct { - CastVotes []CastVote `json:"castvotes"` + Votes []CastVote `json:"votes"` } // EncodeCastVotesReply encodes a CastVotesReply into a JSON byte slice. @@ -421,7 +485,7 @@ func EncodeCastVotesReply(cvr CastVotesReply) ([]byte, error) { } // DecodeCastVotesReply decodes a JSON byte slice into a CastVotesReply. -func DecodeCastVotesReplyVote(payload []byte) (*CastVotesReply, error) { +func DecodeCastVotesReply(payload []byte) (*CastVotesReply, error) { var cvr CastVotesReply err := json.Unmarshal(payload, &cvr) if err != nil { @@ -441,7 +505,7 @@ func EncodeSummaries(s Summaries) ([]byte, error) { } // DecodeSummaries decodes a JSON byte slice into a Summaries. -func DecodeSummariesVote(payload []byte) (*Summaries, error) { +func DecodeSummaries(payload []byte) (*Summaries, error) { var s Summaries err := json.Unmarshal(payload, &s) if err != nil { @@ -455,7 +519,7 @@ func DecodeSummariesVote(payload []byte) (*Summaries, error) { type Result struct { ID string `json:"id"` // Single unique word (e.g. yes) Description string `json:"description"` // Longer description of the vote - Bits uint64 `json:"bits"` // Bits used for this option + VoteBit uint64 `json:"votebit"` // Bits used for this option Votes uint64 `json:"votes"` // Votes cast for this option } @@ -465,7 +529,8 @@ type Summary struct { Status VoteStatusT `json:"status"` Duration uint32 `json:"duration"` StartBlockHeight uint32 `json:"startblockheight"` - EndBlockHeight string `json:"endblockheight"` + StartBlockHash string `json:"startblockhash"` + EndBlockHeight uint32 `json:"endblockheight"` EligibleTickets uint32 `json:"eligibletickets"` QuorumPercentage uint32 `json:"quorumpercentage"` PassPercentage uint32 `json:"passpercentage"` @@ -489,7 +554,7 @@ type SummariesReply struct { // BestBlock is the best block value that was used to prepare the // summaries. - BestBlock uint64 `json:"bestblock"` + BestBlock uint32 `json:"bestblock"` } // EncodeSummariesReply encodes a SummariesReply into a JSON byte slice. @@ -498,7 +563,7 @@ func EncodeSummariesReply(sr SummariesReply) ([]byte, error) { } // DecodeSummariesReply decodes a JSON byte slice into a SummariesReply. -func DecodeSummariesReplyVote(payload []byte) (*SummariesReply, error) { +func DecodeSummariesReply(payload []byte) (*SummariesReply, error) { var sr SummariesReply err := json.Unmarshal(payload, &sr) if err != nil { @@ -517,7 +582,7 @@ func EncodeInventory(i Inventory) ([]byte, error) { } // DecodeInventory decodes a JSON byte slice into a Inventory. -func DecodeInventoryVote(payload []byte) (*Inventory, error) { +func DecodeInventory(payload []byte) (*Inventory, error) { var i Inventory err := json.Unmarshal(payload, &i) if err != nil { @@ -536,7 +601,7 @@ type InventoryReply struct { // BestBlock is the best block value that was used to prepare the // inventory. - BestBlock uint64 `json:"bestblock"` + BestBlock uint32 `json:"bestblock"` } // EncodeInventoryReply encodes a InventoryReply into a JSON byte slice. @@ -545,7 +610,7 @@ func EncodeInventoryReply(ir InventoryReply) ([]byte, error) { } // DecodeInventoryReply decodes a JSON byte slice into a InventoryReply. -func DecodeInventoryReplyVote(payload []byte) (*InventoryReply, error) { +func DecodeInventoryReply(payload []byte) (*InventoryReply, error) { var ir InventoryReply err := json.Unmarshal(payload, &ir) if err != nil { diff --git a/politeiad/backend/tlogbe/plugins/comments.go b/politeiad/backend/tlogbe/plugins/comments.go index 6799ac553..f1fbe34ac 100644 --- a/politeiad/backend/tlogbe/plugins/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments.go @@ -54,6 +54,8 @@ type commentsPlugin struct { mutexes map[string]*sync.Mutex // [string]mutex } +// TODO move these to the plugin API since this is what the inclusion proof +// is for // commentAdd is the structure that is saved to disk when a comment is created // or edited. type commentAdd struct { @@ -266,7 +268,7 @@ func convertCommentAddFromBlobEntry(be store.BlobEntry) (*commentAdd, error) { var c commentAdd err = json.Unmarshal(b, &c) if err != nil { - return nil, fmt.Errorf("unmarshal index: %v", err) + return nil, fmt.Errorf("unmarshal commentAdd: %v", err) } return &c, nil @@ -304,7 +306,7 @@ func convertCommentDelFromBlobEntry(be store.BlobEntry) (*commentDel, error) { var c commentDel err = json.Unmarshal(b, &c) if err != nil { - return nil, fmt.Errorf("unmarshal index: %v", err) + return nil, fmt.Errorf("unmarshal commentDel: %v", err) } return &c, nil @@ -1552,7 +1554,10 @@ func (p *commentsPlugin) Setup() error { } // CommentsPluginNew returns a new comments plugin. -func CommentsPluginNew(id *identity.FullIdentity, backend *tlogbe.Tlogbe) *commentsPlugin { +func CommentsPluginNew(backend *tlogbe.Tlogbe, settings []backend.PluginSetting) *commentsPlugin { + // TODO these should be passed in as plugin settings + id := &identity.FullIdentity{} + return &commentsPlugin{ id: id, backend: backend, diff --git a/politeiad/backend/tlogbe/plugins/dcrdata.go b/politeiad/backend/tlogbe/plugins/dcrdata.go index 9a236b32e..728dee953 100644 --- a/politeiad/backend/tlogbe/plugins/dcrdata.go +++ b/politeiad/backend/tlogbe/plugins/dcrdata.go @@ -5,11 +5,14 @@ package plugins import ( + "bytes" "crypto/tls" "encoding/json" "fmt" "io/ioutil" "net/http" + "strconv" + "strings" "sync" "time" @@ -17,12 +20,17 @@ import ( exptypes "github.com/decred/dcrdata/explorer/types/v2" pstypes "github.com/decred/dcrdata/pubsub/types/v3" "github.com/decred/politeia/plugins/dcrdata" + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/util" "github.com/decred/politeia/wsdcrdata" ) const ( // Dcrdata routes - routeBestBlock = "/api/block/best" + routeBestBlock = "/api/block/best" + routeBlockDetails = "/api/block/{height}" + routeTicketPool = "/api/stake/pool/b/{hash}/full" + routeTxsTrimmed = "/api/txs/trimmed" ) var ( @@ -36,8 +44,8 @@ type dcrdataPlugin struct { client *http.Client // HTTP client ws *wsdcrdata.Client // Websocket client - // bestBlock is the cached best block. This field is kept up to - // date by the websocket connection. If the websocket connection + // bestBlock is the cached best block height. This field is kept up + // to date by the websocket connection. If the websocket connection // drops, the best block is marked as stale and is not marked as // current again until the connection has been re-established and // a new best block message is received. @@ -74,38 +82,124 @@ func (p *dcrdataPlugin) bestBlockIsStale() bool { return p.bestBlockStale } -// bestBlockHTTP fetches the best block from the dcrdata API. -func (p *dcrdataPlugin) bestBlockHTTP() (*v4.BlockDataBasic, error) { - url := p.host + routeBestBlock +func (p *dcrdataPlugin) makeReq(method string, route string, v interface{}) ([]byte, error) { + var ( + url = p.host + route + reqBody []byte + err error + ) - log.Tracef("dcrdata bestBlock: %v", url) + log.Tracef("%v %v", method, url) - r, err := p.client.Get(url) - log.Debugf("http connecting to %v", url) + // Setup request body + if v != nil { + reqBody, err = json.Marshal(v) + if err != nil { + return nil, err + } + } + + // Send request + req, err := http.NewRequest(method, url, bytes.NewReader(reqBody)) + if err != nil { + return nil, err + } + r, err := p.client.Do(req) if err != nil { return nil, err } defer r.Body.Close() + // Handle response if r.StatusCode != http.StatusOK { body, err := ioutil.ReadAll(r.Body) if err != nil { - return nil, fmt.Errorf("dcrdata error: %v %v %v", - r.StatusCode, url, err) + return nil, fmt.Errorf("%v %v %v %v", + r.StatusCode, method, url, err) } - return nil, fmt.Errorf("dcrdata error: %v %v %s", - r.StatusCode, url, body) + return nil, fmt.Errorf("%v %v %v %s", + r.StatusCode, method, url, body) + } + + resBody := util.ConvertBodyToByteArray(r.Body, false) + return resBody, nil +} + +// bestBlockHTTP fetches and returns the best block from the dcrdata http API. +func (p *dcrdataPlugin) bestBlockHTTP() (*v4.BlockDataBasic, error) { + resBody, err := p.makeReq(http.MethodGet, routeBestBlock, nil) + if err != nil { + return nil, err + } + + var bdb v4.BlockDataBasic + err = json.Unmarshal(resBody, &bdb) + if err != nil { + return nil, err + } + + return &bdb, nil +} + +// blockDetailsHTTP fetches and returns the block details from the dcrdata API +// for the provided block height. +func (p *dcrdataPlugin) blockDetailsHTTP(height uint32) (*v4.BlockDataBasic, error) { + h := strconv.FormatUint(uint64(height), 10) + + route := strings.Replace(routeBlockDetails, "{height}", h, 1) + resBody, err := p.makeReq(http.MethodGet, route, nil) + if err != nil { + return nil, err } var bdb v4.BlockDataBasic - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&bdb); err != nil { + err = json.Unmarshal(resBody, &bdb) + if err != nil { return nil, err } return &bdb, nil } +// ticketPoolHTTP fetches and returns the list of tickets in the ticket pool +// from the dcrdata API at the provided block hash. +func (p *dcrdataPlugin) ticketPoolHTTP(blockHash string) ([]string, error) { + route := strings.Replace(routeTicketPool, "{hash}", blockHash, 1) + route += "?sort=true" + resBody, err := p.makeReq(http.MethodGet, route, nil) + if err != nil { + return nil, err + } + + var tickets []string + err = json.Unmarshal(resBody, &tickets) + if err != nil { + return nil, err + } + + return tickets, nil +} + +// txsTrimmedHTTP fetches and returns the TrimmedTx from the dcrdata API for +// the provided tx IDs. +func (p *dcrdataPlugin) txsTrimmedHTTP(txIDs []string) ([]v4.TrimmedTx, error) { + t := v4.Txns{ + Transactions: txIDs, + } + resBody, err := p.makeReq(http.MethodPost, routeTxsTrimmed, t) + if err != nil { + return nil, err + } + + var txs []v4.TrimmedTx + err = json.Unmarshal(resBody, &txs) + if err != nil { + return nil, err + } + + return txs, nil +} + // cmdBestBlock returns the best block. If the dcrdata websocket has been // disconnected the best block will be fetched from the dcrdata API. If dcrdata // cannot be reached then the most recent cached best block will be returned @@ -152,14 +246,14 @@ func (p *dcrdataPlugin) cmdBestBlock(payload string) (string, error) { default: // Unable to fetch the best block manually and there is no // stale cached value to return. - return "", err + return "", fmt.Errorf("bestBlockHTTP: %v", err) } } // Prepare reply bbr := dcrdata.BestBlockReply{ - Status: status, - BestBlock: bb, + Status: status, + Height: bb, } reply, err := dcrdata.EncodeBestBlockReply(bbr) if err != nil { @@ -169,6 +263,125 @@ func (p *dcrdataPlugin) cmdBestBlock(payload string) (string, error) { return string(reply), nil } +func (p *dcrdataPlugin) cmdBlockDetails(payload string) (string, error) { + log.Tracef("dcrdata cmdBlockDetails: %v", payload) + + // Decode payload + bd, err := dcrdata.DecodeBlockDetails([]byte(payload)) + if err != nil { + return "", err + } + + // Fetch block details + bdb, err := p.blockDetailsHTTP(bd.Height) + if err != nil { + return "", fmt.Errorf("blockDetailsHTTP: %v", err) + } + + // Prepare reply + bdr := dcrdata.BlockDetailsReply{ + Block: *bdb, + } + reply, err := dcrdata.EncodeBlockDetailsReply(bdr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *dcrdataPlugin) cmdTicketPool(payload string) (string, error) { + log.Tracef("dcrdata cmdTicketPool: %v", payload) + + // Decode payload + tp, err := dcrdata.DecodeTicketPool([]byte(payload)) + if err != nil { + return "", err + } + + // Get the ticket pool + tickets, err := p.ticketPoolHTTP(tp.BlockHash) + if err != nil { + return "", fmt.Errorf("ticketPoolHTTP: %v", err) + } + + // Prepare reply + tpr := dcrdata.TicketPoolReply{ + Tickets: tickets, + } + reply, err := dcrdata.EncodeTicketPoolReply(tpr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *dcrdataPlugin) cmdTxsTrimmed(payload string) (string, error) { + log.Tracef("cmdTxsTrimmed: %v", payload) + + // Decode payload + tt, err := dcrdata.DecodeTxsTrimmed([]byte(payload)) + if err != nil { + return "", err + } + + // Get trimmed txs + txs, err := p.txsTrimmedHTTP(tt.TxIDs) + if err != nil { + return "", fmt.Errorf("txsTrimmedHTTP: %v", err) + } + + // Prepare reply + ttr := dcrdata.TxsTrimmedReply{ + Txs: txs, + } + reply, err := dcrdata.EncodeTxsTrimmedReply(ttr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +// Cmd executes a plugin command. +// +// This function satisfies the Plugin interface. +func (p *dcrdataPlugin) Cmd(cmd, payload string) (string, error) { + log.Tracef("dcrdata Cmd: %v", cmd) + + switch cmd { + case dcrdata.CmdBestBlock: + return p.cmdBestBlock(payload) + case dcrdata.CmdBlockDetails: + return p.cmdBlockDetails(payload) + case dcrdata.CmdTicketPool: + return p.cmdTicketPool(payload) + case dcrdata.CmdTxsTrimmed: + return p.cmdTxsTrimmed(payload) + } + + return "", ErrInvalidPluginCmd +} + +// Hook executes a plugin hook. +// +// This function satisfies the Plugin interface. +func (p *dcrdataPlugin) Hook(h HookT, payload string) error { + log.Tracef("dcrdata Hook: %v %v", h, payload) + + return nil +} + +// Fsck performs a plugin filesystem check. +// +// This function satisfies the Plugin interface. +func (p *dcrdataPlugin) Fsck() error { + log.Tracef("dcrdata Fsck") + + return nil +} + func (p *dcrdataPlugin) websocketMonitor() { defer func() { log.Infof("Dcrdata websocket closed") @@ -250,38 +463,6 @@ func (p *dcrdataPlugin) websocketSetup() { go p.websocketMonitor() } -// Cmd executes a plugin command. -// -// This function satisfies the Plugin interface. -func (p *dcrdataPlugin) Cmd(cmd, payload string) (string, error) { - log.Tracef("dcrdata Cmd: %v", cmd) - - switch cmd { - case dcrdata.CmdBestBlock: - return p.cmdBestBlock(payload) - } - - return "", ErrInvalidPluginCmd -} - -// Hook executes a plugin hook. -// -// This function satisfies the Plugin interface. -func (p *dcrdataPlugin) Hook(h HookT, payload string) error { - log.Tracef("dcrdata Hook: %v %v", h, payload) - - return nil -} - -// Fsck performs a plugin filesystem check. -// -// This function satisfies the Plugin interface. -func (p *dcrdataPlugin) Fsck() error { - log.Tracef("dcrdata Fsck") - - return nil -} - // Setup performs any plugin setup work that needs to be done. // // This function satisfies the Plugin interface. @@ -297,7 +478,10 @@ func (p *dcrdataPlugin) Setup() error { return nil } -func DcrdataPluginNew(dcrdataHost string) *dcrdataPlugin { +func DcrdataPluginNew(settings []backend.PluginSetting) *dcrdataPlugin { + // TODO these should be passed in as plugin settings + var dcrdataHost string + // Setup http client client := &http.Client{ Timeout: 1 * time.Minute, diff --git a/politeiad/backend/tlogbe/plugins/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote.go index 563978aff..b480d97ca 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote.go @@ -11,9 +11,20 @@ import ( "encoding/json" "errors" "fmt" + "io/ioutil" + "os" + "path/filepath" "strconv" + "strings" + "sync" "time" + "github.com/decred/dcrd/chaincfg" + "github.com/decred/dcrd/chaincfg/chainhash" + "github.com/decred/dcrd/dcrec/secp256k1" + "github.com/decred/dcrd/dcrutil" + "github.com/decred/dcrd/wire" + "github.com/decred/politeia/plugins/dcrdata" "github.com/decred/politeia/plugins/ticketvote" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" @@ -23,50 +34,149 @@ import ( ) const ( + // ticketVoteDirname is the ticket vote data directory name. + ticketVoteDirname = "ticketvote" + + // Filenames of memoized data saved to data dir. + summaryFilename = "{token}-summary.json" + // Blob entry data descriptors - dataDescriptorAuthorize = "ticketvoteauthorize" - dataDescriptorStart = "ticketvotestart" + dataDescriptorAuthorizeVote = "authorizevote" + dataDescriptorStartVote = "startvote" + dataDescriptorCastVote = "castvote" // Prefixes that are appended to key-value store keys before // storing them in the log leaf ExtraData field. - keyPrefixAuthorize = "ticketvoteauthorize" - keyPrefixStart = "ticketvotestart" + keyPrefixAuthorizeVote = "authorizevote:" + keyPrefixStartVote = "startvote:" + keyPrefixCastVote = "castvote:" ) var ( _ Plugin = (*ticketVotePlugin)(nil) + + // Local errors + errRecordNotFound = errors.New("record not found") ) // ticketVotePlugin satsifies the Plugin interface. type ticketVotePlugin struct { - id *identity.FullIdentity + sync.Mutex backend *tlogbe.Tlogbe + + // Plugin settings + id *identity.FullIdentity + activeNetParams *chaincfg.Params + voteDurationMin uint32 // In blocks + voteDurationMax uint32 // In blocks + + // dataDir is the ticket vote plugin data directory. The only data + // that is stored here is memoized data that can be re-created at + // any time by walking the trillian tree. Example, the vote summary + // once a record vote has ended. + dataDir string + + // castVotes contains the cast votes of ongoing record votes. + // TODO load on startup and cleanup once vote is finished. If we + // save these to the data dir instead of memory then there won't + // be anything to build on startup. + castVotes map[string]map[string]string // [token][ticket]voteBit + + // Mutexes contains a mutex for each record. The mutexes are lazy + // loaded. + mutexes map[string]*sync.Mutex // [string]mutex } -// authorize is the structure that is saved to disk when a vote is authorized -// or a previous authorization is revoked. -type authorize struct { - Token string `json:"token"` // Record token - Version uint32 `json:"version"` // Record version - Action string `json:"action"` // Authorize or revoke - PublicKey string `json:"publickey"` // Public key used for signature - Signature string `json:"signature"` // Signature of token+version+action - Timestamp int64 `json:"timestamp"` // Received UNIX timestamp - Receipt string `json:"receipt"` // Server signature of client signature +func (p *ticketVotePlugin) cachedCastVotes(token string) map[string]string { + p.Lock() + defer p.Unlock() + + // Return a copy of the map + cv, ok := p.castVotes[token] + if !ok { + return map[string]string{} + } + c := make(map[string]string, len(cv)) + for k, v := range cv { + c[k] = v + } + + return c } -// start is the structure that is saved to disk when a vote is started. -// -// Signature is a signature of the SHA256 digest of the JSON encoded Vote -// struct. -type start struct { - Vote ticketvote.Vote `json:"vote"` - PublicKey string `json:"publickey"` - Signature string `json:"signature"` - StartBlockHeight uint32 `json:"startblockheight"` - StartBlockHash string `json:"startblockhash"` - EndBlockHeight uint32 `json:"endblockheight"` - EligibleTickets []string `json:"eligibletickets"` // Ticket hashes +func (p *ticketVotePlugin) cachedCastVotesSave(token, ticket, voteBit string) { + p.Lock() + defer p.Unlock() + + _, ok := p.castVotes[token] + if !ok { + p.castVotes[token] = make(map[string]string, 40960) // Ticket pool size + } + + p.castVotes[token][ticket] = voteBit +} + +func (p *ticketVotePlugin) cachedSummaryPath(token string) string { + fn := strings.Replace(summaryFilename, "{token}", token, 1) + return filepath.Join(p.dataDir, fn) +} + +func (p *ticketVotePlugin) cachedSummary(token string) (*ticketvote.Summary, error) { + fp := p.cachedSummaryPath(token) + + p.Lock() + defer p.Unlock() + + b, err := ioutil.ReadFile(fp) + if err != nil { + var e *os.PathError + if errors.As(err, &e) && !os.IsExist(err) { + // File does't exist + return nil, errRecordNotFound + } + return nil, err + } + + var s ticketvote.Summary + err = json.Unmarshal(b, &s) + if err != nil { + return nil, err + } + + return &s, nil +} + +func (p *ticketVotePlugin) cachedSummarySave(token string, s ticketvote.Summary) error { + b, err := json.Marshal(s) + if err != nil { + return err + } + + p.Lock() + defer p.Unlock() + + fp := p.cachedSummaryPath(token) + err = ioutil.WriteFile(fp, b, 0664) + if err != nil { + return err + } + + return nil +} + +// mutex returns the mutex for the specified record. +func (p *ticketVotePlugin) mutex(token string) *sync.Mutex { + p.Lock() + defer p.Unlock() + + m, ok := p.mutexes[token] + if !ok { + // Mutexes is lazy loaded + m = &sync.Mutex{} + p.mutexes[token] = m + } + + return m } func convertTicketVoteErrFromSignatureErr(err error) ticketvote.UserError { @@ -86,7 +196,83 @@ func convertTicketVoteErrFromSignatureErr(err error) ticketvote.UserError { } } -func convertAuthorizeFromBlobEntry(be store.BlobEntry) (*authorize, error) { +func convertAuthorizeVoteFromBlobEntry(be store.BlobEntry) (*ticketvote.AuthorizeVote, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorAuthorizeVote { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorAuthorizeVote) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + hash, err := hex.DecodeString(be.Hash) + if err != nil { + return nil, fmt.Errorf("decode hash: %v", err) + } + if !bytes.Equal(util.Digest(b), hash) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), hash) + } + var av ticketvote.AuthorizeVote + err = json.Unmarshal(b, &av) + if err != nil { + return nil, fmt.Errorf("unmarshal AuthorizeVote: %v", err) + } + + return &av, nil +} + +func convertStartVoteFromBlobEntry(be store.BlobEntry) (*ticketvote.StartVote, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorStartVote { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorStartVote) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + hash, err := hex.DecodeString(be.Hash) + if err != nil { + return nil, fmt.Errorf("decode hash: %v", err) + } + if !bytes.Equal(util.Digest(b), hash) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), hash) + } + var sv ticketvote.StartVote + err = json.Unmarshal(b, &sv) + if err != nil { + return nil, fmt.Errorf("unmarshal StartVote: %v", err) + } + + return &sv, nil +} + +func convertCastVoteFromBlobEntry(be store.BlobEntry) (*ticketvote.CastVote, error) { // Decode and validate data hint b, err := base64.StdEncoding.DecodeString(be.DataHint) if err != nil { @@ -97,9 +283,9 @@ func convertAuthorizeFromBlobEntry(be store.BlobEntry) (*authorize, error) { if err != nil { return nil, fmt.Errorf("unmarshal DataHint: %v", err) } - if dd.Descriptor != dataDescriptorAuthorize { + if dd.Descriptor != dataDescriptorCastVote { return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorAuthorize) + dd.Descriptor, dataDescriptorCastVote) } // Decode data @@ -115,103 +301,1182 @@ func convertAuthorizeFromBlobEntry(be store.BlobEntry) (*authorize, error) { return nil, fmt.Errorf("data is not coherent; got %x, want %x", util.Digest(b), hash) } - var a authorize - err = json.Unmarshal(b, &a) + var cv ticketvote.CastVote + err = json.Unmarshal(b, &cv) + if err != nil { + return nil, fmt.Errorf("unmarshal CastVote: %v", err) + } + + return &cv, nil +} + +func convertBlobEntryFromAuthorizeVote(av ticketvote.AuthorizeVote) (*store.BlobEntry, error) { + data, err := json.Marshal(av) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorAuthorizeVote, + }) + if err != nil { + return nil, err + } + be := store.BlobEntryNew(hint, data) + return &be, nil +} + +func convertBlobEntryFromStartVote(sv ticketvote.StartVote) (*store.BlobEntry, error) { + data, err := json.Marshal(sv) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorStartVote, + }) + if err != nil { + return nil, err + } + be := store.BlobEntryNew(hint, data) + return &be, nil +} + +func convertBlobEntryFromCastVote(cv ticketvote.CastVote) (*store.BlobEntry, error) { + data, err := json.Marshal(cv) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorCastVote, + }) + if err != nil { + return nil, err + } + be := store.BlobEntryNew(hint, data) + return &be, nil +} + +func authorizeVoteSave(client *tlogbe.PluginClient, av ticketvote.AuthorizeVote) error { + // Prepare blob + be, err := convertBlobEntryFromAuthorizeVote(av) + if err != nil { + return err + } + h, err := hex.DecodeString(be.Hash) + if err != nil { + return err + } + b, err := store.Blobify(*be) + if err != nil { + return err + } + + // Save blob + merkles, err := client.BlobsSave(keyPrefixAuthorizeVote, + [][]byte{b}, [][]byte{h}, false) + if err != nil { + return fmt.Errorf("BlobsSave: %v", err) + } + if len(merkles) != 1 { + return fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", + len(merkles)) + } + + return nil +} + +func authorizeVotes(client *tlogbe.PluginClient) ([]ticketvote.AuthorizeVote, error) { + // Retrieve blobs + blobs, err := client.BlobsByKeyPrefix(keyPrefixAuthorizeVote) + if err != nil { + return nil, err + } + + // Decode blobs + auths := make([]ticketvote.AuthorizeVote, 0, len(blobs)) + for _, v := range blobs { + be, err := store.Deblob(v) + if err != nil { + return nil, err + } + a, err := convertAuthorizeVoteFromBlobEntry(*be) + if err != nil { + return nil, err + } + auths = append(auths, *a) + } + + return auths, nil +} + +func startVoteSave(client *tlogbe.PluginClient, sv ticketvote.StartVote) error { + // Prepare blob + be, err := convertBlobEntryFromStartVote(sv) + if err != nil { + return err + } + h, err := hex.DecodeString(be.Hash) + if err != nil { + return err + } + b, err := store.Blobify(*be) + if err != nil { + return err + } + + // Save blob + merkles, err := client.BlobsSave(keyPrefixStartVote, + [][]byte{b}, [][]byte{h}, false) + if err != nil { + return fmt.Errorf("BlobsSave: %v", err) + } + if len(merkles) != 1 { + return fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", + len(merkles)) + } + + return nil +} + +// startVote returns the StartVote for the provided record if one exists. +func startVote(client *tlogbe.PluginClient) (*ticketvote.StartVote, error) { + // Retrieve blobs + blobs, err := client.BlobsByKeyPrefix(keyPrefixStartVote) + if err != nil { + return nil, err + } + switch len(blobs) { + case 0: + // A start vote does not exist + return nil, nil + case 1: + // A start vote exists; continue + default: + // This should not happen. There should only ever be a max of + // one start vote. + return nil, fmt.Errorf("multiple start votes found (%v) for record %x", + len(blobs), client.Token) + } + + // Decode blob + be, err := store.Deblob(blobs[0]) + if err != nil { + return nil, err + } + sv, err := convertStartVoteFromBlobEntry(*be) + if err != nil { + return nil, err + } + + return sv, nil +} + +func castVotes(client *tlogbe.PluginClient) ([]ticketvote.CastVote, error) { + // Retrieve blobs + blobs, err := client.BlobsByKeyPrefix(keyPrefixCastVote) if err != nil { - return nil, fmt.Errorf("unmarshal index: %v", err) + return nil, err + } + + // Decode blobs + votes := make([]ticketvote.CastVote, 0, len(blobs)) + for _, v := range blobs { + be, err := store.Deblob(v) + if err != nil { + return nil, err + } + cv, err := convertCastVoteFromBlobEntry(*be) + if err != nil { + return nil, err + } + votes = append(votes, *cv) + } + + return votes, nil +} + +func castVoteSave(client *tlogbe.PluginClient, cv ticketvote.CastVote) error { + // Prepare blob + be, err := convertBlobEntryFromCastVote(cv) + if err != nil { + return err + } + h, err := hex.DecodeString(be.Hash) + if err != nil { + return err + } + b, err := store.Blobify(*be) + if err != nil { + return err + } + + // Save blob + merkles, err := client.BlobsSave(keyPrefixCastVote, + [][]byte{b}, [][]byte{h}, false) + if err != nil { + return fmt.Errorf("BlobsSave: %v", err) + } + if len(merkles) != 1 { + return fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", + len(merkles)) + } + + return nil +} + +// bestBlock fetches the best block from the dcrdata plugin and returns it. If +// the dcrdata connection is not active, an error WILL NOT BE returned. The +// dcrdata cached best block height will be returned even though it may be +// stale. Use bestBlockSafe() if the caller requires a guarantee that the best +// block is not stale. +func (p *ticketVotePlugin) bestBlock() (uint32, error) { + // Get best block + payload, err := dcrdata.EncodeBestBlock(dcrdata.BestBlock{}) + if err != nil { + return 0, err + } + _, reply, err := p.backend.Plugin(dcrdata.ID, + dcrdata.CmdBestBlock, string(payload)) + if err != nil { + return 0, fmt.Errorf("Plugin %v %v: %v", + dcrdata.ID, dcrdata.CmdBestBlock, err) + } + + // Handle response + bbr, err := dcrdata.DecodeBestBlockReply([]byte(reply)) + if err != nil { + return 0, err + } + if bbr.Height == 0 { + return 0, fmt.Errorf("invalid best block height 0") + } + + return bbr.Height, nil +} + +// bestBlockSafe fetches the best block from the dcrdata plugin and returns it. +// If the dcrdata connection is not active, an error WILL BE returned. +func (p *ticketVotePlugin) bestBlockSafe() (uint32, error) { + // Get best block + payload, err := dcrdata.EncodeBestBlock(dcrdata.BestBlock{}) + if err != nil { + return 0, err + } + _, reply, err := p.backend.Plugin(dcrdata.ID, + dcrdata.CmdBestBlock, string(payload)) + if err != nil { + return 0, fmt.Errorf("Plugin %v %v: %v", + dcrdata.ID, dcrdata.CmdBestBlock, err) + } + + // Handle response + bbr, err := dcrdata.DecodeBestBlockReply([]byte(reply)) + if err != nil { + return 0, err + } + if bbr.Status != dcrdata.StatusConnected { + // The dcrdata connection is down. The best block cannot be + // trusted as being accurate. + return 0, fmt.Errorf("dcrdata connection is down") + } + if bbr.Height == 0 { + return 0, fmt.Errorf("invalid best block height 0") + } + + return bbr.Height, nil +} + +type commitmentAddr struct { + ticket string // Ticket hash + addr string // Commitment address + err error // Error if one occured +} + +func (p *ticketVotePlugin) largestCommitmentAddrs(tickets []string) ([]commitmentAddr, error) { + // Get tx details + tt := dcrdata.TxsTrimmed{ + TxIDs: tickets, + } + payload, err := dcrdata.EncodeTxsTrimmed(tt) + if err != nil { + return nil, err + } + _, reply, err := p.backend.Plugin(dcrdata.ID, + dcrdata.CmdTxsTrimmed, string(payload)) + if err != nil { + return nil, fmt.Errorf("Plugin %v %v: %v", + dcrdata.ID, dcrdata.CmdTxsTrimmed, err) + } + ttr, err := dcrdata.DecodeTxsTrimmedReply([]byte(reply)) + if err != nil { + return nil, err + } + + // Find the largest commitment address for each tx + addrs := make([]commitmentAddr, 0, len(ttr.Txs)) + for _, tx := range ttr.Txs { + var ( + bestAddr string // Addr with largest commitment amount + bestAmt float64 // Largest commitment amount + addrErr error // Error if one is encountered + ) + for _, vout := range tx.Vout { + scriptPubKey := vout.ScriptPubKeyDecoded + switch { + case scriptPubKey.CommitAmt == nil: + // No commitment amount; continue + case len(scriptPubKey.Addresses) == 0: + // No commitment address; continue + case *scriptPubKey.CommitAmt > bestAmt: + // New largest commitment address found + bestAddr = scriptPubKey.Addresses[0] + bestAmt = *scriptPubKey.CommitAmt + } + } + if bestAddr == "" || bestAmt == 0.0 { + addrErr = fmt.Errorf("no largest commitment address found") + } + + // Store result + addrs = append(addrs, commitmentAddr{ + ticket: tx.TxID, + addr: bestAddr, + err: addrErr, + }) + } + + return addrs, nil +} + +// startReply fetches all required data and returns a StartReply. +func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, error) { + // Get the best block height + bb, err := p.bestBlockSafe() + if err != nil { + return nil, fmt.Errorf("bestBlockSafe: %v", err) + } + + // Find the snapshot height. Subtract the ticket maturity from the + // block height to get into unforkable territory. + ticketMaturity := uint32(p.activeNetParams.TicketMaturity) + snapshotHeight := bb - ticketMaturity + + // Fetch the block details for the snapshot height. We need the + // block hash in order to fetch the ticket pool snapshot. + bd := dcrdata.BlockDetails{ + Height: snapshotHeight, + } + payload, err := dcrdata.EncodeBlockDetails(bd) + if err != nil { + return nil, err + } + _, reply, err := p.backend.Plugin(dcrdata.ID, + dcrdata.CmdBlockDetails, string(payload)) + if err != nil { + return nil, fmt.Errorf("Plugin %v %v: %v", + dcrdata.ID, dcrdata.CmdBlockDetails, err) + } + bdr, err := dcrdata.DecodeBlockDetailsReply([]byte(reply)) + if err != nil { + return nil, err + } + if bdr.Block.Hash == "" { + return nil, fmt.Errorf("invalid block hash for height %v", snapshotHeight) + } + snapshotHash := bdr.Block.Hash + + // Fetch the ticket pool snapshot + tp := dcrdata.TicketPool{ + BlockHash: snapshotHash, + } + payload, err = dcrdata.EncodeTicketPool(tp) + if err != nil { + return nil, err + } + _, reply, err = p.backend.Plugin(dcrdata.ID, + dcrdata.CmdTicketPool, string(payload)) + if err != nil { + return nil, fmt.Errorf("Plugin %v %v: %v", + dcrdata.ID, dcrdata.CmdTicketPool, err) + } + tpr, err := dcrdata.DecodeTicketPoolReply([]byte(reply)) + if err != nil { + return nil, err + } + if len(tpr.Tickets) == 0 { + return nil, fmt.Errorf("no tickets found for block %v %v", + snapshotHeight, snapshotHash) + } + + return &ticketvote.StartReply{ + StartBlockHeight: snapshotHeight, + StartBlockHash: snapshotHash, + EndBlockHeight: snapshotHeight + duration, + EligibleTickets: tpr.Tickets, + }, nil +} + +// voteMessageVerify verifies a cast vote message is properly signed. Copied +// from: github.com/decred/dcrd/blob/0fc55252f912756c23e641839b1001c21442c38a/rpcserver.go#L5605 +func (p *ticketVotePlugin) voteMessageVerify(address, message, signature string) (bool, error) { + // Decode the provided address. + addr, err := dcrutil.DecodeAddress(address) + if err != nil { + return false, fmt.Errorf("Could not decode address: %v", + err) + } + + // Only P2PKH addresses are valid for signing. + if _, ok := addr.(*dcrutil.AddressPubKeyHash); !ok { + return false, fmt.Errorf("Address is not a pay-to-pubkey-hash "+ + "address: %v", address) + } + + // Decode base64 signature. + sig, err := base64.StdEncoding.DecodeString(signature) + if err != nil { + return false, fmt.Errorf("Malformed base64 encoding: %v", err) + } + + // Validate the signature - this just shows that it was valid at all. + // we will compare it with the key next. + var buf bytes.Buffer + wire.WriteVarString(&buf, 0, "Decred Signed Message:\n") + wire.WriteVarString(&buf, 0, message) + expectedMessageHash := chainhash.HashB(buf.Bytes()) + pk, wasCompressed, err := secp256k1.RecoverCompact(sig, + expectedMessageHash) + if err != nil { + // Mirror Bitcoin Core behavior, which treats error in + // RecoverCompact as invalid signature. + return false, nil + } + + // Reconstruct the pubkey hash. + dcrPK := pk + var serializedPK []byte + if wasCompressed { + serializedPK = dcrPK.SerializeCompressed() + } else { + serializedPK = dcrPK.SerializeUncompressed() + } + a, err := dcrutil.NewAddressSecpPubKey(serializedPK, p.activeNetParams) + if err != nil { + // Again mirror Bitcoin Core behavior, which treats error in + // public key reconstruction as invalid signature. + return false, nil + } + + // Return boolean if addresses match. + return a.EncodeAddress() == address, nil +} + +func (p *ticketVotePlugin) voteSignatureVerify(v ticketvote.Vote, addr string) error { + msg := v.Token + v.Ticket + v.VoteBit + + // Convert hex signature to base64. The voteMessageVerify function + // expects bas64. + b, err := hex.DecodeString(v.Signature) + if err != nil { + return fmt.Errorf("invalid hex") + } + sig := base64.StdEncoding.EncodeToString(b) + + // Verify message + validated, err := p.voteMessageVerify(addr, msg, sig) + if err != nil { + return err + } + if !validated { + return fmt.Errorf("could not verify message") + } + + return nil +} + +func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { + log.Tracef("ticketvote cmdAuthorize: %v", payload) + + // Decode payload + a, err := ticketvote.DecodeAuthorize([]byte(payload)) + if err != nil { + return "", err + } + + // Verify signature + version := strconv.FormatUint(uint64(a.Version), 10) + msg := a.Token + version + string(a.Action) + err = util.VerifySignature(a.Signature, a.PublicKey, msg) + if err != nil { + return "", convertTicketVoteErrFromSignatureErr(err) + } + + // Get plugin client + tokenb, err := hex.DecodeString(a.Token) + if err != nil { + return "", ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusTokenInvalid, + } + } + client, err := p.backend.PluginClient(tokenb) + if err != nil { + if err == backend.ErrRecordNotFound { + return "", ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusRecordNotFound, + } + } + return "", err + } + if client.State != tlogbe.RecordStateVetted { + return "", ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusRecordStateInvalid, + ErrorContext: []string{"record not vetted"}, + } + } + + // Verify record version + _, err = p.backend.GetVetted(tokenb, version) + if err != nil { + if err == backend.ErrRecordNotFound { + e := fmt.Sprintf("version %v not found", version) + return "", ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusRecordNotFound, + ErrorContext: []string{e}, + } + } + } + + // Verify action + switch a.Action { + case ticketvote.ActionAuthorize: + // This is allowed + case ticketvote.ActionRevoke: + // This is allowed + default: + e := fmt.Sprintf("%v not a valid action", a.Action) + return "", ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusAuthorizationInvalid, + ErrorContext: []string{e}, + } + } + + // The previous authorize votes must be retrieved to validate the + // new autorize vote. The lock must be held for the remainder of + // this function. + m := p.mutex(a.Token) + m.Lock() + defer m.Unlock() + + // Get any previous authorizations to verify that the new action + // is allowed based on the previous action. + auths, err := authorizeVotes(client) + if err != nil { + return "", err + } + var prevAction ticketvote.ActionT + if len(auths) > 0 { + prevAction = ticketvote.ActionT(auths[len(auths)-1].Action) + } + switch { + case len(auths) == 0: + // No previous actions. New action must be an authorize. + if a.Action != ticketvote.ActionAuthorize { + return "", ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusAuthorizationInvalid, + ErrorContext: []string{"no prev action; action must be authorize"}, + } + } + case prevAction == ticketvote.ActionAuthorize: + // Previous action was a authorize. This action must be revoke. + return "", ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusAuthorizationInvalid, + ErrorContext: []string{"prev action was authorize"}, + } + case prevAction == ticketvote.ActionRevoke: + // Previous action was a revoke. This action must be authorize. + return "", ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusAuthorizationInvalid, + ErrorContext: []string{"prev action was revoke"}, + } + } + + // Prepare authorize vote + receipt := p.id.SignMessage([]byte(a.Signature)) + auth := ticketvote.AuthorizeVote{ + Token: a.Token, + Version: a.Version, + Action: string(a.Action), + PublicKey: a.PublicKey, + Signature: a.Signature, + Timestamp: time.Now().Unix(), + Receipt: hex.EncodeToString(receipt[:]), + } + + // Save authorize vote + err = authorizeVoteSave(client, auth) + if err != nil { + return "", err + } + + // Prepare reply + ar := ticketvote.AuthorizeReply{ + Timestamp: auth.Timestamp, + Receipt: auth.Receipt, + } + reply, err := ticketvote.EncodeAuthorizeReply(ar) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func voteBitVerify(options []ticketvote.VoteOption, mask, bit uint64) error { + if len(options) == 0 { + return fmt.Errorf("no vote options found") + } + if bit == 0 { + return fmt.Errorf("invalid bit 0x%x", bit) + } + + // Verify bit is included in mask + if mask&bit != bit { + return fmt.Errorf("invalid mask 0x%x bit 0x%x", mask, bit) + } + + // Verify bit is inlcuded in vote options + for _, v := range options { + if v.Bit == bit { + // Bit matches one of the options. We're done. + return nil + } + } + + return fmt.Errorf("bit 0x%x not found in vote options") +} + +// TODO test this function +func voteVerifyDetails(vote ticketvote.VoteDetails, voteDurationMin, voteDurationMax uint32) error { + // Verify vote type + switch vote.Type { + case ticketvote.VoteTypeStandard: + // This is allowed + case ticketvote.VoteTypeRunoff: + // This is allowed + default: + e := fmt.Sprintf("invalid type %v", vote.Type) + return ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusVoteInvalid, + ErrorContext: []string{e}, + } + } + + // Verify vote params + switch { + case vote.Duration > voteDurationMax: + e := fmt.Sprintf("duration %v exceeds max duration %v", + vote.Duration, voteDurationMax) + return ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusVoteInvalid, + ErrorContext: []string{e}, + } + case vote.Duration < voteDurationMin: + e := fmt.Sprintf("duration %v under min duration %v", + vote.Duration, voteDurationMin) + return ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusVoteInvalid, + ErrorContext: []string{e}, + } + case vote.QuorumPercentage > 100: + e := fmt.Sprintf("quorum percent %v exceeds 100 percent", + vote.QuorumPercentage) + return ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusVoteInvalid, + ErrorContext: []string{e}, + } + case vote.PassPercentage > 100: + e := fmt.Sprintf("pass percent %v exceeds 100 percent", + vote.PassPercentage) + return ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusVoteInvalid, + ErrorContext: []string{e}, + } + } + + // Verify vote options. Different vote types have different + // requirements. + if len(vote.Options) == 0 { + return ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusVoteInvalid, + ErrorContext: []string{"no vote options found"}, + } + } + switch vote.Type { + case ticketvote.VoteTypeStandard, ticketvote.VoteTypeRunoff: + // These vote types only allow for approve/reject votes. Ensure + // that the only options present are approve/reject and that they + // use the vote option IDs specified by the ticketvote API. + if len(vote.Options) != 2 { + e := fmt.Sprintf("vote options count got %v, want 2", + len(vote.Options)) + return ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusVoteInvalid, + ErrorContext: []string{e}, + } + } + // map[optionID]found + options := map[string]bool{ + ticketvote.VoteOptionIDApprove: false, + ticketvote.VoteOptionIDReject: false, + } + for _, v := range vote.Options { + switch v.ID { + case ticketvote.VoteOptionIDApprove: + options[v.ID] = true + case ticketvote.VoteOptionIDReject: + options[v.ID] = true + } + } + missing := make([]string, 0, 2) + for k, v := range options { + if !v { + // Option ID was not found + missing = append(missing, k) + } + } + if len(missing) > 0 { + e := fmt.Sprintf("vote option IDs not found: %v", + strings.Join(missing, ",")) + return ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusVoteInvalid, + ErrorContext: []string{e}, + } + } + } + + // Verify vote bits are somewhat sane + for _, v := range vote.Options { + err := voteBitVerify(vote.Options, vote.Mask, v.Bit) + if err != nil { + return ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusVoteInvalid, + ErrorContext: []string{err.Error()}, + } + } + } + + return nil +} + +func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { + log.Tracef("ticketvote cmdStart: %v", payload) + + // Decode payload + s, err := ticketvote.DecodeStart([]byte(payload)) + if err != nil { + return "", err + } + + // Verify signature + vb, err := json.Marshal(s.Vote) + if err != nil { + return "", err + } + msg := hex.EncodeToString(util.Digest(vb)) + err = util.VerifySignature(s.Signature, s.PublicKey, msg) + if err != nil { + return "", convertTicketVoteErrFromSignatureErr(err) + } + + // Verify vote options and params + err = voteVerifyDetails(s.Vote, p.voteDurationMin, p.voteDurationMax) + if err != nil { + return "", err + } + + // Get plugin client + tokenb, err := hex.DecodeString(s.Vote.Token) + if err != nil { + return "", ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusTokenInvalid, + } + } + client, err := p.backend.PluginClient(tokenb) + if err != nil { + if err == backend.ErrRecordNotFound { + return "", ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusRecordNotFound, + } + } + return "", err + } + if client.State != tlogbe.RecordStateVetted { + return "", ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusRecordStateInvalid, + ErrorContext: []string{"record not vetted"}, + } + } + + // Verify record version + version := strconv.FormatUint(uint64(s.Vote.Version), 10) + _, err = p.backend.GetVetted(tokenb, version) + if err != nil { + if err == backend.ErrRecordNotFound { + e := fmt.Sprintf("version %v not found", version) + return "", ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusRecordNotFound, + ErrorContext: []string{e}, + } + } + } + + // Verify vote authorization + auths, err := authorizeVotes(client) + if err != nil { + return "", err + } + if len(auths) == 0 { + return "", ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusAuthorizationInvalid, + ErrorContext: []string{"authorization not found"}, + } + } + action := ticketvote.ActionT(auths[len(auths)-1].Action) + if action != ticketvote.ActionAuthorize { + return "", ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusAuthorizationInvalid, + ErrorContext: []string{"not authorized"}, + } + } + + // Get vote blockchain data + sr, err := p.startReply(s.Vote.Duration) + if err != nil { + return "", err + } + + // Any previous start vote must be retrieved to verify that a vote + // has not already been started. The lock must be held for the + // remainder of this function. + m := p.mutex(s.Vote.Token) + m.Lock() + defer m.Unlock() + + // Verify vote has not already been started + svp, err := startVote(client) + if err != nil { + return "", err + } + if svp != nil { + // Vote has already been started + return "", ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusVoteStatusInvalid, + ErrorContext: []string{"vote already started"}, + } + } + + // Prepare start vote + sv := ticketvote.StartVote{ + Vote: s.Vote, + PublicKey: s.PublicKey, + Signature: s.Signature, + StartBlockHeight: sr.StartBlockHeight, + StartBlockHash: sr.StartBlockHash, + EndBlockHeight: sr.EndBlockHeight, + EligibleTickets: sr.EligibleTickets, + } + + // Save start vote + err = startVoteSave(client, sv) + if err != nil { + return "", fmt.Errorf("startVoteSave: %v", err) + } + + // Prepare reply + reply, err := ticketvote.EncodeStartReply(*sr) + if err != nil { + return "", err } - return &a, nil + return string(reply), nil } -func convertBlobEntryFromAuthorize(a authorize) (*store.BlobEntry, error) { - data, err := json.Marshal(a) - if err != nil { - return nil, err +func (p *ticketVotePlugin) cmdStartRunoff(payload string) (string, error) { + log.Tracef("ticketvote cmdStartRunoff: %v", payload) + + return "", nil +} + +// ballotExitWithErr applies the provided vote error to each of the cast vote +// replies then returns the encoded ballot reply. +func ballotExitWithErr(votes []ticketvote.Vote, errCode ticketvote.VoteErrorT, errContext string) (string, error) { + token := votes[0].Token + receipts := make([]ticketvote.VoteReply, len(votes)) + for k, v := range votes { + // Its possible that cast votes were provided for different + // records. This is not allowed. Verify the token is the same + // before applying the provided error. + if v.Token != token { + // Token is not the same. Use multiple record vote error. + e := ticketvote.VoteErrorMultipleRecordVotes + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = ticketvote.VoteError[e] + continue + } + + // Use the provided vote error + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = errCode + receipts[k].ErrorContext = errContext } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorAuthorize, - }) + + // Prepare reply + br := ticketvote.BallotReply{ + Receipts: receipts, + } + reply, err := ticketvote.EncodeBallotReply(br) if err != nil { - return nil, err + return "", err } - be := store.BlobEntryNew(hint, data) - return &be, nil + return string(reply), nil } -func authorizeSave(client *tlogbe.PluginClient, a authorize) error { - // Prepare blob - be, err := convertBlobEntryFromAuthorize(a) +// TODO test this when casting large blocks of votes +// cmdBallot casts a ballot of votes. This function will not return a user +// error if one occurs. It will instead return the ballot reply with the error +// included in the invidiual cast vote reply that it applies to. +func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { + log.Tracef("ticketvote cmdBallot: %v", payload) + + // Decode payload + ballot, err := ticketvote.DecodeBallot([]byte(payload)) if err != nil { - return err + return "", err } - h, err := hex.DecodeString(be.Hash) + + // Verify there is work to do + if len(ballot.Votes) == 0 { + // Nothing to do + br := ticketvote.BallotReply{ + Receipts: []ticketvote.VoteReply{}, + } + reply, err := ticketvote.EncodeBallotReply(br) + if err != nil { + return "", err + } + return string(reply), nil + } + + // Get plugin client + var ( + votes = ballot.Votes + token = votes[0].Token + ) + tokenb, err := hex.DecodeString(token) if err != nil { - return err + e := ticketvote.VoteErrorTokenInvalid + c := fmt.Sprintf("%v: not hex", ticketvote.VoteError[e]) + return ballotExitWithErr(votes, e, c) } - b, err := store.Blobify(*be) + client, err := p.backend.PluginClient(tokenb) if err != nil { - return err + if err == backend.ErrRecordNotFound { + e := ticketvote.VoteErrorRecordNotFound + c := fmt.Sprintf("%v: %v", ticketvote.VoteError[e], token) + return ballotExitWithErr(votes, e, c) + } + return "", err + } + if client.State != tlogbe.RecordStateVetted { + e := ticketvote.VoteErrorVoteStatusInvalid + c := fmt.Sprintf("%v: record is unvetted", ticketvote.VoteError[e]) + return ballotExitWithErr(votes, e, c) } - // Save blob - merkles, err := client.BlobsSave(keyPrefixAuthorize, - [][]byte{b}, [][]byte{h}, false) + // Verify record vote status + sv, err := startVote(client) if err != nil { - return fmt.Errorf("BlobsSave: %v", err) + return "", err } - if len(merkles) != 1 { - return fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", - len(merkles)) + if sv == nil { + // Vote has not been started yet + e := ticketvote.VoteErrorVoteStatusInvalid + c := fmt.Sprintf("%v: vote not started", ticketvote.VoteError[e]) + return ballotExitWithErr(votes, e, c) + } + bb, err := p.bestBlockSafe() + if err != nil { + return "", err + } + if bb >= sv.EndBlockHeight { + // Vote has ended + e := ticketvote.VoteErrorVoteStatusInvalid + c := fmt.Sprintf("%v: vote has ended", ticketvote.VoteError[e]) + return ballotExitWithErr(votes, e, c) } - return nil -} + // Put eligible tickets in a map for easy lookups + eligible := make(map[string]struct{}, len(sv.EligibleTickets)) + for _, v := range sv.EligibleTickets { + eligible[v] = struct{}{} + } -func authorizations(client *tlogbe.PluginClient) ([]authorize, error) { - // Retrieve blobs - blobs, err := client.BlobsByKeyPrefix(keyPrefixAuthorize) + // Obtain largest commitment addresses for each ticket. The vote + // must be signed using the largest commitment address. + tickets := make([]string, 0, len(ballot.Votes)) + for _, v := range ballot.Votes { + tickets = append(tickets, v.Ticket) + } + addrs, err := p.largestCommitmentAddrs(tickets) if err != nil { - return nil, err + return "", fmt.Errorf("largestCommitmentAddrs: %v", err) } - // Decode blobs - auths := make([]authorize, 0, len(blobs)) - for _, v := range blobs { - be, err := store.Deblob(v) + // The lock must be held for the remainder of the function to + // ensure duplicate votes cannot be cast. + m := p.mutex(token) + m.Lock() + defer m.Unlock() + + // castVotes contains the tickets that have alread voted + castVotes := p.cachedCastVotes(token) + + // Verify and save votes + receipts := make([]ticketvote.VoteReply, len(votes)) + for k, v := range votes { + // Set receipt ticket + receipts[k].Ticket = v.Ticket + + // Verify token is the same + if v.Token != token { + e := ticketvote.VoteErrorMultipleRecordVotes + receipts[k].ErrorCode = e + receipts[k].ErrorContext = ticketvote.VoteError[e] + continue + } + + // Verify vote bit + bit, err := strconv.ParseUint(v.VoteBit, 16, 64) if err != nil { - return nil, err + e := ticketvote.VoteErrorVoteBitInvalid + receipts[k].ErrorCode = e + receipts[k].ErrorContext = ticketvote.VoteError[e] + continue } - a, err := convertAuthorizeFromBlobEntry(*be) + err = voteBitVerify(sv.Vote.Options, sv.Vote.Mask, bit) if err != nil { - return nil, err + e := ticketvote.VoteErrorVoteBitInvalid + receipts[k].ErrorCode = e + receipts[k].ErrorContext = fmt.Sprintf("%v: %v", + ticketvote.VoteError[e], err) + continue } - auths = append(auths, *a) - } - return auths, nil -} -func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { - log.Tracef("ticketvote cmdAuthorize: %v", payload) + // Verify vote signature + ca := addrs[k] + if ca.ticket != v.Ticket { + t := time.Now().Unix() + log.Errorf("cmdBallot: commitment addr mismatch %v: %v %v", + t, ca.ticket, v.Ticket) + e := ticketvote.VoteErrorInternalError + receipts[k].ErrorCode = e + receipts[k].ErrorContext = fmt.Sprintf("%v: %v", + ticketvote.VoteError[e], t) + continue + } + if ca.err != nil { + t := time.Now().Unix() + log.Errorf("cmdBallot: commitment addr error %v: %v %v", + t, ca.ticket, ca.err) + e := ticketvote.VoteErrorInternalError + receipts[k].ErrorCode = e + receipts[k].ErrorContext = fmt.Sprintf("%v: %v", + ticketvote.VoteError[e], t) + continue + } + err = p.voteSignatureVerify(v, ca.addr) + if err != nil { + e := ticketvote.VoteErrorSignatureInvalid + receipts[k].ErrorCode = e + receipts[k].ErrorContext = fmt.Sprintf("%v: %v", + ticketvote.VoteError[e], err) + continue + } - // Decode payload - a, err := ticketvote.DecodeAuthorize([]byte(payload)) + // Verify ticket is eligible to vote + _, ok := eligible[v.Ticket] + if !ok { + e := ticketvote.VoteErrorTicketNotEligible + receipts[k].ErrorCode = e + receipts[k].ErrorContext = ticketvote.VoteError[e] + continue + } + + // Verify ticket has not already vote + _, ok = castVotes[v.Ticket] + if ok { + e := ticketvote.VoteErrorTicketAlreadyVoted + receipts[k].ErrorCode = e + receipts[k].ErrorContext = ticketvote.VoteError[e] + continue + } + + // Save cast vote + receipt := p.id.SignMessage([]byte(v.Signature)) + cv := ticketvote.CastVote{ + Token: v.Token, + Ticket: v.Ticket, + VoteBit: v.VoteBit, + Signature: v.Signature, + Receipt: hex.EncodeToString(receipt[:]), + } + err = castVoteSave(client, cv) + if err != nil { + t := time.Now().Unix() + log.Errorf("cmdBallot: castVoteSave %v: %v", t, err) + e := ticketvote.VoteErrorInternalError + receipts[k].ErrorCode = e + receipts[k].ErrorContext = fmt.Sprintf("%v: %v", + ticketvote.VoteError[e], t) + continue + } + + // Update receipt + receipts[k].Ticket = cv.Ticket + receipts[k].Receipt = cv.Receipt + + // Update cast votes cache + p.cachedCastVotesSave(v.Token, v.Ticket, v.VoteBit) + } + + // Prepare reply + br := ticketvote.BallotReply{ + Receipts: receipts, + } + reply, err := ticketvote.EncodeBallotReply(br) if err != nil { return "", err } - // Verify signature - version := strconv.FormatUint(uint64(a.Version), 10) - msg := a.Token + version + string(a.Action) - err = util.VerifySignature(a.Signature, a.PublicKey, msg) + return string(reply), nil +} + +func (p *ticketVotePlugin) cmdDetails(payload string) (string, error) { + log.Tracef("ticketvote cmdDetails: %v", payload) + + d, err := ticketvote.DecodeDetails([]byte(payload)) if err != nil { - return "", convertTicketVoteErrFromSignatureErr(err) + return "", err } // Get plugin client - tokenb, err := hex.DecodeString(a.Token) + tokenb, err := hex.DecodeString(d.Token) if err != nil { return "", ticketvote.UserError{ ErrorCode: ticketvote.ErrorStatusTokenInvalid, @@ -226,94 +1491,69 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { } return "", err } - if client.State != tlogbe.RecordStateVetted { - return "", ticketvote.UserError{ - ErrorCode: ticketvote.ErrorStatusRecordStateInvalid, - ErrorContext: []string{"record not vetted"}, - } + + // Get authorize votes + auths, err := authorizeVotes(client) + if err != nil { + return "", fmt.Errorf("authorizeVotes: %v", err) } - // Verify record version - _, err = p.backend.GetVetted(tokenb, version) + // Get start vote + sv, err := startVote(client) if err != nil { - if err == backend.ErrRecordNotFound { - e := fmt.Sprintf("version %v not found", version) - return "", ticketvote.UserError{ - ErrorCode: ticketvote.ErrorStatusRecordNotFound, - ErrorContext: []string{e}, - } - } + return "", fmt.Errorf("startVote: %v", err) } - // Verify action - switch a.Action { - case ticketvote.ActionAuthorize: - // This is allowed - case ticketvote.ActionRevoke: - // This is allowed - default: - return "", ticketvote.UserError{ - ErrorCode: ticketvote.ErrorStatusAuthorizeActionInvalid, - } + // Prepare rely + dr := ticketvote.DetailsReply{ + Auths: auths, + Vote: sv, + } + reply, err := ticketvote.EncodeDetailsReply(dr) + if err != nil { + return "", err } - // Get any previous authorizations to verify that the new action - // is allowed based on the previous action. - auths, err := authorizations(client) + return string(reply), nil +} + +func (p *ticketVotePlugin) cmdCastVotes(payload string) (string, error) { + log.Tracef("ticketvote cmdCastVotes: %v", payload) + + // Decode payload + cv, err := ticketvote.DecodeCastVotes([]byte(payload)) if err != nil { return "", err } - var prevAction ticketvote.ActionT - if len(auths) > 0 { - prevAction = ticketvote.ActionT(auths[len(auths)-1].Action) + + // Get plugin client + tokenb, err := hex.DecodeString(cv.Token) + if err != nil { + return "", ticketvote.UserError{ + ErrorCode: ticketvote.ErrorStatusTokenInvalid, + } } - switch { - case len(auths) == 0: - // No previous actions. New action must be an authorize. - if a.Action != ticketvote.ActionAuthorize { + client, err := p.backend.PluginClient(tokenb) + if err != nil { + if err == backend.ErrRecordNotFound { return "", ticketvote.UserError{ - ErrorCode: ticketvote.ErrorStatusAuthorizeActionInvalid, - ErrorContext: []string{"no prev action; action must be authorize"}, + ErrorCode: ticketvote.ErrorStatusRecordNotFound, } } - case prevAction == ticketvote.ActionAuthorize: - // Previous action was a authorize. This action must be revoke. - return "", ticketvote.UserError{ - ErrorCode: ticketvote.ErrorStatusAuthorizeActionInvalid, - ErrorContext: []string{"prev action was authorize"}, - } - case prevAction == ticketvote.ActionRevoke: - // Previous action was a revoke. This action must be authorize. - return "", ticketvote.UserError{ - ErrorCode: ticketvote.ErrorStatusAuthorizeActionInvalid, - ErrorContext: []string{"prev action was revoke"}, - } - } - - // Prepare authorization - receipt := p.id.SignMessage([]byte(a.Signature)) - auth := authorize{ - Token: a.Token, - Version: a.Version, - Action: string(a.Action), - PublicKey: a.PublicKey, - Signature: a.Signature, - Timestamp: time.Now().Unix(), - Receipt: hex.EncodeToString(receipt[:]), + return "", err } - // Save authorization - err = authorizeSave(client, auth) + // Get cast votes + votes, err := castVotes(client) if err != nil { - return "", err + return "", fmt.Errorf("castVotes: %v", err) } // Prepare reply - ar := ticketvote.AuthorizeReply{ - Timestamp: auth.Timestamp, - Receipt: auth.Receipt, + cvr := ticketvote.CastVotesReply{ + Votes: votes, } - reply, err := ticketvote.EncodeAuthorizeReply(ar) + reply, err := ticketvote.EncodeCastVotesReply(cvr) if err != nil { return "", err } @@ -321,40 +1561,142 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { return string(reply), nil } -func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { - log.Tracef("ticketvote cmdStart: %v", payload) - - return "", nil -} - -func (p *ticketVotePlugin) cmdStartRunoff(payload string) (string, error) { - log.Tracef("ticketvote cmdStartRunoff: %v", payload) - - return "", nil +func summaryPrepare(sv ticketvote.StartVote, castVotes map[string]string, bestBlock uint32) ticketvote.Summary { + // TODO + return ticketvote.Summary{ + Type: sv.Vote.Type, + // Status: , + Duration: sv.Vote.Duration, + StartBlockHeight: sv.StartBlockHeight, + StartBlockHash: sv.StartBlockHash, + EndBlockHeight: sv.EndBlockHeight, + EligibleTickets: uint32(len(sv.EligibleTickets)), + QuorumPercentage: sv.Vote.QuorumPercentage, + PassPercentage: sv.Vote.PassPercentage, + // Results: , + // Approved: , + } } -func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { - log.Tracef("ticketvote cmdBallot: %v", payload) +func (p *ticketVotePlugin) summary(token string, bestBlock uint32) (*ticketvote.Summary, error) { + // Check if the summary has been cached + s, err := p.cachedSummary(token) + switch { + case err == errRecordNotFound: + // Cached summary not found. This is ok; continue. + case err != nil: + // Some other error + return nil, fmt.Errorf("cachedSummary: %v", err) + default: + // Caches summary was found. Return it. + return s, nil + } - return "", nil -} + // Cached summary not found. Get the plugin client. + tokenb, err := hex.DecodeString(token) + if err != nil { + return nil, errRecordNotFound + } + client, err := p.backend.PluginClient(tokenb) + if err != nil { + return nil, errRecordNotFound + } + if client.State != tlogbe.RecordStateVetted { + // Record exists but is unvetted so a vote can not have + // authorized yet. + return &ticketvote.Summary{ + Status: ticketvote.VoteStatusUnauthorized, + Results: []ticketvote.Result{}, + }, nil + } -func (p *ticketVotePlugin) cmdDetails(payload string) (string, error) { - log.Tracef("ticketvote cmdDetails: %v", payload) + // Check if the vote has been authorized + auths, err := authorizeVotes(client) + if err != nil { + return nil, fmt.Errorf("authorizeVotes: %v", err) + } + if len(auths) == 0 { + // Vote has not been authorized yet + return &ticketvote.Summary{ + Status: ticketvote.VoteStatusUnauthorized, + Results: []ticketvote.Result{}, + }, nil + } + lastAuth := auths[len(auths)-1] + switch ticketvote.ActionT(lastAuth.Action) { + case ticketvote.ActionAuthorize: + // Vote has been authorized; continue + case ticketvote.ActionRevoke: + // Vote authorization has been revoked + return &ticketvote.Summary{ + Status: ticketvote.VoteStatusUnauthorized, + Results: []ticketvote.Result{}, + }, nil + } - return "", nil -} + // Vote has been authorized. Check if it has been started yet. + sv, err := startVote(client) + if err != nil { + return nil, fmt.Errorf("startVote: %v", err) + } + if sv == nil { + // Vote has not been started yet + return &ticketvote.Summary{ + Status: ticketvote.VoteStatusAuthorized, + Results: []ticketvote.Result{}, + }, nil + } -func (p *ticketVotePlugin) cmdCastVotes(payload string) (string, error) { - log.Tracef("ticketvote cmdCastVotes: %v", payload) + // Vote has been started. Vote is either still in progress or has + // ended but the summary hasn't been cached yet. Pull the cast + // votes from the cache and calulate the results manually. + votes := p.cachedCastVotes(token) + summary := summaryPrepare(*sv, votes, bestBlock) - return "", nil + return &summary, nil } func (p *ticketVotePlugin) cmdSummaries(payload string) (string, error) { log.Tracef("ticketvote cmdSummaries: %v", payload) - return "", nil + // Decode payload + s, err := ticketvote.DecodeSummaries([]byte(payload)) + if err != nil { + return "", err + } + + // Get best block. This cmd does not write any data so we do not + // have to use the safe best block. + bb, err := p.bestBlock() + if err != nil { + return "", fmt.Errorf("bestBlock: %v", err) + } + + // Get summaries + summaries := make(map[string]ticketvote.Summary, len(s.Tokens)) + for _, v := range s.Tokens { + s, err := p.summary(v, bb) + if err != nil { + if err == errRecordNotFound { + // Record does not exist for token. Do not include this token + // in the reply. + continue + } + return "", fmt.Errorf("summary %v: %v", v, err) + } + summaries[v] = *s + } + + // Prepare reply + sr := ticketvote.SummariesReply{ + Summaries: summaries, + } + reply, err := ticketvote.EncodeSummariesReply(sr) + if err != nil { + return "", err + } + + return string(reply), nil } func (p *ticketVotePlugin) cmdInventory(payload string) (string, error) { @@ -416,13 +1758,27 @@ func (p *ticketVotePlugin) Setup() error { log.Tracef("ticketvote Setup") // Ensure dcrdata plugin has been registered + // Build vote summaries return nil } -func TicketVotePluginNew(id *identity.FullIdentity, backend *tlogbe.Tlogbe) (*ticketVotePlugin, error) { +func TicketVotePluginNew(backend *tlogbe.Tlogbe, settings []backend.PluginSetting) (*ticketVotePlugin, error) { + var ( + // TODO these should be passed in as plugin settings + dataDir string + id = &identity.FullIdentity{} + activeNetParams = &chaincfg.Params{} + voteDurationMin uint32 + voteDurationMax uint32 + ) + return &ticketVotePlugin{ - id: id, - backend: backend, + dataDir: filepath.Join(dataDir, ticketVoteDirname), + backend: backend, + id: id, + activeNetParams: activeNetParams, + voteDurationMin: voteDurationMin, + voteDurationMax: voteDurationMax, }, nil } diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index f79a0071d..61257c027 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -930,6 +930,7 @@ func (p *politeiawww) getUserProps(filter proposalsFilter) ([]www.ProposalRecord } // Get the latest version of all proposals from the cache + // TODO do not use getAllProps all, err := p.getAllProps() if err != nil { return nil, nil, fmt.Errorf("getAllProps: %v", err) @@ -1893,6 +1894,7 @@ func (p *politeiawww) processAllVetted(v www.GetAllVetted) (*www.GetAllVettedRep } // Fetch all proposals from the cache + // TODO do not use getAllProps all, err := p.getAllProps() if err != nil { return nil, fmt.Errorf("getAllProps: %v", err) @@ -2127,6 +2129,7 @@ func (p *politeiawww) processGetAllVoteStatus() (*www.GetAllVoteStatusReply, err } // Get all proposals from cache + // TODO do not use getAllProps all, err := p.getAllProps() if err != nil { return nil, fmt.Errorf("getAllProps: %v", err) diff --git a/util/signature.go b/util/signature.go index b10ccddcb..0357b2478 100644 --- a/util/signature.go +++ b/util/signature.go @@ -32,8 +32,8 @@ func (e SignatureError) Error() string { return fmt.Sprintf("signature error code: %v", e.ErrorCode) } -// VerifySignature verifies a Ed25519 signature. -func VerifySignature(signature, pubkey, msg string) error { +// VerifySignature verifies a hex encoded Ed25519 signature. +func VerifySignature(signature, pubKey, msg string) error { sig, err := ConvertSignature(signature) if err != nil { return SignatureError{ @@ -41,7 +41,7 @@ func VerifySignature(signature, pubkey, msg string) error { ErrorContext: []string{err.Error()}, } } - b, err := hex.DecodeString(pubkey) + b, err := hex.DecodeString(pubKey) if err != nil { return SignatureError{ ErrorCode: ErrorStatusPublicKeyInvalid, From 05cb2a46545adc71d9910ca1b08aa443ce42ae0c Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 12 Aug 2020 19:09:13 -0500 Subject: [PATCH 015/449] cont plugins and tlogbe implementation --- plugins/comments/comments.go | 55 ++ plugins/ticketvote/ticketvote.go | 14 +- politeiad/backend/backend.go | 189 +------ politeiad/backend/tlogbe/anchor.go | 140 ++++- politeiad/backend/tlogbe/encryptionkey.go | 4 +- .../tlogbe/{pluginclient.go => plugin.go} | 65 ++- politeiad/backend/tlogbe/plugins/comments.go | 185 +++---- politeiad/backend/tlogbe/plugins/dcrdata.go | 9 +- politeiad/backend/tlogbe/plugins/plugins.go | 48 -- .../backend/tlogbe/plugins/ticketvote.go | 415 ++++++++++++--- .../tlogbe/store/filesystem/filesystem.go | 260 +++++---- politeiad/backend/tlogbe/store/store.go | 33 +- politeiad/backend/tlogbe/tlog.go | 122 +++-- politeiad/backend/tlogbe/tlogbe.go | 499 +++++++++++++++--- politeiad/backend/tlogbe/trillian.go | 2 +- 15 files changed, 1320 insertions(+), 720 deletions(-) rename politeiad/backend/tlogbe/{pluginclient.go => plugin.go} (79%) delete mode 100644 politeiad/backend/tlogbe/plugins/plugins.go diff --git a/plugins/comments/comments.go b/plugins/comments/comments.go index 57b913d04..0356ddce7 100644 --- a/plugins/comments/comments.go +++ b/plugins/comments/comments.go @@ -12,6 +12,8 @@ import ( type VoteT int type ErrorStatusT int +// TODO add a hint to comments that can be used freely by the client. This +// is how we'll distinguish proposal comments from update comments. const ( Version uint32 = 1 ID = "comments" @@ -93,6 +95,59 @@ type Comment struct { Reason string `json:"reason"` // Reason for deletion } +// CommentAdd is the structure that is saved to disk when a comment is created +// or edited. +type CommentAdd struct { + // Data generated by client + Token string `json:"token"` // Record token + ParentID uint32 `json:"parentid"` // Parent comment ID + Comment string `json:"comment"` // Comment + PublicKey string `json:"publickey"` // Pubkey used for Signature + Signature string `json:"signature"` // Signature of Token+ParentID+Comment + + // Metadata generated by server + CommentID uint32 `json:"commentid"` // Comment ID + Version uint32 `json:"version"` // Comment version + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + Receipt string `json:"receipt"` // Server signature of client signature +} + +// CommentDel is the structure that is saved to disk when a comment is deleted. +// Some additional fields like ParentID and AuthorPublicKey are required to be +// saved since all the comment add records will be deleted and the client needs +// these additional fields to properly display the deleted comment in the +// comment hierarchy. +type CommentDel struct { + // Data generated by client + Token string `json:"token"` // Record token + CommentID uint32 `json:"commentid"` // Comment ID + Reason string `json:"reason"` // Reason for deleting + PublicKey string `json:"publickey"` // Pubkey used for Signature + Signature string `json:"signature"` // Signature of Token+CommentID+Reason + + // Metadata generated by server + ParentID uint32 `json:"parentid"` // Parent comment ID + AuthorPublicKey string `json:"authorpublickey"` // Author public key + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + Receipt string `json:"receipt"` // Server sig of client sig +} + +// CommentVote is the structure that is saved to disk when a comment is voted +// on. +type CommentVote struct { + // Data generated by client + UUID string `json:"uuid"` // Unique user ID + Token string `json:"token"` // Record token + CommentID uint32 `json:"commentid"` // Comment ID + Vote int64 `json:"vote"` // Upvote or downvote + PublicKey string `json:"publickey"` // Public key used for signature + Signature string `json:"signature"` // Signature of Token+CommentID+Vote + + // Metadata generated by server + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + Receipt string `json:"receipt"` // Server signature of client signature +} + // New creates a new comment. A parent ID of 0 indicates that the comment is // a base level comment and not a reply commment. type New struct { diff --git a/plugins/ticketvote/ticketvote.go b/plugins/ticketvote/ticketvote.go index f6c36ec85..5ab289b50 100644 --- a/plugins/ticketvote/ticketvote.go +++ b/plugins/ticketvote/ticketvote.go @@ -174,8 +174,9 @@ type StartVote struct { } // CastVote is the structure that is saved to disk when a vote is cast. -// TODO vote option vote bit is a uint64, but this vote bit is a string. Is -// that on purpose? +// TODO VoteOption.Bit is a uint64, but the CastVote.VoteBit is a string in +// decredplugin. Do we want to make them consistent or was that done on +// purpose? It was probably done that way so that way for the signature. type CastVote struct { // Data generated by client Token string `json:"token"` // Record token @@ -593,6 +594,15 @@ func DecodeInventory(payload []byte) (*Inventory, error) { // InventoryReply is the reply to the Inventory command. It contains the tokens // of all public, non-abandoned records catagorized by vote status. +// TODO +// Sorted by timestamp in descending order: +// Unauthorized, Authorized +// +// Sorted by voting period end block height in descending order: +// Started, Finished +// +// TODO the pi plugin will need to catagorize finished into approved and +// rejected. type InventoryReply struct { Unauthorized []string `json:"unauthorized"` Authorized []string `json:"authorized"` diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index e0e72f7b8..9daf70435 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -5,18 +5,11 @@ package backend import ( - "bytes" - "encoding/base64" "errors" "fmt" - "path/filepath" "regexp" - "strconv" v1 "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/politeiad/api/v1/mime" - "github.com/decred/politeia/util" - "github.com/subosito/gozaru" ) var ( @@ -45,10 +38,17 @@ var ( // archived record. ErrRecordArchived = errors.New("record is archived") - // ErrJournalsNotReplayed is returned when the journals have not been replayed - // and the subsequent code expect it to be replayed + // ErrJournalsNotReplayed is returned when the journals have not + // been replayed and the subsequent code expect it to be replayed. ErrJournalsNotReplayed = errors.New("journals have not been replayed") + // ErrPluginInvalid is emitted when an invalid plugin ID is used. + ErrPluginInvalid = errors.New("plugin invalid") + + // ErrPluginCmdInvalid is emitted when an invalid plugin command is + // used. + ErrPluginCmdInvalid = errors.New("plugin command invalid") + // Plugin names must be all lowercase letters and have a length of <20 PluginRE = regexp.MustCompile(`^[a-z]{1,20}$`) ) @@ -134,161 +134,6 @@ type Record struct { Files []File // User provided files } -// VerifyContent verifies that all provided MetadataStream and File are sane. -func VerifyContent(metadata []MetadataStream, files []File, filesDel []string) error { - // Make sure all metadata is within maxima. - for _, v := range metadata { - if v.ID > v1.MetadataStreamsMax-1 { - return ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidMDID, - ErrorContext: []string{ - strconv.FormatUint(v.ID, 10), - }, - } - } - } - for i := range metadata { - for j := range metadata { - // Skip self and non duplicates. - if i == j || metadata[i].ID != metadata[j].ID { - continue - } - return ContentVerificationError{ - ErrorCode: v1.ErrorStatusDuplicateMDID, - ErrorContext: []string{ - strconv.FormatUint(metadata[i].ID, 10), - }, - } - } - } - - // Prevent paths - for i := range files { - if filepath.Base(files[i].Name) != files[i].Name { - return ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidFilename, - ErrorContext: []string{ - files[i].Name, - }, - } - } - } - for _, v := range filesDel { - if filepath.Base(v) != v { - return ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidFilename, - ErrorContext: []string{ - v, - }, - } - } - } - - // Now check files - if len(files) == 0 { - return ContentVerificationError{ - ErrorCode: v1.ErrorStatusEmpty, - } - } - - // Prevent bad filenames and duplicate filenames - for i := range files { - for j := range files { - if i == j { - continue - } - if files[i].Name == files[j].Name { - return ContentVerificationError{ - ErrorCode: v1.ErrorStatusDuplicateFilename, - ErrorContext: []string{ - files[i].Name, - }, - } - } - } - // Check against filesDel - for _, v := range filesDel { - if files[i].Name == v { - return ContentVerificationError{ - ErrorCode: v1.ErrorStatusDuplicateFilename, - ErrorContext: []string{ - files[i].Name, - }, - } - } - } - } - - for i := range files { - if gozaru.Sanitize(files[i].Name) != files[i].Name { - return ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidFilename, - ErrorContext: []string{ - files[i].Name, - }, - } - } - - // Validate digest - d, ok := util.ConvertDigest(files[i].Digest) - if !ok { - return ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidFileDigest, - ErrorContext: []string{ - files[i].Name, - }, - } - } - - // Decode base64 payload - var err error - payload, err := base64.StdEncoding.DecodeString(files[i].Payload) - if err != nil { - return ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidBase64, - ErrorContext: []string{ - files[i].Name, - }, - } - } - - // Calculate payload digest - dp := util.Digest(payload) - if !bytes.Equal(d[:], dp) { - return ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidFileDigest, - ErrorContext: []string{ - files[i].Name, - }, - } - } - - // Verify MIME - detectedMIMEType := mime.DetectMimeType(payload) - if detectedMIMEType != files[i].MIME { - return ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidMIMEType, - ErrorContext: []string{ - files[i].Name, - detectedMIMEType, - }, - } - } - - if !mime.MimeValid(files[i].MIME) { - return ContentVerificationError{ - ErrorCode: v1.ErrorStatusUnsupportedMIMEType, - ErrorContext: []string{ - files[i].Name, - files[i].MIME, - }, - } - } - } - - return nil -} - // PluginSettings type PluginSetting struct { Key string // Name of setting @@ -302,6 +147,17 @@ type Plugin struct { Settings []PluginSetting // Settings } +// InventoryByStatus contains the record tokens of all records in the inventory +// catagorized by MDStatusT. Each array is sorted by the timestamp of the +// status change from newest to oldest. +type InventoryByStatus struct { + Unvetted []string + IterationUnvetted []string + Vetted []string + Censored []string + Archived []string +} + type Backend interface { // Create new record New([]MetadataStream, []File) (*RecordMetadata, error) @@ -345,8 +201,9 @@ type Backend interface { // Inventory retrieves various record records. Inventory(uint, uint, bool, bool) ([]Record, []Record, error) - // TODO Inventory needs to return the token inventory grouped by - // record status. + // InventoryByStatus returns the record tokens of all records in the + // inventory catagorized by MDStatusT. + InventoryByStatus() (*InventoryByStatus, error) // Obtain plugin settings GetPlugins() ([]Plugin, error) diff --git a/politeiad/backend/tlogbe/anchor.go b/politeiad/backend/tlogbe/anchor.go index b90019fc4..79838d67d 100644 --- a/politeiad/backend/tlogbe/anchor.go +++ b/politeiad/backend/tlogbe/anchor.go @@ -13,7 +13,9 @@ import ( dcrtime "github.com/decred/dcrtime/api/v2" "github.com/decred/politeia/politeiad/backend/tlogbe/store" "github.com/decred/politeia/util" + "github.com/google/trillian" "github.com/google/trillian/types" + "google.golang.org/grpc/codes" ) // TODO handle reorgs. A anchor record may become invalid in the case of a @@ -55,7 +57,6 @@ func (t *tlog) anchorSave(a anchor) error { return fmt.Errorf("verify digest not found") } - // TODO // Save the anchor record to store be, err := convertBlobEntryFromAnchor(a) if err != nil { @@ -65,15 +66,38 @@ func (t *tlog) anchorSave(a anchor) error { if err != nil { return err } - lrb, err := a.LogRoot.MarshalBinary() + keys, err := t.store.Put([][]byte{b}) if err != nil { - return err + return fmt.Errorf("store Put: %v", err) + } + if len(keys) != 1 { + return fmt.Errorf("wrong number of keys: got %v, want 1", + len(keys)) } - logRootHash := util.Hash(lrb)[:] - _ = logRootHash - _ = b // Append anchor leaf to trillian tree + h, err := hex.DecodeString(be.Hash) + if err != nil { + return err + } + prefixedKey := []byte(keyPrefixAnchorRecord + keys[0]) + queued, _, err := t.trillian.leavesAppend(a.TreeID, []*trillian.LogLeaf{ + logLeafNew(h, prefixedKey), + }) + if len(queued) != 1 { + return fmt.Errorf("wrong number of queud leaves: got %v, want 1", + len(queued)) + } + failed := make([]string, 0, len(queued)) + for _, v := range queued { + c := codes.Code(v.QueuedLeaf.GetStatus().GetCode()) + if c != codes.OK { + failed = append(failed, fmt.Sprintf("%v", c)) + } + } + if len(failed) > 0 { + return fmt.Errorf("append leaves failed: %v", failed) + } log.Debugf("Saved %v anchor for tree %v at height %v", t.id, a.TreeID, a.LogRoot.TreeSize) @@ -81,6 +105,57 @@ func (t *tlog) anchorSave(a anchor) error { return nil } +// anchorLatest returns the most recent anchor for the provided tree. A +// errAnchorNotFound is returned if no anchor is found for the provided tree. +func (t *tlog) anchorLatest(treeID int64) (*anchor, error) { + // Get tree leaves + leavesAll, err := t.trillian.leavesAll(treeID) + if err != nil { + return nil, fmt.Errorf("leavesAll: %v", err) + } + + // Find the most recent anchor leaf + var key string + for i := len(leavesAll) - 1; i >= 0; i-- { + if leafIsAnchor(leavesAll[i]) { + // Extract key-value store key + key, err = extractKeyFromLeaf(leavesAll[i]) + if err != nil { + return nil, err + } + } + } + if key == "" { + return nil, errAnchorNotFound + } + + // Pull blob from key-value store + blobs, err := t.store.Get([]string{key}) + if err != nil { + return nil, fmt.Errorf("store Get: %v", err) + } + if len(blobs) != 1 { + return nil, fmt.Errorf("unexpected blobs count: got %v, want 1", + len(blobs)) + } + + // Decode freeze record + b, ok := blobs[key] + if !ok { + return nil, fmt.Errorf("blob not found %v", key) + } + be, err := store.Deblob(b) + if err != nil { + return nil, err + } + a, err := convertAnchorFromBlobEntry(*be) + if err != nil { + return nil, err + } + + return a, nil +} + // anchorWait waits for the anchor to drop. The anchor is not considered // dropped until dcrtime returns the ChainTimestamp in the reply. dcrtime does // not return the ChainTimestamp until the timestamp transaction has 6 @@ -150,6 +225,7 @@ func (t *tlog) anchorWait(anchors []anchor, hashes []string) { retry = true break } + // Transaction will be populated once the tx has been sent, // otherwise is will be a zeroed out SHA256 digest. b := make([]byte, sha256.Size) @@ -158,6 +234,7 @@ func (t *tlog) anchorWait(anchors []anchor, hashes []string) { retry = true break } + // ChainTimestamp will be populated once the tx has 6 // confirmations. if v.ChainInformation.ChainTimestamp == 0 { @@ -239,21 +316,48 @@ func (t *tlog) anchor() { // key-value store. anchors := make([]anchor, 0, len(trees)) - // Find the trees that need to be anchored + // Find the trees that need to be anchored. This is done by pulling + // the most recent anchor from the tree and checking the anchored + // tree height against the current tree height. We cannot rely on + // the anchored being the last leaf in the tree since new leaves + // can be added while the anchor is waiting to be dropped. for _, v := range trees { - // TODO this needs to pull the anchor record from the store and - // check the anchor tree height against the current tree height + // Get latest anchor + a, err := t.anchorLatest(v.TreeId) + switch { + case err == errAnchorNotFound: + // Tree has not been anchored yet. Verify that the tree has + // leaves. A tree with no leaves does not need to be anchored. + leavesAll, err := t.trillian.leavesAll(v.TreeId) + if err != nil { + exitErr = fmt.Errorf("leavesAll: %v", err) + return + } + if len(leavesAll) == 0 { + // Tree does not have any leaves. Nothing to do. + continue + } - // Check if the last leaf is an anchor record - l, err := t.lastLeaf(v.TreeId) - if err != nil { - exitErr = fmt.Errorf("lastLeaf %v: %v", v.TreeId, err) + case err != nil: + // All other errors + exitErr = fmt.Errorf("anchorLatest %v: %v", v.TreeId, err) return - } - if leafIsAnchorRecord(l) { - // Tree has already been anchored at the current height. Check - // the next one. - continue + + default: + // Anchor record found. If the anchor height differs from the + // current height than the tree needs to be anchored. + _, lr, err := t.trillian.signedLogRootForTree(v) + if err != nil { + exitErr = fmt.Errorf("signedLogRoot %v: %v", v.TreeId, err) + return + } + // Subtract one from the current height to account for the + // anchor leaf. + if a.LogRoot.TreeSize == lr.TreeSize-1 { + // Tree has already been anchored at this height. Nothing to + // do. + continue + } } // Tree has not been anchored at current height. Add it to the diff --git a/politeiad/backend/tlogbe/encryptionkey.go b/politeiad/backend/tlogbe/encryptionkey.go index 8717cafa2..07b9bd728 100644 --- a/politeiad/backend/tlogbe/encryptionkey.go +++ b/politeiad/backend/tlogbe/encryptionkey.go @@ -13,7 +13,7 @@ import ( // encryptionKey provides an API for encrypting and decrypting data. The // encryption key is zero'd out on application exit so the lock must be held -// anytime the key is accessed in order to prevent the goland race detector +// anytime the key is accessed in order to prevent the golang race detector // from complaining. type encryptionKey struct { sync.RWMutex @@ -49,7 +49,7 @@ func (e *encryptionKey) Zero() { e.key = nil } -func encryptionKeyNew(key *[32]byte) *encryptionKey { +func newEncryptionKey(key *[32]byte) *encryptionKey { return &encryptionKey{ key: key, } diff --git a/politeiad/backend/tlogbe/pluginclient.go b/politeiad/backend/tlogbe/plugin.go similarity index 79% rename from politeiad/backend/tlogbe/pluginclient.go rename to politeiad/backend/tlogbe/plugin.go index 04a812698..3e7be672f 100644 --- a/politeiad/backend/tlogbe/pluginclient.go +++ b/politeiad/backend/tlogbe/plugin.go @@ -13,6 +13,47 @@ import ( "google.golang.org/grpc/codes" ) +type HookT int + +const ( + // Plugin hooks + HookInvalid HookT = 0 + HookPreNewRecord HookT = 1 + HookPostNewRecord HookT = 2 + HookPreEditRecord HookT = 3 + HookPostEditRecord HookT = 4 + HookPreEditMetadata HookT = 5 + HookPostEditMetadata HookT = 6 + HookPreSetRecordStatus HookT = 7 + HookPostSetRecordStatus HookT = 8 +) + +var ( + // Hooks contains human readable plugin hook descriptions. + Hooks = map[HookT]string{ + HookPostNewRecord: "post new record", + HookPostEditRecord: "post edit record", + HookPostEditMetadata: "post edit metadata", + HookPostSetRecordStatus: "post set record status", + } +) + +// Plugin provides an API for the tlogbe to use when interacting with plugins. +// All tlogbe plugins must implement the Plugin interface. +type Plugin interface { + // Perform plugin setup + Setup() error + + // Execute a plugin command + Cmd(cmd, payload string) (string, error) + + // Execute a plugin hook + Hook(h HookT, payload string) error + + // Perform a plugin file system check + Fsck() error +} + type RecordStateT int const ( @@ -22,9 +63,9 @@ const ( RecordStateVetted RecordStateT = 2 ) -// PluginClient provides an API for plugins to save, retrieve, and delete +// RecordClient provides an API for plugins to save, retrieve, and delete // plugin data for a specific record. Editing data is not allowed. -type PluginClient struct { +type RecordClient struct { Token []byte State RecordStateT treeID int64 @@ -32,8 +73,8 @@ type PluginClient struct { } // hashes and keys must share the same ordering. -func (c *PluginClient) BlobsSave(keyPrefix string, blobs, hashes [][]byte, encrypt bool) ([][]byte, error) { - log.Tracef("BlobsSave: %v %v %v %x", c.treeID, keyPrefix, encrypt, hashes) +func (c *RecordClient) Save(keyPrefix string, blobs, hashes [][]byte, encrypt bool) ([][]byte, error) { + log.Tracef("Save: %x %v %v %x", c.Token, keyPrefix, encrypt, hashes) // Ensure tree exists and is not frozen if !c.tlog.treeExists(c.treeID) { @@ -100,8 +141,8 @@ func (c *PluginClient) BlobsSave(keyPrefix string, blobs, hashes [][]byte, encry return merkles, nil } -func (c *PluginClient) BlobsDel(merkleHashes [][]byte) error { - log.Tracef("BlobsDel: %v %x", c.treeID, merkleHashes) +func (c *RecordClient) Del(merkleHashes [][]byte) error { + log.Tracef("Del: %x %x", c.Token, merkleHashes) // Ensure tree exists. We allow blobs to be deleted from both // frozen and non frozen trees. @@ -145,8 +186,8 @@ func (c *PluginClient) BlobsDel(merkleHashes [][]byte) error { // If a blob does not exist it will not be included in the returned map. It is // the responsibility of the caller to check that a blob is returned for each // of the provided merkle hashes. -func (c *PluginClient) BlobsByMerkleHash(merkleHashes [][]byte) (map[string][]byte, error) { - log.Tracef("BlobsByMerkleHash: %v %x", merkleHashes) +func (c *RecordClient) BlobsByMerkleHash(merkleHashes [][]byte) (map[string][]byte, error) { + log.Tracef("BlobsByMerkleHash: %x %x", c.Token, merkleHashes) // Get leaves leavesAll, err := c.tlog.trillian.leavesAll(c.treeID) @@ -226,7 +267,9 @@ func (c *PluginClient) BlobsByMerkleHash(merkleHashes [][]byte) (map[string][]by return b, nil } -func (c *PluginClient) BlobsByKeyPrefix(keyPrefix string) ([][]byte, error) { +func (c *RecordClient) BlobsByKeyPrefix(keyPrefix string) ([][]byte, error) { + log.Tracef("BlobsByKeyPrefix: %x %x", c.Token, keyPrefix) + // Get leaves leaves, err := c.tlog.trillian.leavesAll(c.treeID) if err != nil { @@ -283,7 +326,7 @@ func (c *PluginClient) BlobsByKeyPrefix(keyPrefix string) ([][]byte, error) { return b, nil } -// TODO -func (t *Tlogbe) PluginClient(token []byte) (*PluginClient, error) { +// TODO implement RecordClient +func (t *Tlogbe) RecordClient(token []byte) (*RecordClient, error) { return nil, nil } diff --git a/politeiad/backend/tlogbe/plugins/comments.go b/politeiad/backend/tlogbe/plugins/comments.go index f1fbe34ac..1b11cbc01 100644 --- a/politeiad/backend/tlogbe/plugins/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments.go @@ -40,7 +40,7 @@ const ( ) var ( - _ Plugin = (*commentsPlugin)(nil) + _ tlogbe.Plugin = (*commentsPlugin)(nil) ) // commentsPlugin satisfies the Plugin interface. @@ -54,61 +54,6 @@ type commentsPlugin struct { mutexes map[string]*sync.Mutex // [string]mutex } -// TODO move these to the plugin API since this is what the inclusion proof -// is for -// commentAdd is the structure that is saved to disk when a comment is created -// or edited. -type commentAdd struct { - // Data generated by client - Token string `json:"token"` // Record token - ParentID uint32 `json:"parentid"` // Parent comment ID - Comment string `json:"comment"` // Comment - PublicKey string `json:"publickey"` // Pubkey used for Signature - Signature string `json:"signature"` // Signature of Token+ParentID+Comment - - // Metadata generated by server - CommentID uint32 `json:"commentid"` // Comment ID - Version uint32 `json:"version"` // Comment version - Timestamp int64 `json:"timestamp"` // Received UNIX timestamp - Receipt string `json:"receipt"` // Server signature of client signature -} - -// commentDel is the structure that is saved to disk when a comment is deleted. -// Some additional fields like ParentID and AuthorPublicKey are required to be -// saved by the server because all comment add records will be deleted and -// the client will still require those fields to properly display the comment -// hierarchy. -type commentDel struct { - // Data generated by client - Token string `json:"token"` // Record token - CommentID uint32 `json:"commentid"` // Comment ID - Reason string `json:"reason"` // Reason for deleting - PublicKey string `json:"publickey"` // Pubkey used for Signature - Signature string `json:"signature"` // Signature of Token+CommentID+Reason - - // Metadata generated by server - ParentID uint32 `json:"parentid"` // Parent comment ID - AuthorPublicKey string `json:"authorpublickey"` // Author public key - Timestamp int64 `json:"timestamp"` // Received UNIX timestamp - Receipt string `json:"receipt"` // Server sig of client sig -} - -// commentVote is the structure that is saved to disk when a comment is voted -// on. -type commentVote struct { - // Data generated by client - UUID string `json:"uuid"` // Unique user ID - Token string `json:"token"` // Record token - CommentID uint32 `json:"commentid"` // Comment ID - Vote int64 `json:"vote"` // Upvote or downvote - PublicKey string `json:"publickey"` // Public key used for signature - Signature string `json:"signature"` // Signature of Token+CommentID+Vote - - // Metadata generated by server - Timestamp int64 `json:"timestamp"` // Received UNIX timestamp - Receipt string `json:"receipt"` // Server signature of client signature -} - type voteIndex struct { Vote comments.VoteT `json:"vote"` MerkleHash []byte `json:"merklehash"` @@ -168,7 +113,7 @@ func convertCommentsErrFromSignatureErr(err error) comments.UserError { } } -func convertBlobEntryFromCommentAdd(c commentAdd) (*store.BlobEntry, error) { +func convertBlobEntryFromCommentAdd(c comments.CommentAdd) (*store.BlobEntry, error) { data, err := json.Marshal(c) if err != nil { return nil, err @@ -185,7 +130,7 @@ func convertBlobEntryFromCommentAdd(c commentAdd) (*store.BlobEntry, error) { return &be, nil } -func convertBlobEntryFromCommentDel(c commentDel) (*store.BlobEntry, error) { +func convertBlobEntryFromCommentDel(c comments.CommentDel) (*store.BlobEntry, error) { data, err := json.Marshal(c) if err != nil { return nil, err @@ -202,7 +147,7 @@ func convertBlobEntryFromCommentDel(c commentDel) (*store.BlobEntry, error) { return &be, nil } -func convertBlobEntryFromCommentVote(c commentVote) (*store.BlobEntry, error) { +func convertBlobEntryFromCommentVote(c comments.CommentVote) (*store.BlobEntry, error) { data, err := json.Marshal(c) if err != nil { return nil, err @@ -236,7 +181,7 @@ func convertBlobEntryFromIndex(idx index) (*store.BlobEntry, error) { return &be, nil } -func convertCommentAddFromBlobEntry(be store.BlobEntry) (*commentAdd, error) { +func convertCommentAddFromBlobEntry(be store.BlobEntry) (*comments.CommentAdd, error) { // Decode and validate data hint b, err := base64.StdEncoding.DecodeString(be.DataHint) if err != nil { @@ -265,16 +210,16 @@ func convertCommentAddFromBlobEntry(be store.BlobEntry) (*commentAdd, error) { return nil, fmt.Errorf("data is not coherent; got %x, want %x", util.Digest(b), hash) } - var c commentAdd + var c comments.CommentAdd err = json.Unmarshal(b, &c) if err != nil { - return nil, fmt.Errorf("unmarshal commentAdd: %v", err) + return nil, fmt.Errorf("unmarshal CommentAdd: %v", err) } return &c, nil } -func convertCommentDelFromBlobEntry(be store.BlobEntry) (*commentDel, error) { +func convertCommentDelFromBlobEntry(be store.BlobEntry) (*comments.CommentDel, error) { // Decode and validate data hint b, err := base64.StdEncoding.DecodeString(be.DataHint) if err != nil { @@ -303,10 +248,10 @@ func convertCommentDelFromBlobEntry(be store.BlobEntry) (*commentDel, error) { return nil, fmt.Errorf("data is not coherent; got %x, want %x", util.Digest(b), hash) } - var c commentDel + var c comments.CommentDel err = json.Unmarshal(b, &c) if err != nil { - return nil, fmt.Errorf("unmarshal commentDel: %v", err) + return nil, fmt.Errorf("unmarshal CommentDel: %v", err) } return &c, nil @@ -350,7 +295,7 @@ func convertIndexFromBlobEntry(be store.BlobEntry) (*index, error) { return &idx, nil } -func convertCommentFromCommentAdd(ca commentAdd) comments.Comment { +func convertCommentFromCommentAdd(ca comments.CommentAdd) comments.Comment { // Score needs to be filled in seperately return comments.Comment{ Token: ca.Token, @@ -368,7 +313,7 @@ func convertCommentFromCommentAdd(ca commentAdd) comments.Comment { } } -func convertCommentFromCommentDel(cd commentDel) comments.Comment { +func convertCommentFromCommentDel(cd comments.CommentDel) comments.Comment { // Score needs to be filled in seperately return comments.Comment{ Token: cd.Token, @@ -386,7 +331,7 @@ func convertCommentFromCommentDel(cd commentDel) comments.Comment { } } -func commentAddSave(client *tlogbe.PluginClient, c commentAdd, encrypt bool) ([]byte, error) { +func commentAddSave(client *tlogbe.RecordClient, c comments.CommentAdd, encrypt bool) ([]byte, error) { // Prepare blob be, err := convertBlobEntryFromCommentAdd(c) if err != nil { @@ -402,10 +347,10 @@ func commentAddSave(client *tlogbe.PluginClient, c commentAdd, encrypt bool) ([] } // Save blob - merkles, err := client.BlobsSave(keyPrefixCommentAdd, + merkles, err := client.Save(keyPrefixCommentAdd, [][]byte{b}, [][]byte{h}, encrypt) if err != nil { - return nil, fmt.Errorf("BlobsSave: %v", err) + return nil, fmt.Errorf("Save: %v", err) } if len(merkles) != 1 { return nil, fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", @@ -415,7 +360,7 @@ func commentAddSave(client *tlogbe.PluginClient, c commentAdd, encrypt bool) ([] return merkles[0], nil } -func commentAdds(client *tlogbe.PluginClient, merkleHashes [][]byte) ([]commentAdd, error) { +func commentAdds(client *tlogbe.RecordClient, merkleHashes [][]byte) ([]comments.CommentAdd, error) { // Retrieve blobs blobs, err := client.BlobsByMerkleHash(merkleHashes) if err != nil { @@ -434,7 +379,7 @@ func commentAdds(client *tlogbe.PluginClient, merkleHashes [][]byte) ([]commentA } // Decode blobs - adds := make([]commentAdd, 0, len(blobs)) + adds := make([]comments.CommentAdd, 0, len(blobs)) for _, v := range blobs { be, err := store.Deblob(v) if err != nil { @@ -450,7 +395,7 @@ func commentAdds(client *tlogbe.PluginClient, merkleHashes [][]byte) ([]commentA return adds, nil } -func commentDelSave(client *tlogbe.PluginClient, c commentDel) ([]byte, error) { +func commentDelSave(client *tlogbe.RecordClient, c comments.CommentDel) ([]byte, error) { // Prepare blob be, err := convertBlobEntryFromCommentDel(c) if err != nil { @@ -466,10 +411,10 @@ func commentDelSave(client *tlogbe.PluginClient, c commentDel) ([]byte, error) { } // Save blob - merkles, err := client.BlobsSave(keyPrefixCommentDel, + merkles, err := client.Save(keyPrefixCommentDel, [][]byte{b}, [][]byte{h}, false) if err != nil { - return nil, fmt.Errorf("BlobsSave: %v", err) + return nil, fmt.Errorf("Save: %v", err) } if len(merkles) != 1 { return nil, fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", @@ -479,7 +424,7 @@ func commentDelSave(client *tlogbe.PluginClient, c commentDel) ([]byte, error) { return merkles[0], nil } -func commentDels(client *tlogbe.PluginClient, merkleHashes [][]byte) ([]commentDel, error) { +func commentDels(client *tlogbe.RecordClient, merkleHashes [][]byte) ([]comments.CommentDel, error) { // Retrieve blobs blobs, err := client.BlobsByMerkleHash(merkleHashes) if err != nil { @@ -498,7 +443,7 @@ func commentDels(client *tlogbe.PluginClient, merkleHashes [][]byte) ([]commentD } // Decode blobs - dels := make([]commentDel, 0, len(blobs)) + dels := make([]comments.CommentDel, 0, len(blobs)) for _, v := range blobs { be, err := store.Deblob(v) if err != nil { @@ -514,7 +459,7 @@ func commentDels(client *tlogbe.PluginClient, merkleHashes [][]byte) ([]commentD return dels, nil } -func commentVoteSave(client *tlogbe.PluginClient, c commentVote) ([]byte, error) { +func commentVoteSave(client *tlogbe.RecordClient, c comments.CommentVote) ([]byte, error) { // Prepare blob be, err := convertBlobEntryFromCommentVote(c) if err != nil { @@ -530,10 +475,10 @@ func commentVoteSave(client *tlogbe.PluginClient, c commentVote) ([]byte, error) } // Save blob - merkles, err := client.BlobsSave(keyPrefixCommentAdd, + merkles, err := client.Save(keyPrefixCommentAdd, [][]byte{b}, [][]byte{h}, false) if err != nil { - return nil, fmt.Errorf("BlobsSave: %v", err) + return nil, fmt.Errorf("Save: %v", err) } if len(merkles) != 1 { return nil, fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", @@ -543,7 +488,7 @@ func commentVoteSave(client *tlogbe.PluginClient, c commentVote) ([]byte, error) return merkles[0], nil } -func indexSave(client *tlogbe.PluginClient, idx index) error { +func indexSave(client *tlogbe.RecordClient, idx index) error { // Prepare blob be, err := convertBlobEntryFromIndex(idx) if err != nil { @@ -559,10 +504,10 @@ func indexSave(client *tlogbe.PluginClient, idx index) error { } // Save blob - merkles, err := client.BlobsSave(keyPrefixCommentsIndex, + merkles, err := client.Save(keyPrefixCommentsIndex, [][]byte{b}, [][]byte{h}, false) if err != nil { - return fmt.Errorf("BlobsSave: %v", err) + return fmt.Errorf("Save: %v", err) } if len(merkles) != 1 { return fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", @@ -578,7 +523,7 @@ func indexSave(client *tlogbe.PluginClient, idx index) error { // Example, a user upvotes a comment that they have already upvoted, the // resulting vote score is 0 due to the second upvote removing the original // upvote. -func indexAddCommentVote(idx index, cv commentVote, merkleHash []byte) index { +func indexAddCommentVote(idx index, cv comments.CommentVote, merkleHash []byte) index { // Get the existing votes for this uuid cidx := idx.Comments[cv.CommentID] votes, ok := cidx.Votes[cv.UUID] @@ -624,7 +569,7 @@ func indexAddCommentVote(idx index, cv commentVote, merkleHash []byte) index { return idx } -func indexLatest(client *tlogbe.PluginClient) (*index, error) { +func indexLatest(client *tlogbe.RecordClient) (*index, error) { // Get all comment indexes blobs, err := client.BlobsByKeyPrefix(keyPrefixCommentsIndex) if err != nil { @@ -678,7 +623,7 @@ func commentExists(idx index, commentID uint32) bool { // the responsibility of the caller to ensure a comment is returned for each of // the provided comment IDs. The comments index that was looked up during this // process is also returned. -func commentsLatest(client *tlogbe.PluginClient, idx index, commentIDs []uint32) (map[uint32]comments.Comment, error) { +func commentsLatest(client *tlogbe.RecordClient, idx index, commentIDs []uint32) (map[uint32]comments.Comment, error) { // Aggregate the merkle hashes for all records that need to be // looked up. If a comment has been deleted then the only record // that will still exist is the comment del record. If the comment @@ -742,14 +687,14 @@ func commentsLatest(client *tlogbe.PluginClient, idx index, commentIDs []uint32) } // This function must be called WITH the record lock held. -func (p *commentsPlugin) new(client *tlogbe.PluginClient, n comments.New, encrypt bool) (*comments.NewReply, error) { +func (p *commentsPlugin) new(client *tlogbe.RecordClient, n comments.New, encrypt bool) (*comments.NewReply, error) { // Pull comments index idx, err := indexLatest(client) if err != nil { return nil, err } - // Ensure parent comment exists if set. A parent ID of 0 means that + // Verify parent comment exists if set. A parent ID of 0 means that // this is a base level comment, not a reply to another comment. if n.ParentID > 0 && !commentExists(*idx, n.ParentID) { return nil, comments.UserError{ @@ -760,7 +705,7 @@ func (p *commentsPlugin) new(client *tlogbe.PluginClient, n comments.New, encryp // Setup comment receipt := p.id.SignMessage([]byte(n.Signature)) - c := commentAdd{ + c := comments.CommentAdd{ Token: n.Token, ParentID: n.ParentID, Comment: n.Comment, @@ -819,14 +764,14 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { return "", convertCommentsErrFromSignatureErr(err) } - // Get plugin client + // Get record client token, err := hex.DecodeString(n.Token) if err != nil { return "", comments.UserError{ ErrorCode: comments.ErrorStatusTokenInvalid, } } - client, err := p.backend.PluginClient(token) + client, err := p.backend.RecordClient(token) if err != nil { if err == backend.ErrRecordNotFound { return "", comments.UserError{ @@ -869,7 +814,7 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { } // This function must be called WITH the record lock held. -func (p *commentsPlugin) edit(client *tlogbe.PluginClient, e comments.Edit, encrypt bool) (*comments.EditReply, error) { +func (p *commentsPlugin) edit(client *tlogbe.RecordClient, e comments.Edit, encrypt bool) (*comments.EditReply, error) { // Get comments index idx, err := indexLatest(client) if err != nil { @@ -906,7 +851,7 @@ func (p *commentsPlugin) edit(client *tlogbe.PluginClient, e comments.Edit, encr // Create a new comment version receipt := p.id.SignMessage([]byte(e.Signature)) - c := commentAdd{ + c := comments.CommentAdd{ Token: e.Token, ParentID: e.ParentID, Comment: e.Comment, @@ -959,14 +904,14 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { return "", convertCommentsErrFromSignatureErr(err) } - // Get plugin client + // Get record client token, err := hex.DecodeString(e.Token) if err != nil { return "", comments.UserError{ ErrorCode: comments.ErrorStatusTokenInvalid, } } - client, err := p.backend.PluginClient(token) + client, err := p.backend.RecordClient(token) if err != nil { if err == backend.ErrRecordNotFound { return "", comments.UserError{ @@ -1009,7 +954,7 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { } // This function must be called WITH the record lock held. -func (p *commentsPlugin) del(client *tlogbe.PluginClient, d comments.Del) (*comments.DelReply, error) { +func (p *commentsPlugin) del(client *tlogbe.RecordClient, d comments.Del) (*comments.DelReply, error) { // Get comments index idx, err := indexLatest(client) if err != nil { @@ -1030,7 +975,7 @@ func (p *commentsPlugin) del(client *tlogbe.PluginClient, d comments.Del) (*comm // Save delete record receipt := p.id.SignMessage([]byte(d.Signature)) - cd := commentDel{ + cd := comments.CommentDel{ Token: d.Token, CommentID: d.CommentID, Reason: d.Reason, @@ -1065,7 +1010,7 @@ func (p *commentsPlugin) del(client *tlogbe.PluginClient, d comments.Del) (*comm for _, v := range cidx.Adds { merkles = append(merkles, v) } - err = client.BlobsDel(merkles) + err = client.Del(merkles) if err != nil { return nil, fmt.Errorf("BlobsDel: %v", err) } @@ -1092,14 +1037,14 @@ func (p *commentsPlugin) cmdDel(payload string) (string, error) { return "", convertCommentsErrFromSignatureErr(err) } - // Get plugin client + // Get record client token, err := hex.DecodeString(d.Token) if err != nil { return "", comments.UserError{ ErrorCode: comments.ErrorStatusTokenInvalid, } } - client, err := p.backend.PluginClient(token) + client, err := p.backend.RecordClient(token) if err != nil { if err == backend.ErrRecordNotFound { return "", comments.UserError{ @@ -1139,14 +1084,14 @@ func (p *commentsPlugin) cmdGet(payload string) (string, error) { return "", err } - // Get plugin client + // Get record client token, err := hex.DecodeString(g.Token) if err != nil { return "", comments.UserError{ ErrorCode: comments.ErrorStatusTokenInvalid, } } - client, err := p.backend.PluginClient(token) + client, err := p.backend.RecordClient(token) if err != nil { if err == backend.ErrRecordNotFound { return "", comments.UserError{ @@ -1189,14 +1134,14 @@ func (p *commentsPlugin) cmdGetAll(payload string) (string, error) { return "", err } - // Get plugin client + // Get record client token, err := hex.DecodeString(ga.Token) if err != nil { return "", comments.UserError{ ErrorCode: comments.ErrorStatusTokenInvalid, } } - client, err := p.backend.PluginClient(token) + client, err := p.backend.RecordClient(token) if err != nil { if err == backend.ErrRecordNotFound { return "", comments.UserError{ @@ -1256,14 +1201,14 @@ func (p *commentsPlugin) cmdGetVersion(payload string) (string, error) { return "", err } - // Get plugin client + // Get record client token, err := hex.DecodeString(gv.Token) if err != nil { return "", comments.UserError{ ErrorCode: comments.ErrorStatusTokenInvalid, } } - client, err := p.backend.PluginClient(token) + client, err := p.backend.RecordClient(token) if err != nil { if err == backend.ErrRecordNotFound { return "", comments.UserError{ @@ -1279,7 +1224,7 @@ func (p *commentsPlugin) cmdGetVersion(payload string) (string, error) { return "", fmt.Errorf("indexLatest: %v", err) } - // Ensure comment exists + // Verify comment exists cidx, ok := idx.Comments[gv.CommentID] if !ok { return "", comments.UserError{ @@ -1337,14 +1282,14 @@ func (p *commentsPlugin) cmdCount(payload string) (string, error) { return "", err } - // Get plugin client + // Get record client token, err := hex.DecodeString(c.Token) if err != nil { return "", comments.UserError{ ErrorCode: comments.ErrorStatusTokenInvalid, } } - client, err := p.backend.PluginClient(token) + client, err := p.backend.RecordClient(token) if err != nil { if err == backend.ErrRecordNotFound { return "", comments.UserError{ @@ -1401,14 +1346,14 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { return "", convertCommentsErrFromSignatureErr(err) } - // Get plugin client + // Get record client token, err := hex.DecodeString(v.Token) if err != nil { return "", comments.UserError{ ErrorCode: comments.ErrorStatusTokenInvalid, } } - client, err := p.backend.PluginClient(token) + client, err := p.backend.RecordClient(token) if err != nil { if err == backend.ErrRecordNotFound { return "", comments.UserError{ @@ -1430,7 +1375,7 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { return "", fmt.Errorf("indexLatest: %v", err) } - // Ensure comment exists + // Verify comment exists cidx, ok := idx.Comments[v.CommentID] if !ok { return "", comments.UserError{ @@ -1438,7 +1383,7 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { } } - // Ensure user has not exceeded max allowed vote changes + // Verify user has not exceeded max allowed vote changes uvotes, ok := cidx.Votes[v.UUID] if !ok { uvotes = make([]voteIndex, 0) @@ -1451,7 +1396,7 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { // Prepare comment vote receipt := p.id.SignMessage([]byte(v.Signature)) - cv := commentVote{ + cv := comments.CommentVote{ UUID: v.UUID, Token: v.Token, CommentID: v.CommentID, @@ -1523,14 +1468,14 @@ func (p *commentsPlugin) Cmd(cmd, payload string) (string, error) { return p.cmdProofs(payload) } - return "", ErrInvalidPluginCmd + return "", backend.ErrPluginCmdInvalid } // Hook executes a plugin hook. // // This function satisfies the Plugin interface. -func (p *commentsPlugin) Hook(h HookT, payload string) error { - log.Tracef("comments Hook: %v", Hook[h]) +func (p *commentsPlugin) Hook(h tlogbe.HookT, payload string) error { + log.Tracef("comments Hook: %v", tlogbe.Hooks[h]) return nil } @@ -1540,7 +1485,7 @@ func (p *commentsPlugin) Hook(h HookT, payload string) error { func (p *commentsPlugin) Fsck() error { log.Tracef("comments Fsck") - // Make sure commentDel blobs were actually deleted + // Make sure CommentDel blobs were actually deleted return nil } @@ -1553,8 +1498,8 @@ func (p *commentsPlugin) Setup() error { return nil } -// CommentsPluginNew returns a new comments plugin. -func CommentsPluginNew(backend *tlogbe.Tlogbe, settings []backend.PluginSetting) *commentsPlugin { +// NewCommentsPlugin returns a new comments plugin. +func NewCommentsPlugin(backend *tlogbe.Tlogbe, settings []backend.PluginSetting) *commentsPlugin { // TODO these should be passed in as plugin settings id := &identity.FullIdentity{} diff --git a/politeiad/backend/tlogbe/plugins/dcrdata.go b/politeiad/backend/tlogbe/plugins/dcrdata.go index 728dee953..3f12dc7be 100644 --- a/politeiad/backend/tlogbe/plugins/dcrdata.go +++ b/politeiad/backend/tlogbe/plugins/dcrdata.go @@ -21,6 +21,7 @@ import ( pstypes "github.com/decred/dcrdata/pubsub/types/v3" "github.com/decred/politeia/plugins/dcrdata" "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe" "github.com/decred/politeia/util" "github.com/decred/politeia/wsdcrdata" ) @@ -34,7 +35,7 @@ const ( ) var ( - _ Plugin = (*dcrdataPlugin)(nil) + _ tlogbe.Plugin = (*dcrdataPlugin)(nil) ) // dcrdataplugin satisfies the Plugin interface. @@ -361,14 +362,14 @@ func (p *dcrdataPlugin) Cmd(cmd, payload string) (string, error) { return p.cmdTxsTrimmed(payload) } - return "", ErrInvalidPluginCmd + return "", backend.ErrPluginCmdInvalid } // Hook executes a plugin hook. // // This function satisfies the Plugin interface. -func (p *dcrdataPlugin) Hook(h HookT, payload string) error { - log.Tracef("dcrdata Hook: %v %v", h, payload) +func (p *dcrdataPlugin) Hook(h tlogbe.HookT, payload string) error { + log.Tracef("dcrdata Hook: %v %v", tlogbe.Hooks[h], payload) return nil } diff --git a/politeiad/backend/tlogbe/plugins/plugins.go b/politeiad/backend/tlogbe/plugins/plugins.go deleted file mode 100644 index deaa2a87f..000000000 --- a/politeiad/backend/tlogbe/plugins/plugins.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) 2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package plugins - -import "errors" - -type HookT int - -const ( - // Plugin hooks - HookInvalid HookT = 0 - HookPostNewRecord HookT = 1 - HookPostEditRecord HookT = 2 - HookPostEditMetadata HookT = 3 - HookPostSetRecordStatus HookT = 4 -) - -var ( - // Human readable plugin hooks - Hook = map[HookT]string{ - HookPostNewRecord: "post new record", - HookPostEditRecord: "post edit record", - HookPostEditMetadata: "post edit metadata", - HookPostSetRecordStatus: "post set record status", - } - - // ErrInvalidPluginCmd is emitted when an invalid plugin command is - // used. - ErrInvalidPluginCmd = errors.New("invalid plugin command") -) - -// Plugin provides an interface for the backend to use when interacting with -// plugins. -type Plugin interface { - // Perform plugin setup - Setup() error - - // Execute a plugin command - Cmd(cmd, payload string) (string, error) - - // Execute a plugin hook - Hook(h HookT, payload string) error - - // Perform a plugin file system check - Fsck() error -} diff --git a/politeiad/backend/tlogbe/plugins/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote.go index b480d97ca..446557ec6 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote.go @@ -37,8 +37,8 @@ const ( // ticketVoteDirname is the ticket vote data directory name. ticketVoteDirname = "ticketvote" - // Filenames of memoized data saved to data dir. - summaryFilename = "{token}-summary.json" + // Filenames of memoized data saved to the data dir. + filenameSummary = "{token}-summary.json" // Blob entry data descriptors dataDescriptorAuthorizeVote = "authorizevote" @@ -53,7 +53,7 @@ const ( ) var ( - _ Plugin = (*ticketVotePlugin)(nil) + _ tlogbe.Plugin = (*ticketVotePlugin)(nil) // Local errors errRecordNotFound = errors.New("record not found") @@ -76,23 +76,74 @@ type ticketVotePlugin struct { // once a record vote has ended. dataDir string - // castVotes contains the cast votes of ongoing record votes. - // TODO load on startup and cleanup once vote is finished. If we - // save these to the data dir instead of memory then there won't - // be anything to build on startup. - castVotes map[string]map[string]string // [token][ticket]voteBit + // inv contains the record inventory catagorized by vote status. + // The inventory will only contain public, non-abandoned records. + // This cache is built on startup. + inv inventory + + // votes contains the cast votes of ongoing record votes. This + // cache is built on startup and record entries are removed once + // the vote has ended and the vote summary has been saved. + votes map[string]map[string]string // [token][ticket]voteBit // Mutexes contains a mutex for each record. The mutexes are lazy // loaded. mutexes map[string]*sync.Mutex // [string]mutex } -func (p *ticketVotePlugin) cachedCastVotes(token string) map[string]string { +type inventory struct { + unauthorized []string // Unauthorized tokens + authorized []string // Authorized tokens + started map[string]uint32 // [token]endHeight + finished []string // Finished tokens + bestBlock uint32 // Height of last inventory update +} + +func (p *ticketVotePlugin) cachedInventory() inventory { + p.Lock() + defer p.Unlock() + + // Return a copy of the inventory + var ( + unauthorized = make([]string, len(p.inv.unauthorized)) + authorized = make([]string, len(p.inv.authorized)) + started = make(map[string]uint32, len(p.inv.started)) + finished = make([]string, len(p.inv.finished)) + ) + for k, v := range p.inv.unauthorized { + unauthorized[k] = v + } + for k, v := range p.inv.authorized { + authorized[k] = v + } + for k, v := range p.inv.started { + started[k] = v + } + for k, v := range p.inv.finished { + finished[k] = v + } + + return inventory{ + unauthorized: unauthorized, + authorized: authorized, + started: started, + finished: finished, + } +} + +func (p *ticketVotePlugin) cachedInventorySet(inv inventory) { + p.Lock() + defer p.Unlock() + + p.inv = inv +} + +func (p *ticketVotePlugin) cachedVotes(token string) map[string]string { p.Lock() defer p.Unlock() // Return a copy of the map - cv, ok := p.castVotes[token] + cv, ok := p.votes[token] if !ok { return map[string]string{} } @@ -104,20 +155,31 @@ func (p *ticketVotePlugin) cachedCastVotes(token string) map[string]string { return c } -func (p *ticketVotePlugin) cachedCastVotesSave(token, ticket, voteBit string) { +func (p *ticketVotePlugin) cachedVotesSet(token, ticket, voteBit string) { p.Lock() defer p.Unlock() - _, ok := p.castVotes[token] + _, ok := p.votes[token] if !ok { - p.castVotes[token] = make(map[string]string, 40960) // Ticket pool size + p.votes[token] = make(map[string]string, 40960) // Ticket pool size } - p.castVotes[token][ticket] = voteBit + p.votes[token][ticket] = voteBit + + log.Debugf("Votes add: %v %v %v", token, ticket, voteBit) +} + +func (p *ticketVotePlugin) cachedVotesDel(token string) { + p.Lock() + defer p.Unlock() + + delete(p.votes, token) + + log.Debugf("Votes del: %v", token) } func (p *ticketVotePlugin) cachedSummaryPath(token string) string { - fn := strings.Replace(summaryFilename, "{token}", token, 1) + fn := strings.Replace(filenameSummary, "{token}", token, 1) return filepath.Join(p.dataDir, fn) } @@ -146,7 +208,7 @@ func (p *ticketVotePlugin) cachedSummary(token string) (*ticketvote.Summary, err return &s, nil } -func (p *ticketVotePlugin) cachedSummarySave(token string, s ticketvote.Summary) error { +func (p *ticketVotePlugin) cachedSummarySet(token string, s ticketvote.Summary) error { b, err := json.Marshal(s) if err != nil { return err @@ -361,7 +423,7 @@ func convertBlobEntryFromCastVote(cv ticketvote.CastVote) (*store.BlobEntry, err return &be, nil } -func authorizeVoteSave(client *tlogbe.PluginClient, av ticketvote.AuthorizeVote) error { +func authorizeVoteSave(client *tlogbe.RecordClient, av ticketvote.AuthorizeVote) error { // Prepare blob be, err := convertBlobEntryFromAuthorizeVote(av) if err != nil { @@ -377,10 +439,10 @@ func authorizeVoteSave(client *tlogbe.PluginClient, av ticketvote.AuthorizeVote) } // Save blob - merkles, err := client.BlobsSave(keyPrefixAuthorizeVote, + merkles, err := client.Save(keyPrefixAuthorizeVote, [][]byte{b}, [][]byte{h}, false) if err != nil { - return fmt.Errorf("BlobsSave: %v", err) + return fmt.Errorf("Save: %v", err) } if len(merkles) != 1 { return fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", @@ -390,7 +452,7 @@ func authorizeVoteSave(client *tlogbe.PluginClient, av ticketvote.AuthorizeVote) return nil } -func authorizeVotes(client *tlogbe.PluginClient) ([]ticketvote.AuthorizeVote, error) { +func authorizeVotes(client *tlogbe.RecordClient) ([]ticketvote.AuthorizeVote, error) { // Retrieve blobs blobs, err := client.BlobsByKeyPrefix(keyPrefixAuthorizeVote) if err != nil { @@ -414,7 +476,7 @@ func authorizeVotes(client *tlogbe.PluginClient) ([]ticketvote.AuthorizeVote, er return auths, nil } -func startVoteSave(client *tlogbe.PluginClient, sv ticketvote.StartVote) error { +func startVoteSave(client *tlogbe.RecordClient, sv ticketvote.StartVote) error { // Prepare blob be, err := convertBlobEntryFromStartVote(sv) if err != nil { @@ -430,10 +492,10 @@ func startVoteSave(client *tlogbe.PluginClient, sv ticketvote.StartVote) error { } // Save blob - merkles, err := client.BlobsSave(keyPrefixStartVote, + merkles, err := client.Save(keyPrefixStartVote, [][]byte{b}, [][]byte{h}, false) if err != nil { - return fmt.Errorf("BlobsSave: %v", err) + return fmt.Errorf("Save: %v", err) } if len(merkles) != 1 { return fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", @@ -444,7 +506,7 @@ func startVoteSave(client *tlogbe.PluginClient, sv ticketvote.StartVote) error { } // startVote returns the StartVote for the provided record if one exists. -func startVote(client *tlogbe.PluginClient) (*ticketvote.StartVote, error) { +func startVote(client *tlogbe.RecordClient) (*ticketvote.StartVote, error) { // Retrieve blobs blobs, err := client.BlobsByKeyPrefix(keyPrefixStartVote) if err != nil { @@ -476,7 +538,7 @@ func startVote(client *tlogbe.PluginClient) (*ticketvote.StartVote, error) { return sv, nil } -func castVotes(client *tlogbe.PluginClient) ([]ticketvote.CastVote, error) { +func castVotes(client *tlogbe.RecordClient) ([]ticketvote.CastVote, error) { // Retrieve blobs blobs, err := client.BlobsByKeyPrefix(keyPrefixCastVote) if err != nil { @@ -500,7 +562,7 @@ func castVotes(client *tlogbe.PluginClient) ([]ticketvote.CastVote, error) { return votes, nil } -func castVoteSave(client *tlogbe.PluginClient, cv ticketvote.CastVote) error { +func castVoteSave(client *tlogbe.RecordClient, cv ticketvote.CastVote) error { // Prepare blob be, err := convertBlobEntryFromCastVote(cv) if err != nil { @@ -516,10 +578,10 @@ func castVoteSave(client *tlogbe.PluginClient, cv ticketvote.CastVote) error { } // Save blob - merkles, err := client.BlobsSave(keyPrefixCastVote, + merkles, err := client.Save(keyPrefixCastVote, [][]byte{b}, [][]byte{h}, false) if err != nil { - return fmt.Errorf("BlobsSave: %v", err) + return fmt.Errorf("Save: %v", err) } if len(merkles) != 1 { return fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", @@ -530,10 +592,7 @@ func castVoteSave(client *tlogbe.PluginClient, cv ticketvote.CastVote) error { } // bestBlock fetches the best block from the dcrdata plugin and returns it. If -// the dcrdata connection is not active, an error WILL NOT BE returned. The -// dcrdata cached best block height will be returned even though it may be -// stale. Use bestBlockSafe() if the caller requires a guarantee that the best -// block is not stale. +// the dcrdata connection is not active, an error will be returned. func (p *ticketVotePlugin) bestBlock() (uint32, error) { // Get best block payload, err := dcrdata.EncodeBestBlock(dcrdata.BestBlock{}) @@ -552,6 +611,11 @@ func (p *ticketVotePlugin) bestBlock() (uint32, error) { if err != nil { return 0, err } + if bbr.Status != dcrdata.StatusConnected { + // The dcrdata connection is down. The best block cannot be + // trusted as being accurate. + return 0, fmt.Errorf("dcrdata connection is down") + } if bbr.Height == 0 { return 0, fmt.Errorf("invalid best block height 0") } @@ -559,9 +623,12 @@ func (p *ticketVotePlugin) bestBlock() (uint32, error) { return bbr.Height, nil } -// bestBlockSafe fetches the best block from the dcrdata plugin and returns it. -// If the dcrdata connection is not active, an error WILL BE returned. -func (p *ticketVotePlugin) bestBlockSafe() (uint32, error) { +// bestBlockUnsafe fetches the best block from the dcrdata plugin and returns +// it. If the dcrdata connection is not active, an error WILL NOT be returned. +// The dcrdata cached best block height will be returned even though it may be +// stale. Use bestBlock() if the caller requires a guarantee that the best +// block is not stale. +func (p *ticketVotePlugin) bestBlockUnsafe() (uint32, error) { // Get best block payload, err := dcrdata.EncodeBestBlock(dcrdata.BestBlock{}) if err != nil { @@ -579,11 +646,6 @@ func (p *ticketVotePlugin) bestBlockSafe() (uint32, error) { if err != nil { return 0, err } - if bbr.Status != dcrdata.StatusConnected { - // The dcrdata connection is down. The best block cannot be - // trusted as being accurate. - return 0, fmt.Errorf("dcrdata connection is down") - } if bbr.Height == 0 { return 0, fmt.Errorf("invalid best block height 0") } @@ -656,9 +718,9 @@ func (p *ticketVotePlugin) largestCommitmentAddrs(tickets []string) ([]commitmen // startReply fetches all required data and returns a StartReply. func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, error) { // Get the best block height - bb, err := p.bestBlockSafe() + bb, err := p.bestBlock() if err != nil { - return nil, fmt.Errorf("bestBlockSafe: %v", err) + return nil, fmt.Errorf("bestBlock: %v", err) } // Find the snapshot height. Subtract the ticket maturity from the @@ -816,14 +878,14 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { return "", convertTicketVoteErrFromSignatureErr(err) } - // Get plugin client + // Get record client tokenb, err := hex.DecodeString(a.Token) if err != nil { return "", ticketvote.UserError{ ErrorCode: ticketvote.ErrorStatusTokenInvalid, } } - client, err := p.backend.PluginClient(tokenb) + client, err := p.backend.RecordClient(tokenb) if err != nil { if err == backend.ErrRecordNotFound { return "", ticketvote.UserError{ @@ -1099,14 +1161,14 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { return "", err } - // Get plugin client + // Get record client tokenb, err := hex.DecodeString(s.Vote.Token) if err != nil { return "", ticketvote.UserError{ ErrorCode: ticketvote.ErrorStatusTokenInvalid, } } - client, err := p.backend.PluginClient(tokenb) + client, err := p.backend.RecordClient(tokenb) if err != nil { if err == backend.ErrRecordNotFound { return "", ticketvote.UserError{ @@ -1273,7 +1335,7 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { return string(reply), nil } - // Get plugin client + // Get record client var ( votes = ballot.Votes token = votes[0].Token @@ -1284,7 +1346,7 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { c := fmt.Sprintf("%v: not hex", ticketvote.VoteError[e]) return ballotExitWithErr(votes, e, c) } - client, err := p.backend.PluginClient(tokenb) + client, err := p.backend.RecordClient(tokenb) if err != nil { if err == backend.ErrRecordNotFound { e := ticketvote.VoteErrorRecordNotFound @@ -1310,7 +1372,7 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { c := fmt.Sprintf("%v: vote not started", ticketvote.VoteError[e]) return ballotExitWithErr(votes, e, c) } - bb, err := p.bestBlockSafe() + bb, err := p.bestBlock() if err != nil { return "", err } @@ -1345,7 +1407,7 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { defer m.Unlock() // castVotes contains the tickets that have alread voted - castVotes := p.cachedCastVotes(token) + castVotes := p.cachedVotes(token) // Verify and save votes receipts := make([]ticketvote.VoteReply, len(votes)) @@ -1452,7 +1514,7 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { receipts[k].Receipt = cv.Receipt // Update cast votes cache - p.cachedCastVotesSave(v.Token, v.Ticket, v.VoteBit) + p.cachedVotesSet(v.Token, v.Ticket, v.VoteBit) } // Prepare reply @@ -1475,14 +1537,14 @@ func (p *ticketVotePlugin) cmdDetails(payload string) (string, error) { return "", err } - // Get plugin client + // Get record client tokenb, err := hex.DecodeString(d.Token) if err != nil { return "", ticketvote.UserError{ ErrorCode: ticketvote.ErrorStatusTokenInvalid, } } - client, err := p.backend.PluginClient(tokenb) + client, err := p.backend.RecordClient(tokenb) if err != nil { if err == backend.ErrRecordNotFound { return "", ticketvote.UserError{ @@ -1526,14 +1588,14 @@ func (p *ticketVotePlugin) cmdCastVotes(payload string) (string, error) { return "", err } - // Get plugin client + // Get record client tokenb, err := hex.DecodeString(cv.Token) if err != nil { return "", ticketvote.UserError{ ErrorCode: ticketvote.ErrorStatusTokenInvalid, } } - client, err := p.backend.PluginClient(tokenb) + client, err := p.backend.RecordClient(tokenb) if err != nil { if err == backend.ErrRecordNotFound { return "", ticketvote.UserError{ @@ -1561,29 +1623,12 @@ func (p *ticketVotePlugin) cmdCastVotes(payload string) (string, error) { return string(reply), nil } -func summaryPrepare(sv ticketvote.StartVote, castVotes map[string]string, bestBlock uint32) ticketvote.Summary { - // TODO - return ticketvote.Summary{ - Type: sv.Vote.Type, - // Status: , - Duration: sv.Vote.Duration, - StartBlockHeight: sv.StartBlockHeight, - StartBlockHash: sv.StartBlockHash, - EndBlockHeight: sv.EndBlockHeight, - EligibleTickets: uint32(len(sv.EligibleTickets)), - QuorumPercentage: sv.Vote.QuorumPercentage, - PassPercentage: sv.Vote.PassPercentage, - // Results: , - // Approved: , - } -} - func (p *ticketVotePlugin) summary(token string, bestBlock uint32) (*ticketvote.Summary, error) { // Check if the summary has been cached s, err := p.cachedSummary(token) switch { case err == errRecordNotFound: - // Cached summary not found. This is ok; continue. + // Cached summary not found case err != nil: // Some other error return nil, fmt.Errorf("cachedSummary: %v", err) @@ -1592,17 +1637,18 @@ func (p *ticketVotePlugin) summary(token string, bestBlock uint32) (*ticketvote. return s, nil } - // Cached summary not found. Get the plugin client. + // Cached summary not found. Get the record client so we can create + // a summary manually. tokenb, err := hex.DecodeString(token) if err != nil { return nil, errRecordNotFound } - client, err := p.backend.PluginClient(tokenb) + client, err := p.backend.RecordClient(tokenb) if err != nil { return nil, errRecordNotFound } if client.State != tlogbe.RecordStateVetted { - // Record exists but is unvetted so a vote can not have + // Record exists but is unvetted so a vote can not have been // authorized yet. return &ticketvote.Summary{ Status: ticketvote.VoteStatusUnauthorized, @@ -1647,11 +1693,100 @@ func (p *ticketVotePlugin) summary(token string, bestBlock uint32) (*ticketvote. }, nil } - // Vote has been started. Vote is either still in progress or has - // ended but the summary hasn't been cached yet. Pull the cast - // votes from the cache and calulate the results manually. - votes := p.cachedCastVotes(token) - summary := summaryPrepare(*sv, votes, bestBlock) + // Vote has been started. Check if it is still in progress or has + // already ended. + var status ticketvote.VoteStatusT + if bestBlock < sv.EndBlockHeight { + status = ticketvote.VoteStatusStarted + } else { + status = ticketvote.VoteStatusFinished + } + + // Pull the cast votes from the cache and calculate the results + // manually. + votes := p.cachedVotes(token) + tally := make(map[string]int, len(sv.Vote.Options)) + for _, voteBit := range votes { + tally[voteBit]++ + } + results := make([]ticketvote.Result, len(sv.Vote.Options)) + for _, v := range sv.Vote.Options { + bit := strconv.FormatUint(v.Bit, 16) + results = append(results, ticketvote.Result{ + ID: v.ID, + Description: v.Description, + VoteBit: v.Bit, + Votes: uint64(tally[bit]), + }) + } + + // Approved can only be calculated on certain types of votes + var approved bool + switch sv.Vote.Type { + case ticketvote.VoteTypeStandard, ticketvote.VoteTypeRunoff: + // Calculate results for a simple approve/reject vote + var total uint64 + for _, v := range results { + total += v.Votes + } + + var ( + eligible = float64(len(sv.EligibleTickets)) + quorumPerc = float64(sv.Vote.QuorumPercentage) + passPerc = float64(sv.Vote.PassPercentage) + quorum = uint64(quorumPerc / 100 * eligible) + pass = uint64(passPerc / 100 * float64(total)) + + approvedVotes uint64 + ) + for _, v := range results { + if v.ID == ticketvote.VoteOptionIDApprove { + approvedVotes++ + } + } + + switch { + case total < quorum: + // Quorum not met + approved = false + case approvedVotes < pass: + // Pass percentage not met + approved = false + default: + // Vote was approved + approved = true + } + } + + // Prepare summary + summary := ticketvote.Summary{ + Type: sv.Vote.Type, + Status: status, + Duration: sv.Vote.Duration, + StartBlockHeight: sv.StartBlockHeight, + StartBlockHash: sv.StartBlockHash, + EndBlockHeight: sv.EndBlockHeight, + EligibleTickets: uint32(len(sv.EligibleTickets)), + QuorumPercentage: sv.Vote.QuorumPercentage, + PassPercentage: sv.Vote.PassPercentage, + Results: results, + Approved: approved, + } + + // Cache the summary if the vote has finished so we don't have to + // calculate these results again. + if status == ticketvote.VoteStatusFinished { + // Save summary + err = p.cachedSummarySet(sv.Vote.Token, summary) + if err != nil { + return nil, fmt.Errorf("cachedSummarySet %v: %v %v", + sv.Vote.Token, err, summary) + } + + // Remove record from the votes cache now that a summary has + // been saved for it. + p.cachedVotesDel(sv.Vote.Token) + } return &summary, nil } @@ -1667,7 +1802,7 @@ func (p *ticketVotePlugin) cmdSummaries(payload string) (string, error) { // Get best block. This cmd does not write any data so we do not // have to use the safe best block. - bb, err := p.bestBlock() + bb, err := p.bestBlockUnsafe() if err != nil { return "", fmt.Errorf("bestBlock: %v", err) } @@ -1699,10 +1834,114 @@ func (p *ticketVotePlugin) cmdSummaries(payload string) (string, error) { return string(reply), nil } +func (p *ticketVotePlugin) inventory(bestBlock uint32) (*ticketvote.InventoryReply, error) { + // Get existing inventory + inv := p.cachedInventory() + + // Get backend inventory to see if there are any new unauthorized + // records. + invBackend, err := p.backend.InventoryByStatus() + if err != nil { + return nil, err + } + l := len(inv.unauthorized) + len(inv.authorized) + + len(inv.started) + len(inv.finished) + if l != len(invBackend.Vetted) { + // There are new unauthorized records. Put all ticket vote + // inventory records into a map so we can easily find what + // backend records are missing. + all := make(map[string]struct{}, l) + for _, v := range inv.unauthorized { + all[v] = struct{}{} + } + for _, v := range inv.authorized { + all[v] = struct{}{} + } + for k := range inv.started { + all[k] = struct{}{} + } + for _, v := range inv.finished { + all[v] = struct{}{} + } + + // Add any missing records to the inventory + for _, v := range invBackend.Vetted { + if _, ok := all[v]; !ok { + inv.unauthorized = append(inv.unauthorized, v) + } + } + + // Update cache + p.cachedInventorySet(inv) + } + + // Check if inventory has already been updated for this block + // height. + if inv.bestBlock == bestBlock { + // Inventory already updated. Nothing else to do. + started := make([]string, 0, len(inv.started)) + for k := range inv.started { + started = append(started, k) + } + return &ticketvote.InventoryReply{ + Unauthorized: inv.unauthorized, + Authorized: inv.authorized, + Started: started, + Finished: inv.finished, + BestBlock: bestBlock, + }, nil + } + + // Inventory has not been updated for this block height. Check if + // any proposal votes have finished. + started := make([]string, 0, len(inv.started)) + for token, endHeight := range inv.started { + if bestBlock >= endHeight { + // Vote has finished + inv.finished = append(inv.finished, token) + } else { + // Vote is still ongoing + started = append(started, token) + } + } + + // Update cache + p.cachedInventorySet(inv) + + return &ticketvote.InventoryReply{ + Unauthorized: inv.unauthorized, + Authorized: inv.authorized, + Started: started, + Finished: inv.finished, + BestBlock: bestBlock, + }, nil +} + func (p *ticketVotePlugin) cmdInventory(payload string) (string, error) { log.Tracef("ticketvote cmdInventory: %v", payload) - return "", nil + // Payload is empty. Nothing to decode. + + // Get best block. This command does not write any data so we can + // use the unsafe best block. + bb, err := p.bestBlockUnsafe() + if err != nil { + return "", err + } + + // Get the inventory + ir, err := p.inventory(bb) + if err != nil { + return "", fmt.Errorf("inventory: %v", err) + } + + // Prepare reply + reply, err := ticketvote.EncodeInventoryReply(*ir) + if err != nil { + return "", err + } + + return string(reply), nil } // Cmd executes a plugin command. @@ -1730,14 +1969,14 @@ func (p *ticketVotePlugin) Cmd(cmd, payload string) (string, error) { return p.cmdInventory(payload) } - return "", ErrInvalidPluginCmd + return "", backend.ErrPluginCmdInvalid } // Hook executes a plugin hook. // // This function satisfies the Plugin interface. -func (p *ticketVotePlugin) Hook(h HookT, payload string) error { - log.Tracef("ticketvote Hook: %v %v", h, payload) +func (p *ticketVotePlugin) Hook(h tlogbe.HookT, payload string) error { + log.Tracef("ticketvote Hook: %v %v", tlogbe.Hooks[h], payload) return nil } @@ -1757,8 +1996,10 @@ func (p *ticketVotePlugin) Fsck() error { func (p *ticketVotePlugin) Setup() error { log.Tracef("ticketvote Setup") + // TODO // Ensure dcrdata plugin has been registered - // Build vote summaries + // Build votes cache + // Build inventory cache return nil } diff --git a/politeiad/backend/tlogbe/store/filesystem/filesystem.go b/politeiad/backend/tlogbe/store/filesystem/filesystem.go index c2a06bc6c..e378a2bcf 100644 --- a/politeiad/backend/tlogbe/store/filesystem/filesystem.go +++ b/politeiad/backend/tlogbe/store/filesystem/filesystem.go @@ -12,32 +12,43 @@ import ( "sync" "github.com/decred/politeia/politeiad/backend/tlogbe/store" + "github.com/google/uuid" ) var ( -// TODO put back -// _ store.Blob = (*fileSystem)(nil) + _ store.Blob = (*fileSystem)(nil) ) // fileSystem implements the Blob interface using the file system. +// +// This implementation should be used for TESTING ONLY. type fileSystem struct { sync.RWMutex root string // Location of files } +// put saves a files to the file system. +// +// This function must be called WITH the lock held. func (f *fileSystem) put(key string, value []byte) error { return ioutil.WriteFile(filepath.Join(f.root, key), value, 0600) } -func (f *fileSystem) Put(key string, value []byte) error { - log.Tracef("Put: %v", key) - - f.Lock() - defer f.Unlock() - - return f.put(key, value) +// This function must be called WITH the lock held. +func (f *fileSystem) del(key string) error { + err := os.Remove(filepath.Join(f.root, key)) + if err != nil { + if os.IsNotExist(err) { + return store.ErrNotFound + } + return err + } + return nil } +// get retrieves a file from the file system. +// +// This function must be called WITH the lock held. func (f *fileSystem) get(key string) ([]byte, error) { b, err := ioutil.ReadFile(filepath.Join(f.root, key)) if err != nil { @@ -49,167 +60,154 @@ func (f *fileSystem) get(key string) ([]byte, error) { return b, nil } -func (f *fileSystem) Get(key string) ([]byte, error) { - log.Tracef("Get: %v", key) - - f.RLock() - defer f.RUnlock() - - return f.get(key) -} - -func (f *fileSystem) del(key string) error { - err := os.Remove(filepath.Join(f.root, key)) - if err != nil { - // Always return not found - return store.ErrNotFound - } - return nil -} - -func (f *fileSystem) Del(key string) error { - log.Tracef("Del: %v", key) +// Put saves the provided blobs to the file system. The keys for the blobs +// are generated in this function and returned. +// +// This function satisfies the Blob interface. +func (f *fileSystem) Put(blobs [][]byte) ([]string, error) { + log.Tracef("Put: %v", len(blobs)) f.Lock() defer f.Unlock() - return f.del(key) -} - -func (f *fileSystem) Enum(cb func(key string, value []byte) error) error { - log.Tracef("Enum") - - f.RLock() - defer f.RUnlock() - - files, err := ioutil.ReadDir(f.root) - if err != nil { - return err - } - - for _, file := range files { - if file.Name() == ".." { - continue - } - blob, err := f.Get(file.Name()) - if err != nil { - return err - } - err = cb(file.Name(), blob) + // Save blobs to file system + keys := make([]string, 0, len(blobs)) + for _, v := range blobs { + key := uuid.New().String() + err := f.put(key, v) if err != nil { - return err + // Unwind blobs that have already been saved + for _, v := range keys { + err2 := f.del(v) + if err2 != nil { + // We're in trouble! + log.Criticalf("Failed to unwind put blob %v: %v", v, err2) + continue + } + } + return nil, err } + keys = append(keys, key) } - return nil -} - -func (f *fileSystem) batch(ops store.Ops) error { - for _, fn := range ops.Del { - log.Tracef("del: %v", fn) - err := f.del(fn) - if err != nil { - return fmt.Errorf("del %v: %v", fn, err) - } - } - for fn, b := range ops.Put { - log.Tracef("put: %v", fn) - err := f.put(fn, b) - if err != nil { - return fmt.Errorf("put %v: %v", fn, err) - } - } - return nil + return keys, nil } -func (f *fileSystem) Batch(ops store.Ops) error { - log.Tracef("Batch") +// Del deletes the files from the file system that correspond to the provided +// keys. +// +// This function satisfies the Blob interface. +func (f *fileSystem) Del(keys []string) error { + log.Tracef("Del: %v", keys) f.Lock() defer f.Unlock() // Temporarily store del files in case we need to unwind - dels := make(map[string][]byte, len(ops.Del)) - for _, fn := range ops.Del { - b, err := f.get(fn) + dels := make(map[string][]byte, len(keys)) + for _, v := range keys { + b, err := f.get(v) if err != nil { - return fmt.Errorf("get %v: %v", fn, err) + return fmt.Errorf("get %v: %v", v, err) } - dels[fn] = b + dels[v] = b } - // Temporarily store existing put files in case we need to unwind. - // An existing put file may or may not exist. - puts := make(map[string][]byte, len(ops.Put)) - for fn := range ops.Put { - b, err := f.get(fn) + // Delete files + deleted := make([]string, 0, len(keys)) + for _, v := range keys { + err := f.del(v) if err != nil { if err == store.ErrNotFound { - // File doesn't exist. This is ok. + // File does not exist. This is ok. continue } - return fmt.Errorf("get %v: %v", fn, err) - } - puts[fn] = b - } - err := f.batch(ops) - if err != nil { - // Unwind puts - for fn := range ops.Put { - err2 := f.del(fn) - if err2 != nil { - // This is ok. It just means the file was never saved before - // the batch function exited with an error. - log.Debugf("batch unwind: del %v: %v", fn, err2) - continue + // File does exist but del failed. Unwind deleted files. + for _, key := range deleted { + b := dels[key] + err2 := f.put(key, b) + if err2 != nil { + // We're in trouble! + log.Criticalf("Failed to unwind del blob %v: %v %x", key, err, b) + continue + } } + return fmt.Errorf("del %v: %v", v, err) } - // Replace existing puts - var unwindFailed bool - for fn, b := range puts { - err2 := f.put(fn, b) - if err2 != nil { - // We're in trouble! - log.Criticalf("batch unwind: unable to put original file back %v: %v", - fn, err2) - unwindFailed = true - continue - } - } + deleted = append(deleted, v) + } - // Unwind deletes - for fn, b := range dels { - _, err2 := f.get(fn) - if err2 == nil { - // File was never deleted. Nothing to do. - continue - } - // File was deleted. Put it back. - err2 = f.put(fn, b) - if err2 != nil { - // We're in trouble! - log.Criticalf("batch unwind: unable to put deleted file back %v: %v", - fn, err2) - unwindFailed = true + return nil +} + +// Get returns blobs from the file system for the provided keys. An entry will +// not exist in the returned map if for any blobs that are not found. It is the +// responsibility of the caller to ensure a blob was returned for all provided +// keys. +// +// This function satisfies the Blob interface. +func (f *fileSystem) Get(keys []string) (map[string][]byte, error) { + log.Tracef("Get: %v", keys) + + f.RLock() + defer f.RUnlock() + + blobs := make(map[string][]byte, len(keys)) + for _, v := range keys { + b, err := f.get(v) + if err != nil { + if err == store.ErrNotFound { + // File does not exist. This is ok. continue } + return nil, fmt.Errorf("get %v: %v", v, err) } + blobs[v] = b + } - if unwindFailed { - // Print orignal error that caused the unwind then panic - // because the unwind failed. - log.Errorf("batch: %v", err) - panic("batch unwind failed") - } + return blobs, nil +} +// Enum enumerates over all blobs in the store, invoking the provided function +// for each blob. +// +// This function satisfies the Blob interface. +func (f *fileSystem) Enum(cb func(key string, blob []byte) error) error { + log.Tracef("Enum") + + f.RLock() + defer f.RUnlock() + + files, err := ioutil.ReadDir(f.root) + if err != nil { return err } + for _, file := range files { + if file.Name() == ".." { + continue + } + blob, err := f.get(file.Name()) + if err != nil { + return err + } + err = cb(file.Name(), blob) + if err != nil { + return err + } + } + return nil } +// Closes closes the blob store connection. +// +// This function satisfies the Blob interface. +func (f *fileSystem) Close() {} + +// New returns a new fileSystem. func New(root string) *fileSystem { return &fileSystem{ root: root, diff --git a/politeiad/backend/tlogbe/store/store.go b/politeiad/backend/tlogbe/store/store.go index b3a3c10ec..9b607d2fb 100644 --- a/politeiad/backend/tlogbe/store/store.go +++ b/politeiad/backend/tlogbe/store/store.go @@ -16,9 +16,8 @@ import ( ) const ( - // Data descriptor types. These may be freely edited since they are - // solely hints to the application. - DataTypeStructure = "struct" // Descriptor contains a structure + // Data descriptor types + DataTypeStructure = "struct" ) var ( @@ -26,20 +25,6 @@ var ( ErrNotFound = errors.New("not found") ) -type Ops struct { - Put map[string][]byte - Del []string -} - -// TODO get rid of this Blob implementation -type Blob interface { - Get(string) ([]byte, error) - Put(string, []byte) error - Del(key string) error - Enum(func(key string, blob []byte) error) error - Batch(Ops) error -} - // DataDescriptor provides hints about a data blob. In practise we JSON encode // this struture and stuff it into BlobEntry.DataHint. type DataDescriptor struct { @@ -94,13 +79,7 @@ func BlobEntryNew(dataHint, data []byte) BlobEntry { } // Blob represents a blob key-value store. -type Blob_ interface { - // Get returns blobs from the store for the provided keys. An entry - // will not exist in the returned map if for any blobs that are not - // found. It is the responsibility of the caller to ensure a blob - // was returned for all provided keys. - Get(keys []string) (map[string][]byte, error) - +type Blob interface { // Put saves the provided blobs to the store. The keys for the // blobs are returned using the same odering that the blobs were // provided in. This operation is performed atomically. @@ -110,6 +89,12 @@ type Blob_ interface { // is performed atomically. Del(keys []string) error + // Get returns blobs from the store for the provided keys. An entry + // will not exist in the returned map if for any blobs that are not + // found. It is the responsibility of the caller to ensure a blob + // was returned for all provided keys. + Get(keys []string) (map[string][]byte, error) + // Enum enumerates over all blobs in the store, invoking the // provided function for each blob. Enum(func(key string, blob []byte) error) error diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index 1e17d492d..bf265cc14 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -68,18 +68,20 @@ var ( // errTreeIsFrozen is emitted when a frozen tree is attempted to be // altered. errTreeIsFrozen = errors.New("tree is frozen") + + // errAnchorNotFound is emitted when a anchor is not found in a + // tree. + errAnchorNotFound = errors.New("anchor not found") ) // We do not unwind. type tlog struct { sync.Mutex - // TODO shutdown bool id string - dataDir string + dcrtimeHost string encryptionKey *encryptionKey trillian *trillianClient - store store.Blob_ - dcrtimeHost string + store store.Blob cron *cron.Cron // droppingAnchor indicates whether tlog is in the process of @@ -140,10 +142,15 @@ func blobIsEncrypted(b []byte) bool { } // treeIsFrozen deterimes if the tree is frozen given the full list of the -// tree's leaves. A tree is considered frozen if the last leaf on the tree -// corresponds to a freeze record. +// tree's leaves. A tree is considered frozen if the tree contains a freeze +// record leaf. func treeIsFrozen(leaves []*trillian.LogLeaf) bool { - return leafIsFreezeRecord(leaves[len(leaves)-1]) + for i := len(leaves) - 1; i >= 0; i-- { + if leafIsFreezeRecord(leaves[i]) { + return true + } + } + return false } func leafIsRecordIndex(l *trillian.LogLeaf) bool { @@ -158,7 +165,7 @@ func leafIsFreezeRecord(l *trillian.LogLeaf) bool { return bytes.HasPrefix(l.ExtraData, []byte(keyPrefixFreezeRecord)) } -func leafIsAnchorRecord(l *trillian.LogLeaf) bool { +func leafIsAnchor(l *trillian.LogLeaf) bool { return bytes.HasPrefix(l.ExtraData, []byte(keyPrefixAnchorRecord)) } @@ -348,6 +355,44 @@ func convertRecordIndexFromBlobEntry(be store.BlobEntry) (*recordIndex, error) { return &ri, nil } +func convertAnchorFromBlobEntry(be store.BlobEntry) (*anchor, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorAnchor { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorAnchor) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + hash, err := hex.DecodeString(be.Hash) + if err != nil { + return nil, fmt.Errorf("decode hash: %v", err) + } + if !bytes.Equal(util.Digest(b), hash) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), hash) + } + var a anchor + err = json.Unmarshal(b, &a) + if err != nil { + return nil, fmt.Errorf("unmarshal freezeRecord: %v", err) + } + + return &a, nil +} + func (t *tlog) treeNew() (int64, error) { tree, _, err := t.trillian.treeNew() if err != nil { @@ -368,19 +413,13 @@ type freezeRecord struct { TreeID int64 `json:"treeid,omitempty"` } -// treeFreeze freezes the trillian tree for the provided token. Once a tree -// has been frozen it is no longer able to be appended to. The last leaf in a -// frozen tree will correspond to a freeze record in the key-value store. A -// tree is frozen when the status of the corresponding record is updated to a -// status that locks the record, such as when a record is censored. -// -// It's possible for this function to fail in between the append leaves call -// and the call that updates the tree status to frozen. If this happens the -// freeze record will still be the last leaf on the tree but the tree state -// will not be frozen. The tree is still considered frozen and no new leaves -// should be appended to it. The tree state will be updated in the next fsck. -// Functions that append new leaves onto the tree MUST ensure that the last -// leaf does not contain a freeze record. +// treeFreeze updates the status of a record and freezes the trillian tree as a +// result of a record status change. Once a freeze record has been appended +// onto the tree the tlog backend considers the tree to be frozen. The only +// thing that can be appended onto a frozen tree is one additional anchor +// record. Once a frozen tree has been anchored, the tlog fsck function will +// update the status of the tree to frozen in trillian, at which point trillian +// will not allow any additional leaves to be appended onto the tree. func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, fr freezeRecord) error { // Get tree leaves leavesAll, err := t.trillian.leavesAll(treeID) @@ -504,12 +543,6 @@ func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []ba return fmt.Errorf("append leaves failed: %v", failed) } - // Update tree status to frozen - _, err = t.trillian.treeFreeze(treeID) - if err != nil { - return fmt.Errorf("treeFreeze: %v", err) - } - return nil } @@ -534,6 +567,7 @@ func (t *tlog) lastLeaf(treeID int64) (*trillian.LogLeaf, error) { return leaves[0], nil } +// TODO the last leaf will be a anchor // freeze record returns the freeze record of the provided tree if one exists. // If one does not exists a errFreezeRecordNotFound error is returned. func (t *tlog) freezeRecord(treeID int64) (*freezeRecord, error) { @@ -1397,10 +1431,19 @@ func (t *tlog) recordLatest(treeID int64) (*backend.Record, error) { return t.recordVersion(treeID, 0) } +// TODO implement recordProof func (t *tlog) recordProof(treeID int64, version uint32) {} +// TODO run fsck episodically func (t *tlog) fsck() { - // Failed freeze + /* + // Freeze any trees with a freeze record + _, err = t.trillian.treeFreeze(treeID) + if err != nil { + return fmt.Errorf("treeFreeze: %v", err) + } + */ + // Failed censor } @@ -1415,6 +1458,25 @@ func (t *tlog) close() { } } -func tlogNew() (*tlog, error) { - return &tlog{}, nil +func newTlog(id, trillianHost, trillianKeyFile, dcrtimeHost string, key *encryptionKey, store store.Blob) (*tlog, error) { + // Setup trillian client + tclient, err := newTrillianClient(homeDir, trillianHost, trillianKeyFile) + if err != nil { + return nil, err + } + + // Launch cron + log.Infof("Launch cron anchor job: %v", id) + c := cron.New() + err = cron.AddFunc(anchorSchedule, func() { + // t.anchorTrees() + }) + if err != nil { + return nil, err + } + cron.Start() + + return &tlog{ + id: id, + }, nil } diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 1e2a89151..913507598 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -7,6 +7,7 @@ package tlogbe import ( "bytes" "crypto/sha256" + "encoding/base64" "encoding/hex" "fmt" "io/ioutil" @@ -19,13 +20,15 @@ import ( "github.com/decred/dcrtime/merkle" pd "github.com/decred/politeia/politeiad/api/v1" + v1 "github.com/decred/politeia/politeiad/api/v1" + "github.com/decred/politeia/politeiad/api/v1/mime" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" "github.com/decred/politeia/util" "github.com/marcopeereboom/sbox" + "github.com/subosito/gozaru" ) -// TODO populate token prefixes cache on startup // TODO testnet vs mainnet trillian databases // TODO fsck // TODO allow token prefix lookups @@ -65,6 +68,14 @@ var ( } ) +// plugin represents a tlogbe plugin. +type plugin struct { + id string + version string + settings []backend.PluginSetting + ctx Plugin +} + // Tlogbe implements the Backend interface. type Tlogbe struct { sync.RWMutex @@ -73,16 +84,15 @@ type Tlogbe struct { dataDir string unvetted *tlog vetted *tlog - plugins []backend.Plugin + plugins map[string]plugin // [pluginID]plugin // prefixes contains the token prefix to full token mapping for all // records. The prefix is the first n characters of the hex encoded // record token, where n is defined by the TokenPrefixLength from // the politeiad API. Record lookups by token prefix are allowed. - // This cache is loaded on tlogbe startup and is used to prevent - // prefix collisions when creating new tokens and to facilitate - // lookups by token prefix. - prefixes map[string][]byte // [tokenPrefix]fullToken + // This cache is used to prevent prefix collisions when creating + // new tokens and to facilitate lookups by token prefix. + prefixes map[string][]byte // [tokenPrefix]token // vettedTreeIDs contains the token to tree ID mapping for vetted // records. The token corresponds to the unvetted tree ID so @@ -90,6 +100,11 @@ type Tlogbe struct { // required pulling the freeze record from the unvetted tree to // get the vetted tree ID. This cache memoizes these results. vettedTreeIDs map[string]int64 // [token]treeID + + // inventory contains the full record inventory grouped by record + // status. Each list of tokens is sorted by the timestamp of the + // status change from newest to oldest. + inventory map[backend.MDStatusT][]string } func tokenPrefix(token []byte) string { @@ -104,12 +119,12 @@ func (t *Tlogbe) prefixExists(fullToken []byte) bool { return ok } -func (t *Tlogbe) prefixSet(fullToken []byte) { +func (t *Tlogbe) prefixAdd(fullToken []byte) { t.Lock() defer t.Unlock() prefix := tokenPrefix(fullToken) - t.prefixes[tokenPrefix(fullToken)] = fullToken + t.prefixes[prefix] = fullToken log.Debugf("Token prefix cached: %v", prefix) } @@ -122,7 +137,7 @@ func (t *Tlogbe) vettedTreeIDGet(token string) (int64, bool) { return treeID, ok } -func (t *Tlogbe) vettedTreeIDSet(token string, treeID int64) { +func (t *Tlogbe) vettedTreeIDAdd(token string, treeID int64) { t.Lock() defer t.Unlock() @@ -131,14 +146,223 @@ func (t *Tlogbe) vettedTreeIDSet(token string, treeID int64) { log.Debugf("Vetted tree ID cached: %v %v", token, treeID) } +func (t *Tlogbe) inventoryGet() map[backend.MDStatusT][]string { + t.RLock() + defer t.RUnlock() + + // Return a copy of the inventory + inv := make(map[backend.MDStatusT][]string, len(t.inventory)) + for status, tokens := range t.inventory { + tokensCopy := make([]string, len(tokens)) + for k, v := range tokens { + tokensCopy[k] = v + } + inv[status] = tokensCopy + } + + return inv +} + +func (t *Tlogbe) inventoryAdd(token string, s backend.MDStatusT) { + t.Lock() + defer t.Unlock() + + t.inventory[s] = append([]string{token}, t.inventory[s]...) + + log.Debugf("Inventory cache added: %v %v", token, backend.MDStatus[s]) +} + +func (t *Tlogbe) inventoryUpdate(token string, currStatus, newStatus backend.MDStatusT) { + t.Lock() + defer t.Unlock() + + // Find the index of the token in its current status list + var idx int + for k, v := range t.inventory[currStatus] { + if v == token { + // Token found + idx = k + } + } + if idx == 0 { + // Token was never found. This should not happen. + e := fmt.Sprintf("inventoryUpdate: token not found: %v %v %v", + token, currStatus, newStatus) + panic(e) + } + + // Remove the token from its current status list + tokens := t.inventory[currStatus] + t.inventory[currStatus] = append(tokens[:idx], tokens[idx+1:]...) + + // Prepend token to new status + t.inventory[newStatus] = append([]string{token}, t.inventory[newStatus]...) + + log.Debugf("Inventory cache updated: %v %v to %v", + token, backend.MDStatus[currStatus], backend.MDStatus[newStatus]) +} + +// verifyContent verifies that all provided MetadataStream and File are sane. +func verifyContent(metadata []backend.MetadataStream, files []backend.File, filesDel []string) error { + // Make sure all metadata is within maxima. + for _, v := range metadata { + if v.ID > v1.MetadataStreamsMax-1 { + return backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidMDID, + ErrorContext: []string{ + strconv.FormatUint(v.ID, 10), + }, + } + } + } + for i := range metadata { + for j := range metadata { + // Skip self and non duplicates. + if i == j || metadata[i].ID != metadata[j].ID { + continue + } + return backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusDuplicateMDID, + ErrorContext: []string{ + strconv.FormatUint(metadata[i].ID, 10), + }, + } + } + } + + // Prevent paths + for i := range files { + if filepath.Base(files[i].Name) != files[i].Name { + return backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidFilename, + ErrorContext: []string{ + files[i].Name, + }, + } + } + } + for _, v := range filesDel { + if filepath.Base(v) != v { + return backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidFilename, + ErrorContext: []string{ + v, + }, + } + } + } + + // Now check files + if len(files) == 0 { + return backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusEmpty, + } + } + + // Prevent bad filenames and duplicate filenames + for i := range files { + for j := range files { + if i == j { + continue + } + if files[i].Name == files[j].Name { + return backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusDuplicateFilename, + ErrorContext: []string{ + files[i].Name, + }, + } + } + } + // Check against filesDel + for _, v := range filesDel { + if files[i].Name == v { + return backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusDuplicateFilename, + ErrorContext: []string{ + files[i].Name, + }, + } + } + } + } + + for i := range files { + if gozaru.Sanitize(files[i].Name) != files[i].Name { + return backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidFilename, + ErrorContext: []string{ + files[i].Name, + }, + } + } + + // Validate digest + d, ok := util.ConvertDigest(files[i].Digest) + if !ok { + return backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidFileDigest, + ErrorContext: []string{ + files[i].Name, + }, + } + } + + // Decode base64 payload + var err error + payload, err := base64.StdEncoding.DecodeString(files[i].Payload) + if err != nil { + return backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidBase64, + ErrorContext: []string{ + files[i].Name, + }, + } + } + + // Calculate payload digest + dp := util.Digest(payload) + if !bytes.Equal(d[:], dp) { + return backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidFileDigest, + ErrorContext: []string{ + files[i].Name, + }, + } + } + + // Verify MIME + detectedMIMEType := mime.DetectMimeType(payload) + if detectedMIMEType != files[i].MIME { + return backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidMIMEType, + ErrorContext: []string{ + files[i].Name, + detectedMIMEType, + }, + } + } + + if !mime.MimeValid(files[i].MIME) { + return backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusUnsupportedMIMEType, + ErrorContext: []string{ + files[i].Name, + files[i].MIME, + }, + } + } + } + + return nil +} + // statusChangeIsAllowed returns whether the provided status change is allowed -// by tlogbe. An invalid 'from' status will panic since the 'from' status -// represents the existing status of a record and should never be invalid. +// by tlogbe. func statusChangeIsAllowed(from, to backend.MDStatusT) bool { allowed, ok := statusChanges[from] if !ok { - e := fmt.Sprintf("status invalid: %v", from) - panic(e) + return false } _, ok = allowed[to] return ok @@ -243,7 +467,7 @@ func (t *Tlogbe) New(metadata []backend.MetadataStream, files []backend.File) (* log.Tracef("New") // Validate record contents - err := backend.VerifyContent(metadata, files, []string{}) + err := verifyContent(metadata, files, []string{}) if err != nil { return nil, err } @@ -260,8 +484,7 @@ func (t *Tlogbe) New(metadata []backend.MetadataStream, files []backend.File) (* // Check for token prefix collisions if !t.prefixExists(token) { - // Not a collision. Update token prefixes cache. - t.prefixSet(token) + // Not a collision. Use this token. break } @@ -281,6 +504,12 @@ func (t *Tlogbe) New(metadata []backend.MetadataStream, files []backend.File) (* return nil, fmt.Errorf("recordSave %x: %v", token, err) } + // Update the prefix cache + t.prefixAdd(token) + + // Update the inventory cache + t.inventoryAdd(hex.EncodeToString(token), backend.MDStatusUnvetted) + log.Infof("New record %x", token) return rm, nil @@ -293,7 +522,7 @@ func (t *Tlogbe) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite []back // Validate record contents. Send in a single metadata array to // verify there are no dups. allMD := append(mdAppend, mdOverwrite...) - err := backend.VerifyContent(allMD, filesAdd, filesDel) + err := verifyContent(allMD, filesAdd, filesDel) if err != nil { e, ok := err.(backend.ContentVerificationError) if !ok { @@ -345,7 +574,13 @@ func (t *Tlogbe) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite []back return nil, fmt.Errorf("recordSave: %v", err) } - // TODO Call plugin hooks + // Update inventory cache. The inventory will only need to be + // updated if there was a status transition. + if r.RecordMetadata.Status != recordMD.Status { + // Status was changed + t.inventoryUpdate(recordMD.Token, r.RecordMetadata.Status, + recordMD.Status) + } // Return updated record r, err = t.unvetted.recordLatest(treeID) @@ -363,7 +598,7 @@ func (t *Tlogbe) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []backen // Validate record contents. Send in a single metadata array to // verify there are no dups. allMD := append(mdAppend, mdOverwrite...) - err := backend.VerifyContent(allMD, filesAdd, filesDel) + err := verifyContent(allMD, filesAdd, filesDel) if err != nil { e, ok := err.(backend.ContentVerificationError) if !ok { @@ -415,8 +650,6 @@ func (t *Tlogbe) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []backen return nil, fmt.Errorf("recordSave: %v", err) } - // TODO Call plugin hooks - // Return updated record r, err = t.vetted.recordLatest(treeID) if err != nil { @@ -431,7 +664,7 @@ func (t *Tlogbe) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite []ba // Validate record contents. Send in a single metadata array to // verify there are no dups. allMD := append(mdAppend, mdOverwrite...) - err := backend.VerifyContent(allMD, []backend.File{}, []string{}) + err := verifyContent(allMD, []backend.File{}, []string{}) if err != nil { e, ok := err.(backend.ContentVerificationError) if !ok { @@ -472,7 +705,12 @@ func (t *Tlogbe) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite []ba metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) // Update metadata - return t.unvetted.recordMetadataUpdate(treeID, r.RecordMetadata, metadata) + err = t.unvetted.recordMetadataUpdate(treeID, r.RecordMetadata, metadata) + if err != nil { + return err + } + + return nil } // This function satisfies the Backend interface. @@ -482,7 +720,7 @@ func (t *Tlogbe) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite []back // Validate record contents. Send in a single metadata array to // verify there are no dups. allMD := append(mdAppend, mdOverwrite...) - err := backend.VerifyContent(allMD, []backend.File{}, []string{}) + err := verifyContent(allMD, []backend.File{}, []string{}) if err != nil { e, ok := err.(backend.ContentVerificationError) if !ok { @@ -523,7 +761,12 @@ func (t *Tlogbe) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite []back metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) // Update metadata - return t.vetted.recordMetadataUpdate(treeID, r.RecordMetadata, metadata) + err = t.vetted.recordMetadataUpdate(treeID, r.RecordMetadata, metadata) + if err != nil { + return err + } + + return nil } // UnvettedExists returns whether the provided token corresponds to an unvetted @@ -608,7 +851,7 @@ func (t *Tlogbe) VettedExists(token []byte) bool { } // Update the vetted cache - t.vettedTreeIDSet(hex.EncodeToString(token), fr.TreeID) + t.vettedTreeIDAdd(hex.EncodeToString(token), fr.TreeID) return true } @@ -657,7 +900,7 @@ func (t *Tlogbe) unvettedPublish(token []byte, rm backend.RecordMetadata, metada // Check for token prefix collisions if !t.prefixExists(vettedToken) { // Not a collision. Update prefixes cache. - t.prefixSet(vettedToken) + t.prefixAdd(vettedToken) break } @@ -686,7 +929,7 @@ func (t *Tlogbe) unvettedPublish(token []byte, rm backend.RecordMetadata, metada log.Debugf("Unvetted record %x frozen", token) // Update the vetted cache - t.vettedTreeIDSet(hex.EncodeToString(token), vettedTreeID) + t.vettedTreeIDAdd(hex.EncodeToString(token), vettedTreeID) return nil } @@ -747,17 +990,18 @@ func (t *Tlogbe) SetUnvettedStatus(token []byte, status backend.MDStatusT, mdApp return nil, fmt.Errorf("recordLatest: %v", err) } rm := r.RecordMetadata + oldStatus := rm.Status // Validate status change - if !statusChangeIsAllowed(rm.Status, status) { + if !statusChangeIsAllowed(oldStatus, status) { return nil, backend.StateTransitionError{ - From: rm.Status, + From: oldStatus, To: status, } } log.Debugf("Status change %x from %v (%v) to %v (%v)", - token, backend.MDStatus[rm.Status], rm.Status, + token, backend.MDStatus[oldStatus], oldStatus, backend.MDStatus[status], status) // Apply status change @@ -790,6 +1034,9 @@ func (t *Tlogbe) SetUnvettedStatus(token []byte, status backend.MDStatusT, mdApp backend.MDStatus[status], status) } + // Update inventory cache + t.inventoryUpdate(rm.Token, oldStatus, status) + // Return the updated record r, err = t.unvetted.recordLatest(treeID) if err != nil { @@ -845,17 +1092,18 @@ func (t *Tlogbe) SetVettedStatus(token []byte, status backend.MDStatusT, mdAppen return nil, fmt.Errorf("recordLatest: %v", err) } rm := r.RecordMetadata + oldStatus := rm.Status // Validate status change if !statusChangeIsAllowed(rm.Status, status) { return nil, backend.StateTransitionError{ - From: rm.Status, + From: oldStatus, To: status, } } log.Debugf("Status change %x from %v (%v) to %v (%v)", - token, backend.MDStatus[rm.Status], rm.Status, + token, backend.MDStatus[oldStatus], oldStatus, backend.MDStatus[status], status) // Apply status change @@ -883,6 +1131,9 @@ func (t *Tlogbe) SetVettedStatus(token []byte, status backend.MDStatusT, mdAppen backend.MDStatus[status], status) } + // Update inventory cache + t.inventoryUpdate(rm.Token, oldStatus, status) + // Return the updated record r, err = t.vetted.recordLatest(treeID) if err != nil { @@ -892,33 +1143,75 @@ func (t *Tlogbe) SetVettedStatus(token []byte, status backend.MDStatusT, mdAppen return r, nil } +// Inventory is not currenctly implemented in tlogbe. +// // This function satisfies the Backend interface. func (t *Tlogbe) Inventory(vettedCount uint, unvettedCount uint, includeFiles, allVersions bool) ([]backend.Record, []backend.Record, error) { log.Tracef("Inventory: %v %v", includeFiles, allVersions) - // TODO implement inventory + return nil, nil, fmt.Errorf("not implemented") +} - // return vetted, unvetted, nil - return nil, nil, nil +// InventoryByStatus returns the record tokens of all records in the inventory +// catagorized by MDStatusT. +// +// This function satisfies the Backend interface. +func (t *Tlogbe) InventoryByStatus() (*backend.InventoryByStatus, error) { + log.Tracef("InventoryByStatus") + + inv := t.inventoryGet() + return &backend.InventoryByStatus{ + Unvetted: inv[backend.MDStatusUnvetted], + IterationUnvetted: inv[backend.MDStatusIterationUnvetted], + Vetted: inv[backend.MDStatusVetted], + Censored: inv[backend.MDStatusCensored], + Archived: inv[backend.MDStatusArchived], + }, nil } +// GetPlugins returns the backend plugins that have been registered and their +// settings. +// +// This function satisfies the Backend interface. func (t *Tlogbe) GetPlugins() ([]backend.Plugin, error) { log.Tracef("GetPlugins") - // TODO implement plugins + plugins := make([]backend.Plugin, 0, len(t.plugins)) + for _, v := range t.plugins { + plugins = append(plugins, backend.Plugin{ + ID: v.id, + Version: v.version, + Settings: v.settings, + }) + } - return t.plugins, nil + return plugins, nil } -// Add commandID to Plugin +// Plugin is a pass-through function for plugin commands. +// +// This function satisfies the Backend interface. func (t *Tlogbe) Plugin(pluginID, command, payload string) (string, string, error) { log.Tracef("Plugin: %v", command) - // TODO implement plugins + // Get plugin + plugin, ok := t.plugins[pluginID] + if !ok { + return "", "", backend.ErrPluginInvalid + } + + // Execute plugin command + reply, err := plugin.ctx.Cmd(command, payload) + if err != nil { + return "", "", err + } - return "", "", nil + return command, reply, nil } +// Close shuts the backend down and performs cleanup. +// +// This function satisfies the Backend interface. func (t *Tlogbe) Close() { log.Tracef("Close") @@ -933,7 +1226,60 @@ func (t *Tlogbe) Close() { t.vetted.close() } -func New(homeDir, dataDir, trillianHost, trillianKeyFile, dcrtimeHost, encryptionKeyFile string) (*Tlogbe, error) { +func (t *Tlogbe) setup() error { + // Get all trees + trees, err := t.unvetted.trillian.treesAll() + if err != nil { + return fmt.Errorf("unvetted treesAll: %v", err) + } + + // Build all memory caches + for _, v := range trees { + // Add tree to prefixes cache + token := tokenFromTreeID(v.TreeId) + t.prefixAdd(token) + + // Check if the tree needs to be added to the vettedTreeIDs cache + // by checking the freeze record of the unvetted tree. + var vettedTreeID int64 + fr, err := t.unvetted.freezeRecord(v.TreeId) + switch err { + case errFreezeRecordNotFound: + // No freeze record means this is not a vetted record. + // Nothing to do. Continue. + case nil: + // A freeze record exists. If a pointer to a vetted tree has + // been set, add it to the vettedTreeIDs cache. + if fr.TreeID != 0 { + vettedTreeID = fr.TreeID + t.vettedTreeIDAdd(hex.EncodeToString(token), vettedTreeID) + } + default: + // All other errors + return fmt.Errorf("freezeRecord %v: %v", v.TreeId, err) + } + + // Add record to the inventory cache + var r *backend.Record + if vettedTreeID != 0 { + r, err = t.GetVetted(token, "") + if err != nil { + return fmt.Errorf("GetVetted %x: %v", token, err) + } + } else { + r, err = t.GetUnvetted(token, "") + if err != nil { + return fmt.Errorf("GetUnvetted %x: %v", token, err) + } + } + t.inventoryAdd(hex.EncodeToString(token), r.RecordMetadata.Status) + } + + return nil +} + +// New returns a new Tlogbe. +func New(homeDir, dataDir, dcrtimeHost, encryptionKeyFile, unvettedTrillianHost, unvettedTrillianKeyFile, vettedTrillianHost, vettedTrillianKeyFile string) (*Tlogbe, error) { // Setup encryption key file if encryptionKeyFile == "" { // No file path was given. Use the default path. @@ -954,20 +1300,6 @@ func New(homeDir, dataDir, trillianHost, trillianKeyFile, dcrtimeHost, encryptio log.Infof("Encryption key created: %v", encryptionKeyFile) } - // Setup trillian client - tlog, err := trillianClientNew(homeDir, trillianHost, trillianKeyFile) - if err != nil { - return nil, err - } - - // Setup key-value store - fp := filepath.Join(dataDir, recordsDirname) - err = os.MkdirAll(fp, 0700) - if err != nil { - return nil, err - } - store := filesystem.New(fp) - // Load encryption key f, err := os.Open(encryptionKeyFile) if err != nil { @@ -982,10 +1314,18 @@ func New(homeDir, dataDir, trillianHost, trillianKeyFile, dcrtimeHost, encryptio return nil, err } f.Close() - encryptionKey := encryptionKeyNew(&key) + encryptionKey := newEncryptionKey(&key) log.Infof("Encryption key loaded") + // Setup key-value store + fp := filepath.Join(dataDir, recordsDirname) + err = os.MkdirAll(fp, 0700) + if err != nil { + return nil, err + } + store := filesystem.New(fp) + // Setup dcrtime host _, err = url.Parse(dcrtimeHost) if err != nil { @@ -993,27 +1333,34 @@ func New(homeDir, dataDir, trillianHost, trillianKeyFile, dcrtimeHost, encryptio } log.Infof("Anchor host: %v", dcrtimeHost) - _ = encryptionKey - _ = dcrtimeHost - _ = store - _ = tlog - t := Tlogbe{ - homeDir: homeDir, - dataDir: dataDir, - // cron: cron.New(), + // Setup tlog instances + unvetted, err := newTlog("unvetted", unvettedTrillianHost, + unvettedTrillianKeyFile, dcrtimeHost, encryptionKey, store) + if err != nil { + return nil, fmt.Errorf("newTlog unvetted: %v", err) + } + vetted, err := newTlog("vetted", vettedTrillianHost, + vettedTrillianKeyFile, dcrtimeHost, encryptionKey, store) + if err != nil { + return nil, fmt.Errorf("newTlog vetted: %v", err) } - /* - // Launch cron - log.Infof("Launch cron anchor job") - err = t.cron.AddFunc(anchorSchedule, func() { - // t.anchorTrees() - }) - if err != nil { - return nil, err - } - t.cron.Start() - */ + t := Tlogbe{ + homeDir: homeDir, + dataDir: dataDir, + unvetted: unvetted, + vetted: vetted, + plugins: make(map[string]plugin), + prefixes: make(map[string][]byte), + vettedTreeIDs: make(map[string]int64), + inventory: map[backend.MDStatusT][]string{ + backend.MDStatusUnvetted: make([]string, 0), + backend.MDStatusIterationUnvetted: make([]string, 0), + backend.MDStatusVetted: make([]string, 0), + backend.MDStatusCensored: make([]string, 0), + backend.MDStatusArchived: make([]string, 0), + }, + } return &t, nil } diff --git a/politeiad/backend/tlogbe/trillian.go b/politeiad/backend/tlogbe/trillian.go index d4f83b004..7c49d1436 100644 --- a/politeiad/backend/tlogbe/trillian.go +++ b/politeiad/backend/tlogbe/trillian.go @@ -439,7 +439,7 @@ func (t *trillianClient) close() { t.grpc.Close() } -func trillianClientNew(homeDir, host, keyFile string) (*trillianClient, error) { +func newTrillianClient(homeDir, host, keyFile string) (*trillianClient, error) { // Setup trillian key file if keyFile == "" { // No file path was given. Use the default path. From e05e9a1521b6fe5fe7b8a3f9ff125fa048684b1b Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 25 Aug 2020 09:44:24 -0500 Subject: [PATCH 016/449] del cache package and get politeiad to build --- plugins/ticketvote/ticketvote.go | 1 + politeiad/api/v1/api.md | 42 - politeiad/api/v1/v1.go | 13 - politeiad/backend/gitbe/decred.go | 7 - politeiad/backend/gitbe/gitbe.go | 84 +- politeiad/backend/tlogbe/anchor.go | 2 +- .../backend/tlogbe/plugins/ticketvote.go | 11 +- politeiad/backend/tlogbe/tlog.go | 69 +- politeiad/backend/tlogbe/tlogbe.go | 67 +- politeiad/backend/tlogbe/trillian.go | 11 +- politeiad/cache/cache.go | 215 -- politeiad/cache/cachestub/cachestub.go | 97 - politeiad/cache/cockroachdb/cms.go | 813 ------ politeiad/cache/cockroachdb/cockroachdb.go | 1043 -------- politeiad/cache/cockroachdb/convert.go | 524 ---- politeiad/cache/cockroachdb/decred.go | 2310 ----------------- politeiad/cache/cockroachdb/log.go | 25 - politeiad/cache/cockroachdb/models.go | 372 --- politeiad/cache/testcache/decred.go | 327 --- politeiad/cache/testcache/testcache.go | 277 -- politeiad/config.go | 160 +- politeiad/log.go | 10 +- politeiad/politeiad.go | 565 +--- politeiad/testpoliteiad/convert.go | 76 - politeiad/testpoliteiad/decred.go | 183 -- politeiad/testpoliteiad/testpoliteiad.go | 95 +- politeiawww/www.go | 6 - 27 files changed, 214 insertions(+), 7191 deletions(-) delete mode 100644 politeiad/cache/cache.go delete mode 100644 politeiad/cache/cachestub/cachestub.go delete mode 100644 politeiad/cache/cockroachdb/cms.go delete mode 100644 politeiad/cache/cockroachdb/cockroachdb.go delete mode 100644 politeiad/cache/cockroachdb/convert.go delete mode 100644 politeiad/cache/cockroachdb/decred.go delete mode 100644 politeiad/cache/cockroachdb/log.go delete mode 100644 politeiad/cache/cockroachdb/models.go delete mode 100644 politeiad/cache/testcache/decred.go delete mode 100644 politeiad/cache/testcache/testcache.go delete mode 100644 politeiad/testpoliteiad/convert.go delete mode 100644 politeiad/testpoliteiad/decred.go diff --git a/plugins/ticketvote/ticketvote.go b/plugins/ticketvote/ticketvote.go index 5ab289b50..1121ba793 100644 --- a/plugins/ticketvote/ticketvote.go +++ b/plugins/ticketvote/ticketvote.go @@ -63,6 +63,7 @@ const ( VoteTypeRunoff VoteT = 2 // Vote duration requirements in blocks + // TODO these are not used anywhere VoteDurationMinMainnet = 2016 VoteDurationMaxMainnet = 4032 VoteDurationMinTestnet = 0 diff --git a/politeiad/api/v1/api.md b/politeiad/api/v1/api.md index 03c17167a..95e6d82f8 100644 --- a/politeiad/api/v1/api.md +++ b/politeiad/api/v1/api.md @@ -20,7 +20,6 @@ censorship mechanism. - [`Update vetted record`](#update-vetted-record) - [`Update vetted metadata`](#update-vetted-metadata) - [`Inventory`](#inventory) -- [`Update readme`](#update-readme) **Error status codes** @@ -703,47 +702,6 @@ Reply: } ``` -### `Update Readme` - -Update the README.md file located in both the unvetted and vetted repositories. - -This command requires administrator privileges. - -**Route**: `POST /v1/updatereadme` - -**Params**: - -| Parameter | Type | Description | Required | -|-|-|-|-| -| challenge | string | 32 byte hex encoded array. | Yes | -| content | string | String containing desired contents of readme file. | Yes | - -**Results**: - -| | Type | Description | -|-|-|-| -| response | string | hex encoded signature of challenge byte array. | - -**Example** - -Request: - -```json -{ - "challenge":"3d41f60ffd17176e7b456e67a2fb712d3223a719edb45db11c87b124b9d9afc1", - "content":"### `Updated readme page` \n content content content \n" -} -``` - -Reply: - -```json -{ - "response":"3652492f3966ac616fe2e3bafddab3e58cf76dc94c7e19fcbc6f0495edcfbc6955f22fbb49cb4cf73cca355097da1c4255a9c4a61ccbfdcd02bf9f7fff78050e" -} -``` - - ### `Inventory` Retrieve all records. This is a very expensive call. diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index 6675609a4..305d29776 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -35,7 +35,6 @@ const ( SetVettedStatusRoute = "/v1/setvettedstatus/" // Set vetted status PluginCommandRoute = "/v1/plugin/" // Send a command to a plugin PluginInventoryRoute = PluginCommandRoute + "inventory/" // Inventory all plugins - UpdateReadmeRoute = "/v1/updatereadme/" // Update README ChallengeSize = 32 // Size of challenge token in bytes TokenSize = 32 // Size of token @@ -333,18 +332,6 @@ type UpdateVettedMetadataReply struct { Response string `json:"response"` // Challenge response } -// UpdateReadme updated the README.md file in the vetted and unvetted repos. -type UpdateReadme struct { - Challenge string `json:"challenge"` // Random challenge - Content string `json:"content"` // New content of README.md -} - -// UpdateReadmeReply returns a response challenge to an -// UpdateReadme command. -type UpdateReadmeReply struct { - Response string `json:"response"` // Challenge response -} - // Inventory sends an (expensive and therefore authenticated) inventory request // for vetted records (master branch) and branches (censored, unpublished etc) // records. This is a very expensive call and should be only issued at start diff --git a/politeiad/backend/gitbe/decred.go b/politeiad/backend/gitbe/decred.go index 5d7c6ee40..19bad4d81 100644 --- a/politeiad/backend/gitbe/decred.go +++ b/politeiad/backend/gitbe/decred.go @@ -40,7 +40,6 @@ const ( decredPluginJournals = "journals" decredPluginVoteDurationMin = "votedurationmin" decredPluginVoteDurationMax = "votedurationmax" - decredPluginEnableCache = "enablecache" defaultCommentIDFilename = "commentid.txt" defaultCommentFilename = "comments.journal" @@ -178,12 +177,6 @@ func getDecredPlugin(dcrdataHost string) backend.Plugin { }, ) - decredPlugin.Settings = append(decredPlugin.Settings, - backend.PluginSetting{ - Key: decredPluginEnableCache, - Value: "", - }) - // Initialize hooks decredPluginHooks = make(map[string]func(string) error) diff --git a/politeiad/backend/gitbe/gitbe.go b/politeiad/backend/gitbe/gitbe.go index 8b02521e3..ad63fcc82 100644 --- a/politeiad/backend/gitbe/gitbe.go +++ b/politeiad/backend/gitbe/gitbe.go @@ -1805,6 +1805,13 @@ func (g *gitBackEnd) UpdateUnvettedRecord(token []byte, mdAppend []backend.Metad false) } +// UpdateUnvettedMetadata is not implemented. +// +// This function satsifies the Backend interface. +func (g *gitBackEnd) UpdateUnvettedMetadata(token []byte, mdAppend []backend.MetadataStream, mdOverwrite []backend.MetadataStream) error { + return fmt.Errorf("not implemented") +} + // updateVettedMetadata updates metadata in the unvetted repo and pushes it // upstream followed by a rebase. Record is not updated. // This function must be called with the lock held. @@ -2418,8 +2425,12 @@ func (g *gitBackEnd) vettedMetadataStreamExists(token []byte, mdstreamID int) bo // unvetted/token directory. // // GetUnvetted satisfies the backend interface. -func (g *gitBackEnd) GetUnvetted(token []byte) (*backend.Record, error) { +func (g *gitBackEnd) GetUnvetted(token []byte, version string) (*backend.Record, error) { log.Tracef("GetUnvetted %x", token) + + // The version argument is not used because gitbe does not version + // unvetted records. + return g.getRecordLock(token, "", g.unvetted, true) } @@ -2773,6 +2784,13 @@ func (g *gitBackEnd) Inventory(vettedCount, branchCount uint, includeFiles, allV return pr, br, nil } +// InventoryByStatus is not implemented. +// +// This function satsifies the Backend interface. +func (g *gitBackEnd) InventoryByStatus() (*backend.InventoryByStatus, error) { + return nil, fmt.Errorf("not implemented") +} + // GetPlugins returns a list of currently supported plugins and their settings. // // GetPlugins satisfies the backend interface. @@ -2786,7 +2804,7 @@ func (g *gitBackEnd) GetPlugins() ([]backend.Plugin, error) { // execute. // // Plugin satisfies the backend interface. -func (g *gitBackEnd) Plugin(command, payload string) (string, string, error) { +func (g *gitBackEnd) Plugin(pluginID, command, payload string) (string, string, error) { log.Tracef("Plugin: %v", command) switch command { case decredplugin.CmdAuthorizeVote: @@ -2984,7 +3002,7 @@ func (g *gitBackEnd) rebasePR(id string) error { } // New returns a gitBackEnd context. It verifies that git is installed. -func New(anp *chaincfg.Params, root string, dcrtimeHost string, gitPath string, id *identity.FullIdentity, gitTrace bool, dcrdataHost string, mode string) (*gitBackEnd, error) { +func New(anp *chaincfg.Params, root string, dcrtimeHost string, gitPath string, id *identity.FullIdentity, gitTrace bool, dcrdataHost string) (*gitBackEnd, error) { // Default to system git if gitPath == "" { @@ -3013,32 +3031,31 @@ func New(anp *chaincfg.Params, root string, dcrtimeHost string, gitPath string, return nil, err } - switch mode { - case piMode: - // Setup decred plugin settings - var voteDurationMin, voteDurationMax string - switch anp.Name { - case chaincfg.MainNetParams.Name: - voteDurationMin = strconv.Itoa(decredplugin.VoteDurationMinMainnet) - voteDurationMax = strconv.Itoa(decredplugin.VoteDurationMaxMainnet) - case chaincfg.TestNet3Params.Name: - voteDurationMin = strconv.Itoa(decredplugin.VoteDurationMinTestnet) - voteDurationMax = strconv.Itoa(decredplugin.VoteDurationMaxTestnet) - default: - return nil, fmt.Errorf("unknown chaincfg params '%v'", anp.Name) - } - setDecredPluginSetting(decredPluginVoteDurationMin, voteDurationMin) - setDecredPluginSetting(decredPluginVoteDurationMax, voteDurationMax) - case cmsMode: - g.plugins = []backend.Plugin{getDecredPlugin(dcrdataHost), - getCMSPlugin(anp.Name != "mainnet")} - - setCMSPluginSetting(cmsPluginIdentity, string(idJSON)) - setCMSPluginSetting(cmsPluginJournals, g.journals) + // Register all plugins + g.plugins = []backend.Plugin{ + getDecredPlugin(dcrdataHost), + getCMSPlugin(anp.Name != chaincfg.MainNetParams.Name), + } + + // Setup cms plugin + setCMSPluginSetting(cmsPluginIdentity, string(idJSON)) + setCMSPluginSetting(cmsPluginJournals, g.journals) + + // Setup decred plugin + var voteDurationMin, voteDurationMax string + switch anp.Name { + case chaincfg.MainNetParams.Name: + voteDurationMin = strconv.Itoa(decredplugin.VoteDurationMinMainnet) + voteDurationMax = strconv.Itoa(decredplugin.VoteDurationMaxMainnet) + case chaincfg.TestNet3Params.Name: + voteDurationMin = strconv.Itoa(decredplugin.VoteDurationMinTestnet) + voteDurationMax = strconv.Itoa(decredplugin.VoteDurationMaxTestnet) default: - return nil, fmt.Errorf("invalid mode") + return nil, fmt.Errorf("unknown chaincfg params '%v'", anp.Name) } + setDecredPluginSetting(decredPluginVoteDurationMin, voteDurationMin) + setDecredPluginSetting(decredPluginVoteDurationMax, voteDurationMax) setDecredPluginSetting(decredPluginIdentity, string(idJSON)) setDecredPluginSetting(decredPluginJournals, g.journals) setDecredPluginHook(PluginPostHookEdit, g.decredPluginPostEdit) @@ -3059,12 +3076,10 @@ func New(anp *chaincfg.Params, root string, dcrtimeHost string, gitPath string, return nil, err } - if mode == cmsMode { - // this function must be called after g.journal is created - err = g.initCMSPluginJournals() - if err != nil { - return nil, err - } + // this function must be called after g.journal is created + err = g.initCMSPluginJournals() + if err != nil { + return nil, err } err = g.newLocked() @@ -3083,10 +3098,7 @@ func New(anp *chaincfg.Params, root string, dcrtimeHost string, gitPath string, err = g.cron.AddFunc(anchorSchedule, func() { // Flush journals g.decredPluginJournalFlusher() - - if mode == cmsMode { - g.cmsPluginJournalFlusher() - } + g.cmsPluginJournalFlusher() // Anchor commit g.anchorAllReposCronJob() diff --git a/politeiad/backend/tlogbe/anchor.go b/politeiad/backend/tlogbe/anchor.go index 79838d67d..e20a698c8 100644 --- a/politeiad/backend/tlogbe/anchor.go +++ b/politeiad/backend/tlogbe/anchor.go @@ -345,7 +345,7 @@ func (t *tlog) anchor() { default: // Anchor record found. If the anchor height differs from the - // current height than the tree needs to be anchored. + // current height then the tree needs to be anchored. _, lr, err := t.trillian.signedLogRootForTree(v) if err != nil { exitErr = fmt.Errorf("signedLogRoot %v: %v", v.TreeId, err) diff --git a/politeiad/backend/tlogbe/plugins/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote.go index 446557ec6..1715271bf 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote.go @@ -1023,7 +1023,7 @@ func voteBitVerify(options []ticketvote.VoteOption, mask, bit uint64) error { } // TODO test this function -func voteVerifyDetails(vote ticketvote.VoteDetails, voteDurationMin, voteDurationMax uint32) error { +func voteDetailsVerify(vote ticketvote.VoteDetails, voteDurationMin, voteDurationMax uint32) error { // Verify vote type switch vote.Type { case ticketvote.VoteTypeStandard: @@ -1156,7 +1156,7 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { } // Verify vote options and params - err = voteVerifyDetails(s.Vote, p.voteDurationMin, p.voteDurationMax) + err = voteDetailsVerify(s.Vote, p.voteDurationMin, p.voteDurationMax) if err != nil { return "", err } @@ -2014,6 +2014,13 @@ func TicketVotePluginNew(backend *tlogbe.Tlogbe, settings []backend.PluginSettin voteDurationMax uint32 ) + /* + switch activeNetParams.Name { + case chaincfg.MainNetParams.Name: + case chaincfg.TestNet3Params.Name: + } + */ + return &ticketVotePlugin{ dataDir: filepath.Join(dataDir, ticketVoteDirname), backend: backend, diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index bf265cc14..2b2432a37 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -12,11 +12,14 @@ import ( "encoding/json" "errors" "fmt" + "os" + "path/filepath" "strconv" "sync" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/store" + "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" "github.com/decred/politeia/util" "github.com/google/trillian" "github.com/robfig/cron" @@ -40,6 +43,7 @@ const ( // data out of the store, which can become an issue in situations // such as searching for a record index that has been buried by // thousands of leaves from plugin data. + // TODO key prefix app-dataID: keyPrefixRecordIndex = "recordindex:" keyPrefixRecordContent = "record:" keyPrefixFreezeRecord = "freeze:" @@ -1458,25 +1462,70 @@ func (t *tlog) close() { } } -func newTlog(id, trillianHost, trillianKeyFile, dcrtimeHost string, key *encryptionKey, store store.Blob) (*tlog, error) { +func newTlog(id, homeDir, dataDir, trillianHost, trillianKeyFile, dcrtimeHost, encryptionKeyFile string) (*tlog, error) { + // Load encryption key if provided. An encryption key is optional. + var ek *encryptionKey + if encryptionKeyFile != "" { + f, err := os.Open(encryptionKeyFile) + if err != nil { + return nil, err + } + var key [32]byte + n, err := f.Read(key[:]) + if n != len(key) { + return nil, fmt.Errorf("invalid encryption key length") + } + if err != nil { + return nil, err + } + f.Close() + ek = newEncryptionKey(&key) + + log.Infof("Encryption key %v: %v", id, encryptionKeyFile) + } + + // Setup key-value store + fp := filepath.Join(dataDir, id) + err := os.MkdirAll(fp, 0700) + if err != nil { + return nil, err + } + store := filesystem.New(fp) + // Setup trillian client - tclient, err := newTrillianClient(homeDir, trillianHost, trillianKeyFile) + if trillianKeyFile == "" { + // No file path was given. Use the default path. + fn := fmt.Sprintf("%v-%v", id, defaultTrillianKeyFilename) + trillianKeyFile = filepath.Join(homeDir, fn) + } + + log.Infof("Trillian key %v: %v", id, trillianKeyFile) + log.Infof("Trillian host %v: %v", id, trillianHost) + + tclient, err := newTrillianClient(trillianHost, trillianKeyFile) if err != nil { return nil, err } + // Setup tlog + t := tlog{ + id: id, + dcrtimeHost: dcrtimeHost, + encryptionKey: ek, + trillian: tclient, + store: store, + cron: cron.New(), + } + // Launch cron - log.Infof("Launch cron anchor job: %v", id) - c := cron.New() - err = cron.AddFunc(anchorSchedule, func() { - // t.anchorTrees() + log.Infof("Launch %v cron anchor job", id) + err = t.cron.AddFunc(anchorSchedule, func() { + t.anchor() }) if err != nil { return nil, err } - cron.Start() + t.cron.Start() - return &tlog{ - id: id, - }, nil + return &t, nil } diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 913507598..3935c3c71 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -12,7 +12,6 @@ import ( "fmt" "io/ioutil" "net/url" - "os" "path/filepath" "strconv" "sync" @@ -23,7 +22,6 @@ import ( v1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/mime" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" "github.com/decred/politeia/util" "github.com/marcopeereboom/sbox" "github.com/subosito/gozaru" @@ -36,8 +34,6 @@ import ( const ( defaultTrillianKeyFilename = "trillian.key" defaultEncryptionKeyFilename = "tlogbe.key" - - recordsDirname = "records" ) var ( @@ -111,6 +107,13 @@ func tokenPrefix(token []byte) string { return hex.EncodeToString(token)[:pd.TokenPrefixLength] } +func (t *Tlogbe) isShutdown() bool { + t.RLock() + defer t.RUnlock() + + return t.shutdown +} + func (t *Tlogbe) prefixExists(fullToken []byte) bool { t.RLock() defer t.RUnlock() @@ -860,6 +863,10 @@ func (t *Tlogbe) VettedExists(token []byte) bool { func (t *Tlogbe) GetUnvetted(token []byte, version string) (*backend.Record, error) { log.Tracef("GetUnvetted: %x", token) + if t.isShutdown() { + return nil, backend.ErrShutdown + } + treeID := treeIDFromToken(token) v, err := strconv.ParseUint(version, 10, 64) if err != nil { @@ -873,6 +880,10 @@ func (t *Tlogbe) GetUnvetted(token []byte, version string) (*backend.Record, err func (t *Tlogbe) GetVetted(token []byte, version string) (*backend.Record, error) { log.Tracef("GetVetted: %x", token) + if t.isShutdown() { + return nil, backend.ErrShutdown + } + treeID := treeIDFromToken(token) v, err := strconv.ParseUint(version, 10, 64) if err != nil { @@ -1194,6 +1205,10 @@ func (t *Tlogbe) GetPlugins() ([]backend.Plugin, error) { func (t *Tlogbe) Plugin(pluginID, command, payload string) (string, string, error) { log.Tracef("Plugin: %v", command) + if t.isShutdown() { + return "", "", backend.ErrShutdown + } + // Get plugin plugin, ok := t.plugins[pluginID] if !ok { @@ -1300,51 +1315,26 @@ func New(homeDir, dataDir, dcrtimeHost, encryptionKeyFile, unvettedTrillianHost, log.Infof("Encryption key created: %v", encryptionKeyFile) } - // Load encryption key - f, err := os.Open(encryptionKeyFile) - if err != nil { - return nil, err - } - var key [32]byte - n, err := f.Read(key[:]) - if n != len(key) { - return nil, fmt.Errorf("invalid encryption key length") - } - if err != nil { - return nil, err - } - f.Close() - encryptionKey := newEncryptionKey(&key) - - log.Infof("Encryption key loaded") - - // Setup key-value store - fp := filepath.Join(dataDir, recordsDirname) - err = os.MkdirAll(fp, 0700) - if err != nil { - return nil, err - } - store := filesystem.New(fp) - - // Setup dcrtime host - _, err = url.Parse(dcrtimeHost) + // Verify dcrtime host + _, err := url.Parse(dcrtimeHost) if err != nil { return nil, fmt.Errorf("parse dcrtime host '%v': %v", dcrtimeHost, err) } log.Infof("Anchor host: %v", dcrtimeHost) // Setup tlog instances - unvetted, err := newTlog("unvetted", unvettedTrillianHost, - unvettedTrillianKeyFile, dcrtimeHost, encryptionKey, store) + unvetted, err := newTlog("unvetted", homeDir, dataDir, unvettedTrillianHost, + unvettedTrillianKeyFile, dcrtimeHost, encryptionKeyFile) if err != nil { return nil, fmt.Errorf("newTlog unvetted: %v", err) } - vetted, err := newTlog("vetted", vettedTrillianHost, - vettedTrillianKeyFile, dcrtimeHost, encryptionKey, store) + vetted, err := newTlog("vetted", homeDir, dataDir, vettedTrillianHost, + vettedTrillianKeyFile, dcrtimeHost, "") if err != nil { return nil, fmt.Errorf("newTlog vetted: %v", err) } + // Setup tlogbe t := Tlogbe{ homeDir: homeDir, dataDir: dataDir, @@ -1362,5 +1352,10 @@ func New(homeDir, dataDir, dcrtimeHost, encryptionKeyFile, unvettedTrillianHost, }, } + err = t.setup() + if err != nil { + return nil, fmt.Errorf("setup: %v", err) + } + return &t, nil } diff --git a/politeiad/backend/tlogbe/trillian.go b/politeiad/backend/tlogbe/trillian.go index 7c49d1436..47fb398ff 100644 --- a/politeiad/backend/tlogbe/trillian.go +++ b/politeiad/backend/tlogbe/trillian.go @@ -11,7 +11,6 @@ import ( "crypto/sha256" "fmt" "io/ioutil" - "path/filepath" "time" "github.com/decred/politeia/util" @@ -439,17 +438,11 @@ func (t *trillianClient) close() { t.grpc.Close() } -func newTrillianClient(homeDir, host, keyFile string) (*trillianClient, error) { +func newTrillianClient(host, keyFile string) (*trillianClient, error) { // Setup trillian key file - if keyFile == "" { - // No file path was given. Use the default path. - keyFile = filepath.Join(homeDir, defaultTrillianKeyFilename) - } if !util.FileExists(keyFile) { // Trillian key file does not exist. Create one. log.Infof("Generating trillian private key") - if keyFile == "" { - } key, err := keys.NewFromSpec(&keyspb.Specification{ // TODO Params: &keyspb.Specification_Ed25519Params{}, Params: &keyspb.Specification_EcdsaParams{}, @@ -469,7 +462,6 @@ func newTrillianClient(homeDir, host, keyFile string) (*trillianClient, error) { } // Setup trillian connection - log.Infof("Trillian host: %v", host) g, err := grpc.Dial(host, grpc.WithInsecure()) if err != nil { return nil, fmt.Errorf("grpc dial: %v", err) @@ -485,7 +477,6 @@ func newTrillianClient(homeDir, host, keyFile string) (*trillianClient, error) { if err != nil { return nil, err } - log.Infof("Trillian key loaded") t := trillianClient{ grpc: g, diff --git a/politeiad/cache/cache.go b/politeiad/cache/cache.go deleted file mode 100644 index 2a326a06d..000000000 --- a/politeiad/cache/cache.go +++ /dev/null @@ -1,215 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package cache - -import ( - "errors" - - "github.com/jinzhu/gorm" -) - -type RecordStatusT int - -var ( - // ErrNoVersionRecord is emitted when no version record exists. - ErrNoVersionRecord = errors.New("no version record") - - // ErrWrongVersion is emitted when the version record does not - // match the implementation version. - ErrWrongVersion = errors.New("wrong version") - - // ErrShutdown is emitted when the cache is shutting down. - ErrShutdown = errors.New("cache is shutting down") - - // ErrRecordNotFound is emitted when a cache record could not be - // found. - ErrRecordNotFound = errors.New("record not found") - - // ErrInvalidPlugin is emitted when an invalid plugin is used. - ErrInvalidPlugin = errors.New("invalid plugin") - - // ErrDuplicatePlugin is emitted when the a plugin is registered - // more than once. - ErrDuplicatePlugin = errors.New("duplicate plugin") - - // ErrInvalidPluginCmd is emitted when an invalid plugin command - // is used. - ErrInvalidPluginCmd = errors.New("invalid plugin command") - - // ErrInvalidPluginCmdArgs is emitted when a plugin command is used - // with invalid arguments. - ErrInvalidPluginCmdArgs = errors.New("invalid plugin command arguments") -) - -const ( - // Record status codes - RecordStatusInvalid RecordStatusT = 0 // Invalid status - RecordStatusNotFound RecordStatusT = 1 // Record not found - RecordStatusNotReviewed RecordStatusT = 2 // Record has not been reviewed - RecordStatusCensored RecordStatusT = 3 // Record has been censored - RecordStatusPublic RecordStatusT = 4 // Record is publicly visible - RecordStatusUnreviewedChanges RecordStatusT = 5 // NotReviewed record that has been changed - RecordStatusArchived RecordStatusT = 6 // Public record that has been archived -) - -// File describes an individual file that is part of the record. -type File struct { - Name string // Basename of the file - MIME string // MIME type - Digest string // SHA256 of decoded Payload - Payload string // base64 encoded file -} - -// MetadataStream identifies a metadata stream by its identity. -type MetadataStream struct { - ID uint64 // Stream identity - Payload string // String encoded metadata -} - -// CensorshipRecord contains the proof that a record was accepted for review. -// The proof is verifiable on the client side. The Merkle field contains the -// ordered merkle root of all files in the record. The Token field contains a -// random censorship token that is signed by the server private key. The token -// can be used on the client to verify the authenticity of the -// CensorshipRecord. -type CensorshipRecord struct { - Token string // Censorship token - Merkle string // Merkle root of record - Signature string // Signature of merkle+token -} - -// Record is an entire record and it's content. -type Record struct { - Version string // Version of this record - Status RecordStatusT // Current status - Timestamp int64 // Last update - CensorshipRecord CensorshipRecord // Censorship record - Metadata []MetadataStream // Metadata streams - Files []File // Files that make up the record -} - -// InventoryStats is a summary of the number of records in the cache grouped -// by record status. Only the latest version of each record is included. -type InventoryStats struct { - Invalid int // Number of invalid records - NotReviewed int // Number of unreviewed records - Censored int // Number of censored records - Public int // Number of public records - UnreviewedChanges int // Number of unreviewed records with edits - Archived int // Number of archived records -} - -// PluginCommand is used to execute a plugin command. The reply payload -// contains the reply from politeiad, which is sometimes required by commands -// that write data to the cache. The reply payload will be empty for commands -// that only read data from the cache. -type PluginCommand struct { - ID string // Plugin identifier - Command string // Command identifier - CommandPayload string // Command payload - ReplyPayload string // Command reply payload -} - -// PluginCommandReply is used to reply to a PluginCommand. -type PluginCommandReply struct { - ID string // Plugin identifier - Command string // Command identifier - Payload string // Actual command reply -} - -// PluginSetting is a structure that holds key/value pairs of a plugin setting. -type PluginSetting struct { - Key string // Name of setting - Value string // Value of setting -} - -// Plugin describes a plugin and its settings. -type Plugin struct { - ID string // Identifier - Version string // Version - Settings []PluginSetting // Settings -} - -// PluginDriver describes the common set of methods that the cache uses to -// build and maintain the cache for a plugin. -// -// All cache plugins must implement the PluginDriver interface. -type PluginDriver interface { - // Check that the correct plugin version is being used - CheckVersion() error - - // Setup the plugin tables - Setup() error - - // Drop the existing plugin tables and rebuild them - Build(payload string) error - - // Execute a plugin command. Some commands are executed by - // politeiad first then fowarded to the cache. If this is the case - // the replyPayload will be populated with the politeiad reply, - // otherwise the replyPayload will be empty. - Exec(cmdID, cmdPayload, replyPayload string) (string, error) - - // Run a plugin hook. The given gorm.DB should be a transaction so - // that the hook actions can be executed atomically. - Hook(tx *gorm.DB, hookID, payload string) error -} - -// Cache describes the interface used for interacting with an external -// politeiad cache. The politeiad backend implementation serves as the source -// of truth for politeiad data and an external cache can be used if more -// performant queries are required. -type Cache interface { - // Create a new record - NewRecord(Record) error - - // Get the latest version of a record - Record(string) (*Record, error) - - // Get the latest version of a record based on its prefix. - // The length of the prefix is defined by TokenPrefixLength in the - // politeiad api. - RecordByPrefix(string) (*Record, error) - - // Get a specific version of a record - RecordVersion(string, string) (*Record, error) - - // Update a record - UpdateRecord(Record) error - - // Update the status of a record - UpdateRecordStatus(string, string, RecordStatusT, int64, - []MetadataStream) error - - // Update the metadata streams of a record - UpdateRecordMetadata(string, []MetadataStream) error - - // Get the latest version of a set of records - Records([]string, bool) (map[string]Record, error) - - // Get the latest version of all records - Inventory() ([]Record, error) - - // Setup the record cache tables - Setup() error - - // Drop existing tables and rebuild them - Build([]Record) error - - // Register a plugin with the cache - RegisterPlugin(Plugin) error - - // Setup the database tables for a plugin - PluginSetup(string) error - - // Drop existing plugin tables and rebuild them - PluginBuild(string, string) error - - // Execute a plugin command - PluginExec(PluginCommand) (*PluginCommandReply, error) - - // Perform cleanup of the cache - Close() -} diff --git a/politeiad/cache/cachestub/cachestub.go b/politeiad/cache/cachestub/cachestub.go deleted file mode 100644 index 3f2d19c14..000000000 --- a/politeiad/cache/cachestub/cachestub.go +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package cachestub - -import "github.com/decred/politeia/politeiad/cache" - -// cachestub implements the cache interface. -type cachestub struct{} - -// NewRecord is a stub to satisfy the cache interface. -func (c *cachestub) NewRecord(r cache.Record) error { - return nil -} - -// RecordByPrefix is a stub to satisfy the cache interface. -func (c *cachestub) RecordByPrefix(prefix string) (*cache.Record, error) { - return &cache.Record{}, nil -} - -// Record is a stub to satisfy the cache interface. -func (c *cachestub) Record(token string) (*cache.Record, error) { - return &cache.Record{}, nil -} - -// Records is a stub to satisfy the cache interface. -func (c *cachestub) Records(token []string, fetchFiles bool) (map[string]cache.Record, error) { - records := make(map[string]cache.Record) - return records, nil -} - -// RecordVersion is a stub to satisfy the cache interface. -func (c *cachestub) RecordVersion(token, version string) (*cache.Record, error) { - return &cache.Record{}, nil -} - -// UpdateRecord is a stub to satisfy the cache interface. -func (c *cachestub) UpdateRecord(r cache.Record) error { - return nil -} - -// UpdateRecordStatus is a stub to satisfy the cache interface. -func (c *cachestub) UpdateRecordStatus(token, version string, status cache.RecordStatusT, timestamp int64, metadata []cache.MetadataStream) error { - return nil -} - -func (c *cachestub) UpdateRecordMetadata(token string, ms []cache.MetadataStream) error { - return nil -} - -// Inventory is a stub to satisfy the cache interface. -func (c *cachestub) Inventory() ([]cache.Record, error) { - return make([]cache.Record, 0), nil -} - -// InventoryStats is a stub to satisfy the cache interface. -func (c *cachestub) InventoryStats() (*cache.InventoryStats, error) { - return &cache.InventoryStats{}, nil -} - -// Setup is a stub to satisfy the cache interface. -func (c *cachestub) Setup() error { - return nil -} - -// Build is a stub to satisfy the cache interface. -func (c *cachestub) Build(records []cache.Record) error { - return nil -} - -func (c *cachestub) RegisterPlugin(p cache.Plugin) error { - return nil -} - -// PluginSetup is a stub to satisfy the cache interface. -func (c *cachestub) PluginSetup(id string) error { - return nil -} - -// PluginBuild is a stub to satisfy the cache interface. -func (c *cachestub) PluginBuild(id, payload string) error { - return nil -} - -// PluginExec is a stub to satisfy the cache interface. -func (c *cachestub) PluginExec(pc cache.PluginCommand) (*cache.PluginCommandReply, error) { - return &cache.PluginCommandReply{}, nil -} - -// Close is a stub to satisfy the cache interface. -func (c *cachestub) Close() {} - -// NewStub returns a new cachestub context. -func New() *cachestub { - return &cachestub{} -} diff --git a/politeiad/cache/cockroachdb/cms.go b/politeiad/cache/cockroachdb/cms.go deleted file mode 100644 index d8e83faf0..000000000 --- a/politeiad/cache/cockroachdb/cms.go +++ /dev/null @@ -1,813 +0,0 @@ -// Copyright (c) 2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package cockroachdb - -import ( - "fmt" - "strconv" - "time" - - "github.com/decred/politeia/cmsplugin" - "github.com/decred/politeia/politeiad/cache" - "github.com/jinzhu/gorm" -) - -const ( - // cmsVersion is the version of the cache implementation of - // cms plugin. This may differ from the cmsplugin package - // version. - cmsVersion = "1" - - tableCastDCCVotes = "cast_dcc_votes" - tableStartDCCVotes = "start_dcc_votes" - tableDCCUserWeights = "user_weights" - tableVoteDCCOptionResults = "vote_dcc_options_results" - tableVoteDCCOptions = "vote_dcc_options" - tableVoteDCCResults = "vote_dcc_results" -) - -// cms implements the PluginDriver interface. -type cms struct { - recordsdb *gorm.DB // Database context - version string // Version of cms cache plugin - settings []cache.PluginSetting // Plugin settings -} - -// newStartDCCVote inserts a StartDCCVote record into the database. This function -// has a database parameter so that it can be called inside of a transaction -// when required. -func (c *cms) newStartDCCVote(db *gorm.DB, sv StartDCCVote) error { - return db.Create(&sv).Error -} - -// cmdStartVote creates a StartDCCVote record using the passed in payloads and -// inserts it into the database. -func (c *cms) cmdStartVote(cmdPayload, replyPayload string) (string, error) { - log.Tracef("cms cmdStartDCCVote") - - sv, err := cmsplugin.DecodeStartVote([]byte(cmdPayload)) - if err != nil { - return "", err - } - svr, err := cmsplugin.DecodeStartVoteReply([]byte(replyPayload)) - if err != nil { - return "", err - } - - s := convertStartVoteFromCMS(sv, svr, svr.EndHeight) - err = c.newStartDCCVote(c.recordsdb, s) - if err != nil { - return "", err - } - - return replyPayload, nil -} - -// cmdVoteDetails returns the StartDCCVote records for the -// passed in record token. -func (c *cms) cmdVoteDetails(payload string) (string, error) { - log.Tracef("cms cmdDCCVoteDetails") - - vd, err := cmsplugin.DecodeVoteDetails([]byte(payload)) - if err != nil { - return "", nil - } - - // Lookup the most recent version of the record - var r Record - err = c.recordsdb. - Where("records.token = ?", vd.Token). - Order("records.version desc"). - Limit(1). - Find(&r). - Error - if err != nil { - if err == gorm.ErrRecordNotFound { - err = cache.ErrRecordNotFound - } - return "", err - } - - // Lookup start vote - var sv StartDCCVote - err = c.recordsdb. - Where("token = ?", vd.Token). - Preload("Options"). - Preload("EligibleUserIDs"). - Find(&sv). - Error - if err == gorm.ErrRecordNotFound { - // A start vote may note exist. This is ok. - } else if err != nil { - return "", fmt.Errorf("start vote lookup failed: %v", err) - } - - // Prepare reply - dsv, dsvr := convertStartVoteToCMS(sv) - vdr := cmsplugin.VoteDetailsReply{ - StartVote: dsv, - StartVoteReply: dsvr, - } - vdrb, err := cmsplugin.EncodeVoteDetailsReply(vdr) - if err != nil { - return "", err - } - - return string(vdrb), nil -} - -// cmdCastVote creates CastVote records using the passed in payloads and -// inserts them into the database. -func (c *cms) cmdCastVote(cmdPayload, replyPayload string) (string, error) { - log.Tracef("cms cmdNewCastVote") - - b, err := cmsplugin.DecodeCastVote([]byte(cmdPayload)) - if err != nil { - return "", err - } - - if b.Signature == "" { - log.Debugf("cmdNewCastVote: vote receipt not found %v %v", - b.Token, b.UserID) - return "", nil - } - cv := convertCastVoteFromCMS(*b) - err = c.recordsdb.Create(&cv).Error - if err != nil { - return "", err - } - - return replyPayload, nil -} - -// cmdProposalVotes returns the StartVote record and all CastVote records for -// the passed in record token. -func (c *cms) cmdDCCVotes(payload string) (string, error) { - log.Tracef("cms cmdProposalVotes") - - vr, err := cmsplugin.DecodeVoteResults([]byte(payload)) - if err != nil { - return "", err - } - - // Lookup start vote - var sv StartDCCVote - err = c.recordsdb. - Where("token = ?", vr.Token). - Preload("Options"). - Find(&sv). - Error - if err == gorm.ErrRecordNotFound { - // A start vote may note exist if the voting period has not - // been started yet. This is ok. - } else if err != nil { - return "", fmt.Errorf("start vote lookup failed: %v", err) - } - - // Lookup all cast votes - var cv []CastDCCVote - err = c.recordsdb. - Where("token = ?", vr.Token). - Find(&cv). - Error - if err == gorm.ErrRecordNotFound { - // No cast votes may exist yet. This is ok. - } else if err != nil { - return "", fmt.Errorf("cast votes lookup failed: %v", err) - } - - // Prepare reply - dsv, _ := convertStartVoteToCMS(sv) - dcv := make([]cmsplugin.CastVote, 0, len(cv)) - for _, v := range cv { - dcv = append(dcv, convertCastVoteToCMS(v)) - } - - vrr := cmsplugin.VoteResultsReply{ - StartVote: dsv, - CastVotes: dcv, - } - - vrrb, err := cmsplugin.EncodeVoteResultsReply(vrr) - if err != nil { - return "", err - } - - return string(vrrb), nil -} - -// cmdInventory returns the cms plugin inventory. -func (c *cms) cmdInventory() (string, error) { - log.Tracef("cms cmdInventory") - - // XXX we don't currently return anything for inventory here - - return "", nil -} - -// cmdLoadVoteResults creates vote results entries for any dccs that have -// a finished voting period but have not yet been added to the vote results -// table. The vote results table is lazy loaded. -func (c *cms) cmdLoadVoteResults(payload string) (string, error) { - log.Tracef("cmdLoadVoteResults") - - lvs, err := cmsplugin.DecodeLoadVoteResults([]byte(payload)) - if err != nil { - return "", err - } - - // Find dccs that have a finished voting period but - // have not yet been added to the vote results table. - q := `SELECT start__dcc_votes.token - FROM start_dcc_votes - LEFT OUTER JOIN vote_dcc_results - ON start_dcc_votes.token = vote_dcc_results.token - WHERE start_dcc_votes.end_height <= ? - AND vote_dcc_results.token IS NULL` - rows, err := c.recordsdb.Raw(q, lvs.BestBlock).Rows() - if err != nil { - return "", fmt.Errorf("no vote results: %v", err) - } - defer rows.Close() - - var token string - tokens := make([]string, 0, 1024) - for rows.Next() { - err = rows.Scan(&token) - if err != nil { - return "", err - } - tokens = append(tokens, token) - } - if err = rows.Err(); err != nil { - return "", err - } - - // Create vote result entries - for _, v := range tokens { - err := c.newVoteResults(v) - if err != nil { - return "", fmt.Errorf("newVoteResults %v: %v", v, err) - } - } - - // Prepare reply - r := cmsplugin.LoadVoteResultsReply{} - reply, err := cmsplugin.EncodeLoadVoteResultsReply(r) - if err != nil { - return "", err - } - - return string(reply), nil -} - -// newVoteResults creates a VoteDCCResults record for a proposal and inserts it -// into the cache. A VoteDCCResults record should only be created for proposals -// once the voting period has ended. -func (c *cms) newVoteResults(token string) error { - log.Tracef("newVoteResults %v", token) - - // Lookup start vote - var sv StartDCCVote - err := c.recordsdb. - Where("token = ?", token). - Preload("Options"). - Find(&sv). - Error - if err != nil { - return fmt.Errorf("lookup start vote: %v", err) - } - - // Lookup cast votes - var cv []CastDCCVote - err = c.recordsdb. - Where("token = ?", token). - Find(&cv). - Error - if err == gorm.ErrRecordNotFound { - // No cast votes exists. In theory, this could - // happen if no one were to vote on a proposal. - // In practice, this shouldn't happen. - } else if err != nil { - return fmt.Errorf("lookup cast votes: %v", err) - } - - // Tally cast votes - tally := make(map[string]uint64) // [voteBit]voteCount - for _, v := range cv { - tally[v.VoteBit]++ - } - - // Create vote option results - results := make([]VoteDCCOptionResult, 0, len(sv.Options)) - for _, v := range sv.Options { - voteBit := strconv.FormatUint(v.Bits, 16) - voteCount := tally[voteBit] - - results = append(results, VoteDCCOptionResult{ - Key: token + voteBit, - Votes: voteCount, - Option: v, - }) - } - - // Check whether vote was approved - var total uint64 - for _, v := range results { - total += v.Votes - } - - eligible := sv.EligibleUserIDs - quorum := uint64(int(sv.QuorumPercentage) / 100 * len(eligible)) - pass := uint64(float64(sv.PassPercentage) / 100 * float64(total)) - - var approvedVotes uint64 - for _, v := range results { - if v.Option.ID == cmsplugin.DCCApprovalString { - approvedVotes = v.Votes - } - } - - var approved bool - switch { - case total < quorum: - // Quorum not met - case approvedVotes < pass: - // Pass percentage not met - default: - // Vote was approved - approved = true - } - - // Create a vote results entry - err = c.recordsdb.Create(&VoteDCCResults{ - Token: token, - Approved: approved, - Results: results, - }).Error - if err != nil { - return fmt.Errorf("new vote results: %v", err) - } - - return nil -} - -// getStartVotes looks up the start votes for records which have been -// authorized to start voting. -func (c *cms) getStartVotes(records map[string]Record) (map[string]StartDCCVote, error) { - startVotes := make(map[string]StartDCCVote) - - if len(records) == 0 { - return startVotes, nil - } - - keys := make([]string, 0, len(records)) - for token, record := range records { - keys = append(keys, token+strconv.FormatUint(record.Version, 10)) - } - - svs := make([]StartDCCVote, 0, len(keys)) - err := c.recordsdb. - Where("key IN (?)", keys). - Preload("Options"). - Find(&svs). - Error - if err != nil { - return nil, err - } - for _, sv := range svs { - startVotes[sv.Token] = sv - } - - return startVotes, nil -} - -// lookupResultsForVoteDCCOptions looks in the CastDCCVote table to see how many -// votes each option has received. -func (c *cms) lookupResultsForVoteDCCOptions(options []VoteDCCOption) ([]cmsplugin.VoteOptionResult, error) { - results := make([]cmsplugin.VoteOptionResult, 0, len(options)) - - for _, v := range options { - var votes uint64 - tokenVoteBit := v.Token + strconv.FormatUint(v.Bits, 16) - err := c.recordsdb. - Model(&CastDCCVote{}). - Where("token_vote_bit = ?", tokenVoteBit). - Count(&votes). - Error - if err != nil { - return nil, err - } - - results = append(results, - cmsplugin.VoteOptionResult{ - ID: v.ID, - Description: v.Description, - Bits: v.Bits, - Votes: votes, - }) - } - - return results, nil -} - -// getVoteResults retrieves vote results for records that have begun the voting -// process. Results are lazily loaded into this table, so some results are -// manually looked up in the CastDCCVote table. -func (c *cms) getVoteResults(startVotes map[string]StartDCCVote) (map[string][]cmsplugin.VoteOptionResult, error) { - results := make(map[string][]cmsplugin.VoteOptionResult) - - if len(startVotes) == 0 { - return results, nil - } - - tokens := make([]string, 0, len(startVotes)) - for token := range startVotes { - tokens = append(tokens, token) - } - - vrs := make([]VoteResults, 0, len(tokens)) - err := c.recordsdb. - Where("token IN (?)", tokens). - Preload("Results"). - Preload("Results.Option"). - Find(&vrs). - Error - if err != nil { - return nil, err - } - - for _, vr := range vrs { - results[vr.Token] = convertVoteOptionResultsToCMS(vr.Results) - } - - for token, sv := range startVotes { - _, ok := results[token] - if ok { - continue - } - - res, err := c.lookupResultsForVoteDCCOptions(sv.Options) - if err != nil { - return nil, err - } - - results[token] = res - } - - return results, nil -} - -func (c *cms) cmdVoteSummary(payload string) (string, error) { - log.Tracef("cms cmdVoteSummary") - - vs, err := cmsplugin.DecodeVoteSummary([]byte(payload)) - if err != nil { - return "", err - } - - // Lookup the most recent record version - var r Record - err = c.recordsdb. - Where("records.token = ?", vs.Token). - Order("records.version desc"). - Limit(1). - Find(&r). - Error - if err != nil { - if err == gorm.ErrRecordNotFound { - err = cache.ErrRecordNotFound - } - return "", err - } - - // Declare here to prevent goto errors - results := make([]cmsplugin.VoteOptionResult, 0, 16) - var ( - sv StartDCCVote - vr VoteResults - ) - - // Lookup start vote - err = c.recordsdb. - Where("token = ?", vs.Token). - Preload("Options"). - Find(&sv). - Error - if err == gorm.ErrRecordNotFound { - // If an start vote doesn't exist then - // there is no need to continue. - goto sendReply - } else if err != nil { - return "", fmt.Errorf("lookup start vote: %v", err) - } - - // Lookup vote results - err = c.recordsdb. - Where("token = ?", vs.Token). - Preload("Results"). - Preload("Results.Option"). - Find(&vr). - Error - if err == gorm.ErrRecordNotFound { - // A vote results record was not found. This means that - // the vote is either still active or has not been lazy - // loaded yet. The vote results will need to be looked - // up manually. - } else if err != nil { - return "", fmt.Errorf("lookup vote results: %v", err) - } else { - // Vote results record exists. We have all of the data - // that we need to send the reply. - vor := convertVoteOptionResultsToCMS(vr.Results) - results = append(results, vor...) - goto sendReply - } - - // Lookup vote results manually - results, err = c.lookupResultsForVoteDCCOptions(sv.Options) - if err != nil { - return "", fmt.Errorf("count cast votes: %v", err) - } - -sendReply: - - vsr := cmsplugin.VoteSummaryReply{ - Duration: sv.Duration, - EndHeight: sv.EndHeight, - PassPercentage: sv.PassPercentage, - Results: results, - } - reply, err := cmsplugin.EncodeVoteSummaryReply(vsr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -// Exec executes a cms plugin command. Plugin commands that write data to -// the cache require both the command payload and the reply payload. Plugin -// commands that fetch data from the cache require only the command payload. -// All commands return the appropriate reply payload. -func (c *cms) Exec(cmd, cmdPayload, replyPayload string) (string, error) { - log.Tracef("cms Exec: %v", cmd) - - switch cmd { - case cmsplugin.CmdStartVote: - return c.cmdStartVote(cmdPayload, replyPayload) - case cmsplugin.CmdVoteDetails: - return c.cmdVoteDetails(cmdPayload) - case cmsplugin.CmdCastVote: - return c.cmdCastVote(cmdPayload, replyPayload) - case cmsplugin.CmdInventory: - return c.cmdInventory() - case cmsplugin.CmdVoteSummary: - return c.cmdVoteSummary(cmdPayload) - case cmsplugin.CmdLoadVoteResults: - return c.cmdLoadVoteResults(cmdPayload) - } - - return "", cache.ErrInvalidPluginCmd -} - -// createTables creates the cache tables needed by the cms plugin if they do -// not already exist. A cms plugin version record is inserted into the -// database during table creation. -// -// This function must be called within a transaction. -func (c *cms) createTables(tx *gorm.DB) error { - log.Tracef("createTables") - - if !tx.HasTable(tableCastDCCVotes) { - err := tx.CreateTable(&CastDCCVote{}).Error - if err != nil { - return err - } - } - if !tx.HasTable(tableVoteDCCOptions) { - err := tx.CreateTable(&VoteDCCOption{}).Error - if err != nil { - return err - } - } - if !tx.HasTable(tableStartDCCVotes) { - err := tx.CreateTable(&StartDCCVote{}).Error - if err != nil { - return err - } - } - if !tx.HasTable(tableVoteDCCOptionResults) { - err := tx.CreateTable(&VoteDCCOptionResult{}).Error - if err != nil { - return err - } - } - if !tx.HasTable(tableVoteDCCResults) { - err := tx.CreateTable(&VoteDCCResults{}).Error - if err != nil { - return err - } - } - if !tx.HasTable(tableDCCUserWeights) { - err := tx.CreateTable(&DCCUserWeight{}).Error - if err != nil { - return err - } - } - - // Check if a cms version record exists. Insert one - // if no version record is found. - if !tx.HasTable(tableVersions) { - // This should never happen - return fmt.Errorf("versions table not found") - } - - var v Version - err := tx.Where("id = ?", cmsplugin.ID).Find(&v).Error - if err == gorm.ErrRecordNotFound { - err = tx.Create( - &Version{ - ID: cmsplugin.ID, - Version: cmsVersion, - Timestamp: time.Now().Unix(), - }).Error - } - - return err -} - -// dropTables drops all cms plugin tables from the cache and remove the -// cms plugin version record. -// -// This function must be called within a transaction. -func (c *cms) dropTables(tx *gorm.DB) error { - // Drop cms plugin tables - err := tx.DropTableIfExists(tableCastDCCVotes, tableStartDCCVotes, - tableVoteDCCOptions, tableVoteDCCOptionResults, tableVoteDCCResults, - tableDCCUserWeights). - Error - if err != nil { - return err - } - - // Remove cms plugin version record - return tx.Delete(&Version{ - ID: cmsplugin.ID, - }).Error -} - -// build the cms plugin cache using the passed in inventory. -// -// This function cannot be called using a transaction because it could -// potentially exceed cockroachdb's transaction size limit. -func (c *cms) build(ir *cmsplugin.InventoryReply) error { - log.Tracef("cms build") - - // Drop all cms plugin tables - tx := c.recordsdb.Begin() - err := c.dropTables(tx) - if err != nil { - tx.Rollback() - return fmt.Errorf("drop tables: %v", err) - } - err = tx.Commit().Error - if err != nil { - return err - } - - // Create cms plugin tables - tx = c.recordsdb.Begin() - err = c.createTables(tx) - if err != nil { - tx.Rollback() - return fmt.Errorf("create tables: %v", err) - } - err = tx.Commit().Error - if err != nil { - return err - } - - // Build start vote cache - log.Tracef("cms: building start vote cache") - for _, v := range ir.StartVoteTuples { - sv := convertStartVoteFromCMS(v.StartVote, - v.StartVoteReply, v.StartVoteReply.EndHeight) - err = c.newStartDCCVote(c.recordsdb, sv) - if err != nil { - log.Debugf("newStartVote failed on '%v'", sv) - return fmt.Errorf("newStartVote: %v", err) - } - } - - // Build cast vote cache - log.Tracef("cms: building cast vote cache") - for _, v := range ir.CastVotes { - cv := convertCastVoteFromCMS(v) - err := c.recordsdb.Create(&cv).Error - if err != nil { - log.Debugf("insert cast vote failed on '%v'", cv) - return fmt.Errorf("insert cast vote: %v", err) - } - } - - return nil -} - -// Build drops all existing cms plugin tables from the database, recreates -// them, then uses the passed in inventory payload to build the cms plugin -// cache. -func (c *cms) Build(payload string) error { - log.Tracef("cms Build") - - // Decode the payload - ir, err := cmsplugin.DecodeInventoryReply([]byte(payload)) - if err != nil { - return fmt.Errorf("DecodeInventoryReply: %v", err) - } - - // Build the cms plugin cache. This is not run using - // a transaction because it could potentially exceed - // cockroachdb's transaction size limit. - err = c.build(ir) - if err != nil { - // Remove the version record. This will - // force a rebuild on the next start up. - err1 := c.recordsdb.Delete(&Version{ - ID: cmsplugin.ID, - }).Error - if err1 != nil { - panic("the cache is out of sync and will not rebuild" + - "automatically; a rebuild must be forced") - } - } - - return err -} - -// Setup creates the cms plugin tables if they do not already exist. A -// cms plugin version record is inserted into the database during table -// creation. -func (c *cms) Setup() error { - log.Tracef("cms: Setup") - - tx := c.recordsdb.Begin() - err := c.createTables(tx) - if err != nil { - tx.Rollback() - return err - } - - return tx.Commit().Error -} - -// CheckVersion retrieves the cms plugin version record from the database, -// if one exists, and checks that it matches the version of the current cms -// plugin cache implementation. -func (c *cms) CheckVersion() error { - log.Tracef("cms: CheckVersion") - - // Sanity check. Ensure version table exists. - if !c.recordsdb.HasTable(tableVersions) { - return fmt.Errorf("versions table not found") - } - - // Lookup version record. If the version is not found or - // if there is a version mismatch, return an error so - // that the cms plugin cache can be built/rebuilt. - var v Version - err := c.recordsdb. - Where("id = ?", cmsplugin.ID). - Find(&v). - Error - if err == gorm.ErrRecordNotFound { - log.Debugf("version record not found for ID '%v'", - cmsplugin.ID) - err = cache.ErrNoVersionRecord - } else if v.Version != cmsVersion { - log.Debugf("version mismatch for ID '%v': got %v, want %v", - cmsplugin.ID, v.Version, cmsVersion) - err = cache.ErrWrongVersion - } - - return err -} - -// Hook executes the given cms plugin hook. -func (d *cms) Hook(tx *gorm.DB, hookID, payload string) error { - log.Tracef("cms Hook: %v", hookID) - - return nil -} - -// newCMSPlugin returns a cache cms plugin context. -func newCMSPlugin(db *gorm.DB, p cache.Plugin) *cms { - log.Tracef("newCMSPlugin") - return &cms{ - recordsdb: db, - version: cmsVersion, - settings: p.Settings, - } -} diff --git a/politeiad/cache/cockroachdb/cockroachdb.go b/politeiad/cache/cockroachdb/cockroachdb.go deleted file mode 100644 index cf592d1b2..000000000 --- a/politeiad/cache/cockroachdb/cockroachdb.go +++ /dev/null @@ -1,1043 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package cockroachdb - -import ( - "encoding/binary" - "encoding/json" - "fmt" - "net/url" - "path/filepath" - "strconv" - "sync" - "time" - - "github.com/decred/politeia/cmsplugin" - "github.com/decred/politeia/decredplugin" - "github.com/decred/politeia/politeiad/cache" - "github.com/jinzhu/gorm" - _ "github.com/jinzhu/gorm/dialects/postgres" -) - -const ( - cacheID = "records" - cacheVersion = "1" - - // Database table names - tableKeyValue = "key_value" - tableVersions = "versions" - tableRecords = "records" - tableMetadataStreams = "metadata_streams" - tableFiles = "files" - - // Plugin hooks - pluginHookPostNewRecord = "postnewrecord" - pluginHookPostUpdateRecord = "postupdaterecord" - pluginHookPostUpdateRecordMetadata = "postupdaterecordmetadata" - - // Database users - UserPoliteiad = "politeiad" // politeiad user (read/write access) - UserPoliteiawww = "politeiawww" // politeiawww user (read access) - - // Key-value store keys - keyBestBlock = "bestblock" -) - -// cockroachdb implements the cache interface. -type cockroachdb struct { - sync.RWMutex - shutdown bool // Backend is shutdown - recordsdb *gorm.DB // Database context - plugins map[string]cache.PluginDriver // [pluginID]PluginDriver -} - -// isShutdown returns whether the backend has been shutdown. -func (c *cockroachdb) isShutdown() bool { - c.RLock() - defer c.RUnlock() - - return c.shutdown -} - -// recordExists returns whether a record exists for the provided token and -// version. -func recordExists(db *gorm.DB, token string, version string) (bool, error) { - var r Record - err := db.Where("key = ?", token+version).Find(&r).Error - if err == gorm.ErrRecordNotFound { - // Record doesn't exist - return false, nil - } else if err != nil { - // All other errors - return false, err - } - - // Record exists - return true, nil -} - -func (c *cockroachdb) newRecord(tx *gorm.DB, r Record) error { - // Insert record - err := tx.Create(&r).Error - if err != nil { - return err - } - - // Call plugin hooks - if c.pluginIsRegistered(decredplugin.ID) { - plugin, err := c.getPlugin(decredplugin.ID) - if err != nil { - return err - } - payload, err := json.Marshal(r) - if err != nil { - return err - } - err = plugin.Hook(tx, pluginHookPostNewRecord, string(payload)) - if err != nil { - return err - } - } - - return nil -} - -// NewRecord creates a new entry in the database for the passed in record. -func (c *cockroachdb) NewRecord(cr cache.Record) error { - log.Tracef("NewRecord: %v", cr.CensorshipRecord.Token) - - if c.isShutdown() { - return cache.ErrShutdown - } - - v, err := strconv.ParseUint(cr.Version, 10, 64) - if err != nil { - return fmt.Errorf("parse version '%v' failed: %v", - cr.Version, err) - } - r := convertRecordFromCache(cr, v) - - tx := c.recordsdb.Begin() - err = c.newRecord(tx, r) - if err != nil { - tx.Rollback() - return err - } - - return tx.Commit().Error -} - -// recordByPrefix gets the most recent version of a record using the prefix -// of its token. The length of the prefix is defined by TokenPrefixLength -// in the politeiad api. -// -// This function has a database parameter so that it can be called inside of a -// transaction when required. -func recordByPrefix(db *gorm.DB, prefix string) (*Record, error) { - var r Record - err := db. - Where("records.token_prefix = ?", prefix). - Order("records.version desc"). - Limit(1). - Preload("Metadata"). - Preload("Files"). - Find(&r). - Error - if err != nil { - if err == gorm.ErrRecordNotFound { - err = cache.ErrRecordNotFound - } - return nil, err - } - return &r, nil -} - -// RecordByPrefix gets the most recent version of a record from the database -// using the prefix of its token. The length of the prefix is defined by -// TokenPrefixLength in the politeiad api. -func (c *cockroachdb) RecordByPrefix(prefix string) (*cache.Record, error) { - log.Tracef("RecordByPrefix %v", prefix) - - if c.isShutdown() { - return nil, cache.ErrShutdown - } - - r, err := recordByPrefix(c.recordsdb, prefix) - if err != nil { - return nil, err - } - - cr := convertRecordToCache(*r) - return &cr, nil -} - -// recordVersion gets the specified version of a record from the database. -// This function has a database parameter so that it can be called inside of -// a transaction when required. -func (c *cockroachdb) recordVersion(db *gorm.DB, token, version string) (*Record, error) { - log.Tracef("getRecordVersion: %v %v", token, version) - - r := Record{ - Key: token + version, - } - err := db.Preload("Metadata"). - Preload("Files"). - Find(&r). - Error - if err != nil { - if err == gorm.ErrRecordNotFound { - err = cache.ErrRecordNotFound - } - return nil, err - } - return &r, nil -} - -// RecordVersion gets the specified version of a record from the database. -func (c *cockroachdb) RecordVersion(token, version string) (*cache.Record, error) { - log.Tracef("RecordVersion: %v %v", token, version) - - if c.isShutdown() { - return nil, cache.ErrShutdown - } - - r, err := c.recordVersion(c.recordsdb, token, version) - if err != nil { - return nil, err - } - - cr := convertRecordToCache(*r) - return &cr, nil -} - -// record gets the most recent version of a record from the database. This -// function has a database parameter so that it can be called inside of a -// transaction when required. -func record(db *gorm.DB, token string) (*Record, error) { - var r Record - err := db. - Where("records.token = ?", token). - Order("records.version desc"). - Limit(1). - Preload("Metadata"). - Preload("Files"). - Find(&r). - Error - if err != nil { - if err == gorm.ErrRecordNotFound { - err = cache.ErrRecordNotFound - } - return nil, err - } - return &r, nil -} - -// Record gets the most recent version of a record from the database. -func (c *cockroachdb) Record(token string) (*cache.Record, error) { - log.Tracef("Record: %v", token) - - if c.isShutdown() { - return nil, cache.ErrShutdown - } - - r, err := record(c.recordsdb, token) - if err != nil { - return nil, err - } - - cr := convertRecordToCache(*r) - return &cr, nil -} - -// updateMetadataStreams updates a record's metadata streams by deleting the -// existing metadata streams then adding the passed in metadata streams to the -// database. -// -// This function must be called using a transaction. -func updateMetadataStreams(tx *gorm.DB, key string, ms []MetadataStream) error { - // Delete existing metadata streams - err := tx.Where("record_key = ?", key). - Delete(MetadataStream{}). - Error - if err != nil { - return fmt.Errorf("delete MD streams: %v", err) - } - - // Add new metadata streams - for _, v := range ms { - err = tx.Create(&MetadataStream{ - RecordKey: key, - ID: v.ID, - Payload: v.Payload, - }).Error - if err != nil { - return fmt.Errorf("create MD stream %v: %v", - v.ID, err) - } - } - - return nil -} - -// updateRecord updates a record in the database. This includes updating the -// record as well as any metadata streams and files that are associated with -// the record. The existing record metadata streams and files are deleted from -// the database before the passed in metadata streams and files are added. -// -// This function must be called within a transaction. -func (c *cockroachdb) updateRecord(tx *gorm.DB, updated Record) error { - log.Tracef("updateRecord: %v %v", updated.Token, updated.Version) - - // Ensure record exists. We need to do this because updates - // will not return an error if you try to update a record that - // does not exist. - record, err := c.recordVersion(tx, updated.Token, - strconv.FormatUint(updated.Version, 10)) - if err != nil { - return err - } - - // Update record - err = tx.Model(&record). - Updates(map[string]interface{}{ - "status": updated.Status, - "timestamp": updated.Timestamp, - "merkle": updated.Merkle, - "signature": updated.Signature, - }).Error - if err != nil { - return fmt.Errorf("update record: %v", err) - } - - // Update metadata - err = updateMetadataStreams(tx, record.Key, updated.Metadata) - if err != nil { - return err - } - - // Delete existing files - err = tx.Where("record_key = ?", record.Key). - Delete(File{}). - Error - if err != nil { - return fmt.Errorf("delete files: %v", err) - } - - // Add new files - for _, f := range updated.Files { - err = tx.Create(&File{ - RecordKey: record.Key, - Name: f.Name, - MIME: f.MIME, - Digest: f.Digest, - Payload: f.Payload, - }).Error - if err != nil { - return fmt.Errorf("create file %v: %v", f.Name, err) - } - } - - // Call plugin hooks - if c.pluginIsRegistered(decredplugin.ID) { - plugin, err := c.getPlugin(decredplugin.ID) - if err != nil { - return err - } - payload, err := json.Marshal(updated) - if err != nil { - return err - } - err = plugin.Hook(tx, pluginHookPostUpdateRecord, string(payload)) - if err != nil { - return err - } - } - - return nil -} - -// UpdateRecord updates a record in the database. This includes updating the -// record as well as any metadata streams and files that are associated with -// the record. -func (c *cockroachdb) UpdateRecord(r cache.Record) error { - log.Tracef("UpdateRecord: %v %v", r.CensorshipRecord.Token, r.Version) - - if c.isShutdown() { - return cache.ErrShutdown - } - - v, err := strconv.ParseUint(r.Version, 10, 64) - if err != nil { - return fmt.Errorf("parse version '%v' failed: %v", - r.Version, err) - } - - // Run update within a transaction - tx := c.recordsdb.Begin() - err = c.updateRecord(tx, convertRecordFromCache(r, v)) - if err != nil { - tx.Rollback() - return err - } - return tx.Commit().Error -} - -// updateRecordStatus updates the status of a record in the database. This -// includes updating the record as well as any metadata streams that are -// associated with the record. The existing metadata streams are deleted from -// the database before the passed in metadata streams are added. -// -// This function must be called within a transaction. -func (c *cockroachdb) updateRecordStatus(tx *gorm.DB, token, version string, status int, timestamp int64, metadata []MetadataStream) error { - log.Tracef("updateRecordStatus: %v %v", token, version) - - // Ensure record exists. We need to do this because updates - // will not return an error if you try to update a record that - // does not exist. - record, err := c.recordVersion(tx, token, version) - if err != nil { - return err - } - - // Update record - err = tx.Model(&record). - Updates(map[string]interface{}{ - "status": status, - "timestamp": timestamp, - }).Error - if err != nil { - return fmt.Errorf("update record: %v", err) - } - - // Update metadata - return updateMetadataStreams(tx, record.Key, metadata) -} - -// UpdateRecordStatus updates the status of a record in the database. This -// includes an update to the record as well as replacing the existing record -// metadata streams with the passed in metadata streams. -func (c *cockroachdb) UpdateRecordStatus(token, version string, status cache.RecordStatusT, timestamp int64, metadata []cache.MetadataStream) error { - log.Tracef("UpdateRecordStatus: %v %v", token, status) - - if c.isShutdown() { - return cache.ErrShutdown - } - - mdStreams := make([]MetadataStream, 0, len(metadata)) - for _, ms := range metadata { - mdStreams = append(mdStreams, convertMDStreamFromCache(ms)) - } - - // Run update within a transaction - tx := c.recordsdb.Begin() - err := c.updateRecordStatus(tx, token, version, int(status), - timestamp, mdStreams) - if err != nil { - tx.Rollback() - return err - } - return tx.Commit().Error -} - -// updateRecordMetadata updates the metadata streams of the given record. It -// does this by first deleting the existing metadata streams then adding the -// passed in metadata streams to the database. -// -// This function must be called using a transaction. -func (c *cockroachdb) updateRecordMetadata(tx *gorm.DB, token string, ms []MetadataStream) error { - // Ensure record exists. This is required because updates - // will not return an error if the record does not exist. - r, err := record(tx, token) - if err != nil { - return err - } - - // Update metadata - err = updateMetadataStreams(tx, r.Key, ms) - if err != nil { - return err - } - - // Call plugin hooks - if c.pluginIsRegistered(decredplugin.ID) { - plugin, err := c.getPlugin(decredplugin.ID) - if err != nil { - return err - } - err = plugin.Hook(tx, pluginHookPostUpdateRecordMetadata, "") - if err != nil { - return err - } - } - - return nil -} - -// UpdateRecordMetadata updates the metadata streams of the given record. It -// does this by first deleting the existing metadata streams then adding the -// passed in metadata streams to the database. -func (c *cockroachdb) UpdateRecordMetadata(token string, ms []cache.MetadataStream) error { - log.Tracef("UpdateRecordMetadata: %v", token) - - if c.isShutdown() { - return cache.ErrShutdown - } - - m := convertMDStreamsFromCache(ms) - - // Run update in a transaction - tx := c.recordsdb.Begin() - err := c.updateRecordMetadata(tx, token, m) - if err != nil { - tx.Rollback() - return err - } - return tx.Commit().Error -} - -// getRecords returns the records for the provided censorship tokens. If a -// record is not found for a provided token, the returned records slice will -// not include an entry for it. -func (c *cockroachdb) getRecords(tokens []string, fetchFiles bool) ([]Record, error) { - // Lookup the latest version of each record specified by - // the provided tokens. - query := `SELECT a.* - FROM records a - LEFT OUTER JOIN records b - ON a.token = b.token - AND a.version < b.version - WHERE b.token IS NULL - AND a.token IN (?)` - rows, err := c.recordsdb.Raw(query, tokens).Rows() - if err != nil { - return nil, err - } - defer rows.Close() - - records := make([]Record, 0, len(tokens)) - for rows.Next() { - var r Record - err := c.recordsdb.ScanRows(rows, &r) - if err != nil { - return nil, err - } - records = append(records, r) - } - if err = rows.Err(); err != nil { - return nil, err - } - - // Compile a list of record primary keys - keys := make([]string, 0, len(records)) - for _, v := range records { - keys = append(keys, v.Key) - } - - if fetchFiles { - // Lookup files and metadata streams for each of the - // previously queried records. - err = c.recordsdb. - Preload("Metadata"). - Preload("Files"). - Where(keys). - Find(&records). - Error - } else { - // Lookup just the metadata streams for each of the - // previously queried records. - err = c.recordsdb. - Preload("Metadata"). - Where(keys). - Find(&records). - Error - } - - return records, err -} - -// Records returns a [token]cache.Record map for the provided censorship -// tokens. If a record is not found, the map will not include an entry for the -// corresponding censorship token. It is the responsibility of the caller to -// ensure that results are returned for all of the provided censorship tokens. -func (c *cockroachdb) Records(tokens []string, fetchFiles bool) (map[string]cache.Record, error) { - log.Tracef("Records: %v", tokens) - - if c.isShutdown() { - return nil, cache.ErrShutdown - } - - records, err := c.getRecords(tokens, fetchFiles) - if err != nil { - return nil, err - } - - // Compile records map - cr := make(map[string]cache.Record, len(records)) // [token]cache.Record - for _, r := range records { - cr[r.Token] = convertRecordToCache(r) - } - - return cr, nil -} - -// inventory returns the latest version of every record in the cache. -func (c *cockroachdb) inventory() ([]Record, error) { - // Lookup the latest version of all records - query := `SELECT a.* - FROM records a - LEFT OUTER JOIN records b - ON a.token = b.token - AND a.version < b.version - WHERE b.token IS NULL` - rows, err := c.recordsdb.Raw(query).Rows() - if err != nil { - return nil, err - } - defer rows.Close() - - records := make([]Record, 0, 1024) // PNOOMA - for rows.Next() { - var r Record - err := c.recordsdb.ScanRows(rows, &r) - if err != nil { - return nil, err - } - records = append(records, r) - } - if err = rows.Err(); err != nil { - return nil, err - } - - // Compile a list of record primary keys - keys := make([]string, 0, len(records)) - for _, v := range records { - keys = append(keys, v.Key) - } - - // Lookup the files and metadata streams for each of the - // previously queried records. - err = c.recordsdb. - Preload("Metadata"). - Preload("Files"). - Where(keys). - Find(&records). - Error - if err != nil { - return nil, err - } - - return records, nil -} - -// Inventory returns the latest version of all records in the cache. -func (c *cockroachdb) Inventory() ([]cache.Record, error) { - log.Tracef("Inventory") - - if c.isShutdown() { - return nil, cache.ErrShutdown - } - - inv, err := c.inventory() - if err != nil { - return nil, err - } - - cr := make([]cache.Record, 0, len(inv)) - for _, v := range inv { - cr = append(cr, convertRecordToCache(v)) - } - - return cr, nil -} - -func (c *cockroachdb) pluginIsRegistered(pluginID string) bool { - c.RLock() - defer c.RUnlock() - - _, ok := c.plugins[pluginID] - return ok -} - -func (c *cockroachdb) getPlugin(id string) (cache.PluginDriver, error) { - c.Lock() - defer c.Unlock() - plugin, ok := c.plugins[id] - if !ok { - return nil, cache.ErrInvalidPlugin - } - return plugin, nil -} - -// PluginExec is a pass through function for plugin commands. -func (c *cockroachdb) PluginExec(pc cache.PluginCommand) (*cache.PluginCommandReply, error) { - log.Tracef("PluginExec: %v", pc.ID) - - if c.isShutdown() { - return nil, cache.ErrShutdown - } - - plugin, err := c.getPlugin(pc.ID) - if err != nil { - return nil, err - } - - payload, err := plugin.Exec(pc.Command, pc.CommandPayload, - pc.ReplyPayload) - if err != nil { - return nil, err - } - - return &cache.PluginCommandReply{ - ID: pc.ID, - Command: pc.Command, - Payload: payload, - }, nil -} - -// PluginSetup sets up the database tables for the passed in plugin. -func (c *cockroachdb) PluginSetup(id string) error { - log.Tracef("PluginSetup: %v", id) - - if c.isShutdown() { - return cache.ErrShutdown - } - - plugin, err := c.getPlugin(id) - if err != nil { - return err - } - - return plugin.Setup() -} - -// RegisterPlugin registers and plugin with the cache and checks to make sure -// that the cache is using the correct plugin version. -func (c *cockroachdb) RegisterPlugin(p cache.Plugin) error { - log.Tracef("RegisterPlugin: %v", p.ID) - - c.Lock() - defer c.Unlock() - - if c.shutdown { - return cache.ErrShutdown - } - - _, ok := c.plugins[p.ID] - if ok { - return cache.ErrDuplicatePlugin - } - - // Register the plugin - var pd cache.PluginDriver - switch p.ID { - case decredplugin.ID: - pd = newDecredPlugin(c.recordsdb, p) - c.plugins[decredplugin.ID] = pd - case cmsplugin.ID: - pd = newCMSPlugin(c.recordsdb, p) - c.plugins[cmsplugin.ID] = pd - default: - return cache.ErrInvalidPlugin - } - - // Ensure we're using the correct plugin version - return pd.CheckVersion() -} - -// PluginBuilds builds the cache for the passed in plugin. -func (c *cockroachdb) PluginBuild(id, payload string) error { - log.Tracef("PluginBuild: %v", id) - - if c.isShutdown() { - return cache.ErrShutdown - } - - plugin, err := c.getPlugin(id) - if err != nil { - return err - } - - log.Infof("Building plugin cache: %v", id) - - return plugin.Build(payload) -} - -// createTables creates the database tables if they do not already exist. A -// version record for the cache is inserted into the database during this -// process if one does not already exist. -// -// This function must be called within a transaction. -func (c *cockroachdb) createTables(tx *gorm.DB) error { - log.Tracef("createTables") - - if !tx.HasTable(tableKeyValue) { - err := tx.CreateTable(&KeyValue{}).Error - if err != nil { - return err - } - } - if !tx.HasTable(tableVersions) { - err := tx.CreateTable(&Version{}).Error - if err != nil { - return err - } - } - if !tx.HasTable(tableRecords) { - err := tx.CreateTable(&Record{}).Error - if err != nil { - return err - } - } - if !tx.HasTable(tableMetadataStreams) { - err := tx.CreateTable(&MetadataStream{}).Error - if err != nil { - return err - } - } - if !tx.HasTable(tableFiles) { - err := tx.CreateTable(&File{}).Error - if err != nil { - return err - } - } - - var v Version - err := tx.Where("id = ?", cacheID). - Find(&v). - Error - if err == gorm.ErrRecordNotFound { - err = tx.Create( - &Version{ - ID: cacheID, - Version: cacheVersion, - Timestamp: time.Now().Unix(), - }).Error - return err - } - - // Insert initial best block if record is not found. - kv := KeyValue{ - Key: keyBestBlock, - } - err = tx.Find(&kv).Error - if err != nil { - if err == gorm.ErrRecordNotFound { - b := make([]byte, 8) - binary.LittleEndian.PutUint64(b, 0) - kv := KeyValue{ - Key: keyBestBlock, - Value: b, - } - err = tx.Save(&kv).Error - } - } - - return err -} - -func (c *cockroachdb) dropTables(tx *gorm.DB) error { - // Drop record tables - err := tx.DropTableIfExists(tableRecords, - tableMetadataStreams, tableFiles).Error - if err != nil { - return err - } - - // Remove cache version record - return tx.Delete(&Version{ - ID: cacheID, - }).Error -} - -// Setup creates the database tables for the records cache if they do not -// already exist. A version record is inserted into the database during table -// creation. -func (c *cockroachdb) Setup() error { - log.Tracef("Setup tables") - - c.Lock() - defer c.Unlock() - - if c.shutdown { - return cache.ErrShutdown - } - - tx := c.recordsdb.Begin() - err := c.createTables(tx) - if err != nil { - tx.Rollback() - return err - } - - return tx.Commit().Error -} - -// build the records cache using the passed in records. -// -// This function cannot be called using a transaction because it could -// potentially exceed cockroachdb's transaction size limit. -func (c *cockroachdb) build(records []Record) error { - log.Tracef("build") - - // Drop record tables - tx := c.recordsdb.Begin() - err := c.dropTables(tx) - if err != nil { - tx.Rollback() - return fmt.Errorf("drop tables: %v", err) - } - err = tx.Commit().Error - if err != nil { - return err - } - - // Create record tables - tx = c.recordsdb.Begin() - err = c.createTables(tx) - if err != nil { - tx.Rollback() - return fmt.Errorf("create tables: %v", err) - } - err = tx.Commit().Error - if err != nil { - return err - } - - // Populate record tables - for _, r := range records { - err := c.recordsdb.Create(&r).Error - if err != nil { - log.Debugf("create record failed on '%v'", r) - return fmt.Errorf("create record: %v", err) - } - } - - return nil -} - -// Build drops all existing tables from the records cache, recreates them, then -// builds the records cache using the passed in records. -func (c *cockroachdb) Build(records []cache.Record) error { - log.Tracef("Build") - - c.Lock() - defer c.Unlock() - - if c.shutdown { - return cache.ErrShutdown - } - - log.Infof("Building records cache") - - r := make([]Record, 0, len(records)) - for _, cr := range records { - v, err := strconv.ParseUint(cr.Version, 10, 64) - if err != nil { - return fmt.Errorf("parse version '%v' failed %v: %v", - cr.Version, cr.CensorshipRecord.Token, err) - } - r = append(r, convertRecordFromCache(cr, v)) - } - - // Build the records cache. This is not run using a - // transaction because it could potentially exceed - // cockroachdb's transaction size limit. - err := c.build(r) - if err != nil { - // Remove the version record. This will - // force a rebuild on the next start up. - err1 := c.recordsdb.Delete(&Version{ - ID: cacheID, - }).Error - if err1 != nil { - panic("the cache is out of sync and will not rebuild" + - "automatically; a rebuild must be forced") - } - } - - return err -} - -// Close shuts down the cache. All interface functions MUST return with -// errShutdown if the backend is shutting down. -func (c *cockroachdb) Close() { - log.Tracef("Close") - - c.Lock() - defer c.Unlock() - - c.shutdown = true - c.recordsdb.Close() -} - -func buildQueryString(user, rootCert, cert, key string) string { - v := url.Values{} - v.Set("sslmode", "require") - v.Set("sslrootcert", filepath.Clean(rootCert)) - v.Set("sslcert", filepath.Join(cert)) - v.Set("sslkey", filepath.Join(key)) - return v.Encode() -} - -// New returns a new cockroachdb context that contains a connection to the -// specified database that was made using the passed in user and certificates. -func New(user, host, net, rootCert, cert, key string) (*cockroachdb, error) { - log.Tracef("New: %v %v %v %v %v %v", user, host, net, rootCert, cert, key) - - // Connect to database - dbName := cacheID + "_" + net - h := "postgresql://" + user + "@" + host + "/" + dbName - u, err := url.Parse(h) - if err != nil { - return nil, fmt.Errorf("parse url '%v': %v", h, err) - } - - qs := buildQueryString(u.User.String(), rootCert, cert, key) - addr := u.String() + "?" + qs - db, err := gorm.Open("postgres", addr) - if err != nil { - return nil, fmt.Errorf("connect to database '%v': %v", addr, err) - } - - // Create context - c := &cockroachdb{ - recordsdb: db, - plugins: make(map[string]cache.PluginDriver), - } - - // Disable gorm logging. This prevents duplicate errors from - // being printed since we handle errors manually. - c.recordsdb.LogMode(false) - - // Disable automatic table name pluralization. We set table - // names manually. - c.recordsdb.SingularTable(true) - - log.Infof("Cache host: %v", h) - - // Return an error if the version record is not found or - // if there is a version mismatch, but also return the - // cache context so that the cache can be built/rebuilt. - if !c.recordsdb.HasTable(tableVersions) { - log.Debugf("table '%v' does not exist", tableVersions) - return c, cache.ErrNoVersionRecord - } - - var v Version - err = c.recordsdb. - Where("id = ?", cacheID). - Find(&v). - Error - if err == gorm.ErrRecordNotFound { - log.Debugf("version record not found for ID '%v'", cacheID) - err = cache.ErrNoVersionRecord - } else if v.Version != cacheVersion { - log.Debugf("version mismatch for ID '%v': got %v, want %v", - cacheID, v.Version, cacheVersion) - err = cache.ErrWrongVersion - } - - return c, err -} diff --git a/politeiad/cache/cockroachdb/convert.go b/politeiad/cache/cockroachdb/convert.go deleted file mode 100644 index 90e038e60..000000000 --- a/politeiad/cache/cockroachdb/convert.go +++ /dev/null @@ -1,524 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package cockroachdb - -import ( - "fmt" - "strconv" - "strings" - - "github.com/decred/politeia/cmsplugin" - "github.com/decred/politeia/decredplugin" - "github.com/decred/politeia/politeiad/cache" - "github.com/decred/politeia/util" -) - -func convertMDStreamFromCache(ms cache.MetadataStream) MetadataStream { - return MetadataStream{ - ID: ms.ID, - Payload: ms.Payload, - } -} - -func convertMDStreamsFromCache(ms []cache.MetadataStream) []MetadataStream { - m := make([]MetadataStream, 0, len(ms)) - for _, v := range ms { - m = append(m, convertMDStreamFromCache(v)) - } - return m -} - -func convertRecordFromCache(r cache.Record, version uint64) Record { - files := make([]File, 0, len(r.Files)) - for _, f := range r.Files { - files = append(files, - File{ - Name: f.Name, - MIME: f.MIME, - Digest: f.Digest, - Payload: f.Payload, - }) - } - - return Record{ - Key: r.CensorshipRecord.Token + r.Version, - Token: r.CensorshipRecord.Token, - Version: version, - Status: int(r.Status), - Timestamp: r.Timestamp, - Merkle: r.CensorshipRecord.Merkle, - Signature: r.CensorshipRecord.Signature, - Metadata: convertMDStreamsFromCache(r.Metadata), - Files: files, - TokenPrefix: util.TokenToPrefix(r.CensorshipRecord.Token), - } -} - -func convertRecordToCache(r Record) cache.Record { - cr := cache.CensorshipRecord{ - Token: r.Token, - Merkle: r.Merkle, - Signature: r.Signature, - } - - metadata := make([]cache.MetadataStream, 0, len(r.Metadata)) - for _, ms := range r.Metadata { - metadata = append(metadata, - cache.MetadataStream{ - ID: ms.ID, - Payload: ms.Payload, - }) - } - - files := make([]cache.File, 0, len(r.Files)) - for _, f := range r.Files { - files = append(files, - cache.File{ - Name: f.Name, - MIME: f.MIME, - Digest: f.Digest, - Payload: f.Payload, - }) - } - - return cache.Record{ - Version: strconv.FormatUint(r.Version, 10), - Status: cache.RecordStatusT(r.Status), - Timestamp: r.Timestamp, - CensorshipRecord: cr, - Metadata: metadata, - Files: files, - } -} - -func convertNewCommentFromDecred(nc decredplugin.NewComment, ncr decredplugin.NewCommentReply) Comment { - return Comment{ - Key: nc.Token + ncr.CommentID, - Token: nc.Token, - ParentID: nc.ParentID, - Comment: nc.Comment, - Signature: nc.Signature, - PublicKey: nc.PublicKey, - CommentID: ncr.CommentID, - Receipt: ncr.Receipt, - Timestamp: ncr.Timestamp, - Censored: false, - } -} - -func convertCommentFromDecred(c decredplugin.Comment) Comment { - return Comment{ - Key: c.Token + c.CommentID, - Token: c.Token, - ParentID: c.ParentID, - Comment: c.Comment, - Signature: c.Signature, - PublicKey: c.PublicKey, - CommentID: c.CommentID, - Receipt: c.Receipt, - Timestamp: c.Timestamp, - Censored: false, - } -} - -func convertCommentToDecred(c Comment) decredplugin.Comment { - return decredplugin.Comment{ - Token: c.Token, - ParentID: c.ParentID, - Comment: c.Comment, - Signature: c.Signature, - PublicKey: c.PublicKey, - CommentID: c.CommentID, - Receipt: c.Receipt, - Timestamp: c.Timestamp, - TotalVotes: 0, - ResultVotes: 0, - Censored: c.Censored, - } -} - -func convertLikeCommentFromDecred(lc decredplugin.LikeComment) LikeComment { - return LikeComment{ - Token: lc.Token, - CommentID: lc.CommentID, - Action: lc.Action, - Signature: lc.Signature, - PublicKey: lc.PublicKey, - } -} - -func convertLikeCommentToDecred(lc LikeComment) decredplugin.LikeComment { - return decredplugin.LikeComment{ - Token: lc.Token, - CommentID: lc.CommentID, - Action: lc.Action, - Signature: lc.Signature, - PublicKey: lc.PublicKey, - } -} - -func convertAuthorizeVoteFromDecred(av decredplugin.AuthorizeVote, avr decredplugin.AuthorizeVoteReply) (*AuthorizeVote, error) { - version, err := strconv.ParseUint(avr.RecordVersion, 10, 64) - if err != nil { - return nil, fmt.Errorf("parse version '%v' failed: %v", - avr.RecordVersion, err) - } - - return &AuthorizeVote{ - Key: av.Token + avr.RecordVersion, - Token: av.Token, - Version: version, - Action: av.Action, - Signature: av.Signature, - PublicKey: av.PublicKey, - Receipt: avr.Receipt, - Timestamp: avr.Timestamp, - }, nil -} - -func convertAuthorizeVoteToDecred(av AuthorizeVote) decredplugin.AuthorizeVote { - return decredplugin.AuthorizeVote{ - Action: av.Action, - Token: av.Token, - Signature: av.Signature, - PublicKey: av.PublicKey, - Receipt: av.Receipt, - Timestamp: av.Timestamp, - } -} - -func convertStartVoteV1FromDecred(sv decredplugin.StartVoteV1, svr decredplugin.StartVoteReply) (*StartVote, error) { - opts := make([]VoteOption, 0, len(sv.Vote.Options)) - for _, v := range sv.Vote.Options { - opts = append(opts, VoteOption{ - Token: sv.Vote.Token, - ID: v.Id, - Description: v.Description, - Bits: v.Bits, - }) - } - startHeight, err := strconv.ParseUint(svr.StartBlockHeight, 10, 32) - if err != nil { - return nil, fmt.Errorf("parse start height '%v': %v", - svr.StartBlockHeight, err) - } - endHeight, err := strconv.ParseUint(svr.EndHeight, 10, 32) - if err != nil { - return nil, fmt.Errorf("parse end height '%v': %v", - svr.EndHeight, err) - } - return &StartVote{ - Token: sv.Vote.Token, - Version: sv.Version, - Type: int(decredplugin.VoteTypeStandard), - Mask: sv.Vote.Mask, - Duration: sv.Vote.Duration, - QuorumPercentage: sv.Vote.QuorumPercentage, - PassPercentage: sv.Vote.PassPercentage, - Options: opts, - PublicKey: sv.PublicKey, - Signature: sv.Signature, - StartBlockHeight: uint32(startHeight), - StartBlockHash: svr.StartBlockHash, - EndHeight: uint32(endHeight), - EligibleTickets: strings.Join(svr.EligibleTickets, ","), - EligibleTicketCount: len(svr.EligibleTickets), - }, nil -} - -func convertStartVoteV2FromDecred(sv decredplugin.StartVoteV2, svr decredplugin.StartVoteReply) (*StartVote, error) { - opts := make([]VoteOption, 0, len(sv.Vote.Options)) - for _, v := range sv.Vote.Options { - opts = append(opts, VoteOption{ - Token: sv.Vote.Token, - ID: v.Id, - Description: v.Description, - Bits: v.Bits, - }) - } - startHeight, err := strconv.ParseUint(svr.StartBlockHeight, 10, 32) - if err != nil { - return nil, fmt.Errorf("parse start height '%v': %v", - svr.StartBlockHeight, err) - } - endHeight, err := strconv.ParseUint(svr.EndHeight, 10, 32) - if err != nil { - return nil, fmt.Errorf("parse end height '%v': %v", - svr.EndHeight, err) - } - // The version must be pulled from decredplugin because the version - // is filled in by the politeiad backend and does not travel to the - // cache. If the cache is being built from scratch the version will - // be present since the data is being read directly from disk. - return &StartVote{ - Token: sv.Vote.Token, - Version: decredplugin.VersionStartVoteV2, - ProposalVersion: sv.Vote.ProposalVersion, - Type: int(sv.Vote.Type), - Mask: sv.Vote.Mask, - Duration: sv.Vote.Duration, - QuorumPercentage: sv.Vote.QuorumPercentage, - PassPercentage: sv.Vote.PassPercentage, - Options: opts, - PublicKey: sv.PublicKey, - Signature: sv.Signature, - StartBlockHeight: uint32(startHeight), - StartBlockHash: svr.StartBlockHash, - EndHeight: uint32(endHeight), - EligibleTickets: strings.Join(svr.EligibleTickets, ","), - EligibleTicketCount: len(svr.EligibleTickets), - }, nil -} - -func convertStartVoteToDecredV1(sv StartVote) (*decredplugin.StartVote, error) { - opts := make([]decredplugin.VoteOption, 0, len(sv.Options)) - for _, v := range sv.Options { - opts = append(opts, decredplugin.VoteOption{ - Id: v.ID, - Description: v.Description, - Bits: v.Bits, - }) - } - dsv := decredplugin.StartVoteV1{ - Version: sv.Version, - PublicKey: sv.PublicKey, - Vote: decredplugin.VoteV1{ - Token: sv.Token, - Mask: sv.Mask, - Duration: sv.Duration, - QuorumPercentage: sv.QuorumPercentage, - PassPercentage: sv.PassPercentage, - Options: opts, - }, - Signature: sv.Signature, - } - svb, err := decredplugin.EncodeStartVoteV1(dsv) - if err != nil { - return nil, err - } - return &decredplugin.StartVote{ - Token: sv.Token, - Version: sv.Version, - Payload: string(svb), - }, nil -} - -func convertStartVoteToDecredV2(sv StartVote) (*decredplugin.StartVote, error) { - opts := make([]decredplugin.VoteOption, 0, len(sv.Options)) - for _, v := range sv.Options { - opts = append(opts, decredplugin.VoteOption{ - Id: v.ID, - Description: v.Description, - Bits: v.Bits, - }) - } - dsv := decredplugin.StartVoteV2{ - Version: sv.Version, - PublicKey: sv.PublicKey, - Vote: decredplugin.VoteV2{ - Token: sv.Token, - ProposalVersion: sv.ProposalVersion, - Type: decredplugin.VoteT(sv.Type), - Mask: sv.Mask, - Duration: sv.Duration, - QuorumPercentage: sv.QuorumPercentage, - PassPercentage: sv.PassPercentage, - Options: opts, - }, - Signature: sv.Signature, - } - svb, err := decredplugin.EncodeStartVoteV2(dsv) - if err != nil { - return nil, err - } - return &decredplugin.StartVote{ - Token: sv.Token, - Version: sv.Version, - Payload: string(svb), - }, nil -} - -func convertStartVoteToDecred(sv StartVote) (*decredplugin.StartVote, *decredplugin.StartVoteReply, error) { - var ( - dsv *decredplugin.StartVote - err error - ) - switch sv.Version { - case decredplugin.VersionStartVoteV1: - dsv, err = convertStartVoteToDecredV1(sv) - if err != nil { - return nil, nil, err - } - case decredplugin.VersionStartVoteV2: - dsv, err = convertStartVoteToDecredV2(sv) - if err != nil { - return nil, nil, err - } - default: - return nil, nil, fmt.Errorf("invalid StartVote version %v %v", - sv.Token, sv.Version) - } - - var tix []string - if sv.EligibleTickets != "" { - tix = strings.Split(sv.EligibleTickets, ",") - } - dsvr := &decredplugin.StartVoteReply{ - StartBlockHeight: strconv.FormatUint(uint64(sv.StartBlockHeight), 10), - StartBlockHash: sv.StartBlockHash, - EndHeight: strconv.FormatUint(uint64(sv.EndHeight), 10), - EligibleTickets: tix, - } - - return dsv, dsvr, nil -} - -func convertCastVoteFromDecred(cv decredplugin.CastVote) CastVote { - return CastVote{ - Token: cv.Token, - Ticket: cv.Ticket, - VoteBit: cv.VoteBit, - Signature: cv.Signature, - TokenVoteBit: cv.Token + cv.VoteBit, - } -} - -func convertCastVoteToDecred(cv CastVote) decredplugin.CastVote { - return decredplugin.CastVote{ - Token: cv.Token, - Ticket: cv.Ticket, - VoteBit: cv.VoteBit, - Signature: cv.Signature, - } -} - -func convertVoteOptionResultToDecred(r VoteOptionResult) decredplugin.VoteOptionResult { - return decredplugin.VoteOptionResult{ - ID: r.Option.ID, - Description: r.Option.Description, - Bits: r.Option.Bits, - Votes: r.Votes, - } -} - -func convertVoteOptionResultsToDecred(r []VoteOptionResult) []decredplugin.VoteOptionResult { - results := make([]decredplugin.VoteOptionResult, 0, len(r)) - for _, v := range r { - results = append(results, convertVoteOptionResultToDecred(v)) - } - return results -} - -func convertStartVoteFromCMS(sv cmsplugin.StartVote, svr cmsplugin.StartVoteReply, endHeight uint32) StartDCCVote { - opts := make([]VoteDCCOption, 0, len(sv.Vote.Options)) - for _, v := range sv.Vote.Options { - opts = append(opts, VoteDCCOption{ - Token: sv.Vote.Token, - ID: v.Id, - Description: v.Description, - Bits: v.Bits, - }) - } - weights := make([]DCCUserWeight, 0, len(sv.UserWeights)) - for _, v := range sv.UserWeights { - weights = append(weights, DCCUserWeight{ - Key: sv.Vote.Token + "-" + v.UserID, - Token: sv.Vote.Token, - UserID: v.UserID, - Weight: v.Weight, - }) - } - return StartDCCVote{ - Token: sv.Vote.Token, - Mask: sv.Vote.Mask, - Duration: sv.Vote.Duration, - QuorumPercentage: sv.Vote.QuorumPercentage, - PassPercentage: sv.Vote.PassPercentage, - Options: opts, - PublicKey: sv.PublicKey, - Signature: sv.Signature, - StartBlockHeight: svr.StartBlockHeight, - StartBlockHash: svr.StartBlockHash, - EndHeight: endHeight, - EligibleUserIDs: weights, - } -} - -func convertStartVoteToCMS(sv StartDCCVote) (cmsplugin.StartVote, cmsplugin.StartVoteReply) { - opts := make([]cmsplugin.VoteOption, 0, len(sv.Options)) - for _, v := range sv.Options { - opts = append(opts, cmsplugin.VoteOption{ - Id: v.ID, - Description: v.Description, - Bits: v.Bits, - }) - } - - weights := make([]cmsplugin.UserWeight, 0, len(sv.EligibleUserIDs)) - for _, v := range sv.EligibleUserIDs { - weights = append(weights, cmsplugin.UserWeight{ - UserID: v.UserID, - Weight: v.Weight, - }) - } - - dsv := cmsplugin.StartVote{ - PublicKey: sv.PublicKey, - Signature: sv.Signature, - Vote: cmsplugin.Vote{ - Token: sv.Token, - Mask: sv.Mask, - Duration: sv.Duration, - QuorumPercentage: sv.QuorumPercentage, - PassPercentage: sv.PassPercentage, - Options: opts, - }, - UserWeights: weights, - } - - dsvr := cmsplugin.StartVoteReply{ - StartBlockHeight: sv.StartBlockHeight, - StartBlockHash: sv.StartBlockHash, - EndHeight: sv.EndHeight, - } - - return dsv, dsvr -} - -func convertCastVoteFromCMS(cv cmsplugin.CastVote) CastDCCVote { - return CastDCCVote{ - Token: cv.Token, - UserID: cv.UserID, - VoteBit: cv.VoteBit, - Signature: cv.Signature, - TokenVoteBit: cv.Token + cv.VoteBit, - } -} - -func convertCastVoteToCMS(cv CastDCCVote) cmsplugin.CastVote { - return cmsplugin.CastVote{ - Token: cv.Token, - UserID: cv.UserID, - VoteBit: cv.VoteBit, - Signature: cv.Signature, - } -} - -func convertVoteOptionResultToCMS(r VoteOptionResult) cmsplugin.VoteOptionResult { - return cmsplugin.VoteOptionResult{ - ID: r.Option.ID, - Description: r.Option.Description, - Bits: r.Option.Bits, - Votes: r.Votes, - } -} - -func convertVoteOptionResultsToCMS(r []VoteOptionResult) []cmsplugin.VoteOptionResult { - results := make([]cmsplugin.VoteOptionResult, 0, len(r)) - for _, v := range r { - results = append(results, convertVoteOptionResultToCMS(v)) - } - return results -} diff --git a/politeiad/cache/cockroachdb/decred.go b/politeiad/cache/cockroachdb/decred.go deleted file mode 100644 index fc0c5a309..000000000 --- a/politeiad/cache/cockroachdb/decred.go +++ /dev/null @@ -1,2310 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package cockroachdb - -import ( - "encoding/base64" - "encoding/binary" - "encoding/json" - "fmt" - "strconv" - "strings" - "sync" - "time" - - "github.com/decred/politeia/decredplugin" - "github.com/decred/politeia/mdstream" - pd "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/politeiad/cache" - "github.com/jinzhu/gorm" -) - -const ( - // decredVersion is the version of the cache implementation of - // decred plugin. This may differ from the decredplugin package - // version. - decredVersion = "1.2" - - // Decred plugin table names - tableProposalMetadata = "proposal_metadata" - tableComments = "comments" - tableCommentLikes = "comment_likes" - tableCastVotes = "cast_votes" - tableAuthorizeVotes = "authorize_votes" - tableVoteOptions = "vote_options" - tableStartVotes = "start_votes" - tableVoteOptionResults = "vote_option_results" - tableVoteResults = "vote_results" - - // Vote option IDs - voteOptionIDApproved = "yes" -) - -// decred implements the PluginDriver interface. -type decred struct { - sync.RWMutex - recordsdb *gorm.DB // Database context - version string // Version of decred cache plugin - settings []cache.PluginSetting // Plugin settings -} - -// publicStatuses returns an array of ints that represents all the publicly -// viewable politeiad record statuses. The statuses are returned as ints so -// that they can be inserted into queries. -func publicStatuses() []int { - return []int{ - int(pd.RecordStatusPublic), - int(pd.RecordStatusArchived), - } -} - -// bestBlockSet sets the best block used to update the VoteResults table -// in the key-value store. -func (d *decred) bestBlockSet(bb uint64) error { - b := make([]byte, binary.MaxVarintLen64) - binary.LittleEndian.PutUint64(b, bb) - kv := KeyValue{ - Key: keyBestBlock, - Value: b, - } - return d.recordsdb.Save(&kv).Error -} - -// BestBlockGet gets the best block used to update the VoteResults table -// from the key-value store. -func (d *decred) bestBlockGet() (uint64, error) { - kv := KeyValue{ - Key: keyBestBlock, - } - err := d.recordsdb.Find(&kv).Error - if err != nil { - return 0, fmt.Errorf("bestBlockGetError: %v", err) - } - return binary.LittleEndian.Uint64(kv.Value), nil -} - -// authorizeVotes returns a map[token]AuthorizeVote for the given records. An -// entry in the returned map will only exist if an AuthorizeVote is found for -// the record. -func (d *decred) authorizeVotes(records []Record) (map[string]AuthorizeVote, error) { - authorizeVotes := make(map[string]AuthorizeVote, len(records)) - - if len(records) == 0 { - return authorizeVotes, nil - } - - keys := make([]string, 0, len(records)) - for _, v := range records { - keys = append(keys, v.Token+strconv.FormatUint(v.Version, 10)) - } - - avs := make([]AuthorizeVote, 0, len(keys)) - err := d.recordsdb. - Where("key IN (?)", keys). - Find(&avs). - Error - if err != nil { - return nil, fmt.Errorf("select AuthorizeVotes in %v: %v", - keys, err) - } - - for _, av := range avs { - authorizeVotes[av.Token] = av - } - - return authorizeVotes, nil -} - -// startVotes returns a map[token]StartVote for the given tokens. An entry in -// the returned map will only exist for tokens where a StartVote was found. -func (d *decred) startVotes(tokens []string) (map[string]StartVote, error) { - startVotes := make(map[string]StartVote, len(tokens)) - - if len(tokens) == 0 { - return startVotes, nil - } - - svs := make([]StartVote, 0, len(tokens)) - err := d.recordsdb. - Where("token IN (?)", tokens). - Preload("Options"). - Find(&svs). - Error - if err != nil { - return nil, fmt.Errorf("select StartVotes in %v: %v", - tokens, err) - } - - for _, v := range svs { - startVotes[v.Token] = v - } - - return startVotes, nil -} - -// voteOptionResults returns the VoteOptionResult for each of the given -// VoteOptions. The results are looked up manually from the CastVotes table -// instead of using the VoteResults table. This allows results to be looked up -// when a VoteResults entry does not yet exist, such as for ongoing votes. -func (d *decred) voteOptionResults(options []VoteOption) ([]VoteOptionResult, error) { - results := make([]VoteOptionResult, 0, len(options)) - - for _, v := range options { - var votes uint64 - tokenVoteBit := v.Token + strconv.FormatUint(v.Bits, 16) - err := d.recordsdb. - Model(&CastVote{}). - Where("token_vote_bit = ?", tokenVoteBit). - Count(&votes). - Error - if err != nil { - return nil, fmt.Errorf("select CastVote count for %v: %v", - tokenVoteBit, err) - } - - results = append(results, VoteOptionResult{ - Key: tokenVoteBit, - Token: v.Token, - Votes: votes, - Option: VoteOption{ - ID: v.ID, - Description: v.Description, - Bits: v.Bits, - }, - }) - } - - return results, nil -} - -// voteResultsMissing returns the tokens of the standard proposal votes and -// runoff proposal votes that have finished voting but are missing from the -// lazy loaded VoteResults table. -func (d *decred) voteResultsMissing(bestBlock uint64) ([]string, []string, error) { - // Check if the vote results table has already been built for this - // block. If so, there is no need to run these queries. - bb, err := d.bestBlockGet() - if err != nil { - return nil, nil, err - } - if bb >= bestBlock { - log.Debugf("voteResultsMissing: table already updated for block "+ - "%v", bestBlock) - return []string{}, []string{}, nil - } - - log.Debugf("voteResultsMissing: checking missing results for block: %v. "+ - "Cached block: %v", bestBlock, bb) - - // Find standard vote proposals that have finished voting but - // have not yet been added to the VoteResults table. - q := `SELECT start_votes.token - FROM start_votes - LEFT OUTER JOIN vote_results - ON start_votes.token = vote_results.token - WHERE start_votes.end_height <= ? - AND start_votes.Type = ? - AND vote_results.token IS NULL` - rows, err := d.recordsdb.Raw(q, bestBlock, - int(decredplugin.VoteTypeStandard)).Rows() - if err != nil { - return nil, nil, fmt.Errorf("lookup missing standard vote results: %v", - err) - } - defer rows.Close() - - standard := make([]string, 0, 1024) - for rows.Next() { - var token string - rows.Scan(&token) - standard = append(standard, token) - } - - // Find runoff vote proposals that have finished voting but - // have not yet been added to the VoteResults table. - q = `SELECT start_votes.token - FROM start_votes - LEFT OUTER JOIN vote_results - ON start_votes.token = vote_results.token - WHERE start_votes.end_height <= ? - AND start_votes.Type = ? - AND vote_results.token IS NULL` - rows, err = d.recordsdb.Raw(q, bestBlock, - int(decredplugin.VoteTypeRunoff)).Rows() - if err != nil { - return nil, nil, fmt.Errorf("lookup missing runoff vote results: %v", - err) - } - defer rows.Close() - - runoff := make([]string, 0, 1024) - for rows.Next() { - var token string - rows.Scan(&token) - runoff = append(runoff, token) - } - - log.Debugf("voteResultsMissing: found %v standard and %v runoff "+ - "proposals", len(standard), len(runoff)) - - return standard, runoff, nil -} - -// voteResultsCompile compiles the results of a standard vote and returns a -// VoteResults. The vote is considered approved if the quorum and pass -// percentage requirements are met. -func voteResultsCompile(sv StartVote, votes []CastVote) VoteResults { - // Tally cast votes - tally := make(map[string]uint64) // [voteBit]voteCount - for _, v := range votes { - tally[v.VoteBit]++ - } - - // Prepare vote option results - results := make([]VoteOptionResult, 0, len(sv.Options)) - for _, v := range sv.Options { - voteBit := strconv.FormatUint(v.Bits, 16) - voteCount := tally[voteBit] - - results = append(results, VoteOptionResult{ - Key: sv.Token + voteBit, - Votes: voteCount, - Option: v, - }) - } - - // Check whether vote was approved - var total uint64 - for _, v := range results { - total += v.Votes - } - - eligible := len(strings.Split(sv.EligibleTickets, ",")) - quorum := uint64(float64(sv.QuorumPercentage) / 100 * float64(eligible)) - pass := uint64(float64(sv.PassPercentage) / 100 * float64(total)) - - // XXX this does not support multiple choice votes yet - var ( - approvedVotes uint64 // Number of approve votes - approved bool // Is the proposal vote approved - ) - for _, v := range results { - if v.Option.ID == decredplugin.VoteOptionIDApprove { - approvedVotes = v.Votes - } - } - switch { - case total < quorum: - // Quorum not met - case approvedVotes < pass: - // Pass percentage not met - default: - // Vote was approved - approved = true - } - - return VoteResults{ - Token: sv.Token, - Approved: approved, - Results: results, - } -} - -// voteResultsInsertStandard calculates the vote results for a standard -// proposal vote and inserts a VoteResults record into the cache. A VoteResults -// record should only be created for proposals once the voting period has -// ended. -func (d *decred) voteResultsInsertStandard(token string) error { - log.Tracef("insertVoteResults %v", token) - - // Lookup start vote - var sv StartVote - err := d.recordsdb. - Where("token = ?", token). - Preload("Options"). - Find(&sv). - Error - if err != nil { - return fmt.Errorf("lookup start vote %v: %v", - token, err) - } - - // Lookup cast votes - var cv []CastVote - err = d.recordsdb. - Where("token = ?", token). - Find(&cv). - Error - if err == gorm.ErrRecordNotFound { - // No cast votes exists. In theory, this could - // happen if no one were to vote on a proposal. - // In practice, this shouldn't happen. - } else if err != nil { - return fmt.Errorf("lookup cast votes: %v", err) - } - - // Create a vote results entry - vr := voteResultsCompile(sv, cv) - err = d.recordsdb.Create(&vr).Error - if err != nil { - return fmt.Errorf("insert vote results: %v", err) - } - - log.Debugf("Standard vote result created %v", vr.Token) - - return nil -} - -// voteResultsInsertRunoff calculates the results of a runoff vote and inserts -// a VoteResults record into the cache for each of the runoff vote submissions. -func (d *decred) voteResultsInsertRunoff(rfpToken string) error { - log.Tracef("voteResultsInsertRunoffVote: %v", rfpToken) - - linkedFrom, err := d.linkedFrom(rfpToken) - if err != nil { - return err - } - - // Compile vote results for all RFP submissions - results := make([]VoteResults, 0, len(linkedFrom)) - for _, token := range linkedFrom { - // Make sure record hasn't been abandoned - var r Record - err = d.recordsdb. - Where("records.token = ?", token). - Order("records.version desc"). - Limit(1). - Preload("Metadata"). - Find(&r). - Error - if err != nil { - return fmt.Errorf("lookup record %v: %v", - token, err) - } - if r.Status == int(pd.RecordStatusArchived) { - // RFP submission has been abandoned - continue - } - - // Lookup start vote - var sv StartVote - err := d.recordsdb. - Where("token = ?", token). - Preload("Options"). - Find(&sv). - Error - if err != nil { - return fmt.Errorf("find start vote %v: %v", - token, err) - } - - // Lookup cast votes - var cv []CastVote - err = d.recordsdb. - Where("token = ?", token). - Find(&cv). - Error - if err == gorm.ErrRecordNotFound { - // No cast votes exists. In theory, this could - // happen if no one were to vote on a proposal. - // In practice, this shouldn't happen. - } else if err != nil { - return fmt.Errorf("lookup cast votes: %v", err) - } - - // Compile vote results. This will mark the vote as - // approved if it meets the specified quorum and pass - // requirements. The actual runoff vote winner is - // determined later in this function. - results = append(results, voteResultsCompile(sv, cv)) - } - - // Determine runoff vote winner. The winner is the vote - // that passed the quorum and pass requirments and has - // the most net approved votes. - var ( - winnerNetApprove int // Net number of approve votes of the winner - winnerToken string // Censorship token of the winner - ) - for _, vr := range results { - if !vr.Approved { - // Vote did not meet quorum and pass requirements - continue - } - - // Check if this proposal has more net approved votes - // then the current highest. - var ( - votesApprove uint64 // Number of approve votes - votesReject uint64 // Number of reject votes - ) - for _, vor := range vr.Results { - switch vor.Option.ID { - case decredplugin.VoteOptionIDApprove: - votesApprove = vor.Votes - case decredplugin.VoteOptionIDReject: - votesReject = vor.Votes - default: - // Runoff vote options can only be approve/reject - return fmt.Errorf("unknown runoff vote option %v %v", - vr.Token, vor.Option.ID) - } - - netApprove := int(votesApprove) - int(votesReject) - if netApprove > winnerNetApprove { - // New winner! - winnerToken = vr.Token - winnerNetApprove = netApprove - } - - // This doesn't handle the unlikely case that the - // runoff vote results in a tie. If that happens - // we can decide how to handle it and rebuild the - // cache with the new rules. - } - } - - // Now that we know the winner we can update the losers - // and insert the results into the cache. - for _, vr := range results { - // Update runoff vote losers - if vr.Token != winnerToken { - vr.Approved = false - } - - // Insert vote results record - err = d.recordsdb.Create(&vr).Error - if err != nil { - return fmt.Errorf("insert vote results %v: %v", - vr.Token, err) - } - - log.Debugf("Runoff vote result created %v", vr.Token) - } - - return nil -} - -func (d *decred) voteResultsLoad(bestBlock uint64) error { - // Check to see if the VoteResults table needs to be updated. - standard, runoff, err := d.voteResultsMissing(bestBlock) - if err != nil { - return err - } - - // Insert vote results for standard votes proposals. - for _, token := range standard { - err := d.voteResultsInsertStandard(token) - if err != nil { - return fmt.Errorf("voteResultsInsertStandard %v: %v", - token, err) - } - } - - // Insert vote results for the runoff vote submissions. Runoff - // votes are identified by the parent RFP proposal token. - done := make(map[string]struct{}, len(runoff)) // [rfpToken]struct{} - for _, token := range runoff { - // Lookup the RFP token for the runoff vote submission - var pm ProposalMetadata - err := d.recordsdb. - Where("token = ?", token). - Find(&pm). - Error - if err != nil { - return fmt.Errorf("lookup ProposalMetadata %v: %v", - token, err) - } - if pm.LinkTo == "" { - return fmt.Errorf("runoff vote linkto not found %v", - token) - } - - if _, ok := done[pm.LinkTo]; ok { - // Results have already been inserted for this RFP. - // This happens because the vote results are built - // using the RFP token and all RFP submissions will - // list the same RFP token in their LinkTo field. - continue - } - - // Insert vote results for the full runoff vote - err = d.voteResultsInsertRunoff(pm.LinkTo) - if err != nil { - return fmt.Errorf("insert runoff vote results %v: %v", - pm.LinkTo, err) - } - - done[pm.LinkTo] = struct{}{} - } - - log.Debugf("voteResultsLoad: table updated for block %v", bestBlock) - - // Keep track of block used to update the table. - err = d.bestBlockSet(bestBlock) - if err != nil { - return err - } - - return nil -} - -// voteResults returns a map[token]VoteResults for the given tokens. -// -// The VoteResults table is lazy loaded. A cache.ErrRecordNotFound error is -// returned if the VoteResults table is not up-to-date. -func (d *decred) voteResults(tokens []string, bestBlock uint64) (map[string]VoteResults, error) { - // Check to see if the VoteResults table needs to be updated. - standard, runoff, err := d.voteResultsMissing(bestBlock) - if err != nil { - return nil, err - } - if len(standard) > 0 || len(runoff) > 0 { - // Return a ErrRecordNotFound to indicate one - // or more vote result records were not found. - return nil, cache.ErrRecordNotFound - } - - voteResults := make(map[string]VoteResults, len(tokens)) - if len(tokens) == 0 { - return voteResults, nil - } - - // Lookup vote results - vrs := make([]VoteResults, 0, len(tokens)) - err = d.recordsdb. - Where("token IN (?)", tokens). - Preload("Results"). - Preload("Results.Option"). - Find(&vrs). - Error - if err != nil { - return nil, fmt.Errorf("select VoteResults in %v: %v", - tokens, err) - } - - for _, v := range vrs { - voteResults[v.Token] = v - } - - return voteResults, nil -} - -// voteSummaries returns a map[string]decredplugin.VoteSummaryReply for the -// given proposal tokens. An entry in the returned map will only exist for -// tokens where a Record was found. -// -// This function pulls data from the the lazy loaded VoteResults table. A -// cache.ErrRecordNotFound error is returned if the VoteResults table is not -// up-to-date. -func (d *decred) voteSummaries(tokens []string, bestBlock uint64) (map[string]decredplugin.VoteSummaryReply, error) { - // This query returns the latest version of the given records. - query := `SELECT a.* - FROM records a - LEFT OUTER JOIN records b - ON a.token = b.token - AND a.version < b.version - WHERE b.token IS NULL - AND a.token IN (?)` - rows, err := d.recordsdb.Raw(query, tokens).Rows() - if err != nil { - return nil, fmt.Errorf("select records %v: %v", - tokens, err) - } - defer rows.Close() - - records := make([]Record, 0, len(tokens)) - for rows.Next() { - var r Record - err := d.recordsdb.ScanRows(rows, &r) - if err != nil { - return nil, err - } - records = append(records, r) - } - if err = rows.Err(); err != nil { - return nil, err - } - - // Lookup the AuthorizeVote for each record - authVotes, err := d.authorizeVotes(records) - if err != nil { - return nil, fmt.Errorf("authorizeVotes: %v", err) - } - - // Lookup the StartVote for each record that has an AuthorizeVote - tokens = make([]string, 0, len(authVotes)) - for token := range authVotes { - tokens = append(tokens, token) - } - startVotes, err := d.startVotes(tokens) - if err != nil { - return nil, fmt.Errorf("startVotes: %v", err) - } - - // Lookup the VoteResults for each record that has a StartVote - tokens = make([]string, 0, len(startVotes)) - for token := range startVotes { - tokens = append(tokens, token) - } - - // Check for missing vote results tables - voteResults, err := d.voteResults(tokens, bestBlock) - if err != nil { - if err == cache.ErrRecordNotFound { - // The VoteResults table needs to be updated. Return - // just the error so the calling function can key off - // of it. - return nil, err - } - return nil, fmt.Errorf("voteResults: %v", err) - } - - if len(tokens) != len(voteResults) { - // There were tokens that do not correspond to a VoteResults - // entry. This happens when the proposal vote has either not - // begun or is ongoing. We know that these tokens have a - // StartVote so the vote must be ongoing. Lookup the results - // manually. - for _, token := range tokens { - _, ok := voteResults[token] - if !ok { - sv := startVotes[token] - results, err := d.voteOptionResults(sv.Options) - if err != nil { - return nil, fmt.Errorf("voteOptionResults %v: %v", - sv.Token, err) - } - voteResults[token] = VoteResults{ - Token: token, - Approved: false, - Results: results, - } - } - } - } - - // Prepare vote summaries - summaries := make(map[string]decredplugin.VoteSummaryReply, len(records)) - for _, v := range records { - av := authVotes[v.Token] - sv := startVotes[v.Token] - vr := voteResults[v.Token] - - // Return "" not "0" if end height doesn't exist - var endHeight string - if sv.EndHeight != 0 { - endHeight = strconv.FormatUint(uint64(sv.EndHeight), 10) - } - - summaries[v.Token] = decredplugin.VoteSummaryReply{ - Authorized: av.Action == decredplugin.AuthVoteActionAuthorize, - Type: decredplugin.VoteT(sv.Type), - Duration: sv.Duration, - EndHeight: endHeight, - EligibleTicketCount: sv.EligibleTicketCount, - QuorumPercentage: sv.QuorumPercentage, - PassPercentage: sv.PassPercentage, - Results: convertVoteOptionResultsToDecred(vr.Results), - Approved: vr.Approved, - } - } - - return summaries, nil -} - -// cmdNewComment creates a Comment record using the passed in payloads and -// inserts it into the database. -func (d *decred) cmdNewComment(cmdPayload, replyPayload string) (string, error) { - log.Tracef("decred cmdNewComment") - - nc, err := decredplugin.DecodeNewComment([]byte(cmdPayload)) - if err != nil { - return "", err - } - ncr, err := decredplugin.DecodeNewCommentReply([]byte(replyPayload)) - if err != nil { - return "", err - } - - c := convertNewCommentFromDecred(*nc, *ncr) - err = d.recordsdb.Create(&c).Error - - return replyPayload, err -} - -// cmdLikeComment creates a LikeComment record using the passed in payloads -// and inserts it into the database. -func (d *decred) cmdLikeComment(cmdPayload, replyPayload string) (string, error) { - log.Tracef("decred cmdLikeComment") - - dlc, err := decredplugin.DecodeLikeComment([]byte(cmdPayload)) - if err != nil { - return "", err - } - - lc := convertLikeCommentFromDecred(*dlc) - err = d.recordsdb.Create(&lc).Error - - return replyPayload, err -} - -// cmdCensorComment censors an existing comment. A censored comment has its -// comment message removed and is marked as censored. -func (d *decred) cmdCensorComment(cmdPayload, replyPayload string) (string, error) { - log.Tracef("decred cmdCensorComment") - - cc, err := decredplugin.DecodeCensorComment([]byte(cmdPayload)) - if err != nil { - return "", err - } - - c := Comment{ - Key: cc.Token + cc.CommentID, - } - err = d.recordsdb.Model(&c). - Updates(map[string]interface{}{ - "comment": "", - "censored": true, - }).Error - - return replyPayload, err -} - -func (d *decred) commentGetByID(token string, commentID string) (*Comment, error) { - c := Comment{ - Key: token + commentID, - } - err := d.recordsdb.Find(&c).Error - if err != nil { - if err == gorm.ErrRecordNotFound { - err = cache.ErrRecordNotFound - } - return nil, err - } - return &c, nil -} - -func (d *decred) commentGetBySignature(token string, sig string) (*Comment, error) { - var c Comment - err := d.recordsdb. - Where("token = ? AND signature = ?", token, sig). - Find(&c). - Error - if err != nil { - if err == gorm.ErrRecordNotFound { - err = cache.ErrRecordNotFound - } - return nil, err - } - return &c, nil -} - -// cmdGetComment retreives the passed in comment from the database. -func (d *decred) cmdGetComment(payload string) (string, error) { - log.Tracef("decred cmdGetComment") - - gc, err := decredplugin.DecodeGetComment([]byte(payload)) - if err != nil { - return "", err - } - - if gc.Token == "" { - return "", cache.ErrInvalidPluginCmdArgs - } - - var c *Comment - switch { - case gc.CommentID != "": - c, err = d.commentGetByID(gc.Token, gc.CommentID) - case gc.Signature != "": - c, err = d.commentGetBySignature(gc.Token, gc.Signature) - default: - return "", cache.ErrInvalidPluginCmdArgs - } - if err != nil { - return "", err - } - - gcr := decredplugin.GetCommentReply{ - Comment: convertCommentToDecred(*c), - } - gcrb, err := decredplugin.EncodeGetCommentReply(gcr) - if err != nil { - return "", err - } - - return string(gcrb), nil -} - -// cmdGetComments returns all of the comments for the passed in record token. -func (d *decred) cmdGetComments(payload string) (string, error) { - log.Tracef("decred cmdGetComments") - - gc, err := decredplugin.DecodeGetComments([]byte(payload)) - if err != nil { - return "", err - } - - comments := make([]Comment, 0, 1024) // PNOOMA - err = d.recordsdb. - Where("token = ?", gc.Token). - Find(&comments). - Error - if err != nil { - return "", err - } - - dpc := make([]decredplugin.Comment, 0, len(comments)) - for _, c := range comments { - dpc = append(dpc, convertCommentToDecred(c)) - } - - gcr := decredplugin.GetCommentsReply{ - Comments: dpc, - } - gcrb, err := decredplugin.EncodeGetCommentsReply(gcr) - if err != nil { - return "", err - } - - return string(gcrb), nil -} - -// cmdGetNumComments returns an encoded plugin reply that contains a -// [token]numComments map for the provided list of censorship tokens. If a -// provided token does not correspond to an actual proposal then it will not -// be included in the returned map. -func (d *decred) cmdGetNumComments(payload string) (string, error) { - log.Tracef("decred cmdGetNumComments") - - gnc, err := decredplugin.DecodeGetNumComments([]byte(payload)) - if err != nil { - return "", err - } - - // Lookup number of comments for provided tokens - type Result struct { - Token string - Counts int - } - results := make([]Result, 0, len(gnc.Tokens)) - err = d.recordsdb. - Table("comments"). - Select("count(*) as counts, token"). - Group("token"). - Where("token IN (?)", gnc.Tokens). - Find(&results). - Error - if err != nil { - return "", err - } - - // Put results into a map - numComments := make(map[string]int, len(results)) // [token]numComments - for _, c := range results { - numComments[c.Token] = c.Counts - } - - // Encode reply - gncr := decredplugin.GetNumCommentsReply{ - NumComments: numComments, - } - gncre, err := decredplugin.EncodeGetNumCommentsReply(gncr) - if err != nil { - return "", err - } - - return string(gncre), nil -} - -// cmdCommentLikes returns all of the comment likes for the passed in comment. -func (d *decred) cmdCommentLikes(payload string) (string, error) { - log.Tracef("decred cmdCommentLikes") - - cl, err := decredplugin.DecodeCommentLikes([]byte(payload)) - if err != nil { - return "", err - } - - likes := make([]LikeComment, 1024) // PNOOMA - err = d.recordsdb. - Where("token = ? AND comment_id = ?", cl.Token, cl.CommentID). - Find(&likes). - Error - if err != nil { - return "", err - } - - lc := make([]decredplugin.LikeComment, 0, len(likes)) - for _, v := range likes { - lc = append(lc, convertLikeCommentToDecred(v)) - } - - clr := decredplugin.CommentLikesReply{ - CommentLikes: lc, - } - clrb, err := decredplugin.EncodeCommentLikesReply(clr) - if err != nil { - return "", err - } - - return string(clrb), nil -} - -// cmdProposalLikes returns all of the comment likes for all comments of the -// passed in record token. -func (d *decred) cmdProposalCommentsLikes(payload string) (string, error) { - log.Tracef("decred cmdProposalCommentsLikes") - - cl, err := decredplugin.DecodeGetProposalCommentsLikes([]byte(payload)) - if err != nil { - return "", err - } - - likes := make([]LikeComment, 0, 1024) // PNOOMA - err = d.recordsdb. - Where("token = ?", cl.Token). - Find(&likes). - Error - if err != nil { - return "", err - } - - lc := make([]decredplugin.LikeComment, 0, len(likes)) - for _, v := range likes { - lc = append(lc, convertLikeCommentToDecred(v)) - } - - clr := decredplugin.GetProposalCommentsLikesReply{ - CommentsLikes: lc, - } - clrb, err := decredplugin.EncodeGetProposalCommentsLikesReply(clr) - if err != nil { - return "", err - } - - return string(clrb), nil -} - -// authorizeVoteInsert creates an AuthorizeVote record and inserts it into the -// database. If a previous AuthorizeVote record exists for the passed in -// proposal and version, it will be deleted before the new AuthorizeVote record -// is inserted. -// -// This function must be called using a transaction. -func authorizeVoteInsert(tx *gorm.DB, av AuthorizeVote) error { - // Delete authorize vote if one exists for this version - err := tx.Where("key = ?", av.Key). - Delete(AuthorizeVote{}). - Error - if err != nil { - return fmt.Errorf("delete authorize vote: %v", err) - } - - // Add new authorize vote - err = tx.Create(&av).Error - if err != nil { - return fmt.Errorf("create authorize vote: %v", err) - } - - return nil -} - -// cmdAuthorizeVote creates a AuthorizeVote record using the passed in payloads -// and inserts it into the database. -func (d *decred) cmdAuthorizeVote(cmdPayload, replyPayload string) (string, error) { - log.Tracef("decred cmdAuthorizeVote") - - av, err := decredplugin.DecodeAuthorizeVote([]byte(cmdPayload)) - if err != nil { - return "", err - } - avr, err := decredplugin.DecodeAuthorizeVoteReply([]byte(replyPayload)) - if err != nil { - return "", err - } - a, err := convertAuthorizeVoteFromDecred(*av, *avr) - if err != nil { - return "", err - } - - // Run update in a transaction - tx := d.recordsdb.Begin() - err = authorizeVoteInsert(tx, *a) - if err != nil { - tx.Rollback() - return "", fmt.Errorf("authorizeVoteInsert: %v", err) - } - - // Commit transaction - err = tx.Commit().Error - if err != nil { - return "", fmt.Errorf("commit transaction: %v", err) - } - - return replyPayload, nil -} - -// startVoteInsert inserts a StartVote record into the database. This function -// has a database parameter so that it can be called inside of a transaction -// when required. -func startVoteInsert(db *gorm.DB, sv StartVote) error { - return db.Create(&sv).Error -} - -// cmdStartVote creates a StartVote record using the passed in payloads and -// inserts it into the database. -func (d *decred) cmdStartVote(cmdPayload, replyPayload string) (string, error) { - log.Tracef("decred cmdStartVote") - - // Handle start vote versioning. This command accepts both v1 and - // v2 start votes in order to allow rebuilding the start vote cache - // using this command. - dsv, err := decredplugin.DecodeStartVote([]byte(cmdPayload)) - if err != nil { - return "", err - } - svr, err := decredplugin.DecodeStartVoteReply([]byte(replyPayload)) - if err != nil { - return "", err - } - - var sv *StartVote - switch dsv.Version { - case 0: - // The version is only going to be present when the cache is - // being rebuilt. When a vote is started normally, gitbe fills in - // the version before writing it to disk, which means the version - // is not going to travel to the cache. This is a side effect of - // the cache being implemented in the wrong layer. If the version - // is 0 then we can assume this is the current start vote version. - svb := []byte(dsv.Payload) - sv2, err := decredplugin.DecodeStartVoteV2(svb) - if err != nil { - return "", fmt.Errorf("decode StartVoteV2 %v: %v", dsv.Token, err) - } - err = sv2.VerifySignature() - if err != nil { - return "", fmt.Errorf("verify signature: %v", err) - } - sv, err = convertStartVoteV2FromDecred(*sv2, *svr) - if err != nil { - return "", err - } - case decredplugin.VersionStartVoteV1: - svb := []byte(dsv.Payload) - sv1, err := decredplugin.DecodeStartVoteV1(svb) - if err != nil { - return "", fmt.Errorf("decode StartVoteV1 %v: %v", dsv.Token, err) - } - err = sv1.VerifySignature() - if err != nil { - return "", fmt.Errorf("verify signature: %v", err) - } - sv, err = convertStartVoteV1FromDecred(*sv1, *svr) - if err != nil { - return "", err - } - case decredplugin.VersionStartVoteV2: - svb := []byte(dsv.Payload) - sv2, err := decredplugin.DecodeStartVoteV2(svb) - if err != nil { - return "", fmt.Errorf("decode StartVoteV2 %v: %v", dsv.Token, err) - } - err = sv2.VerifySignature() - if err != nil { - return "", fmt.Errorf("verify signature: %v", err) - } - sv, err = convertStartVoteV2FromDecred(*sv2, *svr) - if err != nil { - return "", err - } - default: - return "", fmt.Errorf("unknown start vote version %v", dsv.Version) - } - - err = startVoteInsert(d.recordsdb, *sv) - if err != nil { - return "", err - } - - return replyPayload, nil -} - -func (d *decred) cmdStartVoteRunoff(cmdPayload, replyPayload string) (string, error) { - log.Tracef("decred cmdStartVoteRunoff") - - sv, err := decredplugin.DecodeStartVoteRunoff([]byte(cmdPayload)) - if err != nil { - return "", err - } - svr, err := decredplugin.DecodeStartVoteRunoffReply([]byte(replyPayload)) - if err != nil { - return "", err - } - - // Prepare records to be inserted - authVotes := make([]AuthorizeVote, 0, len(sv.AuthorizeVotes)) - for _, v := range sv.AuthorizeVotes { - avr := svr.AuthorizeVoteReplies[v.Token] - a, err := convertAuthorizeVoteFromDecred(v, avr) - if err != nil { - return "", err - } - authVotes = append(authVotes, *a) - } - startVotes := make([]StartVote, 0, len(sv.StartVotes)) - for _, v := range sv.StartVotes { - s, err := convertStartVoteV2FromDecred(v, svr.StartVoteReply) - if err != nil { - return "", err - } - startVotes = append(startVotes, *s) - } - - // Run updates in a transaction - tx := d.recordsdb.Begin() - for _, v := range authVotes { - err = authorizeVoteInsert(tx, v) - if err != nil { - tx.Rollback() - return "", err - } - } - for _, v := range startVotes { - err := startVoteInsert(tx, v) - if err != nil { - tx.Rollback() - return "", err - } - } - err = tx.Commit().Error - if err != nil { - return "", err - } - - return replyPayload, nil -} - -// cmdVoteDetails returns the AuthorizeVote and StartVote records for the -// passed in record token. -func (d *decred) cmdVoteDetails(payload string) (string, error) { - log.Tracef("decred cmdVoteDetails") - - vd, err := decredplugin.DecodeVoteDetails([]byte(payload)) - if err != nil { - return "", nil - } - - // Lookup the most recent version of the record - var r Record - err = d.recordsdb. - Where("records.token = ?", vd.Token). - Order("records.version desc"). - Limit(1). - Find(&r). - Error - if err != nil { - if err == gorm.ErrRecordNotFound { - err = cache.ErrRecordNotFound - } - return "", err - } - - // Lookup authorize vote - var av AuthorizeVote - key := vd.Token + strconv.FormatUint(r.Version, 10) - err = d.recordsdb. - Where("key = ?", key). - Find(&av). - Error - if err == gorm.ErrRecordNotFound { - // An authorize vote may note exist. This is ok. - } else if err != nil { - return "", fmt.Errorf("authorize vote lookup failed: %v", err) - } - - // Lookup start vote - var ( - sv StartVote - dsv decredplugin.StartVote - dsvr decredplugin.StartVoteReply - ) - err = d.recordsdb. - Where("token = ?", vd.Token). - Preload("Options"). - Find(&sv). - Error - if err == gorm.ErrRecordNotFound { - // A start vote may note exist. This is ok. - } else if err != nil { - return "", fmt.Errorf("start vote lookup failed: %v", err) - } - - // Only convert if a StartVote was found, otherwise it will - // throw an invalid version error. - if sv.Version != 0 { - dsvp, dsvrp, err := convertStartVoteToDecred(sv) - if err != nil { - return "", err - } - dsv = *dsvp - dsvr = *dsvrp - } - - // Prepare reply - vdr := decredplugin.VoteDetailsReply{ - AuthorizeVote: convertAuthorizeVoteToDecred(av), - StartVote: dsv, - StartVoteReply: dsvr, - } - vdrb, err := decredplugin.EncodeVoteDetailsReply(vdr) - if err != nil { - return "", err - } - - return string(vdrb), nil -} - -// cmdNewBallot creates CastVote records using the passed in payloads and -// inserts them into the database. -func (d *decred) cmdNewBallot(cmdPayload, replyPayload string) (string, error) { - log.Tracef("decred cmdNewBallot") - - b, err := decredplugin.DecodeBallot([]byte(cmdPayload)) - if err != nil { - return "", err - } - - br, err := decredplugin.DecodeBallotReply([]byte(replyPayload)) - if err != nil { - return "", err - } - - // If the vote contains an error, don't add it. - invalid := make(map[string]struct{}, len(br.Receipts)) - for _, v := range br.Receipts { - if v.ErrorStatus != 0 || v.Error != "" { - invalid[v.ClientSignature] = struct{}{} - } - } - - // Insert cast vote records - for _, v := range b.Votes { - if _, ok := invalid[v.Signature]; ok { - continue - } - - cv := convertCastVoteFromDecred(v) - err := d.recordsdb.Create(&cv).Error - if err != nil { - return "", err - } - } - - return replyPayload, nil -} - -// cmdProposalVotes returns the StartVote record and all CastVote records for -// the passed in record token. -func (d *decred) cmdProposalVotes(payload string) (string, error) { - log.Tracef("decred cmdProposalVotes") - - vr, err := decredplugin.DecodeVoteResults([]byte(payload)) - if err != nil { - return "", err - } - - // Lookup all cast votes - var cv []CastVote - err = d.recordsdb. - Where("token = ?", vr.Token). - Find(&cv). - Error - if err == gorm.ErrRecordNotFound { - // No cast votes may exist yet. This is ok. - } else if err != nil { - return "", fmt.Errorf("cast votes lookup failed: %v", err) - } - - // Prepare reply - dcv := make([]decredplugin.CastVote, 0, len(cv)) - for _, v := range cv { - dcv = append(dcv, convertCastVoteToDecred(v)) - } - - vrr := decredplugin.VoteResultsReply{ - CastVotes: dcv, - } - - vrrb, err := decredplugin.EncodeVoteResultsReply(vrr) - if err != nil { - return "", err - } - - return string(vrrb), nil -} - -// cmdInventory returns the decred plugin inventory. -func (d *decred) cmdInventory() (string, error) { - log.Tracef("decred cmdInventory") - - // XXX the only part of the decred plugin inventory that we return - // at the moment is comments. This is because comments are the only - // thing politeiawww currently needs on startup. - - // Get all comments - var c []Comment - err := d.recordsdb.Find(&c).Error - if err != nil { - return "", err - } - - dc := make([]decredplugin.Comment, 0, len(c)) - for _, v := range c { - dc = append(dc, convertCommentToDecred(v)) - } - - // Prepare inventory reply - ir := decredplugin.InventoryReply{ - Comments: dc, - } - irb, err := decredplugin.EncodeInventoryReply(ir) - if err != nil { - return "", err - } - - return string(irb), err -} - -// cmdLoadVoteResults creates vote results entries for any proposals that have -// a finished voting period but have not yet been added to the vote results -// table. The vote results table is lazy loaded. -func (d *decred) cmdLoadVoteResults(payload string) (string, error) { - log.Tracef("cmdLoadVoteResults") - - lvs, err := decredplugin.DecodeLoadVoteResults([]byte(payload)) - if err != nil { - return "", err - } - - err = d.voteResultsLoad(lvs.BestBlock) - if err != nil { - return "", err - } - - r := decredplugin.LoadVoteResultsReply{} - reply, err := decredplugin.EncodeLoadVoteResultsReply(r) - if err != nil { - return "", err - } - - return string(reply), nil -} - -// cmdTokenInventory returns the tokens of all records in the cache, -// categorized by stage of the voting process. This call relies on the lazy -// loaded VoteResults table. A cache.ErrRecordNotFound is returned if the -// VoteResults table is not up-to-date. -func (d *decred) cmdTokenInventory(payload string) (string, error) { - log.Tracef("decred cmdTokenInventory") - - ti, err := decredplugin.DecodeTokenInventory([]byte(payload)) - if err != nil { - return "", err - } - - // Check to see if the VoteResults table needs to be updated - standard, runoff, err := d.voteResultsMissing(ti.BestBlock) - if err != nil { - return "", err - } - if len(standard) > 0 || len(runoff) > 0 { - // Return a ErrRecordNotFound to indicate one - // or more vote result records were not found. - return "", cache.ErrRecordNotFound - } - - // Pre voting period tokens. This query returns the - // tokens of the most recent version of all records that - // are public and do not have an associated StartVote - // record, ordered by timestamp in descending order. - q := `SELECT a.token - FROM records a - LEFT OUTER JOIN start_votes - ON a.token = start_votes.token - LEFT OUTER JOIN records b - ON a.token = b.token - AND a.version < b.version - WHERE b.token IS NULL - AND start_votes.token IS NULL - AND a.status = ? - ORDER BY a.timestamp DESC` - rows, err := d.recordsdb.Raw(q, pd.RecordStatusPublic).Rows() - if err != nil { - return "", fmt.Errorf("pre: %v", err) - } - defer rows.Close() - - var token string - pre := make([]string, 0, 1024) - for rows.Next() { - err = rows.Scan(&token) - if err != nil { - return "", err - } - pre = append(pre, token) - } - if err = rows.Err(); err != nil { - return "", err - } - - // Active voting period tokens - q = `SELECT token - FROM start_votes - WHERE end_height > ? - ORDER BY end_height DESC` - rows, err = d.recordsdb.Raw(q, ti.BestBlock).Rows() - if err != nil { - return "", fmt.Errorf("active: %v", err) - } - defer rows.Close() - - active := make([]string, 0, 1024) - for rows.Next() { - err = rows.Scan(&token) - if err != nil { - return "", err - } - active = append(active, token) - } - if err = rows.Err(); err != nil { - return "", err - } - - // Approved vote tokens - q = `SELECT vote_results.token - FROM vote_results - INNER JOIN start_votes - ON vote_results.token = start_votes.token - WHERE vote_results.approved = true - ORDER BY start_votes.end_height DESC` - rows, err = d.recordsdb.Raw(q).Rows() - if err != nil { - return "", fmt.Errorf("approved: %v", err) - } - defer rows.Close() - - approved := make([]string, 0, 1024) - for rows.Next() { - err = rows.Scan(&token) - if err != nil { - return "", err - } - approved = append(approved, token) - } - if err = rows.Err(); err != nil { - return "", err - } - - // Rejected vote tokens - q = `SELECT vote_results.token - FROM vote_results - INNER JOIN start_votes - ON vote_results.token = start_votes.token - WHERE vote_results.approved = false - ORDER BY start_votes.end_height DESC` - rows, err = d.recordsdb.Raw(q).Rows() - if err != nil { - return "", fmt.Errorf("rejected: %v", err) - } - defer rows.Close() - - rejected := make([]string, 0, 1024) - for rows.Next() { - err = rows.Scan(&token) - if err != nil { - return "", err - } - rejected = append(rejected, token) - } - if err = rows.Err(); err != nil { - return "", err - } - - // Abandoned tokens - abandoned := make([]string, 0, 1024) - q = `SELECT token - FROM records - WHERE status = ? - ORDER BY timestamp DESC` - rows, err = d.recordsdb.Raw(q, pd.RecordStatusArchived).Rows() - if err != nil { - return "", fmt.Errorf("abandoned: %v", err) - } - defer rows.Close() - - for rows.Next() { - err = rows.Scan(&token) - if err != nil { - return "", err - } - abandoned = append(abandoned, token) - } - if err = rows.Err(); err != nil { - return "", err - } - - // Setup reply - tir := decredplugin.TokenInventoryReply{ - Pre: pre, - Active: active, - Approved: approved, - Rejected: rejected, - Abandoned: abandoned, - Unreviewed: []string{}, - Censored: []string{}, - } - - // Populate unvetted records if specified - if ti.Unvetted { - // Unreviewed tokens. Edits to an unreviewed record do not - // increment the version. Only edits to a public record - // increment the version. This means means we don't need - // to worry about fetching the most recent version here - // because an unreviewed record will only have one version. - unreviewed := make([]string, 0, 1024) - q = `SELECT token - FROM records - WHERE status = ? or status = ? - ORDER BY timestamp DESC` - rows, err = d.recordsdb.Raw(q, pd.RecordStatusNotReviewed, - pd.RecordStatusUnreviewedChanges).Rows() - if err != nil { - return "", fmt.Errorf("unreviewed: %v", err) - } - defer rows.Close() - - for rows.Next() { - err = rows.Scan(&token) - if err != nil { - return "", err - } - unreviewed = append(unreviewed, token) - } - if err = rows.Err(); err != nil { - return "", err - } - // Censored tokens - censored := make([]string, 0, 1024) - q = `SELECT token - FROM records - WHERE status = ? - ORDER BY timestamp DESC` - rows, err = d.recordsdb.Raw(q, pd.RecordStatusCensored).Rows() - if err != nil { - return "", fmt.Errorf("censored: %v", err) - } - defer rows.Close() - - for rows.Next() { - err = rows.Scan(&token) - if err != nil { - return "", err - } - censored = append(censored, token) - } - if err = rows.Err(); err != nil { - return "", err - } - - // Update reply - tir.Unreviewed = unreviewed - tir.Censored = censored - } - - // Encode reply - reply, err := decredplugin.EncodeTokenInventoryReply(tir) - if err != nil { - return "", err - } - - return string(reply), nil -} - -// cmdVoteSummary returns a map[string]VoteSummaryReply for the given proposal -// tokens. Results are returned for all tokens that correspond to a proposal. -// This includes both unvetted and vetted proposals. Tokens that do no -// correspond to a proposal are not included in the returned map. -// -// This function pulls data from the the lazy loaded VoteResults table. A -// cache.ErrRecordNotFound error is returned if the VoteResults table is not -// up-to-date. -func (d *decred) cmdBatchVoteSummary(payload string) (string, error) { - log.Tracef("cmdBatchVoteSummary") - - bvs, err := decredplugin.DecodeBatchVoteSummary([]byte(payload)) - if err != nil { - return "", err - } - - summaries, err := d.voteSummaries(bvs.Tokens, bvs.BestBlock) - if err != nil { - return "", err - } - - bvsr := decredplugin.BatchVoteSummaryReply{ - Summaries: summaries, - } - reply, err := decredplugin.EncodeBatchVoteSummaryReply(bvsr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -// cmdVoteSummary returns a decredplugin.VoteSummaryReply for the given -// proposal token. -// -// This function pulls data from the the lazy loaded VoteResults table. A -// cache.ErrRecordNotFound error is returned if the VoteResults table is not -// up-to-date. -func (d *decred) cmdVoteSummary(payload string) (string, error) { - log.Tracef("cmdVoteSummary") - - vs, err := decredplugin.DecodeVoteSummary([]byte(payload)) - if err != nil { - return "", err - } - - summaries, err := d.voteSummaries([]string{vs.Token}, vs.BestBlock) - if err != nil { - return "", err - } - - vsr := summaries[vs.Token] - reply, err := decredplugin.EncodeVoteSummaryReply(vsr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -// linkedFrom returns the proposal tokens of all publicly viewable proposals -// that have linked to the given proposal token. If the provided token does not -// correspond to an actual proposal record then a nil value is returned instead -// of a []string. -func (d *decred) linkedFrom(token string) ([]string, error) { - // Ensure the token corresponds to an actual record - ok, err := recordExists(d.recordsdb, token, "1") - if err != nil { - return nil, err - } - if !ok { - // Token doesn't correspond to an actual record - return nil, nil - } - - // This query returns the proposals that are publicly viewable, - // have linked to the provided token, and are the most recent - // version of their proposal. - q := `SELECT a.token - FROM records a - LEFT OUTER JOIN records b - ON a.token = b.token - AND a.version < b.version - INNER JOIN proposal_metadata - ON a.token = proposal_metadata.token - WHERE b.token IS NULL - AND proposal_metadata.link_to = ? - AND a.status IN (?)` - rows, err := d.recordsdb. - Raw(q, token, publicStatuses()). - Rows() - if err != nil { - return nil, fmt.Errorf("lookup linked from %v: %v", token, err) - } - defer rows.Close() - - linkedFrom := make([]string, 0, 256) - for rows.Next() { - var token string - rows.Scan(&token) - linkedFrom = append(linkedFrom, token) - } - if err = rows.Err(); err != nil { - return nil, err - } - - return linkedFrom, nil -} - -func (d *decred) cmdLinkedFrom(payload string) (string, error) { - log.Tracef("cmdLinkedFrom") - - lf, err := decredplugin.DecodeLinkedFrom([]byte(payload)) - if err != nil { - return "", err - } - - linkedFrom := make(map[string][]string, len(lf.Tokens)) // [token]linkedFrom - for _, token := range lf.Tokens { - lf, err := d.linkedFrom(token) - if err != nil { - return "", err - } - if lf == nil { - // Token doesn't correspond to an actual record. Don't include - // it in the reply. - continue - } - linkedFrom[token] = lf - } - - lfr := decredplugin.LinkedFromReply{ - LinkedFrom: linkedFrom, - } - reply, err := decredplugin.EncodeLinkedFromReply(lfr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func proposalMetadataNew(r Record) (*ProposalMetadata, error) { - var name string - for _, v := range r.Metadata { - switch v.ID { - case mdstream.IDProposalGeneral: - // Pull the name out of the mdstream - b := []byte(v.Payload) - version, err := mdstream.DecodeVersion(b) - if err != nil { - return nil, err - } - switch version { - case 1: - pg, err := mdstream.DecodeProposalGeneralV1(b) - if err != nil { - return nil, err - } - name = pg.Name - case 2: - // The proposal name was removed from the ProposalGeneralV2 - // mdtream and added to the ProposalMetadata file. - } - } - } - - // Parse the ProposalMetadata from the proposal files. One will - // exists for all new proposals but may not exists for older - // proposals. - var pm ProposalMetadata - for _, f := range r.Files { - switch f.Name { - case mdstream.FilenameProposalMetadata: - b, err := base64.StdEncoding.DecodeString(f.Payload) - if err != nil { - return nil, err - } - err = json.Unmarshal(b, &pm) - if err != nil { - return nil, err - } - name = pm.Name - } - } - - return &ProposalMetadata{ - Token: r.Token, - Name: name, - LinkTo: pm.LinkTo, - LinkBy: pm.LinkBy, - }, nil -} - -// hookPostNewRecord executes the decred plugin post new record hook. This -// includes inserting a ProposalMetadata record for the given proposal. -// -// This function must be called using a transaction. -func (d *decred) hookPostNewRecord(tx *gorm.DB, payload string) error { - var r Record - err := json.Unmarshal([]byte(payload), &r) - if err != nil { - return err - } - pm, err := proposalMetadataNew(r) - if err != nil { - return err - } - if pm.Name == "" { - // XXX Commented out as a temporary workaround for CMS using decred - // plugin. This needs to be fixed once the plugin architecture is - // sorted out. - // - // return fmt.Errorf("proposal user metadata not found") - - return nil - } - - // All prososal versions are stored in the cache which means that - // this new proposal request could be for a brand new proposal or - // it could be for a new proposal version that is the result of a - // proposal edit. We only need to store the ProposalMetadata for - // the most recent version of the proposal. Delete any existing - // metadata. - err = tx.Delete(ProposalMetadata{ - Token: r.Token, - }).Error - if err != nil { - return fmt.Errorf("delete: %v", err) - } - - // Insert new metadata - err = tx.Create(pm).Error - if err != nil { - return fmt.Errorf("create: %v", err) - } - - return nil -} - -// hookPostUpdateRecord executes the decred plugin post update record hook. -// This includes updating the ProposalMetadata in the cache for the given -// proposal. The existing metadata is first deleted before the new metadata is -// inserted. -// -// This function must be called using a transaction. -func (d *decred) hookPostUpdateRecord(tx *gorm.DB, payload string) error { - var r Record - err := json.Unmarshal([]byte(payload), &r) - if err != nil { - return err - } - pm, err := proposalMetadataNew(r) - if err != nil { - return err - } - if pm.Name == "" { - // XXX Commented out as a temporary workaround for CMS using decred - // plugin. This needs to be fixed once the plugin architecture is - // sorted out. - // - // return fmt.Errorf("proposal user metadata not found") - - return nil - } - - // All prososal versions are stored in the cache which means that - // this new proposal request could be for a brand new proposal or - // it could be for a new proposal version that is the result of a - // proposal edit. We only need to store the ProposalMetadata for - // the most recent version of the proposal. Delete any existing - // metadata. - err = tx.Delete(ProposalMetadata{ - Token: r.Token, - }).Error - if err != nil { - return fmt.Errorf("delete: %v", err) - } - - // Insert new metadata - err = tx.Create(pm).Error - if err != nil { - return fmt.Errorf("create: %v", err) - } - - return nil -} - -// hookPostUpdateRecordMetadata executes the decred plugin post update record -// metadata hook. -func (d *decred) hookPostUpdateRecordMetadata(tx *gorm.DB, payload string) error { - // piwww does not currently use the UpdateRecordMetadata route. - // If this changes, this panic is here as a reminder that any piwww - // mdstream tables, such as ProposalMetadata and StartVote, need to - // be properly updated in this hook. - - // XXX Commented out as a temporary workaround for CMS using decred - // plugin. This needs to be fixed once the plugin architecture is - // sorted out. - // - // panic("cache decred plugin: hookPostUpdateRecordMetadata not implemented") - - return nil -} - -// Hook executes the given decred plugin hook. -func (d *decred) Hook(tx *gorm.DB, hookID, payload string) error { - log.Tracef("decred Hook: %v", hookID) - - switch hookID { - case pluginHookPostNewRecord: - return d.hookPostNewRecord(tx, payload) - case pluginHookPostUpdateRecord: - return d.hookPostUpdateRecord(tx, payload) - case pluginHookPostUpdateRecordMetadata: - return d.hookPostUpdateRecordMetadata(tx, payload) - } - - return nil -} - -// Exec executes a decred plugin command. Plugin commands that write data to -// the cache require both the command payload and the reply payload. Plugin -// commands that fetch data from the cache require only the command payload. -// All commands return the appropriate reply payload. -func (d *decred) Exec(cmd, cmdPayload, replyPayload string) (string, error) { - log.Tracef("decred Exec: %v", cmd) - - switch cmd { - case decredplugin.CmdAuthorizeVote: - return d.cmdAuthorizeVote(cmdPayload, replyPayload) - case decredplugin.CmdStartVote: - return d.cmdStartVote(cmdPayload, replyPayload) - case decredplugin.CmdStartVoteRunoff: - return d.cmdStartVoteRunoff(cmdPayload, replyPayload) - case decredplugin.CmdVoteDetails: - return d.cmdVoteDetails(cmdPayload) - case decredplugin.CmdBallot: - return d.cmdNewBallot(cmdPayload, replyPayload) - case decredplugin.CmdBestBlock: - return "", nil - case decredplugin.CmdNewComment: - return d.cmdNewComment(cmdPayload, replyPayload) - case decredplugin.CmdLikeComment: - return d.cmdLikeComment(cmdPayload, replyPayload) - case decredplugin.CmdCensorComment: - return d.cmdCensorComment(cmdPayload, replyPayload) - case decredplugin.CmdGetComment: - return d.cmdGetComment(cmdPayload) - case decredplugin.CmdGetComments: - return d.cmdGetComments(cmdPayload) - case decredplugin.CmdGetNumComments: - return d.cmdGetNumComments(cmdPayload) - case decredplugin.CmdProposalVotes: - return d.cmdProposalVotes(cmdPayload) - case decredplugin.CmdCommentLikes: - return d.cmdCommentLikes(cmdPayload) - case decredplugin.CmdProposalCommentsLikes: - return d.cmdProposalCommentsLikes(cmdPayload) - case decredplugin.CmdInventory: - return d.cmdInventory() - case decredplugin.CmdLoadVoteResults: - return d.cmdLoadVoteResults(cmdPayload) - case decredplugin.CmdTokenInventory: - return d.cmdTokenInventory(cmdPayload) - case decredplugin.CmdVoteSummary: - return d.cmdVoteSummary(cmdPayload) - case decredplugin.CmdBatchVoteSummary: - return d.cmdBatchVoteSummary(cmdPayload) - case decredplugin.CmdLinkedFrom: - return d.cmdLinkedFrom(cmdPayload) - } - - return "", cache.ErrInvalidPluginCmd -} - -// createTables creates the cache tables needed by the decred plugin if they do -// not already exist. A decred plugin version record is inserted into the -// database during table creation. -// -// This function must be called within a transaction. -func (d *decred) createTables(tx *gorm.DB) error { - log.Tracef("createTables") - - // Create decred plugin tables - if !tx.HasTable(tableProposalMetadata) { - err := tx.CreateTable(&ProposalMetadata{}).Error - if err != nil { - return err - } - } - if !tx.HasTable(tableComments) { - err := tx.CreateTable(&Comment{}).Error - if err != nil { - return err - } - } - if !tx.HasTable(tableCommentLikes) { - err := tx.CreateTable(&LikeComment{}).Error - if err != nil { - return err - } - } - if !tx.HasTable(tableCastVotes) { - err := tx.CreateTable(&CastVote{}).Error - if err != nil { - return err - } - } - if !tx.HasTable(tableAuthorizeVotes) { - err := tx.CreateTable(&AuthorizeVote{}).Error - if err != nil { - return err - } - } - if !tx.HasTable(tableVoteOptions) { - err := tx.CreateTable(&VoteOption{}).Error - if err != nil { - return err - } - } - if !tx.HasTable(tableStartVotes) { - err := tx.CreateTable(&StartVote{}).Error - if err != nil { - return err - } - } - if !tx.HasTable(tableVoteOptionResults) { - err := tx.CreateTable(&VoteOptionResult{}).Error - if err != nil { - return err - } - } - if !tx.HasTable(tableVoteResults) { - err := tx.CreateTable(&VoteResults{}).Error - if err != nil { - return err - } - } - - // Check if a decred version record exists. Insert one - // if no version record is found. - if !tx.HasTable(tableVersions) { - // This should never happen - return fmt.Errorf("versions table not found") - } - - var v Version - err := tx.Where("id = ?", decredplugin.ID).Find(&v).Error - if err == gorm.ErrRecordNotFound { - err = tx.Create( - &Version{ - ID: decredplugin.ID, - Version: decredVersion, - Timestamp: time.Now().Unix(), - }).Error - } - - return err -} - -// droptTables drops all decred plugin tables from the cache and remove the -// decred plugin version record. -// -// This function must be called within a transaction. -func (d *decred) dropTables(tx *gorm.DB) error { - // Drop decred plugin tables - err := tx.DropTableIfExists(tableComments, tableCommentLikes, - tableCastVotes, tableAuthorizeVotes, tableVoteOptions, - tableStartVotes, tableVoteOptionResults, tableVoteResults, - tableProposalMetadata).Error - if err != nil { - return err - } - - // Remove decred plugin version record - return tx.Delete(&Version{ - ID: decredplugin.ID, - }).Error -} - -// build the decred plugin cache using the passed in inventory. -// -// This function cannot be called using a transaction because it could -// potentially exceed cockroachdb's transaction size limit. -func (d *decred) build() error { - log.Tracef("decred build") - - // Drop all decred plugin tables - log.Debugf("Dropping decred plugin tables") - tx := d.recordsdb.Begin() - err := d.dropTables(tx) - if err != nil { - tx.Rollback() - return fmt.Errorf("drop tables: %v", err) - } - err = tx.Commit().Error - if err != nil { - return err - } - - // Create decred plugin tables - log.Debugf("Building decred plugin tables") - tx = d.recordsdb.Begin() - err = d.createTables(tx) - if err != nil { - tx.Rollback() - return fmt.Errorf("create tables: %v", err) - } - err = tx.Commit().Error - if err != nil { - return err - } - - // Build the ProposalMetadata cache. This metadata is already in - // the cache as a MetadataStream with an encoded payload. We need - // to pull the ProposalMetadata out so that it can be queried. Only - // the ProposalMetadata for the most recent version of the proposal - // is saved to the cache. - - log.Debugf("Building proposal metadata cache") - - // Lookup latest version of each record - query := `SELECT a.* - FROM records a - LEFT OUTER JOIN records b - ON a.token = b.token - AND a.version < b.version - WHERE b.token IS NULL` - rows, err := d.recordsdb.Raw(query).Rows() - if err != nil { - return fmt.Errorf("lookup latest records: %v", err) - } - defer rows.Close() - - records := make([]Record, 0, 1024) - for rows.Next() { - var r Record - err := d.recordsdb.ScanRows(rows, &r) - if err != nil { - return err - } - records = append(records, r) - } - if err = rows.Err(); err != nil { - return err - } - - // Compile a list of record primary keys - keys := make([]string, 0, len(records)) - for _, v := range records { - keys = append(keys, v.Key) - } - - // Lookup the files and metadata streams for each record - err = d.recordsdb. - Preload("Files"). - Preload("Metadata"). - Where(keys). - Find(&records). - Error - if err != nil { - return fmt.Errorf("lookup record metadata: %v", err) - } - - for _, v := range records { - pm, err := proposalMetadataNew(v) - if err != nil { - return err - } - if pm.Name == "" { - // XXX we cannot return an error here until the plugin - // architecture is sorted out. Right now, politeiad registers - // the decred plugin by default. CMS needs a way to register - // just the functionality it needs so that proposal specific - // tables do not get built for CMS. - // return fmt.Errorf("no proposal user metadata found %v", - // v.Token) - - continue - } - - err = d.recordsdb.Create(pm).Error - if err != nil { - return fmt.Errorf("create: %v", err) - } - } - - return nil -} - -// Build drops all existing decred plugin tables from the database, recreates -// them, then uses the passed in inventory payload to build the decred plugin -// cache. -func (d *decred) Build(payload string) error { - log.Tracef("decred Build") - - // Build the decred plugin cache. This is not run using - // a transaction because it could potentially exceed - // cockroachdb's transaction size limit. - err := d.build() - if err != nil { - // Remove the version record. This will - // force a rebuild on the next start up. - err1 := d.recordsdb.Delete(&Version{ - ID: decredplugin.ID, - }).Error - if err1 != nil { - panic("the cache is out of sync and will not rebuild" + - "automatically; a rebuild must be forced") - } - } - - return err -} - -// Setup creates the decred plugin tables if they do not already exist. A -// decred plugin version record is inserted into the database during table -// creation. -func (d *decred) Setup() error { - log.Tracef("decred: Setup") - - tx := d.recordsdb.Begin() - err := d.createTables(tx) - if err != nil { - tx.Rollback() - return err - } - - return tx.Commit().Error -} - -// CheckVersion retrieves the decred plugin version record from the database, -// if one exists, and checks that it matches the version of the current decred -// plugin cache implementation. -func (d *decred) CheckVersion() error { - log.Tracef("decred: CheckVersion") - - // Sanity check. Ensure version table exists. - if !d.recordsdb.HasTable(tableVersions) { - return fmt.Errorf("versions table not found") - } - - // Lookup version record. If the version is not found or - // if there is a version mismatch, return an error so - // that the decred plugin cache can be built/rebuilt. - var v Version - err := d.recordsdb. - Where("id = ?", decredplugin.ID). - Find(&v). - Error - if err == gorm.ErrRecordNotFound { - log.Debugf("version record not found for ID '%v'", - decredplugin.ID) - err = cache.ErrNoVersionRecord - } else if v.Version != decredVersion { - log.Debugf("version mismatch for ID '%v': got %v, want %v", - decredplugin.ID, v.Version, decredVersion) - err = cache.ErrWrongVersion - } - - return err -} - -// newDecredPlugin returns a cache decred plugin context. -func newDecredPlugin(db *gorm.DB, p cache.Plugin) *decred { - log.Tracef("newDecredPlugin") - return &decred{ - recordsdb: db, - version: decredVersion, - settings: p.Settings, - } -} diff --git a/politeiad/cache/cockroachdb/log.go b/politeiad/cache/cockroachdb/log.go deleted file mode 100644 index ac94cb0e9..000000000 --- a/politeiad/cache/cockroachdb/log.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) 2013-2015 The btcsuite developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package cockroachdb - -import "github.com/decred/slog" - -// log is a logger that is initialized with no output filters. This -// means the package will not perform any logging by default until the caller -// requests it. -var log = slog.Disabled - -// DisableLog disables all library log output. Logging output is disabled -// by default until either UseLogger or SetLogWriter are called. -func DisableLog() { - log = slog.Disabled -} - -// UseLogger uses a specified Logger to output package logging info. -// This should be used in preference to SetLogWriter if the caller is also -// using slog. -func UseLogger(logger slog.Logger) { - log = logger -} diff --git a/politeiad/cache/cockroachdb/models.go b/politeiad/cache/cockroachdb/models.go deleted file mode 100644 index f1eae13d7..000000000 --- a/politeiad/cache/cockroachdb/models.go +++ /dev/null @@ -1,372 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package cockroachdb - -// KeyValue is a generic key-value store for the cache. -type KeyValue struct { - Key string `gorm:"primary_key"` - Value []byte `gorm:"not null"` -} - -// TableName returns the name of the KeyValue table. -func (KeyValue) TableName() string { - return tableKeyValue -} - -// Version describes the version of a record or plugin that the database is -// currently using. -type Version struct { - ID string `gorm:"primary_key"` // Primary key - Version string `gorm:"not null"` // Version - Timestamp int64 `gorm:"not null"` // UNIX timestamp of record creation -} - -// TableName returns the name of the Version database table. -func (Version) TableName() string { - return tableVersions -} - -// File describes an individual file that is part of the record. -type File struct { - Key uint `gorm:"primary_key"` // Primary key - RecordKey string `gorm:"not null"` // Record foreign key - Name string `gorm:"not null"` // Basename of the file - MIME string `gorm:"not null"` // MIME type - Digest string `gorm:"not null;size:64"` // SHA256 of decoded Payload - Payload string `gorm:"not null"` // base64 encoded file -} - -// TableName returns the name of the File database table. -func (File) TableName() string { - return tableFiles -} - -// MetadataStream identifies a metadata stream by its identity. -type MetadataStream struct { - Key uint `gorm:"primary_key"` // Primary key - RecordKey string `gorm:"not null"` // Record foreign key - ID uint64 `gorm:"not null"` // Stream identity - Payload string `gorm:"not null"` // String encoded metadata -} - -// TableName returns the name of the MetadataStream database table. -func (MetadataStream) TableName() string { - return tableMetadataStreams -} - -// Record is an entire record and it's content. -type Record struct { - Key string `gorm:"primary_key"` // Primary key (token+version) - Token string `gorm:"not null"` // Censorship token - TokenPrefix string `gorm:"not null;size:7;index"` // Prefix of token used for lookups - Version uint64 `gorm:"not null"` // Version of files - Status int `gorm:"not null"` // Current status - Timestamp int64 `gorm:"not null"` // UNIX timestamp of last updated - Merkle string `gorm:"not null;size:64"` // Merkle root of all files in record - Signature string `gorm:"not null;size:128"` // Server signature of merkle+token - - Metadata []MetadataStream `gorm:"foreignkey:RecordKey"` // User provided metadata - Files []File `gorm:"foreignkey:RecordKey"` // User provided files -} - -// TableName returns the name of the Record database table. -func (Record) TableName() string { - return tableRecords -} - -// ProposalMetadata represents user defined proposal metadata. -// -// This data is already saved to the cache as a MetadataStream with an encoded -// payload. The ProposalMetadata duplicates existing data, but is necessary so -// that the metadata fields can be queried. ProposalMetadata is only saved for -// the most recent proposal version since this is the only metadata that -// currently needs to be queried. -// -// This is a decred plugin model. -type ProposalMetadata struct { - Token string `gorm:"primary_key"` // Censorship token - Name string `gorm:"not null"` // Proposal name - LinkTo string `gorm:""` // Token of proposal to link to - LinkBy int64 `gorm:""` // UNIX timestamp of RFP deadline -} - -// Comment represents a record comment, including all of the server side -// metadata. -// -// This is a decred plugin model. -type Comment struct { - Key string `gorm:"primary_key"` // Primary key (token+commentID) - Token string `gorm:"not null"` // Censorship token - ParentID string `gorm:"not null"` // Parent comment ID - Comment string `gorm:"not null"` // Comment - Signature string `gorm:"not null;size:128"` // Client Signature of Token+ParentID+Comment - PublicKey string `gorm:"not null;size:64"` // Pubkey used for Signature - CommentID string `gorm:"not null"` // Comment ID - Receipt string `gorm:"not null"` // Server signature of the client Signature - Timestamp int64 `gorm:"not null"` // Received UNIX timestamp - Censored bool `gorm:"not null"` // Has this comment been censored -} - -// TableName returns the name of the Comment database table. -func (Comment) TableName() string { - return tableComments -} - -// LikeComment describes a comment upvote/downvote. The server side metadata -// is not included. -// -// This is a decred plugin model. -type LikeComment struct { - Key uint `gorm:"primary_key"` // Primary key - Token string `gorm:"not null"` // Censorship token - CommentID string `gorm:"not null"` // Comment ID - Action string `gorm:"not null;size:2"` // Up or downvote (1, -1) - Signature string `gorm:"not null;size:128"` // Client Signature of Token+CommentID+Action - PublicKey string `gorm:"not null;size:64"` // Public key used for Signature -} - -// TableName returns the name of the LikeComment database table. -func (LikeComment) TableName() string { - return tableCommentLikes -} - -// AuthorizeVote is used to indicate that a record has been finalized and is -// ready to be voted on. -// -// This is a decred plugin model. -type AuthorizeVote struct { - Key string `gorm:"primary_key"` // Primary key (token+version) - Token string `gorm:"not null"` // Censorship token - Version uint64 `gorm:"not null"` // Version of files - Action string `gorm:"not null"` // Authorize or revoke - Signature string `gorm:"not null;size:128"` // Signature of token+version+action - PublicKey string `gorm:"not null;size:64"` // Pubkey used for signature - Receipt string `gorm:"not null;size:128"` // Server signature of client signature - Timestamp int64 `gorm:"not null"` // Received UNIX timestamp -} - -// TableName returns the name of the AuthorizeVote database table. -func (AuthorizeVote) TableName() string { - return tableAuthorizeVotes -} - -// VoteOption describes a single vote option. -// -// This is a decred plugin model. -type VoteOption struct { - Key uint `gorm:"primary_key"` // Primary key - Token string `gorm:"not null"` // StartVote foreign key - ID string `gorm:"not null"` // Single unique word identifying vote (e.g. yes) - Description string `gorm:"not null"` // Longer description of the vote - Bits uint64 `gorm:"not null"` // Bits used for this option -} - -// TableName returns the name of the VoteOption database table. -func (VoteOption) TableName() string { - return tableVoteOptions -} - -// StartVote records the details of a proposal vote. -// -// ProposalVersion will only be present when StartVote version is >= 2 since -// the decredplugin VoteV1 struct does not contain the proposal version. -// -// QuorumPercentage is the percent of eligible votes required for a quorum. -// -// PassPercentage is the percent of total votes required for the proposal to -// be considered approved. -// -// The data contained in the cache StartVote includes the decredplugin -// StartVote and StartVoteReply mdstreams. These mdstreams are not saved in the -// cache as separate Record.Metadata for the given proposal. This means that -// this mdstream data will not be returned when a proposal record is fetched -// from the cache. The cache StartVote must be queried directly to obtain this -// data. -// -// This is a decred plugin model. -type StartVote struct { - Token string `gorm:"primary_key"` // Censorship token - Version uint `gorm:"not null"` // StartVote struct version - ProposalVersion uint32 `` // Prop version being voted on - Type int `gorm:"not null"` // Vote type - Mask uint64 `gorm:"not null"` // Valid votebits - Duration uint32 `gorm:"not null"` // Duration in blocks - QuorumPercentage uint32 `gorm:"not null"` // Quorum requirement - PassPercentage uint32 `gorm:"not null"` // Approval requirement - Options []VoteOption `gorm:"foreignkey:Token"` // Vote option - PublicKey string `gorm:"not null;size:64"` // Key used for signature - Signature string `gorm:"not null;size:128"` // Signature - StartBlockHeight uint32 `gorm:"not null"` // Block height - StartBlockHash string `gorm:"not null"` // Block hash - EndHeight uint32 `gorm:"not null"` // Height of vote end - EligibleTickets string `gorm:"not null"` // Valid voting tickets - EligibleTicketCount int `gorm:"not null"` // Number of eligible tickets -} - -// TableName returns the name of the StartVote database table. -func (StartVote) TableName() string { - return tableStartVotes -} - -// CastVote records a signed vote. -// -// This is a decred plugin model. -type CastVote struct { - Key uint `gorm:"primary_key"` // Primary key - Token string `gorm:"not null"` // Censorship token - Ticket string `gorm:"not null"` // Ticket ID - VoteBit string `gorm:"not null"` // Hex encoded vote bit that was selected - Signature string `gorm:"not null;size:130"` // Signature of Token+Ticket+VoteBit - - // TokenVoteBit is the Token+VoteBit. Indexing TokenVoteBit allows - // for quick lookups of the number of votes cast for each vote bit. - TokenVoteBit string `gorm:"no null;index"` -} - -// TableName returns the name of the CastVote database table. -func (CastVote) TableName() string { - return tableCastVotes -} - -// VoteOptionResults records the vote result for a vote option. A -// VoteOptionResult should only be created once the proposal vote has finished. -// -// This is a decred plugin model. -type VoteOptionResult struct { - Key string `gorm:"primary_key"` // Primary key (token+votebit) - Token string `gorm:"not null"` // Censorship token (VoteResults foreign key) - Votes uint64 `gorm:"not null"` // Number of votes cast for this option - Option VoteOption `gorm:"not null"` // Vote option - OptionKey uint `gorm:"not null"` // VoteOption foreign key -} - -// TableName returns the name of the VoteOptionResult database table. -func (VoteOptionResult) TableName() string { - return tableVoteOptionResults -} - -// VoteResults records the tallied vote results for a proposal and whether the -// vote was approved/rejected. A vote result entry should only be created once -// the voting period has ended. The vote results table is lazy loaded. -// -// This is a decred plugin model. -type VoteResults struct { - Token string `gorm:"primary_key"` // Censorship token - Approved bool `gorm:"not null"` // Vote was approved - Results []VoteOptionResult `gorm:"foreignkey:Token"` // Results for the vote options -} - -// TableName returns the name of the VoteResults database table. -func (VoteResults) TableName() string { - return tableVoteResults -} - -// VoteDCCOption describes a single vote option. -// -// This is a cms plugin model. -type VoteDCCOption struct { - Key uint `gorm:"primary_key"` // Primary key - Token string `gorm:"not null"` // StartVote foreign key - ID string `gorm:"not null"` // Single unique word identifying vote (e.g. yes) - Description string `gorm:"not null"` // Longer description of the vote - Bits uint64 `gorm:"not null"` // Bits used for this option -} - -// TableName returns the name of the VoteOption database table. -func (VoteDCCOption) TableName() string { - return tableVoteDCCOptions -} - -// StartDCCVote records the details of a dcc proposal vote. -// -// This is a cms plugin model. -type StartDCCVote struct { - Token string `gorm:"primary_key"` // Censorship token - Version uint64 `gorm:"not null"` // Version of files - Mask uint64 `gorm:"not null"` // Valid votebits - Duration uint32 `gorm:"not null"` // Duration in blocks - QuorumPercentage uint32 `gorm:"not null"` // Percent of eligible votes required for quorum - PassPercentage uint32 `gorm:"not null"` // Percent of total votes required to pass - Options []VoteDCCOption `gorm:"foreignkey:Token"` // Vote option - PublicKey string `gorm:"not null;size:64"` // Key used for signature - Signature string `gorm:"not null;size:128"` // Signature of Votehash - StartBlockHeight uint32 `gorm:"not null"` // Block height - StartBlockHash string `gorm:"not null"` // Block hash - EndHeight uint32 `gorm:"not null"` // Height of vote end - EligibleUserIDs []DCCUserWeight `gorm:"foreignkey:Token"` // Valid user weights for DCC Vote -} - -// TableName returns the name of the StartDCCVote database table. -func (StartDCCVote) TableName() string { - return tableStartDCCVotes -} - -// CastDCCVote records a signed dcc vote. -// -// This is a cms plugin model. -type CastDCCVote struct { - Key uint `gorm:"primary_key"` // Primary key - Token string `gorm:"not null"` // Censorship token - UserID string `gorm:"not null"` // User ID - VoteBit string `gorm:"not null"` // Hex encoded vote bit that was selected - Signature string `gorm:"not null;size:130"` // Signature of Token+Ticket+VoteBit - - // TokenVoteBit is the Token+VoteBit. Indexing TokenVoteBit allows - // for quick lookups of the number of votes cast for each vote bit. - TokenVoteBit string `gorm:"no null;index"` -} - -// TableName returns the name of the CastDCCVote database table. -func (CastDCCVote) TableName() string { - return tableCastDCCVotes -} - -// VoteDCCOptionResult records the vote result for a vote option. A -// VoteDCCOptionResult should only be created once the dcc vote has finished. -// -// This is a cms plugin model. -type VoteDCCOptionResult struct { - Key string `gorm:"primary_key"` // Primary key (token+votebit) - Token string `gorm:"not null"` // Censorship token (VoteResults foreign key) - Votes uint64 `gorm:"not null"` // Number of votes cast for this option - Option VoteDCCOption `gorm:"not null"` // Vote option - OptionKey uint `gorm:"not null"` // VoteOption foreign key -} - -// TableName returns the name of the VoteOptionResult database table. -func (VoteDCCOptionResult) TableName() string { - return tableVoteDCCOptionResults -} - -// VoteDCCResults records the tallied vote results for a dcc and whether the -// vote was approved/rejected. A vote result entry should only be created once -// the voting period has ended. The vote results table is lazy loaded. -// -// This is a cms plugin model. -type VoteDCCResults struct { - Token string `gorm:"primary_key"` // Censorship tokenba - Approved bool `gorm:"not null"` // Vote was approved - Results []VoteDCCOptionResult `gorm:"foreignkey:Token"` // Results for the vote options -} - -// TableName returns the name of the VoteResults database table. -func (VoteDCCResults) TableName() string { - return tableVoteDCCResults -} - -// DCCUserWeight records a given userid's weight for a given dcc proposal token. -// -// This is a cms plugin model. -type DCCUserWeight struct { - Key string `gorm:"primary_key"` // Primary Key (token + userid) - Token string `gorm:"not null"` // StartDCCVote foreign key - UserID string `gorm:"not null"` // User ID - Weight int64 `gorm:"not null"` // Weight of User -} - -// TableName returns the name of the DCCUserWeight database table. -func (DCCUserWeight) TableName() string { - return tableDCCUserWeights -} diff --git a/politeiad/cache/testcache/decred.go b/politeiad/cache/testcache/decred.go deleted file mode 100644 index a3f304869..000000000 --- a/politeiad/cache/testcache/decred.go +++ /dev/null @@ -1,327 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package testcache - -import ( - "encoding/base64" - "encoding/json" - "fmt" - "strconv" - - "github.com/decred/politeia/decredplugin" - decred "github.com/decred/politeia/decredplugin" - "github.com/decred/politeia/mdstream" - www "github.com/decred/politeia/politeiawww/api/www/v1" -) - -func (c *testcache) getComments(payload string) (string, error) { - gc, err := decred.DecodeGetComments([]byte(payload)) - if err != nil { - return "", err - } - - c.RLock() - defer c.RUnlock() - - gcrb, err := decred.EncodeGetCommentsReply( - decred.GetCommentsReply{ - Comments: c.comments[gc.Token], - }) - if err != nil { - return "", err - } - - return string(gcrb), nil -} - -func (c *testcache) authorizeVote(cmdPayload, replyPayload string) (string, error) { - av, err := decred.DecodeAuthorizeVote([]byte(cmdPayload)) - if err != nil { - return "", err - } - - avr, err := decred.DecodeAuthorizeVoteReply([]byte(replyPayload)) - if err != nil { - return "", err - } - - av.Receipt = avr.Receipt - av.Timestamp = avr.Timestamp - - c.Lock() - defer c.Unlock() - - _, ok := c.authorizeVotes[av.Token] - if !ok { - c.authorizeVotes[av.Token] = make(map[string]decred.AuthorizeVote) - } - - c.authorizeVotes[av.Token][avr.RecordVersion] = *av - - return replyPayload, nil -} - -func (c *testcache) startVote(cmdPayload, replyPayload string) (string, error) { - sv, err := decred.DecodeStartVoteV2([]byte(cmdPayload)) - if err != nil { - return "", err - } - - svr, err := decred.DecodeStartVoteReply([]byte(replyPayload)) - if err != nil { - return "", err - } - - // Version must be added to the StartVote. This is done by - // politeiad but the updated StartVote does not travel to the - // cache. - sv.Version = decred.VersionStartVote - - c.Lock() - defer c.Unlock() - - // Store start vote data - c.startVotes[sv.Vote.Token] = *sv - c.startVoteReplies[sv.Vote.Token] = *svr - - return replyPayload, nil -} - -func (c *testcache) voteDetails(payload string) (string, error) { - vd, err := decred.DecodeVoteDetails([]byte(payload)) - if err != nil { - return "", err - } - - c.Lock() - defer c.Unlock() - - // Lookup the latest record version - r, err := c.record(vd.Token) - if err != nil { - return "", err - } - - // Prepare reply - _, ok := c.authorizeVotes[vd.Token] - if !ok { - c.authorizeVotes[vd.Token] = make(map[string]decred.AuthorizeVote) - } - - sv := c.startVotes[vd.Token] - svb, err := decredplugin.EncodeStartVoteV2(sv) - if err != nil { - return "", err - } - - vdb, err := decred.EncodeVoteDetailsReply( - decred.VoteDetailsReply{ - AuthorizeVote: c.authorizeVotes[vd.Token][r.Version], - StartVote: decredplugin.StartVote{ - Version: sv.Version, - Payload: string(svb), - }, - StartVoteReply: c.startVoteReplies[vd.Token], - }) - if err != nil { - return "", err - } - - return string(vdb), nil -} - -func (c *testcache) voteSummaryReply(token string) (*decred.VoteSummaryReply, error) { - c.RLock() - defer c.RUnlock() - - r, err := c.record(token) - if err != nil { - return nil, err - } - - av := c.authorizeVotes[token][r.Version] - sv := c.startVotes[token] - - var duration uint32 - svr, ok := c.startVoteReplies[token] - if ok { - start, err := strconv.ParseUint(svr.StartBlockHeight, 10, 32) - if err != nil { - return nil, err - } - end, err := strconv.ParseUint(svr.EndHeight, 10, 32) - if err != nil { - return nil, err - } - duration = uint32(end - start) - } - - vsr := decred.VoteSummaryReply{ - Authorized: av.Action == decred.AuthVoteActionAuthorize, - Type: sv.Vote.Type, - Duration: duration, - EndHeight: svr.EndHeight, - EligibleTicketCount: 0, - QuorumPercentage: sv.Vote.QuorumPercentage, - PassPercentage: sv.Vote.PassPercentage, - Results: []decred.VoteOptionResult{}, - } - - return &vsr, nil -} - -func (c *testcache) voteSummary(cmdPayload string) (string, error) { - vs, err := decred.DecodeVoteSummary([]byte(cmdPayload)) - if err != nil { - return "", err - } - vsr, err := c.voteSummaryReply(vs.Token) - if err != nil { - return "", err - } - reply, err := decred.EncodeVoteSummaryReply(*vsr) - if err != nil { - return "", err - } - return string(reply), nil -} - -func (c *testcache) voteSummaries(cmdPayload string) (string, error) { - bvs, err := decred.DecodeBatchVoteSummary([]byte(cmdPayload)) - if err != nil { - return "", err - } - - s := make(map[string]decred.VoteSummaryReply, len(bvs.Tokens)) - for _, token := range bvs.Tokens { - vsr, err := c.voteSummaryReply(token) - if err != nil { - return "", err - } - s[token] = *vsr - } - - reply, err := decred.EncodeBatchVoteSummaryReply( - decred.BatchVoteSummaryReply{ - Summaries: s, - }) - if err != nil { - return "", err - } - - return string(reply), nil -} - -// findLinkedFrom returns the tokens of any proposals that have linked to -// the given proposal token. -func (c *testcache) findLinkedFrom(token string) ([]string, error) { - linkedFrom := make([]string, 0, len(c.records)) - - // Check all records in the cache to see if they're linked to the - // provided token. - for _, allVersions := range c.records { - // Get the latest version of the proposal - r := allVersions[strconv.Itoa(len(allVersions))] - - // Extract LinkTo from the ProposalMetadata file - for _, f := range r.Files { - if f.Name == mdstream.FilenameProposalMetadata { - b, err := base64.StdEncoding.DecodeString(f.Payload) - if err != nil { - return nil, err - } - var pm www.ProposalMetadata - err = json.Unmarshal(b, &pm) - if err != nil { - return nil, err - } - if pm.LinkTo == token { - // This proposal links to the provided token - linkedFrom = append(linkedFrom, r.CensorshipRecord.Token) - } - - // No need to continue - break - } - } - } - - return linkedFrom, nil -} - -func (c *testcache) linkedFrom(cmdPayload string) (string, error) { - lf, err := decredplugin.DecodeLinkedFrom([]byte(cmdPayload)) - if err != nil { - return "", err - } - - c.RLock() - defer c.RUnlock() - - linkedFromBatch := make(map[string][]string, len(lf.Tokens)) // [token]linkedFrom - for _, token := range lf.Tokens { - linkedFrom, err := c.findLinkedFrom(token) - if err != nil { - return "", err - } - linkedFromBatch[token] = linkedFrom - } - - lfr := decredplugin.LinkedFromReply{ - LinkedFrom: linkedFromBatch, - } - reply, err := decredplugin.EncodeLinkedFromReply(lfr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (c *testcache) getNumComments(payload string) (string, error) { - gnc, err := decred.DecodeGetNumComments([]byte(payload)) - if err != nil { - return "", err - } - - numComments := make(map[string]int) - for _, token := range gnc.Tokens { - numComments[token] = len(c.comments[token]) - } - - gncr, err := decred.EncodeGetNumCommentsReply( - decred.GetNumCommentsReply{ - NumComments: numComments, - }) - - if err != nil { - return "", err - } - - return string(gncr), nil -} - -func (c *testcache) decredExec(cmd, cmdPayload, replyPayload string) (string, error) { - switch cmd { - case decred.CmdGetComments: - return c.getComments(cmdPayload) - case decred.CmdAuthorizeVote: - return c.authorizeVote(cmdPayload, replyPayload) - case decred.CmdStartVote: - return c.startVote(cmdPayload, replyPayload) - case decred.CmdVoteDetails: - return c.voteDetails(cmdPayload) - case decred.CmdGetNumComments: - return c.getNumComments(cmdPayload) - case decred.CmdVoteSummary: - return c.voteSummary(cmdPayload) - case decred.CmdBatchVoteSummary: - return c.voteSummaries(cmdPayload) - case decred.CmdLinkedFrom: - return c.linkedFrom(cmdPayload) - } - - return "", fmt.Errorf("invalid cache plugin command") -} diff --git a/politeiad/cache/testcache/testcache.go b/politeiad/cache/testcache/testcache.go deleted file mode 100644 index ffc7a76f7..000000000 --- a/politeiad/cache/testcache/testcache.go +++ /dev/null @@ -1,277 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package testcache - -import ( - "fmt" - "strconv" - "sync" - - decred "github.com/decred/politeia/decredplugin" - "github.com/decred/politeia/politeiad/cache" -) - -// testcache provides a implementation of the cache interface that stores -// records in memory and that can be used for testing. -type testcache struct { - sync.RWMutex - records map[string]map[string]cache.Record // [token][version]Record - - // Decred plugin - comments map[string][]decred.Comment // [token][]Comment - authorizeVotes map[string]map[string]decred.AuthorizeVote // [token][version]AuthorizeVote - startVotes map[string]decred.StartVoteV2 // [token]StartVote - startVoteReplies map[string]decred.StartVoteReply // [token]StartVoteReply -} - -// NewRecords adds a record to the cache. -func (c *testcache) NewRecord(r cache.Record) error { - c.Lock() - defer c.Unlock() - - token := r.CensorshipRecord.Token - _, ok := c.records[token] - if !ok { - c.records[token] = make(map[string]cache.Record) - } - - c.records[token][r.Version] = r - return nil -} - -// record returns the most recent version of a record from the memory cache. -// -// This function must be called with the lock held. -func (c *testcache) record(token string) (*cache.Record, error) { - records, ok := c.records[token] - if !ok { - return nil, cache.ErrRecordNotFound - } - - var latest int - for version := range records { - v, err := strconv.Atoi(version) - if err != nil { - return nil, fmt.Errorf("parse version '%v' failed: %v", - version, err) - } - - if v > latest { - latest = v - } - } - - // Sanity check - if latest == 0 { - return nil, cache.ErrRecordNotFound - } - - r := records[strconv.Itoa(latest)] - return &r, nil -} - -// Record returns the most recent version of the record. -func (c *testcache) Record(token string) (*cache.Record, error) { - c.RLock() - defer c.RUnlock() - - return c.record(token) -} - -// getTokenFromPrefix searches the cache for a token that is prefixed by the -// specified prefix. -// -// This function must be called with the lock held. -func (c *testcache) getTokenFromPrefix(prefix string) (string, error) { - for token := range c.records { - if len(token) < len(prefix) { - continue - } - if prefix == token[0:len(prefix)] { - return token, nil - } - } - - return "", cache.ErrRecordNotFound -} - -// RecordByPrefix returns the most recent version of the record based on its -// prefix. -func (c *testcache) RecordByPrefix(prefix string) (*cache.Record, error) { - c.RLock() - defer c.RUnlock() - - token, err := c.getTokenFromPrefix(prefix) - if err != nil { - return nil, err - } - - return c.record(token) -} - -// Records returns the most recent version of a set of records. -func (c *testcache) Records(tokens []string, fetchFiles bool) (map[string]cache.Record, error) { - c.RLock() - defer c.RUnlock() - - records := make(map[string]cache.Record, len(tokens)) // [token]Record - for _, token := range tokens { - r, err := c.record(token) - if err != nil { - return nil, err - } - records[token] = *r - } - - return records, nil -} - -// recordVersion retreives a specific version of a record from the memory -// cache. -// -// This function must be called with the lock held. -func (c *testcache) recordVersion(token, version string) (*cache.Record, error) { - _, ok := c.records[token] - if !ok { - return nil, cache.ErrRecordNotFound - } - - r, ok := c.records[token][version] - if !ok { - return nil, cache.ErrRecordNotFound - } - - return &r, nil -} - -// RecordVersion returns a specific version of a record. -func (c *testcache) RecordVersion(token, version string) (*cache.Record, error) { - c.RLock() - defer c.RUnlock() - - return c.recordVersion(token, version) -} - -// UpdateRecord updates a record in the cache. -func (c *testcache) UpdateRecord(r cache.Record) error { - c.Lock() - defer c.Unlock() - - token := r.CensorshipRecord.Token - _, ok := c.records[token] - if !ok { - return cache.ErrRecordNotFound - } - - c.records[token][r.Version] = r - return nil -} - -// UpdateRecordStatus updates the status of a record. -func (c *testcache) UpdateRecordStatus(token, version string, status cache.RecordStatusT, timestamp int64, metadata []cache.MetadataStream) error { - c.Lock() - defer c.Unlock() - - // Lookup record - r, err := c.recordVersion(token, version) - if err != nil { - return err - } - - // Update record - r.Status = status - r.Timestamp = timestamp - r.Metadata = metadata - c.records[token][version] = *r - - return nil -} - -// UpdateRecordMetadata is a stub to satisfy the cache interface. -func (c *testcache) UpdateRecordMetadata(token string, md []cache.MetadataStream) error { - return nil -} - -// inventory returns all records in the cache. -func (c *testcache) inventory() ([]cache.Record, error) { - records := make([]cache.Record, 0, len(c.records)) - version := "1" - - for token := range c.records { - records = append(records, c.records[token][version]) - } - - return records, nil -} - -// Inventory returns all records in the cache. -func (c *testcache) Inventory() ([]cache.Record, error) { - c.RLock() - defer c.RUnlock() - - return c.inventory() -} - -// InventoryStats is a stub to satisfy the cache interface. -func (c *testcache) InventoryStats() (*cache.InventoryStats, error) { - return &cache.InventoryStats{}, nil -} - -// Setup is a stub to satisfy the cache interface. -func (c *testcache) Setup() error { - return nil -} - -// Build is a stub to satisfy the cache interface. -func (c *testcache) Build(records []cache.Record) error { - return nil -} - -func (c *testcache) RegisterPlugin(p cache.Plugin) error { - return nil -} - -// PluginSetup is a stub to satisfy the cache interface. -func (c *testcache) PluginSetup(id string) error { - return nil -} - -// PluginBuild is a stub to satisfy the cache interface. -func (c *testcache) PluginBuild(id, payload string) error { - return nil -} - -// PluginExec is a stub to satisfy the cache interface. -func (c *testcache) PluginExec(pc cache.PluginCommand) (*cache.PluginCommandReply, error) { - var payload string - var err error - switch pc.ID { - case decred.ID: - payload, err = c.decredExec(pc.Command, - pc.CommandPayload, pc.ReplyPayload) - if err != nil { - return nil, err - } - } - return &cache.PluginCommandReply{ - ID: pc.ID, - Command: pc.Command, - Payload: payload, - }, nil -} - -// Close is a stub to satisfy the cache interface. -func (c *testcache) Close() {} - -// New returns a new testcache context. -func New() *testcache { - return &testcache{ - records: make(map[string]map[string]cache.Record), - comments: make(map[string][]decred.Comment), - authorizeVotes: make(map[string]map[string]decred.AuthorizeVote), - startVotes: make(map[string]decred.StartVoteV2), - startVoteReplies: make(map[string]decred.StartVoteReply), - } -} diff --git a/politeiad/config.go b/politeiad/config.go index 886a578e2..df58168dd 100644 --- a/politeiad/config.go +++ b/politeiad/config.go @@ -6,12 +6,8 @@ package main import ( - "crypto/tls" - "crypto/x509" "encoding/base64" - "encoding/pem" "fmt" - "io/ioutil" "net" "os" "os/user" @@ -42,18 +38,13 @@ const ( defaultMainnetDcrdata = "dcrdata.decred.org:443" defaultTestnetDcrdata = "testnet.decred.org:443" - // Currently available modes to run politeia, by default piwww, is used. - politeiaWWWMode = "piwww" - cmsWWWMode = "cmswww" - - defaultWWWMode = politeiaWWWMode - // Backend options backendGit = "git" backendTlog = "tlog" - defaultBackend = backendGit + defaultBackend = backendTlog - defaultTrillianHost = "localhost:8090" + defaultTrillianHostUnvetted = "localhost:8090" + defaultTrillianHostVetted = "localhost:8094" ) var ( @@ -74,40 +65,36 @@ var runServiceCommand func(string) error // // See loadConfig for details on the configuration load process. type config struct { - HomeDir string `short:"A" long:"appdata" description:"Path to application home directory"` - ShowVersion bool `short:"V" long:"version" description:"Display version information and exit"` - ConfigFile string `short:"C" long:"configfile" description:"Path to configuration file"` - DataDir string `short:"b" long:"datadir" description:"Directory to store data"` - LogDir string `long:"logdir" description:"Directory to log output."` - TestNet bool `long:"testnet" description:"Use the test network"` - SimNet bool `long:"simnet" description:"Use the simulation test network"` - Profile string `long:"profile" description:"Enable HTTP profiling on given port -- NOTE port must be between 1024 and 65536"` - CPUProfile string `long:"cpuprofile" description:"Write CPU profile to the specified file"` - MemProfile string `long:"memprofile" description:"Write mem profile to the specified file"` - DebugLevel string `short:"d" long:"debuglevel" description:"Logging level for all subsystems {trace, debug, info, warn, error, critical} -- You may also specify =,=,... to set the log level for individual subsystems -- Use show to list available subsystems"` - Listeners []string `long:"listen" description:"Add an interface/port to listen for connections (default all interfaces port: 49152, testnet: 59152)"` - Version string - HTTPSCert string `long:"httpscert" description:"File containing the https certificate file"` - HTTPSKey string `long:"httpskey" description:"File containing the https certificate key"` - RPCUser string `long:"rpcuser" description:"RPC user name for privileged commands"` - RPCPass string `long:"rpcpass" description:"RPC password for privileged commands"` - DcrtimeHost string `long:"dcrtimehost" description:"Dcrtime ip:port"` - DcrtimeCert string `long:"dcrtimecert" description:"File containing the https certificate file for dcrtimehost"` - EnableCache bool `long:"enablecache" description:"Enable the external cache"` - CacheHost string `long:"cachehost" description:"Cache ip:port"` - CacheRootCert string `long:"cacherootcert" description:"File containing the CA certificate for the cache"` - CacheCert string `long:"cachecert" description:"File containing the politeiad client certificate for the cache"` - CacheKey string `long:"cachekey" description:"File containing the politeiad client certificate key for the cache"` - BuildCache bool `long:"buildcache" description:"Build the cache from scratch"` - Identity string `long:"identity" description:"File containing the politeiad identity file"` - GitTrace bool `long:"gittrace" description:"Enable git tracing in logs"` - DcrdataHost string `long:"dcrdatahost" description:"Dcrdata ip:port"` - - Mode string `long:"mode" description:"Mode www runs as. Supported values: piwww, cmswww"` - Backend string `long:"backend"` - TrillianHost string `long:"trillianhost"` - TrillianKey string `long:"trilliankey"` - EncryptionKey string `long:"encryptionkey"` + HomeDir string `short:"A" long:"appdata" description:"Path to application home directory"` + ShowVersion bool `short:"V" long:"version" description:"Display version information and exit"` + ConfigFile string `short:"C" long:"configfile" description:"Path to configuration file"` + DataDir string `short:"b" long:"datadir" description:"Directory to store data"` + LogDir string `long:"logdir" description:"Directory to log output."` + TestNet bool `long:"testnet" description:"Use the test network"` + SimNet bool `long:"simnet" description:"Use the simulation test network"` + Profile string `long:"profile" description:"Enable HTTP profiling on given port -- NOTE port must be between 1024 and 65536"` + CPUProfile string `long:"cpuprofile" description:"Write CPU profile to the specified file"` + MemProfile string `long:"memprofile" description:"Write mem profile to the specified file"` + DebugLevel string `short:"d" long:"debuglevel" description:"Logging level for all subsystems {trace, debug, info, warn, error, critical} -- You may also specify =,=,... to set the log level for individual subsystems -- Use show to list available subsystems"` + Listeners []string `long:"listen" description:"Add an interface/port to listen for connections (default all interfaces port: 49152, testnet: 59152)"` + Version string + HTTPSCert string `long:"httpscert" description:"File containing the https certificate file"` + HTTPSKey string `long:"httpskey" description:"File containing the https certificate key"` + RPCUser string `long:"rpcuser" description:"RPC user name for privileged commands"` + RPCPass string `long:"rpcpass" description:"RPC password for privileged commands"` + DcrtimeHost string `long:"dcrtimehost" description:"Dcrtime ip:port"` + DcrtimeCert string `long:"dcrtimecert" description:"File containing the https certificate file for dcrtimehost"` + Identity string `long:"identity" description:"File containing the politeiad identity file"` + GitTrace bool `long:"gittrace" description:"Enable git tracing in logs"` + DcrdataHost string `long:"dcrdatahost" description:"Dcrdata ip:port"` + + // TODO validate these config params + Backend string `long:"backend"` + TrillianHostUnvetted string `long:"trillianhostunvetted"` + TrillianHostVetted string `long:"trillianhostvetted"` + TrillianKeyUnvetted string `long:"trilliankeyunvetted"` + TrillianKeyVetted string `long:"trilliankeyvetted"` + EncryptionKey string `long:"encryptionkey"` } // serviceOptions defines the configuration options for the daemon as a service @@ -302,17 +289,17 @@ func newConfigParser(cfg *config, so *serviceOptions, options flags.Options) *fl func loadConfig() (*config, []string, error) { // Default config. cfg := config{ - HomeDir: defaultHomeDir, - ConfigFile: defaultConfigFile, - DebugLevel: defaultLogLevel, - DataDir: defaultDataDir, - LogDir: defaultLogDir, - HTTPSKey: defaultHTTPSKeyFile, - HTTPSCert: defaultHTTPSCertFile, - Version: version.String(), - Mode: defaultWWWMode, - Backend: defaultBackend, - TrillianHost: defaultTrillianHost, + HomeDir: defaultHomeDir, + ConfigFile: defaultConfigFile, + DebugLevel: defaultLogLevel, + DataDir: defaultDataDir, + LogDir: defaultLogDir, + HTTPSKey: defaultHTTPSKeyFile, + HTTPSCert: defaultHTTPSCertFile, + Version: version.String(), + Backend: defaultBackend, + TrillianHostUnvetted: defaultTrillianHostUnvetted, + TrillianHostVetted: defaultTrillianHostVetted, } // Service options which are only added on Windows. @@ -481,55 +468,6 @@ func loadConfig() (*config, []string, error) { os.Exit(0) } - // Validate cache options. - if cfg.EnableCache { - switch { - case cfg.CacheHost == "": - return nil, nil, fmt.Errorf("the enablecache param can " + - "not be used without the cachehost param") - case cfg.CacheRootCert == "": - return nil, nil, fmt.Errorf("the enablecache param can " + - "not be used without the cacherootcert param") - case cfg.CacheCert == "": - return nil, nil, fmt.Errorf("the enablecache param can " + - "not be used without the cachecert param") - case cfg.CacheKey == "": - return nil, nil, fmt.Errorf("the enablecache param can " + - "not be used without the cachekey param") - } - - cfg.CacheRootCert = cleanAndExpandPath(cfg.CacheRootCert) - cfg.CacheCert = cleanAndExpandPath(cfg.CacheCert) - cfg.CacheKey = cleanAndExpandPath(cfg.CacheKey) - - // Validate cache root cert. - b, err := ioutil.ReadFile(cfg.CacheRootCert) - if err != nil { - return nil, nil, fmt.Errorf("read cacherootcert: %v", err) - } - block, _ := pem.Decode(b) - if block == nil { - return nil, nil, fmt.Errorf("%s is not a valid certificate", - cfg.CacheRootCert) - } - _, err = x509.ParseCertificate(block.Bytes) - if err != nil { - return nil, nil, fmt.Errorf("parse cacherootcert: %v", err) - } - - // Validate cache key pair. - _, err = tls.LoadX509KeyPair(cfg.CacheCert, cfg.CacheKey) - if err != nil { - return nil, nil, fmt.Errorf("load key pair cachecert "+ - "and cachekey: %v", err) - } - } - - if cfg.BuildCache && !cfg.EnableCache { - return nil, nil, fmt.Errorf("the buildcache param can " + - "not be used without the enablecache param") - } - // Initialize log rotation. After log rotation has been initialized, // the logger variables may be used. initLogRotator(filepath.Join(cfg.LogDir, defaultLogFilename)) @@ -634,16 +572,6 @@ func loadConfig() (*config, []string, error) { log.Warnf("RPC password not set, using random value") } - // Verify mode - switch cfg.Mode { - case cmsWWWMode: - case politeiaWWWMode: - default: - err := fmt.Errorf("invalid mode: %v", cfg.Mode) - fmt.Fprintln(os.Stderr, err) - return nil, nil, err - } - // Warn about missing config file only after all other configuration is // done. This prevents the warning on help messages and invalid // options. Note this should go directly before the return. diff --git a/politeiad/log.go b/politeiad/log.go index 14bf69cd5..3513117bd 100644 --- a/politeiad/log.go +++ b/politeiad/log.go @@ -13,7 +13,6 @@ import ( "github.com/decred/politeia/politeiad/backend/tlogbe" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" - "github.com/decred/politeia/politeiad/cache/cockroachdb" "github.com/decred/slog" "github.com/jrick/logrotate/rotator" ) @@ -47,17 +46,15 @@ var ( log = backendLog.Logger("POLI") gitbeLog = backendLog.Logger("GITB") - tlogbeLog = backendLog.Logger("TLOG") - cacheLog = backendLog.Logger("CACH") + backLog = backendLog.Logger("BACK") storeLog = backendLog.Logger("STOR") pluginsLog = backendLog.Logger("PLGN") ) // Initialize package-global logger variables. func init() { - cockroachdb.UseLogger(cacheLog) gitbe.UseLogger(gitbeLog) - tlogbe.UseLogger(tlogbeLog) + tlogbe.UseLogger(backLog) filesystem.UseLogger(storeLog) plugins.UseLogger(pluginsLog) } @@ -66,8 +63,7 @@ func init() { var subsystemLoggers = map[string]slog.Logger{ "POLI": log, "GITB": gitbeLog, - "TLOG": tlogbeLog, - "CACH": cacheLog, + "BACK": backLog, "STOR": storeLog, "PLGN": pluginsLog, } diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index b586481f0..d63cd491d 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -15,22 +15,15 @@ import ( "net/http/httputil" "os" "os/signal" - "path/filepath" "runtime/debug" - "strconv" "syscall" "time" - "github.com/decred/politeia/cmsplugin" - "github.com/decred/politeia/decredplugin" v1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/gitbe" "github.com/decred/politeia/politeiad/backend/tlogbe" - "github.com/decred/politeia/politeiad/cache" - "github.com/decred/politeia/politeiad/cache/cachestub" - "github.com/decred/politeia/politeiad/cache/cockroachdb" "github.com/decred/politeia/util" "github.com/decred/politeia/util/version" "github.com/gorilla/mux" @@ -46,7 +39,6 @@ const ( // politeia application context. type politeia struct { backend backend.Backend - cache cache.Cache cfg *config router *mux.Router identity *identity.FullIdentity @@ -189,81 +181,6 @@ func (p *politeia) convertBackendRecord(br backend.Record) v1.Record { return pr } -func convertBackendStatusToCache(status backend.MDStatusT) cache.RecordStatusT { - s := cache.RecordStatusInvalid - switch status { - case backend.MDStatusInvalid: - s = cache.RecordStatusInvalid - case backend.MDStatusUnvetted: - s = cache.RecordStatusNotReviewed - case backend.MDStatusVetted: - s = cache.RecordStatusPublic - case backend.MDStatusCensored: - s = cache.RecordStatusCensored - case backend.MDStatusIterationUnvetted: - s = cache.RecordStatusUnreviewedChanges - case backend.MDStatusArchived: - s = cache.RecordStatusArchived - } - return s -} - -func convertBackendPluginToCache(p backend.Plugin) cache.Plugin { - settings := make([]cache.PluginSetting, 0, len(p.Settings)) - for _, s := range p.Settings { - settings = append(settings, cache.PluginSetting{ - Key: s.Key, - Value: s.Value, - }) - } - return cache.Plugin{ - ID: p.ID, - Version: p.Version, - Settings: settings, - } -} - -func convertMDStreamsToCache(ms []backend.MetadataStream) []cache.MetadataStream { - m := make([]cache.MetadataStream, 0, len(ms)) - for _, v := range ms { - m = append(m, cache.MetadataStream{ - ID: v.ID, - Payload: v.Payload, - }) - } - return m -} - -func (p *politeia) convertBackendRecordToCache(r backend.Record) cache.Record { - msg := []byte(r.RecordMetadata.Merkle + r.RecordMetadata.Token) - signature := p.identity.SignMessage(msg) - cr := cache.CensorshipRecord{ - Token: r.RecordMetadata.Token, - Merkle: r.RecordMetadata.Merkle, - Signature: hex.EncodeToString(signature[:]), - } - - files := make([]cache.File, 0, len(r.Files)) - for _, f := range r.Files { - files = append(files, - cache.File{ - Name: f.Name, - MIME: f.MIME, - Digest: f.Digest, - Payload: f.Payload, - }) - } - - return cache.Record{ - Version: r.Version, - Status: convertBackendStatusToCache(r.RecordMetadata.Status), - Timestamp: r.RecordMetadata.Timestamp, - CensorshipRecord: cr, - Metadata: convertMDStreamsToCache(r.Metadata), - Files: files, - } -} - func (p *politeia) respondWithUserError(w http.ResponseWriter, errorCode v1.ErrorStatusT, errorContext []string) { util.RespondWithJSON(w, http.StatusBadRequest, v1.UserErrorReply{ @@ -340,19 +257,6 @@ func (p *politeia) newRecord(w http.ResponseWriter, r *http.Request) { return } - // Update cache. - record := p.convertBackendRecordToCache(backend.Record{ - RecordMetadata: *rm, - Version: "1", - Metadata: md, - Files: files, - }) - err = p.cache.NewRecord(record) - if err != nil { - log.Criticalf("Cache new record failed %v: %v", - record.CensorshipRecord.Token, err) - } - // Prepare reply. signature := p.identity.SignMessage([]byte(rm.Merkle + rm.Token)) @@ -455,25 +359,6 @@ func (p *politeia) updateRecord(w http.ResponseWriter, r *http.Request, vetted b return } - // Update cache. - cr := p.convertBackendRecordToCache(*record) - if vetted { - // Create a new cache entry for new versions. - err := p.cache.NewRecord(cr) - if err != nil { - log.Criticalf("Cache update vetted failed %v: %v", - cr.CensorshipRecord.Token, err) - } - } else { - // Update existing cache entry for new iterations that are not - // new versions. - err = p.cache.UpdateRecord(cr) - if err != nil { - log.Criticalf("Cache update unvetted failed %v: %v", - cr.CensorshipRecord.Token, err) - } - } - // Prepare reply. response := p.identity.SignMessage(challenge) reply := v1.UpdateRecordReply{ @@ -494,37 +379,6 @@ func (p *politeia) updateVetted(w http.ResponseWriter, r *http.Request) { p.updateRecord(w, r, true) } -func (p *politeia) updateReadme(w http.ResponseWriter, r *http.Request) { - var t v1.UpdateReadme - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&t); err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) - return - } - - challenge, err := hex.DecodeString(t.Challenge) - if err != nil || len(challenge) != v1.ChallengeSize { - p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) - return - } - - response := p.identity.SignMessage(challenge) - - reply := v1.UpdateReadmeReply{ - Response: hex.EncodeToString(response[:]), - } - - err = p.backend.UpdateReadme(t.Content) - if err != nil { - errorCode := time.Now().Unix() - log.Errorf("Error updating readme: %v", err) - p.respondWithServerError(w, errorCode) - return - } - - util.RespondWithJSON(w, http.StatusOK, reply) -} - func (p *politeia) getUnvetted(w http.ResponseWriter, r *http.Request) { var t v1.GetUnvetted decoder := json.NewDecoder(r.Body) @@ -552,7 +406,7 @@ func (p *politeia) getUnvetted(w http.ResponseWriter, r *http.Request) { } // Ask backend about the censorship token. - bpr, err := p.backend.GetUnvetted(token) + bpr, err := p.backend.GetUnvetted(token, "") if err == backend.ErrRecordNotFound { reply.Record.Status = v1.RecordStatusNotFound log.Errorf("Get unvetted record %v: token %v not found", @@ -776,15 +630,6 @@ func (p *politeia) setVettedStatus(w http.ResponseWriter, r *http.Request) { return } - // Update cache. - cr := p.convertBackendRecordToCache(*record) - err = p.cache.UpdateRecordStatus(cr.CensorshipRecord.Token, - cr.Version, cr.Status, cr.Timestamp, cr.Metadata) - if err != nil { - log.Criticalf("Cache set vetted status failed %v: %v", - cr.CensorshipRecord.Token, err) - } - // Prepare reply. reply := v1.SetVettedStatusReply{ Response: hex.EncodeToString(response[:]), @@ -847,15 +692,6 @@ func (p *politeia) setUnvettedStatus(w http.ResponseWriter, r *http.Request) { return } - // Update cache. - cr := p.convertBackendRecordToCache(*record) - err = p.cache.UpdateRecordStatus(cr.CensorshipRecord.Token, - cr.Version, cr.Status, cr.Timestamp, cr.Metadata) - if err != nil { - log.Criticalf("Cache set unvetted status failed %v: %v", - cr.CensorshipRecord.Token, err) - } - // Prepare reply. reply := v1.SetUnvettedStatusReply{ Response: hex.EncodeToString(response[:]), @@ -868,17 +704,6 @@ func (p *politeia) setUnvettedStatus(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, reply) } -func (p *politeia) cacheUpdateVettedMetadata(token []byte) error { - r, err := p.backend.GetVetted(token, "") - if err != nil { - return fmt.Errorf("get vetted: %v", err) - } - - m := convertMDStreamsToCache(r.Metadata) - t := hex.EncodeToString(token) - return p.cache.UpdateRecordMetadata(t, m) -} - func (p *politeia) updateVettedMetadata(w http.ResponseWriter, r *http.Request) { var t v1.UpdateVettedMetadata decoder := json.NewDecoder(r.Body) @@ -931,13 +756,6 @@ func (p *politeia) updateVettedMetadata(w http.ResponseWriter, r *http.Request) return } - // Update the cache - err = p.cacheUpdateVettedMetadata(token) - if err != nil { - log.Criticalf("Cache updated vetted metadata failed %x: %v", - token, err) - } - // Reply reply := v1.UpdateVettedMetadataReply{ Response: hex.EncodeToString(response[:]), @@ -990,7 +808,7 @@ func (p *politeia) pluginCommand(w http.ResponseWriter, r *http.Request) { return } - cid, payload, err := p.backend.Plugin(pc.Command, pc.Payload) + cid, payload, err := p.backend.Plugin("", pc.Command, pc.Payload) if err != nil { // Generic internal error. errorCode := time.Now().Unix() @@ -1001,19 +819,6 @@ func (p *politeia) pluginCommand(w http.ResponseWriter, r *http.Request) { return } - // Send plugin command to cache - _, err = p.cache.PluginExec(cache.PluginCommand{ - ID: pc.ID, - Command: pc.Command, - CommandPayload: pc.Payload, - ReplyPayload: payload, - }) - if err != nil { - log.Criticalf("Cache plugin exec failed: command:%v "+ - "commandPayload:%v replyPayload:%v error:%v", - pc.Command, pc.Payload, payload, err) - } - response := p.identity.SignMessage(challenge) reply := v1.PluginCommandReply{ Response: hex.EncodeToString(response[:]), @@ -1061,208 +866,6 @@ func (p *politeia) addRoute(method string, route string, handler http.HandlerFun p.router.StrictSlash(true).HandleFunc(route, handler).Methods(method) } -func (p *politeia) buildCacheDecredPlugin(tokens [][]byte) error { - log.Infof("Building %v plugin cache", decredplugin.ID) - - // Reset the existing plugin tables - err := p.cache.PluginBuild(decredplugin.ID, "") - if err != nil { - return fmt.Errorf("PluginBuild: %v", err) - } - - // Build the plugin cache for each record - for i, token := range tokens { - log.Infof("Building %v plugin cache for %x (%v/%v)", - decredplugin.ID, token, i+1, len(tokens)) - - // Get plugin data from the backend for this record - ri := decredplugin.Inventory{ - Tokens: []string{hex.EncodeToString(token)}, - } - b, err := decredplugin.EncodeInventory(ri) - if err != nil { - return err - } - _, reply, err := p.backend.Plugin(decredplugin.CmdInventory, string(b)) - if err != nil { - return fmt.Errorf("backend decred plugin inventory %x: %v", token, err) - } - ir, err := decredplugin.DecodeInventoryReply([]byte(reply)) - if err != nil { - return err - } - - // Build the plugin cache for this record - - // Build comments - log.Debugf("Building comments cache (%v comments)", len(ir.Comments)) - for _, v := range ir.Comments { - // Setup payloads - nc := decredplugin.NewComment{ - Token: v.Token, - ParentID: v.ParentID, - Comment: v.Comment, - Signature: v.Signature, - PublicKey: v.PublicKey, - } - ncr := decredplugin.NewCommentReply{ - CommentID: v.CommentID, - Receipt: v.Receipt, - Timestamp: v.Timestamp, - } - cmdPayload, err := decredplugin.EncodeNewComment(nc) - if err != nil { - return err - } - replyPayload, err := decredplugin.EncodeNewCommentReply(ncr) - if err != nil { - return err - } - - // Send plugin command - _, err = p.cache.PluginExec(cache.PluginCommand{ - ID: decredplugin.ID, - Command: decredplugin.CmdNewComment, - CommandPayload: string(cmdPayload), - ReplyPayload: string(replyPayload), - }) - if err != nil { - return fmt.Errorf("PluginExec %v %x %v: %v", - decredplugin.CmdNewComment, token, ncr.CommentID, err) - } - } - - // Build comment likes - log.Debugf("Building comment likes cache (%v likes)", - len(ir.LikeComments)) - for _, v := range ir.LikeComments { - // Setup payloads - cmdPayload, err := decredplugin.EncodeLikeComment(v) - if err != nil { - return err - } - - // Send plugin command - _, err = p.cache.PluginExec(cache.PluginCommand{ - ID: decredplugin.ID, - Command: decredplugin.CmdLikeComment, - CommandPayload: string(cmdPayload), - }) - if err != nil { - return fmt.Errorf("PluginExec %v %x %v: %v", - decredplugin.CmdLikeComment, token, v.CommentID, err) - } - } - - // Build vote authorizations - log.Debugf("Building authorize vote cache") - for k, v := range ir.AuthorizeVotes { - // Setup payloads - cmdPayload, err := decredplugin.EncodeAuthorizeVote(v) - if err != nil { - return err - } - avr := ir.AuthorizeVoteReplies[k] - replyPayload, err := decredplugin.EncodeAuthorizeVoteReply(avr) - if err != nil { - return err - } - - // Send plugin command - _, err = p.cache.PluginExec(cache.PluginCommand{ - ID: decredplugin.ID, - Command: decredplugin.CmdAuthorizeVote, - CommandPayload: string(cmdPayload), - ReplyPayload: string(replyPayload), - }) - if err != nil { - return fmt.Errorf("PluginExec %v %x: %v", - decredplugin.CmdAuthorizeVote, token, err) - } - } - - // Build start votes - log.Debugf("Building start vote cache") - for _, v := range ir.StartVoteTuples { - // Setup payloads. The start vote payload comes in the tuple - // already encoded due to the start vote versioning. - replyPayload, err := decredplugin.EncodeStartVoteReply(v.StartVoteReply) - if err != nil { - return err - } - - // Send plugin command - _, err = p.cache.PluginExec(cache.PluginCommand{ - ID: decredplugin.ID, - Command: decredplugin.CmdStartVote, - CommandPayload: v.StartVote.Payload, - ReplyPayload: string(replyPayload), - }) - if err != nil { - return fmt.Errorf("PluginExec %v %x: %v", - decredplugin.CmdStartVote, token, err) - } - } - - // Build cast votes - log.Debugf("Building cast votes cache (%v votes)", len(ir.CastVotes)) - if len(ir.CastVotes) != 0 { - // Setup payloads - bl := decredplugin.Ballot{ - Votes: ir.CastVotes, - } - cmdPayload, err := decredplugin.EncodeBallot(bl) - if err != nil { - return err - } - receipts := make([]decredplugin.CastVoteReply, 0, len(ir.CastVotes)) - for _, v := range ir.CastVotes { - receipts = append(receipts, decredplugin.CastVoteReply{ - ClientSignature: v.Signature, - }) - } - br := decredplugin.BallotReply{ - Receipts: receipts, - } - replyPayload, err := decredplugin.EncodeBallotReply(br) - if err != nil { - return err - } - - // Send plugin command - _, err = p.cache.PluginExec(cache.PluginCommand{ - ID: decredplugin.ID, - Command: decredplugin.CmdBallot, - CommandPayload: string(cmdPayload), - ReplyPayload: string(replyPayload), - }) - if err != nil { - return fmt.Errorf("PluginExec %v %x: %v", - decredplugin.CmdBallot, token, err) - } - } - } - - return nil -} - -func (p *politeia) buildCacheCMSPlugin() error { - // Fetch plugin inventory - _, payload, err := p.backend.Plugin(cmsplugin.CmdInventory, "") - if err != nil { - return fmt.Errorf("cms plugin inventory: %v", err) - } - - // Build plugin cache - err = p.cache.PluginBuild(cmsplugin.ID, payload) - if err != nil { - return fmt.Errorf("cache PluginBuild %v: %v", - cmsplugin.ID, err) - } - - return nil -} - func _main() error { // Load configuration and parse command line. This function also // initializes logging and configures it accordingly. @@ -1281,12 +884,6 @@ func _main() error { log.Infof("Network : %v", activeNetParams.Params.Name) log.Infof("Home dir: %v", loadedCfg.HomeDir) - // Issue a warning if pi was builded locally and does not - // have the main module info available. - if version.BuildMainVersion() == "(devel)" { - log.Warnf("Warning: no build information available") - } - // Create the data directory in case it does not exist. err = os.MkdirAll(loadedCfg.DataDir, 0700) if err != nil { @@ -1327,7 +924,6 @@ func _main() error { p := &politeia{ cfg: loadedCfg, plugins: make(map[string]v1.Plugin), - cache: cachestub.New(), } // Load identity. @@ -1361,15 +957,16 @@ func _main() error { case backendGit: b, err := gitbe.New(activeNetParams.Params, loadedCfg.DataDir, loadedCfg.DcrtimeHost, "", p.identity, loadedCfg.GitTrace, - loadedCfg.DcrdataHost, loadedCfg.Mode) + loadedCfg.DcrdataHost) if err != nil { return fmt.Errorf("new gitbe: %v", err) } p.backend = b case backendTlog: b, err := tlogbe.New(loadedCfg.HomeDir, loadedCfg.DataDir, - loadedCfg.TrillianHost, loadedCfg.TrillianKey, loadedCfg.DcrtimeHost, - loadedCfg.EncryptionKey) + loadedCfg.DcrtimeHost, loadedCfg.EncryptionKey, + loadedCfg.TrillianHostUnvetted, loadedCfg.TrillianKeyUnvetted, + loadedCfg.TrillianHostVetted, loadedCfg.TrillianKeyVetted) if err != nil { return fmt.Errorf("new tlogbe: %v", err) } @@ -1378,29 +975,6 @@ func _main() error { return fmt.Errorf("invalid backend selected: %v", loadedCfg.Backend) } - // Setup cache - if p.cfg.EnableCache { - // Create a new cache context - net := filepath.Base(p.cfg.DataDir) - db, err := cockroachdb.New(cockroachdb.UserPoliteiad, p.cfg.CacheHost, - net, p.cfg.CacheRootCert, p.cfg.CacheCert, p.cfg.CacheKey) - if err == cache.ErrNoVersionRecord || err == cache.ErrWrongVersion { - // The cache version record was either not found or - // is the wrong version which means that the cache - // needs to be built/rebuilt. - p.cfg.BuildCache = true - } else if err != nil { - return fmt.Errorf("cockroachdb new: %v", err) - } - p.cache = db - - // Setup the cache tables - err = p.cache.Setup() - if err != nil { - return fmt.Errorf("cache setup: %v", err) - } - } - // Setup mux p.router = mux.NewRouter() @@ -1427,8 +1001,6 @@ func _main() error { p.setVettedStatus, permissionAuth) p.addRoute(http.MethodPost, v1.UpdateVettedMetadataRoute, p.updateVettedMetadata, permissionAuth) - p.addRoute(http.MethodPost, v1.UpdateReadmeRoute, - p.updateReadme, permissionAuth) // Setup plugins plugins, err := p.backend.GetPlugins() @@ -1452,134 +1024,10 @@ func _main() error { } p.plugins[v.ID] = convertBackendPlugin(v) - // Register plugin with the cache - cp := convertBackendPluginToCache(v) - err := p.cache.RegisterPlugin(cp) - if err == cache.ErrNoVersionRecord || err == cache.ErrWrongVersion { - // The cache plugin version record was either not found - // or it is the wrong version which means that the cache - // needs to be built/rebuilt. - p.cfg.BuildCache = true - } else if err != nil { - return fmt.Errorf("cache register plugin '%v': %v", - cp.ID, err) - } - - // Setup the cache plugin tables - err = p.cache.PluginSetup(cp.ID) - if err != nil { - return fmt.Errorf("cache plugin setup '%v': %v", - cp.ID, err) - } - log.Infof("Registered plugin: %v", v.ID) } } - // Build the cache - if p.cfg.BuildCache { - // Reset cache tables - err = p.cache.Build([]cache.Record{}) - if err != nil { - return fmt.Errorf("cache Build: %v", err) - } - - // Build unvetted records cache - log.Infof("Building unvettted records cache") - unvetted, err := p.backend.UnvettedTokens() - if err != nil { - return fmt.Errorf("backend UnvettedTokens: %v", err) - } - for _, token := range unvetted { - r, err := p.backend.GetUnvetted(token) - if err != nil { - return fmt.Errorf("backend GetUnvetted %x: %v", token, err) - } - cr := p.convertBackendRecordToCache(*r) - err = p.cache.NewRecord(cr) - if err != nil { - return fmt.Errorf("cache NewRecord %x: %v", token, err) - } - - log.Debugf("Added unvetted record %x", token) - } - - // Build vetted records cache - log.Debugf("Building vetted records cache") - vetted, err := p.backend.VettedTokens() - if err != nil { - return fmt.Errorf("backend VettedTokens: %v", err) - } - for _, token := range vetted { - // Add the most recent version - r, err := p.backend.GetVetted(token, "") - if err != nil { - return fmt.Errorf("backend GetVetted %x: %v", token, err) - } - cr := p.convertBackendRecordToCache(*r) - err = p.cache.NewRecord(cr) - if err != nil { - return fmt.Errorf("cache NewRecord %x: %v", token, err) - } - - log.Debugf("Added vetted record %x version %v", token, r.Version) - - // Add all previous versions - version, err := strconv.ParseUint(r.Version, 10, 64) - if err != nil { - return err - } - version-- - for version > 0 { - v := strconv.FormatUint(version, 10) - r, err := p.backend.GetVetted(token, v) - if err != nil { - return fmt.Errorf("backend GetVetted %x %v: %v", - token, v, err) - } - cr := p.convertBackendRecordToCache(*r) - err = p.cache.NewRecord(cr) - if err != nil { - return fmt.Errorf("cache NewRecord %x %v: %v", - token, v, err) - } - - log.Debugf("Added vetted record %x version %v", token, version) - version-- - } - } - - // Build the cache for plugins - for _, plugin := range p.plugins { - var enableCache bool - for _, s := range plugin.Settings { - if s.Key == "enablecache" { - enableCache = true - } - } - if !enableCache { - continue - } - - switch plugin.ID { - case decredplugin.ID: - // Decred plugin features are only available on vetted - // proposals. - err := p.buildCacheDecredPlugin(vetted) - if err != nil { - return fmt.Errorf("buildCacheDecredPlugin: %v", err) - } - case cmsplugin.ID: - err := p.buildCacheCMSPlugin() - if err != nil { - return fmt.Errorf("buildCacheCMSPlugin: %v", err) - } - default: - return fmt.Errorf("cache enabled for invalid plugin '%v'", plugin.ID) - } - } - } - // Bind to a port and pass our router in listenC := make(chan error) for _, listener := range loadedCfg.Listeners { @@ -1610,7 +1058,6 @@ func _main() error { } } done: - p.cache.Close() p.backend.Close() log.Infof("Exiting") diff --git a/politeiad/testpoliteiad/convert.go b/politeiad/testpoliteiad/convert.go deleted file mode 100644 index 5a3f3997d..000000000 --- a/politeiad/testpoliteiad/convert.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package testpoliteiad - -import ( - v1 "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/politeiad/cache" -) - -func convertRecordStatusToCache(status v1.RecordStatusT) cache.RecordStatusT { - s := cache.RecordStatusInvalid - switch status { - case v1.RecordStatusInvalid: - s = cache.RecordStatusInvalid - case v1.RecordStatusNotReviewed: - s = cache.RecordStatusNotReviewed - case v1.RecordStatusPublic: - s = cache.RecordStatusPublic - case v1.RecordStatusCensored: - s = cache.RecordStatusCensored - case v1.RecordStatusUnreviewedChanges: - s = cache.RecordStatusUnreviewedChanges - case v1.RecordStatusArchived: - s = cache.RecordStatusArchived - } - return s -} - -func convertCensorshipRecordToCache(r v1.CensorshipRecord) cache.CensorshipRecord { - return cache.CensorshipRecord{ - Token: r.Token, - Merkle: r.Merkle, - Signature: r.Signature, - } -} - -func convertMetadataStreamsToCache(m []v1.MetadataStream) []cache.MetadataStream { - cm := make([]cache.MetadataStream, 0, len(m)) - for _, v := range m { - cm = append(cm, cache.MetadataStream{ - ID: v.ID, - Payload: v.Payload, - }) - } - return cm -} - -func convertFileToCache(f v1.File) cache.File { - return cache.File{ - Name: f.Name, - MIME: f.MIME, - Digest: f.Digest, - Payload: f.Payload, - } -} - -func convertFilesToCache(f []v1.File) []cache.File { - files := make([]cache.File, 0, len(f)) - for _, v := range f { - files = append(files, convertFileToCache(v)) - } - return files -} - -func convertRecordToCache(r v1.Record) cache.Record { - return cache.Record{ - Version: r.Version, - Status: convertRecordStatusToCache(r.Status), - Timestamp: r.Timestamp, - CensorshipRecord: convertCensorshipRecordToCache(r.CensorshipRecord), - Metadata: convertMetadataStreamsToCache(r.Metadata), - Files: convertFilesToCache(r.Files), - } -} diff --git a/politeiad/testpoliteiad/decred.go b/politeiad/testpoliteiad/decred.go deleted file mode 100644 index 50f144482..000000000 --- a/politeiad/testpoliteiad/decred.go +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package testpoliteiad - -import ( - "encoding/hex" - "fmt" - "strconv" - "time" - - decred "github.com/decred/politeia/decredplugin" - v1 "github.com/decred/politeia/politeiad/api/v1" -) - -const ( - bestBlock uint32 = 1000 -) - -func (p *TestPoliteiad) authorizeVote(payload string) (string, error) { - av, err := decred.DecodeAuthorizeVote([]byte(payload)) - if err != nil { - return "", err - } - - // Sign authorize vote - s := p.identity.SignMessage([]byte(av.Signature)) - av.Receipt = hex.EncodeToString(s[:]) - av.Timestamp = time.Now().Unix() - av.Version = decred.VersionAuthorizeVote - - p.Lock() - defer p.Unlock() - - // Store authorize vote - _, ok := p.authorizeVotes[av.Token] - if !ok { - p.authorizeVotes[av.Token] = make(map[string]decred.AuthorizeVote) - } - - r, err := p.record(av.Token) - if err != nil { - return "", err - } - - p.authorizeVotes[av.Token][r.Version] = *av - - // Prepare reply - avrb, err := decred.EncodeAuthorizeVoteReply( - decred.AuthorizeVoteReply{ - Action: av.Action, - RecordVersion: r.Version, - Receipt: av.Receipt, - Timestamp: av.Timestamp, - }) - if err != nil { - return "", err - } - - return string(avrb), nil -} - -func (p *TestPoliteiad) startVote(payload string) (string, error) { - sv, err := decred.DecodeStartVoteV2([]byte(payload)) - if err != nil { - return "", err - } - - p.Lock() - defer p.Unlock() - - // Store start vote - sv.Version = decred.VersionStartVote - p.startVotes[sv.Vote.Token] = *sv - - // Prepare reply - endHeight := bestBlock + sv.Vote.Duration - svr := decred.StartVoteReply{ - Version: decred.VersionStartVoteReply, - StartBlockHeight: strconv.FormatUint(uint64(bestBlock), 10), - EndHeight: strconv.FormatUint(uint64(endHeight), 10), - EligibleTickets: []string{}, - } - svrb, err := decred.EncodeStartVoteReply(svr) - if err != nil { - return "", err - } - - // Store reply - p.startVoteReplies[sv.Vote.Token] = svr - - return string(svrb), nil -} - -func (p *TestPoliteiad) startVoteRunoff(payload string) (string, error) { - svr, err := decred.DecodeStartVoteRunoff([]byte(payload)) - if err != nil { - return "", err - } - - p.Lock() - defer p.Unlock() - - // Store authorize votes - avReply := make(map[string]decred.AuthorizeVoteReply) - for _, av := range svr.AuthorizeVotes { - r, err := p.record(av.Token) - if err != nil { - return "", err - } - // Fill client data - s := p.identity.SignMessage([]byte(av.Signature)) - av.Version = decred.VersionAuthorizeVote - av.Receipt = hex.EncodeToString(s[:]) - av.Timestamp = time.Now().Unix() - av.Version = decred.VersionAuthorizeVote - - // Store - _, ok := p.authorizeVotes[av.Token] - if !ok { - p.authorizeVotes[av.Token] = make(map[string]decred.AuthorizeVote) - } - p.authorizeVotes[av.Token][r.Version] = av - - // Prepare response - avr := decred.AuthorizeVoteReply{ - Action: av.Action, - RecordVersion: r.Version, - Receipt: av.Receipt, - Timestamp: av.Timestamp, - } - avReply[av.Token] = avr - } - - // Store start votes - svReply := decred.StartVoteReply{} - for _, sv := range svr.StartVotes { - sv.Version = decred.VersionStartVote - p.startVotes[sv.Vote.Token] = sv - // Prepare response - endHeight := bestBlock + sv.Vote.Duration - svReply.Version = decred.VersionStartVoteReply - svReply.StartBlockHeight = strconv.FormatUint(uint64(bestBlock), 10) - svReply.EndHeight = strconv.FormatUint(uint64(endHeight), 10) - svReply.EligibleTickets = []string{} - } - - // Store start vote runoff - p.startVotesRunoff[svr.Token] = *svr - - response := decred.StartVoteRunoffReply{ - AuthorizeVoteReplies: avReply, - StartVoteReply: svReply, - } - - p.startVotesRunoffReplies[svr.Token] = response - - svrReply, err := decred.EncodeStartVoteRunoffReply(response) - if err != nil { - return "", err - } - - return string(svrReply), nil -} - -// decredExec executes the passed in plugin command. -func (p *TestPoliteiad) decredExec(pc v1.PluginCommand) (string, error) { - switch pc.Command { - case decred.CmdStartVote: - return p.startVote(pc.Payload) - case decred.CmdStartVoteRunoff: - return p.startVoteRunoff(pc.Payload) - case decred.CmdAuthorizeVote: - return p.authorizeVote(pc.Payload) - case decred.CmdBestBlock: - return strconv.FormatUint(uint64(bestBlock), 10), nil - case decred.CmdVoteSummary: - // This is a cache plugin command. No work needed here. - return "", nil - } - return "", fmt.Errorf("invalid testpoliteiad plugin command") -} diff --git a/politeiad/testpoliteiad/testpoliteiad.go b/politeiad/testpoliteiad/testpoliteiad.go index eb7a87d1f..f7223708b 100644 --- a/politeiad/testpoliteiad/testpoliteiad.go +++ b/politeiad/testpoliteiad/testpoliteiad.go @@ -12,7 +12,6 @@ import ( "encoding/json" "errors" "fmt" - "log" "net/http" "net/http/httptest" "strconv" @@ -21,10 +20,8 @@ import ( "time" "github.com/decred/dcrtime/merkle" - decred "github.com/decred/politeia/decredplugin" v1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/identity" - "github.com/decred/politeia/politeiad/cache" "github.com/decred/politeia/util" "github.com/gorilla/mux" ) @@ -45,16 +42,7 @@ type TestPoliteiad struct { identity *identity.FullIdentity server *httptest.Server - cache cache.Cache records map[string]map[string]v1.Record // [token][version]Record - - // Decred plugin - authorizeVotes map[string]map[string]decred.AuthorizeVote // [token][version]AuthorizeVote - startVotes map[string]decred.StartVoteV2 // [token]StartVote - startVoteReplies map[string]decred.StartVoteReply // [token]StartVoteReply - startVotesRunoff map[string]decred.StartVoteRunoff // [token]StartVoteRunoff - startVotesRunoffReplies map[string]decred.StartVoteRunoffReply // [token]StartVoteRunoffReply - } func respondWithUserError(w http.ResponseWriter, @@ -239,13 +227,6 @@ func (p *TestPoliteiad) handleUpdateVettedRecord(w http.ResponseWriter, r *http. // Update record in store p.updateRecord(updated) - // Update record in cache - err = p.cache.UpdateRecord(convertRecordToCache(updated)) - if err != nil { - util.RespondWithJSON(w, http.StatusInternalServerError, err) - return - } - response := p.identity.SignMessage(challenge) util.RespondWithJSON(w, http.StatusOK, v1.UpdateRecordReply{ Response: hex.EncodeToString(response[:]), @@ -303,15 +284,6 @@ func (p *TestPoliteiad) handleSetUnvettedStatus(w http.ResponseWriter, r *http.R rc.Metadata = append(rc.Metadata, t.MDAppend...) p.addRecord(*rc) - // Update cache - s := convertRecordStatusToCache(rc.Status) - m := convertMetadataStreamsToCache(rc.Metadata) - err = p.cache.UpdateRecordStatus(t.Token, rc.Version, - s, rc.Timestamp, m) - if err != nil { - log.Printf("cache update record status: %v", err) - } - // Send response util.RespondWithJSON(w, http.StatusOK, v1.SetUnvettedStatusReply{ @@ -337,11 +309,8 @@ func (p *TestPoliteiad) handlePluginCommand(w http.ResponseWriter, r *http.Reque } response := p.identity.SignMessage(challenge) - payload, err := p.decredExec(t) - if err != nil { - respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) - return - } + // TODO exec plugin command + var payload string util.RespondWithJSON(w, http.StatusOK, v1.PluginCommandReply{ @@ -404,15 +373,6 @@ func (p *TestPoliteiad) handleSetVettedStatus(w http.ResponseWriter, r *http.Req rc.Metadata = append(rc.Metadata, t.MDAppend...) p.addRecord(*rc) - // Update cache - s := convertRecordStatusToCache(rc.Status) - m := convertMetadataStreamsToCache(rc.Metadata) - err = p.cache.UpdateRecordStatus(t.Token, rc.Version, - s, rc.Timestamp, m) - if err != nil { - log.Printf("cache update record status: %v", err) - } - // Send response util.RespondWithJSON(w, http.StatusOK, v1.SetUnvettedStatusReply{ @@ -420,52 +380,25 @@ func (p *TestPoliteiad) handleSetVettedStatus(w http.ResponseWriter, r *http.Req }) } -// Plugin is a pass through function for plugin commands. The plugin command -// is executed in politeiad and is then passed to the cache. This function -// is intended to be used as a way to setup test data. +// Plugin is a pass through function for plugin commands. This function is +// intended to be used as a way to setup test data. func (p *TestPoliteiad) Plugin(t *testing.T, pc v1.PluginCommand) { t.Helper() // Execute plugin command - var payload string - var err error switch pc.ID { - case decred.ID: - payload, err = p.decredExec(pc) default: t.Fatalf("invalid plugin") } - - if err != nil { - t.Fatal(err) - } - - // Send plugin cmd to cache - _, err = p.cache.PluginExec( - cache.PluginCommand{ - ID: pc.ID, - Command: pc.Command, - CommandPayload: pc.Payload, - ReplyPayload: payload, - }) - if err != nil { - t.Fatal(err) - } } -// AddRecord adds a record to the politeiad records store and to the cache. -// This function is intended to be used as a way to setup test data. +// AddRecord adds a record to the politeiad records store. This function is +// intended to be used as a way to setup test data. func (p *TestPoliteiad) AddRecord(t *testing.T, r v1.Record) { t.Helper() // Add record to memory store p.addRecord(r) - - // Add record to cache - err := p.cache.NewRecord(convertRecordToCache(r)) - if err != nil { - t.Fatal(err) - } } // Close shuts down the httptest server. @@ -474,7 +407,7 @@ func (p *TestPoliteiad) Close() { } // New returns a new TestPoliteiad context. -func New(t *testing.T, c cache.Cache) *TestPoliteiad { +func New(t *testing.T) *TestPoliteiad { t.Helper() // Setup politeiad identity @@ -485,16 +418,10 @@ func New(t *testing.T, c cache.Cache) *TestPoliteiad { // Init context p := TestPoliteiad{ - FullIdentity: id, - PublicIdentity: &id.Public, - identity: id, - cache: c, - records: make(map[string]map[string]v1.Record), - authorizeVotes: make(map[string]map[string]decred.AuthorizeVote), - startVotes: make(map[string]decred.StartVoteV2), - startVoteReplies: make(map[string]decred.StartVoteReply), - startVotesRunoff: make(map[string]decred.StartVoteRunoff), - startVotesRunoffReplies: make(map[string]decred.StartVoteRunoffReply), + FullIdentity: id, + PublicIdentity: &id.Public, + identity: id, + records: make(map[string]map[string]v1.Record), } // Setup routes diff --git a/politeiawww/www.go b/politeiawww/www.go index c78c21f36..29024726f 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -277,12 +277,6 @@ func _main() error { log.Infof("Network : %v", activeNetParams.Params.Name) log.Infof("Home dir: %v", loadedCfg.HomeDir) - // Issue a warning if pi was builded locally and does not - // have the main module info available. - if version.BuildMainVersion() == "(devel)" { - log.Warnf("Warning: no build information available") - } - if loadedCfg.PaywallAmount != 0 && loadedCfg.PaywallXpub != "" { paywallAmountInDcr := float64(loadedCfg.PaywallAmount) / 1e8 log.Infof("Paywall : %v DCR", paywallAmountInDcr) From 22be474aaa43f6497adf4a52674a1a3bcde1e776 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 27 Aug 2020 18:13:47 -0500 Subject: [PATCH 017/449] remove politeiawww_dataload --- .../cmd/politeiawww_dataload/README.md | 59 -- .../cmd/politeiawww_dataload/config.go | 193 ----- politeiawww/cmd/politeiawww_dataload/main.go | 701 ------------------ .../sample-politeiawww_dataload.conf | 55 -- 4 files changed, 1008 deletions(-) delete mode 100644 politeiawww/cmd/politeiawww_dataload/README.md delete mode 100644 politeiawww/cmd/politeiawww_dataload/config.go delete mode 100644 politeiawww/cmd/politeiawww_dataload/main.go delete mode 100644 politeiawww/cmd/politeiawww_dataload/sample-politeiawww_dataload.conf diff --git a/politeiawww/cmd/politeiawww_dataload/README.md b/politeiawww/cmd/politeiawww_dataload/README.md deleted file mode 100644 index 0f166acd2..000000000 --- a/politeiawww/cmd/politeiawww_dataload/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# politeiawww_dataload - -politeiawww_dataload is a tool that loads basic data into Politeia to help -speed up full end-to-end testing. It will automatically start and stop -politeiad and politeiawww, and utilize the politeiawwwcli tool to create the following: - -* Admin user -* Regular paid user -* Regular unpaid user -* A proposal for each state (public, censored, etc) -* A couple comments on the public proposal - -Because it starts and stops politeiad and politeiawww automatically, you -will need to ensure that those servers are shut down before running this tool. -It will run the servers with some fixed configuration, although some default -configuration is required, so you should have politeiad.conf and politeiawww.conf -already set up. - -When running politeiawww_dataload twice, the second time will fail because it -can't create duplicate users. - -## Usage - -This tool doesn't require any arguments, but you can specify the following options. -Additionally, you can specify these options in a `politeiawww_dataload.conf` file, -which should be located under `/Users//Library/Application Support/Politeiawww/dataload/`. - -``` - --adminemail admin email address - --adminuser admin username - --adminpass admin password - --paidemail paid user email address - --paiduser paid user username - --paidpass paid user password - --unpaidemail unpaid user email address - --unpaiduser unpaid user username - --unpaidpass unpaid user password - --deletedata before loading the data, delete all existing data - --debuglevel the debug level to set when starting politeiad and politeiawww - server; the servers' log output is stored in the data directory - --datadir specify a different directory to store log files - --configfile specify a different .conf file for config options - -v, --verbose verbose output -``` - -Example: - -``` -politeiawww_dataload --verbose -``` - -## Troubleshooting - -If you encounter an error while running politeiawww_dataload, it's possible that -some program this depends on is out of date. Before opening a Github issue, -make sure to pull the latest from master and build all programs: - - cd $GOPATH/src/github.com/decred/politeia - dep ensure && go install -v ./... diff --git a/politeiawww/cmd/politeiawww_dataload/config.go b/politeiawww/cmd/politeiawww_dataload/config.go deleted file mode 100644 index 2366ee1b1..000000000 --- a/politeiawww/cmd/politeiawww_dataload/config.go +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright (c) 2013-2014 The btcsuite developers -// Copyright (c) 2015-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/decred/politeia/politeiawww/sharedconfig" - flags "github.com/jessevdk/go-flags" -) - -const ( - defaultDataDirname = "dataload" - defaultConfigFilename = "politeiawww_dataload.conf" - defaultPoliteiadLogFilename = "politeiad.log" - defaultPoliteiawwwLogFilename = "politeiawww.log" - defaultLogLevel = "info" -) - -var ( - defaultDataDir = filepath.Join(sharedconfig.DefaultHomeDir, defaultDataDirname) - defaultConfigFile = filepath.Join(defaultDataDir, defaultConfigFilename) -) - -// config defines the configuration options for politeiawww_dataload. -// -// See loadConfig for details on the configuration load process. -type config struct { - AdminEmail string `long:"adminemail" description:"Admin user email address"` - AdminUser string `long:"adminuser" description:"Admin username"` - AdminPass string `long:"adminpass" description:"Admin password"` - PaidEmail string `long:"paidemail" description:"Regular paid user email address"` - PaidUser string `long:"paiduser" description:"Regular paid user username"` - PaidPass string `long:"paidpass" description:"Regular paid user password"` - UnpaidEmail string `long:"unpaidemail" description:"Regular unpaid user email address"` - UnpaidUser string `long:"unpaiduser" description:"Regular unpaid user username"` - UnpaidPass string `long:"unpaidpass" description:"Regular unpaid user password"` - VettedPropsNumber int `long:"vettedproposalsnumber" description:"Number of vetted proposals to be created"` - UnvettedPropsNumber int `long:"unvettedproposalsnumber" description:"Number of unvetted proposals to be created"` - CommentsNumber int `long:"commentsnumber" description:"Number of comments on the firs vetted proposal"` - Verbose bool `short:"v" long:"verbose" description:"Verbose output"` - DataDir string `long:"datadir" description:"Path to config/data directory"` - ConfigFile string `long:"configfile" description:"Path to configuration file"` - DebugLevel string `long:"debuglevel" description:"Logging level to use for servers {trace, debug, info, warn, error, critical}"` - DeleteData bool `long:"deletedata" description:"Delete all existing data from politeiad and politeiawww before loading data"` - PoliteiadLogFile string - PoliteiawwwLogFile string -} - -// cleanAndExpandPath expands environment variables and leading ~ in the -// passed path, cleans the result, and returns it. -func cleanAndExpandPath(path string) string { - // Expand initial ~ to OS specific home directory. - if strings.HasPrefix(path, "~") { - homeDir := filepath.Dir(sharedconfig.DefaultHomeDir) - path = strings.Replace(path, "~", homeDir, 1) - } - - // NOTE: The os.ExpandEnv doesn't work with Windows-style %VARIABLE%, - // but the variables can still be expanded via POSIX-style $VARIABLE. - return filepath.Clean(os.ExpandEnv(path)) -} - -// newConfigParser returns a new command line flags parser. -func newConfigParser(cfg *config, options flags.Options) *flags.Parser { - return flags.NewParser(cfg, options) -} - -// loadConfig initializes and parses the config using a config file and command -// line options. -// -// The configuration proceeds as follows: -// 1) Start with a default config with sane settings -// 2) Pre-parse the command line to check for an alternative config file -// 3) Load configuration file overwriting defaults with any specified options -// 4) Parse CLI options and overwrite/add any specified options -// -// The above results in rpc functioning properly without any config settings -// while still allowing the user to override settings with config files and -// command line options. Command line options always take precedence. -func loadConfig() (*config, error) { - // Default config. - cfg := config{ - AdminEmail: "admin@example.com", - AdminUser: "admin", - AdminPass: "password", - PaidEmail: "paid_user@example.com", - PaidUser: "paid_user", - PaidPass: "password", - UnpaidEmail: "unpaid_user@example.com", - UnpaidUser: "unpaid_user", - UnpaidPass: "password", - VettedPropsNumber: 1, - UnvettedPropsNumber: 2, - CommentsNumber: 2, - DeleteData: false, - Verbose: false, - DataDir: defaultDataDir, - ConfigFile: defaultConfigFile, - DebugLevel: defaultLogLevel, - } - - // Pre-parse the command line options to see if an alternative config - // file or the version flag was specified. Any errors aside from the - // help message error can be ignored here since they will be caught by - // the final parse below. - preCfg := cfg - preParser := newConfigParser(&preCfg, flags.HelpFlag) - _, err := preParser.Parse() - if err != nil { - if e, ok := err.(*flags.Error); ok && e.Type == flags.ErrHelp { - fmt.Fprintln(os.Stderr, err) - os.Exit(0) - } - } - - // Show the version and exit if the version flag was specified. - appName := filepath.Base(os.Args[0]) - appName = strings.TrimSuffix(appName, filepath.Ext(appName)) - usageMessage := fmt.Sprintf("Use %s -h to show usage", appName) - - // Update the data directory if specified. Since the data directory - // is updated, other variables need to be updated to reflect the new changes. - if preCfg.DataDir != "" { - cfg.DataDir, _ = filepath.Abs(preCfg.DataDir) - - if preCfg.ConfigFile == defaultConfigFile { - cfg.ConfigFile = filepath.Join(cfg.DataDir, defaultConfigFilename) - } else { - cfg.ConfigFile = cleanAndExpandPath(preCfg.ConfigFile) - } - } - - // Load additional config from file. - var configFileError error - parser := newConfigParser(&cfg, flags.Default) - err = flags.NewIniParser(parser).ParseFile(cfg.ConfigFile) - if err != nil { - if _, ok := err.(*os.PathError); !ok { - fmt.Fprintf(os.Stderr, "Error parsing config "+ - "file: %v\n", err) - fmt.Fprintln(os.Stderr, usageMessage) - return nil, err - } - configFileError = err - } - - // Parse command line options again to ensure they take precedence. - _, err = parser.Parse() - if err != nil { - if e, ok := err.(*flags.Error); !ok || e.Type != flags.ErrHelp { - fmt.Fprintln(os.Stderr, usageMessage) - } - return nil, err - } - - // Create the data directory if it doesn't already exist. - funcName := "loadConfig" - err = os.MkdirAll(cfg.DataDir, 0700) - if err != nil { - // Show a nicer error message if it's because a symlink is - // linked to a directory that does not exist (probably because - // it's not mounted). - if e, ok := err.(*os.PathError); ok && os.IsExist(err) { - if link, lerr := os.Readlink(e.Path); lerr == nil { - str := "is symlink %s -> %s mounted?" - err = fmt.Errorf(str, e.Path, link) - } - } - - str := "%s: Failed to create data directory: %v" - err := fmt.Errorf(str, funcName, err) - fmt.Fprintln(os.Stderr, err) - return nil, err - } - - if configFileError != nil { - fmt.Printf("WARNING: %v\n", configFileError) - } - - cfg.PoliteiadLogFile = filepath.Join(cfg.DataDir, - defaultPoliteiadLogFilename) - cfg.PoliteiawwwLogFile = filepath.Join(cfg.DataDir, - defaultPoliteiawwwLogFilename) - - return &cfg, nil -} diff --git a/politeiawww/cmd/politeiawww_dataload/main.go b/politeiawww/cmd/politeiawww_dataload/main.go deleted file mode 100644 index 00937c683..000000000 --- a/politeiawww/cmd/politeiawww_dataload/main.go +++ /dev/null @@ -1,701 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import ( - "bufio" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "strconv" - "strings" - - "github.com/decred/dcrd/dcrutil" - "github.com/decred/politeia/politeiawww/api/www/v1" - wwwconfig "github.com/decred/politeia/politeiawww/sharedconfig" -) - -type beforeVerifyReply func() interface{} -type verifyReply func() bool - -const ( - cli = "politeiawwwcli" - dbutil = "politeiawww_dbutil" -) - -var ( - cfg *config - politeiadCmd *exec.Cmd - politeiawwwCmd *exec.Cmd -) - -func executeCommand(args ...string) *exec.Cmd { - if cfg.Verbose { - fmt.Printf(" $ %v\n", strings.Join(args, " ")) - } - return exec.Command(args[0], args[1:]...) -} - -func createPoliteiawwCmd(paywall bool) *exec.Cmd { - var paywallXPub string - var paywallAmount uint64 - if paywall { - paywallXPub = "tpubVobLtToNtTq6TZNw4raWQok35PRPZou53vegZqNubtBTJMMFmuMpWybFCfweJ52N8uZJPZZdHE5SRnBBuuRPfC5jdNstfKjiAs8JtbYG9jx" - paywallAmount = 10000000 - } - - return executeCommand( - "politeiawww", - "--testnet", - "--paywallxpub", paywallXPub, - "--paywallamount", strconv.FormatUint(paywallAmount, 10), - "--debuglevel", cfg.DebugLevel) -} - -func createLogFile(path string) (*os.File, error) { - return os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0644) - /* - if err != nil { - return nil, err - } - - _, err = file.Write([]byte("------------------------------------------------------\n")) - return file, err - */ -} - -func waitForStartOfDay(out io.Reader) { - buf := bufio.NewScanner(out) - for buf.Scan() { - text := buf.Text() - if strings.Contains(text, "Start of day") { - return - } - } -} - -func startPoliteiawww(paywall bool) error { - fmt.Printf("Starting politeiawww\n") - politeiawwwCmd = createPoliteiawwCmd(paywall) - out, _ := politeiawwwCmd.StdoutPipe() - if err := politeiawwwCmd.Start(); err != nil { - politeiawwwCmd = nil - return err - } - - logFile, err := createLogFile(cfg.PoliteiawwwLogFile) - if err != nil { - return err - } - - reader := io.TeeReader(out, logFile) - waitForStartOfDay(reader) - go io.Copy(logFile, out) - - // Get the version for the csrf - return getVersionFromPoliteiawww() -} - -func startPoliteiad() error { - fmt.Printf("Starting politeiad\n") - politeiadCmd = executeCommand("politeiad", "--testnet", "--buildcache") - out, _ := politeiadCmd.StdoutPipe() - if err := politeiadCmd.Start(); err != nil { - politeiadCmd = nil - return err - } - - logFile, err := createLogFile(cfg.PoliteiadLogFile) - if err != nil { - return err - } - - reader := io.TeeReader(out, logFile) - waitForStartOfDay(reader) - return nil -} - -func getVersionFromPoliteiawww() error { - fmt.Printf("Getting version\n") - - var vr *v1.VersionReply - return executeCliCommand( - func() interface{} { - vr = &v1.VersionReply{} - return vr - }, - func() bool { - return vr.PubKey != "" - }, - "version", - ) -} - -func createUserWithPoliteiawww(email, username, password string) error { - fmt.Printf("Creating user: %v\n", email) - - var nur *v1.NewUserReply - receivedNewUserReply := new(bool) - return executeCliCommand( - func() interface{} { - nur = &v1.NewUserReply{} - return nur - }, - func() bool { - if *receivedNewUserReply && nur.VerificationToken == "" { - return true - } - - if !*receivedNewUserReply && nur.VerificationToken != "" { - *receivedNewUserReply = true - } - - return false - }, - "newuser", - email, - username, - password, - "--verify", - ) -} - -func setAdmin(email string) error { - fmt.Printf("Elevating user to admin: %v\n", email) - cmd := executeCommand( - dbutil, - "-testnet", - "-setadmin", - email, - "true") - if err := cmd.Start(); err != nil { - return err - } - return cmd.Wait() -} - -func clearPaywall(userID string) error { - fmt.Printf("Clearing paywall for user with ID: %v\n", userID) - var eur *v1.ManageUserReply - return executeCliCommand( - func() interface{} { - eur = &v1.ManageUserReply{} - return eur - }, - func() bool { - return *eur == (v1.ManageUserReply{}) - }, - "manageuser", - userID, - fmt.Sprintf("%v", v1.UserManageClearUserPaywall), - "politeaiwww_dataload") -} - -func addProposalCredits(email, quantity string) error { - fmt.Printf("Adding %v proposal credits to user account: %v\n", quantity, email) - cmd := executeCommand( - dbutil, - "-testnet", - "-addcredits", - email, - quantity) - if err := cmd.Start(); err != nil { - return err - } - return cmd.Wait() -} - -func me() (*v1.LoginReply, error) { - fmt.Printf("Fetching user details\n") - var lr *v1.LoginReply - err := executeCliCommand( - func() interface{} { - lr = &v1.LoginReply{} - return lr - }, - func() bool { - return lr.UserID != "" - }, - "me") - if err != nil { - return nil, err - } - return lr, nil -} - -func createPaidUsers() error { - err := createUserWithPoliteiawww(cfg.AdminEmail, cfg.AdminUser, cfg.AdminPass) - if err != nil { - return err - } - - err = createUserWithPoliteiawww(cfg.PaidEmail, cfg.PaidUser, cfg.PaidPass) - if err != nil { - return err - } - - stopServers() - - if err = setAdmin(cfg.AdminEmail); err != nil { - return err - } - - if err = addProposalCredits(cfg.AdminEmail, "5"); err != nil { - return err - } - - if err = startPoliteiad(); err != nil { - return err - } - - if err = startPoliteiawww(true); err != nil { - return err - } - - if err = login(cfg.AdminEmail, cfg.AdminPass); err != nil { - return err - } - if err := updateUserKey(); err != nil { - return err - } - - // Fetch userIDs for admin user and paid user - lr, err := me() - if err != nil { - return err - } - adminID := lr.UserID - - if err = login(cfg.PaidEmail, cfg.PaidPass); err != nil { - return err - } - lr, err = me() - if err != nil { - return nil - } - paidID := lr.UserID - - // Log back in with admin and clear paywalls - if err = login(cfg.AdminEmail, cfg.AdminPass); err != nil { - return err - } - - if err = clearPaywall(adminID); err != nil { - return err - } - - return clearPaywall(paidID) -} - -func createUnpaidUsers() error { - return createUserWithPoliteiawww(cfg.UnpaidEmail, cfg.UnpaidUser, cfg.UnpaidPass) -} - -func executeCliCommand(beforeVerify beforeVerifyReply, verify verifyReply, args ...string) error { - fullArgs := make([]string, 0, len(args)+2) - fullArgs = append(fullArgs, cli) - fullArgs = append(fullArgs, "--host") - fullArgs = append(fullArgs, "https://127.0.0.1:4443") - fullArgs = append(fullArgs, "--json") - fullArgs = append(fullArgs, "--skipverify") - fullArgs = append(fullArgs, args...) - cmd := executeCommand(fullArgs...) - - stdout, _ := cmd.StdoutPipe() - stderr, _ := cmd.StderrPipe() - if err := cmd.Start(); err != nil { - return err - } - defer cmd.Wait() - - errBytes, err := ioutil.ReadAll(stderr) - if err != nil { - return err - } - - if len(errBytes) > 0 { - return fmt.Errorf("unexpected error output from %v: %v", cli, - string(errBytes)) - } - - var allText string - buf := bufio.NewScanner(stdout) - for buf.Scan() { - text := buf.Text() - - var lines []string - if strings.Contains(text, "\n") { - lines = strings.Split(text, "\n") - } else { - lines = append(lines, text) - } - - for _, line := range lines { - if cfg.Verbose { - fmt.Printf(" %v\n", line) - } - - var er v1.ErrorReply - err := json.Unmarshal([]byte(line), &er) - if err == nil && er.ErrorCode != int64(v1.ErrorStatusInvalid) { - return fmt.Errorf("error returned from %v: %v %v", cli, - er.ErrorCode, er.ErrorContext) - } - - reply := beforeVerify() - err = json.Unmarshal([]byte(line), reply) - if err == nil && verify() { - return nil - } - - allText += line + "\n" - } - } - - if err := buf.Err(); err != nil { - return err - } - - return fmt.Errorf("unexpected output from %v: %v", cli, allText) -} - -func createProposal() (string, error) { - fmt.Printf("Creating proposal\n") - - var npr *v1.NewProposalReply - err := executeCliCommand( - func() interface{} { - npr = &v1.NewProposalReply{} - return npr - }, - func() bool { - return npr.CensorshipRecord.Token != "" - }, - "newproposal", - "--random", - ) - if err != nil { - return "", err - } - - fmt.Printf("Created proposal with token %v\n", npr.CensorshipRecord.Token) - return npr.CensorshipRecord.Token, nil -} - -func checkProposal(token string) error { - fmt.Printf("Checking proposal with token %v\n", token) - - var pdr *v1.ProposalDetailsReply - err := executeCliCommand( - func() interface{} { - pdr = &v1.ProposalDetailsReply{} - return pdr - }, - func() bool { - return pdr.Proposal.CensorshipRecord.Token == token - }, - "proposaldetails", - token, - ) - if err != nil { - return err - } - - fmt.Printf("Verified proposal\n") - return nil -} - -func createComment(parentID, token string) (string, error) { - fmt.Printf("Creating comment\n") - - var ncr *v1.NewCommentReply - err := executeCliCommand( - func() interface{} { - ncr = &v1.NewCommentReply{} - return ncr - }, - func() bool { - return ncr.Comment.CommentID != "" - }, - "newcomment", - token, - "This is a comment", - parentID) - if err != nil { - return "", err - } - - fmt.Printf("Created comment with id %v\n", ncr.Comment.CommentID) - return ncr.Comment.CommentID, nil -} - -func setProposalStatus(token string, status v1.PropStatusT, message string) error { - fmt.Printf("Setting proposal status to %v\n", status) - - var spsr *v1.SetProposalStatusReply - return executeCliCommand( - func() interface{} { - spsr = &v1.SetProposalStatusReply{} - return spsr - }, - func() bool { - return spsr.Proposal.Status != v1.PropStatusInvalid - }, - "setproposalstatus", - token, - strconv.FormatInt(int64(status), 10), - message, - ) -} - -func publishProposals(number int) ([]string, error) { - var proposalsTokens []string - - for i := 0; i < number; i++ { - token, err := createProposal() - if err != nil { - return nil, err - } - proposalsTokens = append(proposalsTokens, token) - } - - return proposalsTokens, nil -} - -func createProposals(vettedProps int, unvettedProps int, commentsNumber int) error { - // Create the proposals. - if err := login(cfg.PaidEmail, cfg.PaidPass); err != nil { - return err - } - if err := updateUserKey(); err != nil { - return err - } - - vettedProposalTokens, err := publishProposals(vettedProps) - if err != nil { - return err - } - - unvettedProposalTokens, err := publishProposals(unvettedProps) - if err != nil { - return err - } - - for i := 0; i < vettedProps; i++ { - err := checkProposal(vettedProposalTokens[i]) - if err != nil { - return err - } - } - for i := 0; i < unvettedProps; i++ { - err := checkProposal(unvettedProposalTokens[i]) - if err != nil { - return err - } - } - - // Set the proposals' status. - if err := login(cfg.AdminEmail, cfg.AdminPass); err != nil { - return err - } - if err := updateUserKey(); err != nil { - return err - } - - for i := 0; i < vettedProps; i++ { - if err := setProposalStatus(vettedProposalTokens[i], v1.PropStatusPublic, ""); err != nil { - return err - } - } - - // Censor the first unvetted proposal - if len(unvettedProposalTokens) > 0 { - if err := setProposalStatus(unvettedProposalTokens[0], v1.PropStatusCensored, "censor message"); err != nil { - return err - } - } - - if err := logout(); err != nil { - return err - } - - // Create comments on the first public published proposal. - if err := login(cfg.AdminEmail, cfg.AdminPass); err != nil { - return err - } - if err := updateUserKey(); err != nil { - return err - } - - var commentID string - for i := 0; i < commentsNumber; i++ { - if len(vettedProposalTokens) > 0 { - commentID, err = createComment("", vettedProposalTokens[0]) - if err != nil { - return err - } - } - } - - if err := logout(); err != nil { - return err - } - - if err := login(cfg.PaidEmail, cfg.PaidPass); err != nil { - return err - } - if err := updateUserKey(); err != nil { - return err - } - if len(vettedProposalTokens) > 0 { - if _, err := createComment(commentID, vettedProposalTokens[0]); err != nil { - return err - } - } - return logout() -} - -func login(email, password string) error { - fmt.Printf("Logging in as: %v\n", email) - var lr *v1.LoginReply - return executeCliCommand( - func() interface{} { - lr = &v1.LoginReply{} - return lr - }, - func() bool { - return lr.UserID != "" - }, - "login", - email, - password) -} - -func logout() error { - fmt.Printf("Logging out...\n") - return executeCliCommand( - func() interface{} { - return &v1.LogoutReply{} - }, - func() bool { - return true - }, - "logout") -} - -func updateUserKey() error { - fmt.Printf("Updating user key\n") - return executeCliCommand( - func() interface{} { - return &v1.UpdateUserKeyReply{} - }, - func() bool { - return true - }, - "updateuserkey") -} - -func deleteExistingData() error { - fmt.Printf("Deleting existing data\n") - - // politeiad data dir - politeiadDataDir := filepath.Join(dcrutil.AppDataDir("politeiad", false), "data") - if err := os.RemoveAll(politeiadDataDir); err != nil { - return err - } - - // politeiawww data dir - if err := os.RemoveAll(wwwconfig.DefaultDataDir); err != nil { - return err - } - - // politeiawww cli dir - cliDataDir := filepath.Join(wwwconfig.DefaultHomeDir, "cli", "data") - return os.RemoveAll(cliDataDir) -} - -func stopPoliteiad() { - if politeiadCmd != nil { - fmt.Printf("Stopping politeiad\n") - if err := politeiadCmd.Process.Kill(); err != nil { - fmt.Fprintf(os.Stderr, "unable to kill politeiad: %v", err) - } - politeiadCmd = nil - } -} - -func stopPoliteiawww() { - if politeiawwwCmd != nil { - fmt.Printf("Stopping politeiawww\n") - if err := politeiawwwCmd.Process.Kill(); err != nil { - fmt.Fprintf(os.Stderr, "unable to kill politeiawww: %v", err) - } - politeiawwwCmd = nil - } -} - -func stopServers() { - stopPoliteiad() - stopPoliteiawww() -} - -func _main() error { - // Load configuration and parse command line. This function also - // initializes logging and configures it accordingly. - var err error - cfg, err = loadConfig() - if err != nil { - return fmt.Errorf("Could not load configuration file: %v", err) - } - - if cfg.DeleteData { - if err = deleteExistingData(); err != nil { - return err - } - } - - if err = startPoliteiad(); err != nil { - return err - } - - if err = startPoliteiawww(true); err != nil { - return err - } - - if err = createPaidUsers(); err != nil { - return err - } - - if err = createUnpaidUsers(); err != nil { - return err - } - - stopPoliteiawww() - if err = startPoliteiawww(false); err != nil { - return err - } - - err = createProposals(cfg.VettedPropsNumber, cfg.UnvettedPropsNumber, - cfg.CommentsNumber) - if err != nil { - return err - } - - fmt.Printf("Load data complete\n") - return nil -} - -func main() { - err := _main() - stopServers() - if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) - } -} diff --git a/politeiawww/cmd/politeiawww_dataload/sample-politeiawww_dataload.conf b/politeiawww/cmd/politeiawww_dataload/sample-politeiawww_dataload.conf deleted file mode 100644 index 3510bd0cc..000000000 --- a/politeiawww/cmd/politeiawww_dataload/sample-politeiawww_dataload.conf +++ /dev/null @@ -1,55 +0,0 @@ -[Application Options] - -; ------------------------------------------------------------------------------ -; Data settings -; ------------------------------------------------------------------------------ - -; The directory to store data such as the server log files and the dataload -; config file. The default is ~/.politeiawww/dataload on POSIX OSes. -; datadir=~/.politeiawww/dataload - -; The specific file used to load configuration options. -; configfile=~/.politeiawww/dataload/politeiawww_dataload.conf - -; ------------------------------------------------------------------------------ -; Misc configuration options -; ------------------------------------------------------------------------------ - -; Delete all existing data within politeiad and politeiawww before -; attempting to load new data. -; deletedata=true - -; ------------------------------------------------------------------------------ -; Admin user options -; ------------------------------------------------------------------------------ - -; adminemail=admin@example.com -; adminuser=admin -; adminpass=password - -; ------------------------------------------------------------------------------ -; Regular paid user options -; ------------------------------------------------------------------------------ - -; paidemail=paid@example.com -; paiduser=paid_user -; paidpass=password - -; ------------------------------------------------------------------------------ -; Regular unpaid user options -; ------------------------------------------------------------------------------ - -; unpaidemail=unpaid@example.com -; unpaiduser=unpaid_user -; unpaidpass=password - -; ------------------------------------------------------------------------------ -; Debug -; ------------------------------------------------------------------------------ - -; Debug logging level for the politeiad and politeiawww servers. -; Valid levels are {trace, debug, info, warn, error, critical} -; You may also specify =,=,... to set -; log level for individual subsystems. Use politeiawww --debuglevel=show to list -; available subsystems. -; debuglevel=info From d011358e39b7984a5a1f37296e872653aed299e0 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 27 Aug 2020 18:20:29 -0500 Subject: [PATCH 018/449] www refactor --- cms-instructions.md | 51 ++-- politeia.md | 134 +++++++--- politeiawww/cmsaddresswatcher.go | 4 +- politeiawww/comments.go | 164 ++---------- politeiawww/convert.go | 446 +------------------------------ politeiawww/decred.go | 405 +--------------------------- politeiawww/invoices.go | 127 +++++++++ politeiawww/log.go | 22 +- politeiawww/politeiawww.go | 16 +- politeiawww/proposals.go | 335 +++++++++++------------ politeiawww/testing.go | 2 - politeiawww/www.go | 67 +---- scripts/cachesetup.sh | 83 ------ util/signature.go | 2 + 14 files changed, 484 insertions(+), 1374 deletions(-) delete mode 100755 scripts/cachesetup.sh diff --git a/cms-instructions.md b/cms-instructions.md index d049cac15..af7ed016d 100644 --- a/cms-instructions.md +++ b/cms-instructions.md @@ -1,9 +1,10 @@ ## Contractor Management System Instructions -Welcome to Decred's Contractor Management System! This site is being designed -to be a functional interface for contractors to submit and invoices be processed -and, in the future, have a seat at the table in some of the contractor level -decision making, as discussed in the stakeholder approved proposal for the +Welcome to Decred's Contractor Management System! This site is being designed +to be a functional interface for contractors to submit and invoices be +processed and, in the future, have a seat at the table in some of the +contractor level decision making, as discussed in the stakeholder approved +proposal for the [DCC](https://proposals.decred.org/proposals/fa38a3593d9a3f6cb2478a24c25114f5097c572f6dadf24c78bb521ed10992a4). To begin, we will be inviting contractors to the site to begin to submit @@ -17,8 +18,8 @@ be "Approved" for payment. Currently, payments will still be processed by hand until the DAE is fully operational. -Also note that for the initial months of CMS usage invoice DCR/USD rates will be -calculated in the same fashion as they had done before. In the near future, +Also note that for the initial months of CMS usage invoice DCR/USD rates will +be calculated in the same fashion as they had done before. In the near future, once implemented, users will see a given month's DCR/USD rate upon invoice submission. Upon administrator invoice approval, CMS will watch the invoice's payment address for the amount expected. Once observed, the invoice will be @@ -31,30 +32,38 @@ Discord. ### How to become a user -Currently, becoming a user requires one to be invited by an administrator. Once -the administrator issues the invitation, one should receive an email at the -address that the DHG currently has on hand. There will be a link in the email -that will include a verification token. Following this link will reach a -registration page that requires entry of the email, token, username and password. -Once successfully registered a user may login via the form on the right. +Currently, becoming a user requires one to be invited by an administrator. +Once the administrator issues the invitation, one should receive an email at +the address that the DHG currently has on hand. There will be a link in the +email that will include a verification token. Following this link will reach a +registration page that requires entry of the email, token, username and +password. Once successfully registered a user may login via the form on the +right. ### How to create invoices Once registered a user may submit invoices at [https://cms.decred.org/invoices/new](https://cms.decred.org/invoices/new). -This form is relatively self explantory, but here is a quick description of each -of the currently required fields: +This form is relatively self explantory, but here is a quick description of +each of the currently required fields: -* Contractor Name: This is whatever name you identify yourself with the DHG, typically something beyond a mere handle or nick. -* Contractor Location: This is which country you are currently located, or primarily residing. -* Contractor Contact: Contact information incase an administrator would need to reach out to discuss something, typically an email address or chat nick. -* Contractor Rate: This is the previously agreed upon rate you will be performing work. -* Payment Address: This is the DCR address where you would like to receive payment. +* Contractor Name: This is whatever name you identify yourself with the DHG, + typically something beyond a mere handle or nick. +* Contractor Location: This is which country you are currently located, or + primarily residing. +* Contractor Contact: Contact information incase an administrator would need to + reach out to discuss something, typically an email address or chat nick. +* Contractor Rate: This is the previously agreed upon rate you will be + performing work. +* Payment Address: This is the DCR address where you would like to receive + payment. * Line Items: * Type: Currently can be 1 (Labor), 2 (Expense), or 3 (Misc) - * Domain: The broad category of work performed/expenses spent (for example, Development, Marketing, Community etc). - * Subdomain: The specific project or program of which the work or expenses are related (for example, Decrediton, dcrd, NYC Event). + * Domain: The broad category of work performed/expenses spent (for example, + Development, Marketing, Community etc). + * Subdomain: The specific project or program of which the work or expenses + are related (for example, Decrediton, dcrd, NYC Event). * Description: A thorough description of the work or expenses. * Labor: The number of hours of work performed. * Expenses: The cost of the line item (in USD). diff --git a/politeia.md b/politeia.md index 393173a14..34cf7fd56 100644 --- a/politeia.md +++ b/politeia.md @@ -1,34 +1,67 @@ ## Politeia (Pi) introduction -Politeia, or Pi, is the Decred proposal system. It is intended to facilitate the submission, discussion and approval/rejection of governance proposals. +Politeia, or Pi, is the Decred proposal system. It is intended to facilitate +the submission, discussion and approval/rejection of governance proposals. There are two broad types of proposal: -1. Proposals that aim to establish stake-voter support for a course of action, e.g. direction of software development, adopting or changing some policy. -2. Proposals that commit to spending project fund DCR, creating a budget that some entity can draw down against as they demonstrate progress towards the proposal's aim. +1. Proposals that aim to establish stake-voter support for a course of action, + e.g. direction of software development, adopting or changing some policy. +2. Proposals that commit to spending project fund DCR, creating a budget that + some entity can draw down against as they demonstrate progress towards the + proposal's aim. There is a fee for submitting a proposal (0.1 DCR), to limit the potential for proposal spamming. -When proposals are submitted, they are checked by Politeia administrators. Proposals that are deemed spam or invalid will be censored. - -When proposals are submitted, a censorship token is generated, which the proposal's owner can use, in the case that their proposal was censored, to demonstrate that it was submitted but censored. - -Valid proposals will be displayed on the Politeia platform, where they can be seen by all and discussed by Politeia members. - -There is a registration fee (0.1 DCR) for creating a Politeia account. Only members who have paid this fee are eligible to submit proposals and comments, and to make up/down votes on the Politeia web platform. - -Up/down votes do not affect proposal funding decisions, they are used as soft signals and to determine display order. Up/down voting is not anonymous, the up/down voting history of Politeia accounts will be public information. - -When a proposal is submitted and passes screening, it will be displayed on Politeia but voting will not open immediately. The proposer has discretion to participate in discussion with Decred stakeholders and make edits to their proposal, then decide when to trigger the ticket-voting interval. When voting is triggered, edits to the proposal can no longer be made. - -Ticket-voting is used to determine whether proposals are approved by Decred's stake-governors. Ticket-voting is to be performed from a Decred wallet with live tickets, it does not happen directly through the Politeia web platform. - -Politeia's aim is to serve as the decision-making force behind the Decred Decentralized Autonomous Entity (DAE). This is an ambitious aim, Politeia and its accompanying processes are in an experimental stage and thus subject to change. - -Initially at least, the disbursal of funds to successful proposals will be a manual process. Proposals that request funding should specify how much funding they require, denominated in a national currency like $USD. They should also specify a set of milestones or deliverables which will trigger the release of funds. - -When a proposal is approved by ticket-voters, this gives a green light to the proposing entity to begin work. When the first milestone is met, they can make a request for the release of the first tranche of funding. This will be reviewed and, where satisfactory, will be processed. It is expected that, initially at least, all proposals requesting funding are paid in arrears. - -An example is that stakeholders expect developers to write and show code before payment occurs. The existing contractors use this model. For example, Company 0 carries the costs (and risk) for 4-6 weeks before payout occurs. This is a feature and not a bug. Asking for a large sum of money without incurring costs shifts the risk to the DAE and that incentivizes malicious behavior. +When proposals are submitted, they are checked by Politeia administrators. +Proposals that are deemed spam or invalid will be censored. + +When proposals are submitted, a censorship token is generated, which the +proposal's owner can use, in the case that their proposal was censored, to +demonstrate that it was submitted but censored. + +Valid proposals will be displayed on the Politeia platform, where they can be +seen by all and discussed by Politeia members. + +There is a registration fee (0.1 DCR) for creating a Politeia account. Only +members who have paid this fee are eligible to submit proposals and comments, +and to make up/down votes on the Politeia web platform. + +Up/down votes do not affect proposal funding decisions, they are used as soft +signals and to determine display order. Up/down voting is not anonymous, the +up/down voting history of Politeia accounts will be public information. + +When a proposal is submitted and passes screening, it will be displayed on +Politeia but voting will not open immediately. The proposer has discretion to +participate in discussion with Decred stakeholders and make edits to their +proposal, then decide when to trigger the ticket-voting interval. When voting +is triggered, edits to the proposal can no longer be made. + +Ticket-voting is used to determine whether proposals are approved by Decred's +stake-governors. Ticket-voting is to be performed from a Decred wallet with +live tickets, it does not happen directly through the Politeia web platform. + +Politeia's aim is to serve as the decision-making force behind the Decred +Decentralized Autonomous Entity (DAE). This is an ambitious aim, Politeia and +its accompanying processes are in an experimental stage and thus subject to +change. + +Initially at least, the disbursal of funds to successful proposals will be a +manual process. Proposals that request funding should specify how much funding +they require, denominated in a national currency like $USD. They should also +specify a set of milestones or deliverables which will trigger the release of +funds. + +When a proposal is approved by ticket-voters, this gives a green light to the +proposing entity to begin work. When the first milestone is met, they can make +a request for the release of the first tranche of funding. This will be +reviewed and, where satisfactory, will be processed. It is expected that, +initially at least, all proposals requesting funding are paid in arrears. + +An example is that stakeholders expect developers to write and show code before +payment occurs. The existing contractors use this model. For example, Company 0 +carries the costs (and risk) for 4-6 weeks before payout occurs. This is a +feature and not a bug. Asking for a large sum of money without incurring costs +shifts the risk to the DAE and that incentivizes malicious behavior. ## How to submit a Politeia (Pi) proposal @@ -83,7 +116,8 @@ the admins. ### Who -In the *Who* section, describe the entity that is making the proposal, will complete the work, and will draw down on the proposal's budget. +In the *Who* section, describe the entity that is making the proposal, will +complete the work, and will draw down on the proposal's budget. ### When @@ -131,37 +165,63 @@ Week 2 deliverables ## Marketing Example ### What + ``` -This proposal would fund a Decred presence at Real Blockchain Conference 2018, in Dublin, Ireland, November 11-13. It would cover costs for a booth, swag, and people to staff the booth. +This proposal would fund a Decred presence at Real Blockchain Conference 2018, +in Dublin, Ireland, November 11-13. It would cover costs for a booth, swag, and +people to staff the booth. ``` + ### Why + ``` -Real Blockchain Conference is a top cryptocurrency conference and totally not made up. Last year's conference had 5,000 attendees and they seemed cool, good solid Decred stakeholder material. With epic swag and a physical embodiment of Stakey in attendance, a presence at this conference would raise awareness of Decred. +Real Blockchain Conference is a top cryptocurrency conference and totally not +made up. Last year's conference had 5,000 attendees and they seemed cool, good +solid Decred stakeholder material. With epic swag and a physical embodiment of +Stakey in attendance, a presence at this conference would raise awareness of +Decred. ``` + ### How (much) + ``` -I will organize Decred's presence at this event, it will take about 20 hours of my time at 40$/hour. $800 +I will organize Decred's presence at this event, it will take about 20 hours of +my time at 40$/hour: $800 Conference registration/booth fees: $3,000 Booth decorations: $1,000 Decred swag to give away: $2,000 -3 staff on the booth for 3 (10 hour) days each at $30/hr: (3 x 3 x 10 x 30) $2,700 +3 booth staff for 3 (10 hour) days each at $30/hr: (3 x 3 x 10 x 30) $2,700 Stakey costume: $500 -Stakey costume occupant: 3 (10 hour) days at $40/hr (that suit is warm!): $1,200 +Stakey costume occupant: 3 (10 hour) days at $40/hr (that suit is hot!): $1,200 Travel expenses for booth staff: Up to $2,000 -Accommodation for booth staff. We will stay at the conference hotel costing $200/night, it is unlikely that all booth staff need accommodation, but the maximum would be 200 x 3 nights x 4 staff = $2,400 +Accommodation for booth staff. We will stay at the conference hotel costing +$200/night, it is unlikely that all booth staff need accommodation, but the +maximum would be 200 x 3 nights x 4 staff: $2,400 Maximum total budget: $15,600 ``` ### Who + ``` -This proposal is submitted by @AllYourStake (on Slack, /u/StakeGovernor2000 on reddit). You may remember me as the organizer of Decred's presence at such blockchain events as Real Blockchain Conference 2017 and Buckets of Blockchain 2018. -I don't know exactly who the 3 booth staff and 1 Stakey suit wearer will be, I will be one of the staff and @Contributor1 is also interested. +This proposal is submitted by @AllYourStake (on Slack, /u/StakeGovernor2000 on +reddit). You may remember me as the organizer of Decred's presence at such +blockchain events as Real Blockchain Conference 2017 and Buckets of Blockchain +2018. + +I don't know exactly who the 3 booth staff and 1 Stakey suit wearer will be, I +will be one of the staff and @Contributor1 is also interested. ``` ### When + ``` -Registration fees are due by September 30th, I will pay these up-front and request full reimbursement immediately. -I will front the cost of the swag and Stakey suit, and claim this along with my travel/accommodation expenses and payment for my work, after the event. -Booth staff who are already Decred contributors will bill for their hours and expenses directly, I will serve as intermediary for any staff costs not associated with established contributors. -``` +Registration fees are due by September 30th, I will pay these up-front and +request full reimbursement immediately. + +I will front the cost of the swag and Stakey suit, and claim this along with my +travel/accommodation expenses and payment for my work, after the event. + +Booth staff who are already Decred contributors will bill for their hours and +expenses directly, I will serve as intermediary for any staff costs not +associated with established contributors. ``` diff --git a/politeiawww/cmsaddresswatcher.go b/politeiawww/cmsaddresswatcher.go index bd1604719..1e0070129 100644 --- a/politeiawww/cmsaddresswatcher.go +++ b/politeiawww/cmsaddresswatcher.go @@ -16,9 +16,9 @@ import ( pstypes "github.com/decred/dcrdata/pubsub/types/v3" "github.com/decred/politeia/mdstream" pd "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/politeiad/cache" cms "github.com/decred/politeia/politeiawww/api/cms/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" + "github.com/decred/politeia/politeiawww/cmsdatabase" database "github.com/decred/politeia/politeiawww/cmsdatabase" "github.com/decred/politeia/politeiawww/user" "github.com/decred/politeia/util" @@ -377,7 +377,7 @@ func (p *politeiawww) updateInvoicePayment(payment *database.Payments) error { func (p *politeiawww) invoiceStatusPaid(token string) error { dbInvoice, err := p.cmsDB.InvoiceByToken(token) if err != nil { - if err == cache.ErrRecordNotFound { + if err == cmsdatabase.ErrInvoiceNotFound { err = www.UserError{ ErrorCode: cms.ErrorStatusInvoiceNotFound, } diff --git a/politeiawww/comments.go b/politeiawww/comments.go index 86f0c3dee..9ebe44d7c 100644 --- a/politeiawww/comments.go +++ b/politeiawww/comments.go @@ -13,8 +13,6 @@ import ( "github.com/decred/politeia/decredplugin" pd "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/politeiad/cache" - cms "github.com/decred/politeia/politeiawww/api/cms/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/user" "github.com/decred/politeia/util" @@ -235,11 +233,14 @@ func (p *politeiawww) processNewComment(nc www.NewComment, u *user.User) (*www.N // Ensure proposal exists and is public pr, err := p.getProp(nc.Token) if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, + /* + // TODO + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + } } - } + */ return nil, err } @@ -274,8 +275,8 @@ func (p *politeiawww) processNewComment(nc www.NewComment, u *user.User) (*www.N return nil, www.UserError{ ErrorCode: www.ErrorStatusDuplicateComment, } - case cache.ErrRecordNotFound: - // No duplicate comment; continue + // TODO case cache.ErrRecordNotFound: + // No duplicate comment; continue default: // Some other error return nil, fmt.Errorf("decredCommentBySignature: %v", err) @@ -347,131 +348,6 @@ func (p *politeiawww) processNewComment(nc www.NewComment, u *user.User) (*www.N }, nil } -// processNewCommentInvoice sends a new comment decred plugin command to politeaid -// then fetches the new comment from the cache and returns it. -func (p *politeiawww) processNewCommentInvoice(nc www.NewComment, u *user.User) (*www.NewCommentReply, error) { - log.Tracef("processNewComment: %v %v", nc.Token, u.ID) - - ir, err := p.getInvoice(nc.Token) - if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: cms.ErrorStatusInvoiceNotFound, - } - } - return nil, err - } - - // Check to make sure the user is either an admin or the - // author of the invoice. - if !u.Admin && (ir.Username != u.Username) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusUserActionNotAllowed, - } - } - - // Ensure the public key is the user's active key - if nc.PublicKey != u.PublicKey() { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidSigningKey, - } - } - - // Validate signature - msg := nc.Token + nc.ParentID + nc.Comment - err = validateSignature(nc.PublicKey, nc.Signature, msg) - if err != nil { - return nil, err - } - - // Validate comment - err = validateComment(nc) - if err != nil { - return nil, err - } - - // Check to make sure that invoice isn't already approved or paid. - if ir.Status == cms.InvoiceStatusApproved || ir.Status == cms.InvoiceStatusPaid { - return nil, www.UserError{ - ErrorCode: cms.ErrorStatusWrongInvoiceStatus, - } - } - - // Setup plugin command - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - - dnc := convertNewCommentToDecredPlugin(nc) - payload, err := decredplugin.EncodeNewComment(dnc) - if err != nil { - return nil, err - } - - pc := pd.PluginCommand{ - Challenge: hex.EncodeToString(challenge), - ID: decredplugin.ID, - Command: decredplugin.CmdNewComment, - CommandID: decredplugin.CmdNewComment, - Payload: string(payload), - } - - // Send polieiad request - responseBody, err := p.makeRequest(http.MethodPost, - pd.PluginCommandRoute, pc) - if err != nil { - return nil, err - } - - // Handle response - var reply pd.PluginCommandReply - err = json.Unmarshal(responseBody, &reply) - if err != nil { - return nil, fmt.Errorf("could not unmarshal "+ - "PluginCommandReply: %v", err) - } - - err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) - if err != nil { - return nil, err - } - - ncr, err := decredplugin.DecodeNewCommentReply([]byte(reply.Payload)) - if err != nil { - return nil, err - } - - // Add comment to commentVotes in-memory cache - p.Lock() - p.commentVotes[nc.Token+ncr.CommentID] = counters{} - p.Unlock() - - // Get comment from cache - c, err := p.getComment(nc.Token, ncr.CommentID) - if err != nil { - return nil, fmt.Errorf("getComment: %v", err) - } - - if u.Admin { - invoiceUser, err := p.db.UserGetByUsername(ir.Username) - if err != nil { - return nil, fmt.Errorf("failed to get user by username %v %v", - ir.Username, err) - } - // Fire off new invoice comment event - p.fireEvent(EventTypeInvoiceComment, - EventDataInvoiceComment{ - Token: nc.Token, - User: invoiceUser, - }, - ) - } - return &www.NewCommentReply{ - Comment: *c, - }, nil -} - // processLikeComment processes an upvote/downvote on a comment. func (p *politeiawww) processLikeComment(lc www.LikeComment, u *user.User) (*www.LikeCommentReply, error) { log.Debugf("processLikeComment: %v %v %v", lc.Token, lc.CommentID, u.ID) @@ -508,11 +384,14 @@ func (p *politeiawww) processLikeComment(lc www.LikeComment, u *user.User) (*www // Ensure proposal exists and is public pr, err := p.getProp(lc.Token) if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, + /* + // TODO + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + } } - } + */ return nil, err } @@ -541,11 +420,14 @@ func (p *politeiawww) processLikeComment(lc www.LikeComment, u *user.User) (*www // Ensure comment exists and has not been censored. c, err := p.decredCommentGetByID(lc.Token, lc.CommentID) if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusCommentNotFound, + /* + // TODO + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: www.ErrorStatusCommentNotFound, + } } - } + */ return nil, err } if c.Censored { diff --git a/politeiawww/convert.go b/politeiawww/convert.go index 9770facca..bec685d59 100644 --- a/politeiawww/convert.go +++ b/politeiawww/convert.go @@ -16,7 +16,6 @@ import ( "github.com/decred/politeia/decredplugin" "github.com/decred/politeia/mdstream" pd "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/politeiad/cache" cms "github.com/decred/politeia/politeiawww/api/cms/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" www2 "github.com/decred/politeia/politeiawww/api/www/v2" @@ -316,234 +315,6 @@ func convertPropStatusToState(status www.PropStatusT) www.PropStateT { return www.PropStateInvalid } -func convertPropStatusFromCache(s cache.RecordStatusT) www.PropStatusT { - switch s { - case cache.RecordStatusNotFound: - return www.PropStatusNotFound - case cache.RecordStatusNotReviewed: - return www.PropStatusNotReviewed - case cache.RecordStatusCensored: - return www.PropStatusCensored - case cache.RecordStatusPublic: - return www.PropStatusPublic - case cache.RecordStatusUnreviewedChanges: - return www.PropStatusUnreviewedChanges - case cache.RecordStatusArchived: - return www.PropStatusAbandoned - } - return www.PropStatusInvalid -} - -// convertFileFromMetadata returns a politeiawww v1 Metadata that was converted -// from a politeiad File. User specified metadata is store as a file in -// politeiad so that it is included in the merkle root that politeiad -// calculates. -func convertMetadataFromFile(f cache.File) www.Metadata { - var hint string - switch f.Name { - case mdstream.FilenameProposalMetadata: - hint = www.HintProposalMetadata - } - return www.Metadata{ - Digest: f.Digest, - Hint: hint, - Payload: f.Payload, - } -} - -func convertPropFromCache(r cache.Record) (*www.ProposalRecord, error) { - // Decode metadata stream payloads - var ( - // The name was originally saved in the ProposalGeneralV1 - // mdstream but was moved to the ProposalData mdsteam, which - // is saved to politeiad as a File, not a MetadataStream. - name string - pubkey string - sig string - - statusesV1 []mdstream.RecordStatusChangeV1 - statusesV2 []mdstream.RecordStatusChangeV2 - err error - - token = r.CensorshipRecord.Token - ) - for _, ms := range r.Metadata { - switch ms.ID { - case mdstream.IDProposalGeneral: - // General metadata - v, err := mdstream.DecodeVersion([]byte(ms.Payload)) - if err != nil { - return nil, fmt.Errorf("DecodeVersion %v: %v", - ms.ID, err) - } - switch v { - case 1: - pg, err := mdstream.DecodeProposalGeneralV1([]byte(ms.Payload)) - if err != nil { - return nil, fmt.Errorf("DecodeProposalGeneralV1: %v", err) - } - name = pg.Name - pubkey = pg.PublicKey - sig = pg.Signature - case 2: - pg, err := mdstream.DecodeProposalGeneralV2([]byte(ms.Payload)) - if err != nil { - return nil, fmt.Errorf("DecodeProposalGeneralV2: %v", err) - } - pubkey = pg.PublicKey - sig = pg.Signature - default: - return nil, fmt.Errorf("unknown ProposalGeneral version %v", ms) - } - - case mdstream.IDRecordStatusChange: - // Status change metadata - b := []byte(ms.Payload) - statusesV1, statusesV2, err = mdstream.DecodeRecordStatusChanges(b) - if err != nil { - return nil, fmt.Errorf("DecodeRecordStatusChanges: %v", err) - } - - // Verify the signatures - for _, v := range statusesV2 { - err := v.VerifySignature(token) - if err != nil { - // This is not good! - return nil, fmt.Errorf("invalid status change signature: %v", v) - } - } - - case decredplugin.MDStreamAuthorizeVote: - // Valid proposal mdstream but not needed for a ProposalRecord - log.Tracef("convertPropFromCache: skipping mdstream %v", - decredplugin.MDStreamAuthorizeVote) - case decredplugin.MDStreamVoteBits: - // Valid proposal mdstream but not needed for a ProposalRecord - log.Tracef("convertPropFromCache: skipping mdstream %v", - decredplugin.MDStreamVoteBits) - case decredplugin.MDStreamVoteSnapshot: - // Valid proposal mdstream but not needed for a ProposalRecord - log.Tracef("convertPropFromCache: skipping mdstream %v", - decredplugin.MDStreamVoteSnapshot) - default: - return nil, fmt.Errorf("invalid mdstream: %v", ms) - } - } - - // Compile proposal status change metadata - var ( - changeMsg string - changeMsgTimestamp int64 - publishedAt int64 - censoredAt int64 - abandonedAt int64 - ) - for _, v := range statusesV1 { - // Keep the most recent status change message. This is what - // will be returned as part of the ProposalRecord. - if v.Timestamp > changeMsgTimestamp { - changeMsg = v.StatusChangeMessage - changeMsgTimestamp = v.Timestamp - } - - switch convertPropStatusFromPD(v.NewStatus) { - case www.PropStatusPublic: - publishedAt = v.Timestamp - case www.PropStatusCensored: - censoredAt = v.Timestamp - case www.PropStatusAbandoned: - abandonedAt = v.Timestamp - } - } - for _, v := range statusesV2 { - // Keep the most recent status change message. This is what - // will be returned as part of the ProposalRecord. - if v.Timestamp > changeMsgTimestamp { - changeMsg = v.StatusChangeMessage - changeMsgTimestamp = v.Timestamp - } - - switch convertPropStatusFromPD(v.NewStatus) { - case www.PropStatusPublic: - publishedAt = v.Timestamp - case www.PropStatusCensored: - censoredAt = v.Timestamp - case www.PropStatusAbandoned: - abandonedAt = v.Timestamp - } - } - - // Convert files. The ProposalMetadata mdstream is saved to - // politeiad as a File, not as a MetadataStream, so we have to - // account for this when preparing the files. - var ( - pm www.ProposalMetadata - files = make([]www.File, 0, len(r.Files)) - metadata = make([]www.Metadata, 0, len(r.Files)) - ) - for _, f := range r.Files { - switch f.Name { - case mdstream.FilenameProposalMetadata: - metadata = append(metadata, convertMetadataFromFile(f)) - - // Extract the proposal metadata from the payload - b, err := base64.StdEncoding.DecodeString(f.Payload) - if err != nil { - return nil, fmt.Errorf("decode file payload %v: %v", - mdstream.FilenameProposalMetadata, err) - } - err = json.Unmarshal(b, &pm) - if err != nil { - return nil, fmt.Errorf("unmarshal ProposalMetadata: %v", err) - } - - name = pm.Name - - continue - } - - files = append(files, - www.File{ - Name: f.Name, - MIME: f.MIME, - Digest: f.Digest, - Payload: f.Payload, - }) - } - - status := convertPropStatusFromCache(r.Status) - - // The UserId, Username, and NumComments fields are returned - // as zero values since a cache record does not contain that - // data. - return &www.ProposalRecord{ - Name: name, - State: convertPropStatusToState(status), - Status: status, - Timestamp: r.Timestamp, - UserId: "", - Username: "", - PublicKey: pubkey, - Signature: sig, - NumComments: 0, - Version: r.Version, - StatusChangeMessage: changeMsg, - PublishedAt: publishedAt, - CensoredAt: censoredAt, - AbandonedAt: abandonedAt, - LinkTo: pm.LinkTo, - LinkBy: pm.LinkBy, - LinkedFrom: []string{}, - Files: files, - Metadata: metadata, - CensorshipRecord: www.CensorshipRecord{ - Token: r.CensorshipRecord.Token, - Merkle: r.CensorshipRecord.Merkle, - Signature: r.CensorshipRecord.Signature, - }, - }, nil -} - func convertNewCommentToDecredPlugin(nc www.NewComment) decredplugin.NewComment { return decredplugin.NewComment{ Token: nc.Token, @@ -586,7 +357,7 @@ func convertCensorCommentToDecred(cc www.CensorComment) decredplugin.CensorComme func convertCommentFromDecred(c decredplugin.Comment) www.Comment { // Upvotes, Downvotes, UserID, and Username are filled in as zero - // values since a cache comment does not contain this data. + // values since a decred plugin comment does not contain this data. return www.Comment{ Token: c.Token, ParentID: c.ParentID, @@ -605,21 +376,6 @@ func convertCommentFromDecred(c decredplugin.Comment) www.Comment { } } -func convertPluginToCache(p Plugin) cache.Plugin { - settings := make([]cache.PluginSetting, 0, len(p.Settings)) - for _, s := range p.Settings { - settings = append(settings, cache.PluginSetting{ - Key: s.Key, - Value: s.Value, - }) - } - return cache.Plugin{ - ID: p.ID, - Version: p.Version, - Settings: settings, - } -} - func convertVoteOptionFromDecred(vo decredplugin.VoteOption) www.VoteOption { return www.VoteOption{ Id: vo.Id, @@ -1079,124 +835,6 @@ func convertRecordToDatabaseInvoice(p pd.Record) (*cmsdatabase.Invoice, error) { return &dbInvoice, nil } -func convertDCCFromCache(r cache.Record) cms.DCCRecord { - dcc := cms.DCCRecord{} - // Decode metadata streams - var md mdstream.DCCGeneral - var c mdstream.DCCStatusChange - for _, v := range r.Metadata { - switch v.ID { - case mdstream.IDRecordStatusChange: - // Ignore initial stream change since it's just the automatic change from - // unvetted to vetted - continue - case mdstream.IDDCCGeneral: - // General invoice metadata - m, err := mdstream.DecodeDCCGeneral([]byte(v.Payload)) - if err != nil { - log.Errorf("convertDCCFromCache: decode md stream: "+ - "token:%v error:%v payload:%v", - r.CensorshipRecord.Token, err, v) - continue - } - md = *m - - case mdstream.IDDCCStatusChange: - // Invoice status changes - m, err := mdstream.DecodeDCCStatusChange([]byte(v.Payload)) - if err != nil { - log.Errorf("convertDCCFromCache: decode md stream: "+ - "token:%v error:%v payload:%v", - r.CensorshipRecord.Token, err, v) - continue - } - - // Calc submission, approval/rejection timestamps - // Hold the most recent status change. - for _, s := range m { - switch s.NewStatus { - case cms.DCCStatusActive: - dcc.TimeSubmitted = s.Timestamp - case cms.DCCStatusApproved, cms.DCCStatusRejected: - dcc.TimeReviewed = s.Timestamp - } - c = s - } - case mdstream.IDDCCSupportOpposition: - // Support and Opposition - so, err := mdstream.DecodeDCCSupportOpposition([]byte(v.Payload)) - if err != nil { - log.Errorf("convertDCCFromCache: decode md stream: "+ - "token:%v error:%v payload:%v", - r.CensorshipRecord.Token, err, v) - continue - } - supportPubkeys := make([]string, 0, len(so)) - opposePubkeys := make([]string, 0, len(so)) - // Tabulate all support and opposition - for _, s := range so { - if s.Vote == supportString { - supportPubkeys = append(supportPubkeys, s.PublicKey) - } else if s.Vote == opposeString { - opposePubkeys = append(opposePubkeys, s.PublicKey) - } - } - dcc.SupportUserIDs = supportPubkeys - dcc.OppositionUserIDs = opposePubkeys - } - } - - // Convert files - var di cms.DCCInput - - var f www.File - - for _, v := range r.Files { - f = www.File{ - Name: v.Name, - MIME: v.MIME, - Digest: v.Digest, - Payload: v.Payload, - } - - // Parse invoice json - if f.Name == dccFile { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - log.Errorf("convertDCCFromCache: decode dcc: "+ - "token:%v error:%v payload:%v", - r.CensorshipRecord.Token, err, f.Payload) - continue - } - - err = json.Unmarshal(b, &di) - if err != nil { - log.Errorf("convertDCCFromCache: unmarshal DCCInput: "+ - "token:%v error:%v payload:%v", - r.CensorshipRecord.Token, err, f.Payload) - continue - } - } - } - - dcc.Status = c.NewStatus - dcc.StatusChangeReason = c.Reason - dcc.Timestamp = r.Timestamp - dcc.SponsorUserID = "" - dcc.SponsorUsername = "" - dcc.PublicKey = md.PublicKey - dcc.Signature = md.Signature - dcc.File = f - dcc.CensorshipRecord = www.CensorshipRecord{ - Token: r.CensorshipRecord.Token, - Merkle: r.CensorshipRecord.Merkle, - Signature: r.CensorshipRecord.Signature, - } - dcc.DCC = di - - return dcc -} - func convertRecordToDatabaseDCC(p pd.Record) (*cmsdatabase.DCC, error) { dbDCC := cmsdatabase.DCC{ Files: convertRecordFilesToWWW(p.Files), @@ -1268,88 +906,6 @@ func convertRecordToDatabaseDCC(p pd.Record) (*cmsdatabase.DCC, error) { return &dbDCC, nil } -func convertCacheToDatabaseDCC(p cache.Record) (*cmsdatabase.DCC, error) { - dbDCC := cmsdatabase.DCC{ - Token: p.CensorshipRecord.Token, - ServerSignature: p.CensorshipRecord.Signature, - } - - fs := make([]www.File, 0, len(p.Files)) - for _, v := range p.Files { - f := www.File{ - Name: v.Name, - MIME: v.MIME, - Digest: v.Digest, - Payload: v.Payload, - } - fs = append(fs, f) - } - dbDCC.Files = fs - - // Decode invoice file - for _, v := range p.Files { - if v.Name == dccFile { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return nil, err - } - - var dcc cms.DCCInput - err = json.Unmarshal(b, &dcc) - if err != nil { - return nil, fmt.Errorf("could not decode DCC input data: token '%v': %v", - p.CensorshipRecord.Token, err) - } - dbDCC.Type = dcc.Type - dbDCC.NomineeUserID = dcc.NomineeUserID - dbDCC.SponsorStatement = dcc.SponsorStatement - dbDCC.Domain = dcc.Domain - dbDCC.ContractorType = dcc.ContractorType - } - } - - for _, m := range p.Metadata { - switch m.ID { - case mdstream.IDRecordStatusChange: - // Ignore initial stream change since it's just the automatic change from - // unvetted to vetted - continue - case mdstream.IDDCCGeneral: - var mdGeneral mdstream.DCCGeneral - err := json.Unmarshal([]byte(m.Payload), &mdGeneral) - if err != nil { - return nil, fmt.Errorf("could not decode metadata '%v' token '%v': %v", - p.Metadata, p.CensorshipRecord.Token, err) - } - - dbDCC.Timestamp = mdGeneral.Timestamp - dbDCC.PublicKey = mdGeneral.PublicKey - dbDCC.UserSignature = mdGeneral.Signature - - case mdstream.IDDCCStatusChange: - sc, err := mdstream.DecodeDCCStatusChange([]byte(m.Payload)) - if err != nil { - return nil, fmt.Errorf("could not decode metadata '%v' token '%v': %v", - m, p.CensorshipRecord.Token, err) - } - - // We don't need all of the status changes. - // Just the most recent one. - for _, s := range sc { - dbDCC.Status = s.NewStatus - dbDCC.StatusChangeReason = s.Reason - } - default: - // Log error but proceed - log.Errorf("initializeInventory: invalid "+ - "metadata stream ID %v token %v", - m.ID, p.CensorshipRecord.Token) - } - } - - return &dbDCC, nil -} - func convertDCCDatabaseToRecord(dbDCC *cmsdatabase.DCC) cms.DCCRecord { dccRecord := cms.DCCRecord{} diff --git a/politeiawww/decred.go b/politeiawww/decred.go index 71f137c53..51f94194c 100644 --- a/politeiawww/decred.go +++ b/politeiawww/decred.go @@ -5,15 +5,7 @@ package main import ( - "encoding/hex" - "encoding/json" - "fmt" - "net/http" - "github.com/decred/politeia/decredplugin" - pd "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/politeiad/cache" - "github.com/decred/politeia/util" ) // decredGetComment sends the decred plugin getcomment command to the cache and @@ -25,19 +17,10 @@ func (p *politeiawww) decredGetComment(gc decredplugin.GetComment) (*decredplugi return nil, err } - pc := cache.PluginCommand{ - ID: decredplugin.ID, - Command: decredplugin.CmdGetComment, - CommandPayload: string(payload), - } + // TODO this needs to use the politeiad plugin command + var reply string - // Get comment from the cache - reply, err := p.cache.PluginExec(pc) - if err != nil { - return nil, err - } - - gcr, err := decredplugin.DecodeGetCommentReply([]byte(reply.Payload)) + gcr, err := decredplugin.DecodeGetCommentReply([]byte(reply)) if err != nil { return nil, err } @@ -72,25 +55,15 @@ func (p *politeiawww) decredGetComments(token string) ([]decredplugin.Comment, e gc := decredplugin.GetComments{ Token: token, } - payload, err := decredplugin.EncodeGetComments(gc) if err != nil { return nil, err } - pc := cache.PluginCommand{ - ID: decredplugin.ID, - Command: decredplugin.CmdGetComments, - CommandPayload: string(payload), - } - - // Get comments from the cache - reply, err := p.cache.PluginExec(pc) - if err != nil { - return nil, fmt.Errorf("PluginExec: %v", err) - } + // TODO this needs to use the politeiad plugin command + var reply string - gcr, err := decredplugin.DecodeGetCommentsReply([]byte(reply.Payload)) + gcr, err := decredplugin.DecodeGetCommentsReply([]byte(reply)) if err != nil { return nil, err } @@ -114,373 +87,13 @@ func (p *politeiawww) decredGetNumComments(tokens []string) (map[string]int, err return nil, err } - pc := cache.PluginCommand{ - ID: decredplugin.ID, - Command: decredplugin.CmdGetNumComments, - CommandPayload: string(payload), - } + // TODO this needs to use the politeiad plugin command + var reply string - // Send plugin comand - reply, err := p.cache.PluginExec(pc) - if err != nil { - return nil, fmt.Errorf("PluginExec: %v", err) - } - - gncr, err := decredplugin.DecodeGetNumCommentsReply( - []byte(reply.Payload)) + gncr, err := decredplugin.DecodeGetNumCommentsReply([]byte(reply)) if err != nil { return nil, err } return gncr.NumComments, nil } - -// decredCommentLikes sends the decred plugin commentlikes command to the cache -// and returns all of the comment likes for the passed in comment. -func (p *politeiawww) decredCommentLikes(token, commentID string) ([]decredplugin.LikeComment, error) { - // Setup plugin command - cl := decredplugin.CommentLikes{ - Token: token, - CommentID: commentID, - } - - payload, err := decredplugin.EncodeCommentLikes(cl) - if err != nil { - return nil, err - } - - pc := cache.PluginCommand{ - ID: decredplugin.ID, - Command: decredplugin.CmdCommentLikes, - CommandPayload: string(payload), - } - - // Get comment likes from cache - reply, err := p.cache.PluginExec(pc) - if err != nil { - return nil, err - } - - clr, err := decredplugin.DecodeCommentLikesReply([]byte(reply.Payload)) - if err != nil { - return nil, err - } - - return clr.CommentLikes, nil -} - -// decredPropCommentLikes sends the decred plugin proposalcommentslikes command -// to the cache and returns all of the comment likes for the passed in proposal -// token. -func (p *politeiawww) decredPropCommentLikes(token string) ([]decredplugin.LikeComment, error) { - // Setup plugin command - pcl := decredplugin.GetProposalCommentsLikes{ - Token: token, - } - - payload, err := decredplugin.EncodeGetProposalCommentsLikes(pcl) - if err != nil { - return nil, err - } - - pc := cache.PluginCommand{ - ID: decredplugin.ID, - Command: decredplugin.CmdProposalCommentsLikes, - CommandPayload: string(payload), - } - - // Get proposal comment likes from cache - reply, err := p.cache.PluginExec(pc) - if err != nil { - return nil, err - } - - rp := []byte(reply.Payload) - pclr, err := decredplugin.DecodeGetProposalCommentsLikesReply(rp) - if err != nil { - return nil, err - } - - return pclr.CommentsLikes, nil -} - -// decredVoteDetails sends the decred plugin votedetails command to the cache -// and returns the vote details for the passed in proposal. -func (p *politeiawww) decredVoteDetails(token string) (*decredplugin.VoteDetailsReply, error) { - // Setup plugin command - vd := decredplugin.VoteDetails{ - Token: token, - } - - payload, err := decredplugin.EncodeVoteDetails(vd) - if err != nil { - return nil, err - } - - pc := cache.PluginCommand{ - ID: decredplugin.ID, - Command: decredplugin.CmdVoteDetails, - CommandPayload: string(payload), - } - - // Get vote details from cache - reply, err := p.cache.PluginExec(pc) - if err != nil { - return nil, err - } - - vdr, err := decredplugin.DecodeVoteDetailsReply([]byte(reply.Payload)) - if err != nil { - return nil, err - } - - return vdr, nil -} - -// decredProposalVotes sends the decred plugin proposalvotes command to the -// cache and returns the vote results for the passed in proposal. -func (p *politeiawww) decredProposalVotes(token string) (*decredplugin.VoteResultsReply, error) { - // Setup plugin command - vr := decredplugin.VoteResults{ - Token: token, - } - - payload, err := decredplugin.EncodeVoteResults(vr) - if err != nil { - return nil, err - } - - pc := cache.PluginCommand{ - ID: decredplugin.ID, - Command: decredplugin.CmdProposalVotes, - CommandPayload: string(payload), - } - - // Get proposal votes from cache - reply, err := p.cache.PluginExec(pc) - if err != nil { - return nil, err - } - - vrr, err := decredplugin.DecodeVoteResultsReply([]byte(reply.Payload)) - if err != nil { - return nil, err - } - - return vrr, nil -} - -// decredInventory sends the decred plugin inventory command to the cache and -// returns the decred plugin inventory. -func (p *politeiawww) decredInventory() (*decredplugin.InventoryReply, error) { - // Setup plugin command - i := decredplugin.Inventory{} - payload, err := decredplugin.EncodeInventory(i) - if err != nil { - return nil, err - } - - pc := cache.PluginCommand{ - ID: decredplugin.ID, - Command: decredplugin.CmdInventory, - CommandPayload: string(payload), - } - - // Get cache inventory - reply, err := p.cache.PluginExec(pc) - if err != nil { - return nil, err - } - - ir, err := decredplugin.DecodeInventoryReply([]byte(reply.Payload)) - if err != nil { - return nil, err - } - - return ir, nil -} - -// decredTokenInventory sends the decred plugin tokeninventory command to the -// cache. -// -// This function should not be called directly in most circumstances due to its -// reliance on the lazy loaded VoteResults cache table. The politeiawww method -// tokenInventory() should be called instead. -func (p *politeiawww) decredTokenInventory(bestBlock uint64, includeUnvetted bool) (*decredplugin.TokenInventoryReply, error) { - payload, err := decredplugin.EncodeTokenInventory( - decredplugin.TokenInventory{ - BestBlock: bestBlock, - Unvetted: includeUnvetted, - }) - if err != nil { - return nil, err - } - - pc := cache.PluginCommand{ - ID: decredplugin.ID, - Command: decredplugin.CmdTokenInventory, - CommandPayload: string(payload), - } - - reply, err := p.cache.PluginExec(pc) - if err != nil { - return nil, err - } - - tir, err := decredplugin.DecodeTokenInventoryReply([]byte(reply.Payload)) - if err != nil { - return nil, err - } - - return tir, nil -} - -// decredLoadVoteResults sends the loadvotesummaries command to politeiad. -func (p *politeiawww) decredLoadVoteResults(bestBlock uint64) (*decredplugin.LoadVoteResultsReply, error) { - // Setup plugin command - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - - lvr := decredplugin.LoadVoteResults{ - BestBlock: bestBlock, - } - payload, err := decredplugin.EncodeLoadVoteResults(lvr) - if err != nil { - return nil, err - } - - pc := pd.PluginCommand{ - Challenge: hex.EncodeToString(challenge), - ID: decredplugin.ID, - Command: decredplugin.CmdLoadVoteResults, - CommandID: decredplugin.CmdLoadVoteResults, - Payload: string(payload), - } - - // Send plugin command to politeiad - respBody, err := p.makeRequest(http.MethodPost, - pd.PluginCommandRoute, pc) - if err != nil { - return nil, err - } - - // Handle response - var pcr pd.PluginCommandReply - err = json.Unmarshal(respBody, &pcr) - if err != nil { - return nil, err - } - - err = util.VerifyChallenge(p.cfg.Identity, challenge, pcr.Response) - if err != nil { - return nil, err - } - - b := []byte(pcr.Payload) - reply, err := decredplugin.DecodeLoadVoteResultsReply(b) - if err != nil { - return nil, err - } - - return reply, nil -} - -// decredBatchVoteSummary uses the decred plugin batch vote summary command to -// request a vote summary for a set of proposals from the cache. -// -// This function should not be called directly in most circumstances due to its -// reliance on the lazy loaded VoteResults cache table. The politeiawww method -// getVoteSummaries() should be called instead. -func (p *politeiawww) decredBatchVoteSummary(tokens []string, bestBlock uint64) (*decredplugin.BatchVoteSummaryReply, error) { - bvs := decredplugin.BatchVoteSummary{ - Tokens: tokens, - BestBlock: bestBlock, - } - payload, err := decredplugin.EncodeBatchVoteSummary(bvs) - if err != nil { - return nil, err - } - - pc := cache.PluginCommand{ - ID: decredplugin.ID, - Command: decredplugin.CmdBatchVoteSummary, - CommandPayload: string(payload), - } - - res, err := p.cache.PluginExec(pc) - if err != nil { - return nil, err - } - - reply, err := decredplugin.DecodeBatchVoteSummaryReply([]byte(res.Payload)) - if err != nil { - return nil, err - } - - return reply, nil -} - -// decredVoteSummary uses the decred plugin vote summary command to request a -// vote summary for a specific proposal from the cache. -// -// This function should not be called directly in most circumstances due to its -// reliance on the lazy loaded VoteResults cache table. The politeiawww method -// voteSummaryGet() should be called instead. -func (p *politeiawww) decredVoteSummary(token string, bestBlock uint64) (*decredplugin.VoteSummaryReply, error) { - v := decredplugin.VoteSummary{ - Token: token, - BestBlock: bestBlock, - } - payload, err := decredplugin.EncodeVoteSummary(v) - if err != nil { - return nil, err - } - - pc := cache.PluginCommand{ - ID: decredplugin.ID, - Command: decredplugin.CmdVoteSummary, - CommandPayload: string(payload), - } - - resp, err := p.cache.PluginExec(pc) - if err != nil { - return nil, err - } - - reply, err := decredplugin.DecodeVoteSummaryReply([]byte(resp.Payload)) - if err != nil { - return nil, err - } - - return reply, nil -} - -func (p *politeiawww) decredLinkedFrom(tokens []string) (*decredplugin.LinkedFromReply, error) { - lf := decredplugin.LinkedFrom{ - Tokens: tokens, - } - payload, err := decredplugin.EncodeLinkedFrom(lf) - if err != nil { - return nil, err - } - - pc := cache.PluginCommand{ - ID: decredplugin.ID, - Command: decredplugin.CmdLinkedFrom, - CommandPayload: string(payload), - } - - resp, err := p.cache.PluginExec(pc) - if err != nil { - return nil, err - } - - reply, err := decredplugin.DecodeLinkedFromReply([]byte(resp.Payload)) - if err != nil { - return nil, err - } - - return reply, nil -} diff --git a/politeiawww/invoices.go b/politeiawww/invoices.go index 40297d66b..79b500cc9 100644 --- a/politeiawww/invoices.go +++ b/politeiawww/invoices.go @@ -18,11 +18,13 @@ import ( "time" "github.com/decred/dcrd/dcrutil" + "github.com/decred/politeia/decredplugin" "github.com/decred/politeia/mdstream" pd "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/identity" cms "github.com/decred/politeia/politeiawww/api/cms/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" + "github.com/decred/politeia/politeiawww/cmsdatabase" database "github.com/decred/politeia/politeiawww/cmsdatabase" "github.com/decred/politeia/politeiawww/user" wwwutil "github.com/decred/politeia/politeiawww/util" @@ -1560,6 +1562,131 @@ func (p *politeiawww) processInvoices(ai cms.Invoices, u *user.User) (*cms.UserI return &reply, nil } +// processNewCommentInvoice sends a new comment decred plugin command to politeaid +// then fetches the new comment from the cache and returns it. +func (p *politeiawww) processNewCommentInvoice(nc www.NewComment, u *user.User) (*www.NewCommentReply, error) { + log.Tracef("processNewComment: %v %v", nc.Token, u.ID) + + ir, err := p.getInvoice(nc.Token) + if err != nil { + if err == cmsdatabase.ErrInvoiceNotFound { + err = www.UserError{ + ErrorCode: cms.ErrorStatusInvoiceNotFound, + } + } + return nil, err + } + + // Check to make sure the user is either an admin or the + // author of the invoice. + if !u.Admin && (ir.Username != u.Username) { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusUserActionNotAllowed, + } + } + + // Ensure the public key is the user's active key + if nc.PublicKey != u.PublicKey() { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidSigningKey, + } + } + + // Validate signature + msg := nc.Token + nc.ParentID + nc.Comment + err = validateSignature(nc.PublicKey, nc.Signature, msg) + if err != nil { + return nil, err + } + + // Validate comment + err = validateComment(nc) + if err != nil { + return nil, err + } + + // Check to make sure that invoice isn't already approved or paid. + if ir.Status == cms.InvoiceStatusApproved || ir.Status == cms.InvoiceStatusPaid { + return nil, www.UserError{ + ErrorCode: cms.ErrorStatusWrongInvoiceStatus, + } + } + + // Setup plugin command + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return nil, err + } + + dnc := convertNewCommentToDecredPlugin(nc) + payload, err := decredplugin.EncodeNewComment(dnc) + if err != nil { + return nil, err + } + + pc := pd.PluginCommand{ + Challenge: hex.EncodeToString(challenge), + ID: decredplugin.ID, + Command: decredplugin.CmdNewComment, + CommandID: decredplugin.CmdNewComment, + Payload: string(payload), + } + + // Send polieiad request + responseBody, err := p.makeRequest(http.MethodPost, + pd.PluginCommandRoute, pc) + if err != nil { + return nil, err + } + + // Handle response + var reply pd.PluginCommandReply + err = json.Unmarshal(responseBody, &reply) + if err != nil { + return nil, fmt.Errorf("could not unmarshal "+ + "PluginCommandReply: %v", err) + } + + err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) + if err != nil { + return nil, err + } + + ncr, err := decredplugin.DecodeNewCommentReply([]byte(reply.Payload)) + if err != nil { + return nil, err + } + + // Add comment to commentVotes in-memory cache + p.Lock() + p.commentVotes[nc.Token+ncr.CommentID] = counters{} + p.Unlock() + + // Get comment from cache + c, err := p.getComment(nc.Token, ncr.CommentID) + if err != nil { + return nil, fmt.Errorf("getComment: %v", err) + } + + if u.Admin { + invoiceUser, err := p.db.UserGetByUsername(ir.Username) + if err != nil { + return nil, fmt.Errorf("failed to get user by username %v %v", + ir.Username, err) + } + // Fire off new invoice comment event + p.fireEvent(EventTypeInvoiceComment, + EventDataInvoiceComment{ + Token: nc.Token, + User: invoiceUser, + }, + ) + } + return &www.NewCommentReply{ + Comment: *c, + }, nil +} + // processCommentsGet returns all comments for a given proposal. If the user is // logged in the user's last access time for the given comments will also be // returned. diff --git a/politeiawww/log.go b/politeiawww/log.go index 4953ab61c..d98ff1495 100644 --- a/politeiawww/log.go +++ b/politeiawww/log.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017 The Decred developers +// Copyright (c) 2017-2020 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -9,7 +9,6 @@ import ( "os" "path/filepath" - cachedb "github.com/decred/politeia/politeiad/cache/cockroachdb" cmsdb "github.com/decred/politeia/politeiawww/cmsdatabase/cockroachdb" "github.com/decred/politeia/politeiawww/user/cockroachdb" "github.com/decred/politeia/politeiawww/user/localdb" @@ -45,26 +44,25 @@ var ( // application shutdown. logRotator *rotator.Rotator - log = backendLog.Logger("PWWW") - localdbLog = backendLog.Logger("LODB") - cockroachdbLog = backendLog.Logger("CODB") - wsdcrdataLog = backendLog.Logger("WSDD") + log = backendLog.Logger("PWWW") + userdbLog = backendLog.Logger("USDB") + cmsdbLog = backendLog.Logger("CMDB") + wsdcrdataLog = backendLog.Logger("WSDD") ) // Initialize package-global logger variables. func init() { - localdb.UseLogger(localdbLog) - cockroachdb.UseLogger(cockroachdbLog) - cachedb.UseLogger(cockroachdbLog) - cmsdb.UseLogger(cockroachdbLog) + localdb.UseLogger(userdbLog) + cockroachdb.UseLogger(userdbLog) + cmsdb.UseLogger(cmsdbLog) wsdcrdata.UseLogger(wsdcrdataLog) } // subsystemLoggers maps each subsystem identifier to its associated logger. var subsystemLoggers = map[string]slog.Logger{ "PWWW": log, - "LODB": localdbLog, - "CODB": cockroachdbLog, + "USDB": userdbLog, + "CMDB": cmsdbLog, "WSDD": wsdcrdataLog, } diff --git a/politeiawww/politeiawww.go b/politeiawww/politeiawww.go index 74f76ae27..a14585fe9 100644 --- a/politeiawww/politeiawww.go +++ b/politeiawww/politeiawww.go @@ -19,7 +19,6 @@ import ( exptypes "github.com/decred/dcrdata/explorer/types/v2" pstypes "github.com/decred/dcrdata/pubsub/types/v3" "github.com/decred/politeia/politeiad/api/v1/mime" - "github.com/decred/politeia/politeiad/cache" www "github.com/decred/politeia/politeiawww/api/www/v1" www2 "github.com/decred/politeia/politeiawww/api/www/v2" "github.com/decred/politeia/politeiawww/cmsdatabase" @@ -90,14 +89,11 @@ type politeiawww struct { cfg *config router *mux.Router sessions sessions.Store + plugins []Plugin ws map[string]map[string]*wsContext // [uuid][]*context wsMtx sync.RWMutex - // Cache - cache cache.Cache - plugins []Plugin - // Politeiad client client *http.Client @@ -121,10 +117,6 @@ type politeiawww struct { userPaywallPool map[uuid.UUID]paywallPoolMember // [userid][paywallPoolMember] commentVotes map[string]counters // [token+commentID]counters - // voteSummaries is a lazy loaded cache of votes summaries for - // proposals whose voting period has ended. - voteSummaries map[string]www.VoteSummary // [token]VoteSummary - // XXX userEmails is a temporary measure until the user by email // lookups are completely removed from politeiawww. userEmails map[string]uuid.UUID // [email]userID @@ -135,12 +127,6 @@ type politeiawww struct { // wsDcrdata is a dcrdata websocket client wsDcrdata *wsdcrdata.Client - - // The current best block is cached and updated using a websocket - // subscription to dcrdata. If the websocket connection is not active, - // the dcrdata best block route of politeiad is used as a fallback. - bestBlock uint64 - bbMtx sync.RWMutex } // XXX rig this up diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 61257c027..2735838fc 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -22,7 +22,6 @@ import ( "github.com/decred/politeia/mdstream" pd "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/identity" - "github.com/decred/politeia/politeiad/cache" www "github.com/decred/politeia/politeiawww/api/www/v1" www2 "github.com/decred/politeia/politeiawww/api/www/v2" "github.com/decred/politeia/politeiawww/user" @@ -255,18 +254,22 @@ func (p *politeiawww) validateProposalMetadata(pm www.ProposalMetadata) error { // Validate the LinkTo proposal. The only type of proposal that // we currently allow linking to is an RFP. - r, err := p.cache.Record(pm.LinkTo) - if err != nil { - if err == cache.ErrRecordNotFound { - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkTo, + /* + // TODO + r, err := p.cache.Record(pm.LinkTo) + if err != nil { + if err == cache.ErrRecordNotFound { + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidLinkTo, + } } } - } - pr, err := convertPropFromCache(*r) - if err != nil { - return err - } + pr, err := convertPropFromCache(*r) + if err != nil { + return err + } + */ + var pr *www.ProposalRecord bb, err := p.getBestBlock() if err != nil { return err @@ -634,21 +637,24 @@ func (p *politeiawww) fillProposalMissingFields(pr *www.ProposalRecord) error { func (p *politeiawww) getProp(token string) (*www.ProposalRecord, error) { log.Tracef("getProp: %v", token) - var r *cache.Record - var err error - if len(token) == www.TokenPrefixLength { - r, err = p.cache.RecordByPrefix(token) - } else { - r, err = p.cache.Record(token) - } - if err != nil { - return nil, err - } + /* + var r *cache.Record + var err error + if len(token) == www.TokenPrefixLength { + r, err = p.cache.RecordByPrefix(token) + } else { + r, err = p.cache.Record(token) + } + if err != nil { + return nil, err + } - pr, err := convertPropFromCache(*r) - if err != nil { - return nil, err - } + pr, err := convertPropFromCache(*r) + if err != nil { + return nil, err + } + */ + var pr *www.ProposalRecord err = p.fillProposalMissingFields(pr) if err != nil { @@ -667,21 +673,24 @@ func (p *politeiawww) getProps(tokens []string) (map[string]www.ProposalRecord, log.Tracef("getProps: %v", tokens) // Get the proposals from the cache - records, err := p.cache.Records(tokens, true) - if err != nil { - return nil, err - } - - // Use pointers for now so the props can be easily updated - props := make(map[string]*www.ProposalRecord, len(records)) - for _, v := range records { - pr, err := convertPropFromCache(v) + /* + // TODO + records, err := p.cache.Records(tokens, true) if err != nil { - return nil, fmt.Errorf("convertPropFromCache %v: %v", - v.CensorshipRecord.Token, err) + return nil, err } - props[v.CensorshipRecord.Token] = pr - } + + // Use pointers for now so the props can be easily updated + for _, v := range records { + pr, err := convertPropFromCache(v) + if err != nil { + return nil, fmt.Errorf("convertPropFromCache %v: %v", + v.CensorshipRecord.Token, err) + } + props[v.CensorshipRecord.Token] = pr + } + */ + props := make(map[string]*www.ProposalRecord, len(records)) // Get the number of comments for each proposal. Comments // are part of decred plugin so this must be fetched from @@ -755,15 +764,18 @@ func (p *politeiawww) getProps(tokens []string) (map[string]www.ProposalRecord, func (p *politeiawww) getPropVersion(token, version string) (*www.ProposalRecord, error) { log.Tracef("getPropVersion: %v %v", token, version) - r, err := p.cache.RecordVersion(token, version) - if err != nil { - return nil, err - } + /* + r, err := p.cache.RecordVersion(token, version) + if err != nil { + return nil, err + } - pr, err := convertPropFromCache(*r) - if err != nil { - return nil, err - } + pr, err := convertPropFromCache(*r) + if err != nil { + return nil, err + } + */ + var pr *www.ProposalRecord err = p.fillProposalMissingFields(pr) if err != nil { @@ -778,54 +790,9 @@ func (p *politeiawww) getPropVersion(token, version string) (*www.ProposalRecord func (p *politeiawww) getAllProps() ([]www.ProposalRecord, error) { log.Tracef("getAllProps") - // Get proposals from cache - records, err := p.cache.Inventory() - if err != nil { - return nil, err - } + // TODO getAllProps shouldn't exist - // Convert props and fill in missing info - props := make([]www.ProposalRecord, 0, len(records)) - for _, v := range records { - pr, err := convertPropFromCache(v) - if err != nil { - return nil, fmt.Errorf("convertPropFromCache %v: %v", - pr.CensorshipRecord.Token, err) - } - token := pr.CensorshipRecord.Token - - // Fill in num comments - dc, err := p.decredGetComments(token) - if err != nil { - return nil, fmt.Errorf("decredGetComments %v: %v", - pr.CensorshipRecord.Token, err) - } - pr.NumComments = uint(len(dc)) - - // Find linked from proposals - lfr, err := p.decredLinkedFrom([]string{token}) - if err != nil { - return nil, err - } - linkedFrom, ok := lfr.LinkedFrom[token] - if ok { - pr.LinkedFrom = linkedFrom - } - - // Fill in author info - u, err := p.db.UserGetByPubKey(pr.PublicKey) - if err != nil { - return nil, fmt.Errorf("UserGetByPubKey %v %v: %v", - pr.CensorshipRecord.Token, pr.PublicKey, err) - } else { - pr.UserId = u.ID.String() - pr.Username = u.Username - } - - props = append(props, *pr) - } - - return props, nil + return nil, nil } // filterProps filters the given proposals according to the filtering @@ -1043,17 +1010,20 @@ func (p *politeiawww) voteSummariesGet(tokens []string, bestBlock uint64) (map[s for retries := 0; !done && retries <= 1; retries++ { reply, err = p.decredBatchVoteSummary(tokensToLookup, bestBlock) if err != nil { - if err == cache.ErrRecordNotFound { - // There are missing entries in the VoteResults - // cache table. Load them. - _, err := p.decredLoadVoteResults(bestBlock) - if err != nil { - return nil, err + /* + // TODO + if err == cache.ErrRecordNotFound { + // There are missing entries in the VoteResults + // cache table. Load them. + _, err := p.decredLoadVoteResults(bestBlock) + if err != nil { + return nil, err + } + + // Retry the vote summaries call + continue } - - // Retry the vote summaries call - continue - } + */ return nil, err } @@ -1125,7 +1095,8 @@ func (p *politeiawww) voteSummaryGet(token string, bestBlock uint64) (*www.VoteS } vs, ok := s[token] if !ok { - return nil, cache.ErrRecordNotFound + // return nil, cache.ErrRecordNotFound + return nil, fmt.Errorf("record not found") } return &vs, nil } @@ -1156,14 +1127,17 @@ func (p *politeiawww) processNewProposal(np www.NewProposal, user *user.User) (* // submissions. Everything else has already been validated by the // validateProposal function. if pm.LinkTo != "" { - r, err := p.cache.Record(pm.LinkTo) - if err != nil { - return nil, err - } - pr, err := convertPropFromCache(*r) - if err != nil { - return nil, err - } + /* + r, err := p.cache.Record(pm.LinkTo) + if err != nil { + return nil, err + } + pr, err := convertPropFromCache(*r) + if err != nil { + return nil, err + } + */ + var pr *www.ProposalRecord // Once the linkto deadline has expired no new submissions are // allowed. Edits to existing submissions are allowed which is // why this is checked here and not in the validateProposal @@ -1308,11 +1282,14 @@ func (p *politeiawww) processProposalDetails(propDetails www.ProposalsDetails, u prop, err = p.getPropVersion(propDetails.Token, propDetails.Version) } if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, + /* + // TODO + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + } } - } + */ return nil, err } @@ -1512,11 +1489,14 @@ func (p *politeiawww) processSetProposalStatus(sps www.SetProposalStatus, u *use // Get proposal from cache pr, err := p.getProp(sps.Token) if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, + /* + // TODO + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + } } - } + */ return nil, err } @@ -1714,11 +1694,14 @@ func (p *politeiawww) processEditProposal(ep www.EditProposal, u *user.User) (*w // Validate proposal status cachedProp, err := p.getProp(ep.Token) if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, + /* + // TODO + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + } } - } + */ return nil, err } @@ -2088,11 +2071,14 @@ func (p *politeiawww) processVoteStatus(token string) (*www.VoteStatusReply, err // Ensure proposal is vetted pr, err := p.getProp(token) if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, + /* + // TODO + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + } } - } + */ return nil, err } @@ -2242,11 +2228,14 @@ func (p *politeiawww) processVoteResults(token string) (*www.VoteResultsReply, e // Ensure proposal is vetted pr, err := p.getProp(token) if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, + /* + // TODO + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + } } - } + */ return nil, err } @@ -2554,11 +2543,14 @@ func (p *politeiawww) processAuthorizeVote(av www.AuthorizeVote, u *user.User) ( // Validate the vote authorization pr, err := p.getProp(av.Token) if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, + /* + // TODO + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + } } - } + */ return nil, err } bb, err := p.getBestBlock() @@ -2872,11 +2864,14 @@ func (p *politeiawww) processStartVoteV2(sv www2.StartVote, u *user.User) (*www2 } pr, err := p.getProp(sv.Vote.Token) if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, + /* + // TODO + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + } } - } + */ return nil, err } bb, err := p.getBestBlock() @@ -3049,12 +3044,15 @@ func (p *politeiawww) processStartVoteRunoffV2(sv www2.StartVoteRunoff, u *user. } pr, err := p.getProp(token) if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - ErrorContext: []string{token}, + /* + // TODO + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + ErrorContext: []string{token}, + } } - } + */ return nil, err } vs, err := p.voteSummaryGet(token, bb) @@ -3100,12 +3098,15 @@ func (p *politeiawww) processStartVoteRunoffV2(sv www2.StartVoteRunoff, u *user. // Validate the RFP proposal rfp, err := p.getProp(sv.Token) if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - ErrorContext: []string{sv.Token}, + /* + // TODO + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + ErrorContext: []string{sv.Token}, + } } - } + */ return nil, err } switch { @@ -3251,17 +3252,20 @@ func (p *politeiawww) tokenInventory(bestBlock uint64, isAdmin bool) (*www.Token // for non-admins. ti, err := p.decredTokenInventory(bestBlock, isAdmin) if err != nil { - if err == cache.ErrRecordNotFound { - // There are missing entries in the vote - // results cache table. Load them. - _, err := p.decredLoadVoteResults(bestBlock) - if err != nil { - return nil, err + /* + // TODO + if err == cache.ErrRecordNotFound { + // There are missing entries in the vote + // results cache table. Load them. + _, err := p.decredLoadVoteResults(bestBlock) + if err != nil { + return nil, err + } + + // Retry token inventory call + continue } - - // Retry token inventory call - continue - } + */ return nil, err } @@ -3292,11 +3296,14 @@ func (p *politeiawww) processVoteDetailsV2(token string) (*www2.VoteDetailsReply // Validate vote status dvdr, err := p.decredVoteDetails(token) if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, + /* + // TODO + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + } } - } + */ return nil, err } if dvdr.StartVoteReply.StartBlockHash == "" { diff --git a/politeiawww/testing.go b/politeiawww/testing.go index 1f41e2bcc..29575e3f9 100644 --- a/politeiawww/testing.go +++ b/politeiawww/testing.go @@ -28,7 +28,6 @@ import ( pd "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/api/v1/mime" - "github.com/decred/politeia/politeiad/cache/testcache" "github.com/decred/politeia/politeiad/testpoliteiad" www "github.com/decred/politeia/politeiawww/api/www/v1" www2 "github.com/decred/politeia/politeiawww/api/www/v2" @@ -690,7 +689,6 @@ func newTestPoliteiawww(t *testing.T) (*politeiawww, func()) { p := politeiawww{ cfg: cfg, db: db, - cache: testcache.New(), params: &chaincfg.TestNet3Params, router: mux.NewRouter(), sessions: NewSessionStore(db, sessionMaxAge, cookieKey), diff --git a/politeiawww/www.go b/politeiawww/www.go index 51456e1ba..7ea9b44ee 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -26,8 +26,6 @@ import ( "github.com/decred/politeia/mdstream" pd "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/politeiad/cache" - cachedb "github.com/decred/politeia/politeiad/cache/cockroachdb" cms "github.com/decred/politeia/politeiawww/api/cms/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" database "github.com/decred/politeia/politeiawww/cmsdatabase" @@ -389,42 +387,6 @@ func _main() error { return fmt.Errorf("getPluginInventory: %v", err) } - // Setup cache connection - net := filepath.Base(p.cfg.DataDir) - p.cache, err = cachedb.New(cachedb.UserPoliteiawww, p.cfg.DBHost, - net, p.cfg.DBRootCert, p.cfg.DBCert, p.cfg.DBKey) - if err != nil { - switch err { - case cache.ErrNoVersionRecord: - err = fmt.Errorf("cache version record not found; " + - "start politeiad to setup the cache") - case cache.ErrWrongVersion: - err = fmt.Errorf("wrong cache version found; " + - "restart politeiad to rebuild the cache") - } - return fmt.Errorf("cachedb new: %v", err) - } - - // Register plugins with cache - for _, v := range p.plugins { - cp := convertPluginToCache(v) - err = p.cache.RegisterPlugin(cp) - if err != nil { - switch err { - case cache.ErrNoVersionRecord: - err = fmt.Errorf("version record not found;" + - "start politeiad to setup the cache") - case cache.ErrWrongVersion: - err = fmt.Errorf("wrong version found; " + - "restart politeiad to rebuild the cache") - } - return fmt.Errorf("cache register plugin '%v': %v", - v.ID, err) - } - - log.Infof("Registered cache plugin: %v", v.ID) - } - // Setup email-userID map err = p.initUserEmailsCache() if err != nil { @@ -490,16 +452,6 @@ func _main() error { p.router = mux.NewRouter() p.router.Use(recoverMiddleware) - // Setup dcrdata websocket connection - ws, err := wsdcrdata.New(p.dcrdataHostWS()) - if err != nil { - // Continue even if a websocket connection was not able to be - // made. The application specific websocket setup (pi, cms, etc) - // can decide whether to attempt reconnection or to exit. - log.Errorf("wsdcrdata New: %v", err) - } - p.wsDcrdata = ws - switch p.cfg.Mode { case politeiaWWWMode: // Setup routes @@ -515,14 +467,17 @@ func _main() error { p.setupWSDcrdataPi() }() - // Setup VoteResults cache table - log.Infof("Loading vote results cache table") - err = p.initVoteResults() + case cmsWWWMode: + // Setup dcrdata websocket connection + ws, err := wsdcrdata.New(p.dcrdataHostWS()) if err != nil { - return err + // Continue even if a websocket connection was not able to be + // made. The application specific websocket setup (pi, cms, etc) + // can decide whether to attempt reconnection or to exit. + log.Errorf("wsdcrdata New: %v", err) } + p.wsDcrdata = ws - case cmsWWWMode: pluginFound := false for _, plugin := range p.plugins { if plugin.ID == "cms" { @@ -531,7 +486,7 @@ func _main() error { } } if !pluginFound { - return fmt.Errorf("must start politeiad in cmswww mode, cms plugin not found") + return fmt.Errorf("politeiad plugin 'cms' not found") } p.setCMSWWWRoutes() @@ -619,10 +574,10 @@ func _main() error { } } - // Build the cache + // Build the cmsdb err = p.cmsDB.Build(dbInvs, dbDCCs) if err != nil { - return fmt.Errorf("build cache: %v", err) + return fmt.Errorf("build cmsdb: %v", err) } } // Register cms userdb plugin diff --git a/scripts/cachesetup.sh b/scripts/cachesetup.sh deleted file mode 100755 index 313bf04ab..000000000 --- a/scripts/cachesetup.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash -# This script sets up the CockroachDB databases for the politeiad cache and -# assigns user privileges. -# This script requires that you have already created CockroachDB certificates -# using the cockroachcerts.sh script and that you have a CockroachDB instance -# listening on the default port localhost:26257. - -set -ex - -# COCKROACHDB_DIR must be the same directory that was used with the -# cockroachcerts.sh script. -COCKROACHDB_DIR=$1 -if [ "${COCKROACHDB_DIR}" == "" ]; then - COCKROACHDB_DIR="${HOME}/.cockroachdb" -fi - -# ROOT_CERTS_DIR must contain client.root.crt, client.root.key, and ca.crt. -readonly ROOT_CERTS_DIR="${COCKROACHDB_DIR}/certs/clients/root" - -if [ ! -f "${ROOT_CERTS_DIR}/client.root.crt" ]; then - >&2 echo "error: file not found ${ROOT_CERTS_DIR}/client.root.crt" - exit -elif [ ! -f "${ROOT_CERTS_DIR}/client.root.key" ]; then - >&2 echo "error: file not found ${ROOT_CERTS_DIR}/client.root.key" - exit -elif [ ! -f "${ROOT_CERTS_DIR}/ca.crt" ]; then - >&2 echo "error: file not found ${ROOT_CERTS_DIR}/ca.crt" - exit -fi - -# Database names. -readonly DB_MAINNET="records_mainnet" -readonly DB_TESTNET="records_testnet3" - -# Database usernames. -readonly USER_POLITEIAD="politeiad" -readonly USER_POLITEIAWWW="politeiawww" - -# Create the mainnet and testnet databases for the politeiad records cache. -cockroach sql \ - --certs-dir="${ROOT_CERTS_DIR}" \ - --execute "CREATE DATABASE IF NOT EXISTS ${DB_MAINNET}" - -cockroach sql \ - --certs-dir="${ROOT_CERTS_DIR}" \ - --execute "CREATE DATABASE IF NOT EXISTS ${DB_TESTNET}" - -# Create the politeiad user and assign privileges. -cockroach sql \ - --certs-dir="${ROOT_CERTS_DIR}" \ - --execute "CREATE USER IF NOT EXISTS ${USER_POLITEIAD}" - -cockroach sql \ - --certs-dir="${ROOT_CERTS_DIR}" \ - --execute "GRANT CREATE, SELECT, DROP, INSERT, DELETE, UPDATE \ - ON DATABASE ${DB_MAINNET} TO ${USER_POLITEIAD}" - -cockroach sql \ - --certs-dir="${ROOT_CERTS_DIR}" \ - --execute "GRANT CREATE, SELECT, DROP, INSERT, DELETE, UPDATE \ - ON DATABASE ${DB_TESTNET} TO ${USER_POLITEIAD}" - -# Create politeiawww user and assign privileges. -cockroach sql \ - --certs-dir="${ROOT_CERTS_DIR}" \ - --execute "CREATE USER IF NOT EXISTS ${USER_POLITEIAWWW}" - -cockroach sql \ - --certs-dir="${ROOT_CERTS_DIR}" \ - --execute "GRANT SELECT ON DATABASE ${DB_MAINNET} TO ${USER_POLITEIAWWW}" - -cockroach sql \ - --certs-dir="${ROOT_CERTS_DIR}" \ - --execute "GRANT SELECT ON DATABASE ${DB_TESTNET} TO ${USER_POLITEIAWWW}" - -# Disable CockroachDB diagnostic reporting -cockroach sql \ - --certs-dir="${ROOT_CERTS_DIR}" \ - --execute "SET CLUSTER SETTING diagnostics.reporting.enabled=false" - -cockroach sql \ - --certs-dir="${ROOT_CERTS_DIR}" \ - --execute "SET CLUSTER SETTING diagnostics.reporting.send_crash_reports=false" diff --git a/util/signature.go b/util/signature.go index 0357b2478..70002d0b4 100644 --- a/util/signature.go +++ b/util/signature.go @@ -11,6 +11,8 @@ import ( "github.com/decred/politeia/politeiad/api/v1/identity" ) +// TODO this doesn't need to be in util since all signature validation is +// done in the plugins type ErrorStatusT int const ( From a7890624aadd3e451b3fcb2e279bcbf1a2e8a02c Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 27 Aug 2020 19:09:48 -0500 Subject: [PATCH 019/449] dcc changes --- politeiawww/dcc.go | 187 +++++++++++++++++++++++++++------------------ 1 file changed, 114 insertions(+), 73 deletions(-) diff --git a/politeiawww/dcc.go b/politeiawww/dcc.go index b9b22451b..c71aaa5f9 100644 --- a/politeiawww/dcc.go +++ b/politeiawww/dcc.go @@ -21,7 +21,6 @@ import ( "github.com/decred/politeia/mdstream" pd "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/identity" - "github.com/decred/politeia/politeiad/cache" cms "github.com/decred/politeia/politeiawww/api/cms/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/cmsdatabase" @@ -412,11 +411,15 @@ func validateSponsorStatement(statement string) bool { // then fills in any missing user fields before returning the DCC record. func (p *politeiawww) getDCC(token string) (*cms.DCCRecord, error) { // Get dcc from cache - r, err := p.cache.Record(token) - if err != nil { - return nil, err - } - i := convertDCCFromCache(*r) + /* + r, err := p.cache.Record(token) + if err != nil { + return nil, err + } + i := convertDCCFromCache(*r) + */ + // TODO + var i cms.DCCRecord // Check for possible malformed DCC if i.PublicKey == "" { @@ -494,11 +497,14 @@ func (p *politeiawww) processDCCDetails(gd cms.DCCDetails) (*cms.DCCDetailsReply dcc, err := p.getDCC(gd.Token) if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: cms.ErrorStatusDCCNotFound, + /* + // TODO + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: cms.ErrorStatusDCCNotFound, + } } - } + */ return nil, err } @@ -573,11 +579,14 @@ func (p *politeiawww) processSupportOpposeDCC(sd cms.SupportOpposeDCC, u *user.U dcc, err := p.getDCC(sd.Token) if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: cms.ErrorStatusDCCNotFound, + /* + // TODO + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: cms.ErrorStatusDCCNotFound, + } } - } + */ return nil, err } // Check to make sure the user has not SupportOpposeed or Opposed this DCC yet @@ -720,11 +729,14 @@ func (p *politeiawww) processNewCommentDCC(nc www.NewComment, u *user.User) (*ww dcc, err := p.getDCC(nc.Token) if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: cms.ErrorStatusDCCNotFound, + /* + // TODO + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: cms.ErrorStatusDCCNotFound, + } } - } + */ return nil, err } @@ -871,11 +883,14 @@ func (p *politeiawww) processSetDCCStatus(sds cms.SetDCCStatus, u *user.User) (* dcc, err := p.getDCC(sds.Token) if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: cms.ErrorStatusDCCNotFound, + /* + // TODO + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: cms.ErrorStatusDCCNotFound, + } } - } + */ return nil, err } @@ -1118,11 +1133,14 @@ func (p *politeiawww) processVoteDetailsDCC(token string) (*cms.VoteDetailsReply // Validate vote status dvdr, err := p.cmsVoteDetails(token) if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: cms.ErrorStatusDCCNotFound, + /* + // TODO + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: cms.ErrorStatusDCCNotFound, + } } - } + */ return nil, err } if dvdr.StartVoteReply.StartBlockHash == "" { @@ -1153,18 +1171,24 @@ func (p *politeiawww) cmsVoteDetails(token string) (*cmsplugin.VoteDetailsReply, return nil, err } - pc := cache.PluginCommand{ - ID: cmsplugin.ID, - Command: cmsplugin.CmdVoteDetails, - CommandPayload: string(payload), - } + /* + pc := cache.PluginCommand{ + ID: cmsplugin.ID, + Command: cmsplugin.CmdVoteDetails, + CommandPayload: string(payload), + } - // Get vote details from cache - reply, err := p.cache.PluginExec(pc) - if err != nil { - return nil, err - } - vdr, err := cmsplugin.DecodeVoteDetailsReply([]byte(reply.Payload)) + // Get vote details from cache + reply, err := p.cache.PluginExec(pc) + if err != nil { + return nil, err + } + */ + + // TODO + var reply string + + vdr, err := cmsplugin.DecodeVoteDetailsReply([]byte(reply)) if err != nil { return nil, err } @@ -1185,18 +1209,24 @@ func (p *politeiawww) cmsVoteSummary(token string) (*cmsplugin.VoteSummaryReply, return nil, err } - pc := cache.PluginCommand{ - ID: cmsplugin.ID, - Command: cmsplugin.CmdVoteSummary, - CommandPayload: string(payload), - } + /* + pc := cache.PluginCommand{ + ID: cmsplugin.ID, + Command: cmsplugin.CmdVoteSummary, + CommandPayload: string(payload), + } - // Get vote details from cache - reply, err := p.cache.PluginExec(pc) - if err != nil { - return nil, err - } - vdr, err := cmsplugin.DecodeVoteSummaryReply([]byte(reply.Payload)) + // Get vote details from cache + reply, err := p.cache.PluginExec(pc) + if err != nil { + return nil, err + } + */ + + // TODO + var reply string + + vdr, err := cmsplugin.DecodeVoteSummaryReply([]byte(reply)) if err != nil { return nil, err } @@ -1207,20 +1237,24 @@ func (p *politeiawww) cmsVoteSummary(token string) (*cmsplugin.VoteSummaryReply, func (p *politeiawww) processActiveVoteDCC() (*cms.ActiveVoteReply, error) { log.Tracef("processActiveVoteDCC") - vetted, err := p.cache.Inventory() - if err != nil { - return nil, fmt.Errorf("backend inventory: %v", err) - } + /* + vetted, err := p.cache.Inventory() + if err != nil { + return nil, fmt.Errorf("backend inventory: %v", err) + } - active := make([]string, 0, len(vetted)) - for _, r := range vetted { - for _, m := range r.Metadata { - switch m.ID { - case mdstream.IDDCCGeneral: - // Get vote summary and thereby status to check here. + for _, r := range vetted { + for _, m := range r.Metadata { + switch m.ID { + case mdstream.IDDCCGeneral: + // Get vote summary and thereby status to check here. + } } } - } + */ + + // TODO + active := make([]string, 0, len(vetted)) dccs, err := p.getDCCs(active) if err != nil { @@ -1257,18 +1291,22 @@ func (p *politeiawww) processActiveVoteDCC() (*cms.ActiveVoteReply, error) { func (p *politeiawww) getDCCs(tokens []string) (map[string]cms.DCCRecord, error) { log.Tracef("getDCCs: %v", tokens) - // Get the dccs from the cache - records, err := p.cache.Records(tokens, false) - if err != nil { - return nil, err - } + /* + // Get the dccs from the cache + records, err := p.cache.Records(tokens, false) + if err != nil { + return nil, err + } + + // Use pointers for now so the props can be easily updated + for _, v := range records { + dr := convertDCCFromCache(v) + dccs[v.CensorshipRecord.Token] = &dr + } + */ - // Use pointers for now so the props can be easily updated + // TODO dccs := make(map[string]*cms.DCCRecord, len(records)) - for _, v := range records { - dr := convertDCCFromCache(v) - dccs[v.CensorshipRecord.Token] = &dr - } // Compile a list of unique proposal author pubkeys. These // are needed to lookup the proposal author info. @@ -1400,11 +1438,14 @@ func (p *politeiawww) processStartVoteDCC(sv cms.StartVote, u *user.User) (*cms. // Validate proposal version and status pr, err := p.getDCC(sv.Vote.Token) if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, + /* + // TODO + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + } } - } + */ return nil, err } if pr.Status != cms.DCCStatusActive { From d18f4dd1979ecef8ac70abe15ae0ba58c3591908 Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 28 Aug 2020 09:27:02 -0500 Subject: [PATCH 020/449] get politeiawww to build --- politeiawww/cmswww.go | 3 - politeiawww/comments.go | 847 +++++++++++++++++---------------- politeiawww/dcc.go | 6 +- politeiawww/decred.go | 3 + politeiawww/email.go | 12 +- politeiawww/events.go | 40 +- politeiawww/plugin.go | 6 +- politeiawww/politeiawww.go | 115 ----- politeiawww/proposals.go | 949 ++++++++++++++++++------------------- politeiawww/testing.go | 3 +- politeiawww/user.go | 140 +++--- politeiawww/www.go | 10 - 12 files changed, 992 insertions(+), 1142 deletions(-) diff --git a/politeiawww/cmswww.go b/politeiawww/cmswww.go index a598774b5..62d4506d9 100644 --- a/politeiawww/cmswww.go +++ b/politeiawww/cmswww.go @@ -1204,9 +1204,6 @@ func (p *politeiawww) setCMSWWWRoutes() { p.addRoute(http.MethodPost, cms.APIRoute, cms.RouteInviteNewUser, p.handleInviteNewUser, permissionAdmin) - p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, - www.RouteCensorComment, p.handleCensorComment, - permissionAdmin) p.addRoute(http.MethodPost, cms.APIRoute, cms.RouteSetInvoiceStatus, p.handleSetInvoiceStatus, permissionAdmin) diff --git a/politeiawww/comments.go b/politeiawww/comments.go index 9ebe44d7c..2abbc0b76 100644 --- a/politeiawww/comments.go +++ b/politeiawww/comments.go @@ -5,17 +5,10 @@ package main import ( - "encoding/hex" - "encoding/json" "fmt" - "net/http" - "sort" - "github.com/decred/politeia/decredplugin" - pd "github.com/decred/politeia/politeiad/api/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/user" - "github.com/decred/politeia/util" ) // counters is a struct that helps us keep track of up/down votes. @@ -88,90 +81,93 @@ func (p *politeiawww) getCommentVotes(token, commentID string) (counters, error) func (p *politeiawww) updateCommentVotes(token, commentID string) (*counters, error) { log.Tracef("updateCommentVotes: %v %v", token, commentID) - // Fetch all comment likes for the specified comment - likes, err := p.decredCommentLikes(token, commentID) - if err != nil { - return nil, fmt.Errorf("decredLikeComments: %v", err) - } + /* + // Fetch all comment likes for the specified comment + likes, err := p.decredCommentLikes(token, commentID) + if err != nil { + return nil, fmt.Errorf("decredLikeComments: %v", err) + } - // Sanity check. Like comments should already be sorted in - // chronological order. - sort.SliceStable(likes, func(i, j int) bool { - return likes[i].Timestamp < likes[j].Timestamp - }) - - p.Lock() - defer p.Unlock() - - // Compute the comment votes. We have to keep track of each user's most - // recent like action because the net effect of an upvote/downvote is - // dependent on the user's previous action. - // Example: a user upvoting a comment twice results in a net score of 0 - // because the second upvote is actually the user taking away their original - // upvote. - var votes counters - userActions := make(map[string]string) // [userID]action - for _, v := range likes { - // Lookup the userID of the comment author - u, err := p.db.UserGetByPubKey(v.PublicKey) - if err != nil { - return nil, fmt.Errorf("user lookup failed for pubkey %v", - v.PublicKey) - } - userID := u.ID.String() - - // Lookup the previous like comment action that the author - // made on this comment - prevAction := userActions[userID] - - switch { - case prevAction == "": - // No previous action so we add the new action to the - // vote score - switch v.Action { - case www.VoteActionDown: - votes.down += 1 - case www.VoteActionUp: - votes.up += 1 - } - userActions[userID] = v.Action - - case prevAction == v.Action: - // New action is the same as the previous action so we - // remove the previous action from the vote score - switch prevAction { - case www.VoteActionDown: - votes.down -= 1 - case www.VoteActionUp: - votes.up -= 1 - } - delete(userActions, userID) - - case prevAction != v.Action: - // New action is different than the previous action so - // we remove the previous action from the vote score.. - switch prevAction { - case www.VoteActionDown: - votes.down -= 1 - case www.VoteActionUp: - votes.up -= 1 + // Sanity check. Like comments should already be sorted in + // chronological order. + sort.SliceStable(likes, func(i, j int) bool { + return likes[i].Timestamp < likes[j].Timestamp + }) + + p.Lock() + defer p.Unlock() + + // Compute the comment votes. We have to keep track of each user's most + // recent like action because the net effect of an upvote/downvote is + // dependent on the user's previous action. + // Example: a user upvoting a comment twice results in a net score of 0 + // because the second upvote is actually the user taking away their original + // upvote. + var votes counters + userActions := make(map[string]string) // [userID]action + for _, v := range likes { + // Lookup the userID of the comment author + u, err := p.db.UserGetByPubKey(v.PublicKey) + if err != nil { + return nil, fmt.Errorf("user lookup failed for pubkey %v", + v.PublicKey) } + userID := u.ID.String() + + // Lookup the previous like comment action that the author + // made on this comment + prevAction := userActions[userID] + + switch { + case prevAction == "": + // No previous action so we add the new action to the + // vote score + switch v.Action { + case www.VoteActionDown: + votes.down += 1 + case www.VoteActionUp: + votes.up += 1 + } + userActions[userID] = v.Action + + case prevAction == v.Action: + // New action is the same as the previous action so we + // remove the previous action from the vote score + switch prevAction { + case www.VoteActionDown: + votes.down -= 1 + case www.VoteActionUp: + votes.up -= 1 + } + delete(userActions, userID) + + case prevAction != v.Action: + // New action is different than the previous action so + // we remove the previous action from the vote score.. + switch prevAction { + case www.VoteActionDown: + votes.down -= 1 + case www.VoteActionUp: + votes.up -= 1 + } - // ..and then add the new action to the vote score - switch v.Action { - case www.VoteActionDown: - votes.down += 1 - case www.VoteActionUp: - votes.up += 1 + // ..and then add the new action to the vote score + switch v.Action { + case www.VoteActionDown: + votes.down += 1 + case www.VoteActionUp: + votes.up += 1 + } + userActions[userID] = v.Action } - userActions[userID] = v.Action } - } - // Update in-memory cache - p.commentVotes[token+commentID] = votes + // Update in-memory cache + p.commentVotes[token+commentID] = votes - return &votes, nil + return &votes, nil + */ + return nil, nil } func validateComment(c www.NewComment) error { @@ -195,427 +191,428 @@ func validateComment(c www.NewComment) error { func (p *politeiawww) processNewComment(nc www.NewComment, u *user.User) (*www.NewCommentReply, error) { log.Tracef("processNewComment: %v %v", nc.Token, u.ID) - // Make sure token is valid and not a prefix - if !isTokenValid(nc.Token) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: []string{nc.Token}, + /* + // Make sure token is valid and not a prefix + if !isTokenValid(nc.Token) { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidCensorshipToken, + ErrorContext: []string{nc.Token}, + } } - } - // Pay up sucker! - if !p.HasUserPaid(u) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusUserNotPaid, + // Pay up sucker! + if !p.HasUserPaid(u) { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusUserNotPaid, + } } - } - // Ensure the public key is the user's active key - if nc.PublicKey != u.PublicKey() { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidSigningKey, + // Ensure the public key is the user's active key + if nc.PublicKey != u.PublicKey() { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidSigningKey, + } } - } - // Validate signature - msg := nc.Token + nc.ParentID + nc.Comment - err := validateSignature(nc.PublicKey, nc.Signature, msg) - if err != nil { - return nil, err - } + // Validate signature + msg := nc.Token + nc.ParentID + nc.Comment + err := validateSignature(nc.PublicKey, nc.Signature, msg) + if err != nil { + return nil, err + } - // Validate comment - err = validateComment(nc) - if err != nil { - return nil, err - } + // Validate comment + err = validateComment(nc) + if err != nil { + return nil, err + } - // Ensure proposal exists and is public - pr, err := p.getProp(nc.Token) - if err != nil { - /* - // TODO - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, + // Ensure proposal exists and is public + pr, err := p.getProp(nc.Token) + if err != nil { + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + } } + return nil, err + } + + if pr.Status != www.PropStatusPublic { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusWrongStatus, + ErrorContext: []string{"proposal is not public"}, } - */ - return nil, err - } + } - if pr.Status != www.PropStatusPublic { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusWrongStatus, - ErrorContext: []string{"proposal is not public"}, + // Ensure proposal voting has not ended + bb, err := p.getBestBlock() + if err != nil { + return nil, fmt.Errorf("getBestBlock: %v", err) + } + vs, err := p.voteSummaryGet(nc.Token, bb) + if err != nil { + return nil, fmt.Errorf("voteSummaryGet: %v", err) + } + if vs.Status == www.PropVoteStatusFinished { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusWrongVoteStatus, + ErrorContext: []string{"vote is finished"}, + } } - } - // Ensure proposal voting has not ended - bb, err := p.getBestBlock() - if err != nil { - return nil, fmt.Errorf("getBestBlock: %v", err) - } - vs, err := p.voteSummaryGet(nc.Token, bb) - if err != nil { - return nil, fmt.Errorf("voteSummaryGet: %v", err) - } - if vs.Status == www.PropVoteStatusFinished { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - ErrorContext: []string{"vote is finished"}, + // Ensure the comment is not a duplicate + _, err = p.decredCommentGetBySignature(nc.Token, nc.Signature) + switch err { + case nil: + // Duplicate comment was found + return nil, www.UserError{ + ErrorCode: www.ErrorStatusDuplicateComment, + } + // TODO case cache.ErrRecordNotFound: + // No duplicate comment; continue + default: + // Some other error + return nil, fmt.Errorf("decredCommentBySignature: %v", err) } - } - // Ensure the comment is not a duplicate - _, err = p.decredCommentGetBySignature(nc.Token, nc.Signature) - switch err { - case nil: - // Duplicate comment was found - return nil, www.UserError{ - ErrorCode: www.ErrorStatusDuplicateComment, - } - // TODO case cache.ErrRecordNotFound: - // No duplicate comment; continue - default: - // Some other error - return nil, fmt.Errorf("decredCommentBySignature: %v", err) - } + // Setup plugin command + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return nil, err + } - // Setup plugin command - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } + dnc := convertNewCommentToDecredPlugin(nc) + payload, err := decredplugin.EncodeNewComment(dnc) + if err != nil { + return nil, err + } - dnc := convertNewCommentToDecredPlugin(nc) - payload, err := decredplugin.EncodeNewComment(dnc) - if err != nil { - return nil, err - } + pc := pd.PluginCommand{ + Challenge: hex.EncodeToString(challenge), + ID: decredplugin.ID, + Command: decredplugin.CmdNewComment, + CommandID: decredplugin.CmdNewComment, + Payload: string(payload), + } - pc := pd.PluginCommand{ - Challenge: hex.EncodeToString(challenge), - ID: decredplugin.ID, - Command: decredplugin.CmdNewComment, - CommandID: decredplugin.CmdNewComment, - Payload: string(payload), - } + // Send polieiad request + responseBody, err := p.makeRequest(http.MethodPost, + pd.PluginCommandRoute, pc) + if err != nil { + return nil, err + } - // Send polieiad request - responseBody, err := p.makeRequest(http.MethodPost, - pd.PluginCommandRoute, pc) - if err != nil { - return nil, err - } + // Handle response + var reply pd.PluginCommandReply + err = json.Unmarshal(responseBody, &reply) + if err != nil { + return nil, fmt.Errorf("could not unmarshal "+ + "PluginCommandReply: %v", err) + } - // Handle response - var reply pd.PluginCommandReply - err = json.Unmarshal(responseBody, &reply) - if err != nil { - return nil, fmt.Errorf("could not unmarshal "+ - "PluginCommandReply: %v", err) - } + err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) + if err != nil { + return nil, err + } - err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) - if err != nil { - return nil, err - } + ncr, err := decredplugin.DecodeNewCommentReply([]byte(reply.Payload)) + if err != nil { + return nil, err + } - ncr, err := decredplugin.DecodeNewCommentReply([]byte(reply.Payload)) - if err != nil { - return nil, err - } + // Add comment to commentVotes in-memory cache + p.Lock() + p.commentVotes[nc.Token+ncr.CommentID] = counters{} + p.Unlock() - // Add comment to commentVotes in-memory cache - p.Lock() - p.commentVotes[nc.Token+ncr.CommentID] = counters{} - p.Unlock() + // Get comment from cache + c, err := p.getComment(nc.Token, ncr.CommentID) + if err != nil { + return nil, fmt.Errorf("getComment: %v", err) + } - // Get comment from cache - c, err := p.getComment(nc.Token, ncr.CommentID) - if err != nil { - return nil, fmt.Errorf("getComment: %v", err) - } + // Fire off new comment event + p.fireEvent(EventTypeComment, EventDataComment{ + Comment: c, + }) - // Fire off new comment event - p.fireEvent(EventTypeComment, EventDataComment{ - Comment: c, - }) + return &www.NewCommentReply{ + Comment: *c, + }, nil + */ - return &www.NewCommentReply{ - Comment: *c, - }, nil + return nil, nil } // processLikeComment processes an upvote/downvote on a comment. func (p *politeiawww) processLikeComment(lc www.LikeComment, u *user.User) (*www.LikeCommentReply, error) { log.Debugf("processLikeComment: %v %v %v", lc.Token, lc.CommentID, u.ID) - // Make sure token is valid and not a prefix - if !isTokenValid(lc.Token) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: []string{lc.Token}, + /* + // Make sure token is valid and not a prefix + if !isTokenValid(lc.Token) { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidCensorshipToken, + ErrorContext: []string{lc.Token}, + } } - } - // Pay up sucker! - if !p.HasUserPaid(u) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusUserNotPaid, + // Pay up sucker! + if !p.HasUserPaid(u) { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusUserNotPaid, + } } - } - // Ensure the public key is the user's active key - if lc.PublicKey != u.PublicKey() { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidSigningKey, + // Ensure the public key is the user's active key + if lc.PublicKey != u.PublicKey() { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidSigningKey, + } } - } - // Validate signature - msg := lc.Token + lc.CommentID + lc.Action - err := validateSignature(lc.PublicKey, lc.Signature, msg) - if err != nil { - return nil, err - } + // Validate signature + msg := lc.Token + lc.CommentID + lc.Action + err := validateSignature(lc.PublicKey, lc.Signature, msg) + if err != nil { + return nil, err + } - // Ensure proposal exists and is public - pr, err := p.getProp(lc.Token) - if err != nil { - /* - // TODO - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, + // Ensure proposal exists and is public + pr, err := p.getProp(lc.Token) + if err != nil { + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + } } - } - */ - return nil, err - } + return nil, err + } - if pr.Status != www.PropStatusPublic { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusWrongStatus, + if pr.Status != www.PropStatusPublic { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusWrongStatus, + } } - } - // Ensure proposal voting has not ended - bb, err := p.getBestBlock() - if err != nil { - return nil, fmt.Errorf("getBestBlock: %v", err) - } - vs, err := p.voteSummaryGet(pr.CensorshipRecord.Token, bb) - if err != nil { - return nil, err - } - if vs.Status == www.PropVoteStatusFinished { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - ErrorContext: []string{"vote has ended"}, + // Ensure proposal voting has not ended + bb, err := p.getBestBlock() + if err != nil { + return nil, fmt.Errorf("getBestBlock: %v", err) + } + vs, err := p.voteSummaryGet(pr.CensorshipRecord.Token, bb) + if err != nil { + return nil, err + } + if vs.Status == www.PropVoteStatusFinished { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusWrongVoteStatus, + ErrorContext: []string{"vote has ended"}, + } } - } - // Ensure comment exists and has not been censored. - c, err := p.decredCommentGetByID(lc.Token, lc.CommentID) - if err != nil { - /* - // TODO - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusCommentNotFound, + // Ensure comment exists and has not been censored. + c, err := p.decredCommentGetByID(lc.Token, lc.CommentID) + if err != nil { + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: www.ErrorStatusCommentNotFound, + } } + return nil, err + } + if c.Censored { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusCommentIsCensored, } - */ - return nil, err - } - if c.Censored { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusCommentIsCensored, } - } - // Validate action - action := lc.Action - if len(lc.Action) > 10 { - // Clip action to not fill up logs and prevent DOS of sorts - action = lc.Action[0:9] + "..." - } - if action != www.VoteActionUp && action != www.VoteActionDown { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidLikeCommentAction, + // Validate action + action := lc.Action + if len(lc.Action) > 10 { + // Clip action to not fill up logs and prevent DOS of sorts + action = lc.Action[0:9] + "..." + } + if action != www.VoteActionUp && action != www.VoteActionDown { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidLikeCommentAction, + } } - } - // Setup plugin command - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } + // Setup plugin command + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return nil, err + } - dlc := convertLikeCommentToDecred(lc) - payload, err := decredplugin.EncodeLikeComment(dlc) - if err != nil { - return nil, err - } + dlc := convertLikeCommentToDecred(lc) + payload, err := decredplugin.EncodeLikeComment(dlc) + if err != nil { + return nil, err + } - pc := pd.PluginCommand{ - Challenge: hex.EncodeToString(challenge), - ID: decredplugin.ID, - Command: decredplugin.CmdLikeComment, - CommandID: decredplugin.CmdLikeComment, - Payload: string(payload), - } + pc := pd.PluginCommand{ + Challenge: hex.EncodeToString(challenge), + ID: decredplugin.ID, + Command: decredplugin.CmdLikeComment, + CommandID: decredplugin.CmdLikeComment, + Payload: string(payload), + } - // Send plugin command to politeiad - responseBody, err := p.makeRequest(http.MethodPost, - pd.PluginCommandRoute, pc) - if err != nil { - return nil, err - } + // Send plugin command to politeiad + responseBody, err := p.makeRequest(http.MethodPost, + pd.PluginCommandRoute, pc) + if err != nil { + return nil, err + } - // Handle response - var reply pd.PluginCommandReply - err = json.Unmarshal(responseBody, &reply) - if err != nil { - return nil, fmt.Errorf("could not unmarshal "+ - "PluginCommandReply: %v", err) - } + // Handle response + var reply pd.PluginCommandReply + err = json.Unmarshal(responseBody, &reply) + if err != nil { + return nil, fmt.Errorf("could not unmarshal "+ + "PluginCommandReply: %v", err) + } - err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) - if err != nil { - return nil, err - } + err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) + if err != nil { + return nil, err + } - lcr, err := decredplugin.DecodeLikeCommentReply([]byte(reply.Payload)) - if err != nil { - return nil, err - } + lcr, err := decredplugin.DecodeLikeCommentReply([]byte(reply.Payload)) + if err != nil { + return nil, err + } - // Update comment score in the in-memory cache - votes, err := p.updateCommentVotes(lc.Token, lc.CommentID) - if err != nil { - log.Criticalf("processLikeComment: update comment score "+ - "failed token:%v commentID:%v error:%v", lc.Token, - lc.CommentID, err) - } + // Update comment score in the in-memory cache + votes, err := p.updateCommentVotes(lc.Token, lc.CommentID) + if err != nil { + log.Criticalf("processLikeComment: update comment score "+ + "failed token:%v commentID:%v error:%v", lc.Token, + lc.CommentID, err) + } + + return &www.LikeCommentReply{ + Result: int64(votes.up - votes.down), + Upvotes: votes.up, + Downvotes: votes.down, + Receipt: lcr.Receipt, + Error: lcr.Error, + }, nil + */ - return &www.LikeCommentReply{ - Result: int64(votes.up - votes.down), - Upvotes: votes.up, - Downvotes: votes.down, - Receipt: lcr.Receipt, - Error: lcr.Error, - }, nil + return nil, nil } -// processCensorComment sends a censor comment decred plugin command to -// politeiad then returns the censor comment receipt. func (p *politeiawww) processCensorComment(cc www.CensorComment, u *user.User) (*www.CensorCommentReply, error) { log.Tracef("processCensorComment: %v: %v", cc.Token, cc.CommentID) - // Make sure token is valid and not a prefix - if !isTokenValid(cc.Token) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: []string{cc.Token}, + /* + // Make sure token is valid and not a prefix + if !isTokenValid(cc.Token) { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidCensorshipToken, + ErrorContext: []string{cc.Token}, + } } - } - // Ensure the public key is the user's active key - if cc.PublicKey != u.PublicKey() { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidSigningKey, + // Ensure the public key is the user's active key + if cc.PublicKey != u.PublicKey() { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidSigningKey, + } } - } - // Validate signature - msg := cc.Token + cc.CommentID + cc.Reason - err := validateSignature(cc.PublicKey, cc.Signature, msg) - if err != nil { - return nil, err - } + // Validate signature + msg := cc.Token + cc.CommentID + cc.Reason + err := validateSignature(cc.PublicKey, cc.Signature, msg) + if err != nil { + return nil, err + } - // Ensure censor reason is present - if cc.Reason == "" { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusCensorReasonCannotBeBlank, + // Ensure censor reason is present + if cc.Reason == "" { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusCensorReasonCannotBeBlank, + } } - } - // Ensure comment exists and has not already been censored - c, err := p.decredCommentGetByID(cc.Token, cc.CommentID) - if err != nil { - return nil, fmt.Errorf("decredGetComment: %v", err) - } - if c.Censored { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusCommentNotFound, + // Ensure comment exists and has not already been censored + c, err := p.decredCommentGetByID(cc.Token, cc.CommentID) + if err != nil { + return nil, fmt.Errorf("decredGetComment: %v", err) + } + if c.Censored { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusCommentNotFound, + } } - } - // Ensure proposal voting has not ended - bb, err := p.getBestBlock() - if err != nil { - return nil, fmt.Errorf("getBestBlock: %v", err) - } - vs, err := p.voteSummaryGet(cc.Token, bb) - if err != nil { - return nil, err - } - if vs.Status == www.PropVoteStatusFinished { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - ErrorContext: []string{"vote has ended"}, + // Ensure proposal voting has not ended + bb, err := p.getBestBlock() + if err != nil { + return nil, fmt.Errorf("getBestBlock: %v", err) + } + vs, err := p.voteSummaryGet(cc.Token, bb) + if err != nil { + return nil, err + } + if vs.Status == www.PropVoteStatusFinished { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusWrongVoteStatus, + ErrorContext: []string{"vote has ended"}, + } } - } - // Setup plugin command - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } + // Setup plugin command + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return nil, err + } - dcc := convertCensorCommentToDecred(cc) - payload, err := decredplugin.EncodeCensorComment(dcc) - if err != nil { - return nil, err - } + dcc := convertCensorCommentToDecred(cc) + payload, err := decredplugin.EncodeCensorComment(dcc) + if err != nil { + return nil, err + } - pc := pd.PluginCommand{ - Challenge: hex.EncodeToString(challenge), - ID: decredplugin.ID, - Command: decredplugin.CmdCensorComment, - CommandID: decredplugin.CmdCensorComment, - Payload: string(payload), - } + pc := pd.PluginCommand{ + Challenge: hex.EncodeToString(challenge), + ID: decredplugin.ID, + Command: decredplugin.CmdCensorComment, + CommandID: decredplugin.CmdCensorComment, + Payload: string(payload), + } - // Send plugin request - responseBody, err := p.makeRequest(http.MethodPost, - pd.PluginCommandRoute, pc) - if err != nil { - return nil, err - } + // Send plugin request + responseBody, err := p.makeRequest(http.MethodPost, + pd.PluginCommandRoute, pc) + if err != nil { + return nil, err + } - // Handle response - var reply pd.PluginCommandReply - err = json.Unmarshal(responseBody, &reply) - if err != nil { - return nil, err - } + // Handle response + var reply pd.PluginCommandReply + err = json.Unmarshal(responseBody, &reply) + if err != nil { + return nil, err + } - err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) - if err != nil { - return nil, err - } + err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) + if err != nil { + return nil, err + } - ccr, err := decredplugin.DecodeCensorCommentReply([]byte(reply.Payload)) - if err != nil { - return nil, err - } + ccr, err := decredplugin.DecodeCensorCommentReply([]byte(reply.Payload)) + if err != nil { + return nil, err + } + + return &www.CensorCommentReply{ + Receipt: ccr.Receipt, + }, nil + */ - return &www.CensorCommentReply{ - Receipt: ccr.Receipt, - }, nil + return nil, nil } diff --git a/politeiawww/dcc.go b/politeiawww/dcc.go index c71aaa5f9..44ab4e54b 100644 --- a/politeiawww/dcc.go +++ b/politeiawww/dcc.go @@ -1186,6 +1186,7 @@ func (p *politeiawww) cmsVoteDetails(token string) (*cmsplugin.VoteDetailsReply, */ // TODO + _ = payload var reply string vdr, err := cmsplugin.DecodeVoteDetailsReply([]byte(reply)) @@ -1224,6 +1225,7 @@ func (p *politeiawww) cmsVoteSummary(token string) (*cmsplugin.VoteSummaryReply, */ // TODO + _ = payload var reply string vdr, err := cmsplugin.DecodeVoteSummaryReply([]byte(reply)) @@ -1254,7 +1256,7 @@ func (p *politeiawww) processActiveVoteDCC() (*cms.ActiveVoteReply, error) { */ // TODO - active := make([]string, 0, len(vetted)) + active := make([]string, 0, 64) dccs, err := p.getDCCs(active) if err != nil { @@ -1306,7 +1308,7 @@ func (p *politeiawww) getDCCs(tokens []string) (map[string]cms.DCCRecord, error) */ // TODO - dccs := make(map[string]*cms.DCCRecord, len(records)) + dccs := make(map[string]*cms.DCCRecord, len(tokens)) // Compile a list of unique proposal author pubkeys. These // are needed to lookup the proposal author info. diff --git a/politeiawww/decred.go b/politeiawww/decred.go index 51f94194c..2b83ea635 100644 --- a/politeiawww/decred.go +++ b/politeiawww/decred.go @@ -18,6 +18,7 @@ func (p *politeiawww) decredGetComment(gc decredplugin.GetComment) (*decredplugi } // TODO this needs to use the politeiad plugin command + _ = payload var reply string gcr, err := decredplugin.DecodeGetCommentReply([]byte(reply)) @@ -61,6 +62,7 @@ func (p *politeiawww) decredGetComments(token string) ([]decredplugin.Comment, e } // TODO this needs to use the politeiad plugin command + _ = payload var reply string gcr, err := decredplugin.DecodeGetCommentsReply([]byte(reply)) @@ -88,6 +90,7 @@ func (p *politeiawww) decredGetNumComments(tokens []string) (map[string]int, err } // TODO this needs to use the politeiad plugin command + _ = payload var reply string gncr, err := decredplugin.DecodeGetNumCommentsReply([]byte(reply)) diff --git a/politeiawww/email.go b/politeiawww/email.go index 96c547b20..bfc6e0483 100644 --- a/politeiawww/email.go +++ b/politeiawww/email.go @@ -384,22 +384,22 @@ func (p *politeiawww) emailAdminsForNewSubmittedProposal(token string, propName }) } -func (p *politeiawww) emailAdminsForProposalVoteAuthorized(proposal *www.ProposalRecord, authorUser *user.User) error { +func (p *politeiawww) emailAdminsForProposalVoteAuthorized(token, title, authorUsername, authorEmail string) error { if p.smtp.disabled { return nil } - l, err := url.Parse(fmt.Sprintf("%v/proposals/%v", p.cfg.WebServerAddress, - proposal.CensorshipRecord.Token)) + l, err := url.Parse(fmt.Sprintf("%v/proposals/%v", + p.cfg.WebServerAddress, token)) if err != nil { return err } tplData := proposalVoteAuthorizedTemplateData{ Link: l.String(), - Name: proposal.Name, - Username: authorUser.Username, - Email: authorUser.Email, + Name: title, + Username: authorUsername, + Email: authorEmail, } subject := "Proposal Authorized To Start Voting" diff --git a/politeiawww/events.go b/politeiawww/events.go index 956a9651d..c77d15c11 100644 --- a/politeiawww/events.go +++ b/politeiawww/events.go @@ -14,6 +14,8 @@ import ( "github.com/decred/politeia/politeiawww/user" ) +// TODO clean this up + // EventT is the type of event. type EventT int @@ -24,19 +26,30 @@ type EventManager struct { const ( EventTypeInvalid EventT = iota + EventTypeComment + EventTypeUserManage + + // Pi events EventTypeProposalSubmitted EventTypeProposalStatusChange EventTypeProposalEdited EventTypeProposalVoteStarted EventTypeProposalVoteAuthorized - EventTypeComment - EventTypeUserManage + + // CMS events EventTypeInvoiceComment // CMS Type EventTypeInvoiceStatusUpdate // CMS Type EventTypeDCCNew // DCC Type EventTypeDCCSupportOppose // DCC Type ) +type eventDataProposalVoteAuthorized struct { + token string // Proposal token + name string // Proposal name + authorUsername string + authorEmail string +} + type EventDataProposalSubmitted struct { CensorshipRecord *www.CensorshipRecord ProposalName string @@ -58,11 +71,6 @@ type EventDataProposalVoteStarted struct { StartVote www2.StartVote } -type EventDataProposalVoteAuthorized struct { - AuthorizeVote *www.AuthorizeVote - User *user.User -} - type EventDataComment struct { Comment *www.Comment } @@ -427,27 +435,17 @@ func (p *politeiawww) _setupProposalVoteAuthorizedEmailNotification() { ch := make(chan interface{}) go func() { for data := range ch { - pvs, ok := data.(EventDataProposalVoteAuthorized) + pvs, ok := data.(eventDataProposalVoteAuthorized) if !ok { log.Errorf("invalid event data") continue } - token := pvs.AuthorizeVote.Token - record, err := p.cache.Record(token) - if err != nil { - log.Errorf("proposal not found: %v", err) - continue - } - proposal, err := convertPropFromCache(*record) - if err != nil { - log.Errorf("invalid proposal %v", token) - } - - err = p.emailAdminsForProposalVoteAuthorized(proposal, pvs.User) + err := p.emailAdminsForProposalVoteAuthorized(pvs.token, pvs.name, + pvs.authorUsername, pvs.authorEmail) if err != nil { log.Errorf("email all admins for new submitted proposal %v: %v", - token, err) + pvs.token, err) } } }() diff --git a/politeiawww/plugin.go b/politeiawww/plugin.go index 92fff4095..f2eb417e9 100644 --- a/politeiawww/plugin.go +++ b/politeiawww/plugin.go @@ -30,9 +30,9 @@ type Plugin struct { Settings []PluginSetting // Settings } -// getBestBlockDecredPlugin asks the decred plugin what the current best block -// is. -func (p *politeiawww) getBestBlockDecredPlugin() (uint64, error) { +// getBestBlock fetches and returns the best block from politeiad using the +// decred plugin bestblock command. +func (p *politeiawww) getBestBlock() (uint64, error) { challenge, err := util.Random(pd.ChallengeSize) if err != nil { return 0, err diff --git a/politeiawww/politeiawww.go b/politeiawww/politeiawww.go index a14585fe9..b8151cb12 100644 --- a/politeiawww/politeiawww.go +++ b/politeiawww/politeiawww.go @@ -16,8 +16,6 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/decred/dcrd/chaincfg" - exptypes "github.com/decred/dcrdata/explorer/types/v2" - pstypes "github.com/decred/dcrdata/pubsub/types/v3" "github.com/decred/politeia/politeiad/api/v1/mime" www "github.com/decred/politeia/politeiawww/api/www/v1" www2 "github.com/decred/politeia/politeiawww/api/www/v2" @@ -873,119 +871,6 @@ func (p *politeiawww) handleWebsocketWrite(wc *wsContext) { } } -// updateBestBlock updates the cached best block. -func (p *politeiawww) updateBestBlock(bestBlock uint64) { - p.bbMtx.Lock() - defer p.bbMtx.Unlock() - p.bestBlock = bestBlock -} - -// getBestBlock returns the cached best block if there is an active websocket -// connection to dcrdata. Otherwise, it requests the best block from politeiad -// using the the decred plugin best block command. -func (p *politeiawww) getBestBlock() (uint64, error) { - p.bbMtx.RLock() - bb := p.bestBlock - p.bbMtx.RUnlock() - - // the cached best block will equal 0 if there is no active websocket - // connection to dcrdata, or if no new block messages have been received - // since a connection was established. - if bb == 0 { - return p.getBestBlockDecredPlugin() - } - - return bb, nil -} - -// monitorWSDcrdataPi monitors the websocket connection for pi and handles -// new websocket messages. -func (p *politeiawww) monitorWSDcrdataPi() { - defer func() { - log.Infof("Dcrdata websocket closed") - }() - - // Setup messages channel - receiver := p.wsDcrdata.Receive() - - for { - // Monitor for a new message - msg, ok := <-receiver - if !ok { - // Check if the websocket was shut down intentionally or was - // dropped unexpectedly. - if p.wsDcrdata.Status() == wsdcrdata.StatusShutdown { - return - } - log.Infof("Dcrdata websocket connection unexpectedly dropped") - goto reconnect - } - - // Handle new message - switch m := msg.Message.(type) { - case *exptypes.WebsocketBlock: - log.Debugf("wsDcrdata message WebsocketBlock(height=%v)", - m.Block.Height) - - // Update cached best block - bb := uint64(m.Block.Height) - p.updateBestBlock(bb) - - // Keep VoteResults table updated with received best block - _, err := p.decredLoadVoteResults(bb) - if err != nil { - log.Errorf("decredLoadVoteResults: %v", err) - } - - case *pstypes.HangUp: - log.Infof("Dcrdata websocket has hung up. Will reconnect.") - goto reconnect - - case int: - // Ping messages are of type int - - default: - log.Errorf("wsDcrdata message of type %v unhandled: %v", - msg.EventId, m) - } - - // Check for next message - continue - - reconnect: - // Update best block to 0 to indicate that the websocket - // connection is closed. - p.updateBestBlock(0) - - // Reconnect - p.wsDcrdata.Reconnect() - - // Setup a new messages channel using the new connection. - receiver = p.wsDcrdata.Receive() - - log.Infof("Successfully reconnected dcrdata websocket") - } -} - -// setupWSDcrataPi subscribes and listens to websocket messages from dcrdata -// that are needed for pi. -func (p *politeiawww) setupWSDcrdataPi() { - // Ensure connection is open. If connection is closed, establish a - // new connection before continuing. - if p.wsDcrdata.Status() != wsdcrdata.StatusOpen { - p.wsDcrdata.Reconnect() - } - - // Setup subscriptions - err := p.wsDcrdata.NewBlockSubscribe() - if err != nil { - log.Errorf("wsdcrdata NewBlockSubscribe: %v", err) - } - - // Monitor websocket connection in a new go routine - go p.monitorWSDcrdataPi() -} - // handleWebsocket upgrades a regular HTTP connection to a websocket. func (p *politeiawww) handleWebsocket(w http.ResponseWriter, r *http.Request, id string) { log.Tracef("handleWebsocket: %v", id) diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 2735838fc..879af2edf 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -15,7 +15,6 @@ import ( "regexp" "sort" "strconv" - "strings" "time" "github.com/decred/politeia/decredplugin" @@ -619,15 +618,18 @@ func (p *politeiawww) fillProposalMissingFields(pr *www.ProposalRecord) error { pr.UserId = u.ID.String() pr.Username = u.Username - // Find linked from proposals - lfr, err := p.decredLinkedFrom([]string{pr.CensorshipRecord.Token}) - if err != nil { - return err - } - linkedFrom, ok := lfr.LinkedFrom[pr.CensorshipRecord.Token] - if ok { - pr.LinkedFrom = linkedFrom - } + /* + // TODO + // Find linked from proposals + lfr, err := p.decredLinkedFrom([]string{pr.CensorshipRecord.Token}) + if err != nil { + return err + } + linkedFrom, ok := lfr.LinkedFrom[pr.CensorshipRecord.Token] + if ok { + pr.LinkedFrom = linkedFrom + } + */ return nil } @@ -656,7 +658,7 @@ func (p *politeiawww) getProp(token string) (*www.ProposalRecord, error) { */ var pr *www.ProposalRecord - err = p.fillProposalMissingFields(pr) + err := p.fillProposalMissingFields(pr) if err != nil { return nil, err } @@ -670,17 +672,17 @@ func (p *politeiawww) getProp(token string) (*www.ProposalRecord, error) { // the caller to ensure that results are returned for all of the provided // censorship tokens. func (p *politeiawww) getProps(tokens []string) (map[string]www.ProposalRecord, error) { - log.Tracef("getProps: %v", tokens) - - // Get the proposals from the cache /* - // TODO + log.Tracef("getProps: %v", tokens) + + // Get the proposals from the cache records, err := p.cache.Records(tokens, true) if err != nil { return nil, err } // Use pointers for now so the props can be easily updated + props := make(map[string]*www.ProposalRecord, len(records)) for _, v := range records { pr, err := convertPropFromCache(v) if err != nil { @@ -689,74 +691,74 @@ func (p *politeiawww) getProps(tokens []string) (map[string]www.ProposalRecord, } props[v.CensorshipRecord.Token] = pr } - */ - props := make(map[string]*www.ProposalRecord, len(records)) - // Get the number of comments for each proposal. Comments - // are part of decred plugin so this must be fetched from - // the cache separately. - dnc, err := p.decredGetNumComments(tokens) - if err != nil { - return nil, err - } - for token, numComments := range dnc { - props[token].NumComments = uint(numComments) - } + // Get the number of comments for each proposal. Comments + // are part of decred plugin so this must be fetched from + // the cache separately. + dnc, err := p.decredGetNumComments(tokens) + if err != nil { + return nil, err + } + for token, numComments := range dnc { + props[token].NumComments = uint(numComments) + } - // Find linked from proposals - lfr, err := p.decredLinkedFrom(tokens) - if err != nil { - return nil, err - } - for token, linkedFrom := range lfr.LinkedFrom { - props[token].LinkedFrom = linkedFrom - } + // Find linked from proposals + lfr, err := p.decredLinkedFrom(tokens) + if err != nil { + return nil, err + } + for token, linkedFrom := range lfr.LinkedFrom { + props[token].LinkedFrom = linkedFrom + } - // Compile a list of unique proposal author pubkeys. These - // are needed to lookup the proposal author info. - pubKeys := make(map[string]struct{}) - for _, pr := range props { - if _, ok := pubKeys[pr.PublicKey]; !ok { - pubKeys[pr.PublicKey] = struct{}{} + // Compile a list of unique proposal author pubkeys. These + // are needed to lookup the proposal author info. + pubKeys := make(map[string]struct{}) + for _, pr := range props { + if _, ok := pubKeys[pr.PublicKey]; !ok { + pubKeys[pr.PublicKey] = struct{}{} + } } - } - // Lookup proposal authors - pk := make([]string, 0, len(pubKeys)) - for k := range pubKeys { - pk = append(pk, k) - } - users, err := p.db.UsersGetByPubKey(pk) - if err != nil { - return nil, err - } - if len(users) != len(pubKeys) { - // A user is missing from the userdb for one - // or more public keys. We're in trouble! - notFound := make([]string, 0, len(pubKeys)) - for v := range pubKeys { - if _, ok := users[v]; !ok { - notFound = append(notFound, v) + // Lookup proposal authors + pk := make([]string, 0, len(pubKeys)) + for k := range pubKeys { + pk = append(pk, k) + } + users, err := p.db.UsersGetByPubKey(pk) + if err != nil { + return nil, err + } + if len(users) != len(pubKeys) { + // A user is missing from the userdb for one + // or more public keys. We're in trouble! + notFound := make([]string, 0, len(pubKeys)) + for v := range pubKeys { + if _, ok := users[v]; !ok { + notFound = append(notFound, v) + } } + e := fmt.Sprintf("users not found for pubkeys: %v", + strings.Join(notFound, ", ")) + panic(e) } - e := fmt.Sprintf("users not found for pubkeys: %v", - strings.Join(notFound, ", ")) - panic(e) - } - // Fill in proposal author info - for i, pr := range props { - props[i].UserId = users[pr.PublicKey].ID.String() - props[i].Username = users[pr.PublicKey].Username - } + // Fill in proposal author info + for i, pr := range props { + props[i].UserId = users[pr.PublicKey].ID.String() + props[i].Username = users[pr.PublicKey].Username + } - // Convert pointers to values - proposals := make(map[string]www.ProposalRecord, len(props)) - for token, pr := range props { - proposals[token] = *pr - } + // Convert pointers to values + proposals := make(map[string]www.ProposalRecord, len(props)) + for token, pr := range props { + proposals[token] = *pr + } - return proposals, nil + return proposals, nil + */ + return nil, nil } // getPropVersion gets a specific version of a proposal from the cache then @@ -774,15 +776,16 @@ func (p *politeiawww) getPropVersion(token, version string) (*www.ProposalRecord if err != nil { return nil, err } - */ - var pr *www.ProposalRecord - err = p.fillProposalMissingFields(pr) - if err != nil { - return nil, err - } + err = p.fillProposalMissingFields(pr) + if err != nil { + return nil, err + } - return pr, nil + return pr, nil + */ + + return nil, nil } // getAllProps gets the latest version of all proposals from the cache then @@ -978,40 +981,39 @@ func (p *politeiawww) getPropComments(token string) ([]www.Comment, error) { // // This function must be called WITHOUT read/write lock held. func (p *politeiawww) voteSummariesGet(tokens []string, bestBlock uint64) (map[string]www.VoteSummary, error) { - voteSummaries := make(map[string]www.VoteSummary) - tokensToLookup := make([]string, 0, len(tokens)) - - p.RLock() - for _, token := range tokens { - vs, ok := p.voteSummaries[token] - if ok { - voteSummaries[token] = vs - } else { - tokensToLookup = append(tokensToLookup, token) - } - } - p.RUnlock() - - if len(tokensToLookup) == 0 { - return voteSummaries, nil - } - - // Fetch the vote summaries from the cache. This call relies on the - // lazy loaded VoteResults cache table. If the VoteResults table is - // not up-to-date then this function will load it before retrying - // the vote summary call. Since politeiawww only has read access to - // the cache, loading the VoteResults table requires using a - // politeiad decredplugin command. - var ( - done bool - err error - reply *decredplugin.BatchVoteSummaryReply - ) - for retries := 0; !done && retries <= 1; retries++ { - reply, err = p.decredBatchVoteSummary(tokensToLookup, bestBlock) - if err != nil { - /* - // TODO + /* + voteSummaries := make(map[string]www.VoteSummary) + tokensToLookup := make([]string, 0, len(tokens)) + + p.RLock() + for _, token := range tokens { + vs, ok := p.voteSummaries[token] + if ok { + voteSummaries[token] = vs + } else { + tokensToLookup = append(tokensToLookup, token) + } + } + p.RUnlock() + + if len(tokensToLookup) == 0 { + return voteSummaries, nil + } + + // Fetch the vote summaries from the cache. This call relies on the + // lazy loaded VoteResults cache table. If the VoteResults table is + // not up-to-date then this function will load it before retrying + // the vote summary call. Since politeiawww only has read access to + // the cache, loading the VoteResults table requires using a + // politeiad decredplugin command. + var ( + done bool + err error + reply *decredplugin.BatchVoteSummaryReply + ) + for retries := 0; !done && retries <= 1; retries++ { + reply, err = p.decredBatchVoteSummary(tokensToLookup, bestBlock) + if err != nil { if err == cache.ErrRecordNotFound { // There are missing entries in the VoteResults // cache table. Load them. @@ -1023,52 +1025,54 @@ func (p *politeiawww) voteSummariesGet(tokens []string, bestBlock uint64) (map[s // Retry the vote summaries call continue } - */ - return nil, err - } + return nil, err + } - done = true - } + done = true + } - for token, v := range reply.Summaries { - results := convertVoteOptionResultsFromDecred(v.Results) - votet := convertVoteTypeFromDecred(v.Type) + for token, v := range reply.Summaries { + results := convertVoteOptionResultsFromDecred(v.Results) + votet := convertVoteTypeFromDecred(v.Type) - // An endHeight will not exist if the proposal has not gone - // up for vote yet. - var endHeight uint64 - if v.EndHeight != "" { - i, err := strconv.ParseUint(v.EndHeight, 10, 64) - if err != nil { - return nil, fmt.Errorf("failed to parse end height "+ - "'%v' for %v: %v", v.EndHeight, token, err) + // An endHeight will not exist if the proposal has not gone + // up for vote yet. + var endHeight uint64 + if v.EndHeight != "" { + i, err := strconv.ParseUint(v.EndHeight, 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse end height "+ + "'%v' for %v: %v", v.EndHeight, token, err) + } + endHeight = i } - endHeight = i - } - vs := www.VoteSummary{ - Status: voteStatusFromVoteSummary(v, endHeight, bestBlock), - Type: www.VoteT(int(votet)), - Approved: v.Approved, - EligibleTickets: uint32(v.EligibleTicketCount), - Duration: v.Duration, - EndHeight: endHeight, - QuorumPercentage: v.QuorumPercentage, - PassPercentage: v.PassPercentage, - Results: results, - } + vs := www.VoteSummary{ + Status: voteStatusFromVoteSummary(v, endHeight, bestBlock), + Type: www.VoteT(int(votet)), + Approved: v.Approved, + EligibleTickets: uint32(v.EligibleTicketCount), + Duration: v.Duration, + EndHeight: endHeight, + QuorumPercentage: v.QuorumPercentage, + PassPercentage: v.PassPercentage, + Results: results, + } - voteSummaries[token] = vs + voteSummaries[token] = vs - // If the voting period has ended the vote status - // is not going to change so add it to the memory - // cache. - if vs.Status == www.PropVoteStatusFinished { - p.voteSummarySet(token, vs) + // If the voting period has ended the vote status + // is not going to change so add it to the memory + // cache. + if vs.Status == www.PropVoteStatusFinished { + p.voteSummarySet(token, vs) + } } - } - return voteSummaries, nil + return voteSummaries, nil + */ + + return nil, nil } // voteSummaryGet stores the provided VoteSummary in the vote summaries memory @@ -1080,7 +1084,7 @@ func (p *politeiawww) voteSummarySet(token string, voteSummary www.VoteSummary) p.Lock() defer p.Unlock() - p.voteSummaries[token] = voteSummary + // p.voteSummaries[token] = voteSummary } // voteSummaryGet returns the VoteSummary for the given token. A cache @@ -1954,262 +1958,243 @@ func (p *politeiawww) processCommentsGet(token string, u *user.User) (*www.GetCo // ** This fuction is to be removed when the deprecated vote status route is // ** removed. func (p *politeiawww) setVoteStatusReply(v www.VoteStatusReply) error { - p.Lock() - defer p.Unlock() - - endHeight, err := strconv.Atoi(v.EndHeight) - if err != nil { - return err - } - - voteSummary := www.VoteSummary{ - Status: v.Status, - EligibleTickets: uint32(v.NumOfEligibleVotes), - EndHeight: uint64(endHeight), - QuorumPercentage: v.QuorumPercentage, - PassPercentage: v.PassPercentage, - Results: v.OptionsResult, - } - - p.voteSummaries[v.Token] = voteSummary + /* + p.Lock() + defer p.Unlock() - return nil -} + endHeight, err := strconv.Atoi(v.EndHeight) + if err != nil { + return err + } -// getVoteStatusReply retrieves the VoteSummary from the cache for a proposal -// whose voting period has ended and converts it to a VoteStatusReply. -// -// This function must be called without the lock held. -// -// ** This fuction is to be removed when the deprecated vote status route is -// ** removed. -func (p *politeiawww) getVoteStatusReply(token string) (*www.VoteStatusReply, bool) { - p.RLock() - vs, ok := p.voteSummaries[token] - p.RUnlock() + voteSummary := www.VoteSummary{ + Status: v.Status, + EligibleTickets: uint32(v.NumOfEligibleVotes), + EndHeight: uint64(endHeight), + QuorumPercentage: v.QuorumPercentage, + PassPercentage: v.PassPercentage, + Results: v.OptionsResult, + } - if !ok { - return nil, false - } + p.voteSummaries[v.Token] = voteSummary - var totalVotes uint64 - for _, or := range vs.Results { - totalVotes += or.VotesReceived - } - - voteStatusReply := www.VoteStatusReply{ - Token: token, - Status: vs.Status, - TotalVotes: totalVotes, - OptionsResult: vs.Results, - EndHeight: strconv.Itoa(int(vs.EndHeight)), - NumOfEligibleVotes: int(vs.EligibleTickets), - QuorumPercentage: vs.QuorumPercentage, - PassPercentage: vs.PassPercentage, - } + return nil + */ - return &voteStatusReply, true + return nil } func (p *politeiawww) voteStatusReply(token string, bestBlock uint64) (*www.VoteStatusReply, error) { - cachedVsr, ok := p.getVoteStatusReply(token) + /* + cachedVsr, ok := p.getVoteStatusReply(token) - if ok { - cachedVsr.BestBlock = strconv.Itoa(int(bestBlock)) - return cachedVsr, nil - } + if ok { + cachedVsr.BestBlock = strconv.Itoa(int(bestBlock)) + return cachedVsr, nil + } - // Vote status wasn't in the memory cache - // so fetch it from the cache database. - vs, err := p.voteSummaryGet(token, bestBlock) - if err != nil { - return nil, err - } + // Vote status wasn't in the memory cache + // so fetch it from the cache database. + vs, err := p.voteSummaryGet(token, bestBlock) + if err != nil { + return nil, err + } - var total uint64 - for _, v := range vs.Results { - total += v.VotesReceived - } + var total uint64 + for _, v := range vs.Results { + total += v.VotesReceived + } - voteStatusReply := www.VoteStatusReply{ - Token: token, - Status: vs.Status, - TotalVotes: total, - OptionsResult: vs.Results, - EndHeight: strconv.Itoa(int(vs.EndHeight)), - BestBlock: strconv.Itoa(int(bestBlock)), - NumOfEligibleVotes: int(vs.EligibleTickets), - QuorumPercentage: vs.QuorumPercentage, - PassPercentage: vs.PassPercentage, - } + voteStatusReply := www.VoteStatusReply{ + Token: token, + Status: vs.Status, + TotalVotes: total, + OptionsResult: vs.Results, + EndHeight: strconv.Itoa(int(vs.EndHeight)), + BestBlock: strconv.Itoa(int(bestBlock)), + NumOfEligibleVotes: int(vs.EligibleTickets), + QuorumPercentage: vs.QuorumPercentage, + PassPercentage: vs.PassPercentage, + } - // If the voting period has ended the vote status - // is not going to change so add it to the memory - // cache. - if voteStatusReply.Status == www.PropVoteStatusFinished { - err = p.setVoteStatusReply(voteStatusReply) - if err != nil { - return nil, err + // If the voting period has ended the vote status + // is not going to change so add it to the memory + // cache. + if voteStatusReply.Status == www.PropVoteStatusFinished { + err = p.setVoteStatusReply(voteStatusReply) + if err != nil { + return nil, err + } } - } - return &voteStatusReply, nil + return &voteStatusReply, nil + */ + return nil, nil } // processVoteStatus returns the vote status for a given proposal func (p *politeiawww) processVoteStatus(token string) (*www.VoteStatusReply, error) { log.Tracef("ProcessProposalVotingStatus: %v", token) - // Make sure token is valid and not a prefix - if !isTokenValid(token) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: []string{token}, + /* + // Make sure token is valid and not a prefix + if !isTokenValid(token) { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidCensorshipToken, + ErrorContext: []string{token}, + } } - } - // Ensure proposal is vetted - pr, err := p.getProp(token) - if err != nil { - /* - // TODO + // Ensure proposal is vetted + pr, err := p.getProp(token) + if err != nil { if err == cache.ErrRecordNotFound { err = www.UserError{ ErrorCode: www.ErrorStatusProposalNotFound, } } - */ - return nil, err - } + return nil, err + } - if pr.State != www.PropStateVetted { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusWrongStatus, + if pr.State != www.PropStateVetted { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusWrongStatus, + } } - } - // Get best block - bestBlock, err := p.getBestBlock() - if err != nil { - return nil, fmt.Errorf("bestBlock: %v", err) - } + // Get best block + bestBlock, err := p.getBestBlock() + if err != nil { + return nil, fmt.Errorf("bestBlock: %v", err) + } - // Get vote status - vsr, err := p.voteStatusReply(token, bestBlock) - if err != nil { - return nil, fmt.Errorf("voteStatusReply: %v", err) - } + // Get vote status + vsr, err := p.voteStatusReply(token, bestBlock) + if err != nil { + return nil, fmt.Errorf("voteStatusReply: %v", err) + } - return vsr, nil + return vsr, nil + */ + + return nil, nil } // processGetAllVoteStatus returns the vote status of all public proposals. func (p *politeiawww) processGetAllVoteStatus() (*www.GetAllVoteStatusReply, error) { log.Tracef("processGetAllVoteStatus") - // We need to determine best block height here in order - // to set the voting status - bestBlock, err := p.getBestBlock() - if err != nil { - return nil, fmt.Errorf("bestBlock: %v", err) - } - - // Get all proposals from cache - // TODO do not use getAllProps - all, err := p.getAllProps() - if err != nil { - return nil, fmt.Errorf("getAllProps: %v", err) - } - - // Compile votes statuses - vrr := make([]www.VoteStatusReply, 0, len(all)) - for _, v := range all { - // We only need public proposals - if v.Status != www.PropStatusPublic { - continue + /* + // We need to determine best block height here in order + // to set the voting status + bestBlock, err := p.getBestBlock() + if err != nil { + return nil, fmt.Errorf("bestBlock: %v", err) } - // Get vote status for proposal - vs, err := p.voteStatusReply(v.CensorshipRecord.Token, bestBlock) + // Get all proposals from cache + // TODO do not use getAllProps + all, err := p.getAllProps() if err != nil { - return nil, fmt.Errorf("voteStatusReply: %v", err) + return nil, fmt.Errorf("getAllProps: %v", err) } - vrr = append(vrr, *vs) - } + // Compile votes statuses + vrr := make([]www.VoteStatusReply, 0, len(all)) + for _, v := range all { + // We only need public proposals + if v.Status != www.PropStatusPublic { + continue + } - return &www.GetAllVoteStatusReply{ - VotesStatus: vrr, - }, nil + // Get vote status for proposal + vs, err := p.voteStatusReply(v.CensorshipRecord.Token, bestBlock) + if err != nil { + return nil, fmt.Errorf("voteStatusReply: %v", err) + } + + vrr = append(vrr, *vs) + } + + return &www.GetAllVoteStatusReply{ + VotesStatus: vrr, + }, nil + */ + + return nil, nil } func (p *politeiawww) processActiveVote() (*www.ActiveVoteReply, error) { log.Tracef("processActiveVote") - // Fetch proposals that are actively being voted on - bb, err := p.getBestBlock() - if err != nil { - return nil, err - } - tir, err := p.tokenInventory(bb, false) - if err != nil { - return nil, err - } - props, err := p.getProps(tir.Active) - if err != nil { - return nil, err - } - - // Compile proposal vote tuples - pvt := make([]www.ProposalVoteTuple, 0, len(props)) - for _, v := range props { - // Get vote details from cache - vdr, err := p.decredVoteDetails(v.CensorshipRecord.Token) + /* + // Fetch proposals that are actively being voted on + bb, err := p.getBestBlock() + if err != nil { + return nil, err + } + tir, err := p.tokenInventory(bb, false) if err != nil { - return nil, fmt.Errorf("decredVoteDetails %v: %v", - v.CensorshipRecord.Token, err) + return nil, err + } + props, err := p.getProps(tir.Active) + if err != nil { + return nil, err } - // Handle StartVote versioning - var sv www.StartVote - switch vdr.StartVote.Version { - case decredplugin.VersionStartVoteV1: - b := []byte(vdr.StartVote.Payload) - dsv, err := decredplugin.DecodeStartVoteV1(b) + // Compile proposal vote tuples + pvt := make([]www.ProposalVoteTuple, 0, len(props)) + for _, v := range props { + // Get vote details from cache + vdr, err := p.decredVoteDetails(v.CensorshipRecord.Token) if err != nil { - return nil, fmt.Errorf("decode StartVoteV1 %v: %v", + return nil, fmt.Errorf("decredVoteDetails %v: %v", v.CensorshipRecord.Token, err) } - sv = convertStartVoteV1FromDecred(*dsv) - case decredplugin.VersionStartVoteV2: - b := []byte(vdr.StartVote.Payload) - dsv2, err := decredplugin.DecodeStartVoteV2(b) - if err != nil { - return nil, fmt.Errorf("decode StartVoteV2 %v: %v", - v.CensorshipRecord.Token, err) + // Handle StartVote versioning + var sv www.StartVote + switch vdr.StartVote.Version { + case decredplugin.VersionStartVoteV1: + b := []byte(vdr.StartVote.Payload) + dsv, err := decredplugin.DecodeStartVoteV1(b) + if err != nil { + return nil, fmt.Errorf("decode StartVoteV1 %v: %v", + v.CensorshipRecord.Token, err) + } + sv = convertStartVoteV1FromDecred(*dsv) + + case decredplugin.VersionStartVoteV2: + b := []byte(vdr.StartVote.Payload) + dsv2, err := decredplugin.DecodeStartVoteV2(b) + if err != nil { + return nil, fmt.Errorf("decode StartVoteV2 %v: %v", + v.CensorshipRecord.Token, err) + } + sv2 := convertStartVoteV2FromDecred(*dsv2) + // Convert StartVote v2 to v1 since this route returns + // a v1 StartVote. + sv = convertStartVoteV2ToV1(sv2) + + default: + return nil, fmt.Errorf("invalid StartVote version %v %v", + v.CensorshipRecord.Token, vdr.StartVote.Version) } - sv2 := convertStartVoteV2FromDecred(*dsv2) - // Convert StartVote v2 to v1 since this route returns - // a v1 StartVote. - sv = convertStartVoteV2ToV1(sv2) - default: - return nil, fmt.Errorf("invalid StartVote version %v %v", - v.CensorshipRecord.Token, vdr.StartVote.Version) + // Create vote tuple + pvt = append(pvt, www.ProposalVoteTuple{ + Proposal: v, + StartVote: sv, + StartVoteReply: convertStartVoteReplyFromDecred(vdr.StartVoteReply), + }) } - // Create vote tuple - pvt = append(pvt, www.ProposalVoteTuple{ - Proposal: v, - StartVote: sv, - StartVoteReply: convertStartVoteReplyFromDecred(vdr.StartVoteReply), - }) - } + return &www.ActiveVoteReply{ + Votes: pvt, + }, nil + */ - return &www.ActiveVoteReply{ - Votes: pvt, - }, nil + return nil, nil } // processVoteResults returns the vote details for a specific proposal and all @@ -2217,80 +2202,81 @@ func (p *politeiawww) processActiveVote() (*www.ActiveVoteReply, error) { func (p *politeiawww) processVoteResults(token string) (*www.VoteResultsReply, error) { log.Tracef("processVoteResults: %v", token) - // Make sure token is valid and not a prefix - if !isTokenValid(token) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: []string{token}, + /* + // Make sure token is valid and not a prefix + if !isTokenValid(token) { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidCensorshipToken, + ErrorContext: []string{token}, + } } - } - // Ensure proposal is vetted - pr, err := p.getProp(token) - if err != nil { - /* - // TODO - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, + // Ensure proposal is vetted + pr, err := p.getProp(token) + if err != nil { + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + } } - } - */ - return nil, err - } - - if pr.State != www.PropStateVetted { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusWrongStatus, + return nil, err } - } - // Get vote details from cache - vdr, err := p.decredVoteDetails(token) - if err != nil { - return nil, fmt.Errorf("decredVoteDetails: %v", err) - } - if vdr.StartVoteReply.StartBlockHash == "" { - // Vote has not been started yet. No need to continue. - return &www.VoteResultsReply{}, nil - } - - // Get cast votes from cache - vrr, err := p.decredProposalVotes(token) - if err != nil { - return nil, fmt.Errorf("decredProposalVotes: %v", err) - } + if pr.State != www.PropStateVetted { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusWrongStatus, + } + } - // Handle StartVote versioning - var sv www.StartVote - switch vdr.StartVote.Version { - case decredplugin.VersionStartVoteV1: - b := []byte(vdr.StartVote.Payload) - dsv1, err := decredplugin.DecodeStartVoteV1(b) + // Get vote details from cache + vdr, err := p.decredVoteDetails(token) if err != nil { - return nil, err + return nil, fmt.Errorf("decredVoteDetails: %v", err) + } + if vdr.StartVoteReply.StartBlockHash == "" { + // Vote has not been started yet. No need to continue. + return &www.VoteResultsReply{}, nil } - sv = convertStartVoteV1FromDecred(*dsv1) - case decredplugin.VersionStartVoteV2: - b := []byte(vdr.StartVote.Payload) - dsv2, err := decredplugin.DecodeStartVoteV2(b) + + // Get cast votes from cache + vrr, err := p.decredProposalVotes(token) if err != nil { - return nil, err + return nil, fmt.Errorf("decredProposalVotes: %v", err) } - sv2 := convertStartVoteV2FromDecred(*dsv2) - // Convert StartVote v2 to v1 since this route returns - // a v1 StartVote. - sv = convertStartVoteV2ToV1(sv2) - default: - return nil, fmt.Errorf("invalid StartVote version %v %v", - token, vdr.StartVote.Version) - } - return &www.VoteResultsReply{ - StartVote: sv, - StartVoteReply: convertStartVoteReplyFromDecred(vdr.StartVoteReply), - CastVotes: convertCastVotesFromDecred(vrr.CastVotes), - }, nil + // Handle StartVote versioning + var sv www.StartVote + switch vdr.StartVote.Version { + case decredplugin.VersionStartVoteV1: + b := []byte(vdr.StartVote.Payload) + dsv1, err := decredplugin.DecodeStartVoteV1(b) + if err != nil { + return nil, err + } + sv = convertStartVoteV1FromDecred(*dsv1) + case decredplugin.VersionStartVoteV2: + b := []byte(vdr.StartVote.Payload) + dsv2, err := decredplugin.DecodeStartVoteV2(b) + if err != nil { + return nil, err + } + sv2 := convertStartVoteV2FromDecred(*dsv2) + // Convert StartVote v2 to v1 since this route returns + // a v1 StartVote. + sv = convertStartVoteV2ToV1(sv2) + default: + return nil, fmt.Errorf("invalid StartVote version %v %v", + token, vdr.StartVote.Version) + } + + return &www.VoteResultsReply{ + StartVote: sv, + StartVoteReply: convertStartVoteReplyFromDecred(vdr.StartVoteReply), + CastVotes: convertCastVotesFromDecred(vrr.CastVotes), + }, nil + */ + + return nil, nil } // processCastVotes handles the www.Ballot call @@ -2613,9 +2599,11 @@ func (p *politeiawww) processAuthorizeVote(av www.AuthorizeVote, u *user.User) ( if !p.test && avr.Action == decredplugin.AuthVoteActionAuthorize { p.fireEvent(EventTypeProposalVoteAuthorized, - EventDataProposalVoteAuthorized{ - AuthorizeVote: &av, - User: u, + eventDataProposalVoteAuthorized{ + token: av.Token, + name: pr.Name, + authorUsername: u.Email, + authorEmail: u.Username, }, ) } @@ -3244,16 +3232,15 @@ func (p *politeiawww) processStartVoteRunoffV2(sv www2.StartVoteRunoff, u *user. // read access to the cache, loading the VoteResults table requires using a // politeiad decredplugin command. func (p *politeiawww) tokenInventory(bestBlock uint64, isAdmin bool) (*www.TokenInventoryReply, error) { - var done bool - var r www.TokenInventoryReply - for retries := 0; !done && retries <= 1; retries++ { - // Both vetted and unvetted tokens should be returned - // for admins. Only vetted tokens should be returned - // for non-admins. - ti, err := p.decredTokenInventory(bestBlock, isAdmin) - if err != nil { - /* - // TODO + /* + var done bool + var r www.TokenInventoryReply + for retries := 0; !done && retries <= 1; retries++ { + // Both vetted and unvetted tokens should be returned + // for admins. Only vetted tokens should be returned + // for non-admins. + ti, err := p.decredTokenInventory(bestBlock, isAdmin) + if err != nil { if err == cache.ErrRecordNotFound { // There are missing entries in the vote // results cache table. Load them. @@ -3265,15 +3252,17 @@ func (p *politeiawww) tokenInventory(bestBlock uint64, isAdmin bool) (*www.Token // Retry token inventory call continue } - */ - return nil, err + return nil, err + } + + r = convertTokenInventoryReplyFromDecred(*ti) + done = true } - r = convertTokenInventoryReplyFromDecred(*ti) - done = true - } + return &r, nil + */ - return &r, nil + return nil, nil } // processTokenInventory returns the tokens of all proposals in the inventory, @@ -3293,71 +3282,57 @@ func (p *politeiawww) processTokenInventory(isAdmin bool) (*www.TokenInventoryRe func (p *politeiawww) processVoteDetailsV2(token string) (*www2.VoteDetailsReply, error) { log.Tracef("processVoteDetailsV2: %v", token) - // Validate vote status - dvdr, err := p.decredVoteDetails(token) - if err != nil { - /* - // TODO - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - } - } - */ - return nil, err - } - if dvdr.StartVoteReply.StartBlockHash == "" { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - ErrorContext: []string{"voting has not started yet"}, - } - } - - // Handle StartVote versioning - var vdr *www2.VoteDetailsReply - switch dvdr.StartVote.Version { - case decredplugin.VersionStartVoteV1: - b := []byte(dvdr.StartVote.Payload) - dsv1, err := decredplugin.DecodeStartVoteV1(b) - if err != nil { - return nil, err - } - vdr, err = convertDecredStartVoteV1ToVoteDetailsReplyV2(*dsv1, - dvdr.StartVoteReply) - if err != nil { - return nil, err - } - case decredplugin.VersionStartVoteV2: - b := []byte(dvdr.StartVote.Payload) - dsv2, err := decredplugin.DecodeStartVoteV2(b) + /* + // Validate vote status + dvdr, err := p.decredVoteDetails(token) if err != nil { + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + } + } return nil, err } - vdr, err = convertDecredStartVoteV2ToVoteDetailsReplyV2(*dsv2, - dvdr.StartVoteReply) - if err != nil { - return nil, err + if dvdr.StartVoteReply.StartBlockHash == "" { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusWrongVoteStatus, + ErrorContext: []string{"voting has not started yet"}, + } } - default: - return nil, fmt.Errorf("invalid StartVote version %v %v", - token, dvdr.StartVote.Version) - } + // Handle StartVote versioning + var vdr *www2.VoteDetailsReply + switch dvdr.StartVote.Version { + case decredplugin.VersionStartVoteV1: + b := []byte(dvdr.StartVote.Payload) + dsv1, err := decredplugin.DecodeStartVoteV1(b) + if err != nil { + return nil, err + } + vdr, err = convertDecredStartVoteV1ToVoteDetailsReplyV2(*dsv1, + dvdr.StartVoteReply) + if err != nil { + return nil, err + } + case decredplugin.VersionStartVoteV2: + b := []byte(dvdr.StartVote.Payload) + dsv2, err := decredplugin.DecodeStartVoteV2(b) + if err != nil { + return nil, err + } + vdr, err = convertDecredStartVoteV2ToVoteDetailsReplyV2(*dsv2, + dvdr.StartVoteReply) + if err != nil { + return nil, err + } - return vdr, nil -} + default: + return nil, fmt.Errorf("invalid StartVote version %v %v", + token, dvdr.StartVote.Version) + } -// initLoadVoteResults is used to send the LoadVoteResults decred plugin command -// to the cache on www startup. -func (p *politeiawww) initVoteResults() error { - bb, err := p.getBestBlock() - if err != nil { - return err - } - _, err = p.decredLoadVoteResults(bb) - if err != nil { - return err - } + return vdr, nil + */ - return nil + return nil, nil } diff --git a/politeiawww/testing.go b/politeiawww/testing.go index 29575e3f9..aa769e44d 100644 --- a/politeiawww/testing.go +++ b/politeiawww/testing.go @@ -697,7 +697,6 @@ func newTestPoliteiawww(t *testing.T) (*politeiawww, func()) { userEmails: make(map[string]uuid.UUID), userPaywallPool: make(map[uuid.UUID]paywallPoolMember), commentVotes: make(map[string]counters), - voteSummaries: make(map[string]www.VoteSummary), } // Setup routes @@ -731,7 +730,7 @@ func newTestPoliteiawww(t *testing.T) (*politeiawww, func()) { // newTestPoliteiad returns a new TestPoliteiad context. The relevant // politeiawww config params are updated with the TestPoliteiad info. func newTestPoliteiad(t *testing.T, p *politeiawww) *testpoliteiad.TestPoliteiad { - td := testpoliteiad.New(t, p.cache) + td := testpoliteiad.New(t) p.cfg.RPCHost = td.URL p.cfg.Identity = td.PublicIdentity return td diff --git a/politeiawww/user.go b/politeiawww/user.go index 2be72dffd..031f76bdd 100644 --- a/politeiawww/user.go +++ b/politeiawww/user.go @@ -635,83 +635,87 @@ func (p *politeiawww) processEditUser(eu *www.EditUser, user *user.User) (*www.E func (p *politeiawww) processUserCommentsLikes(user *user.User, token string) (*www.UserCommentsLikesReply, error) { log.Tracef("processUserCommentsLikes: %v %v", user.ID, token) - // Make sure token is valid and not a prefix - if !isTokenValid(token) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: []string{token}, + /* + // Make sure token is valid and not a prefix + if !isTokenValid(token) { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidCensorshipToken, + ErrorContext: []string{token}, + } } - } - // Fetch all like comments for the proposal - dlc, err := p.decredPropCommentLikes(token) - if err != nil { - return nil, fmt.Errorf("decredPropLikeComments: %v", err) - } + // Fetch all like comments for the proposal + dlc, err := p.decredPropCommentLikes(token) + if err != nil { + return nil, fmt.Errorf("decredPropLikeComments: %v", err) + } - // Sanity check. Like comments should already be sorted in - // chronological order. - sort.SliceStable(dlc, func(i, j int) bool { - return dlc[i].Timestamp < dlc[j].Timestamp - }) + // Sanity check. Like comments should already be sorted in + // chronological order. + sort.SliceStable(dlc, func(i, j int) bool { + return dlc[i].Timestamp < dlc[j].Timestamp + }) - // Find all the like comments that are from the user - lc := make([]www.LikeComment, 0, len(dlc)) - for _, v := range dlc { - u, err := p.db.UserGetByPubKey(v.PublicKey) - if err != nil { - log.Errorf("getUserCommentLikes: UserGetByPubKey: "+ - "token:%v commentID:%v pubKey:%v err:%v", v.Token, - v.CommentID, v.PublicKey, err) - continue + // Find all the like comments that are from the user + lc := make([]www.LikeComment, 0, len(dlc)) + for _, v := range dlc { + u, err := p.db.UserGetByPubKey(v.PublicKey) + if err != nil { + log.Errorf("getUserCommentLikes: UserGetByPubKey: "+ + "token:%v commentID:%v pubKey:%v err:%v", v.Token, + v.CommentID, v.PublicKey, err) + continue + } + if user.ID.String() == u.ID.String() { + lc = append(lc, convertLikeCommentFromDecred(v)) + } } - if user.ID.String() == u.ID.String() { - lc = append(lc, convertLikeCommentFromDecred(v)) + + // Compute the resulting like comment action for each comment. + // The resulting action depends on the order of the like + // comment actions. + // + // Example: when a user upvotes a comment twice, the second + // upvote cancels out the first upvote and the resulting + // comment score is 0. + // + // Example: when a user upvotes a comment and then downvotes + // the same comment, the downvote takes precedent and the + // resulting comment score is -1. + actions := make(map[string]string) // [commentID]action + for _, v := range lc { + prevAction := actions[v.CommentID] + switch { + case v.Action == prevAction: + // New action is the same as the previous action so + // we undo the previous action. + actions[v.CommentID] = "" + case v.Action != prevAction: + // New action is different than the previous action + // so the new action takes precedent. + actions[v.CommentID] = v.Action + } } - } - // Compute the resulting like comment action for each comment. - // The resulting action depends on the order of the like - // comment actions. - // - // Example: when a user upvotes a comment twice, the second - // upvote cancels out the first upvote and the resulting - // comment score is 0. - // - // Example: when a user upvotes a comment and then downvotes - // the same comment, the downvote takes precedent and the - // resulting comment score is -1. - actions := make(map[string]string) // [commentID]action - for _, v := range lc { - prevAction := actions[v.CommentID] - switch { - case v.Action == prevAction: - // New action is the same as the previous action so - // we undo the previous action. - actions[v.CommentID] = "" - case v.Action != prevAction: - // New action is different than the previous action - // so the new action takes precedent. - actions[v.CommentID] = v.Action - } - } - - cl := make([]www.CommentLike, 0, len(lc)) - for k, v := range actions { - // Skip actions that have been taken away - if v == "" { - continue + cl := make([]www.CommentLike, 0, len(lc)) + for k, v := range actions { + // Skip actions that have been taken away + if v == "" { + continue + } + cl = append(cl, www.CommentLike{ + Token: token, + CommentID: k, + Action: v, + }) } - cl = append(cl, www.CommentLike{ - Token: token, - CommentID: k, - Action: v, - }) - } - return &www.UserCommentsLikesReply{ - CommentsLikes: cl, - }, nil + return &www.UserCommentsLikesReply{ + CommentsLikes: cl, + }, nil + */ + + return nil, nil } // createLoginReply creates a login reply. diff --git a/politeiawww/www.go b/politeiawww/www.go index 7ea9b44ee..bdae4b522 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -323,7 +323,6 @@ func _main() error { userEmails: make(map[string]uuid.UUID), userPaywallPool: make(map[uuid.UUID]paywallPoolMember), commentVotes: make(map[string]counters), - voteSummaries: make(map[string]www.VoteSummary), params: activeNetParams.Params, } @@ -458,15 +457,6 @@ func _main() error { p.setPoliteiaWWWRoutes() p.setUserWWWRoutes() - // Setup dcrdata websocket subscriptions and monitoring. This is - // done in a go routine so politeiawww startup will continue in - // the event that a dcrdata websocket connection was not able to - // be made during client initialization and reconnection attempts - // are required. - go func() { - p.setupWSDcrdataPi() - }() - case cmsWWWMode: // Setup dcrdata websocket connection ws, err := wsdcrdata.New(p.dcrdataHostWS()) From f54ca5f824522b159bcc979ae0767275ea99c11d Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 28 Aug 2020 16:50:03 -0500 Subject: [PATCH 021/449] add pi api --- plugins/comments/comments.go | 3 + plugins/pi/pi.go | 107 +++++++ politeiad/backend/tlogbe/plugins/comments.go | 3 + .../backend/tlogbe/plugins/ticketvote.go | 3 + politeiad/backend/tlogbe/tlogbe.go | 2 + politeiawww/api/pi/v1/v1.go | 240 ++++++++++++++++ politeiawww/api/www/v1/v1.go | 50 ++-- politeiawww/convert.go | 14 + politeiawww/dcc.go | 3 +- politeiawww/invoices.go | 3 +- politeiawww/piwww.go | 263 ++++++++++++++++++ politeiawww/politeiawww.go | 15 +- politeiawww/proposals.go | 71 ++--- politeiawww/util/merkle.go | 11 +- util/signature.go | 2 - 15 files changed, 715 insertions(+), 75 deletions(-) create mode 100644 plugins/pi/pi.go create mode 100644 politeiawww/api/pi/v1/v1.go create mode 100644 politeiawww/piwww.go diff --git a/plugins/comments/comments.go b/plugins/comments/comments.go index 0356ddce7..ef474bbb0 100644 --- a/plugins/comments/comments.go +++ b/plugins/comments/comments.go @@ -34,6 +34,9 @@ const ( VoteDownvote VoteT = -1 VoteUpvote VoteT = 1 + // TODO should these policies be plugin settings? + // TODO PolicyMaxCommentLength + // PolicayMaxVoteChanges is the maximum number times a user can // change their vote on a comment. This prevents a malicious user // from being able to spam comment votes. diff --git a/plugins/pi/pi.go b/plugins/pi/pi.go new file mode 100644 index 000000000..37e72269c --- /dev/null +++ b/plugins/pi/pi.go @@ -0,0 +1,107 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package pi + +import "encoding/json" + +const ( + Version uint32 = 1 + ID = "pi" + + // Plugin commands + CmdLinkedFrom = "linkedfrom" // Get linked from list + + // FilenameProposalMetadata is the filename of the ProposalMetadata + // file that is saved to politeiad. ProposalMetadata is saved to + // politeiad as a file, not as a metadata stream, since it needs to + // be included in the merkle root that politeiad signs. + FilenameProposalMetadata = "proposalmetadata.json" +) + +// ProposalMetadata contains proposal metadata that is provided by the user on +// proposal submission. ProposalMetadata is saved to politeiad as a file, not +// as a metadata stream, since it needs to be included in the merkle root that +// politeiad signs. +type ProposalMetadata struct { + Name string `json:"name"` // Proposal name + + // LinkTo specifies a public proposal token to link this proposal + // to. Ex, an RFP sumbssion must link to the RFP proposal. + LinkTo string `json:"linkto,omitempty"` + + // LinkBy is a UNIX timestamp that serves as a deadline for other + // proposals to link to this proposal. Ex, an RFP submission cannot + // link to an RFP proposal once the RFP's LinkBy deadline is past. + LinkBy int64 `json:"linkby,omitempty"` +} + +// ProposalGeneral represents general proposal metadata that is saved on +// proposal submission. ProposalGeneral is saved to politeiad as a metadata +// stream. +// +// Signature is the client signature of the proposal merkle root. The merkle +// root is the ordered merkle root of all proposal Files and Metadata. +type ProposalGeneral struct { + Timestamp int64 `json:"timestamp"` // Submission UNIX timestamp + PublicKey string `json:"publickey"` // Key used for signature + Signature string `json:"signature"` // Signature of merkle root +} + +// SetStatus represents a proposal status change. +type StatusChange struct { + Status PropStatusT `json:"status"` + Message string `json:"message,omitempty"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` + Timestamp int64 `json:"timestamp"` +} + +// LinkedFrom retrieves the linked from list for each of the provided proposal +// tokens. A linked from list is a list of all the proposals that have linked +// to a given proposal using the LinkTo field in the ProposalMetadata mdstream. +// If a token does not correspond to an actual proposal then it will not be +// included in the returned map. +type LinkedFrom struct { + Tokens []string `json:"tokens"` +} + +// EncodeLinkedFrom encodes a LinkedFrom into a JSON byte slice. +func EncodeLinkedFrom(lf LinkedFrom) ([]byte, error) { + return json.Marshal(lf) +} + +// DecodeLinkedFrom decodes a JSON byte slice into a LinkedFrom. +func DecodeLinkedFrom(payload []byte) (*LinkedFrom, error) { + var lf LinkedFrom + + err := json.Unmarshal(payload, &lf) + if err != nil { + return nil, err + } + + return &lf, nil +} + +// LinkedFromReply is the reply to the LinkedFrom command. +type LinkedFromReply struct { + LinkedFrom map[string][]string `json:"linkedfrom"` +} + +// EncodeLinkedFromReply encodes a LinkedFromReply into a JSON byte slice. +func EncodeLinkedFromReply(reply LinkedFromReply) ([]byte, error) { + return json.Marshal(reply) +} + +// DecodeLinkedFromReply decodes a JSON byte slice into a LinkedFrom. +func DecodeLinkedFromReply(payload []byte) (*LinkedFromReply, error) { + var reply LinkedFromReply + + err := json.Unmarshal(payload, &reply) + if err != nil { + return nil, err + } + + return &reply, nil +} diff --git a/politeiad/backend/tlogbe/plugins/comments.go b/politeiad/backend/tlogbe/plugins/comments.go index 1b11cbc01..769e52208 100644 --- a/politeiad/backend/tlogbe/plugins/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments.go @@ -24,6 +24,9 @@ import ( "github.com/decred/politeia/util" ) +// TODO don't save data to the file system. Save it to the kv store and save +// the key to the file system. This will allow the data to be backed up. + const ( // Blob entry data descriptors dataDescriptorCommentAdd = "commentadd" diff --git a/politeiad/backend/tlogbe/plugins/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote.go index 1715271bf..d3150f45b 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote.go @@ -33,6 +33,9 @@ import ( "github.com/decred/politeia/util" ) +// TODO don't save data to the file system. Save it to the kv store and save +// the key to the file system. This will allow the data to be backed up. + const ( // ticketVoteDirname is the ticket vote data directory name. ticketVoteDirname = "ticketvote" diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 3935c3c71..5ba6262c8 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -30,6 +30,8 @@ import ( // TODO testnet vs mainnet trillian databases // TODO fsck // TODO allow token prefix lookups +// TODO we need to be able to run multiple politeiad instances. Its ok if +// we have a tree that seeds data between the instances. const ( defaultTrillianKeyFilename = "trillian.key" diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go new file mode 100644 index 000000000..af2923833 --- /dev/null +++ b/politeiawww/api/pi/v1/v1.go @@ -0,0 +1,240 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package v1 + +import ( + "fmt" +) + +type ErrorStatusT int +type PropStatusT int +type VoteStatusT int +type VoteT int + +const ( + APIVersion = 1 + + // Proposal routes + RouteProposalNew = "/proposal/new" + RouteProposalEdit = "/proposal/edit" + RouteProposalSetStatus = "/proposal/setstatus" + RouteProposalDetails = "/proposal/details" + RouteProposals = "/proposals" + RouteProposalInventory = "/proposals/inventory" + + // Comment routes + RouteCommentNew = "/comment/new" + RouteCommentVote = "/comment/vote" + RouteCommentCensor = "/comment/censor" + RouteComments = "/comments" + RouteCommentVotes = "/comments/votes" + + // Vote routes + RouteVoteAuthorize = "/vote/authorize" + RouteVoteStart = "/vote/start" + RouteVoteStartRunoff = "/vote/startrunoff" + RouteVoteBallot = "/vote/ballot" + RouteVoteDetails = "/vote/details" + RouteVoteResults = "/vote/results" + RouteVoteSummaries = "/votes/summaries" + RouteVoteInventory = "/votes/inventory" + + // Proposal status codes + PropStatusInvalid PropStatusT = 0 // Invalid status + PropStatusUnvetted PropStatusT = 1 // Prop has not been vetted + PropStatusPublic PropStatusT = 2 // Prop has been made public + PropStatusCensored PropStatusT = 3 // Prop has been censored + PropStatusAbandoned PropStatusT = 4 // Prop has been abandoned + + // Vote status codes + VoteStatusInvalid VoteStatusT = 0 // Invalid status + VoteStatusUnauthorized VoteStatusT = 1 // Vote cannot be started + VoteStatusAuthorized VoteStatusT = 2 // Vote can be started + VoteStatusStarted VoteStatusT = 3 // Vote has been started + VoteStatusFinished VoteStatusT = 4 // Vote has finished + + // Vote types + VoteTypeInvalid VoteT = 0 + + // VoteTypeStandard is used to indicate a simple approve or reject + // vote where the winner is the voting option that has met the + // specified quorum and pass requirements. + VoteTypeStandard VoteT = 1 + + // VoteTypeRunoff specifies a runoff vote that multiple records + // compete in. All records are voted on like normal, but there can + // only be one winner in a runoff vote. The winner is the record + // that meets the quorum requirement, meets the pass requirement, + // and that has the most net yes votes. The winning record is + // considered approved and all other records are considered to be + // rejected. If no records meet the quorum and pass requirements + // then all records are considered rejected. Note, in a runoff vote + // it's possible for a proposal to meet both the quorum and pass + // requirements but still be rejected if it does not have the most + // net yes votes. + VoteTypeRunoff VoteT = 2 +) + +var ( + // APIRoute is the prefix to the v2 API routes. + APIRoute = fmt.Sprintf("/v%v", APIVersion) +) + +// File describes an individual file that is part of the proposal. The +// directory structure must be flattened. +type File struct { + Name string `json:"name"` // Filename + MIME string `json:"mime"` // Mime type + Digest string `json:"digest"` // SHA256 digest of unencoded payload + Payload string `json:"payload"` // File content, base64 encoded +} + +// Metadata describes user specified proposal metadata. +type Metadata struct { + Hint string `json:"hint"` // Hint that describes the payload + Digest string `json:"digest"` // SHA256 digest of unencoded payload + Payload string `json:"payload"` // JSON metadata content, base64 encoded +} + +const ( + // Metadata hints + HintProposalMetadata = "proposalmetadata" +) + +// ProposalMetadata contains metadata that is specified by the user on proposal +// submission. It is attached to a proposal submission as a Metadata object. +type ProposalMetadata struct { + Name string `json:"name"` // Proposal name + + // LinkTo specifies a public proposal token to link this proposal + // to. Ex, an RFP sumbssion must link to the RFP proposal. + LinkTo string `json:"linkto,omitempty"` + + // LinkBy is a UNIX timestamp that serves as a deadline for other + // proposals to link to this proposal. Ex, an RFP submission cannot + // link to an RFP proposal once the RFP's LinkBy deadline is past. + LinkBy int64 `json:"linkby,omitempty"` +} + +// CensorshipRecord contains the proof that a proposal was accepted for review +// by the server. The proof is verifiable on the client side. +type CensorshipRecord struct { + // Token is a random censorship token that is generated by the + // server. It serves as a unique identifier for the proposal. + Token string `json:"token"` + + // Merkle is the ordered merkle root of all files and metadata in + // in the proposal. + Merkle string `json:"merkle"` + + // Signature is the server signature of the Merkle+Token. + Signature string `json:"signature"` +} + +// SetStatus represents a proposal status change. +type StatusChange struct { + Status PropStatusT `json:"status"` + Message string `json:"message,omitempty"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` + Timestamp int64 `json:"timestamp"` +} + +// ProposalRecord is an entire proposal and it's contents. +// +// Signature is the client signature of the proposal merkle root. The merkle +// root is the ordered merkle root of all proposal Files and Metadata. +type ProposalRecord struct { + Version string `json:"version"` // Proposal version + Timestamp int64 `json:"timestamp"` // Submission UNIX timestamp + Status PropStatusT `json:"status"` // Proposal status + UserID string `json:"userid"` // Author ID + Username string `json:"username"` // Author username + PublicKey string `json:"publickey"` // Key used in signature + Signature string `json:"signature"` // Signature of merkle root + Statuses []StatusChange `json:"statuses"` // Status change history + Files []File `json:"files"` // Proposal files + Metadata []Metadata `json:"metadata"` // User defined metadata + + CensorshipRecord CensorshipRecord `json:"censorshiprecord"` +} + +// ProposalNew submits a new proposal. +// +// Metadata must contain a ProposalMetadata object. +// +// Signature is the client signature of the proposal merkle root. The merkle +// root is the ordered merkle root of all proposal Files and Metadata. +type ProposalNew struct { + Files []File `json:"files"` + Metadata []Metadata `json:"metadata"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` +} + +// ProposalNewReply is the reply to the ProposalNew command. +type ProposalNewReply struct { + CensorshipRecord CensorshipRecord `json:"censorshiprecord"` +} + +// ProposalEdit edits an existing proposal. +// +// Metadata must contain a ProposalMetadata object. +// +// Signature is the client signature of the proposal merkle root. The merkle +// root is the ordered merkle root of all proposal Files and Metadata. +type ProposalEdit struct { + Token string `json:"token"` + Files []File `json:"files"` + Metadata []Metadata `json:"metadata"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` +} + +// ProposalEditReply is the reply to the ProposalEdit command. +type ProposalEditReply struct { + CensorshipRecord CensorshipRecord `json:"censorshiprecord"` +} + +// ProposalSetStatus sets the status of a proposal. Some status changes require +// a reason to be included. +// +// Signature is the client signature of the Token+Version+Status+Reason. +type ProposalSetStatus struct { + Token string `json:"token"` // Censorship token + Version string `json:"version"` // Proposal version + Status PropStatusT `json:"status"` // New status + Reason string `json:"reason,omitempty"` // Reason for status change + PublicKey string `json:"publickey"` // Key used for signature + Signature string `json:"signature"` // Client signature +} + +// ProposalSetStatusReply is the reply to the ProposalSetStatus command. +type ProposalSetStatusReply struct{} + +// ProposalDetails retrieves a proposal record. If a version is not specified, +// the latest version will be returned. +type ProposalDetails struct { + Token string `json:"token"` + Version string `json:"version,omitempty"` +} + +// ProposalDetailsReply is the reply to the ProposalDetails command. +type ProposalDetailsReply struct { + Proposal ProposalRecord `json:"proposal"` +} + +// ProposalInventry retrieves the tokens of all proposals in the inventory, +// catagorized by proposal status and ordered by timestamp of the status change +// from newest to oldest. +type ProposalInventory struct{} + +// ProposalInventoryReply is the reply to the ProposalInventory command. +type ProposalInventoryReply struct { + Unvetted []string `json:"unvetted"` + Public []string `json:"public"` + Censored []string `json:"censored"` + Abandoned []string `json:"abandoned"` +} diff --git a/politeiawww/api/www/v1/v1.go b/politeiawww/api/www/v1/v1.go index 6adadebeb..cbe1e659f 100644 --- a/politeiawww/api/www/v1/v1.go +++ b/politeiawww/api/www/v1/v1.go @@ -41,9 +41,7 @@ const ( RouteChangePassword = "/user/password/change" RouteResetPassword = "/user/password/reset" RouteVerifyResetPassword = "/user/password/reset/verify" - RouteUserProposals = "/user/proposals" RouteUserProposalCredits = "/user/proposals/credits" - RouteUserCommentsLikes = "/user/proposals/{token:[A-z0-9]{64}}/commentslikes" RouteVerifyUserPayment = "/user/verifypayment" RouteUserPaymentsRescan = "/user/payments/rescan" RouteManageUser = "/user/manage" @@ -51,30 +49,36 @@ const ( RouteSetTOTP = "/user/totp" RouteVerifyTOTP = "/user/verifytotp" RouteUsers = "/users" - RouteTokenInventory = "/proposals/tokeninventory" - RouteBatchProposals = "/proposals/batch" - RouteBatchVoteSummary = "/proposals/batchvotesummary" - RouteAllVetted = "/proposals/vetted" - RouteNewProposal = "/proposals/new" - RouteEditProposal = "/proposals/edit" - RouteAuthorizeVote = "/proposals/authorizevote" - RouteStartVote = "/proposals/startvote" - RouteActiveVote = "/proposals/activevote" // XXX rename to ActiveVotes - RouteCastVotes = "/proposals/castvotes" - RouteAllVoteStatus = "/proposals/votestatus" - RouteProposalPaywallDetails = "/proposals/paywall" - RouteProposalPaywallPayment = "/proposals/paywallpayment" - RouteProposalDetails = "/proposals/{token:[A-Fa-f0-9]{7,64}}" - RouteSetProposalStatus = "/proposals/{token:[A-Fa-f0-9]{7,64}}/status" - RouteCommentsGet = "/proposals/{token:[A-Fa-f0-9]{7,64}}/comments" - RouteVoteResults = "/proposals/{token:[A-Fa-f0-9]{7,64}}/votes" - RouteVoteStatus = "/proposals/{token:[A-Fa-f0-9]{7,64}}/votestatus" - RouteNewComment = "/comments/new" - RouteLikeComment = "/comments/like" - RouteCensorComment = "/comments/censor" RouteUnauthenticatedWebSocket = "/ws" RouteAuthenticatedWebSocket = "/aws" + // XXX these routes should be user routes + RouteProposalPaywallDetails = "/proposals/paywall" + RouteProposalPaywallPayment = "/proposals/paywallpayment" + + // The following routes have been DEPRECATED. Use the v2 routes. + RouteTokenInventory = "/proposals/tokeninventory" + RouteBatchProposals = "/proposals/batch" + RouteBatchVoteSummary = "/proposals/batchvotesummary" + RouteAllVetted = "/proposals/vetted" + RouteNewProposal = "/proposals/new" + RouteEditProposal = "/proposals/edit" + RouteAuthorizeVote = "/proposals/authorizevote" + RouteStartVote = "/proposals/startvote" + RouteActiveVote = "/proposals/activevote" + RouteCastVotes = "/proposals/castvotes" + RouteAllVoteStatus = "/proposals/votestatus" + RouteProposalDetails = "/proposals/{token:[A-Fa-f0-9]{7,64}}" + RouteSetProposalStatus = "/proposals/{token:[A-Fa-f0-9]{7,64}}/status" + RouteCommentsGet = "/proposals/{token:[A-Fa-f0-9]{7,64}}/comments" + RouteVoteResults = "/proposals/{token:[A-Fa-f0-9]{7,64}}/votes" + RouteVoteStatus = "/proposals/{token:[A-Fa-f0-9]{7,64}}/votestatus" + RouteNewComment = "/comments/new" + RouteLikeComment = "/comments/like" + RouteCensorComment = "/comments/censor" + RouteUserCommentsLikes = "/user/proposals/{token:[A-z0-9]{64}}/commentslikes" + RouteUserProposals = "/user/proposals" + // VerificationTokenSize is the size of verification token in bytes VerificationTokenSize = 32 diff --git a/politeiawww/convert.go b/politeiawww/convert.go index bec685d59..c3153ca4e 100644 --- a/politeiawww/convert.go +++ b/politeiawww/convert.go @@ -17,6 +17,7 @@ import ( "github.com/decred/politeia/mdstream" pd "github.com/decred/politeia/politeiad/api/v1" cms "github.com/decred/politeia/politeiawww/api/cms/v1" + pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" www2 "github.com/decred/politeia/politeiawww/api/www/v2" "github.com/decred/politeia/politeiawww/cmsdatabase" @@ -1134,3 +1135,16 @@ func filterDomainInvoice(inv *cms.InvoiceRecord) cms.InvoiceRecord { return *inv } + +func convertPiFilesFromWWW(files []www.File) []pi.File { + f := make([]pi.File, 0, len(files)) + for _, v := range files { + f = append(f, pi.File{ + Name: v.Name, + MIME: v.MIME, + Digest: v.Digest, + Payload: v.Payload, + }) + } + return f +} diff --git a/politeiawww/dcc.go b/politeiawww/dcc.go index 44ab4e54b..d72aee695 100644 --- a/politeiawww/dcc.go +++ b/politeiawww/dcc.go @@ -366,7 +366,8 @@ func (p *politeiawww) validateDCC(nd cms.NewDCC, u *user.User) error { } // Note that we need validate the string representation of the merkle - mr, err := wwwutil.MerkleRoot([]www.File{nd.File}, nil) + files := convertPiFilesFromWWW([]www.File{nd.File}) + mr, err := wwwutil.MerkleRoot(files, nil) if err != nil { return err } diff --git a/politeiawww/invoices.go b/politeiawww/invoices.go index 79b500cc9..38f2c42c5 100644 --- a/politeiawww/invoices.go +++ b/politeiawww/invoices.go @@ -791,7 +791,8 @@ func (p *politeiawww) validateInvoice(ni cms.NewInvoice, u *user.CMSUser) error } // Note that we need validate the string representation of the merkle - mr, err := wwwutil.MerkleRoot(ni.Files, nil) + files := convertPiFilesFromWWW(ni.Files) + mr, err := wwwutil.MerkleRoot(files, nil) if err != nil { return err } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go new file mode 100644 index 000000000..418d9365f --- /dev/null +++ b/politeiawww/piwww.go @@ -0,0 +1,263 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "encoding/base64" + "errors" + "fmt" + "mime" + "net/http" + + pi "github.com/decred/politeia/politeiawww/api/pi/v1" + www "github.com/decred/politeia/politeiawww/api/www/v1" + "github.com/decred/politeia/politeiawww/user" + wwwutil "github.com/decred/politeia/politeiawww/util" + "github.com/decred/politeia/util" +) + +// TODO use pi errors instead of www errors + +func convertUserErrFromSignatureErr(err error) www.UserError { + var e util.SignatureError + var s www.ErrorStatusT + if errors.As(err, &e) { + switch e.ErrorCode { + case util.ErrorStatusPublicKeyInvalid: + s = www.ErrorStatusInvalidPublicKey + case util.ErrorStatusSignatureInvalid: + s = www.ErrorStatusInvalidSignature + } + } + return www.UserError{ + ErrorCode: s, + ErrorContext: e.ErrorContext, + } +} + +func verifyProposal(files []pi.File, metadata []pi.Metadata, publicKey, signature string) error { + if len(files) == 0 { + return www.UserError{ + ErrorCode: www.ErrorStatusProposalMissingFiles, + ErrorContext: []string{"no files found"}, + } + } + + // Verify the files adhere to all policy requirements + var ( + countTextFiles int + countImageFiles int + foundIndexFile bool + ) + filenames := make(map[string]struct{}, len(files)) + for _, v := range files { + // Validate file name + _, ok := filenames[v.Name] + if ok { + return www.UserError{ + ErrorCode: www.ErrorStatusProposalDuplicateFilenames, + ErrorContext: []string{v.Name}, + } + } + filenames[v.Name] = struct{}{} + + // Validate file payload + if v.Payload == "" { + e := fmt.Sprintf("base64 payload is empty for file '%v'", + v.Name) + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidBase64, + ErrorContext: []string{e}, + } + } + payloadb, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidBase64, + ErrorContext: []string{v.Name}, + } + } + + // Verify computed file digest matches given file digest + digest := util.Digest(payloadb) + d, ok := util.ConvertDigest(v.Digest) + if !ok { + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidFileDigest, + ErrorContext: []string{v.Name}, + } + } + if !bytes.Equal(digest, d[:]) { + e := fmt.Sprintf("computed digest does not match given digest "+ + "for file '%v'", v.Name) + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidFileDigest, + ErrorContext: []string{e}, + } + } + + // Verify detected MIME type matches given mime type + ct := http.DetectContentType(payloadb) + mimePayload, _, err := mime.ParseMediaType(ct) + if err != nil { + return err + } + mimeFile, _, err := mime.ParseMediaType(v.MIME) + if err != nil { + log.Debugf("validateProposal: ParseMediaType(%v): %v", + v.MIME, err) + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidMIMEType, + ErrorContext: []string{v.Name}, + } + } + if mimeFile != mimePayload { + e := fmt.Sprintf("detected mime '%v' does not match '%v' for file '%v'", + mimePayload, mimeFile, v.Name) + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidMIMEType, + ErrorContext: []string{e}, + } + } + + // Run MIME type specific validation + switch mimeFile { + case mimeTypeText: + countTextFiles++ + + // Verify text file size + if len(payloadb) > www.PolicyMaxMDSize { + e := fmt.Sprintf("file size %v exceeds max %v for file '%v'", + len(payloadb), www.PolicyMaxMDSize, v.Name) + return www.UserError{ + ErrorCode: www.ErrorStatusMaxMDSizeExceededPolicy, + ErrorContext: []string{e}, + } + } + + // The only text file that is allowed is the index markdown + // file. + if v.Name != www.PolicyIndexFilename { + return www.UserError{ + ErrorCode: www.ErrorStatusMaxMDsExceededPolicy, + ErrorContext: []string{v.Name}, + } + } + if foundIndexFile { + e := fmt.Sprintf("more than one %v file found", + www.PolicyIndexFilename) + return www.UserError{ + ErrorCode: www.ErrorStatusMaxMDsExceededPolicy, + ErrorContext: []string{e}, + } + } + + // Set index file as being found + foundIndexFile = true + + case mimeTypePNG: + countImageFiles++ + + // Verify image file size + if len(payloadb) > www.PolicyMaxImageSize { + e := fmt.Sprintf("file size %v exceeds max %v for file '%v'", + len(payloadb), www.PolicyMaxImageSize, v.Name) + return www.UserError{ + ErrorCode: www.ErrorStatusMaxImageSizeExceededPolicy, + ErrorContext: []string{e}, + } + } + + default: + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidMIMEType, + ErrorContext: []string{v.MIME}, + } + } + } + + // Verify that an index file is present. + if !foundIndexFile { + e := fmt.Sprintf("%v file not found", www.PolicyIndexFilename) + return www.UserError{ + ErrorCode: www.ErrorStatusProposalMissingFiles, + ErrorContext: []string{e}, + } + } + + // Verify file counts are acceptable + if countTextFiles > www.PolicyMaxMDs { + e := fmt.Sprintf("got %v text files; max is %v", + countTextFiles, www.PolicyMaxMDs) + return www.UserError{ + ErrorCode: www.ErrorStatusMaxMDsExceededPolicy, + ErrorContext: []string{e}, + } + } + if countImageFiles > www.PolicyMaxImages { + e := fmt.Sprintf("got %v image files, max is %v", + countImageFiles, www.PolicyMaxImages) + return www.UserError{ + ErrorCode: www.ErrorStatusMaxImagesExceededPolicy, + ErrorContext: []string{e}, + } + } + + // Verify signature + mr, err := wwwutil.MerkleRoot(files, metadata) + if err != nil { + return fmt.Errorf("MerkleRoot: %v", err) + } + err = util.VerifySignature(signature, publicKey, mr) + if err != nil { + return convertUserErrFromSignatureErr(err) + } + + return nil +} + +func (p *politeiawww) processProposalNew(pn pi.ProposalNew, usr user.User) (*pi.ProposalNewReply, error) { + log.Tracef("processProposalNew: %v", usr.Username) + + // Verify user has paid the registration paywall + if !p.HasUserPaid(&usr) { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusUserNotPaid, + } + } + + // Verify user has a proposal credit to spend + if !p.UserHasProposalCredits(&usr) { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusNoProposalCredits, + } + } + + // Verify the user signed using their active identitiy + if usr.PublicKey() != pn.PublicKey { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidSigningKey, + } + } + + // Verify proposal + err := verifyProposal(pn.Files, pn.Metadata, pn.PublicKey, pn.Signature) + if err != nil { + return nil, err + } + + // Setup metadata stream + + // Send politeiad request + + // Handle response + + // Deduct proposal credit from author's account + + // Fire off a new proposal event + + return nil, nil +} diff --git a/politeiawww/politeiawww.go b/politeiawww/politeiawww.go index b8151cb12..45bd72688 100644 --- a/politeiawww/politeiawww.go +++ b/politeiawww/politeiawww.go @@ -1188,17 +1188,6 @@ func (p *politeiawww) handleVerifyTOTP(w http.ResponseWriter, r *http.Request) { // setPoliteiaWWWRoutes sets up the politeia routes. func (p *politeiawww) setPoliteiaWWWRoutes() { - // Templates - //p.addTemplate(templateNewProposalSubmittedName, - // templateNewProposalSubmittedRaw) - - // Static content. - // XXX disable static for now. This code is broken and it needs to - // point to a sane directory. If a directory is not set it SHALL be - // disabled. - //p.router.PathPrefix("/static/").Handler(http.StripPrefix("/static/", - // http.FileServer(http.Dir(".")))) - // Public routes. p.router.HandleFunc("/", closeBody(logging(p.handleVersion))).Methods(http.MethodGet) p.router.NotFoundHandler = closeBody(p.handleNotFound) @@ -1258,10 +1247,10 @@ func (p *politeiawww) setPoliteiaWWWRoutes() { permissionLogin) p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, www.RouteNewComment, p.handleNewComment, - permissionLogin) // XXX comments need to become a setting + permissionLogin) p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, www.RouteLikeComment, p.handleLikeComment, - permissionLogin) // XXX comments need to become a setting + permissionLogin) p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, www.RouteEditProposal, p.handleEditProposal, permissionLogin) diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 879af2edf..8264800d6 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -20,11 +20,9 @@ import ( "github.com/decred/politeia/decredplugin" "github.com/decred/politeia/mdstream" pd "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/politeiad/api/v1/identity" www "github.com/decred/politeia/politeiawww/api/www/v1" www2 "github.com/decred/politeia/politeiawww/api/www/v2" "github.com/decred/politeia/politeiawww/user" - wwwutil "github.com/decred/politeia/politeiawww/util" "github.com/decred/politeia/util" ) @@ -544,25 +542,28 @@ func (p *politeiawww) validateProposal(np www.NewProposal, u *user.User) (*www.P } // Verify signature. The signature message is the merkle root // of the proposal files. - sig, err := util.ConvertSignature(np.Signature) - if err != nil { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidSignature, + /* + // TODO + sig, err := util.ConvertSignature(np.Signature) + if err != nil { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidSignature, + } } - } - pk, err := identity.PublicIdentityFromBytes(u.ActiveIdentity().Key[:]) - if err != nil { - return nil, err - } - mr, err := wwwutil.MerkleRoot(np.Files, np.Metadata) - if err != nil { - return nil, err - } - if !pk.VerifyMessage([]byte(mr), sig) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidSignature, + pk, err := identity.PublicIdentityFromBytes(u.ActiveIdentity().Key[:]) + if err != nil { + return nil, err } - } + mr, err := wwwutil.MerkleRoot(np.Files, np.Metadata) + if err != nil { + return nil, err + } + if !pk.VerifyMessage([]byte(mr), sig) { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidSignature, + } + } + */ // Verify the user signed using their active identity if u.PublicKey() != np.PublicKey { @@ -1757,22 +1758,26 @@ func (p *politeiawww) processEditProposal(ep www.EditProposal, u *user.User) (*w // Check if there were changes in the proposal by comparing // their merkle roots. This captures changes that were made // to either the files or the metadata. - mr, err := wwwutil.MerkleRoot(ep.Files, ep.Metadata) - if err != nil { - return nil, err - } - if cachedProp.CensorshipRecord.Merkle == mr { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusNoProposalChanges, + _ = pm + /* + // TODO + mr, err := wwwutil.MerkleRoot(ep.Files, ep.Metadata) + if err != nil { + return nil, err } - } - if cachedProp.State == www.PropStateVetted && - cachedProp.LinkTo != pm.LinkTo { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkTo, - ErrorContext: []string{"linkto cannot change once public"}, + if cachedProp.CensorshipRecord.Merkle == mr { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusNoProposalChanges, + } } - } + if cachedProp.State == www.PropStateVetted && + cachedProp.LinkTo != pm.LinkTo { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidLinkTo, + ErrorContext: []string{"linkto cannot change once public"}, + } + } + */ // politeiad only includes files in its merkle root calc, not the // metadata streams. This is why we include the ProposalMetadata diff --git a/politeiawww/util/merkle.go b/politeiawww/util/merkle.go index bf563cfce..b55dc36c7 100644 --- a/politeiawww/util/merkle.go +++ b/politeiawww/util/merkle.go @@ -1,3 +1,7 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + package util import ( @@ -6,14 +10,15 @@ import ( "encoding/hex" "github.com/decred/dcrtime/merkle" - v1 "github.com/decred/politeia/politeiawww/api/www/v1" + pi "github.com/decred/politeia/politeiawww/api/pi/v1" "github.com/decred/politeia/util" ) // MerkleRoot converts the passed in list of files and metadata into SHA256 // digests then calculates and returns the merkle root of the digests. -func MerkleRoot(files []v1.File, md []v1.Metadata) (string, error) { +func MerkleRoot(files []pi.File, md []pi.Metadata) (string, error) { digests := make([]*[sha256.Size]byte, 0, len(files)) + // Calculate file digests for _, f := range files { b, err := base64.StdEncoding.DecodeString(f.Payload) @@ -25,6 +30,7 @@ func MerkleRoot(files []v1.File, md []v1.Metadata) (string, error) { copy(hf[:], digest) digests = append(digests, &hf) } + // Calculate metadata digests for _, v := range md { b, err := base64.StdEncoding.DecodeString(v.Payload) @@ -36,6 +42,7 @@ func MerkleRoot(files []v1.File, md []v1.Metadata) (string, error) { copy(hv[:], digest) digests = append(digests, &hv) } + // Return merkle root return hex.EncodeToString(merkle.Root(digests)[:]), nil } diff --git a/util/signature.go b/util/signature.go index 70002d0b4..0357b2478 100644 --- a/util/signature.go +++ b/util/signature.go @@ -11,8 +11,6 @@ import ( "github.com/decred/politeia/politeiad/api/v1/identity" ) -// TODO this doesn't need to be in util since all signature validation is -// done in the plugins type ErrorStatusT int const ( From ca995464af131da18ec07e961f6cc35f3ebb4b81 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 30 Aug 2020 13:56:37 -0500 Subject: [PATCH 022/449] refactor event manager --- plugins/pi/pi.go | 51 ++++++++----- politeiawww/email.go | 38 ++++------ politeiawww/eventmanager.go | 139 ++++++++++++++++++++++++++++++++++++ politeiawww/events.go | 95 +++++------------------- politeiawww/paywall.go | 4 +- politeiawww/piwww.go | 122 +++++++++++++++++++++++++++++-- politeiawww/politeiawww.go | 6 +- politeiawww/proposals.go | 40 ++++------- politeiawww/smtp.go | 19 +++++ politeiawww/templates.go | 13 ++-- politeiawww/testing.go | 3 +- politeiawww/www.go | 29 ++++---- 12 files changed, 379 insertions(+), 180 deletions(-) create mode 100644 politeiawww/eventmanager.go diff --git a/plugins/pi/pi.go b/plugins/pi/pi.go index 37e72269c..55dcd8327 100644 --- a/plugins/pi/pi.go +++ b/plugins/pi/pi.go @@ -13,11 +13,16 @@ const ( // Plugin commands CmdLinkedFrom = "linkedfrom" // Get linked from list + // Metadata stream IDs. All metadata streams in this plugin will + // use 1xx numbering. + MDStreamIDProposalGeneral = 101 + MDStreamIDStatusChange = 102 + // FilenameProposalMetadata is the filename of the ProposalMetadata // file that is saved to politeiad. ProposalMetadata is saved to // politeiad as a file, not as a metadata stream, since it needs to // be included in the merkle root that politeiad signs. - FilenameProposalMetadata = "proposalmetadata.json" + FilenameProposalMetadata = "proposalmd.json" ) // ProposalMetadata contains proposal metadata that is provided by the user on @@ -44,18 +49,34 @@ type ProposalMetadata struct { // Signature is the client signature of the proposal merkle root. The merkle // root is the ordered merkle root of all proposal Files and Metadata. type ProposalGeneral struct { - Timestamp int64 `json:"timestamp"` // Submission UNIX timestamp PublicKey string `json:"publickey"` // Key used for signature Signature string `json:"signature"` // Signature of merkle root + Timestamp int64 `json:"timestamp"` // Submission UNIX timestamp +} + +func EncodeProposalGeneral(pg ProposalGeneral) ([]byte, error) { + return json.Marshal(pg) } -// SetStatus represents a proposal status change. +func DecodeProposalGeneral(payload []byte) (*ProposalGeneral, error) { + var pg ProposalGeneral + err := json.Unmarshal(payload, &pg) + if err != nil { + return nil, err + } + return &pg, nil +} + +// StatusChange represents a proposal status change. +// +// Signature is the client signature of the Token+Version+Status+Reason. type StatusChange struct { - Status PropStatusT `json:"status"` - Message string `json:"message,omitempty"` - PublicKey string `json:"publickey"` - Signature string `json:"signature"` - Timestamp int64 `json:"timestamp"` + // Status PropStatusT `json:"status"` + Version string `json:"version"` + Message string `json:"message,omitempty"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` + Timestamp int64 `json:"timestamp"` } // LinkedFrom retrieves the linked from list for each of the provided proposal @@ -75,12 +96,10 @@ func EncodeLinkedFrom(lf LinkedFrom) ([]byte, error) { // DecodeLinkedFrom decodes a JSON byte slice into a LinkedFrom. func DecodeLinkedFrom(payload []byte) (*LinkedFrom, error) { var lf LinkedFrom - err := json.Unmarshal(payload, &lf) if err != nil { return nil, err } - return &lf, nil } @@ -90,18 +109,16 @@ type LinkedFromReply struct { } // EncodeLinkedFromReply encodes a LinkedFromReply into a JSON byte slice. -func EncodeLinkedFromReply(reply LinkedFromReply) ([]byte, error) { - return json.Marshal(reply) +func EncodeLinkedFromReply(lfr LinkedFromReply) ([]byte, error) { + return json.Marshal(lfr) } // DecodeLinkedFromReply decodes a JSON byte slice into a LinkedFrom. func DecodeLinkedFromReply(payload []byte) (*LinkedFromReply, error) { - var reply LinkedFromReply - - err := json.Unmarshal(payload, &reply) + var lfr LinkedFromReply + err := json.Unmarshal(payload, &lfr) if err != nil { return nil, err } - - return &reply, nil + return &lfr, nil } diff --git a/politeiawww/email.go b/politeiawww/email.go index bfc6e0483..57a5f2650 100644 --- a/politeiawww/email.go +++ b/politeiawww/email.go @@ -8,6 +8,7 @@ import ( "bytes" "fmt" "net/url" + "strings" "text/template" "time" @@ -18,9 +19,10 @@ import ( ) const ( - // This refers to a route for the GUI registration page. It is used to fill - // in email messages giving the direct URL to the page for users to follow. - RegisterNewUserGuiRoute = "/register" + // GUI routes. These are used in notification emails to direct the + // user to the correct GUI pages. + guiRouteProposalDetails = "/proposals/{token}" + guiRouteRegisterNewUser = "/register" ) func createBody(tpl *template.Template, tplData interface{}) (string, error) { @@ -348,40 +350,26 @@ func (p *politeiawww) emailUsersForProposalVoteStarted(proposal *www.ProposalRec }) } -func (p *politeiawww) emailAdminsForNewSubmittedProposal(token string, propName string, username string, userEmail string) error { - if p.smtp.disabled { - return nil - } - - l, err := url.Parse(p.cfg.WebServerAddress + "/proposals/" + token) +func (p *politeiawww) emailProposalSubmitted(token, name, username string, emails []string) error { + route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) + l, err := url.Parse(p.cfg.WebServerAddress + route) if err != nil { return err } - tplData := newProposalSubmittedTemplateData{ + tplData := proposalSubmittedTemplateData{ Link: l.String(), - Name: propName, + Name: name, Username: username, - Email: userEmail, } subject := "New Proposal Submitted" - body, err := createBody(templateNewProposalSubmitted, &tplData) + body, err := createBody(templateProposalSubmitted, &tplData) if err != nil { return err } - return p.smtp.sendEmail(subject, body, func(msg *goemail.Message) error { - // Add admin emails to the goemail.Message - return p.db.AllUsers(func(u *user.User) { - if !u.Admin || u.Deactivated || - (u.EmailNotifications& - uint64(www.NotificationEmailAdminProposalNew) == 0) { - return - } - msg.AddBCC(u.Email) - }) - }) + return p.smtp.sendEmailTo(subject, body, emails) } func (p *politeiawww) emailAdminsForProposalVoteAuthorized(token, title, authorUsername, authorEmail string) error { @@ -579,7 +567,7 @@ func (p *politeiawww) emailInviteNewUserVerificationLink(email, token string) er return nil } - link, err := p.createEmailLink(RegisterNewUserGuiRoute, "", token, "") + link, err := p.createEmailLink(guiRouteRegisterNewUser, "", token, "") if err != nil { return err } diff --git a/politeiawww/eventmanager.go b/politeiawww/eventmanager.go new file mode 100644 index 000000000..ae900540e --- /dev/null +++ b/politeiawww/eventmanager.go @@ -0,0 +1,139 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "sync" + + www "github.com/decred/politeia/politeiawww/api/www/v1" + "github.com/decred/politeia/politeiawww/user" +) + +type eventT int + +const ( + // Event types + eventTypeInvalid eventT = iota + + // Pi events + eventProposalSubmitted +) + +// eventManager manages event listeners for different event types. +type eventManager struct { + sync.Mutex + listeners map[eventT][]chan interface{} +} + +// register adds adds a listener for the given event type. +func (e *eventManager) register(event eventT, listener chan interface{}) { + e.Lock() + defer e.Unlock() + + l, ok := e.listeners[event] + if !ok { + l = make([]chan interface{}, 0) + } + + l = append(l, listener) + e.listeners[event] = l +} + +// fire fires off an event by passing it to all channels that have been +// registered to listen for the event. +func (e *eventManager) fire(event eventT, data interface{}) { + e.Lock() + defer e.Unlock() + + listeners, ok := e.listeners[event] + if !ok { + log.Errorf("fireEvent: unregistered event %v", event) + return + } + + for _, ch := range listeners { + ch <- data + } +} + +// newEventManager returns a new eventManager context. +func newEventManager() *eventManager { + return &eventManager{ + listeners: make(map[eventT][]chan interface{}), + } +} + +func (p *politeiawww) setupEventListenersPi() { + // Setup process for each event: + // 1. Create a channel for the event + // 2. Register the channel with the event manager + // 3. Launch an event handler to listen for new events + + // Setup proposal submitted event + ch := make(chan interface{}) + p.eventManager.register(eventProposalSubmitted, ch) + go p.handleEventProposalSubmitted(ch) +} + +// notificationIsSet returns whether the provided user has the provided +// notification bit set. +func notificationIsSet(u user.User, n www.EmailNotificationT) bool { + // Notifications should not be sent to deactivated users + if u.Deactivated { + return false + } + + bit := uint64(n) + if u.EmailNotifications&bit == 0 { + // Notification bit not set + return false + } + + // Notification bit is set + return true +} + +type dataProposalSubmitted struct { + token string // Proposal token + name string // Proposal name + username string // Author username +} + +func (p *politeiawww) handleEventProposalSubmitted(ch chan interface{}) { + for msg := range ch { + d, ok := msg.(dataProposalSubmitted) + if !ok { + log.Errorf("handleEventProposalSubmitted invalid msg: %v", msg) + continue + } + + // Compile email notification recipients + emails := make([]string, 0, 256) + err := p.db.AllUsers(func(u *user.User) { + // Proposal submitted notifications should only be sent to + // admins. + if !u.Admin { + return + } + + // Check if notification bit is set + if notificationIsSet(*u, www.NotificationEmailAdminProposalNew) { + emails = append(emails, u.Email) + } + }) + if err != nil { + log.Errorf("handleEventProposalSubmitted: AllUsers: %v", err) + continue + } + + // Send email notification + err = p.emailProposalSubmitted(d.token, d.name, d.username, emails) + if err != nil { + log.Errorf("emailProposalSubmitted %v: %v", err) + } + + log.Debugf("Proposal submitted notification sent") + } +} diff --git a/politeiawww/events.go b/politeiawww/events.go index c77d15c11..a60f4ac03 100644 --- a/politeiawww/events.go +++ b/politeiawww/events.go @@ -14,27 +14,20 @@ import ( "github.com/decred/politeia/politeiawww/user" ) -// TODO clean this up - -// EventT is the type of event. -type EventT int - -// EventManager manages listeners (channels) for different event types. -type EventManager struct { - Listeners map[EventT][]chan interface{} -} - const ( - EventTypeInvalid EventT = iota - EventTypeComment + // XXX these events need to be moved over to eventmanager.go and + // the handlers need to be refactored to conform to the style in + // eventmanager.go. + + // Event types + EventTypeComment eventT = iota + 10 EventTypeUserManage // Pi events - EventTypeProposalSubmitted EventTypeProposalStatusChange EventTypeProposalEdited - EventTypeProposalVoteStarted EventTypeProposalVoteAuthorized + EventTypeProposalVoteStarted // CMS events EventTypeInvoiceComment // CMS Type @@ -50,12 +43,6 @@ type eventDataProposalVoteAuthorized struct { authorEmail string } -type EventDataProposalSubmitted struct { - CensorshipRecord *www.CensorshipRecord - ProposalName string - User *user.User -} - type EventDataProposalStatusChange struct { Proposal *www.ProposalRecord SetProposalStatus *www.SetProposalStatus @@ -124,7 +111,7 @@ func (p *politeiawww) getProposalAndAuthor(token string) (*www.ProposalRecord, * // holds the lock. // // This function must be called WITHOUT the mutex held. -func (p *politeiawww) fireEvent(eventType EventT, data interface{}) { +func (p *politeiawww) fireEvent(eventType eventT, data interface{}) { if p.test { return } @@ -135,21 +122,15 @@ func (p *politeiawww) fireEvent(eventType EventT, data interface{}) { p.eventManager._fireEvent(eventType, data) } -func (p *politeiawww) initEventManager() { +func (p *politeiawww) initEventManagerPi() { + p.Lock() defer p.Unlock() - p.eventManager = &EventManager{} - p._setupProposalStatusChangeLogging() p._setupProposalVoteStartedLogging() p._setupUserManageLogging() - if p.smtp.disabled { - return - } - - p._setupProposalSubmittedEmailNotification() p._setupProposalStatusChangeEmailNotification() p._setupProposalEditedEmailNotification() p._setupProposalVoteStartedEmailNotification() @@ -161,8 +142,6 @@ func (p *politeiawww) initCMSEventManager() { p.Lock() defer p.Unlock() - p.eventManager = &EventManager{} - if p.smtp.disabled { return } @@ -255,28 +234,6 @@ func (p *politeiawww) _setupDCCSupportOpposeEmailNotification() { p.eventManager._register(EventTypeDCCSupportOppose, ch) } -func (p *politeiawww) _setupProposalSubmittedEmailNotification() { - ch := make(chan interface{}) - go func() { - for data := range ch { - ps, ok := data.(EventDataProposalSubmitted) - if !ok { - log.Errorf("invalid event data") - continue - } - - err := p.emailAdminsForNewSubmittedProposal( - ps.CensorshipRecord.Token, ps.ProposalName, - ps.User.Username, ps.User.Email) - if err != nil { - log.Errorf("email all admins for new submitted proposal %v: %v", - ps.CensorshipRecord.Token, err) - } - } - }() - p.eventManager._register(EventTypeProposalSubmitted, ch) -} - func (p *politeiawww) _setupProposalStatusChangeEmailNotification() { ch := make(chan interface{}) go func() { @@ -525,41 +482,23 @@ func (p *politeiawww) _setupUserManageLogging() { p.eventManager._register(EventTypeUserManage, ch) } -// _register adds a listener channel for the given event type. +// register adds a listener channel for the given event type. // // This function must be called WITH the mutex held. -func (e *EventManager) _register(eventType EventT, listenerToAdd chan interface{}) { - if e.Listeners == nil { - e.Listeners = make(map[EventT][]chan interface{}) +func (e *eventManager) _register(eventType eventT, listenerToAdd chan interface{}) { + if e.listeners == nil { + e.listeners = make(map[eventT][]chan interface{}) } - e.Listeners[eventType] = append(e.Listeners[eventType], listenerToAdd) -} - -// _unregister removes the given listener channel for the given event type. -// -// This function must be called WITH the mutex held. -func (e *EventManager) _unregister(eventType EventT, listenerToRemove chan interface{}) { - listeners, ok := e.Listeners[eventType] - if !ok { - return - } - - for i, listener := range listeners { - if listener == listenerToRemove { - e.Listeners[eventType] = append(e.Listeners[eventType][:i], - e.Listeners[eventType][i+1:]...) - break - } - } + e.listeners[eventType] = append(e.listeners[eventType], listenerToAdd) } // _fireEvent iterates all listener channels for the given event type and // passes the given data to it. // // This function must be called WITH the mutex held. -func (e *EventManager) _fireEvent(eventType EventT, data interface{}) { - listeners, ok := e.Listeners[eventType] +func (e *eventManager) _fireEvent(eventType eventT, data interface{}) { + listeners, ok := e.listeners[eventType] if !ok { return } diff --git a/politeiawww/paywall.go b/politeiawww/paywall.go index 88466025f..993bf4ec5 100644 --- a/politeiawww/paywall.go +++ b/politeiawww/paywall.go @@ -280,10 +280,10 @@ func (p *politeiawww) UserHasProposalCredits(u *user.User) bool { return ProposalCreditBalance(u) > 0 } -// SpendProposalCredit updates an unspent proposal credit with the passed in +// spendProposalCredit updates an unspent proposal credit with the passed in // censorship token, moves the credit into the user's spent proposal credits // list, and then updates the user database. -func (p *politeiawww) SpendProposalCredit(u *user.User, token string) error { +func (p *politeiawww) spendProposalCredit(u *user.User, token string) error { // Skip when running unit tests or if paywall is disabled. if !p.paywallIsEnabled() { return nil diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 418d9365f..c4dccff8f 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -7,11 +7,16 @@ package main import ( "bytes" "encoding/base64" + "encoding/hex" + "encoding/json" "errors" "fmt" "mime" "net/http" + "time" + piplugin "github.com/decred/politeia/plugins/pi" + pd "github.com/decred/politeia/politeiad/api/v1" pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/user" @@ -38,6 +43,45 @@ func convertUserErrFromSignatureErr(err error) www.UserError { } } +func convertFileFromMetadata(m pi.Metadata) pd.File { + var name string + switch m.Hint { + case pi.HintProposalMetadata: + name = piplugin.FilenameProposalMetadata + } + return pd.File{ + Name: name, + MIME: mimeTypeTextUTF8, + Digest: m.Digest, + Payload: m.Payload, + } +} + +func convertFileFromPi(f pi.File) pd.File { + return pd.File{ + Name: f.Name, + MIME: f.MIME, + Digest: f.Digest, + Payload: f.Payload, + } +} + +func convertFilesFromPi(files []pi.File) []pd.File { + f := make([]pd.File, 0, len(files)) + for _, v := range files { + f = append(f, convertFileFromPi(v)) + } + return f +} + +func convertCensorshipRecordFromPD(cr pd.CensorshipRecord) pi.CensorshipRecord { + return pi.CensorshipRecord{ + Token: cr.Token, + Merkle: cr.Merkle, + Signature: cr.Signature, + } +} + func verifyProposal(files []pi.File, metadata []pi.Metadata, publicKey, signature string) error { if len(files) == 0 { return www.UserError{ @@ -222,24 +266,25 @@ func verifyProposal(files []pi.File, metadata []pi.Metadata, publicKey, signatur func (p *politeiawww) processProposalNew(pn pi.ProposalNew, usr user.User) (*pi.ProposalNewReply, error) { log.Tracef("processProposalNew: %v", usr.Username) - // Verify user has paid the registration paywall + // Verify user paid registration paywall if !p.HasUserPaid(&usr) { return nil, www.UserError{ ErrorCode: www.ErrorStatusUserNotPaid, } } - // Verify user has a proposal credit to spend + // Verify user bought proposal credit if !p.UserHasProposalCredits(&usr) { return nil, www.UserError{ ErrorCode: www.ErrorStatusNoProposalCredits, } } - // Verify the user signed using their active identitiy + // Verify user signed with active identity if usr.PublicKey() != pn.PublicKey { return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidSigningKey, + ErrorCode: www.ErrorStatusInvalidSigningKey, + ErrorContext: []string{"not user's active identity"}, } } @@ -249,15 +294,82 @@ func (p *politeiawww) processProposalNew(pn pi.ProposalNew, usr user.User) (*pi. return nil, err } + // Setup politeiad files. The Metadata objects are converted to + // politeiad files instead of metadata streams since they contain + // user defined data that needs to be included in the merkle root + // that politeiad signs. + files := convertFilesFromPi(pn.Files) + for _, v := range pn.Metadata { + switch v.Hint { + case pi.HintProposalMetadata: + files = append(files, convertFileFromMetadata(v)) + } + } + // Setup metadata stream + pg := piplugin.ProposalGeneral{ + PublicKey: pn.PublicKey, + Signature: pn.Signature, + Timestamp: time.Now().Unix(), + } + b, err := piplugin.EncodeProposalGeneral(pg) + if err != nil { + return nil, err + } + metadata := []pd.MetadataStream{ + { + ID: piplugin.MDStreamIDProposalGeneral, + Payload: string(b), + }, + } // Send politeiad request + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return nil, err + } + nr := pd.NewRecord{ + Challenge: hex.EncodeToString(challenge), + Metadata: metadata, + Files: files, + } + resBody, err := p.makeRequest(http.MethodPost, pd.NewRecordRoute, nr) + if err != nil { + return nil, err + } // Handle response + var nrr pd.NewRecordReply + err = json.Unmarshal(resBody, &nrr) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(p.cfg.Identity, challenge, nrr.Response) + if err != nil { + return nil, err + } + cr := convertCensorshipRecordFromPD(nrr.CensorshipRecord) // Deduct proposal credit from author's account + err = p.spendProposalCredit(&usr, cr.Token) + if err != nil { + return nil, err + } // Fire off a new proposal event + p.eventManager.fire(eventProposalSubmitted, + dataProposalSubmitted{ + token: cr.Token, + // name: name, + username: usr.Username, + }) + + log.Infof("Submitted proposal: %v", cr.Token) + for k, f := range pn.Files { + log.Infof("%02v: %v %v", k, f.Name, f.Digest) + } - return nil, nil + return &pi.ProposalNewReply{ + CensorshipRecord: cr, + }, nil } diff --git a/politeiawww/politeiawww.go b/politeiawww/politeiawww.go index 45bd72688..c78b9f86c 100644 --- a/politeiawww/politeiawww.go +++ b/politeiawww/politeiawww.go @@ -34,8 +34,8 @@ import ( ) var ( - templateNewProposalSubmitted = template.Must( - template.New("new_proposal_submitted_template").Parse(templateNewProposalSubmittedRaw)) + templateProposalSubmitted = template.Must( + template.New("proposal_submitted_template").Parse(templateProposalSubmittedRaw)) templateProposalVetted = template.Must( template.New("proposal_vetted_template").Parse(templateProposalVettedRaw)) templateProposalEdited = template.Must( @@ -106,7 +106,7 @@ type politeiawww struct { db user.Database // User database XXX GOT TO GO params *chaincfg.Params - eventManager *EventManager + eventManager *eventManager // These properties are only used for testing. test bool diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 8264800d6..c862bc41a 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -57,24 +57,6 @@ type proposalsFilter struct { StateMap map[www.PropStateT]bool } -// convertMetadataFromFile returns a politeiad File that was converted from a -// politiawww v1 Metadata. User specified metadata is store as a file in -// politeiad so that it is included in the merkle root that politeiad -// calculates. -func convertFileFromMetadata(m www.Metadata) pd.File { - var name string - switch m.Hint { - case www.HintProposalMetadata: - name = mdstream.FilenameProposalMetadata - } - return pd.File{ - Name: name, - MIME: mimeTypeTextUTF8, - Digest: m.Digest, - Payload: m.Payload, - } -} - // isValidProposalName returns whether the provided string is a valid proposal // name. func isValidProposalName(str string) bool { @@ -1164,7 +1146,7 @@ func (p *politeiawww) processNewProposal(np www.NewProposal, user *user.User) (* for _, v := range np.Metadata { switch v.Hint { case www.HintProposalMetadata: - files = append(files, convertFileFromMetadata(v)) + // files = append(files, convertFileFromMetadata(v)) } } @@ -1224,19 +1206,21 @@ func (p *politeiawww) processNewProposal(np www.NewProposal, user *user.User) (* cr := convertPropCensorFromPD(pdReply.CensorshipRecord) // Deduct proposal credit from user account - err = p.SpendProposalCredit(user, cr.Token) + err = p.spendProposalCredit(user, cr.Token) if err != nil { return nil, err } // Fire off new proposal event - p.fireEvent(EventTypeProposalSubmitted, - EventDataProposalSubmitted{ - CensorshipRecord: &cr, - ProposalName: pm.Name, - User: user, - }, - ) + /* + p.fireEvent(eventTypeProposalSubmitted, + EventDataProposalSubmitted{ + CensorshipRecord: &cr, + ProposalName: pm.Name, + User: user, + }, + ) + */ return &www.NewProposalReply{ CensorshipRecord: cr, @@ -1788,7 +1772,7 @@ func (p *politeiawww) processEditProposal(ep www.EditProposal, u *user.User) (*w for _, v := range ep.Metadata { switch v.Hint { case www.HintProposalMetadata: - files = append(files, convertFileFromMetadata(v)) + // files = append(files, convertFileFromMetadata(v)) } } diff --git a/politeiawww/smtp.go b/politeiawww/smtp.go index 525b82d73..c9dc81e86 100644 --- a/politeiawww/smtp.go +++ b/politeiawww/smtp.go @@ -18,6 +18,25 @@ type smtp struct { disabled bool // Has email been disabled } +// sendEmailTo sends an email with the given subject and body to the provided +// list of email addresses. +func (s *smtp) sendEmailTo(subject, body string, recipients []string) error { + if s.disabled { + return nil + } + + // Setup email + msg := goemail.NewMessage(s.mailAddress, subject, body) + msg.SetName(s.mailName) + + // Add all recipients to BCC + for _, v := range recipients { + msg.AddBCC(v) + } + + return s.client.Send(msg) +} + // sendEmail sends an email with the given subject and body, and the caller // must supply a function which is used to add email addresses to send the // email to. diff --git a/politeiawww/templates.go b/politeiawww/templates.go index 1a5b568ce..1416a0c57 100644 --- a/politeiawww/templates.go +++ b/politeiawww/templates.go @@ -46,11 +46,10 @@ type userPasswordChangedTemplateData struct { Email string } -type newProposalSubmittedTemplateData struct { - Link string - Name string - Username string - Email string +type proposalSubmittedTemplateData struct { + Link string // GUI proposal details url + Name string // Proposal name + Username string // Author username } type proposalEditedTemplateData struct { @@ -156,8 +155,8 @@ You are receiving this email because someone made too many login attempts for {{.Email}} on Politeia. If that was not you, please notify Politeia administrators. ` -const templateNewProposalSubmittedRaw = ` -A new proposal has been submitted on Politeia by {{.Username}} ({{.Email}}): +const templateProposalSubmittedRaw = ` +A new proposal has been submitted on Politeia by {{.Username}}: {{.Name}} {{.Link}} diff --git a/politeiawww/testing.go b/politeiawww/testing.go index aa769e44d..bbd1f1763 100644 --- a/politeiawww/testing.go +++ b/politeiawww/testing.go @@ -610,7 +610,8 @@ func convertPropToPD(t *testing.T, p www.ProposalRecord) pd.Record { for _, v := range p.Metadata { switch v.Hint { case www.HintProposalMetadata: - files = append(files, convertFileFromMetadata(v)) + // TODO + // files = append(files, convertFileFromMetadata(v)) } } diff --git a/politeiawww/www.go b/politeiawww/www.go index bdae4b522..cc035f958 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -315,9 +315,10 @@ func _main() error { // Setup application context. p := &politeiawww{ - cfg: loadedCfg, - ws: make(map[string]map[string]*wsContext), - templates: make(map[string]*template.Template), + cfg: loadedCfg, + ws: make(map[string]map[string]*wsContext), + templates: make(map[string]*template.Template), + eventManager: newEventManager(), // XXX reevaluate where this goes userEmails: make(map[string]uuid.UUID), @@ -392,17 +393,6 @@ func _main() error { return err } - // Setup the code that checks for paywall payments. - if p.cfg.Mode == "piwww" { - err = p.initPaywallChecker() - if err != nil { - return err - } - p.initEventManager() - } else if p.cfg.Mode == "cmswww" { - p.initCMSEventManager() - } - // Load or create new CSRF key log.Infof("Load CSRF key") csrfKeyFilename := filepath.Join(p.cfg.DataDir, "csrf.key") @@ -457,7 +447,18 @@ func _main() error { p.setPoliteiaWWWRoutes() p.setUserWWWRoutes() + err = p.initPaywallChecker() + if err != nil { + return err + } + + p.initEventManagerPi() + p.setupEventListenersPi() + case cmsWWWMode: + // Setup event manager + p.initCMSEventManager() + // Setup dcrdata websocket connection ws, err := wsdcrdata.New(p.dcrdataHostWS()) if err != nil { From 7611975b0ac12fefcb0de269348d0b22704b9c22 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 30 Aug 2020 19:10:09 -0500 Subject: [PATCH 023/449] proposals refactor --- plugins/pi/pi.go | 5 +- politeiad/api/v1/v1.go | 3 +- politeiawww/comments.go | 2 +- politeiawww/eventmanager.go | 7 +- politeiawww/paywall.go | 25 ++-- politeiawww/piwww.go | 269 ++++++++++++++++++++++++++++++------ politeiawww/proposals.go | 117 +++------------- politeiawww/user.go | 22 +-- 8 files changed, 277 insertions(+), 173 deletions(-) diff --git a/plugins/pi/pi.go b/plugins/pi/pi.go index 55dcd8327..f296a2dc6 100644 --- a/plugins/pi/pi.go +++ b/plugins/pi/pi.go @@ -11,7 +11,7 @@ const ( ID = "pi" // Plugin commands - CmdLinkedFrom = "linkedfrom" // Get linked from list + CmdLinkedFrom = "linkedfrom" // Get linked from lists // Metadata stream IDs. All metadata streams in this plugin will // use 1xx numbering. @@ -30,7 +30,8 @@ const ( // as a metadata stream, since it needs to be included in the merkle root that // politeiad signs. type ProposalMetadata struct { - Name string `json:"name"` // Proposal name + // Name is the name of the proposal. + Name string `json:"name"` // LinkTo specifies a public proposal token to link this proposal // to. Ex, an RFP sumbssion must link to the RFP proposal. diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index 305d29776..b073134cd 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -36,7 +36,8 @@ const ( PluginCommandRoute = "/v1/plugin/" // Send a command to a plugin PluginInventoryRoute = PluginCommandRoute + "inventory/" // Inventory all plugins - ChallengeSize = 32 // Size of challenge token in bytes + ChallengeSize = 32 // Size of challenge token in bytes + // TODO TokenSize needs to be updated TokenSize = 32 // Size of token MetadataStreamsMax = uint64(16) // Maximum number of metadata streams diff --git a/politeiawww/comments.go b/politeiawww/comments.go index 2abbc0b76..cc2e3bc87 100644 --- a/politeiawww/comments.go +++ b/politeiawww/comments.go @@ -178,7 +178,7 @@ func validateComment(c www.NewComment) error { } } // validate token - if !isTokenValid(c.Token) { + if !tokenIsValid(c.Token) { return www.UserError{ ErrorCode: www.ErrorStatusInvalidCensorshipToken, } diff --git a/politeiawww/eventmanager.go b/politeiawww/eventmanager.go index ae900540e..16159cd64 100644 --- a/politeiawww/eventmanager.go +++ b/politeiawww/eventmanager.go @@ -49,7 +49,7 @@ func (e *eventManager) fire(event eventT, data interface{}) { listeners, ok := e.listeners[event] if !ok { - log.Errorf("fireEvent: unregistered event %v", event) + log.Errorf("fire: unregistered event %v", event) return } @@ -112,8 +112,7 @@ func (p *politeiawww) handleEventProposalSubmitted(ch chan interface{}) { // Compile email notification recipients emails := make([]string, 0, 256) err := p.db.AllUsers(func(u *user.User) { - // Proposal submitted notifications should only be sent to - // admins. + // Only send proposal submitted notifications to admins if !u.Admin { return } @@ -134,6 +133,6 @@ func (p *politeiawww) handleEventProposalSubmitted(ch chan interface{}) { log.Errorf("emailProposalSubmitted %v: %v", err) } - log.Debugf("Proposal submitted notification sent") + log.Debugf("Sent proposal submitted notification %v", d.token) } } diff --git a/politeiawww/paywall.go b/politeiawww/paywall.go index 993bf4ec5..4bb4ee225 100644 --- a/politeiawww/paywall.go +++ b/politeiawww/paywall.go @@ -264,20 +264,27 @@ func (p *politeiawww) verifyProposalPayment(u *user.User) (*util.TxDetails, erro return nil, nil } -// ProposalCreditsBalance returns the number of proposal credits that the user -// has available to spend. -func ProposalCreditBalance(u *user.User) uint64 { +// userHasPaid returns whether the user has paid the user registration paywall. +func (p *politeiawww) userHasPaid(u user.User) bool { + // Return true if paywall is disabled + if !p.paywallIsEnabled() { + return true + } + + return u.NewUserPaywallTx != "" +} + +func proposalCreditBalance(u user.User) uint64 { return uint64(len(u.UnspentProposalCredits)) } -// UserHasProposalCredits checks to see if the user has any unspent proposal -// credits. -func (p *politeiawww) UserHasProposalCredits(u *user.User) bool { - // Return true when running unit tests or if paywall is disabled +// userHasProposalCredits returns whether the user has at least 1 unspent +// proposal credit. +func (p *politeiawww) userHasProposalCredits(u user.User) bool { if !p.paywallIsEnabled() { return true } - return ProposalCreditBalance(u) > 0 + return proposalCreditBalance(u) > 0 } // spendProposalCredit updates an unspent proposal credit with the passed in @@ -289,7 +296,7 @@ func (p *politeiawww) spendProposalCredit(u *user.User, token string) error { return nil } - if ProposalCreditBalance(u) == 0 { + if !p.userHasProposalCredits(*u) { return www.UserError{ ErrorCode: www.ErrorStatusNoProposalCredits, } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index c4dccff8f..bb4783e60 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -13,6 +13,8 @@ import ( "fmt" "mime" "net/http" + "regexp" + "strconv" "time" piplugin "github.com/decred/politeia/plugins/pi" @@ -26,6 +28,17 @@ import ( // TODO use pi errors instead of www errors +const ( + // MIME types + mimeTypeText = "text/plain" + mimeTypeTextUTF8 = "text/plain; charset=utf-8" + mimeTypePNG = "image/png" +) + +var ( + validProposalName = regexp.MustCompile(createProposalNameRegex()) +) + func convertUserErrFromSignatureErr(err error) www.UserError { var e util.SignatureError var s www.ErrorStatusT @@ -82,9 +95,122 @@ func convertCensorshipRecordFromPD(cr pd.CensorshipRecord) pi.CensorshipRecord { } } -func verifyProposal(files []pi.File, metadata []pi.Metadata, publicKey, signature string) error { - if len(files) == 0 { +// proposalNameIsValid returns whether the provided string is a valid proposal +// name. +func proposalNameIsValid(str string) bool { + return validProposalName.MatchString(str) +} + +// createProposalNameRegex returns a regex string for validating the proposal +// name. +func createProposalNameRegex() string { + var validProposalNameBuffer bytes.Buffer + validProposalNameBuffer.WriteString("^[") + + for _, supportedChar := range www.PolicyProposalNameSupportedChars { + if len(supportedChar) > 1 { + validProposalNameBuffer.WriteString(supportedChar) + } else { + validProposalNameBuffer.WriteString(`\` + supportedChar) + } + } + minNameLength := strconv.Itoa(www.PolicyMinProposalNameLength) + maxNameLength := strconv.Itoa(www.PolicyMaxProposalNameLength) + validProposalNameBuffer.WriteString("]{") + validProposalNameBuffer.WriteString(minNameLength + ",") + validProposalNameBuffer.WriteString(maxNameLength + "}$") + + return validProposalNameBuffer.String() +} + +// linkByPeriodMin returns the minimum amount of time, in seconds, that the +// LinkBy period must be set to. This is determined by adding 1 week onto the +// minimum voting period so that RFP proposal submissions have at least one +// week to be submitted after the proposal vote ends. +func (p *politeiawww) linkByPeriodMin() int64 { + var ( + submissionPeriod int64 = 604800 // One week in seconds + blockTime int64 // In seconds + ) + switch { + case p.cfg.TestNet: + blockTime = int64(testNet3Params.TargetTimePerBlock.Seconds()) + case p.cfg.SimNet: + blockTime = int64(simNetParams.TargetTimePerBlock.Seconds()) + default: + blockTime = int64(mainNetParams.TargetTimePerBlock.Seconds()) + } + return (int64(p.cfg.VoteDurationMin) * blockTime) + submissionPeriod +} + +// linkByPeriodMax returns the maximum amount of time, in seconds, that the +// LinkBy period can be set to. 3 months is currently hard coded with no real +// reason for deciding on 3 months besides that it sounds like a sufficient +// amount of time. This can be changed if there is a valid reason to. +func (p *politeiawww) linkByPeriodMax() int64 { + return 7776000 // 3 months in seconds +} + +// tokenIsValid returns whether the provided string is a valid politeiad +// censorship record token. +func tokenIsValid(token string) bool { + b, err := hex.DecodeString(token) + if err != nil { + return false + } + if len(b) != pd.TokenSize { + return false + } + return true +} + +func (p *politeiawww) verifyProposalMetadata(pm pi.ProposalMetadata) error { + // Verify name + if !proposalNameIsValid(pm.Name) { return www.UserError{ + ErrorCode: www.ErrorStatusProposalInvalidTitle, + ErrorContext: []string{createProposalNameRegex()}, + } + } + + // Verify linkto + if pm.LinkTo != "" { + if !tokenIsValid(pm.LinkTo) { + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidLinkTo, + ErrorContext: []string{"invalid token"}, + } + } + } + + // Verify linkby + if pm.LinkBy != 0 { + min := time.Now().Unix() + p.linkByPeriodMin() + max := time.Now().Unix() + p.linkByPeriodMax() + switch { + case pm.LinkBy < min: + e := fmt.Sprintf("linkby %v is less than min required of %v", + pm.LinkBy, min) + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidLinkBy, + ErrorContext: []string{e}, + } + case pm.LinkBy > max: + e := fmt.Sprintf("linkby %v is more than max allowed of %v", + pm.LinkBy, max) + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidLinkBy, + ErrorContext: []string{e}, + } + } + } + + return nil +} + +func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, publicKey, signature string) (*pi.ProposalMetadata, error) { + if len(files) == 0 { + return nil, www.UserError{ ErrorCode: www.ErrorStatusProposalMissingFiles, ErrorContext: []string{"no files found"}, } @@ -101,7 +227,7 @@ func verifyProposal(files []pi.File, metadata []pi.Metadata, publicKey, signatur // Validate file name _, ok := filenames[v.Name] if ok { - return www.UserError{ + return nil, www.UserError{ ErrorCode: www.ErrorStatusProposalDuplicateFilenames, ErrorContext: []string{v.Name}, } @@ -110,16 +236,15 @@ func verifyProposal(files []pi.File, metadata []pi.Metadata, publicKey, signatur // Validate file payload if v.Payload == "" { - e := fmt.Sprintf("base64 payload is empty for file '%v'", - v.Name) - return www.UserError{ + e := fmt.Sprintf("empty payload for file '%v'", v.Name) + return nil, www.UserError{ ErrorCode: www.ErrorStatusInvalidBase64, ErrorContext: []string{e}, } } payloadb, err := base64.StdEncoding.DecodeString(v.Payload) if err != nil { - return www.UserError{ + return nil, www.UserError{ ErrorCode: www.ErrorStatusInvalidBase64, ErrorContext: []string{v.Name}, } @@ -129,15 +254,15 @@ func verifyProposal(files []pi.File, metadata []pi.Metadata, publicKey, signatur digest := util.Digest(payloadb) d, ok := util.ConvertDigest(v.Digest) if !ok { - return www.UserError{ + return nil, www.UserError{ ErrorCode: www.ErrorStatusInvalidFileDigest, ErrorContext: []string{v.Name}, } } if !bytes.Equal(digest, d[:]) { - e := fmt.Sprintf("computed digest does not match given digest "+ - "for file '%v'", v.Name) - return www.UserError{ + e := fmt.Sprintf("file '%v' digest got %v, want %x", + v.Name, v.Digest, digest) + return nil, www.UserError{ ErrorCode: www.ErrorStatusInvalidFileDigest, ErrorContext: []string{e}, } @@ -147,21 +272,20 @@ func verifyProposal(files []pi.File, metadata []pi.Metadata, publicKey, signatur ct := http.DetectContentType(payloadb) mimePayload, _, err := mime.ParseMediaType(ct) if err != nil { - return err + return nil, err } mimeFile, _, err := mime.ParseMediaType(v.MIME) if err != nil { - log.Debugf("validateProposal: ParseMediaType(%v): %v", - v.MIME, err) - return www.UserError{ + log.Debugf("ParseMediaType(%v): %v", v.MIME, err) + return nil, www.UserError{ ErrorCode: www.ErrorStatusInvalidMIMEType, ErrorContext: []string{v.Name}, } } if mimeFile != mimePayload { - e := fmt.Sprintf("detected mime '%v' does not match '%v' for file '%v'", - mimePayload, mimeFile, v.Name) - return www.UserError{ + e := fmt.Sprintf("file '%v' mime type got %v, want %v", + v.Name, mimeFile, mimePayload) + return nil, www.UserError{ ErrorCode: www.ErrorStatusInvalidMIMEType, ErrorContext: []string{e}, } @@ -174,9 +298,9 @@ func verifyProposal(files []pi.File, metadata []pi.Metadata, publicKey, signatur // Verify text file size if len(payloadb) > www.PolicyMaxMDSize { - e := fmt.Sprintf("file size %v exceeds max %v for file '%v'", - len(payloadb), www.PolicyMaxMDSize, v.Name) - return www.UserError{ + e := fmt.Sprintf("file '%v' size %v exceeds max size %v", + v.Name, len(payloadb), www.PolicyMaxMDSize) + return nil, www.UserError{ ErrorCode: www.ErrorStatusMaxMDSizeExceededPolicy, ErrorContext: []string{e}, } @@ -185,7 +309,7 @@ func verifyProposal(files []pi.File, metadata []pi.Metadata, publicKey, signatur // The only text file that is allowed is the index markdown // file. if v.Name != www.PolicyIndexFilename { - return www.UserError{ + return nil, www.UserError{ ErrorCode: www.ErrorStatusMaxMDsExceededPolicy, ErrorContext: []string{v.Name}, } @@ -193,7 +317,7 @@ func verifyProposal(files []pi.File, metadata []pi.Metadata, publicKey, signatur if foundIndexFile { e := fmt.Sprintf("more than one %v file found", www.PolicyIndexFilename) - return www.UserError{ + return nil, www.UserError{ ErrorCode: www.ErrorStatusMaxMDsExceededPolicy, ErrorContext: []string{e}, } @@ -207,16 +331,16 @@ func verifyProposal(files []pi.File, metadata []pi.Metadata, publicKey, signatur // Verify image file size if len(payloadb) > www.PolicyMaxImageSize { - e := fmt.Sprintf("file size %v exceeds max %v for file '%v'", - len(payloadb), www.PolicyMaxImageSize, v.Name) - return www.UserError{ + e := fmt.Sprintf("file '%v' size %v exceeds max size %v", + v.Name, len(payloadb), www.PolicyMaxImageSize) + return nil, www.UserError{ ErrorCode: www.ErrorStatusMaxImageSizeExceededPolicy, ErrorContext: []string{e}, } } default: - return www.UserError{ + return nil, www.UserError{ ErrorCode: www.ErrorStatusInvalidMIMEType, ErrorContext: []string{v.MIME}, } @@ -226,7 +350,7 @@ func verifyProposal(files []pi.File, metadata []pi.Metadata, publicKey, signatur // Verify that an index file is present. if !foundIndexFile { e := fmt.Sprintf("%v file not found", www.PolicyIndexFilename) - return www.UserError{ + return nil, www.UserError{ ErrorCode: www.ErrorStatusProposalMissingFiles, ErrorContext: []string{e}, } @@ -234,9 +358,9 @@ func verifyProposal(files []pi.File, metadata []pi.Metadata, publicKey, signatur // Verify file counts are acceptable if countTextFiles > www.PolicyMaxMDs { - e := fmt.Sprintf("got %v text files; max is %v", + e := fmt.Sprintf("got %v text files, max is %v", countTextFiles, www.PolicyMaxMDs) - return www.UserError{ + return nil, www.UserError{ ErrorCode: www.ErrorStatusMaxMDsExceededPolicy, ErrorContext: []string{e}, } @@ -244,37 +368,99 @@ func verifyProposal(files []pi.File, metadata []pi.Metadata, publicKey, signatur if countImageFiles > www.PolicyMaxImages { e := fmt.Sprintf("got %v image files, max is %v", countImageFiles, www.PolicyMaxImages) - return www.UserError{ + return nil, www.UserError{ ErrorCode: www.ErrorStatusMaxImagesExceededPolicy, ErrorContext: []string{e}, } } + // Verify that the metadata contains a ProposalMetadata and only + // a ProposalMetadata. + switch { + case len(metadata) == 0: + return nil, www.UserError{ + ErrorCode: www.ErrorStatusMetadataMissing, + ErrorContext: []string{www.HintProposalMetadata}, + } + case len(metadata) > 1: + e := fmt.Sprintf("metadata should only contain %v", + www.HintProposalMetadata) + return nil, www.UserError{ + ErrorCode: www.ErrorStatusMetadataInvalid, + ErrorContext: []string{e}, + } + } + md := metadata[0] + if md.Hint != www.HintProposalMetadata { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusMetadataInvalid, + ErrorContext: []string{md.Hint}, + } + } + + // Verify metadata fields + b, err := base64.StdEncoding.DecodeString(md.Payload) + if err != nil { + e := fmt.Sprintf("metadata '%v' invalid base64 payload", md.Hint) + return nil, www.UserError{ + ErrorCode: www.ErrorStatusMetadataInvalid, + ErrorContext: []string{e}, + } + } + digest := util.Digest(b) + if md.Digest != hex.EncodeToString(digest) { + e := fmt.Sprintf("metadata '%v' got digest %v, want %v", + md.Hint, md.Digest, hex.EncodeToString(digest)) + return nil, www.UserError{ + ErrorCode: www.ErrorStatusMetadataDigestInvalid, + ErrorContext: []string{e}, + } + } + + // Decode ProposalMetadata + d := json.NewDecoder(bytes.NewReader(b)) + d.DisallowUnknownFields() + var pm pi.ProposalMetadata + err = d.Decode(&pm) + if err != nil { + log.Debugf("Decode ProposalMetadata: %v", err) + return nil, www.UserError{ + ErrorCode: www.ErrorStatusMetadataInvalid, + ErrorContext: []string{md.Hint}, + } + } + + // Verify ProposalMetadata + err = p.verifyProposalMetadata(pm) + if err != nil { + return nil, err + } + // Verify signature mr, err := wwwutil.MerkleRoot(files, metadata) if err != nil { - return fmt.Errorf("MerkleRoot: %v", err) + return nil, err } err = util.VerifySignature(signature, publicKey, mr) if err != nil { - return convertUserErrFromSignatureErr(err) + return nil, convertUserErrFromSignatureErr(err) } - return nil + return &pm, nil } func (p *politeiawww) processProposalNew(pn pi.ProposalNew, usr user.User) (*pi.ProposalNewReply, error) { log.Tracef("processProposalNew: %v", usr.Username) // Verify user paid registration paywall - if !p.HasUserPaid(&usr) { + if !p.userHasPaid(usr) { return nil, www.UserError{ ErrorCode: www.ErrorStatusUserNotPaid, } } - // Verify user bought proposal credit - if !p.UserHasProposalCredits(&usr) { + // Verify user has a proposal credit + if !p.userHasProposalCredits(usr) { return nil, www.UserError{ ErrorCode: www.ErrorStatusNoProposalCredits, } @@ -289,7 +475,8 @@ func (p *politeiawww) processProposalNew(pn pi.ProposalNew, usr user.User) (*pi. } // Verify proposal - err := verifyProposal(pn.Files, pn.Metadata, pn.PublicKey, pn.Signature) + pm, err := p.verifyProposal(pn.Files, pn.Metadata, + pn.PublicKey, pn.Signature) if err != nil { return nil, err } @@ -359,12 +546,12 @@ func (p *politeiawww) processProposalNew(pn pi.ProposalNew, usr user.User) (*pi. // Fire off a new proposal event p.eventManager.fire(eventProposalSubmitted, dataProposalSubmitted{ - token: cr.Token, - // name: name, + token: cr.Token, + name: pm.Name, username: usr.Username, }) - log.Infof("Submitted proposal: %v", cr.Token) + log.Infof("Submitted proposal: %v %v", cr.Token, pm.Name) for k, f := range pn.Files { log.Infof("%02v: %v %v", k, f.Name, f.Digest) } diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index c862bc41a..b7cfa5128 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -12,7 +12,6 @@ import ( "fmt" "mime" "net/http" - "regexp" "sort" "strconv" "time" @@ -26,17 +25,6 @@ import ( "github.com/decred/politeia/util" ) -const ( - // MIME types - mimeTypeText = "text/plain" - mimeTypeTextUTF8 = "text/plain; charset=utf-8" - mimeTypePNG = "image/png" -) - -var ( - validProposalName = regexp.MustCompile(createProposalNameRegex()) -) - // proposalStats is used to provide a summary of the number of proposals // grouped by proposal status. type proposalsSummary struct { @@ -57,34 +45,6 @@ type proposalsFilter struct { StateMap map[www.PropStateT]bool } -// isValidProposalName returns whether the provided string is a valid proposal -// name. -func isValidProposalName(str string) bool { - return validProposalName.MatchString(str) -} - -// createProposalNameRegex returns a regex string for matching the proposal -// name. -func createProposalNameRegex() string { - var validProposalNameBuffer bytes.Buffer - validProposalNameBuffer.WriteString("^[") - - for _, supportedChar := range www.PolicyProposalNameSupportedChars { - if len(supportedChar) > 1 { - validProposalNameBuffer.WriteString(supportedChar) - } else { - validProposalNameBuffer.WriteString(`\` + supportedChar) - } - } - minNameLength := strconv.Itoa(www.PolicyMinProposalNameLength) - maxNameLength := strconv.Itoa(www.PolicyMaxProposalNameLength) - validProposalNameBuffer.WriteString("]{") - validProposalNameBuffer.WriteString(minNameLength + ",") - validProposalNameBuffer.WriteString(maxNameLength + "}$") - - return validProposalNameBuffer.String() -} - // isProposalAuthor returns whether the provided user is the author of the // provided proposal. func isProposalAuthor(pr www.ProposalRecord, u user.User) bool { @@ -111,24 +71,11 @@ func isRFPSubmission(pr www.ProposalRecord) bool { return pr.LinkTo != "" } -// isTokenValid returns whether the provided string is a valid politeiad -// censorship record token. -func isTokenValid(token string) bool { - b, err := hex.DecodeString(token) - if err != nil { - return false - } - if len(b) != pd.TokenSize { - return false - } - return true -} - func getInvalidTokens(tokens []string) []string { invalidTokens := make([]string, 0, len(tokens)) for _, token := range tokens { - if !isTokenValid(token) { + if !tokenIsValid(token) { invalidTokens = append(invalidTokens, token) } } @@ -158,34 +105,6 @@ func validateVoteBit(vote www2.Vote, bit uint64) error { return fmt.Errorf("bit not found 0x%x", bit) } -// linkByPeriodMin returns the minimum amount of time, in seconds, that the -// LinkBy period must be set to. This is determined by adding 1 week onto the -// minimum voting period so that RFP proposal submissions have at least one -// week to be submitted after the proposal vote ends. -func (p *politeiawww) linkByPeriodMin() int64 { - var ( - submissionPeriod int64 = 604800 // One week in seconds - blockTime int64 // In seconds - ) - switch { - case p.cfg.TestNet: - blockTime = int64(testNet3Params.TargetTimePerBlock.Seconds()) - case p.cfg.SimNet: - blockTime = int64(simNetParams.TargetTimePerBlock.Seconds()) - default: - blockTime = int64(mainNetParams.TargetTimePerBlock.Seconds()) - } - return (int64(p.cfg.VoteDurationMin) * blockTime) + submissionPeriod -} - -// linkByPeriodMax returns the maximum amount of time, in seconds, that the -// LinkBy period can be set to. 3 months is currently hard coded with no real -// reason for deciding on 3 months besides that it sounds like a sufficient -// amount of time. This can be changed if there is a valid reason to. -func (p *politeiawww) linkByPeriodMax() int64 { - return 7776000 // 3 months in seconds -} - func (p *politeiawww) linkByValidate(linkBy int64) error { min := time.Now().Unix() + p.linkByPeriodMin() max := time.Now().Unix() + p.linkByPeriodMax() @@ -215,7 +134,7 @@ func (p *politeiawww) linkByValidate(linkBy int64) error { // specific to that action only. func (p *politeiawww) validateProposalMetadata(pm www.ProposalMetadata) error { // Validate Name - if !isValidProposalName(pm.Name) { + if !proposalNameIsValid(pm.Name) { return www.UserError{ ErrorCode: www.ErrorStatusProposalInvalidTitle, ErrorContext: []string{createProposalNameRegex()}, @@ -224,7 +143,7 @@ func (p *politeiawww) validateProposalMetadata(pm www.ProposalMetadata) error { // Validate LinkTo if pm.LinkTo != "" { - if !isTokenValid(pm.LinkTo) { + if !tokenIsValid(pm.LinkTo) { return www.UserError{ ErrorCode: www.ErrorStatusInvalidLinkTo, ErrorContext: []string{"invalid token"}, @@ -1093,13 +1012,13 @@ func (p *politeiawww) processNewProposal(np www.NewProposal, user *user.User) (* log.Tracef("processNewProposal") // Pay up sucker! - if !p.HasUserPaid(user) { + if !p.userHasPaid(*user) { return nil, www.UserError{ ErrorCode: www.ErrorStatusUserNotPaid, } } - if !p.UserHasProposalCredits(user) { + if !p.userHasProposalCredits(*user) { return nil, www.UserError{ ErrorCode: www.ErrorStatusNoProposalCredits, } @@ -1443,7 +1362,7 @@ func (p *politeiawww) processSetProposalStatus(sps www.SetProposalStatus, u *use log.Tracef("processSetProposalStatus %v", sps.Token) // Make sure token is valid and not a prefix - if !isTokenValid(sps.Token) { + if !tokenIsValid(sps.Token) { return nil, www.UserError{ ErrorCode: www.ErrorStatusInvalidCensorshipToken, ErrorContext: []string{sps.Token}, @@ -1673,7 +1592,7 @@ func (p *politeiawww) processEditProposal(ep www.EditProposal, u *user.User) (*w log.Tracef("processEditProposal %v", ep.Token) // Make sure token is valid and not a prefix - if !isTokenValid(ep.Token) { + if !tokenIsValid(ep.Token) { return nil, www.UserError{ ErrorCode: www.ErrorStatusInvalidCensorshipToken, ErrorContext: []string{ep.Token}, @@ -1862,8 +1781,8 @@ func (p *politeiawww) processAllVetted(v www.GetAllVetted) (*www.GetAllVettedRep log.Tracef("processAllVetted") // Validate query params - if (v.Before != "" && !isTokenValid(v.Before)) || - (v.After != "" && !isTokenValid(v.After)) { + if (v.Before != "" && !tokenIsValid(v.Before)) || + (v.After != "" && !tokenIsValid(v.After)) { return nil, www.UserError{ ErrorCode: www.ErrorStatusInvalidCensorshipToken, } @@ -1904,7 +1823,7 @@ func (p *politeiawww) processCommentsGet(token string, u *user.User) (*www.GetCo log.Tracef("ProcessCommentGet: %v", token) // Make sure token is valid and not a prefix - if !isTokenValid(token) { + if !tokenIsValid(token) { return nil, www.UserError{ ErrorCode: www.ErrorStatusInvalidCensorshipToken, ErrorContext: []string{token}, @@ -2027,7 +1946,7 @@ func (p *politeiawww) processVoteStatus(token string) (*www.VoteStatusReply, err /* // Make sure token is valid and not a prefix - if !isTokenValid(token) { + if !tokenIsValid(token) { return nil, www.UserError{ ErrorCode: www.ErrorStatusInvalidCensorshipToken, ErrorContext: []string{token}, @@ -2193,7 +2112,7 @@ func (p *politeiawww) processVoteResults(token string) (*www.VoteResultsReply, e /* // Make sure token is valid and not a prefix - if !isTokenValid(token) { + if !tokenIsValid(token) { return nil, www.UserError{ ErrorCode: www.ErrorStatusInvalidCensorshipToken, ErrorContext: []string{token}, @@ -2279,7 +2198,7 @@ func (p *politeiawww) processCastVotes(ballot *www.Ballot) (*www.BallotReply, er // Verify proposal tokens for _, vote := range ballot.Votes { - if !isTokenValid(vote.Token) { + if !tokenIsValid(vote.Token) { return nil, www.UserError{ ErrorCode: www.ErrorStatusInvalidCensorshipToken, ErrorContext: []string{vote.Token}, @@ -2341,7 +2260,7 @@ func (p *politeiawww) processProposalPaywallDetails(u *user.User) (*www.Proposal // Proposal paywalls cannot be generated until the user has paid their // user registration fee. - if !p.HasUserPaid(u) { + if !p.userHasPaid(*u) { return nil, www.UserError{ ErrorCode: www.ErrorStatusUserNotPaid, } @@ -2508,7 +2427,7 @@ func (p *politeiawww) processAuthorizeVote(av www.AuthorizeVote, u *user.User) ( log.Tracef("processAuthorizeVote %v", av.Token) // Make sure token is valid and not a prefix - if !isTokenValid(av.Token) { + if !tokenIsValid(av.Token) { return nil, www.UserError{ ErrorCode: www.ErrorStatusInvalidCensorshipToken, ErrorContext: []string{av.Token}, @@ -2640,7 +2559,7 @@ func validateVoteOptions(options []www2.VoteOption) error { } func validateStartVote(sv www2.StartVote, u user.User, pr www.ProposalRecord, vs www.VoteSummary, durationMin, durationMax uint32) error { - if !isTokenValid(sv.Vote.Token) { + if !tokenIsValid(sv.Vote.Token) { // Sanity check since proposal has already been looked up and // passed in to this function. return fmt.Errorf("invalid token %v", sv.Vote.Token) @@ -2833,7 +2752,7 @@ func (p *politeiawww) processStartVoteV2(sv www2.StartVote, u *user.User) (*www2 } // Fetch proposal and vote summary - if !isTokenValid(sv.Vote.Token) { + if !tokenIsValid(sv.Vote.Token) { return nil, www.UserError{ ErrorCode: www.ErrorStatusInvalidCensorshipToken, ErrorContext: []string{sv.Vote.Token}, @@ -3013,7 +2932,7 @@ func (p *politeiawww) processStartVoteRunoffV2(sv www2.StartVoteRunoff, u *user. for _, v := range sv.StartVotes { // Fetch proposal and vote summary token := v.Vote.Token - if !isTokenValid(token) { + if !tokenIsValid(token) { return nil, www.UserError{ ErrorCode: www.ErrorStatusInvalidCensorshipToken, ErrorContext: []string{token}, diff --git a/politeiawww/user.go b/politeiawww/user.go index 031f76bdd..b0b205e4f 100644 --- a/politeiawww/user.go +++ b/politeiawww/user.go @@ -111,7 +111,7 @@ func convertWWWUserFromDatabaseUser(user *user.User) www.User { Deactivated: user.Deactivated, Locked: userIsLocked(user.FailedLoginAttempts), Identities: convertWWWIdentitiesFromDatabaseIdentities(user.Identities), - ProposalCredits: ProposalCreditBalance(user), + ProposalCredits: proposalCreditBalance(*user), EmailNotifications: user.EmailNotifications, } } @@ -727,11 +727,11 @@ func (p *politeiawww) createLoginReply(u *user.User, lastLoginTime int64) (*www. Username: u.Username, PublicKey: u.PublicKey(), PaywallTxID: u.NewUserPaywallTx, - ProposalCredits: ProposalCreditBalance(u), + ProposalCredits: proposalCreditBalance(*u), LastLoginTime: lastLoginTime, } - if !p.HasUserPaid(u) { + if !p.userHasPaid(*u) { err := p.GenerateNewUserPaywall(u) if err != nil { return nil, err @@ -2041,7 +2041,7 @@ func (p *politeiawww) processUserPaymentsRescan(upr www.UserPaymentsRescan) (*ww // that in the user database. func (p *politeiawww) processVerifyUserPayment(u *user.User, vupt www.VerifyUserPayment) (*www.VerifyUserPaymentReply, error) { var reply www.VerifyUserPaymentReply - if p.HasUserPaid(u) { + if p.userHasPaid(*u) { reply.HasPaid = true return &reply, nil } @@ -2145,7 +2145,7 @@ func (p *politeiawww) addUsersToPaywallPool() error { } // User paywalls - if p.HasUserPaid(u) { + if p.userHasPaid(*u) { return } if u.NewUserVerificationToken != nil { @@ -2226,7 +2226,7 @@ func (p *politeiawww) checkForUserPayments(pool map[uuid.UUID]paywallPoolMember) log.Tracef("Checking the user paywall address for user %v...", u.Email) - if p.HasUserPaid(u) { + if p.userHasPaid(*u) { // The user could have been marked as paid by // RouteVerifyUserPayment, so just remove him from the // in-memory pool. @@ -2311,16 +2311,6 @@ func (p *politeiawww) GenerateNewUserPaywall(u *user.User) error { return nil } -// HasUserPaid checks that a user has paid the paywall -func (p *politeiawww) HasUserPaid(u *user.User) bool { - // Return true if paywall is disabled - if !p.paywallIsEnabled() { - return true - } - - return u.NewUserPaywallTx != "" -} - // initPaywallCheck is intended to be called func (p *politeiawww) initPaywallChecker() error { if p.cfg.PaywallAmount == 0 { From 7741948965f61af4f99f6a7fddaa9677c820f2f1 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 31 Aug 2020 06:41:10 -0500 Subject: [PATCH 024/449] plugin hook new record pre --- plugins/pi/pi.go | 17 + politeiad/backend/backend.go | 2 +- politeiad/backend/gitbe/gitbe.go | 52 +-- politeiad/backend/tlogbe/plugin.go | 348 +++--------------- politeiad/backend/tlogbe/plugins/pi.go | 79 ++++ .../backend/tlogbe/plugins/ticketvote.go | 10 +- politeiad/backend/tlogbe/recordclient.go | 294 +++++++++++++++ politeiad/backend/tlogbe/tlogbe.go | 64 +++- politeiawww/api/www/v1/v1.go | 2 +- politeiawww/eventmanager.go | 3 +- politeiawww/piwww.go | 2 +- 11 files changed, 532 insertions(+), 341 deletions(-) create mode 100644 politeiad/backend/tlogbe/plugins/pi.go create mode 100644 politeiad/backend/tlogbe/recordclient.go diff --git a/plugins/pi/pi.go b/plugins/pi/pi.go index f296a2dc6..8128ac594 100644 --- a/plugins/pi/pi.go +++ b/plugins/pi/pi.go @@ -43,6 +43,21 @@ type ProposalMetadata struct { LinkBy int64 `json:"linkby,omitempty"` } +// EncodeProposalMetadata encodes a ProposalMetadata into a JSON byte slice. +func EncodeProposalMetadata(pm ProposalMetadata) ([]byte, error) { + return json.Marshal(pm) +} + +// DecodeProposalMetadata decodes a ProposalMetadata into a JSON byte slice. +func DecodeProposalMetadata(payload []byte) (*ProposalMetadata, error) { + var pm ProposalMetadata + err := json.Unmarshal(payload, &pm) + if err != nil { + return nil, err + } + return &pm, nil +} + // ProposalGeneral represents general proposal metadata that is saved on // proposal submission. ProposalGeneral is saved to politeiad as a metadata // stream. @@ -55,10 +70,12 @@ type ProposalGeneral struct { Timestamp int64 `json:"timestamp"` // Submission UNIX timestamp } +// EncodeProposalGeneral encodes a ProposalGeneral into a JSON byte slice. func EncodeProposalGeneral(pg ProposalGeneral) ([]byte, error) { return json.Marshal(pg) } +// DecodeProposalGeneral decodes a ProposalGeneral into a JSON byte slice. func DecodeProposalGeneral(payload []byte) (*ProposalGeneral, error) { var pg ProposalGeneral err := json.Unmarshal(payload, &pg) diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index 9daf70435..9d4fd9205 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -209,7 +209,7 @@ type Backend interface { GetPlugins() ([]Plugin, error) // Plugin pass-through command - Plugin(string, string, string) (string, string, error) // command type, payload, error + Plugin(string, string, string) (string, error) // Close performs cleanup of the backend. Close() diff --git a/politeiad/backend/gitbe/gitbe.go b/politeiad/backend/gitbe/gitbe.go index ad63fcc82..30c8f101c 100644 --- a/politeiad/backend/gitbe/gitbe.go +++ b/politeiad/backend/gitbe/gitbe.go @@ -2804,59 +2804,43 @@ func (g *gitBackEnd) GetPlugins() ([]backend.Plugin, error) { // execute. // // Plugin satisfies the backend interface. -func (g *gitBackEnd) Plugin(pluginID, command, payload string) (string, string, error) { +func (g *gitBackEnd) Plugin(pluginID, command, payload string) (string, error) { log.Tracef("Plugin: %v", command) switch command { case decredplugin.CmdAuthorizeVote: - payload, err := g.pluginAuthorizeVote(payload) - return decredplugin.CmdAuthorizeVote, payload, err + return g.pluginAuthorizeVote(payload) case decredplugin.CmdStartVote: - payload, err := g.pluginStartVote(payload) - return decredplugin.CmdStartVote, payload, err + return g.pluginStartVote(payload) case decredplugin.CmdStartVoteRunoff: - payload, err := g.pluginStartVoteRunoff(payload) - return decredplugin.CmdStartVote, payload, err + return g.pluginStartVoteRunoff(payload) case decredplugin.CmdBallot: - payload, err := g.pluginBallot(payload) - return decredplugin.CmdBallot, payload, err + return g.pluginBallot(payload) case decredplugin.CmdProposalVotes: - payload, err := g.pluginProposalVotes(payload) - return decredplugin.CmdProposalVotes, payload, err + return g.pluginProposalVotes(payload) case decredplugin.CmdBestBlock: - payload, err := g.pluginBestBlock() - return decredplugin.CmdBestBlock, payload, err + return g.pluginBestBlock() case decredplugin.CmdNewComment: - payload, err := g.pluginNewComment(payload) - return decredplugin.CmdNewComment, payload, err + return g.pluginNewComment(payload) case decredplugin.CmdLikeComment: - payload, err := g.pluginLikeComment(payload) - return decredplugin.CmdLikeComment, payload, err + return g.pluginLikeComment(payload) case decredplugin.CmdCensorComment: - payload, err := g.pluginCensorComment(payload) - return decredplugin.CmdCensorComment, payload, err + return g.pluginCensorComment(payload) case decredplugin.CmdGetComments: - payload, err := g.pluginGetComments(payload) - return decredplugin.CmdGetComments, payload, err + return g.pluginGetComments(payload) case decredplugin.CmdProposalCommentsLikes: - payload, err := g.pluginGetProposalCommentsLikes(payload) - return decredplugin.CmdProposalCommentsLikes, payload, err + return g.pluginGetProposalCommentsLikes(payload) case decredplugin.CmdInventory: - payload, err := g.pluginInventory(payload) - return decredplugin.CmdInventory, payload, err + return g.pluginInventory(payload) case decredplugin.CmdLoadVoteResults: - payload, err := g.pluginLoadVoteResults() - return decredplugin.CmdLoadVoteResults, payload, err + return g.pluginLoadVoteResults() case cmsplugin.CmdInventory: - payload, err := g.pluginCMSInventory() - return cmsplugin.CmdInventory, payload, err + return g.pluginCMSInventory() case cmsplugin.CmdStartVote: - payload, err := g.pluginStartDCCVote(payload) - return cmsplugin.CmdStartVote, payload, err + return g.pluginStartDCCVote(payload) case cmsplugin.CmdCastVote: - payload, err := g.pluginCastVote(payload) - return cmsplugin.CmdCastVote, payload, err + return g.pluginCastVote(payload) } - return "", "", fmt.Errorf("invalid payload command") // XXX this needs to become a type error + return "", fmt.Errorf("invalid payload command") // XXX this needs to become a type error } // Close shuts down the backend. It obtains the lock and sets the shutdown diff --git a/politeiad/backend/tlogbe/plugin.go b/politeiad/backend/tlogbe/plugin.go index 3e7be672f..279a35d36 100644 --- a/politeiad/backend/tlogbe/plugin.go +++ b/politeiad/backend/tlogbe/plugin.go @@ -5,12 +5,9 @@ package tlogbe import ( - "bytes" - "encoding/hex" - "fmt" + "encoding/json" - "github.com/google/trillian" - "google.golang.org/grpc/codes" + "github.com/decred/politeia/politeiad/backend" ) type HookT int @@ -18,315 +15,86 @@ type HookT int const ( // Plugin hooks HookInvalid HookT = 0 - HookPreNewRecord HookT = 1 - HookPostNewRecord HookT = 2 - HookPreEditRecord HookT = 3 - HookPostEditRecord HookT = 4 - HookPreEditMetadata HookT = 5 - HookPostEditMetadata HookT = 6 - HookPreSetRecordStatus HookT = 7 - HookPostSetRecordStatus HookT = 8 + HookNewRecordPre HookT = 1 + HookNewRecordPost HookT = 2 + HookEditRecordPre HookT = 3 + HookEditRecordPost HookT = 4 + HookEditMetadataPre HookT = 5 + HookEditMetadataPost HookT = 6 + HookSetRecordStatusPre HookT = 7 + HookSetRecordStatusPost HookT = 8 ) var ( // Hooks contains human readable plugin hook descriptions. Hooks = map[HookT]string{ - HookPostNewRecord: "post new record", - HookPostEditRecord: "post edit record", - HookPostEditMetadata: "post edit metadata", - HookPostSetRecordStatus: "post set record status", + HookNewRecordPre: "new record pre", + HookNewRecordPost: "new record post", + HookEditRecordPre: "edit record pre", + HookEditRecordPost: "edit record post", + HookEditMetadataPre: "edit metadata pre", + HookEditMetadataPost: "edit metadata post", + HookSetRecordStatusPre: "set record status pre", + HookSetRecordStatusPost: "set record status post", } ) -// Plugin provides an API for the tlogbe to use when interacting with plugins. -// All tlogbe plugins must implement the Plugin interface. -type Plugin interface { - // Perform plugin setup - Setup() error - - // Execute a plugin command - Cmd(cmd, payload string) (string, error) - - // Execute a plugin hook - Hook(h HookT, payload string) error - - // Perform a plugin file system check - Fsck() error +// NewRecordPre is the payload for the HookNewRecordPre hook. +type NewRecordPre struct { + RecordMetadata backend.RecordMetadata `json:"recordmetadata"` + Metadata []backend.MetadataStream `json:"metadata"` + Files []backend.File `json:"files"` } -type RecordStateT int - -const ( - // Record types - RecordStateInvalid RecordStateT = 0 - RecordStateUnvetted RecordStateT = 1 - RecordStateVetted RecordStateT = 2 -) - -// RecordClient provides an API for plugins to save, retrieve, and delete -// plugin data for a specific record. Editing data is not allowed. -type RecordClient struct { - Token []byte - State RecordStateT - treeID int64 - tlog *tlog +// EncodeNewRecordPre encodes a NewRecordPre into a JSON byte slice. +func EncodeNewRecordPre(nrp NewRecordPre) ([]byte, error) { + return json.Marshal(nrp) } -// hashes and keys must share the same ordering. -func (c *RecordClient) Save(keyPrefix string, blobs, hashes [][]byte, encrypt bool) ([][]byte, error) { - log.Tracef("Save: %x %v %v %x", c.Token, keyPrefix, encrypt, hashes) - - // Ensure tree exists and is not frozen - if !c.tlog.treeExists(c.treeID) { - return nil, errRecordNotFound - } - _, err := c.tlog.freezeRecord(c.treeID) - if err != errFreezeRecordNotFound { - return nil, errTreeIsFrozen - } - - // Encrypt blobs if specified - if encrypt { - for k, v := range blobs { - e, err := c.tlog.encryptionKey.encrypt(0, v) - if err != nil { - return nil, err - } - blobs[k] = e - } - } - - // Save blobs to store - keys, err := c.tlog.store.Put(blobs) +// DecodeNewRecordPre decodes a JSON byte slice into a NewRecordPre. +func DecodeNewRecordPre(payload []byte) (*NewRecordPre, error) { + var nrp NewRecordPre + err := json.Unmarshal(payload, &nrp) if err != nil { - return nil, fmt.Errorf("store Put: %v", err) - } - if len(keys) != len(blobs) { - return nil, fmt.Errorf("wrong number of keys: got %v, want %v", - len(keys), len(blobs)) - } - - // Prepare log leaves. hashes and keys share the same ordering. - leaves := make([]*trillian.LogLeaf, 0, len(blobs)) - for k := range blobs { - pk := []byte(keyPrefix + keys[k]) - leaves = append(leaves, logLeafNew(hashes[k], pk)) - } - - // Append leaves to trillian tree - queued, _, err := c.tlog.trillian.leavesAppend(c.treeID, leaves) - if err != nil { - return nil, fmt.Errorf("leavesAppend: %v", err) - } - if len(queued) != len(leaves) { - return nil, fmt.Errorf("wrong number of queued leaves: got %v, want %v", - len(queued), len(leaves)) - } - failed := make([]string, 0, len(queued)) - for _, v := range queued { - c := codes.Code(v.QueuedLeaf.GetStatus().GetCode()) - if c != codes.OK { - failed = append(failed, fmt.Sprintf("%v", c)) - } - } - if len(failed) > 0 { - return nil, fmt.Errorf("append leaves failed: %v", failed) + return nil, err } - - merkles := make([][]byte, 0, len(blobs)) - for _, v := range queued { - merkles = append(merkles, v.QueuedLeaf.Leaf.MerkleLeafHash) - } - - return merkles, nil + return &nrp, nil } -func (c *RecordClient) Del(merkleHashes [][]byte) error { - log.Tracef("Del: %x %x", c.Token, merkleHashes) - - // Ensure tree exists. We allow blobs to be deleted from both - // frozen and non frozen trees. - if !c.tlog.treeExists(c.treeID) { - return errRecordNotFound - } - - // Get all tree leaves - leaves, err := c.tlog.trillian.leavesAll(c.treeID) - if err != nil { - return err - } - - // Aggregate the key-value store keys for the provided merkle - // hashes. - merkles := make(map[string]struct{}, len(leaves)) - for _, v := range merkleHashes { - merkles[hex.EncodeToString(v)] = struct{}{} - } - keys := make([]string, 0, len(merkles)) - for _, v := range leaves { - _, ok := merkles[hex.EncodeToString(v.MerkleLeafHash)] - if ok { - key, err := extractKeyFromLeaf(v) - if err != nil { - return err - } - keys = append(keys, key) - } - } - - // Delete file blobs from the store - err = c.tlog.store.Del(keys) - if err != nil { - return fmt.Errorf("store Del: %v", err) - } - - return nil +// NewRecordPost is the payload for the HookNewRecordPost hook. +type NewRecordPost struct { + RecordMetadata backend.RecordMetadata `json:"recordmetadata"` + Metadata []backend.MetadataStream `json:"metadata"` + Files []backend.File `json:"files"` } -// If a blob does not exist it will not be included in the returned map. It is -// the responsibility of the caller to check that a blob is returned for each -// of the provided merkle hashes. -func (c *RecordClient) BlobsByMerkleHash(merkleHashes [][]byte) (map[string][]byte, error) { - log.Tracef("BlobsByMerkleHash: %x %x", c.Token, merkleHashes) - - // Get leaves - leavesAll, err := c.tlog.trillian.leavesAll(c.treeID) - if err != nil { - return nil, fmt.Errorf("leavesAll: %v", err) - } - - // Aggregate the leaves that correspond to the provided merkle - // hashes. - // map[merkleHash]*trillian.LogLeaf - leaves := make(map[string]*trillian.LogLeaf, len(merkleHashes)) - for _, v := range merkleHashes { - leaves[hex.EncodeToString(v)] = nil - } - for _, v := range leavesAll { - m := hex.EncodeToString(v.MerkleLeafHash) - if _, ok := leaves[m]; ok { - leaves[m] = v - } - } - - // Ensure a leaf was found for all provided merkle hashes - for k, v := range leaves { - if v == nil { - return nil, fmt.Errorf("leaf not found for merkle hash: %v", k) - } - } - - // Extract the key-value store keys. These keys MUST be put in the - // same order that the merkle hashes were provided in. - keys := make([]string, 0, len(leaves)) - for _, v := range merkleHashes { - l, ok := leaves[hex.EncodeToString(v)] - if !ok { - return nil, fmt.Errorf("leaf not found for merkle hash: %x", v) - } - k, err := extractKeyFromLeaf(l) - if err != nil { - return nil, err - } - keys = append(keys, k) - } - - // Pull the blobs from the store. If is ok if one or more blobs is - // not found. It is the responsibility of the caller to decide how - // this should be handled. - blobs, err := c.tlog.store.Get(keys) - if err != nil { - return nil, fmt.Errorf("store Get: %v", err) - } - - // Decrypt any encrypted blobs - for k, v := range blobs { - if blobIsEncrypted(v) { - b, _, err := c.tlog.encryptionKey.decrypt(v) - if err != nil { - return nil, err - } - blobs[k] = b - } - } - - // Put blobs in a map so the caller can determine if any of the - // provided merkle hashes did not correspond to a blob in the - // store. - b := make(map[string][]byte, len(blobs)) // [merkleHash]blob - for k, v := range keys { - // The merkle hashes slice and keys slice share the same order - merkleHash := hex.EncodeToString(merkleHashes[k]) - blob, ok := blobs[v] - if !ok { - return nil, fmt.Errorf("blob not found for key %v", v) - } - b[merkleHash] = blob - } - - return b, nil +// EncodeNewRecordPost encodes a NewRecordPost into a JSON byte slice. +func EncodeNewRecordPost(nrp NewRecordPost) ([]byte, error) { + return json.Marshal(nrp) } -func (c *RecordClient) BlobsByKeyPrefix(keyPrefix string) ([][]byte, error) { - log.Tracef("BlobsByKeyPrefix: %x %x", c.Token, keyPrefix) - - // Get leaves - leaves, err := c.tlog.trillian.leavesAll(c.treeID) - if err != nil { - return nil, fmt.Errorf("leavesAll: %v", err) - } - - // Walk leaves and aggregate the key-value store keys for all - // leaves with a matching key prefix. - keys := make([]string, 0, len(leaves)) - for _, v := range leaves { - if bytes.HasPrefix(v.ExtraData, []byte(keyPrefix)) { - k, err := extractKeyFromLeaf(v) - if err != nil { - return nil, err - } - keys = append(keys, k) - } - } - - // Pull the blobs from the store - blobs, err := c.tlog.store.Get(keys) +// DecodeNewRecordPost decodes a JSON byte slice into a NewRecordPost. +func DecodeNewRecordPost(payload []byte) (*NewRecordPost, error) { + var nrp NewRecordPost + err := json.Unmarshal(payload, &nrp) if err != nil { - return nil, fmt.Errorf("store Get: %v", err) - } - if len(blobs) != len(keys) { - // One or more blobs were not found - missing := make([]string, 0, len(keys)) - for _, v := range keys { - _, ok := blobs[v] - if !ok { - missing = append(missing, v) - } - } - return nil, fmt.Errorf("blobs not found: %v", missing) + return nil, err } + return &nrp, nil +} - // Decrypt any encrypted blobs - for k, v := range blobs { - if blobIsEncrypted(v) { - b, _, err := c.tlog.encryptionKey.decrypt(v) - if err != nil { - return nil, err - } - blobs[k] = b - } - } +// Plugin provides an API for the tlogbe to use when interacting with plugins. +// All tlogbe plugins must implement the Plugin interface. +type Plugin interface { + // Perform plugin setup + Setup() error - // Covert blobs from map to slice - b := make([][]byte, 0, len(blobs)) - for _, v := range blobs { - b = append(b, v) - } + // Execute a plugin command + Cmd(cmd, payload string) (string, error) - return b, nil -} + // Execute a plugin hook + Hook(h HookT, payload string) error -// TODO implement RecordClient -func (t *Tlogbe) RecordClient(token []byte) (*RecordClient, error) { - return nil, nil + // Perform a plugin file system check + Fsck() error } diff --git a/politeiad/backend/tlogbe/plugins/pi.go b/politeiad/backend/tlogbe/plugins/pi.go new file mode 100644 index 000000000..52e65cda4 --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/pi.go @@ -0,0 +1,79 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package plugins + +import ( + "encoding/base64" + + "github.com/decred/politeia/plugins/pi" + "github.com/decred/politeia/politeiad/backend/tlogbe" +) + +var ( + _ tlogbe.Plugin = (*piPlugin)(nil) +) + +type piPlugin struct{} + +func (p *piPlugin) Setup() error { + log.Tracef("pi Setup") + + return nil +} + +func (p *piPlugin) Cmd(cmd, payload string) (string, error) { + log.Tracef("pi Cmd: %v %v", cmd, payload) + + return "", nil +} + +func (p *piPlugin) hookNewRecordPre(payload string) error { + nrp, err := tlogbe.DecodeNewRecordPre([]byte(payload)) + if err != nil { + return err + } + + // Decode the ProposalMetadata + // TODO pickup here + var pm *pi.ProposalMetadata + for _, v := range nrp.Files { + if v.Name == pi.FilenameProposalMetadata { + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return err + } + pm, err = pi.DecodeProposalMetadata(b) + if err != nil { + return err + } + break + } + } + if pm == nil { + } + + return nil +} + +func (p *piPlugin) Hook(h tlogbe.HookT, payload string) error { + log.Tracef("pi Hook: %v", tlogbe.Hooks[h]) + + switch h { + case tlogbe.HookNewRecordPre: + return p.hookNewRecordPre(payload) + } + + return nil +} + +func (p *piPlugin) Fsck() error { + log.Tracef("pi Fsck") + + return nil +} + +func NewPiPlugin() *piPlugin { + return &piPlugin{} +} diff --git a/politeiad/backend/tlogbe/plugins/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote.go index d3150f45b..1aa7f4e20 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote.go @@ -602,7 +602,7 @@ func (p *ticketVotePlugin) bestBlock() (uint32, error) { if err != nil { return 0, err } - _, reply, err := p.backend.Plugin(dcrdata.ID, + reply, err := p.backend.Plugin(dcrdata.ID, dcrdata.CmdBestBlock, string(payload)) if err != nil { return 0, fmt.Errorf("Plugin %v %v: %v", @@ -637,7 +637,7 @@ func (p *ticketVotePlugin) bestBlockUnsafe() (uint32, error) { if err != nil { return 0, err } - _, reply, err := p.backend.Plugin(dcrdata.ID, + reply, err := p.backend.Plugin(dcrdata.ID, dcrdata.CmdBestBlock, string(payload)) if err != nil { return 0, fmt.Errorf("Plugin %v %v: %v", @@ -671,7 +671,7 @@ func (p *ticketVotePlugin) largestCommitmentAddrs(tickets []string) ([]commitmen if err != nil { return nil, err } - _, reply, err := p.backend.Plugin(dcrdata.ID, + reply, err := p.backend.Plugin(dcrdata.ID, dcrdata.CmdTxsTrimmed, string(payload)) if err != nil { return nil, fmt.Errorf("Plugin %v %v: %v", @@ -740,7 +740,7 @@ func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, if err != nil { return nil, err } - _, reply, err := p.backend.Plugin(dcrdata.ID, + reply, err := p.backend.Plugin(dcrdata.ID, dcrdata.CmdBlockDetails, string(payload)) if err != nil { return nil, fmt.Errorf("Plugin %v %v: %v", @@ -763,7 +763,7 @@ func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, if err != nil { return nil, err } - _, reply, err = p.backend.Plugin(dcrdata.ID, + reply, err = p.backend.Plugin(dcrdata.ID, dcrdata.CmdTicketPool, string(payload)) if err != nil { return nil, fmt.Errorf("Plugin %v %v: %v", diff --git a/politeiad/backend/tlogbe/recordclient.go b/politeiad/backend/tlogbe/recordclient.go new file mode 100644 index 000000000..f65c82b93 --- /dev/null +++ b/politeiad/backend/tlogbe/recordclient.go @@ -0,0 +1,294 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlogbe + +import ( + "bytes" + "encoding/hex" + "fmt" + + "github.com/google/trillian" + "google.golang.org/grpc/codes" +) + +type RecordStateT int + +const ( + // Record types + RecordStateInvalid RecordStateT = 0 + RecordStateUnvetted RecordStateT = 1 + RecordStateVetted RecordStateT = 2 +) + +// TODO if we make this an interface it will make testing the plugins a whole +// lot easier. Or better yet, make tlog and interface. + +// RecordClient provides an API for plugins to save, retrieve, and delete +// plugin data for a specific record. Editing data is not allowed. +type RecordClient struct { + Token []byte + State RecordStateT + treeID int64 + tlog *tlog +} + +// hashes and keys must share the same ordering. +func (c *RecordClient) Save(keyPrefix string, blobs, hashes [][]byte, encrypt bool) ([][]byte, error) { + log.Tracef("Save: %x %v %v %x", c.Token, keyPrefix, encrypt, hashes) + + // Ensure tree exists and is not frozen + if !c.tlog.treeExists(c.treeID) { + return nil, errRecordNotFound + } + _, err := c.tlog.freezeRecord(c.treeID) + if err != errFreezeRecordNotFound { + return nil, errTreeIsFrozen + } + + // Encrypt blobs if specified + if encrypt { + for k, v := range blobs { + e, err := c.tlog.encryptionKey.encrypt(0, v) + if err != nil { + return nil, err + } + blobs[k] = e + } + } + + // Save blobs to store + keys, err := c.tlog.store.Put(blobs) + if err != nil { + return nil, fmt.Errorf("store Put: %v", err) + } + if len(keys) != len(blobs) { + return nil, fmt.Errorf("wrong number of keys: got %v, want %v", + len(keys), len(blobs)) + } + + // Prepare log leaves. hashes and keys share the same ordering. + leaves := make([]*trillian.LogLeaf, 0, len(blobs)) + for k := range blobs { + pk := []byte(keyPrefix + keys[k]) + leaves = append(leaves, logLeafNew(hashes[k], pk)) + } + + // Append leaves to trillian tree + queued, _, err := c.tlog.trillian.leavesAppend(c.treeID, leaves) + if err != nil { + return nil, fmt.Errorf("leavesAppend: %v", err) + } + if len(queued) != len(leaves) { + return nil, fmt.Errorf("wrong number of queued leaves: got %v, want %v", + len(queued), len(leaves)) + } + failed := make([]string, 0, len(queued)) + for _, v := range queued { + c := codes.Code(v.QueuedLeaf.GetStatus().GetCode()) + if c != codes.OK { + failed = append(failed, fmt.Sprintf("%v", c)) + } + } + if len(failed) > 0 { + return nil, fmt.Errorf("append leaves failed: %v", failed) + } + + merkles := make([][]byte, 0, len(blobs)) + for _, v := range queued { + merkles = append(merkles, v.QueuedLeaf.Leaf.MerkleLeafHash) + } + + return merkles, nil +} + +func (c *RecordClient) Del(merkleHashes [][]byte) error { + log.Tracef("Del: %x %x", c.Token, merkleHashes) + + // Ensure tree exists. We allow blobs to be deleted from both + // frozen and non frozen trees. + if !c.tlog.treeExists(c.treeID) { + return errRecordNotFound + } + + // Get all tree leaves + leaves, err := c.tlog.trillian.leavesAll(c.treeID) + if err != nil { + return err + } + + // Aggregate the key-value store keys for the provided merkle + // hashes. + merkles := make(map[string]struct{}, len(leaves)) + for _, v := range merkleHashes { + merkles[hex.EncodeToString(v)] = struct{}{} + } + keys := make([]string, 0, len(merkles)) + for _, v := range leaves { + _, ok := merkles[hex.EncodeToString(v.MerkleLeafHash)] + if ok { + key, err := extractKeyFromLeaf(v) + if err != nil { + return err + } + keys = append(keys, key) + } + } + + // Delete file blobs from the store + err = c.tlog.store.Del(keys) + if err != nil { + return fmt.Errorf("store Del: %v", err) + } + + return nil +} + +// If a blob does not exist it will not be included in the returned map. It is +// the responsibility of the caller to check that a blob is returned for each +// of the provided merkle hashes. +func (c *RecordClient) BlobsByMerkleHash(merkleHashes [][]byte) (map[string][]byte, error) { + log.Tracef("BlobsByMerkleHash: %x %x", c.Token, merkleHashes) + + // Get leaves + leavesAll, err := c.tlog.trillian.leavesAll(c.treeID) + if err != nil { + return nil, fmt.Errorf("leavesAll: %v", err) + } + + // Aggregate the leaves that correspond to the provided merkle + // hashes. + // map[merkleHash]*trillian.LogLeaf + leaves := make(map[string]*trillian.LogLeaf, len(merkleHashes)) + for _, v := range merkleHashes { + leaves[hex.EncodeToString(v)] = nil + } + for _, v := range leavesAll { + m := hex.EncodeToString(v.MerkleLeafHash) + if _, ok := leaves[m]; ok { + leaves[m] = v + } + } + + // Ensure a leaf was found for all provided merkle hashes + for k, v := range leaves { + if v == nil { + return nil, fmt.Errorf("leaf not found for merkle hash: %v", k) + } + } + + // Extract the key-value store keys. These keys MUST be put in the + // same order that the merkle hashes were provided in. + keys := make([]string, 0, len(leaves)) + for _, v := range merkleHashes { + l, ok := leaves[hex.EncodeToString(v)] + if !ok { + return nil, fmt.Errorf("leaf not found for merkle hash: %x", v) + } + k, err := extractKeyFromLeaf(l) + if err != nil { + return nil, err + } + keys = append(keys, k) + } + + // Pull the blobs from the store. If is ok if one or more blobs is + // not found. It is the responsibility of the caller to decide how + // this should be handled. + blobs, err := c.tlog.store.Get(keys) + if err != nil { + return nil, fmt.Errorf("store Get: %v", err) + } + + // Decrypt any encrypted blobs + for k, v := range blobs { + if blobIsEncrypted(v) { + b, _, err := c.tlog.encryptionKey.decrypt(v) + if err != nil { + return nil, err + } + blobs[k] = b + } + } + + // Put blobs in a map so the caller can determine if any of the + // provided merkle hashes did not correspond to a blob in the + // store. + b := make(map[string][]byte, len(blobs)) // [merkleHash]blob + for k, v := range keys { + // The merkle hashes slice and keys slice share the same order + merkleHash := hex.EncodeToString(merkleHashes[k]) + blob, ok := blobs[v] + if !ok { + return nil, fmt.Errorf("blob not found for key %v", v) + } + b[merkleHash] = blob + } + + return b, nil +} + +func (c *RecordClient) BlobsByKeyPrefix(keyPrefix string) ([][]byte, error) { + log.Tracef("BlobsByKeyPrefix: %x %x", c.Token, keyPrefix) + + // Get leaves + leaves, err := c.tlog.trillian.leavesAll(c.treeID) + if err != nil { + return nil, fmt.Errorf("leavesAll: %v", err) + } + + // Walk leaves and aggregate the key-value store keys for all + // leaves with a matching key prefix. + keys := make([]string, 0, len(leaves)) + for _, v := range leaves { + if bytes.HasPrefix(v.ExtraData, []byte(keyPrefix)) { + k, err := extractKeyFromLeaf(v) + if err != nil { + return nil, err + } + keys = append(keys, k) + } + } + + // Pull the blobs from the store + blobs, err := c.tlog.store.Get(keys) + if err != nil { + return nil, fmt.Errorf("store Get: %v", err) + } + if len(blobs) != len(keys) { + // One or more blobs were not found + missing := make([]string, 0, len(keys)) + for _, v := range keys { + _, ok := blobs[v] + if !ok { + missing = append(missing, v) + } + } + return nil, fmt.Errorf("blobs not found: %v", missing) + } + + // Decrypt any encrypted blobs + for k, v := range blobs { + if blobIsEncrypted(v) { + b, _, err := c.tlog.encryptionKey.decrypt(v) + if err != nil { + return nil, err + } + blobs[k] = b + } + } + + // Covert blobs from map to slice + b := make([][]byte, 0, len(blobs)) + for _, v := range blobs { + b = append(b, v) + } + + return b, nil +} + +// TODO implement RecordClient +func (t *Tlogbe) RecordClient(token []byte) (*RecordClient, error) { + return nil, nil +} diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 5ba6262c8..ad265766c 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -490,6 +490,12 @@ func (t *Tlogbe) New(metadata []backend.MetadataStream, files []backend.File) (* // Check for token prefix collisions if !t.prefixExists(token) { // Not a collision. Use this token. + + // Update the prefix cache. This must be done even if the + // record creation fails since the tree will still exist in + // tlog. + t.prefixAdd(token) + break } @@ -503,14 +509,43 @@ func (t *Tlogbe) New(metadata []backend.MetadataStream, files []backend.File) (* return nil, err } + // Call pre plugin hook + pre := NewRecordPre{ + RecordMetadata: *rm, + Metadata: metadata, + Files: files, + } + b, err := EncodeNewRecordPre(pre) + if err != nil { + return nil, err + } + err = t.pluginHook(HookNewRecordPre, string(b)) + if err != nil { + return nil, fmt.Errorf("pluginHook %v: %v", + Hooks[HookNewRecordPre], err) + } + // Save the record err = t.unvetted.recordSave(treeID, *rm, metadata, files) if err != nil { return nil, fmt.Errorf("recordSave %x: %v", token, err) } - // Update the prefix cache - t.prefixAdd(token) + // Call post plugin hook + post := NewRecordPost{ + RecordMetadata: *rm, + Metadata: metadata, + Files: files, + } + b, err = EncodeNewRecordPost(post) + if err != nil { + return nil, err + } + err = t.pluginHook(HookNewRecordPost, string(b)) + if err != nil { + return nil, fmt.Errorf("pluginHook %v: %v", + Hooks[HookNewRecordPost], err) + } // Update the inventory cache t.inventoryAdd(hex.EncodeToString(token), backend.MDStatusUnvetted) @@ -1182,6 +1217,21 @@ func (t *Tlogbe) InventoryByStatus() (*backend.InventoryByStatus, error) { }, nil } +func (t *Tlogbe) pluginHook(h HookT, payload string) error { + t.RLock() + defer t.RUnlock() + + // Pass hook event and payload to each plugin + for _, v := range t.plugins { + err := v.ctx.Hook(h, payload) + if err != nil { + return fmt.Errorf("Hook %v: %v", v.id, err) + } + } + + return nil +} + // GetPlugins returns the backend plugins that have been registered and their // settings. // @@ -1204,26 +1254,26 @@ func (t *Tlogbe) GetPlugins() ([]backend.Plugin, error) { // Plugin is a pass-through function for plugin commands. // // This function satisfies the Backend interface. -func (t *Tlogbe) Plugin(pluginID, command, payload string) (string, string, error) { +func (t *Tlogbe) Plugin(pluginID, command, payload string) (string, error) { log.Tracef("Plugin: %v", command) if t.isShutdown() { - return "", "", backend.ErrShutdown + return "", backend.ErrShutdown } // Get plugin plugin, ok := t.plugins[pluginID] if !ok { - return "", "", backend.ErrPluginInvalid + return "", backend.ErrPluginInvalid } // Execute plugin command reply, err := plugin.ctx.Cmd(command, payload) if err != nil { - return "", "", err + return "", err } - return command, reply, nil + return reply, nil } // Close shuts the backend down and performs cleanup. diff --git a/politeiawww/api/www/v1/v1.go b/politeiawww/api/www/v1/v1.go index cbe1e659f..01c92ce54 100644 --- a/politeiawww/api/www/v1/v1.go +++ b/politeiawww/api/www/v1/v1.go @@ -56,7 +56,7 @@ const ( RouteProposalPaywallDetails = "/proposals/paywall" RouteProposalPaywallPayment = "/proposals/paywallpayment" - // The following routes have been DEPRECATED. Use the v2 routes. + // The following routes have been DEPRECATED. Use the pi v1 API. RouteTokenInventory = "/proposals/tokeninventory" RouteBatchProposals = "/proposals/batch" RouteBatchVoteSummary = "/proposals/batchvotesummary" diff --git a/politeiawww/eventmanager.go b/politeiawww/eventmanager.go index 16159cd64..f0476ef20 100644 --- a/politeiawww/eventmanager.go +++ b/politeiawww/eventmanager.go @@ -85,8 +85,7 @@ func notificationIsSet(u user.User, n www.EmailNotificationT) bool { return false } - bit := uint64(n) - if u.EmailNotifications&bit == 0 { + if u.EmailNotifications&uint64(n) == 0 { // Notification bit not set return false } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index bb4783e60..325bb4a94 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -347,7 +347,7 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu } } - // Verify that an index file is present. + // Verify that an index file is present if !foundIndexFile { e := fmt.Sprintf("%v file not found", www.PolicyIndexFilename) return nil, www.UserError{ From 87cb7fa920a2e594997ec0d8fcf741b14a29a845 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 31 Aug 2020 10:09:15 -0500 Subject: [PATCH 025/449] event bug fixes --- politeiawww/eventmanager.go | 11 ++++++----- politeiawww/piwww.go | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/politeiawww/eventmanager.go b/politeiawww/eventmanager.go index f0476ef20..d6277450a 100644 --- a/politeiawww/eventmanager.go +++ b/politeiawww/eventmanager.go @@ -27,7 +27,8 @@ type eventManager struct { listeners map[eventT][]chan interface{} } -// register adds adds a listener for the given event type. +// register registers an event listener (channel) to listen for the provided +// event type. func (e *eventManager) register(event eventT, listener chan interface{}) { e.Lock() defer e.Unlock() @@ -41,9 +42,9 @@ func (e *eventManager) register(event eventT, listener chan interface{}) { e.listeners[event] = l } -// fire fires off an event by passing it to all channels that have been -// registered to listen for the event. -func (e *eventManager) fire(event eventT, data interface{}) { +// emit emits an event by passing it to all channels that have been registered +// to listen for the event. +func (e *eventManager) emit(event eventT, data interface{}) { e.Lock() defer e.Unlock() @@ -123,7 +124,7 @@ func (p *politeiawww) handleEventProposalSubmitted(ch chan interface{}) { }) if err != nil { log.Errorf("handleEventProposalSubmitted: AllUsers: %v", err) - continue + return } // Send email notification diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 325bb4a94..4e3cc5cc2 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -544,7 +544,7 @@ func (p *politeiawww) processProposalNew(pn pi.ProposalNew, usr user.User) (*pi. } // Fire off a new proposal event - p.eventManager.fire(eventProposalSubmitted, + p.eventManager.emit(eventProposalSubmitted, dataProposalSubmitted{ token: cr.Token, name: pm.Name, From 2679a5515b650268c7a56fa59306775840a5a6b6 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 1 Sep 2020 10:06:20 -0500 Subject: [PATCH 026/449] tlogbe pi plugin hooks --- plugins/comments/comments.go | 6 +- plugins/dcrdata/dcrdata.go | 2 + plugins/pi/pi.go | 81 ++- plugins/ticketvote/ticketvote.go | 6 +- politeiad/backend/tlogbe/plugin.go | 54 ++ politeiad/backend/tlogbe/plugins/comments.go | 4 +- politeiad/backend/tlogbe/plugins/pi.go | 296 ++++++++++- .../backend/tlogbe/plugins/ticketvote.go | 21 +- politeiad/backend/tlogbe/recordclient.go | 2 +- politeiad/backend/tlogbe/tlog.go | 2 + politeiad/backend/tlogbe/tlogbe.go | 169 ++++-- politeiawww/piwww.go | 2 +- politeiawww/proposals.go | 486 +----------------- 13 files changed, 525 insertions(+), 606 deletions(-) diff --git a/plugins/comments/comments.go b/plugins/comments/comments.go index ef474bbb0..a020c724f 100644 --- a/plugins/comments/comments.go +++ b/plugins/comments/comments.go @@ -2,6 +2,8 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. +// Package comments provides a plugin for adding comment functionality to +// records. package comments import ( @@ -71,7 +73,7 @@ var ( } ) -// UserError represents an error that is cause by something that the user did. +// UserError represents an error that is cause by the user. type UserError struct { ErrorCode ErrorStatusT ErrorContext []string @@ -79,7 +81,7 @@ type UserError struct { // Error satisfies the error interface. func (e UserError) Error() string { - return fmt.Sprintf("comments error code: %v", e.ErrorCode) + return fmt.Sprintf("comments plugin error code: %v", e.ErrorCode) } // Comment represent a record comment. diff --git a/plugins/dcrdata/dcrdata.go b/plugins/dcrdata/dcrdata.go index 83cdb8001..d3757fcf7 100644 --- a/plugins/dcrdata/dcrdata.go +++ b/plugins/dcrdata/dcrdata.go @@ -2,6 +2,8 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. +// Package dcrdata provides a plugin for retrieving data from the dcrdata block +// explorer. package dcrdata import ( diff --git a/plugins/pi/pi.go b/plugins/pi/pi.go index 8128ac594..13d58c49d 100644 --- a/plugins/pi/pi.go +++ b/plugins/pi/pi.go @@ -2,16 +2,24 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. +// Package pi provides a plugin for functionality that is specific to decred's +// proposal system. package pi -import "encoding/json" +import ( + "encoding/json" + "fmt" +) + +type ErrorStatusT int const ( Version uint32 = 1 ID = "pi" // Plugin commands - CmdLinkedFrom = "linkedfrom" // Get linked from lists + CmdProposalDetails = "proposaldetails" + CmdProposals = "proposals" // Metadata stream IDs. All metadata streams in this plugin will // use 1xx numbering. @@ -22,9 +30,32 @@ const ( // file that is saved to politeiad. ProposalMetadata is saved to // politeiad as a file, not as a metadata stream, since it needs to // be included in the merkle root that politeiad signs. - FilenameProposalMetadata = "proposalmd.json" + FilenameProposalMetadata = "proposalmetadata.json" + + // User error status codes + ErrorStatusInvalid ErrorStatusT = 0 + ErrorStatusLinkToInvalid ErrorStatusT = 1 ) +var ( + // ErrorStatus contains human readable user error statuses. + ErrorStatus = map[ErrorStatusT]string{ + ErrorStatusInvalid: "error status invalid", + ErrorStatusLinkToInvalid: "linkto invalid", + } +) + +// UserError represents an error that is caused by the user. +type UserError struct { + ErrorCode ErrorStatusT + ErrorContext []string +} + +// Error satisfies the error interface. +func (e UserError) Error() string { + return fmt.Sprintf("pi plugin error code: %v", e.ErrorCode) +} + // ProposalMetadata contains proposal metadata that is provided by the user on // proposal submission. ProposalMetadata is saved to politeiad as a file, not // as a metadata stream, since it needs to be included in the merkle root that @@ -96,47 +127,3 @@ type StatusChange struct { Signature string `json:"signature"` Timestamp int64 `json:"timestamp"` } - -// LinkedFrom retrieves the linked from list for each of the provided proposal -// tokens. A linked from list is a list of all the proposals that have linked -// to a given proposal using the LinkTo field in the ProposalMetadata mdstream. -// If a token does not correspond to an actual proposal then it will not be -// included in the returned map. -type LinkedFrom struct { - Tokens []string `json:"tokens"` -} - -// EncodeLinkedFrom encodes a LinkedFrom into a JSON byte slice. -func EncodeLinkedFrom(lf LinkedFrom) ([]byte, error) { - return json.Marshal(lf) -} - -// DecodeLinkedFrom decodes a JSON byte slice into a LinkedFrom. -func DecodeLinkedFrom(payload []byte) (*LinkedFrom, error) { - var lf LinkedFrom - err := json.Unmarshal(payload, &lf) - if err != nil { - return nil, err - } - return &lf, nil -} - -// LinkedFromReply is the reply to the LinkedFrom command. -type LinkedFromReply struct { - LinkedFrom map[string][]string `json:"linkedfrom"` -} - -// EncodeLinkedFromReply encodes a LinkedFromReply into a JSON byte slice. -func EncodeLinkedFromReply(lfr LinkedFromReply) ([]byte, error) { - return json.Marshal(lfr) -} - -// DecodeLinkedFromReply decodes a JSON byte slice into a LinkedFrom. -func DecodeLinkedFromReply(payload []byte) (*LinkedFromReply, error) { - var lfr LinkedFromReply - err := json.Unmarshal(payload, &lfr) - if err != nil { - return nil, err - } - return &lfr, nil -} diff --git a/plugins/ticketvote/ticketvote.go b/plugins/ticketvote/ticketvote.go index 1121ba793..d614ee76b 100644 --- a/plugins/ticketvote/ticketvote.go +++ b/plugins/ticketvote/ticketvote.go @@ -2,6 +2,8 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. +// Package ticketvote provides a plugin for creating and managing votes that +// require decred tickets to participate. package ticketvote import ( @@ -130,7 +132,7 @@ var ( } ) -// UserError represents an error that is cause by something that the user did. +// UserError represents an error that is caused by the user. type UserError struct { ErrorCode ErrorStatusT ErrorContext []string @@ -138,7 +140,7 @@ type UserError struct { // Error satisfies the error interface. func (e UserError) Error() string { - return fmt.Sprintf("ticketvote error code: %v", e.ErrorCode) + return fmt.Sprintf("ticketvote plugin error code: %v", e.ErrorCode) } // AuthorizeVote is the structure that is saved to disk when a vote is diff --git a/politeiad/backend/tlogbe/plugin.go b/politeiad/backend/tlogbe/plugin.go index 279a35d36..e36579ccd 100644 --- a/politeiad/backend/tlogbe/plugin.go +++ b/politeiad/backend/tlogbe/plugin.go @@ -83,6 +83,60 @@ func DecodeNewRecordPost(payload []byte) (*NewRecordPost, error) { return &nrp, nil } +// SetRecordStatusPre is the payload for the HookSetRecordStatusPre hook. +type SetRecordStatusPre struct { + Record backend.Record `json:"record"` // Current record + + // Updated fields + RecordMetadata backend.RecordMetadata `json:"recordmetadata"` + MDAppend []backend.MetadataStream `json:"mdappend"` + MDOverwrite []backend.MetadataStream `json:"mdoverwrite"` +} + +// EncodeSetRecordStatusPre encodes a SetRecordStatusPre into a JSON byte +// slice. +func EncodeSetRecordStatusPre(srsp SetRecordStatusPre) ([]byte, error) { + return json.Marshal(srsp) +} + +// DecodeSetRecordStatusPre decodes a JSON byte slice into a +// SetRecordStatusPre. +func DecodeSetRecordStatusPre(payload []byte) (*SetRecordStatusPre, error) { + var srsp SetRecordStatusPre + err := json.Unmarshal(payload, &srsp) + if err != nil { + return nil, err + } + return &srsp, nil +} + +// SetRecordStatusPost is the payload for the HookSetRecordStatusPost hook. +type SetRecordStatusPost struct { + Record backend.Record `json:"record"` // Current record + + // Updated fields + RecordMetadata backend.RecordMetadata `json:"recordmetadata"` + MDAppend []backend.MetadataStream `json:"mdappend"` + MDOverwrite []backend.MetadataStream `json:"mdoverwrite"` +} + +// EncodeSetRecordStatusPost encodes a SetRecordStatusPost into a JSON byte +// slice. +func EncodeSetRecordStatusPost(srsp SetRecordStatusPost) ([]byte, error) { + return json.Marshal(srsp) +} + +// DecodeSetRecordStatusPost decodes a JSON byte slice into a +// SetRecordStatusPost. +func DecodeSetRecordStatusPost(payload []byte) (*SetRecordStatusPost, error) { + var srsp SetRecordStatusPost + err := json.Unmarshal(payload, &srsp) + if err != nil { + return nil, err + } + return &srsp, nil +} + // Plugin provides an API for the tlogbe to use when interacting with plugins. // All tlogbe plugins must implement the Plugin interface. type Plugin interface { diff --git a/politeiad/backend/tlogbe/plugins/comments.go b/politeiad/backend/tlogbe/plugins/comments.go index 769e52208..50599eb80 100644 --- a/politeiad/backend/tlogbe/plugins/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments.go @@ -50,7 +50,7 @@ var ( type commentsPlugin struct { sync.Mutex id *identity.FullIdentity - backend *tlogbe.Tlogbe + backend *tlogbe.TlogBackend // Mutexes contains a mutex for each record. The mutexes are lazy // loaded. @@ -1502,7 +1502,7 @@ func (p *commentsPlugin) Setup() error { } // NewCommentsPlugin returns a new comments plugin. -func NewCommentsPlugin(backend *tlogbe.Tlogbe, settings []backend.PluginSetting) *commentsPlugin { +func NewCommentsPlugin(backend *tlogbe.TlogBackend, settings []backend.PluginSetting) *commentsPlugin { // TODO these should be passed in as plugin settings id := &identity.FullIdentity{} diff --git a/politeiad/backend/tlogbe/plugins/pi.go b/politeiad/backend/tlogbe/plugins/pi.go index 52e65cda4..fa69b1a58 100644 --- a/politeiad/backend/tlogbe/plugins/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi.go @@ -6,16 +6,171 @@ package plugins import ( "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "sync" + "time" "github.com/decred/politeia/plugins/pi" + "github.com/decred/politeia/plugins/ticketvote" + "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe" ) +const ( + // Filenames of memoized data saved to the data dir. + filenameLinkedFrom = "{token}-linkedfrom.json" +) + var ( _ tlogbe.Plugin = (*piPlugin)(nil) ) -type piPlugin struct{} +// piPlugin satisfies the Plugin interface. +type piPlugin struct { + sync.Mutex + backend *tlogbe.TlogBackend + + // dataDir is the pi plugin data directory. The only data that is + // stored here is cached data that can be re-created at any time + // by walking the trillian trees. + dataDir string +} + +func isRFP(pm pi.ProposalMetadata) bool { + return pm.LinkBy != 0 +} + +// proposalMetadataFromFiles parses and returns the ProposalMetadata from the +// provided files. If a ProposalMetadata is not found, nil is returned. +func proposalMetadataFromFiles(files []backend.File) (*pi.ProposalMetadata, error) { + var pm *pi.ProposalMetadata + for _, v := range files { + if v.Name == pi.FilenameProposalMetadata { + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return nil, err + } + pm, err = pi.DecodeProposalMetadata(b) + if err != nil { + return nil, err + } + } + } + return pm, nil +} + +// TODO saving the proposalLinkedFrom to the filesystem is not scalable between +// multiple politeiad instances. The plugin needs to have a tree that can be +// used to share state between the different politeiad instances. + +// proposalLinkedFrom is the the structure that is updated and cached for +// proposal A when proposal B links to proposal A. The list contains all +// proposals that have linked to proposal A. The linked from list will only +// contain public proposals. +// +// Example: an RFP proposal's linked from list will contain all public RFP +// submissions since they have all linked to the RFP proposal. +type proposalLinkedFrom struct { + Tokens map[string]struct{} `json:"tokens"` +} + +func (p *piPlugin) cachedLinkedFromPath(token string) string { + fn := strings.Replace(filenameLinkedFrom, "{token}", token, 1) + return filepath.Join(p.dataDir, fn) +} + +// This function must be called WITH the lock held. +func (p *piPlugin) cachedLinkedFromLocked(token string) (*proposalLinkedFrom, error) { + fp := p.cachedLinkedFromPath(token) + b, err := ioutil.ReadFile(fp) + if err != nil { + if errors.As(err, &e) && !os.IsExist(err) { + // File does't exist + return nil, errRecordNotFound + } + } + + var plf proposalLinkedFrom + err = json.Unmarshal(b, &plf) + if err != nil { + return nil, err + } + + return &plf, nil +} + +func (p *piPlugin) cachedLinkedFrom(token string) (*proposalLinkedFrom, error) { + p.Lock() + defer p.Unlock() + + return p.cachedLinkedFromLocked(token) +} + +func (p *piPlugin) cachedLinkedFromAdd(parentToken, childToken string) error { + p.Lock() + defer p.Unlock() + + // Get existing linked from list + plf, err := p.cachedLinkedFromLocked(parentToken) + if err == errRecordNotFound { + // List doesn't exist. Create a new one. + plf = &proposalLinkedFrom{ + Tokens: make(map[string]struct{}, 0), + } + } else if err != nil { + return fmt.Errorf("cachedLinkedFromLocked %v: %v", parentToken, err) + } + + // Update list + plf.Tokens[childToken] = struct{}{} + + // Save list + b, err := json.Marshal(plf) + if err != nil { + return err + } + fp := p.cachedLinkedFromPath(parentToken) + err = ioutil.WriteFile(fp, b, 0664) + if err != nil { + return fmt.Errorf("WriteFile: %v", err) + } + + return nil +} + +func (p *piPlugin) cachedLinkedFromDel(parentToken, childToken string) error { + p.Lock() + defer p.Unlock() + + // Get existing linked from list + plf, err := p.cachedLinkedFromLocked(parentToken) + if err != nil { + return fmt.Errorf("cachedLinkedFromLocked %v: %v", parentToken, err) + } + + // Update list + delete(plf.Tokens, childToken) + + // Save list + b, err := json.Marshal(plf) + if err != nil { + return err + } + fp := p.cachedLinkedFromPath(parentToken) + err = ioutil.WriteFile(fp, b, 0664) + if err != nil { + return fmt.Errorf("WriteFile: %v", err) + } + + return nil +} func (p *piPlugin) Setup() error { log.Tracef("pi Setup") @@ -35,8 +190,7 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { return err } - // Decode the ProposalMetadata - // TODO pickup here + // Decode ProposalMetadata var pm *pi.ProposalMetadata for _, v := range nrp.Files { if v.Name == pi.FilenameProposalMetadata { @@ -52,6 +206,127 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { } } if pm == nil { + return fmt.Errorf("proposal metadata not found") + } + + // Verify the linkto is an RFP and that the RFP is eligible to be + // linked to. We currently only allow linking to RFP proposals that + // have been approved by a ticket vote. + if pm.LinkTo != "" { + if isRFP(*pm) { + return pi.UserError{ + ErrorCode: pi.ErrorStatusLinkToInvalid, + ErrorContext: []string{"an rfp cannot have linkto set"}, + } + } + tokenb, err := hex.DecodeString(pm.LinkTo) + if err != nil { + return pi.UserError{ + ErrorCode: pi.ErrorStatusLinkToInvalid, + ErrorContext: []string{"invalid hex"}, + } + } + r, err := p.backend.GetVetted(tokenb, "") + if err != nil { + if err == backend.ErrRecordNotFound { + return pi.UserError{ + ErrorCode: pi.ErrorStatusLinkToInvalid, + ErrorContext: []string{"proposal not found"}, + } + } + return err + } + linkToPM, err := proposalMetadataFromFiles(r.Files) + if err != nil { + return err + } + if linkToPM == nil { + return pi.UserError{ + ErrorCode: pi.ErrorStatusLinkToInvalid, + ErrorContext: []string{"proposal not an rfp"}, + } + } + if !isRFP(*linkToPM) { + return pi.UserError{ + ErrorCode: pi.ErrorStatusLinkToInvalid, + ErrorContext: []string{"proposal not an rfp"}, + } + } + if time.Now().Unix() > linkToPM.LinkBy { + // Link by deadline has expired. New links are not allowed. + return pi.UserError{ + ErrorCode: pi.ErrorStatusLinkToInvalid, + ErrorContext: []string{"rfp link by deadline expired"}, + } + } + s := ticketvote.Summaries{ + Tokens: []string{pm.LinkTo}, + } + b, err := ticketvote.EncodeSummaries(s) + if err != nil { + return err + } + reply, err := p.backend.Plugin(ticketvote.ID, + ticketvote.CmdSummaries, string(b)) + if err != nil { + return fmt.Errorf("Plugin %v %v: %v", + ticketvote.ID, ticketvote.CmdSummaries, err) + } + sr, err := ticketvote.DecodeSummariesReply([]byte(reply)) + if err != nil { + return err + } + summary, ok := sr.Summaries[pm.LinkTo] + if !ok { + return fmt.Errorf("summary not found %v", pm.LinkTo) + } + if !summary.Approved { + return pi.UserError{ + ErrorCode: pi.ErrorStatusLinkToInvalid, + ErrorContext: []string{"rfp vote not approved"}, + } + } + } + + return nil +} + +func (p *piPlugin) hookSetRecordStatusPost(payload string) error { + srsp, err := tlogbe.DecodeSetRecordStatusPost([]byte(payload)) + if err != nil { + return err + } + + // If the LinkTo field has been set then the proposalLinkedFrom + // list might need to be updated for the proposal that is being + // linked to, depending on the status change that is being made. + pm, err := proposalMetadataFromFiles(srsp.Record.Files) + if err != nil { + return err + } + if pm != nil && pm.LinkTo != "" { + // Link from has been set. Check if the status change requires + // the parent proposal's linked from list to be updated. + var ( + parentToken = pm.LinkTo + childToken = srsp.RecordMetadata.Token + ) + switch srsp.RecordMetadata.Status { + case backend.MDStatusVetted: + // Proposal has been made public. Add child token to parent + // token's linked from list. + err := p.cachedLinkedFromAdd(parentToken, childToken) + if err != nil { + return fmt.Errorf("cachedLinkedFromAdd: %v", err) + } + case backend.MDStatusCensored: + // Proposal has been censored. Delete child token from parent + // token's linked from list. + err := p.cachedLinkedFromDel(parentToken, childToken) + if err != nil { + return fmt.Errorf("cachedLinkedFromDel: %v", err) + } + } } return nil @@ -63,6 +338,8 @@ func (p *piPlugin) Hook(h tlogbe.HookT, payload string) error { switch h { case tlogbe.HookNewRecordPre: return p.hookNewRecordPre(payload) + case tlogbe.HookSetRecordStatusPost: + return p.hookSetRecordStatusPost(payload) } return nil @@ -71,9 +348,18 @@ func (p *piPlugin) Hook(h tlogbe.HookT, payload string) error { func (p *piPlugin) Fsck() error { log.Tracef("pi Fsck") + // proposalLinkedFrom cache + return nil } -func NewPiPlugin() *piPlugin { - return &piPlugin{} +func NewPiPlugin(backend *tlogbe.TlogBackend, settings []backend.PluginSetting) *piPlugin { + // TODO these should be passed in as plugin settings + var ( + dataDir string + ) + return &piPlugin{ + dataDir: dataDir, + backend: backend, + } } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote.go index 1aa7f4e20..c6576eb8e 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote.go @@ -62,10 +62,10 @@ var ( errRecordNotFound = errors.New("record not found") ) -// ticketVotePlugin satsifies the Plugin interface. +// ticketVotePlugin satisfies the Plugin interface. type ticketVotePlugin struct { sync.Mutex - backend *tlogbe.Tlogbe + backend *tlogbe.TlogBackend // Plugin settings id *identity.FullIdentity @@ -74,9 +74,9 @@ type ticketVotePlugin struct { voteDurationMax uint32 // In blocks // dataDir is the ticket vote plugin data directory. The only data - // that is stored here is memoized data that can be re-created at - // any time by walking the trillian tree. Example, the vote summary - // once a record vote has ended. + // that is stored here is cached data that can be re-created at any + // time by walking the trillian trees. Ex, the vote summary once a + // record vote has ended. dataDir string // inv contains the record inventory catagorized by vote status. @@ -187,11 +187,10 @@ func (p *ticketVotePlugin) cachedSummaryPath(token string) string { } func (p *ticketVotePlugin) cachedSummary(token string) (*ticketvote.Summary, error) { - fp := p.cachedSummaryPath(token) - p.Lock() defer p.Unlock() + fp := p.cachedSummaryPath(token) b, err := ioutil.ReadFile(fp) if err != nil { var e *os.PathError @@ -211,7 +210,7 @@ func (p *ticketVotePlugin) cachedSummary(token string) (*ticketvote.Summary, err return &s, nil } -func (p *ticketVotePlugin) cachedSummarySet(token string, s ticketvote.Summary) error { +func (p *ticketVotePlugin) cachedSummarySave(token string, s ticketvote.Summary) error { b, err := json.Marshal(s) if err != nil { return err @@ -1780,9 +1779,9 @@ func (p *ticketVotePlugin) summary(token string, bestBlock uint32) (*ticketvote. // calculate these results again. if status == ticketvote.VoteStatusFinished { // Save summary - err = p.cachedSummarySet(sv.Vote.Token, summary) + err = p.cachedSummarySave(sv.Vote.Token, summary) if err != nil { - return nil, fmt.Errorf("cachedSummarySet %v: %v %v", + return nil, fmt.Errorf("cachedSummarySave %v: %v %v", sv.Vote.Token, err, summary) } @@ -2007,7 +2006,7 @@ func (p *ticketVotePlugin) Setup() error { return nil } -func TicketVotePluginNew(backend *tlogbe.Tlogbe, settings []backend.PluginSetting) (*ticketVotePlugin, error) { +func TicketVotePluginNew(backend *tlogbe.TlogBackend, settings []backend.PluginSetting) (*ticketVotePlugin, error) { var ( // TODO these should be passed in as plugin settings dataDir string diff --git a/politeiad/backend/tlogbe/recordclient.go b/politeiad/backend/tlogbe/recordclient.go index f65c82b93..256722edf 100644 --- a/politeiad/backend/tlogbe/recordclient.go +++ b/politeiad/backend/tlogbe/recordclient.go @@ -289,6 +289,6 @@ func (c *RecordClient) BlobsByKeyPrefix(keyPrefix string) ([][]byte, error) { } // TODO implement RecordClient -func (t *Tlogbe) RecordClient(token []byte) (*RecordClient, error) { +func (t *TlogBackend) RecordClient(token []byte) (*RecordClient, error) { return nil, nil } diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index 2b2432a37..2d35eccf7 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -44,6 +44,8 @@ const ( // such as searching for a record index that has been buried by // thousands of leaves from plugin data. // TODO key prefix app-dataID: + // TODO the pluginID and the dataID should be passed into the tlog function + // instead of the keyPrefix keyPrefixRecordIndex = "recordindex:" keyPrefixRecordContent = "record:" keyPrefixFreezeRecord = "freeze:" diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index ad265766c..9348d10f2 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -39,9 +39,11 @@ const ( ) var ( - _ backend.Backend = (*Tlogbe)(nil) + _ backend.Backend = (*TlogBackend)(nil) - // statusChanges contains the allowed record status changes. + // statusChanges contains the allowed record status changes. If + // statusChanges[currentStatus][newStatus] exists then the status + // change is allowed. statusChanges = map[backend.MDStatusT]map[backend.MDStatusT]struct{}{ // Unvetted status changes backend.MDStatusUnvetted: map[backend.MDStatusT]struct{}{ @@ -74,8 +76,8 @@ type plugin struct { ctx Plugin } -// Tlogbe implements the Backend interface. -type Tlogbe struct { +// TlogBackend implements the Backend interface. +type TlogBackend struct { sync.RWMutex shutdown bool homeDir string @@ -109,14 +111,14 @@ func tokenPrefix(token []byte) string { return hex.EncodeToString(token)[:pd.TokenPrefixLength] } -func (t *Tlogbe) isShutdown() bool { +func (t *TlogBackend) isShutdown() bool { t.RLock() defer t.RUnlock() return t.shutdown } -func (t *Tlogbe) prefixExists(fullToken []byte) bool { +func (t *TlogBackend) prefixExists(fullToken []byte) bool { t.RLock() defer t.RUnlock() @@ -124,7 +126,7 @@ func (t *Tlogbe) prefixExists(fullToken []byte) bool { return ok } -func (t *Tlogbe) prefixAdd(fullToken []byte) { +func (t *TlogBackend) prefixAdd(fullToken []byte) { t.Lock() defer t.Unlock() @@ -134,7 +136,7 @@ func (t *Tlogbe) prefixAdd(fullToken []byte) { log.Debugf("Token prefix cached: %v", prefix) } -func (t *Tlogbe) vettedTreeIDGet(token string) (int64, bool) { +func (t *TlogBackend) vettedTreeIDGet(token string) (int64, bool) { t.RLock() defer t.RUnlock() @@ -142,7 +144,7 @@ func (t *Tlogbe) vettedTreeIDGet(token string) (int64, bool) { return treeID, ok } -func (t *Tlogbe) vettedTreeIDAdd(token string, treeID int64) { +func (t *TlogBackend) vettedTreeIDAdd(token string, treeID int64) { t.Lock() defer t.Unlock() @@ -151,7 +153,7 @@ func (t *Tlogbe) vettedTreeIDAdd(token string, treeID int64) { log.Debugf("Vetted tree ID cached: %v %v", token, treeID) } -func (t *Tlogbe) inventoryGet() map[backend.MDStatusT][]string { +func (t *TlogBackend) inventoryGet() map[backend.MDStatusT][]string { t.RLock() defer t.RUnlock() @@ -168,7 +170,7 @@ func (t *Tlogbe) inventoryGet() map[backend.MDStatusT][]string { return inv } -func (t *Tlogbe) inventoryAdd(token string, s backend.MDStatusT) { +func (t *TlogBackend) inventoryAdd(token string, s backend.MDStatusT) { t.Lock() defer t.Unlock() @@ -177,7 +179,7 @@ func (t *Tlogbe) inventoryAdd(token string, s backend.MDStatusT) { log.Debugf("Inventory cache added: %v %v", token, backend.MDStatus[s]) } -func (t *Tlogbe) inventoryUpdate(token string, currStatus, newStatus backend.MDStatusT) { +func (t *TlogBackend) inventoryUpdate(token string, currStatus, newStatus backend.MDStatusT) { t.Lock() defer t.Unlock() @@ -468,7 +470,7 @@ func metadataStreamsUpdate(mdCurr, mdAppend, mdOverwrite []backend.MetadataStrea // New satisfies the Backend interface. // // This function satisfies the Backend interface. -func (t *Tlogbe) New(metadata []backend.MetadataStream, files []backend.File) (*backend.RecordMetadata, error) { +func (t *TlogBackend) New(metadata []backend.MetadataStream, files []backend.File) (*backend.RecordMetadata, error) { log.Tracef("New") // Validate record contents @@ -509,7 +511,7 @@ func (t *Tlogbe) New(metadata []backend.MetadataStream, files []backend.File) (* return nil, err } - // Call pre plugin hook + // Call pre plugin hooks pre := NewRecordPre{ RecordMetadata: *rm, Metadata: metadata, @@ -531,7 +533,7 @@ func (t *Tlogbe) New(metadata []backend.MetadataStream, files []backend.File) (* return nil, fmt.Errorf("recordSave %x: %v", token, err) } - // Call post plugin hook + // Call post plugin hooks post := NewRecordPost{ RecordMetadata: *rm, Metadata: metadata, @@ -543,8 +545,7 @@ func (t *Tlogbe) New(metadata []backend.MetadataStream, files []backend.File) (* } err = t.pluginHook(HookNewRecordPost, string(b)) if err != nil { - return nil, fmt.Errorf("pluginHook %v: %v", - Hooks[HookNewRecordPost], err) + log.Errorf("New: pluginHook %v: %v", Hooks[HookNewRecordPost], err) } // Update the inventory cache @@ -556,7 +557,7 @@ func (t *Tlogbe) New(metadata []backend.MetadataStream, files []backend.File) (* } // This function satisfies the Backend interface. -func (t *Tlogbe) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { +func (t *TlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { log.Tracef("UpdateUnvettedRecord: %x", token) // Validate record contents. Send in a single metadata array to @@ -632,7 +633,7 @@ func (t *Tlogbe) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite []back } // This function satisfies the Backend interface. -func (t *Tlogbe) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { +func (t *TlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { log.Tracef("UpdateVettedRecord: %x", token) // Validate record contents. Send in a single metadata array to @@ -700,7 +701,7 @@ func (t *Tlogbe) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []backen } // This function satisfies the Backend interface. -func (t *Tlogbe) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite []backend.MetadataStream) error { +func (t *TlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite []backend.MetadataStream) error { // Validate record contents. Send in a single metadata array to // verify there are no dups. allMD := append(mdAppend, mdOverwrite...) @@ -754,7 +755,7 @@ func (t *Tlogbe) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite []ba } // This function satisfies the Backend interface. -func (t *Tlogbe) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite []backend.MetadataStream) error { +func (t *TlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite []backend.MetadataStream) error { log.Tracef("UpdateVettedMetadata: %x", token) // Validate record contents. Send in a single metadata array to @@ -813,7 +814,7 @@ func (t *Tlogbe) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite []back // record. // // This function satisfies the Backend interface. -func (t *Tlogbe) UnvettedExists(token []byte) bool { +func (t *TlogBackend) UnvettedExists(token []byte) bool { log.Tracef("UnvettedExists %x", token) // If the token is in the vetted cache then we know this is not an @@ -842,7 +843,7 @@ func (t *Tlogbe) UnvettedExists(token []byte) bool { } // This function satisfies the Backend interface. -func (t *Tlogbe) VettedExists(token []byte) bool { +func (t *TlogBackend) VettedExists(token []byte) bool { log.Tracef("VettedExists %x", token) // Check if the token is in the vetted cache. The vetted cache is @@ -897,7 +898,7 @@ func (t *Tlogbe) VettedExists(token []byte) bool { } // This function satisfies the Backend interface. -func (t *Tlogbe) GetUnvetted(token []byte, version string) (*backend.Record, error) { +func (t *TlogBackend) GetUnvetted(token []byte, version string) (*backend.Record, error) { log.Tracef("GetUnvetted: %x", token) if t.isShutdown() { @@ -914,7 +915,7 @@ func (t *Tlogbe) GetUnvetted(token []byte, version string) (*backend.Record, err } // This function satisfies the Backend interface. -func (t *Tlogbe) GetVetted(token []byte, version string) (*backend.Record, error) { +func (t *TlogBackend) GetVetted(token []byte, version string) (*backend.Record, error) { log.Tracef("GetVetted: %x", token) if t.isShutdown() { @@ -931,7 +932,7 @@ func (t *Tlogbe) GetVetted(token []byte, version string) (*backend.Record, error } // This function must be called WITH the unvetted lock held. -func (t *Tlogbe) unvettedPublish(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { +func (t *TlogBackend) unvettedPublish(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { // Create a vetted tree var ( vettedToken []byte @@ -983,7 +984,7 @@ func (t *Tlogbe) unvettedPublish(token []byte, rm backend.RecordMetadata, metada } // This function must be called WITH the unvetted lock held. -func (t *Tlogbe) unvettedCensor(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { +func (t *TlogBackend) unvettedCensor(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { // Freeze the tree treeID := treeIDFromToken(token) err := t.unvetted.treeFreeze(treeID, rm, metadata, freezeRecord{}) @@ -1005,7 +1006,7 @@ func (t *Tlogbe) unvettedCensor(token []byte, rm backend.RecordMetadata, metadat } // This function must be called WITH the unvetted lock held. -func (t *Tlogbe) unvettedArchive(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { +func (t *TlogBackend) unvettedArchive(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { // Freeze the tree. Nothing else needs to be done for an archived // record. treeID := treeIDFromToken(token) @@ -1019,7 +1020,7 @@ func (t *Tlogbe) unvettedArchive(token []byte, rm backend.RecordMetadata, metada return nil } -func (t *Tlogbe) SetUnvettedStatus(token []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { +func (t *TlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { log.Tracef("SetUnvettedStatus: %x %v (%v)", token, status, backend.MDStatus[status]) @@ -1048,10 +1049,6 @@ func (t *Tlogbe) SetUnvettedStatus(token []byte, status backend.MDStatusT, mdApp } } - log.Debugf("Status change %x from %v (%v) to %v (%v)", - token, backend.MDStatus[oldStatus], oldStatus, - backend.MDStatus[status], status) - // Apply status change rm.Status = status rm.Iteration += 1 @@ -1060,6 +1057,23 @@ func (t *Tlogbe) SetUnvettedStatus(token []byte, status backend.MDStatusT, mdApp // Apply metdata changes metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) + // Call pre plugin hooks + pre := SetRecordStatusPre{ + Record: *r, + RecordMetadata: rm, + MDAppend: mdAppend, + MDOverwrite: mdOverwrite, + } + b, err := EncodeSetRecordStatusPre(pre) + if err != nil { + return nil, err + } + err = t.pluginHook(HookSetRecordStatusPre, string(b)) + if err != nil { + return nil, fmt.Errorf("pluginHook %v: %v", + Hooks[HookSetRecordStatusPre], err) + } + // Update record switch status { case backend.MDStatusVetted: @@ -1082,9 +1096,30 @@ func (t *Tlogbe) SetUnvettedStatus(token []byte, status backend.MDStatusT, mdApp backend.MDStatus[status], status) } + // Call post plugin hooks + post := SetRecordStatusPost{ + Record: *r, + RecordMetadata: rm, + MDAppend: mdAppend, + MDOverwrite: mdOverwrite, + } + b, err = EncodeSetRecordStatusPost(post) + if err != nil { + return nil, err + } + err = t.pluginHook(HookSetRecordStatusPost, string(b)) + if err != nil { + log.Errorf("SetUnvettedStatus: pluginHook %v: %v", + Hooks[HookSetRecordStatusPost], err) + } + // Update inventory cache t.inventoryUpdate(rm.Token, oldStatus, status) + log.Debugf("Status change %x from %v (%v) to %v (%v)", + token, backend.MDStatus[oldStatus], oldStatus, + backend.MDStatus[status], status) + // Return the updated record r, err = t.unvetted.recordLatest(treeID) if err != nil { @@ -1095,7 +1130,7 @@ func (t *Tlogbe) SetUnvettedStatus(token []byte, status backend.MDStatusT, mdApp } // This function must be called WITH the vetted lock held. -func (t *Tlogbe) vettedCensor(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { +func (t *TlogBackend) vettedCensor(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { // Freeze the tree treeID := treeIDFromToken(token) err := t.vetted.treeFreeze(treeID, rm, metadata, freezeRecord{}) @@ -1113,7 +1148,7 @@ func (t *Tlogbe) vettedCensor(token []byte, rm backend.RecordMetadata, metadata } // This function must be called WITH the vetted lock held. -func (t *Tlogbe) vettedArchive(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { +func (t *TlogBackend) vettedArchive(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { // Freeze the tree. Nothing else needs to be done for an archived // record. treeID := treeIDFromToken(token) @@ -1121,7 +1156,7 @@ func (t *Tlogbe) vettedArchive(token []byte, rm backend.RecordMetadata, metadata } // This function satisfies the Backend interface. -func (t *Tlogbe) SetVettedStatus(token []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { +func (t *TlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { log.Tracef("SetVettedStatus: %x %v (%v)", token, status, backend.MDStatus[status]) @@ -1150,10 +1185,6 @@ func (t *Tlogbe) SetVettedStatus(token []byte, status backend.MDStatusT, mdAppen } } - log.Debugf("Status change %x from %v (%v) to %v (%v)", - token, backend.MDStatus[oldStatus], oldStatus, - backend.MDStatus[status], status) - // Apply status change rm.Status = status rm.Iteration += 1 @@ -1162,6 +1193,23 @@ func (t *Tlogbe) SetVettedStatus(token []byte, status backend.MDStatusT, mdAppen // Apply metdata changes metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) + // Call pre plugin hooks + pre := SetRecordStatusPre{ + Record: *r, + RecordMetadata: rm, + MDAppend: mdAppend, + MDOverwrite: mdOverwrite, + } + b, err := EncodeSetRecordStatusPre(pre) + if err != nil { + return nil, err + } + err = t.pluginHook(HookSetRecordStatusPre, string(b)) + if err != nil { + return nil, fmt.Errorf("pluginHook %v: %v", + Hooks[HookSetRecordStatusPre], err) + } + // Update record switch status { case backend.MDStatusCensored: @@ -1179,9 +1227,30 @@ func (t *Tlogbe) SetVettedStatus(token []byte, status backend.MDStatusT, mdAppen backend.MDStatus[status], status) } + // Call post plugin hooks + post := SetRecordStatusPost{ + Record: *r, + RecordMetadata: rm, + MDAppend: mdAppend, + MDOverwrite: mdOverwrite, + } + b, err = EncodeSetRecordStatusPost(post) + if err != nil { + return nil, err + } + err = t.pluginHook(HookSetRecordStatusPost, string(b)) + if err != nil { + log.Errorf("SetVettedStatus: pluginHook %v: %v", + Hooks[HookSetRecordStatusPost], err) + } + // Update inventory cache t.inventoryUpdate(rm.Token, oldStatus, status) + log.Debugf("Status change %x from %v (%v) to %v (%v)", + token, backend.MDStatus[oldStatus], oldStatus, + backend.MDStatus[status], status) + // Return the updated record r, err = t.vetted.recordLatest(treeID) if err != nil { @@ -1194,7 +1263,7 @@ func (t *Tlogbe) SetVettedStatus(token []byte, status backend.MDStatusT, mdAppen // Inventory is not currenctly implemented in tlogbe. // // This function satisfies the Backend interface. -func (t *Tlogbe) Inventory(vettedCount uint, unvettedCount uint, includeFiles, allVersions bool) ([]backend.Record, []backend.Record, error) { +func (t *TlogBackend) Inventory(vettedCount uint, unvettedCount uint, includeFiles, allVersions bool) ([]backend.Record, []backend.Record, error) { log.Tracef("Inventory: %v %v", includeFiles, allVersions) return nil, nil, fmt.Errorf("not implemented") @@ -1204,7 +1273,7 @@ func (t *Tlogbe) Inventory(vettedCount uint, unvettedCount uint, includeFiles, a // catagorized by MDStatusT. // // This function satisfies the Backend interface. -func (t *Tlogbe) InventoryByStatus() (*backend.InventoryByStatus, error) { +func (t *TlogBackend) InventoryByStatus() (*backend.InventoryByStatus, error) { log.Tracef("InventoryByStatus") inv := t.inventoryGet() @@ -1217,7 +1286,7 @@ func (t *Tlogbe) InventoryByStatus() (*backend.InventoryByStatus, error) { }, nil } -func (t *Tlogbe) pluginHook(h HookT, payload string) error { +func (t *TlogBackend) pluginHook(h HookT, payload string) error { t.RLock() defer t.RUnlock() @@ -1236,7 +1305,7 @@ func (t *Tlogbe) pluginHook(h HookT, payload string) error { // settings. // // This function satisfies the Backend interface. -func (t *Tlogbe) GetPlugins() ([]backend.Plugin, error) { +func (t *TlogBackend) GetPlugins() ([]backend.Plugin, error) { log.Tracef("GetPlugins") plugins := make([]backend.Plugin, 0, len(t.plugins)) @@ -1254,7 +1323,7 @@ func (t *Tlogbe) GetPlugins() ([]backend.Plugin, error) { // Plugin is a pass-through function for plugin commands. // // This function satisfies the Backend interface. -func (t *Tlogbe) Plugin(pluginID, command, payload string) (string, error) { +func (t *TlogBackend) Plugin(pluginID, command, payload string) (string, error) { log.Tracef("Plugin: %v", command) if t.isShutdown() { @@ -1279,7 +1348,7 @@ func (t *Tlogbe) Plugin(pluginID, command, payload string) (string, error) { // Close shuts the backend down and performs cleanup. // // This function satisfies the Backend interface. -func (t *Tlogbe) Close() { +func (t *TlogBackend) Close() { log.Tracef("Close") t.Lock() @@ -1293,7 +1362,7 @@ func (t *Tlogbe) Close() { t.vetted.close() } -func (t *Tlogbe) setup() error { +func (t *TlogBackend) setup() error { // Get all trees trees, err := t.unvetted.trillian.treesAll() if err != nil { @@ -1345,8 +1414,8 @@ func (t *Tlogbe) setup() error { return nil } -// New returns a new Tlogbe. -func New(homeDir, dataDir, dcrtimeHost, encryptionKeyFile, unvettedTrillianHost, unvettedTrillianKeyFile, vettedTrillianHost, vettedTrillianKeyFile string) (*Tlogbe, error) { +// New returns a new TlogBackend. +func New(homeDir, dataDir, dcrtimeHost, encryptionKeyFile, unvettedTrillianHost, unvettedTrillianKeyFile, vettedTrillianHost, vettedTrillianKeyFile string) (*TlogBackend, error) { // Setup encryption key file if encryptionKeyFile == "" { // No file path was given. Use the default path. @@ -1387,7 +1456,7 @@ func New(homeDir, dataDir, dcrtimeHost, encryptionKeyFile, unvettedTrillianHost, } // Setup tlogbe - t := Tlogbe{ + t := TlogBackend{ homeDir: homeDir, dataDir: dataDir, unvetted: unvetted, diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 4e3cc5cc2..a654f2f24 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -466,7 +466,7 @@ func (p *politeiawww) processProposalNew(pn pi.ProposalNew, usr user.User) (*pi. } } - // Verify user signed with active identity + // Verify user signed using active identity if usr.PublicKey() != pn.PublicKey { return nil, www.UserError{ ErrorCode: www.ErrorStatusInvalidSigningKey, diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index b7cfa5128..be32335c4 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -5,12 +5,9 @@ package main import ( - "bytes" - "encoding/base64" "encoding/hex" "encoding/json" "fmt" - "mime" "net/http" "sort" "strconv" @@ -127,355 +124,6 @@ func (p *politeiawww) linkByValidate(linkBy int64) error { return nil } -// validateProposalMetdata validates the provided proposal metadata to ensure -// it adheres to the api policy. Note that this function only checks validation -// that is applicable to both new proposals and proposal edits. Additional -// validation is done in processNewProposal and processEditProposal that is -// specific to that action only. -func (p *politeiawww) validateProposalMetadata(pm www.ProposalMetadata) error { - // Validate Name - if !proposalNameIsValid(pm.Name) { - return www.UserError{ - ErrorCode: www.ErrorStatusProposalInvalidTitle, - ErrorContext: []string{createProposalNameRegex()}, - } - } - - // Validate LinkTo - if pm.LinkTo != "" { - if !tokenIsValid(pm.LinkTo) { - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkTo, - ErrorContext: []string{"invalid token"}, - } - } - - // Validate the LinkTo proposal. The only type of proposal that - // we currently allow linking to is an RFP. - /* - // TODO - r, err := p.cache.Record(pm.LinkTo) - if err != nil { - if err == cache.ErrRecordNotFound { - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkTo, - } - } - } - pr, err := convertPropFromCache(*r) - if err != nil { - return err - } - */ - var pr *www.ProposalRecord - bb, err := p.getBestBlock() - if err != nil { - return err - } - vs, err := p.voteSummaryGet(pm.LinkTo, bb) - if err != nil { - return err - } - switch { - case !isRFP(*pr): - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkTo, - ErrorContext: []string{"linkto proposal is not an rfp"}, - } - case !voteIsApproved(*vs): - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkTo, - ErrorContext: []string{"rfp proposal vote did not pass"}, - } - case pr.State != www.PropStateVetted: - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkTo, - ErrorContext: []string{"linkto proposal is not vetted"}, - } - case isRFP(*pr) && pm.LinkBy != 0: - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkTo, - ErrorContext: []string{"an rfp cannot link to an rfp"}, - } - } - } - - // Validate LinkBy - if pm.LinkBy != 0 { - err := p.linkByValidate(pm.LinkBy) - if err != nil { - return err - } - } - - return nil -} - -// validateProposal ensures that the given new proposal meets the api policy -// requirements. If a proposal data file exists (currently optional) then it is -// parsed and a ProposalMetadata is returned. -func (p *politeiawww) validateProposal(np www.NewProposal, u *user.User) (*www.ProposalMetadata, error) { - if len(np.Files) == 0 { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusProposalMissingFiles, - ErrorContext: []string{"no files found"}, - } - } - - // Verify the files adhere to all policy requirements - var ( - countTextFiles int - countImageFiles int - foundIndexFile bool - ) - filenames := make(map[string]struct{}, len(np.Files)) - for _, v := range np.Files { - // Validate file name - _, ok := filenames[v.Name] - if ok { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusProposalDuplicateFilenames, - ErrorContext: []string{v.Name}, - } - } - filenames[v.Name] = struct{}{} - - // Validate file payload - if v.Payload == "" { - e := fmt.Sprintf("base64 payload is empty for file '%v'", - v.Name) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidBase64, - ErrorContext: []string{e}, - } - } - payloadb, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidBase64, - ErrorContext: []string{v.Name}, - } - } - - // Verify computed file digest matches given file digest - digest := util.Digest(payloadb) - d, ok := util.ConvertDigest(v.Digest) - if !ok { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidFileDigest, - ErrorContext: []string{v.Name}, - } - } - if !bytes.Equal(digest, d[:]) { - e := fmt.Sprintf("computed digest does not match given digest "+ - "for file '%v'", v.Name) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidFileDigest, - ErrorContext: []string{e}, - } - } - - // Verify detected MIME type matches given mime type - ct := http.DetectContentType(payloadb) - mimePayload, _, err := mime.ParseMediaType(ct) - if err != nil { - return nil, err - } - mimeFile, _, err := mime.ParseMediaType(v.MIME) - if err != nil { - log.Debugf("validateProposal: ParseMediaType(%v): %v", - v.MIME, err) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidMIMEType, - ErrorContext: []string{v.Name}, - } - } - if mimeFile != mimePayload { - e := fmt.Sprintf("detected mime '%v' does not match '%v' for file '%v'", - mimePayload, mimeFile, v.Name) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidMIMEType, - ErrorContext: []string{e}, - } - } - - // Run MIME type specific validation - switch mimeFile { - case mimeTypeText: - countTextFiles++ - - // Verify text file size - if len(payloadb) > www.PolicyMaxMDSize { - e := fmt.Sprintf("file size %v exceeds max %v for file '%v'", - len(payloadb), www.PolicyMaxMDSize, v.Name) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusMaxMDSizeExceededPolicy, - ErrorContext: []string{e}, - } - } - - // The only text file that is allowed is the index markdown - // file. - if v.Name != www.PolicyIndexFilename { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusMaxMDsExceededPolicy, - ErrorContext: []string{v.Name}, - } - } - if foundIndexFile { - e := fmt.Sprintf("more than one %v file found", - www.PolicyIndexFilename) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusMaxMDsExceededPolicy, - ErrorContext: []string{e}, - } - } - - // Set index file as being found - foundIndexFile = true - - case mimeTypePNG: - countImageFiles++ - - // Verify image file size - if len(payloadb) > www.PolicyMaxImageSize { - e := fmt.Sprintf("file size %v exceeds max %v for file '%v'", - len(payloadb), www.PolicyMaxImageSize, v.Name) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusMaxImageSizeExceededPolicy, - ErrorContext: []string{e}, - } - } - - default: - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidMIMEType, - ErrorContext: []string{v.MIME}, - } - } - } - - // Verify that an index file is present. - if !foundIndexFile { - e := fmt.Sprintf("%v file not found", www.PolicyIndexFilename) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusProposalMissingFiles, - ErrorContext: []string{e}, - } - } - - // Verify file counts are acceptable - if countTextFiles > www.PolicyMaxMDs { - e := fmt.Sprintf("got %v text files; max is %v", - countTextFiles, www.PolicyMaxMDs) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusMaxMDsExceededPolicy, - ErrorContext: []string{e}, - } - } - if countImageFiles > www.PolicyMaxImages { - e := fmt.Sprintf("got %v image files, max is %v", - countImageFiles, www.PolicyMaxImages) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusMaxImagesExceededPolicy, - ErrorContext: []string{e}, - } - } - - // Decode and validate metadata - var pm *www.ProposalMetadata - for _, v := range np.Metadata { - // Decode payload - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - e := fmt.Sprintf("invalid base64 for '%v'", v.Hint) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusMetadataInvalid, - ErrorContext: []string{e}, - } - } - d := json.NewDecoder(bytes.NewReader(b)) - d.DisallowUnknownFields() - - // Unmarshal payload - switch v.Hint { - case www.HintProposalMetadata: - var p www.ProposalMetadata - err := d.Decode(&p) - if err != nil { - log.Debugf("validateProposal: decode ProposalMetadata: %v", err) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusMetadataInvalid, - ErrorContext: []string{v.Hint}, - } - } - pm = &p - default: - e := fmt.Sprintf("unknown hint '%v'", v.Hint) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusMetadataInvalid, - ErrorContext: []string{e}, - } - } - - // Validate digest - digest := util.Digest(b) - if v.Digest != hex.EncodeToString(digest) { - e := fmt.Sprintf("%v got digest %v, want %v", - v.Hint, v.Digest, hex.EncodeToString(digest)) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusMetadataDigestInvalid, - ErrorContext: []string{e}, - } - } - } - if pm == nil { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusMetadataMissing, - ErrorContext: []string{www.HintProposalMetadata}, - } - } - - // Validate ProposalMetadata - err := p.validateProposalMetadata(*pm) - if err != nil { - return nil, err - } - // Verify signature. The signature message is the merkle root - // of the proposal files. - /* - // TODO - sig, err := util.ConvertSignature(np.Signature) - if err != nil { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidSignature, - } - } - pk, err := identity.PublicIdentityFromBytes(u.ActiveIdentity().Key[:]) - if err != nil { - return nil, err - } - mr, err := wwwutil.MerkleRoot(np.Files, np.Metadata) - if err != nil { - return nil, err - } - if !pk.VerifyMessage([]byte(mr), sig) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidSignature, - } - } - */ - - // Verify the user signed using their active identity - if u.PublicKey() != np.PublicKey { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidSigningKey, - } - } - - return pm, nil -} - func voteStatusFromVoteSummary(r decredplugin.VoteSummaryReply, endHeight, bestBlock uint64) www.PropVoteStatusT { switch { case !r.Authorized: @@ -1011,139 +659,7 @@ func (p *politeiawww) voteSummaryGet(token string, bestBlock uint64) (*www.VoteS func (p *politeiawww) processNewProposal(np www.NewProposal, user *user.User) (*www.NewProposalReply, error) { log.Tracef("processNewProposal") - // Pay up sucker! - if !p.userHasPaid(*user) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusUserNotPaid, - } - } - - if !p.userHasProposalCredits(*user) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusNoProposalCredits, - } - } - - pm, err := p.validateProposal(np, user) - if err != nil { - return nil, err - } - - // Validate linkto requirements that are specific to new proposal - // submissions. Everything else has already been validated by the - // validateProposal function. - if pm.LinkTo != "" { - /* - r, err := p.cache.Record(pm.LinkTo) - if err != nil { - return nil, err - } - pr, err := convertPropFromCache(*r) - if err != nil { - return nil, err - } - */ - var pr *www.ProposalRecord - // Once the linkto deadline has expired no new submissions are - // allowed. Edits to existing submissions are allowed which is - // why this is checked here and not in the validateProposal - // function. - if time.Now().Unix() > pr.LinkBy { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkTo, - ErrorContext: []string{"linkto proposal deadline is expired"}, - } - } - } - - // politeiad only includes files in its merkle root calc, not the - // metadata streams. This is why we include the ProposalMetadata - // as a politeiad file. - - // Setup politeaid files - files := convertPropFilesFromWWW(np.Files) - for _, v := range np.Metadata { - switch v.Hint { - case www.HintProposalMetadata: - // files = append(files, convertFileFromMetadata(v)) - } - } - - // Setup politeiad metadata - pg := mdstream.ProposalGeneralV2{ - Version: mdstream.VersionProposalGeneral, - Timestamp: time.Now().Unix(), - PublicKey: np.PublicKey, - Signature: np.Signature, - } - pgb, err := mdstream.EncodeProposalGeneralV2(pg) - if err != nil { - return nil, err - } - metadata := []pd.MetadataStream{ - { - ID: mdstream.IDProposalGeneral, - Payload: string(pgb), - }, - } - - // Setup politeiad request - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - n := pd.NewRecord{ - Challenge: hex.EncodeToString(challenge), - Metadata: metadata, - Files: files, - } - - // Send politeiad request - responseBody, err := p.makeRequest(http.MethodPost, - pd.NewRecordRoute, n) - if err != nil { - return nil, err - } - - log.Infof("Submitted proposal name: %v", pm.Name) - for k, f := range n.Files { - log.Infof("%02v: %v %v", k, f.Name, f.Digest) - } - - // Handle response - var pdReply pd.NewRecordReply - err = json.Unmarshal(responseBody, &pdReply) - if err != nil { - return nil, fmt.Errorf("Unmarshal NewProposalReply: %v", err) - } - - err = util.VerifyChallenge(p.cfg.Identity, challenge, pdReply.Response) - if err != nil { - return nil, err - } - - cr := convertPropCensorFromPD(pdReply.CensorshipRecord) - - // Deduct proposal credit from user account - err = p.spendProposalCredit(user, cr.Token) - if err != nil { - return nil, err - } - - // Fire off new proposal event - /* - p.fireEvent(eventTypeProposalSubmitted, - EventDataProposalSubmitted{ - CensorshipRecord: &cr, - ProposalName: pm.Name, - User: user, - }, - ) - */ - - return &www.NewProposalReply{ - CensorshipRecord: cr, - }, nil + return nil, nil } // createProposalDetailsReply makes updates to a proposal record based on the From 2b6b59f2ee1a8cb4b908d404c1857aafa0c40f96 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 2 Sep 2020 20:15:17 -0500 Subject: [PATCH 027/449] proposal routes --- plugins/pi/pi.go | 44 +++--- plugins/ticketvote/ticketvote.go | 8 + politeiad/backend/tlogbe/plugin.go | 98 +++++-------- politeiad/backend/tlogbe/plugins/pi.go | 107 +++++++++++++- politeiad/backend/tlogbe/tlogbe.go | 107 ++++++++------ politeiawww/api/pi/v1/v1.go | 43 ++++-- politeiawww/piwww.go | 168 ++++++++++++++++++++- politeiawww/proposals.go | 193 ++----------------------- 8 files changed, 433 insertions(+), 335 deletions(-) diff --git a/plugins/pi/pi.go b/plugins/pi/pi.go index 13d58c49d..bfb2793fe 100644 --- a/plugins/pi/pi.go +++ b/plugins/pi/pi.go @@ -11,6 +11,7 @@ import ( "fmt" ) +type PropStatusT int type ErrorStatusT int const ( @@ -18,8 +19,6 @@ const ( ID = "pi" // Plugin commands - CmdProposalDetails = "proposaldetails" - CmdProposals = "proposals" // Metadata stream IDs. All metadata streams in this plugin will // use 1xx numbering. @@ -32,16 +31,37 @@ const ( // be included in the merkle root that politeiad signs. FilenameProposalMetadata = "proposalmetadata.json" + // Proposal status codes + PropStatusInvalid PropStatusT = 0 // Invalid status + PropStatusUnvetted PropStatusT = 1 // Prop has not been vetted + PropStatusPublic PropStatusT = 2 // Prop has been made public + PropStatusCensored PropStatusT = 3 // Prop has been censored + PropStatusAbandoned PropStatusT = 4 // Prop has been abandoned + // User error status codes - ErrorStatusInvalid ErrorStatusT = 0 - ErrorStatusLinkToInvalid ErrorStatusT = 1 + ErrorStatusInvalid ErrorStatusT = 0 + ErrorStatusLinkToInvalid ErrorStatusT = 1 + ErrorStatusWrongPropStatus ErrorStatusT = 2 + ErrorStatusWrongVoteStatus ErrorStatusT = 3 ) var ( + // StatusChanges contains the allowed proposal status change + // transitions. If StatusChanges[currentStatus][newStatus] exists + // then the status change is allowed. + StatusChanges = map[PropStatusT]map[PropStatusT]struct{}{ + PropStatusUnvetted: map[PropStatusT]struct{}{}, + PropStatusPublic: map[PropStatusT]struct{}{}, + PropStatusCensored: map[PropStatusT]struct{}{}, + PropStatusAbandoned: map[PropStatusT]struct{}{}, + } + // ErrorStatus contains human readable user error statuses. ErrorStatus = map[ErrorStatusT]string{ - ErrorStatusInvalid: "error status invalid", - ErrorStatusLinkToInvalid: "linkto invalid", + ErrorStatusInvalid: "error status invalid", + ErrorStatusLinkToInvalid: "linkto invalid", + ErrorStatusWrongPropStatus: "wrong proposal status", + ErrorStatusWrongVoteStatus: "wrong vote status", } ) @@ -115,15 +135,3 @@ func DecodeProposalGeneral(payload []byte) (*ProposalGeneral, error) { } return &pg, nil } - -// StatusChange represents a proposal status change. -// -// Signature is the client signature of the Token+Version+Status+Reason. -type StatusChange struct { - // Status PropStatusT `json:"status"` - Version string `json:"version"` - Message string `json:"message,omitempty"` - PublicKey string `json:"publickey"` - Signature string `json:"signature"` - Timestamp int64 `json:"timestamp"` -} diff --git a/plugins/ticketvote/ticketvote.go b/plugins/ticketvote/ticketvote.go index d614ee76b..f879e6e3c 100644 --- a/plugins/ticketvote/ticketvote.go +++ b/plugins/ticketvote/ticketvote.go @@ -104,6 +104,14 @@ const ( ) var ( + VoteStatus = map[VoteStatusT]string{ + VoteStatusInvalid: "vote status invalid", + VoteStatusUnauthorized: "unauthorized", + VoteStatusAuthorized: "authorized", + VoteStatusStarted: "started", + VoteStatusFinished: "finished", + } + VoteError = map[VoteErrorT]string{ VoteErrorInvalid: "vote error invalid", VoteErrorInternalError: "internal server error", diff --git a/politeiad/backend/tlogbe/plugin.go b/politeiad/backend/tlogbe/plugin.go index e36579ccd..d4c17a24e 100644 --- a/politeiad/backend/tlogbe/plugin.go +++ b/politeiad/backend/tlogbe/plugin.go @@ -39,80 +39,62 @@ var ( } ) -// NewRecordPre is the payload for the HookNewRecordPre hook. -type NewRecordPre struct { +// NewRecord is the payload for the HookNewRecordPre and HookNewRecordPost +// hooks. +type NewRecord struct { RecordMetadata backend.RecordMetadata `json:"recordmetadata"` Metadata []backend.MetadataStream `json:"metadata"` Files []backend.File `json:"files"` } -// EncodeNewRecordPre encodes a NewRecordPre into a JSON byte slice. -func EncodeNewRecordPre(nrp NewRecordPre) ([]byte, error) { - return json.Marshal(nrp) +// EncodeNewRecord encodes a NewRecord into a JSON byte slice. +func EncodeNewRecord(nr NewRecord) ([]byte, error) { + return json.Marshal(nr) } -// DecodeNewRecordPre decodes a JSON byte slice into a NewRecordPre. -func DecodeNewRecordPre(payload []byte) (*NewRecordPre, error) { - var nrp NewRecordPre - err := json.Unmarshal(payload, &nrp) +// DecodeNewRecord decodes a JSON byte slice into a NewRecord. +func DecodeNewRecord(payload []byte) (*NewRecord, error) { + var nr NewRecord + err := json.Unmarshal(payload, &nr) if err != nil { return nil, err } - return &nrp, nil + return &nr, nil } -// NewRecordPost is the payload for the HookNewRecordPost hook. -type NewRecordPost struct { - RecordMetadata backend.RecordMetadata `json:"recordmetadata"` - Metadata []backend.MetadataStream `json:"metadata"` - Files []backend.File `json:"files"` -} - -// EncodeNewRecordPost encodes a NewRecordPost into a JSON byte slice. -func EncodeNewRecordPost(nrp NewRecordPost) ([]byte, error) { - return json.Marshal(nrp) -} - -// DecodeNewRecordPost decodes a JSON byte slice into a NewRecordPost. -func DecodeNewRecordPost(payload []byte) (*NewRecordPost, error) { - var nrp NewRecordPost - err := json.Unmarshal(payload, &nrp) - if err != nil { - return nil, err - } - return &nrp, nil -} - -// SetRecordStatusPre is the payload for the HookSetRecordStatusPre hook. -type SetRecordStatusPre struct { - Record backend.Record `json:"record"` // Current record +// EditRecord is the payload for the EditRecordPre and EditRecordPost hooks. +type EditRecord struct { + // Current record + Record backend.Record `json:"record"` // Updated fields RecordMetadata backend.RecordMetadata `json:"recordmetadata"` MDAppend []backend.MetadataStream `json:"mdappend"` MDOverwrite []backend.MetadataStream `json:"mdoverwrite"` + FilesAdd []backend.File `json:"filesadd"` + FilesDel []string `json:"filesdel"` } -// EncodeSetRecordStatusPre encodes a SetRecordStatusPre into a JSON byte -// slice. -func EncodeSetRecordStatusPre(srsp SetRecordStatusPre) ([]byte, error) { - return json.Marshal(srsp) +// EncodeEditRecord encodes an EditRecord into a JSON byte slice. +func EncodeEditRecord(nr EditRecord) ([]byte, error) { + return json.Marshal(nr) } -// DecodeSetRecordStatusPre decodes a JSON byte slice into a -// SetRecordStatusPre. -func DecodeSetRecordStatusPre(payload []byte) (*SetRecordStatusPre, error) { - var srsp SetRecordStatusPre - err := json.Unmarshal(payload, &srsp) +// DecodeEditRecord decodes a JSON byte slice into a EditRecord. +func DecodeEditRecord(payload []byte) (*EditRecord, error) { + var nr EditRecord + err := json.Unmarshal(payload, &nr) if err != nil { return nil, err } - return &srsp, nil + return &nr, nil } -// SetRecordStatusPost is the payload for the HookSetRecordStatusPost hook. -type SetRecordStatusPost struct { - Record backend.Record `json:"record"` // Current record +// SetRecordStatus is the payload for the HookSetRecordStatusPre and +// HookSetRecordStatusPost hooks. +type SetRecordStatus struct { + // Current record + Record backend.Record `json:"record"` // Updated fields RecordMetadata backend.RecordMetadata `json:"recordmetadata"` @@ -120,24 +102,22 @@ type SetRecordStatusPost struct { MDOverwrite []backend.MetadataStream `json:"mdoverwrite"` } -// EncodeSetRecordStatusPost encodes a SetRecordStatusPost into a JSON byte -// slice. -func EncodeSetRecordStatusPost(srsp SetRecordStatusPost) ([]byte, error) { - return json.Marshal(srsp) +// EncodeSetRecordStatus encodes a SetRecordStatus into a JSON byte slice. +func EncodeSetRecordStatus(srs SetRecordStatus) ([]byte, error) { + return json.Marshal(srs) } -// DecodeSetRecordStatusPost decodes a JSON byte slice into a -// SetRecordStatusPost. -func DecodeSetRecordStatusPost(payload []byte) (*SetRecordStatusPost, error) { - var srsp SetRecordStatusPost - err := json.Unmarshal(payload, &srsp) +// DecodeSetRecordStatus decodes a JSON byte slice into a SetRecordStatus. +func DecodeSetRecordStatus(payload []byte) (*SetRecordStatus, error) { + var srs SetRecordStatus + err := json.Unmarshal(payload, &srs) if err != nil { return nil, err } - return &srsp, nil + return &srs, nil } -// Plugin provides an API for the tlogbe to use when interacting with plugins. +/// Plugin provides an API for the tlogbe to use when interacting with plugins. // All tlogbe plugins must implement the Plugin interface. type Plugin interface { // Perform plugin setup diff --git a/politeiad/backend/tlogbe/plugins/pi.go b/politeiad/backend/tlogbe/plugins/pi.go index fa69b1a58..fa53d3c1f 100644 --- a/politeiad/backend/tlogbe/plugins/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi.go @@ -91,6 +91,7 @@ func (p *piPlugin) cachedLinkedFromLocked(token string) (*proposalLinkedFrom, er fp := p.cachedLinkedFromPath(token) b, err := ioutil.ReadFile(fp) if err != nil { + var e *os.PathError if errors.As(err, &e) && !os.IsExist(err) { // File does't exist return nil, errRecordNotFound @@ -175,6 +176,8 @@ func (p *piPlugin) cachedLinkedFromDel(parentToken, childToken string) error { func (p *piPlugin) Setup() error { log.Tracef("pi Setup") + // Verify vote plugin dependency + return nil } @@ -185,14 +188,17 @@ func (p *piPlugin) Cmd(cmd, payload string) (string, error) { } func (p *piPlugin) hookNewRecordPre(payload string) error { - nrp, err := tlogbe.DecodeNewRecordPre([]byte(payload)) + nr, err := tlogbe.DecodeNewRecord([]byte(payload)) if err != nil { return err } + // TODO verify ProposalMetadata signature. This is already done in + // www but we should do it here anyway since its plugin data. + // Decode ProposalMetadata var pm *pi.ProposalMetadata - for _, v := range nrp.Files { + for _, v := range nr.Files { if v.Name == pi.FilenameProposalMetadata { b, err := base64.StdEncoding.DecodeString(v.Payload) if err != nil { @@ -291,8 +297,95 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { return nil } +func convertPropStatusFromMDStatus(s backend.MDStatusT) pi.PropStatusT { + var status pi.PropStatusT + switch s { + case backend.MDStatusUnvetted, backend.MDStatusIterationUnvetted: + status = pi.PropStatusUnvetted + case backend.MDStatusVetted: + status = pi.PropStatusPublic + case backend.MDStatusCensored: + status = pi.PropStatusCensored + case backend.MDStatusArchived: + status = pi.PropStatusAbandoned + } + return status +} + +func (p *piPlugin) hookEditRecordPre(payload string) error { + er, err := tlogbe.DecodeEditRecord([]byte(payload)) + if err != nil { + return err + } + + // TODO verify files were changed. Before adding this, verify that + // politeiad will also error if no files were changed. + + // Verify proposal status + status := convertPropStatusFromMDStatus(er.Record.RecordMetadata.Status) + if status != pi.PropStatusUnvetted && status != pi.PropStatusPublic { + return pi.UserError{ + ErrorCode: pi.ErrorStatusWrongPropStatus, + } + } + + // Verify vote status + token := er.RecordMetadata.Token + s := ticketvote.Summaries{ + Tokens: []string{token}, + } + b, err := ticketvote.EncodeSummaries(s) + if err != nil { + return err + } + reply, err := p.backend.Plugin(ticketvote.ID, + ticketvote.CmdSummaries, string(b)) + if err != nil { + return fmt.Errorf("ticketvote Summaries: %v", err) + } + sr, err := ticketvote.DecodeSummariesReply([]byte(reply)) + if err != nil { + return err + } + summary, ok := sr.Summaries[token] + if !ok { + return fmt.Errorf("ticketvote summmary not found") + } + if summary.Status != ticketvote.VoteStatusUnauthorized { + e := fmt.Sprintf("vote status got %v, want %v", + ticketvote.VoteStatus[summary.Status], + ticketvote.VoteStatus[ticketvote.VoteStatusUnauthorized]) + return pi.UserError{ + ErrorCode: pi.ErrorStatusWrongVoteStatus, + ErrorContext: []string{e}, + } + } + + // Verify that the linkto has not changed. This only applies to + // public proposal. Unvetted proposals are allowed to change their + // linkto. + if status == pi.PropStatusPublic { + pmCurr, err := proposalMetadataFromFiles(er.Record.Files) + if err != nil { + return err + } + pmNew, err := proposalMetadataFromFiles(er.FilesAdd) + if err != nil { + return err + } + if pmCurr.LinkTo != pmNew.LinkTo { + return pi.UserError{ + ErrorCode: pi.ErrorStatusLinkToInvalid, + ErrorContext: []string{"linkto cannot change on public proposal"}, + } + } + } + + return nil +} + func (p *piPlugin) hookSetRecordStatusPost(payload string) error { - srsp, err := tlogbe.DecodeSetRecordStatusPost([]byte(payload)) + srs, err := tlogbe.DecodeSetRecordStatus([]byte(payload)) if err != nil { return err } @@ -300,7 +393,7 @@ func (p *piPlugin) hookSetRecordStatusPost(payload string) error { // If the LinkTo field has been set then the proposalLinkedFrom // list might need to be updated for the proposal that is being // linked to, depending on the status change that is being made. - pm, err := proposalMetadataFromFiles(srsp.Record.Files) + pm, err := proposalMetadataFromFiles(srs.Record.Files) if err != nil { return err } @@ -309,9 +402,9 @@ func (p *piPlugin) hookSetRecordStatusPost(payload string) error { // the parent proposal's linked from list to be updated. var ( parentToken = pm.LinkTo - childToken = srsp.RecordMetadata.Token + childToken = srs.RecordMetadata.Token ) - switch srsp.RecordMetadata.Status { + switch srs.RecordMetadata.Status { case backend.MDStatusVetted: // Proposal has been made public. Add child token to parent // token's linked from list. @@ -338,6 +431,8 @@ func (p *piPlugin) Hook(h tlogbe.HookT, payload string) error { switch h { case tlogbe.HookNewRecordPre: return p.hookNewRecordPre(payload) + case tlogbe.HookEditRecordPre: + return p.hookEditRecordPre(payload) case tlogbe.HookSetRecordStatusPost: return p.hookSetRecordStatusPost(payload) } diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 9348d10f2..c16ae160c 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -41,9 +41,9 @@ const ( var ( _ backend.Backend = (*TlogBackend)(nil) - // statusChanges contains the allowed record status changes. If - // statusChanges[currentStatus][newStatus] exists then the status - // change is allowed. + // statusChanges contains the allowed record status change + // transitions. If statusChanges[currentStatus][newStatus] exists + // then the status change is allowed. statusChanges = map[backend.MDStatusT]map[backend.MDStatusT]struct{}{ // Unvetted status changes backend.MDStatusUnvetted: map[backend.MDStatusT]struct{}{ @@ -512,19 +512,18 @@ func (t *TlogBackend) New(metadata []backend.MetadataStream, files []backend.Fil } // Call pre plugin hooks - pre := NewRecordPre{ + nr := NewRecord{ RecordMetadata: *rm, Metadata: metadata, Files: files, } - b, err := EncodeNewRecordPre(pre) + b, err := EncodeNewRecord(nr) if err != nil { return nil, err } err = t.pluginHook(HookNewRecordPre, string(b)) if err != nil { - return nil, fmt.Errorf("pluginHook %v: %v", - Hooks[HookNewRecordPre], err) + return nil, err } // Save the record @@ -534,15 +533,6 @@ func (t *TlogBackend) New(metadata []backend.MetadataStream, files []backend.Fil } // Call post plugin hooks - post := NewRecordPost{ - RecordMetadata: *rm, - Metadata: metadata, - Files: files, - } - b, err = EncodeNewRecordPost(post) - if err != nil { - return nil, err - } err = t.pluginHook(HookNewRecordPost, string(b)) if err != nil { log.Errorf("New: pluginHook %v: %v", Hooks[HookNewRecordPost], err) @@ -606,6 +596,24 @@ func (t *TlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ return nil, err } + // Call pre plugin hooks + er := EditRecord{ + Record: *r, + RecordMetadata: *recordMD, + MDAppend: mdAppend, + MDOverwrite: mdOverwrite, + FilesAdd: filesAdd, + FilesDel: filesDel, + } + b, err := EncodeEditRecord(er) + if err != nil { + return nil, err + } + err = t.pluginHook(HookEditRecordPre, string(b)) + if err != nil { + return nil, err + } + // Save record err = t.unvetted.recordSave(treeID, *recordMD, metadata, files) if err != nil { @@ -615,6 +623,12 @@ func (t *TlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ return nil, fmt.Errorf("recordSave: %v", err) } + // Call post plugin hooks + err = t.pluginHook(HookEditRecordPost, string(b)) + if err != nil { + log.Errorf("pluginHook %v: %v", Hooks[HookEditRecordPost], err) + } + // Update inventory cache. The inventory will only need to be // updated if there was a status transition. if r.RecordMetadata.Status != recordMD.Status { @@ -682,6 +696,24 @@ func (t *TlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b return nil, err } + // Call pre plugin hooks + er := EditRecord{ + Record: *r, + RecordMetadata: *recordMD, + MDAppend: mdAppend, + MDOverwrite: mdOverwrite, + FilesAdd: filesAdd, + FilesDel: filesDel, + } + b, err := EncodeEditRecord(er) + if err != nil { + return nil, err + } + err = t.pluginHook(HookEditRecordPre, string(b)) + if err != nil { + return nil, err + } + // Save record err = t.vetted.recordSave(treeID, *recordMD, metadata, files) if err != nil { @@ -691,6 +723,12 @@ func (t *TlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b return nil, fmt.Errorf("recordSave: %v", err) } + // Call post plugin hooks + err = t.pluginHook(HookEditRecordPost, string(b)) + if err != nil { + log.Errorf("pluginHook %v: %v", Hooks[HookEditRecordPost], err) + } + // Return updated record r, err = t.vetted.recordLatest(treeID) if err != nil { @@ -1058,20 +1096,19 @@ func (t *TlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) // Call pre plugin hooks - pre := SetRecordStatusPre{ + srs := SetRecordStatus{ Record: *r, RecordMetadata: rm, MDAppend: mdAppend, MDOverwrite: mdOverwrite, } - b, err := EncodeSetRecordStatusPre(pre) + b, err := EncodeSetRecordStatus(srs) if err != nil { return nil, err } err = t.pluginHook(HookSetRecordStatusPre, string(b)) if err != nil { - return nil, fmt.Errorf("pluginHook %v: %v", - Hooks[HookSetRecordStatusPre], err) + return nil, err } // Update record @@ -1097,16 +1134,6 @@ func (t *TlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, } // Call post plugin hooks - post := SetRecordStatusPost{ - Record: *r, - RecordMetadata: rm, - MDAppend: mdAppend, - MDOverwrite: mdOverwrite, - } - b, err = EncodeSetRecordStatusPost(post) - if err != nil { - return nil, err - } err = t.pluginHook(HookSetRecordStatusPost, string(b)) if err != nil { log.Errorf("SetUnvettedStatus: pluginHook %v: %v", @@ -1194,20 +1221,19 @@ func (t *TlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) // Call pre plugin hooks - pre := SetRecordStatusPre{ + srs := SetRecordStatus{ Record: *r, RecordMetadata: rm, MDAppend: mdAppend, MDOverwrite: mdOverwrite, } - b, err := EncodeSetRecordStatusPre(pre) + b, err := EncodeSetRecordStatus(srs) if err != nil { return nil, err } err = t.pluginHook(HookSetRecordStatusPre, string(b)) if err != nil { - return nil, fmt.Errorf("pluginHook %v: %v", - Hooks[HookSetRecordStatusPre], err) + return nil, err } // Update record @@ -1228,16 +1254,6 @@ func (t *TlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md } // Call post plugin hooks - post := SetRecordStatusPost{ - Record: *r, - RecordMetadata: rm, - MDAppend: mdAppend, - MDOverwrite: mdOverwrite, - } - b, err = EncodeSetRecordStatusPost(post) - if err != nil { - return nil, err - } err = t.pluginHook(HookSetRecordStatusPost, string(b)) if err != nil { log.Errorf("SetVettedStatus: pluginHook %v: %v", @@ -1287,9 +1303,6 @@ func (t *TlogBackend) InventoryByStatus() (*backend.InventoryByStatus, error) { } func (t *TlogBackend) pluginHook(h HookT, payload string) error { - t.RLock() - defer t.RUnlock() - // Pass hook event and payload to each plugin for _, v := range t.plugins { err := v.ctx.Hook(h, payload) diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index af2923833..4c05bc1d0 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -109,7 +109,7 @@ type ProposalMetadata struct { Name string `json:"name"` // Proposal name // LinkTo specifies a public proposal token to link this proposal - // to. Ex, an RFP sumbssion must link to the RFP proposal. + // to. Ex, an RFP submission must link to the RFP proposal. LinkTo string `json:"linkto,omitempty"` // LinkBy is a UNIX timestamp that serves as a deadline for other @@ -118,8 +118,8 @@ type ProposalMetadata struct { LinkBy int64 `json:"linkby,omitempty"` } -// CensorshipRecord contains the proof that a proposal was accepted for review -// by the server. The proof is verifiable on the client side. +// CensorshipRecord contains cryptographic proof that a proposal was accepted +// for review by the server. The proof is verifiable by the client. type CensorshipRecord struct { // Token is a random censorship token that is generated by the // server. It serves as a unique identifier for the proposal. @@ -134,8 +134,11 @@ type CensorshipRecord struct { } // SetStatus represents a proposal status change. +// +// Signature is the client signature of the Token+Version+Status+Reason. type StatusChange struct { Status PropStatusT `json:"status"` + Version string `json:"version"` Message string `json:"message,omitempty"` PublicKey string `json:"publickey"` Signature string `json:"signature"` @@ -158,6 +161,14 @@ type ProposalRecord struct { Files []File `json:"files"` // Proposal files Metadata []Metadata `json:"metadata"` // User defined metadata + // LinkedFrom contains a list of public proposals that have linked + // to this proposal. A link is established when a child proposal + // specifies this proposal using the LinkTo field of the + // ProposalMetadata. + LinkedFrom []string `json:"linkedfrom"` + + // CensorshipRecord contains cryptographic proof that the proposal + // was received by the server. CensorshipRecord CensorshipRecord `json:"censorshiprecord"` } @@ -168,15 +179,15 @@ type ProposalRecord struct { // Signature is the client signature of the proposal merkle root. The merkle // root is the ordered merkle root of all proposal Files and Metadata. type ProposalNew struct { - Files []File `json:"files"` - Metadata []Metadata `json:"metadata"` - PublicKey string `json:"publickey"` - Signature string `json:"signature"` + Files []File `json:"files"` // Proposal files + Metadata []Metadata `json:"metadata"` // User defined metadata + PublicKey string `json:"publickey"` // Key used for signature + Signature string `json:"signature"` // Signature of merkle root } // ProposalNewReply is the reply to the ProposalNew command. type ProposalNewReply struct { - CensorshipRecord CensorshipRecord `json:"censorshiprecord"` + Proposal ProposalRecord `json:"proposal"` } // ProposalEdit edits an existing proposal. @@ -186,16 +197,16 @@ type ProposalNewReply struct { // Signature is the client signature of the proposal merkle root. The merkle // root is the ordered merkle root of all proposal Files and Metadata. type ProposalEdit struct { - Token string `json:"token"` - Files []File `json:"files"` - Metadata []Metadata `json:"metadata"` - PublicKey string `json:"publickey"` - Signature string `json:"signature"` + Token string `json:"token"` // Censorship token + Files []File `json:"files"` // Proposal files + Metadata []Metadata `json:"metadata"` // User defined metadata + PublicKey string `json:"publickey"` // Key used for signature + Signature string `json:"signature"` // Signature of merkle root } // ProposalEditReply is the reply to the ProposalEdit command. type ProposalEditReply struct { - CensorshipRecord CensorshipRecord `json:"censorshiprecord"` + Proposal ProposalRecord `json:"proposal"` } // ProposalSetStatus sets the status of a proposal. Some status changes require @@ -212,7 +223,9 @@ type ProposalSetStatus struct { } // ProposalSetStatusReply is the reply to the ProposalSetStatus command. -type ProposalSetStatusReply struct{} +type ProposalSetStatusReply struct { + Timestamp int64 `json:"timestamp"` +} // ProposalDetails retrieves a proposal record. If a version is not specified, // the latest version will be returned. diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index a654f2f24..fa2fbff5a 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -28,17 +28,35 @@ import ( // TODO use pi errors instead of www errors +type propStateT int + const ( // MIME types mimeTypeText = "text/plain" mimeTypeTextUTF8 = "text/plain; charset=utf-8" mimeTypePNG = "image/png" + + // Proposal states. Proposal states correspond that politeiad + // routes. + propStateInvalid propStateT = 0 + propStateUnvetted propStateT = 1 + propStateVetted propStateT = 2 ) var ( validProposalName = regexp.MustCompile(createProposalNameRegex()) ) +func propStateFromStatus(s pi.PropStatusT) propStateT { + switch s { + case pi.PropStatusUnvetted, pi.PropStatusCensored: + return propStateUnvetted + case pi.PropStatusPublic, pi.PropStatusAbandoned: + return propStateVetted + } + return propStateInvalid +} + func convertUserErrFromSignatureErr(err error) www.UserError { var e util.SignatureError var s www.ErrorStatusT @@ -524,8 +542,6 @@ func (p *politeiawww) processProposalNew(pn pi.ProposalNew, usr user.User) (*pi. if err != nil { return nil, err } - - // Handle response var nrr pd.NewRecordReply err = json.Unmarshal(resBody, &nrr) if err != nil { @@ -551,12 +567,152 @@ func (p *politeiawww) processProposalNew(pn pi.ProposalNew, usr user.User) (*pi. username: usr.Username, }) - log.Infof("Submitted proposal: %v %v", cr.Token, pm.Name) + log.Infof("Proposal submitted: %v %v", cr.Token, pm.Name) for k, f := range pn.Files { log.Infof("%02v: %v %v", k, f.Name, f.Digest) } - return &pi.ProposalNewReply{ - CensorshipRecord: cr, - }, nil + // TODO return full proposal + _ = cr + return &pi.ProposalNewReply{}, nil +} + +// TODO implement proposalRecord +func (p *politeiawww) proposalRecord(token string) (*pi.ProposalRecord, error) { + return nil, nil +} + +// filesToDel returns the names of the files that are included in current but +// are not included in updated. These are the files that need to be deleted +// from a proposal on update. +func filesToDel(current []pi.File, updated []pi.File) []string { + curr := make(map[string]struct{}, len(current)) // [name]struct + for _, v := range updated { + curr[v.Name] = struct{}{} + } + + del := make([]string, 0, len(current)) + for _, v := range current { + _, ok := curr[v.Name] + if !ok { + del = append(del, v.Name) + } + } + + return del +} + +func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*pi.ProposalEditReply, error) { + log.Tracef("processProposalEdit: %v", pe.Token) + + // Verify user signed using active identity + if usr.PublicKey() != pe.PublicKey { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidSigningKey, + ErrorContext: []string{"not user's active identity"}, + } + } + + // Verify proposal + pm, err := p.verifyProposal(pe.Files, pe.Metadata, + pe.PublicKey, pe.Signature) + if err != nil { + return nil, err + } + if !tokenIsValid(pe.Token) { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidCensorshipToken, + ErrorContext: []string{pe.Token}, + } + } + + // Get the current proposal + curr, err := p.proposalRecord(pe.Token) + if err != nil { + // TODO www.ErrorStatusProposalNotFound + return nil, err + } + + // Verify the user is the author. The public keys are not static + // values so the user IDs must be compared directly. + if curr.UserID != usr.ID.String() { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusUserNotAuthor, + } + } + + // Setup politeiad files. The Metadata objects are converted to + // politeiad files instead of metadata streams since they contain + // user defined data that needs to be included in the merkle root + // that politeiad signs. + files := convertFilesFromPi(pe.Files) + for _, v := range pe.Metadata { + switch v.Hint { + case pi.HintProposalMetadata: + files = append(files, convertFileFromMetadata(v)) + } + } + + // Setup metadata stream + pg := piplugin.ProposalGeneral{ + PublicKey: pe.PublicKey, + Signature: pe.Signature, + Timestamp: time.Now().Unix(), + } + b, err := piplugin.EncodeProposalGeneral(pg) + if err != nil { + return nil, err + } + metadata := []pd.MetadataStream{ + { + ID: piplugin.MDStreamIDProposalGeneral, + Payload: string(b), + }, + } + + // Send politeiad request + var route string + switch propStateFromStatus(curr.Status) { + case propStateUnvetted: + route = pd.UpdateUnvettedRoute + case propStateVetted: + route = pd.UpdateVettedRoute + } + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return nil, err + } + ur := pd.UpdateRecord{ + Token: pe.Token, + Challenge: hex.EncodeToString(challenge), + MDOverwrite: metadata, + FilesAdd: files, + FilesDel: filesToDel(curr.Files, pe.Files), + } + resBody, err := p.makeRequest(http.MethodPost, route, ur) + if err != nil { + // TODO verify that this will throw an error if no proposal files + // were changed. + // TODO plugin error pass through + return nil, err + } + var urr pd.UpdateRecordReply + err = json.Unmarshal(resBody, &urr) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(p.cfg.Identity, challenge, urr.Response) + if err != nil { + return nil, err + } + + // TODO Emit an edit proposal event + + log.Infof("Proposal edited: %v %v", pe.Token, pm.Name) + for k, f := range pe.Files { + log.Infof("%02v: %v %v", k, f.Name, f.Digest) + } + + // TODO return full proposal + return &pi.ProposalEditReply{}, nil } diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index be32335c4..d560bb680 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -1083,103 +1083,25 @@ func (p *politeiawww) processSetProposalStatus(sps www.SetProposalStatus, u *use }, nil } -// filesToDel returns the names of the files that are included in filesOld but -// are not included in filesNew. These are the files that need to be deleted -// from a proposal on update. -func filesToDel(filesOld []www.File, filesNew []www.File) []string { - newf := make(map[string]struct{}, len(filesOld)) // [name]struct - for _, v := range filesNew { - newf[v.Name] = struct{}{} - } - - del := make([]string, 0, len(filesOld)) - for _, v := range filesOld { - _, ok := newf[v.Name] - if !ok { - del = append(del, v.Name) - } - } - - return del -} - // processEditProposal attempts to edit a proposal on politeiad. func (p *politeiawww) processEditProposal(ep www.EditProposal, u *user.User) (*www.EditProposalReply, error) { log.Tracef("processEditProposal %v", ep.Token) - // Make sure token is valid and not a prefix - if !tokenIsValid(ep.Token) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: []string{ep.Token}, - } - } - - // Validate proposal status - cachedProp, err := p.getProp(ep.Token) - if err != nil { - /* - // TODO + /* + // Validate proposal status + cachedProp, err := p.getProp(ep.Token) + if err != nil { if err == cache.ErrRecordNotFound { err = www.UserError{ ErrorCode: www.ErrorStatusProposalNotFound, } } - */ - return nil, err - } - - if cachedProp.Status == www.PropStatusCensored || - cachedProp.Status == www.PropStatusAbandoned { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusWrongStatus, - } - } - - // Ensure user is the proposal author - if cachedProp.UserId != u.ID.String() { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusUserNotAuthor, - } - } - - // Validate proposal vote status - bb, err := p.getBestBlock() - if err != nil { - return nil, err - } - vs, err := p.voteSummaryGet(ep.Token, bb) - if err != nil { - return nil, err - } - if vs.Status != www.PropVoteStatusNotAuthorized { - e := fmt.Sprintf("got vote status %v, want %v", - vs.Status, www.PropVoteStatusNotAuthorized) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - ErrorContext: []string{e}, + return nil, err } - } - - // Validate proposal. Convert it to www.NewProposal so that - // we can reuse the function validateProposal. - np := www.NewProposal{ - Files: ep.Files, - Metadata: ep.Metadata, - PublicKey: ep.PublicKey, - Signature: ep.Signature, - } - pm, err := p.validateProposal(np, u) - if err != nil { - return nil, err - } - // Check if there were changes in the proposal by comparing - // their merkle roots. This captures changes that were made - // to either the files or the metadata. - _ = pm - /* - // TODO + // Check if there were changes in the proposal by comparing + // their merkle roots. This captures changes that were made + // to either the files or the metadata. mr, err := wwwutil.MerkleRoot(ep.Files, ep.Metadata) if err != nil { return nil, err @@ -1189,106 +1111,9 @@ func (p *politeiawww) processEditProposal(ep www.EditProposal, u *user.User) (*w ErrorCode: www.ErrorStatusNoProposalChanges, } } - if cachedProp.State == www.PropStateVetted && - cachedProp.LinkTo != pm.LinkTo { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkTo, - ErrorContext: []string{"linkto cannot change once public"}, - } - } */ - // politeiad only includes files in its merkle root calc, not the - // metadata streams. This is why we include the ProposalMetadata - // as a politeiad file. - - // Setup files - files := convertPropFilesFromWWW(ep.Files) - for _, v := range ep.Metadata { - switch v.Hint { - case www.HintProposalMetadata: - // files = append(files, convertFileFromMetadata(v)) - } - } - - // Setup metadata streams - pg := mdstream.ProposalGeneralV2{ - Version: mdstream.VersionProposalGeneral, - Timestamp: time.Now().Unix(), - PublicKey: ep.PublicKey, - Signature: ep.Signature, - } - pgb, err := mdstream.EncodeProposalGeneralV2(pg) - if err != nil { - return nil, err - } - mds := []pd.MetadataStream{ - { - ID: mdstream.IDProposalGeneral, - Payload: string(pgb), - }, - } - - // Setup politeiad request - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - - e := pd.UpdateRecord{ - Token: ep.Token, - Challenge: hex.EncodeToString(challenge), - MDOverwrite: mds, - FilesAdd: files, - FilesDel: filesToDel(cachedProp.Files, ep.Files), - } - - var pdRoute string - switch cachedProp.Status { - case www.PropStatusNotReviewed, www.PropStatusUnreviewedChanges: - pdRoute = pd.UpdateUnvettedRoute - case www.PropStatusPublic: - pdRoute = pd.UpdateVettedRoute - default: - return nil, www.UserError{ - ErrorCode: www.ErrorStatusWrongStatus, - } - } - - // Send politeiad request - responseBody, err := p.makeRequest(http.MethodPost, pdRoute, e) - if err != nil { - return nil, err - } - - // Handle response - var pdReply pd.UpdateRecordReply - err = json.Unmarshal(responseBody, &pdReply) - if err != nil { - return nil, fmt.Errorf("Unmarshal UpdateUnvettedReply: %v", err) - } - - err = util.VerifyChallenge(p.cfg.Identity, challenge, pdReply.Response) - if err != nil { - return nil, err - } - - // Get proposal from the cache - updatedProp, err := p.getProp(ep.Token) - if err != nil { - return nil, err - } - - // Fire off edit proposal event - p.fireEvent(EventTypeProposalEdited, - EventDataProposalEdited{ - Proposal: updatedProp, - }, - ) - - return &www.EditProposalReply{ - Proposal: *updatedProp, - }, nil + return nil, nil } // processAllVetted returns an array of vetted proposals. The maximum number From 9ffb3aecb8038200c4f1618151b0fe63c82806a6 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 7 Sep 2020 17:56:51 -0500 Subject: [PATCH 028/449] add plugin user errors and pi user errors --- plugins/ticketvote/ticketvote.go | 8 +- politeiad/api/v1/v1.go | 4 + politeiad/backend/tlogbe/plugins/comments.go | 2 +- .../backend/tlogbe/plugins/ticketvote.go | 22 +- politeiawww/api/pi/v1/v1.go | 33 ++ politeiawww/api/www/v1/v1.go | 32 +- politeiawww/convert.go | 25 - politeiawww/middleware.go | 14 +- politeiawww/politeiad.go | 87 ++++ politeiawww/www.go | 431 ++++++++++++++---- 10 files changed, 508 insertions(+), 150 deletions(-) create mode 100644 politeiawww/politeiad.go diff --git a/plugins/ticketvote/ticketvote.go b/plugins/ticketvote/ticketvote.go index f879e6e3c..598217ea1 100644 --- a/plugins/ticketvote/ticketvote.go +++ b/plugins/ticketvote/ticketvote.go @@ -96,9 +96,9 @@ const ( ErrorStatusPublicKeyInvalid ErrorStatusT = 2 ErrorStatusSignatureInvalid ErrorStatusT = 3 ErrorStatusRecordNotFound ErrorStatusT = 4 - ErrorStatusRecordStateInvalid ErrorStatusT = 5 + ErrorStatusRecordStatusInvalid ErrorStatusT = 5 ErrorStatusAuthorizationInvalid ErrorStatusT = 6 - ErrorStatusVoteInvalid ErrorStatusT = 7 + ErrorStatusVoteDetailsInvalid ErrorStatusT = 7 ErrorStatusVoteStatusInvalid ErrorStatusT = 8 ErrorStatusBallotInvalid ErrorStatusT = 9 ) @@ -132,9 +132,9 @@ var ( ErrorStatusPublicKeyInvalid: "public key invalid", ErrorStatusSignatureInvalid: "signature invalid", ErrorStatusRecordNotFound: "record not found", - ErrorStatusRecordStateInvalid: "record state invalid", + ErrorStatusRecordStatusInvalid: "record status invalid", ErrorStatusAuthorizationInvalid: "authorization invalid", - ErrorStatusVoteInvalid: "vote invalid", + ErrorStatusVoteDetailsInvalid: "vote details invalid", ErrorStatusVoteStatusInvalid: "vote status invalid", ErrorStatusBallotInvalid: "ballot invalid", } diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index b073134cd..19f78f36f 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -365,6 +365,10 @@ type InventoryReply struct { type UserErrorReply struct { ErrorCode ErrorStatusT `json:"errorcode"` // Numeric error code ErrorContext []string `json:"errorcontext,omitempty"` // Additional error information + + // Plugin will be populated with the plugin ID if the error was a + // plugin user error. + Plugin string `json:"plugin,omitempty"` } // ServerErrorReply returns an error code that can be correlated with diff --git a/politeiad/backend/tlogbe/plugins/comments.go b/politeiad/backend/tlogbe/plugins/comments.go index 50599eb80..c849e30b4 100644 --- a/politeiad/backend/tlogbe/plugins/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments.go @@ -702,7 +702,7 @@ func (p *commentsPlugin) new(client *tlogbe.RecordClient, n comments.New, encryp if n.ParentID > 0 && !commentExists(*idx, n.ParentID) { return nil, comments.UserError{ ErrorCode: comments.ErrorStatusParentIDInvalid, - ErrorContext: []string{"comment not found"}, + ErrorContext: []string{"parent ID comment not found"}, } } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote.go index c6576eb8e..fade7a159 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote.go @@ -898,7 +898,7 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { } if client.State != tlogbe.RecordStateVetted { return "", ticketvote.UserError{ - ErrorCode: ticketvote.ErrorStatusRecordStateInvalid, + ErrorCode: ticketvote.ErrorStatusRecordStatusInvalid, ErrorContext: []string{"record not vetted"}, } } @@ -1035,7 +1035,7 @@ func voteDetailsVerify(vote ticketvote.VoteDetails, voteDurationMin, voteDuratio default: e := fmt.Sprintf("invalid type %v", vote.Type) return ticketvote.UserError{ - ErrorCode: ticketvote.ErrorStatusVoteInvalid, + ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, ErrorContext: []string{e}, } } @@ -1046,28 +1046,28 @@ func voteDetailsVerify(vote ticketvote.VoteDetails, voteDurationMin, voteDuratio e := fmt.Sprintf("duration %v exceeds max duration %v", vote.Duration, voteDurationMax) return ticketvote.UserError{ - ErrorCode: ticketvote.ErrorStatusVoteInvalid, + ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, ErrorContext: []string{e}, } case vote.Duration < voteDurationMin: e := fmt.Sprintf("duration %v under min duration %v", vote.Duration, voteDurationMin) return ticketvote.UserError{ - ErrorCode: ticketvote.ErrorStatusVoteInvalid, + ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, ErrorContext: []string{e}, } case vote.QuorumPercentage > 100: e := fmt.Sprintf("quorum percent %v exceeds 100 percent", vote.QuorumPercentage) return ticketvote.UserError{ - ErrorCode: ticketvote.ErrorStatusVoteInvalid, + ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, ErrorContext: []string{e}, } case vote.PassPercentage > 100: e := fmt.Sprintf("pass percent %v exceeds 100 percent", vote.PassPercentage) return ticketvote.UserError{ - ErrorCode: ticketvote.ErrorStatusVoteInvalid, + ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, ErrorContext: []string{e}, } } @@ -1076,7 +1076,7 @@ func voteDetailsVerify(vote ticketvote.VoteDetails, voteDurationMin, voteDuratio // requirements. if len(vote.Options) == 0 { return ticketvote.UserError{ - ErrorCode: ticketvote.ErrorStatusVoteInvalid, + ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, ErrorContext: []string{"no vote options found"}, } } @@ -1089,7 +1089,7 @@ func voteDetailsVerify(vote ticketvote.VoteDetails, voteDurationMin, voteDuratio e := fmt.Sprintf("vote options count got %v, want 2", len(vote.Options)) return ticketvote.UserError{ - ErrorCode: ticketvote.ErrorStatusVoteInvalid, + ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, ErrorContext: []string{e}, } } @@ -1117,7 +1117,7 @@ func voteDetailsVerify(vote ticketvote.VoteDetails, voteDurationMin, voteDuratio e := fmt.Sprintf("vote option IDs not found: %v", strings.Join(missing, ",")) return ticketvote.UserError{ - ErrorCode: ticketvote.ErrorStatusVoteInvalid, + ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, ErrorContext: []string{e}, } } @@ -1128,7 +1128,7 @@ func voteDetailsVerify(vote ticketvote.VoteDetails, voteDurationMin, voteDuratio err := voteBitVerify(vote.Options, vote.Mask, v.Bit) if err != nil { return ticketvote.UserError{ - ErrorCode: ticketvote.ErrorStatusVoteInvalid, + ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, ErrorContext: []string{err.Error()}, } } @@ -1181,7 +1181,7 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { } if client.State != tlogbe.RecordStateVetted { return "", ticketvote.UserError{ - ErrorCode: ticketvote.ErrorStatusRecordStateInvalid, + ErrorCode: ticketvote.ErrorStatusRecordStatusInvalid, ErrorContext: []string{"record not vetted"}, } } diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 4c05bc1d0..7155bd26c 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -75,13 +75,46 @@ const ( // requirements but still be rejected if it does not have the most // net yes votes. VoteTypeRunoff VoteT = 2 + + // Error status codes + ErrorStatusInvalid ErrorStatusT = 0 ) var ( // APIRoute is the prefix to the v2 API routes. APIRoute = fmt.Sprintf("/v%v", APIVersion) + + // ErrorStatus contains human readable error messages. + ErrorStatus = map[ErrorStatusT]string{ + ErrorStatusInvalid: "error status invalid", + } ) +// UserError represents an error that is caused by something that the user +// did (malformed input, bad timing, etc). +type UserError struct { + ErrorCode ErrorStatusT + ErrorContext []string +} + +// Error satisfies the error interface. +func (e UserError) Error() string { + return fmt.Sprintf("user error code: %v", e.ErrorCode) +} + +// ErrorReply are replies that the server returns when it encounters an +// unrecoverable problem while executing a command. The HTTP status code will +// be 500 and the ErrorCode field will contain a UNIX timestamp that the user +// can provide to the server admin to track down the error details in the logs. +type ErrorReply struct { + ErrorCode int64 `json:"errorcode"` +} + +// Error satisfies the error interface. +func (e ErrorReply) Error() string { + return fmt.Sprintf("server error: %v", e.ErrorCode) +} + // File describes an individual file that is part of the proposal. The // directory structure must be flattened. type File struct { diff --git a/politeiawww/api/www/v1/v1.go b/politeiawww/api/www/v1/v1.go index 01c92ce54..c874dad1f 100644 --- a/politeiawww/api/www/v1/v1.go +++ b/politeiawww/api/www/v1/v1.go @@ -552,34 +552,20 @@ func (e UserError) Error() string { return fmt.Sprintf("user error code: %v", e.ErrorCode) } -// PDError is emitted when an HTTP error response is returned from Politeiad -// for a request. It contains the HTTP status code and the JSON response body. -type PDError struct { - HTTPCode int - ErrorReply PDErrorReply -} - -// Error satisfies the error interface. -func (e PDError) Error() string { - return fmt.Sprintf("error from politeiad: %v %v", e.HTTPCode, - e.ErrorReply.ErrorCode) -} - -// PDErrorReply is an error reply returned from Politeiad whenever an -// error occurs. -type PDErrorReply struct { - ErrorCode int - ErrorContext []string -} - -// ErrorReply are replies that the server returns a when it encounters an -// unrecoverable problem while executing a command. The HTTP Error Code -// shall be 500 if it's an internal server error or 4xx if it's a user error. +// ErrorReply are replies that the server returns when it encounters an +// unrecoverable problem while executing a command. The HTTP status code will +// be 500 and the ErrorCode field will contain a UNIX timestamp that the user +// can provide to the server admin to track down the error details in the logs. type ErrorReply struct { ErrorCode int64 `json:"errorcode,omitempty"` ErrorContext []string `json:"errorcontext,omitempty"` } +// Error satisfies the error interface. +func (e ErrorReply) Error() string { + return fmt.Sprintf("server error: %v", e.ErrorCode) +} + // Version command is used to determine the lowest API version that this // backend supports and additionally it provides the route to said API. This // call is required in order to establish CSRF for the session. The client diff --git a/politeiawww/convert.go b/politeiawww/convert.go index c3153ca4e..725e178ba 100644 --- a/politeiawww/convert.go +++ b/politeiawww/convert.go @@ -280,31 +280,6 @@ func convertPropCensorFromWWW(f www.CensorshipRecord) pd.CensorshipRecord { } } -func convertErrorStatusFromPD(s int) www.ErrorStatusT { - switch pd.ErrorStatusT(s) { - case pd.ErrorStatusInvalidFileDigest: - return www.ErrorStatusInvalidFileDigest - case pd.ErrorStatusInvalidBase64: - return www.ErrorStatusInvalidBase64 - case pd.ErrorStatusInvalidMIMEType: - return www.ErrorStatusInvalidMIMEType - case pd.ErrorStatusUnsupportedMIMEType: - return www.ErrorStatusUnsupportedMIMEType - case pd.ErrorStatusInvalidRecordStatusTransition: - return www.ErrorStatusInvalidPropStatusTransition - case pd.ErrorStatusInvalidFilename: - return www.ErrorStatusInvalidFilename - - // These cases are intentionally omitted because - // they are indicative of some internal server error, - // so ErrorStatusInvalid is returned. - // - //case pd.ErrorStatusInvalidRequestPayload - //case pd.ErrorStatusInvalidChallenge - } - return www.ErrorStatusInvalid -} - func convertPropStatusToState(status www.PropStatusT) www.PropStateT { switch status { case www.PropStatusNotReviewed, www.PropStatusUnreviewedChanges, diff --git a/politeiawww/middleware.go b/politeiawww/middleware.go index ee91ff465..86343bf6e 100644 --- a/politeiawww/middleware.go +++ b/politeiawww/middleware.go @@ -24,16 +24,16 @@ func (p *politeiawww) isLoggedIn(f http.HandlerFunc) http.HandlerFunc { id, err := p.getSessionUserID(w, r) if err != nil { - util.RespondWithJSON(w, http.StatusUnauthorized, www.ErrorReply{ - ErrorCode: int64(www.ErrorStatusNotLoggedIn), + util.RespondWithJSON(w, http.StatusUnauthorized, www.UserError{ + ErrorCode: www.ErrorStatusNotLoggedIn, }) return } // Check if user is authenticated if id == "" { - util.RespondWithJSON(w, http.StatusUnauthorized, www.ErrorReply{ - ErrorCode: int64(www.ErrorStatusNotLoggedIn), + util.RespondWithJSON(w, http.StatusUnauthorized, www.UserError{ + ErrorCode: www.ErrorStatusNotLoggedIn, }) return } @@ -63,13 +63,13 @@ func (p *politeiawww) isLoggedInAsAdmin(f http.HandlerFunc) http.HandlerFunc { isAdmin, err := p.isAdmin(w, r) if err != nil { log.Errorf("isLoggedInAsAdmin: isAdmin %v", err) - util.RespondWithJSON(w, http.StatusUnauthorized, www.ErrorReply{ - ErrorCode: int64(www.ErrorStatusNotLoggedIn), + util.RespondWithJSON(w, http.StatusUnauthorized, www.UserError{ + ErrorCode: www.ErrorStatusNotLoggedIn, }) return } if !isAdmin { - util.RespondWithJSON(w, http.StatusForbidden, www.ErrorReply{}) + util.RespondWithJSON(w, http.StatusForbidden, www.UserError{}) return } diff --git a/politeiawww/politeiad.go b/politeiawww/politeiad.go new file mode 100644 index 000000000..52312e8f8 --- /dev/null +++ b/politeiawww/politeiad.go @@ -0,0 +1,87 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + + "github.com/decred/politeia/util" +) + +// pdErrorReply represents the request body that is returned from politeaid +// when an error occurs. PluginID will be populated if this is a plugin error. +type pdErrorReply struct { + ErrorCode int + ErrorContext []string + PluginID string +} + +// pdError represents a politeiad error. +type pdError struct { + HTTPCode int + ErrorReply pdErrorReply +} + +// Error satisfies the error interface. +func (e pdError) Error() string { + return fmt.Sprintf("error from politeiad: %v %v", + e.HTTPCode, e.ErrorReply.ErrorCode) +} + +// makeRequest makes a politeiad http request to the method and route provided, +// serializing the provided object as the request body. A pdError is returned +// if politeiad does not respond with a 200. +func (p *politeiawww) makeRequest(method string, route string, v interface{}) ([]byte, error) { + var ( + requestBody []byte + err error + ) + if v != nil { + requestBody, err = json.Marshal(v) + if err != nil { + return nil, err + } + } + + fullRoute := p.cfg.RPCHost + route + + if p.client == nil { + p.client, err = util.NewClient(false, p.cfg.RPCCert) + if err != nil { + return nil, err + } + } + + req, err := http.NewRequest(method, fullRoute, + bytes.NewReader(requestBody)) + if err != nil { + return nil, err + } + req.SetBasicAuth(p.cfg.RPCUser, p.cfg.RPCPass) + r, err := p.client.Do(req) + if err != nil { + return nil, err + } + defer r.Body.Close() + + if r.StatusCode != http.StatusOK { + var e pdErrorReply + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&e); err != nil { + return nil, err + } + + return nil, pdError{ + HTTPCode: r.StatusCode, + ErrorReply: e, + } + } + + responseBody := util.ConvertBodyToByteArray(r.Body, false) + return responseBody, nil +} diff --git a/politeiawww/www.go b/politeiawww/www.go index cc035f958..d132c23b6 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -6,7 +6,6 @@ package main import ( "bufio" - "bytes" "crypto/elliptic" "crypto/tls" _ "encoding/gob" @@ -25,8 +24,12 @@ import ( "time" "github.com/decred/politeia/mdstream" + "github.com/decred/politeia/plugins/comments" + piplugin "github.com/decred/politeia/plugins/pi" + "github.com/decred/politeia/plugins/ticketvote" pd "github.com/decred/politeia/politeiad/api/v1" cms "github.com/decred/politeia/politeiawww/api/cms/v1" + pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" database "github.com/decred/politeia/politeiawww/cmsdatabase" cmsdb "github.com/decred/politeia/politeiawww/cmsdatabase/cockroachdb" @@ -52,6 +55,206 @@ const ( csrfKeyLength = 32 ) +func convertWWWErrorStatusFromPD(e pd.ErrorStatusT) www.ErrorStatusT { + switch e { + case pd.ErrorStatusInvalidFilename: + return www.ErrorStatusInvalidFilename + case pd.ErrorStatusInvalidFileDigest: + return www.ErrorStatusInvalidFileDigest + case pd.ErrorStatusInvalidBase64: + return www.ErrorStatusInvalidBase64 + case pd.ErrorStatusInvalidMIMEType: + return www.ErrorStatusInvalidMIMEType + case pd.ErrorStatusUnsupportedMIMEType: + return www.ErrorStatusUnsupportedMIMEType + case pd.ErrorStatusInvalidRecordStatusTransition: + return www.ErrorStatusInvalidPropStatusTransition + case pd.ErrorStatusInvalidRequestPayload: + // Intentionally omitted because this indicates a politeiawww + // server error so a ErrorStatusInvalid should be returned. + case pd.ErrorStatusInvalidChallenge: + // Intentionally omitted because this indicates a politeiawww + // server error so a ErrorStatusInvalid should be returned. + } + return www.ErrorStatusInvalid +} + +func convertWWWErrorStatusFromPiPlugin(e piplugin.ErrorStatusT) www.ErrorStatusT { + switch e { + case piplugin.ErrorStatusLinkToInvalid: + return www.ErrorStatusInvalidLinkTo + case piplugin.ErrorStatusWrongPropStatus: + return www.ErrorStatusWrongStatus + case piplugin.ErrorStatusWrongVoteStatus: + return www.ErrorStatusWrongVoteStatus + } + return www.ErrorStatusInvalid +} + +func convertWWWErrorStatusFromComments(e comments.ErrorStatusT) www.ErrorStatusT { + switch e { + case comments.ErrorStatusTokenInvalid: + return www.ErrorStatusInvalidCensorshipToken + case comments.ErrorStatusPublicKeyInvalid: + return www.ErrorStatusInvalidPublicKey + case comments.ErrorStatusSignatureInvalid: + return www.ErrorStatusInvalidSignature + case comments.ErrorStatusRecordNotFound: + return www.ErrorStatusProposalNotFound + case comments.ErrorStatusCommentNotFound: + return www.ErrorStatusCommentNotFound + case comments.ErrorStatusParentIDInvalid: + return www.ErrorStatusCommentNotFound + case comments.ErrorStatusNoCommentChanges: + // Intentionally omitted. The www API does not allow for comment + // changes. + case comments.ErrorStatusVoteInvalid: + return www.ErrorStatusInvalidLikeCommentAction + case comments.ErrorStatusMaxVoteChanges: + return www.ErrorStatusInvalidLikeCommentAction + } + return www.ErrorStatusInvalid +} + +func convertWWWErrorStatusFromTicketVote(e ticketvote.ErrorStatusT) www.ErrorStatusT { + switch e { + case ticketvote.ErrorStatusTokenInvalid: + return www.ErrorStatusInvalidCensorshipToken + case ticketvote.ErrorStatusPublicKeyInvalid: + return www.ErrorStatusInvalidPublicKey + case ticketvote.ErrorStatusSignatureInvalid: + return www.ErrorStatusInvalidSignature + case ticketvote.ErrorStatusRecordNotFound: + return www.ErrorStatusProposalNotFound + case ticketvote.ErrorStatusRecordStatusInvalid: + return www.ErrorStatusWrongStatus + case ticketvote.ErrorStatusVoteDetailsInvalid: + return www.ErrorStatusInvalidPropVoteParams + case ticketvote.ErrorStatusVoteStatusInvalid: + return www.ErrorStatusInvalidPropVoteStatus + case ticketvote.ErrorStatusBallotInvalid: + } + return www.ErrorStatusInvalid +} + +// convertWWWErrorStatus attempts to convert the provided politeiad plugin ID +// and error code into a www ErrorStatusT. If a plugin ID is provided the error +// code is assumed to be a user error code from the specified plugin API. If +// no plugin ID is provided the error code is assumed to be a user error code +// from the politeiad API. +func convertWWWErrorStatus(pluginID string, errCode int) www.ErrorStatusT { + switch pluginID { + case "": + // politeiad API + e := pd.ErrorStatusT(errCode) + return convertWWWErrorStatusFromPD(e) + case piplugin.ID: + // Pi plugin + e := piplugin.ErrorStatusT(errCode) + return convertWWWErrorStatusFromPiPlugin(e) + case comments.ID: + // Comments plugin + e := comments.ErrorStatusT(errCode) + return convertWWWErrorStatusFromComments(e) + case ticketvote.ID: + // Ticket vote plugin + e := ticketvote.ErrorStatusT(errCode) + return convertWWWErrorStatusFromTicketVote(e) + } + + // No corresponding www error status found + return www.ErrorStatusInvalid +} + +// TODO add pi error statuses +func convertPiErrorStatusFromPD(e pd.ErrorStatusT) pi.ErrorStatusT { + switch e { + case pd.ErrorStatusInvalidFilename: + case pd.ErrorStatusInvalidFileDigest: + case pd.ErrorStatusInvalidBase64: + case pd.ErrorStatusInvalidMIMEType: + case pd.ErrorStatusUnsupportedMIMEType: + case pd.ErrorStatusInvalidRecordStatusTransition: + case pd.ErrorStatusInvalidRequestPayload: + // Intentionally omitted because this indicates a politeiawww + // server error so a ErrorStatusInvalid should be returned. + case pd.ErrorStatusInvalidChallenge: + // Intentionally omitted because this indicates a politeiawww + // server error so a ErrorStatusInvalid should be returned. + } + return pi.ErrorStatusInvalid +} + +// TODO add pi error statuses +func convertPiErrorStatusFromPiPlugin(e piplugin.ErrorStatusT) pi.ErrorStatusT { + switch e { + case piplugin.ErrorStatusLinkToInvalid: + case piplugin.ErrorStatusWrongPropStatus: + case piplugin.ErrorStatusWrongVoteStatus: + } + return pi.ErrorStatusInvalid +} + +// TODO add pi error statuses +func convertPiErrorStatusFromComments(e comments.ErrorStatusT) pi.ErrorStatusT { + switch e { + case comments.ErrorStatusTokenInvalid: + case comments.ErrorStatusPublicKeyInvalid: + case comments.ErrorStatusSignatureInvalid: + case comments.ErrorStatusRecordNotFound: + case comments.ErrorStatusCommentNotFound: + case comments.ErrorStatusParentIDInvalid: + case comments.ErrorStatusNoCommentChanges: + case comments.ErrorStatusVoteInvalid: + case comments.ErrorStatusMaxVoteChanges: + } + return pi.ErrorStatusInvalid +} + +// TODO add pi error statuses +func convertPiErrorStatusFromTicketVote(e ticketvote.ErrorStatusT) pi.ErrorStatusT { + switch e { + case ticketvote.ErrorStatusTokenInvalid: + case ticketvote.ErrorStatusPublicKeyInvalid: + case ticketvote.ErrorStatusSignatureInvalid: + case ticketvote.ErrorStatusRecordNotFound: + case ticketvote.ErrorStatusRecordStatusInvalid: + case ticketvote.ErrorStatusVoteDetailsInvalid: + case ticketvote.ErrorStatusVoteStatusInvalid: + case ticketvote.ErrorStatusBallotInvalid: + } + return pi.ErrorStatusInvalid +} + +// convertPiErrorStatus attempts to convert the provided politeiad plugin ID +// and error code into a pi ErrorStatusT. If a plugin ID is provided the error +// code is assumed to be a user error code from the specified plugin API. If +// no plugin ID is provided the error code is assumed to be a user error code +// from the politeiad API. +func convertPiErrorStatus(pluginID string, errCode int) pi.ErrorStatusT { + switch pluginID { + case "": + // politeiad API + e := pd.ErrorStatusT(errCode) + return convertPiErrorStatusFromPD(e) + case piplugin.ID: + // Pi plugin + e := piplugin.ErrorStatusT(errCode) + return convertPiErrorStatusFromPiPlugin(e) + case comments.ID: + // Comments plugin + e := comments.ErrorStatusT(errCode) + return convertPiErrorStatusFromComments(e) + case ticketvote.ID: + // Ticket vote plugin + e := ticketvote.ErrorStatusT(errCode) + return convertPiErrorStatusFromTicketVote(e) + } + + // No corresponding pi error status found + return pi.ErrorStatusInvalid +} + // Fetch remote identity func (p *politeiawww) getIdentity() error { id, err := util.RemoteIdentity(false, p.cfg.RPCHost, p.cfg.RPCCert) @@ -91,6 +294,99 @@ func (p *politeiawww) getIdentity() error { return nil } +// respondWithPiError returns an HTTP error status to the client. If it's a pi +// user error, it returns a 4xx HTTP status and the specific user error code. +// If it's an internal server error, it returns 500 and a UNIX timestamp which +// is also outputted to the logs so that it can be correlated later if the user +// files a complaint. +func respondWithPiError(w http.ResponseWriter, r *http.Request, format string, err error) { + // Check for pi user error + if ue, ok := err.(pi.UserError); ok { + // Error is a pi user error. Log it and return a 400. + if len(ue.ErrorContext) == 0 { + log.Infof("Pi user error: %v %v %v", + remoteAddr(r), int64(ue.ErrorCode), + pi.ErrorStatus[ue.ErrorCode]) + } else { + log.Errorf("Pi user error: %v %v %v: %v", + remoteAddr(r), int64(ue.ErrorCode), + pi.ErrorStatus[ue.ErrorCode], + strings.Join(ue.ErrorContext, ", ")) + } + + util.RespondWithJSON(w, http.StatusBadRequest, + pi.UserError{ + ErrorCode: ue.ErrorCode, + ErrorContext: ue.ErrorContext, + }) + return + } + + // Check for politeiad error + if pdErr, ok := err.(pdError); ok { + var ( + pluginID = pdErr.ErrorReply.PluginID + errCode = pdErr.ErrorReply.ErrorCode + errContext = pdErr.ErrorReply.ErrorContext + ) + + // Check if the politeiad error corresponds to a pi user error + piErrCode := convertPiErrorStatus(pluginID, errCode) + if piErrCode == pi.ErrorStatusInvalid { + // politeiad error does not correspond to a pi user error. Log + // it and return a 500. + t := time.Now().Unix() + if pluginID == "" { + log.Errorf("%v %v %v %v Internal error %v: error "+ + "code from politeiad: %v", remoteAddr(r), r.Method, + r.URL, r.Proto, t, errCode) + } else { + log.Errorf("%v %v %v %v Internal error %v: error "+ + "code from politeiad plugin %v: %v", remoteAddr(r), + r.Method, r.URL, r.Proto, t, pluginID, errCode) + } + + util.RespondWithJSON(w, http.StatusInternalServerError, + pi.ErrorReply{ + ErrorCode: t, + }) + return + } + + // politeiad error does correspond to a pi user error. Log it and + // return a 400. + if len(errContext) == 0 { + log.Infof("Pi user error: %v %v %v", + remoteAddr(r), int64(piErrCode), + pi.ErrorStatus[piErrCode]) + } else { + log.Infof("Pi user error: %v %v %v: %v", + remoteAddr(r), int64(piErrCode), + pi.ErrorStatus[piErrCode], + strings.Join(errContext, ", ")) + } + + util.RespondWithJSON(w, http.StatusBadRequest, + pi.UserError{ + ErrorCode: piErrCode, + ErrorContext: errContext, + }) + return + + } + + // Error is a politeiawww server error. Log it and return a 500. + t := time.Now().Unix() + log.Errorf("%v %v %v %v Internal error %v: %v", + remoteAddr(r), r.Method, r.URL, r.Proto, t, format) + log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) + + util.RespondWithJSON(w, http.StatusInternalServerError, + pi.ErrorReply{ + ErrorCode: t, + }) +} + // userErrorStatus retrieves the human readable error message for an error // status code. The status code can be from either the pi or cms api. func userErrorStatus(e www.ErrorStatusT) string { @@ -117,64 +413,95 @@ func RespondWithError(w http.ResponseWriter, r *http.Request, userHttpCode int, // if err == nil -> internal error using format + args // if err != nil -> if defined error -> return defined error + log.Errorf format+args // if err != nil -> if !defined error -> return + log.Errorf format+args + + // Check for www user error if userErr, ok := args[0].(www.UserError); ok { + // Error is a www user error. Log it and return a 400. if userHttpCode == 0 { userHttpCode = http.StatusBadRequest } if len(userErr.ErrorContext) == 0 { - log.Errorf("RespondWithError: %v %v %v", - remoteAddr(r), - int64(userErr.ErrorCode), + log.Infof("WWW user error: %v %v %v", + remoteAddr(r), int64(userErr.ErrorCode), userErrorStatus(userErr.ErrorCode)) } else { - log.Errorf("RespondWithError: %v %v %v: %v", - remoteAddr(r), - int64(userErr.ErrorCode), + log.Infof("WWW user error: %v %v %v: %v", + remoteAddr(r), int64(userErr.ErrorCode), userErrorStatus(userErr.ErrorCode), strings.Join(userErr.ErrorContext, ", ")) } util.RespondWithJSON(w, userHttpCode, - www.ErrorReply{ - ErrorCode: int64(userErr.ErrorCode), + www.UserError{ + ErrorCode: userErr.ErrorCode, ErrorContext: userErr.ErrorContext, }) return } - if pdError, ok := args[0].(www.PDError); ok { - pdErrorCode := convertErrorStatusFromPD(pdError.ErrorReply.ErrorCode) - if pdErrorCode == www.ErrorStatusInvalid { - errorCode := time.Now().Unix() - log.Errorf("%v %v %v %v Internal error %v: error "+ - "code from politeiad: %v", remoteAddr(r), - r.Method, r.URL, r.Proto, errorCode, - pdError.ErrorReply.ErrorCode) + // Check for politeiad error + if pdError, ok := args[0].(pdError); ok { + var ( + pluginID = pdError.ErrorReply.PluginID + errCode = pdError.ErrorReply.ErrorCode + errContext = pdError.ErrorReply.ErrorContext + ) + + // Check if the politeiad error corresponds to a www user error + wwwErrCode := convertWWWErrorStatus(pluginID, errCode) + if wwwErrCode == www.ErrorStatusInvalid { + // politeiad error does not correspond to a www user error. Log + // it and return a 500. + t := time.Now().Unix() + if pluginID == "" { + log.Errorf("%v %v %v %v Internal error %v: error "+ + "code from politeiad: %v", remoteAddr(r), r.Method, + r.URL, r.Proto, t, errCode) + } else { + log.Errorf("%v %v %v %v Internal error %v: error "+ + "code from politeiad plugin %v: %v", remoteAddr(r), + r.Method, r.URL, r.Proto, t, pluginID, errCode) + } + util.RespondWithJSON(w, http.StatusInternalServerError, www.ErrorReply{ - ErrorCode: errorCode, + ErrorCode: t, }) return } - util.RespondWithJSON(w, pdError.HTTPCode, - www.ErrorReply{ - ErrorCode: int64(pdErrorCode), - ErrorContext: pdError.ErrorReply.ErrorContext, + // politeiad error does correspond to a www user error. Log it + // and return a 400. + if len(errContext) == 0 { + log.Infof("WWW user error: %v %v %v", + remoteAddr(r), int64(wwwErrCode), + userErrorStatus(wwwErrCode)) + } else { + log.Infof("WWW user error: %v %v %v: %v", + remoteAddr(r), int64(wwwErrCode), + userErrorStatus(wwwErrCode), + strings.Join(errContext, ", ")) + } + + util.RespondWithJSON(w, http.StatusBadRequest, + www.UserError{ + ErrorCode: wwwErrCode, + ErrorContext: errContext, }) return } - errorCode := time.Now().Unix() + // Error is a politeiawww server error. Log it and return a 500. + t := time.Now().Unix() ec := fmt.Sprintf("%v %v %v %v Internal error %v: ", remoteAddr(r), - r.Method, r.URL, r.Proto, errorCode) + r.Method, r.URL, r.Proto, t) log.Errorf(ec+format, args...) log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) util.RespondWithJSON(w, http.StatusInternalServerError, www.ErrorReply{ - ErrorCode: errorCode, + ErrorCode: t, }) } @@ -205,60 +532,6 @@ func (p *politeiawww) addRoute(method string, routeVersion string, route string, } } -// makeRequest makes an http request to the method and route provided, -// serializing the provided object as the request body. -// -// XXX doesn't belong in this file but stuff it here for now. -func (p *politeiawww) makeRequest(method string, route string, v interface{}) ([]byte, error) { - var ( - requestBody []byte - err error - ) - if v != nil { - requestBody, err = json.Marshal(v) - if err != nil { - return nil, err - } - } - - fullRoute := p.cfg.RPCHost + route - - if p.client == nil { - p.client, err = util.NewClient(false, p.cfg.RPCCert) - if err != nil { - return nil, err - } - } - - req, err := http.NewRequest(method, fullRoute, - bytes.NewReader(requestBody)) - if err != nil { - return nil, err - } - req.SetBasicAuth(p.cfg.RPCUser, p.cfg.RPCPass) - r, err := p.client.Do(req) - if err != nil { - return nil, err - } - defer r.Body.Close() - - if r.StatusCode != http.StatusOK { - var pdErrorReply www.PDErrorReply - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&pdErrorReply); err != nil { - return nil, err - } - - return nil, www.PDError{ - HTTPCode: r.StatusCode, - ErrorReply: pdErrorReply, - } - } - - responseBody := util.ConvertBodyToByteArray(r.Body, false) - return responseBody, nil -} - func _main() error { // Load configuration and parse command line. This function also // initializes logging and configures it accordingly. From 2deedb311e143c130dbc5a9845586f9ac414ba5e Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 8 Sep 2020 09:07:58 -0500 Subject: [PATCH 029/449] proposals pi plugin cmd --- plugins/pi/pi.go | 52 +++++++++++++- politeiad/backend/tlogbe/plugins/pi.go | 99 +++++++++++++++++--------- politeiawww/piwww.go | 9 ++- 3 files changed, 121 insertions(+), 39 deletions(-) diff --git a/plugins/pi/pi.go b/plugins/pi/pi.go index bfb2793fe..43c0ba9d7 100644 --- a/plugins/pi/pi.go +++ b/plugins/pi/pi.go @@ -19,6 +19,7 @@ const ( ID = "pi" // Plugin commands + CmdProposals = "proposals" // Get plugin data of proposals // Metadata stream IDs. All metadata streams in this plugin will // use 1xx numbering. @@ -27,8 +28,9 @@ const ( // FilenameProposalMetadata is the filename of the ProposalMetadata // file that is saved to politeiad. ProposalMetadata is saved to - // politeiad as a file, not as a metadata stream, since it needs to - // be included in the merkle root that politeiad signs. + // politeiad as a file, not as a metadata stream, since it contains + // user provided metadata and needs to be included in the merkle + // root that politeiad signs. FilenameProposalMetadata = "proposalmetadata.json" // Proposal status codes @@ -135,3 +137,49 @@ func DecodeProposalGeneral(payload []byte) (*ProposalGeneral, error) { } return &pg, nil } + +// Proposals requests the pi plugin data for the provided proposals. +type Proposals struct { + Tokens []string `json:"tokens"` +} + +// EncodeProposals encodes a Proposals into a JSON byte slice. +func EncodeProposals(p Proposals) ([]byte, error) { + return json.Marshal(p) +} + +// DecodeProposals decodes a Proposals into a JSON byte slice. +func DecodeProposals(payload []byte) (*Proposals, error) { + var p Proposals + err := json.Unmarshal(payload, &p) + if err != nil { + return nil, err + } + return &p, nil +} + +// ProposalData represents the pi plugin data of a proposal. +type ProposalData struct { + LinkedFrom []string `json:"linkedfrom"` +} + +// ProposalsReply is the reply to the Proposals command. The proposals map will +// not contain an entry for tokens that do not correspond to actual proposals. +type ProposalsReply struct { + Proposals map[string]ProposalData `json:"proposals"` +} + +// EncodeProposalsReply encodes a ProposalsReply into a JSON byte slice. +func EncodeProposalsReply(pr ProposalsReply) ([]byte, error) { + return json.Marshal(pr) +} + +// DecodeProposalsReply decodes a ProposalsReply into a JSON byte slice. +func DecodeProposalsReply(payload []byte) (*ProposalsReply, error) { + var pr ProposalsReply + err := json.Unmarshal(payload, &pr) + if err != nil { + return nil, err + } + return &pr, nil +} diff --git a/politeiad/backend/tlogbe/plugins/pi.go b/politeiad/backend/tlogbe/plugins/pi.go index fa53d3c1f..fb7926d3c 100644 --- a/politeiad/backend/tlogbe/plugins/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi.go @@ -66,18 +66,18 @@ func proposalMetadataFromFiles(files []backend.File) (*pi.ProposalMetadata, erro return pm, nil } -// TODO saving the proposalLinkedFrom to the filesystem is not scalable between +// TODO saving the linkedFrom to the filesystem is not scalable between // multiple politeiad instances. The plugin needs to have a tree that can be // used to share state between the different politeiad instances. -// proposalLinkedFrom is the the structure that is updated and cached for -// proposal A when proposal B links to proposal A. The list contains all -// proposals that have linked to proposal A. The linked from list will only -// contain public proposals. +// linkedFrom is the the structure that is updated and cached for proposal A +// when proposal B links to proposal A. The list contains all proposals that +// have linked to proposal A. The linked from list will only contain public +// proposals. // // Example: an RFP proposal's linked from list will contain all public RFP // submissions since they have all linked to the RFP proposal. -type proposalLinkedFrom struct { +type linkedFrom struct { Tokens map[string]struct{} `json:"tokens"` } @@ -87,7 +87,7 @@ func (p *piPlugin) cachedLinkedFromPath(token string) string { } // This function must be called WITH the lock held. -func (p *piPlugin) cachedLinkedFromLocked(token string) (*proposalLinkedFrom, error) { +func (p *piPlugin) cachedLinkedFromLocked(token string) (*linkedFrom, error) { fp := p.cachedLinkedFromPath(token) b, err := ioutil.ReadFile(fp) if err != nil { @@ -98,16 +98,16 @@ func (p *piPlugin) cachedLinkedFromLocked(token string) (*proposalLinkedFrom, er } } - var plf proposalLinkedFrom - err = json.Unmarshal(b, &plf) + var lf linkedFrom + err = json.Unmarshal(b, &lf) if err != nil { return nil, err } - return &plf, nil + return &lf, nil } -func (p *piPlugin) cachedLinkedFrom(token string) (*proposalLinkedFrom, error) { +func (p *piPlugin) cachedLinkedFrom(token string) (*linkedFrom, error) { p.Lock() defer p.Unlock() @@ -119,10 +119,10 @@ func (p *piPlugin) cachedLinkedFromAdd(parentToken, childToken string) error { defer p.Unlock() // Get existing linked from list - plf, err := p.cachedLinkedFromLocked(parentToken) + lf, err := p.cachedLinkedFromLocked(parentToken) if err == errRecordNotFound { // List doesn't exist. Create a new one. - plf = &proposalLinkedFrom{ + lf = &linkedFrom{ Tokens: make(map[string]struct{}, 0), } } else if err != nil { @@ -130,10 +130,10 @@ func (p *piPlugin) cachedLinkedFromAdd(parentToken, childToken string) error { } // Update list - plf.Tokens[childToken] = struct{}{} + lf.Tokens[childToken] = struct{}{} // Save list - b, err := json.Marshal(plf) + b, err := json.Marshal(lf) if err != nil { return err } @@ -151,16 +151,16 @@ func (p *piPlugin) cachedLinkedFromDel(parentToken, childToken string) error { defer p.Unlock() // Get existing linked from list - plf, err := p.cachedLinkedFromLocked(parentToken) + lf, err := p.cachedLinkedFromLocked(parentToken) if err != nil { return fmt.Errorf("cachedLinkedFromLocked %v: %v", parentToken, err) } // Update list - delete(plf.Tokens, childToken) + delete(lf.Tokens, childToken) // Save list - b, err := json.Marshal(plf) + b, err := json.Marshal(lf) if err != nil { return err } @@ -173,20 +173,6 @@ func (p *piPlugin) cachedLinkedFromDel(parentToken, childToken string) error { return nil } -func (p *piPlugin) Setup() error { - log.Tracef("pi Setup") - - // Verify vote plugin dependency - - return nil -} - -func (p *piPlugin) Cmd(cmd, payload string) (string, error) { - log.Tracef("pi Cmd: %v %v", cmd, payload) - - return "", nil -} - func (p *piPlugin) hookNewRecordPre(payload string) error { nr, err := tlogbe.DecodeNewRecord([]byte(payload)) if err != nil { @@ -390,7 +376,7 @@ func (p *piPlugin) hookSetRecordStatusPost(payload string) error { return err } - // If the LinkTo field has been set then the proposalLinkedFrom + // If the LinkTo field has been set then the linkedFrom // list might need to be updated for the proposal that is being // linked to, depending on the status change that is being made. pm, err := proposalMetadataFromFiles(srs.Record.Files) @@ -425,6 +411,51 @@ func (p *piPlugin) hookSetRecordStatusPost(payload string) error { return nil } +func (p *piPlugin) Setup() error { + log.Tracef("pi Setup") + + // Verify vote plugin dependency + + return nil +} + +func (p *piPlugin) cmdProposals(payload string) (string, error) { + ps, err := pi.DecodeProposals([]byte(payload)) + if err != nil { + return "", err + } + _ = ps + + /* + // TODO just because a cached linked from doesn't exist doesn't + // mean the token isn't valid. We need to check if the token + // corresponds to a real proposal. + proposals := make(map[string]pi.ProposalData, len(ps.Tokens)) + for _, v := range ps.Tokens { + lf, err := p.cachedLinkedFrom(v) + if err != nil { + if err == errRecordNotFound { + continue + } + return "", fmt.Errorf("cachedLinkedFrom %v: %v", v, err) + } + } + */ + + return "", nil +} + +func (p *piPlugin) Cmd(cmd, payload string) (string, error) { + log.Tracef("pi Cmd: %v %v", cmd, payload) + + switch cmd { + case pi.CmdProposals: + return p.cmdProposals(payload) + } + + return "", nil +} + func (p *piPlugin) Hook(h tlogbe.HookT, payload string) error { log.Tracef("pi Hook: %v", tlogbe.Hooks[h]) @@ -443,7 +474,7 @@ func (p *piPlugin) Hook(h tlogbe.HookT, payload string) error { func (p *piPlugin) Fsck() error { log.Tracef("pi Fsck") - // proposalLinkedFrom cache + // linkedFrom cache return nil } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index fa2fbff5a..6cd631972 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -47,7 +47,7 @@ var ( validProposalName = regexp.MustCompile(createProposalNameRegex()) ) -func propStateFromStatus(s pi.PropStatusT) propStateT { +func convertPropStateFromStatus(s pi.PropStatusT) propStateT { switch s { case pi.PropStatusUnvetted, pi.PropStatusCensored: return propStateUnvetted @@ -578,7 +578,11 @@ func (p *politeiawww) processProposalNew(pn pi.ProposalNew, usr user.User) (*pi. } // TODO implement proposalRecord +// proposalRecord returns the proposal record for the provided token. func (p *politeiawww) proposalRecord(token string) (*pi.ProposalRecord, error) { + // Get politeiad record + // Get proposal plugin data + // Get user data return nil, nil } @@ -672,7 +676,7 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p // Send politeiad request var route string - switch propStateFromStatus(curr.Status) { + switch convertPropStateFromStatus(curr.Status) { case propStateUnvetted: route = pd.UpdateUnvettedRoute case propStateVetted: @@ -693,7 +697,6 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p if err != nil { // TODO verify that this will throw an error if no proposal files // were changed. - // TODO plugin error pass through return nil, err } var urr pd.UpdateRecordReply From 6af8cfa1fb36d589da107d8ae300a51248a62c32 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 8 Sep 2020 12:11:34 -0500 Subject: [PATCH 030/449] proposalRecord and proposalRecords --- plugins/pi/pi.go | 15 +++-- politeiawww/api/pi/v1/v1.go | 1 + politeiawww/piwww.go | 109 +++++++++++++++++++++++++++++------- 3 files changed, 100 insertions(+), 25 deletions(-) diff --git a/plugins/pi/pi.go b/plugins/pi/pi.go index 43c0ba9d7..6b4f10c7d 100644 --- a/plugins/pi/pi.go +++ b/plugins/pi/pi.go @@ -19,7 +19,7 @@ const ( ID = "pi" // Plugin commands - CmdProposals = "proposals" // Get plugin data of proposals + CmdProposals = "proposals" // Get proposals plugin data // Metadata stream IDs. All metadata streams in this plugin will // use 1xx numbering. @@ -138,7 +138,9 @@ func DecodeProposalGeneral(payload []byte) (*ProposalGeneral, error) { return &pg, nil } -// Proposals requests the pi plugin data for the provided proposals. +// Proposals requests the plugin data for the provided proposals. This includes +// pi plugin data as well as other plugin data such as comments plugin data. +// This command aggregates all proposal plugin data into a single call. type Proposals struct { Tokens []string `json:"tokens"` } @@ -158,15 +160,16 @@ func DecodeProposals(payload []byte) (*Proposals, error) { return &p, nil } -// ProposalData represents the pi plugin data of a proposal. -type ProposalData struct { - LinkedFrom []string `json:"linkedfrom"` +// ProposalPluginData represents the plugin data of a proposal. +type ProposalPluginData struct { + Comments uint64 `json:"comments"` // Number of comments + LinkedFrom []string `json:"linkedfrom"` // Linked from list } // ProposalsReply is the reply to the Proposals command. The proposals map will // not contain an entry for tokens that do not correspond to actual proposals. type ProposalsReply struct { - Proposals map[string]ProposalData `json:"proposals"` + Proposals map[string]ProposalPluginData `json:"proposals"` } // EncodeProposalsReply encodes a ProposalsReply into a JSON byte slice. diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 7155bd26c..7cdd926fc 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -190,6 +190,7 @@ type ProposalRecord struct { Username string `json:"username"` // Author username PublicKey string `json:"publickey"` // Key used in signature Signature string `json:"signature"` // Signature of merkle root + Comments uint64 `json:"comments"` // Number of comments Statuses []StatusChange `json:"statuses"` // Status change history Files []File `json:"files"` // Proposal files Metadata []Metadata `json:"metadata"` // User defined metadata diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 6cd631972..4a655da3c 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -36,8 +36,7 @@ const ( mimeTypeTextUTF8 = "text/plain; charset=utf-8" mimeTypePNG = "image/png" - // Proposal states. Proposal states correspond that politeiad - // routes. + // Proposal states. These correspond to the politeiad routes. propStateInvalid propStateT = 0 propStateUnvetted propStateT = 1 propStateVetted propStateT = 2 @@ -113,6 +112,19 @@ func convertCensorshipRecordFromPD(cr pd.CensorshipRecord) pi.CensorshipRecord { } } +// tokenIsValid returns whether the provided string is a valid politeiad +// censorship record token. +func tokenIsValid(token string) bool { + b, err := hex.DecodeString(token) + if err != nil { + return false + } + if len(b) != pd.TokenSize { + return false + } + return true +} + // proposalNameIsValid returns whether the provided string is a valid proposal // name. func proposalNameIsValid(str string) bool { @@ -169,17 +181,85 @@ func (p *politeiawww) linkByPeriodMax() int64 { return 7776000 // 3 months in seconds } -// tokenIsValid returns whether the provided string is a valid politeiad -// censorship record token. -func tokenIsValid(token string) bool { - b, err := hex.DecodeString(token) +// proposalsPluginData fetches the plugin data for the provided proposals using +// the pi proposals plugin command. +func (p *politeiawww) proposalsPluginData(tokens []string) (map[string]piplugin.ProposalPluginData, error) { + // Setup plugin command + ps := piplugin.Proposals{ + Tokens: tokens, + } + payload, err := piplugin.EncodeProposals(ps) if err != nil { - return false + return nil, err } - if len(b) != pd.TokenSize { - return false + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return nil, err } - return true + pc := pd.PluginCommand{ + Challenge: hex.EncodeToString(challenge), + ID: piplugin.ID, + Command: piplugin.CmdProposals, + Payload: string(payload), + } + + // Send plugin command + resBody, err := p.makeRequest(http.MethodPost, pd.PluginCommandRoute, pc) + if err != nil { + return nil, err + } + var pcr pd.PluginCommandReply + err = json.Unmarshal(resBody, &pcr) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(p.cfg.Identity, challenge, pcr.Response) + if err != nil { + return nil, err + } + pr, err := piplugin.DecodeProposalsReply([]byte(pcr.Payload)) + if err != nil { + return nil, err + } + + return pr.Proposals, nil +} + +// proposalRecord returns the proposal record for the provided token. +func (p *politeiawww) proposalRecord(token string) (*pi.ProposalRecord, error) { + // TODO Get politeiad record + var pr *pi.ProposalRecord + + // Get proposal plugin data + ppd, err := p.proposalsPluginData([]string{token}) + if err != nil { + return nil, err + } + pd, ok := ppd[token] + if !ok { + return nil, fmt.Errorf("proposal plugin data not found %v", token) + } + pr.Comments = pd.Comments + pr.LinkedFrom = pd.LinkedFrom + + // Get user data + u, err := p.db.UserGetByPubKey(pr.PublicKey) + if err != nil { + return nil, err + } + pr.UserID = u.ID.String() + pr.Username = u.Username + + return nil, nil +} + +func (p *politeiawww) proposalRecords(tokens []string) (map[string]pi.ProposalRecord, error) { + // Get politeiad records + // Get proposals plugin data + + // Get user data + + return nil, nil } func (p *politeiawww) verifyProposalMetadata(pm pi.ProposalMetadata) error { @@ -577,15 +657,6 @@ func (p *politeiawww) processProposalNew(pn pi.ProposalNew, usr user.User) (*pi. return &pi.ProposalNewReply{}, nil } -// TODO implement proposalRecord -// proposalRecord returns the proposal record for the provided token. -func (p *politeiawww) proposalRecord(token string) (*pi.ProposalRecord, error) { - // Get politeiad record - // Get proposal plugin data - // Get user data - return nil, nil -} - // filesToDel returns the names of the files that are included in current but // are not included in updated. These are the files that need to be deleted // from a proposal on update. From 77c6a8ab08147e162321f1dbc4d4c00bcba4bf8a Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 8 Sep 2020 13:29:57 -0500 Subject: [PATCH 031/449] map pi api to politeiad api --- politeiad/api/v1/v1.go | 1 + politeiawww/api/pi/v1/v1.go | 50 ++++++++++++++++++++++++++----------- politeiawww/piwww.go | 23 ++++++++--------- 3 files changed, 47 insertions(+), 27 deletions(-) diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index 19f78f36f..414f880dc 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -315,6 +315,7 @@ type UpdateRecord struct { // UpdateRecordReply returns a CensorshipRecord which may or may not have // changed. Metadata only updates do not create a new CensorshipRecord. type UpdateRecordReply struct { + // TODO add censorship record Response string `json:"response"` // Challenge response } diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 7cdd926fc..87d6869c8 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -9,6 +9,7 @@ import ( ) type ErrorStatusT int +type PropStateT int type PropStatusT int type VoteStatusT int type VoteT int @@ -20,7 +21,6 @@ const ( RouteProposalNew = "/proposal/new" RouteProposalEdit = "/proposal/edit" RouteProposalSetStatus = "/proposal/setstatus" - RouteProposalDetails = "/proposal/details" RouteProposals = "/proposals" RouteProposalInventory = "/proposals/inventory" @@ -36,19 +36,24 @@ const ( RouteVoteStart = "/vote/start" RouteVoteStartRunoff = "/vote/startrunoff" RouteVoteBallot = "/vote/ballot" - RouteVoteDetails = "/vote/details" - RouteVoteResults = "/vote/results" + RouteVotes = "/votes" + RouteVoteResults = "/votes/results" RouteVoteSummaries = "/votes/summaries" RouteVoteInventory = "/votes/inventory" - // Proposal status codes + // Proposal states + PropStateInvalid PropStateT = 0 + PropStateUnvetted PropStateT = 1 + PropStateVetted PropStateT = 2 + + // Proposal statuses PropStatusInvalid PropStatusT = 0 // Invalid status PropStatusUnvetted PropStatusT = 1 // Prop has not been vetted PropStatusPublic PropStatusT = 2 // Prop has been made public PropStatusCensored PropStatusT = 3 // Prop has been censored PropStatusAbandoned PropStatusT = 4 // Prop has been abandoned - // Vote status codes + // Vote statuses VoteStatusInvalid VoteStatusT = 0 // Invalid status VoteStatusUnauthorized VoteStatusT = 1 // Vote cannot be started VoteStatusAuthorized VoteStatusT = 2 // Vote can be started @@ -185,6 +190,7 @@ type StatusChange struct { type ProposalRecord struct { Version string `json:"version"` // Proposal version Timestamp int64 `json:"timestamp"` // Submission UNIX timestamp + State PropStateT `json:"state"` // Proposal state Status PropStatusT `json:"status"` // Proposal status UserID string `json:"userid"` // Author ID Username string `json:"username"` // Author username @@ -221,7 +227,8 @@ type ProposalNew struct { // ProposalNewReply is the reply to the ProposalNew command. type ProposalNewReply struct { - Proposal ProposalRecord `json:"proposal"` + Timestamp int64 `json:"timestamp"` + CensorshipRecord CensorshipRecord `json:"censorshiprecord"` } // ProposalEdit edits an existing proposal. @@ -232,6 +239,7 @@ type ProposalNewReply struct { // root is the ordered merkle root of all proposal Files and Metadata. type ProposalEdit struct { Token string `json:"token"` // Censorship token + State PropStateT `json:"state"` // Proposal state Files []File `json:"files"` // Proposal files Metadata []Metadata `json:"metadata"` // User defined metadata PublicKey string `json:"publickey"` // Key used for signature @@ -240,7 +248,9 @@ type ProposalEdit struct { // ProposalEditReply is the reply to the ProposalEdit command. type ProposalEditReply struct { - Proposal ProposalRecord `json:"proposal"` + Version string `json:"version"` + Timestamp int64 `json:"timestamp"` + CensorshipRecord CensorshipRecord `json:"censorshiprecord"` } // ProposalSetStatus sets the status of a proposal. Some status changes require @@ -249,6 +259,7 @@ type ProposalEditReply struct { // Signature is the client signature of the Token+Version+Status+Reason. type ProposalSetStatus struct { Token string `json:"token"` // Censorship token + State PropStateT `json:"state"` // Proposal state Version string `json:"version"` // Proposal version Status PropStatusT `json:"status"` // New status Reason string `json:"reason,omitempty"` // Reason for status change @@ -261,16 +272,25 @@ type ProposalSetStatusReply struct { Timestamp int64 `json:"timestamp"` } -// ProposalDetails retrieves a proposal record. If a version is not specified, -// the latest version will be returned. -type ProposalDetails struct { - Token string `json:"token"` - Version string `json:"version,omitempty"` +// ProposalRequest is used to request the ProposalRecord of the provided +// proposal token and version. If the version is omitted, the most recent +// version will be returned. +type ProposalRequest struct { + Token string `json:"token"` + State PropStateT `json:"state"` + Version string `json:"version,omitempty"` + IncludeFiles bool `json:"includefiles,omitempty"` +} + +// Proposals retrieves the ProposalRecord for each of the provided proposal +// requests. +type Proposals struct { + Requests []ProposalRequest `json:"requests"` } -// ProposalDetailsReply is the reply to the ProposalDetails command. -type ProposalDetailsReply struct { - Proposal ProposalRecord `json:"proposal"` +// ProposalsReply is the reply to the Proposals command. +type ProposalsReply struct { + Proposals []ProposalRecord `json:"proposals"` } // ProposalInventry retrieves the tokens of all proposals in the inventory, diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 4a655da3c..ed4ddb0fd 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -28,25 +28,18 @@ import ( // TODO use pi errors instead of www errors -type propStateT int - const ( // MIME types mimeTypeText = "text/plain" mimeTypeTextUTF8 = "text/plain; charset=utf-8" mimeTypePNG = "image/png" - - // Proposal states. These correspond to the politeiad routes. - propStateInvalid propStateT = 0 - propStateUnvetted propStateT = 1 - propStateVetted propStateT = 2 ) var ( validProposalName = regexp.MustCompile(createProposalNameRegex()) ) -func convertPropStateFromStatus(s pi.PropStatusT) propStateT { +func convertPropStateFromStatus(s pi.PropStatusT) pi.PropStateT { switch s { case pi.PropStatusUnvetted, pi.PropStatusCensored: return propStateUnvetted @@ -745,13 +738,17 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p }, } - // Send politeiad request + // Setup politeiad request var route string - switch convertPropStateFromStatus(curr.Status) { - case propStateUnvetted: + switch pe.State { + case pi.PropStateUnvetted: route = pd.UpdateUnvettedRoute - case propStateVetted: + case pi.PropStateVetted: route = pd.UpdateVettedRoute + default: + return nil, pi.UserError{ + // TODO ErrorCode: pi.ErrorStatusPropStateInvalid, + } } challenge, err := util.Random(pd.ChallengeSize) if err != nil { @@ -764,6 +761,8 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p FilesAdd: files, FilesDel: filesToDel(curr.Files, pe.Files), } + + // Send politeiad request resBody, err := p.makeRequest(http.MethodPost, route, ur) if err != nil { // TODO verify that this will throw an error if no proposal files From 49caeab8df92bbd0ab6f705a75027d68d9591874 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 8 Sep 2020 16:46:42 -0500 Subject: [PATCH 032/449] pi api error statuses --- plugins/pi/pi.go | 16 +- politeiad/backend/tlogbe/plugins/pi.go | 20 +- politeiawww/api/pi/v1/v1.go | 51 ++++- politeiawww/piwww.go | 293 +++++++++++++------------ politeiawww/www.go | 42 +++- 5 files changed, 247 insertions(+), 175 deletions(-) diff --git a/plugins/pi/pi.go b/plugins/pi/pi.go index 6b4f10c7d..2562d257a 100644 --- a/plugins/pi/pi.go +++ b/plugins/pi/pi.go @@ -41,10 +41,10 @@ const ( PropStatusAbandoned PropStatusT = 4 // Prop has been abandoned // User error status codes - ErrorStatusInvalid ErrorStatusT = 0 - ErrorStatusLinkToInvalid ErrorStatusT = 1 - ErrorStatusWrongPropStatus ErrorStatusT = 2 - ErrorStatusWrongVoteStatus ErrorStatusT = 3 + ErrorStatusInvalid ErrorStatusT = 0 + ErrorStatusPropLinkToInvalid ErrorStatusT = 1 + ErrorStatusPropStatusInvalid ErrorStatusT = 2 + ErrorStatusVoteStatusInvalid ErrorStatusT = 3 ) var ( @@ -60,10 +60,10 @@ var ( // ErrorStatus contains human readable user error statuses. ErrorStatus = map[ErrorStatusT]string{ - ErrorStatusInvalid: "error status invalid", - ErrorStatusLinkToInvalid: "linkto invalid", - ErrorStatusWrongPropStatus: "wrong proposal status", - ErrorStatusWrongVoteStatus: "wrong vote status", + ErrorStatusInvalid: "error status invalid", + ErrorStatusPropLinkToInvalid: "proposal link to invalid", + ErrorStatusPropStatusInvalid: "proposal status invalid", + ErrorStatusVoteStatusInvalid: "vote status invalid", } ) diff --git a/politeiad/backend/tlogbe/plugins/pi.go b/politeiad/backend/tlogbe/plugins/pi.go index fb7926d3c..ebe335432 100644 --- a/politeiad/backend/tlogbe/plugins/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi.go @@ -207,14 +207,14 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { if pm.LinkTo != "" { if isRFP(*pm) { return pi.UserError{ - ErrorCode: pi.ErrorStatusLinkToInvalid, + ErrorCode: pi.ErrorStatusPropLinkToInvalid, ErrorContext: []string{"an rfp cannot have linkto set"}, } } tokenb, err := hex.DecodeString(pm.LinkTo) if err != nil { return pi.UserError{ - ErrorCode: pi.ErrorStatusLinkToInvalid, + ErrorCode: pi.ErrorStatusPropLinkToInvalid, ErrorContext: []string{"invalid hex"}, } } @@ -222,7 +222,7 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { if err != nil { if err == backend.ErrRecordNotFound { return pi.UserError{ - ErrorCode: pi.ErrorStatusLinkToInvalid, + ErrorCode: pi.ErrorStatusPropLinkToInvalid, ErrorContext: []string{"proposal not found"}, } } @@ -234,20 +234,20 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { } if linkToPM == nil { return pi.UserError{ - ErrorCode: pi.ErrorStatusLinkToInvalid, + ErrorCode: pi.ErrorStatusPropLinkToInvalid, ErrorContext: []string{"proposal not an rfp"}, } } if !isRFP(*linkToPM) { return pi.UserError{ - ErrorCode: pi.ErrorStatusLinkToInvalid, + ErrorCode: pi.ErrorStatusPropLinkToInvalid, ErrorContext: []string{"proposal not an rfp"}, } } if time.Now().Unix() > linkToPM.LinkBy { // Link by deadline has expired. New links are not allowed. return pi.UserError{ - ErrorCode: pi.ErrorStatusLinkToInvalid, + ErrorCode: pi.ErrorStatusPropLinkToInvalid, ErrorContext: []string{"rfp link by deadline expired"}, } } @@ -274,7 +274,7 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { } if !summary.Approved { return pi.UserError{ - ErrorCode: pi.ErrorStatusLinkToInvalid, + ErrorCode: pi.ErrorStatusPropLinkToInvalid, ErrorContext: []string{"rfp vote not approved"}, } } @@ -311,7 +311,7 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { status := convertPropStatusFromMDStatus(er.Record.RecordMetadata.Status) if status != pi.PropStatusUnvetted && status != pi.PropStatusPublic { return pi.UserError{ - ErrorCode: pi.ErrorStatusWrongPropStatus, + ErrorCode: pi.ErrorStatusPropStatusInvalid, } } @@ -342,7 +342,7 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { ticketvote.VoteStatus[summary.Status], ticketvote.VoteStatus[ticketvote.VoteStatusUnauthorized]) return pi.UserError{ - ErrorCode: pi.ErrorStatusWrongVoteStatus, + ErrorCode: pi.ErrorStatusVoteStatusInvalid, ErrorContext: []string{e}, } } @@ -361,7 +361,7 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { } if pmCurr.LinkTo != pmNew.LinkTo { return pi.UserError{ - ErrorCode: pi.ErrorStatusLinkToInvalid, + ErrorCode: pi.ErrorStatusPropLinkToInvalid, ErrorContext: []string{"linkto cannot change on public proposal"}, } } diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 87d6869c8..0a4613957 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -83,13 +83,62 @@ const ( // Error status codes ErrorStatusInvalid ErrorStatusT = 0 + + // User errors + ErrorStatusUserRegistrationNotPaid ErrorStatusT = 1 + ErrorStatusUserBalanceInsufficient ErrorStatusT = 2 + ErrorStatusUserIsNotAuthor ErrorStatusT = 3 + + // Signature errors + ErrorStatusPublicKeyInvalid ErrorStatusT = 100 + ErrorStatusSignatureInvalid ErrorStatusT = 101 + + // Proposal errors + ErrorStatusFileCountInvalid ErrorStatusT = 202 + ErrorStatusFileNameInvalid ErrorStatusT = 203 + ErrorStatusFileMIMEInvalid ErrorStatusT = 204 + ErrorStatusFileDigestInvalid ErrorStatusT = 205 + ErrorStatusFilePayloadInvalid ErrorStatusT = 206 + ErrorStatusIndexFileNameInvalid ErrorStatusT = 207 + ErrorStatusIndexFileCountInvalid ErrorStatusT = 207 + ErrorStatusIndexFileSizeInvalid ErrorStatusT = 208 + ErrorStatusTextFileCountInvalid ErrorStatusT = 209 + ErrorStatusImageFileCountInvalid ErrorStatusT = 210 + ErrorStatusImageFileSizeInvalid ErrorStatusT = 211 + ErrorStatusMetadataCountInvalid ErrorStatusT = 212 + ErrorStatusMetadataHintInvalid ErrorStatusT = 213 + ErrorStatusMetadataDigestInvalid ErrorStatusT = 214 + ErrorStatusMetadataPayloadInvalid ErrorStatusT = 215 + ErrorStatusPropNameInvalid ErrorStatusT = 216 + ErrorStatusPropLinkToInvalid ErrorStatusT = 217 + ErrorStatusPropLinkByInvalid ErrorStatusT = 218 + + // TODO make normal + ErrorStatusPropTokenInvalid ErrorStatusT = iota + ErrorStatusPropNotFound + ErrorStatusPropStateInvalid + ErrorStatusPropStatusInvalid + ErrorStatusPropStatusChangeInvalid + + // Comment errors + ErrorStatusCommentTextInvalid + ErrorStatusCommentParentIDInvalid + ErrorStatusCommentVoteInvalid + ErrorStatusCommentNotFound + ErrorStatusCommentMaxVoteChanges + + // Vote errors + ErrorStatusVoteStatusInvalid + ErrorStatusVoteDetailsInvalid + ErrorStatusBallotInvalid ) var ( - // APIRoute is the prefix to the v2 API routes. + // APIRoute is the prefix to all API routes. APIRoute = fmt.Sprintf("/v%v", APIVersion) // ErrorStatus contains human readable error messages. + // TODO fill in error status messages ErrorStatus = map[ErrorStatusT]string{ ErrorStatusInvalid: "error status invalid", } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index ed4ddb0fd..0019c130f 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -26,7 +26,7 @@ import ( "github.com/decred/politeia/util" ) -// TODO use pi errors instead of www errors +// TODO use pi policies const ( // MIME types @@ -39,28 +39,59 @@ var ( validProposalName = regexp.MustCompile(createProposalNameRegex()) ) -func convertPropStateFromStatus(s pi.PropStatusT) pi.PropStateT { - switch s { - case pi.PropStatusUnvetted, pi.PropStatusCensored: - return propStateUnvetted - case pi.PropStatusPublic, pi.PropStatusAbandoned: - return propStateVetted +// tokenIsValid returns whether the provided string is a valid politeiad +// censorship record token. +func tokenIsValid(token string) bool { + b, err := hex.DecodeString(token) + if err != nil { + return false + } + if len(b) != pd.TokenSize { + return false + } + return true +} + +// proposalNameIsValid returns whether the provided name is a valid proposal +// name. +func proposalNameIsValid(name string) bool { + return validProposalName.MatchString(name) +} + +// createProposalNameRegex returns a regex string for validating the proposal +// name. +func createProposalNameRegex() string { + var validProposalNameBuffer bytes.Buffer + validProposalNameBuffer.WriteString("^[") + + for _, supportedChar := range www.PolicyProposalNameSupportedChars { + if len(supportedChar) > 1 { + validProposalNameBuffer.WriteString(supportedChar) + } else { + validProposalNameBuffer.WriteString(`\` + supportedChar) + } } - return propStateInvalid + minNameLength := strconv.Itoa(www.PolicyMinProposalNameLength) + maxNameLength := strconv.Itoa(www.PolicyMaxProposalNameLength) + validProposalNameBuffer.WriteString("]{") + validProposalNameBuffer.WriteString(minNameLength + ",") + validProposalNameBuffer.WriteString(maxNameLength + "}$") + + return validProposalNameBuffer.String() } -func convertUserErrFromSignatureErr(err error) www.UserError { +func convertUserErrFromSignatureErr(err error) pi.UserError { var e util.SignatureError - var s www.ErrorStatusT + var s pi.ErrorStatusT if errors.As(err, &e) { switch e.ErrorCode { case util.ErrorStatusPublicKeyInvalid: - s = www.ErrorStatusInvalidPublicKey + s = pi.ErrorStatusPublicKeyInvalid case util.ErrorStatusSignatureInvalid: - s = www.ErrorStatusInvalidSignature + s = pi.ErrorStatusSignatureInvalid } } - return www.UserError{ + return pi.UserError{ ErrorCode: s, ErrorContext: e.ErrorContext, } @@ -105,47 +136,6 @@ func convertCensorshipRecordFromPD(cr pd.CensorshipRecord) pi.CensorshipRecord { } } -// tokenIsValid returns whether the provided string is a valid politeiad -// censorship record token. -func tokenIsValid(token string) bool { - b, err := hex.DecodeString(token) - if err != nil { - return false - } - if len(b) != pd.TokenSize { - return false - } - return true -} - -// proposalNameIsValid returns whether the provided string is a valid proposal -// name. -func proposalNameIsValid(str string) bool { - return validProposalName.MatchString(str) -} - -// createProposalNameRegex returns a regex string for validating the proposal -// name. -func createProposalNameRegex() string { - var validProposalNameBuffer bytes.Buffer - validProposalNameBuffer.WriteString("^[") - - for _, supportedChar := range www.PolicyProposalNameSupportedChars { - if len(supportedChar) > 1 { - validProposalNameBuffer.WriteString(supportedChar) - } else { - validProposalNameBuffer.WriteString(`\` + supportedChar) - } - } - minNameLength := strconv.Itoa(www.PolicyMinProposalNameLength) - maxNameLength := strconv.Itoa(www.PolicyMaxProposalNameLength) - validProposalNameBuffer.WriteString("]{") - validProposalNameBuffer.WriteString(minNameLength + ",") - validProposalNameBuffer.WriteString(maxNameLength + "}$") - - return validProposalNameBuffer.String() -} - // linkByPeriodMin returns the minimum amount of time, in seconds, that the // LinkBy period must be set to. This is determined by adding 1 week onto the // minimum voting period so that RFP proposal submissions have at least one @@ -258,8 +248,8 @@ func (p *politeiawww) proposalRecords(tokens []string) (map[string]pi.ProposalRe func (p *politeiawww) verifyProposalMetadata(pm pi.ProposalMetadata) error { // Verify name if !proposalNameIsValid(pm.Name) { - return www.UserError{ - ErrorCode: www.ErrorStatusProposalInvalidTitle, + return pi.UserError{ + ErrorCode: pi.ErrorStatusPropNameInvalid, ErrorContext: []string{createProposalNameRegex()}, } } @@ -267,8 +257,8 @@ func (p *politeiawww) verifyProposalMetadata(pm pi.ProposalMetadata) error { // Verify linkto if pm.LinkTo != "" { if !tokenIsValid(pm.LinkTo) { - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkTo, + return pi.UserError{ + ErrorCode: pi.ErrorStatusPropLinkToInvalid, ErrorContext: []string{"invalid token"}, } } @@ -282,15 +272,15 @@ func (p *politeiawww) verifyProposalMetadata(pm pi.ProposalMetadata) error { case pm.LinkBy < min: e := fmt.Sprintf("linkby %v is less than min required of %v", pm.LinkBy, min) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkBy, + return pi.UserError{ + ErrorCode: pi.ErrorStatusPropLinkByInvalid, ErrorContext: []string{e}, } case pm.LinkBy > max: e := fmt.Sprintf("linkby %v is more than max allowed of %v", pm.LinkBy, max) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkBy, + return pi.UserError{ + ErrorCode: pi.ErrorStatusPropLinkByInvalid, ErrorContext: []string{e}, } } @@ -301,8 +291,8 @@ func (p *politeiawww) verifyProposalMetadata(pm pi.ProposalMetadata) error { func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, publicKey, signature string) (*pi.ProposalMetadata, error) { if len(files) == 0 { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusProposalMissingFiles, + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusFileCountInvalid, ErrorContext: []string{"no files found"}, } } @@ -318,26 +308,28 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu // Validate file name _, ok := filenames[v.Name] if ok { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusProposalDuplicateFilenames, - ErrorContext: []string{v.Name}, + e := fmt.Sprintf("duplicate name %v", v.Name) + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusFileNameInvalid, + ErrorContext: []string{e}, } } filenames[v.Name] = struct{}{} // Validate file payload if v.Payload == "" { - e := fmt.Sprintf("empty payload for file '%v'", v.Name) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidBase64, + e := fmt.Sprintf("file %v empty payload", v.Name) + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusFilePayloadInvalid, ErrorContext: []string{e}, } } payloadb, err := base64.StdEncoding.DecodeString(v.Payload) if err != nil { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidBase64, - ErrorContext: []string{v.Name}, + e := fmt.Sprintf("file %v invalid base64", v.Name) + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusFilePayloadInvalid, + ErrorContext: []string{e}, } } @@ -345,16 +337,16 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu digest := util.Digest(payloadb) d, ok := util.ConvertDigest(v.Digest) if !ok { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidFileDigest, + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusFileDigestInvalid, ErrorContext: []string{v.Name}, } } if !bytes.Equal(digest, d[:]) { - e := fmt.Sprintf("file '%v' digest got %v, want %x", + e := fmt.Sprintf("file %v digest got %v, want %x", v.Name, v.Digest, digest) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidFileDigest, + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusFileDigestInvalid, ErrorContext: []string{e}, } } @@ -367,17 +359,17 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu } mimeFile, _, err := mime.ParseMediaType(v.MIME) if err != nil { - log.Debugf("ParseMediaType(%v): %v", v.MIME, err) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidMIMEType, - ErrorContext: []string{v.Name}, + e := fmt.Sprintf("file %v mime '%v' not parsable", v.Name, v.MIME) + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusFileMIMEInvalid, + ErrorContext: []string{e}, } } if mimeFile != mimePayload { - e := fmt.Sprintf("file '%v' mime type got %v, want %v", + e := fmt.Sprintf("file %v mime got %v, want %v", v.Name, mimeFile, mimePayload) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidMIMEType, + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusFileMIMEInvalid, ErrorContext: []string{e}, } } @@ -389,10 +381,10 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu // Verify text file size if len(payloadb) > www.PolicyMaxMDSize { - e := fmt.Sprintf("file '%v' size %v exceeds max size %v", + e := fmt.Sprintf("file %v size %v exceeds max size %v", v.Name, len(payloadb), www.PolicyMaxMDSize) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusMaxMDSizeExceededPolicy, + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusIndexFileSizeInvalid, ErrorContext: []string{e}, } } @@ -400,16 +392,17 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu // The only text file that is allowed is the index markdown // file. if v.Name != www.PolicyIndexFilename { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusMaxMDsExceededPolicy, - ErrorContext: []string{v.Name}, + e := fmt.Sprint("want %v, got %v", www.PolicyIndexFilename, v.Name) + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusIndexFileNameInvalid, + ErrorContext: []string{e}, } } if foundIndexFile { e := fmt.Sprintf("more than one %v file found", www.PolicyIndexFilename) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusMaxMDsExceededPolicy, + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusIndexFileCountInvalid, ErrorContext: []string{e}, } } @@ -422,17 +415,17 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu // Verify image file size if len(payloadb) > www.PolicyMaxImageSize { - e := fmt.Sprintf("file '%v' size %v exceeds max size %v", + e := fmt.Sprintf("image %v size %v exceeds max size %v", v.Name, len(payloadb), www.PolicyMaxImageSize) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusMaxImageSizeExceededPolicy, + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusImageFileSizeInvalid, ErrorContext: []string{e}, } } default: - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidMIMEType, + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusFileMIMEInvalid, ErrorContext: []string{v.MIME}, } } @@ -441,8 +434,8 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu // Verify that an index file is present if !foundIndexFile { e := fmt.Sprintf("%v file not found", www.PolicyIndexFilename) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusProposalMissingFiles, + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusIndexFileCountInvalid, ErrorContext: []string{e}, } } @@ -451,16 +444,16 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu if countTextFiles > www.PolicyMaxMDs { e := fmt.Sprintf("got %v text files, max is %v", countTextFiles, www.PolicyMaxMDs) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusMaxMDsExceededPolicy, + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusTextFileCountInvalid, ErrorContext: []string{e}, } } if countImageFiles > www.PolicyMaxImages { e := fmt.Sprintf("got %v image files, max is %v", countImageFiles, www.PolicyMaxImages) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusMaxImagesExceededPolicy, + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusImageFileCountInvalid, ErrorContext: []string{e}, } } @@ -469,41 +462,44 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu // a ProposalMetadata. switch { case len(metadata) == 0: - return nil, www.UserError{ - ErrorCode: www.ErrorStatusMetadataMissing, - ErrorContext: []string{www.HintProposalMetadata}, + e := fmt.Sprintf("metadata with hint %v not found", + www.HintProposalMetadata) + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusMetadataCountInvalid, + ErrorContext: []string{e}, } case len(metadata) > 1: e := fmt.Sprintf("metadata should only contain %v", www.HintProposalMetadata) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusMetadataInvalid, + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusMetadataCountInvalid, ErrorContext: []string{e}, } } md := metadata[0] if md.Hint != www.HintProposalMetadata { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusMetadataInvalid, - ErrorContext: []string{md.Hint}, + e := fmt.Sprintf("unknown metadata hint %v", md.Hint) + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusMetadataHintInvalid, + ErrorContext: []string{e}, } } // Verify metadata fields b, err := base64.StdEncoding.DecodeString(md.Payload) if err != nil { - e := fmt.Sprintf("metadata '%v' invalid base64 payload", md.Hint) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusMetadataInvalid, + e := fmt.Sprintf("metadata with hint %v invalid base64 payload", md.Hint) + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusMetadataPayloadInvalid, ErrorContext: []string{e}, } } digest := util.Digest(b) if md.Digest != hex.EncodeToString(digest) { - e := fmt.Sprintf("metadata '%v' got digest %v, want %v", + e := fmt.Sprintf("metadata with hint %v got digest %v, want %v", md.Hint, md.Digest, hex.EncodeToString(digest)) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusMetadataDigestInvalid, + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusMetadataDigestInvalid, ErrorContext: []string{e}, } } @@ -514,10 +510,10 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu var pm pi.ProposalMetadata err = d.Decode(&pm) if err != nil { - log.Debugf("Decode ProposalMetadata: %v", err) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusMetadataInvalid, - ErrorContext: []string{md.Hint}, + e := fmt.Sprintf("unable to decode %v payload", md.Hint) + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusMetadataPayloadInvalid, + ErrorContext: []string{e}, } } @@ -543,24 +539,24 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu func (p *politeiawww) processProposalNew(pn pi.ProposalNew, usr user.User) (*pi.ProposalNewReply, error) { log.Tracef("processProposalNew: %v", usr.Username) - // Verify user paid registration paywall + // Verify user has paid registration paywall if !p.userHasPaid(usr) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusUserNotPaid, + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusUserRegistrationNotPaid, } } // Verify user has a proposal credit if !p.userHasProposalCredits(usr) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusNoProposalCredits, + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusUserBalanceInsufficient, } } // Verify user signed using active identity if usr.PublicKey() != pn.PublicKey { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidSigningKey, + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusPublicKeyInvalid, ErrorContext: []string{"not user's active identity"}, } } @@ -585,10 +581,11 @@ func (p *politeiawww) processProposalNew(pn pi.ProposalNew, usr user.User) (*pi. } // Setup metadata stream + timestamp := time.Now().Unix() pg := piplugin.ProposalGeneral{ PublicKey: pn.PublicKey, Signature: pn.Signature, - Timestamp: time.Now().Unix(), + Timestamp: timestamp, } b, err := piplugin.EncodeProposalGeneral(pg) if err != nil { @@ -645,9 +642,10 @@ func (p *politeiawww) processProposalNew(pn pi.ProposalNew, usr user.User) (*pi. log.Infof("%02v: %v %v", k, f.Name, f.Digest) } - // TODO return full proposal - _ = cr - return &pi.ProposalNewReply{}, nil + return &pi.ProposalNewReply{ + Timestamp: timestamp, + CensorshipRecord: cr, + }, nil } // filesToDel returns the names of the files that are included in current but @@ -675,8 +673,8 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p // Verify user signed using active identity if usr.PublicKey() != pe.PublicKey { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidSigningKey, + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusPublicKeyInvalid, ErrorContext: []string{"not user's active identity"}, } } @@ -687,25 +685,25 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p if err != nil { return nil, err } + + // Get the current proposal if !tokenIsValid(pe.Token) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusPropTokenInvalid, ErrorContext: []string{pe.Token}, } } - - // Get the current proposal curr, err := p.proposalRecord(pe.Token) if err != nil { - // TODO www.ErrorStatusProposalNotFound + // TODO pi.ErrorStatusProposalNotFound return nil, err } // Verify the user is the author. The public keys are not static // values so the user IDs must be compared directly. if curr.UserID != usr.ID.String() { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusUserNotAuthor, + return nil, pi.UserError{ + ErrorCode: pi.ErrorStatusUserIsNotAuthor, } } @@ -722,10 +720,11 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p } // Setup metadata stream + timestamp := time.Now().Unix() pg := piplugin.ProposalGeneral{ PublicKey: pe.PublicKey, Signature: pe.Signature, - Timestamp: time.Now().Unix(), + Timestamp: timestamp, } b, err := piplugin.EncodeProposalGeneral(pg) if err != nil { @@ -747,7 +746,7 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p route = pd.UpdateVettedRoute default: return nil, pi.UserError{ - // TODO ErrorCode: pi.ErrorStatusPropStateInvalid, + ErrorCode: pi.ErrorStatusPropStateInvalid, } } challenge, err := util.Random(pd.ChallengeSize) @@ -786,6 +785,8 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p log.Infof("%02v: %v %v", k, f.Name, f.Digest) } - // TODO return full proposal - return &pi.ProposalEditReply{}, nil + return &pi.ProposalEditReply{ + // TODO CensorshipRecord: cr, + Timestamp: timestamp, + }, nil } diff --git a/politeiawww/www.go b/politeiawww/www.go index d132c23b6..0908daab1 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -81,11 +81,11 @@ func convertWWWErrorStatusFromPD(e pd.ErrorStatusT) www.ErrorStatusT { func convertWWWErrorStatusFromPiPlugin(e piplugin.ErrorStatusT) www.ErrorStatusT { switch e { - case piplugin.ErrorStatusLinkToInvalid: + case piplugin.ErrorStatusPropLinkToInvalid: return www.ErrorStatusInvalidLinkTo - case piplugin.ErrorStatusWrongPropStatus: + case piplugin.ErrorStatusPropStatusInvalid: return www.ErrorStatusWrongStatus - case piplugin.ErrorStatusWrongVoteStatus: + case piplugin.ErrorStatusVoteStatusInvalid: return www.ErrorStatusWrongVoteStatus } return www.ErrorStatusInvalid @@ -166,15 +166,20 @@ func convertWWWErrorStatus(pluginID string, errCode int) www.ErrorStatusT { return www.ErrorStatusInvalid } -// TODO add pi error statuses func convertPiErrorStatusFromPD(e pd.ErrorStatusT) pi.ErrorStatusT { switch e { case pd.ErrorStatusInvalidFilename: + return pi.ErrorStatusFileNameInvalid case pd.ErrorStatusInvalidFileDigest: + return pi.ErrorStatusFileDigestInvalid case pd.ErrorStatusInvalidBase64: + return pi.ErrorStatusFilePayloadInvalid case pd.ErrorStatusInvalidMIMEType: + return pi.ErrorStatusFileMIMEInvalid case pd.ErrorStatusUnsupportedMIMEType: + return pi.ErrorStatusFileMIMEInvalid case pd.ErrorStatusInvalidRecordStatusTransition: + return pi.ErrorStatusPropStatusChangeInvalid case pd.ErrorStatusInvalidRequestPayload: // Intentionally omitted because this indicates a politeiawww // server error so a ErrorStatusInvalid should be returned. @@ -185,43 +190,60 @@ func convertPiErrorStatusFromPD(e pd.ErrorStatusT) pi.ErrorStatusT { return pi.ErrorStatusInvalid } -// TODO add pi error statuses func convertPiErrorStatusFromPiPlugin(e piplugin.ErrorStatusT) pi.ErrorStatusT { switch e { - case piplugin.ErrorStatusLinkToInvalid: - case piplugin.ErrorStatusWrongPropStatus: - case piplugin.ErrorStatusWrongVoteStatus: + case piplugin.ErrorStatusPropLinkToInvalid: + return pi.ErrorStatusPropLinkToInvalid + case piplugin.ErrorStatusPropStatusInvalid: + return pi.ErrorStatusPropStatusInvalid + case piplugin.ErrorStatusVoteStatusInvalid: + return pi.ErrorStatusVoteStatusInvalid } return pi.ErrorStatusInvalid } -// TODO add pi error statuses func convertPiErrorStatusFromComments(e comments.ErrorStatusT) pi.ErrorStatusT { switch e { case comments.ErrorStatusTokenInvalid: + return pi.ErrorStatusPropTokenInvalid case comments.ErrorStatusPublicKeyInvalid: + return pi.ErrorStatusPublicKeyInvalid case comments.ErrorStatusSignatureInvalid: + return pi.ErrorStatusSignatureInvalid case comments.ErrorStatusRecordNotFound: + return pi.ErrorStatusPropNotFound case comments.ErrorStatusCommentNotFound: + return pi.ErrorStatusCommentNotFound case comments.ErrorStatusParentIDInvalid: + return pi.ErrorStatusCommentParentIDInvalid case comments.ErrorStatusNoCommentChanges: + return pi.ErrorStatusCommentTextInvalid case comments.ErrorStatusVoteInvalid: + return pi.ErrorStatusCommentVoteInvalid case comments.ErrorStatusMaxVoteChanges: + return pi.ErrorStatusCommentMaxVoteChanges } return pi.ErrorStatusInvalid } -// TODO add pi error statuses func convertPiErrorStatusFromTicketVote(e ticketvote.ErrorStatusT) pi.ErrorStatusT { switch e { case ticketvote.ErrorStatusTokenInvalid: + return pi.ErrorStatusPropTokenInvalid case ticketvote.ErrorStatusPublicKeyInvalid: + return pi.ErrorStatusPublicKeyInvalid case ticketvote.ErrorStatusSignatureInvalid: + return pi.ErrorStatusSignatureInvalid case ticketvote.ErrorStatusRecordNotFound: + return pi.ErrorStatusPropNotFound case ticketvote.ErrorStatusRecordStatusInvalid: + return pi.ErrorStatusPropStatusInvalid case ticketvote.ErrorStatusVoteDetailsInvalid: + return pi.ErrorStatusVoteDetailsInvalid case ticketvote.ErrorStatusVoteStatusInvalid: + return pi.ErrorStatusVoteStatusInvalid case ticketvote.ErrorStatusBallotInvalid: + return pi.ErrorStatusBallotInvalid } return pi.ErrorStatusInvalid } From 859b1d2687424d8aa4cfe72890ad6e5c1d9971b6 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 9 Sep 2020 13:20:29 -0500 Subject: [PATCH 033/449] fix politeiad and politeiawww build issues --- politeiad/politeiad.go | 4 ++-- politeiawww/testing.go | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index d63cd491d..beea25bfc 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -808,7 +808,7 @@ func (p *politeia) pluginCommand(w http.ResponseWriter, r *http.Request) { return } - cid, payload, err := p.backend.Plugin("", pc.Command, pc.Payload) + payload, err := p.backend.Plugin("", pc.Command, pc.Payload) if err != nil { // Generic internal error. errorCode := time.Now().Unix() @@ -823,7 +823,7 @@ func (p *politeia) pluginCommand(w http.ResponseWriter, r *http.Request) { reply := v1.PluginCommandReply{ Response: hex.EncodeToString(response[:]), ID: pc.ID, - Command: cid, + Command: pc.CommandID, CommandID: pc.CommandID, Payload: payload, } diff --git a/politeiawww/testing.go b/politeiawww/testing.go index 9c27c9814..a95d88fb1 100644 --- a/politeiawww/testing.go +++ b/politeiawww/testing.go @@ -796,7 +796,6 @@ func newTestCMSwww(t *testing.T) (*politeiawww, func()) { p := politeiawww{ cfg: cfg, db: db, - cache: testcache.New(), params: &chaincfg.TestNet3Params, router: mux.NewRouter(), sessions: NewSessionStore(db, sessionMaxAge, cookieKey), @@ -805,7 +804,6 @@ func newTestCMSwww(t *testing.T) (*politeiawww, func()) { userEmails: make(map[string]uuid.UUID), userPaywallPool: make(map[uuid.UUID]paywallPoolMember), commentVotes: make(map[string]counters), - voteSummaries: make(map[string]www.VoteSummary), } // Setup routes From f98929542d30ecfa88bf5884c8f21bf401c9e2c7 Mon Sep 17 00:00:00 2001 From: Thiago Figueiredo Date: Tue, 8 Sep 2020 17:51:53 -0300 Subject: [PATCH 034/449] eventmanager: events refactored to new event manager design --- politeiawww/email.go | 284 ++++++++++++++++++++++++++++-- politeiawww/eventmanager.go | 342 ++++++++++++++++++++++++++++++++++-- politeiawww/templates.go | 30 ++-- 3 files changed, 620 insertions(+), 36 deletions(-) diff --git a/politeiawww/email.go b/politeiawww/email.go index 57a5f2650..77d0e9a6d 100644 --- a/politeiawww/email.go +++ b/politeiawww/email.go @@ -23,6 +23,7 @@ const ( // user to the correct GUI pages. guiRouteProposalDetails = "/proposals/{token}" guiRouteRegisterNewUser = "/register" + guiRouteDCCDetails = "/dcc/{token}" ) func createBody(tpl *template.Template, tplData interface{}) (string, error) { @@ -138,9 +139,9 @@ func (p *politeiawww) emailResetPasswordVerificationLink(email, username, token return p.sendEmailTo(subject, body, email) } -// emailAuthorForVettedProposal sends an email notification for a new proposal -// becoming vetted to the proposal's author. -func (p *politeiawww) emailAuthorForVettedProposal(proposal *www.ProposalRecord, authorUser *user.User, adminUser *user.User) error { +// emailAuthorForCensoredProposal sends an email notification for a new +// proposal becoming censored to the proposal's author. +func (p *politeiawww) emailAuthorForCensoredProposal(proposal *www.ProposalRecord, authorUser *user.User, adminUser *user.User) error { if p.smtp.disabled { return nil } @@ -162,8 +163,8 @@ func (p *politeiawww) emailAuthorForVettedProposal(proposal *www.ProposalRecord, StatusChangeReason: proposal.StatusChangeMessage, } - subject := "Your Proposal Has Been Published" - body, err := createBody(templateProposalVettedForAuthor, &tplData) + subject := "Your Proposal Has Been Censored" + body, err := createBody(templateProposalCensoredForAuthor, &tplData) if err != nil { return err } @@ -171,9 +172,77 @@ func (p *politeiawww) emailAuthorForVettedProposal(proposal *www.ProposalRecord, return p.sendEmailTo(subject, body, authorUser.Email) } -// emailAuthorForCensoredProposal sends an email notification for a new -// proposal becoming censored to the proposal's author. -func (p *politeiawww) emailAuthorForCensoredProposal(proposal *www.ProposalRecord, authorUser *user.User, adminUser *user.User) error { +// emailProposalStatusChange sends emails regarding the proposal status change +// event. Sends email for the author and the users with this notification +// bit turned on. This function replaces tree functions from the old event +// manager: emailAuthorForVettedProposal, emailUsersForVettedProposal, +// emailAuthorForCensoredProposal +func (p *politeiawww) emailProposalStatusChange(data dataProposalStatusChange, emails []string) error { + if p.smtp.disabled { + return nil + } + + route := strings.Replace(guiRouteProposalDetails, "{token}", data.token, 1) + l, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err + } + + var subject string + var template *template.Template + + // Prepare and send author's email if author notification bit + // is set + if notificationIsSet(data.emailNotifications, + www.NotificationEmailMyProposalStatusChange) { + switch data.status { + case www.PropStatusCensored: + subject = "Your Proposal Has Been Censored" + template = templateProposalCensoredForAuthor + case www.PropStatusPublic: + subject = "Your Proposal Has Been Published" + template = templateProposalVettedForAuthor + } + authorTplData := proposalStatusChangeTemplateData{ + Link: l.String(), + Name: data.name, + StatusChangeReason: data.statusChangeMessage, + } + body, err := createBody(template, &authorTplData) + if err != nil { + return err + } + err = p.sendEmailTo(subject, body, data.email) + if err != nil { + return err + } + } + + // Prepare and send user's email, if there is any + if len(emails) > 0 { + subject = "New Proposal Published" + template = templateProposalVetted + usersTplData := proposalStatusChangeTemplateData{ + Link: l.String(), + Name: data.name, + Username: data.username, + } + body, err := createBody(template, &usersTplData) + if err != nil { + return err + } + err = p.smtp.sendEmailTo(subject, body, emails) + if err != nil { + return err + } + } + + return nil +} + +// emailAuthorForVettedProposal sends an email notification for a new proposal +// becoming vetted to the proposal's author. +func (p *politeiawww) emailAuthorForVettedProposal(proposal *www.ProposalRecord, authorUser *user.User, adminUser *user.User) error { if p.smtp.disabled { return nil } @@ -195,8 +264,8 @@ func (p *politeiawww) emailAuthorForCensoredProposal(proposal *www.ProposalRecor StatusChangeReason: proposal.StatusChangeMessage, } - subject := "Your Proposal Has Been Censored" - body, err := createBody(templateProposalCensoredForAuthor, &tplData) + subject := "Your Proposal Has Been Published" + body, err := createBody(templateProposalVettedForAuthor, &tplData) if err != nil { return err } @@ -247,6 +316,35 @@ func (p *politeiawww) emailUsersForVettedProposal(proposal *www.ProposalRecord, }) } +// emailProposalEdited sends email regarding the proposal edits event. +// Sends to all users with this notification bit turned on. +func (p *politeiawww) emailProposalEdited(name, username, token, version string, emails []string) error { + if p.smtp.disabled { + return nil + } + + route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) + l, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err + } + + tplData := proposalEditedTemplateData{ + Link: l.String(), + Name: name, + Version: version, + Username: username, + } + + subject := "Proposal Edited" + body, err := createBody(templateProposalEdited, &tplData) + if err != nil { + return err + } + + return p.smtp.sendEmailTo(subject, body, emails) +} + // emailUsersForEditedProposal sends an email notification for a proposal being // edited. func (p *politeiawww) emailUsersForEditedProposal(proposal *www.ProposalRecord, authorUser *user.User) error { @@ -291,6 +389,49 @@ func (p *politeiawww) emailUsersForEditedProposal(proposal *www.ProposalRecord, }) } +// emailProposalVoteStarted sends email for the proposal vote started event. +// Sends email to author and users with this notification bit set on. +func (p *politeiawww) emailProposalVoteStarted(token, name, username, email string, emailNotifications uint64, emails []string) error { + if p.smtp.disabled { + return nil + } + + route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) + l, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err + } + + tplData := proposalVoteStartedTemplateData{ + Link: l.String(), + Name: name, + Username: username, + } + + if emailNotifications& + uint64(www.NotificationEmailMyProposalVoteStarted) != 0 { + + subject := "Your Proposal Has Started Voting" + body, err := createBody(templateProposalVoteStartedForAuthor, &tplData) + if err != nil { + return err + } + + err = p.sendEmailTo(subject, body, email) + if err != nil { + return err + } + } + + subject := "Voting Started for Proposal" + body, err := createBody(templateProposalVoteStarted, &tplData) + if err != nil { + return err + } + + return p.smtp.sendEmailTo(subject, body, emails) +} + // emailUsersForProposalVoteStarted sends an email notification for a proposal // entering the voting state. func (p *politeiawww) emailUsersForProposalVoteStarted(proposal *www.ProposalRecord, authorUser *user.User, adminUser *user.User) error { @@ -350,6 +491,8 @@ func (p *politeiawww) emailUsersForProposalVoteStarted(proposal *www.ProposalRec }) } +// emailProposalSubmitted sends email notification for a new proposal becoming +// vetted. Sends to the author and for users with this notification setting. func (p *politeiawww) emailProposalSubmitted(token, name, username string, emails []string) error { route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) l, err := url.Parse(p.cfg.WebServerAddress + route) @@ -372,6 +515,35 @@ func (p *politeiawww) emailProposalSubmitted(token, name, username string, email return p.smtp.sendEmailTo(subject, body, emails) } +// emailProposalVoteAuthorized sends email notification for the proposal vote +// authorized event. Sends to all admins with this notification bit set on. +func (p *politeiawww) emailProposalVoteAuthorized(token, name, username, email string, emails []string) error { + if p.smtp.disabled { + return nil + } + + route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) + l, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err + } + + tplData := proposalVoteAuthorizedTemplateData{ + Link: l.String(), + Name: name, + Username: username, + Email: email, + } + + subject := "Proposal Authorized To Start Voting" + body, err := createBody(templateProposalVoteAuthorized, &tplData) + if err != nil { + return err + } + + return p.smtp.sendEmailTo(subject, body, emails) +} + func (p *politeiawww) emailAdminsForProposalVoteAuthorized(token, title, authorUsername, authorEmail string) error { if p.smtp.disabled { return nil @@ -629,6 +801,24 @@ func (p *politeiawww) emailInvoiceNotifications(email, username string) error { return p.sendEmailTo(subject, body, email) } +// emailInvoiceComment sends email for the invoice comment event. Sends +// email to the user regarding that invoice. +func (p *politeiawww) emailInvoiceComment(userEmail string) error { + if p.smtp.disabled { + return nil + } + + var tplData interface{} + subject := "New Invoice Comment" + + body, err := createBody(templateNewInvoiceComment, tplData) + if err != nil { + return err + } + + return p.sendEmailTo(subject, body, userEmail) +} + func (p *politeiawww) emailUserInvoiceComment(userEmail string) error { if p.smtp.disabled { return nil @@ -645,6 +835,26 @@ func (p *politeiawww) emailUserInvoiceComment(userEmail string) error { return p.sendEmailTo(subject, body, userEmail) } +// emailInvoiceStatusUpdate sends email for the invoice status update event. +// Sends email for the user regarding that invoice. +func (p *politeiawww) emailInvoiceStatusUpdate(invoiceToken, userEmail string) error { + if p.smtp.disabled { + return nil + } + + tplData := newInvoiceStatusUpdateTemplate{ + Token: invoiceToken, + } + + subject := "Invoice status has been updated" + body, err := createBody(templateNewInvoiceStatusUpdate, &tplData) + if err != nil { + return err + } + + return p.sendEmailTo(subject, body, userEmail) +} + func (p *politeiawww) emailUserInvoiceStatusUpdate(userEmail, invoiceToken string) error { if p.smtp.disabled { return nil @@ -663,12 +873,38 @@ func (p *politeiawww) emailUserInvoiceStatusUpdate(userEmail, invoiceToken strin return p.sendEmailTo(subject, body, userEmail) } +// emailDCCNew sends email regarding the DCC New event. Sends email +// to all admins. +func (p *politeiawww) emailDCCNew(token string, emails []string) error { + if p.smtp.disabled { + return nil + } + + route := strings.Replace(guiRouteDCCDetails, "{token}", token, 1) + l, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err + } + + tplData := newDCCSubmittedTemplateData{ + Link: l.String(), + } + + subject := "New DCC Submitted" + body, err := createBody(templateNewDCCSubmitted, &tplData) + if err != nil { + return err + } + + return p.smtp.sendEmailTo(subject, body, emails) +} + func (p *politeiawww) emailAdminsForNewDCC(token string) error { if p.smtp.disabled { return nil } - l, err := url.Parse(p.cfg.WebServerAddress + "/dcc/" + token) + l, err := url.Parse(p.cfg.WebServerAddress + "{token}" + token) if err != nil { return err } @@ -694,6 +930,32 @@ func (p *politeiawww) emailAdminsForNewDCC(token string) error { }) } +// emailDCCSupportOppose sends emails regarding dcc support/oppose event. +// Sends emails to all admin users. +func (p *politeiawww) emailDCCSupportOppose(token string, emails []string) error { + if p.smtp.disabled { + return nil + } + + route := strings.Replace(guiRouteDCCDetails, "{token}", token, 1) + l, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err + } + + tplData := newDCCSupportOpposeTemplateData{ + Link: l.String(), + } + + subject := "New DCC Support/Opposition Submitted" + body, err := createBody(templateNewDCCSupportOppose, &tplData) + if err != nil { + return err + } + + return p.smtp.sendEmailTo(subject, body, emails) +} + func (p *politeiawww) emailAdminsForNewDCCSupportOppose(token string) error { if p.smtp.disabled { return nil diff --git a/politeiawww/eventmanager.go b/politeiawww/eventmanager.go index d6277450a..8ebf1080e 100644 --- a/politeiawww/eventmanager.go +++ b/politeiawww/eventmanager.go @@ -9,6 +9,7 @@ import ( www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/user" + "github.com/google/uuid" ) type eventT int @@ -17,8 +18,18 @@ const ( // Event types eventTypeInvalid eventT = iota - // Pi events + // Politeia events eventProposalSubmitted + eventProposalStatusChange + eventProposalEdited + eventProposalVoteAuthorized + eventProposalVoteStarted + + // CMS events + eventInvoiceComment + eventInvoiceStatusUpdate + eventDCCNew + eventDCCSupportOppose ) // eventManager manages event listeners for different event types. @@ -76,17 +87,52 @@ func (p *politeiawww) setupEventListenersPi() { ch := make(chan interface{}) p.eventManager.register(eventProposalSubmitted, ch) go p.handleEventProposalSubmitted(ch) + + // Setup proposal status change event + ch = make(chan interface{}) + p.eventManager.register(eventProposalStatusChange, ch) + go p.handleEventProposalStatusChange(ch) + + // Setup proposal edit event + ch = make(chan interface{}) + p.eventManager.register(eventProposalEdited, ch) + go p.handleEventProposalEdited(ch) + + // Setup proposal vote authorized event + ch = make(chan interface{}) + p.eventManager.register(eventProposalVoteAuthorized, ch) + go p.handleEventProposalVoteAuthorized(ch) + + // Setup proposal vote started event + ch = make(chan interface{}) + p.eventManager.register(eventProposalVoteStarted, ch) + go p.handleEventProposalVoteStarted(ch) + + // Setup invoice comment event + ch = make(chan interface{}) + p.eventManager.register(eventInvoiceComment, ch) + go p.handleEventInvoiceComment(ch) + + // Setup invoice status update event + ch = make(chan interface{}) + p.eventManager.register(eventInvoiceStatusUpdate, ch) + go p.handleEventInvoiceStatusUpdate(ch) + + // Setup DCC new update event + ch = make(chan interface{}) + p.eventManager.register(eventDCCNew, ch) + go p.handleEventDCCNew(ch) + + // Setup DCC support/oppose event + ch = make(chan interface{}) + p.eventManager.register(eventDCCSupportOppose, ch) + go p.handleEventDCCSupportOppose(ch) } // notificationIsSet returns whether the provided user has the provided // notification bit set. -func notificationIsSet(u user.User, n www.EmailNotificationT) bool { - // Notifications should not be sent to deactivated users - if u.Deactivated { - return false - } - - if u.EmailNotifications&uint64(n) == 0 { +func notificationIsSet(emailNotifications uint64, n www.EmailNotificationT) bool { + if emailNotifications&uint64(n) == 0 { // Notification bit not set return false } @@ -117,8 +163,14 @@ func (p *politeiawww) handleEventProposalSubmitted(ch chan interface{}) { return } + // Notifications should not be sent to deactivated users + if u.Deactivated { + return + } + // Check if notification bit is set - if notificationIsSet(*u, www.NotificationEmailAdminProposalNew) { + if notificationIsSet(u.EmailNotifications, + www.NotificationEmailAdminProposalNew) { emails = append(emails, u.Email) } }) @@ -130,9 +182,279 @@ func (p *politeiawww) handleEventProposalSubmitted(ch chan interface{}) { // Send email notification err = p.emailProposalSubmitted(d.token, d.name, d.username, emails) if err != nil { - log.Errorf("emailProposalSubmitted %v: %v", err) + log.Errorf("emailProposalSubmitted: %v", err) } log.Debugf("Sent proposal submitted notification %v", d.token) } } + +type dataProposalStatusChange struct { + name string // Proposal name + token string // Proposal censorship token + adminID uuid.UUID // Admin uuid + id uuid.UUID // Author uuid + email string // Author user email + emailNotifications uint64 // Author notification settings + username string // Author username + status www.PropStatusT // Proposal status + statusChangeMessage string // Status change message +} + +func (p *politeiawww) handleEventProposalStatusChange(ch chan interface{}) { + for msg := range ch { + d, ok := msg.(dataProposalStatusChange) + if !ok { + log.Errorf("handleProposalStatusChange invalid msg: %v", msg) + continue + } + + if d.status != www.PropStatusPublic && + d.status != www.PropStatusCensored { + continue + } + + emails := make([]string, 0, 256) + err := p.db.AllUsers(func(u *user.User) { + // Check circunstances where we don't notify + if u.Deactivated || u.ID == d.adminID || u.ID == d.id || + !notificationIsSet(u.EmailNotifications, + www.NotificationEmailRegularProposalVetted) { + return + } + + emails = append(emails, u.Email) + }) + + err = p.emailProposalStatusChange(d, emails) + if err != nil { + log.Errorf("emailProposalStatusChange: %v", err) + } + + log.Debugf("Sent proposal status change notifications %v", d.token) + } +} + +type dataProposalEdited struct { + id uuid.UUID // Author id + username string // Author username + token string // Proposal censorship token + name string // Proposal name + version string // Proposal version +} + +func (p *politeiawww) handleEventProposalEdited(ch chan interface{}) { + for msg := range ch { + d, ok := msg.(dataProposalEdited) + if !ok { + log.Errorf("handleEventProposalEdited invalid msg: %v", msg) + continue + } + + emails := make([]string, 0, 256) + err := p.db.AllUsers(func(u *user.User) { + // Check circunstances where we don't notify + if u.NewUserPaywallTx == "" || u.Deactivated || u.ID == d.id || + !notificationIsSet(u.EmailNotifications, + www.NotificationEmailRegularProposalEdited) { + return + } + + emails = append(emails, u.Email) + }) + + err = p.emailProposalEdited(d.name, d.username, d.token, d.version, + emails) + if err != nil { + log.Errorf("emailProposalEdited: %v", err) + } + + log.Debugf("Sent proposal edited notifications %v", d.token) + } +} + +type dataProposalVoteAuthorized struct { + token string // Proposal censhorship token + name string // Proposal name + username string // Author username + email string // Author email +} + +func (p *politeiawww) handleEventProposalVoteAuthorized(ch chan interface{}) { + for msg := range ch { + d, ok := msg.(dataProposalVoteAuthorized) + if !ok { + log.Errorf("handleEventProposalVoteAuthorized invalid msg: %v", msg) + continue + } + + emails := make([]string, 0, 256) + err := p.db.AllUsers(func(u *user.User) { + // Check circunstances where we don't notify + if !u.Admin || u.Deactivated || + !notificationIsSet(u.EmailNotifications, + www.NotificationEmailAdminProposalVoteAuthorized) { + return + } + + emails = append(emails, u.Email) + }) + + err = p.emailProposalVoteAuthorized(d.token, d.name, d.username, + d.email, emails) + if err != nil { + log.Errorf("emailProposalVoteAuthorized: %v", err) + } + + log.Debugf("Sent proposal vote authorized notifications %v", d.token) + } +} + +type dataProposalVoteStarted struct { + token string // Proposal censhorship token + name string // Proposal name + adminID uuid.UUID // Admin uuid + id uuid.UUID // Author uuid + username string // Author username + email string // Author email + emailNotifications uint64 // Author notifications bits +} + +func (p *politeiawww) handleEventProposalVoteStarted(ch chan interface{}) { + for msg := range ch { + d, ok := msg.(dataProposalVoteStarted) + if !ok { + log.Errorf("handleEventProposalVoteStarted invalid msg: %v", msg) + continue + } + + emails := make([]string, 0, 256) + err := p.db.AllUsers(func(u *user.User) { + // Check circunstances where we don't notify + if u.NewUserPaywallTx == "" || u.Deactivated || + u.ID == d.adminID || u.ID == d.id || + !notificationIsSet(u.EmailNotifications, + www.NotificationEmailRegularProposalVoteStarted) { + return + } + + emails = append(emails, u.Email) + }) + + err = p.emailProposalVoteStarted(d.token, d.name, d.username, + d.email, d.emailNotifications, emails) + if err != nil { + log.Errorf("emailProposalVoteAuthorized: %v", err) + } + + log.Debugf("Sent proposal authorized vote notifications %v", d.token) + } +} + +type dataInvoiceComment struct { + token string // Comment token + email string // User email +} + +func (p *politeiawww) handleEventInvoiceComment(ch chan interface{}) { + for msg := range ch { + d, ok := msg.(dataInvoiceComment) + if !ok { + log.Errorf("handleEventInvoiceComment invalid msg: %v", msg) + continue + } + + err := p.emailInvoiceComment(d.email) + if err != nil { + log.Errorf("emailInvoiceComment %v: %v", err) + } + + log.Debugf("Sent invoice comment notification %v", d.token) + } +} + +type dataInvoiceStatusUpdate struct { + token string // Comment token + email string // User email +} + +func (p *politeiawww) handleEventInvoiceStatusUpdate(ch chan interface{}) { + for msg := range ch { + d, ok := msg.(dataInvoiceComment) + if !ok { + log.Errorf("handleEventInvoiceStatusUpdate invalid msg: %v", msg) + continue + } + + err := p.emailInvoiceStatusUpdate(d.token, d.email) + if err != nil { + log.Errorf("emailInvoiceComment %v: %v", err) + } + + log.Debugf("Sent invoice comment notification %v", d.token) + } +} + +type dataDCCNew struct { + token string // DCC token + email string // User email +} + +func (p *politeiawww) handleEventDCCNew(ch chan interface{}) { + for msg := range ch { + d, ok := msg.(dataDCCNew) + if !ok { + log.Errorf("handleEventDCCNew invalid msg: %v", msg) + continue + } + + emails := make([]string, 0, 256) + err := p.db.AllUsers(func(u *user.User) { + // Check circunstances where we don't notify + if !u.Admin || u.Deactivated { + return + } + + emails = append(emails, u.Email) + }) + + err = p.emailDCCNew(d.token, emails) + if err != nil { + log.Errorf("emailDCCNew %v: %v", err) + } + + log.Debugf("Sent DCC new notification %v", d.token) + } +} + +type dataDCCSupportOppose struct { + token string // DCC token + email string // User email +} + +func (p *politeiawww) handleEventDCCSupportOppose(ch chan interface{}) { + for msg := range ch { + d, ok := msg.(dataDCCSupportOppose) + if !ok { + log.Errorf("handleEventDCCSupportOppose invalid msg: %v", msg) + continue + } + + emails := make([]string, 0, 256) + err := p.db.AllUsers(func(u *user.User) { + // Check circunstances where we don't notify + if !u.Admin || u.Deactivated { + return + } + + emails = append(emails, u.Email) + }) + + err = p.emailDCCSupportOppose(d.token, emails) + if err != nil { + log.Errorf("emailDCCSupportOppose %v: %v", err) + } + + log.Debugf("Sent DCC support/oppose notification %v", d.token) + } +} diff --git a/politeiawww/templates.go b/politeiawww/templates.go index 1416a0c57..43ae4fee3 100644 --- a/politeiawww/templates.go +++ b/politeiawww/templates.go @@ -53,30 +53,30 @@ type proposalSubmittedTemplateData struct { } type proposalEditedTemplateData struct { - Link string - Name string - Version string - Username string + Link string // GUI proposal details url + Name string // Proposal name + Version string // ProposalVersion + Username string // Author username } type proposalVoteStartedTemplateData struct { - Link string - Name string - Username string + Link string // GUI proposal details url + Name string // Proposal name + Username string // Author username } type proposalStatusChangeTemplateData struct { - Link string - Name string - Username string - StatusChangeReason string + Link string // GUI proposal details url + Name string // Proposal name + Username string // Author username + StatusChangeReason string // Proposal status change reason } type proposalVoteAuthorizedTemplateData struct { - Link string - Name string - Username string - Email string + Link string // GUI proposal details url + Name string // Proposal name + Username string // Author username + Email string // Author email } type commentReplyOnProposalTemplateData struct { From aebee3664f7ac2a465505a77df627cf11de4f9fd Mon Sep 17 00:00:00 2001 From: Thiago Figueiredo Date: Wed, 9 Sep 2020 12:55:31 -0300 Subject: [PATCH 035/449] review nits; create separate setup event functions cms/pi; remove redundant smtp disabled checks and user deactivated; wrap user checks in userNotificationEnabled; --- politeiawww/email.go | 287 +++++++++++++----------------------- politeiawww/eventmanager.go | 81 +++++----- politeiawww/www.go | 1 + 3 files changed, 147 insertions(+), 222 deletions(-) diff --git a/politeiawww/email.go b/politeiawww/email.go index 77d0e9a6d..8de15011e 100644 --- a/politeiawww/email.go +++ b/politeiawww/email.go @@ -14,6 +14,7 @@ import ( "github.com/dajohi/goemail" + v1 "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/user" ) @@ -57,25 +58,9 @@ func (p *politeiawww) createEmailLink(path, email, token, username string) (stri return l.String(), nil } -// sendEmailTo sends an email with the given subject and body to a single -// address. -func (p *politeiawww) sendEmailTo(subject, body, toAddress string) error { - if p.smtp.disabled { - return nil - } - return p.smtp.sendEmail(subject, body, func(msg *goemail.Message) error { - msg.AddTo(toAddress) - return nil - }) -} - // emailNewUserVerificationLink emails the link with the new user verification // token if the email server is set up. func (p *politeiawww) emailNewUserVerificationLink(email, token, username string) error { - if p.smtp.disabled { - return nil - } - link, err := p.createEmailLink(www.RouteVerifyNewUser, email, token, username) if err != nil { @@ -93,8 +78,9 @@ func (p *politeiawww) emailNewUserVerificationLink(email, token, username string if err != nil { return err } + recipients := []string{email} - return p.sendEmailTo(subject, body, email) + return p.smtp.sendEmailTo(subject, body, recipients) } func (p *politeiawww) newVerificationURL(route, token string) (*url.URL, error) { @@ -113,10 +99,6 @@ func (p *politeiawww) newVerificationURL(route, token string) (*url.URL, error) // emailResetPasswordVerificationLink emails the link with the reset password // verification token if the email server is set up. func (p *politeiawww) emailResetPasswordVerificationLink(email, username, token string) error { - if p.smtp.disabled { - return nil - } - u, err := p.newVerificationURL(www.RouteResetPassword, token) if err != nil { return err @@ -135,17 +117,14 @@ func (p *politeiawww) emailResetPasswordVerificationLink(email, username, token if err != nil { return err } + recipients := []string{email} - return p.sendEmailTo(subject, body, email) + return p.smtp.sendEmailTo(subject, body, recipients) } // emailAuthorForCensoredProposal sends an email notification for a new // proposal becoming censored to the proposal's author. func (p *politeiawww) emailAuthorForCensoredProposal(proposal *www.ProposalRecord, authorUser *user.User, adminUser *user.User) error { - if p.smtp.disabled { - return nil - } - l, err := url.Parse(p.cfg.WebServerAddress + "/proposals/" + proposal.CensorshipRecord.Token) if err != nil { @@ -168,85 +147,95 @@ func (p *politeiawww) emailAuthorForCensoredProposal(proposal *www.ProposalRecor if err != nil { return err } + recipients := []string{authorUser.Email} - return p.sendEmailTo(subject, body, authorUser.Email) + return p.smtp.sendEmailTo(subject, body, recipients) } // emailProposalStatusChange sends emails regarding the proposal status change // event. Sends email for the author and the users with this notification -// bit turned on. This function replaces tree functions from the old event -// manager: emailAuthorForVettedProposal, emailUsersForVettedProposal, -// emailAuthorForCensoredProposal +// bit set on func (p *politeiawww) emailProposalStatusChange(data dataProposalStatusChange, emails []string) error { - if p.smtp.disabled { - return nil - } - route := strings.Replace(guiRouteProposalDetails, "{token}", data.token, 1) l, err := url.Parse(p.cfg.WebServerAddress + route) if err != nil { return err } + // Prepare and send author's email + err = p.emailAuthorProposalStatusChange(data.name, data.email, l.String(), + data.statusChangeMessage, data.emailNotifications, data.status, emails) + if err != nil { + return err + } + + // Prepare and send user's email + err = p.emailUsersProposalStatusChange(data.name, data.username, l.String(), + emails) + if err != nil { + return err + } + + return nil +} + +// emailAuthorProposalStatusChange sends email for the author of the proposal +// in which the status has changed, if his notification bit is set on. +func (p *politeiawww) emailAuthorProposalStatusChange(name, email, link, statusChangeMsg string, emailNotifications uint64, status v1.PropStatusT, emails []string) error { + if !notificationIsSet(emailNotifications, + www.NotificationEmailMyProposalStatusChange) { + return nil + } + var subject string var template *template.Template - // Prepare and send author's email if author notification bit - // is set - if notificationIsSet(data.emailNotifications, - www.NotificationEmailMyProposalStatusChange) { - switch data.status { - case www.PropStatusCensored: - subject = "Your Proposal Has Been Censored" - template = templateProposalCensoredForAuthor - case www.PropStatusPublic: - subject = "Your Proposal Has Been Published" - template = templateProposalVettedForAuthor - } - authorTplData := proposalStatusChangeTemplateData{ - Link: l.String(), - Name: data.name, - StatusChangeReason: data.statusChangeMessage, - } - body, err := createBody(template, &authorTplData) - if err != nil { - return err - } - err = p.sendEmailTo(subject, body, data.email) - if err != nil { - return err - } + switch status { + case v1.PropStatusCensored: + subject = "Your Proposal Has Been Censored" + template = templateProposalCensoredForAuthor + case v1.PropStatusPublic: + subject = "Your Proposal Has Been Published" + template = templateProposalVettedForAuthor } - // Prepare and send user's email, if there is any - if len(emails) > 0 { - subject = "New Proposal Published" - template = templateProposalVetted - usersTplData := proposalStatusChangeTemplateData{ - Link: l.String(), - Name: data.name, - Username: data.username, - } - body, err := createBody(template, &usersTplData) - if err != nil { - return err - } - err = p.smtp.sendEmailTo(subject, body, emails) - if err != nil { - return err - } + authorTplData := proposalStatusChangeTemplateData{ + Link: link, + Name: name, + StatusChangeReason: statusChangeMsg, } + body, err := createBody(template, &authorTplData) + if err != nil { + return err + } + recipients := []string{email} - return nil + return p.smtp.sendEmailTo(subject, body, recipients) } -// emailAuthorForVettedProposal sends an email notification for a new proposal -// becoming vetted to the proposal's author. -func (p *politeiawww) emailAuthorForVettedProposal(proposal *www.ProposalRecord, authorUser *user.User, adminUser *user.User) error { - if p.smtp.disabled { +// emailUsersProposalStatusChange sends email for all users with this +// notification bit set on. +func (p *politeiawww) emailUsersProposalStatusChange(name, username, link string, emails []string) error { + if len(emails) > 0 { return nil } + subject := "New Proposal Published" + template := templateProposalVetted + usersTplData := proposalStatusChangeTemplateData{ + Link: link, + Name: name, + Username: username, + } + body, err := createBody(template, &usersTplData) + if err != nil { + return err + } + return p.smtp.sendEmailTo(subject, body, emails) +} +// emailAuthorForVettedProposal sends an email notification for a new proposal +// becoming vetted to the proposal's author. +func (p *politeiawww) emailAuthorForVettedProposal(proposal *www.ProposalRecord, authorUser *user.User, adminUser *user.User) error { l, err := url.Parse(p.cfg.WebServerAddress + "/proposals/" + proposal.CensorshipRecord.Token) if err != nil { @@ -269,17 +258,14 @@ func (p *politeiawww) emailAuthorForVettedProposal(proposal *www.ProposalRecord, if err != nil { return err } + recipients := []string{authorUser.Email} - return p.sendEmailTo(subject, body, authorUser.Email) + return p.smtp.sendEmailTo(subject, body, recipients) } // emailUsersForVettedProposal sends an email notification for a new proposal // becoming vetted. func (p *politeiawww) emailUsersForVettedProposal(proposal *www.ProposalRecord, authorUser *user.User, adminUser *user.User) error { - if p.smtp.disabled { - return nil - } - // Create the template data. l, err := url.Parse(p.cfg.WebServerAddress + "/proposals/" + proposal.CensorshipRecord.Token) @@ -319,10 +305,6 @@ func (p *politeiawww) emailUsersForVettedProposal(proposal *www.ProposalRecord, // emailProposalEdited sends email regarding the proposal edits event. // Sends to all users with this notification bit turned on. func (p *politeiawww) emailProposalEdited(name, username, token, version string, emails []string) error { - if p.smtp.disabled { - return nil - } - route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) l, err := url.Parse(p.cfg.WebServerAddress + route) if err != nil { @@ -348,10 +330,6 @@ func (p *politeiawww) emailProposalEdited(name, username, token, version string, // emailUsersForEditedProposal sends an email notification for a proposal being // edited. func (p *politeiawww) emailUsersForEditedProposal(proposal *www.ProposalRecord, authorUser *user.User) error { - if p.smtp.disabled { - return nil - } - // Create the template data. l, err := url.Parse(p.cfg.WebServerAddress + "/proposals/" + proposal.CensorshipRecord.Token) @@ -392,10 +370,6 @@ func (p *politeiawww) emailUsersForEditedProposal(proposal *www.ProposalRecord, // emailProposalVoteStarted sends email for the proposal vote started event. // Sends email to author and users with this notification bit set on. func (p *politeiawww) emailProposalVoteStarted(token, name, username, email string, emailNotifications uint64, emails []string) error { - if p.smtp.disabled { - return nil - } - route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) l, err := url.Parse(p.cfg.WebServerAddress + route) if err != nil { @@ -416,8 +390,9 @@ func (p *politeiawww) emailProposalVoteStarted(token, name, username, email stri if err != nil { return err } + recipients := []string{email} - err = p.sendEmailTo(subject, body, email) + err = p.smtp.sendEmailTo(subject, body, recipients) if err != nil { return err } @@ -435,10 +410,6 @@ func (p *politeiawww) emailProposalVoteStarted(token, name, username, email stri // emailUsersForProposalVoteStarted sends an email notification for a proposal // entering the voting state. func (p *politeiawww) emailUsersForProposalVoteStarted(proposal *www.ProposalRecord, authorUser *user.User, adminUser *user.User) error { - if p.smtp.disabled { - return nil - } - // Create the template data. l, err := url.Parse(p.cfg.WebServerAddress + "/proposals/" + proposal.CensorshipRecord.Token) @@ -461,8 +432,9 @@ func (p *politeiawww) emailUsersForProposalVoteStarted(proposal *www.ProposalRec if err != nil { return err } + recipients := []string{authorUser.Email} - err = p.sendEmailTo(subject, body, authorUser.Email) + err = p.smtp.sendEmailTo(subject, body, recipients) if err != nil { return err } @@ -518,10 +490,6 @@ func (p *politeiawww) emailProposalSubmitted(token, name, username string, email // emailProposalVoteAuthorized sends email notification for the proposal vote // authorized event. Sends to all admins with this notification bit set on. func (p *politeiawww) emailProposalVoteAuthorized(token, name, username, email string, emails []string) error { - if p.smtp.disabled { - return nil - } - route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) l, err := url.Parse(p.cfg.WebServerAddress + route) if err != nil { @@ -545,10 +513,6 @@ func (p *politeiawww) emailProposalVoteAuthorized(token, name, username, email s } func (p *politeiawww) emailAdminsForProposalVoteAuthorized(token, title, authorUsername, authorEmail string) error { - if p.smtp.disabled { - return nil - } - l, err := url.Parse(fmt.Sprintf("%v/proposals/%v", p.cfg.WebServerAddress, token)) if err != nil { @@ -584,10 +548,6 @@ func (p *politeiawww) emailAdminsForProposalVoteAuthorized(token, title, authorU // emailAuthorForCommentOnProposal sends an email notification to a proposal // author for a new comment. func (p *politeiawww) emailAuthorForCommentOnProposal(proposal *www.ProposalRecord, authorUser *user.User, commentID, username string) error { - if p.smtp.disabled { - return nil - } - l, err := url.Parse(fmt.Sprintf("%v/proposals/%v/comments/%v", p.cfg.WebServerAddress, proposal.CensorshipRecord.Token, commentID)) if err != nil { @@ -615,17 +575,14 @@ func (p *politeiawww) emailAuthorForCommentOnProposal(proposal *www.ProposalReco if err != nil { return err } + recipients := []string{authorUser.Email} - return p.sendEmailTo(subject, body, authorUser.Email) + return p.smtp.sendEmailTo(subject, body, recipients) } // emailAuthorForCommentOnComment sends an email notification to a comment // author for a new comment reply. func (p *politeiawww) emailAuthorForCommentOnComment(proposal *www.ProposalRecord, authorUser *user.User, commentID, username string) error { - if p.smtp.disabled { - return nil - } - l, err := url.Parse(fmt.Sprintf("%v/proposals/%v/comments/%v", p.cfg.WebServerAddress, proposal.CensorshipRecord.Token, commentID)) if err != nil { @@ -653,17 +610,14 @@ func (p *politeiawww) emailAuthorForCommentOnComment(proposal *www.ProposalRecor if err != nil { return err } + recipients := []string{authorUser.Email} - return p.sendEmailTo(subject, body, authorUser.Email) + return p.smtp.sendEmailTo(subject, body, recipients) } // emailUpdateUserKeyVerificationLink emails the link with the verification // token used for setting a new key pair if the email server is set up. func (p *politeiawww) emailUpdateUserKeyVerificationLink(email, publicKey, token string) error { - if p.smtp.disabled { - return nil - } - link, err := p.createEmailLink(www.RouteVerifyUpdateUserKey, "", token, "") if err != nil { return err @@ -680,17 +634,14 @@ func (p *politeiawww) emailUpdateUserKeyVerificationLink(email, publicKey, token if err != nil { return err } + recipients := []string{email} - return p.sendEmailTo(subject, body, email) + return p.smtp.sendEmailTo(subject, body, recipients) } // emailUserPasswordChanged notifies the user that his password was changed, // and verifies if he was the author of this action, for security purposes. func (p *politeiawww) emailUserPasswordChanged(email string) error { - if p.smtp.disabled { - return nil - } - tplData := userPasswordChangedTemplateData{ Email: email, } @@ -700,18 +651,15 @@ func (p *politeiawww) emailUserPasswordChanged(email string) error { if err != nil { return err } + recipients := []string{email} - return p.sendEmailTo(subject, body, email) + return p.smtp.sendEmailTo(subject, body, recipients) } // emailUserLocked notifies the user its account has been locked and emails the // link with the reset password verification token if the email server is set // up. func (p *politeiawww) emailUserLocked(email string) error { - if p.smtp.disabled { - return nil - } - link, err := p.createEmailLink(ResetPasswordGuiRoute, email, "", "") if err != nil { @@ -728,17 +676,14 @@ func (p *politeiawww) emailUserLocked(email string) error { if err != nil { return err } + recipients := []string{email} - return p.sendEmailTo(subject, body, email) + return p.smtp.sendEmailTo(subject, body, recipients) } // emailInviteNewUserVerificationLink emails the link to invite a user to // join the Contractor Management System, if the email server is set up. func (p *politeiawww) emailInviteNewUserVerificationLink(email, token string) error { - if p.smtp.disabled { - return nil - } - link, err := p.createEmailLink(guiRouteRegisterNewUser, "", token, "") if err != nil { return err @@ -754,17 +699,14 @@ func (p *politeiawww) emailInviteNewUserVerificationLink(email, token string) er if err != nil { return err } + recipients := []string{email} - return p.sendEmailTo(subject, body, email) + return p.smtp.sendEmailTo(subject, body, recipients) } // emailApproveDCCVerificationLink emails the link to invite a user that // has been approved by the other contractors from a DCC proposal. func (p *politeiawww) emailApproveDCCVerificationLink(email string) error { - if p.smtp.disabled { - return nil - } - tplData := approveDCCUserEmailTemplateData{ Email: email, } @@ -774,16 +716,14 @@ func (p *politeiawww) emailApproveDCCVerificationLink(email string) error { if err != nil { return err } + recipients := []string{email} - return p.sendEmailTo(subject, body, email) + return p.smtp.sendEmailTo(subject, body, recipients) } // emailInvoiceNotifications emails users that have not yet submitted an invoice // for the given month/year func (p *politeiawww) emailInvoiceNotifications(email, username string) error { - if p.smtp.disabled { - return nil - } // Set the date to the first day of the previous month. newDate := time.Date(time.Now().Year(), time.Now().Month()-1, 1, 0, 0, 0, 0, time.UTC) tplData := invoiceNotificationEmailData{ @@ -797,17 +737,14 @@ func (p *politeiawww) emailInvoiceNotifications(email, username string) error { if err != nil { return err } + recipients := []string{email} - return p.sendEmailTo(subject, body, email) + return p.smtp.sendEmailTo(subject, body, recipients) } // emailInvoiceComment sends email for the invoice comment event. Sends // email to the user regarding that invoice. func (p *politeiawww) emailInvoiceComment(userEmail string) error { - if p.smtp.disabled { - return nil - } - var tplData interface{} subject := "New Invoice Comment" @@ -815,15 +752,12 @@ func (p *politeiawww) emailInvoiceComment(userEmail string) error { if err != nil { return err } + recipients := []string{userEmail} - return p.sendEmailTo(subject, body, userEmail) + return p.smtp.sendEmailTo(subject, body, recipients) } func (p *politeiawww) emailUserInvoiceComment(userEmail string) error { - if p.smtp.disabled { - return nil - } - tplData := newInvoiceCommentTemplateData{} subject := "New Invoice Comment" @@ -831,17 +765,14 @@ func (p *politeiawww) emailUserInvoiceComment(userEmail string) error { if err != nil { return err } + recipients := []string{userEmail} - return p.sendEmailTo(subject, body, userEmail) + return p.smtp.sendEmailTo(subject, body, recipients) } // emailInvoiceStatusUpdate sends email for the invoice status update event. // Sends email for the user regarding that invoice. func (p *politeiawww) emailInvoiceStatusUpdate(invoiceToken, userEmail string) error { - if p.smtp.disabled { - return nil - } - tplData := newInvoiceStatusUpdateTemplate{ Token: invoiceToken, } @@ -851,15 +782,12 @@ func (p *politeiawww) emailInvoiceStatusUpdate(invoiceToken, userEmail string) e if err != nil { return err } + recipients := []string{userEmail} - return p.sendEmailTo(subject, body, userEmail) + return p.smtp.sendEmailTo(subject, body, recipients) } func (p *politeiawww) emailUserInvoiceStatusUpdate(userEmail, invoiceToken string) error { - if p.smtp.disabled { - return nil - } - tplData := newInvoiceStatusUpdateTemplate{ Token: invoiceToken, } @@ -869,17 +797,14 @@ func (p *politeiawww) emailUserInvoiceStatusUpdate(userEmail, invoiceToken strin if err != nil { return err } + recipients := []string{userEmail} - return p.sendEmailTo(subject, body, userEmail) + return p.smtp.sendEmailTo(subject, body, recipients) } // emailDCCNew sends email regarding the DCC New event. Sends email // to all admins. func (p *politeiawww) emailDCCNew(token string, emails []string) error { - if p.smtp.disabled { - return nil - } - route := strings.Replace(guiRouteDCCDetails, "{token}", token, 1) l, err := url.Parse(p.cfg.WebServerAddress + route) if err != nil { @@ -900,10 +825,6 @@ func (p *politeiawww) emailDCCNew(token string, emails []string) error { } func (p *politeiawww) emailAdminsForNewDCC(token string) error { - if p.smtp.disabled { - return nil - } - l, err := url.Parse(p.cfg.WebServerAddress + "{token}" + token) if err != nil { return err @@ -933,10 +854,6 @@ func (p *politeiawww) emailAdminsForNewDCC(token string) error { // emailDCCSupportOppose sends emails regarding dcc support/oppose event. // Sends emails to all admin users. func (p *politeiawww) emailDCCSupportOppose(token string, emails []string) error { - if p.smtp.disabled { - return nil - } - route := strings.Replace(guiRouteDCCDetails, "{token}", token, 1) l, err := url.Parse(p.cfg.WebServerAddress + route) if err != nil { @@ -957,10 +874,6 @@ func (p *politeiawww) emailDCCSupportOppose(token string, emails []string) error } func (p *politeiawww) emailAdminsForNewDCCSupportOppose(token string) error { - if p.smtp.disabled { - return nil - } - l, err := url.Parse(p.cfg.WebServerAddress + "/dcc/" + token) if err != nil { return err diff --git a/politeiawww/eventmanager.go b/politeiawww/eventmanager.go index 8ebf1080e..93c2f2843 100644 --- a/politeiawww/eventmanager.go +++ b/politeiawww/eventmanager.go @@ -7,6 +7,7 @@ package main import ( "sync" + v1 "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/user" "github.com/google/uuid" @@ -18,7 +19,7 @@ const ( // Event types eventTypeInvalid eventT = iota - // Politeia events + // Pi events eventProposalSubmitted eventProposalStatusChange eventProposalEdited @@ -108,8 +109,11 @@ func (p *politeiawww) setupEventListenersPi() { p.eventManager.register(eventProposalVoteStarted, ch) go p.handleEventProposalVoteStarted(ch) +} + +func (p *politeiawww) setupEventListenersCms() { // Setup invoice comment event - ch = make(chan interface{}) + ch := make(chan interface{}) p.eventManager.register(eventInvoiceComment, ch) go p.handleEventInvoiceComment(ch) @@ -136,11 +140,24 @@ func notificationIsSet(emailNotifications uint64, n www.EmailNotificationT) bool // Notification bit not set return false } - // Notification bit is set return true } +// userNotificationEnabled wraps all user checks to see if he is in correct +// state to receive notifications +func userNotificationEnabled(u user.User, n www.EmailNotificationT) bool { + // Never send notification to deactivated users + if u.Deactivated { + return false + } + // Check if notification bit is set + if !notificationIsSet(u.EmailNotifications, n) { + return false + } + return true +} + type dataProposalSubmitted struct { token string // Proposal token name string // Proposal name @@ -158,21 +175,16 @@ func (p *politeiawww) handleEventProposalSubmitted(ch chan interface{}) { // Compile email notification recipients emails := make([]string, 0, 256) err := p.db.AllUsers(func(u *user.User) { - // Only send proposal submitted notifications to admins - if !u.Admin { - return + // Check if user is able to receive notification + if userNotificationEnabled(*u, + www.NotificationEmailAdminProposalNew) { + emails = append(emails, u.Email) } - // Notifications should not be sent to deactivated users - if u.Deactivated { + // Only send proposal submitted notifications to admins + if !u.Admin { return } - - // Check if notification bit is set - if notificationIsSet(u.EmailNotifications, - www.NotificationEmailAdminProposalNew) { - emails = append(emails, u.Email) - } }) if err != nil { log.Errorf("handleEventProposalSubmitted: AllUsers: %v", err) @@ -190,15 +202,15 @@ func (p *politeiawww) handleEventProposalSubmitted(ch chan interface{}) { } type dataProposalStatusChange struct { - name string // Proposal name - token string // Proposal censorship token - adminID uuid.UUID // Admin uuid - id uuid.UUID // Author uuid - email string // Author user email - emailNotifications uint64 // Author notification settings - username string // Author username - status www.PropStatusT // Proposal status - statusChangeMessage string // Status change message + name string // Proposal name + token string // Proposal censorship token + adminID uuid.UUID // Admin uuid + id uuid.UUID // Author uuid + email string // Author user email + emailNotifications uint64 // Author notification settings + username string // Author username + status v1.PropStatusT // Proposal status + statusChangeMessage string // Status change message } func (p *politeiawww) handleEventProposalStatusChange(ch chan interface{}) { @@ -209,16 +221,17 @@ func (p *politeiawww) handleEventProposalStatusChange(ch chan interface{}) { continue } - if d.status != www.PropStatusPublic && - d.status != www.PropStatusCensored { + // Check if proposal is in correct status for notification + if d.status != v1.PropStatusPublic && + d.status != v1.PropStatusCensored { continue } emails := make([]string, 0, 256) err := p.db.AllUsers(func(u *user.User) { // Check circunstances where we don't notify - if u.Deactivated || u.ID == d.adminID || u.ID == d.id || - !notificationIsSet(u.EmailNotifications, + if u.ID == d.adminID || u.ID == d.id || + !userNotificationEnabled(*u, www.NotificationEmailRegularProposalVetted) { return } @@ -254,8 +267,8 @@ func (p *politeiawww) handleEventProposalEdited(ch chan interface{}) { emails := make([]string, 0, 256) err := p.db.AllUsers(func(u *user.User) { // Check circunstances where we don't notify - if u.NewUserPaywallTx == "" || u.Deactivated || u.ID == d.id || - !notificationIsSet(u.EmailNotifications, + if u.NewUserPaywallTx == "" || u.ID == d.id || + !userNotificationEnabled(*u, www.NotificationEmailRegularProposalEdited) { return } @@ -291,9 +304,8 @@ func (p *politeiawww) handleEventProposalVoteAuthorized(ch chan interface{}) { emails := make([]string, 0, 256) err := p.db.AllUsers(func(u *user.User) { // Check circunstances where we don't notify - if !u.Admin || u.Deactivated || - !notificationIsSet(u.EmailNotifications, - www.NotificationEmailAdminProposalVoteAuthorized) { + if !u.Admin || !userNotificationEnabled(*u, + www.NotificationEmailAdminProposalVoteAuthorized) { return } @@ -331,9 +343,8 @@ func (p *politeiawww) handleEventProposalVoteStarted(ch chan interface{}) { emails := make([]string, 0, 256) err := p.db.AllUsers(func(u *user.User) { // Check circunstances where we don't notify - if u.NewUserPaywallTx == "" || u.Deactivated || - u.ID == d.adminID || u.ID == d.id || - !notificationIsSet(u.EmailNotifications, + if u.NewUserPaywallTx == "" || u.ID == d.adminID || u.ID == d.id || + !userNotificationEnabled(*u, www.NotificationEmailRegularProposalVoteStarted) { return } diff --git a/politeiawww/www.go b/politeiawww/www.go index 0908daab1..3219b5a3b 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -753,6 +753,7 @@ func _main() error { case cmsWWWMode: // Setup event manager p.initCMSEventManager() + p.setupEventListenersCms() // Setup dcrdata websocket connection ws, err := wsdcrdata.New(p.dcrdataHostWS()) From 803f48369987fd7470ed22c2dd131109fe753920 Mon Sep 17 00:00:00 2001 From: Thiago Figueiredo Date: Thu, 10 Sep 2020 15:53:28 -0300 Subject: [PATCH 036/449] emit cms events, fully working. minor code cleanup --- politeiawww/dcc.go | 17 +++++++++-------- politeiawww/eventmanager.go | 4 +--- politeiawww/invoices.go | 10 ++++++---- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/politeiawww/dcc.go b/politeiawww/dcc.go index a7742ac70..f594747ad 100644 --- a/politeiawww/dcc.go +++ b/politeiawww/dcc.go @@ -239,10 +239,10 @@ func (p *politeiawww) processNewDCC(nd cms.NewDCC, u *user.User) (*cms.NewDCCRep return nil, err } - // Fire new event notification upon new DCC being submitted. - p.fireEvent(EventTypeDCCNew, - EventDataDCCNew{ - Token: pdReply.CensorshipRecord.Token, + // Emit event notification for new DCC being submitted + p.eventManager.emit(eventDCCNew, + dataDCCNew{ + token: pdReply.CensorshipRecord.Token, }) cr := convertPropCensorFromPD(pdReply.CensorshipRecord) @@ -663,10 +663,11 @@ func (p *politeiawww) processSupportOpposeDCC(sd cms.SupportOpposeDCC, u *user.U if err != nil { return nil, err } - // Fire new event notification upon new DCC being supported or opposed. - p.fireEvent(EventTypeDCCSupportOppose, - EventDataDCCSupportOppose{ - Token: sd.Token, + + // Emit event notification for a DCC being supported/opposed + p.eventManager.emit(eventDCCSupportOppose, + dataDCCSupportOppose{ + token: sd.Token, }) return &cms.SupportOpposeDCCReply{}, nil diff --git a/politeiawww/eventmanager.go b/politeiawww/eventmanager.go index 93c2f2843..fd1dc0399 100644 --- a/politeiawww/eventmanager.go +++ b/politeiawww/eventmanager.go @@ -391,7 +391,7 @@ type dataInvoiceStatusUpdate struct { func (p *politeiawww) handleEventInvoiceStatusUpdate(ch chan interface{}) { for msg := range ch { - d, ok := msg.(dataInvoiceComment) + d, ok := msg.(dataInvoiceStatusUpdate) if !ok { log.Errorf("handleEventInvoiceStatusUpdate invalid msg: %v", msg) continue @@ -408,7 +408,6 @@ func (p *politeiawww) handleEventInvoiceStatusUpdate(ch chan interface{}) { type dataDCCNew struct { token string // DCC token - email string // User email } func (p *politeiawww) handleEventDCCNew(ch chan interface{}) { @@ -440,7 +439,6 @@ func (p *politeiawww) handleEventDCCNew(ch chan interface{}) { type dataDCCSupportOppose struct { token string // DCC token - email string // User email } func (p *politeiawww) handleEventDCCSupportOppose(ch chan interface{}) { diff --git a/politeiawww/invoices.go b/politeiawww/invoices.go index 38f2c42c5..2aa642263 100644 --- a/politeiawww/invoices.go +++ b/politeiawww/invoices.go @@ -990,10 +990,12 @@ func (p *politeiawww) processSetInvoiceStatus(sis cms.SetInvoiceStatus, u *user. if c.NewStatus == cms.InvoiceStatusApproved { p.addWatchAddress(dbInvoice.PaymentAddress) } - p.fireEvent(EventTypeInvoiceStatusUpdate, - EventDataInvoiceStatusUpdate{ - Token: sis.Token, - User: invoiceUser, + + // Emit event notification for invoice status update + p.eventManager.emit(eventInvoiceStatusUpdate, + dataInvoiceStatusUpdate{ + token: dbInvoice.Token, + email: invoiceUser.Email, }) } From 6a9067458a3b3b8fdddf01aad9ce94e7ab1c5e82 Mon Sep 17 00:00:00 2001 From: Thiago Figueiredo Date: Fri, 11 Sep 2020 14:53:26 -0300 Subject: [PATCH 037/449] emit all events from new event manager; nuke all references to old events.go and delete file; email.go code cleanup --- politeiawww/comments.go | 14 +- politeiawww/email.go | 381 +-------------------------- politeiawww/eventmanager.go | 90 ++++++- politeiawww/events.go | 511 ------------------------------------ politeiawww/invoices.go | 13 +- politeiawww/proposals.go | 97 ++++--- politeiawww/user.go | 8 - politeiawww/www.go | 4 +- 8 files changed, 176 insertions(+), 942 deletions(-) delete mode 100644 politeiawww/events.go diff --git a/politeiawww/comments.go b/politeiawww/comments.go index cc2e3bc87..4c4abb01d 100644 --- a/politeiawww/comments.go +++ b/politeiawww/comments.go @@ -332,10 +332,16 @@ func (p *politeiawww) processNewComment(nc www.NewComment, u *user.User) (*www.N return nil, fmt.Errorf("getComment: %v", err) } - // Fire off new comment event - p.fireEvent(EventTypeComment, EventDataComment{ - Comment: c, - }) + // Emit event notification for a proposal comment + p.eventManager.emit(eventProposalComment, + dataProposalComment{ + token: pr.CensorshipRecord.Token, + name: pr.Name, + username: pr.Username, + parentID: c.ParentID, + commentID: c.CommentID, + commentUsername: c.Username, + }) return &www.NewCommentReply{ Comment: *c, diff --git a/politeiawww/email.go b/politeiawww/email.go index 8de15011e..bb6e85133 100644 --- a/politeiawww/email.go +++ b/politeiawww/email.go @@ -6,25 +6,22 @@ package main import ( "bytes" - "fmt" "net/url" "strings" "text/template" "time" - "github.com/dajohi/goemail" - v1 "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" - "github.com/decred/politeia/politeiawww/user" ) const ( // GUI routes. These are used in notification emails to direct the // user to the correct GUI pages. - guiRouteProposalDetails = "/proposals/{token}" - guiRouteRegisterNewUser = "/register" - guiRouteDCCDetails = "/dcc/{token}" + guiRouteProposalDetails = "/proposals/{token}" + guirouteProposalComments = "/proposals/{token}/comments/{id}" + guiRouteRegisterNewUser = "/register" + guiRouteDCCDetails = "/dcc/{token}" ) func createBody(tpl *template.Template, tplData interface{}) (string, error) { @@ -122,36 +119,6 @@ func (p *politeiawww) emailResetPasswordVerificationLink(email, username, token return p.smtp.sendEmailTo(subject, body, recipients) } -// emailAuthorForCensoredProposal sends an email notification for a new -// proposal becoming censored to the proposal's author. -func (p *politeiawww) emailAuthorForCensoredProposal(proposal *www.ProposalRecord, authorUser *user.User, adminUser *user.User) error { - l, err := url.Parse(p.cfg.WebServerAddress + "/proposals/" + - proposal.CensorshipRecord.Token) - if err != nil { - return err - } - - if authorUser.EmailNotifications& - uint64(www.NotificationEmailMyProposalStatusChange) == 0 { - return nil - } - - tplData := proposalStatusChangeTemplateData{ - Link: l.String(), - Name: proposal.Name, - StatusChangeReason: proposal.StatusChangeMessage, - } - - subject := "Your Proposal Has Been Censored" - body, err := createBody(templateProposalCensoredForAuthor, &tplData) - if err != nil { - return err - } - recipients := []string{authorUser.Email} - - return p.smtp.sendEmailTo(subject, body, recipients) -} - // emailProposalStatusChange sends emails regarding the proposal status change // event. Sends email for the author and the users with this notification // bit set on @@ -233,75 +200,6 @@ func (p *politeiawww) emailUsersProposalStatusChange(name, username, link string return p.smtp.sendEmailTo(subject, body, emails) } -// emailAuthorForVettedProposal sends an email notification for a new proposal -// becoming vetted to the proposal's author. -func (p *politeiawww) emailAuthorForVettedProposal(proposal *www.ProposalRecord, authorUser *user.User, adminUser *user.User) error { - l, err := url.Parse(p.cfg.WebServerAddress + "/proposals/" + - proposal.CensorshipRecord.Token) - if err != nil { - return err - } - - if authorUser.EmailNotifications& - uint64(www.NotificationEmailMyProposalStatusChange) == 0 { - return nil - } - - tplData := proposalStatusChangeTemplateData{ - Link: l.String(), - Name: proposal.Name, - StatusChangeReason: proposal.StatusChangeMessage, - } - - subject := "Your Proposal Has Been Published" - body, err := createBody(templateProposalVettedForAuthor, &tplData) - if err != nil { - return err - } - recipients := []string{authorUser.Email} - - return p.smtp.sendEmailTo(subject, body, recipients) -} - -// emailUsersForVettedProposal sends an email notification for a new proposal -// becoming vetted. -func (p *politeiawww) emailUsersForVettedProposal(proposal *www.ProposalRecord, authorUser *user.User, adminUser *user.User) error { - // Create the template data. - l, err := url.Parse(p.cfg.WebServerAddress + "/proposals/" + - proposal.CensorshipRecord.Token) - if err != nil { - return err - } - - tplData := proposalStatusChangeTemplateData{ - Link: l.String(), - Name: proposal.Name, - Username: authorUser.Username, - } - - // Send email to users. - subject := "New Proposal Published" - body, err := createBody(templateProposalVetted, &tplData) - if err != nil { - return err - } - - return p.smtp.sendEmail(subject, body, func(msg *goemail.Message) error { - // Add user emails to the goemail.Message - return p.db.AllUsers(func(u *user.User) { - // Don't notify the user under certain conditions. - if u.NewUserPaywallTx == "" || u.Deactivated || - u.ID == adminUser.ID || u.ID == authorUser.ID || - (u.EmailNotifications& - uint64(www.NotificationEmailRegularProposalVetted)) == 0 { - return - } - - msg.AddBCC(u.Email) - }) - }) -} - // emailProposalEdited sends email regarding the proposal edits event. // Sends to all users with this notification bit turned on. func (p *politeiawww) emailProposalEdited(name, username, token, version string, emails []string) error { @@ -327,46 +225,6 @@ func (p *politeiawww) emailProposalEdited(name, username, token, version string, return p.smtp.sendEmailTo(subject, body, emails) } -// emailUsersForEditedProposal sends an email notification for a proposal being -// edited. -func (p *politeiawww) emailUsersForEditedProposal(proposal *www.ProposalRecord, authorUser *user.User) error { - // Create the template data. - l, err := url.Parse(p.cfg.WebServerAddress + "/proposals/" + - proposal.CensorshipRecord.Token) - if err != nil { - return err - } - - tplData := proposalEditedTemplateData{ - Link: l.String(), - Name: proposal.Name, - Version: proposal.Version, - Username: authorUser.Username, - } - - // Send email to users. - subject := "Proposal Edited" - body, err := createBody(templateProposalEdited, &tplData) - if err != nil { - return err - } - - return p.smtp.sendEmail(subject, body, func(msg *goemail.Message) error { - // Add user emails to the goemail.Message - return p.db.AllUsers(func(u *user.User) { - // Don't notify the user under certain conditions. - if u.NewUserPaywallTx == "" || u.Deactivated || - u.ID == authorUser.ID || - (u.EmailNotifications& - uint64(www.NotificationEmailRegularProposalEdited)) == 0 { - return - } - - msg.AddBCC(u.Email) - }) - }) -} - // emailProposalVoteStarted sends email for the proposal vote started event. // Sends email to author and users with this notification bit set on. func (p *politeiawww) emailProposalVoteStarted(token, name, username, email string, emailNotifications uint64, emails []string) error { @@ -407,62 +265,6 @@ func (p *politeiawww) emailProposalVoteStarted(token, name, username, email stri return p.smtp.sendEmailTo(subject, body, emails) } -// emailUsersForProposalVoteStarted sends an email notification for a proposal -// entering the voting state. -func (p *politeiawww) emailUsersForProposalVoteStarted(proposal *www.ProposalRecord, authorUser *user.User, adminUser *user.User) error { - // Create the template data. - l, err := url.Parse(p.cfg.WebServerAddress + "/proposals/" + - proposal.CensorshipRecord.Token) - if err != nil { - return err - } - - tplData := proposalVoteStartedTemplateData{ - Link: l.String(), - Name: proposal.Name, - Username: authorUser.Username, - } - - // Send email to author. - if authorUser.EmailNotifications& - uint64(www.NotificationEmailMyProposalVoteStarted) != 0 { - - subject := "Your Proposal Has Started Voting" - body, err := createBody(templateProposalVoteStartedForAuthor, &tplData) - if err != nil { - return err - } - recipients := []string{authorUser.Email} - - err = p.smtp.sendEmailTo(subject, body, recipients) - if err != nil { - return err - } - } - - subject := "Voting Started for Proposal" - body, err := createBody(templateProposalVoteStarted, &tplData) - if err != nil { - return err - } - - return p.smtp.sendEmail(subject, body, func(msg *goemail.Message) error { - // Add user emails to the goemail.Message - return p.db.AllUsers(func(u *user.User) { - // Don't notify the user under certain conditions. - if u.NewUserPaywallTx == "" || u.Deactivated || - u.ID == adminUser.ID || - u.ID == authorUser.ID || - (u.EmailNotifications& - uint64(www.NotificationEmailRegularProposalVoteStarted)) == 0 { - return - } - - msg.AddBCC(u.Email) - }) - }) -} - // emailProposalSubmitted sends email notification for a new proposal becoming // vetted. Sends to the author and for users with this notification setting. func (p *politeiawww) emailProposalSubmitted(token, name, username string, emails []string) error { @@ -512,61 +314,17 @@ func (p *politeiawww) emailProposalVoteAuthorized(token, name, username, email s return p.smtp.sendEmailTo(subject, body, emails) } -func (p *politeiawww) emailAdminsForProposalVoteAuthorized(token, title, authorUsername, authorEmail string) error { - l, err := url.Parse(fmt.Sprintf("%v/proposals/%v", - p.cfg.WebServerAddress, token)) - if err != nil { - return err - } - - tplData := proposalVoteAuthorizedTemplateData{ - Link: l.String(), - Name: title, - Username: authorUsername, - Email: authorEmail, - } - - subject := "Proposal Authorized To Start Voting" - body, err := createBody(templateProposalVoteAuthorized, &tplData) - if err != nil { - return err - } - - return p.smtp.sendEmail(subject, body, func(msg *goemail.Message) error { - // Add admin emails to the goemail.Message - return p.db.AllUsers(func(u *user.User) { - if !u.Admin || u.Deactivated || - (u.EmailNotifications& - uint64(www.NotificationEmailAdminProposalVoteAuthorized) == 0) { - return - } - msg.AddBCC(u.Email) - }) - }) -} - -// emailAuthorForCommentOnProposal sends an email notification to a proposal -// author for a new comment. -func (p *politeiawww) emailAuthorForCommentOnProposal(proposal *www.ProposalRecord, authorUser *user.User, commentID, username string) error { - l, err := url.Parse(fmt.Sprintf("%v/proposals/%v/comments/%v", - p.cfg.WebServerAddress, proposal.CensorshipRecord.Token, commentID)) +func (p *politeiawww) emailProposalComment(token, commentID, commentUsername, name, email string) error { + route := strings.Replace(guirouteProposalComments, "{token}", token, 1) + route = strings.Replace(route, "{id}", commentID, 1) + l, err := url.Parse(p.cfg.WebServerAddress + route) if err != nil { return err } - if authorUser.EmailNotifications& - uint64(www.NotificationEmailCommentOnMyProposal) == 0 { - return nil - } - - // Don't send email when author comments on own proposal - if username == authorUser.Username { - return nil - } - tplData := commentReplyOnProposalTemplateData{ - Commenter: username, - ProposalName: proposal.Name, + Commenter: commentUsername, + ProposalName: name, CommentLink: l.String(), } @@ -575,42 +333,7 @@ func (p *politeiawww) emailAuthorForCommentOnProposal(proposal *www.ProposalReco if err != nil { return err } - recipients := []string{authorUser.Email} - - return p.smtp.sendEmailTo(subject, body, recipients) -} - -// emailAuthorForCommentOnComment sends an email notification to a comment -// author for a new comment reply. -func (p *politeiawww) emailAuthorForCommentOnComment(proposal *www.ProposalRecord, authorUser *user.User, commentID, username string) error { - l, err := url.Parse(fmt.Sprintf("%v/proposals/%v/comments/%v", - p.cfg.WebServerAddress, proposal.CensorshipRecord.Token, commentID)) - if err != nil { - return err - } - - if authorUser.EmailNotifications& - uint64(www.NotificationEmailCommentOnMyComment) == 0 { - return nil - } - - // Don't send email when author replies to his own comment - if username == authorUser.Username { - return nil - } - - tplData := commentReplyOnCommentTemplateData{ - Commenter: username, - ProposalName: proposal.Name, - CommentLink: l.String(), - } - - subject := "New Comment On Your Comment" - body, err := createBody(templateCommentReplyOnComment, &tplData) - if err != nil { - return err - } - recipients := []string{authorUser.Email} + recipients := []string{email} return p.smtp.sendEmailTo(subject, body, recipients) } @@ -757,19 +480,6 @@ func (p *politeiawww) emailInvoiceComment(userEmail string) error { return p.smtp.sendEmailTo(subject, body, recipients) } -func (p *politeiawww) emailUserInvoiceComment(userEmail string) error { - tplData := newInvoiceCommentTemplateData{} - - subject := "New Invoice Comment" - body, err := createBody(templateNewInvoiceComment, &tplData) - if err != nil { - return err - } - recipients := []string{userEmail} - - return p.smtp.sendEmailTo(subject, body, recipients) -} - // emailInvoiceStatusUpdate sends email for the invoice status update event. // Sends email for the user regarding that invoice. func (p *politeiawww) emailInvoiceStatusUpdate(invoiceToken, userEmail string) error { @@ -787,21 +497,6 @@ func (p *politeiawww) emailInvoiceStatusUpdate(invoiceToken, userEmail string) e return p.smtp.sendEmailTo(subject, body, recipients) } -func (p *politeiawww) emailUserInvoiceStatusUpdate(userEmail, invoiceToken string) error { - tplData := newInvoiceStatusUpdateTemplate{ - Token: invoiceToken, - } - - subject := "Invoice status has been updated" - body, err := createBody(templateNewInvoiceStatusUpdate, &tplData) - if err != nil { - return err - } - recipients := []string{userEmail} - - return p.smtp.sendEmailTo(subject, body, recipients) -} - // emailDCCNew sends email regarding the DCC New event. Sends email // to all admins. func (p *politeiawww) emailDCCNew(token string, emails []string) error { @@ -824,33 +519,6 @@ func (p *politeiawww) emailDCCNew(token string, emails []string) error { return p.smtp.sendEmailTo(subject, body, emails) } -func (p *politeiawww) emailAdminsForNewDCC(token string) error { - l, err := url.Parse(p.cfg.WebServerAddress + "{token}" + token) - if err != nil { - return err - } - - tplData := newDCCSubmittedTemplateData{ - Link: l.String(), - } - - subject := "New DCC Submitted" - body, err := createBody(templateNewDCCSubmitted, &tplData) - if err != nil { - return err - } - - return p.smtp.sendEmail(subject, body, func(msg *goemail.Message) error { - // Add admin emails to the goemail.Message - return p.db.AllUsers(func(u *user.User) { - if !u.Admin || u.Deactivated { - return - } - msg.AddBCC(u.Email) - }) - }) -} - // emailDCCSupportOppose sends emails regarding dcc support/oppose event. // Sends emails to all admin users. func (p *politeiawww) emailDCCSupportOppose(token string, emails []string) error { @@ -872,30 +540,3 @@ func (p *politeiawww) emailDCCSupportOppose(token string, emails []string) error return p.smtp.sendEmailTo(subject, body, emails) } - -func (p *politeiawww) emailAdminsForNewDCCSupportOppose(token string) error { - l, err := url.Parse(p.cfg.WebServerAddress + "/dcc/" + token) - if err != nil { - return err - } - - tplData := newDCCSupportOpposeTemplateData{ - Link: l.String(), - } - - subject := "New DCC Support or Opposition Submitted" - body, err := createBody(templateNewDCCSupportOppose, &tplData) - if err != nil { - return err - } - - return p.smtp.sendEmail(subject, body, func(msg *goemail.Message) error { - // Add admin emails to the goemail.Message - return p.db.AllUsers(func(u *user.User) { - if !u.Admin || u.Deactivated { - return - } - msg.AddBCC(u.Email) - }) - }) -} diff --git a/politeiawww/eventmanager.go b/politeiawww/eventmanager.go index fd1dc0399..5021e0599 100644 --- a/politeiawww/eventmanager.go +++ b/politeiawww/eventmanager.go @@ -20,6 +20,7 @@ const ( eventTypeInvalid eventT = iota // Pi events + eventProposalComment eventProposalSubmitted eventProposalStatusChange eventProposalEdited @@ -84,8 +85,13 @@ func (p *politeiawww) setupEventListenersPi() { // 2. Register the channel with the event manager // 3. Launch an event handler to listen for new events - // Setup proposal submitted event + // Setup proposal comment event ch := make(chan interface{}) + p.eventManager.register(eventProposalComment, ch) + go p.handleEventProposalComment(ch) + + // Setup proposal submitted event + ch = make(chan interface{}) p.eventManager.register(eventProposalSubmitted, ch) go p.handleEventProposalSubmitted(ch) @@ -108,7 +114,6 @@ func (p *politeiawww) setupEventListenersPi() { ch = make(chan interface{}) p.eventManager.register(eventProposalVoteStarted, ch) go p.handleEventProposalVoteStarted(ch) - } func (p *politeiawww) setupEventListenersCms() { @@ -158,6 +163,75 @@ func userNotificationEnabled(u user.User, n www.EmailNotificationT) bool { return true } +type dataProposalComment struct { + token string // Proposal token + name string // Proposal name + username string // Author username + parentID string // Parent comment id + commentID string // Comment id + commentUsername string // Comment user username +} + +func (p *politeiawww) handleEventProposalComment(ch chan interface{}) { + for msg := range ch { + d, ok := msg.(dataProposalComment) + if !ok { + log.Errorf("handleEventProposalComment invalid msg: %v", msg) + continue + } + + // Fetch proposal author + author, err := p.db.UserGetByUsername(d.username) + if err != nil { + log.Error(err) + continue + } + + // Check if user notification is enabled + if !userNotificationEnabled(*author, + www.NotificationEmailCommentOnMyProposal) { + continue + } + + // Don't notify when author comments on own proposal + if d.commentUsername == author.Username { + continue + } + + // Top-level comment + if d.parentID == "0" { + err := p.emailProposalComment(d.token, d.commentID, + d.commentUsername, d.name, author.Email) + if err != nil { + log.Errorf("emailProposalComment: %v", err) + } + continue + } + + // Nested comment reply. Fetch parent comment in order to fetch + // parent comment author + parent, err := p.decredCommentGetByID(d.token, d.parentID) + if err != nil { + log.Errorf("decredCommentGetByID: %v", err) + continue + } + + author, err = p.db.UserGetByPubKey(parent.PublicKey) + if err != nil { + log.Errorf("UserGetByPubKey: %v", err) + continue + } + + err = p.emailProposalComment(d.token, d.commentID, + d.commentUsername, d.name, author.Email) + if err != nil { + log.Errorf("emailProposalComment: %v", err) + } + + log.Debugf("Sent proposal commment notification %v", d.token) + } +} + type dataProposalSubmitted struct { token string // Proposal token name string // Proposal name @@ -204,13 +278,13 @@ func (p *politeiawww) handleEventProposalSubmitted(ch chan interface{}) { type dataProposalStatusChange struct { name string // Proposal name token string // Proposal censorship token + status v1.PropStatusT // Proposal status + statusChangeMessage string // Status change message adminID uuid.UUID // Admin uuid id uuid.UUID // Author uuid email string // Author user email emailNotifications uint64 // Author notification settings username string // Author username - status v1.PropStatusT // Proposal status - statusChangeMessage string // Status change message } func (p *politeiawww) handleEventProposalStatusChange(ch chan interface{}) { @@ -355,10 +429,10 @@ func (p *politeiawww) handleEventProposalVoteStarted(ch chan interface{}) { err = p.emailProposalVoteStarted(d.token, d.name, d.username, d.email, d.emailNotifications, emails) if err != nil { - log.Errorf("emailProposalVoteAuthorized: %v", err) + log.Errorf("emailProposalVoteStarted: %v", err) } - log.Debugf("Sent proposal authorized vote notifications %v", d.token) + log.Debugf("Sent proposal vote started notifications %v", d.token) } } @@ -399,10 +473,10 @@ func (p *politeiawww) handleEventInvoiceStatusUpdate(ch chan interface{}) { err := p.emailInvoiceStatusUpdate(d.token, d.email) if err != nil { - log.Errorf("emailInvoiceComment %v: %v", err) + log.Errorf("emailInvoiceStatusUpdate %v: %v", err) } - log.Debugf("Sent invoice comment notification %v", d.token) + log.Debugf("Sent invoice status update notification %v", d.token) } } diff --git a/politeiawww/events.go b/politeiawww/events.go deleted file mode 100644 index a60f4ac03..000000000 --- a/politeiawww/events.go +++ /dev/null @@ -1,511 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" - - "github.com/google/uuid" - - www "github.com/decred/politeia/politeiawww/api/www/v1" - www2 "github.com/decred/politeia/politeiawww/api/www/v2" - "github.com/decred/politeia/politeiawww/user" -) - -const ( - // XXX these events need to be moved over to eventmanager.go and - // the handlers need to be refactored to conform to the style in - // eventmanager.go. - - // Event types - EventTypeComment eventT = iota + 10 - EventTypeUserManage - - // Pi events - EventTypeProposalStatusChange - EventTypeProposalEdited - EventTypeProposalVoteAuthorized - EventTypeProposalVoteStarted - - // CMS events - EventTypeInvoiceComment // CMS Type - EventTypeInvoiceStatusUpdate // CMS Type - EventTypeDCCNew // DCC Type - EventTypeDCCSupportOppose // DCC Type -) - -type eventDataProposalVoteAuthorized struct { - token string // Proposal token - name string // Proposal name - authorUsername string - authorEmail string -} - -type EventDataProposalStatusChange struct { - Proposal *www.ProposalRecord - SetProposalStatus *www.SetProposalStatus - AdminUser *user.User -} - -type EventDataProposalEdited struct { - Proposal *www.ProposalRecord -} - -type EventDataProposalVoteStarted struct { - AdminUser *user.User - StartVote www2.StartVote -} - -type EventDataComment struct { - Comment *www.Comment -} - -type EventDataUserManage struct { - AdminUser *user.User - User *user.User - ManageUser *www.ManageUser -} - -type EventDataInvoiceComment struct { - Token string - User *user.User -} - -type EventDataInvoiceStatusUpdate struct { - Token string - User *user.User -} - -type EventDataDCCNew struct { - Token string -} - -type EventDataDCCSupportOppose struct { - Token string -} - -func (p *politeiawww) getProposalAndAuthor(token string) (*www.ProposalRecord, *user.User, error) { - proposal, err := p.getProp(token) - if err != nil { - return nil, nil, err - } - - userID, err := uuid.Parse(proposal.UserId) - if err != nil { - return nil, nil, fmt.Errorf("cannot parse UUID %v: %v", - proposal.UserId, err) - } - - author, err := p.db.UserGetById(userID) - if err != nil { - return nil, nil, fmt.Errorf("user lookup failed for userID %v: %v", - userID, err) - } - - return proposal, author, nil -} - -// fireEvent is a convenience wrapper for EventManager._fireEvent which -// holds the lock. -// -// This function must be called WITHOUT the mutex held. -func (p *politeiawww) fireEvent(eventType eventT, data interface{}) { - if p.test { - return - } - - p.Lock() - defer p.Unlock() - - p.eventManager._fireEvent(eventType, data) -} - -func (p *politeiawww) initEventManagerPi() { - - p.Lock() - defer p.Unlock() - - p._setupProposalStatusChangeLogging() - p._setupProposalVoteStartedLogging() - p._setupUserManageLogging() - - p._setupProposalStatusChangeEmailNotification() - p._setupProposalEditedEmailNotification() - p._setupProposalVoteStartedEmailNotification() - p._setupProposalVoteAuthorizedEmailNotification() - p._setupCommentReplyEmailNotifications() -} - -func (p *politeiawww) initCMSEventManager() { - p.Lock() - defer p.Unlock() - - if p.smtp.disabled { - return - } - - p._setupInvoiceCommentEmailNotification() - p._setupInvoiceStatusUpdateEmailNotification() - p._setupDCCNewEmailNotification() - p._setupDCCSupportOpposeEmailNotification() - //p._setupInvoiceEditedEmailNotification() - //p._setupInvoiceCommentEmailNotification() -} - -func (p *politeiawww) _setupInvoiceCommentEmailNotification() { - ch := make(chan interface{}) - go func() { - for data := range ch { - ic, ok := data.(EventDataInvoiceComment) - if !ok { - log.Errorf("invalid event data") - continue - } - - err := p.emailUserInvoiceComment(ic.User.Email) - if err != nil { - log.Errorf("email for new admin comment %v: %v", - ic.Token, err) - } - } - }() - p.eventManager._register(EventTypeInvoiceComment, ch) -} - -func (p *politeiawww) _setupInvoiceStatusUpdateEmailNotification() { - ch := make(chan interface{}) - go func() { - for data := range ch { - isu, ok := data.(EventDataInvoiceStatusUpdate) - if !ok { - log.Errorf("invalid event data") - continue - } - - err := p.emailUserInvoiceStatusUpdate(isu.User.Email, isu.Token) - if err != nil { - log.Errorf("email for new admin comment %v: %v", - isu.Token, err) - } - } - }() - p.eventManager._register(EventTypeInvoiceStatusUpdate, ch) -} - -func (p *politeiawww) _setupDCCNewEmailNotification() { - ch := make(chan interface{}) - go func() { - for data := range ch { - ic, ok := data.(EventDataDCCNew) - if !ok { - log.Errorf("invalid event data") - continue - } - - err := p.emailAdminsForNewDCC(ic.Token) - if err != nil { - log.Errorf("email all admins for new dcc %v: %v", - ic.Token, err) - } - } - }() - p.eventManager._register(EventTypeDCCNew, ch) -} - -func (p *politeiawww) _setupDCCSupportOpposeEmailNotification() { - ch := make(chan interface{}) - go func() { - for data := range ch { - ic, ok := data.(EventDataDCCSupportOppose) - if !ok { - log.Errorf("invalid event data") - continue - } - - err := p.emailAdminsForNewDCCSupportOppose(ic.Token) - if err != nil { - log.Errorf("email all admins for new dcc suppoort oppose %v: %v", - ic.Token, err) - } - } - }() - p.eventManager._register(EventTypeDCCSupportOppose, ch) -} - -func (p *politeiawww) _setupProposalStatusChangeEmailNotification() { - ch := make(chan interface{}) - go func() { - for data := range ch { - psc, ok := data.(EventDataProposalStatusChange) - if !ok { - log.Errorf("invalid event data") - continue - } - - if psc.SetProposalStatus.ProposalStatus != www.PropStatusPublic && - psc.SetProposalStatus.ProposalStatus != www.PropStatusCensored { - continue - } - - author, err := p.db.UserGetByPubKey(psc.Proposal.PublicKey) - if err != nil { - log.Errorf("cannot fetch author for proposal: %v", err) - continue - } - - switch psc.SetProposalStatus.ProposalStatus { - case www.PropStatusPublic: - err = p.emailAuthorForVettedProposal(psc.Proposal, author, - psc.AdminUser) - if err != nil { - log.Errorf("email author for vetted proposal %v: %v", - psc.Proposal.CensorshipRecord.Token, err) - } - err = p.emailUsersForVettedProposal(psc.Proposal, author, - psc.AdminUser) - if err != nil { - log.Errorf("email users for vetted proposal %v: %v", - psc.Proposal.CensorshipRecord.Token, err) - } - case www.PropStatusCensored: - err = p.emailAuthorForCensoredProposal(psc.Proposal, author, - psc.AdminUser) - if err != nil { - log.Errorf("email author for censored proposal %v: %v", - psc.Proposal.CensorshipRecord.Token, err) - } - default: - } - } - }() - p.eventManager._register(EventTypeProposalStatusChange, ch) -} - -func (p *politeiawww) _setupProposalStatusChangeLogging() { - ch := make(chan interface{}) - go func() { - for data := range ch { - psc, ok := data.(EventDataProposalStatusChange) - if !ok { - log.Errorf("invalid event data") - continue - } - - // Log the action in the admin log. - err := p.logAdminProposalAction(psc.AdminUser, - psc.Proposal.CensorshipRecord.Token, - fmt.Sprintf("set proposal status to %v", - www.PropStatus[psc.SetProposalStatus.ProposalStatus]), - psc.SetProposalStatus.StatusChangeMessage) - - if err != nil { - log.Errorf("could not log action to file: %v", err) - } - } - }() - p.eventManager._register(EventTypeProposalStatusChange, ch) -} - -func (p *politeiawww) _setupProposalEditedEmailNotification() { - ch := make(chan interface{}) - go func() { - for data := range ch { - pe, ok := data.(EventDataProposalEdited) - if !ok { - log.Errorf("invalid event data") - continue - } - - if pe.Proposal.Status != www.PropStatusPublic { - continue - } - - author, err := p.db.UserGetByPubKey(pe.Proposal.PublicKey) - if err != nil { - log.Errorf("cannot fetch author for proposal: %v", err) - continue - } - - err = p.emailUsersForEditedProposal(pe.Proposal, author) - if err != nil { - log.Errorf("email users for edited proposal %v: %v", - pe.Proposal.CensorshipRecord.Token, err) - } - } - }() - p.eventManager._register(EventTypeProposalEdited, ch) -} - -func (p *politeiawww) _setupProposalVoteStartedEmailNotification() { - ch := make(chan interface{}) - go func() { - for data := range ch { - pvs, ok := data.(EventDataProposalVoteStarted) - if !ok { - log.Errorf("invalid event data") - continue - } - - token := pvs.StartVote.Vote.Token - proposal, author, err := p.getProposalAndAuthor( - token) - if err != nil { - log.Error(err) - continue - } - - err = p.emailUsersForProposalVoteStarted(proposal, author, - pvs.AdminUser) - if err != nil { - log.Errorf("email all admins for new submitted proposal %v: %v", - token, err) - } - } - }() - p.eventManager._register(EventTypeProposalVoteStarted, ch) -} - -func (p *politeiawww) _setupProposalVoteStartedLogging() { - ch := make(chan interface{}) - go func() { - for data := range ch { - pvs, ok := data.(EventDataProposalVoteStarted) - if !ok { - log.Errorf("invalid event data") - continue - } - - // Log the action in the admin log. - err := p.logAdminProposalAction(pvs.AdminUser, - pvs.StartVote.Vote.Token, "start vote", "") - if err != nil { - log.Errorf("could not log action to file: %v", err) - } - } - }() - p.eventManager._register(EventTypeProposalVoteStarted, ch) -} - -func (p *politeiawww) _setupProposalVoteAuthorizedEmailNotification() { - ch := make(chan interface{}) - go func() { - for data := range ch { - pvs, ok := data.(eventDataProposalVoteAuthorized) - if !ok { - log.Errorf("invalid event data") - continue - } - - err := p.emailAdminsForProposalVoteAuthorized(pvs.token, pvs.name, - pvs.authorUsername, pvs.authorEmail) - if err != nil { - log.Errorf("email all admins for new submitted proposal %v: %v", - pvs.token, err) - } - } - }() - p.eventManager._register(EventTypeProposalVoteAuthorized, ch) -} - -func (p *politeiawww) _setupCommentReplyEmailNotifications() { - ch := make(chan interface{}) - go func() { - for data := range ch { - c, ok := data.(EventDataComment) - if !ok { - log.Errorf("invalid event data") - continue - } - - token := c.Comment.Token - proposal, author, err := p.getProposalAndAuthor(token) - if err != nil { - log.Error(err) - continue - } - - if c.Comment.ParentID == "0" { - // Top-level comment - err := p.emailAuthorForCommentOnProposal(proposal, author, - c.Comment.CommentID, c.Comment.Username) - if err != nil { - log.Errorf("email author of proposal %v for new comment %v: %v", - c.Comment.Token, c.Comment.CommentID, err) - } - } else { - parent, err := p.decredCommentGetByID(token, c.Comment.ParentID) - if err != nil { - log.Errorf("EventManager: getComment failed for token %v "+ - "commentID %v: %v", token, c.Comment.ParentID, err) - continue - } - - author, err := p.db.UserGetByPubKey(parent.PublicKey) - if err != nil { - log.Errorf("cannot fetch author for comment: %v", err) - continue - } - - // Comment reply to another comment - err = p.emailAuthorForCommentOnComment(proposal, author, - c.Comment.CommentID, c.Comment.Username) - if err != nil { - log.Errorf("email author of comment %v for new comment %v: %v", - c.Comment.CommentID, c.Comment.ParentID, err) - } - } - } - }() - p.eventManager._register(EventTypeComment, ch) -} - -func (p *politeiawww) _setupUserManageLogging() { - ch := make(chan interface{}) - go func() { - for data := range ch { - ue, ok := data.(EventDataUserManage) - if !ok { - log.Errorf("invalid event data") - continue - } - - // Log the action in the admin log. - err := p.logAdminUserAction(ue.AdminUser, ue.User, - ue.ManageUser.Action, ue.ManageUser.Reason) - if err != nil { - log.Errorf("could not log action to file: %v", err) - } - } - }() - p.eventManager._register(EventTypeUserManage, ch) -} - -// register adds a listener channel for the given event type. -// -// This function must be called WITH the mutex held. -func (e *eventManager) _register(eventType eventT, listenerToAdd chan interface{}) { - if e.listeners == nil { - e.listeners = make(map[eventT][]chan interface{}) - } - - e.listeners[eventType] = append(e.listeners[eventType], listenerToAdd) -} - -// _fireEvent iterates all listener channels for the given event type and -// passes the given data to it. -// -// This function must be called WITH the mutex held. -func (e *eventManager) _fireEvent(eventType eventT, data interface{}) { - listeners, ok := e.listeners[eventType] - if !ok { - return - } - - for _, listener := range listeners { - go func(listener chan interface{}) { - listener <- data - }(listener) - } -} diff --git a/politeiawww/invoices.go b/politeiawww/invoices.go index 2aa642263..9dcffaf39 100644 --- a/politeiawww/invoices.go +++ b/politeiawww/invoices.go @@ -1677,13 +1677,12 @@ func (p *politeiawww) processNewCommentInvoice(nc www.NewComment, u *user.User) return nil, fmt.Errorf("failed to get user by username %v %v", ir.Username, err) } - // Fire off new invoice comment event - p.fireEvent(EventTypeInvoiceComment, - EventDataInvoiceComment{ - Token: nc.Token, - User: invoiceUser, - }, - ) + // Emit event notification for a invoice comment + p.eventManager.emit(eventInvoiceComment, + dataInvoiceComment{ + token: nc.Token, + email: invoiceUser.Email, + }) } return &www.NewCommentReply{ Comment: *c, diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index d560bb680..62e422f27 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -926,6 +926,7 @@ func (p *politeiawww) processSetProposalStatus(sps www.SetProposalStatus, u *use // The only time admins are allowed to change the status of // their own proposals is on testnet + var author *user.User if !p.cfg.TestNet { author, err := p.db.UserGetByPubKey(pr.PublicKey) if err != nil { @@ -1069,14 +1070,23 @@ func (p *politeiawww) processSetProposalStatus(sps www.SetProposalStatus, u *use return nil, err } - // Fire off proposal status change event - p.fireEvent(EventTypeProposalStatusChange, - EventDataProposalStatusChange{ - Proposal: updatedProp, - AdminUser: u, - SetProposalStatus: &sps, - }, - ) + // Emit event notification for proposal status change + // TODO: updatedProp is a record from the old api, therefore we + // get a mismatch on the Status prop, that expects a PropStatusT. + // We need to update this call to use a proposal record from the + // new api. + p.eventManager.emit(eventProposalStatusChange, + dataProposalStatusChange{ + // status: updatedProp.Status, + name: updatedProp.Name, + token: updatedProp.CensorshipRecord.Token, + statusChangeMessage: updatedProp.StatusChangeMessage, + adminID: u.ID, + id: author.ID, + email: author.Email, + emailNotifications: author.EmailNotifications, + username: author.Username, + }) return &www.SetProposalStatusReply{ Proposal: *updatedProp, @@ -1847,14 +1857,14 @@ func (p *politeiawww) processAuthorizeVote(av www.AuthorizeVote, u *user.User) ( } if !p.test && avr.Action == decredplugin.AuthVoteActionAuthorize { - p.fireEvent(EventTypeProposalVoteAuthorized, - eventDataProposalVoteAuthorized{ - token: av.Token, - name: pr.Name, - authorUsername: u.Email, - authorEmail: u.Username, - }, - ) + // Emit event notification for proposal vote authorized + p.eventManager.emit(eventProposalVoteAuthorized, + dataProposalVoteAuthorized{ + token: av.Token, + name: pr.Name, + username: u.Username, + email: u.Email, + }) } return &www.AuthorizeVoteReply{ @@ -2169,13 +2179,23 @@ func (p *politeiawww) processStartVoteV2(sv www2.StartVote, u *user.User) (*www2 return nil, err } - // Fire off start vote event - p.fireEvent(EventTypeProposalVoteStarted, - EventDataProposalVoteStarted{ - AdminUser: u, - StartVote: sv, - }, - ) + // Get author data + author, err := p.db.UserGetByPubKey(pr.PublicKey) + if err != nil { + return nil, err + } + + // Emit event notification for proposal start vote + p.eventManager.emit(eventProposalVoteStarted, + dataProposalVoteStarted{ + token: pr.CensorshipRecord.Token, + name: pr.Name, + adminID: u.ID, + id: author.ID, + username: author.Username, + email: author.Email, + emailNotifications: author.EmailNotifications, + }) return svr, nil } @@ -2269,6 +2289,10 @@ func (p *politeiawww) processStartVoteRunoffV2(sv www2.StartVoteRunoff, u *user. } } + // Slice used to send event notification for each proposal + // starting a vote, at the end of this function. + var proposalNotifications []*www.ProposalRecord + // Validate authorize votes and start votes for _, v := range sv.StartVotes { // Fetch proposal and vote summary @@ -2292,6 +2316,9 @@ func (p *politeiawww) processStartVoteRunoffV2(sv www2.StartVoteRunoff, u *user. */ return nil, err } + + proposalNotifications = append(proposalNotifications, pr) + vs, err := p.voteSummaryGet(token, bb) if err != nil { return nil, err @@ -2456,14 +2483,22 @@ func (p *politeiawww) processStartVoteRunoffV2(sv www2.StartVoteRunoff, u *user. return nil, err } - // Fire off a start vote events for each rfp submission - for _, v := range sv.StartVotes { - p.fireEvent(EventTypeProposalVoteStarted, - EventDataProposalVoteStarted{ - AdminUser: u, - StartVote: v, - }, - ) + // Emit event notification for each proposal starting vote + for _, pn := range proposalNotifications { + author, err := p.db.UserGetByPubKey(pn.PublicKey) + if err != nil { + return nil, err + } + p.eventManager.emit(eventProposalVoteStarted, + dataProposalVoteStarted{ + token: pn.CensorshipRecord.Token, + name: pn.Name, + adminID: u.ID, + id: author.ID, + username: author.Username, + email: author.Email, + emailNotifications: author.EmailNotifications, + }) } return &www2.StartVoteRunoffReply{ diff --git a/politeiawww/user.go b/politeiawww/user.go index b0b205e4f..e34e17038 100644 --- a/politeiawww/user.go +++ b/politeiawww/user.go @@ -1750,14 +1750,6 @@ func (p *politeiawww) processManageUser(mu *www.ManageUser, adminUser *user.User return nil, err } - if !p.test { - p.fireEvent(EventTypeUserManage, EventDataUserManage{ - AdminUser: adminUser, - User: user, - ManageUser: mu, - }) - } - return &www.ManageUserReply{}, nil } diff --git a/politeiawww/www.go b/politeiawww/www.go index 3219b5a3b..5dc779ebc 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -746,13 +746,11 @@ func _main() error { if err != nil { return err } - - p.initEventManagerPi() + // Setup event manager p.setupEventListenersPi() case cmsWWWMode: // Setup event manager - p.initCMSEventManager() p.setupEventListenersCms() // Setup dcrdata websocket connection From 80c85fb3b1edb33a4aeb69997c220d5aa293f4ed Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 9 Sep 2020 17:34:55 -0500 Subject: [PATCH 038/449] proposal routes --- plugins/pi/pi.go | 50 ++- politeiad/api/v1/v1.go | 2 + politeiawww/api/pi/v1/v1.go | 42 +-- politeiawww/convert.go | 20 +- politeiawww/piwww.go | 643 +++++++++++++++++++++++++++++------- politeiawww/politeiad.go | 166 ++++++++++ politeiawww/politeiawww.go | 106 ------ politeiawww/proposals.go | 288 ---------------- politeiawww/user.go | 15 +- politeiawww/www.go | 10 +- 10 files changed, 771 insertions(+), 571 deletions(-) diff --git a/plugins/pi/pi.go b/plugins/pi/pi.go index 2562d257a..59d46890f 100644 --- a/plugins/pi/pi.go +++ b/plugins/pi/pi.go @@ -11,6 +11,7 @@ import ( "fmt" ) +type PropStateT int type PropStatusT int type ErrorStatusT int @@ -24,14 +25,19 @@ const ( // Metadata stream IDs. All metadata streams in this plugin will // use 1xx numbering. MDStreamIDProposalGeneral = 101 - MDStreamIDStatusChange = 102 + MDStreamIDStatusChanges = 102 - // FilenameProposalMetadata is the filename of the ProposalMetadata + // FileNameProposalMetadata is the filename of the ProposalMetadata // file that is saved to politeiad. ProposalMetadata is saved to // politeiad as a file, not as a metadata stream, since it contains // user provided metadata and needs to be included in the merkle // root that politeiad signs. - FilenameProposalMetadata = "proposalmetadata.json" + FileNameProposalMetadata = "proposalmetadata.json" + + // Proposal states + PropStateInvalid PropStateT = 0 + PropStateUnvetted PropStateT = 1 + PropStateVetted PropStateT = 2 // Proposal status codes PropStatusInvalid PropStatusT = 0 // Invalid status @@ -52,8 +58,14 @@ var ( // transitions. If StatusChanges[currentStatus][newStatus] exists // then the status change is allowed. StatusChanges = map[PropStatusT]map[PropStatusT]struct{}{ - PropStatusUnvetted: map[PropStatusT]struct{}{}, - PropStatusPublic: map[PropStatusT]struct{}{}, + PropStatusUnvetted: map[PropStatusT]struct{}{ + PropStatusPublic: struct{}{}, + PropStatusCensored: struct{}{}, + }, + PropStatusPublic: map[PropStatusT]struct{}{ + PropStatusAbandoned: struct{}{}, + PropStatusCensored: struct{}{}, + }, PropStatusCensored: map[PropStatusT]struct{}{}, PropStatusAbandoned: map[PropStatusT]struct{}{}, } @@ -67,6 +79,7 @@ var ( } ) +// TODO change this to UserErrorReply as well as in all the other plugins. // UserError represents an error that is caused by the user. type UserError struct { ErrorCode ErrorStatusT @@ -138,11 +151,32 @@ func DecodeProposalGeneral(payload []byte) (*ProposalGeneral, error) { return &pg, nil } +// StatusChange represents a proposal status change. +// +// Signature is the client signature of the Token+Version+Status+Reason. +type StatusChange struct { + Token string `json:"token"` + Version string `json:"version"` + Status PropStatusT `json:"status"` + Reason string `json:"message,omitempty"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` + Timestamp int64 `json:"timestamp"` +} + +// EncodeStatusChange encodes a StatusChange into a JSON byte slice. +func EncodeStatusChange(sc StatusChange) ([]byte, error) { + return json.Marshal(sc) +} + +// TODO DecodeStatusChanges + // Proposals requests the plugin data for the provided proposals. This includes -// pi plugin data as well as other plugin data such as comments plugin data. +// pi plugin data as well as other plugin data such as comment plugin data. // This command aggregates all proposal plugin data into a single call. type Proposals struct { - Tokens []string `json:"tokens"` + State PropStateT `json:"state"` + Tokens []string `json:"tokens"` } // EncodeProposals encodes a Proposals into a JSON byte slice. @@ -160,7 +194,7 @@ func DecodeProposals(payload []byte) (*Proposals, error) { return &p, nil } -// ProposalPluginData represents the plugin data of a proposal. +// ProposalPluginData contains all the plugin data for a proposal. type ProposalPluginData struct { Comments uint64 `json:"comments"` // Number of comments LinkedFrom []string `json:"linkedfrom"` // Linked from list diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index 414f880dc..d439b7435 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -242,9 +242,11 @@ type NewRecordReply struct { } // GetUnvetted requests an unvetted record from the server. +// TODO Implement Version. Unvetted didn't previously have a version. type GetUnvetted struct { Challenge string `json:"challenge"` // Random challenge Token string `json:"token"` // Censorship token + Version string `json:"version"` // Record version } // GetUnvettedReply returns an unvetted record. It retrieves the censorship diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 0a4613957..b7dd21256 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -88,6 +88,7 @@ const ( ErrorStatusUserRegistrationNotPaid ErrorStatusT = 1 ErrorStatusUserBalanceInsufficient ErrorStatusT = 2 ErrorStatusUserIsNotAuthor ErrorStatusT = 3 + ErrorStatusUserIsNotAdmin ErrorStatusT = 4 // Signature errors ErrorStatusPublicKeyInvalid ErrorStatusT = 100 @@ -119,6 +120,7 @@ const ( ErrorStatusPropStateInvalid ErrorStatusPropStatusInvalid ErrorStatusPropStatusChangeInvalid + ErrorStatusPropStatusChangeReasonInvalid // Comment errors ErrorStatusCommentTextInvalid @@ -144,28 +146,29 @@ var ( } ) -// UserError represents an error that is caused by something that the user -// did (malformed input, bad timing, etc). -type UserError struct { +// UserErrorReply is the reply that the server returns when it encounters an +// error that is caused by something that the user did (malformed input, bad +// timing, etc). The HTTP status code will be 400. +type UserErrorReply struct { ErrorCode ErrorStatusT ErrorContext []string } // Error satisfies the error interface. -func (e UserError) Error() string { +func (e UserErrorReply) Error() string { return fmt.Sprintf("user error code: %v", e.ErrorCode) } -// ErrorReply are replies that the server returns when it encounters an -// unrecoverable problem while executing a command. The HTTP status code will -// be 500 and the ErrorCode field will contain a UNIX timestamp that the user -// can provide to the server admin to track down the error details in the logs. -type ErrorReply struct { +// ServerErrorReply is the reply that the server returns when it encounters an +// unrecoverable error while executing a command. The HTTP status code will be +// 500 and the ErrorCode field will contain a UNIX timestamp that the user can +// provide to the server admin to track down the error details in the logs. +type ServerErrorReply struct { ErrorCode int64 `json:"errorcode"` } // Error satisfies the error interface. -func (e ErrorReply) Error() string { +func (e ServerErrorReply) Error() string { return fmt.Sprintf("server error: %v", e.ErrorCode) } @@ -220,7 +223,7 @@ type CensorshipRecord struct { Signature string `json:"signature"` } -// SetStatus represents a proposal status change. +// StatusChange represents a proposal status change. // // Signature is the client signature of the Token+Version+Status+Reason. type StatusChange struct { @@ -325,21 +328,22 @@ type ProposalSetStatusReply struct { // proposal token and version. If the version is omitted, the most recent // version will be returned. type ProposalRequest struct { - Token string `json:"token"` - State PropStateT `json:"state"` - Version string `json:"version,omitempty"` - IncludeFiles bool `json:"includefiles,omitempty"` + Token string `json:"token"` + Version string `json:"version,omitempty"` } // Proposals retrieves the ProposalRecord for each of the provided proposal -// requests. +// requests. Unvetted proposal files are only returned to admins. type Proposals struct { - Requests []ProposalRequest `json:"requests"` + State PropStateT `json:"state"` + Requests []ProposalRequest `json:"requests"` + IncludeFiles bool `json:"includefiles,omitempty"` } -// ProposalsReply is the reply to the Proposals command. +// ProposalsReply is the reply to the Proposals command. Any tokens that did +// not correspond to a ProposalRecord will not be included in the reply. type ProposalsReply struct { - Proposals []ProposalRecord `json:"proposals"` + Proposals map[string]ProposalRecord `json:"proposals"` // [token]Proposal } // ProposalInventry retrieves the tokens of all proposals in the inventory, diff --git a/politeiawww/convert.go b/politeiawww/convert.go index 725e178ba..41423ab9a 100644 --- a/politeiawww/convert.go +++ b/politeiawww/convert.go @@ -23,6 +23,8 @@ import ( "github.com/decred/politeia/politeiawww/cmsdatabase" ) +// TODO cleanup all unused convert functions + func convertCastVoteReplyFromDecredPlugin(cvr decredplugin.CastVoteReply) www.CastVoteReply { return www.CastVoteReply{ ClientSignature: cvr.ClientSignature, @@ -246,24 +248,6 @@ func convertPropFilesFromWWW(f []www.File) []pd.File { return files } -func convertPropStatusFromPD(s pd.RecordStatusT) www.PropStatusT { - switch s { - case pd.RecordStatusNotFound: - return www.PropStatusNotFound - case pd.RecordStatusNotReviewed: - return www.PropStatusNotReviewed - case pd.RecordStatusCensored: - return www.PropStatusCensored - case pd.RecordStatusPublic: - return www.PropStatusPublic - case pd.RecordStatusUnreviewedChanges: - return www.PropStatusUnreviewedChanges - case pd.RecordStatusArchived: - return www.PropStatusAbandoned - } - return www.PropStatusInvalid -} - func convertPropCensorFromPD(f pd.CensorshipRecord) www.CensorshipRecord { return www.CensorshipRecord{ Token: f.Token, diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 0019c130f..8a3b1dfe2 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -36,11 +36,24 @@ const ( ) var ( - validProposalName = regexp.MustCompile(createProposalNameRegex()) + // validProposalName contains the regex that matches a valid + // proposal name. + validProposalName = regexp.MustCompile(proposalNameRegex()) + + // statusReasonRequired contains the list of proposal statuses that + // require an accompanying reason to be given for the status change. + statusReasonRequired = map[pi.PropStatusT]struct{}{ + pi.PropStatusCensored: struct{}{}, + pi.PropStatusAbandoned: struct{}{}, + } ) // tokenIsValid returns whether the provided string is a valid politeiad -// censorship record token. +// censorship record token. This CAN BE EITHER the short token or the full +// length token. +// +// Short tokens should only be used when retrieving data. Data that is written +// to disk should always reference the full length token. func tokenIsValid(token string) bool { b, err := hex.DecodeString(token) if err != nil { @@ -52,15 +65,28 @@ func tokenIsValid(token string) bool { return true } +// tokenIsFullLength returns whether the provided string a is valid, full +// length politeiad censorship record token. Short tokens are considered +// invalid by this function. +func tokenIsFullLength(token string) bool { + b, err := hex.DecodeString(token) + if err != nil { + return false + } + if len(b) != pd.TokenSize { + return false + } + return true +} + // proposalNameIsValid returns whether the provided name is a valid proposal // name. func proposalNameIsValid(name string) bool { return validProposalName.MatchString(name) } -// createProposalNameRegex returns a regex string for validating the proposal -// name. -func createProposalNameRegex() string { +// proposalNameRegex returns a regex string for validating the proposal name. +func proposalNameRegex() string { var validProposalNameBuffer bytes.Buffer validProposalNameBuffer.WriteString("^[") @@ -80,7 +106,7 @@ func createProposalNameRegex() string { return validProposalNameBuffer.String() } -func convertUserErrFromSignatureErr(err error) pi.UserError { +func convertUserErrFromSignatureErr(err error) pi.UserErrorReply { var e util.SignatureError var s pi.ErrorStatusT if errors.As(err, &e) { @@ -91,17 +117,51 @@ func convertUserErrFromSignatureErr(err error) pi.UserError { s = pi.ErrorStatusSignatureInvalid } } - return pi.UserError{ + return pi.UserErrorReply{ ErrorCode: s, ErrorContext: e.ErrorContext, } } +func convertPropStateFromPropStatus(s pi.PropStatusT) pi.PropStateT { + switch s { + case pi.PropStatusUnvetted, pi.PropStatusCensored: + return pi.PropStateUnvetted + case pi.PropStatusPublic, pi.PropStatusAbandoned: + return pi.PropStateVetted + } + return pi.PropStateInvalid +} + +func convertPropStateFromPi(s pi.PropStateT) piplugin.PropStateT { + switch s { + case pi.PropStateUnvetted: + return piplugin.PropStateUnvetted + case pi.PropStateVetted: + return piplugin.PropStateVetted + } + return piplugin.PropStateInvalid +} + +func convertRecordStatusFromPropStatus(s pi.PropStatusT) pd.RecordStatusT { + switch s { + case pi.PropStatusUnvetted: + return pd.RecordStatusNotReviewed + case pi.PropStatusPublic: + return pd.RecordStatusPublic + case pi.PropStatusCensored: + return pd.RecordStatusCensored + case pi.PropStatusAbandoned: + return pd.RecordStatusArchived + } + return pd.RecordStatusInvalid +} + func convertFileFromMetadata(m pi.Metadata) pd.File { var name string switch m.Hint { case pi.HintProposalMetadata: - name = piplugin.FilenameProposalMetadata + name = piplugin.FileNameProposalMetadata } return pd.File{ Name: name, @@ -128,6 +188,24 @@ func convertFilesFromPi(files []pi.File) []pd.File { return f } +func convertPropStatusFromPD(s pd.RecordStatusT) pi.PropStatusT { + switch s { + case pd.RecordStatusNotFound: + // Intentionally omitted. No corresponding PropStatusT. + case pd.RecordStatusNotReviewed: + return pi.PropStatusUnvetted + case pd.RecordStatusCensored: + return pi.PropStatusCensored + case pd.RecordStatusPublic: + return pi.PropStatusPublic + case pd.RecordStatusUnreviewedChanges: + return pi.PropStatusUnvetted + case pd.RecordStatusArchived: + return pi.PropStatusAbandoned + } + return pi.PropStatusInvalid +} + func convertCensorshipRecordFromPD(cr pd.CensorshipRecord) pi.CensorshipRecord { return pi.CensorshipRecord{ Token: cr.Token, @@ -136,6 +214,74 @@ func convertCensorshipRecordFromPD(cr pd.CensorshipRecord) pi.CensorshipRecord { } } +func convertFilesFromPD(f []pd.File) ([]pi.File, []pi.Metadata) { + files := make([]pi.File, 0, len(f)) + metadata := make([]pi.Metadata, 0, len(f)) + for _, v := range f { + switch v.Name { + case piplugin.FileNameProposalMetadata: + metadata = append(metadata, pi.Metadata{ + Hint: pi.HintProposalMetadata, + Digest: v.Digest, + Payload: v.Payload, + }) + default: + files = append(files, pi.File{ + Name: v.Name, + MIME: v.MIME, + Digest: v.Digest, + Payload: v.Payload, + }) + } + } + return files, metadata +} + +func convertProposalRecordFromPD(r pd.Record) (*pi.ProposalRecord, error) { + // Decode metadata streams + var ( + pg *piplugin.ProposalGeneral + statuses = make([]pi.StatusChange, 0, 16) + err error + ) + for _, v := range r.Metadata { + switch v.ID { + case piplugin.MDStreamIDProposalGeneral: + pg, err = piplugin.DecodeProposalGeneral([]byte(v.Payload)) + if err != nil { + return nil, err + } + case piplugin.MDStreamIDStatusChanges: + // TODO decode status changes + } + } + + // Convert files and status + files, metadata := convertFilesFromPD(r.Files) + status := convertPropStatusFromPD(r.Status) + state := convertPropStateFromPropStatus(status) + + // Some fields are intentionally omitted because they are either + // user data that needs to be pulled from the user database or they + // are plugin data that needs to be retrieved using a plugin cmd. + return &pi.ProposalRecord{ + Version: r.Version, + Timestamp: pg.Timestamp, + State: state, + Status: status, + UserID: "", // Intentionally omitted + Username: "", // Intentionally omitted + PublicKey: pg.PublicKey, + Signature: pg.Signature, + Comments: 0, // Intentionally omitted + Statuses: statuses, + Files: files, + Metadata: metadata, + LinkedFrom: []string{}, // Intentionally omitted + CensorshipRecord: convertCensorshipRecordFromPD(r.CensorshipRecord), + }, nil +} + // linkByPeriodMin returns the minimum amount of time, in seconds, that the // LinkBy period must be set to. This is determined by adding 1 week onto the // minimum voting period so that RFP proposal submissions have at least one @@ -164,13 +310,10 @@ func (p *politeiawww) linkByPeriodMax() int64 { return 7776000 // 3 months in seconds } -// proposalsPluginData fetches the plugin data for the provided proposals using -// the pi proposals plugin command. -func (p *politeiawww) proposalsPluginData(tokens []string) (map[string]piplugin.ProposalPluginData, error) { +// proposalPluginData fetches the plugin data for the provided proposals using +// the pi plugin proposals command. +func (p *politeiawww) proposalPluginData(ps piplugin.Proposals) (*piplugin.ProposalsReply, error) { // Setup plugin command - ps := piplugin.Proposals{ - Tokens: tokens, - } payload, err := piplugin.EncodeProposals(ps) if err != nil { return nil, err @@ -205,25 +348,50 @@ func (p *politeiawww) proposalsPluginData(tokens []string) (map[string]piplugin. return nil, err } - return pr.Proposals, nil + return pr, nil } -// proposalRecord returns the proposal record for the provided token. -func (p *politeiawww) proposalRecord(token string) (*pi.ProposalRecord, error) { - // TODO Get politeiad record - var pr *pi.ProposalRecord +// proposalRecord returns the proposal record for the provided token and +// version. +func (p *politeiawww) proposalRecord(state pi.PropStateT, token, version string) (*pi.ProposalRecord, error) { + // Get politeiad record + var r *pd.Record + var err error + switch state { + case pi.PropStateUnvetted: + r, err = p.getUnvetted(token, version) + if err != nil { + return nil, err + } + case pi.PropStateVetted: + r, err = p.getVetted(token, version) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unknown state %v", state) + } + + pr, err := convertProposalRecordFromPD(*r) + if err != nil { + return nil, err + } // Get proposal plugin data - ppd, err := p.proposalsPluginData([]string{token}) + ps := piplugin.Proposals{ + State: convertPropStateFromPi(state), + Tokens: []string{token}, + } + psr, err := p.proposalPluginData(ps) if err != nil { return nil, err } - pd, ok := ppd[token] + d, ok := psr.Proposals[token] if !ok { return nil, fmt.Errorf("proposal plugin data not found %v", token) } - pr.Comments = pd.Comments - pr.LinkedFrom = pd.LinkedFrom + pr.Comments = d.Comments + pr.LinkedFrom = d.LinkedFrom // Get user data u, err := p.db.UserGetByPubKey(pr.PublicKey) @@ -233,31 +401,124 @@ func (p *politeiawww) proposalRecord(token string) (*pi.ProposalRecord, error) { pr.UserID = u.ID.String() pr.Username = u.Username - return nil, nil + return pr, nil } -func (p *politeiawww) proposalRecords(tokens []string) (map[string]pi.ProposalRecord, error) { +// proposalRecordLatest returns the latest version of the proposal record for +// the provided token. +func (p *politeiawww) proposalRecordLatest(state pi.PropStateT, token string) (*pi.ProposalRecord, error) { + return p.proposalRecord(state, token, "") +} + +// proposalRecords returns the ProposalRecord for each of the provided proposal +// requests. If a token does not correspond to an actual proposal then it will +// not be included in the returned map. +// +// XXX politeiad needs batched calls for retrieving unvetted and vetted +// records. This call should have an includeFiles option. +func (p *politeiawww) proposalRecords(state pi.PropStateT, reqs []pi.ProposalRequest, includeFiles bool) (map[string]pi.ProposalRecord, error) { // Get politeiad records - // Get proposals plugin data + props := make([]pi.ProposalRecord, 0, len(reqs)) + for _, v := range reqs { + var r *pd.Record + var err error + switch state { + case pi.PropStateUnvetted: + // Unvetted politeiad record + r, err = p.getUnvetted(v.Token, v.Version) + if err != nil { + return nil, fmt.Errorf("getUnvetted %v: %v", v.Token, err) + } + case pi.PropStateVetted: + // Vetted politeiad record + r, err = p.getVetted(v.Token, v.Version) + if err != nil { + return nil, fmt.Errorf("getVetted %v: %v", v.Token, err) + } + default: + return nil, fmt.Errorf("unknown state %v", state) + } + + pr, err := convertProposalRecordFromPD(*r) + if err != nil { + return nil, fmt.Errorf("convertProposalRecordFromPD %v: %v", + v.Token, err) + } + + // Remove files if specified + if !includeFiles { + pr.Files = []pi.File{} + pr.Metadata = []pi.Metadata{} + } + + props = append(props, *pr) + } + + // Get proposal plugin data + tokens := make([]string, 0, len(reqs)) + for _, v := range reqs { + tokens = append(tokens, v.Token) + } + ps := piplugin.Proposals{ + State: convertPropStateFromPi(state), + Tokens: tokens, + } + psr, err := p.proposalPluginData(ps) + if err != nil { + return nil, fmt.Errorf("proposalPluginData: %v", err) + } + for k, v := range props { + token := v.CensorshipRecord.Token + d, ok := psr.Proposals[token] + if !ok { + return nil, fmt.Errorf("proposal plugin data not found %v", token) + } + props[k].Comments = d.Comments + props[k].LinkedFrom = d.LinkedFrom + } // Get user data + pubkeys := make([]string, 0, len(props)) + for _, v := range props { + pubkeys = append(pubkeys, v.PublicKey) + } + ur, err := p.db.UsersGetByPubKey(pubkeys) + if err != nil { + return nil, err + } + for k, v := range props { + token := v.CensorshipRecord.Token + u, ok := ur[v.PublicKey] + if !ok { + return nil, fmt.Errorf("user not found for pubkey %v from proposal %v", + v.PublicKey, token) + } + props[k].UserID = u.ID.String() + props[k].Username = u.Username + } + + // Convert proposals to a map + proposals := make(map[string]pi.ProposalRecord, len(props)) + for _, v := range props { + proposals[v.CensorshipRecord.Token] = v + } - return nil, nil + return proposals, nil } func (p *politeiawww) verifyProposalMetadata(pm pi.ProposalMetadata) error { // Verify name if !proposalNameIsValid(pm.Name) { - return pi.UserError{ + return pi.UserErrorReply{ ErrorCode: pi.ErrorStatusPropNameInvalid, - ErrorContext: []string{createProposalNameRegex()}, + ErrorContext: []string{proposalNameRegex()}, } } // Verify linkto if pm.LinkTo != "" { - if !tokenIsValid(pm.LinkTo) { - return pi.UserError{ + if !tokenIsFullLength(pm.LinkTo) { + return pi.UserErrorReply{ ErrorCode: pi.ErrorStatusPropLinkToInvalid, ErrorContext: []string{"invalid token"}, } @@ -272,14 +533,14 @@ func (p *politeiawww) verifyProposalMetadata(pm pi.ProposalMetadata) error { case pm.LinkBy < min: e := fmt.Sprintf("linkby %v is less than min required of %v", pm.LinkBy, min) - return pi.UserError{ + return pi.UserErrorReply{ ErrorCode: pi.ErrorStatusPropLinkByInvalid, ErrorContext: []string{e}, } case pm.LinkBy > max: e := fmt.Sprintf("linkby %v is more than max allowed of %v", pm.LinkBy, max) - return pi.UserError{ + return pi.UserErrorReply{ ErrorCode: pi.ErrorStatusPropLinkByInvalid, ErrorContext: []string{e}, } @@ -291,7 +552,7 @@ func (p *politeiawww) verifyProposalMetadata(pm pi.ProposalMetadata) error { func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, publicKey, signature string) (*pi.ProposalMetadata, error) { if len(files) == 0 { - return nil, pi.UserError{ + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusFileCountInvalid, ErrorContext: []string{"no files found"}, } @@ -309,7 +570,7 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu _, ok := filenames[v.Name] if ok { e := fmt.Sprintf("duplicate name %v", v.Name) - return nil, pi.UserError{ + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusFileNameInvalid, ErrorContext: []string{e}, } @@ -319,7 +580,7 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu // Validate file payload if v.Payload == "" { e := fmt.Sprintf("file %v empty payload", v.Name) - return nil, pi.UserError{ + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusFilePayloadInvalid, ErrorContext: []string{e}, } @@ -327,7 +588,7 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu payloadb, err := base64.StdEncoding.DecodeString(v.Payload) if err != nil { e := fmt.Sprintf("file %v invalid base64", v.Name) - return nil, pi.UserError{ + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusFilePayloadInvalid, ErrorContext: []string{e}, } @@ -337,7 +598,7 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu digest := util.Digest(payloadb) d, ok := util.ConvertDigest(v.Digest) if !ok { - return nil, pi.UserError{ + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusFileDigestInvalid, ErrorContext: []string{v.Name}, } @@ -345,7 +606,7 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu if !bytes.Equal(digest, d[:]) { e := fmt.Sprintf("file %v digest got %v, want %x", v.Name, v.Digest, digest) - return nil, pi.UserError{ + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusFileDigestInvalid, ErrorContext: []string{e}, } @@ -360,7 +621,7 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu mimeFile, _, err := mime.ParseMediaType(v.MIME) if err != nil { e := fmt.Sprintf("file %v mime '%v' not parsable", v.Name, v.MIME) - return nil, pi.UserError{ + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusFileMIMEInvalid, ErrorContext: []string{e}, } @@ -368,7 +629,7 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu if mimeFile != mimePayload { e := fmt.Sprintf("file %v mime got %v, want %v", v.Name, mimeFile, mimePayload) - return nil, pi.UserError{ + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusFileMIMEInvalid, ErrorContext: []string{e}, } @@ -383,7 +644,7 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu if len(payloadb) > www.PolicyMaxMDSize { e := fmt.Sprintf("file %v size %v exceeds max size %v", v.Name, len(payloadb), www.PolicyMaxMDSize) - return nil, pi.UserError{ + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusIndexFileSizeInvalid, ErrorContext: []string{e}, } @@ -393,7 +654,7 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu // file. if v.Name != www.PolicyIndexFilename { e := fmt.Sprint("want %v, got %v", www.PolicyIndexFilename, v.Name) - return nil, pi.UserError{ + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusIndexFileNameInvalid, ErrorContext: []string{e}, } @@ -401,7 +662,7 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu if foundIndexFile { e := fmt.Sprintf("more than one %v file found", www.PolicyIndexFilename) - return nil, pi.UserError{ + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusIndexFileCountInvalid, ErrorContext: []string{e}, } @@ -417,14 +678,14 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu if len(payloadb) > www.PolicyMaxImageSize { e := fmt.Sprintf("image %v size %v exceeds max size %v", v.Name, len(payloadb), www.PolicyMaxImageSize) - return nil, pi.UserError{ + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusImageFileSizeInvalid, ErrorContext: []string{e}, } } default: - return nil, pi.UserError{ + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusFileMIMEInvalid, ErrorContext: []string{v.MIME}, } @@ -434,7 +695,7 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu // Verify that an index file is present if !foundIndexFile { e := fmt.Sprintf("%v file not found", www.PolicyIndexFilename) - return nil, pi.UserError{ + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusIndexFileCountInvalid, ErrorContext: []string{e}, } @@ -444,7 +705,7 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu if countTextFiles > www.PolicyMaxMDs { e := fmt.Sprintf("got %v text files, max is %v", countTextFiles, www.PolicyMaxMDs) - return nil, pi.UserError{ + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusTextFileCountInvalid, ErrorContext: []string{e}, } @@ -452,7 +713,7 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu if countImageFiles > www.PolicyMaxImages { e := fmt.Sprintf("got %v image files, max is %v", countImageFiles, www.PolicyMaxImages) - return nil, pi.UserError{ + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusImageFileCountInvalid, ErrorContext: []string{e}, } @@ -464,14 +725,14 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu case len(metadata) == 0: e := fmt.Sprintf("metadata with hint %v not found", www.HintProposalMetadata) - return nil, pi.UserError{ + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusMetadataCountInvalid, ErrorContext: []string{e}, } case len(metadata) > 1: e := fmt.Sprintf("metadata should only contain %v", www.HintProposalMetadata) - return nil, pi.UserError{ + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusMetadataCountInvalid, ErrorContext: []string{e}, } @@ -479,7 +740,7 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu md := metadata[0] if md.Hint != www.HintProposalMetadata { e := fmt.Sprintf("unknown metadata hint %v", md.Hint) - return nil, pi.UserError{ + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusMetadataHintInvalid, ErrorContext: []string{e}, } @@ -489,7 +750,7 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu b, err := base64.StdEncoding.DecodeString(md.Payload) if err != nil { e := fmt.Sprintf("metadata with hint %v invalid base64 payload", md.Hint) - return nil, pi.UserError{ + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusMetadataPayloadInvalid, ErrorContext: []string{e}, } @@ -498,7 +759,7 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu if md.Digest != hex.EncodeToString(digest) { e := fmt.Sprintf("metadata with hint %v got digest %v, want %v", md.Hint, md.Digest, hex.EncodeToString(digest)) - return nil, pi.UserError{ + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusMetadataDigestInvalid, ErrorContext: []string{e}, } @@ -511,7 +772,7 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu err = d.Decode(&pm) if err != nil { e := fmt.Sprintf("unable to decode %v payload", md.Hint) - return nil, pi.UserError{ + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusMetadataPayloadInvalid, ErrorContext: []string{e}, } @@ -541,21 +802,21 @@ func (p *politeiawww) processProposalNew(pn pi.ProposalNew, usr user.User) (*pi. // Verify user has paid registration paywall if !p.userHasPaid(usr) { - return nil, pi.UserError{ + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusUserRegistrationNotPaid, } } // Verify user has a proposal credit if !p.userHasProposalCredits(usr) { - return nil, pi.UserError{ + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusUserBalanceInsufficient, } } // Verify user signed using active identity if usr.PublicKey() != pn.PublicKey { - return nil, pi.UserError{ + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusPublicKeyInvalid, ErrorContext: []string{"not user's active identity"}, } @@ -599,29 +860,11 @@ func (p *politeiawww) processProposalNew(pn pi.ProposalNew, usr user.User) (*pi. } // Send politeiad request - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - nr := pd.NewRecord{ - Challenge: hex.EncodeToString(challenge), - Metadata: metadata, - Files: files, - } - resBody, err := p.makeRequest(http.MethodPost, pd.NewRecordRoute, nr) + dcr, err := p.newRecord(metadata, files) if err != nil { return nil, err } - var nrr pd.NewRecordReply - err = json.Unmarshal(resBody, &nrr) - if err != nil { - return nil, err - } - err = util.VerifyChallenge(p.cfg.Identity, challenge, nrr.Response) - if err != nil { - return nil, err - } - cr := convertCensorshipRecordFromPD(nrr.CensorshipRecord) + cr := convertCensorshipRecordFromPD(*dcr) // Deduct proposal credit from author's account err = p.spendProposalCredit(&usr, cr.Token) @@ -671,11 +914,20 @@ func filesToDel(current []pi.File, updated []pi.File) []string { func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*pi.ProposalEditReply, error) { log.Tracef("processProposalEdit: %v", pe.Token) - // Verify user signed using active identity - if usr.PublicKey() != pe.PublicKey { - return nil, pi.UserError{ - ErrorCode: pi.ErrorStatusPublicKeyInvalid, - ErrorContext: []string{"not user's active identity"}, + // Verify token + if !tokenIsFullLength(pe.Token) { + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusPropTokenInvalid, + } + } + + // Verify state + switch pe.State { + case pi.PropStateUnvetted, pi.PropStateVetted: + // Allowed; continue + default: + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusPropStateInvalid, } } @@ -686,40 +938,49 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p return nil, err } - // Get the current proposal - if !tokenIsValid(pe.Token) { - return nil, pi.UserError{ - ErrorCode: pi.ErrorStatusPropTokenInvalid, - ErrorContext: []string{pe.Token}, + // Verify user signed using active identity + if usr.PublicKey() != pe.PublicKey { + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusPublicKeyInvalid, + ErrorContext: []string{"not user's active identity"}, } } - curr, err := p.proposalRecord(pe.Token) + + // Get the current proposal + curr, err := p.proposalRecordLatest(pe.State, pe.Token) if err != nil { - // TODO pi.ErrorStatusProposalNotFound + // TODO pi.ErrorStatusPropNotFound return nil, err } // Verify the user is the author. The public keys are not static // values so the user IDs must be compared directly. if curr.UserID != usr.ID.String() { - return nil, pi.UserError{ + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusUserIsNotAuthor, } } + // Verification that requires retrieving the existing proposal is + // done in the politeiad pi plugin hook. This includes: + // -Verify proposal status + // -Verify vote status + // -Verify linkto + // Setup politeiad files. The Metadata objects are converted to // politeiad files instead of metadata streams since they contain // user defined data that needs to be included in the merkle root // that politeiad signs. - files := convertFilesFromPi(pe.Files) + filesAdd := convertFilesFromPi(pe.Files) for _, v := range pe.Metadata { switch v.Hint { case pi.HintProposalMetadata: - files = append(files, convertFileFromMetadata(v)) + filesAdd = append(filesAdd, convertFileFromMetadata(v)) } } + filesDel := filesToDel(curr.Files, pe.Files) - // Setup metadata stream + // Setup politeiad metadata timestamp := time.Now().Unix() pg := piplugin.ProposalGeneral{ PublicKey: pe.PublicKey, @@ -730,63 +991,197 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p if err != nil { return nil, err } - metadata := []pd.MetadataStream{ + mdOverwrite := []pd.MetadataStream{ { ID: piplugin.MDStreamIDProposalGeneral, Payload: string(b), }, } + mdAppend := []pd.MetadataStream{} - // Setup politeiad request - var route string + // Send politeiad request + // TODO verify that this will throw an error if no proposal files + // were changed. switch pe.State { case pi.PropStateUnvetted: - route = pd.UpdateUnvettedRoute + err = p.updateUnvetted(pe.Token, mdAppend, mdOverwrite, + filesAdd, filesDel) + if err != nil { + return nil, err + } case pi.PropStateVetted: - route = pd.UpdateVettedRoute + err = p.updateVetted(pe.Token, mdAppend, mdOverwrite, + filesAdd, filesDel) + if err != nil { + return nil, err + } default: - return nil, pi.UserError{ + return nil, fmt.Errorf("unknown state %v", pe.State) + } + + // TODO Emit an edit proposal event + + log.Infof("Proposal edited: %v %v", pe.Token, pm.Name) + for k, f := range pe.Files { + log.Infof("%02v: %v %v", k, f.Name, f.Digest) + } + + return &pi.ProposalEditReply{ + // TODO CensorshipRecord: cr, + Timestamp: timestamp, + }, nil +} + +func (p *politeiawww) processProposalSetStatus(pss pi.ProposalSetStatus, usr user.User) (*pi.ProposalSetStatusReply, error) { + log.Tracef("processProposalSetStatus: %v %v", pss.Token, pss.Status) + + // Verify token + if !tokenIsFullLength(pss.Token) { + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusPropTokenInvalid, + } + } + + // Verify state + switch pss.State { + case pi.PropStateUnvetted, pi.PropStateVetted: + // Allowed; continue + default: + return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusPropStateInvalid, } } - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err + + // Verify reason + _, required := statusReasonRequired[pss.Status] + if required && pss.Reason == "" { + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusPropStatusChangeReasonInvalid, + ErrorContext: []string{"reason not given"}, + } } - ur := pd.UpdateRecord{ - Token: pe.Token, - Challenge: hex.EncodeToString(challenge), - MDOverwrite: metadata, - FilesAdd: files, - FilesDel: filesToDel(curr.Files, pe.Files), + + // Verify user is an admin + if !usr.Admin { + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusUserIsNotAdmin, + } } - // Send politeiad request - resBody, err := p.makeRequest(http.MethodPost, route, ur) + // Verify user signed with their active identity + if usr.PublicKey() != pss.PublicKey { + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusPublicKeyInvalid, + ErrorContext: []string{"not user's active identity"}, + } + } + + // Verify signature + msg := pss.Token + pss.Version + strconv.Itoa(int(pss.Status)) + pss.Reason + err := util.VerifySignature(pss.Signature, pss.PublicKey, msg) if err != nil { - // TODO verify that this will throw an error if no proposal files - // were changed. - return nil, err + return nil, convertUserErrFromSignatureErr(err) } - var urr pd.UpdateRecordReply - err = json.Unmarshal(resBody, &urr) + + // TODO + // Verification that requires retrieving the existing proposal is + // done in the politeiad pi plugin hook. This includes: + // -Verify token corresponds to a proposal + // -Verify proposal state is correct + // -Verify version is the latest version + // -Verify status change is allowed + + // Setup metadata + timestamp := time.Now().Unix() + sc := piplugin.StatusChange{ + Token: pss.Token, + Version: pss.Version, + Status: piplugin.PropStatusT(pss.Status), + Reason: pss.Reason, + PublicKey: pss.PublicKey, + Signature: pss.Signature, + Timestamp: timestamp, + } + b, err := piplugin.EncodeStatusChange(sc) if err != nil { return nil, err } - err = util.VerifyChallenge(p.cfg.Identity, challenge, urr.Response) + mdAppend := []pd.MetadataStream{ + { + ID: piplugin.MDStreamIDStatusChanges, + Payload: string(b), + }, + } + mdOverwrite := []pd.MetadataStream{} + + // Send politeiad request + status := convertRecordStatusFromPropStatus(pss.Status) + switch pss.State { + case pi.PropStateUnvetted: + err = p.setUnvettedStatus(pss.Token, status, mdAppend, mdOverwrite) + if err != nil { + return nil, err + } + case pi.PropStateVetted: + err = p.setVettedStatus(pss.Token, status, mdAppend, mdOverwrite) + if err != nil { + return nil, err + } + } + + // TODO Emit status change event + + return &pi.ProposalSetStatusReply{ + Timestamp: timestamp, + }, nil +} + +// processProposalsPublic retrieves and returns the proposal records for each +// of the provided proposal requests. This is a public route that removes all +// unvetted proposal files from unvetted proposals before returning them. +func (p *politeiawww) processProposalsPublic(ps pi.Proposals) (*pi.ProposalsReply, error) { + log.Tracef("processProposals: %v", ps.Requests) + + props, err := p.proposalRecords(ps.State, ps.Requests, ps.IncludeFiles) if err != nil { return nil, err } - // TODO Emit an edit proposal event + // Strip unvetted proposals of their files before returning them. + for k, v := range props { + if v.State == pi.PropStateVetted { + continue + } + v.Files = []pi.File{} + v.Metadata = []pi.Metadata{} + props[k] = v + } - log.Infof("Proposal edited: %v %v", pe.Token, pm.Name) - for k, f := range pe.Files { - log.Infof("%02v: %v %v", k, f.Name, f.Digest) + return &pi.ProposalsReply{ + Proposals: props, + }, nil +} + +// processProposalsAdmin retrieves and returns the proposal records for each of +// the provided proposal requests. This is an admin route that returns unvetted +// proposals in their entirety. +func (p *politeiawww) processProposalsAdmin(ps pi.Proposals) (*pi.ProposalsReply, error) { + log.Tracef("processProposals: %v", ps.Requests) + + props, err := p.proposalRecords(ps.State, ps.Requests, ps.IncludeFiles) + if err != nil { + return nil, err } - return &pi.ProposalEditReply{ - // TODO CensorshipRecord: cr, - Timestamp: timestamp, + return &pi.ProposalsReply{ + Proposals: props, }, nil } + +func (p *politeiawww) processProposalInventory() (*pi.ProposalInventoryReply, error) { + log.Tracef("processProposalInventory") + + // TODO politeiad needs a InventoryByStatus route + + return &pi.ProposalInventoryReply{}, nil +} diff --git a/politeiawww/politeiad.go b/politeiawww/politeiad.go index 52312e8f8..2594bb0b8 100644 --- a/politeiawww/politeiad.go +++ b/politeiawww/politeiad.go @@ -6,13 +6,30 @@ package main import ( "bytes" + "encoding/hex" "encoding/json" "fmt" "net/http" + pd "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/util" ) +// TODO politeiad API changes that are needed +// -The politeiad UpdateRecordReply is suppose to return the censorship record +// but it doesn't. +// -Add an updateUnvettedMetadata route. This has been added to the politeiad +// Backend interface, but a corresponding route has not been added yet. +// -Add a InventoryByStatus route. This has been added to the politeiad +// Backend interface, but a corresponding route has not been added yet. +// +// TODO add functions to this file for: +// -setUnvettedStatus +// -setVettedStatus +// -updateUnvettedMetadata +// -updatedVettedMetadata +// -plugin + // pdErrorReply represents the request body that is returned from politeaid // when an error occurs. PluginID will be populated if this is a plugin error. type pdErrorReply struct { @@ -85,3 +102,152 @@ func (p *politeiawww) makeRequest(method string, route string, v interface{}) ([ responseBody := util.ConvertBodyToByteArray(r.Body, false) return responseBody, nil } + +func (p *politeiawww) newRecord(metadata []pd.MetadataStream, files []pd.File) (*pd.CensorshipRecord, error) { + // Setup request + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return nil, err + } + nr := pd.NewRecord{ + Challenge: hex.EncodeToString(challenge), + Metadata: metadata, + Files: files, + } + + // Send request + resBody, err := p.makeRequest(http.MethodPost, pd.NewRecordRoute, nr) + if err != nil { + return nil, err + } + + var nrr pd.NewRecordReply + err = json.Unmarshal(resBody, &nrr) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(p.cfg.Identity, challenge, nrr.Response) + if err != nil { + return nil, err + } + + return &nrr.CensorshipRecord, nil +} + +// updateRecord updates a record in politeiad. This can be used to update +// unvetted or vetted records depending on the route that is provided. +func (p *politeiawww) updateRecord(route, token string, mdAppend, mdOverwrite []pd.MetadataStream, filesAdd []pd.File, filesDel []string) error { + // Setup request + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return err + } + ur := pd.UpdateRecord{ + Token: token, + Challenge: hex.EncodeToString(challenge), + MDOverwrite: mdOverwrite, + FilesAdd: filesAdd, + FilesDel: filesDel, + } + + // Send request + resBody, err := p.makeRequest(http.MethodPost, route, ur) + if err != nil { + return nil + } + var urr pd.UpdateRecordReply + err = json.Unmarshal(resBody, &urr) + if err != nil { + return err + } + err = util.VerifyChallenge(p.cfg.Identity, challenge, urr.Response) + if err != nil { + return err + } + + return nil +} + +// updateUnvetted updates an unvetted record in politeiad. +func (p *politeiawww) updateUnvetted(token string, mdAppend, mdOverwrite []pd.MetadataStream, filesAdd []pd.File, filesDel []string) error { + return p.updateRecord(pd.UpdateUnvettedRoute, token, + mdAppend, mdOverwrite, filesAdd, filesDel) +} + +// updateVetted updates a vetted record in politeiad. +func (p *politeiawww) updateVetted(token string, mdAppend, mdOverwrite []pd.MetadataStream, filesAdd []pd.File, filesDel []string) error { + return p.updateRecord(pd.UpdateVettedRoute, token, + mdAppend, mdOverwrite, filesAdd, filesDel) +} + +func (p *politeiawww) setUnvettedStatus(token string, status pd.RecordStatusT, mdAppend, mdOverwrite []pd.MetadataStream) error { + + return nil +} + +func (p *politeiawww) setVettedStatus(token string, status pd.RecordStatusT, mdAppend, mdOverwrite []pd.MetadataStream) error { + + return nil +} + +// getUnvetted retrieves an unvetted record from politeiad. +func (p *politeiawww) getUnvetted(token, version string) (*pd.Record, error) { + // Setup request + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return nil, err + } + gu := pd.GetUnvetted{ + Challenge: hex.EncodeToString(challenge), + Token: token, + Version: version, + } + + // Send request + resBody, err := p.makeRequest(http.MethodPost, pd.GetUnvettedRoute, gu) + if err != nil { + return nil, err + } + var gur pd.GetUnvettedReply + err = json.Unmarshal(resBody, &gur) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(p.cfg.Identity, challenge, gur.Response) + if err != nil { + return nil, err + } + + return &gur.Record, nil +} + +// getVetted retrieves a vetted record from politeiad. +func (p *politeiawww) getVetted(token, version string) (*pd.Record, error) { + // Setup request + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return nil, err + } + gu := pd.GetVetted{ + Challenge: hex.EncodeToString(challenge), + Token: token, + Version: version, + } + + // Send request + resBody, err := p.makeRequest(http.MethodPost, pd.GetVettedRoute, gu) + if err != nil { + return nil, err + } + var gvr pd.GetVettedReply + err = json.Unmarshal(resBody, &gvr) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(p.cfg.Identity, challenge, gvr.Response) + if err != nil { + return nil, err + } + + return &gvr.Record, nil +} diff --git a/politeiawww/politeiawww.go b/politeiawww/politeiawww.go index c78b9f86c..3063ea2ee 100644 --- a/politeiawww/politeiawww.go +++ b/politeiawww/politeiawww.go @@ -565,37 +565,6 @@ func (p *politeiawww) handleProposalPaywallDetails(w http.ResponseWriter, r *htt util.RespondWithJSON(w, http.StatusOK, reply) } -// handleNewProposal handles the incoming new proposal command. -func (p *politeiawww) handleNewProposal(w http.ResponseWriter, r *http.Request) { - // Get the new proposal command. - log.Tracef("handleNewProposal") - var np www.NewProposal - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&np); err != nil { - RespondWithError(w, r, 0, "handleNewProposal: unmarshal", www.UserError{ - ErrorCode: www.ErrorStatusInvalidInput, - }) - return - } - - user, err := p.getSessionUser(w, r) - if err != nil { - RespondWithError(w, r, 0, - "handleNewProposal: getSessionUser %v", err) - return - } - - reply, err := p.processNewProposal(np, user) - if err != nil { - RespondWithError(w, r, 0, - "handleNewProposal: processNewProposal %v", err) - return - } - - // Reply with the challenge response and censorship token. - util.RespondWithJSON(w, http.StatusOK, reply) -} - // handleNewComment handles incomming comments. func (p *politeiawww) handleNewComment(w http.ResponseWriter, r *http.Request) { log.Tracef("handleNewComment") @@ -658,37 +627,6 @@ func (p *politeiawww) handleLikeComment(w http.ResponseWriter, r *http.Request) util.RespondWithJSON(w, http.StatusOK, cr) } -// handleEditProposal attempts to edit a proposal -func (p *politeiawww) handleEditProposal(w http.ResponseWriter, r *http.Request) { - var ep www.EditProposal - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&ep); err != nil { - RespondWithError(w, r, 0, "handleEditProposal: unmarshal", - www.UserError{ - ErrorCode: www.ErrorStatusInvalidInput, - }) - return - } - - user, err := p.getSessionUser(w, r) - if err != nil { - RespondWithError(w, r, 0, - "handleEditProposal: getSessionUser %v", err) - return - } - - log.Debugf("handleEditProposal: %v", ep.Token) - - epr, err := p.processEditProposal(ep, user) - if err != nil { - RespondWithError(w, r, 0, - "handleEditProposal: processEditProposal %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, epr) -} - // handleAuthorizeVote handles authorizing a proposal vote. func (p *politeiawww) handleAuthorizeVote(w http.ResponseWriter, r *http.Request) { log.Tracef("handleAuthorizeVote") @@ -981,41 +919,6 @@ func (p *politeiawww) handleAuthenticatedWebsocket(w http.ResponseWriter, r *htt p.handleWebsocket(w, r, id) } -// handleSetProposalStatus handles the incoming set proposal status command. -// It's used for either publishing or censoring a proposal. -func (p *politeiawww) handleSetProposalStatus(w http.ResponseWriter, r *http.Request) { - // Get the proposal status command. - log.Tracef("handleSetProposalStatus") - var sps www.SetProposalStatus - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&sps); err != nil { - RespondWithError(w, r, 0, "handleSetProposalStatus: unmarshal", - www.UserError{ - ErrorCode: www.ErrorStatusInvalidInput, - }) - return - } - - user, err := p.getSessionUser(w, r) - if err != nil { - RespondWithError(w, r, 0, - "handleSetProposalStatus: getSessionUser %v", err) - return - } - - // Set status - reply, err := p.processSetProposalStatus(sps, user) - if err != nil { - RespondWithError(w, r, 0, - "handleSetProposalStatus: processSetProposalStatus %v", - err) - return - } - - // Reply with the new proposal status. - util.RespondWithJSON(w, http.StatusOK, reply) -} - // handleStartVote handles the v2 StartVote route. func (p *politeiawww) handleStartVoteV2(w http.ResponseWriter, r *http.Request) { log.Tracef("handleStartVoteV2") @@ -1242,18 +1145,12 @@ func (p *politeiawww) setPoliteiaWWWRoutes() { p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, www.RouteProposalPaywallDetails, p.handleProposalPaywallDetails, permissionLogin) - p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, - www.RouteNewProposal, p.handleNewProposal, - permissionLogin) p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, www.RouteNewComment, p.handleNewComment, permissionLogin) p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, www.RouteLikeComment, p.handleLikeComment, permissionLogin) - p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, - www.RouteEditProposal, p.handleEditProposal, - permissionLogin) p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, www.RouteAuthorizeVote, p.handleAuthorizeVote, permissionLogin) @@ -1277,9 +1174,6 @@ func (p *politeiawww) setPoliteiaWWWRoutes() { permissionLogin) // Routes that require being logged in as an admin user. - p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, - www.RouteSetProposalStatus, p.handleSetProposalStatus, - permissionAdmin) p.addRoute(http.MethodPost, www2.APIRoute, www2.RouteStartVote, p.handleStartVoteV2, permissionAdmin) diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index d560bb680..60d0f3d77 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -14,7 +14,6 @@ import ( "time" "github.com/decred/politeia/decredplugin" - "github.com/decred/politeia/mdstream" pd "github.com/decred/politeia/politeiad/api/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" www2 "github.com/decred/politeia/politeiawww/api/www/v2" @@ -139,17 +138,6 @@ func voteStatusFromVoteSummary(r decredplugin.VoteSummaryReply, endHeight, bestB return www.PropVoteStatusInvalid } -// convertWWWPropCreditFromDatabasePropCredit coverts a database proposal -// credit to a v1 proposal credit. -func convertWWWPropCreditFromDatabasePropCredit(credit user.ProposalCredit) www.ProposalCredit { - return www.ProposalCredit{ - PaywallID: credit.PaywallID, - Price: credit.Price, - DatePurchased: credit.DatePurchased, - TxID: credit.TxID, - } -} - // fillProposalMissingFields populates a ProposalRecord struct with the fields // that are not stored in the cache. func (p *politeiawww) fillProposalMissingFields(pr *www.ProposalRecord) error { @@ -655,13 +643,6 @@ func (p *politeiawww) voteSummaryGet(token string, bestBlock uint64) (*www.VoteS return &vs, nil } -// processNewProposal tries to submit a new proposal to politeiad. -func (p *politeiawww) processNewProposal(np www.NewProposal, user *user.User) (*www.NewProposalReply, error) { - log.Tracef("processNewProposal") - - return nil, nil -} - // createProposalDetailsReply makes updates to a proposal record based on the // user who made the request, and puts it into a ProposalDetailsReply. func createProposalDetailsReply(prop *www.ProposalRecord, user *user.User) *www.ProposalDetailsReply { @@ -847,275 +828,6 @@ func (p *politeiawww) processBatchProposals(bp www.BatchProposals, user *user.Us }, nil } -// verifyStatusChange verifies that the proposal status change is a valid -// status transition. This only applies to manual status transitions that are -// initiated by an admin. It does not apply to status changes that are caused -// by editing a proposal. -func verifyStatusChange(current, next www.PropStatusT) error { - var err error - switch { - case current == www.PropStatusNotReviewed && - (next == www.PropStatusCensored || - next == www.PropStatusPublic): - // allowed; continue - case current == www.PropStatusUnreviewedChanges && - (next == www.PropStatusCensored || - next == www.PropStatusPublic): - // allowed; continue - case current == www.PropStatusPublic && - next == www.PropStatusAbandoned: - // allowed; continue - default: - err = www.UserError{ - ErrorCode: www.ErrorStatusInvalidPropStatusTransition, - } - } - return err -} - -// processSetProposalStatus changes the status of an existing proposal. -func (p *politeiawww) processSetProposalStatus(sps www.SetProposalStatus, u *user.User) (*www.SetProposalStatusReply, error) { - log.Tracef("processSetProposalStatus %v", sps.Token) - - // Make sure token is valid and not a prefix - if !tokenIsValid(sps.Token) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: []string{sps.Token}, - } - } - - // Ensure the status change message is not blank if the - // proposal is being censored or abandoned. - if sps.StatusChangeMessage == "" && - (sps.ProposalStatus == www.PropStatusCensored || - sps.ProposalStatus == www.PropStatusAbandoned) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusChangeMessageCannotBeBlank, - } - } - - // Ensure the provided public key is the user's active key. - if sps.PublicKey != u.PublicKey() { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidSigningKey, - } - } - - // Validate signature - msg := sps.Token + strconv.Itoa(int(sps.ProposalStatus)) + - sps.StatusChangeMessage - err := validateSignature(sps.PublicKey, sps.Signature, msg) - if err != nil { - return nil, err - } - - // Get proposal from cache - pr, err := p.getProp(sps.Token) - if err != nil { - /* - // TODO - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - } - } - */ - return nil, err - } - - // The only time admins are allowed to change the status of - // their own proposals is on testnet - if !p.cfg.TestNet { - author, err := p.db.UserGetByPubKey(pr.PublicKey) - if err != nil { - return nil, err - } - if author.ID.String() == u.ID.String() { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusReviewerAdminEqualsAuthor, - } - } - } - - // Create change record - newStatus := convertPropStatusFromWWW(sps.ProposalStatus) - blob, err := mdstream.EncodeRecordStatusChangeV2( - mdstream.RecordStatusChangeV2{ - Version: mdstream.VersionRecordStatusChange, - Timestamp: time.Now().Unix(), - NewStatus: newStatus, - Signature: sps.Signature, - AdminPubKey: u.PublicKey(), - StatusChangeMessage: sps.StatusChangeMessage, - }) - if err != nil { - return nil, err - } - - // Create challenge - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - - // Ensure status change is allowed - err = verifyStatusChange(pr.Status, sps.ProposalStatus) - if err != nil { - return nil, err - } - - var challengeResponse string - switch { - case pr.State == www.PropStateUnvetted: - // Unvetted status change - - // Setup request - sus := pd.SetUnvettedStatus{ - Token: sps.Token, - Status: newStatus, - Challenge: hex.EncodeToString(challenge), - MDAppend: []pd.MetadataStream{ - { - ID: mdstream.IDRecordStatusChange, - Payload: string(blob), - }, - }, - } - - // Send unvetted status change request - responseBody, err := p.makeRequest(http.MethodPost, - pd.SetUnvettedStatusRoute, sus) - if err != nil { - return nil, err - } - - var susr pd.SetUnvettedStatusReply - err = json.Unmarshal(responseBody, &susr) - if err != nil { - return nil, fmt.Errorf("could not unmarshal "+ - "SetUnvettedStatusReply: %v", err) - } - challengeResponse = susr.Response - - case pr.State == www.PropStateVetted: - // Vetted status change - - // Ensure voting has not been started or authorized yet - bb, err := p.getBestBlock() - if err != nil { - return nil, fmt.Errorf("getBestBlock: %v", err) - } - vs, err := p.voteSummaryGet(pr.CensorshipRecord.Token, bb) - if err != nil { - return nil, err - } - switch { - case vs.EndHeight != 0: - return nil, www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - ErrorContext: []string{"vote has started"}, - } - case vs.Status == www.PropVoteStatusAuthorized: - return nil, www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - ErrorContext: []string{"vote has been authorized"}, - } - } - - // Setup request - svs := pd.SetVettedStatus{ - Token: sps.Token, - Status: newStatus, - Challenge: hex.EncodeToString(challenge), - MDAppend: []pd.MetadataStream{ - { - ID: mdstream.IDRecordStatusChange, - Payload: string(blob), - }, - }, - } - - // Send vetted status change request - responseBody, err := p.makeRequest(http.MethodPost, - pd.SetVettedStatusRoute, svs) - if err != nil { - return nil, err - } - - var svsr pd.SetVettedStatusReply - err = json.Unmarshal(responseBody, &svsr) - if err != nil { - return nil, fmt.Errorf("could not unmarshal "+ - "SetVettedStatusReply: %v", err) - } - challengeResponse = svsr.Response - - default: - panic(fmt.Sprintf("invalid proposal state %v %v", - pr.CensorshipRecord.Token, pr.State)) - } - - // Verify the challenge - err = util.VerifyChallenge(p.cfg.Identity, challenge, - challengeResponse) - if err != nil { - return nil, err - } - - // Get record from the cache - updatedProp, err := p.getPropVersion(pr.CensorshipRecord.Token, pr.Version) - if err != nil { - return nil, err - } - - // Fire off proposal status change event - p.fireEvent(EventTypeProposalStatusChange, - EventDataProposalStatusChange{ - Proposal: updatedProp, - AdminUser: u, - SetProposalStatus: &sps, - }, - ) - - return &www.SetProposalStatusReply{ - Proposal: *updatedProp, - }, nil -} - -// processEditProposal attempts to edit a proposal on politeiad. -func (p *politeiawww) processEditProposal(ep www.EditProposal, u *user.User) (*www.EditProposalReply, error) { - log.Tracef("processEditProposal %v", ep.Token) - - /* - // Validate proposal status - cachedProp, err := p.getProp(ep.Token) - if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - } - } - return nil, err - } - - // Check if there were changes in the proposal by comparing - // their merkle roots. This captures changes that were made - // to either the files or the metadata. - mr, err := wwwutil.MerkleRoot(ep.Files, ep.Metadata) - if err != nil { - return nil, err - } - if cachedProp.CensorshipRecord.Merkle == mr { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusNoProposalChanges, - } - } - */ - - return nil, nil -} - // processAllVetted returns an array of vetted proposals. The maximum number // of proposals returned is dictated by www.ProposalListPageSize. func (p *politeiawww) processAllVetted(v www.GetAllVetted) (*www.GetAllVettedReply, error) { diff --git a/politeiawww/user.go b/politeiawww/user.go index b0b205e4f..baf2eaefc 100644 --- a/politeiawww/user.go +++ b/politeiawww/user.go @@ -1589,17 +1589,26 @@ func (p *politeiawww) processVerifyResetPassword(vrp www.VerifyResetPassword) (* return &www.VerifyResetPasswordReply{}, nil } +func convertProposalCreditFromUserDB(credit user.ProposalCredit) www.ProposalCredit { + return www.ProposalCredit{ + PaywallID: credit.PaywallID, + Price: credit.Price, + DatePurchased: credit.DatePurchased, + TxID: credit.TxID, + } +} + // processUserProposalCredits returns a list of the user's unspent proposal // credits and a list of the user's spent proposal credits. func processUserProposalCredits(u *user.User) (*www.UserProposalCreditsReply, error) { // Convert from database proposal credits to www proposal credits. upc := make([]www.ProposalCredit, len(u.UnspentProposalCredits)) for i, credit := range u.UnspentProposalCredits { - upc[i] = convertWWWPropCreditFromDatabasePropCredit(credit) + upc[i] = convertProposalCreditFromUserDB(credit) } spc := make([]www.ProposalCredit, len(u.SpentProposalCredits)) for i, credit := range u.SpentProposalCredits { - spc[i] = convertWWWPropCreditFromDatabasePropCredit(credit) + spc[i] = convertProposalCreditFromUserDB(credit) } return &www.UserProposalCreditsReply{ @@ -2028,7 +2037,7 @@ func (p *politeiawww) processUserPaymentsRescan(upr www.UserPaymentsRescan) (*ww // Convert database credits to www credits newCreditsWWW := make([]www.ProposalCredit, len(newCredits)) for i, credit := range newCredits { - newCreditsWWW[i] = convertWWWPropCreditFromDatabasePropCredit(credit) + newCreditsWWW[i] = convertProposalCreditFromUserDB(credit) } return &www.UserPaymentsRescanReply{ diff --git a/politeiawww/www.go b/politeiawww/www.go index 0908daab1..676feb3de 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -323,7 +323,7 @@ func (p *politeiawww) getIdentity() error { // files a complaint. func respondWithPiError(w http.ResponseWriter, r *http.Request, format string, err error) { // Check for pi user error - if ue, ok := err.(pi.UserError); ok { + if ue, ok := err.(pi.UserErrorReply); ok { // Error is a pi user error. Log it and return a 400. if len(ue.ErrorContext) == 0 { log.Infof("Pi user error: %v %v %v", @@ -337,7 +337,7 @@ func respondWithPiError(w http.ResponseWriter, r *http.Request, format string, e } util.RespondWithJSON(w, http.StatusBadRequest, - pi.UserError{ + pi.UserErrorReply{ ErrorCode: ue.ErrorCode, ErrorContext: ue.ErrorContext, }) @@ -369,7 +369,7 @@ func respondWithPiError(w http.ResponseWriter, r *http.Request, format string, e } util.RespondWithJSON(w, http.StatusInternalServerError, - pi.ErrorReply{ + pi.ServerErrorReply{ ErrorCode: t, }) return @@ -389,7 +389,7 @@ func respondWithPiError(w http.ResponseWriter, r *http.Request, format string, e } util.RespondWithJSON(w, http.StatusBadRequest, - pi.UserError{ + pi.UserErrorReply{ ErrorCode: piErrCode, ErrorContext: errContext, }) @@ -404,7 +404,7 @@ func respondWithPiError(w http.ResponseWriter, r *http.Request, format string, e log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) util.RespondWithJSON(w, http.StatusInternalServerError, - pi.ErrorReply{ + pi.ServerErrorReply{ ErrorCode: t, }) } From 5c9f6b4a807667a44cf861dc75925ca54800fd38 Mon Sep 17 00:00:00 2001 From: Thiago Figueiredo Date: Fri, 11 Sep 2020 16:58:08 -0300 Subject: [PATCH 039/449] uppercase CMS --- politeiawww/eventmanager.go | 2 +- politeiawww/www.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/politeiawww/eventmanager.go b/politeiawww/eventmanager.go index 5021e0599..89319df2d 100644 --- a/politeiawww/eventmanager.go +++ b/politeiawww/eventmanager.go @@ -116,7 +116,7 @@ func (p *politeiawww) setupEventListenersPi() { go p.handleEventProposalVoteStarted(ch) } -func (p *politeiawww) setupEventListenersCms() { +func (p *politeiawww) setupEventListenersCMS() { // Setup invoice comment event ch := make(chan interface{}) p.eventManager.register(eventInvoiceComment, ch) diff --git a/politeiawww/www.go b/politeiawww/www.go index b40732c42..27fffb538 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -751,7 +751,7 @@ func _main() error { case cmsWWWMode: // Setup event manager - p.setupEventListenersCms() + p.setupEventListenersCMS() // Setup dcrdata websocket connection ws, err := wsdcrdata.New(p.dcrdataHostWS()) From 36d45de4efb03fece9ec5a6054c706ac148ef491 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 14 Sep 2020 06:33:35 -0500 Subject: [PATCH 040/449] events update --- politeiawww/email.go | 134 +++++++++--------- politeiawww/eventmanager.go | 263 ++++++++++++++++++++---------------- politeiawww/piwww.go | 80 ++++++++++- politeiawww/politeiad.go | 12 ++ politeiawww/politeiawww.go | 23 ---- politeiawww/templates.go | 179 +++++++++++++++--------- 6 files changed, 411 insertions(+), 280 deletions(-) diff --git a/politeiawww/email.go b/politeiawww/email.go index bb6e85133..a6cd8192d 100644 --- a/politeiawww/email.go +++ b/politeiawww/email.go @@ -6,12 +6,13 @@ package main import ( "bytes" + "fmt" "net/url" "strings" "text/template" "time" - v1 "github.com/decred/politeia/politeiawww/api/pi/v1" + pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" ) @@ -119,84 +120,79 @@ func (p *politeiawww) emailResetPasswordVerificationLink(email, username, token return p.smtp.sendEmailTo(subject, body, recipients) } -// emailProposalStatusChange sends emails regarding the proposal status change -// event. Sends email for the author and the users with this notification -// bit set on -func (p *politeiawww) emailProposalStatusChange(data dataProposalStatusChange, emails []string) error { - route := strings.Replace(guiRouteProposalDetails, "{token}", data.token, 1) +// emailAuthorProposalStatusChange sends an email to the author of the proposal +// notifying them of the proposal status change. +func (p *politeiawww) emailProposalStatusChangeToAuthor(d dataProposalStatusChange) error { + route := strings.Replace(guiRouteProposalDetails, "{token}", d.token, 1) l, err := url.Parse(p.cfg.WebServerAddress + route) if err != nil { return err } - // Prepare and send author's email - err = p.emailAuthorProposalStatusChange(data.name, data.email, l.String(), - data.statusChangeMessage, data.emailNotifications, data.status, emails) - if err != nil { - return err - } - - // Prepare and send user's email - err = p.emailUsersProposalStatusChange(data.name, data.username, l.String(), - emails) - if err != nil { - return err - } + var ( + subject string + body string + ) + switch d.status { + case pi.PropStatusPublic: + subject = "Your Proposal Has Been Published" + tmplData := tmplDataProposalVettedForAuthor{ + Name: d.name, + Link: l.String(), + } + body, err = createBody(tmplProposalVettedForAuthor, &tmplData) + if err != nil { + return err + } - return nil -} + case pi.PropStatusCensored: + subject = "Your Proposal Has Been Censored" + tmplData := tmplDataProposalCensoredForAuthor{ + Name: d.name, + Reason: d.reason, + Link: l.String(), + } + body, err = createBody(tmplProposalCensoredForAuthor, &tmplData) + if err != nil { + return err + } -// emailAuthorProposalStatusChange sends email for the author of the proposal -// in which the status has changed, if his notification bit is set on. -func (p *politeiawww) emailAuthorProposalStatusChange(name, email, link, statusChangeMsg string, emailNotifications uint64, status v1.PropStatusT, emails []string) error { - if !notificationIsSet(emailNotifications, - www.NotificationEmailMyProposalStatusChange) { - return nil + default: + return fmt.Errorf("no author notification for prop status %v", d.status) } - var subject string - var template *template.Template - - switch status { - case v1.PropStatusCensored: - subject = "Your Proposal Has Been Censored" - template = templateProposalCensoredForAuthor - case v1.PropStatusPublic: - subject = "Your Proposal Has Been Published" - template = templateProposalVettedForAuthor - } + return p.smtp.sendEmailTo(subject, body, []string{d.author.Email}) +} - authorTplData := proposalStatusChangeTemplateData{ - Link: link, - Name: name, - StatusChangeReason: statusChangeMsg, - } - body, err := createBody(template, &authorTplData) +// emailProposalStatusChangeToUsers sends an email to the provided users +// notifying them of the proposal status change. +func (p *politeiawww) emailProposalStatusChangeToUsers(d dataProposalStatusChange, emails []string) error { + route := strings.Replace(guiRouteProposalDetails, "{token}", d.token, 1) + l, err := url.Parse(p.cfg.WebServerAddress + route) if err != nil { return err } - recipients := []string{email} - return p.smtp.sendEmailTo(subject, body, recipients) -} + var ( + subject string + body string + ) + switch d.status { + case pi.PropStatusPublic: + subject = "New Proposal Published" + tmplData := tmplDataProposalVetted{ + Name: d.name, + Link: l.String(), + } + body, err = createBody(tmplProposalVetted, &tmplData) + if err != nil { + return err + } -// emailUsersProposalStatusChange sends email for all users with this -// notification bit set on. -func (p *politeiawww) emailUsersProposalStatusChange(name, username, link string, emails []string) error { - if len(emails) > 0 { - return nil - } - subject := "New Proposal Published" - template := templateProposalVetted - usersTplData := proposalStatusChangeTemplateData{ - Link: link, - Name: name, - Username: username, - } - body, err := createBody(template, &usersTplData) - if err != nil { - return err + default: + return fmt.Errorf("no user notification for prop status %v", d.status) } + return p.smtp.sendEmailTo(subject, body, emails) } @@ -209,7 +205,7 @@ func (p *politeiawww) emailProposalEdited(name, username, token, version string, return err } - tplData := proposalEditedTemplateData{ + tmplData := tmplDataProposalEdited{ Link: l.String(), Name: name, Version: version, @@ -217,7 +213,7 @@ func (p *politeiawww) emailProposalEdited(name, username, token, version string, } subject := "Proposal Edited" - body, err := createBody(templateProposalEdited, &tplData) + body, err := createBody(tmplProposalEdited, &tmplData) if err != nil { return err } @@ -274,14 +270,14 @@ func (p *politeiawww) emailProposalSubmitted(token, name, username string, email return err } - tplData := proposalSubmittedTemplateData{ - Link: l.String(), - Name: name, + tmplData := tmplDataProposalSubmitted{ Username: username, + Name: name, + Link: l.String(), } subject := "New Proposal Submitted" - body, err := createBody(templateProposalSubmitted, &tplData) + body, err := createBody(tmplProposalSubmitted, &tmplData) if err != nil { return err } diff --git a/politeiawww/eventmanager.go b/politeiawww/eventmanager.go index 89319df2d..d94a9825f 100644 --- a/politeiawww/eventmanager.go +++ b/politeiawww/eventmanager.go @@ -7,7 +7,7 @@ package main import ( "sync" - v1 "github.com/decred/politeia/politeiawww/api/pi/v1" + pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/user" "github.com/google/uuid" @@ -20,10 +20,10 @@ const ( eventTypeInvalid eventT = iota // Pi events - eventProposalComment eventProposalSubmitted - eventProposalStatusChange eventProposalEdited + eventProposalStatusChange + eventProposalComment eventProposalVoteAuthorized eventProposalVoteStarted @@ -149,8 +149,8 @@ func notificationIsSet(emailNotifications uint64, n www.EmailNotificationT) bool return true } -// userNotificationEnabled wraps all user checks to see if he is in correct -// state to receive notifications +// userNotificationEnabled returns whether the user should receive the provided +// notification. func userNotificationEnabled(u user.User, n www.EmailNotificationT) bool { // Never send notification to deactivated users if u.Deactivated { @@ -163,128 +163,94 @@ func userNotificationEnabled(u user.User, n www.EmailNotificationT) bool { return true } -type dataProposalComment struct { - token string // Proposal token - name string // Proposal name - username string // Author username - parentID string // Parent comment id - commentID string // Comment id - commentUsername string // Comment user username +type dataProposalSubmitted struct { + token string // Proposal token + name string // Proposal name + username string // Author username } -func (p *politeiawww) handleEventProposalComment(ch chan interface{}) { +func (p *politeiawww) handleEventProposalSubmitted(ch chan interface{}) { for msg := range ch { - d, ok := msg.(dataProposalComment) + d, ok := msg.(dataProposalSubmitted) if !ok { - log.Errorf("handleEventProposalComment invalid msg: %v", msg) - continue - } - - // Fetch proposal author - author, err := p.db.UserGetByUsername(d.username) - if err != nil { - log.Error(err) - continue - } - - // Check if user notification is enabled - if !userNotificationEnabled(*author, - www.NotificationEmailCommentOnMyProposal) { - continue - } - - // Don't notify when author comments on own proposal - if d.commentUsername == author.Username { + log.Errorf("handleEventProposalSubmitted invalid msg: %v", msg) continue } - // Top-level comment - if d.parentID == "0" { - err := p.emailProposalComment(d.token, d.commentID, - d.commentUsername, d.name, author.Email) - if err != nil { - log.Errorf("emailProposalComment: %v", err) + // Compile email notification recipients + emails := make([]string, 0, 256) + err := p.db.AllUsers(func(u *user.User) { + // Check if user is able to receive notification + if userNotificationEnabled(*u, www.NotificationEmailAdminProposalNew) { + emails = append(emails, u.Email) } - continue - } - - // Nested comment reply. Fetch parent comment in order to fetch - // parent comment author - parent, err := p.decredCommentGetByID(d.token, d.parentID) - if err != nil { - log.Errorf("decredCommentGetByID: %v", err) - continue - } - - author, err = p.db.UserGetByPubKey(parent.PublicKey) + }) if err != nil { - log.Errorf("UserGetByPubKey: %v", err) - continue + log.Errorf("handleEventProposalSubmitted: AllUsers: %v", err) + return } - err = p.emailProposalComment(d.token, d.commentID, - d.commentUsername, d.name, author.Email) + // Send email notification + err = p.emailProposalSubmitted(d.token, d.name, d.username, emails) if err != nil { - log.Errorf("emailProposalComment: %v", err) + log.Errorf("emailProposalSubmitted: %v", err) } - log.Debugf("Sent proposal commment notification %v", d.token) + log.Debugf("Sent proposal submitted notification %v", d.token) } } -type dataProposalSubmitted struct { - token string // Proposal token - name string // Proposal name +type dataProposalEdited struct { + userID string // Author id username string // Author username + token string // Proposal censorship token + name string // Proposal name + version string // Proposal version } -func (p *politeiawww) handleEventProposalSubmitted(ch chan interface{}) { +func (p *politeiawww) handleEventProposalEdited(ch chan interface{}) { for msg := range ch { - d, ok := msg.(dataProposalSubmitted) + d, ok := msg.(dataProposalEdited) if !ok { - log.Errorf("handleEventProposalSubmitted invalid msg: %v", msg) + log.Errorf("handleEventProposalEdited invalid msg: %v", msg) continue } - // Compile email notification recipients + // Compile list of emails to send notification to emails := make([]string, 0, 256) err := p.db.AllUsers(func(u *user.User) { - // Check if user is able to receive notification - if userNotificationEnabled(*u, - www.NotificationEmailAdminProposalNew) { - emails = append(emails, u.Email) - } - - // Only send proposal submitted notifications to admins - if !u.Admin { + // Check circumstances where we don't notify + switch { + case u.ID.String() == d.userID: + // User is the author + return + case !userNotificationEnabled(*u, + www.NotificationEmailRegularProposalEdited): + // User doesn't have notification bit set return } + + // Add user to notification list + emails = append(emails, u.Email) }) - if err != nil { - log.Errorf("handleEventProposalSubmitted: AllUsers: %v", err) - return - } - // Send email notification - err = p.emailProposalSubmitted(d.token, d.name, d.username, emails) + err = p.emailProposalEdited(d.name, d.username, d.token, d.version, + emails) if err != nil { - log.Errorf("emailProposalSubmitted: %v", err) + log.Errorf("emailProposalEdited: %v", err) } - log.Debugf("Sent proposal submitted notification %v", d.token) + log.Debugf("Sent proposal edited notifications %v", d.token) } } type dataProposalStatusChange struct { - name string // Proposal name - token string // Proposal censorship token - status v1.PropStatusT // Proposal status - statusChangeMessage string // Status change message - adminID uuid.UUID // Admin uuid - id uuid.UUID // Author uuid - email string // Author user email - emailNotifications uint64 // Author notification settings - username string // Author username + name string // Proposal name + token string // Proposal censorship token + status pi.PropStatusT // Proposal status + reason string // Status change reason + adminID string // Admin uuid + author user.User // Proposal author } func (p *politeiawww) handleEventProposalStatusChange(ch chan interface{}) { @@ -296,67 +262,124 @@ func (p *politeiawww) handleEventProposalStatusChange(ch chan interface{}) { } // Check if proposal is in correct status for notification - if d.status != v1.PropStatusPublic && - d.status != v1.PropStatusCensored { + switch d.status { + case pi.PropStatusPublic, pi.PropStatusCensored: + // The status requires a notification be sent + default: + // The status does not require a notification be sent. Listen + // for next event. continue } + // Compile list of emails to sent notification to emails := make([]string, 0, 256) + notification := www.NotificationEmailRegularProposalVetted err := p.db.AllUsers(func(u *user.User) { - // Check circunstances where we don't notify - if u.ID == d.adminID || u.ID == d.id || - !userNotificationEnabled(*u, - www.NotificationEmailRegularProposalVetted) { + // Check circumstances where we don't notify + switch { + case u.ID.String() == d.adminID: + // User is the admin that made the status change + return + case u.ID.String() == d.author.ID.String(): + // User is the author. The author is sent a notification but + // the notification is not the same as the normal user + // notification so don't include the author's emails in the + // list of user emails. + return + case !userNotificationEnabled(*u, notification): + // User does not have notification bit set return } emails = append(emails, u.Email) }) - err = p.emailProposalStatusChange(d, emails) - if err != nil { - log.Errorf("emailProposalStatusChange: %v", err) + // Email author + if userNotificationEnabled(d.author, notification) { + err = p.emailProposalStatusChangeToAuthor(d) + if err != nil { + log.Errorf("emailProposalStatusChangeToAuthor: %v", err) + } + } + + // Email users + if len(emails) > 0 { + err = p.emailProposalStatusChangeToUsers(d, emails) + if err != nil { + log.Errorf("emailProposalStatusChangeToUsers: %v", err) + } } log.Debugf("Sent proposal status change notifications %v", d.token) } } -type dataProposalEdited struct { - id uuid.UUID // Author id - username string // Author username - token string // Proposal censorship token - name string // Proposal name - version string // Proposal version +type dataProposalComment struct { + token string // Proposal token + name string // Proposal name + username string // Author username + parentID string // Parent comment id + commentID string // Comment id + commentUsername string // Comment user username } -func (p *politeiawww) handleEventProposalEdited(ch chan interface{}) { +func (p *politeiawww) handleEventProposalComment(ch chan interface{}) { for msg := range ch { - d, ok := msg.(dataProposalEdited) + d, ok := msg.(dataProposalComment) if !ok { - log.Errorf("handleEventProposalEdited invalid msg: %v", msg) + log.Errorf("handleEventProposalComment invalid msg: %v", msg) continue } - emails := make([]string, 0, 256) - err := p.db.AllUsers(func(u *user.User) { - // Check circunstances where we don't notify - if u.NewUserPaywallTx == "" || u.ID == d.id || - !userNotificationEnabled(*u, - www.NotificationEmailRegularProposalEdited) { - return + // Fetch proposal author + author, err := p.db.UserGetByUsername(d.username) + if err != nil { + log.Error(err) + continue + } + + // Check if user notification is enabled + if !userNotificationEnabled(*author, + www.NotificationEmailCommentOnMyProposal) { + continue + } + + // Don't notify when author comments on own proposal + if d.commentUsername == author.Username { + continue + } + + // Top-level comment + if d.parentID == "0" { + err := p.emailProposalComment(d.token, d.commentID, + d.commentUsername, d.name, author.Email) + if err != nil { + log.Errorf("emailProposalComment: %v", err) } + continue + } - emails = append(emails, u.Email) - }) + // Nested comment reply. Fetch parent comment in order to fetch + // parent comment author + parent, err := p.decredCommentGetByID(d.token, d.parentID) + if err != nil { + log.Errorf("decredCommentGetByID: %v", err) + continue + } - err = p.emailProposalEdited(d.name, d.username, d.token, d.version, - emails) + author, err = p.db.UserGetByPubKey(parent.PublicKey) if err != nil { - log.Errorf("emailProposalEdited: %v", err) + log.Errorf("UserGetByPubKey: %v", err) + continue } - log.Debugf("Sent proposal edited notifications %v", d.token) + err = p.emailProposalComment(d.token, d.commentID, + d.commentUsername, d.name, author.Email) + if err != nil { + log.Errorf("emailProposalComment: %v", err) + } + + log.Debugf("Sent proposal commment notification %v", d.token) } } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 8a3b1dfe2..f032807d1 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -282,6 +282,27 @@ func convertProposalRecordFromPD(r pd.Record) (*pi.ProposalRecord, error) { }, nil } +// parseProposalName parsed the proposal name from the ProposalMetadata and +// returns it. An empty string will be returned if any errors occur or if a +// name is not found. +func parseProposalName(pr pi.ProposalRecord) string { + var name string + for _, v := range pr.Metadata { + if v.Hint == pi.HintProposalMetadata { + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return "" + } + pm, err := piplugin.DecodeProposalMetadata(b) + if err != nil { + return "" + } + name = pm.Name + } + } + return name +} + // linkByPeriodMin returns the minimum amount of time, in seconds, that the // LinkBy period must be set to. This is determined by adding 1 week onto the // minimum voting period so that RFP proposal submissions have at least one @@ -872,7 +893,7 @@ func (p *politeiawww) processProposalNew(pn pi.ProposalNew, usr user.User) (*pi. return nil, err } - // Fire off a new proposal event + // Emit a new proposal event p.eventManager.emit(eventProposalSubmitted, dataProposalSubmitted{ token: cr.Token, @@ -1019,7 +1040,14 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p return nil, fmt.Errorf("unknown state %v", pe.State) } - // TODO Emit an edit proposal event + // Emit an edit proposal event + p.eventManager.emit(eventProposalEdited, dataProposalEdited{ + userID: usr.ID.String(), + username: usr.Username, + token: pe.Token, + name: pm.Name, + // TODO version: version, + }) log.Infof("Proposal edited: %v %v", pe.Token, pm.Name) for k, f := range pe.Files { @@ -1129,7 +1157,53 @@ func (p *politeiawww) processProposalSetStatus(pss pi.ProposalSetStatus, usr use } } - // TODO Emit status change event + // Emit status change event + var ( + r *pd.Record + pr *pi.ProposalRecord + author *user.User + ) + switch pss.State { + case pi.PropStateUnvetted: + r, err = p.getUnvettedLatest(pss.Token) + if err != nil { + err = fmt.Errorf("getUnvettedLatest: %v", err) + goto reply + } + case pi.PropStateVetted: + r, err = p.getVettedLatest(pss.Token) + if err != nil { + err = fmt.Errorf("getVettedLatest: %v", err) + goto reply + } + } + pr, err = convertProposalRecordFromPD(*r) + if err != nil { + err = fmt.Errorf("convertProposalRecordFromPD: %v", err) + goto reply + } + author, err = p.db.UserGetByPubKey(pr.PublicKey) + if err != nil { + err = fmt.Errorf("UserGetByPubKey: %v", err) + goto reply + } + p.eventManager.emit(eventProposalStatusChange, + dataProposalStatusChange{ + name: parseProposalName(*pr), + token: pss.Token, + status: pss.Status, + reason: pss.Reason, + adminID: usr.ID.String(), + author: *author, + }) + +reply: + // If an error exists at this point it means the error was from + // an action that occured after the status had been updated in + // politeiad. Log it and return a normal reply. + if err != nil { + log.Errorf("processProposalSetStatus: %v", err) + } return &pi.ProposalSetStatusReply{ Timestamp: timestamp, diff --git a/politeiawww/politeiad.go b/politeiawww/politeiad.go index 2594bb0b8..ab356f075 100644 --- a/politeiawww/politeiad.go +++ b/politeiawww/politeiad.go @@ -221,6 +221,12 @@ func (p *politeiawww) getUnvetted(token, version string) (*pd.Record, error) { return &gur.Record, nil } +// getUnvettedLatest returns the latest version of the unvetted record for the +// provided token. +func (p *politeiawww) getUnvettedLatest(token string) (*pd.Record, error) { + return p.getUnvetted(token, "") +} + // getVetted retrieves a vetted record from politeiad. func (p *politeiawww) getVetted(token, version string) (*pd.Record, error) { // Setup request @@ -251,3 +257,9 @@ func (p *politeiawww) getVetted(token, version string) (*pd.Record, error) { return &gvr.Record, nil } + +// getVettedLatest returns the latest version of the vvetted record for the +// provided token. +func (p *politeiawww) getVettedLatest(token string) (*pd.Record, error) { + return p.getVetted(token, "") +} diff --git a/politeiawww/politeiawww.go b/politeiawww/politeiawww.go index 3063ea2ee..90cf557fa 100644 --- a/politeiawww/politeiawww.go +++ b/politeiawww/politeiawww.go @@ -33,29 +33,6 @@ import ( "github.com/robfig/cron" ) -var ( - templateProposalSubmitted = template.Must( - template.New("proposal_submitted_template").Parse(templateProposalSubmittedRaw)) - templateProposalVetted = template.Must( - template.New("proposal_vetted_template").Parse(templateProposalVettedRaw)) - templateProposalEdited = template.Must( - template.New("proposal_edited_template").Parse(templateProposalEditedRaw)) - templateProposalVoteStarted = template.Must( - template.New("proposal_vote_started_template").Parse(templateProposalVoteStartedRaw)) - templateProposalVoteAuthorized = template.Must( - template.New("proposal_vote_authorized_template").Parse(templateProposalVoteAuthorizedRaw)) - templateProposalVettedForAuthor = template.Must( - template.New("proposal_vetted_for_author_template").Parse(templateProposalVettedForAuthorRaw)) - templateProposalCensoredForAuthor = template.Must( - template.New("proposal_censored_for_author_template").Parse(templateProposalCensoredForAuthorRaw)) - templateProposalVoteStartedForAuthor = template.Must( - template.New("proposal_vote_started_for_author_template").Parse(templateProposalVoteStartedForAuthorRaw)) - templateCommentReplyOnProposal = template.Must( - template.New("comment_reply_on_proposal").Parse(templateCommentReplyOnProposalRaw)) - templateCommentReplyOnComment = template.Must( - template.New("comment_reply_on_comment").Parse(templateCommentReplyOnCommentRaw)) -) - // wsContext is the websocket context. If uuid == "" then it is an // unauthenticated websocket. type wsContext struct { diff --git a/politeiawww/templates.go b/politeiawww/templates.go index 43ae4fee3..c8469301d 100644 --- a/politeiawww/templates.go +++ b/politeiawww/templates.go @@ -1,9 +1,122 @@ -// Copyright (c) 2017-2019 The Decred developers +// Copyright (c) 2017-2020 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. package main +import "text/template" + +var ( + templateProposalVoteStarted = template.Must( + template.New("proposal_vote_started_template").Parse(templateProposalVoteStartedRaw)) + templateProposalVoteAuthorized = template.Must( + template.New("proposal_vote_authorized_template").Parse(templateProposalVoteAuthorizedRaw)) + templateProposalVoteStartedForAuthor = template.Must( + template.New("proposal_vote_started_for_author_template").Parse(templateProposalVoteStartedForAuthorRaw)) + templateCommentReplyOnProposal = template.Must( + template.New("comment_reply_on_proposal").Parse(templateCommentReplyOnProposalRaw)) + templateCommentReplyOnComment = template.Must( + template.New("comment_reply_on_comment").Parse(templateCommentReplyOnCommentRaw)) +) + +// Proposal submitted +type tmplDataProposalSubmitted struct { + Username string // Author username + Name string // Proposal name + Link string // GUI proposal details url +} + +const tmplTextProposalSubmitted = ` +A new proposal has been submitted on Politeia by {{.Username}}: + +{{.Name}} +{{.Link}} +` + +var tmplProposalSubmitted = template.Must( + template.New("proposal_submitted"). + Parse(tmplTextProposalSubmitted)) + +// Proposal edited +type tmplDataProposalEdited struct { + Name string // Proposal name + Version string // ProposalVersion + Username string // Author username + Link string // GUI proposal details url +} + +const tmplTextProposalEdited = ` +A proposal by {{.Username}} has just been edited: + +{{.Name}} (Version: {{.Version}}) +{{.Link}} +` + +var tmplProposalEdited = template.Must( + template.New("proposal_edited"). + Parse(tmplTextProposalEdited)) + +// Proposal status change - Vetted - Send to author +type tmplDataProposalVettedForAuthor struct { + Name string // Proposal name + Link string // GUI proposal details url +} + +const tmplTextProposalVettedForAuthor = ` +Your proposal has just been approved on Politeia! + +You will need to authorize a proposal vote before an administrator will be +allowed to start the voting period on your proposal. You can authorize a +proposal vote by opening the proposal page and clicking on the "Authorize +Voting to Start" button. + +You should allow sufficient time for the community to discuss your proposal +before authorizing the vote. + +{{.Name}} +{{.Link}} +` + +var tmplProposalVettedForAuthor = template.Must( + template.New("proposal_vetted_for_author"). + Parse(tmplTextProposalVettedForAuthor)) + +// Proposal status change - Censored - Send to author +type tmplDataProposalCensoredForAuthor struct { + Name string // Proposal name + Reason string // Reason for censoring + Link string // GUI proposal details url +} + +const tmplTextProposalCensoredForAuthor = ` +Your proposal on Politeia has been censored: + +{{.Name}} +Reason: {{.Reason}} +{{.Link}} +` + +var tmplProposalCensoredForAuthor = template.Must( + template.New("proposal_censored_for_author"). + Parse(tmplTextProposalCensoredForAuthor)) + +// Proposal status change - Vetted - Send to users +type tmplDataProposalVetted struct { + Name string + Link string +} + +const tmplTextProposalVetted = ` +A new proposal has just been published on Politeia. + +{{.Name}} +{{.Link}} +` + +var tmplProposalVetted = template.Must( + template.New("proposal_vetted"). + Parse(tmplTextProposalVetted)) + type invoiceNotificationEmailData struct { Username string Month string @@ -46,32 +159,12 @@ type userPasswordChangedTemplateData struct { Email string } -type proposalSubmittedTemplateData struct { - Link string // GUI proposal details url - Name string // Proposal name - Username string // Author username -} - -type proposalEditedTemplateData struct { - Link string // GUI proposal details url - Name string // Proposal name - Version string // ProposalVersion - Username string // Author username -} - type proposalVoteStartedTemplateData struct { Link string // GUI proposal details url Name string // Proposal name Username string // Author username } -type proposalStatusChangeTemplateData struct { - Link string // GUI proposal details url - Name string // Proposal name - Username string // Author username - StatusChangeReason string // Proposal status change reason -} - type proposalVoteAuthorizedTemplateData struct { Link string // GUI proposal details url Name string // Proposal name @@ -155,27 +248,6 @@ You are receiving this email because someone made too many login attempts for {{.Email}} on Politeia. If that was not you, please notify Politeia administrators. ` -const templateProposalSubmittedRaw = ` -A new proposal has been submitted on Politeia by {{.Username}}: - -{{.Name}} -{{.Link}} -` - -const templateProposalVettedRaw = ` -A new proposal has just been approved on Politeia, authored by {{.Username}}: - -{{.Name}} -{{.Link}} -` - -const templateProposalEditedRaw = ` -A proposal by {{.Username}} has just been edited: - -{{.Name}} (Version: {{.Version}}) -{{.Link}} -` - const templateProposalVoteStartedRaw = ` Voting has started for the following proposal on Politeia, authored by {{.Username}}: @@ -190,29 +262,6 @@ Voting has been authorized for the following proposal on Politeia by {{.Username {{.Link}} ` -const templateProposalVettedForAuthorRaw = ` -Your proposal has just been approved on Politeia! - -You will need to authorize a proposal vote before an administrator will be -allowed to start the voting period on your proposal. You can authorize a -proposal vote by opening the proposal page and clicking on the "Authorize -Voting to Start" button. - -You must authorize a proposal vote within 14 days. If you fail to do so, your -proposal will be considered abandoned. - -{{.Name}} -{{.Link}} -` - -const templateProposalCensoredForAuthorRaw = ` -Your proposal on Politeia has been censored: - -{{.Name}} -{{.Link}} -Reason: {{.StatusChangeReason}} -` - const templateProposalVoteStartedForAuthorRaw = ` Voting has just started for your proposal on Politeia! From 59102806233b3ac3e7c7f44717f0c8de9aa08767 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 14 Sep 2020 10:12:17 -0500 Subject: [PATCH 041/449] proposal route changes --- mdstream/mdstream.go | 114 --------------- plugins/comments/comments.go | 6 +- plugins/pi/pi.go | 56 ++++++-- plugins/ticketvote/ticketvote.go | 6 +- politeiad/backend/tlogbe/plugin.go | 4 +- politeiad/backend/tlogbe/plugins/comments.go | 58 ++++---- politeiad/backend/tlogbe/plugins/pi.go | 89 ++++++++++-- .../backend/tlogbe/plugins/ticketvote.go | 60 ++++---- politeiad/backend/tlogbe/tlogbe.go | 8 +- politeiawww/api/pi/v1/v1.go | 58 ++++---- politeiawww/piwww.go | 132 ++++++++++-------- politeiawww/testing.go | 41 ------ 12 files changed, 287 insertions(+), 345 deletions(-) diff --git a/mdstream/mdstream.go b/mdstream/mdstream.go index 85d305812..421c343a6 100644 --- a/mdstream/mdstream.go +++ b/mdstream/mdstream.go @@ -20,7 +20,6 @@ import ( const ( // mdstream IDs - IDProposalGeneral = 0 IDRecordStatusChange = 2 IDInvoiceGeneral = 3 IDInvoiceStatusChange = 4 @@ -29,12 +28,7 @@ const ( IDDCCStatusChange = 7 IDDCCSupportOpposition = 8 - // Note that 13 is in use by the decred plugin - // Note that 14 is in use by the decred plugin - // Note that 15 is in use by the decred plugin - // mdstream current supported versions - VersionProposalGeneral = 2 VersionRecordStatusChange = 2 VersionInvoiceGeneral = 1 VersionInvoiceStatusChange = 1 @@ -42,11 +36,6 @@ const ( VersionDCCGeneral = 1 VersionDCCStatusChange = 1 VersionDCCSupposeOpposition = 1 - - // Filenames of user defined metadata that is stored as politeiad - // files instead of politeiad metadata streams. This is done so - // that the metadata is included in the politeiad merkle root calc. - FilenameProposalMetadata = "proposalmetadata.json" ) // DecodeVersion returns the version of the provided mstream payload. This @@ -65,108 +54,6 @@ func DecodeVersion(payload []byte) (uint, error) { return version, nil } -// ProposalGeneralV1 represents general metadata for a proposal. -// -// Signature is the signature of the merkle root where the merkle root contains -// the proposal file payloads. -type ProposalGeneralV1 struct { - Version uint64 `json:"version"` // Struct version - Timestamp int64 `json:"timestamp"` // Last update of proposal - Name string `json:"name"` // Provided proposal name - PublicKey string `json:"publickey"` // Key used for signature - Signature string `json:"signature"` // Proposal signature -} - -// EncodeProposalGeneralV1 encodes a ProposalGeneralV1 into a JSON byte slice. -func EncodeProposalGeneralV1(md ProposalGeneralV1) ([]byte, error) { - b, err := json.Marshal(md) - if err != nil { - return nil, err - } - return b, nil -} - -// DecodeProposalGeneralV1 decodes a JSON byte slice into a ProposalGeneralV1. -func DecodeProposalGeneralV1(payload []byte) (*ProposalGeneralV1, error) { - var md ProposalGeneralV1 - err := json.Unmarshal(payload, &md) - if err != nil { - return nil, err - } - return &md, nil -} - -// ProposalGeneralV2 represents general metadata for a proposal. -// -// Signature is the signature of the proposal merkle root. The merkle root -// contains the ordered files and metadata digests. The file digests are first -// in the ordering. -// -// Differences between v1 and v2: -// * Name has been removed and is now part of proposal metadata. -// * Signature has been updated to include propoposal metadata. -type ProposalGeneralV2 struct { - Version uint64 `json:"version"` // Struct version - Timestamp int64 `json:"timestamp"` // Last update of proposal - PublicKey string `json:"publickey"` // Key used for signature - Signature string `json:"signature"` // Proposal signature -} - -// EncodeProposalGeneralV2 encodes a ProposalGeneralV2 into a JSON byte slice. -func EncodeProposalGeneralV2(md ProposalGeneralV2) ([]byte, error) { - b, err := json.Marshal(md) - if err != nil { - return nil, err - } - return b, nil -} - -// DecodeProposalGeneralV2 decodes a JSON byte slice into a ProposalGeneralV2. -func DecodeProposalGeneralV2(payload []byte) (*ProposalGeneralV2, error) { - var md ProposalGeneralV2 - err := json.Unmarshal(payload, &md) - if err != nil { - return nil, err - } - return &md, nil -} - -// ProposalMetadata contains metadata that is specified by the user on proposal -// submission. It is attached to a proposal submission as a politeiawww -// Metadata object and is saved to politeiad as a File, not as a -// MetadataStream. The filename is defined by FilenameProposalMetadata. -// -// The reason it is saved to politeiad as a File is because politeiad only -// includes Files in the merkle root calculation. This is user defined metadata -// so it must be included in the proposal signature on submission. If it were -// saved to politeiad as a MetadataStream then it would not be included in the -// merkle root, thus causing an error where the client calculated merkle root -// if different than the politeiad calculated merkle root. -type ProposalMetadata struct { - Name string `json:"name"` // Proposal name - LinkTo string `json:"linkto,omitempty"` // Token of proposal to link to - LinkBy int64 `json:"linkby,omitempty"` // UNIX timestamp of RFP deadline -} - -// EncodeProposalMetadata encodes a ProposalMetadata into a JSON byte slice. -func EncodeProposalMetadata(md ProposalMetadata) ([]byte, error) { - b, err := json.Marshal(md) - if err != nil { - return nil, err - } - return b, nil -} - -// DecodeProposalMetadata decodes a JSON byte slice into a ProposalMetadata. -func DecodeProposalMetadata(payload []byte) (*ProposalMetadata, error) { - var md ProposalMetadata - err := json.Unmarshal(payload, &md) - if err != nil { - return nil, err - } - return &md, nil -} - // RecordStatusChangeV1 represents a politeiad record status change and is used // to store additional status change metadata that would not otherwise be // captured by the politeiad status change routes. @@ -514,7 +401,6 @@ func EncodeDCCSupportOpposition(md DCCSupportOpposition) ([]byte, error) { // DCCSupportOpposition. func DecodeDCCSupportOpposition(payload []byte) ([]DCCSupportOpposition, error) { var md []DCCSupportOpposition - d := json.NewDecoder(strings.NewReader(string(payload))) for { var m DCCSupportOpposition diff --git a/plugins/comments/comments.go b/plugins/comments/comments.go index a020c724f..05a72bcc8 100644 --- a/plugins/comments/comments.go +++ b/plugins/comments/comments.go @@ -73,14 +73,14 @@ var ( } ) -// UserError represents an error that is cause by the user. -type UserError struct { +// UserErrorReply represents an error that is cause by the user. +type UserErrorReply struct { ErrorCode ErrorStatusT ErrorContext []string } // Error satisfies the error interface. -func (e UserError) Error() string { +func (e UserErrorReply) Error() string { return fmt.Sprintf("comments plugin error code: %v", e.ErrorCode) } diff --git a/plugins/pi/pi.go b/plugins/pi/pi.go index 59d46890f..a02aa575f 100644 --- a/plugins/pi/pi.go +++ b/plugins/pi/pi.go @@ -9,6 +9,8 @@ package pi import ( "encoding/json" "fmt" + "io" + "strings" ) type PropStateT int @@ -47,10 +49,13 @@ const ( PropStatusAbandoned PropStatusT = 4 // Prop has been abandoned // User error status codes - ErrorStatusInvalid ErrorStatusT = 0 - ErrorStatusPropLinkToInvalid ErrorStatusT = 1 - ErrorStatusPropStatusInvalid ErrorStatusT = 2 - ErrorStatusVoteStatusInvalid ErrorStatusT = 3 + // TODO number error codes + ErrorStatusInvalid ErrorStatusT = iota + ErrorStatusPropVersionInvalid + ErrorStatusPropStatusInvalid + ErrorStatusPropStatusChangeInvalid + ErrorStatusPropLinkToInvalid + ErrorStatusVoteStatusInvalid ) var ( @@ -79,15 +84,14 @@ var ( } ) -// TODO change this to UserErrorReply as well as in all the other plugins. -// UserError represents an error that is caused by the user. -type UserError struct { +// UserErrorReply represents an error that is caused by the user. +type UserErrorReply struct { ErrorCode ErrorStatusT ErrorContext []string } // Error satisfies the error interface. -func (e UserError) Error() string { +func (e UserErrorReply) Error() string { return fmt.Sprintf("pi plugin error code: %v", e.ErrorCode) } @@ -114,7 +118,7 @@ func EncodeProposalMetadata(pm ProposalMetadata) ([]byte, error) { return json.Marshal(pm) } -// DecodeProposalMetadata decodes a ProposalMetadata into a JSON byte slice. +// DecodeProposalMetadata decodes a JSON byte slice into a ProposalMetadata. func DecodeProposalMetadata(payload []byte) (*ProposalMetadata, error) { var pm ProposalMetadata err := json.Unmarshal(payload, &pm) @@ -141,7 +145,7 @@ func EncodeProposalGeneral(pg ProposalGeneral) ([]byte, error) { return json.Marshal(pg) } -// DecodeProposalGeneral decodes a ProposalGeneral into a JSON byte slice. +// DecodeProposalGeneral decodes a JSON byte slice into a ProposalGeneral. func DecodeProposalGeneral(payload []byte) (*ProposalGeneral, error) { var pg ProposalGeneral err := json.Unmarshal(payload, &pg) @@ -169,7 +173,33 @@ func EncodeStatusChange(sc StatusChange) ([]byte, error) { return json.Marshal(sc) } -// TODO DecodeStatusChanges +// DecodeStatusChange decodes a JSON byte slice into a StatusChange. +func DecodeStatusChange(payload []byte) (*StatusChange, error) { + var sc StatusChange + err := json.Unmarshal(payload, &sc) + if err != nil { + return nil, err + } + return &sc, nil +} + +// DecodeStatusChanges decodes a JSON byte slice into a []StatusChange. +func DecodeStatusChanges(payload []byte) ([]StatusChange, error) { + var statuses []StatusChange + d := json.NewDecoder(strings.NewReader(string(payload))) + for { + var sc StatusChange + err := d.Decode(&sc) + if err == io.EOF { + break + } else if err != nil { + return nil, err + } + statuses = append(statuses, sc) + } + + return statuses, nil +} // Proposals requests the plugin data for the provided proposals. This includes // pi plugin data as well as other plugin data such as comment plugin data. @@ -184,7 +214,7 @@ func EncodeProposals(p Proposals) ([]byte, error) { return json.Marshal(p) } -// DecodeProposals decodes a Proposals into a JSON byte slice. +// DecodeProposals decodes a JSON byte slice into a Proposals. func DecodeProposals(payload []byte) (*Proposals, error) { var p Proposals err := json.Unmarshal(payload, &p) @@ -211,7 +241,7 @@ func EncodeProposalsReply(pr ProposalsReply) ([]byte, error) { return json.Marshal(pr) } -// DecodeProposalsReply decodes a ProposalsReply into a JSON byte slice. +// DecodeProposalsReply decodes a JSON byte slice into a ProposalsReply. func DecodeProposalsReply(payload []byte) (*ProposalsReply, error) { var pr ProposalsReply err := json.Unmarshal(payload, &pr) diff --git a/plugins/ticketvote/ticketvote.go b/plugins/ticketvote/ticketvote.go index 598217ea1..528a30c04 100644 --- a/plugins/ticketvote/ticketvote.go +++ b/plugins/ticketvote/ticketvote.go @@ -140,14 +140,14 @@ var ( } ) -// UserError represents an error that is caused by the user. -type UserError struct { +// UserErrorReply represents an error that is caused by the user. +type UserErrorReply struct { ErrorCode ErrorStatusT ErrorContext []string } // Error satisfies the error interface. -func (e UserError) Error() string { +func (e UserErrorReply) Error() string { return fmt.Sprintf("ticketvote plugin error code: %v", e.ErrorCode) } diff --git a/politeiad/backend/tlogbe/plugin.go b/politeiad/backend/tlogbe/plugin.go index d4c17a24e..7e4aca9df 100644 --- a/politeiad/backend/tlogbe/plugin.go +++ b/politeiad/backend/tlogbe/plugin.go @@ -65,7 +65,7 @@ func DecodeNewRecord(payload []byte) (*NewRecord, error) { // EditRecord is the payload for the EditRecordPre and EditRecordPost hooks. type EditRecord struct { // Current record - Record backend.Record `json:"record"` + Current backend.Record `json:"record"` // Updated fields RecordMetadata backend.RecordMetadata `json:"recordmetadata"` @@ -94,7 +94,7 @@ func DecodeEditRecord(payload []byte) (*EditRecord, error) { // HookSetRecordStatusPost hooks. type SetRecordStatus struct { // Current record - Record backend.Record `json:"record"` + Current backend.Record `json:"record"` // Updated fields RecordMetadata backend.RecordMetadata `json:"recordmetadata"` diff --git a/politeiad/backend/tlogbe/plugins/comments.go b/politeiad/backend/tlogbe/plugins/comments.go index c849e30b4..e68a83022 100644 --- a/politeiad/backend/tlogbe/plugins/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments.go @@ -99,7 +99,7 @@ func (p *commentsPlugin) mutex(token string) *sync.Mutex { return m } -func convertCommentsErrFromSignatureErr(err error) comments.UserError { +func convertCommentsErrFromSignatureErr(err error) comments.UserErrorReply { var e util.SignatureError var s comments.ErrorStatusT if errors.As(err, &e) { @@ -110,7 +110,7 @@ func convertCommentsErrFromSignatureErr(err error) comments.UserError { s = comments.ErrorStatusSignatureInvalid } } - return comments.UserError{ + return comments.UserErrorReply{ ErrorCode: s, ErrorContext: e.ErrorContext, } @@ -700,7 +700,7 @@ func (p *commentsPlugin) new(client *tlogbe.RecordClient, n comments.New, encryp // Verify parent comment exists if set. A parent ID of 0 means that // this is a base level comment, not a reply to another comment. if n.ParentID > 0 && !commentExists(*idx, n.ParentID) { - return nil, comments.UserError{ + return nil, comments.UserErrorReply{ ErrorCode: comments.ErrorStatusParentIDInvalid, ErrorContext: []string{"parent ID comment not found"}, } @@ -770,14 +770,14 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { // Get record client token, err := hex.DecodeString(n.Token) if err != nil { - return "", comments.UserError{ + return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusTokenInvalid, } } client, err := p.backend.RecordClient(token) if err != nil { if err == backend.ErrRecordNotFound { - return "", comments.UserError{ + return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusRecordNotFound, } } @@ -831,7 +831,7 @@ func (p *commentsPlugin) edit(client *tlogbe.RecordClient, e comments.Edit, encr } existing, ok := cs[e.CommentID] if !ok { - return nil, comments.UserError{ + return nil, comments.UserErrorReply{ ErrorCode: comments.ErrorStatusCommentNotFound, } } @@ -841,13 +841,13 @@ func (p *commentsPlugin) edit(client *tlogbe.RecordClient, e comments.Edit, encr if e.ParentID != existing.ParentID { e := fmt.Sprintf("parent id cannot change; got %v, want %v", e.ParentID, existing.ParentID) - return nil, comments.UserError{ + return nil, comments.UserErrorReply{ ErrorCode: comments.ErrorStatusParentIDInvalid, ErrorContext: []string{e}, } } if e.Comment == existing.Comment { - return nil, comments.UserError{ + return nil, comments.UserErrorReply{ ErrorCode: comments.ErrorStatusNoCommentChanges, } } @@ -910,14 +910,14 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { // Get record client token, err := hex.DecodeString(e.Token) if err != nil { - return "", comments.UserError{ + return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusTokenInvalid, } } client, err := p.backend.RecordClient(token) if err != nil { if err == backend.ErrRecordNotFound { - return "", comments.UserError{ + return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusRecordNotFound, } } @@ -971,7 +971,7 @@ func (p *commentsPlugin) del(client *tlogbe.RecordClient, d comments.Del) (*comm } comment, ok := cs[d.CommentID] if !ok { - return nil, comments.UserError{ + return nil, comments.UserErrorReply{ ErrorCode: comments.ErrorStatusCommentNotFound, } } @@ -1043,14 +1043,14 @@ func (p *commentsPlugin) cmdDel(payload string) (string, error) { // Get record client token, err := hex.DecodeString(d.Token) if err != nil { - return "", comments.UserError{ + return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusTokenInvalid, } } client, err := p.backend.RecordClient(token) if err != nil { if err == backend.ErrRecordNotFound { - return "", comments.UserError{ + return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusRecordNotFound, } } @@ -1090,14 +1090,14 @@ func (p *commentsPlugin) cmdGet(payload string) (string, error) { // Get record client token, err := hex.DecodeString(g.Token) if err != nil { - return "", comments.UserError{ + return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusTokenInvalid, } } client, err := p.backend.RecordClient(token) if err != nil { if err == backend.ErrRecordNotFound { - return "", comments.UserError{ + return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusRecordNotFound, } } @@ -1140,14 +1140,14 @@ func (p *commentsPlugin) cmdGetAll(payload string) (string, error) { // Get record client token, err := hex.DecodeString(ga.Token) if err != nil { - return "", comments.UserError{ + return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusTokenInvalid, } } client, err := p.backend.RecordClient(token) if err != nil { if err == backend.ErrRecordNotFound { - return "", comments.UserError{ + return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusRecordNotFound, } } @@ -1207,14 +1207,14 @@ func (p *commentsPlugin) cmdGetVersion(payload string) (string, error) { // Get record client token, err := hex.DecodeString(gv.Token) if err != nil { - return "", comments.UserError{ + return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusTokenInvalid, } } client, err := p.backend.RecordClient(token) if err != nil { if err == backend.ErrRecordNotFound { - return "", comments.UserError{ + return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusRecordNotFound, } } @@ -1230,12 +1230,12 @@ func (p *commentsPlugin) cmdGetVersion(payload string) (string, error) { // Verify comment exists cidx, ok := idx.Comments[gv.CommentID] if !ok { - return "", comments.UserError{ + return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusCommentNotFound, } } if cidx.Del != nil { - return "", comments.UserError{ + return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusCommentNotFound, ErrorContext: []string{"comment has been deleted"}, } @@ -1244,7 +1244,7 @@ func (p *commentsPlugin) cmdGetVersion(payload string) (string, error) { if !ok { e := fmt.Sprintf("comment %v does not have version %v", gv.CommentID, gv.Version) - return "", comments.UserError{ + return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusCommentNotFound, ErrorContext: []string{e}, } @@ -1288,14 +1288,14 @@ func (p *commentsPlugin) cmdCount(payload string) (string, error) { // Get record client token, err := hex.DecodeString(c.Token) if err != nil { - return "", comments.UserError{ + return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusTokenInvalid, } } client, err := p.backend.RecordClient(token) if err != nil { if err == backend.ErrRecordNotFound { - return "", comments.UserError{ + return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusRecordNotFound, } } @@ -1336,7 +1336,7 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { case comments.VoteUpvote: // This is allowed default: - return "", comments.UserError{ + return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusVoteInvalid, } } @@ -1352,14 +1352,14 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { // Get record client token, err := hex.DecodeString(v.Token) if err != nil { - return "", comments.UserError{ + return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusTokenInvalid, } } client, err := p.backend.RecordClient(token) if err != nil { if err == backend.ErrRecordNotFound { - return "", comments.UserError{ + return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusRecordNotFound, } } @@ -1381,7 +1381,7 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { // Verify comment exists cidx, ok := idx.Comments[v.CommentID] if !ok { - return "", comments.UserError{ + return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusCommentNotFound, } } @@ -1392,7 +1392,7 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { uvotes = make([]voteIndex, 0) } if len(uvotes) > comments.PolicyMaxVoteChanges { - return "", comments.UserError{ + return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusMaxVoteChanges, } } diff --git a/politeiad/backend/tlogbe/plugins/pi.go b/politeiad/backend/tlogbe/plugins/pi.go index ebe335432..4728c0d6c 100644 --- a/politeiad/backend/tlogbe/plugins/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi.go @@ -52,7 +52,7 @@ func isRFP(pm pi.ProposalMetadata) bool { func proposalMetadataFromFiles(files []backend.File) (*pi.ProposalMetadata, error) { var pm *pi.ProposalMetadata for _, v := range files { - if v.Name == pi.FilenameProposalMetadata { + if v.Name == pi.FileNameProposalMetadata { b, err := base64.StdEncoding.DecodeString(v.Payload) if err != nil { return nil, err @@ -185,7 +185,7 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { // Decode ProposalMetadata var pm *pi.ProposalMetadata for _, v := range nr.Files { - if v.Name == pi.FilenameProposalMetadata { + if v.Name == pi.FileNameProposalMetadata { b, err := base64.StdEncoding.DecodeString(v.Payload) if err != nil { return err @@ -206,14 +206,14 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { // have been approved by a ticket vote. if pm.LinkTo != "" { if isRFP(*pm) { - return pi.UserError{ + return pi.UserErrorReply{ ErrorCode: pi.ErrorStatusPropLinkToInvalid, ErrorContext: []string{"an rfp cannot have linkto set"}, } } tokenb, err := hex.DecodeString(pm.LinkTo) if err != nil { - return pi.UserError{ + return pi.UserErrorReply{ ErrorCode: pi.ErrorStatusPropLinkToInvalid, ErrorContext: []string{"invalid hex"}, } @@ -221,7 +221,7 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { r, err := p.backend.GetVetted(tokenb, "") if err != nil { if err == backend.ErrRecordNotFound { - return pi.UserError{ + return pi.UserErrorReply{ ErrorCode: pi.ErrorStatusPropLinkToInvalid, ErrorContext: []string{"proposal not found"}, } @@ -233,20 +233,20 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { return err } if linkToPM == nil { - return pi.UserError{ + return pi.UserErrorReply{ ErrorCode: pi.ErrorStatusPropLinkToInvalid, ErrorContext: []string{"proposal not an rfp"}, } } if !isRFP(*linkToPM) { - return pi.UserError{ + return pi.UserErrorReply{ ErrorCode: pi.ErrorStatusPropLinkToInvalid, ErrorContext: []string{"proposal not an rfp"}, } } if time.Now().Unix() > linkToPM.LinkBy { // Link by deadline has expired. New links are not allowed. - return pi.UserError{ + return pi.UserErrorReply{ ErrorCode: pi.ErrorStatusPropLinkToInvalid, ErrorContext: []string{"rfp link by deadline expired"}, } @@ -273,7 +273,7 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { return fmt.Errorf("summary not found %v", pm.LinkTo) } if !summary.Approved { - return pi.UserError{ + return pi.UserErrorReply{ ErrorCode: pi.ErrorStatusPropLinkToInvalid, ErrorContext: []string{"rfp vote not approved"}, } @@ -308,9 +308,9 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { // politeiad will also error if no files were changed. // Verify proposal status - status := convertPropStatusFromMDStatus(er.Record.RecordMetadata.Status) + status := convertPropStatusFromMDStatus(er.Current.RecordMetadata.Status) if status != pi.PropStatusUnvetted && status != pi.PropStatusPublic { - return pi.UserError{ + return pi.UserErrorReply{ ErrorCode: pi.ErrorStatusPropStatusInvalid, } } @@ -341,7 +341,7 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { e := fmt.Sprintf("vote status got %v, want %v", ticketvote.VoteStatus[summary.Status], ticketvote.VoteStatus[ticketvote.VoteStatusUnauthorized]) - return pi.UserError{ + return pi.UserErrorReply{ ErrorCode: pi.ErrorStatusVoteStatusInvalid, ErrorContext: []string{e}, } @@ -351,7 +351,7 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { // public proposal. Unvetted proposals are allowed to change their // linkto. if status == pi.PropStatusPublic { - pmCurr, err := proposalMetadataFromFiles(er.Record.Files) + pmCurr, err := proposalMetadataFromFiles(er.Current.Files) if err != nil { return err } @@ -360,7 +360,7 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { return err } if pmCurr.LinkTo != pmNew.LinkTo { - return pi.UserError{ + return pi.UserErrorReply{ ErrorCode: pi.ErrorStatusPropLinkToInvalid, ErrorContext: []string{"linkto cannot change on public proposal"}, } @@ -376,10 +376,69 @@ func (p *piPlugin) hookSetRecordStatusPost(payload string) error { return err } + // Parse the status change metadata + var sc *pi.StatusChange + for _, v := range srs.MDAppend { + if v.ID != pi.MDStreamIDStatusChanges { + continue + } + + sc, err = pi.DecodeStatusChange([]byte(v.Payload)) + if err != nil { + return err + } + break + } + if sc == nil { + return fmt.Errorf("status change append metadata not found") + } + + // Parse the existing status changes metadata stream + var statuses []pi.StatusChange + for _, v := range srs.Current.Metadata { + if v.ID != pi.MDStreamIDStatusChanges { + continue + } + + statuses, err = pi.DecodeStatusChanges([]byte(v.Payload)) + if err != nil { + return err + } + break + } + + // Verify version is the latest version + if sc.Version != srs.Current.Version { + e := fmt.Sprintf("version not current: got %v, want %v", + sc.Version, srs.Current.Version) + return pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusPropVersionInvalid, + ErrorContext: []string{e}, + } + } + + // Verify status change is allowed + var from pi.PropStatusT + if len(statuses) == 0 { + // No previous status changes exist. Proposal is unvetted. + from = pi.PropStatusUnvetted + } else { + from = statuses[len(statuses)-1].Status + } + _, isAllowed := pi.StatusChanges[from][sc.Status] + if !isAllowed { + e := fmt.Sprintf("from %v to %v status change not allowed", + from, sc.Status) + return pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusPropStatusChangeInvalid, + ErrorContext: []string{e}, + } + } + // If the LinkTo field has been set then the linkedFrom // list might need to be updated for the proposal that is being // linked to, depending on the status change that is being made. - pm, err := proposalMetadataFromFiles(srs.Record.Files) + pm, err := proposalMetadataFromFiles(srs.Current.Files) if err != nil { return err } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote.go index fade7a159..1b843e758 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote.go @@ -243,7 +243,7 @@ func (p *ticketVotePlugin) mutex(token string) *sync.Mutex { return m } -func convertTicketVoteErrFromSignatureErr(err error) ticketvote.UserError { +func convertTicketVoteErrFromSignatureErr(err error) ticketvote.UserErrorReply { var e util.SignatureError var s ticketvote.ErrorStatusT if errors.As(err, &e) { @@ -254,7 +254,7 @@ func convertTicketVoteErrFromSignatureErr(err error) ticketvote.UserError { s = ticketvote.ErrorStatusSignatureInvalid } } - return ticketvote.UserError{ + return ticketvote.UserErrorReply{ ErrorCode: s, ErrorContext: e.ErrorContext, } @@ -883,21 +883,21 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { // Get record client tokenb, err := hex.DecodeString(a.Token) if err != nil { - return "", ticketvote.UserError{ + return "", ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusTokenInvalid, } } client, err := p.backend.RecordClient(tokenb) if err != nil { if err == backend.ErrRecordNotFound { - return "", ticketvote.UserError{ + return "", ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusRecordNotFound, } } return "", err } if client.State != tlogbe.RecordStateVetted { - return "", ticketvote.UserError{ + return "", ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusRecordStatusInvalid, ErrorContext: []string{"record not vetted"}, } @@ -908,7 +908,7 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { if err != nil { if err == backend.ErrRecordNotFound { e := fmt.Sprintf("version %v not found", version) - return "", ticketvote.UserError{ + return "", ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusRecordNotFound, ErrorContext: []string{e}, } @@ -923,7 +923,7 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { // This is allowed default: e := fmt.Sprintf("%v not a valid action", a.Action) - return "", ticketvote.UserError{ + return "", ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusAuthorizationInvalid, ErrorContext: []string{e}, } @@ -950,20 +950,20 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { case len(auths) == 0: // No previous actions. New action must be an authorize. if a.Action != ticketvote.ActionAuthorize { - return "", ticketvote.UserError{ + return "", ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusAuthorizationInvalid, ErrorContext: []string{"no prev action; action must be authorize"}, } } case prevAction == ticketvote.ActionAuthorize: // Previous action was a authorize. This action must be revoke. - return "", ticketvote.UserError{ + return "", ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusAuthorizationInvalid, ErrorContext: []string{"prev action was authorize"}, } case prevAction == ticketvote.ActionRevoke: // Previous action was a revoke. This action must be authorize. - return "", ticketvote.UserError{ + return "", ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusAuthorizationInvalid, ErrorContext: []string{"prev action was revoke"}, } @@ -1034,7 +1034,7 @@ func voteDetailsVerify(vote ticketvote.VoteDetails, voteDurationMin, voteDuratio // This is allowed default: e := fmt.Sprintf("invalid type %v", vote.Type) - return ticketvote.UserError{ + return ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, ErrorContext: []string{e}, } @@ -1045,28 +1045,28 @@ func voteDetailsVerify(vote ticketvote.VoteDetails, voteDurationMin, voteDuratio case vote.Duration > voteDurationMax: e := fmt.Sprintf("duration %v exceeds max duration %v", vote.Duration, voteDurationMax) - return ticketvote.UserError{ + return ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, ErrorContext: []string{e}, } case vote.Duration < voteDurationMin: e := fmt.Sprintf("duration %v under min duration %v", vote.Duration, voteDurationMin) - return ticketvote.UserError{ + return ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, ErrorContext: []string{e}, } case vote.QuorumPercentage > 100: e := fmt.Sprintf("quorum percent %v exceeds 100 percent", vote.QuorumPercentage) - return ticketvote.UserError{ + return ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, ErrorContext: []string{e}, } case vote.PassPercentage > 100: e := fmt.Sprintf("pass percent %v exceeds 100 percent", vote.PassPercentage) - return ticketvote.UserError{ + return ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, ErrorContext: []string{e}, } @@ -1075,7 +1075,7 @@ func voteDetailsVerify(vote ticketvote.VoteDetails, voteDurationMin, voteDuratio // Verify vote options. Different vote types have different // requirements. if len(vote.Options) == 0 { - return ticketvote.UserError{ + return ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, ErrorContext: []string{"no vote options found"}, } @@ -1088,7 +1088,7 @@ func voteDetailsVerify(vote ticketvote.VoteDetails, voteDurationMin, voteDuratio if len(vote.Options) != 2 { e := fmt.Sprintf("vote options count got %v, want 2", len(vote.Options)) - return ticketvote.UserError{ + return ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, ErrorContext: []string{e}, } @@ -1116,7 +1116,7 @@ func voteDetailsVerify(vote ticketvote.VoteDetails, voteDurationMin, voteDuratio if len(missing) > 0 { e := fmt.Sprintf("vote option IDs not found: %v", strings.Join(missing, ",")) - return ticketvote.UserError{ + return ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, ErrorContext: []string{e}, } @@ -1127,7 +1127,7 @@ func voteDetailsVerify(vote ticketvote.VoteDetails, voteDurationMin, voteDuratio for _, v := range vote.Options { err := voteBitVerify(vote.Options, vote.Mask, v.Bit) if err != nil { - return ticketvote.UserError{ + return ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, ErrorContext: []string{err.Error()}, } @@ -1166,21 +1166,21 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { // Get record client tokenb, err := hex.DecodeString(s.Vote.Token) if err != nil { - return "", ticketvote.UserError{ + return "", ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusTokenInvalid, } } client, err := p.backend.RecordClient(tokenb) if err != nil { if err == backend.ErrRecordNotFound { - return "", ticketvote.UserError{ + return "", ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusRecordNotFound, } } return "", err } if client.State != tlogbe.RecordStateVetted { - return "", ticketvote.UserError{ + return "", ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusRecordStatusInvalid, ErrorContext: []string{"record not vetted"}, } @@ -1192,7 +1192,7 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { if err != nil { if err == backend.ErrRecordNotFound { e := fmt.Sprintf("version %v not found", version) - return "", ticketvote.UserError{ + return "", ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusRecordNotFound, ErrorContext: []string{e}, } @@ -1205,14 +1205,14 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { return "", err } if len(auths) == 0 { - return "", ticketvote.UserError{ + return "", ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusAuthorizationInvalid, ErrorContext: []string{"authorization not found"}, } } action := ticketvote.ActionT(auths[len(auths)-1].Action) if action != ticketvote.ActionAuthorize { - return "", ticketvote.UserError{ + return "", ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusAuthorizationInvalid, ErrorContext: []string{"not authorized"}, } @@ -1238,7 +1238,7 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { } if svp != nil { // Vote has already been started - return "", ticketvote.UserError{ + return "", ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusVoteStatusInvalid, ErrorContext: []string{"vote already started"}, } @@ -1542,14 +1542,14 @@ func (p *ticketVotePlugin) cmdDetails(payload string) (string, error) { // Get record client tokenb, err := hex.DecodeString(d.Token) if err != nil { - return "", ticketvote.UserError{ + return "", ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusTokenInvalid, } } client, err := p.backend.RecordClient(tokenb) if err != nil { if err == backend.ErrRecordNotFound { - return "", ticketvote.UserError{ + return "", ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusRecordNotFound, } } @@ -1593,14 +1593,14 @@ func (p *ticketVotePlugin) cmdCastVotes(payload string) (string, error) { // Get record client tokenb, err := hex.DecodeString(cv.Token) if err != nil { - return "", ticketvote.UserError{ + return "", ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusTokenInvalid, } } client, err := p.backend.RecordClient(tokenb) if err != nil { if err == backend.ErrRecordNotFound { - return "", ticketvote.UserError{ + return "", ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusRecordNotFound, } } diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index c16ae160c..5339f5aea 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -598,7 +598,7 @@ func (t *TlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ // Call pre plugin hooks er := EditRecord{ - Record: *r, + Current: *r, RecordMetadata: *recordMD, MDAppend: mdAppend, MDOverwrite: mdOverwrite, @@ -698,7 +698,7 @@ func (t *TlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b // Call pre plugin hooks er := EditRecord{ - Record: *r, + Current: *r, RecordMetadata: *recordMD, MDAppend: mdAppend, MDOverwrite: mdOverwrite, @@ -1097,7 +1097,7 @@ func (t *TlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, // Call pre plugin hooks srs := SetRecordStatus{ - Record: *r, + Current: *r, RecordMetadata: rm, MDAppend: mdAppend, MDOverwrite: mdOverwrite, @@ -1222,7 +1222,7 @@ func (t *TlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md // Call pre plugin hooks srs := SetRecordStatus{ - Record: *r, + Current: *r, RecordMetadata: rm, MDAppend: mdAppend, MDOverwrite: mdOverwrite, diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index b7dd21256..0353cb140 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -95,35 +95,34 @@ const ( ErrorStatusSignatureInvalid ErrorStatusT = 101 // Proposal errors - ErrorStatusFileCountInvalid ErrorStatusT = 202 - ErrorStatusFileNameInvalid ErrorStatusT = 203 - ErrorStatusFileMIMEInvalid ErrorStatusT = 204 - ErrorStatusFileDigestInvalid ErrorStatusT = 205 - ErrorStatusFilePayloadInvalid ErrorStatusT = 206 - ErrorStatusIndexFileNameInvalid ErrorStatusT = 207 - ErrorStatusIndexFileCountInvalid ErrorStatusT = 207 - ErrorStatusIndexFileSizeInvalid ErrorStatusT = 208 - ErrorStatusTextFileCountInvalid ErrorStatusT = 209 - ErrorStatusImageFileCountInvalid ErrorStatusT = 210 - ErrorStatusImageFileSizeInvalid ErrorStatusT = 211 - ErrorStatusMetadataCountInvalid ErrorStatusT = 212 - ErrorStatusMetadataHintInvalid ErrorStatusT = 213 - ErrorStatusMetadataDigestInvalid ErrorStatusT = 214 - ErrorStatusMetadataPayloadInvalid ErrorStatusT = 215 - ErrorStatusPropNameInvalid ErrorStatusT = 216 - ErrorStatusPropLinkToInvalid ErrorStatusT = 217 - ErrorStatusPropLinkByInvalid ErrorStatusT = 218 - - // TODO make normal - ErrorStatusPropTokenInvalid ErrorStatusT = iota - ErrorStatusPropNotFound - ErrorStatusPropStateInvalid - ErrorStatusPropStatusInvalid - ErrorStatusPropStatusChangeInvalid - ErrorStatusPropStatusChangeReasonInvalid + ErrorStatusFileCountInvalid ErrorStatusT = 202 + ErrorStatusFileNameInvalid ErrorStatusT = 203 + ErrorStatusFileMIMEInvalid ErrorStatusT = 204 + ErrorStatusFileDigestInvalid ErrorStatusT = 205 + ErrorStatusFilePayloadInvalid ErrorStatusT = 206 + ErrorStatusIndexFileNameInvalid ErrorStatusT = 207 + ErrorStatusIndexFileCountInvalid ErrorStatusT = 207 + ErrorStatusIndexFileSizeInvalid ErrorStatusT = 208 + ErrorStatusTextFileCountInvalid ErrorStatusT = 209 + ErrorStatusImageFileCountInvalid ErrorStatusT = 210 + ErrorStatusImageFileSizeInvalid ErrorStatusT = 211 + ErrorStatusMetadataCountInvalid ErrorStatusT = 212 + ErrorStatusMetadataHintInvalid ErrorStatusT = 213 + ErrorStatusMetadataDigestInvalid ErrorStatusT = 214 + ErrorStatusMetadataPayloadInvalid ErrorStatusT = 215 + ErrorStatusPropNameInvalid ErrorStatusT = 216 + ErrorStatusPropLinkToInvalid ErrorStatusT = 217 + ErrorStatusPropLinkByInvalid ErrorStatusT = 218 + ErrorStatusPropTokenInvalid ErrorStatusT = 219 + ErrorStatusPropNotFound ErrorStatusT = 220 + ErrorStatusPropStateInvalid ErrorStatusT = 221 + ErrorStatusPropStatusInvalid ErrorStatusT = 222 + ErrorStatusPropStatusChangeInvalid ErrorStatusT = 223 + ErrorStatusPropStatusChangeReasonInvalid ErrorStatusT = 224 // Comment errors - ErrorStatusCommentTextInvalid + // TODO make normal + ErrorStatusCommentTextInvalid ErrorStatusT = iota ErrorStatusCommentParentIDInvalid ErrorStatusCommentVoteInvalid ErrorStatusCommentNotFound @@ -227,9 +226,10 @@ type CensorshipRecord struct { // // Signature is the client signature of the Token+Version+Status+Reason. type StatusChange struct { - Status PropStatusT `json:"status"` + Token string `json:"token"` Version string `json:"version"` - Message string `json:"message,omitempty"` + Status PropStatusT `json:"status"` + Reason string `json:"message,omitempty"` PublicKey string `json:"publickey"` Signature string `json:"signature"` Timestamp int64 `json:"timestamp"` diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index f032807d1..015833348 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -26,7 +26,8 @@ import ( "github.com/decred/politeia/util" ) -// TODO use pi policies +// TODO use pi policies. Should the policies be defined in the pi plugin +// or the pi api spec? const ( // MIME types @@ -106,6 +107,27 @@ func proposalNameRegex() string { return validProposalNameBuffer.String() } +// proposalName parses the proposal name from the ProposalMetadata and returns +// it. An empty string will be returned if any errors occur or if a name is not +// found. +func proposalName(pr pi.ProposalRecord) string { + var name string + for _, v := range pr.Metadata { + if v.Hint == pi.HintProposalMetadata { + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return "" + } + pm, err := piplugin.DecodeProposalMetadata(b) + if err != nil { + return "" + } + name = pm.Name + } + } + return name +} + func convertUserErrFromSignatureErr(err error) pi.UserErrorReply { var e util.SignatureError var s pi.ErrorStatusT @@ -240,9 +262,9 @@ func convertFilesFromPD(f []pd.File) ([]pi.File, []pi.Metadata) { func convertProposalRecordFromPD(r pd.Record) (*pi.ProposalRecord, error) { // Decode metadata streams var ( - pg *piplugin.ProposalGeneral - statuses = make([]pi.StatusChange, 0, 16) - err error + pg *piplugin.ProposalGeneral + sc = make([]piplugin.StatusChange, 0, 16) + err error ) for _, v := range r.Metadata { switch v.ID { @@ -252,18 +274,35 @@ func convertProposalRecordFromPD(r pd.Record) (*pi.ProposalRecord, error) { return nil, err } case piplugin.MDStreamIDStatusChanges: - // TODO decode status changes + sc, err = piplugin.DecodeStatusChanges([]byte(v.Payload)) + if err != nil { + return nil, err + } } } - // Convert files and status + // Convert to pi types files, metadata := convertFilesFromPD(r.Files) status := convertPropStatusFromPD(r.Status) state := convertPropStateFromPropStatus(status) + statuses := make([]pi.StatusChange, 0, len(sc)) + for _, v := range sc { + statuses = append(statuses, pi.StatusChange{ + Token: v.Token, + Version: v.Version, + Status: pi.PropStatusT(v.Status), + Reason: v.Reason, + PublicKey: v.PublicKey, + Signature: v.Signature, + Timestamp: v.Timestamp, + }) + } + // Some fields are intentionally omitted because they are either // user data that needs to be pulled from the user database or they - // are plugin data that needs to be retrieved using a plugin cmd. + // are politeiad plugin data that needs to be retrieved using a + // plugin command. return &pi.ProposalRecord{ Version: r.Version, Timestamp: pg.Timestamp, @@ -282,27 +321,6 @@ func convertProposalRecordFromPD(r pd.Record) (*pi.ProposalRecord, error) { }, nil } -// parseProposalName parsed the proposal name from the ProposalMetadata and -// returns it. An empty string will be returned if any errors occur or if a -// name is not found. -func parseProposalName(pr pi.ProposalRecord) string { - var name string - for _, v := range pr.Metadata { - if v.Hint == pi.HintProposalMetadata { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return "" - } - pm, err := piplugin.DecodeProposalMetadata(b) - if err != nil { - return "" - } - name = pm.Name - } - } - return name -} - // linkByPeriodMin returns the minimum amount of time, in seconds, that the // LinkBy period must be set to. This is determined by adding 1 week onto the // minimum voting period so that RFP proposal submissions have at least one @@ -1111,13 +1129,12 @@ func (p *politeiawww) processProposalSetStatus(pss pi.ProposalSetStatus, usr use return nil, convertUserErrFromSignatureErr(err) } - // TODO // Verification that requires retrieving the existing proposal is - // done in the politeiad pi plugin hook. This includes: - // -Verify token corresponds to a proposal - // -Verify proposal state is correct - // -Verify version is the latest version - // -Verify status change is allowed + // done in politeiad. This includes: + // -Verify proposal exists (politeiad) + // -Verify proposal state is correct (politeiad) + // -Verify version is the latest version (politeiad pi plugin) + // -Verify status change is allowed (politeiad pi plugin) // Setup metadata timestamp := time.Now().Unix() @@ -1143,6 +1160,8 @@ func (p *politeiawww) processProposalSetStatus(pss pi.ProposalSetStatus, usr use mdOverwrite := []pd.MetadataStream{} // Send politeiad request + // TODO verify proposal not found error is returned when wrong + // token or state is used status := convertRecordStatusFromPropStatus(pss.Status) switch pss.State { case pi.PropStateUnvetted: @@ -1189,7 +1208,7 @@ func (p *politeiawww) processProposalSetStatus(pss pi.ProposalSetStatus, usr use } p.eventManager.emit(eventProposalStatusChange, dataProposalStatusChange{ - name: parseProposalName(*pr), + name: proposalName(*pr), token: pss.Token, status: pss.Status, reason: pss.Reason, @@ -1210,10 +1229,11 @@ reply: }, nil } -// processProposalsPublic retrieves and returns the proposal records for each -// of the provided proposal requests. This is a public route that removes all -// unvetted proposal files from unvetted proposals before returning them. -func (p *politeiawww) processProposalsPublic(ps pi.Proposals) (*pi.ProposalsReply, error) { +// processProposals retrieves and returns the proposal records for each of the +// provided proposal requests. If unvetted proposals are requested by a +// non-admin then the unvetted proposal files are removed before the proposal +// is returned. +func (p *politeiawww) processProposalsPublic(ps pi.Proposals, isAdmin bool) (*pi.ProposalsReply, error) { log.Tracef("processProposals: %v", ps.Requests) props, err := p.proposalRecords(ps.State, ps.Requests, ps.IncludeFiles) @@ -1221,30 +1241,18 @@ func (p *politeiawww) processProposalsPublic(ps pi.Proposals) (*pi.ProposalsRepl return nil, err } - // Strip unvetted proposals of their files before returning them. - for k, v := range props { - if v.State == pi.PropStateVetted { - continue + // Only admins are allowed to retrieve unvetted proposal files. + // Remove all unvetted proposal files and user defined metadata if + // the user is not an admin. + if !isAdmin { + for k, v := range props { + if v.State == pi.PropStateVetted { + continue + } + v.Files = []pi.File{} + v.Metadata = []pi.Metadata{} + props[k] = v } - v.Files = []pi.File{} - v.Metadata = []pi.Metadata{} - props[k] = v - } - - return &pi.ProposalsReply{ - Proposals: props, - }, nil -} - -// processProposalsAdmin retrieves and returns the proposal records for each of -// the provided proposal requests. This is an admin route that returns unvetted -// proposals in their entirety. -func (p *politeiawww) processProposalsAdmin(ps pi.Proposals) (*pi.ProposalsReply, error) { - log.Tracef("processProposals: %v", ps.Requests) - - props, err := p.proposalRecords(ps.State, ps.Requests, ps.IncludeFiles) - if err != nil { - return nil, err } return &pi.ProposalsReply{ diff --git a/politeiawww/testing.go b/politeiawww/testing.go index a95d88fb1..7b7622da9 100644 --- a/politeiawww/testing.go +++ b/politeiawww/testing.go @@ -24,7 +24,6 @@ import ( "github.com/decred/dcrd/chaincfg" "github.com/decred/dcrtime/merkle" "github.com/decred/politeia/decredplugin" - "github.com/decred/politeia/mdstream" pd "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/api/v1/mime" @@ -602,46 +601,6 @@ func makeProposalRFPSubmissions(t *testing.T, prs []*www.ProposalRecord, linkto } } -func convertPropToPD(t *testing.T, p www.ProposalRecord) pd.Record { - t.Helper() - - // Attach ProposalMetadata as a politeiad file - files := convertPropFilesFromWWW(p.Files) - for _, v := range p.Metadata { - switch v.Hint { - case www.HintProposalMetadata: - // TODO - // files = append(files, convertFileFromMetadata(v)) - } - } - - // Create a ProposalGeneralV2 mdstream - md, err := mdstream.EncodeProposalGeneralV2( - mdstream.ProposalGeneralV2{ - Version: mdstream.VersionProposalGeneral, - Timestamp: time.Now().Unix(), - PublicKey: p.PublicKey, - Signature: p.Signature, - }) - if err != nil { - t.Fatal(err) - } - - mdStreams := []pd.MetadataStream{{ - ID: mdstream.IDProposalGeneral, - Payload: string(md), - }} - - return pd.Record{ - Status: convertPropStatusFromWWW(p.Status), - Timestamp: p.Timestamp, - Version: p.Version, - Metadata: mdStreams, - CensorshipRecord: convertPropCensorFromWWW(p.CensorshipRecord), - Files: files, - } -} - // newTestPoliteiawww returns a new politeiawww context that is setup for // testing and a closure that cleans up the test environment when invoked. func newTestPoliteiawww(t *testing.T) (*politeiawww, func()) { From 04dc68ab5c436aa18e13cd8885fbb4520a31a0d3 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 14 Sep 2020 10:24:59 -0500 Subject: [PATCH 042/449] www route cleanup --- politeiawww/api/www/v1/v1.go | 10 +- politeiawww/proposals.go | 939 +---------------------------------- 2 files changed, 25 insertions(+), 924 deletions(-) diff --git a/politeiawww/api/www/v1/v1.go b/politeiawww/api/www/v1/v1.go index c874dad1f..d2c07a354 100644 --- a/politeiawww/api/www/v1/v1.go +++ b/politeiawww/api/www/v1/v1.go @@ -56,16 +56,20 @@ const ( RouteProposalPaywallDetails = "/proposals/paywall" RouteProposalPaywallPayment = "/proposals/paywallpayment" - // The following routes have been DEPRECATED. Use the pi v1 API. + // The following routes are to be DEPRECATED in the near future and + // should not be used. The pi v1 API should be used instead. + RouteActiveVote = "/proposals/activevote" + RouteAllVetted = "/proposals/vetted" + + // The following routes have been DEPRECATED. The pi v1 API should + // be used instead. RouteTokenInventory = "/proposals/tokeninventory" RouteBatchProposals = "/proposals/batch" RouteBatchVoteSummary = "/proposals/batchvotesummary" - RouteAllVetted = "/proposals/vetted" RouteNewProposal = "/proposals/new" RouteEditProposal = "/proposals/edit" RouteAuthorizeVote = "/proposals/authorizevote" RouteStartVote = "/proposals/startvote" - RouteActiveVote = "/proposals/activevote" RouteCastVotes = "/proposals/castvotes" RouteAllVoteStatus = "/proposals/votestatus" RouteProposalDetails = "/proposals/{token:[A-Fa-f0-9]{7,64}}" diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 0f91b1e24..9f4a64d14 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -138,201 +138,19 @@ func voteStatusFromVoteSummary(r decredplugin.VoteSummaryReply, endHeight, bestB return www.PropVoteStatusInvalid } -// fillProposalMissingFields populates a ProposalRecord struct with the fields -// that are not stored in the cache. -func (p *politeiawww) fillProposalMissingFields(pr *www.ProposalRecord) error { - // Find the number of comments for the proposal - nc, err := p.decredGetNumComments([]string{pr.CensorshipRecord.Token}) - if err != nil { - return err - } - pr.NumComments = uint(nc[pr.CensorshipRecord.Token]) - - // Fill in proposal author info - u, err := p.db.UserGetByPubKey(pr.PublicKey) - if err != nil { - return err - } - pr.UserId = u.ID.String() - pr.Username = u.Username - - /* - // TODO - // Find linked from proposals - lfr, err := p.decredLinkedFrom([]string{pr.CensorshipRecord.Token}) - if err != nil { - return err - } - linkedFrom, ok := lfr.LinkedFrom[pr.CensorshipRecord.Token] - if ok { - pr.LinkedFrom = linkedFrom - } - */ - - return nil -} - -// getProp gets the most recent verions of the given proposal from the cache -// then fills in any missing fields before returning the proposal. func (p *politeiawww) getProp(token string) (*www.ProposalRecord, error) { - log.Tracef("getProp: %v", token) - - /* - var r *cache.Record - var err error - if len(token) == www.TokenPrefixLength { - r, err = p.cache.RecordByPrefix(token) - } else { - r, err = p.cache.Record(token) - } - if err != nil { - return nil, err - } - - pr, err := convertPropFromCache(*r) - if err != nil { - return nil, err - } - */ - var pr *www.ProposalRecord - - err := p.fillProposalMissingFields(pr) - if err != nil { - return nil, err - } - - return pr, nil + return nil, nil } -// getProps returns a [token]www.ProposalRecord map for the provided list of -// censorship tokens. If a proposal is not found, the map will not include an -// entry for the corresponding censorship token. It is the responsibility of -// the caller to ensure that results are returned for all of the provided -// censorship tokens. func (p *politeiawww) getProps(tokens []string) (map[string]www.ProposalRecord, error) { - /* - log.Tracef("getProps: %v", tokens) - - // Get the proposals from the cache - records, err := p.cache.Records(tokens, true) - if err != nil { - return nil, err - } - - // Use pointers for now so the props can be easily updated - props := make(map[string]*www.ProposalRecord, len(records)) - for _, v := range records { - pr, err := convertPropFromCache(v) - if err != nil { - return nil, fmt.Errorf("convertPropFromCache %v: %v", - v.CensorshipRecord.Token, err) - } - props[v.CensorshipRecord.Token] = pr - } - - // Get the number of comments for each proposal. Comments - // are part of decred plugin so this must be fetched from - // the cache separately. - dnc, err := p.decredGetNumComments(tokens) - if err != nil { - return nil, err - } - for token, numComments := range dnc { - props[token].NumComments = uint(numComments) - } - - // Find linked from proposals - lfr, err := p.decredLinkedFrom(tokens) - if err != nil { - return nil, err - } - for token, linkedFrom := range lfr.LinkedFrom { - props[token].LinkedFrom = linkedFrom - } - - // Compile a list of unique proposal author pubkeys. These - // are needed to lookup the proposal author info. - pubKeys := make(map[string]struct{}) - for _, pr := range props { - if _, ok := pubKeys[pr.PublicKey]; !ok { - pubKeys[pr.PublicKey] = struct{}{} - } - } - - // Lookup proposal authors - pk := make([]string, 0, len(pubKeys)) - for k := range pubKeys { - pk = append(pk, k) - } - users, err := p.db.UsersGetByPubKey(pk) - if err != nil { - return nil, err - } - if len(users) != len(pubKeys) { - // A user is missing from the userdb for one - // or more public keys. We're in trouble! - notFound := make([]string, 0, len(pubKeys)) - for v := range pubKeys { - if _, ok := users[v]; !ok { - notFound = append(notFound, v) - } - } - e := fmt.Sprintf("users not found for pubkeys: %v", - strings.Join(notFound, ", ")) - panic(e) - } - - // Fill in proposal author info - for i, pr := range props { - props[i].UserId = users[pr.PublicKey].ID.String() - props[i].Username = users[pr.PublicKey].Username - } - - // Convert pointers to values - proposals := make(map[string]www.ProposalRecord, len(props)) - for token, pr := range props { - proposals[token] = *pr - } - - return proposals, nil - */ return nil, nil } -// getPropVersion gets a specific version of a proposal from the cache then -// fills in any misssing fields before returning the proposal. func (p *politeiawww) getPropVersion(token, version string) (*www.ProposalRecord, error) { - log.Tracef("getPropVersion: %v %v", token, version) - - /* - r, err := p.cache.RecordVersion(token, version) - if err != nil { - return nil, err - } - - pr, err := convertPropFromCache(*r) - if err != nil { - return nil, err - } - - err = p.fillProposalMissingFields(pr) - if err != nil { - return nil, err - } - - return pr, nil - */ - return nil, nil } -// getAllProps gets the latest version of all proposals from the cache then -// fills any missing fields before returning the proposals. func (p *politeiawww) getAllProps() ([]www.ProposalRecord, error) { - log.Tracef("getAllProps") - - // TODO getAllProps shouldn't exist - return nil, nil } @@ -425,6 +243,18 @@ func filterProps(filter proposalsFilter, all []www.ProposalRecord) []www.Proposa return proposals } +func (p *politeiawww) processProposalDetails(pd www.ProposalsDetails, u *user.User) (*www.ProposalDetailsReply, error) { + return nil, nil +} + +func (p *politeiawww) processBatchVoteSummary(bvs www.BatchVoteSummary) (*www.BatchVoteSummaryReply, error) { + return nil, nil +} + +func (p *politeiawww) processBatchProposals(bp www.BatchProposals, u *user.User) (*www.BatchProposalsReply, error) { + return nil, nil +} + // getUserProps gets the latest version of all proposals from the cache and // then filters the proposals according to the specified proposalsFilter, which // is required to contain a userID. In addition to a page of filtered user @@ -477,770 +307,33 @@ func (p *politeiawww) getUserProps(filter proposalsFilter) ([]www.ProposalRecord } func (p *politeiawww) getPropComments(token string) ([]www.Comment, error) { - log.Tracef("getPropComments: %v", token) - - dc, err := p.decredGetComments(token) - if err != nil { - return nil, fmt.Errorf("decredGetComments: %v", err) - } - - // Convert comments and fill in author info - comments := make([]www.Comment, 0, len(dc)) - for _, v := range dc { - c := convertCommentFromDecred(v) - u, err := p.db.UserGetByPubKey(c.PublicKey) - if err != nil { - log.Errorf("getPropComments: UserGetByPubKey: "+ - "token:%v commentID:%v pubKey:%v err:%v", - token, c.CommentID, c.PublicKey, err) - } else { - c.UserID = u.ID.String() - c.Username = u.Username - } - comments = append(comments, c) - } - - for i, v := range comments { - votes, err := p.getCommentVotes(v.Token, v.CommentID) - if err != nil { - return nil, err - } - comments[i].ResultVotes = int64(votes.up - votes.down) - comments[i].Upvotes = votes.up - comments[i].Downvotes = votes.down - } - - return comments, nil -} - -// voteSummariesGet returns a map[string]www.VoteSummary for the given proposal -// tokens. An entry in the returned map will only exist for tokens where a -// proposal record was found. -// -// This function must be called WITHOUT read/write lock held. -func (p *politeiawww) voteSummariesGet(tokens []string, bestBlock uint64) (map[string]www.VoteSummary, error) { - /* - voteSummaries := make(map[string]www.VoteSummary) - tokensToLookup := make([]string, 0, len(tokens)) - - p.RLock() - for _, token := range tokens { - vs, ok := p.voteSummaries[token] - if ok { - voteSummaries[token] = vs - } else { - tokensToLookup = append(tokensToLookup, token) - } - } - p.RUnlock() - - if len(tokensToLookup) == 0 { - return voteSummaries, nil - } - - // Fetch the vote summaries from the cache. This call relies on the - // lazy loaded VoteResults cache table. If the VoteResults table is - // not up-to-date then this function will load it before retrying - // the vote summary call. Since politeiawww only has read access to - // the cache, loading the VoteResults table requires using a - // politeiad decredplugin command. - var ( - done bool - err error - reply *decredplugin.BatchVoteSummaryReply - ) - for retries := 0; !done && retries <= 1; retries++ { - reply, err = p.decredBatchVoteSummary(tokensToLookup, bestBlock) - if err != nil { - if err == cache.ErrRecordNotFound { - // There are missing entries in the VoteResults - // cache table. Load them. - _, err := p.decredLoadVoteResults(bestBlock) - if err != nil { - return nil, err - } - - // Retry the vote summaries call - continue - } - return nil, err - } - - done = true - } - - for token, v := range reply.Summaries { - results := convertVoteOptionResultsFromDecred(v.Results) - votet := convertVoteTypeFromDecred(v.Type) - - // An endHeight will not exist if the proposal has not gone - // up for vote yet. - var endHeight uint64 - if v.EndHeight != "" { - i, err := strconv.ParseUint(v.EndHeight, 10, 64) - if err != nil { - return nil, fmt.Errorf("failed to parse end height "+ - "'%v' for %v: %v", v.EndHeight, token, err) - } - endHeight = i - } - - vs := www.VoteSummary{ - Status: voteStatusFromVoteSummary(v, endHeight, bestBlock), - Type: www.VoteT(int(votet)), - Approved: v.Approved, - EligibleTickets: uint32(v.EligibleTicketCount), - Duration: v.Duration, - EndHeight: endHeight, - QuorumPercentage: v.QuorumPercentage, - PassPercentage: v.PassPercentage, - Results: results, - } - - voteSummaries[token] = vs - - // If the voting period has ended the vote status - // is not going to change so add it to the memory - // cache. - if vs.Status == www.PropVoteStatusFinished { - p.voteSummarySet(token, vs) - } - } - - return voteSummaries, nil - */ - return nil, nil } -// voteSummaryGet stores the provided VoteSummary in the vote summaries memory -// cache. This is to only be used for proposals whose voting period has ended -// so that we don't have to worry about cache invalidation issues. -// -// This function must be called WITHOUT the read/write lock held. -func (p *politeiawww) voteSummarySet(token string, voteSummary www.VoteSummary) { - p.Lock() - defer p.Unlock() - - // p.voteSummaries[token] = voteSummary -} - -// voteSummaryGet returns the VoteSummary for the given token. A cache -// ErrRecordNotFound error is returned if the token does actually not -// correspond to a proposal. -// -// This function must be called WITHOUT the read/write lock held. -func (p *politeiawww) voteSummaryGet(token string, bestBlock uint64) (*www.VoteSummary, error) { - s, err := p.voteSummariesGet([]string{token}, bestBlock) - if err != nil { - return nil, err - } - vs, ok := s[token] - if !ok { - // return nil, cache.ErrRecordNotFound - return nil, fmt.Errorf("record not found") - } - return &vs, nil -} - -// createProposalDetailsReply makes updates to a proposal record based on the -// user who made the request, and puts it into a ProposalDetailsReply. -func createProposalDetailsReply(prop *www.ProposalRecord, user *user.User) *www.ProposalDetailsReply { - // Vetted proposals are viewable by everyone. The contents of - // an unvetted proposal is only viewable by admins and the - // proposal author. Unvetted proposal metadata is viewable by - // everyone. - if prop.State == www.PropStateUnvetted { - var isAuthor bool - var isAdmin bool - // This is a public route so a user may not exist - if user != nil { - isAdmin = user.Admin - isAuthor = (prop.UserId == user.ID.String()) - } - - // Strip the non-public proposal contents if user is - // not the author or an admin - if !isAuthor && !isAdmin { - prop.Name = "" - prop.Files = make([]www.File, 0) - } - } - - return &www.ProposalDetailsReply{ - Proposal: *prop, - } -} - -// processProposalDetails fetches a specific proposal version from the records -// cache and returns it. -func (p *politeiawww) processProposalDetails(propDetails www.ProposalsDetails, user *user.User) (*www.ProposalDetailsReply, error) { - log.Tracef("processProposalDetails: %v", propDetails.Token) - - // Version is an optional query param. Fetch latest version - // when query param is not specified. - var prop *www.ProposalRecord - var err error - if propDetails.Version == "" { - prop, err = p.getProp(propDetails.Token) - } else { - prop, err = p.getPropVersion(propDetails.Token, propDetails.Version) - } - if err != nil { - /* - // TODO - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - } - } - */ - return nil, err - } - - return createProposalDetailsReply(prop, user), nil -} - -// processBatchVoteSummary returns the vote summaries for the provided list -// of proposals. -func (p *politeiawww) processBatchVoteSummary(batchVoteSummary www.BatchVoteSummary) (*www.BatchVoteSummaryReply, error) { - log.Tracef("processBatchVoteSummary") - - if len(batchVoteSummary.Tokens) > www.ProposalListPageSize { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusMaxProposalsExceededPolicy, - } - } - - invalidTokens := getInvalidTokens(batchVoteSummary.Tokens) - if len(invalidTokens) > 0 { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: invalidTokens, - } - } - - bb, err := p.getBestBlock() - if err != nil { - return nil, err - } - - summaries, err := p.voteSummariesGet(batchVoteSummary.Tokens, bb) - if err != nil { - return nil, err - } - - if len(summaries) != len(batchVoteSummary.Tokens) { - tokensNotFound := make([]string, 0, - len(batchVoteSummary.Tokens)-len(summaries)) - - for _, token := range batchVoteSummary.Tokens { - if _, exists := summaries[token]; !exists { - tokensNotFound = append(tokensNotFound, token) - } - } - - return nil, www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - ErrorContext: tokensNotFound, - } - } - - return &www.BatchVoteSummaryReply{ - BestBlock: bb, - Summaries: summaries, - }, nil -} - -// processBatchProposals fetches a list of proposals from the records cache -// and returns them. The returned proposals do not include the proposal files. -func (p *politeiawww) processBatchProposals(bp www.BatchProposals, user *user.User) (*www.BatchProposalsReply, error) { - log.Tracef("processBatchProposals") - - // Validate censorship tokens - if len(bp.Tokens) > www.ProposalListPageSize { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusMaxProposalsExceededPolicy, - } - } - - invalidTokens := getInvalidTokens(bp.Tokens) - if len(invalidTokens) > 0 { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: invalidTokens, - } - } - - // Lookup proposals - props, err := p.getProps(bp.Tokens) - if err != nil { - return nil, err - } - if len(props) != len(bp.Tokens) { - // A proposal was not found for one or more of the - // provided tokens. Figure out which ones they were. - notFound := make([]string, 0, len(bp.Tokens)) - for _, v := range bp.Tokens { - if _, ok := props[v]; !ok { - notFound = append(notFound, v) - } - } - return nil, www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - ErrorContext: notFound, - } - } - - for token, pr := range props { - // Vetted proposals are viewable by everyone. The contents of - // an unvetted proposal is only viewable by admins and the - // proposal author. Unvetted proposal metadata is viewable by - // everyone. - if pr.State == www.PropStateUnvetted { - var isAuthor bool - var isAdmin bool - // This is a public route so a user may not exist - if user != nil { - isAdmin = user.Admin - isAuthor = (pr.UserId == user.ID.String()) - } - - // Strip the non-public proposal contents if user is - // not the author or an admin. The files are already - // not included in this request. - if !isAuthor && !isAdmin { - prop := props[token] - prop.Name = "" - props[token] = prop - } - } - } - - // Convert proposals map to a slice - proposals := make([]www.ProposalRecord, 0, len(props)) - for _, v := range props { - proposals = append(proposals, v) - } - - return &www.BatchProposalsReply{ - Proposals: proposals, - }, nil -} - -// processAllVetted returns an array of vetted proposals. The maximum number -// of proposals returned is dictated by www.ProposalListPageSize. func (p *politeiawww) processAllVetted(v www.GetAllVetted) (*www.GetAllVettedReply, error) { - log.Tracef("processAllVetted") - - // Validate query params - if (v.Before != "" && !tokenIsValid(v.Before)) || - (v.After != "" && !tokenIsValid(v.After)) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - } - } - - // Fetch all proposals from the cache - // TODO do not use getAllProps - all, err := p.getAllProps() - if err != nil { - return nil, fmt.Errorf("getAllProps: %v", err) - } - - // Filter for vetted proposals - filter := proposalsFilter{ - After: v.After, - Before: v.Before, - StateMap: map[www.PropStateT]bool{ - www.PropStateVetted: true, - }, - } - props := filterProps(filter, all) - - // Remove files from proposals - for i, p := range props { - p.Files = make([]www.File, 0) - props[i] = p - } - - return &www.GetAllVettedReply{ - Proposals: props, - }, nil + return nil, nil } -// processCommentsGet returns all comments for a given proposal. If the user is -// logged in the user's last access time for the given comments will also be -// returned. func (p *politeiawww) processCommentsGet(token string, u *user.User) (*www.GetCommentsReply, error) { - log.Tracef("ProcessCommentGet: %v", token) - - // Make sure token is valid and not a prefix - if !tokenIsValid(token) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: []string{token}, - } - } - - // Fetch proposal comments from cache - c, err := p.getPropComments(token) - if err != nil { - return nil, err - } - - // Get the last time the user accessed these comments. This is - // a public route so a user may not exist. - var accessTime int64 - if u != nil { - if u.ProposalCommentsAccessTimes == nil { - u.ProposalCommentsAccessTimes = make(map[string]int64) - } - accessTime = u.ProposalCommentsAccessTimes[token] - u.ProposalCommentsAccessTimes[token] = time.Now().Unix() - err = p.db.UserUpdate(*u) - if err != nil { - return nil, err - } - } - - return &www.GetCommentsReply{ - Comments: c, - AccessTime: accessTime, - }, nil -} - -// setVoteStatusReply converts a VoteStatusReply to a VoteSummary and stores it -// in memory. This is to only be used for proposals whose voting period has -// ended so that we don't have to worry about cache invalidation issues. -// -// This function must be called without the lock held. -// -// ** This fuction is to be removed when the deprecated vote status route is -// ** removed. -func (p *politeiawww) setVoteStatusReply(v www.VoteStatusReply) error { - /* - p.Lock() - defer p.Unlock() - - endHeight, err := strconv.Atoi(v.EndHeight) - if err != nil { - return err - } - - voteSummary := www.VoteSummary{ - Status: v.Status, - EligibleTickets: uint32(v.NumOfEligibleVotes), - EndHeight: uint64(endHeight), - QuorumPercentage: v.QuorumPercentage, - PassPercentage: v.PassPercentage, - Results: v.OptionsResult, - } - - p.voteSummaries[v.Token] = voteSummary - - return nil - */ - - return nil -} - -func (p *politeiawww) voteStatusReply(token string, bestBlock uint64) (*www.VoteStatusReply, error) { - /* - cachedVsr, ok := p.getVoteStatusReply(token) - - if ok { - cachedVsr.BestBlock = strconv.Itoa(int(bestBlock)) - return cachedVsr, nil - } - - // Vote status wasn't in the memory cache - // so fetch it from the cache database. - vs, err := p.voteSummaryGet(token, bestBlock) - if err != nil { - return nil, err - } - - var total uint64 - for _, v := range vs.Results { - total += v.VotesReceived - } - - voteStatusReply := www.VoteStatusReply{ - Token: token, - Status: vs.Status, - TotalVotes: total, - OptionsResult: vs.Results, - EndHeight: strconv.Itoa(int(vs.EndHeight)), - BestBlock: strconv.Itoa(int(bestBlock)), - NumOfEligibleVotes: int(vs.EligibleTickets), - QuorumPercentage: vs.QuorumPercentage, - PassPercentage: vs.PassPercentage, - } - - // If the voting period has ended the vote status - // is not going to change so add it to the memory - // cache. - if voteStatusReply.Status == www.PropVoteStatusFinished { - err = p.setVoteStatusReply(voteStatusReply) - if err != nil { - return nil, err - } - } - - return &voteStatusReply, nil - */ return nil, nil } -// processVoteStatus returns the vote status for a given proposal func (p *politeiawww) processVoteStatus(token string) (*www.VoteStatusReply, error) { - log.Tracef("ProcessProposalVotingStatus: %v", token) - - /* - // Make sure token is valid and not a prefix - if !tokenIsValid(token) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: []string{token}, - } - } - - // Ensure proposal is vetted - pr, err := p.getProp(token) - if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - } - } - return nil, err - } - - if pr.State != www.PropStateVetted { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusWrongStatus, - } - } - - // Get best block - bestBlock, err := p.getBestBlock() - if err != nil { - return nil, fmt.Errorf("bestBlock: %v", err) - } - - // Get vote status - vsr, err := p.voteStatusReply(token, bestBlock) - if err != nil { - return nil, fmt.Errorf("voteStatusReply: %v", err) - } - - return vsr, nil - */ - return nil, nil } -// processGetAllVoteStatus returns the vote status of all public proposals. func (p *politeiawww) processGetAllVoteStatus() (*www.GetAllVoteStatusReply, error) { - log.Tracef("processGetAllVoteStatus") - - /* - // We need to determine best block height here in order - // to set the voting status - bestBlock, err := p.getBestBlock() - if err != nil { - return nil, fmt.Errorf("bestBlock: %v", err) - } - - // Get all proposals from cache - // TODO do not use getAllProps - all, err := p.getAllProps() - if err != nil { - return nil, fmt.Errorf("getAllProps: %v", err) - } - - // Compile votes statuses - vrr := make([]www.VoteStatusReply, 0, len(all)) - for _, v := range all { - // We only need public proposals - if v.Status != www.PropStatusPublic { - continue - } - - // Get vote status for proposal - vs, err := p.voteStatusReply(v.CensorshipRecord.Token, bestBlock) - if err != nil { - return nil, fmt.Errorf("voteStatusReply: %v", err) - } - - vrr = append(vrr, *vs) - } - - return &www.GetAllVoteStatusReply{ - VotesStatus: vrr, - }, nil - */ - return nil, nil } func (p *politeiawww) processActiveVote() (*www.ActiveVoteReply, error) { - log.Tracef("processActiveVote") - - /* - // Fetch proposals that are actively being voted on - bb, err := p.getBestBlock() - if err != nil { - return nil, err - } - tir, err := p.tokenInventory(bb, false) - if err != nil { - return nil, err - } - props, err := p.getProps(tir.Active) - if err != nil { - return nil, err - } - - // Compile proposal vote tuples - pvt := make([]www.ProposalVoteTuple, 0, len(props)) - for _, v := range props { - // Get vote details from cache - vdr, err := p.decredVoteDetails(v.CensorshipRecord.Token) - if err != nil { - return nil, fmt.Errorf("decredVoteDetails %v: %v", - v.CensorshipRecord.Token, err) - } - - // Handle StartVote versioning - var sv www.StartVote - switch vdr.StartVote.Version { - case decredplugin.VersionStartVoteV1: - b := []byte(vdr.StartVote.Payload) - dsv, err := decredplugin.DecodeStartVoteV1(b) - if err != nil { - return nil, fmt.Errorf("decode StartVoteV1 %v: %v", - v.CensorshipRecord.Token, err) - } - sv = convertStartVoteV1FromDecred(*dsv) - - case decredplugin.VersionStartVoteV2: - b := []byte(vdr.StartVote.Payload) - dsv2, err := decredplugin.DecodeStartVoteV2(b) - if err != nil { - return nil, fmt.Errorf("decode StartVoteV2 %v: %v", - v.CensorshipRecord.Token, err) - } - sv2 := convertStartVoteV2FromDecred(*dsv2) - // Convert StartVote v2 to v1 since this route returns - // a v1 StartVote. - sv = convertStartVoteV2ToV1(sv2) - - default: - return nil, fmt.Errorf("invalid StartVote version %v %v", - v.CensorshipRecord.Token, vdr.StartVote.Version) - } - - // Create vote tuple - pvt = append(pvt, www.ProposalVoteTuple{ - Proposal: v, - StartVote: sv, - StartVoteReply: convertStartVoteReplyFromDecred(vdr.StartVoteReply), - }) - } - - return &www.ActiveVoteReply{ - Votes: pvt, - }, nil - */ - return nil, nil } -// processVoteResults returns the vote details for a specific proposal and all -// of the votes that have been cast. func (p *politeiawww) processVoteResults(token string) (*www.VoteResultsReply, error) { - log.Tracef("processVoteResults: %v", token) - - /* - // Make sure token is valid and not a prefix - if !tokenIsValid(token) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: []string{token}, - } - } - - // Ensure proposal is vetted - pr, err := p.getProp(token) - if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - } - } - return nil, err - } - - if pr.State != www.PropStateVetted { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusWrongStatus, - } - } - - // Get vote details from cache - vdr, err := p.decredVoteDetails(token) - if err != nil { - return nil, fmt.Errorf("decredVoteDetails: %v", err) - } - if vdr.StartVoteReply.StartBlockHash == "" { - // Vote has not been started yet. No need to continue. - return &www.VoteResultsReply{}, nil - } - - // Get cast votes from cache - vrr, err := p.decredProposalVotes(token) - if err != nil { - return nil, fmt.Errorf("decredProposalVotes: %v", err) - } - - // Handle StartVote versioning - var sv www.StartVote - switch vdr.StartVote.Version { - case decredplugin.VersionStartVoteV1: - b := []byte(vdr.StartVote.Payload) - dsv1, err := decredplugin.DecodeStartVoteV1(b) - if err != nil { - return nil, err - } - sv = convertStartVoteV1FromDecred(*dsv1) - case decredplugin.VersionStartVoteV2: - b := []byte(vdr.StartVote.Payload) - dsv2, err := decredplugin.DecodeStartVoteV2(b) - if err != nil { - return nil, err - } - sv2 := convertStartVoteV2FromDecred(*dsv2) - // Convert StartVote v2 to v1 since this route returns - // a v1 StartVote. - sv = convertStartVoteV2ToV1(sv2) - default: - return nil, fmt.Errorf("invalid StartVote version %v %v", - token, vdr.StartVote.Version) - } - - return &www.VoteResultsReply{ - StartVote: sv, - StartVoteReply: convertStartVoteReplyFromDecred(vdr.StartVoteReply), - CastVotes: convertCastVotesFromDecred(vrr.CastVotes), - }, nil - */ - return nil, nil } -// processCastVotes handles the www.Ballot call func (p *politeiawww) processCastVotes(ballot *www.Ballot) (*www.BallotReply, error) { log.Tracef("processCastVotes") @@ -1474,6 +567,10 @@ func validateAuthorizeVoteRunoff(av www.AuthorizeVote, u user.User, pr www.Propo return nil } +func (p *politeiawww) voteSummaryGet(token string, bestBlock uint64) (*www.VoteSummary, error) { + return nil, nil +} + // processAuthorizeVote sends the authorizevote command to decred plugin to // indicate that a proposal has been finalized and is ready to be voted on. func (p *politeiawww) processAuthorizeVote(av www.AuthorizeVote, u *user.User) (*www.AuthorizeVoteReply, error) { From 0f517bde998705de2c800f6855f6430f713c34f7 Mon Sep 17 00:00:00 2001 From: Thiago Figueiredo Date: Mon, 14 Sep 2020 15:48:24 -0300 Subject: [PATCH 043/449] eventmanager: minor refactor and cleanup code --- politeiawww/email.go | 39 ++++++---- politeiawww/eventmanager.go | 139 +++++++++++++++++++++--------------- politeiawww/proposals.go | 22 +++--- 3 files changed, 115 insertions(+), 85 deletions(-) diff --git a/politeiawww/email.go b/politeiawww/email.go index a6cd8192d..e1c434641 100644 --- a/politeiawww/email.go +++ b/politeiawww/email.go @@ -221,9 +221,9 @@ func (p *politeiawww) emailProposalEdited(name, username, token, version string, return p.smtp.sendEmailTo(subject, body, emails) } -// emailProposalVoteStarted sends email for the proposal vote started event. -// Sends email to author and users with this notification bit set on. -func (p *politeiawww) emailProposalVoteStarted(token, name, username, email string, emailNotifications uint64, emails []string) error { +// emailProposalVoteStarted sends email to the proposal author for the +// proposal vote started event. +func (p *politeiawww) emailProposalVoteStartedToAuthor(token, name, username, email string) error { route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) l, err := url.Parse(p.cfg.WebServerAddress + route) if err != nil { @@ -236,20 +236,29 @@ func (p *politeiawww) emailProposalVoteStarted(token, name, username, email stri Username: username, } - if emailNotifications& - uint64(www.NotificationEmailMyProposalVoteStarted) != 0 { + subject := "Your Proposal Has Started Voting" + body, err := createBody(templateProposalVoteStartedForAuthor, &tplData) + if err != nil { + return err + } + recipients := []string{email} - subject := "Your Proposal Has Started Voting" - body, err := createBody(templateProposalVoteStartedForAuthor, &tplData) - if err != nil { - return err - } - recipients := []string{email} + return p.smtp.sendEmailTo(subject, body, recipients) +} - err = p.smtp.sendEmailTo(subject, body, recipients) - if err != nil { - return err - } +// emailProposalVoteStartedToUsers sends email to users with this notification +// bit set on for the proposal vote started event. +func (p *politeiawww) emailProposalVoteStartedToUsers(token, name, username string, emails []string) error { + route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) + l, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err + } + + tplData := proposalVoteStartedTemplateData{ + Link: l.String(), + Name: name, + Username: username, } subject := "Voting Started for Proposal" diff --git a/politeiawww/eventmanager.go b/politeiawww/eventmanager.go index d94a9825f..1468d50a2 100644 --- a/politeiawww/eventmanager.go +++ b/politeiawww/eventmanager.go @@ -10,7 +10,6 @@ import ( pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/user" - "github.com/google/uuid" ) type eventT int @@ -85,25 +84,25 @@ func (p *politeiawww) setupEventListenersPi() { // 2. Register the channel with the event manager // 3. Launch an event handler to listen for new events - // Setup proposal comment event - ch := make(chan interface{}) - p.eventManager.register(eventProposalComment, ch) - go p.handleEventProposalComment(ch) - // Setup proposal submitted event - ch = make(chan interface{}) + ch := make(chan interface{}) p.eventManager.register(eventProposalSubmitted, ch) go p.handleEventProposalSubmitted(ch) + // Setup proposal edit event + ch = make(chan interface{}) + p.eventManager.register(eventProposalEdited, ch) + go p.handleEventProposalEdited(ch) + // Setup proposal status change event ch = make(chan interface{}) p.eventManager.register(eventProposalStatusChange, ch) go p.handleEventProposalStatusChange(ch) - // Setup proposal edit event + // Setup proposal comment event ch = make(chan interface{}) - p.eventManager.register(eventProposalEdited, ch) - go p.handleEventProposalEdited(ch) + p.eventManager.register(eventProposalComment, ch) + go p.handleEventProposalComment(ch) // Setup proposal vote authorized event ch = make(chan interface{}) @@ -181,7 +180,8 @@ func (p *politeiawww) handleEventProposalSubmitted(ch chan interface{}) { emails := make([]string, 0, 256) err := p.db.AllUsers(func(u *user.User) { // Check if user is able to receive notification - if userNotificationEnabled(*u, www.NotificationEmailAdminProposalNew) { + if userNotificationEnabled(*u, + www.NotificationEmailAdminProposalNew) { emails = append(emails, u.Email) } }) @@ -271,7 +271,7 @@ func (p *politeiawww) handleEventProposalStatusChange(ch chan interface{}) { continue } - // Compile list of emails to sent notification to + // Compile list of emails to send notification to emails := make([]string, 0, 256) notification := www.NotificationEmailRegularProposalVetted err := p.db.AllUsers(func(u *user.User) { @@ -338,39 +338,32 @@ func (p *politeiawww) handleEventProposalComment(ch chan interface{}) { continue } - // Check if user notification is enabled - if !userNotificationEnabled(*author, - www.NotificationEmailCommentOnMyProposal) { + // Check circumstances where we don't notify + switch { + case d.commentUsername == author.Username: + // Don't notify when author comments on own proposal continue - } - - // Don't notify when author comments on own proposal - if d.commentUsername == author.Username { + case !userNotificationEnabled(*author, + www.NotificationEmailCommentOnMyProposal): + // User does not have notification bit set on continue } - // Top-level comment - if d.parentID == "0" { - err := p.emailProposalComment(d.token, d.commentID, - d.commentUsername, d.name, author.Email) + // Check if is top-level comment + if d.parentID != "0" { + // Nested comment reply. Fetch parent comment in order to fetch + // comment author + parent, err := p.decredCommentGetByID(d.token, d.parentID) if err != nil { - log.Errorf("emailProposalComment: %v", err) + log.Errorf("decredCommentGetByID: %v", err) + continue } - continue - } - // Nested comment reply. Fetch parent comment in order to fetch - // parent comment author - parent, err := p.decredCommentGetByID(d.token, d.parentID) - if err != nil { - log.Errorf("decredCommentGetByID: %v", err) - continue - } - - author, err = p.db.UserGetByPubKey(parent.PublicKey) - if err != nil { - log.Errorf("UserGetByPubKey: %v", err) - continue + author, err = p.db.UserGetByPubKey(parent.PublicKey) + if err != nil { + log.Errorf("UserGetByPubKey: %v", err) + continue + } } err = p.emailProposalComment(d.token, d.commentID, @@ -401,8 +394,13 @@ func (p *politeiawww) handleEventProposalVoteAuthorized(ch chan interface{}) { emails := make([]string, 0, 256) err := p.db.AllUsers(func(u *user.User) { // Check circunstances where we don't notify - if !u.Admin || !userNotificationEnabled(*u, - www.NotificationEmailAdminProposalVoteAuthorized) { + switch { + case !u.Admin: + // Only notify admin users + return + case !userNotificationEnabled(*u, + www.NotificationEmailAdminProposalVoteAuthorized): + // User does not have this notification bit set on return } @@ -420,13 +418,10 @@ func (p *politeiawww) handleEventProposalVoteAuthorized(ch chan interface{}) { } type dataProposalVoteStarted struct { - token string // Proposal censhorship token - name string // Proposal name - adminID uuid.UUID // Admin uuid - id uuid.UUID // Author uuid - username string // Author username - email string // Author email - emailNotifications uint64 // Author notifications bits + token string // Proposal censhorship token + name string // Proposal name + adminID string // Admin uuid + author user.User // Proposal author } func (p *politeiawww) handleEventProposalVoteStarted(ch chan interface{}) { @@ -438,21 +433,43 @@ func (p *politeiawww) handleEventProposalVoteStarted(ch chan interface{}) { } emails := make([]string, 0, 256) + notification := www.NotificationEmailRegularProposalVoteStarted err := p.db.AllUsers(func(u *user.User) { // Check circunstances where we don't notify - if u.NewUserPaywallTx == "" || u.ID == d.adminID || u.ID == d.id || - !userNotificationEnabled(*u, - www.NotificationEmailRegularProposalVoteStarted) { + switch { + case u.NewUserPaywallTx == "": + // User did not pay paywall + return + case u.ID.String() == d.adminID: + // Don't notify admin who started the vote + return + case u.ID.String() == d.author.ID.String(): + // Notify author separately from this users batch + return + case !userNotificationEnabled(*u, notification): + // User does not have notification bit set on return } emails = append(emails, u.Email) }) - err = p.emailProposalVoteStarted(d.token, d.name, d.username, - d.email, d.emailNotifications, emails) - if err != nil { - log.Errorf("emailProposalVoteStarted: %v", err) + // Email author + if userNotificationEnabled(d.author, notification) { + err = p.emailProposalVoteStartedToAuthor(d.token, d.name, + d.author.Username, d.author.Email) + if err != nil { + log.Errorf("emailProposalVoteStartedToAuthor: %v", err) + } + } + + // Email users + if len(emails) > 0 { + err = p.emailProposalVoteStartedToUsers(d.token, d.name, + d.author.Username, emails) + if err != nil { + log.Errorf("emailProposalVoteStartedToUsers: %v", err) + } } log.Debugf("Sent proposal vote started notifications %v", d.token) @@ -518,7 +535,12 @@ func (p *politeiawww) handleEventDCCNew(ch chan interface{}) { emails := make([]string, 0, 256) err := p.db.AllUsers(func(u *user.User) { // Check circunstances where we don't notify - if !u.Admin || u.Deactivated { + switch { + case !u.Admin: + // Only notify admin users + return + case u.Deactivated: + // Never notify deactivated users return } @@ -549,7 +571,12 @@ func (p *politeiawww) handleEventDCCSupportOppose(ch chan interface{}) { emails := make([]string, 0, 256) err := p.db.AllUsers(func(u *user.User) { // Check circunstances where we don't notify - if !u.Admin || u.Deactivated { + switch { + case !u.Admin: + // Only notify admin users + return + case u.Deactivated: + // Never notify deactivated users return } diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 9f4a64d14..b86110902 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -987,13 +987,10 @@ func (p *politeiawww) processStartVoteV2(sv www2.StartVote, u *user.User) (*www2 // Emit event notification for proposal start vote p.eventManager.emit(eventProposalVoteStarted, dataProposalVoteStarted{ - token: pr.CensorshipRecord.Token, - name: pr.Name, - adminID: u.ID, - id: author.ID, - username: author.Username, - email: author.Email, - emailNotifications: author.EmailNotifications, + token: pr.CensorshipRecord.Token, + name: pr.Name, + adminID: u.ID.String(), + author: *author, }) return svr, nil @@ -1290,13 +1287,10 @@ func (p *politeiawww) processStartVoteRunoffV2(sv www2.StartVoteRunoff, u *user. } p.eventManager.emit(eventProposalVoteStarted, dataProposalVoteStarted{ - token: pn.CensorshipRecord.Token, - name: pn.Name, - adminID: u.ID, - id: author.ID, - username: author.Username, - email: author.Email, - emailNotifications: author.EmailNotifications, + token: pn.CensorshipRecord.Token, + name: pn.Name, + adminID: u.ID.String(), + author: *author, }) } From cf83659d882cc39cc0b3c62aed44602cc770f77d Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 14 Sep 2020 13:20:14 -0500 Subject: [PATCH 044/449] hook everything up --- politeiad/backend/gitbe/decred.go | 178 +- politeiawww/api/pi/v1/v1.go | 17 +- politeiawww/api/www/v2/v2.go | 2 + politeiawww/cmd/cmswww/cmswww.go | 21 +- politeiawww/cmd/cmswww/editinvoice.go | 4 +- politeiawww/cmd/cmswww/newdcc.go | 2 +- politeiawww/cmd/cmswww/newinvoice.go | 2 +- politeiawww/cmd/cmswww/testrun.go | 4 +- politeiawww/cmd/piwww/editproposal.go | 288 -- politeiawww/cmd/piwww/help.go | 104 +- politeiawww/cmd/piwww/piwww.go | 134 +- politeiawww/cmd/piwww/proposaledit.go | 263 ++ .../piwww/{newproposal.go => proposalnew.go} | 176 +- politeiawww/cmd/piwww/proposalsetstatus.go | 123 + politeiawww/cmd/piwww/setproposalstatus.go | 130 - politeiawww/cmd/piwww/testrun.go | 2709 ++++++++--------- politeiawww/cmd/shared/client.go | 107 +- politeiawww/cmd/shared/shared.go | 17 +- politeiawww/piwww.go | 119 +- politeiawww/util/merkle.go | 33 + politeiawww/www.go | 2 + 21 files changed, 2293 insertions(+), 2142 deletions(-) delete mode 100644 politeiawww/cmd/piwww/editproposal.go create mode 100644 politeiawww/cmd/piwww/proposaledit.go rename politeiawww/cmd/piwww/{newproposal.go => proposalnew.go} (51%) create mode 100644 politeiawww/cmd/piwww/proposalsetstatus.go delete mode 100644 politeiawww/cmd/piwww/setproposalstatus.go diff --git a/politeiad/backend/gitbe/decred.go b/politeiad/backend/gitbe/decred.go index 19bad4d81..f3ad3b2a4 100644 --- a/politeiad/backend/gitbe/decred.go +++ b/politeiad/backend/gitbe/decred.go @@ -27,7 +27,6 @@ import ( "github.com/decred/dcrd/wire" dcrdataapi "github.com/decred/dcrdata/api/types/v4" "github.com/decred/politeia/decredplugin" - "github.com/decred/politeia/mdstream" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/util" @@ -1694,49 +1693,6 @@ func prepareStartVoteReply(voteDuration, ticketMaturity uint32) (*decredplugin.S }, nil } -var ( - errProposalMetadataNotFound = errors.New("proposal metadata not found") -) - -// vettedProposalMetadata returns the www.ProposalMetadata for the provided -// token. All new proposal records are required to contain a ProposalMetadata, -// but older records may not. A errProposalMetadata not found error is returned -// if a ProposalMetadata was not found. -// -// This function must be called WITH the lock held. -func (g *gitBackEnd) vettedProposalMetadata(token string) (*mdstream.ProposalMetadata, error) { - tokenb, err := hex.DecodeString(token) - if err != nil { - return nil, err - } - r, err := g.getRecord(tokenb, "", g.vetted, true) - if err != nil { - return nil, err - } - - // ProposalMetadata is stored as a politeiad File, not an mdstream. - var pm *mdstream.ProposalMetadata - for _, v := range r.Files { - if v.Name != mdstream.FilenameProposalMetadata { - continue - } - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return nil, err - } - pm, err = mdstream.DecodeProposalMetadata(b) - if err != nil { - return nil, err - } - } - - if pm == nil { - return nil, errProposalMetadataNotFound - } - - return pm, nil -} - func (g *gitBackEnd) pluginStartVote(payload string) (string, error) { sv, err := decredplugin.DecodeStartVoteV2([]byte(payload)) if err != nil { @@ -1775,29 +1731,32 @@ func (g *gitBackEnd) pluginStartVote(payload string) (string, error) { // Ensure proposal is not an RFP submissions. The plugin command // startvoterunoff must be used to start a runoff vote between RFP // submissions. - pm, err := g.vettedProposalMetadata(token) - switch { - case err == errProposalMetadataNotFound: - // Proposal is not an RFP submission. This is ok. - case err != nil: - // All other errors - return "", err - case pm.LinkTo != "": - // ProposalMetadata exists and a linkto was set. Check if this - // proposal is an RFP submission. - linkToPM, err := g.vettedProposalMetadata(pm.LinkTo) - if err != nil { + /* + // TODO + pm, err := g.vettedProposalMetadata(token) + switch { + case err == errProposalMetadataNotFound: + // Proposal is not an RFP submission. This is ok. + case err != nil: + // All other errors return "", err + case pm.LinkTo != "": + // ProposalMetadata exists and a linkto was set. Check if this + // proposal is an RFP submission. + linkToPM, err := g.vettedProposalMetadata(pm.LinkTo) + if err != nil { + return "", err + } + if linkToPM.LinkBy != 0 { + // LinkBy will only be set on RFP proposals + return "", fmt.Errorf("proposal is an rfp submission: %v", + token) + } + default: + // ProposalMetadata exists, but this proposal is not linked to + // another proposal. This is ok. } - if linkToPM.LinkBy != 0 { - // LinkBy will only be set on RFP proposals - return "", fmt.Errorf("proposal is an rfp submission: %v", - token) - } - default: - // ProposalMetadata exists, but this proposal is not linked to - // another proposal. This is ok. - } + */ // Verify proposal state tokenb, err := util.ConvertStringToken(token) @@ -2000,53 +1959,56 @@ func (g *gitBackEnd) pluginStartVoteRunoff(payload string) (string, error) { return "", backend.ErrShutdown } - // Verify this proposal is indeed an RFP - pm, err := g.vettedProposalMetadata(sv.Token) - switch { - case err == errProposalMetadataNotFound: - // No ProposalMetadata. This is not an RFP. - return "", fmt.Errorf("proposal is not an rfp: %v", sv.Token) - case err != nil: - // All other errors - return "", err - case pm.LinkBy == 0: - // ProposalMetadata found but this is not an RFP - return "", fmt.Errorf("proposal is not an rfp: %v", sv.Token) - case pm.LinkBy > 0: - // This proposal is an RFP. This is what we want. - default: - return "", fmt.Errorf("unknown proposal state") - } - - // Validate proposal state of all rfp submissions. The authorize - // vote metadata is intentionally not checked. RFP submissions - // are not required to have the vote authorized by the proposal - // author. - for _, v := range sv.StartVotes { - tokenb, err := util.ConvertStringToken(v.Vote.Token) - if err != nil { + /* + // TODO + // Verify this proposal is indeed an RFP + pm, err := g.vettedProposalMetadata(sv.Token) + switch { + case err == errProposalMetadataNotFound: + // No ProposalMetadata. This is not an RFP. + return "", fmt.Errorf("proposal is not an rfp: %v", sv.Token) + case err != nil: + // All other errors return "", err + case pm.LinkBy == 0: + // ProposalMetadata found but this is not an RFP + return "", fmt.Errorf("proposal is not an rfp: %v", sv.Token) + case pm.LinkBy > 0: + // This proposal is an RFP. This is what we want. + default: + return "", fmt.Errorf("unknown proposal state") } - authVoteExists := g.vettedMetadataStreamExists(tokenb, - decredplugin.MDStreamAuthorizeVote) - voteBitsExist := g.vettedMetadataStreamExists(tokenb, - decredplugin.MDStreamVoteBits) - voteSnapshotExists := g.vettedMetadataStreamExists(tokenb, - decredplugin.MDStreamVoteSnapshot) - switch { - case !authVoteExists && !voteBitsExist && !voteSnapshotExists: - // Vote has not started, continue - case authVoteExists && voteBitsExist && voteSnapshotExists: - // Vote has started - return "", fmt.Errorf("vote already started: %x", - tokenb) - default: - // This is bad, both files should exist or not exist - return "", fmt.Errorf("proposal in unknown vote state: %x", - tokenb) + // Validate proposal state of all rfp submissions. The authorize + // vote metadata is intentionally not checked. RFP submissions + // are not required to have the vote authorized by the proposal + // author. + for _, v := range sv.StartVotes { + tokenb, err := util.ConvertStringToken(v.Vote.Token) + if err != nil { + return "", err + } + + authVoteExists := g.vettedMetadataStreamExists(tokenb, + decredplugin.MDStreamAuthorizeVote) + voteBitsExist := g.vettedMetadataStreamExists(tokenb, + decredplugin.MDStreamVoteBits) + voteSnapshotExists := g.vettedMetadataStreamExists(tokenb, + decredplugin.MDStreamVoteSnapshot) + switch { + case !authVoteExists && !voteBitsExist && !voteSnapshotExists: + // Vote has not started, continue + case authVoteExists && voteBitsExist && voteSnapshotExists: + // Vote has started + return "", fmt.Errorf("vote already started: %x", + tokenb) + default: + // This is bad, both files should exist or not exist + return "", fmt.Errorf("proposal in unknown vote state: %x", + tokenb) + } } - } + */ // Get identity fiJSON, ok := decredPluginSettings[decredPluginIdentity] diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 0353cb140..c55ab82a1 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -41,7 +41,9 @@ const ( RouteVoteSummaries = "/votes/summaries" RouteVoteInventory = "/votes/inventory" - // Proposal states + // Proposal states. A proposal state can be either unvetted or + // vetted. The PropStatusT type further breaks down these two + // states into more granular statuses. PropStateInvalid PropStateT = 0 PropStateUnvetted PropStateT = 1 PropStateVetted PropStateT = 2 @@ -82,13 +84,14 @@ const ( VoteTypeRunoff VoteT = 2 // Error status codes - ErrorStatusInvalid ErrorStatusT = 0 + ErrorStatusInvalid ErrorStatusT = 0 + ErrorStatusInvalidInput ErrorStatusT = 1 // User errors - ErrorStatusUserRegistrationNotPaid ErrorStatusT = 1 - ErrorStatusUserBalanceInsufficient ErrorStatusT = 2 - ErrorStatusUserIsNotAuthor ErrorStatusT = 3 - ErrorStatusUserIsNotAdmin ErrorStatusT = 4 + ErrorStatusUserRegistrationNotPaid ErrorStatusT = 2 + ErrorStatusUserBalanceInsufficient ErrorStatusT = 3 + ErrorStatusUserIsNotAuthor ErrorStatusT = 4 + ErrorStatusUserIsNotAdmin ErrorStatusT = 5 // Signature errors ErrorStatusPublicKeyInvalid ErrorStatusT = 100 @@ -121,7 +124,7 @@ const ( ErrorStatusPropStatusChangeReasonInvalid ErrorStatusT = 224 // Comment errors - // TODO make normal + // TODO number error codes ErrorStatusCommentTextInvalid ErrorStatusT = iota ErrorStatusCommentParentIDInvalid ErrorStatusCommentVoteInvalid diff --git a/politeiawww/api/www/v2/v2.go b/politeiawww/api/www/v2/v2.go index fd67cea03..e3ed886b4 100644 --- a/politeiawww/api/www/v2/v2.go +++ b/politeiawww/api/www/v2/v2.go @@ -14,6 +14,8 @@ type VoteT int const ( APIVersion = 2 + // All www/v2 routes have been deprecated. The pi/v1 API should be + // used instead. RouteStartVote = "/vote/start" RouteStartVoteRunoff = "/vote/startrunoff" RouteVoteDetails = "/vote/{token:[A-z0-9]{64}}" diff --git a/politeiawww/cmd/cmswww/cmswww.go b/politeiawww/cmd/cmswww/cmswww.go index 0c78394cd..09f517fe4 100644 --- a/politeiawww/cmd/cmswww/cmswww.go +++ b/politeiawww/cmd/cmswww/cmswww.go @@ -7,6 +7,7 @@ package main import ( "bufio" "encoding/csv" + "encoding/hex" "errors" "fmt" "net/url" @@ -15,6 +16,7 @@ import ( "strings" "github.com/decred/dcrd/dcrutil" + "github.com/decred/politeia/politeiad/api/v1/identity" cms "github.com/decred/politeia/politeiawww/api/cms/v1" pi "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/cmd/shared" @@ -110,6 +112,21 @@ type cmswww struct { VoteDetails VoteDetailsCmd `command:"votedetails" description:"(user) get the details for a dcc vote"` } +// signedMerkleRoot calculates the merkle root of the passed in list of files +// and metadata, signs the merkle root with the passed in identity and returns +// the signature. +func signedMerkleRoot(files []pi.File, md []pi.Metadata, id *identity.FullIdentity) (string, error) { + if len(files) == 0 { + return "", fmt.Errorf("no proposal files found") + } + mr, err := wwwutil.MerkleRootWWW(files, md) + if err != nil { + return "", err + } + sig := id.SignMessage([]byte(mr)) + return hex.EncodeToString(sig[:]), nil +} + // verifyInvoice verifies a invoice's merkle root, author signature, and // censorship record. func verifyInvoice(p cms.InvoiceRecord, serverPubKey string) error { @@ -120,7 +137,7 @@ func verifyInvoice(p cms.InvoiceRecord, serverPubKey string) error { return err } // Verify merkle root - mr, err := wwwutil.MerkleRoot(p.Files, nil) + mr, err := wwwutil.MerkleRootWWW(p.Files, nil) if err != nil { return err } @@ -284,7 +301,7 @@ func verifyDCC(p cms.DCCRecord, serverPubKey string) error { return err } // Verify merkel root - mr, err := wwwutil.MerkleRoot(files, nil) + mr, err := wwwutil.MerkleRootWWW(files, nil) if err != nil { return err } diff --git a/politeiawww/cmd/cmswww/editinvoice.go b/politeiawww/cmd/cmswww/editinvoice.go index b475f9e5c..4c42ad2d6 100644 --- a/politeiawww/cmd/cmswww/editinvoice.go +++ b/politeiawww/cmd/cmswww/editinvoice.go @@ -17,7 +17,7 @@ import ( "strings" "github.com/decred/politeia/politeiad/api/v1/mime" - "github.com/decred/politeia/politeiawww/api/cms/v1" + v1 "github.com/decred/politeia/politeiawww/api/cms/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/cmd/shared" "github.com/decred/politeia/util" @@ -151,7 +151,7 @@ func (cmd *EditInvoiceCmd) Execute(args []string) error { } // Compute merkle root and sign it - sig, err := shared.SignedMerkleRoot(files, nil, cfg.Identity) + sig, err := signedMerkleRoot(files, nil, cfg.Identity) if err != nil { return fmt.Errorf("SignMerkleRoot: %v", err) } diff --git a/politeiawww/cmd/cmswww/newdcc.go b/politeiawww/cmd/cmswww/newdcc.go index 543209db3..5c22d8fdd 100644 --- a/politeiawww/cmd/cmswww/newdcc.go +++ b/politeiawww/cmd/cmswww/newdcc.go @@ -219,7 +219,7 @@ func (cmd *NewDCCCmd) Execute(args []string) error { files = append(files, f) // Compute merkle root and sign it - sig, err := shared.SignedMerkleRoot(files, nil, cfg.Identity) + sig, err := signedMerkleRoot(files, nil, cfg.Identity) if err != nil { return fmt.Errorf("SignMerkleRoot: %v", err) } diff --git a/politeiawww/cmd/cmswww/newinvoice.go b/politeiawww/cmd/cmswww/newinvoice.go index e54ed2995..f2849692b 100644 --- a/politeiawww/cmd/cmswww/newinvoice.go +++ b/politeiawww/cmd/cmswww/newinvoice.go @@ -168,7 +168,7 @@ func (cmd *NewInvoiceCmd) Execute(args []string) error { } // Compute merkle root and sign it - sig, err := shared.SignedMerkleRoot(files, nil, cfg.Identity) + sig, err := signedMerkleRoot(files, nil, cfg.Identity) if err != nil { return fmt.Errorf("SignMerkleRoot: %v", err) } diff --git a/politeiawww/cmd/cmswww/testrun.go b/politeiawww/cmd/cmswww/testrun.go index 8e031b7d3..a29023389 100644 --- a/politeiawww/cmd/cmswww/testrun.go +++ b/politeiawww/cmd/cmswww/testrun.go @@ -312,7 +312,7 @@ func invoiceNew(u user, contractorRate uint, labor float64) (*www.CensorshipReco Payload: base64.StdEncoding.EncodeToString(b), }, } - sig, err := shared.SignedMerkleRoot(files, nil, cfg.Identity) + sig, err := signedMerkleRoot(files, nil, cfg.Identity) if err != nil { return nil, err } @@ -445,7 +445,7 @@ func dccNew(sponsor user, nomineeID string, dcct cms.DCCTypeT, dt cms.DomainType Payload: base64.StdEncoding.EncodeToString(b), } files := []www.File{f} - sig, err := shared.SignedMerkleRoot(files, nil, cfg.Identity) + sig, err := signedMerkleRoot(files, nil, cfg.Identity) if err != nil { return nil, err } diff --git a/politeiawww/cmd/piwww/editproposal.go b/politeiawww/cmd/piwww/editproposal.go deleted file mode 100644 index 63fe1c757..000000000 --- a/politeiawww/cmd/piwww/editproposal.go +++ /dev/null @@ -1,288 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import ( - "bytes" - "encoding/base64" - "encoding/hex" - "encoding/json" - "fmt" - "io/ioutil" - "path/filepath" - "time" - - "github.com/decred/politeia/politeiad/api/v1/mime" - v1 "github.com/decred/politeia/politeiawww/api/www/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" - "github.com/decred/politeia/util" -) - -// EditProposalCmd edits an existing proposal. -type EditProposalCmd struct { - Args struct { - Token string `positional-arg-name:"token" required:"true"` - Markdown string `positional-arg-name:"markdownfile"` - Attachments []string `positional-arg-name:"attachmentfiles"` - } `positional-args:"true" optional:"true"` - Name string `long:"name" optional:"true"` - LinkTo string `long:"linkto" optional:"true"` - LinkBy int64 `long:"linkby" optional:"true"` - - // Random can be used in place of editing proposal name & data. When - // specified, random proposal name & data will be created and submitted. - Random bool `long:"random" optional:"true"` - - // RFP is a flag that is intended to make editing an RFP easier - // by calculating and inserting a linkby timestamp automatically - // instead of having to pass in a specific timestamp using the - // --linkby flag. - RFP bool `long:"rfp" optional:"true"` - - // Usemd is a flag that is intended to make editing propsoal metadata easier - // by using exisiting proposal metadata values instead of having to pass in - // specific values - UseMd bool `long:"usemd" optional:"true"` -} - -// Execute executes the edit proposal command. -func (cmd *EditProposalCmd) Execute(args []string) error { - token := cmd.Args.Token - mdFile := cmd.Args.Markdown - attachmentFiles := cmd.Args.Attachments - - if !cmd.Random && mdFile == "" { - return errProposalMDNotFound - } - - // Check for user identity - if cfg.Identity == nil { - return shared.ErrUserIdentityNotFound - } - - // RFP & linkby flags conflict - if cmd.RFP && cmd.LinkBy != 0 { - return errEditProposalRfpAndLinkbyFound - } - - // Random & name flags conflict - if cmd.Random && cmd.Name != "" { - return errEditProposalRandomAndNameFound - } - - // Get server public key - vr, err := client.Version() - if err != nil { - return err - } - - var md []byte - files := make([]v1.File, 0, v1.PolicyMaxImages+1) - if cmd.Random { - // Generate random proposal markdown text - var b bytes.Buffer - b.WriteString("This is the proposal title\n") - - for i := 0; i < 10; i++ { - r, err := util.Random(32) - if err != nil { - return err - } - b.WriteString(base64.StdEncoding.EncodeToString(r) + "\n") - } - - md = b.Bytes() - } else { - // Read markdown file into memory and convert to type File - fpath := util.CleanAndExpandPath(mdFile) - - var err error - md, err = ioutil.ReadFile(fpath) - if err != nil { - return fmt.Errorf("ReadFile %v: %v", fpath, err) - } - } - - f := v1.File{ - Name: "index.md", - MIME: mime.DetectMimeType(md), - Digest: hex.EncodeToString(util.Digest(md)), - Payload: base64.StdEncoding.EncodeToString(md), - } - - files = append(files, f) - - // Read attachment files into memory and convert to type File - for _, file := range attachmentFiles { - path := util.CleanAndExpandPath(file) - attachment, err := ioutil.ReadFile(path) - if err != nil { - return fmt.Errorf("ReadFile %v: %v", path, err) - } - - f := v1.File{ - Name: filepath.Base(file), - MIME: mime.DetectMimeType(attachment), - Digest: hex.EncodeToString(util.Digest(attachment)), - Payload: base64.StdEncoding.EncodeToString(attachment), - } - - files = append(files, f) - } - - // Setup metadata - var pm v1.ProposalMetadata - - if cmd.UseMd { - pdr, err := client.ProposalDetails(cmd.Args.Token, - &v1.ProposalsDetails{}) - if err != nil { - return err - } - // Prefill existing metadata - pm.Name = pdr.Proposal.Name - pm.LinkTo = pdr.Proposal.LinkTo - pm.LinkBy = pdr.Proposal.LinkBy - } - if cmd.Random { - // Generate random name - r, err := util.Random(v1.PolicyMinProposalNameLength) - if err != nil { - return err - } - pm.Name = hex.EncodeToString(r) - } - if cmd.RFP { - // Set linkby to a month from now - pm.LinkBy = time.Now().Add(time.Hour * 24 * 30).Unix() - } - if cmd.Name != "" { - pm.Name = cmd.Name - } - if cmd.LinkBy != 0 { - pm.LinkBy = cmd.LinkBy - } - if cmd.LinkTo != "" { - pm.LinkTo = cmd.LinkTo - } - pmb, err := json.Marshal(pm) - if err != nil { - return err - } - metadata := []v1.Metadata{ - { - Digest: hex.EncodeToString(util.Digest(pmb)), - Hint: v1.HintProposalMetadata, - Payload: base64.StdEncoding.EncodeToString(pmb), - }, - } - - // Compute merkle root and sign it - sig, err := shared.SignedMerkleRoot(files, metadata, cfg.Identity) - if err != nil { - return fmt.Errorf("SignMerkleRoot: %v", err) - } - - // Setup edit proposal request - ep := &v1.EditProposal{ - Token: token, - Files: files, - Metadata: metadata, - PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), - Signature: sig, - } - - // Print request details - err = shared.PrintJSON(ep) - if err != nil { - return err - } - - // Send request - epr, err := client.EditProposal(ep) - if err != nil { - return err - } - - // Verify proposal censorship record - err = shared.VerifyProposal(epr.Proposal, vr.PubKey) - if err != nil { - return fmt.Errorf("unable to verify proposal %v: %v", - epr.Proposal.CensorshipRecord.Token, err) - } - - // Print response details - return shared.PrintJSON(epr) -} - -// editProposalHelpMsg is the output of the help command when 'editproposal' -// is specified. -const editProposalHelpMsg = `editproposal [flags] "token" "markdownfile" "attachmentfiles" - -Edit a proposal. - -Arguments: -1. token (string, required) Proposal censorship token -2. markdownfile (string, required) Edited proposal -3. attachmentfiles (string, optional) Attachments - -Flags: - --random (bool, optional) Generate a random proposal name & files to submit. - If this flag is used then the markdown file - argument is no longer required and any provided files will be - ignored. - --usemd (bool, optional) Use the existing metadata if value isn't provided explicitly. - --name (string, optional) The name of the proposal - --linkto (string, optional) Token of an existing public proposal to link to. The token is - used to populate the LinkTo field in the proposal data JSON file. - --linkby (int64, optional) UNIX timestamp of RFP deadline. - --rfp (bool, optional) Make the proposal an RFP by inserting a LinkBy timestamp into the - proposal data JSON file. The LinkBy timestamp is set to be one - month from the current time. - This is intended to be used in place of --linkby. - -Request: -{ - "token": (string) Censorship token - "files": [ - { - "name": (string) Filename - "mime": (string) Mime type - "digest": (string) File digest - "payload": (string) File payload - } - ], - "publickey": (string) Public key used to sign proposal - "signature": (string) Signature of the merkle root -} - -Response: -{ - "proposal": { - "name": (string) Suggested short proposal name - "state": (PropStateT) Current state of proposal - "status": (PropStatusT) Current status of proposal - "timestamp": (int64) Timestamp of last update of proposal - "userid": (string) ID of user who submitted proposal - "username": (string) Username of user who submitted proposal - "publickey": (string) Public key used to sign proposal - "signature": (string) Signature of merkle root - "files": [ - { - "name": (string) Filename - "mime": (string) Mime type - "digest": (string) File digest - "payload": (string) File payload - } - ], - "numcomments": (uint) Number of comments on the proposal - "version": (string) Version of proposal - "censorshiprecord": { - "token": (string) Censorship token - "merkle": (string) Merkle root of proposal - "signature": (string) Server side signature of []byte(Merkle+Token) - } - } -}` diff --git a/politeiawww/cmd/piwww/help.go b/politeiawww/cmd/piwww/help.go index 951be998b..ea7097d00 100644 --- a/politeiawww/cmd/piwww/help.go +++ b/politeiawww/cmd/piwww/help.go @@ -25,58 +25,35 @@ func (cmd *HelpCmd) Execute(args []string) error { } switch cmd.Args.Topic { + // Server commands + case "version": + fmt.Printf("%s\n", shared.VersionHelpMsg) + case "policy": + fmt.Printf("%s\n", policyHelpMsg) + + // User commands case "login": fmt.Printf("%s\n", shared.LoginHelpMsg) case "logout": fmt.Printf("%s\n", shared.LogoutHelpMsg) - case "authorizevote": - fmt.Printf("%s\n", authorizeVoteHelpMsg) case "newuser": fmt.Printf("%s\n", newUserHelpMsg) - case "newproposal": - fmt.Printf("%s\n", newProposalHelpMsg) case "changepassword": fmt.Printf("%s\n", shared.ChangePasswordHelpMsg) case "changeusername": fmt.Printf("%s\n", shared.ChangeUsernameHelpMsg) - case "sendfaucettx": - fmt.Printf("%s\n", sendFaucetTxHelpMsg) case "userdetails": fmt.Printf("%s\n", userDetailsHelpMsg) - case "proposaldetails": - fmt.Printf("%s\n", proposalDetailsHelpMsg) - case "userproposals": - fmt.Printf("%s\n", userProposalsHelpMsg) - case "vettedproposals": - fmt.Printf("%s\n", vettedProposalsHelpMsg) - case "setproposalstatus": - fmt.Printf("%s\n", setProposalStatusHelpMsg) - case "newcomment": - fmt.Printf("%s\n", shared.NewCommentHelpMsg) - case "proposalcomments": - fmt.Printf("%s\n", proposalCommentsHelpMsg) - case "censorcomment": - fmt.Printf("%s\n", shared.CensorCommentHelpMsg) - case "likecomment": - fmt.Printf("%s\n", likeCommentHelpMsg) - case "editproposal": - fmt.Printf("%s\n", editProposalHelpMsg) case "manageuser": fmt.Printf("%s\n", shared.ManageUserHelpMsg) case "users": fmt.Printf("%s\n", shared.UsersHelpMsg) case "verifyuseremail": fmt.Printf("%s\n", verifyUserEmailHelpMsg) - case "version": - fmt.Printf("%s\n", shared.VersionHelpMsg) case "edituser": fmt.Printf("%s\n", editUserHelpMsg) - case "subscribe": - fmt.Printf("%s\n", subscribeHelpMsg) case "me": fmt.Printf("%s\n", shared.MeHelpMsg) - case "policy": - fmt.Printf("%s\n", policyHelpMsg) case "resetpassword": fmt.Printf("%s\n", shared.ResetPasswordHelpMsg) case "updateuserkey": @@ -89,37 +66,76 @@ func (cmd *HelpCmd) Execute(args []string) error { fmt.Printf("%s\n", rescanUserPaymentsHelpMsg) case "verifyuserpayment": fmt.Printf("%s\n", verifyUserPaymentHelpMsg) + case "resendverification": + fmt.Printf("%s\n", resendVerificationHelpMsg) + + // Proposal commands + case "proposalnew": + fmt.Printf("%s\n", proposalNewHelpMsg) + case "proposaledit": + fmt.Printf("%s\n", proposalEditHelpMsg) + case "proposalsetstatus": + fmt.Printf("%s\n", proposalSetStatusHelpMsg) + + case "proposaldetails": + fmt.Printf("%s\n", proposalDetailsHelpMsg) + case "userproposals": + fmt.Printf("%s\n", userProposalsHelpMsg) + case "vettedproposals": + fmt.Printf("%s\n", vettedProposalsHelpMsg) + case "batchproposals": + fmt.Printf("%s\n", shared.BatchProposalsHelpMsg) + + // Comment commands + case "newcomment": + fmt.Printf("%s\n", shared.NewCommentHelpMsg) + case "proposalcomments": + fmt.Printf("%s\n", proposalCommentsHelpMsg) + case "censorcomment": + fmt.Printf("%s\n", shared.CensorCommentHelpMsg) + case "likecomment": + fmt.Printf("%s\n", likeCommentHelpMsg) + case "userlikecomments": + fmt.Printf("%s\n", userLikeCommentsHelpMsg) + + // Vote commands + case "authorizevote": + fmt.Printf("%s\n", authorizeVoteHelpMsg) case "startvote": fmt.Printf("%s\n", startVoteHelpMsg) case "startvoterunoff": fmt.Printf("%s\n", startVoteRunoffHelpMsg) case "voteresults": fmt.Printf("%s\n", voteResultsHelpMsg) - case "inventory": - fmt.Printf("%s\n", inventoryHelpMsg) - case "tally": - fmt.Printf("%s\n", tallyHelpMsg) - case "userlikecomments": - fmt.Printf("%s\n", userLikeCommentsHelpMsg) case "activevotes": fmt.Printf("%s\n", activeVotesHelpMsg) case "votestatus": fmt.Printf("%s\n", voteStatusHelpMsg) case "votestatuses": fmt.Printf("%s\n", voteStatusesHelpMsg) - case "vote": - fmt.Printf("%s\n", voteHelpMsg) - case "testrun": - fmt.Printf("%s\n", testRunHelpMsg) - case "resendverification": - fmt.Printf("%s\n", resendVerificationHelpMsg) - case "batchproposals": - fmt.Printf("%s\n", shared.BatchProposalsHelpMsg) case "batchvotesummary": fmt.Printf("%s\n", batchVoteSummaryHelpMsg) case "votedetails": fmt.Printf("%s\n", voteDetailsHelpMsg) + // Websocket commands + case "subscribe": + fmt.Printf("%s\n", subscribeHelpMsg) + + // Other commands + case "testrun": + fmt.Printf("%s\n", testRunHelpMsg) + case "sendfaucettx": + fmt.Printf("%s\n", sendFaucetTxHelpMsg) + + // politeiavoter mock commands + case "inventory": + fmt.Printf("%s\n", inventoryHelpMsg) + case "tally": + fmt.Printf("%s\n", tallyHelpMsg) + case "vote": + fmt.Printf("%s\n", voteHelpMsg) + default: fmt.Printf("invalid command: use 'piwww -h' " + "to view a list of valid commands\n") diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index 97f3782d5..d2dba7d4f 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -1,14 +1,13 @@ -// Copyright (c) 2017-2019 The Decred developers +// Copyright (c) 2017-2020 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. package main import ( - "bytes" "encoding/base64" "encoding/hex" - "errors" + "encoding/json" "fmt" "net/url" "os" @@ -17,10 +16,10 @@ import ( "github.com/decred/dcrd/chaincfg/chainhash" "github.com/decred/dcrd/dcrutil" - "github.com/decred/politeia/politeiad/api/v1/mime" - v1 "github.com/decred/politeia/politeiawww/api/www/v1" + "github.com/decred/politeia/politeiad/api/v1/identity" + pi "github.com/decred/politeia/politeiawww/api/pi/v1" "github.com/decred/politeia/politeiawww/cmd/shared" - "github.com/decred/politeia/util" + wwwutil "github.com/decred/politeia/politeiawww/util" ) const ( @@ -37,21 +36,6 @@ var ( // Config settings defaultHomeDir = dcrutil.AppDataDir(defaultHomeDirname, false) - - // errProposalMDNotFound is emitted when a proposal markdown file - // is required but has not been provided. - errProposalMDNotFound = errors.New("proposal markdown file not " + - "found; you must either provide a markdown file or use the " + - "flag --random") - // errEditProposalRandomAndNameFound is emitted when both --name - // and --random flags found in editproposal command - errEditProposalRandomAndNameFound = errors.New("--random and --name " + - "can't be used together, as --random generates a random name") - // errEditProposalRfpAndLinkbyFound is emitted when both --rfp - // and --linkby flags found in editproposal command - errEditProposalRfpAndLinkbyFound = errors.New("--rfp and --linkby can't " + - "be used together, as --rfp sets the linkby one month " + - "from now") ) type piwww struct { @@ -60,6 +44,11 @@ type piwww struct { // piwww help message. This is handled by go-flags. Config shared.Config + // Proposal commands + ProposalNew ProposalNewCmd `command:"proposalnew"` + ProposalEdit ProposalEditCmd `command:"proposaledit"` + ProposalSetStatus ProposalSetStatusCmd `command:"proposalsetstatus"` + // Commands ActiveVotes ActiveVotesCmd `command:"activevotes" description:"(public) get the proposals that are being voted on"` AuthorizeVote AuthorizeVoteCmd `command:"authorizevote" description:"(user) authorize a proposal vote (must be proposal author)"` @@ -68,7 +57,6 @@ type piwww struct { CensorComment shared.CensorCommentCmd `command:"censorcomment" description:"(admin) censor a comment"` ChangePassword shared.ChangePasswordCmd `command:"changepassword" description:"(user) change the password for the logged in user"` ChangeUsername shared.ChangeUsernameCmd `command:"changeusername" description:"(user) change the username for the logged in user"` - EditProposal EditProposalCmd `command:"editproposal" description:"(user) edit a proposal"` EditUser EditUserCmd `command:"edituser" description:"(user) edit the preferences of the logged in user"` Help HelpCmd `command:"help" description:" print a detailed help message for a specific command"` Inventory InventoryCmd `command:"inventory" description:"(public) get the proposals that are being voted on"` @@ -78,7 +66,6 @@ type piwww struct { ManageUser shared.ManageUserCmd `command:"manageuser" description:"(admin) edit certain properties of the specified user"` Me shared.MeCmd `command:"me" description:"(user) get user details for the logged in user"` NewComment shared.NewCommentCmd `command:"newcomment" description:"(user) create a new comment"` - NewProposal NewProposalCmd `command:"newproposal" description:"(user) create a new proposal"` NewUser NewUserCmd `command:"newuser" description:"(public) create a new user"` Policy PolicyCmd `command:"policy" description:"(public) get the server policy"` ProposalComments ProposalCommentsCmd `command:"proposalcomments" description:"(public) get the comments for a proposal"` @@ -89,7 +76,6 @@ type piwww struct { ResetPassword shared.ResetPasswordCmd `command:"resetpassword" description:"(public) reset the password for a user that is not logged in"` Secret shared.SecretCmd `command:"secret" description:"(user) ping politeiawww"` SendFaucetTx SendFaucetTxCmd `command:"sendfaucettx" description:" send a DCR transaction using the Decred testnet faucet"` - SetProposalStatus SetProposalStatusCmd `command:"setproposalstatus" description:"(admin) set the status of a proposal"` SetTOTP shared.SetTOTPCmd `command:"settotp" description:"(user) set the key for TOTP"` StartVote StartVoteCmd `command:"startvote" description:"(admin) start the voting period on a proposal"` StartVoteRunoff StartVoteRunoffCmd `command:"startvoterunoff" description:"(admin) start a runoff using the submissions to an RFP"` @@ -115,26 +101,19 @@ type piwww struct { VoteStatuses VoteStatusesCmd `command:"votestatuses" description:"(public) get the vote status for all public proposals"` } -// createMDFile returns a File object that was created using a markdown file -// filled with random text. -func createMDFile() (*v1.File, error) { - var b bytes.Buffer - b.WriteString("This is the proposal title\n") - - for i := 0; i < 10; i++ { - r, err := util.Random(32) - if err != nil { - return nil, err - } - b.WriteString(base64.StdEncoding.EncodeToString(r) + "\n") +// signedMerkleRoot calculates the merkle root of the passed in list of files +// and metadata, signs the merkle root with the passed in identity and returns +// the signature. +func signedMerkleRoot(files []pi.File, md []pi.Metadata, id *identity.FullIdentity) (string, error) { + if len(files) == 0 { + return "", fmt.Errorf("no proposal files found") } - - return &v1.File{ - Name: "index.md", - MIME: mime.DetectMimeType(b.Bytes()), - Digest: hex.EncodeToString(util.Digest(b.Bytes())), - Payload: base64.StdEncoding.EncodeToString(b.Bytes()), - }, nil + mr, err := wwwutil.MerkleRoot(files, md) + if err != nil { + return "", err + } + sig := id.SignMessage([]byte(mr)) + return hex.EncodeToString(sig[:]), nil } // convertTicketHashes converts a slice of hexadecimal ticket hashes into @@ -151,27 +130,76 @@ func convertTicketHashes(h []string) ([][]byte, error) { return hashes, nil } +// proposalRecord returns the ProposalRecord for the provided token and +// version. +func proposalRecord(state pi.PropStateT, token, version string) (*pi.ProposalRecord, error) { + ps := pi.Proposals{ + State: state, + Requests: []pi.ProposalRequest{ + { + Token: token, + }, + }, + IncludeFiles: false, + } + psr, err := client.Proposals(ps) + if err != nil { + return nil, err + } + pr, ok := psr.Proposals[token] + if !ok { + return nil, fmt.Errorf("proposal not found") + } + + return &pr, nil +} + +// proposalRecord returns the latest ProposalRecrord version for the provided +// token. +func proposalRecordLatest(state pi.PropStateT, token string) (*pi.ProposalRecord, error) { + return proposalRecord(state, token, "") +} + +// decodeProposalMetadata decodes and returns a ProposalMetadata given the +// metadata array from a ProposalRecord. +func decodeProposalMetadata(metadata []pi.Metadata) (*pi.ProposalMetadata, error) { + var pm *pi.ProposalMetadata + for _, v := range metadata { + if v.Hint == pi.HintProposalMetadata { + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return nil, err + } + err = json.Unmarshal(b, pm) + if err != nil { + return nil, err + } + } + } + if pm == nil { + return nil, fmt.Errorf("proposal metadata not found") + } + return pm, nil +} + func _main() error { - // Load config - _cfg, err := shared.LoadConfig(defaultHomeDir, + // Load config. The config variable is a CLI global variable. + var err error + cfg, err = shared.LoadConfig(defaultHomeDir, defaultDataDirname, defaultConfigFilename) if err != nil { return fmt.Errorf("load config: %v", err) } - // Load client - _client, err := shared.NewClient(_cfg) + // Load client. The client variable is a CLI global variable. + client, err = shared.NewClient(cfg) if err != nil { return fmt.Errorf("load client: %v", err) } - // Setup global variables for piwww commands - cfg = _cfg - client = _client - // Setup global variables for shared commands - shared.SetConfig(_cfg) - shared.SetClient(_client) + shared.SetConfig(cfg) + shared.SetClient(client) // Get politeiawww CSRF token if cfg.CSRF == "" { diff --git a/politeiawww/cmd/piwww/proposaledit.go b/politeiawww/cmd/piwww/proposaledit.go new file mode 100644 index 000000000..e4f6b9526 --- /dev/null +++ b/politeiawww/cmd/piwww/proposaledit.go @@ -0,0 +1,263 @@ +// Copyright (c) 2017-2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "io/ioutil" + "path/filepath" + "time" + + "github.com/decred/politeia/politeiad/api/v1/mime" + pi "github.com/decred/politeia/politeiawww/api/pi/v1" + v1 "github.com/decred/politeia/politeiawww/api/www/v1" + "github.com/decred/politeia/politeiawww/cmd/shared" + "github.com/decred/politeia/util" +) + +// ProposalEditCmd edits an existing proposal. +type ProposalEditCmd struct { + Args struct { + Token string `positional-arg-name:"token" required:"true"` + IndexFile string `positional-arg-name:"indexfile"` + Attachments []string `positional-arg-name:"attachmets"` + } `positional-args:"true" optional:"true"` + + // CLI flags + Vetted bool `long:"vetted" optional:"true"` + Unvetted bool `long:"unvetted" optional:"true"` + Name string `long:"name" optional:"true"` + LinkTo string `long:"linkto" optional:"true"` + LinkBy int64 `long:"linkby" optional:"true"` + + // Random generates random proposal data. An IndexFile and + // Attachments are not required when using this flag. + Random bool `long:"random" optional:"true"` + + // RFP is a flag that is intended to make editing an RFP easier + // by calculating and inserting a linkby timestamp automatically + // instead of having to pass in a specific timestamp using the + // --linkby flag. + RFP bool `long:"rfp" optional:"true"` + + // UseMD is a flag that is intended to make editing proposal + // metadata easier by using exisiting proposal metadata values + // instead of having to pass in specific values + UseMD bool `long:"usemd" optional:"true"` +} + +// Execute executes the proposal edit command. +func (cmd *ProposalEditCmd) Execute(args []string) error { + token := cmd.Args.Token + indexFile := cmd.Args.IndexFile + attachments := cmd.Args.Attachments + + // Verify arguments + switch { + case !cmd.Random && indexFile == "": + return fmt.Errorf("index file not found; you must either provide an " + + "index.md file or use --random") + case cmd.RFP && cmd.LinkBy != 0: + return fmt.Errorf("--rfp and --linkby can not be used together, as " + + "--rfp sets the linkby one month from now") + case cmd.Random && cmd.Name != "": + return fmt.Errorf("--random and --name can not be used together, as " + + "--random generates a random name and random proposal data") + } + + // Verify state + var state pi.PropStateT + switch { + case cmd.Vetted && cmd.Unvetted: + return fmt.Errorf("cannot use --vetted and --unvetted simultaneously") + case cmd.Unvetted: + state = pi.PropStateUnvetted + case cmd.Vetted: + state = pi.PropStateVetted + default: + return fmt.Errorf("must specify either --vetted or unvetted") + } + + // Check for user identity. A user identity is required to sign + // the proposal files and metadata. + if cfg.Identity == nil { + return shared.ErrUserIdentityNotFound + } + + // Prepare index file + var payload []byte + if cmd.Random { + // Generate random text for the index file + var b bytes.Buffer + for i := 0; i < 10; i++ { + r, err := util.Random(32) + if err != nil { + return err + } + b.WriteString(base64.StdEncoding.EncodeToString(r) + "\n") + } + payload = b.Bytes() + } else { + // Read the index file from disk + fp := util.CleanAndExpandPath(indexFile) + var err error + payload, err = ioutil.ReadFile(fp) + if err != nil { + return fmt.Errorf("ReadFile %v: %v", fp, err) + } + } + + files := []pi.File{ + { + Name: v1.PolicyIndexFilename, + MIME: mime.DetectMimeType(payload), + Digest: hex.EncodeToString(util.Digest(payload)), + Payload: base64.StdEncoding.EncodeToString(payload), + }, + } + + // Prepare attachment files + for _, fn := range attachments { + fp := util.CleanAndExpandPath(fn) + payload, err := ioutil.ReadFile(fp) + if err != nil { + return fmt.Errorf("ReadFile %v: %v", fp, err) + } + + files = append(files, pi.File{ + Name: filepath.Base(fn), + MIME: mime.DetectMimeType(payload), + Digest: hex.EncodeToString(util.Digest(payload)), + Payload: base64.StdEncoding.EncodeToString(payload), + }) + } + + // Setup metadata + var pm pi.ProposalMetadata + if cmd.UseMD { + // Get the existing proposal metadata + pr, err := proposalRecordLatest(state, cmd.Args.Token) + if err != nil { + return err + } + pmCurr, err := decodeProposalMetadata(pr.Metadata) + if err != nil { + return err + } + + // Prefill proposal metadata with existing values + pm.Name = pmCurr.Name + pm.LinkTo = pmCurr.LinkTo + pm.LinkBy = pmCurr.LinkBy + } + if cmd.Random { + // Generate random name + r, err := util.Random(v1.PolicyMinProposalNameLength) + if err != nil { + return err + } + pm.Name = hex.EncodeToString(r) + } + if cmd.RFP { + // Set linkby to a month from now + pm.LinkBy = time.Now().Add(time.Hour * 24 * 30).Unix() + } + if cmd.Name != "" { + pm.Name = cmd.Name + } + if cmd.LinkBy != 0 { + pm.LinkBy = cmd.LinkBy + } + if cmd.LinkTo != "" { + pm.LinkTo = cmd.LinkTo + } + pmb, err := json.Marshal(pm) + if err != nil { + return err + } + metadata := []pi.Metadata{ + { + Digest: hex.EncodeToString(util.Digest(pmb)), + Hint: pi.HintProposalMetadata, + Payload: base64.StdEncoding.EncodeToString(pmb), + }, + } + + // Setup edit proposal request + sig, err := signedMerkleRoot(files, metadata, cfg.Identity) + if err != nil { + return err + } + pe := pi.ProposalEdit{ + Token: token, + State: state, + Files: files, + Metadata: metadata, + PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), + Signature: sig, + } + + // Send request. The request and response details are printed to + // the console based on the logging flags that were used. + err = shared.PrintJSON(pe) + if err != nil { + return err + } + per, err := client.ProposalEdit(pe) + if err != nil { + return err + } + err = shared.PrintJSON(per) + if err != nil { + return err + } + + // Verify proposal censorship record + /* + // TODO + vr, err := client.Version() + if err != nil { + return err + } + err = shared.VerifyProposal(per.Proposal, vr.PubKey) + if err != nil { + return fmt.Errorf("unable to verify proposal %v: %v", + per.Proposal.CensorshipRecord.Token, err) + } + */ + + return nil +} + +// proposalEditHelpMsg is the output of the help command. +const proposalEditHelpMsg = `editproposal [flags] "token" "indexfile" "attachments" + +Edit a proposal. + +Arguments: +1. token (string, required) Proposal censorship token +2. indexfile (string, required) Index file +3. attachments (string, optional) Attachment files + +Flags: + --random (bool, optional) Generate a random proposal name & files to + submit. If this flag is used then the markdown + file argument is no longer required and any + provided files will be ignored. + --usemd (bool, optional) Use the existing proposal metadata. + --name (string, optional) The name of the proposal. + --linkto (string, optional) Censorship token of an existing public proposal + to link to. + --linkby (int64, optional) UNIX timestamp of RFP deadline. + --rfp (bool, optional) Make the proposal an RFP by inserting a LinkBy + timestamp into the proposal metadata. The LinkBy + timestamp is set to be one month from the + current time. This is intended to be used in + place of --linkby. +` diff --git a/politeiawww/cmd/piwww/newproposal.go b/politeiawww/cmd/piwww/proposalnew.go similarity index 51% rename from politeiawww/cmd/piwww/newproposal.go rename to politeiawww/cmd/piwww/proposalnew.go index 2e0e69672..be55041e6 100644 --- a/politeiawww/cmd/piwww/newproposal.go +++ b/politeiawww/cmd/piwww/proposalnew.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Decred developers +// Copyright (c) 2017-2020 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -15,25 +15,28 @@ import ( "time" "github.com/decred/politeia/politeiad/api/v1/mime" + pi "github.com/decred/politeia/politeiawww/api/pi/v1" v1 "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/cmd/shared" "github.com/decred/politeia/util" ) -// NewProposalCmd submits a new proposal. -type NewProposalCmd struct { +// TODO replace www policies with pi policies + +// ProposalNewCmd submits a new proposal. +type ProposalNewCmd struct { Args struct { - Markdown string `positional-arg-name:"markdownfile"` - Attachments []string `positional-arg-name:"attachmentfiles"` + IndexFile string `positional-arg-name:"indexfile"` + Attachments []string `positional-arg-name:"attachments"` } `positional-args:"true" optional:"true"` + + // CLI flags Name string `long:"name" optional:"true"` LinkTo string `long:"linkto" optional:"true"` LinkBy int64 `long:"linkby" optional:"true"` - // Random can be used in place of submitting proposal files. When - // specified, random proposal data will be created and submitted. - // The --random flag cannot be used if proposal files are provided - // as arguments. + // Random generates random proposal data. An IndexFile and + // Attachments are not required when using this flag. Random bool `long:"random" optional:"true"` // RFP is a flag that is intended to make submitting an RFP easier @@ -44,20 +47,24 @@ type NewProposalCmd struct { } // Execute executes the new proposal command. -func (cmd *NewProposalCmd) Execute(args []string) error { - mdFile := cmd.Args.Markdown - attachmentFiles := cmd.Args.Attachments +func (cmd *ProposalNewCmd) Execute(args []string) error { + indexFile := cmd.Args.IndexFile + attachments := cmd.Args.Attachments // Validate arguments switch { - case cmd.Random && mdFile != "": + case !cmd.Random && indexFile == "": + return fmt.Errorf("index file not found; you must either provide an " + + "index.md file or use --random") + + case cmd.Random && indexFile != "": return fmt.Errorf("you cannot provide file arguments and use the " + "--random flag at the same time") - case !cmd.Random && mdFile == "": - return errProposalMDNotFound + case !cmd.Random && cmd.Name == "": return fmt.Errorf("you must either provide a proposal name using the " + "--name flag or use the --random flag to generate a random name") + case cmd.RFP && cmd.LinkBy != 0: return fmt.Errorf("you cannot use both the --rfp and --linkby flags " + "at the same time") @@ -69,17 +76,10 @@ func (cmd *NewProposalCmd) Execute(args []string) error { return shared.ErrUserIdentityNotFound } - // Get server public key. This will be used to verify the reply. - vr, err := client.Version() - if err != nil { - return err - } - - // Prepare proposal index file - var md []byte - files := make([]v1.File, 0, v1.PolicyMaxImages+1) + // Prepare index file + var payload []byte if cmd.Random { - // Generate random proposal markdown text + // Generate random text for the index file var b bytes.Buffer for i := 0; i < 10; i++ { r, err := util.Random(32) @@ -88,44 +88,40 @@ func (cmd *NewProposalCmd) Execute(args []string) error { } b.WriteString(base64.StdEncoding.EncodeToString(r) + "\n") } - - md = b.Bytes() + payload = b.Bytes() } else { - // Read markdown file into memory and convert to type File - fpath := util.CleanAndExpandPath(mdFile) - + // Read the index file from disk + fp := util.CleanAndExpandPath(indexFile) var err error - md, err = ioutil.ReadFile(fpath) + payload, err = ioutil.ReadFile(fp) if err != nil { - return fmt.Errorf("ReadFile %v: %v", fpath, err) + return fmt.Errorf("ReadFile %v: %v", fp, err) } } - f := v1.File{ - Name: v1.PolicyIndexFilename, - MIME: mime.DetectMimeType(md), - Digest: hex.EncodeToString(util.Digest(md)), - Payload: base64.StdEncoding.EncodeToString(md), + files := []pi.File{ + { + Name: v1.PolicyIndexFilename, + MIME: mime.DetectMimeType(payload), + Digest: hex.EncodeToString(util.Digest(payload)), + Payload: base64.StdEncoding.EncodeToString(payload), + }, } - files = append(files, f) - // Prepare attachment files - for _, file := range attachmentFiles { - path := util.CleanAndExpandPath(file) - attachment, err := ioutil.ReadFile(path) + for _, fn := range attachments { + fp := util.CleanAndExpandPath(fn) + payload, err := ioutil.ReadFile(fp) if err != nil { - return fmt.Errorf("ReadFile %v: %v", path, err) - } - - f := v1.File{ - Name: filepath.Base(file), - MIME: mime.DetectMimeType(attachment), - Digest: hex.EncodeToString(util.Digest(attachment)), - Payload: base64.StdEncoding.EncodeToString(attachment), + return fmt.Errorf("ReadFile %v: %v", fp, err) } - files = append(files, f) + files = append(files, pi.File{ + Name: filepath.Base(fn), + MIME: mime.DetectMimeType(payload), + Digest: hex.EncodeToString(util.Digest(payload)), + Payload: base64.StdEncoding.EncodeToString(payload), + }) } // Setup metadata @@ -140,7 +136,7 @@ func (cmd *NewProposalCmd) Execute(args []string) error { // Set linkby to a month from now cmd.LinkBy = time.Now().Add(time.Hour * 24 * 30).Unix() } - pm := v1.ProposalMetadata{ + pm := pi.ProposalMetadata{ Name: cmd.Name, LinkTo: cmd.LinkTo, LinkBy: cmd.LinkBy, @@ -149,60 +145,67 @@ func (cmd *NewProposalCmd) Execute(args []string) error { if err != nil { return err } - metadata := []v1.Metadata{ + metadata := []pi.Metadata{ { Digest: hex.EncodeToString(util.Digest(pmb)), - Hint: v1.HintProposalMetadata, + Hint: pi.HintProposalMetadata, Payload: base64.StdEncoding.EncodeToString(pmb), }, } // Setup new proposal request - sig, err := shared.SignedMerkleRoot(files, metadata, cfg.Identity) + sig, err := signedMerkleRoot(files, metadata, cfg.Identity) if err != nil { return err } - np := &v1.NewProposal{ + pn := pi.ProposalNew{ Files: files, Metadata: metadata, PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), Signature: sig, } - // Send the new proposal request. The request and response details - // are printed to the console based on the logging flags that are - // used. - err = shared.PrintJSON(np) + // Send request. The request and response details are printed to + // the console based on the logging flags that were used. + err = shared.PrintJSON(pn) if err != nil { return err } - npr, err := client.NewProposal(np) + pnr, err := client.ProposalNew(pn) if err != nil { return err } - err = shared.PrintJSON(npr) + err = shared.PrintJSON(pnr) if err != nil { return err } // Verify the censorship record - pr := v1.ProposalRecord{ - Files: np.Files, - Metadata: np.Metadata, - PublicKey: np.PublicKey, - Signature: np.Signature, - CensorshipRecord: npr.CensorshipRecord, - } - err = shared.VerifyProposal(pr, vr.PubKey) - if err != nil { - return fmt.Errorf("unable to verify proposal %v: %v", - pr.CensorshipRecord.Token, err) - } + /* + // TODO implement this using pi types + vr, err := client.Version() + if err != nil { + return err + } + pr := v1.ProposalRecord{ + Files: np.Files, + Metadata: np.Metadata, + PublicKey: np.PublicKey, + Signature: np.Signature, + CensorshipRecord: npr.CensorshipRecord, + } + err = shared.VerifyProposal(pr, vr.PubKey) + if err != nil { + return fmt.Errorf("unable to verify proposal %v: %v", + pr.CensorshipRecord.Token, err) + } + */ return nil } -const newProposalHelpMsg = `newproposal [flags] "markdownFile" "attachmentFiles" +// proposalNewHelpMsg is the output of the help command. +const proposalNewHelpMsg = `proposalnew [flags] "indexfile" "attachments" Submit a new proposal to Politeia. A proposal is defined as a single markdown file with the filename "index.md" and optional attachment png files. No other @@ -216,18 +219,19 @@ A proposal can be submitted as an RFP submission by using the --linkto flag to link to and existing RFP proposal. Arguments: -1. markdownFile (string, required) Proposal -2. attachmentFiles (string, optional) Attachments +1. indexfile (string, required) Index file +2. attachments (string, optional) Attachment files Flags: - --name (string, optional) The name of the proposal + --name (string, optional) The name of the proposal. --linkto (string, optional) Token of an existing public proposal to link to. - --linkby (int64, optional) UNIX timestamp of RFP deadline. Setting the linkby of a proposal will - make the proposal an RFP with a submission deadline specified by the - linkby. - --random (bool, optional) Generate a random proposal. If this flag is used then the markdown - file argument is no longer required and any provided files will be - ignored. - --rfp (bool, optional) Make the proposal an RFP by setting the linkby to one month from the - current time. This is intended to be used in place of --linkby. + --linkby (int64, optional) UNIX timestamp of the RFP deadline. Setting this + field will make the proposal an RFP with a + submission deadline specified by the linkby. + --random (bool, optional) Generate a random proposal. If this flag is used + then the markdownfile argument is no longer + required and any provided files will be ignored. + --rfp (bool, optional) Make the proposal an RFP by setting the linkby to + one month from the current time. This is intended + to be used in place of --linkby. ` diff --git a/politeiawww/cmd/piwww/proposalsetstatus.go b/politeiawww/cmd/piwww/proposalsetstatus.go new file mode 100644 index 000000000..540640116 --- /dev/null +++ b/politeiawww/cmd/piwww/proposalsetstatus.go @@ -0,0 +1,123 @@ +// Copyright (c) 2017-2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "encoding/hex" + "fmt" + "strconv" + + pi "github.com/decred/politeia/politeiawww/api/pi/v1" + "github.com/decred/politeia/politeiawww/cmd/shared" +) + +// ProposalSetStatusCmd sets the status of a proposal. +type ProposalSetStatusCmd struct { + Args struct { + Token string `positional-arg-name:"token" required:"true"` + Status string `positional-arg-name:"status" required:"true"` + Reason string `positional-arg-name:"reason"` + } `positional-args:"true"` + Vetted bool `long:"vetted" optional:"true"` + Unvetted bool `long:"unvetted" optional:"true"` +} + +// Execute executes the set proposal status command. +func (cmd *ProposalSetStatusCmd) Execute(args []string) error { + propStatus := map[string]pi.PropStatusT{ + "public": pi.PropStatusPublic, + "censored": pi.PropStatusCensored, + "abandoned": pi.PropStatusAbandoned, + } + + // Verify state + var state pi.PropStateT + switch { + case cmd.Vetted && cmd.Unvetted: + return fmt.Errorf("cannot use --vetted and --unvetted simultaneously") + case cmd.Unvetted: + state = pi.PropStateUnvetted + case cmd.Vetted: + state = pi.PropStateVetted + default: + return fmt.Errorf("must specify either --vetted or unvetted") + } + + // Validate user identity + if cfg.Identity == nil { + return shared.ErrUserIdentityNotFound + } + + // Parse proposal status. This can be either the numeric status + // code or the human readable equivalent. + var status pi.PropStatusT + s, err := strconv.ParseUint(cmd.Args.Status, 10, 32) + if err == nil { + // Numeric status code found + status = pi.PropStatusT(s) + } else if s, ok := propStatus[cmd.Args.Status]; ok { + // Human readable status code found + status = s + } else { + return fmt.Errorf("Invalid proposal status '%v'. Valid statuses are:\n"+ + " public make a proposal public\n"+ + " censored censor a proposal\n"+ + " abandoned declare a public proposal abandoned", + cmd.Args.Status) + } + + // Get the proposal. The latest proposal version number is needed + // for the set status request. + pr, err := proposalRecordLatest(state, cmd.Args.Token) + if err != nil { + return err + } + + // Setup request + msg := cmd.Args.Token + pr.Version + cmd.Args.Status + cmd.Args.Reason + sig := cfg.Identity.SignMessage([]byte(msg)) + pss := pi.ProposalSetStatus{ + Token: cmd.Args.Token, + State: state, + Version: pr.Version, + Status: status, + Reason: cmd.Args.Reason, + Signature: hex.EncodeToString(sig[:]), + PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), + } + + // Send request. The request and response details are printed to + // the console based on the logging flags that were used. + err = shared.PrintJSON(pss) + if err != nil { + return err + } + pssr, err := client.ProposalSetStatus(pss) + if err != nil { + return err + } + err = shared.PrintJSON(pssr) + if err != nil { + return err + } + + return nil +} + +// proposalSetStatusHelpMsg is the output of the help command. +const proposalSetStatusHelpMsg = `proposalsetstatus "token" "status" "reason" + +Set the status of a proposal. Requires admin privileges. + +Valid statuses: + public + censored + abandoned + +Arguments: +1. token (string, required) Proposal censorship token +2. status (string, required) New status +3. message (string, optional) Status change message +` diff --git a/politeiawww/cmd/piwww/setproposalstatus.go b/politeiawww/cmd/piwww/setproposalstatus.go deleted file mode 100644 index 0b3729df1..000000000 --- a/politeiawww/cmd/piwww/setproposalstatus.go +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import ( - "encoding/hex" - "fmt" - "strconv" - - "github.com/decred/politeia/politeiawww/api/www/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" -) - -// SetProposalStatusCmd sets the status of a proposal. -type SetProposalStatusCmd struct { - Args struct { - Token string `positional-arg-name:"token" required:"true"` // Censorship token - Status string `positional-arg-name:"status" required:"true"` // New status - Message string `positional-arg-name:"message"` // Change message - } `positional-args:"true"` -} - -// Execute executes the set proposal status command. -func (cmd *SetProposalStatusCmd) Execute(args []string) error { - PropStatus := map[string]v1.PropStatusT{ - "censored": v1.PropStatusCensored, - "public": v1.PropStatusPublic, - "abandoned": v1.PropStatusAbandoned, - } - - // Validate user identity - if cfg.Identity == nil { - return shared.ErrUserIdentityNotFound - } - - // Parse proposal status. This can be either the numeric - // status code or the human readable equivalent. - var status v1.PropStatusT - s, err := strconv.ParseUint(cmd.Args.Status, 10, 32) - if err == nil { - // Numeric status code found - status = v1.PropStatusT(s) - } else if s, ok := PropStatus[cmd.Args.Status]; ok { - // Human readable status code found - status = s - } else { - return fmt.Errorf("Invalid proposal status '%v'. "+ - "Valid statuses are:\n"+ - " censored censor a proposal\n"+ - " public make a proposal public\n"+ - " abandoned declare a public proposal abandoned", - cmd.Args.Status) - } - - // Setup request - sig := cfg.Identity.SignMessage([]byte(cmd.Args.Token + - strconv.Itoa(int(status)) + cmd.Args.Message)) - sps := &v1.SetProposalStatus{ - Token: cmd.Args.Token, - ProposalStatus: status, - StatusChangeMessage: cmd.Args.Message, - Signature: hex.EncodeToString(sig[:]), - PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), - } - - // Print request details - err = shared.PrintJSON(sps) - if err != nil { - return err - } - - // Send request - spsr, err := client.SetProposalStatus(sps) - if err != nil { - return err - } - - // Print response details - return shared.PrintJSON(spsr) -} - -// setProposalStatusHelpMsg is the output of the help command when -// "setproposalstatus" is specified. -const setProposalStatusHelpMsg = `setproposalstatus "token" "status" - -Set the status of a proposal. Requires admin privileges. - -Arguments: -1. token (string, required) Proposal censorship token -2. status (string, required) New status (censored, public, abandoned) -3. message (string, required if censoring proposal) Status change message - -Request: -{ - "token": (string) Censorship token - "proposalstatus": (PropStatusT) Proposal status code - "signature": (string) Signature of proposal status change - "publickey": (string) Public key of user changing proposal status -} - -Response: -{ - "proposal": { - "name": (string) Suggested short proposal name - "state": (PropStateT) Current state of proposal - "status": (PropStatusT) Current status of proposal - "timestamp": (int64) Timestamp of last update of proposal - "userid": (string) ID of user who submitted proposal - "username": (string) Username of user who submitted proposal - "publickey": (string) Public key used to sign proposal - "signature": (string) Signature of merkle root - "files": [ - { - "name": (string) Filename - "mime": (string) Mime type - "digest": (string) File digest - "payload": (string) File payload - } - ], - "numcomments": (uint) Number of comments on proposal - "version": (string) Version of proposal - "censorshiprecord": { - "token": (string) Censorship token - "merkle": (string) Merkle root of proposal - "signature": (string) Server side signature of []byte(Merkle+Token) - } - } -}` diff --git a/politeiawww/cmd/piwww/testrun.go b/politeiawww/cmd/piwww/testrun.go index 4fc1cfe51..db20c602e 100644 --- a/politeiawww/cmd/piwww/testrun.go +++ b/politeiawww/cmd/piwww/testrun.go @@ -5,19 +5,11 @@ package main import ( - "encoding/base64" - "encoding/hex" - "encoding/json" "fmt" - "strconv" "strings" - "time" - "github.com/decred/dcrwallet/rpc/walletrpc" - "github.com/decred/politeia/decredplugin" v1 "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/cmd/shared" - "github.com/decred/politeia/util" ) // TestRunCmd performs a test run of all the politeiawww routes. @@ -66,44 +58,48 @@ func newNormalProposal() (*v1.NewProposal, error) { // markdown text and a signature from the logged in user. If given `rfp` bool // is true it creates an RFP. If given `linkto` it creates a RFP submission. func newProposal(rfp bool, linkto string) (*v1.NewProposal, error) { - md, err := createMDFile() - if err != nil { - return nil, fmt.Errorf("create MD file: %v", err) - } - files := []v1.File{*md} + /* + // TODO + md, err := createMDFile() + if err != nil { + return nil, fmt.Errorf("create MD file: %v", err) + } + files := []v1.File{*md} - pm := v1.ProposalMetadata{ - Name: "Some proposal name", - } - if rfp { - pm.LinkBy = time.Now().Add(time.Hour * 24 * 30).Unix() - } - if linkto != "" { - pm.LinkTo = linkto - } - pmb, err := json.Marshal(pm) - if err != nil { - return nil, err - } - metadata := []v1.Metadata{ - { - Digest: hex.EncodeToString(util.Digest(pmb)), - Hint: v1.HintProposalMetadata, - Payload: base64.StdEncoding.EncodeToString(pmb), - }, - } + pm := v1.ProposalMetadata{ + Name: "Some proposal name", + } + if rfp { + pm.LinkBy = time.Now().Add(time.Hour * 24 * 30).Unix() + } + if linkto != "" { + pm.LinkTo = linkto + } + pmb, err := json.Marshal(pm) + if err != nil { + return nil, err + } + metadata := []v1.Metadata{ + { + Digest: hex.EncodeToString(util.Digest(pmb)), + Hint: v1.HintProposalMetadata, + Payload: base64.StdEncoding.EncodeToString(pmb), + }, + } - sig, err := shared.SignedMerkleRoot(files, metadata, cfg.Identity) - if err != nil { - return nil, fmt.Errorf("sign merkle root: %v", err) - } + sig, err := shared.SignedMerkleRoot(files, metadata, cfg.Identity) + if err != nil { + return nil, fmt.Errorf("sign merkle root: %v", err) + } - return &v1.NewProposal{ - Files: files, - Metadata: metadata, - PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), - Signature: sig, - }, nil + return &v1.NewProposal{ + Files: files, + Metadata: metadata, + PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), + Signature: sig, + }, nil + */ + return nil, nil } // castVotes casts votes on a proposal with a given voteId. If it fails it @@ -137,1407 +133,1319 @@ func castVotes(token string, voteID string) (bool, error) { // Execute executes the test run command. func (cmd *TestRunCmd) Execute(args []string) error { - const ( - // sleepInterval is the time to wait in between requests - // when polling politeiawww for paywall tx confirmations - // or RFP vote results. - sleepInterval = 15 * time.Second - - // Comment actions - commentActionUpvote = "upvote" - commentActionDownvote = "downvote" - ) - - var ( - // paywallEnabled represents whether the politeiawww paywall - // has been enabled. A disabled paywall will have a paywall - // address of "" and a paywall amount of 0. - paywallEnabled bool - - // numCredits is the number of proposal credits that will be - // purchased using the testnet faucet. - numCredits = v1.ProposalListPageSize * 2 - - // Test users - user testUser - admin testUser - ) - - // Suppress output from cli commands - cfg.Silent = true - - // Policy - fmt.Printf("Policy\n") - policy, err := client.Policy() - if err != nil { - return err - } - - // Version (CSRF tokens) - fmt.Printf("Version\n") - version, err := client.Version() - if err != nil { - return err - } - - // We only allow this to be run on testnet for right now. - // Running it on mainnet would require changing the user - // email verification flow. - // We ensure vote duration isn't longer than - // 3 blocks as we need to approve an RFP and it's - // submission as part of our tests. - switch { - case !version.TestNet: - return fmt.Errorf("this command must be run on testnet") - case policy.MinVoteDuration > 3: - return fmt.Errorf("--votedurationmin flag should be <= 3, as the " + - "tests include RFP & submssions voting") - } - - // Ensure admin credentials are valid and that the admin has - // paid their user registration fee. - fmt.Printf("Validating admin credentials\n") - admin.email = cmd.Args.AdminEmail - admin.password = cmd.Args.AdminPassword - err = login(admin.email, admin.password) - if err != nil { - return err - } - - vupr, err := client.VerifyUserPayment() - if err != nil { - return err - } - if !vupr.HasPaid { - return fmt.Errorf("admin has not paid registration fee") - } - - lc := shared.LogoutCmd{} - err = lc.Execute(nil) - if err != nil { - return err - } - - // Create user and verify email - b, err := util.Random(int(policy.MinPasswordLength)) - if err != nil { - return err - } - - email := hex.EncodeToString(b) + "@example.com" - username := hex.EncodeToString(b) - password := hex.EncodeToString(b) - - fmt.Printf("Creating user: %v\n", email) - - nuc := NewUserCmd{ - Verify: true, - } - nuc.Args.Email = email - nuc.Args.Username = username - nuc.Args.Password = password - err = nuc.Execute(nil) - if err != nil { - return err - } - - // Login and store user details - fmt.Printf("Login user\n") - lr, err := client.Login( - &v1.Login{ - Email: email, - Password: shared.DigestSHA3(password), - }) - if err != nil { - return err - } + /* + const ( + // sleepInterval is the time to wait in between requests + // when polling politeiawww for paywall tx confirmations + // or RFP vote results. + sleepInterval = 15 * time.Second + + // Comment actions + commentActionUpvote = "upvote" + commentActionDownvote = "downvote" + ) + + var ( + // paywallEnabled represents whether the politeiawww paywall + // has been enabled. A disabled paywall will have a paywall + // address of "" and a paywall amount of 0. + paywallEnabled bool + + // numCredits is the number of proposal credits that will be + // purchased using the testnet faucet. + numCredits = v1.ProposalListPageSize * 2 + + // Test users + user testUser + admin testUser + ) + + // Suppress output from cli commands + cfg.Silent = true + + // Policy + fmt.Printf("Policy\n") + policy, err := client.Policy() + if err != nil { + return err + } - user = testUser{ - ID: lr.UserID, - email: email, - username: username, - password: password, - publicKey: lr.PublicKey, - } + // Version (CSRF tokens) + fmt.Printf("Version\n") + version, err := client.Version() + if err != nil { + return err + } - // Check if paywall is enabled. Paywall address and paywall - // amount will be zero values if paywall has been disabled. - if lr.PaywallAddress != "" && lr.PaywallAmount != 0 { - paywallEnabled = true - } else { - fmt.Printf("WARNING: politeiawww paywall is disabled\n") - } + // We only allow this to be run on testnet for right now. + // Running it on mainnet would require changing the user + // email verification flow. + // We ensure vote duration isn't longer than + // 3 blocks as we need to approve an RFP and it's + // submission as part of our tests. + switch { + case !version.TestNet: + return fmt.Errorf("this command must be run on testnet") + case policy.MinVoteDuration > 3: + return fmt.Errorf("--votedurationmin flag should be <= 3, as the " + + "tests include RFP & submssions voting") + } - // Run user routes. These are the routes - // that reqiure the user to be logged in. - fmt.Printf("Running user routes\n") + // Ensure admin credentials are valid and that the admin has + // paid their user registration fee. + fmt.Printf("Validating admin credentials\n") + admin.email = cmd.Args.AdminEmail + admin.password = cmd.Args.AdminPassword + err = login(admin.email, admin.password) + if err != nil { + return err + } - // Pay user registration fee - if paywallEnabled { - // New proposal failure - registration fee not paid - fmt.Printf(" New proposal failure: registration fee not paid\n") - npc := NewProposalCmd{ - Random: true, + vupr, err := client.VerifyUserPayment() + if err != nil { + return err } - err = npc.Execute(nil) - if err == nil { - return fmt.Errorf("submited proposal without " + - "paying registration fee") + if !vupr.HasPaid { + return fmt.Errorf("admin has not paid registration fee") } - // Pay user registration fee - fmt.Printf(" Paying user registration fee\n") - txID, err := util.PayWithTestnetFaucet(cfg.FaucetHost, - lr.PaywallAddress, lr.PaywallAmount, "") + lc := shared.LogoutCmd{} + err = lc.Execute(nil) if err != nil { return err } - dcr := float64(lr.PaywallAmount) / 1e8 - fmt.Printf(" Paid %v DCR to %v with txID %v\n", - dcr, lr.PaywallAddress, txID) - } - - // Wait for user registration payment confirmations - // If the paywall has been disable this will be marked - // as true. If the paywall has been enabled this will - // be true once the payment tx has the required number - // of confirmations. - for !vupr.HasPaid { - vupr, err = client.VerifyUserPayment() + // Create user and verify email + b, err := util.Random(int(policy.MinPasswordLength)) if err != nil { return err } - fmt.Printf(" Verify user payment: waiting for tx confirmations...\n") - time.Sleep(sleepInterval) - } + email := hex.EncodeToString(b) + "@example.com" + username := hex.EncodeToString(b) + password := hex.EncodeToString(b) - // Purchase proposal credits - fmt.Printf(" Proposal paywall details\n") - ppdr, err := client.ProposalPaywallDetails() - if err != nil { - return err - } + fmt.Printf("Creating user: %v\n", email) - if paywallEnabled { - // New proposal failure - no proposal credits - fmt.Printf(" New proposal failure: no proposal credits\n") - npc := NewProposalCmd{ - Random: true, + nuc := NewUserCmd{ + Verify: true, } - err = npc.Execute(nil) - if err == nil { - return fmt.Errorf("submited proposal without " + - "purchasing any proposal credits") + nuc.Args.Email = email + nuc.Args.Username = username + nuc.Args.Password = password + err = nuc.Execute(nil) + if err != nil { + return err } - // Purchase proposal credits - fmt.Printf(" Purchasing %v proposal credits\n", numCredits) - - atoms := ppdr.CreditPrice * uint64(numCredits) - txID, err := util.PayWithTestnetFaucet(cfg.FaucetHost, - ppdr.PaywallAddress, atoms, "") + // Login and store user details + fmt.Printf("Login user\n") + lr, err := client.Login( + &v1.Login{ + Email: email, + Password: shared.DigestSHA3(password), + }) if err != nil { return err } - fmt.Printf(" Paid %v DCR to %v with txID %v\n", - float64(atoms)/1e8, lr.PaywallAddress, txID) - } + user = testUser{ + ID: lr.UserID, + email: email, + username: username, + password: password, + publicKey: lr.PublicKey, + } - // Keep track of when the pending proposal credit payment - // receives the required number of confirmations. - for { - pppr, err := client.ProposalPaywallPayment() - if err != nil { - return err + // Check if paywall is enabled. Paywall address and paywall + // amount will be zero values if paywall has been disabled. + if lr.PaywallAddress != "" && lr.PaywallAmount != 0 { + paywallEnabled = true + } else { + fmt.Printf("WARNING: politeiawww paywall is disabled\n") } - // TxID will be blank if the paywall has been disabled - // or if the payment is no longer pending. - if pppr.TxID == "" { - // Verify that the correct number of proposal credits - // have been added to the user's account. - upcr, err := client.UserProposalCredits() + // Run user routes. These are the routes + // that reqiure the user to be logged in. + fmt.Printf("Running user routes\n") + + // Pay user registration fee + if paywallEnabled { + // New proposal failure - registration fee not paid + fmt.Printf(" New proposal failure: registration fee not paid\n") + npc := NewProposalCmd{ + Random: true, + } + err = npc.Execute(nil) + if err == nil { + return fmt.Errorf("submited proposal without " + + "paying registration fee") + } + + // Pay user registration fee + fmt.Printf(" Paying user registration fee\n") + txID, err := util.PayWithTestnetFaucet(cfg.FaucetHost, + lr.PaywallAddress, lr.PaywallAmount, "") if err != nil { return err } - if !paywallEnabled || len(upcr.UnspentCredits) == numCredits { - break - } + dcr := float64(lr.PaywallAmount) / 1e8 + fmt.Printf(" Paid %v DCR to %v with txID %v\n", + dcr, lr.PaywallAddress, txID) } - fmt.Printf(" Proposal paywall payment: waiting for tx confirmations...\n") - time.Sleep(sleepInterval) - } + // Wait for user registration payment confirmations + // If the paywall has been disable this will be marked + // as true. If the paywall has been enabled this will + // be true once the payment tx has the required number + // of confirmations. + for !vupr.HasPaid { + vupr, err = client.VerifyUserPayment() + if err != nil { + return err + } - // Me - fmt.Printf(" Me\n") - _, err = client.Me() - if err != nil { - return err - } + fmt.Printf(" Verify user payment: waiting for tx confirmations...\n") + time.Sleep(sleepInterval) + } - // Change password - fmt.Printf(" Change password\n") - b, err = util.Random(int(policy.MinPasswordLength)) - if err != nil { - return err - } - cpc := shared.ChangePasswordCmd{} - cpc.Args.Password = user.password - cpc.Args.NewPassword = hex.EncodeToString(b) - err = cpc.Execute(nil) - if err != nil { - return err - } - user.password = cpc.Args.NewPassword - - // Change username - fmt.Printf(" Change username\n") - cuc := shared.ChangeUsernameCmd{} - cuc.Args.Password = user.password - cuc.Args.NewUsername = hex.EncodeToString(b) - err = cuc.Execute(nil) - if err != nil { - return err - } - user.username = cuc.Args.NewUsername - - // Edit user - fmt.Printf(" Edit user\n") - var n uint64 = 1 << 0 - _, err = client.EditUser( - &v1.EditUser{ - EmailNotifications: &n, - }) - if err != nil { - return err - } + // Purchase proposal credits + fmt.Printf(" Proposal paywall details\n") + ppdr, err := client.ProposalPaywallDetails() + if err != nil { + return err + } - // Update user key - fmt.Printf(" Update user key\n") - var uukc shared.UpdateUserKeyCmd - err = uukc.Execute(nil) - if err != nil { - return err - } + if paywallEnabled { + // New proposal failure - no proposal credits + fmt.Printf(" New proposal failure: no proposal credits\n") + npc := NewProposalCmd{ + Random: true, + } + err = npc.Execute(nil) + if err == nil { + return fmt.Errorf("submited proposal without " + + "purchasing any proposal credits") + } - // Submit new proposal - fmt.Printf(" New proposal\n") - np, err := newNormalProposal() - if err != nil { - return err - } - npr, err := client.NewProposal(np) - if err != nil { - return err - } + // Purchase proposal credits + fmt.Printf(" Purchasing %v proposal credits\n", numCredits) - // Verify proposal censorship record - pr := v1.ProposalRecord{ - Files: np.Files, - Metadata: np.Metadata, - PublicKey: np.PublicKey, - Signature: np.Signature, - CensorshipRecord: npr.CensorshipRecord, - } - err = shared.VerifyProposal(pr, version.PubKey) - if err != nil { - return fmt.Errorf("verify proposal failed: %v", err) - } + atoms := ppdr.CreditPrice * uint64(numCredits) + txID, err := util.PayWithTestnetFaucet(cfg.FaucetHost, + ppdr.PaywallAddress, atoms, "") + if err != nil { + return err + } - // This is the proposal that we'll use for most of the tests - token := pr.CensorshipRecord.Token - fmt.Printf(" Proposal submitted: %v\n", token) + fmt.Printf(" Paid %v DCR to %v with txID %v\n", + float64(atoms)/1e8, lr.PaywallAddress, txID) + } - // Edit unvetted proposal - fmt.Printf(" Edit unvetted proposal\n") - epc := EditProposalCmd{ - Random: true, - } - epc.Args.Token = token - err = epc.Execute(nil) - if err != nil { - return err - } + // Keep track of when the pending proposal credit payment + // receives the required number of confirmations. + for { + pppr, err := client.ProposalPaywallPayment() + if err != nil { + return err + } - // Login with admin and make the proposal public - fmt.Printf(" Login admin\n") - err = login(admin.email, admin.password) - if err != nil { - return err - } + // TxID will be blank if the paywall has been disabled + // or if the payment is no longer pending. + if pppr.TxID == "" { + // Verify that the correct number of proposal credits + // have been added to the user's account. + upcr, err := client.UserProposalCredits() + if err != nil { + return err + } - fmt.Printf(" Set proposal status: public\n") - spsc := SetProposalStatusCmd{} - spsc.Args.Token = token - spsc.Args.Status = strconv.Itoa(int(v1.PropStatusPublic)) - err = spsc.Execute(nil) - if err != nil { - return err - } + if !paywallEnabled || len(upcr.UnspentCredits) == numCredits { + break + } + } - // Log back in with user - fmt.Printf(" Login user\n") - err = login(user.email, user.password) - if err != nil { - return err - } + fmt.Printf(" Proposal paywall payment: waiting for tx confirmations...\n") + time.Sleep(sleepInterval) + } - // Edit vetted proposal - fmt.Printf(" Edit vetted proposal\n") - err = epc.Execute(nil) - if err != nil { - return err - } + // Me + fmt.Printf(" Me\n") + _, err = client.Me() + if err != nil { + return err + } - // New comment - parent - fmt.Printf(" New comment: parent\n") - ncc := shared.NewCommentCmd{} - ncc.Args.Token = token - ncc.Args.Comment = "this is a comment" - ncc.Args.ParentID = "0" - err = ncc.Execute(nil) - if err != nil { - return err - } + // Change password + fmt.Printf(" Change password\n") + b, err = util.Random(int(policy.MinPasswordLength)) + if err != nil { + return err + } + cpc := shared.ChangePasswordCmd{} + cpc.Args.Password = user.password + cpc.Args.NewPassword = hex.EncodeToString(b) + err = cpc.Execute(nil) + if err != nil { + return err + } + user.password = cpc.Args.NewPassword + + // Change username + fmt.Printf(" Change username\n") + cuc := shared.ChangeUsernameCmd{} + cuc.Args.Password = user.password + cuc.Args.NewUsername = hex.EncodeToString(b) + err = cuc.Execute(nil) + if err != nil { + return err + } + user.username = cuc.Args.NewUsername + + // Edit user + fmt.Printf(" Edit user\n") + var n uint64 = 1 << 0 + _, err = client.EditUser( + &v1.EditUser{ + EmailNotifications: &n, + }) + if err != nil { + return err + } - // New comment - reply - fmt.Printf(" New comment: reply\n") - ncc.Args.Token = token - ncc.Args.Comment = "this is a comment reply" - ncc.Args.ParentID = "1" - err = ncc.Execute(nil) - if err != nil { - return err - } + // Update user key + fmt.Printf(" Update user key\n") + var uukc shared.UpdateUserKeyCmd + err = uukc.Execute(nil) + if err != nil { + return err + } - // Validate comments - fmt.Printf(" Proposal details\n") - pdr, err := client.ProposalDetails(token, nil) - if err != nil { - return err - } + // Submit new proposal + fmt.Printf(" New proposal\n") + np, err := newNormalProposal() + if err != nil { + return err + } + npr, err := client.NewProposal(np) + if err != nil { + return err + } - err = shared.VerifyProposal(pdr.Proposal, version.PubKey) - if err != nil { - return fmt.Errorf("verify proposal failed: %v", err) - } + // Verify proposal censorship record + pr := v1.ProposalRecord{ + Files: np.Files, + Metadata: np.Metadata, + PublicKey: np.PublicKey, + Signature: np.Signature, + CensorshipRecord: npr.CensorshipRecord, + } + err = shared.VerifyProposal(pr, version.PubKey) + if err != nil { + return fmt.Errorf("verify proposal failed: %v", err) + } - if pdr.Proposal.NumComments != 2 { - return fmt.Errorf("proposal num comments got %v, want 2", - pdr.Proposal.NumComments) - } + // This is the proposal that we'll use for most of the tests + token := pr.CensorshipRecord.Token + fmt.Printf(" Proposal submitted: %v\n", token) - fmt.Printf(" Proposal comments\n") - gcr, err := client.GetComments(token) - if err != nil { - return fmt.Errorf("GetComments: %v", err) - } + // Edit unvetted proposal + fmt.Printf(" Edit unvetted proposal\n") + epc := EditProposalCmd{ + Random: true, + } + epc.Args.Token = token + err = epc.Execute(nil) + if err != nil { + return err + } - if len(gcr.Comments) != 2 { - return fmt.Errorf("num comments got %v, want 2", - len(gcr.Comments)) - } + // Login with admin and make the proposal public + fmt.Printf(" Login admin\n") + err = login(admin.email, admin.password) + if err != nil { + return err + } - for _, v := range gcr.Comments { - // We check the userID because userIDs are not part of - // the politeiad comment record. UserIDs are stored in - // in politeiawww and are added to the comments at the - // time of the request. This introduces the potential - // for errors. - if v.UserID != user.ID { - return fmt.Errorf("comment userID got %v, want %v", - v.UserID, user.ID) + fmt.Printf(" Set proposal status: public\n") + spsc := SetProposalStatusCmd{} + spsc.Args.Token = token + spsc.Args.Status = strconv.Itoa(int(v1.PropStatusPublic)) + err = spsc.Execute(nil) + if err != nil { + return err } - } - // Like comment sequence - lcc := LikeCommentCmd{} - lcc.Args.Token = pr.CensorshipRecord.Token - lcc.Args.CommentID = "1" - lcc.Args.Action = commentActionUpvote + // Log back in with user + fmt.Printf(" Login user\n") + err = login(user.email, user.password) + if err != nil { + return err + } - fmt.Printf(" Like comment: upvote\n") - err = lcc.Execute(nil) - if err != nil { - return err - } + // Edit vetted proposal + fmt.Printf(" Edit vetted proposal\n") + err = epc.Execute(nil) + if err != nil { + return err + } - fmt.Printf(" Like comment: upvote\n") - err = lcc.Execute(nil) - if err != nil { - return err - } + // New comment - parent + fmt.Printf(" New comment: parent\n") + ncc := shared.NewCommentCmd{} + ncc.Args.Token = token + ncc.Args.Comment = "this is a comment" + ncc.Args.ParentID = "0" + err = ncc.Execute(nil) + if err != nil { + return err + } - fmt.Printf(" Like comment: upvote\n") - err = lcc.Execute(nil) - if err != nil { - return err - } + // New comment - reply + fmt.Printf(" New comment: reply\n") + ncc.Args.Token = token + ncc.Args.Comment = "this is a comment reply" + ncc.Args.ParentID = "1" + err = ncc.Execute(nil) + if err != nil { + return err + } - fmt.Printf(" Like comment: downvote\n") - lcc.Args.Action = commentActionDownvote - err = lcc.Execute(nil) - if err != nil { - return err - } + // Validate comments + fmt.Printf(" Proposal details\n") + pdr, err := client.ProposalDetails(token, nil) + if err != nil { + return err + } - // Validate like comments - fmt.Printf(" Proposal comments\n") - gcr, err = client.GetComments(token) - if err != nil { - return err - } + err = shared.VerifyProposal(pdr.Proposal, version.PubKey) + if err != nil { + return fmt.Errorf("verify proposal failed: %v", err) + } - for _, v := range gcr.Comments { - if v.CommentID == "1" { - switch { - case v.Upvotes != 0: - return fmt.Errorf("comment result up votes got %v, want 0", - v.Upvotes) - case v.Downvotes != 1: - return fmt.Errorf("comment result down votes got %v, want 1", - v.Downvotes) - case v.ResultVotes != -1: - return fmt.Errorf("comment result vote score got %v, want -1", - v.ResultVotes) - } + if pdr.Proposal.NumComments != 2 { + return fmt.Errorf("proposal num comments got %v, want 2", + pdr.Proposal.NumComments) } - } - fmt.Printf(" User like comments\n") - crv, err := client.UserCommentsLikes(token) - if err != nil { - return err - } + fmt.Printf(" Proposal comments\n") + gcr, err := client.GetComments(token) + if err != nil { + return fmt.Errorf("GetComments: %v", err) + } - switch { - case len(crv.CommentsLikes) != 1: - return fmt.Errorf("user like comments got %v, want 1", - len(crv.CommentsLikes)) + if len(gcr.Comments) != 2 { + return fmt.Errorf("num comments got %v, want 2", + len(gcr.Comments)) + } - case crv.CommentsLikes[0].Action != v1.VoteActionDown: - return fmt.Errorf("user like comment action got %v, want %v", - crv.CommentsLikes[0].Action, v1.VoteActionDown) - } + for _, v := range gcr.Comments { + // We check the userID because userIDs are not part of + // the politeiad comment record. UserIDs are stored in + // in politeiawww and are added to the comments at the + // time of the request. This introduces the potential + // for errors. + if v.UserID != user.ID { + return fmt.Errorf("comment userID got %v, want %v", + v.UserID, user.ID) + } + } - // Authorize vote then revoke - fmt.Printf(" Authorize vote: authorize\n") - avc := AuthorizeVoteCmd{} - avc.Args.Token = token - avc.Args.Action = decredplugin.AuthVoteActionAuthorize - err = avc.Execute(nil) - if err != nil { - return err - } + // Like comment sequence + lcc := LikeCommentCmd{} + lcc.Args.Token = pr.CensorshipRecord.Token + lcc.Args.CommentID = "1" + lcc.Args.Action = commentActionUpvote - fmt.Printf(" Authorize vote: revoke\n") - avc.Args.Action = decredplugin.AuthVoteActionRevoke - err = avc.Execute(nil) - if err != nil { - return err - } + fmt.Printf(" Like comment: upvote\n") + err = lcc.Execute(nil) + if err != nil { + return err + } - // Validate vote status - fmt.Printf(" Vote status\n") - vsr, err := client.VoteStatus(token) - if err != nil { - return err - } + fmt.Printf(" Like comment: upvote\n") + err = lcc.Execute(nil) + if err != nil { + return err + } - if vsr.Status != v1.PropVoteStatusNotAuthorized { - return fmt.Errorf("vote status got %v, want %v", - vsr.Status, v1.PropVoteStatusNotAuthorized) - } + fmt.Printf(" Like comment: upvote\n") + err = lcc.Execute(nil) + if err != nil { + return err + } - // Authorize vote - fmt.Printf(" Authorize vote: authorize\n") - avc.Args.Action = decredplugin.AuthVoteActionAuthorize - err = avc.Execute(nil) - if err != nil { - return err - } + fmt.Printf(" Like comment: downvote\n") + lcc.Args.Action = commentActionDownvote + err = lcc.Execute(nil) + if err != nil { + return err + } - // Validate vote status - fmt.Printf(" Vote status\n") - vsr, err = client.VoteStatus(token) - if err != nil { - return err - } + // Validate like comments + fmt.Printf(" Proposal comments\n") + gcr, err = client.GetComments(token) + if err != nil { + return err + } - if vsr.Status != v1.PropVoteStatusAuthorized { - return fmt.Errorf("vote status got %v, want %v", - vsr.Status, v1.PropVoteStatusNotAuthorized) - } + for _, v := range gcr.Comments { + if v.CommentID == "1" { + switch { + case v.Upvotes != 0: + return fmt.Errorf("comment result up votes got %v, want 0", + v.Upvotes) + case v.Downvotes != 1: + return fmt.Errorf("comment result down votes got %v, want 1", + v.Downvotes) + case v.ResultVotes != -1: + return fmt.Errorf("comment result vote score got %v, want -1", + v.ResultVotes) + } + } + } - // Logout - fmt.Printf(" Logout\n") - lc = shared.LogoutCmd{} - err = lc.Execute(nil) - if err != nil { - return err - } + fmt.Printf(" User like comments\n") + crv, err := client.UserCommentsLikes(token) + if err != nil { + return err + } - // Admin routes are routes that only - // admins can access. - fmt.Printf("Running admin routes\n") + switch { + case len(crv.CommentsLikes) != 1: + return fmt.Errorf("user like comments got %v, want 1", + len(crv.CommentsLikes)) - // Login - fmt.Printf(" Login admin\n") - err = login(admin.email, admin.password) - if err != nil { - return err - } + case crv.CommentsLikes[0].Action != v1.VoteActionDown: + return fmt.Errorf("user like comment action got %v, want %v", + crv.CommentsLikes[0].Action, v1.VoteActionDown) + } - // Start vote - fmt.Printf(" Start vote\n") - svc := StartVoteCmd{} - svc.Args.Token = token - err = svc.Execute(nil) - if err != nil { - return err - } + // Authorize vote then revoke + fmt.Printf(" Authorize vote: authorize\n") + avc := AuthorizeVoteCmd{} + avc.Args.Token = token + avc.Args.Action = decredplugin.AuthVoteActionAuthorize + err = avc.Execute(nil) + if err != nil { + return err + } - // Censor comment - fmt.Printf(" Censor comment\n") - ccc := shared.CensorCommentCmd{} - ccc.Args.Token = token - ccc.Args.CommentID = "2" - ccc.Args.Reason = "comment is spam" - err = ccc.Execute(nil) - if err != nil { - return err - } + fmt.Printf(" Authorize vote: revoke\n") + avc.Args.Action = decredplugin.AuthVoteActionRevoke + err = avc.Execute(nil) + if err != nil { + return err + } - // Validate censored comment - fmt.Printf(" Get comments\n") - gcr, err = client.GetComments(token) - if err != nil { - return err - } + // Validate vote status + fmt.Printf(" Vote status\n") + vsr, err := client.VoteStatus(token) + if err != nil { + return err + } - if len(gcr.Comments) != 2 { - return fmt.Errorf("num comments got %v, want 2", - len(gcr.Comments)) - } + if vsr.Status != v1.PropVoteStatusNotAuthorized { + return fmt.Errorf("vote status got %v, want %v", + vsr.Status, v1.PropVoteStatusNotAuthorized) + } - c := gcr.Comments[1] - switch { - case c.CommentID != "2": - return fmt.Errorf("commentID got %v, want 2", - c.CommentID) + // Authorize vote + fmt.Printf(" Authorize vote: authorize\n") + avc.Args.Action = decredplugin.AuthVoteActionAuthorize + err = avc.Execute(nil) + if err != nil { + return err + } - case c.Comment != "": - return fmt.Errorf("censored comment text got %v, want empty string", - c.Comment) + // Validate vote status + fmt.Printf(" Vote status\n") + vsr, err = client.VoteStatus(token) + if err != nil { + return err + } - case !c.Censored: - return fmt.Errorf("censored comment not marked as censored") - } + if vsr.Status != v1.PropVoteStatusAuthorized { + return fmt.Errorf("vote status got %v, want %v", + vsr.Status, v1.PropVoteStatusNotAuthorized) + } - // Login with user in order to submit proposals that we can - // use to test the set proposal status route. - fmt.Printf(" Login user\n") - err = login(user.email, user.password) - if err != nil { - return err - } + // Logout + fmt.Printf(" Logout\n") + lc = shared.LogoutCmd{} + err = lc.Execute(nil) + if err != nil { + return err + } - // Submit proposals that can be used to test the set proposal - // status command. - fmt.Printf(" Creating proposals for set proposal status test\n") - var ( - // Censorship tokens - notReviewed1 string - notReviewed2 string - unreviewedChanges1 string - unreviewedChanges2 string - - // We don't need these now but will need - // them when we test the public routes. - censoredPropToken string - unreviewedPropToken string - ) - - np, err = newNormalProposal() - if err != nil { - return err - } - npr, err = client.NewProposal(np) - if err != nil { - return err - } - notReviewed1 = npr.CensorshipRecord.Token + // Admin routes are routes that only + // admins can access. + fmt.Printf("Running admin routes\n") - np, err = newNormalProposal() - if err != nil { - return err - } - npr, err = client.NewProposal(np) - if err != nil { - return err - } - notReviewed2 = npr.CensorshipRecord.Token + // Login + fmt.Printf(" Login admin\n") + err = login(admin.email, admin.password) + if err != nil { + return err + } - np, err = newNormalProposal() - if err != nil { - return err - } - npr, err = client.NewProposal(np) - if err != nil { - return err - } - epc = EditProposalCmd{ - Random: true, - } - epc.Args.Token = npr.CensorshipRecord.Token - err = epc.Execute(nil) - if err != nil { - return err - } - unreviewedChanges1 = npr.CensorshipRecord.Token + // Start vote + fmt.Printf(" Start vote\n") + svc := StartVoteCmd{} + svc.Args.Token = token + err = svc.Execute(nil) + if err != nil { + return err + } - np, err = newNormalProposal() - if err != nil { - return err - } - npr, err = client.NewProposal(np) - if err != nil { - return err - } - epc = EditProposalCmd{ - Random: true, - } - epc.Args.Token = npr.CensorshipRecord.Token - err = epc.Execute(nil) - if err != nil { - return err - } - unreviewedChanges2 = npr.CensorshipRecord.Token + // Censor comment + fmt.Printf(" Censor comment\n") + ccc := shared.CensorCommentCmd{} + ccc.Args.Token = token + ccc.Args.CommentID = "2" + ccc.Args.Reason = "comment is spam" + err = ccc.Execute(nil) + if err != nil { + return err + } - np, err = newNormalProposal() - if err != nil { - return err - } - npr, err = client.NewProposal(np) - if err != nil { - return err - } - // We don't use this proposal for testing the set - // proposal status routes, but will need it when - // we are testing the public routes. - unreviewedPropToken = npr.CensorshipRecord.Token - - // Log back in with admin - fmt.Printf(" Login admin\n") - err = login(admin.email, admin.password) - if err != nil { - return err - } + // Validate censored comment + fmt.Printf(" Get comments\n") + gcr, err = client.GetComments(token) + if err != nil { + return err + } - // Validate the proposal statuses before we attempt to - // change them. - pdr, err = client.ProposalDetails(notReviewed1, nil) - if err != nil { - return err - } + if len(gcr.Comments) != 2 { + return fmt.Errorf("num comments got %v, want 2", + len(gcr.Comments)) + } - if pdr.Proposal.Status != v1.PropStatusNotReviewed { - return fmt.Errorf("Proposal status got %v, want %v", - pdr.Proposal.Status, v1.PropStatusNotReviewed) - } + c := gcr.Comments[1] + switch { + case c.CommentID != "2": + return fmt.Errorf("commentID got %v, want 2", + c.CommentID) - pdr, err = client.ProposalDetails(unreviewedChanges1, nil) - if err != nil { - return err - } + case c.Comment != "": + return fmt.Errorf("censored comment text got %v, want empty string", + c.Comment) - if pdr.Proposal.Status != v1.PropStatusUnreviewedChanges { - return fmt.Errorf("Proposal status got %v, want %v", - pdr.Proposal.Status, v1.PropStatusUnreviewedChanges) - } + case !c.Censored: + return fmt.Errorf("censored comment not marked as censored") + } - // Set proposal status - not reviewed to censored - fmt.Printf(" Set proposal status: not reviewed to censored\n") - spsc = SetProposalStatusCmd{} - spsc.Args.Token = notReviewed1 - spsc.Args.Status = "censored" - spsc.Args.Message = "proposal is spam" - err = spsc.Execute(nil) - if err != nil { - return err - } + // Login with user in order to submit proposals that we can + // use to test the set proposal status route. + fmt.Printf(" Login user\n") + err = login(user.email, user.password) + if err != nil { + return err + } - pdr, err = client.ProposalDetails(spsc.Args.Token, nil) - if err != nil { - return err - } + // Submit proposals that can be used to test the set proposal + // status command. + fmt.Printf(" Creating proposals for set proposal status test\n") + var ( + // Censorship tokens + notReviewed1 string + notReviewed2 string + unreviewedChanges1 string + unreviewedChanges2 string + + // We don't need these now but will need + // them when we test the public routes. + censoredPropToken string + unreviewedPropToken string + ) - if pdr.Proposal.Status != v1.PropStatusCensored { - return fmt.Errorf("Proposal status got %v, want %v", - pdr.Proposal.Status, v1.PropStatusCensored) - } + np, err = newNormalProposal() + if err != nil { + return err + } + npr, err = client.NewProposal(np) + if err != nil { + return err + } + notReviewed1 = npr.CensorshipRecord.Token - // Save this token. We will need it when - // we test the public routes. - censoredPropToken = spsc.Args.Token + np, err = newNormalProposal() + if err != nil { + return err + } + npr, err = client.NewProposal(np) + if err != nil { + return err + } + notReviewed2 = npr.CensorshipRecord.Token - // Set proposal status - not reviewed to public - fmt.Printf(" Set proposal status: not reviewed to public\n") - spsc = SetProposalStatusCmd{} - spsc.Args.Token = notReviewed2 - spsc.Args.Status = strconv.Itoa(int(v1.PropStatusPublic)) - err = spsc.Execute(nil) - if err != nil { - return err - } + np, err = newNormalProposal() + if err != nil { + return err + } + npr, err = client.NewProposal(np) + if err != nil { + return err + } + epc = EditProposalCmd{ + Random: true, + } + epc.Args.Token = npr.CensorshipRecord.Token + err = epc.Execute(nil) + if err != nil { + return err + } + unreviewedChanges1 = npr.CensorshipRecord.Token - pdr, err = client.ProposalDetails(spsc.Args.Token, nil) - if err != nil { - return err - } + np, err = newNormalProposal() + if err != nil { + return err + } + npr, err = client.NewProposal(np) + if err != nil { + return err + } + epc = EditProposalCmd{ + Random: true, + } + epc.Args.Token = npr.CensorshipRecord.Token + err = epc.Execute(nil) + if err != nil { + return err + } + unreviewedChanges2 = npr.CensorshipRecord.Token - if pdr.Proposal.Status != v1.PropStatusPublic { - return fmt.Errorf("Proposal status got %v, want %v", - pdr.Proposal.Status, v1.PropStatusPublic) - } + np, err = newNormalProposal() + if err != nil { + return err + } + npr, err = client.NewProposal(np) + if err != nil { + return err + } + // We don't use this proposal for testing the set + // proposal status routes, but will need it when + // we are testing the public routes. + unreviewedPropToken = npr.CensorshipRecord.Token + + // Log back in with admin + fmt.Printf(" Login admin\n") + err = login(admin.email, admin.password) + if err != nil { + return err + } - // Set proposal status - unreviewed changes to censored - fmt.Printf(" Set proposal status: unreviewed changes to censored\n") - spsc = SetProposalStatusCmd{} - spsc.Args.Token = unreviewedChanges1 - spsc.Args.Status = "censored" - spsc.Args.Message = "this is spam" - err = spsc.Execute(nil) - if err != nil { - return err - } + // Validate the proposal statuses before we attempt to + // change them. + pdr, err = client.ProposalDetails(notReviewed1, nil) + if err != nil { + return err + } - pdr, err = client.ProposalDetails(spsc.Args.Token, nil) - if err != nil { - return err - } + if pdr.Proposal.Status != v1.PropStatusNotReviewed { + return fmt.Errorf("Proposal status got %v, want %v", + pdr.Proposal.Status, v1.PropStatusNotReviewed) + } - if pdr.Proposal.Status != v1.PropStatusCensored { - return fmt.Errorf("Proposal status got %v, want %v", - pdr.Proposal.Status, v1.PropStatusCensored) - } + pdr, err = client.ProposalDetails(unreviewedChanges1, nil) + if err != nil { + return err + } - // Set proposal status - unreviewed changes to public - fmt.Printf(" Set proposal status: unreviewed changes to public\n") - spsc = SetProposalStatusCmd{} - spsc.Args.Token = unreviewedChanges2 - spsc.Args.Status = strconv.Itoa(int(v1.PropStatusPublic)) - err = spsc.Execute(nil) - if err != nil { - return err - } + if pdr.Proposal.Status != v1.PropStatusUnreviewedChanges { + return fmt.Errorf("Proposal status got %v, want %v", + pdr.Proposal.Status, v1.PropStatusUnreviewedChanges) + } - pdr, err = client.ProposalDetails(spsc.Args.Token, nil) - if err != nil { - return err - } + // Set proposal status - not reviewed to censored + fmt.Printf(" Set proposal status: not reviewed to censored\n") + spsc = SetProposalStatusCmd{} + spsc.Args.Token = notReviewed1 + spsc.Args.Status = "censored" + spsc.Args.Message = "proposal is spam" + err = spsc.Execute(nil) + if err != nil { + return err + } - if pdr.Proposal.Status != v1.PropStatusPublic { - return fmt.Errorf("Proposal status got %v, want %v", - pdr.Proposal.Status, v1.PropStatusPublic) - } + pdr, err = client.ProposalDetails(spsc.Args.Token, nil) + if err != nil { + return err + } - // Set proposal status - public to abandoned - fmt.Printf(" Set proposal status: public to abandoned\n") - spsc.Args.Status = "abandoned" - spsc.Args.Message = "no activity for two weeks" - err = spsc.Execute(nil) - if err != nil { - return err - } + if pdr.Proposal.Status != v1.PropStatusCensored { + return fmt.Errorf("Proposal status got %v, want %v", + pdr.Proposal.Status, v1.PropStatusCensored) + } - pdr, err = client.ProposalDetails(spsc.Args.Token, nil) - if err != nil { - return err - } + // Save this token. We will need it when + // we test the public routes. + censoredPropToken = spsc.Args.Token - if pdr.Proposal.Status != v1.PropStatusAbandoned { - return fmt.Errorf("Proposal status got %v, want %v", - pdr.Proposal.Status, v1.PropStatusAbandoned) - } + // Set proposal status - not reviewed to public + fmt.Printf(" Set proposal status: not reviewed to public\n") + spsc = SetProposalStatusCmd{} + spsc.Args.Token = notReviewed2 + spsc.Args.Status = strconv.Itoa(int(v1.PropStatusPublic)) + err = spsc.Execute(nil) + if err != nil { + return err + } - // Users - filter by email - fmt.Printf(" Users: filter by email\n") - ur, err := client.Users( - &v1.Users{ - Email: user.email, - }) - if err != nil { - return err - } + pdr, err = client.ProposalDetails(spsc.Args.Token, nil) + if err != nil { + return err + } - switch { - case ur.TotalMatches != 1: - return fmt.Errorf("total matches got %v, want 1", - ur.TotalMatches) + if pdr.Proposal.Status != v1.PropStatusPublic { + return fmt.Errorf("Proposal status got %v, want %v", + pdr.Proposal.Status, v1.PropStatusPublic) + } - case ur.Users[0].ID != user.ID: - return fmt.Errorf("user ID got %v, want %v", - ur.Users[0].ID, user.ID) - } + // Set proposal status - unreviewed changes to censored + fmt.Printf(" Set proposal status: unreviewed changes to censored\n") + spsc = SetProposalStatusCmd{} + spsc.Args.Token = unreviewedChanges1 + spsc.Args.Status = "censored" + spsc.Args.Message = "this is spam" + err = spsc.Execute(nil) + if err != nil { + return err + } - // Users - filter by username - fmt.Printf(" Users: filter by username\n") - ur, err = client.Users( - &v1.Users{ - Username: user.username, - }) - if err != nil { - return err - } + pdr, err = client.ProposalDetails(spsc.Args.Token, nil) + if err != nil { + return err + } - switch { - case ur.TotalMatches != 1: - return fmt.Errorf("total matches got %v, want 1", - ur.TotalMatches) + if pdr.Proposal.Status != v1.PropStatusCensored { + return fmt.Errorf("Proposal status got %v, want %v", + pdr.Proposal.Status, v1.PropStatusCensored) + } - case ur.Users[0].ID != user.ID: - return fmt.Errorf("user ID got %v, want %v", - ur.Users[0].ID, user.ID) - } + // Set proposal status - unreviewed changes to public + fmt.Printf(" Set proposal status: unreviewed changes to public\n") + spsc = SetProposalStatusCmd{} + spsc.Args.Token = unreviewedChanges2 + spsc.Args.Status = strconv.Itoa(int(v1.PropStatusPublic)) + err = spsc.Execute(nil) + if err != nil { + return err + } - // Rescan user payments - fmt.Printf(" Rescan user payments\n") - rupc := RescanUserPaymentsCmd{} - rupc.Args.UserID = user.ID - err = rupc.Execute(nil) - if err != nil { - return err - } + pdr, err = client.ProposalDetails(spsc.Args.Token, nil) + if err != nil { + return err + } - // Logout - fmt.Printf(" Logout\n") - lc = shared.LogoutCmd{} - err = lc.Execute(nil) - if err != nil { - return err - } + if pdr.Proposal.Status != v1.PropStatusPublic { + return fmt.Errorf("Proposal status got %v, want %v", + pdr.Proposal.Status, v1.PropStatusPublic) + } - // Public routes - fmt.Printf("Running public routes\n") + // Set proposal status - public to abandoned + fmt.Printf(" Set proposal status: public to abandoned\n") + spsc.Args.Status = "abandoned" + spsc.Args.Message = "no activity for two weeks" + err = spsc.Execute(nil) + if err != nil { + return err + } - // Me failure - _, err = client.Me() - if err == nil { - return fmt.Errorf("admin should be logged out") - } + pdr, err = client.ProposalDetails(spsc.Args.Token, nil) + if err != nil { + return err + } - // Proposal details - fmt.Printf(" Proposal details\n") - pdr, err = client.ProposalDetails(token, nil) - if err != nil { - return err - } + if pdr.Proposal.Status != v1.PropStatusAbandoned { + return fmt.Errorf("Proposal status got %v, want %v", + pdr.Proposal.Status, v1.PropStatusAbandoned) + } - if pdr.Proposal.Version != "2" { - return fmt.Errorf("proposal details version got %v, want 2", - pdr.Proposal.Version) - } + // Users - filter by email + fmt.Printf(" Users: filter by email\n") + ur, err := client.Users( + &v1.Users{ + Email: user.email, + }) + if err != nil { + return err + } - // Proposal details version - fmt.Printf(" Proposal details version\n") - pdr, err = client.ProposalDetails(token, - &v1.ProposalsDetails{ - Version: "1", - }) - if err != nil { - return err - } + switch { + case ur.TotalMatches != 1: + return fmt.Errorf("total matches got %v, want 1", + ur.TotalMatches) - if pdr.Proposal.Version != "1" { - return fmt.Errorf("proposal details version got %v, want 1", - pdr.Proposal.Version) - } + case ur.Users[0].ID != user.ID: + return fmt.Errorf("user ID got %v, want %v", + ur.Users[0].ID, user.ID) + } - // Proposal details - unreviewed - fmt.Printf(" Proposal details: unreviewed proposal\n") - pdr, err = client.ProposalDetails(unreviewedPropToken, nil) - if err != nil { - return err - } + // Users - filter by username + fmt.Printf(" Users: filter by username\n") + ur, err = client.Users( + &v1.Users{ + Username: user.username, + }) + if err != nil { + return err + } - switch { - case pdr.Proposal.Name != "": - return fmt.Errorf("proposal name should be empty string") + switch { + case ur.TotalMatches != 1: + return fmt.Errorf("total matches got %v, want 1", + ur.TotalMatches) - case len(pdr.Proposal.Files) != 0: - return fmt.Errorf("proposal files should not be included") - } + case ur.Users[0].ID != user.ID: + return fmt.Errorf("user ID got %v, want %v", + ur.Users[0].ID, user.ID) + } - // Proposal details - censored - fmt.Printf(" Proposal details: censored proposal\n") - pdr, err = client.ProposalDetails(censoredPropToken, nil) - if err != nil { - return err - } + // Rescan user payments + fmt.Printf(" Rescan user payments\n") + rupc := RescanUserPaymentsCmd{} + rupc.Args.UserID = user.ID + err = rupc.Execute(nil) + if err != nil { + return err + } - switch { - case pdr.Proposal.Name != "": - return fmt.Errorf("proposal name should be empty string") + // Logout + fmt.Printf(" Logout\n") + lc = shared.LogoutCmd{} + err = lc.Execute(nil) + if err != nil { + return err + } - case len(pdr.Proposal.Files) != 0: - return fmt.Errorf("proposal files should not be included") + // Public routes + fmt.Printf("Running public routes\n") - case pdr.Proposal.CensoredAt == 0: - return fmt.Errorf("proposal should have a CensoredAt timestamp") - } + // Me failure + _, err = client.Me() + if err == nil { + return fmt.Errorf("admin should be logged out") + } - // Proposal comments - fmt.Printf(" Get comments\n") - gcr, err = client.GetComments(token) - if err != nil { - return err - } + // Proposal details + fmt.Printf(" Proposal details\n") + pdr, err = client.ProposalDetails(token, nil) + if err != nil { + return err + } - if len(gcr.Comments) != 2 { - return fmt.Errorf("num comments got %v, want 2", - len(gcr.Comments)) - } + if pdr.Proposal.Version != "2" { + return fmt.Errorf("proposal details version got %v, want 2", + pdr.Proposal.Version) + } - c0 := gcr.Comments[0] - c1 := gcr.Comments[1] - switch { - case c0.CommentID != "1": - return fmt.Errorf("comment ID got %v, want 1", - c0.CommentID) + // Proposal details version + fmt.Printf(" Proposal details version\n") + pdr, err = client.ProposalDetails(token, + &v1.ProposalsDetails{ + Version: "1", + }) + if err != nil { + return err + } - case c0.Upvotes != 0: - return fmt.Errorf("comment %v result up votes got %v, want 0", - c0.CommentID, c0.Upvotes) + if pdr.Proposal.Version != "1" { + return fmt.Errorf("proposal details version got %v, want 1", + pdr.Proposal.Version) + } - case c0.Downvotes != 1: - return fmt.Errorf("comment %v result down votes got %v, want 1", - c0.CommentID, c0.Downvotes) + // Proposal details - unreviewed + fmt.Printf(" Proposal details: unreviewed proposal\n") + pdr, err = client.ProposalDetails(unreviewedPropToken, nil) + if err != nil { + return err + } - case c0.ResultVotes != -1: - return fmt.Errorf("comment %v result vote score got %v, want -1", - c0.CommentID, c0.Downvotes) + switch { + case pdr.Proposal.Name != "": + return fmt.Errorf("proposal name should be empty string") - case c1.CommentID != "2": - return fmt.Errorf("comment ID got %v, want 2", - c1.CommentID) + case len(pdr.Proposal.Files) != 0: + return fmt.Errorf("proposal files should not be included") + } - case c1.Comment != "": - return fmt.Errorf("censored comment text got '%v', want ''", - c1.Comment) + // Proposal details - censored + fmt.Printf(" Proposal details: censored proposal\n") + pdr, err = client.ProposalDetails(censoredPropToken, nil) + if err != nil { + return err + } - case !c1.Censored: - return fmt.Errorf("censored comment not marked as censored") - } + switch { + case pdr.Proposal.Name != "": + return fmt.Errorf("proposal name should be empty string") - // Vetted proposals. We need to submit a page of proposals and - // make them public first in order to test the vetted route. - fmt.Printf(" Login admin\n") - err = login(admin.email, admin.password) - if err != nil { - return err - } + case len(pdr.Proposal.Files) != 0: + return fmt.Errorf("proposal files should not be included") - fmt.Printf(" Submitting a page of proposals to test vetted route\n") - for i := 0; i < v1.ProposalListPageSize; i++ { - np, err = newNormalProposal() - if err != nil { - return err + case pdr.Proposal.CensoredAt == 0: + return fmt.Errorf("proposal should have a CensoredAt timestamp") } - _, err = client.NewProposal(np) + + // Proposal comments + fmt.Printf(" Get comments\n") + gcr, err = client.GetComments(token) if err != nil { return err } - } - fmt.Printf(" Token inventory\n") - tir, err := client.TokenInventory() - if err != nil { - return err - } + if len(gcr.Comments) != 2 { + return fmt.Errorf("num comments got %v, want 2", + len(gcr.Comments)) + } - fmt.Printf(" Batch proposals\n") - bpr, err := client.BatchProposals(&v1.BatchProposals{ - Tokens: tir.Unreviewed[:v1.ProposalListPageSize], - }) - if err != nil { - return err - } + c0 := gcr.Comments[0] + c1 := gcr.Comments[1] + switch { + case c0.CommentID != "1": + return fmt.Errorf("comment ID got %v, want 1", + c0.CommentID) - fmt.Printf(" Making unvetted proposals public\n") - for _, v := range bpr.Proposals { - spsc := SetProposalStatusCmd{} - spsc.Args.Token = v.CensorshipRecord.Token - spsc.Args.Status = strconv.Itoa(int(v1.PropStatusPublic)) - err = spsc.Execute(nil) - if err != nil { - return err - } - } + case c0.Upvotes != 0: + return fmt.Errorf("comment %v result up votes got %v, want 0", + c0.CommentID, c0.Upvotes) - fmt.Printf(" Logout\n") - lc = shared.LogoutCmd{} - err = lc.Execute(nil) - if err != nil { - return err - } + case c0.Downvotes != 1: + return fmt.Errorf("comment %v result down votes got %v, want 1", + c0.CommentID, c0.Downvotes) - fmt.Printf(" Vetted\n") - gavr, err := client.GetAllVetted(&v1.GetAllVetted{}) - if err != nil { - return err - } + case c0.ResultVotes != -1: + return fmt.Errorf("comment %v result vote score got %v, want -1", + c0.CommentID, c0.Downvotes) - if len(gavr.Proposals) != v1.ProposalListPageSize { - return fmt.Errorf("proposals page size got %v, want %v", - len(gavr.Proposals), v1.ProposalListPageSize) - } + case c1.CommentID != "2": + return fmt.Errorf("comment ID got %v, want 2", + c1.CommentID) + + case c1.Comment != "": + return fmt.Errorf("censored comment text got '%v', want ''", + c1.Comment) - for _, v := range gavr.Proposals { - err = shared.VerifyProposal(v, version.PubKey) + case !c1.Censored: + return fmt.Errorf("censored comment not marked as censored") + } + + // Vetted proposals. We need to submit a page of proposals and + // make them public first in order to test the vetted route. + fmt.Printf(" Login admin\n") + err = login(admin.email, admin.password) if err != nil { - return fmt.Errorf("verify proposal failed %v: %v", - v.CensorshipRecord.Token, err) + return err } - } - // User details - fmt.Printf(" User details\n") - udr, err := client.UserDetails(user.ID) - if err != nil { - return err - } + fmt.Printf(" Submitting a page of proposals to test vetted route\n") + for i := 0; i < v1.ProposalListPageSize; i++ { + np, err = newNormalProposal() + if err != nil { + return err + } + _, err = client.NewProposal(np) + if err != nil { + return err + } + } - if udr.User.ID != user.ID { - return fmt.Errorf("user ID got %v, want %v", - udr.User.ID, user.ID) - } + fmt.Printf(" Token inventory\n") + tir, err := client.TokenInventory() + if err != nil { + return err + } - // User proposals - fmt.Printf(" User proposals\n") - upr, err := client.UserProposals( - &v1.UserProposals{ - UserId: user.ID, + fmt.Printf(" Batch proposals\n") + bpr, err := client.BatchProposals(&v1.BatchProposals{ + Tokens: tir.Unreviewed[:v1.ProposalListPageSize], }) - if err != nil { - return err - } - - // userPropCount is the total number of proposals that we've - // submitted with the test user during the test run that are - // public. - userPropCount := 3 - if upr.NumOfProposals != userPropCount { - return fmt.Errorf("user proposal count got %v, want %v", - upr.NumOfProposals, userPropCount) - } - - for _, v := range upr.Proposals { - err := shared.VerifyProposal(v, version.PubKey) if err != nil { - return fmt.Errorf("verify proposal failed %v: %v", - v.CensorshipRecord.Token, err) + return err } - } - // Vote details - fmt.Printf(" Vote details\n") - vdr, err := client.VoteDetailsV2(token) - if err != nil { - return err - } - switch vdr.Version { - case 2: - // Validate signature - vote, err := decredplugin.DecodeVoteV2([]byte(vdr.Vote)) + fmt.Printf(" Making unvetted proposals public\n") + for _, v := range bpr.Proposals { + spsc := SetProposalStatusCmd{} + spsc.Args.Token = v.CensorshipRecord.Token + spsc.Args.Status = strconv.Itoa(int(v1.PropStatusPublic)) + err = spsc.Execute(nil) + if err != nil { + return err + } + } + + fmt.Printf(" Logout\n") + lc = shared.LogoutCmd{} + err = lc.Execute(nil) if err != nil { return err } - dsv := decredplugin.StartVoteV2{ - PublicKey: vdr.PublicKey, - Vote: *vote, - Signature: vdr.Signature, - } - err = dsv.VerifySignature() + + fmt.Printf(" Vetted\n") + gavr, err := client.GetAllVetted(&v1.GetAllVetted{}) if err != nil { return err } - default: - return fmt.Errorf("unknown start vote version %v", vdr.Version) - } - // Vote status - fmt.Printf(" Vote status\n") - vsr, err = client.VoteStatus(token) - if err != nil { - return err - } + if len(gavr.Proposals) != v1.ProposalListPageSize { + return fmt.Errorf("proposals page size got %v, want %v", + len(gavr.Proposals), v1.ProposalListPageSize) + } - if vsr.Status != v1.PropVoteStatusStarted { - return fmt.Errorf("vote status got %v, want %v", - vsr.Status, v1.PropVoteStatusStarted) - } + for _, v := range gavr.Proposals { + err = shared.VerifyProposal(v, version.PubKey) + if err != nil { + return fmt.Errorf("verify proposal failed %v: %v", + v.CensorshipRecord.Token, err) + } + } + + // User details + fmt.Printf(" User details\n") + udr, err := client.UserDetails(user.ID) + if err != nil { + return err + } - // Active votes - fmt.Printf(" Active votes\n") - avr, err := client.ActiveVotes() - if err != nil { - return err - } + if udr.User.ID != user.ID { + return fmt.Errorf("user ID got %v, want %v", + udr.User.ID, user.ID) + } - var found bool - for _, v := range avr.Votes { - if v.Proposal.CensorshipRecord.Token == token { - found = true + // User proposals + fmt.Printf(" User proposals\n") + upr, err := client.UserProposals( + &v1.UserProposals{ + UserId: user.ID, + }) + if err != nil { + return err } - } - if !found { - return fmt.Errorf("proposal %v not found in active votes", - token) - } - // Cast votes - fmt.Printf(" Cast votes\n") - dcrwalletFailed, err := castVotes(token, vsr.OptionsResult[0].Option.Id) + // userPropCount is the total number of proposals that we've + // submitted with the test user during the test run that are + // public. + userPropCount := 3 + if upr.NumOfProposals != userPropCount { + return fmt.Errorf("user proposal count got %v, want %v", + upr.NumOfProposals, userPropCount) + } - // Find how many votes the user cast so that - // we can compare it against the vote results. - var voteCount int - if !dcrwalletFailed { - // Get proposal vote details - var pvt v1.ProposalVoteTuple - for _, v := range avr.Votes { - if v.Proposal.CensorshipRecord.Token == token { - pvt = v - break + for _, v := range upr.Proposals { + err := shared.VerifyProposal(v, version.PubKey) + if err != nil { + return fmt.Errorf("verify proposal failed %v: %v", + v.CensorshipRecord.Token, err) } } - // Get the number of eligible tickets the user had - ticketPool, err := convertTicketHashes(pvt.StartVoteReply.EligibleTickets) + // Vote details + fmt.Printf(" Vote details\n") + vdr, err := client.VoteDetailsV2(token) if err != nil { return err } + switch vdr.Version { + case 2: + // Validate signature + vote, err := decredplugin.DecodeVoteV2([]byte(vdr.Vote)) + if err != nil { + return err + } + dsv := decredplugin.StartVoteV2{ + PublicKey: vdr.PublicKey, + Vote: *vote, + Signature: vdr.Signature, + } + err = dsv.VerifySignature() + if err != nil { + return err + } + default: + return fmt.Errorf("unknown start vote version %v", vdr.Version) + } - err = client.LoadWalletClient() + // Vote status + fmt.Printf(" Vote status\n") + vsr, err = client.VoteStatus(token) if err != nil { return err } - defer client.Close() - ctr, err := client.CommittedTickets( - &walletrpc.CommittedTicketsRequest{ - Tickets: ticketPool, - }) + if vsr.Status != v1.PropVoteStatusStarted { + return fmt.Errorf("vote status got %v, want %v", + vsr.Status, v1.PropVoteStatusStarted) + } + + // Active votes + fmt.Printf(" Active votes\n") + avr, err := client.ActiveVotes() if err != nil { return err } - voteCount = len(ctr.TicketAddresses) - } - - // Vote results - fmt.Printf(" Vote results\n") - vrr, err := client.VoteResults(token) - if err != nil { - return err - } + var found bool + for _, v := range avr.Votes { + if v.Proposal.CensorshipRecord.Token == token { + found = true + } + } + if !found { + return fmt.Errorf("proposal %v not found in active votes", + token) + } - if len(vrr.CastVotes) != voteCount { - return fmt.Errorf("num cast votes got %v, want %v", - len(vrr.CastVotes), voteCount) - } + // Cast votes + fmt.Printf(" Cast votes\n") + dcrwalletFailed, err := castVotes(token, vsr.OptionsResult[0].Option.Id) - // RFP routes - fmt.Println("Running RFP routes") + // Find how many votes the user cast so that + // we can compare it against the vote results. + var voteCount int + if !dcrwalletFailed { + // Get proposal vote details + var pvt v1.ProposalVoteTuple + for _, v := range avr.Votes { + if v.Proposal.CensorshipRecord.Token == token { + pvt = v + break + } + } - // Login with admin to create an RFP - // and make it public. - fmt.Printf(" Login admin\n") - err = login(admin.email, admin.password) - if err != nil { - return err - } + // Get the number of eligible tickets the user had + ticketPool, err := convertTicketHashes(pvt.StartVoteReply.EligibleTickets) + if err != nil { + return err + } - // Create RFP - fmt.Println(" Create a RFP") - np, err = newRFPProposal() - if err != nil { - return err - } - npr, err = client.NewProposal(np) - if err != nil { - return err - } + err = client.LoadWalletClient() + if err != nil { + return err + } + defer client.Close() - // Verify RFP censorship record - rpr := v1.ProposalRecord{ - Files: np.Files, - Metadata: np.Metadata, - PublicKey: np.PublicKey, - Signature: np.Signature, - CensorshipRecord: npr.CensorshipRecord, - } - err = shared.VerifyProposal(rpr, version.PubKey) - if err != nil { - return fmt.Errorf("verify RFP failed: %v", err) - } + ctr, err := client.CommittedTickets( + &walletrpc.CommittedTicketsRequest{ + Tickets: ticketPool, + }) + if err != nil { + return err + } - token = rpr.CensorshipRecord.Token - fmt.Printf(" RFP submitted: %v\n", token) + voteCount = len(ctr.TicketAddresses) + } - // Make RFP public - fmt.Printf(" Set RFP status: not reviewed to public\n") - spsc = SetProposalStatusCmd{} - spsc.Args.Token = token - spsc.Args.Status = strconv.Itoa(int(v1.PropStatusPublic)) - err = spsc.Execute(nil) - if err != nil { - return err - } + // Vote results + fmt.Printf(" Vote results\n") + vrr, err := client.VoteResults(token) + if err != nil { + return err + } - pdr, err = client.ProposalDetails(spsc.Args.Token, nil) - if err != nil { - return err - } + if len(vrr.CastVotes) != voteCount { + return fmt.Errorf("num cast votes got %v, want %v", + len(vrr.CastVotes), voteCount) + } - if pdr.Proposal.Status != v1.PropStatusPublic { - return fmt.Errorf("RFP status got %v, want %v", - pdr.Proposal.Status, v1.PropStatusPublic) - } + // RFP routes + fmt.Println("Running RFP routes") - // Try to submit a RFP submission before RFP approval & expect to fail - fmt.Println(" Try submitting a submission before RFP voting") - np, err = newSubmissionProposal(token) - if err != nil { - return err - } - npr, err = client.NewProposal(np) - if err != nil { - switch { - case strings.Contains(err.Error(), "rfp proposal vote did not pass"): - fmt.Println(" Submission failed with expected error") - default: + // Login with admin to create an RFP + // and make it public. + fmt.Printf(" Login admin\n") + err = login(admin.email, admin.password) + if err != nil { return err } - } - - // Authorize vote - fmt.Printf(" Authorize vote: authorize\n") - avc = AuthorizeVoteCmd{} - avc.Args.Token = token - avc.Args.Action = decredplugin.AuthVoteActionAuthorize - err = avc.Execute(nil) - if err != nil { - return err - } - // Start RFP vote - fmt.Printf(" Start RFP vote\n") - svc = StartVoteCmd{} - svc.Args.Token = token - svc.Args.Duration = policy.MinVoteDuration - svc.Args.PassPercentage = "1" - svc.Args.QuorumPercentage = "1" - err = svc.Execute(nil) - if err != nil { - return err - } - - // Wait to RFP to finish voting - var vs v1.VoteSummary - for vs.Status != v1.PropVoteStatusFinished { - bvs := v1.BatchVoteSummary{ - Tokens: []string{token}, + // Create RFP + fmt.Println(" Create a RFP") + np, err = newRFPProposal() + if err != nil { + return err } - bvsr, err := client.BatchVoteSummary(&bvs) + npr, err = client.NewProposal(np) if err != nil { return err } - vs = bvsr.Summaries[token] + // Verify RFP censorship record + rpr := v1.ProposalRecord{ + Files: np.Files, + Metadata: np.Metadata, + PublicKey: np.PublicKey, + Signature: np.Signature, + CensorshipRecord: npr.CensorshipRecord, + } + err = shared.VerifyProposal(rpr, version.PubKey) + if err != nil { + return fmt.Errorf("verify RFP failed: %v", err) + } - fmt.Printf(" RFP voting still going on, block %v\\%v \n", - bvsr.BestBlock, vs.EndHeight) - time.Sleep(sleepInterval) - } - if !vs.Approved { - fmt.Println(" RFP rejected") - } else { - return fmt.Errorf("RFP approved? %v, want false", - vs.Approved) - } + token = rpr.CensorshipRecord.Token + fmt.Printf(" RFP submitted: %v\n", token) - // Try to submit a RFP submission on rejected RFP - fmt.Println(" Try submitting a submission on rejected RFP") - np, err = newSubmissionProposal(token) - if err != nil { - return err - } - npr, err = client.NewProposal(np) - if err != nil { - switch { - case strings.Contains(err.Error(), "rfp proposal vote did not pass"): - fmt.Println(" Submission failed with expected error") - default: + // Make RFP public + fmt.Printf(" Set RFP status: not reviewed to public\n") + spsc = SetProposalStatusCmd{} + spsc.Args.Token = token + spsc.Args.Status = strconv.Itoa(int(v1.PropStatusPublic)) + err = spsc.Execute(nil) + if err != nil { return err } - } - // Create another RFP - fmt.Println(" Create another RFP") - np, err = newRFPProposal() - if err != nil { - return err - } - npr, err = client.NewProposal(np) - if err != nil { - return err - } - token = npr.CensorshipRecord.Token - - // Make second RFP public - fmt.Printf(" Set RFP status: not reviewed to public\n") - spsc = SetProposalStatusCmd{} - spsc.Args.Token = token - spsc.Args.Status = strconv.Itoa(int(v1.PropStatusPublic)) - err = spsc.Execute(nil) - if err != nil { - return err - } + pdr, err = client.ProposalDetails(spsc.Args.Token, nil) + if err != nil { + return err + } - // Authorize vote - fmt.Printf(" Authorize vote: authorize\n") - avc = AuthorizeVoteCmd{} - avc.Args.Token = token - avc.Args.Action = decredplugin.AuthVoteActionAuthorize - err = avc.Execute(nil) - if err != nil { - return err - } + if pdr.Proposal.Status != v1.PropStatusPublic { + return fmt.Errorf("RFP status got %v, want %v", + pdr.Proposal.Status, v1.PropStatusPublic) + } - // Start RFP vote - fmt.Printf(" Start RFP vote\n") - svc = StartVoteCmd{} - svc.Args.Token = token - svc.Args.Duration = policy.MinVoteDuration - svc.Args.PassPercentage = "0" - svc.Args.QuorumPercentage = "0" - err = svc.Execute(nil) - if err != nil { - return err - } + // Try to submit a RFP submission before RFP approval & expect to fail + fmt.Println(" Try submitting a submission before RFP voting") + np, err = newSubmissionProposal(token) + if err != nil { + return err + } + npr, err = client.NewProposal(np) + if err != nil { + switch { + case strings.Contains(err.Error(), "rfp proposal vote did not pass"): + fmt.Println(" Submission failed with expected error") + default: + return err + } + } + + // Authorize vote + fmt.Printf(" Authorize vote: authorize\n") + avc = AuthorizeVoteCmd{} + avc.Args.Token = token + avc.Args.Action = decredplugin.AuthVoteActionAuthorize + err = avc.Execute(nil) + if err != nil { + return err + } - // Cast RFP votes - fmt.Printf(" Cast 'Yes' votes\n") - dcrwalletFailed, err = castVotes(token, vsr.OptionsResult[0].Option.Id) + // Start RFP vote + fmt.Printf(" Start RFP vote\n") + svc = StartVoteCmd{} + svc.Args.Token = token + svc.Args.Duration = policy.MinVoteDuration + svc.Args.PassPercentage = "1" + svc.Args.QuorumPercentage = "1" + err = svc.Execute(nil) + if err != nil { + return err + } - if !dcrwalletFailed { // Wait to RFP to finish voting var vs v1.VoteSummary for vs.Status != v1.PropVoteStatusFinished { @@ -1556,63 +1464,31 @@ func (cmd *TestRunCmd) Execute(args []string) error { time.Sleep(sleepInterval) } if !vs.Approved { - return fmt.Errorf("RFP approved? %v, want true", + fmt.Println(" RFP rejected") + } else { + return fmt.Errorf("RFP approved? %v, want false", vs.Approved) } - fmt.Printf(" RFP approved successfully\n") - // Create 4 RFP submissions. - // 1 Unreviewd - fmt.Println(" Create unreviewed RFP submission") + // Try to submit a RFP submission on rejected RFP + fmt.Println(" Try submitting a submission on rejected RFP") np, err = newSubmissionProposal(token) if err != nil { return err } npr, err = client.NewProposal(np) if err != nil { - return err - } - // 2 Public - fmt.Println(" Create 2 more submissions & make them public") - np, err = newSubmissionProposal(token) - if err != nil { - return err - } - npr, err = client.NewProposal(np) - if err != nil { - return err - } - firststoken := npr.CensorshipRecord.Token - fmt.Printf(" Set first submission status: not reviewed to" + - " public\n") - spsc = SetProposalStatusCmd{} - spsc.Args.Token = firststoken - spsc.Args.Status = strconv.Itoa(int(v1.PropStatusPublic)) - err = spsc.Execute(nil) - if err != nil { - return err - } - np, err = newSubmissionProposal(token) - if err != nil { - return err - } - npr, err = client.NewProposal(np) - if err != nil { - return err - } - secondstoken := npr.CensorshipRecord.Token - fmt.Printf(" Set second submission status: not reviewed to" + - " public\n") - spsc = SetProposalStatusCmd{} - spsc.Args.Token = secondstoken - spsc.Args.Status = strconv.Itoa(int(v1.PropStatusPublic)) - err = spsc.Execute(nil) - if err != nil { - return err + switch { + case strings.Contains(err.Error(), "rfp proposal vote did not pass"): + fmt.Println(" Submission failed with expected error") + default: + return err + } } - // 1 Abandoned, first make public - // then abandon - np, err = newSubmissionProposal(token) + + // Create another RFP + fmt.Println(" Create another RFP") + np, err = newRFPProposal() if err != nil { return err } @@ -1620,121 +1496,244 @@ func (cmd *TestRunCmd) Execute(args []string) error { if err != nil { return err } - thirdstoken := npr.CensorshipRecord.Token - fmt.Printf(" Set third submission status: not reviewed to" + - " abandoned\n") + token = npr.CensorshipRecord.Token + + // Make second RFP public + fmt.Printf(" Set RFP status: not reviewed to public\n") spsc = SetProposalStatusCmd{} - spsc.Args.Token = thirdstoken + spsc.Args.Token = token spsc.Args.Status = strconv.Itoa(int(v1.PropStatusPublic)) err = spsc.Execute(nil) if err != nil { return err } - spsc.Args.Status = "abandoned" - spsc.Args.Message = "this is spam" - err = spsc.Execute(nil) - if err != nil { - return err - } - // Start runoff vote - fmt.Printf(" Start RFP submissions runoff vote\n") - svrc := StartVoteRunoffCmd{} - svrc.Args.TokenRFP = token - svrc.Args.Duration = policy.MinVoteDuration - svrc.Args.PassPercentage = "0" - svrc.Args.QuorumPercentage = "0" - err = svrc.Execute(nil) + + // Authorize vote + fmt.Printf(" Authorize vote: authorize\n") + avc = AuthorizeVoteCmd{} + avc.Args.Token = token + avc.Args.Action = decredplugin.AuthVoteActionAuthorize + err = avc.Execute(nil) if err != nil { return err } - // Cast first submission votes - fmt.Printf(" Cast first submission votes\n") - dcrwalletFailed, err = castVotes(firststoken, vsr.OptionsResult[0].Option.Id) + + // Start RFP vote + fmt.Printf(" Start RFP vote\n") + svc = StartVoteCmd{} + svc.Args.Token = token + svc.Args.Duration = policy.MinVoteDuration + svc.Args.PassPercentage = "0" + svc.Args.QuorumPercentage = "0" + err = svc.Execute(nil) if err != nil { return err } - if !dcrwalletFailed { - // Try cast votes on abandoned & expect error - fmt.Println(" Try casting votes on abandoned RFP submission") - _, err = castVotes(thirdstoken, vsr.OptionsResult[0].Option.Id) - if err != nil { - switch { - case strings.Contains(err.Error(), "proposal not found"): - fmt.Println(" Casting votes on abandoned submission failed") - default: - return err - } - } - // Wait to runoff vote finish + // Cast RFP votes + fmt.Printf(" Cast 'Yes' votes\n") + dcrwalletFailed, err = castVotes(token, vsr.OptionsResult[0].Option.Id) + + if !dcrwalletFailed { + // Wait to RFP to finish voting var vs v1.VoteSummary for vs.Status != v1.PropVoteStatusFinished { bvs := v1.BatchVoteSummary{ - Tokens: []string{firststoken}, + Tokens: []string{token}, } bvsr, err := client.BatchVoteSummary(&bvs) if err != nil { return err } - vs = bvsr.Summaries[firststoken] + vs = bvsr.Summaries[token] - fmt.Printf(" Runoff vote still going on, block %v\\%v \n", + fmt.Printf(" RFP voting still going on, block %v\\%v \n", bvsr.BestBlock, vs.EndHeight) time.Sleep(sleepInterval) } if !vs.Approved { - return fmt.Errorf("First submission approved? %v, want true", + return fmt.Errorf("RFP approved? %v, want true", vs.Approved) } - fmt.Printf(" First submission approved successfully\n") + + fmt.Printf(" RFP approved successfully\n") + // Create 4 RFP submissions. + // 1 Unreviewd + fmt.Println(" Create unreviewed RFP submission") + np, err = newSubmissionProposal(token) + if err != nil { + return err + } + npr, err = client.NewProposal(np) + if err != nil { + return err + } + // 2 Public + fmt.Println(" Create 2 more submissions & make them public") + np, err = newSubmissionProposal(token) + if err != nil { + return err + } + npr, err = client.NewProposal(np) + if err != nil { + return err + } + firststoken := npr.CensorshipRecord.Token + fmt.Printf(" Set first submission status: not reviewed to" + + " public\n") + spsc = SetProposalStatusCmd{} + spsc.Args.Token = firststoken + spsc.Args.Status = strconv.Itoa(int(v1.PropStatusPublic)) + err = spsc.Execute(nil) + if err != nil { + return err + } + np, err = newSubmissionProposal(token) + if err != nil { + return err + } + npr, err = client.NewProposal(np) + if err != nil { + return err + } + secondstoken := npr.CensorshipRecord.Token + fmt.Printf(" Set second submission status: not reviewed to" + + " public\n") + spsc = SetProposalStatusCmd{} + spsc.Args.Token = secondstoken + spsc.Args.Status = strconv.Itoa(int(v1.PropStatusPublic)) + err = spsc.Execute(nil) + if err != nil { + return err + } + // 1 Abandoned, first make public + // then abandon + np, err = newSubmissionProposal(token) + if err != nil { + return err + } + npr, err = client.NewProposal(np) + if err != nil { + return err + } + thirdstoken := npr.CensorshipRecord.Token + fmt.Printf(" Set third submission status: not reviewed to" + + " abandoned\n") + spsc = SetProposalStatusCmd{} + spsc.Args.Token = thirdstoken + spsc.Args.Status = strconv.Itoa(int(v1.PropStatusPublic)) + err = spsc.Execute(nil) + if err != nil { + return err + } + spsc.Args.Status = "abandoned" + spsc.Args.Message = "this is spam" + err = spsc.Execute(nil) + if err != nil { + return err + } + // Start runoff vote + fmt.Printf(" Start RFP submissions runoff vote\n") + svrc := StartVoteRunoffCmd{} + svrc.Args.TokenRFP = token + svrc.Args.Duration = policy.MinVoteDuration + svrc.Args.PassPercentage = "0" + svrc.Args.QuorumPercentage = "0" + err = svrc.Execute(nil) + if err != nil { + return err + } + // Cast first submission votes + fmt.Printf(" Cast first submission votes\n") + dcrwalletFailed, err = castVotes(firststoken, vsr.OptionsResult[0].Option.Id) + if err != nil { + return err + } + if !dcrwalletFailed { + // Try cast votes on abandoned & expect error + fmt.Println(" Try casting votes on abandoned RFP submission") + _, err = castVotes(thirdstoken, vsr.OptionsResult[0].Option.Id) + if err != nil { + switch { + case strings.Contains(err.Error(), "proposal not found"): + fmt.Println(" Casting votes on abandoned submission failed") + default: + return err + } + } + + // Wait to runoff vote finish + var vs v1.VoteSummary + for vs.Status != v1.PropVoteStatusFinished { + bvs := v1.BatchVoteSummary{ + Tokens: []string{firststoken}, + } + bvsr, err := client.BatchVoteSummary(&bvs) + if err != nil { + return err + } + + vs = bvsr.Summaries[firststoken] + + fmt.Printf(" Runoff vote still going on, block %v\\%v \n", + bvsr.BestBlock, vs.EndHeight) + time.Sleep(sleepInterval) + } + if !vs.Approved { + return fmt.Errorf("First submission approved? %v, want true", + vs.Approved) + } + fmt.Printf(" First submission approved successfully\n") + } } - } - // Logout - fmt.Printf(" Logout\n") - lc = shared.LogoutCmd{} - err = lc.Execute(nil) - if err != nil { - return err - } + // Logout + fmt.Printf(" Logout\n") + lc = shared.LogoutCmd{} + err = lc.Execute(nil) + if err != nil { + return err + } - // Websockets - fmt.Printf("Running websocket routes\n") + // Websockets + fmt.Printf("Running websocket routes\n") - // Websocket - unauthenticated ping - fmt.Printf(" Websocket: unauthenticated ping: ") - sc := SubscribeCmd{ - Close: true, - } - err = sc.Execute([]string{"ping"}) - if err != nil { - return err - } + // Websocket - unauthenticated ping + fmt.Printf(" Websocket: unauthenticated ping: ") + sc := SubscribeCmd{ + Close: true, + } + err = sc.Execute([]string{"ping"}) + if err != nil { + return err + } - // Login with user - fmt.Printf(" Login user\n") - err = login(user.email, user.password) - if err != nil { - return err - } + // Login with user + fmt.Printf(" Login user\n") + err = login(user.email, user.password) + if err != nil { + return err + } - // Websocket - authenticated ping - fmt.Printf(" Websocket: authenticated ping: ") - err = sc.Execute([]string{"auth", "ping"}) - if err != nil { - return err - } + // Websocket - authenticated ping + fmt.Printf(" Websocket: authenticated ping: ") + err = sc.Execute([]string{"auth", "ping"}) + if err != nil { + return err + } - // Logout - fmt.Printf(" Logout\n") - lc = shared.LogoutCmd{} - err = lc.Execute(nil) - if err != nil { - return err - } + // Logout + fmt.Printf(" Logout\n") + lc = shared.LogoutCmd{} + err = lc.Execute(nil) + if err != nil { + return err + } - fmt.Printf("Test run successful!\n") + fmt.Printf("Test run successful!\n") + return nil + */ return nil } diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index bc5b4b3b4..d2d7dc839 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -19,6 +19,7 @@ import ( "github.com/decred/dcrwallet/rpc/walletrpc" cms "github.com/decred/politeia/politeiawww/api/cms/v1" + pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" www2 "github.com/decred/politeia/politeiawww/api/www/v2" "github.com/decred/politeia/util" @@ -699,53 +700,100 @@ func (c *Client) ProposalPaywallDetails() (*www.ProposalPaywallDetailsReply, err return &ppdr, nil } -// NewProposal submits the specified proposal to politeiawww for the logged in -// user. -func (c *Client) NewProposal(np *www.NewProposal) (*www.NewProposalReply, error) { +// ProposalNew submits a new proposal. +func (c *Client) ProposalNew(pn pi.ProposalNew) (*pi.ProposalNewReply, error) { + responseBody, err := c.makeRequest(http.MethodPost, + pi.APIRoute, pi.RouteProposalNew, pn) + if err != nil { + return nil, err + } + + var pnr pi.ProposalNewReply + err = json.Unmarshal(responseBody, &pnr) + if err != nil { + return nil, fmt.Errorf("unmarshal ProposalNewReply: %v", err) + } + + if c.cfg.Verbose { + err := prettyPrintJSON(pnr) + if err != nil { + return nil, err + } + } + + return &pnr, nil +} + +// ProposalEdit edits a proposal. +func (c *Client) ProposalEdit(pe pi.ProposalEdit) (*pi.ProposalEditReply, error) { + responseBody, err := c.makeRequest(http.MethodPost, + pi.APIRoute, pi.RouteProposalEdit, pe) + if err != nil { + return nil, err + } + + var per pi.ProposalEditReply + err = json.Unmarshal(responseBody, &per) + if err != nil { + return nil, fmt.Errorf("unmarshal ProposalEditReply: %v", err) + } + + if c.cfg.Verbose { + err := prettyPrintJSON(per) + if err != nil { + return nil, err + } + } + + return &per, nil +} + +// ProposalSetStatus sets the status of a proposal +func (c *Client) ProposalSetStatus(pss pi.ProposalSetStatus) (*pi.ProposalSetStatusReply, error) { responseBody, err := c.makeRequest(http.MethodPost, - www.PoliteiaWWWAPIRoute, www.RouteNewProposal, np) + pi.APIRoute, pi.RouteProposalSetStatus, pss) if err != nil { return nil, err } - var npr www.NewProposalReply - err = json.Unmarshal(responseBody, &npr) + var pssr pi.ProposalSetStatusReply + err = json.Unmarshal(responseBody, &pssr) if err != nil { - return nil, fmt.Errorf("unmarshal NewProposalReply: %v", err) + return nil, fmt.Errorf("unmarshal ProposalSetStatusReply: %v", err) } if c.cfg.Verbose { - err := prettyPrintJSON(npr) + err := prettyPrintJSON(pssr) if err != nil { return nil, err } } - return &npr, nil + return &pssr, nil } -// EditProposal edits the specified proposal with the logged in user. -func (c *Client) EditProposal(ep *www.EditProposal) (*www.EditProposalReply, error) { +// Proposals retrieves a proposal for each of the provided proposal requests. +func (c *Client) Proposals(p pi.Proposals) (*pi.ProposalsReply, error) { responseBody, err := c.makeRequest(http.MethodPost, - www.PoliteiaWWWAPIRoute, www.RouteEditProposal, ep) + pi.APIRoute, pi.RouteProposals, p) if err != nil { return nil, err } - var epr www.EditProposalReply - err = json.Unmarshal(responseBody, &epr) + var pr pi.ProposalsReply + err = json.Unmarshal(responseBody, &pr) if err != nil { - return nil, fmt.Errorf("unmarshal EditProposalReply: %v", err) + return nil, fmt.Errorf("unmarshal ProposalsReply: %v", err) } if c.cfg.Verbose { - err := prettyPrintJSON(epr) + err := prettyPrintJSON(pr) if err != nil { return nil, err } } - return &epr, nil + return &pr, nil } // NewInvoice submits the specified invoice to politeiawww for the logged in @@ -1013,31 +1061,6 @@ func (c *Client) PayInvoices(pi *cms.PayInvoices) (*cms.PayInvoicesReply, error) return &pir, nil } -// SetProposalStatus changes the status of the specified proposal. -func (c *Client) SetProposalStatus(sps *www.SetProposalStatus) (*www.SetProposalStatusReply, error) { - route := "/proposals/" + sps.Token + "/status" - responseBody, err := c.makeRequest(http.MethodPost, - www.PoliteiaWWWAPIRoute, route, sps) - if err != nil { - return nil, err - } - - var spsr www.SetProposalStatusReply - err = json.Unmarshal(responseBody, &spsr) - if err != nil { - return nil, fmt.Errorf("unmarshal SetProposalStatusReply: %v", err) - } - - if c.cfg.Verbose { - err := prettyPrintJSON(spsr) - if err != nil { - return nil, err - } - } - - return &spsr, nil -} - // BatchProposals retrieves a list of proposals func (c *Client) BatchProposals(bp *www.BatchProposals) (*www.BatchProposalsReply, error) { responseBody, err := c.makeRequest(http.MethodPost, www.PoliteiaWWWAPIRoute, diff --git a/politeiawww/cmd/shared/shared.go b/politeiawww/cmd/shared/shared.go index 62a9bad5b..c51320cdd 100644 --- a/politeiawww/cmd/shared/shared.go +++ b/politeiawww/cmd/shared/shared.go @@ -105,21 +105,6 @@ func ValidateDigests(files []v1.File, md []v1.Metadata) error { return nil } -// SignedMerkleRoot calculates the merkle root of the passed in list of files -// and metadata, signs the merkle root with the passed in identity and returns -// the signature. -func SignedMerkleRoot(files []v1.File, md []v1.Metadata, id *identity.FullIdentity) (string, error) { - if len(files) == 0 { - return "", fmt.Errorf("no proposal files found") - } - mr, err := wwwutil.MerkleRoot(files, md) - if err != nil { - return "", err - } - sig := id.SignMessage([]byte(mr)) - return hex.EncodeToString(sig[:]), nil -} - // DigestSHA3 returns the hex encoded SHA3-256 of a string. func DigestSHA3(s string) string { h := sha3.New256() @@ -167,7 +152,7 @@ func VerifyProposal(p v1.ProposalRecord, serverPubKey string) error { return err } // Verify merkle root - mr, err := wwwutil.MerkleRoot(p.Files, p.Metadata) + mr, err := wwwutil.MerkleRootWWW(p.Files, p.Metadata) if err != nil { return err } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 015833348..8d756bba7 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -453,7 +453,7 @@ func (p *politeiawww) proposalRecordLatest(state pi.PropStateT, token string) (* // requests. If a token does not correspond to an actual proposal then it will // not be included in the returned map. // -// XXX politeiad needs batched calls for retrieving unvetted and vetted +// TODO politeiad needs batched calls for retrieving unvetted and vetted // records. This call should have an includeFiles option. func (p *politeiawww) proposalRecords(state pi.PropStateT, reqs []pi.ProposalRequest, includeFiles bool) (map[string]pi.ProposalRecord, error) { // Get politeiad records @@ -1230,10 +1230,9 @@ reply: } // processProposals retrieves and returns the proposal records for each of the -// provided proposal requests. If unvetted proposals are requested by a -// non-admin then the unvetted proposal files are removed before the proposal -// is returned. -func (p *politeiawww) processProposalsPublic(ps pi.Proposals, isAdmin bool) (*pi.ProposalsReply, error) { +// provided proposal requests. Unvetted proposal files are only returned to +// admins. +func (p *politeiawww) processProposals(ps pi.Proposals, isAdmin bool) (*pi.ProposalsReply, error) { log.Tracef("processProposals: %v", ps.Requests) props, err := p.proposalRecords(ps.State, ps.Requests, ps.IncludeFiles) @@ -1267,3 +1266,113 @@ func (p *politeiawww) processProposalInventory() (*pi.ProposalInventoryReply, er return &pi.ProposalInventoryReply{}, nil } + +func (p *politeiawww) handleProposalNew(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleProposalNew") + + var pn pi.ProposalNew + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&pn); err != nil { + respondWithPiError(w, r, "handleProposalNew: unmarshal", + pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusInvalidInput, + }) + return + } + + user, err := p.getSessionUser(w, r) + if err != nil { + respondWithPiError(w, r, + "handleProposalNew: getSessionUser: %v", err) + return + } + + pnr, err := p.processProposalNew(pn, *user) + if err != nil { + respondWithPiError(w, r, + "handleProposalNew: processProposalNew: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, pnr) +} + +func (p *politeiawww) handleProposalEdit(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleProposalEdit") + + var pe pi.ProposalEdit + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&pe); err != nil { + respondWithPiError(w, r, "handleProposalEdit: unmarshal", + pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusInvalidInput, + }) + return + } + + user, err := p.getSessionUser(w, r) + if err != nil { + respondWithPiError(w, r, + "handleProposalEdit: getSessionUser: %v", err) + return + } + + per, err := p.processProposalEdit(pe, *user) + if err != nil { + respondWithPiError(w, r, + "handleProposalEdit: processProposalEdit: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, per) +} + +func (p *politeiawww) handleProposalSetStatus(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleProposalSetStatus") + + var pss pi.ProposalSetStatus + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&pss); err != nil { + respondWithPiError(w, r, "handleProposalSetStatus: unmarshal", + pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusInvalidInput, + }) + return + } + + user, err := p.getSessionUser(w, r) + if err != nil { + respondWithPiError(w, r, + "handleProposalSetStatus: getSessionUser: %v", err) + return + } + + pssr, err := p.processProposalSetStatus(pss, *user) + if err != nil { + respondWithPiError(w, r, + "handleProposalSetStatus: processProposalSetStatus: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, pssr) +} + +func (p *politeiawww) setPiRoutes() { + // Public routes + + // Logged in routes + p.addRoute(http.MethodPost, pi.APIRoute, + pi.RouteProposalNew, p.handleProposalNew, + permissionLogin) + + p.addRoute(http.MethodPost, pi.APIRoute, + pi.RouteProposalEdit, p.handleProposalEdit, + permissionLogin) + + p.addRoute(http.MethodPost, pi.APIRoute, + pi.RouteProposalSetStatus, p.handleProposalSetStatus, + permissionLogin) + + // Admin routes + +} diff --git a/politeiawww/util/merkle.go b/politeiawww/util/merkle.go index b55dc36c7..f6c3d726e 100644 --- a/politeiawww/util/merkle.go +++ b/politeiawww/util/merkle.go @@ -11,6 +11,7 @@ import ( "github.com/decred/dcrtime/merkle" pi "github.com/decred/politeia/politeiawww/api/pi/v1" + www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/util" ) @@ -46,3 +47,35 @@ func MerkleRoot(files []pi.File, md []pi.Metadata) (string, error) { // Return merkle root return hex.EncodeToString(merkle.Root(digests)[:]), nil } + +// TODO remove this once cli has been converted over to use pi types. +func MerkleRootWWW(files []www.File, md []www.Metadata) (string, error) { + digests := make([]*[sha256.Size]byte, 0, len(files)) + + // Calculate file digests + for _, f := range files { + b, err := base64.StdEncoding.DecodeString(f.Payload) + if err != nil { + return "", err + } + digest := util.Digest(b) + var hf [sha256.Size]byte + copy(hf[:], digest) + digests = append(digests, &hf) + } + + // Calculate metadata digests + for _, v := range md { + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return "", err + } + digest := util.Digest(b) + var hv [sha256.Size]byte + copy(hv[:], digest) + digests = append(digests, &hv) + } + + // Return merkle root + return hex.EncodeToString(merkle.Root(digests)[:]), nil +} diff --git a/politeiawww/www.go b/politeiawww/www.go index 27fffb538..087ffa7b9 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -742,6 +742,8 @@ func _main() error { p.setPoliteiaWWWRoutes() p.setUserWWWRoutes() + p.setPiRoutes() + err = p.initPaywallChecker() if err != nil { return err From b7f58a738c2a0628f970ede3e8a8f755d8776010 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 15 Sep 2020 09:57:59 -0500 Subject: [PATCH 045/449] hook up politeiad plugins --- plugins/comments/comments.go | 3 +- plugins/dcrdata/dcrdata.go | 3 +- plugins/pi/pi.go | 3 +- plugins/ticketvote/ticketvote.go | 3 +- politeiad/backend/backend.go | 12 +++-- politeiad/backend/gitbe/gitbe.go | 14 +++++ politeiad/backend/tlogbe/tlogbe.go | 38 +++++++++----- politeiad/config.go | 13 ++--- politeiad/politeiad.go | 84 ++++++++++++++++++++++++------ 9 files changed, 129 insertions(+), 44 deletions(-) diff --git a/plugins/comments/comments.go b/plugins/comments/comments.go index 05a72bcc8..a72f3c654 100644 --- a/plugins/comments/comments.go +++ b/plugins/comments/comments.go @@ -17,8 +17,7 @@ type ErrorStatusT int // TODO add a hint to comments that can be used freely by the client. This // is how we'll distinguish proposal comments from update comments. const ( - Version uint32 = 1 - ID = "comments" + ID = "comments" // Plugin commands CmdNew = "new" // Create a new comment diff --git a/plugins/dcrdata/dcrdata.go b/plugins/dcrdata/dcrdata.go index d3757fcf7..cbe2edce4 100644 --- a/plugins/dcrdata/dcrdata.go +++ b/plugins/dcrdata/dcrdata.go @@ -15,8 +15,7 @@ import ( type StatusT int const ( - Version uint32 = 1 - ID = "dcrdata" + ID = "dcrdata" // Plugin commands CmdBestBlock = "bestblock" // Get best block diff --git a/plugins/pi/pi.go b/plugins/pi/pi.go index a02aa575f..25ff2373a 100644 --- a/plugins/pi/pi.go +++ b/plugins/pi/pi.go @@ -18,8 +18,7 @@ type PropStatusT int type ErrorStatusT int const ( - Version uint32 = 1 - ID = "pi" + ID = "pi" // Plugin commands CmdProposals = "proposals" // Get proposals plugin data diff --git a/plugins/ticketvote/ticketvote.go b/plugins/ticketvote/ticketvote.go index 528a30c04..ece52779c 100644 --- a/plugins/ticketvote/ticketvote.go +++ b/plugins/ticketvote/ticketvote.go @@ -18,8 +18,7 @@ type VoteErrorT int type ErrorStatusT int const ( - Version uint32 = 1 - ID = "ticketvote" + ID = "ticketvote" // Plugin commands CmdAuthorize = "authorize" // Authorize a vote diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index 9d4fd9205..36c20bf67 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -148,8 +148,8 @@ type Plugin struct { } // InventoryByStatus contains the record tokens of all records in the inventory -// catagorized by MDStatusT. Each array is sorted by the timestamp of the -// status change from newest to oldest. +// catagorized by MDStatusT. Each list is sorted by the timestamp of the status +// change from newest to oldest. type InventoryByStatus struct { Unvetted []string IterationUnvetted []string @@ -205,11 +205,17 @@ type Backend interface { // inventory catagorized by MDStatusT. InventoryByStatus() (*InventoryByStatus, error) + // Register a plugin with the backend + RegisterPlugin(Plugin) error + + // Perform any plugin setup that is required + SetupPlugin(pluginID string) error + // Obtain plugin settings GetPlugins() ([]Plugin, error) // Plugin pass-through command - Plugin(string, string, string) (string, error) + Plugin(pluginID, cmd, payload string) (string, error) // Close performs cleanup of the backend. Close() diff --git a/politeiad/backend/gitbe/gitbe.go b/politeiad/backend/gitbe/gitbe.go index d302b5429..4b29b305b 100644 --- a/politeiad/backend/gitbe/gitbe.go +++ b/politeiad/backend/gitbe/gitbe.go @@ -2791,6 +2791,20 @@ func (g *gitBackEnd) InventoryByStatus() (*backend.InventoryByStatus, error) { return nil, fmt.Errorf("not implemented") } +// RegisterPlugin registers a plugin. +func (g *gitBackEnd) RegisterPlugin(p backend.Plugin) error { + log.Tracef("RegisterPlugin: %v", p.ID) + + return nil +} + +// SetupPlugin performs any required plugin setup. +func (g *gitBackEnd) SetupPlugin(pluginID string) error { + log.Tracef("SetupPlugin: %v", pluginID) + + return nil +} + // GetPlugins returns a list of currently supported plugins and their settings. // // GetPlugins satisfies the backend interface. diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 5339f5aea..e95fbf43d 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -1302,16 +1302,18 @@ func (t *TlogBackend) InventoryByStatus() (*backend.InventoryByStatus, error) { }, nil } -func (t *TlogBackend) pluginHook(h HookT, payload string) error { - // Pass hook event and payload to each plugin - for _, v := range t.plugins { - err := v.ctx.Hook(h, payload) - if err != nil { - return fmt.Errorf("Hook %v: %v", v.id, err) - } - } +// TODO +func (t *TlogBackend) RegisterPlugin(p backend.Plugin) error { + log.Tracef("RegisterPlugin: %v", p.ID) - return nil + return fmt.Errorf("not implemented") +} + +// TODO +func (t *TlogBackend) SetupPlugin(pluginID string) error { + log.Tracef("SetupPlugin: %v", pluginID) + + return fmt.Errorf("not implemented") } // GetPlugins returns the backend plugins that have been registered and their @@ -1336,8 +1338,8 @@ func (t *TlogBackend) GetPlugins() ([]backend.Plugin, error) { // Plugin is a pass-through function for plugin commands. // // This function satisfies the Backend interface. -func (t *TlogBackend) Plugin(pluginID, command, payload string) (string, error) { - log.Tracef("Plugin: %v", command) +func (t *TlogBackend) Plugin(pluginID, cmd, payload string) (string, error) { + log.Tracef("Plugin: %v %v", pluginID, cmd) if t.isShutdown() { return "", backend.ErrShutdown @@ -1350,7 +1352,7 @@ func (t *TlogBackend) Plugin(pluginID, command, payload string) (string, error) } // Execute plugin command - reply, err := plugin.ctx.Cmd(command, payload) + reply, err := plugin.ctx.Cmd(cmd, payload) if err != nil { return "", err } @@ -1358,6 +1360,18 @@ func (t *TlogBackend) Plugin(pluginID, command, payload string) (string, error) return reply, nil } +func (t *TlogBackend) pluginHook(h HookT, payload string) error { + // Pass hook event and payload to each plugin + for _, v := range t.plugins { + err := v.ctx.Hook(h, payload) + if err != nil { + return fmt.Errorf("Hook %v: %v", v.id, err) + } + } + + return nil +} + // Close shuts the backend down and performs cleanup. // // This function satisfies the Backend interface. diff --git a/politeiad/config.go b/politeiad/config.go index df58168dd..1368f8f08 100644 --- a/politeiad/config.go +++ b/politeiad/config.go @@ -89,12 +89,13 @@ type config struct { DcrdataHost string `long:"dcrdatahost" description:"Dcrdata ip:port"` // TODO validate these config params - Backend string `long:"backend"` - TrillianHostUnvetted string `long:"trillianhostunvetted"` - TrillianHostVetted string `long:"trillianhostvetted"` - TrillianKeyUnvetted string `long:"trilliankeyunvetted"` - TrillianKeyVetted string `long:"trilliankeyvetted"` - EncryptionKey string `long:"encryptionkey"` + Backend string `long:"backend"` + TrillianHostUnvetted string `long:"trillianhostunvetted"` + TrillianHostVetted string `long:"trillianhostvetted"` + TrillianKeyUnvetted string `long:"trilliankeyunvetted"` + TrillianKeyVetted string `long:"trilliankeyvetted"` + EncryptionKey string `long:"encryptionkey"` + Plugins []string `long:"plugins"` } // serviceOptions defines the configuration options for the daemon as a service diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index beea25bfc..50986f61e 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -19,6 +19,12 @@ import ( "syscall" "time" + "github.com/decred/politeia/cmsplugin" + "github.com/decred/politeia/decredplugin" + "github.com/decred/politeia/plugins/comments" + "github.com/decred/politeia/plugins/dcrdata" + "github.com/decred/politeia/plugins/pi" + "github.com/decred/politeia/plugins/ticketvote" v1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" @@ -808,7 +814,7 @@ func (p *politeia) pluginCommand(w http.ResponseWriter, r *http.Request) { return } - payload, err := p.backend.Plugin("", pc.Command, pc.Payload) + payload, err := p.backend.Plugin(pc.ID, pc.Command, pc.Payload) if err != nil { // Generic internal error. errorCode := time.Now().Unix() @@ -823,7 +829,7 @@ func (p *politeia) pluginCommand(w http.ResponseWriter, r *http.Request) { reply := v1.PluginCommandReply{ Response: hex.EncodeToString(response[:]), ID: pc.ID, - Command: pc.CommandID, + Command: pc.Command, CommandID: pc.CommandID, Payload: payload, } @@ -1003,28 +1009,76 @@ func _main() error { p.updateVettedMetadata, permissionAuth) // Setup plugins - plugins, err := p.backend.GetPlugins() - if err != nil { - return err - } - if len(plugins) > 0 { + /* + plugins, err := p.backend.GetPlugins() + if err != nil { + return err + } + if len(plugins) > 0 { + // Set plugin routes. Requires auth. + p.addRoute(http.MethodPost, v1.PluginCommandRoute, p.pluginCommand, + permissionAuth) + p.addRoute(http.MethodPost, v1.PluginInventoryRoute, p.pluginInventory, + permissionAuth) + + for _, v := range plugins { + // make sure we only have lowercase names + if backend.PluginRE.FindString(v.ID) != v.ID { + return fmt.Errorf("invalid plugin id: %v", v.ID) + } + if _, found := p.plugins[v.ID]; found { + return fmt.Errorf("duplicate plugin: %v", v.ID) + } + p.plugins[v.ID] = convertBackendPlugin(v) + + log.Infof("Registered plugin: %v", v.ID) + } + } + */ + + // Setup plugins + if len(loadedCfg.Plugins) > 0 { // Set plugin routes. Requires auth. p.addRoute(http.MethodPost, v1.PluginCommandRoute, p.pluginCommand, permissionAuth) p.addRoute(http.MethodPost, v1.PluginInventoryRoute, p.pluginInventory, permissionAuth) - for _, v := range plugins { - // make sure we only have lowercase names - if backend.PluginRE.FindString(v.ID) != v.ID { - return fmt.Errorf("invalid plugin id: %v", v.ID) + // Register plugins + for _, v := range loadedCfg.Plugins { + var plugin backend.Plugin + switch v { + case comments.ID: + case dcrdata.ID: + case pi.ID: + case ticketvote.ID: + case decredplugin.ID: + // TODO plugin setup for cms + case cmsplugin.ID: + // TODO plugin setup for cms + default: + return fmt.Errorf("unknown plugin '%v'", v) } - if _, found := p.plugins[v.ID]; found { - return fmt.Errorf("duplicate plugin: %v", v.ID) + + // Register with backend + err := p.backend.RegisterPlugin(plugin) + if err != nil { + return fmt.Errorf("RegisterPlugin %v: %v", v, err) } - p.plugins[v.ID] = convertBackendPlugin(v) - log.Infof("Registered plugin: %v", v.ID) + // Add to politeiad context + p.plugins[plugin.ID] = convertBackendPlugin(plugin) + + log.Infof("Registered plugin: %v", v) + } + + // Setup plugins + for _, v := range loadedCfg.Plugins { + log.Infof("Performing plugin setup for %v plugin", v) + err := p.backend.SetupPlugin(v) + if err != nil { + return fmt.Errorf("SetupPlugin %v: %v", v, err) + } } } From 6735d961b1f3a948ccfe03c49696fc8d51e76318 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 15 Sep 2020 10:18:36 -0500 Subject: [PATCH 046/449] get tests to build --- politeiad/backend/gitbe/gitbe_test.go | 10 +- politeiawww/api/www/v1/v1.go | 16 +- politeiawww/piwww.go | 2 +- politeiawww/politeiawww_test.go | 129 -- politeiawww/proposals_test.go | 2080 +++++++++++++------------ politeiawww/testing.go | 43 + 6 files changed, 1107 insertions(+), 1173 deletions(-) diff --git a/politeiad/backend/gitbe/gitbe_test.go b/politeiad/backend/gitbe/gitbe_test.go index 02b69934a..9526d6f35 100644 --- a/politeiad/backend/gitbe/gitbe_test.go +++ b/politeiad/backend/gitbe/gitbe_test.go @@ -86,7 +86,7 @@ func TestAnchorWithCommits(t *testing.T) { // Initialize stuff we need g, err := New(&chaincfg.TestNet3Params, dir, "", "", nil, - testing.Verbose(), "", "piwww") + testing.Verbose(), "") if err != nil { t.Fatal(err) } @@ -152,7 +152,7 @@ func TestAnchorWithCommits(t *testing.T) { if err != nil { t.Fatal(err) } - pru, err := g.GetUnvetted(token) + pru, err := g.GetUnvetted(token, "") if err != nil { t.Fatalf("%v", err) } @@ -475,7 +475,7 @@ func TestUpdateReadme(t *testing.T) { defer os.RemoveAll(dir) g, err := New(&chaincfg.TestNet3Params, dir, "", "", nil, - testing.Verbose(), "", "piwww") + testing.Verbose(), "") if err != nil { t.Fatal(err) } @@ -583,7 +583,7 @@ func TestTokenPrefixGeneration(t *testing.T) { defer os.RemoveAll(dir) g, err := New(&chaincfg.TestNet3Params, dir, "", "", nil, - testing.Verbose(), "", "piwww") + testing.Verbose(), "") if err != nil { t.Fatal(err) } @@ -679,7 +679,7 @@ func TestTokenPrefixGeneration(t *testing.T) { // the prefix cache is populated correctly. oldPrefixCache := g.prefixCache g, err = New(&chaincfg.TestNet3Params, dir, "", "", nil, - testing.Verbose(), "", "piwww") + testing.Verbose(), "") if err != nil { t.Fatal(err) } diff --git a/politeiawww/api/www/v1/v1.go b/politeiawww/api/www/v1/v1.go index d2c07a354..4ed06b3af 100644 --- a/politeiawww/api/www/v1/v1.go +++ b/politeiawww/api/www/v1/v1.go @@ -58,24 +58,24 @@ const ( // The following routes are to be DEPRECATED in the near future and // should not be used. The pi v1 API should be used instead. - RouteActiveVote = "/proposals/activevote" - RouteAllVetted = "/proposals/vetted" + RouteTokenInventory = "/proposals/tokeninventory" + RouteBatchProposals = "/proposals/batch" + RouteBatchVoteSummary = "/proposals/batchvotesummary" + RouteProposalDetails = "/proposals/{token:[A-Fa-f0-9]{7,64}}" + RouteVoteResults = "/proposals/{token:[A-Fa-f0-9]{7,64}}/votes" + RouteCastVotes = "/proposals/castvotes" // The following routes have been DEPRECATED. The pi v1 API should // be used instead. - RouteTokenInventory = "/proposals/tokeninventory" - RouteBatchProposals = "/proposals/batch" - RouteBatchVoteSummary = "/proposals/batchvotesummary" + RouteActiveVote = "/proposals/activevote" + RouteAllVetted = "/proposals/vetted" RouteNewProposal = "/proposals/new" RouteEditProposal = "/proposals/edit" RouteAuthorizeVote = "/proposals/authorizevote" RouteStartVote = "/proposals/startvote" - RouteCastVotes = "/proposals/castvotes" RouteAllVoteStatus = "/proposals/votestatus" - RouteProposalDetails = "/proposals/{token:[A-Fa-f0-9]{7,64}}" RouteSetProposalStatus = "/proposals/{token:[A-Fa-f0-9]{7,64}}/status" RouteCommentsGet = "/proposals/{token:[A-Fa-f0-9]{7,64}}/comments" - RouteVoteResults = "/proposals/{token:[A-Fa-f0-9]{7,64}}/votes" RouteVoteStatus = "/proposals/{token:[A-Fa-f0-9]{7,64}}/votestatus" RouteNewComment = "/comments/new" RouteLikeComment = "/comments/like" diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 8d756bba7..834dc30ca 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -692,7 +692,7 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu // The only text file that is allowed is the index markdown // file. if v.Name != www.PolicyIndexFilename { - e := fmt.Sprint("want %v, got %v", www.PolicyIndexFilename, v.Name) + e := fmt.Sprintf("want %v, got %v", www.PolicyIndexFilename, v.Name) return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusIndexFileNameInvalid, ErrorContext: []string{e}, diff --git a/politeiawww/politeiawww_test.go b/politeiawww/politeiawww_test.go index 26ca6ce8a..831e09502 100644 --- a/politeiawww/politeiawww_test.go +++ b/politeiawww/politeiawww_test.go @@ -79,135 +79,6 @@ func TestHandleVersion(t *testing.T) { } } -func TestHandleAllVetted(t *testing.T) { - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - d := newTestPoliteiad(t, p) - defer d.Close() - - admin, id := newUser(t, p, true, true) - - propPublic := newProposalRecord(t, admin, id, www.PropStatusPublic) - propAbandoned := newProposalRecord(t, admin, id, www.PropStatusAbandoned) - propUnvetted := newProposalRecord(t, admin, id, www.PropStatusNotReviewed) - propCensored := newProposalRecord(t, admin, id, www.PropStatusCensored) - - d.AddRecord(t, convertPropToPD(t, propPublic)) - d.AddRecord(t, convertPropToPD(t, propAbandoned)) - d.AddRecord(t, convertPropToPD(t, propUnvetted)) - d.AddRecord(t, convertPropToPD(t, propCensored)) - - var tests = []struct { - name string - params www.GetAllVetted - badParams bool - wantProps []string - wantStatus int - wantError error - }{ - { - "bad request parameters", - www.GetAllVetted{}, - true, - []string{}, - http.StatusBadRequest, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidInput, - }, - }, - { - "bad data in request parameters", - www.GetAllVetted{ - After: "bad-token", - }, - false, - []string{}, - http.StatusBadRequest, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - }, - }, - { - "success", - www.GetAllVetted{}, - false, - []string{ - propPublic.CensorshipRecord.Token, - propAbandoned.CensorshipRecord.Token, - }, - http.StatusOK, - nil, - }, - } - - for _, v := range tests { - t.Run(v.name, func(t *testing.T) { - // Prepare request and receive reply - r := httptest.NewRequest(http.MethodGet, www.RouteAllVetted, nil) - w := httptest.NewRecorder() - - q := r.URL.Query() - - if v.badParams { - q.Add("bad", "param") - } else { - q.Add("before", v.params.Before) - q.Add("after", v.params.After) - } - - r.URL.RawQuery = q.Encode() - - p.handleAllVetted(w, r) - res := w.Result() - body, _ := ioutil.ReadAll(res.Body) - res.Body.Close() - - var gotReply www.GetAllVettedReply - err := json.Unmarshal(body, &gotReply) - if err != nil { - t.Errorf("unmarshal error with body %v", body) - } - - // Validate http status code - if res.StatusCode != v.wantStatus { - t.Errorf("got status code %v, want %v", - res.StatusCode, v.wantStatus) - } - - // Make sure that correct proposals were received - tokensMap := make(map[string]string) - for _, prop := range gotReply.Proposals { - token := prop.CensorshipRecord.Token - tokensMap[token] = token - } - for _, token := range v.wantProps { - if _, ok := tokensMap[token]; !ok { - t.Errorf("proposal %v not present in reply", token) - } - } - - // Check if request was successful - if res.StatusCode == http.StatusOK { - return - } - - // Receive user error when request fails - var ue www.UserError - err = json.Unmarshal(body, &ue) - if err != nil { - t.Errorf("unmarshal UserError: %v", err) - } - - got := errToStr(ue) - want := errToStr(v.wantError) - if got != want { - t.Errorf("got error %v, want %v", got, want) - } - }) - } -} - func TestHandleProposalDetails(t *testing.T) { p, cleanup := newTestPoliteiawww(t) defer cleanup() diff --git a/politeiawww/proposals_test.go b/politeiawww/proposals_test.go index 4185a2607..fd4a5adcc 100644 --- a/politeiawww/proposals_test.go +++ b/politeiawww/proposals_test.go @@ -14,7 +14,6 @@ import ( "github.com/decred/politeia/decredplugin" pd "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/politeiad/testpoliteiad" www "github.com/decred/politeia/politeiawww/api/www/v1" www2 "github.com/decred/politeia/politeiawww/api/www/v2" "github.com/decred/politeia/politeiawww/user" @@ -92,7 +91,7 @@ func TestIsValidProposalName(t *testing.T) { } for _, test := range tests { - isValid := isValidProposalName(test.name) + isValid := proposalNameIsValid(test.name) if isValid != test.want { t.Errorf("got %v, want %v", isValid, test.want) } @@ -289,419 +288,425 @@ func TestVoteIsApproved(t *testing.T) { } func TestValidateProposalMetadata(t *testing.T) { - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - d := newTestPoliteiad(t, p) - defer d.Close() - - usr, id := newUser(t, p, true, false) - - // Helper variables - no := decredplugin.VoteOptionIDReject - yes := decredplugin.VoteOptionIDApprove - public := www.PropStatusPublic - linkFrom := []string{} - linkBy := time.Now().Add(time.Hour * 24 * 30).Unix() - invalidName := "bad" - validName := "valid name" - invalidToken := "abcdfe" - randomToken, err := util.Random(pd.TokenSize) - if err != nil { - t.Fatal(err) - } - rToken := hex.EncodeToString(randomToken) - - // Public proposal - proposal := newProposalRecord(t, usr, id, public) - token := proposal.CensorshipRecord.Token - d.AddRecord(t, convertPropToPD(t, proposal)) - - // RFP proposal approved - rfpProposal := newProposalRecord(t, usr, id, public) - rfpToken := rfpProposal.CensorshipRecord.Token - makeProposalRFP(t, &rfpProposal, linkFrom, linkBy) - d.AddRecord(t, convertPropToPD(t, rfpProposal)) - - // RFP proposal not approved - rfpProposalNotApproved := newProposalRecord(t, usr, id, public) - rfpTokenNotApproved := rfpProposalNotApproved.CensorshipRecord.Token - makeProposalRFP(t, &rfpProposalNotApproved, linkFrom, linkBy) - d.AddRecord(t, convertPropToPD(t, rfpProposalNotApproved)) - - // RFP bad state - rfpBadState := newProposalRecord(t, usr, id, www.PropStatusNotReviewed) - rfpBadStateToken := rfpBadState.CensorshipRecord.Token - makeProposalRFP(t, &rfpBadState, linkFrom, linkBy) - d.AddRecord(t, convertPropToPD(t, rfpBadState)) - - // Approved VoteSummary for proposal - approved := []www.VoteOptionResult{ - newVoteOptionResult(t, no, "not approve", 1, 2), - newVoteOptionResult(t, yes, "approve", 2, 8), - } - vsApproved := newVoteSummary(t, www.PropVoteStatusFinished, approved) - p.voteSummarySet(rfpToken, vsApproved) - - // Not approved VoteSummary for proposal - notApproved := []www.VoteOptionResult{ - newVoteOptionResult(t, no, "not approve", 1, 8), - newVoteOptionResult(t, yes, "approve", 2, 2), - } - vsNotApproved := newVoteSummary(t, www.PropVoteStatusFinished, notApproved) - p.voteSummarySet(rfpTokenNotApproved, vsNotApproved) - p.voteSummarySet(rfpBadStateToken, vsApproved) - - // Metadatas to validate and test - _, mdInvalidName := newProposalMetadata(t, invalidName, "", 0) - // LinkTo validations - _, mdInvalidLinkTo := newProposalMetadata(t, validName, invalidToken, 0) - _, mdProposalNotFound := newProposalMetadata(t, validName, rToken, 0) - _, mdProposalNotRFP := newProposalMetadata(t, validName, token, 0) - _, mdProposalNotApproved := newProposalMetadata(t, validName, - rfpTokenNotApproved, 0) - _, mdProposalBadState := newProposalMetadata(t, validName, - rfpBadStateToken, 0) - _, mdProposalBothRFP := newProposalMetadata(t, validName, rfpToken, - time.Now().Unix()) - // LinkBy validations - _, mdLinkByMin := newProposalMetadata(t, validName, "", - p.linkByPeriodMin()-1) - _, mdLinkByMax := newProposalMetadata(t, validName, "", - p.linkByPeriodMax()+1) - _, mdSuccess := newProposalMetadata(t, validName, rfpToken, 0) + // TODO + /* + p, cleanup := newTestPoliteiawww(t) + defer cleanup() + + d := newTestPoliteiad(t, p) + defer d.Close() + + usr, id := newUser(t, p, true, false) + + // Helper variables + no := decredplugin.VoteOptionIDReject + yes := decredplugin.VoteOptionIDApprove + public := www.PropStatusPublic + linkFrom := []string{} + linkBy := time.Now().Add(time.Hour * 24 * 30).Unix() + invalidName := "bad" + validName := "valid name" + invalidToken := "abcdfe" + randomToken, err := util.Random(pd.TokenSize) + if err != nil { + t.Fatal(err) + } + rToken := hex.EncodeToString(randomToken) + + // Public proposal + proposal := newProposalRecord(t, usr, id, public) + token := proposal.CensorshipRecord.Token + d.AddRecord(t, convertPropToPD(t, proposal)) + + // RFP proposal approved + rfpProposal := newProposalRecord(t, usr, id, public) + rfpToken := rfpProposal.CensorshipRecord.Token + makeProposalRFP(t, &rfpProposal, linkFrom, linkBy) + d.AddRecord(t, convertPropToPD(t, rfpProposal)) + + // RFP proposal not approved + rfpProposalNotApproved := newProposalRecord(t, usr, id, public) + rfpTokenNotApproved := rfpProposalNotApproved.CensorshipRecord.Token + makeProposalRFP(t, &rfpProposalNotApproved, linkFrom, linkBy) + d.AddRecord(t, convertPropToPD(t, rfpProposalNotApproved)) + + // RFP bad state + rfpBadState := newProposalRecord(t, usr, id, www.PropStatusNotReviewed) + rfpBadStateToken := rfpBadState.CensorshipRecord.Token + makeProposalRFP(t, &rfpBadState, linkFrom, linkBy) + d.AddRecord(t, convertPropToPD(t, rfpBadState)) + + // Approved VoteSummary for proposal + approved := []www.VoteOptionResult{ + newVoteOptionResult(t, no, "not approve", 1, 2), + newVoteOptionResult(t, yes, "approve", 2, 8), + } + vsApproved := newVoteSummary(t, www.PropVoteStatusFinished, approved) + p.voteSummarySet(rfpToken, vsApproved) - var tests = []struct { - name string - metadata www.ProposalMetadata - want error - }{ - { - "invalid proposal name", - mdInvalidName, - www.UserError{ - ErrorCode: www.ErrorStatusProposalInvalidTitle, + // Not approved VoteSummary for proposal + notApproved := []www.VoteOptionResult{ + newVoteOptionResult(t, no, "not approve", 1, 8), + newVoteOptionResult(t, yes, "approve", 2, 2), + } + vsNotApproved := newVoteSummary(t, www.PropVoteStatusFinished, notApproved) + p.voteSummarySet(rfpTokenNotApproved, vsNotApproved) + p.voteSummarySet(rfpBadStateToken, vsApproved) + + // Metadatas to validate and test + _, mdInvalidName := newProposalMetadata(t, invalidName, "", 0) + // LinkTo validations + _, mdInvalidLinkTo := newProposalMetadata(t, validName, invalidToken, 0) + _, mdProposalNotFound := newProposalMetadata(t, validName, rToken, 0) + _, mdProposalNotRFP := newProposalMetadata(t, validName, token, 0) + _, mdProposalNotApproved := newProposalMetadata(t, validName, + rfpTokenNotApproved, 0) + _, mdProposalBadState := newProposalMetadata(t, validName, + rfpBadStateToken, 0) + _, mdProposalBothRFP := newProposalMetadata(t, validName, rfpToken, + time.Now().Unix()) + // LinkBy validations + _, mdLinkByMin := newProposalMetadata(t, validName, "", + p.linkByPeriodMin()-1) + _, mdLinkByMax := newProposalMetadata(t, validName, "", + p.linkByPeriodMax()+1) + _, mdSuccess := newProposalMetadata(t, validName, rfpToken, 0) + + var tests = []struct { + name string + metadata www.ProposalMetadata + want error + }{ + { + "invalid proposal name", + mdInvalidName, + www.UserError{ + ErrorCode: www.ErrorStatusProposalInvalidTitle, + }, }, - }, - { - "invalid linkTo bad token", - mdInvalidLinkTo, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkTo, + { + "invalid linkTo bad token", + mdInvalidLinkTo, + www.UserError{ + ErrorCode: www.ErrorStatusInvalidLinkTo, + }, }, - }, - { - "invalid linkTo token proposal not found", - mdProposalNotFound, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkTo, + { + "invalid linkTo token proposal not found", + mdProposalNotFound, + www.UserError{ + ErrorCode: www.ErrorStatusInvalidLinkTo, + }, }, - }, - { - "invalid linkTo not a RFP proposal", - mdProposalNotRFP, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkTo, + { + "invalid linkTo not a RFP proposal", + mdProposalNotRFP, + www.UserError{ + ErrorCode: www.ErrorStatusInvalidLinkTo, + }, }, - }, - { - "invalid linkTo RFP proposal vote not approved", - mdProposalNotApproved, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkTo, + { + "invalid linkTo RFP proposal vote not approved", + mdProposalNotApproved, + www.UserError{ + ErrorCode: www.ErrorStatusInvalidLinkTo, + }, }, - }, - { - "invalid linkTo proposal is not vetted", - mdProposalBadState, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkTo, + { + "invalid linkTo proposal is not vetted", + mdProposalBadState, + www.UserError{ + ErrorCode: www.ErrorStatusInvalidLinkTo, + }, }, - }, - { - "invalid linkTo rfp proposal linked to another rfp", - mdProposalBothRFP, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkTo, + { + "invalid linkTo rfp proposal linked to another rfp", + mdProposalBothRFP, + www.UserError{ + ErrorCode: www.ErrorStatusInvalidLinkTo, + }, }, - }, - { - "invalid linkBy shorter than min", - mdLinkByMin, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkBy, + { + "invalid linkBy shorter than min", + mdLinkByMin, + www.UserError{ + ErrorCode: www.ErrorStatusInvalidLinkBy, + }, }, - }, - { - "invalid linkBy greather than max", - mdLinkByMax, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkBy, + { + "invalid linkBy greather than max", + mdLinkByMax, + www.UserError{ + ErrorCode: www.ErrorStatusInvalidLinkBy, + }, }, - }, - { - "validation success", - mdSuccess, - nil, - }, - } + { + "validation success", + mdSuccess, + nil, + }, + } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - err := p.validateProposalMetadata(test.metadata) - got := errToStr(err) - want := errToStr(test.want) - if got != want { - t.Errorf("got %v, want %v", got, want) - } - }) - } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := p.validateProposalMetadata(test.metadata) + got := errToStr(err) + want := errToStr(test.want) + if got != want { + t.Errorf("got %v, want %v", got, want) + } + }) + } + */ } func TestValidateProposal(t *testing.T) { - // Setup politeiawww and a test user - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - usr, id := newUser(t, p, true, false) - - // Create test data - md := createFileMD(t, 8) - png := createFilePNG(t, false) - np := createNewProposal(t, id, []www.File{*md, *png}, "") - - // Invalid signature - propInvalidSig := createNewProposal(t, id, []www.File{*md}, "") - propInvalidSig.Signature = "abc" - - // Signature is valid but incorrect - propBadSig := createNewProposal(t, id, []www.File{*md}, "") - propBadSig.Signature = np.Signature - - // No files - propNoFiles := createNewProposal(t, id, []www.File{}, "") - - // Invalid markdown filename - mdBadFilename := *md - mdBadFilename.Name = "bad_filename.md" - propBadFilename := createNewProposal(t, id, []www.File{mdBadFilename}, "") - - // Duplicate filenames - propDupFiles := createNewProposal(t, id, []www.File{*md, *png, *png}, "") - - // Too many markdown files. We need one correctly named md - // file and the rest must have their names changed so that we - // don't get a duplicate filename error. - files := make([]www.File, 0, www.PolicyMaxMDs+1) - files = append(files, *md) - for i := 0; i < www.PolicyMaxMDs; i++ { - m := *md - m.Name = fmt.Sprintf("%v.md", i) - files = append(files, m) - } - propMaxMDFiles := createNewProposal(t, id, files, "") - - // Too many image files. All of their names must be different - // so that we don't get a duplicate filename error. - files = make([]www.File, 0, www.PolicyMaxImages+2) - files = append(files, *md) - for i := 0; i <= www.PolicyMaxImages; i++ { - p := *png - p.Name = fmt.Sprintf("%v.png", i) - files = append(files, p) - } - propMaxImages := createNewProposal(t, id, files, "") - - // Markdown file too large - mdLarge := createFileMD(t, www.PolicyMaxMDSize) - propMDLarge := createNewProposal(t, id, []www.File{*mdLarge, *png}, "") - - // Image too large - pngLarge := createFilePNG(t, true) - propImageLarge := createNewProposal(t, id, []www.File{*md, *pngLarge}, "") - - // Invalid proposal title - mdBadTitle := createFileMD(t, 8) - propBadTitle := createNewProposal(t, id, - []www.File{*mdBadTitle}, "{invalid-title}") - - // Empty file payload - propEmptyFile := createNewProposal(t, id, []www.File{*md}, "") - emptyFile := createFileMD(t, 8) - emptyFile.Payload = "" - propEmptyFile.Files = []www.File{*emptyFile} - - // Invalid file digest - propInvalidDigest := createNewProposal(t, id, []www.File{*md}, "") - invalidDigestFile := createFilePNG(t, false) - invalidDigestFile.Digest = "" - propInvalidDigest.Files = append(propInvalidDigest.Files, *invalidDigestFile) - - // Invalid MIME type - propInvalidMIME := createNewProposal(t, id, []www.File{*md}, "") - invalidMIMEFile := createFilePNG(t, false) - invalidMIMEFile.MIME = "image/jpeg" - propInvalidMIME.Files = append(propInvalidMIME.Files, *invalidMIMEFile) - - // Invalid metadata - propInvalidMetadata := createNewProposal(t, id, []www.File{*md}, "") - invalidMd := www.Metadata{ - Digest: "", - Payload: "", - Hint: www.HintProposalMetadata, - } - propInvalidMetadata.Metadata = append(propInvalidMetadata.Metadata, invalidMd) - - // Missing metadata - propMissingMetadata := createNewProposal(t, id, []www.File{*md}, "") - propMissingMetadata.Metadata = nil - - // Setup test cases - var tests = []struct { - name string - newProposal www.NewProposal - user *user.User - want error - }{ - { - "correct proposal", - *np, - usr, - nil, - }, - { - "invalid signature", - *propInvalidSig, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidSignature, + // TODO + /* + // Setup politeiawww and a test user + p, cleanup := newTestPoliteiawww(t) + defer cleanup() + + usr, id := newUser(t, p, true, false) + + // Create test data + md := createFileMD(t, 8) + png := createFilePNG(t, false) + np := createNewProposal(t, id, []www.File{*md, *png}, "") + + // Invalid signature + propInvalidSig := createNewProposal(t, id, []www.File{*md}, "") + propInvalidSig.Signature = "abc" + + // Signature is valid but incorrect + propBadSig := createNewProposal(t, id, []www.File{*md}, "") + propBadSig.Signature = np.Signature + + // No files + propNoFiles := createNewProposal(t, id, []www.File{}, "") + + // Invalid markdown filename + mdBadFilename := *md + mdBadFilename.Name = "bad_filename.md" + propBadFilename := createNewProposal(t, id, []www.File{mdBadFilename}, "") + + // Duplicate filenames + propDupFiles := createNewProposal(t, id, []www.File{*md, *png, *png}, "") + + // Too many markdown files. We need one correctly named md + // file and the rest must have their names changed so that we + // don't get a duplicate filename error. + files := make([]www.File, 0, www.PolicyMaxMDs+1) + files = append(files, *md) + for i := 0; i < www.PolicyMaxMDs; i++ { + m := *md + m.Name = fmt.Sprintf("%v.md", i) + files = append(files, m) + } + propMaxMDFiles := createNewProposal(t, id, files, "") + + // Too many image files. All of their names must be different + // so that we don't get a duplicate filename error. + files = make([]www.File, 0, www.PolicyMaxImages+2) + files = append(files, *md) + for i := 0; i <= www.PolicyMaxImages; i++ { + p := *png + p.Name = fmt.Sprintf("%v.png", i) + files = append(files, p) + } + propMaxImages := createNewProposal(t, id, files, "") + + // Markdown file too large + mdLarge := createFileMD(t, www.PolicyMaxMDSize) + propMDLarge := createNewProposal(t, id, []www.File{*mdLarge, *png}, "") + + // Image too large + pngLarge := createFilePNG(t, true) + propImageLarge := createNewProposal(t, id, []www.File{*md, *pngLarge}, "") + + // Invalid proposal title + mdBadTitle := createFileMD(t, 8) + propBadTitle := createNewProposal(t, id, + []www.File{*mdBadTitle}, "{invalid-title}") + + // Empty file payload + propEmptyFile := createNewProposal(t, id, []www.File{*md}, "") + emptyFile := createFileMD(t, 8) + emptyFile.Payload = "" + propEmptyFile.Files = []www.File{*emptyFile} + + // Invalid file digest + propInvalidDigest := createNewProposal(t, id, []www.File{*md}, "") + invalidDigestFile := createFilePNG(t, false) + invalidDigestFile.Digest = "" + propInvalidDigest.Files = append(propInvalidDigest.Files, *invalidDigestFile) + + // Invalid MIME type + propInvalidMIME := createNewProposal(t, id, []www.File{*md}, "") + invalidMIMEFile := createFilePNG(t, false) + invalidMIMEFile.MIME = "image/jpeg" + propInvalidMIME.Files = append(propInvalidMIME.Files, *invalidMIMEFile) + + // Invalid metadata + propInvalidMetadata := createNewProposal(t, id, []www.File{*md}, "") + invalidMd := www.Metadata{ + Digest: "", + Payload: "", + Hint: www.HintProposalMetadata, + } + propInvalidMetadata.Metadata = append(propInvalidMetadata.Metadata, invalidMd) + + // Missing metadata + propMissingMetadata := createNewProposal(t, id, []www.File{*md}, "") + propMissingMetadata.Metadata = nil + + // Setup test cases + var tests = []struct { + name string + newProposal www.NewProposal + user *user.User + want error + }{ + { + "correct proposal", + *np, + usr, + nil, + }, + { + "invalid signature", + *propInvalidSig, + usr, + www.UserError{ + ErrorCode: www.ErrorStatusInvalidSignature, + }, }, - }, - { - "incorrect signature", - *propBadSig, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidSignature, + { + "incorrect signature", + *propBadSig, + usr, + www.UserError{ + ErrorCode: www.ErrorStatusInvalidSignature, + }, }, - }, - { - "missing files", - *propNoFiles, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusProposalMissingFiles, + { + "missing files", + *propNoFiles, + usr, + www.UserError{ + ErrorCode: www.ErrorStatusProposalMissingFiles, + }, }, - }, - { - "bad md filename", - *propBadFilename, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusMaxMDsExceededPolicy, + { + "bad md filename", + *propBadFilename, + usr, + www.UserError{ + ErrorCode: www.ErrorStatusMaxMDsExceededPolicy, + }, }, - }, - { - "duplicate filenames", - *propDupFiles, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusProposalDuplicateFilenames, + { + "duplicate filenames", + *propDupFiles, + usr, + www.UserError{ + ErrorCode: www.ErrorStatusProposalDuplicateFilenames, + }, }, - }, - { - "empty file payload", - *propEmptyFile, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidBase64, + { + "empty file payload", + *propEmptyFile, + usr, + www.UserError{ + ErrorCode: www.ErrorStatusInvalidBase64, + }, }, - }, - { - "invalid file digest", - *propInvalidDigest, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidFileDigest, + { + "invalid file digest", + *propInvalidDigest, + usr, + www.UserError{ + ErrorCode: www.ErrorStatusInvalidFileDigest, + }, }, - }, - { - "invalid file MIME type", - *propInvalidMIME, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidMIMEType, + { + "invalid file MIME type", + *propInvalidMIME, + usr, + www.UserError{ + ErrorCode: www.ErrorStatusInvalidMIMEType, + }, }, - }, - { - "invalid metadata", - *propInvalidMetadata, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusMetadataInvalid, + { + "invalid metadata", + *propInvalidMetadata, + usr, + www.UserError{ + ErrorCode: www.ErrorStatusMetadataInvalid, + }, }, - }, - { - "missing metadata", - *propMissingMetadata, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusMetadataMissing, + { + "missing metadata", + *propMissingMetadata, + usr, + www.UserError{ + ErrorCode: www.ErrorStatusMetadataMissing, + }, }, - }, - { - "too may md files", - *propMaxMDFiles, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusMaxMDsExceededPolicy, + { + "too may md files", + *propMaxMDFiles, + usr, + www.UserError{ + ErrorCode: www.ErrorStatusMaxMDsExceededPolicy, + }, }, - }, - { - "too many images", - *propMaxImages, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusMaxImagesExceededPolicy, + { + "too many images", + *propMaxImages, + usr, + www.UserError{ + ErrorCode: www.ErrorStatusMaxImagesExceededPolicy, + }, }, - }, - { - "md file too large", - *propMDLarge, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusMaxMDSizeExceededPolicy, + { + "md file too large", + *propMDLarge, + usr, + www.UserError{ + ErrorCode: www.ErrorStatusMaxMDSizeExceededPolicy, + }, }, - }, - { - "image too large", - *propImageLarge, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusMaxImageSizeExceededPolicy, + { + "image too large", + *propImageLarge, + usr, + www.UserError{ + ErrorCode: www.ErrorStatusMaxImageSizeExceededPolicy, + }, }, - }, - { - "invalid title", - *propBadTitle, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusProposalInvalidTitle, + { + "invalid title", + *propBadTitle, + usr, + www.UserError{ + ErrorCode: www.ErrorStatusProposalInvalidTitle, + }, }, - }, - } + } - // Run test cases - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - _, err := p.validateProposal(test.newProposal, test.user) - got := errToStr(err) - want := errToStr(test.want) - if got != want { - t.Errorf("got %v, want %v", got, want) - } - }) - } + // Run test cases + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := p.validateProposal(test.newProposal, test.user) + got := errToStr(err) + want := errToStr(test.want) + if got != want { + t.Errorf("got %v, want %v", got, want) + } + }) + } + */ } func TestValidateAuthorizeVote(t *testing.T) { @@ -1727,544 +1732,553 @@ func TestFilterProposals(t *testing.T) { } func TestProcessNewProposal(t *testing.T) { - // Setup test environment - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - td := testpoliteiad.New(t, p.cache) - defer td.Close() - - p.cfg.RPCHost = td.URL - p.cfg.Identity = td.PublicIdentity - - // Create a user that has not paid their registration fee. - usrUnpaid, _ := newUser(t, p, true, false) + // TODO + /* + // Setup test environment + p, cleanup := newTestPoliteiawww(t) + defer cleanup() + + td := testpoliteiad.New(t) + defer td.Close() + + p.cfg.RPCHost = td.URL + p.cfg.Identity = td.PublicIdentity + + // Create a user that has not paid their registration fee. + usrUnpaid, _ := newUser(t, p, true, false) + + // Create a user that has paid their registration + // fee but does not have any proposal credits. + usrNoCredits, _ := newUser(t, p, true, false) + payRegistrationFee(t, p, usrNoCredits) + + // Create a user that has paid their registration + // fee and has purchased proposal credits. + usr, id := newUser(t, p, true, false) + payRegistrationFee(t, p, usr) + addProposalCredits(t, p, usr, 10) + + // Create a new proposal + f := newFileRandomMD(t) + np := createNewProposal(t, id, []www.File{f}, "") + + // Invalid proposal + propInvalid := createNewProposal(t, id, []www.File{f}, "") + propInvalid.Signature = "" + + // Expired deadline RFP proposal + rfpProp := newProposalRecord(t, usr, id, www.PropStatusPublic) + rfpToken := rfpProp.CensorshipRecord.Token + makeProposalRFP(t, &rfpProp, []string{}, time.Now().Unix()-p.linkByPeriodMin()) + td.AddRecord(t, convertPropToPD(t, rfpProp)) + + // Set vote summary for expired deadline proposal + no := decredplugin.VoteOptionIDReject + yes := decredplugin.VoteOptionIDApprove + approved := []www.VoteOptionResult{ + newVoteOptionResult(t, no, "not approve", 1, 2), + newVoteOptionResult(t, yes, "approve", 2, 8), + } + vsApproved := newVoteSummary(t, www.PropVoteStatusFinished, approved) + p.voteSummarySet(rfpToken, vsApproved) + + // Set expired deadline rfp metadata and signature + propMetadata, _ := newProposalMetadata(t, "valid name", rfpToken, 0) + propExpired := createNewProposal(t, id, []www.File{f}, "") + propExpired.Metadata = propMetadata + root := merkleRoot(t, propExpired.Files, propExpired.Metadata) + s := id.SignMessage([]byte(root)) + sig := hex.EncodeToString(s[:]) + propExpired.Signature = sig + + // Setup tests + var tests = []struct { + name string + np *www.NewProposal + usr *user.User + want error + }{ + { + "unpaid registration fee", + np, + usrUnpaid, + www.UserError{ + ErrorCode: www.ErrorStatusUserNotPaid, + }, + }, + { + "no proposal credits", + np, + usrNoCredits, + www.UserError{ + ErrorCode: www.ErrorStatusNoProposalCredits, + }, + }, + { + "invalid proposal", + propInvalid, + usr, + www.UserError{ + ErrorCode: www.ErrorStatusInvalidSignature, + }, + }, + { + "linkby deadline expired", + propExpired, + usr, + www.UserError{ + ErrorCode: www.ErrorStatusInvalidLinkTo, + }, + }, + { + "success", + np, + usr, + nil, + }, + } - // Create a user that has paid their registration - // fee but does not have any proposal credits. - usrNoCredits, _ := newUser(t, p, true, false) - payRegistrationFee(t, p, usrNoCredits) + // Run tests + for _, v := range tests { + t.Run(v.name, func(t *testing.T) { + npr, err := p.processNewProposal(*v.np, v.usr) + got := errToStr(err) + want := errToStr(v.want) - // Create a user that has paid their registration - // fee and has purchased proposal credits. - usr, id := newUser(t, p, true, false) - payRegistrationFee(t, p, usr) - addProposalCredits(t, p, usr, 10) + if got != want { + t.Errorf("got error %v, want %v", + got, want) + } - // Create a new proposal - f := newFileRandomMD(t) - np := createNewProposal(t, id, []www.File{f}, "") + if v.want != nil { + // Test case passes + return + } - // Invalid proposal - propInvalid := createNewProposal(t, id, []www.File{f}, "") - propInvalid.Signature = "" + // Validate success case + if npr == nil { + t.Errorf("NewProposalReply is nil") + } - // Expired deadline RFP proposal - rfpProp := newProposalRecord(t, usr, id, www.PropStatusPublic) - rfpToken := rfpProp.CensorshipRecord.Token - makeProposalRFP(t, &rfpProp, []string{}, time.Now().Unix()-p.linkByPeriodMin()) - td.AddRecord(t, convertPropToPD(t, rfpProp)) + // Ensure a proposal credit has been deducted + // from the user's account. + u, err := p.db.UserGetById(v.usr.ID) + if err != nil { + t.Error(err) + } - // Set vote summary for expired deadline proposal - no := decredplugin.VoteOptionIDReject - yes := decredplugin.VoteOptionIDApprove - approved := []www.VoteOptionResult{ - newVoteOptionResult(t, no, "not approve", 1, 2), - newVoteOptionResult(t, yes, "approve", 2, 8), - } - vsApproved := newVoteSummary(t, www.PropVoteStatusFinished, approved) - p.voteSummarySet(rfpToken, vsApproved) - - // Set expired deadline rfp metadata and signature - propMetadata, _ := newProposalMetadata(t, "valid name", rfpToken, 0) - propExpired := createNewProposal(t, id, []www.File{f}, "") - propExpired.Metadata = propMetadata - root := merkleRoot(t, propExpired.Files, propExpired.Metadata) - s := id.SignMessage([]byte(root)) - sig := hex.EncodeToString(s[:]) - propExpired.Signature = sig + gotCredits := len(u.UnspentProposalCredits) + wantCredits := len(v.usr.UnspentProposalCredits) + if gotCredits != wantCredits { + t.Errorf("got num proposal credits %v, want %v", + gotCredits, wantCredits) + } + }) + } + */ +} - // Setup tests - var tests = []struct { - name string - np *www.NewProposal - usr *user.User - want error - }{ - { - "unpaid registration fee", - np, - usrUnpaid, - www.UserError{ - ErrorCode: www.ErrorStatusUserNotPaid, +func TestProcessEditProposal(t *testing.T) { + // TODO + /* + p, cleanup := newTestPoliteiawww(t) + defer cleanup() + + d := newTestPoliteiad(t, p) + defer d.Close() + + usr, id := newUser(t, p, true, false) + notAuthorUser, _ := newUser(t, p, true, false) + + // Public proposal to be edited + propPublic := newProposalRecord(t, usr, id, www.PropStatusPublic) + tokenPropPublic := propPublic.CensorshipRecord.Token + d.AddRecord(t, convertPropToPD(t, propPublic)) + + root := merkleRoot(t, propPublic.Files, propPublic.Metadata) + s := id.SignMessage([]byte(root)) + sigPropPublic := hex.EncodeToString(s[:]) + + // Edited public proposal + newMD := newFileRandomMD(t) + png := createFilePNG(t, false) + newFiles := []www.File{newMD, *png} + + root = merkleRoot(t, newFiles, propPublic.Metadata) + s = id.SignMessage([]byte(root)) + sigPropPublicEdited := hex.EncodeToString(s[:]) + + // Censored proposal to test error case + propCensored := newProposalRecord(t, usr, id, www.PropStatusCensored) + tokenPropCensored := propCensored.CensorshipRecord.Token + d.AddRecord(t, convertPropToPD(t, propCensored)) + + // Authorized vote proposal to test error case + propVoteAuthorized := newProposalRecord(t, usr, id, www.PropStatusPublic) + tokenVoteAuthorized := propVoteAuthorized.CensorshipRecord.Token + d.AddRecord(t, convertPropToPD(t, propVoteAuthorized)) + + cmd := newAuthorizeVoteCmd(t, tokenVoteAuthorized, + propVoteAuthorized.Version, decredplugin.AuthVoteActionAuthorize, id) + d.Plugin(t, cmd) + + var tests = []struct { + name string + user *user.User + editProp www.EditProposal + wantError error + }{ + { + "invalid proposal token", + usr, + www.EditProposal{ + Token: "invalid-token", + }, + www.UserError{ + ErrorCode: www.ErrorStatusInvalidCensorshipToken, + }, }, - }, - { - "no proposal credits", - np, - usrNoCredits, - www.UserError{ - ErrorCode: www.ErrorStatusNoProposalCredits, - }, - }, - { - "invalid proposal", - propInvalid, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidSignature, - }, - }, - { - "linkby deadline expired", - propExpired, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkTo, - }, - }, - { - "success", - np, - usr, - nil, - }, - } - - // Run tests - for _, v := range tests { - t.Run(v.name, func(t *testing.T) { - npr, err := p.processNewProposal(*v.np, v.usr) - got := errToStr(err) - want := errToStr(v.want) - - if got != want { - t.Errorf("got error %v, want %v", - got, want) - } - - if v.want != nil { - // Test case passes - return - } - - // Validate success case - if npr == nil { - t.Errorf("NewProposalReply is nil") - } - - // Ensure a proposal credit has been deducted - // from the user's account. - u, err := p.db.UserGetById(v.usr.ID) - if err != nil { - t.Error(err) - } - - gotCredits := len(u.UnspentProposalCredits) - wantCredits := len(v.usr.UnspentProposalCredits) - if gotCredits != wantCredits { - t.Errorf("got num proposal credits %v, want %v", - gotCredits, wantCredits) - } - }) - } -} - -func TestProcessEditProposal(t *testing.T) { - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - d := newTestPoliteiad(t, p) - defer d.Close() - - usr, id := newUser(t, p, true, false) - notAuthorUser, _ := newUser(t, p, true, false) - - // Public proposal to be edited - propPublic := newProposalRecord(t, usr, id, www.PropStatusPublic) - tokenPropPublic := propPublic.CensorshipRecord.Token - d.AddRecord(t, convertPropToPD(t, propPublic)) - - root := merkleRoot(t, propPublic.Files, propPublic.Metadata) - s := id.SignMessage([]byte(root)) - sigPropPublic := hex.EncodeToString(s[:]) - - // Edited public proposal - newMD := newFileRandomMD(t) - png := createFilePNG(t, false) - newFiles := []www.File{newMD, *png} - - root = merkleRoot(t, newFiles, propPublic.Metadata) - s = id.SignMessage([]byte(root)) - sigPropPublicEdited := hex.EncodeToString(s[:]) - - // Censored proposal to test error case - propCensored := newProposalRecord(t, usr, id, www.PropStatusCensored) - tokenPropCensored := propCensored.CensorshipRecord.Token - d.AddRecord(t, convertPropToPD(t, propCensored)) - - // Authorized vote proposal to test error case - propVoteAuthorized := newProposalRecord(t, usr, id, www.PropStatusPublic) - tokenVoteAuthorized := propVoteAuthorized.CensorshipRecord.Token - d.AddRecord(t, convertPropToPD(t, propVoteAuthorized)) - - cmd := newAuthorizeVoteCmd(t, tokenVoteAuthorized, - propVoteAuthorized.Version, decredplugin.AuthVoteActionAuthorize, id) - d.Plugin(t, cmd) - - var tests = []struct { - name string - user *user.User - editProp www.EditProposal - wantError error - }{ - { - "invalid proposal token", - usr, - www.EditProposal{ - Token: "invalid-token", - }, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - }, - }, - { - "wrong proposal status", - usr, - www.EditProposal{ - Token: tokenPropCensored, - }, - www.UserError{ - ErrorCode: www.ErrorStatusWrongStatus, - }, - }, - { - "user is not the author", - notAuthorUser, - www.EditProposal{ - Token: tokenPropPublic, - }, - www.UserError{ - ErrorCode: www.ErrorStatusUserNotAuthor, - }, - }, - { - "wrong proposal vote status", - usr, - www.EditProposal{ - Token: tokenVoteAuthorized, + { + "wrong proposal status", + usr, + www.EditProposal{ + Token: tokenPropCensored, + }, + www.UserError{ + ErrorCode: www.ErrorStatusWrongStatus, + }, }, - www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, + { + "user is not the author", + notAuthorUser, + www.EditProposal{ + Token: tokenPropPublic, + }, + www.UserError{ + ErrorCode: www.ErrorStatusUserNotAuthor, + }, }, - }, - { - "no changes in proposal md file", - usr, - www.EditProposal{ - Token: tokenPropPublic, - Files: propPublic.Files, - Metadata: propPublic.Metadata, - PublicKey: usr.PublicKey(), - Signature: sigPropPublic, + { + "wrong proposal vote status", + usr, + www.EditProposal{ + Token: tokenVoteAuthorized, + }, + www.UserError{ + ErrorCode: www.ErrorStatusWrongVoteStatus, + }, }, - www.UserError{ - ErrorCode: www.ErrorStatusNoProposalChanges, + { + "no changes in proposal md file", + usr, + www.EditProposal{ + Token: tokenPropPublic, + Files: propPublic.Files, + Metadata: propPublic.Metadata, + PublicKey: usr.PublicKey(), + Signature: sigPropPublic, + }, + www.UserError{ + ErrorCode: www.ErrorStatusNoProposalChanges, + }, }, - }, - { - "success", - usr, - www.EditProposal{ - Token: tokenPropPublic, - Files: newFiles, - Metadata: propPublic.Metadata, - PublicKey: usr.PublicKey(), - Signature: sigPropPublicEdited, + { + "success", + usr, + www.EditProposal{ + Token: tokenPropPublic, + Files: newFiles, + Metadata: propPublic.Metadata, + PublicKey: usr.PublicKey(), + Signature: sigPropPublicEdited, + }, + nil, }, - nil, - }, - } + } - for _, v := range tests { - t.Run(v.name, func(t *testing.T) { - _, err := p.processEditProposal(v.editProp, v.user) - got := errToStr(err) - want := errToStr(v.wantError) + for _, v := range tests { + t.Run(v.name, func(t *testing.T) { + _, err := p.processEditProposal(v.editProp, v.user) + got := errToStr(err) + want := errToStr(v.wantError) - // Test if we got expected error - if got != want { - t.Errorf("got error %v, want %v", got, want) - } - }) - } + // Test if we got expected error + if got != want { + t.Errorf("got error %v, want %v", got, want) + } + }) + } + */ } func TestProcessSetProposalStatus(t *testing.T) { - // Setup test environment - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - d := newTestPoliteiad(t, p) - defer d.Close() - - // Create test data - admin, id := newUser(t, p, true, true) - - changeMsgCensored := "proposal did not meet guidelines" - changeMsgAbandoned := "no activity" - - statusPublic := strconv.Itoa(int(www.PropStatusPublic)) - statusCensored := strconv.Itoa(int(www.PropStatusCensored)) - statusAbandoned := strconv.Itoa(int(www.PropStatusAbandoned)) - - propNotReviewed := newProposalRecord(t, admin, id, www.PropStatusNotReviewed) - propPublic := newProposalRecord(t, admin, id, www.PropStatusPublic) - - tokenNotReviewed := propNotReviewed.CensorshipRecord.Token - tokenPublic := propPublic.CensorshipRecord.Token - tokenNotFound := "abc" - - msg := fmt.Sprintf("%s%s", tokenNotReviewed, statusPublic) - s := id.SignMessage([]byte(msg)) - sigNotReviewedToPublic := hex.EncodeToString(s[:]) - - msg = fmt.Sprintf("%s%s%s", tokenNotReviewed, - statusCensored, changeMsgCensored) - s = id.SignMessage([]byte(msg)) - sigNotReviewedToCensored := hex.EncodeToString(s[:]) - - msg = fmt.Sprintf("%s%s%s", tokenPublic, - statusAbandoned, changeMsgAbandoned) - s = id.SignMessage([]byte(msg)) - sigPublicToAbandoned := hex.EncodeToString(s[:]) - - msg = fmt.Sprintf("%s%s", tokenNotFound, statusPublic) - s = id.SignMessage([]byte(msg)) - sigNotFound := hex.EncodeToString(s[:]) - - msg = fmt.Sprintf("%s%s%s", tokenNotReviewed, - statusAbandoned, changeMsgAbandoned) - s = id.SignMessage([]byte(msg)) - sigNotReviewedToAbandoned := hex.EncodeToString(s[:]) - - // Add success case proposals to politeiad - d.AddRecord(t, convertPropToPD(t, propNotReviewed)) - d.AddRecord(t, convertPropToPD(t, propPublic)) - - // Create a proposal whose vote has been authorized - propVoteAuthorized := newProposalRecord(t, admin, id, www.PropStatusPublic) - tokenVoteAuthorized := propVoteAuthorized.CensorshipRecord.Token - - msg = fmt.Sprintf("%s%s%s", tokenVoteAuthorized, - statusAbandoned, changeMsgAbandoned) - s = id.SignMessage([]byte(msg)) - sigVoteAuthorizedToAbandoned := hex.EncodeToString(s[:]) - - d.AddRecord(t, convertPropToPD(t, propVoteAuthorized)) - cmd := newAuthorizeVoteCmd(t, tokenVoteAuthorized, - propVoteAuthorized.Version, decredplugin.AuthVoteActionAuthorize, id) - d.Plugin(t, cmd) - - // Create a proposal whose voting period has started - propVoteStarted := newProposalRecord(t, admin, id, www.PropStatusPublic) - tokenVoteStarted := propVoteStarted.CensorshipRecord.Token - - msg = fmt.Sprintf("%s%s%s", tokenVoteStarted, - statusAbandoned, changeMsgAbandoned) - s = id.SignMessage([]byte(msg)) - sigVoteStartedToAbandoned := hex.EncodeToString(s[:]) - - d.AddRecord(t, convertPropToPD(t, propVoteStarted)) - cmd = newStartVoteCmd(t, tokenVoteStarted, 1, 2016, id) - d.Plugin(t, cmd) - - // Ensure that admins are not allowed to change the status of - // their own proposal on mainnet. This is run individually - // because it requires flipping the testnet config setting. - t.Run("admin is author", func(t *testing.T) { - p.cfg.TestNet = false - defer func() { - p.cfg.TestNet = true - }() - - sps := www.SetProposalStatus{ - Token: tokenNotReviewed, - ProposalStatus: www.PropStatusPublic, - Signature: sigNotReviewedToPublic, - PublicKey: admin.PublicKey(), - } - - _, err := p.processSetProposalStatus(sps, admin) - got := errToStr(err) - want := www.ErrorStatus[www.ErrorStatusReviewerAdminEqualsAuthor] - if got != want { - t.Errorf("got error %v, want %v", - got, want) - } - }) - - // Setup tests - var tests = []struct { - name string - usr *user.User - sps www.SetProposalStatus - want error - }{ - // This is an admin route so it can be assumed that the - // user has been validated and is an admin. - - { - "no change message for censored", - admin, - www.SetProposalStatus{ - Token: tokenNotReviewed, - ProposalStatus: www.PropStatusCensored, - Signature: sigNotReviewedToCensored, - PublicKey: admin.PublicKey(), - }, - www.UserError{ - ErrorCode: www.ErrorStatusChangeMessageCannotBeBlank, - }}, - - { - "no change message for abandoned", - admin, - www.SetProposalStatus{ - Token: tokenPublic, - ProposalStatus: www.PropStatusAbandoned, - Signature: sigPublicToAbandoned, - PublicKey: admin.PublicKey(), - }, - www.UserError{ - ErrorCode: www.ErrorStatusChangeMessageCannotBeBlank, - }}, - - { - "invalid public key", - admin, - www.SetProposalStatus{ + // TODO + /* + // Setup test environment + p, cleanup := newTestPoliteiawww(t) + defer cleanup() + + d := newTestPoliteiad(t, p) + defer d.Close() + + // Create test data + admin, id := newUser(t, p, true, true) + + changeMsgCensored := "proposal did not meet guidelines" + changeMsgAbandoned := "no activity" + + statusPublic := strconv.Itoa(int(www.PropStatusPublic)) + statusCensored := strconv.Itoa(int(www.PropStatusCensored)) + statusAbandoned := strconv.Itoa(int(www.PropStatusAbandoned)) + + propNotReviewed := newProposalRecord(t, admin, id, www.PropStatusNotReviewed) + propPublic := newProposalRecord(t, admin, id, www.PropStatusPublic) + + tokenNotReviewed := propNotReviewed.CensorshipRecord.Token + tokenPublic := propPublic.CensorshipRecord.Token + tokenNotFound := "abc" + + msg := fmt.Sprintf("%s%s", tokenNotReviewed, statusPublic) + s := id.SignMessage([]byte(msg)) + sigNotReviewedToPublic := hex.EncodeToString(s[:]) + + msg = fmt.Sprintf("%s%s%s", tokenNotReviewed, + statusCensored, changeMsgCensored) + s = id.SignMessage([]byte(msg)) + sigNotReviewedToCensored := hex.EncodeToString(s[:]) + + msg = fmt.Sprintf("%s%s%s", tokenPublic, + statusAbandoned, changeMsgAbandoned) + s = id.SignMessage([]byte(msg)) + sigPublicToAbandoned := hex.EncodeToString(s[:]) + + msg = fmt.Sprintf("%s%s", tokenNotFound, statusPublic) + s = id.SignMessage([]byte(msg)) + sigNotFound := hex.EncodeToString(s[:]) + + msg = fmt.Sprintf("%s%s%s", tokenNotReviewed, + statusAbandoned, changeMsgAbandoned) + s = id.SignMessage([]byte(msg)) + sigNotReviewedToAbandoned := hex.EncodeToString(s[:]) + + // Add success case proposals to politeiad + d.AddRecord(t, convertPropToPD(t, propNotReviewed)) + d.AddRecord(t, convertPropToPD(t, propPublic)) + + // Create a proposal whose vote has been authorized + propVoteAuthorized := newProposalRecord(t, admin, id, www.PropStatusPublic) + tokenVoteAuthorized := propVoteAuthorized.CensorshipRecord.Token + + msg = fmt.Sprintf("%s%s%s", tokenVoteAuthorized, + statusAbandoned, changeMsgAbandoned) + s = id.SignMessage([]byte(msg)) + sigVoteAuthorizedToAbandoned := hex.EncodeToString(s[:]) + + d.AddRecord(t, convertPropToPD(t, propVoteAuthorized)) + cmd := newAuthorizeVoteCmd(t, tokenVoteAuthorized, + propVoteAuthorized.Version, decredplugin.AuthVoteActionAuthorize, id) + d.Plugin(t, cmd) + + // Create a proposal whose voting period has started + propVoteStarted := newProposalRecord(t, admin, id, www.PropStatusPublic) + tokenVoteStarted := propVoteStarted.CensorshipRecord.Token + + msg = fmt.Sprintf("%s%s%s", tokenVoteStarted, + statusAbandoned, changeMsgAbandoned) + s = id.SignMessage([]byte(msg)) + sigVoteStartedToAbandoned := hex.EncodeToString(s[:]) + + d.AddRecord(t, convertPropToPD(t, propVoteStarted)) + cmd = newStartVoteCmd(t, tokenVoteStarted, 1, 2016, id) + d.Plugin(t, cmd) + + // Ensure that admins are not allowed to change the status of + // their own proposal on mainnet. This is run individually + // because it requires flipping the testnet config setting. + t.Run("admin is author", func(t *testing.T) { + p.cfg.TestNet = false + defer func() { + p.cfg.TestNet = true + }() + + sps := www.SetProposalStatus{ Token: tokenNotReviewed, ProposalStatus: www.PropStatusPublic, Signature: sigNotReviewedToPublic, - PublicKey: "", - }, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidSigningKey, - }}, - - { - "invalid signature", - admin, - www.SetProposalStatus{ - Token: tokenNotReviewed, - ProposalStatus: www.PropStatusPublic, - Signature: "", PublicKey: admin.PublicKey(), - }, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidSignature, - }}, - - { - "invalid proposal token", - admin, - www.SetProposalStatus{ - Token: tokenNotFound, - ProposalStatus: www.PropStatusPublic, - Signature: sigNotFound, - PublicKey: admin.PublicKey(), - }, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - }}, - - { - "invalid status change", - admin, - www.SetProposalStatus{ - Token: tokenNotReviewed, - ProposalStatus: www.PropStatusAbandoned, - StatusChangeMessage: changeMsgAbandoned, - Signature: sigNotReviewedToAbandoned, - PublicKey: admin.PublicKey(), - }, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidPropStatusTransition, - }}, - { - "unvetted success", - admin, - www.SetProposalStatus{ - Token: tokenNotReviewed, - ProposalStatus: www.PropStatusPublic, - Signature: sigNotReviewedToPublic, - PublicKey: admin.PublicKey(), - }, nil}, - - { - "vote already authorized", - admin, - www.SetProposalStatus{ - Token: tokenVoteAuthorized, - ProposalStatus: www.PropStatusAbandoned, - StatusChangeMessage: changeMsgAbandoned, - Signature: sigVoteAuthorizedToAbandoned, - PublicKey: admin.PublicKey(), - }, - www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - }}, - - { - "vote already started", - admin, - www.SetProposalStatus{ - Token: tokenVoteStarted, - ProposalStatus: www.PropStatusAbandoned, - StatusChangeMessage: changeMsgAbandoned, - Signature: sigVoteStartedToAbandoned, - PublicKey: admin.PublicKey(), - }, - www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - }}, - - { - "vetted success", - admin, - www.SetProposalStatus{ - Token: tokenPublic, - ProposalStatus: www.PropStatusAbandoned, - StatusChangeMessage: changeMsgAbandoned, - Signature: sigPublicToAbandoned, - PublicKey: admin.PublicKey(), - }, nil}, - } + } - // Run tests - for _, v := range tests { - t.Run(v.name, func(t *testing.T) { - reply, err := p.processSetProposalStatus(v.sps, v.usr) + _, err := p.processSetProposalStatus(sps, admin) got := errToStr(err) - want := errToStr(v.want) + want := www.ErrorStatus[www.ErrorStatusReviewerAdminEqualsAuthor] if got != want { t.Errorf("got error %v, want %v", got, want) } + }) - if err != nil { - // Test case passes - return - } + // Setup tests + var tests = []struct { + name string + usr *user.User + sps www.SetProposalStatus + want error + }{ + // This is an admin route so it can be assumed that the + // user has been validated and is an admin. + + { + "no change message for censored", + admin, + www.SetProposalStatus{ + Token: tokenNotReviewed, + ProposalStatus: www.PropStatusCensored, + Signature: sigNotReviewedToCensored, + PublicKey: admin.PublicKey(), + }, + www.UserError{ + ErrorCode: www.ErrorStatusChangeMessageCannotBeBlank, + }}, + + { + "no change message for abandoned", + admin, + www.SetProposalStatus{ + Token: tokenPublic, + ProposalStatus: www.PropStatusAbandoned, + Signature: sigPublicToAbandoned, + PublicKey: admin.PublicKey(), + }, + www.UserError{ + ErrorCode: www.ErrorStatusChangeMessageCannotBeBlank, + }}, + + { + "invalid public key", + admin, + www.SetProposalStatus{ + Token: tokenNotReviewed, + ProposalStatus: www.PropStatusPublic, + Signature: sigNotReviewedToPublic, + PublicKey: "", + }, + www.UserError{ + ErrorCode: www.ErrorStatusInvalidSigningKey, + }}, + + { + "invalid signature", + admin, + www.SetProposalStatus{ + Token: tokenNotReviewed, + ProposalStatus: www.PropStatusPublic, + Signature: "", + PublicKey: admin.PublicKey(), + }, + www.UserError{ + ErrorCode: www.ErrorStatusInvalidSignature, + }}, + + { + "invalid proposal token", + admin, + www.SetProposalStatus{ + Token: tokenNotFound, + ProposalStatus: www.PropStatusPublic, + Signature: sigNotFound, + PublicKey: admin.PublicKey(), + }, + www.UserError{ + ErrorCode: www.ErrorStatusInvalidCensorshipToken, + }}, + + { + "invalid status change", + admin, + www.SetProposalStatus{ + Token: tokenNotReviewed, + ProposalStatus: www.PropStatusAbandoned, + StatusChangeMessage: changeMsgAbandoned, + Signature: sigNotReviewedToAbandoned, + PublicKey: admin.PublicKey(), + }, + www.UserError{ + ErrorCode: www.ErrorStatusInvalidPropStatusTransition, + }}, + { + "unvetted success", + admin, + www.SetProposalStatus{ + Token: tokenNotReviewed, + ProposalStatus: www.PropStatusPublic, + Signature: sigNotReviewedToPublic, + PublicKey: admin.PublicKey(), + }, nil}, + + { + "vote already authorized", + admin, + www.SetProposalStatus{ + Token: tokenVoteAuthorized, + ProposalStatus: www.PropStatusAbandoned, + StatusChangeMessage: changeMsgAbandoned, + Signature: sigVoteAuthorizedToAbandoned, + PublicKey: admin.PublicKey(), + }, + www.UserError{ + ErrorCode: www.ErrorStatusWrongVoteStatus, + }}, + + { + "vote already started", + admin, + www.SetProposalStatus{ + Token: tokenVoteStarted, + ProposalStatus: www.PropStatusAbandoned, + StatusChangeMessage: changeMsgAbandoned, + Signature: sigVoteStartedToAbandoned, + PublicKey: admin.PublicKey(), + }, + www.UserError{ + ErrorCode: www.ErrorStatusWrongVoteStatus, + }}, + + { + "vetted success", + admin, + www.SetProposalStatus{ + Token: tokenPublic, + ProposalStatus: www.PropStatusAbandoned, + StatusChangeMessage: changeMsgAbandoned, + Signature: sigPublicToAbandoned, + PublicKey: admin.PublicKey(), + }, nil}, + } - // Validate updated proposal - if reply.Proposal.Status != v.sps.ProposalStatus { - t.Errorf("got proposal status %v, want %v", - reply.Proposal.Status, v.sps.ProposalStatus) - } - }) - } + // Run tests + for _, v := range tests { + t.Run(v.name, func(t *testing.T) { + reply, err := p.processSetProposalStatus(v.sps, v.usr) + got := errToStr(err) + want := errToStr(v.want) + if got != want { + t.Errorf("got error %v, want %v", + got, want) + } + + if err != nil { + // Test case passes + return + } + + // Validate updated proposal + if reply.Proposal.Status != v.sps.ProposalStatus { + t.Errorf("got proposal status %v, want %v", + reply.Proposal.Status, v.sps.ProposalStatus) + } + }) + } + */ } func TestProcessAllVetted(t *testing.T) { @@ -2451,83 +2465,86 @@ func TestProcessAuthorizeVote(t *testing.T) { } func TestProcessStartVoteV2(t *testing.T) { - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - d := newTestPoliteiad(t, p) - defer d.Close() - - usr, id := newUser(t, p, true, true) - usrNotAdmin, _ := newUser(t, p, false, false) - - prop := newProposalRecord(t, usr, id, www.PropStatusPublic) - token := prop.CensorshipRecord.Token - d.AddRecord(t, convertPropToPD(t, prop)) + // TODO + /* + p, cleanup := newTestPoliteiawww(t) + defer cleanup() - sv := newStartVote(t, token, 1, minDuration, www2.VoteTypeStandard, id) - vs := newVoteSummary(t, www.PropVoteStatusAuthorized, []www.VoteOptionResult{}) - p.voteSummarySet(token, vs) + d := newTestPoliteiad(t, p) + defer d.Close() - randomToken, err := util.Random(pd.TokenSize) - if err != nil { - t.Fatal(err) - } - rToken := hex.EncodeToString(randomToken) + usr, id := newUser(t, p, true, true) + usrNotAdmin, _ := newUser(t, p, false, false) - svInvalidToken := newStartVote(t, token, 1, minDuration, - www2.VoteTypeStandard, id) - svInvalidToken.Vote.Token = "" + prop := newProposalRecord(t, usr, id, www.PropStatusPublic) + token := prop.CensorshipRecord.Token + d.AddRecord(t, convertPropToPD(t, prop)) - svRandomToken := newStartVote(t, token, 1, minDuration, - www2.VoteTypeStandard, id) - svRandomToken.Vote.Token = rToken + sv := newStartVote(t, token, 1, minDuration, www2.VoteTypeStandard, id) + vs := newVoteSummary(t, www.PropVoteStatusAuthorized, []www.VoteOptionResult{}) + p.voteSummarySet(token, vs) - var tests = []struct { - name string - user *user.User - sv www2.StartVote - wantErr error - }{ - { - "user not admin", - usrNotAdmin, - sv, - fmt.Errorf("user is not an admin"), - }, - { - "invalid token", - usr, - svInvalidToken, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, + randomToken, err := util.Random(pd.TokenSize) + if err != nil { + t.Fatal(err) + } + rToken := hex.EncodeToString(randomToken) + + svInvalidToken := newStartVote(t, token, 1, minDuration, + www2.VoteTypeStandard, id) + svInvalidToken.Vote.Token = "" + + svRandomToken := newStartVote(t, token, 1, minDuration, + www2.VoteTypeStandard, id) + svRandomToken.Vote.Token = rToken + + var tests = []struct { + name string + user *user.User + sv www2.StartVote + wantErr error + }{ + { + "user not admin", + usrNotAdmin, + sv, + fmt.Errorf("user is not an admin"), + }, + { + "invalid token", + usr, + svInvalidToken, + www.UserError{ + ErrorCode: www.ErrorStatusInvalidCensorshipToken, + }, }, - }, - { - "proposal not found", - usr, - svRandomToken, - www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, + { + "proposal not found", + usr, + svRandomToken, + www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + }, }, - }, - { - "success", - usr, - sv, - nil, - }, - } + { + "success", + usr, + sv, + nil, + }, + } - for _, v := range tests { - t.Run(v.name, func(t *testing.T) { - _, err := p.processStartVoteV2(v.sv, v.user) - got := errToStr(err) - want := errToStr(v.wantErr) - if got != want { - t.Errorf("got %v, want %v", got, want) - } - }) - } + for _, v := range tests { + t.Run(v.name, func(t *testing.T) { + _, err := p.processStartVoteV2(v.sv, v.user) + got := errToStr(err) + want := errToStr(v.wantErr) + if got != want { + t.Errorf("got %v, want %v", got, want) + } + }) + } + */ } func TestProcessStartVoteRunoffV2(t *testing.T) { @@ -2789,75 +2806,78 @@ func TestProcessStartVoteRunoffV2(t *testing.T) { } func TestVerifyStatusChange(t *testing.T) { - invalid := www.PropStatusInvalid - notFound := www.PropStatusNotFound - notReviewed := www.PropStatusNotReviewed - censored := www.PropStatusCensored - public := www.PropStatusPublic - unreviewedChanges := www.PropStatusUnreviewedChanges - abandoned := www.PropStatusAbandoned - - // Setup tests - var tests = []struct { - name string - current www.PropStatusT - next www.PropStatusT - wantError bool - }{ - {"not reviewed to invalid", notReviewed, invalid, true}, - {"not reviewed to not found", notReviewed, notFound, true}, - {"not reviewed to censored", notReviewed, censored, false}, - {"not reviewed to public", notReviewed, public, false}, - {"not reviewed to unreviewed changes", notReviewed, unreviewedChanges, - true}, - {"not reviewed to abandoned", notReviewed, abandoned, true}, - {"censored to invalid", censored, invalid, true}, - {"censored to not found", censored, notFound, true}, - {"censored to not reviewed", censored, notReviewed, true}, - {"censored to public", censored, public, true}, - {"censored to unreviewed changes", censored, unreviewedChanges, true}, - {"censored to abandoned", censored, abandoned, true}, - {"public to invalid", public, invalid, true}, - {"public to not found", public, notFound, true}, - {"public to not reviewed", public, notReviewed, true}, - {"public to censored", public, censored, true}, - {"public to unreviewed changes", public, unreviewedChanges, true}, - {"public to abandoned", public, abandoned, false}, - {"unreviewed changes to invalid", unreviewedChanges, invalid, true}, - {"unreviewed changes to not found", unreviewedChanges, notFound, true}, - {"unreviewed changes to not reviewed", unreviewedChanges, notReviewed, - true}, - {"unreviewed changes to censored", unreviewedChanges, censored, false}, - {"unreviewed changes to public", unreviewedChanges, public, false}, - {"unreviewed changes to abandoned", unreviewedChanges, abandoned, true}, - {"abandoned to invalid", abandoned, invalid, true}, - {"abandoned to not found", abandoned, notFound, true}, - {"abandoned to not reviewed", abandoned, notReviewed, true}, - {"abandoned to censored", abandoned, censored, true}, - {"abandoned to public", abandoned, public, true}, - {"abandoned to unreviewed changes", abandoned, unreviewedChanges, true}, - } + // TODO + /* + invalid := www.PropStatusInvalid + notFound := www.PropStatusNotFound + notReviewed := www.PropStatusNotReviewed + censored := www.PropStatusCensored + public := www.PropStatusPublic + unreviewedChanges := www.PropStatusUnreviewedChanges + abandoned := www.PropStatusAbandoned + + // Setup tests + var tests = []struct { + name string + current www.PropStatusT + next www.PropStatusT + wantError bool + }{ + {"not reviewed to invalid", notReviewed, invalid, true}, + {"not reviewed to not found", notReviewed, notFound, true}, + {"not reviewed to censored", notReviewed, censored, false}, + {"not reviewed to public", notReviewed, public, false}, + {"not reviewed to unreviewed changes", notReviewed, unreviewedChanges, + true}, + {"not reviewed to abandoned", notReviewed, abandoned, true}, + {"censored to invalid", censored, invalid, true}, + {"censored to not found", censored, notFound, true}, + {"censored to not reviewed", censored, notReviewed, true}, + {"censored to public", censored, public, true}, + {"censored to unreviewed changes", censored, unreviewedChanges, true}, + {"censored to abandoned", censored, abandoned, true}, + {"public to invalid", public, invalid, true}, + {"public to not found", public, notFound, true}, + {"public to not reviewed", public, notReviewed, true}, + {"public to censored", public, censored, true}, + {"public to unreviewed changes", public, unreviewedChanges, true}, + {"public to abandoned", public, abandoned, false}, + {"unreviewed changes to invalid", unreviewedChanges, invalid, true}, + {"unreviewed changes to not found", unreviewedChanges, notFound, true}, + {"unreviewed changes to not reviewed", unreviewedChanges, notReviewed, + true}, + {"unreviewed changes to censored", unreviewedChanges, censored, false}, + {"unreviewed changes to public", unreviewedChanges, public, false}, + {"unreviewed changes to abandoned", unreviewedChanges, abandoned, true}, + {"abandoned to invalid", abandoned, invalid, true}, + {"abandoned to not found", abandoned, notFound, true}, + {"abandoned to not reviewed", abandoned, notReviewed, true}, + {"abandoned to censored", abandoned, censored, true}, + {"abandoned to public", abandoned, public, true}, + {"abandoned to unreviewed changes", abandoned, unreviewedChanges, true}, + } - // Run tests - for _, v := range tests { - t.Run(v.name, func(t *testing.T) { - err := verifyStatusChange(v.current, v.next) - got := errToStr(err) - if v.wantError { - want := www.ErrorStatus[www.ErrorStatusInvalidPropStatusTransition] - if got != want { - t.Errorf("got error %v, want %v", - got, want) + // Run tests + for _, v := range tests { + t.Run(v.name, func(t *testing.T) { + err := verifyStatusChange(v.current, v.next) + got := errToStr(err) + if v.wantError { + want := www.ErrorStatus[www.ErrorStatusInvalidPropStatusTransition] + if got != want { + t.Errorf("got error %v, want %v", + got, want) + } + + // Test case passes + return } - // Test case passes - return - } - - if err != nil { - t.Errorf("got error %v, want nil", - got) - } - }) - } + if err != nil { + t.Errorf("got error %v, want nil", + got) + } + }) + } + */ } diff --git a/politeiawww/testing.go b/politeiawww/testing.go index 7b7622da9..b25860678 100644 --- a/politeiawww/testing.go +++ b/politeiawww/testing.go @@ -53,6 +53,49 @@ func errToStr(e error) string { return e.Error() } +func convertPropToPD(t *testing.T, p www.ProposalRecord) pd.Record { + t.Helper() + + // TODO + /* + // Attach ProposalMetadata as a politeiad file + files := convertPropFilesFromWWW(p.Files) + for _, v := range p.Metadata { + switch v.Hint { + case www.HintProposalMetadata: + files = append(files, convertFileFromMetadata(v)) + } + } + + // Create a ProposalGeneralV2 mdstream + md, err := mdstream.EncodeProposalGeneralV2( + mdstream.ProposalGeneralV2{ + Version: mdstream.VersionProposalGeneral, + Timestamp: time.Now().Unix(), + PublicKey: p.PublicKey, + Signature: p.Signature, + }) + if err != nil { + t.Fatal(err) + } + + mdStreams := []pd.MetadataStream{{ + ID: mdstream.IDProposalGeneral, + Payload: string(md), + }} + + return pd.Record{ + Status: convertPropStatusFromWWW(p.Status), + Timestamp: p.Timestamp, + Version: p.Version, + Metadata: mdStreams, + CensorshipRecord: convertPropCensorFromWWW(p.CensorshipRecord), + Files: files, + } + */ + return pd.Record{} +} + func payRegistrationFee(t *testing.T, p *politeiawww, u *user.User) { t.Helper() From 415d9536e08cf8b3cd5adf75b626510287c32ecb Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 15 Sep 2020 10:31:09 -0500 Subject: [PATCH 047/449] cleanup deadcode --- politeiad/backend/gitbe/cms.go | 11 - politeiad/backend/gitbe/decred.go | 11 - politeiad/backend/gitbe/gitbe.go | 3 - politeiad/backend/tlogbe/plugins/comments.go | 2 +- politeiawww/convert.go | 261 ------------------- politeiawww/proposals.go | 27 -- politeiawww/templates.go | 9 - 7 files changed, 1 insertion(+), 323 deletions(-) diff --git a/politeiad/backend/gitbe/cms.go b/politeiad/backend/gitbe/cms.go index 669908888..780fbfa15 100644 --- a/politeiad/backend/gitbe/cms.go +++ b/politeiad/backend/gitbe/cms.go @@ -47,17 +47,6 @@ func encodeCastDCCVoteJournal(cvj CastDCCVoteJournal) ([]byte, error) { return b, nil } -func decodeCastDCCVoteJournal(payload []byte) (*CastDCCVoteJournal, error) { - var cvj CastDCCVoteJournal - - err := json.Unmarshal(payload, &cvj) - if err != nil { - return nil, err - } - - return &cvj, nil -} - var ( cmsPluginSettings map[string]string // [key]setting cmsPluginHooks map[string]func(string) error // [key]func(token) error diff --git a/politeiad/backend/gitbe/decred.go b/politeiad/backend/gitbe/decred.go index f3ad3b2a4..94d5bccb8 100644 --- a/politeiad/backend/gitbe/decred.go +++ b/politeiad/backend/gitbe/decred.go @@ -100,17 +100,6 @@ func encodeCastVoteJournal(cvj CastVoteJournal) ([]byte, error) { return b, nil } -func decodeCastVoteJournal(payload []byte) (*CastVoteJournal, error) { - var cvj CastVoteJournal - - err := json.Unmarshal(payload, &cvj) - if err != nil { - return nil, err - } - - return &cvj, nil -} - var ( decredPluginSettings map[string]string // [key]setting decredPluginHooks map[string]func(string) error // [key]func(token) error diff --git a/politeiad/backend/gitbe/gitbe.go b/politeiad/backend/gitbe/gitbe.go index 4b29b305b..fa3e1d42f 100644 --- a/politeiad/backend/gitbe/gitbe.go +++ b/politeiad/backend/gitbe/gitbe.go @@ -88,9 +88,6 @@ const ( // where an anchor confirmation has been committed. This value is // parsed and therefore must be a const. markerAnchorConfirmation = "Anchor confirmation" - - piMode = "piwww" - cmsMode = "cmswww" ) var ( diff --git a/politeiad/backend/tlogbe/plugins/comments.go b/politeiad/backend/tlogbe/plugins/comments.go index e68a83022..b6845710e 100644 --- a/politeiad/backend/tlogbe/plugins/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments.go @@ -478,7 +478,7 @@ func commentVoteSave(client *tlogbe.RecordClient, c comments.CommentVote) ([]byt } // Save blob - merkles, err := client.Save(keyPrefixCommentAdd, + merkles, err := client.Save(keyPrefixCommentVote, [][]byte{b}, [][]byte{h}, false) if err != nil { return nil, fmt.Errorf("Save: %v", err) diff --git a/politeiawww/convert.go b/politeiawww/convert.go index 41423ab9a..f9540fe8e 100644 --- a/politeiawww/convert.go +++ b/politeiawww/convert.go @@ -98,14 +98,6 @@ func convertVoteOptionFromWWW(vo www.VoteOption) decredplugin.VoteOption { } } -func convertVoteOptionsFromWWW(vo []www.VoteOption) []decredplugin.VoteOption { - vor := make([]decredplugin.VoteOption, 0, len(vo)) - for _, v := range vo { - vor = append(vor, convertVoteOptionFromWWW(v)) - } - return vor -} - func convertVoteOptionV2ToDecred(vo www2.VoteOption) decredplugin.VoteOption { return decredplugin.VoteOption{ Id: vo.Id, @@ -161,76 +153,6 @@ func convertStartVotesV2ToDecred(sv []www2.StartVote) []decredplugin.StartVoteV2 return dsv } -func convertDecredStartVoteV1ToVoteDetailsReplyV2(sv decredplugin.StartVoteV1, svr decredplugin.StartVoteReply) (*www2.VoteDetailsReply, error) { - startHeight, err := strconv.ParseUint(svr.StartBlockHeight, 10, 32) - if err != nil { - return nil, fmt.Errorf("parse start height '%v': %v", - svr.StartBlockHeight, err) - } - endHeight, err := strconv.ParseUint(svr.EndHeight, 10, 32) - if err != nil { - return nil, fmt.Errorf("parse end height '%v': %v", - svr.EndHeight, err) - } - voteb, err := decredplugin.EncodeVoteV1(sv.Vote) - if err != nil { - return nil, err - } - return &www2.VoteDetailsReply{ - Version: uint32(sv.Version), - Vote: string(voteb), - PublicKey: sv.PublicKey, - Signature: sv.Signature, - StartBlockHeight: uint32(startHeight), - StartBlockHash: svr.StartBlockHash, - EndBlockHeight: uint32(endHeight), - EligibleTickets: svr.EligibleTickets, - }, nil -} - -func convertDecredStartVoteV2ToVoteDetailsReplyV2(sv decredplugin.StartVoteV2, svr decredplugin.StartVoteReply) (*www2.VoteDetailsReply, error) { - startHeight, err := strconv.ParseUint(svr.StartBlockHeight, 10, 32) - if err != nil { - return nil, fmt.Errorf("parse start height '%v': %v", - svr.StartBlockHeight, err) - } - endHeight, err := strconv.ParseUint(svr.EndHeight, 10, 32) - if err != nil { - return nil, fmt.Errorf("parse end height '%v': %v", - svr.EndHeight, err) - } - voteb, err := decredplugin.EncodeVoteV2(sv.Vote) - if err != nil { - return nil, err - } - return &www2.VoteDetailsReply{ - Version: uint32(sv.Version), - Vote: string(voteb), - PublicKey: sv.PublicKey, - Signature: sv.Signature, - StartBlockHeight: uint32(startHeight), - StartBlockHash: svr.StartBlockHash, - EndBlockHeight: uint32(endHeight), - EligibleTickets: svr.EligibleTickets, - }, nil -} - -func convertPropStatusFromWWW(s www.PropStatusT) pd.RecordStatusT { - switch s { - case www.PropStatusNotFound: - return pd.RecordStatusNotFound - case www.PropStatusNotReviewed: - return pd.RecordStatusNotReviewed - case www.PropStatusCensored: - return pd.RecordStatusCensored - case www.PropStatusPublic: - return pd.RecordStatusPublic - case www.PropStatusAbandoned: - return pd.RecordStatusArchived - } - return pd.RecordStatusInvalid -} - func convertPropFileFromWWW(f www.File) pd.File { return pd.File{ Name: f.Name, @@ -256,14 +178,6 @@ func convertPropCensorFromPD(f pd.CensorshipRecord) www.CensorshipRecord { } } -func convertPropCensorFromWWW(f www.CensorshipRecord) pd.CensorshipRecord { - return pd.CensorshipRecord{ - Token: f.Token, - Merkle: f.Merkle, - Signature: f.Signature, - } -} - func convertPropStatusToState(status www.PropStatusT) www.PropStateT { switch status { case www.PropStatusNotReviewed, www.PropStatusUnreviewedChanges, @@ -285,36 +199,6 @@ func convertNewCommentToDecredPlugin(nc www.NewComment) decredplugin.NewComment } } -func convertLikeCommentToDecred(lc www.LikeComment) decredplugin.LikeComment { - return decredplugin.LikeComment{ - Token: lc.Token, - CommentID: lc.CommentID, - Action: lc.Action, - Signature: lc.Signature, - PublicKey: lc.PublicKey, - } -} - -func convertLikeCommentFromDecred(lc decredplugin.LikeComment) www.LikeComment { - return www.LikeComment{ - Token: lc.Token, - CommentID: lc.CommentID, - Action: lc.Action, - Signature: lc.Signature, - PublicKey: lc.PublicKey, - } -} - -func convertCensorCommentToDecred(cc www.CensorComment) decredplugin.CensorComment { - return decredplugin.CensorComment{ - Token: cc.Token, - CommentID: cc.CommentID, - Reason: cc.Reason, - Signature: cc.Signature, - PublicKey: cc.PublicKey, - } -} - func convertCommentFromDecred(c decredplugin.Comment) www.Comment { // Upvotes, Downvotes, UserID, and Username are filled in as zero // values since a decred plugin comment does not contain this data. @@ -368,21 +252,6 @@ func convertVoteOptionsV2FromDecred(options []decredplugin.VoteOption) []www2.Vo return opts } -func convertStartVoteV1FromDecred(sv decredplugin.StartVoteV1) www.StartVote { - return www.StartVote{ - PublicKey: sv.PublicKey, - Vote: www.Vote{ - Token: sv.Vote.Token, - Mask: sv.Vote.Mask, - Duration: sv.Vote.Duration, - QuorumPercentage: sv.Vote.QuorumPercentage, - PassPercentage: sv.Vote.PassPercentage, - Options: convertVoteOptionsFromDecred(sv.Vote.Options), - }, - Signature: sv.Signature, - } -} - func convertVoteTypeFromDecred(v decredplugin.VoteT) www2.VoteT { switch v { case decredplugin.VoteTypeStandard: @@ -393,22 +262,6 @@ func convertVoteTypeFromDecred(v decredplugin.VoteT) www2.VoteT { return www2.VoteTypeInvalid } -func convertStartVoteV2FromDecred(sv decredplugin.StartVoteV2) www2.StartVote { - return www2.StartVote{ - PublicKey: sv.PublicKey, - Vote: www2.Vote{ - Token: sv.Vote.Token, - Type: convertVoteTypeFromDecred(sv.Vote.Type), - Mask: sv.Vote.Mask, - Duration: sv.Vote.Duration, - QuorumPercentage: sv.Vote.QuorumPercentage, - PassPercentage: sv.Vote.PassPercentage, - Options: convertVoteOptionsV2FromDecred(sv.Vote.Options), - }, - Signature: sv.Signature, - } -} - func convertVoteOptionsV2ToV1(optsV2 []www2.VoteOption) []www.VoteOption { optsV1 := make([]www.VoteOption, 0, len(optsV2)) for _, v := range optsV2 { @@ -421,30 +274,6 @@ func convertVoteOptionsV2ToV1(optsV2 []www2.VoteOption) []www.VoteOption { return optsV1 } -func convertStartVoteV2ToV1(sv www2.StartVote) www.StartVote { - return www.StartVote{ - PublicKey: sv.PublicKey, - Vote: www.Vote{ - Token: sv.Vote.Token, - Mask: sv.Vote.Mask, - Duration: sv.Vote.Duration, - QuorumPercentage: sv.Vote.QuorumPercentage, - PassPercentage: sv.Vote.PassPercentage, - Options: convertVoteOptionsV2ToV1(sv.Vote.Options), - }, - Signature: sv.Signature, - } -} - -func convertStartVoteReplyFromDecred(svr decredplugin.StartVoteReply) www.StartVoteReply { - return www.StartVoteReply{ - StartBlockHeight: svr.StartBlockHeight, - StartBlockHash: svr.StartBlockHash, - EndHeight: svr.EndHeight, - EligibleTickets: svr.EligibleTickets, - } -} - func convertStartVoteReplyV2FromDecred(svr decredplugin.StartVoteReply) (*www2.StartVoteReply, error) { startHeight, err := strconv.ParseUint(svr.StartBlockHeight, 10, 32) if err != nil { @@ -473,14 +302,6 @@ func convertCastVoteFromDecred(cv decredplugin.CastVote) www.CastVote { } } -func convertCastVotesFromDecred(cv []decredplugin.CastVote) []www.CastVote { - cvr := make([]www.CastVote, 0, len(cv)) - for _, v := range cv { - cvr = append(cvr, convertCastVoteFromDecred(v)) - } - return cvr -} - func convertPluginSettingFromPD(ps pd.PluginSetting) PluginSetting { return PluginSetting{ Key: ps.Key, @@ -499,32 +320,6 @@ func convertPluginFromPD(p pd.Plugin) Plugin { Settings: ps, } } -func convertVoteOptionResultsFromDecred(vor []decredplugin.VoteOptionResult) []www.VoteOptionResult { - r := make([]www.VoteOptionResult, 0, len(vor)) - for _, v := range vor { - r = append(r, www.VoteOptionResult{ - Option: www.VoteOption{ - Id: v.ID, - Description: v.Description, - Bits: v.Bits, - }, - VotesReceived: v.Votes, - }) - } - return r -} - -func convertTokenInventoryReplyFromDecred(r decredplugin.TokenInventoryReply) www.TokenInventoryReply { - return www.TokenInventoryReply{ - Pre: r.Pre, - Active: r.Active, - Approved: r.Approved, - Rejected: r.Rejected, - Abandoned: r.Abandoned, - Unreviewed: r.Unreviewed, - Censored: r.Censored, - } -} func convertInvoiceCensorFromWWW(f www.CensorshipRecord) pd.CensorshipRecord { return pd.CensorshipRecord{ @@ -665,22 +460,6 @@ func convertLineItemsToDatabase(token string, l []cms.LineItemsInput) []cmsdatab return dl } -func convertDatabaseToLineItems(dl []cmsdatabase.LineItem) []cms.LineItemsInput { - l := make([]cms.LineItemsInput, 0, len(dl)) - for _, v := range dl { - l = append(l, cms.LineItemsInput{ - Type: v.Type, - Domain: v.Domain, - Subdomain: v.Subdomain, - Description: v.Description, - ProposalToken: v.ProposalURL, - Labor: v.Labor, - Expenses: v.Expenses, - }) - } - return l -} - func convertRecordToDatabaseInvoice(p pd.Record) (*cmsdatabase.Invoice, error) { dbInvoice := cmsdatabase.Invoice{ Files: convertRecordFilesToWWW(p.Files), @@ -891,46 +670,6 @@ func convertDCCDatabaseToRecord(dbDCC *cmsdatabase.DCC) cms.DCCRecord { return dccRecord } -func convertDCCDatabaseFromDCCRecord(dccRecord cms.DCCRecord) cmsdatabase.DCC { - dbDCC := cmsdatabase.DCC{} - - dbDCC.Type = dccRecord.DCC.Type - dbDCC.NomineeUserID = dccRecord.DCC.NomineeUserID - dbDCC.SponsorStatement = dccRecord.DCC.SponsorStatement - dbDCC.Domain = dccRecord.DCC.Domain - dbDCC.ContractorType = dccRecord.DCC.ContractorType - dbDCC.Status = dccRecord.Status - dbDCC.StatusChangeReason = dccRecord.StatusChangeReason - dbDCC.Timestamp = dccRecord.Timestamp - dbDCC.Token = dccRecord.CensorshipRecord.Token - dbDCC.PublicKey = dccRecord.PublicKey - dbDCC.ServerSignature = dccRecord.Signature - dbDCC.SponsorUserID = dccRecord.SponsorUserID - dbDCC.Token = dccRecord.CensorshipRecord.Token - - supportUserIDs := "" - for i, s := range dccRecord.SupportUserIDs { - if i == 0 { - supportUserIDs += s - } else { - supportUserIDs += "," + s - } - } - dbDCC.SupportUserIDs = supportUserIDs - - oppositionUserIDs := "" - for i, s := range dccRecord.OppositionUserIDs { - if i == 0 { - oppositionUserIDs += s - } else { - oppositionUserIDs += "," + s - } - } - dbDCC.OppositionUserIDs = oppositionUserIDs - - return dbDCC -} - func convertDatabaseInvoiceToProposalLineItems(inv cmsdatabase.Invoice) cms.ProposalLineItems { return cms.ProposalLineItems{ Month: int(inv.Month), diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 9f4a64d14..774843553 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -67,18 +67,6 @@ func isRFPSubmission(pr www.ProposalRecord) bool { return pr.LinkTo != "" } -func getInvalidTokens(tokens []string) []string { - invalidTokens := make([]string, 0, len(tokens)) - - for _, token := range tokens { - if !tokenIsValid(token) { - invalidTokens = append(invalidTokens, token) - } - } - - return invalidTokens -} - // validateVoteBit ensures that bit is a valid vote bit. func validateVoteBit(vote www2.Vote, bit uint64) error { if len(vote.Options) == 0 { @@ -123,21 +111,6 @@ func (p *politeiawww) linkByValidate(linkBy int64) error { return nil } -func voteStatusFromVoteSummary(r decredplugin.VoteSummaryReply, endHeight, bestBlock uint64) www.PropVoteStatusT { - switch { - case !r.Authorized: - return www.PropVoteStatusNotAuthorized - case r.Authorized && endHeight == 0: - return www.PropVoteStatusAuthorized - case bestBlock < endHeight: - return www.PropVoteStatusStarted - case bestBlock >= endHeight: - return www.PropVoteStatusFinished - } - - return www.PropVoteStatusInvalid -} - func (p *politeiawww) getProp(token string) (*www.ProposalRecord, error) { return nil, nil } diff --git a/politeiawww/templates.go b/politeiawww/templates.go index c8469301d..2df757066 100644 --- a/politeiawww/templates.go +++ b/politeiawww/templates.go @@ -178,15 +178,6 @@ type commentReplyOnProposalTemplateData struct { CommentLink string } -type commentReplyOnCommentTemplateData struct { - Commenter string - ProposalName string - CommentLink string -} - -type newInvoiceCommentTemplateData struct { -} - type newInvoiceStatusUpdateTemplate struct { Token string } From 3294730fa16d58059723ea9e32cc48fb35affc5b Mon Sep 17 00:00:00 2001 From: amass Date: Tue, 15 Sep 2020 22:20:40 +0300 Subject: [PATCH 048/449] politeiawww: add processProposalDetails --- politeiawww/api/pi/v1/v1.go | 3 +- politeiawww/proposals.go | 128 +++++++++++++++++++++++++++++++++++- 2 files changed, 129 insertions(+), 2 deletions(-) diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index c55ab82a1..926838c6e 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -190,8 +190,9 @@ type Metadata struct { Payload string `json:"payload"` // JSON metadata content, base64 encoded } +// Metadata hints const ( - // Metadata hints + // HintProposalMetadata is the proposal metadata hint HintProposalMetadata = "proposalmetadata" ) diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 128683e50..47a5888ef 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -5,6 +5,7 @@ package main import ( + "encoding/base64" "encoding/hex" "encoding/json" "fmt" @@ -14,7 +15,9 @@ import ( "time" "github.com/decred/politeia/decredplugin" + piplugin "github.com/decred/politeia/plugins/pi" pd "github.com/decred/politeia/politeiad/api/v1" + pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" www2 "github.com/decred/politeia/politeiawww/api/www/v2" "github.com/decred/politeia/politeiawww/user" @@ -216,8 +219,131 @@ func filterProps(filter proposalsFilter, all []www.ProposalRecord) []www.Proposa return proposals } +func convertStateToWWW(state pi.PropStateT) www.PropStateT { + switch state { + case pi.PropStateInvalid: + return www.PropStateInvalid + case pi.PropStateUnvetted: + return www.PropStateUnvetted + case pi.PropStateVetted: + return www.PropStateVetted + default: + return www.PropStateInvalid + } +} + +func convertStatusToWWW(status pi.PropStatusT) www.PropStatusT { + switch status { + case pi.PropStatusInvalid: + return www.PropStatusInvalid + case pi.PropStatusPublic: + return www.PropStatusPublic + case pi.PropStatusCensored: + return www.PropStatusCensored + case pi.PropStatusAbandoned: + return www.PropStatusAbandoned + default: + return www.PropStatusInvalid + } +} + +func (p *politeiawww) convertProposalToWWW(pr *pi.ProposalRecord) (*www.ProposalRecord, error) { + // Decode metadata + var pm *piplugin.ProposalMetadata + for _, v := range pr.Metadata { + if v.Hint == pi.HintProposalMetadata { + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return nil, err + } + pm, err = piplugin.DecodeProposalMetadata(b) + if err != nil { + return nil, err + } + } + } + // Fill info + pw := www.ProposalRecord{ + Name: pm.Name, + LinkBy: pm.LinkBy, + LinkTo: pm.LinkTo, + Status: convertStatusToWWW(pr.Status), + State: convertStateToWWW(pr.State), + Timestamp: pr.Timestamp, + UserId: pr.UserID, + Username: pr.Username, + PublicKey: pr.PublicKey, + Signature: pr.Signature, + NumComments: uint(pr.Comments), + Version: pr.Version, + LinkedFrom: pr.LinkedFrom, + CensorshipRecord: www.CensorshipRecord{ + Token: pr.CensorshipRecord.Token, + Merkle: pr.CensorshipRecord.Merkle, + Signature: pr.CensorshipRecord.Signature, + }, + } + + files := make([]www.File, len(pr.Files)) + for _, f := range pr.Files { + files = append(files, www.File{ + Name: f.Name, + MIME: f.MIME, + Digest: f.Digest, + Payload: f.Payload, + }) + } + pw.Files = files + + metadata := make([]www.Metadata, len(pr.Metadata)) + for _, md := range pr.Metadata { + metadata = append(metadata, www.Metadata{ + Digest: md.Digest, + Hint: md.Hint, + Payload: md.Payload, + }) + } + pw.Metadata = metadata + + var ( + changeMsg string + changeMsgTimestamp int64 + ) + for _, v := range pr.Statuses { + if v.Timestamp > changeMsgTimestamp { + changeMsgTimestamp = v.Timestamp + changeMsg = v.Reason + } + switch v.Status { + case pi.PropStatusPublic: + pw.PublishedAt = v.Timestamp + case pi.PropStatusCensored: + pw.CensoredAt = v.Timestamp + case pi.PropStatusAbandoned: + pw.AbandonedAt = v.Timestamp + } + } + pw.StatusChangeMessage = changeMsg + + return &pw, nil +} + func (p *politeiawww) processProposalDetails(pd www.ProposalsDetails, u *user.User) (*www.ProposalDetailsReply, error) { - return nil, nil + log.Tracef("processProposalDetails: %v", pd.Token) + + pr, err := p.proposalRecordLatest(pi.PropStateVetted, pd.Token) + if err != nil { + return nil, err + } + pw, err := p.convertProposalToWWW(pr) + if err != nil { + return nil, err + } + pdr := www.ProposalDetailsReply{ + Proposal: *pw, + } + + return &pdr, nil } func (p *politeiawww) processBatchVoteSummary(bvs www.BatchVoteSummary) (*www.BatchVoteSummaryReply, error) { From 8c8d184e02ffb3304910f3e87d12ceaafd2ae5a4 Mon Sep 17 00:00:00 2001 From: amass Date: Tue, 15 Sep 2020 22:57:56 +0300 Subject: [PATCH 049/449] add comments regarding proposal state & use provided version --- politeiawww/api/www/v1/api.md | 3 ++- politeiawww/api/www/v1/v1.go | 3 ++- politeiawww/proposals.go | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/politeiawww/api/www/v1/api.md b/politeiawww/api/www/v1/api.md index cc0a4ec30..44a6c59a1 100644 --- a/politeiawww/api/www/v1/api.md +++ b/politeiawww/api/www/v1/api.md @@ -1527,7 +1527,8 @@ Reply: ### `Proposal details` Retrieve proposal and its details. This request can be made with the full -censorship token or its 7 character prefix. +censorship token or its 7 character prefix. This route will return only +vetted proposals. **Routes:** `GET /v1/proposals/{token}` diff --git a/politeiawww/api/www/v1/v1.go b/politeiawww/api/www/v1/v1.go index 4ed06b3af..a29a30216 100644 --- a/politeiawww/api/www/v1/v1.go +++ b/politeiawww/api/www/v1/v1.go @@ -858,7 +858,8 @@ type NewProposalReply struct { // ProposalsDetails is used to retrieve a proposal by it's token // and by the proposal version (optional). If the version isn't specified -// the latest proposal version will be returned by default. +// the latest proposal version will be returned by default. Returns only +// vetted proposals type ProposalsDetails struct { Token string `json:"token"` // Censorship token Version string `json:"version,omitempty"` // Proposal version diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 47a5888ef..ce92608d8 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -331,7 +331,7 @@ func (p *politeiawww) convertProposalToWWW(pr *pi.ProposalRecord) (*www.Proposal func (p *politeiawww) processProposalDetails(pd www.ProposalsDetails, u *user.User) (*www.ProposalDetailsReply, error) { log.Tracef("processProposalDetails: %v", pd.Token) - pr, err := p.proposalRecordLatest(pi.PropStateVetted, pd.Token) + pr, err := p.proposalRecord(pi.PropStateVetted, pd.Token, pd.Version) if err != nil { return nil, err } From 623576c46047cd1db0df13acb473ba5dd80b79bd Mon Sep 17 00:00:00 2001 From: amass Date: Tue, 15 Sep 2020 23:02:20 +0300 Subject: [PATCH 050/449] ops --- politeiawww/api/www/v1/v1.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/politeiawww/api/www/v1/v1.go b/politeiawww/api/www/v1/v1.go index a29a30216..4ee63d70d 100644 --- a/politeiawww/api/www/v1/v1.go +++ b/politeiawww/api/www/v1/v1.go @@ -859,7 +859,7 @@ type NewProposalReply struct { // ProposalsDetails is used to retrieve a proposal by it's token // and by the proposal version (optional). If the version isn't specified // the latest proposal version will be returned by default. Returns only -// vetted proposals +// vetted proposals. type ProposalsDetails struct { Token string `json:"token"` // Censorship token Version string `json:"version,omitempty"` // Proposal version From 56ae1233a13fa08c2627b22c28105dfb4b4a7157 Mon Sep 17 00:00:00 2001 From: Thiago Figueiredo Date: Mon, 14 Sep 2020 18:32:08 -0300 Subject: [PATCH 051/449] www/d: hookup functions and route setups --- politeiad/api/v1/v1.go | 47 ++++++-- politeiad/politeiad.go | 106 +++++++++++++++++ politeiawww/politeiad.go | 246 ++++++++++++++++++++++++++++++++++++--- 3 files changed, 375 insertions(+), 24 deletions(-) diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index d439b7435..58e318a61 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -21,13 +21,15 @@ type RecordStatusT int const ( // Routes - IdentityRoute = "/v1/identity/" // Retrieve identity - NewRecordRoute = "/v1/newrecord/" // New record - UpdateUnvettedRoute = "/v1/updateunvetted/" // Update unvetted record - UpdateVettedRoute = "/v1/updatevetted/" // Update vetted record - UpdateVettedMetadataRoute = "/v1/updatevettedmd/" // Update vetted metadata - GetUnvettedRoute = "/v1/getunvetted/" // Retrieve unvetted record - GetVettedRoute = "/v1/getvetted/" // Retrieve vetted record + IdentityRoute = "/v1/identity/" // Retrieve identity + NewRecordRoute = "/v1/newrecord/" // New record + UpdateUnvettedRoute = "/v1/updateunvetted/" // Update unvetted record + UpdateVettedRoute = "/v1/updatevetted/" // Update vetted record + UpdateVettedMetadataRoute = "/v1/updatevettedmd/" // Update vetted metadata + UpdateUnvettedMetadataRoute = "/v1/updateunvettedmd/" // Update unvetted metadata + GetUnvettedRoute = "/v1/getunvetted/" // Retrieve unvetted record + GetVettedRoute = "/v1/getvetted/" // Retrieve vetted record + InventoryByStatusRoute = "/v1/inventorybystatus/" // Inventory record tokens by status // Auth required InventoryRoute = "/v1/inventory/" // Inventory records @@ -317,8 +319,8 @@ type UpdateRecord struct { // UpdateRecordReply returns a CensorshipRecord which may or may not have // changed. Metadata only updates do not create a new CensorshipRecord. type UpdateRecordReply struct { - // TODO add censorship record Response string `json:"response"` // Challenge response + Token string `json:"token"` // Censorship token } // UpdateVettedMetadata update a vetted metadata. This is allowed for @@ -336,6 +338,19 @@ type UpdateVettedMetadataReply struct { Response string `json:"response"` // Challenge response } +// UpdateUnvettedMetadata update a unvetted metadata. +type UpdateUnvettedMetadata struct { + Challenge string `json:"challenge"` // Random challenge + Token string `json:"token"` // Censorship token + MDAppend []MetadataStream `json:"mdappend"` // Metadata streams to append + MDOverwrite []MetadataStream `json:"mdoverwrite"` // Metadata streams to overwrite +} + +// UpdateUnvettedMetadataReply returns a response challenge. +type UpdateUnvettedMetadataReply struct { + Response string `json:"response"` // Challenge response +} + // Inventory sends an (expensive and therefore authenticated) inventory request // for vetted records (master branch) and branches (censored, unpublished etc) // records. This is a very expensive call and should be only issued at start @@ -363,6 +378,22 @@ type InventoryReply struct { Branches []Record `json:"branches"` // Last N branches (censored, new etc) } +// InventoryByStatus requests for the censhorship tokens from all records +// filtered by their status. +type InventoryByStatus struct { + Challenge string `json:"challenge"` // Random challenge +} + +// InventoryByStatusReply returns all censorship record tokens by status. +type InventoryByStatusReply struct { + Response string `json:"response"` // Challenge response + Unvetted []string `json:"unvetted"` // Unvetted censorship tokens + IterationUnvetted []string `json:"iterationunvetted"` // Iteration unvetted censorship tokens + Vetted []string `json:"vetted"` // Vetted censorship tokens + Censored []string `json:"censored"` // Censored censorship tokens + Archived []string `json:"archived"` // Archived censorship tokens +} + // UserErrorReply returns details about an error that occurred while trying to // execute a command due to bad input from the client. type UserErrorReply struct { diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 50986f61e..7ae9a3ec1 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -369,6 +369,7 @@ func (p *politeia) updateRecord(w http.ResponseWriter, r *http.Request, vetted b response := p.identity.SignMessage(challenge) reply := v1.UpdateRecordReply{ Response: hex.EncodeToString(response[:]), + Token: t.Token, } log.Infof("Update %v record %v: token %v", cmd, remoteAddr(r), @@ -561,6 +562,45 @@ func (p *politeia) inventory(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, reply) } +func (p *politeia) inventoryByStatus(w http.ResponseWriter, r *http.Request) { + var ibs v1.InventoryByStatus + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&ibs); err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) + return + } + + challenge, err := hex.DecodeString(ibs.Challenge) + if err != nil || len(challenge) != v1.ChallengeSize { + p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) + return + } + + ps, err := p.backend.InventoryByStatus() + if err != nil { + // Generic internal error. + errorCode := time.Now().Unix() + log.Errorf("%v InventoryByStatus error code %v: %v", remoteAddr(r), + errorCode, err) + + p.respondWithServerError(w, errorCode) + return + } + + // Prepare reply + response := p.identity.SignMessage(challenge) + reply := v1.InventoryByStatusReply{ + Response: hex.EncodeToString(response[:]), + Unvetted: ps.Unvetted, + IterationUnvetted: ps.IterationUnvetted, + Vetted: ps.Vetted, + Censored: ps.Censored, + Archived: ps.Archived, + } + + util.RespondWithJSON(w, http.StatusOK, reply) +} + func (p *politeia) check(user, pass string) bool { if user != p.cfg.RPCUser || pass != p.cfg.RPCPass { return false @@ -772,6 +812,68 @@ func (p *politeia) updateVettedMetadata(w http.ResponseWriter, r *http.Request) util.RespondWithJSON(w, http.StatusOK, reply) } +func (p *politeia) updateUnvettedMetadata(w http.ResponseWriter, r *http.Request) { + var t v1.UpdateUnvettedMetadata + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&t); err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) + return + } + + challenge, err := hex.DecodeString(t.Challenge) + if err != nil || len(challenge) != v1.ChallengeSize { + p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) + return + } + + token, err := util.ConvertStringToken(t.Token) + if err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) + return + } + + log.Infof("Update unvetted metadata submitted %v: %x", remoteAddr(r), + token) + + err = p.backend.UpdateUnvettedMetadata(token, + convertFrontendMetadataStream(t.MDAppend), + convertFrontendMetadataStream(t.MDOverwrite)) + if err != nil { + // Reply with error if there were no changes + if err == backend.ErrNoChanges { + log.Errorf("%v update unvetted metadata no changes: %x", + remoteAddr(r), token) + p.respondWithUserError(w, v1.ErrorStatusNoChanges, nil) + return + } + // Check for content error. + if contentErr, ok := err.(backend.ContentVerificationError); ok { + log.Errorf("%v update unvetted metadata content error: %v", + remoteAddr(r), contentErr) + p.respondWithUserError(w, contentErr.ErrorCode, + contentErr.ErrorContext) + return + } + + // Generic internal error. + errorCode := time.Now().Unix() + log.Errorf("%v Update unvetted metadata error code %v: %v", + remoteAddr(r), errorCode, err) + p.respondWithServerError(w, errorCode) + return + } + + // Reply with challenge response + response := p.identity.SignMessage(challenge) + reply := v1.UpdateUnvettedMetadataReply{ + Response: hex.EncodeToString(response[:]), + } + + log.Infof("Update unvetted metadata %v: token %x", remoteAddr(r), token) + + util.RespondWithJSON(w, http.StatusOK, reply) +} + func (p *politeia) pluginInventory(w http.ResponseWriter, r *http.Request) { var pi v1.PluginInventory decoder := json.NewDecoder(r.Body) @@ -997,6 +1099,8 @@ func _main() error { permissionPublic) p.addRoute(http.MethodPost, v1.GetVettedRoute, p.getVetted, permissionPublic) + p.addRoute(http.MethodPost, v1.InventoryByStatusRoute, + p.inventoryByStatus, permissionPublic) // Routes that require auth p.addRoute(http.MethodPost, v1.InventoryRoute, p.inventory, @@ -1007,6 +1111,8 @@ func _main() error { p.setVettedStatus, permissionAuth) p.addRoute(http.MethodPost, v1.UpdateVettedMetadataRoute, p.updateVettedMetadata, permissionAuth) + p.addRoute(http.MethodPost, v1.UpdateUnvettedMetadataRoute, + p.updateUnvettedMetadata, permissionAuth) // Setup plugins /* diff --git a/politeiawww/politeiad.go b/politeiawww/politeiad.go index ab356f075..10d915e8c 100644 --- a/politeiawww/politeiad.go +++ b/politeiawww/politeiad.go @@ -15,21 +15,6 @@ import ( "github.com/decred/politeia/util" ) -// TODO politeiad API changes that are needed -// -The politeiad UpdateRecordReply is suppose to return the censorship record -// but it doesn't. -// -Add an updateUnvettedMetadata route. This has been added to the politeiad -// Backend interface, but a corresponding route has not been added yet. -// -Add a InventoryByStatus route. This has been added to the politeiad -// Backend interface, but a corresponding route has not been added yet. -// -// TODO add functions to this file for: -// -setUnvettedStatus -// -setVettedStatus -// -updateUnvettedMetadata -// -updatedVettedMetadata -// -plugin - // pdErrorReply represents the request body that is returned from politeaid // when an error occurs. PluginID will be populated if this is a plugin error. type pdErrorReply struct { @@ -103,6 +88,8 @@ func (p *politeiawww) makeRequest(method string, route string, v interface{}) ([ return responseBody, nil } +// newRecord creates a record in politeiad. This route returns the censorship +// record from the new created record. func (p *politeiawww) newRecord(metadata []pd.MetadataStream, files []pd.File) (*pd.CensorshipRecord, error) { // Setup request challenge, err := util.Random(pd.ChallengeSize) @@ -121,11 +108,14 @@ func (p *politeiawww) newRecord(metadata []pd.MetadataStream, files []pd.File) ( return nil, err } + // Receive reply var nrr pd.NewRecordReply err = json.Unmarshal(resBody, &nrr) if err != nil { return nil, err } + + // Verify challenge err = util.VerifyChallenge(p.cfg.Identity, challenge, nrr.Response) if err != nil { return nil, err @@ -155,11 +145,15 @@ func (p *politeiawww) updateRecord(route, token string, mdAppend, mdOverwrite [] if err != nil { return nil } + + // Receive reply var urr pd.UpdateRecordReply err = json.Unmarshal(resBody, &urr) if err != nil { return err } + + // Verify challenge err = util.VerifyChallenge(p.cfg.Identity, challenge, urr.Response) if err != nil { return err @@ -180,12 +174,152 @@ func (p *politeiawww) updateVetted(token string, mdAppend, mdOverwrite []pd.Meta mdAppend, mdOverwrite, filesAdd, filesDel) } +// updateUnvettedMetadata updates the metadata of a unvetted record in politeiad. +func (p *politeiawww) updateUnvettedMetadata(token string, mdAppend, mdOverwrite []pd.MetadataStream) error { + // Setup request + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return err + } + uum := pd.UpdateUnvettedMetadata{ + Challenge: hex.EncodeToString(challenge), + Token: token, + MDAppend: mdAppend, + MDOverwrite: mdOverwrite, + } + + // Send request + resBody, err := p.makeRequest(http.MethodPost, + pd.UpdateUnvettedMetadataRoute, uum) + if err != nil { + return nil + } + + // Receive reply + var urr pd.UpdateUnvettedMetadataReply + err = json.Unmarshal(resBody, &urr) + if err != nil { + return err + } + + // Verify challenge + err = util.VerifyChallenge(p.cfg.Identity, challenge, urr.Response) + if err != nil { + return err + } + + return nil +} + +// updateVettedMetadata updates the metadata of a vetted record in politeiad. +func (p *politeiawww) updateVettedMetadata(token string, mdAppend, mdOverwrite []pd.MetadataStream) error { + // Setup request + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return err + } + uum := pd.UpdateVettedMetadata{ + Challenge: hex.EncodeToString(challenge), + Token: token, + MDAppend: mdAppend, + MDOverwrite: mdOverwrite, + } + + // Send request + resBody, err := p.makeRequest(http.MethodPost, + pd.UpdateVettedMetadataRoute, uum) + if err != nil { + return nil + } + + // Receive reply + var urr pd.UpdateVettedMetadataReply + err = json.Unmarshal(resBody, &urr) + if err != nil { + return err + } + + // Verify challenge + err = util.VerifyChallenge(p.cfg.Identity, challenge, urr.Response) + if err != nil { + return err + } + + return nil +} + +// setUnvettedStatus sets the status of a unvetted record in politeiad. func (p *politeiawww) setUnvettedStatus(token string, status pd.RecordStatusT, mdAppend, mdOverwrite []pd.MetadataStream) error { + // Setup request + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return err + } + suv := pd.SetUnvettedStatus{ + Challenge: hex.EncodeToString(challenge), + Token: token, + Status: status, + MDAppend: mdAppend, + MDOverwrite: mdOverwrite, + } + + // Send request + resBody, err := p.makeRequest(http.MethodPost, pd.SetUnvettedStatusRoute, + suv) + if err != nil { + return err + } + + // Receive reply + var suvr pd.SetUnvettedStatusReply + err = json.Unmarshal(resBody, &suvr) + if err != nil { + return err + } + + // Verify challenge + err = util.VerifyChallenge(p.cfg.Identity, challenge, suvr.Response) + if err != nil { + return err + } return nil } +// setVettedStatus sets the status of a vetted record in politeiad. func (p *politeiawww) setVettedStatus(token string, status pd.RecordStatusT, mdAppend, mdOverwrite []pd.MetadataStream) error { + // Setup request + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return err + } + svs := pd.SetVettedStatus{ + Challenge: hex.EncodeToString(challenge), + Token: token, + Status: status, + MDAppend: mdAppend, + MDOverwrite: mdOverwrite, + } + + // Send request + resBody, err := p.makeRequest(http.MethodPost, pd.SetVettedStatusRoute, + svs) + if err != nil { + return err + } + + // Receive reply + var svvr pd.SetVettedStatusReply + err = json.Unmarshal(resBody, &svvr) + if err != nil { + return err + } + + // Verify challenge + err = util.VerifyChallenge(p.cfg.Identity, challenge, svvr.Response) + if err != nil { + return err + } return nil } @@ -208,11 +342,15 @@ func (p *politeiawww) getUnvetted(token, version string) (*pd.Record, error) { if err != nil { return nil, err } + + // Receive reply var gur pd.GetUnvettedReply err = json.Unmarshal(resBody, &gur) if err != nil { return nil, err } + + // Verify challenge err = util.VerifyChallenge(p.cfg.Identity, challenge, gur.Response) if err != nil { return nil, err @@ -245,11 +383,15 @@ func (p *politeiawww) getVetted(token, version string) (*pd.Record, error) { if err != nil { return nil, err } + + // Receive reply var gvr pd.GetVettedReply err = json.Unmarshal(resBody, &gvr) if err != nil { return nil, err } + + // Verify challenge err = util.VerifyChallenge(p.cfg.Identity, challenge, gvr.Response) if err != nil { return nil, err @@ -258,8 +400,80 @@ func (p *politeiawww) getVetted(token, version string) (*pd.Record, error) { return &gvr.Record, nil } -// getVettedLatest returns the latest version of the vvetted record for the +// getVettedLatest returns the latest version of the vetted record for the // provided token. func (p *politeiawww) getVettedLatest(token string) (*pd.Record, error) { return p.getVetted(token, "") } + +// pluginInventory requests the plugin inventory from politeiad and returns +// the available plugins slice. +func (p *politeiawww) pluginInventory() ([]pd.Plugin, error) { + // Setup request + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return nil, err + } + pi := pd.PluginInventory{ + Challenge: hex.EncodeToString(challenge), + } + + // Send request + resBody, err := p.makeRequest(http.MethodPost, pd.PluginInventoryRoute, pc) + if err != nil { + return nil, err + } + + // Receive reply + var pir pd.PluginInventoryReply + err = json.Unmarshal(resBody, &pir) + if err != nil { + return nil, err + } + + // Verify challenge + err = util.VerifyChallenge(p.cfg.Identity, challenge, pir.Response) + if err != nil { + return nil, err + } + + return pir.Plugins, nil +} + +// pluginCommand fires a plugin command on politeiad and returns the reply +// payload. +func (p *politeiawww) pluginCommand(pluginID, cmd, cmdID, payload string) (payload string, error) { + // Setup request + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return nil, err + } + pc := pd.PluginCommand{ + Challenge: hex.EncodeToString(challenge), + ID: pluginID, + Command: cmd, + CommandID: cmdID, + Payload: payload, + } + + // Send request + resBody, err := p.makeRequest(http.MethodPost, pd.PluginCommandRoute, pc) + if err != nil { + return nil, err + } + + // Receive reply + var pcr pd.PluginCommandReply + err = json.Unmarshal(resBody, &pcr) + if err != nil { + return nil, err + } + + // Verify challenge + err = util.VerifyChallenge(p.cfg.Identity, challenge, pcr.Response) + if err != nil { + return nil, err + } + + return pcr.Payload, nil +} From 7a295b664305be7ad179f6cc9cb501622e697ba8 Mon Sep 17 00:00:00 2001 From: Thiago Figueiredo Date: Tue, 15 Sep 2020 17:07:08 -0300 Subject: [PATCH 052/449] minor code cleanup --- politeiad/politeiad.go | 5 ++-- politeiawww/politeiad.go | 50 +++++++++++++++++++++++++++++++++------- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 7ae9a3ec1..ba79abd56 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -854,16 +854,15 @@ func (p *politeia) updateUnvettedMetadata(w http.ResponseWriter, r *http.Request contentErr.ErrorContext) return } - // Generic internal error. errorCode := time.Now().Unix() - log.Errorf("%v Update unvetted metadata error code %v: %v", + log.Errorf("%v update unvetted metadata error code %v: %v", remoteAddr(r), errorCode, err) p.respondWithServerError(w, errorCode) return } - // Reply with challenge response + // Prepare reply response := p.identity.SignMessage(challenge) reply := v1.UpdateUnvettedMetadataReply{ Response: hex.EncodeToString(response[:]), diff --git a/politeiawww/politeiad.go b/politeiawww/politeiad.go index 10d915e8c..cc5288047 100644 --- a/politeiawww/politeiad.go +++ b/politeiawww/politeiad.go @@ -407,8 +407,42 @@ func (p *politeiawww) getVettedLatest(token string) (*pd.Record, error) { } // pluginInventory requests the plugin inventory from politeiad and returns +// inventoryByStatus retrieves the censorship record tokens filtered by status. +func (p *politeiawww) inventoryByStatus() (pd.InventoryByStatusReply, error) { + // Setup request + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return pd.InventoryByStatusReply{}, err + } + ibs := pd.InventoryByStatus{ + Challenge: hex.EncodeToString(challenge), + } + + // Send request + resBody, err := p.makeRequest(http.MethodPost, pd.InventoryByStatusRoute, ibs) + if err != nil { + return pd.InventoryByStatusReply{}, err + } + + // Receive reply + var ibsr pd.InventoryByStatusReply + err = json.Unmarshal(resBody, &ibsr) + if err != nil { + return pd.InventoryByStatusReply{}, err + } + + // Verify challenge + err = util.VerifyChallenge(p.cfg.Identity, challenge, ibsr.Response) + if err != nil { + return pd.InventoryByStatusReply{}, err + } + + return ibsr, nil +} + +// pluginInventory2 requests the plugin inventory from politeiad and returns // the available plugins slice. -func (p *politeiawww) pluginInventory() ([]pd.Plugin, error) { +func (p *politeiawww) pluginInventory2() ([]pd.Plugin, error) { // Setup request challenge, err := util.Random(pd.ChallengeSize) if err != nil { @@ -419,7 +453,7 @@ func (p *politeiawww) pluginInventory() ([]pd.Plugin, error) { } // Send request - resBody, err := p.makeRequest(http.MethodPost, pd.PluginInventoryRoute, pc) + resBody, err := p.makeRequest(http.MethodPost, pd.PluginInventoryRoute, pi) if err != nil { return nil, err } @@ -440,13 +474,13 @@ func (p *politeiawww) pluginInventory() ([]pd.Plugin, error) { return pir.Plugins, nil } -// pluginCommand fires a plugin command on politeiad and returns the reply +// pluginCommand fires a plugin command on politeiad and returns the reply // payload. -func (p *politeiawww) pluginCommand(pluginID, cmd, cmdID, payload string) (payload string, error) { +func (p *politeiawww) pluginCommand(pluginID, cmd, cmdID, payload string) (string, error) { // Setup request challenge, err := util.Random(pd.ChallengeSize) if err != nil { - return nil, err + return "", err } pc := pd.PluginCommand{ Challenge: hex.EncodeToString(challenge), @@ -459,20 +493,20 @@ func (p *politeiawww) pluginCommand(pluginID, cmd, cmdID, payload string) (paylo // Send request resBody, err := p.makeRequest(http.MethodPost, pd.PluginCommandRoute, pc) if err != nil { - return nil, err + return "", err } // Receive reply var pcr pd.PluginCommandReply err = json.Unmarshal(resBody, &pcr) if err != nil { - return nil, err + return "", err } // Verify challenge err = util.VerifyChallenge(p.cfg.Identity, challenge, pcr.Response) if err != nil { - return nil, err + return "", err } return pcr.Payload, nil From faaeada075aa33e222d4d4e32f5de97848e75d34 Mon Sep 17 00:00:00 2001 From: Thiago Figueiredo Date: Tue, 15 Sep 2020 17:38:00 -0300 Subject: [PATCH 053/449] move pluginInventory func to correct place; overall styling nits --- politeiawww/plugin.go | 38 ------------------------------------ politeiawww/politeiad.go | 42 +++++++++++++++++++++++----------------- 2 files changed, 24 insertions(+), 56 deletions(-) diff --git a/politeiawww/plugin.go b/politeiawww/plugin.go index f2eb417e9..a5b9265b8 100644 --- a/politeiawww/plugin.go +++ b/politeiawww/plugin.go @@ -73,44 +73,6 @@ func (p *politeiawww) getBestBlock() (uint64, error) { return bestBlock, nil } -func (p *politeiawww) pluginInventory() ([]Plugin, error) { - // Setup politeiad request - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - - pi := pd.PluginInventory{ - Challenge: hex.EncodeToString(challenge), - } - - // Send politeiad request - responseBody, err := p.makeRequest(http.MethodPost, - pd.PluginInventoryRoute, pi) - if err != nil { - return nil, fmt.Errorf("makeRequest: %v", err) - } - - // Handle response - var reply pd.PluginInventoryReply - err = json.Unmarshal(responseBody, &reply) - if err != nil { - return nil, err - } - - err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) - if err != nil { - return nil, err - } - - plugins := make([]Plugin, 0, len(reply.Plugins)) - for _, v := range reply.Plugins { - plugins = append(plugins, convertPluginFromPD(v)) - } - - return plugins, nil -} - // getPluginInventory obtains the politeiad plugin inventory. If a politeiad // connection cannot be made, the call will be retried every 5 seconds for up // to 1000 tries. diff --git a/politeiawww/politeiad.go b/politeiawww/politeiad.go index cc5288047..2f6ba1e55 100644 --- a/politeiawww/politeiad.go +++ b/politeiawww/politeiad.go @@ -196,14 +196,14 @@ func (p *politeiawww) updateUnvettedMetadata(token string, mdAppend, mdOverwrite } // Receive reply - var urr pd.UpdateUnvettedMetadataReply - err = json.Unmarshal(resBody, &urr) + var uumr pd.UpdateUnvettedMetadataReply + err = json.Unmarshal(resBody, &uumr) if err != nil { return err } // Verify challenge - err = util.VerifyChallenge(p.cfg.Identity, challenge, urr.Response) + err = util.VerifyChallenge(p.cfg.Identity, challenge, uumr.Response) if err != nil { return err } @@ -218,7 +218,7 @@ func (p *politeiawww) updateVettedMetadata(token string, mdAppend, mdOverwrite [ if err != nil { return err } - uum := pd.UpdateVettedMetadata{ + uvm := pd.UpdateVettedMetadata{ Challenge: hex.EncodeToString(challenge), Token: token, MDAppend: mdAppend, @@ -227,20 +227,20 @@ func (p *politeiawww) updateVettedMetadata(token string, mdAppend, mdOverwrite [ // Send request resBody, err := p.makeRequest(http.MethodPost, - pd.UpdateVettedMetadataRoute, uum) + pd.UpdateVettedMetadataRoute, uvm) if err != nil { return nil } // Receive reply - var urr pd.UpdateVettedMetadataReply - err = json.Unmarshal(resBody, &urr) + var uvmr pd.UpdateVettedMetadataReply + err = json.Unmarshal(resBody, &uvmr) if err != nil { return err } // Verify challenge - err = util.VerifyChallenge(p.cfg.Identity, challenge, urr.Response) + err = util.VerifyChallenge(p.cfg.Identity, challenge, uvmr.Response) if err != nil { return err } @@ -255,7 +255,7 @@ func (p *politeiawww) setUnvettedStatus(token string, status pd.RecordStatusT, m if err != nil { return err } - suv := pd.SetUnvettedStatus{ + sus := pd.SetUnvettedStatus{ Challenge: hex.EncodeToString(challenge), Token: token, Status: status, @@ -265,20 +265,20 @@ func (p *politeiawww) setUnvettedStatus(token string, status pd.RecordStatusT, m // Send request resBody, err := p.makeRequest(http.MethodPost, pd.SetUnvettedStatusRoute, - suv) + sus) if err != nil { return err } // Receive reply - var suvr pd.SetUnvettedStatusReply - err = json.Unmarshal(resBody, &suvr) + var susr pd.SetUnvettedStatusReply + err = json.Unmarshal(resBody, &susr) if err != nil { return err } // Verify challenge - err = util.VerifyChallenge(p.cfg.Identity, challenge, suvr.Response) + err = util.VerifyChallenge(p.cfg.Identity, challenge, susr.Response) if err != nil { return err } @@ -309,14 +309,14 @@ func (p *politeiawww) setVettedStatus(token string, status pd.RecordStatusT, mdA } // Receive reply - var svvr pd.SetVettedStatusReply - err = json.Unmarshal(resBody, &svvr) + var svsr pd.SetVettedStatusReply + err = json.Unmarshal(resBody, &svsr) if err != nil { return err } // Verify challenge - err = util.VerifyChallenge(p.cfg.Identity, challenge, svvr.Response) + err = util.VerifyChallenge(p.cfg.Identity, challenge, svsr.Response) if err != nil { return err } @@ -442,7 +442,7 @@ func (p *politeiawww) inventoryByStatus() (pd.InventoryByStatusReply, error) { // pluginInventory2 requests the plugin inventory from politeiad and returns // the available plugins slice. -func (p *politeiawww) pluginInventory2() ([]pd.Plugin, error) { +func (p *politeiawww) pluginInventory() ([]Plugin, error) { // Setup request challenge, err := util.Random(pd.ChallengeSize) if err != nil { @@ -471,7 +471,13 @@ func (p *politeiawww) pluginInventory2() ([]pd.Plugin, error) { return nil, err } - return pir.Plugins, nil + // Convert politeiad plugin types + plugins := make([]Plugin, 0, len(pir.Plugins)) + for _, v := range pir.Plugins { + plugins = append(plugins, convertPluginFromPD(v)) + } + + return plugins, nil } // pluginCommand fires a plugin command on politeiad and returns the reply From af847ac30bcb3af5444333db23dc5e1108e476cd Mon Sep 17 00:00:00 2001 From: Thiago Figueiredo Date: Tue, 15 Sep 2020 17:49:58 -0300 Subject: [PATCH 054/449] solve conflict --- politeiawww/politeiad.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/politeiawww/politeiad.go b/politeiawww/politeiad.go index 2f6ba1e55..d193a3215 100644 --- a/politeiawww/politeiad.go +++ b/politeiawww/politeiad.go @@ -440,7 +440,7 @@ func (p *politeiawww) inventoryByStatus() (pd.InventoryByStatusReply, error) { return ibsr, nil } -// pluginInventory2 requests the plugin inventory from politeiad and returns +// pluginInventory requests the plugin inventory from politeiad and returns // the available plugins slice. func (p *politeiawww) pluginInventory() ([]Plugin, error) { // Setup request From 06dc4d61a7e0b94bdb602fe85569d1afe87fde9a Mon Sep 17 00:00:00 2001 From: amass Date: Wed, 16 Sep 2020 00:20:57 +0300 Subject: [PATCH 055/449] politeiawww: add processBatchProposals --- politeiawww/proposals.go | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index ce92608d8..6f6a84e5e 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -351,7 +351,34 @@ func (p *politeiawww) processBatchVoteSummary(bvs www.BatchVoteSummary) (*www.Ba } func (p *politeiawww) processBatchProposals(bp www.BatchProposals, u *user.User) (*www.BatchProposalsReply, error) { - return nil, nil + log.Tracef("processBatchProposals: %v", bp.Tokens) + + // Prep proposals requests + prs := make([]pi.ProposalRequest, len(bp.Tokens)) + for _, t := range bp.Tokens { + prs = append(prs, pi.ProposalRequest{ + Token: t, + }) + } + + props, err := p.proposalRecords(pi.PropStateVetted, prs, false) + if err != nil { + return nil, err + } + + // Convert proposals records + propsw := make([]www.ProposalRecord, len(bp.Tokens)) + for _, pr := range props { + propw, err := p.convertProposalToWWW(&pr) + if err != nil { + return nil, err + } + propsw = append(propsw, *propw) + } + + return &www.BatchProposalsReply{ + Proposals: propsw, + }, nil } // getUserProps gets the latest version of all proposals from the cache and From 6dbdc01859e756c943c30db1bbb46f5d84b01117 Mon Sep 17 00:00:00 2001 From: amass Date: Wed, 16 Sep 2020 00:29:01 +0300 Subject: [PATCH 056/449] add comments regarding proposals state --- politeiawww/api/www/v1/api.md | 2 +- politeiawww/api/www/v1/v1.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/politeiawww/api/www/v1/api.md b/politeiawww/api/www/v1/api.md index 44a6c59a1..d14967195 100644 --- a/politeiawww/api/www/v1/api.md +++ b/politeiawww/api/www/v1/api.md @@ -1588,7 +1588,7 @@ Reply: Retrieve the proposal details for a list of proposals. This route wil not return the proposal files. The number of proposals that may be requested is limited by the `ProposalListPageSize` property, which is provided via -[`Policy`](#policy). +[`Policy`](#policy). This route will return only vetted proposals. **Routes:** `POST /v1/proposals/batch` diff --git a/politeiawww/api/www/v1/v1.go b/politeiawww/api/www/v1/v1.go index 4ee63d70d..106ae0cec 100644 --- a/politeiawww/api/www/v1/v1.go +++ b/politeiawww/api/www/v1/v1.go @@ -872,7 +872,7 @@ type ProposalDetailsReply struct { // BatchProposals is used to request the proposal details for each of the // provided censorship tokens. The returned proposals do not include the -// proposal files. +// proposal files. Returns only vetted proposals. type BatchProposals struct { Tokens []string `json:"tokens"` // Censorship tokens } From ef245a192bebed98a302af6aac6274797aa78e61 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 15 Sep 2020 15:15:04 -0500 Subject: [PATCH 057/449] remove dead comment code --- politeiad/api/v1/v1.go | 8 +- politeiad/backend/tlogbe/tlogbe.go | 34 ++++- politeiad/testpoliteiad/testpoliteiad.go | 2 +- politeiawww/comments.go | 162 ----------------------- politeiawww/dcc.go | 25 +++- politeiawww/decred.go | 37 ------ politeiawww/invoices.go | 32 +++-- politeiawww/piwww.go | 21 ++- politeiawww/politeiawww.go | 1 - politeiawww/templates.go | 19 ++- politeiawww/testing.go | 4 +- politeiawww/www.go | 1 - util/convert.go | 10 +- 13 files changed, 111 insertions(+), 245 deletions(-) diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index d439b7435..3a4acb99d 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -37,8 +37,12 @@ const ( PluginInventoryRoute = PluginCommandRoute + "inventory/" // Inventory all plugins ChallengeSize = 32 // Size of challenge token in bytes - // TODO TokenSize needs to be updated - TokenSize = 32 // Size of token + + // Token sizes. The size of the token depends on the politeiad + // backend configuration, but will always be within this range. + TokenSizeMin = 10 + TokenSizeMax = 32 + MetadataStreamsMax = uint64(16) // Maximum number of metadata streams // Error status codes diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index e95fbf43d..249428521 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -18,10 +18,15 @@ import ( "time" "github.com/decred/dcrtime/merkle" + "github.com/decred/politeia/plugins/comments" + "github.com/decred/politeia/plugins/dcrdata" + "github.com/decred/politeia/plugins/pi" + "github.com/decred/politeia/plugins/ticketvote" pd "github.com/decred/politeia/politeiad/api/v1" v1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/mime" "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/util" "github.com/marcopeereboom/sbox" "github.com/subosito/gozaru" @@ -1302,18 +1307,39 @@ func (t *TlogBackend) InventoryByStatus() (*backend.InventoryByStatus, error) { }, nil } -// TODO func (t *TlogBackend) RegisterPlugin(p backend.Plugin) error { log.Tracef("RegisterPlugin: %v", p.ID) - return fmt.Errorf("not implemented") + var ctx Plugin + switch p.ID { + case comments.ID: + ctx = plugins.NewCommentsPlugin() + case dcrdata.ID: + case pi.ID: + case ticketvote.ID: + default: + return backend.ErrPluginInvalid + } + + t.plugins[p.ID] = plugin{ + id: p.ID, + version: p.Version, + settings: p.Settings, + ctx: ctx, + } + + return nil } -// TODO func (t *TlogBackend) SetupPlugin(pluginID string) error { log.Tracef("SetupPlugin: %v", pluginID) - return fmt.Errorf("not implemented") + plugin, ok := t.plugins[pluginID] + if !ok { + return backend.ErrPluginInvalid + } + + return plugin.ctx.Setup() } // GetPlugins returns the backend plugins that have been registered and their diff --git a/politeiad/testpoliteiad/testpoliteiad.go b/politeiad/testpoliteiad/testpoliteiad.go index f7223708b..14e480dc2 100644 --- a/politeiad/testpoliteiad/testpoliteiad.go +++ b/politeiad/testpoliteiad/testpoliteiad.go @@ -159,7 +159,7 @@ func (p *TestPoliteiad) handleNewRecord(w http.ResponseWriter, r *http.Request) } // Prepare response - tokenb, err := util.Random(v1.TokenSize) + tokenb, err := util.Random(v1.TokenSizeMax) if err != nil { util.RespondWithJSON(w, http.StatusInternalServerError, err) return diff --git a/politeiawww/comments.go b/politeiawww/comments.go index 4c4abb01d..6410c2449 100644 --- a/politeiawww/comments.go +++ b/politeiawww/comments.go @@ -5,171 +5,10 @@ package main import ( - "fmt" - www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/user" ) -// counters is a struct that helps us keep track of up/down votes. -type counters struct { - up uint64 - down uint64 -} - -// getComment retreives the specified comment from the cache then fills in -// politeiawww specific data for the comment. -func (p *politeiawww) getComment(token, commentID string) (*www.Comment, error) { - // Fetch comment from the cache - dc, err := p.decredCommentGetByID(token, commentID) - if err != nil { - return nil, fmt.Errorf("decredGetComment: %v", err) - } - c := convertCommentFromDecred(*dc) - - // Lookup author info - u, err := p.db.UserGetByPubKey(c.PublicKey) - if err != nil { - log.Errorf("getComment: UserGetByPubKey: token:%v commentID:%v "+ - "pubKey:%v err:%v", token, commentID, c.PublicKey, err) - } else { - c.UserID = u.ID.String() - c.Username = u.Username - } - - // Lookup comment votes - votes, err := p.getCommentVotes(token, commentID) - if err != nil { - return nil, err - } - c.ResultVotes = int64(votes.up - votes.down) - c.Upvotes = votes.up - c.Downvotes = votes.down - - return &c, nil -} - -// getCommentVotes tries to get comment votes from the cache. If votes are -// not stored, fetch them and update cache. -// -// This function must be called WITHOUT the lock held. -func (p *politeiawww) getCommentVotes(token, commentID string) (counters, error) { - log.Tracef("getCommentVotes: %v %v", token, commentID) - - // Check if comment votes is already cached - var votes counters - p.RLock() - vs, ok := p.commentVotes[token+commentID] - p.RUnlock() - votes = vs - // If not in cache, fetch comment votes and update cache - if !ok { - vsUpdated, err := p.updateCommentVotes(token, commentID) - if err != nil { - log.Errorf("getCommentVotes: comment votes update "+ - "failed: token:%v commentID:%v", token, commentID) - return counters{}, err - } - votes = *vsUpdated - } - - return votes, nil -} - -// updateCommentVotes calculates the up/down votes for the specified comment, -// updates the in-memory comment votes cache with these and returns them. -func (p *politeiawww) updateCommentVotes(token, commentID string) (*counters, error) { - log.Tracef("updateCommentVotes: %v %v", token, commentID) - - /* - // Fetch all comment likes for the specified comment - likes, err := p.decredCommentLikes(token, commentID) - if err != nil { - return nil, fmt.Errorf("decredLikeComments: %v", err) - } - - // Sanity check. Like comments should already be sorted in - // chronological order. - sort.SliceStable(likes, func(i, j int) bool { - return likes[i].Timestamp < likes[j].Timestamp - }) - - p.Lock() - defer p.Unlock() - - // Compute the comment votes. We have to keep track of each user's most - // recent like action because the net effect of an upvote/downvote is - // dependent on the user's previous action. - // Example: a user upvoting a comment twice results in a net score of 0 - // because the second upvote is actually the user taking away their original - // upvote. - var votes counters - userActions := make(map[string]string) // [userID]action - for _, v := range likes { - // Lookup the userID of the comment author - u, err := p.db.UserGetByPubKey(v.PublicKey) - if err != nil { - return nil, fmt.Errorf("user lookup failed for pubkey %v", - v.PublicKey) - } - userID := u.ID.String() - - // Lookup the previous like comment action that the author - // made on this comment - prevAction := userActions[userID] - - switch { - case prevAction == "": - // No previous action so we add the new action to the - // vote score - switch v.Action { - case www.VoteActionDown: - votes.down += 1 - case www.VoteActionUp: - votes.up += 1 - } - userActions[userID] = v.Action - - case prevAction == v.Action: - // New action is the same as the previous action so we - // remove the previous action from the vote score - switch prevAction { - case www.VoteActionDown: - votes.down -= 1 - case www.VoteActionUp: - votes.up -= 1 - } - delete(userActions, userID) - - case prevAction != v.Action: - // New action is different than the previous action so - // we remove the previous action from the vote score.. - switch prevAction { - case www.VoteActionDown: - votes.down -= 1 - case www.VoteActionUp: - votes.up -= 1 - } - - // ..and then add the new action to the vote score - switch v.Action { - case www.VoteActionDown: - votes.down += 1 - case www.VoteActionUp: - votes.up += 1 - } - userActions[userID] = v.Action - } - } - - // Update in-memory cache - p.commentVotes[token+commentID] = votes - - return &votes, nil - */ - return nil, nil -} - func validateComment(c www.NewComment) error { // max length if len(c.Comment) > www.PolicyMaxCommentLength { @@ -269,7 +108,6 @@ func (p *politeiawww) processNewComment(nc www.NewComment, u *user.User) (*www.N return nil, www.UserError{ ErrorCode: www.ErrorStatusDuplicateComment, } - // TODO case cache.ErrRecordNotFound: // No duplicate comment; continue default: // Some other error diff --git a/politeiawww/dcc.go b/politeiawww/dcc.go index f594747ad..60b59e51e 100644 --- a/politeiawww/dcc.go +++ b/politeiawww/dcc.go @@ -783,8 +783,8 @@ func (p *politeiawww) processNewCommentDCC(nc www.NewComment, u *user.User) (*ww return nil, err } - // Get comment from cache - c, err := p.getComment(nc.Token, ncr.CommentID) + // Get comment + c, err := p.getDCCComment(nc.Token, ncr.CommentID) if err != nil { return nil, fmt.Errorf("getComment: %v", err) } @@ -854,6 +854,27 @@ func (p *politeiawww) getDCCComments(token string) ([]www.Comment, error) { return comments, nil } +// getDCCComment retrieves a comment from politeiad using the decred plugin +// command then fills in the missing user information. +func (p *politeiawww) getDCCComment(token, commentID string) (*www.Comment, error) { + // Fetch comment + dc, err := p.decredCommentGetByID(token, commentID) + if err != nil { + return nil, fmt.Errorf("decredGetComment: %v", err) + } + c := convertCommentFromDecred(*dc) + + // Lookup author info + u, err := p.db.UserGetByPubKey(c.PublicKey) + if err != nil { + return nil, fmt.Errorf("UserGetbyPubKey: %v", err) + } + c.UserID = u.ID.String() + c.Username = u.Username + + return &c, nil +} + func (p *politeiawww) processSetDCCStatus(sds cms.SetDCCStatus, u *user.User) (*cms.SetDCCStatusReply, error) { log.Tracef("processSetDCCStatus: %v", u.PublicKey()) diff --git a/politeiawww/decred.go b/politeiawww/decred.go index 2b83ea635..87c333bfc 100644 --- a/politeiawww/decred.go +++ b/politeiawww/decred.go @@ -40,15 +40,6 @@ func (p *politeiawww) decredCommentGetByID(token, commentID string) (*decredplug } // decredCommentGetBySignature retrieves the specified decred plugin comment -// from the cache. -func (p *politeiawww) decredCommentGetBySignature(token, sig string) (*decredplugin.Comment, error) { - gc := decredplugin.GetComment{ - Token: token, - Signature: sig, - } - return p.decredGetComment(gc) -} - // decredGetComments sends the decred plugin getcomments command to the cache // and returns all of the comments for the passed in proposal token. func (p *politeiawww) decredGetComments(token string) ([]decredplugin.Comment, error) { @@ -72,31 +63,3 @@ func (p *politeiawww) decredGetComments(token string) ([]decredplugin.Comment, e return gcr.Comments, nil } - -// decredGetNumComments sends the decred plugin command GetNumComments to the -// cache and returns the number of comments for each of the specified -// proposals. If a provided token does not correspond to an actual proposal -// then it will not be included in the returned map. It is the responability -// of the caller to ensure results are returned for all of the provided tokens. -func (p *politeiawww) decredGetNumComments(tokens []string) (map[string]int, error) { - // Setup plugin command - gnc := decredplugin.GetNumComments{ - Tokens: tokens, - } - - payload, err := decredplugin.EncodeGetNumComments(gnc) - if err != nil { - return nil, err - } - - // TODO this needs to use the politeiad plugin command - _ = payload - var reply string - - gncr, err := decredplugin.DecodeGetNumCommentsReply([]byte(reply)) - if err != nil { - return nil, err - } - - return gncr.NumComments, nil -} diff --git a/politeiawww/invoices.go b/politeiawww/invoices.go index 9dcffaf39..d349bc231 100644 --- a/politeiawww/invoices.go +++ b/politeiawww/invoices.go @@ -347,7 +347,7 @@ func (p *politeiawww) processNewInvoice(ni cms.NewInvoice, u *user.User) (*cms.N // Handle test case if p.test { - tokenBytes, err := util.Random(pd.TokenSize) + tokenBytes, err := util.Random(pd.TokenSizeMax) if err != nil { return nil, err } @@ -1660,13 +1660,8 @@ func (p *politeiawww) processNewCommentInvoice(nc www.NewComment, u *user.User) return nil, err } - // Add comment to commentVotes in-memory cache - p.Lock() - p.commentVotes[nc.Token+ncr.CommentID] = counters{} - p.Unlock() - - // Get comment from cache - c, err := p.getComment(nc.Token, ncr.CommentID) + // Get comment + c, err := p.getInvoiceComment(nc.Token, ncr.CommentID) if err != nil { return nil, fmt.Errorf("getComment: %v", err) } @@ -1768,6 +1763,27 @@ func (p *politeiawww) getInvoiceComments(token string) ([]www.Comment, error) { return comments, nil } +// getInvoiceComment retrieves an invoice comment from politeiad using the +// decred plugin command then fills in the missing user information. +func (p *politeiawww) getInvoiceComment(token, commentID string) (*www.Comment, error) { + // Fetch comment + dc, err := p.decredCommentGetByID(token, commentID) + if err != nil { + return nil, fmt.Errorf("decredGetComment: %v", err) + } + c := convertCommentFromDecred(*dc) + + // Lookup author info + u, err := p.db.UserGetByPubKey(c.PublicKey) + if err != nil { + return nil, fmt.Errorf("UserGetbyPubKey: %v", err) + } + c.UserID = u.ID.String() + c.Username = u.Username + + return &c, nil +} + // processPayInvoices looks for all approved invoices and then goes about // changing their statuses' to paid. func (p *politeiawww) processPayInvoices(u *user.User) (*cms.PayInvoicesReply, error) { diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 834dc30ca..ac477ef75 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -56,11 +56,18 @@ var ( // Short tokens should only be used when retrieving data. Data that is written // to disk should always reference the full length token. func tokenIsValid(token string) bool { - b, err := hex.DecodeString(token) - if err != nil { + switch { + case len(token) == pd.TokenPrefixLength: + // Token is a short proposal token + case len(token) == pd.TokenSizeMin*2: + // Token is a full length token + default: + // Unknown token size return false } - if len(b) != pd.TokenSize { + _, err := hex.DecodeString(token) + if err != nil { + // Token is not valid hex return false } return true @@ -74,7 +81,7 @@ func tokenIsFullLength(token string) bool { if err != nil { return false } - if len(b) != pd.TokenSize { + if len(b) != pd.TokenSizeMin { return false } return true @@ -128,7 +135,7 @@ func proposalName(pr pi.ProposalRecord) string { return name } -func convertUserErrFromSignatureErr(err error) pi.UserErrorReply { +func convertUserErrorFromSignatureError(err error) pi.UserErrorReply { var e util.SignatureError var s pi.ErrorStatusT if errors.As(err, &e) { @@ -830,7 +837,7 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu } err = util.VerifySignature(signature, publicKey, mr) if err != nil { - return nil, convertUserErrFromSignatureErr(err) + return nil, convertUserErrorFromSignatureError(err) } return &pm, nil @@ -1126,7 +1133,7 @@ func (p *politeiawww) processProposalSetStatus(pss pi.ProposalSetStatus, usr use msg := pss.Token + pss.Version + strconv.Itoa(int(pss.Status)) + pss.Reason err := util.VerifySignature(pss.Signature, pss.PublicKey, msg) if err != nil { - return nil, convertUserErrFromSignatureErr(err) + return nil, convertUserErrorFromSignatureError(err) } // Verification that requires retrieving the existing proposal is diff --git a/politeiawww/politeiawww.go b/politeiawww/politeiawww.go index 90cf557fa..2669cda4b 100644 --- a/politeiawww/politeiawww.go +++ b/politeiawww/politeiawww.go @@ -90,7 +90,6 @@ type politeiawww struct { // Following entries require locks userPaywallPool map[uuid.UUID]paywallPoolMember // [userid][paywallPoolMember] - commentVotes map[string]counters // [token+commentID]counters // XXX userEmails is a temporary measure until the user by email // lookups are completely removed from politeiawww. diff --git a/politeiawww/templates.go b/politeiawww/templates.go index 2df757066..1feb07acc 100644 --- a/politeiawww/templates.go +++ b/politeiawww/templates.go @@ -33,9 +33,8 @@ A new proposal has been submitted on Politeia by {{.Username}}: {{.Link}} ` -var tmplProposalSubmitted = template.Must( - template.New("proposal_submitted"). - Parse(tmplTextProposalSubmitted)) +var tmplProposalSubmitted = template.Must(template.New("proposalSubmitted"). + Parse(tmplTextProposalSubmitted)) // Proposal edited type tmplDataProposalEdited struct { @@ -52,9 +51,8 @@ A proposal by {{.Username}} has just been edited: {{.Link}} ` -var tmplProposalEdited = template.Must( - template.New("proposal_edited"). - Parse(tmplTextProposalEdited)) +var tmplProposalEdited = template.Must(template.New("proposalEdited"). + Parse(tmplTextProposalEdited)) // Proposal status change - Vetted - Send to author type tmplDataProposalVettedForAuthor struct { @@ -78,7 +76,7 @@ before authorizing the vote. ` var tmplProposalVettedForAuthor = template.Must( - template.New("proposal_vetted_for_author"). + template.New("proposalVettedForAuthor"). Parse(tmplTextProposalVettedForAuthor)) // Proposal status change - Censored - Send to author @@ -97,7 +95,7 @@ Reason: {{.Reason}} ` var tmplProposalCensoredForAuthor = template.Must( - template.New("proposal_censored_for_author"). + template.New("proposalCensoredForAuthor"). Parse(tmplTextProposalCensoredForAuthor)) // Proposal status change - Vetted - Send to users @@ -113,9 +111,8 @@ A new proposal has just been published on Politeia. {{.Link}} ` -var tmplProposalVetted = template.Must( - template.New("proposal_vetted"). - Parse(tmplTextProposalVetted)) +var tmplProposalVetted = template.Must(template.New("proposalVetted"). + Parse(tmplTextProposalVetted)) type invoiceNotificationEmailData struct { Username string diff --git a/politeiawww/testing.go b/politeiawww/testing.go index b25860678..c79011306 100644 --- a/politeiawww/testing.go +++ b/politeiawww/testing.go @@ -531,7 +531,7 @@ func newProposalRecord(t *testing.T, u *user.User, id *identity.FullIdentity, s abandonedAt = time.Now().Unix() } - tokenb, err := util.Random(pd.TokenSize) + tokenb, err := util.Random(pd.TokenSizeMax) if err != nil { t.Fatal(err) } @@ -699,7 +699,6 @@ func newTestPoliteiawww(t *testing.T) (*politeiawww, func()) { test: true, userEmails: make(map[string]uuid.UUID), userPaywallPool: make(map[uuid.UUID]paywallPoolMember), - commentVotes: make(map[string]counters), } // Setup routes @@ -805,7 +804,6 @@ func newTestCMSwww(t *testing.T) (*politeiawww, func()) { test: true, userEmails: make(map[string]uuid.UUID), userPaywallPool: make(map[uuid.UUID]paywallPoolMember), - commentVotes: make(map[string]counters), } // Setup routes diff --git a/politeiawww/www.go b/politeiawww/www.go index 087ffa7b9..f010ff32b 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -618,7 +618,6 @@ func _main() error { // XXX reevaluate where this goes userEmails: make(map[string]uuid.UUID), userPaywallPool: make(map[uuid.UUID]paywallPoolMember), - commentVotes: make(map[string]counters), params: activeNetParams.Params, } diff --git a/util/convert.go b/util/convert.go index b69fce20a..f861464ef 100644 --- a/util/convert.go +++ b/util/convert.go @@ -33,12 +33,10 @@ func ConvertSignature(s string) ([identity.SignatureSize]byte, error) { // ConvertStringToken verifies and converts a string token to a proper sized // []byte. func ConvertStringToken(token string) ([]byte, error) { - // TODO add new token size - /* - if len(token) != pd.TokenSize*2 { - return nil, fmt.Errorf("invalid censorship token size") - } - */ + if len(token) > pd.TokenSizeMax*2 || + len(token) < pd.TokenSizeMin*2 { + return nil, fmt.Errorf("invalid censorship token size") + } blob, err := hex.DecodeString(token) if err != nil { return nil, err From f7d3c7baa73a852bf9568a5e108b358e0021ffc6 Mon Sep 17 00:00:00 2001 From: Thiago Figueiredo Date: Tue, 15 Sep 2020 18:44:54 -0300 Subject: [PATCH 058/449] address review comments --- politeiad/api/v1/v1.go | 18 +++++++++--------- politeiad/politeiad.go | 11 +++++++++-- politeiawww/politeiad.go | 22 ++++++++-------------- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index 58e318a61..25d69cf73 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -239,8 +239,8 @@ type NewRecord struct { // NewRecordReply returns the CensorshipRecord that is associated with a valid // record. A valid record is not always going to be published. type NewRecordReply struct { - Response string `json:"response"` // Challenge response - CensorshipRecord CensorshipRecord `json:"censorshiprecord"` + Response string `json:"response"` // Challenge response + CensorshipRecord CensorshipRecord `json:"censorshiprecord"` // Censorship record } // GetUnvetted requests an unvetted record from the server. @@ -319,8 +319,8 @@ type UpdateRecord struct { // UpdateRecordReply returns a CensorshipRecord which may or may not have // changed. Metadata only updates do not create a new CensorshipRecord. type UpdateRecordReply struct { - Response string `json:"response"` // Challenge response - Token string `json:"token"` // Censorship token + Response string `json:"response"` // Challenge response + CensorshipRecord CensorshipRecord `json:"censorshiprecord"` // Censorship record } // UpdateVettedMetadata update a vetted metadata. This is allowed for @@ -387,11 +387,11 @@ type InventoryByStatus struct { // InventoryByStatusReply returns all censorship record tokens by status. type InventoryByStatusReply struct { Response string `json:"response"` // Challenge response - Unvetted []string `json:"unvetted"` // Unvetted censorship tokens - IterationUnvetted []string `json:"iterationunvetted"` // Iteration unvetted censorship tokens - Vetted []string `json:"vetted"` // Vetted censorship tokens - Censored []string `json:"censored"` // Censored censorship tokens - Archived []string `json:"archived"` // Archived censorship tokens + Unvetted []string `json:"unvetted"` // Unvetted tokens + IterationUnvetted []string `json:"iterationunvetted"` // Iteration unvetted tokens + Vetted []string `json:"vetted"` // Vetted tokens + Censored []string `json:"censored"` // Censored tokens + Archived []string `json:"archived"` // Archived tokens } // UserErrorReply returns details about an error that occurred while trying to diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index ba79abd56..ea5d300b6 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -366,10 +366,17 @@ func (p *politeia) updateRecord(w http.ResponseWriter, r *http.Request, vetted b } // Prepare reply. + merkle := record.RecordMetadata.Merkle + signature := p.identity.SignMessage([]byte(merkle + t.Token)) response := p.identity.SignMessage(challenge) + reply := v1.UpdateRecordReply{ Response: hex.EncodeToString(response[:]), - Token: t.Token, + CensorshipRecord: v1.CensorshipRecord{ + Merkle: merkle, + Token: t.Token, + Signature: hex.EncodeToString(signature[:]), + }, } log.Infof("Update %v record %v: token %v", cmd, remoteAddr(r), @@ -413,7 +420,7 @@ func (p *politeia) getUnvetted(w http.ResponseWriter, r *http.Request) { } // Ask backend about the censorship token. - bpr, err := p.backend.GetUnvetted(token, "") + bpr, err := p.backend.GetUnvetted(token, t.Version) if err == backend.ErrRecordNotFound { reply.Record.Status = v1.RecordStatusNotFound log.Errorf("Get unvetted record %v: token %v not found", diff --git a/politeiawww/politeiad.go b/politeiawww/politeiad.go index d193a3215..5ad2945fa 100644 --- a/politeiawww/politeiad.go +++ b/politeiawww/politeiad.go @@ -408,11 +408,11 @@ func (p *politeiawww) getVettedLatest(token string) (*pd.Record, error) { // pluginInventory requests the plugin inventory from politeiad and returns // inventoryByStatus retrieves the censorship record tokens filtered by status. -func (p *politeiawww) inventoryByStatus() (pd.InventoryByStatusReply, error) { +func (p *politeiawww) inventoryByStatus() (*pd.InventoryByStatusReply, error) { // Setup request challenge, err := util.Random(pd.ChallengeSize) if err != nil { - return pd.InventoryByStatusReply{}, err + return nil, err } ibs := pd.InventoryByStatus{ Challenge: hex.EncodeToString(challenge), @@ -421,28 +421,28 @@ func (p *politeiawww) inventoryByStatus() (pd.InventoryByStatusReply, error) { // Send request resBody, err := p.makeRequest(http.MethodPost, pd.InventoryByStatusRoute, ibs) if err != nil { - return pd.InventoryByStatusReply{}, err + return nil, err } // Receive reply var ibsr pd.InventoryByStatusReply err = json.Unmarshal(resBody, &ibsr) if err != nil { - return pd.InventoryByStatusReply{}, err + return nil, err } // Verify challenge err = util.VerifyChallenge(p.cfg.Identity, challenge, ibsr.Response) if err != nil { - return pd.InventoryByStatusReply{}, err + return nil, err } - return ibsr, nil + return &ibsr, nil } // pluginInventory requests the plugin inventory from politeiad and returns // the available plugins slice. -func (p *politeiawww) pluginInventory() ([]Plugin, error) { +func (p *politeiawww) pluginInventory() ([]pd.Plugin, error) { // Setup request challenge, err := util.Random(pd.ChallengeSize) if err != nil { @@ -471,13 +471,7 @@ func (p *politeiawww) pluginInventory() ([]Plugin, error) { return nil, err } - // Convert politeiad plugin types - plugins := make([]Plugin, 0, len(pir.Plugins)) - for _, v := range pir.Plugins { - plugins = append(plugins, convertPluginFromPD(v)) - } - - return plugins, nil + return pir.Plugins, nil } // pluginCommand fires a plugin command on politeiad and returns the reply From 7cd5fe9ccf79ff2f05fc2cc554d23189e38684cd Mon Sep 17 00:00:00 2001 From: Thiago Figueiredo Date: Tue, 15 Sep 2020 18:47:18 -0300 Subject: [PATCH 059/449] remove TODO string comment --- politeiad/api/v1/v1.go | 1 - 1 file changed, 1 deletion(-) diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index 25d69cf73..c09976f3b 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -244,7 +244,6 @@ type NewRecordReply struct { } // GetUnvetted requests an unvetted record from the server. -// TODO Implement Version. Unvetted didn't previously have a version. type GetUnvetted struct { Challenge string `json:"challenge"` // Random challenge Token string `json:"token"` // Censorship token From a42395f070bbde6e41e6e8b9f658dba15be862af Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 15 Sep 2020 17:04:44 -0500 Subject: [PATCH 060/449] cleanup --- politeiawww/convert.go | 19 --------------- politeiawww/plugin.go | 47 ++++++++++++++++++++++++++------------ politeiawww/politeiawww.go | 2 +- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/politeiawww/convert.go b/politeiawww/convert.go index f9540fe8e..20ba6abb2 100644 --- a/politeiawww/convert.go +++ b/politeiawww/convert.go @@ -302,25 +302,6 @@ func convertCastVoteFromDecred(cv decredplugin.CastVote) www.CastVote { } } -func convertPluginSettingFromPD(ps pd.PluginSetting) PluginSetting { - return PluginSetting{ - Key: ps.Key, - Value: ps.Value, - } -} - -func convertPluginFromPD(p pd.Plugin) Plugin { - ps := make([]PluginSetting, 0, len(p.Settings)) - for _, v := range p.Settings { - ps = append(ps, convertPluginSettingFromPD(v)) - } - return Plugin{ - ID: p.ID, - Version: p.Version, - Settings: ps, - } -} - func convertInvoiceCensorFromWWW(f www.CensorshipRecord) pd.CensorshipRecord { return pd.CensorshipRecord{ Token: f.Token, diff --git a/politeiawww/plugin.go b/politeiawww/plugin.go index a5b9265b8..3669cce76 100644 --- a/politeiawww/plugin.go +++ b/politeiawww/plugin.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Decred developers +// Copyright (c) 2017-2020 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -17,17 +17,36 @@ import ( "github.com/decred/politeia/util" ) -// PluginSetting is a structure that holds key/value pairs of a plugin setting. -type PluginSetting struct { +// pluginSetting is a structure that holds key/value pairs of a plugin setting. +type pluginSetting struct { Key string // Name of setting Value string // Value of setting } -// Plugin describes a plugin and its settings. -type Plugin struct { +// plugin describes a plugin and its settings. +type plugin struct { ID string // Identifier Version string // Version - Settings []PluginSetting // Settings + Settings []pluginSetting // Settings +} + +func convertPluginSettingFromPD(ps pd.PluginSetting) pluginSetting { + return pluginSetting{ + Key: ps.Key, + Value: ps.Value, + } +} + +func convertPluginFromPD(p pd.Plugin) plugin { + ps := make([]pluginSetting, 0, len(p.Settings)) + for _, v := range p.Settings { + ps = append(ps, convertPluginSettingFromPD(v)) + } + return plugin{ + ID: p.ID, + Version: p.Version, + Settings: ps, + } } // getBestBlock fetches and returns the best block from politeiad using the @@ -73,33 +92,33 @@ func (p *politeiawww) getBestBlock() (uint64, error) { return bestBlock, nil } -// getPluginInventory obtains the politeiad plugin inventory. If a politeiad +// getPluginInventory returns the politeiad plugin inventory. If a politeiad // connection cannot be made, the call will be retried every 5 seconds for up // to 1000 tries. -func (p *politeiawww) getPluginInventory() ([]Plugin, error) { - log.Tracef("getPluginInventory") - +func (p *politeiawww) getPluginInventory() ([]plugin, error) { // Attempt to fetch the plugin inventory from politeiad until // either it is successful or the maxRetries has been exceeded. var ( + done bool maxRetries = 1000 sleepInterval = 5 * time.Second - done bool - plugins []Plugin + plugins = make([]plugin, 0, 16) ) for retries := 0; !done; retries++ { if retries == maxRetries { return nil, fmt.Errorf("max retries exceeded") } - p, err := p.pluginInventory() + pi, err := p.pluginInventory() if err != nil { log.Infof("cannot get politeiad plugin inventory: %v", err) time.Sleep(sleepInterval) continue } + for _, v := range pi { + plugins = append(plugins, convertPluginFromPD(v)) + } - plugins = p done = true } diff --git a/politeiawww/politeiawww.go b/politeiawww/politeiawww.go index 2669cda4b..46448b9bc 100644 --- a/politeiawww/politeiawww.go +++ b/politeiawww/politeiawww.go @@ -64,7 +64,7 @@ type politeiawww struct { cfg *config router *mux.Router sessions sessions.Store - plugins []Plugin + plugins []plugin ws map[string]map[string]*wsContext // [uuid][]*context wsMtx sync.RWMutex From ce0ff8e70f7941b80977d7d93bbe8971826fbf57 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 15 Sep 2020 20:22:10 -0500 Subject: [PATCH 061/449] template cleanup --- politeiad/backend/tlogbe/tlogbe.go | 2 - politeiawww/email.go | 54 ++++++++++----------- politeiawww/templates.go | 77 +++++++++++++++--------------- 3 files changed, 65 insertions(+), 68 deletions(-) diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 249428521..feba4f49b 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -26,7 +26,6 @@ import ( v1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/mime" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/util" "github.com/marcopeereboom/sbox" "github.com/subosito/gozaru" @@ -1313,7 +1312,6 @@ func (t *TlogBackend) RegisterPlugin(p backend.Plugin) error { var ctx Plugin switch p.ID { case comments.ID: - ctx = plugins.NewCommentsPlugin() case dcrdata.ID: case pi.ID: case ticketvote.ID: diff --git a/politeiawww/email.go b/politeiawww/email.go index e1c434641..f74c83911 100644 --- a/politeiawww/email.go +++ b/politeiawww/email.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Decred developers +// Copyright (c) 2018-2020 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -72,7 +72,7 @@ func (p *politeiawww) emailNewUserVerificationLink(email, token, username string } subject := "Verify Your Email" - body, err := createBody(templateNewUserEmail, &tplData) + body, err := createBody(templateNewUserEmail, tplData) if err != nil { return err } @@ -111,7 +111,7 @@ func (p *politeiawww) emailResetPasswordVerificationLink(email, username, token } subject := "Reset Your Password" - body, err := createBody(templateResetPasswordEmail, &tplData) + body, err := createBody(templateResetPasswordEmail, tplData) if err != nil { return err } @@ -136,23 +136,23 @@ func (p *politeiawww) emailProposalStatusChangeToAuthor(d dataProposalStatusChan switch d.status { case pi.PropStatusPublic: subject = "Your Proposal Has Been Published" - tmplData := tmplDataProposalVettedForAuthor{ + tmplData := proposalVettedToAuthor{ Name: d.name, Link: l.String(), } - body, err = createBody(tmplProposalVettedForAuthor, &tmplData) + body, err = createBody(proposalVettedToAuthorTmpl, tmplData) if err != nil { return err } case pi.PropStatusCensored: subject = "Your Proposal Has Been Censored" - tmplData := tmplDataProposalCensoredForAuthor{ + tmplData := proposalCensoredToAuthor{ Name: d.name, Reason: d.reason, Link: l.String(), } - body, err = createBody(tmplProposalCensoredForAuthor, &tmplData) + body, err = createBody(tmplProposalCensoredForAuthor, tmplData) if err != nil { return err } @@ -180,11 +180,11 @@ func (p *politeiawww) emailProposalStatusChangeToUsers(d dataProposalStatusChang switch d.status { case pi.PropStatusPublic: subject = "New Proposal Published" - tmplData := tmplDataProposalVetted{ + tmplData := proposalVetted{ Name: d.name, Link: l.String(), } - body, err = createBody(tmplProposalVetted, &tmplData) + body, err = createBody(tmplProposalVetted, tmplData) if err != nil { return err } @@ -205,15 +205,15 @@ func (p *politeiawww) emailProposalEdited(name, username, token, version string, return err } - tmplData := tmplDataProposalEdited{ - Link: l.String(), + tmplData := proposalEdited{ Name: name, Version: version, Username: username, + Link: l.String(), } subject := "Proposal Edited" - body, err := createBody(tmplProposalEdited, &tmplData) + body, err := createBody(proposalEditedTmpl, tmplData) if err != nil { return err } @@ -237,7 +237,7 @@ func (p *politeiawww) emailProposalVoteStartedToAuthor(token, name, username, em } subject := "Your Proposal Has Started Voting" - body, err := createBody(templateProposalVoteStartedForAuthor, &tplData) + body, err := createBody(templateProposalVoteStartedForAuthor, tplData) if err != nil { return err } @@ -262,7 +262,7 @@ func (p *politeiawww) emailProposalVoteStartedToUsers(token, name, username stri } subject := "Voting Started for Proposal" - body, err := createBody(templateProposalVoteStarted, &tplData) + body, err := createBody(templateProposalVoteStarted, tplData) if err != nil { return err } @@ -279,14 +279,14 @@ func (p *politeiawww) emailProposalSubmitted(token, name, username string, email return err } - tmplData := tmplDataProposalSubmitted{ + tmplData := proposalSubmitted{ Username: username, Name: name, Link: l.String(), } subject := "New Proposal Submitted" - body, err := createBody(tmplProposalSubmitted, &tmplData) + body, err := createBody(proposalSubmittedTmpl, tmplData) if err != nil { return err } @@ -311,7 +311,7 @@ func (p *politeiawww) emailProposalVoteAuthorized(token, name, username, email s } subject := "Proposal Authorized To Start Voting" - body, err := createBody(templateProposalVoteAuthorized, &tplData) + body, err := createBody(templateProposalVoteAuthorized, tplData) if err != nil { return err } @@ -334,7 +334,7 @@ func (p *politeiawww) emailProposalComment(token, commentID, commentUsername, na } subject := "New Comment On Your Proposal" - body, err := createBody(templateCommentReplyOnProposal, &tplData) + body, err := createBody(templateCommentReplyOnProposal, tplData) if err != nil { return err } @@ -358,7 +358,7 @@ func (p *politeiawww) emailUpdateUserKeyVerificationLink(email, publicKey, token } subject := "Verify Your New Identity" - body, err := createBody(templateUpdateUserKeyEmail, &tplData) + body, err := createBody(templateUpdateUserKeyEmail, tplData) if err != nil { return err } @@ -375,7 +375,7 @@ func (p *politeiawww) emailUserPasswordChanged(email string) error { } subject := "Password Changed - Security Verification" - body, err := createBody(templateUserPasswordChanged, &tplData) + body, err := createBody(templateUserPasswordChanged, tplData) if err != nil { return err } @@ -400,7 +400,7 @@ func (p *politeiawww) emailUserLocked(email string) error { } subject := "Locked Account - Reset Your Password" - body, err := createBody(templateUserLockedResetPassword, &tplData) + body, err := createBody(templateUserLockedResetPassword, tplData) if err != nil { return err } @@ -423,7 +423,7 @@ func (p *politeiawww) emailInviteNewUserVerificationLink(email, token string) er } subject := "Welcome to the Contractor Management System" - body, err := createBody(templateInviteNewUserEmail, &tplData) + body, err := createBody(templateInviteNewUserEmail, tplData) if err != nil { return err } @@ -440,7 +440,7 @@ func (p *politeiawww) emailApproveDCCVerificationLink(email string) error { } subject := "Congratulations, You've been approved!" - body, err := createBody(templateApproveDCCUserEmail, &tplData) + body, err := createBody(templateApproveDCCUserEmail, tplData) if err != nil { return err } @@ -461,7 +461,7 @@ func (p *politeiawww) emailInvoiceNotifications(email, username string) error { } subject := "Awaiting Monthly Invoice" - body, err := createBody(templateInvoiceNotification, &tplData) + body, err := createBody(templateInvoiceNotification, tplData) if err != nil { return err } @@ -493,7 +493,7 @@ func (p *politeiawww) emailInvoiceStatusUpdate(invoiceToken, userEmail string) e } subject := "Invoice status has been updated" - body, err := createBody(templateNewInvoiceStatusUpdate, &tplData) + body, err := createBody(templateNewInvoiceStatusUpdate, tplData) if err != nil { return err } @@ -516,7 +516,7 @@ func (p *politeiawww) emailDCCNew(token string, emails []string) error { } subject := "New DCC Submitted" - body, err := createBody(templateNewDCCSubmitted, &tplData) + body, err := createBody(templateNewDCCSubmitted, tplData) if err != nil { return err } @@ -538,7 +538,7 @@ func (p *politeiawww) emailDCCSupportOppose(token string, emails []string) error } subject := "New DCC Support/Opposition Submitted" - body, err := createBody(templateNewDCCSupportOppose, &tplData) + body, err := createBody(templateNewDCCSupportOppose, tplData) if err != nil { return err } diff --git a/politeiawww/templates.go b/politeiawww/templates.go index 1feb07acc..6f32856fe 100644 --- a/politeiawww/templates.go +++ b/politeiawww/templates.go @@ -16,51 +16,68 @@ var ( templateCommentReplyOnProposal = template.Must( template.New("comment_reply_on_proposal").Parse(templateCommentReplyOnProposalRaw)) templateCommentReplyOnComment = template.Must( - template.New("comment_reply_on_comment").Parse(templateCommentReplyOnCommentRaw)) + template.New("comment_reply_on_comment"). + Parse(templateCommentReplyOnCommentRaw)) ) // Proposal submitted -type tmplDataProposalSubmitted struct { +type proposalSubmitted struct { Username string // Author username Name string // Proposal name - Link string // GUI proposal details url + Link string // GUI proposal details URL } -const tmplTextProposalSubmitted = ` +const proposalSubmittedText = ` A new proposal has been submitted on Politeia by {{.Username}}: {{.Name}} {{.Link}} ` -var tmplProposalSubmitted = template.Must(template.New("proposalSubmitted"). - Parse(tmplTextProposalSubmitted)) +var proposalSubmittedTmpl = template.Must( + template.New("proposalSubmitted").Parse(proposalSubmittedText)) // Proposal edited -type tmplDataProposalEdited struct { +type proposalEdited struct { Name string // Proposal name Version string // ProposalVersion Username string // Author username - Link string // GUI proposal details url + Link string // GUI proposal details URL } -const tmplTextProposalEdited = ` +const proposalEditedText = ` A proposal by {{.Username}} has just been edited: {{.Name}} (Version: {{.Version}}) {{.Link}} ` -var tmplProposalEdited = template.Must(template.New("proposalEdited"). - Parse(tmplTextProposalEdited)) +var proposalEditedTmpl = template.Must( + template.New("proposalEdited").Parse(proposalEditedText)) + +// Proposal status change - Vetted - Send to users +type proposalVetted struct { + Name string // Proposal name + Link string // GUI proposal details URL +} + +const proposalVettedText = ` +A new proposal has just been published on Politeia. + +{{.Name}} +{{.Link}} +` + +var tmplProposalVetted = template.Must( + template.New("proposalVetted").Parse(proposalVettedText)) // Proposal status change - Vetted - Send to author -type tmplDataProposalVettedForAuthor struct { +type proposalVettedToAuthor struct { Name string // Proposal name - Link string // GUI proposal details url + Link string // GUI proposal details URL } -const tmplTextProposalVettedForAuthor = ` +const proposalVettedToAuthorText = ` Your proposal has just been approved on Politeia! You will need to authorize a proposal vote before an administrator will be @@ -75,19 +92,18 @@ before authorizing the vote. {{.Link}} ` -var tmplProposalVettedForAuthor = template.Must( - template.New("proposalVettedForAuthor"). - Parse(tmplTextProposalVettedForAuthor)) +var proposalVettedToAuthorTmpl = template.Must( + template.New("proposalVettedToAuthor").Parse(proposalVettedToAuthorText)) // Proposal status change - Censored - Send to author -type tmplDataProposalCensoredForAuthor struct { +type proposalCensoredToAuthor struct { Name string // Proposal name Reason string // Reason for censoring - Link string // GUI proposal details url + Link string // GUI proposal details URL } -const tmplTextProposalCensoredForAuthor = ` -Your proposal on Politeia has been censored: +const proposalCensoredToAuthorText = ` +Your proposal on Politeia has been censored. {{.Name}} Reason: {{.Reason}} @@ -95,24 +111,7 @@ Reason: {{.Reason}} ` var tmplProposalCensoredForAuthor = template.Must( - template.New("proposalCensoredForAuthor"). - Parse(tmplTextProposalCensoredForAuthor)) - -// Proposal status change - Vetted - Send to users -type tmplDataProposalVetted struct { - Name string - Link string -} - -const tmplTextProposalVetted = ` -A new proposal has just been published on Politeia. - -{{.Name}} -{{.Link}} -` - -var tmplProposalVetted = template.Must(template.New("proposalVetted"). - Parse(tmplTextProposalVetted)) + template.New("proposalCensoredToAuthor").Parse(proposalCensoredToAuthorText)) type invoiceNotificationEmailData struct { Username string From e461339cff3cc13cbf7dc8466a530d450daad2cd Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Wed, 16 Sep 2020 11:04:26 -0300 Subject: [PATCH 062/449] politeiawww: add ProposalInventory --- politeiawww/piwww.go | 57 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index ac477ef75..1f736110f 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -328,6 +328,18 @@ func convertProposalRecordFromPD(r pd.Record) (*pi.ProposalRecord, error) { }, nil } +func convertInventoryReplyFromPD(i pd.InventoryByStatusReply) pi.ProposalInventoryReply { + // Concatenate both unvetted status from d + unvetted := append(i.Unvetted, i.IterationUnvetted...) + + return pi.ProposalInventoryReply{ + Unvetted: unvetted, + Public: i.Vetted, + Censored: i.Censored, + Abandoned: i.Archived, + } +} + // linkByPeriodMin returns the minimum amount of time, in seconds, that the // LinkBy period must be set to. This is determined by adding 1 week onto the // minimum voting period so that RFP proposal submissions have at least one @@ -1266,12 +1278,24 @@ func (p *politeiawww) processProposals(ps pi.Proposals, isAdmin bool) (*pi.Propo }, nil } -func (p *politeiawww) processProposalInventory() (*pi.ProposalInventoryReply, error) { +// processProposalInventory retrieves the censorship tokens from all records, +// separated by their status. +func (p *politeiawww) processProposalInventory(isAdmin bool) (*pi.ProposalInventoryReply, error) { log.Tracef("processProposalInventory") - // TODO politeiad needs a InventoryByStatus route + i, err := p.inventoryByStatus() + if err != nil { + return nil, err + } + + reply := convertInventoryReplyFromPD(*i) + + // Remove unvetted data from non-admin users + if !isAdmin { + reply.Unvetted = []string{} + } - return &pi.ProposalInventoryReply{}, nil + return &reply, nil } func (p *politeiawww) handleProposalNew(w http.ResponseWriter, r *http.Request) { @@ -1364,8 +1388,35 @@ func (p *politeiawww) handleProposalSetStatus(w http.ResponseWriter, r *http.Req util.RespondWithJSON(w, http.StatusOK, pssr) } +func (p *politeiawww) handleProposalInventory(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleProposalInventory") + + // Get data from session user. This is a public route, so we can + // ignore the session not found error. This is done to strip + // non-admin users from unvetted record tokens. + user, err := p.getSessionUser(w, r) + if err != nil && err != errSessionNotFound { + respondWithPiError(w, r, + "handleProposalInventory: getSessionUser: %v", err) + return + } + isAdmin := user != nil && user.Admin + + ppi, err := p.processProposalInventory(isAdmin) + if err != nil { + respondWithPiError(w, r, + "handleProposalInventory: processProposalInventory: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, ppi) +} + func (p *politeiawww) setPiRoutes() { // Public routes + p.addRoute(http.MethodGet, pi.APIRoute, + pi.RouteProposalInventory, p.handleProposalInventory, + permissionPublic) // Logged in routes p.addRoute(http.MethodPost, pi.APIRoute, From dd456cf4955134d44291a1985aaf8bb3acdc6de2 Mon Sep 17 00:00:00 2001 From: Amir Massarwa Date: Wed, 16 Sep 2020 18:31:29 +0300 Subject: [PATCH 063/449] politeiawww: add processVoteResults --- plugins/ticketvote/ticketvote.go | 2 +- politeiad/backend/gitbe/gitbe.go | 2 +- politeiawww/api/www/v1/v1.go | 18 ++++---- politeiawww/proposals.go | 78 +++++++++++++++++++++++++++++++- 4 files changed, 89 insertions(+), 11 deletions(-) diff --git a/plugins/ticketvote/ticketvote.go b/plugins/ticketvote/ticketvote.go index ece52779c..98176dc65 100644 --- a/plugins/ticketvote/ticketvote.go +++ b/plugins/ticketvote/ticketvote.go @@ -409,7 +409,7 @@ type BallotReply struct { Receipts []VoteReply `json:"receipts"` } -// EncodeBallot encodes a Ballot into a JSON byte slice. +// EncodeBallotReply encodes a Ballot into a JSON byte slice. func EncodeBallotReply(b BallotReply) ([]byte, error) { return json.Marshal(b) } diff --git a/politeiad/backend/gitbe/gitbe.go b/politeiad/backend/gitbe/gitbe.go index fa3e1d42f..b187897f0 100644 --- a/politeiad/backend/gitbe/gitbe.go +++ b/politeiad/backend/gitbe/gitbe.go @@ -1281,7 +1281,7 @@ func (g *gitBackEnd) populateTokenPrefixCache() error { func (g *gitBackEnd) randomUniqueToken() ([]byte, error) { TRIES := 1000 for i := 0; i < TRIES; i++ { - token, err := util.Random(pd.TokenSize) + token, err := util.Random(pd.TokenSizeMax) if err != nil { return nil, err } diff --git a/politeiawww/api/www/v1/v1.go b/politeiawww/api/www/v1/v1.go index 106ae0cec..69546a3fd 100644 --- a/politeiawww/api/www/v1/v1.go +++ b/politeiawww/api/www/v1/v1.go @@ -1046,11 +1046,6 @@ type CastVote struct { Signature string `json:"signature"` // Signature of Token+Ticket+VoteBit } -// Ballot is a batch of votes that are sent to the server. -type Ballot struct { - Votes []CastVote `json:"votes"` -} - // CastVoteReply is the answer to the CastVote command. The Error and // ErrorStatus fields will only be populated if something went wrong while // attempting to cast the vote. @@ -1061,7 +1056,12 @@ type CastVoteReply struct { ErrorStatus decredplugin.ErrorStatusT `json:"errorstatus,omitempty"` // Error status code } -// CastVotesReply is a reply to a batched list of votes. +// Ballot is a batch of votes that are sent to the server. +type Ballot struct { + Votes []CastVote `json:"votes"` +} + +// BallotReply is a reply to a batched list of votes. type BallotReply struct { Receipts []CastVoteReply `json:"receipts"` } @@ -1133,8 +1133,10 @@ type GetCommentsReply struct { } const ( - VoteActionDown = "-1" // User votes down a comment - VoteActionUp = "1" // User votes up a comment + // VoteActionDown used when user votes down a comment + VoteActionDown = "-1" + // VoteActionUp used when user votes up a comment + VoteActionUp = "1" ) // LikeComment allows a user to up or down vote a comment. diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 6f6a84e5e..704654daa 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -16,6 +16,7 @@ import ( "github.com/decred/politeia/decredplugin" piplugin "github.com/decred/politeia/plugins/pi" + ticketvote "github.com/decred/politeia/plugins/ticketvote" pd "github.com/decred/politeia/politeiad/api/v1" pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" @@ -457,7 +458,82 @@ func (p *politeiawww) processActiveVote() (*www.ActiveVoteReply, error) { } func (p *politeiawww) processVoteResults(token string) (*www.VoteResultsReply, error) { - return nil, nil + log.Tracef("processVoteResults: %v", token) + + // Prep vote details payload + vdp := ticketvote.Details{ + Token: token, + } + payload, err := ticketvote.EncodeDetails(vdp) + if err != nil { + return nil, err + } + + r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdDetails, "", + string(payload)) + vd, err := ticketvote.DecodeDetailsReply([]byte(r)) + if err != nil { + return nil, err + } + + // Convert reply to www + res := www.VoteResultsReply{ + StartVote: www.StartVote{ + PublicKey: vd.Vote.PublicKey, + Signature: vd.Vote.Signature, + Vote: www.Vote{ + Token: vd.Vote.Vote.Token, + Mask: vd.Vote.Vote.Mask, + Duration: vd.Vote.Vote.Duration, + QuorumPercentage: vd.Vote.Vote.QuorumPercentage, + PassPercentage: vd.Vote.Vote.PassPercentage, + }, + }, + StartVoteReply: www.StartVoteReply{ + StartBlockHeight: strconv.FormatUint(uint64(vd.Vote.StartBlockHeight), + 10), + StartBlockHash: vd.Vote.StartBlockHash, + EndHeight: strconv.FormatUint(uint64(vd.Vote.EndBlockHeight), 10), + EligibleTickets: vd.Vote.EligibleTickets, + }, + } + + // Transalte vote options + vo := make([]www.VoteOption, len(vd.Vote.Vote.Options)) + for _, o := range vd.Vote.Vote.Options { + vo = append(vo, www.VoteOption{ + Id: o.ID, + Description: o.Description, + Bits: o.Bit, + }) + } + res.StartVote.Vote.Options = vo + + // Prep cast votes payload + csp := ticketvote.CastVotes{ + Token: token, + } + payload, err = ticketvote.EncodeCastVotes(csp) + + r, err = p.pluginCommand(ticketvote.ID, ticketvote.CmdCastVotes, "", + string(payload)) + cv, err := ticketvote.DecodeCastVotesReply([]byte(r)) + if err != nil { + return nil, err + } + + votes := make([]www.CastVote, len(cv.Votes)) + for _, v := range cv.Votes { + votes = append(votes, www.CastVote{ + Token: v.Token, + Ticket: v.Ticket, + VoteBit: v.VoteBit, + Signature: v.Signature, + }) + } + res.CastVotes = votes + + return &res, nil } func (p *politeiawww) processCastVotes(ballot *www.Ballot) (*www.BallotReply, error) { From 7e62328c66b95875a48cf36d94d1fdade3beee11 Mon Sep 17 00:00:00 2001 From: Amir Massarwa Date: Thu, 17 Sep 2020 02:23:48 +0300 Subject: [PATCH 064/449] politeiawww: Add processBatchVoteSummary. --- politeiawww/proposals.go | 99 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 94 insertions(+), 5 deletions(-) diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 704654daa..bca9afca5 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Decred developers +// Copyright (c) 2017-2020 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -347,10 +347,6 @@ func (p *politeiawww) processProposalDetails(pd www.ProposalsDetails, u *user.Us return &pdr, nil } -func (p *politeiawww) processBatchVoteSummary(bvs www.BatchVoteSummary) (*www.BatchVoteSummaryReply, error) { - return nil, nil -} - func (p *politeiawww) processBatchProposals(bp www.BatchProposals, u *user.User) (*www.BatchProposalsReply, error) { log.Tracef("processBatchProposals: %v", bp.Tokens) @@ -517,6 +513,9 @@ func (p *politeiawww) processVoteResults(token string) (*www.VoteResultsReply, e r, err = p.pluginCommand(ticketvote.ID, ticketvote.CmdCastVotes, "", string(payload)) + if err != nil { + return nil, err + } cv, err := ticketvote.DecodeCastVotesReply([]byte(r)) if err != nil { return nil, err @@ -536,6 +535,96 @@ func (p *politeiawww) processVoteResults(token string) (*www.VoteResultsReply, e return &res, nil } +func convertVoteStatusToWWW(status ticketvote.VoteStatusT) www.PropVoteStatusT { + switch status { + case ticketvote.VoteStatusInvalid: + return www.PropVoteStatusInvalid + case ticketvote.VoteStatusUnauthorized: + return www.PropVoteStatusNotAuthorized + case ticketvote.VoteStatusAuthorized: + return www.PropVoteStatusAuthorized + case ticketvote.VoteStatusStarted: + return www.PropVoteStatusStarted + case ticketvote.VoteStatusFinished: + return www.PropVoteStatusFinished + default: + return www.PropVoteStatusInvalid + } +} + +func convertVoteTypeToWWW(t ticketvote.VoteT) www.VoteT { + switch t { + case ticketvote.VoteTypeInvalid: + return www.VoteTypeInvalid + case ticketvote.VoteTypeStandard: + return www.VoteTypeStandard + case ticketvote.VoteTypeRunoff: + return www.VoteTypeRunoff + default: + return www.VoteTypeInvalid + } +} + +func (p *politeiawww) processBatchVoteSummary(bvs www.BatchVoteSummary) (*www.BatchVoteSummaryReply, error) { + log.Tracef("processBatchVoteSummary: %v", bvs.Tokens) + + // Prep plugin command + smp := ticketvote.Summaries{ + Tokens: bvs.Tokens, + } + payload, err := ticketvote.EncodeSummaries(smp) + if err != nil { + return nil, err + } + r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdSummaries, "", + string(payload)) + if err != nil { + return nil, err + } + sm, err := ticketvote.DecodeSummariesReply([]byte(r)) + if err != nil { + return nil, err + } + + // Covert reply to www + res := www.BatchVoteSummaryReply{ + BestBlock: uint64(sm.BestBlock), + } + // Translate summaries + summaries := make(map[string]www.VoteSummary, len(sm.Summaries)) + for t, sum := range sm.Summaries { + vs := www.VoteSummary{ + Status: convertVoteStatusToWWW(sum.Status), + Type: convertVoteTypeToWWW(sum.Type), + Approved: sum.Approved, + EligibleTickets: sum.EligibleTickets, + Duration: sum.Duration, + EndHeight: uint64(sum.EndBlockHeight), + QuorumPercentage: sum.QuorumPercentage, + PassPercentage: sum.PassPercentage, + } + + // Translate vote options + results := make([]www.VoteOptionResult, len(sum.Results)) + for _, r := range sum.Results { + results = append(results, www.VoteOptionResult{ + VotesReceived: r.Votes, + Option: www.VoteOption{ + Id: r.ID, + Description: r.Description, + Bits: r.VoteBit, + }, + }) + } + vs.Results = results + summaries[t] = vs + + } + res.Summaries = summaries + + return &res, nil +} + func (p *politeiawww) processCastVotes(ballot *www.Ballot) (*www.BallotReply, error) { log.Tracef("processCastVotes") From daa547c5fe1e11bfd9b96e592e7b92e1e68ecb33 Mon Sep 17 00:00:00 2001 From: Amir Massarwa Date: Thu, 17 Sep 2020 16:36:46 +0300 Subject: [PATCH 065/449] politeiawww: add processCastVotes --- politeiawww/proposals.go | 103 +++++++++++++++++++++------------------ 1 file changed, 56 insertions(+), 47 deletions(-) diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index bca9afca5..efc5a88e5 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -285,7 +285,7 @@ func (p *politeiawww) convertProposalToWWW(pr *pi.ProposalRecord) (*www.Proposal }, } - files := make([]www.File, len(pr.Files)) + files := make([]www.File, 0, len(pr.Files)) for _, f := range pr.Files { files = append(files, www.File{ Name: f.Name, @@ -296,7 +296,7 @@ func (p *politeiawww) convertProposalToWWW(pr *pi.ProposalRecord) (*www.Proposal } pw.Files = files - metadata := make([]www.Metadata, len(pr.Metadata)) + metadata := make([]www.Metadata, 0, len(pr.Metadata)) for _, md := range pr.Metadata { metadata = append(metadata, www.Metadata{ Digest: md.Digest, @@ -351,7 +351,7 @@ func (p *politeiawww) processBatchProposals(bp www.BatchProposals, u *user.User) log.Tracef("processBatchProposals: %v", bp.Tokens) // Prep proposals requests - prs := make([]pi.ProposalRequest, len(bp.Tokens)) + prs := make([]pi.ProposalRequest, 0, len(bp.Tokens)) for _, t := range bp.Tokens { prs = append(prs, pi.ProposalRequest{ Token: t, @@ -364,7 +364,7 @@ func (p *politeiawww) processBatchProposals(bp www.BatchProposals, u *user.User) } // Convert proposals records - propsw := make([]www.ProposalRecord, len(bp.Tokens)) + propsw := make([]www.ProposalRecord, 0, len(bp.Tokens)) for _, pr := range props { propw, err := p.convertProposalToWWW(&pr) if err != nil { @@ -495,7 +495,7 @@ func (p *politeiawww) processVoteResults(token string) (*www.VoteResultsReply, e } // Transalte vote options - vo := make([]www.VoteOption, len(vd.Vote.Vote.Options)) + vo := make([]www.VoteOption, 0, len(vd.Vote.Vote.Options)) for _, o := range vd.Vote.Vote.Options { vo = append(vo, www.VoteOption{ Id: o.ID, @@ -521,7 +521,7 @@ func (p *politeiawww) processVoteResults(token string) (*www.VoteResultsReply, e return nil, err } - votes := make([]www.CastVote, len(cv.Votes)) + votes := make([]www.CastVote, 0, len(cv.Votes)) for _, v := range cv.Votes { votes = append(votes, www.CastVote{ Token: v.Token, @@ -576,6 +576,7 @@ func (p *politeiawww) processBatchVoteSummary(bvs www.BatchVoteSummary) (*www.Ba if err != nil { return nil, err } + r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdSummaries, "", string(payload)) if err != nil { @@ -603,9 +604,8 @@ func (p *politeiawww) processBatchVoteSummary(bvs www.BatchVoteSummary) (*www.Ba QuorumPercentage: sum.QuorumPercentage, PassPercentage: sum.PassPercentage, } - // Translate vote options - results := make([]www.VoteOptionResult, len(sum.Results)) + results := make([]www.VoteOptionResult, 0, len(sum.Results)) for _, r := range sum.Results { results = append(results, www.VoteOptionResult{ VotesReceived: r.Votes, @@ -625,62 +625,71 @@ func (p *politeiawww) processBatchVoteSummary(bvs www.BatchVoteSummary) (*www.Ba return &res, nil } +func convertVoteErrorCodeToWWW(errcode ticketvote.VoteErrorT) decredplugin.ErrorStatusT { + switch errcode { + case ticketvote.VoteErrorInvalid: + return decredplugin.ErrorStatusInvalid + case ticketvote.VoteErrorInternalError: + return decredplugin.ErrorStatusInternalError + case ticketvote.VoteErrorRecordNotFound: + return decredplugin.ErrorStatusProposalNotFound + case ticketvote.VoteErrorVoteBitInvalid: + return decredplugin.ErrorStatusInvalidVoteBit + case ticketvote.VoteErrorVoteStatusInvalid: + return decredplugin.ErrorStatusVoteHasEnded + case ticketvote.VoteErrorTicketAlreadyVoted: + return decredplugin.ErrorStatusDuplicateVote + case ticketvote.VoteErrorTicketNotEligible: + return decredplugin.ErrorStatusIneligibleTicket + default: + return decredplugin.ErrorStatusInternalError + } +} + func (p *politeiawww) processCastVotes(ballot *www.Ballot) (*www.BallotReply, error) { log.Tracef("processCastVotes") - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - - // Verify proposal tokens + // Prep plugin command + var bp ticketvote.Ballot + // Transale votes + votes := make([]ticketvote.Vote, 0, len(ballot.Votes)) for _, vote := range ballot.Votes { - if !tokenIsValid(vote.Token) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: []string{vote.Token}, - } - } + votes = append(votes, ticketvote.Vote{ + Token: vote.Ticket, + Ticket: vote.Ticket, + VoteBit: vote.VoteBit, + Signature: vote.Signature, + }) } - - payload, err := decredplugin.EncodeBallot(convertBallotFromWWW(*ballot)) + bp.Votes = votes + payload, err := ticketvote.EncodeBallot(bp) if err != nil { return nil, err } - pc := pd.PluginCommand{ - Challenge: hex.EncodeToString(challenge), - ID: decredplugin.ID, - Command: decredplugin.CmdBallot, - CommandID: decredplugin.CmdBallot, - Payload: string(payload), - } - responseBody, err := p.makeRequest(http.MethodPost, - pd.PluginCommandRoute, pc) + r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdBallot, "", + string(payload)) if err != nil { return nil, err } - - var reply pd.PluginCommandReply - err = json.Unmarshal(responseBody, &reply) - if err != nil { - return nil, fmt.Errorf("Could not unmarshal "+ - "PluginCommandReply: %v", err) - } - - // Verify the challenge. - err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) + b, err := ticketvote.DecodeBallotReply([]byte(r)) if err != nil { return nil, err } - // Decode plugin reply - br, err := decredplugin.DecodeBallotReply([]byte(reply.Payload)) - if err != nil { - return nil, err + // Translate reply to www + res := www.BallotReply{} + rps := make([]www.CastVoteReply, 0, len(b.Receipts)) + for i, rp := range b.Receipts { + rps = append(rps, www.CastVoteReply{ + ClientSignature: ballot.Votes[i].Signature, + Signature: rp.Receipt, + Error: rp.ErrorContext, + ErrorStatus: convertVoteErrorCodeToWWW(rp.ErrorCode), + }) } - brr := convertBallotReplyFromDecredPlugin(*br) - return &brr, nil + + return &res, nil } // processProposalPaywallDetails returns a proposal paywall that enables the From 024923dee0945199aa45de1a650bc1f63554e755 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 16 Sep 2020 11:30:48 -0500 Subject: [PATCH 066/449] tlogbe plugins refactor --- plugins/ticketvote/ticketvote.go | 6 +- politeiad/backend/backend.go | 2 +- politeiad/backend/gitbe/gitbe.go | 2 +- politeiad/backend/tlogbe/plugin.go | 134 -------- politeiad/backend/tlogbe/pluginclient.go | 135 ++++++++ politeiad/backend/tlogbe/plugins/pi.go | 4 +- .../backend/tlogbe/plugins/ticketvote.go | 10 +- politeiad/backend/tlogbe/recordclient.go | 294 ----------------- politeiad/backend/tlogbe/tlog.go | 270 +++++++++++++++- politeiad/backend/tlogbe/tlogbe.go | 297 ++++++++++-------- politeiad/backend/tlogbe/tlogclient.go | 147 +++++++++ politeiad/politeiad.go | 3 +- 12 files changed, 730 insertions(+), 574 deletions(-) delete mode 100644 politeiad/backend/tlogbe/plugin.go create mode 100644 politeiad/backend/tlogbe/pluginclient.go delete mode 100644 politeiad/backend/tlogbe/recordclient.go create mode 100644 politeiad/backend/tlogbe/tlogclient.go diff --git a/plugins/ticketvote/ticketvote.go b/plugins/ticketvote/ticketvote.go index 98176dc65..5bc55f34d 100644 --- a/plugins/ticketvote/ticketvote.go +++ b/plugins/ticketvote/ticketvote.go @@ -2,8 +2,8 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -// Package ticketvote provides a plugin for creating and managing votes that -// require decred tickets to participate. +// Package ticketvote provides a plugin for running votes that require decred +// tickets to participate. package ticketvote import ( @@ -322,7 +322,7 @@ func DecodeStartReplyVote(payload []byte) (*StartReply, error) { type StartRunoff struct { Token string `json:"token"` // RFP token Authorizations []Authorize `json:"authorizations"` - Votes []Start `json:"votes"` + Starts []Start `json:"starts"` } // EncodeStartRunoff encodes a StartRunoff into a JSON byte slice. diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index 36c20bf67..390f1a0b9 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -215,7 +215,7 @@ type Backend interface { GetPlugins() ([]Plugin, error) // Plugin pass-through command - Plugin(pluginID, cmd, payload string) (string, error) + Plugin(pluginID, cmd, cmdID, payload string) (string, error) // Close performs cleanup of the backend. Close() diff --git a/politeiad/backend/gitbe/gitbe.go b/politeiad/backend/gitbe/gitbe.go index b187897f0..747fd9f91 100644 --- a/politeiad/backend/gitbe/gitbe.go +++ b/politeiad/backend/gitbe/gitbe.go @@ -2815,7 +2815,7 @@ func (g *gitBackEnd) GetPlugins() ([]backend.Plugin, error) { // execute. // // Plugin satisfies the backend interface. -func (g *gitBackEnd) Plugin(pluginID, command, payload string) (string, error) { +func (g *gitBackEnd) Plugin(pluginID, command, commandID, payload string) (string, error) { log.Tracef("Plugin: %v", command) switch command { case decredplugin.CmdAuthorizeVote: diff --git a/politeiad/backend/tlogbe/plugin.go b/politeiad/backend/tlogbe/plugin.go deleted file mode 100644 index 7e4aca9df..000000000 --- a/politeiad/backend/tlogbe/plugin.go +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright (c) 2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package tlogbe - -import ( - "encoding/json" - - "github.com/decred/politeia/politeiad/backend" -) - -type HookT int - -const ( - // Plugin hooks - HookInvalid HookT = 0 - HookNewRecordPre HookT = 1 - HookNewRecordPost HookT = 2 - HookEditRecordPre HookT = 3 - HookEditRecordPost HookT = 4 - HookEditMetadataPre HookT = 5 - HookEditMetadataPost HookT = 6 - HookSetRecordStatusPre HookT = 7 - HookSetRecordStatusPost HookT = 8 -) - -var ( - // Hooks contains human readable plugin hook descriptions. - Hooks = map[HookT]string{ - HookNewRecordPre: "new record pre", - HookNewRecordPost: "new record post", - HookEditRecordPre: "edit record pre", - HookEditRecordPost: "edit record post", - HookEditMetadataPre: "edit metadata pre", - HookEditMetadataPost: "edit metadata post", - HookSetRecordStatusPre: "set record status pre", - HookSetRecordStatusPost: "set record status post", - } -) - -// NewRecord is the payload for the HookNewRecordPre and HookNewRecordPost -// hooks. -type NewRecord struct { - RecordMetadata backend.RecordMetadata `json:"recordmetadata"` - Metadata []backend.MetadataStream `json:"metadata"` - Files []backend.File `json:"files"` -} - -// EncodeNewRecord encodes a NewRecord into a JSON byte slice. -func EncodeNewRecord(nr NewRecord) ([]byte, error) { - return json.Marshal(nr) -} - -// DecodeNewRecord decodes a JSON byte slice into a NewRecord. -func DecodeNewRecord(payload []byte) (*NewRecord, error) { - var nr NewRecord - err := json.Unmarshal(payload, &nr) - if err != nil { - return nil, err - } - return &nr, nil -} - -// EditRecord is the payload for the EditRecordPre and EditRecordPost hooks. -type EditRecord struct { - // Current record - Current backend.Record `json:"record"` - - // Updated fields - RecordMetadata backend.RecordMetadata `json:"recordmetadata"` - MDAppend []backend.MetadataStream `json:"mdappend"` - MDOverwrite []backend.MetadataStream `json:"mdoverwrite"` - FilesAdd []backend.File `json:"filesadd"` - FilesDel []string `json:"filesdel"` -} - -// EncodeEditRecord encodes an EditRecord into a JSON byte slice. -func EncodeEditRecord(nr EditRecord) ([]byte, error) { - return json.Marshal(nr) -} - -// DecodeEditRecord decodes a JSON byte slice into a EditRecord. -func DecodeEditRecord(payload []byte) (*EditRecord, error) { - var nr EditRecord - err := json.Unmarshal(payload, &nr) - if err != nil { - return nil, err - } - return &nr, nil -} - -// SetRecordStatus is the payload for the HookSetRecordStatusPre and -// HookSetRecordStatusPost hooks. -type SetRecordStatus struct { - // Current record - Current backend.Record `json:"record"` - - // Updated fields - RecordMetadata backend.RecordMetadata `json:"recordmetadata"` - MDAppend []backend.MetadataStream `json:"mdappend"` - MDOverwrite []backend.MetadataStream `json:"mdoverwrite"` -} - -// EncodeSetRecordStatus encodes a SetRecordStatus into a JSON byte slice. -func EncodeSetRecordStatus(srs SetRecordStatus) ([]byte, error) { - return json.Marshal(srs) -} - -// DecodeSetRecordStatus decodes a JSON byte slice into a SetRecordStatus. -func DecodeSetRecordStatus(payload []byte) (*SetRecordStatus, error) { - var srs SetRecordStatus - err := json.Unmarshal(payload, &srs) - if err != nil { - return nil, err - } - return &srs, nil -} - -/// Plugin provides an API for the tlogbe to use when interacting with plugins. -// All tlogbe plugins must implement the Plugin interface. -type Plugin interface { - // Perform plugin setup - Setup() error - - // Execute a plugin command - Cmd(cmd, payload string) (string, error) - - // Execute a plugin hook - Hook(h HookT, payload string) error - - // Perform a plugin file system check - Fsck() error -} diff --git a/politeiad/backend/tlogbe/pluginclient.go b/politeiad/backend/tlogbe/pluginclient.go new file mode 100644 index 000000000..e60e0b147 --- /dev/null +++ b/politeiad/backend/tlogbe/pluginclient.go @@ -0,0 +1,135 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlogbe + +import ( + "encoding/json" + + "github.com/decred/politeia/politeiad/backend" +) + +type hookT int + +const ( + // Plugin hooks + hookInvalid hookT = 0 + hookNewRecordPre hookT = 1 + hookNewRecordPost hookT = 2 + hookEditRecordPre hookT = 3 + hookEditRecordPost hookT = 4 + hookEditMetadataPre hookT = 5 + hookEditMetadataPost hookT = 6 + hookSetRecordStatusPre hookT = 7 + hookSetRecordStatusPost hookT = 8 +) + +var ( + // hooks contains human readable plugin hook descriptions. + hooks = map[hookT]string{ + hookNewRecordPre: "new record pre", + hookNewRecordPost: "new record post", + hookEditRecordPre: "edit record pre", + hookEditRecordPost: "edit record post", + hookEditMetadataPre: "edit metadata pre", + hookEditMetadataPost: "edit metadata post", + hookSetRecordStatusPre: "set record status pre", + hookSetRecordStatusPost: "set record status post", + } +) + +// newRecord is the payload for the hookNewRecordPre and hookNewRecordPost +// hooks. +type newRecord struct { + RecordMetadata backend.RecordMetadata `json:"recordmetadata"` + Metadata []backend.MetadataStream `json:"metadata"` + Files []backend.File `json:"files"` +} + +// encodeNewRecord encodes a newRecord into a JSON byte slice. +func encodeNewRecord(nr newRecord) ([]byte, error) { + return json.Marshal(nr) +} + +// decodeNewRecord decodes a JSON byte slice into a newRecord. +func decodeNewRecord(payload []byte) (*newRecord, error) { + var nr newRecord + err := json.Unmarshal(payload, &nr) + if err != nil { + return nil, err + } + return &nr, nil +} + +// editRecord is the payload for the hookEditRecordPre and hookEditRecordPost +// hooks. +type editRecord struct { + // Current record + Current backend.Record `json:"record"` + + // Updated fields + RecordMetadata backend.RecordMetadata `json:"recordmetadata"` + MDAppend []backend.MetadataStream `json:"mdappend"` + MDOverwrite []backend.MetadataStream `json:"mdoverwrite"` + FilesAdd []backend.File `json:"filesadd"` + FilesDel []string `json:"filesdel"` +} + +// encodeEditRecord encodes an editRecord into a JSON byte slice. +func encodeEditRecord(er editRecord) ([]byte, error) { + return json.Marshal(er) +} + +// decodeEditRecord decodes a JSON byte slice into a editRecord. +func decodeEditRecord(payload []byte) (*editRecord, error) { + var er editRecord + err := json.Unmarshal(payload, &er) + if err != nil { + return nil, err + } + return &er, nil +} + +// setRecordStatus is the payload for the hookSetRecordStatusPre and +// hookSetRecordStatusPost hooks. +type setRecordStatus struct { + // Current record + Current backend.Record `json:"record"` + + // Updated fields + RecordMetadata backend.RecordMetadata `json:"recordmetadata"` + MDAppend []backend.MetadataStream `json:"mdappend"` + MDOverwrite []backend.MetadataStream `json:"mdoverwrite"` +} + +// encodeSetRecordStatus encodes a setRecordStatus into a JSON byte slice. +func encodeSetRecordStatus(srs setRecordStatus) ([]byte, error) { + return json.Marshal(srs) +} + +// decodeSetRecordStatus decodes a JSON byte slice into a setRecordStatus. +func decodeSetRecordStatus(payload []byte) (*setRecordStatus, error) { + var srs setRecordStatus + err := json.Unmarshal(payload, &srs) + if err != nil { + return nil, err + } + return &srs, nil +} + +// pluginClient provides an API for the tlog backend to use when interacting +// with plugins. All tlogbe plugins must implement the pluginClient interface. +type pluginClient interface { + // setup performs any required plugin setup. + setup() error + + // cmd executes the provided plugin command. + cmd(cmd, payload string) (string, error) + + // hook executes the provided plugin hook. + hook(h hookT, payload string) error + + // fsck performs a plugin file system check. + fsck() error +} diff --git a/politeiad/backend/tlogbe/plugins/pi.go b/politeiad/backend/tlogbe/plugins/pi.go index 4728c0d6c..1a435a61b 100644 --- a/politeiad/backend/tlogbe/plugins/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi.go @@ -259,7 +259,7 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { return err } reply, err := p.backend.Plugin(ticketvote.ID, - ticketvote.CmdSummaries, string(b)) + ticketvote.CmdSummaries, "", string(b)) if err != nil { return fmt.Errorf("Plugin %v %v: %v", ticketvote.ID, ticketvote.CmdSummaries, err) @@ -325,7 +325,7 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { return err } reply, err := p.backend.Plugin(ticketvote.ID, - ticketvote.CmdSummaries, string(b)) + ticketvote.CmdSummaries, "", string(b)) if err != nil { return fmt.Errorf("ticketvote Summaries: %v", err) } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote.go index 1b843e758..5e052abe7 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote.go @@ -602,7 +602,7 @@ func (p *ticketVotePlugin) bestBlock() (uint32, error) { return 0, err } reply, err := p.backend.Plugin(dcrdata.ID, - dcrdata.CmdBestBlock, string(payload)) + dcrdata.CmdBestBlock, "", string(payload)) if err != nil { return 0, fmt.Errorf("Plugin %v %v: %v", dcrdata.ID, dcrdata.CmdBestBlock, err) @@ -637,7 +637,7 @@ func (p *ticketVotePlugin) bestBlockUnsafe() (uint32, error) { return 0, err } reply, err := p.backend.Plugin(dcrdata.ID, - dcrdata.CmdBestBlock, string(payload)) + dcrdata.CmdBestBlock, "", string(payload)) if err != nil { return 0, fmt.Errorf("Plugin %v %v: %v", dcrdata.ID, dcrdata.CmdBestBlock, err) @@ -671,7 +671,7 @@ func (p *ticketVotePlugin) largestCommitmentAddrs(tickets []string) ([]commitmen return nil, err } reply, err := p.backend.Plugin(dcrdata.ID, - dcrdata.CmdTxsTrimmed, string(payload)) + dcrdata.CmdTxsTrimmed, "", string(payload)) if err != nil { return nil, fmt.Errorf("Plugin %v %v: %v", dcrdata.ID, dcrdata.CmdTxsTrimmed, err) @@ -740,7 +740,7 @@ func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, return nil, err } reply, err := p.backend.Plugin(dcrdata.ID, - dcrdata.CmdBlockDetails, string(payload)) + dcrdata.CmdBlockDetails, "", string(payload)) if err != nil { return nil, fmt.Errorf("Plugin %v %v: %v", dcrdata.ID, dcrdata.CmdBlockDetails, err) @@ -763,7 +763,7 @@ func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, return nil, err } reply, err = p.backend.Plugin(dcrdata.ID, - dcrdata.CmdTicketPool, string(payload)) + dcrdata.CmdTicketPool, "", string(payload)) if err != nil { return nil, fmt.Errorf("Plugin %v %v: %v", dcrdata.ID, dcrdata.CmdTicketPool, err) diff --git a/politeiad/backend/tlogbe/recordclient.go b/politeiad/backend/tlogbe/recordclient.go deleted file mode 100644 index 256722edf..000000000 --- a/politeiad/backend/tlogbe/recordclient.go +++ /dev/null @@ -1,294 +0,0 @@ -// Copyright (c) 2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package tlogbe - -import ( - "bytes" - "encoding/hex" - "fmt" - - "github.com/google/trillian" - "google.golang.org/grpc/codes" -) - -type RecordStateT int - -const ( - // Record types - RecordStateInvalid RecordStateT = 0 - RecordStateUnvetted RecordStateT = 1 - RecordStateVetted RecordStateT = 2 -) - -// TODO if we make this an interface it will make testing the plugins a whole -// lot easier. Or better yet, make tlog and interface. - -// RecordClient provides an API for plugins to save, retrieve, and delete -// plugin data for a specific record. Editing data is not allowed. -type RecordClient struct { - Token []byte - State RecordStateT - treeID int64 - tlog *tlog -} - -// hashes and keys must share the same ordering. -func (c *RecordClient) Save(keyPrefix string, blobs, hashes [][]byte, encrypt bool) ([][]byte, error) { - log.Tracef("Save: %x %v %v %x", c.Token, keyPrefix, encrypt, hashes) - - // Ensure tree exists and is not frozen - if !c.tlog.treeExists(c.treeID) { - return nil, errRecordNotFound - } - _, err := c.tlog.freezeRecord(c.treeID) - if err != errFreezeRecordNotFound { - return nil, errTreeIsFrozen - } - - // Encrypt blobs if specified - if encrypt { - for k, v := range blobs { - e, err := c.tlog.encryptionKey.encrypt(0, v) - if err != nil { - return nil, err - } - blobs[k] = e - } - } - - // Save blobs to store - keys, err := c.tlog.store.Put(blobs) - if err != nil { - return nil, fmt.Errorf("store Put: %v", err) - } - if len(keys) != len(blobs) { - return nil, fmt.Errorf("wrong number of keys: got %v, want %v", - len(keys), len(blobs)) - } - - // Prepare log leaves. hashes and keys share the same ordering. - leaves := make([]*trillian.LogLeaf, 0, len(blobs)) - for k := range blobs { - pk := []byte(keyPrefix + keys[k]) - leaves = append(leaves, logLeafNew(hashes[k], pk)) - } - - // Append leaves to trillian tree - queued, _, err := c.tlog.trillian.leavesAppend(c.treeID, leaves) - if err != nil { - return nil, fmt.Errorf("leavesAppend: %v", err) - } - if len(queued) != len(leaves) { - return nil, fmt.Errorf("wrong number of queued leaves: got %v, want %v", - len(queued), len(leaves)) - } - failed := make([]string, 0, len(queued)) - for _, v := range queued { - c := codes.Code(v.QueuedLeaf.GetStatus().GetCode()) - if c != codes.OK { - failed = append(failed, fmt.Sprintf("%v", c)) - } - } - if len(failed) > 0 { - return nil, fmt.Errorf("append leaves failed: %v", failed) - } - - merkles := make([][]byte, 0, len(blobs)) - for _, v := range queued { - merkles = append(merkles, v.QueuedLeaf.Leaf.MerkleLeafHash) - } - - return merkles, nil -} - -func (c *RecordClient) Del(merkleHashes [][]byte) error { - log.Tracef("Del: %x %x", c.Token, merkleHashes) - - // Ensure tree exists. We allow blobs to be deleted from both - // frozen and non frozen trees. - if !c.tlog.treeExists(c.treeID) { - return errRecordNotFound - } - - // Get all tree leaves - leaves, err := c.tlog.trillian.leavesAll(c.treeID) - if err != nil { - return err - } - - // Aggregate the key-value store keys for the provided merkle - // hashes. - merkles := make(map[string]struct{}, len(leaves)) - for _, v := range merkleHashes { - merkles[hex.EncodeToString(v)] = struct{}{} - } - keys := make([]string, 0, len(merkles)) - for _, v := range leaves { - _, ok := merkles[hex.EncodeToString(v.MerkleLeafHash)] - if ok { - key, err := extractKeyFromLeaf(v) - if err != nil { - return err - } - keys = append(keys, key) - } - } - - // Delete file blobs from the store - err = c.tlog.store.Del(keys) - if err != nil { - return fmt.Errorf("store Del: %v", err) - } - - return nil -} - -// If a blob does not exist it will not be included in the returned map. It is -// the responsibility of the caller to check that a blob is returned for each -// of the provided merkle hashes. -func (c *RecordClient) BlobsByMerkleHash(merkleHashes [][]byte) (map[string][]byte, error) { - log.Tracef("BlobsByMerkleHash: %x %x", c.Token, merkleHashes) - - // Get leaves - leavesAll, err := c.tlog.trillian.leavesAll(c.treeID) - if err != nil { - return nil, fmt.Errorf("leavesAll: %v", err) - } - - // Aggregate the leaves that correspond to the provided merkle - // hashes. - // map[merkleHash]*trillian.LogLeaf - leaves := make(map[string]*trillian.LogLeaf, len(merkleHashes)) - for _, v := range merkleHashes { - leaves[hex.EncodeToString(v)] = nil - } - for _, v := range leavesAll { - m := hex.EncodeToString(v.MerkleLeafHash) - if _, ok := leaves[m]; ok { - leaves[m] = v - } - } - - // Ensure a leaf was found for all provided merkle hashes - for k, v := range leaves { - if v == nil { - return nil, fmt.Errorf("leaf not found for merkle hash: %v", k) - } - } - - // Extract the key-value store keys. These keys MUST be put in the - // same order that the merkle hashes were provided in. - keys := make([]string, 0, len(leaves)) - for _, v := range merkleHashes { - l, ok := leaves[hex.EncodeToString(v)] - if !ok { - return nil, fmt.Errorf("leaf not found for merkle hash: %x", v) - } - k, err := extractKeyFromLeaf(l) - if err != nil { - return nil, err - } - keys = append(keys, k) - } - - // Pull the blobs from the store. If is ok if one or more blobs is - // not found. It is the responsibility of the caller to decide how - // this should be handled. - blobs, err := c.tlog.store.Get(keys) - if err != nil { - return nil, fmt.Errorf("store Get: %v", err) - } - - // Decrypt any encrypted blobs - for k, v := range blobs { - if blobIsEncrypted(v) { - b, _, err := c.tlog.encryptionKey.decrypt(v) - if err != nil { - return nil, err - } - blobs[k] = b - } - } - - // Put blobs in a map so the caller can determine if any of the - // provided merkle hashes did not correspond to a blob in the - // store. - b := make(map[string][]byte, len(blobs)) // [merkleHash]blob - for k, v := range keys { - // The merkle hashes slice and keys slice share the same order - merkleHash := hex.EncodeToString(merkleHashes[k]) - blob, ok := blobs[v] - if !ok { - return nil, fmt.Errorf("blob not found for key %v", v) - } - b[merkleHash] = blob - } - - return b, nil -} - -func (c *RecordClient) BlobsByKeyPrefix(keyPrefix string) ([][]byte, error) { - log.Tracef("BlobsByKeyPrefix: %x %x", c.Token, keyPrefix) - - // Get leaves - leaves, err := c.tlog.trillian.leavesAll(c.treeID) - if err != nil { - return nil, fmt.Errorf("leavesAll: %v", err) - } - - // Walk leaves and aggregate the key-value store keys for all - // leaves with a matching key prefix. - keys := make([]string, 0, len(leaves)) - for _, v := range leaves { - if bytes.HasPrefix(v.ExtraData, []byte(keyPrefix)) { - k, err := extractKeyFromLeaf(v) - if err != nil { - return nil, err - } - keys = append(keys, k) - } - } - - // Pull the blobs from the store - blobs, err := c.tlog.store.Get(keys) - if err != nil { - return nil, fmt.Errorf("store Get: %v", err) - } - if len(blobs) != len(keys) { - // One or more blobs were not found - missing := make([]string, 0, len(keys)) - for _, v := range keys { - _, ok := blobs[v] - if !ok { - missing = append(missing, v) - } - } - return nil, fmt.Errorf("blobs not found: %v", missing) - } - - // Decrypt any encrypted blobs - for k, v := range blobs { - if blobIsEncrypted(v) { - b, _, err := c.tlog.encryptionKey.decrypt(v) - if err != nil { - return nil, err - } - blobs[k] = b - } - } - - // Covert blobs from map to slice - b := make([][]byte, 0, len(blobs)) - for _, v := range blobs { - b = append(b, v) - } - - return b, nil -} - -// TODO implement RecordClient -func (t *TlogBackend) RecordClient(token []byte) (*RecordClient, error) { - return nil, nil -} diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index 2d35eccf7..169846ae9 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -1261,7 +1261,7 @@ func (t *tlog) recordDel(treeID int64) error { return nil } -func (t *tlog) recordVersion(treeID int64, version uint32) (*backend.Record, error) { +func (t *tlog) record(treeID int64, version uint32) (*backend.Record, error) { // Ensure tree exists if !t.treeExists(treeID) { return nil, errRecordNotFound @@ -1434,12 +1434,278 @@ func (t *tlog) recordVersion(treeID int64, version uint32) (*backend.Record, err } func (t *tlog) recordLatest(treeID int64) (*backend.Record, error) { - return t.recordVersion(treeID, 0) + return t.record(treeID, 0) } // TODO implement recordProof func (t *tlog) recordProof(treeID int64, version uint32) {} +// blobsSave saves the provided blobs to the key-value store then appends them +// onto the trillian tree. Note, hashes contains the hashes of the data encoded +// in the blobs. The hashes must share the same ordering as the blobs. +func (t *tlog) blobsSave(treeID int64, keyPrefix string, blobs, hashes [][]byte, encrypt bool) ([][]byte, error) { + // Verify tree exists + if !t.treeExists(treeID) { + return nil, errRecordNotFound + } + + // Verify tree is not frozen + _, err := t.freezeRecord(treeID) + switch err { + case nil: + // Freeze record was found. Tree is frozen. + return nil, errTreeIsFrozen + case errFreezeRecordNotFound: + // Tree is not frozen; continue + default: + // All other errors + return nil, err + } + + // Encrypt blobs if specified + if encrypt { + for k, v := range blobs { + e, err := t.encryptionKey.encrypt(0, v) + if err != nil { + return nil, err + } + blobs[k] = e + } + } + + // Save blobs to store + keys, err := t.store.Put(blobs) + if err != nil { + return nil, fmt.Errorf("store Put: %v", err) + } + if len(keys) != len(blobs) { + return nil, fmt.Errorf("wrong number of keys: got %v, want %v", + len(keys), len(blobs)) + } + + // Prepare log leaves. hashes and keys share the same ordering. + leaves := make([]*trillian.LogLeaf, 0, len(blobs)) + for k := range blobs { + pk := []byte(keyPrefix + keys[k]) + leaves = append(leaves, logLeafNew(hashes[k], pk)) + } + + // Append leaves to trillian tree + queued, _, err := t.trillian.leavesAppend(treeID, leaves) + if err != nil { + return nil, fmt.Errorf("leavesAppend: %v", err) + } + if len(queued) != len(leaves) { + return nil, fmt.Errorf("wrong number of queued leaves: got %v, want %v", + len(queued), len(leaves)) + } + failed := make([]string, 0, len(queued)) + for _, v := range queued { + c := codes.Code(v.QueuedLeaf.GetStatus().GetCode()) + if c != codes.OK { + failed = append(failed, fmt.Sprintf("%v", c)) + } + } + if len(failed) > 0 { + return nil, fmt.Errorf("append leaves failed: %v", failed) + } + + // Parse and return the merkle leaf hashes + merkles := make([][]byte, 0, len(blobs)) + for _, v := range queued { + merkles = append(merkles, v.QueuedLeaf.Leaf.MerkleLeafHash) + } + + return merkles, nil +} + +// del deletes the blobs that correspond to the provided merkle leaf hashes. +func (t *tlog) blobsDel(treeID int64, merkles [][]byte) error { + // Ensure tree exists. We allow blobs to be deleted from both + // frozen and non frozen trees. + if !t.treeExists(treeID) { + return errRecordNotFound + } + + // Get all tree leaves + leaves, err := t.trillian.leavesAll(treeID) + if err != nil { + return err + } + + // Put merkle leaf hashes into a map so that we can tell if a leaf + // corresponds to one of the target merkle leaf hashes in O(n) + // time. + merkleHashes := make(map[string]struct{}, len(leaves)) + for _, v := range merkles { + merkleHashes[hex.EncodeToString(v)] = struct{}{} + } + + // Aggregate the key-value store keys for the provided merkle leaf + // hashes. + keys := make([]string, 0, len(merkles)) + for _, v := range leaves { + _, ok := merkleHashes[hex.EncodeToString(v.MerkleLeafHash)] + if ok { + key, err := extractKeyFromLeaf(v) + if err != nil { + return err + } + keys = append(keys, key) + } + } + + // Delete file blobs from the store + err = t.store.Del(keys) + if err != nil { + return fmt.Errorf("store Del: %v", err) + } + + return nil +} + +// blobsByMerkle returns the blobs with the provided merkle leaf hashes. +// +// If a blob does not exist it will not be included in the returned map. It is +// the responsibility of the caller to check that a blob is returned for each +// of the provided merkle hashes. +func (t *tlog) blobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, error) { + // Get leaves + leavesAll, err := t.trillian.leavesAll(treeID) + if err != nil { + return nil, fmt.Errorf("leavesAll: %v", err) + } + + // Aggregate the leaves that correspond to the provided merkle + // hashes. + // map[merkleHash]*trillian.LogLeaf + leaves := make(map[string]*trillian.LogLeaf, len(merkles)) + for _, v := range merkles { + leaves[hex.EncodeToString(v)] = nil + } + for _, v := range leavesAll { + m := hex.EncodeToString(v.MerkleLeafHash) + if _, ok := leaves[m]; ok { + leaves[m] = v + } + } + + // Ensure a leaf was found for all provided merkle hashes + for k, v := range leaves { + if v == nil { + return nil, fmt.Errorf("leaf not found for merkle hash: %v", k) + } + } + + // Extract the key-value store keys. These keys MUST be put in the + // same order that the merkle hashes were provided in. + keys := make([]string, 0, len(leaves)) + for _, v := range merkles { + l, ok := leaves[hex.EncodeToString(v)] + if !ok { + return nil, fmt.Errorf("leaf not found for merkle hash: %x", v) + } + k, err := extractKeyFromLeaf(l) + if err != nil { + return nil, err + } + keys = append(keys, k) + } + + // Pull the blobs from the store. If is ok if one or more blobs is + // not found. It is the responsibility of the caller to decide how + // this should be handled. + blobs, err := t.store.Get(keys) + if err != nil { + return nil, fmt.Errorf("store Get: %v", err) + } + + // Decrypt any encrypted blobs + for k, v := range blobs { + if blobIsEncrypted(v) { + b, _, err := t.encryptionKey.decrypt(v) + if err != nil { + return nil, err + } + blobs[k] = b + } + } + + // Put blobs in a map so the caller can determine if any of the + // provided merkle hashes did not correspond to a blob in the + // store. + b := make(map[string][]byte, len(blobs)) // [merkleHash]blob + for k, v := range keys { + // The merkle hashes slice and keys slice share the same order + merkleHash := hex.EncodeToString(merkles[k]) + blob, ok := blobs[v] + if !ok { + return nil, fmt.Errorf("blob not found for key %v", v) + } + b[merkleHash] = blob + } + + return b, nil +} + +// blobsByKeyPrefix returns all blobs that match the provided key prefix. +func (t *tlog) blobsByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) { + // Get leaves + leaves, err := t.trillian.leavesAll(treeID) + if err != nil { + return nil, fmt.Errorf("leavesAll: %v", err) + } + + // Walk leaves and aggregate the key-value store keys for all + // leaves with a matching key prefix. + keys := make([]string, 0, len(leaves)) + for _, v := range leaves { + if bytes.HasPrefix(v.ExtraData, []byte(keyPrefix)) { + k, err := extractKeyFromLeaf(v) + if err != nil { + return nil, err + } + keys = append(keys, k) + } + } + + // Pull the blobs from the store + blobs, err := t.store.Get(keys) + if err != nil { + return nil, fmt.Errorf("store Get: %v", err) + } + if len(blobs) != len(keys) { + // One or more blobs were not found + missing := make([]string, 0, len(keys)) + for _, v := range keys { + _, ok := blobs[v] + if !ok { + missing = append(missing, v) + } + } + return nil, fmt.Errorf("blobs not found: %v", missing) + } + + // Decrypt any encrypted blobs + for k, v := range blobs { + if blobIsEncrypted(v) { + b, _, err := t.encryptionKey.decrypt(v) + if err != nil { + return nil, err + } + blobs[k] = b + } + } + + // Covert blobs from map to slice + b := make([][]byte, 0, len(blobs)) + for _, v := range blobs { + b = append(b, v) + } + + return b, nil +} + // TODO run fsck episodically func (t *tlog) fsck() { /* diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index feba4f49b..0f9287db6 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -34,16 +34,18 @@ import ( // TODO testnet vs mainnet trillian databases // TODO fsck // TODO allow token prefix lookups -// TODO we need to be able to run multiple politeiad instances. Its ok if -// we have a tree that seeds data between the instances. const ( defaultTrillianKeyFilename = "trillian.key" defaultEncryptionKeyFilename = "tlogbe.key" + + // Tlog instance IDs + tlogIDUnvetted = "unvetted" + tlogIDVetted = "vetted" ) var ( - _ backend.Backend = (*TlogBackend)(nil) + _ backend.Backend = (*tlogBackend)(nil) // statusChanges contains the allowed record status change // transitions. If statusChanges[currentStatus][newStatus] exists @@ -77,11 +79,11 @@ type plugin struct { id string version string settings []backend.PluginSetting - ctx Plugin + client pluginClient } -// TlogBackend implements the Backend interface. -type TlogBackend struct { +// tlogBackend implements the Backend interface. +type tlogBackend struct { sync.RWMutex shutdown bool homeDir string @@ -115,14 +117,14 @@ func tokenPrefix(token []byte) string { return hex.EncodeToString(token)[:pd.TokenPrefixLength] } -func (t *TlogBackend) isShutdown() bool { +func (t *tlogBackend) isShutdown() bool { t.RLock() defer t.RUnlock() return t.shutdown } -func (t *TlogBackend) prefixExists(fullToken []byte) bool { +func (t *tlogBackend) prefixExists(fullToken []byte) bool { t.RLock() defer t.RUnlock() @@ -130,7 +132,7 @@ func (t *TlogBackend) prefixExists(fullToken []byte) bool { return ok } -func (t *TlogBackend) prefixAdd(fullToken []byte) { +func (t *tlogBackend) prefixAdd(fullToken []byte) { t.Lock() defer t.Unlock() @@ -140,7 +142,7 @@ func (t *TlogBackend) prefixAdd(fullToken []byte) { log.Debugf("Token prefix cached: %v", prefix) } -func (t *TlogBackend) vettedTreeIDGet(token string) (int64, bool) { +func (t *tlogBackend) vettedTreeIDGet(token string) (int64, bool) { t.RLock() defer t.RUnlock() @@ -148,7 +150,7 @@ func (t *TlogBackend) vettedTreeIDGet(token string) (int64, bool) { return treeID, ok } -func (t *TlogBackend) vettedTreeIDAdd(token string, treeID int64) { +func (t *tlogBackend) vettedTreeIDAdd(token string, treeID int64) { t.Lock() defer t.Unlock() @@ -157,7 +159,62 @@ func (t *TlogBackend) vettedTreeIDAdd(token string, treeID int64) { log.Debugf("Vetted tree ID cached: %v %v", token, treeID) } -func (t *TlogBackend) inventoryGet() map[backend.MDStatusT][]string { +// vettedTreeIDFromToken returns the vetted tree ID that corresponds to the +// provided token. If a tree ID is not found then the returned bool will be +// false. +func (t *tlogBackend) vettedTreeIDFromToken(token []byte) (int64, bool) { + // Check if the token is in the vetted cache. The vetted cache is + // lazy loaded if the token is not present then we need to check + // manually. + treeID, ok := t.vettedTreeIDGet(hex.EncodeToString(token)) + if ok { + return treeID, true + } + + // The token is derived from the unvetted tree ID. Check if the + // token corresponds to an unvetted tree. + treeID = treeIDFromToken(token) + if !t.unvetted.treeExists(treeID) { + // Unvetted tree does not exists. This token does not correspond + // to any record. + return 0, false + } + + // Unvetted tree exists. Get the freeze record to see if it + // contains a pointer to a vetted tree. + fr, err := t.unvetted.freezeRecord(treeID) + if err != nil { + if err == errFreezeRecordNotFound { + // Unvetted tree exists and is not frozen. This is an unvetted + // record. + return 0, false + } + e := fmt.Sprintf("unvetted freezeRecord %v: %v", treeID, err) + panic(e) + } + if fr.TreeID == 0 { + // Unvetted tree has been frozen but does not contain a pointer + // to another tree. This means it was frozen for some other + // reason (ex. censored). This is not a vetted record. + return 0, false + } + + // Ensure the freeze record tree ID points to a valid vetted tree. + // This should not fail. + if !t.vetted.treeExists(fr.TreeID) { + // We're in trouble! + e := fmt.Sprintf("freeze record of unvetted tree %v points to "+ + "an invalid vetted tree %v", treeID, fr.TreeID) + panic(e) + } + + // Update the vetted cache + t.vettedTreeIDAdd(hex.EncodeToString(token), fr.TreeID) + + return fr.TreeID, true +} + +func (t *tlogBackend) inventoryGet() map[backend.MDStatusT][]string { t.RLock() defer t.RUnlock() @@ -174,7 +231,7 @@ func (t *TlogBackend) inventoryGet() map[backend.MDStatusT][]string { return inv } -func (t *TlogBackend) inventoryAdd(token string, s backend.MDStatusT) { +func (t *tlogBackend) inventoryAdd(token string, s backend.MDStatusT) { t.Lock() defer t.Unlock() @@ -183,7 +240,7 @@ func (t *TlogBackend) inventoryAdd(token string, s backend.MDStatusT) { log.Debugf("Inventory cache added: %v %v", token, backend.MDStatus[s]) } -func (t *TlogBackend) inventoryUpdate(token string, currStatus, newStatus backend.MDStatusT) { +func (t *tlogBackend) inventoryUpdate(token string, currStatus, newStatus backend.MDStatusT) { t.Lock() defer t.Unlock() @@ -474,7 +531,7 @@ func metadataStreamsUpdate(mdCurr, mdAppend, mdOverwrite []backend.MetadataStrea // New satisfies the Backend interface. // // This function satisfies the Backend interface. -func (t *TlogBackend) New(metadata []backend.MetadataStream, files []backend.File) (*backend.RecordMetadata, error) { +func (t *tlogBackend) New(metadata []backend.MetadataStream, files []backend.File) (*backend.RecordMetadata, error) { log.Tracef("New") // Validate record contents @@ -516,16 +573,16 @@ func (t *TlogBackend) New(metadata []backend.MetadataStream, files []backend.Fil } // Call pre plugin hooks - nr := NewRecord{ + nr := newRecord{ RecordMetadata: *rm, Metadata: metadata, Files: files, } - b, err := EncodeNewRecord(nr) + b, err := encodeNewRecord(nr) if err != nil { return nil, err } - err = t.pluginHook(HookNewRecordPre, string(b)) + err = t.pluginHook(hookNewRecordPre, string(b)) if err != nil { return nil, err } @@ -537,9 +594,9 @@ func (t *TlogBackend) New(metadata []backend.MetadataStream, files []backend.Fil } // Call post plugin hooks - err = t.pluginHook(HookNewRecordPost, string(b)) + err = t.pluginHook(hookNewRecordPost, string(b)) if err != nil { - log.Errorf("New: pluginHook %v: %v", Hooks[HookNewRecordPost], err) + log.Errorf("New: pluginHook %v: %v", hooks[hookNewRecordPost], err) } // Update the inventory cache @@ -551,7 +608,7 @@ func (t *TlogBackend) New(metadata []backend.MetadataStream, files []backend.Fil } // This function satisfies the Backend interface. -func (t *TlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { +func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { log.Tracef("UpdateUnvettedRecord: %x", token) // Validate record contents. Send in a single metadata array to @@ -601,7 +658,7 @@ func (t *TlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ } // Call pre plugin hooks - er := EditRecord{ + er := editRecord{ Current: *r, RecordMetadata: *recordMD, MDAppend: mdAppend, @@ -609,11 +666,11 @@ func (t *TlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ FilesAdd: filesAdd, FilesDel: filesDel, } - b, err := EncodeEditRecord(er) + b, err := encodeEditRecord(er) if err != nil { return nil, err } - err = t.pluginHook(HookEditRecordPre, string(b)) + err = t.pluginHook(hookEditRecordPre, string(b)) if err != nil { return nil, err } @@ -628,9 +685,9 @@ func (t *TlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ } // Call post plugin hooks - err = t.pluginHook(HookEditRecordPost, string(b)) + err = t.pluginHook(hookEditRecordPost, string(b)) if err != nil { - log.Errorf("pluginHook %v: %v", Hooks[HookEditRecordPost], err) + log.Errorf("pluginHook %v: %v", hooks[hookEditRecordPost], err) } // Update inventory cache. The inventory will only need to be @@ -651,7 +708,7 @@ func (t *TlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ } // This function satisfies the Backend interface. -func (t *TlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { +func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { log.Tracef("UpdateVettedRecord: %x", token) // Validate record contents. Send in a single metadata array to @@ -671,6 +728,12 @@ func (t *TlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b } } + // Get vetted tree ID + treeID, ok := t.vettedTreeIDFromToken(token) + if !ok { + return nil, backend.ErrRecordNotFound + } + // Apply the record changes and save the new version. The lock // needs to be held for the remainder of the function. t.vetted.Lock() @@ -680,7 +743,6 @@ func (t *TlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b } // Get existing record - treeID := treeIDFromToken(token) r, err := t.vetted.recordLatest(treeID) if err != nil { if err == errRecordNotFound { @@ -701,7 +763,7 @@ func (t *TlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b } // Call pre plugin hooks - er := EditRecord{ + er := editRecord{ Current: *r, RecordMetadata: *recordMD, MDAppend: mdAppend, @@ -709,11 +771,11 @@ func (t *TlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b FilesAdd: filesAdd, FilesDel: filesDel, } - b, err := EncodeEditRecord(er) + b, err := encodeEditRecord(er) if err != nil { return nil, err } - err = t.pluginHook(HookEditRecordPre, string(b)) + err = t.pluginHook(hookEditRecordPre, string(b)) if err != nil { return nil, err } @@ -728,9 +790,9 @@ func (t *TlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b } // Call post plugin hooks - err = t.pluginHook(HookEditRecordPost, string(b)) + err = t.pluginHook(hookEditRecordPost, string(b)) if err != nil { - log.Errorf("pluginHook %v: %v", Hooks[HookEditRecordPost], err) + log.Errorf("pluginHook %v: %v", hooks[hookEditRecordPost], err) } // Return updated record @@ -743,7 +805,7 @@ func (t *TlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b } // This function satisfies the Backend interface. -func (t *TlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite []backend.MetadataStream) error { +func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite []backend.MetadataStream) error { // Validate record contents. Send in a single metadata array to // verify there are no dups. allMD := append(mdAppend, mdOverwrite...) @@ -797,7 +859,7 @@ func (t *TlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite } // This function satisfies the Backend interface. -func (t *TlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite []backend.MetadataStream) error { +func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite []backend.MetadataStream) error { log.Tracef("UpdateVettedMetadata: %x", token) // Validate record contents. Send in a single metadata array to @@ -822,6 +884,12 @@ func (t *TlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ } } + // Get vetted tree ID + treeID, ok := t.vettedTreeIDFromToken(token) + if !ok { + return backend.ErrRecordNotFound + } + // Pull the existing record and apply the metadata updates. The // vetted lock must be held for the remainder of this function. t.vetted.Lock() @@ -831,7 +899,6 @@ func (t *TlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ } // Get existing record - treeID := treeIDFromToken(token) r, err := t.vetted.recordLatest(treeID) if err != nil { if err == errRecordNotFound { @@ -856,7 +923,7 @@ func (t *TlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ // record. // // This function satisfies the Backend interface. -func (t *TlogBackend) UnvettedExists(token []byte) bool { +func (t *tlogBackend) UnvettedExists(token []byte) bool { log.Tracef("UnvettedExists %x", token) // If the token is in the vetted cache then we know this is not an @@ -885,62 +952,15 @@ func (t *TlogBackend) UnvettedExists(token []byte) bool { } // This function satisfies the Backend interface. -func (t *TlogBackend) VettedExists(token []byte) bool { +func (t *tlogBackend) VettedExists(token []byte) bool { log.Tracef("VettedExists %x", token) - // Check if the token is in the vetted cache. The vetted cache is - // lazy loaded if the token is not present then we need to check - // manually. - _, ok := t.vettedTreeIDGet(hex.EncodeToString(token)) - if ok { - return true - } - - // The token is derived from the unvetted tree ID. Check if the - // token corresponds to an unvetted tree. - treeID := treeIDFromToken(token) - if !t.unvetted.treeExists(treeID) { - // Unvetted tree does not exists. This token does not correspond - // to any record. - return false - } - - // Unvetted tree exists. Get the freeze record to see if it - // contains a pointer to a vetted tree. - fr, err := t.unvetted.freezeRecord(treeID) - if err != nil { - if err == errFreezeRecordNotFound { - // Unvetted tree exists and is not frozen. This is an unvetted - // record. - return false - } - log.Errorf("unvetted freezeRecord %v: %v", treeID, err) - return false - } - if fr.TreeID == 0 { - // Unvetted tree has been frozen but does not contain a pointer - // to another tree. This means it was frozen for some other - // reason (ex. censored). This is not a vetted record. - return false - } - - // Ensure the freeze record tree ID points to a valid vetted tree. - // This should not fail. - if !t.vetted.treeExists(fr.TreeID) { - // We're in trouble! - e := fmt.Sprintf("freeze record of unvetted tree %v points to "+ - "an invalid vetted tree %v", treeID, fr.TreeID) - panic(e) - } - - // Update the vetted cache - t.vettedTreeIDAdd(hex.EncodeToString(token), fr.TreeID) - - return true + _, ok := t.vettedTreeIDFromToken(token) + return ok } // This function satisfies the Backend interface. -func (t *TlogBackend) GetUnvetted(token []byte, version string) (*backend.Record, error) { +func (t *tlogBackend) GetUnvetted(token []byte, version string) (*backend.Record, error) { log.Tracef("GetUnvetted: %x", token) if t.isShutdown() { @@ -953,28 +973,34 @@ func (t *TlogBackend) GetUnvetted(token []byte, version string) (*backend.Record return nil, backend.ErrRecordNotFound } - return t.unvetted.recordVersion(treeID, uint32(v)) + return t.unvetted.record(treeID, uint32(v)) } // This function satisfies the Backend interface. -func (t *TlogBackend) GetVetted(token []byte, version string) (*backend.Record, error) { +func (t *tlogBackend) GetVetted(token []byte, version string) (*backend.Record, error) { log.Tracef("GetVetted: %x", token) if t.isShutdown() { return nil, backend.ErrShutdown } - treeID := treeIDFromToken(token) + // Get tree ID + treeID, ok := t.vettedTreeIDFromToken(token) + if !ok { + return nil, backend.ErrRecordNotFound + } + + // Parse version v, err := strconv.ParseUint(version, 10, 64) if err != nil { return nil, backend.ErrRecordNotFound } - return t.vetted.recordVersion(treeID, uint32(v)) + return t.vetted.record(treeID, uint32(v)) } // This function must be called WITH the unvetted lock held. -func (t *TlogBackend) unvettedPublish(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { +func (t *tlogBackend) unvettedPublish(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { // Create a vetted tree var ( vettedToken []byte @@ -1026,7 +1052,7 @@ func (t *TlogBackend) unvettedPublish(token []byte, rm backend.RecordMetadata, m } // This function must be called WITH the unvetted lock held. -func (t *TlogBackend) unvettedCensor(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { +func (t *tlogBackend) unvettedCensor(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { // Freeze the tree treeID := treeIDFromToken(token) err := t.unvetted.treeFreeze(treeID, rm, metadata, freezeRecord{}) @@ -1048,7 +1074,7 @@ func (t *TlogBackend) unvettedCensor(token []byte, rm backend.RecordMetadata, me } // This function must be called WITH the unvetted lock held. -func (t *TlogBackend) unvettedArchive(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { +func (t *tlogBackend) unvettedArchive(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { // Freeze the tree. Nothing else needs to be done for an archived // record. treeID := treeIDFromToken(token) @@ -1062,7 +1088,7 @@ func (t *TlogBackend) unvettedArchive(token []byte, rm backend.RecordMetadata, m return nil } -func (t *TlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { +func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { log.Tracef("SetUnvettedStatus: %x %v (%v)", token, status, backend.MDStatus[status]) @@ -1100,17 +1126,17 @@ func (t *TlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) // Call pre plugin hooks - srs := SetRecordStatus{ + srs := setRecordStatus{ Current: *r, RecordMetadata: rm, MDAppend: mdAppend, MDOverwrite: mdOverwrite, } - b, err := EncodeSetRecordStatus(srs) + b, err := encodeSetRecordStatus(srs) if err != nil { return nil, err } - err = t.pluginHook(HookSetRecordStatusPre, string(b)) + err = t.pluginHook(hookSetRecordStatusPre, string(b)) if err != nil { return nil, err } @@ -1138,10 +1164,10 @@ func (t *TlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, } // Call post plugin hooks - err = t.pluginHook(HookSetRecordStatusPost, string(b)) + err = t.pluginHook(hookSetRecordStatusPost, string(b)) if err != nil { log.Errorf("SetUnvettedStatus: pluginHook %v: %v", - Hooks[HookSetRecordStatusPost], err) + hooks[hookSetRecordStatusPost], err) } // Update inventory cache @@ -1161,7 +1187,7 @@ func (t *TlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, } // This function must be called WITH the vetted lock held. -func (t *TlogBackend) vettedCensor(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { +func (t *tlogBackend) vettedCensor(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { // Freeze the tree treeID := treeIDFromToken(token) err := t.vetted.treeFreeze(treeID, rm, metadata, freezeRecord{}) @@ -1179,7 +1205,7 @@ func (t *TlogBackend) vettedCensor(token []byte, rm backend.RecordMetadata, meta } // This function must be called WITH the vetted lock held. -func (t *TlogBackend) vettedArchive(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { +func (t *tlogBackend) vettedArchive(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { // Freeze the tree. Nothing else needs to be done for an archived // record. treeID := treeIDFromToken(token) @@ -1187,10 +1213,16 @@ func (t *TlogBackend) vettedArchive(token []byte, rm backend.RecordMetadata, met } // This function satisfies the Backend interface. -func (t *TlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { +func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { log.Tracef("SetVettedStatus: %x %v (%v)", token, status, backend.MDStatus[status]) + // Get vetted tree ID + treeID, ok := t.vettedTreeIDFromToken(token) + if !ok { + return nil, backend.ErrRecordNotFound + } + // The existing record must be pulled and updated. The vetted lock // must be held for the rest of this function. t.vetted.Lock() @@ -1200,7 +1232,6 @@ func (t *TlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md } // Get existing record - treeID := treeIDFromToken(token) r, err := t.vetted.recordLatest(treeID) if err != nil { return nil, fmt.Errorf("recordLatest: %v", err) @@ -1225,17 +1256,17 @@ func (t *TlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) // Call pre plugin hooks - srs := SetRecordStatus{ + srs := setRecordStatus{ Current: *r, RecordMetadata: rm, MDAppend: mdAppend, MDOverwrite: mdOverwrite, } - b, err := EncodeSetRecordStatus(srs) + b, err := encodeSetRecordStatus(srs) if err != nil { return nil, err } - err = t.pluginHook(HookSetRecordStatusPre, string(b)) + err = t.pluginHook(hookSetRecordStatusPre, string(b)) if err != nil { return nil, err } @@ -1258,10 +1289,10 @@ func (t *TlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md } // Call post plugin hooks - err = t.pluginHook(HookSetRecordStatusPost, string(b)) + err = t.pluginHook(hookSetRecordStatusPost, string(b)) if err != nil { log.Errorf("SetVettedStatus: pluginHook %v: %v", - Hooks[HookSetRecordStatusPost], err) + hooks[hookSetRecordStatusPost], err) } // Update inventory cache @@ -1280,10 +1311,13 @@ func (t *TlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md return r, nil } -// Inventory is not currenctly implemented in tlogbe. +// Inventory is not currenctly implemented in tlogbe. If the caller which to +// pull records from the inventory then they should use the InventoryByStatus +// call to get the tokens of all records in the inventory and pull the required +// records individually. // // This function satisfies the Backend interface. -func (t *TlogBackend) Inventory(vettedCount uint, unvettedCount uint, includeFiles, allVersions bool) ([]backend.Record, []backend.Record, error) { +func (t *tlogBackend) Inventory(vettedCount uint, unvettedCount uint, includeFiles, allVersions bool) ([]backend.Record, []backend.Record, error) { log.Tracef("Inventory: %v %v", includeFiles, allVersions) return nil, nil, fmt.Errorf("not implemented") @@ -1293,7 +1327,7 @@ func (t *TlogBackend) Inventory(vettedCount uint, unvettedCount uint, includeFil // catagorized by MDStatusT. // // This function satisfies the Backend interface. -func (t *TlogBackend) InventoryByStatus() (*backend.InventoryByStatus, error) { +func (t *tlogBackend) InventoryByStatus() (*backend.InventoryByStatus, error) { log.Tracef("InventoryByStatus") inv := t.inventoryGet() @@ -1306,10 +1340,10 @@ func (t *TlogBackend) InventoryByStatus() (*backend.InventoryByStatus, error) { }, nil } -func (t *TlogBackend) RegisterPlugin(p backend.Plugin) error { +func (t *tlogBackend) RegisterPlugin(p backend.Plugin) error { log.Tracef("RegisterPlugin: %v", p.ID) - var ctx Plugin + var client pluginClient switch p.ID { case comments.ID: case dcrdata.ID: @@ -1323,13 +1357,13 @@ func (t *TlogBackend) RegisterPlugin(p backend.Plugin) error { id: p.ID, version: p.Version, settings: p.Settings, - ctx: ctx, + client: client, } return nil } -func (t *TlogBackend) SetupPlugin(pluginID string) error { +func (t *tlogBackend) SetupPlugin(pluginID string) error { log.Tracef("SetupPlugin: %v", pluginID) plugin, ok := t.plugins[pluginID] @@ -1337,14 +1371,14 @@ func (t *TlogBackend) SetupPlugin(pluginID string) error { return backend.ErrPluginInvalid } - return plugin.ctx.Setup() + return plugin.client.setup() } // GetPlugins returns the backend plugins that have been registered and their // settings. // // This function satisfies the Backend interface. -func (t *TlogBackend) GetPlugins() ([]backend.Plugin, error) { +func (t *tlogBackend) GetPlugins() ([]backend.Plugin, error) { log.Tracef("GetPlugins") plugins := make([]backend.Plugin, 0, len(t.plugins)) @@ -1362,7 +1396,7 @@ func (t *TlogBackend) GetPlugins() ([]backend.Plugin, error) { // Plugin is a pass-through function for plugin commands. // // This function satisfies the Backend interface. -func (t *TlogBackend) Plugin(pluginID, cmd, payload string) (string, error) { +func (t *tlogBackend) Plugin(pluginID, cmd, cmdID, payload string) (string, error) { log.Tracef("Plugin: %v %v", pluginID, cmd) if t.isShutdown() { @@ -1376,7 +1410,7 @@ func (t *TlogBackend) Plugin(pluginID, cmd, payload string) (string, error) { } // Execute plugin command - reply, err := plugin.ctx.Cmd(cmd, payload) + reply, err := plugin.client.cmd(cmd, payload) if err != nil { return "", err } @@ -1384,10 +1418,10 @@ func (t *TlogBackend) Plugin(pluginID, cmd, payload string) (string, error) { return reply, nil } -func (t *TlogBackend) pluginHook(h HookT, payload string) error { +func (t *tlogBackend) pluginHook(h hookT, payload string) error { // Pass hook event and payload to each plugin for _, v := range t.plugins { - err := v.ctx.Hook(h, payload) + err := v.client.hook(h, payload) if err != nil { return fmt.Errorf("Hook %v: %v", v.id, err) } @@ -1399,7 +1433,7 @@ func (t *TlogBackend) pluginHook(h HookT, payload string) error { // Close shuts the backend down and performs cleanup. // // This function satisfies the Backend interface. -func (t *TlogBackend) Close() { +func (t *tlogBackend) Close() { log.Tracef("Close") t.Lock() @@ -1413,7 +1447,7 @@ func (t *TlogBackend) Close() { t.vetted.close() } -func (t *TlogBackend) setup() error { +func (t *tlogBackend) setup() error { // Get all trees trees, err := t.unvetted.trillian.treesAll() if err != nil { @@ -1465,8 +1499,8 @@ func (t *TlogBackend) setup() error { return nil } -// New returns a new TlogBackend. -func New(homeDir, dataDir, dcrtimeHost, encryptionKeyFile, unvettedTrillianHost, unvettedTrillianKeyFile, vettedTrillianHost, vettedTrillianKeyFile string) (*TlogBackend, error) { +// New returns a new tlogBackend. +func New(homeDir, dataDir, dcrtimeHost, encryptionKeyFile, unvettedTrillianHost, unvettedTrillianKeyFile, vettedTrillianHost, vettedTrillianKeyFile string) (*tlogBackend, error) { // Setup encryption key file if encryptionKeyFile == "" { // No file path was given. Use the default path. @@ -1495,19 +1529,20 @@ func New(homeDir, dataDir, dcrtimeHost, encryptionKeyFile, unvettedTrillianHost, log.Infof("Anchor host: %v", dcrtimeHost) // Setup tlog instances - unvetted, err := newTlog("unvetted", homeDir, dataDir, unvettedTrillianHost, - unvettedTrillianKeyFile, dcrtimeHost, encryptionKeyFile) + unvetted, err := newTlog(tlogIDUnvetted, homeDir, dataDir, + unvettedTrillianHost, unvettedTrillianKeyFile, dcrtimeHost, + encryptionKeyFile) if err != nil { return nil, fmt.Errorf("newTlog unvetted: %v", err) } - vetted, err := newTlog("vetted", homeDir, dataDir, vettedTrillianHost, + vetted, err := newTlog(tlogIDVetted, homeDir, dataDir, vettedTrillianHost, vettedTrillianKeyFile, dcrtimeHost, "") if err != nil { return nil, fmt.Errorf("newTlog vetted: %v", err) } // Setup tlogbe - t := TlogBackend{ + t := tlogBackend{ homeDir: homeDir, dataDir: dataDir, unvetted: unvetted, diff --git a/politeiad/backend/tlogbe/tlogclient.go b/politeiad/backend/tlogbe/tlogclient.go new file mode 100644 index 000000000..2f19919e2 --- /dev/null +++ b/politeiad/backend/tlogbe/tlogclient.go @@ -0,0 +1,147 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlogbe + +import ( + "fmt" +) + +// tlogClient provides an interface for plugins to interact with the tlog +// backend. Plugins are allowed to save, delete, and get plugin data to/from +// the tlog backend. Editing plugin data is not allowed. +type tlogClient interface { + // save saves the provided blobs to the tlog backend. Note, hashes + // contains the hashes of the data encoded in the blobs. The hashes + // must share the same ordering as the blobs. + save(tlogID string, token []byte, keyPrefix string, + blobs, hashes [][]byte, encrypt bool) ([][]byte, error) + + // del deletes the blobs that correspond to the provided merkle + // leaf hashes. + del(tlogID string, token []byte, merkles [][]byte) error + + // blobsByMerkle returns the blobs with the provided merkle leaf + // hashes. If a blob does not exist it will not be included in the + // returned map. + blobsByMerkle(tlogID string, token []byte, + merkles [][]byte) (map[string][]byte, error) + + // blobsByKeyPrefix returns all blobs that match the provided key + // prefix. + blobsByKeyPrefix(tlogID string, token []byte, + keyPrefix string) ([][]byte, error) +} + +var ( + _ tlogClient = (*backendClient)(nil) +) + +// backendClient implements the tlogClient interface. +type backendClient struct { + backend *tlogBackend +} + +// tlogByID returns the tlog instance that corresponds to the provided ID. +func (c *backendClient) tlogByID(tlogID string) (*tlog, error) { + switch tlogID { + case tlogIDUnvetted: + return c.backend.unvetted, nil + case tlogIDVetted: + return c.backend.vetted, nil + } + return nil, fmt.Errorf("unknown tlog id '%v'", tlogID) +} + +// treeIDFromToken returns the treeID for the provided tlog instance ID and +// token. +func (c *backendClient) treeIDFromToken(tlogID string, token []byte) (int64, error) { + switch tlogID { + case tlogIDUnvetted: + return treeIDFromToken(token), nil + case tlogIDVetted: + treeID, ok := c.backend.vettedTreeIDFromToken(token) + if !ok { + return 0, errRecordNotFound + } + return treeID, nil + } + return 0, fmt.Errorf("unknown tlog id '%v'", tlogID) +} + +// save saves the provided blobs to the tlog backend. Note, hashes contains the +// hashes of the data encoded in the blobs. The hashes must share the same +// ordering as the blobs. +func (c *backendClient) save(tlogID string, token []byte, keyPrefix string, blobs, hashes [][]byte, encrypt bool) ([][]byte, error) { + log.Tracef("save: %x %v %v %x", token, keyPrefix, encrypt, hashes) + + // Get tlog instance and treeID + tlog, err := c.tlogByID(tlogID) + if err != nil { + return nil, err + } + treeID, err := c.treeIDFromToken(tlogID, token) + if err != nil { + return nil, err + } + + // Save blobs + return tlog.blobsSave(treeID, keyPrefix, blobs, hashes, encrypt) +} + +// del deletes the blobs that correspond to the provided merkle leaf hashes. +func (c *backendClient) del(tlogID string, token []byte, merkles [][]byte) error { + log.Tracef("del: %x %x", token, merkles) + + // Get tlog instance and treeID + tlog, err := c.tlogByID(tlogID) + if err != nil { + return err + } + treeID, err := c.treeIDFromToken(tlogID, token) + if err != nil { + return err + } + + // Save blobs + return tlog.blobsDel(treeID, merkles) +} + +// blobsByMerkle returns the blobs with the provided merkle leaf hashes. +// +// If a blob does not exist it will not be included in the returned map. It is +// the responsibility of the caller to check that a blob is returned for each +// of the provided merkle hashes. +func (c *backendClient) blobsByMerkle(tlogID string, token []byte, merkles [][]byte) (map[string][]byte, error) { + log.Tracef("blobsByMerkle: %x %x", token, merkles) + + // Get tlog instance and treeID + tlog, err := c.tlogByID(tlogID) + if err != nil { + return nil, err + } + treeID, err := c.treeIDFromToken(tlogID, token) + if err != nil { + return nil, err + } + + return tlog.blobsByMerkle(treeID, merkles) +} + +// blobsByKeyPrefix returns all blobs that match the provided key prefix. +func (c *backendClient) blobsByKeyPrefix(tlogID string, token []byte, keyPrefix string) ([][]byte, error) { + log.Tracef("blobsByKeyPrefix: %x %x", token, keyPrefix) + + // Get tlog instance and treeID + tlog, err := c.tlogByID(tlogID) + if err != nil { + return nil, err + } + treeID, err := c.treeIDFromToken(tlogID, token) + if err != nil { + return nil, err + } + + return tlog.blobsByKeyPrefix(treeID, keyPrefix) +} diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index ea5d300b6..185a8f70f 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -922,7 +922,8 @@ func (p *politeia) pluginCommand(w http.ResponseWriter, r *http.Request) { return } - payload, err := p.backend.Plugin(pc.ID, pc.Command, pc.Payload) + payload, err := p.backend.Plugin(pc.ID, pc.Command, + pc.CommandID, pc.Payload) if err != nil { // Generic internal error. errorCode := time.Now().Unix() From 506dd9322a759e972e75a5980dfadba1a882b633 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 17 Sep 2020 09:31:14 -0500 Subject: [PATCH 067/449] Add pi CmdVoteInventory --- plugins/pi/pi.go | 52 +++++++++++++++++++++++++- plugins/ticketvote/ticketvote.go | 3 -- politeiad/backend/tlogbe/plugins/pi.go | 3 +- politeiad/backend/tlogbe/tlogbe.go | 1 + 4 files changed, 54 insertions(+), 5 deletions(-) diff --git a/plugins/pi/pi.go b/plugins/pi/pi.go index 25ff2373a..44b809dd5 100644 --- a/plugins/pi/pi.go +++ b/plugins/pi/pi.go @@ -21,7 +21,8 @@ const ( ID = "pi" // Plugin commands - CmdProposals = "proposals" // Get proposals plugin data + CmdProposals = "proposals" // Get plugin data for proposals + CmdVoteInventory = "voteinventory" // Get inventory by vote status // Metadata stream IDs. All metadata streams in this plugin will // use 1xx numbering. @@ -249,3 +250,52 @@ func DecodeProposalsReply(payload []byte) (*ProposalsReply, error) { } return &pr, nil } + +// VoteInventory requests the tokens of all proposals in the inventory +// catagorized by their vote status. The difference between this call and the +// ticketvote Inventory call is that this call breaks the Finished vote status +// out into Approved and Rejected catagories, which is specific to pi. +type VoteInventory struct{} + +// EncodeVoteInventory encodes a VoteInventory into a JSON byte slice. +func EncodeVoteInventory(vi VoteInventory) ([]byte, error) { + return json.Marshal(vi) +} + +// DecodeVoteInventory decodes a JSON byte slice into a VoteInventory. +func DecodeVoteInventory(payload []byte) (*VoteInventory, error) { + var vi VoteInventory + err := json.Unmarshal(payload, &vi) + if err != nil { + return nil, err + } + return &vi, nil +} + +// VoteInventoryReply is the reply to the VoteInventory command. +type VoteInventoryReply struct { + Unauthorized []string `json:"unauthorized"` + Authorized []string `json:"authorized"` + Started []string `json:"started"` + Approved []string `json:"approved"` + Rejected []string `json:"rejected"` + + // BestBlock is the best block value that was used to prepare the + // inventory. + BestBlock uint32 `json:"bestblock"` +} + +// EncodeVoteInventoryReply encodes a VoteInventoryReply into a JSON byte slice. +func EncodeVoteInventoryReply(vir VoteInventoryReply) ([]byte, error) { + return json.Marshal(vir) +} + +// DecodeVoteInventoryReply decodes a JSON byte slice into a VoteInventoryReply. +func DecodeVoteInventoryReply(payload []byte) (*VoteInventoryReply, error) { + var vir VoteInventoryReply + err := json.Unmarshal(payload, &vir) + if err != nil { + return nil, err + } + return &vir, nil +} diff --git a/plugins/ticketvote/ticketvote.go b/plugins/ticketvote/ticketvote.go index 5bc55f34d..a7edb9652 100644 --- a/plugins/ticketvote/ticketvote.go +++ b/plugins/ticketvote/ticketvote.go @@ -610,9 +610,6 @@ func DecodeInventory(payload []byte) (*Inventory, error) { // // Sorted by voting period end block height in descending order: // Started, Finished -// -// TODO the pi plugin will need to catagorize finished into approved and -// rejected. type InventoryReply struct { Unauthorized []string `json:"unauthorized"` Authorized []string `json:"authorized"` diff --git a/politeiad/backend/tlogbe/plugins/pi.go b/politeiad/backend/tlogbe/plugins/pi.go index 1a435a61b..b6b8ead06 100644 --- a/politeiad/backend/tlogbe/plugins/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi.go @@ -510,9 +510,10 @@ func (p *piPlugin) Cmd(cmd, payload string) (string, error) { switch cmd { case pi.CmdProposals: return p.cmdProposals(payload) + // TODO case pi.CmdVoteInventory } - return "", nil + return "", backend.ErrPluginCmdInvalid } func (p *piPlugin) Hook(h tlogbe.HookT, payload string) error { diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 0f9287db6..b22cbc3ae 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -31,6 +31,7 @@ import ( "github.com/subosito/gozaru" ) +// TODO we need an unvetted censored status // TODO testnet vs mainnet trillian databases // TODO fsck // TODO allow token prefix lookups From 76e301d0cc8a3aac57306d8e75d13a469d2041f7 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 17 Sep 2020 10:15:22 -0500 Subject: [PATCH 068/449] Add record state to comment plugin --- plugins/comments/comments.go | 48 +++++++++++++++++--- politeiad/backend/tlogbe/plugins/comments.go | 2 + 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/plugins/comments/comments.go b/plugins/comments/comments.go index a72f3c654..396f20de6 100644 --- a/plugins/comments/comments.go +++ b/plugins/comments/comments.go @@ -11,6 +11,7 @@ import ( "fmt" ) +type StateT int type VoteT int type ErrorStatusT int @@ -30,6 +31,11 @@ const ( CmdVote = "vote" // Vote on a comment CmdProofs = "proofs" // Get inclusion proofs + // Record states + StateInvalid StateT = 0 + StateUnvetted StateT = 1 + StateVetted StateT = 2 + // Comment vote types VoteInvalid VoteT = 0 VoteDownvote VoteT = -1 @@ -84,7 +90,10 @@ func (e UserErrorReply) Error() string { } // Comment represent a record comment. +// +// Signature is the client signature of State+Token+ParentID+Comment. type Comment struct { + State StateT `json:"state"` // Record state Token string `json:"token"` // Record token ParentID uint32 `json:"parentid"` // Parent comment ID if reply Comment string `json:"comment"` // Comment text @@ -101,13 +110,16 @@ type Comment struct { // CommentAdd is the structure that is saved to disk when a comment is created // or edited. +// +// Signature is the client signature of State+Token+ParentID+Comment. type CommentAdd struct { // Data generated by client + State StateT `json:"state"` // Record state Token string `json:"token"` // Record token ParentID uint32 `json:"parentid"` // Parent comment ID Comment string `json:"comment"` // Comment PublicKey string `json:"publickey"` // Pubkey used for Signature - Signature string `json:"signature"` // Signature of Token+ParentID+Comment + Signature string `json:"signature"` // Client signature // Metadata generated by server CommentID uint32 `json:"commentid"` // Comment ID @@ -121,13 +133,16 @@ type CommentAdd struct { // saved since all the comment add records will be deleted and the client needs // these additional fields to properly display the deleted comment in the // comment hierarchy. +// +// Signature is the client signature of the State+Token+CommentID+Reason type CommentDel struct { // Data generated by client + State StateT `json:"state"` // Record state Token string `json:"token"` // Record token CommentID uint32 `json:"commentid"` // Comment ID Reason string `json:"reason"` // Reason for deleting PublicKey string `json:"publickey"` // Pubkey used for Signature - Signature string `json:"signature"` // Signature of Token+CommentID+Reason + Signature string `json:"signature"` // Client signature // Metadata generated by server ParentID uint32 `json:"parentid"` // Parent comment ID @@ -138,14 +153,17 @@ type CommentDel struct { // CommentVote is the structure that is saved to disk when a comment is voted // on. +// +// Signature is the client signature of the State+Token+CommentID+Vote. type CommentVote struct { // Data generated by client + State StateT `json:"state"` // Record state UUID string `json:"uuid"` // Unique user ID Token string `json:"token"` // Record token CommentID uint32 `json:"commentid"` // Comment ID Vote int64 `json:"vote"` // Upvote or downvote PublicKey string `json:"publickey"` // Public key used for signature - Signature string `json:"signature"` // Signature of Token+CommentID+Vote + Signature string `json:"signature"` // Client signature // Metadata generated by server Timestamp int64 `json:"timestamp"` // Received UNIX timestamp @@ -154,12 +172,15 @@ type CommentVote struct { // New creates a new comment. A parent ID of 0 indicates that the comment is // a base level comment and not a reply commment. +// +// Signature is the client signature of State+Token+ParentID+Comment. type New struct { + State StateT `json:"state"` // Record state Token string `json:"token"` // Record token ParentID uint32 `json:"parentid"` // Parent comment ID Comment string `json:"comment"` // Comment text PublicKey string `json:"publickey"` // Pubkey used for Signature - Signature string `json:"signature"` // Signature of Token+ParentID+Comment + Signature string `json:"signature"` // Client signature } // EncodeNew encodes a New into a JSON byte slice. @@ -200,13 +221,16 @@ func DecodeNewReply(payload []byte) (*NewReply, error) { } // Edit edits an existing comment. +// +// Signature is the client signature of State+Token+ParentID+Comment. type Edit struct { + State StateT `json:"state"` // Record state Token string `json:"token"` // Record token ParentID uint32 `json:"parentid"` // Parent comment ID CommentID uint32 `json:"commentid"` // Comment ID Comment string `json:"comment"` // Comment text PublicKey string `json:"publickey"` // Pubkey used for Signature - Signature string `json:"signature"` // Signature of Token+ParentID+Comment + Signature string `json:"signature"` // Client signature } // EncodeEdit encodes a Edit into a JSON byte slice. @@ -247,12 +271,15 @@ func DecodeEditReply(payload []byte) (*EditReply, error) { } // Del permanently deletes all versions of the provided comment. +// +// Signature is the client signature of the State+Token+CommentID+Reason type Del struct { + State StateT `json:"state"` // Record state Token string `json:"token"` // Record token CommentID uint32 `json:"commentid"` // Comment ID Reason string `json:"reason"` // Reason for deletion PublicKey string `json:"publickey"` // Public key used for signature - Signature string `json:"signature"` // Signature of Token+CommentID+Reason + Signature string `json:"signature"` // Client signature } // EncodeDel encodes a Del into a JSON byte slice. @@ -295,6 +322,7 @@ func DecodeDelReply(payload []byte) (*DelReply, error) { // An error is not returned if a comment is not found for one or more of the // comment IDs. Those entries will simply not be included in the reply. type Get struct { + State StateT `json:"state"` Token string `json:"token"` CommentIDs []uint32 `json:"commentids"` } @@ -339,6 +367,7 @@ func DecodeGetReply(payload []byte) (*GetReply, error) { // GetAll returns the latest version off all comments for the provided record. type GetAll struct { + State StateT `json:"state"` Token string `json:"token"` } @@ -379,6 +408,7 @@ func DecodeGetAllReply(payload []byte) (*GetAllReply, error) { // GetVersion returns a specific version of a comment. type GetVersion struct { + State StateT `json:"state"` Token string `json:"token"` CommentID uint32 `json:"commentid"` Version uint32 `json:"version"` @@ -421,6 +451,7 @@ func DecodeGetVersionReply(payload []byte) (*GetVersionReply, error) { // Count returns the comments count for the provided record. type Count struct { + State StateT `json:"state"` Token string `json:"token"` } @@ -466,13 +497,16 @@ func DecodeCountReply(payload []byte) (*CountReply, error) { // comment that they have already upvoted, the resulting vote score is 0 due to // the second upvote removing the original upvote. The public key cannot be // relied on to remain the same for each user so a uuid must be included. +// +// Signature is the client signature of the State+Token+CommentID+Vote. type Vote struct { + State StateT `json:"state"` // Record state UUID string `json:"uuid"` // Unique user ID Token string `json:"token"` // Record token CommentID uint32 `json:"commentid"` // Comment ID Vote VoteT `json:"vote"` // Upvote or downvote PublicKey string `json:"publickey"` // Public key used for signature - Signature string `json:"signature"` // Signature of Token+CommentID+Vote + Signature string `json:"signature"` // Client signature } // EncodeVote encodes a Vote into a JSON byte slice. diff --git a/politeiad/backend/tlogbe/plugins/comments.go b/politeiad/backend/tlogbe/plugins/comments.go index b6845710e..bab76d1b9 100644 --- a/politeiad/backend/tlogbe/plugins/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments.go @@ -27,6 +27,8 @@ import ( // TODO don't save data to the file system. Save it to the kv store and save // the key to the file system. This will allow the data to be backed up. +// TODO comment signature messages need to have state added to them. + const ( // Blob entry data descriptors dataDescriptorCommentAdd = "commentadd" From d58c357b1d6c4948750ac6cc343044121952dbb7 Mon Sep 17 00:00:00 2001 From: Amir Massarwa Date: Fri, 18 Sep 2020 03:22:59 +0300 Subject: [PATCH 069/449] politeiawww: Add processTokenInventory. --- go.sum | 1 + politeiawww/proposals.go | 79 +++++++++++++++++++--------------------- 2 files changed, 38 insertions(+), 42 deletions(-) diff --git a/go.sum b/go.sum index fbd538b5e..e88d6aa10 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,7 @@ cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7h github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkByhMAwYaFAX9w2l7vxvBQ5NMoxDrkhqhtn4= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Sereal/Sereal v0.0.0-20190618215532-0b8ac451a863 h1:BRrxwOZBolJN4gIwvZMJY1tzqBvQgpaZiQRuIDD40jM= diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index efc5a88e5..0a0c1519b 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -1602,57 +1602,52 @@ func (p *politeiawww) processStartVoteRunoffV2(sv www2.StartVoteRunoff, u *user. }, nil } -// tokenInventory fetches the token inventory from the cache and returns a -// TokenInventoryReply. This call relies on the lazy loaded VoteResults cache -// table. If the VoteResults table is not up-to-date then this function will -// load it before retrying the token inventory call. Since politeiawww only has -// read access to the cache, loading the VoteResults table requires using a -// politeiad decredplugin command. -func (p *politeiawww) tokenInventory(bestBlock uint64, isAdmin bool) (*www.TokenInventoryReply, error) { - /* - var done bool - var r www.TokenInventoryReply - for retries := 0; !done && retries <= 1; retries++ { - // Both vetted and unvetted tokens should be returned - // for admins. Only vetted tokens should be returned - // for non-admins. - ti, err := p.decredTokenInventory(bestBlock, isAdmin) - if err != nil { - if err == cache.ErrRecordNotFound { - // There are missing entries in the vote - // results cache table. Load them. - _, err := p.decredLoadVoteResults(bestBlock) - if err != nil { - return nil, err - } - - // Retry token inventory call - continue - } - return nil, err - } - - r = convertTokenInventoryReplyFromDecred(*ti) - done = true - } - - return &r, nil - */ - - return nil, nil -} - // processTokenInventory returns the tokens of all proposals in the inventory, // categorized by stage of the voting process. func (p *politeiawww) processTokenInventory(isAdmin bool) (*www.TokenInventoryReply, error) { log.Tracef("processTokenInventory") - bb, err := p.getBestBlock() + // Prep plugin command to get tokens by vote statuses + var vip piplugin.VoteInventory + payload, err := piplugin.EncodeVoteInventory(vip) if err != nil { return nil, err } - return p.tokenInventory(bb, isAdmin) + r, err := p.pluginCommand(piplugin.ID, piplugin.CmdVoteInventory, "", + string(payload)) + if err != nil { + return nil, err + } + vi, err := piplugin.DecodeVoteInventoryReply([]byte(r)) + if err != nil { + return nil, err + } + + // Translate reply to www + res := www.TokenInventoryReply{ + Pre: append(vi.Unauthorized, vi.Authorized...), + Active: vi.Started, + Approved: vi.Approved, + Rejected: vi.Rejected, + } + + // Call politeiad to get tokens by record statuses + isReply, err := p.inventoryByStatus() + if err != nil { + return nil, err + } + + // Fill info + res.Abandoned = isReply.Archived + + // Add admins only data + if isAdmin { + res.Censored = isReply.Censored + res.Unreviewed = isReply.Unvetted + } + + return &res, nil } // processVoteDetailsV2 returns the vote details for the given proposal token. From a39620299d8a50b8da1cf6565ff9b5a77f4dfc40 Mon Sep 17 00:00:00 2001 From: Amir Massarwa Date: Fri, 18 Sep 2020 18:01:33 +0300 Subject: [PATCH 070/449] politeiawww: Separate ticketvote plugin logic. --- politeiawww/proposals.go | 81 +++++----------------------- politeiawww/ticketvote.go | 110 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 69 deletions(-) create mode 100644 politeiawww/ticketvote.go diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 0a0c1519b..cb6d43e18 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -248,7 +248,7 @@ func convertStatusToWWW(status pi.PropStatusT) www.PropStatusT { } } -func (p *politeiawww) convertProposalToWWW(pr *pi.ProposalRecord) (*www.ProposalRecord, error) { +func convertProposalToWWW(pr *pi.ProposalRecord) (*www.ProposalRecord, error) { // Decode metadata var pm *piplugin.ProposalMetadata for _, v := range pr.Metadata { @@ -336,7 +336,7 @@ func (p *politeiawww) processProposalDetails(pd www.ProposalsDetails, u *user.Us if err != nil { return nil, err } - pw, err := p.convertProposalToWWW(pr) + pw, err := convertProposalToWWW(pr) if err != nil { return nil, err } @@ -366,7 +366,7 @@ func (p *politeiawww) processBatchProposals(bp www.BatchProposals, u *user.User) // Convert proposals records propsw := make([]www.ProposalRecord, 0, len(bp.Tokens)) for _, pr := range props { - propw, err := p.convertProposalToWWW(&pr) + propw, err := convertProposalToWWW(&pr) if err != nil { return nil, err } @@ -456,18 +456,8 @@ func (p *politeiawww) processActiveVote() (*www.ActiveVoteReply, error) { func (p *politeiawww) processVoteResults(token string) (*www.VoteResultsReply, error) { log.Tracef("processVoteResults: %v", token) - // Prep vote details payload - vdp := ticketvote.Details{ - Token: token, - } - payload, err := ticketvote.EncodeDetails(vdp) - if err != nil { - return nil, err - } - - r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdDetails, "", - string(payload)) - vd, err := ticketvote.DecodeDetailsReply([]byte(r)) + // Call ticketvote plugin + vd, err := p.voteDetails(token) if err != nil { return nil, err } @@ -505,22 +495,10 @@ func (p *politeiawww) processVoteResults(token string) (*www.VoteResultsReply, e } res.StartVote.Vote.Options = vo - // Prep cast votes payload - csp := ticketvote.CastVotes{ - Token: token, - } - payload, err = ticketvote.EncodeCastVotes(csp) - - r, err = p.pluginCommand(ticketvote.ID, ticketvote.CmdCastVotes, "", - string(payload)) - if err != nil { - return nil, err - } - cv, err := ticketvote.DecodeCastVotesReply([]byte(r)) - if err != nil { - return nil, err - } + // Get cast votes information + cv, err := p.castVotes(token) + // Transalte to www votes := make([]www.CastVote, 0, len(cv.Votes)) for _, v := range cv.Votes { votes = append(votes, www.CastVote{ @@ -568,21 +546,8 @@ func convertVoteTypeToWWW(t ticketvote.VoteT) www.VoteT { func (p *politeiawww) processBatchVoteSummary(bvs www.BatchVoteSummary) (*www.BatchVoteSummaryReply, error) { log.Tracef("processBatchVoteSummary: %v", bvs.Tokens) - // Prep plugin command - smp := ticketvote.Summaries{ - Tokens: bvs.Tokens, - } - payload, err := ticketvote.EncodeSummaries(smp) - if err != nil { - return nil, err - } - - r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdSummaries, "", - string(payload)) - if err != nil { - return nil, err - } - sm, err := ticketvote.DecodeSummariesReply([]byte(r)) + // Call ticketvote plugin to get vote summaries + sm, err := p.voteSummaries(bvs.Tokens) if err != nil { return nil, err } @@ -649,30 +614,8 @@ func convertVoteErrorCodeToWWW(errcode ticketvote.VoteErrorT) decredplugin.Error func (p *politeiawww) processCastVotes(ballot *www.Ballot) (*www.BallotReply, error) { log.Tracef("processCastVotes") - // Prep plugin command - var bp ticketvote.Ballot - // Transale votes - votes := make([]ticketvote.Vote, 0, len(ballot.Votes)) - for _, vote := range ballot.Votes { - votes = append(votes, ticketvote.Vote{ - Token: vote.Ticket, - Ticket: vote.Ticket, - VoteBit: vote.VoteBit, - Signature: vote.Signature, - }) - } - bp.Votes = votes - payload, err := ticketvote.EncodeBallot(bp) - if err != nil { - return nil, err - } - - r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdBallot, "", - string(payload)) - if err != nil { - return nil, err - } - b, err := ticketvote.DecodeBallotReply([]byte(r)) + // Call ticketvote plugin to cast votes + b, err := p.ballot(ballot) if err != nil { return nil, err } diff --git a/politeiawww/ticketvote.go b/politeiawww/ticketvote.go new file mode 100644 index 000000000..8c32d14e6 --- /dev/null +++ b/politeiawww/ticketvote.go @@ -0,0 +1,110 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + ticketvote "github.com/decred/politeia/plugins/ticketvote" + www "github.com/decred/politeia/politeiawww/api/www/v1" +) + +// voteDetails calls the ticketvote plugin command to get vote details. +func (p *politeiawww) voteDetails(token string) (*ticketvote.DetailsReply, error) { + // Prep vote details payload + vdp := ticketvote.Details{ + Token: token, + } + payload, err := ticketvote.EncodeDetails(vdp) + if err != nil { + return nil, err + } + + r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdDetails, "", + string(payload)) + vd, err := ticketvote.DecodeDetailsReply([]byte(r)) + if err != nil { + return nil, err + } + + return vd, nil +} + +// castVotes calls the ticketvote plugin to retrieve cast votes. +func (p *politeiawww) castVotes(token string) (*ticketvote.CastVotesReply, error) { + // Prep cast votes payload + csp := ticketvote.CastVotes{ + Token: token, + } + payload, err := ticketvote.EncodeCastVotes(csp) + + r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdCastVotes, "", + string(payload)) + if err != nil { + return nil, err + } + cv, err := ticketvote.DecodeCastVotesReply([]byte(r)) + if err != nil { + return nil, err + } + + return cv, nil +} + +// ballot calls the ticketvote plugin to cast a ballot of votes. +func (p *politeiawww) ballot(ballot *www.Ballot) (*ticketvote.BallotReply, error) { + // Prep plugin command + var bp ticketvote.Ballot + + // Transale votes + votes := make([]ticketvote.Vote, 0, len(ballot.Votes)) + for _, vote := range ballot.Votes { + votes = append(votes, ticketvote.Vote{ + Token: vote.Ticket, + Ticket: vote.Ticket, + VoteBit: vote.VoteBit, + Signature: vote.Signature, + }) + } + bp.Votes = votes + payload, err := ticketvote.EncodeBallot(bp) + if err != nil { + return nil, err + } + + r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdBallot, "", + string(payload)) + if err != nil { + return nil, err + } + b, err := ticketvote.DecodeBallotReply([]byte(r)) + if err != nil { + return nil, err + } + + return b, nil +} + +// summaries calls the ticketvote plugin to get vote summary information. +func (p *politeiawww) voteSummaries(tokens []string) (*ticketvote.SummariesReply, error) { + // Prep plugin command + smp := ticketvote.Summaries{ + Tokens: tokens, + } + payload, err := ticketvote.EncodeSummaries(smp) + if err != nil { + return nil, err + } + + r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdSummaries, "", + string(payload)) + if err != nil { + return nil, err + } + sm, err := ticketvote.DecodeSummariesReply([]byte(r)) + if err != nil { + return nil, err + } + + return sm, nil +} From 7755c0fe74f4e53802600c98a0bcd012ea8220f9 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 17 Sep 2020 17:23:16 -0500 Subject: [PATCH 071/449] Update pi comments plugin and api --- plugins/comments/comments.go | 130 +++++++------ plugins/pi/pi.go | 184 +++++++++++++++++- plugins/ticketvote/ticketvote.go | 24 +-- politeiad/backend/tlogbe/plugins/comments.go | 1 + politeiawww/api/pi/v1/v1.go | 189 +++++++++++++++++++ politeiawww/api/www/v1/v1.go | 4 +- 6 files changed, 454 insertions(+), 78 deletions(-) diff --git a/plugins/comments/comments.go b/plugins/comments/comments.go index 396f20de6..a55a2b1f4 100644 --- a/plugins/comments/comments.go +++ b/plugins/comments/comments.go @@ -31,6 +31,8 @@ const ( CmdVote = "vote" // Vote on a comment CmdProofs = "proofs" // Get inclusion proofs + // TODO CmdUserVotes = "uservotes" // Get votes from a specific uuid + // Record states StateInvalid StateT = 0 StateUnvetted StateT = 1 @@ -41,8 +43,8 @@ const ( VoteDownvote VoteT = -1 VoteUpvote VoteT = 1 - // TODO should these policies be plugin settings? - // TODO PolicyMaxCommentLength + // TODO Add plugin policies + // PolicyMaxCommentLength // PolicayMaxVoteChanges is the maximum number times a user can // change their vote on a comment. This prevents a malicious user @@ -93,6 +95,7 @@ func (e UserErrorReply) Error() string { // // Signature is the client signature of State+Token+ParentID+Comment. type Comment struct { + UUID string `json:"uuid"` // Unique user ID State StateT `json:"state"` // Record state Token string `json:"token"` // Record token ParentID uint32 `json:"parentid"` // Parent comment ID if reply @@ -114,6 +117,7 @@ type Comment struct { // Signature is the client signature of State+Token+ParentID+Comment. type CommentAdd struct { // Data generated by client + UUID string `json:"uuid"` // Unique user ID State StateT `json:"state"` // Record state Token string `json:"token"` // Record token ParentID uint32 `json:"parentid"` // Parent comment ID @@ -157,8 +161,8 @@ type CommentDel struct { // Signature is the client signature of the State+Token+CommentID+Vote. type CommentVote struct { // Data generated by client - State StateT `json:"state"` // Record state UUID string `json:"uuid"` // Unique user ID + State StateT `json:"state"` // Record state Token string `json:"token"` // Record token CommentID uint32 `json:"commentid"` // Comment ID Vote int64 `json:"vote"` // Upvote or downvote @@ -170,11 +174,14 @@ type CommentVote struct { Receipt string `json:"receipt"` // Server signature of client signature } -// New creates a new comment. A parent ID of 0 indicates that the comment is -// a base level comment and not a reply commment. +// New creates a new comment. +// +// The parent ID is used to reply to an existing comment. A parent ID of 0 +// indicates that the comment is a base level comment and not a reply commment. // // Signature is the client signature of State+Token+ParentID+Comment. type New struct { + UUID string `json:"uuid"` // Unique user ID State StateT `json:"state"` // Record state Token string `json:"token"` // Record token ParentID uint32 `json:"parentid"` // Parent comment ID @@ -206,24 +213,25 @@ type NewReply struct { } // EncodeNew encodes a NewReply into a JSON byte slice. -func EncodeNewReply(r NewReply) ([]byte, error) { - return json.Marshal(r) +func EncodeNewReply(nr NewReply) ([]byte, error) { + return json.Marshal(nr) } // DecodeNew decodes a JSON byte slice into a NewReply. func DecodeNewReply(payload []byte) (*NewReply, error) { - var r NewReply - err := json.Unmarshal(payload, &r) + var nr NewReply + err := json.Unmarshal(payload, &nr) if err != nil { return nil, err } - return &r, nil + return &nr, nil } // Edit edits an existing comment. // // Signature is the client signature of State+Token+ParentID+Comment. type Edit struct { + UUID string `json:"uuid"` // Unique user ID State StateT `json:"state"` // Record state Token string `json:"token"` // Record token ParentID uint32 `json:"parentid"` // Parent comment ID @@ -256,18 +264,18 @@ type EditReply struct { } // EncodeEdit encodes a EditReply into a JSON byte slice. -func EncodeEditReply(r EditReply) ([]byte, error) { - return json.Marshal(r) +func EncodeEditReply(er EditReply) ([]byte, error) { + return json.Marshal(er) } // DecodeEdit decodes a JSON byte slice into a EditReply. func DecodeEditReply(payload []byte) (*EditReply, error) { - var r EditReply - err := json.Unmarshal(payload, &r) + var er EditReply + err := json.Unmarshal(payload, &er) if err != nil { return nil, err } - return &r, nil + return &er, nil } // Del permanently deletes all versions of the provided comment. @@ -304,18 +312,18 @@ type DelReply struct { } // EncodeDelReply encodes a DelReply into a JSON byte slice. -func EncodeDelReply(d DelReply) ([]byte, error) { - return json.Marshal(d) +func EncodeDelReply(dr DelReply) ([]byte, error) { + return json.Marshal(dr) } // DecodeDelReply decodes a JSON byte slice into a DelReply. func DecodeDelReply(payload []byte) (*DelReply, error) { - var d DelReply - err := json.Unmarshal(payload, &d) + var dr DelReply + err := json.Unmarshal(payload, &dr) if err != nil { return nil, err } - return &d, nil + return &dr, nil } // Get returns the latest version of the comments for the provided comment IDs. @@ -351,18 +359,18 @@ type GetReply struct { } // EncodeGetReply encodes a GetReply into a JSON byte slice. -func EncodeGetReply(g GetReply) ([]byte, error) { - return json.Marshal(g) +func EncodeGetReply(gr GetReply) ([]byte, error) { + return json.Marshal(gr) } // DecodeGetReply decodes a JSON byte slice into a GetReply. func DecodeGetReply(payload []byte) (*GetReply, error) { - var g GetReply - err := json.Unmarshal(payload, &g) + var gr GetReply + err := json.Unmarshal(payload, &gr) if err != nil { return nil, err } - return &g, nil + return &gr, nil } // GetAll returns the latest version off all comments for the provided record. @@ -372,18 +380,18 @@ type GetAll struct { } // EncodeGetAll encodes a GetAll into a JSON byte slice. -func EncodeGetAll(g GetAll) ([]byte, error) { - return json.Marshal(g) +func EncodeGetAll(ga GetAll) ([]byte, error) { + return json.Marshal(ga) } // DecodeGetAll decodes a JSON byte slice into a GetAll. func DecodeGetAll(payload []byte) (*GetAll, error) { - var g GetAll - err := json.Unmarshal(payload, &g) + var ga GetAll + err := json.Unmarshal(payload, &ga) if err != nil { return nil, err } - return &g, nil + return &ga, nil } // GetAllReply is the reply to the GetAll command. @@ -392,18 +400,18 @@ type GetAllReply struct { } // EncodeGetAllReply encodes a GetAllReply into a JSON byte slice. -func EncodeGetAllReply(g GetAllReply) ([]byte, error) { - return json.Marshal(g) +func EncodeGetAllReply(gar GetAllReply) ([]byte, error) { + return json.Marshal(gar) } // DecodeGetAllReply decodes a JSON byte slice into a GetAllReply. func DecodeGetAllReply(payload []byte) (*GetAllReply, error) { - var g GetAllReply - err := json.Unmarshal(payload, &g) + var gar GetAllReply + err := json.Unmarshal(payload, &gar) if err != nil { return nil, err } - return &g, nil + return &gar, nil } // GetVersion returns a specific version of a comment. @@ -415,18 +423,18 @@ type GetVersion struct { } // EncodeGetVersion encodes a GetVersion into a JSON byte slice. -func EncodeGetVersion(g GetVersion) ([]byte, error) { - return json.Marshal(g) +func EncodeGetVersion(gv GetVersion) ([]byte, error) { + return json.Marshal(gv) } // DecodeGetVersion decodes a JSON byte slice into a GetVersion. func DecodeGetVersion(payload []byte) (*GetVersion, error) { - var g GetVersion - err := json.Unmarshal(payload, &g) + var gv GetVersion + err := json.Unmarshal(payload, &gv) if err != nil { return nil, err } - return &g, nil + return &gv, nil } // GetVersionReply is the reply to the GetVersion command. @@ -435,18 +443,18 @@ type GetVersionReply struct { } // EncodeGetVersionReply encodes a GetVersionReply into a JSON byte slice. -func EncodeGetVersionReply(g GetVersionReply) ([]byte, error) { - return json.Marshal(g) +func EncodeGetVersionReply(gvr GetVersionReply) ([]byte, error) { + return json.Marshal(gvr) } // DecodeGetVersionReply decodes a JSON byte slice into a GetVersionReply. func DecodeGetVersionReply(payload []byte) (*GetVersionReply, error) { - var g GetVersionReply - err := json.Unmarshal(payload, &g) + var gvr GetVersionReply + err := json.Unmarshal(payload, &gvr) if err != nil { return nil, err } - return &g, nil + return &gvr, nil } // Count returns the comments count for the provided record. @@ -476,32 +484,32 @@ type CountReply struct { } // EncodeCountReply encodes a CountReply into a JSON byte slice. -func EncodeCountReply(c CountReply) ([]byte, error) { - return json.Marshal(c) +func EncodeCountReply(cr CountReply) ([]byte, error) { + return json.Marshal(cr) } // DecodeCountReply decodes a JSON byte slice into a CountReply. func DecodeCountReply(payload []byte) (*CountReply, error) { - var c CountReply - err := json.Unmarshal(payload, &c) + var cr CountReply + err := json.Unmarshal(payload, &cr) if err != nil { return nil, err } - return &c, nil + return &cr, nil } // Vote casts a comment vote (upvote or downvote). // -// The uuid is required because the effect of a new vote on a comment score -// depends on the previous vote from that uuid. Example, a user upvotes a -// comment that they have already upvoted, the resulting vote score is 0 due to -// the second upvote removing the original upvote. The public key cannot be -// relied on to remain the same for each user so a uuid must be included. +// The effect of a new vote on a comment score depends on the previous vote +// from that uuid. Example, a user upvotes a comment that they have already +// upvoted, the resulting vote score is 0 due to the second upvote removing the +// original upvote. The public key cannot be relied on to remain the same for +// each user so a uuid must be included. // // Signature is the client signature of the State+Token+CommentID+Vote. type Vote struct { - State StateT `json:"state"` // Record state UUID string `json:"uuid"` // Unique user ID + State StateT `json:"state"` // Record state Token string `json:"token"` // Record token CommentID uint32 `json:"commentid"` // Comment ID Vote VoteT `json:"vote"` // Upvote or downvote @@ -532,16 +540,16 @@ type VoteReply struct { } // EncodeVoteReply encodes a VoteReply into a JSON byte slice. -func EncodeVoteReply(v VoteReply) ([]byte, error) { - return json.Marshal(v) +func EncodeVoteReply(vr VoteReply) ([]byte, error) { + return json.Marshal(vr) } // DecodeVoteReply decodes a JSON byte slice into a VoteReply. func DecodeVoteReply(payload []byte) (*VoteReply, error) { - var v VoteReply - err := json.Unmarshal(payload, &v) + var vr VoteReply + err := json.Unmarshal(payload, &vr) if err != nil { return nil, err } - return &v, nil + return &vr, nil } diff --git a/plugins/pi/pi.go b/plugins/pi/pi.go index 44b809dd5..57de17eb6 100644 --- a/plugins/pi/pi.go +++ b/plugins/pi/pi.go @@ -16,12 +16,19 @@ import ( type PropStateT int type PropStatusT int type ErrorStatusT int +type VoteT int const ( ID = "pi" - // Plugin commands + // Plugin commands. Many of these plugin commands rely on the + // commands from other plugins, but perform additional validation + // that is specific to pi or add additional functionality on top of + // the existing plugin commands that is specific to pi. CmdProposals = "proposals" // Get plugin data for proposals + CmdCommentNew = "commentnew" // Create a new comment + CmdCommentCensor = "commentcensor" // Censor a comment + CmdCommentVote = "commentvote" // Upvote/downvote a comment CmdVoteInventory = "voteinventory" // Get inventory by vote status // Metadata stream IDs. All metadata streams in this plugin will @@ -48,6 +55,11 @@ const ( PropStatusCensored PropStatusT = 3 // Prop has been censored PropStatusAbandoned PropStatusT = 4 // Prop has been abandoned + // Comment vote types + VoteInvalid VoteT = 0 + VoteDownvote VoteT = -1 + VoteUpvote VoteT = 1 + // User error status codes // TODO number error codes ErrorStatusInvalid ErrorStatusT = iota @@ -251,10 +263,174 @@ func DecodeProposalsReply(payload []byte) (*ProposalsReply, error) { return &pr, nil } +// CommentNew creates a new comment. This command relies on the comments plugin +// New command, but also performs additional vote status validation that is +// specific to pi. +// +// The parent ID is used to reply to an existing comment. A parent ID of 0 +// indicates that the comment is a base level comment and not a reply commment. +// +// Signature is the client signature of State+Token+ParentID+Comment. +type CommentNew struct { + UUID string `json:"uuid"` // Unique user ID + State PropStateT `json:"state"` // Record state + Token string `json:"token"` // Record token + ParentID uint32 `json:"parentid"` // Parent comment ID + Comment string `json:"comment"` // Comment text + PublicKey string `json:"publickey"` // Pubkey used for Signature + Signature string `json:"signature"` // Client signature +} + +// EncodeCommentNew encodes a CommentNew into a JSON byte slice. +func EncodeCommentNew(cn CommentNew) ([]byte, error) { + return json.Marshal(cn) +} + +// DecodeCommentNew decodes a JSON byte slice into a CommentNew. +func DecodeCommentNew(payload []byte) (*CommentNew, error) { + var cn CommentNew + err := json.Unmarshal(payload, &cn) + if err != nil { + return nil, err + } + return &cn, nil +} + +// CommentNewReply is the reply to the CommentNew command. +type CommentNewReply struct { + CommentID uint32 `json:"commentid"` // Comment ID + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + Receipt string `json:"receipt"` // Server sig of client sig +} + +// EncodeCommentNew encodes a CommentNewReply into a JSON byte slice. +func EncodeCommentNewReply(cnr CommentNewReply) ([]byte, error) { + return json.Marshal(cnr) +} + +// DecodeCommentNew decodes a JSON byte slice into a CommentNewReply. +func DecodeCommentNewReply(payload []byte) (*CommentNewReply, error) { + var cnr CommentNewReply + err := json.Unmarshal(payload, &cnr) + if err != nil { + return nil, err + } + return &cnr, nil +} + +// CommentCensor permanently deletes the provided comment. This command relies +// on the comments plugin Del command, but also performs additional vote status +// validation that is specific to pi. +// +// Signature is the client signature of the State+Token+CommentID+Reason +type CommentCensor struct { + State PropStateT `json:"state"` // Record state + Token string `json:"token"` // Record token + CommentID uint32 `json:"commentid"` // Comment ID + Reason string `json:"reason"` // Reason for deletion + PublicKey string `json:"publickey"` // Public key used for signature + Signature string `json:"signature"` // Client signature +} + +// EncodeCommentCensor encodes a CommentCensor into a JSON byte slice. +func EncodeCommentCensor(cc CommentCensor) ([]byte, error) { + return json.Marshal(cc) +} + +// DecodeCommentCensor decodes a JSON byte slice into a CommentCensor. +func DecodeCommentCensor(payload []byte) (*CommentCensor, error) { + var cc CommentCensor + err := json.Unmarshal(payload, &cc) + if err != nil { + return nil, err + } + return &cc, nil +} + +// CommentCensorReply is the reply to the CommentCensor command. +type CommentCensorReply struct { + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + Receipt string `json:"receipt"` // Server signature of client signature +} + +// EncodeCommentCensorReply encodes a CommentCensorReply into a JSON byte +// slice. +func EncodeCommentCensorReply(ccr CommentCensorReply) ([]byte, error) { + return json.Marshal(ccr) +} + +// DecodeCommentCensorReply decodes a JSON byte slice into CommentCensorReply. +func DecodeCommentCensorReply(payload []byte) (*CommentCensorReply, error) { + var d CommentCensorReply + err := json.Unmarshal(payload, &d) + if err != nil { + return nil, err + } + return &d, nil +} + +// CommentVote casts a comment vote (upvote or downvote). This command relies +// on the comments plugin Del command, but also performs additional vote status +// validation that is specific to pi. +// +// The effect of a new vote on a comment score depends on the previous vote +// from that uuid. Example, a user upvotes a comment that they have already +// upvoted, the resulting vote score is 0 due to the second upvote removing the +// original upvote. The public key cannot be relied on to remain the same for +// each user so a uuid must be included. +// +// Signature is the client signature of the State+Token+CommentID+Vote. +type CommentVote struct { + State PropStateT `json:"state"` // Record state + UUID string `json:"uuid"` // Unique user ID + Token string `json:"token"` // Record token + CommentID uint32 `json:"commentid"` // Comment ID + Vote VoteT `json:"vote"` // Upvote or downvote + PublicKey string `json:"publickey"` // Public key used for signature + Signature string `json:"signature"` // Client signature +} + +// EncodeCommentVote encodes a CommentVote into a JSON byte slice. +func EncodeCommentVote(cv CommentVote) ([]byte, error) { + return json.Marshal(cv) +} + +// DecodeCommentVote decodes a JSON byte slice into a CommentVote. +func DecodeCommentVote(payload []byte) (*CommentVote, error) { + var cv CommentVote + err := json.Unmarshal(payload, &cv) + if err != nil { + return nil, err + } + return &cv, nil +} + +// CommentVoteReply is the reply to the CommentVote command. +type CommentVoteReply struct { + Score int64 `json:"score"` // Overall comment vote score + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + Receipt string `json:"receipt"` // Server signature of client signature +} + +// EncodeCommentVoteReply encodes a CommentVoteReply into a JSON byte slice. +func EncodeCommentVoteReply(cvr CommentVoteReply) ([]byte, error) { + return json.Marshal(cvr) +} + +// DecodeCommentVoteReply decodes a JSON byte slice into a CommentVoteReply. +func DecodeCommentVoteReply(payload []byte) (*CommentVoteReply, error) { + var cvr CommentVoteReply + err := json.Unmarshal(payload, &cvr) + if err != nil { + return nil, err + } + return &cvr, nil +} + // VoteInventory requests the tokens of all proposals in the inventory -// catagorized by their vote status. The difference between this call and the -// ticketvote Inventory call is that this call breaks the Finished vote status -// out into Approved and Rejected catagories, which is specific to pi. +// catagorized by their vote status. This call relies on the ticketvote +// Inventory call, but breaks the Finished vote status out into Approved and +// Rejected catagories. This functionality is specific to pi. type VoteInventory struct{} // EncodeVoteInventory encodes a VoteInventory into a JSON byte slice. diff --git a/plugins/ticketvote/ticketvote.go b/plugins/ticketvote/ticketvote.go index a7edb9652..a0b9eec9e 100644 --- a/plugins/ticketvote/ticketvote.go +++ b/plugins/ticketvote/ticketvote.go @@ -11,9 +11,9 @@ import ( "fmt" ) -type ActionT string -type VoteT int type VoteStatusT int +type AuthActionT string +type VoteT int type VoteErrorT int type ErrorStatusT int @@ -31,10 +31,6 @@ const ( CmdInventory = "inventory" // Get inventory grouped by vote status CmdProofs = "proofs" // Get inclusion proofs - // Authorize vote actions - ActionAuthorize ActionT = "authorize" - ActionRevoke ActionT = "revoke" - // Vote statuses VoteStatusInvalid VoteStatusT = 0 // Invalid status VoteStatusUnauthorized VoteStatusT = 1 // Vote cannot be started @@ -42,6 +38,10 @@ const ( VoteStatusStarted VoteStatusT = 3 // Vote has been started VoteStatusFinished VoteStatusT = 4 // Vote has finished + // Authorize vote actions + ActionAuthorize AuthActionT = "authorize" + ActionRevoke AuthActionT = "revoke" + // Vote types VoteTypeInvalid VoteT = 0 @@ -199,12 +199,14 @@ type CastVote struct { } // Authorize authorizes a ticket vote or revokes a previous authorization. +// +// Signature contains the client signature of the Token+Version+Action. type Authorize struct { - Token string `json:"token"` // Record token - Version uint32 `json:"version"` // Record version - Action ActionT `json:"action"` // Authorize or revoke - PublicKey string `json:"publickey"` // Public key used for signature - Signature string `json:"signature"` // Signature of token+version+action + Token string `json:"token"` // Record token + Version uint32 `json:"version"` // Record version + Action AuthActionT `json:"action"` // Authorize or revoke + PublicKey string `json:"publickey"` // Public key used for signature + Signature string `json:"signature"` // Client signature } // EncodeAuthorize encodes an Authorize into a JSON byte slice. diff --git a/politeiad/backend/tlogbe/plugins/comments.go b/politeiad/backend/tlogbe/plugins/comments.go index bab76d1b9..5a3245e13 100644 --- a/politeiad/backend/tlogbe/plugins/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments.go @@ -28,6 +28,7 @@ import ( // the key to the file system. This will allow the data to be backed up. // TODO comment signature messages need to have state added to them. +// TODO uuid has been added to New, Edit, and CommentAdd. const ( // Blob entry data descriptors diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 926838c6e..08bf2cbe6 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -11,7 +11,9 @@ import ( type ErrorStatusT int type PropStateT int type PropStatusT int +type CommentVoteT int type VoteStatusT int +type VoteAuthActionT string type VoteT int const ( @@ -55,6 +57,11 @@ const ( PropStatusCensored PropStatusT = 3 // Prop has been censored PropStatusAbandoned PropStatusT = 4 // Prop has been abandoned + // Comment vote types + CommentVoteInvalid CommentVoteT = 0 + CommentVoteDownvote CommentVoteT = -1 + CommentVoteUpvote CommentVoteT = 1 + // Vote statuses VoteStatusInvalid VoteStatusT = 0 // Invalid status VoteStatusUnauthorized VoteStatusT = 1 // Vote cannot be started @@ -62,6 +69,10 @@ const ( VoteStatusStarted VoteStatusT = 3 // Vote has been started VoteStatusFinished VoteStatusT = 4 // Vote has finished + // Vote authorization actions + VoteAuthActionAuthorize VoteAuthActionT = "authorize" + VoteAuthActionRevoke VoteAuthActionT = "revoke" + // Vote types VoteTypeInvalid VoteT = 0 @@ -362,3 +373,181 @@ type ProposalInventoryReply struct { Censored []string `json:"censored"` Abandoned []string `json:"abandoned"` } + +// Comment represent a proposal comment. +// +// The parent ID is used to reply to an existing comment. A parent ID of 0 +// indicates that the comment is a base level comment and not a reply commment. +// +// Signature is the client signature of State+Token+ParentID+Comment. +type Comment struct { + UserID string `json:"userid"` // User ID + Username string `json:"username"` // Username + State PropStateT `json:"state"` // Proposal state + Token string `json:"token"` // Proposal token + ParentID uint32 `json:"parentid"` // Parent comment ID + Comment string `json:"comment"` // Comment text + PublicKey string `json:"publickey"` // Public key used for Signature + Signature string `json:"signature"` // Client signature + CommentID uint32 `json:"commentid"` // Comment ID + Version uint32 `json:"version"` // Comment version + Timestamp int64 `json:"timestamp"` // UNIX timestamp of last edit + Receipt string `json:"receipt"` // Server sig of client sig + Score int64 `json:"score"` // Vote score + Deleted bool `json:"deleted"` // Comment has been deleted + Reason string `json:"reason"` // Reason for deletion +} + +// CommentNew creates a new comment. +// +// The parent ID is used to reply to an existing comment. A parent ID of 0 +// indicates that the comment is a base level comment and not a reply commment. +// +// Signature is the client signature of State+Token+ParentID+Comment. +type CommentNew struct { + State PropStateT `json:"state"` + Token string `json:"token"` + ParentID uint32 `json:"parentid"` + Comment string `json:"comment"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` +} + +// CommentNewReply is the reply to the CommentNew command. +// +// Receipt is the server signature of the client signature. This is proof that +// the server received and processed the CommentNew command. +type CommentNewReply struct { + CommentID uint32 `json:"commentid"` + Timestamp int64 `json:"timestamp"` + Receipt string `json:"receipt"` +} + +// CommentCensor permanently censors a comment. The comment will be deleted +// and cannot be retrieved once censored. Only admins can censor a comment. +// +// Reason contains the reason why the comment is being censored and must always +// be included. +type CommentCensor struct { + State PropStateT `json:"state"` + Token string `json:"token"` + CommentID uint32 `json:"commentid"` + Reason string `json:"reason"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` +} + +// CommentCensorReply is the reply to the CommentCensor command. +// +// Receipt is the server signature of the client signature. This is proof that +// the server received and processed the CommentCensor command. +type CommentCensorReply struct { + Timestamp int64 `json:"timestamp"` + Receipt string `json:"receipt"` +} + +// CommentVote casts a comment vote (upvote or downvote). +// +// The effect of a new vote on a comment score depends on the previous vote +// from that uuid. Example, a user upvotes a comment that they have already +// upvoted, the resulting vote score is 0 due to the second upvote removing the +// original upvote. +// +// Signature is the client signature of the State+Token+CommentID+Vote. +type CommentVote struct { + State PropStateT `json:"state"` + Token string `json:"token"` + CommentID uint32 `json:"commentid"` + Vote CommentVoteT `json:"vote"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` +} + +// CommentVoteReply is the reply to the CommentVote command. +// +// Receipt is the server signature of the client signature. This is proof that +// the server received and processed the CommentVote command. +type CommentVoteReply struct { + Score int64 `json:"score"` // Overall comment vote score + Timestamp int64 `json:"timestamp"` + Receipt string `json:"receipt"` +} + +// Comments returns all comments for a proposal. +type Comments struct { + State PropStateT `json:"state"` + Token string `json:"token"` +} + +// CommentsReply is the reply to the comments command. +type CommentsReply struct { + Comments []Comment `json:"comments"` +} + +// UserCommentVote represents a comment vote made by a user. This struct +// contains all the information in a CommentVote and a CommentVoteReply. +type UserCommentVote struct { + State PropStateT `json:"state"` + Token string `json:"token"` + CommentID uint32 `json:"commentid"` + Vote CommentVoteT `json:"vote"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` + Timestamp int64 `json:"timestamp"` + Receipt string `json:"receipt"` +} + +// CommentVotes returns all comment votes made a specific user on a proposal. +type CommentVotes struct { + State PropStateT `json:"state"` + Token string `json:"token"` + UserID string `json:"userid"` +} + +// CommentVotesReply is the reply to the CommentVotes command. +type CommentVotesReply struct { + Votes []UserCommentVote `json:"votes"` +} + +// VoteAuthorize authorizes a proposal vote or revokes a previous vote +// authorization. All proposal votes must be authorized by the proposal author +// before an admin is able to start the voting process. +// +// Signature contains the client signature of the Token+Version+Action. +type VoteAuthorize struct { + Token string `json:"token"` + Version uint32 `json:"version"` + Action VoteAuthActionT `json:"action"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` +} + +// VoteAuthorizeReply is the reply to the VoteAuthorize command. +// +// Receipt is the server signature of the client signature. This is proof that +// the server received and processed the VoteAuthorize command. +type VoteAuthorizeReply struct { + Timestamp int64 `json:"timestamp"` + Receipt string `json:"receipt"` +} + +type VoteStart struct{} +type VoteStartReply struct{} + +type VoteStartRunoff struct{} +type VoteStartRunoffReply struct{} + +type VoteBallot struct{} +type VoteBallotReply struct{} + +type Votes struct{} +type VotesReply struct{} + +type VoteResults struct{} +type VoteResultsReply struct{} + +type VoteSummaries struct{} +type VoteSummariesReply struct{} + +type VoteInventory struct{} +type VoteInventoryReply struct{} diff --git a/politeiawww/api/www/v1/v1.go b/politeiawww/api/www/v1/v1.go index 69546a3fd..64b75962f 100644 --- a/politeiawww/api/www/v1/v1.go +++ b/politeiawww/api/www/v1/v1.go @@ -56,7 +56,7 @@ const ( RouteProposalPaywallDetails = "/proposals/paywall" RouteProposalPaywallPayment = "/proposals/paywallpayment" - // The following routes are to be DEPRECATED in the near future and + // The following routes WILL BE DEPRECATED in the near future and // should not be used. The pi v1 API should be used instead. RouteTokenInventory = "/proposals/tokeninventory" RouteBatchProposals = "/proposals/batch" @@ -65,7 +65,7 @@ const ( RouteVoteResults = "/proposals/{token:[A-Fa-f0-9]{7,64}}/votes" RouteCastVotes = "/proposals/castvotes" - // The following routes have been DEPRECATED. The pi v1 API should + // The following route HAVE BEEN DEPRECATED. The pi v1 API should // be used instead. RouteActiveVote = "/proposals/activevote" RouteAllVetted = "/proposals/vetted" From ed4481ba8fb7609ea22ede2026322d7fbec7fdc6 Mon Sep 17 00:00:00 2001 From: Amir Massarwa Date: Sun, 20 Sep 2020 17:34:57 +0300 Subject: [PATCH 072/449] politeiawww: Add processCommentNew. --- politeiawww/cmd/cmswww/cmswww.go | 2 +- politeiawww/cmd/cmswww/help.go | 2 +- .../cmd/{shared => cmswww}/newcomment.go | 24 +++--- politeiawww/cmd/piwww/commentnew.go | 75 +++++++++++++++++++ politeiawww/cmd/piwww/help.go | 4 +- politeiawww/cmd/piwww/piwww.go | 4 +- politeiawww/cmd/shared/client.go | 30 +++++++- politeiawww/cmd/shared/shared.go | 2 +- politeiawww/pi.go | 30 ++++++++ politeiawww/piwww.go | 72 ++++++++++++++++++ politeiawww/ticketvote.go | 3 + 11 files changed, 226 insertions(+), 22 deletions(-) rename politeiawww/cmd/{shared => cmswww}/newcomment.go (85%) create mode 100644 politeiawww/cmd/piwww/commentnew.go create mode 100644 politeiawww/pi.go diff --git a/politeiawww/cmd/cmswww/cmswww.go b/politeiawww/cmd/cmswww/cmswww.go index 09f517fe4..938eb6e71 100644 --- a/politeiawww/cmd/cmswww/cmswww.go +++ b/politeiawww/cmd/cmswww/cmswww.go @@ -59,6 +59,7 @@ type cmswww struct { // Commands ActiveVotes ActiveVotesCmd `command:"activevotes" description:"(user) get the dccs that are being voted on"` BatchProposals shared.BatchProposalsCmd `command:"batchproposals" description:"(user) retrieve a set of proposals"` + NewComment NewCommentCmd `command:"newcomment" description:"(user) create a new comment"` CensorComment shared.CensorCommentCmd `command:"censorcomment" description:"(admin) censor a comment"` ChangePassword shared.ChangePasswordCmd `command:"changepassword" description:"(user) change the password for the logged in user"` ChangeUsername shared.ChangeUsernameCmd `command:"changeusername" description:"(user) change the username for the logged in user"` @@ -81,7 +82,6 @@ type cmswww struct { CMSManageUser CMSManageUserCmd `command:"cmsmanageuser" description:"(admin) edit certain properties of the specified user"` ManageUser shared.ManageUserCmd `command:"manageuser" description:"(admin) edit certain properties of the specified user"` Me shared.MeCmd `command:"me" description:"(user) get user details for the logged in user"` - NewComment shared.NewCommentCmd `command:"newcomment" description:"(user) create a new comment"` NewDCC NewDCCCmd `command:"newdcc" description:"(user) creates a new dcc proposal"` NewDCCComment NewDCCCommentCmd `command:"newdcccomment" description:"(user) creates a new comment on a dcc proposal"` NewInvoice NewInvoiceCmd `command:"newinvoice" description:"(user) create a new invoice"` diff --git a/politeiawww/cmd/cmswww/help.go b/politeiawww/cmd/cmswww/help.go index 9b6e11546..78aee23f2 100644 --- a/politeiawww/cmd/cmswww/help.go +++ b/politeiawww/cmd/cmswww/help.go @@ -36,7 +36,7 @@ func (cmd *HelpCmd) Execute(args []string) error { case "changeusername": fmt.Printf("%s\n", shared.ChangeUsernameHelpMsg) case "newcomment": - fmt.Printf("%s\n", shared.NewCommentHelpMsg) + fmt.Printf("%s\n", newCommentHelpMsg) case "censorcomment": fmt.Printf("%s\n", shared.CensorCommentHelpMsg) case "manageuser": diff --git a/politeiawww/cmd/shared/newcomment.go b/politeiawww/cmd/cmswww/newcomment.go similarity index 85% rename from politeiawww/cmd/shared/newcomment.go rename to politeiawww/cmd/cmswww/newcomment.go index 5c9b2d477..9cad104ed 100644 --- a/politeiawww/cmd/shared/newcomment.go +++ b/politeiawww/cmd/cmswww/newcomment.go @@ -1,12 +1,14 @@ -// Copyright (c) 2017-2019 The Decred developers +// Copyright (c) 2017-2020 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package shared +package main import ( "encoding/hex" - "github.com/decred/politeia/politeiawww/api/www/v1" + + v1 "github.com/decred/politeia/politeiawww/api/www/v1" + "github.com/decred/politeia/politeiawww/cmd/shared" ) // NewCommentCmd submits a new proposal comment. @@ -26,7 +28,7 @@ func (cmd *NewCommentCmd) Execute(args []string) error { // Check for user identity if cfg.Identity == nil { - return ErrUserIdentityNotFound + return shared.ErrUserIdentityNotFound } // Setup new comment request @@ -40,32 +42,29 @@ func (cmd *NewCommentCmd) Execute(args []string) error { } // Print request details - err := PrintJSON(nc) + err := shared.PrintJSON(nc) if err != nil { return err } // Send request - ncr, err := client.NewComment(nc) + ncr, err := client.WWWNewComment(nc) if err != nil { return err } // Print response details - return PrintJSON(ncr) + return shared.PrintJSON(ncr) } -// NewCommentHelpMsg is the output of the help command when 'newcomment' is +// newCommentHelpMsg is the output of the help command when 'newcomment' is // specified. -const NewCommentHelpMsg = `newcomment "token" "comment" - +const newCommentHelpMsg = `newcomment "token" "comment" Comment on proposal as logged in user. - Arguments: 1. token (string, required) Proposal censorship token 2. comment (string, required) Comment 3. parentID (string, required if replying to comment) Id of commment - Request: { "token": (string) Censorship token @@ -74,7 +73,6 @@ Request: "signature": (string) Signature of comment (token+parentID+comment) "publickey": (string) Public key of user commenting } - Response: { "comment": { diff --git a/politeiawww/cmd/piwww/commentnew.go b/politeiawww/cmd/piwww/commentnew.go new file mode 100644 index 000000000..d395c23d6 --- /dev/null +++ b/politeiawww/cmd/piwww/commentnew.go @@ -0,0 +1,75 @@ +// Copyright (c) 2017-2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "encoding/hex" + "strconv" + + pi "github.com/decred/politeia/politeiawww/api/pi/v1" + "github.com/decred/politeia/politeiawww/cmd/shared" +) + +// CommentNewCmd submits a new proposal comment. +type CommentNewCmd struct { + Args struct { + Token string `positional-arg-name:"token" required:"true"` // Censorship token + Comment string `positional-arg-name:"comment" required:"true"` // Comment text + ParentID string `positional-arg-name:"parentID"` // Comment parent ID + } `positional-args:"true"` +} + +// Execute executes the new comment command. +func (cmd *CommentNewCmd) Execute(args []string) error { + token := cmd.Args.Token + comment := cmd.Args.Comment + parentID := cmd.Args.ParentID + + // Check for user identity + if cfg.Identity == nil { + return shared.ErrUserIdentityNotFound + } + + // Setup new comment request + sig := cfg.Identity.SignMessage([]byte(token + parentID + comment)) + // Parse provided parent id + piUint, err := strconv.ParseUint(parentID, 10, 32) + if err != nil { + return err + } + cn := pi.CommentNew{ + Token: token, + ParentID: uint32(piUint), + Comment: comment, + Signature: hex.EncodeToString(sig[:]), + PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), + } + + // Print request details + err = shared.PrintJSON(cn) + if err != nil { + return err + } + + // Send request + ncr, err := client.CommentNew(cn) + if err != nil { + return err + } + + // Print response details + return shared.PrintJSON(ncr) +} + +// commentNewHelpMsg is the output of the help command when 'newcomment' is +// specified. +const commentNewHelpMsg = `commentnew "token" "comment" + +Comment on proposal as logged in user. + +Arguments: +1. token (string, required) Proposal censorship token +2. comment (string, required) Comment +3. parentID (string, required if replying to comment) Id of commment` diff --git a/politeiawww/cmd/piwww/help.go b/politeiawww/cmd/piwww/help.go index ea7097d00..2c73fc628 100644 --- a/politeiawww/cmd/piwww/help.go +++ b/politeiawww/cmd/piwww/help.go @@ -87,8 +87,8 @@ func (cmd *HelpCmd) Execute(args []string) error { fmt.Printf("%s\n", shared.BatchProposalsHelpMsg) // Comment commands - case "newcomment": - fmt.Printf("%s\n", shared.NewCommentHelpMsg) + case "commentnew": + fmt.Printf("%s\n", commentNewHelpMsg) case "proposalcomments": fmt.Printf("%s\n", proposalCommentsHelpMsg) case "censorcomment": diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index d2dba7d4f..406c37d99 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -49,6 +49,9 @@ type piwww struct { ProposalEdit ProposalEditCmd `command:"proposaledit"` ProposalSetStatus ProposalSetStatusCmd `command:"proposalsetstatus"` + // Comments commands + CommentNew CommentNewCmd `command:"commentnew" description:"(user) create a new comment"` + // Commands ActiveVotes ActiveVotesCmd `command:"activevotes" description:"(public) get the proposals that are being voted on"` AuthorizeVote AuthorizeVoteCmd `command:"authorizevote" description:"(user) authorize a proposal vote (must be proposal author)"` @@ -65,7 +68,6 @@ type piwww struct { Logout shared.LogoutCmd `command:"logout" description:"(public) logout of Politeia"` ManageUser shared.ManageUserCmd `command:"manageuser" description:"(admin) edit certain properties of the specified user"` Me shared.MeCmd `command:"me" description:"(user) get user details for the logged in user"` - NewComment shared.NewCommentCmd `command:"newcomment" description:"(user) create a new comment"` NewUser NewUserCmd `command:"newuser" description:"(public) create a new user"` Policy PolicyCmd `command:"policy" description:"(public) get the server policy"` ProposalComments ProposalCommentsCmd `command:"proposalcomments" description:"(public) get the comments for a proposal"` diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index d2d7dc839..8d2c548db 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Decred developers +// Copyright (c) 2017-2020 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -1134,8 +1134,8 @@ func (c *Client) GetAllVetted(gav *www.GetAllVetted) (*www.GetAllVettedReply, er return &gavr, nil } -// NewComment submits a new proposal comment for the logged in user. -func (c *Client) NewComment(nc *www.NewComment) (*www.NewCommentReply, error) { +// WWWNewComment submits a new proposal comment for the logged in user. +func (c *Client) WWWNewComment(nc *www.NewComment) (*www.NewCommentReply, error) { responseBody, err := c.makeRequest(http.MethodPost, www.PoliteiaWWWAPIRoute, www.RouteNewComment, nc) if err != nil { @@ -1158,6 +1158,30 @@ func (c *Client) NewComment(nc *www.NewComment) (*www.NewCommentReply, error) { return &ncr, nil } +// CommentNew submits a new proposal comment for the logged in user. +func (c *Client) CommentNew(cn pi.CommentNew) (*pi.CommentNewReply, error) { + responseBody, err := c.makeRequest(http.MethodPost, + pi.APIRoute, pi.RouteCommentNew, cn) + if err != nil { + return nil, err + } + + var cnr pi.CommentNewReply + err = json.Unmarshal(responseBody, &cnr) + if err != nil { + return nil, fmt.Errorf("unmarshal CommentNewReply: %v", err) + } + + if c.cfg.Verbose { + err := prettyPrintJSON(cnr) + if err != nil { + return nil, err + } + } + + return &cnr, nil +} + // GetComments retrieves the comments for the specified proposal. func (c *Client) GetComments(token string) (*www.GetCommentsReply, error) { route := "/proposals/" + token + "/comments" diff --git a/politeiawww/cmd/shared/shared.go b/politeiawww/cmd/shared/shared.go index c51320cdd..51a58c78a 100644 --- a/politeiawww/cmd/shared/shared.go +++ b/politeiawww/cmd/shared/shared.go @@ -26,7 +26,7 @@ var ( cfg *Config client *Client - // errUserIdentityNotFound is emitted when a user identity is + // ErrUserIdentityNotFound is emitted when a user identity is // required but the config object does not contain one. ErrUserIdentityNotFound = errors.New("user identity not found; " + "you must either create a new user or use the updateuserkey " + diff --git a/politeiawww/pi.go b/politeiawww/pi.go new file mode 100644 index 000000000..eede5025e --- /dev/null +++ b/politeiawww/pi.go @@ -0,0 +1,30 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + piplugin "github.com/decred/politeia/plugins/pi" +) + +// piCommentNew calls the pi plugin to add new comment. +func (p *politeiawww) piCommentNew(ncp *piplugin.CommentNew) (*piplugin.CommentNewReply, error) { + // Prep new comment payload + payload, err := piplugin.EncodeCommentNew(*ncp) + if err != nil { + return nil, err + } + + r, err := p.pluginCommand(piplugin.ID, piplugin.CmdCommentNew, "", + string(payload)) + if err != nil { + return nil, err + } + cnr, err := piplugin.DecodeCommentNewReply([]byte(r)) + if err != nil { + return nil, err + } + + return cnr, nil +} diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 1f736110f..1e5ae6305 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -1412,6 +1412,75 @@ func (p *politeiawww) handleProposalInventory(w http.ResponseWriter, r *http.Req util.RespondWithJSON(w, http.StatusOK, ppi) } +func (p *politeiawww) processCommentNew(cn pi.CommentNew, usr *user.User) (*pi.CommentNewReply, error) { + log.Tracef("processCommentNew: %v", usr.Username) + + // Verify user has paid registration paywall + if !p.userHasPaid(*usr) { + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusUserRegistrationNotPaid, + } + } + + // Verify user signed using active identity + if usr.PublicKey() != cn.PublicKey { + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusPublicKeyInvalid, + ErrorContext: []string{"not user's active identity"}, + } + } + + // Call pi plugin to add new comment + reply, err := p.piCommentNew(&piplugin.CommentNew{ + UUID: usr.ID.String(), + Token: cn.Token, + ParentID: cn.ParentID, + Comment: cn.Comment, + PublicKey: cn.PublicKey, + Signature: cn.Signature, + State: convertPropStateFromPi(cn.State), + }) + if err != nil { + return nil, err + } + + return &pi.CommentNewReply{ + CommentID: reply.CommentID, + Timestamp: reply.Timestamp, + Receipt: reply.Receipt, + }, nil +} + +func (p *politeiawww) handleCommentNew(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleCommentNew") + + var cn pi.CommentNew + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&cn); err != nil { + respondWithPiError(w, r, "handleCommentNew: unmarshal", + pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusInvalidInput, + }) + return + } + + user, err := p.getSessionUser(w, r) + if err != nil { + respondWithPiError(w, r, + "handleCommentNew: getSessionUser: %v", err) + return + } + + cnr, err := p.processCommentNew(cn, user) + if err != nil { + respondWithPiError(w, r, + "handleCommentNew: processCommentNew: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, cnr) +} + func (p *politeiawww) setPiRoutes() { // Public routes p.addRoute(http.MethodGet, pi.APIRoute, @@ -1431,6 +1500,9 @@ func (p *politeiawww) setPiRoutes() { pi.RouteProposalSetStatus, p.handleProposalSetStatus, permissionLogin) + p.addRoute(http.MethodPost, pi.APIRoute, + pi.RouteCommentNew, p.handleCommentNew, permissionLogin) + // Admin routes } diff --git a/politeiawww/ticketvote.go b/politeiawww/ticketvote.go index 8c32d14e6..21a7ec4db 100644 --- a/politeiawww/ticketvote.go +++ b/politeiawww/ticketvote.go @@ -37,6 +37,9 @@ func (p *politeiawww) castVotes(token string) (*ticketvote.CastVotesReply, error Token: token, } payload, err := ticketvote.EncodeCastVotes(csp) + if err != nil { + return nil, err + } r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdCastVotes, "", string(payload)) From 529fbc7766fec61aff80e64a63b6b6377bac3a55 Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Sun, 20 Sep 2020 11:38:56 -0300 Subject: [PATCH 073/449] politeiawww: Refactor getBestBlock plugin cmd. --- decredplugin/decredplugin.go | 38 ++++++++++++++++++++++++++++++++ politeiawww/api/pi/v1/v1.go | 2 +- politeiawww/decred.go | 20 +++++++++++++++++ politeiawww/plugin.go | 42 ++---------------------------------- 4 files changed, 61 insertions(+), 41 deletions(-) diff --git a/decredplugin/decredplugin.go b/decredplugin/decredplugin.go index fd4ad4cc7..ba764e299 100644 --- a/decredplugin/decredplugin.go +++ b/decredplugin/decredplugin.go @@ -1389,3 +1389,41 @@ func DecodeLinkedFromReply(payload []byte) (*LinkedFromReply, error) { return &reply, nil } + +// BestBlock is a command to request the best block data. +type BestBlock struct{} + +// EncodeBestBlock encodes an BestBlock into a JSON byte slice. +func EncodeBestBlock(bb BestBlock) ([]byte, error) { + return json.Marshal(bb) +} + +// DecodeBestBlock decodes a JSON byte slice into a BestBlock. +func DecodeBestBlock(payload []byte) (*BestBlock, error) { + var bb BestBlock + err := json.Unmarshal(payload, &bb) + if err != nil { + return nil, err + } + return &bb, nil +} + +// BestBlockReply is the reply to the BestBlock command. +type BestBlockReply struct { + Height uint32 `json:"height"` +} + +// EncodeBestBlockReply encodes an BestBlockReply into a JSON byte slice. +func EncodeBestBlockReply(bbr BestBlockReply) ([]byte, error) { + return json.Marshal(bbr) +} + +// DecodeBestBlockReply decodes a JSON byte slice into a BestBlockReply. +func DecodeBestBlockReply(payload []byte) (*BestBlockReply, error) { + var bbr BestBlockReply + err := json.Unmarshal(payload, &bbr) + if err != nil { + return nil, err + } + return &bbr, nil +} diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 08bf2cbe6..3edd5fedc 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -368,7 +368,7 @@ type ProposalInventory struct{} // ProposalInventoryReply is the reply to the ProposalInventory command. type ProposalInventoryReply struct { - Unvetted []string `json:"unvetted"` + Unvetted []string `json:"unvetted,omitempty"` Public []string `json:"public"` Censored []string `json:"censored"` Abandoned []string `json:"abandoned"` diff --git a/politeiawww/decred.go b/politeiawww/decred.go index 87c333bfc..c2ae20461 100644 --- a/politeiawww/decred.go +++ b/politeiawww/decred.go @@ -63,3 +63,23 @@ func (p *politeiawww) decredGetComments(token string) ([]decredplugin.Comment, e return gcr.Comments, nil } + +func (p *politeiawww) decredBestBlock() (*decredplugin.BestBlockReply, error) { + // Setup plugin command + payload, err := decredplugin.EncodeBestBlock(decredplugin.BestBlock{}) + if err != nil { + return nil, err + } + + // Execute plugin command + reply, err := p.pluginCommand(decredplugin.ID, decredplugin.CmdBestBlock, + decredplugin.CmdBestBlock, string(payload)) + + // Receive plugin command reply + bbr, err := decredplugin.DecodeBestBlockReply([]byte(reply)) + if err != nil { + return nil, err + } + + return bbr, nil +} diff --git a/politeiawww/plugin.go b/politeiawww/plugin.go index 3669cce76..c0a55d26e 100644 --- a/politeiawww/plugin.go +++ b/politeiawww/plugin.go @@ -5,16 +5,10 @@ package main import ( - "encoding/hex" - "encoding/json" "fmt" - "net/http" - "strconv" "time" - "github.com/decred/politeia/decredplugin" pd "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/util" ) // pluginSetting is a structure that holds key/value pairs of a plugin setting. @@ -52,44 +46,12 @@ func convertPluginFromPD(p pd.Plugin) plugin { // getBestBlock fetches and returns the best block from politeiad using the // decred plugin bestblock command. func (p *politeiawww) getBestBlock() (uint64, error) { - challenge, err := util.Random(pd.ChallengeSize) + bb, err := p.decredBestBlock() if err != nil { return 0, err } - pc := pd.PluginCommand{ - Challenge: hex.EncodeToString(challenge), - ID: decredplugin.ID, - Command: decredplugin.CmdBestBlock, - CommandID: decredplugin.CmdBestBlock, - Payload: "", - } - - responseBody, err := p.makeRequest(http.MethodPost, - pd.PluginCommandRoute, pc) - if err != nil { - return 0, err - } - - var reply pd.PluginCommandReply - err = json.Unmarshal(responseBody, &reply) - if err != nil { - return 0, fmt.Errorf("Could not unmarshal "+ - "PluginCommandReply: %v", err) - } - - // Verify the challenge. - err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) - if err != nil { - return 0, err - } - - bestBlock, err := strconv.ParseUint(reply.Payload, 10, 64) - if err != nil { - return 0, err - } - - return bestBlock, nil + return uint64(bb.Height), nil } // getPluginInventory returns the politeiad plugin inventory. If a politeiad From ced05b66ee92684a127bdb5d165ad0a678d0b3ee Mon Sep 17 00:00:00 2001 From: Amir Massarwa Date: Sun, 20 Sep 2020 18:58:57 +0300 Subject: [PATCH 074/449] politeiawww: Add processCommentVote. --- .../piwww/{likecomment.go => commentvote.go} | 59 +++++------- politeiawww/cmd/piwww/help.go | 4 +- politeiawww/cmd/piwww/piwww.go | 4 +- politeiawww/cmd/shared/client.go | 16 ++-- politeiawww/pi.go | 25 +++++- politeiawww/piwww.go | 89 ++++++++++++++++++- 6 files changed, 145 insertions(+), 52 deletions(-) rename politeiawww/cmd/piwww/{likecomment.go => commentvote.go} (53%) diff --git a/politeiawww/cmd/piwww/likecomment.go b/politeiawww/cmd/piwww/commentvote.go similarity index 53% rename from politeiawww/cmd/piwww/likecomment.go rename to politeiawww/cmd/piwww/commentvote.go index 028f58f83..d8f5baf36 100644 --- a/politeiawww/cmd/piwww/likecomment.go +++ b/politeiawww/cmd/piwww/commentvote.go @@ -7,14 +7,15 @@ package main import ( "encoding/hex" "fmt" + "strconv" - v1 "github.com/decred/politeia/politeiawww/api/www/v1" + pi "github.com/decred/politeia/politeiawww/api/pi/v1" "github.com/decred/politeia/politeiawww/cmd/shared" ) -// LikeCommentCmd is used to upvote/downvote a proposal comment using the +// CommentVoteCmd is used to upvote/downvote a proposal comment using the // logged in the user. -type LikeCommentCmd struct { +type CommentVoteCmd struct { Args struct { Token string `positional-arg-name:"token"` // Censorship token CommentID string `positional-arg-name:"commentID"` // Comment ID @@ -23,7 +24,7 @@ type LikeCommentCmd struct { } // Execute executes the like comment command. -func (cmd *LikeCommentCmd) Execute(args []string) error { +func (cmd *CommentVoteCmd) Execute(args []string) error { const actionUpvote = "upvote" const actionDownvote = "downvote" @@ -42,64 +43,52 @@ func (cmd *LikeCommentCmd) Execute(args []string) error { return shared.ErrUserIdentityNotFound } - // Setup like comment request - var actionCode string + // Setup pi comment vote request + var vote pi.CommentVoteT switch action { case actionUpvote: - actionCode = v1.VoteActionUp + vote = pi.CommentVoteUpvote case actionDownvote: - actionCode = v1.VoteActionDown + vote = pi.CommentVoteDownvote } - sig := cfg.Identity.SignMessage([]byte(token + commentID + actionCode)) - lc := &v1.LikeComment{ + sig := cfg.Identity.SignMessage([]byte(token + commentID + string(vote))) + // Parse provided parent id + ciUint, err := strconv.ParseUint(commentID, 10, 32) + if err != nil { + return err + } + cv := &pi.CommentVote{ Token: token, - CommentID: commentID, - Action: actionCode, + CommentID: uint32(ciUint), + Vote: vote, Signature: hex.EncodeToString(sig[:]), PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), } // Print request details - err := shared.PrintJSON(lc) + err = shared.PrintJSON(cv) if err != nil { return err } // Send request - lcr, err := client.LikeComment(lc) + cvr, err := client.CommentVote(cv) if err != nil { return err } // Print response details - return shared.PrintJSON(lcr) + return shared.PrintJSON(cvr) } -// likeCommentHelpMsg is the output for the help command when 'likecomment' is +// commentVoteHelpMsg is the output for the help command when 'commentvote' is // specified. -const likeCommentHelpMsg = `votecomment "token" "commentID" "action" +const commentVoteHelpMsg = `commentvote "token" "commentID" "action" Vote on a comment. Arguments: 1. token (string, required) Proposal censorship token 2. commentID (string, required) Id of the comment -3. action (string, required) Vote (upvote or downvote) - -Request: -{ - "token": (string) Censorship token - "commentid": (string) Id of comment - "action": (string) actionCode (upvote = '1', downvote = '-1') - "signature": (string) Signature of vote (token + commentID + actionCode) - "publickey": (string) Public key used for signature -} - -Response: -{ - "total": (uint64) Total number of up and down votes - "result": (int64) Current tally of likes (can be negative) - "receipt": (string) Server signature of vote signature - "error": (string) Error if something went wrong during liking a comment -}` +3. action (string, required) Vote (upvote or downvote)` diff --git a/politeiawww/cmd/piwww/help.go b/politeiawww/cmd/piwww/help.go index 2c73fc628..04b03b607 100644 --- a/politeiawww/cmd/piwww/help.go +++ b/politeiawww/cmd/piwww/help.go @@ -93,8 +93,8 @@ func (cmd *HelpCmd) Execute(args []string) error { fmt.Printf("%s\n", proposalCommentsHelpMsg) case "censorcomment": fmt.Printf("%s\n", shared.CensorCommentHelpMsg) - case "likecomment": - fmt.Printf("%s\n", likeCommentHelpMsg) + case "commentvote": + fmt.Printf("%s\n", commentVoteHelpMsg) case "userlikecomments": fmt.Printf("%s\n", userLikeCommentsHelpMsg) diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index 406c37d99..8bb3ab159 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -50,7 +50,8 @@ type piwww struct { ProposalSetStatus ProposalSetStatusCmd `command:"proposalsetstatus"` // Comments commands - CommentNew CommentNewCmd `command:"commentnew" description:"(user) create a new comment"` + CommentNew CommentNewCmd `command:"commentnew" description:"(user) create a new comment"` + CommentVote CommentVoteCmd `command:"commentvote" description:"(user) upvote/downvote a comment"` // Commands ActiveVotes ActiveVotesCmd `command:"activevotes" description:"(public) get the proposals that are being voted on"` @@ -63,7 +64,6 @@ type piwww struct { EditUser EditUserCmd `command:"edituser" description:"(user) edit the preferences of the logged in user"` Help HelpCmd `command:"help" description:" print a detailed help message for a specific command"` Inventory InventoryCmd `command:"inventory" description:"(public) get the proposals that are being voted on"` - LikeComment LikeCommentCmd `command:"likecomment" description:"(user) upvote/downvote a comment"` Login shared.LoginCmd `command:"login" description:"(public) login to Politeia"` Logout shared.LogoutCmd `command:"logout" description:"(public) logout of Politeia"` ManageUser shared.ManageUserCmd `command:"manageuser" description:"(admin) edit certain properties of the specified user"` diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index 8d2c548db..a827e1cbc 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -1258,29 +1258,29 @@ func (c *Client) UserCommentsLikes(token string) (*www.UserCommentsLikesReply, e return &uclr, nil } -// LikeComment casts a like comment action (upvote/downvote) for the logged in +// CommentVote casts a like comment action (upvote/downvote) for the logged in // user. -func (c *Client) LikeComment(lc *www.LikeComment) (*www.LikeCommentReply, error) { +func (c *Client) CommentVote(cv *pi.CommentVote) (*pi.CommentVoteReply, error) { responseBody, err := c.makeRequest(http.MethodPost, - www.PoliteiaWWWAPIRoute, www.RouteLikeComment, lc) + pi.APIRoute, pi.RouteCommentVote, cv) if err != nil { return nil, err } - var lcr www.LikeCommentReply - err = json.Unmarshal(responseBody, &lcr) + var cvr pi.CommentVoteReply + err = json.Unmarshal(responseBody, &cvr) if err != nil { - return nil, fmt.Errorf("unmarshal LikeCommentReply: %v", err) + return nil, fmt.Errorf("unmarshal CommentVoteReply: %v", err) } if c.cfg.Verbose { - err := prettyPrintJSON(lcr) + err := prettyPrintJSON(cvr) if err != nil { return nil, err } } - return &lcr, nil + return &cvr, nil } // CensorComment censors the specified proposal comment. diff --git a/politeiawww/pi.go b/politeiawww/pi.go index eede5025e..4ace7f15b 100644 --- a/politeiawww/pi.go +++ b/politeiawww/pi.go @@ -8,10 +8,31 @@ import ( piplugin "github.com/decred/politeia/plugins/pi" ) +// piCommentVote calls the pi plugin to vote on a comment. +func (p *politeiawww) piCommentVote(cvp *piplugin.CommentVote) (*piplugin.CommentVoteReply, error) { + // Prep comment vote payload + payload, err := piplugin.EncodeCommentVote(*cvp) + if err != nil { + return nil, err + } + + r, err := p.pluginCommand(piplugin.ID, piplugin.CmdCommentVote, "", + string(payload)) + if err != nil { + return nil, err + } + cvr, err := piplugin.DecodeCommentVoteReply([]byte(r)) + if err != nil { + return nil, err + } + + return cvr, nil +} + // piCommentNew calls the pi plugin to add new comment. -func (p *politeiawww) piCommentNew(ncp *piplugin.CommentNew) (*piplugin.CommentNewReply, error) { +func (p *politeiawww) piCommentNew(cnp *piplugin.CommentNew) (*piplugin.CommentNewReply, error) { // Prep new comment payload - payload, err := piplugin.EncodeCommentNew(*ncp) + payload, err := piplugin.EncodeCommentNew(*cnp) if err != nil { return nil, err } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 1e5ae6305..44dd46e79 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -1412,11 +1412,11 @@ func (p *politeiawww) handleProposalInventory(w http.ResponseWriter, r *http.Req util.RespondWithJSON(w, http.StatusOK, ppi) } -func (p *politeiawww) processCommentNew(cn pi.CommentNew, usr *user.User) (*pi.CommentNewReply, error) { +func (p *politeiawww) processCommentNew(cn pi.CommentNew, usr user.User) (*pi.CommentNewReply, error) { log.Tracef("processCommentNew: %v", usr.Username) // Verify user has paid registration paywall - if !p.userHasPaid(*usr) { + if !p.userHasPaid(usr) { return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusUserRegistrationNotPaid, } @@ -1471,7 +1471,7 @@ func (p *politeiawww) handleCommentNew(w http.ResponseWriter, r *http.Request) { return } - cnr, err := p.processCommentNew(cn, user) + cnr, err := p.processCommentNew(cn, *user) if err != nil { respondWithPiError(w, r, "handleCommentNew: processCommentNew: %v", err) @@ -1481,6 +1481,86 @@ func (p *politeiawww) handleCommentNew(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, cnr) } +func convertVoteFromPi(v pi.CommentVoteT) piplugin.VoteT { + switch v { + case pi.CommentVoteInvalid: + return piplugin.VoteInvalid + case pi.CommentVoteDownvote: + return piplugin.VoteDownvote + case pi.CommentVoteUpvote: + return piplugin.VoteUpvote + default: + return piplugin.VoteInvalid + } +} + +func (p *politeiawww) processCommentVote(cv pi.CommentVote, usr user.User) (*pi.CommentVoteReply, error) { + log.Tracef("processCommentVote: %v %v %v", cv.Token, cv.CommentID, cv.Vote) + + // Verify user has paid registration paywall + if !p.userHasPaid(usr) { + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusUserRegistrationNotPaid, + } + } + + // Verify user signed using active identity + if usr.PublicKey() != cv.PublicKey { + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusPublicKeyInvalid, + ErrorContext: []string{"not user's active identity"}, + } + } + + // Call pi plugin to add new comment + reply, err := p.piCommentVote(&piplugin.CommentVote{ + UUID: usr.ID.String(), + Token: cv.Token, + CommentID: cv.CommentID, + Vote: convertVoteFromPi(cv.Vote), + PublicKey: cv.PublicKey, + Signature: cv.Signature, + State: convertPropStateFromPi(cv.State), + }) + if err != nil { + return nil, err + } + + return &pi.CommentVoteReply{ + Score: reply.Score, + Timestamp: reply.Timestamp, + Receipt: reply.Receipt, + }, nil +} + +func (p *politeiawww) handleCommentVote(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleCommentVote") + + var cv pi.CommentVote + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&cv); err != nil { + respondWithPiError(w, r, "handleCommentVote: unmarshal", + pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusInvalidInput, + }) + return + } + + user, err := p.getSessionUser(w, r) + if err != nil { + respondWithPiError(w, r, + "handleCommentVote: getSessionUser: %v", err) + } + + vcr, err := p.processCommentVote(cv, *user) + if err != nil { + respondWithPiError(w, r, + "handleCommentVote: processCommentVote: %v", err) + } + + util.RespondWithJSON(w, http.StatusOK, vcr) +} + func (p *politeiawww) setPiRoutes() { // Public routes p.addRoute(http.MethodGet, pi.APIRoute, @@ -1503,6 +1583,9 @@ func (p *politeiawww) setPiRoutes() { p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteCommentNew, p.handleCommentNew, permissionLogin) + p.addRoute(http.MethodPost, pi.APIRoute, + pi.RouteCommentVote, p.handleCommentVote, permissionLogin) + // Admin routes } From 968f06e9d3157a3af777d8464702c8e93865dbbe Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 18 Sep 2020 13:41:17 -0500 Subject: [PATCH 075/449] move comments plugin into tlogbe --- go.sum | 1 + .../backend/tlogbe/{plugins => }/comments.go | 434 +++++++++--------- politeiad/backend/tlogbe/pluginclient.go | 59 +-- politeiad/backend/tlogbe/plugins/log.go | 25 - politeiad/backend/tlogbe/tlogbe.go | 20 +- politeiad/backend/tlogbe/tlogclient.go | 23 +- politeiad/log.go | 16 +- politeiawww/email.go | 1 - politeiawww/templates.go | 22 +- 9 files changed, 298 insertions(+), 303 deletions(-) rename politeiad/backend/tlogbe/{plugins => }/comments.go (87%) delete mode 100644 politeiad/backend/tlogbe/plugins/log.go diff --git a/go.sum b/go.sum index e88d6aa10..97e091920 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,7 @@ cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7h github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkByhMAwYaFAX9w2l7vxvBQ5NMoxDrkhqhtn4= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= diff --git a/politeiad/backend/tlogbe/plugins/comments.go b/politeiad/backend/tlogbe/comments.go similarity index 87% rename from politeiad/backend/tlogbe/plugins/comments.go rename to politeiad/backend/tlogbe/comments.go index 5a3245e13..00c4564e2 100644 --- a/politeiad/backend/tlogbe/plugins/comments.go +++ b/politeiad/backend/tlogbe/comments.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package plugins +package tlogbe import ( "bytes" @@ -11,7 +11,6 @@ import ( "encoding/json" "errors" "fmt" - "sort" "strconv" "sync" "time" @@ -19,7 +18,6 @@ import ( "github.com/decred/politeia/plugins/comments" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe" "github.com/decred/politeia/politeiad/backend/tlogbe/store" "github.com/decred/politeia/util" ) @@ -46,14 +44,15 @@ const ( ) var ( - _ tlogbe.Plugin = (*commentsPlugin)(nil) + _ pluginClient = (*commentsPlugin)(nil) ) -// commentsPlugin satisfies the Plugin interface. +// commentsPlugin satisfies the pluginClient interface. type commentsPlugin struct { sync.Mutex id *identity.FullIdentity - backend *tlogbe.TlogBackend + backend backend.Backend + tlog tlogClient // Mutexes contains a mutex for each record. The mutexes are lazy // loaded. @@ -79,11 +78,8 @@ type commentIndex struct { Votes map[string][]voteIndex `json:"votes"` // [uuid]votes } -// TODO this is not very efficient and probably needs to be improved. -// It duplicates a lot of data and requires fetching all the tree -// leaves to get. This may be problematic when there are 20,000 vote -// leaves and you just want to get the comments count for a record. -type index struct { +// TODO the comments index should be cached in the data dir +type commentsIndex struct { Comments map[uint32]commentIndex `json:"comments"` // [commentID]comment } @@ -102,7 +98,17 @@ func (p *commentsPlugin) mutex(token string) *sync.Mutex { return m } -func convertCommentsErrFromSignatureErr(err error) comments.UserErrorReply { +func tlogIDForCommentState(s comments.StateT) string { + switch s { + case comments.StateUnvetted: + return tlogIDUnvetted + case comments.StateVetted: + return tlogIDVetted + } + return "" +} + +func convertCommentsErrorFromSignatureError(err error) comments.UserErrorReply { var e util.SignatureError var s comments.ErrorStatusT if errors.As(err, &e) { @@ -170,8 +176,8 @@ func convertBlobEntryFromCommentVote(c comments.CommentVote) (*store.BlobEntry, return &be, nil } -func convertBlobEntryFromIndex(idx index) (*store.BlobEntry, error) { - data, err := json.Marshal(idx) +func convertBlobEntryFromCommentsIndex(ci commentsIndex) (*store.BlobEntry, error) { + data, err := json.Marshal(ci) if err != nil { return nil, err } @@ -263,7 +269,7 @@ func convertCommentDelFromBlobEntry(be store.BlobEntry) (*comments.CommentDel, e return &c, nil } -func convertIndexFromBlobEntry(be store.BlobEntry) (*index, error) { +func convertCommentsIndexFromBlobEntry(be store.BlobEntry) (*commentsIndex, error) { // Decode and validate data hint b, err := base64.StdEncoding.DecodeString(be.DataHint) if err != nil { @@ -292,13 +298,13 @@ func convertIndexFromBlobEntry(be store.BlobEntry) (*index, error) { return nil, fmt.Errorf("data is not coherent; got %x, want %x", util.Digest(b), hash) } - var idx index - err = json.Unmarshal(b, &idx) + var ci commentsIndex + err = json.Unmarshal(b, &ci) if err != nil { - return nil, fmt.Errorf("unmarshal index: %v", err) + return nil, fmt.Errorf("unmarshal commentsIndex: %v", err) } - return &idx, nil + return &ci, nil } func convertCommentFromCommentAdd(ca comments.CommentAdd) comments.Comment { @@ -337,9 +343,37 @@ func convertCommentFromCommentDel(cd comments.CommentDel) comments.Comment { } } -func commentAddSave(client *tlogbe.RecordClient, c comments.CommentAdd, encrypt bool) ([]byte, error) { +// commentVersionLatest returns the latest comment version. +func commentVersionLatest(cidx commentIndex) uint32 { + var maxVersion uint32 + for version := range cidx.Adds { + if version > maxVersion { + maxVersion = version + } + } + return maxVersion +} + +// commentExists returns whether the provided comment ID exists. +func commentExists(idx commentsIndex, commentID uint32) bool { + _, ok := idx.Comments[commentID] + return ok +} + +// commentIDLatest returns the latest comment ID. +func commentIDLatest(idx commentsIndex) uint32 { + var maxID uint32 + for id := range idx.Comments { + if id > maxID { + maxID = id + } + } + return maxID +} + +func (p *commentsPlugin) commentAddSave(ca comments.CommentAdd) ([]byte, error) { // Prepare blob - be, err := convertBlobEntryFromCommentAdd(c) + be, err := convertBlobEntryFromCommentAdd(ca) if err != nil { return nil, err } @@ -352,8 +386,24 @@ func commentAddSave(client *tlogbe.RecordClient, c comments.CommentAdd, encrypt return nil, err } + // Prepare tlog args + tlogID := tlogIDForCommentState(ca.State) + token, err := hex.DecodeString(ca.Token) + if err != nil { + return nil, err + } + var encrypt bool + switch ca.State { + case comments.StateUnvetted: + encrypt = true + case comments.StateVetted: + encrypt = false + default: + return nil, fmt.Errorf("unknown state %v", ca.State) + } + // Save blob - merkles, err := client.Save(keyPrefixCommentAdd, + merkles, err := p.tlog.save(tlogID, token, keyPrefixCommentAdd, [][]byte{b}, [][]byte{h}, encrypt) if err != nil { return nil, fmt.Errorf("Save: %v", err) @@ -366,6 +416,7 @@ func commentAddSave(client *tlogbe.RecordClient, c comments.CommentAdd, encrypt return merkles[0], nil } +/* func commentAdds(client *tlogbe.RecordClient, merkleHashes [][]byte) ([]comments.CommentAdd, error) { // Retrieve blobs blobs, err := client.BlobsByMerkleHash(merkleHashes) @@ -494,35 +545,6 @@ func commentVoteSave(client *tlogbe.RecordClient, c comments.CommentVote) ([]byt return merkles[0], nil } -func indexSave(client *tlogbe.RecordClient, idx index) error { - // Prepare blob - be, err := convertBlobEntryFromIndex(idx) - if err != nil { - return err - } - h, err := hex.DecodeString(be.Hash) - if err != nil { - return err - } - b, err := store.Blobify(*be) - if err != nil { - return err - } - - // Save blob - merkles, err := client.Save(keyPrefixCommentsIndex, - [][]byte{b}, [][]byte{h}, false) - if err != nil { - return fmt.Errorf("Save: %v", err) - } - if len(merkles) != 1 { - return fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", - len(merkles)) - } - - return nil -} - // indexAddCommentVote adds the provided comment vote to the index and // calculates the new vote score. The updated index is returned. The effect of // a new vote on a comment depends on the previous vote from that uuid. @@ -575,53 +597,7 @@ func indexAddCommentVote(idx index, cv comments.CommentVote, merkleHash []byte) return idx } -func indexLatest(client *tlogbe.RecordClient) (*index, error) { - // Get all comment indexes - blobs, err := client.BlobsByKeyPrefix(keyPrefixCommentsIndex) - if err != nil { - return nil, err - } - if len(blobs) == 0 { - // A comments index does not exist. This can happen when no - // comments have been made on the record yet. Return a new one. - return &index{ - Comments: make(map[uint32]commentIndex), - }, nil - } - - // Decode the most recent index - b := blobs[len(blobs)-1] - be, err := store.Deblob(b) - if err != nil { - return nil, err - } - return convertIndexFromBlobEntry(*be) -} - -func commentIDLatest(idx index) uint32 { - var maxID uint32 - for id := range idx.Comments { - if id > maxID { - maxID = id - } - } - return maxID -} - -func commentVersionLatest(cidx commentIndex) uint32 { - var maxVersion uint32 - for version := range cidx.Adds { - if version > maxVersion { - maxVersion = version - } - } - return maxVersion -} -func commentExists(idx index, commentID uint32) bool { - _, ok := idx.Comments[commentID] - return ok -} // commentsLatest returns the most recent version of the specified comments. // Deleted comments are returned with limited data. Comment IDs that do not @@ -691,67 +667,63 @@ func commentsLatest(client *tlogbe.RecordClient, idx index, commentIDs []uint32) return cs, nil } +*/ -// This function must be called WITH the record lock held. -func (p *commentsPlugin) new(client *tlogbe.RecordClient, n comments.New, encrypt bool) (*comments.NewReply, error) { - // Pull comments index - idx, err := indexLatest(client) +func (p *commentsPlugin) commentsIndex(state comments.StateT, token []byte) (*commentsIndex, error) { + // Get all comment indexes + tlogID := tlogIDForCommentState(state) + blobs, err := p.tlog.blobsByKeyPrefix(tlogID, token, keyPrefixCommentsIndex) if err != nil { return nil, err } - - // Verify parent comment exists if set. A parent ID of 0 means that - // this is a base level comment, not a reply to another comment. - if n.ParentID > 0 && !commentExists(*idx, n.ParentID) { - return nil, comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusParentIDInvalid, - ErrorContext: []string{"parent ID comment not found"}, - } - } - - // Setup comment - receipt := p.id.SignMessage([]byte(n.Signature)) - c := comments.CommentAdd{ - Token: n.Token, - ParentID: n.ParentID, - Comment: n.Comment, - PublicKey: n.PublicKey, - Signature: n.Signature, - CommentID: commentIDLatest(*idx) + 1, - Version: 1, - Timestamp: time.Now().Unix(), - Receipt: hex.EncodeToString(receipt[:]), + if len(blobs) == 0 { + // A comments index does not exist. This can happen when no + // comments have been made on the record yet. Return a new one. + return &commentsIndex{ + Comments: make(map[uint32]commentIndex), + }, nil } - // Save comment - merkleHash, err := commentAddSave(client, c, encrypt) + // Decode the most recent index + b := blobs[len(blobs)-1] + be, err := store.Deblob(b) if err != nil { - return nil, fmt.Errorf("commentSave: %v", err) + return nil, err } + return convertCommentsIndexFromBlobEntry(*be) +} - // Update index - idx.Comments[c.CommentID] = commentIndex{ - Adds: map[uint32][]byte{ - 1: merkleHash, - }, - Del: nil, - Votes: make(map[string][]voteIndex), - } +func (p *commentsPlugin) commentsIndexSave(token []byte, idx commentsIndex) error { + /* + // Prepare blob + be, err := convertBlobEntryFromCommentsIndex(idx) + if err != nil { + return err + } + h, err := hex.DecodeString(be.Hash) + if err != nil { + return err + } + b, err := store.Blobify(*be) + if err != nil { + return err + } - // Save index - err = indexSave(client, *idx) - if err != nil { - return nil, fmt.Errorf("indexSave: %v", err) - } + // Prepare - log.Debugf("Comment saved to record %v comment ID %v", - c.Token, c.CommentID) + // Save blob + merkles, err := client.Save(keyPrefixCommentsIndex, + [][]byte{b}, [][]byte{h}, false) + if err != nil { + return fmt.Errorf("Save: %v", err) + } + if len(merkles) != 1 { + return fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", + len(merkles)) + } + */ - return &comments.NewReply{ - CommentID: c.CommentID, - Timestamp: c.Timestamp, - Receipt: c.Receipt, - }, nil + return nil } func (p *commentsPlugin) cmdNew(payload string) (string, error) { @@ -764,10 +736,11 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { } // Verify signature - msg := n.Token + strconv.FormatUint(uint64(n.ParentID), 10) + n.Comment + msg := strconv.Itoa(int(n.State)) + n.Token + + strconv.FormatUint(uint64(n.ParentID), 10) + n.Comment err = util.VerifySignature(n.Signature, n.PublicKey, msg) if err != nil { - return "", convertCommentsErrFromSignatureErr(err) + return "", convertCommentsErrorFromSignatureError(err) } // Get record client @@ -777,15 +750,18 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { ErrorCode: comments.ErrorStatusTokenInvalid, } } - client, err := p.backend.RecordClient(token) - if err != nil { - if err == backend.ErrRecordNotFound { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusRecordNotFound, + + /* + client, err := p.backend.RecordClient(token) + if err != nil { + if err == backend.ErrRecordNotFound { + return "", comments.UserErrorReply{ + ErrorCode: comments.ErrorStatusRecordNotFound, + } } + return "", err } - return "", err - } + */ // The comments index must be pulled and updated. The record lock // must be held for the remainder of this function. @@ -793,25 +769,67 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { m.Lock() defer m.Unlock() - // Save new comment - var nr *comments.NewReply - switch client.State { - case tlogbe.RecordStateUnvetted: - nr, err = p.new(client, *n, true) - if err != nil { - return "", err - } - case tlogbe.RecordStateVetted: - nr, err = p.new(client, *n, false) - if err != nil { - return "", err + // Pull comments index + idx, err := p.commentsIndex(n.State, token) + if err != nil { + return "", err + } + + // Verify parent comment exists if set. A parent ID of 0 means that + // this is a base level comment, not a reply to another comment. + if n.ParentID > 0 && !commentExists(*idx, n.ParentID) { + return "", comments.UserErrorReply{ + ErrorCode: comments.ErrorStatusParentIDInvalid, + ErrorContext: []string{"parent ID comment not found"}, } - default: - return "", fmt.Errorf("invalid record state %v", client.State) } + // Setup comment + receipt := p.id.SignMessage([]byte(n.Signature)) + ca := comments.CommentAdd{ + UUID: n.UUID, + Token: n.Token, + ParentID: n.ParentID, + Comment: n.Comment, + PublicKey: n.PublicKey, + Signature: n.Signature, + CommentID: commentIDLatest(*idx) + 1, + Version: 1, + Timestamp: time.Now().Unix(), + Receipt: hex.EncodeToString(receipt[:]), + } + + // Save comment + merkleHash, err := p.commentAddSave(ca) + if err != nil { + return "", fmt.Errorf("commentSave: %v", err) + } + + // Update index + idx.Comments[ca.CommentID] = commentIndex{ + Adds: map[uint32][]byte{ + 1: merkleHash, + }, + Del: nil, + Votes: make(map[string][]voteIndex), + } + + // Save index + err = p.commentsIndexSave(token, *idx) + if err != nil { + return "", fmt.Errorf("indexSave: %v", err) + } + + log.Debugf("Comment saved to record %v comment ID %v", + ca.Token, ca.CommentID) + // Prepare reply - reply, err := comments.EncodeNewReply(*nr) + nr := comments.NewReply{ + CommentID: ca.CommentID, + Timestamp: ca.Timestamp, + Receipt: ca.Receipt, + } + reply, err := comments.EncodeNewReply(nr) if err != nil { return "", err } @@ -819,6 +837,7 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { return string(reply), nil } +/* // This function must be called WITH the record lock held. func (p *commentsPlugin) edit(client *tlogbe.RecordClient, e comments.Edit, encrypt bool) (*comments.EditReply, error) { // Get comments index @@ -1441,55 +1460,52 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { return string(reply), nil } +*/ -func (p *commentsPlugin) cmdProofs(payload string) (string, error) { - log.Tracef("comments cmdProof: %v", payload) - return "", nil -} - -// Cmd executes a plugin command. +// cmd executes a plugin command. // -// This function satisfies the Plugin interface. -func (p *commentsPlugin) Cmd(cmd, payload string) (string, error) { - log.Tracef("comments Cmd: %v", cmd) +// This function satisfies the pluginClient interface. +func (p *commentsPlugin) cmd(cmd, payload string) (string, error) { + log.Tracef("comments cmd: %v", cmd) switch cmd { case comments.CmdNew: return p.cmdNew(payload) - case comments.CmdEdit: - return p.cmdEdit(payload) - case comments.CmdDel: - return p.cmdDel(payload) - case comments.CmdGet: - return p.cmdGet(payload) - case comments.CmdGetAll: - return p.cmdGetAll(payload) - case comments.CmdGetVersion: - return p.cmdGetVersion(payload) - case comments.CmdCount: - return p.cmdCount(payload) - case comments.CmdVote: - return p.cmdVote(payload) - case comments.CmdProofs: - return p.cmdProofs(payload) + /* + case comments.CmdEdit: + return p.cmdEdit(payload) + case comments.CmdDel: + return p.cmdDel(payload) + case comments.CmdGet: + return p.cmdGet(payload) + case comments.CmdGetAll: + return p.cmdGetAll(payload) + case comments.CmdGetVersion: + return p.cmdGetVersion(payload) + case comments.CmdCount: + return p.cmdCount(payload) + case comments.CmdVote: + return p.cmdVote(payload) + */ } return "", backend.ErrPluginCmdInvalid } -// Hook executes a plugin hook. +// hook executes a plugin hook. // -// This function satisfies the Plugin interface. -func (p *commentsPlugin) Hook(h tlogbe.HookT, payload string) error { - log.Tracef("comments Hook: %v", tlogbe.Hooks[h]) +// This function satisfies the pluginClient interface. +func (p *commentsPlugin) hook(h hookT, payload string) error { + log.Tracef("comments hook: %v", hooks[h]) + return nil } -// Fsck performs a plugin filesystem check. +// fsck performs a plugin filesystem check. // -// This function satisfies the Plugin interface. -func (p *commentsPlugin) Fsck() error { - log.Tracef("comments Fsck") +// This function satisfies the pluginClient interface. +func (p *commentsPlugin) fsck() error { + log.Tracef("comments fsck") // Make sure CommentDel blobs were actually deleted @@ -1498,20 +1514,22 @@ func (p *commentsPlugin) Fsck() error { // Setup performs any plugin setup work that needs to be done. // -// This function satisfies the Plugin interface. -func (p *commentsPlugin) Setup() error { - log.Tracef("comments Setup") +// This function satisfies the pluginClient interface. +func (p *commentsPlugin) setup() error { + log.Tracef("comments setup") + return nil } -// NewCommentsPlugin returns a new comments plugin. -func NewCommentsPlugin(backend *tlogbe.TlogBackend, settings []backend.PluginSetting) *commentsPlugin { +// newCommentsPlugin returns a new comments plugin. +func newCommentsPlugin(backend backend.Backend, tlog tlogClient, settings []backend.PluginSetting) *commentsPlugin { // TODO these should be passed in as plugin settings id := &identity.FullIdentity{} return &commentsPlugin{ id: id, backend: backend, + tlog: tlog, mutexes: make(map[string]*sync.Mutex), } } diff --git a/politeiad/backend/tlogbe/pluginclient.go b/politeiad/backend/tlogbe/pluginclient.go index e60e0b147..6379d2699 100644 --- a/politeiad/backend/tlogbe/pluginclient.go +++ b/politeiad/backend/tlogbe/pluginclient.go @@ -26,7 +26,7 @@ const ( ) var ( - // hooks contains human readable plugin hook descriptions. + // hooks contains human readable descriptions of the plugin hooks. hooks = map[hookT]string{ hookNewRecordPre: "new record pre", hookNewRecordPost: "new record post", @@ -39,32 +39,28 @@ var ( } ) -// newRecord is the payload for the hookNewRecordPre and hookNewRecordPost -// hooks. -type newRecord struct { +// hookNewRecord is the payload for the new record hooks. +type hookNewRecord struct { RecordMetadata backend.RecordMetadata `json:"recordmetadata"` Metadata []backend.MetadataStream `json:"metadata"` Files []backend.File `json:"files"` } -// encodeNewRecord encodes a newRecord into a JSON byte slice. -func encodeNewRecord(nr newRecord) ([]byte, error) { - return json.Marshal(nr) +func encodeHookNewRecord(hnr hookNewRecord) ([]byte, error) { + return json.Marshal(hnr) } -// decodeNewRecord decodes a JSON byte slice into a newRecord. -func decodeNewRecord(payload []byte) (*newRecord, error) { - var nr newRecord - err := json.Unmarshal(payload, &nr) +func decodeHookNewRecord(payload []byte) (*hookNewRecord, error) { + var hnr hookNewRecord + err := json.Unmarshal(payload, &hnr) if err != nil { return nil, err } - return &nr, nil + return &hnr, nil } -// editRecord is the payload for the hookEditRecordPre and hookEditRecordPost -// hooks. -type editRecord struct { +// hookEditRecord is the payload for the edit record hooks. +type hookEditRecord struct { // Current record Current backend.Record `json:"record"` @@ -76,24 +72,21 @@ type editRecord struct { FilesDel []string `json:"filesdel"` } -// encodeEditRecord encodes an editRecord into a JSON byte slice. -func encodeEditRecord(er editRecord) ([]byte, error) { - return json.Marshal(er) +func encodeHookEditRecord(her hookEditRecord) ([]byte, error) { + return json.Marshal(her) } -// decodeEditRecord decodes a JSON byte slice into a editRecord. -func decodeEditRecord(payload []byte) (*editRecord, error) { - var er editRecord - err := json.Unmarshal(payload, &er) +func decodeHookEditRecord(payload []byte) (*hookEditRecord, error) { + var her hookEditRecord + err := json.Unmarshal(payload, &her) if err != nil { return nil, err } - return &er, nil + return &her, nil } -// setRecordStatus is the payload for the hookSetRecordStatusPre and -// hookSetRecordStatusPost hooks. -type setRecordStatus struct { +// hookSetRecordStatus is the payload for the set record status hooks. +type hookSetRecordStatus struct { // Current record Current backend.Record `json:"record"` @@ -103,19 +96,17 @@ type setRecordStatus struct { MDOverwrite []backend.MetadataStream `json:"mdoverwrite"` } -// encodeSetRecordStatus encodes a setRecordStatus into a JSON byte slice. -func encodeSetRecordStatus(srs setRecordStatus) ([]byte, error) { - return json.Marshal(srs) +func encodeHookSetRecordStatus(hsrs hookSetRecordStatus) ([]byte, error) { + return json.Marshal(hsrs) } -// decodeSetRecordStatus decodes a JSON byte slice into a setRecordStatus. -func decodeSetRecordStatus(payload []byte) (*setRecordStatus, error) { - var srs setRecordStatus - err := json.Unmarshal(payload, &srs) +func decodeHookSetRecordStatus(payload []byte) (*hookSetRecordStatus, error) { + var hsrs hookSetRecordStatus + err := json.Unmarshal(payload, &hsrs) if err != nil { return nil, err } - return &srs, nil + return &hsrs, nil } // pluginClient provides an API for the tlog backend to use when interacting diff --git a/politeiad/backend/tlogbe/plugins/log.go b/politeiad/backend/tlogbe/plugins/log.go deleted file mode 100644 index 18e5ec634..000000000 --- a/politeiad/backend/tlogbe/plugins/log.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) 2013-2015 The btcsuite developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package plugins - -import "github.com/decred/slog" - -// log is a logger that is initialized with no output filters. This -// means the package will not perform any logging by default until the caller -// requests it. -var log = slog.Disabled - -// DisableLog disables all library log output. Logging output is disabled -// by default until either UseLogger or SetLogWriter are called. -func DisableLog() { - log = slog.Disabled -} - -// UseLogger uses a specified Logger to output package logging info. -// This should be used in preference to SetLogWriter if the caller is also -// using slog. -func UseLogger(logger slog.Logger) { - log = logger -} diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index b22cbc3ae..8007e1d6b 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -574,12 +574,12 @@ func (t *tlogBackend) New(metadata []backend.MetadataStream, files []backend.Fil } // Call pre plugin hooks - nr := newRecord{ + hnr := hookNewRecord{ RecordMetadata: *rm, Metadata: metadata, Files: files, } - b, err := encodeNewRecord(nr) + b, err := encodeHookNewRecord(hnr) if err != nil { return nil, err } @@ -659,7 +659,7 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ } // Call pre plugin hooks - er := editRecord{ + her := hookEditRecord{ Current: *r, RecordMetadata: *recordMD, MDAppend: mdAppend, @@ -667,7 +667,7 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ FilesAdd: filesAdd, FilesDel: filesDel, } - b, err := encodeEditRecord(er) + b, err := encodeHookEditRecord(her) if err != nil { return nil, err } @@ -764,7 +764,7 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b } // Call pre plugin hooks - er := editRecord{ + her := hookEditRecord{ Current: *r, RecordMetadata: *recordMD, MDAppend: mdAppend, @@ -772,7 +772,7 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b FilesAdd: filesAdd, FilesDel: filesDel, } - b, err := encodeEditRecord(er) + b, err := encodeHookEditRecord(her) if err != nil { return nil, err } @@ -1127,13 +1127,13 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) // Call pre plugin hooks - srs := setRecordStatus{ + hsrs := hookSetRecordStatus{ Current: *r, RecordMetadata: rm, MDAppend: mdAppend, MDOverwrite: mdOverwrite, } - b, err := encodeSetRecordStatus(srs) + b, err := encodeHookSetRecordStatus(hsrs) if err != nil { return nil, err } @@ -1257,13 +1257,13 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) // Call pre plugin hooks - srs := setRecordStatus{ + srs := hookSetRecordStatus{ Current: *r, RecordMetadata: rm, MDAppend: mdAppend, MDOverwrite: mdOverwrite, } - b, err := encodeSetRecordStatus(srs) + b, err := encodeHookSetRecordStatus(srs) if err != nil { return nil, err } diff --git a/politeiad/backend/tlogbe/tlogclient.go b/politeiad/backend/tlogbe/tlogclient.go index 2f19919e2..b3fc9161e 100644 --- a/politeiad/backend/tlogbe/tlogclient.go +++ b/politeiad/backend/tlogbe/tlogclient.go @@ -8,7 +8,7 @@ import ( "fmt" ) -// tlogClient provides an interface for plugins to interact with the tlog +// tlogClient provides an API for the plugins to use to interact with the tlog // backend. Plugins are allowed to save, delete, and get plugin data to/from // the tlog backend. Editing plugin data is not allowed. type tlogClient interface { @@ -76,11 +76,13 @@ func (c *backendClient) treeIDFromToken(tlogID string, token []byte) (int64, err func (c *backendClient) save(tlogID string, token []byte, keyPrefix string, blobs, hashes [][]byte, encrypt bool) ([][]byte, error) { log.Tracef("save: %x %v %v %x", token, keyPrefix, encrypt, hashes) - // Get tlog instance and treeID + // Get tlog instance tlog, err := c.tlogByID(tlogID) if err != nil { return nil, err } + + // Get tree ID treeID, err := c.treeIDFromToken(tlogID, token) if err != nil { return nil, err @@ -94,17 +96,20 @@ func (c *backendClient) save(tlogID string, token []byte, keyPrefix string, blob func (c *backendClient) del(tlogID string, token []byte, merkles [][]byte) error { log.Tracef("del: %x %x", token, merkles) - // Get tlog instance and treeID + // Get tlog instance tlog, err := c.tlogByID(tlogID) if err != nil { return err + } + + // Get tree ID treeID, err := c.treeIDFromToken(tlogID, token) if err != nil { return err } - // Save blobs + // Delete blobs return tlog.blobsDel(treeID, merkles) } @@ -116,16 +121,19 @@ func (c *backendClient) del(tlogID string, token []byte, merkles [][]byte) error func (c *backendClient) blobsByMerkle(tlogID string, token []byte, merkles [][]byte) (map[string][]byte, error) { log.Tracef("blobsByMerkle: %x %x", token, merkles) - // Get tlog instance and treeID + // Get tlog instance tlog, err := c.tlogByID(tlogID) if err != nil { return nil, err } + + // Get tree ID treeID, err := c.treeIDFromToken(tlogID, token) if err != nil { return nil, err } + // Get blobs return tlog.blobsByMerkle(treeID, merkles) } @@ -133,15 +141,18 @@ func (c *backendClient) blobsByMerkle(tlogID string, token []byte, merkles [][]b func (c *backendClient) blobsByKeyPrefix(tlogID string, token []byte, keyPrefix string) ([][]byte, error) { log.Tracef("blobsByKeyPrefix: %x %x", token, keyPrefix) - // Get tlog instance and treeID + // Get tlog instance tlog, err := c.tlogByID(tlogID) if err != nil { return nil, err } + + // Get tree ID treeID, err := c.treeIDFromToken(tlogID, token) if err != nil { return nil, err } + // Get blobs return tlog.blobsByKeyPrefix(treeID, keyPrefix) } diff --git a/politeiad/log.go b/politeiad/log.go index 3513117bd..2b161f12b 100644 --- a/politeiad/log.go +++ b/politeiad/log.go @@ -11,7 +11,6 @@ import ( "github.com/decred/politeia/politeiad/backend/gitbe" "github.com/decred/politeia/politeiad/backend/tlogbe" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" "github.com/decred/slog" "github.com/jrick/logrotate/rotator" @@ -44,28 +43,25 @@ var ( // application shutdown. logRotator *rotator.Rotator - log = backendLog.Logger("POLI") - gitbeLog = backendLog.Logger("GITB") - backLog = backendLog.Logger("BACK") - storeLog = backendLog.Logger("STOR") - pluginsLog = backendLog.Logger("PLGN") + log = backendLog.Logger("POLI") + gitbeLog = backendLog.Logger("GITB") + tlogLog = backendLog.Logger("TLOG") + storeLog = backendLog.Logger("STOR") ) // Initialize package-global logger variables. func init() { gitbe.UseLogger(gitbeLog) - tlogbe.UseLogger(backLog) + tlogbe.UseLogger(tlogLog) filesystem.UseLogger(storeLog) - plugins.UseLogger(pluginsLog) } // subsystemLoggers maps each subsystem identifier to its associated logger. var subsystemLoggers = map[string]slog.Logger{ "POLI": log, "GITB": gitbeLog, - "BACK": backLog, + "TLOG": tlogLog, "STOR": storeLog, - "PLGN": pluginsLog, } // initLogRotator initializes the logging rotater to write logs to logFile and diff --git a/politeiawww/email.go b/politeiawww/email.go index f74c83911..7585f6fe1 100644 --- a/politeiawww/email.go +++ b/politeiawww/email.go @@ -150,7 +150,6 @@ func (p *politeiawww) emailProposalStatusChangeToAuthor(d dataProposalStatusChan tmplData := proposalCensoredToAuthor{ Name: d.name, Reason: d.reason, - Link: l.String(), } body, err = createBody(tmplProposalCensoredForAuthor, tmplData) if err != nil { diff --git a/politeiawww/templates.go b/politeiawww/templates.go index 6f32856fe..fa6fb9687 100644 --- a/politeiawww/templates.go +++ b/politeiawww/templates.go @@ -78,18 +78,24 @@ type proposalVettedToAuthor struct { } const proposalVettedToAuthorText = ` -Your proposal has just been approved on Politeia! +Your proposal has just been made public on Politeia! -You will need to authorize a proposal vote before an administrator will be -allowed to start the voting period on your proposal. You can authorize a -proposal vote by opening the proposal page and clicking on the "Authorize -Voting to Start" button. +Your proposal has now entered the discussion phase where the community can +leave comments and provide feedback. Be sure to keep an eye out for new +comments and to answer any questions that the community may have. You are +allowed to edit your proposal at any point prior to the start of voting. -You should allow sufficient time for the community to discuss your proposal -before authorizing the vote. +Once you feel that enough time has been given for discussion you may authorize +the vote to commence on your proposal. An admin is not able to start the +voting process until you explicitly authorize it. You can authorize a proposal +vote by opening the proposal page and clicking on the "Authorize Voting to +Start" button. {{.Name}} {{.Link}} + +If you have any questions, drop by the proposals channel on matrix: +https://chat.decred.org/#/room/#proposals:decred.org ` var proposalVettedToAuthorTmpl = template.Must( @@ -99,7 +105,6 @@ var proposalVettedToAuthorTmpl = template.Must( type proposalCensoredToAuthor struct { Name string // Proposal name Reason string // Reason for censoring - Link string // GUI proposal details URL } const proposalCensoredToAuthorText = ` @@ -107,7 +112,6 @@ Your proposal on Politeia has been censored. {{.Name}} Reason: {{.Reason}} -{{.Link}} ` var tmplProposalCensoredForAuthor = template.Must( From e9246c81eac65873ccaecf6dc8e8ac21dd31eb86 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 20 Sep 2020 14:54:27 -0500 Subject: [PATCH 076/449] template changes --- politeiawww/email.go | 56 +++++--- politeiawww/eventmanager.go | 166 ++++++++++++++++------- politeiawww/piwww.go | 48 +------ politeiawww/politeiad.go | 16 +-- politeiawww/templates.go | 257 +++++++++++++++++++----------------- 5 files changed, 309 insertions(+), 234 deletions(-) diff --git a/politeiawww/email.go b/politeiawww/email.go index 7585f6fe1..11a8ee272 100644 --- a/politeiawww/email.go +++ b/politeiawww/email.go @@ -122,7 +122,7 @@ func (p *politeiawww) emailResetPasswordVerificationLink(email, username, token // emailAuthorProposalStatusChange sends an email to the author of the proposal // notifying them of the proposal status change. -func (p *politeiawww) emailProposalStatusChangeToAuthor(d dataProposalStatusChange) error { +func (p *politeiawww) emailProposalStatusChangeToAuthor(d dataProposalStatusChange, proposalName, authorEmail string) error { route := strings.Replace(guiRouteProposalDetails, "{token}", d.token, 1) l, err := url.Parse(p.cfg.WebServerAddress + route) if err != nil { @@ -137,7 +137,7 @@ func (p *politeiawww) emailProposalStatusChangeToAuthor(d dataProposalStatusChan case pi.PropStatusPublic: subject = "Your Proposal Has Been Published" tmplData := proposalVettedToAuthor{ - Name: d.name, + Name: proposalName, Link: l.String(), } body, err = createBody(proposalVettedToAuthorTmpl, tmplData) @@ -148,7 +148,7 @@ func (p *politeiawww) emailProposalStatusChangeToAuthor(d dataProposalStatusChan case pi.PropStatusCensored: subject = "Your Proposal Has Been Censored" tmplData := proposalCensoredToAuthor{ - Name: d.name, + Name: proposalName, Reason: d.reason, } body, err = createBody(tmplProposalCensoredForAuthor, tmplData) @@ -160,12 +160,12 @@ func (p *politeiawww) emailProposalStatusChangeToAuthor(d dataProposalStatusChan return fmt.Errorf("no author notification for prop status %v", d.status) } - return p.smtp.sendEmailTo(subject, body, []string{d.author.Email}) + return p.smtp.sendEmailTo(subject, body, []string{authorEmail}) } // emailProposalStatusChangeToUsers sends an email to the provided users // notifying them of the proposal status change. -func (p *politeiawww) emailProposalStatusChangeToUsers(d dataProposalStatusChange, emails []string) error { +func (p *politeiawww) emailProposalStatusChangeToUsers(d dataProposalStatusChange, proposalName string, emails []string) error { route := strings.Replace(guiRouteProposalDetails, "{token}", d.token, 1) l, err := url.Parse(p.cfg.WebServerAddress + route) if err != nil { @@ -180,7 +180,7 @@ func (p *politeiawww) emailProposalStatusChangeToUsers(d dataProposalStatusChang case pi.PropStatusPublic: subject = "New Proposal Published" tmplData := proposalVetted{ - Name: d.name, + Name: proposalName, Link: l.String(), } body, err = createBody(tmplProposalVetted, tmplData) @@ -318,7 +318,8 @@ func (p *politeiawww) emailProposalVoteAuthorized(token, name, username, email s return p.smtp.sendEmailTo(subject, body, emails) } -func (p *politeiawww) emailProposalComment(token, commentID, commentUsername, name, email string) error { +func (p *politeiawww) emailProposalCommentSubmitted(token, commentID, commentUsername, proposalName, proposalAuthorEmail string) error { + // Setup comment URL route := strings.Replace(guirouteProposalComments, "{token}", token, 1) route = strings.Replace(route, "{id}", commentID, 1) l, err := url.Parse(p.cfg.WebServerAddress + route) @@ -326,20 +327,45 @@ func (p *politeiawww) emailProposalComment(token, commentID, commentUsername, na return err } - tplData := commentReplyOnProposalTemplateData{ - Commenter: commentUsername, - ProposalName: name, - CommentLink: l.String(), + // Setup email + subject := "New Comment On Your Proposal" + tmplData := proposalCommentSubmitted{ + Username: commentUsername, + Name: proposalName, + Link: l.String(), + } + body, err := createBody(proposalCommentSubmittedTmpl, tmplData) + if err != nil { + return err } - subject := "New Comment On Your Proposal" - body, err := createBody(templateCommentReplyOnProposal, tplData) + // Send email + return p.smtp.sendEmailTo(subject, body, []string{proposalAuthorEmail}) +} + +func (p *politeiawww) emailProposalCommentReply(token, commentID, commentUsername, proposalName, parentCommentEmail string) error { + // Setup comment URL + route := strings.Replace(guirouteProposalComments, "{token}", token, 1) + route = strings.Replace(route, "{id}", commentID, 1) + l, err := url.Parse(p.cfg.WebServerAddress + route) if err != nil { return err } - recipients := []string{email} - return p.smtp.sendEmailTo(subject, body, recipients) + // Setup email + subject := "New Reply To Your Comment" + tmplData := proposalCommentReply{ + Username: commentUsername, + Name: proposalName, + Link: l.String(), + } + body, err := createBody(proposalCommentReplyTmpl, tmplData) + if err != nil { + return err + } + + // Send email + return p.smtp.sendEmailTo(subject, body, []string{parentCommentEmail}) } // emailUpdateUserKeyVerificationLink emails the link with the verification diff --git a/politeiawww/eventmanager.go b/politeiawww/eventmanager.go index 1468d50a2..d7a8ae90d 100644 --- a/politeiawww/eventmanager.go +++ b/politeiawww/eventmanager.go @@ -5,11 +5,14 @@ package main import ( + "fmt" + "strconv" "sync" pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/user" + "github.com/google/uuid" ) type eventT int @@ -245,12 +248,10 @@ func (p *politeiawww) handleEventProposalEdited(ch chan interface{}) { } type dataProposalStatusChange struct { - name string // Proposal name token string // Proposal censorship token status pi.PropStatusT // Proposal status reason string // Status change reason adminID string // Admin uuid - author user.User // Proposal author } func (p *politeiawww) handleEventProposalStatusChange(ch chan interface{}) { @@ -271,16 +272,31 @@ func (p *politeiawww) handleEventProposalStatusChange(ch chan interface{}) { continue } + // Get the proposal author + state := convertPropStateFromPropStatus(d.status) + pr, err := p.proposalRecordLatest(state, d.token) + if err != nil { + log.Errorf("handleEventProposalStatusChange: proposalRecordLatest "+ + "%v %v: %v", state, d.token, err) + continue + } + author, err := p.db.UserGetByPubKey(pr.PublicKey) + if err != nil { + log.Errorf("handleEventProposalStatusChange: UserGetByPubKey %v: %v", + pr.PublicKey, err) + continue + } + // Compile list of emails to send notification to emails := make([]string, 0, 256) notification := www.NotificationEmailRegularProposalVetted - err := p.db.AllUsers(func(u *user.User) { + err = p.db.AllUsers(func(u *user.User) { // Check circumstances where we don't notify switch { case u.ID.String() == d.adminID: // User is the admin that made the status change return - case u.ID.String() == d.author.ID.String(): + case u.ID.String() == author.ID.String(): // User is the author. The author is sent a notification but // the notification is not the same as the normal user // notification so don't include the author's emails in the @@ -295,8 +311,9 @@ func (p *politeiawww) handleEventProposalStatusChange(ch chan interface{}) { }) // Email author - if userNotificationEnabled(d.author, notification) { - err = p.emailProposalStatusChangeToAuthor(d) + proposalName := proposalName(*pr) + if userNotificationEnabled(*author, notification) { + err = p.emailProposalStatusChangeToAuthor(d, proposalName, author.Email) if err != nil { log.Errorf("emailProposalStatusChangeToAuthor: %v", err) } @@ -304,7 +321,7 @@ func (p *politeiawww) handleEventProposalStatusChange(ch chan interface{}) { // Email users if len(emails) > 0 { - err = p.emailProposalStatusChangeToUsers(d, emails) + err = p.emailProposalStatusChangeToUsers(d, proposalName, emails) if err != nil { log.Errorf("emailProposalStatusChangeToUsers: %v", err) } @@ -315,12 +332,85 @@ func (p *politeiawww) handleEventProposalStatusChange(ch chan interface{}) { } type dataProposalComment struct { - token string // Proposal token - name string // Proposal name - username string // Author username - parentID string // Parent comment id - commentID string // Comment id - commentUsername string // Comment user username + state pi.PropStateT + token string + commentID uint32 + parentID uint32 + username string // Comment author username +} + +func (p *politeiawww) notifyProposalAuthorOnComment(d dataProposalComment) error { + // Lookup proposal author to see if they should be sent a + // notification. + pr, err := p.proposalRecordLatest(d.state, d.token) + if err != nil { + return fmt.Errorf("proposalRecordLatest %v %v: %v", + d.state, d.token, err) + } + userID, err := uuid.Parse(pr.UserID) + if err != nil { + return err + } + author, err := p.db.UserGetById(userID) + if err != nil { + return fmt.Errorf("UserGetByID %v: %v", userID.String(), err) + } + + // Check if notification should be sent to author + switch { + case d.username == author.Username: + // Author commented on their own proposal + return nil + case !userNotificationEnabled(*author, + www.NotificationEmailCommentOnMyProposal): + // Author does not have notification bit set on + return nil + } + + // Send notification eamil + commentID := strconv.FormatUint(uint64(d.commentID), 10) + return p.emailProposalCommentSubmitted(d.token, commentID, d.username, + proposalName(*pr), author.Email) +} + +func (p *politeiawww) notifyParentAuthorOnComment(d dataProposalComment) error { + // Verify this is a reply comment + if d.parentID == 0 { + return nil + } + + // Lookup the parent comment author to check if they should receive + // a reply notification. + + // Get the parent comment + // TODO + + // Lookup the parent comment author + var author *user.User + + // Check if notification should be sent to author + switch { + case d.username == author.Username: + // Author commented on their own proposal + return nil + case !userNotificationEnabled(*author, + www.NotificationEmailCommentOnMyProposal): + // Author does not have notification bit set on + return nil + } + + // Get proposal. We need this proposal name for the notification. + pr, err := p.proposalRecordLatest(d.state, d.token) + if err != nil { + return fmt.Errorf("proposalRecordLatest %v %v: %v", + d.state, d.token, err) + } + + // Send notification eamil + commentID := strconv.FormatUint(uint64(d.commentID), 10) + + return p.emailProposalCommentReply(d.token, commentID, d.username, + proposalName(*pr), author.Email) } func (p *politeiawww) handleEventProposalComment(ch chan interface{}) { @@ -331,48 +421,28 @@ func (p *politeiawww) handleEventProposalComment(ch chan interface{}) { continue } - // Fetch proposal author - author, err := p.db.UserGetByUsername(d.username) + // Notify the proposal author + err := p.notifyProposalAuthorOnComment(d) if err != nil { - log.Error(err) - continue + err = fmt.Errorf("notifyProposalAuthorOnComment: %v", err) + goto next } - // Check circumstances where we don't notify - switch { - case d.commentUsername == author.Username: - // Don't notify when author comments on own proposal - continue - case !userNotificationEnabled(*author, - www.NotificationEmailCommentOnMyProposal): - // User does not have notification bit set on - continue - } - - // Check if is top-level comment - if d.parentID != "0" { - // Nested comment reply. Fetch parent comment in order to fetch - // comment author - parent, err := p.decredCommentGetByID(d.token, d.parentID) - if err != nil { - log.Errorf("decredCommentGetByID: %v", err) - continue - } - - author, err = p.db.UserGetByPubKey(parent.PublicKey) - if err != nil { - log.Errorf("UserGetByPubKey: %v", err) - continue - } - } - - err = p.emailProposalComment(d.token, d.commentID, - d.commentUsername, d.name, author.Email) + // Notify the parent comment author + err = p.notifyParentAuthorOnComment(d) if err != nil { - log.Errorf("emailProposalComment: %v", err) + err = fmt.Errorf("notifyParentAuthorOnComment: %v", err) + goto next } + // Notifications successfully sent log.Debugf("Sent proposal commment notification %v", d.token) + continue + + next: + // If we made it here then there was an error. Log the error + // before listening for the next event. + log.Errorf("handleEventProposalComment: %v", err) } } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 44dd46e79..dc8ceca32 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -1060,15 +1060,16 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p // Send politeiad request // TODO verify that this will throw an error if no proposal files // were changed. + var cr *pd.CensorshipRecord switch pe.State { case pi.PropStateUnvetted: - err = p.updateUnvetted(pe.Token, mdAppend, mdOverwrite, + cr, err = p.updateUnvetted(pe.Token, mdAppend, mdOverwrite, filesAdd, filesDel) if err != nil { return nil, err } case pi.PropStateVetted: - err = p.updateVetted(pe.Token, mdAppend, mdOverwrite, + cr, err = p.updateVetted(pe.Token, mdAppend, mdOverwrite, filesAdd, filesDel) if err != nil { return nil, err @@ -1092,8 +1093,8 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p } return &pi.ProposalEditReply{ - // TODO CensorshipRecord: cr, - Timestamp: timestamp, + CensorshipRecord: convertCensorshipRecordFromPD(*cr), + Timestamp: timestamp, }, nil } @@ -1196,53 +1197,14 @@ func (p *politeiawww) processProposalSetStatus(pss pi.ProposalSetStatus, usr use } // Emit status change event - var ( - r *pd.Record - pr *pi.ProposalRecord - author *user.User - ) - switch pss.State { - case pi.PropStateUnvetted: - r, err = p.getUnvettedLatest(pss.Token) - if err != nil { - err = fmt.Errorf("getUnvettedLatest: %v", err) - goto reply - } - case pi.PropStateVetted: - r, err = p.getVettedLatest(pss.Token) - if err != nil { - err = fmt.Errorf("getVettedLatest: %v", err) - goto reply - } - } - pr, err = convertProposalRecordFromPD(*r) - if err != nil { - err = fmt.Errorf("convertProposalRecordFromPD: %v", err) - goto reply - } - author, err = p.db.UserGetByPubKey(pr.PublicKey) - if err != nil { - err = fmt.Errorf("UserGetByPubKey: %v", err) - goto reply - } p.eventManager.emit(eventProposalStatusChange, dataProposalStatusChange{ - name: proposalName(*pr), token: pss.Token, status: pss.Status, reason: pss.Reason, adminID: usr.ID.String(), - author: *author, }) -reply: - // If an error exists at this point it means the error was from - // an action that occured after the status had been updated in - // politeiad. Log it and return a normal reply. - if err != nil { - log.Errorf("processProposalSetStatus: %v", err) - } - return &pi.ProposalSetStatusReply{ Timestamp: timestamp, }, nil diff --git a/politeiawww/politeiad.go b/politeiawww/politeiad.go index 5ad2945fa..b052c8f92 100644 --- a/politeiawww/politeiad.go +++ b/politeiawww/politeiad.go @@ -126,11 +126,11 @@ func (p *politeiawww) newRecord(metadata []pd.MetadataStream, files []pd.File) ( // updateRecord updates a record in politeiad. This can be used to update // unvetted or vetted records depending on the route that is provided. -func (p *politeiawww) updateRecord(route, token string, mdAppend, mdOverwrite []pd.MetadataStream, filesAdd []pd.File, filesDel []string) error { +func (p *politeiawww) updateRecord(route, token string, mdAppend, mdOverwrite []pd.MetadataStream, filesAdd []pd.File, filesDel []string) (*pd.CensorshipRecord, error) { // Setup request challenge, err := util.Random(pd.ChallengeSize) if err != nil { - return err + return nil, err } ur := pd.UpdateRecord{ Token: token, @@ -143,33 +143,33 @@ func (p *politeiawww) updateRecord(route, token string, mdAppend, mdOverwrite [] // Send request resBody, err := p.makeRequest(http.MethodPost, route, ur) if err != nil { - return nil + return nil, nil } // Receive reply var urr pd.UpdateRecordReply err = json.Unmarshal(resBody, &urr) if err != nil { - return err + return nil, err } // Verify challenge err = util.VerifyChallenge(p.cfg.Identity, challenge, urr.Response) if err != nil { - return err + return nil, err } - return nil + return &urr.CensorshipRecord, nil } // updateUnvetted updates an unvetted record in politeiad. -func (p *politeiawww) updateUnvetted(token string, mdAppend, mdOverwrite []pd.MetadataStream, filesAdd []pd.File, filesDel []string) error { +func (p *politeiawww) updateUnvetted(token string, mdAppend, mdOverwrite []pd.MetadataStream, filesAdd []pd.File, filesDel []string) (*pd.CensorshipRecord, error) { return p.updateRecord(pd.UpdateUnvettedRoute, token, mdAppend, mdOverwrite, filesAdd, filesDel) } // updateVetted updates a vetted record in politeiad. -func (p *politeiawww) updateVetted(token string, mdAppend, mdOverwrite []pd.MetadataStream, filesAdd []pd.File, filesDel []string) error { +func (p *politeiawww) updateVetted(token string, mdAppend, mdOverwrite []pd.MetadataStream, filesAdd []pd.File, filesDel []string) (*pd.CensorshipRecord, error) { return p.updateRecord(pd.UpdateVettedRoute, token, mdAppend, mdOverwrite, filesAdd, filesDel) } diff --git a/politeiawww/templates.go b/politeiawww/templates.go index fa6fb9687..2a1b06032 100644 --- a/politeiawww/templates.go +++ b/politeiawww/templates.go @@ -6,19 +6,7 @@ package main import "text/template" -var ( - templateProposalVoteStarted = template.Must( - template.New("proposal_vote_started_template").Parse(templateProposalVoteStartedRaw)) - templateProposalVoteAuthorized = template.Must( - template.New("proposal_vote_authorized_template").Parse(templateProposalVoteAuthorizedRaw)) - templateProposalVoteStartedForAuthor = template.Must( - template.New("proposal_vote_started_for_author_template").Parse(templateProposalVoteStartedForAuthorRaw)) - templateCommentReplyOnProposal = template.Must( - template.New("comment_reply_on_proposal").Parse(templateCommentReplyOnProposalRaw)) - templateCommentReplyOnComment = template.Must( - template.New("comment_reply_on_comment"). - Parse(templateCommentReplyOnCommentRaw)) -) +var () // Proposal submitted type proposalSubmitted struct { @@ -117,77 +105,92 @@ Reason: {{.Reason}} var tmplProposalCensoredForAuthor = template.Must( template.New("proposalCensoredToAuthor").Parse(proposalCensoredToAuthorText)) -type invoiceNotificationEmailData struct { - Username string - Month string - Year int +// Proposal comment submitted - Send to proposal author +type proposalCommentSubmitted struct { + Username string // Comment author username + Name string // Proposal name + Link string // Comment link } -type newUserEmailTemplateData struct { - Username string - Link string - Email string -} +const proposalCommentSubmittedText = ` +{{.Username}} has commented on your proposal! -type newInviteUserEmailTemplateData struct { - Email string - Link string -} +Proposal: {{.Name}} +Comment: {{.Link}} +` -type approveDCCUserEmailTemplateData struct { - Email string - Link string -} +var proposalCommentSubmittedTmpl = template.Must( + template.New("proposalCommentSubmitted").Parse(proposalCommentSubmittedText)) -type updateUserKeyEmailTemplateData struct { - Link string - PublicKey string - Email string +// Proposal comment reply - Send to parent comment author +type proposalCommentReply struct { + Username string // Comment author username + Name string // Proposal name + Link string // Comment link } -type resetPasswordEmailTemplateData struct { - Link string - Email string -} +const proposalCommentReplyText = ` +{{.Username}} has replied to your comment! -type userLockedResetPasswordEmailTemplateData struct { - Link string - Email string -} +Proposal: {{.Name}} +Comment: {{.Link}} +` -type userPasswordChangedTemplateData struct { - Email string -} +var proposalCommentReplyTmpl = template.Must( + template.New("proposalCommentReply").Parse(proposalCommentReplyText)) -type proposalVoteStartedTemplateData struct { +// Pi events +type proposalVoteAuthorizedTemplateData struct { Link string // GUI proposal details url Name string // Proposal name Username string // Author username + Email string // Author email } -type proposalVoteAuthorizedTemplateData struct { +const templateProposalVoteAuthorizedRaw = ` +Voting has been authorized for the following proposal on Politeia by {{.Username}} ({{.Email}}): + +{{.Name}} +{{.Link}} +` + +var templateProposalVoteAuthorized = template.Must( + template.New("proposal_vote_authorized_template"). + Parse(templateProposalVoteAuthorizedRaw)) + +type proposalVoteStartedTemplateData struct { Link string // GUI proposal details url Name string // Proposal name Username string // Author username - Email string // Author email } -type commentReplyOnProposalTemplateData struct { - Commenter string - ProposalName string - CommentLink string -} +const templateProposalVoteStartedRaw = ` +Voting has started for the following proposal on Politeia, authored by {{.Username}}: -type newInvoiceStatusUpdateTemplate struct { - Token string -} +{{.Name}} +{{.Link}} +` -type newDCCSubmittedTemplateData struct { - Link string -} +var templateProposalVoteStarted = template.Must( + template.New("proposal_vote_started_template"). + Parse(templateProposalVoteStartedRaw)) -type newDCCSupportOpposeTemplateData struct { - Link string +const templateProposalVoteStartedForAuthorRaw = ` +Voting has just started for your proposal on Politeia! + +{{.Name}} +{{.Link}} +` + +var templateProposalVoteStartedForAuthor = template.Must( + template.New("proposal_vote_started_for_author_template"). + Parse(templateProposalVoteStartedForAuthorRaw)) + +// User events +type newUserEmailTemplateData struct { + Username string + Link string + Email string } const templateNewUserEmailRaw = ` @@ -201,23 +204,11 @@ You are receiving this email because {{.Email}} was used to register for Politei If you did not perform this action, please ignore this email. ` -const templateResetPasswordEmailRaw = ` -Click the link below to continue resetting your password: - -{{.Link}} - -You are receiving this email because a password reset was initiated for {{.Email}} -on Politeia. If you did not perform this action, it is possible that your account has been -compromised. Please contact Politeia administrators through Matrix on the -#politeia:decred.org channel. -` - -const templateUserPasswordChangedRaw = ` -You are receiving this email to notify you that your password has changed for -{{.Email}} on Politeia. If you did not perform this action, it is possible that -your account has been compromised. Please contact Politeia administrators -through Matrix on the #politeia:decred.org channel for further instructions. -` +type updateUserKeyEmailTemplateData struct { + Link string + PublicKey string + Email string +} const templateUpdateUserKeyEmailRaw = ` Click the link below to verify your new identity: @@ -229,50 +220,53 @@ was generated for {{.Email}} on Politeia. If you did not perform this action, please contact Politeia administrators. ` -const templateUserLockedResetPasswordRaw = ` -Your account was locked due to too many login attempts. You need to reset your -password in order to unlock your account: +type resetPasswordEmailTemplateData struct { + Link string + Email string +} + +const templateResetPasswordEmailRaw = ` +Click the link below to continue resetting your password: {{.Link}} -You are receiving this email because someone made too many login attempts for -{{.Email}} on Politeia. If that was not you, please notify Politeia administrators. +You are receiving this email because a password reset was initiated for {{.Email}} +on Politeia. If you did not perform this action, it is possible that your account has been +compromised. Please contact Politeia administrators through Matrix on the +#politeia:decred.org channel. ` -const templateProposalVoteStartedRaw = ` -Voting has started for the following proposal on Politeia, authored by {{.Username}}: - -{{.Name}} -{{.Link}} -` +type userLockedResetPasswordEmailTemplateData struct { + Link string + Email string +} -const templateProposalVoteAuthorizedRaw = ` -Voting has been authorized for the following proposal on Politeia by {{.Username}} ({{.Email}}): +const templateUserLockedResetPasswordRaw = ` +Your account was locked due to too many login attempts. You need to reset your +password in order to unlock your account: -{{.Name}} {{.Link}} -` -const templateProposalVoteStartedForAuthorRaw = ` -Voting has just started for your proposal on Politeia! - -{{.Name}} -{{.Link}} +You are receiving this email because someone made too many login attempts for +{{.Email}} on Politeia. If that was not you, please notify Politeia administrators. ` -const templateCommentReplyOnProposalRaw = ` -{{.Commenter}} has commented on your proposal! +type userPasswordChangedTemplateData struct { + Email string +} -Proposal: {{.ProposalName}} -Comment: {{.CommentLink}} +const templateUserPasswordChangedRaw = ` +You are receiving this email to notify you that your password has changed for +{{.Email}} on Politeia. If you did not perform this action, it is possible that +your account has been compromised. Please contact Politeia administrators +through Matrix on the #politeia:decred.org channel for further instructions. ` -const templateCommentReplyOnCommentRaw = ` -{{.Commenter}} has replied to your comment! - -Proposal: {{.ProposalName}} -Comment: {{.CommentLink}} -` +// CMS events +type newInviteUserEmailTemplateData struct { + Email string + Link string +} const templateInviteNewUserEmailRaw = ` You are invited to join Decred as a contractor! To complete your registration, you will need to use the following link and register on the CMS site: @@ -283,6 +277,11 @@ You are receiving this email because {{.Email}} was used to be invited to Decred If you do not recognize this, please ignore this email. ` +type approveDCCUserEmailTemplateData struct { + Email string + Link string +} + const templateApproveDCCUserEmailRaw = ` Congratulations! Your Decred Contractor Clearance Proposal has been approved! @@ -293,18 +292,9 @@ You are receiving this email because {{.Email}} was used to be invited to Decred If you do not recognize this, please ignore this email. ` -const templateInvoiceNotificationRaw = ` -{{.Username}}, - -You have not yet submitted an invoice for {{.Month}} {{.Year}}. Please do so as soon as possible, so your invoice may be reviewed and paid out in a timely manner. - -Regards, -Contractor Management System -` - -const templateNewInvoiceCommentRaw = ` -An administrator has submitted a new comment to your invoice, please login to cms.decred.org to view the message. -` +type newInvoiceStatusUpdateTemplate struct { + Token string +} const templateNewInvoiceStatusUpdateRaw = ` An invoice's status has been updated, please login to cms.decred.org to review the changes. @@ -315,6 +305,10 @@ Regards, Contractor Management System ` +type newDCCSubmittedTemplateData struct { + Link string +} + const templateNewDCCSubmittedRaw = ` A new DCC has been submitted. @@ -324,6 +318,10 @@ Regards, Contractor Management System ` +type newDCCSupportOpposeTemplateData struct { + Link string +} + const templateNewDCCSupportOpposeRaw = ` A DCC has received new support or opposition. @@ -332,3 +330,22 @@ A DCC has received new support or opposition. Regards, Contractor Management System ` + +type invoiceNotificationEmailData struct { + Username string + Month string + Year int +} + +const templateInvoiceNotificationRaw = ` +{{.Username}}, + +You have not yet submitted an invoice for {{.Month}} {{.Year}}. Please do so as soon as possible, so your invoice may be reviewed and paid out in a timely manner. + +Regards, +Contractor Management System +` + +const templateNewInvoiceCommentRaw = ` +An administrator has submitted a new comment to your invoice, please login to cms.decred.org to view the message. +` From f7106cbdc20083bacee277e4b55d1c9c5054f453 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 20 Sep 2020 15:01:55 -0500 Subject: [PATCH 077/449] typo --- politeiawww/eventmanager.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/politeiawww/eventmanager.go b/politeiawww/eventmanager.go index d7a8ae90d..2cfe2f72a 100644 --- a/politeiawww/eventmanager.go +++ b/politeiawww/eventmanager.go @@ -331,14 +331,6 @@ func (p *politeiawww) handleEventProposalStatusChange(ch chan interface{}) { } } -type dataProposalComment struct { - state pi.PropStateT - token string - commentID uint32 - parentID uint32 - username string // Comment author username -} - func (p *politeiawww) notifyProposalAuthorOnComment(d dataProposalComment) error { // Lookup proposal author to see if they should be sent a // notification. @@ -413,6 +405,14 @@ func (p *politeiawww) notifyParentAuthorOnComment(d dataProposalComment) error { proposalName(*pr), author.Email) } +type dataProposalComment struct { + state pi.PropStateT + token string + commentID uint32 + parentID uint32 + username string // Comment author username +} + func (p *politeiawww) handleEventProposalComment(ch chan interface{}) { for msg := range ch { d, ok := msg.(dataProposalComment) From a7b3527ba978cd365513d4616360bf451788d796 Mon Sep 17 00:00:00 2001 From: Amir Massarwa Date: Sun, 20 Sep 2020 23:47:52 +0300 Subject: [PATCH 078/449] politeiawww: Add processComments. --- politeiawww/cmd/piwww/commentnew.go | 29 +++++- politeiawww/cmd/piwww/commentvote.go | 28 ++++- politeiawww/cmd/piwww/piwww.go | 2 +- politeiawww/cmd/piwww/proposalcomments.go | 63 ++++++----- politeiawww/cmd/piwww/proposaledit.go | 31 +++--- politeiawww/cmd/piwww/proposalsetstatus.go | 10 +- politeiawww/cmd/shared/client.go | 21 ++-- politeiawww/comments.go | 23 ++++ politeiawww/pi.go | 8 +- politeiawww/piwww.go | 116 ++++++++++++++++++++- 10 files changed, 265 insertions(+), 66 deletions(-) diff --git a/politeiawww/cmd/piwww/commentnew.go b/politeiawww/cmd/piwww/commentnew.go index d395c23d6..44192cfb3 100644 --- a/politeiawww/cmd/piwww/commentnew.go +++ b/politeiawww/cmd/piwww/commentnew.go @@ -6,6 +6,7 @@ package main import ( "encoding/hex" + "fmt" "strconv" pi "github.com/decred/politeia/politeiawww/api/pi/v1" @@ -19,6 +20,10 @@ type CommentNewCmd struct { Comment string `positional-arg-name:"comment" required:"true"` // Comment text ParentID string `positional-arg-name:"parentID"` // Comment parent ID } `positional-args:"true"` + + // CLI flags + Vetted bool `long:"vetted" optional:"true"` + Unvetted bool `long:"unvetted" optional:"true"` } // Execute executes the new comment command. @@ -27,13 +32,27 @@ func (cmd *CommentNewCmd) Execute(args []string) error { comment := cmd.Args.Comment parentID := cmd.Args.ParentID + // Verify state + var state pi.PropStateT + switch { + case cmd.Vetted && cmd.Unvetted: + return fmt.Errorf("cannot use --vetted and --unvetted simultaneously") + case cmd.Unvetted: + state = pi.PropStateUnvetted + case cmd.Vetted: + state = pi.PropStateVetted + default: + return fmt.Errorf("must specify either --vetted or unvetted") + } + // Check for user identity if cfg.Identity == nil { return shared.ErrUserIdentityNotFound } // Setup new comment request - sig := cfg.Identity.SignMessage([]byte(token + parentID + comment)) + sig := cfg.Identity.SignMessage([]byte(string(state) + token + parentID + + comment)) // Parse provided parent id piUint, err := strconv.ParseUint(parentID, 10, 32) if err != nil { @@ -41,6 +60,7 @@ func (cmd *CommentNewCmd) Execute(args []string) error { } cn := pi.CommentNew{ Token: token, + State: state, ParentID: uint32(piUint), Comment: comment, Signature: hex.EncodeToString(sig[:]), @@ -72,4 +92,9 @@ Comment on proposal as logged in user. Arguments: 1. token (string, required) Proposal censorship token 2. comment (string, required) Comment -3. parentID (string, required if replying to comment) Id of commment` +3. parentID (string, required if replying to comment) Id of commment + +Flags: + --vetted (bool, optional) Comment on vetted record. + --unvetted (bool, optional) Comment on unvetted reocrd. +` diff --git a/politeiawww/cmd/piwww/commentvote.go b/politeiawww/cmd/piwww/commentvote.go index d8f5baf36..45eb89bf7 100644 --- a/politeiawww/cmd/piwww/commentvote.go +++ b/politeiawww/cmd/piwww/commentvote.go @@ -21,6 +21,10 @@ type CommentVoteCmd struct { CommentID string `positional-arg-name:"commentID"` // Comment ID Action string `positional-arg-name:"action"` // Upvote/downvote action } `positional-args:"true" required:"true"` + + // CLI flags + Vetted bool `long:"vetted" optional:"true"` + Unvetted bool `long:"unvetted" optional:"true"` } // Execute executes the like comment command. @@ -32,6 +36,19 @@ func (cmd *CommentVoteCmd) Execute(args []string) error { commentID := cmd.Args.CommentID action := cmd.Args.Action + // Verify state + var state pi.PropStateT + switch { + case cmd.Vetted && cmd.Unvetted: + return fmt.Errorf("cannot use --vetted and --unvetted simultaneously") + case cmd.Unvetted: + state = pi.PropStateUnvetted + case cmd.Vetted: + state = pi.PropStateVetted + default: + return fmt.Errorf("must specify either --vetted or unvetted") + } + // Validate action if action != actionUpvote && action != actionDownvote { return fmt.Errorf("invalid action %s; the action must be either "+ @@ -52,7 +69,8 @@ func (cmd *CommentVoteCmd) Execute(args []string) error { vote = pi.CommentVoteDownvote } - sig := cfg.Identity.SignMessage([]byte(token + commentID + string(vote))) + sig := cfg.Identity.SignMessage([]byte(string(state) + token + commentID + + string(vote))) // Parse provided parent id ciUint, err := strconv.ParseUint(commentID, 10, 32) if err != nil { @@ -60,6 +78,7 @@ func (cmd *CommentVoteCmd) Execute(args []string) error { } cv := &pi.CommentVote{ Token: token, + State: state, CommentID: uint32(ciUint), Vote: vote, Signature: hex.EncodeToString(sig[:]), @@ -91,4 +110,9 @@ Vote on a comment. Arguments: 1. token (string, required) Proposal censorship token 2. commentID (string, required) Id of the comment -3. action (string, required) Vote (upvote or downvote)` +3. action (string, required) Vote (upvote or downvote) + +Flags: + --vetted (bool, optional) Comment's record is vetted. + --unvetted (bool, optional) Comment's record is unvetted. +` diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index 8bb3ab159..8d942f258 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -52,6 +52,7 @@ type piwww struct { // Comments commands CommentNew CommentNewCmd `command:"commentnew" description:"(user) create a new comment"` CommentVote CommentVoteCmd `command:"commentvote" description:"(user) upvote/downvote a comment"` + Comments CommentsCmd `command:"comments" description:"(public) get the comments for a proposal"` // Commands ActiveVotes ActiveVotesCmd `command:"activevotes" description:"(public) get the proposals that are being voted on"` @@ -70,7 +71,6 @@ type piwww struct { Me shared.MeCmd `command:"me" description:"(user) get user details for the logged in user"` NewUser NewUserCmd `command:"newuser" description:"(public) create a new user"` Policy PolicyCmd `command:"policy" description:"(public) get the server policy"` - ProposalComments ProposalCommentsCmd `command:"proposalcomments" description:"(public) get the comments for a proposal"` ProposalDetails ProposalDetailsCmd `command:"proposaldetails" description:"(public) get the details of a proposal"` ProposalPaywall ProposalPaywallCmd `command:"proposalpaywall" description:"(user) get proposal paywall details for the logged in user"` RescanUserPayments RescanUserPaymentsCmd `command:"rescanuserpayments" description:"(admin) rescan a user's payments to check for missed payments"` diff --git a/politeiawww/cmd/piwww/proposalcomments.go b/politeiawww/cmd/piwww/proposalcomments.go index 835c4fe6d..9d1eed4b8 100644 --- a/politeiawww/cmd/piwww/proposalcomments.go +++ b/politeiawww/cmd/piwww/proposalcomments.go @@ -4,21 +4,49 @@ package main -import "github.com/decred/politeia/politeiawww/cmd/shared" +import ( + "fmt" -// ProposalCommentsCmd retreives the comments for the specified proposal. -type ProposalCommentsCmd struct { + pi "github.com/decred/politeia/politeiawww/api/pi/v1" + "github.com/decred/politeia/politeiawww/cmd/shared" +) + +// CommentsCmd retreives the comments for the specified proposal. +type CommentsCmd struct { Args struct { Token string `positional-arg-name:"token"` // Censorship token } `positional-args:"true" required:"true"` + + // CLI flags + Vetted bool `long:"vetted" optional:"true"` + Unvetted bool `long:"unvetted" optional:"true"` } // Execute executes the proposal comments command. -func (cmd *ProposalCommentsCmd) Execute(args []string) error { - gcr, err := client.GetComments(cmd.Args.Token) +func (cmd *CommentsCmd) Execute(args []string) error { + token := cmd.Args.Token + + // Verify state + var state pi.PropStateT + switch { + case cmd.Vetted && cmd.Unvetted: + return fmt.Errorf("cannot use --vetted and --unvetted simultaneously") + case cmd.Unvetted: + state = pi.PropStateUnvetted + case cmd.Vetted: + state = pi.PropStateVetted + default: + return fmt.Errorf("must specify either --vetted or unvetted") + } + + gcr, err := client.Comments(pi.Comments{ + Token: token, + State: state, + }) if err != nil { return err } + return shared.PrintJSON(gcr) } @@ -31,24 +59,7 @@ Get the comments for a proposal. Arguments: 1. token (string, required) Proposal censorship token -Result: -{ - "comments": [ - { - "token": (string) Censorship token - "parentid": (string) Id of comment (defaults to '0' (top-level)) - "comment": (string) Comment - "signature": (string) Signature of token+parentID+comment - "publickey": (string) Public key of user - "commentid": (string) Id of the comment - "receipt": (string) Server signature of the comment signature - "timestamp": (int64) Received UNIX timestamp - "resultvotes": (int64) Vote score - "upvotes": (uint64) Pro votes - "downvotes": (uint64) Contra votes - "censored": (bool) If comment has been censored - "userid": (string) User id - "username": (string) Username - } - ] -}` +Flags: + --vetted (bool, optional) Comment on vetted record. + --unvetted (bool, optional) Comment on unvetted reocrd. +` diff --git a/politeiawww/cmd/piwww/proposaledit.go b/politeiawww/cmd/piwww/proposaledit.go index e4f6b9526..13c04aa1a 100644 --- a/politeiawww/cmd/piwww/proposaledit.go +++ b/politeiawww/cmd/piwww/proposaledit.go @@ -246,18 +246,19 @@ Arguments: 3. attachments (string, optional) Attachment files Flags: - --random (bool, optional) Generate a random proposal name & files to - submit. If this flag is used then the markdown - file argument is no longer required and any - provided files will be ignored. - --usemd (bool, optional) Use the existing proposal metadata. - --name (string, optional) The name of the proposal. - --linkto (string, optional) Censorship token of an existing public proposal - to link to. - --linkby (int64, optional) UNIX timestamp of RFP deadline. - --rfp (bool, optional) Make the proposal an RFP by inserting a LinkBy - timestamp into the proposal metadata. The LinkBy - timestamp is set to be one month from the - current time. This is intended to be used in - place of --linkby. -` + --vetted (bool, optional) Comment on vetted record. + --unvetted (bool, optional) Comment on unvetted reocrd. + --random (bool, optional) Generate a random proposal name & files to + submit. If this flag is used then the markdown + file argument is no longer required and any + provided files will be ignored. + --usemd (bool, optional) Use the existing proposal metadata. + --name (string, optional) The name of the proposal. + --linkto (string, optional) Censorship token of an existing public proposal + to link to. + --linkby (int64, optional) UNIX timestamp of RFP deadline. + --rfp (bool, optional) Make the proposal an RFP by inserting a LinkBy + timestamp into the proposal metadata. The LinkBy + timestamp is set to be one month from the + current time. This is intended to be used in + place of --linkby.` diff --git a/politeiawww/cmd/piwww/proposalsetstatus.go b/politeiawww/cmd/piwww/proposalsetstatus.go index 540640116..a8409b2a9 100644 --- a/politeiawww/cmd/piwww/proposalsetstatus.go +++ b/politeiawww/cmd/piwww/proposalsetstatus.go @@ -117,7 +117,11 @@ Valid statuses: abandoned Arguments: -1. token (string, required) Proposal censorship token -2. status (string, required) New status -3. message (string, optional) Status change message +1. token (string, required) Proposal censorship token +2. status (string, required) New status +3. message (string, optional) Status change message + +Flags: + --vetted (bool, optional) Set status of a vetted record. + --unvetted (bool, optional) Set status of an unvetted reocrd. ` diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index a827e1cbc..f6d1d8c56 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -1182,32 +1182,31 @@ func (c *Client) CommentNew(cn pi.CommentNew) (*pi.CommentNewReply, error) { return &cnr, nil } -// GetComments retrieves the comments for the specified proposal. -func (c *Client) GetComments(token string) (*www.GetCommentsReply, error) { - route := "/proposals/" + token + "/comments" - responseBody, err := c.makeRequest(http.MethodGet, - www.PoliteiaWWWAPIRoute, route, nil) +// Comments retrieves the comments for the specified proposal. +func (c *Client) Comments(cs pi.Comments) (*pi.CommentsReply, error) { + responseBody, err := c.makeRequest(http.MethodPost, + pi.APIRoute, pi.RouteComments, &cs) if err != nil { return nil, err } - var gcr www.GetCommentsReply - err = json.Unmarshal(responseBody, &gcr) + var cr pi.CommentsReply + err = json.Unmarshal(responseBody, &cr) if err != nil { - return nil, fmt.Errorf("unmarshal GetCommentsReply: %v", err) + return nil, fmt.Errorf("unmarshal CommentsReply: %v", err) } if c.cfg.Verbose { - err := prettyPrintJSON(gcr) + err := prettyPrintJSON(cr) if err != nil { return nil, err } } - return &gcr, nil + return &cr, nil } -// GetComments retrieves the comments for the specified proposal. +// InvoiceComments retrieves the comments for the specified proposal. func (c *Client) InvoiceComments(token string) (*www.GetCommentsReply, error) { route := "/invoices/" + token + "/comments" responseBody, err := c.makeRequest(http.MethodGet, diff --git a/politeiawww/comments.go b/politeiawww/comments.go index 6410c2449..b0445b8ae 100644 --- a/politeiawww/comments.go +++ b/politeiawww/comments.go @@ -5,10 +5,33 @@ package main import ( + "github.com/decred/politeia/plugins/comments" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/user" ) +// comments call the comments plugin to get record's comments. +func (p *politeiawww) comments(cp comments.GetAll) (*comments.GetAllReply, error) { + // Prep plugin payload + payload, err := comments.EncodeGetAll(cp) + if err != nil { + return nil, err + } + + r, err := p.pluginCommand(comments.ID, comments.CmdGetAll, "", + string(payload)) + if err != nil { + return nil, err + } + cr, err := comments.DecodeGetAllReply([]byte(r)) + if err != nil { + return nil, err + } + + return cr, nil + +} + func validateComment(c www.NewComment) error { // max length if len(c.Comment) > www.PolicyMaxCommentLength { diff --git a/politeiawww/pi.go b/politeiawww/pi.go index 4ace7f15b..7779a2672 100644 --- a/politeiawww/pi.go +++ b/politeiawww/pi.go @@ -9,9 +9,9 @@ import ( ) // piCommentVote calls the pi plugin to vote on a comment. -func (p *politeiawww) piCommentVote(cvp *piplugin.CommentVote) (*piplugin.CommentVoteReply, error) { +func (p *politeiawww) piCommentVote(cvp piplugin.CommentVote) (*piplugin.CommentVoteReply, error) { // Prep comment vote payload - payload, err := piplugin.EncodeCommentVote(*cvp) + payload, err := piplugin.EncodeCommentVote(cvp) if err != nil { return nil, err } @@ -30,9 +30,9 @@ func (p *politeiawww) piCommentVote(cvp *piplugin.CommentVote) (*piplugin.Commen } // piCommentNew calls the pi plugin to add new comment. -func (p *politeiawww) piCommentNew(cnp *piplugin.CommentNew) (*piplugin.CommentNewReply, error) { +func (p *politeiawww) piCommentNew(cnp piplugin.CommentNew) (*piplugin.CommentNewReply, error) { // Prep new comment payload - payload, err := piplugin.EncodeCommentNew(*cnp) + payload, err := piplugin.EncodeCommentNew(cnp) if err != nil { return nil, err } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index dc8ceca32..de9e03444 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -17,6 +17,7 @@ import ( "strconv" "time" + "github.com/decred/politeia/plugins/comments" piplugin "github.com/decred/politeia/plugins/pi" pd "github.com/decred/politeia/politeiad/api/v1" pi "github.com/decred/politeia/politeiawww/api/pi/v1" @@ -24,6 +25,7 @@ import ( "github.com/decred/politeia/politeiawww/user" wwwutil "github.com/decred/politeia/politeiawww/util" "github.com/decred/politeia/util" + "github.com/google/uuid" ) // TODO use pi policies. Should the policies be defined in the pi plugin @@ -243,6 +245,46 @@ func convertCensorshipRecordFromPD(cr pd.CensorshipRecord) pi.CensorshipRecord { } } +func convertCommentsPluginPropStateFromPi(s pi.PropStateT) comments.StateT { + switch s { + case pi.PropStateUnvetted: + return comments.StateUnvetted + case pi.PropStateVetted: + return comments.StateVetted + } + return comments.StateInvalid +} + +func convertPropStateFromComments(s comments.StateT) pi.PropStateT { + switch s { + case comments.StateUnvetted: + return pi.PropStateUnvetted + case comments.StateVetted: + return pi.PropStateVetted + } + return pi.PropStateInvalid +} + +func convertCommentFromPlugin(cm comments.Comment) pi.Comment { + return pi.Comment{ + UserID: cm.UUID, + Username: "", // Intentionally omitted, needs to be pulled from userdb + State: convertPropStateFromComments(cm.State), + Token: cm.Token, + ParentID: cm.ParentID, + Comment: cm.Comment, + PublicKey: cm.PublicKey, + Signature: cm.Signature, + CommentID: cm.CommentID, + Version: cm.Version, + Timestamp: cm.Timestamp, + Receipt: cm.Receipt, + Score: cm.Score, + Deleted: cm.Deleted, + Reason: cm.Reason, + } +} + func convertFilesFromPD(f []pd.File) ([]pi.File, []pi.Metadata) { files := make([]pi.File, 0, len(f)) metadata := make([]pi.Metadata, 0, len(f)) @@ -1393,7 +1435,7 @@ func (p *politeiawww) processCommentNew(cn pi.CommentNew, usr user.User) (*pi.Co } // Call pi plugin to add new comment - reply, err := p.piCommentNew(&piplugin.CommentNew{ + reply, err := p.piCommentNew(piplugin.CommentNew{ UUID: usr.ID.String(), Token: cn.Token, ParentID: cn.ParentID, @@ -1406,6 +1448,16 @@ func (p *politeiawww) processCommentNew(cn pi.CommentNew, usr user.User) (*pi.Co return nil, err } + // Emit event notification for a proposal comment + p.eventManager.emit(eventProposalComment, + dataProposalComment{ + state: cn.State, + token: cn.Token, + username: usr.Username, + parentID: cn.ParentID, + commentID: reply.CommentID, + }) + return &pi.CommentNewReply{ CommentID: reply.CommentID, Timestamp: reply.Timestamp, @@ -1475,7 +1527,7 @@ func (p *politeiawww) processCommentVote(cv pi.CommentVote, usr user.User) (*pi. } // Call pi plugin to add new comment - reply, err := p.piCommentVote(&piplugin.CommentVote{ + reply, err := p.piCommentVote(piplugin.CommentVote{ UUID: usr.ID.String(), Token: cv.Token, CommentID: cv.CommentID, @@ -1523,12 +1575,72 @@ func (p *politeiawww) handleCommentVote(w http.ResponseWriter, r *http.Request) util.RespondWithJSON(w, http.StatusOK, vcr) } +func (p *politeiawww) processComments(c pi.Comments) (*pi.CommentsReply, error) { + log.Tracef("processComments: %v", c.Token) + + // Call comments plugin to get comments + reply, err := p.comments(comments.GetAll{ + Token: c.Token, + State: convertCommentsPluginPropStateFromPi(c.State), + }) + if err != nil { + return nil, err + } + + var cr pi.CommentsReply + // Transalte comments + cs := make([]pi.Comment, 0, len(reply.Comments)) + for _, cm := range reply.Comments { + // Convert comment to pi + pic := convertCommentFromPlugin(cm) + // Get comment's author username + // Parse string uuid + uuid, err := uuid.Parse(cm.UUID) + if err != nil { + return nil, err + } + // Get user + u, err := p.db.UserGetById(uuid) + if err != nil { + return nil, err + } + pic.Username = u.Username + cs = append(cs, pic) + } + cr.Comments = cs + + return &cr, nil +} + +func (p *politeiawww) handleComments(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleComments") + + var c pi.Comments + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&c); err != nil { + respondWithPiError(w, r, "handleComments: unmarshal", + pi.UserErrorReply{}) + return + } + + cr, err := p.processComments(c) + if err != nil { + respondWithPiError(w, r, + "handleCommentVote: processComments: %v", err) + } + + util.RespondWithJSON(w, http.StatusOK, cr) +} + func (p *politeiawww) setPiRoutes() { // Public routes p.addRoute(http.MethodGet, pi.APIRoute, pi.RouteProposalInventory, p.handleProposalInventory, permissionPublic) + p.addRoute(http.MethodPost, pi.APIRoute, + pi.RouteComments, p.handleComments, permissionPublic) + // Logged in routes p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteProposalNew, p.handleProposalNew, From 1e6eeb69d94fdb4b6a861da3394180a26b6489a7 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 20 Sep 2020 16:03:47 -0500 Subject: [PATCH 079/449] cleanup events and templates --- politeiawww/email.go | 289 ++++++++++++++++++------------------ politeiawww/eventmanager.go | 116 ++++++++------- politeiawww/smtp.go | 3 + politeiawww/templates.go | 53 +++---- 4 files changed, 233 insertions(+), 228 deletions(-) diff --git a/politeiawww/email.go b/politeiawww/email.go index 11a8ee272..908d37ac5 100644 --- a/politeiawww/email.go +++ b/politeiawww/email.go @@ -56,73 +56,58 @@ func (p *politeiawww) createEmailLink(path, email, token, username string) (stri return l.String(), nil } -// emailNewUserVerificationLink emails the link with the new user verification -// token if the email server is set up. -func (p *politeiawww) emailNewUserVerificationLink(email, token, username string) error { - link, err := p.createEmailLink(www.RouteVerifyNewUser, email, - token, username) +// emailProposalSubmitted send a proposal submitted notification email to +// the provided list of emails. +func (p *politeiawww) emailProposalSubmitted(token, name, username string, emails []string) error { + route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) + l, err := url.Parse(p.cfg.WebServerAddress + route) if err != nil { return err } - tplData := newUserEmailTemplateData{ + tmplData := proposalSubmitted{ Username: username, - Email: email, - Link: link, + Name: name, + Link: l.String(), } - subject := "Verify Your Email" - body, err := createBody(templateNewUserEmail, tplData) + subject := "New Proposal Submitted" + body, err := createBody(proposalSubmittedTmpl, tmplData) if err != nil { return err } - recipients := []string{email} - - return p.smtp.sendEmailTo(subject, body, recipients) -} - -func (p *politeiawww) newVerificationURL(route, token string) (*url.URL, error) { - u, err := url.Parse(p.cfg.WebServerAddress + route) - if err != nil { - return nil, err - } - q := u.Query() - q.Set("verificationtoken", token) - u.RawQuery = q.Encode() - - return u, nil + return p.smtp.sendEmailTo(subject, body, emails) } -// emailResetPasswordVerificationLink emails the link with the reset password -// verification token if the email server is set up. -func (p *politeiawww) emailResetPasswordVerificationLink(email, username, token string) error { - u, err := p.newVerificationURL(www.RouteResetPassword, token) +// emailProposalEdited sends a proposal edited notification email to the +// provided list of emails. +func (p *politeiawww) emailProposalEdited(name, username, token, version string, emails []string) error { + route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) + l, err := url.Parse(p.cfg.WebServerAddress + route) if err != nil { return err } - q := u.Query() - q.Set("username", username) - u.RawQuery = q.Encode() - tplData := resetPasswordEmailTemplateData{ - Email: email, - Link: u.String(), + tmplData := proposalEdited{ + Name: name, + Version: version, + Username: username, + Link: l.String(), } - subject := "Reset Your Password" - body, err := createBody(templateResetPasswordEmail, tplData) + subject := "Proposal Edited" + body, err := createBody(proposalEditedTmpl, tmplData) if err != nil { return err } - recipients := []string{email} - return p.smtp.sendEmailTo(subject, body, recipients) + return p.smtp.sendEmailTo(subject, body, emails) } -// emailAuthorProposalStatusChange sends an email to the author of the proposal -// notifying them of the proposal status change. -func (p *politeiawww) emailProposalStatusChangeToAuthor(d dataProposalStatusChange, proposalName, authorEmail string) error { +// emailProposalStatusChange sends a proposal status change email to the +// provided email addresses. +func (p *politeiawww) emailProposalStatusChange(d dataProposalStatusChange, proposalName string, emails []string) error { route := strings.Replace(guiRouteProposalDetails, "{token}", d.token, 1) l, err := url.Parse(p.cfg.WebServerAddress + route) if err != nil { @@ -135,37 +120,26 @@ func (p *politeiawww) emailProposalStatusChangeToAuthor(d dataProposalStatusChan ) switch d.status { case pi.PropStatusPublic: - subject = "Your Proposal Has Been Published" - tmplData := proposalVettedToAuthor{ + subject = "New Proposal Published" + tmplData := proposalVetted{ Name: proposalName, Link: l.String(), } - body, err = createBody(proposalVettedToAuthorTmpl, tmplData) - if err != nil { - return err - } - - case pi.PropStatusCensored: - subject = "Your Proposal Has Been Censored" - tmplData := proposalCensoredToAuthor{ - Name: proposalName, - Reason: d.reason, - } - body, err = createBody(tmplProposalCensoredForAuthor, tmplData) + body, err = createBody(tmplProposalVetted, tmplData) if err != nil { return err } default: - return fmt.Errorf("no author notification for prop status %v", d.status) + return fmt.Errorf("no user notification for prop status %v", d.status) } - return p.smtp.sendEmailTo(subject, body, []string{authorEmail}) + return p.smtp.sendEmailTo(subject, body, emails) } -// emailProposalStatusChangeToUsers sends an email to the provided users -// notifying them of the proposal status change. -func (p *politeiawww) emailProposalStatusChangeToUsers(d dataProposalStatusChange, proposalName string, emails []string) error { +// emailProposalStatusChangeAuthor sends a proposal status change notification +// email to the provided email address. +func (p *politeiawww) emailProposalStatusChangeToAuthor(d dataProposalStatusChange, proposalName, authorEmail string) error { route := strings.Replace(guiRouteProposalDetails, "{token}", d.token, 1) l, err := url.Parse(p.cfg.WebServerAddress + route) if err != nil { @@ -178,194 +152,215 @@ func (p *politeiawww) emailProposalStatusChangeToUsers(d dataProposalStatusChang ) switch d.status { case pi.PropStatusPublic: - subject = "New Proposal Published" - tmplData := proposalVetted{ + subject = "Your Proposal Has Been Published" + tmplData := proposalVettedToAuthor{ Name: proposalName, Link: l.String(), } - body, err = createBody(tmplProposalVetted, tmplData) + body, err = createBody(proposalVettedToAuthorTmpl, tmplData) + if err != nil { + return err + } + + case pi.PropStatusCensored: + subject = "Your Proposal Has Been Censored" + tmplData := proposalCensoredToAuthor{ + Name: proposalName, + Reason: d.reason, + } + body, err = createBody(tmplProposalCensoredForAuthor, tmplData) if err != nil { return err } default: - return fmt.Errorf("no user notification for prop status %v", d.status) + return fmt.Errorf("no author notification for prop status %v", d.status) } - return p.smtp.sendEmailTo(subject, body, emails) + return p.smtp.sendEmailTo(subject, body, []string{authorEmail}) } -// emailProposalEdited sends email regarding the proposal edits event. -// Sends to all users with this notification bit turned on. -func (p *politeiawww) emailProposalEdited(name, username, token, version string, emails []string) error { - route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) +// emailProposalCommentSubmitted sends a proposal comment submitted email to +// the provided email address. +func (p *politeiawww) emailProposalCommentSubmitted(token, commentID, commentUsername, proposalName, proposalAuthorEmail string) error { + // Setup comment URL + route := strings.Replace(guirouteProposalComments, "{token}", token, 1) + route = strings.Replace(route, "{id}", commentID, 1) l, err := url.Parse(p.cfg.WebServerAddress + route) if err != nil { return err } - tmplData := proposalEdited{ - Name: name, - Version: version, - Username: username, + // Setup email + subject := "New Comment On Your Proposal" + tmplData := proposalCommentSubmitted{ + Username: commentUsername, + Name: proposalName, Link: l.String(), } - - subject := "Proposal Edited" - body, err := createBody(proposalEditedTmpl, tmplData) + body, err := createBody(proposalCommentSubmittedTmpl, tmplData) if err != nil { return err } - return p.smtp.sendEmailTo(subject, body, emails) + // Send email + return p.smtp.sendEmailTo(subject, body, []string{proposalAuthorEmail}) } -// emailProposalVoteStarted sends email to the proposal author for the -// proposal vote started event. -func (p *politeiawww) emailProposalVoteStartedToAuthor(token, name, username, email string) error { - route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) +// emailProposalCommentReply sends a proposal comment reply email to the +// provided email address. +func (p *politeiawww) emailProposalCommentReply(token, commentID, commentUsername, proposalName, parentCommentEmail string) error { + // Setup comment URL + route := strings.Replace(guirouteProposalComments, "{token}", token, 1) + route = strings.Replace(route, "{id}", commentID, 1) l, err := url.Parse(p.cfg.WebServerAddress + route) if err != nil { return err } - tplData := proposalVoteStartedTemplateData{ + // Setup email + subject := "New Reply To Your Comment" + tmplData := proposalCommentReply{ + Username: commentUsername, + Name: proposalName, Link: l.String(), - Name: name, - Username: username, } - - subject := "Your Proposal Has Started Voting" - body, err := createBody(templateProposalVoteStartedForAuthor, tplData) + body, err := createBody(proposalCommentReplyTmpl, tmplData) if err != nil { return err } - recipients := []string{email} - return p.smtp.sendEmailTo(subject, body, recipients) + // Send email + return p.smtp.sendEmailTo(subject, body, []string{parentCommentEmail}) } -// emailProposalVoteStartedToUsers sends email to users with this notification -// bit set on for the proposal vote started event. -func (p *politeiawww) emailProposalVoteStartedToUsers(token, name, username string, emails []string) error { +// emailProposalVoteAuthorized sends a proposal vote authorized email to the +// provided list of emails. +func (p *politeiawww) emailProposalVoteAuthorized(token, name, username string, emails []string) error { + // Setup URL route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) l, err := url.Parse(p.cfg.WebServerAddress + route) if err != nil { return err } - tplData := proposalVoteStartedTemplateData{ - Link: l.String(), - Name: name, + // Setup email + subject := "Proposal Authorized To Start Voting" + tplData := proposalVoteAuthorized{ Username: username, + Name: name, + Link: l.String(), } - - subject := "Voting Started for Proposal" - body, err := createBody(templateProposalVoteStarted, tplData) + body, err := createBody(proposalVoteAuthorizedTmpl, tplData) if err != nil { return err } + // Send email return p.smtp.sendEmailTo(subject, body, emails) } -// emailProposalSubmitted sends email notification for a new proposal becoming -// vetted. Sends to the author and for users with this notification setting. -func (p *politeiawww) emailProposalSubmitted(token, name, username string, emails []string) error { +// emailProposalVoteStarted sends a proposal vote started email notification +// to the provided email addresses. +func (p *politeiawww) emailProposalVoteStarted(token, name string, emails []string) error { + // Setup URL route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) l, err := url.Parse(p.cfg.WebServerAddress + route) if err != nil { return err } - tmplData := proposalSubmitted{ - Username: username, - Name: name, - Link: l.String(), + // Setup email + subject := "Voting Started for Proposal" + tplData := proposalVoteStarted{ + Name: name, + Link: l.String(), } - - subject := "New Proposal Submitted" - body, err := createBody(proposalSubmittedTmpl, tmplData) + body, err := createBody(proposalVoteStartedTmpl, tplData) if err != nil { return err } + // Send email return p.smtp.sendEmailTo(subject, body, emails) } -// emailProposalVoteAuthorized sends email notification for the proposal vote -// authorized event. Sends to all admins with this notification bit set on. -func (p *politeiawww) emailProposalVoteAuthorized(token, name, username, email string, emails []string) error { +// emailProposalVoteStartedToAuthor sends a proposal vote started email to +// the provided email address. +func (p *politeiawww) emailProposalVoteStartedToAuthor(token, name, username, email string) error { + // Setup URL route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) l, err := url.Parse(p.cfg.WebServerAddress + route) if err != nil { return err } - tplData := proposalVoteAuthorizedTemplateData{ - Link: l.String(), - Name: name, - Username: username, - Email: email, + // Setup email + subject := "Your Proposal Has Started Voting" + tplData := proposalVoteStartedToAuthor{ + Name: name, + Link: l.String(), } - - subject := "Proposal Authorized To Start Voting" - body, err := createBody(templateProposalVoteAuthorized, tplData) + body, err := createBody(proposalVoteStartedToAuthorTmpl, tplData) if err != nil { return err } - return p.smtp.sendEmailTo(subject, body, emails) + // Send email + return p.smtp.sendEmailTo(subject, body, []string{email}) } -func (p *politeiawww) emailProposalCommentSubmitted(token, commentID, commentUsername, proposalName, proposalAuthorEmail string) error { - // Setup comment URL - route := strings.Replace(guirouteProposalComments, "{token}", token, 1) - route = strings.Replace(route, "{id}", commentID, 1) - l, err := url.Parse(p.cfg.WebServerAddress + route) +// emailNewUserVerificationLink sends a new user verification email to the +// provided email address. +func (p *politeiawww) emailNewUserVerificationLink(email, token, username string) error { + link, err := p.createEmailLink(www.RouteVerifyNewUser, email, + token, username) if err != nil { return err } - // Setup email - subject := "New Comment On Your Proposal" - tmplData := proposalCommentSubmitted{ - Username: commentUsername, - Name: proposalName, - Link: l.String(), + tplData := newUserEmailTemplateData{ + Username: username, + Email: email, + Link: link, } - body, err := createBody(proposalCommentSubmittedTmpl, tmplData) + + subject := "Verify Your Email" + body, err := createBody(templateNewUserEmail, tplData) if err != nil { return err } + recipients := []string{email} - // Send email - return p.smtp.sendEmailTo(subject, body, []string{proposalAuthorEmail}) + return p.smtp.sendEmailTo(subject, body, recipients) } -func (p *politeiawww) emailProposalCommentReply(token, commentID, commentUsername, proposalName, parentCommentEmail string) error { - // Setup comment URL - route := strings.Replace(guirouteProposalComments, "{token}", token, 1) - route = strings.Replace(route, "{id}", commentID, 1) - l, err := url.Parse(p.cfg.WebServerAddress + route) +// emailResetPasswordVerificationLink emails the link with the reset password +// verification token if the email server is set up. +func (p *politeiawww) emailResetPasswordVerificationLink(email, username, token string) error { + // Setup URL + u, err := url.Parse(p.cfg.WebServerAddress + www.RouteResetPassword) if err != nil { return err } + q := u.Query() + q.Set("verificationtoken", token) + q.Set("username", username) + u.RawQuery = q.Encode() // Setup email - subject := "New Reply To Your Comment" - tmplData := proposalCommentReply{ - Username: commentUsername, - Name: proposalName, - Link: l.String(), + subject := "Reset Your Password" + tplData := resetPasswordEmailTemplateData{ + Email: email, + Link: u.String(), } - body, err := createBody(proposalCommentReplyTmpl, tmplData) + body, err := createBody(templateResetPasswordEmail, tplData) if err != nil { return err } // Send email - return p.smtp.sendEmailTo(subject, body, []string{parentCommentEmail}) + return p.smtp.sendEmailTo(subject, body, []string{email}) } // emailUpdateUserKeyVerificationLink emails the link with the verification diff --git a/politeiawww/eventmanager.go b/politeiawww/eventmanager.go index 2cfe2f72a..ed4e39710 100644 --- a/politeiawww/eventmanager.go +++ b/politeiawww/eventmanager.go @@ -179,14 +179,21 @@ func (p *politeiawww) handleEventProposalSubmitted(ch chan interface{}) { continue } - // Compile email notification recipients + // Compile a list of users to send the notification to emails := make([]string, 0, 256) err := p.db.AllUsers(func(u *user.User) { - // Check if user is able to receive notification - if userNotificationEnabled(*u, - www.NotificationEmailAdminProposalNew) { - emails = append(emails, u.Email) + switch { + case !u.Admin: + // Only admins get this notification + return + case !userNotificationEnabled(*u, + www.NotificationEmailAdminProposalNew): + // Admin doesn't have notification bit set + return } + + // Add user to notification list + emails = append(emails, u.Email) }) if err != nil { log.Errorf("handleEventProposalSubmitted: AllUsers: %v", err) @@ -219,7 +226,7 @@ func (p *politeiawww) handleEventProposalEdited(ch chan interface{}) { continue } - // Compile list of emails to send notification to + // Compile a list of users to send the notification to emails := make([]string, 0, 256) err := p.db.AllUsers(func(u *user.User) { // Check circumstances where we don't notify @@ -237,10 +244,11 @@ func (p *politeiawww) handleEventProposalEdited(ch chan interface{}) { emails = append(emails, u.Email) }) - err = p.emailProposalEdited(d.name, d.username, d.token, d.version, - emails) + err = p.emailProposalEdited(d.name, d.username, + d.token, d.version, emails) if err != nil { log.Errorf("emailProposalEdited: %v", err) + continue } log.Debugf("Sent proposal edited notifications %v", d.token) @@ -287,44 +295,42 @@ func (p *politeiawww) handleEventProposalStatusChange(ch chan interface{}) { continue } - // Compile list of emails to send notification to - emails := make([]string, 0, 256) + // Email author + proposalName := proposalName(*pr) notification := www.NotificationEmailRegularProposalVetted + if userNotificationEnabled(*author, notification) { + err = p.emailProposalStatusChangeToAuthor(d, proposalName, author.Email) + if err != nil { + log.Errorf("emailProposalStatusChangeToAuthor: %v", err) + continue + } + } + + // Compile list of users to send the notification to + emails := make([]string, 0, 256) err = p.db.AllUsers(func(u *user.User) { - // Check circumstances where we don't notify switch { case u.ID.String() == d.adminID: // User is the admin that made the status change return case u.ID.String() == author.ID.String(): - // User is the author. The author is sent a notification but - // the notification is not the same as the normal user - // notification so don't include the author's emails in the - // list of user emails. + // User is the author. The author is sent a different + // notification. Don't include them in the users list. return case !userNotificationEnabled(*u, notification): // User does not have notification bit set return } + // Add user to notification list emails = append(emails, u.Email) }) - // Email author - proposalName := proposalName(*pr) - if userNotificationEnabled(*author, notification) { - err = p.emailProposalStatusChangeToAuthor(d, proposalName, author.Email) - if err != nil { - log.Errorf("emailProposalStatusChangeToAuthor: %v", err) - } - } - // Email users - if len(emails) > 0 { - err = p.emailProposalStatusChangeToUsers(d, proposalName, emails) - if err != nil { - log.Errorf("emailProposalStatusChangeToUsers: %v", err) - } + err = p.emailProposalStatusChange(d, proposalName, emails) + if err != nil { + log.Errorf("emailProposalStatusChange: %v", err) + continue } log.Debugf("Sent proposal status change notifications %v", d.token) @@ -443,6 +449,7 @@ func (p *politeiawww) handleEventProposalComment(ch chan interface{}) { // If we made it here then there was an error. Log the error // before listening for the next event. log.Errorf("handleEventProposalComment: %v", err) + continue } } @@ -461,26 +468,28 @@ func (p *politeiawww) handleEventProposalVoteAuthorized(ch chan interface{}) { continue } + // Compile a list of emails to send the notification to. emails := make([]string, 0, 256) err := p.db.AllUsers(func(u *user.User) { - // Check circunstances where we don't notify switch { case !u.Admin: // Only notify admin users return case !userNotificationEnabled(*u, www.NotificationEmailAdminProposalVoteAuthorized): - // User does not have this notification bit set on + // User does not have notification bit set return } + // Add user to notification list emails = append(emails, u.Email) }) - err = p.emailProposalVoteAuthorized(d.token, d.name, d.username, - d.email, emails) + // Send notification email + err = p.emailProposalVoteAuthorized(d.token, d.name, d.username, emails) if err != nil { log.Errorf("emailProposalVoteAuthorized: %v", err) + continue } log.Debugf("Sent proposal vote authorized notifications %v", d.token) @@ -502,44 +511,41 @@ func (p *politeiawww) handleEventProposalVoteStarted(ch chan interface{}) { continue } - emails := make([]string, 0, 256) + // Email author notification := www.NotificationEmailRegularProposalVoteStarted + if userNotificationEnabled(d.author, notification) { + err := p.emailProposalVoteStartedToAuthor(d.token, d.name, + d.author.Username, d.author.Email) + if err != nil { + log.Errorf("emailProposalVoteStartedToAuthor: %v", err) + continue + } + } + + // Compile a list of users to send the notification to. + emails := make([]string, 0, 256) err := p.db.AllUsers(func(u *user.User) { - // Check circunstances where we don't notify switch { - case u.NewUserPaywallTx == "": - // User did not pay paywall - return case u.ID.String() == d.adminID: // Don't notify admin who started the vote return case u.ID.String() == d.author.ID.String(): - // Notify author separately from this users batch + // Don't send this notification to the author return case !userNotificationEnabled(*u, notification): - // User does not have notification bit set on + // User does not have notification bit set return } + // Add user to notification list emails = append(emails, u.Email) }) - // Email author - if userNotificationEnabled(d.author, notification) { - err = p.emailProposalVoteStartedToAuthor(d.token, d.name, - d.author.Username, d.author.Email) - if err != nil { - log.Errorf("emailProposalVoteStartedToAuthor: %v", err) - } - } - // Email users - if len(emails) > 0 { - err = p.emailProposalVoteStartedToUsers(d.token, d.name, - d.author.Username, emails) - if err != nil { - log.Errorf("emailProposalVoteStartedToUsers: %v", err) - } + err = p.emailProposalVoteStarted(d.token, d.name, emails) + if err != nil { + log.Errorf("emailProposalVoteStartedToUsers: %v", err) + continue } log.Debugf("Sent proposal vote started notifications %v", d.token) diff --git a/politeiawww/smtp.go b/politeiawww/smtp.go index c9dc81e86..c4e470f0b 100644 --- a/politeiawww/smtp.go +++ b/politeiawww/smtp.go @@ -24,6 +24,9 @@ func (s *smtp) sendEmailTo(subject, body string, recipients []string) error { if s.disabled { return nil } + if len(recipients) == 0 { + return nil + } // Setup email msg := goemail.NewMessage(s.mailAddress, subject, body) diff --git a/politeiawww/templates.go b/politeiawww/templates.go index 2a1b06032..2d383f2fe 100644 --- a/politeiawww/templates.go +++ b/politeiawww/templates.go @@ -6,8 +6,6 @@ package main import "text/template" -var () - // Proposal submitted type proposalSubmitted struct { Username string // Author username @@ -139,52 +137,55 @@ Comment: {{.Link}} var proposalCommentReplyTmpl = template.Must( template.New("proposalCommentReply").Parse(proposalCommentReplyText)) -// Pi events -type proposalVoteAuthorizedTemplateData struct { - Link string // GUI proposal details url - Name string // Proposal name +// Proposal vote authorized - Send to admins +type proposalVoteAuthorized struct { Username string // Author username - Email string // Author email + Name string // Proposal name + Link string // GUI proposal details url } -const templateProposalVoteAuthorizedRaw = ` -Voting has been authorized for the following proposal on Politeia by {{.Username}} ({{.Email}}): +const proposalVoteAuthorizedText = ` +{{.Username}} has authorized a vote on their proposal. {{.Name}} {{.Link}} ` -var templateProposalVoteAuthorized = template.Must( - template.New("proposal_vote_authorized_template"). - Parse(templateProposalVoteAuthorizedRaw)) +var proposalVoteAuthorizedTmpl = template.Must( + template.New("proposalVoteAuthorized").Parse(proposalVoteAuthorizedText)) -type proposalVoteStartedTemplateData struct { - Link string // GUI proposal details url - Name string // Proposal name - Username string // Author username +// Proposal vote started - Send to users +type proposalVoteStarted struct { + Name string // Proposal name + Link string // GUI proposal details url } -const templateProposalVoteStartedRaw = ` -Voting has started for the following proposal on Politeia, authored by {{.Username}}: +const proposalVoteStartedText = ` +Voting has started for the following proposal on Politeia. {{.Name}} {{.Link}} ` -var templateProposalVoteStarted = template.Must( - template.New("proposal_vote_started_template"). - Parse(templateProposalVoteStartedRaw)) +var proposalVoteStartedTmpl = template.Must( + template.New("proposalVoteStarted").Parse(proposalVoteStartedText)) + +// Proposal vote started - Send to author +type proposalVoteStartedToAuthor struct { + Name string // Proposal name + Link string // GUI proposal details url +} -const templateProposalVoteStartedForAuthorRaw = ` -Voting has just started for your proposal on Politeia! +const proposalVoteStartedToAuthorText = ` +Voting has just started on your Politeia proposal! {{.Name}} {{.Link}} ` -var templateProposalVoteStartedForAuthor = template.Must( - template.New("proposal_vote_started_for_author_template"). - Parse(templateProposalVoteStartedForAuthorRaw)) +var proposalVoteStartedToAuthorTmpl = template.Must( + template.New("proposalVoteStartedToAuthor"). + Parse(proposalVoteStartedToAuthorText)) // User events type newUserEmailTemplateData struct { From 4db5356c53348af226baea40591e87232e19d6c5 Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Mon, 21 Sep 2020 18:53:10 -0300 Subject: [PATCH 080/449] www: decredplugin functions refactor. --- politeiawww/decred.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/politeiawww/decred.go b/politeiawww/decred.go index c2ae20461..298b04e5f 100644 --- a/politeiawww/decred.go +++ b/politeiawww/decred.go @@ -17,10 +17,11 @@ func (p *politeiawww) decredGetComment(gc decredplugin.GetComment) (*decredplugi return nil, err } - // TODO this needs to use the politeiad plugin command - _ = payload - var reply string + // Execute plugin command + reply, err := p.pluginCommand(decredplugin.ID, decredplugin.CmdGetComment, + decredplugin.CmdGetComment, string(payload)) + // Receive plugin command reply gcr, err := decredplugin.DecodeGetCommentReply([]byte(reply)) if err != nil { return nil, err @@ -39,7 +40,6 @@ func (p *politeiawww) decredCommentGetByID(token, commentID string) (*decredplug return p.decredGetComment(gc) } -// decredCommentGetBySignature retrieves the specified decred plugin comment // decredGetComments sends the decred plugin getcomments command to the cache // and returns all of the comments for the passed in proposal token. func (p *politeiawww) decredGetComments(token string) ([]decredplugin.Comment, error) { @@ -52,10 +52,11 @@ func (p *politeiawww) decredGetComments(token string) ([]decredplugin.Comment, e return nil, err } - // TODO this needs to use the politeiad plugin command - _ = payload - var reply string + // Execute plugin command + reply, err := p.pluginCommand(decredplugin.ID, decredplugin.CmdGetComments, + decredplugin.CmdGetComments, string(payload)) + // Receive plugin command reply gcr, err := decredplugin.DecodeGetCommentsReply([]byte(reply)) if err != nil { return nil, err From 6082af258ce4bb783eb311f40c7d181a03bc34e8 Mon Sep 17 00:00:00 2001 From: Amir Massarwa Date: Tue, 22 Sep 2020 01:43:43 +0300 Subject: [PATCH 081/449] politeiawww: Add processCommentCensor. --- .../cmd/{shared => cmswww}/censorcomment.go | 32 ++--- politeiawww/cmd/cmswww/cmswww.go | 4 +- politeiawww/cmd/cmswww/help.go | 2 +- politeiawww/cmd/piwww/commentcensor.go | 120 ++++++++++++++++++ politeiawww/cmd/piwww/help.go | 4 +- politeiawww/cmd/piwww/piwww.go | 8 +- politeiawww/cmd/shared/client.go | 28 +++- politeiawww/comments.go | 24 +++- politeiawww/piwww.go | 65 +++++++++- 9 files changed, 251 insertions(+), 36 deletions(-) rename politeiawww/cmd/{shared => cmswww}/censorcomment.go (74%) create mode 100644 politeiawww/cmd/piwww/commentcensor.go diff --git a/politeiawww/cmd/shared/censorcomment.go b/politeiawww/cmd/cmswww/censorcomment.go similarity index 74% rename from politeiawww/cmd/shared/censorcomment.go rename to politeiawww/cmd/cmswww/censorcomment.go index 33e41a7f0..40214fd00 100644 --- a/politeiawww/cmd/shared/censorcomment.go +++ b/politeiawww/cmd/cmswww/censorcomment.go @@ -1,14 +1,15 @@ -// Copyright (c) 2017-2019 The Decred developers +// Copyright (c) 2017-2020 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package shared +package main import ( "encoding/hex" "fmt" v1 "github.com/decred/politeia/politeiawww/api/www/v1" + "github.com/decred/politeia/politeiawww/cmd/shared" "github.com/decred/politeia/util" ) @@ -29,7 +30,7 @@ func (cmd *CensorCommentCmd) Execute(args []string) error { // Check for user identity if cfg.Identity == nil { - return ErrUserIdentityNotFound + return shared.ErrUserIdentityNotFound } // Get server public key @@ -50,13 +51,13 @@ func (cmd *CensorCommentCmd) Execute(args []string) error { } // Print request details - err = PrintJSON(cc) + err = shared.PrintJSON(cc) if err != nil { return err } // Send request - ccr, err := client.CensorComment(cc) + ccr, err := client.WWWCensorComment(cc) if err != nil { return err } @@ -75,12 +76,12 @@ func (cmd *CensorCommentCmd) Execute(args []string) error { } // Print response details - return PrintJSON(ccr) + return shared.PrintJSON(ccr) } -// CensorCommentHelpMsg is the output of the help command when 'censorcomment' +// censorCommentHelpMsg is the output of the help command when 'censorcomment' // is specified. -const CensorCommentHelpMsg = `censorcomment "token" "commentID" "reason" +const censorCommentHelpMsg = `censorcomment "token" "commentID" "reason" Censor a user comment. Requires admin privileges. @@ -88,17 +89,4 @@ Arguments: 1. token (string, required) Proposal censorship token 2. commentID (string, required) Id of the comment 3. reason (string, required) Reason for censoring the comment - -Request: -{ - "token": (string) Censorship token - "commentid": (string) Id of comment - "reason": (string) Reason for censoring the comment - "signature": (string) Signature of censor comment (Token+CommentID+Reason) - "publickey": (string) Public key used for signature -} - -Response: -{ - "receipt": (string) Server signature of comment sensor signature -}` +` diff --git a/politeiawww/cmd/cmswww/cmswww.go b/politeiawww/cmd/cmswww/cmswww.go index 938eb6e71..49f669a78 100644 --- a/politeiawww/cmd/cmswww/cmswww.go +++ b/politeiawww/cmd/cmswww/cmswww.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Decred developers +// Copyright (c) 2017-2020 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -60,7 +60,7 @@ type cmswww struct { ActiveVotes ActiveVotesCmd `command:"activevotes" description:"(user) get the dccs that are being voted on"` BatchProposals shared.BatchProposalsCmd `command:"batchproposals" description:"(user) retrieve a set of proposals"` NewComment NewCommentCmd `command:"newcomment" description:"(user) create a new comment"` - CensorComment shared.CensorCommentCmd `command:"censorcomment" description:"(admin) censor a comment"` + CensorComment CensorCommentCmd `command:"censorcomment" description:"(admin) censor a comment"` ChangePassword shared.ChangePasswordCmd `command:"changepassword" description:"(user) change the password for the logged in user"` ChangeUsername shared.ChangeUsernameCmd `command:"changeusername" description:"(user) change the username for the logged in user"` CMSUsers CMSUsersCmd `command:"cmsusers" description:"(user) get a list of cms users"` diff --git a/politeiawww/cmd/cmswww/help.go b/politeiawww/cmd/cmswww/help.go index 78aee23f2..2f47ef4a2 100644 --- a/politeiawww/cmd/cmswww/help.go +++ b/politeiawww/cmd/cmswww/help.go @@ -38,7 +38,7 @@ func (cmd *HelpCmd) Execute(args []string) error { case "newcomment": fmt.Printf("%s\n", newCommentHelpMsg) case "censorcomment": - fmt.Printf("%s\n", shared.CensorCommentHelpMsg) + fmt.Printf("%s\n", censorCommentHelpMsg) case "manageuser": fmt.Printf("%s\n", shared.ManageUserHelpMsg) case "cmsmanageuser": diff --git a/politeiawww/cmd/piwww/commentcensor.go b/politeiawww/cmd/piwww/commentcensor.go new file mode 100644 index 000000000..e3d3f380a --- /dev/null +++ b/politeiawww/cmd/piwww/commentcensor.go @@ -0,0 +1,120 @@ +// Copyright (c) 2017-2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "encoding/hex" + "fmt" + "strconv" + + pi "github.com/decred/politeia/politeiawww/api/pi/v1" + "github.com/decred/politeia/politeiawww/cmd/shared" + "github.com/decred/politeia/util" +) + +// CommentCensorCmd censors a proposal comment. +type CommentCensorCmd struct { + Args struct { + Token string `positional-arg-name:"token"` // Censorship token + CommentID string `positional-arg-name:"commentID"` // Comment ID + Reason string `positional-arg-name:"reason"` // Reason for censoring + } `positional-args:"true" required:"true"` + + // CLI flags + Vetted bool `long:"vetted" optional:"true"` + Unvetted bool `long:"unvetted" optional:"true"` +} + +// Execute executes the censor comment command. +func (cmd *CommentCensorCmd) Execute(args []string) error { + token := cmd.Args.Token + commentID := cmd.Args.CommentID + reason := cmd.Args.Reason + + // Verify state + var state pi.PropStateT + switch { + case cmd.Vetted && cmd.Unvetted: + return fmt.Errorf("cannot use --vetted and --unvetted simultaneously") + case cmd.Unvetted: + state = pi.PropStateUnvetted + case cmd.Vetted: + state = pi.PropStateVetted + default: + return fmt.Errorf("must specify either --vetted or unvetted") + } + + // Check for user identity + if cfg.Identity == nil { + return shared.ErrUserIdentityNotFound + } + + // Get server public key + vr, err := client.Version() + if err != nil { + return err + } + + // Setup comment censor request + s := cfg.Identity.SignMessage([]byte(string(state) + token + commentID + reason)) + signature := hex.EncodeToString(s[:]) + // Parse provided comment id + ciUint, err := strconv.ParseUint(commentID, 10, 32) + if err != nil { + return err + } + cc := &pi.CommentCensor{ + Token: token, + State: state, + CommentID: uint32(ciUint), + Reason: reason, + Signature: signature, + PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), + } + + // Print request details + err = shared.PrintJSON(cc) + if err != nil { + return err + } + + // Send request + ccr, err := client.CommentCensor(cc) + if err != nil { + return err + } + + // Validate censor comment receipt + serverID, err := util.IdentityFromString(vr.PubKey) + if err != nil { + return err + } + receiptB, err := util.ConvertSignature(ccr.Receipt) + if err != nil { + return err + } + if !serverID.VerifyMessage([]byte(signature), receiptB) { + return fmt.Errorf("could not verify receipt signature") + } + + // Print response details + return shared.PrintJSON(ccr) +} + +// commentCensorHelpMsg is the output of the help command when 'commentcensor' +// is specified. +const commentCensorHelpMsg = `commentcensor "token" "commentID" "reason" + +Censor a user comment. Requires admin privileges. + +Arguments: +1. token (string, required) Proposal censorship token +2. commentID (string, required) Id of the comment +3. reason (string, required) Reason for censoring the comment + +Flags: + --vetted (bool, optional) Comment on vetted record. + --unvetted (bool, optional) Comment on unvetted reocrd. +` diff --git a/politeiawww/cmd/piwww/help.go b/politeiawww/cmd/piwww/help.go index 04b03b607..ceb3cb02e 100644 --- a/politeiawww/cmd/piwww/help.go +++ b/politeiawww/cmd/piwww/help.go @@ -91,8 +91,8 @@ func (cmd *HelpCmd) Execute(args []string) error { fmt.Printf("%s\n", commentNewHelpMsg) case "proposalcomments": fmt.Printf("%s\n", proposalCommentsHelpMsg) - case "censorcomment": - fmt.Printf("%s\n", shared.CensorCommentHelpMsg) + case "commentcensor": + fmt.Printf("%s\n", commentCensorHelpMsg) case "commentvote": fmt.Printf("%s\n", commentVoteHelpMsg) case "userlikecomments": diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index 8d942f258..e47c9b942 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -50,16 +50,16 @@ type piwww struct { ProposalSetStatus ProposalSetStatusCmd `command:"proposalsetstatus"` // Comments commands - CommentNew CommentNewCmd `command:"commentnew" description:"(user) create a new comment"` - CommentVote CommentVoteCmd `command:"commentvote" description:"(user) upvote/downvote a comment"` - Comments CommentsCmd `command:"comments" description:"(public) get the comments for a proposal"` + CommentNew CommentNewCmd `command:"commentnew" description:"(user) create a new comment"` + CommentVote CommentVoteCmd `command:"commentvote" description:"(user) upvote/downvote a comment"` + CommentCensor CommentCensorCmd `command:"commentcensor" description:"(admin) censor a comment"` + Comments CommentsCmd `command:"comments" description:"(public) get the comments for a proposal"` // Commands ActiveVotes ActiveVotesCmd `command:"activevotes" description:"(public) get the proposals that are being voted on"` AuthorizeVote AuthorizeVoteCmd `command:"authorizevote" description:"(user) authorize a proposal vote (must be proposal author)"` BatchProposals shared.BatchProposalsCmd `command:"batchproposals" description:"(user) retrieve a set of proposals"` BatchVoteSummary BatchVoteSummaryCmd `command:"batchvotesummary" description:"(user) retrieve the vote summary for a set of proposals"` - CensorComment shared.CensorCommentCmd `command:"censorcomment" description:"(admin) censor a comment"` ChangePassword shared.ChangePasswordCmd `command:"changepassword" description:"(user) change the password for the logged in user"` ChangeUsername shared.ChangeUsernameCmd `command:"changeusername" description:"(user) change the username for the logged in user"` EditUser EditUserCmd `command:"edituser" description:"(user) edit the preferences of the logged in user"` diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index f6d1d8c56..7b88d76be 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -1282,8 +1282,32 @@ func (c *Client) CommentVote(cv *pi.CommentVote) (*pi.CommentVoteReply, error) { return &cvr, nil } -// CensorComment censors the specified proposal comment. -func (c *Client) CensorComment(cc *www.CensorComment) (*www.CensorCommentReply, error) { +// CommentCensor censors the specified proposal comment. +func (c *Client) CommentCensor(cc *pi.CommentCensor) (*pi.CommentCensorReply, error) { + responseBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, + pi.RouteCommentCensor, cc) + if err != nil { + return nil, err + } + + var ccr pi.CommentCensorReply + err = json.Unmarshal(responseBody, &ccr) + if err != nil { + return nil, fmt.Errorf("unmarshal CensorCommentReply: %v", err) + } + + if c.cfg.Verbose { + err := prettyPrintJSON(ccr) + if err != nil { + return nil, err + } + } + + return &ccr, nil +} + +// WWWCensorComment censors the specified proposal comment. +func (c *Client) WWWCensorComment(cc *www.CensorComment) (*www.CensorCommentReply, error) { responseBody, err := c.makeRequest(http.MethodPost, www.PoliteiaWWWAPIRoute, www.RouteCensorComment, cc) if err != nil { diff --git a/politeiawww/comments.go b/politeiawww/comments.go index b0445b8ae..fb2853b1c 100644 --- a/politeiawww/comments.go +++ b/politeiawww/comments.go @@ -10,7 +10,28 @@ import ( "github.com/decred/politeia/politeiawww/user" ) -// comments call the comments plugin to get record's comments. +// commentCensor calls the comments plugin to censor a given comment. +func (p *politeiawww) commentCensor(cc comments.Del) (*comments.DelReply, error) { + // Prep plugin payload + payload, err := comments.EncodeDel(cc) + if err != nil { + return nil, err + } + + r, err := p.pluginCommand(comments.ID, comments.CmdDel, "", + string(payload)) + if err != nil { + return nil, err + } + ccr, err := comments.DecodeDelReply(([]byte(r))) + if err != nil { + return nil, err + } + + return ccr, nil +} + +// comments calls the comments plugin to get record's comments. func (p *politeiawww) comments(cp comments.GetAll) (*comments.GetAllReply, error) { // Prep plugin payload payload, err := comments.EncodeGetAll(cp) @@ -29,7 +50,6 @@ func (p *politeiawww) comments(cp comments.GetAll) (*comments.GetAllReply, error } return cr, nil - } func validateComment(c www.NewComment) error { diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index de9e03444..be476e38c 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -1632,6 +1632,68 @@ func (p *politeiawww) handleComments(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, cr) } +func (p *politeiawww) processCommentCensor(cc pi.CommentCensor, usr user.User) (*pi.CommentCensorReply, error) { + log.Tracef("processCommentCensor: %v %v", cc.Token, cc.CommentID) + + // Sanity check + if !usr.Admin { + return nil, fmt.Errorf("user not admin") + } + + // Verify user signed with their active identity + if usr.PublicKey() != cc.PublicKey { + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusPublicKeyInvalid, + ErrorContext: []string{"not user's active identity"}, + } + } + + // Call comments plugin to censor comment + reply, err := p.commentCensor(comments.Del{ + State: convertCommentsPluginPropStateFromPi(cc.State), + Token: cc.Token, + CommentID: cc.CommentID, + Reason: cc.Reason, + PublicKey: cc.PublicKey, + Signature: cc.Signature, + }) + if err != nil { + return nil, err + } + + return &pi.CommentCensorReply{ + Timestamp: reply.Timestamp, + Receipt: reply.Receipt, + }, nil +} + +func (p *politeiawww) handleCommentCensor(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleCommentCensor") + + var cc pi.CommentCensor + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&cc); err != nil { + respondWithPiError(w, r, "handleCommentCensor: unmarshal", + pi.UserErrorReply{}) + return + } + + user, err := p.getSessionUser(w, r) + if err != nil { + respondWithPiError(w, r, + "handleCommentCensor: getSessionUser: %v", err) + return + } + + ccr, err := p.processCommentCensor(cc, *user) + if err != nil { + respondWithPiError(w, r, + "handleCommentCensor: processCommentCensor: %v", err) + } + + util.RespondWithJSON(w, http.StatusOK, ccr) +} + func (p *politeiawww) setPiRoutes() { // Public routes p.addRoute(http.MethodGet, pi.APIRoute, @@ -1661,5 +1723,6 @@ func (p *politeiawww) setPiRoutes() { pi.RouteCommentVote, p.handleCommentVote, permissionLogin) // Admin routes - + p.addRoute(http.MethodPost, pi.APIRoute, + pi.RouteCommentCensor, p.handleCommentCensor, permissionAdmin) } From ef1fe94099c52350410e7e7e335e9ae6fd0f4d70 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 20 Sep 2020 20:11:57 -0500 Subject: [PATCH 082/449] refactor tlog comments plugin --- plugins/comments/comments.go | 162 ++- politeiad/backend/tlogbe/comments.go | 1091 +++++++++-------- .../backend/tlogbe/plugins/ticketvote.go | 4 +- politeiad/backend/tlogbe/store/store.go | 19 +- politeiad/backend/tlogbe/tlog.go | 26 +- politeiad/backend/tlogbe/tlogclient.go | 8 +- politeiawww/api/pi/v1/v1.go | 14 +- politeiawww/templates.go | 6 +- 8 files changed, 747 insertions(+), 583 deletions(-) diff --git a/plugins/comments/comments.go b/plugins/comments/comments.go index a55a2b1f4..fd4124d05 100644 --- a/plugins/comments/comments.go +++ b/plugins/comments/comments.go @@ -24,15 +24,14 @@ const ( CmdNew = "new" // Create a new comment CmdEdit = "edit" // Edit a comment CmdDel = "del" // Del a comment + CmdVote = "vote" // Vote on a comment CmdGet = "get" // Get specified comments CmdGetAll = "getall" // Get all comments for a record CmdGetVersion = "getversion" // Get specified version of a comment CmdCount = "count" // Get comments count for a record - CmdVote = "vote" // Vote on a comment + CmdVotes = "votes" // Get comment votes CmdProofs = "proofs" // Get inclusion proofs - // TODO CmdUserVotes = "uservotes" // Get votes from a specific uuid - // Record states StateInvalid StateT = 0 StateUnvetted StateT = 1 @@ -43,25 +42,28 @@ const ( VoteDownvote VoteT = -1 VoteUpvote VoteT = 1 - // TODO Add plugin policies - // PolicyMaxCommentLength + // PolicyCommentLengthMax is the maximum number of characters + // accepted for comments. + PolicyCommentLengthMax = 8000 - // PolicayMaxVoteChanges is the maximum number times a user can + // PolicayVoteChangesMax is the maximum number times a user can // change their vote on a comment. This prevents a malicious user // from being able to spam comment votes. - PolicyMaxVoteChanges = 5 + PolicyVoteChangesMax = 5 // Error status codes ErrorStatusInvalid ErrorStatusT = 0 ErrorStatusTokenInvalid ErrorStatusT = 1 ErrorStatusPublicKeyInvalid ErrorStatusT = 2 ErrorStatusSignatureInvalid ErrorStatusT = 3 - ErrorStatusRecordNotFound ErrorStatusT = 4 - ErrorStatusCommentNotFound ErrorStatusT = 5 - ErrorStatusParentIDInvalid ErrorStatusT = 6 - ErrorStatusNoCommentChanges ErrorStatusT = 7 - ErrorStatusVoteInvalid ErrorStatusT = 8 - ErrorStatusMaxVoteChanges ErrorStatusT = 9 + ErrorStatusCommentLengthMax ErrorStatusT = 4 + ErrorStatusRecordNotFound ErrorStatusT = 5 + ErrorStatusCommentNotFound ErrorStatusT = 6 + ErrorStatusUserIDInvalid ErrorStatusT = 7 + ErrorStatusParentIDInvalid ErrorStatusT = 8 + ErrorStatusNoCommentChanges ErrorStatusT = 9 + ErrorStatusVoteInvalid ErrorStatusT = 10 + ErrorStatusVoteChangesMax ErrorStatusT = 11 ) var ( @@ -76,7 +78,7 @@ var ( ErrorStatusParentIDInvalid: "parent id invalid", ErrorStatusNoCommentChanges: "comment did not change", ErrorStatusVoteInvalid: "invalid vote", - ErrorStatusMaxVoteChanges: "max vote changes exceeded", + ErrorStatusVoteChangesMax: "vote changes max exceeded", } ) @@ -95,7 +97,7 @@ func (e UserErrorReply) Error() string { // // Signature is the client signature of State+Token+ParentID+Comment. type Comment struct { - UUID string `json:"uuid"` // Unique user ID + UserID string `json:"userid"` // Unique user ID State StateT `json:"state"` // Record state Token string `json:"token"` // Record token ParentID uint32 `json:"parentid"` // Parent comment ID if reply @@ -117,7 +119,7 @@ type Comment struct { // Signature is the client signature of State+Token+ParentID+Comment. type CommentAdd struct { // Data generated by client - UUID string `json:"uuid"` // Unique user ID + UserID string `json:"userid"` // Unique user ID State StateT `json:"state"` // Record state Token string `json:"token"` // Record token ParentID uint32 `json:"parentid"` // Parent comment ID @@ -133,10 +135,10 @@ type CommentAdd struct { } // CommentDel is the structure that is saved to disk when a comment is deleted. -// Some additional fields like ParentID and AuthorPublicKey are required to be -// saved since all the comment add records will be deleted and the client needs -// these additional fields to properly display the deleted comment in the -// comment hierarchy. +// Some additional fields like ParentID and UserID are required to be saved since +// all the CommentAdd records will be deleted and the client needs these +// additional fields to properly display the deleted comment in the comment +// hierarchy. // // Signature is the client signature of the State+Token+CommentID+Reason type CommentDel struct { @@ -149,10 +151,10 @@ type CommentDel struct { Signature string `json:"signature"` // Client signature // Metadata generated by server - ParentID uint32 `json:"parentid"` // Parent comment ID - AuthorPublicKey string `json:"authorpublickey"` // Author public key - Timestamp int64 `json:"timestamp"` // Received UNIX timestamp - Receipt string `json:"receipt"` // Server sig of client sig + ParentID uint32 `json:"parentid"` // Parent comment ID + UserID string `json:"userid"` // Author user ID + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + Receipt string `json:"receipt"` // Server sig of client sig } // CommentVote is the structure that is saved to disk when a comment is voted @@ -161,7 +163,7 @@ type CommentDel struct { // Signature is the client signature of the State+Token+CommentID+Vote. type CommentVote struct { // Data generated by client - UUID string `json:"uuid"` // Unique user ID + UserID string `json:"userid"` // Unique user ID State StateT `json:"state"` // Record state Token string `json:"token"` // Record token CommentID uint32 `json:"commentid"` // Comment ID @@ -181,7 +183,7 @@ type CommentVote struct { // // Signature is the client signature of State+Token+ParentID+Comment. type New struct { - UUID string `json:"uuid"` // Unique user ID + UserID string `json:"userid"` // Unique user ID State StateT `json:"state"` // Record state Token string `json:"token"` // Record token ParentID uint32 `json:"parentid"` // Parent comment ID @@ -231,7 +233,7 @@ func DecodeNewReply(payload []byte) (*NewReply, error) { // // Signature is the client signature of State+Token+ParentID+Comment. type Edit struct { - UUID string `json:"uuid"` // Unique user ID + UserID string `json:"userid"` // Unique user ID State StateT `json:"state"` // Record state Token string `json:"token"` // Record token ParentID uint32 `json:"parentid"` // Parent comment ID @@ -326,6 +328,62 @@ func DecodeDelReply(payload []byte) (*DelReply, error) { return &dr, nil } +// Vote casts a comment vote (upvote or downvote). +// +// The effect of a new vote on a comment score depends on the previous vote +// from that user ID. Example, a user upvotes a comment that they have already +// upvoted, the resulting vote score is 0 due to the second upvote removing the +// original upvote. The public key cannot be relied on to remain the same for +// each user so a user ID must be included. +// +// Signature is the client signature of the State+Token+CommentID+Vote. +type Vote struct { + UserID string `json:"userid"` // Unique user ID + State StateT `json:"state"` // Record state + Token string `json:"token"` // Record token + CommentID uint32 `json:"commentid"` // Comment ID + Vote VoteT `json:"vote"` // Upvote or downvote + PublicKey string `json:"publickey"` // Public key used for signature + Signature string `json:"signature"` // Client signature +} + +// EncodeVote encodes a Vote into a JSON byte slice. +func EncodeVote(v Vote) ([]byte, error) { + return json.Marshal(v) +} + +// DecodeVote decodes a JSON byte slice into a Vote. +func DecodeVote(payload []byte) (*Vote, error) { + var v Vote + err := json.Unmarshal(payload, &v) + if err != nil { + return nil, err + } + return &v, nil +} + +// VoteReply is the reply to the Vote command. +type VoteReply struct { + Score int64 `json:"score"` // Overall comment vote score + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + Receipt string `json:"receipt"` // Server signature of client signature +} + +// EncodeVoteReply encodes a VoteReply into a JSON byte slice. +func EncodeVoteReply(vr VoteReply) ([]byte, error) { + return json.Marshal(vr) +} + +// DecodeVoteReply decodes a JSON byte slice into a VoteReply. +func DecodeVoteReply(payload []byte) (*VoteReply, error) { + var vr VoteReply + err := json.Unmarshal(payload, &vr) + if err != nil { + return nil, err + } + return &vr, nil +} + // Get returns the latest version of the comments for the provided comment IDs. // An error is not returned if a comment is not found for one or more of the // comment IDs. Those entries will simply not be included in the reply. @@ -498,33 +556,21 @@ func DecodeCountReply(payload []byte) (*CountReply, error) { return &cr, nil } -// Vote casts a comment vote (upvote or downvote). -// -// The effect of a new vote on a comment score depends on the previous vote -// from that uuid. Example, a user upvotes a comment that they have already -// upvoted, the resulting vote score is 0 due to the second upvote removing the -// original upvote. The public key cannot be relied on to remain the same for -// each user so a uuid must be included. -// -// Signature is the client signature of the State+Token+CommentID+Vote. -type Vote struct { - UUID string `json:"uuid"` // Unique user ID - State StateT `json:"state"` // Record state - Token string `json:"token"` // Record token - CommentID uint32 `json:"commentid"` // Comment ID - Vote VoteT `json:"vote"` // Upvote or downvote - PublicKey string `json:"publickey"` // Public key used for signature - Signature string `json:"signature"` // Client signature +// Votes returns the comment votes that meet the provided filtering criteria. +type Votes struct { + State StateT `json:"state"` + Token string `json:"token"` + UserID string `json:"userid"` } -// EncodeVote encodes a Vote into a JSON byte slice. -func EncodeVote(v Vote) ([]byte, error) { +// EncodeVotes encodes a Votes into a JSON byte slice. +func EncodeVotes(v Votes) ([]byte, error) { return json.Marshal(v) } -// DecodeVote decodes a JSON byte slice into a Vote. -func DecodeVote(payload []byte) (*Vote, error) { - var v Vote +// DecodeVotes decodes a JSON byte slice into a Votes. +func DecodeVotes(payload []byte) (*Votes, error) { + var v Votes err := json.Unmarshal(payload, &v) if err != nil { return nil, err @@ -532,21 +578,19 @@ func DecodeVote(payload []byte) (*Vote, error) { return &v, nil } -// VoteReply is the reply to the Vote command. -type VoteReply struct { - Score int64 `json:"score"` // Overall comment vote score - Timestamp int64 `json:"timestamp"` // Received UNIX timestamp - Receipt string `json:"receipt"` // Server signature of client signature +// VotesReply is the reply to the Votes command. +type VotesReply struct { + Votes []CommentVote `json:"votes"` } -// EncodeVoteReply encodes a VoteReply into a JSON byte slice. -func EncodeVoteReply(vr VoteReply) ([]byte, error) { +// EncodeVotesReply encodes a VotesReply into a JSON byte slice. +func EncodeVotesReply(vr VotesReply) ([]byte, error) { return json.Marshal(vr) } -// DecodeVoteReply decodes a JSON byte slice into a VoteReply. -func DecodeVoteReply(payload []byte) (*VoteReply, error) { - var vr VoteReply +// DecodeVotesReply decodes a JSON byte slice into a VotesReply. +func DecodeVotesReply(payload []byte) (*VotesReply, error) { + var vr VotesReply err := json.Unmarshal(payload, &vr) if err != nil { return nil, err diff --git a/politeiad/backend/tlogbe/comments.go b/politeiad/backend/tlogbe/comments.go index 00c4564e2..d2aecf609 100644 --- a/politeiad/backend/tlogbe/comments.go +++ b/politeiad/backend/tlogbe/comments.go @@ -11,7 +11,12 @@ import ( "encoding/json" "errors" "fmt" + "io/ioutil" + "os" + "path/filepath" + "sort" "strconv" + "strings" "sync" "time" @@ -22,31 +27,30 @@ import ( "github.com/decred/politeia/util" ) -// TODO don't save data to the file system. Save it to the kv store and save -// the key to the file system. This will allow the data to be backed up. - -// TODO comment signature messages need to have state added to them. -// TODO uuid has been added to New, Edit, and CommentAdd. - const ( // Blob entry data descriptors - dataDescriptorCommentAdd = "commentadd" - dataDescriptorCommentDel = "commentdel" - dataDescriptorCommentVote = "commentvote" - dataDescriptorCommentsIndex = "commentsindex" + dataDescriptorCommentAdd = "commentadd" + dataDescriptorCommentDel = "commentdel" + dataDescriptorCommentVote = "commentvote" // Prefixes that are appended to key-value store keys before // storing them in the log leaf ExtraData field. - keyPrefixCommentAdd = "commentadd:" - keyPrefixCommentDel = "commentdel:" - keyPrefixCommentVote = "commentvote:" - keyPrefixCommentsIndex = "commentsindex:" + keyPrefixCommentAdd = "commentadd:" + keyPrefixCommentDel = "commentdel:" + keyPrefixCommentVote = "commentvote:" + + // Filenames of cached data saved to the plugin data dir. Brackets + // are used to indicate a variable that should be replaced in the + // filename. + filenameCommentsIndex = "{state}-{token}-commentsindex.json" ) var ( _ pluginClient = (*commentsPlugin)(nil) ) +// commentsPlugin is the tlog backend implementation of the comments plugin. +// // commentsPlugin satisfies the pluginClient interface. type commentsPlugin struct { sync.Mutex @@ -54,14 +58,19 @@ type commentsPlugin struct { backend backend.Backend tlog tlogClient + // dataDir is the comments plugin data directory. The only data + // that is stored here is cached data that can be re-created at any + // time by walking the trillian trees. + dataDir string + // Mutexes contains a mutex for each record. The mutexes are lazy // loaded. mutexes map[string]*sync.Mutex // [string]mutex } type voteIndex struct { - Vote comments.VoteT `json:"vote"` - MerkleHash []byte `json:"merklehash"` + Vote comments.VoteT `json:"vote"` + Merkle []byte `json:"merkle"` // Log leaf merkle leaf hash } type commentIndex struct { @@ -78,7 +87,7 @@ type commentIndex struct { Votes map[string][]voteIndex `json:"votes"` // [uuid]votes } -// TODO the comments index should be cached in the data dir +// commentsIndex contains the indexes for all comments made on a record. type commentsIndex struct { Comments map[uint32]commentIndex `json:"comments"` // [commentID]comment } @@ -98,14 +107,103 @@ func (p *commentsPlugin) mutex(token string) *sync.Mutex { return m } -func tlogIDForCommentState(s comments.StateT) string { +func (p *commentsPlugin) commentsIndexPath(s comments.StateT, token string) string { + fn := filenameCommentsIndex + switch s { + case comments.StateUnvetted: + fn = strings.Replace(fn, "{state}", "unvetted", 1) + case comments.StateVetted: + fn = strings.Replace(fn, "{state}", "vetted", 1) + default: + e := fmt.Errorf("unknown comments state: %v", s) + panic(e) + } + fn = strings.Replace(fn, "{token}", token, 1) + return filepath.Join(p.dataDir, fn) +} + +// commentsIndexLocked returns the cached commentsIndex for the provided +// record. If a cached commentsIndex does not exist, a new one will be +// returned. +// +// This function must be called WITH the lock held. +func (p *commentsPlugin) commentsIndexLocked(s comments.StateT, token []byte) (*commentsIndex, error) { + fp := p.commentsIndexPath(s, hex.EncodeToString(token)) + b, err := ioutil.ReadFile(fp) + if err != nil { + var e *os.PathError + if errors.As(err, &e) && !os.IsExist(err) { + // File does't exist. Return a new commentsIndex instead. + return &commentsIndex{ + Comments: make(map[uint32]commentIndex), + }, nil + } + return nil, err + } + + var idx commentsIndex + err = json.Unmarshal(b, &idx) + if err != nil { + return nil, err + } + + return &idx, nil +} + +// commentsIndex returns the cached commentsIndex for the provided +// record. If a cached commentsIndex does not exist, a new one will be +// returned. +// +// This function must be called WITHOUT the lock held. +func (p *commentsPlugin) commentsIndex(s comments.StateT, token []byte) (*commentsIndex, error) { + m := p.mutex(hex.EncodeToString(token)) + m.Lock() + defer m.Unlock() + + return p.commentsIndexLocked(s, token) +} + +// commentsIndexSaveLocked saves the provided commentsIndex to the comments +// plugin data dir. +// +// This function must be called WITH the lock held. +func (p *commentsPlugin) commentsIndexSaveLocked(s comments.StateT, token []byte, idx commentsIndex) error { + b, err := json.Marshal(idx) + if err != nil { + return err + } + + fp := p.commentsIndexPath(s, hex.EncodeToString(token)) + err = ioutil.WriteFile(fp, b, 0664) + if err != nil { + return err + } + + return nil +} + +func tlogIDFromCommentState(s comments.StateT) string { switch s { case comments.StateUnvetted: return tlogIDUnvetted case comments.StateVetted: return tlogIDVetted + default: + e := fmt.Sprintf("unknown state %v", s) + panic(e) + } +} + +func encryptFromCommentState(s comments.StateT) bool { + switch s { + case comments.StateUnvetted: + return true + case comments.StateVetted: + return false + default: + e := fmt.Sprintf("unknown state %v", s) + panic(e) } - return "" } func convertCommentsErrorFromSignatureError(err error) comments.UserErrorReply { @@ -138,7 +236,7 @@ func convertBlobEntryFromCommentAdd(c comments.CommentAdd) (*store.BlobEntry, er if err != nil { return nil, err } - be := store.BlobEntryNew(hint, data) + be := store.NewBlobEntry(hint, data) return &be, nil } @@ -155,7 +253,7 @@ func convertBlobEntryFromCommentDel(c comments.CommentDel) (*store.BlobEntry, er if err != nil { return nil, err } - be := store.BlobEntryNew(hint, data) + be := store.NewBlobEntry(hint, data) return &be, nil } @@ -172,24 +270,7 @@ func convertBlobEntryFromCommentVote(c comments.CommentVote) (*store.BlobEntry, if err != nil { return nil, err } - be := store.BlobEntryNew(hint, data) - return &be, nil -} - -func convertBlobEntryFromCommentsIndex(ci commentsIndex) (*store.BlobEntry, error) { - data, err := json.Marshal(ci) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorCommentsIndex, - }) - if err != nil { - return nil, err - } - be := store.BlobEntryNew(hint, data) + be := store.NewBlobEntry(hint, data) return &be, nil } @@ -269,7 +350,7 @@ func convertCommentDelFromBlobEntry(be store.BlobEntry) (*comments.CommentDel, e return &c, nil } -func convertCommentsIndexFromBlobEntry(be store.BlobEntry) (*commentsIndex, error) { +func convertCommentVoteFromBlobEntry(be store.BlobEntry) (*comments.CommentVote, error) { // Decode and validate data hint b, err := base64.StdEncoding.DecodeString(be.DataHint) if err != nil { @@ -280,9 +361,9 @@ func convertCommentsIndexFromBlobEntry(be store.BlobEntry) (*commentsIndex, erro if err != nil { return nil, fmt.Errorf("unmarshal DataHint: %v", err) } - if dd.Descriptor != dataDescriptorCommentsIndex { + if dd.Descriptor != dataDescriptorCommentVote { return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorCommentsIndex) + dd.Descriptor, dataDescriptorCommentVote) } // Decode data @@ -298,18 +379,19 @@ func convertCommentsIndexFromBlobEntry(be store.BlobEntry) (*commentsIndex, erro return nil, fmt.Errorf("data is not coherent; got %x, want %x", util.Digest(b), hash) } - var ci commentsIndex - err = json.Unmarshal(b, &ci) + var cv comments.CommentVote + err = json.Unmarshal(b, &cv) if err != nil { - return nil, fmt.Errorf("unmarshal commentsIndex: %v", err) + return nil, fmt.Errorf("unmarshal CommentVote: %v", err) } - return &ci, nil + return &cv, nil } func convertCommentFromCommentAdd(ca comments.CommentAdd) comments.Comment { // Score needs to be filled in seperately return comments.Comment{ + UserID: ca.UserID, Token: ca.Token, ParentID: ca.ParentID, Comment: ca.Comment, @@ -328,10 +410,10 @@ func convertCommentFromCommentAdd(ca comments.CommentAdd) comments.Comment { func convertCommentFromCommentDel(cd comments.CommentDel) comments.Comment { // Score needs to be filled in seperately return comments.Comment{ + UserID: cd.UserID, Token: cd.Token, ParentID: cd.ParentID, Comment: "", - PublicKey: cd.AuthorPublicKey, Signature: "", CommentID: cd.CommentID, Version: 0, @@ -387,26 +469,18 @@ func (p *commentsPlugin) commentAddSave(ca comments.CommentAdd) ([]byte, error) } // Prepare tlog args - tlogID := tlogIDForCommentState(ca.State) + tlogID := tlogIDFromCommentState(ca.State) + encrypt := encryptFromCommentState(ca.State) token, err := hex.DecodeString(ca.Token) if err != nil { return nil, err } - var encrypt bool - switch ca.State { - case comments.StateUnvetted: - encrypt = true - case comments.StateVetted: - encrypt = false - default: - return nil, fmt.Errorf("unknown state %v", ca.State) - } // Save blob merkles, err := p.tlog.save(tlogID, token, keyPrefixCommentAdd, [][]byte{b}, [][]byte{h}, encrypt) if err != nil { - return nil, fmt.Errorf("Save: %v", err) + return nil, err } if len(merkles) != 1 { return nil, fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", @@ -416,16 +490,17 @@ func (p *commentsPlugin) commentAddSave(ca comments.CommentAdd) ([]byte, error) return merkles[0], nil } -/* -func commentAdds(client *tlogbe.RecordClient, merkleHashes [][]byte) ([]comments.CommentAdd, error) { +// commentAdds returns the commentAdd for all specified merkle hashes. +func (p *commentsPlugin) commentAdds(s comments.StateT, token []byte, merkles [][]byte) ([]comments.CommentAdd, error) { // Retrieve blobs - blobs, err := client.BlobsByMerkleHash(merkleHashes) + tlogID := tlogIDFromCommentState(s) + blobs, err := p.tlog.blobsByMerkle(tlogID, token, merkles) if err != nil { return nil, err } - if len(blobs) != len(merkleHashes) { + if len(blobs) != len(merkles) { notFound := make([]string, 0, len(blobs)) - for _, v := range merkleHashes { + for _, v := range merkles { m := hex.EncodeToString(v) _, ok := blobs[m] if !ok { @@ -452,9 +527,9 @@ func commentAdds(client *tlogbe.RecordClient, merkleHashes [][]byte) ([]comments return adds, nil } -func commentDelSave(client *tlogbe.RecordClient, c comments.CommentDel) ([]byte, error) { +func (p *commentsPlugin) commentDelSave(cd comments.CommentDel) ([]byte, error) { // Prepare blob - be, err := convertBlobEntryFromCommentDel(c) + be, err := convertBlobEntryFromCommentDel(cd) if err != nil { return nil, err } @@ -467,11 +542,18 @@ func commentDelSave(client *tlogbe.RecordClient, c comments.CommentDel) ([]byte, return nil, err } + // Prepare tlog args + tlogID := tlogIDFromCommentState(cd.State) + token, err := hex.DecodeString(cd.Token) + if err != nil { + return nil, err + } + // Save blob - merkles, err := client.Save(keyPrefixCommentDel, + merkles, err := p.tlog.save(tlogID, token, keyPrefixCommentDel, [][]byte{b}, [][]byte{h}, false) if err != nil { - return nil, fmt.Errorf("Save: %v", err) + return nil, err } if len(merkles) != 1 { return nil, fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", @@ -481,15 +563,16 @@ func commentDelSave(client *tlogbe.RecordClient, c comments.CommentDel) ([]byte, return merkles[0], nil } -func commentDels(client *tlogbe.RecordClient, merkleHashes [][]byte) ([]comments.CommentDel, error) { +func (p *commentsPlugin) commentDels(s comments.StateT, token []byte, merkles [][]byte) ([]comments.CommentDel, error) { // Retrieve blobs - blobs, err := client.BlobsByMerkleHash(merkleHashes) + tlogID := tlogIDFromCommentState(s) + blobs, err := p.tlog.blobsByMerkle(tlogID, token, merkles) if err != nil { return nil, err } - if len(blobs) != len(merkleHashes) { + if len(blobs) != len(merkles) { notFound := make([]string, 0, len(blobs)) - for _, v := range merkleHashes { + for _, v := range merkles { m := hex.EncodeToString(v) _, ok := blobs[m] if !ok { @@ -516,9 +599,9 @@ func commentDels(client *tlogbe.RecordClient, merkleHashes [][]byte) ([]comments return dels, nil } -func commentVoteSave(client *tlogbe.RecordClient, c comments.CommentVote) ([]byte, error) { +func (p *commentsPlugin) commentVoteSave(cv comments.CommentVote) ([]byte, error) { // Prepare blob - be, err := convertBlobEntryFromCommentVote(c) + be, err := convertBlobEntryFromCommentVote(cv) if err != nil { return nil, err } @@ -531,11 +614,18 @@ func commentVoteSave(client *tlogbe.RecordClient, c comments.CommentVote) ([]byt return nil, err } + // Prepare tlog args + tlogID := tlogIDFromCommentState(cv.State) + token, err := hex.DecodeString(cv.Token) + if err != nil { + return nil, err + } + // Save blob - merkles, err := client.Save(keyPrefixCommentVote, + merkles, err := p.tlog.save(tlogID, token, keyPrefixCommentVote, [][]byte{b}, [][]byte{h}, false) if err != nil { - return nil, fmt.Errorf("Save: %v", err) + return nil, err } if len(merkles) != 1 { return nil, fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", @@ -545,75 +635,57 @@ func commentVoteSave(client *tlogbe.RecordClient, c comments.CommentVote) ([]byt return merkles[0], nil } -// indexAddCommentVote adds the provided comment vote to the index and -// calculates the new vote score. The updated index is returned. The effect of -// a new vote on a comment depends on the previous vote from that uuid. -// Example, a user upvotes a comment that they have already upvoted, the -// resulting vote score is 0 due to the second upvote removing the original -// upvote. -func indexAddCommentVote(idx index, cv comments.CommentVote, merkleHash []byte) index { - // Get the existing votes for this uuid - cidx := idx.Comments[cv.CommentID] - votes, ok := cidx.Votes[cv.UUID] - if !ok { - // This uuid has not cast any votes - votes = make([]voteIndex, 0, 1) +func (p *commentsPlugin) commentVotes(s comments.StateT, token []byte, merkles [][]byte) ([]comments.CommentVote, error) { + // Retrieve blobs + tlogID := tlogIDFromCommentState(s) + blobs, err := p.tlog.blobsByMerkle(tlogID, token, merkles) + if err != nil { + return nil, err } - - // Get the previous vote that this uuid made - var votePrev comments.VoteT - if len(votes) != 0 { - votePrev = votes[len(votes)-1].Vote + if len(blobs) != len(merkles) { + notFound := make([]string, 0, len(blobs)) + for _, v := range merkles { + m := hex.EncodeToString(v) + _, ok := blobs[m] + if !ok { + notFound = append(notFound, m) + } + } + return nil, fmt.Errorf("blobs not found: %v", notFound) } - // Update index vote score - voteNew := comments.VoteT(cv.Vote) - switch { - case votePrev == 0: - // No previous vote. Add the new vote to the score. - cidx.Score += int64(voteNew) - - case voteNew == votePrev: - // New vote is the same as the previous vote. Remove the previous - // vote from the score. - cidx.Score -= int64(votePrev) - - case voteNew != votePrev: - // New vote is different than the previous vote. Remove the - // previous vote from the score and add the new vote to the - // score. - cidx.Score -= int64(votePrev) - cidx.Score += int64(voteNew) + // Decode blobs + votes := make([]comments.CommentVote, 0, len(blobs)) + for _, v := range blobs { + be, err := store.Deblob(v) + if err != nil { + return nil, err + } + c, err := convertCommentVoteFromBlobEntry(*be) + if err != nil { + return nil, err + } + votes = append(votes, *c) } - // Update the index - votes = append(votes, voteIndex{ - Vote: comments.VoteT(cv.Vote), - MerkleHash: merkleHash, - }) - cidx.Votes[cv.UUID] = votes - idx.Comments[cv.CommentID] = cidx - - return idx + return votes, nil } - - -// commentsLatest returns the most recent version of the specified comments. -// Deleted comments are returned with limited data. Comment IDs that do not -// correspond to an actual comment are not included in the returned map. It is -// the responsibility of the caller to ensure a comment is returned for each of -// the provided comment IDs. The comments index that was looked up during this +// comments returns the most recent version of the specified comments. Deleted +// comments are returned with limited data. Comment IDs that do not correspond +// to an actual comment are not included in the returned map. It is the +// responsibility of the caller to ensure a comment is returned for each of the +// provided comment IDs. The comments index that was looked up during this // process is also returned. -func commentsLatest(client *tlogbe.RecordClient, idx index, commentIDs []uint32) (map[uint32]comments.Comment, error) { +func (p *commentsPlugin) comments(s comments.StateT, token []byte, idx commentsIndex, commentIDs []uint32) (map[uint32]comments.Comment, error) { // Aggregate the merkle hashes for all records that need to be // looked up. If a comment has been deleted then the only record // that will still exist is the comment del record. If the comment // has not been deleted then the comment add record will need to be // retrieved for the latest version of the comment. var ( - merklesAdd = make([][]byte, 0, len(commentIDs)) - merklesDel = make([][]byte, 0, len(commentIDs)) + merkleAdds = make([][]byte, 0, len(commentIDs)) + merkleDels = make([][]byte, 0, len(commentIDs)) ) for _, v := range commentIDs { cidx, ok := idx.Comments[v] @@ -624,33 +696,36 @@ func commentsLatest(client *tlogbe.RecordClient, idx index, commentIDs []uint32) // Comment del record if cidx.Del != nil { - merklesDel = append(merklesDel, cidx.Del) + merkleDels = append(merkleDels, cidx.Del) continue } // Comment add record version := commentVersionLatest(cidx) - merklesAdd = append(merklesAdd, cidx.Adds[version]) + merkleAdds = append(merkleAdds, cidx.Adds[version]) } // Get comment add records - adds, err := commentAdds(client, merklesAdd) + adds, err := p.commentAdds(s, token, merkleAdds) if err != nil { + if err == errRecordNotFound { + return nil, err + } return nil, fmt.Errorf("commentAdds: %v", err) } - if len(adds) != len(merklesAdd) { + if len(adds) != len(merkleAdds) { return nil, fmt.Errorf("wrong comment adds count; got %v, want %v", - len(adds), len(merklesAdd)) + len(adds), len(merkleAdds)) } // Get comment del records - dels, err := commentDels(client, merklesDel) + dels, err := p.commentDels(s, token, merkleDels) if err != nil { return nil, fmt.Errorf("commentDels: %v", err) } - if len(dels) != len(merklesDel) { + if len(dels) != len(merkleDels) { return nil, fmt.Errorf("wrong comment dels count; got %v, want %v", - len(dels), len(merklesDel)) + len(dels), len(merkleDels)) } // Prepare comments @@ -667,64 +742,6 @@ func commentsLatest(client *tlogbe.RecordClient, idx index, commentIDs []uint32) return cs, nil } -*/ - -func (p *commentsPlugin) commentsIndex(state comments.StateT, token []byte) (*commentsIndex, error) { - // Get all comment indexes - tlogID := tlogIDForCommentState(state) - blobs, err := p.tlog.blobsByKeyPrefix(tlogID, token, keyPrefixCommentsIndex) - if err != nil { - return nil, err - } - if len(blobs) == 0 { - // A comments index does not exist. This can happen when no - // comments have been made on the record yet. Return a new one. - return &commentsIndex{ - Comments: make(map[uint32]commentIndex), - }, nil - } - - // Decode the most recent index - b := blobs[len(blobs)-1] - be, err := store.Deblob(b) - if err != nil { - return nil, err - } - return convertCommentsIndexFromBlobEntry(*be) -} - -func (p *commentsPlugin) commentsIndexSave(token []byte, idx commentsIndex) error { - /* - // Prepare blob - be, err := convertBlobEntryFromCommentsIndex(idx) - if err != nil { - return err - } - h, err := hex.DecodeString(be.Hash) - if err != nil { - return err - } - b, err := store.Blobify(*be) - if err != nil { - return err - } - - // Prepare - - // Save blob - merkles, err := client.Save(keyPrefixCommentsIndex, - [][]byte{b}, [][]byte{h}, false) - if err != nil { - return fmt.Errorf("Save: %v", err) - } - if len(merkles) != 1 { - return fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", - len(merkles)) - } - */ - - return nil -} func (p *commentsPlugin) cmdNew(payload string) (string, error) { log.Tracef("comments cmdNew: %v", payload) @@ -735,6 +752,14 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { return "", err } + // Verify token + token, err := hex.DecodeString(n.Token) + if err != nil { + return "", comments.UserErrorReply{ + ErrorCode: comments.ErrorStatusTokenInvalid, + } + } + // Verify signature msg := strconv.Itoa(int(n.State)) + n.Token + strconv.FormatUint(uint64(n.ParentID), 10) + n.Comment @@ -743,34 +768,21 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { return "", convertCommentsErrorFromSignatureError(err) } - // Get record client - token, err := hex.DecodeString(n.Token) - if err != nil { + // Verify comment + if len(n.Comment) > comments.PolicyCommentLengthMax { return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusTokenInvalid, + ErrorCode: comments.ErrorStatusCommentLengthMax, } } - /* - client, err := p.backend.RecordClient(token) - if err != nil { - if err == backend.ErrRecordNotFound { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusRecordNotFound, - } - } - return "", err - } - */ - // The comments index must be pulled and updated. The record lock // must be held for the remainder of this function. m := p.mutex(n.Token) m.Lock() defer m.Unlock() - // Pull comments index - idx, err := p.commentsIndex(n.State, token) + // Get comments index + idx, err := p.commentsIndexLocked(n.State, token) if err != nil { return "", err } @@ -787,7 +799,8 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { // Setup comment receipt := p.id.SignMessage([]byte(n.Signature)) ca := comments.CommentAdd{ - UUID: n.UUID, + UserID: n.UserID, + State: n.State, Token: n.Token, ParentID: n.ParentID, Comment: n.Comment, @@ -802,7 +815,12 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { // Save comment merkleHash, err := p.commentAddSave(ca) if err != nil { - return "", fmt.Errorf("commentSave: %v", err) + if err == errRecordNotFound { + return "", comments.UserErrorReply{ + ErrorCode: comments.ErrorStatusRecordNotFound, + } + } + return "", fmt.Errorf("commentAddSave: %v", err) } // Update index @@ -815,9 +833,9 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { } // Save index - err = p.commentsIndexSave(token, *idx) + err = p.commentsIndexSaveLocked(n.State, token, *idx) if err != nil { - return "", fmt.Errorf("indexSave: %v", err) + return "", err } log.Debugf("Comment saved to record %v comment ID %v", @@ -837,46 +855,93 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { return string(reply), nil } -/* -// This function must be called WITH the record lock held. -func (p *commentsPlugin) edit(client *tlogbe.RecordClient, e comments.Edit, encrypt bool) (*comments.EditReply, error) { +func (p *commentsPlugin) cmdEdit(payload string) (string, error) { + log.Tracef("comments cmdEdit: %v", payload) + + // Decode payload + e, err := comments.DecodeEdit([]byte(payload)) + if err != nil { + return "", err + } + + // Verify token + token, err := hex.DecodeString(e.Token) + if err != nil { + return "", comments.UserErrorReply{ + ErrorCode: comments.ErrorStatusTokenInvalid, + } + } + + // Verify signature + msg := strconv.Itoa(int(e.State)) + e.Token + + strconv.FormatUint(uint64(e.ParentID), 10) + e.Comment + err = util.VerifySignature(e.Signature, e.PublicKey, msg) + if err != nil { + return "", convertCommentsErrorFromSignatureError(err) + } + + // Verify comment + if len(e.Comment) > comments.PolicyCommentLengthMax { + return "", comments.UserErrorReply{ + ErrorCode: comments.ErrorStatusCommentLengthMax, + } + } + + // The comments index must be pulled and updated. The record lock + // must be held for the remainder of this function. + m := p.mutex(e.Token) + m.Lock() + defer m.Unlock() + // Get comments index - idx, err := indexLatest(client) + idx, err := p.commentsIndexLocked(e.State, token) if err != nil { - return nil, fmt.Errorf("indexLatest: %v", err) + return "", err } // Get the existing comment - cs, err := commentsLatest(client, *idx, []uint32{e.CommentID}) + cs, err := p.comments(e.State, token, *idx, []uint32{e.CommentID}) if err != nil { - return nil, fmt.Errorf("commentsLatest %v: %v", e.CommentID, err) + return "", fmt.Errorf("comments %v: %v", e.CommentID, err) } existing, ok := cs[e.CommentID] if !ok { - return nil, comments.UserErrorReply{ + return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusCommentNotFound, } } - // Validate the comment edit. The parent ID must remain the same. - // The comment text must be different. + // Verify the user ID + if e.UserID != existing.UserID { + e := fmt.Sprintf("user id cannot change; got %v, want %v", + e.UserID, existing.UserID) + return "", comments.UserErrorReply{ + ErrorCode: comments.ErrorStatusUserIDInvalid, + ErrorContext: []string{e}, + } + } + + // Verify the parent ID if e.ParentID != existing.ParentID { e := fmt.Sprintf("parent id cannot change; got %v, want %v", e.ParentID, existing.ParentID) - return nil, comments.UserErrorReply{ + return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusParentIDInvalid, ErrorContext: []string{e}, } } + + // Verify comment changes if e.Comment == existing.Comment { - return nil, comments.UserErrorReply{ + return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusNoCommentChanges, } } // Create a new comment version receipt := p.id.SignMessage([]byte(e.Signature)) - c := comments.CommentAdd{ + ca := comments.CommentAdd{ + UserID: e.UserID, Token: e.Token, ParentID: e.ParentID, Comment: e.Comment, @@ -889,145 +954,130 @@ func (p *commentsPlugin) edit(client *tlogbe.RecordClient, e comments.Edit, encr } // Save comment - merkleHash, err := commentAddSave(client, c, encrypt) + merkle, err := p.commentAddSave(ca) if err != nil { - return nil, fmt.Errorf("commentSave: %v", err) + if err == errRecordNotFound { + return "", comments.UserErrorReply{ + ErrorCode: comments.ErrorStatusRecordNotFound, + } + } + return "", fmt.Errorf("commentSave: %v", err) } // Update index - idx.Comments[c.CommentID].Adds[c.Version] = merkleHash + idx.Comments[ca.CommentID].Adds[ca.Version] = merkle // Save index - err = indexSave(client, *idx) + err = p.commentsIndexSaveLocked(e.State, token, *idx) if err != nil { - return nil, fmt.Errorf("indexSave: %v", err) + return "", err } log.Debugf("Comment edited on record %v comment ID %v", - c.Token, c.CommentID) + ca.Token, ca.CommentID) - return &comments.EditReply{ - Version: c.Version, - Timestamp: c.Timestamp, - Receipt: c.Receipt, - }, nil + // Prepare reply + er := comments.EditReply{ + Version: ca.Version, + Timestamp: ca.Timestamp, + Receipt: ca.Receipt, + } + reply, err := comments.EncodeEditReply(er) + if err != nil { + return "", err + } + + return string(reply), nil } -func (p *commentsPlugin) cmdEdit(payload string) (string, error) { - log.Tracef("comments cmdEdit: %v", payload) +func (p *commentsPlugin) cmdDel(payload string) (string, error) { + log.Tracef("comments cmdDel: %v", payload) // Decode payload - e, err := comments.DecodeEdit([]byte(payload)) + d, err := comments.DecodeDel([]byte(payload)) if err != nil { return "", err } - // Verify signature - msg := e.Token + strconv.FormatUint(uint64(e.ParentID), 10) + e.Comment - err = util.VerifySignature(e.Signature, e.PublicKey, msg) - if err != nil { - return "", convertCommentsErrFromSignatureErr(err) - } - - // Get record client - token, err := hex.DecodeString(e.Token) + // Verify token + token, err := hex.DecodeString(d.Token) if err != nil { return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusTokenInvalid, } } - client, err := p.backend.RecordClient(token) + + // Verify signature + msg := strconv.Itoa(int(d.State)) + d.Token + + strconv.FormatUint(uint64(d.CommentID), 10) + d.Reason + err = util.VerifySignature(d.Signature, d.PublicKey, msg) if err != nil { - if err == backend.ErrRecordNotFound { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusRecordNotFound, - } - } - return "", err + return "", convertCommentsErrorFromSignatureError(err) } - // The existing comment must be pulled to validate the edit. The - // record lock must be held for the remainder of this function. - m := p.mutex(e.Token) + // The comments index must be pulled and updated. The record lock + // must be held for the remainder of this function. + m := p.mutex(d.Token) m.Lock() defer m.Unlock() - // Edit comment - var er *comments.EditReply - switch client.State { - case tlogbe.RecordStateUnvetted: - er, err = p.edit(client, *e, true) - if err != nil { - return "", err - } - case tlogbe.RecordStateVetted: - er, err = p.edit(client, *e, false) - if err != nil { - return "", err - } - default: - return "", fmt.Errorf("invalid record state %v", client.State) - } - - // Prepare reply - reply, err := comments.EncodeEditReply(*er) - if err != nil { - return "", err - } - - return string(reply), nil -} - -// This function must be called WITH the record lock held. -func (p *commentsPlugin) del(client *tlogbe.RecordClient, d comments.Del) (*comments.DelReply, error) { // Get comments index - idx, err := indexLatest(client) + idx, err := p.commentsIndexLocked(d.State, token) if err != nil { - return nil, fmt.Errorf("indexLatest: %v", err) + return "", err } - // Get comment - cs, err := commentsLatest(client, *idx, []uint32{d.CommentID}) + // Get the existing comment + cs, err := p.comments(d.State, token, *idx, []uint32{d.CommentID}) if err != nil { - return nil, fmt.Errorf("commentsLatest %v: %v", d.CommentID, err) + return "", fmt.Errorf("comments %v: %v", d.CommentID, err) } - comment, ok := cs[d.CommentID] + existing, ok := cs[d.CommentID] if !ok { - return nil, comments.UserErrorReply{ + return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusCommentNotFound, } } - // Save delete record + // Prepare comment delete receipt := p.id.SignMessage([]byte(d.Signature)) cd := comments.CommentDel{ - Token: d.Token, - CommentID: d.CommentID, - Reason: d.Reason, - PublicKey: d.PublicKey, - Signature: d.Signature, - ParentID: comment.ParentID, - AuthorPublicKey: comment.PublicKey, - Timestamp: time.Now().Unix(), - Receipt: hex.EncodeToString(receipt[:]), + Token: d.Token, + CommentID: d.CommentID, + Reason: d.Reason, + PublicKey: d.PublicKey, + Signature: d.Signature, + ParentID: existing.ParentID, + UserID: existing.UserID, + Timestamp: time.Now().Unix(), + Receipt: hex.EncodeToString(receipt[:]), } - merkleHash, err := commentDelSave(client, cd) + + // Save comment del + merkle, err := p.commentDelSave(cd) if err != nil { - return nil, fmt.Errorf("commentDelSave: %v", err) + if err == errRecordNotFound { + return "", comments.UserErrorReply{ + ErrorCode: comments.ErrorStatusRecordNotFound, + } + } + return "", fmt.Errorf("commentDelSave: %v", err) } // Update index cidx, ok := idx.Comments[d.CommentID] if !ok { - return nil, fmt.Errorf("comment not found in index: %v", d.CommentID) + // This should not be possible + e := fmt.Sprintf("comment not found in index: %v", d.CommentID) + panic(e) } - cidx.Del = merkleHash + cidx.Del = merkle idx.Comments[d.CommentID] = cidx // Save index - err = indexSave(client, *idx) + err = p.commentsIndexSaveLocked(d.State, token, *idx) if err != nil { - return nil, fmt.Errorf("indexSave: %v", err) + return "", err } // Delete all comment versions @@ -1035,64 +1085,162 @@ func (p *commentsPlugin) del(client *tlogbe.RecordClient, d comments.Del) (*comm for _, v := range cidx.Adds { merkles = append(merkles, v) } - err = client.Del(merkles) + tlogID := tlogIDFromCommentState(d.State) + err = p.tlog.del(tlogID, token, merkles) if err != nil { - return nil, fmt.Errorf("BlobsDel: %v", err) + return "", fmt.Errorf("del: %v", err) } - return &comments.DelReply{ + // Prepare reply + dr := comments.DelReply{ Timestamp: cd.Timestamp, Receipt: cd.Receipt, - }, nil + } + reply, err := comments.EncodeDelReply(dr) + if err != nil { + return "", err + } + + return string(reply), nil } -func (p *commentsPlugin) cmdDel(payload string) (string, error) { - log.Tracef("comments cmdDel: %v", payload) +func (p *commentsPlugin) cmdVote(payload string) (string, error) { + log.Tracef("comments cmdVote: %v", payload) // Decode payload - d, err := comments.DecodeDel([]byte(payload)) + v, err := comments.DecodeVote([]byte(payload)) if err != nil { return "", err } + // Verify token + token, err := hex.DecodeString(v.Token) + if err != nil { + return "", comments.UserErrorReply{ + ErrorCode: comments.ErrorStatusTokenInvalid, + } + } + + // Verify vote + switch v.Vote { + case comments.VoteDownvote, comments.VoteUpvote: + // These are allowed + default: + return "", comments.UserErrorReply{ + ErrorCode: comments.ErrorStatusVoteInvalid, + } + } + // Verify signature - msg := d.Token + strconv.FormatUint(uint64(d.CommentID), 10) + d.Reason - err = util.VerifySignature(d.Signature, d.PublicKey, msg) + msg := strconv.Itoa(int(v.State)) + v.Token + + strconv.FormatUint(uint64(v.CommentID), 10) + + strconv.FormatInt(int64(v.Vote), 10) + err = util.VerifySignature(v.Signature, v.PublicKey, msg) if err != nil { - return "", convertCommentsErrFromSignatureErr(err) + return "", convertCommentsErrorFromSignatureError(err) } - // Get record client - token, err := hex.DecodeString(d.Token) + // The comments index must be pulled and updated. The record lock + // must be held for the remainder of this function. + m := p.mutex(v.Token) + m.Lock() + defer m.Unlock() + + // Get comments index + idx, err := p.commentsIndexLocked(v.State, token) if err != nil { + return "", err + } + + // Verify comment exists + cidx, ok := idx.Comments[v.CommentID] + if !ok { return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusTokenInvalid, + ErrorCode: comments.ErrorStatusCommentNotFound, + } + } + + // Verify user has not exceeded max allowed vote changes + uvotes, ok := cidx.Votes[v.UserID] + if !ok { + uvotes = make([]voteIndex, 0) + } + if len(uvotes) > comments.PolicyVoteChangesMax { + return "", comments.UserErrorReply{ + ErrorCode: comments.ErrorStatusVoteChangesMax, } } - client, err := p.backend.RecordClient(token) + + // Verify user is not voting on their own comment + cs, err := p.comments(v.State, token, *idx, []uint32{v.CommentID}) if err != nil { - if err == backend.ErrRecordNotFound { + return "", fmt.Errorf("comments %v: %v", v.CommentID, err) + } + c, ok := cs[v.CommentID] + if !ok { + return "", fmt.Errorf("comment not found %v", v.CommentID) + } + if v.UserID == c.UserID { + return "", comments.UserErrorReply{ + ErrorCode: comments.ErrorStatusVoteInvalid, + ErrorContext: []string{"user cannot vote on their own comment"}, + } + } + + // Prepare comment vote + receipt := p.id.SignMessage([]byte(v.Signature)) + cv := comments.CommentVote{ + UserID: v.UserID, + Token: v.Token, + CommentID: v.CommentID, + Vote: int64(v.Vote), + PublicKey: v.PublicKey, + Signature: v.Signature, + Timestamp: time.Now().Unix(), + Receipt: hex.EncodeToString(receipt[:]), + } + + // Save comment vote + merkle, err := p.commentVoteSave(cv) + if err != nil { + if err == errRecordNotFound { return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusRecordNotFound, } } - return "", err + return "", fmt.Errorf("commentVoteSave: %v", err) } - // The comments index must be pulled and updated. The record lock - // must be held for the remainder of this function. - m := p.mutex(d.Token) - m.Lock() - defer m.Unlock() + // Add vote to the comment index + votes, ok := cidx.Votes[cv.UserID] + if !ok { + votes = make([]voteIndex, 0, 1) + } + votes = append(votes, voteIndex{ + Vote: comments.VoteT(cv.Vote), + Merkle: merkle, + }) + cidx.Votes[cv.UserID] = votes - // Delete comment - cr, err := p.del(client, *d) + // Update the comment vote score + cidx.Score = calcVoteScore(cidx, cv) + + // Update the comments index + idx.Comments[cv.CommentID] = cidx + + // Save index + err = p.commentsIndexSaveLocked(cv.State, token, *idx) if err != nil { return "", err } // Prepare reply - reply, err := comments.EncodeDelReply(*cr) + vr := comments.VoteReply{ + Timestamp: cv.Timestamp, + Receipt: cv.Receipt, + Score: cidx.Score, + } + reply, err := comments.EncodeVoteReply(vr) if err != nil { return "", err } @@ -1109,33 +1257,29 @@ func (p *commentsPlugin) cmdGet(payload string) (string, error) { return "", err } - // Get record client + // Verify token token, err := hex.DecodeString(g.Token) if err != nil { return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusTokenInvalid, } } - client, err := p.backend.RecordClient(token) - if err != nil { - if err == backend.ErrRecordNotFound { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusRecordNotFound, - } - } - return "", err - } // Get comments index - idx, err := indexLatest(client) + idx, err := p.commentsIndex(g.State, token) if err != nil { - return "", fmt.Errorf("indexLatest: %v", err) + return "", err } // Get comments - cs, err := commentsLatest(client, *idx, g.CommentIDs) + cs, err := p.comments(g.State, token, *idx, g.CommentIDs) if err != nil { - return "", fmt.Errorf("commentsLatest: %v", err) + if err == errRecordNotFound { + return "", comments.UserErrorReply{ + ErrorCode: comments.ErrorStatusRecordNotFound, + } + } + return "", fmt.Errorf("comments: %v", err) } // Prepare reply @@ -1159,27 +1303,18 @@ func (p *commentsPlugin) cmdGetAll(payload string) (string, error) { return "", err } - // Get record client + // Verify token token, err := hex.DecodeString(ga.Token) if err != nil { return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusTokenInvalid, } } - client, err := p.backend.RecordClient(token) - if err != nil { - if err == backend.ErrRecordNotFound { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusRecordNotFound, - } - } - return "", err - } // Get comments index - idx, err := indexLatest(client) + idx, err := p.commentsIndex(ga.State, token) if err != nil { - return "", fmt.Errorf("indexLatest: %v", err) + return "", err } // Compile comment IDs @@ -1189,9 +1324,14 @@ func (p *commentsPlugin) cmdGetAll(payload string) (string, error) { } // Get comments - c, err := commentsLatest(client, *idx, commentIDs) + c, err := p.comments(ga.State, token, *idx, commentIDs) if err != nil { - return "", fmt.Errorf("commentsLatest: %v", err) + if err == errRecordNotFound { + return "", comments.UserErrorReply{ + ErrorCode: comments.ErrorStatusRecordNotFound, + } + } + return "", fmt.Errorf("comments: %v", err) } // Convert comments from a map to a slice @@ -1226,27 +1366,18 @@ func (p *commentsPlugin) cmdGetVersion(payload string) (string, error) { return "", err } - // Get record client + // Verify token token, err := hex.DecodeString(gv.Token) if err != nil { return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusTokenInvalid, } } - client, err := p.backend.RecordClient(token) - if err != nil { - if err == backend.ErrRecordNotFound { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusRecordNotFound, - } - } - return "", err - } // Get comments index - idx, err := indexLatest(client) + idx, err := p.commentsIndex(gv.State, token) if err != nil { - return "", fmt.Errorf("indexLatest: %v", err) + return "", err } // Verify comment exists @@ -1262,7 +1393,7 @@ func (p *commentsPlugin) cmdGetVersion(payload string) (string, error) { ErrorContext: []string{"comment has been deleted"}, } } - merkleHash, ok := cidx.Adds[gv.Version] + merkle, ok := cidx.Adds[gv.Version] if !ok { e := fmt.Sprintf("comment %v does not have version %v", gv.CommentID, gv.Version) @@ -1273,7 +1404,7 @@ func (p *commentsPlugin) cmdGetVersion(payload string) (string, error) { } // Get comment add record - adds, err := commentAdds(client, [][]byte{merkleHash}) + adds, err := p.commentAdds(gv.State, token, [][]byte{merkle}) if err != nil { return "", fmt.Errorf("commentAdds: %v", err) } @@ -1307,27 +1438,18 @@ func (p *commentsPlugin) cmdCount(payload string) (string, error) { return "", err } - // Get record client + // Verify token token, err := hex.DecodeString(c.Token) if err != nil { return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusTokenInvalid, } } - client, err := p.backend.RecordClient(token) - if err != nil { - if err == backend.ErrRecordNotFound { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusRecordNotFound, - } - } - return "", err - } // Get comments index - idx, err := indexLatest(client) + idx, err := p.commentsIndex(c.State, token) if err != nil { - return "", fmt.Errorf("indexLatest: %v", err) + return "", err } // Prepare reply @@ -1342,125 +1464,102 @@ func (p *commentsPlugin) cmdCount(payload string) (string, error) { return string(reply), nil } -func (p *commentsPlugin) cmdVote(payload string) (string, error) { - log.Tracef("comments cmdVote: %v", payload) +func (p *commentsPlugin) cmdVotes(payload string) (string, error) { + log.Tracef("comments cmdVotes: %v", payload) // Decode payload - v, err := comments.DecodeVote([]byte(payload)) + v, err := comments.DecodeVotes([]byte(payload)) if err != nil { return "", err } - // Validate vote - switch v.Vote { - case comments.VoteDownvote: - // This is allowed - case comments.VoteUpvote: - // This is allowed - default: - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusVoteInvalid, - } - } - - // Validate signature - msg := v.Token + strconv.FormatUint(uint64(v.CommentID), 10) + - strconv.FormatInt(int64(v.Vote), 10) - err = util.VerifySignature(v.Signature, v.PublicKey, msg) - if err != nil { - return "", convertCommentsErrFromSignatureErr(err) - } - - // Get record client + // Verify token token, err := hex.DecodeString(v.Token) if err != nil { return "", comments.UserErrorReply{ ErrorCode: comments.ErrorStatusTokenInvalid, } } - client, err := p.backend.RecordClient(token) - if err != nil { - if err == backend.ErrRecordNotFound { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusRecordNotFound, - } - } - return "", err - } - - // The comments index must be pulled and updated. The record lock - // must be held for the remainder of this function. - m := p.mutex(v.Token) - m.Lock() - defer m.Unlock() // Get comments index - idx, err := indexLatest(client) + idx, err := p.commentsIndex(v.State, token) if err != nil { - return "", fmt.Errorf("indexLatest: %v", err) + return "", err } - // Verify comment exists - cidx, ok := idx.Comments[v.CommentID] - if !ok { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusCommentNotFound, + // Compile the comment vote merkles for all votes that were cast + // by the specified user. + merkles := make([][]byte, 0, 256) + for _, cidx := range idx.Comments { + voteIdxs, ok := cidx.Votes[v.UserID] + if !ok { + // User has not cast any votes for this comment + continue } - } - // Verify user has not exceeded max allowed vote changes - uvotes, ok := cidx.Votes[v.UUID] - if !ok { - uvotes = make([]voteIndex, 0) - } - if len(uvotes) > comments.PolicyMaxVoteChanges { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusMaxVoteChanges, + // User has cast votes on this comment + for _, vidx := range voteIdxs { + merkles = append(merkles, vidx.Merkle) } } - // Prepare comment vote - receipt := p.id.SignMessage([]byte(v.Signature)) - cv := comments.CommentVote{ - UUID: v.UUID, - Token: v.Token, - CommentID: v.CommentID, - Vote: int64(v.Vote), - PublicKey: v.PublicKey, - Signature: v.Signature, - Timestamp: time.Now().Unix(), - Receipt: hex.EncodeToString(receipt[:]), - } - - // Save comment vote - merkleHash, err := commentVoteSave(client, cv) + // Lookup votes + votes, err := p.commentVotes(v.State, token, merkles) if err != nil { - return "", fmt.Errorf("commentVoteSave: %v", err) - } - - // Update index - updatedIdx := indexAddCommentVote(*idx, cv, merkleHash) - - // Save index - err = indexSave(client, updatedIdx) - if err != nil { - return "", fmt.Errorf("indexSave: %v", err) + return "", fmt.Errorf("commentVotes: %v", err) } // Prepare reply - vr := comments.VoteReply{ - Timestamp: cv.Timestamp, - Receipt: cv.Receipt, - Score: updatedIdx.Comments[cv.CommentID].Score, + vr := comments.VotesReply{ + Votes: votes, } - reply, err := comments.EncodeVoteReply(vr) + reply, err := comments.EncodeVotesReply(vr) if err != nil { return "", err } return string(reply), nil } -*/ + +// calcVoteScore returns the updated vote score after the provided CommentVote +// has been added to it. +func calcVoteScore(cidx commentIndex, cv comments.CommentVote) int64 { + // Get the previous vote that this uuid made + var votePrev comments.VoteT + votes, ok := cidx.Votes[cv.UserID] + if !ok && len(votes) != 0 { + votePrev = votes[len(votes)-1].Vote + } + + // Get the existing score + score := cidx.Score + + // Update vote score. The effect of a new vote on a comment score + // depends on the previous vote from that uuid. Example, a user + // upvotes a comment that they have already upvoted, the resulting + // vote score is 0 due to the second upvote removing the original + // upvote. + voteNew := comments.VoteT(cv.Vote) + switch { + case votePrev == 0: + // No previous vote. Add the new vote to the score. + score += int64(voteNew) + + case voteNew == votePrev: + // New vote is the same as the previous vote. Remove the previous + // vote from the score. + score -= int64(votePrev) + + case voteNew != votePrev: + // New vote is different than the previous vote. Remove the + // previous vote from the score and add the new vote to the + // score. + score -= int64(votePrev) + score += int64(voteNew) + } + + return score +} // cmd executes a plugin command. // @@ -1471,22 +1570,22 @@ func (p *commentsPlugin) cmd(cmd, payload string) (string, error) { switch cmd { case comments.CmdNew: return p.cmdNew(payload) - /* - case comments.CmdEdit: - return p.cmdEdit(payload) - case comments.CmdDel: - return p.cmdDel(payload) - case comments.CmdGet: - return p.cmdGet(payload) - case comments.CmdGetAll: - return p.cmdGetAll(payload) - case comments.CmdGetVersion: - return p.cmdGetVersion(payload) - case comments.CmdCount: - return p.cmdCount(payload) - case comments.CmdVote: - return p.cmdVote(payload) - */ + case comments.CmdEdit: + return p.cmdEdit(payload) + case comments.CmdDel: + return p.cmdDel(payload) + case comments.CmdVote: + return p.cmdVote(payload) + case comments.CmdGet: + return p.cmdGet(payload) + case comments.CmdGetAll: + return p.cmdGetAll(payload) + case comments.CmdGetVersion: + return p.cmdGetVersion(payload) + case comments.CmdCount: + return p.cmdCount(payload) + case comments.CmdVotes: + return p.cmdVotes(payload) } return "", backend.ErrPluginCmdInvalid @@ -1525,11 +1624,13 @@ func (p *commentsPlugin) setup() error { func newCommentsPlugin(backend backend.Backend, tlog tlogClient, settings []backend.PluginSetting) *commentsPlugin { // TODO these should be passed in as plugin settings id := &identity.FullIdentity{} + dataDir := "" return &commentsPlugin{ id: id, backend: backend, tlog: tlog, + dataDir: dataDir, mutexes: make(map[string]*sync.Mutex), } } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote.go index 5e052abe7..a4b5e7f57 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote.go @@ -40,7 +40,9 @@ const ( // ticketVoteDirname is the ticket vote data directory name. ticketVoteDirname = "ticketvote" - // Filenames of memoized data saved to the data dir. + // Filenames of cached data saved to the plugin data dir. Brackets + // are used to indicate a variable that should be replaced in the + // filename. filenameSummary = "{token}-summary.json" // Blob entry data descriptors diff --git a/politeiad/backend/tlogbe/store/store.go b/politeiad/backend/tlogbe/store/store.go index 9b607d2fb..ecb68ac45 100644 --- a/politeiad/backend/tlogbe/store/store.go +++ b/politeiad/backend/tlogbe/store/store.go @@ -41,6 +41,16 @@ type BlobEntry struct { Data string `json:"data"` // Data payload, base64 encoded } +// NewBlobEntry returns a new BlobEntry. +func NewBlobEntry(dataHint, data []byte) BlobEntry { + return BlobEntry{ + Hash: hex.EncodeToString(util.Digest(data)), + DataHint: base64.StdEncoding.EncodeToString(dataHint), + Data: base64.StdEncoding.EncodeToString(data), + } +} + +// Blobify encodes the provided BlobEntry into a gzipped byte slice. func Blobify(be BlobEntry) ([]byte, error) { var b bytes.Buffer zw := gzip.NewWriter(&b) @@ -56,6 +66,7 @@ func Blobify(be BlobEntry) ([]byte, error) { return b.Bytes(), nil } +// Deblob decodes the provided gzipped byte slice into a BlobEntry. func Deblob(blob []byte) (*BlobEntry, error) { zr, err := gzip.NewReader(bytes.NewReader(blob)) if err != nil { @@ -70,14 +81,6 @@ func Deblob(blob []byte) (*BlobEntry, error) { return &be, nil } -func BlobEntryNew(dataHint, data []byte) BlobEntry { - return BlobEntry{ - Hash: hex.EncodeToString(util.Digest(data)), - DataHint: base64.StdEncoding.EncodeToString(dataHint), - Data: base64.StdEncoding.EncodeToString(data), - } -} - // Blob represents a blob key-value store. type Blob interface { // Put saves the provided blobs to the store. The keys for the diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index 169846ae9..d94c0be82 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -44,8 +44,8 @@ const ( // such as searching for a record index that has been buried by // thousands of leaves from plugin data. // TODO key prefix app-dataID: - // TODO the pluginID and the dataID should be passed into the tlog function - // instead of the keyPrefix + // TODO the leaf ExtraData field should be hinted. Similar to what + // we do for blobs. keyPrefixRecordIndex = "recordindex:" keyPrefixRecordContent = "record:" keyPrefixFreezeRecord = "freeze:" @@ -196,7 +196,7 @@ func convertBlobEntryFromFile(f backend.File) (*store.BlobEntry, error) { if err != nil { return nil, err } - be := store.BlobEntryNew(hint, data) + be := store.NewBlobEntry(hint, data) return &be, nil } @@ -213,7 +213,7 @@ func convertBlobEntryFromMetadataStream(ms backend.MetadataStream) (*store.BlobE if err != nil { return nil, err } - be := store.BlobEntryNew(hint, data) + be := store.NewBlobEntry(hint, data) return &be, nil } @@ -230,7 +230,7 @@ func convertBlobEntryFromRecordMetadata(rm backend.RecordMetadata) (*store.BlobE if err != nil { return nil, err } - be := store.BlobEntryNew(hint, data) + be := store.NewBlobEntry(hint, data) return &be, nil } @@ -247,7 +247,7 @@ func convertBlobEntryFromRecordIndex(ri recordIndex) (*store.BlobEntry, error) { if err != nil { return nil, err } - be := store.BlobEntryNew(hint, data) + be := store.NewBlobEntry(hint, data) return &be, nil } @@ -264,7 +264,7 @@ func convertBlobEntryFromFreezeRecord(fr freezeRecord) (*store.BlobEntry, error) if err != nil { return nil, err } - be := store.BlobEntryNew(hint, data) + be := store.NewBlobEntry(hint, data) return &be, nil } @@ -281,7 +281,7 @@ func convertBlobEntryFromAnchor(a anchor) (*store.BlobEntry, error) { if err != nil { return nil, err } - be := store.BlobEntryNew(hint, data) + be := store.NewBlobEntry(hint, data) return &be, nil } @@ -1570,6 +1570,11 @@ func (t *tlog) blobsDel(treeID int64, merkles [][]byte) error { // the responsibility of the caller to check that a blob is returned for each // of the provided merkle hashes. func (t *tlog) blobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, error) { + // Verify tree exists + if !t.treeExists(treeID) { + return nil, errRecordNotFound + } + // Get leaves leavesAll, err := t.trillian.leavesAll(treeID) if err != nil { @@ -1650,6 +1655,11 @@ func (t *tlog) blobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, // blobsByKeyPrefix returns all blobs that match the provided key prefix. func (t *tlog) blobsByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) { + // Verify tree exists + if !t.treeExists(treeID) { + return nil, errRecordNotFound + } + // Get leaves leaves, err := t.trillian.leavesAll(treeID) if err != nil { diff --git a/politeiad/backend/tlogbe/tlogclient.go b/politeiad/backend/tlogbe/tlogclient.go index b3fc9161e..3a12fa1de 100644 --- a/politeiad/backend/tlogbe/tlogclient.go +++ b/politeiad/backend/tlogbe/tlogclient.go @@ -74,7 +74,7 @@ func (c *backendClient) treeIDFromToken(tlogID string, token []byte) (int64, err // hashes of the data encoded in the blobs. The hashes must share the same // ordering as the blobs. func (c *backendClient) save(tlogID string, token []byte, keyPrefix string, blobs, hashes [][]byte, encrypt bool) ([][]byte, error) { - log.Tracef("save: %x %v %v %x", token, keyPrefix, encrypt, hashes) + log.Tracef("tlogClient save: %x %v %v %x", token, keyPrefix, encrypt, hashes) // Get tlog instance tlog, err := c.tlogByID(tlogID) @@ -94,7 +94,7 @@ func (c *backendClient) save(tlogID string, token []byte, keyPrefix string, blob // del deletes the blobs that correspond to the provided merkle leaf hashes. func (c *backendClient) del(tlogID string, token []byte, merkles [][]byte) error { - log.Tracef("del: %x %x", token, merkles) + log.Tracef("tlogClient del: %x %x", token, merkles) // Get tlog instance tlog, err := c.tlogByID(tlogID) @@ -119,7 +119,7 @@ func (c *backendClient) del(tlogID string, token []byte, merkles [][]byte) error // the responsibility of the caller to check that a blob is returned for each // of the provided merkle hashes. func (c *backendClient) blobsByMerkle(tlogID string, token []byte, merkles [][]byte) (map[string][]byte, error) { - log.Tracef("blobsByMerkle: %x %x", token, merkles) + log.Tracef("tlogClient blobsByMerkle: %x %x", token, merkles) // Get tlog instance tlog, err := c.tlogByID(tlogID) @@ -139,7 +139,7 @@ func (c *backendClient) blobsByMerkle(tlogID string, token []byte, merkles [][]b // blobsByKeyPrefix returns all blobs that match the provided key prefix. func (c *backendClient) blobsByKeyPrefix(tlogID string, token []byte, keyPrefix string) ([][]byte, error) { - log.Tracef("blobsByKeyPrefix: %x %x", token, keyPrefix) + log.Tracef("tlogClient blobsByKeyPrefix: %x %x", token, keyPrefix) // Get tlog instance tlog, err := c.tlogByID(tlogID) diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 3edd5fedc..8465d0972 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -19,6 +19,8 @@ type VoteT int const ( APIVersion = 1 + // TODO the plugin policies should be returned in a route + // Proposal routes RouteProposalNew = "/proposal/new" RouteProposalEdit = "/proposal/edit" @@ -484,9 +486,10 @@ type CommentsReply struct { Comments []Comment `json:"comments"` } -// UserCommentVote represents a comment vote made by a user. This struct -// contains all the information in a CommentVote and a CommentVoteReply. -type UserCommentVote struct { +// CommentVoteDetails represents all user generated data and server generated +// metadata for a comment vote. +type CommentVoteDetails struct { + UserID string `json:"userid"` State PropStateT `json:"state"` Token string `json:"token"` CommentID uint32 `json:"commentid"` @@ -497,7 +500,8 @@ type UserCommentVote struct { Receipt string `json:"receipt"` } -// CommentVotes returns all comment votes made a specific user on a proposal. +// CommentVotes returns all comment votes that meet the provided filtering +// criteria. type CommentVotes struct { State PropStateT `json:"state"` Token string `json:"token"` @@ -506,7 +510,7 @@ type CommentVotes struct { // CommentVotesReply is the reply to the CommentVotes command. type CommentVotesReply struct { - Votes []UserCommentVote `json:"votes"` + Votes []CommentVoteDetails `json:"votes"` } // VoteAuthorize authorizes a proposal vote or revokes a previous vote diff --git a/politeiawww/templates.go b/politeiawww/templates.go index 2d383f2fe..038f74050 100644 --- a/politeiawww/templates.go +++ b/politeiawww/templates.go @@ -6,7 +6,7 @@ package main import "text/template" -// Proposal submitted +// Proposal submitted - Send to admins type proposalSubmitted struct { Username string // Author username Name string // Proposal name @@ -23,7 +23,7 @@ A new proposal has been submitted on Politeia by {{.Username}}: var proposalSubmittedTmpl = template.Must( template.New("proposalSubmitted").Parse(proposalSubmittedText)) -// Proposal edited +// Proposal edited - Send to users type proposalEdited struct { Name string // Proposal name Version string // ProposalVersion @@ -161,7 +161,7 @@ type proposalVoteStarted struct { } const proposalVoteStartedText = ` -Voting has started for the following proposal on Politeia. +Voting has started on a Politeia proposal! {{.Name}} {{.Link}} From e3331e70dcab9ed57a5775edddd176d86017e845 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 21 Sep 2020 19:22:52 -0500 Subject: [PATCH 083/449] bug fixes --- politeiawww/api/pi/v1/v1.go | 2 +- politeiawww/piwww.go | 4 ++-- politeiawww/www.go | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 8465d0972..27228d366 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -142,7 +142,7 @@ const ( ErrorStatusCommentParentIDInvalid ErrorStatusCommentVoteInvalid ErrorStatusCommentNotFound - ErrorStatusCommentMaxVoteChanges + ErrorStatusCommentVoteChangesMax // Vote errors ErrorStatusVoteStatusInvalid diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index be476e38c..db1a08985 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -267,7 +267,7 @@ func convertPropStateFromComments(s comments.StateT) pi.PropStateT { func convertCommentFromPlugin(cm comments.Comment) pi.Comment { return pi.Comment{ - UserID: cm.UUID, + UserID: cm.UserID, Username: "", // Intentionally omitted, needs to be pulled from userdb State: convertPropStateFromComments(cm.State), Token: cm.Token, @@ -1595,7 +1595,7 @@ func (p *politeiawww) processComments(c pi.Comments) (*pi.CommentsReply, error) pic := convertCommentFromPlugin(cm) // Get comment's author username // Parse string uuid - uuid, err := uuid.Parse(cm.UUID) + uuid, err := uuid.Parse(cm.UserID) if err != nil { return nil, err } diff --git a/politeiawww/www.go b/politeiawww/www.go index f010ff32b..922252a38 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -110,7 +110,7 @@ func convertWWWErrorStatusFromComments(e comments.ErrorStatusT) www.ErrorStatusT // changes. case comments.ErrorStatusVoteInvalid: return www.ErrorStatusInvalidLikeCommentAction - case comments.ErrorStatusMaxVoteChanges: + case comments.ErrorStatusVoteChangesMax: return www.ErrorStatusInvalidLikeCommentAction } return www.ErrorStatusInvalid @@ -220,8 +220,8 @@ func convertPiErrorStatusFromComments(e comments.ErrorStatusT) pi.ErrorStatusT { return pi.ErrorStatusCommentTextInvalid case comments.ErrorStatusVoteInvalid: return pi.ErrorStatusCommentVoteInvalid - case comments.ErrorStatusMaxVoteChanges: - return pi.ErrorStatusCommentMaxVoteChanges + case comments.ErrorStatusVoteChangesMax: + return pi.ErrorStatusCommentVoteChangesMax } return pi.ErrorStatusInvalid } From f3cac5274ec9f0b511dbeed63e209c303dcfceec Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 22 Sep 2020 08:58:37 -0500 Subject: [PATCH 084/449] remove unused decredplugin commands --- decredplugin/decredplugin.go | 713 +----------------------------- politeiad/backend/gitbe/decred.go | 378 +--------------- politeiad/backend/gitbe/gitbe.go | 16 +- politeiawww/dcc.go | 34 +- politeiawww/decred.go | 32 -- politeiawww/invoices.go | 32 +- 6 files changed, 43 insertions(+), 1162 deletions(-) diff --git a/decredplugin/decredplugin.go b/decredplugin/decredplugin.go index ba764e299..c9518e5a8 100644 --- a/decredplugin/decredplugin.go +++ b/decredplugin/decredplugin.go @@ -1,3 +1,7 @@ +// Copyright (c) 2017-2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + package decredplugin import ( @@ -14,32 +18,19 @@ type VoteT int // Plugin settings, kinda doesn;t go here but for now it is fine const ( - Version = "1" - ID = "decred" - CmdAuthorizeVote = "authorizevote" - CmdStartVote = "startvote" - CmdStartVoteRunoff = "startvoterunoff" - CmdVoteDetails = "votedetails" - CmdVoteSummary = "votesummary" - CmdBatchVoteSummary = "batchvotesummary" - CmdLoadVoteResults = "loadvoteresults" - CmdBallot = "ballot" - CmdBestBlock = "bestblock" - CmdNewComment = "newcomment" - CmdLikeComment = "likecomment" - CmdCensorComment = "censorcomment" - CmdGetComment = "getcomment" - CmdGetComments = "getcomments" - CmdGetNumComments = "getnumcomments" - CmdProposalVotes = "proposalvotes" - CmdCommentLikes = "commentlikes" - CmdProposalCommentsLikes = "proposalcommentslikes" - CmdInventory = "inventory" - CmdTokenInventory = "tokeninventory" - CmdLinkedFrom = "linkedfrom" - MDStreamAuthorizeVote = 13 // Vote authorization by proposal author - MDStreamVoteBits = 14 // Vote bits and mask - MDStreamVoteSnapshot = 15 // Vote tickets and start/end parameters + Version = "1" + ID = "decred" + CmdAuthorizeVote = "authorizevote" + CmdStartVote = "startvote" + CmdStartVoteRunoff = "startvoterunoff" + CmdBallot = "ballot" + CmdBestBlock = "bestblock" + CmdNewComment = "newcomment" + CmdCensorComment = "censorcomment" + CmdGetComments = "getcomments" + MDStreamAuthorizeVote = 13 // Vote authorization by proposal author + MDStreamVoteBits = 14 // Vote bits and mask + MDStreamVoteSnapshot = 15 // Vote tickets and start/end parameters // Vote duration requirements for proposal votes (in blocks) VoteDurationMinMainnet = 2016 @@ -536,211 +527,6 @@ func DecodeStartVoteRunoffReply(payload []byte) (*StartVoteRunoffReply, error) { return &v, nil } -// VoteDetails is used to retrieve the voting period details for a record. -type VoteDetails struct { - Token string `json:"token"` // Censorship token -} - -// EncodeVoteDetails encodes VoteDetails into a JSON byte slice. -func EncodeVoteDetails(vd VoteDetails) ([]byte, error) { - return json.Marshal(vd) -} - -// DecodeVoteDetails decodes a JSON byte slice into a VoteDetails. -func DecodeVoteDetails(payload []byte) (*VoteDetails, error) { - var vd VoteDetails - - err := json.Unmarshal(payload, &vd) - if err != nil { - return nil, err - } - - return &vd, nil -} - -// VoteDetailsReply is the reply to VoteDetails. -type VoteDetailsReply struct { - AuthorizeVote AuthorizeVote `json:"authorizevote"` // Vote authorization - StartVote StartVote `json:"startvote"` // Vote ballot - StartVoteReply StartVoteReply `json:"startvotereply"` // Start vote snapshot -} - -// EncodeVoteDetailsReply encodes VoteDetailsReply into a JSON byte slice. -func EncodeVoteDetailsReply(vdr VoteDetailsReply) ([]byte, error) { - return json.Marshal(vdr) -} - -// DecodeVoteReply decodes a JSON byte slice into a VoteDetailsReply. -func DecodeVoteDetailsReply(payload []byte) (*VoteDetailsReply, error) { - var vdr VoteDetailsReply - - err := json.Unmarshal(payload, &vdr) - if err != nil { - return nil, err - } - - return &vdr, nil -} - -type VoteResults struct { - Token string `json:"token"` // Censorship token -} - -type VoteResultsReply struct { - CastVotes []CastVote `json:"castvotes"` // All votes -} - -// EncodeVoteResults encodes VoteResults into a JSON byte slice. -func EncodeVoteResults(v VoteResults) ([]byte, error) { - return json.Marshal(v) -} - -// DecodeVoteResults decodes a JSON byte slice into a VoteResults. -func DecodeVoteResults(payload []byte) (*VoteResults, error) { - var v VoteResults - - err := json.Unmarshal(payload, &v) - if err != nil { - return nil, err - } - - return &v, nil -} - -// EncodeVoteResultsReply encodes VoteResults into a JSON byte slice. -func EncodeVoteResultsReply(v VoteResultsReply) ([]byte, error) { - return json.Marshal(v) -} - -// DecodeVoteResultsReply decodes a JSON byte slice into a VoteResults. -func DecodeVoteResultsReply(payload []byte) (*VoteResultsReply, error) { - var v VoteResultsReply - - err := json.Unmarshal(payload, &v) - if err != nil { - return nil, err - } - - return &v, nil -} - -// VoteSummary requests a summary of a proposal vote. This includes certain -// voting period parameters and a summary of the vote results. -type VoteSummary struct { - Token string `json:"token"` // Censorship token - BestBlock uint64 `json:"bestblock"` // Best block -} - -// EncodeVoteSummary encodes VoteSummary into a JSON byte slice. -func EncodeVoteSummary(v VoteSummary) ([]byte, error) { - return json.Marshal(v) -} - -// DecodeVoteSummary decodes a JSON byte slice into a VoteSummary. -func DecodeVoteSummary(payload []byte) (*VoteSummary, error) { - var v VoteSummary - - err := json.Unmarshal(payload, &v) - if err != nil { - return nil, err - } - - return &v, nil -} - -// VoteOptionResult describes a vote option and the total number of votes that -// have been cast for this option. -type VoteOptionResult struct { - ID string `json:"id"` // Single unique word identifying vote (e.g. yes) - Description string `json:"description"` // Longer description of the vote. - Bits uint64 `json:"bits"` // Bits used for this option - Votes uint64 `json:"votes"` // Number of votes cast for this option -} - -// VoteSummaryReply is the reply to the VoteSummary command and returns certain -// voting period parameters as well as a summary of the vote results. -type VoteSummaryReply struct { - Authorized bool `json:"authorized"` // Vote is authorized - Type VoteT `json:"type"` // Vote type - Duration uint32 `json:"duration"` // Vote duration - EndHeight string `json:"endheight"` // End block height - EligibleTicketCount int `json:"eligibleticketcount"` // Number of eligible tickets - QuorumPercentage uint32 `json:"quorumpercentage"` // Percent of eligible votes required for quorum - PassPercentage uint32 `json:"passpercentage"` // Percent of total votes required to pass - Results []VoteOptionResult `json:"results"` // Vote results - Approved bool `json:"approved"` // Was vote approved -} - -// EncodeVoteSummaryReply encodes VoteSummary into a JSON byte slice. -func EncodeVoteSummaryReply(v VoteSummaryReply) ([]byte, error) { - return json.Marshal(v) -} - -// DecodeVoteSummaryReply decodes a JSON byte slice into a VoteSummaryReply. -func DecodeVoteSummaryReply(payload []byte) (*VoteSummaryReply, error) { - var v VoteSummaryReply - - err := json.Unmarshal(payload, &v) - if err != nil { - return nil, err - } - - return &v, nil -} - -// BatchVoteSummary requests a summary of a set of proposal votes. This -// includes certain voting period parameters and a summary of the vote -// results. -type BatchVoteSummary struct { - Tokens []string `json:"token"` // Censorship token - BestBlock uint64 `json:"bestblock"` // Best block -} - -// EncodeBatchVoteSummary encodes BatchVoteSummary into a JSON byte slice. -func EncodeBatchVoteSummary(v BatchVoteSummary) ([]byte, error) { - return json.Marshal(v) -} - -// DecodeBatchVoteSummary decodes a JSON byte slice into a BatchVoteSummary. -func DecodeBatchVoteSummary(payload []byte) (*BatchVoteSummary, error) { - var bv BatchVoteSummary - - err := json.Unmarshal(payload, &bv) - if err != nil { - return nil, err - } - - return &bv, nil -} - -// BatchVoteSummaryReply is the reply to the VoteSummary command and returns -// certain voting period parameters as well as a summary of the vote results. -// Results are returned for all tokens that correspond to a proposal. This -// includes both unvetted and vetted proposals. Tokens that do no correspond to -// a proposal are not included in the returned map. -type BatchVoteSummaryReply struct { - Summaries map[string]VoteSummaryReply `json:"summaries"` // Vote summaries -} - -// EncodeBatchVoteSummaryReply encodes BatchVoteSummaryReply into a JSON byte -// slice. -func EncodeBatchVoteSummaryReply(v BatchVoteSummaryReply) ([]byte, error) { - return json.Marshal(v) -} - -// DecodeBatchVoteSummaryReply decodes a JSON byte slice into a -// BatchVoteSummaryReply. -func DecodeBatchVoteSummaryReply(payload []byte) (*BatchVoteSummaryReply, error) { - var bv BatchVoteSummaryReply - - err := json.Unmarshal(payload, &bv) - if err != nil { - return nil, err - } - - return &bv, nil -} - // Comment is the structure that describes the full server side content. It // includes server side meta-data as well. Note that the receipt is the server // side. @@ -830,61 +616,6 @@ func DecodeNewCommentReply(payload []byte) (*NewCommentReply, error) { return &ncr, nil } -// LikeComment records an up or down vote from a user on a comment. -type LikeComment struct { - Token string `json:"token"` // Censorship token - CommentID string `json:"commentid"` // Comment ID - Action string `json:"action"` // Up or downvote (1, -1) - Signature string `json:"signature"` // Client Signature of Token+CommentID+Action - PublicKey string `json:"publickey"` // Pubkey used for Signature - - // Only used on disk - Receipt string `json:"receipt,omitempty"` // Signature of Signature - Timestamp int64 `json:"timestamp,omitempty"` // Received UNIX timestamp -} - -// EncodeLikeComment encodes LikeComment into a JSON byte slice. -func EncodeLikeComment(lc LikeComment) ([]byte, error) { - return json.Marshal(lc) -} - -// DecodeLikeComment decodes a JSON byte slice into a LikeComment. -func DecodeLikeComment(payload []byte) (*LikeComment, error) { - var lc LikeComment - - err := json.Unmarshal(payload, &lc) - if err != nil { - return nil, err - } - - return &lc, nil -} - -// LikeCommentReply returns the result of an up or down vote. -type LikeCommentReply struct { - Total uint64 `json:"total"` // Total number of up and down votes - Result int64 `json:"result"` // Current tally of likes, can be negative - Receipt string `json:"receipt"` // Server signature of client signature - Error string `json:"error,omitempty"` // Error if something wen't wrong during liking a comment -} - -// EncodeLikeCommentReply encodes LikeCommentReply into a JSON byte slice. -func EncodeLikeCommentReply(lcr LikeCommentReply) ([]byte, error) { - return json.Marshal(lcr) -} - -// DecodeLikeCommentReply decodes a JSON byte slice into a LikeCommentReply. -func DecodeLikeCommentReply(payload []byte) (*LikeCommentReply, error) { - var lcr LikeCommentReply - - err := json.Unmarshal(payload, &lcr) - if err != nil { - return nil, err - } - - return &lcr, nil -} - // CensorComment is a journal entry for a censored comment. The signature and // public key are from the admin that censored this comment. type CensorComment struct { @@ -935,53 +666,6 @@ func DecodeCensorCommentReply(payload []byte) (*CensorCommentReply, error) { return &ccr, nil } -// GetComment retrieves a single comment. The comment can be retrieved by -// either comment ID or by signature. -type GetComment struct { - Token string `json:"token"` // Proposal ID - CommentID string `json:"commentid,omitempty"` // Comment ID - Signature string `json:"signature,omitempty"` // Client signature -} - -// EncodeGetComment encodes a GetComment into a JSON byte slice. -func EncodeGetComment(gc GetComment) ([]byte, error) { - return json.Marshal(gc) -} - -// DecodeGetComment decodes a JSON byte slice into a GetComment. -func DecodeGetComment(payload []byte) (*GetComment, error) { - var gc GetComment - - err := json.Unmarshal(payload, &gc) - if err != nil { - return nil, err - } - - return &gc, nil -} - -// GetCommentReply returns the provided comment. -type GetCommentReply struct { - Comment Comment `json:"comment"` // Comment -} - -// EncodeGetCommentReply encodes a GetCommentReply into a JSON byte slice. -func EncodeGetCommentReply(gcr GetCommentReply) ([]byte, error) { - return json.Marshal(gcr) -} - -// DecodeGetCommentReply decodes a JSON byte slice into a GetCommentReply. -func DecodeGetCommentReply(payload []byte) (*GetCommentReply, error) { - var gcr GetCommentReply - - err := json.Unmarshal(payload, &gcr) - if err != nil { - return nil, err - } - - return &gcr, nil -} - // GetComments retrieve all comments for a given proposal. This call returns // the cooked comments; deleted/censored comments are not returned. type GetComments struct { @@ -1027,369 +711,6 @@ func DecodeGetCommentsReply(payload []byte) (*GetCommentsReply, error) { return &gcr, nil } -// GetNumComments returns a map that contains the number of comments for the -// provided list of censorship tokens. If a provided token does not corresond -// to an actual proposal then the token will not be included in the returned -// map. It is the responsibility of the caller to ensure that results are -// returned for all of the provided tokens. -type GetNumComments struct { - Tokens []string `json:"tokens"` // List of censorship tokens -} - -// EncodeGetNumComments encodes GetBatchComments into a JSON byte slice. -func EncodeGetNumComments(gnc GetNumComments) ([]byte, error) { - return json.Marshal(gnc) -} - -// DecodeGetNumComments decodes a JSON byte slice into a GetBatchComments. -func DecodeGetNumComments(payload []byte) (*GetNumComments, error) { - var gnc GetNumComments - - err := json.Unmarshal(payload, &gnc) - if err != nil { - return nil, err - } - - return &gnc, nil -} - -// GetNumCommentsReply is the reply to the GetNumComments command. -type GetNumCommentsReply struct { - NumComments map[string]int `json:"numcomments"` // [token]numComments -} - -// EncodeGetNumCommentsReply encodes GetNumCommentsReply into a -// JSON byte slice. -func EncodeGetNumCommentsReply(gncr GetNumCommentsReply) ([]byte, error) { - return json.Marshal(gncr) -} - -// DecodeGetNumCommentsReply decodes a JSON byte slice into a -// GetNumCommentsReply. -func DecodeGetNumCommentsReply(payload []byte) (*GetNumCommentsReply, error) { - var gncr GetNumCommentsReply - - err := json.Unmarshal(payload, &gncr) - if err != nil { - return nil, err - } - - return &gncr, nil -} - -// CommentLikes is used to retrieve all of the comment likes for a single -// record comment. -type CommentLikes struct { - Token string `json:"token"` // Censorship token - CommentID string `json:"commentid"` // Comment ID -} - -// EncodeCommentLikes encodes CommentLikes into a JSON byte slice. -func EncodeCommentLikes(gpcv CommentLikes) ([]byte, error) { - return json.Marshal(gpcv) -} - -// DecodeCommentLikes decodes a JSON byte slice into a CommentLikes. -func DecodeCommentLikes(payload []byte) (*CommentLikes, error) { - var cl CommentLikes - - err := json.Unmarshal(payload, &cl) - if err != nil { - return nil, err - } - - return &cl, nil -} - -// CommentLikesReply is the reply to CommentLikes and returns all of the -// upvote/downvote actions for the specified comment. -type CommentLikesReply struct { - CommentLikes []LikeComment `json:"commentlikes"` -} - -// EncodeCommentLikesReply encodes EncodeCommentLikesReply into a JSON byte -// slice. -func EncodeCommentLikesReply(clr CommentLikesReply) ([]byte, error) { - return json.Marshal(clr) -} - -// DecodeCommentLikesReply decodes a JSON byte slice into a CommentLikesReply. -func DecodeCommentLikesReply(payload []byte) (*CommentLikesReply, error) { - var clr CommentLikesReply - - err := json.Unmarshal(payload, &clr) - if err != nil { - return nil, err - } - - return &clr, nil -} - -// GetProposalCommentsLikes is a command to fetch all vote actions -// on the comments of a given proposal -type GetProposalCommentsLikes struct { - Token string `json:"token"` // Censorship token -} - -// EncodeGetProposalCommentsLikes encodes GetProposalCommentsLikes into a JSON byte slice. -func EncodeGetProposalCommentsLikes(gpcv GetProposalCommentsLikes) ([]byte, error) { - return json.Marshal(gpcv) -} - -// DecodeGetProposalCommentsLikes decodes a JSON byte slice into a GetProposalCommentsLikes. -func DecodeGetProposalCommentsLikes(payload []byte) (*GetProposalCommentsLikes, error) { - var gpcl GetProposalCommentsLikes - - err := json.Unmarshal(payload, &gpcl) - if err != nil { - return nil, err - } - - return &gpcl, nil -} - -// GetProposalCommentsLikesReply is a reply with all vote actions -// for the comments of a given proposal -type GetProposalCommentsLikesReply struct { - CommentsLikes []LikeComment `json:"commentslikes"` -} - -// EncodeGetProposalCommentsLikesReply encodes EncodeGetProposalCommentsLikesReply into a JSON byte slice. -func EncodeGetProposalCommentsLikesReply(gpclr GetProposalCommentsLikesReply) ([]byte, error) { - return json.Marshal(gpclr) -} - -// DecodeGetProposalCommentsLikesReply decodes a JSON byte slice into a GetProposalCommentsLikesReply. -func DecodeGetProposalCommentsLikesReply(payload []byte) (*GetProposalCommentsLikesReply, error) { - var gpclr GetProposalCommentsLikesReply - - err := json.Unmarshal(payload, &gpclr) - if err != nil { - return nil, err - } - - return &gpclr, nil -} - -// Inventory is used to retrieve the decred plugin inventory for all versions -// of the provided record tokens. If no tokens are provided, the decred plugin -// inventory for all versions of all records will be returned. -type Inventory struct { - Tokens []string `json:"tokens,omitempty"` -} - -// EncodeInventory encodes Inventory into a JSON byte slice. -func EncodeInventory(i Inventory) ([]byte, error) { - return json.Marshal(i) -} - -// DecodeInventory decodes a JSON byte slice into a Inventory. -func DecodeInventory(payload []byte) (*Inventory, error) { - var i Inventory - - err := json.Unmarshal(payload, &i) - if err != nil { - return nil, err - } - - return &i, nil -} - -// StartVoteTuple is used to return the StartVote and StartVoteReply for a -// record. StartVoteReply does not contain any record identifying data so it -// must be returned with the StartVote in order to know what record it belongs -// to. -type StartVoteTuple struct { - StartVote StartVote `json:"startvote"` // Start vote - StartVoteReply StartVoteReply `json:"startvotereply"` // Start vote reply -} - -// InventoryReply returns the decred plugin inventory. -type InventoryReply struct { - Comments []Comment `json:"comments"` // Comments - LikeComments []LikeComment `json:"likecomments"` // Like comments - AuthorizeVotes []AuthorizeVote `json:"authorizevotes"` // Authorize votes - AuthorizeVoteReplies []AuthorizeVoteReply `json:"authorizevotereplies"` // Authorize vote replies - StartVoteTuples []StartVoteTuple `json:"startvotetuples"` // Start vote tuples - CastVotes []CastVote `json:"castvotes"` // Cast votes -} - -// EncodeInventoryReply encodes a InventoryReply into a JSON byte slice. -func EncodeInventoryReply(ir InventoryReply) ([]byte, error) { - return json.Marshal(ir) -} - -// DecodeInventoryReply decodes a JSON byte slice into a inventory. -func DecodeInventoryReply(payload []byte) (*InventoryReply, error) { - var ir InventoryReply - - err := json.Unmarshal(payload, &ir) - if err != nil { - return nil, err - } - - return &ir, nil -} - -// TokenInventory requests the tokens of the records in the inventory, -// categorized by stage of the voting process. By default, only vetted -// records are returned. -type TokenInventory struct { - BestBlock uint64 `json:"bestblock"` // Best block - Unvetted bool `json:"unvetted"` // Include unvetted records -} - -// EncodeTokenInventory encodes a TokenInventory into a JSON byte slice. -func EncodeTokenInventory(i TokenInventory) ([]byte, error) { - return json.Marshal(i) -} - -// DecodeTokenInventory decodes a JSON byte slice into a TokenInventory. -func DecodeTokenInventory(payload []byte) (*TokenInventory, error) { - var i TokenInventory - - err := json.Unmarshal(payload, &i) - if err != nil { - return nil, err - } - - return &i, nil -} - -// TokenInventoryReply is the response to the TokenInventory command and -// returns the tokens of all records in the inventory. The tokens are -// categorized by stage of the voting process and are sorted according to -// the following rule. -// -// Sorted by record timestamp in descending order: -// Pre, Abandonded, Unreviewed, Censored -// -// Sorted by voting period end block height in descending order: -// Active, Approved, Rejected -type TokenInventoryReply struct { - // Vetted Records - Pre []string `json:"pre"` // Tokens of records that are pre-vote - Active []string `json:"active"` // Tokens of records with an active voting period - Approved []string `json:"approved"` // Tokens of records that have been approved by a vote - Rejected []string `json:"rejected"` // Tokens of records that have been rejected by a vote - Abandoned []string `json:"abandoned"` // Tokens of records that have been abandoned - - // Unvetted records - Unreviewed []string `json:"unreviewied"` // Tokens of records that are unreviewed - Censored []string `json:"censored"` // Tokens of records that have been censored -} - -// EncodeTokenInventoryReply encodes a TokenInventoryReply into a JSON byte -// slice. -func EncodeTokenInventoryReply(itr TokenInventoryReply) ([]byte, error) { - return json.Marshal(itr) -} - -// DecodeTokenInventoryReply decodes a JSON byte slice into a inventory. -func DecodeTokenInventoryReply(payload []byte) (*TokenInventoryReply, error) { - var itr TokenInventoryReply - - err := json.Unmarshal(payload, &itr) - if err != nil { - return nil, err - } - - return &itr, nil -} - -// LoadVoteResults creates a vote results entry in the cache for any proposals -// that have finsished voting but have not yet been added to the lazy loaded -// vote results table. -type LoadVoteResults struct { - BestBlock uint64 `json:"bestblock"` // Best block height -} - -// EncodeLoadVoteResults encodes a LoadVoteResults into a JSON byte slice. -func EncodeLoadVoteResults(lvr LoadVoteResults) ([]byte, error) { - return json.Marshal(lvr) -} - -// DecodeLoadVoteResults decodes a JSON byte slice into a LoadVoteResults. -func DecodeLoadVoteResults(payload []byte) (*LoadVoteResults, error) { - var lvr LoadVoteResults - - err := json.Unmarshal(payload, &lvr) - if err != nil { - return nil, err - } - - return &lvr, nil -} - -// LoadVoteResultsReply is the reply to the LoadVoteResults command. -type LoadVoteResultsReply struct{} - -// EncodeLoadVoteResultsReply encodes a LoadVoteResultsReply into a JSON -// byte slice. -func EncodeLoadVoteResultsReply(reply LoadVoteResultsReply) ([]byte, error) { - return json.Marshal(reply) -} - -// DecodeLoadVoteResultsReply decodes a JSON byte slice into a LoadVoteResults. -func DecodeLoadVoteResultsReply(payload []byte) (*LoadVoteResultsReply, error) { - var reply LoadVoteResultsReply - - err := json.Unmarshal(payload, &reply) - if err != nil { - return nil, err - } - - return &reply, nil -} - -// LinkedFrom returns a map[token][]token that contains the linked from list -// for each of the given proposal tokens. A linked from list is a list of all -// the proposals that have linked to a given proposal using the LinkTo field in -// the ProposalMetadata mdstream. If a token does not correspond to an actual -// proposal then it will not be included in the returned map. -type LinkedFrom struct { - Tokens []string `json:"tokens"` -} - -// EncodeLinkedFrom encodes a LinkedFrom into a JSON byte slice. -func EncodeLinkedFrom(lf LinkedFrom) ([]byte, error) { - return json.Marshal(lf) -} - -// DecodeLinkedFrom decodes a JSON byte slice into a LinkedFrom. -func DecodeLinkedFrom(payload []byte) (*LinkedFrom, error) { - var lf LinkedFrom - - err := json.Unmarshal(payload, &lf) - if err != nil { - return nil, err - } - - return &lf, nil -} - -// LinkedFromReply is the reply to the LinkedFrom command. -type LinkedFromReply struct { - LinkedFrom map[string][]string `json:"linkedfrom"` -} - -// EncodeLinkedFromReply encodes a LinkedFromReply into a JSON byte slice. -func EncodeLinkedFromReply(reply LinkedFromReply) ([]byte, error) { - return json.Marshal(reply) -} - -// DecodeLinkedFromReply decodes a JSON byte slice into a LinkedFrom. -func DecodeLinkedFromReply(payload []byte) (*LinkedFromReply, error) { - var reply LinkedFromReply - - err := json.Unmarshal(payload, &reply) - if err != nil { - return nil, err - } - - return &reply, nil -} - // BestBlock is a command to request the best block data. type BestBlock struct{} diff --git a/politeiad/backend/gitbe/decred.go b/politeiad/backend/gitbe/decred.go index 94d5bccb8..81a0641de 100644 --- a/politeiad/backend/gitbe/decred.go +++ b/politeiad/backend/gitbe/decred.go @@ -117,9 +117,8 @@ var ( pluginDataDir = filepath.Join("plugins", "decred") // Cached values, requires lock. These caches are built on startup. - decredPluginVotesCache = make(map[string]map[string]struct{}) // [token][ticket]struct{} - decredPluginCommentsCache = make(map[string]map[string]decredplugin.Comment) // [token][commentid]comment - decredPluginCommentsLikesCache = make(map[string][]decredplugin.LikeComment) // [token]LikeComment + decredPluginVotesCache = make(map[string]map[string]struct{}) // [token][ticket]struct{} + decredPluginCommentsCache = make(map[string]map[string]decredplugin.Comment) // [token][commentid]comment journalsReplayed bool = false ) @@ -1090,121 +1089,6 @@ func (g *gitBackEnd) pluginNewComment(payload string) (string, error) { return string(ncrb), nil } -// pluginLikeComment handles up and down votes of comments. -func (g *gitBackEnd) pluginLikeComment(payload string) (string, error) { - log.Tracef("pluginLikeComment") - - // Check if journals were replayed - if !journalsReplayed { - return "", backend.ErrJournalsNotReplayed - } - - // XXX this should become part of some sort of context - fiJSON, ok := decredPluginSettings[decredPluginIdentity] - if !ok { - return "", fmt.Errorf("full identity not set") - } - fi, err := identity.UnmarshalFullIdentity([]byte(fiJSON)) - if err != nil { - return "", err - } - - // Decode comment - like, err := decredplugin.DecodeLikeComment([]byte(payload)) - if err != nil { - return "", fmt.Errorf("DecodeLikeComment: %v", err) - } - - // Make sure action makes sense - if like.Action != "-1" && like.Action != "1" { - return "", fmt.Errorf("invalid action") - } - - // Verify proposal exists, we can run this lockless - if !g.vettedPropExists(like.Token) { - return "", fmt.Errorf("unknown proposal: %v", like.Token) - } - - // XXX make sure comment id exists and is in the right prop - - // Sign signature - r := fi.SignMessage([]byte(like.Signature)) - receipt := hex.EncodeToString(r[:]) - - // Comment journal filename - flushFilename := pijoin(g.journals, like.Token, - defaultCommentsFlushed) - - // Ensure proposal exists in comments cache - g.Lock() - - // Mark comment journal dirty - _ = os.Remove(flushFilename) - - // Verify cache - c, ok := decredPluginCommentsCache[like.Token][like.CommentID] - if !ok { - g.Unlock() - return "", fmt.Errorf("comment not found %v:%v", - like.Token, like.CommentID) - } - - cc := decredPluginCommentsLikesCache[like.Token] - - // Update cache - decredPluginCommentsLikesCache[like.Token] = append(cc, *like) - g.Unlock() - - // We create an unwind function that MUST be called from all error - // paths. If everything works ok it is a no-op. - unwind := func() { - g.Lock() - decredPluginCommentsLikesCache[like.Token] = cc - g.Unlock() - } - - // Create Journal entry - lc := decredplugin.LikeComment{ - Token: like.Token, - CommentID: like.CommentID, - Action: like.Action, - Signature: like.Signature, - PublicKey: like.PublicKey, - Receipt: receipt, - Timestamp: time.Now().Unix(), - } - blob, err := decredplugin.EncodeLikeComment(lc) - if err != nil { - unwind() - return "", fmt.Errorf("EncodeLikeComment: %v", err) - } - - // Add comment to journal - cfilename := pijoin(g.journals, like.Token, - defaultCommentFilename) - err = g.journal.Journal(cfilename, string(journalAddLike)+ - string(blob)) - if err != nil { - unwind() - return "", fmt.Errorf("could not journal %v: %v", lc.Token, err) - } - - // Encode reply - lcr := decredplugin.LikeCommentReply{ - Total: c.TotalVotes, - Result: c.ResultVotes, - Receipt: receipt, - } - lcrb, err := decredplugin.EncodeLikeCommentReply(lcr) - if err != nil { - unwind() - return "", fmt.Errorf("EncodeLikeCommentReply: %v", err) - } - - // return success and encoded answer - return string(lcrb), nil -} - func (g *gitBackEnd) pluginCensorComment(payload string) (string, error) { log.Tracef("pluginCensorComment") @@ -1377,7 +1261,6 @@ func (g *gitBackEnd) replayComments(token string) (map[string]decredplugin.Comme }() comments := make(map[string]decredplugin.Comment) - commentsLikes := make([]decredplugin.LikeComment, 0, 1024) for { err = g.journal.Replay(cfilename, func(s string) error { @@ -1431,16 +1314,6 @@ func (g *gitBackEnd) replayComments(token string) (map[string]decredplugin.Comme c.Censored = true comments[cc.CommentID] = c - case journalActionAddLike: - var lc decredplugin.LikeComment - err = d.Decode(&lc) - if err != nil { - return fmt.Errorf("journal addlike: %v", - err) - } - - commentsLikes = append(commentsLikes, lc) - default: return fmt.Errorf("invalid action: %v", action.Action) @@ -1456,32 +1329,11 @@ func (g *gitBackEnd) replayComments(token string) (map[string]decredplugin.Comme g.Lock() decredPluginCommentsCache[token] = comments - decredPluginCommentsLikesCache[token] = commentsLikes g.Unlock() return comments, nil } -// pluginGetProposalCommentLikes return all UserCommentVotes for a given proposal -func (g *gitBackEnd) pluginGetProposalCommentsLikes(payload string) (string, error) { - var gpclr decredplugin.GetProposalCommentsLikesReply - - gpcl, err := decredplugin.DecodeGetProposalCommentsLikes([]byte(payload)) - if err != nil { - return "", fmt.Errorf("DecodeGetProposalCommentsLikes: %v", err) - } - - g.Lock() - gpclr.CommentsLikes = decredPluginCommentsLikesCache[gpcl.Token] - g.Unlock() - - egpclr, err := decredplugin.EncodeGetProposalCommentsLikesReply(gpclr) - if err != nil { - return "", fmt.Errorf("EncodeGetProposalCommentsLikesReply: %v", err) - } - return string(egpclr), nil -} - func (g *gitBackEnd) pluginGetComments(payload string) (string, error) { log.Tracef("pluginGetComments") @@ -2695,229 +2547,3 @@ func (g *gitBackEnd) tallyVotes(token string) ([]decredplugin.CastVote, error) { return cv, nil } - -// pluginProposalVotes tallies all votes for a proposal. We can run the tally -// unlocked and just replay the journal. If the replay becomes an issue we -// could cache it. The Vote that is returned does have to be locked. -func (g *gitBackEnd) pluginProposalVotes(payload string) (string, error) { - log.Tracef("pluginProposalVotes: %v", payload) - - vote, err := decredplugin.DecodeVoteResults([]byte(payload)) - if err != nil { - return "", fmt.Errorf("DecodeVoteResults %v", err) - } - - // Verify proposal exists, we can run this lockless - if !g.vettedPropExists(vote.Token) { - return "", fmt.Errorf("proposal not found: %v", vote.Token) - } - - // This portion is must run locked - - g.Lock() - defer g.Unlock() - - if g.shutdown { - return "", backend.ErrShutdown - } - - // Prepare reply - var vrr decredplugin.VoteResultsReply - - // Fill out cast votes - vrr.CastVotes, err = g.tallyVotes(vote.Token) - if err != nil { - return "", fmt.Errorf("Could not tally votes: %v", err) - } - - // git checkout master - err = g.gitCheckout(g.vetted, "master") - if err != nil { - return "", err - } - - // Prepare reply - reply, err := decredplugin.EncodeVoteResultsReply(vrr) - if err != nil { - return "", fmt.Errorf("Could not encode VoteResultsReply: %v", - err) - } - - return string(reply), nil -} - -// pluginInventory returns the decred plugin inventory for the specified -// records. If no record tokens are specified then the decred plugin inventory -// for all vetted records will be returned. -func (g *gitBackEnd) pluginInventory(payload string) (string, error) { - log.Tracef("pluginInventory") - - inv, err := decredplugin.DecodeInventory([]byte(payload)) - if err != nil { - return "", err - } - - var tokens []string - if len(inv.Tokens) == 0 { - // No records specified. Return the decred plugin data for all - // vetted records. - g.Lock() - tokens, err = g.getVettedTokens() - g.Unlock() - if err != nil { - return "", err - } - } else { - // Records were specified. Only return the decred plugin data for - // the specified records. - tokens = inv.Tokens - } - - log.Debugf("Fetching decred plugin inventory for %x record", len(tokens)) - - // Convert tokens to a map - include := make(map[string]struct{}, len(tokens)) - for _, v := range tokens { - include[v] = struct{}{} - } - - // Compile decred plugin metadata streams for all versions of the - // specified records. - var ( - authVotes = make([]decredplugin.AuthorizeVote, 0, len(include)) - authVoteReplies = make([]decredplugin.AuthorizeVoteReply, 0, len(include)) - voteTuples = make([]decredplugin.StartVoteTuple, 0, len(include)) - ) - for token := range include { - tokenb, err := hex.DecodeString(token) - if err != nil { - return "", err - } - - // Find the most recent vesion number for this record - r, err := g.GetVetted(tokenb, "") - if err != nil { - return "", fmt.Errorf("GetVetted %v version 0: %v", token, err) - } - version, err := strconv.ParseUint(r.Version, 10, 64) - if err != nil { - return "", err - } - - // Compile decred plugin metadata streams from all versions of - // the record. - for version > 0 { - // Lookup record - r, err := g.GetVetted(tokenb, strconv.FormatUint(version, 10)) - if err != nil { - return "", fmt.Errorf("GetVetted %v version %v: %v", - token, version, err) - } - - // Check for decred plugin metadata streams - var svt decredplugin.StartVoteTuple - for _, v := range r.Metadata { - switch v.ID { - case decredplugin.MDStreamAuthorizeVote: - // Authorize vote - av, err := decredplugin.DecodeAuthorizeVote([]byte(v.Payload)) - if err != nil { - return "", err - } - avr := decredplugin.AuthorizeVoteReply{ - Action: av.Action, - RecordVersion: r.Version, - Receipt: av.Receipt, - Timestamp: av.Timestamp, - } - authVotes = append(authVotes, *av) - authVoteReplies = append(authVoteReplies, avr) - case decredplugin.MDStreamVoteBits: - // Start vote - sv, err := decredplugin.DecodeStartVote([]byte(v.Payload)) - if err != nil { - return "", err - } - svt.StartVote = *sv - case decredplugin.MDStreamVoteSnapshot: - // Start vote reply - svr, err := decredplugin.DecodeStartVoteReply([]byte(v.Payload)) - if err != nil { - return "", err - } - svt.StartVoteReply = *svr - } - // Check if this record version had vote metadata - if svt.StartVote.Version != 0 && svt.StartVoteReply.Version != 0 { - voteTuples = append(voteTuples, svt) - } - } - - // Decrement record version - version-- - } - } - - // Compile the journal data. This requires the lock. - g.Lock() - defer g.Unlock() - - // Compile comments and like comments. These can be pulled from the - // memory caches. - comments := make([]decredplugin.Comment, 0, 1024) - likeComments := make([]decredplugin.LikeComment, 0, 1024) - for token := range include { - // Comments - rcomments, ok := decredPluginCommentsCache[token] - if !ok { - continue - } - for _, v := range rcomments { - comments = append(comments, v) - } - - // Like comments - lc, ok := decredPluginCommentsLikesCache[token] - if !ok { - continue - } - likeComments = append(likeComments, lc...) - } - - // Compile cast votes - votes := make([]decredplugin.CastVote, 0, len(include)*41000) - for token := range include { - cv, err := g.tallyVotes(token) - if err != nil { - return "", fmt.Errorf("tallyVotes %v: %v", token, err) - } - votes = append(votes, cv...) - } - - // Prepare reply - ivr := decredplugin.InventoryReply{ - Comments: comments, - LikeComments: likeComments, - AuthorizeVotes: authVotes, - AuthorizeVoteReplies: authVoteReplies, - StartVoteTuples: voteTuples, - CastVotes: votes, - } - reply, err := decredplugin.EncodeInventoryReply(ivr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -// pluginLoadVoteResults is a pass through function. CmdLoadVoteResults does -// not require any work to be performed in gitBackEnd. -func (g *gitBackEnd) pluginLoadVoteResults() (string, error) { - r := decredplugin.LoadVoteResultsReply{} - reply, err := decredplugin.EncodeLoadVoteResultsReply(r) - if err != nil { - return "", err - } - return string(reply), nil -} diff --git a/politeiad/backend/gitbe/gitbe.go b/politeiad/backend/gitbe/gitbe.go index 747fd9f91..0aa7f82be 100644 --- a/politeiad/backend/gitbe/gitbe.go +++ b/politeiad/backend/gitbe/gitbe.go @@ -2818,6 +2818,7 @@ func (g *gitBackEnd) GetPlugins() ([]backend.Plugin, error) { func (g *gitBackEnd) Plugin(pluginID, command, commandID, payload string) (string, error) { log.Tracef("Plugin: %v", command) switch command { + // Decred plugin case decredplugin.CmdAuthorizeVote: return g.pluginAuthorizeVote(payload) case decredplugin.CmdStartVote: @@ -2826,24 +2827,16 @@ func (g *gitBackEnd) Plugin(pluginID, command, commandID, payload string) (strin return g.pluginStartVoteRunoff(payload) case decredplugin.CmdBallot: return g.pluginBallot(payload) - case decredplugin.CmdProposalVotes: - return g.pluginProposalVotes(payload) case decredplugin.CmdBestBlock: return g.pluginBestBlock() case decredplugin.CmdNewComment: return g.pluginNewComment(payload) - case decredplugin.CmdLikeComment: - return g.pluginLikeComment(payload) case decredplugin.CmdCensorComment: return g.pluginCensorComment(payload) case decredplugin.CmdGetComments: return g.pluginGetComments(payload) - case decredplugin.CmdProposalCommentsLikes: - return g.pluginGetProposalCommentsLikes(payload) - case decredplugin.CmdInventory: - return g.pluginInventory(payload) - case decredplugin.CmdLoadVoteResults: - return g.pluginLoadVoteResults() + + // CMS plugin case cmsplugin.CmdInventory: return g.pluginCMSInventory() case cmsplugin.CmdStartVote: @@ -2857,7 +2850,8 @@ func (g *gitBackEnd) Plugin(pluginID, command, commandID, payload string) (strin case cmsplugin.CmdVoteSummary: return g.pluginDCCVoteSummary(payload) } - return "", fmt.Errorf("invalid payload command") // XXX this needs to become a type error + + return "", fmt.Errorf("invalid payload command") } // Close shuts down the backend. It obtains the lock and sets the shutdown diff --git a/politeiawww/dcc.go b/politeiawww/dcc.go index 60b59e51e..a93e1512f 100644 --- a/politeiawww/dcc.go +++ b/politeiawww/dcc.go @@ -784,13 +784,20 @@ func (p *politeiawww) processNewCommentDCC(nc www.NewComment, u *user.User) (*ww } // Get comment - c, err := p.getDCCComment(nc.Token, ncr.CommentID) + comments, err := p.getDCCComments(nc.Token) if err != nil { - return nil, fmt.Errorf("getComment: %v", err) + return nil, fmt.Errorf("getComments: %v", err) + } + var c www.Comment + for _, v := range comments { + if v.CommentID == ncr.CommentID { + c = v + break + } } return &www.NewCommentReply{ - Comment: *c, + Comment: c, }, nil } @@ -854,27 +861,6 @@ func (p *politeiawww) getDCCComments(token string) ([]www.Comment, error) { return comments, nil } -// getDCCComment retrieves a comment from politeiad using the decred plugin -// command then fills in the missing user information. -func (p *politeiawww) getDCCComment(token, commentID string) (*www.Comment, error) { - // Fetch comment - dc, err := p.decredCommentGetByID(token, commentID) - if err != nil { - return nil, fmt.Errorf("decredGetComment: %v", err) - } - c := convertCommentFromDecred(*dc) - - // Lookup author info - u, err := p.db.UserGetByPubKey(c.PublicKey) - if err != nil { - return nil, fmt.Errorf("UserGetbyPubKey: %v", err) - } - c.UserID = u.ID.String() - c.Username = u.Username - - return &c, nil -} - func (p *politeiawww) processSetDCCStatus(sds cms.SetDCCStatus, u *user.User) (*cms.SetDCCStatusReply, error) { log.Tracef("processSetDCCStatus: %v", u.PublicKey()) diff --git a/politeiawww/decred.go b/politeiawww/decred.go index 298b04e5f..27730e46a 100644 --- a/politeiawww/decred.go +++ b/politeiawww/decred.go @@ -8,38 +8,6 @@ import ( "github.com/decred/politeia/decredplugin" ) -// decredGetComment sends the decred plugin getcomment command to the cache and -// returns the specified comment. -func (p *politeiawww) decredGetComment(gc decredplugin.GetComment) (*decredplugin.Comment, error) { - // Setup plugin command - payload, err := decredplugin.EncodeGetComment(gc) - if err != nil { - return nil, err - } - - // Execute plugin command - reply, err := p.pluginCommand(decredplugin.ID, decredplugin.CmdGetComment, - decredplugin.CmdGetComment, string(payload)) - - // Receive plugin command reply - gcr, err := decredplugin.DecodeGetCommentReply([]byte(reply)) - if err != nil { - return nil, err - } - - return &gcr.Comment, nil -} - -// decredCommentGetByID retrieves the specified decred plugin comment from the -// cache. -func (p *politeiawww) decredCommentGetByID(token, commentID string) (*decredplugin.Comment, error) { - gc := decredplugin.GetComment{ - Token: token, - CommentID: commentID, - } - return p.decredGetComment(gc) -} - // decredGetComments sends the decred plugin getcomments command to the cache // and returns all of the comments for the passed in proposal token. func (p *politeiawww) decredGetComments(token string) ([]decredplugin.Comment, error) { diff --git a/politeiawww/invoices.go b/politeiawww/invoices.go index 99f1602f7..d52a4833a 100644 --- a/politeiawww/invoices.go +++ b/politeiawww/invoices.go @@ -1661,10 +1661,17 @@ func (p *politeiawww) processNewCommentInvoice(nc www.NewComment, u *user.User) } // Get comment - c, err := p.getInvoiceComment(nc.Token, ncr.CommentID) + comments, err := p.getInvoiceComments(nc.Token) if err != nil { return nil, fmt.Errorf("getComment: %v", err) } + var c www.Comment + for _, v := range comments { + if v.CommentID == ncr.CommentID { + c = v + break + } + } if u.Admin { invoiceUser, err := p.db.UserGetByUsername(ir.Username) @@ -1680,7 +1687,7 @@ func (p *politeiawww) processNewCommentInvoice(nc www.NewComment, u *user.User) }) } return &www.NewCommentReply{ - Comment: *c, + Comment: c, }, nil } @@ -1763,27 +1770,6 @@ func (p *politeiawww) getInvoiceComments(token string) ([]www.Comment, error) { return comments, nil } -// getInvoiceComment retrieves an invoice comment from politeiad using the -// decred plugin command then fills in the missing user information. -func (p *politeiawww) getInvoiceComment(token, commentID string) (*www.Comment, error) { - // Fetch comment - dc, err := p.decredCommentGetByID(token, commentID) - if err != nil { - return nil, fmt.Errorf("decredGetComment: %v", err) - } - c := convertCommentFromDecred(*dc) - - // Lookup author info - u, err := p.db.UserGetByPubKey(c.PublicKey) - if err != nil { - return nil, fmt.Errorf("UserGetbyPubKey: %v", err) - } - c.UserID = u.ID.String() - c.Username = u.Username - - return &c, nil -} - // processPayInvoices looks for all approved invoices and then goes about // changing their statuses' to paid. func (p *politeiawww) processPayInvoices(u *user.User) (*cms.PayInvoicesReply, error) { From 58a36bce49db61b8bb2b0dbf5485384690343635 Mon Sep 17 00:00:00 2001 From: Amir Massarwa Date: Tue, 22 Sep 2020 18:33:33 +0300 Subject: [PATCH 085/449] politeiawww: Add processCommentVotes. --- plugins/comments/comments.go | 2 +- politeiawww/cmd/piwww/commentvote.go | 25 +------- politeiawww/cmd/piwww/commentvotes.go | 46 +++++++++++++++ politeiawww/cmd/piwww/help.go | 4 +- politeiawww/cmd/piwww/piwww.go | 8 +-- politeiawww/cmd/piwww/userlikecomments.go | 45 -------------- politeiawww/cmd/shared/client.go | 21 ++++--- politeiawww/comments.go | 20 +++++++ politeiawww/piwww.go | 71 +++++++++++++++++++++++ 9 files changed, 156 insertions(+), 86 deletions(-) create mode 100644 politeiawww/cmd/piwww/commentvotes.go delete mode 100644 politeiawww/cmd/piwww/userlikecomments.go diff --git a/plugins/comments/comments.go b/plugins/comments/comments.go index fd4124d05..ea9b67648 100644 --- a/plugins/comments/comments.go +++ b/plugins/comments/comments.go @@ -167,7 +167,7 @@ type CommentVote struct { State StateT `json:"state"` // Record state Token string `json:"token"` // Record token CommentID uint32 `json:"commentid"` // Comment ID - Vote int64 `json:"vote"` // Upvote or downvote + Vote VoteT `json:"vote"` // Upvote or downvote PublicKey string `json:"publickey"` // Public key used for signature Signature string `json:"signature"` // Client signature diff --git a/politeiawww/cmd/piwww/commentvote.go b/politeiawww/cmd/piwww/commentvote.go index 45eb89bf7..40b16a681 100644 --- a/politeiawww/cmd/piwww/commentvote.go +++ b/politeiawww/cmd/piwww/commentvote.go @@ -21,10 +21,6 @@ type CommentVoteCmd struct { CommentID string `positional-arg-name:"commentID"` // Comment ID Action string `positional-arg-name:"action"` // Upvote/downvote action } `positional-args:"true" required:"true"` - - // CLI flags - Vetted bool `long:"vetted" optional:"true"` - Unvetted bool `long:"unvetted" optional:"true"` } // Execute executes the like comment command. @@ -36,19 +32,6 @@ func (cmd *CommentVoteCmd) Execute(args []string) error { commentID := cmd.Args.CommentID action := cmd.Args.Action - // Verify state - var state pi.PropStateT - switch { - case cmd.Vetted && cmd.Unvetted: - return fmt.Errorf("cannot use --vetted and --unvetted simultaneously") - case cmd.Unvetted: - state = pi.PropStateUnvetted - case cmd.Vetted: - state = pi.PropStateVetted - default: - return fmt.Errorf("must specify either --vetted or unvetted") - } - // Validate action if action != actionUpvote && action != actionDownvote { return fmt.Errorf("invalid action %s; the action must be either "+ @@ -69,7 +52,7 @@ func (cmd *CommentVoteCmd) Execute(args []string) error { vote = pi.CommentVoteDownvote } - sig := cfg.Identity.SignMessage([]byte(string(state) + token + commentID + + sig := cfg.Identity.SignMessage([]byte(string(pi.PropStateVetted) + token + commentID + string(vote))) // Parse provided parent id ciUint, err := strconv.ParseUint(commentID, 10, 32) @@ -78,7 +61,7 @@ func (cmd *CommentVoteCmd) Execute(args []string) error { } cv := &pi.CommentVote{ Token: token, - State: state, + State: pi.PropStateVetted, CommentID: uint32(ciUint), Vote: vote, Signature: hex.EncodeToString(sig[:]), @@ -111,8 +94,4 @@ Arguments: 1. token (string, required) Proposal censorship token 2. commentID (string, required) Id of the comment 3. action (string, required) Vote (upvote or downvote) - -Flags: - --vetted (bool, optional) Comment's record is vetted. - --unvetted (bool, optional) Comment's record is unvetted. ` diff --git a/politeiawww/cmd/piwww/commentvotes.go b/politeiawww/cmd/piwww/commentvotes.go new file mode 100644 index 000000000..b395ff7f3 --- /dev/null +++ b/politeiawww/cmd/piwww/commentvotes.go @@ -0,0 +1,46 @@ +// Copyright (c) 2017-2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + pi "github.com/decred/politeia/politeiawww/api/pi/v1" + "github.com/decred/politeia/politeiawww/cmd/shared" +) + +// CommentVotesCmd retreives like comment objects for +// the specified proposal from the provided user. +type CommentVotesCmd struct { + Args struct { + Token string `positional-arg-name:"token"` // Censorship token + UserID string `positional-arg-name:"userid"` // User id + } `positional-args:"true" required:"true"` +} + +// Execute executes the user comment likes command. +func (cmd *CommentVotesCmd) Execute(args []string) error { + token := cmd.Args.Token + userID := cmd.Args.UserID + + cvr, err := client.CommentVotes(pi.CommentVotes{ + Token: token, + State: pi.PropStateVetted, + UserID: userID, + }) + if err != nil { + return err + } + return shared.PrintJSON(cvr) +} + +// commentVotesHelpMsg is the output for the help command when +// 'commentvotes' is specified. +const commentVotesHelpMsg = `commentvotes "token" "userid" + +Get the provided user comment upvote/downvotes for a proposal. + +Arguments: +1. token (string, required) Proposal censorship token +2. userid (string, required) User id +` diff --git a/politeiawww/cmd/piwww/help.go b/politeiawww/cmd/piwww/help.go index ceb3cb02e..3f0475ea7 100644 --- a/politeiawww/cmd/piwww/help.go +++ b/politeiawww/cmd/piwww/help.go @@ -95,8 +95,8 @@ func (cmd *HelpCmd) Execute(args []string) error { fmt.Printf("%s\n", commentCensorHelpMsg) case "commentvote": fmt.Printf("%s\n", commentVoteHelpMsg) - case "userlikecomments": - fmt.Printf("%s\n", userLikeCommentsHelpMsg) + case "commentvotes": + fmt.Printf("%s\n", commentVotesHelpMsg) // Vote commands case "authorizevote": diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index e47c9b942..f19b3d9f5 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -50,10 +50,11 @@ type piwww struct { ProposalSetStatus ProposalSetStatusCmd `command:"proposalsetstatus"` // Comments commands - CommentNew CommentNewCmd `command:"commentnew" description:"(user) create a new comment"` - CommentVote CommentVoteCmd `command:"commentvote" description:"(user) upvote/downvote a comment"` - CommentCensor CommentCensorCmd `command:"commentcensor" description:"(admin) censor a comment"` + CommentNew CommentNewCmd `command:"commentnew" description:"(user) create a new comment"` + CommentVote CommentVoteCmd `command:"commentvote" description:"(user) upvote/downvote a comment"` + CommentCensor CommentCensorCmd `command:"commentcensor" description:"(admin) censor a comment"` Comments CommentsCmd `command:"comments" description:"(public) get the comments for a proposal"` + CommentVotes CommentVotesCmd `command:"commentvotes" description:"(user) get comment upvotes/downvotes for a proposal from the provided user"` // Commands ActiveVotes ActiveVotesCmd `command:"activevotes" description:"(public) get the proposals that are being voted on"` @@ -87,7 +88,6 @@ type piwww struct { TokenInventory shared.TokenInventoryCmd `command:"tokeninventory" description:"(public) get the censorship record tokens of all proposals"` UpdateUserKey shared.UpdateUserKeyCmd `command:"updateuserkey" description:"(user) generate a new identity for the logged in user"` UserDetails UserDetailsCmd `command:"userdetails" description:"(public) get the details of a user profile"` - UserLikeComments UserLikeCommentsCmd `command:"userlikecomments" description:"(user) get the logged in user's comment upvotes/downvotes for a proposal"` UserPendingPayment UserPendingPaymentCmd `command:"userpendingpayment" description:"(user) get details for a pending payment for the logged in user"` UserProposals UserProposalsCmd `command:"userproposals" description:"(public) get all proposals submitted by a specific user"` Users shared.UsersCmd `command:"users" description:"(public) get a list of users"` diff --git a/politeiawww/cmd/piwww/userlikecomments.go b/politeiawww/cmd/piwww/userlikecomments.go deleted file mode 100644 index f6cc681ac..000000000 --- a/politeiawww/cmd/piwww/userlikecomments.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import "github.com/decred/politeia/politeiawww/cmd/shared" - -// UserLikeCommentsCmd retreives the logged in user's like comment objects for -// the specified proposal. -type UserLikeCommentsCmd struct { - Args struct { - Token string `positional-arg-name:"token"` // Censorship token - } `positional-args:"true" required:"true"` -} - -// Execute executes the user comment likes command. -func (cmd *UserLikeCommentsCmd) Execute(args []string) error { - cvr, err := client.UserCommentsLikes(cmd.Args.Token) - if err != nil { - return err - } - return shared.PrintJSON(cvr) -} - -// userLikeCommentsHelpMsg is the output for the help command when -// 'userlikecomments' is specified. -const userLikeCommentsHelpMsg = `userlikecomments "token" - -Get the logged in user's comment upvote/downvotes for a proposal. - -Arguments: -1. token (string, required) Proposal censorship token - -Result: - -{ - "commentslikes": [ - { - "action": (string) Vote (upvote or downvote) - "commentid" (string) ID of the comment - "token": (string) Proposal censorship token - }, - ] -}` diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index 7b88d76be..9e4ea44d3 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -1231,30 +1231,29 @@ func (c *Client) InvoiceComments(token string) (*www.GetCommentsReply, error) { return &gcr, nil } -// UserCommentsLikes retrieves the comment likes (upvotes/downvotes) for the -// specified proposal that are from the logged in user. -func (c *Client) UserCommentsLikes(token string) (*www.UserCommentsLikesReply, error) { - route := "/user/proposals/" + token + "/commentslikes" - responseBody, err := c.makeRequest(http.MethodGet, - www.PoliteiaWWWAPIRoute, route, nil) +// CommentVotes retrieves the comment likes (upvotes/downvotes) for the +// specified proposal that are from the privoded user. +func (c *Client) CommentVotes(cv pi.CommentVotes) (*pi.CommentVotesReply, error) { + responseBody, err := c.makeRequest(http.MethodPost, + pi.APIRoute, pi.RouteCommentVotes, cv) if err != nil { return nil, err } - var uclr www.UserCommentsLikesReply - err = json.Unmarshal(responseBody, &uclr) + var cvr pi.CommentVotesReply + err = json.Unmarshal(responseBody, &cvr) if err != nil { - return nil, fmt.Errorf("unmarshal UserCommentsLikesReply: %v", err) + return nil, fmt.Errorf("unmarshal CommentVotes: %v", err) } if c.cfg.Verbose { - err := prettyPrintJSON(uclr) + err := prettyPrintJSON(cvr) if err != nil { return nil, err } } - return &uclr, nil + return &cvr, nil } // CommentVote casts a like comment action (upvote/downvote) for the logged in diff --git a/politeiawww/comments.go b/politeiawww/comments.go index fb2853b1c..64777427d 100644 --- a/politeiawww/comments.go +++ b/politeiawww/comments.go @@ -31,6 +31,26 @@ func (p *politeiawww) commentCensor(cc comments.Del) (*comments.DelReply, error) return ccr, nil } +func (p *politeiawww) commentVotes(vs comments.Votes) (*comments.VotesReply, error) { + // Prep plugin payload + payload, err := comments.EncodeVotes(vs) + if err != nil { + return nil, err + } + + r, err := p.pluginCommand(comments.ID, comments.CmdVotes, "", + string(payload)) + if err != nil { + return nil, err + } + vsr, err := comments.DecodeVotesReply([]byte(r)) + if err != nil { + return nil, err + } + + return vsr, nil +} + // comments calls the comments plugin to get record's comments. func (p *politeiawww) comments(cp comments.GetAll) (*comments.GetAllReply, error) { // Prep plugin payload diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index db1a08985..999a8acfd 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -285,6 +285,30 @@ func convertCommentFromPlugin(cm comments.Comment) pi.Comment { } } +func convertVoteFromComments(v comments.VoteT) pi.CommentVoteT { + switch v { + case comments.VoteDownvote: + return pi.CommentVoteDownvote + case comments.VoteUpvote: + return pi.CommentVoteUpvote + } + return pi.CommentVoteInvalid +} + +func convertCommentVoteFromPlugin(cv comments.CommentVote) pi.CommentVoteDetails { + return pi.CommentVoteDetails{ + UserID: cv.UserID, + State: convertPropStateFromComments(cv.State), + Token: cv.Token, + CommentID: cv.CommentID, + Vote: convertVoteFromComments(cv.Vote), + PublicKey: cv.PublicKey, + Signature: cv.Signature, + Timestamp: cv.Timestamp, + Receipt: cv.Receipt, + } +} + func convertFilesFromPD(f []pd.File) ([]pi.File, []pi.Metadata) { files := make([]pi.File, 0, len(f)) metadata := make([]pi.Metadata, 0, len(f)) @@ -1632,6 +1656,50 @@ func (p *politeiawww) handleComments(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, cr) } +func (p *politeiawww) processCommentVotes(cvs pi.CommentVotes) (*pi.CommentVotesReply, error) { + log.Tracef("processCommentVotes: %v %v", cvs.Token, cvs.UserID) + + // Call comments plugin to get filtered record's comment votes + reply, err := p.commentVotes(comments.Votes{ + Token: cvs.Token, + State: convertCommentsPluginPropStateFromPi(cvs.State), + UserID: cvs.UserID, + }) + if err != nil { + return nil, err + } + + // Translate to pi + var cvsr pi.CommentVotesReply + ucvs := make([]pi.CommentVoteDetails, 0, len(reply.Votes)) + for _, cv := range reply.Votes { + ucvs = append(ucvs, convertCommentVoteFromPlugin(cv)) + } + cvsr.Votes = ucvs + + return &cvsr, nil +} + +func (p *politeiawww) handleCommentVotes(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleCommentVotes") + + var cvs pi.CommentVotes + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&cvs); err != nil { + respondWithPiError(w, r, "handleCommentVotes: unmarshal", + pi.UserErrorReply{}) + return + } + + cvsr, err := p.processCommentVotes(cvs) + if err != nil { + respondWithPiError(w, r, + "handleCommentVotes: processCommentVotes: %v", err) + } + + util.RespondWithJSON(w, http.StatusOK, cvsr) +} + func (p *politeiawww) processCommentCensor(cc pi.CommentCensor, usr user.User) (*pi.CommentCensorReply, error) { log.Tracef("processCommentCensor: %v %v", cc.Token, cc.CommentID) @@ -1722,6 +1790,9 @@ func (p *politeiawww) setPiRoutes() { p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteCommentVote, p.handleCommentVote, permissionLogin) + p.addRoute(http.MethodPost, pi.APIRoute, + pi.RouteCommentVotes, p.handleCommentVotes, permissionLogin) + // Admin routes p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteCommentCensor, p.handleCommentCensor, permissionAdmin) From 37c1b752edb53079e601c98eb4cfe627af86cb28 Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Tue, 22 Sep 2020 15:04:50 -0300 Subject: [PATCH 086/449] multi: Return record on politeiad api calls. --- politeiad/api/v1/v1.go | 8 +++++--- politeiad/politeiad.go | 11 +++-------- politeiawww/eventmanager.go | 1 + politeiawww/piwww.go | 18 ++++++++++-------- politeiawww/politeiad.go | 32 ++++++++++++++++---------------- 5 files changed, 35 insertions(+), 35 deletions(-) diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index ff8dbf026..1bad37fcc 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -286,10 +286,11 @@ type SetUnvettedStatus struct { MDOverwrite []MetadataStream `json:"mdoverwrite"` // Metadata streams to overwrite } -// SetUnvettedStatus is a response to a SetUnvettedStatus. It returns the +// SetUnvettedStatusReply is a response to a SetUnvettedStatus. It returns the // potentially modified record without the Files. type SetUnvettedStatusReply struct { Response string `json:"response"` // Challenge response + Record Record `json:"record"` // Record } // SetVettedStatus updates the status of a vetted record. This is used to @@ -306,6 +307,7 @@ type SetVettedStatus struct { // potentially modified record without the Files. type SetVettedStatusReply struct { Response string `json:"response"` // Challenge response + Record Record `json:"record"` // Record } // UpdateRecord updates a record. This is used for both unvetted and vetted @@ -322,8 +324,8 @@ type UpdateRecord struct { // UpdateRecordReply returns a CensorshipRecord which may or may not have // changed. Metadata only updates do not create a new CensorshipRecord. type UpdateRecordReply struct { - Response string `json:"response"` // Challenge response - CensorshipRecord CensorshipRecord `json:"censorshiprecord"` // Censorship record + Response string `json:"response"` // Challenge response + Record Record `json:"record"` // Record } // UpdateVettedMetadata update a vetted metadata. This is allowed for diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 185a8f70f..fa7c19fc6 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -366,17 +366,10 @@ func (p *politeia) updateRecord(w http.ResponseWriter, r *http.Request, vetted b } // Prepare reply. - merkle := record.RecordMetadata.Merkle - signature := p.identity.SignMessage([]byte(merkle + t.Token)) response := p.identity.SignMessage(challenge) - reply := v1.UpdateRecordReply{ Response: hex.EncodeToString(response[:]), - CensorshipRecord: v1.CensorshipRecord{ - Merkle: merkle, - Token: t.Token, - Signature: hex.EncodeToString(signature[:]), - }, + Record: p.convertBackendRecord(*record), } log.Infof("Update %v record %v: token %v", cmd, remoteAddr(r), @@ -686,6 +679,7 @@ func (p *politeia) setVettedStatus(w http.ResponseWriter, r *http.Request) { // Prepare reply. reply := v1.SetVettedStatusReply{ Response: hex.EncodeToString(response[:]), + Record: p.convertBackendRecord(*record), } s := convertBackendStatus(record.RecordMetadata.Status) @@ -748,6 +742,7 @@ func (p *politeia) setUnvettedStatus(w http.ResponseWriter, r *http.Request) { // Prepare reply. reply := v1.SetUnvettedStatusReply{ Response: hex.EncodeToString(response[:]), + Record: p.convertBackendRecord(*record), } s := convertBackendStatus(record.RecordMetadata.Status) diff --git a/politeiawww/eventmanager.go b/politeiawww/eventmanager.go index ed4e39710..d8756d83d 100644 --- a/politeiawww/eventmanager.go +++ b/politeiawww/eventmanager.go @@ -258,6 +258,7 @@ func (p *politeiawww) handleEventProposalEdited(ch chan interface{}) { type dataProposalStatusChange struct { token string // Proposal censorship token status pi.PropStatusT // Proposal status + version string // Proposal version reason string // Status change reason adminID string // Admin uuid } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 999a8acfd..9c229332c 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -1126,16 +1126,16 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p // Send politeiad request // TODO verify that this will throw an error if no proposal files // were changed. - var cr *pd.CensorshipRecord + var r *pd.Record switch pe.State { case pi.PropStateUnvetted: - cr, err = p.updateUnvetted(pe.Token, mdAppend, mdOverwrite, + r, err = p.updateUnvetted(pe.Token, mdAppend, mdOverwrite, filesAdd, filesDel) if err != nil { return nil, err } case pi.PropStateVetted: - cr, err = p.updateVetted(pe.Token, mdAppend, mdOverwrite, + r, err = p.updateVetted(pe.Token, mdAppend, mdOverwrite, filesAdd, filesDel) if err != nil { return nil, err @@ -1150,7 +1150,7 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p username: usr.Username, token: pe.Token, name: pm.Name, - // TODO version: version, + version: r.Version, }) log.Infof("Proposal edited: %v %v", pe.Token, pm.Name) @@ -1159,7 +1159,7 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p } return &pi.ProposalEditReply{ - CensorshipRecord: convertCensorshipRecordFromPD(*cr), + CensorshipRecord: convertCensorshipRecordFromPD(r.CensorshipRecord), Timestamp: timestamp, }, nil } @@ -1248,15 +1248,16 @@ func (p *politeiawww) processProposalSetStatus(pss pi.ProposalSetStatus, usr use // Send politeiad request // TODO verify proposal not found error is returned when wrong // token or state is used + var r *pd.Record status := convertRecordStatusFromPropStatus(pss.Status) switch pss.State { case pi.PropStateUnvetted: - err = p.setUnvettedStatus(pss.Token, status, mdAppend, mdOverwrite) + r, err = p.setUnvettedStatus(pss.Token, status, mdAppend, mdOverwrite) if err != nil { return nil, err } case pi.PropStateVetted: - err = p.setVettedStatus(pss.Token, status, mdAppend, mdOverwrite) + r, err = p.setVettedStatus(pss.Token, status, mdAppend, mdOverwrite) if err != nil { return nil, err } @@ -1266,7 +1267,8 @@ func (p *politeiawww) processProposalSetStatus(pss pi.ProposalSetStatus, usr use p.eventManager.emit(eventProposalStatusChange, dataProposalStatusChange{ token: pss.Token, - status: pss.Status, + status: convertPropStatusFromPD(r.Status), + version: r.Version, reason: pss.Reason, adminID: usr.ID.String(), }) diff --git a/politeiawww/politeiad.go b/politeiawww/politeiad.go index b052c8f92..46bbcc22a 100644 --- a/politeiawww/politeiad.go +++ b/politeiawww/politeiad.go @@ -126,7 +126,7 @@ func (p *politeiawww) newRecord(metadata []pd.MetadataStream, files []pd.File) ( // updateRecord updates a record in politeiad. This can be used to update // unvetted or vetted records depending on the route that is provided. -func (p *politeiawww) updateRecord(route, token string, mdAppend, mdOverwrite []pd.MetadataStream, filesAdd []pd.File, filesDel []string) (*pd.CensorshipRecord, error) { +func (p *politeiawww) updateRecord(route, token string, mdAppend, mdOverwrite []pd.MetadataStream, filesAdd []pd.File, filesDel []string) (*pd.Record, error) { // Setup request challenge, err := util.Random(pd.ChallengeSize) if err != nil { @@ -159,17 +159,17 @@ func (p *politeiawww) updateRecord(route, token string, mdAppend, mdOverwrite [] return nil, err } - return &urr.CensorshipRecord, nil + return &urr.Record, nil } // updateUnvetted updates an unvetted record in politeiad. -func (p *politeiawww) updateUnvetted(token string, mdAppend, mdOverwrite []pd.MetadataStream, filesAdd []pd.File, filesDel []string) (*pd.CensorshipRecord, error) { +func (p *politeiawww) updateUnvetted(token string, mdAppend, mdOverwrite []pd.MetadataStream, filesAdd []pd.File, filesDel []string) (*pd.Record, error) { return p.updateRecord(pd.UpdateUnvettedRoute, token, mdAppend, mdOverwrite, filesAdd, filesDel) } // updateVetted updates a vetted record in politeiad. -func (p *politeiawww) updateVetted(token string, mdAppend, mdOverwrite []pd.MetadataStream, filesAdd []pd.File, filesDel []string) (*pd.CensorshipRecord, error) { +func (p *politeiawww) updateVetted(token string, mdAppend, mdOverwrite []pd.MetadataStream, filesAdd []pd.File, filesDel []string) (*pd.Record, error) { return p.updateRecord(pd.UpdateVettedRoute, token, mdAppend, mdOverwrite, filesAdd, filesDel) } @@ -249,11 +249,11 @@ func (p *politeiawww) updateVettedMetadata(token string, mdAppend, mdOverwrite [ } // setUnvettedStatus sets the status of a unvetted record in politeiad. -func (p *politeiawww) setUnvettedStatus(token string, status pd.RecordStatusT, mdAppend, mdOverwrite []pd.MetadataStream) error { +func (p *politeiawww) setUnvettedStatus(token string, status pd.RecordStatusT, mdAppend, mdOverwrite []pd.MetadataStream) (*pd.Record, error) { // Setup request challenge, err := util.Random(pd.ChallengeSize) if err != nil { - return err + return nil, err } sus := pd.SetUnvettedStatus{ Challenge: hex.EncodeToString(challenge), @@ -267,31 +267,31 @@ func (p *politeiawww) setUnvettedStatus(token string, status pd.RecordStatusT, m resBody, err := p.makeRequest(http.MethodPost, pd.SetUnvettedStatusRoute, sus) if err != nil { - return err + return nil, err } // Receive reply var susr pd.SetUnvettedStatusReply err = json.Unmarshal(resBody, &susr) if err != nil { - return err + return nil, err } // Verify challenge err = util.VerifyChallenge(p.cfg.Identity, challenge, susr.Response) if err != nil { - return err + return nil, err } - return nil + return &susr.Record, nil } // setVettedStatus sets the status of a vetted record in politeiad. -func (p *politeiawww) setVettedStatus(token string, status pd.RecordStatusT, mdAppend, mdOverwrite []pd.MetadataStream) error { +func (p *politeiawww) setVettedStatus(token string, status pd.RecordStatusT, mdAppend, mdOverwrite []pd.MetadataStream) (*pd.Record, error) { // Setup request challenge, err := util.Random(pd.ChallengeSize) if err != nil { - return err + return nil, err } svs := pd.SetVettedStatus{ Challenge: hex.EncodeToString(challenge), @@ -305,23 +305,23 @@ func (p *politeiawww) setVettedStatus(token string, status pd.RecordStatusT, mdA resBody, err := p.makeRequest(http.MethodPost, pd.SetVettedStatusRoute, svs) if err != nil { - return err + return nil, err } // Receive reply var svsr pd.SetVettedStatusReply err = json.Unmarshal(resBody, &svsr) if err != nil { - return err + return nil, err } // Verify challenge err = util.VerifyChallenge(p.cfg.Identity, challenge, svsr.Response) if err != nil { - return err + return nil, err } - return nil + return &svsr.Record, nil } // getUnvetted retrieves an unvetted record from politeiad. From fa5dbb503a8277f627ae4a531fde998dca59ef8b Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 22 Sep 2020 09:49:11 -0500 Subject: [PATCH 087/449] move piplugin into tlog --- plugins/comments/comments.go | 23 +-- plugins/pi/pi.go | 5 +- politeiad/backend/tlogbe/comments.go | 90 +++++++++ politeiad/backend/tlogbe/encryptionkey.go | 2 +- politeiad/backend/tlogbe/{plugins => }/pi.go | 184 +++++++++++-------- politeiad/backend/tlogbe/tlog.go | 4 +- politeiad/backend/tlogbe/tlogbe.go | 16 +- 7 files changed, 219 insertions(+), 105 deletions(-) rename politeiad/backend/tlogbe/{plugins => }/pi.go (82%) diff --git a/plugins/comments/comments.go b/plugins/comments/comments.go index ea9b67648..00a1c001b 100644 --- a/plugins/comments/comments.go +++ b/plugins/comments/comments.go @@ -53,17 +53,18 @@ const ( // Error status codes ErrorStatusInvalid ErrorStatusT = 0 - ErrorStatusTokenInvalid ErrorStatusT = 1 - ErrorStatusPublicKeyInvalid ErrorStatusT = 2 - ErrorStatusSignatureInvalid ErrorStatusT = 3 - ErrorStatusCommentLengthMax ErrorStatusT = 4 - ErrorStatusRecordNotFound ErrorStatusT = 5 - ErrorStatusCommentNotFound ErrorStatusT = 6 - ErrorStatusUserIDInvalid ErrorStatusT = 7 - ErrorStatusParentIDInvalid ErrorStatusT = 8 - ErrorStatusNoCommentChanges ErrorStatusT = 9 - ErrorStatusVoteInvalid ErrorStatusT = 10 - ErrorStatusVoteChangesMax ErrorStatusT = 11 + ErrorStatusStateInvalid ErrorStatusT = 1 + ErrorStatusTokenInvalid ErrorStatusT = 2 + ErrorStatusPublicKeyInvalid ErrorStatusT = 3 + ErrorStatusSignatureInvalid ErrorStatusT = 4 + ErrorStatusCommentLengthMax ErrorStatusT = 5 + ErrorStatusRecordNotFound ErrorStatusT = 6 + ErrorStatusCommentNotFound ErrorStatusT = 7 + ErrorStatusUserIDInvalid ErrorStatusT = 8 + ErrorStatusParentIDInvalid ErrorStatusT = 9 + ErrorStatusNoCommentChanges ErrorStatusT = 10 + ErrorStatusVoteInvalid ErrorStatusT = 11 + ErrorStatusVoteChangesMax ErrorStatusT = 12 ) var ( diff --git a/plugins/pi/pi.go b/plugins/pi/pi.go index 57de17eb6..51304f4b8 100644 --- a/plugins/pi/pi.go +++ b/plugins/pi/pi.go @@ -461,12 +461,13 @@ type VoteInventoryReply struct { BestBlock uint32 `json:"bestblock"` } -// EncodeVoteInventoryReply encodes a VoteInventoryReply into a JSON byte slice. +// EncodeVoteInventoryReply encodes a VoteInventoryReply into a JSON byte +// slice. func EncodeVoteInventoryReply(vir VoteInventoryReply) ([]byte, error) { return json.Marshal(vir) } -// DecodeVoteInventoryReply decodes a JSON byte slice into a VoteInventoryReply. +// DecodeVoteInventoryReply decodes a JSON byte slice into VoteInventoryReply. func DecodeVoteInventoryReply(payload []byte) (*VoteInventoryReply, error) { var vir VoteInventoryReply err := json.Unmarshal(payload, &vir) diff --git a/politeiad/backend/tlogbe/comments.go b/politeiad/backend/tlogbe/comments.go index d2aecf609..ff1923f0f 100644 --- a/politeiad/backend/tlogbe/comments.go +++ b/politeiad/backend/tlogbe/comments.go @@ -752,6 +752,16 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { return "", err } + // Verify state + switch n.State { + case comments.StateUnvetted, comments.StateVetted: + // Allowed; continue + default: + return "", comments.UserErrorReply{ + ErrorCode: comments.ErrorStatusTokenInvalid, + } + } + // Verify token token, err := hex.DecodeString(n.Token) if err != nil { @@ -864,6 +874,16 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { return "", err } + // Verify state + switch e.State { + case comments.StateUnvetted, comments.StateVetted: + // Allowed; continue + default: + return "", comments.UserErrorReply{ + ErrorCode: comments.ErrorStatusTokenInvalid, + } + } + // Verify token token, err := hex.DecodeString(e.Token) if err != nil { @@ -999,6 +1019,16 @@ func (p *commentsPlugin) cmdDel(payload string) (string, error) { return "", err } + // Verify state + switch d.State { + case comments.StateUnvetted, comments.StateVetted: + // Allowed; continue + default: + return "", comments.UserErrorReply{ + ErrorCode: comments.ErrorStatusTokenInvalid, + } + } + // Verify token token, err := hex.DecodeString(d.Token) if err != nil { @@ -1113,6 +1143,16 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { return "", err } + // Verify state + switch v.State { + case comments.StateUnvetted, comments.StateVetted: + // Allowed; continue + default: + return "", comments.UserErrorReply{ + ErrorCode: comments.ErrorStatusTokenInvalid, + } + } + // Verify token token, err := hex.DecodeString(v.Token) if err != nil { @@ -1257,6 +1297,16 @@ func (p *commentsPlugin) cmdGet(payload string) (string, error) { return "", err } + // Verify state + switch g.State { + case comments.StateUnvetted, comments.StateVetted: + // Allowed; continue + default: + return "", comments.UserErrorReply{ + ErrorCode: comments.ErrorStatusTokenInvalid, + } + } + // Verify token token, err := hex.DecodeString(g.Token) if err != nil { @@ -1303,6 +1353,16 @@ func (p *commentsPlugin) cmdGetAll(payload string) (string, error) { return "", err } + // Verify state + switch ga.State { + case comments.StateUnvetted, comments.StateVetted: + // Allowed; continue + default: + return "", comments.UserErrorReply{ + ErrorCode: comments.ErrorStatusTokenInvalid, + } + } + // Verify token token, err := hex.DecodeString(ga.Token) if err != nil { @@ -1366,6 +1426,16 @@ func (p *commentsPlugin) cmdGetVersion(payload string) (string, error) { return "", err } + // Verify state + switch gv.State { + case comments.StateUnvetted, comments.StateVetted: + // Allowed; continue + default: + return "", comments.UserErrorReply{ + ErrorCode: comments.ErrorStatusTokenInvalid, + } + } + // Verify token token, err := hex.DecodeString(gv.Token) if err != nil { @@ -1438,6 +1508,16 @@ func (p *commentsPlugin) cmdCount(payload string) (string, error) { return "", err } + // Verify state + switch c.State { + case comments.StateUnvetted, comments.StateVetted: + // Allowed; continue + default: + return "", comments.UserErrorReply{ + ErrorCode: comments.ErrorStatusTokenInvalid, + } + } + // Verify token token, err := hex.DecodeString(c.Token) if err != nil { @@ -1473,6 +1553,16 @@ func (p *commentsPlugin) cmdVotes(payload string) (string, error) { return "", err } + // Verify state + switch v.State { + case comments.StateUnvetted, comments.StateVetted: + // Allowed; continue + default: + return "", comments.UserErrorReply{ + ErrorCode: comments.ErrorStatusTokenInvalid, + } + } + // Verify token token, err := hex.DecodeString(v.Token) if err != nil { diff --git a/politeiad/backend/tlogbe/encryptionkey.go b/politeiad/backend/tlogbe/encryptionkey.go index 07b9bd728..fd8b15f47 100644 --- a/politeiad/backend/tlogbe/encryptionkey.go +++ b/politeiad/backend/tlogbe/encryptionkey.go @@ -41,7 +41,7 @@ func (e *encryptionKey) decrypt(blob []byte) ([]byte, uint32, error) { } // Zero zeroes out the encryption key. -func (e *encryptionKey) Zero() { +func (e *encryptionKey) zero() { e.Lock() defer e.Unlock() diff --git a/politeiad/backend/tlogbe/plugins/pi.go b/politeiad/backend/tlogbe/pi.go similarity index 82% rename from politeiad/backend/tlogbe/plugins/pi.go rename to politeiad/backend/tlogbe/pi.go index b6b8ead06..b946fd560 100644 --- a/politeiad/backend/tlogbe/plugins/pi.go +++ b/politeiad/backend/tlogbe/pi.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package plugins +package tlogbe import ( "encoding/base64" @@ -20,7 +20,6 @@ import ( "github.com/decred/politeia/plugins/pi" "github.com/decred/politeia/plugins/ticketvote" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe" ) const ( @@ -29,13 +28,14 @@ const ( ) var ( - _ tlogbe.Plugin = (*piPlugin)(nil) + _ pluginClient = (*piPlugin)(nil) ) // piPlugin satisfies the Plugin interface. type piPlugin struct { sync.Mutex - backend *tlogbe.TlogBackend + backend backend.Backend + tlog tlogClient // dataDir is the pi plugin data directory. The only data that is // stored here is cached data that can be re-created at any time @@ -66,9 +66,20 @@ func proposalMetadataFromFiles(files []backend.File) (*pi.ProposalMetadata, erro return pm, nil } -// TODO saving the linkedFrom to the filesystem is not scalable between -// multiple politeiad instances. The plugin needs to have a tree that can be -// used to share state between the different politeiad instances. +func convertPropStatusFromMDStatus(s backend.MDStatusT) pi.PropStatusT { + var status pi.PropStatusT + switch s { + case backend.MDStatusUnvetted, backend.MDStatusIterationUnvetted: + status = pi.PropStatusUnvetted + case backend.MDStatusVetted: + status = pi.PropStatusPublic + case backend.MDStatusCensored: + status = pi.PropStatusCensored + case backend.MDStatusArchived: + status = pi.PropStatusAbandoned + } + return status +} // linkedFrom is the the structure that is updated and cached for proposal A // when proposal B links to proposal A. The list contains all proposals that @@ -81,14 +92,14 @@ type linkedFrom struct { Tokens map[string]struct{} `json:"tokens"` } -func (p *piPlugin) cachedLinkedFromPath(token string) string { +func (p *piPlugin) linkedFromPath(token string) string { fn := strings.Replace(filenameLinkedFrom, "{token}", token, 1) return filepath.Join(p.dataDir, fn) } // This function must be called WITH the lock held. -func (p *piPlugin) cachedLinkedFromLocked(token string) (*linkedFrom, error) { - fp := p.cachedLinkedFromPath(token) +func (p *piPlugin) linkedFromLocked(token string) (*linkedFrom, error) { + fp := p.linkedFromPath(token) b, err := ioutil.ReadFile(fp) if err != nil { var e *os.PathError @@ -107,26 +118,26 @@ func (p *piPlugin) cachedLinkedFromLocked(token string) (*linkedFrom, error) { return &lf, nil } -func (p *piPlugin) cachedLinkedFrom(token string) (*linkedFrom, error) { +func (p *piPlugin) linkedFrom(token string) (*linkedFrom, error) { p.Lock() defer p.Unlock() - return p.cachedLinkedFromLocked(token) + return p.linkedFromLocked(token) } -func (p *piPlugin) cachedLinkedFromAdd(parentToken, childToken string) error { +func (p *piPlugin) linkedFromAdd(parentToken, childToken string) error { p.Lock() defer p.Unlock() // Get existing linked from list - lf, err := p.cachedLinkedFromLocked(parentToken) + lf, err := p.linkedFromLocked(parentToken) if err == errRecordNotFound { // List doesn't exist. Create a new one. lf = &linkedFrom{ Tokens: make(map[string]struct{}, 0), } } else if err != nil { - return fmt.Errorf("cachedLinkedFromLocked %v: %v", parentToken, err) + return fmt.Errorf("linkedFromLocked %v: %v", parentToken, err) } // Update list @@ -137,7 +148,7 @@ func (p *piPlugin) cachedLinkedFromAdd(parentToken, childToken string) error { if err != nil { return err } - fp := p.cachedLinkedFromPath(parentToken) + fp := p.linkedFromPath(parentToken) err = ioutil.WriteFile(fp, b, 0664) if err != nil { return fmt.Errorf("WriteFile: %v", err) @@ -146,14 +157,14 @@ func (p *piPlugin) cachedLinkedFromAdd(parentToken, childToken string) error { return nil } -func (p *piPlugin) cachedLinkedFromDel(parentToken, childToken string) error { +func (p *piPlugin) linkedFromDel(parentToken, childToken string) error { p.Lock() defer p.Unlock() // Get existing linked from list - lf, err := p.cachedLinkedFromLocked(parentToken) + lf, err := p.linkedFromLocked(parentToken) if err != nil { - return fmt.Errorf("cachedLinkedFromLocked %v: %v", parentToken, err) + return fmt.Errorf("linkedFromLocked %v: %v", parentToken, err) } // Update list @@ -164,7 +175,7 @@ func (p *piPlugin) cachedLinkedFromDel(parentToken, childToken string) error { if err != nil { return err } - fp := p.cachedLinkedFromPath(parentToken) + fp := p.linkedFromPath(parentToken) err = ioutil.WriteFile(fp, b, 0664) if err != nil { return fmt.Errorf("WriteFile: %v", err) @@ -173,8 +184,52 @@ func (p *piPlugin) cachedLinkedFromDel(parentToken, childToken string) error { return nil } +func (p *piPlugin) cmdProposals(payload string) (string, error) { + // TODO + + /* + // Just because a cached linked from doesn't exist doesn't + // mean the token isn't valid. We need to check if the token + // corresponds to a real proposal. + proposals := make(map[string]pi.ProposalData, len(ps.Tokens)) + for _, v := range ps.Tokens { + lf, err := p.linkedFrom(v) + if err != nil { + if err == errRecordNotFound { + continue + } + return "", fmt.Errorf("linkedFrom %v: %v", v, err) + } + } + */ + + return "", nil +} + +func (p *piPlugin) cmdCommentNew(payload string) (string, error) { + // TODO + // Only allow commenting on vetted + return "", nil +} + +func (p *piPlugin) cmdCommentCensor(payload string) (string, error) { + // TODO + return "", nil +} + +func (p *piPlugin) cmdCommentVote(payload string) (string, error) { + // TODO + // Only allow voting on vetted + return "", nil +} + +func (p *piPlugin) cmdVoteInventory(payload string) (string, error) { + // TODO + return "", nil +} + func (p *piPlugin) hookNewRecordPre(payload string) error { - nr, err := tlogbe.DecodeNewRecord([]byte(payload)) + nr, err := decodeHookNewRecord([]byte(payload)) if err != nil { return err } @@ -283,23 +338,8 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { return nil } -func convertPropStatusFromMDStatus(s backend.MDStatusT) pi.PropStatusT { - var status pi.PropStatusT - switch s { - case backend.MDStatusUnvetted, backend.MDStatusIterationUnvetted: - status = pi.PropStatusUnvetted - case backend.MDStatusVetted: - status = pi.PropStatusPublic - case backend.MDStatusCensored: - status = pi.PropStatusCensored - case backend.MDStatusArchived: - status = pi.PropStatusAbandoned - } - return status -} - func (p *piPlugin) hookEditRecordPre(payload string) error { - er, err := tlogbe.DecodeEditRecord([]byte(payload)) + er, err := decodeHookEditRecord([]byte(payload)) if err != nil { return err } @@ -371,7 +411,7 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { } func (p *piPlugin) hookSetRecordStatusPost(payload string) error { - srs, err := tlogbe.DecodeSetRecordStatus([]byte(payload)) + srs, err := decodeHookSetRecordStatus([]byte(payload)) if err != nil { return err } @@ -453,16 +493,16 @@ func (p *piPlugin) hookSetRecordStatusPost(payload string) error { case backend.MDStatusVetted: // Proposal has been made public. Add child token to parent // token's linked from list. - err := p.cachedLinkedFromAdd(parentToken, childToken) + err := p.linkedFromAdd(parentToken, childToken) if err != nil { - return fmt.Errorf("cachedLinkedFromAdd: %v", err) + return fmt.Errorf("linkedFromAdd: %v", err) } case backend.MDStatusCensored: // Proposal has been censored. Delete child token from parent // token's linked from list. - err := p.cachedLinkedFromDel(parentToken, childToken) + err := p.linkedFromDel(parentToken, childToken) if err != nil { - return fmt.Errorf("cachedLinkedFromDel: %v", err) + return fmt.Errorf("linkedFromDel: %v", err) } } } @@ -470,76 +510,57 @@ func (p *piPlugin) hookSetRecordStatusPost(payload string) error { return nil } -func (p *piPlugin) Setup() error { - log.Tracef("pi Setup") +func (p *piPlugin) setup() error { + log.Tracef("pi setup") // Verify vote plugin dependency return nil } -func (p *piPlugin) cmdProposals(payload string) (string, error) { - ps, err := pi.DecodeProposals([]byte(payload)) - if err != nil { - return "", err - } - _ = ps - - /* - // TODO just because a cached linked from doesn't exist doesn't - // mean the token isn't valid. We need to check if the token - // corresponds to a real proposal. - proposals := make(map[string]pi.ProposalData, len(ps.Tokens)) - for _, v := range ps.Tokens { - lf, err := p.cachedLinkedFrom(v) - if err != nil { - if err == errRecordNotFound { - continue - } - return "", fmt.Errorf("cachedLinkedFrom %v: %v", v, err) - } - } - */ - - return "", nil -} - -func (p *piPlugin) Cmd(cmd, payload string) (string, error) { - log.Tracef("pi Cmd: %v %v", cmd, payload) +func (p *piPlugin) cmd(cmd, payload string) (string, error) { + log.Tracef("pi cmd: %v %v", cmd, payload) switch cmd { case pi.CmdProposals: return p.cmdProposals(payload) - // TODO case pi.CmdVoteInventory + case pi.CmdCommentNew: + return p.cmdCommentNew(payload) + case pi.CmdCommentCensor: + return p.cmdCommentCensor(payload) + case pi.CmdCommentVote: + return p.cmdCommentVote(payload) + case pi.CmdVoteInventory: + return p.cmdVoteInventory(payload) } return "", backend.ErrPluginCmdInvalid } -func (p *piPlugin) Hook(h tlogbe.HookT, payload string) error { - log.Tracef("pi Hook: %v", tlogbe.Hooks[h]) +func (p *piPlugin) hook(h hookT, payload string) error { + log.Tracef("pi hook: %v", hooks[h]) switch h { - case tlogbe.HookNewRecordPre: + case hookNewRecordPre: return p.hookNewRecordPre(payload) - case tlogbe.HookEditRecordPre: + case hookEditRecordPre: return p.hookEditRecordPre(payload) - case tlogbe.HookSetRecordStatusPost: + case hookSetRecordStatusPost: return p.hookSetRecordStatusPost(payload) } return nil } -func (p *piPlugin) Fsck() error { - log.Tracef("pi Fsck") +func (p *piPlugin) fsck() error { + log.Tracef("pi fsck") // linkedFrom cache return nil } -func NewPiPlugin(backend *tlogbe.TlogBackend, settings []backend.PluginSetting) *piPlugin { +func newPiPlugin(backend backend.Backend, tlog tlogClient, settings []backend.PluginSetting) *piPlugin { // TODO these should be passed in as plugin settings var ( dataDir string @@ -547,5 +568,6 @@ func NewPiPlugin(backend *tlogbe.TlogBackend, settings []backend.PluginSetting) return &piPlugin{ dataDir: dataDir, backend: backend, + tlog: tlog, } } diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index d94c0be82..cd5f80f47 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -1734,9 +1734,9 @@ func (t *tlog) close() { t.store.Close() t.trillian.close() - // Zero out encryption key + // Zero out encryption key. An encryption key is optional. if t.encryptionKey != nil { - t.encryptionKey.Zero() + t.encryptionKey.zero() } } diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 8007e1d6b..308bfc967 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -75,14 +75,6 @@ var ( } ) -// plugin represents a tlogbe plugin. -type plugin struct { - id string - version string - settings []backend.PluginSetting - client pluginClient -} - // tlogBackend implements the Backend interface. type tlogBackend struct { sync.RWMutex @@ -114,6 +106,14 @@ type tlogBackend struct { inventory map[backend.MDStatusT][]string } +// plugin represents a tlogbe plugin. +type plugin struct { + id string + version string + settings []backend.PluginSetting + client pluginClient +} + func tokenPrefix(token []byte) string { return hex.EncodeToString(token)[:pd.TokenPrefixLength] } From 8a3bfd899b7d9373f765922b703205bb225f3143 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 22 Sep 2020 10:21:05 -0500 Subject: [PATCH 088/449] move dcrdata plugin into tlogbe --- politeiad/backend/tlogbe/comments.go | 2 +- .../backend/tlogbe/{plugins => }/dcrdata.go | 91 +++++++++---------- politeiad/backend/tlogbe/pi.go | 14 ++- 3 files changed, 59 insertions(+), 48 deletions(-) rename politeiad/backend/tlogbe/{plugins => }/dcrdata.go (92%) diff --git a/politeiad/backend/tlogbe/comments.go b/politeiad/backend/tlogbe/comments.go index ff1923f0f..a2498e0a6 100644 --- a/politeiad/backend/tlogbe/comments.go +++ b/politeiad/backend/tlogbe/comments.go @@ -1701,7 +1701,7 @@ func (p *commentsPlugin) fsck() error { return nil } -// Setup performs any plugin setup work that needs to be done. +// setup performs any plugin setup work that needs to be done. // // This function satisfies the pluginClient interface. func (p *commentsPlugin) setup() error { diff --git a/politeiad/backend/tlogbe/plugins/dcrdata.go b/politeiad/backend/tlogbe/dcrdata.go similarity index 92% rename from politeiad/backend/tlogbe/plugins/dcrdata.go rename to politeiad/backend/tlogbe/dcrdata.go index 3f12dc7be..dc1fd9208 100644 --- a/politeiad/backend/tlogbe/plugins/dcrdata.go +++ b/politeiad/backend/tlogbe/dcrdata.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package plugins +package tlogbe import ( "bytes" @@ -21,7 +21,6 @@ import ( pstypes "github.com/decred/dcrdata/pubsub/types/v3" "github.com/decred/politeia/plugins/dcrdata" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe" "github.com/decred/politeia/util" "github.com/decred/politeia/wsdcrdata" ) @@ -35,10 +34,10 @@ const ( ) var ( - _ tlogbe.Plugin = (*dcrdataPlugin)(nil) + _ pluginClient = (*dcrdataPlugin)(nil) ) -// dcrdataplugin satisfies the Plugin interface. +// dcrdataplugin satisfies the pluginClient interface. type dcrdataPlugin struct { sync.Mutex host string @@ -345,44 +344,6 @@ func (p *dcrdataPlugin) cmdTxsTrimmed(payload string) (string, error) { return string(reply), nil } -// Cmd executes a plugin command. -// -// This function satisfies the Plugin interface. -func (p *dcrdataPlugin) Cmd(cmd, payload string) (string, error) { - log.Tracef("dcrdata Cmd: %v", cmd) - - switch cmd { - case dcrdata.CmdBestBlock: - return p.cmdBestBlock(payload) - case dcrdata.CmdBlockDetails: - return p.cmdBlockDetails(payload) - case dcrdata.CmdTicketPool: - return p.cmdTicketPool(payload) - case dcrdata.CmdTxsTrimmed: - return p.cmdTxsTrimmed(payload) - } - - return "", backend.ErrPluginCmdInvalid -} - -// Hook executes a plugin hook. -// -// This function satisfies the Plugin interface. -func (p *dcrdataPlugin) Hook(h tlogbe.HookT, payload string) error { - log.Tracef("dcrdata Hook: %v %v", tlogbe.Hooks[h], payload) - - return nil -} - -// Fsck performs a plugin filesystem check. -// -// This function satisfies the Plugin interface. -func (p *dcrdataPlugin) Fsck() error { - log.Tracef("dcrdata Fsck") - - return nil -} - func (p *dcrdataPlugin) websocketMonitor() { defer func() { log.Infof("Dcrdata websocket closed") @@ -464,11 +425,11 @@ func (p *dcrdataPlugin) websocketSetup() { go p.websocketMonitor() } -// Setup performs any plugin setup work that needs to be done. +// setup performs any plugin setup work that needs to be done. // // This function satisfies the Plugin interface. -func (p *dcrdataPlugin) Setup() error { - log.Tracef("dcrdata Setup") +func (p *dcrdataPlugin) setup() error { + log.Tracef("dcrdata setup") // Setup dcrdata websocket subscriptions and monitoring. This is // done in a go routine so setup will continue in the event that @@ -479,7 +440,45 @@ func (p *dcrdataPlugin) Setup() error { return nil } -func DcrdataPluginNew(settings []backend.PluginSetting) *dcrdataPlugin { +// cmd executes a plugin command. +// +// This function satisfies the pluginClient interface. +func (p *dcrdataPlugin) cmd(cmd, payload string) (string, error) { + log.Tracef("dcrdata cmd: %v", cmd) + + switch cmd { + case dcrdata.CmdBestBlock: + return p.cmdBestBlock(payload) + case dcrdata.CmdBlockDetails: + return p.cmdBlockDetails(payload) + case dcrdata.CmdTicketPool: + return p.cmdTicketPool(payload) + case dcrdata.CmdTxsTrimmed: + return p.cmdTxsTrimmed(payload) + } + + return "", backend.ErrPluginCmdInvalid +} + +// hook executes a plugin hook. +// +// This function satisfies the pluginClient interface. +func (p *dcrdataPlugin) hook(h hookT, payload string) error { + log.Tracef("dcrdata hook: %v %v", hooks[h], payload) + + return nil +} + +// fsck performs a plugin filesystem check. +// +// This function satisfies the pluginClient interface. +func (p *dcrdataPlugin) fsck() error { + log.Tracef("dcrdata fsck") + + return nil +} + +func newDcrdataPlugin(settings []backend.PluginSetting) *dcrdataPlugin { // TODO these should be passed in as plugin settings var dcrdataHost string diff --git a/politeiad/backend/tlogbe/pi.go b/politeiad/backend/tlogbe/pi.go index b946fd560..bc59be5e3 100644 --- a/politeiad/backend/tlogbe/pi.go +++ b/politeiad/backend/tlogbe/pi.go @@ -31,7 +31,7 @@ var ( _ pluginClient = (*piPlugin)(nil) ) -// piPlugin satisfies the Plugin interface. +// piPlugin satisfies the pluginClient interface. type piPlugin struct { sync.Mutex backend backend.Backend @@ -510,6 +510,9 @@ func (p *piPlugin) hookSetRecordStatusPost(payload string) error { return nil } +// setup performs any plugin setup work that needs to be done. +// +// This function satisfies the Plugin interface. func (p *piPlugin) setup() error { log.Tracef("pi setup") @@ -518,6 +521,9 @@ func (p *piPlugin) setup() error { return nil } +// cmd executes a plugin command. +// +// This function satisfies the pluginClient interface. func (p *piPlugin) cmd(cmd, payload string) (string, error) { log.Tracef("pi cmd: %v %v", cmd, payload) @@ -537,6 +543,9 @@ func (p *piPlugin) cmd(cmd, payload string) (string, error) { return "", backend.ErrPluginCmdInvalid } +// hook executes a plugin hook. +// +// This function satisfies the pluginClient interface. func (p *piPlugin) hook(h hookT, payload string) error { log.Tracef("pi hook: %v", hooks[h]) @@ -552,6 +561,9 @@ func (p *piPlugin) hook(h hookT, payload string) error { return nil } +// fsck performs a plugin filesystem check. +// +// This function satisfies the pluginClient interface. func (p *piPlugin) fsck() error { log.Tracef("pi fsck") From 126e84f3835fb6a4bce0d20255c36a2c990646e3 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 22 Sep 2020 14:11:31 -0500 Subject: [PATCH 089/449] remove tlog package --- tlog/README.md | 65 --- tlog/api/v1/api.go | 206 -------- tlog/tclient/tclient.go | 897 ------------------------------- tlog/tserver/anchor.go | 657 ----------------------- tlog/tserver/blob.go | 182 ------- tlog/tserver/blob_test.go | 144 ----- tlog/tserver/config.go | 561 -------------------- tlog/tserver/log.go | 107 ---- tlog/tserver/params.go | 68 --- tlog/tserver/tserver.go | 1053 ------------------------------------- tlog/util/util.go | 189 ------- 11 files changed, 4129 deletions(-) delete mode 100644 tlog/README.md delete mode 100644 tlog/api/v1/api.go delete mode 100644 tlog/tclient/tclient.go delete mode 100644 tlog/tserver/anchor.go delete mode 100644 tlog/tserver/blob.go delete mode 100644 tlog/tserver/blob_test.go delete mode 100644 tlog/tserver/config.go delete mode 100644 tlog/tserver/log.go delete mode 100644 tlog/tserver/params.go delete mode 100644 tlog/tserver/tserver.go delete mode 100644 tlog/util/util.go diff --git a/tlog/README.md b/tlog/README.md deleted file mode 100644 index 1935c431e..000000000 --- a/tlog/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# trillian test platform - -tserver and tclient provide a test platform for google trillian -https://github.com/google/trillian - -# trillian quick setup - -Setup `MySQL` with an empty root password. This is obviously only for testing! - -``` -mkdir $GOPATH/src/github.com/google/ -cd $GOPATH/src/github.com/google/ -git clone git@github.com:google/trillian.git -cd trillian -go get -t -u -v ./... -``` - -Reset the test database -``` -./scripts/resetdb.sh -``` - -Launch log server -``` -trillian_log_server --logtostderr ... -``` - -Launch log signer daemom -``` -trillian_log_signer --logtostderr --force_master --http_endpoint=localhost:8092 --rpc_endpoint=localhost:8093 --batch_size=1000 --sequencer_guard_window=0 --sequencer_interval=200ms -``` - -# tserver - -Launch `tserver` -``` -tserver --testnet -``` - -# tclient - -tclient is used to interact with the tserver API. - -``` -Commands: - list list all records - publickey retrieve the tserver public signing key and save it - recordnew create a new record - recordappend append a record - recordget retrieve a record - recordentriesget retrieve the entries for a record - fsck run fsck on a record -``` - -You must retrieve the tserver identity before you can submit a record. - -``` -tclient --testnet publickey -``` - -Submit a new record. - -``` -tclient --testnet recordnew README.md -``` diff --git a/tlog/api/v1/api.go b/tlog/api/v1/api.go deleted file mode 100644 index f96a2dcdf..000000000 --- a/tlog/api/v1/api.go +++ /dev/null @@ -1,206 +0,0 @@ -package v1 - -import ( - "fmt" - - dcrtime "github.com/decred/dcrtime/api/v1" - "github.com/google/trillian" -) - -type ErrorStatusT int - -const ( - // Routes - RouteList = "/v1/list/" // list all records - RoutePublicKey = "/v1/publickey/" // public signing key - RouteRecordNew = "/v1/recordnew/" // new record - RouteRecordGet = "/v1/recordget/" // retrieve record and proofs - RouteRecordEntriesGet = "/v1/recordentriesget/" // retrieve record entries and proofs - RouteRecordAppend = "/v1/append/" // append data to record - RouteRecordFsck = "/v1/fsck/" // fsck record - - // Error status codes - ErrorStatusInvalid ErrorStatusT = 0 - ErrorStatusInvalidInput ErrorStatusT = 1 - - Forward = "X-Forwarded-For" -) - -var ( - // ErrorStatus converts error status codes to human readable text. - ErrorStatus = map[ErrorStatusT]string{ - ErrorStatusInvalid: "invalid status", - ErrorStatusInvalidInput: "invalid input", - } -) - -// UserError represents an error that is caused by something that the user -// did (malformed input, bad timing, etc). -type UserError struct { - ErrorCode ErrorStatusT - ErrorContext []string -} - -// Error satisfies the error interface. -func (e UserError) Error() string { - return fmt.Sprintf("user error code: %v", e.ErrorCode) -} - -// ErrorReply are replies that the server returns a when it encounters an -// unrecoverable problem while executing a command. The HTTP Error Code -// shall be 500 if it's an internal server error or 4xx if it's a user error. -type ErrorReply struct { - ErrorCode int64 `json:"errorcode,omitempty"` - ErrorContext []string `json:"errorcontext,omitempty"` -} - -const ( - // DataDescriptor.Type values. These may be freely edited since they - // are solely hints to the application. - DataTypeKeyValue = "kv" // Descriptor is empty but data is key/value - DataTypeMime = "mime" // Descriptor contains a mime type - DataTypeStructure = "struct" // Descriptor contains a structure - - DataDescriptorAnchor = "anchor" // Data is JSON Anchor structure -) - -// DataDescriptor provides hints about a data blob. In practise we JSON encode -// this struture and stuff it into RecordEntry.DataHint. -type DataDescriptor struct { - Type string `json:"type,omitempty"` // Type of data that is stored - Descriptor string `json:"descriptor,omitempty"` // Description of the data - ExtraData string `json:"extradata,omitempty"` // Value to be freely used by caller -} - -// DataKeyValue is an encoded key/value pair. -type DataKeyValue struct { - Key string `json:"key"` // Key - Value string `json:"value"` // Value -} - -// DataAnchor describes what is stored in dcrtime. We store the SHA256 hash of -// STH.LogRoot in dcrtime -type DataAnchor struct { - RecordId int64 `json:"recordid"` // Record ID this STH belongs to - STH trillian.SignedLogRoot `json:"sth"` // Signed tree head - VerifyDigest dcrtime.VerifyDigest `json:"verifydigest"` // dcrtime digest structure -} - -// Record contains user provided data and user attestation. -type RecordEntry struct { - PublicKey string `json:"publickey"` // Hex encoded public key used sign Data - Hash string `json:"hash"` // Hex encoded hash of the string data - Signature string `json:"signature"` // Hex encoded client ed25519 signature of Hash - DataHint string `json:"datahint"` // Hint that describes the data, base64 encoded - Data string `json:"data"` // Data payload, base64 encoded -} - -// PublicKey retrieves the server public signing key. -type PublicKey struct{} - -// PublicKeyReply returns the server's signing key. It is a base64 encoded DER -// format. -type PublicKeyReply struct { - SigningKey string `json:"signingkey"` // base64 encoded DER key -} - -// List request a list of all trees. -type List struct{} - -// ListReply returns a list of all trees. -type ListReply struct { - Trees []*trillian.Tree `json:"trees"` -} - -// RecordNew creates a new record that consists of several record entries. The -// server will not interpret the data at all. It will simply verify that the -// Data is signed with PublicKey. -type RecordNew struct { - RecordEntries []RecordEntry `json:"recordentries"` // Entries to be stored -} - -// QueuedLeafProof contains a queued log leaf and an inclusion proof for the -// leaf. A queued log leaf will not have a leaf index so any client side -// verification must be done using the leaf hash. -type QueuedLeafProof struct { - QueuedLeaf trillian.QueuedLogLeaf `json:"queuedleaf"` // A queued leaf and its status - Proof *trillian.Proof `json:"proof,omitempty"` // Leaf inclusion proof -} - -// RecordNewReply returns all pertinent information about a record. It returns -// trillian types so that the client can perform verifications. -type RecordNewReply struct { - Tree trillian.Tree `json:"tree"` // TreeId is the record id - InitialRoot trillian.SignedLogRoot `json:"initialroot"` // Tree creation root - STH trillian.SignedLogRoot `json:"sth"` // Signed tree head after record addition - Proofs []QueuedLeafProof `json:"proofs"` // Queued leaves and their proofs -} - -// RecordAppend adds new record entries to a record. The server will not -// interpret the data at all. It will simply verify that the Data is signed -// with PublicKey. It also does not overwrite or delete items. The caller is -// expected to keep track of ordering etc. Note that The leafs do have -// timestamps. -type RecordAppend struct { - Id int64 `json:"id"` // Record ID - RecordEntries []RecordEntry `json:"recordentries"` // Entries to be stored -} - -// RecordAppendReply returns all pertinent information about the record entries -// that were append to a record. It returns trillian types so that the client -// can perform verifications. -type RecordAppendReply struct { - STH trillian.SignedLogRoot `json:"sth"` // Signed tree head after record addition - Proofs []QueuedLeafProof `json:"proofs"` // Queued leaves and their proofs -} - -// RecordGet retrieves the entire record including proofs. This is an expensive -// call. -type RecordGet struct { - Id int64 `json:"id"` // Record ID -} - -// RecordGetReply returns all record entries and the proofs. -type RecordGetReply struct { - Proofs []RecordEntryProof `json:"recordentries"` // All entries and proofs. This may be big - STH trillian.SignedLogRoot `json:"sth"` // Signed tree head -} - -// RecordEntryIdentifier uniquely identifies a single leaf+data+proof. -type RecordEntryIdentifier struct { - Id int64 `json:"id"` // Record ID - MerkleHash string `json:"merklehash"` // Merkle hash -} - -// RecordEntryProof contains an entire record entry and anchor proof. The STH, -// Proof, and Anchor will ony be present once the RecordEntry has been -// successfully anchored. The STH corresponds to the LogRoot that was anchored. -// It is not the STH of the current tree. If error is set the record could not -// be retrieved. -type RecordEntryProof struct { - RecordEntry *RecordEntry `json:"recordentry,omitempty"` // Data - Leaf *trillian.LogLeaf `json:"leaf,omitempty"` // Requested Leaf - STH *trillian.SignedLogRoot `json:"sth,omitempty"` // Signed tree head - Proof *trillian.Proof `json:"proof,omitempty"` // Inclusion proof for STH - Anchor *dcrtime.ChainInformation `json:"anchor,omitempty"` // Anchor info for STH - Error string `json:"error,omitempty"` // Error is set when record could not be retrieved -} - -// RecordEntriesGet attempts to retrieve a batch of record entries and their -// proofs. -type RecordEntriesGet struct { - Entries []RecordEntryIdentifier `json:"entries"` // Entries to retrieve -} - -// RecordEntriesGetReply is the array of the requested record entries and proofs. -type RecordEntriesGetReply struct { - Proofs []RecordEntryProof `json:"proofs"` // All proofs and data -} - -// RecordFsck performs an fsck on the record to ensure integrity. -type RecordFsck struct { - Id int64 `json:"id"` // Record ID -} - -// RecordFsckReply is the reply to the RecordFsck command. -type RecordFsckReply struct{} diff --git a/tlog/tclient/tclient.go b/tlog/tclient/tclient.go deleted file mode 100644 index 157f3a2fe..000000000 --- a/tlog/tclient/tclient.go +++ /dev/null @@ -1,897 +0,0 @@ -package main - -import ( - "bufio" - "bytes" - "crypto" - _ "crypto/sha256" - "encoding/base64" - "encoding/json" - "flag" - "fmt" - "io/ioutil" - "net/http" - "net/url" - "os" - "path/filepath" - "strconv" - "strings" - - "github.com/davecgh/go-spew/spew" - "github.com/decred/dcrd/dcrutil" - dcrtime "github.com/decred/dcrtime/api/v1" - "github.com/decred/politeia/politeiad/api/v1/identity" - v1 "github.com/decred/politeia/tlog/api/v1" - tlogutil "github.com/decred/politeia/tlog/util" - "github.com/decred/politeia/util" - "github.com/google/trillian" - "github.com/google/trillian/client" - tcrypto "github.com/google/trillian/crypto" - "github.com/google/trillian/crypto/keys/der" - _ "github.com/google/trillian/merkle/rfc6962" - "github.com/google/trillian/types" -) - -var ( - // Server defaults - defaultHomeDir = dcrutil.AppDataDir("tclient", false) - defaultHTTPSCertFile = filepath.Join(defaultHomeDir, "https.cert") - - rpcuser = flag.String("rpcuser", "", "RPC user name for privileged calls") - rpcpass = flag.String("rpcpass", "", "RPC password for privileged calls") - rpchost = flag.String("rpchost", "127.0.0.1", "RPC host") - rpccert = flag.String("rpccert", defaultHTTPSCertFile, "RPC certificate") - testnet = flag.Bool("testnet", false, "Use testnet port") - printJson = flag.Bool("json", false, "Print JSON") - identityFilename = flag.String("identity", filepath.Join(defaultHomeDir, "identity.bin"), "Client identity") - publicKeyFilename = flag.String("publickey", filepath.Join(defaultHomeDir, "server.der"), "Server identity") - - verify = false // Don't validate server TLS certificate - - myID *identity.FullIdentity // our identity - publicKey crypto.PublicKey // remote server signing key -) - -// getErrorFromResponse extracts a user-readable string from the response from -// tlog, which will contain a JSON error. -func getErrorFromResponse(r *http.Response) (string, error) { - var errMsg string - decoder := json.NewDecoder(r.Body) - if r.StatusCode == http.StatusInternalServerError { - var e v1.ErrorReply - if err := decoder.Decode(&e); err != nil { - return "", err - } - errMsg = fmt.Sprintf("%v", e.ErrorCode) - } else { - var e v1.UserError - if err := decoder.Decode(&e); err != nil { - return "", err - } - errMsg = v1.ErrorStatus[e.ErrorCode] + " " - if e.ErrorContext != nil && len(e.ErrorContext) > 0 { - errMsg += strings.Join(e.ErrorContext, ", ") - } - } - - return errMsg, nil -} - -func usage() { - fmt.Fprintf(os.Stderr, "usage: tclient [flags] [arguments]\n") - fmt.Fprintf(os.Stderr, " flags:\n") - flag.PrintDefaults() - fmt.Fprintf(os.Stderr, "\n actions:\n") - fmt.Fprintf(os.Stderr, " list - List all records "+ - "\n") - fmt.Fprintf(os.Stderr, " publickey - Retrieve the tserver public "+ - "key\n") - fmt.Fprintf(os.Stderr, " recordnew - Create a new record "+ - "[key=value]... ...\n") - fmt.Fprintf(os.Stderr, " recordappend - Append a record "+ - " [key=value]... ...\n") - fmt.Fprintf(os.Stderr, " recordget - Retrieve a record "+ - "\n") - fmt.Fprintf(os.Stderr, " recordentriesget - Retrieve the entries for a "+ - "record ...\n") - fmt.Fprintf(os.Stderr, " fsck - Run fsck on a record "+ - "\n") - fmt.Fprintf(os.Stderr, "\n") -} - -// handleFile returns hash, signature of the string hash and data as strings to -// send to server. -func handleFile(filename string) (*v1.RecordEntry, error) { - mime, data, err := util.LoadFile2(filename) - if err != nil { - return nil, err - } - - // Encode data descriptor - dd, err := json.Marshal(v1.DataDescriptor{ - Type: v1.DataTypeMime, - Descriptor: mime, - }) - if err != nil { - return nil, err - } - - re := tlogutil.RecordEntryNew(myID, dd, data) - return &re, nil -} - -func handleError(r *http.Response) error { - if r.StatusCode != http.StatusOK { - e, err := getErrorFromResponse(r) - if err != nil { - return fmt.Errorf("%v", r.Status) - } - return fmt.Errorf("%v: %v", r.Status, e) - } - return nil -} - -func printTree(tree trillian.Tree) { - fmt.Printf("TreeId : %v\n", tree.TreeId) - fmt.Printf("TreeState : %v\n", tree.TreeState) - fmt.Printf("TreeType : %v\n", tree.TreeType) - fmt.Printf("HashStrategy : %v\n", tree.HashStrategy) - fmt.Printf("HashAlgorithm : %v\n", tree.HashAlgorithm) - fmt.Printf("SignatureAlgorithm: %v\n", tree.SignatureAlgorithm) - fmt.Printf("DisplayName : %v\n", tree.DisplayName) - fmt.Printf("Description : %v\n", tree.Description) - fmt.Printf("PublicKey : %v\n", tree.PublicKey) - fmt.Printf("MaxRootDuration : %v\n", tree.MaxRootDuration) - fmt.Printf("CreateTime : %v\n", tree.CreateTime) - fmt.Printf("UpdateTime : %v\n", tree.UpdateTime) - fmt.Printf("Deleted : %v\n", tree.Deleted) - fmt.Printf("DeleteTime : %v\n", tree.DeleteTime) -} - -func printRoot(root trillian.SignedLogRoot) { - fmt.Printf("KeyHint : %x\n", root.KeyHint) - fmt.Printf("LogRoot : %x\n", root.LogRoot) - fmt.Printf("LogRootSignature: %x\n", root.LogRootSignature) -} - -func printLogRootV1(l types.LogRootV1) { - fmt.Printf("TreeSize : %v\n", l.TreeSize) - fmt.Printf("RootHash : %x\n", l.RootHash) - fmt.Printf("TimestampNanos: %v\n", l.TimestampNanos) - fmt.Printf("Revision : %v\n", l.Revision) - fmt.Printf("Metadata : %x\n", l.Metadata) -} - -func printLeaf(leaf trillian.LogLeaf) { - fmt.Printf("MerkleLeafHash : %x\n", leaf.MerkleLeafHash) - fmt.Printf("LeafValue : %x\n", leaf.LeafValue) - fmt.Printf("ExtraData : %s\n", leaf.ExtraData) - fmt.Printf("LeafIndex : %v\n", leaf.LeafIndex) - fmt.Printf("LeafIdentityHash : %x\n", leaf.LeafIdentityHash) - fmt.Printf("QueueTimestamp : %v\n", leaf.QueueTimestamp) - fmt.Printf("IntegrateTimestamp: %v\n", leaf.IntegrateTimestamp) -} - -func printQueuedLeaf(ql trillian.QueuedLogLeaf) { - printLeaf(*ql.Leaf) - fmt.Printf("Status : %v\n", ql.Status) -} - -func printRecordEntry(r v1.RecordEntry) { - fmt.Printf("PublicKey: %v\n", r.PublicKey) - fmt.Printf("Hash : %v\n", r.Hash) - fmt.Printf("Signature: %v\n", r.Signature) - fmt.Printf("DataHint : %v\n", r.DataHint) - data, err := base64.StdEncoding.DecodeString(r.Data) - if err != nil { - panic(err) // for now - } - if len(data) > 40 { - fmt.Printf("Data : ...\n") - } else { - fmt.Printf("Data : %s\n", data) // Assume string for now - } -} - -func printProof(p trillian.Proof) { - fmt.Printf("LeafIndex: %v\n", p.LeafIndex) - for k, v := range p.Hashes { - fmt.Printf("Hash(%3v): %x\n", k, v) - } -} - -func printAnchor(a dcrtime.ChainInformation) { - fmt.Printf("Timestamp : %v\n", a.ChainTimestamp) - fmt.Printf("Tx : %v\n", a.Transaction) - fmt.Printf("MerkleRoot: %v\n", a.MerkleRoot) - for k, v := range a.MerklePath.Hashes { - fmt.Printf("Hash(%3v) : %x\n", k, v) - } -} - -func list() error { - // convert to JSON and sent it to server - b, err := json.Marshal(v1.List{}) - if err != nil { - return err - } - - if *printJson { - fmt.Println(string(b)) - } - - c, err := util.NewClient(verify, *rpccert) - if err != nil { - return err - } - r, err := c.Post(*rpchost+v1.RouteList, "application/json", - bytes.NewReader(b)) - if err != nil { - return err - } - defer r.Body.Close() - - err = handleError(r) - if err != nil { - return err - } - - body := util.ConvertBodyToByteArray(r.Body, *printJson) - var lr v1.ListReply - err = json.Unmarshal(body, &lr) - if err != nil { - return fmt.Errorf("Could no unmarshal ListReply: %v", err) - } - - if !*printJson { - for _, v := range lr.Trees { - fmt.Printf("\n") - printTree(*v) - } - } - - return nil -} - -func getPublicKey() error { - // Make sure we won't overwrite the key file - pkf := util.CleanAndExpandPath(*publicKeyFilename) - if util.FileExists(pkf) { - return fmt.Errorf("public key file already exists") - } - - // convert to JSON and sent it to server - b, err := json.Marshal(v1.PublicKey{}) - if err != nil { - return err - } - - if *printJson { - fmt.Println(string(b)) - } - - c, err := util.NewClient(verify, *rpccert) - if err != nil { - return err - } - r, err := c.Post(*rpchost+v1.RoutePublicKey, "application/json", - bytes.NewReader(b)) - if err != nil { - return err - } - defer r.Body.Close() - - err = handleError(r) - if err != nil { - return err - } - - body := util.ConvertBodyToByteArray(r.Body, *printJson) - var pkr v1.PublicKeyReply - err = json.Unmarshal(body, &pkr) - if err != nil { - return fmt.Errorf("Could not unmarshal PublicKeyReply: %v", err) - } - - keyb, err := base64.StdEncoding.DecodeString(pkr.SigningKey) - if err != nil { - return err - } - publicKey, err = der.UnmarshalPublicKey(keyb) - if err != nil { - return err - } - - // Save? - spew.Dump(publicKey) - fmt.Printf("\nSave to %v or ctrl-c to abort ", pkf) - scanner := bufio.NewScanner(os.Stdin) - scanner.Scan() - if err = scanner.Err(); err != nil { - return err - } - if len(scanner.Text()) != 0 { - pkf = scanner.Text() - } - - // Save public key - err = ioutil.WriteFile(pkf, keyb, 0600) - if err != nil { - return err - } - - return nil -} - -func recordParse(index int) ([]v1.RecordEntry, error) { - flags := flag.Args()[index:] // Chop off action. - if len(flags) == 0 { - return nil, fmt.Errorf("not enough arguments, expected " + - "[key=value]... ...") - } - - re := make([]v1.RecordEntry, 0, len(flags)) - for _, v := range flags { - // See if we are key value or a filename - _, err := os.Stat(v) - if err != nil { - // maybe key value - if !strings.Contains(v, "=") { - return nil, err - } - // parse key=value - a := strings.SplitN(v, "=", 2) - if len(a) != 2 { - return nil, fmt.Errorf("not a valid "+ - "'key=value': %v", v) - } - - // Encode data descriptor - dd, err := json.Marshal(v1.DataDescriptor{ - Type: v1.DataTypeKeyValue, - }) - if err != nil { - return nil, err - } - // Encode data - kv, err := json.Marshal(v1.DataKeyValue{ - Key: a[0], - Value: a[1], - }) - if err != nil { - return nil, err - } - re = append(re, tlogutil.RecordEntryNew(myID, dd, kv)) - continue - } - r, err := handleFile(v) - if err != nil { - return nil, err - } - re = append(re, *r) - } - - return re, nil -} - -func recordNew() error { - re, err := recordParse(1) - if err != nil { - return err - } - rn := v1.RecordNew{RecordEntries: re} - - // convert to JSON and sent it to server - b, err := json.Marshal(rn) - if err != nil { - return err - } - - if *printJson { - fmt.Println(string(b)) - } - - c, err := util.NewClient(verify, *rpccert) - if err != nil { - return err - } - r, err := c.Post(*rpchost+v1.RouteRecordNew, "application/json", - bytes.NewReader(b)) - if err != nil { - return err - } - defer r.Body.Close() - - err = handleError(r) - if err != nil { - return err - } - - body := util.ConvertBodyToByteArray(r.Body, *printJson) - var rnr v1.RecordNewReply - err = json.Unmarshal(body, &rnr) - if err != nil { - return fmt.Errorf("Could not unmarshal RecordNewReply: %v", err) - } - - // Let's verify what came back - verifier, err := client.NewLogVerifierFromTree(&rnr.Tree) - if err != nil { - return err - } - lrInitial, err := tcrypto.VerifySignedLogRoot(verifier.PubKey, - crypto.SHA256, &rnr.InitialRoot) - if err != nil { - return err - } - lrSTH, err := tcrypto.VerifySignedLogRoot(verifier.PubKey, - crypto.SHA256, &rnr.STH) - if err != nil { - return err - } - for _, v := range rnr.Proofs { - err := tlogutil.QueuedLeafProofVerify(publicKey, lrSTH, v) - if err != nil { - return err - } - } - - if !*printJson { - fmt.Printf("\nTree:\n") - printTree(rnr.Tree) - - fmt.Printf("\nInitial root:\n") - printRoot(rnr.InitialRoot) - fmt.Printf("\nInitial root LogRootV1:\n") - printLogRootV1(*lrInitial) - - fmt.Printf("\nSTH:\n") - printRoot(rnr.STH) - fmt.Printf("\nSTH LogRootV1:\n") - printLogRootV1(*lrSTH) - - fmt.Printf("\nQueued leaves:\n") - for _, v := range rnr.Proofs { - printQueuedLeaf(v.QueuedLeaf) - fmt.Printf("\n") - } - } - - return nil -} - -func recordAppend() error { - // get tree id - flags := flag.Args()[1:] // Chop off action. - if len(flags) < 2 { - return fmt.Errorf("not enough arguments, expected " + - " [key=value]... ...") - } - id, err := strconv.ParseInt(flags[0], 10, 64) - if err != nil { - return err - } - - re, err := recordParse(2) - if err != nil { - return err - } - ra := v1.RecordAppend{ - Id: id, - RecordEntries: re, - } - - // convert to JSON and sent it to server - b, err := json.Marshal(ra) - if err != nil { - return err - } - - if *printJson { - fmt.Println(string(b)) - } - - c, err := util.NewClient(verify, *rpccert) - if err != nil { - return err - } - r, err := c.Post(*rpchost+v1.RouteRecordAppend, "application/json", - bytes.NewReader(b)) - if err != nil { - return err - } - defer r.Body.Close() - - err = handleError(r) - if err != nil { - return err - } - - body := util.ConvertBodyToByteArray(r.Body, *printJson) - var rar v1.RecordAppendReply - err = json.Unmarshal(body, &rar) - if err != nil { - return fmt.Errorf("Could not unmarshal RecordAppendReply: %v", - err) - } - - // Let's verify what came back - lrv1, err := tcrypto.VerifySignedLogRoot(publicKey, - crypto.SHA256, &rar.STH) - if err != nil { - return fmt.Errorf("VerifySignedLogRoot: %v", err) - } - for _, v := range rar.Proofs { - err := tlogutil.QueuedLeafProofVerify(publicKey, lrv1, v) - if err != nil { - return err - } - } - - if !*printJson { - fmt.Printf("\nSTH:\n") - printRoot(rar.STH) - - fmt.Printf("\nQueuedLeaves\n") - for _, v := range rar.Proofs { - printQueuedLeaf(v.QueuedLeaf) - fmt.Printf("\n") - } - } - - return nil -} - -func recordGet() error { - flags := flag.Args()[1:] // Chop off action. - if len(flags) != 1 { - return fmt.Errorf("not enough arguments, expected " + - "") - } - - id, err := strconv.ParseInt(flags[0], 10, 64) - if err != nil { - return err - } - - // convert to JSON and sent it to server - b, err := json.Marshal(v1.RecordGet{Id: id}) - if err != nil { - return err - } - - if *printJson { - fmt.Println(string(b)) - } - - c, err := util.NewClient(verify, *rpccert) - if err != nil { - return err - } - r, err := c.Post(*rpchost+v1.RouteRecordGet, "application/json", - bytes.NewReader(b)) - if err != nil { - return err - } - defer r.Body.Close() - - err = handleError(r) - if err != nil { - return err - } - - body := util.ConvertBodyToByteArray(r.Body, *printJson) - var rgr v1.RecordGetReply - err = json.Unmarshal(body, &rgr) - if err != nil { - return fmt.Errorf("Could not unmarshal RecordGetReply: %v", err) - } - - // Verify STH - _, err = tcrypto.VerifySignedLogRoot(publicKey, crypto.SHA256, - &rgr.STH) - if err != nil { - return err - } - - // Verify record entry proofs - for _, v := range rgr.Proofs { - // XXX anchor records won't have a public key or signature. - // This is a temporary workaround until trillian properly - // supports ed25519. Skip verification for anchor records - // until this is fixed. - b, err := base64.StdEncoding.DecodeString(v.RecordEntry.DataHint) - if err != nil { - return err - } - var dh v1.DataDescriptor - err = json.Unmarshal(b, &dh) - if err != nil { - return err - } - if dh.Type == v1.DataTypeStructure && - dh.Descriptor == v1.DataDescriptorAnchor { - continue - } - - err = tlogutil.RecordEntryProofVerify(publicKey, v) - if err != nil { - return err - } - } - - if !*printJson { - fmt.Printf("\nSTH:\n") - printRoot(rgr.STH) - - fmt.Printf("\nLeaves:\n") - for _, v := range rgr.Proofs { - printLeaf(*v.Leaf) - fmt.Printf("\n") - } - - fmt.Printf("\nRecords:\n") - for _, v := range rgr.Proofs { - printRecordEntry(*v.RecordEntry) - fmt.Printf("\n") - } - } - - return nil -} - -func recordEntriesGet() error { - flags := flag.Args()[1:] // Chop off action. - if len(flags) == 0 { - return fmt.Errorf("not enough arguments, expected " + - "...") - } - - reg := v1.RecordEntriesGet{ - Entries: make([]v1.RecordEntryIdentifier, 0, len(flags)), - } - for _, v := range flags { - a := strings.SplitN(v, ",", 2) - if len(a) != 2 { - return fmt.Errorf("invalid format, expected: " + - "id,merklehash") - } - id, err := strconv.ParseInt(a[0], 10, 64) - if err != nil { - return err - } - if !util.IsDigest(a[1]) { - return fmt.Errorf("invalid hash: %v", a[1]) - } - reg.Entries = append(reg.Entries, v1.RecordEntryIdentifier{ - Id: id, - MerkleHash: a[1], - }) - } - - // convert to JSON and sent it to server - b, err := json.Marshal(reg) - if err != nil { - return err - } - - if *printJson { - fmt.Println(string(b)) - } - - c, err := util.NewClient(verify, *rpccert) - if err != nil { - return err - } - r, err := c.Post(*rpchost+v1.RouteRecordEntriesGet, "application/json", - bytes.NewReader(b)) - if err != nil { - return err - } - defer r.Body.Close() - - err = handleError(r) - if err != nil { - return err - } - - body := util.ConvertBodyToByteArray(r.Body, *printJson) - var rgr v1.RecordEntriesGetReply - err = json.Unmarshal(body, &rgr) - if err != nil { - return fmt.Errorf("Could not unmarshal RecordEntriesGetReply: %v", err) - } - - // Verify record entry proofs - for _, v := range rgr.Proofs { - err := tlogutil.RecordEntryProofVerify(publicKey, v) - if err != nil { - return err - } - } - - if !*printJson { - fmt.Printf("\nLeaves:\n") - for _, v := range rgr.Proofs { - printLeaf(*v.Leaf) - fmt.Printf("\n") - } - - fmt.Printf("\nRecords:\n") - for _, v := range rgr.Proofs { - printRecordEntry(*v.RecordEntry) - fmt.Printf("\n") - } - - fmt.Printf("\nAnchors:\n") - for _, v := range rgr.Proofs { - if v.Anchor == nil { - fmt.Printf("No anchor yet\n\n") - continue - } - printAnchor(*v.Anchor) - fmt.Printf("\n") - } - - fmt.Printf("\nProofs:\n") - for _, v := range rgr.Proofs { - if v.Proof == nil { - fmt.Printf("No proof yet\n\n") - continue - } - printProof(*v.Proof) - fmt.Printf("\n") - } - } - - return nil -} - -func recordFsck() error { - flags := flag.Args()[1:] // Chop off action. - if len(flags) != 1 { - return fmt.Errorf("not enough arguments, expected " + - "") - } - - id, err := strconv.ParseInt(flags[0], 10, 64) - if err != nil { - return err - } - - // convert to JSON and sent it to server - b, err := json.Marshal(v1.RecordFsck{Id: id}) - if err != nil { - return err - } - - if *printJson { - fmt.Println(string(b)) - } - - c, err := util.NewClient(verify, *rpccert) - if err != nil { - return err - } - r, err := c.Post(*rpchost+v1.RouteRecordFsck, "application/json", - bytes.NewReader(b)) - if err != nil { - return err - } - defer r.Body.Close() - - err = handleError(r) - if err != nil { - return err - } - - body := util.ConvertBodyToByteArray(r.Body, *printJson) - var rfr v1.RecordFsckReply - err = json.Unmarshal(body, &rfr) - if err != nil { - return fmt.Errorf("Could not unmarshal RecordFsckReply: %v", err) - } - - return nil -} - -func _main() error { - flag.Parse() - if len(flag.Args()) == 0 { - usage() - return fmt.Errorf("must provide action") - } - - port := "65535" - if *testnet { - port = "65534" - } - - *rpchost = util.NormalizeAddress(*rpchost, port) - - // Set port if not specified. - u, err := url.Parse("https://" + *rpchost) - if err != nil { - return err - } - *rpchost = u.String() - - // Generate ed25519 identity to save messages, tokens etc. - idf := util.CleanAndExpandPath(*identityFilename) - if !util.FileExists(idf) { - err = os.MkdirAll(defaultHomeDir, 0700) - if err != nil { - return err - } - fmt.Println("Generating signing identity...") - id, err := identity.New() - if err != nil { - return err - } - err = id.Save(idf) - if err != nil { - return err - } - fmt.Println("Signing identity created...") - } - - // Load identity. - myID, err = identity.LoadFullIdentity(idf) - if err != nil { - return err - } - - // See if we have a remote identity stored - pkf := util.CleanAndExpandPath(*publicKeyFilename) - if !util.FileExists(pkf) { - if len(flag.Args()) != 1 || flag.Args()[0] != "publickey" { - return fmt.Errorf("Missing remote signing key. Use " + - "the 'publickey' command to retrieve it") - } - } else { - // Load public key - pk, err := ioutil.ReadFile(pkf) - if err != nil { - return err - } - publicKey, err = der.UnmarshalPublicKey(pk) - if err != nil { - return err - } - } - - // Scan through command line arguments. - for i, a := range flag.Args() { - // Select action - if i == 0 { - switch a { - case "list": - return list() - case "publickey": - return getPublicKey() - case "recordappend": - return recordAppend() - case "recordnew": - return recordNew() - case "recordget": - return recordGet() - case "recordentriesget": - return recordEntriesGet() - case "fsck": - return recordFsck() - default: - return fmt.Errorf("invalid action: %v", a) - } - } - } - - return nil -} - -func main() { - err := _main() - if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) - } -} diff --git a/tlog/tserver/anchor.go b/tlog/tserver/anchor.go deleted file mode 100644 index bd541e7c6..000000000 --- a/tlog/tserver/anchor.go +++ /dev/null @@ -1,657 +0,0 @@ -package main - -import ( - "bytes" - "crypto" - "crypto/sha256" - "encoding/base64" - "encoding/hex" - "encoding/json" - "fmt" - "time" - - "github.com/davecgh/go-spew/spew" - v1 "github.com/decred/politeia/tlog/api/v1" - tlogutil "github.com/decred/politeia/tlog/util" - "github.com/decred/politeia/util" - "github.com/google/trillian" - "github.com/google/trillian/client" - tcrypto "github.com/google/trillian/crypto" - "github.com/google/trillian/merkle" - "github.com/google/trillian/merkle/hashers" - "github.com/google/trillian/types" - "google.golang.org/grpc/codes" -) - -const ( - // anchorSchedule determines how often we anchor records - // Seconds Minutes Hours Days Months DayOfWeek - anchorSchedule = "0 56 * * * *" // At 56 minutes every hour -) - -// findLeafAnchor returns the DataAnchor for the provided leaf. If there is no -// anchor it returns nil. -func (t *tserver) findLeafAnchor(treeId int64, leaf *trillian.LogLeaf) (*v1.DataAnchor, error) { - log.Tracef("findAnchorByLeafHash %v %x", treeId, leaf.MerkleLeafHash) - - // Retrieve STH - reply, err := t.client.GetLatestSignedLogRoot(t.ctx, - &trillian.GetLatestSignedLogRootRequest{ - LogId: treeId, - }) - if err != nil { - return nil, err - } - var lrv1 types.LogRootV1 - err = lrv1.UnmarshalBinary(reply.SignedLogRoot.LogRoot) - if err != nil { - return nil, err - } - - // Start at the provided leaf and scan the tree for - // an anchor. We request a page of leaves at a time. - var ( - anchor *v1.DataAnchor - pageSize int64 = 10 - startIndex int64 = leaf.LeafIndex + 1 - ) - for anchor == nil && startIndex < int64(lrv1.TreeSize) { - // Retrieve a page of leaves - glbrr, err := t.client.GetLeavesByRange(t.ctx, - &trillian.GetLeavesByRangeRequest{ - LogId: treeId, - StartIndex: startIndex, - Count: pageSize, - }) - if err != nil { - return nil, err - } - - // Scan leaves for an anchor - for _, v := range glbrr.Leaves { - // Retrieve leaf payload from backend - payload, err := t.s.Get(v.ExtraData) - if err != nil { - return nil, err - } - re, err := deblob(payload) - if err != nil { - return nil, err - } - - // Investigate data hint - dhb, err := base64.StdEncoding.DecodeString(re.DataHint) - if err != nil { - return nil, err - } - var dh v1.DataDescriptor - err = json.Unmarshal(dhb, &dh) - if err != nil { - return nil, err - } - if !(dh.Type == v1.DataTypeStructure && - dh.Descriptor == v1.DataDescriptorAnchor) { - // Not a anchor. Try the next one. - continue - } - - // Found one! - data, err := base64.StdEncoding.DecodeString(re.Data) - if err != nil { - return nil, err - } - var da v1.DataAnchor - err = json.Unmarshal(data, &da) - if err != nil { - return nil, err - } - anchor = &da - break - } - - // Increment startIndex and try again - startIndex += pageSize - } - - return anchor, nil -} - -// findLatestAnchor scans all leaves in a tree and returns the latest anchor. -// If there is no anchor it returns nil. -func (t *tserver) findLatestAnchor(tree *trillian.Tree, lrv1 *types.LogRootV1) (*v1.DataAnchor, error) { - log.Tracef("findLatestAnchor") - - // Get leaves - startIndex := int64(lrv1.TreeSize) - 1 - if startIndex < 0 { - startIndex = 1 - } - glbrr, err := t.client.GetLeavesByRange(t.ctx, - &trillian.GetLeavesByRangeRequest{ - LogId: tree.TreeId, - StartIndex: startIndex, - Count: int64(lrv1.TreeSize), - }) - if err != nil { - return nil, err - } - - // We can be clever and request only the top leaf and see if it is an - // anchor. Note that the FSCK code must walk the entire tree backwards. - log.Tracef("findLatestAnchor get: %s", glbrr.Leaves[0].ExtraData) - payload, err := t.s.Get(glbrr.Leaves[0].ExtraData) - if err != nil { - return nil, err - } - re, err := deblob(payload) - if err != nil { - return nil, err - } - - // investigate data hint - dhb, err := base64.StdEncoding.DecodeString(re.DataHint) - if err != nil { - return nil, err - } - var dh v1.DataDescriptor - err = json.Unmarshal(dhb, &dh) - if err != nil { - log.Errorf("findLatestAnchor invalid datahint %v", dh.Type) - return nil, err - } - if !(dh.Type == v1.DataTypeStructure && - dh.Descriptor == v1.DataDescriptorAnchor) { - return nil, nil - } - - // Found one! - data, err := base64.StdEncoding.DecodeString(re.Data) - if err != nil { - return nil, err - } - var da v1.DataAnchor - err = json.Unmarshal(data, &da) - if err != nil { - log.Errorf("findLatestAnchor invalid DataAnchor %v", err) - return nil, err - } - return &da, nil -} - -// scanAllRecords scans all records and determines if a record is dirty and at -// what height. -func (t *tserver) scanAllRecords() error { - log.Tracef("scanAllRecords") - // List trees - ltr, err := t.admin.ListTrees(t.ctx, &trillian.ListTreesRequest{}) - if err != nil { - return err - } - - if len(ltr.Tree) == 0 { - log.Infof("scanAllRecords: nothing dirty") - return nil - } - - // Get all records - log.Debugf("scanAllRecords scanning records: %v", len(ltr.Tree)) - for _, tree := range ltr.Tree { - // Retrieve STH - _, lrv1, err := t.getLatestSignedLogRoot(tree) - if err != nil { - return err - } - log.Tracef("scanAllRecords scanning %v %v", lrv1.TreeSize, - tree.TreeId) - - // Load data from backend to find anchors - anchor, err := t.findLatestAnchor(tree, lrv1) - if err != nil { - return err - } - - if anchor == nil { - // No anchor yet in this record - t.Lock() - t.dirty[tree.TreeId] = int64(lrv1.TreeSize) - t.Unlock() - continue - } - } - - t.Lock() - r := len(t.dirty) - t.Unlock() - log.Infof("Unanchored records: %v", r) - - return nil -} - -// anchorRecords is a function that is periodically called to anchor dirty -// records. -func (t *tserver) anchorRecords() { - var exitError error // Set on exit if there is an error - defer func() { - if exitError != nil { - log.Errorf("anchorRecords %v", exitError) - } - }() - - // Copy dirty records - t.Lock() - anchors := make([]v1.DataAnchor, 0, len(t.dirty)) - for k := range t.dirty { - anchors = append(anchors, v1.DataAnchor{RecordId: k}) - } - t.Unlock() - if len(anchors) == 0 { - log.Infof("Nothing to anchor") - return - } - - // XXX Abort entire run if there is an error for now. - - // Scan over the dirty records - trees := make([]*trillian.Tree, len(anchors)) // Cache trees - hashes := make([]*[sha256.Size]byte, len(anchors)) // Coalesce records - lrv1s := make([]*types.LogRootV1, len(anchors)) // We need TreeSize - for k, v := range anchors { - log.Tracef("Obtaining anchoring data: %v", v.RecordId) - - tree, err := t.getTree(v.RecordId) - if err != nil { - exitError = fmt.Errorf("getTree %v", err) - return - } - sth, lrv1, err := t.getLatestSignedLogRoot(tree) - if err != nil { - exitError = fmt.Errorf("getLatestSignedLogRoot: %v", - err) - return - } - - trees[k] = tree - anchors[k].STH = *sth - lrv1s[k] = lrv1 - hashes[k] = util.Hash(sth.LogRoot) - } - - // Ensure we aren't reentrant. This is done deliberately late in the - // process. - t.Lock() - if t.droppingAnchor { - // This shouldn't happen so let's warn the user of something - // misbehaving. - t.Unlock() - log.Infof("Dropping anchor already in progress") - return - } - t.Unlock() - - log.Infof("Anchoring records: %v", len(anchors)) - - err := util.Timestamp("tserver", t.cfg.DcrtimeHost, hashes) - if err != nil { - exitError = err - return - } - - go t.waitForAchor(trees, anchors, hashes, lrv1s) -} - -// waitForAchor waits until an anchor drops. -func (t *tserver) waitForAchor(trees []*trillian.Tree, anchors []v1.DataAnchor, hashes []*[sha256.Size]byte, lrv1s []*types.LogRootV1) { - // Ensure we are not reentrant - t.Lock() - if t.droppingAnchor { - log.Errorf("waitForAchor: called reentrantly") - return - } - t.droppingAnchor = true - t.Unlock() - - // Whatever happens in this function we must clear droppingAnchor. - defer func() { - t.Lock() - t.droppingAnchor = false - t.Unlock() - }() - - // Wait for anchor to drop - log.Infof("Waiting for anchor to drop") - - // Construct verify command - waitFor := make([]string, 0, len(hashes)) - for _, v := range hashes { - waitFor = append(waitFor, hex.EncodeToString(v[:])) - } - - period := time.Duration(1) // in minutes - retries := 30 / int(period) // wait up to 30 minutes - ticker := time.NewTicker(period * time.Minute) - defer ticker.Stop() - for try := 0; try < retries; try++ { - restart: - <-ticker.C - - log.Tracef("anchorRecords checking anchor") - - vr, err := util.Verify("tserver", t.cfg.DcrtimeHost, waitFor) - if err != nil { - if _, ok := err.(util.ErrNotAnchored); ok { - // Anchor not dropped, try again - log.Tracef("anchorRecords: try %v %v", try, err) - continue - } - log.Errorf("waitForAnchor: %v", err) - return - } - - // Make sure we are actually anchored. - for _, v := range vr.Digests { - if v.ChainInformation.ChainTimestamp == 0 { - log.Tracef("anchorRecords ChainTimestamp 0: %v", - v.Digest) - goto restart - } - } - - log.Tracef("%T %v", vr, spew.Sdump(vr)) - - log.Infof("Anchor dropped") - - // Drop anchor records - for k, v := range anchors { - da := v1.DataAnchor{ - RecordId: v.RecordId, - STH: v.STH, - VerifyDigest: vr.Digests[k], - } - log.Tracef("DataAnchor: %v", spew.Sdump(da)) - - data, err := json.Marshal(da) - if err != nil { - log.Errorf("waitForAchor: marshal data: %v", - err) - continue - } - - // construct a RecordEntry - dd, err := json.Marshal(v1.DataDescriptor{ - Type: v1.DataTypeStructure, - Descriptor: v1.DataDescriptorAnchor, - }) - if err != nil { - log.Errorf("waitForAchor: marshal desc: %v", - err) - continue - } - re := tlogutil.RecordEntryNew(nil, dd, data) - - treeID := trees[k].TreeId - proofs, sth, err := t.appendRecord(trees[k], &v.STH, - []v1.RecordEntry{re}) - if err != nil { - log.Errorf("waitForAchor appendRecord %v: %v", - treeID, err) - continue - } - - // Check QueuedLogLeafProofs - if len(proofs) != 1 { - log.Errorf("waitForAchor %v: QueuedLogLeaveProofs != 1", - treeID) - continue - } - ql := proofs[0].QueuedLeaf - c := codes.Code(ql.GetStatus().GetCode()) - if c != codes.OK { - log.Errorf("waitForAnchor leaf not appended %v: %v", - treeID, ql.GetStatus().GetMessage()) - continue - } - if proofs[0].Proof == nil { - log.Errorf("waitForAnchor %v: no proof", - treeID) - continue - } - - // Verify STH - verifier, err := client.NewLogVerifierFromTree(trees[k]) - if err != nil { - log.Errorf("waitForAnchor NewLogVerifierFromTree %v: %v", - treeID, err) - continue - } - lrv1, err := tcrypto.VerifySignedLogRoot(verifier.PubKey, - crypto.SHA256, sth) - if err != nil { - log.Errorf("waitForAnchor VerifySignedLogRoot %v: %v", - treeID, err) - continue - } - - // Verify inclusion proof - err = verifier.VerifyInclusionByHash(lrv1, - ql.Leaf.MerkleLeafHash, proofs[0].Proof) - if err != nil { - log.Errorf("waitForAnchor VerifyInclusionByHash %v: %v", - treeID, err) - continue - } - } - - // Fixup dirty structure - t.Lock() - defer t.Unlock() - - // Go over anchors and compare TreeSize to see if the tree was - // made dirty during anchor process. - for k, v := range anchors { - size, ok := t.dirty[v.RecordId] - if !ok { - // XXX panic? - log.Criticalf("anchorRecords id disappeared: %v", - v.RecordId) - return - } - - if int64(lrv1s[k].TreeSize) != size { - log.Tracef("anchorRecords record changed, remains "+ - "dirty %v %v != %v", v.RecordId, - lrv1s[k].TreeSize, size) - - // Updte size - t.dirty[v.RecordId] = int64(lrv1s[k].TreeSize) - continue - } - - log.Tracef("anchorRecords marking record clean: %v", - v.RecordId) - delete(t.dirty, v.RecordId) - } - - return - } - - log.Errorf("Anchor drop timeout, waited for: %v", period*time.Minute) -} - -func NewSHA256(data []byte) []byte { - hash := sha256.Sum256(data) - return hash[:] -} - -func (t *tserver) fsckRecord(tree *trillian.Tree, lrv1 *types.LogRootV1) error { - log.Tracef("fsckRecord") - - // Get leaves - glbrr, err := t.client.GetLeavesByRange(t.ctx, - &trillian.GetLeavesByRangeRequest{ - LogId: tree.TreeId, - StartIndex: 0, - Count: int64(lrv1.TreeSize), - }) - if err != nil { - return err - } - - // Perform tree coherency test - gcprr, err := t.client.GetConsistencyProof(t.ctx, - &trillian.GetConsistencyProofRequest{ - LogId: tree.TreeId, - FirstTreeSize: 1, - SecondTreeSize: int64(lrv1.TreeSize), - }) - if err != nil { - return err - } - lh, err := hashers.NewLogHasher(trillian.HashStrategy_RFC6962_SHA256) - if err != nil { - return err - } - verifier := merkle.NewLogVerifier(lh) - // The log root hash when TreeSize=1 is just the MerkleLeafHash - // of the single leaf. - var root1 []byte - for _, v := range glbrr.Leaves { - if v.LeafIndex == 0 { - root1 = v.MerkleLeafHash - break - } - } - if root1 == nil { - return fmt.Errorf("leaf index 0 not found") - } - err = verifier.VerifyConsistencyProof(1, int64(lrv1.TreeSize), - root1, lrv1.RootHash, gcprr.Proof.Hashes) - if err != nil { - return err - } - - // Walk leaves backwards - for x := len(glbrr.Leaves) - 1; x >= 0; x-- { - log.Tracef("fsckRecord get: %s", glbrr.Leaves[x].ExtraData) - payload, err := t.s.Get(glbrr.Leaves[x].ExtraData) - if err != nil { - // Absence of a record entry does not necessarily mean something - // is wrong. If there was an error during the record append call - // the trillian leaves will still exist but the record entry - // blobs would have been unwound. - log.Debugf("Record entry not found %v %v: %v", - glbrr.Leaves[x].MerkleLeafHash, - glbrr.Leaves[x].ExtraData, err) - continue - } - re, err := deblob(payload) - if err != nil { - return err - } - - // investigate data hint - dhb, err := base64.StdEncoding.DecodeString(re.DataHint) - if err != nil { - return err - } - var dh v1.DataDescriptor - err = json.Unmarshal(dhb, &dh) - if err != nil { - return fmt.Errorf("fsckRecord invalid datahint %v %v", - x, dh.Type) - } - - switch dh.Type { - case v1.DataTypeKeyValue: - data, err := base64.StdEncoding.DecodeString(re.Data) - if err != nil { - return fmt.Errorf("fsckRecord base64 %v %v", - x, err) - } - log.Tracef("fsckRecord kv: %s", data) - hash := NewSHA256(data) - if !bytes.Equal(hash, glbrr.Leaves[x].LeafValue) { - return fmt.Errorf("fsckRecord data key/value "+ - "corruption %s", - glbrr.Leaves[x].ExtraData) - } - log.Tracef("fsckRecord %s Data integrity OK", - glbrr.Leaves[x].ExtraData) - continue - case v1.DataTypeMime: - log.Tracef("fsckRecord mime: %v", dh.Descriptor) - - data, err := base64.StdEncoding.DecodeString(re.Data) - if err != nil { - return fmt.Errorf("fsckRecord base64 %v %v", - x, err) - } - hash := NewSHA256(data) - if !bytes.Equal(hash, glbrr.Leaves[x].LeafValue) { - return fmt.Errorf("fsckRecord data corruption %s", - glbrr.Leaves[x].ExtraData) - } - log.Tracef("fsckRecord %s Data integrity OK", - glbrr.Leaves[x].ExtraData) - continue - case v1.DataTypeStructure: - if !(dh.Descriptor == v1.DataDescriptorAnchor) { - log.Tracef("fsckRecord skipping %v %v", - x, dh.Type) - continue - } - log.Tracef("fsckRecord struct: %v", dh.Descriptor) - data, err := base64.StdEncoding.DecodeString(re.Data) - if err != nil { - return fmt.Errorf("fsckRecord base64 %v %v", - x, err) - } - var da v1.DataAnchor - err = json.Unmarshal(data, &da) - if err != nil { - return fmt.Errorf("fsckRecord invalid "+ - "DataAnchor %v %v", x, err) - } - - // Verify hash - hash := NewSHA256(data) - if !bytes.Equal(hash, glbrr.Leaves[x].LeafValue) { - return fmt.Errorf("fsckRecord data structure "+ - "corruption %s", - glbrr.Leaves[x].ExtraData) - } - log.Tracef("fsckRecord %s Data integrity OK", - glbrr.Leaves[x].ExtraData) - - // Verify anchor - _, err = util.Verify("tserver", t.cfg.DcrtimeHost, - []string{da.VerifyDigest.Digest}) - if err != nil { - return fmt.Errorf("fsckRecord failed "+ - "anchor %v", err) - } - - log.Tracef("fsckRecord %s Anchor OK", - glbrr.Leaves[x].ExtraData) - continue - default: - return fmt.Errorf("fsckRecord unknown type: %v", dh.Type) - } - } - - return nil -} - -func (t *tserver) fsck(f v1.RecordFsck) error { - // Get tree and STH to perform fsck on - tree, err := t.getTree(f.Id) - if err != nil { - return err - } - sth, lrv1, err := t.getLatestSignedLogRoot(tree) - if err != nil { - return err - } - _ = sth - - return t.fsckRecord(tree, lrv1) -} diff --git a/tlog/tserver/blob.go b/tlog/tserver/blob.go deleted file mode 100644 index 17086df24..000000000 --- a/tlog/tserver/blob.go +++ /dev/null @@ -1,182 +0,0 @@ -package main - -import ( - "bytes" - "compress/gzip" - "crypto/rand" - "encoding/gob" - "errors" - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - - v1 "github.com/decred/politeia/tlog/api/v1" - "github.com/google/uuid" - "golang.org/x/crypto/nacl/secretbox" -) - -func blobify(re v1.RecordEntry) ([]byte, error) { - var b bytes.Buffer - zw := gzip.NewWriter(&b) - enc := gob.NewEncoder(zw) - err := enc.Encode(re) - if err != nil { - return nil, err - } - err = zw.Close() // we must flush gzip buffers - if err != nil { - return nil, err - } - return b.Bytes(), nil -} - -func deblob(blob []byte) (*v1.RecordEntry, error) { - zr, err := gzip.NewReader(bytes.NewReader(blob)) - if err != nil { - return nil, err - } - r := gob.NewDecoder(zr) - var re v1.RecordEntry - err = r.Decode(&re) - if err != nil { - return nil, err - } - return &re, nil -} - -func NewKey() (*[32]byte, error) { - var k [32]byte - - _, err := io.ReadFull(rand.Reader, k[:]) - if err != nil { - return nil, err - } - - return &k, nil -} - -func encryptAndPack(data []byte, key *[32]byte) ([]byte, error) { - var nonce [24]byte - - // random nonce - _, err := io.ReadFull(rand.Reader, nonce[:]) - if err != nil { - return nil, err - } - - // encrypt data - blob := secretbox.Seal(nil, data, &nonce, key) - - // pack all the things - packed := make([]byte, len(nonce)+len(blob)) - copy(packed[0:], nonce[:]) - copy(packed[24:], blob) - - return packed, nil -} - -func unpackAndDecrypt(key *[32]byte, packed []byte) ([]byte, error) { - if len(packed) < 24 { - return nil, errors.New("not an sbox file") - } - - var nonce [24]byte - copy(nonce[:], packed[0:24]) - - decrypted, ok := secretbox.Open(nil, packed[24:], &nonce, key) - if !ok { - return nil, fmt.Errorf("could not decrypt") - } - return decrypted, nil -} - -type Blob interface { - Put([]byte) ([]byte, error) // Store blob and return identifier - Get([]byte) ([]byte, error) // Get blob by identifier - Del([]byte) error // Attempt to delete object - Enum(func([]byte, []byte) error) error // Enumerate over all objects -} - -// Unencrypted filesystem -var ( - ErrDoesntExist = errors.New("doesn't exist") - - _ Blob = (*blobFilesystem)(nil) -) - -// blobFilesystem provides a blob filesystem that is encrypted if a private key -// is provided. -type blobFilesystem struct { - path string // Location of files - privateKey *[32]byte // Private key -} - -func (b *blobFilesystem) Put(blob []byte) ([]byte, error) { - var err error - if b.privateKey != nil { - blob, err = encryptAndPack(blob, b.privateKey) - if err != nil { - return nil, err - } - } - filename := uuid.New().String() - err = ioutil.WriteFile(filepath.Join(b.path, filename), blob, 0600) - if err != nil { - return nil, err - } - return []byte(filename), nil -} - -func (b *blobFilesystem) Get(id []byte) ([]byte, error) { - blob, err := ioutil.ReadFile(filepath.Join(b.path, string(id))) - if err != nil { - return nil, err - } - - if b.privateKey != nil { - return unpackAndDecrypt(b.privateKey, blob) - } - return blob, nil -} - -func (b *blobFilesystem) Del(id []byte) error { - err := os.Remove(filepath.Join(b.path, string(id))) - if err != nil { - // Always return doesn't exist - return (ErrDoesntExist) - } - return nil -} - -func (b *blobFilesystem) Enum(f func([]byte, []byte) error) error { - files, err := ioutil.ReadDir(b.path) - if err != nil { - return err - } - - for _, file := range files { - if file.Name() == ".." { - continue - } - // XXX should we return filesystem errors or wrap them? - blob, err := b.Get([]byte(file.Name())) - if err != nil { - return err - } - err = f([]byte(file.Name()), blob) - if err != nil { - return err - } - } - - return nil -} - -func BlobFilesystemNew(privateKey *[32]byte, path string) (Blob, error) { - return &blobFilesystem{ - path: path, - privateKey: privateKey, - }, nil -} diff --git a/tlog/tserver/blob_test.go b/tlog/tserver/blob_test.go deleted file mode 100644 index e0f51993c..000000000 --- a/tlog/tserver/blob_test.go +++ /dev/null @@ -1,144 +0,0 @@ -package main - -import ( - "bytes" - "io/ioutil" - "os" - "reflect" - "strings" - "testing" - - "github.com/davecgh/go-spew/spew" - v1 "github.com/decred/politeia/tlog/api/v1" -) - -func TestBlob(t *testing.T) { - re := v1.RecordEntry{ - PublicKey: "ma key yo", - Hash: "ma hash yo", - Signature: "ma signature yo", - DataHint: "ma data hint yo", - Data: strings.Repeat("ma data yo", 1000), - } - blob, err := blobify(re) - if err != nil { - t.Fatal(err) - } - red, err := deblob(blob) - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(re, *red) { - t.Fatalf("want %v, got %v", spew.Sdump(re), spew.Sdump(red)) - } -} - -func TestFilesystemEncrypt(t *testing.T) { - key, err := NewKey() - if err != nil { - t.Fatal(err) - } - - dir, err := ioutil.TempDir("", "fsencrypted") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - - fs, err := BlobFilesystemNew(key, dir) - if err != nil { - t.Fatal(err) - } - - // use a key as random data - b, err := NewKey() - if err != nil { - t.Fatal(err) - } - blob := make([]byte, 32) - copy(blob, b[:]) - id, err := fs.Put(blob) - if err != nil { - t.Fatal(err) - } - - data, err := fs.Get(id) - if err != nil { - t.Fatal(err) - } - - if !bytes.Equal(data, blob) { - t.Fatal("data corruption") - } -} - -func TestFilesystem(t *testing.T) { - dir, err := ioutil.TempDir("", "fs") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - - fs, err := BlobFilesystemNew(nil, dir) - if err != nil { - t.Fatal(err) - } - - // use a key as random data - b, err := NewKey() - if err != nil { - t.Fatal(err) - } - blob := make([]byte, 32) - copy(blob, b[:]) - id, err := fs.Put(blob) - if err != nil { - t.Fatal(err) - } - - data, err := fs.Get(id) - if err != nil { - t.Fatal(err) - } - - if !bytes.Equal(data, blob) { - t.Fatal("data corruption") - } - - // Test Del - err = fs.Del(id) - if err != nil { - t.Fatal(err) - } - - // Test enum - files := 10 - objects := make(map[string]struct{}, files) - for i := 0; i < files; i++ { - id, err := fs.Put(blob) - if err != nil { - t.Fatal(err) - } - objects[string(id)] = struct{}{} - } - x := 0 - f := func(myId []byte, myBlob []byte) error { - x++ - t.Logf("%s", string(myId)) - if !bytes.Equal(data, myBlob) { - t.Fatalf("data corruption %v %v", x, string(myId)) - } - delete(objects, string(myId)) - return nil - } - err = fs.Enum(f) - if err != nil { - t.Fatal(err) - } - if len(objects) != 0 { - t.Fatalf("invalid map count got %v, want %v", len(objects), 0) - } - if x != files { - t.Fatalf("invalid blob count got %v, want %v", x, files) - } -} diff --git a/tlog/tserver/config.go b/tlog/tserver/config.go deleted file mode 100644 index 96da1ff93..000000000 --- a/tlog/tserver/config.go +++ /dev/null @@ -1,561 +0,0 @@ -// Copyright (c) 2013-2014 The btcsuite developers -// Copyright (c) 2015-2017 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import ( - "encoding/base64" - "fmt" - "net" - "os" - "os/user" - "path/filepath" - "runtime" - "sort" - "strconv" - "strings" - - "github.com/decred/dcrd/dcrutil" - v1 "github.com/decred/dcrtime/api/v1" - "github.com/decred/politeia/util" - "github.com/decred/politeia/util/version" - flags "github.com/jessevdk/go-flags" -) - -const ( - defaultConfigFilename = "tserver.conf" - defaultDataDirname = "data" - defaultLogLevel = "info" - defaultLogDirname = "logs" - defaultLogFilename = "tserver.log" - defaultTrillianHost = "localhost:8090" - - defaultMainnetPort = "65535" - defaultTestnetPort = "65534" -) - -var ( - defaultHomeDir = dcrutil.AppDataDir("tserver", false) - defaultConfigFile = filepath.Join(defaultHomeDir, defaultConfigFilename) - defaultDataDir = filepath.Join(defaultHomeDir, defaultDataDirname) - defaultSigningKeyFile = filepath.Join(defaultHomeDir, "signingkey.der") - defaultEncryptionKeyFile = filepath.Join(defaultHomeDir, "encryption.key") - defaultHTTPSKeyFile = filepath.Join(defaultHomeDir, "https.key") - defaultHTTPSCertFile = filepath.Join(defaultHomeDir, "https.cert") - defaultLogDir = filepath.Join(defaultHomeDir, defaultLogDirname) -) - -// runServiceCommand is only set to a real function on Windows. It is used -// to parse and execute service commands specified via the -s flag. -var runServiceCommand func(string) error - -// config defines the configuration options for dcrd. -// -// See loadConfig for details on the configuration load process. -type config struct { - HomeDir string `short:"A" long:"appdata" description:"Path to application home directory"` - ShowVersion bool `short:"V" long:"version" description:"Display version information and exit"` - ConfigFile string `short:"C" long:"configfile" description:"Path to configuration file"` - DataDir string `short:"b" long:"datadir" description:"Directory to store data"` - LogDir string `long:"logdir" description:"Directory to log output."` - TestNet bool `long:"testnet" description:"Use the test network"` - SimNet bool `long:"simnet" description:"Use the simulation test network"` - Profile string `long:"profile" description:"Enable HTTP profiling on given port -- NOTE port must be between 1024 and 65536"` - CPUProfile string `long:"cpuprofile" description:"Write CPU profile to the specified file"` - MemProfile string `long:"memprofile" description:"Write mem profile to the specified file"` - DebugLevel string `short:"d" long:"debuglevel" description:"Logging level for all subsystems {trace, debug, info, warn, error, critical} -- You may also specify =,=,... to set the log level for individual subsystems -- Use show to list available subsystems"` - Listeners []string `long:"listen" description:"Add an interface/port to listen for connections (default all interfaces port: 49152, testnet: 59152)"` - Version string - SigningKey string `long:"signingkey" description:"File containing the signing private key file"` - EncryptionKey string `long:"encryptionkey" description:"File containing the encryption private key file"` - HTTPSCert string `long:"httpscert" description:"File containing the https certificate file"` - HTTPSKey string `long:"httpskey" description:"File containing the https certificate key"` - RPCUser string `long:"rpcuser" description:"RPC user name for privileged commands"` - RPCPass string `long:"rpcpass" description:"RPC password for privileged commands"` - TrillianHost string `long:"trillianhost" description:"Trillian log host"` - DcrtimeHost string `long:"dcrtimehost" description:"Dcrtime ip:port"` - DcrtimeCert string `long:"dcrtimecert" description:"File containing the https certificate file for dcrtimehost"` -} - -// serviceOptions defines the configuration options for the daemon as a service -// on Windows. -type serviceOptions struct { - ServiceCommand string `short:"s" long:"service" description:"Service command {install, remove, start, stop}"` -} - -// cleanAndExpandPath expands environment variables and leading ~ in the -// passed path, cleans the result, and returns it. -func cleanAndExpandPath(path string) string { - // Nothing to do when no path is given. - if path == "" { - return path - } - - // NOTE: The os.ExpandEnv doesn't work with Windows cmd.exe-style - // %VARIABLE%, but the variables can still be expanded via POSIX-style - // $VARIABLE. - path = os.ExpandEnv(path) - - if !strings.HasPrefix(path, "~") { - return filepath.Clean(path) - } - - // Expand initial ~ to the current user's home directory, or ~otheruser - // to otheruser's home directory. On Windows, both forward and backward - // slashes can be used. - path = path[1:] - - var pathSeparators string - if runtime.GOOS == "windows" { - pathSeparators = string(os.PathSeparator) + "/" - } else { - pathSeparators = string(os.PathSeparator) - } - - userName := "" - if i := strings.IndexAny(path, pathSeparators); i != -1 { - userName = path[:i] - path = path[i:] - } - - homeDir := "" - var u *user.User - var err error - if userName == "" { - u, err = user.Current() - } else { - u, err = user.Lookup(userName) - } - if err == nil { - homeDir = u.HomeDir - } - // Fallback to CWD if user lookup fails or user has no home directory. - if homeDir == "" { - homeDir = "." - } - - return filepath.Join(homeDir, path) -} - -// validLogLevel returns whether or not logLevel is a valid debug log level. -func validLogLevel(logLevel string) bool { - switch logLevel { - case "trace": - fallthrough - case "debug": - fallthrough - case "info": - fallthrough - case "warn": - fallthrough - case "error": - fallthrough - case "critical": - return true - } - return false -} - -// supportedSubsystems returns a sorted slice of the supported subsystems for -// logging purposes. -func supportedSubsystems() []string { - // Convert the subsystemLoggers map keys to a slice. - subsystems := make([]string, 0, len(subsystemLoggers)) - for subsysID := range subsystemLoggers { - subsystems = append(subsystems, subsysID) - } - - // Sort the subsytems for stable display. - sort.Strings(subsystems) - return subsystems -} - -// parseAndSetDebugLevels attempts to parse the specified debug level and set -// the levels accordingly. An appropriate error is returned if anything is -// invalid. -func parseAndSetDebugLevels(debugLevel string) error { - // When the specified string doesn't have any delimters, treat it as - // the log level for all subsystems. - if !strings.Contains(debugLevel, ",") && !strings.Contains(debugLevel, "=") { - // Validate debug log level. - if !validLogLevel(debugLevel) { - str := "The specified debug level [%v] is invalid" - return fmt.Errorf(str, debugLevel) - } - - // Change the logging level for all subsystems. - setLogLevels(debugLevel) - - return nil - } - - // Split the specified string into subsystem/level pairs while detecting - // issues and update the log levels accordingly. - for _, logLevelPair := range strings.Split(debugLevel, ",") { - if !strings.Contains(logLevelPair, "=") { - str := "The specified debug level contains an invalid " + - "subsystem/level pair [%v]" - return fmt.Errorf(str, logLevelPair) - } - - // Extract the specified subsystem and log level. - fields := strings.Split(logLevelPair, "=") - subsysID, logLevel := fields[0], fields[1] - - // Validate subsystem. - if _, exists := subsystemLoggers[subsysID]; !exists { - str := "The specified subsystem [%v] is invalid -- " + - "supported subsytems %v" - return fmt.Errorf(str, subsysID, supportedSubsystems()) - } - - // Validate log level. - if !validLogLevel(logLevel) { - str := "The specified debug level [%v] is invalid" - return fmt.Errorf(str, logLevel) - } - - setLogLevel(subsysID, logLevel) - } - - return nil -} - -// removeDuplicateAddresses returns a new slice with all duplicate entries in -// addrs removed. -func removeDuplicateAddresses(addrs []string) []string { - result := make([]string, 0, len(addrs)) - seen := map[string]struct{}{} - for _, val := range addrs { - if _, ok := seen[val]; !ok { - result = append(result, val) - seen[val] = struct{}{} - } - } - return result -} - -// normalizeAddresses returns a new slice with all the passed peer addresses -// normalized with the given default port, and all duplicates removed. -func normalizeAddresses(addrs []string, defaultPort string) []string { - for i, addr := range addrs { - addrs[i] = util.NormalizeAddress(addr, defaultPort) - } - - return removeDuplicateAddresses(addrs) -} - -// newConfigParser returns a new command line flags parser. -func newConfigParser(cfg *config, so *serviceOptions, options flags.Options) *flags.Parser { - parser := flags.NewParser(cfg, options) - if runtime.GOOS == "windows" { - parser.AddGroup("Service Options", "Service Options", so) - } - return parser -} - -// loadConfig initializes and parses the config using a config file and command -// line options. -// -// The configuration proceeds as follows: -// 1) Start with a default config with sane settings -// 2) Pre-parse the command line to check for an alternative config file -// 3) Load configuration file overwriting defaults with any specified options -// 4) Parse CLI options and overwrite/add any specified options -// -// The above results in daemon functioning properly without any config settings -// while still allowing the user to override settings with config files and -// command line options. Command line options always take precedence. -func loadConfig() (*config, []string, error) { - // Default config. - cfg := config{ - HomeDir: defaultHomeDir, - ConfigFile: defaultConfigFile, - DebugLevel: defaultLogLevel, - DataDir: defaultDataDir, - LogDir: defaultLogDir, - HTTPSKey: defaultHTTPSKeyFile, - HTTPSCert: defaultHTTPSCertFile, - SigningKey: defaultSigningKeyFile, - EncryptionKey: defaultEncryptionKeyFile, - TrillianHost: defaultTrillianHost, - Version: version.String(), - } - - // Service options which are only added on Windows. - serviceOpts := serviceOptions{} - - // Pre-parse the command line options to see if an alternative config - // file or the version flag was specified. Any errors aside from the - // help message error can be ignored here since they will be caught by - // the final parse below. - preCfg := cfg - preParser := newConfigParser(&preCfg, &serviceOpts, flags.HelpFlag) - _, err := preParser.Parse() - if err != nil { - if e, ok := err.(*flags.Error); ok && e.Type == flags.ErrHelp { - fmt.Fprintln(os.Stderr, err) - os.Exit(0) - } - } - - // Show the version and exit if the version flag was specified. - appName := filepath.Base(os.Args[0]) - appName = strings.TrimSuffix(appName, filepath.Ext(appName)) - usageMessage := fmt.Sprintf("Use %s -h to show usage", appName) - if preCfg.ShowVersion { - fmt.Printf("%s version %s (Go version %s %s/%s)\n", appName, - version.String(), runtime.Version(), runtime.GOOS, - runtime.GOARCH) - os.Exit(0) - } - - // Perform service command and exit if specified. Invalid service - // commands show an appropriate error. Only runs on Windows since - // the runServiceCommand function will be nil when not on Windows. - if serviceOpts.ServiceCommand != "" && runServiceCommand != nil { - err := runServiceCommand(serviceOpts.ServiceCommand) - if err != nil { - fmt.Fprintln(os.Stderr, err) - } - os.Exit(0) - } - - // Update the home directory for stakepoold if specified. Since the - // home directory is updated, other variables need to be updated to - // reflect the new changes. - if preCfg.HomeDir != "" { - cfg.HomeDir, _ = filepath.Abs(preCfg.HomeDir) - - if preCfg.ConfigFile == defaultConfigFile { - cfg.ConfigFile = filepath.Join(cfg.HomeDir, defaultConfigFilename) - } else { - cfg.ConfigFile = preCfg.ConfigFile - } - if preCfg.DataDir == defaultDataDir { - cfg.DataDir = filepath.Join(cfg.HomeDir, defaultDataDirname) - } else { - cfg.DataDir = preCfg.DataDir - } - if preCfg.SigningKey == defaultSigningKeyFile { - cfg.SigningKey = filepath.Join(cfg.HomeDir, "signingkey.der") - } else { - cfg.SigningKey = preCfg.SigningKey - } - if preCfg.EncryptionKey == defaultEncryptionKeyFile { - cfg.EncryptionKey = filepath.Join(cfg.HomeDir, "encryption.key") - } else { - cfg.EncryptionKey = preCfg.EncryptionKey - } - if preCfg.HTTPSKey == defaultHTTPSKeyFile { - cfg.HTTPSKey = filepath.Join(cfg.HomeDir, "https.key") - } else { - cfg.HTTPSKey = preCfg.HTTPSKey - } - if preCfg.HTTPSCert == defaultHTTPSCertFile { - cfg.HTTPSCert = filepath.Join(cfg.HomeDir, "https.cert") - } else { - cfg.HTTPSCert = preCfg.HTTPSCert - } - if preCfg.LogDir == defaultLogDir { - cfg.LogDir = filepath.Join(cfg.HomeDir, defaultLogDirname) - } else { - cfg.LogDir = preCfg.LogDir - } - } - - // Load additional config from file. - var configFileError error - parser := newConfigParser(&cfg, &serviceOpts, flags.Default) - if !(preCfg.SimNet) || cfg.ConfigFile != defaultConfigFile { - err := flags.NewIniParser(parser).ParseFile(cfg.ConfigFile) - if err != nil { - if _, ok := err.(*os.PathError); !ok { - fmt.Fprintf(os.Stderr, "Error parsing config "+ - "file: %v\n", err) - fmt.Fprintln(os.Stderr, usageMessage) - return nil, nil, err - } - configFileError = err - } - } - - // Parse command line options again to ensure they take precedence. - remainingArgs, err := parser.Parse() - if err != nil { - if e, ok := err.(*flags.Error); !ok || e.Type != flags.ErrHelp { - fmt.Fprintln(os.Stderr, usageMessage) - } - return nil, nil, err - } - - // Create the home directory if it doesn't already exist. - funcName := "loadConfig" - err = os.MkdirAll(defaultHomeDir, 0700) - if err != nil { - // Show a nicer error message if it's because a symlink is - // linked to a directory that does not exist (probably because - // it's not mounted). - if e, ok := err.(*os.PathError); ok && os.IsExist(err) { - if link, lerr := os.Readlink(e.Path); lerr == nil { - str := "is symlink %s -> %s mounted?" - err = fmt.Errorf(str, e.Path, link) - } - } - - str := "%s: Failed to create home directory: %v" - err := fmt.Errorf(str, funcName, err) - fmt.Fprintln(os.Stderr, err) - return nil, nil, err - } - - // Multiple networks can't be selected simultaneously. - numNets := 0 - - // Count number of network flags passed; assign active network params - // while we're at it - port := defaultMainnetPort - activeNetParams = &mainNetParams - if cfg.TestNet { - numNets++ - activeNetParams = &testNet3Params - port = defaultTestnetPort - } - if cfg.SimNet { - numNets++ - // Also disable dns seeding on the simulation test network. - activeNetParams = &simNetParams - } - if numNets > 1 { - str := "%s: The testnet and simnet params can't be " + - "used together -- choose one of the three" - err := fmt.Errorf(str, funcName) - fmt.Fprintln(os.Stderr, err) - fmt.Fprintln(os.Stderr, usageMessage) - return nil, nil, err - } - - // Append the network type to the data directory so it is "namespaced" - // per network. In addition to the block database, there are other - // pieces of data that are saved to disk such as address manager state. - // All data is specific to a network, so namespacing the data directory - // means each individual piece of serialized data does not have to - // worry about changing names per network and such. - cfg.DataDir = cleanAndExpandPath(cfg.DataDir) - cfg.DataDir = filepath.Join(cfg.DataDir, netName(activeNetParams)) - - // Append the network type to the log directory so it is "namespaced" - // per network in the same fashion as the data directory. - cfg.LogDir = cleanAndExpandPath(cfg.LogDir) - cfg.LogDir = filepath.Join(cfg.LogDir, netName(activeNetParams)) - - cfg.HTTPSKey = cleanAndExpandPath(cfg.HTTPSKey) - cfg.HTTPSCert = cleanAndExpandPath(cfg.HTTPSCert) - - // Special show command to list supported subsystems and exit. - if cfg.DebugLevel == "show" { - fmt.Println("Supported subsystems", supportedSubsystems()) - os.Exit(0) - } - - // Initialize log rotation. After log rotation has been initialized, - // the logger variables may be used. - initLogRotator(filepath.Join(cfg.LogDir, defaultLogFilename)) - - // Parse, validate, and set debug log level(s). - if err := parseAndSetDebugLevels(cfg.DebugLevel); err != nil { - err := fmt.Errorf("%s: %v", funcName, err.Error()) - fmt.Fprintln(os.Stderr, err) - fmt.Fprintln(os.Stderr, usageMessage) - return nil, nil, err - } - - // Validate profile port number - if cfg.Profile != "" { - profilePort, err := strconv.Atoi(cfg.Profile) - if err != nil || profilePort < 1024 || profilePort > 65535 { - str := "%s: The profile port must be between 1024 and 65535" - err := fmt.Errorf(str, funcName) - fmt.Fprintln(os.Stderr, err) - fmt.Fprintln(os.Stderr, usageMessage) - return nil, nil, err - } - } - - // Add the default listener if none were specified. The default - // listener is all addresses on the listen port for the network - // we are to connect to. - if len(cfg.Listeners) == 0 { - cfg.Listeners = []string{ - net.JoinHostPort("", port), - } - } - - // Add default port to all listener addresses if needed and remove - // duplicate addresses. - cfg.Listeners = normalizeAddresses(cfg.Listeners, port) - - if cfg.TestNet { - var timeHost string - if len(cfg.DcrtimeHost) == 0 { - timeHost = v1.DefaultTestnetTimeHost - } else { - timeHost = cfg.DcrtimeHost - } - cfg.DcrtimeHost = util.NormalizeAddress(timeHost, - v1.DefaultTestnetTimePort) - } else { - var timeHost string - if len(cfg.DcrtimeHost) == 0 { - timeHost = v1.DefaultMainnetTimeHost - } else { - timeHost = cfg.DcrtimeHost - } - cfg.DcrtimeHost = util.NormalizeAddress(timeHost, - v1.DefaultMainnetTimePort) - } - cfg.DcrtimeHost = "https://" + cfg.DcrtimeHost - - if len(cfg.DcrtimeCert) != 0 && !util.FileExists(cfg.DcrtimeCert) { - cfg.DcrtimeCert = cleanAndExpandPath(cfg.DcrtimeCert) - path := filepath.Join(cfg.HomeDir, cfg.DcrtimeCert) - if !util.FileExists(path) { - str := "%s: dcrtimecert " + cfg.DcrtimeCert + " and " + - path + " don't exist" - err := fmt.Errorf(str, funcName) - fmt.Fprintln(os.Stderr, err) - return nil, nil, err - } - - cfg.DcrtimeCert = path - } - - // Set random username and password when not specified - if cfg.RPCUser == "" { - name, err := util.Random(32) - if err != nil { - return nil, nil, err - } - cfg.RPCUser = base64.StdEncoding.EncodeToString(name) - log.Warnf("RPC user name not set, using random value") - } - if cfg.RPCPass == "" { - pass, err := util.Random(32) - if err != nil { - return nil, nil, err - } - cfg.RPCPass = base64.StdEncoding.EncodeToString(pass) - log.Warnf("RPC password not set, using random value") - } - - // Warn about missing config file only after all other configuration is - // done. This prevents the warning on help messages and invalid - // options. Note this should go directly before the return. - if configFileError != nil { - log.Warnf("%v", configFileError) - } - - return &cfg, remainingArgs, nil -} diff --git a/tlog/tserver/log.go b/tlog/tserver/log.go deleted file mode 100644 index 955ed83ee..000000000 --- a/tlog/tserver/log.go +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" - "os" - "path/filepath" - - "github.com/decred/slog" - "github.com/jrick/logrotate/rotator" -) - -// logWriter implements an io.Writer that outputs to both standard output and -// the write-end pipe of an initialized log rotator. -type logWriter struct{} - -func (logWriter) Write(p []byte) (n int, err error) { - os.Stdout.Write(p) - return logRotator.Write(p) -} - -// Loggers per subsystem. A single backend logger is created and all subsytem -// loggers created from it will write to the backend. When adding new -// subsystems, add the subsystem logger variable here and to the -// subsystemLoggers map. -// -// Loggers can not be used before the log rotator has been initialized with a -// log file. This must be performed early during application startup by calling -// initLogRotator. -var ( - // backendLog is the logging backend used to create all subsystem loggers. - // The backend must not be used before the log rotator has been initialized, - // or data races and/or nil pointer dereferences will occur. - backendLog = slog.NewBackend(logWriter{}) - - // logRotator is one of the logging outputs. It should be closed on - // application shutdown. - logRotator *rotator.Rotator - - log = backendLog.Logger("TSRV") -) - -// subsystemLoggers maps each subsystem identifier to its associated logger. -var subsystemLoggers = map[string]slog.Logger{ - "TSRV": log, -} - -// initLogRotator initializes the logging rotater to write logs to logFile and -// create roll files in the same directory. It must be called before the -// package-global log rotater variables are used. -func initLogRotator(logFile string) { - logDir, _ := filepath.Split(logFile) - err := os.MkdirAll(logDir, 0700) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to create log directory: %v\n", err) - os.Exit(1) - } - r, err := rotator.New(logFile, 10*1024, false, 3) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to create file rotator: %v\n", err) - os.Exit(1) - } - - logRotator = r -} - -// setLogLevel sets the logging level for provided subsystem. Invalid -// subsystems are ignored. Uninitialized subsystems are dynamically created as -// needed. -func setLogLevel(subsystemID string, logLevel string) { - // Ignore invalid subsystems. - logger, ok := subsystemLoggers[subsystemID] - if !ok { - return - } - - // Defaults to info if the log level is invalid. - level, _ := slog.LevelFromString(logLevel) - logger.SetLevel(level) -} - -// setLogLevels sets the log level for all subsystem loggers to the passed -// level. It also dynamically creates the subsystem loggers as needed, so it -// can be used to initialize the logging system. -func setLogLevels(logLevel string) { - // Configure all sub-systems with the new logging level. Dynamically - // create loggers as needed. - for subsystemID := range subsystemLoggers { - setLogLevel(subsystemID, logLevel) - } -} - -// LogClosure is a closure that can be printed with %v to be used to -// generate expensive-to-create data for a detailed log level and avoid doing -// the work if the data isn't printed. -type logClosure func() string - -func (c logClosure) String() string { - return c() -} - -func newLogClosure(c func() string) logClosure { - return logClosure(c) -} diff --git a/tlog/tserver/params.go b/tlog/tserver/params.go deleted file mode 100644 index a5788a0d5..000000000 --- a/tlog/tserver/params.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) 2013-2014 The btcsuite developers -// Copyright (c) 2015-2017 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import ( - "github.com/decred/dcrd/chaincfg" - "github.com/decred/dcrd/wire" - "github.com/decred/dcrwallet/netparams" -) - -// activeNetParams is a pointer to the parameters specific to the -// currently active decred network. -var activeNetParams = &mainNetParams - -// params is used to group parameters for various networks such as the main -// network and test networks. -type params struct { - *chaincfg.Params - WalletRPCServerPort string -} - -// mainNetParams contains parameters specific to the main network -// (wire.MainNet). NOTE: The RPC port is intentionally different than the -// reference implementation because dcrd does not handle wallet requests. The -// separate wallet process listens on the well-known port and forwards requests -// it does not handle on to dcrd. This approach allows the wallet process -// to emulate the full reference implementation RPC API. -var mainNetParams = params{ - Params: &chaincfg.MainNetParams, - WalletRPCServerPort: netparams.MainNetParams.GRPCServerPort, -} - -// testNet3Params contains parameters specific to the test network (version 0) -// (wire.TestNet). NOTE: The RPC port is intentionally different than the -// reference implementation - see the mainNetParams comment for details. - -var testNet3Params = params{ - Params: &chaincfg.TestNet3Params, - WalletRPCServerPort: netparams.TestNet3Params.GRPCServerPort, -} - -// simNetParams contains parameters specific to the simulation test network -// (wire.SimNet). -var simNetParams = params{ - Params: &chaincfg.SimNetParams, - WalletRPCServerPort: netparams.SimNetParams.GRPCServerPort, -} - -// netName returns the name used when referring to a decred network. At the -// time of writing, dcrd currently places blocks for testnet version 0 in the -// data and log directory "testnet", which does not match the Name field of the -// chaincfg parameters. This function can be used to override this directory name -// as "testnet" when the passed active network matches wire.TestNet. -// -// A proper upgrade to move the data and log directories for this network to -// "testnet" is planned for the future, at which point this function can be -// removed and the network parameter's name used instead. -func netName(chainParams *params) string { - switch chainParams.Net { - case wire.TestNet3: - return "testnet3" - default: - return chainParams.Name - } -} diff --git a/tlog/tserver/tserver.go b/tlog/tserver/tserver.go deleted file mode 100644 index 4fec06476..000000000 --- a/tlog/tserver/tserver.go +++ /dev/null @@ -1,1053 +0,0 @@ -package main - -import ( - "context" - "crypto" - "crypto/elliptic" - "crypto/x509" - "encoding/base64" - "encoding/hex" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "net/http/httputil" - "os" - "os/signal" - "runtime/debug" - "strings" - "sync" - "syscall" - "time" - - v1 "github.com/decred/politeia/tlog/api/v1" - tlogutil "github.com/decred/politeia/tlog/util" - "github.com/decred/politeia/util" - "github.com/decred/politeia/util/version" - "github.com/golang/protobuf/ptypes" - "github.com/google/trillian" - "github.com/google/trillian/client" - tcrypto "github.com/google/trillian/crypto" - "github.com/google/trillian/crypto/keys" - "github.com/google/trillian/crypto/keys/der" - "github.com/google/trillian/crypto/keyspb" - "github.com/google/trillian/crypto/sigpb" - _ "github.com/google/trillian/merkle/rfc6962" - "github.com/google/trillian/types" - "github.com/gorilla/mux" - "github.com/robfig/cron" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -type tserver struct { - sync.RWMutex - - // dirty keeps track of which tree is dirty at what height. At - // start-of-day we scan all records and look for STH that have not been - // anchored. Note that we only anchor the latest STH and do so - // opportunistically. If the application is closed and restarted it - // simply will drop a new anchor at the next interval; it will not try - // to finish a prior outstanding anchor drop. - dirty map[int64]int64 // [treeid]height - droppingAnchor bool // anchor dropping is in progress - - s Blob // Storage interface - - cron *cron.Cron // Scheduler for periodic tasks - - cfg *config - router *mux.Router - client trillian.TrillianLogClient - admin trillian.TrillianAdminClient - ctx context.Context - - signingKey *keyspb.PrivateKey // trillian signing key - publicKeyDER []byte // DER encoded public key - encryptionKey [32]byte // secretbox key for data at rest -} - -//func convertTrillianDuration(p *durpb.Duration) int64 { -// d, err := ptypes.Duration(p) -// if err != nil { -// panic(err) -// } -// return int64(d) -//} -// -//func convertTrillianTimestamp(ts *timestamp.Timestamp) int64 { -// if ts == nil { -// return 0 -// } -// return time.Unix(ts.Seconds, int64(ts.Nanos)).Unix() -//} - -func remoteAddr(r *http.Request) string { - via := r.RemoteAddr - xff := r.Header.Get(v1.Forward) - if xff != "" { - return fmt.Sprintf("%v via %v", xff, r.RemoteAddr) - } - return via -} - -// closeBody closes the request body after the provided handler is called. -func closeBody(f http.HandlerFunc) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - f(w, r) - r.Body.Close() - } -} - -func logging(f http.HandlerFunc) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - // Trace incoming request - log.Tracef("%v", newLogClosure(func() string { - trace, err := httputil.DumpRequest(r, true) - if err != nil { - trace = []byte(fmt.Sprintf("logging: "+ - "DumpRequest %v", err)) - } - return string(trace) - })) - - // Log incoming connection - log.Infof("%v %v %v %v", remoteAddr(r), r.Method, r.URL, r.Proto) - f(w, r) - } -} - -// RespondWithError returns an HTTP error status to the client. If it's a user -// error, it returns a 4xx HTTP status and the specific user error code. If it's -// an internal server error, it returns 500 and an error code which is also -// outputted to the logs so that it can be correlated later if the user -// files a complaint. -func RespondWithError(w http.ResponseWriter, r *http.Request, userHttpCode int, format string, args ...interface{}) { - // XXX this function needs to get an error in and a format + args - // instead of what it is doing now. - // So inError error, format string, args ...interface{} - // if err == nil -> internal error using format + args - // if err != nil -> if defined error -> return defined error + log.Errorf format+args - // if err != nil -> if !defined error -> return + log.Errorf format+args - if userErr, ok := args[0].(v1.UserError); ok { - if userHttpCode == 0 { - userHttpCode = http.StatusBadRequest - } - - if len(userErr.ErrorContext) == 0 { - log.Errorf("RespondWithError: %v %v %v", - remoteAddr(r), - int64(userErr.ErrorCode), - v1.ErrorStatus[userErr.ErrorCode]) - } else { - log.Errorf("RespondWithError: %v %v %v: %v", - remoteAddr(r), - int64(userErr.ErrorCode), - v1.ErrorStatus[userErr.ErrorCode], - strings.Join(userErr.ErrorContext, ", ")) - } - - util.RespondWithJSON(w, userHttpCode, - v1.ErrorReply{ - ErrorCode: int64(userErr.ErrorCode), - ErrorContext: userErr.ErrorContext, - }) - return - } - - errorCode := time.Now().Unix() - ec := fmt.Sprintf("%v %v %v %v Internal error %v: ", remoteAddr(r), - r.Method, r.URL, r.Proto, errorCode) - log.Errorf(ec+format, args...) - log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) - - util.RespondWithJSON(w, http.StatusInternalServerError, - v1.ErrorReply{ - ErrorCode: errorCode, - }) -} - -func (t *tserver) addRoute(method string, route string, handler http.HandlerFunc) { - handler = closeBody(logging(handler)) - - t.router.StrictSlash(true).HandleFunc(route, handler).Methods(method) -} - -func (t *tserver) getTree(treeId int64) (*trillian.Tree, error) { - // Verify tree exists - tree, err := t.admin.GetTree(t.ctx, &trillian.GetTreeRequest{ - TreeId: treeId, - }) - if err != nil { - return nil, err - } - if tree.TreeId != treeId { - // XXX really shouldn't happen - return nil, fmt.Errorf("invalid tree returned got %v wanted %v", - tree.TreeId, treeId) - } - return tree, nil -} - -func (t *tserver) list(w http.ResponseWriter, r *http.Request) { - // Ignore structure since it is empty - ltr, err := t.admin.ListTrees(t.ctx, &trillian.ListTreesRequest{}) - if err != nil { - RespondWithError(w, r, 0, "list: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, v1.ListReply{Trees: ltr.Tree}) -} - -// getLatestSignedLogRoot retrieves the latest signed root and verifies the -// signatures. -func (t *tserver) getLatestSignedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoot, *types.LogRootV1, error) { - // get latest signed root - resp, err := t.client.GetLatestSignedLogRoot(t.ctx, - &trillian.GetLatestSignedLogRootRequest{LogId: tree.TreeId}) - if err != nil { - return nil, nil, err - } - - // verify root - verifier, err := client.NewLogVerifierFromTree(tree) - if err != nil { - return nil, nil, err - } - lrv1, err := tcrypto.VerifySignedLogRoot(verifier.PubKey, - crypto.SHA256, resp.SignedLogRoot) - if err != nil { - return nil, nil, err - } - - return resp.SignedLogRoot, lrv1, nil -} - -// createTree creates a new trillian tree and verifies that the signatures are -// correct. It returns the tree and the signed log root which can be externally -// verified. -func (t *tserver) createTree() (*trillian.Tree, *trillian.SignedLogRoot, error) { - k, err := ptypes.MarshalAny(t.signingKey) - if err != nil { - return nil, nil, err - } - - // Create new trillian tree - tree, err := t.admin.CreateTree(t.ctx, &trillian.CreateTreeRequest{ - Tree: &trillian.Tree{ - TreeState: trillian.TreeState_ACTIVE, - TreeType: trillian.TreeType_LOG, - HashStrategy: trillian.HashStrategy_RFC6962_SHA256, - HashAlgorithm: sigpb.DigitallySigned_SHA256, - SignatureAlgorithm: sigpb.DigitallySigned_ECDSA, - //SignatureAlgorithm: sigpb.DigitallySigned_ED25519, - DisplayName: "", - Description: "", - MaxRootDuration: ptypes.DurationProto(0), - PrivateKey: k, - }, - }) - if err != nil { - return nil, nil, err - } - - // Init tree or signer goes bananas - ilr, err := t.client.InitLog(t.ctx, &trillian.InitLogRequest{ - LogId: tree.TreeId, - }) - if err != nil { - return nil, nil, err - } - - // Check trillian errors - switch code := status.Code(err); code { - case codes.Unavailable: - err = fmt.Errorf("log server unavailable: %v", err) - case codes.AlreadyExists: - err = fmt.Errorf("just-created Log (%v) is already initialised: %v", - tree.TreeId, err) - case codes.OK: - log.Debugf("Initialised Log: %v", tree.TreeId) - default: - err = fmt.Errorf("failed to InitLog (unknown error)") - } - if err != nil { - return nil, nil, err - } - - // Verify root signature - verifier, err := client.NewLogVerifierFromTree(tree) - if err != nil { - return nil, nil, err - } - _, err = tcrypto.VerifySignedLogRoot(verifier.PubKey, - crypto.SHA256, ilr.Created) - if err != nil { - return nil, nil, err - } - - return tree, ilr.Created, nil -} - -// waitForRootUpdate waits until the trillian root is updated. This code is -// clunky because we need a trillian.client context which we have to construct -// from known information. We probably should create our own client structure -// that does this with a saner API. -func (t *tserver) waitForRootUpdate(tree *trillian.Tree, root *trillian.SignedLogRoot) error { - // Wait for update - var logRoot types.LogRootV1 - err := logRoot.UnmarshalBinary(root.LogRoot) - if err != nil { - return err - } - c, err := client.NewFromTree(t.client, tree, logRoot) - if err != nil { - return err - } - _, err = c.WaitForRootUpdate(t.ctx) - if err != nil { - return err - } - return nil -} - -func (t *tserver) countErrors(qlr *trillian.QueueLeavesResponse) int { - var n int - for k := range qlr.QueuedLeaves { - c := codes.Code(qlr.QueuedLeaves[k].GetStatus().GetCode()) - if c != codes.OK { - n++ - } - } - return n -} - -// getEntry returns the record entry proof for the provided leaf. -func (t *tserver) getEntry(id int64, leaf *trillian.LogLeaf) (re v1.RecordEntryProof) { - log.Tracef("getEntry: %v %x", id, leaf.MerkleLeafHash) - - // Retrieve record entry - payload, err := t.s.Get(leaf.ExtraData) - if err != nil { - re.Error = fmt.Sprintf("Get: %v", err) - return - } - entry, err := deblob(payload) - if err != nil { - re.Error = fmt.Sprintf("deblob: %v", err) - return - } - - // Retrieve anchor - da, err := t.findLeafAnchor(id, leaf) - if err != nil { - re.Error = fmt.Sprintf("findLeafAnchor: %v", err) - return - } - if da == nil { - // Entry hasn't been anchored yet. No need to continue. - re.RecordEntry = entry - re.Leaf = leaf - return - } - var lrv1 types.LogRootV1 - err = lrv1.UnmarshalBinary(da.STH.LogRoot) - if err != nil { - re.Error = fmt.Sprintf("unmarshal LogRootV1: %v", err) - return - } - - // Retrieve inclusion proof for anchored LogRoot - gipr, err := t.client.GetInclusionProof(t.ctx, - &trillian.GetInclusionProofRequest{ - LogId: id, - LeafIndex: leaf.LeafIndex, - TreeSize: int64(lrv1.TreeSize), // tree size when anchor was dropped - }) - if err != nil { - re.Error = fmt.Sprintf("GetInclusionProof: %v", err) - return - } - - // Fill record out - re.RecordEntry = entry - re.Leaf = leaf - re.STH = &da.STH - re.Proof = gipr.Proof - re.Anchor = &da.VerifyDigest.ChainInformation - - return -} - -// getEntries returns the record entry proofs for the provided leaf hashes. -func (t *tserver) getEntries(id int64, leafHashes [][]byte) ([]v1.RecordEntryProof, error) { - log.Tracef("getEntries: %v %x", id, leafHashes) - - // Retrieve leaves - glbhr, err := t.client.GetLeavesByHash(t.ctx, - &trillian.GetLeavesByHashRequest{ - LogId: id, - LeafHash: leafHashes, - }) - if err != nil { - return nil, fmt.Errorf("GetLeavesByHashRequest: %v", err) - } - - // Retrieve record entry proofs - rep := make([]v1.RecordEntryProof, 0, len(glbhr.Leaves)) - for _, v := range glbhr.Leaves { - rep = append(rep, t.getEntry(id, v)) - } - - return rep, nil -} - -// unwindBlobs deletes the passed in blobs from the backend. This function -// panics if anything goes wrong. -func (t *tserver) unwindBlobs(blobIDs map[string]struct{}) { - var failed bool - for id := range blobIDs { - log.Debugf("Unwinding blob %s", id) - err := t.s.Del([]byte(id)) - if err != nil { - log.Critical("del blob %s: %v", id, err) - failed = true - } - } - if failed { - // We are in trouble! - panic("could not unwind blobs") - } -} - -// appendRecord stores pointers to record entries into trillian and data into a -// backend. -func (t *tserver) appendRecord(tree *trillian.Tree, root *trillian.SignedLogRoot, re []v1.RecordEntry) ([]v1.QueuedLeafProof, *trillian.SignedLogRoot, error) { - ll := make([]*trillian.LogLeaf, 0, len(re)) - blobIDs := make(map[string]struct{}, len(re)) // [blobID]struct{} - for _, v := range re { - blob, err := blobify(v) - if err != nil { - t.unwindBlobs(blobIDs) - return nil, nil, fmt.Errorf("blobify: %v", err) - } - id, err := t.s.Put(blob) - if err != nil { - t.unwindBlobs(blobIDs) - return nil, nil, fmt.Errorf("Put: %v", err) - } - blobIDs[string(id)] = struct{}{} - - h, err := hex.DecodeString(v.Hash) - if err != nil { - // Shouldn't happen, really should be a panic - t.unwindBlobs(blobIDs) - return nil, nil, fmt.Errorf("DecodeString: %v", err) - } - - ll = append(ll, &trillian.LogLeaf{ - LeafValue: h, // use hash data so that we can collide dups - ExtraData: id, - }) - } - log.Debugf("Stored record entries: %v %v", len(ll), tree.TreeId) - - // Store all records as leafs - qlr, err := t.client.QueueLeaves(t.ctx, &trillian.QueueLeavesRequest{ - LogId: tree.TreeId, - Leaves: ll, - }) - if err != nil { - t.unwindBlobs(blobIDs) - return nil, nil, fmt.Errorf("QueueLeaves: %v", err) - } - - // Count errors to see if we need to wait - n := t.countErrors(qlr) - log.Debugf("Stored/Ignored leaves: %v/%v %v", len(ll)-n, n, tree.TreeId) - - // Unwind blobs of ignored leaves - if n != 0 { - ignored := make(map[string]struct{}) - for _, v := range qlr.QueuedLeaves { - c := codes.Code(v.GetStatus().GetCode()) - if c != codes.OK { - // Blob ID is stored in Leaf.ExtraData - ignored[string(v.Leaf.ExtraData)] = struct{}{} - delete(blobIDs, string(v.Leaf.ExtraData)) - } - } - t.unwindBlobs(ignored) - } - - // Only wait if we actually updated the tree - if len(ll)-n != 0 { - // Wait for update - log.Debugf("Waiting for update: %v", tree.TreeId) - err = t.waitForRootUpdate(tree, root) - if err != nil { - t.unwindBlobs(blobIDs) - return nil, nil, fmt.Errorf("waitForRootUpdate: %v", err) - } - } - - // Get latest signed tree head - sth, lrv1, err := t.getLatestSignedLogRoot(tree) - if err != nil { - t.unwindBlobs(blobIDs) - return nil, nil, fmt.Errorf("getLatestSignedLogRoot: %v", err) - } - - // Get inclusion proofs - proofs := make([]v1.QueuedLeafProof, 0, len(qlr.QueuedLeaves)) - for _, v := range qlr.QueuedLeaves { - qllp := v1.QueuedLeafProof{ - QueuedLeaf: *v, - } - c := codes.Code(v.GetStatus().GetCode()) - if c == codes.OK { - // LeafIndex of a QueuedLogLeaf will not be set so - // get the inclusion proof by hash. - resp, err := t.client.GetInclusionProofByHash(t.ctx, - &trillian.GetInclusionProofByHashRequest{ - LogId: tree.TreeId, - LeafHash: v.Leaf.MerkleLeafHash, - TreeSize: int64(lrv1.TreeSize), - }) - if err != nil { - t.unwindBlobs(blobIDs) - return nil, nil, fmt.Errorf("GetInclusionProof: %v", err) - } - if len(resp.Proof) != 1 { - t.unwindBlobs(blobIDs) - return nil, nil, fmt.Errorf("invalid number of proofs "+ - "for leaf %v: got %v, want 1", v.Leaf.MerkleLeafHash, - len(resp.Proof)) - } - qllp.Proof = resp.Proof[0] - } - proofs = append(proofs, qllp) - } - - // Mark dirty - t.Lock() - t.dirty[tree.TreeId] = int64(lrv1.TreeSize) - t.Unlock() - - return proofs, sth, nil -} - -// publicKey returns the public key to the caller. -func (t *tserver) publicKey(w http.ResponseWriter, r *http.Request) { - log.Tracef("publicKey") - - util.RespondWithJSON(w, http.StatusOK, v1.PublicKeyReply{ - SigningKey: base64.StdEncoding.EncodeToString(t.publicKeyDER), - }) -} - -// recordNew creates a new record that consists of various entries. -func (t *tserver) recordNew(w http.ResponseWriter, r *http.Request) { - log.Tracef("recordNew") - - // Decode incoming record - var rn v1.RecordNew - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&rn); err != nil { - RespondWithError(w, r, 0, "recordNew: Unmarshal", - v1.UserError{ - ErrorCode: v1.ErrorStatusInvalidInput, - }) - return - } - - // Verify individual record entries - for k := range rn.RecordEntries { - err := tlogutil.RecordEntryVerify(rn.RecordEntries[k]) - if err != nil { - // Abort entire thing if any RecordEntry is invalid - e := fmt.Sprintf("recordNew RecordEntryVerify(%v): %v", - k, err) - RespondWithError(w, r, 0, "", - v1.UserError{ - ErrorCode: v1.ErrorStatusInvalidInput, - ErrorContext: []string{e}, - }) - return - } - } - - // Check for duplicates - h := make(map[string]struct{}, len(rn.RecordEntries)) - for _, v := range rn.RecordEntries { - _, ok := h[v.Hash] - if ok { - e := fmt.Sprintf("recordNew duplicate record entry %v", - v.Hash) - RespondWithError(w, r, 0, "", - v1.UserError{ - ErrorCode: v1.ErrorStatusInvalidInput, - ErrorContext: []string{e}, - }) - return - } - h[v.Hash] = struct{}{} - } - - // Create tree to hold record - tree, root, err := t.createTree() - if err != nil { - RespondWithError(w, r, 0, "recordNew createTree: %v", err) - return - } - log.Debugf("Created tree: %v", tree.TreeId) - - // Append record entries and data - proofs, sth, err := t.appendRecord(tree, root, rn.RecordEntries) - if err != nil { - RespondWithError(w, r, 0, "recordNew appendRecord: %v", err) - return - } - - // Return the good news - util.RespondWithJSON(w, http.StatusOK, v1.RecordNewReply{ - Tree: *tree, - InitialRoot: *root, - STH: *sth, - Proofs: proofs, - }) -} - -// recordAppend appends record entries to a specified tree. -func (t *tserver) recordAppend(w http.ResponseWriter, r *http.Request) { - log.Tracef("recordAppend") - - // Decode incoming record - var ra v1.RecordAppend - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&ra); err != nil { - RespondWithError(w, r, 0, "recordAppend: Unmarshal", - v1.UserError{ - ErrorCode: v1.ErrorStatusInvalidInput, - }) - return - } - - // Verify individual record entries - for k := range ra.RecordEntries { - err := tlogutil.RecordEntryVerify(ra.RecordEntries[k]) - if err != nil { - // Abort entire thing if any RecordEntry is invalid - e := fmt.Sprintf("recordAppend RecordEntryVerify(%v): %v", - k, err) - RespondWithError(w, r, 0, "", - v1.UserError{ - ErrorCode: v1.ErrorStatusInvalidInput, - ErrorContext: []string{e}, - }) - return - } - } - - // Retrieve tree - tree, err := t.getTree(ra.Id) - if err != nil { - e := fmt.Sprintf("invalid record id: %v", ra.Id) - RespondWithError(w, r, 0, "recordAppend: getTree", - v1.UserError{ - ErrorCode: v1.ErrorStatusInvalidInput, - ErrorContext: []string{e}, - }) - return - } - - // Retrieve STH - root, _, err := t.getLatestSignedLogRoot(tree) - if err != nil { - RespondWithError(w, r, 0, "recordAppend "+ - "getLatestSignedLogRoot: %v", - err) - return - } - // Append record entries and data - proofs, sth, err := t.appendRecord(tree, root, ra.RecordEntries) - if err != nil { - RespondWithError(w, r, 0, "recordAppend appendRecord: %v", err) - return - } - - // Return the good news - util.RespondWithJSON(w, http.StatusOK, v1.RecordAppendReply{ - STH: *sth, - Proofs: proofs, - }) -} - -// recordGet returns the entire trillian tree and corresponding data. -func (t *tserver) recordGet(w http.ResponseWriter, r *http.Request) { - // Decode incoming record - var rg v1.RecordGet - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&rg); err != nil { - RespondWithError(w, r, 0, "recordGet: Unmarshal", - v1.UserError{ - ErrorCode: v1.ErrorStatusInvalidInput, - }) - return - } - - // Retrieve tree - tree, err := t.getTree(rg.Id) - if err != nil { - e := fmt.Sprintf("invalid record id: %v", rg.Id) - RespondWithError(w, r, 0, "recordGet: getTree", - v1.UserError{ - ErrorCode: v1.ErrorStatusInvalidInput, - ErrorContext: []string{e}, - }) - return - } - - // Retrieve STH - sth, lrv1, err := t.getLatestSignedLogRoot(tree) - if err != nil { - RespondWithError(w, r, 0, "recordGet getLatestSignedLogRoot: %v", - err) - return - } - - // Get leaves - glbrr, err := t.client.GetLeavesByRange(t.ctx, - &trillian.GetLeavesByRangeRequest{ - LogId: tree.TreeId, - StartIndex: 0, - Count: int64(lrv1.TreeSize), - }) - if err != nil { - e := fmt.Sprintf("GetLeavesByRange: %v", err) - RespondWithError(w, r, 0, "recordGet: getTree", - v1.UserError{ - ErrorCode: v1.ErrorStatusInvalidInput, - ErrorContext: []string{e}, - }) - return - } - - // Retrieve record entry proofs - rep := make([]v1.RecordEntryProof, 0, len(glbrr.Leaves)) - for _, v := range glbrr.Leaves { - rep = append(rep, t.getEntry(tree.TreeId, v)) - } - - // Return the good news - util.RespondWithJSON(w, http.StatusOK, v1.RecordGetReply{ - STH: *sth, - Proofs: rep, - }) -} - -// recordEntriesGet returns batched record entries and their proofs. -func (t *tserver) recordEntriesGet(w http.ResponseWriter, r *http.Request) { - // Decode incoming record - var reg v1.RecordEntriesGet - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(®); err != nil { - RespondWithError(w, r, 0, "recordEntriesGet: Unmarshal", - v1.UserError{ - ErrorCode: v1.ErrorStatusInvalidInput, - }) - return - } - - // Coalesce requests - trees := make(map[int64][][]byte) // [treeId]merkleHashes - for _, v := range reg.Entries { - _, ok := trees[v.Id] - if !ok { - trees[v.Id] = make([][]byte, 0, len(reg.Entries)) - } - h, err := hex.DecodeString(v.MerkleHash) - if err != nil { - e := fmt.Sprintf("recordEntriesGet DecodeString(%v): %v", - v.MerkleHash, err) - RespondWithError(w, r, 0, "", - v1.UserError{ - ErrorCode: v1.ErrorStatusInvalidInput, - ErrorContext: []string{e}, - }) - return - } - trees[v.Id] = append(trees[v.Id], h) - } - - // Retrieve record entries - rep := make([]v1.RecordEntryProof, 0, len(reg.Entries)) - for id, hashes := range trees { - entries, err := t.getEntries(id, hashes) - if err != nil { - e := fmt.Sprintf("recordEntriesGet getEntries %v: %v", - id, err) - RespondWithError(w, r, 0, "", - v1.UserError{ - ErrorCode: v1.ErrorStatusInvalidInput, - ErrorContext: []string{e}, - }) - return - } - rep = append(rep, entries...) - } - - util.RespondWithJSON(w, http.StatusOK, v1.RecordEntriesGetReply{ - Proofs: rep, - }) -} - -// recordFsck run fsck on a record. -func (t *tserver) recordFsck(w http.ResponseWriter, r *http.Request) { - // Decode incoming record - var rf v1.RecordFsck - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&rf); err != nil { - RespondWithError(w, r, 0, "recordFsck: Unmarshal", - v1.UserError{ - ErrorCode: v1.ErrorStatusInvalidInput, - }) - return - } - - err := t.fsck(rf) - if err != nil { - RespondWithError(w, r, 0, "fsck: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, v1.RecordFsckReply{}) -} - -func _main() error { - // Load configuration and parse command line. This function also - // initializes logging and configures it accordingly. - loadedCfg, _, err := loadConfig() - if err != nil { - return fmt.Errorf("Could not load configuration file: %v", err) - } - defer func() { - if logRotator != nil { - logRotator.Close() - } - }() - - log.Infof("Version : %v", version.String()) - log.Infof("Network : %v", activeNetParams.Params.Name) - log.Infof("Home dir: %v", loadedCfg.HomeDir) - - // Create the data directory in case it does not exist. - err = os.MkdirAll(loadedCfg.DataDir, 0700) - if err != nil { - return err - } - - // Generate the TLS cert and key file if both don't already - // exist. - if !util.FileExists(loadedCfg.HTTPSKey) && - !util.FileExists(loadedCfg.HTTPSCert) { - log.Infof("Generating HTTPS keypair...") - - err := util.GenCertPair(elliptic.P521(), "tserver", - loadedCfg.HTTPSCert, loadedCfg.HTTPSKey) - if err != nil { - return fmt.Errorf("unable to create https keypair: %v", - err) - } - - log.Infof("HTTPS keypair created...") - } - - // Create new signing key - if !util.FileExists(loadedCfg.SigningKey) { - log.Infof("Generating signing key...") - signingKey, err := keys.NewFromSpec(&keyspb.Specification{ - //Params: &keyspb.Specification_Ed25519Params{}, - Params: &keyspb.Specification_EcdsaParams{}, - }) - if err != nil { - return err - } - b, err := der.MarshalPrivateKey(signingKey) - if err != nil { - return err - } - err = ioutil.WriteFile(loadedCfg.SigningKey, b, 0400) - if err != nil { - return err - } - - log.Infof("Signing Key created...") - } - - // Create new encryption key - if !util.FileExists(loadedCfg.EncryptionKey) { - log.Infof("Generating encryption key...") - key, err := NewKey() - if err != nil { - return err - } - err = ioutil.WriteFile(loadedCfg.EncryptionKey, key[:], 0400) - if err != nil { - return err - } - - log.Infof("EncryptionKey Key created...") - } - - // Connect to trillian - log.Infof("Trillian log server: %v", loadedCfg.TrillianHost) - g, err := grpc.Dial(loadedCfg.TrillianHost, grpc.WithInsecure()) - if err != nil { - return err - } - defer g.Close() - - // Dcrtime host - log.Infof("Anchor host: %v", loadedCfg.DcrtimeHost) - - // Setup application context. - t := &tserver{ - cfg: loadedCfg, - cron: cron.New(), - client: trillian.NewTrillianLogClient(g), - admin: trillian.NewTrillianAdminClient(g), - ctx: context.Background(), - signingKey: &keyspb.PrivateKey{}, - dirty: make(map[int64]int64), - } - - // Load certs, if there. If they aren't there assume OS is used to - // resolve cert validity. - if len(loadedCfg.DcrtimeCert) != 0 { - var certPool *x509.CertPool - if !util.FileExists(loadedCfg.DcrtimeCert) { - return fmt.Errorf("unable to find dcrtime cert %v", - loadedCfg.DcrtimeCert) - } - dcrtimeCert, err := ioutil.ReadFile(loadedCfg.DcrtimeCert) - if err != nil { - return fmt.Errorf("unable to read dcrtime cert %v: %v", - loadedCfg.DcrtimeCert, err) - } - certPool = x509.NewCertPool() - if !certPool.AppendCertsFromPEM(dcrtimeCert) { - return fmt.Errorf("unable to load cert") - } - } - - // Load signing key - t.signingKey.Der, err = ioutil.ReadFile(loadedCfg.SigningKey) - if err != nil { - return err - } - // Verify that it is DER encoded and extract public key DER - privKey, err := der.UnmarshalPrivateKey(t.signingKey.Der) - if err != nil { - return err - } - t.publicKeyDER, err = der.MarshalPublicKey(privKey.Public()) - if err != nil { - return err - } - - // Load encryption key - f, err := os.Open(loadedCfg.EncryptionKey) - if err != nil { - return err - } - n, err := f.Read(t.encryptionKey[:]) - if n != len(t.encryptionKey) { - return fmt.Errorf("invalid key length") - } - if err != nil { - return err - } - f.Close() - - // Setup storage - t.s, err = BlobFilesystemNew(&t.encryptionKey, t.cfg.DataDir) - if err != nil { - return err - } - - // Scan for unanchored records - log.Infof("Scanning for unanchored records") - err = t.scanAllRecords() - if err != nil { - return err - } - - // Launch cron. - err = t.cron.AddFunc(anchorSchedule, func() { - t.anchorRecords() - }) - if err != nil { - return err - } - t.cron.Start() - - // XXX remove, this is for test only - t.anchorRecords() - - // Setup mux - t.router = mux.NewRouter() - - // Unprivileged routes - t.addRoute(http.MethodPost, v1.RouteList, t.list) - t.addRoute(http.MethodPost, v1.RoutePublicKey, t.publicKey) - t.addRoute(http.MethodPost, v1.RouteRecordNew, t.recordNew) - t.addRoute(http.MethodPost, v1.RouteRecordGet, t.recordGet) - t.addRoute(http.MethodPost, v1.RouteRecordEntriesGet, t.recordEntriesGet) - t.addRoute(http.MethodPost, v1.RouteRecordAppend, t.recordAppend) - t.addRoute(http.MethodPost, v1.RouteRecordFsck, t.recordFsck) - - // Bind to a port and pass our router in - listenC := make(chan error) - for _, listener := range loadedCfg.Listeners { - listen := listener - go func() { - log.Infof("Listen: %v", listen) - listenC <- http.ListenAndServeTLS(listen, - loadedCfg.HTTPSCert, loadedCfg.HTTPSKey, - t.router) - }() - } - - // Tell user we are ready to go. - log.Infof("Start of day") - - // Setup OS signals - sigs := make(chan os.Signal, 1) - signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) - signal.Notify(sigs, syscall.SIGINT, syscall.SIGINT) - for { - select { - case sig := <-sigs: - log.Infof("Terminating with %v", sig) - goto done - case err := <-listenC: - log.Errorf("%v", err) - goto done - } - } -done: - log.Infof("Exiting") - - return nil -} - -func main() { - err := _main() - if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) - } -} diff --git a/tlog/util/util.go b/tlog/util/util.go deleted file mode 100644 index 2cddaee85..000000000 --- a/tlog/util/util.go +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright (c) 2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package util - -import ( - "bytes" - "crypto" - "crypto/sha256" - "encoding/base64" - "encoding/hex" - "fmt" - - "github.com/decred/dcrtime/merkle" - "github.com/decred/politeia/politeiad/api/v1/identity" - v1 "github.com/decred/politeia/tlog/api/v1" - "github.com/decred/politeia/util" - "github.com/google/trillian" - "github.com/google/trillian/client" - tcrypto "github.com/google/trillian/crypto" - "github.com/google/trillian/merkle/hashers" - "github.com/google/trillian/types" - "google.golang.org/grpc/codes" -) - -// RecordEntryNew returns an encoded tlog RecordEntry structure. -func RecordEntryNew(myId *identity.FullIdentity, dataHint, data []byte) v1.RecordEntry { - // Calculate hash - h := sha256.New() - h.Write(data) - - // Create record - re := v1.RecordEntry{ - Hash: hex.EncodeToString(h.Sum(nil)), - DataHint: base64.StdEncoding.EncodeToString(dataHint), - Data: base64.StdEncoding.EncodeToString(data), - } - - // XXX don't sign when we don't have an identity. This is not - // acceptable and only a temporary workaround until trillian properly - // supports ed25519. - if myId != nil { - re.PublicKey = hex.EncodeToString(myId.Public.Key[:]) - - // Sign - signature := myId.SignMessage([]byte(re.Hash)) - re.Signature = hex.EncodeToString(signature[:]) - } - - return re -} - -// RecordEntryVerify ensures that a tlog RecordEntry is valid. -func RecordEntryVerify(record v1.RecordEntry) error { - // Decode identity - id, err := util.IdentityFromString(record.PublicKey) - if err != nil { - return fmt.Errorf("invalid pubkey: %v", err) - } - - // Decode hash - hash, err := hex.DecodeString(record.Hash) - if err != nil { - return fmt.Errorf("invalid record hash: %v", err) - } - - // Decode signature - s, err := hex.DecodeString(record.Signature) - if err != nil { - return fmt.Errorf("invalid signature: %v", err) - } - var signature [64]byte - copy(signature[:], s) - - // Decode data - data, err := base64.StdEncoding.DecodeString(record.Data) - if err != nil { - return fmt.Errorf("invalid record data: %v", err) - } - - // Verify hash - h := sha256.New() - h.Write(data) - if !bytes.Equal(hash, h.Sum(nil)) { - return fmt.Errorf("invalid hash") - } - - // Verify signature - if !id.VerifyMessage([]byte(record.Hash), signature) { - return fmt.Errorf("invalid signature") - } - - return nil -} - -// QueuedLeafProof ensures that a queued leaf and the inclusion proof for the -// leaf are valid. -func QueuedLeafProofVerify(pk crypto.PublicKey, lrv1 *types.LogRootV1, qlp v1.QueuedLeafProof) error { - // Check queued leaf status - c := codes.Code(qlp.QueuedLeaf.GetStatus().GetCode()) - if c != codes.OK { - return fmt.Errorf("queued leaf status: %v", - qlp.QueuedLeaf.GetStatus().GetMessage()) - } - - // Verify inclusion proof. A queued log leaf does not have - // a leaf index so this must be done using the leaf hash. - lh, err := hashers.NewLogHasher(trillian.HashStrategy_RFC6962_SHA256) - if err != nil { - return err - } - verifier := client.NewLogVerifier(lh, pk, crypto.SHA256) - err = verifier.VerifyInclusionByHash(lrv1, - qlp.QueuedLeaf.Leaf.MerkleLeafHash, qlp.Proof) - if err != nil { - return fmt.Errorf("VerifyInclusionByHash: %v", err) - } - - return nil -} - -// RecordEntryProofVerify ensures that a RecordEntry and the inclusion proof -// for the RecordEntry anchor is valid. -func RecordEntryProofVerify(pk crypto.PublicKey, rep v1.RecordEntryProof) error { - if rep.Error != "" { - return fmt.Errorf("%v", rep.Error) - } - - // Verify record - err := RecordEntryVerify(*rep.RecordEntry) - if err != nil { - return fmt.Errorf("RecordEntryVerify: %v", err) - } - - if rep.Anchor == nil { - // If an achor does not exist then - // there is nothing else to verify. - return nil - } - - // Verify STH - lrv1, err := tcrypto.VerifySignedLogRoot(pk, crypto.SHA256, rep.STH) - if err != nil { - return fmt.Errorf("VerifySignedLogRoot: %v", err) - } - - // Verify inclusion proof - lh, err := hashers.NewLogHasher(trillian.HashStrategy_RFC6962_SHA256) - if err != nil { - return err - } - - verifier := client.NewLogVerifier(lh, pk, crypto.SHA256) - err = verifier.VerifyInclusionAtIndex(lrv1, - rep.Leaf.LeafValue, rep.Leaf.LeafIndex, - rep.Proof.Hashes) - if err != nil { - return fmt.Errorf("VerifyInclusionAtIndex: %v", err) - } - - // Also verify by hash - err = verifier.VerifyInclusionByHash(lrv1, - rep.Leaf.MerkleLeafHash, rep.Proof) - if err != nil { - return fmt.Errorf("VerifyInclusionByHash: %v", err) - } - - // Verify anchor merkle path - _, err = merkle.VerifyAuthPath(&rep.Anchor.MerklePath) - if err != nil { - return fmt.Errorf("VerifyAuthPath: %v", err) - } - - // Verify that the log root hash is included in the anchor - var found bool - h := util.Hash(rep.STH.LogRoot) - for _, v := range rep.Anchor.MerklePath.Hashes { - if bytes.Equal(h[:], v[:]) { - found = true - break - } - } - if !found { - return fmt.Errorf("anchor does not contain log root hash") - } - - return nil -} From f29c372683b5c88d29e992116ab7fcb88d544eee Mon Sep 17 00:00:00 2001 From: Amir Massarwa Date: Wed, 23 Sep 2020 00:01:26 +0300 Subject: [PATCH 090/449] politeiawww: Add processVoteAuthorize. --- politeiawww/cmd/piwww/commentcensor.go | 2 +- politeiawww/cmd/piwww/commentvote.go | 2 +- politeiawww/cmd/piwww/help.go | 4 +- politeiawww/cmd/piwww/piwww.go | 4 +- .../{authorizevote.go => voteauthorize.go} | 45 ++++++------ politeiawww/cmd/shared/client.go | 22 +++--- politeiawww/piwww.go | 70 ++++++++++++++++++- politeiawww/ticketvote.go | 23 ++++++ 8 files changed, 128 insertions(+), 44 deletions(-) rename politeiawww/cmd/piwww/{authorizevote.go => voteauthorize.go} (67%) diff --git a/politeiawww/cmd/piwww/commentcensor.go b/politeiawww/cmd/piwww/commentcensor.go index e3d3f380a..e0857346b 100644 --- a/politeiawww/cmd/piwww/commentcensor.go +++ b/politeiawww/cmd/piwww/commentcensor.go @@ -65,7 +65,7 @@ func (cmd *CommentCensorCmd) Execute(args []string) error { if err != nil { return err } - cc := &pi.CommentCensor{ + cc := pi.CommentCensor{ Token: token, State: state, CommentID: uint32(ciUint), diff --git a/politeiawww/cmd/piwww/commentvote.go b/politeiawww/cmd/piwww/commentvote.go index 40b16a681..996476b07 100644 --- a/politeiawww/cmd/piwww/commentvote.go +++ b/politeiawww/cmd/piwww/commentvote.go @@ -59,7 +59,7 @@ func (cmd *CommentVoteCmd) Execute(args []string) error { if err != nil { return err } - cv := &pi.CommentVote{ + cv := pi.CommentVote{ Token: token, State: pi.PropStateVetted, CommentID: uint32(ciUint), diff --git a/politeiawww/cmd/piwww/help.go b/politeiawww/cmd/piwww/help.go index 3f0475ea7..fe482ed52 100644 --- a/politeiawww/cmd/piwww/help.go +++ b/politeiawww/cmd/piwww/help.go @@ -99,8 +99,8 @@ func (cmd *HelpCmd) Execute(args []string) error { fmt.Printf("%s\n", commentVotesHelpMsg) // Vote commands - case "authorizevote": - fmt.Printf("%s\n", authorizeVoteHelpMsg) + case "voteauthorize": + fmt.Printf("%s\n", voteAuthorizeHelpMsg) case "startvote": fmt.Printf("%s\n", startVoteHelpMsg) case "startvoterunoff": diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index f19b3d9f5..8ae89d7d5 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -56,9 +56,11 @@ type piwww struct { Comments CommentsCmd `command:"comments" description:"(public) get the comments for a proposal"` CommentVotes CommentVotesCmd `command:"commentvotes" description:"(user) get comment upvotes/downvotes for a proposal from the provided user"` + // Vote commands + VoteAuthorize VoteAuthorizeCmd `command:"voteauthorize" description:"(user) authorize a proposal vote (must be proposal author)"` + // Commands ActiveVotes ActiveVotesCmd `command:"activevotes" description:"(public) get the proposals that are being voted on"` - AuthorizeVote AuthorizeVoteCmd `command:"authorizevote" description:"(user) authorize a proposal vote (must be proposal author)"` BatchProposals shared.BatchProposalsCmd `command:"batchproposals" description:"(user) retrieve a set of proposals"` BatchVoteSummary BatchVoteSummaryCmd `command:"batchvotesummary" description:"(user) retrieve the vote summary for a set of proposals"` ChangePassword shared.ChangePasswordCmd `command:"changepassword" description:"(user) change the password for the logged in user"` diff --git a/politeiawww/cmd/piwww/authorizevote.go b/politeiawww/cmd/piwww/voteauthorize.go similarity index 67% rename from politeiawww/cmd/piwww/authorizevote.go rename to politeiawww/cmd/piwww/voteauthorize.go index 9cbacb5ef..18577a597 100644 --- a/politeiawww/cmd/piwww/authorizevote.go +++ b/politeiawww/cmd/piwww/voteauthorize.go @@ -8,15 +8,14 @@ import ( "encoding/hex" "fmt" - "github.com/decred/politeia/decredplugin" - "github.com/decred/politeia/politeiawww/api/www/v1" + pi "github.com/decred/politeia/politeiawww/api/pi/v1" "github.com/decred/politeia/politeiawww/cmd/shared" "github.com/decred/politeia/util" ) -// AuthorizeVoteCmd authorizes a proposal vote. The AuthorizeVoteCmd must be +// VoteAuthorizeCmd authorizes a proposal vote. The VoteAuthorizeCmd must be // sent by the proposal author to be valid. -type AuthorizeVoteCmd struct { +type VoteAuthorizeCmd struct { Args struct { Token string `positional-arg-name:"token" required:"true"` // Censorship token Action string `positional-arg-name:"action"` // Authorize or revoke action @@ -24,7 +23,7 @@ type AuthorizeVoteCmd struct { } // Execute executes the authorize vote command. -func (cmd *AuthorizeVoteCmd) Execute(args []string) error { +func (cmd *VoteAuthorizeCmd) Execute(args []string) error { token := cmd.Args.Token // Check for user identity @@ -33,13 +32,15 @@ func (cmd *AuthorizeVoteCmd) Execute(args []string) error { } // Validate action + var action pi.VoteAuthActionT switch cmd.Args.Action { - case decredplugin.AuthVoteActionAuthorize, - decredplugin.AuthVoteActionRevoke: - // This is correct; continue + case "authorize": + action = pi.VoteAuthActionAuthorize + case "revoke": + action = pi.VoteAuthActionRevoke case "": // Default to authorize - cmd.Args.Action = decredplugin.AuthVoteActionAuthorize + action = pi.VoteAuthActionAuthorize default: return fmt.Errorf("Invalid action. Valid actions are:\n " + "authorize (default) authorize a vote\n " + @@ -61,21 +62,21 @@ func (cmd *AuthorizeVoteCmd) Execute(args []string) error { // Setup authorize vote request sig := cfg.Identity.SignMessage([]byte(token + pdr.Proposal.Version + cmd.Args.Action)) - av := &v1.AuthorizeVote{ - Action: cmd.Args.Action, + va := pi.VoteAuthorize{ + Action: action, Token: token, PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), Signature: hex.EncodeToString(sig[:]), } // Print request details - err = shared.PrintJSON(av) + err = shared.PrintJSON(va) if err != nil { return err } // Send request - avr, err := client.AuthorizeVote(av) + varep, err := client.VoteAuthorize(va) if err != nil { return err } @@ -85,21 +86,21 @@ func (cmd *AuthorizeVoteCmd) Execute(args []string) error { if err != nil { return err } - s, err := util.ConvertSignature(avr.Receipt) + s, err := util.ConvertSignature(varep.Receipt) if err != nil { return err } - if !serverID.VerifyMessage([]byte(av.Signature), s) { + if !serverID.VerifyMessage([]byte(va.Signature), s) { return fmt.Errorf("could not verify authorize vote receipt") } // Print response details - return shared.PrintJSON(avr) + return shared.PrintJSON(vr) } -// authorizeVoteHelpMsg is the output of the help command when 'authorizevote' +// voteAuthorizeHelpMsg is the output of the help command when 'voteauthorize' // is specified. -const authorizeVoteHelpMsg = `authorizevote "token" "action" +const voteAuthorizeHelpMsg = `voteauthorize "token" "action" Authorize or revoke proposal vote. Only the proposal author (owner of censorship token) can authorize or revoke vote. @@ -108,10 +109,4 @@ Arguments: 1. token (string, required) Proposal censorship token 2. action (string, optional) Valid actions are 'authorize' or 'revoke' (defaults to 'authorize') - -Result: -{ - "action": (string) Action that was executed - "receipt": (string) Server signature of client signature - (signed token+version+action) -}` +` diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index 9e4ea44d3..09c54cf19 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -1258,7 +1258,7 @@ func (c *Client) CommentVotes(cv pi.CommentVotes) (*pi.CommentVotesReply, error) // CommentVote casts a like comment action (upvote/downvote) for the logged in // user. -func (c *Client) CommentVote(cv *pi.CommentVote) (*pi.CommentVoteReply, error) { +func (c *Client) CommentVote(cv pi.CommentVote) (*pi.CommentVoteReply, error) { responseBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, pi.RouteCommentVote, cv) if err != nil { @@ -1282,7 +1282,7 @@ func (c *Client) CommentVote(cv *pi.CommentVote) (*pi.CommentVoteReply, error) { } // CommentCensor censors the specified proposal comment. -func (c *Client) CommentCensor(cc *pi.CommentCensor) (*pi.CommentCensorReply, error) { +func (c *Client) CommentCensor(cc pi.CommentCensor) (*pi.CommentCensorReply, error) { responseBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, pi.RouteCommentCensor, cc) if err != nil { @@ -1581,29 +1581,29 @@ func (c *Client) EditUser(eu *www.EditUser) (*www.EditUserReply, error) { return &eur, nil } -// AuthorizeVote authorizes the voting period for the specified proposal using +// VoteAuthorize authorizes the voting period for the specified proposal using // the logged in user. -func (c *Client) AuthorizeVote(av *www.AuthorizeVote) (*www.AuthorizeVoteReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, www.PoliteiaWWWAPIRoute, - www.RouteAuthorizeVote, av) +func (c *Client) VoteAuthorize(va pi.VoteAuthorize) (*pi.VoteAuthorizeReply, error) { + responseBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, + pi.RouteVoteAuthorize, va) if err != nil { return nil, err } - var avr www.AuthorizeVoteReply - err = json.Unmarshal(responseBody, &avr) + var vr pi.VoteAuthorizeReply + err = json.Unmarshal(responseBody, &vr) if err != nil { - return nil, fmt.Errorf("unmarshal AuthorizeVoteReply: %v", err) + return nil, fmt.Errorf("unmarshal VoteAuthorizeReply: %v", err) } if c.cfg.Verbose { - err := prettyPrintJSON(avr) + err := prettyPrintJSON(vr) if err != nil { return nil, err } } - return &avr, nil + return &vr, nil } // VoteStatus retrieves the vote status for the specified proposal. diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 9c229332c..b0aacce7f 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -19,6 +19,7 @@ import ( "github.com/decred/politeia/plugins/comments" piplugin "github.com/decred/politeia/plugins/pi" + "github.com/decred/politeia/plugins/ticketvote" pd "github.com/decred/politeia/politeiad/api/v1" pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" @@ -1645,7 +1646,9 @@ func (p *politeiawww) handleComments(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&c); err != nil { respondWithPiError(w, r, "handleComments: unmarshal", - pi.UserErrorReply{}) + pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusInvalidInput, + }) return } @@ -1689,7 +1692,9 @@ func (p *politeiawww) handleCommentVotes(w http.ResponseWriter, r *http.Request) decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&cvs); err != nil { respondWithPiError(w, r, "handleCommentVotes: unmarshal", - pi.UserErrorReply{}) + pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusInvalidInput, + }) return } @@ -1744,7 +1749,9 @@ func (p *politeiawww) handleCommentCensor(w http.ResponseWriter, r *http.Request decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&cc); err != nil { respondWithPiError(w, r, "handleCommentCensor: unmarshal", - pi.UserErrorReply{}) + pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusInvalidInput, + }) return } @@ -1764,6 +1771,60 @@ func (p *politeiawww) handleCommentCensor(w http.ResponseWriter, r *http.Request util.RespondWithJSON(w, http.StatusOK, ccr) } +func convertVoteAuthActionFromPi(a pi.VoteAuthActionT) ticketvote.AuthActionT { + switch a { + case pi.VoteAuthActionAuthorize: + return ticketvote.ActionAuthorize + case pi.VoteAuthActionRevoke: + return ticketvote.ActionRevoke + default: + return ticketvote.ActionAuthorize + } +} + +func (p *politeiawww) processVoteAuthorize(va pi.VoteAuthorize) (*pi.VoteAuthorizeReply, error) { + log.Tracef("processVoteAuthorize: %v", va.Token) + + // Call ticketvote plugin to authorize vote + reply, err := p.authorizeVote(ticketvote.Authorize{ + Token: va.Token, + Version: va.Version, + Action: convertVoteAuthActionFromPi(va.Action), + PublicKey: va.PublicKey, + Signature: va.Signature, + }) + if err != nil { + return nil, err + } + + return &pi.VoteAuthorizeReply{ + Timestamp: reply.Timestamp, + Receipt: reply.Receipt, + }, nil +} + +func (p *politeiawww) handleVoteAuthorize(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleVoteAuthorize") + + var va pi.VoteAuthorize + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&va); err != nil { + respondWithPiError(w, r, "handleVoteAuthorize: unmarshal", + pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusInvalidInput, + }) + return + } + + vr, err := p.processVoteAuthorize(va) + if err != nil { + respondWithPiError(w, r, + "handleVoteAuthorize: processVoteAuthorize: %v", err) + } + + util.RespondWithJSON(w, http.StatusOK, vr) +} + func (p *politeiawww) setPiRoutes() { // Public routes p.addRoute(http.MethodGet, pi.APIRoute, @@ -1773,6 +1834,9 @@ func (p *politeiawww) setPiRoutes() { p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteComments, p.handleComments, permissionPublic) + p.addRoute(http.MethodPost, pi.APIRoute, + pi.RouteVoteAuthorize, p.handleVoteAuthorize, permissionPublic) + // Logged in routes p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteProposalNew, p.handleProposalNew, diff --git a/politeiawww/ticketvote.go b/politeiawww/ticketvote.go index 21a7ec4db..b64098783 100644 --- a/politeiawww/ticketvote.go +++ b/politeiawww/ticketvote.go @@ -9,6 +9,26 @@ import ( www "github.com/decred/politeia/politeiawww/api/www/v1" ) +func (p *politeiawww) authorizeVote(a ticketvote.Authorize) (*ticketvote.AuthorizeReply, error) { + // Prop plugin command + payload, err := ticketvote.EncodeAuthorize(a) + if err != nil { + return nil, err + } + + r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdAuthorize, "", + string(payload)) + if err != nil { + return nil, err + } + va, err := ticketvote.DecodeAuthorizeReply([]byte(r)) + if err != nil { + return nil, err + } + + return va, nil +} + // voteDetails calls the ticketvote plugin command to get vote details. func (p *politeiawww) voteDetails(token string) (*ticketvote.DetailsReply, error) { // Prep vote details payload @@ -22,6 +42,9 @@ func (p *politeiawww) voteDetails(token string) (*ticketvote.DetailsReply, error r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdDetails, "", string(payload)) + if err != nil { + return nil, err + } vd, err := ticketvote.DecodeDetailsReply([]byte(r)) if err != nil { return nil, err From e6eb38338b40e9ad467d28052c41ae3705f489f5 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 22 Sep 2020 15:16:41 -0500 Subject: [PATCH 091/449] cleanup deprecated api routes and deadcode --- examples/bugbounty/README.md | 35 - examples/bugbounty/admin_censor.sh | 3 - examples/bugbounty/admin_inventory.sh | 3 - examples/bugbounty/admin_publish.sh | 3 - examples/bugbounty/badbug1.txt | 3 - examples/bugbounty/get_unvetted.sh | 3 - examples/bugbounty/notabug1.txt | 3 - examples/bugbounty/report_bug.sh | 3 - examples/bugbounty/report_silly_bug.sh | 3 - examples/bugbounty/settings.sh | 6 - politeiawww/api/pi/v1/v1.go | 2 + politeiawww/api/www/v1/v1.go | 4 +- .../politeiawww_dbutil/politeiawww_dbutil.go | 13 +- politeiawww/convert.go | 213 --- politeiawww/piwww.go | 3 +- politeiawww/politeiawww.go | 693 ++------ politeiawww/proposals.go | 1421 ++++++++--------- politeiawww/proposals_test.go | 2 +- politeiawww/testing.go | 75 +- politeiawww/user.go | 41 - politeiawww/userwww.go | 68 + politeiawww/www.go | 2 - wsdcrdata/wsdcrdata.go | 2 + 23 files changed, 854 insertions(+), 1750 deletions(-) delete mode 100644 examples/bugbounty/README.md delete mode 100644 examples/bugbounty/admin_censor.sh delete mode 100644 examples/bugbounty/admin_inventory.sh delete mode 100644 examples/bugbounty/admin_publish.sh delete mode 100644 examples/bugbounty/badbug1.txt delete mode 100644 examples/bugbounty/get_unvetted.sh delete mode 100644 examples/bugbounty/notabug1.txt delete mode 100644 examples/bugbounty/report_bug.sh delete mode 100644 examples/bugbounty/report_silly_bug.sh delete mode 100644 examples/bugbounty/settings.sh diff --git a/examples/bugbounty/README.md b/examples/bugbounty/README.md deleted file mode 100644 index cad623efc..000000000 --- a/examples/bugbounty/README.md +++ /dev/null @@ -1,35 +0,0 @@ -Launch politead -``` -dep ensure -go install -v ./... && LOGFLAGS=shortfile politeiad --testnet --rpcuser=user --rpcpass=pass --gittrace -``` - -Install politeia -``` -cd politeiad/cmd/politeia -go install -``` - -Run example scripts -``` -cd examples/bugbounty -bash -x report_bug.sh -``` - -Example output: -``` -+ . settings.sh -++ RPCHOST=127.0.0.1 -++ RPCUSER=user -++ RPCPASS=pass -++ EFLAGS='-v -testnet' -++ USERFLAGS='-v -testnet -rpchost 127.0.0.1' -++ ADMINFLAGS='-v -testnet -rpchost 127.0.0.1 -rpcuser user -rpcpass pass' -+ politeia -v -testnet -rpchost 127.0.0.1 new '{"name":"Marco", "description":"Bad bug #1"}' badbug1.txt -00: 512cd8bc7980a6186fd36e7a095310f33b8cda1a696185bf77c6de97a7f2cfcb badbug1.txt text/plain; charset=utf-8 -Record submitted - Censorship record: - Merkle : 512cd8bc7980a6186fd36e7a095310f33b8cda1a696185bf77c6de97a7f2cfcb - Token : 141aed9b800e49bb8db9b30d32994a1b56154bc3c64842e177ba80e0e1715883 - Signature: 643e142d24004883a824bb42c083622bfb0069de9659ae0dd3ee296bc20cc1a9d07c7dd6e97b6fb01d28240aaa34405df42eec15febcdda57b98eb109f5bf30d -``` diff --git a/examples/bugbounty/admin_censor.sh b/examples/bugbounty/admin_censor.sh deleted file mode 100644 index 7a8f7b8c5..000000000 --- a/examples/bugbounty/admin_censor.sh +++ /dev/null @@ -1,3 +0,0 @@ -. settings.sh - -politeia ${ADMINFLAGS} setunvettedstatus censor $1 diff --git a/examples/bugbounty/admin_inventory.sh b/examples/bugbounty/admin_inventory.sh deleted file mode 100644 index 134b77885..000000000 --- a/examples/bugbounty/admin_inventory.sh +++ /dev/null @@ -1,3 +0,0 @@ -. settings.sh - -politeia ${ADMINFLAGS} inventory 1 1 diff --git a/examples/bugbounty/admin_publish.sh b/examples/bugbounty/admin_publish.sh deleted file mode 100644 index 3e7061dcb..000000000 --- a/examples/bugbounty/admin_publish.sh +++ /dev/null @@ -1,3 +0,0 @@ -. settings.sh - -politeia ${ADMINFLAGS} setunvettedstatus publish $1 diff --git a/examples/bugbounty/badbug1.txt b/examples/bugbounty/badbug1.txt deleted file mode 100644 index 38caa24be..000000000 --- a/examples/bugbounty/badbug1.txt +++ /dev/null @@ -1,3 +0,0 @@ -This is an example bug report. - -macOS High Sierra allows root logins without a password. diff --git a/examples/bugbounty/get_unvetted.sh b/examples/bugbounty/get_unvetted.sh deleted file mode 100644 index c2122ed21..000000000 --- a/examples/bugbounty/get_unvetted.sh +++ /dev/null @@ -1,3 +0,0 @@ -. settings.sh - -politeia ${USERFLAGS} getunvetted $1 diff --git a/examples/bugbounty/notabug1.txt b/examples/bugbounty/notabug1.txt deleted file mode 100644 index 056beecad..000000000 --- a/examples/bugbounty/notabug1.txt +++ /dev/null @@ -1,3 +0,0 @@ -This is a pretend bug report from Tr0llZ0rz - -For some reason I am not getting paid for bad bug reports, HALP! diff --git a/examples/bugbounty/report_bug.sh b/examples/bugbounty/report_bug.sh deleted file mode 100644 index b36f8912f..000000000 --- a/examples/bugbounty/report_bug.sh +++ /dev/null @@ -1,3 +0,0 @@ -. settings.sh - -politeia ${USERFLAGS} new '{"name":"Marco", "description":"Bad bug #1"}' badbug1.txt diff --git a/examples/bugbounty/report_silly_bug.sh b/examples/bugbounty/report_silly_bug.sh deleted file mode 100644 index 73d0f49bd..000000000 --- a/examples/bugbounty/report_silly_bug.sh +++ /dev/null @@ -1,3 +0,0 @@ -. settings.sh - -politeia ${USERFLAGS} new '{"name":"Tr0llZ0rz", "description":"Silly bug #1"}' notabug1.txt diff --git a/examples/bugbounty/settings.sh b/examples/bugbounty/settings.sh deleted file mode 100644 index 31b3a5029..000000000 --- a/examples/bugbounty/settings.sh +++ /dev/null @@ -1,6 +0,0 @@ -RPCHOST="127.0.0.1" -RPCUSER="user" -RPCPASS="pass" -EFLAGS="-v -testnet" -USERFLAGS="${EFLAGS} -rpchost ${RPCHOST}" -ADMINFLAGS="${EFLAGS} -rpchost ${RPCHOST} -rpcuser ${RPCUSER} -rpcpass ${RPCPASS}" diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 27228d366..39f5f1485 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -20,6 +20,8 @@ const ( APIVersion = 1 // TODO the plugin policies should be returned in a route + // TODO the proposals route should allow filtering by user ID + // TODO max page sizes should be added to RouteProposals // Proposal routes RouteProposalNew = "/proposal/new" diff --git a/politeiawww/api/www/v1/v1.go b/politeiawww/api/www/v1/v1.go index 64b75962f..275d97437 100644 --- a/politeiawww/api/www/v1/v1.go +++ b/politeiawww/api/www/v1/v1.go @@ -59,11 +59,11 @@ const ( // The following routes WILL BE DEPRECATED in the near future and // should not be used. The pi v1 API should be used instead. RouteTokenInventory = "/proposals/tokeninventory" - RouteBatchProposals = "/proposals/batch" - RouteBatchVoteSummary = "/proposals/batchvotesummary" RouteProposalDetails = "/proposals/{token:[A-Fa-f0-9]{7,64}}" + RouteBatchProposals = "/proposals/batch" RouteVoteResults = "/proposals/{token:[A-Fa-f0-9]{7,64}}/votes" RouteCastVotes = "/proposals/castvotes" + RouteBatchVoteSummary = "/proposals/batchvotesummary" // The following route HAVE BEEN DEPRECATED. The pi v1 API should // be used instead. diff --git a/politeiawww/cmd/politeiawww_dbutil/politeiawww_dbutil.go b/politeiawww/cmd/politeiawww_dbutil/politeiawww_dbutil.go index 243a59748..53ca4a355 100644 --- a/politeiawww/cmd/politeiawww_dbutil/politeiawww_dbutil.go +++ b/politeiawww/cmd/politeiawww_dbutil/politeiawww_dbutil.go @@ -51,9 +51,8 @@ const ( proposalMDFilename = "00.metadata.txt" // Journal actions - journalActionAdd = "add" // Add entry - journalActionDel = "del" // Delete entry - journalActionAddLike = "addlike" // Add comment like + journalActionAdd = "add" // Add entry + journalActionDel = "del" // Delete entry ) var ( @@ -451,14 +450,6 @@ func replayCommentsJournal(path string, pubkeys map[string]struct{}) error { } pubkeys[cc.PublicKey] = struct{}{} - case journalActionAddLike: - var lc decredplugin.LikeComment - err = d.Decode(&lc) - if err != nil { - return fmt.Errorf("journal addlike: %v", err) - } - pubkeys[lc.PublicKey] = struct{}{} - default: return fmt.Errorf("invalid action: %v", action.Action) diff --git a/politeiawww/convert.go b/politeiawww/convert.go index 20ba6abb2..3df6114d6 100644 --- a/politeiawww/convert.go +++ b/politeiawww/convert.go @@ -19,140 +19,9 @@ import ( cms "github.com/decred/politeia/politeiawww/api/cms/v1" pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" - www2 "github.com/decred/politeia/politeiawww/api/www/v2" "github.com/decred/politeia/politeiawww/cmsdatabase" ) -// TODO cleanup all unused convert functions - -func convertCastVoteReplyFromDecredPlugin(cvr decredplugin.CastVoteReply) www.CastVoteReply { - return www.CastVoteReply{ - ClientSignature: cvr.ClientSignature, - Signature: cvr.Signature, - Error: cvr.Error, - ErrorStatus: cvr.ErrorStatus, - } -} - -func convertCastVoteFromWWW(b www.CastVote) decredplugin.CastVote { - return decredplugin.CastVote{ - Token: b.Token, - Ticket: b.Ticket, - VoteBit: b.VoteBit, - Signature: b.Signature, - } -} - -func convertBallotFromWWW(b www.Ballot) decredplugin.Ballot { - br := decredplugin.Ballot{ - Votes: make([]decredplugin.CastVote, 0, len(b.Votes)), - } - for _, v := range b.Votes { - br.Votes = append(br.Votes, convertCastVoteFromWWW(v)) - } - return br -} - -func convertBallotReplyFromDecredPlugin(b decredplugin.BallotReply) www.BallotReply { - br := www.BallotReply{ - Receipts: make([]www.CastVoteReply, 0, len(b.Receipts)), - } - for _, v := range b.Receipts { - br.Receipts = append(br.Receipts, - convertCastVoteReplyFromDecredPlugin(v)) - } - return br -} - -func convertAuthorizeVoteToDecred(av www.AuthorizeVote) decredplugin.AuthorizeVote { - return decredplugin.AuthorizeVote{ - Action: av.Action, - Token: av.Token, - PublicKey: av.PublicKey, - Signature: av.Signature, - } -} - -func convertAuthorizeVoteV2ToDecred(av www2.AuthorizeVote) decredplugin.AuthorizeVote { - return decredplugin.AuthorizeVote{ - Action: av.Action, - Token: av.Token, - PublicKey: av.PublicKey, - Signature: av.Signature, - } -} - -func convertAuthorizeVotesV2ToDecred(av []www2.AuthorizeVote) []decredplugin.AuthorizeVote { - dav := make([]decredplugin.AuthorizeVote, 0, len(av)) - for _, v := range av { - dav = append(dav, convertAuthorizeVoteV2ToDecred(v)) - } - return dav -} - -func convertVoteOptionFromWWW(vo www.VoteOption) decredplugin.VoteOption { - return decredplugin.VoteOption{ - Id: vo.Id, - Description: vo.Description, - Bits: vo.Bits, - } -} - -func convertVoteOptionV2ToDecred(vo www2.VoteOption) decredplugin.VoteOption { - return decredplugin.VoteOption{ - Id: vo.Id, - Description: vo.Description, - Bits: vo.Bits, - } -} - -func convertVoteOptionsV2ToDecred(vo []www2.VoteOption) []decredplugin.VoteOption { - dvo := make([]decredplugin.VoteOption, 0, len(vo)) - for _, v := range vo { - dvo = append(dvo, convertVoteOptionV2ToDecred(v)) - } - return dvo -} - -func convertVoteTypeV2ToDecred(v www2.VoteT) decredplugin.VoteT { - switch v { - case www2.VoteTypeStandard: - return decredplugin.VoteTypeStandard - case www2.VoteTypeRunoff: - return decredplugin.VoteTypeRunoff - } - return decredplugin.VoteTypeInvalid -} - -func convertVoteV2ToDecred(v www2.Vote) decredplugin.VoteV2 { - return decredplugin.VoteV2{ - Token: v.Token, - ProposalVersion: v.ProposalVersion, - Type: convertVoteTypeV2ToDecred(v.Type), - Mask: v.Mask, - Duration: v.Duration, - QuorumPercentage: v.QuorumPercentage, - PassPercentage: v.PassPercentage, - Options: convertVoteOptionsV2ToDecred(v.Options), - } -} - -func convertStartVoteV2ToDecred(sv www2.StartVote) decredplugin.StartVoteV2 { - return decredplugin.StartVoteV2{ - PublicKey: sv.PublicKey, - Vote: convertVoteV2ToDecred(sv.Vote), - Signature: sv.Signature, - } -} - -func convertStartVotesV2ToDecred(sv []www2.StartVote) []decredplugin.StartVoteV2 { - dsv := make([]decredplugin.StartVoteV2, 0, len(sv)) - for _, v := range sv { - dsv = append(dsv, convertStartVoteV2ToDecred(v)) - } - return dsv -} - func convertPropFileFromWWW(f www.File) pd.File { return pd.File{ Name: f.Name, @@ -220,88 +89,6 @@ func convertCommentFromDecred(c decredplugin.Comment) www.Comment { } } -func convertVoteOptionFromDecred(vo decredplugin.VoteOption) www.VoteOption { - return www.VoteOption{ - Id: vo.Id, - Description: vo.Description, - Bits: vo.Bits, - } -} - -func convertVoteOptionsFromDecred(options []decredplugin.VoteOption) []www.VoteOption { - opts := make([]www.VoteOption, 0, len(options)) - for _, v := range options { - opts = append(opts, convertVoteOptionFromDecred(v)) - } - return opts -} - -func convertVoteOptionV2FromDecred(vo decredplugin.VoteOption) www2.VoteOption { - return www2.VoteOption{ - Id: vo.Id, - Description: vo.Description, - Bits: vo.Bits, - } -} - -func convertVoteOptionsV2FromDecred(options []decredplugin.VoteOption) []www2.VoteOption { - opts := make([]www2.VoteOption, 0, len(options)) - for _, v := range options { - opts = append(opts, convertVoteOptionV2FromDecred(v)) - } - return opts -} - -func convertVoteTypeFromDecred(v decredplugin.VoteT) www2.VoteT { - switch v { - case decredplugin.VoteTypeStandard: - return www2.VoteTypeStandard - case decredplugin.VoteTypeRunoff: - return www2.VoteTypeRunoff - } - return www2.VoteTypeInvalid -} - -func convertVoteOptionsV2ToV1(optsV2 []www2.VoteOption) []www.VoteOption { - optsV1 := make([]www.VoteOption, 0, len(optsV2)) - for _, v := range optsV2 { - optsV1 = append(optsV1, www.VoteOption{ - Id: v.Id, - Description: v.Description, - Bits: v.Bits, - }) - } - return optsV1 -} - -func convertStartVoteReplyV2FromDecred(svr decredplugin.StartVoteReply) (*www2.StartVoteReply, error) { - startHeight, err := strconv.ParseUint(svr.StartBlockHeight, 10, 32) - if err != nil { - return nil, fmt.Errorf("parse start height '%v': %v", - svr.StartBlockHeight, err) - } - endHeight, err := strconv.ParseUint(svr.EndHeight, 10, 32) - if err != nil { - return nil, fmt.Errorf("parse end height '%v': %v", - svr.EndHeight, err) - } - return &www2.StartVoteReply{ - StartBlockHeight: uint32(startHeight), - StartBlockHash: svr.StartBlockHash, - EndBlockHeight: uint32(endHeight), - EligibleTickets: svr.EligibleTickets, - }, nil -} - -func convertCastVoteFromDecred(cv decredplugin.CastVote) www.CastVote { - return www.CastVote{ - Token: cv.Token, - Ticket: cv.Ticket, - VoteBit: cv.VoteBit, - Signature: cv.Signature, - } -} - func convertInvoiceCensorFromWWW(f www.CensorshipRecord) pd.CensorshipRecord { return pd.CensorshipRecord{ Token: f.Token, diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index b0aacce7f..50a2de7b2 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -90,8 +90,7 @@ func tokenIsFullLength(token string) bool { return true } -// proposalNameIsValid returns whether the provided name is a valid proposal -// name. +// proposalNameIsValid returns whether the provided proposal name is a valid. func proposalNameIsValid(name string) bool { return validProposalName.MatchString(name) } diff --git a/politeiawww/politeiawww.go b/politeiawww/politeiawww.go index 46448b9bc..96d1798de 100644 --- a/politeiawww/politeiawww.go +++ b/politeiawww/politeiawww.go @@ -11,14 +11,12 @@ import ( "net/http" "net/http/httputil" "sync" - "text/template" "time" "github.com/davecgh/go-spew/spew" "github.com/decred/dcrd/chaincfg" "github.com/decred/politeia/politeiad/api/v1/mime" www "github.com/decred/politeia/politeiawww/api/www/v1" - www2 "github.com/decred/politeia/politeiawww/api/www/v2" "github.com/decred/politeia/politeiawww/cmsdatabase" "github.com/decred/politeia/politeiawww/user" utilwww "github.com/decred/politeia/politeiawww/util" @@ -59,64 +57,60 @@ func (w *wsContext) isAuthenticated() bool { return w.uuid != "" } -// politeiawww application context. +// politeiawww represents the politeiawww server. type politeiawww struct { - cfg *config - router *mux.Router - sessions sessions.Store - plugins []plugin - - ws map[string]map[string]*wsContext // [uuid][]*context - wsMtx sync.RWMutex - - // Politeiad client - client *http.Client - - // SMTP client - smtp *smtp - - templates map[string]*template.Template - tmplMtx sync.RWMutex - - // XXX This needs to be abstracted away - sync.RWMutex // XXX This needs to be the first entry in struct - - db user.Database // User database XXX GOT TO GO + sync.RWMutex + cfg *config params *chaincfg.Params + router *mux.Router + sessions sessions.Store + client *http.Client + smtp *smtp + db user.Database eventManager *eventManager + plugins []plugin - // These properties are only used for testing. - test bool - - // Following entries require locks - userPaywallPool map[uuid.UUID]paywallPoolMember // [userid][paywallPoolMember] + // Client websocket connections + ws map[string]map[string]*wsContext // [uuid][]*context + wsMtx sync.RWMutex - // XXX userEmails is a temporary measure until the user by email - // lookups are completely removed from politeiawww. + // userEmails contains a mapping of all user emails to user ID. + // This is required for now because the email is stored as part of + // the encrypted user blob in the user database, but we also allow + // the user to sign in using their email address, requiring a user + // lookup by email. This is a temporary measure and should be + // removed once all user by email lookups have been taken out. userEmails map[string]uuid.UUID // [email]userID - // Following entries are use only during cmswww mode - cmsDB cmsdatabase.Database - cron *cron.Cron + // These fields are only used during piwww mode + userPaywallPool map[uuid.UUID]paywallPoolMember // [userid][paywallPoolMember] - // wsDcrdata is a dcrdata websocket client + // These fields are use only during cmswww mode + cmsDB cmsdatabase.Database + cron *cron.Cron wsDcrdata *wsdcrdata.Client + + // The following fields are only used during testing. + test bool } -// XXX rig this up -func (p *politeiawww) addTemplate(templateName, templateContent string) { - p.tmplMtx.Lock() - defer p.tmplMtx.Unlock() +// handleNotFound is a generic handler for an invalid route. +func (p *politeiawww) handleNotFound(w http.ResponseWriter, r *http.Request) { + // Log incoming connection + log.Debugf("Invalid route: %v %v %v %v", remoteAddr(r), r.Method, r.URL, + r.Proto) - p.templates[templateName] = template.Must( - template.New(templateName).Parse(templateContent)) -} + // Trace incoming request + log.Tracef("%v", newLogClosure(func() string { + trace, err := httputil.DumpRequest(r, true) + if err != nil { + trace = []byte(fmt.Sprintf("logging: "+ + "DumpRequest %v", err)) + } + return string(trace) + })) -// XXX rig this up -func (p *politeiawww) getTemplate(templateName string) *template.Template { - p.tmplMtx.RLock() - defer p.tmplMtx.RUnlock() - return p.templates[templateName] + util.RespondWithJSON(w, http.StatusNotFound, www.ErrorReply{}) } // version is an HTTP GET to determine the lowest API route version that this @@ -157,48 +151,59 @@ func (p *politeiawww) handleVersion(w http.ResponseWriter, r *http.Request) { w.Write(vr) } -// handleNotFound is a generic handler for an invalid route. -func (p *politeiawww) handleNotFound(w http.ResponseWriter, r *http.Request) { - // Log incoming connection - log.Debugf("Invalid route: %v %v %v %v", remoteAddr(r), r.Method, r.URL, - r.Proto) +func (p *politeiawww) handlePolicy(w http.ResponseWriter, r *http.Request) { + // Get the policy command. + log.Tracef("handlePolicy") - // Trace incoming request - log.Tracef("%v", newLogClosure(func() string { - trace, err := httputil.DumpRequest(r, true) - if err != nil { - trace = []byte(fmt.Sprintf("logging: "+ - "DumpRequest %v", err)) - } - return string(trace) - })) + reply := &www.PolicyReply{ + MinPasswordLength: www.PolicyMinPasswordLength, + MinUsernameLength: www.PolicyMinUsernameLength, + MaxUsernameLength: www.PolicyMaxUsernameLength, + UsernameSupportedChars: www.PolicyUsernameSupportedChars, + ProposalListPageSize: www.ProposalListPageSize, + UserListPageSize: www.UserListPageSize, + MaxImages: www.PolicyMaxImages, + MaxImageSize: www.PolicyMaxImageSize, + MaxMDs: www.PolicyMaxMDs, + MaxMDSize: www.PolicyMaxMDSize, + PaywallEnabled: p.paywallIsEnabled(), + ValidMIMETypes: mime.ValidMimeTypes(), + MinProposalNameLength: www.PolicyMinProposalNameLength, + MaxProposalNameLength: www.PolicyMaxProposalNameLength, + ProposalNameSupportedChars: www.PolicyProposalNameSupportedChars, + MaxCommentLength: www.PolicyMaxCommentLength, + TokenPrefixLength: www.TokenPrefixLength, + BuildInformation: version.BuildInformation(), + IndexFilename: www.PolicyIndexFilename, + MinLinkByPeriod: p.linkByPeriodMin(), + MaxLinkByPeriod: p.linkByPeriodMax(), + MinVoteDuration: p.cfg.VoteDurationMin, + MaxVoteDuration: p.cfg.VoteDurationMax, + } - util.RespondWithJSON(w, http.StatusNotFound, www.ErrorReply{}) + util.RespondWithJSON(w, http.StatusOK, reply) } -// handleAllVetted replies with the list of vetted proposals. -func (p *politeiawww) handleAllVetted(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleAllVetted") +// handleTokenInventory returns the tokens of all proposals in the inventory. +func (p *politeiawww) handleTokenInventory(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleTokenInventory") - // Get the all vetted command. - var v www.GetAllVetted - err := util.ParseGetParams(r, &v) - if err != nil { - RespondWithError(w, r, 0, "handleAllVetted: ParseGetParams", - www.UserError{ - ErrorCode: www.ErrorStatusInvalidInput, - }) + // Get session user. This is a public route so one might not exist. + user, err := p.getSessionUser(w, r) + if err != nil && err != errSessionNotFound { + RespondWithError(w, r, 0, + "handleTokenInventory: getSessionUser %v", err) return } - vr, err := p.processAllVetted(v) + isAdmin := user != nil && user.Admin + reply, err := p.processTokenInventory(isAdmin) if err != nil { RespondWithError(w, r, 0, - "handleAllVetted: processAllVetted %v", err) + "handleTokenInventory: processTokenInventory: %v", err) return } - - util.RespondWithJSON(w, http.StatusOK, vr) + util.RespondWithJSON(w, http.StatusOK, reply) } // handleProposalDetails handles the incoming proposal details command. It @@ -240,31 +245,6 @@ func (p *politeiawww) handleProposalDetails(w http.ResponseWriter, r *http.Reque util.RespondWithJSON(w, http.StatusOK, reply) } -// handleBatchVoteSummary handles the incoming batch vote summary command. It -// returns a VoteSummary for each of the provided censorship tokens. -func (p *politeiawww) handleBatchVoteSummary(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleBatchVoteSummary") - - var bvs www.BatchVoteSummary - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&bvs); err != nil { - RespondWithError(w, r, 0, "handleBatchVoteSummary: unmarshal", - www.UserError{ - ErrorCode: www.ErrorStatusInvalidInput, - }) - return - } - - reply, err := p.processBatchVoteSummary(bvs) - if err != nil { - RespondWithError(w, r, 0, - "handleBatchVoteSummary: processBatchVoteSummary %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, reply) -} - // handleBatchProposals handles the incoming batch proposals command. It // returns a ProposalRecord for each of the provided censorship tokens. func (p *politeiawww) handleBatchProposals(w http.ResponseWriter, r *http.Request) { @@ -298,123 +278,7 @@ func (p *politeiawww) handleBatchProposals(w http.ResponseWriter, r *http.Reques util.RespondWithJSON(w, http.StatusOK, reply) } -func (p *politeiawww) handlePolicy(w http.ResponseWriter, r *http.Request) { - // Get the policy command. - log.Tracef("handlePolicy") - - reply := &www.PolicyReply{ - MinPasswordLength: www.PolicyMinPasswordLength, - MinUsernameLength: www.PolicyMinUsernameLength, - MaxUsernameLength: www.PolicyMaxUsernameLength, - UsernameSupportedChars: www.PolicyUsernameSupportedChars, - ProposalListPageSize: www.ProposalListPageSize, - UserListPageSize: www.UserListPageSize, - MaxImages: www.PolicyMaxImages, - MaxImageSize: www.PolicyMaxImageSize, - MaxMDs: www.PolicyMaxMDs, - MaxMDSize: www.PolicyMaxMDSize, - PaywallEnabled: p.paywallIsEnabled(), - ValidMIMETypes: mime.ValidMimeTypes(), - MinProposalNameLength: www.PolicyMinProposalNameLength, - MaxProposalNameLength: www.PolicyMaxProposalNameLength, - ProposalNameSupportedChars: www.PolicyProposalNameSupportedChars, - MaxCommentLength: www.PolicyMaxCommentLength, - TokenPrefixLength: www.TokenPrefixLength, - BuildInformation: version.BuildInformation(), - IndexFilename: www.PolicyIndexFilename, - MinLinkByPeriod: p.linkByPeriodMin(), - MaxLinkByPeriod: p.linkByPeriodMax(), - MinVoteDuration: p.cfg.VoteDurationMin, - MaxVoteDuration: p.cfg.VoteDurationMax, - } - - util.RespondWithJSON(w, http.StatusOK, reply) -} - -// handleCommentsGet handles batched comments get. -func (p *politeiawww) handleCommentsGet(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleCommentsGet") - - pathParams := mux.Vars(r) - token := pathParams["token"] - - // Get session user. This is a public route so one might not exist. - user, err := p.getSessionUser(w, r) - if err != nil && err != errSessionNotFound { - RespondWithError(w, r, 0, - "handleCommentsGet: getSessionUser %v", err) - return - } - - gcr, err := p.processCommentsGet(token, user) - if err != nil { - RespondWithError(w, r, 0, - "handleCommentsGet: processCommentsGet %v", err) - return - } - util.RespondWithJSON(w, http.StatusOK, gcr) -} - -// handleUserProposals returns the proposals for the given user. -func (p *politeiawww) handleUserProposals(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleUserProposals") - - // Get the user proposals command. - var up www.UserProposals - err := util.ParseGetParams(r, &up) - if err != nil { - RespondWithError(w, r, 0, "handleUserProposals: ParseGetParams", - www.UserError{ - ErrorCode: www.ErrorStatusInvalidInput, - }) - return - } - - userId, err := uuid.Parse(up.UserId) - if err != nil { - RespondWithError(w, r, 0, "handleUserProposals: ParseUint", - www.UserError{ - ErrorCode: www.ErrorStatusInvalidInput, - }) - return - } - - // Get session user. This is a public route so one might not exist. - user, err := p.getSessionUser(w, r) - if err != nil && err != errSessionNotFound { - RespondWithError(w, r, 0, - "handleUserProposals: getSessionUser %v", err) - return - } - - upr, err := p.processUserProposals( - &up, - user != nil && user.ID == userId, - user != nil && user.Admin) - if err != nil { - RespondWithError(w, r, 0, - "handleUserProposals: processUserProposals %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, upr) -} - -// handleActiveVote returns all active proposals that have an active vote. -func (p *politeiawww) handleActiveVote(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleActiveVote") - - avr, err := p.processActiveVote() - if err != nil { - RespondWithError(w, r, 0, - "handleActiveVote: processActiveVote %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, avr) -} - -// handleCastVotes records the user votes in politeiad. +// handleCastVotes casts dcr ticket votes for a proposal vote. func (p *politeiawww) handleCastVotes(w http.ResponseWriter, r *http.Request) { log.Tracef("handleCastVotes") @@ -455,67 +319,28 @@ func (p *politeiawww) handleVoteResults(w http.ResponseWriter, r *http.Request) util.RespondWithJSON(w, http.StatusOK, vrr) } -// handleVoteDetails returns the vote details for the given proposal token. -func (p *politeiawww) handleVoteDetailsV2(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleVoteDetailsV2") - - pathParams := mux.Vars(r) - token := pathParams["token"] - - vrr, err := p.processVoteDetailsV2(token) - if err != nil { - RespondWithError(w, r, 0, - "handleVoteDetailsV2: processVoteDetailsV2: %v", - err) - return - } - - util.RespondWithJSON(w, http.StatusOK, vrr) -} +// handleBatchVoteSummary handles the incoming batch vote summary command. It +// returns a VoteSummary for each of the provided censorship tokens. +func (p *politeiawww) handleBatchVoteSummary(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleBatchVoteSummary") -// handleGetAllVoteStatus returns the voting status of all public proposals. -func (p *politeiawww) handleGetAllVoteStatus(w http.ResponseWriter, r *http.Request) { - gasvr, err := p.processGetAllVoteStatus() - if err != nil { - RespondWithError(w, r, 0, - "handleGetAllVoteStatus: processGetAllVoteStatus %v", err) + var bvs www.BatchVoteSummary + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&bvs); err != nil { + RespondWithError(w, r, 0, "handleBatchVoteSummary: unmarshal", + www.UserError{ + ErrorCode: www.ErrorStatusInvalidInput, + }) return } - util.RespondWithJSON(w, http.StatusOK, gasvr) -} - -// handleVoteStatus returns the vote status for a given proposal. -func (p *politeiawww) handleVoteStatus(w http.ResponseWriter, r *http.Request) { - pathParams := mux.Vars(r) - vsr, err := p.processVoteStatus(pathParams["token"]) + reply, err := p.processBatchVoteSummary(bvs) if err != nil { RespondWithError(w, r, 0, - "handleVoteStatus: ProcessVoteStatus: %v", err) - return - } - util.RespondWithJSON(w, http.StatusOK, vsr) -} - -// handleTokenInventory returns the tokens of all proposals in the inventory. -func (p *politeiawww) handleTokenInventory(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleTokenInventory") - - // Get session user. This is a public route so one might not exist. - user, err := p.getSessionUser(w, r) - if err != nil && err != errSessionNotFound { - RespondWithError(w, r, 0, - "handleTokenInventory: getSessionUser %v", err) + "handleBatchVoteSummary: processBatchVoteSummary %v", err) return } - isAdmin := user != nil && user.Admin - reply, err := p.processTokenInventory(isAdmin) - if err != nil { - RespondWithError(w, r, 0, - "handleTokenInventory: processTokenInventory: %v", err) - return - } util.RespondWithJSON(w, http.StatusOK, reply) } @@ -541,95 +366,6 @@ func (p *politeiawww) handleProposalPaywallDetails(w http.ResponseWriter, r *htt util.RespondWithJSON(w, http.StatusOK, reply) } -// handleNewComment handles incomming comments. -func (p *politeiawww) handleNewComment(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleNewComment") - - var sc www.NewComment - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&sc); err != nil { - RespondWithError(w, r, 0, "handleNewComment: unmarshal", - www.UserError{ - ErrorCode: www.ErrorStatusInvalidInput, - }) - return - } - - user, err := p.getSessionUser(w, r) - if err != nil { - RespondWithError(w, r, 0, - "handleNewComment: getSessionUser %v", err) - return - } - - cr, err := p.processNewComment(sc, user) - if err != nil { - RespondWithError(w, r, 0, - "handleNewComment: processNewComment: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, cr) -} - -// handleLikeComment handles up or down voting of commentd. -func (p *politeiawww) handleLikeComment(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleLikeComment") - - var lc www.LikeComment - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&lc); err != nil { - RespondWithError(w, r, 0, "handleLikeComment: unmarshal", - www.UserError{ - ErrorCode: www.ErrorStatusInvalidInput, - }) - return - } - - user, err := p.getSessionUser(w, r) - if err != nil { - RespondWithError(w, r, 0, - "handleLikeComment: getSessionUser %v", err) - return - } - - cr, err := p.processLikeComment(lc, user) - if err != nil { - RespondWithError(w, r, 0, - "handleLikeComment: processLikeComment %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, cr) -} - -// handleAuthorizeVote handles authorizing a proposal vote. -func (p *politeiawww) handleAuthorizeVote(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleAuthorizeVote") - var av www.AuthorizeVote - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&av); err != nil { - RespondWithError(w, r, 0, "handleAuthorizeVote: unmarshal", - www.UserError{ - ErrorCode: www.ErrorStatusInvalidInput, - }) - return - } - user, err := p.getSessionUser(w, r) - if err != nil { - RespondWithError(w, r, 0, - "handleAuthorizeVote: getSessionUser %v", err) - return - } - avr, err := p.processAuthorizeVote(av, user) - if err != nil { - RespondWithError(w, r, 0, - "handleAuthorizeVote: processAuthorizeVote %v", err) - return - } - util.RespondWithJSON(w, http.StatusOK, avr) -} - // handleProposalPaywallPayment returns the payment details for a pending // proposal paywall payment. func (p *politeiawww) handleProposalPaywallPayment(w http.ResponseWriter, r *http.Request) { @@ -895,223 +631,36 @@ func (p *politeiawww) handleAuthenticatedWebsocket(w http.ResponseWriter, r *htt p.handleWebsocket(w, r, id) } -// handleStartVote handles the v2 StartVote route. -func (p *politeiawww) handleStartVoteV2(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleStartVoteV2") - - var sv www2.StartVote - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&sv); err != nil { - RespondWithError(w, r, 0, "handleStartVoteV2: unmarshal", - www.UserError{ - ErrorCode: www.ErrorStatusInvalidInput, - }) - return - } - - user, err := p.getSessionUser(w, r) - if err != nil { - RespondWithError(w, r, 0, - "handleStartVoteV2: getSessionUser %v", err) - return - } - - // Sanity - if !user.Admin { - RespondWithError(w, r, 0, - "handleStartVoteV2: admin %v", user.Admin) - return - } - - svr, err := p.processStartVoteV2(sv, user) - if err != nil { - RespondWithError(w, r, 0, - "handleStartVoteV2: processStartVoteV2 %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, svr) -} - -// handleStartVoteRunoffV2 handles starting a runoff vote for RFP proposal -// submissions. -func (p *politeiawww) handleStartVoteRunoffV2(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleStartVoteRunoffV2") - - var sv www2.StartVoteRunoff - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&sv); err != nil { - RespondWithError(w, r, 0, "handleStartVoteRunoffV2: unmarshal", - www.UserError{ - ErrorCode: www.ErrorStatusInvalidInput, - }) - return - } - - user, err := p.getSessionUser(w, r) - if err != nil { - RespondWithError(w, r, 0, - "handleStartVoteRunoffV2: getSessionUser %v", err) - return - } - - // Sanity - if !user.Admin { - RespondWithError(w, r, 0, - "handleStartVoteRunoffV2: admin %v", user.Admin) - return - } - - svr, err := p.processStartVoteRunoffV2(sv, user) - if err != nil { - RespondWithError(w, r, 0, - "handleStartVoteRunoffV2: processStartVoteRunoff %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, svr) -} - -// handleCensorComment handles the censoring of a comment by an admin. -func (p *politeiawww) handleCensorComment(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleCensorComment") - - var cc www.CensorComment - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&cc); err != nil { - RespondWithError(w, r, 0, "handleCensorComment: unmarshal", - www.UserError{ - ErrorCode: www.ErrorStatusInvalidInput, - }) - return - } - - user, err := p.getSessionUser(w, r) - if err != nil { - RespondWithError(w, r, 0, - "handleCensorComment: getSessionUser %v", err) - return - } - - cr, err := p.processCensorComment(cc, user) - if err != nil { - RespondWithError(w, r, 0, - "handleCensorComment: processCensorComment %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, cr) -} - -// handleSetTOTP handles the setting of TOTP Key -func (p *politeiawww) handleSetTOTP(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleSetTOTP") - - var st www.SetTOTP - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&st); err != nil { - RespondWithError(w, r, 0, "handleSetTOTP: unmarshal", - www.UserError{ - ErrorCode: www.ErrorStatusInvalidInput, - }) - return - } - - u, err := p.getSessionUser(w, r) - if err != nil { - RespondWithError(w, r, 0, - "handleSetTOTP: getSessionUser %v", err) - return - } - - str, err := p.processSetTOTP(st, u) - if err != nil { - RespondWithError(w, r, 0, - "handleSetTOTP: processSetTOTP %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, str) -} - -// handleVerifyTOTP handles the request to verify a set TOTP Key. -func (p *politeiawww) handleVerifyTOTP(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleVerifyTOTP") - - var vt www.VerifyTOTP - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&vt); err != nil { - RespondWithError(w, r, 0, "handleVerifyTOTP: unmarshal", - www.UserError{ - ErrorCode: www.ErrorStatusInvalidInput, - }) - return - } - - u, err := p.getSessionUser(w, r) - if err != nil { - RespondWithError(w, r, 0, - "handleVerifyTOTP: getSessionUser %v", err) - return - } - - vtr, err := p.processVerifyTOTP(vt, u) - if err != nil { - RespondWithError(w, r, 0, - "handleVerifyTOTP: processVerifyTOTP %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, vtr) -} - // setPoliteiaWWWRoutes sets up the politeia routes. func (p *politeiawww) setPoliteiaWWWRoutes() { - // Public routes. - p.router.HandleFunc("/", closeBody(logging(p.handleVersion))).Methods(http.MethodGet) + // Home + p.router.HandleFunc("/", closeBody(logging(p.handleVersion))). + Methods(http.MethodGet) + + // Not found p.router.NotFoundHandler = closeBody(p.handleNotFound) - p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteVersion, p.handleVersion, - permissionPublic) + // Public routes p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteAllVetted, p.handleAllVetted, - permissionPublic) - p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteProposalDetails, p.handleProposalDetails, + www.RouteVersion, p.handleVersion, permissionPublic) p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, www.RoutePolicy, p.handlePolicy, permissionPublic) p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteCommentsGet, p.handleCommentsGet, - permissionPublic) - p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteUserProposals, p.handleUserProposals, + www.RouteTokenInventory, p.handleTokenInventory, permissionPublic) p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteActiveVote, p.handleActiveVote, + www.RouteProposalDetails, p.handleProposalDetails, permissionPublic) p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, - www.RouteCastVotes, p.handleCastVotes, - permissionPublic) - p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteVoteResults, p.handleVoteResults, - permissionPublic) - p.addRoute(http.MethodGet, www2.APIRoute, - www2.RouteVoteDetails, p.handleVoteDetailsV2, - permissionPublic) - p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteAllVoteStatus, p.handleGetAllVoteStatus, + www.RouteBatchProposals, p.handleBatchProposals, permissionPublic) p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteVoteStatus, p.handleVoteStatus, + www.RouteCastVotes, p.handleCastVotes, permissionPublic) p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteTokenInventory, p.handleTokenInventory, - permissionPublic) - p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, - www.RouteBatchProposals, p.handleBatchProposals, + www.RouteVoteResults, p.handleVoteResults, permissionPublic) p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, www.RouteBatchVoteSummary, p.handleBatchVoteSummary, @@ -1121,24 +670,9 @@ func (p *politeiawww) setPoliteiaWWWRoutes() { p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, www.RouteProposalPaywallDetails, p.handleProposalPaywallDetails, permissionLogin) - p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, - www.RouteNewComment, p.handleNewComment, - permissionLogin) - p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, - www.RouteLikeComment, p.handleLikeComment, - permissionLogin) - p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, - www.RouteAuthorizeVote, p.handleAuthorizeVote, - permissionLogin) p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, www.RouteProposalPaywallPayment, p.handleProposalPaywallPayment, permissionLogin) - p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, - www.RouteSetTOTP, p.handleSetTOTP, - permissionLogin) - p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, - www.RouteVerifyTOTP, p.handleVerifyTOTP, - permissionLogin) // Unauthenticated websocket p.addRoute("", www.PoliteiaWWWAPIRoute, @@ -1148,15 +682,4 @@ func (p *politeiawww) setPoliteiaWWWRoutes() { p.addRoute("", www.PoliteiaWWWAPIRoute, www.RouteAuthenticatedWebSocket, p.handleAuthenticatedWebsocket, permissionLogin) - - // Routes that require being logged in as an admin user. - p.addRoute(http.MethodPost, www2.APIRoute, - www2.RouteStartVote, p.handleStartVoteV2, - permissionAdmin) - p.addRoute(http.MethodPost, www2.APIRoute, - www2.RouteStartVoteRunoff, p.handleStartVoteRunoffV2, - permissionAdmin) - p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, - www.RouteCensorComment, p.handleCensorComment, - permissionAdmin) } diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index cb6d43e18..7ec428343 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -6,57 +6,19 @@ package main import ( "encoding/base64" - "encoding/hex" - "encoding/json" "fmt" - "net/http" - "sort" "strconv" "time" "github.com/decred/politeia/decredplugin" piplugin "github.com/decred/politeia/plugins/pi" ticketvote "github.com/decred/politeia/plugins/ticketvote" - pd "github.com/decred/politeia/politeiad/api/v1" pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" www2 "github.com/decred/politeia/politeiawww/api/www/v2" "github.com/decred/politeia/politeiawww/user" - "github.com/decred/politeia/util" ) -// proposalStats is used to provide a summary of the number of proposals -// grouped by proposal status. -type proposalsSummary struct { - Invalid int - NotReviewed int - Censored int - Public int - UnreviewedChanges int - Abandoned int -} - -// proposalsFilter is used to pass filtering parameters into the filterProps -// function. -type proposalsFilter struct { - After string - Before string - UserID string - StateMap map[www.PropStateT]bool -} - -// isProposalAuthor returns whether the provided user is the author of the -// provided proposal. -func isProposalAuthor(pr www.ProposalRecord, u user.User) bool { - var isAuthor bool - for _, v := range u.Identities { - if v.String() == pr.PublicKey { - isAuthor = true - } - } - return isAuthor -} - // isRFP returns whether the proposal is a Request For Proposals (RFP). func isRFP(pr www.ProposalRecord) bool { return pr.LinkBy != 0 @@ -131,95 +93,6 @@ func (p *politeiawww) getAllProps() ([]www.ProposalRecord, error) { return nil, nil } -// filterProps filters the given proposals according to the filtering -// parameters specified by the passed in proposalsFilter. filterProps will -// only return a single page of proposals regardless of how many proposals are -// passed in. -func filterProps(filter proposalsFilter, all []www.ProposalRecord) []www.ProposalRecord { - log.Tracef("filterProps") - - sort.Slice(all, func(i, j int) bool { - // Sort by older timestamp first, if timestamps are different - // from each other - if all[i].Timestamp != all[j].Timestamp { - return all[i].Timestamp < all[j].Timestamp - } - - // Otherwise sort by token - return all[i].CensorshipRecord.Token > - all[j].CensorshipRecord.Token - }) - - // pageStarted stores whether or not it's okay to start adding - // proposals to the array. If the after or before parameter is - // supplied, we must find the beginning (or end) of the page first. - pageStarted := (filter.After == "" && filter.Before == "") - beforeIdx := -1 - proposals := make([]www.ProposalRecord, 0, len(all)) - - // Iterate in reverse order because they're sorted by oldest - // timestamp first. - for i := len(all) - 1; i >= 0; i-- { - proposal := all[i] - - // Filter by user if it's provided. - if (filter.UserID != "") && (filter.UserID != proposal.UserId) { - continue - } - - // Filter by the state. - if val, ok := filter.StateMap[proposal.State]; !ok || !val { - continue - } - - if pageStarted { - proposals = append(proposals, proposal) - if len(proposals) >= www.ProposalListPageSize { - break - } - } else if filter.After != "" { - // The beginning of the page has been found, so - // the next public proposal is added. - pageStarted = proposal.CensorshipRecord.Token == filter.After - } else if filter.Before != "" { - // The end of the page has been found, so we'll - // have to iterate in the other direction to - // add the proposals; save the current index. - if proposal.CensorshipRecord.Token == filter.Before { - beforeIdx = i - break - } - } - } - - // If beforeIdx is set, the caller is asking for vetted proposals - // whose last result is before the provided proposal. - if beforeIdx >= 0 { - for _, proposal := range all[beforeIdx+1:] { - // Filter by user if it's provided. - if (filter.UserID != "") && (filter.UserID != proposal.UserId) { - continue - } - - // Filter by the state. - if val, ok := filter.StateMap[proposal.State]; !ok || !val { - continue - } - - // The iteration direction is oldest -> newest, - // so proposals are prepended to the array so - // the result will be newest -> oldest. - proposals = append([]www.ProposalRecord{proposal}, - proposals...) - if len(proposals) >= www.ProposalListPageSize { - break - } - } - } - - return proposals -} - func convertStateToWWW(state pi.PropStateT) www.PropStateT { switch state { case pi.PropStateInvalid: @@ -378,57 +251,6 @@ func (p *politeiawww) processBatchProposals(bp www.BatchProposals, u *user.User) }, nil } -// getUserProps gets the latest version of all proposals from the cache and -// then filters the proposals according to the specified proposalsFilter, which -// is required to contain a userID. In addition to a page of filtered user -// proposals, this function also returns summary statistics for all of the -// proposals that the user has submitted grouped by proposal status. -func (p *politeiawww) getUserProps(filter proposalsFilter) ([]www.ProposalRecord, *proposalsSummary, error) { - log.Tracef("getUserProps: %v", filter.UserID) - - if filter.UserID == "" { - return nil, nil, fmt.Errorf("filter missing userID") - } - - // Get the latest version of all proposals from the cache - // TODO do not use getAllProps - all, err := p.getAllProps() - if err != nil { - return nil, nil, fmt.Errorf("getAllProps: %v", err) - } - - // Find proposal summary statistics for the user. This - // includes statistics on ALL of the proposals that the user - // has submitted. Not just the single page of proposals that - // is going to be returned. - var ps proposalsSummary - for _, v := range all { - if v.UserId != filter.UserID { - continue - } - switch v.Status { - case www.PropStatusNotReviewed: - ps.NotReviewed++ - case www.PropStatusCensored: - ps.Censored++ - case www.PropStatusPublic: - ps.Public++ - case www.PropStatusUnreviewedChanges: - ps.UnreviewedChanges++ - case www.PropStatusAbandoned: - ps.Abandoned++ - default: - ps.Invalid++ - } - } - - // Filter proposals according to the proposalsFilter. Only - // a single page of proposals will be returned. - filtered := filterProps(filter, all) - - return filtered, &ps, nil -} - func (p *politeiawww) getPropComments(token string) ([]www.Comment, error) { return nil, nil } @@ -761,28 +583,30 @@ func validateAuthorizeVote(av www.AuthorizeVote, u user.User, pr www.ProposalRec // is participating in a standard vote. A UserError is returned if any of the // validation fails. func validateAuthorizeVoteStandard(av www.AuthorizeVote, u user.User, pr www.ProposalRecord, vs www.VoteSummary) error { - err := validateAuthorizeVote(av, u, pr, vs) - if err != nil { - return err - } - - // The rest of the validation is specific to authorize votes for - // standard votes. - switch { - case isRFPSubmission(pr): - return www.UserError{ - ErrorCode: www.ErrorStatusWrongProposalType, - ErrorContext: []string{"proposal is an rfp submission"}, + /* + err := validateAuthorizeVote(av, u, pr, vs) + if err != nil { + return err } - case pr.PublicKey != av.PublicKey: - // User is not the author. First make sure the author didn't - // submit the proposal using an old identity. - if !isProposalAuthor(pr, u) { + + // The rest of the validation is specific to authorize votes for + // standard votes. + switch { + case isRFPSubmission(pr): return www.UserError{ - ErrorCode: www.ErrorStatusUserNotAuthor, + ErrorCode: www.ErrorStatusWrongProposalType, + ErrorContext: []string{"proposal is an rfp submission"}, + } + case pr.PublicKey != av.PublicKey: + // User is not the author. First make sure the author didn't + // submit the proposal using an old identity. + if !isProposalAuthor(pr, u) { + return www.UserError{ + ErrorCode: www.ErrorStatusUserNotAuthor, + } } } - } + */ return nil } @@ -819,317 +643,329 @@ func (p *politeiawww) voteSummaryGet(token string, bestBlock uint64) (*www.VoteS func (p *politeiawww) processAuthorizeVote(av www.AuthorizeVote, u *user.User) (*www.AuthorizeVoteReply, error) { log.Tracef("processAuthorizeVote %v", av.Token) - // Make sure token is valid and not a prefix - if !tokenIsValid(av.Token) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: []string{av.Token}, + /* + // Make sure token is valid and not a prefix + if !tokenIsValid(av.Token) { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidCensorshipToken, + ErrorContext: []string{av.Token}, + } } - } - // Validate the vote authorization - pr, err := p.getProp(av.Token) - if err != nil { - /* - // TODO + // Validate the vote authorization + pr, err := p.getProp(av.Token) + if err != nil { if err == cache.ErrRecordNotFound { err = www.UserError{ ErrorCode: www.ErrorStatusProposalNotFound, } } - */ - return nil, err - } - bb, err := p.getBestBlock() - if err != nil { - return nil, err - } - vs, err := p.voteSummaryGet(av.Token, bb) - if err != nil { - return nil, err - } - err = validateAuthorizeVoteStandard(av, *u, *pr, *vs) - if err != nil { - return nil, err - } + return nil, err + } + bb, err := p.getBestBlock() + if err != nil { + return nil, err + } + vs, err := p.voteSummaryGet(av.Token, bb) + if err != nil { + return nil, err + } + err = validateAuthorizeVoteStandard(av, *u, *pr, *vs) + if err != nil { + return nil, err + } - // Setup plugin command - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, fmt.Errorf("Random: %v", err) - } + // Setup plugin command + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return nil, fmt.Errorf("Random: %v", err) + } - dav := convertAuthorizeVoteToDecred(av) - payload, err := decredplugin.EncodeAuthorizeVote(dav) - if err != nil { - return nil, fmt.Errorf("EncodeAuthorizeVote: %v", err) - } + dav := convertAuthorizeVoteToDecred(av) + payload, err := decredplugin.EncodeAuthorizeVote(dav) + if err != nil { + return nil, fmt.Errorf("EncodeAuthorizeVote: %v", err) + } - pc := pd.PluginCommand{ - Challenge: hex.EncodeToString(challenge), - ID: decredplugin.ID, - Command: decredplugin.CmdAuthorizeVote, - CommandID: decredplugin.CmdAuthorizeVote + " " + av.Token, - Payload: string(payload), - } + pc := pd.PluginCommand{ + Challenge: hex.EncodeToString(challenge), + ID: decredplugin.ID, + Command: decredplugin.CmdAuthorizeVote, + CommandID: decredplugin.CmdAuthorizeVote + " " + av.Token, + Payload: string(payload), + } - // Send authorizevote plugin request - responseBody, err := p.makeRequest(http.MethodPost, - pd.PluginCommandRoute, pc) - if err != nil { - return nil, err - } + // Send authorizevote plugin request + responseBody, err := p.makeRequest(http.MethodPost, + pd.PluginCommandRoute, pc) + if err != nil { + return nil, err + } - var reply pd.PluginCommandReply - err = json.Unmarshal(responseBody, &reply) - if err != nil { - return nil, fmt.Errorf("Unmarshal PluginCommandReply: %v", err) - } + var reply pd.PluginCommandReply + err = json.Unmarshal(responseBody, &reply) + if err != nil { + return nil, fmt.Errorf("Unmarshal PluginCommandReply: %v", err) + } - // Verify challenge - err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) - if err != nil { - return nil, fmt.Errorf("VerifyChallenge: %v", err) - } + // Verify challenge + err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) + if err != nil { + return nil, fmt.Errorf("VerifyChallenge: %v", err) + } - // Decode plugin reply - avr, err := decredplugin.DecodeAuthorizeVoteReply([]byte(reply.Payload)) - if err != nil { - return nil, fmt.Errorf("DecodeAuthorizeVoteReply: %v", err) - } + // Decode plugin reply + avr, err := decredplugin.DecodeAuthorizeVoteReply([]byte(reply.Payload)) + if err != nil { + return nil, fmt.Errorf("DecodeAuthorizeVoteReply: %v", err) + } - if !p.test && avr.Action == decredplugin.AuthVoteActionAuthorize { - // Emit event notification for proposal vote authorized - p.eventManager.emit(eventProposalVoteAuthorized, - dataProposalVoteAuthorized{ - token: av.Token, - name: pr.Name, - username: u.Username, - email: u.Email, - }) - } + if !p.test && avr.Action == decredplugin.AuthVoteActionAuthorize { + // Emit event notification for proposal vote authorized + p.eventManager.emit(eventProposalVoteAuthorized, + dataProposalVoteAuthorized{ + token: av.Token, + name: pr.Name, + username: u.Username, + email: u.Email, + }) + } - return &www.AuthorizeVoteReply{ - Action: avr.Action, - Receipt: avr.Receipt, - }, nil + return &www.AuthorizeVoteReply{ + Action: avr.Action, + Receipt: avr.Receipt, + }, nil + */ + return nil, nil } // validateVoteOptions verifies that the provided vote options // specify a simple approve/reject vote and nothing else. A UserError is // returned if this validation fails. func validateVoteOptions(options []www2.VoteOption) error { - if len(options) == 0 { - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidVoteOptions, - ErrorContext: []string{"no vote options found"}, - } - } - optionIDs := map[string]bool{ - decredplugin.VoteOptionIDApprove: false, - decredplugin.VoteOptionIDReject: false, - } - for _, vo := range options { - if _, ok := optionIDs[vo.Id]; !ok { - e := fmt.Sprintf("invalid vote option id '%v'", vo.Id) + /* + if len(options) == 0 { return www.UserError{ ErrorCode: www.ErrorStatusInvalidVoteOptions, - ErrorContext: []string{e}, + ErrorContext: []string{"no vote options found"}, } } - optionIDs[vo.Id] = true - } - for k, wasFound := range optionIDs { - if !wasFound { - e := fmt.Sprintf("missing vote option id '%v'", k) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidVoteOptions, - ErrorContext: []string{e}, + optionIDs := map[string]bool{ + decredplugin.VoteOptionIDApprove: false, + decredplugin.VoteOptionIDReject: false, + } + for _, vo := range options { + if _, ok := optionIDs[vo.Id]; !ok { + e := fmt.Sprintf("invalid vote option id '%v'", vo.Id) + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidVoteOptions, + ErrorContext: []string{e}, + } + } + optionIDs[vo.Id] = true + } + for k, wasFound := range optionIDs { + if !wasFound { + e := fmt.Sprintf("missing vote option id '%v'", k) + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidVoteOptions, + ErrorContext: []string{e}, + } } } - } + return nil + */ return nil } func validateStartVote(sv www2.StartVote, u user.User, pr www.ProposalRecord, vs www.VoteSummary, durationMin, durationMax uint32) error { - if !tokenIsValid(sv.Vote.Token) { - // Sanity check since proposal has already been looked up and - // passed in to this function. - return fmt.Errorf("invalid token %v", sv.Vote.Token) - } + /* + if !tokenIsValid(sv.Vote.Token) { + // Sanity check since proposal has already been looked up and + // passed in to this function. + return fmt.Errorf("invalid token %v", sv.Vote.Token) + } - // Validate vote bits - for _, v := range sv.Vote.Options { - err := validateVoteBit(sv.Vote, v.Bits) - if err != nil { - log.Debugf("validateStartVote: validateVoteBit '%v': %v", - v.Id, err) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidPropVoteBits, + // Validate vote bits + for _, v := range sv.Vote.Options { + err := validateVoteBit(sv.Vote, v.Bits) + if err != nil { + log.Debugf("validateStartVote: validateVoteBit '%v': %v", + v.Id, err) + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidPropVoteBits, + } } } - } - - // Validate vote options. Only simple yes/no votes are currently - // allowed. - err := validateVoteOptions(sv.Vote.Options) - if err != nil { - return err - } - // Validate vote params - switch { - case sv.Vote.Duration < durationMin: - // Duration not large enough - e := fmt.Sprintf("vote duration must be >= %v", durationMin) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidPropVoteParams, - ErrorContext: []string{e}, - } - case sv.Vote.Duration > durationMax: - // Duration too large - e := fmt.Sprintf("vote duration must be <= %v", durationMax) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidPropVoteParams, - ErrorContext: []string{e}, - } - case sv.Vote.QuorumPercentage > 100: - // Quorum too large - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidPropVoteParams, - ErrorContext: []string{"quorum percentage cannot be >100"}, - } - case sv.Vote.PassPercentage > 100: - // Pass percentage too large - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidPropVoteParams, - ErrorContext: []string{"pass percentage cannot be >100"}, + // Validate vote options. Only simple yes/no votes are currently + // allowed. + err := validateVoteOptions(sv.Vote.Options) + if err != nil { + return err } - } - // Ensure the public key is the user's active key - if sv.PublicKey != u.PublicKey() { - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidSigningKey, + // Validate vote params + switch { + case sv.Vote.Duration < durationMin: + // Duration not large enough + e := fmt.Sprintf("vote duration must be >= %v", durationMin) + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidPropVoteParams, + ErrorContext: []string{e}, + } + case sv.Vote.Duration > durationMax: + // Duration too large + e := fmt.Sprintf("vote duration must be <= %v", durationMax) + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidPropVoteParams, + ErrorContext: []string{e}, + } + case sv.Vote.QuorumPercentage > 100: + // Quorum too large + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidPropVoteParams, + ErrorContext: []string{"quorum percentage cannot be >100"}, + } + case sv.Vote.PassPercentage > 100: + // Pass percentage too large + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidPropVoteParams, + ErrorContext: []string{"pass percentage cannot be >100"}, + } } - } - // Validate signature - dsv := convertStartVoteV2ToDecred(sv) - err = dsv.VerifySignature() - if err != nil { - log.Debugf("validateStartVote: VerifySignature: %v", err) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidSignature, + // Ensure the public key is the user's active key + if sv.PublicKey != u.PublicKey() { + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidSigningKey, + } } - } - // Validate proposal - votePropVersion := strconv.FormatUint(uint64(sv.Vote.ProposalVersion), 10) - switch { - case pr.Version != votePropVersion: - // Vote is specifying the wrong version - e := fmt.Sprintf("got %v, want %v", votePropVersion, pr.Version) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidProposalVersion, - ErrorContext: []string{e}, - } - case pr.Status != www.PropStatusPublic: - // Proposal is not public - return www.UserError{ - ErrorCode: www.ErrorStatusWrongStatus, - ErrorContext: []string{"proposal is not public"}, + // Validate signature + dsv := convertStartVoteV2ToDecred(sv) + err = dsv.VerifySignature() + if err != nil { + log.Debugf("validateStartVote: VerifySignature: %v", err) + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidSignature, + } } - case vs.EndHeight != 0: - // Vote has already started - return www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - ErrorContext: []string{"vote already started"}, + + // Validate proposal + votePropVersion := strconv.FormatUint(uint64(sv.Vote.ProposalVersion), 10) + switch { + case pr.Version != votePropVersion: + // Vote is specifying the wrong version + e := fmt.Sprintf("got %v, want %v", votePropVersion, pr.Version) + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidProposalVersion, + ErrorContext: []string{e}, + } + case pr.Status != www.PropStatusPublic: + // Proposal is not public + return www.UserError{ + ErrorCode: www.ErrorStatusWrongStatus, + ErrorContext: []string{"proposal is not public"}, + } + case vs.EndHeight != 0: + // Vote has already started + return www.UserError{ + ErrorCode: www.ErrorStatusWrongVoteStatus, + ErrorContext: []string{"vote already started"}, + } } - } + return nil + */ return nil } func (p *politeiawww) validateStartVoteStandard(sv www2.StartVote, u user.User, pr www.ProposalRecord, vs www.VoteSummary) error { - err := validateStartVote(sv, u, pr, vs, p.cfg.VoteDurationMin, - p.cfg.VoteDurationMax) - if err != nil { - return err - } - - // The remaining validation is specific to a VoteTypeStandard. - switch { - case sv.Vote.Type != www2.VoteTypeStandard: - // Not a standard vote - e := fmt.Sprintf("vote type must be %v", www2.VoteTypeStandard) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidVoteType, - ErrorContext: []string{e}, - } - case vs.Status != www.PropVoteStatusAuthorized: - // Vote has not been authorized - return www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - ErrorContext: []string{"vote not authorized"}, + /* + err := validateStartVote(sv, u, pr, vs, p.cfg.VoteDurationMin, + p.cfg.VoteDurationMax) + if err != nil { + return err } - case isRFPSubmission(pr): - // The proposal is an an RFP submission. The voting period for - // RFP submissions can only be started using the StartVoteRunoff - // route. - return www.UserError{ - ErrorCode: www.ErrorStatusWrongProposalType, - ErrorContext: []string{"cannot be an rfp submission"}, + + // The remaining validation is specific to a VoteTypeStandard. + switch { + case sv.Vote.Type != www2.VoteTypeStandard: + // Not a standard vote + e := fmt.Sprintf("vote type must be %v", www2.VoteTypeStandard) + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidVoteType, + ErrorContext: []string{e}, + } + case vs.Status != www.PropVoteStatusAuthorized: + // Vote has not been authorized + return www.UserError{ + ErrorCode: www.ErrorStatusWrongVoteStatus, + ErrorContext: []string{"vote not authorized"}, + } + case isRFPSubmission(pr): + // The proposal is an an RFP submission. The voting period for + // RFP submissions can only be started using the StartVoteRunoff + // route. + return www.UserError{ + ErrorCode: www.ErrorStatusWrongProposalType, + ErrorContext: []string{"cannot be an rfp submission"}, + } } - } - // Verify the LinkBy deadline for RFP proposals. The LinkBy policy - // requirements are enforced at the time of starting the vote - // because its purpose is to ensure that there is enough time for - // RFP submissions to be submitted. - if isRFP(pr) { - err := p.linkByValidate(pr.LinkBy) - if err != nil { - return err + // Verify the LinkBy deadline for RFP proposals. The LinkBy policy + // requirements are enforced at the time of starting the vote + // because its purpose is to ensure that there is enough time for + // RFP submissions to be submitted. + if isRFP(pr) { + err := p.linkByValidate(pr.LinkBy) + if err != nil { + return err + } } - } + return nil + */ return nil } func validateStartVoteRunoff(sv www2.StartVote, u user.User, pr www.ProposalRecord, vs www.VoteSummary, durationMin, durationMax uint32) error { - err := validateStartVote(sv, u, pr, vs, durationMin, durationMax) - if err != nil { - return err - } - - // The remaining validation is specific to a VoteTypeRunoff. - - token := sv.Vote.Token - switch { - case sv.Vote.Type != www2.VoteTypeRunoff: - // Not a runoff vote - e := fmt.Sprintf("%v vote type must be %v", - token, www2.VoteTypeRunoff) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidVoteType, - ErrorContext: []string{e}, + /* + err := validateStartVote(sv, u, pr, vs, durationMin, durationMax) + if err != nil { + return err } - case !isRFPSubmission(pr): - // The proposal is not an RFP submission - e := fmt.Sprintf("%v is not an rfp submission", token) - return www.UserError{ - ErrorCode: www.ErrorStatusWrongProposalType, - ErrorContext: []string{e}, - } + // The remaining validation is specific to a VoteTypeRunoff. - case vs.Status != www.PropVoteStatusNotAuthorized: - // Sanity check. This should not be possible. - return fmt.Errorf("%v got vote status %v, want %v", - token, vs.Status, www.PropVoteStatusNotAuthorized) - } + token := sv.Vote.Token + switch { + case sv.Vote.Type != www2.VoteTypeRunoff: + // Not a runoff vote + e := fmt.Sprintf("%v vote type must be %v", + token, www2.VoteTypeRunoff) + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidVoteType, + ErrorContext: []string{e}, + } + + case !isRFPSubmission(pr): + // The proposal is not an RFP submission + e := fmt.Sprintf("%v is not an rfp submission", token) + return www.UserError{ + ErrorCode: www.ErrorStatusWrongProposalType, + ErrorContext: []string{e}, + } + + case vs.Status != www.PropVoteStatusNotAuthorized: + // Sanity check. This should not be possible. + return fmt.Errorf("%v got vote status %v, want %v", + token, vs.Status, www.PropVoteStatusNotAuthorized) + } + return nil + */ return nil } @@ -1139,104 +975,105 @@ func validateStartVoteRunoff(sv www2.StartVote, u user.User, pr www.ProposalReco func (p *politeiawww) processStartVoteV2(sv www2.StartVote, u *user.User) (*www2.StartVoteReply, error) { log.Tracef("processStartVoteV2 %v", sv.Vote.Token) - // Sanity check - if !u.Admin { - return nil, fmt.Errorf("user is not an admin") - } + /* + // Sanity check + if !u.Admin { + return nil, fmt.Errorf("user is not an admin") + } - // Fetch proposal and vote summary - if !tokenIsValid(sv.Vote.Token) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: []string{sv.Vote.Token}, + // Fetch proposal and vote summary + if !tokenIsValid(sv.Vote.Token) { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidCensorshipToken, + ErrorContext: []string{sv.Vote.Token}, + } } - } - pr, err := p.getProp(sv.Vote.Token) - if err != nil { - /* - // TODO + pr, err := p.getProp(sv.Vote.Token) + if err != nil { if err == cache.ErrRecordNotFound { err = www.UserError{ ErrorCode: www.ErrorStatusProposalNotFound, } } - */ - return nil, err - } - bb, err := p.getBestBlock() - if err != nil { - return nil, err - } - vs, err := p.voteSummaryGet(sv.Vote.Token, bb) - if err != nil { - return nil, err - } + return nil, err + } + bb, err := p.getBestBlock() + if err != nil { + return nil, err + } + vs, err := p.voteSummaryGet(sv.Vote.Token, bb) + if err != nil { + return nil, err + } - // Validate the start vote - err = p.validateStartVoteStandard(sv, *u, *pr, *vs) - if err != nil { - return nil, err - } + // Validate the start vote + err = p.validateStartVoteStandard(sv, *u, *pr, *vs) + if err != nil { + return nil, err + } - // Tell decred plugin to start voting - dsv := convertStartVoteV2ToDecred(sv) - payload, err := decredplugin.EncodeStartVoteV2(dsv) - if err != nil { - return nil, err - } - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - pc := pd.PluginCommand{ - Challenge: hex.EncodeToString(challenge), - ID: decredplugin.ID, - Command: decredplugin.CmdStartVote, - CommandID: decredplugin.CmdStartVote + " " + sv.Vote.Token, - Payload: string(payload), - } - responseBody, err := p.makeRequest(http.MethodPost, - pd.PluginCommandRoute, pc) - if err != nil { - return nil, err - } + // Tell decred plugin to start voting + dsv := convertStartVoteV2ToDecred(sv) + payload, err := decredplugin.EncodeStartVoteV2(dsv) + if err != nil { + return nil, err + } + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return nil, err + } + pc := pd.PluginCommand{ + Challenge: hex.EncodeToString(challenge), + ID: decredplugin.ID, + Command: decredplugin.CmdStartVote, + CommandID: decredplugin.CmdStartVote + " " + sv.Vote.Token, + Payload: string(payload), + } + responseBody, err := p.makeRequest(http.MethodPost, + pd.PluginCommandRoute, pc) + if err != nil { + return nil, err + } - // Handle reply - var reply pd.PluginCommandReply - err = json.Unmarshal(responseBody, &reply) - if err != nil { - return nil, fmt.Errorf("could not unmarshal "+ - "PluginCommandReply: %v", err) - } - err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) - if err != nil { - return nil, err - } - dsvr, err := decredplugin.DecodeStartVoteReply([]byte(reply.Payload)) - if err != nil { - return nil, err - } - svr, err := convertStartVoteReplyV2FromDecred(*dsvr) - if err != nil { - return nil, err - } + // Handle reply + var reply pd.PluginCommandReply + err = json.Unmarshal(responseBody, &reply) + if err != nil { + return nil, fmt.Errorf("could not unmarshal "+ + "PluginCommandReply: %v", err) + } + err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) + if err != nil { + return nil, err + } + dsvr, err := decredplugin.DecodeStartVoteReply([]byte(reply.Payload)) + if err != nil { + return nil, err + } + svr, err := convertStartVoteReplyV2FromDecred(*dsvr) + if err != nil { + return nil, err + } - // Get author data - author, err := p.db.UserGetByPubKey(pr.PublicKey) - if err != nil { - return nil, err - } + // Get author data + author, err := p.db.UserGetByPubKey(pr.PublicKey) + if err != nil { + return nil, err + } - // Emit event notification for proposal start vote - p.eventManager.emit(eventProposalVoteStarted, - dataProposalVoteStarted{ - token: pr.CensorshipRecord.Token, - name: pr.Name, - adminID: u.ID.String(), - author: *author, - }) + // Emit event notification for proposal start vote + p.eventManager.emit(eventProposalVoteStarted, + dataProposalVoteStarted{ + token: pr.CensorshipRecord.Token, + name: pr.Name, + adminID: u.ID.String(), + author: *author, + }) + + return svr, nil + */ - return svr, nil + return nil, nil } // voteIsApproved returns whether the provided VoteSummary met the quorum @@ -1279,318 +1116,268 @@ func voteIsApproved(vs www.VoteSummary) bool { func (p *politeiawww) processStartVoteRunoffV2(sv www2.StartVoteRunoff, u *user.User) (*www2.StartVoteRunoffReply, error) { log.Tracef("processStartVoteRunoffV2 %v", sv.Token) - // Sanity check - if !u.Admin { - return nil, fmt.Errorf("user is not an admin") - } + /* + // Sanity check + if !u.Admin { + return nil, fmt.Errorf("user is not an admin") + } - bb, err := p.getBestBlock() - if err != nil { - return nil, err - } + bb, err := p.getBestBlock() + if err != nil { + return nil, err + } - // Ensure authorize votes and start votes match - auths := make(map[string]www2.AuthorizeVote, len(sv.AuthorizeVotes)) - starts := make(map[string]www2.StartVote, len(sv.StartVotes)) - for _, v := range sv.AuthorizeVotes { - auths[v.Token] = v - } - for _, v := range sv.StartVotes { - _, ok := auths[v.Vote.Token] - if !ok { - e := fmt.Sprintf("start vote found without matching authorize vote %v", - v.Vote.Token) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidRunoffVote, - ErrorContext: []string{e}, + // Ensure authorize votes and start votes match + auths := make(map[string]www2.AuthorizeVote, len(sv.AuthorizeVotes)) + starts := make(map[string]www2.StartVote, len(sv.StartVotes)) + for _, v := range sv.AuthorizeVotes { + auths[v.Token] = v + } + for _, v := range sv.StartVotes { + _, ok := auths[v.Vote.Token] + if !ok { + e := fmt.Sprintf("start vote found without matching authorize vote %v", + v.Vote.Token) + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidRunoffVote, + ErrorContext: []string{e}, + } } } - } - for _, v := range sv.StartVotes { - starts[v.Vote.Token] = v - } - for _, v := range sv.AuthorizeVotes { - _, ok := starts[v.Token] - if !ok { - e := fmt.Sprintf("authorize vote found without matching start vote %v", - v.Token) + for _, v := range sv.StartVotes { + starts[v.Vote.Token] = v + } + for _, v := range sv.AuthorizeVotes { + _, ok := starts[v.Token] + if !ok { + e := fmt.Sprintf("authorize vote found without matching start vote %v", + v.Token) + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidRunoffVote, + ErrorContext: []string{e}, + } + } + } + if len(auths) == 0 { + e := "start votes and authorize votes cannot be empty" return nil, www.UserError{ ErrorCode: www.ErrorStatusInvalidRunoffVote, ErrorContext: []string{e}, } } - } - if len(auths) == 0 { - e := "start votes and authorize votes cannot be empty" - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidRunoffVote, - ErrorContext: []string{e}, - } - } - // Slice used to send event notification for each proposal - // starting a vote, at the end of this function. - var proposalNotifications []*www.ProposalRecord + // Slice used to send event notification for each proposal + // starting a vote, at the end of this function. + var proposalNotifications []*www.ProposalRecord - // Validate authorize votes and start votes - for _, v := range sv.StartVotes { - // Fetch proposal and vote summary - token := v.Vote.Token - if !tokenIsValid(token) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: []string{token}, + // Validate authorize votes and start votes + for _, v := range sv.StartVotes { + // Fetch proposal and vote summary + token := v.Vote.Token + if !tokenIsValid(token) { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidCensorshipToken, + ErrorContext: []string{token}, + } } - } - pr, err := p.getProp(token) - if err != nil { - /* - // TODO + pr, err := p.getProp(token) + if err != nil { if err == cache.ErrRecordNotFound { err = www.UserError{ ErrorCode: www.ErrorStatusProposalNotFound, ErrorContext: []string{token}, } } - */ - return nil, err - } + return nil, err + } - proposalNotifications = append(proposalNotifications, pr) + proposalNotifications = append(proposalNotifications, pr) - vs, err := p.voteSummaryGet(token, bb) - if err != nil { - return nil, err - } + vs, err := p.voteSummaryGet(token, bb) + if err != nil { + return nil, err + } - // Validate authorize vote. The validation function requires a v1 - // AuthorizeVote. This is fine. There is no difference between v1 - // and v2. - av := auths[v.Vote.Token] - av1 := www.AuthorizeVote{ - Token: av.Token, - Action: av.Action, - PublicKey: av.PublicKey, - Signature: av.Signature, - } - err = validateAuthorizeVoteRunoff(av1, *u, *pr, *vs) - if err != nil { - // Attach the token to the error so the user knows which one - // failed. - if ue, ok := err.(*www.UserError); ok { - ue.ErrorContext = append(ue.ErrorContext, token) - err = ue + // Validate authorize vote. The validation function requires a v1 + // AuthorizeVote. This is fine. There is no difference between v1 + // and v2. + av := auths[v.Vote.Token] + av1 := www.AuthorizeVote{ + Token: av.Token, + Action: av.Action, + PublicKey: av.PublicKey, + Signature: av.Signature, + } + err = validateAuthorizeVoteRunoff(av1, *u, *pr, *vs) + if err != nil { + // Attach the token to the error so the user knows which one + // failed. + if ue, ok := err.(*www.UserError); ok { + ue.ErrorContext = append(ue.ErrorContext, token) + err = ue + } + return nil, err } - return nil, err - } - // Validate start vote - err = validateStartVoteRunoff(v, *u, *pr, *vs, - p.cfg.VoteDurationMin, p.cfg.VoteDurationMax) - if err != nil { - // Attach the token to the error so the user knows which one - // failed. - if ue, ok := err.(*www.UserError); ok { - ue.ErrorContext = append(ue.ErrorContext, token) - err = ue + // Validate start vote + err = validateStartVoteRunoff(v, *u, *pr, *vs, + p.cfg.VoteDurationMin, p.cfg.VoteDurationMax) + if err != nil { + // Attach the token to the error so the user knows which one + // failed. + if ue, ok := err.(*www.UserError); ok { + ue.ErrorContext = append(ue.ErrorContext, token) + err = ue + } + return nil, err } - return nil, err } - } - // Validate the RFP proposal - rfp, err := p.getProp(sv.Token) - if err != nil { - /* - // TODO + // Validate the RFP proposal + rfp, err := p.getProp(sv.Token) + if err != nil { if err == cache.ErrRecordNotFound { err = www.UserError{ ErrorCode: www.ErrorStatusProposalNotFound, ErrorContext: []string{sv.Token}, } } - */ - return nil, err - } - switch { - case rfp.LinkBy > time.Now().Unix() && !p.cfg.TestNet: - // Vote cannot start on RFP submissions until the RFP linkby - // deadline has been met. This validation is skipped when on - // testnet. - return nil, www.UserError{ - ErrorCode: www.ErrorStatusLinkByDeadlineNotMet, - } - case len(rfp.LinkedFrom) == 0: - return nil, www.UserError{ - ErrorCode: www.ErrorStatusNoLinkedProposals, - } - } - - // Compile a list of the public, non-abandoned RFP submissions. - // This list will be used to ensure a StartVote exists for each - // of the public, non-abandoned submissions. - linkedFromProps, err := p.getProps(rfp.LinkedFrom) - if err != nil { - return nil, err - } - submissions := make(map[string]bool, len(rfp.LinkedFrom)) // [token]startVoteFound - for _, v := range linkedFromProps { - // Filter out abandoned submissions. These are not allowed - // to be included in a runoff vote. - if v.Status != www.PropStatusPublic { - continue + return nil, err } - - // Set to false for now until we check that a StartVote - // was included for this proposal. - submissions[v.CensorshipRecord.Token] = false - } - - // Verify that a StartVote exists for all public, non-abandoned - // submissions and that there are no extra StartVotes. - for _, v := range sv.StartVotes { - _, ok := submissions[v.Vote.Token] - if !ok { - e := fmt.Sprintf("invalid start vote submission: %v", - v.Vote.Token) + switch { + case rfp.LinkBy > time.Now().Unix() && !p.cfg.TestNet: + // Vote cannot start on RFP submissions until the RFP linkby + // deadline has been met. This validation is skipped when on + // testnet. return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidRunoffVote, - ErrorContext: []string{e}, + ErrorCode: www.ErrorStatusLinkByDeadlineNotMet, } - } - - // A StartVote was included for this proposal - submissions[v.Vote.Token] = true - } - for token, startVoteFound := range submissions { - if !startVoteFound { - e := fmt.Sprintf("missing start vote for rfp submission: %v", - token) + case len(rfp.LinkedFrom) == 0: return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidRunoffVote, - ErrorContext: []string{e}, + ErrorCode: www.ErrorStatusNoLinkedProposals, } } - } - // Setup plugin command - dav := convertAuthorizeVotesV2ToDecred(sv.AuthorizeVotes) - dsv := convertStartVotesV2ToDecred(sv.StartVotes) - payload, err := decredplugin.EncodeStartVoteRunoff( - decredplugin.StartVoteRunoff{ - Token: sv.Token, - AuthorizeVotes: dav, - StartVotes: dsv, - }) - if err != nil { - return nil, err - } - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - pc := pd.PluginCommand{ - Challenge: hex.EncodeToString(challenge), - ID: decredplugin.ID, - Command: decredplugin.CmdStartVoteRunoff, - Payload: string(payload), - } - - // Send plugin command - responseBody, err := p.makeRequest(http.MethodPost, - pd.PluginCommandRoute, pc) - if err != nil { - return nil, err - } - - // Handle response - var reply pd.PluginCommandReply - err = json.Unmarshal(responseBody, &reply) - if err != nil { - return nil, err - } - err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) - if err != nil { - return nil, err - } - dsvr, err := decredplugin.DecodeStartVoteRunoffReply([]byte(reply.Payload)) - if err != nil { - return nil, err - } - svr, err := convertStartVoteReplyV2FromDecred(dsvr.StartVoteReply) - if err != nil { - return nil, err - } - - // Emit event notification for each proposal starting vote - for _, pn := range proposalNotifications { - author, err := p.db.UserGetByPubKey(pn.PublicKey) + // Compile a list of the public, non-abandoned RFP submissions. + // This list will be used to ensure a StartVote exists for each + // of the public, non-abandoned submissions. + linkedFromProps, err := p.getProps(rfp.LinkedFrom) if err != nil { return nil, err } - p.eventManager.emit(eventProposalVoteStarted, - dataProposalVoteStarted{ - token: pn.CensorshipRecord.Token, - name: pn.Name, - adminID: u.ID.String(), - author: *author, - }) - } - - return &www2.StartVoteRunoffReply{ - StartBlockHeight: svr.StartBlockHeight, - StartBlockHash: svr.StartBlockHash, - EndBlockHeight: svr.EndBlockHeight, - EligibleTickets: svr.EligibleTickets, - }, nil -} - -// processTokenInventory returns the tokens of all proposals in the inventory, -// categorized by stage of the voting process. -func (p *politeiawww) processTokenInventory(isAdmin bool) (*www.TokenInventoryReply, error) { - log.Tracef("processTokenInventory") + submissions := make(map[string]bool, len(rfp.LinkedFrom)) // [token]startVoteFound + for _, v := range linkedFromProps { + // Filter out abandoned submissions. These are not allowed + // to be included in a runoff vote. + if v.Status != www.PropStatusPublic { + continue + } - // Prep plugin command to get tokens by vote statuses - var vip piplugin.VoteInventory - payload, err := piplugin.EncodeVoteInventory(vip) - if err != nil { - return nil, err - } + // Set to false for now until we check that a StartVote + // was included for this proposal. + submissions[v.CensorshipRecord.Token] = false + } + + // Verify that a StartVote exists for all public, non-abandoned + // submissions and that there are no extra StartVotes. + for _, v := range sv.StartVotes { + _, ok := submissions[v.Vote.Token] + if !ok { + e := fmt.Sprintf("invalid start vote submission: %v", + v.Vote.Token) + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidRunoffVote, + ErrorContext: []string{e}, + } + } - r, err := p.pluginCommand(piplugin.ID, piplugin.CmdVoteInventory, "", - string(payload)) - if err != nil { - return nil, err - } - vi, err := piplugin.DecodeVoteInventoryReply([]byte(r)) - if err != nil { - return nil, err - } + // A StartVote was included for this proposal + submissions[v.Vote.Token] = true + } + for token, startVoteFound := range submissions { + if !startVoteFound { + e := fmt.Sprintf("missing start vote for rfp submission: %v", + token) + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidRunoffVote, + ErrorContext: []string{e}, + } + } + } - // Translate reply to www - res := www.TokenInventoryReply{ - Pre: append(vi.Unauthorized, vi.Authorized...), - Active: vi.Started, - Approved: vi.Approved, - Rejected: vi.Rejected, - } + // Setup plugin command + dav := convertAuthorizeVotesV2ToDecred(sv.AuthorizeVotes) + dsv := convertStartVotesV2ToDecred(sv.StartVotes) + payload, err := decredplugin.EncodeStartVoteRunoff( + decredplugin.StartVoteRunoff{ + Token: sv.Token, + AuthorizeVotes: dav, + StartVotes: dsv, + }) + if err != nil { + return nil, err + } + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return nil, err + } + pc := pd.PluginCommand{ + Challenge: hex.EncodeToString(challenge), + ID: decredplugin.ID, + Command: decredplugin.CmdStartVoteRunoff, + Payload: string(payload), + } - // Call politeiad to get tokens by record statuses - isReply, err := p.inventoryByStatus() - if err != nil { - return nil, err - } + // Send plugin command + responseBody, err := p.makeRequest(http.MethodPost, + pd.PluginCommandRoute, pc) + if err != nil { + return nil, err + } - // Fill info - res.Abandoned = isReply.Archived + // Handle response + var reply pd.PluginCommandReply + err = json.Unmarshal(responseBody, &reply) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) + if err != nil { + return nil, err + } + dsvr, err := decredplugin.DecodeStartVoteRunoffReply([]byte(reply.Payload)) + if err != nil { + return nil, err + } + svr, err := convertStartVoteReplyV2FromDecred(dsvr.StartVoteReply) + if err != nil { + return nil, err + } - // Add admins only data - if isAdmin { - res.Censored = isReply.Censored - res.Unreviewed = isReply.Unvetted - } + // Emit event notification for each proposal starting vote + for _, pn := range proposalNotifications { + author, err := p.db.UserGetByPubKey(pn.PublicKey) + if err != nil { + return nil, err + } + p.eventManager.emit(eventProposalVoteStarted, + dataProposalVoteStarted{ + token: pn.CensorshipRecord.Token, + name: pn.Name, + adminID: u.ID.String(), + author: *author, + }) + } + + return &www2.StartVoteRunoffReply{ + StartBlockHeight: svr.StartBlockHeight, + StartBlockHash: svr.StartBlockHash, + EndBlockHeight: svr.EndBlockHeight, + EligibleTickets: svr.EligibleTickets, + }, nil + */ - return &res, nil + return nil, nil } // processVoteDetailsV2 returns the vote details for the given proposal token. @@ -1651,3 +1438,51 @@ func (p *politeiawww) processVoteDetailsV2(token string) (*www2.VoteDetailsReply return nil, nil } + +// processTokenInventory returns the tokens of all proposals in the inventory, +// categorized by stage of the voting process. +func (p *politeiawww) processTokenInventory(isAdmin bool) (*www.TokenInventoryReply, error) { + log.Tracef("processTokenInventory") + + // Prep plugin command to get tokens by vote statuses + var vip piplugin.VoteInventory + payload, err := piplugin.EncodeVoteInventory(vip) + if err != nil { + return nil, err + } + + r, err := p.pluginCommand(piplugin.ID, piplugin.CmdVoteInventory, "", + string(payload)) + if err != nil { + return nil, err + } + vi, err := piplugin.DecodeVoteInventoryReply([]byte(r)) + if err != nil { + return nil, err + } + + // Translate reply to www + res := www.TokenInventoryReply{ + Pre: append(vi.Unauthorized, vi.Authorized...), + Active: vi.Started, + Approved: vi.Approved, + Rejected: vi.Rejected, + } + + // Call politeiad to get tokens by record statuses + isReply, err := p.inventoryByStatus() + if err != nil { + return nil, err + } + + // Fill info + res.Abandoned = isReply.Archived + + // Add admins only data + if isAdmin { + res.Censored = isReply.Censored + res.Unreviewed = isReply.Unvetted + } + + return &res, nil +} diff --git a/politeiawww/proposals_test.go b/politeiawww/proposals_test.go index fd4a5adcc..43991568c 100644 --- a/politeiawww/proposals_test.go +++ b/politeiawww/proposals_test.go @@ -2663,7 +2663,7 @@ func TestProcessStartVoteRunoffV2(t *testing.T) { svInvalidToken) // Proposal submission record not found - randomToken, err := util.Random(pd.TokenSize) + randomToken, err := util.Random(pd.TokenSizeMax) if err != nil { t.Fatal(err) } diff --git a/politeiawww/testing.go b/politeiawww/testing.go index c79011306..27e40e381 100644 --- a/politeiawww/testing.go +++ b/politeiawww/testing.go @@ -23,7 +23,6 @@ import ( "github.com/decred/dcrd/chaincfg" "github.com/decred/dcrtime/merkle" - "github.com/decred/politeia/decredplugin" pd "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/api/v1/mime" @@ -423,25 +422,28 @@ func newStartVote(t *testing.T, token string, v uint32, d uint32, vt www2.VoteT, func newStartVoteCmd(t *testing.T, token string, proposalVersion uint32, d uint32, id *identity.FullIdentity) pd.PluginCommand { t.Helper() - sv := newStartVote(t, token, proposalVersion, d, www2.VoteTypeStandard, id) - dsv := convertStartVoteV2ToDecred(sv) - payload, err := decredplugin.EncodeStartVoteV2(dsv) - if err != nil { - t.Fatal(err) - } + /* + sv := newStartVote(t, token, proposalVersion, d, www2.VoteTypeStandard, id) + dsv := convertStartVoteV2ToDecred(sv) + payload, err := decredplugin.EncodeStartVoteV2(dsv) + if err != nil { + t.Fatal(err) + } - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - t.Fatal(err) - } + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + t.Fatal(err) + } - return pd.PluginCommand{ - Challenge: hex.EncodeToString(challenge), - ID: decredplugin.ID, - Command: decredplugin.CmdStartVote, - CommandID: decredplugin.CmdStartVote + " " + sv.Vote.Token, - Payload: string(payload), - } + return pd.PluginCommand{ + Challenge: hex.EncodeToString(challenge), + ID: decredplugin.ID, + Command: decredplugin.CmdStartVote, + CommandID: decredplugin.CmdStartVote + " " + sv.Vote.Token, + Payload: string(payload), + } + */ + return pd.PluginCommand{} } func newStartVoteRunoff(t *testing.T, tk string, avs []www2.AuthorizeVote, svs []www2.StartVote) www2.StartVoteRunoff { @@ -481,25 +483,28 @@ func newAuthorizeVote(t *testing.T, token, version, action string, id *identity. func newAuthorizeVoteCmd(t *testing.T, token, version, action string, id *identity.FullIdentity) pd.PluginCommand { t.Helper() - av := newAuthorizeVote(t, token, version, action, id) - dav := convertAuthorizeVoteToDecred(av) - payload, err := decredplugin.EncodeAuthorizeVote(dav) - if err != nil { - t.Fatal(err) - } + /* + av := newAuthorizeVote(t, token, version, action, id) + dav := convertAuthorizeVoteToDecred(av) + payload, err := decredplugin.EncodeAuthorizeVote(dav) + if err != nil { + t.Fatal(err) + } - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - t.Fatal(err) - } + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + t.Fatal(err) + } - return pd.PluginCommand{ - Challenge: hex.EncodeToString(challenge), - ID: decredplugin.ID, - Command: decredplugin.CmdAuthorizeVote, - CommandID: decredplugin.CmdAuthorizeVote + " " + av.Token, - Payload: string(payload), - } + return pd.PluginCommand{ + Challenge: hex.EncodeToString(challenge), + ID: decredplugin.ID, + Command: decredplugin.CmdAuthorizeVote, + CommandID: decredplugin.CmdAuthorizeVote + " " + av.Token, + Payload: string(payload), + } + */ + return pd.PluginCommand{} } func newProposalRecord(t *testing.T, u *user.User, id *identity.FullIdentity, s www.PropStatusT) www.ProposalRecord { diff --git a/politeiawww/user.go b/politeiawww/user.go index 41ffe75fc..01f81bafb 100644 --- a/politeiawww/user.go +++ b/politeiawww/user.go @@ -1617,47 +1617,6 @@ func processUserProposalCredits(u *user.User) (*www.UserProposalCreditsReply, er }, nil } -// processUserProposals returns a page of proposals for the given user. -func (p *politeiawww) processUserProposals(up *www.UserProposals, isCurrentUser, isAdminUser bool) (*www.UserProposalsReply, error) { - // Verify user exists - _, err := p.userByIDStr(up.UserId) - if err != nil { - return nil, err - } - - // Get a page of user proposals - props, ps, err := p.getUserProps(proposalsFilter{ - After: up.After, - Before: up.Before, - UserID: up.UserId, - StateMap: map[www.PropStateT]bool{ - www.PropStateUnvetted: isCurrentUser || isAdminUser, - www.PropStateVetted: true, - }, - }) - if err != nil { - return nil, err - } - - // Find the number of proposals the user has submitted. This - // number will be different depending on who is requesting it. - // Non-public proposals are included in the calculation when - // an admin or the author is requesting the data. - numProposals := ps.Public + ps.Abandoned - if isCurrentUser || isAdminUser { - numProposals += ps.NotReviewed + ps.UnreviewedChanges + ps.Censored - } - - return &www.UserProposalsReply{ - Proposals: props, - NumOfProposals: numProposals, - }, nil -} - -// -// admin user code follows -// - // _logAdminAction logs a string to the admin log file. // // This function must be called WITH the mutex held. diff --git a/politeiawww/userwww.go b/politeiawww/userwww.go index 0365db577..1e4dd8040 100644 --- a/politeiawww/userwww.go +++ b/politeiawww/userwww.go @@ -727,6 +727,68 @@ func (p *politeiawww) handleRegisterUser(w http.ResponseWriter, r *http.Request) util.RespondWithJSON(w, http.StatusOK, reply) } +// handleSetTOTP handles the setting of TOTP Key +func (p *politeiawww) handleSetTOTP(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleSetTOTP") + + var st www.SetTOTP + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&st); err != nil { + RespondWithError(w, r, 0, "handleSetTOTP: unmarshal", + www.UserError{ + ErrorCode: www.ErrorStatusInvalidInput, + }) + return + } + + u, err := p.getSessionUser(w, r) + if err != nil { + RespondWithError(w, r, 0, + "handleSetTOTP: getSessionUser %v", err) + return + } + + str, err := p.processSetTOTP(st, u) + if err != nil { + RespondWithError(w, r, 0, + "handleSetTOTP: processSetTOTP %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, str) +} + +// handleVerifyTOTP handles the request to verify a set TOTP Key. +func (p *politeiawww) handleVerifyTOTP(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleVerifyTOTP") + + var vt www.VerifyTOTP + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&vt); err != nil { + RespondWithError(w, r, 0, "handleVerifyTOTP: unmarshal", + www.UserError{ + ErrorCode: www.ErrorStatusInvalidInput, + }) + return + } + + u, err := p.getSessionUser(w, r) + if err != nil { + RespondWithError(w, r, 0, + "handleVerifyTOTP: getSessionUser %v", err) + return + } + + vtr, err := p.processVerifyTOTP(vt, u) + if err != nil { + RespondWithError(w, r, 0, + "handleVerifyTOTP: processVerifyTOTP %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, vtr) +} + // setUserWWWRoutes setsup the user routes. func (p *politeiawww) setUserWWWRoutes() { // Public routes @@ -849,6 +911,12 @@ func (p *politeiawww) setCMSUserWWWRoutes() { p.addRoute(http.MethodGet, cms.APIRoute, cms.RouteCMSUsers, p.handleCMSUsers, permissionLogin) + p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, + www.RouteSetTOTP, p.handleSetTOTP, + permissionLogin) + p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, + www.RouteVerifyTOTP, p.handleVerifyTOTP, + permissionLogin) // Routes that require being logged in as an admin user. p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, diff --git a/politeiawww/www.go b/politeiawww/www.go index 922252a38..2e841dd31 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -20,7 +20,6 @@ import ( "runtime/debug" "strings" "syscall" - "text/template" "time" "github.com/decred/politeia/mdstream" @@ -612,7 +611,6 @@ func _main() error { p := &politeiawww{ cfg: loadedCfg, ws: make(map[string]map[string]*wsContext), - templates: make(map[string]*template.Template), eventManager: newEventManager(), // XXX reevaluate where this goes diff --git a/wsdcrdata/wsdcrdata.go b/wsdcrdata/wsdcrdata.go index fbb3ae917..e88d9d46a 100644 --- a/wsdcrdata/wsdcrdata.go +++ b/wsdcrdata/wsdcrdata.go @@ -2,6 +2,8 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. +// Package wsdcrdata provides a client for managing dcrdata websocket +// subscriptions. package wsdcrdata import ( From b3d0a53c3b9ba09e13a5f12b5e22c22d8f19bdf4 Mon Sep 17 00:00:00 2001 From: Amir Massarwa Date: Wed, 23 Sep 2020 01:46:10 +0300 Subject: [PATCH 092/449] politeiawww: Add processVoteStart. --- plugins/ticketvote/ticketvote.go | 2 +- politeiawww/api/pi/v1/v1.go | 52 +++++++++- politeiawww/cmd/piwww/help.go | 4 +- politeiawww/cmd/piwww/piwww.go | 2 +- .../cmd/piwww/{startvote.go => votestart.go} | 53 +++++------ politeiawww/cmd/shared/client.go | 18 ++-- politeiawww/piwww.go | 94 +++++++++++++++++-- politeiawww/ticketvote.go | 24 ++++- 8 files changed, 194 insertions(+), 55 deletions(-) rename politeiawww/cmd/piwww/{startvote.go => votestart.go} (73%) diff --git a/plugins/ticketvote/ticketvote.go b/plugins/ticketvote/ticketvote.go index a0b9eec9e..2da014cf5 100644 --- a/plugins/ticketvote/ticketvote.go +++ b/plugins/ticketvote/ticketvote.go @@ -310,7 +310,7 @@ func EncodeStartReply(sr StartReply) ([]byte, error) { } // DecodeStartReply decodes a JSON byte slice into a StartReply. -func DecodeStartReplyVote(payload []byte) (*StartReply, error) { +func DecodeStartReply(payload []byte) (*StartReply, error) { var sr StartReply err := json.Unmarshal(payload, &sr) if err != nil { diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 39f5f1485..1617cad58 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -365,7 +365,7 @@ type ProposalsReply struct { Proposals map[string]ProposalRecord `json:"proposals"` // [token]Proposal } -// ProposalInventry retrieves the tokens of all proposals in the inventory, +// ProposalInventory retrieves the tokens of all proposals in the inventory, // catagorized by proposal status and ordered by timestamp of the status change // from newest to oldest. type ProposalInventory struct{} @@ -537,8 +537,54 @@ type VoteAuthorizeReply struct { Receipt string `json:"receipt"` } -type VoteStart struct{} -type VoteStartReply struct{} +// VoteOption describes a single vote option. +type VoteOption struct { + ID string `json:"id"` // Single, unique word (e.g. yes) + Description string `json:"description"` // Longer description of the vote + Bit uint64 `json:"bit"` // Bit used for this option +} + +// VoteDetails includes all data required by server to start vote on a +// proposal. +type VoteDetails struct { + Token string `json:"token"` // Proposal token + Version uint32 `json:"version"` // Proposal version + Type VoteT `json:"type"` // Vote type + Mask uint64 `json:"mask"` // Valid vote bits + Duration uint32 `json:"duration"` // Duration in blocks + + // QuorumPercentage is the percent of elligible votes required for + // the vote to meet a quorum. + QuorumPercentage uint32 `json:"quorumpercentage"` + + // PassPercentage is the percent of total votes that are required + // to consider a vote option as passed. + PassPercentage uint32 `json:"passpercentage"` + + Options []VoteOption `json:"options"` +} + +// VoteStart starts a proposal vote. All proposal votes must be authorized +// by the proposal author before an admin is able to start the voting process. +// +// Signature is the signature of a SHA256 digest of the JSON +// encoded Vote structure. +type VoteStart struct { + Vote VoteDetails `json:"vote"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` +} + +// VoteStartReply is the reply to the VoteStart command. +// +// Receipt is the server signature of the client signature. This is proof that +// the server received and processed the VoteStart command. +type VoteStartReply struct { + StartBlockHeight uint32 `json:"startblockheight"` + StartBlockHash string `json:"startblockhash"` + EndBlockHeight uint32 `json:"endblockheight"` + EligibleTickets []string `json:"eligibletickets"` +} type VoteStartRunoff struct{} type VoteStartRunoffReply struct{} diff --git a/politeiawww/cmd/piwww/help.go b/politeiawww/cmd/piwww/help.go index fe482ed52..e52391941 100644 --- a/politeiawww/cmd/piwww/help.go +++ b/politeiawww/cmd/piwww/help.go @@ -101,8 +101,8 @@ func (cmd *HelpCmd) Execute(args []string) error { // Vote commands case "voteauthorize": fmt.Printf("%s\n", voteAuthorizeHelpMsg) - case "startvote": - fmt.Printf("%s\n", startVoteHelpMsg) + case "votestart": + fmt.Printf("%s\n", voteStartHelpMsg) case "startvoterunoff": fmt.Printf("%s\n", startVoteRunoffHelpMsg) case "voteresults": diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index 8ae89d7d5..52ff38f6b 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -57,6 +57,7 @@ type piwww struct { CommentVotes CommentVotesCmd `command:"commentvotes" description:"(user) get comment upvotes/downvotes for a proposal from the provided user"` // Vote commands + VoteStart VoteStartCmd `command:"votestart" description:"(admin) start the voting period on a proposal"` VoteAuthorize VoteAuthorizeCmd `command:"voteauthorize" description:"(user) authorize a proposal vote (must be proposal author)"` // Commands @@ -82,7 +83,6 @@ type piwww struct { Secret shared.SecretCmd `command:"secret" description:"(user) ping politeiawww"` SendFaucetTx SendFaucetTxCmd `command:"sendfaucettx" description:" send a DCR transaction using the Decred testnet faucet"` SetTOTP shared.SetTOTPCmd `command:"settotp" description:"(user) set the key for TOTP"` - StartVote StartVoteCmd `command:"startvote" description:"(admin) start the voting period on a proposal"` StartVoteRunoff StartVoteRunoffCmd `command:"startvoterunoff" description:"(admin) start a runoff using the submissions to an RFP"` Subscribe SubscribeCmd `command:"subscribe" description:"(public) subscribe to all websocket commands and do not exit tool"` Tally TallyCmd `command:"tally" description:"(public) get the vote tally for a proposal"` diff --git a/politeiawww/cmd/piwww/startvote.go b/politeiawww/cmd/piwww/votestart.go similarity index 73% rename from politeiawww/cmd/piwww/startvote.go rename to politeiawww/cmd/piwww/votestart.go index 48136488b..e099e0e39 100644 --- a/politeiawww/cmd/piwww/startvote.go +++ b/politeiawww/cmd/piwww/votestart.go @@ -10,17 +10,17 @@ import ( "strconv" "github.com/decred/politeia/decredplugin" - v2 "github.com/decred/politeia/politeiawww/api/www/v2" + pi "github.com/decred/politeia/politeiawww/api/pi/v1" "github.com/decred/politeia/politeiawww/cmd/shared" "github.com/decred/politeia/util" ) -// StartVoteCmd starts the voting period on the specified proposal. +// VoteStartCmd starts the voting period on the specified proposal. // // The QuorumPercentage and PassPercentage are strings and not uint32 so that a // value of 0 can be passed in and not be overwritten by the defaults. This is // sometimes desirable when testing. -type StartVoteCmd struct { +type VoteStartCmd struct { Args struct { Token string `positional-arg-name:"token" required:"true"` Duration uint32 `positional-arg-name:"duration"` @@ -30,13 +30,13 @@ type StartVoteCmd struct { } // Execute executes the start vote command. -func (cmd *StartVoteCmd) Execute(args []string) error { +func (cmd *VoteStartCmd) Execute(args []string) error { // Check for user identity if cfg.Identity == nil { return shared.ErrUserIdentityNotFound } - // Get proposal version. This is needed for the VoteV2. + // Get proposal version. This is needed for the pi route. pdr, err := client.ProposalDetails(cmd.Args.Token, nil) if err != nil { return err @@ -71,63 +71,64 @@ func (cmd *StartVoteCmd) Execute(args []string) error { pass = uint32(i) } - // Create StartVote - vote := v2.Vote{ + // Create VoteStart + vote := pi.VoteDetails{ Token: cmd.Args.Token, - ProposalVersion: uint32(version), - Type: v2.VoteTypeStandard, + Version: uint32(version), + Type: pi.VoteTypeStandard, Mask: 0x03, // bit 0 no, bit 1 yes Duration: duration, QuorumPercentage: quorum, PassPercentage: pass, - Options: []v2.VoteOption{ + Options: []pi.VoteOption{ { - Id: decredplugin.VoteOptionIDApprove, + ID: decredplugin.VoteOptionIDApprove, Description: "Approve proposal", - Bits: 0x01, + Bit: 0x01, }, { - Id: decredplugin.VoteOptionIDReject, + ID: decredplugin.VoteOptionIDReject, Description: "Don't approve proposal", - Bits: 0x02, + Bit: 0x02, }, }, } + vb, err := json.Marshal(vote) if err != nil { return err } msg := hex.EncodeToString(util.Digest(vb)) sig := cfg.Identity.SignMessage([]byte(msg)) - sv := v2.StartVote{ + vs := pi.VoteStart{ Vote: vote, PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), Signature: hex.EncodeToString(sig[:]), } // Print request details - err = shared.PrintJSON(sv) + err = shared.PrintJSON(vs) if err != nil { return err } // Send request - svr, err := client.StartVoteV2(sv) + vsr, err := client.VoteStart(vs) if err != nil { return err } // Remove ticket snapshot from the response so that the output // is legible - svr.EligibleTickets = []string{"removed by piwww for readability"} + vsr.EligibleTickets = []string{"removed by piwww for readability"} // Print response details - return shared.PrintJSON(svr) + return shared.PrintJSON(vsr) } -// startVoteHelpMsg is the output of the help command when 'startvote' is +// voteStartHelpMsg is the output of the help command when 'votestart' is // specified. -var startVoteHelpMsg = `startvote +var voteStartHelpMsg = `votestart Start voting period for a proposal. Requires admin privileges. @@ -140,12 +141,4 @@ Arguments: 2. duration (uint32, optional) Duration of vote in blocks (default: 2016) 3. quorumpercentage (string, optional) Percent of votes required for quorum (default: 10) 4. passpercentage (string, optional) Percent of votes required to pass (default: 60) - -Result: - -{ - "startblockheight" (string) Block height at start of vote - "startblockhash" (string) Hash of first block of vote interval - "endheight" (string) Height of vote end - "eligibletickets" ([]string) Valid voting tickets -}` +` diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index 09c54cf19..9dc6c6020 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -1329,29 +1329,29 @@ func (c *Client) WWWCensorComment(cc *www.CensorComment) (*www.CensorCommentRepl return &ccr, nil } -// StartVoteV2 sends the provided v2 StartVote to the politeiawww backend. -func (c *Client) StartVoteV2(sv www2.StartVote) (*www2.StartVoteReply, error) { +// VoteStart sends the provided VoteStart to pi. +func (c *Client) VoteStart(vs pi.VoteStart) (*pi.VoteStartReply, error) { responseBody, err := c.makeRequest(http.MethodPost, - www2.APIRoute, www2.RouteStartVote, sv) + pi.APIRoute, pi.RouteVoteStart, vs) if err != nil { return nil, err } - var svr www2.StartVoteReply - err = json.Unmarshal(responseBody, &svr) + var vsr pi.VoteStartReply + err = json.Unmarshal(responseBody, &vsr) if err != nil { - return nil, fmt.Errorf("unmarshal StartVoteReply: %v", err) + return nil, fmt.Errorf("unmarshal VoteStartReply: %v", err) } if c.cfg.Verbose { - svr.EligibleTickets = []string{"removed by piwww for readability"} - err := prettyPrintJSON(svr) + vsr.EligibleTickets = []string{"removed by piwww for readability"} + err := prettyPrintJSON(vsr) if err != nil { return nil, err } } - return &svr, nil + return &vsr, nil } // StartVoteRunoffV2 sends the given StartVoteRunoff to the politeiawww v2 diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 50a2de7b2..f52ae3867 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -1460,7 +1460,7 @@ func (p *politeiawww) processCommentNew(cn pi.CommentNew, usr user.User) (*pi.Co } } - // Call pi plugin to add new comment + // Call the pi plugin to add new comment reply, err := p.piCommentNew(piplugin.CommentNew{ UUID: usr.ID.String(), Token: cn.Token, @@ -1552,7 +1552,7 @@ func (p *politeiawww) processCommentVote(cv pi.CommentVote, usr user.User) (*pi. } } - // Call pi plugin to add new comment + // Call the pi plugin to add new comment reply, err := p.piCommentVote(piplugin.CommentVote{ UUID: usr.ID.String(), Token: cv.Token, @@ -1604,7 +1604,7 @@ func (p *politeiawww) handleCommentVote(w http.ResponseWriter, r *http.Request) func (p *politeiawww) processComments(c pi.Comments) (*pi.CommentsReply, error) { log.Tracef("processComments: %v", c.Token) - // Call comments plugin to get comments + // Call the comments plugin to get comments reply, err := p.comments(comments.GetAll{ Token: c.Token, State: convertCommentsPluginPropStateFromPi(c.State), @@ -1663,7 +1663,7 @@ func (p *politeiawww) handleComments(w http.ResponseWriter, r *http.Request) { func (p *politeiawww) processCommentVotes(cvs pi.CommentVotes) (*pi.CommentVotesReply, error) { log.Tracef("processCommentVotes: %v %v", cvs.Token, cvs.UserID) - // Call comments plugin to get filtered record's comment votes + // Call the comments plugin to get filtered record's comment votes reply, err := p.commentVotes(comments.Votes{ Token: cvs.Token, State: convertCommentsPluginPropStateFromPi(cvs.State), @@ -1722,7 +1722,7 @@ func (p *politeiawww) processCommentCensor(cc pi.CommentCensor, usr user.User) ( } } - // Call comments plugin to censor comment + // Call the comments plugin to censor comment reply, err := p.commentCensor(comments.Del{ State: convertCommentsPluginPropStateFromPi(cc.State), Token: cc.Token, @@ -1784,8 +1784,8 @@ func convertVoteAuthActionFromPi(a pi.VoteAuthActionT) ticketvote.AuthActionT { func (p *politeiawww) processVoteAuthorize(va pi.VoteAuthorize) (*pi.VoteAuthorizeReply, error) { log.Tracef("processVoteAuthorize: %v", va.Token) - // Call ticketvote plugin to authorize vote - reply, err := p.authorizeVote(ticketvote.Authorize{ + // Call the ticketvote plugin to authorize vote + reply, err := p.voteAuthorize(ticketvote.Authorize{ Token: va.Token, Version: va.Version, Action: convertVoteAuthActionFromPi(va.Action), @@ -1824,6 +1824,83 @@ func (p *politeiawww) handleVoteAuthorize(w http.ResponseWriter, r *http.Request util.RespondWithJSON(w, http.StatusOK, vr) } +func convertVoteTypeFromPi(t pi.VoteT) ticketvote.VoteT { + switch t { + case pi.VoteTypeStandard: + return ticketvote.VoteTypeStandard + case pi.VoteTypeRunoff: + return ticketvote.VoteTypeRunoff + } + return ticketvote.VoteTypeInvalid +} + +func convertVoteDetailsFromPi(v pi.VoteDetails) ticketvote.VoteDetails { + tv := ticketvote.VoteDetails{ + Token: v.Token, + Version: v.Version, + Type: convertVoteTypeFromPi(v.Type), + Mask: v.Mask, + Duration: v.Duration, + QuorumPercentage: v.QuorumPercentage, + PassPercentage: v.PassPercentage, + } + // Convert vote options + vo := make([]ticketvote.VoteOption, 0, len(v.Options)) + for _, vi := range v.Options { + vo = append(vo, ticketvote.VoteOption{ + ID: vi.ID, + Description: vi.Description, + Bit: vi.Bit, + }) + } + tv.Options = vo + + return tv +} + +func (p *politeiawww) processVoteStart(vs pi.VoteStart) (*pi.VoteStartReply, error) { + log.Tracef("processVoteStart: %v", vs.Vote.Token) + + // Call the ticketvote plugin to start vote + reply, err := p.voteStart(ticketvote.Start{ + Vote: convertVoteDetailsFromPi(vs.Vote), + PublicKey: vs.PublicKey, + Signature: vs.Signature, + }) + if err != nil { + return nil, err + } + + return &pi.VoteStartReply{ + StartBlockHeight: reply.StartBlockHeight, + StartBlockHash: reply.StartBlockHash, + EndBlockHeight: reply.EndBlockHeight, + EligibleTickets: reply.EligibleTickets, + }, nil +} + +func (p *politeiawww) handleVoteStart(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleVoteStart") + + var vs pi.VoteStart + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&vs); err != nil { + respondWithPiError(w, r, "handleVoteStart: unmarshal", + pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusInvalidInput, + }) + return + } + + vsr, err := p.processVoteStart(vs) + if err != nil { + respondWithPiError(w, r, + "handleVoteStart: processVoteStart: %v", err) + } + + util.RespondWithJSON(w, http.StatusOK, vsr) +} + func (p *politeiawww) setPiRoutes() { // Public routes p.addRoute(http.MethodGet, pi.APIRoute, @@ -1836,6 +1913,9 @@ func (p *politeiawww) setPiRoutes() { p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteVoteAuthorize, p.handleVoteAuthorize, permissionPublic) + p.addRoute(http.MethodPost, pi.APIRoute, + pi.RouteVoteStart, p.handleVoteStart, permissionPublic) + // Logged in routes p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteProposalNew, p.handleProposalNew, diff --git a/politeiawww/ticketvote.go b/politeiawww/ticketvote.go index b64098783..c17e095df 100644 --- a/politeiawww/ticketvote.go +++ b/politeiawww/ticketvote.go @@ -9,8 +9,28 @@ import ( www "github.com/decred/politeia/politeiawww/api/www/v1" ) -func (p *politeiawww) authorizeVote(a ticketvote.Authorize) (*ticketvote.AuthorizeReply, error) { - // Prop plugin command +func (p *politeiawww) voteStart(vs ticketvote.Start) (*ticketvote.StartReply, error) { + // Prep plugin command + payload, err := ticketvote.EncodeStart(vs) + if err != nil { + return nil, err + } + + r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdStart, "", + string(payload)) + if err != nil { + return nil, err + } + vsr, err := ticketvote.DecodeStartReply([]byte(r)) + if err != nil { + return nil, err + } + + return vsr, nil +} + +func (p *politeiawww) voteAuthorize(a ticketvote.Authorize) (*ticketvote.AuthorizeReply, error) { + // Prep plugin command payload, err := ticketvote.EncodeAuthorize(a) if err != nil { return nil, err From bd0e93a1315f6255808dd935257952e3023704b7 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 22 Sep 2020 19:43:48 -0500 Subject: [PATCH 093/449] move ticketvote plugin to tlogbe --- plugins/ticketvote/ticketvote.go | 101 ++-- politeiad/backend/tlogbe/comments.go | 2 +- .../tlogbe/{plugins => }/ticketvote.go | 512 ++++++++---------- 3 files changed, 276 insertions(+), 339 deletions(-) rename politeiad/backend/tlogbe/{plugins => }/ticketvote.go (81%) diff --git a/plugins/ticketvote/ticketvote.go b/plugins/ticketvote/ticketvote.go index 2da014cf5..d355bdc3d 100644 --- a/plugins/ticketvote/ticketvote.go +++ b/plugins/ticketvote/ticketvote.go @@ -17,6 +17,9 @@ type VoteT int type VoteErrorT int type ErrorStatusT int +// TODO StartDetails, StartReply, StartRunoffReply should contain a receipt. +// The receipt should be the server signature of Signature+StartBlockHash. + const ( ID = "ticketvote" @@ -150,9 +153,36 @@ func (e UserErrorReply) Error() string { return fmt.Sprintf("ticketvote plugin error code: %v", e.ErrorCode) } -// AuthorizeVote is the structure that is saved to disk when a vote is -// authorized or a previous authorization is revoked. -type AuthorizeVote struct { +// VoteOption describes a single vote option. +type VoteOption struct { + ID string `json:"id"` // Single, unique word (e.g. yes) + Description string `json:"description"` // Longer description of the vote + Bit uint64 `json:"bit"` // Bit used for this option +} + +// VoteDetails describes the options and parameters of a ticket vote. +type VoteDetails struct { + Token string `json:"token"` // Record token + Version uint32 `json:"version"` // Record version + Type VoteT `json:"type"` // Vote type + Mask uint64 `json:"mask"` // Valid vote bits + Duration uint32 `json:"duration"` // Duration in blocks + + // QuorumPercentage is the percent of elligible votes required for + // the vote to meet a quorum. + QuorumPercentage uint32 `json:"quorumpercentage"` + + // PassPercentage is the percent of total votes that are required + // to consider a vote option as passed. + PassPercentage uint32 `json:"passpercentage"` + + Options []VoteOption `json:"options"` +} + +// AuthorizeDetails is the structure that is saved to disk when a vote is +// authorized or a previous authorization is revoked. It contains all the +// fields from a Authorize and a AuthorizeReply. +type AuthorizeDetails struct { // Data generated by client Token string `json:"token"` // Record token Version uint32 `json:"version"` // Record version @@ -165,16 +195,18 @@ type AuthorizeVote struct { Receipt string `json:"receipt"` // Server signature of client signature } -// StartVote is the structure that is saved to disk when a vote is started. +// StartDetails is the structure that is saved to disk when a vote is started. +// It contains all of the fields from a Start and a StartReply. +// +// Signature is the client signature of the SHA256 digest of the JSON encoded +// Vote struct. +// // TODO does this need a receipt? -type StartVote struct { +type StartDetails struct { // Data generated by client Vote VoteDetails `json:"vote"` PublicKey string `json:"publickey"` - - // Signature is the client signature of the SHA256 digest of the - // JSON encoded Vote struct. - Signature string `json:"signature"` + Signature string `json:"signature"` // Metadata generated by server StartBlockHeight uint32 `json:"startblockheight"` @@ -183,11 +215,12 @@ type StartVote struct { EligibleTickets []string `json:"eligibletickets"` // Ticket hashes } -// CastVote is the structure that is saved to disk when a vote is cast. +// CastVoteDetails is the structure that is saved to disk when a vote is cast. +// // TODO VoteOption.Bit is a uint64, but the CastVote.VoteBit is a string in // decredplugin. Do we want to make them consistent or was that done on // purpose? It was probably done that way so that way for the signature. -type CastVote struct { +type CastVoteDetails struct { // Data generated by client Token string `json:"token"` // Record token Ticket string `json:"ticket"` // Ticket hash @@ -245,40 +278,14 @@ func DecodeAuthorizeReply(payload []byte) (*AuthorizeReply, error) { return &ar, nil } -// VoteOption describes a single vote option. -type VoteOption struct { - ID string `json:"id"` // Single, unique word (e.g. yes) - Description string `json:"description"` // Longer description of the vote - Bit uint64 `json:"bit"` // Bit used for this option -} - -// VoteDetails describes the options and parameters of a ticket vote. -type VoteDetails struct { - Token string `json:"token"` // Record token - Version uint32 `json:"version"` // Record version - Type VoteT `json:"type"` // Vote type - Mask uint64 `json:"mask"` // Valid vote bits - Duration uint32 `json:"duration"` // Duration in blocks - - // QuorumPercentage is the percent of elligible votes required for - // the vote to meet a quorum. - QuorumPercentage uint32 `json:"quorumpercentage"` - - // PassPercentage is the percent of total votes that are required - // to consider a vote option as passed. - PassPercentage uint32 `json:"passpercentage"` - - Options []VoteOption `json:"options"` -} - // Start starts a ticket vote. +// +// Signature is the signature of a SHA256 digest of the JSON encoded Vote +// structure. type Start struct { - Vote VoteDetails `json:"vote"` // Vote options and params + Vote VoteDetails `json:"vote"` PublicKey string `json:"publickey"` // Public key used for signature - - // Signature is the signature of a SHA256 digest of the JSON - // encoded Vote structure. - Signature string `json:"signature"` + Signature string `json:"signature"` // Client signature } // EncodeStart encodes a Start into a JSON byte slice. @@ -297,6 +304,8 @@ func DecodeStart(payload []byte) (*Start, error) { } // StartReply is the reply to the Start command. +// +// TODO should this return a receipt? type StartReply struct { StartBlockHeight uint32 `json:"startblockheight"` StartBlockHash string `json:"startblockhash"` @@ -343,6 +352,8 @@ func DecodeStartRunoff(payload []byte) (*StartRunoff, error) { } // StartRunoffReply is the reply to the StartRunoff command. +// +// TODO should this return a receipt? type StartRunoffReply struct { StartBlockHeight uint32 `json:"startblockheight"` StartBlockHash string `json:"startblockhash"` @@ -448,8 +459,8 @@ func DecodeDetails(payload []byte) (*Details, error) { // DetailsReply is the reply to the Details command. type DetailsReply struct { - Auths []AuthorizeVote `json:"auths"` - Vote *StartVote `json:"vote"` + Auths []AuthorizeDetails `json:"auths"` + StartDetails *StartDetails `json:"startdetails"` } // EncodeDetailsReply encodes a DetailsReply into a JSON byte slice. @@ -489,7 +500,7 @@ func DecodeCastVotes(payload []byte) (*CastVotes, error) { // CastVotesReply is the rely to the CastVotes command. type CastVotesReply struct { - Votes []CastVote `json:"votes"` + Votes []CastVoteDetails `json:"votes"` } // EncodeCastVotesReply encodes a CastVotesReply into a JSON byte slice. diff --git a/politeiad/backend/tlogbe/comments.go b/politeiad/backend/tlogbe/comments.go index a2498e0a6..4a4abfb25 100644 --- a/politeiad/backend/tlogbe/comments.go +++ b/politeiad/backend/tlogbe/comments.go @@ -1233,7 +1233,7 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { UserID: v.UserID, Token: v.Token, CommentID: v.CommentID, - Vote: int64(v.Vote), + Vote: v.Vote, PublicKey: v.PublicKey, Signature: v.Signature, Timestamp: time.Now().Unix(), diff --git a/politeiad/backend/tlogbe/plugins/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go similarity index 81% rename from politeiad/backend/tlogbe/plugins/ticketvote.go rename to politeiad/backend/tlogbe/ticketvote.go index a4b5e7f57..13e448302 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package plugins +package tlogbe import ( "bytes" @@ -28,14 +28,10 @@ import ( "github.com/decred/politeia/plugins/ticketvote" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe" "github.com/decred/politeia/politeiad/backend/tlogbe/store" "github.com/decred/politeia/util" ) -// TODO don't save data to the file system. Save it to the kv store and save -// the key to the file system. This will allow the data to be backed up. - const ( // ticketVoteDirname is the ticket vote data directory name. ticketVoteDirname = "ticketvote" @@ -46,28 +42,29 @@ const ( filenameSummary = "{token}-summary.json" // Blob entry data descriptors - dataDescriptorAuthorizeVote = "authorizevote" - dataDescriptorStartVote = "startvote" - dataDescriptorCastVote = "castvote" + dataDescriptorAuthorizeDetails = "authorizedetails" + dataDescriptorStartDetails = "startdetails" + dataDescriptorCastVoteDetails = "castvotedetails" // Prefixes that are appended to key-value store keys before // storing them in the log leaf ExtraData field. - keyPrefixAuthorizeVote = "authorizevote:" - keyPrefixStartVote = "startvote:" - keyPrefixCastVote = "castvote:" + keyPrefixAuthorizeDetails = "authorizedetails:" + keyPrefixStartDetails = "startdetails:" + keyPrefixCastVoteDetails = "castvotedetails:" ) var ( - _ tlogbe.Plugin = (*ticketVotePlugin)(nil) - - // Local errors - errRecordNotFound = errors.New("record not found") + _ pluginClient = (*ticketVotePlugin)(nil) ) -// ticketVotePlugin satisfies the Plugin interface. +// TODO holding the lock before verifying the token can allow the mutexes to +// be spammed. Create an infinite amount of them with invalid tokens. + +// ticketVotePlugin satisfies the pluginClient interface. type ticketVotePlugin struct { sync.Mutex - backend *tlogbe.TlogBackend + backend backend.Backend + tlog tlogClient // Plugin settings id *identity.FullIdentity @@ -143,12 +140,12 @@ func (p *ticketVotePlugin) cachedInventorySet(inv inventory) { p.inv = inv } -func (p *ticketVotePlugin) cachedVotes(token string) map[string]string { +func (p *ticketVotePlugin) cachedVotes(token []byte) map[string]string { p.Lock() defer p.Unlock() // Return a copy of the map - cv, ok := p.votes[token] + cv, ok := p.votes[hex.EncodeToString(token)] if !ok { return map[string]string{} } @@ -262,7 +259,7 @@ func convertTicketVoteErrFromSignatureErr(err error) ticketvote.UserErrorReply { } } -func convertAuthorizeVoteFromBlobEntry(be store.BlobEntry) (*ticketvote.AuthorizeVote, error) { +func convertAuthorizeDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.AuthorizeDetails, error) { // Decode and validate data hint b, err := base64.StdEncoding.DecodeString(be.DataHint) if err != nil { @@ -273,9 +270,9 @@ func convertAuthorizeVoteFromBlobEntry(be store.BlobEntry) (*ticketvote.Authoriz if err != nil { return nil, fmt.Errorf("unmarshal DataHint: %v", err) } - if dd.Descriptor != dataDescriptorAuthorizeVote { + if dd.Descriptor != dataDescriptorAuthorizeDetails { return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorAuthorizeVote) + dd.Descriptor, dataDescriptorAuthorizeDetails) } // Decode data @@ -291,16 +288,16 @@ func convertAuthorizeVoteFromBlobEntry(be store.BlobEntry) (*ticketvote.Authoriz return nil, fmt.Errorf("data is not coherent; got %x, want %x", util.Digest(b), hash) } - var av ticketvote.AuthorizeVote - err = json.Unmarshal(b, &av) + var ad ticketvote.AuthorizeDetails + err = json.Unmarshal(b, &ad) if err != nil { - return nil, fmt.Errorf("unmarshal AuthorizeVote: %v", err) + return nil, fmt.Errorf("unmarshal AuthorizeDetails: %v", err) } - return &av, nil + return &ad, nil } -func convertStartVoteFromBlobEntry(be store.BlobEntry) (*ticketvote.StartVote, error) { +func convertStartDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.StartDetails, error) { // Decode and validate data hint b, err := base64.StdEncoding.DecodeString(be.DataHint) if err != nil { @@ -311,9 +308,9 @@ func convertStartVoteFromBlobEntry(be store.BlobEntry) (*ticketvote.StartVote, e if err != nil { return nil, fmt.Errorf("unmarshal DataHint: %v", err) } - if dd.Descriptor != dataDescriptorStartVote { + if dd.Descriptor != dataDescriptorStartDetails { return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorStartVote) + dd.Descriptor, dataDescriptorStartDetails) } // Decode data @@ -329,16 +326,16 @@ func convertStartVoteFromBlobEntry(be store.BlobEntry) (*ticketvote.StartVote, e return nil, fmt.Errorf("data is not coherent; got %x, want %x", util.Digest(b), hash) } - var sv ticketvote.StartVote - err = json.Unmarshal(b, &sv) + var sd ticketvote.StartDetails + err = json.Unmarshal(b, &sd) if err != nil { - return nil, fmt.Errorf("unmarshal StartVote: %v", err) + return nil, fmt.Errorf("unmarshal StartDetails: %v", err) } - return &sv, nil + return &sd, nil } -func convertCastVoteFromBlobEntry(be store.BlobEntry) (*ticketvote.CastVote, error) { +func convertCastVoteDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.CastVoteDetails, error) { // Decode and validate data hint b, err := base64.StdEncoding.DecodeString(be.DataHint) if err != nil { @@ -349,9 +346,9 @@ func convertCastVoteFromBlobEntry(be store.BlobEntry) (*ticketvote.CastVote, err if err != nil { return nil, fmt.Errorf("unmarshal DataHint: %v", err) } - if dd.Descriptor != dataDescriptorCastVote { + if dd.Descriptor != dataDescriptorCastVoteDetails { return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorCastVote) + dd.Descriptor, dataDescriptorCastVoteDetails) } // Decode data @@ -367,50 +364,50 @@ func convertCastVoteFromBlobEntry(be store.BlobEntry) (*ticketvote.CastVote, err return nil, fmt.Errorf("data is not coherent; got %x, want %x", util.Digest(b), hash) } - var cv ticketvote.CastVote + var cv ticketvote.CastVoteDetails err = json.Unmarshal(b, &cv) if err != nil { - return nil, fmt.Errorf("unmarshal CastVote: %v", err) + return nil, fmt.Errorf("unmarshal CastVoteDetails: %v", err) } return &cv, nil } -func convertBlobEntryFromAuthorizeVote(av ticketvote.AuthorizeVote) (*store.BlobEntry, error) { - data, err := json.Marshal(av) +func convertBlobEntryFromAuthorizeDetails(ad ticketvote.AuthorizeDetails) (*store.BlobEntry, error) { + data, err := json.Marshal(ad) if err != nil { return nil, err } hint, err := json.Marshal( store.DataDescriptor{ Type: store.DataTypeStructure, - Descriptor: dataDescriptorAuthorizeVote, + Descriptor: dataDescriptorAuthorizeDetails, }) if err != nil { return nil, err } - be := store.BlobEntryNew(hint, data) + be := store.NewBlobEntry(hint, data) return &be, nil } -func convertBlobEntryFromStartVote(sv ticketvote.StartVote) (*store.BlobEntry, error) { - data, err := json.Marshal(sv) +func convertBlobEntryFromStartDetails(sd ticketvote.StartDetails) (*store.BlobEntry, error) { + data, err := json.Marshal(sd) if err != nil { return nil, err } hint, err := json.Marshal( store.DataDescriptor{ Type: store.DataTypeStructure, - Descriptor: dataDescriptorStartVote, + Descriptor: dataDescriptorStartDetails, }) if err != nil { return nil, err } - be := store.BlobEntryNew(hint, data) + be := store.NewBlobEntry(hint, data) return &be, nil } -func convertBlobEntryFromCastVote(cv ticketvote.CastVote) (*store.BlobEntry, error) { +func convertBlobEntryFromCastVoteDetails(cv ticketvote.CastVoteDetails) (*store.BlobEntry, error) { data, err := json.Marshal(cv) if err != nil { return nil, err @@ -418,18 +415,23 @@ func convertBlobEntryFromCastVote(cv ticketvote.CastVote) (*store.BlobEntry, err hint, err := json.Marshal( store.DataDescriptor{ Type: store.DataTypeStructure, - Descriptor: dataDescriptorCastVote, + Descriptor: dataDescriptorCastVoteDetails, }) if err != nil { return nil, err } - be := store.BlobEntryNew(hint, data) + be := store.NewBlobEntry(hint, data) return &be, nil } -func authorizeVoteSave(client *tlogbe.RecordClient, av ticketvote.AuthorizeVote) error { +func (p *ticketVotePlugin) authorizeSave(ad ticketvote.AuthorizeDetails) error { + token, err := hex.DecodeString(ad.Token) + if err != nil { + return err + } + // Prepare blob - be, err := convertBlobEntryFromAuthorizeVote(av) + be, err := convertBlobEntryFromAuthorizeDetails(ad) if err != nil { return err } @@ -443,10 +445,10 @@ func authorizeVoteSave(client *tlogbe.RecordClient, av ticketvote.AuthorizeVote) } // Save blob - merkles, err := client.Save(keyPrefixAuthorizeVote, + merkles, err := p.tlog.save(tlogIDVetted, token, keyPrefixAuthorizeDetails, [][]byte{b}, [][]byte{h}, false) if err != nil { - return fmt.Errorf("Save: %v", err) + return fmt.Errorf("save: %v", err) } if len(merkles) != 1 { return fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", @@ -456,21 +458,22 @@ func authorizeVoteSave(client *tlogbe.RecordClient, av ticketvote.AuthorizeVote) return nil } -func authorizeVotes(client *tlogbe.RecordClient) ([]ticketvote.AuthorizeVote, error) { +func (p *ticketVotePlugin) authorizes(token []byte) ([]ticketvote.AuthorizeDetails, error) { // Retrieve blobs - blobs, err := client.BlobsByKeyPrefix(keyPrefixAuthorizeVote) + blobs, err := p.tlog.blobsByKeyPrefix(tlogIDVetted, token, + keyPrefixAuthorizeDetails) if err != nil { return nil, err } // Decode blobs - auths := make([]ticketvote.AuthorizeVote, 0, len(blobs)) + auths := make([]ticketvote.AuthorizeDetails, 0, len(blobs)) for _, v := range blobs { be, err := store.Deblob(v) if err != nil { return nil, err } - a, err := convertAuthorizeVoteFromBlobEntry(*be) + a, err := convertAuthorizeDetailsFromBlobEntry(*be) if err != nil { return nil, err } @@ -480,9 +483,14 @@ func authorizeVotes(client *tlogbe.RecordClient) ([]ticketvote.AuthorizeVote, er return auths, nil } -func startVoteSave(client *tlogbe.RecordClient, sv ticketvote.StartVote) error { +func (p *ticketVotePlugin) startSave(sd ticketvote.StartDetails) error { + token, err := hex.DecodeString(sd.Vote.Token) + if err != nil { + return err + } + // Prepare blob - be, err := convertBlobEntryFromStartVote(sv) + be, err := convertBlobEntryFromStartDetails(sd) if err != nil { return err } @@ -496,7 +504,7 @@ func startVoteSave(client *tlogbe.RecordClient, sv ticketvote.StartVote) error { } // Save blob - merkles, err := client.Save(keyPrefixStartVote, + merkles, err := p.tlog.save(tlogIDVetted, token, keyPrefixStartDetails, [][]byte{b}, [][]byte{h}, false) if err != nil { return fmt.Errorf("Save: %v", err) @@ -509,10 +517,11 @@ func startVoteSave(client *tlogbe.RecordClient, sv ticketvote.StartVote) error { return nil } -// startVote returns the StartVote for the provided record if one exists. -func startVote(client *tlogbe.RecordClient) (*ticketvote.StartVote, error) { +// startDetails returns the StartDetails for the provided record if one exists. +func (p *ticketVotePlugin) startDetails(token []byte) (*ticketvote.StartDetails, error) { // Retrieve blobs - blobs, err := client.BlobsByKeyPrefix(keyPrefixStartVote) + blobs, err := p.tlog.blobsByKeyPrefix(tlogIDVetted, token, + keyPrefixStartDetails) if err != nil { return nil, err } @@ -526,7 +535,7 @@ func startVote(client *tlogbe.RecordClient) (*ticketvote.StartVote, error) { // This should not happen. There should only ever be a max of // one start vote. return nil, fmt.Errorf("multiple start votes found (%v) for record %x", - len(blobs), client.Token) + len(blobs), token) } // Decode blob @@ -534,41 +543,22 @@ func startVote(client *tlogbe.RecordClient) (*ticketvote.StartVote, error) { if err != nil { return nil, err } - sv, err := convertStartVoteFromBlobEntry(*be) + sd, err := convertStartDetailsFromBlobEntry(*be) if err != nil { return nil, err } - return sv, nil + return sd, nil } -func castVotes(client *tlogbe.RecordClient) ([]ticketvote.CastVote, error) { - // Retrieve blobs - blobs, err := client.BlobsByKeyPrefix(keyPrefixCastVote) +func (p *ticketVotePlugin) castVoteSave(cv ticketvote.CastVoteDetails) error { + token, err := hex.DecodeString(cv.Token) if err != nil { - return nil, err - } - - // Decode blobs - votes := make([]ticketvote.CastVote, 0, len(blobs)) - for _, v := range blobs { - be, err := store.Deblob(v) - if err != nil { - return nil, err - } - cv, err := convertCastVoteFromBlobEntry(*be) - if err != nil { - return nil, err - } - votes = append(votes, *cv) + return err } - return votes, nil -} - -func castVoteSave(client *tlogbe.RecordClient, cv ticketvote.CastVote) error { // Prepare blob - be, err := convertBlobEntryFromCastVote(cv) + be, err := convertBlobEntryFromCastVoteDetails(cv) if err != nil { return err } @@ -582,10 +572,10 @@ func castVoteSave(client *tlogbe.RecordClient, cv ticketvote.CastVote) error { } // Save blob - merkles, err := client.Save(keyPrefixCastVote, + merkles, err := p.tlog.save(tlogIDVetted, token, keyPrefixCastVoteDetails, [][]byte{b}, [][]byte{h}, false) if err != nil { - return fmt.Errorf("Save: %v", err) + return fmt.Errorf("save: %v", err) } if len(merkles) != 1 { return fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", @@ -595,6 +585,31 @@ func castVoteSave(client *tlogbe.RecordClient, cv ticketvote.CastVote) error { return nil } +func (p *ticketVotePlugin) castVotes(token []byte) ([]ticketvote.CastVoteDetails, error) { + // Retrieve blobs + blobs, err := p.tlog.blobsByKeyPrefix(tlogIDVetted, token, + keyPrefixCastVoteDetails) + if err != nil { + return nil, err + } + + // Decode blobs + votes := make([]ticketvote.CastVoteDetails, 0, len(blobs)) + for _, v := range blobs { + be, err := store.Deblob(v) + if err != nil { + return nil, err + } + cv, err := convertCastVoteDetailsFromBlobEntry(*be) + if err != nil { + return nil, err + } + votes = append(votes, *cv) + } + + return votes, nil +} + // bestBlock fetches the best block from the dcrdata plugin and returns it. If // the dcrdata connection is not active, an error will be returned. func (p *ticketVotePlugin) bestBlock() (uint32, error) { @@ -874,47 +889,20 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { return "", err } - // Verify signature - version := strconv.FormatUint(uint64(a.Version), 10) - msg := a.Token + version + string(a.Action) - err = util.VerifySignature(a.Signature, a.PublicKey, msg) - if err != nil { - return "", convertTicketVoteErrFromSignatureErr(err) - } - - // Get record client - tokenb, err := hex.DecodeString(a.Token) + // Verify token + token, err := hex.DecodeString(a.Token) if err != nil { return "", ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusTokenInvalid, } } - client, err := p.backend.RecordClient(tokenb) - if err != nil { - if err == backend.ErrRecordNotFound { - return "", ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusRecordNotFound, - } - } - return "", err - } - if client.State != tlogbe.RecordStateVetted { - return "", ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusRecordStatusInvalid, - ErrorContext: []string{"record not vetted"}, - } - } - // Verify record version - _, err = p.backend.GetVetted(tokenb, version) + // Verify signature + version := strconv.FormatUint(uint64(a.Version), 10) + msg := a.Token + version + string(a.Action) + err = util.VerifySignature(a.Signature, a.PublicKey, msg) if err != nil { - if err == backend.ErrRecordNotFound { - e := fmt.Sprintf("version %v not found", version) - return "", ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusRecordNotFound, - ErrorContext: []string{e}, - } - } + return "", convertTicketVoteErrFromSignatureErr(err) } // Verify action @@ -940,13 +928,13 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { // Get any previous authorizations to verify that the new action // is allowed based on the previous action. - auths, err := authorizeVotes(client) + auths, err := p.authorizes(token) if err != nil { return "", err } - var prevAction ticketvote.ActionT + var prevAction ticketvote.AuthActionT if len(auths) > 0 { - prevAction = ticketvote.ActionT(auths[len(auths)-1].Action) + prevAction = ticketvote.AuthActionT(auths[len(auths)-1].Action) } switch { case len(auths) == 0: @@ -973,7 +961,7 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { // Prepare authorize vote receipt := p.id.SignMessage([]byte(a.Signature)) - auth := ticketvote.AuthorizeVote{ + auth := ticketvote.AuthorizeDetails{ Token: a.Token, Version: a.Version, Action: string(a.Action), @@ -984,7 +972,7 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { } // Save authorize vote - err = authorizeVoteSave(client, auth) + err = p.authorizeSave(auth) if err != nil { return "", err } @@ -1148,6 +1136,14 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { return "", err } + // Verify token + token, err := hex.DecodeString(s.Vote.Token) + if err != nil { + return "", ticketvote.UserErrorReply{ + ErrorCode: ticketvote.ErrorStatusTokenInvalid, + } + } + // Verify signature vb, err := json.Marshal(s.Vote) if err != nil { @@ -1165,32 +1161,9 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { return "", err } - // Get record client - tokenb, err := hex.DecodeString(s.Vote.Token) - if err != nil { - return "", ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusTokenInvalid, - } - } - client, err := p.backend.RecordClient(tokenb) - if err != nil { - if err == backend.ErrRecordNotFound { - return "", ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusRecordNotFound, - } - } - return "", err - } - if client.State != tlogbe.RecordStateVetted { - return "", ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusRecordStatusInvalid, - ErrorContext: []string{"record not vetted"}, - } - } - // Verify record version version := strconv.FormatUint(uint64(s.Vote.Version), 10) - _, err = p.backend.GetVetted(tokenb, version) + _, err = p.backend.GetVetted(token, version) if err != nil { if err == backend.ErrRecordNotFound { e := fmt.Sprintf("version %v not found", version) @@ -1202,7 +1175,7 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { } // Verify vote authorization - auths, err := authorizeVotes(client) + auths, err := p.authorizes(token) if err != nil { return "", err } @@ -1212,7 +1185,7 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { ErrorContext: []string{"authorization not found"}, } } - action := ticketvote.ActionT(auths[len(auths)-1].Action) + action := ticketvote.AuthActionT(auths[len(auths)-1].Action) if action != ticketvote.ActionAuthorize { return "", ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusAuthorizationInvalid, @@ -1234,7 +1207,7 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { defer m.Unlock() // Verify vote has not already been started - svp, err := startVote(client) + svp, err := p.startDetails(token) if err != nil { return "", err } @@ -1247,7 +1220,7 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { } // Prepare start vote - sv := ticketvote.StartVote{ + sd := ticketvote.StartDetails{ Vote: s.Vote, PublicKey: s.PublicKey, Signature: s.Signature, @@ -1258,9 +1231,9 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { } // Save start vote - err = startVoteSave(client, sv) + err = p.startSave(sd) if err != nil { - return "", fmt.Errorf("startVoteSave: %v", err) + return "", fmt.Errorf("startSave: %v", err) } // Prepare reply @@ -1325,9 +1298,10 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { if err != nil { return "", err } + votes := ballot.Votes // Verify there is work to do - if len(ballot.Votes) == 0 { + if len(votes) == 0 { // Nothing to do br := ticketvote.BallotReply{ Receipts: []ticketvote.VoteReply{}, @@ -1339,38 +1313,20 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { return string(reply), nil } - // Get record client - var ( - votes = ballot.Votes - token = votes[0].Token - ) - tokenb, err := hex.DecodeString(token) + // Verify token + token, err := hex.DecodeString(votes[0].Token) if err != nil { e := ticketvote.VoteErrorTokenInvalid c := fmt.Sprintf("%v: not hex", ticketvote.VoteError[e]) return ballotExitWithErr(votes, e, c) } - client, err := p.backend.RecordClient(tokenb) - if err != nil { - if err == backend.ErrRecordNotFound { - e := ticketvote.VoteErrorRecordNotFound - c := fmt.Sprintf("%v: %v", ticketvote.VoteError[e], token) - return ballotExitWithErr(votes, e, c) - } - return "", err - } - if client.State != tlogbe.RecordStateVetted { - e := ticketvote.VoteErrorVoteStatusInvalid - c := fmt.Sprintf("%v: record is unvetted", ticketvote.VoteError[e]) - return ballotExitWithErr(votes, e, c) - } // Verify record vote status - sv, err := startVote(client) + sd, err := p.startDetails(token) if err != nil { return "", err } - if sv == nil { + if sd == nil { // Vote has not been started yet e := ticketvote.VoteErrorVoteStatusInvalid c := fmt.Sprintf("%v: vote not started", ticketvote.VoteError[e]) @@ -1380,7 +1336,7 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { if err != nil { return "", err } - if bb >= sv.EndBlockHeight { + if bb >= sd.EndBlockHeight { // Vote has ended e := ticketvote.VoteErrorVoteStatusInvalid c := fmt.Sprintf("%v: vote has ended", ticketvote.VoteError[e]) @@ -1388,8 +1344,8 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { } // Put eligible tickets in a map for easy lookups - eligible := make(map[string]struct{}, len(sv.EligibleTickets)) - for _, v := range sv.EligibleTickets { + eligible := make(map[string]struct{}, len(sd.EligibleTickets)) + for _, v := range sd.EligibleTickets { eligible[v] = struct{}{} } @@ -1406,7 +1362,7 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { // The lock must be held for the remainder of the function to // ensure duplicate votes cannot be cast. - m := p.mutex(token) + m := p.mutex(hex.EncodeToString(token)) m.Lock() defer m.Unlock() @@ -1420,7 +1376,7 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { receipts[k].Ticket = v.Ticket // Verify token is the same - if v.Token != token { + if v.Token != hex.EncodeToString(token) { e := ticketvote.VoteErrorMultipleRecordVotes receipts[k].ErrorCode = e receipts[k].ErrorContext = ticketvote.VoteError[e] @@ -1435,7 +1391,7 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { receipts[k].ErrorContext = ticketvote.VoteError[e] continue } - err = voteBitVerify(sv.Vote.Options, sv.Vote.Mask, bit) + err = voteBitVerify(sd.Vote.Options, sd.Vote.Mask, bit) if err != nil { e := ticketvote.VoteErrorVoteBitInvalid receipts[k].ErrorCode = e @@ -1495,14 +1451,14 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { // Save cast vote receipt := p.id.SignMessage([]byte(v.Signature)) - cv := ticketvote.CastVote{ + cv := ticketvote.CastVoteDetails{ Token: v.Token, Ticket: v.Ticket, VoteBit: v.VoteBit, Signature: v.Signature, Receipt: hex.EncodeToString(receipt[:]), } - err = castVoteSave(client, cv) + err = p.castVoteSave(cv) if err != nil { t := time.Now().Unix() log.Errorf("cmdBallot: castVoteSave %v: %v", t, err) @@ -1541,39 +1497,30 @@ func (p *ticketVotePlugin) cmdDetails(payload string) (string, error) { return "", err } - // Get record client - tokenb, err := hex.DecodeString(d.Token) + // Verify token + token, err := hex.DecodeString(d.Token) if err != nil { return "", ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusTokenInvalid, } } - client, err := p.backend.RecordClient(tokenb) - if err != nil { - if err == backend.ErrRecordNotFound { - return "", ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusRecordNotFound, - } - } - return "", err - } // Get authorize votes - auths, err := authorizeVotes(client) + auths, err := p.authorizes(token) if err != nil { - return "", fmt.Errorf("authorizeVotes: %v", err) + return "", err } // Get start vote - sv, err := startVote(client) + sd, err := p.startDetails(token) if err != nil { - return "", fmt.Errorf("startVote: %v", err) + return "", fmt.Errorf("startDetails: %v", err) } // Prepare rely dr := ticketvote.DetailsReply{ - Auths: auths, - Vote: sv, + Auths: auths, + StartDetails: sd, } reply, err := ticketvote.EncodeDetailsReply(dr) if err != nil { @@ -1592,27 +1539,18 @@ func (p *ticketVotePlugin) cmdCastVotes(payload string) (string, error) { return "", err } - // Get record client - tokenb, err := hex.DecodeString(cv.Token) + // Verify token + token, err := hex.DecodeString(cv.Token) if err != nil { return "", ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusTokenInvalid, } } - client, err := p.backend.RecordClient(tokenb) - if err != nil { - if err == backend.ErrRecordNotFound { - return "", ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusRecordNotFound, - } - } - return "", err - } // Get cast votes - votes, err := castVotes(client) + votes, err := p.castVotes(token) if err != nil { - return "", fmt.Errorf("castVotes: %v", err) + return "", err } // Prepare reply @@ -1627,9 +1565,9 @@ func (p *ticketVotePlugin) cmdCastVotes(payload string) (string, error) { return string(reply), nil } -func (p *ticketVotePlugin) summary(token string, bestBlock uint32) (*ticketvote.Summary, error) { +func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote.Summary, error) { // Check if the summary has been cached - s, err := p.cachedSummary(token) + s, err := p.cachedSummary(hex.EncodeToString(token)) switch { case err == errRecordNotFound: // Cached summary not found @@ -1641,29 +1579,12 @@ func (p *ticketVotePlugin) summary(token string, bestBlock uint32) (*ticketvote. return s, nil } - // Cached summary not found. Get the record client so we can create - // a summary manually. - tokenb, err := hex.DecodeString(token) - if err != nil { - return nil, errRecordNotFound - } - client, err := p.backend.RecordClient(tokenb) - if err != nil { - return nil, errRecordNotFound - } - if client.State != tlogbe.RecordStateVetted { - // Record exists but is unvetted so a vote can not have been - // authorized yet. - return &ticketvote.Summary{ - Status: ticketvote.VoteStatusUnauthorized, - Results: []ticketvote.Result{}, - }, nil - } + // Summary has not been cached. Get it manually. // Check if the vote has been authorized - auths, err := authorizeVotes(client) + auths, err := p.authorizes(token) if err != nil { - return nil, fmt.Errorf("authorizeVotes: %v", err) + return nil, fmt.Errorf("authorizes: %v", err) } if len(auths) == 0 { // Vote has not been authorized yet @@ -1673,7 +1594,7 @@ func (p *ticketVotePlugin) summary(token string, bestBlock uint32) (*ticketvote. }, nil } lastAuth := auths[len(auths)-1] - switch ticketvote.ActionT(lastAuth.Action) { + switch ticketvote.AuthActionT(lastAuth.Action) { case ticketvote.ActionAuthorize: // Vote has been authorized; continue case ticketvote.ActionRevoke: @@ -1685,11 +1606,11 @@ func (p *ticketVotePlugin) summary(token string, bestBlock uint32) (*ticketvote. } // Vote has been authorized. Check if it has been started yet. - sv, err := startVote(client) + sd, err := p.startDetails(token) if err != nil { - return nil, fmt.Errorf("startVote: %v", err) + return nil, fmt.Errorf("startDetails: %v", err) } - if sv == nil { + if sd == nil { // Vote has not been started yet return &ticketvote.Summary{ Status: ticketvote.VoteStatusAuthorized, @@ -1700,7 +1621,7 @@ func (p *ticketVotePlugin) summary(token string, bestBlock uint32) (*ticketvote. // Vote has been started. Check if it is still in progress or has // already ended. var status ticketvote.VoteStatusT - if bestBlock < sv.EndBlockHeight { + if bestBlock < sd.EndBlockHeight { status = ticketvote.VoteStatusStarted } else { status = ticketvote.VoteStatusFinished @@ -1709,12 +1630,12 @@ func (p *ticketVotePlugin) summary(token string, bestBlock uint32) (*ticketvote. // Pull the cast votes from the cache and calculate the results // manually. votes := p.cachedVotes(token) - tally := make(map[string]int, len(sv.Vote.Options)) + tally := make(map[string]int, len(sd.Vote.Options)) for _, voteBit := range votes { tally[voteBit]++ } - results := make([]ticketvote.Result, len(sv.Vote.Options)) - for _, v := range sv.Vote.Options { + results := make([]ticketvote.Result, len(sd.Vote.Options)) + for _, v := range sd.Vote.Options { bit := strconv.FormatUint(v.Bit, 16) results = append(results, ticketvote.Result{ ID: v.ID, @@ -1726,7 +1647,7 @@ func (p *ticketVotePlugin) summary(token string, bestBlock uint32) (*ticketvote. // Approved can only be calculated on certain types of votes var approved bool - switch sv.Vote.Type { + switch sd.Vote.Type { case ticketvote.VoteTypeStandard, ticketvote.VoteTypeRunoff: // Calculate results for a simple approve/reject vote var total uint64 @@ -1735,9 +1656,9 @@ func (p *ticketVotePlugin) summary(token string, bestBlock uint32) (*ticketvote. } var ( - eligible = float64(len(sv.EligibleTickets)) - quorumPerc = float64(sv.Vote.QuorumPercentage) - passPerc = float64(sv.Vote.PassPercentage) + eligible = float64(len(sd.EligibleTickets)) + quorumPerc = float64(sd.Vote.QuorumPercentage) + passPerc = float64(sd.Vote.PassPercentage) quorum = uint64(quorumPerc / 100 * eligible) pass = uint64(passPerc / 100 * float64(total)) @@ -1764,15 +1685,15 @@ func (p *ticketVotePlugin) summary(token string, bestBlock uint32) (*ticketvote. // Prepare summary summary := ticketvote.Summary{ - Type: sv.Vote.Type, + Type: sd.Vote.Type, Status: status, - Duration: sv.Vote.Duration, - StartBlockHeight: sv.StartBlockHeight, - StartBlockHash: sv.StartBlockHash, - EndBlockHeight: sv.EndBlockHeight, - EligibleTickets: uint32(len(sv.EligibleTickets)), - QuorumPercentage: sv.Vote.QuorumPercentage, - PassPercentage: sv.Vote.PassPercentage, + Duration: sd.Vote.Duration, + StartBlockHeight: sd.StartBlockHeight, + StartBlockHash: sd.StartBlockHash, + EndBlockHeight: sd.EndBlockHeight, + EligibleTickets: uint32(len(sd.EligibleTickets)), + QuorumPercentage: sd.Vote.QuorumPercentage, + PassPercentage: sd.Vote.PassPercentage, Results: results, Approved: approved, } @@ -1781,15 +1702,15 @@ func (p *ticketVotePlugin) summary(token string, bestBlock uint32) (*ticketvote. // calculate these results again. if status == ticketvote.VoteStatusFinished { // Save summary - err = p.cachedSummarySave(sv.Vote.Token, summary) + err = p.cachedSummarySave(sd.Vote.Token, summary) if err != nil { return nil, fmt.Errorf("cachedSummarySave %v: %v %v", - sv.Vote.Token, err, summary) + sd.Vote.Token, err, summary) } // Remove record from the votes cache now that a summary has // been saved for it. - p.cachedVotesDel(sv.Vote.Token) + p.cachedVotesDel(sd.Vote.Token) } return &summary, nil @@ -1814,7 +1735,11 @@ func (p *ticketVotePlugin) cmdSummaries(payload string) (string, error) { // Get summaries summaries := make(map[string]ticketvote.Summary, len(s.Tokens)) for _, v := range s.Tokens { - s, err := p.summary(v, bb) + token, err := hex.DecodeString(v) + if err != nil { + return "", err + } + s, err := p.summary(token, bb) if err != nil { if err == errRecordNotFound { // Record does not exist for token. Do not include this token @@ -1948,11 +1873,25 @@ func (p *ticketVotePlugin) cmdInventory(payload string) (string, error) { return string(reply), nil } -// Cmd executes a plugin command. +// setup performs any plugin setup work that needs to be done. // -// This function satisfies the Plugin interface. -func (p *ticketVotePlugin) Cmd(cmd, payload string) (string, error) { - log.Tracef("ticketvote Cmd: %v", cmd) +// This function satisfies the pluginClient interface. +func (p *ticketVotePlugin) setup() error { + log.Tracef("ticketvote setup") + + // TODO + // Ensure dcrdata plugin has been registered + // Build votes cache + // Build inventory cache + + return nil +} + +// cmd executes a plugin command. +// +// This function satisfies the pluginClient interface. +func (p *ticketVotePlugin) cmd(cmd, payload string) (string, error) { + log.Tracef("ticketvote cmd: %v %v", cmd, payload) switch cmd { case ticketvote.CmdAuthorize: @@ -1976,39 +1915,25 @@ func (p *ticketVotePlugin) Cmd(cmd, payload string) (string, error) { return "", backend.ErrPluginCmdInvalid } -// Hook executes a plugin hook. +// hook executes a plugin hook. // -// This function satisfies the Plugin interface. -func (p *ticketVotePlugin) Hook(h tlogbe.HookT, payload string) error { - log.Tracef("ticketvote Hook: %v %v", tlogbe.Hooks[h], payload) +// This function satisfies the pluginClient interface. +func (p *ticketVotePlugin) hook(h hookT, payload string) error { + log.Tracef("ticketvote hook: %v %v", hooks[h], payload) return nil } // Fsck performs a plugin filesystem check. // -// This function satisfies the Plugin interface. -func (p *ticketVotePlugin) Fsck() error { - log.Tracef("ticketvote Fsck") - - return nil -} - -// Setup performs any plugin setup work that needs to be done. -// -// This function satisfies the Plugin interface. -func (p *ticketVotePlugin) Setup() error { - log.Tracef("ticketvote Setup") - - // TODO - // Ensure dcrdata plugin has been registered - // Build votes cache - // Build inventory cache +// This function satisfies the pluginClient interface. +func (p *ticketVotePlugin) fsck() error { + log.Tracef("ticketvote fsck") return nil } -func TicketVotePluginNew(backend *tlogbe.TlogBackend, settings []backend.PluginSetting) (*ticketVotePlugin, error) { +func TicketVotePluginNew(backend backend.Backend, tlog tlogClient, settings []backend.PluginSetting) (*ticketVotePlugin, error) { var ( // TODO these should be passed in as plugin settings dataDir string @@ -2028,6 +1953,7 @@ func TicketVotePluginNew(backend *tlogbe.TlogBackend, settings []backend.PluginS return &ticketVotePlugin{ dataDir: filepath.Join(dataDir, ticketVoteDirname), backend: backend, + tlog: tlog, id: id, activeNetParams: activeNetParams, voteDurationMin: voteDurationMin, From 3c36539aca4ac6699ee26740f9b2a9585c48e914 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 23 Sep 2020 07:41:35 -0500 Subject: [PATCH 094/449] ticketvote and pi vote api refactor --- plugins/ticketvote/ticketvote.go | 85 +++++++------- politeiad/backend/tlogbe/ticketvote.go | 156 ++++++++++++------------- politeiawww/api/pi/v1/v1.go | 145 +++++++++++++++++------ politeiawww/piwww.go | 8 +- politeiawww/proposals.go | 25 ++-- politeiawww/ticketvote.go | 4 +- politeiawww/www.go | 6 +- 7 files changed, 252 insertions(+), 177 deletions(-) diff --git a/plugins/ticketvote/ticketvote.go b/plugins/ticketvote/ticketvote.go index d355bdc3d..4dc201140 100644 --- a/plugins/ticketvote/ticketvote.go +++ b/plugins/ticketvote/ticketvote.go @@ -17,7 +17,7 @@ type VoteT int type VoteErrorT int type ErrorStatusT int -// TODO StartDetails, StartReply, StartRunoffReply should contain a receipt. +// TODO VoteDetails, StartReply, StartRunoffReply should contain a receipt. // The receipt should be the server signature of Signature+StartBlockHash. const ( @@ -61,7 +61,7 @@ const ( // considered approved and all other records are considered to be // rejected. If no records meet the quorum and pass requirements // then all records are considered rejected. Note, in a runoff vote - // it's possible for a proposal to meet both the quorum and pass + // it's possible for a record to meet both the quorum and pass // requirements but still be rejected if it does not have the most // net yes votes. VoteTypeRunoff VoteT = 2 @@ -100,7 +100,7 @@ const ( ErrorStatusRecordNotFound ErrorStatusT = 4 ErrorStatusRecordStatusInvalid ErrorStatusT = 5 ErrorStatusAuthorizationInvalid ErrorStatusT = 6 - ErrorStatusVoteDetailsInvalid ErrorStatusT = 7 + ErrorStatusVoteParamsInvalid ErrorStatusT = 7 ErrorStatusVoteStatusInvalid ErrorStatusT = 8 ErrorStatusBallotInvalid ErrorStatusT = 9 ) @@ -136,7 +136,7 @@ var ( ErrorStatusRecordNotFound: "record not found", ErrorStatusRecordStatusInvalid: "record status invalid", ErrorStatusAuthorizationInvalid: "authorization invalid", - ErrorStatusVoteDetailsInvalid: "vote details invalid", + ErrorStatusVoteParamsInvalid: "vote params invalid", ErrorStatusVoteStatusInvalid: "vote status invalid", ErrorStatusBallotInvalid: "ballot invalid", } @@ -160,8 +160,24 @@ type VoteOption struct { Bit uint64 `json:"bit"` // Bit used for this option } -// VoteDetails describes the options and parameters of a ticket vote. -type VoteDetails struct { +// AuthorizeDetails is the structure that is saved to disk when a vote is +// authorized or a previous authorization is revoked. It contains all the +// fields from a Authorize and a AuthorizeReply. +type AuthorizeDetails struct { + // Data generated by client + Token string `json:"token"` // Record token + Version uint32 `json:"version"` // Record version + Action string `json:"action"` // Authorize or revoke + PublicKey string `json:"publickey"` // Public key used for signature + Signature string `json:"signature"` // Signature of token+version+action + + // Metadata generated by server + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + Receipt string `json:"receipt"` // Server signature of client signature +} + +// VoteParams describes the options and parameters of a ticket vote. +type VoteParams struct { Token string `json:"token"` // Record token Version uint32 `json:"version"` // Record version Type VoteT `json:"type"` // Vote type @@ -179,34 +195,18 @@ type VoteDetails struct { Options []VoteOption `json:"options"` } -// AuthorizeDetails is the structure that is saved to disk when a vote is -// authorized or a previous authorization is revoked. It contains all the -// fields from a Authorize and a AuthorizeReply. -type AuthorizeDetails struct { - // Data generated by client - Token string `json:"token"` // Record token - Version uint32 `json:"version"` // Record version - Action string `json:"action"` // Authorize or revoke - PublicKey string `json:"publickey"` // Public key used for signature - Signature string `json:"signature"` // Signature of token+version+action - - // Metadata generated by server - Timestamp int64 `json:"timestamp"` // Received UNIX timestamp - Receipt string `json:"receipt"` // Server signature of client signature -} - -// StartDetails is the structure that is saved to disk when a vote is started. +// VoteDetails is the structure that is saved to disk when a vote is started. // It contains all of the fields from a Start and a StartReply. // // Signature is the client signature of the SHA256 digest of the JSON encoded // Vote struct. // // TODO does this need a receipt? -type StartDetails struct { +type VoteDetails struct { // Data generated by client - Vote VoteDetails `json:"vote"` - PublicKey string `json:"publickey"` - Signature string `json:"signature"` + Params VoteParams `json:"params"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` // Metadata generated by server StartBlockHeight uint32 `json:"startblockheight"` @@ -283,9 +283,9 @@ func DecodeAuthorizeReply(payload []byte) (*AuthorizeReply, error) { // Signature is the signature of a SHA256 digest of the JSON encoded Vote // structure. type Start struct { - Vote VoteDetails `json:"vote"` - PublicKey string `json:"publickey"` // Public key used for signature - Signature string `json:"signature"` // Client signature + Params VoteParams `json:"params"` + PublicKey string `json:"publickey"` // Public key used for signature + Signature string `json:"signature"` // Client signature } // EncodeStart encodes a Start into a JSON byte slice. @@ -331,9 +331,9 @@ func DecodeStartReply(payload []byte) (*StartReply, error) { // StartRunoff starts a runoff vote between the provided submissions. Each // submission is required to have its own Authorize and Start. type StartRunoff struct { - Token string `json:"token"` // RFP token - Authorizations []Authorize `json:"authorizations"` - Starts []Start `json:"starts"` + Token string `json:"token"` // RFP token + Auths []Authorize `json:"auths"` + Starts []Start `json:"starts"` } // EncodeStartRunoff encodes a StartRunoff into a JSON byte slice. @@ -376,17 +376,17 @@ func DecodeStartRunoffReply(payload []byte) (*StartRunoffReply, error) { return &srr, nil } -// Vote is a signed ticket vote. This structure gets saved to disk when +// CastVote is a signed ticket vote. This structure gets saved to disk when // a vote is cast. -type Vote struct { +type CastVote struct { Token string `json:"token"` // Record token Ticket string `json:"ticket"` // Ticket ID VoteBit string `json:"votebits"` // Selected vote bit, hex encoded Signature string `json:"signature"` // Signature of Token+Ticket+VoteBit } -// VoteReply contains the receipt for the cast vote. -type VoteReply struct { +// CastVoteReply contains the receipt for the cast vote. +type CastVoteReply struct { Ticket string `json:"ticket"` // Ticket ID Receipt string `json:"receipt"` // Server signature of client signature @@ -399,7 +399,7 @@ type VoteReply struct { // Ballot is a batch of votes that are sent to the server. A ballot can only // contain the votes for a single record. type Ballot struct { - Votes []Vote `json:"votes"` + Votes []CastVote `json:"votes"` } // EncodeBallot encodes a Ballot into a JSON byte slice. @@ -419,7 +419,7 @@ func DecodeBallot(payload []byte) (*Ballot, error) { // BallotReply is a reply to a batched list of votes. type BallotReply struct { - Receipts []VoteReply `json:"receipts"` + Receipts []CastVoteReply `json:"receipts"` } // EncodeBallotReply encodes a Ballot into a JSON byte slice. @@ -457,10 +457,11 @@ func DecodeDetails(payload []byte) (*Details, error) { return &d, nil } -// DetailsReply is the reply to the Details command. +// DetailsReply is the reply to the Details command. The VoteDetails will be +// nil if the vote has been started. type DetailsReply struct { - Auths []AuthorizeDetails `json:"auths"` - StartDetails *StartDetails `json:"startdetails"` + Auths []AuthorizeDetails `json:"auths"` + Vote *VoteDetails `json:"vote"` } // EncodeDetailsReply encodes a DetailsReply into a JSON byte slice. diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index 13e448302..b56e8eb0b 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -43,13 +43,13 @@ const ( // Blob entry data descriptors dataDescriptorAuthorizeDetails = "authorizedetails" - dataDescriptorStartDetails = "startdetails" + dataDescriptorVoteDetails = "votedetails" dataDescriptorCastVoteDetails = "castvotedetails" // Prefixes that are appended to key-value store keys before // storing them in the log leaf ExtraData field. keyPrefixAuthorizeDetails = "authorizedetails:" - keyPrefixStartDetails = "startdetails:" + keyPrefixVoteDetails = "votedetails:" keyPrefixCastVoteDetails = "castvotedetails:" ) @@ -297,7 +297,7 @@ func convertAuthorizeDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.Autho return &ad, nil } -func convertStartDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.StartDetails, error) { +func convertVoteDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.VoteDetails, error) { // Decode and validate data hint b, err := base64.StdEncoding.DecodeString(be.DataHint) if err != nil { @@ -308,9 +308,9 @@ func convertStartDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.StartDeta if err != nil { return nil, fmt.Errorf("unmarshal DataHint: %v", err) } - if dd.Descriptor != dataDescriptorStartDetails { + if dd.Descriptor != dataDescriptorVoteDetails { return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorStartDetails) + dd.Descriptor, dataDescriptorVoteDetails) } // Decode data @@ -326,13 +326,13 @@ func convertStartDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.StartDeta return nil, fmt.Errorf("data is not coherent; got %x, want %x", util.Digest(b), hash) } - var sd ticketvote.StartDetails - err = json.Unmarshal(b, &sd) + var vd ticketvote.VoteDetails + err = json.Unmarshal(b, &vd) if err != nil { - return nil, fmt.Errorf("unmarshal StartDetails: %v", err) + return nil, fmt.Errorf("unmarshal VoteDetails: %v", err) } - return &sd, nil + return &vd, nil } func convertCastVoteDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.CastVoteDetails, error) { @@ -390,15 +390,15 @@ func convertBlobEntryFromAuthorizeDetails(ad ticketvote.AuthorizeDetails) (*stor return &be, nil } -func convertBlobEntryFromStartDetails(sd ticketvote.StartDetails) (*store.BlobEntry, error) { - data, err := json.Marshal(sd) +func convertBlobEntryFromVoteDetails(vd ticketvote.VoteDetails) (*store.BlobEntry, error) { + data, err := json.Marshal(vd) if err != nil { return nil, err } hint, err := json.Marshal( store.DataDescriptor{ Type: store.DataTypeStructure, - Descriptor: dataDescriptorStartDetails, + Descriptor: dataDescriptorVoteDetails, }) if err != nil { return nil, err @@ -483,14 +483,14 @@ func (p *ticketVotePlugin) authorizes(token []byte) ([]ticketvote.AuthorizeDetai return auths, nil } -func (p *ticketVotePlugin) startSave(sd ticketvote.StartDetails) error { - token, err := hex.DecodeString(sd.Vote.Token) +func (p *ticketVotePlugin) startSave(vd ticketvote.VoteDetails) error { + token, err := hex.DecodeString(vd.Params.Token) if err != nil { return err } // Prepare blob - be, err := convertBlobEntryFromStartDetails(sd) + be, err := convertBlobEntryFromVoteDetails(vd) if err != nil { return err } @@ -504,7 +504,7 @@ func (p *ticketVotePlugin) startSave(sd ticketvote.StartDetails) error { } // Save blob - merkles, err := p.tlog.save(tlogIDVetted, token, keyPrefixStartDetails, + merkles, err := p.tlog.save(tlogIDVetted, token, keyPrefixVoteDetails, [][]byte{b}, [][]byte{h}, false) if err != nil { return fmt.Errorf("Save: %v", err) @@ -517,11 +517,11 @@ func (p *ticketVotePlugin) startSave(sd ticketvote.StartDetails) error { return nil } -// startDetails returns the StartDetails for the provided record if one exists. -func (p *ticketVotePlugin) startDetails(token []byte) (*ticketvote.StartDetails, error) { +// startDetails returns the VoteDetails for the provided record if one exists. +func (p *ticketVotePlugin) startDetails(token []byte) (*ticketvote.VoteDetails, error) { // Retrieve blobs blobs, err := p.tlog.blobsByKeyPrefix(tlogIDVetted, token, - keyPrefixStartDetails) + keyPrefixVoteDetails) if err != nil { return nil, err } @@ -543,12 +543,12 @@ func (p *ticketVotePlugin) startDetails(token []byte) (*ticketvote.StartDetails, if err != nil { return nil, err } - sd, err := convertStartDetailsFromBlobEntry(*be) + vd, err := convertVoteDetailsFromBlobEntry(*be) if err != nil { return nil, err } - return sd, nil + return vd, nil } func (p *ticketVotePlugin) castVoteSave(cv ticketvote.CastVoteDetails) error { @@ -857,12 +857,12 @@ func (p *ticketVotePlugin) voteMessageVerify(address, message, signature string) return a.EncodeAddress() == address, nil } -func (p *ticketVotePlugin) voteSignatureVerify(v ticketvote.Vote, addr string) error { - msg := v.Token + v.Ticket + v.VoteBit +func (p *ticketVotePlugin) castVoteSignatureVerify(cv ticketvote.CastVote, addr string) error { + msg := cv.Token + cv.Ticket + cv.VoteBit // Convert hex signature to base64. The voteMessageVerify function // expects bas64. - b, err := hex.DecodeString(v.Signature) + b, err := hex.DecodeString(cv.Signature) if err != nil { return fmt.Errorf("invalid hex") } @@ -1015,7 +1015,7 @@ func voteBitVerify(options []ticketvote.VoteOption, mask, bit uint64) error { } // TODO test this function -func voteDetailsVerify(vote ticketvote.VoteDetails, voteDurationMin, voteDurationMax uint32) error { +func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationMax uint32) error { // Verify vote type switch vote.Type { case ticketvote.VoteTypeStandard: @@ -1025,7 +1025,7 @@ func voteDetailsVerify(vote ticketvote.VoteDetails, voteDurationMin, voteDuratio default: e := fmt.Sprintf("invalid type %v", vote.Type) return ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, + ErrorCode: ticketvote.ErrorStatusVoteParamsInvalid, ErrorContext: []string{e}, } } @@ -1036,28 +1036,28 @@ func voteDetailsVerify(vote ticketvote.VoteDetails, voteDurationMin, voteDuratio e := fmt.Sprintf("duration %v exceeds max duration %v", vote.Duration, voteDurationMax) return ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, + ErrorCode: ticketvote.ErrorStatusVoteParamsInvalid, ErrorContext: []string{e}, } case vote.Duration < voteDurationMin: e := fmt.Sprintf("duration %v under min duration %v", vote.Duration, voteDurationMin) return ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, + ErrorCode: ticketvote.ErrorStatusVoteParamsInvalid, ErrorContext: []string{e}, } case vote.QuorumPercentage > 100: e := fmt.Sprintf("quorum percent %v exceeds 100 percent", vote.QuorumPercentage) return ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, + ErrorCode: ticketvote.ErrorStatusVoteParamsInvalid, ErrorContext: []string{e}, } case vote.PassPercentage > 100: e := fmt.Sprintf("pass percent %v exceeds 100 percent", vote.PassPercentage) return ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, + ErrorCode: ticketvote.ErrorStatusVoteParamsInvalid, ErrorContext: []string{e}, } } @@ -1066,7 +1066,7 @@ func voteDetailsVerify(vote ticketvote.VoteDetails, voteDurationMin, voteDuratio // requirements. if len(vote.Options) == 0 { return ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, + ErrorCode: ticketvote.ErrorStatusVoteParamsInvalid, ErrorContext: []string{"no vote options found"}, } } @@ -1079,7 +1079,7 @@ func voteDetailsVerify(vote ticketvote.VoteDetails, voteDurationMin, voteDuratio e := fmt.Sprintf("vote options count got %v, want 2", len(vote.Options)) return ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, + ErrorCode: ticketvote.ErrorStatusVoteParamsInvalid, ErrorContext: []string{e}, } } @@ -1107,7 +1107,7 @@ func voteDetailsVerify(vote ticketvote.VoteDetails, voteDurationMin, voteDuratio e := fmt.Sprintf("vote option IDs not found: %v", strings.Join(missing, ",")) return ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, + ErrorCode: ticketvote.ErrorStatusVoteParamsInvalid, ErrorContext: []string{e}, } } @@ -1118,7 +1118,7 @@ func voteDetailsVerify(vote ticketvote.VoteDetails, voteDurationMin, voteDuratio err := voteBitVerify(vote.Options, vote.Mask, v.Bit) if err != nil { return ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusVoteDetailsInvalid, + ErrorCode: ticketvote.ErrorStatusVoteParamsInvalid, ErrorContext: []string{err.Error()}, } } @@ -1137,7 +1137,7 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { } // Verify token - token, err := hex.DecodeString(s.Vote.Token) + token, err := hex.DecodeString(s.Params.Token) if err != nil { return "", ticketvote.UserErrorReply{ ErrorCode: ticketvote.ErrorStatusTokenInvalid, @@ -1145,7 +1145,7 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { } // Verify signature - vb, err := json.Marshal(s.Vote) + vb, err := json.Marshal(s.Params) if err != nil { return "", err } @@ -1156,13 +1156,13 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { } // Verify vote options and params - err = voteDetailsVerify(s.Vote, p.voteDurationMin, p.voteDurationMax) + err = voteParamsVerify(s.Params, p.voteDurationMin, p.voteDurationMax) if err != nil { return "", err } // Verify record version - version := strconv.FormatUint(uint64(s.Vote.Version), 10) + version := strconv.FormatUint(uint64(s.Params.Version), 10) _, err = p.backend.GetVetted(token, version) if err != nil { if err == backend.ErrRecordNotFound { @@ -1194,7 +1194,7 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { } // Get vote blockchain data - sr, err := p.startReply(s.Vote.Duration) + sr, err := p.startReply(s.Params.Duration) if err != nil { return "", err } @@ -1202,7 +1202,7 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { // Any previous start vote must be retrieved to verify that a vote // has not already been started. The lock must be held for the // remainder of this function. - m := p.mutex(s.Vote.Token) + m := p.mutex(s.Params.Token) m.Lock() defer m.Unlock() @@ -1220,8 +1220,8 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { } // Prepare start vote - sd := ticketvote.StartDetails{ - Vote: s.Vote, + vd := ticketvote.VoteDetails{ + Params: s.Params, PublicKey: s.PublicKey, Signature: s.Signature, StartBlockHeight: sr.StartBlockHeight, @@ -1231,7 +1231,7 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { } // Save start vote - err = p.startSave(sd) + err = p.startSave(vd) if err != nil { return "", fmt.Errorf("startSave: %v", err) } @@ -1253,9 +1253,9 @@ func (p *ticketVotePlugin) cmdStartRunoff(payload string) (string, error) { // ballotExitWithErr applies the provided vote error to each of the cast vote // replies then returns the encoded ballot reply. -func ballotExitWithErr(votes []ticketvote.Vote, errCode ticketvote.VoteErrorT, errContext string) (string, error) { +func ballotExitWithErr(votes []ticketvote.CastVote, errCode ticketvote.VoteErrorT, errContext string) (string, error) { token := votes[0].Token - receipts := make([]ticketvote.VoteReply, len(votes)) + receipts := make([]ticketvote.CastVoteReply, len(votes)) for k, v := range votes { // Its possible that cast votes were provided for different // records. This is not allowed. Verify the token is the same @@ -1304,7 +1304,7 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { if len(votes) == 0 { // Nothing to do br := ticketvote.BallotReply{ - Receipts: []ticketvote.VoteReply{}, + Receipts: []ticketvote.CastVoteReply{}, } reply, err := ticketvote.EncodeBallotReply(br) if err != nil { @@ -1322,11 +1322,11 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { } // Verify record vote status - sd, err := p.startDetails(token) + vd, err := p.startDetails(token) if err != nil { return "", err } - if sd == nil { + if vd == nil { // Vote has not been started yet e := ticketvote.VoteErrorVoteStatusInvalid c := fmt.Sprintf("%v: vote not started", ticketvote.VoteError[e]) @@ -1336,7 +1336,7 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { if err != nil { return "", err } - if bb >= sd.EndBlockHeight { + if bb >= vd.EndBlockHeight { // Vote has ended e := ticketvote.VoteErrorVoteStatusInvalid c := fmt.Sprintf("%v: vote has ended", ticketvote.VoteError[e]) @@ -1344,8 +1344,8 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { } // Put eligible tickets in a map for easy lookups - eligible := make(map[string]struct{}, len(sd.EligibleTickets)) - for _, v := range sd.EligibleTickets { + eligible := make(map[string]struct{}, len(vd.EligibleTickets)) + for _, v := range vd.EligibleTickets { eligible[v] = struct{}{} } @@ -1370,7 +1370,7 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { castVotes := p.cachedVotes(token) // Verify and save votes - receipts := make([]ticketvote.VoteReply, len(votes)) + receipts := make([]ticketvote.CastVoteReply, len(votes)) for k, v := range votes { // Set receipt ticket receipts[k].Ticket = v.Ticket @@ -1391,7 +1391,7 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { receipts[k].ErrorContext = ticketvote.VoteError[e] continue } - err = voteBitVerify(sd.Vote.Options, sd.Vote.Mask, bit) + err = voteBitVerify(vd.Params.Options, vd.Params.Mask, bit) if err != nil { e := ticketvote.VoteErrorVoteBitInvalid receipts[k].ErrorCode = e @@ -1422,7 +1422,7 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { ticketvote.VoteError[e], t) continue } - err = p.voteSignatureVerify(v, ca.addr) + err = p.castVoteSignatureVerify(v, ca.addr) if err != nil { e := ticketvote.VoteErrorSignatureInvalid receipts[k].ErrorCode = e @@ -1512,15 +1512,15 @@ func (p *ticketVotePlugin) cmdDetails(payload string) (string, error) { } // Get start vote - sd, err := p.startDetails(token) + vd, err := p.startDetails(token) if err != nil { return "", fmt.Errorf("startDetails: %v", err) } // Prepare rely dr := ticketvote.DetailsReply{ - Auths: auths, - StartDetails: sd, + Auths: auths, + Vote: vd, } reply, err := ticketvote.EncodeDetailsReply(dr) if err != nil { @@ -1606,11 +1606,11 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. } // Vote has been authorized. Check if it has been started yet. - sd, err := p.startDetails(token) + vd, err := p.startDetails(token) if err != nil { return nil, fmt.Errorf("startDetails: %v", err) } - if sd == nil { + if vd == nil { // Vote has not been started yet return &ticketvote.Summary{ Status: ticketvote.VoteStatusAuthorized, @@ -1621,7 +1621,7 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. // Vote has been started. Check if it is still in progress or has // already ended. var status ticketvote.VoteStatusT - if bestBlock < sd.EndBlockHeight { + if bestBlock < vd.EndBlockHeight { status = ticketvote.VoteStatusStarted } else { status = ticketvote.VoteStatusFinished @@ -1630,12 +1630,12 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. // Pull the cast votes from the cache and calculate the results // manually. votes := p.cachedVotes(token) - tally := make(map[string]int, len(sd.Vote.Options)) + tally := make(map[string]int, len(vd.Params.Options)) for _, voteBit := range votes { tally[voteBit]++ } - results := make([]ticketvote.Result, len(sd.Vote.Options)) - for _, v := range sd.Vote.Options { + results := make([]ticketvote.Result, len(vd.Params.Options)) + for _, v := range vd.Params.Options { bit := strconv.FormatUint(v.Bit, 16) results = append(results, ticketvote.Result{ ID: v.ID, @@ -1647,7 +1647,7 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. // Approved can only be calculated on certain types of votes var approved bool - switch sd.Vote.Type { + switch vd.Params.Type { case ticketvote.VoteTypeStandard, ticketvote.VoteTypeRunoff: // Calculate results for a simple approve/reject vote var total uint64 @@ -1656,9 +1656,9 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. } var ( - eligible = float64(len(sd.EligibleTickets)) - quorumPerc = float64(sd.Vote.QuorumPercentage) - passPerc = float64(sd.Vote.PassPercentage) + eligible = float64(len(vd.EligibleTickets)) + quorumPerc = float64(vd.Params.QuorumPercentage) + passPerc = float64(vd.Params.PassPercentage) quorum = uint64(quorumPerc / 100 * eligible) pass = uint64(passPerc / 100 * float64(total)) @@ -1685,15 +1685,15 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. // Prepare summary summary := ticketvote.Summary{ - Type: sd.Vote.Type, + Type: vd.Params.Type, Status: status, - Duration: sd.Vote.Duration, - StartBlockHeight: sd.StartBlockHeight, - StartBlockHash: sd.StartBlockHash, - EndBlockHeight: sd.EndBlockHeight, - EligibleTickets: uint32(len(sd.EligibleTickets)), - QuorumPercentage: sd.Vote.QuorumPercentage, - PassPercentage: sd.Vote.PassPercentage, + Duration: vd.Params.Duration, + StartBlockHeight: vd.StartBlockHeight, + StartBlockHash: vd.StartBlockHash, + EndBlockHeight: vd.EndBlockHeight, + EligibleTickets: uint32(len(vd.EligibleTickets)), + QuorumPercentage: vd.Params.QuorumPercentage, + PassPercentage: vd.Params.PassPercentage, Results: results, Approved: approved, } @@ -1702,15 +1702,15 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. // calculate these results again. if status == ticketvote.VoteStatusFinished { // Save summary - err = p.cachedSummarySave(sd.Vote.Token, summary) + err = p.cachedSummarySave(vd.Params.Token, summary) if err != nil { return nil, fmt.Errorf("cachedSummarySave %v: %v %v", - sd.Vote.Token, err, summary) + vd.Params.Token, err, summary) } // Remove record from the votes cache now that a summary has // been saved for it. - p.cachedVotesDel(sd.Vote.Token) + p.cachedVotesDel(vd.Params.Token) } return &summary, nil diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 1617cad58..d1441c98a 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -148,7 +148,7 @@ const ( // Vote errors ErrorStatusVoteStatusInvalid - ErrorStatusVoteDetailsInvalid + ErrorStatusVoteParamsInvalid ErrorStatusBallotInvalid ) @@ -515,26 +515,15 @@ type CommentVotesReply struct { Votes []CommentVoteDetails `json:"votes"` } -// VoteAuthorize authorizes a proposal vote or revokes a previous vote -// authorization. All proposal votes must be authorized by the proposal author -// before an admin is able to start the voting process. -// -// Signature contains the client signature of the Token+Version+Action. -type VoteAuthorize struct { - Token string `json:"token"` - Version uint32 `json:"version"` - Action VoteAuthActionT `json:"action"` - PublicKey string `json:"publickey"` - Signature string `json:"signature"` -} - -// VoteAuthorizeReply is the reply to the VoteAuthorize command. -// -// Receipt is the server signature of the client signature. This is proof that -// the server received and processed the VoteAuthorize command. -type VoteAuthorizeReply struct { - Timestamp int64 `json:"timestamp"` - Receipt string `json:"receipt"` +// AuthorizeDetails contains the details of a vote authorization. +type AuthorizeDetails struct { + Token string `json:"token"` // Proposal token + Version uint32 `json:"version"` // Proposal version + Action string `json:"action"` // Authorize or revoke + PublicKey string `json:"publickey"` // Public key used for signature + Signature string `json:"signature"` // Signature of token+version+action + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + Receipt string `json:"receipt"` // Server signature of client signature } // VoteOption describes a single vote option. @@ -544,9 +533,9 @@ type VoteOption struct { Bit uint64 `json:"bit"` // Bit used for this option } -// VoteDetails includes all data required by server to start vote on a -// proposal. -type VoteDetails struct { +// VoteParams contains all client defined vote params required by server to +// start a proposal vote. +type VoteParams struct { Token string `json:"token"` // Proposal token Version uint32 `json:"version"` // Proposal version Type VoteT `json:"type"` // Vote type @@ -564,21 +553,95 @@ type VoteDetails struct { Options []VoteOption `json:"options"` } +// VoteDetails contains the details of a proposal vote. +// +// Signature is the client signature of the SHA256 digest of the JSON encoded +// Vote struct. +type VoteDetails struct { + Params VoteParams `json:"params"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` + StartBlockHeight uint32 `json:"startblockheight"` + StartBlockHash string `json:"startblockhash"` + EndBlockHeight uint32 `json:"endblockheight"` + EligibleTickets []string `json:"eligibletickets"` // Ticket hashes +} + +// ProposalVote contains all vote authorizations and the vote details for a +// proposal vote. The vote details will be null if the proposal vote has not +// been started yet. +type ProposalVote struct { + Auths []AuthorizeDetails `json:"auths"` + Vote *VoteDetails `json:"vote"` +} + +// CastVoteDetails contains the details of a cast vote. +type CastVoteDetails struct { + Token string `json:"token"` // Record token + Ticket string `json:"ticket"` // Ticket hash + VoteBit string `json:"votebits"` // Selected vote bit, hex encoded + Signature string `json:"signature"` // Signature of Token+Ticket+VoteBit + Receipt string `json:"receipt"` // Server signature of client signature +} + +// VoteResult describes a vote option and the total number of votes that have +// been cast for this option. +type VoteResult struct { + ID string `json:"id"` // Single unique word (e.g. yes) + Description string `json:"description"` // Longer description of the vote + VoteBit uint64 `json:"votebit"` // Bits used for this option + Votes uint64 `json:"votes"` // Votes cast for this option +} + +// VoteSummary summarizes the vote params and results of a proposal vote. +type VoteSummary struct { + Type VoteT `json:"type"` + Status VoteStatusT `json:"status"` + Duration uint32 `json:"duration"` + StartBlockHeight uint32 `json:"startblockheight"` + StartBlockHash string `json:"startblockhash"` + EndBlockHeight uint32 `json:"endblockheight"` + EligibleTickets uint32 `json:"eligibletickets"` + QuorumPercentage uint32 `json:"quorumpercentage"` + PassPercentage uint32 `json:"passpercentage"` + Results []VoteResult `json:"results"` + Approved bool `json:"approved"` +} + +// VoteAuthorize authorizes a proposal vote or revokes a previous vote +// authorization. All proposal votes must be authorized by the proposal author +// before an admin is able to start the voting process. +// +// Signature contains the client signature of the Token+Version+Action. +type VoteAuthorize struct { + Token string `json:"token"` + Version uint32 `json:"version"` + Action VoteAuthActionT `json:"action"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` +} + +// VoteAuthorizeReply is the reply to the VoteAuthorize command. +// +// Receipt is the server signature of the client signature. This is proof that +// the server received and processed the VoteAuthorize command. +type VoteAuthorizeReply struct { + Timestamp int64 `json:"timestamp"` + Receipt string `json:"receipt"` +} + // VoteStart starts a proposal vote. All proposal votes must be authorized // by the proposal author before an admin is able to start the voting process. // -// Signature is the signature of a SHA256 digest of the JSON -// encoded Vote structure. +// Signature is the signature of a SHA256 digest of the JSON encoded Vote +// structure. type VoteStart struct { - Vote VoteDetails `json:"vote"` - PublicKey string `json:"publickey"` - Signature string `json:"signature"` + Params VoteParams `json:"params"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` } // VoteStartReply is the reply to the VoteStart command. -// -// Receipt is the server signature of the client signature. This is proof that -// the server received and processed the VoteStart command. type VoteStartReply struct { StartBlockHeight uint32 `json:"startblockheight"` StartBlockHash string `json:"startblockhash"` @@ -592,11 +655,21 @@ type VoteStartRunoffReply struct{} type VoteBallot struct{} type VoteBallotReply struct{} -type Votes struct{} -type VotesReply struct{} +type Votes struct { + Tokens []string `json:"tokens"` +} + +type VotesReply struct { + Votes map[string]ProposalVote `json:"votes"` +} -type VoteResults struct{} -type VoteResultsReply struct{} +type VoteResults struct { + Token string `json:"token"` +} + +type VoteResultsReply struct { + Votes []CastVoteDetails `json:"votes"` +} type VoteSummaries struct{} type VoteSummariesReply struct{} diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index f52ae3867..ac0b7a8aa 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -1834,8 +1834,8 @@ func convertVoteTypeFromPi(t pi.VoteT) ticketvote.VoteT { return ticketvote.VoteTypeInvalid } -func convertVoteDetailsFromPi(v pi.VoteDetails) ticketvote.VoteDetails { - tv := ticketvote.VoteDetails{ +func convertVoteParamsFromPi(v pi.VoteParams) ticketvote.VoteParams { + tv := ticketvote.VoteParams{ Token: v.Token, Version: v.Version, Type: convertVoteTypeFromPi(v.Type), @@ -1859,11 +1859,11 @@ func convertVoteDetailsFromPi(v pi.VoteDetails) ticketvote.VoteDetails { } func (p *politeiawww) processVoteStart(vs pi.VoteStart) (*pi.VoteStartReply, error) { - log.Tracef("processVoteStart: %v", vs.Vote.Token) + log.Tracef("processVoteStart: %v", vs.Params.Token) // Call the ticketvote plugin to start vote reply, err := p.voteStart(ticketvote.Start{ - Vote: convertVoteDetailsFromPi(vs.Vote), + Params: convertVoteParamsFromPi(vs.Params), PublicKey: vs.PublicKey, Signature: vs.Signature, }) diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 7ec428343..a49bcee32 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -285,30 +285,31 @@ func (p *politeiawww) processVoteResults(token string) (*www.VoteResultsReply, e } // Convert reply to www + startHeight := strconv.FormatUint(uint64(vd.Vote.StartBlockHeight), 10) + endHeight := strconv.FormatUint(uint64(vd.Vote.EndBlockHeight), 10) res := www.VoteResultsReply{ StartVote: www.StartVote{ PublicKey: vd.Vote.PublicKey, Signature: vd.Vote.Signature, Vote: www.Vote{ - Token: vd.Vote.Vote.Token, - Mask: vd.Vote.Vote.Mask, - Duration: vd.Vote.Vote.Duration, - QuorumPercentage: vd.Vote.Vote.QuorumPercentage, - PassPercentage: vd.Vote.Vote.PassPercentage, + Token: vd.Vote.Params.Token, + Mask: vd.Vote.Params.Mask, + Duration: vd.Vote.Params.Duration, + QuorumPercentage: vd.Vote.Params.QuorumPercentage, + PassPercentage: vd.Vote.Params.PassPercentage, }, }, StartVoteReply: www.StartVoteReply{ - StartBlockHeight: strconv.FormatUint(uint64(vd.Vote.StartBlockHeight), - 10), - StartBlockHash: vd.Vote.StartBlockHash, - EndHeight: strconv.FormatUint(uint64(vd.Vote.EndBlockHeight), 10), - EligibleTickets: vd.Vote.EligibleTickets, + StartBlockHeight: startHeight, + StartBlockHash: vd.Vote.StartBlockHash, + EndHeight: endHeight, + EligibleTickets: vd.Vote.EligibleTickets, }, } // Transalte vote options - vo := make([]www.VoteOption, 0, len(vd.Vote.Vote.Options)) - for _, o := range vd.Vote.Vote.Options { + vo := make([]www.VoteOption, 0, len(vd.Vote.Params.Options)) + for _, o := range vd.Vote.Params.Options { vo = append(vo, www.VoteOption{ Id: o.ID, Description: o.Description, diff --git a/politeiawww/ticketvote.go b/politeiawww/ticketvote.go index c17e095df..a6fd4d239 100644 --- a/politeiawww/ticketvote.go +++ b/politeiawww/ticketvote.go @@ -103,9 +103,9 @@ func (p *politeiawww) ballot(ballot *www.Ballot) (*ticketvote.BallotReply, error var bp ticketvote.Ballot // Transale votes - votes := make([]ticketvote.Vote, 0, len(ballot.Votes)) + votes := make([]ticketvote.CastVote, 0, len(ballot.Votes)) for _, vote := range ballot.Votes { - votes = append(votes, ticketvote.Vote{ + votes = append(votes, ticketvote.CastVote{ Token: vote.Ticket, Ticket: vote.Ticket, VoteBit: vote.VoteBit, diff --git a/politeiawww/www.go b/politeiawww/www.go index 2e841dd31..b2646bfe4 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -127,7 +127,7 @@ func convertWWWErrorStatusFromTicketVote(e ticketvote.ErrorStatusT) www.ErrorSta return www.ErrorStatusProposalNotFound case ticketvote.ErrorStatusRecordStatusInvalid: return www.ErrorStatusWrongStatus - case ticketvote.ErrorStatusVoteDetailsInvalid: + case ticketvote.ErrorStatusVoteParamsInvalid: return www.ErrorStatusInvalidPropVoteParams case ticketvote.ErrorStatusVoteStatusInvalid: return www.ErrorStatusInvalidPropVoteStatus @@ -237,8 +237,8 @@ func convertPiErrorStatusFromTicketVote(e ticketvote.ErrorStatusT) pi.ErrorStatu return pi.ErrorStatusPropNotFound case ticketvote.ErrorStatusRecordStatusInvalid: return pi.ErrorStatusPropStatusInvalid - case ticketvote.ErrorStatusVoteDetailsInvalid: - return pi.ErrorStatusVoteDetailsInvalid + case ticketvote.ErrorStatusVoteParamsInvalid: + return pi.ErrorStatusVoteParamsInvalid case ticketvote.ErrorStatusVoteStatusInvalid: return pi.ErrorStatusVoteStatusInvalid case ticketvote.ErrorStatusBallotInvalid: From b71d243a130273d2a85b8384116d9d6a4f42dc1b Mon Sep 17 00:00:00 2001 From: Amir Massarwa Date: Wed, 23 Sep 2020 16:07:47 +0300 Subject: [PATCH 095/449] politeiawww: Add processVoteStartRunoff. --- politeiawww/api/pi/v1/v1.go | 17 +++- politeiawww/cmd/piwww/help.go | 4 +- politeiawww/cmd/piwww/piwww.go | 6 +- politeiawww/cmd/piwww/votestart.go | 4 +- ...{startvoterunoff.go => votestartrunoff.go} | 54 +++++------ politeiawww/cmd/shared/client.go | 18 ++-- politeiawww/piwww.go | 89 ++++++++++++++++--- politeiawww/ticketvote.go | 20 +++++ 8 files changed, 156 insertions(+), 56 deletions(-) rename politeiawww/cmd/piwww/{startvoterunoff.go => votestartrunoff.go} (78%) diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index d1441c98a..146ed3d40 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -649,8 +649,21 @@ type VoteStartReply struct { EligibleTickets []string `json:"eligibletickets"` } -type VoteStartRunoff struct{} -type VoteStartRunoffReply struct{} +// VoteStartRunoff starts a runoff vote between the provided submissions. Each +// submission is required to have its own Authorize and Start. +type VoteStartRunoff struct { + Token string `json:"token"` // RFP token + Authorizations []VoteAuthorize `json:"authorizations"` + Starts []VoteStart `json:"starts"` +} + +// VoteStartRunoffReply is the reply to the VoteStartRunoff command. +type VoteStartRunoffReply struct { + StartBlockHeight uint32 `json:"startblockheight"` + StartBlockHash string `json:"startblockhash"` + EndBlockHeight uint32 `json:"endblockheight"` + EligibleTickets []string `json:"eligibletickets"` // Ticket hashes +} type VoteBallot struct{} type VoteBallotReply struct{} diff --git a/politeiawww/cmd/piwww/help.go b/politeiawww/cmd/piwww/help.go index e52391941..173dc5f66 100644 --- a/politeiawww/cmd/piwww/help.go +++ b/politeiawww/cmd/piwww/help.go @@ -103,8 +103,8 @@ func (cmd *HelpCmd) Execute(args []string) error { fmt.Printf("%s\n", voteAuthorizeHelpMsg) case "votestart": fmt.Printf("%s\n", voteStartHelpMsg) - case "startvoterunoff": - fmt.Printf("%s\n", startVoteRunoffHelpMsg) + case "votestartrunoff": + fmt.Printf("%s\n", voteStartRunoffHelpMsg) case "voteresults": fmt.Printf("%s\n", voteResultsHelpMsg) case "activevotes": diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index 52ff38f6b..c5ca53ad9 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -57,8 +57,9 @@ type piwww struct { CommentVotes CommentVotesCmd `command:"commentvotes" description:"(user) get comment upvotes/downvotes for a proposal from the provided user"` // Vote commands - VoteStart VoteStartCmd `command:"votestart" description:"(admin) start the voting period on a proposal"` - VoteAuthorize VoteAuthorizeCmd `command:"voteauthorize" description:"(user) authorize a proposal vote (must be proposal author)"` + VoteStartRunoff VoteStartRunoffCmd `command:"votestartrunoff" description:"(admin) start a runoff using the submissions to an RFP"` + VoteStart VoteStartCmd `command:"votestart" description:"(admin) start the voting period on a proposal"` + VoteAuthorize VoteAuthorizeCmd `command:"voteauthorize" description:"(user) authorize a proposal vote (must be proposal author)"` // Commands ActiveVotes ActiveVotesCmd `command:"activevotes" description:"(public) get the proposals that are being voted on"` @@ -83,7 +84,6 @@ type piwww struct { Secret shared.SecretCmd `command:"secret" description:"(user) ping politeiawww"` SendFaucetTx SendFaucetTxCmd `command:"sendfaucettx" description:" send a DCR transaction using the Decred testnet faucet"` SetTOTP shared.SetTOTPCmd `command:"settotp" description:"(user) set the key for TOTP"` - StartVoteRunoff StartVoteRunoffCmd `command:"startvoterunoff" description:"(admin) start a runoff using the submissions to an RFP"` Subscribe SubscribeCmd `command:"subscribe" description:"(public) subscribe to all websocket commands and do not exit tool"` Tally TallyCmd `command:"tally" description:"(public) get the vote tally for a proposal"` TestRun TestRunCmd `command:"testrun" description:" run a series of tests on the politeiawww routes (dev use only)"` diff --git a/politeiawww/cmd/piwww/votestart.go b/politeiawww/cmd/piwww/votestart.go index e099e0e39..9b2d26630 100644 --- a/politeiawww/cmd/piwww/votestart.go +++ b/politeiawww/cmd/piwww/votestart.go @@ -72,7 +72,7 @@ func (cmd *VoteStartCmd) Execute(args []string) error { } // Create VoteStart - vote := pi.VoteDetails{ + vote := pi.VoteParams{ Token: cmd.Args.Token, Version: uint32(version), Type: pi.VoteTypeStandard, @@ -101,7 +101,7 @@ func (cmd *VoteStartCmd) Execute(args []string) error { msg := hex.EncodeToString(util.Digest(vb)) sig := cfg.Identity.SignMessage([]byte(msg)) vs := pi.VoteStart{ - Vote: vote, + Params: vote, PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), Signature: hex.EncodeToString(sig[:]), } diff --git a/politeiawww/cmd/piwww/startvoterunoff.go b/politeiawww/cmd/piwww/votestartrunoff.go similarity index 78% rename from politeiawww/cmd/piwww/startvoterunoff.go rename to politeiawww/cmd/piwww/votestartrunoff.go index 37b5370d4..afb5c7f63 100644 --- a/politeiawww/cmd/piwww/startvoterunoff.go +++ b/politeiawww/cmd/piwww/votestartrunoff.go @@ -10,19 +10,19 @@ import ( "strconv" "github.com/decred/politeia/decredplugin" + pi "github.com/decred/politeia/politeiawww/api/pi/v1" v1 "github.com/decred/politeia/politeiawww/api/www/v1" - v2 "github.com/decred/politeia/politeiawww/api/www/v2" "github.com/decred/politeia/politeiawww/cmd/shared" "github.com/decred/politeia/util" ) -// StartVoteRunoffCmd starts the voting period on all public submissions to a +// VoteStartRunoffCmd starts the voting period on all public submissions to a // request for proposals (RFP). // // The QuorumPercentage and PassPercentage are strings and not uint32 so that a // value of 0 can be passed in and not be overwritten by the defaults. This is // sometimes desirable when testing. -type StartVoteRunoffCmd struct { +type VoteStartRunoffCmd struct { Args struct { TokenRFP string `positional-arg-name:"token" required:"true"` // RFP censorship token Duration uint32 `positional-arg-name:"duration"` // Duration in blocks @@ -32,7 +32,7 @@ type StartVoteRunoffCmd struct { } // Execute executes the StartVoteRunoff command. -func (cmd *StartVoteRunoffCmd) Execute(args []string) error { +func (cmd *VoteStartRunoffCmd) Execute(args []string) error { // Check for user identity if cfg.Identity == nil { return shared.ErrUserIdentityNotFound @@ -85,13 +85,13 @@ func (cmd *StartVoteRunoffCmd) Execute(args []string) error { } } - // Prepare AuthorizeVote for each submission - authVotes := make([]v2.AuthorizeVote, 0, len(submissions)) + // Prepare VoteAuthorize for each submission + auths := make([]pi.VoteAuthorize, 0, len(submissions)) for _, v := range submissions { - action := v2.AuthVoteActionAuthorize - msg := v.CensorshipRecord.Token + v.Version + action + action := pi.VoteAuthActionAuthorize + msg := v.CensorshipRecord.Token + v.Version + string(action) sig := cfg.Identity.SignMessage([]byte(msg)) - authVotes = append(authVotes, v2.AuthorizeVote{ + auths = append(auths, pi.VoteAuthorize{ Token: v.CensorshipRecord.Token, Action: action, PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), @@ -99,32 +99,32 @@ func (cmd *StartVoteRunoffCmd) Execute(args []string) error { }) } - // Prepare StartVote for each submission - startVotes := make([]v2.StartVote, 0, len(submissions)) + // Prepare VoteStart for each submission + starts := make([]pi.VoteStart, 0, len(submissions)) for _, v := range submissions { version, err := strconv.ParseUint(v.Version, 10, 32) if err != nil { return err } - vote := v2.Vote{ + vote := pi.VoteParams{ Token: v.CensorshipRecord.Token, - ProposalVersion: uint32(version), - Type: v2.VoteTypeRunoff, + Version: uint32(version), + Type: pi.VoteTypeRunoff, Mask: 0x03, // bit 0 no, bit 1 yes Duration: duration, QuorumPercentage: quorum, PassPercentage: pass, - Options: []v2.VoteOption{ + Options: []pi.VoteOption{ { - Id: decredplugin.VoteOptionIDApprove, + ID: decredplugin.VoteOptionIDApprove, Description: "Approve proposal", - Bits: 0x01, + Bit: 0x01, }, { - Id: decredplugin.VoteOptionIDReject, + ID: decredplugin.VoteOptionIDReject, Description: "Don't approve proposal", - Bits: 0x02, + Bit: 0x02, }, }, } @@ -135,24 +135,24 @@ func (cmd *StartVoteRunoffCmd) Execute(args []string) error { msg := hex.EncodeToString(util.Digest(vb)) sig := cfg.Identity.SignMessage([]byte(msg)) - startVotes = append(startVotes, v2.StartVote{ - Vote: vote, + starts = append(starts, pi.VoteStart{ + Params: vote, PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), Signature: hex.EncodeToString(sig[:]), }) } // Prepare and send request - svr := v2.StartVoteRunoff{ + svr := pi.VoteStartRunoff{ Token: cmd.Args.TokenRFP, - AuthorizeVotes: authVotes, - StartVotes: startVotes, + Authorizations: auths, + Starts: starts, } err = shared.PrintJSON(svr) if err != nil { return err } - svrr, err := client.StartVoteRunoffV2(svr) + svrr, err := client.VoteStartRunoff(svr) if err != nil { return err } @@ -170,8 +170,8 @@ func (cmd *StartVoteRunoffCmd) Execute(args []string) error { return nil } -// startVoteRunoffHelpMsg is the help command output for 'startvoterunoff'. -var startVoteRunoffHelpMsg = `startvoterunoff +// voteStartRunoffHelpMsg is the help command output for 'votestartrunoff'. +var voteStartRunoffHelpMsg = `votestartrunoff Start the voting period on all public submissions to an RFP proposal. The optional arguments must either all be used or none be used. diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index 9dc6c6020..193527536 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -1354,30 +1354,30 @@ func (c *Client) VoteStart(vs pi.VoteStart) (*pi.VoteStartReply, error) { return &vsr, nil } -// StartVoteRunoffV2 sends the given StartVoteRunoff to the politeiawww v2 -// StartVoteRunoffRoute and returns the reply. -func (c *Client) StartVoteRunoffV2(svr www2.StartVoteRunoff) (*www2.StartVoteRunoffReply, error) { +// VoteStartRunoff sends the given VoteStartRunoff to the pi api +// RouteVoteStartRunoff and returns the reply. +func (c *Client) VoteStartRunoff(vsr pi.VoteStartRunoff) (*pi.VoteStartRunoffReply, error) { responseBody, err := c.makeRequest(http.MethodPost, - www2.APIRoute, www2.RouteStartVoteRunoff, svr) + pi.APIRoute, pi.RouteVoteStartRunoff, vsr) if err != nil { return nil, err } - var svrr www2.StartVoteRunoffReply - err = json.Unmarshal(responseBody, &svrr) + var vsrr pi.VoteStartRunoffReply + err = json.Unmarshal(responseBody, &vsrr) if err != nil { return nil, err } if c.cfg.Verbose { - svrr.EligibleTickets = []string{"removed by piwww for readability"} - err := prettyPrintJSON(svr) + vsrr.EligibleTickets = []string{"removed by piwww for readability"} + err := prettyPrintJSON(vsrr) if err != nil { return nil, err } } - return &svrr, nil + return &vsrr, nil } // VerifyUserPayment checks whether the logged in user has paid their user diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index ac0b7a8aa..cd2900649 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -1781,17 +1781,21 @@ func convertVoteAuthActionFromPi(a pi.VoteAuthActionT) ticketvote.AuthActionT { } } -func (p *politeiawww) processVoteAuthorize(va pi.VoteAuthorize) (*pi.VoteAuthorizeReply, error) { - log.Tracef("processVoteAuthorize: %v", va.Token) - - // Call the ticketvote plugin to authorize vote - reply, err := p.voteAuthorize(ticketvote.Authorize{ +func convertVoteAuthorizeFromPi(va pi.VoteAuthorize) ticketvote.Authorize { + return ticketvote.Authorize{ Token: va.Token, Version: va.Version, Action: convertVoteAuthActionFromPi(va.Action), PublicKey: va.PublicKey, Signature: va.Signature, - }) + } +} + +func (p *politeiawww) processVoteAuthorize(va pi.VoteAuthorize) (*pi.VoteAuthorizeReply, error) { + log.Tracef("processVoteAuthorize: %v", va.Token) + + // Call the ticketvote plugin to authorize vote + reply, err := p.voteAuthorize(convertVoteAuthorizeFromPi(va)) if err != nil { return nil, err } @@ -1858,15 +1862,19 @@ func convertVoteParamsFromPi(v pi.VoteParams) ticketvote.VoteParams { return tv } +func convertVoteStartFromPi(vs pi.VoteStart) ticketvote.Start { + return ticketvote.Start{ + Params: convertVoteParamsFromPi(vs.Params), + PublicKey: vs.PublicKey, + Signature: vs.Signature, + } +} + func (p *politeiawww) processVoteStart(vs pi.VoteStart) (*pi.VoteStartReply, error) { log.Tracef("processVoteStart: %v", vs.Params.Token) // Call the ticketvote plugin to start vote - reply, err := p.voteStart(ticketvote.Start{ - Params: convertVoteParamsFromPi(vs.Params), - PublicKey: vs.PublicKey, - Signature: vs.Signature, - }) + reply, err := p.voteStart(convertVoteStartFromPi(vs)) if err != nil { return nil, err } @@ -1901,6 +1909,62 @@ func (p *politeiawww) handleVoteStart(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, vsr) } +func (p *politeiawww) processVoteStartRunoff(cvr pi.VoteStartRunoff) (*pi.VoteStartRunoffReply, error) { + log.Tracef("processVoteStartRunoff: %v", cvr.Token) + + // Call the ticketvote plugin to start runoff vote + // Transalte payload + pcvr := ticketvote.StartRunoff{ + Token: cvr.Token, + } + // Transalte submissions' vote authorizations structs + auths := make([]ticketvote.Authorize, 0, len(cvr.Authorizations)) + for _, auth := range cvr.Authorizations { + auths = append(auths, convertVoteAuthorizeFromPi(auth)) + } + pcvr.Auths = auths + // Transate submissions' vote start structs + starts := make([]ticketvote.Start, 0, len(cvr.Starts)) + for _, s := range cvr.Starts { + starts = append(starts, convertVoteStartFromPi(s)) + } + pcvr.Starts = starts + + reply, err := p.voteStartRunoff(pcvr) + if err != nil { + return nil, err + } + + return &pi.VoteStartRunoffReply{ + StartBlockHeight: reply.StartBlockHeight, + StartBlockHash: reply.StartBlockHash, + EndBlockHeight: reply.EndBlockHeight, + EligibleTickets: reply.EligibleTickets, + }, nil +} + +func (p *politeiawww) handleVoteStartRunoff(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleVoteStartRunoff") + + var vsr pi.VoteStartRunoff + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&vsr); err != nil { + respondWithPiError(w, r, "handleVoteStartRunoff: unmarshal", + pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusInvalidInput, + }) + return + } + + vsrr, err := p.processVoteStartRunoff(vsr) + if err != nil { + respondWithPiError(w, r, + "handleVoteStartRunoff: processVoteStartRunoff: %v", err) + } + + util.RespondWithJSON(w, http.StatusOK, vsrr) +} + func (p *politeiawww) setPiRoutes() { // Public routes p.addRoute(http.MethodGet, pi.APIRoute, @@ -1916,6 +1980,9 @@ func (p *politeiawww) setPiRoutes() { p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteVoteStart, p.handleVoteStart, permissionPublic) + p.addRoute(http.MethodPost, pi.APIRoute, + pi.RouteVoteStartRunoff, p.handleVoteStartRunoff, permissionPublic) + // Logged in routes p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteProposalNew, p.handleProposalNew, diff --git a/politeiawww/ticketvote.go b/politeiawww/ticketvote.go index a6fd4d239..bfffaa253 100644 --- a/politeiawww/ticketvote.go +++ b/politeiawww/ticketvote.go @@ -9,6 +9,26 @@ import ( www "github.com/decred/politeia/politeiawww/api/www/v1" ) +func (p *politeiawww) voteStartRunoff(vsr ticketvote.StartRunoff) (*ticketvote.StartRunoffReply, error) { + // Prep plugin command + payload, err := ticketvote.EncodeStartRunoff(vsr) + if err != nil { + return nil, err + } + + r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdStartRunoff, "", + string(payload)) + if err != nil { + return nil, err + } + vsrr, err := ticketvote.DecodeStartRunoffReply([]byte(r)) + if err != nil { + return nil, err + } + + return vsrr, nil +} + func (p *politeiawww) voteStart(vs ticketvote.Start) (*ticketvote.StartReply, error) { // Prep plugin command payload, err := ticketvote.EncodeStart(vs) From b39fe28900dc2140b264fe900116d064a3e54201 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 23 Sep 2020 10:10:45 -0500 Subject: [PATCH 096/449] make vote details cmd a batched call --- plugins/pi/pi.go | 3 +- plugins/ticketvote/ticketvote.go | 41 ++++++++++----- politeiad/backend/tlogbe/ticketvote.go | 70 ++++++++++++++------------ politeiawww/api/pi/v1/v1.go | 40 +++++++++------ politeiawww/api/www/v2/api.md | 7 +++ politeiawww/api/www/v2/v2.go | 2 +- politeiawww/templates.go | 2 +- 7 files changed, 104 insertions(+), 61 deletions(-) diff --git a/plugins/pi/pi.go b/plugins/pi/pi.go index 51304f4b8..9d53c3d47 100644 --- a/plugins/pi/pi.go +++ b/plugins/pi/pi.go @@ -61,13 +61,14 @@ const ( VoteUpvote VoteT = 1 // User error status codes - // TODO number error codes + // TODO number error codes and add human readable error messages ErrorStatusInvalid ErrorStatusT = iota ErrorStatusPropVersionInvalid ErrorStatusPropStatusInvalid ErrorStatusPropStatusChangeInvalid ErrorStatusPropLinkToInvalid ErrorStatusVoteStatusInvalid + ErrorStatusPageSizeExceeded ) var ( diff --git a/plugins/ticketvote/ticketvote.go b/plugins/ticketvote/ticketvote.go index 4dc201140..38dbfc3f8 100644 --- a/plugins/ticketvote/ticketvote.go +++ b/plugins/ticketvote/ticketvote.go @@ -28,12 +28,17 @@ const ( CmdStart = "start" // Start a vote CmdStartRunoff = "startrunoff" // Start a runoff vote CmdBallot = "ballot" // Cast a ballot of votes - CmdDetails = "details" // Get details of a vote + CmdDetails = "details" // Get vote details CmdCastVotes = "castvotes" // Get cast votes CmdSummaries = "summaries" // Get vote summaries CmdInventory = "inventory" // Get inventory grouped by vote status CmdProofs = "proofs" // Get inclusion proofs + // TODO implement PolicyVotesPageSize + // PolicyVotesPageSize is the maximum number of results that can be + // returned from any of the batched vote commands. + PolicyVotesPageSize = 20 + // Vote statuses VoteStatusInvalid VoteStatusT = 0 // Invalid status VoteStatusUnauthorized VoteStatusT = 1 // Vote cannot be started @@ -103,6 +108,7 @@ const ( ErrorStatusVoteParamsInvalid ErrorStatusT = 7 ErrorStatusVoteStatusInvalid ErrorStatusT = 8 ErrorStatusBallotInvalid ErrorStatusT = 9 + ErrorStatusPageSizeExceeded ErrorStatusT = 10 ) var ( @@ -139,6 +145,7 @@ var ( ErrorStatusVoteParamsInvalid: "vote params invalid", ErrorStatusVoteStatusInvalid: "vote status invalid", ErrorStatusBallotInvalid: "ballot invalid", + ErrorStatusPageSizeExceeded: "page size exceeded", } ) @@ -153,13 +160,6 @@ func (e UserErrorReply) Error() string { return fmt.Sprintf("ticketvote plugin error code: %v", e.ErrorCode) } -// VoteOption describes a single vote option. -type VoteOption struct { - ID string `json:"id"` // Single, unique word (e.g. yes) - Description string `json:"description"` // Longer description of the vote - Bit uint64 `json:"bit"` // Bit used for this option -} - // AuthorizeDetails is the structure that is saved to disk when a vote is // authorized or a previous authorization is revoked. It contains all the // fields from a Authorize and a AuthorizeReply. @@ -176,6 +176,13 @@ type AuthorizeDetails struct { Receipt string `json:"receipt"` // Server signature of client signature } +// VoteOption describes a single vote option. +type VoteOption struct { + ID string `json:"id"` // Single, unique word (e.g. yes) + Description string `json:"description"` // Longer description of the vote + Bit uint64 `json:"bit"` // Bit used for this option +} + // VoteParams describes the options and parameters of a ticket vote. type VoteParams struct { Token string `json:"token"` // Record token @@ -437,9 +444,9 @@ func DecodeBallotReply(payload []byte) (*BallotReply, error) { return &b, nil } -// Details requests the vote details for the specified record token. +// Details returns the vote details for each of the provided record tokens. type Details struct { - Token string `json:"token"` + Tokens []string `json:"tokens"` } // EncodeDetails encodes a Details into a JSON byte slice. @@ -457,13 +464,21 @@ func DecodeDetails(payload []byte) (*Details, error) { return &d, nil } -// DetailsReply is the reply to the Details command. The VoteDetails will be -// nil if the vote has been started. -type DetailsReply struct { +// RecordVote contains all vote authorizations and the vote details for a +// record. The VoteDetails will be nil if the vote has been started. +type RecordVote struct { Auths []AuthorizeDetails `json:"auths"` Vote *VoteDetails `json:"vote"` } +// DetailsReply is the reply to the Details command. The returned map will not +// contain an entry for any tokens that did not correspond to an actual record. +// It is the callers responsibility to ensure that a entry is returned for all +// of the provided tokens. +type DetailsReply struct { + Votes map[string]RecordVote `json:"votes"` +} + // EncodeDetailsReply encodes a DetailsReply into a JSON byte slice. func EncodeDetailsReply(dr DetailsReply) ([]byte, error) { return json.Marshal(dr) diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index b56e8eb0b..6acfbac89 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -483,7 +483,7 @@ func (p *ticketVotePlugin) authorizes(token []byte) ([]ticketvote.AuthorizeDetai return auths, nil } -func (p *ticketVotePlugin) startSave(vd ticketvote.VoteDetails) error { +func (p *ticketVotePlugin) voteSave(vd ticketvote.VoteDetails) error { token, err := hex.DecodeString(vd.Params.Token) if err != nil { return err @@ -517,8 +517,7 @@ func (p *ticketVotePlugin) startSave(vd ticketvote.VoteDetails) error { return nil } -// startDetails returns the VoteDetails for the provided record if one exists. -func (p *ticketVotePlugin) startDetails(token []byte) (*ticketvote.VoteDetails, error) { +func (p *ticketVotePlugin) voteDetails(token []byte) (*ticketvote.VoteDetails, error) { // Retrieve blobs blobs, err := p.tlog.blobsByKeyPrefix(tlogIDVetted, token, keyPrefixVoteDetails) @@ -527,14 +526,14 @@ func (p *ticketVotePlugin) startDetails(token []byte) (*ticketvote.VoteDetails, } switch len(blobs) { case 0: - // A start vote does not exist + // A vote details does not exist return nil, nil case 1: - // A start vote exists; continue + // A vote details exists; continue default: // This should not happen. There should only ever be a max of - // one start vote. - return nil, fmt.Errorf("multiple start votes found (%v) for record %x", + // one vote details. + return nil, fmt.Errorf("multiple vote detailss found (%v) for record %x", len(blobs), token) } @@ -1199,7 +1198,7 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { return "", err } - // Any previous start vote must be retrieved to verify that a vote + // Any previous vote details must be retrieved to verify that a vote // has not already been started. The lock must be held for the // remainder of this function. m := p.mutex(s.Params.Token) @@ -1207,7 +1206,7 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { defer m.Unlock() // Verify vote has not already been started - svp, err := p.startDetails(token) + svp, err := p.voteDetails(token) if err != nil { return "", err } @@ -1219,7 +1218,7 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { } } - // Prepare start vote + // Prepare vote details vd := ticketvote.VoteDetails{ Params: s.Params, PublicKey: s.PublicKey, @@ -1230,8 +1229,8 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { EligibleTickets: sr.EligibleTickets, } - // Save start vote - err = p.startSave(vd) + // Save vote details + err = p.voteSave(vd) if err != nil { return "", fmt.Errorf("startSave: %v", err) } @@ -1322,7 +1321,7 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { } // Verify record vote status - vd, err := p.startDetails(token) + vd, err := p.voteDetails(token) if err != nil { return "", err } @@ -1497,30 +1496,39 @@ func (p *ticketVotePlugin) cmdDetails(payload string) (string, error) { return "", err } - // Verify token - token, err := hex.DecodeString(d.Token) - if err != nil { - return "", ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusTokenInvalid, + votes := make(map[string]ticketvote.RecordVote, len(d.Tokens)) + for _, v := range d.Tokens { + // Verify token + token, err := hex.DecodeString(v) + if err != nil { + continue } - } - // Get authorize votes - auths, err := p.authorizes(token) - if err != nil { - return "", err - } + // Get authorize votes + auths, err := p.authorizes(token) + if err != nil { + if err == errRecordNotFound { + continue + } + return "", fmt.Errorf("authorizes: %v", err) + } - // Get start vote - vd, err := p.startDetails(token) - if err != nil { - return "", fmt.Errorf("startDetails: %v", err) + // Get vote details + vd, err := p.voteDetails(token) + if err != nil { + return "", fmt.Errorf("startDetails: %v", err) + } + + // Add record vote + votes[v] = ticketvote.RecordVote{ + Auths: auths, + Vote: vd, + } } // Prepare rely dr := ticketvote.DetailsReply{ - Auths: auths, - Vote: vd, + Votes: votes, } reply, err := ticketvote.EncodeDetailsReply(dr) if err != nil { @@ -1606,7 +1614,7 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. } // Vote has been authorized. Check if it has been started yet. - vd, err := p.startDetails(token) + vd, err := p.voteDetails(token) if err != nil { return nil, fmt.Errorf("startDetails: %v", err) } diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 146ed3d40..e2c060afc 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -16,12 +16,14 @@ type VoteStatusT int type VoteAuthActionT string type VoteT int +// TODO the plugin policies should be returned in a route +// TODO the proposals route should allow filtering by user ID + const ( APIVersion = 1 - // TODO the plugin policies should be returned in a route - // TODO the proposals route should allow filtering by user ID - // TODO max page sizes should be added to RouteProposals + // APIRoute is prefixed onto all routes. + APIRoute = "/v1" // Proposal routes RouteProposalNew = "/proposal/new" @@ -47,6 +49,11 @@ const ( RouteVoteSummaries = "/votes/summaries" RouteVoteInventory = "/votes/inventory" + // TODO implement PolicyProposalPageSize + // PolicyProposalsPageSize is the maximum number of results that can + // be returned from any of the batched proposal commands. + PolicyProposalsPageSize = 10 + // Proposal states. A proposal state can be either unvetted or // vetted. The PropStatusT type further breaks down these two // states into more granular statuses. @@ -137,6 +144,7 @@ const ( ErrorStatusPropStatusInvalid ErrorStatusT = 222 ErrorStatusPropStatusChangeInvalid ErrorStatusT = 223 ErrorStatusPropStatusChangeReasonInvalid ErrorStatusT = 224 + ErrorStatusPropPageSizeExceeded ErrorStatusT = 225 // Comment errors // TODO number error codes @@ -150,12 +158,10 @@ const ( ErrorStatusVoteStatusInvalid ErrorStatusVoteParamsInvalid ErrorStatusBallotInvalid + ErrorStatusVotePageSizeExceeded ) var ( - // APIRoute is the prefix to all API routes. - APIRoute = fmt.Sprintf("/v%v", APIVersion) - // ErrorStatus contains human readable error messages. // TODO fill in error status messages ErrorStatus = map[ErrorStatusT]string{ @@ -567,14 +573,6 @@ type VoteDetails struct { EligibleTickets []string `json:"eligibletickets"` // Ticket hashes } -// ProposalVote contains all vote authorizations and the vote details for a -// proposal vote. The vote details will be null if the proposal vote has not -// been started yet. -type ProposalVote struct { - Auths []AuthorizeDetails `json:"auths"` - Vote *VoteDetails `json:"vote"` -} - // CastVoteDetails contains the details of a cast vote. type CastVoteDetails struct { Token string `json:"token"` // Record token @@ -668,10 +666,24 @@ type VoteStartRunoffReply struct { type VoteBallot struct{} type VoteBallotReply struct{} +// ProposalVote contains all vote authorizations and the vote details for a +// proposal vote. The vote details will be null if the proposal vote has not +// been started yet. +type ProposalVote struct { + Auths []AuthorizeDetails `json:"auths"` + Vote *VoteDetails `json:"vote"` +} + +// Votes returns the vote authorizations and vote details for each of the +// provided proposal tokens. type Votes struct { Tokens []string `json:"tokens"` } +// VoteReply is the reply to the Votes command. The returned map will not +// contain an entry for any tokens that did not correspond to an actual +// proposal. It is the callers responsibility to ensure that a entry is +// returned for all of the provided tokens. type VotesReply struct { Votes map[string]ProposalVote `json:"votes"` } diff --git a/politeiawww/api/www/v2/api.md b/politeiawww/api/www/v2/api.md index e40613d37..d0741c18a 100644 --- a/politeiawww/api/www/v2/api.md +++ b/politeiawww/api/www/v2/api.md @@ -14,6 +14,8 @@ using a JSON REST API. ### `Start vote` +THIS ROUTE HAS BEEN DEPRECATED. The pi/v1 routes should be used instead. + Start the voting period on the given proposal. Signature is a signature of the hex encoded SHA256 digest of the JSON encoded @@ -116,6 +118,9 @@ Note: eligibletickets is abbreviated for readability. ### `Start vote runoff` + +THIS ROUTE HAS BEEN DEPRECATED. The pi/v1 routes should be used instead. + Start the runoff voting process on all public, non-abandoned RFP submissions for the provided RFP token. @@ -297,6 +302,8 @@ Note: eligibletickets is abbreviated for readability. ### `Vote details` +THIS ROUTE HAS BEEN DEPRECATED. The pi/v1 routes should be used instead. + Vote details returns all of the relevant proposal vote information for the given proposal token. This includes all of the vote parameters and voting options. diff --git a/politeiawww/api/www/v2/v2.go b/politeiawww/api/www/v2/v2.go index e3ed886b4..8a7ac1d83 100644 --- a/politeiawww/api/www/v2/v2.go +++ b/politeiawww/api/www/v2/v2.go @@ -14,7 +14,7 @@ type VoteT int const ( APIVersion = 2 - // All www/v2 routes have been deprecated. The pi/v1 API should be + // All www/v2 routes have been DEPRECATED. The pi/v1 API should be // used instead. RouteStartVote = "/vote/start" RouteStartVoteRunoff = "/vote/startrunoff" diff --git a/politeiawww/templates.go b/politeiawww/templates.go index 038f74050..1509a42ba 100644 --- a/politeiawww/templates.go +++ b/politeiawww/templates.go @@ -80,7 +80,7 @@ Start" button. {{.Name}} {{.Link}} -If you have any questions, drop by the proposals channel on matrix: +If you have any questions, drop by the proposals channel on matrix. https://chat.decred.org/#/room/#proposals:decred.org ` From 7b21199c3a0e4754468fb97bcff4dd8cd5484259 Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Wed, 23 Sep 2020 12:23:09 -0300 Subject: [PATCH 097/449] politeiawww: Refactor user email templates. --- politeiawww/email.go | 44 ++++++------- politeiawww/templates.go | 63 ++++++++++++------- politeiawww/user.go | 16 ++--- .../user/cockroachdb/cockroachdb_test.go | 1 - politeiawww/userwww.go | 10 --- 5 files changed, 72 insertions(+), 62 deletions(-) diff --git a/politeiawww/email.go b/politeiawww/email.go index 908d37ac5..1411042a0 100644 --- a/politeiawww/email.go +++ b/politeiawww/email.go @@ -310,23 +310,23 @@ func (p *politeiawww) emailProposalVoteStartedToAuthor(token, name, username, em return p.smtp.sendEmailTo(subject, body, []string{email}) } -// emailNewUserVerificationLink sends a new user verification email to the +// emailUserEmailVerify sends a new user verification email to the // provided email address. -func (p *politeiawww) emailNewUserVerificationLink(email, token, username string) error { +func (p *politeiawww) emailUserEmailVerify(email, token, username string) error { link, err := p.createEmailLink(www.RouteVerifyNewUser, email, token, username) if err != nil { return err } - tplData := newUserEmailTemplateData{ + tplData := userEmailVerify{ Username: username, Email: email, Link: link, } subject := "Verify Your Email" - body, err := createBody(templateNewUserEmail, tplData) + body, err := createBody(userEmailVerifyTmpl, tplData) if err != nil { return err } @@ -335,9 +335,9 @@ func (p *politeiawww) emailNewUserVerificationLink(email, token, username string return p.smtp.sendEmailTo(subject, body, recipients) } -// emailResetPasswordVerificationLink emails the link with the reset password -// verification token if the email server is set up. -func (p *politeiawww) emailResetPasswordVerificationLink(email, username, token string) error { +// emailUserPasswordReset emails the link with the reset password verification +// token if the email server is set up. +func (p *politeiawww) emailUserPasswordReset(email, username, token string) error { // Setup URL u, err := url.Parse(p.cfg.WebServerAddress + www.RouteResetPassword) if err != nil { @@ -350,11 +350,11 @@ func (p *politeiawww) emailResetPasswordVerificationLink(email, username, token // Setup email subject := "Reset Your Password" - tplData := resetPasswordEmailTemplateData{ + tplData := userPasswordReset{ Email: email, Link: u.String(), } - body, err := createBody(templateResetPasswordEmail, tplData) + body, err := createBody(userPasswordResetTmpl, tplData) if err != nil { return err } @@ -363,22 +363,22 @@ func (p *politeiawww) emailResetPasswordVerificationLink(email, username, token return p.smtp.sendEmailTo(subject, body, []string{email}) } -// emailUpdateUserKeyVerificationLink emails the link with the verification -// token used for setting a new key pair if the email server is set up. -func (p *politeiawww) emailUpdateUserKeyVerificationLink(email, publicKey, token string) error { +// emailUserKeyUpdate emails the link with the verification token used for +// setting a new key pair if the email server is set up. +func (p *politeiawww) emailUserKeyUpdate(email, publicKey, token string) error { link, err := p.createEmailLink(www.RouteVerifyUpdateUserKey, "", token, "") if err != nil { return err } - tplData := updateUserKeyEmailTemplateData{ + tplData := userKeyUpdate{ Email: email, PublicKey: publicKey, Link: link, } subject := "Verify Your New Identity" - body, err := createBody(templateUpdateUserKeyEmail, tplData) + body, err := createBody(userKeyUpdateTmpl, tplData) if err != nil { return err } @@ -390,12 +390,12 @@ func (p *politeiawww) emailUpdateUserKeyVerificationLink(email, publicKey, token // emailUserPasswordChanged notifies the user that his password was changed, // and verifies if he was the author of this action, for security purposes. func (p *politeiawww) emailUserPasswordChanged(email string) error { - tplData := userPasswordChangedTemplateData{ + tplData := userPasswordChanged{ Email: email, } subject := "Password Changed - Security Verification" - body, err := createBody(templateUserPasswordChanged, tplData) + body, err := createBody(userPasswordChangedTmpl, tplData) if err != nil { return err } @@ -404,23 +404,23 @@ func (p *politeiawww) emailUserPasswordChanged(email string) error { return p.smtp.sendEmailTo(subject, body, recipients) } -// emailUserLocked notifies the user its account has been locked and emails the -// link with the reset password verification token if the email server is set -// up. -func (p *politeiawww) emailUserLocked(email string) error { +// emailUserAccountLocked notifies the user its account has been locked and +// emails the link with the reset password verification token if the email +// server is set up. +func (p *politeiawww) emailUserAccountLocked(email string) error { link, err := p.createEmailLink(ResetPasswordGuiRoute, email, "", "") if err != nil { return err } - tplData := userLockedResetPasswordEmailTemplateData{ + tplData := userAccountLocked{ Email: email, Link: link, } subject := "Locked Account - Reset Your Password" - body, err := createBody(templateUserLockedResetPassword, tplData) + body, err := createBody(userAccountLockedTmpl, tplData) if err != nil { return err } diff --git a/politeiawww/templates.go b/politeiawww/templates.go index 1509a42ba..f4a8b5ff5 100644 --- a/politeiawww/templates.go +++ b/politeiawww/templates.go @@ -188,13 +188,15 @@ var proposalVoteStartedToAuthorTmpl = template.Must( Parse(proposalVoteStartedToAuthorText)) // User events -type newUserEmailTemplateData struct { - Username string - Link string - Email string + +// User email verify - Send verification link to new user +type userEmailVerify struct { + Username string // User username + Email string // User email + Link string // Verification link } -const templateNewUserEmailRaw = ` +const userEmailVerifyText = ` Thanks for joining Politeia, {{.Username}}! Click the link below to verify your email and complete your registration: @@ -205,13 +207,17 @@ You are receiving this email because {{.Email}} was used to register for Politei If you did not perform this action, please ignore this email. ` -type updateUserKeyEmailTemplateData struct { - Link string - PublicKey string - Email string +var userEmailVerifyTmpl = template.Must( + template.New("userEmailVerify").Parse(userEmailVerifyText)) + +// User key update - Send key verification link to user +type userKeyUpdate struct { + Link string // Verify key link + PublicKey string // User new public key + Email string // User } -const templateUpdateUserKeyEmailRaw = ` +const userKeyUpdateText = ` Click the link below to verify your new identity: {{.Link}} @@ -221,12 +227,16 @@ was generated for {{.Email}} on Politeia. If you did not perform this action, please contact Politeia administrators. ` -type resetPasswordEmailTemplateData struct { - Link string - Email string +var userKeyUpdateTmpl = template.Must( + template.New("userKeyUpdate").Parse(userKeyUpdateText)) + +// User password reset - Send password reset link to user +type userPasswordReset struct { + Link string // Password reset link + Email string // User email } -const templateResetPasswordEmailRaw = ` +const userPasswordResetText = ` Click the link below to continue resetting your password: {{.Link}} @@ -237,12 +247,16 @@ compromised. Please contact Politeia administrators through Matrix on the #politeia:decred.org channel. ` -type userLockedResetPasswordEmailTemplateData struct { - Link string - Email string +var userPasswordResetTmpl = template.Must( + template.New("userPasswordReset").Parse(userPasswordResetText)) + +// User account locked - Send reset password link to user +type userAccountLocked struct { + Link string // Reset password link + Email string // User email } -const templateUserLockedResetPasswordRaw = ` +const userAccountLockedText = ` Your account was locked due to too many login attempts. You need to reset your password in order to unlock your account: @@ -252,17 +266,24 @@ You are receiving this email because someone made too many login attempts for {{.Email}} on Politeia. If that was not you, please notify Politeia administrators. ` -type userPasswordChangedTemplateData struct { - Email string +var userAccountLockedTmpl = template.Must( + template.New("userAccountLocked").Parse(userAccountLockedText)) + +// User password changed - Send to user +type userPasswordChanged struct { + Email string // User email } -const templateUserPasswordChangedRaw = ` +const userPasswordChangedText = ` You are receiving this email to notify you that your password has changed for {{.Email}} on Politeia. If you did not perform this action, it is possible that your account has been compromised. Please contact Politeia administrators through Matrix on the #politeia:decred.org channel for further instructions. ` +var userPasswordChangedTmpl = template.Must( + template.New("userPasswordChanged").Parse(userPasswordChangedText)) + // CMS events type newInviteUserEmailTemplateData struct { Email string diff --git a/politeiawww/user.go b/politeiawww/user.go index 01f81bafb..6f854b91b 100644 --- a/politeiawww/user.go +++ b/politeiawww/user.go @@ -455,7 +455,7 @@ func (p *politeiawww) processNewUser(nu www.NewUser) (*www.NewUserReply, error) // Email the verification token before updating the // database. If the email fails, the database won't // be updated. - err = p.emailNewUserVerificationLink(u.Email, + err = p.emailUserEmailVerify(u.Email, hex.EncodeToString(tokenb), u.Username) if err != nil { log.Errorf("processNewUser: mail verification "+ @@ -548,7 +548,7 @@ func (p *politeiawww) processNewUser(nu www.NewUser) (*www.NewUserReply, error) // then the new user won't be created. // // This is conditional on the email server being setup. - err = p.emailNewUserVerificationLink(newUser.Email, + err = p.emailUserEmailVerify(newUser.Email, hex.EncodeToString(tokenb), newUser.Username) if err != nil { log.Errorf("processNewUser: mail verification token "+ @@ -922,7 +922,7 @@ func (p *politeiawww) processResendVerification(rv *www.ResendVerification) (*ww // the user won't be updated. // // This is conditional on the email server being setup. - err = p.emailNewUserVerificationLink(u.Email, + err = p.emailUserEmailVerify(u.Email, hex.EncodeToString(token), u.Username) if err != nil { log.Errorf("processResendVerification: email verification "+ @@ -1001,7 +1001,7 @@ func (p *politeiawww) processUpdateUserKey(usr *user.User, uuk www.UpdateUserKey // // This is conditional on the email server being setup. token := hex.EncodeToString(tokenb) - err = p.emailUpdateUserKeyVerificationLink(usr.Email, uuk.PublicKey, + err = p.emailUserKeyUpdate(usr.Email, uuk.PublicKey, token) if err != nil { return nil, err @@ -1133,7 +1133,7 @@ func (p *politeiawww) login(l www.Login) loginResult { // send them an email informing them their account is // now locked. if userIsLocked(u.FailedLoginAttempts) { - err := p.emailUserLocked(u.Email) + err := p.emailUserAccountLocked(u.Email) if err != nil { return loginResult{ reply: nil, @@ -1267,9 +1267,9 @@ func (p *politeiawww) processLogin(l www.Login) (*www.LoginReply, error) { // login attempts, send the user an email to notify them // that their account is locked. if userIsLocked(u.FailedLoginAttempts) { - err := p.emailUserLocked(u.Email) + err := p.emailUserAccountLocked(u.Email) if err != nil { - log.Errorf("processLogin: emailUserLocked '%v': %v", + log.Errorf("processLogin: emailUserAccountLocked '%v': %v", u.Email, err) } } @@ -1457,7 +1457,7 @@ func (p *politeiawww) resetPassword(rp www.ResetPassword) resetPasswordResult { // Try to email the verification link first. If it fails, the // user record won't be updated in the database. - err = p.emailResetPasswordVerificationLink(rp.Email, rp.Username, + err = p.emailUserPasswordReset(rp.Email, rp.Username, hex.EncodeToString(tokenb)) if err != nil { return resetPasswordResult{ diff --git a/politeiawww/user/cockroachdb/cockroachdb_test.go b/politeiawww/user/cockroachdb/cockroachdb_test.go index e3c1cc733..d7de4aaa8 100644 --- a/politeiawww/user/cockroachdb/cockroachdb_test.go +++ b/politeiawww/user/cockroachdb/cockroachdb_test.go @@ -38,7 +38,6 @@ func (a AnyTime) Match(v driver.Value) bool { // Helpers var ( - errUpdate = fmt.Errorf("update user error") errSelect = fmt.Errorf("select user error") errDelete = fmt.Errorf("delete user error") ) diff --git a/politeiawww/userwww.go b/politeiawww/userwww.go index 1e4dd8040..a9b1c85fa 100644 --- a/politeiawww/userwww.go +++ b/politeiawww/userwww.go @@ -18,16 +18,6 @@ import ( ) var ( - templateNewUserEmail = template.Must( - template.New("new_user_email_template").Parse(templateNewUserEmailRaw)) - templateResetPasswordEmail = template.Must( - template.New("reset_password_email_template").Parse(templateResetPasswordEmailRaw)) - templateUpdateUserKeyEmail = template.Must( - template.New("update_user_key_email_template").Parse(templateUpdateUserKeyEmailRaw)) - templateUserLockedResetPassword = template.Must( - template.New("user_locked_reset_password").Parse(templateUserLockedResetPasswordRaw)) - templateUserPasswordChanged = template.Must( - template.New("user_changed_password").Parse(templateUserPasswordChangedRaw)) templateInviteNewUserEmail = template.Must( template.New("invite_new_user_email_template").Parse(templateInviteNewUserEmailRaw)) templateApproveDCCUserEmail = template.Must( From 9858a5dec173b2e7bebf9162f0b665d44cfc6ce2 Mon Sep 17 00:00:00 2001 From: Amir Massarwa Date: Wed, 23 Sep 2020 21:47:32 +0300 Subject: [PATCH 098/449] politeiawww: Add processVoteBallot. --- politeiawww/api/pi/v1/v1.go | 46 +++++++++++++++-- politeiawww/cmd/piwww/vote.go | 30 ++++++------ politeiawww/cmd/shared/client.go | 16 +++--- politeiawww/piwww.go | 84 ++++++++++++++++++++++++++++++++ politeiawww/proposals.go | 37 ++++++++------ politeiawww/ticketvote.go | 24 ++++++++- 6 files changed, 194 insertions(+), 43 deletions(-) diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index e2c060afc..ec7f7f901 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -15,6 +15,7 @@ type CommentVoteT int type VoteStatusT int type VoteAuthActionT string type VoteT int +type VoteErrorT int // TODO the plugin policies should be returned in a route // TODO the proposals route should allow filtering by user ID @@ -159,6 +160,18 @@ const ( ErrorStatusVoteParamsInvalid ErrorStatusBallotInvalid ErrorStatusVotePageSizeExceeded + + // Cast vote errors + VoteErrorInvalid VoteErrorT = 0 + VoteErrorInternalError VoteErrorT = 1 + VoteErrorTokenInvalid VoteErrorT = 2 + VoteErrorRecordNotFound VoteErrorT = 3 + VoteErrorMultipleRecordVotes VoteErrorT = 4 + VoteErrorVoteStatusInvalid VoteErrorT = 5 + VoteErrorVoteBitInvalid VoteErrorT = 6 + VoteErrorSignatureInvalid VoteErrorT = 7 + VoteErrorTicketNotEligible VoteErrorT = 8 + VoteErrorTicketAlreadyVoted VoteErrorT = 9 ) var ( @@ -575,7 +588,7 @@ type VoteDetails struct { // CastVoteDetails contains the details of a cast vote. type CastVoteDetails struct { - Token string `json:"token"` // Record token + Token string `json:"token"` // Proposal token Ticket string `json:"ticket"` // Ticket hash VoteBit string `json:"votebits"` // Selected vote bit, hex encoded Signature string `json:"signature"` // Signature of Token+Ticket+VoteBit @@ -663,8 +676,35 @@ type VoteStartRunoffReply struct { EligibleTickets []string `json:"eligibletickets"` // Ticket hashes } -type VoteBallot struct{} -type VoteBallotReply struct{} +// CastVote is a signed ticket vote. +type CastVote struct { + Token string `json:"token"` // Proposal token + Ticket string `json:"ticket"` // Ticket ID + VoteBit string `json:"votebits"` // Selected vote bit, hex encoded + Signature string `json:"signature"` // Signature of Token+Ticket+VoteBit +} + +// CastVoteReply contains the receipt for the cast vote. +type CastVoteReply struct { + Ticket string `json:"ticket"` // Ticket ID + Receipt string `json:"receipt"` // Server signature of client signature + + // The follwing fields will only be present if an error occured + // while attempting to cast the vote. + ErrorCode VoteErrorT `json:"errorcode,omitempty"` + ErrorContext string `json:"errorcontext,omitempty"` +} + +// VoteBallot is a batch of votes that are sent to the server. A ballot can only +// contain the votes for a single record. +type VoteBallot struct { + Votes []CastVote `json:"votes"` +} + +// VoteBallotReply is a reply to a batched list of votes. +type VoteBallotReply struct { + Receipts []CastVoteReply `json:"receipts"` +} // ProposalVote contains all vote authorizations and the vote details for a // proposal vote. The vote details will be null if the proposal vote has not diff --git a/politeiawww/cmd/piwww/vote.go b/politeiawww/cmd/piwww/vote.go index f9409853b..5b5f3d324 100644 --- a/politeiawww/cmd/piwww/vote.go +++ b/politeiawww/cmd/piwww/vote.go @@ -14,7 +14,8 @@ import ( "github.com/decred/dcrd/chaincfg/chainhash" "github.com/decred/dcrwallet/rpc/walletrpc" "github.com/decred/politeia/politeiad/api/v1/identity" - "github.com/decred/politeia/politeiawww/api/www/v1" + pi "github.com/decred/politeia/politeiawww/api/pi/v1" + v1 "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/util" "golang.org/x/crypto/ssh/terminal" ) @@ -154,10 +155,10 @@ func (cmd *VoteCmd) Execute(args []string) error { } // Setup cast votes request - votes := make([]v1.CastVote, 0, len(eligibleTickets)) + votes := make([]pi.CastVote, 0, len(eligibleTickets)) for i, ticket := range eligibleTickets { // eligibleTickets and sigs use the same index - votes = append(votes, v1.CastVote{ + votes = append(votes, pi.CastVote{ Token: token, Ticket: ticket, VoteBit: voteBits, @@ -166,7 +167,7 @@ func (cmd *VoteCmd) Execute(args []string) error { } // Cast proposal votes - br, err := client.CastVotes(&v1.Ballot{ + br, err := client.VoteBallot(&pi.VoteBallot{ Votes: votes, }) if err != nil { @@ -177,7 +178,7 @@ func (cmd *VoteCmd) Execute(args []string) error { // the ticket hash so in order to associate a failed // receipt with a specific ticket, we need to lookup the // ticket hash and store it separately. - failedReceipts := make([]v1.CastVoteReply, 0, len(br.Receipts)) + failedReceipts := make([]pi.CastVoteReply, 0, len(br.Receipts)) failedTickets := make([]string, 0, len(eligibleTickets)) for i, v := range br.Receipts { // Lookup ticket hash @@ -185,23 +186,24 @@ func (cmd *VoteCmd) Execute(args []string) error { h := eligibleTickets[i] // Check for voting error - if v.Error != "" { + if v.ErrorContext != "" { failedReceipts = append(failedReceipts, v) failedTickets = append(failedTickets, h) continue } // Validate server signature - sig, err := identity.SignatureFromString(v.Signature) + sig, err := identity.SignatureFromString(v.Receipt) if err != nil { - v.Error = err.Error() + v.ErrorContext = err.Error() failedReceipts = append(failedReceipts, v) failedTickets = append(failedTickets, h) continue } - if !serverID.VerifyMessage([]byte(v.ClientSignature), *sig) { - v.Error = "Could not verify receipt " + v.ClientSignature + clientSig := votes[i].Signature + if !serverID.VerifyMessage([]byte(clientSig), *sig) { + v.ErrorContext = "Could not verify receipt " + clientSig failedReceipts = append(failedReceipts, v) failedTickets = append(failedTickets, h) } @@ -212,7 +214,7 @@ func (cmd *VoteCmd) Execute(args []string) error { fmt.Printf("Votes succeeded: %v\n", len(br.Receipts)-len(failedReceipts)) fmt.Printf("Votes failed : %v\n", len(failedReceipts)) for i, v := range failedReceipts { - fmt.Printf("Failed vote : %v %v\n", failedTickets[i], v.Error) + fmt.Printf("Failed vote : %v %v\n", failedTickets[i], v.ErrorContext) } } @@ -227,8 +229,4 @@ Cast ticket votes for a proposal. Arguments: 1. token (string, optional) Proposal censorship token 2. voteid (string, optional) A single word identifying vote (e.g. yes) - -Result: -Enter the private passphrase of your wallet: -Votes succeeded: (int) Number of successful votes -Votes failed : (int) Number of failed votes` +` diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index 193527536..d291ce8e6 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -1703,28 +1703,28 @@ func (c *Client) ActiveVotesDCC() (*cms.ActiveVoteReply, error) { return &avr, nil } -// CastVotes casts votes for a proposal. -func (c *Client) CastVotes(b *www.Ballot) (*www.BallotReply, error) { +// VoteBallot casts ballot of votes for a proposal. +func (c *Client) VoteBallot(vb *pi.VoteBallot) (*pi.VoteBallotReply, error) { responseBody, err := c.makeRequest(http.MethodPost, - www.PoliteiaWWWAPIRoute, www.RouteCastVotes, &b) + pi.APIRoute, pi.RouteVoteBallot, &vb) if err != nil { return nil, err } - var br www.BallotReply - err = json.Unmarshal(responseBody, &br) + var vbr pi.VoteBallotReply + err = json.Unmarshal(responseBody, &vbr) if err != nil { - return nil, fmt.Errorf("unmarshal BallotReply: %v", err) + return nil, fmt.Errorf("unmarshal VoteBallotReply: %v", err) } if c.cfg.Verbose { - err := prettyPrintJSON(br) + err := prettyPrintJSON(vbr) if err != nil { return nil, err } } - return &br, nil + return &vbr, nil } // UpdateUserKey updates the identity of the logged in user. diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index cd2900649..69529ef4b 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -1965,6 +1965,87 @@ func (p *politeiawww) handleVoteStartRunoff(w http.ResponseWriter, r *http.Reque util.RespondWithJSON(w, http.StatusOK, vsrr) } +func convertPiVoteErrorFromTicketVote(e ticketvote.VoteErrorT) pi.VoteErrorT { + switch e { + case ticketvote.VoteErrorInvalid: + return pi.VoteErrorInvalid + case ticketvote.VoteErrorInternalError: + return pi.VoteErrorInternalError + case ticketvote.VoteErrorRecordNotFound: + return pi.VoteErrorRecordNotFound + case ticketvote.VoteErrorVoteBitInvalid: + return pi.VoteErrorVoteBitInvalid + case ticketvote.VoteErrorVoteStatusInvalid: + return pi.VoteErrorVoteStatusInvalid + case ticketvote.VoteErrorTicketAlreadyVoted: + return pi.VoteErrorTicketAlreadyVoted + case ticketvote.VoteErrorTicketNotEligible: + return pi.VoteErrorTicketNotEligible + default: + return pi.VoteErrorInternalError + } +} + +func (p *politeiawww) processVoteBallot(vb pi.VoteBallot) (*pi.VoteBallotReply, error) { + log.Tracef("processVoteBallot") + + // Call the ticketvote to cast ballot of votes + // Transalte payload + var vbp ticketvote.Ballot + votes := make([]ticketvote.CastVote, 0, len(vb.Votes)) + for _, v := range vb.Votes { + votes = append(votes, ticketvote.CastVote{ + Token: v.Token, + Ticket: v.Ticket, + VoteBit: v.VoteBit, + Signature: v.Signature, + }) + } + vbp.Votes = votes + + reply, err := p.voteBallot(vbp) + if err != nil { + return nil, err + } + + // Translate reply + var vbr pi.VoteBallotReply + vrs := make([]pi.CastVoteReply, 0, len(reply.Receipts)) + for _, vr := range reply.Receipts { + vrs = append(vrs, pi.CastVoteReply{ + Ticket: vr.Ticket, + Receipt: vr.Receipt, + ErrorCode: convertPiVoteErrorFromTicketVote(vr.ErrorCode), + ErrorContext: vr.ErrorContext, + }) + } + vbr.Receipts = vrs + + return &vbr, nil +} + +func (p *politeiawww) handleVoteBallot(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleVoteBallot") + + var vb pi.VoteBallot + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&vb); err != nil { + respondWithPiError(w, r, "handleVoteBallot: unmarshal", + pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusInvalidInput, + }) + return + } + + vbr, err := p.processVoteBallot(vb) + if err != nil { + respondWithPiError(w, r, + "handleVoteBallot: processVoteBallot: %v", err) + } + + util.RespondWithJSON(w, http.StatusOK, vbr) +} + func (p *politeiawww) setPiRoutes() { // Public routes p.addRoute(http.MethodGet, pi.APIRoute, @@ -1983,6 +2064,9 @@ func (p *politeiawww) setPiRoutes() { p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteVoteStartRunoff, p.handleVoteStartRunoff, permissionPublic) + p.addRoute(http.MethodPost, pi.APIRoute, + pi.RouteVoteBallot, p.handleVoteBallot, permissionPublic) + // Logged in routes p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteProposalNew, p.handleProposalNew, diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index a49bcee32..8c6e836ea 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -279,37 +279,46 @@ func (p *politeiawww) processVoteResults(token string) (*www.VoteResultsReply, e log.Tracef("processVoteResults: %v", token) // Call ticketvote plugin - vd, err := p.voteDetails(token) + vd, err := p.voteDetails([]string{token}) if err != nil { return nil, err } // Convert reply to www - startHeight := strconv.FormatUint(uint64(vd.Vote.StartBlockHeight), 10) - endHeight := strconv.FormatUint(uint64(vd.Vote.EndBlockHeight), 10) + var ( + vote ticketvote.RecordVote + ok bool + ) + if vote, ok = vd.Votes[token]; !ok { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + } + } + startHeight := strconv.FormatUint(uint64(vote.Vote.StartBlockHeight), 10) + endHeight := strconv.FormatUint(uint64(vote.Vote.EndBlockHeight), 10) res := www.VoteResultsReply{ StartVote: www.StartVote{ - PublicKey: vd.Vote.PublicKey, - Signature: vd.Vote.Signature, + PublicKey: vote.Vote.PublicKey, + Signature: vote.Vote.Signature, Vote: www.Vote{ - Token: vd.Vote.Params.Token, - Mask: vd.Vote.Params.Mask, - Duration: vd.Vote.Params.Duration, - QuorumPercentage: vd.Vote.Params.QuorumPercentage, - PassPercentage: vd.Vote.Params.PassPercentage, + Token: vote.Vote.Params.Token, + Mask: vote.Vote.Params.Mask, + Duration: vote.Vote.Params.Duration, + QuorumPercentage: vote.Vote.Params.QuorumPercentage, + PassPercentage: vote.Vote.Params.PassPercentage, }, }, StartVoteReply: www.StartVoteReply{ StartBlockHeight: startHeight, - StartBlockHash: vd.Vote.StartBlockHash, + StartBlockHash: vote.Vote.StartBlockHash, EndHeight: endHeight, - EligibleTickets: vd.Vote.EligibleTickets, + EligibleTickets: vote.Vote.EligibleTickets, }, } // Transalte vote options - vo := make([]www.VoteOption, 0, len(vd.Vote.Params.Options)) - for _, o := range vd.Vote.Params.Options { + vo := make([]www.VoteOption, 0, len(vote.Vote.Params.Options)) + for _, o := range vote.Vote.Params.Options { vo = append(vo, www.VoteOption{ Id: o.ID, Description: o.Description, diff --git a/politeiawww/ticketvote.go b/politeiawww/ticketvote.go index bfffaa253..23561bb3f 100644 --- a/politeiawww/ticketvote.go +++ b/politeiawww/ticketvote.go @@ -9,6 +9,26 @@ import ( www "github.com/decred/politeia/politeiawww/api/www/v1" ) +func (p *politeiawww) voteBallot(vb ticketvote.Ballot) (*ticketvote.BallotReply, error) { + // Prep plugin command + payload, err := ticketvote.EncodeBallot(vb) + if err != nil { + return nil, err + } + + r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdBallot, "", + string(payload)) + if err != nil { + return nil, err + } + vbr, err := ticketvote.DecodeBallotReply([]byte(r)) + if err != nil { + return nil, err + } + + return vbr, nil +} + func (p *politeiawww) voteStartRunoff(vsr ticketvote.StartRunoff) (*ticketvote.StartRunoffReply, error) { // Prep plugin command payload, err := ticketvote.EncodeStartRunoff(vsr) @@ -70,10 +90,10 @@ func (p *politeiawww) voteAuthorize(a ticketvote.Authorize) (*ticketvote.Authori } // voteDetails calls the ticketvote plugin command to get vote details. -func (p *politeiawww) voteDetails(token string) (*ticketvote.DetailsReply, error) { +func (p *politeiawww) voteDetails(tokens []string) (*ticketvote.DetailsReply, error) { // Prep vote details payload vdp := ticketvote.Details{ - Token: token, + Tokens: tokens, } payload, err := ticketvote.EncodeDetails(vdp) if err != nil { From 2cae29e59eafefcc5447309c820fcc01adf2b0cd Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 23 Sep 2020 14:44:59 -0500 Subject: [PATCH 099/449] remove email from all email templates --- politeiawww/email.go | 18 +++++------ politeiawww/templates.go | 66 +++++++++++++++++++++++----------------- politeiawww/user.go | 7 ++--- 3 files changed, 49 insertions(+), 42 deletions(-) diff --git a/politeiawww/email.go b/politeiawww/email.go index 1411042a0..0f4edd270 100644 --- a/politeiawww/email.go +++ b/politeiawww/email.go @@ -321,7 +321,6 @@ func (p *politeiawww) emailUserEmailVerify(email, token, username string) error tplData := userEmailVerify{ Username: username, - Email: email, Link: link, } @@ -351,8 +350,7 @@ func (p *politeiawww) emailUserPasswordReset(email, username, token string) erro // Setup email subject := "Reset Your Password" tplData := userPasswordReset{ - Email: email, - Link: u.String(), + Link: u.String(), } body, err := createBody(userPasswordResetTmpl, tplData) if err != nil { @@ -365,15 +363,15 @@ func (p *politeiawww) emailUserPasswordReset(email, username, token string) erro // emailUserKeyUpdate emails the link with the verification token used for // setting a new key pair if the email server is set up. -func (p *politeiawww) emailUserKeyUpdate(email, publicKey, token string) error { +func (p *politeiawww) emailUserKeyUpdate(username, email, publicKey, token string) error { link, err := p.createEmailLink(www.RouteVerifyUpdateUserKey, "", token, "") if err != nil { return err } tplData := userKeyUpdate{ - Email: email, PublicKey: publicKey, + Username: username, Link: link, } @@ -389,9 +387,9 @@ func (p *politeiawww) emailUserKeyUpdate(email, publicKey, token string) error { // emailUserPasswordChanged notifies the user that his password was changed, // and verifies if he was the author of this action, for security purposes. -func (p *politeiawww) emailUserPasswordChanged(email string) error { +func (p *politeiawww) emailUserPasswordChanged(username, email string) error { tplData := userPasswordChanged{ - Email: email, + Username: username, } subject := "Password Changed - Security Verification" @@ -407,7 +405,7 @@ func (p *politeiawww) emailUserPasswordChanged(email string) error { // emailUserAccountLocked notifies the user its account has been locked and // emails the link with the reset password verification token if the email // server is set up. -func (p *politeiawww) emailUserAccountLocked(email string) error { +func (p *politeiawww) emailUserAccountLocked(username, email string) error { link, err := p.createEmailLink(ResetPasswordGuiRoute, email, "", "") if err != nil { @@ -415,8 +413,8 @@ func (p *politeiawww) emailUserAccountLocked(email string) error { } tplData := userAccountLocked{ - Email: email, - Link: link, + Link: link, + Username: username, } subject := "Locked Account - Reset Your Password" diff --git a/politeiawww/templates.go b/politeiawww/templates.go index f4a8b5ff5..708fea9e0 100644 --- a/politeiawww/templates.go +++ b/politeiawww/templates.go @@ -187,24 +187,22 @@ var proposalVoteStartedToAuthorTmpl = template.Must( template.New("proposalVoteStartedToAuthor"). Parse(proposalVoteStartedToAuthorText)) -// User events - // User email verify - Send verification link to new user type userEmailVerify struct { Username string // User username - Email string // User email Link string // Verification link } const userEmailVerifyText = ` Thanks for joining Politeia, {{.Username}}! -Click the link below to verify your email and complete your registration: +Click the link below to verify your email and complete your registration. {{.Link}} -You are receiving this email because {{.Email}} was used to register for Politeia. -If you did not perform this action, please ignore this email. +You are receiving this notification because this email address was used to +register a Politeia account. If you did not perform this action, please ignore +this email. ` var userEmailVerifyTmpl = template.Must( @@ -212,9 +210,9 @@ var userEmailVerifyTmpl = template.Must( // User key update - Send key verification link to user type userKeyUpdate struct { - Link string // Verify key link PublicKey string // User new public key - Email string // User + Username string + Link string // Verify key link } const userKeyUpdateText = ` @@ -222,9 +220,15 @@ Click the link below to verify your new identity: {{.Link}} -You are receiving this email because a new identity (public key: {{.PublicKey}}) -was generated for {{.Email}} on Politeia. If you did not perform this action, -please contact Politeia administrators. +You are receiving this notification because a new identity was generated for +{{.Username}} on Politeia with the following public key. + +Public key: {{.PublicKey}} + +If you did not perform this action, please contact a Politeia administrators in +the Politeia channel on Matrix. + +https://chat.decred.org/#/room/#politeia:decred.org ` var userKeyUpdateTmpl = template.Must( @@ -232,8 +236,7 @@ var userKeyUpdateTmpl = template.Must( // User password reset - Send password reset link to user type userPasswordReset struct { - Link string // Password reset link - Email string // User email + Link string // Password reset link } const userPasswordResetText = ` @@ -241,10 +244,11 @@ Click the link below to continue resetting your password: {{.Link}} -You are receiving this email because a password reset was initiated for {{.Email}} -on Politeia. If you did not perform this action, it is possible that your account has been -compromised. Please contact Politeia administrators through Matrix on the -#politeia:decred.org channel. +A password reset was initiated for this Politeia account. If you did not +perform this action, it's possible that your account has been compromised. +Please contact a Politeia administrator in the Politeia channel on Matrix. + +https://chat.decred.org/#/room/#politeia:decred.org ` var userPasswordResetTmpl = template.Must( @@ -252,18 +256,20 @@ var userPasswordResetTmpl = template.Must( // User account locked - Send reset password link to user type userAccountLocked struct { - Link string // Reset password link - Email string // User email + Link string // Reset password link + Username string } const userAccountLockedText = ` -Your account was locked due to too many login attempts. You need to reset your -password in order to unlock your account: +The Politeia account for {{.Username}} was locked due to too many login +attempts. You need to reset your password in order to unlock your account: {{.Link}} -You are receiving this email because someone made too many login attempts for -{{.Email}} on Politeia. If that was not you, please notify Politeia administrators. +If these login attempts were not made by you, please notify a Politeia +administrators in the Politeia channel on Matrix. + +https://chat.decred.org/#/room/#politeia:decred.org ` var userAccountLockedTmpl = template.Must( @@ -271,14 +277,18 @@ var userAccountLockedTmpl = template.Must( // User password changed - Send to user type userPasswordChanged struct { - Email string // User email + Username string } const userPasswordChangedText = ` -You are receiving this email to notify you that your password has changed for -{{.Email}} on Politeia. If you did not perform this action, it is possible that -your account has been compromised. Please contact Politeia administrators -through Matrix on the #politeia:decred.org channel for further instructions. +The password has been changed for your Politeia account with the username +{{.Username}}. + +If you did not perform this action, it's possible that your account has been +compromised. Please contact a Politeia administrator in the Politeia channel +on Matrix. + +https://chat.decred.org/#/room/#politeia:decred.org ` var userPasswordChangedTmpl = template.Must( diff --git a/politeiawww/user.go b/politeiawww/user.go index 6f854b91b..17b5761e4 100644 --- a/politeiawww/user.go +++ b/politeiawww/user.go @@ -1001,8 +1001,7 @@ func (p *politeiawww) processUpdateUserKey(usr *user.User, uuk www.UpdateUserKey // // This is conditional on the email server being setup. token := hex.EncodeToString(tokenb) - err = p.emailUserKeyUpdate(usr.Email, uuk.PublicKey, - token) + err = p.emailUserKeyUpdate(usr.Username, usr.Email, uuk.PublicKey, token) if err != nil { return nil, err } @@ -1133,7 +1132,7 @@ func (p *politeiawww) login(l www.Login) loginResult { // send them an email informing them their account is // now locked. if userIsLocked(u.FailedLoginAttempts) { - err := p.emailUserAccountLocked(u.Email) + err := p.emailUserAccountLocked(u.Username, u.Email) if err != nil { return loginResult{ reply: nil, @@ -1405,7 +1404,7 @@ func (p *politeiawww) processChangePassword(email string, cp www.ChangePassword) return nil, err } - err = p.emailUserPasswordChanged(email) + err = p.emailUserPasswordChanged(u.Username, email) if err != nil { return nil, err } From 73717268a43fbc710b8bd14f3e730a25da5cbc4f Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Wed, 23 Sep 2020 17:50:46 -0300 Subject: [PATCH 100/449] politeiawww: Refactor CMS/DCC email templates. --- politeiawww/cmsnotifications.go | 2 +- politeiawww/cmsuser.go | 5 +- politeiawww/cmswww.go | 14 --- politeiawww/email.go | 198 ++++++++++++++++---------------- politeiawww/eventmanager.go | 8 +- politeiawww/templates.go | 84 +++++++++----- politeiawww/userwww.go | 8 -- 7 files changed, 162 insertions(+), 157 deletions(-) diff --git a/politeiawww/cmsnotifications.go b/politeiawww/cmsnotifications.go index 703aa8b8d..510f19642 100644 --- a/politeiawww/cmsnotifications.go +++ b/politeiawww/cmsnotifications.go @@ -68,7 +68,7 @@ func (p *politeiawww) checkInvoiceNotifications() { log.Tracef("Checked user: %v sending email? %v", user.Username, !invoiceFound) if !invoiceFound { - err = p.emailInvoiceNotifications(user.Email, user.Username) + err = p.emailInvoiceNotSent(user.Email, user.Username) if err != nil { log.Errorf("Error sending email: %v %v", err, user.Email) } diff --git a/politeiawww/cmsuser.go b/politeiawww/cmsuser.go index 0a0e5e9b1..d80237af5 100644 --- a/politeiawww/cmsuser.go +++ b/politeiawww/cmsuser.go @@ -85,8 +85,7 @@ func (p *politeiawww) processInviteNewUser(u cms.InviteNewUser) (*cms.InviteNewU // the new user won't be created. // // This is conditional on the email server being setup. - err = p.emailInviteNewUserVerificationLink(u.Email, - hex.EncodeToString(token)) + err = p.emailUserCMSInvite(u.Email, hex.EncodeToString(token)) if err != nil { log.Errorf("processInviteNewUser: verification email "+ "failed for '%v': %v", u.Email, err) @@ -731,7 +730,7 @@ func (p *politeiawww) issuanceDCCUser(userid, sponsorUserID string, domain, cont // the new user won't be created. // // This is conditional on the email server being setup. - err = p.emailApproveDCCVerificationLink(nominatedUser.Email) + err = p.emailUserDCCApproved(nominatedUser.Email) if err != nil { log.Errorf("processApproveDCC: verification email "+ "failed for '%v': %v", nominatedUser.Email, err) diff --git a/politeiawww/cmswww.go b/politeiawww/cmswww.go index 62d4506d9..a56a4e7d7 100644 --- a/politeiawww/cmswww.go +++ b/politeiawww/cmswww.go @@ -8,7 +8,6 @@ import ( "bytes" "encoding/json" "net/http" - "text/template" cms "github.com/decred/politeia/politeiawww/api/cms/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" @@ -17,19 +16,6 @@ import ( "github.com/gorilla/mux" ) -var ( - templateInvoiceNotification = template.Must( - template.New("invoice_notification").Parse(templateInvoiceNotificationRaw)) - templateNewInvoiceComment = template.Must( - template.New("invoice_comment").Parse(templateNewInvoiceCommentRaw)) - templateNewInvoiceStatusUpdate = template.Must( - template.New("invoice_status_update").Parse(templateNewInvoiceStatusUpdateRaw)) - templateNewDCCSubmitted = template.Must( - template.New("dcc_new").Parse(templateNewDCCSubmittedRaw)) - templateNewDCCSupportOppose = template.Must( - template.New("dcc_support_oppose").Parse(templateNewDCCSupportOpposeRaw)) -) - // handleInviteNewUser handles the invitation of a new contractor by an // administrator for the Contractor Management System. func (p *politeiawww) handleInviteNewUser(w http.ResponseWriter, r *http.Request) { diff --git a/politeiawww/email.go b/politeiawww/email.go index 0f4edd270..fbc2e6f03 100644 --- a/politeiawww/email.go +++ b/politeiawww/email.go @@ -334,33 +334,6 @@ func (p *politeiawww) emailUserEmailVerify(email, token, username string) error return p.smtp.sendEmailTo(subject, body, recipients) } -// emailUserPasswordReset emails the link with the reset password verification -// token if the email server is set up. -func (p *politeiawww) emailUserPasswordReset(email, username, token string) error { - // Setup URL - u, err := url.Parse(p.cfg.WebServerAddress + www.RouteResetPassword) - if err != nil { - return err - } - q := u.Query() - q.Set("verificationtoken", token) - q.Set("username", username) - u.RawQuery = q.Encode() - - // Setup email - subject := "Reset Your Password" - tplData := userPasswordReset{ - Link: u.String(), - } - body, err := createBody(userPasswordResetTmpl, tplData) - if err != nil { - return err - } - - // Send email - return p.smtp.sendEmailTo(subject, body, []string{email}) -} - // emailUserKeyUpdate emails the link with the verification token used for // setting a new key pair if the email server is set up. func (p *politeiawww) emailUserKeyUpdate(username, email, publicKey, token string) error { @@ -385,21 +358,31 @@ func (p *politeiawww) emailUserKeyUpdate(username, email, publicKey, token strin return p.smtp.sendEmailTo(subject, body, recipients) } -// emailUserPasswordChanged notifies the user that his password was changed, -// and verifies if he was the author of this action, for security purposes. -func (p *politeiawww) emailUserPasswordChanged(username, email string) error { - tplData := userPasswordChanged{ - Username: username, +// emailUserPasswordReset emails the link with the reset password verification +// token to the provided email address. +func (p *politeiawww) emailUserPasswordReset(email, username, token string) error { + // Setup URL + u, err := url.Parse(p.cfg.WebServerAddress + www.RouteResetPassword) + if err != nil { + return err } + q := u.Query() + q.Set("verificationtoken", token) + q.Set("username", username) + u.RawQuery = q.Encode() - subject := "Password Changed - Security Verification" - body, err := createBody(userPasswordChangedTmpl, tplData) + // Setup email + subject := "Reset Your Password" + tplData := userPasswordReset{ + Link: u.String(), + } + body, err := createBody(userPasswordResetTmpl, tplData) if err != nil { return err } - recipients := []string{email} - return p.smtp.sendEmailTo(subject, body, recipients) + // Send email + return p.smtp.sendEmailTo(subject, body, []string{email}) } // emailUserAccountLocked notifies the user its account has been locked and @@ -427,21 +410,38 @@ func (p *politeiawww) emailUserAccountLocked(username, email string) error { return p.smtp.sendEmailTo(subject, body, recipients) } -// emailInviteNewUserVerificationLink emails the link to invite a user to -// join the Contractor Management System, if the email server is set up. -func (p *politeiawww) emailInviteNewUserVerificationLink(email, token string) error { +// emailUserPasswordChanged notifies the user that his password was changed, +// and verifies if he was the author of this action, for security purposes. +func (p *politeiawww) emailUserPasswordChanged(username, email string) error { + tplData := userPasswordChanged{ + Username: username, + } + + subject := "Password Changed - Security Verification" + body, err := createBody(userPasswordChangedTmpl, tplData) + if err != nil { + return err + } + recipients := []string{email} + + return p.smtp.sendEmailTo(subject, body, recipients) +} + +// emailUserCMSInvite emails the invitation link for the Contractor Management +// System to the provided user email address. +func (p *politeiawww) emailUserCMSInvite(email, token string) error { link, err := p.createEmailLink(guiRouteRegisterNewUser, "", token, "") if err != nil { return err } - tplData := newInviteUserEmailTemplateData{ + tplData := userCMSInvite{ Email: email, Link: link, } subject := "Welcome to the Contractor Management System" - body, err := createBody(templateInviteNewUserEmail, tplData) + body, err := createBody(userCMSInviteTmpl, tplData) if err != nil { return err } @@ -450,15 +450,15 @@ func (p *politeiawww) emailInviteNewUserVerificationLink(email, token string) er return p.smtp.sendEmailTo(subject, body, recipients) } -// emailApproveDCCVerificationLink emails the link to invite a user that -// has been approved by the other contractors from a DCC proposal. -func (p *politeiawww) emailApproveDCCVerificationLink(email string) error { - tplData := approveDCCUserEmailTemplateData{ +// emailUserDCCApproved emails the link to invite a user that has been approved +// by the other contractors from a DCC proposal. +func (p *politeiawww) emailUserDCCApproved(email string) error { + tplData := userDCCApproved{ Email: email, } subject := "Congratulations, You've been approved!" - body, err := createBody(templateApproveDCCUserEmail, tplData) + body, err := createBody(userDCCApprovedTmpl, tplData) if err != nil { return err } @@ -467,51 +467,59 @@ func (p *politeiawww) emailApproveDCCVerificationLink(email string) error { return p.smtp.sendEmailTo(subject, body, recipients) } -// emailInvoiceNotifications emails users that have not yet submitted an invoice -// for the given month/year -func (p *politeiawww) emailInvoiceNotifications(email, username string) error { - // Set the date to the first day of the previous month. - newDate := time.Date(time.Now().Year(), time.Now().Month()-1, 1, 0, 0, 0, 0, time.UTC) - tplData := invoiceNotificationEmailData{ - Username: username, - Month: newDate.Month().String(), - Year: newDate.Year(), +// emailDCCSubmitted sends email regarding the DCC New event. Sends email +// to the provided email addresses. +func (p *politeiawww) emailDCCSubmitted(token string, emails []string) error { + route := strings.Replace(guiRouteDCCDetails, "{token}", token, 1) + l, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err } - subject := "Awaiting Monthly Invoice" - body, err := createBody(templateInvoiceNotification, tplData) + tplData := dccSubmitted{ + Link: l.String(), + } + + subject := "New DCC Submitted" + body, err := createBody(dccSubmittedTmpl, tplData) if err != nil { return err } - recipients := []string{email} - return p.smtp.sendEmailTo(subject, body, recipients) + return p.smtp.sendEmailTo(subject, body, emails) } -// emailInvoiceComment sends email for the invoice comment event. Sends -// email to the user regarding that invoice. -func (p *politeiawww) emailInvoiceComment(userEmail string) error { - var tplData interface{} - subject := "New Invoice Comment" +// emailDCCSupportOppose sends emails regarding dcc support/oppose event. +// Sends emails to the provided email addresses. +func (p *politeiawww) emailDCCSupportOppose(token string, emails []string) error { + route := strings.Replace(guiRouteDCCDetails, "{token}", token, 1) + l, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err + } + + tplData := dccSupportOppose{ + Link: l.String(), + } - body, err := createBody(templateNewInvoiceComment, tplData) + subject := "New DCC Support/Opposition Submitted" + body, err := createBody(dccSupportOpposeTmpl, tplData) if err != nil { return err } - recipients := []string{userEmail} - return p.smtp.sendEmailTo(subject, body, recipients) + return p.smtp.sendEmailTo(subject, body, emails) } // emailInvoiceStatusUpdate sends email for the invoice status update event. -// Sends email for the user regarding that invoice. +// Send email for the provided user email address. func (p *politeiawww) emailInvoiceStatusUpdate(invoiceToken, userEmail string) error { - tplData := newInvoiceStatusUpdateTemplate{ + tplData := invoiceStatusUpdate{ Token: invoiceToken, } subject := "Invoice status has been updated" - body, err := createBody(templateNewInvoiceStatusUpdate, tplData) + body, err := createBody(invoiceStatusUpdateTmpl, tplData) if err != nil { return err } @@ -520,46 +528,38 @@ func (p *politeiawww) emailInvoiceStatusUpdate(invoiceToken, userEmail string) e return p.smtp.sendEmailTo(subject, body, recipients) } -// emailDCCNew sends email regarding the DCC New event. Sends email -// to all admins. -func (p *politeiawww) emailDCCNew(token string, emails []string) error { - route := strings.Replace(guiRouteDCCDetails, "{token}", token, 1) - l, err := url.Parse(p.cfg.WebServerAddress + route) - if err != nil { - return err - } - - tplData := newDCCSubmittedTemplateData{ - Link: l.String(), +// emailInvoiceNotSent sends a invoice not sent email notification to the +// provided email address. +func (p *politeiawww) emailInvoiceNotSent(email, username string) error { + // Set the date to the first day of the previous month. + newDate := time.Date(time.Now().Year(), time.Now().Month()-1, 1, 0, 0, 0, 0, time.UTC) + tplData := invoiceNotSent{ + Username: username, + Month: newDate.Month().String(), + Year: newDate.Year(), } - subject := "New DCC Submitted" - body, err := createBody(templateNewDCCSubmitted, tplData) + subject := "Awaiting Monthly Invoice" + body, err := createBody(invoiceNotSentTmpl, tplData) if err != nil { return err } + recipients := []string{email} - return p.smtp.sendEmailTo(subject, body, emails) + return p.smtp.sendEmailTo(subject, body, recipients) } -// emailDCCSupportOppose sends emails regarding dcc support/oppose event. -// Sends emails to all admin users. -func (p *politeiawww) emailDCCSupportOppose(token string, emails []string) error { - route := strings.Replace(guiRouteDCCDetails, "{token}", token, 1) - l, err := url.Parse(p.cfg.WebServerAddress + route) - if err != nil { - return err - } - - tplData := newDCCSupportOpposeTemplateData{ - Link: l.String(), - } +// emailInvoiceNewComment sends email for the invoice new comment event. Send +// email to the provided user email address. +func (p *politeiawww) emailInvoiceNewComment(userEmail string) error { + var tplData interface{} + subject := "New Invoice Comment" - subject := "New DCC Support/Opposition Submitted" - body, err := createBody(templateNewDCCSupportOppose, tplData) + body, err := createBody(invoiceNewCommentTmpl, tplData) if err != nil { return err } + recipients := []string{userEmail} - return p.smtp.sendEmailTo(subject, body, emails) + return p.smtp.sendEmailTo(subject, body, recipients) } diff --git a/politeiawww/eventmanager.go b/politeiawww/eventmanager.go index d8756d83d..f68b07cc7 100644 --- a/politeiawww/eventmanager.go +++ b/politeiawww/eventmanager.go @@ -566,9 +566,9 @@ func (p *politeiawww) handleEventInvoiceComment(ch chan interface{}) { continue } - err := p.emailInvoiceComment(d.email) + err := p.emailInvoiceNewComment(d.email) if err != nil { - log.Errorf("emailInvoiceComment %v: %v", err) + log.Errorf("emailInvoiceNewComment %v: %v", err) } log.Debugf("Sent invoice comment notification %v", d.token) @@ -624,9 +624,9 @@ func (p *politeiawww) handleEventDCCNew(ch chan interface{}) { emails = append(emails, u.Email) }) - err = p.emailDCCNew(d.token, emails) + err = p.emailDCCSubmitted(d.token, emails) if err != nil { - log.Errorf("emailDCCNew %v: %v", err) + log.Errorf("emailDCCSubmitted %v: %v", err) } log.Debugf("Sent DCC new notification %v", d.token) diff --git a/politeiawww/templates.go b/politeiawww/templates.go index 708fea9e0..2a64ca01d 100644 --- a/politeiawww/templates.go +++ b/politeiawww/templates.go @@ -295,12 +295,14 @@ var userPasswordChangedTmpl = template.Must( template.New("userPasswordChanged").Parse(userPasswordChangedText)) // CMS events -type newInviteUserEmailTemplateData struct { - Email string - Link string + +// User CMS invite - Send to user being invited +type userCMSInvite struct { + Email string // User email + Link string // Registration link } -const templateInviteNewUserEmailRaw = ` +const userCMSInviteText = ` You are invited to join Decred as a contractor! To complete your registration, you will need to use the following link and register on the CMS site: {{.Link}} @@ -309,12 +311,15 @@ You are receiving this email because {{.Email}} was used to be invited to Decred If you do not recognize this, please ignore this email. ` -type approveDCCUserEmailTemplateData struct { - Email string - Link string +var userCMSInviteTmpl = template.Must( + template.New("userCMSInvite").Parse(userCMSInviteText)) + +// User DCC approved - Send to approved user +type userDCCApproved struct { + Email string // User email } -const templateApproveDCCUserEmailRaw = ` +const userDCCApprovedText = ` Congratulations! Your Decred Contractor Clearance Proposal has been approved! You are now a fully registered contractor and may now submit invoices. You should also be receiving an invitation to the contractors room on matrix. @@ -324,25 +329,33 @@ You are receiving this email because {{.Email}} was used to be invited to Decred If you do not recognize this, please ignore this email. ` -type newInvoiceStatusUpdateTemplate struct { - Token string +var userDCCApprovedTmpl = template.Must( + template.New("userDCCApproved").Parse(userDCCApprovedText)) + +// DCC submitted - Send to admins +type dccSubmitted struct { + Link string // DCC gui link } -const templateNewInvoiceStatusUpdateRaw = ` -An invoice's status has been updated, please login to cms.decred.org to review the changes. +const dccSubmittedText = ` +A new DCC has been submitted. -Updated Invoice Token: {{.Token}} +{{.Link}} Regards, Contractor Management System ` -type newDCCSubmittedTemplateData struct { - Link string +var dccSubmittedTmpl = template.Must( + template.New("dccSubmitted").Parse(dccSubmittedText)) + +// DCC support/oppose - Send to admins +type dccSupportOppose struct { + Link string // DCC gui link } -const templateNewDCCSubmittedRaw = ` -A new DCC has been submitted. +const dccSupportOpposeText = ` +A DCC has received new support or opposition. {{.Link}} @@ -350,26 +363,34 @@ Regards, Contractor Management System ` -type newDCCSupportOpposeTemplateData struct { - Link string +var dccSupportOpposeTmpl = template.Must( + template.New("dccSupportOppose").Parse(dccSupportOpposeText)) + +// Invoice status update - Send to invoice owner +type invoiceStatusUpdate struct { + Token string // Invoice token } -const templateNewDCCSupportOpposeRaw = ` -A DCC has received new support or opposition. +const invoiceStatusUpdateText = ` +An invoice's status has been updated, please login to cms.decred.org to review the changes. -{{.Link}} +Updated Invoice Token: {{.Token}} Regards, Contractor Management System ` -type invoiceNotificationEmailData struct { - Username string - Month string - Year int +var invoiceStatusUpdateTmpl = template.Must( + template.New("invoiceStatusUpdate").Parse(invoiceStatusUpdateText)) + +// Invoice not sent - Send to users that did not send monthly invoice yet +type invoiceNotSent struct { + Username string // User username + Month string // Current month + Year int // Current year } -const templateInvoiceNotificationRaw = ` +const invoiceNotSentText = ` {{.Username}}, You have not yet submitted an invoice for {{.Month}} {{.Year}}. Please do so as soon as possible, so your invoice may be reviewed and paid out in a timely manner. @@ -378,6 +399,13 @@ Regards, Contractor Management System ` -const templateNewInvoiceCommentRaw = ` +var invoiceNotSentTmpl = template.Must( + template.New("invoiceNotSent").Parse(invoiceNotSentText)) + +// Invoice new comment - Send to invoice owner +const invoiceNewCommentText = ` An administrator has submitted a new comment to your invoice, please login to cms.decred.org to view the message. ` + +var invoiceNewCommentTmpl = template.Must( + template.New("invoiceNewComment").Parse(invoiceNewCommentText)) diff --git a/politeiawww/userwww.go b/politeiawww/userwww.go index a9b1c85fa..6a8e63e7b 100644 --- a/politeiawww/userwww.go +++ b/politeiawww/userwww.go @@ -8,7 +8,6 @@ import ( "encoding/json" "fmt" "net/http" - "text/template" cms "github.com/decred/politeia/politeiawww/api/cms/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" @@ -17,13 +16,6 @@ import ( "github.com/gorilla/mux" ) -var ( - templateInviteNewUserEmail = template.Must( - template.New("invite_new_user_email_template").Parse(templateInviteNewUserEmailRaw)) - templateApproveDCCUserEmail = template.Must( - template.New("invite_approved_dcc_user").Parse(templateApproveDCCUserEmailRaw)) -) - // handleNewUser handles the incoming new user command. It verifies that the new user // doesn't already exist, and then creates a new user in the db and generates a random // code used for verification. The code is intended to be sent to the specified email. From dd0aff61cb1c8872dcfd924f22941e41ab9ce736 Mon Sep 17 00:00:00 2001 From: Amir Massarwa Date: Thu, 24 Sep 2020 15:21:58 +0300 Subject: [PATCH 101/449] politeiawww: Use pi plugin to censor a comment. --- politeiawww/comments.go | 21 --------------------- politeiawww/pi.go | 21 +++++++++++++++++++++ politeiawww/piwww.go | 6 +++--- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/politeiawww/comments.go b/politeiawww/comments.go index 64777427d..9e75efc31 100644 --- a/politeiawww/comments.go +++ b/politeiawww/comments.go @@ -10,27 +10,6 @@ import ( "github.com/decred/politeia/politeiawww/user" ) -// commentCensor calls the comments plugin to censor a given comment. -func (p *politeiawww) commentCensor(cc comments.Del) (*comments.DelReply, error) { - // Prep plugin payload - payload, err := comments.EncodeDel(cc) - if err != nil { - return nil, err - } - - r, err := p.pluginCommand(comments.ID, comments.CmdDel, "", - string(payload)) - if err != nil { - return nil, err - } - ccr, err := comments.DecodeDelReply(([]byte(r))) - if err != nil { - return nil, err - } - - return ccr, nil -} - func (p *politeiawww) commentVotes(vs comments.Votes) (*comments.VotesReply, error) { // Prep plugin payload payload, err := comments.EncodeVotes(vs) diff --git a/politeiawww/pi.go b/politeiawww/pi.go index 7779a2672..a897b5323 100644 --- a/politeiawww/pi.go +++ b/politeiawww/pi.go @@ -8,6 +8,27 @@ import ( piplugin "github.com/decred/politeia/plugins/pi" ) +// commentCensor calls the pi plugin to censor a given comment. +func (p *politeiawww) commentCensor(cc piplugin.CommentCensor) (*piplugin.CommentCensorReply, error) { + // Prep plugin payload + payload, err := piplugin.EncodeCommentCensor(cc) + if err != nil { + return nil, err + } + + r, err := p.pluginCommand(piplugin.ID, piplugin.CmdCommentCensor, "", + string(payload)) + if err != nil { + return nil, err + } + ccr, err := piplugin.DecodeCommentCensorReply(([]byte(r))) + if err != nil { + return nil, err + } + + return ccr, nil +} + // piCommentVote calls the pi plugin to vote on a comment. func (p *politeiawww) piCommentVote(cvp piplugin.CommentVote) (*piplugin.CommentVoteReply, error) { // Prep comment vote payload diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 69529ef4b..c30c0b03c 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -1722,9 +1722,9 @@ func (p *politeiawww) processCommentCensor(cc pi.CommentCensor, usr user.User) ( } } - // Call the comments plugin to censor comment - reply, err := p.commentCensor(comments.Del{ - State: convertCommentsPluginPropStateFromPi(cc.State), + // Call the pi plugin to censor comment + reply, err := p.commentCensor(piplugin.CommentCensor{ + State: convertPropStateFromPi(cc.State), Token: cc.Token, CommentID: cc.CommentID, Reason: cc.Reason, From 3ba1b958c70022f07ba5ff62b35e85ed2d0b0195 Mon Sep 17 00:00:00 2001 From: Amir Massarwa Date: Thu, 24 Sep 2020 15:23:26 +0300 Subject: [PATCH 102/449] politeiawww: Add processVotes. --- politeiawww/api/pi/v1/v1.go | 2 +- politeiawww/cmd/piwww/help.go | 2 + politeiawww/cmd/piwww/piwww.go | 9 ++- politeiawww/cmd/piwww/tally.go | 4 - politeiawww/cmd/piwww/votes.go | 67 +++++++++++++++++ politeiawww/cmd/shared/client.go | 24 ++++++ politeiawww/piwww.go | 122 +++++++++++++++++++++++++++++++ 7 files changed, 222 insertions(+), 8 deletions(-) create mode 100644 politeiawww/cmd/piwww/votes.go diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index ec7f7f901..336622e97 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -720,7 +720,7 @@ type Votes struct { Tokens []string `json:"tokens"` } -// VoteReply is the reply to the Votes command. The returned map will not +// VotesReply is the reply to the Votes command. The returned map will not // contain an entry for any tokens that did not correspond to an actual // proposal. It is the callers responsibility to ensure that a entry is // returned for all of the provided tokens. diff --git a/politeiawww/cmd/piwww/help.go b/politeiawww/cmd/piwww/help.go index 173dc5f66..a256cf0db 100644 --- a/politeiawww/cmd/piwww/help.go +++ b/politeiawww/cmd/piwww/help.go @@ -105,6 +105,8 @@ func (cmd *HelpCmd) Execute(args []string) error { fmt.Printf("%s\n", voteStartHelpMsg) case "votestartrunoff": fmt.Printf("%s\n", voteStartRunoffHelpMsg) + case "votes": + fmt.Printf("%s\n", votesHelpMsg) case "voteresults": fmt.Printf("%s\n", voteResultsHelpMsg) case "activevotes": diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index c5ca53ad9..d586fc281 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -60,6 +60,12 @@ type piwww struct { VoteStartRunoff VoteStartRunoffCmd `command:"votestartrunoff" description:"(admin) start a runoff using the submissions to an RFP"` VoteStart VoteStartCmd `command:"votestart" description:"(admin) start the voting period on a proposal"` VoteAuthorize VoteAuthorizeCmd `command:"voteauthorize" description:"(user) authorize a proposal vote (must be proposal author)"` + Votes VotesCmd `command:"votes" description:"(public) get the vote tally for a proposal"` + + // XXX will go + Tally TallyCmd `command:"tally" description:"(public) get the vote tally for a proposal"` + Inventory InventoryCmd `command:"inventory" description:"(public) get the proposals that are being voted on"` + Vote VoteCmd `command:"vote" description:"(public) cast votes for a proposal"` // Commands ActiveVotes ActiveVotesCmd `command:"activevotes" description:"(public) get the proposals that are being voted on"` @@ -69,7 +75,6 @@ type piwww struct { ChangeUsername shared.ChangeUsernameCmd `command:"changeusername" description:"(user) change the username for the logged in user"` EditUser EditUserCmd `command:"edituser" description:"(user) edit the preferences of the logged in user"` Help HelpCmd `command:"help" description:" print a detailed help message for a specific command"` - Inventory InventoryCmd `command:"inventory" description:"(public) get the proposals that are being voted on"` Login shared.LoginCmd `command:"login" description:"(public) login to Politeia"` Logout shared.LogoutCmd `command:"logout" description:"(public) logout of Politeia"` ManageUser shared.ManageUserCmd `command:"manageuser" description:"(admin) edit certain properties of the specified user"` @@ -85,7 +90,6 @@ type piwww struct { SendFaucetTx SendFaucetTxCmd `command:"sendfaucettx" description:" send a DCR transaction using the Decred testnet faucet"` SetTOTP shared.SetTOTPCmd `command:"settotp" description:"(user) set the key for TOTP"` Subscribe SubscribeCmd `command:"subscribe" description:"(public) subscribe to all websocket commands and do not exit tool"` - Tally TallyCmd `command:"tally" description:"(public) get the vote tally for a proposal"` TestRun TestRunCmd `command:"testrun" description:" run a series of tests on the politeiawww routes (dev use only)"` TokenInventory shared.TokenInventoryCmd `command:"tokeninventory" description:"(public) get the censorship record tokens of all proposals"` UpdateUserKey shared.UpdateUserKeyCmd `command:"updateuserkey" description:"(user) generate a new identity for the logged in user"` @@ -98,7 +102,6 @@ type piwww struct { VerifyTOTP shared.VerifyTOTPCmd `command:"verifytotp" description:"(user) verify the set code for TOTP"` Version shared.VersionCmd `command:"version" description:"(public) get server info and CSRF token"` VettedProposals VettedProposalsCmd `command:"vettedproposals" description:"(public) get a page of vetted proposals"` - Vote VoteCmd `command:"vote" description:"(public) cast votes for a proposal"` VoteDetails VoteDetailsCmd `command:"votedetails" description:"(public) get the details for a proposal vote"` VoteResults VoteResultsCmd `command:"voteresults" description:"(public) get vote results for a proposal"` VoteStatus VoteStatusCmd `command:"votestatus" description:"(public) get the vote status of a proposal"` diff --git a/politeiawww/cmd/piwww/tally.go b/politeiawww/cmd/piwww/tally.go index 01b0db6d1..b9494e513 100644 --- a/politeiawww/cmd/piwww/tally.go +++ b/politeiawww/cmd/piwww/tally.go @@ -58,14 +58,10 @@ func (cmd *TallyCmd) Execute(args []string) error { // tallyHelpMsg is the output for the help command when 'tally' is specified. const tallyHelpMsg = `tally "token" - Fetch the vote tally for a proposal. - Arguments: 1. token (string, required) Proposal censorship token - Response: - Vote Option: ID : (string) Unique word identifying vote (e.g. 'no') Description : (string) Longer description of the vote diff --git a/politeiawww/cmd/piwww/votes.go b/politeiawww/cmd/piwww/votes.go new file mode 100644 index 000000000..188ad3881 --- /dev/null +++ b/politeiawww/cmd/piwww/votes.go @@ -0,0 +1,67 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + + pi "github.com/decred/politeia/politeiawww/api/pi/v1" + "github.com/decred/politeia/politeiawww/cmd/shared" +) + +// VotesCmd retrieves vote details for a proposal, tallies the votes, +// and displays the result. +type VotesCmd struct { + Args struct { + Token string `positional-arg-name:"token"` // Censorship token + } `positional-args:"true" required:"true"` +} + +// Execute executes the tally command. +func (cmd *VotesCmd) Execute(args []string) error { + token := cmd.Args.Token + + // Prep request payload + v := pi.Votes{ + Tokens: []string{token}, + } + + // Print request details + err := shared.PrintJSON(v) + if err != nil { + return err + } + + // Get vote detials for proposal + vrr, err := client.Votes(v) + if err != nil { + return fmt.Errorf("ProposalVotes: %v", err) + } + + // Remove eligible tickets snapshot from response + // so that the output is legible + var ( + pv pi.ProposalVote + ok bool + ) + if pv, ok = vrr.Votes[token]; ok && !cfg.RawJSON { + pv.Vote.EligibleTickets = []string{ + "removed by politeiawwwcli for readability", + } + vrr.Votes[token] = pv + } + + // Print response details + return shared.PrintJSON(vrr) +} + +// votesHelpMsg is the output for the help command when 'tally' is specified. +const votesHelpMsg = `votes "token" + +Fetch the vote details for a proposal. + +Arguments: +1. token (string, required) Proposal censorship token +` diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index d291ce8e6..dbab4a602 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -1231,6 +1231,30 @@ func (c *Client) InvoiceComments(token string) (*www.GetCommentsReply, error) { return &gcr, nil } +// Votes rerieves the vote details for a given proposal. +func (c *Client) Votes(vs pi.Votes) (*pi.VotesReply, error) { + responseBody, err := c.makeRequest(http.MethodPost, + pi.APIRoute, pi.RouteVotes, vs) + if err != nil { + return nil, err + } + + var vsr pi.VotesReply + err = json.Unmarshal(responseBody, &vsr) + if err != nil { + return nil, fmt.Errorf("unmarshal Votes: %v", err) + } + + if c.cfg.Verbose { + err := prettyPrintJSON(vsr) + if err != nil { + return nil, err + } + } + + return &vsr, nil +} + // CommentVotes retrieves the comment likes (upvotes/downvotes) for the // specified proposal that are from the privoded user. func (c *Client) CommentVotes(cv pi.CommentVotes) (*pi.CommentVotesReply, error) { diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index c30c0b03c..685a05deb 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -2046,6 +2046,125 @@ func (p *politeiawww) handleVoteBallot(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, vbr) } +func convertVoteTypeFromPlugin(t ticketvote.VoteT) pi.VoteT { + switch t { + case ticketvote.VoteTypeStandard: + return pi.VoteTypeStandard + case ticketvote.VoteTypeRunoff: + return pi.VoteTypeRunoff + } + return pi.VoteTypeInvalid + +} + +func convertVoteParamsFromPlugin(v ticketvote.VoteParams) pi.VoteParams { + vp := pi.VoteParams{ + Token: v.Token, + Version: v.Version, + Type: convertVoteTypeFromPlugin(v.Type), + Mask: v.Mask, + Duration: v.Duration, + QuorumPercentage: v.QuorumPercentage, + PassPercentage: v.PassPercentage, + } + vo := make([]pi.VoteOption, 0, len(v.Options)) + for _, o := range v.Options { + vo = append(vo, pi.VoteOption{ + ID: o.ID, + Description: o.Description, + Bit: o.Bit, + }) + } + vp.Options = vo + + return vp +} + +func convertVoteDetailsFromPlugin(vd ticketvote.VoteDetails) *pi.VoteDetails { + return &pi.VoteDetails{ + Params: convertVoteParamsFromPlugin(vd.Params), + PublicKey: vd.PublicKey, + Signature: vd.Signature, + StartBlockHeight: vd.StartBlockHeight, + StartBlockHash: vd.StartBlockHash, + EndBlockHeight: vd.EndBlockHeight, + EligibleTickets: vd.EligibleTickets, + } +} + +func convertAuthorizeDetailsFromPlugin(ad ticketvote.AuthorizeDetails) pi.AuthorizeDetails { + return pi.AuthorizeDetails{ + Token: ad.Token, + Version: ad.Version, + Action: ad.Action, + PublicKey: ad.PublicKey, + Signature: ad.Signature, + Timestamp: ad.Timestamp, + Receipt: ad.Receipt, + } +} + +func (p *politeiawww) processVotes(v pi.Votes) (*pi.VotesReply, error) { + log.Tracef("processVotes: %v", v.Tokens) + + // Call the ticketvote plugin to get vote details + vd, err := p.voteDetails(v.Tokens) + if err != nil { + return nil, err + } + + // Convert reply to pi + var ( + vote ticketvote.RecordVote + ok bool + vr pi.VotesReply + ) + votes := make(map[string]pi.ProposalVote) + for _, token := range v.Tokens { + if vote, ok = vd.Votes[token]; !ok { + // No related vote details, skip token + continue + } + pv := pi.ProposalVote{ + Vote: convertVoteDetailsFromPlugin(*(vote.Vote)), + } + + // Transalte vote auth + auths := make([]pi.AuthorizeDetails, 0, len(vote.Auths)) + for _, a := range vote.Auths { + auths = append(auths, convertAuthorizeDetailsFromPlugin(a)) + } + pv.Auths = auths + + votes[token] = pv + } + vr.Votes = votes + + return &vr, nil +} + +func (p *politeiawww) handleVotes(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleVotes") + + var v pi.Votes + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&v); err != nil { + respondWithPiError(w, r, "handleVotes: unmarshal", + pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusInvalidInput, + }) + return + } + + vr, err := p.processVotes(v) + if err != nil { + respondWithPiError(w, r, + "handleVotes: processVotes: %v", err) + } + + util.RespondWithJSON(w, http.StatusOK, vr) +} + func (p *politeiawww) setPiRoutes() { // Public routes p.addRoute(http.MethodGet, pi.APIRoute, @@ -2067,6 +2186,9 @@ func (p *politeiawww) setPiRoutes() { p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteVoteBallot, p.handleVoteBallot, permissionPublic) + p.addRoute(http.MethodPost, pi.APIRoute, + pi.RouteVotes, p.handleVotes, permissionPublic) + // Logged in routes p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteProposalNew, p.handleProposalNew, From 12be4691bc2183afa4d45e0446efa9e5dd519485 Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Thu, 24 Sep 2020 12:33:26 -0300 Subject: [PATCH 103/449] piwww: Add proposalinventory command. --- politeiawww/cmd/piwww/help.go | 2 ++ politeiawww/cmd/piwww/piwww.go | 1 + politeiawww/cmd/piwww/proposalinventory.go | 31 ++++++++++++++++++++++ politeiawww/cmd/shared/client.go | 25 +++++++++++++++++ 4 files changed, 59 insertions(+) create mode 100644 politeiawww/cmd/piwww/proposalinventory.go diff --git a/politeiawww/cmd/piwww/help.go b/politeiawww/cmd/piwww/help.go index a256cf0db..e81b92393 100644 --- a/politeiawww/cmd/piwww/help.go +++ b/politeiawww/cmd/piwww/help.go @@ -76,6 +76,8 @@ func (cmd *HelpCmd) Execute(args []string) error { fmt.Printf("%s\n", proposalEditHelpMsg) case "proposalsetstatus": fmt.Printf("%s\n", proposalSetStatusHelpMsg) + case "proposalinventory": + fmt.Printf("%s\n", proposalInventoryHelpMsg) case "proposaldetails": fmt.Printf("%s\n", proposalDetailsHelpMsg) diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index d586fc281..a12cbbd56 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -48,6 +48,7 @@ type piwww struct { ProposalNew ProposalNewCmd `command:"proposalnew"` ProposalEdit ProposalEditCmd `command:"proposaledit"` ProposalSetStatus ProposalSetStatusCmd `command:"proposalsetstatus"` + ProposalInventory ProposalInventoryCmd `command:"proposalinventory"` // Comments commands CommentNew CommentNewCmd `command:"commentnew" description:"(user) create a new comment"` diff --git a/politeiawww/cmd/piwww/proposalinventory.go b/politeiawww/cmd/piwww/proposalinventory.go new file mode 100644 index 000000000..77a8ef83d --- /dev/null +++ b/politeiawww/cmd/piwww/proposalinventory.go @@ -0,0 +1,31 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "github.com/decred/politeia/politeiawww/cmd/shared" +) + +// ProposalInventoryCmd retrieves the censorship record tokens of all proposals in +// the inventory. +type ProposalInventoryCmd struct{} + +// Execute executes the proposal inventory command. +func (cmd *ProposalInventoryCmd) Execute(args []string) error { + reply, err := client.ProposalInventory() + if err != nil { + return err + } + + return shared.PrintJSON(reply) +} + +// proposalInventoryHelpMsg is the output of the help command when +// 'proposalinventory' is specified. +const proposalInventoryHelpMsg = `proposalinventory + +Fetch the censorship record tokens for all proposals, separated by their +status. The unvetted tokens are only returned if the logged in user is an +admin.` diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index dbab4a602..c2c61fdc9 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -1061,6 +1061,31 @@ func (c *Client) PayInvoices(pi *cms.PayInvoices) (*cms.PayInvoicesReply, error) return &pir, nil } +// ProposalInventory retrieves the censorship tokens of all proposals, +// separated by their status. +func (c *Client) ProposalInventory() (*pi.ProposalInventoryReply, error) { + respondeBody, err := c.makeRequest(http.MethodGet, pi.APIRoute, + pi.RouteProposalInventory, nil) + if err != nil { + return nil, err + } + + var pir pi.ProposalInventoryReply + err = json.Unmarshal(respondeBody, &pir) + if err != nil { + return nil, fmt.Errorf("unmarshal ProposalInventory: %v", err) + } + + if c.cfg.Verbose { + err := prettyPrintJSON(pir) + if err != nil { + return nil, err + } + } + + return &pir, nil +} + // BatchProposals retrieves a list of proposals func (c *Client) BatchProposals(bp *www.BatchProposals) (*www.BatchProposalsReply, error) { responseBody, err := c.makeRequest(http.MethodPost, www.PoliteiaWWWAPIRoute, From d2f20440e499e45495157245694d63f4fbd18cf2 Mon Sep 17 00:00:00 2001 From: Amir Massarwa Date: Thu, 24 Sep 2020 20:53:02 +0300 Subject: [PATCH 104/449] politeiawww: Add processVoteResults. --- politeiawww/api/pi/v1/v1.go | 3 ++ politeiawww/cmd/piwww/help.go | 2 - politeiawww/cmd/piwww/piwww.go | 3 +- politeiawww/cmd/piwww/tally.go | 76 ---------------------------- politeiawww/cmd/piwww/voteresults.go | 61 ++++++---------------- politeiawww/cmd/shared/client.go | 7 ++- politeiawww/piwww.go | 50 ++++++++++++++++++ politeiawww/politeiawww.go | 12 ++--- politeiawww/proposals.go | 4 +- 9 files changed, 81 insertions(+), 137 deletions(-) delete mode 100644 politeiawww/cmd/piwww/tally.go diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 336622e97..0585fbcb4 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -728,10 +728,13 @@ type VotesReply struct { Votes map[string]ProposalVote `json:"votes"` } +// VoteResults returns the votes that have been cast for the specified +// proposal. type VoteResults struct { Token string `json:"token"` } +// VoteResultsReply is the reply to the VoteResults command. type VoteResultsReply struct { Votes []CastVoteDetails `json:"votes"` } diff --git a/politeiawww/cmd/piwww/help.go b/politeiawww/cmd/piwww/help.go index e81b92393..422110e78 100644 --- a/politeiawww/cmd/piwww/help.go +++ b/politeiawww/cmd/piwww/help.go @@ -135,8 +135,6 @@ func (cmd *HelpCmd) Execute(args []string) error { // politeiavoter mock commands case "inventory": fmt.Printf("%s\n", inventoryHelpMsg) - case "tally": - fmt.Printf("%s\n", tallyHelpMsg) case "vote": fmt.Printf("%s\n", voteHelpMsg) diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index a12cbbd56..ee79a3770 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -62,9 +62,9 @@ type piwww struct { VoteStart VoteStartCmd `command:"votestart" description:"(admin) start the voting period on a proposal"` VoteAuthorize VoteAuthorizeCmd `command:"voteauthorize" description:"(user) authorize a proposal vote (must be proposal author)"` Votes VotesCmd `command:"votes" description:"(public) get the vote tally for a proposal"` + VoteResults VoteResultsCmd `command:"voteresults" description:"(public) get vote results for a proposal"` // XXX will go - Tally TallyCmd `command:"tally" description:"(public) get the vote tally for a proposal"` Inventory InventoryCmd `command:"inventory" description:"(public) get the proposals that are being voted on"` Vote VoteCmd `command:"vote" description:"(public) cast votes for a proposal"` @@ -104,7 +104,6 @@ type piwww struct { Version shared.VersionCmd `command:"version" description:"(public) get server info and CSRF token"` VettedProposals VettedProposalsCmd `command:"vettedproposals" description:"(public) get a page of vetted proposals"` VoteDetails VoteDetailsCmd `command:"votedetails" description:"(public) get the details for a proposal vote"` - VoteResults VoteResultsCmd `command:"voteresults" description:"(public) get vote results for a proposal"` VoteStatus VoteStatusCmd `command:"votestatus" description:"(public) get the vote status of a proposal"` VoteStatuses VoteStatusesCmd `command:"votestatuses" description:"(public) get the vote status for all public proposals"` } diff --git a/politeiawww/cmd/piwww/tally.go b/politeiawww/cmd/piwww/tally.go deleted file mode 100644 index b9494e513..000000000 --- a/politeiawww/cmd/piwww/tally.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" - "strconv" -) - -// TallyCmd retrieves all of the cast votes for a proposal, tallies the votes, -// and displays the result. -type TallyCmd struct { - Args struct { - Token string `positional-arg-name:"token"` // Censorship token - } `positional-args:"true" required:"true"` -} - -// Execute executes the tally command. -func (cmd *TallyCmd) Execute(args []string) error { - // Get vote results for proposal - vrr, err := client.VoteResults(cmd.Args.Token) - if err != nil { - return fmt.Errorf("ProposalVotes: %v", err) - } - - // Tally votes - var total uint - tally := make(map[uint64]uint) - for _, v := range vrr.CastVotes { - bits, err := strconv.ParseUint(v.VoteBit, 10, 64) - if err != nil { - return err - } - tally[bits]++ - total++ - } - - if total == 0 { - return fmt.Errorf("no votes recorded") - } - - // Print results - for _, vo := range vrr.StartVote.Vote.Options { - votes := tally[vo.Bits] - fmt.Printf("Vote Option:\n") - fmt.Printf(" ID : %v\n", vo.Id) - fmt.Printf(" Description : %v\n", vo.Description) - fmt.Printf(" Bits : %v\n", vo.Bits) - fmt.Printf(" Votes received : %v\n", votes) - fmt.Printf(" Percentage : %v%%\n", - float64(votes)/float64(total)*100) - } - - return nil -} - -// tallyHelpMsg is the output for the help command when 'tally' is specified. -const tallyHelpMsg = `tally "token" -Fetch the vote tally for a proposal. -Arguments: -1. token (string, required) Proposal censorship token -Response: -Vote Option: - ID : (string) Unique word identifying vote (e.g. 'no') - Description : (string) Longer description of the vote - Bits : (uint64) Bits used for this option (e.g. '1') - Votes received : (uint) Number of votes received - Percentage : (float64) Percentage of votes for vote option -Vote Option: - ID : (string) Unique word identifying vote (e.g. 'yes') - Description : (string) Longer description of the vote - Bits : (uint64) Bits used for this option (e.g. '2') - Votes received : (uint) Number of votes received - Percentage : (float64) Percentage of votes for vote option` diff --git a/politeiawww/cmd/piwww/voteresults.go b/politeiawww/cmd/piwww/voteresults.go index b2eab7881..ed0ffb47c 100644 --- a/politeiawww/cmd/piwww/voteresults.go +++ b/politeiawww/cmd/piwww/voteresults.go @@ -1,10 +1,13 @@ -// Copyright (c) 2017-2019 The Decred developers +// Copyright (c) 2017-2020 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. package main -import "github.com/decred/politeia/politeiawww/cmd/shared" +import ( + pi "github.com/decred/politeia/politeiawww/api/pi/v1" + "github.com/decred/politeia/politeiawww/cmd/shared" +) // VoteResultsCmd gets the votes that have been cast for the specified // proposal. @@ -16,17 +19,20 @@ type VoteResultsCmd struct { // Execute executes the proposal votes command. func (cmd *VoteResultsCmd) Execute(args []string) error { - vrr, err := client.VoteResults(cmd.Args.Token) + // Prep request payload + vr := pi.VoteResults{ + Token: cmd.Args.Token, + } + + // Print request details + err := shared.PrintJSON(vr) if err != nil { return err } - // Remove eligible tickets snapshot from response - // so that the output is legible - if !cfg.RawJSON { - vrr.StartVoteReply.EligibleTickets = []string{ - "removed by politeiawwwcli for readability", - } + vrr, err := client.VoteResults(vr) + if err != nil { + return err } return shared.PrintJSON(vrr) @@ -40,39 +46,4 @@ Fetch vote results for a proposal. Arguments: 1. token (string, required) Proposal censorship token - -Request: -{ - "token": (string) Proposal censorship token -} - -Response: -{ - "startvote": { - "publickey" (string) Public key of user that submitted proposal - "vote": { - "token": (string) Censorship token - "mask" (uint64) Valid votebits - "duration": (uint32) Duration of vote in blocks - "quorumpercentage" (uint32) Percent of votes required for quorum - "passpercentage": (uint32) Percent of votes required to pass - "options": [ - { - "id" (string) Unique word identifying vote (e.g. yes) - "description" (string) Longer description of the vote - "bits": (uint64) Bits used for this option - }, - ] - }, - "signature" (string) Signature of Votehash - }, - "castvotes": [], - "startvotereply": { - "startblockheight": (string) Block height at start of vote - "startblockhash": (string) Hash of first block of vote interval - "endheight": (string) Block height at end of vote - "eligibletickets": [ - "removed by politeiawwwcli for readability" - ] - } -}` +` diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index c2c61fdc9..e6afa8c18 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -1455,15 +1455,14 @@ func (c *Client) VerifyUserPayment() (*www.VerifyUserPaymentReply, error) { } // VoteResults retrieves the vote results for the specified proposal. -func (c *Client) VoteResults(token string) (*www.VoteResultsReply, error) { - route := "/proposals/" + token + "/votes" +func (c *Client) VoteResults(vr pi.VoteResults) (*pi.VoteResultsReply, error) { responseBody, err := c.makeRequest(http.MethodGet, - www.PoliteiaWWWAPIRoute, route, nil) + pi.APIRoute, pi.RouteVoteResults, vr) if err != nil { return nil, err } - var vrr www.VoteResultsReply + var vrr pi.VoteResultsReply err = json.Unmarshal(responseBody, &vrr) if err != nil { return nil, fmt.Errorf("unmarshal ProposalVotesReply: %v", err) diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 685a05deb..e318e4890 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -2165,6 +2165,53 @@ func (p *politeiawww) handleVotes(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, vr) } +func (p *politeiawww) processVoteResults(vr pi.VoteResults) (*pi.VoteResultsReply, error) { + log.Tracef("processVoteResults: %v", vr.Token) + + // Get cast votes information + cv, err := p.castVotes(vr.Token) + if err != nil { + return nil, err + } + + // Transalte to pi + var vrr pi.VoteResultsReply + votes := make([]pi.CastVoteDetails, 0, len(cv.Votes)) + for _, v := range cv.Votes { + votes = append(votes, pi.CastVoteDetails{ + Token: v.Token, + Ticket: v.Ticket, + VoteBit: v.VoteBit, + Signature: v.Signature, + }) + } + vrr.Votes = votes + + return &vrr, nil +} + +func (p *politeiawww) handleVoteResults(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleVoteResults") + + var vr pi.VoteResults + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&vr); err != nil { + respondWithPiError(w, r, "handleVoteResults: unmarshal", + pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusInvalidInput, + }) + return + } + + vrr, err := p.processVoteResults(vr) + if err != nil { + respondWithPiError(w, r, + "handleVoteResults: prcoessVoteResults: %v", err) + } + + util.RespondWithJSON(w, http.StatusOK, vrr) +} + func (p *politeiawww) setPiRoutes() { // Public routes p.addRoute(http.MethodGet, pi.APIRoute, @@ -2189,6 +2236,9 @@ func (p *politeiawww) setPiRoutes() { p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteVotes, p.handleVotes, permissionPublic) + p.addRoute(http.MethodPost, pi.APIRoute, + pi.RouteVoteResults, p.handleVoteResults, permissionPublic) + // Logged in routes p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteProposalNew, p.handleProposalNew, diff --git a/politeiawww/politeiawww.go b/politeiawww/politeiawww.go index 96d1798de..3196711d5 100644 --- a/politeiawww/politeiawww.go +++ b/politeiawww/politeiawww.go @@ -301,17 +301,17 @@ func (p *politeiawww) handleCastVotes(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, avr) } -// handleVoteResults returns a proposal + all voting action. -func (p *politeiawww) handleVoteResults(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleVoteResults") +// handleVoteResultsWWW returns a proposal + all voting action. +func (p *politeiawww) handleVoteResultsWWW(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleVoteResultsWWW") pathParams := mux.Vars(r) token := pathParams["token"] - vrr, err := p.processVoteResults(token) + vrr, err := p.processVoteResultsWWW(token) if err != nil { RespondWithError(w, r, 0, - "handleVoteResults: processVoteResults %v", + "handleVoteResultsWWW: processVoteResultsWWW %v", err) return } @@ -660,7 +660,7 @@ func (p *politeiawww) setPoliteiaWWWRoutes() { www.RouteCastVotes, p.handleCastVotes, permissionPublic) p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteVoteResults, p.handleVoteResults, + www.RouteVoteResults, p.handleVoteResultsWWW, permissionPublic) p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, www.RouteBatchVoteSummary, p.handleBatchVoteSummary, diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 8c6e836ea..cbce272ec 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -275,8 +275,8 @@ func (p *politeiawww) processActiveVote() (*www.ActiveVoteReply, error) { return nil, nil } -func (p *politeiawww) processVoteResults(token string) (*www.VoteResultsReply, error) { - log.Tracef("processVoteResults: %v", token) +func (p *politeiawww) processVoteResultsWWW(token string) (*www.VoteResultsReply, error) { + log.Tracef("processVoteResultsWWW: %v", token) // Call ticketvote plugin vd, err := p.voteDetails([]string{token}) From 0644967285ec9e1e4f3d05745d3645afd6000f8b Mon Sep 17 00:00:00 2001 From: Amir Massarwa Date: Fri, 25 Sep 2020 00:49:30 +0300 Subject: [PATCH 105/449] politeiawww: Add processVoteSummaries. --- politeiawww/api/pi/v1/v1.go | 21 +++++- politeiawww/cmd/piwww/batchvotesummary.go | 58 -------------- politeiawww/cmd/piwww/help.go | 4 +- politeiawww/cmd/piwww/piwww.go | 2 +- politeiawww/cmd/piwww/votesummaries.go | 35 +++++++++ politeiawww/cmd/shared/client.go | 16 ++-- politeiawww/piwww.go | 92 +++++++++++++++++++++++ 7 files changed, 157 insertions(+), 71 deletions(-) delete mode 100644 politeiawww/cmd/piwww/batchvotesummary.go create mode 100644 politeiawww/cmd/piwww/votesummaries.go diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 0585fbcb4..54fd97e6a 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -739,8 +739,25 @@ type VoteResultsReply struct { Votes []CastVoteDetails `json:"votes"` } -type VoteSummaries struct{} -type VoteSummariesReply struct{} +// VoteSummaries summarizes the vote params and results for a ticket vote. +type VoteSummaries struct { + Tokens []string `json:"tokens"` +} + +// VoteSummariesReply is the reply to the VoteSummaries command. +// +// Summaries field contains a vote summary for each of the provided +// tokens. The map will not contain an entry for any tokens that +// did not correspond to an actual record. It is the callers +// responsibility to ensure that a summary is returned for all of +// the provided tokens. +type VoteSummariesReply struct { + Summaries map[string]VoteSummary `json:"summaries"` // [token]Summary + + // BestBlock is the best block value that was used to prepare the + // summaries. + BestBlock uint32 `json:"bestblock"` +} type VoteInventory struct{} type VoteInventoryReply struct{} diff --git a/politeiawww/cmd/piwww/batchvotesummary.go b/politeiawww/cmd/piwww/batchvotesummary.go deleted file mode 100644 index 0cc4dd014..000000000 --- a/politeiawww/cmd/piwww/batchvotesummary.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import ( - v1 "github.com/decred/politeia/politeiawww/api/www/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" -) - -// BatchVoteSummaryCmd retrieves a set of proposal vote summaries. -type BatchVoteSummaryCmd struct{} - -// Execute executes the batch vote summaries command. -func (cmd *BatchVoteSummaryCmd) Execute(args []string) error { - bpr, err := client.BatchVoteSummary(&v1.BatchVoteSummary{ - Tokens: args, - }) - if err != nil { - return err - } - - return shared.PrintJSON(bpr) -} - -// batchVoteSummaryHelpMsg is the output for the help command when -// 'batchvotesummary' is specified. -const batchVoteSummaryHelpMsg = `batchvotesummary - -Fetch a summary of the voting process for a list of proposals. - -Example: -batchvotesummary token1 token2 - -Result: -{ - "statuses": { - "token": {( (string) Censorship token of proposal - "status": (int) Vote status code, - "eligibletickets": (uint32) Number of tickets eligible to vote - "endheight": (uint64) Final voting block of proposal - "bestblock": (uint64) Current block - "quorumpercentage": (uint32) Percent of eligible votes required for quorum - "passpercentage": (uint32) Percent of total votes required to pass - "results": [ - { - "option": { - "id": (string) Unique word identifying vote (e.g. 'yes') - "description": (string) Longer description of the vote - "bits": (uint64) Bits used for this option - }, - "votesreceived": (uint64) Number of votes received - } - ] - } - } -}` diff --git a/politeiawww/cmd/piwww/help.go b/politeiawww/cmd/piwww/help.go index 422110e78..7589ceb9f 100644 --- a/politeiawww/cmd/piwww/help.go +++ b/politeiawww/cmd/piwww/help.go @@ -117,8 +117,8 @@ func (cmd *HelpCmd) Execute(args []string) error { fmt.Printf("%s\n", voteStatusHelpMsg) case "votestatuses": fmt.Printf("%s\n", voteStatusesHelpMsg) - case "batchvotesummary": - fmt.Printf("%s\n", batchVoteSummaryHelpMsg) + case "votesummaries": + fmt.Printf("%s\n", voteSummariesHelpMsg) case "votedetails": fmt.Printf("%s\n", voteDetailsHelpMsg) diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index ee79a3770..f160a141f 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -63,6 +63,7 @@ type piwww struct { VoteAuthorize VoteAuthorizeCmd `command:"voteauthorize" description:"(user) authorize a proposal vote (must be proposal author)"` Votes VotesCmd `command:"votes" description:"(public) get the vote tally for a proposal"` VoteResults VoteResultsCmd `command:"voteresults" description:"(public) get vote results for a proposal"` + VoteSummaries VoteSummariesCmd `command:"votesummaries" description:"(user) retrieve the vote summary for a set of proposals"` // XXX will go Inventory InventoryCmd `command:"inventory" description:"(public) get the proposals that are being voted on"` @@ -71,7 +72,6 @@ type piwww struct { // Commands ActiveVotes ActiveVotesCmd `command:"activevotes" description:"(public) get the proposals that are being voted on"` BatchProposals shared.BatchProposalsCmd `command:"batchproposals" description:"(user) retrieve a set of proposals"` - BatchVoteSummary BatchVoteSummaryCmd `command:"batchvotesummary" description:"(user) retrieve the vote summary for a set of proposals"` ChangePassword shared.ChangePasswordCmd `command:"changepassword" description:"(user) change the password for the logged in user"` ChangeUsername shared.ChangeUsernameCmd `command:"changeusername" description:"(user) change the username for the logged in user"` EditUser EditUserCmd `command:"edituser" description:"(user) edit the preferences of the logged in user"` diff --git a/politeiawww/cmd/piwww/votesummaries.go b/politeiawww/cmd/piwww/votesummaries.go new file mode 100644 index 000000000..50ed42a20 --- /dev/null +++ b/politeiawww/cmd/piwww/votesummaries.go @@ -0,0 +1,35 @@ +// Copyright (c) 2017-2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + pi "github.com/decred/politeia/politeiawww/api/pi/v1" + "github.com/decred/politeia/politeiawww/cmd/shared" +) + +// VoteSummariesCmd retrieves a set of proposal vote summaries. +type VoteSummariesCmd struct{} + +// Execute executes the batch vote summaries command. +func (cmd *VoteSummariesCmd) Execute(args []string) error { + bpr, err := client.VoteSummaries(&pi.VoteSummaries{ + Tokens: args, + }) + if err != nil { + return err + } + + return shared.PrintJSON(bpr) +} + +// voteSummariesHelpMsg is the output for the help command when +// 'votesummaries' is specified. +const voteSummariesHelpMsg = `votesummaries + +Fetch a summary of the voting process for a list of proposals. + +Example: +votesummaries token1 token2 +` diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index e6afa8c18..e76541fdf 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -1110,29 +1110,29 @@ func (c *Client) BatchProposals(bp *www.BatchProposals) (*www.BatchProposalsRepl return &bpr, nil } -// BatchVoteSummary retrieves a summary of the voting process for a set of +// VoteSummaries retrieves a summary of the voting process for a set of // proposals. -func (c *Client) BatchVoteSummary(bvs *www.BatchVoteSummary) (*www.BatchVoteSummaryReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, www.PoliteiaWWWAPIRoute, - www.RouteBatchVoteSummary, bvs) +func (c *Client) VoteSummaries(vs *pi.VoteSummaries) (*pi.VoteSummariesReply, error) { + responseBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, + pi.RouteVoteSummaries, vs) if err != nil { return nil, err } - var bvsr www.BatchVoteSummaryReply - err = json.Unmarshal(responseBody, &bvsr) + var vsr pi.VoteSummariesReply + err = json.Unmarshal(responseBody, &vsr) if err != nil { return nil, fmt.Errorf("unmarshal BatchVoteSummary: %v", err) } if c.cfg.Verbose { - err := prettyPrintJSON(bvsr) + err := prettyPrintJSON(vsr) if err != nil { return nil, err } } - return &bvsr, nil + return &vsr, nil } // GetAllVetted retrieves a page of vetted proposals. diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index e318e4890..3be68b1a3 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -2212,6 +2212,95 @@ func (p *politeiawww) handleVoteResults(w http.ResponseWriter, r *http.Request) util.RespondWithJSON(w, http.StatusOK, vrr) } +func convertVoteStatusFromPlugin(s ticketvote.VoteStatusT) pi.VoteStatusT { + switch s { + case ticketvote.VoteStatusInvalid: + return pi.VoteStatusInvalid + case ticketvote.VoteStatusUnauthorized: + return pi.VoteStatusUnauthorized + case ticketvote.VoteStatusAuthorized: + return pi.VoteStatusAuthorized + case ticketvote.VoteStatusStarted: + return pi.VoteStatusStarted + case ticketvote.VoteStatusFinished: + return pi.VoteStatusFinished + default: + return pi.VoteStatusInvalid + } +} + +func convertVoteSummaryFromPlugin(s ticketvote.Summary) pi.VoteSummary { + vs := pi.VoteSummary{ + Type: convertVoteTypeFromPlugin(s.Type), + Status: convertVoteStatusFromPlugin(s.Status), + Duration: s.Duration, + StartBlockHeight: s.StartBlockHeight, + StartBlockHash: s.StartBlockHash, + EndBlockHeight: s.EndBlockHeight, + EligibleTickets: s.EligibleTickets, + QuorumPercentage: s.QuorumPercentage, + Approved: s.Approved, + } + // Transalte results + rs := make([]pi.VoteResult, 0, len(s.Results)) + for _, v := range s.Results { + rs = append(rs, pi.VoteResult{ + ID: v.ID, + Description: v.Description, + VoteBit: v.VoteBit, + Votes: v.Votes, + }) + } + vs.Results = rs + + return vs +} + +func (p *politeiawww) processVoteSummaries(vs pi.VoteSummaries) (*pi.VoteSummariesReply, error) { + log.Tracef("processVoteSummaries: %v", vs.Tokens) + + // Call the ticket vote to get the vote summaries + r, err := p.voteSummaries(vs.Tokens) + if err != nil { + return nil, err + } + + // Translate to pi + vsr := pi.VoteSummariesReply{ + BestBlock: r.BestBlock, + } + // Translate summaries + sms := make(map[string]pi.VoteSummary) + for t, s := range r.Summaries { + sms[t] = convertVoteSummaryFromPlugin(s) + } + vsr.Summaries = sms + + return &vsr, nil +} + +func (p *politeiawww) handleVoteSummaries(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleVoteSummaries") + + var vs pi.VoteSummaries + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&vs); err != nil { + respondWithPiError(w, r, "handleVoteSummaries: unmarshal", + pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusInvalidInput, + }) + return + } + + vsr, err := p.processVoteSummaries(vs) + if err != nil { + respondWithPiError(w, r, "handleVoteSummaries: processVoteSummaries: %v", + err) + } + + util.RespondWithJSON(w, http.StatusOK, vsr) +} + func (p *politeiawww) setPiRoutes() { // Public routes p.addRoute(http.MethodGet, pi.APIRoute, @@ -2239,6 +2328,9 @@ func (p *politeiawww) setPiRoutes() { p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteVoteResults, p.handleVoteResults, permissionPublic) + p.addRoute(http.MethodPost, pi.APIRoute, + pi.RouteVoteSummaries, p.handleVoteSummaries, permissionPublic) + // Logged in routes p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteProposalNew, p.handleProposalNew, From 840cb7db9db3a0b69b1cb2f325c6cabb7e106d5a Mon Sep 17 00:00:00 2001 From: Amir Massarwa Date: Fri, 25 Sep 2020 15:05:30 +0300 Subject: [PATCH 106/449] politeiawww: Add processVoteInventory. --- politeiawww/api/pi/v1/v1.go | 16 ++- politeiawww/cmd/piwww/help.go | 4 +- politeiawww/cmd/piwww/inventory.go | 138 ------------------------- politeiawww/cmd/piwww/piwww.go | 6 +- politeiawww/cmd/piwww/voteinventory.go | 31 ++++++ politeiawww/cmd/shared/client.go | 25 +++++ politeiawww/pi.go | 33 ++++-- politeiawww/piwww.go | 50 ++++++++- 8 files changed, 150 insertions(+), 153 deletions(-) delete mode 100644 politeiawww/cmd/piwww/inventory.go create mode 100644 politeiawww/cmd/piwww/voteinventory.go diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 54fd97e6a..176473c88 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -759,5 +759,19 @@ type VoteSummariesReply struct { BestBlock uint32 `json:"bestblock"` } +// VoteInventory retrieves the tokens of all public, non-abandoned proposals +// catagorized by their vote status. type VoteInventory struct{} -type VoteInventoryReply struct{} + +// VoteInventoryReply in the reply to the VoteInventory command. +type VoteInventoryReply struct { + Unauthorized []string `json:"unauthorized"` + Authorized []string `json:"authorized"` + Started []string `json:"started"` + Approved []string `json:"approved"` + Rejected []string `json:"rejected"` + + // BestBlock is the best block value that was used to prepare the + // inventory. + BestBlock uint32 `json:"bestblock"` +} diff --git a/politeiawww/cmd/piwww/help.go b/politeiawww/cmd/piwww/help.go index 7589ceb9f..ea5199913 100644 --- a/politeiawww/cmd/piwww/help.go +++ b/politeiawww/cmd/piwww/help.go @@ -121,6 +121,8 @@ func (cmd *HelpCmd) Execute(args []string) error { fmt.Printf("%s\n", voteSummariesHelpMsg) case "votedetails": fmt.Printf("%s\n", voteDetailsHelpMsg) + case "voteinventory": + fmt.Printf("%s\n", voteInventoryHelpMsg) // Websocket commands case "subscribe": @@ -133,8 +135,6 @@ func (cmd *HelpCmd) Execute(args []string) error { fmt.Printf("%s\n", sendFaucetTxHelpMsg) // politeiavoter mock commands - case "inventory": - fmt.Printf("%s\n", inventoryHelpMsg) case "vote": fmt.Printf("%s\n", voteHelpMsg) diff --git a/politeiawww/cmd/piwww/inventory.go b/politeiawww/cmd/piwww/inventory.go deleted file mode 100644 index 43745c7a6..000000000 --- a/politeiawww/cmd/piwww/inventory.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" - "strconv" - - "github.com/decred/dcrwallet/rpc/walletrpc" -) - -// InventoryCmd retreives the proposals that are being voted on. -type InventoryCmd struct{} - -// Execute executes the inventory command. -func (cmd *InventoryCmd) Execute(args []string) error { - // Connect to user's wallet - err := client.LoadWalletClient() - if err != nil { - return fmt.Errorf("LoadWalletClient: %v", err) - } - defer client.Close() - - // Get all active proposal votes - avr, err := client.ActiveVotes() - if err != nil { - return fmt.Errorf("ActiveVotes: %v", err) - } - - // Get current block height - ar, err := client.WalletAccounts() - if err != nil { - return fmt.Errorf("WalletAccounts: %v", err) - } - - // Validate active votes and print the details of the - // votes that the user is eligible to vote in - for _, v := range avr.Votes { - // Ensure a CensorshipRecord exists - if v.Proposal.CensorshipRecord.Token == "" { - // This should not happen - fmt.Printf("skipping empty CensorshipRecord\n") - continue - } - - // Ensure vote bits are valid - if v.StartVote.Vote.Token == "" || v.StartVote.Vote.Mask == 0 || - v.StartVote.Vote.Options == nil { - // This should not happen - fmt.Printf("invalid vote bits: %v", v.Proposal.CensorshipRecord.Token) - continue - } - - // Ensure vote has not expired - endHeight, err := strconv.ParseInt(v.StartVoteReply.EndHeight, 10, 32) - if err != nil { - return err - } - - if int64(ar.CurrentBlockHeight) > endHeight { - // This should not happen - fmt.Printf("Vote expired: current %v > end %v %v\n", - endHeight, ar.CurrentBlockHeight, v.StartVote.Vote.Token) - continue - } - - // Ensure user has eligible tickets for this proposal vote - ticketPool, err := convertTicketHashes(v.StartVoteReply.EligibleTickets) - if err != nil { - return err - } - - ctr, err := client.CommittedTickets( - &walletrpc.CommittedTicketsRequest{ - Tickets: ticketPool, - }) - if err != nil { - return fmt.Errorf("CommittedTickets: %v", err) - } - - if len(ctr.TicketAddresses) == 0 { - // User doesn't have any eligible tickets - fmt.Printf("Token: %v\n", v.StartVote.Vote.Token) - fmt.Printf(" Proposal : %v\n", v.Proposal.Name) - fmt.Printf(" Eligible tickets: %v\n", len(ctr.TicketAddresses)) - continue - } - - // Print details for the active proposal votes where - // the user has eligible tickets - fmt.Printf("Token: %v\n", v.StartVote.Vote.Token) - fmt.Printf(" Proposal : %v\n", v.Proposal.Name) - fmt.Printf(" Eligible tickets: %v\n", len(ctr.TicketAddresses)) - fmt.Printf(" Start block : %v\n", v.StartVoteReply.StartBlockHeight) - fmt.Printf(" End block : %v\n", v.StartVoteReply.EndHeight) - fmt.Printf(" Mask : %v\n", v.StartVote.Vote.Mask) - for _, vo := range v.StartVote.Vote.Options { - fmt.Printf(" Vote Option:\n") - fmt.Printf(" ID : %v\n", vo.Id) - fmt.Printf(" Description : %v\n", - vo.Description) - fmt.Printf(" Bits : %v\n", vo.Bits) - fmt.Printf(" To choose this option: piwww vote %v %v\n", - v.StartVote.Vote.Token, vo.Id) - } - } - - return nil -} - -// inventoryHelpMsg is the output of the help command when 'inventory' is -// specified. -const inventoryHelpMsg = `inventory - -Fetch the proposals that are being voted on. - -Arguments: -None - -Response: - -Token: (string) Proposal censorship token - Proposal : (string) Proposal name - Eligible tickets: (int) Number of eligible tickets - Start block : (string) Block height at start of vote - End block : (string) Block height at end of vote - Mask : (uint64) Valid votebits - Vote Option: - ID : (string) Unique word identifying vote (e.g. 'no') - Description : (string) Longer description of the vote - Bits : (uint64) Bits used for this option (e.g. '1') - Vote Option: - ID : (string) Unique word identifying vote (e.g. 'yes') - Description : (string) Longer description of the vote - Bits : (uint64) Bits used for this option (e.g. '2') - To choose this option: piwww vote 'Token' 'ID'` diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index f160a141f..ebe64ba8c 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -63,11 +63,11 @@ type piwww struct { VoteAuthorize VoteAuthorizeCmd `command:"voteauthorize" description:"(user) authorize a proposal vote (must be proposal author)"` Votes VotesCmd `command:"votes" description:"(public) get the vote tally for a proposal"` VoteResults VoteResultsCmd `command:"voteresults" description:"(public) get vote results for a proposal"` - VoteSummaries VoteSummariesCmd `command:"votesummaries" description:"(user) retrieve the vote summary for a set of proposals"` + VoteSummaries VoteSummariesCmd `command:"votesummaries" description:"(public) retrieve the vote summary for a set of proposals"` + VoteInventory VoteInventoryCmd `command:"voteinventory" description:"(public) retrieve the tokens of all public, non-abandoned proposal separated by vote status"` // XXX will go - Inventory InventoryCmd `command:"inventory" description:"(public) get the proposals that are being voted on"` - Vote VoteCmd `command:"vote" description:"(public) cast votes for a proposal"` + Vote VoteCmd `command:"vote" description:"(public) cast votes for a proposal"` // Commands ActiveVotes ActiveVotesCmd `command:"activevotes" description:"(public) get the proposals that are being voted on"` diff --git a/politeiawww/cmd/piwww/voteinventory.go b/politeiawww/cmd/piwww/voteinventory.go new file mode 100644 index 000000000..03b49fda0 --- /dev/null +++ b/politeiawww/cmd/piwww/voteinventory.go @@ -0,0 +1,31 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "github.com/decred/politeia/politeiawww/cmd/shared" +) + +// VoteInventoryCmd retrieves the censorship record tokens of all public, +// non-abandoned proposals in inventory categorized by their vote status. +type VoteInventoryCmd struct{} + +// Execute executes the vote inventory command. +func (cmd *VoteInventoryCmd) Execute(args []string) error { + reply, err := client.VoteInventory() + if err != nil { + return err + } + + return shared.PrintJSON(reply) +} + +// voteInventoryHelpMsg is the output of the help command when +// 'voteinventory' is specified. +const voteInventoryHelpMsg = `voteinventory + +Fetch the censorship record tokens for all proposals, separated by their +vote status. The unvetted tokens are only returned if the logged in user is an +admin.` diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index e76541fdf..1ddf265ac 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -1061,6 +1061,31 @@ func (c *Client) PayInvoices(pi *cms.PayInvoices) (*cms.PayInvoicesReply, error) return &pir, nil } +// VoteInventory retrieves the tokens of all proposals in the inventory +// catagorized by their vote status. +func (c *Client) VoteInventory() (*pi.VoteInventoryReply, error) { + responseBody, err := c.makeRequest(http.MethodGet, pi.APIRoute, + pi.RouteVoteInventory, nil) + if err != nil { + return nil, err + } + + var vir pi.VoteInventoryReply + err = json.Unmarshal(responseBody, &vir) + if err != nil { + return nil, fmt.Errorf("unmarshal VoteInventory: %v", err) + } + + if c.cfg.Verbose { + err := prettyPrintJSON(vir) + if err != nil { + return nil, err + } + } + + return &vir, nil +} + // ProposalInventory retrieves the censorship tokens of all proposals, // separated by their status. func (c *Client) ProposalInventory() (*pi.ProposalInventoryReply, error) { diff --git a/politeiawww/pi.go b/politeiawww/pi.go index a897b5323..7f64fb3e6 100644 --- a/politeiawww/pi.go +++ b/politeiawww/pi.go @@ -8,8 +8,29 @@ import ( piplugin "github.com/decred/politeia/plugins/pi" ) -// commentCensor calls the pi plugin to censor a given comment. -func (p *politeiawww) commentCensor(cc piplugin.CommentCensor) (*piplugin.CommentCensorReply, error) { +// voteInventoryPi calls the pi plugin to retrieve the token inventory. +func (p *politeiawww) voteInventoryPi(vi piplugin.VoteInventory) (*piplugin.VoteInventoryReply, error) { + // Prep plugin payload + payload, err := piplugin.EncodeVoteInventory(vi) + if err != nil { + return nil, err + } + + r, err := p.pluginCommand(piplugin.ID, piplugin.CmdVoteInventory, "", + string(payload)) + if err != nil { + return nil, err + } + vir, err := piplugin.DecodeVoteInventoryReply(([]byte(r))) + if err != nil { + return nil, err + } + + return vir, nil +} + +// commentCensorPi calls the pi plugin to censor a given comment. +func (p *politeiawww) commentCensorPi(cc piplugin.CommentCensor) (*piplugin.CommentCensorReply, error) { // Prep plugin payload payload, err := piplugin.EncodeCommentCensor(cc) if err != nil { @@ -29,8 +50,8 @@ func (p *politeiawww) commentCensor(cc piplugin.CommentCensor) (*piplugin.Commen return ccr, nil } -// piCommentVote calls the pi plugin to vote on a comment. -func (p *politeiawww) piCommentVote(cvp piplugin.CommentVote) (*piplugin.CommentVoteReply, error) { +// commentVotePi calls the pi plugin to vote on a comment. +func (p *politeiawww) commentVotePi(cvp piplugin.CommentVote) (*piplugin.CommentVoteReply, error) { // Prep comment vote payload payload, err := piplugin.EncodeCommentVote(cvp) if err != nil { @@ -50,8 +71,8 @@ func (p *politeiawww) piCommentVote(cvp piplugin.CommentVote) (*piplugin.Comment return cvr, nil } -// piCommentNew calls the pi plugin to add new comment. -func (p *politeiawww) piCommentNew(cnp piplugin.CommentNew) (*piplugin.CommentNewReply, error) { +// commentNewPi calls the pi plugin to add new comment. +func (p *politeiawww) commentNewPi(cnp piplugin.CommentNew) (*piplugin.CommentNewReply, error) { // Prep new comment payload payload, err := piplugin.EncodeCommentNew(cnp) if err != nil { diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 3be68b1a3..6027d126c 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -1461,7 +1461,7 @@ func (p *politeiawww) processCommentNew(cn pi.CommentNew, usr user.User) (*pi.Co } // Call the pi plugin to add new comment - reply, err := p.piCommentNew(piplugin.CommentNew{ + reply, err := p.commentNewPi(piplugin.CommentNew{ UUID: usr.ID.String(), Token: cn.Token, ParentID: cn.ParentID, @@ -1553,7 +1553,7 @@ func (p *politeiawww) processCommentVote(cv pi.CommentVote, usr user.User) (*pi. } // Call the pi plugin to add new comment - reply, err := p.piCommentVote(piplugin.CommentVote{ + reply, err := p.commentVotePi(piplugin.CommentVote{ UUID: usr.ID.String(), Token: cv.Token, CommentID: cv.CommentID, @@ -1723,7 +1723,7 @@ func (p *politeiawww) processCommentCensor(cc pi.CommentCensor, usr user.User) ( } // Call the pi plugin to censor comment - reply, err := p.commentCensor(piplugin.CommentCensor{ + reply, err := p.commentCensorPi(piplugin.CommentCensor{ State: convertPropStateFromPi(cc.State), Token: cc.Token, CommentID: cc.CommentID, @@ -2301,6 +2301,47 @@ func (p *politeiawww) handleVoteSummaries(w http.ResponseWriter, r *http.Request util.RespondWithJSON(w, http.StatusOK, vsr) } +func (p *politeiawww) processVoteInventory() (*pi.VoteInventoryReply, error) { + log.Tracef("processVoteInventory") + + // Call the pi plugin vote inventory + r, err := p.voteInventoryPi(piplugin.VoteInventory{}) + if err != nil { + return nil, err + } + + return &pi.VoteInventoryReply{ + Unauthorized: r.Unauthorized, + Authorized: r.Authorized, + Started: r.Started, + Approved: r.Approved, + Rejected: r.Rejected, + BestBlock: r.BestBlock, + }, nil +} + +func (p *politeiawww) handleVoteInventory(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleVoteInventory") + + var vi pi.VoteInventory + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&vi); err != nil { + respondWithPiError(w, r, "handleVoteInventory: unmarshal", + pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusInvalidInput, + }) + return + } + + vir, err := p.processVoteInventory() + if err != nil { + respondWithPiError(w, r, "handleVoteInventory: processVoteInventory: %v", + err) + } + + util.RespondWithJSON(w, http.StatusOK, vir) +} + func (p *politeiawww) setPiRoutes() { // Public routes p.addRoute(http.MethodGet, pi.APIRoute, @@ -2331,6 +2372,9 @@ func (p *politeiawww) setPiRoutes() { p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteVoteSummaries, p.handleVoteSummaries, permissionPublic) + p.addRoute(http.MethodGet, pi.APIRoute, + pi.RouteVoteInventory, p.handleVoteInventory, permissionPublic) + // Logged in routes p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteProposalNew, p.handleProposalNew, From bfe9af15d596522c1af72eeaeaa76bd376448471 Mon Sep 17 00:00:00 2001 From: Amir Massarwa Date: Fri, 25 Sep 2020 21:32:43 +0300 Subject: [PATCH 107/449] piwww: Replace vote command with voteBallot. --- politeiawww/cmd/piwww/activevotes.go | 98 ------------------- politeiawww/cmd/piwww/help.go | 8 +- politeiawww/cmd/piwww/piwww.go | 5 +- politeiawww/cmd/piwww/testrun.go | 8 +- .../cmd/piwww/{vote.go => voteballot.go} | 55 +++++------ politeiawww/cmd/shared/client.go | 24 ----- 6 files changed, 31 insertions(+), 167 deletions(-) delete mode 100644 politeiawww/cmd/piwww/activevotes.go rename politeiawww/cmd/piwww/{vote.go => voteballot.go} (82%) diff --git a/politeiawww/cmd/piwww/activevotes.go b/politeiawww/cmd/piwww/activevotes.go deleted file mode 100644 index d7bb13076..000000000 --- a/politeiawww/cmd/piwww/activevotes.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import "github.com/decred/politeia/politeiawww/cmd/shared" - -// ActiveVotesCmd retreives all proposals that are currently being voted on. -type ActiveVotesCmd struct{} - -// Execute executes the active votes command. -func (cmd *ActiveVotesCmd) Execute(args []string) error { - // Send request - avr, err := client.ActiveVotes() - if err != nil { - return err - } - - // Remove the ticket snapshots from the response so that the - // output is legible - if !cfg.RawJSON { - for k := range avr.Votes { - avr.Votes[k].StartVoteReply.EligibleTickets = []string{ - "removed by politeiawwwcli for readability", - } - } - } - - // Print response details - return shared.PrintJSON(avr) -} - -// activeVotesHelpMsg is the output for the help command when 'activevotes' -// is specified. -const activeVotesHelpMsg = `activevotes "token" - -Retrieve all proposals that are currently being voted on. - -Arguments: None - -Result: -{ - "votes": [ - "proposal": { - "name": (string) Suggested short proposal name - "state": (PropStateT) Current state of proposal - "status": (PropStatusT) Current status of proposal - "timestamp": (int64) Timestamp of last update of proposal - "userid": (string) ID of user who submitted proposal - "username": (string) Username of user who submitted proposal - "publickey": (string) Public key used to sign proposal - "signature": (string) Signature of merkle root - "files": [ - { - "name": (string) Filename - "mime": (string) Mime type - "digest": (string) File digest - "payload": (string) File payload - } - ], - "numcomments": (uint) Number of comments on the proposal - "version": (string) Version of proposal - "censorshiprecord": { - "token": (string) Censorship token - "merkle": (string) Merkle root of proposal - "signature": (string) Server side signature of []byte(Merkle+Token) - } - }, - "startvote": { - "publickey" (string) Public key of user that submitted proposal - "vote": { - "token": (string) Censorship token - "mask" (uint64) Valid votebits - "duration": (uint32) Duration of vote in blocks - "quorumpercentage" (uint32) Percent of votes required for quorum - "passpercentage": (uint32) Percent of votes required to pass - "options": [ - { - "id" (string) Unique word identifying vote (e.g. yes) - "description" (string) Longer description of the vote - "bits": (uint64) Bits used for this option - }, - ] - }, - "signature" (string) Signature of Votehash - }, - "startvotereply": { - "startblockheight": (string) Block height at start of vote - "startblockhash": (string) Hash of first block of vote interval - "endheight": (string) Block height at end of vote - "eligibletickets": [ - "removed by politeiawwwcli for readability" - ] - } - ] -} -` diff --git a/politeiawww/cmd/piwww/help.go b/politeiawww/cmd/piwww/help.go index ea5199913..888835f9a 100644 --- a/politeiawww/cmd/piwww/help.go +++ b/politeiawww/cmd/piwww/help.go @@ -111,8 +111,6 @@ func (cmd *HelpCmd) Execute(args []string) error { fmt.Printf("%s\n", votesHelpMsg) case "voteresults": fmt.Printf("%s\n", voteResultsHelpMsg) - case "activevotes": - fmt.Printf("%s\n", activeVotesHelpMsg) case "votestatus": fmt.Printf("%s\n", voteStatusHelpMsg) case "votestatuses": @@ -123,6 +121,8 @@ func (cmd *HelpCmd) Execute(args []string) error { fmt.Printf("%s\n", voteDetailsHelpMsg) case "voteinventory": fmt.Printf("%s\n", voteInventoryHelpMsg) + case "voteballot": + fmt.Printf("%s\n", voteInventoryHelpMsg) // Websocket commands case "subscribe": @@ -134,10 +134,6 @@ func (cmd *HelpCmd) Execute(args []string) error { case "sendfaucettx": fmt.Printf("%s\n", sendFaucetTxHelpMsg) - // politeiavoter mock commands - case "vote": - fmt.Printf("%s\n", voteHelpMsg) - default: fmt.Printf("invalid command: use 'piwww -h' " + "to view a list of valid commands\n") diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index ebe64ba8c..dad61c7a2 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -65,12 +65,9 @@ type piwww struct { VoteResults VoteResultsCmd `command:"voteresults" description:"(public) get vote results for a proposal"` VoteSummaries VoteSummariesCmd `command:"votesummaries" description:"(public) retrieve the vote summary for a set of proposals"` VoteInventory VoteInventoryCmd `command:"voteinventory" description:"(public) retrieve the tokens of all public, non-abandoned proposal separated by vote status"` - - // XXX will go - Vote VoteCmd `command:"vote" description:"(public) cast votes for a proposal"` + VoteBallot VoteBallotCmd `command:"voteballot" description:"(public) cast ballot of votes for a proposal"` // Commands - ActiveVotes ActiveVotesCmd `command:"activevotes" description:"(public) get the proposals that are being voted on"` BatchProposals shared.BatchProposalsCmd `command:"batchproposals" description:"(user) retrieve a set of proposals"` ChangePassword shared.ChangePasswordCmd `command:"changepassword" description:"(user) change the password for the logged in user"` ChangeUsername shared.ChangeUsernameCmd `command:"changeusername" description:"(user) change the username for the logged in user"` diff --git a/politeiawww/cmd/piwww/testrun.go b/politeiawww/cmd/piwww/testrun.go index db20c602e..7bc3291a2 100644 --- a/politeiawww/cmd/piwww/testrun.go +++ b/politeiawww/cmd/piwww/testrun.go @@ -5,9 +5,6 @@ package main import ( - "fmt" - "strings" - v1 "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/cmd/shared" ) @@ -106,7 +103,7 @@ func newProposal(rfp bool, linkto string) (*v1.NewProposal, error) { // returns the error and in case of dcrwallet connection error it returns // true as first returned value func castVotes(token string, voteID string) (bool, error) { - var vc VoteCmd + /* var vc VoteCmd vc.Args.Token = token vc.Args.VoteID = voteID err := vc.Execute(nil) @@ -128,7 +125,8 @@ func castVotes(token string, voteID string) (bool, error) { return false, err } } - return false, err + return false, err*/ + return false, nil } // Execute executes the test run command. diff --git a/politeiawww/cmd/piwww/vote.go b/politeiawww/cmd/piwww/voteballot.go similarity index 82% rename from politeiawww/cmd/piwww/vote.go rename to politeiawww/cmd/piwww/voteballot.go index 5b5f3d324..64fd35cb8 100644 --- a/politeiawww/cmd/piwww/vote.go +++ b/politeiawww/cmd/piwww/voteballot.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Decred developers +// Copyright (c) 2017-2020 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -15,21 +15,20 @@ import ( "github.com/decred/dcrwallet/rpc/walletrpc" "github.com/decred/politeia/politeiad/api/v1/identity" pi "github.com/decred/politeia/politeiawww/api/pi/v1" - v1 "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/util" "golang.org/x/crypto/ssh/terminal" ) -// VoteCmd casts a proposal ballot for the specified proposal. -type VoteCmd struct { +// VoteBallotCmd casts a votes ballot for the specified proposal. +type VoteBallotCmd struct { Args struct { Token string `positional-arg-name:"token"` // Censorship token VoteID string `positional-arg-name:"voteid"` // Vote choice ID } `positional-args:"true" required:"true"` } -// Execute executes the vote command. -func (cmd *VoteCmd) Execute(args []string) error { +// Execute executes the vote ballot command. +func (cmd *VoteBallotCmd) Execute(args []string) error { token := cmd.Args.Token voteID := cmd.Args.VoteID @@ -51,42 +50,37 @@ func (cmd *VoteCmd) Execute(args []string) error { return err } - // Get all active proposal votes - avr, err := client.ActiveVotes() + // Get vote details of provided proposal + avr, err := client.Votes(pi.Votes{ + Tokens: []string{token}, + }) if err != nil { - return fmt.Errorf("ActiveVotes: %v", err) + return fmt.Errorf("Votes: %v", err) } // Find the proposal that the user wants to vote on - var pvt v1.ProposalVoteTuple - for _, v := range avr.Votes { - if token == v.Proposal.CensorshipRecord.Token { - pvt = v - break - } - } - - if pvt.Proposal.Name == "" { + pvt, ok := avr.Votes[token] + if !ok { return fmt.Errorf("proposal not found: %v", token) } // Ensure that the passed in voteID is one of the // proposal's voting options and save the vote bits - var voteBits string - for _, option := range pvt.StartVote.Vote.Options { - if voteID == option.Id { - voteBits = strconv.FormatUint(option.Bits, 16) + var voteBit string + for _, option := range pvt.Vote.Params.Options { + if voteID == option.ID { + voteBit = strconv.FormatUint(option.Bit, 16) break } } - if voteBits == "" { + if voteBit == "" { return fmt.Errorf("vote id not found: %v", voteID) } // Find user's tickets that are eligible to vote on this // proposal - ticketPool, err := convertTicketHashes(pvt.StartVoteReply.EligibleTickets) + ticketPool, err := convertTicketHashes(pvt.Vote.EligibleTickets) if err != nil { return err } @@ -101,7 +95,7 @@ func (cmd *VoteCmd) Execute(args []string) error { if len(ctr.TicketAddresses) == 0 { return fmt.Errorf("user has no eligible tickets: %v", - pvt.StartVote.Vote.Token) + token) } // Create slice of hexadecimal ticket hashes to represent @@ -132,7 +126,7 @@ func (cmd *VoteCmd) Execute(args []string) error { len(eligibleTickets)) for i, v := range ctr.TicketAddresses { // ctr.TicketAddresses and eligibleTickets use the same index - msg := token + eligibleTickets[i] + voteBits + msg := token + eligibleTickets[i] + voteBit messages = append(messages, &walletrpc.SignMessagesRequest_Message{ Address: v.Address, Message: msg, @@ -161,7 +155,7 @@ func (cmd *VoteCmd) Execute(args []string) error { votes = append(votes, pi.CastVote{ Token: token, Ticket: ticket, - VoteBit: voteBits, + VoteBit: voteBit, Signature: hex.EncodeToString(sigs.Replies[i].Signature), }) } @@ -171,7 +165,7 @@ func (cmd *VoteCmd) Execute(args []string) error { Votes: votes, }) if err != nil { - return fmt.Errorf("CastVotes: %v", err) + return fmt.Errorf("VoteBallot: %v", err) } // Check for any failed votes. Vote receipts don't include @@ -221,8 +215,9 @@ func (cmd *VoteCmd) Execute(args []string) error { return nil } -// voteHelpMsg is the output of the help command when 'vote' is specified. -const voteHelpMsg = `vote "token" "voteid" +// voteBallotHelpMsg is the output of the help command when 'voteballot' +// is specified. +const voteBallotHelpMsg = `voteballot "token" "voteid" Cast ticket votes for a proposal. diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index 1ddf265ac..72b46ff98 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -1728,30 +1728,6 @@ func (c *Client) GetAllVoteStatus() (*www.GetAllVoteStatusReply, error) { return &avsr, nil } -// ActiveVotes retreives all proposals that are currently being voted on. -func (c *Client) ActiveVotes() (*www.ActiveVoteReply, error) { - responseBody, err := c.makeRequest(http.MethodGet, - www.PoliteiaWWWAPIRoute, www.RouteActiveVote, nil) - if err != nil { - return nil, err - } - - var avr www.ActiveVoteReply - err = json.Unmarshal(responseBody, &avr) - if err != nil { - return nil, fmt.Errorf("unmarshal ActiveVoteReply: %v", err) - } - - if c.cfg.Verbose { - err := prettyPrintJSON(avr) - if err != nil { - return nil, err - } - } - - return &avr, nil -} - // ActiveVotesDCC retreives all dccs that are currently being voted on. func (c *Client) ActiveVotesDCC() (*cms.ActiveVoteReply, error) { responseBody, err := c.makeRequest(http.MethodGet, From 2c2f41125fad59f91a8df618289666e789f8219f Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 23 Sep 2020 19:30:40 -0500 Subject: [PATCH 108/449] add user ID to ProposalGeneral --- plugins/pi/pi.go | 9 +++++---- politeiawww/api/pi/v1/v1.go | 4 +++- politeiawww/piwww.go | 10 ++++++---- politeiawww/politeiawww.go | 2 +- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/plugins/pi/pi.go b/plugins/pi/pi.go index 9d53c3d47..d0691701e 100644 --- a/plugins/pi/pi.go +++ b/plugins/pi/pi.go @@ -148,6 +148,7 @@ func DecodeProposalMetadata(payload []byte) (*ProposalMetadata, error) { // Signature is the client signature of the proposal merkle root. The merkle // root is the ordered merkle root of all proposal Files and Metadata. type ProposalGeneral struct { + UserID string `json:"userid"` // Unique user ID PublicKey string `json:"publickey"` // Key used for signature Signature string `json:"signature"` // Signature of merkle root Timestamp int64 `json:"timestamp"` // Submission UNIX timestamp @@ -273,7 +274,7 @@ func DecodeProposalsReply(payload []byte) (*ProposalsReply, error) { // // Signature is the client signature of State+Token+ParentID+Comment. type CommentNew struct { - UUID string `json:"uuid"` // Unique user ID + UserID string `json:"userid"` // Unique user ID State PropStateT `json:"state"` // Record state Token string `json:"token"` // Record token ParentID uint32 `json:"parentid"` // Parent comment ID @@ -375,15 +376,15 @@ func DecodeCommentCensorReply(payload []byte) (*CommentCensorReply, error) { // validation that is specific to pi. // // The effect of a new vote on a comment score depends on the previous vote -// from that uuid. Example, a user upvotes a comment that they have already +// from that user ID. Example, a user upvotes a comment that they have already // upvoted, the resulting vote score is 0 due to the second upvote removing the // original upvote. The public key cannot be relied on to remain the same for -// each user so a uuid must be included. +// each user so a user ID must be included. // // Signature is the client signature of the State+Token+CommentID+Vote. type CommentVote struct { + UserID string `json:"userid"` // Unique user ID State PropStateT `json:"state"` // Record state - UUID string `json:"uuid"` // Unique user ID Token string `json:"token"` // Record token CommentID uint32 `json:"commentid"` // Comment ID Vote VoteT `json:"vote"` // Upvote or downvote diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 176473c88..66ce1f32f 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -18,7 +18,9 @@ type VoteT int type VoteErrorT int // TODO the plugin policies should be returned in a route -// TODO the proposals route should allow filtering by user ID +// TODO the proposals route should allow filtering by user ID. Actually, this +// is going to have to wait until after the intial release. This is non-trivial +// to accomplish and is outside the scope of the core functionality. const ( APIVersion = 1 diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 6027d126c..1421ddc54 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -968,6 +968,7 @@ func (p *politeiawww) processProposalNew(pn pi.ProposalNew, usr user.User) (*pi. // Setup metadata stream timestamp := time.Now().Unix() pg := piplugin.ProposalGeneral{ + UserID: usr.ID.String(), PublicKey: pn.PublicKey, Signature: pn.Signature, Timestamp: timestamp, @@ -1107,6 +1108,7 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p // Setup politeiad metadata timestamp := time.Now().Unix() pg := piplugin.ProposalGeneral{ + UserID: usr.ID.String(), PublicKey: pe.PublicKey, Signature: pe.Signature, Timestamp: timestamp, @@ -1461,8 +1463,8 @@ func (p *politeiawww) processCommentNew(cn pi.CommentNew, usr user.User) (*pi.Co } // Call the pi plugin to add new comment - reply, err := p.commentNewPi(piplugin.CommentNew{ - UUID: usr.ID.String(), + reply, err := p.piCommentNew(piplugin.CommentNew{ + UserID: usr.ID.String(), Token: cn.Token, ParentID: cn.ParentID, Comment: cn.Comment, @@ -1553,8 +1555,8 @@ func (p *politeiawww) processCommentVote(cv pi.CommentVote, usr user.User) (*pi. } // Call the pi plugin to add new comment - reply, err := p.commentVotePi(piplugin.CommentVote{ - UUID: usr.ID.String(), + reply, err := p.piCommentVote(piplugin.CommentVote{ + UserID: usr.ID.String(), Token: cv.Token, CommentID: cv.CommentID, Vote: convertVoteFromPi(cv.Vote), diff --git a/politeiawww/politeiawww.go b/politeiawww/politeiawww.go index 3196711d5..caa1fb164 100644 --- a/politeiawww/politeiawww.go +++ b/politeiawww/politeiawww.go @@ -63,10 +63,10 @@ type politeiawww struct { cfg *config params *chaincfg.Params router *mux.Router - sessions sessions.Store client *http.Client smtp *smtp db user.Database + sessions sessions.Store eventManager *eventManager plugins []plugin From 951fddee118c01af0e35bc5f57f323410ad2ec1a Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 24 Sep 2020 17:28:16 -0500 Subject: [PATCH 109/449] hooking things up and cleanup --- plugins/comments/comments.go | 3 +- plugins/dcrdata/dcrdata.go | 3 +- plugins/pi/pi.go | 3 +- plugins/ticketvote/ticketvote.go | 3 +- politeiad/backend/tlogbe/anchor.go | 2 +- politeiad/backend/tlogbe/dcrdata.go | 56 ++- politeiad/backend/tlogbe/ticketvote.go | 4 +- politeiad/backend/tlogbe/tlogbe.go | 20 +- politeiad/backend/tlogbe/tlogclient.go | 6 + politeiad/backend/tlogbe/trillian.go | 6 +- politeiad/log.go | 12 +- politeiad/politeiad.go | 49 ++- politeiawww/api/pi/v1/v1.go | 2 + politeiawww/piwww.go | 2 + politeiawww/politeiad.go | 18 +- politeiawww/sessionstore.go | 4 +- politeiawww/smtp.go | 1 + politeiawww/templates.go | 2 + politeiawww/testing.go | 4 +- politeiawww/www.go | 524 +++++++++++++------------ util/net.go | 11 +- 21 files changed, 421 insertions(+), 314 deletions(-) diff --git a/plugins/comments/comments.go b/plugins/comments/comments.go index 00a1c001b..1af727ab2 100644 --- a/plugins/comments/comments.go +++ b/plugins/comments/comments.go @@ -18,7 +18,8 @@ type ErrorStatusT int // TODO add a hint to comments that can be used freely by the client. This // is how we'll distinguish proposal comments from update comments. const ( - ID = "comments" + ID = "comments" + Version = "1" // Plugin commands CmdNew = "new" // Create a new comment diff --git a/plugins/dcrdata/dcrdata.go b/plugins/dcrdata/dcrdata.go index cbe2edce4..58180a78c 100644 --- a/plugins/dcrdata/dcrdata.go +++ b/plugins/dcrdata/dcrdata.go @@ -15,7 +15,8 @@ import ( type StatusT int const ( - ID = "dcrdata" + ID = "dcrdata" + Version = "1" // Plugin commands CmdBestBlock = "bestblock" // Get best block diff --git a/plugins/pi/pi.go b/plugins/pi/pi.go index d0691701e..346d67f36 100644 --- a/plugins/pi/pi.go +++ b/plugins/pi/pi.go @@ -19,7 +19,8 @@ type ErrorStatusT int type VoteT int const ( - ID = "pi" + ID = "pi" + Version = "1" // Plugin commands. Many of these plugin commands rely on the // commands from other plugins, but perform additional validation diff --git a/plugins/ticketvote/ticketvote.go b/plugins/ticketvote/ticketvote.go index 38dbfc3f8..d602996be 100644 --- a/plugins/ticketvote/ticketvote.go +++ b/plugins/ticketvote/ticketvote.go @@ -21,7 +21,8 @@ type ErrorStatusT int // The receipt should be the server signature of Signature+StartBlockHash. const ( - ID = "ticketvote" + ID = "ticketvote" + Version = "1" // Plugin commands CmdAuthorize = "authorize" // Authorize a vote diff --git a/politeiad/backend/tlogbe/anchor.go b/politeiad/backend/tlogbe/anchor.go index e20a698c8..66f757a60 100644 --- a/politeiad/backend/tlogbe/anchor.go +++ b/politeiad/backend/tlogbe/anchor.go @@ -386,7 +386,7 @@ func (t *tlog) anchor() { t.id, v.TreeId, lr.TreeSize) } if len(anchors) == 0 { - log.Infof("Nothing to anchor") + log.Infof("No %v trees to to anchor", t.id) return } diff --git a/politeiad/backend/tlogbe/dcrdata.go b/politeiad/backend/tlogbe/dcrdata.go index dc1fd9208..b768da0ba 100644 --- a/politeiad/backend/tlogbe/dcrdata.go +++ b/politeiad/backend/tlogbe/dcrdata.go @@ -6,7 +6,6 @@ package tlogbe import ( "bytes" - "crypto/tls" "encoding/json" "fmt" "io/ioutil" @@ -14,7 +13,6 @@ import ( "strconv" "strings" "sync" - "time" v4 "github.com/decred/dcrdata/api/types/v4" exptypes "github.com/decred/dcrdata/explorer/types/v2" @@ -40,9 +38,10 @@ var ( // dcrdataplugin satisfies the pluginClient interface. type dcrdataPlugin struct { sync.Mutex - host string - client *http.Client // HTTP client - ws *wsdcrdata.Client // Websocket client + hostHTTP string // dcrdata HTTP host + hostWS string // dcrdata websocket host + client *http.Client // HTTP client + ws *wsdcrdata.Client // Websocket client // bestBlock is the cached best block height. This field is kept up // to date by the websocket connection. If the websocket connection @@ -84,7 +83,7 @@ func (p *dcrdataPlugin) bestBlockIsStale() bool { func (p *dcrdataPlugin) makeReq(method string, route string, v interface{}) ([]byte, error) { var ( - url = p.host + route + url = p.hostHTTP + route reqBody []byte err error ) @@ -398,7 +397,7 @@ func (p *dcrdataPlugin) websocketMonitor() { // Setup a new messages channel using the new connection. receiver = p.ws.Receive() - log.Infof("Successfully reconnected dcrdata websocket") + log.Infof("Dcrdata websocket successfully reconnected") } } @@ -409,7 +408,7 @@ func (p *dcrdataPlugin) websocketSetup() { // Best block err := p.ws.NewBlockSubscribe() if err != nil && err != wsdcrdata.ErrDuplicateSub { - log.Errorf("NewBlockSubscribe: %v", err) + log.Errorf("dcrdataPlugin: NewBlockSubscribe: %v", err) goto reconnect } @@ -431,6 +430,15 @@ func (p *dcrdataPlugin) websocketSetup() { func (p *dcrdataPlugin) setup() error { log.Tracef("dcrdata setup") + // Setup websocket client + ws, err := wsdcrdata.New(p.hostWS) + if err != nil { + // Continue even if a websocket connection was not able to be + // made. Reconnection attempts will be made in the plugin setup. + log.Errorf("wsdcrdata New: %v", err) + } + p.ws = ws + // Setup dcrdata websocket subscriptions and monitoring. This is // done in a go routine so setup will continue in the event that // a dcrdata websocket connection was not able to be made during @@ -478,33 +486,19 @@ func (p *dcrdataPlugin) fsck() error { return nil } -func newDcrdataPlugin(settings []backend.PluginSetting) *dcrdataPlugin { +func newDcrdataPlugin(settings []backend.PluginSetting) (*dcrdataPlugin, error) { // TODO these should be passed in as plugin settings - var dcrdataHost string + dcrdataHostHTTP := "https://dcrdata.decred.org/" + dcrdataHostWS := "wss://dcrdata.decred.org/ps" - // Setup http client - client := &http.Client{ - Timeout: 1 * time.Minute, - Transport: &http.Transport{ - IdleConnTimeout: 1 * time.Minute, - ResponseHeaderTimeout: 1 * time.Minute, - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: false, - }, - }, - } - - // Setup websocket client - ws, err := wsdcrdata.New(dcrdataHost) + client, err := util.NewClient(false, "") if err != nil { - // Continue even if a websocket connection was not able to be - // made. Reconnection attempts will be made in the plugin setup. - log.Errorf("wsdcrdata New: %v", err) + return nil, err } return &dcrdataPlugin{ - host: dcrdataHost, - client: client, - ws: ws, - } + hostHTTP: dcrdataHostHTTP, + hostWS: dcrdataHostWS, + client: client, + }, nil } diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index 6acfbac89..a62e1321e 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -1941,7 +1941,7 @@ func (p *ticketVotePlugin) fsck() error { return nil } -func TicketVotePluginNew(backend backend.Backend, tlog tlogClient, settings []backend.PluginSetting) (*ticketVotePlugin, error) { +func newTicketVotePlugin(backend backend.Backend, tlog tlogClient, settings []backend.PluginSetting) *ticketVotePlugin { var ( // TODO these should be passed in as plugin settings dataDir string @@ -1966,5 +1966,5 @@ func TicketVotePluginNew(backend backend.Backend, tlog tlogClient, settings []ba activeNetParams: activeNetParams, voteDurationMin: voteDurationMin, voteDurationMax: voteDurationMax, - }, nil + } } diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 308bfc967..fa617474a 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -31,7 +31,6 @@ import ( "github.com/subosito/gozaru" ) -// TODO we need an unvetted censored status // TODO testnet vs mainnet trillian databases // TODO fsck // TODO allow token prefix lookups @@ -1344,21 +1343,30 @@ func (t *tlogBackend) InventoryByStatus() (*backend.InventoryByStatus, error) { func (t *tlogBackend) RegisterPlugin(p backend.Plugin) error { log.Tracef("RegisterPlugin: %v", p.ID) - var client pluginClient + var ( + client pluginClient + err error + ) switch p.ID { case comments.ID: + client = newCommentsPlugin(t, newBackendClient(t), p.Settings) case dcrdata.ID: + client, err = newDcrdataPlugin(p.Settings) + if err != nil { + return err + } case pi.ID: + client = newPiPlugin(t, newBackendClient(t), p.Settings) case ticketvote.ID: + client = newTicketVotePlugin(t, newBackendClient(t), p.Settings) default: return backend.ErrPluginInvalid } t.plugins[p.ID] = plugin{ - id: p.ID, - version: p.Version, - settings: p.Settings, - client: client, + id: p.ID, + version: p.Version, + client: client, } return nil diff --git a/politeiad/backend/tlogbe/tlogclient.go b/politeiad/backend/tlogbe/tlogclient.go index 3a12fa1de..99c3ec594 100644 --- a/politeiad/backend/tlogbe/tlogclient.go +++ b/politeiad/backend/tlogbe/tlogclient.go @@ -43,6 +43,12 @@ type backendClient struct { backend *tlogBackend } +func newBackendClient(tlog *tlogBackend) *backendClient { + return &backendClient{ + backend: tlog, + } +} + // tlogByID returns the tlog instance that corresponds to the provided ID. func (c *backendClient) tlogByID(tlogID string) (*tlog, error) { switch tlogID { diff --git a/politeiad/backend/tlogbe/trillian.go b/politeiad/backend/tlogbe/trillian.go index 47fb398ff..2ddbe5a36 100644 --- a/politeiad/backend/tlogbe/trillian.go +++ b/politeiad/backend/tlogbe/trillian.go @@ -462,6 +462,7 @@ func newTrillianClient(host, keyFile string) (*trillianClient, error) { } // Setup trillian connection + // TODO should this be WithInsecure? g, err := grpc.Dial(host, grpc.WithInsecure()) if err != nil { return nil, fmt.Errorf("grpc dial: %v", err) @@ -487,10 +488,13 @@ func newTrillianClient(host, keyFile string) (*trillianClient, error) { publicKey: signer.Public(), } + // The grpc dial requires a little time to connect + time.Sleep(time.Second) + // Ensure trillian is up and running for t.grpc.GetState() != connectivity.Ready { wait := 15 * time.Second - log.Infof("Cannot connect to trillian; retry in %v ", wait) + log.Infof("Cannot connect to trillian at %v; retry in %v ", host, wait) time.Sleep(wait) } diff --git a/politeiad/log.go b/politeiad/log.go index 2b161f12b..0ebcdaa2b 100644 --- a/politeiad/log.go +++ b/politeiad/log.go @@ -12,6 +12,7 @@ import ( "github.com/decred/politeia/politeiad/backend/gitbe" "github.com/decred/politeia/politeiad/backend/tlogbe" "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" + "github.com/decred/politeia/wsdcrdata" "github.com/decred/slog" "github.com/jrick/logrotate/rotator" ) @@ -43,10 +44,11 @@ var ( // application shutdown. logRotator *rotator.Rotator - log = backendLog.Logger("POLI") - gitbeLog = backendLog.Logger("GITB") - tlogLog = backendLog.Logger("TLOG") - storeLog = backendLog.Logger("STOR") + log = backendLog.Logger("POLI") + gitbeLog = backendLog.Logger("GITB") + tlogLog = backendLog.Logger("TLOG") + storeLog = backendLog.Logger("STOR") + wsdcrdataLog = backendLog.Logger("WSDD") ) // Initialize package-global logger variables. @@ -54,6 +56,7 @@ func init() { gitbe.UseLogger(gitbeLog) tlogbe.UseLogger(tlogLog) filesystem.UseLogger(storeLog) + wsdcrdata.UseLogger(wsdcrdataLog) } // subsystemLoggers maps each subsystem identifier to its associated logger. @@ -62,6 +65,7 @@ var subsystemLoggers = map[string]slog.Logger{ "GITB": gitbeLog, "TLOG": tlogLog, "STOR": storeLog, + "WSDD": wsdcrdataLog, } // initLogRotator initializes the logging rotater to write logs to logFile and diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index fa7c19fc6..99b24fc70 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -69,7 +69,8 @@ func convertBackendPluginSetting(bpi backend.PluginSetting) v1.PluginSetting { func convertBackendPlugin(bpi backend.Plugin) v1.Plugin { p := v1.Plugin{ - ID: bpi.ID, + ID: bpi.ID, + Version: bpi.Version, } for _, v := range bpi.Settings { p.Settings = append(p.Settings, convertBackendPluginSetting(v)) @@ -187,6 +188,25 @@ func (p *politeia) convertBackendRecord(br backend.Record) v1.Record { return pr } +// handleNotFound is a generic handler for an invalid route. +func (p *politeia) handleNotFound(w http.ResponseWriter, r *http.Request) { + // Log incoming connection + log.Debugf("Invalid route: %v %v %v %v", remoteAddr(r), r.Method, r.URL, + r.Proto) + + // Trace incoming request + log.Tracef("%v", newLogClosure(func() string { + trace, err := httputil.DumpRequest(r, true) + if err != nil { + trace = []byte(fmt.Sprintf("logging: "+ + "DumpRequest %v", err)) + } + return string(trace) + })) + + util.RespondWithJSON(w, http.StatusNotFound, v1.ServerErrorReply{}) +} + func (p *politeia) respondWithUserError(w http.ResponseWriter, errorCode v1.ErrorStatusT, errorContext []string) { util.RespondWithJSON(w, http.StatusBadRequest, v1.UserErrorReply{ @@ -1088,6 +1108,9 @@ func _main() error { // Setup mux p.router = mux.NewRouter() + // Not found + p.router.NotFoundHandler = closeBody(p.handleNotFound) + // Unprivileged routes p.addRoute(http.MethodPost, v1.IdentityRoute, p.getIdentity, permissionPublic) @@ -1145,6 +1168,8 @@ func _main() error { */ // Setup plugins + // TODO fix this + loadedCfg.Plugins = []string{"comments", "dcrdata", "pi", "ticketvote"} if len(loadedCfg.Plugins) > 0 { // Set plugin routes. Requires auth. p.addRoute(http.MethodPost, v1.PluginCommandRoute, p.pluginCommand, @@ -1157,9 +1182,29 @@ func _main() error { var plugin backend.Plugin switch v { case comments.ID: + plugin = backend.Plugin{ + ID: comments.ID, + Version: comments.Version, + Settings: make([]backend.PluginSetting, 0), + } case dcrdata.ID: + plugin = backend.Plugin{ + ID: dcrdata.ID, + Version: dcrdata.Version, + Settings: make([]backend.PluginSetting, 0), + } case pi.ID: + plugin = backend.Plugin{ + ID: pi.ID, + Version: pi.Version, + Settings: make([]backend.PluginSetting, 0), + } case ticketvote.ID: + plugin = backend.Plugin{ + ID: ticketvote.ID, + Version: ticketvote.Version, + Settings: make([]backend.PluginSetting, 0), + } case decredplugin.ID: // TODO plugin setup for cms case cmsplugin.ID: @@ -1182,7 +1227,7 @@ func _main() error { // Setup plugins for _, v := range loadedCfg.Plugins { - log.Infof("Performing plugin setup for %v plugin", v) + log.Infof("Setting up plugin: %v", v) err := p.backend.SetupPlugin(v) if err != nil { return fmt.Errorf("SetupPlugin %v: %v", v, err) diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 66ce1f32f..1b72d5f58 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -17,6 +17,8 @@ type VoteAuthActionT string type VoteT int type VoteErrorT int +// TODO I screwed up comments. A comment should contain fields for upvotes and +// downvotes instead of just a overall vote score. // TODO the plugin policies should be returned in a route // TODO the proposals route should allow filtering by user ID. Actually, this // is going to have to wait until after the intial release. This is non-trivial diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 1421ddc54..4bda02839 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -32,6 +32,8 @@ import ( // TODO use pi policies. Should the policies be defined in the pi plugin // or the pi api spec? +// TODO politeiad needs to return the plugin with the error. + const ( // MIME types mimeTypeText = "text/plain" diff --git a/politeiawww/politeiad.go b/politeiawww/politeiad.go index 46bbcc22a..5763d47c0 100644 --- a/politeiawww/politeiad.go +++ b/politeiawww/politeiad.go @@ -40,11 +40,11 @@ func (e pdError) Error() string { // if politeiad does not respond with a 200. func (p *politeiawww) makeRequest(method string, route string, v interface{}) ([]byte, error) { var ( - requestBody []byte - err error + reqBody []byte + err error ) if v != nil { - requestBody, err = json.Marshal(v) + reqBody, err = json.Marshal(v) if err != nil { return nil, err } @@ -52,15 +52,9 @@ func (p *politeiawww) makeRequest(method string, route string, v interface{}) ([ fullRoute := p.cfg.RPCHost + route - if p.client == nil { - p.client, err = util.NewClient(false, p.cfg.RPCCert) - if err != nil { - return nil, err - } - } + log.Debugf("%v %v %+v", method, fullRoute, v) - req, err := http.NewRequest(method, fullRoute, - bytes.NewReader(requestBody)) + req, err := http.NewRequest(method, fullRoute, bytes.NewReader(reqBody)) if err != nil { return nil, err } @@ -75,7 +69,7 @@ func (p *politeiawww) makeRequest(method string, route string, v interface{}) ([ var e pdErrorReply decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&e); err != nil { - return nil, err + return nil, fmt.Errorf("status code %v: %v", r.StatusCode, err) } return nil, pdError{ diff --git a/politeiawww/sessionstore.go b/politeiawww/sessionstore.go index 75d857151..15489c32e 100644 --- a/politeiawww/sessionstore.go +++ b/politeiawww/sessionstore.go @@ -193,7 +193,7 @@ func newSessionOptions() *sessions.Options { } } -// NewSessionStore returns a new SessionStore. +// newSessionStore returns a new SessionStore. // // Keys are defined in pairs to allow key rotation, but the common case is // to set a single authentication key and optionally an encryption key. @@ -205,7 +205,7 @@ func newSessionOptions() *sessions.Options { // It is recommended to use an authentication key with 32 or 64 bytes. // The encryption key, if set, must be either 16, 24, or 32 bytes to select // AES-128, AES-192, or AES-256 modes. -func NewSessionStore(db user.Database, sessionMaxAge int, keyPairs ...[]byte) *SessionStore { +func newSessionStore(db user.Database, sessionMaxAge int, keyPairs ...[]byte) *SessionStore { // Set the maxAge for each securecookie instance codecs := securecookie.CodecsFromPairs(keyPairs...) for _, codec := range codecs { diff --git a/politeiawww/smtp.go b/politeiawww/smtp.go index c4e470f0b..7ea111ef8 100644 --- a/politeiawww/smtp.go +++ b/politeiawww/smtp.go @@ -62,6 +62,7 @@ func (s *smtp) sendEmail(subject, body string, addToAddressesFn func(*goemail.Me func newSMTP(host, user, password, emailAddress string, systemCerts *x509.CertPool, skipVerify bool) (*smtp, error) { // Check if email has been disabled if host == "" || user == "" || password == "" { + log.Infof("Email: DISABLED") return &smtp{ disabled: true, }, nil diff --git a/politeiawww/templates.go b/politeiawww/templates.go index 2a64ca01d..f0a3c6b9a 100644 --- a/politeiawww/templates.go +++ b/politeiawww/templates.go @@ -6,6 +6,8 @@ package main import "text/template" +// TODO move templates to the email file where they're being used + // Proposal submitted - Send to admins type proposalSubmitted struct { Username string // Author username diff --git a/politeiawww/testing.go b/politeiawww/testing.go index 27e40e381..4583807ce 100644 --- a/politeiawww/testing.go +++ b/politeiawww/testing.go @@ -699,7 +699,7 @@ func newTestPoliteiawww(t *testing.T) (*politeiawww, func()) { db: db, params: &chaincfg.TestNet3Params, router: mux.NewRouter(), - sessions: NewSessionStore(db, sessionMaxAge, cookieKey), + sessions: newSessionStore(db, sessionMaxAge, cookieKey), smtp: smtp, test: true, userEmails: make(map[string]uuid.UUID), @@ -804,7 +804,7 @@ func newTestCMSwww(t *testing.T) (*politeiawww, func()) { db: db, params: &chaincfg.TestNet3Params, router: mux.NewRouter(), - sessions: NewSessionStore(db, sessionMaxAge, cookieKey), + sessions: newSessionStore(db, sessionMaxAge, cookieKey), smtp: smtp, test: true, userEmails: make(map[string]uuid.UUID), diff --git a/politeiawww/www.go b/politeiawww/www.go index b2646bfe4..9a1a18e95 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -33,7 +33,7 @@ import ( database "github.com/decred/politeia/politeiawww/cmsdatabase" cmsdb "github.com/decred/politeia/politeiawww/cmsdatabase/cockroachdb" "github.com/decred/politeia/politeiawww/user" - userdb "github.com/decred/politeia/politeiawww/user/cockroachdb" + "github.com/decred/politeia/politeiawww/user/cockroachdb" "github.com/decred/politeia/politeiawww/user/localdb" "github.com/decred/politeia/util" "github.com/decred/politeia/util/version" @@ -277,8 +277,8 @@ func convertPiErrorStatus(pluginID string, errCode int) pi.ErrorStatusT { } // Fetch remote identity -func (p *politeiawww) getIdentity() error { - id, err := util.RemoteIdentity(false, p.cfg.RPCHost, p.cfg.RPCCert) +func getIdentity(rpcHost, rpcCert, rpcIdentityFile, interactive string) error { + id, err := util.RemoteIdentity(false, rpcHost, rpcCert) if err != nil { return err } @@ -288,29 +288,29 @@ func (p *politeiawww) getIdentity() error { log.Infof("Key : %x", id.Key) log.Infof("Fingerprint: %v", id.Fingerprint()) - if p.cfg.Interactive != allowInteractive { + if interactive != allowInteractive { // Ask user if we like this identity log.Infof("Press enter to save to %v or ctrl-c to abort", - p.cfg.RPCIdentityFile) + rpcIdentityFile) scanner := bufio.NewScanner(os.Stdin) scanner.Scan() if err = scanner.Err(); err != nil { return err } } else { - log.Infof("Saving identity to %v", p.cfg.RPCIdentityFile) + log.Infof("Saving identity to %v", rpcIdentityFile) } // Save identity - err = os.MkdirAll(filepath.Dir(p.cfg.RPCIdentityFile), 0700) + err = os.MkdirAll(filepath.Dir(rpcIdentityFile), 0700) if err != nil { return err } - err = id.SavePublicIdentity(p.cfg.RPCIdentityFile) + err = id.SavePublicIdentity(rpcIdentityFile) if err != nil { return err } - log.Infof("Identity saved to: %v", p.cfg.RPCIdentityFile) + log.Infof("Identity saved to: %v", rpcIdentityFile) return nil } @@ -553,6 +553,187 @@ func (p *politeiawww) addRoute(method string, routeVersion string, route string, } } +func (p *politeiawww) setupPi() error { + // Setup routes + p.setPoliteiaWWWRoutes() + p.setUserWWWRoutes() + p.setPiRoutes() + + // Verify paywall settings + switch { + case p.cfg.PaywallAmount != 0 && p.cfg.PaywallXpub != "": + // Paywall is enabled + paywallAmountInDcr := float64(p.cfg.PaywallAmount) / 1e8 + log.Infof("Paywall : %v DCR", paywallAmountInDcr) + + case p.cfg.PaywallAmount == 0 && p.cfg.PaywallXpub == "": + // Paywall is disabled + log.Infof("Paywall: DISABLED") + + default: + // Invalid paywall setting + return fmt.Errorf("paywall settings invalid, both an amount " + + "and public key MUST be set") + } + + // Setup paywall pool + p.userPaywallPool = make(map[uuid.UUID]paywallPoolMember) + err := p.initPaywallChecker() + if err != nil { + return err + } + + // Setup event manager + p.setupEventListenersPi() + + // TODO Verify politeiad plugins + + return nil +} + +func (p *politeiawww) setupCMS() error { + // Setup routes + p.setCMSWWWRoutes() + p.setCMSUserWWWRoutes() + + // Setup event manager + p.setupEventListenersCMS() + + // Setup dcrdata websocket connection + ws, err := wsdcrdata.New(p.dcrdataHostWS()) + if err != nil { + // Continue even if a websocket connection was not able to be + // made. The application specific websocket setup (pi, cms, etc) + // can decide whether to attempt reconnection or to exit. + log.Errorf("wsdcrdata New: %v", err) + } + p.wsDcrdata = ws + + // Verify politeiad plugins + pluginFound := false + for _, plugin := range p.plugins { + if plugin.ID == "cms" { + pluginFound = true + break + } + } + if !pluginFound { + return fmt.Errorf("politeiad plugin 'cms' not found") + } + + // Setup cmsdb + net := filepath.Base(p.cfg.DataDir) + p.cmsDB, err = cmsdb.New(p.cfg.DBHost, net, p.cfg.DBRootCert, + p.cfg.DBCert, p.cfg.DBKey) + if err == database.ErrNoVersionRecord || err == database.ErrWrongVersion { + // The cmsdb version record was either not found or + // is the wrong version which means that the cmsdb + // needs to be built/rebuilt. + p.cfg.BuildCMSDB = true + } else if err != nil { + return err + } + err = p.cmsDB.Setup() + if err != nil { + return fmt.Errorf("cmsdb setup: %v", err) + } + + // Build the cms database + if p.cfg.BuildCMSDB { + // Request full record inventory from backend + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return err + } + + pdCommand := pd.Inventory{ + Challenge: hex.EncodeToString(challenge), + IncludeFiles: true, + AllVersions: true, + } + + responseBody, err := p.makeRequest(http.MethodPost, + pd.InventoryRoute, pdCommand) + if err != nil { + return err + } + + var pdReply pd.InventoryReply + err = json.Unmarshal(responseBody, &pdReply) + if err != nil { + return fmt.Errorf("Could not unmarshal InventoryReply: %v", + err) + } + + // Verify the UpdateVettedMetadata challenge. + err = util.VerifyChallenge(p.cfg.Identity, challenge, pdReply.Response) + if err != nil { + return err + } + + vetted := pdReply.Vetted + dbInvs := make([]database.Invoice, 0, len(vetted)) + dbDCCs := make([]database.DCC, 0, len(vetted)) + for _, r := range vetted { + for _, m := range r.Metadata { + switch m.ID { + case mdstream.IDInvoiceGeneral: + i, err := convertRecordToDatabaseInvoice(r) + if err != nil { + log.Errorf("convertRecordToDatabaseInvoice: %v", err) + break + } + u, err := p.db.UserGetByPubKey(i.PublicKey) + if err != nil { + log.Errorf("usergetbypubkey: %v %v", err, i.PublicKey) + break + } + i.UserID = u.ID.String() + i.Username = u.Username + dbInvs = append(dbInvs, *i) + case mdstream.IDDCCGeneral: + d, err := convertRecordToDatabaseDCC(r) + if err != nil { + log.Errorf("convertRecordToDatabaseDCC: %v", err) + break + } + dbDCCs = append(dbDCCs, *d) + } + } + } + + // Build the cmsdb + err = p.cmsDB.Build(dbInvs, dbDCCs) + if err != nil { + return fmt.Errorf("build cmsdb: %v", err) + } + } + // Register cms userdb plugin + plugin := user.Plugin{ + ID: user.CMSPluginID, + Version: user.CMSPluginVersion, + } + err = p.db.RegisterPlugin(plugin) + if err != nil { + return fmt.Errorf("register userdb plugin: %v", err) + } + + // Setup invoice notifications + p.cron = cron.New() + p.checkInvoiceNotifications() + + // Setup dcrdata websocket subscriptions and monitoring. This is + // done in a go routine so cmswww startup will continue in + // the event that a dcrdata websocket connection was not able to + // be made during client initialization and reconnection attempts + // are required. + go func() { + p.setupCMSAddressWatcher() + }() + + return nil +} + func _main() error { // Load configuration and parse command line. This function also // initializes logging and configures it accordingly. @@ -571,28 +752,20 @@ func _main() error { log.Infof("Network : %v", activeNetParams.Params.Name) log.Infof("Home dir: %v", loadedCfg.HomeDir) - if loadedCfg.PaywallAmount != 0 && loadedCfg.PaywallXpub != "" { - paywallAmountInDcr := float64(loadedCfg.PaywallAmount) / 1e8 - log.Infof("Paywall : %v DCR", paywallAmountInDcr) - } else if loadedCfg.PaywallAmount == 0 && loadedCfg.PaywallXpub == "" { - log.Infof("Paywall : DISABLED") - } else { - return fmt.Errorf("Paywall settings invalid, both an amount " + - "and public key MUST be set") - } - - if loadedCfg.MailHost == "" { - log.Infof("Email : DISABLED") - } - // Create the data directory in case it does not exist. err = os.MkdirAll(loadedCfg.DataDir, 0700) if err != nil { return err } - // Generate the TLS cert and key file if both don't already - // exist. + // Check if this command is being run to fetch the politeiad + // identity. + if loadedCfg.FetchIdentity { + return getIdentity(loadedCfg.RPCHost, loadedCfg.RPCCert, + loadedCfg.RPCIdentityFile, loadedCfg.Interactive) + } + + // Generate the TLS cert and key file if both don't already exist. if !fileExists(loadedCfg.HTTPSKey) && !fileExists(loadedCfg.HTTPSCert) { log.Infof("Generating HTTPS keypair...") @@ -604,73 +777,98 @@ func _main() error { err) } - log.Infof("HTTPS keypair created...") + log.Infof("HTTPS keypair created") } - // Setup application context. - p := &politeiawww{ - cfg: loadedCfg, - ws: make(map[string]map[string]*wsContext), - eventManager: newEventManager(), - - // XXX reevaluate where this goes - userEmails: make(map[string]uuid.UUID), - userPaywallPool: make(map[uuid.UUID]paywallPoolMember), - params: activeNetParams.Params, - } + // Setup router + router := mux.NewRouter() + router.Use(recoverMiddleware) - // Check if this command is being run to fetch the identity. - if p.cfg.FetchIdentity { - return p.getIdentity() + // Setup smtp client + smtp, err := newSMTP(loadedCfg.MailHost, loadedCfg.MailUser, + loadedCfg.MailPass, loadedCfg.MailAddress, loadedCfg.SystemCerts, + loadedCfg.SMTPSkipVerify) + if err != nil { + return fmt.Errorf("newSMTP: %v", err) } - // Setup email - smtp, err := newSMTP(p.cfg.MailHost, p.cfg.MailUser, - p.cfg.MailPass, p.cfg.MailAddress, p.cfg.SystemCerts, - p.cfg.SMTPSkipVerify) + // Setup politeiad client + client, err := util.NewClient(false, loadedCfg.RPCCert) if err != nil { - return fmt.Errorf("unable to initialize SMTP client: %v", - err) + return err } - p.smtp = smtp // Setup user database - switch p.cfg.UserDB { + var userDB user.Database + switch loadedCfg.UserDB { case userDBLevel: - db, err := localdb.New(p.cfg.DataDir) + db, err := localdb.New(loadedCfg.DataDir) if err != nil { return err } - p.db = db + userDB = db + case userDBCockroach: // If old encryption key is set it means that we need // to open a db connection using the old key and then // rotate keys. var encryptionKey string - if p.cfg.OldEncryptionKey != "" { - encryptionKey = p.cfg.OldEncryptionKey + if loadedCfg.OldEncryptionKey != "" { + encryptionKey = loadedCfg.OldEncryptionKey } else { - encryptionKey = p.cfg.EncryptionKey + encryptionKey = loadedCfg.EncryptionKey } // Open db connection - network := filepath.Base(p.cfg.DataDir) - db, err := userdb.New(p.cfg.DBHost, network, p.cfg.DBRootCert, - p.cfg.DBCert, p.cfg.DBKey, encryptionKey) + network := filepath.Base(loadedCfg.DataDir) + db, err := cockroachdb.New(loadedCfg.DBHost, network, + loadedCfg.DBRootCert, loadedCfg.DBCert, loadedCfg.DBKey, + encryptionKey) if err != nil { return fmt.Errorf("new cockroachdb: %v", err) } - p.db = db + userDB = db // Rotate keys - if p.cfg.OldEncryptionKey != "" { - err = db.RotateKeys(p.cfg.EncryptionKey) + if loadedCfg.OldEncryptionKey != "" { + err = db.RotateKeys(loadedCfg.EncryptionKey) if err != nil { return fmt.Errorf("rotate userdb keys: %v", err) } } + default: - return fmt.Errorf("no user db option found") + return fmt.Errorf("invalid userdb '%v'", loadedCfg.UserDB) + } + + // Setup sessions store + var cookieKey []byte + if cookieKey, err = ioutil.ReadFile(loadedCfg.CookieKeyFile); err != nil { + log.Infof("Cookie key not found, generating one...") + cookieKey, err = util.Random(32) + if err != nil { + return err + } + err = ioutil.WriteFile(loadedCfg.CookieKeyFile, cookieKey, 0400) + if err != nil { + return err + } + log.Infof("Cookie key generated") + } + sessions := newSessionStore(userDB, sessionMaxAge, cookieKey) + + // Setup application context + p := &politeiawww{ + cfg: loadedCfg, + params: activeNetParams.Params, + router: router, + client: client, + smtp: smtp, + db: userDB, + sessions: sessions, + eventManager: newEventManager(), + ws: make(map[string]map[string]*wsContext), + userEmails: make(map[string]uuid.UUID), } // Get plugins from politeiad @@ -679,12 +877,28 @@ func _main() error { return fmt.Errorf("getPluginInventory: %v", err) } - // Setup email-userID map + // Setup email-userID cache err = p.initUserEmailsCache() if err != nil { return err } + // Perform application specific setup + switch p.cfg.Mode { + case politeiaWWWMode: + err = p.setupPi() + if err != nil { + return fmt.Errorf("setupPi: %v", err) + } + case cmsWWWMode: + err = p.setupCMS() + if err != nil { + return fmt.Errorf("setupCMS: %v", err) + } + default: + return fmt.Errorf("unknown mode: %v", p.cfg.Mode) + } + // Load or create new CSRF key log.Infof("Load CSRF key") csrfKeyFilename := filepath.Join(p.cfg.DataDir, "csrf.key") @@ -730,187 +944,6 @@ func _main() error { csrf.MaxAge(sessionMaxAge), ) - p.router = mux.NewRouter() - p.router.Use(recoverMiddleware) - - switch p.cfg.Mode { - case politeiaWWWMode: - // Setup routes - p.setPoliteiaWWWRoutes() - p.setUserWWWRoutes() - - p.setPiRoutes() - - err = p.initPaywallChecker() - if err != nil { - return err - } - // Setup event manager - p.setupEventListenersPi() - - case cmsWWWMode: - // Setup event manager - p.setupEventListenersCMS() - - // Setup dcrdata websocket connection - ws, err := wsdcrdata.New(p.dcrdataHostWS()) - if err != nil { - // Continue even if a websocket connection was not able to be - // made. The application specific websocket setup (pi, cms, etc) - // can decide whether to attempt reconnection or to exit. - log.Errorf("wsdcrdata New: %v", err) - } - p.wsDcrdata = ws - - pluginFound := false - for _, plugin := range p.plugins { - if plugin.ID == "cms" { - pluginFound = true - break - } - } - if !pluginFound { - return fmt.Errorf("politeiad plugin 'cms' not found") - } - - p.setCMSWWWRoutes() - // XXX setup user routes - p.setCMSUserWWWRoutes() - - // Setup cmsdb - net := filepath.Base(p.cfg.DataDir) - p.cmsDB, err = cmsdb.New(p.cfg.DBHost, net, p.cfg.DBRootCert, - p.cfg.DBCert, p.cfg.DBKey) - if err == database.ErrNoVersionRecord || err == database.ErrWrongVersion { - // The cmsdb version record was either not found or - // is the wrong version which means that the cmsdb - // needs to be built/rebuilt. - p.cfg.BuildCMSDB = true - } else if err != nil { - return err - } - err = p.cmsDB.Setup() - if err != nil { - return fmt.Errorf("cmsdb setup: %v", err) - } - - // Build the cms database - if p.cfg.BuildCMSDB { - // Request full record inventory from backend - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return err - } - - pdCommand := pd.Inventory{ - Challenge: hex.EncodeToString(challenge), - IncludeFiles: true, - AllVersions: true, - } - - responseBody, err := p.makeRequest(http.MethodPost, - pd.InventoryRoute, pdCommand) - if err != nil { - return err - } - - var pdReply pd.InventoryReply - err = json.Unmarshal(responseBody, &pdReply) - if err != nil { - return fmt.Errorf("Could not unmarshal InventoryReply: %v", - err) - } - - // Verify the UpdateVettedMetadata challenge. - err = util.VerifyChallenge(p.cfg.Identity, challenge, pdReply.Response) - if err != nil { - return err - } - - vetted := pdReply.Vetted - dbInvs := make([]database.Invoice, 0, len(vetted)) - dbDCCs := make([]database.DCC, 0, len(vetted)) - for _, r := range vetted { - for _, m := range r.Metadata { - switch m.ID { - case mdstream.IDInvoiceGeneral: - i, err := convertRecordToDatabaseInvoice(r) - if err != nil { - log.Errorf("convertRecordToDatabaseInvoice: %v", err) - break - } - u, err := p.db.UserGetByPubKey(i.PublicKey) - if err != nil { - log.Errorf("usergetbypubkey: %v %v", err, i.PublicKey) - break - } - i.UserID = u.ID.String() - i.Username = u.Username - dbInvs = append(dbInvs, *i) - case mdstream.IDDCCGeneral: - d, err := convertRecordToDatabaseDCC(r) - if err != nil { - log.Errorf("convertRecordToDatabaseDCC: %v", err) - break - } - dbDCCs = append(dbDCCs, *d) - } - } - } - - // Build the cmsdb - err = p.cmsDB.Build(dbInvs, dbDCCs) - if err != nil { - return fmt.Errorf("build cmsdb: %v", err) - } - } - // Register cms userdb plugin - plugin := user.Plugin{ - ID: user.CMSPluginID, - Version: user.CMSPluginVersion, - } - err = p.db.RegisterPlugin(plugin) - if err != nil { - return fmt.Errorf("register userdb plugin: %v", err) - } - - // Setup invoice notifications - p.cron = cron.New() - p.checkInvoiceNotifications() - - // Setup dcrdata websocket subscriptions and monitoring. This is - // done in a go routine so cmswww startup will continue in - // the event that a dcrdata websocket connection was not able to - // be made during client initialization and reconnection attempts - // are required. - go func() { - p.setupCMSAddressWatcher() - }() - - default: - return fmt.Errorf("unknown mode: %v", p.cfg.Mode) - } - // Persist session cookies. - var cookieKey []byte - if cookieKey, err = ioutil.ReadFile(p.cfg.CookieKeyFile); err != nil { - log.Infof("Cookie key not found, generating one...") - cookieKey, err = util.Random(32) - if err != nil { - return err - } - err = ioutil.WriteFile(p.cfg.CookieKeyFile, cookieKey, 0400) - if err != nil { - return err - } - log.Infof("Cookie key generated.") - } - sessionsDir := filepath.Join(p.cfg.DataDir, "sessions") - err = os.MkdirAll(sessionsDir, 0700) - if err != nil { - return err - } - p.sessions = NewSessionStore(p.db, sessionMaxAge, cookieKey) - // Bind to a port and pass our router in listenC := make(chan error) for _, listener := range loadedCfg.Listeners { @@ -968,8 +1001,11 @@ done: // Close user db connection p.db.Close() - // Shutdown all dcrdata websockets - if p.wsDcrdata != nil { + // Perform application specific shutdown + switch p.cfg.Mode { + case politeiaWWWMode: + // Nothing to do + case cmsWWWMode: p.wsDcrdata.Close() } diff --git a/util/net.go b/util/net.go index bc98c5ca0..a3d4c7a1c 100644 --- a/util/net.go +++ b/util/net.go @@ -14,6 +14,7 @@ import ( "net" "net/http" "os" + "time" ) // NormalizeAddress returns addr with the passed default port appended if @@ -43,9 +44,13 @@ func NewClient(skipVerify bool, certFilename string) (*http.Client, error) { tlsConfig.RootCAs = certPool } - return &http.Client{Transport: &http.Transport{ - TLSClientConfig: tlsConfig, - }}, nil + return &http.Client{ + Timeout: 1 * time.Minute, + Transport: &http.Transport{ + IdleConnTimeout: 1 * time.Minute, + ResponseHeaderTimeout: 1 * time.Minute, + TLSClientConfig: tlsConfig, + }}, nil } // ConvertBodyToByteArray converts a response body into a byte array From 94c6febc66bd4da285523ba5f4d99b2fb683d15e Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 25 Sep 2020 13:38:40 -0500 Subject: [PATCH 110/449] debuging tlogbe --- plugins/pi/pi.go | 7 ++- politeiad/backend/tlogbe/tlog.go | 80 ++++++++++++++++++++-------- politeiad/backend/tlogbe/tlogbe.go | 37 +++++++++---- politeiad/backend/tlogbe/trillian.go | 26 ++++++--- politeiawww/politeiad.go | 2 +- 5 files changed, 107 insertions(+), 45 deletions(-) diff --git a/plugins/pi/pi.go b/plugins/pi/pi.go index 346d67f36..7080107e7 100644 --- a/plugins/pi/pi.go +++ b/plugins/pi/pi.go @@ -32,10 +32,9 @@ const ( CmdCommentVote = "commentvote" // Upvote/downvote a comment CmdVoteInventory = "voteinventory" // Get inventory by vote status - // Metadata stream IDs. All metadata streams in this plugin will - // use 1xx numbering. - MDStreamIDProposalGeneral = 101 - MDStreamIDStatusChanges = 102 + // Metadata stream IDs + MDStreamIDProposalGeneral = 1 + MDStreamIDStatusChanges = 2 // FileNameProposalMetadata is the filename of the ProposalMetadata // file that is saved to politeiad. ProposalMetadata is saved to diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index cd5f80f47..2b4777507 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -400,18 +400,24 @@ func convertAnchorFromBlobEntry(be store.BlobEntry) (*anchor, error) { } func (t *tlog) treeNew() (int64, error) { + log.Tracef("tlog treeNew") + tree, _, err := t.trillian.treeNew() if err != nil { return 0, err } + return tree.TreeId, nil } func (t *tlog) treeExists(treeID int64) bool { + log.Tracef("tlog treeExists: %v", treeID) + _, err := t.trillian.tree(treeID) if err == nil { return true } + return false } @@ -427,6 +433,8 @@ type freezeRecord struct { // update the status of the tree to frozen in trillian, at which point trillian // will not allow any additional leaves to be appended onto the tree. func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, fr freezeRecord) error { + log.Tracef("tlog treeFreeze: %v") + // Get tree leaves leavesAll, err := t.trillian.leavesAll(treeID) if err != nil { @@ -552,8 +560,14 @@ func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []ba return nil } -// lastLeaf returns the last leaf of the given trillian tree. -func (t *tlog) lastLeaf(treeID int64) (*trillian.LogLeaf, error) { +// freeze record returns the freeze record of the provided tree if one exists. +// If one does not exists a errFreezeRecordNotFound error is returned. +func (t *tlog) freezeRecord(treeID int64) (*freezeRecord, error) { + log.Tracef("tlog freezeRecord: %v", treeID) + + // Check if the tree contains a freeze record. The last two leaves + // are checked because the last leaf will be the final anchor drop, + // which may not exist yet if the tree was recently frozen. tree, err := t.trillian.tree(treeID) if err != nil { return nil, errRecordNotFound @@ -562,32 +576,36 @@ func (t *tlog) lastLeaf(treeID int64) (*trillian.LogLeaf, error) { if err != nil { return nil, fmt.Errorf("signedLogRootForTree: %v", err) } - leaves, err := t.trillian.leavesByRange(treeID, int64(lr.TreeSize)-1, 1) - if err != nil { - return nil, fmt.Errorf("leavesByRange: %v", err) - } - if len(leaves) != 1 { - return nil, fmt.Errorf("unexpected leaves count: got %v, want 1", - len(leaves)) + + var startIndex, count int64 + switch lr.TreeSize { + case 0: + return nil, errFreezeRecordNotFound + case 1: + startIndex = 0 + count = 1 + default: + startIndex = int64(lr.TreeSize) - 1 + count = 2 } - return leaves[0], nil -} -// TODO the last leaf will be a anchor -// freeze record returns the freeze record of the provided tree if one exists. -// If one does not exists a errFreezeRecordNotFound error is returned. -func (t *tlog) freezeRecord(treeID int64) (*freezeRecord, error) { - // Check if the last leaf of the tree is a freeze record - l, err := t.lastLeaf(treeID) + leaves, err := t.trillian.leavesByRange(treeID, startIndex, count) if err != nil { - return nil, fmt.Errorf("lastLeaf %v: %v", treeID, err) + return nil, fmt.Errorf("leavesByRange: %v", err) + } + var l *trillian.LogLeaf + for _, v := range leaves { + if leafIsFreezeRecord(l) { + l = v + break + } } - if !leafIsFreezeRecord(l) { - // Leaf is not a freeze record + if l == nil { + // No freeze record was found return nil, errFreezeRecordNotFound } - // The leaf is a freeze record. Get it from the store. + // A freeze record was found. Pull it from the store. k, err := extractKeyFromLeaf(l) if err != nil { return nil, err @@ -1044,6 +1062,8 @@ func (t *tlog) recordBlobsSave(treeID int64, rbpr recordBlobsPrepareReply) (*rec // We do not unwind. func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { + log.Tracef("tlog recordSave: %v", treeID) + // Ensure tree exists if !t.treeExists(treeID) { return errRecordNotFound @@ -1125,6 +1145,8 @@ func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []ba } func (t *tlog) recordMetadataUpdate(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { + log.Tracef("tlog recordMetadataUpdate: %v", treeID) + // Ensure tree exists if !t.treeExists(treeID) { return errRecordNotFound @@ -1208,6 +1230,8 @@ func (t *tlog) recordMetadataUpdate(treeID int64, rm backend.RecordMetadata, met // recordDel walks the provided tree and deletes all blobs in the store that // correspond to record files. This is done for all versions of the record. func (t *tlog) recordDel(treeID int64) error { + log.Tracef("tlog recordDel: %v", treeID) + // Ensure tree exists if !t.treeExists(treeID) { return errRecordNotFound @@ -1262,6 +1286,8 @@ func (t *tlog) recordDel(treeID int64) error { } func (t *tlog) record(treeID int64, version uint32) (*backend.Record, error) { + log.Tracef("tlog record: %v %v", treeID, version) + // Ensure tree exists if !t.treeExists(treeID) { return nil, errRecordNotFound @@ -1434,6 +1460,8 @@ func (t *tlog) record(treeID int64, version uint32) (*backend.Record, error) { } func (t *tlog) recordLatest(treeID int64) (*backend.Record, error) { + log.Tracef("tlog recordLatest: %v", treeID) + return t.record(treeID, 0) } @@ -1444,6 +1472,8 @@ func (t *tlog) recordProof(treeID int64, version uint32) {} // onto the trillian tree. Note, hashes contains the hashes of the data encoded // in the blobs. The hashes must share the same ordering as the blobs. func (t *tlog) blobsSave(treeID int64, keyPrefix string, blobs, hashes [][]byte, encrypt bool) ([][]byte, error) { + log.Tracef("tlog blobsSave: %v %v %v", treeID, keyPrefix, encrypt) + // Verify tree exists if !t.treeExists(treeID) { return nil, errRecordNotFound @@ -1521,6 +1551,8 @@ func (t *tlog) blobsSave(treeID int64, keyPrefix string, blobs, hashes [][]byte, // del deletes the blobs that correspond to the provided merkle leaf hashes. func (t *tlog) blobsDel(treeID int64, merkles [][]byte) error { + log.Tracef("tlog blobsDel: %v", treeID) + // Ensure tree exists. We allow blobs to be deleted from both // frozen and non frozen trees. if !t.treeExists(treeID) { @@ -1570,6 +1602,8 @@ func (t *tlog) blobsDel(treeID int64, merkles [][]byte) error { // the responsibility of the caller to check that a blob is returned for each // of the provided merkle hashes. func (t *tlog) blobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, error) { + log.Tracef("tlog blobsByMerkle: %v", treeID) + // Verify tree exists if !t.treeExists(treeID) { return nil, errRecordNotFound @@ -1655,6 +1689,8 @@ func (t *tlog) blobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, // blobsByKeyPrefix returns all blobs that match the provided key prefix. func (t *tlog) blobsByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) { + log.Tracef("tlog blobsByKeyPrefix: %v %v", treeID, keyPrefix) + // Verify tree exists if !t.treeExists(treeID) { return nil, errRecordNotFound @@ -1730,6 +1766,8 @@ func (t *tlog) fsck() { } func (t *tlog) close() { + log.Tracef("tlog close: %v", t.id) + // Close connections t.store.Close() t.trillian.close() diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index fa617474a..2f5c2af6d 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -961,24 +961,28 @@ func (t *tlogBackend) VettedExists(token []byte) bool { // This function satisfies the Backend interface. func (t *tlogBackend) GetUnvetted(token []byte, version string) (*backend.Record, error) { - log.Tracef("GetUnvetted: %x", token) + log.Tracef("GetUnvetted: %x %v", token, version) if t.isShutdown() { return nil, backend.ErrShutdown } treeID := treeIDFromToken(token) - v, err := strconv.ParseUint(version, 10, 64) - if err != nil { - return nil, backend.ErrRecordNotFound + var v uint32 + if version != "" { + u, err := strconv.ParseUint(version, 10, 64) + if err != nil { + return nil, backend.ErrRecordNotFound + } + v = uint32(u) } - return t.unvetted.record(treeID, uint32(v)) + return t.unvetted.record(treeID, v) } // This function satisfies the Backend interface. func (t *tlogBackend) GetVetted(token []byte, version string) (*backend.Record, error) { - log.Tracef("GetVetted: %x", token) + log.Tracef("GetVetted: %x %v", token, version) if t.isShutdown() { return nil, backend.ErrShutdown @@ -991,12 +995,16 @@ func (t *tlogBackend) GetVetted(token []byte, version string) (*backend.Record, } // Parse version - v, err := strconv.ParseUint(version, 10, 64) - if err != nil { - return nil, backend.ErrRecordNotFound + var v uint32 + if version != "" { + u, err := strconv.ParseUint(version, 10, 64) + if err != nil { + return nil, backend.ErrRecordNotFound + } + v = uint32(u) } - return t.vetted.record(treeID, uint32(v)) + return t.vetted.record(treeID, v) } // This function must be called WITH the unvetted lock held. @@ -1457,16 +1465,23 @@ func (t *tlogBackend) Close() { } func (t *tlogBackend) setup() error { + log.Tracef("setup") + // Get all trees trees, err := t.unvetted.trillian.treesAll() if err != nil { return fmt.Errorf("unvetted treesAll: %v", err) } + log.Infof("Building backend caches") + // Build all memory caches for _, v := range trees { - // Add tree to prefixes cache token := tokenFromTreeID(v.TreeId) + + log.Debugf("Building cache: %v %x", v.TreeId, token) + + // Add tree to prefixes cache t.prefixAdd(token) // Check if the tree needs to be added to the vettedTreeIDs cache diff --git a/politeiad/backend/tlogbe/trillian.go b/politeiad/backend/tlogbe/trillian.go index 2ddbe5a36..84fd626e8 100644 --- a/politeiad/backend/tlogbe/trillian.go +++ b/politeiad/backend/tlogbe/trillian.go @@ -37,6 +37,7 @@ import ( // for the backend to use and ensures that proper verification of all trillian // responses is performed. type trillianClient struct { + host string grpc *grpc.ClientConn client trillian.TrillianLogClient admin trillian.TrillianAdminClient @@ -80,7 +81,7 @@ func logLeafNew(value []byte, extraData []byte) *trillian.LogLeaf { // correct. It returns the tree and the signed log root which can be externally // verified. func (t *trillianClient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { - log.Tracef("treeNew") + log.Tracef("trillian treeNew") pk, err := ptypes.MarshalAny(t.privateKey) if err != nil { @@ -145,6 +146,8 @@ func (t *trillianClient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, err } func (t *trillianClient) treeFreeze(treeID int64) (*trillian.Tree, error) { + log.Tracef("trillian treeFreeze: %v", treeID) + // Get the current tree tree, err := t.tree(treeID) if err != nil { @@ -169,7 +172,7 @@ func (t *trillianClient) treeFreeze(treeID int64) (*trillian.Tree, error) { } func (t *trillianClient) tree(treeID int64) (*trillian.Tree, error) { - log.Tracef("tree: %v", treeID) + log.Tracef("trillian tree: %v", treeID) tree, err := t.admin.GetTree(t.ctx, &trillian.GetTreeRequest{ TreeId: treeID, @@ -187,7 +190,7 @@ func (t *trillianClient) tree(treeID int64) (*trillian.Tree, error) { } func (t *trillianClient) treesAll() ([]*trillian.Tree, error) { - log.Tracef("treesAll") + log.Tracef("trillian treesAll") ltr, err := t.admin.ListTrees(t.ctx, &trillian.ListTreesRequest{}) if err != nil { @@ -198,7 +201,7 @@ func (t *trillianClient) treesAll() ([]*trillian.Tree, error) { } func (t *trillianClient) inclusionProof(treeID int64, merkleLeafHash []byte, lrv1 *types.LogRootV1) (*trillian.Proof, error) { - log.Tracef("inclusionProof: %v %x", treeID, merkleLeafHash) + log.Tracef("tillian inclusionProof: %v %x", treeID, merkleLeafHash) resp, err := t.client.GetInclusionProofByHash(t.ctx, &trillian.GetInclusionProofByHashRequest{ @@ -254,7 +257,7 @@ func (t *trillianClient) signedLogRootForTree(tree *trillian.Tree) (*trillian.Si // signedLogRoot returns the signed log root for the provided tree ID at its // current height. The log root is structure is decoded an returned as well. func (t *trillianClient) signedLogRoot(treeID int64) (*trillian.SignedLogRoot, *types.LogRootV1, error) { - log.Tracef("SignedLogRoot: %v", treeID) + log.Tracef("trillian signedLogRoot: %v", treeID) tree, err := t.tree(treeID) if err != nil { @@ -275,7 +278,7 @@ func (t *trillianClient) signedLogRoot(treeID int64) (*trillian.SignedLogRoot, * // fail to be appended. Note leaves that are duplicates will fail and it is the // callers responsibility to determine how they should be handled. func (t *trillianClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, *types.LogRootV1, error) { - log.Tracef("LeavesAppend: %v", treeID) + log.Tracef("trillian leavesAppend: %v", treeID) // Get the latest signed log root tree, err := t.tree(treeID) @@ -377,6 +380,8 @@ func (t *trillianClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) } func (t *trillianClient) leavesByRange(treeID int64, startIndex, count int64) ([]*trillian.LogLeaf, error) { + log.Tracef("trillian leavesByRange: %v %v %v", treeID, startIndex, count) + glbrr, err := t.client.GetLeavesByRange(t.ctx, &trillian.GetLeavesByRangeRequest{ LogId: treeID, @@ -391,6 +396,8 @@ func (t *trillianClient) leavesByRange(treeID int64, startIndex, count int64) ([ // leavesAll returns all of the leaves for the provided treeID. func (t *trillianClient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { + log.Tracef("trillian leavesAll: %v", treeID) + // Get log root _, lr, err := t.signedLogRoot(treeID) if err != nil { @@ -405,7 +412,8 @@ func (t *trillianClient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { // hashes. The inclusion proof returned in the leafProof is for the tree height // specified by the provided LogRootV1. func (t *trillianClient) leafProofs(treeID int64, merkleLeafHashes [][]byte, lr *types.LogRootV1) ([]leafProof, error) { - log.Tracef("leafProofs: %v %v %x", treeID, lr.TreeSize, merkleLeafHashes) + log.Tracef("trillian leafProofs: %v %v %x", + treeID, lr.TreeSize, merkleLeafHashes) // Retrieve leaves r, err := t.client.GetLeavesByHash(t.ctx, @@ -433,8 +441,10 @@ func (t *trillianClient) leafProofs(treeID int64, merkleLeafHashes [][]byte, lr return proofs, nil } -// Close closes the trillian grpc connection. +// close closes the trillian grpc connection. func (t *trillianClient) close() { + log.Tracef("trillian close %v", t.host) + t.grpc.Close() } diff --git a/politeiawww/politeiad.go b/politeiawww/politeiad.go index 5763d47c0..af7676dd9 100644 --- a/politeiawww/politeiad.go +++ b/politeiawww/politeiad.go @@ -52,7 +52,7 @@ func (p *politeiawww) makeRequest(method string, route string, v interface{}) ([ fullRoute := p.cfg.RPCHost + route - log.Debugf("%v %v %+v", method, fullRoute, v) + log.Debugf("%v %v", method, fullRoute) req, err := http.NewRequest(method, fullRoute, bytes.NewReader(reqBody)) if err != nil { From ed78b86ff26bc508e66b10e7f03cfeed8b98c028 Mon Sep 17 00:00:00 2001 From: Amir Massarwa Date: Fri, 25 Sep 2020 22:49:29 +0300 Subject: [PATCH 111/449] cleanup pi func names --- politeiawww/piwww.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 4bda02839..42efb8950 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -1465,7 +1465,7 @@ func (p *politeiawww) processCommentNew(cn pi.CommentNew, usr user.User) (*pi.Co } // Call the pi plugin to add new comment - reply, err := p.piCommentNew(piplugin.CommentNew{ + reply, err := p.commentNewPi(piplugin.CommentNew{ UserID: usr.ID.String(), Token: cn.Token, ParentID: cn.ParentID, @@ -1557,7 +1557,7 @@ func (p *politeiawww) processCommentVote(cv pi.CommentVote, usr user.User) (*pi. } // Call the pi plugin to add new comment - reply, err := p.piCommentVote(piplugin.CommentVote{ + reply, err := p.commentVotePi(piplugin.CommentVote{ UserID: usr.ID.String(), Token: cv.Token, CommentID: cv.CommentID, From 449508fe34504561c322c6a99e43929d1cc93ae0 Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Fri, 25 Sep 2020 18:02:08 -0300 Subject: [PATCH 112/449] piwww: Add proposals command. --- politeiawww/cmd/piwww/help.go | 2 + politeiawww/cmd/piwww/piwww.go | 1 + politeiawww/cmd/piwww/proposals.go | 98 ++++++++++++++++++++++++++++++ politeiawww/cmd/shared/client.go | 50 +++++++-------- 4 files changed, 126 insertions(+), 25 deletions(-) create mode 100644 politeiawww/cmd/piwww/proposals.go diff --git a/politeiawww/cmd/piwww/help.go b/politeiawww/cmd/piwww/help.go index 888835f9a..07b2c40d6 100644 --- a/politeiawww/cmd/piwww/help.go +++ b/politeiawww/cmd/piwww/help.go @@ -76,6 +76,8 @@ func (cmd *HelpCmd) Execute(args []string) error { fmt.Printf("%s\n", proposalEditHelpMsg) case "proposalsetstatus": fmt.Printf("%s\n", proposalSetStatusHelpMsg) + case "proposals": + fmt.Printf("%s\n", proposalsHelpMsg) case "proposalinventory": fmt.Printf("%s\n", proposalInventoryHelpMsg) diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index dad61c7a2..3b68d1a29 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -48,6 +48,7 @@ type piwww struct { ProposalNew ProposalNewCmd `command:"proposalnew"` ProposalEdit ProposalEditCmd `command:"proposaledit"` ProposalSetStatus ProposalSetStatusCmd `command:"proposalsetstatus"` + Proposals ProposalsCmd `command:"proposals"` ProposalInventory ProposalInventoryCmd `command:"proposalinventory"` // Comments commands diff --git a/politeiawww/cmd/piwww/proposals.go b/politeiawww/cmd/piwww/proposals.go new file mode 100644 index 000000000..d0f141c73 --- /dev/null +++ b/politeiawww/cmd/piwww/proposals.go @@ -0,0 +1,98 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "strings" + + pi "github.com/decred/politeia/politeiawww/api/pi/v1" + "github.com/decred/politeia/politeiawww/cmd/shared" +) + +// ProposalsCmd retrieves the proposal records of the requested tokens and versions. +type ProposalsCmd struct { + Args struct { + Proposals []string `positional-arg-name:"proposals" required:"true"` + } `positional-args:"true" optional:"true"` + + // Unvetted requests for unvetted proposals instead of vetted ones. + Unvetted bool `long:"unvetted" optional:"true"` + + // IncludeFiles adds the proposals files to the response payload. + IncludeFiles bool `long:"includefiles" optional:"true"` +} + +// Execute executes the proposals command. +func (cmd *ProposalsCmd) Execute(args []string) error { + proposals := cmd.Args.Proposals + + // Set state to get unvetted or vetted proposals. Defaults + // to vetted unless the unvetted flag is used. + var state pi.PropStateT + switch { + case cmd.Unvetted: + state = pi.PropStateUnvetted + default: + state = pi.PropStateVetted + } + + // Build proposals request + var requests []pi.ProposalRequest + for _, p := range proposals { + // Parse token and version + var r pi.ProposalRequest + tokenAndVersion := strings.Split(p, ",") + switch len(tokenAndVersion) { + case 1: + // No version provided + r.Token = tokenAndVersion[0] + case 2: + // Version provided + r.Token = tokenAndVersion[0] + r.Version = tokenAndVersion[1] + default: + return fmt.Errorf("invalid format for proposal request. check " + + "the help command for usage example") + } + + requests = append(requests, r) + } + + // Setup and send request + props := pi.Proposals{ + State: state, + Requests: requests, + IncludeFiles: cmd.IncludeFiles, + } + + reply, err := client.Proposals(props) + if err != nil { + return err + } + + return shared.PrintJSON(reply) +} + +// proposalsHelpMsg is the output of the help command. +const proposalsHelpMsg = `proposals [flags] "proposals" + +Fetch the proposal record for the requested tokens in "proposals". A request +is set by providing the censorship record token and the desired version, +comma-separated. Providing only the token will default to the latest proposal +version. + +Arguments: +1. proposals ([]string, required) Proposals request + +Flags: + --unvetted (bool, optional) Request for unvetted proposals instead of + vetted ones. + --includefiles (bool, optional) Include proposal files in the returned + proposal records. + +Example: + piwww proposals ... +` diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index 72b46ff98..574165e75 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -796,6 +796,31 @@ func (c *Client) Proposals(p pi.Proposals) (*pi.ProposalsReply, error) { return &pr, nil } +// ProposalInventory retrieves the censorship tokens of all proposals, +// separated by their status. +func (c *Client) ProposalInventory() (*pi.ProposalInventoryReply, error) { + respondeBody, err := c.makeRequest(http.MethodGet, pi.APIRoute, + pi.RouteProposalInventory, nil) + if err != nil { + return nil, err + } + + var pir pi.ProposalInventoryReply + err = json.Unmarshal(respondeBody, &pir) + if err != nil { + return nil, fmt.Errorf("unmarshal ProposalInventory: %v", err) + } + + if c.cfg.Verbose { + err := prettyPrintJSON(pir) + if err != nil { + return nil, err + } + } + + return &pir, nil +} + // NewInvoice submits the specified invoice to politeiawww for the logged in // user. func (c *Client) NewInvoice(ni *cms.NewInvoice) (*cms.NewInvoiceReply, error) { @@ -1086,31 +1111,6 @@ func (c *Client) VoteInventory() (*pi.VoteInventoryReply, error) { return &vir, nil } -// ProposalInventory retrieves the censorship tokens of all proposals, -// separated by their status. -func (c *Client) ProposalInventory() (*pi.ProposalInventoryReply, error) { - respondeBody, err := c.makeRequest(http.MethodGet, pi.APIRoute, - pi.RouteProposalInventory, nil) - if err != nil { - return nil, err - } - - var pir pi.ProposalInventoryReply - err = json.Unmarshal(respondeBody, &pir) - if err != nil { - return nil, fmt.Errorf("unmarshal ProposalInventory: %v", err) - } - - if c.cfg.Verbose { - err := prettyPrintJSON(pir) - if err != nil { - return nil, err - } - } - - return &pir, nil -} - // BatchProposals retrieves a list of proposals func (c *Client) BatchProposals(bp *www.BatchProposals) (*www.BatchProposalsReply, error) { responseBody, err := c.makeRequest(http.MethodPost, www.PoliteiaWWWAPIRoute, From e475e3adf37bbeb78e83cebe5289852519e12a10 Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Sat, 26 Sep 2020 13:17:09 -0300 Subject: [PATCH 113/449] piwww: Remove www proposal/comment cmds. --- .../cmd/{shared => cmswww}/batchproposals.go | 9 +- politeiawww/cmd/cmswww/cmswww.go | 4 +- politeiawww/cmd/cmswww/help.go | 2 + .../cmd/{shared => cmswww}/tokeninventory.go | 8 +- .../{proposalcomments.go => comments.go} | 0 politeiawww/cmd/piwww/help.go | 29 +++--- politeiawww/cmd/piwww/piwww.go | 9 +- politeiawww/cmd/piwww/proposaldetails.go | 94 ------------------- politeiawww/cmd/piwww/proposalpaywall.go | 19 ++-- politeiawww/cmd/shared/client.go | 90 +++++++++--------- 10 files changed, 83 insertions(+), 181 deletions(-) rename politeiawww/cmd/{shared => cmswww}/batchproposals.go (91%) rename politeiawww/cmd/{shared => cmswww}/tokeninventory.go (67%) rename politeiawww/cmd/piwww/{proposalcomments.go => comments.go} (100%) delete mode 100644 politeiawww/cmd/piwww/proposaldetails.go diff --git a/politeiawww/cmd/shared/batchproposals.go b/politeiawww/cmd/cmswww/batchproposals.go similarity index 91% rename from politeiawww/cmd/shared/batchproposals.go rename to politeiawww/cmd/cmswww/batchproposals.go index b95e4d567..e4572fa53 100644 --- a/politeiawww/cmd/shared/batchproposals.go +++ b/politeiawww/cmd/cmswww/batchproposals.go @@ -2,12 +2,13 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package shared +package main import ( "fmt" v1 "github.com/decred/politeia/politeiawww/api/www/v1" + "github.com/decred/politeia/politeiawww/cmd/shared" ) // BatchProposalsCmd retrieves a set of proposals. @@ -31,7 +32,7 @@ func (cmd *BatchProposalsCmd) Execute(args []string) error { // Verify proposal censorship records for _, p := range bpr.Proposals { - err = VerifyProposal(p, vr.PubKey) + err = shared.VerifyProposal(p, vr.PubKey) if err != nil { return fmt.Errorf("unable to verify proposal %v: %v", p.CensorshipRecord.Token, err) @@ -39,12 +40,12 @@ func (cmd *BatchProposalsCmd) Execute(args []string) error { } // Print proposals - return PrintJSON(bpr) + return shared.PrintJSON(bpr) } // batchProposalsHelpMsg is the output for the help command when // 'batchproposals' is specified. -const BatchProposalsHelpMsg = `batchproposals +const batchProposalsHelpMsg = `batchproposals Fetch a list of proposals. diff --git a/politeiawww/cmd/cmswww/cmswww.go b/politeiawww/cmd/cmswww/cmswww.go index 49f669a78..831ce1c49 100644 --- a/politeiawww/cmd/cmswww/cmswww.go +++ b/politeiawww/cmd/cmswww/cmswww.go @@ -58,7 +58,7 @@ type cmswww struct { // Commands ActiveVotes ActiveVotesCmd `command:"activevotes" description:"(user) get the dccs that are being voted on"` - BatchProposals shared.BatchProposalsCmd `command:"batchproposals" description:"(user) retrieve a set of proposals"` + BatchProposals BatchProposalsCmd `command:"batchproposals" description:"(user) retrieve a set of proposals"` NewComment NewCommentCmd `command:"newcomment" description:"(user) create a new comment"` CensorComment CensorCommentCmd `command:"censorcomment" description:"(admin) censor a comment"` ChangePassword shared.ChangePasswordCmd `command:"changepassword" description:"(user) change the password for the logged in user"` @@ -99,7 +99,7 @@ type cmswww struct { StartVote StartVoteCmd `command:"startvote" description:"(admin) start the voting period on a dcc"` SupportOpposeDCC SupportOpposeDCCCmd `command:"supportopposedcc" description:"(user) support or oppose a given DCC"` TestRun TestRunCmd `command:"testrun" description:" test cmswww routes"` - TokenInventory shared.TokenInventoryCmd `command:"tokeninventory" description:"(user) get the censorship record tokens of all proposals (passthrough)"` + TokenInventory TokenInventoryCmd `command:"tokeninventory" description:"(user) get the censorship record tokens of all proposals (passthrough)"` UpdateUserKey shared.UpdateUserKeyCmd `command:"updateuserkey" description:"(user) generate a new identity for the logged in user"` UserDetails UserDetailsCmd `command:"userdetails" description:"(user) get current cms user details"` UserInvoices UserInvoicesCmd `command:"userinvoices" description:"(user) get all invoices submitted by a specific user"` diff --git a/politeiawww/cmd/cmswww/help.go b/politeiawww/cmd/cmswww/help.go index 2f47ef4a2..67eb3fb25 100644 --- a/politeiawww/cmd/cmswww/help.go +++ b/politeiawww/cmd/cmswww/help.go @@ -77,6 +77,8 @@ func (cmd *HelpCmd) Execute(args []string) error { fmt.Printf("%s\n", dccCommentsHelpMsg) case "newdcccomment": fmt.Printf("%s\n", newDCCCommentHelpMsg) + case "batchproposals": + fmt.Printf("%s\n", batchProposalsHelpMsg) default: fmt.Printf("invalid command: use 'cmswww -h' " + diff --git a/politeiawww/cmd/shared/tokeninventory.go b/politeiawww/cmd/cmswww/tokeninventory.go similarity index 67% rename from politeiawww/cmd/shared/tokeninventory.go rename to politeiawww/cmd/cmswww/tokeninventory.go index c7a44aa18..588567b86 100644 --- a/politeiawww/cmd/shared/tokeninventory.go +++ b/politeiawww/cmd/cmswww/tokeninventory.go @@ -2,9 +2,11 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package shared +package main -// TokenInventory retrieves the censorship record tokens of all proposals in +import "github.com/decred/politeia/politeiawww/cmd/shared" + +// TokenInventoryCmd retrieves the censorship record tokens of all proposals in // the inventory. type TokenInventoryCmd struct{} @@ -15,5 +17,5 @@ func (cmd *TokenInventoryCmd) Execute(args []string) error { return err } - return PrintJSON(reply) + return shared.PrintJSON(reply) } diff --git a/politeiawww/cmd/piwww/proposalcomments.go b/politeiawww/cmd/piwww/comments.go similarity index 100% rename from politeiawww/cmd/piwww/proposalcomments.go rename to politeiawww/cmd/piwww/comments.go diff --git a/politeiawww/cmd/piwww/help.go b/politeiawww/cmd/piwww/help.go index 07b2c40d6..a2a8d2d42 100644 --- a/politeiawww/cmd/piwww/help.go +++ b/politeiawww/cmd/piwww/help.go @@ -60,10 +60,10 @@ func (cmd *HelpCmd) Execute(args []string) error { fmt.Printf("%s\n", shared.UpdateUserKeyHelpMsg) case "userpendingpayment": fmt.Printf("%s\n", userPendingPaymentHelpMsg) - case "proposalpaywall": - fmt.Printf("%s\n", proposalPaywallHelpMsg) case "rescanuserpayments": fmt.Printf("%s\n", rescanUserPaymentsHelpMsg) + case "proposalpaywall": + fmt.Printf("%s\n", proposalPaywallHelpMsg) case "verifyuserpayment": fmt.Printf("%s\n", verifyUserPaymentHelpMsg) case "resendverification": @@ -81,24 +81,20 @@ func (cmd *HelpCmd) Execute(args []string) error { case "proposalinventory": fmt.Printf("%s\n", proposalInventoryHelpMsg) - case "proposaldetails": - fmt.Printf("%s\n", proposalDetailsHelpMsg) case "userproposals": fmt.Printf("%s\n", userProposalsHelpMsg) case "vettedproposals": fmt.Printf("%s\n", vettedProposalsHelpMsg) - case "batchproposals": - fmt.Printf("%s\n", shared.BatchProposalsHelpMsg) // Comment commands case "commentnew": fmt.Printf("%s\n", commentNewHelpMsg) - case "proposalcomments": - fmt.Printf("%s\n", proposalCommentsHelpMsg) - case "commentcensor": - fmt.Printf("%s\n", commentCensorHelpMsg) case "commentvote": fmt.Printf("%s\n", commentVoteHelpMsg) + case "commentcensor": + fmt.Printf("%s\n", commentCensorHelpMsg) + case "comments": + fmt.Printf("%s\n", proposalCommentsHelpMsg) case "commentvotes": fmt.Printf("%s\n", commentVotesHelpMsg) @@ -109,22 +105,23 @@ func (cmd *HelpCmd) Execute(args []string) error { fmt.Printf("%s\n", voteStartHelpMsg) case "votestartrunoff": fmt.Printf("%s\n", voteStartRunoffHelpMsg) + case "voteballot": + fmt.Printf("%s\n", voteInventoryHelpMsg) case "votes": fmt.Printf("%s\n", votesHelpMsg) case "voteresults": fmt.Printf("%s\n", voteResultsHelpMsg) + case "votesummaries": + fmt.Printf("%s\n", voteSummariesHelpMsg) + case "voteinventory": + fmt.Printf("%s\n", voteInventoryHelpMsg) + case "votestatus": fmt.Printf("%s\n", voteStatusHelpMsg) case "votestatuses": fmt.Printf("%s\n", voteStatusesHelpMsg) - case "votesummaries": - fmt.Printf("%s\n", voteSummariesHelpMsg) case "votedetails": fmt.Printf("%s\n", voteDetailsHelpMsg) - case "voteinventory": - fmt.Printf("%s\n", voteInventoryHelpMsg) - case "voteballot": - fmt.Printf("%s\n", voteInventoryHelpMsg) // Websocket commands case "subscribe": diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index 3b68d1a29..ccabb6836 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -59,17 +59,16 @@ type piwww struct { CommentVotes CommentVotesCmd `command:"commentvotes" description:"(user) get comment upvotes/downvotes for a proposal from the provided user"` // Vote commands - VoteStartRunoff VoteStartRunoffCmd `command:"votestartrunoff" description:"(admin) start a runoff using the submissions to an RFP"` - VoteStart VoteStartCmd `command:"votestart" description:"(admin) start the voting period on a proposal"` VoteAuthorize VoteAuthorizeCmd `command:"voteauthorize" description:"(user) authorize a proposal vote (must be proposal author)"` + VoteStart VoteStartCmd `command:"votestart" description:"(admin) start the voting period on a proposal"` + VoteStartRunoff VoteStartRunoffCmd `command:"votestartrunoff" description:"(admin) start a runoff using the submissions to an RFP"` + VoteBallot VoteBallotCmd `command:"voteballot" description:"(public) cast ballot of votes for a proposal"` Votes VotesCmd `command:"votes" description:"(public) get the vote tally for a proposal"` VoteResults VoteResultsCmd `command:"voteresults" description:"(public) get vote results for a proposal"` VoteSummaries VoteSummariesCmd `command:"votesummaries" description:"(public) retrieve the vote summary for a set of proposals"` VoteInventory VoteInventoryCmd `command:"voteinventory" description:"(public) retrieve the tokens of all public, non-abandoned proposal separated by vote status"` - VoteBallot VoteBallotCmd `command:"voteballot" description:"(public) cast ballot of votes for a proposal"` // Commands - BatchProposals shared.BatchProposalsCmd `command:"batchproposals" description:"(user) retrieve a set of proposals"` ChangePassword shared.ChangePasswordCmd `command:"changepassword" description:"(user) change the password for the logged in user"` ChangeUsername shared.ChangeUsernameCmd `command:"changeusername" description:"(user) change the username for the logged in user"` EditUser EditUserCmd `command:"edituser" description:"(user) edit the preferences of the logged in user"` @@ -80,7 +79,6 @@ type piwww struct { Me shared.MeCmd `command:"me" description:"(user) get user details for the logged in user"` NewUser NewUserCmd `command:"newuser" description:"(public) create a new user"` Policy PolicyCmd `command:"policy" description:"(public) get the server policy"` - ProposalDetails ProposalDetailsCmd `command:"proposaldetails" description:"(public) get the details of a proposal"` ProposalPaywall ProposalPaywallCmd `command:"proposalpaywall" description:"(user) get proposal paywall details for the logged in user"` RescanUserPayments RescanUserPaymentsCmd `command:"rescanuserpayments" description:"(admin) rescan a user's payments to check for missed payments"` ResendVerification ResendVerificationCmd `command:"resendverification" description:"(public) resend the user verification email"` @@ -90,7 +88,6 @@ type piwww struct { SetTOTP shared.SetTOTPCmd `command:"settotp" description:"(user) set the key for TOTP"` Subscribe SubscribeCmd `command:"subscribe" description:"(public) subscribe to all websocket commands and do not exit tool"` TestRun TestRunCmd `command:"testrun" description:" run a series of tests on the politeiawww routes (dev use only)"` - TokenInventory shared.TokenInventoryCmd `command:"tokeninventory" description:"(public) get the censorship record tokens of all proposals"` UpdateUserKey shared.UpdateUserKeyCmd `command:"updateuserkey" description:"(user) generate a new identity for the logged in user"` UserDetails UserDetailsCmd `command:"userdetails" description:"(public) get the details of a user profile"` UserPendingPayment UserPendingPaymentCmd `command:"userpendingpayment" description:"(user) get details for a pending payment for the logged in user"` diff --git a/politeiawww/cmd/piwww/proposaldetails.go b/politeiawww/cmd/piwww/proposaldetails.go deleted file mode 100644 index bb0f5b3b0..000000000 --- a/politeiawww/cmd/piwww/proposaldetails.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" - - www "github.com/decred/politeia/politeiawww/api/www/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" -) - -// ProposalDetailsCmd retrieves the details of a proposal. -type ProposalDetailsCmd struct { - Args struct { - Token string `positional-arg-name:"token" required:"true"` // Censorship token - Version string `positional-arg-name:"version"` // Proposal version - } `positional-args:"true"` -} - -// Execute executes the proposal details command. -func (cmd *ProposalDetailsCmd) Execute(args []string) error { - // Get server's public key - vr, err := client.Version() - if err != nil { - return err - } - - if len(cmd.Args.Token) == www.TokenPrefixLength && cmd.Args.Version != "" { - fmt.Println("VERSION ARGUMENT CANNOT BE USED WITH TOKEN PREFIX!!") - } - - // Get proposal - pdr, err := client.ProposalDetails(cmd.Args.Token, - &www.ProposalsDetails{ - Version: cmd.Args.Version, - }) - if err != nil { - return err - } - - // Verify proposal censorship record - err = shared.VerifyProposal(pdr.Proposal, vr.PubKey) - if err != nil { - return fmt.Errorf("unable to verify proposal %v: %v", - pdr.Proposal.CensorshipRecord.Token, err) - } - - // Print proposal details - return shared.PrintJSON(pdr) -} - -// proposalDetailsHelpMsg is the output for the help command when -// 'proposaldetails' is specified. -const proposalDetailsHelpMsg = `proposaldetails "token" "version" - -Get a proposal. - -The 7 character prefix of the token can also be used instead of the full token, -but when using the token prefix, only the latest version can be retrieved. - -Arguments: -1. token (string, required) Censorship token -2. version (string, optional) Proposal version - -Result: -{ - "proposal": { - "name": (string) Suggested short proposal name - "state": (PropStateT) Current state of proposal - "status": (PropStatusT) Current status of proposal - "timestamp": (int64) Timestamp of last update of proposal - "userid": (string) ID of user who submitted proposal - "username": (string) Username of user who submitted proposal - "publickey": (string) Public key used to sign proposal - "signature": (string) Signature of merkle root - "files": [ - { - "name": (string) Filename - "mime": (string) Mime type - "digest": (string) File digest - "payload": (string) File payload - } - ], - "numcomments": (uint) Number of comments on the proposal - "version": (string) Version of proposal - "censorshiprecord": { - "token": (string) Censorship token - "merkle": (string) Merkle root of proposal - "signature": (string) Server side signature of []byte(Merkle+Token) - } - } -}` diff --git a/politeiawww/cmd/piwww/proposalpaywall.go b/politeiawww/cmd/piwww/proposalpaywall.go index 4bc695973..f212b96be 100644 --- a/politeiawww/cmd/piwww/proposalpaywall.go +++ b/politeiawww/cmd/piwww/proposalpaywall.go @@ -20,15 +20,12 @@ func (cmd *ProposalPaywallCmd) Execute(args []string) error { // proposalPaywallHelpMsg is the output of the help command when // 'proposalpaywall' is specified. -const proposalPaywallHelpMsg = `proposalpaywall - -Fetch proposal paywall details. - -Arguments: None - -Response: -{ - "creditprice" (uint64) Price per proposal credit in atoms - "paywalladdress" (string) Proposal paywall address - "paywalltxnotbefore" (string) Minimum timestamp for paywall tx +const proposalPaywallHelpMsg = `proposalpaywall +Fetch proposal paywall details. +Arguments: None +Response: +{ + "creditprice" (uint64) Price per proposal credit in atoms + "paywalladdress" (string) Proposal paywall address + "paywalltxnotbefore" (string) Minimum timestamp for paywall tx }` diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index 574165e75..bb8e725b0 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -1232,77 +1232,77 @@ func (c *Client) CommentNew(cn pi.CommentNew) (*pi.CommentNewReply, error) { return &cnr, nil } -// Comments retrieves the comments for the specified proposal. -func (c *Client) Comments(cs pi.Comments) (*pi.CommentsReply, error) { +// CommentVote casts a like comment action (upvote/downvote) for the logged in +// user. +func (c *Client) CommentVote(cv pi.CommentVote) (*pi.CommentVoteReply, error) { responseBody, err := c.makeRequest(http.MethodPost, - pi.APIRoute, pi.RouteComments, &cs) + pi.APIRoute, pi.RouteCommentVote, cv) if err != nil { return nil, err } - var cr pi.CommentsReply - err = json.Unmarshal(responseBody, &cr) + var cvr pi.CommentVoteReply + err = json.Unmarshal(responseBody, &cvr) if err != nil { - return nil, fmt.Errorf("unmarshal CommentsReply: %v", err) + return nil, fmt.Errorf("unmarshal CommentVoteReply: %v", err) } if c.cfg.Verbose { - err := prettyPrintJSON(cr) + err := prettyPrintJSON(cvr) if err != nil { return nil, err } } - return &cr, nil + return &cvr, nil } -// InvoiceComments retrieves the comments for the specified proposal. -func (c *Client) InvoiceComments(token string) (*www.GetCommentsReply, error) { - route := "/invoices/" + token + "/comments" - responseBody, err := c.makeRequest(http.MethodGet, - cms.APIRoute, route, nil) +// CommentCensor censors the specified proposal comment. +func (c *Client) CommentCensor(cc pi.CommentCensor) (*pi.CommentCensorReply, error) { + responseBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, + pi.RouteCommentCensor, cc) if err != nil { return nil, err } - var gcr www.GetCommentsReply - err = json.Unmarshal(responseBody, &gcr) + var ccr pi.CommentCensorReply + err = json.Unmarshal(responseBody, &ccr) if err != nil { - return nil, fmt.Errorf("unmarshal InvoiceCommentsReply: %v", err) + return nil, fmt.Errorf("unmarshal CensorCommentReply: %v", err) } if c.cfg.Verbose { - err := prettyPrintJSON(gcr) + err := prettyPrintJSON(ccr) if err != nil { return nil, err } } - return &gcr, nil + return &ccr, nil } -// Votes rerieves the vote details for a given proposal. -func (c *Client) Votes(vs pi.Votes) (*pi.VotesReply, error) { +// Comments retrieves the comments for the specified proposal. +func (c *Client) Comments(cs pi.Comments) (*pi.CommentsReply, error) { responseBody, err := c.makeRequest(http.MethodPost, - pi.APIRoute, pi.RouteVotes, vs) + pi.APIRoute, pi.RouteComments, &cs) if err != nil { return nil, err } - var vsr pi.VotesReply - err = json.Unmarshal(responseBody, &vsr) + var cr pi.CommentsReply + err = json.Unmarshal(responseBody, &cr) if err != nil { - return nil, fmt.Errorf("unmarshal Votes: %v", err) + return nil, fmt.Errorf("unmarshal CommentsReply: %v", err) } if c.cfg.Verbose { - err := prettyPrintJSON(vsr) + err := prettyPrintJSON(cr) if err != nil { return nil, err } } - return &vsr, nil + return &cr, nil } // CommentVotes retrieves the comment likes (upvotes/downvotes) for the @@ -1330,53 +1330,53 @@ func (c *Client) CommentVotes(cv pi.CommentVotes) (*pi.CommentVotesReply, error) return &cvr, nil } -// CommentVote casts a like comment action (upvote/downvote) for the logged in -// user. -func (c *Client) CommentVote(cv pi.CommentVote) (*pi.CommentVoteReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, - pi.APIRoute, pi.RouteCommentVote, cv) +// InvoiceComments retrieves the comments for the specified proposal. +func (c *Client) InvoiceComments(token string) (*www.GetCommentsReply, error) { + route := "/invoices/" + token + "/comments" + responseBody, err := c.makeRequest(http.MethodGet, + cms.APIRoute, route, nil) if err != nil { return nil, err } - var cvr pi.CommentVoteReply - err = json.Unmarshal(responseBody, &cvr) + var gcr www.GetCommentsReply + err = json.Unmarshal(responseBody, &gcr) if err != nil { - return nil, fmt.Errorf("unmarshal CommentVoteReply: %v", err) + return nil, fmt.Errorf("unmarshal InvoiceCommentsReply: %v", err) } if c.cfg.Verbose { - err := prettyPrintJSON(cvr) + err := prettyPrintJSON(gcr) if err != nil { return nil, err } } - return &cvr, nil + return &gcr, nil } -// CommentCensor censors the specified proposal comment. -func (c *Client) CommentCensor(cc pi.CommentCensor) (*pi.CommentCensorReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, - pi.RouteCommentCensor, cc) +// Votes rerieves the vote details for a given proposal. +func (c *Client) Votes(vs pi.Votes) (*pi.VotesReply, error) { + responseBody, err := c.makeRequest(http.MethodPost, + pi.APIRoute, pi.RouteVotes, vs) if err != nil { return nil, err } - var ccr pi.CommentCensorReply - err = json.Unmarshal(responseBody, &ccr) + var vsr pi.VotesReply + err = json.Unmarshal(responseBody, &vsr) if err != nil { - return nil, fmt.Errorf("unmarshal CensorCommentReply: %v", err) + return nil, fmt.Errorf("unmarshal Votes: %v", err) } if c.cfg.Verbose { - err := prettyPrintJSON(ccr) + err := prettyPrintJSON(vsr) if err != nil { return nil, err } } - return &ccr, nil + return &vsr, nil } // WWWCensorComment censors the specified proposal comment. From 339cc1e5c4fb83e157b4c2916bf27f2246547a8c Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Sun, 27 Sep 2020 12:37:57 -0300 Subject: [PATCH 114/449] piwww: Naming refactor and cleanup. --- politeiawww/cmd/cmswww/cmswww.go | 106 ++++++++--------- politeiawww/cmd/cmswww/help.go | 10 +- politeiawww/cmd/piwww/commentcensor.go | 6 +- politeiawww/cmd/piwww/commentnew.go | 8 +- politeiawww/cmd/piwww/comments.go | 12 +- politeiawww/cmd/piwww/commentvote.go | 6 +- politeiawww/cmd/piwww/commentvotes.go | 6 +- politeiawww/cmd/piwww/help.go | 110 +++++++++-------- politeiawww/cmd/piwww/piwww.go | 111 ++++++++++-------- politeiawww/cmd/piwww/policy.go | 30 +---- politeiawww/cmd/piwww/proposaledit.go | 6 +- politeiawww/cmd/piwww/proposalinventory.go | 6 +- politeiawww/cmd/piwww/proposalnew.go | 6 +- politeiawww/cmd/piwww/proposalpaywall.go | 16 +-- politeiawww/cmd/piwww/proposals.go | 6 +- politeiawww/cmd/piwww/proposalsetstatus.go | 6 +- politeiawww/cmd/piwww/sendfaucettx.go | 6 +- politeiawww/cmd/piwww/subscribe.go | 13 +- politeiawww/cmd/piwww/testrun.go | 6 +- politeiawww/cmd/piwww/userdetails.go | 41 +------ .../cmd/piwww/{edituser.go => useredit.go} | 22 ++-- ...{verifyuseremail.go => useremailverify.go} | 19 ++- .../cmd/piwww/{newuser.go => usernew.go} | 29 ++--- ...nuserpayments.go => userpaymentsrescan.go} | 19 ++- politeiawww/cmd/piwww/userpaymentverify.go | 28 +++++ politeiawww/cmd/piwww/userpendingpayment.go | 15 +-- politeiawww/cmd/piwww/userproposals.go | 90 -------------- ...ification.go => userverificationresend.go} | 21 ++-- politeiawww/cmd/piwww/verifyuserpayment.go | 36 ------ politeiawww/cmd/piwww/vettedproposals.go | 99 ---------------- politeiawww/cmd/piwww/voteauthorize.go | 6 +- politeiawww/cmd/piwww/voteballot.go | 6 +- politeiawww/cmd/piwww/votedetails.go | 6 +- politeiawww/cmd/piwww/voteinventory.go | 6 +- politeiawww/cmd/piwww/voteresults.go | 6 +- politeiawww/cmd/piwww/votes.go | 8 +- politeiawww/cmd/piwww/votestart.go | 6 +- politeiawww/cmd/piwww/votestartrunoff.go | 6 +- politeiawww/cmd/piwww/votestatus.go | 34 +----- politeiawww/cmd/piwww/votestatuses.go | 33 +----- politeiawww/cmd/piwww/votesummaries.go | 6 +- politeiawww/cmd/shared/logout.go | 5 +- .../{updateuserkey.go => userkeyupdate.go} | 20 ++-- .../shared/{manageuser.go => usermanage.go} | 22 +--- ...hangepassword.go => userpasswordchange.go} | 25 ++-- ...{resetpassword.go => userpasswordreset.go} | 13 +- politeiawww/cmd/shared/users.go | 33 +----- .../cmd/shared/{settotp.go => usertotpset.go} | 6 +- .../{verifytotp.go => usertotpverify.go} | 6 +- ...hangeusername.go => userusernamechange.go} | 25 ++-- 50 files changed, 379 insertions(+), 794 deletions(-) rename politeiawww/cmd/piwww/{edituser.go => useredit.go} (88%) rename politeiawww/cmd/piwww/{verifyuseremail.go => useremailverify.go} (71%) rename politeiawww/cmd/piwww/{newuser.go => usernew.go} (87%) rename politeiawww/cmd/piwww/{rescanuserpayments.go => userpaymentsrescan.go} (64%) create mode 100644 politeiawww/cmd/piwww/userpaymentverify.go delete mode 100644 politeiawww/cmd/piwww/userproposals.go rename politeiawww/cmd/piwww/{resendverification.go => userverificationresend.go} (71%) delete mode 100644 politeiawww/cmd/piwww/verifyuserpayment.go delete mode 100644 politeiawww/cmd/piwww/vettedproposals.go rename politeiawww/cmd/shared/{updateuserkey.go => userkeyupdate.go} (78%) rename politeiawww/cmd/shared/{manageuser.go => usermanage.go} (86%) rename politeiawww/cmd/shared/{changepassword.go => userpasswordchange.go} (68%) rename politeiawww/cmd/shared/{resetpassword.go => userpasswordreset.go} (83%) rename politeiawww/cmd/shared/{settotp.go => usertotpset.go} (83%) rename politeiawww/cmd/shared/{verifytotp.go => usertotpverify.go} (82%) rename politeiawww/cmd/shared/{changeusername.go => userusernamechange.go} (61%) diff --git a/politeiawww/cmd/cmswww/cmswww.go b/politeiawww/cmd/cmswww/cmswww.go index 831ce1c49..5dd058db6 100644 --- a/politeiawww/cmd/cmswww/cmswww.go +++ b/politeiawww/cmd/cmswww/cmswww.go @@ -57,59 +57,59 @@ type cmswww struct { Config shared.Config // Commands - ActiveVotes ActiveVotesCmd `command:"activevotes" description:"(user) get the dccs that are being voted on"` - BatchProposals BatchProposalsCmd `command:"batchproposals" description:"(user) retrieve a set of proposals"` - NewComment NewCommentCmd `command:"newcomment" description:"(user) create a new comment"` - CensorComment CensorCommentCmd `command:"censorcomment" description:"(admin) censor a comment"` - ChangePassword shared.ChangePasswordCmd `command:"changepassword" description:"(user) change the password for the logged in user"` - ChangeUsername shared.ChangeUsernameCmd `command:"changeusername" description:"(user) change the username for the logged in user"` - CMSUsers CMSUsersCmd `command:"cmsusers" description:"(user) get a list of cms users"` - DCCComments DCCCommentsCmd `command:"dcccomments" description:"(user) get the comments for a dcc proposal"` - DCCDetails DCCDetailsCmd `command:"dccdetails" description:"(user) get the details of a dcc"` - EditInvoice EditInvoiceCmd `command:"editinvoice" description:"(user) edit a invoice"` - EditUser EditUserCmd `command:"edituser" description:"(user) edit current cms user information"` - GeneratePayouts GeneratePayoutsCmd `command:"generatepayouts" description:"(admin) generate a list of payouts with addresses and amounts to pay"` - GetDCCs GetDCCsCmd `command:"getdccs" description:"(user) get all dccs (optional by status)"` - Help HelpCmd `command:"help" description:" print a detailed help message for a specific command"` - InvoiceComments InvoiceCommentsCmd `command:"invoicecomments" description:"(user) get the comments for a invoice"` - InvoiceExchangeRate InvoiceExchangeRateCmd `command:"invoiceexchangerate" description:"(user) get exchange rate for a given month/year"` - InviteNewUser InviteNewUserCmd `command:"invite" description:"(admin) invite a new user"` - InvoiceDetails InvoiceDetailsCmd `command:"invoicedetails" description:"(public) get the details of a proposal"` - InvoicePayouts InvoicePayoutsCmd `command:"invoicepayouts" description:"(admin) generate paid invoice list for a given date range"` - Invoices InvoicesCmd `command:"invoices" description:"(user) get all invoices (optional with optional parameters)"` - Login shared.LoginCmd `command:"login" description:"(public) login to Politeia"` - Logout shared.LogoutCmd `command:"logout" description:"(public) logout of Politeia"` - CMSManageUser CMSManageUserCmd `command:"cmsmanageuser" description:"(admin) edit certain properties of the specified user"` - ManageUser shared.ManageUserCmd `command:"manageuser" description:"(admin) edit certain properties of the specified user"` - Me shared.MeCmd `command:"me" description:"(user) get user details for the logged in user"` - NewDCC NewDCCCmd `command:"newdcc" description:"(user) creates a new dcc proposal"` - NewDCCComment NewDCCCommentCmd `command:"newdcccomment" description:"(user) creates a new comment on a dcc proposal"` - NewInvoice NewInvoiceCmd `command:"newinvoice" description:"(user) create a new invoice"` - PayInvoices PayInvoicesCmd `command:"payinvoices" description:"(admin) set all approved invoices to paid"` - Policy PolicyCmd `command:"policy" description:"(public) get the server policy"` - ProposalOwner ProposalOwnerCmd `command:"proposalowner" description:"(user) get owners of a proposal"` - ProposalBilling ProposalBillingCmd `command:"proposalbilling" description:"(user) get billing information for a proposal"` - ProposalBillingDetails ProposalBillingDetailsCmd `command:"proposalbillingdetails" description:"(admin) get billing information for a proposal"` - ProposalBillingSummary ProposalBillingSummaryCmd `command:"proposalbillingsummary" description:"(admin) get all approved proposal billing information"` - RegisterUser RegisterUserCmd `command:"register" description:"(public) register an invited user to cms"` - ResetPassword shared.ResetPasswordCmd `command:"resetpassword" description:"(public) reset the password for a user that is not logged in"` - SetDCCStatus SetDCCStatusCmd `command:"setdccstatus" description:"(admin) set the status of a DCC"` - SetInvoiceStatus SetInvoiceStatusCmd `command:"setinvoicestatus" description:"(admin) set the status of an invoice"` - SetTOTP shared.SetTOTPCmd `command:"settotp" description:"(user) set the key for TOTP"` - StartVote StartVoteCmd `command:"startvote" description:"(admin) start the voting period on a dcc"` - SupportOpposeDCC SupportOpposeDCCCmd `command:"supportopposedcc" description:"(user) support or oppose a given DCC"` - TestRun TestRunCmd `command:"testrun" description:" test cmswww routes"` - TokenInventory TokenInventoryCmd `command:"tokeninventory" description:"(user) get the censorship record tokens of all proposals (passthrough)"` - UpdateUserKey shared.UpdateUserKeyCmd `command:"updateuserkey" description:"(user) generate a new identity for the logged in user"` - UserDetails UserDetailsCmd `command:"userdetails" description:"(user) get current cms user details"` - UserInvoices UserInvoicesCmd `command:"userinvoices" description:"(user) get all invoices submitted by a specific user"` - UserSubContractors UserSubContractorsCmd `command:"usersubcontractors" description:"(user) get all users that are linked to the user"` - Users shared.UsersCmd `command:"users" description:"(user) get a list of users"` - Secret shared.SecretCmd `command:"secret" description:"(user) ping politeiawww"` - VerifyTOTP shared.VerifyTOTPCmd `command:"verifytotp" description:"(user) verify the set code for TOTP"` - Version shared.VersionCmd `command:"version" description:"(public) get server info and CSRF token"` - VoteDCC VoteDCCCmd `command:"votedcc" description:"(user) vote for a given DCC during an all contractor vote"` - VoteDetails VoteDetailsCmd `command:"votedetails" description:"(user) get the details for a dcc vote"` + ActiveVotes ActiveVotesCmd `command:"activevotes" description:"(user) get the dccs that are being voted on"` + BatchProposals BatchProposalsCmd `command:"batchproposals" description:"(user) retrieve a set of proposals"` + NewComment NewCommentCmd `command:"newcomment" description:"(user) create a new comment"` + CensorComment CensorCommentCmd `command:"censorcomment" description:"(admin) censor a comment"` + ChangePassword shared.UserPasswordChangeCmd `command:"changepassword" description:"(user) change the password for the logged in user"` + ChangeUsername shared.UserUsernameChangeCmd `command:"changeusername" description:"(user) change the username for the logged in user"` + CMSUsers CMSUsersCmd `command:"cmsusers" description:"(user) get a list of cms users"` + DCCComments DCCCommentsCmd `command:"dcccomments" description:"(user) get the comments for a dcc proposal"` + DCCDetails DCCDetailsCmd `command:"dccdetails" description:"(user) get the details of a dcc"` + EditInvoice EditInvoiceCmd `command:"editinvoice" description:"(user) edit a invoice"` + EditUser EditUserCmd `command:"edituser" description:"(user) edit current cms user information"` + GeneratePayouts GeneratePayoutsCmd `command:"generatepayouts" description:"(admin) generate a list of payouts with addresses and amounts to pay"` + GetDCCs GetDCCsCmd `command:"getdccs" description:"(user) get all dccs (optional by status)"` + Help HelpCmd `command:"help" description:" print a detailed help message for a specific command"` + InvoiceComments InvoiceCommentsCmd `command:"invoicecomments" description:"(user) get the comments for a invoice"` + InvoiceExchangeRate InvoiceExchangeRateCmd `command:"invoiceexchangerate" description:"(user) get exchange rate for a given month/year"` + InviteNewUser InviteNewUserCmd `command:"invite" description:"(admin) invite a new user"` + InvoiceDetails InvoiceDetailsCmd `command:"invoicedetails" description:"(public) get the details of a proposal"` + InvoicePayouts InvoicePayoutsCmd `command:"invoicepayouts" description:"(admin) generate paid invoice list for a given date range"` + Invoices InvoicesCmd `command:"invoices" description:"(user) get all invoices (optional with optional parameters)"` + Login shared.LoginCmd `command:"login" description:"(public) login to Politeia"` + Logout shared.LogoutCmd `command:"logout" description:"(public) logout of Politeia"` + CMSManageUser CMSManageUserCmd `command:"cmsmanageuser" description:"(admin) edit certain properties of the specified user"` + ManageUser shared.UserManageCmd `command:"manageuser" description:"(admin) edit certain properties of the specified user"` + Me shared.MeCmd `command:"me" description:"(user) get user details for the logged in user"` + NewDCC NewDCCCmd `command:"newdcc" description:"(user) creates a new dcc proposal"` + NewDCCComment NewDCCCommentCmd `command:"newdcccomment" description:"(user) creates a new comment on a dcc proposal"` + NewInvoice NewInvoiceCmd `command:"newinvoice" description:"(user) create a new invoice"` + PayInvoices PayInvoicesCmd `command:"payinvoices" description:"(admin) set all approved invoices to paid"` + Policy PolicyCmd `command:"policy" description:"(public) get the server policy"` + ProposalOwner ProposalOwnerCmd `command:"proposalowner" description:"(user) get owners of a proposal"` + ProposalBilling ProposalBillingCmd `command:"proposalbilling" description:"(user) get billing information for a proposal"` + ProposalBillingDetails ProposalBillingDetailsCmd `command:"proposalbillingdetails" description:"(admin) get billing information for a proposal"` + ProposalBillingSummary ProposalBillingSummaryCmd `command:"proposalbillingsummary" description:"(admin) get all approved proposal billing information"` + RegisterUser RegisterUserCmd `command:"register" description:"(public) register an invited user to cms"` + ResetPassword shared.UserPasswordResetCmd `command:"resetpassword" description:"(public) reset the password for a user that is not logged in"` + SetDCCStatus SetDCCStatusCmd `command:"setdccstatus" description:"(admin) set the status of a DCC"` + SetInvoiceStatus SetInvoiceStatusCmd `command:"setinvoicestatus" description:"(admin) set the status of an invoice"` + SetTOTP shared.UserTOTPSetCmd `command:"settotp" description:"(user) set the key for TOTP"` + StartVote StartVoteCmd `command:"startvote" description:"(admin) start the voting period on a dcc"` + SupportOpposeDCC SupportOpposeDCCCmd `command:"supportopposedcc" description:"(user) support or oppose a given DCC"` + TestRun TestRunCmd `command:"testrun" description:" test cmswww routes"` + TokenInventory TokenInventoryCmd `command:"tokeninventory" description:"(user) get the censorship record tokens of all proposals (passthrough)"` + UpdateUserKey shared.UserKeyUpdateCmd `command:"updateuserkey" description:"(user) generate a new identity for the logged in user"` + UserDetails UserDetailsCmd `command:"userdetails" description:"(user) get current cms user details"` + UserInvoices UserInvoicesCmd `command:"userinvoices" description:"(user) get all invoices submitted by a specific user"` + UserSubContractors UserSubContractorsCmd `command:"usersubcontractors" description:"(user) get all users that are linked to the user"` + Users shared.UsersCmd `command:"users" description:"(user) get a list of users"` + Secret shared.SecretCmd `command:"secret" description:"(user) ping politeiawww"` + VerifyTOTP shared.UserTOTPVerifyCmd `command:"verifytotp" description:"(user) verify the set code for TOTP"` + Version shared.VersionCmd `command:"version" description:"(public) get server info and CSRF token"` + VoteDCC VoteDCCCmd `command:"votedcc" description:"(user) vote for a given DCC during an all contractor vote"` + VoteDetails VoteDetailsCmd `command:"votedetails" description:"(user) get the details for a dcc vote"` } // signedMerkleRoot calculates the merkle root of the passed in list of files diff --git a/politeiawww/cmd/cmswww/help.go b/politeiawww/cmd/cmswww/help.go index 67eb3fb25..f2001756e 100644 --- a/politeiawww/cmd/cmswww/help.go +++ b/politeiawww/cmd/cmswww/help.go @@ -32,15 +32,15 @@ func (cmd *HelpCmd) Execute(args []string) error { case "logout": fmt.Printf("%s\n", shared.LogoutHelpMsg) case "changepassword": - fmt.Printf("%s\n", shared.ChangePasswordHelpMsg) + fmt.Printf("%s\n", shared.UserPasswordChangeHelpMsg) case "changeusername": - fmt.Printf("%s\n", shared.ChangeUsernameHelpMsg) + fmt.Printf("%s\n", shared.UserUsernameChangeHelpMsg) case "newcomment": fmt.Printf("%s\n", newCommentHelpMsg) case "censorcomment": fmt.Printf("%s\n", censorCommentHelpMsg) case "manageuser": - fmt.Printf("%s\n", shared.ManageUserHelpMsg) + fmt.Printf("%s\n", shared.UserManageHelpMsg) case "cmsmanageuser": fmt.Printf("%s\n", cmsManageUserHelpMsg) case "version": @@ -48,9 +48,9 @@ func (cmd *HelpCmd) Execute(args []string) error { case "me": fmt.Printf("%s\n", shared.MeHelpMsg) case "resetpassword": - fmt.Printf("%s\n", shared.ResetPasswordHelpMsg) + fmt.Printf("%s\n", shared.UserPasswordResetHelpMsg) case "updateuserkey": - fmt.Printf("%s\n", shared.UpdateUserKeyHelpMsg) + fmt.Printf("%s\n", shared.UserKeyUpdateHelpMsg) case "users": fmt.Printf("%s\n", shared.UsersHelpMsg) case "userdetails": diff --git a/politeiawww/cmd/piwww/commentcensor.go b/politeiawww/cmd/piwww/commentcensor.go index e0857346b..3ec661c2e 100644 --- a/politeiawww/cmd/piwww/commentcensor.go +++ b/politeiawww/cmd/piwww/commentcensor.go @@ -14,8 +14,8 @@ import ( "github.com/decred/politeia/util" ) -// CommentCensorCmd censors a proposal comment. -type CommentCensorCmd struct { +// commentCensorCmd censors a proposal comment. +type commentCensorCmd struct { Args struct { Token string `positional-arg-name:"token"` // Censorship token CommentID string `positional-arg-name:"commentID"` // Comment ID @@ -28,7 +28,7 @@ type CommentCensorCmd struct { } // Execute executes the censor comment command. -func (cmd *CommentCensorCmd) Execute(args []string) error { +func (cmd *commentCensorCmd) Execute(args []string) error { token := cmd.Args.Token commentID := cmd.Args.CommentID reason := cmd.Args.Reason diff --git a/politeiawww/cmd/piwww/commentnew.go b/politeiawww/cmd/piwww/commentnew.go index 44192cfb3..89ad7c681 100644 --- a/politeiawww/cmd/piwww/commentnew.go +++ b/politeiawww/cmd/piwww/commentnew.go @@ -13,8 +13,8 @@ import ( "github.com/decred/politeia/politeiawww/cmd/shared" ) -// CommentNewCmd submits a new proposal comment. -type CommentNewCmd struct { +// commentNewCmd submits a new proposal comment. +type commentNewCmd struct { Args struct { Token string `positional-arg-name:"token" required:"true"` // Censorship token Comment string `positional-arg-name:"comment" required:"true"` // Comment text @@ -27,7 +27,7 @@ type CommentNewCmd struct { } // Execute executes the new comment command. -func (cmd *CommentNewCmd) Execute(args []string) error { +func (cmd *commentNewCmd) Execute(args []string) error { token := cmd.Args.Token comment := cmd.Args.Comment parentID := cmd.Args.ParentID @@ -83,7 +83,7 @@ func (cmd *CommentNewCmd) Execute(args []string) error { return shared.PrintJSON(ncr) } -// commentNewHelpMsg is the output of the help command when 'newcomment' is +// commentNewHelpMsg is the output of the help command when 'commentnew' is // specified. const commentNewHelpMsg = `commentnew "token" "comment" diff --git a/politeiawww/cmd/piwww/comments.go b/politeiawww/cmd/piwww/comments.go index 9d1eed4b8..08e40474c 100644 --- a/politeiawww/cmd/piwww/comments.go +++ b/politeiawww/cmd/piwww/comments.go @@ -11,8 +11,8 @@ import ( "github.com/decred/politeia/politeiawww/cmd/shared" ) -// CommentsCmd retreives the comments for the specified proposal. -type CommentsCmd struct { +// commentsCmd retreives the comments for the specified proposal. +type commentsCmd struct { Args struct { Token string `positional-arg-name:"token"` // Censorship token } `positional-args:"true" required:"true"` @@ -23,7 +23,7 @@ type CommentsCmd struct { } // Execute executes the proposal comments command. -func (cmd *CommentsCmd) Execute(args []string) error { +func (cmd *commentsCmd) Execute(args []string) error { token := cmd.Args.Token // Verify state @@ -50,9 +50,9 @@ func (cmd *CommentsCmd) Execute(args []string) error { return shared.PrintJSON(gcr) } -// proposalCommentsHelpMsg is the output for the help command when -// 'proposalcomments' is specified. -const proposalCommentsHelpMsg = `proposalcomments "token" +// commentsHelpMsg is the output for the help command when 'comments' +// is specified. +const commentsHelpMsg = `comments "token" Get the comments for a proposal. diff --git a/politeiawww/cmd/piwww/commentvote.go b/politeiawww/cmd/piwww/commentvote.go index 996476b07..e7defa852 100644 --- a/politeiawww/cmd/piwww/commentvote.go +++ b/politeiawww/cmd/piwww/commentvote.go @@ -13,9 +13,9 @@ import ( "github.com/decred/politeia/politeiawww/cmd/shared" ) -// CommentVoteCmd is used to upvote/downvote a proposal comment using the +// commentVoteCmd is used to upvote/downvote a proposal comment using the // logged in the user. -type CommentVoteCmd struct { +type commentVoteCmd struct { Args struct { Token string `positional-arg-name:"token"` // Censorship token CommentID string `positional-arg-name:"commentID"` // Comment ID @@ -24,7 +24,7 @@ type CommentVoteCmd struct { } // Execute executes the like comment command. -func (cmd *CommentVoteCmd) Execute(args []string) error { +func (cmd *commentVoteCmd) Execute(args []string) error { const actionUpvote = "upvote" const actionDownvote = "downvote" diff --git a/politeiawww/cmd/piwww/commentvotes.go b/politeiawww/cmd/piwww/commentvotes.go index b395ff7f3..e97a9f52a 100644 --- a/politeiawww/cmd/piwww/commentvotes.go +++ b/politeiawww/cmd/piwww/commentvotes.go @@ -9,9 +9,9 @@ import ( "github.com/decred/politeia/politeiawww/cmd/shared" ) -// CommentVotesCmd retreives like comment objects for +// commentVotesCmd retreives like comment objects for // the specified proposal from the provided user. -type CommentVotesCmd struct { +type commentVotesCmd struct { Args struct { Token string `positional-arg-name:"token"` // Censorship token UserID string `positional-arg-name:"userid"` // User id @@ -19,7 +19,7 @@ type CommentVotesCmd struct { } // Execute executes the user comment likes command. -func (cmd *CommentVotesCmd) Execute(args []string) error { +func (cmd *commentVotesCmd) Execute(args []string) error { token := cmd.Args.Token userID := cmd.Args.UserID diff --git a/politeiawww/cmd/piwww/help.go b/politeiawww/cmd/piwww/help.go index a2a8d2d42..c6d58e256 100644 --- a/politeiawww/cmd/piwww/help.go +++ b/politeiawww/cmd/piwww/help.go @@ -10,65 +10,21 @@ import ( "github.com/decred/politeia/politeiawww/cmd/shared" ) -// HelpCmd prints a detailed help message for the specified command. -type HelpCmd struct { +// helpCmd prints a detailed help message for the specified command. +type helpCmd struct { Args struct { Topic string `positional-arg-name:"topic"` // Topic to print help message for } `positional-args:"true"` } // Execute executes the help command. -func (cmd *HelpCmd) Execute(args []string) error { +func (cmd *helpCmd) Execute(args []string) error { if cmd.Args.Topic == "" { return fmt.Errorf("Specify a command to print a detailed help " + "message for. Example: piwww help login") } switch cmd.Args.Topic { - // Server commands - case "version": - fmt.Printf("%s\n", shared.VersionHelpMsg) - case "policy": - fmt.Printf("%s\n", policyHelpMsg) - - // User commands - case "login": - fmt.Printf("%s\n", shared.LoginHelpMsg) - case "logout": - fmt.Printf("%s\n", shared.LogoutHelpMsg) - case "newuser": - fmt.Printf("%s\n", newUserHelpMsg) - case "changepassword": - fmt.Printf("%s\n", shared.ChangePasswordHelpMsg) - case "changeusername": - fmt.Printf("%s\n", shared.ChangeUsernameHelpMsg) - case "userdetails": - fmt.Printf("%s\n", userDetailsHelpMsg) - case "manageuser": - fmt.Printf("%s\n", shared.ManageUserHelpMsg) - case "users": - fmt.Printf("%s\n", shared.UsersHelpMsg) - case "verifyuseremail": - fmt.Printf("%s\n", verifyUserEmailHelpMsg) - case "edituser": - fmt.Printf("%s\n", editUserHelpMsg) - case "me": - fmt.Printf("%s\n", shared.MeHelpMsg) - case "resetpassword": - fmt.Printf("%s\n", shared.ResetPasswordHelpMsg) - case "updateuserkey": - fmt.Printf("%s\n", shared.UpdateUserKeyHelpMsg) - case "userpendingpayment": - fmt.Printf("%s\n", userPendingPaymentHelpMsg) - case "rescanuserpayments": - fmt.Printf("%s\n", rescanUserPaymentsHelpMsg) - case "proposalpaywall": - fmt.Printf("%s\n", proposalPaywallHelpMsg) - case "verifyuserpayment": - fmt.Printf("%s\n", verifyUserPaymentHelpMsg) - case "resendverification": - fmt.Printf("%s\n", resendVerificationHelpMsg) - // Proposal commands case "proposalnew": fmt.Printf("%s\n", proposalNewHelpMsg) @@ -81,11 +37,6 @@ func (cmd *HelpCmd) Execute(args []string) error { case "proposalinventory": fmt.Printf("%s\n", proposalInventoryHelpMsg) - case "userproposals": - fmt.Printf("%s\n", userProposalsHelpMsg) - case "vettedproposals": - fmt.Printf("%s\n", vettedProposalsHelpMsg) - // Comment commands case "commentnew": fmt.Printf("%s\n", commentNewHelpMsg) @@ -94,7 +45,7 @@ func (cmd *HelpCmd) Execute(args []string) error { case "commentcensor": fmt.Printf("%s\n", commentCensorHelpMsg) case "comments": - fmt.Printf("%s\n", proposalCommentsHelpMsg) + fmt.Printf("%s\n", commentsHelpMsg) case "commentvotes": fmt.Printf("%s\n", commentVotesHelpMsg) @@ -116,18 +67,65 @@ func (cmd *HelpCmd) Execute(args []string) error { case "voteinventory": fmt.Printf("%s\n", voteInventoryHelpMsg) + case "votedetails": + fmt.Printf("%s\n", voteDetailsHelpMsg) case "votestatus": fmt.Printf("%s\n", voteStatusHelpMsg) case "votestatuses": fmt.Printf("%s\n", voteStatusesHelpMsg) - case "votedetails": - fmt.Printf("%s\n", voteDetailsHelpMsg) + + // Server commands + case "version": + fmt.Printf("%s\n", shared.VersionHelpMsg) + case "policy": + fmt.Printf("%s\n", policyHelpMsg) + + // User commands + case "usernew": + fmt.Printf("%s\n", userNewHelpMsg) + case "useredit": + fmt.Printf("%s\n", userEditHelpMsg) + case "userdetails": + fmt.Printf("%s\n", userDetailsHelpMsg) + case "userpaymentsrescan": + fmt.Printf("%s\n", userPaymentsRescanHelpMsg) + case "userpendingpayment": + fmt.Printf("%s\n", userPendingPaymentHelpMsg) + case "useremailverify": + fmt.Printf("%s\n", userEmailVerifyHelpMsg) + case "userpaymentverify": + fmt.Printf("%s\n", userPaymentVerifyHelpMsg) + case "usermanage": + fmt.Printf("%s\n", shared.UserManageHelpMsg) + case "userkeyupdate": + fmt.Printf("%s\n", shared.UserKeyUpdateHelpMsg) + case "userverificationresend": + fmt.Printf("%s\n", userVerificationResendHelpMsg) + case "userusernamechange": + fmt.Printf("%s\n", shared.UserUsernameChangeHelpMsg) + case "userpasswordchange": + fmt.Printf("%s\n", shared.UserPasswordChangeHelpMsg) + case "userpasswordreset": + fmt.Printf("%s\n", shared.UserPasswordResetHelpMsg) + case "users": + fmt.Printf("%s\n", shared.UsersHelpMsg) + + case "proposalpaywall": + fmt.Printf("%s\n", proposalPaywallHelpMsg) + + // Basic commands + case "login": + fmt.Printf("%s\n", shared.LoginHelpMsg) + case "logout": + fmt.Printf("%s\n", shared.LogoutHelpMsg) + case "me": + fmt.Printf("%s\n", shared.MeHelpMsg) // Websocket commands case "subscribe": fmt.Printf("%s\n", subscribeHelpMsg) - // Other commands + // Dev commands case "testrun": fmt.Printf("%s\n", testRunHelpMsg) case "sendfaucettx": diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index ccabb6836..25f48ef55 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -45,62 +45,69 @@ type piwww struct { Config shared.Config // Proposal commands - ProposalNew ProposalNewCmd `command:"proposalnew"` - ProposalEdit ProposalEditCmd `command:"proposaledit"` - ProposalSetStatus ProposalSetStatusCmd `command:"proposalsetstatus"` - Proposals ProposalsCmd `command:"proposals"` - ProposalInventory ProposalInventoryCmd `command:"proposalinventory"` + ProposalNew proposalNewCmd `command:"proposalnew"` + ProposalEdit proposalEditCmd `command:"proposaledit"` + ProposalSetStatus proposalSetStatusCmd `command:"proposalsetstatus"` + Proposals proposalsCmd `command:"proposals"` + ProposalInventory proposalInventoryCmd `command:"proposalinventory"` // Comments commands - CommentNew CommentNewCmd `command:"commentnew" description:"(user) create a new comment"` - CommentVote CommentVoteCmd `command:"commentvote" description:"(user) upvote/downvote a comment"` - CommentCensor CommentCensorCmd `command:"commentcensor" description:"(admin) censor a comment"` - Comments CommentsCmd `command:"comments" description:"(public) get the comments for a proposal"` - CommentVotes CommentVotesCmd `command:"commentvotes" description:"(user) get comment upvotes/downvotes for a proposal from the provided user"` + CommentNew commentNewCmd `command:"commentnew" description:"(user) create a new comment"` + CommentVote commentVoteCmd `command:"commentvote" description:"(user) upvote/downvote a comment"` + CommentCensor commentCensorCmd `command:"commentcensor" description:"(admin) censor a comment"` + Comments commentsCmd `command:"comments" description:"(public) get the comments for a proposal"` + CommentVotes commentVotesCmd `command:"commentvotes" description:"(user) get comment upvotes/downvotes for a proposal from the provided user"` // Vote commands - VoteAuthorize VoteAuthorizeCmd `command:"voteauthorize" description:"(user) authorize a proposal vote (must be proposal author)"` - VoteStart VoteStartCmd `command:"votestart" description:"(admin) start the voting period on a proposal"` - VoteStartRunoff VoteStartRunoffCmd `command:"votestartrunoff" description:"(admin) start a runoff using the submissions to an RFP"` - VoteBallot VoteBallotCmd `command:"voteballot" description:"(public) cast ballot of votes for a proposal"` - Votes VotesCmd `command:"votes" description:"(public) get the vote tally for a proposal"` - VoteResults VoteResultsCmd `command:"voteresults" description:"(public) get vote results for a proposal"` - VoteSummaries VoteSummariesCmd `command:"votesummaries" description:"(public) retrieve the vote summary for a set of proposals"` - VoteInventory VoteInventoryCmd `command:"voteinventory" description:"(public) retrieve the tokens of all public, non-abandoned proposal separated by vote status"` - - // Commands - ChangePassword shared.ChangePasswordCmd `command:"changepassword" description:"(user) change the password for the logged in user"` - ChangeUsername shared.ChangeUsernameCmd `command:"changeusername" description:"(user) change the username for the logged in user"` - EditUser EditUserCmd `command:"edituser" description:"(user) edit the preferences of the logged in user"` - Help HelpCmd `command:"help" description:" print a detailed help message for a specific command"` - Login shared.LoginCmd `command:"login" description:"(public) login to Politeia"` - Logout shared.LogoutCmd `command:"logout" description:"(public) logout of Politeia"` - ManageUser shared.ManageUserCmd `command:"manageuser" description:"(admin) edit certain properties of the specified user"` - Me shared.MeCmd `command:"me" description:"(user) get user details for the logged in user"` - NewUser NewUserCmd `command:"newuser" description:"(public) create a new user"` - Policy PolicyCmd `command:"policy" description:"(public) get the server policy"` - ProposalPaywall ProposalPaywallCmd `command:"proposalpaywall" description:"(user) get proposal paywall details for the logged in user"` - RescanUserPayments RescanUserPaymentsCmd `command:"rescanuserpayments" description:"(admin) rescan a user's payments to check for missed payments"` - ResendVerification ResendVerificationCmd `command:"resendverification" description:"(public) resend the user verification email"` - ResetPassword shared.ResetPasswordCmd `command:"resetpassword" description:"(public) reset the password for a user that is not logged in"` - Secret shared.SecretCmd `command:"secret" description:"(user) ping politeiawww"` - SendFaucetTx SendFaucetTxCmd `command:"sendfaucettx" description:" send a DCR transaction using the Decred testnet faucet"` - SetTOTP shared.SetTOTPCmd `command:"settotp" description:"(user) set the key for TOTP"` - Subscribe SubscribeCmd `command:"subscribe" description:"(public) subscribe to all websocket commands and do not exit tool"` - TestRun TestRunCmd `command:"testrun" description:" run a series of tests on the politeiawww routes (dev use only)"` - UpdateUserKey shared.UpdateUserKeyCmd `command:"updateuserkey" description:"(user) generate a new identity for the logged in user"` - UserDetails UserDetailsCmd `command:"userdetails" description:"(public) get the details of a user profile"` - UserPendingPayment UserPendingPaymentCmd `command:"userpendingpayment" description:"(user) get details for a pending payment for the logged in user"` - UserProposals UserProposalsCmd `command:"userproposals" description:"(public) get all proposals submitted by a specific user"` - Users shared.UsersCmd `command:"users" description:"(public) get a list of users"` - VerifyUserEmail VerifyUserEmailCmd `command:"verifyuseremail" description:"(public) verify a user's email address"` - VerifyUserPayment VerifyUserPaymentCmd `command:"verifyuserpayment" description:"(user) check if the logged in user has paid their user registration fee"` - VerifyTOTP shared.VerifyTOTPCmd `command:"verifytotp" description:"(user) verify the set code for TOTP"` - Version shared.VersionCmd `command:"version" description:"(public) get server info and CSRF token"` - VettedProposals VettedProposalsCmd `command:"vettedproposals" description:"(public) get a page of vetted proposals"` - VoteDetails VoteDetailsCmd `command:"votedetails" description:"(public) get the details for a proposal vote"` - VoteStatus VoteStatusCmd `command:"votestatus" description:"(public) get the vote status of a proposal"` - VoteStatuses VoteStatusesCmd `command:"votestatuses" description:"(public) get the vote status for all public proposals"` + VoteAuthorize voteAuthorizeCmd `command:"voteauthorize" description:"(user) authorize a proposal vote (must be proposal author)"` + VoteStart voteStartCmd `command:"votestart" description:"(admin) start the voting period on a proposal"` + VoteStartRunoff voteStartRunoffCmd `command:"votestartrunoff" description:"(admin) start a runoff using the submissions to an RFP"` + VoteBallot voteBallotCmd `command:"voteballot" description:"(public) cast ballot of votes for a proposal"` + Votes votesCmd `command:"votes" description:"(public) get the vote tally for a proposal"` + VoteResults voteResultsCmd `command:"voteresults" description:"(public) get vote results for a proposal"` + VoteSummaries voteSummariesCmd `command:"votesummaries" description:"(public) retrieve the vote summary for a set of proposals"` + VoteInventory voteInventoryCmd `command:"voteinventory" description:"(public) retrieve the tokens of all public, non-abandoned proposal separated by vote status"` + // XXX www vote routes + VoteDetails voteDetailsCmd `command:"votedetails" description:"(public) get the details for a proposal vote"` + VoteStatus voteStatusCmd `command:"votestatus" description:"(public) get the vote status of a proposal"` + VoteStatuses voteStatusesCmd `command:"votestatuses" description:"(public) get the vote status for all public proposals"` + + // User commands + UserNew userNewCmd `command:"usernew" description:"(public) create a new user"` + UserEdit userEditCmd `command:"useredit" description:"(user) edit the preferences of the logged in user"` + UserDetails userDetailsCmd `command:"userdetails" description:"(public) get the details of a user profile"` + UserPaymentsRescan userPaymentsRescanCmd `command:"userpaymentsrescan" description:"(admin) rescan a user's payments to check for missed payments"` + UserPendingPayment userPendingPaymentCmd `command:"userpendingpayment" description:"(user) get details for a pending payment for the logged in user"` + UserEmailVerify userEmailVerifyCmd `command:"useremailverify" description:"(public) verify a user's email address"` + UserPaymentVerify userPaymentVerifyCmd `command:"userpaymentverify" description:"(user) check if the logged in user has paid their user registration fee"` + UserVerificationResend userVerificationResendCmd `command:"userverificationresend" description:"(public) resend the user verification email"` + UserManage shared.UserManageCmd `command:"usermanage" description:"(admin) edit certain properties of the specified user"` + UserKeyUpdate shared.UserKeyUpdateCmd `command:"userkeyupdate" description:"(user) generate a new identity for the logged in user"` + UserUsernameChange shared.UserUsernameChangeCmd `command:"userusernamechange" description:"(user) change the username for the logged in user"` + UserPasswordChange shared.UserPasswordChangeCmd `command:"userpasswordchange" description:"(user) change the password for the logged in user"` + UserPasswordReset shared.UserPasswordResetCmd `command:"userpasswordreset" description:"(public) reset the password for a user that is not logged in"` + UserTOTPSet shared.UserTOTPSetCmd `command:"usertotpset" description:"(user) set the key for TOTP"` + UserTOTPVerify shared.UserTOTPVerifyCmd `command:"usertotpverify" description:"(user) verify the set code for TOTP"` + Users shared.UsersCmd `command:"users" description:"(public) get a list of users"` + + // XXX will be factored to a user route + ProposalPaywall proposalPaywallCmd `command:"proposalpaywall" description:"(user) get proposal paywall details for the logged in user"` + + // Basic commands + Login shared.LoginCmd `command:"login" description:"(public) login to Politeia"` + Logout shared.LogoutCmd `command:"logout" description:"(public) logout of Politeia"` + Me shared.MeCmd `command:"me" description:"(user) get user details for the logged in user"` + Secret shared.SecretCmd `command:"secret" description:"(user) ping politeiawww"` + Version shared.VersionCmd `command:"version" description:"(public) get server info and CSRF token"` + Policy policyCmd `command:"policy" description:"(public) get the server policy"` + Help helpCmd `command:"help" description:" print a detailed help message for a specific command"` + + // Websocket commands + Subscribe subscribeCmd `command:"subscribe" description:"(public) subscribe to all websocket commands and do not exit tool"` + + // Dev commands + TestRun testRunCmd `command:"testrun" description:"(dev) run a series of tests on the politeiawww routes"` + SendFaucetTx sendFaucetTxCmd `command:"sendfaucettx" description:"(dev) send a DCR transaction using the Decred testnet faucet"` } // signedMerkleRoot calculates the merkle root of the passed in list of files diff --git a/politeiawww/cmd/piwww/policy.go b/politeiawww/cmd/piwww/policy.go index 77880ad57..be8e5380c 100644 --- a/politeiawww/cmd/piwww/policy.go +++ b/politeiawww/cmd/piwww/policy.go @@ -6,11 +6,11 @@ package main import "github.com/decred/politeia/politeiawww/cmd/shared" -// PolicyCmd gets the server policy information. -type PolicyCmd struct{} +// policyCmd gets the server policy information. +type policyCmd struct{} // Execute executes the policy command. -func (cmd *PolicyCmd) Execute(args []string) error { +func (cmd *policyCmd) Execute(args []string) error { pr, err := client.Policy() if err != nil { return err @@ -24,26 +24,4 @@ const policyHelpMsg = `policy Fetch server policy. Arguments: -None - -Response: -{ - "minpasswordlength" (uint) Minimum password length - "minusernamelength" (uint) Minimum username length - "maxusernamelength" (uint) Maximum username length - "usernamesupportedchars" ([]string) List of unsupported characters - "proposallistpagesize" (uint) Maximum proposals per page - "userlistpagesize" (uint) Maximum users per page - "maximages" (uint) Maximum number of proposal images - "maximagesize" (uint) Maximum image file size (in bytes) - "maxmds" (uint) Maximum number of markdown files - "maxmdsize" (uint) Maximum markdown file size (bytes) - "validmimetypes" ([]string) List of acceptable MIME types - "minproposalnamelength" (uint) Minimum length of a proposal name - "maxproposalnamelength" (uint) Maximum length of a proposal name - "proposalnamesupportedchars" ([]string) Regex of a valid proposal name - "maxcommentlength" (uint) Maximum characters in comments - "backendpublickey" (string) Backend public key - "minvoteduration" (uint) Minimum vote duration - "maxvoteduration" (uint) Maximum vote duration -}` +None` diff --git a/politeiawww/cmd/piwww/proposaledit.go b/politeiawww/cmd/piwww/proposaledit.go index 13c04aa1a..814ecd940 100644 --- a/politeiawww/cmd/piwww/proposaledit.go +++ b/politeiawww/cmd/piwww/proposaledit.go @@ -21,8 +21,8 @@ import ( "github.com/decred/politeia/util" ) -// ProposalEditCmd edits an existing proposal. -type ProposalEditCmd struct { +// proposalEditCmd edits an existing proposal. +type proposalEditCmd struct { Args struct { Token string `positional-arg-name:"token" required:"true"` IndexFile string `positional-arg-name:"indexfile"` @@ -53,7 +53,7 @@ type ProposalEditCmd struct { } // Execute executes the proposal edit command. -func (cmd *ProposalEditCmd) Execute(args []string) error { +func (cmd *proposalEditCmd) Execute(args []string) error { token := cmd.Args.Token indexFile := cmd.Args.IndexFile attachments := cmd.Args.Attachments diff --git a/politeiawww/cmd/piwww/proposalinventory.go b/politeiawww/cmd/piwww/proposalinventory.go index 77a8ef83d..69ae8df0c 100644 --- a/politeiawww/cmd/piwww/proposalinventory.go +++ b/politeiawww/cmd/piwww/proposalinventory.go @@ -8,12 +8,12 @@ import ( "github.com/decred/politeia/politeiawww/cmd/shared" ) -// ProposalInventoryCmd retrieves the censorship record tokens of all proposals in +// proposalInventoryCmd retrieves the censorship record tokens of all proposals in // the inventory. -type ProposalInventoryCmd struct{} +type proposalInventoryCmd struct{} // Execute executes the proposal inventory command. -func (cmd *ProposalInventoryCmd) Execute(args []string) error { +func (cmd *proposalInventoryCmd) Execute(args []string) error { reply, err := client.ProposalInventory() if err != nil { return err diff --git a/politeiawww/cmd/piwww/proposalnew.go b/politeiawww/cmd/piwww/proposalnew.go index be55041e6..d3a14391a 100644 --- a/politeiawww/cmd/piwww/proposalnew.go +++ b/politeiawww/cmd/piwww/proposalnew.go @@ -23,8 +23,8 @@ import ( // TODO replace www policies with pi policies -// ProposalNewCmd submits a new proposal. -type ProposalNewCmd struct { +// proposalNewCmd submits a new proposal. +type proposalNewCmd struct { Args struct { IndexFile string `positional-arg-name:"indexfile"` Attachments []string `positional-arg-name:"attachments"` @@ -47,7 +47,7 @@ type ProposalNewCmd struct { } // Execute executes the new proposal command. -func (cmd *ProposalNewCmd) Execute(args []string) error { +func (cmd *proposalNewCmd) Execute(args []string) error { indexFile := cmd.Args.IndexFile attachments := cmd.Args.Attachments diff --git a/politeiawww/cmd/piwww/proposalpaywall.go b/politeiawww/cmd/piwww/proposalpaywall.go index f212b96be..0133f9d87 100644 --- a/politeiawww/cmd/piwww/proposalpaywall.go +++ b/politeiawww/cmd/piwww/proposalpaywall.go @@ -6,11 +6,11 @@ package main import "github.com/decred/politeia/politeiawww/cmd/shared" -// ProposalPaywallCmd gets paywall info for the logged in user. -type ProposalPaywallCmd struct{} +// proposalPaywallCmd gets paywall info for the logged in user. +type proposalPaywallCmd struct{} // Execute executes the proposal paywall command. -func (cmd *ProposalPaywallCmd) Execute(args []string) error { +func (cmd *proposalPaywallCmd) Execute(args []string) error { ppdr, err := client.ProposalPaywallDetails() if err != nil { return err @@ -21,11 +21,7 @@ func (cmd *ProposalPaywallCmd) Execute(args []string) error { // proposalPaywallHelpMsg is the output of the help command when // 'proposalpaywall' is specified. const proposalPaywallHelpMsg = `proposalpaywall + Fetch proposal paywall details. -Arguments: None -Response: -{ - "creditprice" (uint64) Price per proposal credit in atoms - "paywalladdress" (string) Proposal paywall address - "paywalltxnotbefore" (string) Minimum timestamp for paywall tx -}` + +Arguments: None` diff --git a/politeiawww/cmd/piwww/proposals.go b/politeiawww/cmd/piwww/proposals.go index d0f141c73..45637eef1 100644 --- a/politeiawww/cmd/piwww/proposals.go +++ b/politeiawww/cmd/piwww/proposals.go @@ -12,8 +12,8 @@ import ( "github.com/decred/politeia/politeiawww/cmd/shared" ) -// ProposalsCmd retrieves the proposal records of the requested tokens and versions. -type ProposalsCmd struct { +// proposalsCmd retrieves the proposal records of the requested tokens and versions. +type proposalsCmd struct { Args struct { Proposals []string `positional-arg-name:"proposals" required:"true"` } `positional-args:"true" optional:"true"` @@ -26,7 +26,7 @@ type ProposalsCmd struct { } // Execute executes the proposals command. -func (cmd *ProposalsCmd) Execute(args []string) error { +func (cmd *proposalsCmd) Execute(args []string) error { proposals := cmd.Args.Proposals // Set state to get unvetted or vetted proposals. Defaults diff --git a/politeiawww/cmd/piwww/proposalsetstatus.go b/politeiawww/cmd/piwww/proposalsetstatus.go index a8409b2a9..45ce4f7fa 100644 --- a/politeiawww/cmd/piwww/proposalsetstatus.go +++ b/politeiawww/cmd/piwww/proposalsetstatus.go @@ -13,8 +13,8 @@ import ( "github.com/decred/politeia/politeiawww/cmd/shared" ) -// ProposalSetStatusCmd sets the status of a proposal. -type ProposalSetStatusCmd struct { +// proposalSetStatusCmd sets the status of a proposal. +type proposalSetStatusCmd struct { Args struct { Token string `positional-arg-name:"token" required:"true"` Status string `positional-arg-name:"status" required:"true"` @@ -25,7 +25,7 @@ type ProposalSetStatusCmd struct { } // Execute executes the set proposal status command. -func (cmd *ProposalSetStatusCmd) Execute(args []string) error { +func (cmd *proposalSetStatusCmd) Execute(args []string) error { propStatus := map[string]pi.PropStatusT{ "public": pi.PropStatusPublic, "censored": pi.PropStatusCensored, diff --git a/politeiawww/cmd/piwww/sendfaucettx.go b/politeiawww/cmd/piwww/sendfaucettx.go index 1c62de8fe..1b3b09c35 100644 --- a/politeiawww/cmd/piwww/sendfaucettx.go +++ b/politeiawww/cmd/piwww/sendfaucettx.go @@ -10,9 +10,9 @@ import ( "github.com/decred/politeia/util" ) -// SendFaucetTxCmd uses the Decred testnet faucet to send the specified amount +// sendFaucetTxCmd uses the Decred testnet faucet to send the specified amount // of DCR (in atoms) to the specified address. -type SendFaucetTxCmd struct { +type sendFaucetTxCmd struct { Args struct { Address string `positional-arg-name:"address" required:"true"` // DCR address Amount uint64 `positional-arg-name:"amount" required:"true"` // Amount in atoms @@ -21,7 +21,7 @@ type SendFaucetTxCmd struct { } // Execute executes the send faucet tx command. -func (cmd *SendFaucetTxCmd) Execute(args []string) error { +func (cmd *sendFaucetTxCmd) Execute(args []string) error { address := cmd.Args.Address atoms := cmd.Args.Amount dcr := float64(atoms) / 1e8 diff --git a/politeiawww/cmd/piwww/subscribe.go b/politeiawww/cmd/piwww/subscribe.go index 28536ce1f..176cc8fcd 100644 --- a/politeiawww/cmd/piwww/subscribe.go +++ b/politeiawww/cmd/piwww/subscribe.go @@ -20,13 +20,13 @@ import ( "golang.org/x/net/publicsuffix" ) -// SubscribeCmd opens a websocket connect to politeiawww. -type SubscribeCmd struct { +// subscribeCmd opens a websocket connect to politeiawww. +type subscribeCmd struct { Close bool `long:"close" optional:"true"` // Do not keep connetion alive } // Execute executes the subscribe command. -func (cmd *SubscribeCmd) Execute(args []string) error { +func (cmd *subscribeCmd) Execute(args []string) error { // Parse args route := v1.RouteUnauthenticatedWebSocket subscribe := make([]string, 0, len(args)) @@ -119,9 +119,4 @@ Flags: --close (bool, optional) Do not keep the websocket connection alive Supported commands: - - ping (does not require authentication) - -Request: -{ -} -` + - ping (does not require authentication)` diff --git a/politeiawww/cmd/piwww/testrun.go b/politeiawww/cmd/piwww/testrun.go index 7bc3291a2..1c2882486 100644 --- a/politeiawww/cmd/piwww/testrun.go +++ b/politeiawww/cmd/piwww/testrun.go @@ -9,8 +9,8 @@ import ( "github.com/decred/politeia/politeiawww/cmd/shared" ) -// TestRunCmd performs a test run of all the politeiawww routes. -type TestRunCmd struct { +// testRunCmd performs a test run of all the politeiawww routes. +type testRunCmd struct { Args struct { AdminEmail string `positional-arg-name:"adminemail"` AdminPassword string `positional-arg-name:"adminpassword"` @@ -130,7 +130,7 @@ func castVotes(token string, voteID string) (bool, error) { } // Execute executes the test run command. -func (cmd *TestRunCmd) Execute(args []string) error { +func (cmd *testRunCmd) Execute(args []string) error { /* const ( // sleepInterval is the time to wait in between requests diff --git a/politeiawww/cmd/piwww/userdetails.go b/politeiawww/cmd/piwww/userdetails.go index 68d6ac24e..93231b014 100644 --- a/politeiawww/cmd/piwww/userdetails.go +++ b/politeiawww/cmd/piwww/userdetails.go @@ -6,15 +6,15 @@ package main import "github.com/decred/politeia/politeiawww/cmd/shared" -// UserDetailsCmd gets the user details for the specified user. -type UserDetailsCmd struct { +// userDetailsCmd gets the user details for the specified user. +type userDetailsCmd struct { Args struct { UserID string `positional-arg-name:"userid"` // User ID } `positional-args:"true" required:"true"` } // Execute executes the user details command. -func (cmd *UserDetailsCmd) Execute(args []string) error { +func (cmd *userDetailsCmd) Execute(args []string) error { udr, err := client.UserDetails(cmd.Args.UserID) if err != nil { return err @@ -29,37 +29,4 @@ const userDetailsHelpMsg = `userdetails "userid" Fetch user details by user id. Arguments: -1. userid (string, required) User id - -Result: -{ - "user": { - "id": (uuid.UUID) Unique user uuid - "email": (string) Email address + lookup key - "username": (string) Unique username - "isadmin": (bool) Is user an admin - "newuserpaywalladdress": (string) Address for paywall payment - "newuserpaywallamount": (uint64) Paywall amount - "newuserpaywalltx": (string) Paywall transaction id - "newuserpaywalltxnotbefore": (int64) Txs before this time are not valid - "newuserpaywallpollexpiry": (int64) Time to stop polling paywall address - "newuserverificationtoken": ([]byte) Registration verification token - "newuserverificationexpiry": (int64) Registration verification expiration - "updatekeyverificationtoken": ([]byte) Keypair update verification token - "updatekeyverificationexpiry": (int64) Verification expiration - "resetpasswordverificationtoken": ([]byte) Reset password token - "resetpasswordverificationexpiry": (int64) Reset password token expiration - "lastlogintime": (int64) Unix timestamp of last user login - "failedloginattempts": (uint64) Number of sequential failed login attempts - "isdeactivated": (bool) Whether the account is deactivated or not - "islocked": (bool) Whether the account is locked or not - "identities": [ - { - "pubkey": (string) User's public key - "isactive": (bool) Whether user's identity is active or not - } - ], - "proposalcredits": (uint64) Number of available proposal credits - "emailnotifications": (uint64) Whether to notify via emails - } -}` +1. userid (string, required) User id` diff --git a/politeiawww/cmd/piwww/edituser.go b/politeiawww/cmd/piwww/useredit.go similarity index 88% rename from politeiawww/cmd/piwww/edituser.go rename to politeiawww/cmd/piwww/useredit.go index 1b343bf5a..8c04d04bb 100644 --- a/politeiawww/cmd/piwww/edituser.go +++ b/politeiawww/cmd/piwww/useredit.go @@ -9,19 +9,19 @@ import ( "strconv" "strings" - "github.com/decred/politeia/politeiawww/api/www/v1" + v1 "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/cmd/shared" ) -// EditUserCmd edits the preferences of the logged in user. -type EditUserCmd struct { +// userEditCmd edits the preferences of the logged in user. +type userEditCmd struct { Args struct { NotifType string `long:"emailnotifications"` // Email notification bit field } `positional-args:"true" required:"true"` } // Execute executes the edit user command. -func (cmd *EditUserCmd) Execute(args []string) error { +func (cmd *userEditCmd) Execute(args []string) error { emailNotifs := map[string]v1.EmailNotificationT{ "userproposalchange": v1.NotificationEmailMyProposalStatusChange, "userproposalvotingstarted": v1.NotificationEmailMyProposalVoteStarted, @@ -83,9 +83,9 @@ func (cmd *EditUserCmd) Execute(args []string) error { return shared.PrintJSON(eur) } -// editUserHelpMsg is the output of the help command when 'edituser' is +// userEditHelpMsg is the output of the help command when 'edituser' is // specified. -const editUserHelpMsg = `edituser "emailnotifications" +const userEditHelpMsg = `useredit "emailnotifications" Edit user settings for the logged in user. @@ -102,12 +102,4 @@ Valid options are: 32. newproposal Notify when proposal is submitted (admin only) 64. userauthorizedvote Notify when user authorizes vote (admin only) 128. commentonproposal Notify when comment is made on my proposal -256. commentoncomment Notify when comment is made on my comment - -Request: -{ - "emailnotifications": (uint64) Bit field -} - -Response: -{}` +256. commentoncomment Notify when comment is made on my comment` diff --git a/politeiawww/cmd/piwww/verifyuseremail.go b/politeiawww/cmd/piwww/useremailverify.go similarity index 71% rename from politeiawww/cmd/piwww/verifyuseremail.go rename to politeiawww/cmd/piwww/useremailverify.go index 8fbee0055..566714bdc 100644 --- a/politeiawww/cmd/piwww/verifyuseremail.go +++ b/politeiawww/cmd/piwww/useremailverify.go @@ -7,12 +7,12 @@ package main import ( "encoding/hex" - "github.com/decred/politeia/politeiawww/api/www/v1" + v1 "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/cmd/shared" ) -// VerifyUserEmailCmd is used to verify a user's email address. -type VerifyUserEmailCmd struct { +// userEmailVerifyCmd is used to verify a user's email address. +type userEmailVerifyCmd struct { Args struct { Username string `positional-arg-name:"username"` // Username Email string `positional-arg-name:"email"` // User email address @@ -21,7 +21,7 @@ type VerifyUserEmailCmd struct { } // Execute executes the verify user email command. -func (cmd *VerifyUserEmailCmd) Execute(args []string) error { +func (cmd *userEmailVerifyCmd) Execute(args []string) error { // Load user identity id, err := cfg.LoadIdentity(cmd.Args.Username) if err != nil { @@ -44,15 +44,12 @@ func (cmd *VerifyUserEmailCmd) Execute(args []string) error { return shared.PrintJSON(vnur) } -// verifyUserEmailHelpMsg is the output for the help command when -// 'verifyuseremail' is specified. -var verifyUserEmailHelpMsg = `verifyuseremail "email" "token" +// userEmailVerifyHelpMsg is the output for the help command when +// 'useremailverify' is specified. +var userEmailVerifyHelpMsg = `useremailverify "email" "token" Verify user's email address. Arguments: 1. email (string, optional) Email of user -2. token (string, optional) Verification token - -Result: -{}` +2. token (string, optional) Verification token` diff --git a/politeiawww/cmd/piwww/newuser.go b/politeiawww/cmd/piwww/usernew.go similarity index 87% rename from politeiawww/cmd/piwww/newuser.go rename to politeiawww/cmd/piwww/usernew.go index 71219bae7..24aa361e4 100644 --- a/politeiawww/cmd/piwww/newuser.go +++ b/politeiawww/cmd/piwww/usernew.go @@ -8,13 +8,13 @@ import ( "encoding/hex" "fmt" - "github.com/decred/politeia/politeiawww/api/www/v1" + v1 "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/cmd/shared" "github.com/decred/politeia/util" ) -// NewUserCmd creates a new politeia user. -type NewUserCmd struct { +// userNewCmd creates a new politeia user. +type userNewCmd struct { Args struct { Email string `positional-arg-name:"email"` // Email address Username string `positional-arg-name:"username"` // Username @@ -27,7 +27,7 @@ type NewUserCmd struct { } // Execute executes the new user command. -func (cmd *NewUserCmd) Execute(args []string) error { +func (cmd *userNewCmd) Execute(args []string) error { email := cmd.Args.Email username := cmd.Args.Username password := cmd.Args.Password @@ -138,7 +138,7 @@ func (cmd *NewUserCmd) Execute(args []string) error { // Pays paywall fee using faucet if cmd.Paywall { - faucet := SendFaucetTxCmd{} + faucet := sendFaucetTxCmd{} faucet.Args.Address = lr.PaywallAddress faucet.Args.Amount = lr.PaywallAmount err = faucet.Execute(nil) @@ -150,9 +150,9 @@ func (cmd *NewUserCmd) Execute(args []string) error { return nil } -// newUserHelpMsg is the output of the help command when 'newuser' is +// userNewHelpMsg is the output of the help command when 'usernew' is // specified. -const newUserHelpMsg = `newuser [flags] "email" "username" "password" +const userNewHelpMsg = `usernew [flags] "email" "username" "password" Create a new Politeia user. Users can be created by supplying all the arguments below, or supplying the --random flag. If --random is used, Politeia will @@ -167,17 +167,4 @@ Flags: --random (bool, optional) Generate a random email/password for the user --paywall (bool, optional) Satisfy the paywall fee using testnet faucet --verify (bool, optional) Verify the user's email address - --nosave (bool, optional) Do not save the user identity to disk - -Request: -{ - "email": (string) User email - "password": (string) Password - "publickey": (string) Active public key - "username": (string) Username -} - -Response: -{ - "verificationtoken": (string) Server verification token -}` + --nosave (bool, optional) Do not save the user identity to disk` diff --git a/politeiawww/cmd/piwww/rescanuserpayments.go b/politeiawww/cmd/piwww/userpaymentsrescan.go similarity index 64% rename from politeiawww/cmd/piwww/rescanuserpayments.go rename to politeiawww/cmd/piwww/userpaymentsrescan.go index 0321fb4ac..3237746e6 100644 --- a/politeiawww/cmd/piwww/rescanuserpayments.go +++ b/politeiawww/cmd/piwww/userpaymentsrescan.go @@ -5,20 +5,20 @@ package main import ( - "github.com/decred/politeia/politeiawww/api/www/v1" + v1 "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/cmd/shared" ) -// RescanUserPaymentsCmd rescans the logged in user's paywall address and +// userPaymentsRescanCmd rescans the logged in user's paywall address and // makes sure that all payments have been credited to the user's account. -type RescanUserPaymentsCmd struct { +type userPaymentsRescanCmd struct { Args struct { UserID string `positional-arg-name:"userid"` // User ID } `positional-args:"true" required:"true"` } // Execute executes the rescan user payments command. -func (cmd *RescanUserPaymentsCmd) Execute(args []string) error { +func (cmd *userPaymentsRescanCmd) Execute(args []string) error { upr := &v1.UserPaymentsRescan{ UserID: cmd.Args.UserID, } @@ -36,16 +36,11 @@ func (cmd *RescanUserPaymentsCmd) Execute(args []string) error { return shared.PrintJSON(uprr) } -// rescanUserPaymentsHelpMsg is the output of the help command when +// userPaymentsRescanHelpMsg is the output of the help command when // 'rescanuserpayments' is specified. -var rescanUserPaymentsHelpMsg = `rescanuserpayments +var userPaymentsRescanHelpMsg = `userpaymentsrescan Rescan user payments to check for missed payments. Arguments: -1. userid (string, required) User id - -Result: -{ - "newcredits" ([]uint64) Credits that were created by the rescan -}` +1. userid (string, required) User id` diff --git a/politeiawww/cmd/piwww/userpaymentverify.go b/politeiawww/cmd/piwww/userpaymentverify.go new file mode 100644 index 000000000..a006d32a8 --- /dev/null +++ b/politeiawww/cmd/piwww/userpaymentverify.go @@ -0,0 +1,28 @@ +// Copyright (c) 2017-2019 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import "github.com/decred/politeia/politeiawww/cmd/shared" + +// userPaymentVerifyCmd checks on the status of the logged in user's +// registration payment. +type userPaymentVerifyCmd struct{} + +// Execute executes the verify user payment command. +func (cmd *userPaymentVerifyCmd) Execute(args []string) error { + vupr, err := client.VerifyUserPayment() + if err != nil { + return err + } + return shared.PrintJSON(vupr) +} + +// userPaymentVerifyHelpMsg is the output of the help command when +// 'userpaymentverify' is specified. +var userPaymentVerifyHelpMsg = `userpaymentverify + +Check if the currently logged in user has paid their user registration fee. + +Arguments: None` diff --git a/politeiawww/cmd/piwww/userpendingpayment.go b/politeiawww/cmd/piwww/userpendingpayment.go index 917068752..4b319f3ef 100644 --- a/politeiawww/cmd/piwww/userpendingpayment.go +++ b/politeiawww/cmd/piwww/userpendingpayment.go @@ -6,12 +6,12 @@ package main import "github.com/decred/politeia/politeiawww/cmd/shared" -// UserPendingPaymentCmd sretrieve the payment details for a pending payment, +// userPendingPaymentCmd sretrieve the payment details for a pending payment, // if one exists, for the logged in user. -type UserPendingPaymentCmd struct{} +type userPendingPaymentCmd struct{} // Execute executes the user pending payment command. -func (cmd *UserPendingPaymentCmd) Execute(args []string) error { +func (cmd *userPendingPaymentCmd) Execute(args []string) error { pppr, err := client.ProposalPaywallPayment() if err != nil { return err @@ -25,11 +25,4 @@ const userPendingPaymentHelpMsg = `userpendingpayment Get pending payment details for the logged in user. -Arguments: None - -Response: -{ - "txid" (string) Transaction id - "amount" (uint64) Amount sent to paywall address in atoms - "confirmations" (uint64) Number of confirmations of payment tx -}` +Arguments: None` diff --git a/politeiawww/cmd/piwww/userproposals.go b/politeiawww/cmd/piwww/userproposals.go deleted file mode 100644 index f273afb9a..000000000 --- a/politeiawww/cmd/piwww/userproposals.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" - - v1 "github.com/decred/politeia/politeiawww/api/www/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" -) - -// UserProposalsCmd gets the proposals for the specified user. -type UserProposalsCmd struct { - Args struct { - UserID string `positional-arg-name:"userID"` // User ID - } `positional-args:"true" required:"true"` -} - -// Execute executes the user proposals command. -func (cmd *UserProposalsCmd) Execute(args []string) error { - // Get server public key - vr, err := client.Version() - if err != nil { - return err - } - - // Get user proposals - upr, err := client.UserProposals( - &v1.UserProposals{ - UserId: cmd.Args.UserID, - }) - if err != nil { - return err - } - - // Verify proposal censorship records - for _, p := range upr.Proposals { - err := shared.VerifyProposal(p, vr.PubKey) - if err != nil { - return fmt.Errorf("unable to verify proposal %v: %v", - p.CensorshipRecord.Token, err) - } - } - - // Print user proposals - return shared.PrintJSON(upr) -} - -// userProposalsHelpMsg is the output of the help command when 'userproposals' -// is specified. -const userProposalsHelpMsg = `userproposals "userID" - -Fetch all proposals submitted by a specific user. - -Arguments: -1. userID (string, required) User id - -Result: -{ - "proposals": [ - { - "name": (string) Suggested short proposal name - "state": (PropStateT) Current state of proposal - "status": (PropStatusT) Current status of proposal - "timestamp": (int64) Timestamp of last update of proposal - "userid": (string) ID of user who submitted proposal - "username": (string) Username of user who submitted proposal - "publickey": (string) Public key used to sign proposal - "signature": (string) Signature of merkle root - "files": [ - { - "name": (string) Filename - "mime": (string) Mime type - "digest": (string) File digest - "payload": (string) File payload - } - ], - "numcomments": (uint) Number of comments on the proposal - "version": (string) Version of proposal - "censorshiprecord": { - "token": (string) Censorship token - "merkle": (string) Merkle root of proposal - "signature": (string) Server side signature of []byte(Merkle+Token) - } - } - ], - "numofproposals": (int) Number of proposals submitted by user -}` diff --git a/politeiawww/cmd/piwww/resendverification.go b/politeiawww/cmd/piwww/userverificationresend.go similarity index 71% rename from politeiawww/cmd/piwww/resendverification.go rename to politeiawww/cmd/piwww/userverificationresend.go index 0b29e471e..42f83390f 100644 --- a/politeiawww/cmd/piwww/resendverification.go +++ b/politeiawww/cmd/piwww/userverificationresend.go @@ -5,13 +5,13 @@ package main import ( - "github.com/decred/politeia/politeiawww/api/www/v1" + v1 "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/cmd/shared" ) -// ResendVerificationCmd re-sends the user verification email for an unverified +// userVerificationResendCmd re-sends the user verification email for an unverified // user. -type ResendVerificationCmd struct { +type userVerificationResendCmd struct { Args struct { Email string `positional-arg-name:"email"` // User email PublicKey string `positional-arg-name:"publickey"` // User public key @@ -19,7 +19,7 @@ type ResendVerificationCmd struct { } // Execute executes the resend verification command. -func (cmd *ResendVerificationCmd) Execute(args []string) error { +func (cmd *userVerificationResendCmd) Execute(args []string) error { rv := v1.ResendVerification{ Email: cmd.Args.Email, PublicKey: cmd.Args.PublicKey, @@ -38,9 +38,9 @@ func (cmd *ResendVerificationCmd) Execute(args []string) error { return shared.PrintJSON(rvr) } -// resendVerificationHelpMsg is the output of the help command when -// 'resendverification' is specified. -var resendVerificationHelpMsg = `resendverification +// userVerificationResendHelpMsg is the output of the help command when +// 'userverificationresend' is specified. +var userVerificationResendHelpMsg = `userverificationresend Resend the user verification email. The user is only allowed to resend the verification email one time before they must wait for the verification token to @@ -53,9 +53,4 @@ been disabled on politeiawww. Arguments: 1. email (string, required) User email address -2. publickey (string, required) User public key - -Result: -{ - "verificationtoken" (string) Verification token for the user -}` +2. publickey (string, required) User public key` diff --git a/politeiawww/cmd/piwww/verifyuserpayment.go b/politeiawww/cmd/piwww/verifyuserpayment.go deleted file mode 100644 index 5b7333898..000000000 --- a/politeiawww/cmd/piwww/verifyuserpayment.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import "github.com/decred/politeia/politeiawww/cmd/shared" - -// VerifyUserPaymentCmd checks on the status of the logged in user's -// registration payment. -type VerifyUserPaymentCmd struct{} - -// Execute executes the verify user payment command. -func (cmd *VerifyUserPaymentCmd) Execute(args []string) error { - vupr, err := client.VerifyUserPayment() - if err != nil { - return err - } - return shared.PrintJSON(vupr) -} - -// verifyUserPaymentHelpMsg is the output of the help command when -// 'verifyuserpayment' is specified. -var verifyUserPaymentHelpMsg = `verifyuserpayment - -Check if the currently logged in user has paid their user registration fee. - -Arguments: None - -Result: -{ - "haspaid" (bool) Has paid or not - "paywalladdress" (string) Registration paywall address - "paywallamount" (uint64) Registration paywall amount in atoms - "paywalltxnotbefore" (int64) Minimum timestamp for paywall tx -}` diff --git a/politeiawww/cmd/piwww/vettedproposals.go b/politeiawww/cmd/piwww/vettedproposals.go deleted file mode 100644 index 039b3f897..000000000 --- a/politeiawww/cmd/piwww/vettedproposals.go +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" - - v1 "github.com/decred/politeia/politeiawww/api/www/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" -) - -// VettedProposalsCmd retreives a page of vetted proposals. -type VettedProposalsCmd struct { - Before string `long:"before"` // Before censorship token - After string `long:"after"` // After censorship token -} - -// Execute executs the vetted proposals command. -func (cmd *VettedProposalsCmd) Execute(args []string) error { - if cmd.Before != "" && cmd.After != "" { - return fmt.Errorf("the 'before' and 'after' flags " + - "cannot be used at the same time") - } - - // Get server's public key - vr, err := client.Version() - if err != nil { - return err - } - - // Get a page of vetted proposals - gavr, err := client.GetAllVetted(&v1.GetAllVetted{ - Before: cmd.Before, - After: cmd.After, - }) - if err != nil { - return err - } - - // Verify proposal censorship records - for _, p := range gavr.Proposals { - err = shared.VerifyProposal(p, vr.PubKey) - if err != nil { - return fmt.Errorf("unable to verify proposal %v: %v", - p.CensorshipRecord.Token, err) - } - } - - // Print vetted proposals - return shared.PrintJSON(gavr) -} - -// vettedproposalsHelpMsg is the output for the help command when -// 'vettedproposals' is specified. -const vettedProposalsHelpMsg = `vettedproposals [flags] - -Fetch a page of vetted proposals. - -Arguments: None - -Flags: - --before (string, optional) Get proposals before this proposal (token) - --after (string, optional) Get proposals after this proposal (token) - -Example: -getvetted --after=[token] - -Result: -{ - "proposals": [ - { - "name": (string) Suggested short proposal name - "state": (PropStateT) Current state of proposal - "status": (PropStatusT) Current status of proposal - "timestamp": (int64) Timestamp of last update of proposal - "userid": (string) ID of user who submitted proposal - "username": (string) Username of user who submitted proposal - "publickey": (string) Public key used to sign proposal - "signature": (string) Signature of merkle root - "files": [ - { - "name": (string) Filename - "mime": (string) Mime type - "digest": (string) File digest - "payload": (string) File payload - } - ], - "numcomments": (uint) Number of comments on the proposal - "version": (string) Version of proposal - "censorshiprecord": { - "token": (string) Censorship token - "merkle": (string) Merkle root of proposal - "signature": (string) Server side signature of []byte(Merkle+Token) - } - } - ] -}` diff --git a/politeiawww/cmd/piwww/voteauthorize.go b/politeiawww/cmd/piwww/voteauthorize.go index 18577a597..da71413b4 100644 --- a/politeiawww/cmd/piwww/voteauthorize.go +++ b/politeiawww/cmd/piwww/voteauthorize.go @@ -13,9 +13,9 @@ import ( "github.com/decred/politeia/util" ) -// VoteAuthorizeCmd authorizes a proposal vote. The VoteAuthorizeCmd must be +// voteAuthorizeCmd authorizes a proposal vote. The VoteAuthorizeCmd must be // sent by the proposal author to be valid. -type VoteAuthorizeCmd struct { +type voteAuthorizeCmd struct { Args struct { Token string `positional-arg-name:"token" required:"true"` // Censorship token Action string `positional-arg-name:"action"` // Authorize or revoke action @@ -23,7 +23,7 @@ type VoteAuthorizeCmd struct { } // Execute executes the authorize vote command. -func (cmd *VoteAuthorizeCmd) Execute(args []string) error { +func (cmd *voteAuthorizeCmd) Execute(args []string) error { token := cmd.Args.Token // Check for user identity diff --git a/politeiawww/cmd/piwww/voteballot.go b/politeiawww/cmd/piwww/voteballot.go index 64fd35cb8..cc78ac265 100644 --- a/politeiawww/cmd/piwww/voteballot.go +++ b/politeiawww/cmd/piwww/voteballot.go @@ -19,8 +19,8 @@ import ( "golang.org/x/crypto/ssh/terminal" ) -// VoteBallotCmd casts a votes ballot for the specified proposal. -type VoteBallotCmd struct { +// voteBallotCmd casts a votes ballot for the specified proposal. +type voteBallotCmd struct { Args struct { Token string `positional-arg-name:"token"` // Censorship token VoteID string `positional-arg-name:"voteid"` // Vote choice ID @@ -28,7 +28,7 @@ type VoteBallotCmd struct { } // Execute executes the vote ballot command. -func (cmd *VoteBallotCmd) Execute(args []string) error { +func (cmd *voteBallotCmd) Execute(args []string) error { token := cmd.Args.Token voteID := cmd.Args.VoteID diff --git a/politeiawww/cmd/piwww/votedetails.go b/politeiawww/cmd/piwww/votedetails.go index 3252d9732..7aeb40ac0 100644 --- a/politeiawww/cmd/piwww/votedetails.go +++ b/politeiawww/cmd/piwww/votedetails.go @@ -8,16 +8,16 @@ import ( "github.com/decred/politeia/politeiawww/cmd/shared" ) -// VoteDetailsCmd fetches the vote parameters and vote options from the +// voteDetailsCmd fetches the vote parameters and vote options from the // politeiawww v2 VoteDetails routes. -type VoteDetailsCmd struct { +type voteDetailsCmd struct { Args struct { Token string `positional-arg-name:"token"` // Proposal token } `positional-args:"true" required:"true"` } // Execute executes the vote details command. -func (cmd *VoteDetailsCmd) Execute(args []string) error { +func (cmd *voteDetailsCmd) Execute(args []string) error { vdr, err := client.VoteDetailsV2(cmd.Args.Token) if err != nil { return err diff --git a/politeiawww/cmd/piwww/voteinventory.go b/politeiawww/cmd/piwww/voteinventory.go index 03b49fda0..ec05b0e28 100644 --- a/politeiawww/cmd/piwww/voteinventory.go +++ b/politeiawww/cmd/piwww/voteinventory.go @@ -8,12 +8,12 @@ import ( "github.com/decred/politeia/politeiawww/cmd/shared" ) -// VoteInventoryCmd retrieves the censorship record tokens of all public, +// voteInventoryCmd retrieves the censorship record tokens of all public, // non-abandoned proposals in inventory categorized by their vote status. -type VoteInventoryCmd struct{} +type voteInventoryCmd struct{} // Execute executes the vote inventory command. -func (cmd *VoteInventoryCmd) Execute(args []string) error { +func (cmd *voteInventoryCmd) Execute(args []string) error { reply, err := client.VoteInventory() if err != nil { return err diff --git a/politeiawww/cmd/piwww/voteresults.go b/politeiawww/cmd/piwww/voteresults.go index ed0ffb47c..e9b88fee5 100644 --- a/politeiawww/cmd/piwww/voteresults.go +++ b/politeiawww/cmd/piwww/voteresults.go @@ -9,16 +9,16 @@ import ( "github.com/decred/politeia/politeiawww/cmd/shared" ) -// VoteResultsCmd gets the votes that have been cast for the specified +// voteResultsCmd gets the votes that have been cast for the specified // proposal. -type VoteResultsCmd struct { +type voteResultsCmd struct { Args struct { Token string `positional-arg-name:"token"` // Censorship token } `positional-args:"true" required:"true"` } // Execute executes the proposal votes command. -func (cmd *VoteResultsCmd) Execute(args []string) error { +func (cmd *voteResultsCmd) Execute(args []string) error { // Prep request payload vr := pi.VoteResults{ Token: cmd.Args.Token, diff --git a/politeiawww/cmd/piwww/votes.go b/politeiawww/cmd/piwww/votes.go index 188ad3881..290847a39 100644 --- a/politeiawww/cmd/piwww/votes.go +++ b/politeiawww/cmd/piwww/votes.go @@ -11,16 +11,16 @@ import ( "github.com/decred/politeia/politeiawww/cmd/shared" ) -// VotesCmd retrieves vote details for a proposal, tallies the votes, +// votesCmd retrieves vote details for a proposal, tallies the votes, // and displays the result. -type VotesCmd struct { +type votesCmd struct { Args struct { Token string `positional-arg-name:"token"` // Censorship token } `positional-args:"true" required:"true"` } // Execute executes the tally command. -func (cmd *VotesCmd) Execute(args []string) error { +func (cmd *votesCmd) Execute(args []string) error { token := cmd.Args.Token // Prep request payload @@ -57,7 +57,7 @@ func (cmd *VotesCmd) Execute(args []string) error { return shared.PrintJSON(vrr) } -// votesHelpMsg is the output for the help command when 'tally' is specified. +// votesHelpMsg is the output for the help command when 'votes' is specified. const votesHelpMsg = `votes "token" Fetch the vote details for a proposal. diff --git a/politeiawww/cmd/piwww/votestart.go b/politeiawww/cmd/piwww/votestart.go index 9b2d26630..50812a0aa 100644 --- a/politeiawww/cmd/piwww/votestart.go +++ b/politeiawww/cmd/piwww/votestart.go @@ -15,12 +15,12 @@ import ( "github.com/decred/politeia/util" ) -// VoteStartCmd starts the voting period on the specified proposal. +// voteStartCmd starts the voting period on the specified proposal. // // The QuorumPercentage and PassPercentage are strings and not uint32 so that a // value of 0 can be passed in and not be overwritten by the defaults. This is // sometimes desirable when testing. -type VoteStartCmd struct { +type voteStartCmd struct { Args struct { Token string `positional-arg-name:"token" required:"true"` Duration uint32 `positional-arg-name:"duration"` @@ -30,7 +30,7 @@ type VoteStartCmd struct { } // Execute executes the start vote command. -func (cmd *VoteStartCmd) Execute(args []string) error { +func (cmd *voteStartCmd) Execute(args []string) error { // Check for user identity if cfg.Identity == nil { return shared.ErrUserIdentityNotFound diff --git a/politeiawww/cmd/piwww/votestartrunoff.go b/politeiawww/cmd/piwww/votestartrunoff.go index afb5c7f63..200c1cc5b 100644 --- a/politeiawww/cmd/piwww/votestartrunoff.go +++ b/politeiawww/cmd/piwww/votestartrunoff.go @@ -16,13 +16,13 @@ import ( "github.com/decred/politeia/util" ) -// VoteStartRunoffCmd starts the voting period on all public submissions to a +// voteStartRunoffCmd starts the voting period on all public submissions to a // request for proposals (RFP). // // The QuorumPercentage and PassPercentage are strings and not uint32 so that a // value of 0 can be passed in and not be overwritten by the defaults. This is // sometimes desirable when testing. -type VoteStartRunoffCmd struct { +type voteStartRunoffCmd struct { Args struct { TokenRFP string `positional-arg-name:"token" required:"true"` // RFP censorship token Duration uint32 `positional-arg-name:"duration"` // Duration in blocks @@ -32,7 +32,7 @@ type VoteStartRunoffCmd struct { } // Execute executes the StartVoteRunoff command. -func (cmd *VoteStartRunoffCmd) Execute(args []string) error { +func (cmd *voteStartRunoffCmd) Execute(args []string) error { // Check for user identity if cfg.Identity == nil { return shared.ErrUserIdentityNotFound diff --git a/politeiawww/cmd/piwww/votestatus.go b/politeiawww/cmd/piwww/votestatus.go index 7084db75e..908dfebe7 100644 --- a/politeiawww/cmd/piwww/votestatus.go +++ b/politeiawww/cmd/piwww/votestatus.go @@ -6,15 +6,15 @@ package main import "github.com/decred/politeia/politeiawww/cmd/shared" -// VoteStatusCmd gets the vote status of the specified proposal. -type VoteStatusCmd struct { +// voteStatusCmd gets the vote status of the specified proposal. +type voteStatusCmd struct { Args struct { Token string `positional-arg-name:"token"` // Censorship token } `positional-args:"true" required:"true"` } // Execute executes the vote status command. -func (cmd *VoteStatusCmd) Execute(args []string) error { +func (cmd *voteStatusCmd) Execute(args []string) error { vsr, err := client.VoteStatus(cmd.Args.Token) if err != nil { return err @@ -38,30 +38,4 @@ Proposal vote status codes: '5' - Proposal doesn't exist Arguments: -1. token (string, required) Proposal censorship token - -Request: -{ - "token": (string) Proposal censorship token -} - -Response: -{ - "token": (string) Public key of user that submitted proposal - "status": (int) Vote status code - "totalvotes": (uint64) Total number of votes on proposal - "optionsresult": [ - { - "option": { - "id": (string) Unique word identifying vote (e.g. 'yes') - "description": (string) Longer description of the vote - "bits": (uint64) Bits used for this option - }, - "votesreceived": (uint64) Number of votes received - }, - ], - "endheight": (string) String encoded final block height of the vote - "numofeligiblevotes": (int) Total number of eligible votes - "quorumpercentage": (uint32) Percent of eligible votes required for quorum - "passpercentage": (uint32) Percent of total votes required to pass -}` +1. token (string, required) Proposal censorship token` diff --git a/politeiawww/cmd/piwww/votestatuses.go b/politeiawww/cmd/piwww/votestatuses.go index a59dac2c9..ff3be57bf 100644 --- a/politeiawww/cmd/piwww/votestatuses.go +++ b/politeiawww/cmd/piwww/votestatuses.go @@ -6,11 +6,11 @@ package main import "github.com/decred/politeia/politeiawww/cmd/shared" -// VoteStatusesCmd retreives the vote status of all public proposals. -type VoteStatusesCmd struct{} +// voteStatusesCmd retreives the vote status of all public proposals. +type voteStatusesCmd struct{} // Execute executes the vote statuses command. -func (cmd *VoteStatusesCmd) Execute(args []string) error { +func (cmd *voteStatusesCmd) Execute(args []string) error { avsr, err := client.GetAllVoteStatus() if err != nil { return err @@ -33,29 +33,4 @@ Proposal vote status codes: '4' - Proposal vote has been finished '5' - Proposal doesn't exist -Arguments: None - -Response: -{ - "votestatus": [ - { - "token": (string) Public key of user that submitted proposal - "status": (int) Vote status code - "totalvotes": (uint64) Total number of votes on proposal - "optionsresult": [ - { - "option": { - "id": (string) Unique word identifying vote (e.g. 'yes') - "description": (string) Longer description of the vote - "bits": (uint64) Bits used for this option - }, - "votesreceived": (uint64) Number of votes received - }, - ], - "endheight": (string) String encoded final block height of the vote - "numofeligiblevotes": (int) Total number of eligible votes - "quorumpercentage": (uint32) Percent of eligible votes required for quorum - "passpercentage": (uint32) Percent of total votes required to pass - } - ] -}` +Arguments: None` diff --git a/politeiawww/cmd/piwww/votesummaries.go b/politeiawww/cmd/piwww/votesummaries.go index 50ed42a20..5562c0a5c 100644 --- a/politeiawww/cmd/piwww/votesummaries.go +++ b/politeiawww/cmd/piwww/votesummaries.go @@ -9,11 +9,11 @@ import ( "github.com/decred/politeia/politeiawww/cmd/shared" ) -// VoteSummariesCmd retrieves a set of proposal vote summaries. -type VoteSummariesCmd struct{} +// voteSummariesCmd retrieves a set of proposal vote summaries. +type voteSummariesCmd struct{} // Execute executes the batch vote summaries command. -func (cmd *VoteSummariesCmd) Execute(args []string) error { +func (cmd *voteSummariesCmd) Execute(args []string) error { bpr, err := client.VoteSummaries(&pi.VoteSummaries{ Tokens: args, }) diff --git a/politeiawww/cmd/shared/logout.go b/politeiawww/cmd/shared/logout.go index f86784a03..6b909e719 100644 --- a/politeiawww/cmd/shared/logout.go +++ b/politeiawww/cmd/shared/logout.go @@ -31,7 +31,4 @@ const LogoutHelpMsg = `logout Logout as a user or admin. Arguments: -None - -Result: -{}` +None` diff --git a/politeiawww/cmd/shared/updateuserkey.go b/politeiawww/cmd/shared/userkeyupdate.go similarity index 78% rename from politeiawww/cmd/shared/updateuserkey.go rename to politeiawww/cmd/shared/userkeyupdate.go index 319bd2b5d..f61d983f8 100644 --- a/politeiawww/cmd/shared/updateuserkey.go +++ b/politeiawww/cmd/shared/userkeyupdate.go @@ -8,16 +8,16 @@ import ( "encoding/hex" "fmt" - "github.com/decred/politeia/politeiawww/api/www/v1" + v1 "github.com/decred/politeia/politeiawww/api/www/v1" ) -// UpdateUserKeyCmd creates a new identity for the logged in user. -type UpdateUserKeyCmd struct { +// UserKeyUpdateCmd creates a new identity for the logged in user. +type UserKeyUpdateCmd struct { NoSave bool `long:"nosave"` // Don't save new identity to disk } // Execute executes the update user key command. -func (cmd *UpdateUserKeyCmd) Execute(args []string) error { +func (cmd *UserKeyUpdateCmd) Execute(args []string) error { // Get the logged in user's username. We need // this when we save the new identity to disk. me, err := client.Me() @@ -67,17 +67,11 @@ func (cmd *UpdateUserKeyCmd) Execute(args []string) error { return PrintJSON(vuukr) } -// UpdateUserKeyHelpMsg is the output of the help command when 'updateuserkey' +// UserKeyUpdateHelpMsg is the output of the help command when 'userkeyupdate' // is specified. -const UpdateUserKeyHelpMsg = `updateuserkey +const UserKeyUpdateHelpMsg = `userkeyupdate Generate a new public key for the currently logged in user. Arguments: -None - -Result: -{ - "publickey" (string) User's public key -} -{}` +None` diff --git a/politeiawww/cmd/shared/manageuser.go b/politeiawww/cmd/shared/usermanage.go similarity index 86% rename from politeiawww/cmd/shared/manageuser.go rename to politeiawww/cmd/shared/usermanage.go index bca811601..566d0e823 100644 --- a/politeiawww/cmd/shared/manageuser.go +++ b/politeiawww/cmd/shared/usermanage.go @@ -11,9 +11,9 @@ import ( v1 "github.com/decred/politeia/politeiawww/api/www/v1" ) -// ManageUserCmd allows an admin to edit certain properties of the specified +// UserManageCmd allows an admin to edit certain properties of the specified // user. -type ManageUserCmd struct { +type UserManageCmd struct { Args struct { UserID string `positional-arg-name:"userid"` // User ID Action string `positional-arg-name:"action"` // Edit user action @@ -22,7 +22,7 @@ type ManageUserCmd struct { } // Execute executes the manage user command. -func (cmd *ManageUserCmd) Execute(args []string) error { +func (cmd *UserManageCmd) Execute(args []string) error { ManageActions := map[string]v1.UserManageActionT{ "expirenewuser": v1.UserManageExpireNewUserVerification, "expireupdatekey": v1.UserManageExpireUpdateKeyVerification, @@ -77,9 +77,9 @@ func (cmd *ManageUserCmd) Execute(args []string) error { return PrintJSON(mur) } -// ManageUserHelpMsg is the output of the help command when 'edituser' is +// UserManageHelpMsg is the output of the help command when 'edituser' is // specified. -const ManageUserHelpMsg = `manageuser "userid" "action" "reason" +const UserManageHelpMsg = `usermanage "userid" "action" "reason" Edit the details for the given user id. Requires admin privileges. @@ -95,14 +95,4 @@ Valid actions are: 4. clearpaywall Clears user registration paywall 5. unlocks Unlocks user account from failed logins 6. deactivates Deactivates user account -7. reactivate Reactivates user account - -Request: -{ - "userid": (string) User id - "action": (string) Edit user action - "reason": (string) Reason for action -} - -Response: -{}` +7. reactivate Reactivates user account` diff --git a/politeiawww/cmd/shared/changepassword.go b/politeiawww/cmd/shared/userpasswordchange.go similarity index 68% rename from politeiawww/cmd/shared/changepassword.go rename to politeiawww/cmd/shared/userpasswordchange.go index 480751aaf..bffd79862 100644 --- a/politeiawww/cmd/shared/changepassword.go +++ b/politeiawww/cmd/shared/userpasswordchange.go @@ -7,11 +7,11 @@ package shared import ( "fmt" - "github.com/decred/politeia/politeiawww/api/www/v1" + v1 "github.com/decred/politeia/politeiawww/api/www/v1" ) -// ChangePasswordCmd changes the password for the logged in user. -type ChangePasswordCmd struct { +// UserPasswordChangeCmd changes the password for the logged in user. +type UserPasswordChangeCmd struct { Args struct { Password string `positional-arg-name:"currentPassword"` // Current password NewPassword string `positional-arg-name:"newPassword"` // New password @@ -19,7 +19,7 @@ type ChangePasswordCmd struct { } // Execute executes the change password command. -func (cmd *ChangePasswordCmd) Execute(args []string) error { +func (cmd *UserPasswordChangeCmd) Execute(args []string) error { // Get password requirements pr, err := client.Policy() if err != nil { @@ -54,21 +54,12 @@ func (cmd *ChangePasswordCmd) Execute(args []string) error { return PrintJSON(cpr) } -// ChangePasswordHelpMsg is the output of the help command when -// 'changepassword' is specified. -const ChangePasswordHelpMsg = `changepassword "currentPassword" "newPassword" +// UserPasswordChangeHelpMsg is the output of the help command when +// 'userpasswordchange' is specified. +const UserPasswordChangeHelpMsg = `userpasswordchange "currentPassword" "newPassword" Change password for the currently logged in user. Arguments: 1. currentPassword (string, required) Current password -2. newPassword (string, required) New password - -Request: -{ - "currentpassword": (string) Current password - "newpassword": (string) New password -} - -Response: -{}` +2. newPassword (string, required) New password` diff --git a/politeiawww/cmd/shared/resetpassword.go b/politeiawww/cmd/shared/userpasswordreset.go similarity index 83% rename from politeiawww/cmd/shared/resetpassword.go rename to politeiawww/cmd/shared/userpasswordreset.go index 02a981ccd..538c5620a 100644 --- a/politeiawww/cmd/shared/resetpassword.go +++ b/politeiawww/cmd/shared/userpasswordreset.go @@ -10,8 +10,8 @@ import ( www "github.com/decred/politeia/politeiawww/api/www/v1" ) -// ResetPasswordCmd resets the password of the specified user. -type ResetPasswordCmd struct { +// UserPasswordResetCmd resets the password of the specified user. +type UserPasswordResetCmd struct { Args struct { Username string `positional-arg-name:"username"` // Username Email string `positional-arg-name:"email"` // User email address @@ -20,7 +20,7 @@ type ResetPasswordCmd struct { } // Execute executes the reset password command. -func (cmd *ResetPasswordCmd) Execute(args []string) error { +func (cmd *UserPasswordResetCmd) Execute(args []string) error { username := cmd.Args.Username email := cmd.Args.Email newPassword := cmd.Args.NewPassword @@ -86,14 +86,13 @@ func (cmd *ResetPasswordCmd) Execute(args []string) error { return PrintJSON(vrpr) } -// ResetPasswordHelpMsg is the output of the help command when 'resetpassword' +// UserPasswordResetHelpMsg is the output of the help command when 'userpasswordreset' // is specified. -const ResetPasswordHelpMsg = `resetpassword "username" "email" "newpassword" +const UserPasswordResetHelpMsg = `userpasswordreset "username" "email" "newpassword" Reset the password for a user that is not logged in. Arguments: 1. username (string, required) Username of user 2. email (string, required) Email address of user -3. password (string, required) New password -` +3. password (string, required) New password` diff --git a/politeiawww/cmd/shared/users.go b/politeiawww/cmd/shared/users.go index 9baa38b2c..7b01e01a4 100644 --- a/politeiawww/cmd/shared/users.go +++ b/politeiawww/cmd/shared/users.go @@ -4,9 +4,7 @@ package shared -import ( - "github.com/decred/politeia/politeiawww/api/www/v1" -) +import v1 "github.com/decred/politeia/politeiawww/api/www/v1" // UsersCmd retreives a list of users that have been filtered using the // specified filtering params. @@ -50,31 +48,4 @@ Flags: Example (Admin): users --email=user@example.com --username=user --pubkey=0b2283a91f6bf95f2c121 -14c7c1259c1396756bea4f64be43fe0f73b383bdf92 - -Result (Admin): -{ - "totalusers": (uint64) Total number of all users in the database - "totalmatches": (uint64) Total number of users that match the filters - "users": [ - { - "id": (string) User id - "email": (string) User email address - "username": (string) Username - } - ] -} - -Example (non Admin): ---pubkey=0b2283a91f6bf95f2c12114c7c1259c1396756bea4f64be43fe0f73b383bdf92 - -Result (Non admin): -{ - "users": [ - { - "id": (string) User id - "username": (string) Username - } - ] -} -` +14c7c1259c1396756bea4f64be43fe0f73b383bdf92` diff --git a/politeiawww/cmd/shared/settotp.go b/politeiawww/cmd/shared/usertotpset.go similarity index 83% rename from politeiawww/cmd/shared/settotp.go rename to politeiawww/cmd/shared/usertotpset.go index 7d0317a3e..9f8e86032 100644 --- a/politeiawww/cmd/shared/settotp.go +++ b/politeiawww/cmd/shared/usertotpset.go @@ -10,15 +10,15 @@ import ( v1 "github.com/decred/politeia/politeiawww/api/www/v1" ) -// SetTOTPCmd sets the TOTP key for the logged in user. -type SetTOTPCmd struct { +// UserTOTPSetCmd sets the TOTP key for the logged in user. +type UserTOTPSetCmd struct { Args struct { Code string `positional-arg-name:"code"` } `positional-args:"true"` } // Execute executes the set totp command. -func (cmd *SetTOTPCmd) Execute(args []string) error { +func (cmd *UserTOTPSetCmd) Execute(args []string) error { // Setup new user request st := &v1.SetTOTP{ Code: cmd.Args.Code, diff --git a/politeiawww/cmd/shared/verifytotp.go b/politeiawww/cmd/shared/usertotpverify.go similarity index 82% rename from politeiawww/cmd/shared/verifytotp.go rename to politeiawww/cmd/shared/usertotpverify.go index 3212c0670..4285a2088 100644 --- a/politeiawww/cmd/shared/verifytotp.go +++ b/politeiawww/cmd/shared/usertotpverify.go @@ -10,15 +10,15 @@ import ( v1 "github.com/decred/politeia/politeiawww/api/www/v1" ) -// VerifyTOTPCmd sets the TOTP key for the logged in user. -type VerifyTOTPCmd struct { +// UserTOTPVerifyCmd sets the TOTP key for the logged in user. +type UserTOTPVerifyCmd struct { Args struct { Code string `positional-arg-name:"code"` } `positional-args:"true"` } // Execute executes the set totp command. -func (cmd *VerifyTOTPCmd) Execute(args []string) error { +func (cmd *UserTOTPVerifyCmd) Execute(args []string) error { // Setup new user request st := &v1.VerifyTOTP{ Code: cmd.Args.Code, diff --git a/politeiawww/cmd/shared/changeusername.go b/politeiawww/cmd/shared/userusernamechange.go similarity index 61% rename from politeiawww/cmd/shared/changeusername.go rename to politeiawww/cmd/shared/userusernamechange.go index 0ed55bddb..672e0a3cf 100644 --- a/politeiawww/cmd/shared/changeusername.go +++ b/politeiawww/cmd/shared/userusernamechange.go @@ -4,10 +4,10 @@ package shared -import "github.com/decred/politeia/politeiawww/api/www/v1" +import v1 "github.com/decred/politeia/politeiawww/api/www/v1" -// ChangeUsernameCmd changes the username for the logged in user. -type ChangeUsernameCmd struct { +// UserUsernameChangeCmd changes the username for the logged in user. +type UserUsernameChangeCmd struct { Args struct { Password string `positional-arg-name:"password"` // User password NewUsername string `positional-arg-name:"newusername"` // New username @@ -15,7 +15,7 @@ type ChangeUsernameCmd struct { } // Execute executes the change username command. -func (cmd *ChangeUsernameCmd) Execute(args []string) error { +func (cmd *UserUsernameChangeCmd) Execute(args []string) error { cu := &v1.ChangeUsername{ Password: DigestSHA3(cmd.Args.Password), NewUsername: cmd.Args.NewUsername, @@ -37,21 +37,12 @@ func (cmd *ChangeUsernameCmd) Execute(args []string) error { return PrintJSON(cur) } -// ChangeUsernameHelpMsg is the output of the help command when -// 'changeusername' is specified. -var ChangeUsernameHelpMsg = `changeusername "password" "newusername" +// UserUsernameChangeHelpMsg is the output of the help command when +// 'userusernamechange' is specified. +var UserUsernameChangeHelpMsg = `userusernamechange "password" "newusername" Change the username for the currently logged in user. Arguments: 1. password (string, required) Current password -2. newusername (string, required) New username - -Request: -{ - "password": (string) Current password - "newusername": (string) New username -} - -Response: -{}` +2. newusername (string, required) New username` From 3e59a07bfc435ddd6299b5bf2f061df969a0400c Mon Sep 17 00:00:00 2001 From: luke Date: Sat, 26 Sep 2020 16:08:47 -0500 Subject: [PATCH 115/449] fix tlobe plugin user errors --- plugins/comments/comments.go | 12 -- plugins/pi/pi.go | 14 +- plugins/ticketvote/ticketvote.go | 12 -- politeiad/api/v1/v1.go | 10 +- politeiad/backend/backend.go | 13 ++ politeiad/backend/tlogbe/comments.go | 202 ++++++++++++-------- politeiad/backend/tlogbe/pi.go | 61 +++--- politeiad/backend/tlogbe/ticketvote.go | 106 +++++----- politeiad/backend/tlogbe/tlog.go | 10 +- politeiad/backend/tlogbe/tlogbe.go | 39 +++- politeiad/backend/tlogbe/trillian.go | 3 + politeiad/cmd/politeia/README.md | 4 +- politeiad/politeiad.go | 81 +++++++- politeiawww/api/pi/v1/v1.go | 56 +++--- politeiawww/api/www/v1/v1.go | 10 +- politeiawww/piwww.go | 43 ++--- politeiawww/user/cockroachdb/cockroachdb.go | 2 +- politeiawww/www.go | 28 +-- 18 files changed, 435 insertions(+), 271 deletions(-) diff --git a/plugins/comments/comments.go b/plugins/comments/comments.go index 1af727ab2..d741ed49b 100644 --- a/plugins/comments/comments.go +++ b/plugins/comments/comments.go @@ -8,7 +8,6 @@ package comments import ( "encoding/json" - "fmt" ) type StateT int @@ -84,17 +83,6 @@ var ( } ) -// UserErrorReply represents an error that is cause by the user. -type UserErrorReply struct { - ErrorCode ErrorStatusT - ErrorContext []string -} - -// Error satisfies the error interface. -func (e UserErrorReply) Error() string { - return fmt.Sprintf("comments plugin error code: %v", e.ErrorCode) -} - // Comment represent a record comment. // // Signature is the client signature of State+Token+ParentID+Comment. diff --git a/plugins/pi/pi.go b/plugins/pi/pi.go index 7080107e7..105a5e577 100644 --- a/plugins/pi/pi.go +++ b/plugins/pi/pi.go @@ -8,7 +8,6 @@ package pi import ( "encoding/json" - "fmt" "io" "strings" ) @@ -63,10 +62,10 @@ const ( // User error status codes // TODO number error codes and add human readable error messages ErrorStatusInvalid ErrorStatusT = iota + ErrorStatusPropLinkToInvalid ErrorStatusPropVersionInvalid ErrorStatusPropStatusInvalid ErrorStatusPropStatusChangeInvalid - ErrorStatusPropLinkToInvalid ErrorStatusVoteStatusInvalid ErrorStatusPageSizeExceeded ) @@ -97,17 +96,6 @@ var ( } ) -// UserErrorReply represents an error that is caused by the user. -type UserErrorReply struct { - ErrorCode ErrorStatusT - ErrorContext []string -} - -// Error satisfies the error interface. -func (e UserErrorReply) Error() string { - return fmt.Sprintf("pi plugin error code: %v", e.ErrorCode) -} - // ProposalMetadata contains proposal metadata that is provided by the user on // proposal submission. ProposalMetadata is saved to politeiad as a file, not // as a metadata stream, since it needs to be included in the merkle root that diff --git a/plugins/ticketvote/ticketvote.go b/plugins/ticketvote/ticketvote.go index d602996be..38d08ce3b 100644 --- a/plugins/ticketvote/ticketvote.go +++ b/plugins/ticketvote/ticketvote.go @@ -8,7 +8,6 @@ package ticketvote import ( "encoding/json" - "fmt" ) type VoteStatusT int @@ -150,17 +149,6 @@ var ( } ) -// UserErrorReply represents an error that is caused by the user. -type UserErrorReply struct { - ErrorCode ErrorStatusT - ErrorContext []string -} - -// Error satisfies the error interface. -func (e UserErrorReply) Error() string { - return fmt.Sprintf("ticketvote plugin error code: %v", e.ErrorCode) -} - // AuthorizeDetails is the structure that is saved to disk when a vote is // authorized or a previous authorization is revoked. It contains all the // fields from a Authorize and a AuthorizeReply. diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index 1bad37fcc..6cc3ab210 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -404,10 +404,14 @@ type InventoryByStatusReply struct { type UserErrorReply struct { ErrorCode ErrorStatusT `json:"errorcode"` // Numeric error code ErrorContext []string `json:"errorcontext,omitempty"` // Additional error information +} - // Plugin will be populated with the plugin ID if the error was a - // plugin user error. - Plugin string `json:"plugin,omitempty"` +// UserErrorReply returns details about a plugin error that occurred while +// trying to execute a command due to bad input from the client. +type PluginUserErrorReply struct { + Plugin string `json:"plugin"` + ErrorCode int `json:"errorcode"` + ErrorContext []string `json:"errorcontext,omitempty"` } // ServerErrorReply returns an error code that can be correlated with diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index 390f1a0b9..3bea609ac 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -101,11 +101,24 @@ type StateTransitionError struct { To MDStatusT } +// Error satisfies the error interface. func (s StateTransitionError) Error() string { return fmt.Sprintf("invalid record status transition %v (%v) -> %v (%v)", s.From, MDStatus[s.From], s.To, MDStatus[s.To]) } +// PluginUserError represents a plugin error that is caused by the user. +type PluginUserError struct { + PluginID string + ErrorCode int + ErrorContext []string +} + +// Error satisfies the error interface. +func (e PluginUserError) Error() string { + return fmt.Sprintf("plugin error code: %v", e.ErrorCode) +} + // RecordMetadata is the metadata of a record. const VersionRecordMD = 1 diff --git a/politeiad/backend/tlogbe/comments.go b/politeiad/backend/tlogbe/comments.go index 4a4abfb25..4cc45810a 100644 --- a/politeiad/backend/tlogbe/comments.go +++ b/politeiad/backend/tlogbe/comments.go @@ -206,7 +206,7 @@ func encryptFromCommentState(s comments.StateT) bool { } } -func convertCommentsErrorFromSignatureError(err error) comments.UserErrorReply { +func convertCommentsErrorFromSignatureError(err error) backend.PluginUserError { var e util.SignatureError var s comments.ErrorStatusT if errors.As(err, &e) { @@ -217,8 +217,9 @@ func convertCommentsErrorFromSignatureError(err error) comments.UserErrorReply { s = comments.ErrorStatusSignatureInvalid } } - return comments.UserErrorReply{ - ErrorCode: s, + return backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(s), ErrorContext: e.ErrorContext, } } @@ -757,16 +758,18 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { case comments.StateUnvetted, comments.StateVetted: // Allowed; continue default: - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusTokenInvalid, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), } } // Verify token token, err := hex.DecodeString(n.Token) if err != nil { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusTokenInvalid, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), } } @@ -780,8 +783,9 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { // Verify comment if len(n.Comment) > comments.PolicyCommentLengthMax { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusCommentLengthMax, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusCommentLengthMax), } } @@ -800,8 +804,9 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { // Verify parent comment exists if set. A parent ID of 0 means that // this is a base level comment, not a reply to another comment. if n.ParentID > 0 && !commentExists(*idx, n.ParentID) { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusParentIDInvalid, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusParentIDInvalid), ErrorContext: []string{"parent ID comment not found"}, } } @@ -826,8 +831,9 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { merkleHash, err := p.commentAddSave(ca) if err != nil { if err == errRecordNotFound { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusRecordNotFound, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusRecordNotFound), } } return "", fmt.Errorf("commentAddSave: %v", err) @@ -879,16 +885,18 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { case comments.StateUnvetted, comments.StateVetted: // Allowed; continue default: - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusTokenInvalid, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), } } // Verify token token, err := hex.DecodeString(e.Token) if err != nil { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusTokenInvalid, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), } } @@ -902,8 +910,9 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { // Verify comment if len(e.Comment) > comments.PolicyCommentLengthMax { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusCommentLengthMax, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusCommentLengthMax), } } @@ -926,8 +935,9 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { } existing, ok := cs[e.CommentID] if !ok { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusCommentNotFound, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusCommentNotFound), } } @@ -935,8 +945,9 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { if e.UserID != existing.UserID { e := fmt.Sprintf("user id cannot change; got %v, want %v", e.UserID, existing.UserID) - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusUserIDInvalid, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusUserIDInvalid), ErrorContext: []string{e}, } } @@ -945,16 +956,18 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { if e.ParentID != existing.ParentID { e := fmt.Sprintf("parent id cannot change; got %v, want %v", e.ParentID, existing.ParentID) - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusParentIDInvalid, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusParentIDInvalid), ErrorContext: []string{e}, } } // Verify comment changes if e.Comment == existing.Comment { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusNoCommentChanges, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusNoCommentChanges), } } @@ -977,8 +990,9 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { merkle, err := p.commentAddSave(ca) if err != nil { if err == errRecordNotFound { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusRecordNotFound, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusRecordNotFound), } } return "", fmt.Errorf("commentSave: %v", err) @@ -1024,16 +1038,18 @@ func (p *commentsPlugin) cmdDel(payload string) (string, error) { case comments.StateUnvetted, comments.StateVetted: // Allowed; continue default: - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusTokenInvalid, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), } } // Verify token token, err := hex.DecodeString(d.Token) if err != nil { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusTokenInvalid, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), } } @@ -1064,8 +1080,9 @@ func (p *commentsPlugin) cmdDel(payload string) (string, error) { } existing, ok := cs[d.CommentID] if !ok { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusCommentNotFound, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusCommentNotFound), } } @@ -1087,8 +1104,9 @@ func (p *commentsPlugin) cmdDel(payload string) (string, error) { merkle, err := p.commentDelSave(cd) if err != nil { if err == errRecordNotFound { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusRecordNotFound, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusRecordNotFound), } } return "", fmt.Errorf("commentDelSave: %v", err) @@ -1148,16 +1166,18 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { case comments.StateUnvetted, comments.StateVetted: // Allowed; continue default: - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusTokenInvalid, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), } } // Verify token token, err := hex.DecodeString(v.Token) if err != nil { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusTokenInvalid, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), } } @@ -1166,8 +1186,9 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { case comments.VoteDownvote, comments.VoteUpvote: // These are allowed default: - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusVoteInvalid, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusVoteInvalid), } } @@ -1195,8 +1216,9 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { // Verify comment exists cidx, ok := idx.Comments[v.CommentID] if !ok { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusCommentNotFound, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusCommentNotFound), } } @@ -1206,8 +1228,9 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { uvotes = make([]voteIndex, 0) } if len(uvotes) > comments.PolicyVoteChangesMax { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusVoteChangesMax, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusVoteChangesMax), } } @@ -1221,8 +1244,9 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { return "", fmt.Errorf("comment not found %v", v.CommentID) } if v.UserID == c.UserID { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusVoteInvalid, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusVoteInvalid), ErrorContext: []string{"user cannot vote on their own comment"}, } } @@ -1244,8 +1268,9 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { merkle, err := p.commentVoteSave(cv) if err != nil { if err == errRecordNotFound { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusRecordNotFound, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusRecordNotFound), } } return "", fmt.Errorf("commentVoteSave: %v", err) @@ -1302,16 +1327,18 @@ func (p *commentsPlugin) cmdGet(payload string) (string, error) { case comments.StateUnvetted, comments.StateVetted: // Allowed; continue default: - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusTokenInvalid, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), } } // Verify token token, err := hex.DecodeString(g.Token) if err != nil { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusTokenInvalid, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), } } @@ -1325,8 +1352,9 @@ func (p *commentsPlugin) cmdGet(payload string) (string, error) { cs, err := p.comments(g.State, token, *idx, g.CommentIDs) if err != nil { if err == errRecordNotFound { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusRecordNotFound, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusRecordNotFound), } } return "", fmt.Errorf("comments: %v", err) @@ -1358,16 +1386,18 @@ func (p *commentsPlugin) cmdGetAll(payload string) (string, error) { case comments.StateUnvetted, comments.StateVetted: // Allowed; continue default: - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusTokenInvalid, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), } } // Verify token token, err := hex.DecodeString(ga.Token) if err != nil { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusTokenInvalid, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), } } @@ -1387,8 +1417,9 @@ func (p *commentsPlugin) cmdGetAll(payload string) (string, error) { c, err := p.comments(ga.State, token, *idx, commentIDs) if err != nil { if err == errRecordNotFound { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusRecordNotFound, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusRecordNotFound), } } return "", fmt.Errorf("comments: %v", err) @@ -1431,16 +1462,18 @@ func (p *commentsPlugin) cmdGetVersion(payload string) (string, error) { case comments.StateUnvetted, comments.StateVetted: // Allowed; continue default: - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusTokenInvalid, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), } } // Verify token token, err := hex.DecodeString(gv.Token) if err != nil { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusTokenInvalid, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), } } @@ -1453,13 +1486,15 @@ func (p *commentsPlugin) cmdGetVersion(payload string) (string, error) { // Verify comment exists cidx, ok := idx.Comments[gv.CommentID] if !ok { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusCommentNotFound, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusCommentNotFound), } } if cidx.Del != nil { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusCommentNotFound, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusCommentNotFound), ErrorContext: []string{"comment has been deleted"}, } } @@ -1467,8 +1502,9 @@ func (p *commentsPlugin) cmdGetVersion(payload string) (string, error) { if !ok { e := fmt.Sprintf("comment %v does not have version %v", gv.CommentID, gv.Version) - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusCommentNotFound, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusCommentNotFound), ErrorContext: []string{e}, } } @@ -1513,16 +1549,18 @@ func (p *commentsPlugin) cmdCount(payload string) (string, error) { case comments.StateUnvetted, comments.StateVetted: // Allowed; continue default: - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusTokenInvalid, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), } } // Verify token token, err := hex.DecodeString(c.Token) if err != nil { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusTokenInvalid, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), } } @@ -1558,16 +1596,18 @@ func (p *commentsPlugin) cmdVotes(payload string) (string, error) { case comments.StateUnvetted, comments.StateVetted: // Allowed; continue default: - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusTokenInvalid, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), } } // Verify token token, err := hex.DecodeString(v.Token) if err != nil { - return "", comments.UserErrorReply{ - ErrorCode: comments.ErrorStatusTokenInvalid, + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), } } diff --git a/politeiad/backend/tlogbe/pi.go b/politeiad/backend/tlogbe/pi.go index bc59be5e3..6c05ea227 100644 --- a/politeiad/backend/tlogbe/pi.go +++ b/politeiad/backend/tlogbe/pi.go @@ -261,23 +261,27 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { // have been approved by a ticket vote. if pm.LinkTo != "" { if isRFP(*pm) { - return pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropLinkToInvalid, + return backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropLinkToInvalid), ErrorContext: []string{"an rfp cannot have linkto set"}, } } tokenb, err := hex.DecodeString(pm.LinkTo) if err != nil { - return pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropLinkToInvalid, + return backend.PluginUserError{ + + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropLinkToInvalid), ErrorContext: []string{"invalid hex"}, } } r, err := p.backend.GetVetted(tokenb, "") if err != nil { if err == backend.ErrRecordNotFound { - return pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropLinkToInvalid, + return backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropLinkToInvalid), ErrorContext: []string{"proposal not found"}, } } @@ -288,21 +292,24 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { return err } if linkToPM == nil { - return pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropLinkToInvalid, + return backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropLinkToInvalid), ErrorContext: []string{"proposal not an rfp"}, } } if !isRFP(*linkToPM) { - return pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropLinkToInvalid, + return backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropLinkToInvalid), ErrorContext: []string{"proposal not an rfp"}, } } if time.Now().Unix() > linkToPM.LinkBy { // Link by deadline has expired. New links are not allowed. - return pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropLinkToInvalid, + return backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropLinkToInvalid), ErrorContext: []string{"rfp link by deadline expired"}, } } @@ -328,8 +335,9 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { return fmt.Errorf("summary not found %v", pm.LinkTo) } if !summary.Approved { - return pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropLinkToInvalid, + return backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropLinkToInvalid), ErrorContext: []string{"rfp vote not approved"}, } } @@ -350,8 +358,9 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { // Verify proposal status status := convertPropStatusFromMDStatus(er.Current.RecordMetadata.Status) if status != pi.PropStatusUnvetted && status != pi.PropStatusPublic { - return pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropStatusInvalid, + return backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropStatusInvalid), } } @@ -381,8 +390,9 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { e := fmt.Sprintf("vote status got %v, want %v", ticketvote.VoteStatus[summary.Status], ticketvote.VoteStatus[ticketvote.VoteStatusUnauthorized]) - return pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusVoteStatusInvalid, + return backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusVoteStatusInvalid), ErrorContext: []string{e}, } } @@ -400,8 +410,9 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { return err } if pmCurr.LinkTo != pmNew.LinkTo { - return pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropLinkToInvalid, + return backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropLinkToInvalid), ErrorContext: []string{"linkto cannot change on public proposal"}, } } @@ -451,8 +462,9 @@ func (p *piPlugin) hookSetRecordStatusPost(payload string) error { if sc.Version != srs.Current.Version { e := fmt.Sprintf("version not current: got %v, want %v", sc.Version, srs.Current.Version) - return pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropVersionInvalid, + return backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropVersionInvalid), ErrorContext: []string{e}, } } @@ -469,8 +481,9 @@ func (p *piPlugin) hookSetRecordStatusPost(payload string) error { if !isAllowed { e := fmt.Sprintf("from %v to %v status change not allowed", from, sc.Status) - return pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropStatusChangeInvalid, + return backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropStatusChangeInvalid), ErrorContext: []string{e}, } } diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index a62e1321e..a44f6f565 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -242,7 +242,7 @@ func (p *ticketVotePlugin) mutex(token string) *sync.Mutex { return m } -func convertTicketVoteErrFromSignatureErr(err error) ticketvote.UserErrorReply { +func convertTicketVoteErrFromSignatureErr(err error) backend.PluginUserError { var e util.SignatureError var s ticketvote.ErrorStatusT if errors.As(err, &e) { @@ -253,8 +253,9 @@ func convertTicketVoteErrFromSignatureErr(err error) ticketvote.UserErrorReply { s = ticketvote.ErrorStatusSignatureInvalid } } - return ticketvote.UserErrorReply{ - ErrorCode: s, + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(s), ErrorContext: e.ErrorContext, } } @@ -891,8 +892,9 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { // Verify token token, err := hex.DecodeString(a.Token) if err != nil { - return "", ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusTokenInvalid, + return "", backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusTokenInvalid), } } @@ -912,8 +914,9 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { // This is allowed default: e := fmt.Sprintf("%v not a valid action", a.Action) - return "", ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusAuthorizationInvalid, + return "", backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusAuthorizationInvalid), ErrorContext: []string{e}, } } @@ -939,21 +942,23 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { case len(auths) == 0: // No previous actions. New action must be an authorize. if a.Action != ticketvote.ActionAuthorize { - return "", ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusAuthorizationInvalid, + return "", backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusAuthorizationInvalid), ErrorContext: []string{"no prev action; action must be authorize"}, } } case prevAction == ticketvote.ActionAuthorize: // Previous action was a authorize. This action must be revoke. - return "", ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusAuthorizationInvalid, + return "", backend.PluginUserError{ + ErrorCode: int(ticketvote.ErrorStatusAuthorizationInvalid), ErrorContext: []string{"prev action was authorize"}, } case prevAction == ticketvote.ActionRevoke: // Previous action was a revoke. This action must be authorize. - return "", ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusAuthorizationInvalid, + return "", backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusAuthorizationInvalid), ErrorContext: []string{"prev action was revoke"}, } } @@ -1023,8 +1028,9 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM // This is allowed default: e := fmt.Sprintf("invalid type %v", vote.Type) - return ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusVoteParamsInvalid, + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), ErrorContext: []string{e}, } } @@ -1034,29 +1040,33 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM case vote.Duration > voteDurationMax: e := fmt.Sprintf("duration %v exceeds max duration %v", vote.Duration, voteDurationMax) - return ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusVoteParamsInvalid, + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), ErrorContext: []string{e}, } case vote.Duration < voteDurationMin: e := fmt.Sprintf("duration %v under min duration %v", vote.Duration, voteDurationMin) - return ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusVoteParamsInvalid, + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), ErrorContext: []string{e}, } case vote.QuorumPercentage > 100: e := fmt.Sprintf("quorum percent %v exceeds 100 percent", vote.QuorumPercentage) - return ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusVoteParamsInvalid, + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), ErrorContext: []string{e}, } case vote.PassPercentage > 100: e := fmt.Sprintf("pass percent %v exceeds 100 percent", vote.PassPercentage) - return ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusVoteParamsInvalid, + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), ErrorContext: []string{e}, } } @@ -1064,8 +1074,9 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM // Verify vote options. Different vote types have different // requirements. if len(vote.Options) == 0 { - return ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusVoteParamsInvalid, + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), ErrorContext: []string{"no vote options found"}, } } @@ -1077,8 +1088,9 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM if len(vote.Options) != 2 { e := fmt.Sprintf("vote options count got %v, want 2", len(vote.Options)) - return ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusVoteParamsInvalid, + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), ErrorContext: []string{e}, } } @@ -1105,8 +1117,9 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM if len(missing) > 0 { e := fmt.Sprintf("vote option IDs not found: %v", strings.Join(missing, ",")) - return ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusVoteParamsInvalid, + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), ErrorContext: []string{e}, } } @@ -1116,8 +1129,9 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM for _, v := range vote.Options { err := voteBitVerify(vote.Options, vote.Mask, v.Bit) if err != nil { - return ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusVoteParamsInvalid, + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), ErrorContext: []string{err.Error()}, } } @@ -1138,8 +1152,9 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { // Verify token token, err := hex.DecodeString(s.Params.Token) if err != nil { - return "", ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusTokenInvalid, + return "", backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusTokenInvalid), } } @@ -1166,8 +1181,9 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { if err != nil { if err == backend.ErrRecordNotFound { e := fmt.Sprintf("version %v not found", version) - return "", ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusRecordNotFound, + return "", backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusRecordNotFound), ErrorContext: []string{e}, } } @@ -1179,15 +1195,17 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { return "", err } if len(auths) == 0 { - return "", ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusAuthorizationInvalid, + return "", backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusAuthorizationInvalid), ErrorContext: []string{"authorization not found"}, } } action := ticketvote.AuthActionT(auths[len(auths)-1].Action) if action != ticketvote.ActionAuthorize { - return "", ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusAuthorizationInvalid, + return "", backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusAuthorizationInvalid), ErrorContext: []string{"not authorized"}, } } @@ -1212,8 +1230,9 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { } if svp != nil { // Vote has already been started - return "", ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusVoteStatusInvalid, + return "", backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusVoteStatusInvalid), ErrorContext: []string{"vote already started"}, } } @@ -1550,8 +1569,9 @@ func (p *ticketVotePlugin) cmdCastVotes(payload string) (string, error) { // Verify token token, err := hex.DecodeString(cv.Token) if err != nil { - return "", ticketvote.UserErrorReply{ - ErrorCode: ticketvote.ErrorStatusTokenInvalid, + return "", backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusTokenInvalid), } } diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index 2b4777507..e00f55ec1 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -151,7 +151,7 @@ func blobIsEncrypted(b []byte) bool { // tree's leaves. A tree is considered frozen if the tree contains a freeze // record leaf. func treeIsFrozen(leaves []*trillian.LogLeaf) bool { - for i := len(leaves) - 1; i >= 0; i-- { + for i := 0; i < len(leaves); i++ { if leafIsFreezeRecord(leaves[i]) { return true } @@ -818,10 +818,8 @@ type recordBlobsPrepareReply struct { // TODO test this function func recordBlobsPrepare(args recordBlobsPrepareArgs) (*recordBlobsPrepareReply, error) { - // Ensure tree is not frozen. A tree is considered frozen if the - // last leaf on the tree is a freeze record. - lastLeaf := args.leaves[len(args.leaves)-1] - if leafIsFreezeRecord(lastLeaf) { + // Ensure tree is not frozen + if treeIsFrozen(args.leaves) { return nil, errTreeIsFrozen } @@ -1688,6 +1686,8 @@ func (t *tlog) blobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, } // blobsByKeyPrefix returns all blobs that match the provided key prefix. +// +// This function satisfies the backendClient interface. func (t *tlog) blobsByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) { log.Tracef("tlog blobsByKeyPrefix: %v %v", treeID, keyPrefix) diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 2f5c2af6d..4642ea46f 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -9,6 +9,7 @@ import ( "crypto/sha256" "encoding/base64" "encoding/hex" + "errors" "fmt" "io/ioutil" "net/url" @@ -977,7 +978,15 @@ func (t *tlogBackend) GetUnvetted(token []byte, version string) (*backend.Record v = uint32(u) } - return t.unvetted.record(treeID, v) + r, err := t.unvetted.record(treeID, v) + if err != nil { + if err == errRecordNotFound { + err = backend.ErrRecordNotFound + } + return nil, err + } + + return r, nil } // This function satisfies the Backend interface. @@ -1004,7 +1013,15 @@ func (t *tlogBackend) GetVetted(token []byte, version string) (*backend.Record, v = uint32(u) } - return t.vetted.record(treeID, v) + r, err := t.vetted.record(treeID, v) + if err != nil { + if err == errRecordNotFound { + err = backend.ErrRecordNotFound + } + return nil, err + } + + return r, nil } // This function must be called WITH the unvetted lock held. @@ -1440,7 +1457,11 @@ func (t *tlogBackend) pluginHook(h hookT, payload string) error { for _, v := range t.plugins { err := v.client.hook(h, payload) if err != nil { - return fmt.Errorf("Hook %v: %v", v.id, err) + var e backend.PluginUserError + if errors.As(err, &e) { + return err + } + return fmt.Errorf("hook %v: %v", v.id, err) } } @@ -1509,11 +1530,23 @@ func (t *tlogBackend) setup() error { if vettedTreeID != 0 { r, err = t.GetVetted(token, "") if err != nil { + if err == backend.ErrRecordNotFound { + // A tree that was created but no record was appended onto + // it for whatever reason. This can happen if there is a + // network failure or internal server error. + continue + } return fmt.Errorf("GetVetted %x: %v", token, err) } } else { r, err = t.GetUnvetted(token, "") if err != nil { + if err == backend.ErrRecordNotFound { + // A tree that was created but no record was appended onto + // it for whatever reason. This can happen if there is a + // network failure or internal server error. + continue + } return fmt.Errorf("GetUnvetted %x: %v", token, err) } } diff --git a/politeiad/backend/tlogbe/trillian.go b/politeiad/backend/tlogbe/trillian.go index 84fd626e8..f3880c5cb 100644 --- a/politeiad/backend/tlogbe/trillian.go +++ b/politeiad/backend/tlogbe/trillian.go @@ -403,6 +403,9 @@ func (t *trillianClient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { if err != nil { return nil, fmt.Errorf("SignedLogRoot: %v", err) } + if lr.TreeSize == 0 { + return []*trillian.LogLeaf{}, nil + } // Get all leaves return t.leavesByRange(treeID, 0, int64(lr.TreeSize)) diff --git a/politeiad/cmd/politeia/README.md b/politeiad/cmd/politeia/README.md index 6fc972ac1..7f79231d5 100644 --- a/politeiad/cmd/politeia/README.md +++ b/politeiad/cmd/politeia/README.md @@ -9,8 +9,8 @@ $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass identity Key : 8f627e9da14322626d7e81d789f7fcafd25f62235a95377f39cbc7293c4944ad Fingerprint: j2J+naFDImJtfoHXiff8r9JfYiNalTd/OcvHKTxJRK0= -Save to /home/marco/.politeia/identity.json or ctrl-c to abort -Identity saved to: /home/marco/.politeia/identity.json +Save to /home/user/.politeia/identity.json or ctrl-c to abort +Identity saved to: /home/user/.politeia/identity.json ``` ## Add a new record diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 99b24fc70..d5f9019eb 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -9,6 +9,7 @@ import ( "crypto/x509" "encoding/hex" "encoding/json" + "errors" "fmt" "io/ioutil" "net/http" @@ -207,14 +208,21 @@ func (p *politeia) handleNotFound(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusNotFound, v1.ServerErrorReply{}) } -func (p *politeia) respondWithUserError(w http.ResponseWriter, - errorCode v1.ErrorStatusT, errorContext []string) { +func (p *politeia) respondWithUserError(w http.ResponseWriter, errorCode v1.ErrorStatusT, errorContext []string) { util.RespondWithJSON(w, http.StatusBadRequest, v1.UserErrorReply{ ErrorCode: errorCode, ErrorContext: errorContext, }) } +func (p *politeia) respondWithPluginUserError(w http.ResponseWriter, plugin string, errorCode int, errorContext []string) { + util.RespondWithJSON(w, http.StatusBadRequest, v1.PluginUserErrorReply{ + Plugin: plugin, + ErrorCode: errorCode, + ErrorContext: errorContext, + }) +} + func (p *politeia) respondWithServerError(w http.ResponseWriter, errorCode int64) { log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) util.RespondWithJSON(w, http.StatusInternalServerError, v1.ServerErrorReply{ @@ -275,6 +283,16 @@ func (p *politeia) newRecord(w http.ResponseWriter, r *http.Request) { return } + // Check for plugin error + var e backend.PluginUserError + if errors.As(err, &e) { + log.Debugf("%v plugin user error: %v %v", + remoteAddr(r), e.PluginID, e.ErrorCode) + p.respondWithPluginUserError(w, e.PluginID, e.ErrorCode, + e.ErrorContext) + return + } + // Generic internal error. errorCode := time.Now().Unix() log.Errorf("%v New record error code %v: %v", remoteAddr(r), @@ -376,7 +394,15 @@ func (p *politeia) updateRecord(w http.ResponseWriter, r *http.Request, vetted b contentErr.ErrorContext) return } - + // Check for plugin error + var e backend.PluginUserError + if errors.As(err, &e) { + log.Debugf("%v plugin user error: %v %v", + remoteAddr(r), e.PluginID, e.ErrorCode) + p.respondWithPluginUserError(w, e.PluginID, e.ErrorCode, + e.ErrorContext) + return + } // Generic internal error. errorCode := time.Now().Unix() log.Errorf("%v Update %v record error code %v: %v", @@ -687,6 +713,15 @@ func (p *politeia) setVettedStatus(w http.ResponseWriter, r *http.Request) { p.respondWithUserError(w, v1.ErrorStatusInvalidRecordStatusTransition, nil) return } + // Check for plugin error + var e backend.PluginUserError + if errors.As(err, &e) { + log.Debugf("%v plugin user error: %v %v", + remoteAddr(r), e.PluginID, e.ErrorCode) + p.respondWithPluginUserError(w, e.PluginID, e.ErrorCode, + e.ErrorContext) + return + } // Generic internal error. errorCode := time.Now().Unix() log.Errorf("%v Set status error code %v: %v", @@ -750,6 +785,15 @@ func (p *politeia) setUnvettedStatus(w http.ResponseWriter, r *http.Request) { p.respondWithUserError(w, v1.ErrorStatusInvalidRecordStatusTransition, nil) return } + // Check for plugin error + var e backend.PluginUserError + if errors.As(err, &e) { + log.Debugf("%v plugin user error: %v %v", + remoteAddr(r), e.PluginID, e.ErrorCode) + p.respondWithPluginUserError(w, e.PluginID, e.ErrorCode, + e.ErrorContext) + return + } // Generic internal error. errorCode := time.Now().Unix() log.Errorf("%v Set unvetted status error code %v: %v", @@ -815,7 +859,15 @@ func (p *politeia) updateVettedMetadata(w http.ResponseWriter, r *http.Request) contentErr.ErrorContext) return } - + // Check for plugin error + var e backend.PluginUserError + if errors.As(err, &e) { + log.Debugf("%v plugin user error: %v %v", + remoteAddr(r), e.PluginID, e.ErrorCode) + p.respondWithPluginUserError(w, e.PluginID, e.ErrorCode, + e.ErrorContext) + return + } // Generic internal error. errorCode := time.Now().Unix() log.Errorf("%v Update vetted metadata error code %v: %v", @@ -876,6 +928,15 @@ func (p *politeia) updateUnvettedMetadata(w http.ResponseWriter, r *http.Request contentErr.ErrorContext) return } + // Check for plugin error + var e backend.PluginUserError + if errors.As(err, &e) { + log.Debugf("%v plugin user error: %v %v", + remoteAddr(r), e.PluginID, e.ErrorCode) + p.respondWithPluginUserError(w, e.PluginID, e.ErrorCode, + e.ErrorContext) + return + } // Generic internal error. errorCode := time.Now().Unix() log.Errorf("%v update unvetted metadata error code %v: %v", @@ -940,6 +1001,16 @@ func (p *politeia) pluginCommand(w http.ResponseWriter, r *http.Request) { payload, err := p.backend.Plugin(pc.ID, pc.Command, pc.CommandID, pc.Payload) if err != nil { + // Check for a user error + var e backend.PluginUserError + if errors.As(err, &e) { + log.Debugf("%v plugin user error: %v %v", + remoteAddr(r), e.PluginID, e.ErrorCode) + p.respondWithPluginUserError(w, e.PluginID, e.ErrorCode, + e.ErrorContext) + return + } + // Generic internal error. errorCode := time.Now().Unix() log.Errorf("%v %v: backend plugin failed with "+ @@ -1169,7 +1240,7 @@ func _main() error { // Setup plugins // TODO fix this - loadedCfg.Plugins = []string{"comments", "dcrdata", "pi", "ticketvote"} + // loadedCfg.Plugins = []string{"comments", "dcrdata", "pi", "ticketvote"} if len(loadedCfg.Plugins) > 0 { // Set plugin routes. Requires auth. p.addRoute(http.MethodPost, v1.PluginCommandRoute, p.pluginCommand, diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 1b72d5f58..d6c8fded7 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -112,7 +112,7 @@ const ( // Error status codes ErrorStatusInvalid ErrorStatusT = 0 - ErrorStatusInvalidInput ErrorStatusT = 1 + ErrorStatusInputInvalid ErrorStatusT = 1 // User errors ErrorStatusUserRegistrationNotPaid ErrorStatusT = 2 @@ -125,35 +125,35 @@ const ( ErrorStatusSignatureInvalid ErrorStatusT = 101 // Proposal errors - ErrorStatusFileCountInvalid ErrorStatusT = 202 - ErrorStatusFileNameInvalid ErrorStatusT = 203 - ErrorStatusFileMIMEInvalid ErrorStatusT = 204 - ErrorStatusFileDigestInvalid ErrorStatusT = 205 - ErrorStatusFilePayloadInvalid ErrorStatusT = 206 - ErrorStatusIndexFileNameInvalid ErrorStatusT = 207 - ErrorStatusIndexFileCountInvalid ErrorStatusT = 207 - ErrorStatusIndexFileSizeInvalid ErrorStatusT = 208 - ErrorStatusTextFileCountInvalid ErrorStatusT = 209 - ErrorStatusImageFileCountInvalid ErrorStatusT = 210 - ErrorStatusImageFileSizeInvalid ErrorStatusT = 211 - ErrorStatusMetadataCountInvalid ErrorStatusT = 212 - ErrorStatusMetadataHintInvalid ErrorStatusT = 213 - ErrorStatusMetadataDigestInvalid ErrorStatusT = 214 - ErrorStatusMetadataPayloadInvalid ErrorStatusT = 215 - ErrorStatusPropNameInvalid ErrorStatusT = 216 - ErrorStatusPropLinkToInvalid ErrorStatusT = 217 - ErrorStatusPropLinkByInvalid ErrorStatusT = 218 - ErrorStatusPropTokenInvalid ErrorStatusT = 219 - ErrorStatusPropNotFound ErrorStatusT = 220 - ErrorStatusPropStateInvalid ErrorStatusT = 221 - ErrorStatusPropStatusInvalid ErrorStatusT = 222 - ErrorStatusPropStatusChangeInvalid ErrorStatusT = 223 - ErrorStatusPropStatusChangeReasonInvalid ErrorStatusT = 224 - ErrorStatusPropPageSizeExceeded ErrorStatusT = 225 + // TODO number error codes + ErrorStatusFileCountInvalid ErrorStatusT = 200 + ErrorStatusFileNameInvalid ErrorStatusT = iota + ErrorStatusFileMIMEInvalid + ErrorStatusFileDigestInvalid + ErrorStatusFilePayloadInvalid + ErrorStatusIndexFileNameInvalid + ErrorStatusIndexFileCountInvalid + ErrorStatusIndexFileSizeInvalid + ErrorStatusTextFileCountInvalid + ErrorStatusImageFileCountInvalid + ErrorStatusImageFileSizeInvalid + ErrorStatusMetadataCountInvalid + ErrorStatusMetadataDigestInvalid + ErrorStatusMetadataPayloadInvalid + ErrorStatusPropMetadataNotFound + ErrorStatusPropNameInvalid + ErrorStatusPropLinkToInvalid + ErrorStatusPropLinkByInvalid + ErrorStatusPropTokenInvalid + ErrorStatusPropNotFound + ErrorStatusPropStateInvalid + ErrorStatusPropStatusInvalid + ErrorStatusPropStatusChangeInvalid + ErrorStatusPropStatusChangeReasonInvalid + ErrorStatusPropPageSizeExceeded // Comment errors - // TODO number error codes - ErrorStatusCommentTextInvalid ErrorStatusT = iota + ErrorStatusCommentTextInvalid ErrorStatusCommentParentIDInvalid ErrorStatusCommentVoteInvalid ErrorStatusCommentNotFound diff --git a/politeiawww/api/www/v1/v1.go b/politeiawww/api/www/v1/v1.go index 275d97437..7ae7916cf 100644 --- a/politeiawww/api/www/v1/v1.go +++ b/politeiawww/api/www/v1/v1.go @@ -52,7 +52,15 @@ const ( RouteUnauthenticatedWebSocket = "/ws" RouteAuthenticatedWebSocket = "/aws" - // XXX these routes should be user routes + // TODO the user payment routes are a mess. The route naming + // convention should be updated to be the following. + // RouteUserRegistrationPayment = "/user/payments/registration" + // RouteUserProposalPaywall = "/user/payments/paywall" + // RouteUserProposalPaywallTx = "/user/payments/paywalltx" + // RouteUserProposalCredits = "/user/payments/credits" + // RouteUserPaymentsRescan = "/user/payments/rescan" + + // TODO these routes should be user routes RouteProposalPaywallDetails = "/proposals/paywall" RouteProposalPaywallPayment = "/proposals/paywallpayment" diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 42efb8950..96a25a3e7 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -849,11 +849,8 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu // a ProposalMetadata. switch { case len(metadata) == 0: - e := fmt.Sprintf("metadata with hint %v not found", - www.HintProposalMetadata) return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusMetadataCountInvalid, - ErrorContext: []string{e}, + ErrorCode: pi.ErrorStatusPropMetadataNotFound, } case len(metadata) > 1: e := fmt.Sprintf("metadata should only contain %v", @@ -865,10 +862,8 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu } md := metadata[0] if md.Hint != www.HintProposalMetadata { - e := fmt.Sprintf("unknown metadata hint %v", md.Hint) return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusMetadataHintInvalid, - ErrorContext: []string{e}, + ErrorCode: pi.ErrorStatusPropMetadataNotFound, } } @@ -1340,7 +1335,7 @@ func (p *politeiawww) handleProposalNew(w http.ResponseWriter, r *http.Request) if err := decoder.Decode(&pn); err != nil { respondWithPiError(w, r, "handleProposalNew: unmarshal", pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInvalidInput, + ErrorCode: pi.ErrorStatusInputInvalid, }) return } @@ -1370,7 +1365,7 @@ func (p *politeiawww) handleProposalEdit(w http.ResponseWriter, r *http.Request) if err := decoder.Decode(&pe); err != nil { respondWithPiError(w, r, "handleProposalEdit: unmarshal", pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInvalidInput, + ErrorCode: pi.ErrorStatusInputInvalid, }) return } @@ -1400,7 +1395,7 @@ func (p *politeiawww) handleProposalSetStatus(w http.ResponseWriter, r *http.Req if err := decoder.Decode(&pss); err != nil { respondWithPiError(w, r, "handleProposalSetStatus: unmarshal", pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInvalidInput, + ErrorCode: pi.ErrorStatusInputInvalid, }) return } @@ -1503,7 +1498,7 @@ func (p *politeiawww) handleCommentNew(w http.ResponseWriter, r *http.Request) { if err := decoder.Decode(&cn); err != nil { respondWithPiError(w, r, "handleCommentNew: unmarshal", pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInvalidInput, + ErrorCode: pi.ErrorStatusInputInvalid, }) return } @@ -1556,7 +1551,7 @@ func (p *politeiawww) processCommentVote(cv pi.CommentVote, usr user.User) (*pi. } } - // Call the pi plugin to add new comment + // Call the pi plugin to vote on a comment reply, err := p.commentVotePi(piplugin.CommentVote{ UserID: usr.ID.String(), Token: cv.Token, @@ -1585,7 +1580,7 @@ func (p *politeiawww) handleCommentVote(w http.ResponseWriter, r *http.Request) if err := decoder.Decode(&cv); err != nil { respondWithPiError(w, r, "handleCommentVote: unmarshal", pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInvalidInput, + ErrorCode: pi.ErrorStatusInputInvalid, }) return } @@ -1650,7 +1645,7 @@ func (p *politeiawww) handleComments(w http.ResponseWriter, r *http.Request) { if err := decoder.Decode(&c); err != nil { respondWithPiError(w, r, "handleComments: unmarshal", pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInvalidInput, + ErrorCode: pi.ErrorStatusInputInvalid, }) return } @@ -1696,7 +1691,7 @@ func (p *politeiawww) handleCommentVotes(w http.ResponseWriter, r *http.Request) if err := decoder.Decode(&cvs); err != nil { respondWithPiError(w, r, "handleCommentVotes: unmarshal", pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInvalidInput, + ErrorCode: pi.ErrorStatusInputInvalid, }) return } @@ -1753,7 +1748,7 @@ func (p *politeiawww) handleCommentCensor(w http.ResponseWriter, r *http.Request if err := decoder.Decode(&cc); err != nil { respondWithPiError(w, r, "handleCommentCensor: unmarshal", pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInvalidInput, + ErrorCode: pi.ErrorStatusInputInvalid, }) return } @@ -1818,7 +1813,7 @@ func (p *politeiawww) handleVoteAuthorize(w http.ResponseWriter, r *http.Request if err := decoder.Decode(&va); err != nil { respondWithPiError(w, r, "handleVoteAuthorize: unmarshal", pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInvalidInput, + ErrorCode: pi.ErrorStatusInputInvalid, }) return } @@ -1899,7 +1894,7 @@ func (p *politeiawww) handleVoteStart(w http.ResponseWriter, r *http.Request) { if err := decoder.Decode(&vs); err != nil { respondWithPiError(w, r, "handleVoteStart: unmarshal", pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInvalidInput, + ErrorCode: pi.ErrorStatusInputInvalid, }) return } @@ -1955,7 +1950,7 @@ func (p *politeiawww) handleVoteStartRunoff(w http.ResponseWriter, r *http.Reque if err := decoder.Decode(&vsr); err != nil { respondWithPiError(w, r, "handleVoteStartRunoff: unmarshal", pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInvalidInput, + ErrorCode: pi.ErrorStatusInputInvalid, }) return } @@ -2036,7 +2031,7 @@ func (p *politeiawww) handleVoteBallot(w http.ResponseWriter, r *http.Request) { if err := decoder.Decode(&vb); err != nil { respondWithPiError(w, r, "handleVoteBallot: unmarshal", pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInvalidInput, + ErrorCode: pi.ErrorStatusInputInvalid, }) return } @@ -2155,7 +2150,7 @@ func (p *politeiawww) handleVotes(w http.ResponseWriter, r *http.Request) { if err := decoder.Decode(&v); err != nil { respondWithPiError(w, r, "handleVotes: unmarshal", pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInvalidInput, + ErrorCode: pi.ErrorStatusInputInvalid, }) return } @@ -2202,7 +2197,7 @@ func (p *politeiawww) handleVoteResults(w http.ResponseWriter, r *http.Request) if err := decoder.Decode(&vr); err != nil { respondWithPiError(w, r, "handleVoteResults: unmarshal", pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInvalidInput, + ErrorCode: pi.ErrorStatusInputInvalid, }) return } @@ -2291,7 +2286,7 @@ func (p *politeiawww) handleVoteSummaries(w http.ResponseWriter, r *http.Request if err := decoder.Decode(&vs); err != nil { respondWithPiError(w, r, "handleVoteSummaries: unmarshal", pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInvalidInput, + ErrorCode: pi.ErrorStatusInputInvalid, }) return } @@ -2332,7 +2327,7 @@ func (p *politeiawww) handleVoteInventory(w http.ResponseWriter, r *http.Request if err := decoder.Decode(&vi); err != nil { respondWithPiError(w, r, "handleVoteInventory: unmarshal", pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInvalidInput, + ErrorCode: pi.ErrorStatusInputInvalid, }) return } diff --git a/politeiawww/user/cockroachdb/cockroachdb.go b/politeiawww/user/cockroachdb/cockroachdb.go index 2fa1eaeb5..af4b35087 100644 --- a/politeiawww/user/cockroachdb/cockroachdb.go +++ b/politeiawww/user/cockroachdb/cockroachdb.go @@ -857,7 +857,7 @@ func New(host, network, sslRootCert, sslCert, sslKey, encryptionKey string) (*co u.String(), err) } - log.Infof("UserDB host: %v", h) + log.Infof("Host: %v", h) // Load encryption key key, err := loadEncryptionKey(encryptionKey) diff --git a/politeiawww/www.go b/politeiawww/www.go index 9a1a18e95..38b89d163 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -784,20 +784,6 @@ func _main() error { router := mux.NewRouter() router.Use(recoverMiddleware) - // Setup smtp client - smtp, err := newSMTP(loadedCfg.MailHost, loadedCfg.MailUser, - loadedCfg.MailPass, loadedCfg.MailAddress, loadedCfg.SystemCerts, - loadedCfg.SMTPSkipVerify) - if err != nil { - return fmt.Errorf("newSMTP: %v", err) - } - - // Setup politeiad client - client, err := util.NewClient(false, loadedCfg.RPCCert) - if err != nil { - return err - } - // Setup user database var userDB user.Database switch loadedCfg.UserDB { @@ -857,6 +843,20 @@ func _main() error { } sessions := newSessionStore(userDB, sessionMaxAge, cookieKey) + // Setup smtp client + smtp, err := newSMTP(loadedCfg.MailHost, loadedCfg.MailUser, + loadedCfg.MailPass, loadedCfg.MailAddress, loadedCfg.SystemCerts, + loadedCfg.SMTPSkipVerify) + if err != nil { + return fmt.Errorf("newSMTP: %v", err) + } + + // Setup politeiad client + client, err := util.NewClient(false, loadedCfg.RPCCert) + if err != nil { + return err + } + // Setup application context p := &politeiawww{ cfg: loadedCfg, From c630f36f157e1ac4bee5bdae38c412323e310fa8 Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Mon, 28 Sep 2020 17:14:03 -0300 Subject: [PATCH 116/449] politeiawww: refactor user payments routes. --- politeiawww/api/www/v1/v1.go | 120 +++++++------ politeiawww/cmd/piwww/userpaymentverify.go | 2 +- politeiawww/cmd/shared/client.go | 34 ++-- politeiawww/paywall.go | 2 +- politeiawww/politeiawww.go | 53 ------ politeiawww/proposals.go | 68 -------- politeiawww/user.go | 192 ++++++++++++++------- politeiawww/userwww.go | 168 +++++++++++------- 8 files changed, 311 insertions(+), 328 deletions(-) diff --git a/politeiawww/api/www/v1/v1.go b/politeiawww/api/www/v1/v1.go index 7ae7916cf..0c1900ed7 100644 --- a/politeiawww/api/www/v1/v1.go +++ b/politeiawww/api/www/v1/v1.go @@ -41,9 +41,6 @@ const ( RouteChangePassword = "/user/password/change" RouteResetPassword = "/user/password/reset" RouteVerifyResetPassword = "/user/password/reset/verify" - RouteUserProposalCredits = "/user/proposals/credits" - RouteVerifyUserPayment = "/user/verifypayment" - RouteUserPaymentsRescan = "/user/payments/rescan" RouteManageUser = "/user/manage" RouteEditUser = "/user/edit" RouteSetTOTP = "/user/totp" @@ -52,17 +49,12 @@ const ( RouteUnauthenticatedWebSocket = "/ws" RouteAuthenticatedWebSocket = "/aws" - // TODO the user payment routes are a mess. The route naming - // convention should be updated to be the following. - // RouteUserRegistrationPayment = "/user/payments/registration" - // RouteUserProposalPaywall = "/user/payments/paywall" - // RouteUserProposalPaywallTx = "/user/payments/paywalltx" - // RouteUserProposalCredits = "/user/payments/credits" - // RouteUserPaymentsRescan = "/user/payments/rescan" - - // TODO these routes should be user routes - RouteProposalPaywallDetails = "/proposals/paywall" - RouteProposalPaywallPayment = "/proposals/paywallpayment" + // User payments routes + RouteUserRegistrationPayment = "/user/payments/registration" + RouteUserProposalPaywall = "/user/payments/paywall" + RouteUserProposalPaywallTx = "/user/payments/paywalltx" + RouteUserProposalCredits = "/user/payments/credits" + RouteUserPaymentsRescan = "/user/payments/rescan" // The following routes WILL BE DEPRECATED in the near future and // should not be used. The pi v1 API should be used instead. @@ -703,33 +695,6 @@ type VerifyResetPassword struct { // command. type VerifyResetPasswordReply struct{} -// UserProposalCredits is used to request a list of all the user's unspent -// proposal credits and a list of all of the user's spent proposal credits. -// A spent credit means that the credit was used to submit a proposal. Spent -// credits have a proposal censorship token associated with them to signify -// that they have been spent. -type UserProposalCredits struct{} - -// UserProposalCredits is used to reply to the UserProposalCredits command. -type UserProposalCreditsReply struct { - UnspentCredits []ProposalCredit `json:"unspentcredits"` // credits that the user has purchased, but have not yet been used to submit proposals (credit price in atoms) - SpentCredits []ProposalCredit `json:"spentcredits"` // credits that the user has purchased and that have already been used to submit proposals (credit price in atoms) -} - -// UserPaymentsRescan allows an admin to rescan a user's paywall address to -// check for any payments that may have been missed by paywall polling. Any -// proposal credits that are created as a result of the rescan are returned in -// the UserPaymentsRescanReply. This call isn't RESTful, but a PUT request is -// used since it's idempotent. -type UserPaymentsRescan struct { - UserID string `json:"userid"` // ID of user to rescan -} - -// UserPaymentsRescanReply is used to reply to the UserPaymentsRescan command. -type UserPaymentsRescanReply struct { - NewCredits []ProposalCredit `json:"newcredits"` // Credits that were created by the rescan -} - // UserProposals is used to request a list of proposals that the // user has submitted. This command optionally takes either a Before // or After parameter, which specify a proposal's censorship token. @@ -750,19 +715,6 @@ type UserProposalsReply struct { NumOfProposals int `json:"numofproposals"` // number of proposals submitted by the user } -// VerifyUserPayment is used to request the server to check for the -// provided transaction on the Decred blockchain and verify that it -// satisfies the requirements for a user to pay his registration fee. -type VerifyUserPayment struct { -} - -type VerifyUserPaymentReply struct { - HasPaid bool `json:"haspaid"` - PaywallAddress string `json:"paywalladdress"` // Registration paywall address - PaywallAmount uint64 `json:"paywallamount"` // Registration paywall amount in atoms - PaywallTxNotBefore int64 `json:"paywalltxnotbefore"` // Minimum timestamp for paywall tx -} - // Users is used to request a list of users given a filter. type Users struct { Username string `json:"username"` // String which should match or partially match a username @@ -820,30 +772,74 @@ type LogoutReply struct{} // for this endpoint. type Me struct{} -// ProposalPaywallDetails is used to request proposal paywall details from the +// UserRegistrationPayment is used to request the server to check for the +// provided transaction on the Decred blockchain and verify that it +// satisfies the requirements for a user to pay his registration fee. +type UserRegistrationPayment struct{} + +// UserRegistrationPaymentReply is used to reply to the UserRegistrationPayment +// command. +type UserRegistrationPaymentReply struct { + HasPaid bool `json:"haspaid"` + PaywallAddress string `json:"paywalladdress"` // Registration paywall address + PaywallAmount uint64 `json:"paywallamount"` // Registration paywall amount in atoms + PaywallTxNotBefore int64 `json:"paywalltxnotbefore"` // Minimum timestamp for paywall tx +} + +// UserProposalPaywall is used to request proposal paywall details from the // server that the user needs in order to purchase paywall credits. -type ProposalPaywallDetails struct{} +type UserProposalPaywall struct{} -// ProposalPaywallDetailsReply is used to reply to the ProposalPaywallDetails +// UserProposalPaywallReply is used to reply to the ProposalPaywallDetails // command. -type ProposalPaywallDetailsReply struct { +type UserProposalPaywallReply struct { CreditPrice uint64 `json:"creditprice"` // Cost per proposal credit in atoms PaywallAddress string `json:"paywalladdress"` // Proposal paywall address PaywallTxNotBefore int64 `json:"paywalltxnotbefore"` // Minimum timestamp for paywall tx } -// ProposalPaywallPayment is used to request payment details for a pending +// UserProposalPaywallTx is used to request payment details for a pending // proposal paywall payment. -type ProposalPaywallPayment struct{} +type UserProposalPaywallTx struct{} -// ProposalPaywallPaymentReply is used to reply to the ProposalPaywallPayment +// UserProposalPaywallTxReply is used to reply to the ProposalPaywallPayment // command. -type ProposalPaywallPaymentReply struct { +type UserProposalPaywallTxReply struct { TxID string `json:"txid"` // Transaction ID TxAmount uint64 `json:"amount"` // Transaction amount in atoms Confirmations uint64 `json:"confirmations"` // Number of block confirmations } +// UserProposalCredits is used to request a list of all the user's unspent +// proposal credits and a list of all of the user's spent proposal credits. +// A spent credit means that the credit was used to submit a proposal. Spent +// credits have a proposal censorship token associated with them to signify +// that they have been spent. +type UserProposalCredits struct{} + +// UserProposalCreditsReply is used to reply to the UserProposalCredits command. +// It contains unspent credits that the user purchased but did not yet use, and +// the credits the user already spent to submit proposals. +type UserProposalCreditsReply struct { + UnspentCredits []ProposalCredit `json:"unspentcredits"` + SpentCredits []ProposalCredit `json:"spentcredits"` +} + +// UserPaymentsRescan allows an admin to rescan a user's paywall address to +// check for any payments that may have been missed by paywall polling. Any +// proposal credits that are created as a result of the rescan are returned in +// the UserPaymentsRescanReply. This call isn't RESTful, but a PUT request is +// used since it's idempotent. +type UserPaymentsRescan struct { + UserID string `json:"userid"` // ID of user to rescan +} + +// UserPaymentsRescanReply is used to reply to the UserPaymentsRescan command. +// Returns the credits that were created by the rescan. +type UserPaymentsRescanReply struct { + NewCredits []ProposalCredit `json:"newcredits"` +} + // NewProposal attempts to submit a new proposal. // // Metadata is required to include a ProposalMetadata for all proposal diff --git a/politeiawww/cmd/piwww/userpaymentverify.go b/politeiawww/cmd/piwww/userpaymentverify.go index a006d32a8..ff0eccff7 100644 --- a/politeiawww/cmd/piwww/userpaymentverify.go +++ b/politeiawww/cmd/piwww/userpaymentverify.go @@ -12,7 +12,7 @@ type userPaymentVerifyCmd struct{} // Execute executes the verify user payment command. func (cmd *userPaymentVerifyCmd) Execute(args []string) error { - vupr, err := client.VerifyUserPayment() + vupr, err := client.UserPaymentVerify() if err != nil { return err } diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index bb8e725b0..d2138ad2a 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -677,14 +677,14 @@ func (c *Client) VerifyResetPassword(vrp www.VerifyResetPassword) (*www.VerifyRe // ProposalPaywallDetails retrieves proposal credit paywall information for the // logged in user. -func (c *Client) ProposalPaywallDetails() (*www.ProposalPaywallDetailsReply, error) { +func (c *Client) ProposalPaywallDetails() (*www.UserProposalPaywallReply, error) { responseBody, err := c.makeRequest(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteProposalPaywallDetails, nil) + www.RouteUserProposalPaywall, nil) if err != nil { return nil, err } - var ppdr www.ProposalPaywallDetailsReply + var ppdr www.UserProposalPaywallReply err = json.Unmarshal(responseBody, &ppdr) if err != nil { return nil, fmt.Errorf("unmarshal ProposalPaywalDetailsReply: %v", err) @@ -1454,29 +1454,29 @@ func (c *Client) VoteStartRunoff(vsr pi.VoteStartRunoff) (*pi.VoteStartRunoffRep return &vsrr, nil } -// VerifyUserPayment checks whether the logged in user has paid their user +// UserPaymentVerify checks whether the logged in user has paid their user // registration fee. -func (c *Client) VerifyUserPayment() (*www.VerifyUserPaymentReply, error) { +func (c *Client) UserPaymentVerify() (*www.UserRegistrationPaymentReply, error) { responseBody, err := c.makeRequest(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteVerifyUserPayment, nil) + www.RouteUserRegistrationPayment, nil) if err != nil { return nil, err } - var vupr www.VerifyUserPaymentReply - err = json.Unmarshal(responseBody, &vupr) + var urpr www.UserRegistrationPaymentReply + err = json.Unmarshal(responseBody, &urpr) if err != nil { - return nil, fmt.Errorf("unmarshal VerifyUserPaymentReply: %v", err) + return nil, fmt.Errorf("unmarshal UserRegistrationPaymentReply: %v", err) } if c.cfg.Verbose { - err := prettyPrintJSON(vupr) + err := prettyPrintJSON(urpr) if err != nil { return nil, err } } - return &vupr, nil + return &urpr, nil } // VoteResults retrieves the vote results for the specified proposal. @@ -1826,27 +1826,27 @@ func (c *Client) VerifyUpdateUserKey(vuuk *www.VerifyUpdateUserKey) (*www.Verify // ProposalPaywallPayment retrieves payment details of any pending proposal // credit payment from the logged in user. -func (c *Client) ProposalPaywallPayment() (*www.ProposalPaywallPaymentReply, error) { +func (c *Client) ProposalPaywallPayment() (*www.UserProposalPaywallTxReply, error) { responseBody, err := c.makeRequest(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteProposalPaywallPayment, nil) + www.RouteUserProposalPaywallTx, nil) if err != nil { return nil, err } - var pppr www.ProposalPaywallPaymentReply - err = json.Unmarshal(responseBody, &pppr) + var upptxr www.UserProposalPaywallTxReply + err = json.Unmarshal(responseBody, &upptxr) if err != nil { return nil, fmt.Errorf("unmarshal ProposalPaywallPaymentReply: %v", err) } if c.cfg.Verbose { - err := prettyPrintJSON(pppr) + err := prettyPrintJSON(upptxr) if err != nil { return nil, err } } - return &pppr, nil + return &upptxr, nil } // UserPaymentsRescan scans the specified user's paywall address and makes sure diff --git a/politeiawww/paywall.go b/politeiawww/paywall.go index 4bb4ee225..09b6b1eb7 100644 --- a/politeiawww/paywall.go +++ b/politeiawww/paywall.go @@ -28,7 +28,7 @@ type paywallPoolMember struct { const ( // paywallExpiryDuration is the amount of time the server will watch a paywall address // for transactions. It gets reset when the user logs in or makes a call to - // RouteVerifyUserPayment. + // RouteUserRegistrationPayment. paywallExpiryDuration = time.Hour * 24 // paywallCheckGap is the amount of time the server sleeps after polling for diff --git a/politeiawww/politeiawww.go b/politeiawww/politeiawww.go index caa1fb164..b9697c85d 100644 --- a/politeiawww/politeiawww.go +++ b/politeiawww/politeiawww.go @@ -344,51 +344,6 @@ func (p *politeiawww) handleBatchVoteSummary(w http.ResponseWriter, r *http.Requ util.RespondWithJSON(w, http.StatusOK, reply) } -// handleProposalPaywallDetails returns paywall details that allows the user to -// purchase proposal credits. -func (p *politeiawww) handleProposalPaywallDetails(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleProposalPaywallDetails") - - user, err := p.getSessionUser(w, r) - if err != nil { - RespondWithError(w, r, 0, - "handleProposalPaywallDetails: getSessionUser %v", err) - return - } - - reply, err := p.processProposalPaywallDetails(user) - if err != nil { - RespondWithError(w, r, 0, - "handleProposalPaywallDetails: processProposalPaywallDetails %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, reply) -} - -// handleProposalPaywallPayment returns the payment details for a pending -// proposal paywall payment. -func (p *politeiawww) handleProposalPaywallPayment(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleProposalPaywallPayment") - - user, err := p.getSessionUser(w, r) - if err != nil { - RespondWithError(w, r, 0, - "handleProposalPaywallPayment: getSessionUser %v", err) - return - } - - reply, err := p.processProposalPaywallPayment(user) - if err != nil { - RespondWithError(w, r, 0, - "handleProposalPaywallPayment: "+ - "processProposalPaywallPayment %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, reply) -} - // websocketPing is used to verify that websockets are operational. func (p *politeiawww) websocketPing(id string) { log.Tracef("websocketPing %v", id) @@ -666,14 +621,6 @@ func (p *politeiawww) setPoliteiaWWWRoutes() { www.RouteBatchVoteSummary, p.handleBatchVoteSummary, permissionPublic) - // Routes that require being logged in. - p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteProposalPaywallDetails, p.handleProposalPaywallDetails, - permissionLogin) - p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteProposalPaywallPayment, p.handleProposalPaywallPayment, - permissionLogin) - // Unauthenticated websocket p.addRoute("", www.PoliteiaWWWAPIRoute, www.RouteUnauthenticatedWebSocket, p.handleUnauthenticatedWebsocket, diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index cbce272ec..8edc3ed6e 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -467,74 +467,6 @@ func (p *politeiawww) processCastVotes(ballot *www.Ballot) (*www.BallotReply, er return &res, nil } -// processProposalPaywallDetails returns a proposal paywall that enables the -// the user to purchase proposal credits. The user can only have one paywall -// active at a time. If no paywall currently exists, a new one is created and -// the user is added to the paywall pool. -func (p *politeiawww) processProposalPaywallDetails(u *user.User) (*www.ProposalPaywallDetailsReply, error) { - log.Tracef("processProposalPaywallDetails") - - // Ensure paywall is enabled - if !p.paywallIsEnabled() { - return &www.ProposalPaywallDetailsReply{}, nil - } - - // Proposal paywalls cannot be generated until the user has paid their - // user registration fee. - if !p.userHasPaid(*u) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusUserNotPaid, - } - } - - var pp *user.ProposalPaywall - if p.userHasValidProposalPaywall(u) { - // Don't create a new paywall if a valid one already exists. - pp = p.mostRecentProposalPaywall(u) - } else { - // Create a new paywall. - var err error - pp, err = p.generateProposalPaywall(u) - if err != nil { - return nil, err - } - } - - return &www.ProposalPaywallDetailsReply{ - CreditPrice: pp.CreditPrice, - PaywallAddress: pp.Address, - PaywallTxNotBefore: pp.TxNotBefore, - }, nil -} - -// processProposalPaywallPayment checks if the user has a pending paywall -// payment and returns the payment details if one is found. -func (p *politeiawww) processProposalPaywallPayment(u *user.User) (*www.ProposalPaywallPaymentReply, error) { - log.Tracef("processProposalPaywallPayment") - - var ( - txID string - txAmount uint64 - confirmations uint64 - ) - - p.RLock() - defer p.RUnlock() - - poolMember, ok := p.userPaywallPool[u.ID] - if ok { - txID = poolMember.txID - txAmount = poolMember.txAmount - confirmations = poolMember.txConfirmations - } - - return &www.ProposalPaywallPaymentReply{ - TxID: txID, - TxAmount: txAmount, - Confirmations: confirmations, - }, nil -} - // validateAuthorizeVote validates the authorize vote fields. A UserError is // returned if any of the validation fails. func validateAuthorizeVote(av www.AuthorizeVote, u user.User, pr www.ProposalRecord, vs www.VoteSummary) error { diff --git a/politeiawww/user.go b/politeiawww/user.go index 17b5761e4..ce4a3efe9 100644 --- a/politeiawww/user.go +++ b/politeiawww/user.go @@ -1597,25 +1597,6 @@ func convertProposalCreditFromUserDB(credit user.ProposalCredit) www.ProposalCre } } -// processUserProposalCredits returns a list of the user's unspent proposal -// credits and a list of the user's spent proposal credits. -func processUserProposalCredits(u *user.User) (*www.UserProposalCreditsReply, error) { - // Convert from database proposal credits to www proposal credits. - upc := make([]www.ProposalCredit, len(u.UnspentProposalCredits)) - for i, credit := range u.UnspentProposalCredits { - upc[i] = convertProposalCreditFromUserDB(credit) - } - spc := make([]www.ProposalCredit, len(u.SpentProposalCredits)) - for i, credit := range u.SpentProposalCredits { - spc[i] = convertProposalCreditFromUserDB(credit) - } - - return &www.UserProposalCreditsReply{ - UnspentCredits: upc, - SpentCredits: spc, - }, nil -} - // _logAdminAction logs a string to the admin log file. // // This function must be called WITH the mutex held. @@ -1852,6 +1833,135 @@ func (p *politeiawww) processUsers(users *www.Users, isAdmin bool) (*www.UsersRe }, nil } +// processUserRegistrationPayment verifies that the provided transaction +// meets the minimum requirements to mark the user as paid, and then does +// that in the user database. +func (p *politeiawww) processUserRegistrationPayment(u *user.User) (*www.UserRegistrationPaymentReply, error) { + var reply www.UserRegistrationPaymentReply + if p.userHasPaid(*u) { + reply.HasPaid = true + return &reply, nil + } + + if paywallHasExpired(u.NewUserPaywallPollExpiry) { + err := p.GenerateNewUserPaywall(u) + if err != nil { + return nil, err + } + reply.PaywallAddress = u.NewUserPaywallAddress + reply.PaywallAmount = u.NewUserPaywallAmount + reply.PaywallTxNotBefore = u.NewUserPaywallTxNotBefore + return &reply, nil + } + + tx, _, err := util.FetchTxWithBlockExplorers(u.NewUserPaywallAddress, + u.NewUserPaywallAmount, u.NewUserPaywallTxNotBefore, + p.cfg.MinConfirmationsRequired, p.dcrdataHostHTTP()) + if err != nil { + return nil, err + } + + if tx != "" { + reply.HasPaid = true + + err = p.updateUserAsPaid(u, tx) + if err != nil { + return nil, err + } + } else { + // TODO: Add the user to the in-memory pool. + } + + return &reply, nil +} + +// processUserProposalPaywall returns a proposal paywall that enables the +// the user to purchase proposal credits. The user can only have one paywall +// active at a time. If no paywall currently exists, a new one is created and +// the user is added to the paywall pool. +func (p *politeiawww) processUserProposalPaywall(u *user.User) (*www.UserProposalPaywallReply, error) { + log.Tracef("processUserProposalPaywall") + + // Ensure paywall is enabled + if !p.paywallIsEnabled() { + return &www.UserProposalPaywallReply{}, nil + } + + // Proposal paywalls cannot be generated until the user has paid their + // user registration fee. + if !p.userHasPaid(*u) { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusUserNotPaid, + } + } + + var pp *user.ProposalPaywall + if p.userHasValidProposalPaywall(u) { + // Don't create a new paywall if a valid one already exists. + pp = p.mostRecentProposalPaywall(u) + } else { + // Create a new paywall. + var err error + pp, err = p.generateProposalPaywall(u) + if err != nil { + return nil, err + } + } + + return &www.UserProposalPaywallReply{ + CreditPrice: pp.CreditPrice, + PaywallAddress: pp.Address, + PaywallTxNotBefore: pp.TxNotBefore, + }, nil +} + +// processUserProposalPaywallTx checks if the user has a pending paywall +// payment and returns the payment details if one is found. +func (p *politeiawww) processUserProposalPaywallTx(u *user.User) (*www.UserProposalPaywallTxReply, error) { + log.Tracef("processUserProposalPaywallTx") + + var ( + txID string + txAmount uint64 + confirmations uint64 + ) + + p.RLock() + defer p.RUnlock() + + poolMember, ok := p.userPaywallPool[u.ID] + if ok { + txID = poolMember.txID + txAmount = poolMember.txAmount + confirmations = poolMember.txConfirmations + } + + return &www.UserProposalPaywallTxReply{ + TxID: txID, + TxAmount: txAmount, + Confirmations: confirmations, + }, nil +} + +// processUserProposalCredits returns a list of the user's unspent proposal +// credits and a list of the user's spent proposal credits. +func processUserProposalCredits(u *user.User) (*www.UserProposalCreditsReply, error) { + // Convert from database proposal credits to www proposal credits. + upc := make([]www.ProposalCredit, len(u.UnspentProposalCredits)) + for i, credit := range u.UnspentProposalCredits { + upc[i] = convertProposalCreditFromUserDB(credit) + } + spc := make([]www.ProposalCredit, len(u.SpentProposalCredits)) + for i, credit := range u.SpentProposalCredits { + spc[i] = convertProposalCreditFromUserDB(credit) + } + + return &www.UserProposalCreditsReply{ + UnspentCredits: upc, + SpentCredits: spc, + }, nil +} + // processUserPaymentsRescan allows an admin to rescan a user's paywall address // to check for any payments that may have been missed by paywall polling. func (p *politeiawww) processUserPaymentsRescan(upr www.UserPaymentsRescan) (*www.UserPaymentsRescanReply, error) { @@ -1995,48 +2105,6 @@ func (p *politeiawww) processUserPaymentsRescan(upr www.UserPaymentsRescan) (*ww }, nil } -// processVerifyUserPayment verifies that the provided transaction -// meets the minimum requirements to mark the user as paid, and then does -// that in the user database. -func (p *politeiawww) processVerifyUserPayment(u *user.User, vupt www.VerifyUserPayment) (*www.VerifyUserPaymentReply, error) { - var reply www.VerifyUserPaymentReply - if p.userHasPaid(*u) { - reply.HasPaid = true - return &reply, nil - } - - if paywallHasExpired(u.NewUserPaywallPollExpiry) { - err := p.GenerateNewUserPaywall(u) - if err != nil { - return nil, err - } - reply.PaywallAddress = u.NewUserPaywallAddress - reply.PaywallAmount = u.NewUserPaywallAmount - reply.PaywallTxNotBefore = u.NewUserPaywallTxNotBefore - return &reply, nil - } - - tx, _, err := util.FetchTxWithBlockExplorers(u.NewUserPaywallAddress, - u.NewUserPaywallAmount, u.NewUserPaywallTxNotBefore, - p.cfg.MinConfirmationsRequired, p.dcrdataHostHTTP()) - if err != nil { - return nil, err - } - - if tx != "" { - reply.HasPaid = true - - err = p.updateUserAsPaid(u, tx) - if err != nil { - return nil, err - } - } else { - // TODO: Add the user to the in-memory pool. - } - - return &reply, nil -} - // removeUsersFromPool removes the provided user IDs from the the poll pool. // // Currently, updating the user db and removing the user from pool isn't an @@ -2187,7 +2255,7 @@ func (p *politeiawww) checkForUserPayments(pool map[uuid.UUID]paywallPoolMember) if p.userHasPaid(*u) { // The user could have been marked as paid by - // RouteVerifyUserPayment, so just remove him from the + // RouteUserRegistrationPayment, so just remove him from the // in-memory pool. userIDsToRemove = append(userIDsToRemove, userID) log.Tracef(" removing from polling, user already paid") diff --git a/politeiawww/userwww.go b/politeiawww/userwww.go index 6a8e63e7b..55fc19472 100644 --- a/politeiawww/userwww.go +++ b/politeiawww/userwww.go @@ -457,41 +457,6 @@ func (p *politeiawww) handleChangePassword(w http.ResponseWriter, r *http.Reques util.RespondWithJSON(w, http.StatusOK, reply) } -// handleVerifyUserPayment checks whether the provided transaction -// is on the blockchain and meets the requirements to consider the user -// registration fee as paid. -func (p *politeiawww) handleVerifyUserPayment(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleVerifyUserPayment") - - // Get the verify user payment tx command. - var vupt www.VerifyUserPayment - err := util.ParseGetParams(r, &vupt) - if err != nil { - RespondWithError(w, r, 0, "handleVerifyUserPayment: ParseGetParams", - www.UserError{ - ErrorCode: www.ErrorStatusInvalidInput, - }) - return - } - - user, err := p.getSessionUser(w, r) - if err != nil { - RespondWithError(w, r, 0, - "handleVerifyUserPayment: getSessionUser %v", err) - return - } - - vuptr, err := p.processVerifyUserPayment(user, vupt) - if err != nil { - RespondWithError(w, r, 0, - "handleVerifyUserPayment: processVerifyUserPayment %v", - err) - return - } - - util.RespondWithJSON(w, http.StatusOK, vuptr) -} - // handleEditUser handles editing a user's preferences. func (p *politeiawww) handleEditUser(w http.ResponseWriter, r *http.Request) { log.Tracef("handleEditUser") @@ -581,32 +546,6 @@ func (p *politeiawww) handleCMSUsers(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, cur) } -// handleUserPaymentsRescan allows an admin to rescan a user's paywall address -// to check for any payments that may have been missed by paywall polling. -func (p *politeiawww) handleUserPaymentsRescan(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleUserPaymentsRescan") - - var upr www.UserPaymentsRescan - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&upr); err != nil { - RespondWithError(w, r, 0, "handleUserPaymentsRescan: unmarshal", - www.UserError{ - ErrorCode: www.ErrorStatusInvalidInput, - }) - return - } - - reply, err := p.processUserPaymentsRescan(upr) - if err != nil { - RespondWithError(w, r, 0, - "handleUserPaymentsRescan: processUserPaymentsRescan: %v", - err) - return - } - - util.RespondWithJSON(w, http.StatusOK, reply) -} - // handleManageUser handles editing a user's details. func (p *politeiawww) handleManageUser(w http.ResponseWriter, r *http.Request) { log.Tracef("handleManageUser") @@ -662,6 +601,75 @@ func (p *politeiawww) handleUserCommentsLikes(w http.ResponseWriter, r *http.Req util.RespondWithJSON(w, http.StatusOK, uclr) } +// handleUserRegistrationPayment checks whether the provided transaction +// is on the blockchain and meets the requirements to consider the user +// registration fee as paid. +func (p *politeiawww) handleUserRegistrationPayment(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleUserRegistrationPayment") + + user, err := p.getSessionUser(w, r) + if err != nil { + RespondWithError(w, r, 0, + "handleUserRegistrationPayment: getSessionUser %v", err) + return + } + + vuptr, err := p.processUserRegistrationPayment(user) + if err != nil { + RespondWithError(w, r, 0, + "handleUserRegistrationPayment: processUserRegistrationPayment %v", + err) + return + } + + util.RespondWithJSON(w, http.StatusOK, vuptr) +} + +// handleUserProposalPaywall returns paywall details that allows the user to +// purchase proposal credits. +func (p *politeiawww) handleUserProposalPaywall(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleUserProposalPaywall") + + user, err := p.getSessionUser(w, r) + if err != nil { + RespondWithError(w, r, 0, + "handleUserProposalPaywall: getSessionUser %v", err) + return + } + + reply, err := p.processUserProposalPaywall(user) + if err != nil { + RespondWithError(w, r, 0, + "handleUserProposalPaywall: processUserProposalPaywall %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, reply) +} + +// handleUserProposalPaywallTx returns the payment details for a pending +// proposal paywall payment. +func (p *politeiawww) handleUserProposalPaywallTx(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleUserProposalPaywallTx") + + user, err := p.getSessionUser(w, r) + if err != nil { + RespondWithError(w, r, 0, + "handleUserProposalPaywallTx: getSessionUser %v", err) + return + } + + reply, err := p.processUserProposalPaywallTx(user) + if err != nil { + RespondWithError(w, r, 0, + "handleUserProposalPaywallTx: "+ + "processUserProposalPaywallTx %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, reply) +} + // handleUserProposalCredits returns the spent and unspent proposal credits for // the logged in user. func (p *politeiawww) handleUserProposalCredits(w http.ResponseWriter, r *http.Request) { @@ -684,6 +692,32 @@ func (p *politeiawww) handleUserProposalCredits(w http.ResponseWriter, r *http.R util.RespondWithJSON(w, http.StatusOK, reply) } +// handleUserPaymentsRescan allows an admin to rescan a user's paywall address +// to check for any payments that may have been missed by paywall polling. +func (p *politeiawww) handleUserPaymentsRescan(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleUserPaymentsRescan") + + var upr www.UserPaymentsRescan + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&upr); err != nil { + RespondWithError(w, r, 0, "handleUserPaymentsRescan: unmarshal", + www.UserError{ + ErrorCode: www.ErrorStatusInvalidInput, + }) + return + } + + reply, err := p.processUserPaymentsRescan(upr) + if err != nil { + RespondWithError(w, r, 0, + "handleUserPaymentsRescan: processUserPaymentsRescan: %v", + err) + return + } + + util.RespondWithJSON(w, http.StatusOK, reply) +} + // handleRegisterUser handles the completion of registration by invited users of // the Contractor Management System. func (p *politeiawww) handleRegisterUser(w http.ResponseWriter, r *http.Request) { @@ -821,15 +855,21 @@ func (p *politeiawww) setUserWWWRoutes() { p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, www.RouteChangePassword, p.handleChangePassword, permissionLogin) - p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteVerifyUserPayment, p.handleVerifyUserPayment, - permissionLogin) p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, www.RouteEditUser, p.handleEditUser, permissionLogin) p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, www.RouteUserCommentsLikes, p.handleUserCommentsLikes, permissionLogin) // XXX comments need to become a setting + p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, + www.RouteUserRegistrationPayment, p.handleUserRegistrationPayment, + permissionLogin) + p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, + www.RouteUserProposalPaywall, p.handleUserProposalPaywall, + permissionLogin) + p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, + www.RouteUserProposalPaywallTx, p.handleUserProposalPaywallTx, + permissionLogin) p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, www.RouteUserProposalCredits, p.handleUserProposalCredits, permissionLogin) From b925607bd3c735e32d4046ea49d5df289239528d Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 27 Sep 2020 15:04:00 -0500 Subject: [PATCH 117/449] tlogbe fixes --- politeiad/backend/tlogbe/tlog.go | 3 ++- politeiad/backend/tlogbe/trillian.go | 12 +++++++++++- politeiad/cmd/politeia/README.md | 9 +++++---- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index e00f55ec1..f2197c370 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -595,7 +595,7 @@ func (t *tlog) freezeRecord(treeID int64) (*freezeRecord, error) { } var l *trillian.LogLeaf for _, v := range leaves { - if leafIsFreezeRecord(l) { + if leafIsFreezeRecord(v) { l = v break } @@ -1108,6 +1108,7 @@ func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []ba if err == errRecordNotFound { // No record versions exist yet. This is fine. The version and // iteration will be incremented to 1. + oldIdx = &recordIndex{} } else if err != nil { return fmt.Errorf("recordIndexLatest: %v", err) } diff --git a/politeiad/backend/tlogbe/trillian.go b/politeiad/backend/tlogbe/trillian.go index f3880c5cb..2074f50f4 100644 --- a/politeiad/backend/tlogbe/trillian.go +++ b/politeiad/backend/tlogbe/trillian.go @@ -391,6 +391,7 @@ func (t *trillianClient) leavesByRange(treeID int64, startIndex, count int64) ([ if err != nil { return nil, err } + return glbrr.Leaves, nil } @@ -407,6 +408,10 @@ func (t *trillianClient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { return []*trillian.LogLeaf{}, nil } + // Default gprc max message size is 4MB (4194304 bytes). We need to + // increase this when fetching all leaves. + // maxMsgSize := grpc.MaxCallSendMsgSize(6000000) + // Get all leaves return t.leavesByRange(treeID, 0, int64(lr.TreeSize)) } @@ -474,9 +479,14 @@ func newTrillianClient(host, keyFile string) (*trillianClient, error) { log.Infof("Trillian private key created: %v", keyFile) } + // Default gprc max message size is ~4MB (4194304 bytes). This is + // not large enough for trees with tens of thousands of leaves. + // Increase it to 20MB. + maxMsgSize := grpc.WithMaxMsgSize(20000000) + // Setup trillian connection // TODO should this be WithInsecure? - g, err := grpc.Dial(host, grpc.WithInsecure()) + g, err := grpc.Dial(host, grpc.WithInsecure(), maxMsgSize) if err != nil { return nil, fmt.Errorf("grpc dial: %v", err) } diff --git a/politeiad/cmd/politeia/README.md b/politeiad/cmd/politeia/README.md index 7f79231d5..eacb90362 100644 --- a/politeiad/cmd/politeia/README.md +++ b/politeiad/cmd/politeia/README.md @@ -27,24 +27,25 @@ $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass new 'metad Record submitted Censorship record: Merkle : 22e88c7d6da9b73fbb515ed6a8f6d133c680527a799e3069ca7ce346d90649b2 - Token : 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 + Token : 07ca1d1b5f7ca84f0000 Signature: 28c75019fb15af4e81ee1607deff58a8a82896d6bb1af4e813c5c996069ad7872505e4f25e067e8f310af82981aca1b02050ee23029f6d1e87b8ea8f0b3bcd08 ``` ## Get unvetted record ``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass getunvetted 43c2d4a2c846c188ab0b49012ed17e5f2c16bd6e276cfbb42e30352dffb1743f +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass getunvetted 07ca1d1b5f7ca84f0000 + Unvetted record: Status : censored Timestamp : 2017-12-14 17:08:33 +0000 UTC Censorship record: Merkle : 22e88c7d6da9b73fbb515ed6a8f6d133c680527a799e3069ca7ce346d90649b2 - Token : 43c2d4a2c846c188ab0b49012ed17e5f2c16bd6e276cfbb42e30352dffb1743f + Token : 07ca1d1b5f7ca84f0000 Signature: 5c28d2a93ff9cfe35e8a6b465ae06fa596b08bfe7b980ff9dbe68877e7d860010ec3c4fd8c8b739dc4ceeda3a2381899c7741896323856f0f267abf9a40b8003 Metadata : [{2 {"foo":"bar"}} {12 "zap"}] File (00) : - Name : a + Name : filename.txt MIME : text/plain; charset=utf-8 Digest : 22e88c7d6da9b73fbb515ed6a8f6d133c680527a799e3069ca7ce346d90649b2 ``` From 7177a74669a78b6933132854bb3d91a52df53dab Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 28 Sep 2020 15:32:51 -0500 Subject: [PATCH 118/449] fixes and cleanups --- politeiad/backend/tlogbe/tlog.go | 2 +- politeiad/cmd/politeia/README.md | 45 ++++++----- politeiad/cmd/politeia/politeia.go | 32 ++++---- politeiawww/api/pi/v1/v1.go | 7 +- politeiawww/cmd/shared/client.go | 25 ------ politeiawww/piwww.go | 126 +++++++++++++++++++---------- politeiawww/user.go | 88 -------------------- politeiawww/userwww.go | 27 ------- 8 files changed, 130 insertions(+), 222 deletions(-) diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index f2197c370..f665a70e1 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -1451,7 +1451,7 @@ func (t *tlog) record(treeID int64, version uint32) (*backend.Record, error) { } return &backend.Record{ - Version: strconv.FormatUint(uint64(version), 10), + Version: strconv.FormatUint(uint64(index.Version), 10), RecordMetadata: *recordMD, Metadata: metadata, Files: files, diff --git a/politeiad/cmd/politeia/README.md b/politeiad/cmd/politeia/README.md index eacb90362..49571b0d6 100644 --- a/politeiad/cmd/politeia/README.md +++ b/politeiad/cmd/politeia/README.md @@ -15,39 +15,40 @@ Identity saved to: /home/user/.politeia/identity.json ## Add a new record -At least one file must be included. This is the `filepath` argument in the -example below. The provided file must already exist. Arguments are matched -against the regex `^metadata[\d]{1,2}:` to determine if the string is record -metadata. Arguments that are not classified as metadata are assumed to be file -paths. +At least one file must be submitted. This example uses an `index.md` file. + +Arguments are matched against the regex `^metadata[\d]{1,2}:` to determine if +the string is record metadata. Arguments that are not classified as metadata +are assumed to be file paths. ``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass new 'metadata12:{"moo":"lala"}' 'metadata2:{"foo":"bar"}' filepath -00: 22e88c7d6da9b73fbb515ed6a8f6d133c680527a799e3069ca7ce346d90649b2 a text/plain; charset=utf-8 +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass new 'metadata12:{"moo":"lala"}' 'metadata2:{"foo":"bar"}' ~/index.md +00: 4bde9f923b61e26147c79500e6d6dfa27291559a74cd878c29a7f96984dd48bb index.md text/plain; charset=utf-8 Record submitted Censorship record: - Merkle : 22e88c7d6da9b73fbb515ed6a8f6d133c680527a799e3069ca7ce346d90649b2 - Token : 07ca1d1b5f7ca84f0000 - Signature: 28c75019fb15af4e81ee1607deff58a8a82896d6bb1af4e813c5c996069ad7872505e4f25e067e8f310af82981aca1b02050ee23029f6d1e87b8ea8f0b3bcd08 + Merkle : 4bde9f923b61e26147c79500e6d6dfa27291559a74cd878c29a7f96984dd48bb + Token : 0e4a82a370228b710000 + Signature: a0c4afd301d5452d787ac1c9835fb6f3a32443d21c92cd4575e8ef6d5ef6c4f9199a02f67893aa7b7a610055d2a6d56899ccd73c0a48ffeab72d788d1c4d4a01 ``` ## Get unvetted record ``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass getunvetted 07ca1d1b5f7ca84f0000 - +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass getunvetted 0e4a82a370228b710000 Unvetted record: - Status : censored - Timestamp : 2017-12-14 17:08:33 +0000 UTC + Status : not reviewed + Timestamp : 2020-09-28 14:20:14 +0000 UTC Censorship record: - Merkle : 22e88c7d6da9b73fbb515ed6a8f6d133c680527a799e3069ca7ce346d90649b2 - Token : 07ca1d1b5f7ca84f0000 - Signature: 5c28d2a93ff9cfe35e8a6b465ae06fa596b08bfe7b980ff9dbe68877e7d860010ec3c4fd8c8b739dc4ceeda3a2381899c7741896323856f0f267abf9a40b8003 - Metadata : [{2 {"foo":"bar"}} {12 "zap"}] + Merkle : 4bde9f923b61e26147c79500e6d6dfa27291559a74cd878c29a7f96984dd48bb + Token : 0e4a82a370228b710000 + Signature: a0c4afd301d5452d787ac1c9835fb6f3a32443d21c92cd4575e8ef6d5ef6c4f9199a02f67893aa7b7a610055d2a6d56899ccd73c0a48ffeab72d788d1c4d4a01 + Metadata : [{2 {"foo":"bar"}} {12 {"moo":"lala"}}] + Version : 1 File (00) : - Name : filename.txt + Name : index.md MIME : text/plain; charset=utf-8 - Digest : 22e88c7d6da9b73fbb515ed6a8f6d133c680527a799e3069ca7ce346d90649b2 + Digest : 4bde9f923b61e26147c79500e6d6dfa27291559a74cd878c29a7f96984dd48bb + ``` ## Update an unvetted record @@ -64,7 +65,9 @@ Metadata provided using the `overwritemetadata` argument does not have to already exist. The token argument should be prefixed with `token:`. ``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass updateunvetted 'appendmetadata12:{"foo":"bar"}' 'overwritemetadata2:{"12foo":"12bar"}' del:filename add filepath token:72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass updateunvetted \ + 'appendmetadata12:{"foo":"bar"}' 'overwritemetadata2:{"12foo":"12bar"}' \ + del:index.md add:~/updated.md token:0e4a82a370228b710000 Update record: 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 Files add : 00: 12a31b5e662dfa0a572e9fc523eb703f9708de5e2d53aba74f8ebcebbdb706f7 b text/plain; charset=utf-8 Files delete : a diff --git a/politeiad/cmd/politeia/politeia.go b/politeiad/cmd/politeia/politeia.go index 019e6c4e4..9a09e4b9a 100644 --- a/politeiad/cmd/politeia/politeia.go +++ b/politeiad/cmd/politeia/politeia.go @@ -23,7 +23,7 @@ import ( "github.com/decred/dcrd/dcrutil" "github.com/decred/dcrtime/merkle" - "github.com/decred/politeia/politeiad/api/v1" + v1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/api/v1/mime" "github.com/decred/politeia/util" @@ -1270,28 +1270,30 @@ func _main() error { // Select action if i == 0 { switch a { - case "new": - return newRecord() case "identity": return getIdentity() - case "plugin": - return plugin() - case "plugininventory": - return getPluginInventory() - case "inventory": - return inventory() - case "getunvetted": - return getUnvetted() - case "getvetted": - return getVetted() - case "setunvettedstatus": - return setUnvettedStatus() + case "new": + return newRecord() case "updateunvetted": return updateRecord(false) + // TODO case "updateunvettedmd" + case "setunvettedstatus": + return setUnvettedStatus() + case "getunvetted": + return getUnvetted() case "updatevetted": return updateRecord(true) case "updatevettedmd": return updateVettedMetadata() + // TODO case "setvettedstatus": + case "getvetted": + return getVetted() + case "inventory": + return inventory() + case "plugin": + return plugin() + case "plugininventory": + return getPluginInventory() default: return fmt.Errorf("invalid action: %v", a) } diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index d6c8fded7..c2c5f42ef 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -375,7 +375,12 @@ type ProposalRequest struct { } // Proposals retrieves the ProposalRecord for each of the provided proposal -// requests. Unvetted proposal files are only returned to admins. +// requests. Unvetted proposals are stripped of their user defined proposal +// files and metadata when being returned to non-admins. +// +// IncludeFiles specifies whether the proposal files should be returned. The +// user defined metadata will still be returned even when IncludeFiles is set +// to false. type Proposals struct { State PropStateT `json:"state"` Requests []ProposalRequest `json:"requests"` diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index d2138ad2a..c0b779835 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -895,31 +895,6 @@ func (c *Client) ProposalDetails(token string, pd *www.ProposalsDetails) (*www.P return &pr, nil } -// UserProposals retrieves the proposals that have been submitted by the -// specified user. -func (c *Client) UserProposals(up *www.UserProposals) (*www.UserProposalsReply, error) { - responseBody, err := c.makeRequest(http.MethodGet, - www.PoliteiaWWWAPIRoute, www.RouteUserProposals, up) - if err != nil { - return nil, err - } - - var upr www.UserProposalsReply - err = json.Unmarshal(responseBody, &upr) - if err != nil { - return nil, fmt.Errorf("unmarshal UserProposalsReply: %v", err) - } - - if c.cfg.Verbose { - err := prettyPrintJSON(upr) - if err != nil { - return nil, err - } - } - - return &upr, nil -} - // UserInvoices retrieves the proposals that have been submitted by the // specified user. func (c *Client) UserInvoices(up *cms.UserInvoices) (*cms.UserInvoicesReply, error) { diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 96a25a3e7..a1eae6b8e 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -542,6 +542,10 @@ func (p *politeiawww) proposalRecordLatest(state pi.PropStateT, token string) (* // // TODO politeiad needs batched calls for retrieving unvetted and vetted // records. This call should have an includeFiles option. +// TODO this presents a challenge because the proposal Metadata still needs to +// be returned even if the proposal Files are not returned, which means that we +// will always need to fetch the record from politeiad with the files attached +// since the proposal Metadata is saved to politeiad as a politeiad File. func (p *politeiawww) proposalRecords(state pi.PropStateT, reqs []pi.ProposalRequest, includeFiles bool) (map[string]pi.ProposalRecord, error) { // Get politeiad records props := make([]pi.ProposalRecord, 0, len(reqs)) @@ -571,10 +575,10 @@ func (p *politeiawww) proposalRecords(state pi.PropStateT, reqs []pi.ProposalReq v.Token, err) } - // Remove files if specified + // Remove files if specified. The Metadata objects will still be + // returned. if !includeFiles { pr.Files = []pi.File{} - pr.Metadata = []pi.Metadata{} } props = append(props, *pr) @@ -1417,20 +1421,52 @@ func (p *politeiawww) handleProposalSetStatus(w http.ResponseWriter, r *http.Req util.RespondWithJSON(w, http.StatusOK, pssr) } +func (p *politeiawww) handleProposals(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleProposals") + + var ps pi.Proposals + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&ps); err != nil { + respondWithPiError(w, r, "handleProposals: unmarshal", + pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusInputInvalid, + }) + return + } + + // Lookup session user. This is a public route so a session may not + // exist. Ignore any session not found errors. + user, err := p.getSessionUser(w, r) + if err != nil && err != errSessionNotFound { + respondWithPiError(w, r, + "handleProposals: getSessionUser: %v", err) + return + } + + isAdmin := user != nil && user.Admin + ppi, err := p.processProposals(ps, isAdmin) + if err != nil { + respondWithPiError(w, r, + "handleProposals: processProposals: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, ppi) +} + func (p *politeiawww) handleProposalInventory(w http.ResponseWriter, r *http.Request) { log.Tracef("handleProposalInventory") - // Get data from session user. This is a public route, so we can - // ignore the session not found error. This is done to strip - // non-admin users from unvetted record tokens. + // Lookup session user. This is a public route so a session may not + // exist. Ignore any session not found errors. user, err := p.getSessionUser(w, r) if err != nil && err != errSessionNotFound { respondWithPiError(w, r, "handleProposalInventory: getSessionUser: %v", err) return } - isAdmin := user != nil && user.Admin + isAdmin := user != nil && user.Admin ppi, err := p.processProposalInventory(isAdmin) if err != nil { respondWithPiError(w, r, @@ -2342,61 +2378,63 @@ func (p *politeiawww) handleVoteInventory(w http.ResponseWriter, r *http.Request } func (p *politeiawww) setPiRoutes() { - // Public routes - p.addRoute(http.MethodGet, pi.APIRoute, - pi.RouteProposalInventory, p.handleProposalInventory, - permissionPublic) - + // Proposal routes p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteComments, p.handleComments, permissionPublic) - + pi.RouteProposalNew, p.handleProposalNew, + permissionLogin) p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteVoteAuthorize, p.handleVoteAuthorize, permissionPublic) - + pi.RouteProposalEdit, p.handleProposalEdit, + permissionLogin) p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteVoteStart, p.handleVoteStart, permissionPublic) - + pi.RouteProposalSetStatus, p.handleProposalSetStatus, + permissionLogin) p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteVoteStartRunoff, p.handleVoteStartRunoff, permissionPublic) - + pi.RouteProposals, p.handleProposals, + permissionPublic) p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteVoteBallot, p.handleVoteBallot, permissionPublic) + pi.RouteProposalInventory, p.handleProposalInventory, + permissionPublic) + // Comment routes p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteVotes, p.handleVotes, permissionPublic) - + pi.RouteCommentNew, p.handleCommentNew, + permissionLogin) p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteVoteResults, p.handleVoteResults, permissionPublic) - + pi.RouteCommentVote, p.handleCommentVote, + permissionLogin) p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteVoteSummaries, p.handleVoteSummaries, permissionPublic) - - p.addRoute(http.MethodGet, pi.APIRoute, - pi.RouteVoteInventory, p.handleVoteInventory, permissionPublic) - - // Logged in routes + pi.RouteCommentCensor, p.handleCommentCensor, + permissionAdmin) p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteProposalNew, p.handleProposalNew, - permissionLogin) - + pi.RouteComments, p.handleComments, + permissionPublic) p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteProposalEdit, p.handleProposalEdit, + pi.RouteCommentVotes, p.handleCommentVotes, permissionLogin) + // Vote routes p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteProposalSetStatus, p.handleProposalSetStatus, + pi.RouteVoteAuthorize, p.handleVoteAuthorize, permissionLogin) - p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteCommentNew, p.handleCommentNew, permissionLogin) - + pi.RouteVoteStart, p.handleVoteStart, + permissionAdmin) p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteCommentVote, p.handleCommentVote, permissionLogin) - + pi.RouteVoteStartRunoff, p.handleVoteStartRunoff, + permissionAdmin) p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteCommentVotes, p.handleCommentVotes, permissionLogin) - - // Admin routes + pi.RouteVoteBallot, p.handleVoteBallot, + permissionPublic) + p.addRoute(http.MethodPost, pi.APIRoute, + pi.RouteVotes, p.handleVotes, + permissionPublic) + p.addRoute(http.MethodPost, pi.APIRoute, + pi.RouteVoteResults, p.handleVoteResults, + permissionPublic) + p.addRoute(http.MethodPost, pi.APIRoute, + pi.RouteVoteSummaries, p.handleVoteSummaries, + permissionPublic) p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteCommentCensor, p.handleCommentCensor, permissionAdmin) + pi.RouteVoteInventory, p.handleVoteInventory, + permissionPublic) } diff --git a/politeiawww/user.go b/politeiawww/user.go index ce4a3efe9..6b8c6055d 100644 --- a/politeiawww/user.go +++ b/politeiawww/user.go @@ -630,94 +630,6 @@ func (p *politeiawww) processEditUser(eu *www.EditUser, user *user.User) (*www.E return &www.EditUserReply{}, nil } -// processUserCommentsLikes returns all of the user's comment likes for the -// passed in proposal. -func (p *politeiawww) processUserCommentsLikes(user *user.User, token string) (*www.UserCommentsLikesReply, error) { - log.Tracef("processUserCommentsLikes: %v %v", user.ID, token) - - /* - // Make sure token is valid and not a prefix - if !isTokenValid(token) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: []string{token}, - } - } - - // Fetch all like comments for the proposal - dlc, err := p.decredPropCommentLikes(token) - if err != nil { - return nil, fmt.Errorf("decredPropLikeComments: %v", err) - } - - // Sanity check. Like comments should already be sorted in - // chronological order. - sort.SliceStable(dlc, func(i, j int) bool { - return dlc[i].Timestamp < dlc[j].Timestamp - }) - - // Find all the like comments that are from the user - lc := make([]www.LikeComment, 0, len(dlc)) - for _, v := range dlc { - u, err := p.db.UserGetByPubKey(v.PublicKey) - if err != nil { - log.Errorf("getUserCommentLikes: UserGetByPubKey: "+ - "token:%v commentID:%v pubKey:%v err:%v", v.Token, - v.CommentID, v.PublicKey, err) - continue - } - if user.ID.String() == u.ID.String() { - lc = append(lc, convertLikeCommentFromDecred(v)) - } - } - - // Compute the resulting like comment action for each comment. - // The resulting action depends on the order of the like - // comment actions. - // - // Example: when a user upvotes a comment twice, the second - // upvote cancels out the first upvote and the resulting - // comment score is 0. - // - // Example: when a user upvotes a comment and then downvotes - // the same comment, the downvote takes precedent and the - // resulting comment score is -1. - actions := make(map[string]string) // [commentID]action - for _, v := range lc { - prevAction := actions[v.CommentID] - switch { - case v.Action == prevAction: - // New action is the same as the previous action so - // we undo the previous action. - actions[v.CommentID] = "" - case v.Action != prevAction: - // New action is different than the previous action - // so the new action takes precedent. - actions[v.CommentID] = v.Action - } - } - - cl := make([]www.CommentLike, 0, len(lc)) - for k, v := range actions { - // Skip actions that have been taken away - if v == "" { - continue - } - cl = append(cl, www.CommentLike{ - Token: token, - CommentID: k, - Action: v, - }) - } - - return &www.UserCommentsLikesReply{ - CommentsLikes: cl, - }, nil - */ - - return nil, nil -} - // createLoginReply creates a login reply. func (p *politeiawww) createLoginReply(u *user.User, lastLoginTime int64) (*www.LoginReply, error) { reply := www.LoginReply{ diff --git a/politeiawww/userwww.go b/politeiawww/userwww.go index 55fc19472..42a189716 100644 --- a/politeiawww/userwww.go +++ b/politeiawww/userwww.go @@ -577,30 +577,6 @@ func (p *politeiawww) handleManageUser(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, mur) } -// handleUserCommentsLikes returns the user votes on comments of a given proposal. -func (p *politeiawww) handleUserCommentsLikes(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleUserCommentsLikes") - - pathParams := mux.Vars(r) - token := pathParams["token"] - - user, err := p.getSessionUser(w, r) - if err != nil { - RespondWithError(w, r, 0, - "handleUserCommentsLikes: getSessionUser %v", err) - return - } - - uclr, err := p.processUserCommentsLikes(user, token) - if err != nil { - RespondWithError(w, r, 0, - "handleUserCommentsLikes: processUserCommentsLikes %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, uclr) -} - // handleUserRegistrationPayment checks whether the provided transaction // is on the blockchain and meets the requirements to consider the user // registration fee as paid. @@ -858,9 +834,6 @@ func (p *politeiawww) setUserWWWRoutes() { p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, www.RouteEditUser, p.handleEditUser, permissionLogin) - p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteUserCommentsLikes, p.handleUserCommentsLikes, - permissionLogin) // XXX comments need to become a setting p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, www.RouteUserRegistrationPayment, p.handleUserRegistrationPayment, permissionLogin) From b971df2bd426ada97551e1e7ed3a3f12457a0177 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 29 Sep 2020 08:46:29 -0500 Subject: [PATCH 119/449] cleanup and fixes --- plugins/pi/pi.go | 22 +- politeiad/cmd/politeia/README.md | 3 +- politeiawww/api/pi/v1/v1.go | 7 +- politeiawww/api/www/v1/v1.go | 16 +- politeiawww/cmd/piwww/votestartrunoff.go | 10 +- politeiawww/comments.go | 487 +----- politeiawww/convert.go | 616 ------- politeiawww/dcc.go | 277 ++- politeiawww/decred.go | 22 +- politeiawww/invoices.go | 358 +++- politeiawww/pi.go | 71 +- politeiawww/piwww.go | 1748 ++++++++++--------- politeiawww/plugin.go | 18 +- politeiawww/politeiad.go | 4 +- politeiawww/proposals.go | 2038 +++++++++++----------- politeiawww/testing.go | 4 +- politeiawww/ticketvote.go | 144 +- politeiawww/userwww.go | 14 +- politeiawww/www.go | 2 +- 19 files changed, 2628 insertions(+), 3233 deletions(-) delete mode 100644 politeiawww/convert.go diff --git a/plugins/pi/pi.go b/plugins/pi/pi.go index 105a5e577..ffb5207d9 100644 --- a/plugins/pi/pi.go +++ b/plugins/pi/pi.go @@ -15,7 +15,7 @@ import ( type PropStateT int type PropStatusT int type ErrorStatusT int -type VoteT int +type CommentVoteT int const ( ID = "pi" @@ -55,9 +55,9 @@ const ( PropStatusAbandoned PropStatusT = 4 // Prop has been abandoned // Comment vote types - VoteInvalid VoteT = 0 - VoteDownvote VoteT = -1 - VoteUpvote VoteT = 1 + VoteInvalid CommentVoteT = 0 + VoteDownvote CommentVoteT = -1 + VoteUpvote CommentVoteT = 1 // User error status codes // TODO number error codes and add human readable error messages @@ -371,13 +371,13 @@ func DecodeCommentCensorReply(payload []byte) (*CommentCensorReply, error) { // // Signature is the client signature of the State+Token+CommentID+Vote. type CommentVote struct { - UserID string `json:"userid"` // Unique user ID - State PropStateT `json:"state"` // Record state - Token string `json:"token"` // Record token - CommentID uint32 `json:"commentid"` // Comment ID - Vote VoteT `json:"vote"` // Upvote or downvote - PublicKey string `json:"publickey"` // Public key used for signature - Signature string `json:"signature"` // Client signature + UserID string `json:"userid"` // Unique user ID + State PropStateT `json:"state"` // Record state + Token string `json:"token"` // Record token + CommentID uint32 `json:"commentid"` // Comment ID + Vote CommentVoteT `json:"vote"` // Upvote or downvote + PublicKey string `json:"publickey"` // Public key used for signature + Signature string `json:"signature"` // Client signature } // EncodeCommentVote encodes a CommentVote into a JSON byte slice. diff --git a/politeiad/cmd/politeia/README.md b/politeiad/cmd/politeia/README.md index 49571b0d6..62c78c5db 100644 --- a/politeiad/cmd/politeia/README.md +++ b/politeiad/cmd/politeia/README.md @@ -22,7 +22,8 @@ the string is record metadata. Arguments that are not classified as metadata are assumed to be file paths. ``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass new 'metadata12:{"moo":"lala"}' 'metadata2:{"foo":"bar"}' ~/index.md +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass new \ + 'metadata12:{"moo":"lala"}' 'metadata2:{"foo":"bar"}' ~/index.md 00: 4bde9f923b61e26147c79500e6d6dfa27291559a74cd878c29a7f96984dd48bb index.md text/plain; charset=utf-8 Record submitted Censorship record: diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index c2c5f42ef..cac91e926 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -166,6 +166,7 @@ const ( ErrorStatusVotePageSizeExceeded // Cast vote errors + // TODO these need human readable equivalents VoteErrorInvalid VoteErrorT = 0 VoteErrorInternalError VoteErrorT = 1 VoteErrorTokenInvalid VoteErrorT = 2 @@ -672,9 +673,9 @@ type VoteStartReply struct { // VoteStartRunoff starts a runoff vote between the provided submissions. Each // submission is required to have its own Authorize and Start. type VoteStartRunoff struct { - Token string `json:"token"` // RFP token - Authorizations []VoteAuthorize `json:"authorizations"` - Starts []VoteStart `json:"starts"` + Token string `json:"token"` // RFP token + Auths []VoteAuthorize `json:"auths"` + Starts []VoteStart `json:"starts"` } // VoteStartRunoffReply is the reply to the VoteStartRunoff command. diff --git a/politeiawww/api/www/v1/v1.go b/politeiawww/api/www/v1/v1.go index 0c1900ed7..3bb991ba4 100644 --- a/politeiawww/api/www/v1/v1.go +++ b/politeiawww/api/www/v1/v1.go @@ -31,31 +31,29 @@ const ( RouteLogin = "/login" RouteLogout = "/logout" RouteUserMe = "/user/me" - RouteUserDetails = "/user/{userid:[0-9a-zA-Z-]{36}}" RouteNewUser = "/user/new" RouteResendVerification = "/user/new/resend" RouteVerifyNewUser = "/user/verify" + RouteEditUser = "/user/edit" RouteUpdateUserKey = "/user/key" RouteVerifyUpdateUserKey = "/user/key/verify" RouteChangeUsername = "/user/username/change" RouteChangePassword = "/user/password/change" RouteResetPassword = "/user/password/reset" RouteVerifyResetPassword = "/user/password/reset/verify" + RouteUserRegistrationPayment = "/user/payments/registration" + RouteUserProposalPaywall = "/user/payments/paywall" + RouteUserProposalPaywallTx = "/user/payments/paywalltx" + RouteUserProposalCredits = "/user/payments/credits" + RouteUserPaymentsRescan = "/user/payments/rescan" RouteManageUser = "/user/manage" - RouteEditUser = "/user/edit" RouteSetTOTP = "/user/totp" RouteVerifyTOTP = "/user/verifytotp" + RouteUserDetails = "/user/{userid:[0-9a-zA-Z-]{36}}" RouteUsers = "/users" RouteUnauthenticatedWebSocket = "/ws" RouteAuthenticatedWebSocket = "/aws" - // User payments routes - RouteUserRegistrationPayment = "/user/payments/registration" - RouteUserProposalPaywall = "/user/payments/paywall" - RouteUserProposalPaywallTx = "/user/payments/paywalltx" - RouteUserProposalCredits = "/user/payments/credits" - RouteUserPaymentsRescan = "/user/payments/rescan" - // The following routes WILL BE DEPRECATED in the near future and // should not be used. The pi v1 API should be used instead. RouteTokenInventory = "/proposals/tokeninventory" diff --git a/politeiawww/cmd/piwww/votestartrunoff.go b/politeiawww/cmd/piwww/votestartrunoff.go index 200c1cc5b..b92d05cf8 100644 --- a/politeiawww/cmd/piwww/votestartrunoff.go +++ b/politeiawww/cmd/piwww/votestartrunoff.go @@ -24,8 +24,8 @@ import ( // sometimes desirable when testing. type voteStartRunoffCmd struct { Args struct { - TokenRFP string `positional-arg-name:"token" required:"true"` // RFP censorship token - Duration uint32 `positional-arg-name:"duration"` // Duration in blocks + TokenRFP string `positional-arg-name:"token" required:"true"` + Duration uint32 `positional-arg-name:"duration"` QuorumPercentage string `positional-arg-name:"quorumpercentage"` PassPercentage string `positional-arg-name:"passpercentage"` } `positional-args:"true"` @@ -144,9 +144,9 @@ func (cmd *voteStartRunoffCmd) Execute(args []string) error { // Prepare and send request svr := pi.VoteStartRunoff{ - Token: cmd.Args.TokenRFP, - Authorizations: auths, - Starts: starts, + Token: cmd.Args.TokenRFP, + Auths: auths, + Starts: starts, } err = shared.PrintJSON(svr) if err != nil { diff --git a/politeiawww/comments.go b/politeiawww/comments.go index 9e75efc31..45919b675 100644 --- a/politeiawww/comments.go +++ b/politeiawww/comments.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Decred developers +// Copyright (c) 2017-2020 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -6,499 +6,38 @@ package main import ( "github.com/decred/politeia/plugins/comments" - www "github.com/decred/politeia/politeiawww/api/www/v1" - "github.com/decred/politeia/politeiawww/user" ) -func (p *politeiawww) commentVotes(vs comments.Votes) (*comments.VotesReply, error) { - // Prep plugin payload - payload, err := comments.EncodeVotes(vs) +// commentsAll returns all comments for the provided record. +func (p *politeiawww) commentsAll(cp comments.GetAll) (*comments.GetAllReply, error) { + b, err := comments.EncodeGetAll(cp) if err != nil { return nil, err } - - r, err := p.pluginCommand(comments.ID, comments.CmdVotes, "", - string(payload)) + r, err := p.pluginCommand(comments.ID, comments.CmdGetAll, string(b)) if err != nil { return nil, err } - vsr, err := comments.DecodeVotesReply([]byte(r)) + cr, err := comments.DecodeGetAllReply([]byte(r)) if err != nil { return nil, err } - - return vsr, nil + return cr, nil } -// comments calls the comments plugin to get record's comments. -func (p *politeiawww) comments(cp comments.GetAll) (*comments.GetAllReply, error) { - // Prep plugin payload - payload, err := comments.EncodeGetAll(cp) +// commentVotes returns the comment votes that meet the provided criteria. +func (p *politeiawww) commentVotes(vs comments.Votes) (*comments.VotesReply, error) { + b, err := comments.EncodeVotes(vs) if err != nil { return nil, err } - - r, err := p.pluginCommand(comments.ID, comments.CmdGetAll, "", - string(payload)) + r, err := p.pluginCommand(comments.ID, comments.CmdVotes, string(b)) if err != nil { return nil, err } - cr, err := comments.DecodeGetAllReply([]byte(r)) + vsr, err := comments.DecodeVotesReply([]byte(r)) if err != nil { return nil, err } - - return cr, nil -} - -func validateComment(c www.NewComment) error { - // max length - if len(c.Comment) > www.PolicyMaxCommentLength { - return www.UserError{ - ErrorCode: www.ErrorStatusCommentLengthExceededPolicy, - } - } - // validate token - if !tokenIsValid(c.Token) { - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - } - } - return nil -} - -// processNewComment sends a new comment decred plugin command to politeaid -// then fetches the new comment from the cache and returns it. -func (p *politeiawww) processNewComment(nc www.NewComment, u *user.User) (*www.NewCommentReply, error) { - log.Tracef("processNewComment: %v %v", nc.Token, u.ID) - - /* - // Make sure token is valid and not a prefix - if !isTokenValid(nc.Token) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: []string{nc.Token}, - } - } - - // Pay up sucker! - if !p.HasUserPaid(u) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusUserNotPaid, - } - } - - // Ensure the public key is the user's active key - if nc.PublicKey != u.PublicKey() { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidSigningKey, - } - } - - // Validate signature - msg := nc.Token + nc.ParentID + nc.Comment - err := validateSignature(nc.PublicKey, nc.Signature, msg) - if err != nil { - return nil, err - } - - // Validate comment - err = validateComment(nc) - if err != nil { - return nil, err - } - - // Ensure proposal exists and is public - pr, err := p.getProp(nc.Token) - if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - } - } - return nil, err - } - - if pr.Status != www.PropStatusPublic { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusWrongStatus, - ErrorContext: []string{"proposal is not public"}, - } - } - - // Ensure proposal voting has not ended - bb, err := p.getBestBlock() - if err != nil { - return nil, fmt.Errorf("getBestBlock: %v", err) - } - vs, err := p.voteSummaryGet(nc.Token, bb) - if err != nil { - return nil, fmt.Errorf("voteSummaryGet: %v", err) - } - if vs.Status == www.PropVoteStatusFinished { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - ErrorContext: []string{"vote is finished"}, - } - } - - // Ensure the comment is not a duplicate - _, err = p.decredCommentGetBySignature(nc.Token, nc.Signature) - switch err { - case nil: - // Duplicate comment was found - return nil, www.UserError{ - ErrorCode: www.ErrorStatusDuplicateComment, - } - // No duplicate comment; continue - default: - // Some other error - return nil, fmt.Errorf("decredCommentBySignature: %v", err) - } - - // Setup plugin command - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - - dnc := convertNewCommentToDecredPlugin(nc) - payload, err := decredplugin.EncodeNewComment(dnc) - if err != nil { - return nil, err - } - - pc := pd.PluginCommand{ - Challenge: hex.EncodeToString(challenge), - ID: decredplugin.ID, - Command: decredplugin.CmdNewComment, - CommandID: decredplugin.CmdNewComment, - Payload: string(payload), - } - - // Send polieiad request - responseBody, err := p.makeRequest(http.MethodPost, - pd.PluginCommandRoute, pc) - if err != nil { - return nil, err - } - - // Handle response - var reply pd.PluginCommandReply - err = json.Unmarshal(responseBody, &reply) - if err != nil { - return nil, fmt.Errorf("could not unmarshal "+ - "PluginCommandReply: %v", err) - } - - err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) - if err != nil { - return nil, err - } - - ncr, err := decredplugin.DecodeNewCommentReply([]byte(reply.Payload)) - if err != nil { - return nil, err - } - - // Add comment to commentVotes in-memory cache - p.Lock() - p.commentVotes[nc.Token+ncr.CommentID] = counters{} - p.Unlock() - - // Get comment from cache - c, err := p.getComment(nc.Token, ncr.CommentID) - if err != nil { - return nil, fmt.Errorf("getComment: %v", err) - } - - // Emit event notification for a proposal comment - p.eventManager.emit(eventProposalComment, - dataProposalComment{ - token: pr.CensorshipRecord.Token, - name: pr.Name, - username: pr.Username, - parentID: c.ParentID, - commentID: c.CommentID, - commentUsername: c.Username, - }) - - return &www.NewCommentReply{ - Comment: *c, - }, nil - */ - - return nil, nil -} - -// processLikeComment processes an upvote/downvote on a comment. -func (p *politeiawww) processLikeComment(lc www.LikeComment, u *user.User) (*www.LikeCommentReply, error) { - log.Debugf("processLikeComment: %v %v %v", lc.Token, lc.CommentID, u.ID) - - /* - // Make sure token is valid and not a prefix - if !isTokenValid(lc.Token) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: []string{lc.Token}, - } - } - - // Pay up sucker! - if !p.HasUserPaid(u) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusUserNotPaid, - } - } - - // Ensure the public key is the user's active key - if lc.PublicKey != u.PublicKey() { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidSigningKey, - } - } - - // Validate signature - msg := lc.Token + lc.CommentID + lc.Action - err := validateSignature(lc.PublicKey, lc.Signature, msg) - if err != nil { - return nil, err - } - - // Ensure proposal exists and is public - pr, err := p.getProp(lc.Token) - if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - } - } - return nil, err - } - - if pr.Status != www.PropStatusPublic { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusWrongStatus, - } - } - - // Ensure proposal voting has not ended - bb, err := p.getBestBlock() - if err != nil { - return nil, fmt.Errorf("getBestBlock: %v", err) - } - vs, err := p.voteSummaryGet(pr.CensorshipRecord.Token, bb) - if err != nil { - return nil, err - } - if vs.Status == www.PropVoteStatusFinished { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - ErrorContext: []string{"vote has ended"}, - } - } - - // Ensure comment exists and has not been censored. - c, err := p.decredCommentGetByID(lc.Token, lc.CommentID) - if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusCommentNotFound, - } - } - return nil, err - } - if c.Censored { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusCommentIsCensored, - } - } - - // Validate action - action := lc.Action - if len(lc.Action) > 10 { - // Clip action to not fill up logs and prevent DOS of sorts - action = lc.Action[0:9] + "..." - } - if action != www.VoteActionUp && action != www.VoteActionDown { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidLikeCommentAction, - } - } - - // Setup plugin command - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - - dlc := convertLikeCommentToDecred(lc) - payload, err := decredplugin.EncodeLikeComment(dlc) - if err != nil { - return nil, err - } - - pc := pd.PluginCommand{ - Challenge: hex.EncodeToString(challenge), - ID: decredplugin.ID, - Command: decredplugin.CmdLikeComment, - CommandID: decredplugin.CmdLikeComment, - Payload: string(payload), - } - - // Send plugin command to politeiad - responseBody, err := p.makeRequest(http.MethodPost, - pd.PluginCommandRoute, pc) - if err != nil { - return nil, err - } - - // Handle response - var reply pd.PluginCommandReply - err = json.Unmarshal(responseBody, &reply) - if err != nil { - return nil, fmt.Errorf("could not unmarshal "+ - "PluginCommandReply: %v", err) - } - - err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) - if err != nil { - return nil, err - } - - lcr, err := decredplugin.DecodeLikeCommentReply([]byte(reply.Payload)) - if err != nil { - return nil, err - } - - // Update comment score in the in-memory cache - votes, err := p.updateCommentVotes(lc.Token, lc.CommentID) - if err != nil { - log.Criticalf("processLikeComment: update comment score "+ - "failed token:%v commentID:%v error:%v", lc.Token, - lc.CommentID, err) - } - - return &www.LikeCommentReply{ - Result: int64(votes.up - votes.down), - Upvotes: votes.up, - Downvotes: votes.down, - Receipt: lcr.Receipt, - Error: lcr.Error, - }, nil - */ - - return nil, nil -} - -func (p *politeiawww) processCensorComment(cc www.CensorComment, u *user.User) (*www.CensorCommentReply, error) { - log.Tracef("processCensorComment: %v: %v", cc.Token, cc.CommentID) - - /* - // Make sure token is valid and not a prefix - if !isTokenValid(cc.Token) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: []string{cc.Token}, - } - } - - // Ensure the public key is the user's active key - if cc.PublicKey != u.PublicKey() { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidSigningKey, - } - } - - // Validate signature - msg := cc.Token + cc.CommentID + cc.Reason - err := validateSignature(cc.PublicKey, cc.Signature, msg) - if err != nil { - return nil, err - } - - // Ensure censor reason is present - if cc.Reason == "" { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusCensorReasonCannotBeBlank, - } - } - - // Ensure comment exists and has not already been censored - c, err := p.decredCommentGetByID(cc.Token, cc.CommentID) - if err != nil { - return nil, fmt.Errorf("decredGetComment: %v", err) - } - if c.Censored { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusCommentNotFound, - } - } - - // Ensure proposal voting has not ended - bb, err := p.getBestBlock() - if err != nil { - return nil, fmt.Errorf("getBestBlock: %v", err) - } - vs, err := p.voteSummaryGet(cc.Token, bb) - if err != nil { - return nil, err - } - if vs.Status == www.PropVoteStatusFinished { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - ErrorContext: []string{"vote has ended"}, - } - } - - // Setup plugin command - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - - dcc := convertCensorCommentToDecred(cc) - payload, err := decredplugin.EncodeCensorComment(dcc) - if err != nil { - return nil, err - } - - pc := pd.PluginCommand{ - Challenge: hex.EncodeToString(challenge), - ID: decredplugin.ID, - Command: decredplugin.CmdCensorComment, - CommandID: decredplugin.CmdCensorComment, - Payload: string(payload), - } - - // Send plugin request - responseBody, err := p.makeRequest(http.MethodPost, - pd.PluginCommandRoute, pc) - if err != nil { - return nil, err - } - - // Handle response - var reply pd.PluginCommandReply - err = json.Unmarshal(responseBody, &reply) - if err != nil { - return nil, err - } - - err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) - if err != nil { - return nil, err - } - - ccr, err := decredplugin.DecodeCensorCommentReply([]byte(reply.Payload)) - if err != nil { - return nil, err - } - - return &www.CensorCommentReply{ - Receipt: ccr.Receipt, - }, nil - */ - - return nil, nil + return vsr, nil } diff --git a/politeiawww/convert.go b/politeiawww/convert.go deleted file mode 100644 index 3df6114d6..000000000 --- a/politeiawww/convert.go +++ /dev/null @@ -1,616 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import ( - "encoding/base64" - "encoding/json" - "fmt" - "strconv" - "strings" - - "github.com/decred/dcrd/dcrutil" - "github.com/decred/politeia/cmsplugin" - "github.com/decred/politeia/decredplugin" - "github.com/decred/politeia/mdstream" - pd "github.com/decred/politeia/politeiad/api/v1" - cms "github.com/decred/politeia/politeiawww/api/cms/v1" - pi "github.com/decred/politeia/politeiawww/api/pi/v1" - www "github.com/decred/politeia/politeiawww/api/www/v1" - "github.com/decred/politeia/politeiawww/cmsdatabase" -) - -func convertPropFileFromWWW(f www.File) pd.File { - return pd.File{ - Name: f.Name, - MIME: f.MIME, - Digest: f.Digest, - Payload: f.Payload, - } -} - -func convertPropFilesFromWWW(f []www.File) []pd.File { - files := make([]pd.File, 0, len(f)) - for _, v := range f { - files = append(files, convertPropFileFromWWW(v)) - } - return files -} - -func convertPropCensorFromPD(f pd.CensorshipRecord) www.CensorshipRecord { - return www.CensorshipRecord{ - Token: f.Token, - Merkle: f.Merkle, - Signature: f.Signature, - } -} - -func convertPropStatusToState(status www.PropStatusT) www.PropStateT { - switch status { - case www.PropStatusNotReviewed, www.PropStatusUnreviewedChanges, - www.PropStatusCensored: - return www.PropStateUnvetted - case www.PropStatusPublic, www.PropStatusAbandoned: - return www.PropStateVetted - } - return www.PropStateInvalid -} - -func convertNewCommentToDecredPlugin(nc www.NewComment) decredplugin.NewComment { - return decredplugin.NewComment{ - Token: nc.Token, - ParentID: nc.ParentID, - Comment: nc.Comment, - Signature: nc.Signature, - PublicKey: nc.PublicKey, - } -} - -func convertCommentFromDecred(c decredplugin.Comment) www.Comment { - // Upvotes, Downvotes, UserID, and Username are filled in as zero - // values since a decred plugin comment does not contain this data. - return www.Comment{ - Token: c.Token, - ParentID: c.ParentID, - Comment: c.Comment, - Signature: c.Signature, - PublicKey: c.PublicKey, - CommentID: c.CommentID, - Receipt: c.Receipt, - Timestamp: c.Timestamp, - ResultVotes: 0, - Upvotes: 0, - Downvotes: 0, - UserID: "", - Username: "", - Censored: c.Censored, - } -} - -func convertInvoiceCensorFromWWW(f www.CensorshipRecord) pd.CensorshipRecord { - return pd.CensorshipRecord{ - Token: f.Token, - Merkle: f.Merkle, - Signature: f.Signature, - } -} - -func convertRecordFileToWWW(f pd.File) www.File { - return www.File{ - Name: f.Name, - MIME: f.MIME, - Digest: f.Digest, - Payload: f.Payload, - } -} - -func convertRecordFilesToWWW(f []pd.File) []www.File { - files := make([]www.File, 0, len(f)) - for _, v := range f { - files = append(files, convertRecordFileToWWW(v)) - } - return files -} - -func convertDatabaseInvoiceToInvoiceRecord(dbInvoice cmsdatabase.Invoice) (cms.InvoiceRecord, error) { - invRec := cms.InvoiceRecord{} - invRec.Status = dbInvoice.Status - invRec.Timestamp = dbInvoice.Timestamp - invRec.UserID = dbInvoice.UserID - invRec.Username = dbInvoice.Username - invRec.PublicKey = dbInvoice.PublicKey - invRec.Version = dbInvoice.Version - invRec.Signature = dbInvoice.UserSignature - invRec.CensorshipRecord = www.CensorshipRecord{ - Token: dbInvoice.Token, - } - invInput := cms.InvoiceInput{ - ContractorContact: dbInvoice.ContractorContact, - ContractorRate: dbInvoice.ContractorRate, - ContractorName: dbInvoice.ContractorName, - ContractorLocation: dbInvoice.ContractorLocation, - PaymentAddress: dbInvoice.PaymentAddress, - Month: dbInvoice.Month, - Year: dbInvoice.Year, - ExchangeRate: dbInvoice.ExchangeRate, - } - invInputLineItems := make([]cms.LineItemsInput, 0, len(dbInvoice.LineItems)) - for _, dbLineItem := range dbInvoice.LineItems { - lineItem := cms.LineItemsInput{ - Type: dbLineItem.Type, - Domain: dbLineItem.Domain, - Subdomain: dbLineItem.Subdomain, - Description: dbLineItem.Description, - ProposalToken: dbLineItem.ProposalURL, - Labor: dbLineItem.Labor, - Expenses: dbLineItem.Expenses, - SubRate: dbLineItem.ContractorRate, - SubUserID: dbLineItem.SubUserID, - } - invInputLineItems = append(invInputLineItems, lineItem) - } - - payout, err := calculatePayout(dbInvoice) - if err != nil { - return invRec, err - } - invRec.Total = int64(payout.Total) - - invInput.LineItems = invInputLineItems - invRec.Input = invInput - invRec.Input.LineItems = invInputLineItems - txIDs := strings.Split(dbInvoice.Payments.TxIDs, ",") - payment := cms.PaymentInformation{ - Token: dbInvoice.Payments.InvoiceToken, - Address: dbInvoice.Payments.Address, - TxIDs: txIDs, - AmountReceived: dcrutil.Amount(dbInvoice.Payments.AmountReceived), - TimeLastUpdated: dbInvoice.Payments.TimeLastUpdated, - } - invRec.Payment = payment - return invRec, nil -} - -func convertInvoiceRecordToDatabaseInvoice(invRec *cms.InvoiceRecord) *cmsdatabase.Invoice { - dbInvoice := &cmsdatabase.Invoice{} - dbInvoice.Status = invRec.Status - dbInvoice.Timestamp = invRec.Timestamp - dbInvoice.UserID = invRec.UserID - dbInvoice.PublicKey = invRec.PublicKey - dbInvoice.Version = invRec.Version - dbInvoice.ContractorContact = invRec.Input.ContractorContact - dbInvoice.ContractorRate = invRec.Input.ContractorRate - dbInvoice.ContractorName = invRec.Input.ContractorName - dbInvoice.ContractorLocation = invRec.Input.ContractorLocation - dbInvoice.PaymentAddress = invRec.Input.PaymentAddress - dbInvoice.Month = invRec.Input.Month - dbInvoice.Year = invRec.Input.Year - dbInvoice.ExchangeRate = invRec.Input.ExchangeRate - dbInvoice.Token = invRec.CensorshipRecord.Token - dbInvoice.ServerSignature = invRec.Signature - - dbInvoice.LineItems = make([]cmsdatabase.LineItem, 0, len(invRec.Input.LineItems)) - for _, lineItem := range invRec.Input.LineItems { - dbLineItem := cmsdatabase.LineItem{ - Type: lineItem.Type, - Domain: lineItem.Domain, - Subdomain: lineItem.Subdomain, - Description: lineItem.Description, - ProposalURL: lineItem.ProposalToken, - Labor: lineItem.Labor, - Expenses: lineItem.Expenses, - ContractorRate: lineItem.SubRate, - SubUserID: lineItem.SubUserID, - } - dbInvoice.LineItems = append(dbInvoice.LineItems, dbLineItem) - } - return dbInvoice -} - -func convertLineItemsToDatabase(token string, l []cms.LineItemsInput) []cmsdatabase.LineItem { - dl := make([]cmsdatabase.LineItem, 0, len(l)) - for _, v := range l { - dl = append(dl, cmsdatabase.LineItem{ - InvoiceToken: token, - Type: v.Type, - Domain: v.Domain, - Subdomain: v.Subdomain, - Description: v.Description, - ProposalURL: v.ProposalToken, - Labor: v.Labor, - Expenses: v.Expenses, - // If subrate is populated, use the existing contractor rate field. - ContractorRate: v.SubRate, - }) - } - return dl -} - -func convertRecordToDatabaseInvoice(p pd.Record) (*cmsdatabase.Invoice, error) { - dbInvoice := cmsdatabase.Invoice{ - Files: convertRecordFilesToWWW(p.Files), - Token: p.CensorshipRecord.Token, - ServerSignature: p.CensorshipRecord.Signature, - Version: p.Version, - } - - // Decode invoice file - for _, v := range p.Files { - if v.Name == invoiceFile { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return nil, err - } - - var ii cms.InvoiceInput - err = json.Unmarshal(b, &ii) - if err != nil { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidInput, - } - } - - dbInvoice.Month = ii.Month - dbInvoice.Year = ii.Year - dbInvoice.ExchangeRate = ii.ExchangeRate - dbInvoice.LineItems = convertLineItemsToDatabase(dbInvoice.Token, - ii.LineItems) - dbInvoice.ContractorContact = ii.ContractorContact - dbInvoice.ContractorLocation = ii.ContractorLocation - dbInvoice.ContractorRate = ii.ContractorRate - dbInvoice.ContractorName = ii.ContractorName - dbInvoice.PaymentAddress = ii.PaymentAddress - } - } - payout, err := calculatePayout(dbInvoice) - if err != nil { - return nil, err - } - payment := cmsdatabase.Payments{ - Address: dbInvoice.PaymentAddress, - AmountNeeded: int64(payout.DCRTotal), - } - for _, m := range p.Metadata { - switch m.ID { - case mdstream.IDRecordStatusChange: - // Ignore initial stream change since it's just the automatic change from - // unvetted to vetted - continue - case mdstream.IDInvoiceGeneral: - var mdGeneral mdstream.InvoiceGeneral - err := json.Unmarshal([]byte(m.Payload), &mdGeneral) - if err != nil { - return nil, fmt.Errorf("could not decode metadata '%v' token '%v': %v", - p.Metadata, p.CensorshipRecord.Token, err) - } - - dbInvoice.Timestamp = mdGeneral.Timestamp - dbInvoice.PublicKey = mdGeneral.PublicKey - dbInvoice.UserSignature = mdGeneral.Signature - case mdstream.IDInvoiceStatusChange: - sc, err := mdstream.DecodeInvoiceStatusChange([]byte(m.Payload)) - if err != nil { - return nil, fmt.Errorf("could not decode metadata '%v' token '%v': %v", - m, p.CensorshipRecord.Token, err) - } - - invChanges := make([]cmsdatabase.InvoiceChange, 0, len(sc)) - for _, s := range sc { - invChange := cmsdatabase.InvoiceChange{ - AdminPublicKey: s.AdminPublicKey, - NewStatus: s.NewStatus, - Reason: s.Reason, - Timestamp: s.Timestamp, - } - invChanges = append(invChanges, invChange) - // Capture information about payments - dbInvoice.Status = s.NewStatus - if s.NewStatus == cms.InvoiceStatusApproved { - payment.Status = cms.PaymentStatusWatching - payment.TimeStarted = s.Timestamp - } else if s.NewStatus == cms.InvoiceStatusPaid { - payment.Status = cms.PaymentStatusPaid - } - } - dbInvoice.Changes = invChanges - - case mdstream.IDInvoicePayment: - ip, err := mdstream.DecodeInvoicePayment([]byte(m.Payload)) - if err != nil { - return nil, fmt.Errorf("could not decode metadata '%v' token '%v': %v", - m, p.CensorshipRecord.Token, err) - } - - // We don't need all of the payments. - // Just the most recent one. - for _, s := range ip { - payment.TxIDs = s.TxIDs - payment.TimeLastUpdated = s.Timestamp - payment.AmountReceived = s.AmountReceived - } - dbInvoice.Payments = payment - default: - // Log error but proceed - log.Errorf("initializeInventory: invalid "+ - "metadata stream ID %v token %v", - m.ID, p.CensorshipRecord.Token) - } - } - - return &dbInvoice, nil -} - -func convertRecordToDatabaseDCC(p pd.Record) (*cmsdatabase.DCC, error) { - dbDCC := cmsdatabase.DCC{ - Files: convertRecordFilesToWWW(p.Files), - Token: p.CensorshipRecord.Token, - ServerSignature: p.CensorshipRecord.Signature, - } - - // Decode invoice file - for _, v := range p.Files { - if v.Name == dccFile { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return nil, err - } - - var dcc cms.DCCInput - err = json.Unmarshal(b, &dcc) - if err != nil { - return nil, fmt.Errorf("could not decode DCC input data: token '%v': %v", - p.CensorshipRecord.Token, err) - } - dbDCC.Type = dcc.Type - dbDCC.NomineeUserID = dcc.NomineeUserID - dbDCC.SponsorStatement = dcc.SponsorStatement - dbDCC.Domain = dcc.Domain - dbDCC.ContractorType = dcc.ContractorType - } - } - - for _, m := range p.Metadata { - switch m.ID { - case mdstream.IDRecordStatusChange: - // Ignore initial stream change since it's just the automatic change from - // unvetted to vetted - continue - case mdstream.IDDCCGeneral: - var mdGeneral mdstream.DCCGeneral - err := json.Unmarshal([]byte(m.Payload), &mdGeneral) - if err != nil { - return nil, fmt.Errorf("could not decode metadata '%v' token '%v': %v", - p.Metadata, p.CensorshipRecord.Token, err) - } - - dbDCC.Timestamp = mdGeneral.Timestamp - dbDCC.PublicKey = mdGeneral.PublicKey - dbDCC.UserSignature = mdGeneral.Signature - - case mdstream.IDDCCStatusChange: - sc, err := mdstream.DecodeDCCStatusChange([]byte(m.Payload)) - if err != nil { - return nil, fmt.Errorf("could not decode metadata '%v' token '%v': %v", - m, p.CensorshipRecord.Token, err) - } - - // We don't need all of the status changes. - // Just the most recent one. - for _, s := range sc { - dbDCC.Status = s.NewStatus - dbDCC.StatusChangeReason = s.Reason - } - default: - // Log error but proceed - log.Errorf("initializeInventory: invalid "+ - "metadata stream ID %v token %v", - m.ID, p.CensorshipRecord.Token) - } - } - - return &dbDCC, nil -} - -func convertDCCDatabaseToRecord(dbDCC *cmsdatabase.DCC) cms.DCCRecord { - dccRecord := cms.DCCRecord{} - - dccRecord.DCC.Type = dbDCC.Type - dccRecord.DCC.NomineeUserID = dbDCC.NomineeUserID - dccRecord.DCC.SponsorStatement = dbDCC.SponsorStatement - dccRecord.DCC.Domain = dbDCC.Domain - dccRecord.DCC.ContractorType = dbDCC.ContractorType - dccRecord.Status = dbDCC.Status - dccRecord.StatusChangeReason = dbDCC.StatusChangeReason - dccRecord.Timestamp = dbDCC.Timestamp - dccRecord.CensorshipRecord = www.CensorshipRecord{ - Token: dbDCC.Token, - } - dccRecord.PublicKey = dbDCC.PublicKey - dccRecord.Signature = dbDCC.ServerSignature - dccRecord.SponsorUserID = dbDCC.SponsorUserID - supportUserIDs := strings.Split(dbDCC.SupportUserIDs, ",") - dccRecord.SupportUserIDs = supportUserIDs - oppositionUserIDs := strings.Split(dbDCC.OppositionUserIDs, ",") - dccRecord.OppositionUserIDs = oppositionUserIDs - - return dccRecord -} - -func convertDatabaseInvoiceToProposalLineItems(inv cmsdatabase.Invoice) cms.ProposalLineItems { - return cms.ProposalLineItems{ - Month: int(inv.Month), - Year: int(inv.Year), - UserID: inv.UserID, - Username: inv.Username, - LineItem: cms.LineItemsInput{ - Type: inv.LineItems[0].Type, - Domain: inv.LineItems[0].Domain, - Subdomain: inv.LineItems[0].Subdomain, - Description: inv.LineItems[0].Description, - ProposalToken: inv.LineItems[0].ProposalURL, - Labor: inv.LineItems[0].Labor, - Expenses: inv.LineItems[0].Expenses, - SubRate: inv.LineItems[0].ContractorRate, - }, - } -} - -func convertCastVoteFromCMS(b cms.CastVote) cmsplugin.CastVote { - return cmsplugin.CastVote{ - VoteBit: b.VoteBit, - Token: b.Token, - UserID: b.UserID, - Signature: b.Signature, - } -} - -func convertCastVoteReplyToCMS(cv *cmsplugin.CastVoteReply) *cms.CastVoteReply { - return &cms.CastVoteReply{ - ClientSignature: cv.ClientSignature, - Signature: cv.Signature, - Error: cv.Error, - ErrorStatus: cv.ErrorStatus, - } -} - -func convertUserWeightToCMS(uw []cmsplugin.UserWeight) []cms.DCCWeight { - dccWeight := make([]cms.DCCWeight, 0, len(uw)) - for _, w := range uw { - dccWeight = append(dccWeight, cms.DCCWeight{ - UserID: w.UserID, - Weight: w.Weight, - }) - } - return dccWeight -} - -func convertVoteOptionResultsToCMS(vr []cmsplugin.VoteOptionResult) []cms.VoteOptionResult { - votes := make([]cms.VoteOptionResult, 0, len(vr)) - for _, w := range vr { - votes = append(votes, cms.VoteOptionResult{ - Option: cms.VoteOption{ - Id: w.ID, - Description: w.Description, - Bits: w.Bits, - }, - VotesReceived: w.Votes, - }) - } - return votes -} -func convertCMSStartVoteToCMSVoteDetailsReply(sv cmsplugin.StartVote, svr cmsplugin.StartVoteReply) (*cms.VoteDetailsReply, error) { - voteb, err := cmsplugin.EncodeVote(sv.Vote) - if err != nil { - return nil, err - } - userWeights := make([]string, 0, len(sv.UserWeights)) - for _, weights := range sv.UserWeights { - userWeight := weights.UserID + "-" + strconv.Itoa(int(weights.Weight)) - userWeights = append(userWeights, userWeight) - } - return &cms.VoteDetailsReply{ - Version: uint32(sv.Version), - Vote: string(voteb), - PublicKey: sv.PublicKey, - Signature: sv.Signature, - StartBlockHeight: svr.StartBlockHeight, - StartBlockHash: svr.StartBlockHash, - EndBlockHeight: svr.EndHeight, - UserWeights: userWeights, - }, nil -} - -func convertCMSStartVoteToCMS(sv cmsplugin.StartVote) cms.StartVote { - vote := cms.Vote{ - Token: sv.Vote.Token, - Mask: sv.Vote.Mask, - Duration: sv.Vote.Duration, - QuorumPercentage: sv.Vote.QuorumPercentage, - PassPercentage: sv.Vote.PassPercentage, - } - - voteOptions := make([]cms.VoteOption, 0, len(sv.Vote.Options)) - for _, option := range sv.Vote.Options { - voteOption := cms.VoteOption{ - Id: option.Id, - Description: option.Description, - Bits: option.Bits, - } - voteOptions = append(voteOptions, voteOption) - } - vote.Options = voteOptions - - return cms.StartVote{ - Vote: vote, - PublicKey: sv.PublicKey, - Signature: sv.Signature, - } -} - -func convertCMSStartVoteReplyToCMS(svr cmsplugin.StartVoteReply) cms.StartVoteReply { - return cms.StartVoteReply{ - StartBlockHeight: svr.StartBlockHeight, - StartBlockHash: svr.StartBlockHash, - EndBlockHeight: svr.EndHeight, - } -} - -func convertStartVoteToCMS(sv cms.StartVote) cmsplugin.StartVote { - vote := cmsplugin.Vote{ - Token: sv.Vote.Token, - Mask: sv.Vote.Mask, - Duration: sv.Vote.Duration, - QuorumPercentage: sv.Vote.QuorumPercentage, - PassPercentage: sv.Vote.PassPercentage, - } - - voteOptions := make([]cmsplugin.VoteOption, 0, len(sv.Vote.Options)) - for _, option := range sv.Vote.Options { - voteOption := cmsplugin.VoteOption{ - Id: option.Id, - Description: option.Description, - Bits: option.Bits, - } - voteOptions = append(voteOptions, voteOption) - } - vote.Options = voteOptions - - return cmsplugin.StartVote{ - Token: sv.Vote.Token, - Vote: vote, - PublicKey: sv.PublicKey, - Signature: sv.Signature, - } - -} - -func filterDomainInvoice(inv *cms.InvoiceRecord) cms.InvoiceRecord { - inv.Files = nil - inv.Input.ContractorContact = "" - inv.Input.ContractorLocation = "" - inv.Input.ContractorName = "" - inv.Input.ContractorRate = 0 - - for i, li := range inv.Input.LineItems { - li.Expenses = 0 - li.SubRate = 0 - inv.Input.LineItems[i] = li - } - - return *inv -} - -func convertPiFilesFromWWW(files []www.File) []pi.File { - f := make([]pi.File, 0, len(files)) - for _, v := range files { - f = append(f, pi.File{ - Name: v.Name, - MIME: v.MIME, - Digest: v.Digest, - Payload: v.Payload, - }) - } - return f -} diff --git a/politeiawww/dcc.go b/politeiawww/dcc.go index a93e1512f..9d193708c 100644 --- a/politeiawww/dcc.go +++ b/politeiawww/dcc.go @@ -22,6 +22,7 @@ import ( pd "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/identity" cms "github.com/decred/politeia/politeiawww/api/cms/v1" + pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/cmsdatabase" "github.com/decred/politeia/politeiawww/user" @@ -79,6 +80,244 @@ func createSponsorStatementRegex() string { return buf.String() } +func convertRecordToDatabaseDCC(p pd.Record) (*cmsdatabase.DCC, error) { + dbDCC := cmsdatabase.DCC{ + Files: convertWWWFilesFromPD(p.Files), + Token: p.CensorshipRecord.Token, + ServerSignature: p.CensorshipRecord.Signature, + } + + // Decode invoice file + for _, v := range p.Files { + if v.Name == dccFile { + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return nil, err + } + + var dcc cms.DCCInput + err = json.Unmarshal(b, &dcc) + if err != nil { + return nil, fmt.Errorf("could not decode DCC input data: token '%v': %v", + p.CensorshipRecord.Token, err) + } + dbDCC.Type = dcc.Type + dbDCC.NomineeUserID = dcc.NomineeUserID + dbDCC.SponsorStatement = dcc.SponsorStatement + dbDCC.Domain = dcc.Domain + dbDCC.ContractorType = dcc.ContractorType + } + } + + for _, m := range p.Metadata { + switch m.ID { + case mdstream.IDRecordStatusChange: + // Ignore initial stream change since it's just the automatic change from + // unvetted to vetted + continue + case mdstream.IDDCCGeneral: + var mdGeneral mdstream.DCCGeneral + err := json.Unmarshal([]byte(m.Payload), &mdGeneral) + if err != nil { + return nil, fmt.Errorf("could not decode metadata '%v' token '%v': %v", + p.Metadata, p.CensorshipRecord.Token, err) + } + + dbDCC.Timestamp = mdGeneral.Timestamp + dbDCC.PublicKey = mdGeneral.PublicKey + dbDCC.UserSignature = mdGeneral.Signature + + case mdstream.IDDCCStatusChange: + sc, err := mdstream.DecodeDCCStatusChange([]byte(m.Payload)) + if err != nil { + return nil, fmt.Errorf("could not decode metadata '%v' token '%v': %v", + m, p.CensorshipRecord.Token, err) + } + + // We don't need all of the status changes. + // Just the most recent one. + for _, s := range sc { + dbDCC.Status = s.NewStatus + dbDCC.StatusChangeReason = s.Reason + } + default: + // Log error but proceed + log.Errorf("initializeInventory: invalid "+ + "metadata stream ID %v token %v", + m.ID, p.CensorshipRecord.Token) + } + } + + return &dbDCC, nil +} + +func convertDCCDatabaseToRecord(dbDCC *cmsdatabase.DCC) cms.DCCRecord { + dccRecord := cms.DCCRecord{} + + dccRecord.DCC.Type = dbDCC.Type + dccRecord.DCC.NomineeUserID = dbDCC.NomineeUserID + dccRecord.DCC.SponsorStatement = dbDCC.SponsorStatement + dccRecord.DCC.Domain = dbDCC.Domain + dccRecord.DCC.ContractorType = dbDCC.ContractorType + dccRecord.Status = dbDCC.Status + dccRecord.StatusChangeReason = dbDCC.StatusChangeReason + dccRecord.Timestamp = dbDCC.Timestamp + dccRecord.CensorshipRecord = www.CensorshipRecord{ + Token: dbDCC.Token, + } + dccRecord.PublicKey = dbDCC.PublicKey + dccRecord.Signature = dbDCC.ServerSignature + dccRecord.SponsorUserID = dbDCC.SponsorUserID + supportUserIDs := strings.Split(dbDCC.SupportUserIDs, ",") + dccRecord.SupportUserIDs = supportUserIDs + oppositionUserIDs := strings.Split(dbDCC.OppositionUserIDs, ",") + dccRecord.OppositionUserIDs = oppositionUserIDs + + return dccRecord +} + +func convertCastVoteFromCMS(b cms.CastVote) cmsplugin.CastVote { + return cmsplugin.CastVote{ + VoteBit: b.VoteBit, + Token: b.Token, + UserID: b.UserID, + Signature: b.Signature, + } +} + +func convertCastVoteReplyToCMS(cv *cmsplugin.CastVoteReply) *cms.CastVoteReply { + return &cms.CastVoteReply{ + ClientSignature: cv.ClientSignature, + Signature: cv.Signature, + Error: cv.Error, + ErrorStatus: cv.ErrorStatus, + } +} + +func convertUserWeightToCMS(uw []cmsplugin.UserWeight) []cms.DCCWeight { + dccWeight := make([]cms.DCCWeight, 0, len(uw)) + for _, w := range uw { + dccWeight = append(dccWeight, cms.DCCWeight{ + UserID: w.UserID, + Weight: w.Weight, + }) + } + return dccWeight +} + +func convertVoteOptionResultsToCMS(vr []cmsplugin.VoteOptionResult) []cms.VoteOptionResult { + votes := make([]cms.VoteOptionResult, 0, len(vr)) + for _, w := range vr { + votes = append(votes, cms.VoteOptionResult{ + Option: cms.VoteOption{ + Id: w.ID, + Description: w.Description, + Bits: w.Bits, + }, + VotesReceived: w.Votes, + }) + } + return votes +} +func convertCMSStartVoteToCMSVoteDetailsReply(sv cmsplugin.StartVote, svr cmsplugin.StartVoteReply) (*cms.VoteDetailsReply, error) { + voteb, err := cmsplugin.EncodeVote(sv.Vote) + if err != nil { + return nil, err + } + userWeights := make([]string, 0, len(sv.UserWeights)) + for _, weights := range sv.UserWeights { + userWeight := weights.UserID + "-" + strconv.Itoa(int(weights.Weight)) + userWeights = append(userWeights, userWeight) + } + return &cms.VoteDetailsReply{ + Version: uint32(sv.Version), + Vote: string(voteb), + PublicKey: sv.PublicKey, + Signature: sv.Signature, + StartBlockHeight: svr.StartBlockHeight, + StartBlockHash: svr.StartBlockHash, + EndBlockHeight: svr.EndHeight, + UserWeights: userWeights, + }, nil +} + +func convertCMSStartVoteToCMS(sv cmsplugin.StartVote) cms.StartVote { + vote := cms.Vote{ + Token: sv.Vote.Token, + Mask: sv.Vote.Mask, + Duration: sv.Vote.Duration, + QuorumPercentage: sv.Vote.QuorumPercentage, + PassPercentage: sv.Vote.PassPercentage, + } + + voteOptions := make([]cms.VoteOption, 0, len(sv.Vote.Options)) + for _, option := range sv.Vote.Options { + voteOption := cms.VoteOption{ + Id: option.Id, + Description: option.Description, + Bits: option.Bits, + } + voteOptions = append(voteOptions, voteOption) + } + vote.Options = voteOptions + + return cms.StartVote{ + Vote: vote, + PublicKey: sv.PublicKey, + Signature: sv.Signature, + } +} + +func convertCMSStartVoteReplyToCMS(svr cmsplugin.StartVoteReply) cms.StartVoteReply { + return cms.StartVoteReply{ + StartBlockHeight: svr.StartBlockHeight, + StartBlockHash: svr.StartBlockHash, + EndBlockHeight: svr.EndHeight, + } +} + +func convertStartVoteToCMS(sv cms.StartVote) cmsplugin.StartVote { + vote := cmsplugin.Vote{ + Token: sv.Vote.Token, + Mask: sv.Vote.Mask, + Duration: sv.Vote.Duration, + QuorumPercentage: sv.Vote.QuorumPercentage, + PassPercentage: sv.Vote.PassPercentage, + } + + voteOptions := make([]cmsplugin.VoteOption, 0, len(sv.Vote.Options)) + for _, option := range sv.Vote.Options { + voteOption := cmsplugin.VoteOption{ + Id: option.Id, + Description: option.Description, + Bits: option.Bits, + } + voteOptions = append(voteOptions, voteOption) + } + vote.Options = voteOptions + + return cmsplugin.StartVote{ + Token: sv.Vote.Token, + Vote: vote, + PublicKey: sv.PublicKey, + Signature: sv.Signature, + } + +} + +func convertPiFilesFromWWW(files []www.File) []pi.File { + f := make([]pi.File, 0, len(files)) + for _, v := range files { + f = append(f, pi.File{ + Name: v.Name, + MIME: v.MIME, + Digest: v.Digest, + Payload: v.Payload, + }) + } + return f +} + func (p *politeiawww) processNewDCC(nd cms.NewDCC, u *user.User) (*cms.NewDCCReply, error) { reply := &cms.NewDCCReply{} @@ -142,7 +381,7 @@ func (p *politeiawww) processNewDCC(nd cms.NewDCC, u *user.User) (*cms.NewDCCRep Payload: string(scb), }, }, - Files: convertPropFilesFromWWW(files), + Files: convertPDFilesFromWWW(files), } // Send the newrecord politeiad request @@ -245,7 +484,7 @@ func (p *politeiawww) processNewDCC(nd cms.NewDCC, u *user.User) (*cms.NewDCCRep token: pdReply.CensorshipRecord.Token, }) - cr := convertPropCensorFromPD(pdReply.CensorshipRecord) + cr := convertWWWCensorFromPD(pdReply.CensorshipRecord) reply.CensorshipRecord = cr return reply, nil @@ -683,13 +922,29 @@ func stringInSlice(arr []string, str string) bool { return false } +func validateNewComment(c www.NewComment) error { + // Validate token + if !tokenIsValid(c.Token) { + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidCensorshipToken, + } + } + // Validate max length + if len(c.Comment) > www.PolicyMaxCommentLength { + return www.UserError{ + ErrorCode: www.ErrorStatusCommentLengthExceededPolicy, + } + } + return nil +} + // processNewCommentDCC sends a new comment decred plugin command to politeaid // then fetches the new comment from the cache and returns it. func (p *politeiawww) processNewCommentDCC(nc www.NewComment, u *user.User) (*www.NewCommentReply, error) { log.Tracef("processNewCommentDCC: %v %v", nc.Token, u.ID) // Validate comment - err := validateComment(nc) + err := validateNewComment(nc) if err != nil { return nil, err } @@ -901,7 +1156,7 @@ func (p *politeiawww) processSetDCCStatus(sds cms.SetDCCStatus, u *user.User) (* // Only allow voting on All Vote DCC proposals // Get vote summary to check vote status - bb, err := p.getBestBlock() + bb, err := p.decredBestBlock() if err != nil { return nil, err } @@ -1063,13 +1318,13 @@ func (p *politeiawww) processCastVoteDCC(cv cms.CastVote, u *user.User) (*cms.Ca // Only allow voting on All Vote DCC proposals // Get vote summary to check vote status - bb, err := p.getBestBlock() + bb, err := p.decredBestBlock() if err != nil { return nil, err } // Check to make sure that the Vote hasn't ended yet. - if uint64(vdr.StartVoteReply.EndHeight) < bb { + if vdr.StartVoteReply.EndHeight < bb { return nil, www.UserError{ ErrorCode: cms.ErrorStatusDCCVoteEnded, } @@ -1276,7 +1531,7 @@ func (p *politeiawww) processActiveVoteDCC() (*cms.ActiveVoteReply, error) { } vetted := pdReply.Vetted - bb, err := p.getBestBlock() + bb, err := p.decredBestBlock() if err != nil { return nil, err } @@ -1292,7 +1547,7 @@ func (p *politeiawww) processActiveVoteDCC() (*cms.ActiveVoteReply, error) { "%v %v", r.CensorshipRecord.Token, err) continue } - if uint64(vs.EndHeight) > bb { + if vs.EndHeight > bb { active = append(active, r.CensorshipRecord.Token) } } @@ -1497,7 +1752,7 @@ func (p *politeiawww) processStartVoteDCC(sv cms.StartVote, u *user.User) (*cms. // Only allow voting on All Vote DCC proposals // Get vote summary to check vote status - bb, err := p.getBestBlock() + bb, err := p.decredBestBlock() if err != nil { return nil, err } @@ -1589,12 +1844,12 @@ func validateVoteBitDCC(vote cms.Vote, bit uint64) error { return fmt.Errorf("bit not found 0x%x", bit) } -func dccVoteStatusFromVoteSummary(r cmsplugin.VoteSummaryReply, bestBlock uint64) cms.DCCVoteStatusT { +func dccVoteStatusFromVoteSummary(r cmsplugin.VoteSummaryReply, bestBlock uint32) cms.DCCVoteStatusT { switch { case r.EndHeight == 0: return cms.DCCVoteStatusNotStarted default: - if bestBlock < uint64(r.EndHeight) { + if bestBlock < r.EndHeight { return cms.DCCVoteStatusStarted } diff --git a/politeiawww/decred.go b/politeiawww/decred.go index 27730e46a..8a73657f9 100644 --- a/politeiawww/decred.go +++ b/politeiawww/decred.go @@ -5,6 +5,8 @@ package main import ( + "fmt" + "github.com/decred/politeia/decredplugin" ) @@ -22,7 +24,11 @@ func (p *politeiawww) decredGetComments(token string) ([]decredplugin.Comment, e // Execute plugin command reply, err := p.pluginCommand(decredplugin.ID, decredplugin.CmdGetComments, - decredplugin.CmdGetComments, string(payload)) + string(payload)) + if err != nil { + return nil, fmt.Errorf("pluginCommand %v %v: %v", + decredplugin.ID, decredplugin.CmdGetComments, err) + } // Receive plugin command reply gcr, err := decredplugin.DecodeGetCommentsReply([]byte(reply)) @@ -33,22 +39,26 @@ func (p *politeiawww) decredGetComments(token string) ([]decredplugin.Comment, e return gcr.Comments, nil } -func (p *politeiawww) decredBestBlock() (*decredplugin.BestBlockReply, error) { +func (p *politeiawww) decredBestBlock() (uint32, error) { // Setup plugin command payload, err := decredplugin.EncodeBestBlock(decredplugin.BestBlock{}) if err != nil { - return nil, err + return 0, err } // Execute plugin command reply, err := p.pluginCommand(decredplugin.ID, decredplugin.CmdBestBlock, - decredplugin.CmdBestBlock, string(payload)) + string(payload)) + if err != nil { + return 0, fmt.Errorf("pluginCommand %v %v: %v", + decredplugin.ID, decredplugin.CmdBestBlock, err) + } // Receive plugin command reply bbr, err := decredplugin.DecodeBestBlockReply([]byte(reply)) if err != nil { - return nil, err + return 0, err } - return bbr, nil + return bbr.Height, nil } diff --git a/politeiawww/invoices.go b/politeiawww/invoices.go index d52a4833a..07fe4f4e1 100644 --- a/politeiawww/invoices.go +++ b/politeiawww/invoices.go @@ -82,6 +82,334 @@ var ( validContact = regexp.MustCompile(createContactRegex()) ) +func convertPDFileFromWWW(f www.File) pd.File { + return pd.File{ + Name: f.Name, + MIME: f.MIME, + Digest: f.Digest, + Payload: f.Payload, + } +} + +func convertPDFilesFromWWW(f []www.File) []pd.File { + files := make([]pd.File, 0, len(f)) + for _, v := range f { + files = append(files, convertPDFileFromWWW(v)) + } + return files +} + +func convertPDCensorFromWWW(f www.CensorshipRecord) pd.CensorshipRecord { + return pd.CensorshipRecord{ + Token: f.Token, + Merkle: f.Merkle, + Signature: f.Signature, + } +} + +func convertWWWFileFromPD(f pd.File) www.File { + return www.File{ + Name: f.Name, + MIME: f.MIME, + Digest: f.Digest, + Payload: f.Payload, + } +} + +func convertWWWFilesFromPD(f []pd.File) []www.File { + files := make([]www.File, 0, len(f)) + for _, v := range f { + files = append(files, convertWWWFileFromPD(v)) + } + return files +} + +func convertWWWCensorFromPD(f pd.CensorshipRecord) www.CensorshipRecord { + return www.CensorshipRecord{ + Token: f.Token, + Merkle: f.Merkle, + Signature: f.Signature, + } +} + +func convertNewCommentToDecredPlugin(nc www.NewComment) decredplugin.NewComment { + return decredplugin.NewComment{ + Token: nc.Token, + ParentID: nc.ParentID, + Comment: nc.Comment, + Signature: nc.Signature, + PublicKey: nc.PublicKey, + } +} + +func convertCommentFromDecred(c decredplugin.Comment) www.Comment { + // Upvotes, Downvotes, UserID, and Username are filled in as zero + // values since a decred plugin comment does not contain this data. + return www.Comment{ + Token: c.Token, + ParentID: c.ParentID, + Comment: c.Comment, + Signature: c.Signature, + PublicKey: c.PublicKey, + CommentID: c.CommentID, + Receipt: c.Receipt, + Timestamp: c.Timestamp, + ResultVotes: 0, + Upvotes: 0, + Downvotes: 0, + UserID: "", + Username: "", + Censored: c.Censored, + } +} + +func convertDatabaseInvoiceToInvoiceRecord(dbInvoice cmsdatabase.Invoice) (cms.InvoiceRecord, error) { + invRec := cms.InvoiceRecord{} + invRec.Status = dbInvoice.Status + invRec.Timestamp = dbInvoice.Timestamp + invRec.UserID = dbInvoice.UserID + invRec.Username = dbInvoice.Username + invRec.PublicKey = dbInvoice.PublicKey + invRec.Version = dbInvoice.Version + invRec.Signature = dbInvoice.UserSignature + invRec.CensorshipRecord = www.CensorshipRecord{ + Token: dbInvoice.Token, + } + invInput := cms.InvoiceInput{ + ContractorContact: dbInvoice.ContractorContact, + ContractorRate: dbInvoice.ContractorRate, + ContractorName: dbInvoice.ContractorName, + ContractorLocation: dbInvoice.ContractorLocation, + PaymentAddress: dbInvoice.PaymentAddress, + Month: dbInvoice.Month, + Year: dbInvoice.Year, + ExchangeRate: dbInvoice.ExchangeRate, + } + invInputLineItems := make([]cms.LineItemsInput, 0, len(dbInvoice.LineItems)) + for _, dbLineItem := range dbInvoice.LineItems { + lineItem := cms.LineItemsInput{ + Type: dbLineItem.Type, + Domain: dbLineItem.Domain, + Subdomain: dbLineItem.Subdomain, + Description: dbLineItem.Description, + ProposalToken: dbLineItem.ProposalURL, + Labor: dbLineItem.Labor, + Expenses: dbLineItem.Expenses, + SubRate: dbLineItem.ContractorRate, + SubUserID: dbLineItem.SubUserID, + } + invInputLineItems = append(invInputLineItems, lineItem) + } + + payout, err := calculatePayout(dbInvoice) + if err != nil { + return invRec, err + } + invRec.Total = int64(payout.Total) + + invInput.LineItems = invInputLineItems + invRec.Input = invInput + invRec.Input.LineItems = invInputLineItems + txIDs := strings.Split(dbInvoice.Payments.TxIDs, ",") + payment := cms.PaymentInformation{ + Token: dbInvoice.Payments.InvoiceToken, + Address: dbInvoice.Payments.Address, + TxIDs: txIDs, + AmountReceived: dcrutil.Amount(dbInvoice.Payments.AmountReceived), + TimeLastUpdated: dbInvoice.Payments.TimeLastUpdated, + } + invRec.Payment = payment + return invRec, nil +} + +func convertInvoiceRecordToDatabaseInvoice(invRec *cms.InvoiceRecord) *cmsdatabase.Invoice { + dbInvoice := &cmsdatabase.Invoice{} + dbInvoice.Status = invRec.Status + dbInvoice.Timestamp = invRec.Timestamp + dbInvoice.UserID = invRec.UserID + dbInvoice.PublicKey = invRec.PublicKey + dbInvoice.Version = invRec.Version + dbInvoice.ContractorContact = invRec.Input.ContractorContact + dbInvoice.ContractorRate = invRec.Input.ContractorRate + dbInvoice.ContractorName = invRec.Input.ContractorName + dbInvoice.ContractorLocation = invRec.Input.ContractorLocation + dbInvoice.PaymentAddress = invRec.Input.PaymentAddress + dbInvoice.Month = invRec.Input.Month + dbInvoice.Year = invRec.Input.Year + dbInvoice.ExchangeRate = invRec.Input.ExchangeRate + dbInvoice.Token = invRec.CensorshipRecord.Token + dbInvoice.ServerSignature = invRec.Signature + + dbInvoice.LineItems = make([]cmsdatabase.LineItem, 0, len(invRec.Input.LineItems)) + for _, lineItem := range invRec.Input.LineItems { + dbLineItem := cmsdatabase.LineItem{ + Type: lineItem.Type, + Domain: lineItem.Domain, + Subdomain: lineItem.Subdomain, + Description: lineItem.Description, + ProposalURL: lineItem.ProposalToken, + Labor: lineItem.Labor, + Expenses: lineItem.Expenses, + ContractorRate: lineItem.SubRate, + SubUserID: lineItem.SubUserID, + } + dbInvoice.LineItems = append(dbInvoice.LineItems, dbLineItem) + } + return dbInvoice +} + +func convertLineItemsToDatabase(token string, l []cms.LineItemsInput) []cmsdatabase.LineItem { + dl := make([]cmsdatabase.LineItem, 0, len(l)) + for _, v := range l { + dl = append(dl, cmsdatabase.LineItem{ + InvoiceToken: token, + Type: v.Type, + Domain: v.Domain, + Subdomain: v.Subdomain, + Description: v.Description, + ProposalURL: v.ProposalToken, + Labor: v.Labor, + Expenses: v.Expenses, + // If subrate is populated, use the existing contractor rate field. + ContractorRate: v.SubRate, + }) + } + return dl +} + +func convertRecordToDatabaseInvoice(p pd.Record) (*cmsdatabase.Invoice, error) { + dbInvoice := cmsdatabase.Invoice{ + Files: convertWWWFilesFromPD(p.Files), + Token: p.CensorshipRecord.Token, + ServerSignature: p.CensorshipRecord.Signature, + Version: p.Version, + } + + // Decode invoice file + for _, v := range p.Files { + if v.Name == invoiceFile { + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return nil, err + } + + var ii cms.InvoiceInput + err = json.Unmarshal(b, &ii) + if err != nil { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidInput, + } + } + + dbInvoice.Month = ii.Month + dbInvoice.Year = ii.Year + dbInvoice.ExchangeRate = ii.ExchangeRate + dbInvoice.LineItems = convertLineItemsToDatabase(dbInvoice.Token, + ii.LineItems) + dbInvoice.ContractorContact = ii.ContractorContact + dbInvoice.ContractorLocation = ii.ContractorLocation + dbInvoice.ContractorRate = ii.ContractorRate + dbInvoice.ContractorName = ii.ContractorName + dbInvoice.PaymentAddress = ii.PaymentAddress + } + } + payout, err := calculatePayout(dbInvoice) + if err != nil { + return nil, err + } + payment := cmsdatabase.Payments{ + Address: dbInvoice.PaymentAddress, + AmountNeeded: int64(payout.DCRTotal), + } + for _, m := range p.Metadata { + switch m.ID { + case mdstream.IDRecordStatusChange: + // Ignore initial stream change since it's just the automatic change from + // unvetted to vetted + continue + case mdstream.IDInvoiceGeneral: + var mdGeneral mdstream.InvoiceGeneral + err := json.Unmarshal([]byte(m.Payload), &mdGeneral) + if err != nil { + return nil, fmt.Errorf("could not decode metadata '%v' token '%v': %v", + p.Metadata, p.CensorshipRecord.Token, err) + } + + dbInvoice.Timestamp = mdGeneral.Timestamp + dbInvoice.PublicKey = mdGeneral.PublicKey + dbInvoice.UserSignature = mdGeneral.Signature + case mdstream.IDInvoiceStatusChange: + sc, err := mdstream.DecodeInvoiceStatusChange([]byte(m.Payload)) + if err != nil { + return nil, fmt.Errorf("could not decode metadata '%v' token '%v': %v", + m, p.CensorshipRecord.Token, err) + } + + invChanges := make([]cmsdatabase.InvoiceChange, 0, len(sc)) + for _, s := range sc { + invChange := cmsdatabase.InvoiceChange{ + AdminPublicKey: s.AdminPublicKey, + NewStatus: s.NewStatus, + Reason: s.Reason, + Timestamp: s.Timestamp, + } + invChanges = append(invChanges, invChange) + // Capture information about payments + dbInvoice.Status = s.NewStatus + if s.NewStatus == cms.InvoiceStatusApproved { + payment.Status = cms.PaymentStatusWatching + payment.TimeStarted = s.Timestamp + } else if s.NewStatus == cms.InvoiceStatusPaid { + payment.Status = cms.PaymentStatusPaid + } + } + dbInvoice.Changes = invChanges + + case mdstream.IDInvoicePayment: + ip, err := mdstream.DecodeInvoicePayment([]byte(m.Payload)) + if err != nil { + return nil, fmt.Errorf("could not decode metadata '%v' token '%v': %v", + m, p.CensorshipRecord.Token, err) + } + + // We don't need all of the payments. + // Just the most recent one. + for _, s := range ip { + payment.TxIDs = s.TxIDs + payment.TimeLastUpdated = s.Timestamp + payment.AmountReceived = s.AmountReceived + } + dbInvoice.Payments = payment + default: + // Log error but proceed + log.Errorf("initializeInventory: invalid "+ + "metadata stream ID %v token %v", + m.ID, p.CensorshipRecord.Token) + } + } + + return &dbInvoice, nil +} + +func convertDatabaseInvoiceToProposalLineItems(inv cmsdatabase.Invoice) cms.ProposalLineItems { + return cms.ProposalLineItems{ + Month: int(inv.Month), + Year: int(inv.Year), + UserID: inv.UserID, + Username: inv.Username, + LineItem: cms.LineItemsInput{ + Type: inv.LineItems[0].Type, + Domain: inv.LineItems[0].Domain, + Subdomain: inv.LineItems[0].Subdomain, + Description: inv.LineItems[0].Description, + ProposalToken: inv.LineItems[0].ProposalURL, + Labor: inv.LineItems[0].Labor, + Expenses: inv.LineItems[0].Expenses, + SubRate: inv.LineItems[0].ContractorRate, + }, + } +} + // formatInvoiceField normalizes an invoice field without leading and // trailing spaces. func formatInvoiceField(field string) string { @@ -342,7 +670,7 @@ func (p *politeiawww) processNewInvoice(ni cms.NewInvoice, u *user.User) (*cms.N Payload: string(scb), }, }, - Files: convertPropFilesFromWWW(ni.Files), + Files: convertPDFilesFromWWW(ni.Files), } // Handle test case @@ -359,7 +687,7 @@ func (p *politeiawww) processNewInvoice(ni cms.NewInvoice, u *user.User) (*cms.N } return &cms.NewInvoiceReply{ - CensorshipRecord: convertPropCensorFromPD(testReply.CensorshipRecord), + CensorshipRecord: convertWWWCensorFromPD(testReply.CensorshipRecord), }, nil } @@ -459,7 +787,7 @@ func (p *politeiawww) processNewInvoice(ni cms.NewInvoice, u *user.User) (*cms.N if err != nil { return nil, err } - cr := convertPropCensorFromPD(pdReply.CensorshipRecord) + cr := convertWWWCensorFromPD(pdReply.CensorshipRecord) return &cms.NewInvoiceReply{ CensorshipRecord: cr, @@ -805,6 +1133,22 @@ func (p *politeiawww) validateInvoice(ni cms.NewInvoice, u *user.CMSUser) error return nil } +func filterDomainInvoice(inv *cms.InvoiceRecord) cms.InvoiceRecord { + inv.Files = nil + inv.Input.ContractorContact = "" + inv.Input.ContractorLocation = "" + inv.Input.ContractorName = "" + inv.Input.ContractorRate = 0 + + for i, li := range inv.Input.LineItems { + li.Expenses = 0 + li.SubRate = 0 + inv.Input.LineItems[i] = li + } + + return *inv +} + // processInvoiceDetails fetches a specific proposal version from the invoice // db and returns it. func (p *politeiawww) processInvoiceDetails(invDetails cms.InvoiceDetails, u *user.User) (*cms.InvoiceDetailsReply, error) { @@ -1177,7 +1521,7 @@ func (p *politeiawww) processEditInvoice(ei cms.EditInvoice, u *user.User) (*cms Token: ei.Token, Challenge: hex.EncodeToString(challenge), MDOverwrite: mds, - FilesAdd: convertPropFilesFromWWW(ei.Files), + FilesAdd: convertPDFilesFromWWW(ei.Files), FilesDel: delFiles, } @@ -1255,9 +1599,9 @@ func (p *politeiawww) processEditInvoice(ei cms.EditInvoice, u *user.User) (*cms // Update the cmsdb dbInvoice, err := convertRecordToDatabaseInvoice(pd.Record{ - Files: convertPropFilesFromWWW(ei.Files), + Files: convertPDFilesFromWWW(ei.Files), Metadata: mds, - CensorshipRecord: convertInvoiceCensorFromWWW(invRec.CensorshipRecord), + CensorshipRecord: convertPDCensorFromWWW(invRec.CensorshipRecord), Version: invRec.Version, }) if err != nil { @@ -1603,7 +1947,7 @@ func (p *politeiawww) processNewCommentInvoice(nc www.NewComment, u *user.User) } // Validate comment - err = validateComment(nc) + err = validateNewComment(nc) if err != nil { return nil, err } diff --git a/politeiawww/pi.go b/politeiawww/pi.go index 7f64fb3e6..4b1d909ef 100644 --- a/politeiawww/pi.go +++ b/politeiawww/pi.go @@ -8,58 +8,47 @@ import ( piplugin "github.com/decred/politeia/plugins/pi" ) -// voteInventoryPi calls the pi plugin to retrieve the token inventory. -func (p *politeiawww) voteInventoryPi(vi piplugin.VoteInventory) (*piplugin.VoteInventoryReply, error) { - // Prep plugin payload - payload, err := piplugin.EncodeVoteInventory(vi) +// piProposals returns the pi plugin data for the provided proposals. +func (p *politeiawww) piProposals(ps piplugin.Proposals) (*piplugin.ProposalsReply, error) { + b, err := piplugin.EncodeProposals(ps) if err != nil { return nil, err } - - r, err := p.pluginCommand(piplugin.ID, piplugin.CmdVoteInventory, "", - string(payload)) + r, err := p.pluginCommand(piplugin.ID, piplugin.CmdProposals, string(b)) if err != nil { return nil, err } - vir, err := piplugin.DecodeVoteInventoryReply(([]byte(r))) + pr, err := piplugin.DecodeProposalsReply([]byte(r)) if err != nil { return nil, err } - - return vir, nil + return pr, nil } -// commentCensorPi calls the pi plugin to censor a given comment. -func (p *politeiawww) commentCensorPi(cc piplugin.CommentCensor) (*piplugin.CommentCensorReply, error) { - // Prep plugin payload - payload, err := piplugin.EncodeCommentCensor(cc) +// piCommentNew uses the pi plugin to submit a new comment. +func (p *politeiawww) piCommentNew(cn piplugin.CommentNew) (*piplugin.CommentNewReply, error) { + b, err := piplugin.EncodeCommentNew(cn) if err != nil { return nil, err } - - r, err := p.pluginCommand(piplugin.ID, piplugin.CmdCommentCensor, "", - string(payload)) + r, err := p.pluginCommand(piplugin.ID, piplugin.CmdCommentNew, string(b)) if err != nil { return nil, err } - ccr, err := piplugin.DecodeCommentCensorReply(([]byte(r))) + cnr, err := piplugin.DecodeCommentNewReply([]byte(r)) if err != nil { return nil, err } - - return ccr, nil + return cnr, nil } -// commentVotePi calls the pi plugin to vote on a comment. -func (p *politeiawww) commentVotePi(cvp piplugin.CommentVote) (*piplugin.CommentVoteReply, error) { - // Prep comment vote payload - payload, err := piplugin.EncodeCommentVote(cvp) +// piCommentVote uses the pi plugin to vote on a comment. +func (p *politeiawww) piCommentVote(cvp piplugin.CommentVote) (*piplugin.CommentVoteReply, error) { + b, err := piplugin.EncodeCommentVote(cvp) if err != nil { return nil, err } - - r, err := p.pluginCommand(piplugin.ID, piplugin.CmdCommentVote, "", - string(payload)) + r, err := p.pluginCommand(piplugin.ID, piplugin.CmdCommentVote, string(b)) if err != nil { return nil, err } @@ -67,27 +56,35 @@ func (p *politeiawww) commentVotePi(cvp piplugin.CommentVote) (*piplugin.Comment if err != nil { return nil, err } - return cvr, nil } -// commentNewPi calls the pi plugin to add new comment. -func (p *politeiawww) commentNewPi(cnp piplugin.CommentNew) (*piplugin.CommentNewReply, error) { - // Prep new comment payload - payload, err := piplugin.EncodeCommentNew(cnp) +// piCommentCensor uses the pi plugin to censor a proposal comment. +func (p *politeiawww) piCommentCensor(cc piplugin.CommentCensor) (*piplugin.CommentCensorReply, error) { + b, err := piplugin.EncodeCommentCensor(cc) if err != nil { return nil, err } - - r, err := p.pluginCommand(piplugin.ID, piplugin.CmdCommentNew, "", - string(payload)) + r, err := p.pluginCommand(piplugin.ID, piplugin.CmdCommentCensor, string(b)) if err != nil { return nil, err } - cnr, err := piplugin.DecodeCommentNewReply([]byte(r)) + ccr, err := piplugin.DecodeCommentCensorReply(([]byte(r))) if err != nil { return nil, err } + return ccr, nil +} - return cnr, nil +// piVoteInventory returns the pi plugin vote inventory. +func (p *politeiawww) piVoteInventory() (*piplugin.VoteInventoryReply, error) { + r, err := p.pluginCommand(piplugin.ID, piplugin.CmdVoteInventory, "") + if err != nil { + return nil, err + } + vir, err := piplugin.DecodeVoteInventoryReply(([]byte(r))) + if err != nil { + return nil, err + } + return vir, nil } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index a1eae6b8e..00740f40d 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -247,70 +247,6 @@ func convertCensorshipRecordFromPD(cr pd.CensorshipRecord) pi.CensorshipRecord { } } -func convertCommentsPluginPropStateFromPi(s pi.PropStateT) comments.StateT { - switch s { - case pi.PropStateUnvetted: - return comments.StateUnvetted - case pi.PropStateVetted: - return comments.StateVetted - } - return comments.StateInvalid -} - -func convertPropStateFromComments(s comments.StateT) pi.PropStateT { - switch s { - case comments.StateUnvetted: - return pi.PropStateUnvetted - case comments.StateVetted: - return pi.PropStateVetted - } - return pi.PropStateInvalid -} - -func convertCommentFromPlugin(cm comments.Comment) pi.Comment { - return pi.Comment{ - UserID: cm.UserID, - Username: "", // Intentionally omitted, needs to be pulled from userdb - State: convertPropStateFromComments(cm.State), - Token: cm.Token, - ParentID: cm.ParentID, - Comment: cm.Comment, - PublicKey: cm.PublicKey, - Signature: cm.Signature, - CommentID: cm.CommentID, - Version: cm.Version, - Timestamp: cm.Timestamp, - Receipt: cm.Receipt, - Score: cm.Score, - Deleted: cm.Deleted, - Reason: cm.Reason, - } -} - -func convertVoteFromComments(v comments.VoteT) pi.CommentVoteT { - switch v { - case comments.VoteDownvote: - return pi.CommentVoteDownvote - case comments.VoteUpvote: - return pi.CommentVoteUpvote - } - return pi.CommentVoteInvalid -} - -func convertCommentVoteFromPlugin(cv comments.CommentVote) pi.CommentVoteDetails { - return pi.CommentVoteDetails{ - UserID: cv.UserID, - State: convertPropStateFromComments(cv.State), - Token: cv.Token, - CommentID: cv.CommentID, - Vote: convertVoteFromComments(cv.Vote), - PublicKey: cv.PublicKey, - Signature: cv.Signature, - Timestamp: cv.Timestamp, - Receipt: cv.Receipt, - } -} - func convertFilesFromPD(f []pd.File) ([]pi.File, []pi.Metadata) { files := make([]pi.File, 0, len(f)) metadata := make([]pi.Metadata, 0, len(f)) @@ -396,138 +332,429 @@ func convertProposalRecordFromPD(r pd.Record) (*pi.ProposalRecord, error) { }, nil } -func convertInventoryReplyFromPD(i pd.InventoryByStatusReply) pi.ProposalInventoryReply { - // Concatenate both unvetted status from d - unvetted := append(i.Unvetted, i.IterationUnvetted...) - - return pi.ProposalInventoryReply{ - Unvetted: unvetted, - Public: i.Vetted, - Censored: i.Censored, - Abandoned: i.Archived, +func convertCommentsStateFromPi(s pi.PropStateT) comments.StateT { + switch s { + case pi.PropStateUnvetted: + return comments.StateUnvetted + case pi.PropStateVetted: + return comments.StateVetted } + return comments.StateInvalid } -// linkByPeriodMin returns the minimum amount of time, in seconds, that the -// LinkBy period must be set to. This is determined by adding 1 week onto the -// minimum voting period so that RFP proposal submissions have at least one -// week to be submitted after the proposal vote ends. -func (p *politeiawww) linkByPeriodMin() int64 { - var ( - submissionPeriod int64 = 604800 // One week in seconds - blockTime int64 // In seconds - ) - switch { - case p.cfg.TestNet: - blockTime = int64(testNet3Params.TargetTimePerBlock.Seconds()) - case p.cfg.SimNet: - blockTime = int64(simNetParams.TargetTimePerBlock.Seconds()) - default: - blockTime = int64(mainNetParams.TargetTimePerBlock.Seconds()) +func convertPropStateFromComments(s comments.StateT) pi.PropStateT { + switch s { + case comments.StateUnvetted: + return pi.PropStateUnvetted + case comments.StateVetted: + return pi.PropStateVetted } - return (int64(p.cfg.VoteDurationMin) * blockTime) + submissionPeriod + return pi.PropStateInvalid } -// linkByPeriodMax returns the maximum amount of time, in seconds, that the -// LinkBy period can be set to. 3 months is currently hard coded with no real -// reason for deciding on 3 months besides that it sounds like a sufficient -// amount of time. This can be changed if there is a valid reason to. -func (p *politeiawww) linkByPeriodMax() int64 { - return 7776000 // 3 months in seconds +func convertCommentFromPlugin(cm comments.Comment) pi.Comment { + return pi.Comment{ + UserID: cm.UserID, + Username: "", // Intentionally omitted, needs to be pulled from userdb + State: convertPropStateFromComments(cm.State), + Token: cm.Token, + ParentID: cm.ParentID, + Comment: cm.Comment, + PublicKey: cm.PublicKey, + Signature: cm.Signature, + CommentID: cm.CommentID, + Version: cm.Version, + Timestamp: cm.Timestamp, + Receipt: cm.Receipt, + Score: cm.Score, + Deleted: cm.Deleted, + Reason: cm.Reason, + } } -// proposalPluginData fetches the plugin data for the provided proposals using -// the pi plugin proposals command. -func (p *politeiawww) proposalPluginData(ps piplugin.Proposals) (*piplugin.ProposalsReply, error) { - // Setup plugin command - payload, err := piplugin.EncodeProposals(ps) - if err != nil { - return nil, err - } - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - pc := pd.PluginCommand{ - Challenge: hex.EncodeToString(challenge), - ID: piplugin.ID, - Command: piplugin.CmdProposals, - Payload: string(payload), +func convertCommentVoteFromPlugin(v comments.VoteT) pi.CommentVoteT { + switch v { + case comments.VoteDownvote: + return pi.CommentVoteDownvote + case comments.VoteUpvote: + return pi.CommentVoteUpvote } + return pi.CommentVoteInvalid +} - // Send plugin command - resBody, err := p.makeRequest(http.MethodPost, pd.PluginCommandRoute, pc) - if err != nil { - return nil, err - } - var pcr pd.PluginCommandReply - err = json.Unmarshal(resBody, &pcr) - if err != nil { - return nil, err - } - err = util.VerifyChallenge(p.cfg.Identity, challenge, pcr.Response) - if err != nil { - return nil, err - } - pr, err := piplugin.DecodeProposalsReply([]byte(pcr.Payload)) - if err != nil { - return nil, err +func convertCommentVoteDetailsFromPlugin(cv comments.CommentVote) pi.CommentVoteDetails { + return pi.CommentVoteDetails{ + UserID: cv.UserID, + State: convertPropStateFromComments(cv.State), + Token: cv.Token, + CommentID: cv.CommentID, + Vote: convertCommentVoteFromPlugin(cv.Vote), + PublicKey: cv.PublicKey, + Signature: cv.Signature, + Timestamp: cv.Timestamp, + Receipt: cv.Receipt, } +} - return pr, nil +func convertCommentVoteFromPi(v pi.CommentVoteT) piplugin.CommentVoteT { + switch v { + case pi.CommentVoteInvalid: + return piplugin.VoteInvalid + case pi.CommentVoteDownvote: + return piplugin.VoteDownvote + case pi.CommentVoteUpvote: + return piplugin.VoteUpvote + default: + return piplugin.VoteInvalid + } } -// proposalRecord returns the proposal record for the provided token and -// version. -func (p *politeiawww) proposalRecord(state pi.PropStateT, token, version string) (*pi.ProposalRecord, error) { - // Get politeiad record - var r *pd.Record - var err error - switch state { - case pi.PropStateUnvetted: - r, err = p.getUnvetted(token, version) - if err != nil { - return nil, err - } - case pi.PropStateVetted: - r, err = p.getVetted(token, version) - if err != nil { - return nil, err - } +func convertVoteAuthActionFromPi(a pi.VoteAuthActionT) ticketvote.AuthActionT { + switch a { + case pi.VoteAuthActionAuthorize: + return ticketvote.ActionAuthorize + case pi.VoteAuthActionRevoke: + return ticketvote.ActionRevoke default: - return nil, fmt.Errorf("unknown state %v", state) + return ticketvote.ActionAuthorize } +} - pr, err := convertProposalRecordFromPD(*r) - if err != nil { - return nil, err +func convertVoteAuthorizeFromPi(va pi.VoteAuthorize) ticketvote.Authorize { + return ticketvote.Authorize{ + Token: va.Token, + Version: va.Version, + Action: convertVoteAuthActionFromPi(va.Action), + PublicKey: va.PublicKey, + Signature: va.Signature, } +} - // Get proposal plugin data - ps := piplugin.Proposals{ - State: convertPropStateFromPi(state), - Tokens: []string{token}, +func convertVoteAuthsFromPi(auths []pi.VoteAuthorize) []ticketvote.Authorize { + a := make([]ticketvote.Authorize, 0, len(auths)) + for _, v := range auths { + a = append(a, convertVoteAuthorizeFromPi(v)) } - psr, err := p.proposalPluginData(ps) - if err != nil { - return nil, err + return a +} + +func convertVoteTypeFromPi(t pi.VoteT) ticketvote.VoteT { + switch t { + case pi.VoteTypeStandard: + return ticketvote.VoteTypeStandard + case pi.VoteTypeRunoff: + return ticketvote.VoteTypeRunoff } - d, ok := psr.Proposals[token] - if !ok { - return nil, fmt.Errorf("proposal plugin data not found %v", token) + return ticketvote.VoteTypeInvalid +} + +func convertVoteParamsFromPi(v pi.VoteParams) ticketvote.VoteParams { + tv := ticketvote.VoteParams{ + Token: v.Token, + Version: v.Version, + Type: convertVoteTypeFromPi(v.Type), + Mask: v.Mask, + Duration: v.Duration, + QuorumPercentage: v.QuorumPercentage, + PassPercentage: v.PassPercentage, } - pr.Comments = d.Comments - pr.LinkedFrom = d.LinkedFrom + // Convert vote options + vo := make([]ticketvote.VoteOption, 0, len(v.Options)) + for _, vi := range v.Options { + vo = append(vo, ticketvote.VoteOption{ + ID: vi.ID, + Description: vi.Description, + Bit: vi.Bit, + }) + } + tv.Options = vo - // Get user data - u, err := p.db.UserGetByPubKey(pr.PublicKey) - if err != nil { - return nil, err + return tv +} + +func convertVoteStartFromPi(vs pi.VoteStart) ticketvote.Start { + return ticketvote.Start{ + Params: convertVoteParamsFromPi(vs.Params), + PublicKey: vs.PublicKey, + Signature: vs.Signature, } - pr.UserID = u.ID.String() - pr.Username = u.Username +} - return pr, nil +func convertVoteStartsFromPi(starts []pi.VoteStart) []ticketvote.Start { + s := make([]ticketvote.Start, 0, len(starts)) + for _, v := range starts { + s = append(s, convertVoteStartFromPi(v)) + } + return s +} + +func convertCastVotesFromPi(votes []pi.CastVote) []ticketvote.CastVote { + cv := make([]ticketvote.CastVote, 0, len(votes)) + for _, v := range votes { + cv = append(cv, ticketvote.CastVote{ + Token: v.Token, + Ticket: v.Ticket, + VoteBit: v.VoteBit, + Signature: v.Signature, + }) + } + return cv +} + +func convertVoteErrorFromPlugin(e ticketvote.VoteErrorT) pi.VoteErrorT { + switch e { + case ticketvote.VoteErrorInvalid: + return pi.VoteErrorInvalid + case ticketvote.VoteErrorInternalError: + return pi.VoteErrorInternalError + case ticketvote.VoteErrorRecordNotFound: + return pi.VoteErrorRecordNotFound + case ticketvote.VoteErrorVoteBitInvalid: + return pi.VoteErrorVoteBitInvalid + case ticketvote.VoteErrorVoteStatusInvalid: + return pi.VoteErrorVoteStatusInvalid + case ticketvote.VoteErrorTicketAlreadyVoted: + return pi.VoteErrorTicketAlreadyVoted + case ticketvote.VoteErrorTicketNotEligible: + return pi.VoteErrorTicketNotEligible + default: + return pi.VoteErrorInternalError + } +} + +func convertVoteTypeFromPlugin(t ticketvote.VoteT) pi.VoteT { + switch t { + case ticketvote.VoteTypeStandard: + return pi.VoteTypeStandard + case ticketvote.VoteTypeRunoff: + return pi.VoteTypeRunoff + } + return pi.VoteTypeInvalid + +} + +func convertVoteParamsFromPlugin(v ticketvote.VoteParams) pi.VoteParams { + vp := pi.VoteParams{ + Token: v.Token, + Version: v.Version, + Type: convertVoteTypeFromPlugin(v.Type), + Mask: v.Mask, + Duration: v.Duration, + QuorumPercentage: v.QuorumPercentage, + PassPercentage: v.PassPercentage, + } + vo := make([]pi.VoteOption, 0, len(v.Options)) + for _, o := range v.Options { + vo = append(vo, pi.VoteOption{ + ID: o.ID, + Description: o.Description, + Bit: o.Bit, + }) + } + vp.Options = vo + + return vp +} + +func convertCastVoteRepliesFromPlugin(replies []ticketvote.CastVoteReply) []pi.CastVoteReply { + r := make([]pi.CastVoteReply, 0, len(replies)) + for _, v := range replies { + r = append(r, pi.CastVoteReply{ + Ticket: v.Ticket, + Receipt: v.Receipt, + ErrorCode: convertVoteErrorFromPlugin(v.ErrorCode), + ErrorContext: v.ErrorContext, + }) + } + return r +} + +func convertVoteDetailsFromPlugin(vd ticketvote.VoteDetails) pi.VoteDetails { + return pi.VoteDetails{ + Params: convertVoteParamsFromPlugin(vd.Params), + PublicKey: vd.PublicKey, + Signature: vd.Signature, + StartBlockHeight: vd.StartBlockHeight, + StartBlockHash: vd.StartBlockHash, + EndBlockHeight: vd.EndBlockHeight, + EligibleTickets: vd.EligibleTickets, + } +} + +func convertAuthorizeDetailsFromPlugin(auths []ticketvote.AuthorizeDetails) []pi.AuthorizeDetails { + a := make([]pi.AuthorizeDetails, 0, len(auths)) + for _, v := range auths { + a = append(a, pi.AuthorizeDetails{ + Token: v.Token, + Version: v.Version, + Action: v.Action, + PublicKey: v.PublicKey, + Signature: v.Signature, + Timestamp: v.Timestamp, + Receipt: v.Receipt, + }) + } + return a +} + +func convertCastVoteDetailsFromPlugin(votes []ticketvote.CastVoteDetails) []pi.CastVoteDetails { + vs := make([]pi.CastVoteDetails, 0, len(votes)) + for _, v := range votes { + vs = append(vs, pi.CastVoteDetails{ + Token: v.Token, + Ticket: v.Ticket, + VoteBit: v.VoteBit, + Signature: v.Signature, + Receipt: v.Receipt, + }) + } + return vs +} + +func convertProposalVotesFromPlugin(votes map[string]ticketvote.RecordVote) map[string]pi.ProposalVote { + pv := make(map[string]pi.ProposalVote, len(votes)) + for k, v := range votes { + var vdp *pi.VoteDetails + if v.Vote != nil { + vd := convertVoteDetailsFromPlugin(*v.Vote) + vdp = &vd + } + pv[k] = pi.ProposalVote{ + Auths: convertAuthorizeDetailsFromPlugin(v.Auths), + Vote: vdp, + } + } + return pv +} + +func convertVoteStatusFromPlugin(s ticketvote.VoteStatusT) pi.VoteStatusT { + switch s { + case ticketvote.VoteStatusInvalid: + return pi.VoteStatusInvalid + case ticketvote.VoteStatusUnauthorized: + return pi.VoteStatusUnauthorized + case ticketvote.VoteStatusAuthorized: + return pi.VoteStatusAuthorized + case ticketvote.VoteStatusStarted: + return pi.VoteStatusStarted + case ticketvote.VoteStatusFinished: + return pi.VoteStatusFinished + default: + return pi.VoteStatusInvalid + } +} + +func convertVoteSummaryFromPlugin(s ticketvote.Summary) pi.VoteSummary { + results := make([]pi.VoteResult, 0, len(s.Results)) + for _, v := range s.Results { + results = append(results, pi.VoteResult{ + ID: v.ID, + Description: v.Description, + VoteBit: v.VoteBit, + Votes: v.Votes, + }) + } + return pi.VoteSummary{ + Type: convertVoteTypeFromPlugin(s.Type), + Status: convertVoteStatusFromPlugin(s.Status), + Duration: s.Duration, + StartBlockHeight: s.StartBlockHeight, + StartBlockHash: s.StartBlockHash, + EndBlockHeight: s.EndBlockHeight, + EligibleTickets: s.EligibleTickets, + QuorumPercentage: s.QuorumPercentage, + Results: results, + Approved: s.Approved, + } +} + +func convertVoteSummariesFromPlugin(ts map[string]ticketvote.Summary) map[string]pi.VoteSummary { + s := make(map[string]pi.VoteSummary, len(ts)) + for k, v := range ts { + s[k] = convertVoteSummaryFromPlugin(v) + } + return s +} + +// linkByPeriodMin returns the minimum amount of time, in seconds, that the +// LinkBy period must be set to. This is determined by adding 1 week onto the +// minimum voting period so that RFP proposal submissions have at least one +// week to be submitted after the proposal vote ends. +func (p *politeiawww) linkByPeriodMin() int64 { + var ( + submissionPeriod int64 = 604800 // One week in seconds + blockTime int64 // In seconds + ) + switch { + case p.cfg.TestNet: + blockTime = int64(testNet3Params.TargetTimePerBlock.Seconds()) + case p.cfg.SimNet: + blockTime = int64(simNetParams.TargetTimePerBlock.Seconds()) + default: + blockTime = int64(mainNetParams.TargetTimePerBlock.Seconds()) + } + return (int64(p.cfg.VoteDurationMin) * blockTime) + submissionPeriod +} + +// linkByPeriodMax returns the maximum amount of time, in seconds, that the +// LinkBy period can be set to. 3 months is currently hard coded with no real +// reason for deciding on 3 months besides that it sounds like a sufficient +// amount of time. This can be changed if there is a valid reason to. +func (p *politeiawww) linkByPeriodMax() int64 { + return 7776000 // 3 months in seconds +} + +// proposalRecord returns the proposal record for the provided token and +// version. +func (p *politeiawww) proposalRecord(state pi.PropStateT, token, version string) (*pi.ProposalRecord, error) { + // Get politeiad record + var r *pd.Record + var err error + switch state { + case pi.PropStateUnvetted: + r, err = p.getUnvetted(token, version) + if err != nil { + return nil, err + } + case pi.PropStateVetted: + r, err = p.getVetted(token, version) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unknown state %v", state) + } + + pr, err := convertProposalRecordFromPD(*r) + if err != nil { + return nil, err + } + + // Get proposal plugin data + ps := piplugin.Proposals{ + State: convertPropStateFromPi(state), + Tokens: []string{token}, + } + psr, err := p.piProposals(ps) + if err != nil { + return nil, err + } + d, ok := psr.Proposals[token] + if !ok { + return nil, fmt.Errorf("proposal plugin data not found %v", token) + } + pr.Comments = d.Comments + pr.LinkedFrom = d.LinkedFrom + + // Get user data + u, err := p.db.UserGetByPubKey(pr.PublicKey) + if err != nil { + return nil, err + } + pr.UserID = u.ID.String() + pr.Username = u.Username + + return pr, nil } // proposalRecordLatest returns the latest version of the proposal record for @@ -593,7 +820,7 @@ func (p *politeiawww) proposalRecords(state pi.PropStateT, reqs []pi.ProposalReq State: convertPropStateFromPi(state), Tokens: tokens, } - psr, err := p.proposalPluginData(ps) + psr, err := p.piProposals(ps) if err != nil { return nil, fmt.Errorf("proposalPluginData: %v", err) } @@ -943,7 +1170,7 @@ func (p *politeiawww) processProposalNew(pn pi.ProposalNew, usr user.User) (*pi. if usr.PublicKey() != pn.PublicKey { return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusPublicKeyInvalid, - ErrorContext: []string{"not user's active identity"}, + ErrorContext: []string{"not active identity"}, } } @@ -1068,7 +1295,7 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p if usr.PublicKey() != pe.PublicKey { return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusPublicKeyInvalid, - ErrorContext: []string{"not user's active identity"}, + ErrorContext: []string{"not active identity"}, } } @@ -1207,7 +1434,7 @@ func (p *politeiawww) processProposalSetStatus(pss pi.ProposalSetStatus, usr use if usr.PublicKey() != pss.PublicKey { return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusPublicKeyInvalid, - ErrorContext: []string{"not user's active identity"}, + ErrorContext: []string{"not active identity"}, } } @@ -1281,9 +1508,6 @@ func (p *politeiawww) processProposalSetStatus(pss pi.ProposalSetStatus, usr use }, nil } -// processProposals retrieves and returns the proposal records for each of the -// provided proposal requests. Unvetted proposal files are only returned to -// admins. func (p *politeiawww) processProposals(ps pi.Proposals, isAdmin bool) (*pi.ProposalsReply, error) { log.Tracef("processProposals: %v", ps.Requests) @@ -1311,310 +1535,431 @@ func (p *politeiawww) processProposals(ps pi.Proposals, isAdmin bool) (*pi.Propo }, nil } -// processProposalInventory retrieves the censorship tokens from all records, -// separated by their status. func (p *politeiawww) processProposalInventory(isAdmin bool) (*pi.ProposalInventoryReply, error) { log.Tracef("processProposalInventory") - i, err := p.inventoryByStatus() + ir, err := p.inventoryByStatus() if err != nil { return nil, err } - - reply := convertInventoryReplyFromPD(*i) + reply := pi.ProposalInventoryReply{ + Unvetted: append(ir.Unvetted, ir.IterationUnvetted...), + Public: ir.Vetted, + Censored: ir.Censored, + Abandoned: ir.Archived, + } // Remove unvetted data from non-admin users if !isAdmin { reply.Unvetted = []string{} + reply.Censored = []string{} } return &reply, nil } -func (p *politeiawww) handleProposalNew(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleProposalNew") - - var pn pi.ProposalNew - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&pn); err != nil { - respondWithPiError(w, r, "handleProposalNew: unmarshal", - pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInputInvalid, - }) - return - } +func (p *politeiawww) processCommentNew(cn pi.CommentNew, usr user.User) (*pi.CommentNewReply, error) { + log.Tracef("processCommentNew: %v", usr.Username) - user, err := p.getSessionUser(w, r) - if err != nil { - respondWithPiError(w, r, - "handleProposalNew: getSessionUser: %v", err) - return + // Verify user has paid registration paywall + if !p.userHasPaid(usr) { + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusUserRegistrationNotPaid, + } } - pnr, err := p.processProposalNew(pn, *user) - if err != nil { - respondWithPiError(w, r, - "handleProposalNew: processProposalNew: %v", err) - return + // Verify user signed using active identity + if usr.PublicKey() != cn.PublicKey { + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusPublicKeyInvalid, + ErrorContext: []string{"not active identity"}, + } } - util.RespondWithJSON(w, http.StatusOK, pnr) + // Send plugin command + pcn := piplugin.CommentNew{ + UserID: usr.ID.String(), + State: convertPropStateFromPi(cn.State), + Token: cn.Token, + ParentID: cn.ParentID, + Comment: cn.Comment, + PublicKey: cn.PublicKey, + Signature: cn.Signature, + } + cnr, err := p.piCommentNew(pcn) + if err != nil { + return nil, err + } + + // Emit event + p.eventManager.emit(eventProposalComment, + dataProposalComment{ + state: cn.State, + token: cn.Token, + commentID: cnr.CommentID, + parentID: cn.ParentID, + username: usr.Username, + }) + + return &pi.CommentNewReply{ + CommentID: cnr.CommentID, + Timestamp: cnr.Timestamp, + Receipt: cnr.Receipt, + }, nil } -func (p *politeiawww) handleProposalEdit(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleProposalEdit") +func (p *politeiawww) processCommentVote(cv pi.CommentVote, usr user.User) (*pi.CommentVoteReply, error) { + log.Tracef("processCommentVote: %v %v %v", cv.Token, cv.CommentID, cv.Vote) - var pe pi.ProposalEdit - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&pe); err != nil { - respondWithPiError(w, r, "handleProposalEdit: unmarshal", - pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInputInvalid, - }) - return + // Verify user has paid registration paywall + if !p.userHasPaid(usr) { + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusUserRegistrationNotPaid, + } } - user, err := p.getSessionUser(w, r) - if err != nil { - respondWithPiError(w, r, - "handleProposalEdit: getSessionUser: %v", err) - return + // Verify user signed using active identity + if usr.PublicKey() != cv.PublicKey { + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusPublicKeyInvalid, + ErrorContext: []string{"not active identity"}, + } } - per, err := p.processProposalEdit(pe, *user) + // Send plugin command + pcv := piplugin.CommentVote{ + UserID: usr.ID.String(), + State: convertPropStateFromPi(cv.State), + Token: cv.Token, + CommentID: cv.CommentID, + Vote: convertCommentVoteFromPi(cv.Vote), + PublicKey: cv.PublicKey, + Signature: cv.Signature, + } + cvr, err := p.piCommentVote(pcv) if err != nil { - respondWithPiError(w, r, - "handleProposalEdit: processProposalEdit: %v", err) - return + return nil, err } - util.RespondWithJSON(w, http.StatusOK, per) + return &pi.CommentVoteReply{ + Score: cvr.Score, + Timestamp: cvr.Timestamp, + Receipt: cvr.Receipt, + }, nil } -func (p *politeiawww) handleProposalSetStatus(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleProposalSetStatus") +func (p *politeiawww) processCommentCensor(cc pi.CommentCensor, usr user.User) (*pi.CommentCensorReply, error) { + log.Tracef("processCommentCensor: %v %v", cc.Token, cc.CommentID) - var pss pi.ProposalSetStatus - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&pss); err != nil { - respondWithPiError(w, r, "handleProposalSetStatus: unmarshal", - pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInputInvalid, - }) - return + // Sanity check + if !usr.Admin { + return nil, fmt.Errorf("user not admin") } - user, err := p.getSessionUser(w, r) - if err != nil { - respondWithPiError(w, r, - "handleProposalSetStatus: getSessionUser: %v", err) - return + // Verify user signed with their active identity + if usr.PublicKey() != cc.PublicKey { + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusPublicKeyInvalid, + ErrorContext: []string{"not active identity"}, + } } - pssr, err := p.processProposalSetStatus(pss, *user) + // Send plugin command + pcc := piplugin.CommentCensor{ + State: convertPropStateFromPi(cc.State), + Token: cc.Token, + CommentID: cc.CommentID, + Reason: cc.Reason, + PublicKey: cc.PublicKey, + Signature: cc.Signature, + } + ccr, err := p.piCommentCensor(pcc) if err != nil { - respondWithPiError(w, r, - "handleProposalSetStatus: processProposalSetStatus: %v", err) - return + return nil, err } - util.RespondWithJSON(w, http.StatusOK, pssr) + return &pi.CommentCensorReply{ + Timestamp: ccr.Timestamp, + Receipt: ccr.Receipt, + }, nil } -func (p *politeiawww) handleProposals(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleProposals") +func (p *politeiawww) processComments(c pi.Comments) (*pi.CommentsReply, error) { + log.Tracef("processComments: %v", c.Token) - var ps pi.Proposals - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&ps); err != nil { - respondWithPiError(w, r, "handleProposals: unmarshal", - pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInputInvalid, - }) - return + reply, err := p.commentsAll(comments.GetAll{ + State: convertCommentsStateFromPi(c.State), + Token: c.Token, + }) + if err != nil { + return nil, err } - // Lookup session user. This is a public route so a session may not - // exist. Ignore any session not found errors. - user, err := p.getSessionUser(w, r) - if err != nil && err != errSessionNotFound { - respondWithPiError(w, r, - "handleProposals: getSessionUser: %v", err) - return + // Compile comments. Comments contain user data that needs to be + // pulled from the user database. + cs := make([]pi.Comment, 0, len(reply.Comments)) + for _, cm := range reply.Comments { + // Convert comment + pic := convertCommentFromPlugin(cm) + + // Get comment user data + uuid, err := uuid.Parse(cm.UserID) + if err != nil { + return nil, err + } + u, err := p.db.UserGetById(uuid) + if err != nil { + return nil, err + } + pic.Username = u.Username + + // Add comment + cs = append(cs, pic) } - isAdmin := user != nil && user.Admin - ppi, err := p.processProposals(ps, isAdmin) + return &pi.CommentsReply{ + Comments: cs, + }, nil +} + +func (p *politeiawww) processCommentVotes(cv pi.CommentVotes) (*pi.CommentVotesReply, error) { + log.Tracef("processCommentVotes: %v %v", cv.Token, cv.UserID) + + // Send plugin command + v := comments.Votes{ + State: convertCommentsStateFromPi(cv.State), + Token: cv.Token, + UserID: cv.UserID, + } + cvr, err := p.commentVotes(v) if err != nil { - respondWithPiError(w, r, - "handleProposals: processProposals: %v", err) - return + return nil, err } - util.RespondWithJSON(w, http.StatusOK, ppi) + votes := make([]pi.CommentVoteDetails, 0, len(cvr.Votes)) + for _, v := range cvr.Votes { + votes = append(votes, convertCommentVoteDetailsFromPlugin(v)) + } + + return &pi.CommentVotesReply{ + Votes: votes, + }, nil } -func (p *politeiawww) handleProposalInventory(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleProposalInventory") +func (p *politeiawww) processVoteAuthorize(va pi.VoteAuthorize, usr user.User) (*pi.VoteAuthorizeReply, error) { + log.Tracef("processVoteAuthorize: %v", va.Token) - // Lookup session user. This is a public route so a session may not - // exist. Ignore any session not found errors. - user, err := p.getSessionUser(w, r) - if err != nil && err != errSessionNotFound { - respondWithPiError(w, r, - "handleProposalInventory: getSessionUser: %v", err) - return + // Verify user signed with their active identity + if usr.PublicKey() != va.PublicKey { + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusPublicKeyInvalid, + ErrorContext: []string{"not active identity"}, + } } - isAdmin := user != nil && user.Admin - ppi, err := p.processProposalInventory(isAdmin) + // Send plugin command + ar, err := p.voteAuthorize(convertVoteAuthorizeFromPi(va)) if err != nil { - respondWithPiError(w, r, - "handleProposalInventory: processProposalInventory: %v", err) - return + return nil, err } - util.RespondWithJSON(w, http.StatusOK, ppi) + return &pi.VoteAuthorizeReply{ + Timestamp: ar.Timestamp, + Receipt: ar.Receipt, + }, nil } -func (p *politeiawww) processCommentNew(cn pi.CommentNew, usr user.User) (*pi.CommentNewReply, error) { - log.Tracef("processCommentNew: %v", usr.Username) +func (p *politeiawww) processVoteStart(vs pi.VoteStart, usr user.User) (*pi.VoteStartReply, error) { + log.Tracef("processVoteStart: %v", vs.Params.Token) - // Verify user has paid registration paywall - if !p.userHasPaid(usr) { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusUserRegistrationNotPaid, - } + // Sanity check + if !usr.Admin { + return nil, fmt.Errorf("not an admin") } - // Verify user signed using active identity - if usr.PublicKey() != cn.PublicKey { + // Verify admin signed with their active identity + if usr.PublicKey() != vs.PublicKey { return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusPublicKeyInvalid, - ErrorContext: []string{"not user's active identity"}, + ErrorContext: []string{"not active identity"}, } } - // Call the pi plugin to add new comment - reply, err := p.commentNewPi(piplugin.CommentNew{ - UserID: usr.ID.String(), - Token: cn.Token, - ParentID: cn.ParentID, - Comment: cn.Comment, - PublicKey: cn.PublicKey, - Signature: cn.Signature, - State: convertPropStateFromPi(cn.State), - }) + // Call the ticketvote plugin to start vote + reply, err := p.voteStart(convertVoteStartFromPi(vs)) if err != nil { return nil, err } - // Emit event notification for a proposal comment - p.eventManager.emit(eventProposalComment, - dataProposalComment{ - state: cn.State, - token: cn.Token, - username: usr.Username, - parentID: cn.ParentID, - commentID: reply.CommentID, - }) + return &pi.VoteStartReply{ + StartBlockHeight: reply.StartBlockHeight, + StartBlockHash: reply.StartBlockHash, + EndBlockHeight: reply.EndBlockHeight, + EligibleTickets: reply.EligibleTickets, + }, nil +} - return &pi.CommentNewReply{ - CommentID: reply.CommentID, - Timestamp: reply.Timestamp, - Receipt: reply.Receipt, +func (p *politeiawww) processVoteStartRunoff(vsr pi.VoteStartRunoff, usr user.User) (*pi.VoteStartRunoffReply, error) { + log.Tracef("processVoteStartRunoff: %v", vsr.Token) + + // Sanity check + if !usr.Admin { + return nil, fmt.Errorf("not an admin") + } + + // Verify admin signed all authorizations and starts using their + // active identity. + for _, v := range vsr.Auths { + if usr.PublicKey() != v.PublicKey { + e := fmt.Sprintf("authorize %v public key is not the active identity", + v.Token) + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusPublicKeyInvalid, + ErrorContext: []string{e}, + } + } + } + for _, v := range vsr.Starts { + if usr.PublicKey() != v.PublicKey { + e := fmt.Sprintf("start %v public key is not the active identity", + v.Params.Token) + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusPublicKeyInvalid, + ErrorContext: []string{e}, + } + } + } + + // Send plugin command + tsr := ticketvote.StartRunoff{ + Token: vsr.Token, + Auths: convertVoteAuthsFromPi(vsr.Auths), + Starts: convertVoteStartsFromPi(vsr.Starts), + } + srr, err := p.voteStartRunoff(tsr) + if err != nil { + return nil, err + } + + return &pi.VoteStartRunoffReply{ + StartBlockHeight: srr.StartBlockHeight, + StartBlockHash: srr.StartBlockHash, + EndBlockHeight: srr.EndBlockHeight, + EligibleTickets: srr.EligibleTickets, }, nil } -func (p *politeiawww) handleCommentNew(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleCommentNew") +func (p *politeiawww) processVoteBallot(vb pi.VoteBallot) (*pi.VoteBallotReply, error) { + log.Tracef("processVoteBallot") - var cn pi.CommentNew - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&cn); err != nil { - respondWithPiError(w, r, "handleCommentNew: unmarshal", - pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInputInvalid, - }) - return + b := ticketvote.Ballot{ + Votes: convertCastVotesFromPi(vb.Votes), + } + reply, err := p.voteBallot(b) + if err != nil { + return nil, err } - user, err := p.getSessionUser(w, r) + return &pi.VoteBallotReply{ + Receipts: convertCastVoteRepliesFromPlugin(reply.Receipts), + }, nil +} + +func (p *politeiawww) processVotes(v pi.Votes) (*pi.VotesReply, error) { + log.Tracef("processVotes: %v", v.Tokens) + + vd, err := p.voteDetails(v.Tokens) if err != nil { - respondWithPiError(w, r, - "handleCommentNew: getSessionUser: %v", err) - return + return nil, err } - cnr, err := p.processCommentNew(cn, *user) + return &pi.VotesReply{ + Votes: convertProposalVotesFromPlugin(vd.Votes), + }, nil +} + +func (p *politeiawww) processVoteResults(vr pi.VoteResults) (*pi.VoteResultsReply, error) { + log.Tracef("processVoteResults: %v", vr.Token) + + cvr, err := p.castVotes(vr.Token) if err != nil { - respondWithPiError(w, r, - "handleCommentNew: processCommentNew: %v", err) - return + return nil, err } - util.RespondWithJSON(w, http.StatusOK, cnr) + return &pi.VoteResultsReply{ + Votes: convertCastVoteDetailsFromPlugin(cvr.Votes), + }, nil } -func convertVoteFromPi(v pi.CommentVoteT) piplugin.VoteT { - switch v { - case pi.CommentVoteInvalid: - return piplugin.VoteInvalid - case pi.CommentVoteDownvote: - return piplugin.VoteDownvote - case pi.CommentVoteUpvote: - return piplugin.VoteUpvote - default: - return piplugin.VoteInvalid +func (p *politeiawww) processVoteSummaries(vs pi.VoteSummaries) (*pi.VoteSummariesReply, error) { + log.Tracef("processVoteSummaries: %v", vs.Tokens) + + r, err := p.voteSummaries(vs.Tokens) + if err != nil { + return nil, err } + + return &pi.VoteSummariesReply{ + Summaries: convertVoteSummariesFromPlugin(r.Summaries), + BestBlock: r.BestBlock, + }, nil } -func (p *politeiawww) processCommentVote(cv pi.CommentVote, usr user.User) (*pi.CommentVoteReply, error) { - log.Tracef("processCommentVote: %v %v %v", cv.Token, cv.CommentID, cv.Vote) +func (p *politeiawww) processVoteInventory() (*pi.VoteInventoryReply, error) { + log.Tracef("processVoteInventory") - // Verify user has paid registration paywall - if !p.userHasPaid(usr) { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusUserRegistrationNotPaid, - } + r, err := p.piVoteInventory() + if err != nil { + return nil, err } - // Verify user signed using active identity - if usr.PublicKey() != cv.PublicKey { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPublicKeyInvalid, - ErrorContext: []string{"not user's active identity"}, - } + return &pi.VoteInventoryReply{ + Unauthorized: r.Unauthorized, + Authorized: r.Authorized, + Started: r.Started, + Approved: r.Approved, + Rejected: r.Rejected, + BestBlock: r.BestBlock, + }, nil +} + +func (p *politeiawww) handleProposalNew(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleProposalNew") + + var pn pi.ProposalNew + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&pn); err != nil { + respondWithPiError(w, r, "handleProposalNew: unmarshal", + pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusInputInvalid, + }) + return + } + + user, err := p.getSessionUser(w, r) + if err != nil { + respondWithPiError(w, r, + "handleProposalNew: getSessionUser: %v", err) + return } - // Call the pi plugin to vote on a comment - reply, err := p.commentVotePi(piplugin.CommentVote{ - UserID: usr.ID.String(), - Token: cv.Token, - CommentID: cv.CommentID, - Vote: convertVoteFromPi(cv.Vote), - PublicKey: cv.PublicKey, - Signature: cv.Signature, - State: convertPropStateFromPi(cv.State), - }) + pnr, err := p.processProposalNew(pn, *user) if err != nil { - return nil, err + respondWithPiError(w, r, + "handleProposalNew: processProposalNew: %v", err) + return } - return &pi.CommentVoteReply{ - Score: reply.Score, - Timestamp: reply.Timestamp, - Receipt: reply.Receipt, - }, nil + util.RespondWithJSON(w, http.StatusOK, pnr) } -func (p *politeiawww) handleCommentVote(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleCommentVote") +func (p *politeiawww) handleProposalEdit(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleProposalEdit") - var cv pi.CommentVote + var pe pi.ProposalEdit decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&cv); err != nil { - respondWithPiError(w, r, "handleCommentVote: unmarshal", + if err := decoder.Decode(&pe); err != nil { + respondWithPiError(w, r, "handleProposalEdit: unmarshal", pi.UserErrorReply{ ErrorCode: pi.ErrorStatusInputInvalid, }) @@ -1624,156 +1969,162 @@ func (p *politeiawww) handleCommentVote(w http.ResponseWriter, r *http.Request) user, err := p.getSessionUser(w, r) if err != nil { respondWithPiError(w, r, - "handleCommentVote: getSessionUser: %v", err) + "handleProposalEdit: getSessionUser: %v", err) + return } - vcr, err := p.processCommentVote(cv, *user) + per, err := p.processProposalEdit(pe, *user) if err != nil { respondWithPiError(w, r, - "handleCommentVote: processCommentVote: %v", err) + "handleProposalEdit: processProposalEdit: %v", err) + return } - util.RespondWithJSON(w, http.StatusOK, vcr) + util.RespondWithJSON(w, http.StatusOK, per) } -func (p *politeiawww) processComments(c pi.Comments) (*pi.CommentsReply, error) { - log.Tracef("processComments: %v", c.Token) +func (p *politeiawww) handleProposalSetStatus(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleProposalSetStatus") - // Call the comments plugin to get comments - reply, err := p.comments(comments.GetAll{ - Token: c.Token, - State: convertCommentsPluginPropStateFromPi(c.State), - }) + var pss pi.ProposalSetStatus + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&pss); err != nil { + respondWithPiError(w, r, "handleProposalSetStatus: unmarshal", + pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusInputInvalid, + }) + return + } + + usr, err := p.getSessionUser(w, r) if err != nil { - return nil, err + respondWithPiError(w, r, + "handleProposalSetStatus: getSessionUser: %v", err) + return } - var cr pi.CommentsReply - // Transalte comments - cs := make([]pi.Comment, 0, len(reply.Comments)) - for _, cm := range reply.Comments { - // Convert comment to pi - pic := convertCommentFromPlugin(cm) - // Get comment's author username - // Parse string uuid - uuid, err := uuid.Parse(cm.UserID) - if err != nil { - return nil, err - } - // Get user - u, err := p.db.UserGetById(uuid) - if err != nil { - return nil, err - } - pic.Username = u.Username - cs = append(cs, pic) + pssr, err := p.processProposalSetStatus(pss, *usr) + if err != nil { + respondWithPiError(w, r, + "handleProposalSetStatus: processProposalSetStatus: %v", err) + return } - cr.Comments = cs - return &cr, nil + util.RespondWithJSON(w, http.StatusOK, pssr) } -func (p *politeiawww) handleComments(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleComments") +func (p *politeiawww) handleProposals(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleProposals") - var c pi.Comments + var ps pi.Proposals decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&c); err != nil { - respondWithPiError(w, r, "handleComments: unmarshal", + if err := decoder.Decode(&ps); err != nil { + respondWithPiError(w, r, "handleProposals: unmarshal", pi.UserErrorReply{ ErrorCode: pi.ErrorStatusInputInvalid, }) return } - cr, err := p.processComments(c) + // Lookup session user. This is a public route so a session may not + // exist. Ignore any session not found errors. + usr, err := p.getSessionUser(w, r) + if err != nil && err != errSessionNotFound { + respondWithPiError(w, r, + "handleProposals: getSessionUser: %v", err) + return + } + + isAdmin := usr != nil && usr.Admin + ppi, err := p.processProposals(ps, isAdmin) if err != nil { respondWithPiError(w, r, - "handleCommentVote: processComments: %v", err) + "handleProposals: processProposals: %v", err) + return } - util.RespondWithJSON(w, http.StatusOK, cr) + util.RespondWithJSON(w, http.StatusOK, ppi) } -func (p *politeiawww) processCommentVotes(cvs pi.CommentVotes) (*pi.CommentVotesReply, error) { - log.Tracef("processCommentVotes: %v %v", cvs.Token, cvs.UserID) +func (p *politeiawww) handleProposalInventory(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleProposalInventory") - // Call the comments plugin to get filtered record's comment votes - reply, err := p.commentVotes(comments.Votes{ - Token: cvs.Token, - State: convertCommentsPluginPropStateFromPi(cvs.State), - UserID: cvs.UserID, - }) - if err != nil { - return nil, err + // Lookup session user. This is a public route so a session may not + // exist. Ignore any session not found errors. + usr, err := p.getSessionUser(w, r) + if err != nil && err != errSessionNotFound { + respondWithPiError(w, r, + "handleProposalInventory: getSessionUser: %v", err) + return } - // Translate to pi - var cvsr pi.CommentVotesReply - ucvs := make([]pi.CommentVoteDetails, 0, len(reply.Votes)) - for _, cv := range reply.Votes { - ucvs = append(ucvs, convertCommentVoteFromPlugin(cv)) + isAdmin := usr != nil && usr.Admin + ppi, err := p.processProposalInventory(isAdmin) + if err != nil { + respondWithPiError(w, r, + "handleProposalInventory: processProposalInventory: %v", err) + return } - cvsr.Votes = ucvs - return &cvsr, nil + util.RespondWithJSON(w, http.StatusOK, ppi) } -func (p *politeiawww) handleCommentVotes(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleCommentVotes") +func (p *politeiawww) handleCommentNew(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleCommentNew") - var cvs pi.CommentVotes + var cn pi.CommentNew decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&cvs); err != nil { - respondWithPiError(w, r, "handleCommentVotes: unmarshal", + if err := decoder.Decode(&cn); err != nil { + respondWithPiError(w, r, "handleCommentNew: unmarshal", pi.UserErrorReply{ ErrorCode: pi.ErrorStatusInputInvalid, }) return } - cvsr, err := p.processCommentVotes(cvs) + usr, err := p.getSessionUser(w, r) if err != nil { respondWithPiError(w, r, - "handleCommentVotes: processCommentVotes: %v", err) + "handleCommentNew: getSessionUser: %v", err) + return } - util.RespondWithJSON(w, http.StatusOK, cvsr) + cnr, err := p.processCommentNew(cn, *usr) + if err != nil { + respondWithPiError(w, r, + "handleCommentNew: processCommentNew: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, cnr) } -func (p *politeiawww) processCommentCensor(cc pi.CommentCensor, usr user.User) (*pi.CommentCensorReply, error) { - log.Tracef("processCommentCensor: %v %v", cc.Token, cc.CommentID) +func (p *politeiawww) handleCommentVote(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleCommentVote") - // Sanity check - if !usr.Admin { - return nil, fmt.Errorf("user not admin") + var cv pi.CommentVote + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&cv); err != nil { + respondWithPiError(w, r, "handleCommentVote: unmarshal", + pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusInputInvalid, + }) + return } - // Verify user signed with their active identity - if usr.PublicKey() != cc.PublicKey { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPublicKeyInvalid, - ErrorContext: []string{"not user's active identity"}, - } + usr, err := p.getSessionUser(w, r) + if err != nil { + respondWithPiError(w, r, + "handleCommentVote: getSessionUser: %v", err) } - // Call the pi plugin to censor comment - reply, err := p.commentCensorPi(piplugin.CommentCensor{ - State: convertPropStateFromPi(cc.State), - Token: cc.Token, - CommentID: cc.CommentID, - Reason: cc.Reason, - PublicKey: cc.PublicKey, - Signature: cc.Signature, - }) + vcr, err := p.processCommentVote(cv, *usr) if err != nil { - return nil, err + respondWithPiError(w, r, + "handleCommentVote: processCommentVote: %v", err) } - return &pi.CommentCensorReply{ - Timestamp: reply.Timestamp, - Receipt: reply.Receipt, - }, nil + util.RespondWithJSON(w, http.StatusOK, vcr) } func (p *politeiawww) handleCommentCensor(w http.ResponseWriter, r *http.Request) { @@ -1789,14 +2140,14 @@ func (p *politeiawww) handleCommentCensor(w http.ResponseWriter, r *http.Request return } - user, err := p.getSessionUser(w, r) + usr, err := p.getSessionUser(w, r) if err != nil { respondWithPiError(w, r, "handleCommentCensor: getSessionUser: %v", err) return } - ccr, err := p.processCommentCensor(cc, *user) + ccr, err := p.processCommentCensor(cc, *usr) if err != nil { respondWithPiError(w, r, "handleCommentCensor: processCommentCensor: %v", err) @@ -1805,121 +2156,76 @@ func (p *politeiawww) handleCommentCensor(w http.ResponseWriter, r *http.Request util.RespondWithJSON(w, http.StatusOK, ccr) } -func convertVoteAuthActionFromPi(a pi.VoteAuthActionT) ticketvote.AuthActionT { - switch a { - case pi.VoteAuthActionAuthorize: - return ticketvote.ActionAuthorize - case pi.VoteAuthActionRevoke: - return ticketvote.ActionRevoke - default: - return ticketvote.ActionAuthorize - } -} - -func convertVoteAuthorizeFromPi(va pi.VoteAuthorize) ticketvote.Authorize { - return ticketvote.Authorize{ - Token: va.Token, - Version: va.Version, - Action: convertVoteAuthActionFromPi(va.Action), - PublicKey: va.PublicKey, - Signature: va.Signature, - } -} - -func (p *politeiawww) processVoteAuthorize(va pi.VoteAuthorize) (*pi.VoteAuthorizeReply, error) { - log.Tracef("processVoteAuthorize: %v", va.Token) - - // Call the ticketvote plugin to authorize vote - reply, err := p.voteAuthorize(convertVoteAuthorizeFromPi(va)) - if err != nil { - return nil, err - } - - return &pi.VoteAuthorizeReply{ - Timestamp: reply.Timestamp, - Receipt: reply.Receipt, - }, nil -} - -func (p *politeiawww) handleVoteAuthorize(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleVoteAuthorize") +func (p *politeiawww) handleComments(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleComments") - var va pi.VoteAuthorize + var c pi.Comments decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&va); err != nil { - respondWithPiError(w, r, "handleVoteAuthorize: unmarshal", + if err := decoder.Decode(&c); err != nil { + respondWithPiError(w, r, "handleComments: unmarshal", pi.UserErrorReply{ ErrorCode: pi.ErrorStatusInputInvalid, }) return } - vr, err := p.processVoteAuthorize(va) + cr, err := p.processComments(c) if err != nil { respondWithPiError(w, r, - "handleVoteAuthorize: processVoteAuthorize: %v", err) + "handleCommentVote: processComments: %v", err) } - util.RespondWithJSON(w, http.StatusOK, vr) + util.RespondWithJSON(w, http.StatusOK, cr) } -func convertVoteTypeFromPi(t pi.VoteT) ticketvote.VoteT { - switch t { - case pi.VoteTypeStandard: - return ticketvote.VoteTypeStandard - case pi.VoteTypeRunoff: - return ticketvote.VoteTypeRunoff - } - return ticketvote.VoteTypeInvalid -} +func (p *politeiawww) handleCommentVotes(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleCommentVotes") -func convertVoteParamsFromPi(v pi.VoteParams) ticketvote.VoteParams { - tv := ticketvote.VoteParams{ - Token: v.Token, - Version: v.Version, - Type: convertVoteTypeFromPi(v.Type), - Mask: v.Mask, - Duration: v.Duration, - QuorumPercentage: v.QuorumPercentage, - PassPercentage: v.PassPercentage, - } - // Convert vote options - vo := make([]ticketvote.VoteOption, 0, len(v.Options)) - for _, vi := range v.Options { - vo = append(vo, ticketvote.VoteOption{ - ID: vi.ID, - Description: vi.Description, - Bit: vi.Bit, - }) + var cvs pi.CommentVotes + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&cvs); err != nil { + respondWithPiError(w, r, "handleCommentVotes: unmarshal", + pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusInputInvalid, + }) + return } - tv.Options = vo - return tv + cvsr, err := p.processCommentVotes(cvs) + if err != nil { + respondWithPiError(w, r, + "handleCommentVotes: processCommentVotes: %v", err) + } + + util.RespondWithJSON(w, http.StatusOK, cvsr) } -func convertVoteStartFromPi(vs pi.VoteStart) ticketvote.Start { - return ticketvote.Start{ - Params: convertVoteParamsFromPi(vs.Params), - PublicKey: vs.PublicKey, - Signature: vs.Signature, +func (p *politeiawww) handleVoteAuthorize(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleVoteAuthorize") + + var va pi.VoteAuthorize + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&va); err != nil { + respondWithPiError(w, r, "handleVoteAuthorize: unmarshal", + pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusInputInvalid, + }) + return } -} -func (p *politeiawww) processVoteStart(vs pi.VoteStart) (*pi.VoteStartReply, error) { - log.Tracef("processVoteStart: %v", vs.Params.Token) + usr, err := p.getSessionUser(w, r) + if err != nil { + respondWithPiError(w, r, + "handleVoteAuthorize: getSessionUser: %v", err) + } - // Call the ticketvote plugin to start vote - reply, err := p.voteStart(convertVoteStartFromPi(vs)) + vr, err := p.processVoteAuthorize(va, *usr) if err != nil { - return nil, err + respondWithPiError(w, r, + "handleVoteAuthorize: processVoteAuthorize: %v", err) } - return &pi.VoteStartReply{ - StartBlockHeight: reply.StartBlockHeight, - StartBlockHash: reply.StartBlockHash, - EndBlockHeight: reply.EndBlockHeight, - EligibleTickets: reply.EligibleTickets, - }, nil + util.RespondWithJSON(w, http.StatusOK, vr) } func (p *politeiawww) handleVoteStart(w http.ResponseWriter, r *http.Request) { @@ -1935,47 +2241,19 @@ func (p *politeiawww) handleVoteStart(w http.ResponseWriter, r *http.Request) { return } - vsr, err := p.processVoteStart(vs) + usr, err := p.getSessionUser(w, r) if err != nil { respondWithPiError(w, r, - "handleVoteStart: processVoteStart: %v", err) - } - - util.RespondWithJSON(w, http.StatusOK, vsr) -} - -func (p *politeiawww) processVoteStartRunoff(cvr pi.VoteStartRunoff) (*pi.VoteStartRunoffReply, error) { - log.Tracef("processVoteStartRunoff: %v", cvr.Token) - - // Call the ticketvote plugin to start runoff vote - // Transalte payload - pcvr := ticketvote.StartRunoff{ - Token: cvr.Token, - } - // Transalte submissions' vote authorizations structs - auths := make([]ticketvote.Authorize, 0, len(cvr.Authorizations)) - for _, auth := range cvr.Authorizations { - auths = append(auths, convertVoteAuthorizeFromPi(auth)) + "handleVoteStart: getSessionUser: %v", err) } - pcvr.Auths = auths - // Transate submissions' vote start structs - starts := make([]ticketvote.Start, 0, len(cvr.Starts)) - for _, s := range cvr.Starts { - starts = append(starts, convertVoteStartFromPi(s)) - } - pcvr.Starts = starts - reply, err := p.voteStartRunoff(pcvr) + vsr, err := p.processVoteStart(vs, *usr) if err != nil { - return nil, err + respondWithPiError(w, r, + "handleVoteStart: processVoteStart: %v", err) } - return &pi.VoteStartRunoffReply{ - StartBlockHeight: reply.StartBlockHeight, - StartBlockHash: reply.StartBlockHash, - EndBlockHeight: reply.EndBlockHeight, - EligibleTickets: reply.EligibleTickets, - }, nil + util.RespondWithJSON(w, http.StatusOK, vsr) } func (p *politeiawww) handleVoteStartRunoff(w http.ResponseWriter, r *http.Request) { @@ -1991,72 +2269,19 @@ func (p *politeiawww) handleVoteStartRunoff(w http.ResponseWriter, r *http.Reque return } - vsrr, err := p.processVoteStartRunoff(vsr) + usr, err := p.getSessionUser(w, r) if err != nil { respondWithPiError(w, r, - "handleVoteStartRunoff: processVoteStartRunoff: %v", err) + "handleVoteStartRunoff: getSessionUser: %v", err) } - util.RespondWithJSON(w, http.StatusOK, vsrr) -} - -func convertPiVoteErrorFromTicketVote(e ticketvote.VoteErrorT) pi.VoteErrorT { - switch e { - case ticketvote.VoteErrorInvalid: - return pi.VoteErrorInvalid - case ticketvote.VoteErrorInternalError: - return pi.VoteErrorInternalError - case ticketvote.VoteErrorRecordNotFound: - return pi.VoteErrorRecordNotFound - case ticketvote.VoteErrorVoteBitInvalid: - return pi.VoteErrorVoteBitInvalid - case ticketvote.VoteErrorVoteStatusInvalid: - return pi.VoteErrorVoteStatusInvalid - case ticketvote.VoteErrorTicketAlreadyVoted: - return pi.VoteErrorTicketAlreadyVoted - case ticketvote.VoteErrorTicketNotEligible: - return pi.VoteErrorTicketNotEligible - default: - return pi.VoteErrorInternalError - } -} - -func (p *politeiawww) processVoteBallot(vb pi.VoteBallot) (*pi.VoteBallotReply, error) { - log.Tracef("processVoteBallot") - - // Call the ticketvote to cast ballot of votes - // Transalte payload - var vbp ticketvote.Ballot - votes := make([]ticketvote.CastVote, 0, len(vb.Votes)) - for _, v := range vb.Votes { - votes = append(votes, ticketvote.CastVote{ - Token: v.Token, - Ticket: v.Ticket, - VoteBit: v.VoteBit, - Signature: v.Signature, - }) - } - vbp.Votes = votes - - reply, err := p.voteBallot(vbp) + vsrr, err := p.processVoteStartRunoff(vsr, *usr) if err != nil { - return nil, err - } - - // Translate reply - var vbr pi.VoteBallotReply - vrs := make([]pi.CastVoteReply, 0, len(reply.Receipts)) - for _, vr := range reply.Receipts { - vrs = append(vrs, pi.CastVoteReply{ - Ticket: vr.Ticket, - Receipt: vr.Receipt, - ErrorCode: convertPiVoteErrorFromTicketVote(vr.ErrorCode), - ErrorContext: vr.ErrorContext, - }) + respondWithPiError(w, r, + "handleVoteStartRunoff: processVoteStartRunoff: %v", err) } - vbr.Receipts = vrs - return &vbr, nil + util.RespondWithJSON(w, http.StatusOK, vsrr) } func (p *politeiawww) handleVoteBallot(w http.ResponseWriter, r *http.Request) { @@ -2081,103 +2306,6 @@ func (p *politeiawww) handleVoteBallot(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, vbr) } -func convertVoteTypeFromPlugin(t ticketvote.VoteT) pi.VoteT { - switch t { - case ticketvote.VoteTypeStandard: - return pi.VoteTypeStandard - case ticketvote.VoteTypeRunoff: - return pi.VoteTypeRunoff - } - return pi.VoteTypeInvalid - -} - -func convertVoteParamsFromPlugin(v ticketvote.VoteParams) pi.VoteParams { - vp := pi.VoteParams{ - Token: v.Token, - Version: v.Version, - Type: convertVoteTypeFromPlugin(v.Type), - Mask: v.Mask, - Duration: v.Duration, - QuorumPercentage: v.QuorumPercentage, - PassPercentage: v.PassPercentage, - } - vo := make([]pi.VoteOption, 0, len(v.Options)) - for _, o := range v.Options { - vo = append(vo, pi.VoteOption{ - ID: o.ID, - Description: o.Description, - Bit: o.Bit, - }) - } - vp.Options = vo - - return vp -} - -func convertVoteDetailsFromPlugin(vd ticketvote.VoteDetails) *pi.VoteDetails { - return &pi.VoteDetails{ - Params: convertVoteParamsFromPlugin(vd.Params), - PublicKey: vd.PublicKey, - Signature: vd.Signature, - StartBlockHeight: vd.StartBlockHeight, - StartBlockHash: vd.StartBlockHash, - EndBlockHeight: vd.EndBlockHeight, - EligibleTickets: vd.EligibleTickets, - } -} - -func convertAuthorizeDetailsFromPlugin(ad ticketvote.AuthorizeDetails) pi.AuthorizeDetails { - return pi.AuthorizeDetails{ - Token: ad.Token, - Version: ad.Version, - Action: ad.Action, - PublicKey: ad.PublicKey, - Signature: ad.Signature, - Timestamp: ad.Timestamp, - Receipt: ad.Receipt, - } -} - -func (p *politeiawww) processVotes(v pi.Votes) (*pi.VotesReply, error) { - log.Tracef("processVotes: %v", v.Tokens) - - // Call the ticketvote plugin to get vote details - vd, err := p.voteDetails(v.Tokens) - if err != nil { - return nil, err - } - - // Convert reply to pi - var ( - vote ticketvote.RecordVote - ok bool - vr pi.VotesReply - ) - votes := make(map[string]pi.ProposalVote) - for _, token := range v.Tokens { - if vote, ok = vd.Votes[token]; !ok { - // No related vote details, skip token - continue - } - pv := pi.ProposalVote{ - Vote: convertVoteDetailsFromPlugin(*(vote.Vote)), - } - - // Transalte vote auth - auths := make([]pi.AuthorizeDetails, 0, len(vote.Auths)) - for _, a := range vote.Auths { - auths = append(auths, convertAuthorizeDetailsFromPlugin(a)) - } - pv.Auths = auths - - votes[token] = pv - } - vr.Votes = votes - - return &vr, nil -} - func (p *politeiawww) handleVotes(w http.ResponseWriter, r *http.Request) { log.Tracef("handleVotes") @@ -2200,31 +2328,6 @@ func (p *politeiawww) handleVotes(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, vr) } -func (p *politeiawww) processVoteResults(vr pi.VoteResults) (*pi.VoteResultsReply, error) { - log.Tracef("processVoteResults: %v", vr.Token) - - // Get cast votes information - cv, err := p.castVotes(vr.Token) - if err != nil { - return nil, err - } - - // Transalte to pi - var vrr pi.VoteResultsReply - votes := make([]pi.CastVoteDetails, 0, len(cv.Votes)) - for _, v := range cv.Votes { - votes = append(votes, pi.CastVoteDetails{ - Token: v.Token, - Ticket: v.Ticket, - VoteBit: v.VoteBit, - Signature: v.Signature, - }) - } - vrr.Votes = votes - - return &vrr, nil -} - func (p *politeiawww) handleVoteResults(w http.ResponseWriter, r *http.Request) { log.Tracef("handleVoteResults") @@ -2247,73 +2350,6 @@ func (p *politeiawww) handleVoteResults(w http.ResponseWriter, r *http.Request) util.RespondWithJSON(w, http.StatusOK, vrr) } -func convertVoteStatusFromPlugin(s ticketvote.VoteStatusT) pi.VoteStatusT { - switch s { - case ticketvote.VoteStatusInvalid: - return pi.VoteStatusInvalid - case ticketvote.VoteStatusUnauthorized: - return pi.VoteStatusUnauthorized - case ticketvote.VoteStatusAuthorized: - return pi.VoteStatusAuthorized - case ticketvote.VoteStatusStarted: - return pi.VoteStatusStarted - case ticketvote.VoteStatusFinished: - return pi.VoteStatusFinished - default: - return pi.VoteStatusInvalid - } -} - -func convertVoteSummaryFromPlugin(s ticketvote.Summary) pi.VoteSummary { - vs := pi.VoteSummary{ - Type: convertVoteTypeFromPlugin(s.Type), - Status: convertVoteStatusFromPlugin(s.Status), - Duration: s.Duration, - StartBlockHeight: s.StartBlockHeight, - StartBlockHash: s.StartBlockHash, - EndBlockHeight: s.EndBlockHeight, - EligibleTickets: s.EligibleTickets, - QuorumPercentage: s.QuorumPercentage, - Approved: s.Approved, - } - // Transalte results - rs := make([]pi.VoteResult, 0, len(s.Results)) - for _, v := range s.Results { - rs = append(rs, pi.VoteResult{ - ID: v.ID, - Description: v.Description, - VoteBit: v.VoteBit, - Votes: v.Votes, - }) - } - vs.Results = rs - - return vs -} - -func (p *politeiawww) processVoteSummaries(vs pi.VoteSummaries) (*pi.VoteSummariesReply, error) { - log.Tracef("processVoteSummaries: %v", vs.Tokens) - - // Call the ticket vote to get the vote summaries - r, err := p.voteSummaries(vs.Tokens) - if err != nil { - return nil, err - } - - // Translate to pi - vsr := pi.VoteSummariesReply{ - BestBlock: r.BestBlock, - } - // Translate summaries - sms := make(map[string]pi.VoteSummary) - for t, s := range r.Summaries { - sms[t] = convertVoteSummaryFromPlugin(s) - } - vsr.Summaries = sms - - return &vsr, nil -} - func (p *politeiawww) handleVoteSummaries(w http.ResponseWriter, r *http.Request) { log.Tracef("handleVoteSummaries") @@ -2336,25 +2372,6 @@ func (p *politeiawww) handleVoteSummaries(w http.ResponseWriter, r *http.Request util.RespondWithJSON(w, http.StatusOK, vsr) } -func (p *politeiawww) processVoteInventory() (*pi.VoteInventoryReply, error) { - log.Tracef("processVoteInventory") - - // Call the pi plugin vote inventory - r, err := p.voteInventoryPi(piplugin.VoteInventory{}) - if err != nil { - return nil, err - } - - return &pi.VoteInventoryReply{ - Unauthorized: r.Unauthorized, - Authorized: r.Authorized, - Started: r.Started, - Approved: r.Approved, - Rejected: r.Rejected, - BestBlock: r.BestBlock, - }, nil -} - func (p *politeiawww) handleVoteInventory(w http.ResponseWriter, r *http.Request) { log.Tracef("handleVoteInventory") @@ -2377,7 +2394,8 @@ func (p *politeiawww) handleVoteInventory(w http.ResponseWriter, r *http.Request util.RespondWithJSON(w, http.StatusOK, vir) } -func (p *politeiawww) setPiRoutes() { +// setupPiRoutes sets up the pi API routes. +func (p *politeiawww) setupPiRoutes() { // Proposal routes p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteProposalNew, p.handleProposalNew, diff --git a/politeiawww/plugin.go b/politeiawww/plugin.go index c0a55d26e..b94fd3904 100644 --- a/politeiawww/plugin.go +++ b/politeiawww/plugin.go @@ -11,10 +11,11 @@ import ( pd "github.com/decred/politeia/politeiad/api/v1" ) -// pluginSetting is a structure that holds key/value pairs of a plugin setting. +// pluginSetting is a structure that holds the key-value pairs of a plugin +// setting. type pluginSetting struct { - Key string // Name of setting - Value string // Value of setting + Key string + Value string } // plugin describes a plugin and its settings. @@ -43,17 +44,6 @@ func convertPluginFromPD(p pd.Plugin) plugin { } } -// getBestBlock fetches and returns the best block from politeiad using the -// decred plugin bestblock command. -func (p *politeiawww) getBestBlock() (uint64, error) { - bb, err := p.decredBestBlock() - if err != nil { - return 0, err - } - - return uint64(bb.Height), nil -} - // getPluginInventory returns the politeiad plugin inventory. If a politeiad // connection cannot be made, the call will be retried every 5 seconds for up // to 1000 tries. diff --git a/politeiawww/politeiad.go b/politeiawww/politeiad.go index af7676dd9..53236eb54 100644 --- a/politeiawww/politeiad.go +++ b/politeiawww/politeiad.go @@ -470,7 +470,7 @@ func (p *politeiawww) pluginInventory() ([]pd.Plugin, error) { // pluginCommand fires a plugin command on politeiad and returns the reply // payload. -func (p *politeiawww) pluginCommand(pluginID, cmd, cmdID, payload string) (string, error) { +func (p *politeiawww) pluginCommand(pluginID, cmd, payload string) (string, error) { // Setup request challenge, err := util.Random(pd.ChallengeSize) if err != nil { @@ -480,7 +480,7 @@ func (p *politeiawww) pluginCommand(pluginID, cmd, cmdID, payload string) (strin Challenge: hex.EncodeToString(challenge), ID: pluginID, Command: cmd, - CommandID: cmdID, + CommandID: "", Payload: payload, } diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 8edc3ed6e..41d27a9d5 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -6,93 +6,16 @@ package main import ( "encoding/base64" - "fmt" "strconv" - "time" "github.com/decred/politeia/decredplugin" piplugin "github.com/decred/politeia/plugins/pi" ticketvote "github.com/decred/politeia/plugins/ticketvote" pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" - www2 "github.com/decred/politeia/politeiawww/api/www/v2" "github.com/decred/politeia/politeiawww/user" ) -// isRFP returns whether the proposal is a Request For Proposals (RFP). -func isRFP(pr www.ProposalRecord) bool { - return pr.LinkBy != 0 -} - -func isRFPSubmission(pr www.ProposalRecord) bool { - // Right now the only proposals that we allow linking to - // are RFPs so if the linkto is set than this is an RFP - // submission. This may change in the future, at which - // point we'll actually have to check the linkto proposal - // to see if its an RFP. - return pr.LinkTo != "" -} - -// validateVoteBit ensures that bit is a valid vote bit. -func validateVoteBit(vote www2.Vote, bit uint64) error { - if len(vote.Options) == 0 { - return fmt.Errorf("vote corrupt") - } - if bit == 0 { - return fmt.Errorf("invalid bit 0x%x", bit) - } - if vote.Mask&bit != bit { - return fmt.Errorf("invalid mask 0x%x bit 0x%x", - vote.Mask, bit) - } - - for _, v := range vote.Options { - if v.Bits == bit { - return nil - } - } - - return fmt.Errorf("bit not found 0x%x", bit) -} - -func (p *politeiawww) linkByValidate(linkBy int64) error { - min := time.Now().Unix() + p.linkByPeriodMin() - max := time.Now().Unix() + p.linkByPeriodMax() - switch { - case linkBy < min: - e := fmt.Sprintf("linkby %v is less than min required of %v", - linkBy, min) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkBy, - ErrorContext: []string{e}, - } - case linkBy > max: - e := fmt.Sprintf("linkby %v is more than max allowed of %v", - linkBy, max) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkBy, - ErrorContext: []string{e}, - } - } - return nil -} - -func (p *politeiawww) getProp(token string) (*www.ProposalRecord, error) { - return nil, nil -} - -func (p *politeiawww) getProps(tokens []string) (map[string]www.ProposalRecord, error) { - return nil, nil -} - -func (p *politeiawww) getPropVersion(token, version string) (*www.ProposalRecord, error) { - return nil, nil -} - -func (p *politeiawww) getAllProps() ([]www.ProposalRecord, error) { - return nil, nil -} - func convertStateToWWW(state pi.PropStateT) www.PropStateT { switch state { case pi.PropStateInvalid: @@ -136,28 +59,8 @@ func convertProposalToWWW(pr *pi.ProposalRecord) (*www.ProposalRecord, error) { } } } - // Fill info - pw := www.ProposalRecord{ - Name: pm.Name, - LinkBy: pm.LinkBy, - LinkTo: pm.LinkTo, - Status: convertStatusToWWW(pr.Status), - State: convertStateToWWW(pr.State), - Timestamp: pr.Timestamp, - UserId: pr.UserID, - Username: pr.Username, - PublicKey: pr.PublicKey, - Signature: pr.Signature, - NumComments: uint(pr.Comments), - Version: pr.Version, - LinkedFrom: pr.LinkedFrom, - CensorshipRecord: www.CensorshipRecord{ - Token: pr.CensorshipRecord.Token, - Merkle: pr.CensorshipRecord.Merkle, - Signature: pr.CensorshipRecord.Signature, - }, - } + // Convert files files := make([]www.File, 0, len(pr.Files)) for _, f := range pr.Files { files = append(files, www.File{ @@ -167,8 +70,8 @@ func convertProposalToWWW(pr *pi.ProposalRecord) (*www.ProposalRecord, error) { Payload: f.Payload, }) } - pw.Files = files + // Convert metadata metadata := make([]www.Metadata, 0, len(pr.Metadata)) for _, md := range pr.Metadata { metadata = append(metadata, www.Metadata{ @@ -177,29 +80,104 @@ func convertProposalToWWW(pr *pi.ProposalRecord) (*www.ProposalRecord, error) { Payload: md.Payload, }) } - pw.Metadata = metadata var ( - changeMsg string - changeMsgTimestamp int64 + publishedAt, censoredAt, abandonedAt int64 + changeMsg string + changeMsgTimestamp int64 ) for _, v := range pr.Statuses { if v.Timestamp > changeMsgTimestamp { - changeMsgTimestamp = v.Timestamp changeMsg = v.Reason + changeMsgTimestamp = v.Timestamp } switch v.Status { case pi.PropStatusPublic: - pw.PublishedAt = v.Timestamp + publishedAt = v.Timestamp case pi.PropStatusCensored: - pw.CensoredAt = v.Timestamp + censoredAt = v.Timestamp case pi.PropStatusAbandoned: - pw.AbandonedAt = v.Timestamp + abandonedAt = v.Timestamp } } - pw.StatusChangeMessage = changeMsg - return &pw, nil + return &www.ProposalRecord{ + Name: pm.Name, + State: convertStateToWWW(pr.State), + Status: convertStatusToWWW(pr.Status), + Timestamp: pr.Timestamp, + UserId: pr.UserID, + Username: pr.Username, + PublicKey: pr.PublicKey, + Signature: pr.Signature, + NumComments: uint(pr.Comments), + Version: pr.Version, + StatusChangeMessage: changeMsg, + PublishedAt: publishedAt, + CensoredAt: censoredAt, + AbandonedAt: abandonedAt, + LinkTo: pm.LinkTo, + LinkBy: pm.LinkBy, + LinkedFrom: pr.LinkedFrom, + Files: files, + Metadata: metadata, + CensorshipRecord: www.CensorshipRecord{ + Token: pr.CensorshipRecord.Token, + Merkle: pr.CensorshipRecord.Merkle, + Signature: pr.CensorshipRecord.Signature, + }, + }, nil +} + +func convertVoteStatusToWWW(status ticketvote.VoteStatusT) www.PropVoteStatusT { + switch status { + case ticketvote.VoteStatusInvalid: + return www.PropVoteStatusInvalid + case ticketvote.VoteStatusUnauthorized: + return www.PropVoteStatusNotAuthorized + case ticketvote.VoteStatusAuthorized: + return www.PropVoteStatusAuthorized + case ticketvote.VoteStatusStarted: + return www.PropVoteStatusStarted + case ticketvote.VoteStatusFinished: + return www.PropVoteStatusFinished + default: + return www.PropVoteStatusInvalid + } +} + +func convertVoteTypeToWWW(t ticketvote.VoteT) www.VoteT { + switch t { + case ticketvote.VoteTypeInvalid: + return www.VoteTypeInvalid + case ticketvote.VoteTypeStandard: + return www.VoteTypeStandard + case ticketvote.VoteTypeRunoff: + return www.VoteTypeRunoff + default: + return www.VoteTypeInvalid + } +} + +func convertVoteErrorCodeToWWW(errcode ticketvote.VoteErrorT) decredplugin.ErrorStatusT { + switch errcode { + case ticketvote.VoteErrorInvalid: + return decredplugin.ErrorStatusInvalid + case ticketvote.VoteErrorInternalError: + return decredplugin.ErrorStatusInternalError + case ticketvote.VoteErrorRecordNotFound: + return decredplugin.ErrorStatusProposalNotFound + case ticketvote.VoteErrorVoteBitInvalid: + return decredplugin.ErrorStatusInvalidVoteBit + case ticketvote.VoteErrorVoteStatusInvalid: + return decredplugin.ErrorStatusVoteHasEnded + case ticketvote.VoteErrorTicketAlreadyVoted: + return decredplugin.ErrorStatusDuplicateVote + case ticketvote.VoteErrorTicketNotEligible: + return decredplugin.ErrorStatusIneligibleTicket + default: + return decredplugin.ErrorStatusInternalError + } } func (p *politeiawww) processProposalDetails(pd www.ProposalsDetails, u *user.User) (*www.ProposalDetailsReply, error) { @@ -213,17 +191,16 @@ func (p *politeiawww) processProposalDetails(pd www.ProposalsDetails, u *user.Us if err != nil { return nil, err } - pdr := www.ProposalDetailsReply{ - Proposal: *pw, - } - return &pdr, nil + return &www.ProposalDetailsReply{ + Proposal: *pw, + }, nil } func (p *politeiawww) processBatchProposals(bp www.BatchProposals, u *user.User) (*www.BatchProposalsReply, error) { log.Tracef("processBatchProposals: %v", bp.Tokens) - // Prep proposals requests + // Setup requests prs := make([]pi.ProposalRequest, 0, len(bp.Tokens)) for _, t := range bp.Tokens { prs = append(prs, pi.ProposalRequest{ @@ -231,12 +208,13 @@ func (p *politeiawww) processBatchProposals(bp www.BatchProposals, u *user.User) }) } + // Get proposals props, err := p.proposalRecords(pi.PropStateVetted, prs, false) if err != nil { return nil, err } - // Convert proposals records + // Prepare reply propsw := make([]www.ProposalRecord, 0, len(bp.Tokens)) for _, pr := range props { propw, err := convertProposalToWWW(&pr) @@ -251,86 +229,40 @@ func (p *politeiawww) processBatchProposals(bp www.BatchProposals, u *user.User) }, nil } -func (p *politeiawww) getPropComments(token string) ([]www.Comment, error) { - return nil, nil -} - -func (p *politeiawww) processAllVetted(v www.GetAllVetted) (*www.GetAllVettedReply, error) { - return nil, nil -} - -func (p *politeiawww) processCommentsGet(token string, u *user.User) (*www.GetCommentsReply, error) { - return nil, nil -} - -func (p *politeiawww) processVoteStatus(token string) (*www.VoteStatusReply, error) { - return nil, nil -} - -func (p *politeiawww) processGetAllVoteStatus() (*www.GetAllVoteStatusReply, error) { - return nil, nil -} - -func (p *politeiawww) processActiveVote() (*www.ActiveVoteReply, error) { - return nil, nil -} - func (p *politeiawww) processVoteResultsWWW(token string) (*www.VoteResultsReply, error) { log.Tracef("processVoteResultsWWW: %v", token) - // Call ticketvote plugin + // Get vote details vd, err := p.voteDetails([]string{token}) if err != nil { return nil, err } - - // Convert reply to www - var ( - vote ticketvote.RecordVote - ok bool - ) - if vote, ok = vd.Votes[token]; !ok { + vote, ok := vd.Votes[token] + if !ok { return nil, www.UserError{ ErrorCode: www.ErrorStatusProposalNotFound, } } + + // Convert to www startHeight := strconv.FormatUint(uint64(vote.Vote.StartBlockHeight), 10) endHeight := strconv.FormatUint(uint64(vote.Vote.EndBlockHeight), 10) - res := www.VoteResultsReply{ - StartVote: www.StartVote{ - PublicKey: vote.Vote.PublicKey, - Signature: vote.Vote.Signature, - Vote: www.Vote{ - Token: vote.Vote.Params.Token, - Mask: vote.Vote.Params.Mask, - Duration: vote.Vote.Params.Duration, - QuorumPercentage: vote.Vote.Params.QuorumPercentage, - PassPercentage: vote.Vote.Params.PassPercentage, - }, - }, - StartVoteReply: www.StartVoteReply{ - StartBlockHeight: startHeight, - StartBlockHash: vote.Vote.StartBlockHash, - EndHeight: endHeight, - EligibleTickets: vote.Vote.EligibleTickets, - }, - } - - // Transalte vote options - vo := make([]www.VoteOption, 0, len(vote.Vote.Params.Options)) + options := make([]www.VoteOption, 0, len(vote.Vote.Params.Options)) for _, o := range vote.Vote.Params.Options { - vo = append(vo, www.VoteOption{ + options = append(options, www.VoteOption{ Id: o.ID, Description: o.Description, Bits: o.Bit, }) } - res.StartVote.Vote.Options = vo - // Get cast votes information + // Get cast votes cv, err := p.castVotes(token) + if err != nil { + return nil, err + } - // Transalte to www + // Convert to www votes := make([]www.CastVote, 0, len(cv.Votes)) for _, v := range cv.Votes { votes = append(votes, www.CastVote{ @@ -340,70 +272,44 @@ func (p *politeiawww) processVoteResultsWWW(token string) (*www.VoteResultsReply Signature: v.Signature, }) } - res.CastVotes = votes - - return &res, nil -} - -func convertVoteStatusToWWW(status ticketvote.VoteStatusT) www.PropVoteStatusT { - switch status { - case ticketvote.VoteStatusInvalid: - return www.PropVoteStatusInvalid - case ticketvote.VoteStatusUnauthorized: - return www.PropVoteStatusNotAuthorized - case ticketvote.VoteStatusAuthorized: - return www.PropVoteStatusAuthorized - case ticketvote.VoteStatusStarted: - return www.PropVoteStatusStarted - case ticketvote.VoteStatusFinished: - return www.PropVoteStatusFinished - default: - return www.PropVoteStatusInvalid - } -} -func convertVoteTypeToWWW(t ticketvote.VoteT) www.VoteT { - switch t { - case ticketvote.VoteTypeInvalid: - return www.VoteTypeInvalid - case ticketvote.VoteTypeStandard: - return www.VoteTypeStandard - case ticketvote.VoteTypeRunoff: - return www.VoteTypeRunoff - default: - return www.VoteTypeInvalid - } + return &www.VoteResultsReply{ + StartVote: www.StartVote{ + PublicKey: vote.Vote.PublicKey, + Signature: vote.Vote.Signature, + Vote: www.Vote{ + Token: vote.Vote.Params.Token, + Mask: vote.Vote.Params.Mask, + Duration: vote.Vote.Params.Duration, + QuorumPercentage: vote.Vote.Params.QuorumPercentage, + PassPercentage: vote.Vote.Params.PassPercentage, + Options: options, + }, + }, + StartVoteReply: www.StartVoteReply{ + StartBlockHeight: startHeight, + StartBlockHash: vote.Vote.StartBlockHash, + EndHeight: endHeight, + EligibleTickets: vote.Vote.EligibleTickets, + }, + CastVotes: votes, + }, nil } func (p *politeiawww) processBatchVoteSummary(bvs www.BatchVoteSummary) (*www.BatchVoteSummaryReply, error) { log.Tracef("processBatchVoteSummary: %v", bvs.Tokens) - // Call ticketvote plugin to get vote summaries + // Get vote summaries sm, err := p.voteSummaries(bvs.Tokens) if err != nil { return nil, err } - // Covert reply to www - res := www.BatchVoteSummaryReply{ - BestBlock: uint64(sm.BestBlock), - } - // Translate summaries + // Prepare reply summaries := make(map[string]www.VoteSummary, len(sm.Summaries)) - for t, sum := range sm.Summaries { - vs := www.VoteSummary{ - Status: convertVoteStatusToWWW(sum.Status), - Type: convertVoteTypeToWWW(sum.Type), - Approved: sum.Approved, - EligibleTickets: sum.EligibleTickets, - Duration: sum.Duration, - EndHeight: uint64(sum.EndBlockHeight), - QuorumPercentage: sum.QuorumPercentage, - PassPercentage: sum.PassPercentage, - } - // Translate vote options - results := make([]www.VoteOptionResult, 0, len(sum.Results)) - for _, r := range sum.Results { + for k, v := range sm.Summaries { + results := make([]www.VoteOptionResult, 0, len(v.Results)) + for _, r := range v.Results { results = append(results, www.VoteOptionResult{ VotesReceived: r.Votes, Option: www.VoteOption{ @@ -413,1018 +319,1026 @@ func (p *politeiawww) processBatchVoteSummary(bvs www.BatchVoteSummary) (*www.Ba }, }) } - vs.Results = results - summaries[t] = vs - + summaries[k] = www.VoteSummary{ + Status: convertVoteStatusToWWW(v.Status), + Type: convertVoteTypeToWWW(v.Type), + Approved: v.Approved, + EligibleTickets: v.EligibleTickets, + Duration: v.Duration, + EndHeight: uint64(v.EndBlockHeight), + QuorumPercentage: v.QuorumPercentage, + PassPercentage: v.PassPercentage, + Results: results, + } } - res.Summaries = summaries - - return &res, nil -} -func convertVoteErrorCodeToWWW(errcode ticketvote.VoteErrorT) decredplugin.ErrorStatusT { - switch errcode { - case ticketvote.VoteErrorInvalid: - return decredplugin.ErrorStatusInvalid - case ticketvote.VoteErrorInternalError: - return decredplugin.ErrorStatusInternalError - case ticketvote.VoteErrorRecordNotFound: - return decredplugin.ErrorStatusProposalNotFound - case ticketvote.VoteErrorVoteBitInvalid: - return decredplugin.ErrorStatusInvalidVoteBit - case ticketvote.VoteErrorVoteStatusInvalid: - return decredplugin.ErrorStatusVoteHasEnded - case ticketvote.VoteErrorTicketAlreadyVoted: - return decredplugin.ErrorStatusDuplicateVote - case ticketvote.VoteErrorTicketNotEligible: - return decredplugin.ErrorStatusIneligibleTicket - default: - return decredplugin.ErrorStatusInternalError - } + return &www.BatchVoteSummaryReply{ + Summaries: summaries, + BestBlock: uint64(sm.BestBlock), + }, nil } func (p *politeiawww) processCastVotes(ballot *www.Ballot) (*www.BallotReply, error) { log.Tracef("processCastVotes") - // Call ticketvote plugin to cast votes - b, err := p.ballot(ballot) + // Prepare plugin command + votes := make([]ticketvote.CastVote, 0, len(ballot.Votes)) + for _, vote := range ballot.Votes { + votes = append(votes, ticketvote.CastVote{ + Token: vote.Ticket, + Ticket: vote.Ticket, + VoteBit: vote.VoteBit, + Signature: vote.Signature, + }) + } + b := ticketvote.Ballot{ + Votes: votes, + } + payload, err := ticketvote.EncodeBallot(b) if err != nil { return nil, err } - // Translate reply to www - res := www.BallotReply{} - rps := make([]www.CastVoteReply, 0, len(b.Receipts)) - for i, rp := range b.Receipts { - rps = append(rps, www.CastVoteReply{ - ClientSignature: ballot.Votes[i].Signature, - Signature: rp.Receipt, - Error: rp.ErrorContext, - ErrorStatus: convertVoteErrorCodeToWWW(rp.ErrorCode), - }) + // Send plugin command + r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdBallot, + string(payload)) + if err != nil { + return nil, err } - - return &res, nil -} - -// validateAuthorizeVote validates the authorize vote fields. A UserError is -// returned if any of the validation fails. -func validateAuthorizeVote(av www.AuthorizeVote, u user.User, pr www.ProposalRecord, vs www.VoteSummary) error { - // Ensure the public key is the user's active key - if av.PublicKey != u.PublicKey() { - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidSigningKey, - } - } - - // Validate signature - msg := av.Token + pr.Version + av.Action - err := validateSignature(av.PublicKey, av.Signature, msg) + br, err := ticketvote.DecodeBallotReply([]byte(r)) if err != nil { - return err + return nil, err } - // Verify record is in the right state and that the authorize - // vote request is valid. A vote authorization may already - // exist. We also allow vote authorizations to be revoked. - switch { - case pr.Status != www.PropStatusPublic: - // Record not public - return www.UserError{ - ErrorCode: www.ErrorStatusWrongStatus, - } - case vs.EndHeight != 0: - // Vote has already started - return www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - } - case av.Action != decredplugin.AuthVoteActionAuthorize && - av.Action != decredplugin.AuthVoteActionRevoke: - // Invalid authorize vote action - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidAuthVoteAction, - } - case av.Action == decredplugin.AuthVoteActionAuthorize && - vs.Status == www.PropVoteStatusAuthorized: - // Cannot authorize vote; vote has already been authorized - return www.UserError{ - ErrorCode: www.ErrorStatusVoteAlreadyAuthorized, - } - case av.Action == decredplugin.AuthVoteActionRevoke && - vs.Status != www.PropVoteStatusAuthorized: - // Cannot revoke authorization; vote has not been authorized - return www.UserError{ - ErrorCode: www.ErrorStatusVoteNotAuthorized, - } + // Prepare reply + receipts := make([]www.CastVoteReply, 0, len(br.Receipts)) + for k, v := range br.Receipts { + receipts = append(receipts, www.CastVoteReply{ + ClientSignature: ballot.Votes[k].Signature, + Signature: v.Receipt, + Error: v.ErrorContext, + ErrorStatus: convertVoteErrorCodeToWWW(v.ErrorCode), + }) } - return nil + return &www.BallotReply{ + Receipts: receipts, + }, nil } -// validateAuthorizeVoteStandard validates the authorize vote for a proposal that -// is participating in a standard vote. A UserError is returned if any of the -// validation fails. -func validateAuthorizeVoteStandard(av www.AuthorizeVote, u user.User, pr www.ProposalRecord, vs www.VoteSummary) error { - /* - err := validateAuthorizeVote(av, u, pr, vs) - if err != nil { - return err - } - - // The rest of the validation is specific to authorize votes for - // standard votes. - switch { - case isRFPSubmission(pr): - return www.UserError{ - ErrorCode: www.ErrorStatusWrongProposalType, - ErrorContext: []string{"proposal is an rfp submission"}, - } - case pr.PublicKey != av.PublicKey: - // User is not the author. First make sure the author didn't - // submit the proposal using an old identity. - if !isProposalAuthor(pr, u) { - return www.UserError{ - ErrorCode: www.ErrorStatusUserNotAuthor, - } - } - } - */ - - return nil -} +func (p *politeiawww) processTokenInventory(isAdmin bool) (*www.TokenInventoryReply, error) { + log.Tracef("processTokenInventory") -// validateAuthorizeVoteRunoff validates the authorize vote for a proposal that -// is participating in a runoff vote. A UserError is returned if any of the -// validation fails. -func validateAuthorizeVoteRunoff(av www.AuthorizeVote, u user.User, pr www.ProposalRecord, vs www.VoteSummary) error { - err := validateAuthorizeVote(av, u, pr, vs) + // Get record inventory + ri, err := p.inventoryByStatus() if err != nil { - return err + return nil, err } - // The rest of the validation is specific to authorize votes for - // runoff votes. - switch { - case !u.Admin: - // User is not an admin - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidSigningKey, - ErrorContext: []string{"user not an admin"}, - } + // Get vote inventory + vi, err := p.piVoteInventory() + if err != nil { + return nil, err } - return nil -} + // Prepare reply + tir := www.TokenInventoryReply{ + Pre: append(vi.Unauthorized, vi.Authorized...), + Active: vi.Started, + Approved: vi.Approved, + Rejected: vi.Rejected, + Abandoned: ri.Archived, + } + if isAdmin { + tir.Unreviewed = ri.Unvetted + tir.Censored = ri.Censored + } -func (p *politeiawww) voteSummaryGet(token string, bestBlock uint64) (*www.VoteSummary, error) { - return nil, nil + return &tir, nil } -// processAuthorizeVote sends the authorizevote command to decred plugin to -// indicate that a proposal has been finalized and is ready to be voted on. +/* func (p *politeiawww) processAuthorizeVote(av www.AuthorizeVote, u *user.User) (*www.AuthorizeVoteReply, error) { - log.Tracef("processAuthorizeVote %v", av.Token) - - /* - // Make sure token is valid and not a prefix - if !tokenIsValid(av.Token) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: []string{av.Token}, - } + // Make sure token is valid and not a prefix + if !tokenIsValid(av.Token) { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidCensorshipToken, + ErrorContext: []string{av.Token}, } + } - // Validate the vote authorization - pr, err := p.getProp(av.Token) - if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - } + // Validate the vote authorization + pr, err := p.getProp(av.Token) + if err != nil { + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, } - return nil, err - } - bb, err := p.getBestBlock() - if err != nil { - return nil, err - } - vs, err := p.voteSummaryGet(av.Token, bb) - if err != nil { - return nil, err - } - err = validateAuthorizeVoteStandard(av, *u, *pr, *vs) - if err != nil { - return nil, err - } - - // Setup plugin command - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, fmt.Errorf("Random: %v", err) } + return nil, err + } + bb, err := p.getBestBlock() + if err != nil { + return nil, err + } + vs, err := p.voteSummaryGet(av.Token, bb) + if err != nil { + return nil, err + } + err = validateAuthorizeVoteStandard(av, *u, *pr, *vs) + if err != nil { + return nil, err + } - dav := convertAuthorizeVoteToDecred(av) - payload, err := decredplugin.EncodeAuthorizeVote(dav) - if err != nil { - return nil, fmt.Errorf("EncodeAuthorizeVote: %v", err) - } + // Setup plugin command + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return nil, fmt.Errorf("Random: %v", err) + } - pc := pd.PluginCommand{ - Challenge: hex.EncodeToString(challenge), - ID: decredplugin.ID, - Command: decredplugin.CmdAuthorizeVote, - CommandID: decredplugin.CmdAuthorizeVote + " " + av.Token, - Payload: string(payload), - } + dav := convertAuthorizeVoteToDecred(av) + payload, err := decredplugin.EncodeAuthorizeVote(dav) + if err != nil { + return nil, fmt.Errorf("EncodeAuthorizeVote: %v", err) + } - // Send authorizevote plugin request - responseBody, err := p.makeRequest(http.MethodPost, - pd.PluginCommandRoute, pc) - if err != nil { - return nil, err - } + pc := pd.PluginCommand{ + Challenge: hex.EncodeToString(challenge), + ID: decredplugin.ID, + Command: decredplugin.CmdAuthorizeVote, + CommandID: decredplugin.CmdAuthorizeVote + " " + av.Token, + Payload: string(payload), + } - var reply pd.PluginCommandReply - err = json.Unmarshal(responseBody, &reply) - if err != nil { - return nil, fmt.Errorf("Unmarshal PluginCommandReply: %v", err) - } + // Send authorizevote plugin request + responseBody, err := p.makeRequest(http.MethodPost, + pd.PluginCommandRoute, pc) + if err != nil { + return nil, err + } - // Verify challenge - err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) - if err != nil { - return nil, fmt.Errorf("VerifyChallenge: %v", err) - } + var reply pd.PluginCommandReply + err = json.Unmarshal(responseBody, &reply) + if err != nil { + return nil, fmt.Errorf("Unmarshal PluginCommandReply: %v", err) + } - // Decode plugin reply - avr, err := decredplugin.DecodeAuthorizeVoteReply([]byte(reply.Payload)) - if err != nil { - return nil, fmt.Errorf("DecodeAuthorizeVoteReply: %v", err) - } - - if !p.test && avr.Action == decredplugin.AuthVoteActionAuthorize { - // Emit event notification for proposal vote authorized - p.eventManager.emit(eventProposalVoteAuthorized, - dataProposalVoteAuthorized{ - token: av.Token, - name: pr.Name, - username: u.Username, - email: u.Email, - }) - } - - return &www.AuthorizeVoteReply{ - Action: avr.Action, - Receipt: avr.Receipt, - }, nil - */ - return nil, nil -} + // Verify challenge + err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) + if err != nil { + return nil, fmt.Errorf("VerifyChallenge: %v", err) + } -// validateVoteOptions verifies that the provided vote options -// specify a simple approve/reject vote and nothing else. A UserError is -// returned if this validation fails. -func validateVoteOptions(options []www2.VoteOption) error { - /* - if len(options) == 0 { - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidVoteOptions, - ErrorContext: []string{"no vote options found"}, - } - } - optionIDs := map[string]bool{ - decredplugin.VoteOptionIDApprove: false, - decredplugin.VoteOptionIDReject: false, - } - for _, vo := range options { - if _, ok := optionIDs[vo.Id]; !ok { - e := fmt.Sprintf("invalid vote option id '%v'", vo.Id) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidVoteOptions, - ErrorContext: []string{e}, - } - } - optionIDs[vo.Id] = true - } - for k, wasFound := range optionIDs { - if !wasFound { - e := fmt.Sprintf("missing vote option id '%v'", k) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidVoteOptions, - ErrorContext: []string{e}, - } - } - } - return nil - */ - return nil -} + // Decode plugin reply + avr, err := decredplugin.DecodeAuthorizeVoteReply([]byte(reply.Payload)) + if err != nil { + return nil, fmt.Errorf("DecodeAuthorizeVoteReply: %v", err) + } -func validateStartVote(sv www2.StartVote, u user.User, pr www.ProposalRecord, vs www.VoteSummary, durationMin, durationMax uint32) error { - /* - if !tokenIsValid(sv.Vote.Token) { - // Sanity check since proposal has already been looked up and - // passed in to this function. - return fmt.Errorf("invalid token %v", sv.Vote.Token) - } + if !p.test && avr.Action == decredplugin.AuthVoteActionAuthorize { + // Emit event notification for proposal vote authorized + p.eventManager.emit(eventProposalVoteAuthorized, + dataProposalVoteAuthorized{ + token: av.Token, + name: pr.Name, + username: u.Username, + email: u.Email, + }) + } - // Validate vote bits - for _, v := range sv.Vote.Options { - err := validateVoteBit(sv.Vote, v.Bits) - if err != nil { - log.Debugf("validateStartVote: validateVoteBit '%v': %v", - v.Id, err) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidPropVoteBits, - } - } - } + return &www.AuthorizeVoteReply{ + Action: avr.Action, + Receipt: avr.Receipt, + }, nil +} - // Validate vote options. Only simple yes/no votes are currently - // allowed. - err := validateVoteOptions(sv.Vote.Options) - if err != nil { - return err - } +func (p *politeiawww) processStartVoteV2(sv www2.StartVote, u *user.User) (*www2.StartVoteReply, error) { + log.Tracef("processStartVoteV2 %v", sv.Vote.Token) - // Validate vote params - switch { - case sv.Vote.Duration < durationMin: - // Duration not large enough - e := fmt.Sprintf("vote duration must be >= %v", durationMin) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidPropVoteParams, - ErrorContext: []string{e}, - } - case sv.Vote.Duration > durationMax: - // Duration too large - e := fmt.Sprintf("vote duration must be <= %v", durationMax) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidPropVoteParams, - ErrorContext: []string{e}, - } - case sv.Vote.QuorumPercentage > 100: - // Quorum too large - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidPropVoteParams, - ErrorContext: []string{"quorum percentage cannot be >100"}, - } - case sv.Vote.PassPercentage > 100: - // Pass percentage too large - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidPropVoteParams, - ErrorContext: []string{"pass percentage cannot be >100"}, - } - } + // Sanity check + if !u.Admin { + return nil, fmt.Errorf("user is not an admin") + } - // Ensure the public key is the user's active key - if sv.PublicKey != u.PublicKey() { - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidSigningKey, - } + // Fetch proposal and vote summary + if !tokenIsValid(sv.Vote.Token) { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidCensorshipToken, + ErrorContext: []string{sv.Vote.Token}, } - - // Validate signature - dsv := convertStartVoteV2ToDecred(sv) - err = dsv.VerifySignature() - if err != nil { - log.Debugf("validateStartVote: VerifySignature: %v", err) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidSignature, + } + pr, err := p.getProp(sv.Vote.Token) + if err != nil { + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, } } + return nil, err + } + bb, err := p.getBestBlock() + if err != nil { + return nil, err + } + vs, err := p.voteSummaryGet(sv.Vote.Token, bb) + if err != nil { + return nil, err + } - // Validate proposal - votePropVersion := strconv.FormatUint(uint64(sv.Vote.ProposalVersion), 10) - switch { - case pr.Version != votePropVersion: - // Vote is specifying the wrong version - e := fmt.Sprintf("got %v, want %v", votePropVersion, pr.Version) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidProposalVersion, - ErrorContext: []string{e}, - } - case pr.Status != www.PropStatusPublic: - // Proposal is not public - return www.UserError{ - ErrorCode: www.ErrorStatusWrongStatus, - ErrorContext: []string{"proposal is not public"}, - } - case vs.EndHeight != 0: - // Vote has already started - return www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - ErrorContext: []string{"vote already started"}, - } - } + // Validate the start vote + err = p.validateStartVoteStandard(sv, *u, *pr, *vs) + if err != nil { + return nil, err + } - return nil - */ - return nil -} + // Tell decred plugin to start voting + dsv := convertStartVoteV2ToDecred(sv) + payload, err := decredplugin.EncodeStartVoteV2(dsv) + if err != nil { + return nil, err + } + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return nil, err + } + pc := pd.PluginCommand{ + Challenge: hex.EncodeToString(challenge), + ID: decredplugin.ID, + Command: decredplugin.CmdStartVote, + CommandID: decredplugin.CmdStartVote + " " + sv.Vote.Token, + Payload: string(payload), + } + responseBody, err := p.makeRequest(http.MethodPost, + pd.PluginCommandRoute, pc) + if err != nil { + return nil, err + } -func (p *politeiawww) validateStartVoteStandard(sv www2.StartVote, u user.User, pr www.ProposalRecord, vs www.VoteSummary) error { - /* - err := validateStartVote(sv, u, pr, vs, p.cfg.VoteDurationMin, - p.cfg.VoteDurationMax) - if err != nil { - return err - } + // Handle reply + var reply pd.PluginCommandReply + err = json.Unmarshal(responseBody, &reply) + if err != nil { + return nil, fmt.Errorf("could not unmarshal "+ + "PluginCommandReply: %v", err) + } + err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) + if err != nil { + return nil, err + } + dsvr, err := decredplugin.DecodeStartVoteReply([]byte(reply.Payload)) + if err != nil { + return nil, err + } + svr, err := convertStartVoteReplyV2FromDecred(*dsvr) + if err != nil { + return nil, err + } - // The remaining validation is specific to a VoteTypeStandard. - switch { - case sv.Vote.Type != www2.VoteTypeStandard: - // Not a standard vote - e := fmt.Sprintf("vote type must be %v", www2.VoteTypeStandard) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidVoteType, - ErrorContext: []string{e}, - } - case vs.Status != www.PropVoteStatusAuthorized: - // Vote has not been authorized - return www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - ErrorContext: []string{"vote not authorized"}, - } - case isRFPSubmission(pr): - // The proposal is an an RFP submission. The voting period for - // RFP submissions can only be started using the StartVoteRunoff - // route. - return www.UserError{ - ErrorCode: www.ErrorStatusWrongProposalType, - ErrorContext: []string{"cannot be an rfp submission"}, - } - } + // Get author data + author, err := p.db.UserGetByPubKey(pr.PublicKey) + if err != nil { + return nil, err + } - // Verify the LinkBy deadline for RFP proposals. The LinkBy policy - // requirements are enforced at the time of starting the vote - // because its purpose is to ensure that there is enough time for - // RFP submissions to be submitted. - if isRFP(pr) { - err := p.linkByValidate(pr.LinkBy) - if err != nil { - return err - } - } + // Emit event notification for proposal start vote + p.eventManager.emit(eventProposalVoteStarted, + dataProposalVoteStarted{ + token: pr.CensorshipRecord.Token, + name: pr.Name, + adminID: u.ID.String(), + author: *author, + }) - return nil - */ - return nil + return svr, nil } -func validateStartVoteRunoff(sv www2.StartVote, u user.User, pr www.ProposalRecord, vs www.VoteSummary, durationMin, durationMax uint32) error { - /* - err := validateStartVote(sv, u, pr, vs, durationMin, durationMax) - if err != nil { - return err - } +func (p *politeiawww) processStartVoteRunoffV2(sv www2.StartVoteRunoff, u *user.User) (*www2.StartVoteRunoffReply, error) { + log.Tracef("processStartVoteRunoffV2 %v", sv.Token) + + // Sanity check + if !u.Admin { + return nil, fmt.Errorf("user is not an admin") + } - // The remaining validation is specific to a VoteTypeRunoff. + bb, err := p.getBestBlock() + if err != nil { + return nil, err + } - token := sv.Vote.Token - switch { - case sv.Vote.Type != www2.VoteTypeRunoff: - // Not a runoff vote - e := fmt.Sprintf("%v vote type must be %v", - token, www2.VoteTypeRunoff) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidVoteType, + // Ensure authorize votes and start votes match + auths := make(map[string]www2.AuthorizeVote, len(sv.AuthorizeVotes)) + starts := make(map[string]www2.StartVote, len(sv.StartVotes)) + for _, v := range sv.AuthorizeVotes { + auths[v.Token] = v + } + for _, v := range sv.StartVotes { + _, ok := auths[v.Vote.Token] + if !ok { + e := fmt.Sprintf("start vote found without matching authorize vote %v", + v.Vote.Token) + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidRunoffVote, ErrorContext: []string{e}, } - - case !isRFPSubmission(pr): - // The proposal is not an RFP submission - e := fmt.Sprintf("%v is not an rfp submission", token) - return www.UserError{ - ErrorCode: www.ErrorStatusWrongProposalType, + } + } + for _, v := range sv.StartVotes { + starts[v.Vote.Token] = v + } + for _, v := range sv.AuthorizeVotes { + _, ok := starts[v.Token] + if !ok { + e := fmt.Sprintf("authorize vote found without matching start vote %v", + v.Token) + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidRunoffVote, ErrorContext: []string{e}, } - - case vs.Status != www.PropVoteStatusNotAuthorized: - // Sanity check. This should not be possible. - return fmt.Errorf("%v got vote status %v, want %v", - token, vs.Status, www.PropVoteStatusNotAuthorized) } - - return nil - */ - return nil -} - -// processStartVoteV2 starts the voting period on a proposal using the provided -// v2 StartVote. Proposals that are RFP submissions cannot use this route. They -// must sue the StartVoteRunoff route. -func (p *politeiawww) processStartVoteV2(sv www2.StartVote, u *user.User) (*www2.StartVoteReply, error) { - log.Tracef("processStartVoteV2 %v", sv.Vote.Token) - - /* - // Sanity check - if !u.Admin { - return nil, fmt.Errorf("user is not an admin") + } + if len(auths) == 0 { + e := "start votes and authorize votes cannot be empty" + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidRunoffVote, + ErrorContext: []string{e}, } + } + + // Slice used to send event notification for each proposal + // starting a vote, at the end of this function. + var proposalNotifications []*www.ProposalRecord + // Validate authorize votes and start votes + for _, v := range sv.StartVotes { // Fetch proposal and vote summary - if !tokenIsValid(sv.Vote.Token) { + token := v.Vote.Token + if !tokenIsValid(token) { return nil, www.UserError{ ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: []string{sv.Vote.Token}, + ErrorContext: []string{token}, } } - pr, err := p.getProp(sv.Vote.Token) + pr, err := p.getProp(token) if err != nil { if err == cache.ErrRecordNotFound { err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, + ErrorCode: www.ErrorStatusProposalNotFound, + ErrorContext: []string{token}, } } return nil, err } - bb, err := p.getBestBlock() + + proposalNotifications = append(proposalNotifications, pr) + + vs, err := p.voteSummaryGet(token, bb) if err != nil { return nil, err } - vs, err := p.voteSummaryGet(sv.Vote.Token, bb) + + // Validate authorize vote. The validation function requires a v1 + // AuthorizeVote. This is fine. There is no difference between v1 + // and v2. + av := auths[v.Vote.Token] + av1 := www.AuthorizeVote{ + Token: av.Token, + Action: av.Action, + PublicKey: av.PublicKey, + Signature: av.Signature, + } + err = validateAuthorizeVoteRunoff(av1, *u, *pr, *vs) if err != nil { + // Attach the token to the error so the user knows which one + // failed. + if ue, ok := err.(*www.UserError); ok { + ue.ErrorContext = append(ue.ErrorContext, token) + err = ue + } return nil, err } - // Validate the start vote - err = p.validateStartVoteStandard(sv, *u, *pr, *vs) + // Validate start vote + err = validateStartVoteRunoff(v, *u, *pr, *vs, + p.cfg.VoteDurationMin, p.cfg.VoteDurationMax) if err != nil { + // Attach the token to the error so the user knows which one + // failed. + if ue, ok := err.(*www.UserError); ok { + ue.ErrorContext = append(ue.ErrorContext, token) + err = ue + } return nil, err } + } - // Tell decred plugin to start voting - dsv := convertStartVoteV2ToDecred(sv) - payload, err := decredplugin.EncodeStartVoteV2(dsv) - if err != nil { - return nil, err + // Validate the RFP proposal + rfp, err := p.getProp(sv.Token) + if err != nil { + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + ErrorContext: []string{sv.Token}, + } } - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err + return nil, err + } + switch { + case rfp.LinkBy > time.Now().Unix() && !p.cfg.TestNet: + // Vote cannot start on RFP submissions until the RFP linkby + // deadline has been met. This validation is skipped when on + // testnet. + return nil, www.UserError{ + ErrorCode: www.ErrorStatusLinkByDeadlineNotMet, + } + case len(rfp.LinkedFrom) == 0: + return nil, www.UserError{ + ErrorCode: www.ErrorStatusNoLinkedProposals, + } + } + + // Compile a list of the public, non-abandoned RFP submissions. + // This list will be used to ensure a StartVote exists for each + // of the public, non-abandoned submissions. + linkedFromProps, err := p.getProps(rfp.LinkedFrom) + if err != nil { + return nil, err + } + submissions := make(map[string]bool, len(rfp.LinkedFrom)) // [token]startVoteFound + for _, v := range linkedFromProps { + // Filter out abandoned submissions. These are not allowed + // to be included in a runoff vote. + if v.Status != www.PropStatusPublic { + continue + } + + // Set to false for now until we check that a StartVote + // was included for this proposal. + submissions[v.CensorshipRecord.Token] = false + } + + // Verify that a StartVote exists for all public, non-abandoned + // submissions and that there are no extra StartVotes. + for _, v := range sv.StartVotes { + _, ok := submissions[v.Vote.Token] + if !ok { + e := fmt.Sprintf("invalid start vote submission: %v", + v.Vote.Token) + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidRunoffVote, + ErrorContext: []string{e}, + } } - pc := pd.PluginCommand{ - Challenge: hex.EncodeToString(challenge), - ID: decredplugin.ID, - Command: decredplugin.CmdStartVote, - CommandID: decredplugin.CmdStartVote + " " + sv.Vote.Token, - Payload: string(payload), + + // A StartVote was included for this proposal + submissions[v.Vote.Token] = true + } + for token, startVoteFound := range submissions { + if !startVoteFound { + e := fmt.Sprintf("missing start vote for rfp submission: %v", + token) + return nil, www.UserError{ + ErrorCode: www.ErrorStatusInvalidRunoffVote, + ErrorContext: []string{e}, + } } - responseBody, err := p.makeRequest(http.MethodPost, - pd.PluginCommandRoute, pc) + } + + // Setup plugin command + dav := convertAuthorizeVotesV2ToDecred(sv.AuthorizeVotes) + dsv := convertStartVotesV2ToDecred(sv.StartVotes) + payload, err := decredplugin.EncodeStartVoteRunoff( + decredplugin.StartVoteRunoff{ + Token: sv.Token, + AuthorizeVotes: dav, + StartVotes: dsv, + }) + if err != nil { + return nil, err + } + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return nil, err + } + pc := pd.PluginCommand{ + Challenge: hex.EncodeToString(challenge), + ID: decredplugin.ID, + Command: decredplugin.CmdStartVoteRunoff, + Payload: string(payload), + } + + // Send plugin command + responseBody, err := p.makeRequest(http.MethodPost, + pd.PluginCommandRoute, pc) + if err != nil { + return nil, err + } + + // Handle response + var reply pd.PluginCommandReply + err = json.Unmarshal(responseBody, &reply) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) + if err != nil { + return nil, err + } + dsvr, err := decredplugin.DecodeStartVoteRunoffReply([]byte(reply.Payload)) + if err != nil { + return nil, err + } + svr, err := convertStartVoteReplyV2FromDecred(dsvr.StartVoteReply) + if err != nil { + return nil, err + } + + // Emit event notification for each proposal starting vote + for _, pn := range proposalNotifications { + author, err := p.db.UserGetByPubKey(pn.PublicKey) if err != nil { return nil, err } + p.eventManager.emit(eventProposalVoteStarted, + dataProposalVoteStarted{ + token: pn.CensorshipRecord.Token, + name: pn.Name, + adminID: u.ID.String(), + author: *author, + }) + } - // Handle reply - var reply pd.PluginCommandReply - err = json.Unmarshal(responseBody, &reply) - if err != nil { - return nil, fmt.Errorf("could not unmarshal "+ - "PluginCommandReply: %v", err) + return &www2.StartVoteRunoffReply{ + StartBlockHeight: svr.StartBlockHeight, + StartBlockHash: svr.StartBlockHash, + EndBlockHeight: svr.EndBlockHeight, + EligibleTickets: svr.EligibleTickets, + }, nil +} + +func (p *politeiawww) processVoteDetailsV2(token string) (*www2.VoteDetailsReply, error) { + log.Tracef("processVoteDetailsV2: %v", token) + + // Validate vote status + dvdr, err := p.decredVoteDetails(token) + if err != nil { + if err == cache.ErrRecordNotFound { + err = www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + } + } + return nil, err + } + if dvdr.StartVoteReply.StartBlockHash == "" { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusWrongVoteStatus, + ErrorContext: []string{"voting has not started yet"}, } - err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) + } + + // Handle StartVote versioning + var vdr *www2.VoteDetailsReply + switch dvdr.StartVote.Version { + case decredplugin.VersionStartVoteV1: + b := []byte(dvdr.StartVote.Payload) + dsv1, err := decredplugin.DecodeStartVoteV1(b) if err != nil { return nil, err } - dsvr, err := decredplugin.DecodeStartVoteReply([]byte(reply.Payload)) + vdr, err = convertDecredStartVoteV1ToVoteDetailsReplyV2(*dsv1, + dvdr.StartVoteReply) if err != nil { return nil, err } - svr, err := convertStartVoteReplyV2FromDecred(*dsvr) + case decredplugin.VersionStartVoteV2: + b := []byte(dvdr.StartVote.Payload) + dsv2, err := decredplugin.DecodeStartVoteV2(b) if err != nil { return nil, err } - - // Get author data - author, err := p.db.UserGetByPubKey(pr.PublicKey) + vdr, err = convertDecredStartVoteV2ToVoteDetailsReplyV2(*dsv2, + dvdr.StartVoteReply) if err != nil { return nil, err } - // Emit event notification for proposal start vote - p.eventManager.emit(eventProposalVoteStarted, - dataProposalVoteStarted{ - token: pr.CensorshipRecord.Token, - name: pr.Name, - adminID: u.ID.String(), - author: *author, - }) + default: + return nil, fmt.Errorf("invalid StartVote version %v %v", + token, dvdr.StartVote.Version) + } - return svr, nil - */ + return vdr, nil +} - return nil, nil +// isRFP returns whether the proposal is a Request For Proposals (RFP). +func isRFP(pr www.ProposalRecord) bool { + return pr.LinkBy != 0 } -// voteIsApproved returns whether the provided VoteSummary met the quorum -// and pass requirements. This function should only be called on simple -// approve/reject votes that use the decredplugin VoteOptionIDApprove. -func voteIsApproved(vs www.VoteSummary) bool { - if vs.Status != www.PropVoteStatusFinished { - // Vote has not ended yet - return false +func isRFPSubmission(pr www.ProposalRecord) bool { + // Right now the only proposals that we allow linking to + // are RFPs so if the linkto is set than this is an RFP + // submission. This may change in the future, at which + // point we'll actually have to check the linkto proposal + // to see if its an RFP. + return pr.LinkTo != "" +} + +// validateVoteBit ensures that bit is a valid vote bit. +func validateVoteBit(vote www2.Vote, bit uint64) error { + if len(vote.Options) == 0 { + return fmt.Errorf("vote corrupt") + } + if bit == 0 { + return fmt.Errorf("invalid bit 0x%x", bit) + } + if vote.Mask&bit != bit { + return fmt.Errorf("invalid mask 0x%x bit 0x%x", + vote.Mask, bit) } - var ( - total uint64 - approve uint64 - ) - for _, v := range vs.Results { - total += v.VotesReceived - if v.Option.Id == decredplugin.VoteOptionIDApprove { - approve = v.VotesReceived + for _, v := range vote.Options { + if v.Bits == bit { + return nil } } - quorum := uint64(float64(vs.QuorumPercentage) / 100 * float64(vs.EligibleTickets)) - pass := uint64(float64(vs.PassPercentage) / 100 * float64(total)) + + return fmt.Errorf("bit not found 0x%x", bit) +} + +func (p *politeiawww) linkByValidate(linkBy int64) error { + min := time.Now().Unix() + p.linkByPeriodMin() + max := time.Now().Unix() + p.linkByPeriodMax() switch { - case total < quorum: - // Quorum not met - return false - case approve < pass: - // Pass percentage not met - return false + case linkBy < min: + e := fmt.Sprintf("linkby %v is less than min required of %v", + linkBy, min) + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidLinkBy, + ErrorContext: []string{e}, + } + case linkBy > max: + e := fmt.Sprintf("linkby %v is more than max allowed of %v", + linkBy, max) + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidLinkBy, + ErrorContext: []string{e}, + } + } + return nil +} + +// validateAuthorizeVote validates the authorize vote fields. A UserError is +// returned if any of the validation fails. +func validateAuthorizeVote(av www.AuthorizeVote, u user.User, pr www.ProposalRecord, vs www.VoteSummary) error { + // Ensure the public key is the user's active key + if av.PublicKey != u.PublicKey() { + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidSigningKey, + } + } + + // Validate signature + msg := av.Token + pr.Version + av.Action + err := validateSignature(av.PublicKey, av.Signature, msg) + if err != nil { + return err + } + + // Verify record is in the right state and that the authorize + // vote request is valid. A vote authorization may already + // exist. We also allow vote authorizations to be revoked. + switch { + case pr.Status != www.PropStatusPublic: + // Record not public + return www.UserError{ + ErrorCode: www.ErrorStatusWrongStatus, + } + case vs.EndHeight != 0: + // Vote has already started + return www.UserError{ + ErrorCode: www.ErrorStatusWrongVoteStatus, + } + case av.Action != decredplugin.AuthVoteActionAuthorize && + av.Action != decredplugin.AuthVoteActionRevoke: + // Invalid authorize vote action + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidAuthVoteAction, + } + case av.Action == decredplugin.AuthVoteActionAuthorize && + vs.Status == www.PropVoteStatusAuthorized: + // Cannot authorize vote; vote has already been authorized + return www.UserError{ + ErrorCode: www.ErrorStatusVoteAlreadyAuthorized, + } + case av.Action == decredplugin.AuthVoteActionRevoke && + vs.Status != www.PropVoteStatusAuthorized: + // Cannot revoke authorization; vote has not been authorized + return www.UserError{ + ErrorCode: www.ErrorStatusVoteNotAuthorized, + } } - return true + return nil } -// processStartVoteRunoffV2 starts the runoff voting process on all public, -// non-abandoned RFP submissions for the provided RFP token. If politeiad fails -// to start the voting period on any of the RFP submissions, all work is -// unwound and an error is returned. -func (p *politeiawww) processStartVoteRunoffV2(sv www2.StartVoteRunoff, u *user.User) (*www2.StartVoteRunoffReply, error) { - log.Tracef("processStartVoteRunoffV2 %v", sv.Token) - - /* - // Sanity check - if !u.Admin { - return nil, fmt.Errorf("user is not an admin") - } - - bb, err := p.getBestBlock() +// validateAuthorizeVoteStandard validates the authorize vote for a proposal that +// is participating in a standard vote. A UserError is returned if any of the +// validation fails. +func validateAuthorizeVoteStandard(av www.AuthorizeVote, u user.User, pr www.ProposalRecord, vs www.VoteSummary) error { + err := validateAuthorizeVote(av, u, pr, vs) if err != nil { - return nil, err + return err } - // Ensure authorize votes and start votes match - auths := make(map[string]www2.AuthorizeVote, len(sv.AuthorizeVotes)) - starts := make(map[string]www2.StartVote, len(sv.StartVotes)) - for _, v := range sv.AuthorizeVotes { - auths[v.Token] = v - } - for _, v := range sv.StartVotes { - _, ok := auths[v.Vote.Token] - if !ok { - e := fmt.Sprintf("start vote found without matching authorize vote %v", - v.Vote.Token) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidRunoffVote, - ErrorContext: []string{e}, - } + // The rest of the validation is specific to authorize votes for + // standard votes. + switch { + case isRFPSubmission(pr): + return www.UserError{ + ErrorCode: www.ErrorStatusWrongProposalType, + ErrorContext: []string{"proposal is an rfp submission"}, } - } - for _, v := range sv.StartVotes { - starts[v.Vote.Token] = v - } - for _, v := range sv.AuthorizeVotes { - _, ok := starts[v.Token] - if !ok { - e := fmt.Sprintf("authorize vote found without matching start vote %v", - v.Token) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidRunoffVote, - ErrorContext: []string{e}, + case pr.PublicKey != av.PublicKey: + // User is not the author. First make sure the author didn't + // submit the proposal using an old identity. + if !isProposalAuthor(pr, u) { + return www.UserError{ + ErrorCode: www.ErrorStatusUserNotAuthor, } } } - if len(auths) == 0 { - e := "start votes and authorize votes cannot be empty" - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidRunoffVote, - ErrorContext: []string{e}, - } - } - - // Slice used to send event notification for each proposal - // starting a vote, at the end of this function. - var proposalNotifications []*www.ProposalRecord - // Validate authorize votes and start votes - for _, v := range sv.StartVotes { - // Fetch proposal and vote summary - token := v.Vote.Token - if !tokenIsValid(token) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: []string{token}, - } - } - pr, err := p.getProp(token) - if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - ErrorContext: []string{token}, - } - } - return nil, err - } + return nil +} - proposalNotifications = append(proposalNotifications, pr) +// validateAuthorizeVoteRunoff validates the authorize vote for a proposal that +// is participating in a runoff vote. A UserError is returned if any of the +// validation fails. +func validateAuthorizeVoteRunoff(av www.AuthorizeVote, u user.User, pr www.ProposalRecord, vs www.VoteSummary) error { + err := validateAuthorizeVote(av, u, pr, vs) + if err != nil { + return err + } - vs, err := p.voteSummaryGet(token, bb) - if err != nil { - return nil, err - } + // The rest of the validation is specific to authorize votes for + // runoff votes. + switch { + case !u.Admin: + // User is not an admin + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidSigningKey, + ErrorContext: []string{"user not an admin"}, + } + } - // Validate authorize vote. The validation function requires a v1 - // AuthorizeVote. This is fine. There is no difference between v1 - // and v2. - av := auths[v.Vote.Token] - av1 := www.AuthorizeVote{ - Token: av.Token, - Action: av.Action, - PublicKey: av.PublicKey, - Signature: av.Signature, - } - err = validateAuthorizeVoteRunoff(av1, *u, *pr, *vs) - if err != nil { - // Attach the token to the error so the user knows which one - // failed. - if ue, ok := err.(*www.UserError); ok { - ue.ErrorContext = append(ue.ErrorContext, token) - err = ue - } - return nil, err - } + return nil +} - // Validate start vote - err = validateStartVoteRunoff(v, *u, *pr, *vs, - p.cfg.VoteDurationMin, p.cfg.VoteDurationMax) - if err != nil { - // Attach the token to the error so the user knows which one - // failed. - if ue, ok := err.(*www.UserError); ok { - ue.ErrorContext = append(ue.ErrorContext, token) - err = ue - } - return nil, err - } +// validateVoteOptions verifies that the provided vote options +// specify a simple approve/reject vote and nothing else. A UserError is +// returned if this validation fails. +func validateVoteOptions(options []www2.VoteOption) error { + if len(options) == 0 { + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidVoteOptions, + ErrorContext: []string{"no vote options found"}, } - - // Validate the RFP proposal - rfp, err := p.getProp(sv.Token) - if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - ErrorContext: []string{sv.Token}, - } + } + optionIDs := map[string]bool{ + decredplugin.VoteOptionIDApprove: false, + decredplugin.VoteOptionIDReject: false, + } + for _, vo := range options { + if _, ok := optionIDs[vo.Id]; !ok { + e := fmt.Sprintf("invalid vote option id '%v'", vo.Id) + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidVoteOptions, + ErrorContext: []string{e}, } - return nil, err } - switch { - case rfp.LinkBy > time.Now().Unix() && !p.cfg.TestNet: - // Vote cannot start on RFP submissions until the RFP linkby - // deadline has been met. This validation is skipped when on - // testnet. - return nil, www.UserError{ - ErrorCode: www.ErrorStatusLinkByDeadlineNotMet, - } - case len(rfp.LinkedFrom) == 0: - return nil, www.UserError{ - ErrorCode: www.ErrorStatusNoLinkedProposals, + optionIDs[vo.Id] = true + } + for k, wasFound := range optionIDs { + if !wasFound { + e := fmt.Sprintf("missing vote option id '%v'", k) + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidVoteOptions, + ErrorContext: []string{e}, } } + } + return nil +} + +func validateStartVote(sv www2.StartVote, u user.User, pr www.ProposalRecord, vs www.VoteSummary, durationMin, durationMax uint32) error { + if !tokenIsValid(sv.Vote.Token) { + // Sanity check since proposal has already been looked up and + // passed in to this function. + return fmt.Errorf("invalid token %v", sv.Vote.Token) + } - // Compile a list of the public, non-abandoned RFP submissions. - // This list will be used to ensure a StartVote exists for each - // of the public, non-abandoned submissions. - linkedFromProps, err := p.getProps(rfp.LinkedFrom) + // Validate vote bits + for _, v := range sv.Vote.Options { + err := validateVoteBit(sv.Vote, v.Bits) if err != nil { - return nil, err - } - submissions := make(map[string]bool, len(rfp.LinkedFrom)) // [token]startVoteFound - for _, v := range linkedFromProps { - // Filter out abandoned submissions. These are not allowed - // to be included in a runoff vote. - if v.Status != www.PropStatusPublic { - continue + log.Debugf("validateStartVote: validateVoteBit '%v': %v", + v.Id, err) + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidPropVoteBits, } + } + } - // Set to false for now until we check that a StartVote - // was included for this proposal. - submissions[v.CensorshipRecord.Token] = false - } - - // Verify that a StartVote exists for all public, non-abandoned - // submissions and that there are no extra StartVotes. - for _, v := range sv.StartVotes { - _, ok := submissions[v.Vote.Token] - if !ok { - e := fmt.Sprintf("invalid start vote submission: %v", - v.Vote.Token) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidRunoffVote, - ErrorContext: []string{e}, - } - } + // Validate vote options. Only simple yes/no votes are currently + // allowed. + err := validateVoteOptions(sv.Vote.Options) + if err != nil { + return err + } - // A StartVote was included for this proposal - submissions[v.Vote.Token] = true - } - for token, startVoteFound := range submissions { - if !startVoteFound { - e := fmt.Sprintf("missing start vote for rfp submission: %v", - token) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidRunoffVote, - ErrorContext: []string{e}, - } - } + // Validate vote params + switch { + case sv.Vote.Duration < durationMin: + // Duration not large enough + e := fmt.Sprintf("vote duration must be >= %v", durationMin) + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidPropVoteParams, + ErrorContext: []string{e}, } - - // Setup plugin command - dav := convertAuthorizeVotesV2ToDecred(sv.AuthorizeVotes) - dsv := convertStartVotesV2ToDecred(sv.StartVotes) - payload, err := decredplugin.EncodeStartVoteRunoff( - decredplugin.StartVoteRunoff{ - Token: sv.Token, - AuthorizeVotes: dav, - StartVotes: dsv, - }) - if err != nil { - return nil, err + case sv.Vote.Duration > durationMax: + // Duration too large + e := fmt.Sprintf("vote duration must be <= %v", durationMax) + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidPropVoteParams, + ErrorContext: []string{e}, } - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err + case sv.Vote.QuorumPercentage > 100: + // Quorum too large + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidPropVoteParams, + ErrorContext: []string{"quorum percentage cannot be >100"}, } - pc := pd.PluginCommand{ - Challenge: hex.EncodeToString(challenge), - ID: decredplugin.ID, - Command: decredplugin.CmdStartVoteRunoff, - Payload: string(payload), + case sv.Vote.PassPercentage > 100: + // Pass percentage too large + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidPropVoteParams, + ErrorContext: []string{"pass percentage cannot be >100"}, } + } - // Send plugin command - responseBody, err := p.makeRequest(http.MethodPost, - pd.PluginCommandRoute, pc) - if err != nil { - return nil, err + // Ensure the public key is the user's active key + if sv.PublicKey != u.PublicKey() { + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidSigningKey, } + } - // Handle response - var reply pd.PluginCommandReply - err = json.Unmarshal(responseBody, &reply) - if err != nil { - return nil, err + // Validate signature + dsv := convertStartVoteV2ToDecred(sv) + err = dsv.VerifySignature() + if err != nil { + log.Debugf("validateStartVote: VerifySignature: %v", err) + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidSignature, } - err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) - if err != nil { - return nil, err + } + + // Validate proposal + votePropVersion := strconv.FormatUint(uint64(sv.Vote.ProposalVersion), 10) + switch { + case pr.Version != votePropVersion: + // Vote is specifying the wrong version + e := fmt.Sprintf("got %v, want %v", votePropVersion, pr.Version) + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidProposalVersion, + ErrorContext: []string{e}, } - dsvr, err := decredplugin.DecodeStartVoteRunoffReply([]byte(reply.Payload)) - if err != nil { - return nil, err + case pr.Status != www.PropStatusPublic: + // Proposal is not public + return www.UserError{ + ErrorCode: www.ErrorStatusWrongStatus, + ErrorContext: []string{"proposal is not public"}, } - svr, err := convertStartVoteReplyV2FromDecred(dsvr.StartVoteReply) - if err != nil { - return nil, err + case vs.EndHeight != 0: + // Vote has already started + return www.UserError{ + ErrorCode: www.ErrorStatusWrongVoteStatus, + ErrorContext: []string{"vote already started"}, } + } - // Emit event notification for each proposal starting vote - for _, pn := range proposalNotifications { - author, err := p.db.UserGetByPubKey(pn.PublicKey) - if err != nil { - return nil, err - } - p.eventManager.emit(eventProposalVoteStarted, - dataProposalVoteStarted{ - token: pn.CensorshipRecord.Token, - name: pn.Name, - adminID: u.ID.String(), - author: *author, - }) - } - - return &www2.StartVoteRunoffReply{ - StartBlockHeight: svr.StartBlockHeight, - StartBlockHash: svr.StartBlockHash, - EndBlockHeight: svr.EndBlockHeight, - EligibleTickets: svr.EligibleTickets, - }, nil - */ - - return nil, nil + return nil } -// processVoteDetailsV2 returns the vote details for the given proposal token. -func (p *politeiawww) processVoteDetailsV2(token string) (*www2.VoteDetailsReply, error) { - log.Tracef("processVoteDetailsV2: %v", token) +func (p *politeiawww) validateStartVoteStandard(sv www2.StartVote, u user.User, pr www.ProposalRecord, vs www.VoteSummary) error { + err := validateStartVote(sv, u, pr, vs, p.cfg.VoteDurationMin, + p.cfg.VoteDurationMax) + if err != nil { + return err + } - /* - // Validate vote status - dvdr, err := p.decredVoteDetails(token) - if err != nil { - if err == cache.ErrRecordNotFound { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - } - } - return nil, err + // The remaining validation is specific to a VoteTypeStandard. + switch { + case sv.Vote.Type != www2.VoteTypeStandard: + // Not a standard vote + e := fmt.Sprintf("vote type must be %v", www2.VoteTypeStandard) + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidVoteType, + ErrorContext: []string{e}, } - if dvdr.StartVoteReply.StartBlockHash == "" { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - ErrorContext: []string{"voting has not started yet"}, - } + case vs.Status != www.PropVoteStatusAuthorized: + // Vote has not been authorized + return www.UserError{ + ErrorCode: www.ErrorStatusWrongVoteStatus, + ErrorContext: []string{"vote not authorized"}, } - - // Handle StartVote versioning - var vdr *www2.VoteDetailsReply - switch dvdr.StartVote.Version { - case decredplugin.VersionStartVoteV1: - b := []byte(dvdr.StartVote.Payload) - dsv1, err := decredplugin.DecodeStartVoteV1(b) - if err != nil { - return nil, err - } - vdr, err = convertDecredStartVoteV1ToVoteDetailsReplyV2(*dsv1, - dvdr.StartVoteReply) - if err != nil { - return nil, err - } - case decredplugin.VersionStartVoteV2: - b := []byte(dvdr.StartVote.Payload) - dsv2, err := decredplugin.DecodeStartVoteV2(b) - if err != nil { - return nil, err - } - vdr, err = convertDecredStartVoteV2ToVoteDetailsReplyV2(*dsv2, - dvdr.StartVoteReply) - if err != nil { - return nil, err - } - - default: - return nil, fmt.Errorf("invalid StartVote version %v %v", - token, dvdr.StartVote.Version) + case isRFPSubmission(pr): + // The proposal is an an RFP submission. The voting period for + // RFP submissions can only be started using the StartVoteRunoff + // route. + return www.UserError{ + ErrorCode: www.ErrorStatusWrongProposalType, + ErrorContext: []string{"cannot be an rfp submission"}, } + } - return vdr, nil - */ + // Verify the LinkBy deadline for RFP proposals. The LinkBy policy + // requirements are enforced at the time of starting the vote + // because its purpose is to ensure that there is enough time for + // RFP submissions to be submitted. + if isRFP(pr) { + err := p.linkByValidate(pr.LinkBy) + if err != nil { + return err + } + } - return nil, nil + return nil } -// processTokenInventory returns the tokens of all proposals in the inventory, -// categorized by stage of the voting process. -func (p *politeiawww) processTokenInventory(isAdmin bool) (*www.TokenInventoryReply, error) { - log.Tracef("processTokenInventory") - - // Prep plugin command to get tokens by vote statuses - var vip piplugin.VoteInventory - payload, err := piplugin.EncodeVoteInventory(vip) +func validateStartVoteRunoff(sv www2.StartVote, u user.User, pr www.ProposalRecord, vs www.VoteSummary, durationMin, durationMax uint32) error { + err := validateStartVote(sv, u, pr, vs, durationMin, durationMax) if err != nil { - return nil, err + return err } - r, err := p.pluginCommand(piplugin.ID, piplugin.CmdVoteInventory, "", - string(payload)) - if err != nil { - return nil, err - } - vi, err := piplugin.DecodeVoteInventoryReply([]byte(r)) - if err != nil { - return nil, err - } + // The remaining validation is specific to a VoteTypeRunoff. - // Translate reply to www - res := www.TokenInventoryReply{ - Pre: append(vi.Unauthorized, vi.Authorized...), - Active: vi.Started, - Approved: vi.Approved, - Rejected: vi.Rejected, - } + token := sv.Vote.Token + switch { + case sv.Vote.Type != www2.VoteTypeRunoff: + // Not a runoff vote + e := fmt.Sprintf("%v vote type must be %v", + token, www2.VoteTypeRunoff) + return www.UserError{ + ErrorCode: www.ErrorStatusInvalidVoteType, + ErrorContext: []string{e}, + } - // Call politeiad to get tokens by record statuses - isReply, err := p.inventoryByStatus() - if err != nil { - return nil, err + case !isRFPSubmission(pr): + // The proposal is not an RFP submission + e := fmt.Sprintf("%v is not an rfp submission", token) + return www.UserError{ + ErrorCode: www.ErrorStatusWrongProposalType, + ErrorContext: []string{e}, + } + + case vs.Status != www.PropVoteStatusNotAuthorized: + // Sanity check. This should not be possible. + return fmt.Errorf("%v got vote status %v, want %v", + token, vs.Status, www.PropVoteStatusNotAuthorized) } - // Fill info - res.Abandoned = isReply.Archived + return nil +} - // Add admins only data - if isAdmin { - res.Censored = isReply.Censored - res.Unreviewed = isReply.Unvetted +func voteIsApproved(vs www.VoteSummary) bool { + if vs.Status != www.PropVoteStatusFinished { + // Vote has not ended yet + return false + } + + var ( + total uint64 + approve uint64 + ) + for _, v := range vs.Results { + total += v.VotesReceived + if v.Option.Id == decredplugin.VoteOptionIDApprove { + approve = v.VotesReceived + } + } + quorum := uint64(float64(vs.QuorumPercentage) / 100 * float64(vs.EligibleTickets)) + pass := uint64(float64(vs.PassPercentage) / 100 * float64(total)) + switch { + case total < quorum: + // Quorum not met + return false + case approve < pass: + // Pass percentage not met + return false } - return &res, nil + return true } +*/ diff --git a/politeiawww/testing.go b/politeiawww/testing.go index 4583807ce..796b5bb20 100644 --- a/politeiawww/testing.go +++ b/politeiawww/testing.go @@ -545,8 +545,8 @@ func newProposalRecord(t *testing.T, u *user.User, id *identity.FullIdentity, s // generates the token locally to make setting up tests easier. // The censorship record signature is left intentionally blank. return www.ProposalRecord{ - Name: name, - State: convertPropStatusToState(s), + Name: name, + // State: convertPropStatusToState(s), Status: s, Timestamp: time.Now().Unix(), UserId: u.ID.String(), diff --git a/politeiawww/ticketvote.go b/politeiawww/ticketvote.go index 23561bb3f..24aa2f015 100644 --- a/politeiawww/ticketvote.go +++ b/politeiawww/ticketvote.go @@ -6,191 +6,133 @@ package main import ( ticketvote "github.com/decred/politeia/plugins/ticketvote" - www "github.com/decred/politeia/politeiawww/api/www/v1" ) -func (p *politeiawww) voteBallot(vb ticketvote.Ballot) (*ticketvote.BallotReply, error) { - // Prep plugin command - payload, err := ticketvote.EncodeBallot(vb) +// voteAuthorize uses the ticketvote plugin to authorize a vote. +func (p *politeiawww) voteAuthorize(a ticketvote.Authorize) (*ticketvote.AuthorizeReply, error) { + b, err := ticketvote.EncodeAuthorize(a) if err != nil { return nil, err } - - r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdBallot, "", - string(payload)) + r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdAuthorize, string(b)) if err != nil { return nil, err } - vbr, err := ticketvote.DecodeBallotReply([]byte(r)) + va, err := ticketvote.DecodeAuthorizeReply([]byte(r)) if err != nil { return nil, err } - - return vbr, nil + return va, nil } -func (p *politeiawww) voteStartRunoff(vsr ticketvote.StartRunoff) (*ticketvote.StartRunoffReply, error) { - // Prep plugin command - payload, err := ticketvote.EncodeStartRunoff(vsr) +// voteStart uses the ticketvote plugin to start a vote. +func (p *politeiawww) voteStart(s ticketvote.Start) (*ticketvote.StartReply, error) { + b, err := ticketvote.EncodeStart(s) if err != nil { return nil, err } - - r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdStartRunoff, "", - string(payload)) + r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdStart, string(b)) if err != nil { return nil, err } - vsrr, err := ticketvote.DecodeStartRunoffReply([]byte(r)) + sr, err := ticketvote.DecodeStartReply([]byte(r)) if err != nil { return nil, err } - - return vsrr, nil + return sr, nil } -func (p *politeiawww) voteStart(vs ticketvote.Start) (*ticketvote.StartReply, error) { - // Prep plugin command - payload, err := ticketvote.EncodeStart(vs) +// voteStartRunoff uses the ticketvote plugin to start a runoff vote. +func (p *politeiawww) voteStartRunoff(sr ticketvote.StartRunoff) (*ticketvote.StartRunoffReply, error) { + b, err := ticketvote.EncodeStartRunoff(sr) if err != nil { return nil, err } - - r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdStart, "", - string(payload)) + r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdStartRunoff, + string(b)) if err != nil { return nil, err } - vsr, err := ticketvote.DecodeStartReply([]byte(r)) + srr, err := ticketvote.DecodeStartRunoffReply([]byte(r)) if err != nil { return nil, err } - - return vsr, nil + return srr, nil } -func (p *politeiawww) voteAuthorize(a ticketvote.Authorize) (*ticketvote.AuthorizeReply, error) { - // Prep plugin command - payload, err := ticketvote.EncodeAuthorize(a) +// voteBallot uses the ticketvote plugin to cast a ballot of votes. +func (p *politeiawww) voteBallot(tb ticketvote.Ballot) (*ticketvote.BallotReply, error) { + b, err := ticketvote.EncodeBallot(tb) if err != nil { return nil, err } - - r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdAuthorize, "", - string(payload)) + r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdBallot, string(b)) if err != nil { return nil, err } - va, err := ticketvote.DecodeAuthorizeReply([]byte(r)) + br, err := ticketvote.DecodeBallotReply([]byte(r)) if err != nil { return nil, err } - - return va, nil + return br, nil } -// voteDetails calls the ticketvote plugin command to get vote details. +// voteDetails uses the ticketvote plugin to fetch the details of a vote. func (p *politeiawww) voteDetails(tokens []string) (*ticketvote.DetailsReply, error) { - // Prep vote details payload - vdp := ticketvote.Details{ + d := ticketvote.Details{ Tokens: tokens, } - payload, err := ticketvote.EncodeDetails(vdp) + b, err := ticketvote.EncodeDetails(d) if err != nil { return nil, err } - - r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdDetails, "", - string(payload)) + r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdDetails, string(b)) if err != nil { return nil, err } - vd, err := ticketvote.DecodeDetailsReply([]byte(r)) + dr, err := ticketvote.DecodeDetailsReply([]byte(r)) if err != nil { return nil, err } - - return vd, nil + return dr, nil } -// castVotes calls the ticketvote plugin to retrieve cast votes. +// castVotes uses the ticketvote plugin to fetch cast votes for a record. func (p *politeiawww) castVotes(token string) (*ticketvote.CastVotesReply, error) { - // Prep cast votes payload - csp := ticketvote.CastVotes{ + cv := ticketvote.CastVotes{ Token: token, } - payload, err := ticketvote.EncodeCastVotes(csp) + b, err := ticketvote.EncodeCastVotes(cv) if err != nil { return nil, err } - - r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdCastVotes, "", - string(payload)) + r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdCastVotes, string(b)) if err != nil { return nil, err } - cv, err := ticketvote.DecodeCastVotesReply([]byte(r)) + cvr, err := ticketvote.DecodeCastVotesReply([]byte(r)) if err != nil { return nil, err } - - return cv, nil + return cvr, nil } -// ballot calls the ticketvote plugin to cast a ballot of votes. -func (p *politeiawww) ballot(ballot *www.Ballot) (*ticketvote.BallotReply, error) { - // Prep plugin command - var bp ticketvote.Ballot - - // Transale votes - votes := make([]ticketvote.CastVote, 0, len(ballot.Votes)) - for _, vote := range ballot.Votes { - votes = append(votes, ticketvote.CastVote{ - Token: vote.Ticket, - Ticket: vote.Ticket, - VoteBit: vote.VoteBit, - Signature: vote.Signature, - }) - } - bp.Votes = votes - payload, err := ticketvote.EncodeBallot(bp) - if err != nil { - return nil, err - } - - r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdBallot, "", - string(payload)) - if err != nil { - return nil, err - } - b, err := ticketvote.DecodeBallotReply([]byte(r)) - if err != nil { - return nil, err - } - - return b, nil -} - -// summaries calls the ticketvote plugin to get vote summary information. +// voteSummaries uses the ticketvote plugin to fetch vote summaries. func (p *politeiawww) voteSummaries(tokens []string) (*ticketvote.SummariesReply, error) { - // Prep plugin command - smp := ticketvote.Summaries{ + s := ticketvote.Summaries{ Tokens: tokens, } - payload, err := ticketvote.EncodeSummaries(smp) + b, err := ticketvote.EncodeSummaries(s) if err != nil { return nil, err } - - r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdSummaries, "", - string(payload)) + r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdSummaries, string(b)) if err != nil { return nil, err } - sm, err := ticketvote.DecodeSummariesReply([]byte(r)) + sr, err := ticketvote.DecodeSummariesReply([]byte(r)) if err != nil { return nil, err } - - return sm, nil + return sr, nil } diff --git a/politeiawww/userwww.go b/politeiawww/userwww.go index 42a189716..0902937f6 100644 --- a/politeiawww/userwww.go +++ b/politeiawww/userwww.go @@ -16,9 +16,10 @@ import ( "github.com/gorilla/mux" ) -// handleNewUser handles the incoming new user command. It verifies that the new user -// doesn't already exist, and then creates a new user in the db and generates a random -// code used for verification. The code is intended to be sent to the specified email. +// handleNewUser handles the incoming new user command. It verifies that the +// new user doesn't already exist, and then creates a new user in the db and +// generates a random code used for verification. The code is intended to be +// sent to the specified email. func (p *politeiawww) handleNewUser(w http.ResponseWriter, r *http.Request) { log.Tracef("handleNewUser") @@ -42,9 +43,10 @@ func (p *politeiawww) handleNewUser(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, reply) } -// handleVerifyNewUser handles the incoming new user verify command. It verifies -// that the user with the provided email has a verification token that matches -// the provided token and that the verification token has not yet expired. +// handleVerifyNewUser handles the incoming new user verify command. It +// verifies that the user with the provided email has a verification token that +// matches the provided token and that the verification token has not yet +// expired. func (p *politeiawww) handleVerifyNewUser(w http.ResponseWriter, r *http.Request) { log.Tracef("handleVerifyNewUser") diff --git a/politeiawww/www.go b/politeiawww/www.go index 38b89d163..9c0c8fe9d 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -557,7 +557,7 @@ func (p *politeiawww) setupPi() error { // Setup routes p.setPoliteiaWWWRoutes() p.setUserWWWRoutes() - p.setPiRoutes() + p.setupPiRoutes() // Verify paywall settings switch { From fd9bfd69099f30064cf585988afeafca55e43574 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 30 Sep 2020 09:29:00 -0500 Subject: [PATCH 120/449] tlog bug fixes and doco --- politeiad/backend/tlogbe/anchor.go | 7 + politeiad/backend/tlogbe/pluginclient.go | 34 ++- politeiad/backend/tlogbe/ticketvote.go | 4 +- politeiad/backend/tlogbe/tlog.go | 257 ++++++++++++++--------- politeiad/backend/tlogbe/tlogbe.go | 119 ++++++++--- politeiad/cmd/politeia/README.md | 33 ++- politeiad/cmd/politeia/politeia.go | 2 +- politeiawww/piwww.go | 58 ++--- politeiawww/plugin.go | 12 +- politeiawww/www.go | 2 +- 10 files changed, 350 insertions(+), 178 deletions(-) diff --git a/politeiad/backend/tlogbe/anchor.go b/politeiad/backend/tlogbe/anchor.go index 66f757a60..9ea6d508f 100644 --- a/politeiad/backend/tlogbe/anchor.go +++ b/politeiad/backend/tlogbe/anchor.go @@ -7,6 +7,7 @@ package tlogbe import ( "crypto/sha256" "encoding/hex" + "errors" "fmt" "time" @@ -105,6 +106,12 @@ func (t *tlog) anchorSave(a anchor) error { return nil } +var ( + // errAnchorNotFound is emitted when a anchor is not found when + // requesting the anchor record from a tree. + errAnchorNotFound = errors.New("anchor not found") +) + // anchorLatest returns the most recent anchor for the provided tree. A // errAnchorNotFound is returned if no anchor is found for the provided tree. func (t *tlog) anchorLatest(treeID int64) (*anchor, error) { diff --git a/politeiad/backend/tlogbe/pluginclient.go b/politeiad/backend/tlogbe/pluginclient.go index 6379d2699..5b373c919 100644 --- a/politeiad/backend/tlogbe/pluginclient.go +++ b/politeiad/backend/tlogbe/pluginclient.go @@ -41,9 +41,14 @@ var ( // hookNewRecord is the payload for the new record hooks. type hookNewRecord struct { - RecordMetadata backend.RecordMetadata `json:"recordmetadata"` - Metadata []backend.MetadataStream `json:"metadata"` - Files []backend.File `json:"files"` + Metadata []backend.MetadataStream `json:"metadata"` + Files []backend.File `json:"files"` + + // RecordMetadata will only be present on the post new record hook. + // This is because the record metadata requires the creation of a + // trillian tree and the pre new record hook should execute before + // any politeiad state is changed in case of validation errors. + RecordMetadata *backend.RecordMetadata `json:"recordmetadata"` } func encodeHookNewRecord(hnr hookNewRecord) ([]byte, error) { @@ -85,6 +90,29 @@ func decodeHookEditRecord(payload []byte) (*hookEditRecord, error) { return &her, nil } +// hookEditMetadata is the payload for the edit metadata hooks. +type hookEditMetadata struct { + // Current record + Current backend.Record `json:"record"` + + // Updated fields + MDAppend []backend.MetadataStream `json:"mdappend"` + MDOverwrite []backend.MetadataStream `json:"mdoverwrite"` +} + +func encodeHookEditMetadata(hem hookEditMetadata) ([]byte, error) { + return json.Marshal(hem) +} + +func decodeHookEditMetadata(payload []byte) (*hookEditMetadata, error) { + var hem hookEditMetadata + err := json.Unmarshal(payload, &hem) + if err != nil { + return nil, err + } + return &hem, nil +} + // hookSetRecordStatus is the payload for the set record status hooks. type hookSetRecordStatus struct { // Current record diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index a44f6f565..115860519 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -58,7 +58,9 @@ var ( ) // TODO holding the lock before verifying the token can allow the mutexes to -// be spammed. Create an infinite amount of them with invalid tokens. +// be spammed. Create an infinite amount of them with invalid tokens. The fix +// is to add an exists() method onto the tlogClient and have the mutexes +// function ensure a token is valid before holding the lock on it. // ticketVotePlugin satisfies the pluginClient interface. type ticketVotePlugin struct { diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index f665a70e1..f932a5ec3 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -14,6 +14,7 @@ import ( "fmt" "os" "path/filepath" + "sort" "strconv" "sync" @@ -74,10 +75,6 @@ var ( // errTreeIsFrozen is emitted when a frozen tree is attempted to be // altered. errTreeIsFrozen = errors.New("tree is frozen") - - // errAnchorNotFound is emitted when a anchor is not found in a - // tree. - errAnchorNotFound = errors.New("anchor not found") ) // We do not unwind. @@ -98,14 +95,27 @@ type tlog struct { // recordIndex contains the merkle leaf hashes of all the record content for a // specific record version and iteration. The record index can be used to -// lookup the trillian log leaves for the record content. The ExtraData field -// of each log leaf contains the key-value store key for the record content -// data blob. +// lookup the trillian log leaves for the record content and the log leaves can +// be used to lookup the kv store blobs. +// +// A record is updated in three steps: +// +// 1. Record content blobs are saved to the kv store. +// +// 2. The kv store keys are stuffed into the LogLeaf.ExtraData field and the +// log leaves are appended onto the trillian tree. // -// Appending the record index leaf to the trillian tree is the last operation -// that occurs when updating a record, so if a record index leaf exists then -// the record update is considered valid and you can be sure that the data -// blobs were successfully saved to the key-value store. +// 3. If there failures in steps 1 or 2 for any of the blobs then the update +// will exit without completing. No unwinding is performed. Blobs will be +// left in the kv store as orphaned blobs. The trillian tree is append only +// so once a leaf is appended, it is there permanently. If steps 1 and 2 are +// successful then a recordIndex will be created, saved to the kv store, and +// appended onto the trillian tree. +// +// Appending a recordIndex onto the trillian tree is the last operation that +// occurs during a record update. If a recordIndex exists in the tree then the +// update is considered successful. Any record content leaves that are not part +// of a recordIndex are considered to be orphaned and can be disregarded. type recordIndex struct { // Version represents the version of the record. The version is // only incremented when the record files are updated. @@ -129,6 +139,11 @@ type recordIndex struct { Files map[string][]byte `json:"files"` // [filename]merkle } +// TODO add comment explaining what a freeze record is +type freezeRecord struct { + TreeID int64 `json:"treeid,omitempty"` +} + func treeIDFromToken(token []byte) int64 { return int64(binary.LittleEndian.Uint64(token)) } @@ -421,10 +436,6 @@ func (t *tlog) treeExists(treeID int64) bool { return false } -type freezeRecord struct { - TreeID int64 `json:"treeid,omitempty"` -} - // treeFreeze updates the status of a record and freezes the trillian tree as a // result of a record status change. Once a freeze record has been appended // onto the tree the tlog backend considers the tree to be frozen. The only @@ -442,13 +453,8 @@ func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []ba } // Prepare kv store blobs for metadata - args := recordBlobsPrepareArgs{ - encryptionKey: t.encryptionKey, - leaves: leavesAll, - recordMD: rm, - metadata: metadata, - } - rbpr, err := recordBlobsPrepare(args) + rbpr, err := recordBlobsPrepare(leavesAll, rm, metadata, + []backend.File{}, t.encryptionKey) if err != nil { return err } @@ -660,10 +666,10 @@ func (t *tlog) recordIndexSave(treeID int64, ri recordIndex) error { if err != nil { return err } - prefixedKey := []byte(keyPrefixRecordIndex + keys[0]) - queued, _, err := t.trillian.leavesAppend(treeID, []*trillian.LogLeaf{ - logLeafNew(h, prefixedKey), - }) + leaves := []*trillian.LogLeaf{ + logLeafNew(h, []byte(keyPrefixRecordIndex+keys[0])), + } + queued, _, err := t.trillian.leavesAppend(treeID, leaves) if len(queued) != 1 { return fmt.Errorf("wrong number of queud leaves: got %v, want 1", len(queued)) @@ -769,6 +775,12 @@ func (t *tlog) recordIndexes(leaves []*trillian.LogLeaf) ([]recordIndex, error) indexes = append(indexes, *ri) } + // Sort indexes by iteration, smallest to largets. The leaves + // ordering was not preserved in the returned blobs map. + sort.SliceStable(indexes, func(i, j int) bool { + return indexes[i].Iteration < indexes[j].Iteration + }) + // Sanity check. Index iterations should start with 1 and be // sequential. Index versions should start with 1 and also be // sequential, but duplicate versions can exist as long as the @@ -799,27 +811,36 @@ type recordHashes struct { files map[string]string // [hash]filename } -type recordBlobsPrepareArgs struct { - encryptionKey *encryptionKey - leaves []*trillian.LogLeaf - recordMD backend.RecordMetadata - metadata []backend.MetadataStream - files []backend.File -} - type recordBlobsPrepareReply struct { - recordIndex recordIndex + // recordIndex is the index for the record content. It is created + // during the blobs prepare step so that it can be populated with + // the merkle leaf hashes of duplicate data, i.e. data that remains + // unchanged between two versions of a record. It will be fully + // populated once the unique blobs haves been saved to the kv store + // and appended onto the trillian tree. + recordIndex recordIndex + + // recordHashes contains a mapping of the record content hashes to + // the record content type. This is used to populate the record + // index once the leaves have been appended onto the trillian tree. recordHashes recordHashes - // blobs and hashes MUST share the same ordering + // blobs contains the blobified record content that needs to be + // saved to the kv store. Hashes contains the hashes of the record + // content prior to being blobified. + // + // blobs and hashes MUST share the same ordering. blobs [][]byte hashes [][]byte } +// recordBlobsPrepare prepares the provided record content to be saved to +// the blob kv store and appended onto a trillian tree. +// // TODO test this function -func recordBlobsPrepare(args recordBlobsPrepareArgs) (*recordBlobsPrepareReply, error) { - // Ensure tree is not frozen - if treeIsFrozen(args.leaves) { +func recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File, encryptionKey *encryptionKey) (*recordBlobsPrepareReply, error) { + // Verify tree state + if treeIsFrozen(leavesAll) { return nil, errTreeIsFrozen } @@ -832,15 +853,15 @@ func recordBlobsPrepare(args recordBlobsPrepareArgs) (*recordBlobsPrepareReply, // Compute record content hashes rhashes := recordHashes{ - metadata: make(map[string]uint64, len(args.metadata)), - files: make(map[string]string, len(args.files)), + metadata: make(map[string]uint64, len(metadata)), + files: make(map[string]string, len(files)), } - b, err := json.Marshal(args.recordMD) + b, err := json.Marshal(recordMD) if err != nil { return nil, err } rhashes.recordMetadata = hex.EncodeToString(util.Digest(b)) - for _, v := range args.metadata { + for _, v := range metadata { b, err := json.Marshal(v) if err != nil { return nil, err @@ -848,7 +869,7 @@ func recordBlobsPrepare(args recordBlobsPrepareArgs) (*recordBlobsPrepareReply, h := hex.EncodeToString(util.Digest(b)) rhashes.metadata[h] = v.ID } - for _, v := range args.files { + for _, v := range files { b, err := json.Marshal(v) if err != nil { return nil, err @@ -860,17 +881,17 @@ func recordBlobsPrepare(args recordBlobsPrepareArgs) (*recordBlobsPrepareReply, // Compare leaf data to record content hashes to find duplicates var ( // Dups tracks duplicates so we know which blobs should be - // skipped when saving the blob to the store. + // skipped when blobifying record content. dups = make(map[string]struct{}, 64) // Any duplicates that are found are added to the record index // since we already have the leaf data for them. index = recordIndex{ - Metadata: make(map[uint64][]byte, len(args.metadata)), - Files: make(map[string][]byte, len(args.files)), + Metadata: make(map[uint64][]byte, len(metadata)), + Files: make(map[string][]byte, len(files)), } ) - for _, v := range args.leaves { + for _, v := range leavesAll { h := hex.EncodeToString(v.LeafValue) // Check record metadata @@ -900,10 +921,12 @@ func recordBlobsPrepare(args recordBlobsPrepareArgs) (*recordBlobsPrepareReply, // Prepare kv store blobs. The hashes of the record content are // also aggregated and will be used to create the log leaves that // are appended to the trillian tree. - l := len(args.metadata) + len(args.files) + 1 + l := len(metadata) + len(files) + 1 hashes := make([][]byte, 0, l) blobs := make([][]byte, 0, l) - be, err := convertBlobEntryFromRecordMetadata(args.recordMD) + + // Prepare record metadata blob + be, err := convertBlobEntryFromRecordMetadata(recordMD) if err != nil { return nil, err } @@ -922,7 +945,8 @@ func recordBlobsPrepare(args recordBlobsPrepareArgs) (*recordBlobsPrepareReply, blobs = append(blobs, b) } - for _, v := range args.metadata { + // Prepare metadata blobs + for _, v := range metadata { be, err := convertBlobEntryFromMetadataStream(v) if err != nil { return nil, err @@ -943,7 +967,8 @@ func recordBlobsPrepare(args recordBlobsPrepareArgs) (*recordBlobsPrepareReply, } } - for _, v := range args.files { + // Prepare file blobs + for _, v := range files { be, err := convertBlobEntryFromFile(v) if err != nil { return nil, err @@ -957,8 +982,8 @@ func recordBlobsPrepare(args recordBlobsPrepareArgs) (*recordBlobsPrepareReply, return nil, err } // Encypt file blobs if encryption key has been set - if args.encryptionKey != nil { - b, err = args.encryptionKey.encrypt(0, b) + if encryptionKey != nil { + b, err = encryptionKey.encrypt(0, b) if err != nil { return nil, err } @@ -979,7 +1004,12 @@ func recordBlobsPrepare(args recordBlobsPrepareArgs) (*recordBlobsPrepareReply, }, nil } +// recordBlobsSave saves the provided blobs to the kv store, appends a leaf +// to the trillian tree for each blob, then updates the record index with the +// trillian leaf information and returns it. func (t *tlog) recordBlobsSave(treeID int64, rbpr recordBlobsPrepareReply) (*recordIndex, error) { + log.Tracef("recordBlobsSave: %v", treeID) + var ( index = rbpr.recordIndex rhashes = rbpr.recordHashes @@ -1058,11 +1088,14 @@ func (t *tlog) recordBlobsSave(treeID int64, rbpr recordBlobsPrepareReply) (*rec return &index, nil } -// We do not unwind. +// recordSave saves the provided record to tlog, creating a new version of the +// record. Once the record contents have been successfully saved to tlog, a +// recordIndex is created for this version of the record and saved to tlog as +// well. func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { log.Tracef("tlog recordSave: %v", treeID) - // Ensure tree exists + // Verify tree exists if !t.treeExists(treeID) { return errRecordNotFound } @@ -1073,26 +1106,69 @@ func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []ba return fmt.Errorf("leavesAll %v: %v", treeID, err) } - // Ensure tree is not frozen + // Verify tree state if treeIsFrozen(leavesAll) { return errTreeIsFrozen } - // Prepare kv store blobs - args := recordBlobsPrepareArgs{ - encryptionKey: t.encryptionKey, - leaves: leavesAll, - recordMD: rm, - metadata: metadata, - files: files, + // Get the existing record index + currIdx, err := t.recordIndexLatest(leavesAll) + if err == errRecordNotFound { + // No record versions exist yet. This is ok. + currIdx = &recordIndex{ + Metadata: make(map[uint64][]byte), + Files: make(map[string][]byte), + } + } else if err != nil { + return fmt.Errorf("recordIndexLatest: %v", err) } - rbpr, err := recordBlobsPrepare(args) + + // Prepare kv store blobs + rbpr, err := recordBlobsPrepare(leavesAll, rm, metadata, + files, t.encryptionKey) if err != nil { return err } - // Ensure file changes are being made - if len(rbpr.recordIndex.Files) == len(files) { + // Verify file changes are being made. + var fileChanges bool + for _, v := range files { + // Duplicate blobs have already been added to the new record + // index by the recordBlobsPrepare function. If a file is in the + // new record index it means that the file has existed in one of + // the previous versions of the record. + newMerkle, ok := rbpr.recordIndex.Files[v.Name] + if !ok { + // File does not exist in the index. It is new. + fileChanges = true + break + } + + // We now know the file has existed in a previous version of the + // record, but it may not have be the most recent version. If the + // file is not part of the current record index then it means + // there are file changes between the current version and new + // version. + currMerkle, ok := currIdx.Files[v.Name] + if !ok { + // File is not part of the current version. + fileChanges = true + break + } + + // We now know that the new file has existed in some previous + // version of the record and the there is a file in the current + // version of the record that has the same filename as the new + // file. Check if the merkles match. If the merkles are different + // then it means the files are different, they just use the same + // filename. + if !bytes.Equal(newMerkle, currMerkle) { + // Files share the same name but have different content. + fileChanges = true + break + } + } + if !fileChanges { return errNoFileChanges } @@ -1102,28 +1178,18 @@ func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []ba return fmt.Errorf("blobsSave: %v", err) } - // Get the existing record index and use it to bump the version and - // iteration of the new record index. - oldIdx, err := t.recordIndexLatest(leavesAll) - if err == errRecordNotFound { - // No record versions exist yet. This is fine. The version and - // iteration will be incremented to 1. - oldIdx = &recordIndex{} - } else if err != nil { - return fmt.Errorf("recordIndexLatest: %v", err) - } - idx.Version = oldIdx.Version + 1 - idx.Iteration = oldIdx.Iteration + 1 + // Bump the index version and iteration + idx.Version = currIdx.Version + 1 + idx.Iteration = currIdx.Iteration + 1 - // Sanity check. The record index should be fully populated at this - // point. + // Sanity checks switch { - case idx.Version != oldIdx.Version+1: + case idx.Version != currIdx.Version+1: return fmt.Errorf("invalid index version: got %v, want %v", - idx.Version, oldIdx.Version+1) - case idx.Iteration != oldIdx.Iteration+1: + idx.Version, currIdx.Version+1) + case idx.Iteration != currIdx.Iteration+1: return fmt.Errorf("invalid index iteration: got %v, want %v", - idx.Iteration, oldIdx.Iteration+1) + idx.Iteration, currIdx.Iteration+1) case idx.RecordMetadata == nil: return fmt.Errorf("invalid index record metadata") case len(idx.Metadata) != len(metadata): @@ -1146,7 +1212,7 @@ func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []ba func (t *tlog) recordMetadataUpdate(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { log.Tracef("tlog recordMetadataUpdate: %v", treeID) - // Ensure tree exists + // Verify tree exists if !t.treeExists(treeID) { return errRecordNotFound } @@ -1157,24 +1223,19 @@ func (t *tlog) recordMetadataUpdate(treeID int64, rm backend.RecordMetadata, met return fmt.Errorf("leavesAll: %v", err) } - // Ensure tree is not frozen + // Verify tree state if treeIsFrozen(leavesAll) { return errTreeIsFrozen } // Prepare kv store blobs - args := recordBlobsPrepareArgs{ - encryptionKey: t.encryptionKey, - leaves: leavesAll, - recordMD: rm, - metadata: metadata, - } - bpr, err := recordBlobsPrepare(args) + bpr, err := recordBlobsPrepare(leavesAll, rm, metadata, + []backend.File{}, t.encryptionKey) if err != nil { return err } - // Ensure metadata has been changed + // Verify changes are being made if len(bpr.blobs) == 0 { return errNoMetadataChanges } @@ -1231,7 +1292,7 @@ func (t *tlog) recordMetadataUpdate(treeID int64, rm backend.RecordMetadata, met func (t *tlog) recordDel(treeID int64) error { log.Tracef("tlog recordDel: %v", treeID) - // Ensure tree exists + // Verify tree exists if !t.treeExists(treeID) { return errRecordNotFound } @@ -1287,7 +1348,7 @@ func (t *tlog) recordDel(treeID int64) error { func (t *tlog) record(treeID int64, version uint32) (*backend.Record, error) { log.Tracef("tlog record: %v %v", treeID, version) - // Ensure tree exists + // Verify tree exists if !t.treeExists(treeID) { return nil, errRecordNotFound } @@ -1552,7 +1613,7 @@ func (t *tlog) blobsSave(treeID int64, keyPrefix string, blobs, hashes [][]byte, func (t *tlog) blobsDel(treeID int64, merkles [][]byte) error { log.Tracef("tlog blobsDel: %v", treeID) - // Ensure tree exists. We allow blobs to be deleted from both + // Verify tree exists. We allow blobs to be deleted from both // frozen and non frozen trees. if !t.treeExists(treeID) { return errRecordNotFound diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 4642ea46f..bbb78ef48 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -247,13 +247,16 @@ func (t *tlogBackend) inventoryUpdate(token string, currStatus, newStatus backen // Find the index of the token in its current status list var idx int + var found bool for k, v := range t.inventory[currStatus] { if v == token { // Token found idx = k + found = true + break } } - if idx == 0 { + if !found { // Token was never found. This should not happen. e := fmt.Sprintf("inventoryUpdate: token not found: %v %v %v", token, currStatus, newStatus) @@ -521,7 +524,7 @@ func metadataStreamsUpdate(mdCurr, mdAppend, mdOverwrite []backend.MetadataStrea } // Convert metadata back to a slice - metadata := make([]backend.MetadataStream, len(md)) + metadata := make([]backend.MetadataStream, 0, len(md)) for _, v := range md { metadata = append(metadata, v) } @@ -541,6 +544,20 @@ func (t *tlogBackend) New(metadata []backend.MetadataStream, files []backend.Fil return nil, err } + // Call pre plugin hooks + hnr := hookNewRecord{ + Metadata: metadata, + Files: files, + } + b, err := encodeHookNewRecord(hnr) + if err != nil { + return nil, err + } + err = t.pluginHook(hookNewRecordPre, string(b)) + if err != nil { + return nil, err + } + // Create a new token var token []byte var treeID int64 @@ -573,21 +590,6 @@ func (t *tlogBackend) New(metadata []backend.MetadataStream, files []backend.Fil return nil, err } - // Call pre plugin hooks - hnr := hookNewRecord{ - RecordMetadata: *rm, - Metadata: metadata, - Files: files, - } - b, err := encodeHookNewRecord(hnr) - if err != nil { - return nil, err - } - err = t.pluginHook(hookNewRecordPre, string(b)) - if err != nil { - return nil, err - } - // Save the record err = t.unvetted.recordSave(treeID, *rm, metadata, files) if err != nil { @@ -595,9 +597,18 @@ func (t *tlogBackend) New(metadata []backend.MetadataStream, files []backend.Fil } // Call post plugin hooks + hnr = hookNewRecord{ + Metadata: metadata, + Files: files, + RecordMetadata: rm, + } + b, err = encodeHookNewRecord(hnr) + if err != nil { + return nil, err + } err = t.pluginHook(hookNewRecordPost, string(b)) if err != nil { - log.Errorf("New: pluginHook %v: %v", hooks[hookNewRecordPost], err) + log.Errorf("New %x: pluginHook newRecordPost: %v", token, err) } // Update the inventory cache @@ -688,15 +699,15 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ // Call post plugin hooks err = t.pluginHook(hookEditRecordPost, string(b)) if err != nil { - log.Errorf("pluginHook %v: %v", hooks[hookEditRecordPost], err) + log.Errorf("UpdateUnvettedRecord %x: pluginHook editRecordPost: %v", + token, err) } // Update inventory cache. The inventory will only need to be // updated if there was a status transition. if r.RecordMetadata.Status != recordMD.Status { // Status was changed - t.inventoryUpdate(recordMD.Token, r.RecordMetadata.Status, - recordMD.Status) + t.inventoryUpdate(recordMD.Token, r.RecordMetadata.Status, recordMD.Status) } // Return updated record @@ -793,7 +804,8 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b // Call post plugin hooks err = t.pluginHook(hookEditRecordPost, string(b)) if err != nil { - log.Errorf("pluginHook %v: %v", hooks[hookEditRecordPost], err) + log.Errorf("UpdateVettedRecord %x: pluginHook editRecordPost: %v", + token, err) } // Return updated record @@ -847,15 +859,40 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite return fmt.Errorf("recordLatest: %v", err) } + // Call pre plugin hooks + hem := hookEditMetadata{ + Current: *r, + MDAppend: mdAppend, + MDOverwrite: mdOverwrite, + } + b, err := encodeHookEditMetadata(hem) + if err != nil { + return err + } + err = t.pluginHook(hookEditMetadataPre, string(b)) + if err != nil { + return err + } + // Apply changes metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) // Update metadata err = t.unvetted.recordMetadataUpdate(treeID, r.RecordMetadata, metadata) if err != nil { + if err == errNoMetadataChanges { + return backend.ErrNoChanges + } return err } + // Call post plugin hooks + err = t.pluginHook(hookEditMetadataPost, string(b)) + if err != nil { + log.Errorf("UpdateUnvettedMetadata %x: pluginHook editMetadataPost: %v", + token, err) + } + return nil } @@ -908,15 +945,40 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ return fmt.Errorf("recordLatest: %v", err) } + // Call pre plugin hooks + hem := hookEditMetadata{ + Current: *r, + MDAppend: mdAppend, + MDOverwrite: mdOverwrite, + } + b, err := encodeHookEditMetadata(hem) + if err != nil { + return err + } + err = t.pluginHook(hookEditMetadataPre, string(b)) + if err != nil { + return err + } + // Apply changes metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) // Update metadata err = t.vetted.recordMetadataUpdate(treeID, r.RecordMetadata, metadata) if err != nil { + if err == errNoMetadataChanges { + return backend.ErrNoChanges + } return err } + // Call post plugin hooks + err = t.pluginHook(hookEditMetadataPost, string(b)) + if err != nil { + log.Errorf("UpdateVettedMetadata %x: pluginHook editMetadataPost: %v", + token, err) + } + return nil } @@ -1191,8 +1253,8 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, // Call post plugin hooks err = t.pluginHook(hookSetRecordStatusPost, string(b)) if err != nil { - log.Errorf("SetUnvettedStatus: pluginHook %v: %v", - hooks[hookSetRecordStatusPost], err) + log.Errorf("SetUnvettedStatus %x: pluginHook setRecordStatusPost: %v", + token, err) } // Update inventory cache @@ -1316,8 +1378,8 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md // Call post plugin hooks err = t.pluginHook(hookSetRecordStatusPost, string(b)) if err != nil { - log.Errorf("SetVettedStatus: pluginHook %v: %v", - hooks[hookSetRecordStatusPost], err) + log.Errorf("SetVettedStatus %x: pluginHook setRecordStatusPost: %v", + token, err) } // Update inventory cache @@ -1342,9 +1404,8 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md // records individually. // // This function satisfies the Backend interface. -func (t *tlogBackend) Inventory(vettedCount uint, unvettedCount uint, includeFiles, allVersions bool) ([]backend.Record, []backend.Record, error) { - log.Tracef("Inventory: %v %v", includeFiles, allVersions) - +func (t *tlogBackend) Inventory(vettedCount, vettedStart, unvettedCount uint, includeFiles, allVersions bool) ([]backend.Record, []backend.Record, error) { + log.Tracef("Inventory") return nil, nil, fmt.Errorf("not implemented") } diff --git a/politeiad/cmd/politeia/README.md b/politeiad/cmd/politeia/README.md index 62c78c5db..22f5e3316 100644 --- a/politeiad/cmd/politeia/README.md +++ b/politeiad/cmd/politeia/README.md @@ -22,8 +22,9 @@ the string is record metadata. Arguments that are not classified as metadata are assumed to be file paths. ``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass new \ - 'metadata12:{"moo":"lala"}' 'metadata2:{"foo":"bar"}' ~/index.md +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass new \ + 'metadata12:{"moo":"lala"}' 'metadata2:{"foo":"bar"}' index.md + 00: 4bde9f923b61e26147c79500e6d6dfa27291559a74cd878c29a7f96984dd48bb index.md text/plain; charset=utf-8 Record submitted Censorship record: @@ -36,6 +37,7 @@ Record submitted ``` $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass getunvetted 0e4a82a370228b710000 + Unvetted record: Status : not reviewed Timestamp : 2020-09-28 14:20:14 +0000 UTC @@ -54,21 +56,25 @@ Unvetted record: ## Update an unvetted record -Files can be updated using the arguments: -- `add:[filepath]` -- `del:[filename]` +Metadata can be updated using the arguments: +`'appendmetadata[ID]:[metadataJSON]'` +`'overwritemetadata[ID]:[metadataJSON]'` + +Files can be updated using the arguments: +`add:[filepath]` +`del:[filename]` -Metadata can be updated using the arguments: -- `'appendmetadata[ID]:[metadataJSON]'` -- `'overwritemetadata[ID]:[metadataJSON]'` +The token is specified using the argument: +`token:[token]` Metadata provided using the `overwritemetadata` argument does not have to -already exist. The token argument should be prefixed with `token:`. +already exist. ``` $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass updateunvetted \ 'appendmetadata12:{"foo":"bar"}' 'overwritemetadata2:{"12foo":"12bar"}' \ - del:index.md add:~/updated.md token:0e4a82a370228b710000 + del:index.md add:updated.md token:0e4a82a370228b710000 + Update record: 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 Files add : 00: 12a31b5e662dfa0a572e9fc523eb703f9708de5e2d53aba74f8ebcebbdb706f7 b text/plain; charset=utf-8 Files delete : a @@ -86,7 +92,8 @@ statuses: Note `token:` is not prefixed to the token in this command. ``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass setunvettedstatus publish 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass setunvettedstatus publish 0e4a82a370228b710000 + Set record status: Status : public ``` @@ -95,6 +102,7 @@ Set record status: ``` $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass getvetted 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 + Vetted record: Status : public Timestamp : 2017-12-14 17:06:21 +0000 UTC @@ -113,6 +121,7 @@ Vetted record: ``` $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass updatevetted 'appendmetadata12:{"foo":"bar"}' 'overwritemetadata2:{"12foo":"12bar"}' del:a add:b token:72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 + Update record: 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 Files add : 00: 12a31b5e662dfa0a572e9fc523eb703f9708de5e2d53aba74f8ebcebbdb706f7 b text/plain; charset=utf-8 Files delete : a @@ -126,6 +135,7 @@ Censor a record (and zap metadata stream 12): ``` $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass setunvettedstatus censor 43c2d4a2c846c188ab0b49012ed17e5f2c16bd6e276cfbb42e30352dffb1743f 'overwritemetadata12:"zap"' + Set record status: Status : censored ``` @@ -134,6 +144,7 @@ Set record status: ``` politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass inventory 1 1 + Vetted record: Status : public Timestamp : 2017-12-14 17:06:21 +0000 UTC diff --git a/politeiad/cmd/politeia/politeia.go b/politeiad/cmd/politeia/politeia.go index 9a09e4b9a..ac812cede 100644 --- a/politeiad/cmd/politeia/politeia.go +++ b/politeiad/cmd/politeia/politeia.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017 The Decred developers +// Copyright (c) 2017-2020 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 00740f40d..50ef51f77 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -33,6 +33,8 @@ import ( // or the pi api spec? // TODO politeiad needs to return the plugin with the error. +// TODO ensure plugins can't write data using short proposal token. +// TODO move proposal validation to pi plugin const ( // MIME types @@ -382,18 +384,22 @@ func convertCommentVoteFromPlugin(v comments.VoteT) pi.CommentVoteT { return pi.CommentVoteInvalid } -func convertCommentVoteDetailsFromPlugin(cv comments.CommentVote) pi.CommentVoteDetails { - return pi.CommentVoteDetails{ - UserID: cv.UserID, - State: convertPropStateFromComments(cv.State), - Token: cv.Token, - CommentID: cv.CommentID, - Vote: convertCommentVoteFromPlugin(cv.Vote), - PublicKey: cv.PublicKey, - Signature: cv.Signature, - Timestamp: cv.Timestamp, - Receipt: cv.Receipt, +func convertCommentVoteDetailsFromPlugin(cv []comments.CommentVote) []pi.CommentVoteDetails { + c := make([]pi.CommentVoteDetails, 0, len(cv)) + for _, v := range cv { + c = append(c, pi.CommentVoteDetails{ + UserID: v.UserID, + State: convertPropStateFromComments(v.State), + Token: v.Token, + CommentID: v.CommentID, + Vote: convertCommentVoteFromPlugin(v.Vote), + PublicKey: v.PublicKey, + Signature: v.Signature, + Timestamp: v.Timestamp, + Receipt: v.Receipt, + }) } + return c } func convertCommentVoteFromPi(v pi.CommentVoteT) piplugin.CommentVoteT { @@ -1397,6 +1403,11 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p func (p *politeiawww) processProposalSetStatus(pss pi.ProposalSetStatus, usr user.User) (*pi.ProposalSetStatusReply, error) { log.Tracef("processProposalSetStatus: %v %v", pss.Token, pss.Status) + // Sanity check + if !usr.Admin { + return nil, fmt.Errorf("not an admin") + } + // Verify token if !tokenIsFullLength(pss.Token) { return nil, pi.UserErrorReply{ @@ -1653,7 +1664,7 @@ func (p *politeiawww) processCommentCensor(cc pi.CommentCensor, usr user.User) ( // Sanity check if !usr.Admin { - return nil, fmt.Errorf("user not admin") + return nil, fmt.Errorf("not an admin") } // Verify user signed with their active identity @@ -1687,6 +1698,7 @@ func (p *politeiawww) processCommentCensor(cc pi.CommentCensor, usr user.User) ( func (p *politeiawww) processComments(c pi.Comments) (*pi.CommentsReply, error) { log.Tracef("processComments: %v", c.Token) + // Send plugin command reply, err := p.commentsAll(comments.GetAll{ State: convertCommentsStateFromPi(c.State), Token: c.Token, @@ -1695,7 +1707,7 @@ func (p *politeiawww) processComments(c pi.Comments) (*pi.CommentsReply, error) return nil, err } - // Compile comments. Comments contain user data that needs to be + // Prepare reply. Comments contain user data that needs to be // pulled from the user database. cs := make([]pi.Comment, 0, len(reply.Comments)) for _, cm := range reply.Comments { @@ -1725,7 +1737,6 @@ func (p *politeiawww) processComments(c pi.Comments) (*pi.CommentsReply, error) func (p *politeiawww) processCommentVotes(cv pi.CommentVotes) (*pi.CommentVotesReply, error) { log.Tracef("processCommentVotes: %v %v", cv.Token, cv.UserID) - // Send plugin command v := comments.Votes{ State: convertCommentsStateFromPi(cv.State), Token: cv.Token, @@ -1736,13 +1747,8 @@ func (p *politeiawww) processCommentVotes(cv pi.CommentVotes) (*pi.CommentVotesR return nil, err } - votes := make([]pi.CommentVoteDetails, 0, len(cvr.Votes)) - for _, v := range cvr.Votes { - votes = append(votes, convertCommentVoteDetailsFromPlugin(v)) - } - return &pi.CommentVotesReply{ - Votes: votes, + Votes: convertCommentVoteDetailsFromPlugin(cvr.Votes), }, nil } @@ -2181,9 +2187,9 @@ func (p *politeiawww) handleComments(w http.ResponseWriter, r *http.Request) { func (p *politeiawww) handleCommentVotes(w http.ResponseWriter, r *http.Request) { log.Tracef("handleCommentVotes") - var cvs pi.CommentVotes + var cv pi.CommentVotes decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&cvs); err != nil { + if err := decoder.Decode(&cv); err != nil { respondWithPiError(w, r, "handleCommentVotes: unmarshal", pi.UserErrorReply{ ErrorCode: pi.ErrorStatusInputInvalid, @@ -2191,13 +2197,13 @@ func (p *politeiawww) handleCommentVotes(w http.ResponseWriter, r *http.Request) return } - cvsr, err := p.processCommentVotes(cvs) + cvr, err := p.processCommentVotes(cv) if err != nil { respondWithPiError(w, r, "handleCommentVotes: processCommentVotes: %v", err) } - util.RespondWithJSON(w, http.StatusOK, cvsr) + util.RespondWithJSON(w, http.StatusOK, cvr) } func (p *politeiawww) handleVoteAuthorize(w http.ResponseWriter, r *http.Request) { @@ -2405,7 +2411,7 @@ func (p *politeiawww) setupPiRoutes() { permissionLogin) p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteProposalSetStatus, p.handleProposalSetStatus, - permissionLogin) + permissionAdmin) p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteProposals, p.handleProposals, permissionPublic) @@ -2428,7 +2434,7 @@ func (p *politeiawww) setupPiRoutes() { permissionPublic) p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteCommentVotes, p.handleCommentVotes, - permissionLogin) + permissionPublic) // Vote routes p.addRoute(http.MethodPost, pi.APIRoute, diff --git a/politeiawww/plugin.go b/politeiawww/plugin.go index b94fd3904..3ad50a71a 100644 --- a/politeiawww/plugin.go +++ b/politeiawww/plugin.go @@ -25,17 +25,13 @@ type plugin struct { Settings []pluginSetting // Settings } -func convertPluginSettingFromPD(ps pd.PluginSetting) pluginSetting { - return pluginSetting{ - Key: ps.Key, - Value: ps.Value, - } -} - func convertPluginFromPD(p pd.Plugin) plugin { ps := make([]pluginSetting, 0, len(p.Settings)) for _, v := range p.Settings { - ps = append(ps, convertPluginSettingFromPD(v)) + ps = append(ps, pluginSetting{ + Key: v.Key, + Value: v.Value, + }) } return plugin{ ID: p.ID, diff --git a/politeiawww/www.go b/politeiawww/www.go index 578f84b99..4cac66407 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -884,7 +884,7 @@ func _main() error { userEmails: make(map[string]uuid.UUID), } - // Get plugins from politeiad + // Setup politeiad plugins p.plugins, err = p.getPluginInventory() if err != nil { return fmt.Errorf("getPluginInventory: %v", err) From 28a5602580f755483a871f7b9150b971f0c6ca18 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 30 Sep 2020 15:42:36 -0500 Subject: [PATCH 121/449] working through more tlogbe issues --- politeiad/backend/tlogbe/tlog.go | 24 ++++---- politeiad/backend/tlogbe/tlogbe.go | 99 +++++++++++++----------------- politeiad/cmd/politeia/README.md | 5 +- politeiawww/api/pi/v1/v1.go | 3 + 4 files changed, 61 insertions(+), 70 deletions(-) diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index f932a5ec3..4d06093e4 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -103,14 +103,14 @@ type tlog struct { // 1. Record content blobs are saved to the kv store. // // 2. The kv store keys are stuffed into the LogLeaf.ExtraData field and the -// log leaves are appended onto the trillian tree. +// leaves are appended onto the trillian tree. // -// 3. If there failures in steps 1 or 2 for any of the blobs then the update -// will exit without completing. No unwinding is performed. Blobs will be -// left in the kv store as orphaned blobs. The trillian tree is append only -// so once a leaf is appended, it is there permanently. If steps 1 and 2 are -// successful then a recordIndex will be created, saved to the kv store, and -// appended onto the trillian tree. +// 3. If there are failures in steps 1 or 2 for any of the blobs then the +// update will exit without completing. No unwinding is performed. Blobs +// will be left in the kv store as orphaned blobs. The trillian tree is +// append only so once a leaf is appended, it's there permanently. If steps +// 1 and 2 are successful then a recordIndex will be created, saved to the +// kv store, and appended onto the trillian tree. // // Appending a recordIndex onto the trillian tree is the last operation that // occurs during a record update. If a recordIndex exists in the tree then the @@ -483,13 +483,12 @@ func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []ba // Increment the iteration idx.Iteration = oldIdx.Iteration + 1 - // Sanity check. The record index should be fully populated at this - // point. + // Sanity check switch { case idx.Version != oldIdx.Version: return fmt.Errorf("invalid index version: got %v, want %v", idx.Version, oldIdx.Version) - case idx.Version != oldIdx.Iteration+1: + case idx.Iteration != oldIdx.Iteration+1: return fmt.Errorf("invalid index iteration: got %v, want %v", idx.Iteration, oldIdx.Iteration+1) case idx.RecordMetadata == nil: @@ -502,7 +501,7 @@ func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []ba len(idx.Files), len(oldIdx.Files)) } - // Blobify the record index and freeze record + // Blobify the record index blobs := make([][]byte, 0, 2) be, err := convertBlobEntryFromRecordIndex(*idx) if err != nil { @@ -518,6 +517,7 @@ func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []ba } blobs = append(blobs, b) + // Blobify the freeze record be, err = convertBlobEntryFromFreezeRecord(fr) if err != nil { return err @@ -532,7 +532,7 @@ func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []ba } blobs = append(blobs, b) - // Save record index and freeze record blobs to the store + // Save blobs to the kv store keys, err := t.store.Put(blobs) if err != nil { return fmt.Errorf("store Put: %v", err) diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index bbb78ef48..1451f048a 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -48,25 +48,27 @@ const ( var ( _ backend.Backend = (*tlogBackend)(nil) - // statusChanges contains the allowed record status change - // transitions. If statusChanges[currentStatus][newStatus] exists - // then the status change is allowed. + // statusChanges contains the allowed record status changes. If + // statusChanges[currentStatus][newStatus] exists then the status + // change is allowed. + // + // Note, the tlog backend does not make use of the status + // MDStatusIterationUnvetted. The original purpose of this status + // was to show when an unvetted record had been altered since + // unvetted records were not versioned in the git backend. The tlog + // backend versions unvetted records and thus does not need to use + // this additional status. statusChanges = map[backend.MDStatusT]map[backend.MDStatusT]struct{}{ // Unvetted status changes backend.MDStatusUnvetted: map[backend.MDStatusT]struct{}{ - backend.MDStatusIterationUnvetted: struct{}{}, - backend.MDStatusVetted: struct{}{}, - backend.MDStatusCensored: struct{}{}, - }, - backend.MDStatusIterationUnvetted: map[backend.MDStatusT]struct{}{ backend.MDStatusVetted: struct{}{}, backend.MDStatusCensored: struct{}{}, }, // Vetted status changes backend.MDStatusVetted: map[backend.MDStatusT]struct{}{ - backend.MDStatusArchived: struct{}{}, backend.MDStatusCensored: struct{}{}, + backend.MDStatusArchived: struct{}{}, }, // Statuses that do not allow any further transitions @@ -143,11 +145,11 @@ func (t *tlogBackend) prefixAdd(fullToken []byte) { log.Debugf("Token prefix cached: %v", prefix) } -func (t *tlogBackend) vettedTreeIDGet(token string) (int64, bool) { +func (t *tlogBackend) vettedTreeID(token []byte) (int64, bool) { t.RLock() defer t.RUnlock() - treeID, ok := t.vettedTreeIDs[token] + treeID, ok := t.vettedTreeIDs[hex.EncodeToString(token)] return treeID, ok } @@ -167,7 +169,7 @@ func (t *tlogBackend) vettedTreeIDFromToken(token []byte) (int64, bool) { // Check if the token is in the vetted cache. The vetted cache is // lazy loaded if the token is not present then we need to check // manually. - treeID, ok := t.vettedTreeIDGet(hex.EncodeToString(token)) + treeID, ok := t.vettedTreeID(token) if ok { return treeID, true } @@ -640,6 +642,11 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ } } + // Verify record exists and is unvetted + if !t.UnvettedExists(token) { + return nil, backend.ErrRecordNotFound + } + // Apply the record changes and save the new version. The lock // needs to be held for the remainder of the function. t.unvetted.Lock() @@ -652,9 +659,6 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ treeID := treeIDFromToken(token) r, err := t.unvetted.recordLatest(treeID) if err != nil { - if err == errRecordNotFound { - return nil, backend.ErrRecordNotFound - } return nil, fmt.Errorf("recordLatest: %v", err) } @@ -664,7 +668,7 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ // Create record metadata recordMD, err := recordMetadataNew(token, files, - backend.MDStatusIterationUnvetted, r.RecordMetadata.Iteration+1) + backend.MDStatusUnvetted, r.RecordMetadata.Iteration+1) if err != nil { return nil, err } @@ -703,13 +707,6 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ token, err) } - // Update inventory cache. The inventory will only need to be - // updated if there was a status transition. - if r.RecordMetadata.Status != recordMD.Status { - // Status was changed - t.inventoryUpdate(recordMD.Token, r.RecordMetadata.Status, recordMD.Status) - } - // Return updated record r, err = t.unvetted.recordLatest(treeID) if err != nil { @@ -841,6 +838,11 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite } } + // Verify record exists and is unvetted + if !t.UnvettedExists(token) { + return backend.ErrRecordNotFound + } + // Pull the existing record and apply the metadata updates. The // unvetted lock must be held for the remainder of this function. t.unvetted.Lock() @@ -853,9 +855,6 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite treeID := treeIDFromToken(token) r, err := t.unvetted.recordLatest(treeID) if err != nil { - if err == errRecordNotFound { - return backend.ErrRecordNotFound - } return fmt.Errorf("recordLatest: %v", err) } @@ -991,7 +990,7 @@ func (t *tlogBackend) UnvettedExists(token []byte) bool { // If the token is in the vetted cache then we know this is not an // unvetted record without having to make any network requests. - _, ok := t.vettedTreeIDGet(hex.EncodeToString(token)) + _, ok := t.vettedTreeID(token) if ok { return false } @@ -1040,12 +1039,15 @@ func (t *tlogBackend) GetUnvetted(token []byte, version string) (*backend.Record v = uint32(u) } + // Verify record exists and is unvetted + if !t.UnvettedExists(token) { + return nil, backend.ErrRecordNotFound + } + + // Get unvetted record r, err := t.unvetted.record(treeID, v) if err != nil { - if err == errRecordNotFound { - err = backend.ErrRecordNotFound - } - return nil, err + return nil, fmt.Errorf("unvetted record: %v", err) } return r, nil @@ -1160,25 +1162,15 @@ func (t *tlogBackend) unvettedCensor(token []byte, rm backend.RecordMetadata, me return nil } -// This function must be called WITH the unvetted lock held. -func (t *tlogBackend) unvettedArchive(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { - // Freeze the tree. Nothing else needs to be done for an archived - // record. - treeID := treeIDFromToken(token) - err := t.unvetted.treeFreeze(treeID, rm, metadata, freezeRecord{}) - if err != nil { - return err - } - - log.Debugf("Unvetted record %x frozen", token) - - return nil -} - func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { log.Tracef("SetUnvettedStatus: %x %v (%v)", token, status, backend.MDStatus[status]) + // Verify record exists and is unvetted + if !t.UnvettedExists(token) { + return nil, backend.ErrRecordNotFound + } + // The existing record must be pulled and updated. The unvetted // lock must be held for the rest of this function. t.unvetted.Lock() @@ -1194,12 +1186,12 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, return nil, fmt.Errorf("recordLatest: %v", err) } rm := r.RecordMetadata - oldStatus := rm.Status + currStatus := rm.Status // Validate status change - if !statusChangeIsAllowed(oldStatus, status) { + if !statusChangeIsAllowed(currStatus, status) { return nil, backend.StateTransitionError{ - From: oldStatus, + From: currStatus, To: status, } } @@ -1240,11 +1232,6 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, if err != nil { return nil, fmt.Errorf("unvettedCensor: %v", err) } - case backend.MDStatusArchived: - err := t.unvettedArchive(token, rm, metadata) - if err != nil { - return nil, fmt.Errorf("unvettedArchive: %v", err) - } default: return nil, fmt.Errorf("unknown status: %v (%v)", backend.MDStatus[status], status) @@ -1258,10 +1245,10 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, } // Update inventory cache - t.inventoryUpdate(rm.Token, oldStatus, status) + t.inventoryUpdate(rm.Token, currStatus, status) log.Debugf("Status change %x from %v (%v) to %v (%v)", - token, backend.MDStatus[oldStatus], oldStatus, + token, backend.MDStatus[currStatus], currStatus, backend.MDStatus[status], status) // Return the updated record diff --git a/politeiad/cmd/politeia/README.md b/politeiad/cmd/politeia/README.md index 22f5e3316..0271a52a2 100644 --- a/politeiad/cmd/politeia/README.md +++ b/politeiad/cmd/politeia/README.md @@ -6,6 +6,7 @@ The retrieved identity is used to verify replies from politeiad. ``` $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass identity + Key : 8f627e9da14322626d7e81d789f7fcafd25f62235a95377f39cbc7293c4944ad Fingerprint: j2J+naFDImJtfoHXiff8r9JfYiNalTd/OcvHKTxJRK0= @@ -101,14 +102,14 @@ Set record status: ## Get vetted record ``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass getvetted 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass getvetted 0e4a82a370228b710000 Vetted record: Status : public Timestamp : 2017-12-14 17:06:21 +0000 UTC Censorship record: Merkle : 12a31b5e662dfa0a572e9fc523eb703f9708de5e2d53aba74f8ebcebbdb706f7 - Token : 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 + Token : 0e4a82a370228b710000 Signature: 25483966ec6e8df90398c197e3bdb74fe5356df0c96927d771b06e83a7fb29e069751118f4496e42d02a63feb74d67b777c69bb8f356aeafca873325aaf8010f Metadata : [{2 {"12foo":"12bar"}} {12 {"moo":"lala"}{"foo":"bar"}}] File (00) : diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index cac91e926..205f43674 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -23,6 +23,9 @@ type VoteErrorT int // TODO the proposals route should allow filtering by user ID. Actually, this // is going to have to wait until after the intial release. This is non-trivial // to accomplish and is outside the scope of the core functionality. +// TODO show the difference between unvetted censored and vetted censored +// in the proposal inventory route since fetching them requires specifying +// the state. const ( APIVersion = 1 From bc0a4dd42fa0fd740a61fb07a95d08452f134ab5 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 30 Sep 2020 19:03:07 -0500 Subject: [PATCH 122/449] fix errors from merge --- go.mod | 3 --- go.sum | 1 - politeiad/backend/gitbe/gitbe.go | 6 +++--- politeiad/backend/tlogbe/ticketvote.go | 12 ++++++------ .../cmd/politeiawww_dbutil/politeiawww_dbutil.go | 13 ------------- 5 files changed, 9 insertions(+), 26 deletions(-) diff --git a/go.mod b/go.mod index 6157383f9..1b648d1c0 100644 --- a/go.mod +++ b/go.mod @@ -14,13 +14,10 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/decred/dcrd/blockchain/stake/v3 v3.0.0-20200921185235-6d75c7ec1199 github.com/decred/dcrd/certgen v1.1.1-0.20200921185235-6d75c7ec1199 - github.com/decred/dcrd/chaincfg v1.1.1 github.com/decred/dcrd/chaincfg/chainhash v1.0.3-0.20200921185235-6d75c7ec1199 github.com/decred/dcrd/chaincfg/v3 v3.0.0 github.com/decred/dcrd/dcrec v1.0.1-0.20200921185235-6d75c7ec1199 - github.com/decred/dcrd/dcrec/secp256k1 v1.0.2 github.com/decred/dcrd/dcrec/secp256k1/v3 v3.0.0 - github.com/decred/dcrd/dcrutil v1.1.1 github.com/decred/dcrd/dcrutil/v3 v3.0.0 github.com/decred/dcrd/hdkeychain/v3 v3.0.0 github.com/decred/dcrd/wire v1.4.0 diff --git a/go.sum b/go.sum index eae7f69d1..4c8e6e321 100644 --- a/go.sum +++ b/go.sum @@ -295,7 +295,6 @@ github.com/decred/dcrdata/txhelpers/v4 v4.0.1 h1:jNPPSP5HzE4cfddj5zIJhrIEus/Tvd2 github.com/decred/dcrdata/txhelpers/v4 v4.0.1/go.mod h1:cUJbgsIzzI42llHDS0nkPlG49vPJ0cW6IZGbfu5sFrA= github.com/decred/dcrtime v0.0.0-20191018193024-8d8b4ef0458e h1:sNDR7vx6gaA3WD+WoEofTvtdjfwHAiogtjB3kt8iFco= github.com/decred/dcrtime v0.0.0-20191018193024-8d8b4ef0458e/go.mod h1:IyZnyBE3E6RBFsEjwEs21FrO/UsrLrL15hUnpZZQxpU= -github.com/decred/dcrtime v0.0.0-20200912200806-b1e4dbc46be9 h1:/S6B5sB2MxhtYcy+bdXOqja9/tSeXX7n7OrtbaI4pjk= github.com/decred/dcrtime/api/v2 v2.0.0-20200912200806-b1e4dbc46be9 h1:LtZvUp0JwILQNi+fJbutWGm4v2Ia+Vje83pO3D8c9VA= github.com/decred/dcrtime/api/v2 v2.0.0-20200912200806-b1e4dbc46be9/go.mod h1:JdIX208vnNj4TdU6hDRaN+ccxmxp1I1R6sWGZNK1BAQ= github.com/decred/dcrwallet v1.2.2 h1:NdI13wxP+OsWKXPqjWQQ9VSGAl4VoSLBfAunzFreeJg= diff --git a/politeiad/backend/gitbe/gitbe.go b/politeiad/backend/gitbe/gitbe.go index a978a7560..cfb8484ba 100644 --- a/politeiad/backend/gitbe/gitbe.go +++ b/politeiad/backend/gitbe/gitbe.go @@ -3038,7 +3038,7 @@ func New(anp *chaincfg.Params, root string, dcrtimeHost string, gitPath string, // Register all plugins g.plugins = []backend.Plugin{ getDecredPlugin(dcrdataHost), - getCMSPlugin(anp.Name != chaincfg.MainNetParams.Name), + getCMSPlugin(anp.Name != chaincfg.MainNetParams().Name), } // Setup cms plugin @@ -3048,10 +3048,10 @@ func New(anp *chaincfg.Params, root string, dcrtimeHost string, gitPath string, // Setup decred plugin var voteDurationMin, voteDurationMax string switch anp.Name { - case chaincfg.MainNetParams.Name: + case chaincfg.MainNetParams().Name: voteDurationMin = strconv.Itoa(decredplugin.VoteDurationMinMainnet) voteDurationMax = strconv.Itoa(decredplugin.VoteDurationMaxMainnet) - case chaincfg.TestNet3Params.Name: + case chaincfg.TestNet3Params().Name: voteDurationMin = strconv.Itoa(decredplugin.VoteDurationMinTestnet) voteDurationMax = strconv.Itoa(decredplugin.VoteDurationMaxTestnet) default: diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index 115860519..2443f0e86 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -19,10 +19,10 @@ import ( "sync" "time" - "github.com/decred/dcrd/chaincfg" "github.com/decred/dcrd/chaincfg/chainhash" - "github.com/decred/dcrd/dcrec/secp256k1" - "github.com/decred/dcrd/dcrutil" + "github.com/decred/dcrd/chaincfg/v3" + "github.com/decred/dcrd/dcrec/secp256k1/v3/ecdsa" + "github.com/decred/dcrd/dcrutil/v3" "github.com/decred/dcrd/wire" "github.com/decred/politeia/plugins/dcrdata" "github.com/decred/politeia/plugins/ticketvote" @@ -808,7 +808,7 @@ func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, // from: github.com/decred/dcrd/blob/0fc55252f912756c23e641839b1001c21442c38a/rpcserver.go#L5605 func (p *ticketVotePlugin) voteMessageVerify(address, message, signature string) (bool, error) { // Decode the provided address. - addr, err := dcrutil.DecodeAddress(address) + addr, err := dcrutil.DecodeAddress(address, p.activeNetParams) if err != nil { return false, fmt.Errorf("Could not decode address: %v", err) @@ -832,7 +832,7 @@ func (p *ticketVotePlugin) voteMessageVerify(address, message, signature string) wire.WriteVarString(&buf, 0, "Decred Signed Message:\n") wire.WriteVarString(&buf, 0, message) expectedMessageHash := chainhash.HashB(buf.Bytes()) - pk, wasCompressed, err := secp256k1.RecoverCompact(sig, + pk, wasCompressed, err := ecdsa.RecoverCompact(sig, expectedMessageHash) if err != nil { // Mirror Bitcoin Core behavior, which treats error in @@ -856,7 +856,7 @@ func (p *ticketVotePlugin) voteMessageVerify(address, message, signature string) } // Return boolean if addresses match. - return a.EncodeAddress() == address, nil + return a.Address() == address, nil } func (p *ticketVotePlugin) castVoteSignatureVerify(cv ticketvote.CastVote, addr string) error { diff --git a/politeiawww/cmd/politeiawww_dbutil/politeiawww_dbutil.go b/politeiawww/cmd/politeiawww_dbutil/politeiawww_dbutil.go index 17d40114c..94d879b9c 100644 --- a/politeiawww/cmd/politeiawww_dbutil/politeiawww_dbutil.go +++ b/politeiawww/cmd/politeiawww_dbutil/politeiawww_dbutil.go @@ -27,7 +27,6 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/decred/dcrd/chaincfg/v3" "github.com/decred/politeia/decredplugin" - "github.com/decred/politeia/mdstream" "github.com/decred/politeia/politeiad/backend/gitbe" "github.com/decred/politeia/politeiawww/sharedconfig" "github.com/decred/politeia/politeiawww/user" @@ -372,18 +371,6 @@ func cmdStubUsers() error { if err != nil { return err } - case proposalMDFilename: - b, err := ioutil.ReadFile(path) - if err != nil { - return err - } - - var md mdstream.ProposalGeneralV2 - err = json.Unmarshal(b, &md) - if err != nil { - return fmt.Errorf("proposal md: %v", err) - } - pubkeys[md.PublicKey] = struct{}{} } return nil From 785861b607c20bdf4fbd550d3b2378e2e813c9c3 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 1 Oct 2020 15:40:57 -0500 Subject: [PATCH 123/449] tlogbe fully operational! --- politeiad/backend/tlogbe/anchor.go | 2 +- politeiad/backend/tlogbe/tlog.go | 106 +++++++++-------- politeiad/backend/tlogbe/tlogbe.go | 18 +-- politeiad/backend/tlogbe/tlogclient.go | 13 +- politeiad/cmd/politeia/README.md | 120 +++++++++++-------- politeiad/cmd/politeia/politeia.go | 157 ++++++------------------- politeiad/politeiad.go | 87 ++++++++------ 7 files changed, 238 insertions(+), 265 deletions(-) diff --git a/politeiad/backend/tlogbe/anchor.go b/politeiad/backend/tlogbe/anchor.go index 9ea6d508f..56db5809b 100644 --- a/politeiad/backend/tlogbe/anchor.go +++ b/politeiad/backend/tlogbe/anchor.go @@ -302,7 +302,7 @@ func (t *tlog) anchor() { var exitErr error // Set on exit if there is an error defer func() { if exitErr != nil { - log.Errorf("anchorTrees: %v", exitErr) + log.Errorf("anchor %v: %v", t.id, exitErr) } }() diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index 4d06093e4..90f76be90 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -75,6 +75,10 @@ var ( // errTreeIsFrozen is emitted when a frozen tree is attempted to be // altered. errTreeIsFrozen = errors.New("tree is frozen") + + // errTreeIsNotFrozen is emitted when a tree is expected to be + // frozen but is actually not frozen. + errTreeIsNotFrozen = errors.New("tree is not frozen") ) // We do not unwind. @@ -126,8 +130,14 @@ type recordIndex struct { // file changes that bump the version as well metadata stream and // record metadata changes that don't bump the version. // - // Note this is not the same as the RecordMetadata iteration field, - // which does not get incremented on metadata updates. + // Note, this field is not the same as the backend RecordMetadata + // iteration field, which does not get incremented on metadata + // updates. + // + // TODO maybe it should be the same. The original iteration field + // was to track unvetted changes in gitbe since unvetted gitbe + // records are not versioned. tlogbe unvetted records are versioned + // so the original use for the iteration field isn't needed anymore. Iteration uint32 `json:"iteration"` // The following fields contain the merkle leaf hashes of the @@ -453,19 +463,19 @@ func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []ba } // Prepare kv store blobs for metadata - rbpr, err := recordBlobsPrepare(leavesAll, rm, metadata, + bpr, err := recordBlobsPrepare(leavesAll, rm, metadata, []backend.File{}, t.encryptionKey) if err != nil { return err } // Ensure metadata has been changed - if len(rbpr.blobs) == 0 { + if len(bpr.blobs) == 0 { return errNoMetadataChanges } // Save metadata blobs - idx, err := t.recordBlobsSave(treeID, *rbpr) + idx, err := t.recordBlobsSave(treeID, *bpr) if err != nil { return fmt.Errorf("blobsSave: %v", err) } @@ -1089,9 +1099,9 @@ func (t *tlog) recordBlobsSave(treeID int64, rbpr recordBlobsPrepareReply) (*rec } // recordSave saves the provided record to tlog, creating a new version of the -// record. Once the record contents have been successfully saved to tlog, a -// recordIndex is created for this version of the record and saved to tlog as -// well. +// record (the record iteration also gets incremented on new versions). Once +// the record contents have been successfully saved to tlog, a recordIndex is +// created for this version of the record and saved to tlog as well. func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { log.Tracef("tlog recordSave: %v", treeID) @@ -1124,7 +1134,7 @@ func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []ba } // Prepare kv store blobs - rbpr, err := recordBlobsPrepare(leavesAll, rm, metadata, + bpr, err := recordBlobsPrepare(leavesAll, rm, metadata, files, t.encryptionKey) if err != nil { return err @@ -1137,7 +1147,7 @@ func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []ba // index by the recordBlobsPrepare function. If a file is in the // new record index it means that the file has existed in one of // the previous versions of the record. - newMerkle, ok := rbpr.recordIndex.Files[v.Name] + newMerkle, ok := bpr.recordIndex.Files[v.Name] if !ok { // File does not exist in the index. It is new. fileChanges = true @@ -1173,7 +1183,7 @@ func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []ba } // Save blobs - idx, err := t.recordBlobsSave(treeID, *rbpr) + idx, err := t.recordBlobsSave(treeID, *bpr) if err != nil { return fmt.Errorf("blobsSave: %v", err) } @@ -1209,8 +1219,12 @@ func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []ba return nil } -func (t *tlog) recordMetadataUpdate(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { - log.Tracef("tlog recordMetadataUpdate: %v", treeID) +// recordMetadataSave saves the provided metadata to tlog, creating a new +// iteration of the record while keeping the record version the same. Once the +// metadata has been successfully saved to tlog, a recordIndex is created for +// this iteration of the record and saved to tlog as well. +func (t *tlog) recordMetadataSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { + log.Tracef("tlog recordMetadataSave: %v", treeID) // Verify tree exists if !t.treeExists(treeID) { @@ -1235,8 +1249,12 @@ func (t *tlog) recordMetadataUpdate(treeID int64, rm backend.RecordMetadata, met return err } - // Verify changes are being made - if len(bpr.blobs) == 0 { + // Verify changes are being made. The record index returned from + // recordBlobsPrepare() will contain duplicate blobs, i.e. metadata + // blobs that already existed prior to this update. If the record + // index already contains all of the metadata blobs it means that + // no metadata changes are being made. + if len(metadata) == len(bpr.recordIndex.Metadata) { return errNoMetadataChanges } @@ -1259,13 +1277,12 @@ func (t *tlog) recordMetadataUpdate(treeID int64, rm backend.RecordMetadata, met // Increment the iteration idx.Iteration = oldIdx.Iteration + 1 - // Sanity check. The record index should be fully populated at this - // point. + // Sanity check switch { case idx.Version != oldIdx.Version: return fmt.Errorf("invalid index version: got %v, want %v", idx.Version, oldIdx.Version) - case idx.Version != oldIdx.Iteration+1: + case idx.Iteration != oldIdx.Iteration+1: return fmt.Errorf("invalid index iteration: got %v, want %v", idx.Iteration, oldIdx.Iteration+1) case idx.RecordMetadata == nil: @@ -1287,8 +1304,10 @@ func (t *tlog) recordMetadataUpdate(treeID int64, rm backend.RecordMetadata, met return nil } -// recordDel walks the provided tree and deletes all blobs in the store that -// correspond to record files. This is done for all versions of the record. +// recordDel walks the provided tree and deletes all file blobs in the store +// that correspond to record files. This is done for all versions and all +// iterations of the record. Record metadata and metadata stream blobs are not +// deleted. func (t *tlog) recordDel(treeID int64) error { log.Tracef("tlog recordDel: %v", treeID) @@ -1305,8 +1324,8 @@ func (t *tlog) recordDel(treeID int64) error { // Ensure tree is frozen. Deleting files from the store is only // allowed on frozen trees. - if treeIsFrozen(leaves) { - return errTreeIsFrozen + if !treeIsFrozen(leaves) { + return errTreeIsNotFrozen } // Retrieve all the record indexes @@ -1403,16 +1422,12 @@ func (t *tlog) record(treeID int64, version uint32) (*backend.Record, error) { if err != nil { return nil, fmt.Errorf("store Get: %v", err) } - if len(blobs) != len(keys) { - // One or more blobs were not found - missing := make([]string, 0, len(keys)) - for _, v := range keys { - _, ok := blobs[v] - if !ok { - missing = append(missing, v) - } - } - return nil, fmt.Errorf("blobs not found: %v", missing) + if len(keys) != len(blobs) { + // One or more blobs were not found. This is allowed since the + // blobs for a censored record will not exist, but the record + // metadata and metadata streams should still be returned. + log.Tracef("Blobs not found %v: want %v, got %v", + treeID, len(keys), len(blobs)) } // Decode blobs @@ -1506,9 +1521,6 @@ func (t *tlog) record(treeID int64, version uint32) (*backend.Record, error) { case len(metadata) != len(index.Metadata): return nil, fmt.Errorf("invalid number of metadata; got %v, want %v", len(metadata), len(index.Metadata)) - case len(files) != len(index.Files): - return nil, fmt.Errorf("invalid number of files; got %v, want %v", - len(files), len(index.Files)) } return &backend.Record{ @@ -1531,6 +1543,8 @@ func (t *tlog) recordProof(treeID int64, version uint32) {} // blobsSave saves the provided blobs to the key-value store then appends them // onto the trillian tree. Note, hashes contains the hashes of the data encoded // in the blobs. The hashes must share the same ordering as the blobs. +// +// This function satisfies the tlogClient interface. func (t *tlog) blobsSave(treeID int64, keyPrefix string, blobs, hashes [][]byte, encrypt bool) ([][]byte, error) { log.Tracef("tlog blobsSave: %v %v %v", treeID, keyPrefix, encrypt) @@ -1609,7 +1623,11 @@ func (t *tlog) blobsSave(treeID int64, keyPrefix string, blobs, hashes [][]byte, return merkles, nil } -// del deletes the blobs that correspond to the provided merkle leaf hashes. +// del deletes the blobs in the kv store that correspond to the provided merkle +// leaf hashes. The kv store keys in store in the ExtraData field of the leaves +// specified by the provided merkle leaf hashes. +// +// This function satisfies the tlogClient interface. func (t *tlog) blobsDel(treeID int64, merkles [][]byte) error { log.Tracef("tlog blobsDel: %v", treeID) @@ -1661,6 +1679,8 @@ func (t *tlog) blobsDel(treeID int64, merkles [][]byte) error { // If a blob does not exist it will not be included in the returned map. It is // the responsibility of the caller to check that a blob is returned for each // of the provided merkle hashes. +// +// This function satisfies the tlogClient interface. func (t *tlog) blobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, error) { log.Tracef("tlog blobsByMerkle: %v", treeID) @@ -1749,7 +1769,7 @@ func (t *tlog) blobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, // blobsByKeyPrefix returns all blobs that match the provided key prefix. // -// This function satisfies the backendClient interface. +// This function satisfies the tlogClient interface. func (t *tlog) blobsByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) { log.Tracef("tlog blobsByKeyPrefix: %v %v", treeID, keyPrefix) @@ -1816,15 +1836,9 @@ func (t *tlog) blobsByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error // TODO run fsck episodically func (t *tlog) fsck() { - /* - // Freeze any trees with a freeze record - _, err = t.trillian.treeFreeze(treeID) - if err != nil { - return fmt.Errorf("treeFreeze: %v", err) - } - */ - - // Failed censor + // Set tree status to frozen for any trees with a freeze record. + // Failed censor. Ensure all blobs have been deleted from all + // record versions of a censored record. } func (t *tlog) close() { diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 1451f048a..b78a55027 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -142,7 +142,7 @@ func (t *tlogBackend) prefixAdd(fullToken []byte) { prefix := tokenPrefix(fullToken) t.prefixes[prefix] = fullToken - log.Debugf("Token prefix cached: %v", prefix) + log.Debugf("Add token prefix: %v", prefix) } func (t *tlogBackend) vettedTreeID(token []byte) (int64, bool) { @@ -159,7 +159,7 @@ func (t *tlogBackend) vettedTreeIDAdd(token string, treeID int64) { t.vettedTreeIDs[token] = treeID - log.Debugf("Vetted tree ID cached: %v %v", token, treeID) + log.Debugf("Add vetted tree ID: %v %v", token, treeID) } // vettedTreeIDFromToken returns the vetted tree ID that corresponds to the @@ -240,7 +240,7 @@ func (t *tlogBackend) inventoryAdd(token string, s backend.MDStatusT) { t.inventory[s] = append([]string{token}, t.inventory[s]...) - log.Debugf("Inventory cache added: %v %v", token, backend.MDStatus[s]) + log.Debugf("Add to inventory: %v %v", token, backend.MDStatus[s]) } func (t *tlogBackend) inventoryUpdate(token string, currStatus, newStatus backend.MDStatusT) { @@ -272,7 +272,7 @@ func (t *tlogBackend) inventoryUpdate(token string, currStatus, newStatus backen // Prepend token to new status t.inventory[newStatus] = append([]string{token}, t.inventory[newStatus]...) - log.Debugf("Inventory cache updated: %v %v to %v", + log.Debugf("Update inventory: %v %v to %v", token, backend.MDStatus[currStatus], backend.MDStatus[newStatus]) } @@ -877,7 +877,7 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) // Update metadata - err = t.unvetted.recordMetadataUpdate(treeID, r.RecordMetadata, metadata) + err = t.unvetted.recordMetadataSave(treeID, r.RecordMetadata, metadata) if err != nil { if err == errNoMetadataChanges { return backend.ErrNoChanges @@ -963,7 +963,7 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) // Update metadata - err = t.vetted.recordMetadataUpdate(treeID, r.RecordMetadata, metadata) + err = t.vetted.recordMetadataSave(treeID, r.RecordMetadata, metadata) if err != nil { if err == errNoMetadataChanges { return backend.ErrNoChanges @@ -1149,7 +1149,7 @@ func (t *tlogBackend) unvettedCensor(token []byte, rm backend.RecordMetadata, me return fmt.Errorf("treeFreeze %v: %v", treeID, err) } - log.Debugf("Unvetted record %x frozen", token) + log.Debugf("Unvetted record frozen %v", token) // Delete all record files err = t.unvetted.recordDel(treeID) @@ -1157,7 +1157,7 @@ func (t *tlogBackend) unvettedCensor(token []byte, rm backend.RecordMetadata, me return fmt.Errorf("recordDel %v: %v", treeID, err) } - log.Debug("Unvetted record %x files deleted", token) + log.Debugf("Unvetted record files deleted %v", token) return nil } @@ -1548,7 +1548,7 @@ func (t *tlogBackend) setup() error { for _, v := range trees { token := tokenFromTreeID(v.TreeId) - log.Debugf("Building cache: %v %x", v.TreeId, token) + log.Debugf("Building memory caches for %x", token) // Add tree to prefixes cache t.prefixAdd(token) diff --git a/politeiad/backend/tlogbe/tlogclient.go b/politeiad/backend/tlogbe/tlogclient.go index 99c3ec594..33ac8064a 100644 --- a/politeiad/backend/tlogbe/tlogclient.go +++ b/politeiad/backend/tlogbe/tlogclient.go @@ -43,12 +43,6 @@ type backendClient struct { backend *tlogBackend } -func newBackendClient(tlog *tlogBackend) *backendClient { - return &backendClient{ - backend: tlog, - } -} - // tlogByID returns the tlog instance that corresponds to the provided ID. func (c *backendClient) tlogByID(tlogID string) (*tlog, error) { switch tlogID { @@ -162,3 +156,10 @@ func (c *backendClient) blobsByKeyPrefix(tlogID string, token []byte, keyPrefix // Get blobs return tlog.blobsByKeyPrefix(treeID, keyPrefix) } + +// newBackendClient returns a new backendClient. +func newBackendClient(tlog *tlogBackend) *backendClient { + return &backendClient{ + backend: tlog, + } +} diff --git a/politeiad/cmd/politeia/README.md b/politeiad/cmd/politeia/README.md index 0271a52a2..3afcb7eed 100644 --- a/politeiad/cmd/politeia/README.md +++ b/politeiad/cmd/politeia/README.md @@ -1,5 +1,18 @@ # politeia refclient examples +Available commands: +`identity` +`new` +`updateunvetted` +`updateunvettedmd` +`setunvettedstatus` +`getunvetted` +`updatevetted` +`updatevettedmd` +`getvetted` +`plugin` +`plugininventory` + ## Obtain politeiad identity The retrieved identity is used to verify replies from politeiad. @@ -30,29 +43,29 @@ $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass new \ Record submitted Censorship record: Merkle : 4bde9f923b61e26147c79500e6d6dfa27291559a74cd878c29a7f96984dd48bb - Token : 0e4a82a370228b710000 - Signature: a0c4afd301d5452d787ac1c9835fb6f3a32443d21c92cd4575e8ef6d5ef6c4f9199a02f67893aa7b7a610055d2a6d56899ccd73c0a48ffeab72d788d1c4d4a01 + Token : 9dfe084fccb7f27c0000 + Signature: e69a38b6e6c21021db2fe37c6b38886ef987c7347bb881e2358feb766974577a742e535d34cd4d7a140b2555b3771a194fea4be942cbd99247c143d07419bc06 +$ ``` ## Get unvetted record ``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass getunvetted 0e4a82a370228b710000 +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass getunvetted 9dfe084fccb7f27c0000 Unvetted record: Status : not reviewed - Timestamp : 2020-09-28 14:20:14 +0000 UTC + Timestamp : 2020-10-01 14:36:11 +0000 UTC Censorship record: Merkle : 4bde9f923b61e26147c79500e6d6dfa27291559a74cd878c29a7f96984dd48bb - Token : 0e4a82a370228b710000 - Signature: a0c4afd301d5452d787ac1c9835fb6f3a32443d21c92cd4575e8ef6d5ef6c4f9199a02f67893aa7b7a610055d2a6d56899ccd73c0a48ffeab72d788d1c4d4a01 + Token : 9dfe084fccb7f27c0000 + Signature: e69a38b6e6c21021db2fe37c6b38886ef987c7347bb881e2358feb766974577a742e535d34cd4d7a140b2555b3771a194fea4be942cbd99247c143d07419bc06 Metadata : [{2 {"foo":"bar"}} {12 {"moo":"lala"}}] Version : 1 File (00) : Name : index.md MIME : text/plain; charset=utf-8 Digest : 4bde9f923b61e26147c79500e6d6dfa27291559a74cd878c29a7f96984dd48bb - ``` ## Update an unvetted record @@ -74,15 +87,17 @@ already exist. ``` $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass updateunvetted \ 'appendmetadata12:{"foo":"bar"}' 'overwritemetadata2:{"12foo":"12bar"}' \ - del:index.md add:updated.md token:0e4a82a370228b710000 + del:index.md add:updated.md token:9dfe084fccb7f27c0000 -Update record: 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 - Files add : 00: 12a31b5e662dfa0a572e9fc523eb703f9708de5e2d53aba74f8ebcebbdb706f7 b text/plain; charset=utf-8 - Files delete : a +Update record: 9dfe084fccb7f27c0000 + Files add : 00: 22036b8b67a7c54f2bae29e1f9a11551cf62a33a038788b8f2e8f8d6e7f60425 updated.md text/plain; charset=utf-8 + Files delete : index.md Metadata overwrite: 2 Metadata append : 12 ``` +## Update unvetted metadata only + ## Set unvetted status You can update the status of an unvetted record using one of the following @@ -93,7 +108,7 @@ statuses: Note `token:` is not prefixed to the token in this command. ``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass setunvettedstatus publish 0e4a82a370228b710000 +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass setunvettedstatus publish 9dfe084fccb7f27c0000 Set record status: Status : public @@ -102,64 +117,69 @@ Set record status: ## Get vetted record ``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass getvetted 0e4a82a370228b710000 +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass getvetted 9dfe084fccb7f27c0000 Vetted record: Status : public - Timestamp : 2017-12-14 17:06:21 +0000 UTC + Timestamp : 2020-10-01 14:38:43 +0000 UTC Censorship record: - Merkle : 12a31b5e662dfa0a572e9fc523eb703f9708de5e2d53aba74f8ebcebbdb706f7 - Token : 0e4a82a370228b710000 - Signature: 25483966ec6e8df90398c197e3bdb74fe5356df0c96927d771b06e83a7fb29e069751118f4496e42d02a63feb74d67b777c69bb8f356aeafca873325aaf8010f + Merkle : 22036b8b67a7c54f2bae29e1f9a11551cf62a33a038788b8f2e8f8d6e7f60425 + Token : 9dfe084fccb7f27c0000 + Signature: 531e5103e9f8905d52d7bf3c6fdb40070cca4f88e69f3b6c647baf8bd84148471e378b5c137014a1f3f46a2cb9a40cdc302dea4bf828fb6dd09a858fa2748c0e Metadata : [{2 {"12foo":"12bar"}} {12 {"moo":"lala"}{"foo":"bar"}}] + Version : 1 File (00) : - Name : b + Name : updated.md MIME : text/plain; charset=utf-8 - Digest : 12a31b5e662dfa0a572e9fc523eb703f9708de5e2d53aba74f8ebcebbdb706f7 + Digest : 22036b8b67a7c54f2bae29e1f9a11551cf62a33a038788b8f2e8f8d6e7f60425 ``` ## Update a vetted record +Metadata can be updated using the arguments: +`'appendmetadata[ID]:[metadataJSON]'` +`'overwritemetadata[ID]:[metadataJSON]'` + +Files can be updated using the arguments: +`add:[filepath]` +`del:[filename]` + +The token is specified using the argument: +`token:[token]` + +Metadata provided using the `overwritemetadata` argument does not have to +already exist. + ``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass updatevetted 'appendmetadata12:{"foo":"bar"}' 'overwritemetadata2:{"12foo":"12bar"}' del:a add:b token:72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass updatevetted \ + 'appendmetadata12:{"foo":"bar"}' 'overwritemetadata2:{"12foo":"12bar"}' \ + del:updated add:newfile.md token:9dfe084fccb7f27c0000 -Update record: 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 - Files add : 00: 12a31b5e662dfa0a572e9fc523eb703f9708de5e2d53aba74f8ebcebbdb706f7 b text/plain; charset=utf-8 - Files delete : a +Update record: 9dfe084fccb7f27c0000 + Files add : 00: 22036b8b67a7c54f2bae29e1f9a11551cf62a33a038788b8f2e8f8d6e7f60425 newfile.md text/plain; charset=utf-8 + Files delete : updated Metadata overwrite: 2 Metadata append : 12 ``` -## Set vetted status +## Update vetted metadata only -Censor a record (and zap metadata stream 12): +Metadata can be updated using the arguments: +`'appendmetadata[ID]:[metadataJSON]'` +`'overwritemetadata[ID]:[metadataJSON]'` -``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass setunvettedstatus censor 43c2d4a2c846c188ab0b49012ed17e5f2c16bd6e276cfbb42e30352dffb1743f 'overwritemetadata12:"zap"' +The token is specified using the argument: +`token:[token]` -Set record status: - Status : censored +Metadata provided using the `overwritemetadata` argument does not have to +already exist. ``` +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass updatevettedmd \ + 'appendmetadata12:{"foo":"bar"}' token:9dfe084fccb7f27c0000 -## Inventory all records - -``` -politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass inventory 1 1 +Update vetted metadata: 9dfe084fccb7f27c0000 + Metadata append : 12 +` -Vetted record: - Status : public - Timestamp : 2017-12-14 17:06:21 +0000 UTC - Censorship record: - Merkle : 12a31b5e662dfa0a572e9fc523eb703f9708de5e2d53aba74f8ebcebbdb706f7 - Token : 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 - Signature: 25483966ec6e8df90398c197e3bdb74fe5356df0c96927d771b06e83a7fb29e069751118f4496e42d02a63feb74d67b777c69bb8f356aeafca873325aaf8010f - Metadata : [{2 {"12foo":"12bar"}} {12 {"moo":"lala"}{"foo":"bar"}}] -Unvetted record: - Status : censored - Timestamp : 2017-12-14 17:08:33 +0000 UTC - Censorship record: - Merkle : 22e88c7d6da9b73fbb515ed6a8f6d133c680527a799e3069ca7ce346d90649b2 - Token : 43c2d4a2c846c188ab0b49012ed17e5f2c16bd6e276cfbb42e30352dffb1743f - Signature: 5c28d2a93ff9cfe35e8a6b465ae06fa596b08bfe7b980ff9dbe68877e7d860010ec3c4fd8c8b739dc4ceeda3a2381899c7741896323856f0f267abf9a40b8003 - Metadata : [{2 {"foo":"bar"}} {12 {"moo":"lala"}}] -``` +## Set vetted status +TODO diff --git a/politeiad/cmd/politeia/politeia.go b/politeiad/cmd/politeia/politeia.go index b83397211..a57b711d6 100644 --- a/politeiad/cmd/politeia/politeia.go +++ b/politeiad/cmd/politeia/politeia.go @@ -342,91 +342,6 @@ func getPluginInventory() error { return nil } -func remoteInventory() (*v1.InventoryReply, error) { - challenge, err := util.Random(v1.ChallengeSize) - if err != nil { - return nil, err - } - b, err := json.Marshal(v1.Inventory{ - Challenge: hex.EncodeToString(challenge), - }) - if err != nil { - return nil, err - } - - if *printJson { - fmt.Println(string(b)) - } - - c, err := util.NewClient(verify, *rpccert) - if err != nil { - return nil, err - } - req, err := http.NewRequest("POST", *rpchost+v1.InventoryRoute, - bytes.NewReader(b)) - if err != nil { - return nil, err - } - req.SetBasicAuth(*rpcuser, *rpcpass) - r, err := c.Do(req) - if err != nil { - return nil, err - } - defer r.Body.Close() - if r.StatusCode != http.StatusOK { - e, err := getErrorFromResponse(r) - if err != nil { - return nil, fmt.Errorf("%v", r.Status) - } - return nil, fmt.Errorf("%v: %v", r.Status, e) - } - - bodyBytes := util.ConvertBodyToByteArray(r.Body, *printJson) - - var ir v1.InventoryReply - err = json.Unmarshal(bodyBytes, &ir) - if err != nil { - return nil, fmt.Errorf("Could node unmarshal "+ - "InventoryReply: %v", err) - } - - // Fetch remote identity - id, err := identity.LoadPublicIdentity(*identityFilename) - if err != nil { - return nil, err - } - - err = util.VerifyChallenge(id, challenge, ir.Response) - if err != nil { - return nil, err - } - - return &ir, nil -} - -func inventory() error { - flags := flag.Args()[1:] // Chop off action. - if len(flags) < 2 { - return fmt.Errorf("vetted and branches counts expected") - } - - i, err := remoteInventory() - if err != nil { - return err - } - - if !*printJson { - for _, v := range i.Vetted { - printRecord("Vetted record", v) - } - for _, v := range i.Branches { - printRecord("Unvetted record", v) - } - } - - return nil -} - func getFile(filename string) (*v1.File, *[sha256.Size]byte, error) { var err error @@ -964,29 +879,32 @@ func getUnvetted() error { return err } - // Verify status - if reply.Record.Status == v1.RecordStatusInvalid || - reply.Record.Status == v1.RecordStatusNotFound { - // Pretty print record + switch reply.Record.Status { + case v1.RecordStatusInvalid, v1.RecordStatusNotFound: status, ok := v1.RecordStatus[reply.Record.Status] if !ok { status = v1.RecordStatus[v1.RecordStatusInvalid] } fmt.Printf("Record : %v\n", flags[0]) fmt.Printf(" Status : %v\n", status) - return nil - } - - // Verify content - err = v1.Verify(*id, reply.Record.CensorshipRecord, - reply.Record.Files) - if err != nil { - return err + case v1.RecordStatusCensored: + // Censored records will not contain any file so the verification + // is skipped. + if !*printJson { + printRecord("Unvetted record", reply.Record) + } + default: + // Verify content + err = v1.Verify(*id, reply.Record.CensorshipRecord, + reply.Record.Files) + if err != nil { + return err + } + if !*printJson { + printRecord("Unvetted record", reply.Record) + } } - if !*printJson { - printRecord("Unvetted record", reply.Record) - } return nil } @@ -1065,29 +983,32 @@ func getVetted() error { return err } - // Verify status - if reply.Record.Status == v1.RecordStatusInvalid || - reply.Record.Status == v1.RecordStatusNotFound { - // Pretty print record + switch reply.Record.Status { + case v1.RecordStatusInvalid, v1.RecordStatusNotFound: status, ok := v1.RecordStatus[reply.Record.Status] if !ok { status = v1.RecordStatus[v1.RecordStatusInvalid] } - fmt.Printf("Record : %v\n", flags[0]) - fmt.Printf(" Status : %v\n", status) - return nil - } - - // Verify content - err = v1.Verify(*id, reply.Record.CensorshipRecord, - reply.Record.Files) - if err != nil { - return err + fmt.Printf("Record : %v\n", flags[0]) + fmt.Printf(" Status : %v\n", status) + case v1.RecordStatusCensored: + // Censored records will not contain any file so the verification + // is skipped. + if !*printJson { + printRecord("Vetted record", reply.Record) + } + default: + // Verify content + err = v1.Verify(*id, reply.Record.CensorshipRecord, + reply.Record.Files) + if err != nil { + return err + } + if !*printJson { + printRecord("Vetted record", reply.Record) + } } - if !*printJson { - printRecord("Vetted record", reply.Record) - } return nil } @@ -1288,8 +1209,6 @@ func _main() error { // TODO case "setvettedstatus": case "getvetted": return getVetted() - case "inventory": - return inventory() case "plugin": return plugin() case "plugininventory": diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 451c6a687..700d97393 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -263,7 +263,7 @@ func (p *politeia) newRecord(w http.ResponseWriter, r *http.Request) { challenge, err := hex.DecodeString(t.Challenge) if err != nil || len(challenge) != v1.ChallengeSize { - log.Errorf("%v newRecord: invalid challenge", remoteAddr(r)) + log.Infof("%v newRecord: invalid challenge", remoteAddr(r)) p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) return } @@ -277,7 +277,7 @@ func (p *politeia) newRecord(w http.ResponseWriter, r *http.Request) { // Check for content error. var contentErr backend.ContentVerificationError if errors.As(err, &contentErr) { - log.Errorf("%v New record content error: %v", + log.Infof("%v New record content error: %v", remoteAddr(r), contentErr) p.respondWithUserError(w, contentErr.ErrorCode, contentErr.ErrorContext) @@ -337,7 +337,7 @@ func (p *politeia) updateRecord(w http.ResponseWriter, r *http.Request, vetted b challenge, err := hex.DecodeString(t.Challenge) if err != nil || len(challenge) != v1.ChallengeSize { - log.Errorf("%v update %v record: invalid challenge", + log.Infof("%v update %v record: invalid challenge", remoteAddr(r), cmd) p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) return @@ -368,21 +368,21 @@ func (p *politeia) updateRecord(w http.ResponseWriter, r *http.Request, vetted b } if err != nil { if errors.Is(err, backend.ErrRecordFound) { - log.Errorf("%v update %v record found: %x", + log.Infof("%v update %v record found: %x", remoteAddr(r), cmd, token) p.respondWithUserError(w, v1.ErrorStatusRecordFound, nil) return } if errors.Is(err, backend.ErrRecordNotFound) { - log.Errorf("%v update %v record not found: %x", + log.Infof("%v update %v record not found: %x", remoteAddr(r), cmd, token) p.respondWithUserError(w, v1.ErrorStatusRecordFound, nil) return } if errors.Is(err, backend.ErrNoChanges) { - log.Errorf("%v update %v record no changes: %x", + log.Infof("%v update %v record no changes: %x", remoteAddr(r), cmd, token) p.respondWithUserError(w, v1.ErrorStatusNoChanges, nil) return @@ -390,7 +390,7 @@ func (p *politeia) updateRecord(w http.ResponseWriter, r *http.Request, vetted b // Check for content error. var contentErr backend.ContentVerificationError if errors.As(err, &contentErr) { - log.Errorf("%v update %v record content error: %v", + log.Infof("%v update %v record content error: %v", remoteAddr(r), cmd, contentErr) p.respondWithUserError(w, contentErr.ErrorCode, contentErr.ErrorContext) @@ -462,19 +462,30 @@ func (p *politeia) getUnvetted(w http.ResponseWriter, r *http.Request) { // Ask backend about the censorship token. bpr, err := p.backend.GetUnvetted(token, t.Version) - if errors.Is(err, backend.ErrRecordNotFound) { + switch { + case errors.Is(err, backend.ErrRecordNotFound): + // Record not found reply.Record.Status = v1.RecordStatusNotFound - log.Errorf("Get unvetted record %v: token %v not found", + log.Infof("Get unvetted record %v: token %v not found", remoteAddr(r), t.Token) - } else if err != nil { - // Generic internal error. + + case err != nil: + // Generic internal error errorCode := time.Now().Unix() log.Errorf("%v Get unvetted record error code %v: %v", remoteAddr(r), errorCode, err) - p.respondWithServerError(w, errorCode) return - } else { + + case bpr.RecordMetadata.Status == backend.MDStatusCensored: + // Record has been censored. The default case will verify the + // record before sending it off. This will fail for censored + // records since the files will not exist, they've been deleted, + // so skip the verification step. + reply.Record = p.convertBackendRecord(*bpr) + log.Infof("Get unvetted record %v: token %v", remoteAddr(r), t.Token) + + default: reply.Record = p.convertBackendRecord(*bpr) // Double check record bits before sending them off @@ -486,13 +497,11 @@ func (p *politeia) getUnvetted(w http.ResponseWriter, r *http.Request) { log.Errorf("%v Get unvetted record CORRUPTION "+ "error code %v: %v", remoteAddr(r), errorCode, err) - p.respondWithServerError(w, errorCode) return } - log.Infof("Get unvetted record %v: token %v", remoteAddr(r), - t.Token) + log.Infof("Get unvetted record %v: token %v", remoteAddr(r), t.Token) } util.RespondWithJSON(w, http.StatusOK, reply) @@ -526,19 +535,30 @@ func (p *politeia) getVetted(w http.ResponseWriter, r *http.Request) { // Ask backend about the censorship token. bpr, err := p.backend.GetVetted(token, t.Version) - if errors.Is(err, backend.ErrRecordNotFound) { + switch { + case errors.Is(err, backend.ErrRecordNotFound): + // Record not found reply.Record.Status = v1.RecordStatusNotFound - log.Errorf("Get vetted record %v: token %v not found", + log.Infof("Get vetted record %v: token %v not found", remoteAddr(r), t.Token) - } else if err != nil { - // Generic internal error. + + case err != nil: + // Generic internal error errorCode := time.Now().Unix() log.Errorf("%v Get vetted record error code %v: %v", remoteAddr(r), errorCode, err) - p.respondWithServerError(w, errorCode) return - } else { + + case bpr.RecordMetadata.Status == backend.MDStatusCensored: + // Record has been censored. The default case will verify the + // record before sending it off. This will fail for censored + // records since the files will not exist, they've been deleted, + // so skip the verification step. + reply.Record = p.convertBackendRecord(*bpr) + log.Infof("Get vetted record %v: token %v", remoteAddr(r), t.Token) + + default: reply.Record = p.convertBackendRecord(*bpr) // Double check record bits before sending them off @@ -550,12 +570,11 @@ func (p *politeia) getVetted(w http.ResponseWriter, r *http.Request) { log.Errorf("%v Get vetted record CORRUPTION "+ "error code %v: %v", remoteAddr(r), errorCode, err) - p.respondWithServerError(w, errorCode) return } - log.Infof("Get vetted record %v: token %v", remoteAddr(r), - t.Token) + + log.Infof("Get vetted record %v: token %v", remoteAddr(r), t.Token) } util.RespondWithJSON(w, http.StatusOK, reply) @@ -660,7 +679,7 @@ func (p *politeia) auth(fn http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { user, pass, ok := r.BasicAuth() if !ok || !p.check(user, pass) { - log.Errorf("%v Unauthorized access for: %v", + log.Infof("%v Unauthorized access for: %v", remoteAddr(r), user) w.Header().Set("WWW-Authenticate", `Basic realm="Politeiad"`) @@ -704,7 +723,7 @@ func (p *politeia) setVettedStatus(w http.ResponseWriter, r *http.Request) { if err != nil { // Check for specific errors if errors.Is(err, backend.ErrRecordNotFound) { - log.Errorf("%v updateStatus record not "+ + log.Infof("%v updateStatus record not "+ "found: %x", remoteAddr(r), token) p.respondWithUserError(w, v1.ErrorStatusRecordFound, nil) @@ -712,7 +731,7 @@ func (p *politeia) setVettedStatus(w http.ResponseWriter, r *http.Request) { } var serr backend.StateTransitionError if errors.As(err, &serr) { - log.Errorf("%v %v %v", remoteAddr(r), t.Token, err) + log.Infof("%v %v %v", remoteAddr(r), t.Token, err) p.respondWithUserError(w, v1.ErrorStatusInvalidRecordStatusTransition, nil) return } @@ -777,7 +796,7 @@ func (p *politeia) setUnvettedStatus(w http.ResponseWriter, r *http.Request) { if err != nil { // Check for specific errors if errors.Is(err, backend.ErrRecordNotFound) { - log.Errorf("%v updateUnvettedStatus record not "+ + log.Infof("%v updateUnvettedStatus record not "+ "found: %x", remoteAddr(r), token) p.respondWithUserError(w, v1.ErrorStatusRecordFound, nil) @@ -785,7 +804,7 @@ func (p *politeia) setUnvettedStatus(w http.ResponseWriter, r *http.Request) { } var serr backend.StateTransitionError if errors.As(err, &serr) { - log.Errorf("%v %v %v", remoteAddr(r), t.Token, err) + log.Infof("%v %v %v", remoteAddr(r), t.Token, err) p.respondWithUserError(w, v1.ErrorStatusInvalidRecordStatusTransition, nil) return } @@ -850,7 +869,7 @@ func (p *politeia) updateVettedMetadata(w http.ResponseWriter, r *http.Request) convertFrontendMetadataStream(t.MDOverwrite)) if err != nil { if errors.Is(err, backend.ErrNoChanges) { - log.Errorf("%v update vetted metadata no changes: %x", + log.Infof("%v update vetted metadata no changes: %x", remoteAddr(r), token) p.respondWithUserError(w, v1.ErrorStatusNoChanges, nil) return @@ -858,7 +877,7 @@ func (p *politeia) updateVettedMetadata(w http.ResponseWriter, r *http.Request) // Check for content error. var contentErr backend.ContentVerificationError if errors.As(err, &contentErr) { - log.Errorf("%v update vetted metadata content error: %v", + log.Infof("%v update vetted metadata content error: %v", remoteAddr(r), contentErr) p.respondWithUserError(w, contentErr.ErrorCode, contentErr.ErrorContext) @@ -920,14 +939,14 @@ func (p *politeia) updateUnvettedMetadata(w http.ResponseWriter, r *http.Request if err != nil { // Reply with error if there were no changes if err == backend.ErrNoChanges { - log.Errorf("%v update unvetted metadata no changes: %x", + log.Infof("%v update unvetted metadata no changes: %x", remoteAddr(r), token) p.respondWithUserError(w, v1.ErrorStatusNoChanges, nil) return } // Check for content error. if contentErr, ok := err.(backend.ContentVerificationError); ok { - log.Errorf("%v update unvetted metadata content error: %v", + log.Infof("%v update unvetted metadata content error: %v", remoteAddr(r), contentErr) p.respondWithUserError(w, contentErr.ErrorCode, contentErr.ErrorContext) From 04cc6a2130f709d620d665586d0c8411952c632d Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Fri, 2 Oct 2020 10:27:28 -0300 Subject: [PATCH 124/449] politeia: Add updateunvettedmd command. --- politeiad/cmd/politeia/README.md | 42 ++++++++ politeiad/cmd/politeia/politeia.go | 165 ++++++++++++++++++++++++----- 2 files changed, 183 insertions(+), 24 deletions(-) diff --git a/politeiad/cmd/politeia/README.md b/politeiad/cmd/politeia/README.md index 3afcb7eed..61d93dd19 100644 --- a/politeiad/cmd/politeia/README.md +++ b/politeiad/cmd/politeia/README.md @@ -98,6 +98,26 @@ Update record: 9dfe084fccb7f27c0000 ## Update unvetted metadata only +Metadata can be updated using the arguments: +`'appendmetadata[ID]:[metadataJSON]'` +`'overwritemetadata[ID]:[metadataJSON]'` + +The token is specified using the argument: +`token:[token]` + +Metadata provided using the `overwritemetadata` argument does not have to +already exist. + +``` +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass updateunvettedmd \ + 'appendmetadata12:{"foo":"bar"}' 'overwritemetadata2:{"12foo":"12bar"}' \ + token:0e4a82a370228b710000 + +Update record: 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 + Metadata overwrite: 2 + Metadata append : 12 +``` + ## Set unvetted status You can update the status of an unvetted record using one of the following @@ -171,6 +191,28 @@ Metadata can be updated using the arguments: The token is specified using the argument: `token:[token]` +Metadata provided using the `overwritemetadata` argument does not have to +already exist. + +``` +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass updatevettedmd \ + 'appendmetadata12:{"foo":"bar"}' 'overwritemetadata2:{"12foo":"12bar"}' \ + token:0e4a82a370228b710000 + +Update record: 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 + Metadata overwrite: 2 + Metadata append : 12 +``` + +## Set vetted status + +Metadata can be updated using the arguments: +`'appendmetadata[ID]:[metadataJSON]'` +`'overwritemetadata[ID]:[metadataJSON]'` + +The token is specified using the argument: +`token:[token]` + Metadata provided using the `overwritemetadata` argument does not have to already exist. ``` diff --git a/politeiad/cmd/politeia/politeia.go b/politeiad/cmd/politeia/politeia.go index a57b711d6..2ce457307 100644 --- a/politeiad/cmd/politeia/politeia.go +++ b/politeiad/cmd/politeia/politeia.go @@ -79,6 +79,8 @@ func usage() { fmt.Fprintf(os.Stderr, " updateunvetted - Update unvetted record "+ "[actionmdid:metadata]... ... "+ "token:\n") + fmt.Fprintf(os.Stderr, " updateunvettedmd - Update unvetted record "+ + "metadata [actionmdid:metadata]... token:\n") fmt.Fprintf(os.Stderr, " updatevetted - Update vetted record "+ "[actionmdid:metadata]... ... "+ "token:\n") @@ -512,19 +514,10 @@ func newRecord() error { return nil } -func updateVettedMetadata() error { - flags := flag.Args()[1:] // Chop off action. - - // Create New command - challenge, err := util.Random(v1.ChallengeSize) - if err != nil { - return err - } - n := v1.UpdateVettedMetadata{ - Challenge: hex.EncodeToString(challenge), - } - - // Fish out metadata records and filenames +func validateMetadataFlags(flags []string) ([]v1.MetadataStream, []v1.MetadataStream, string, error) { + var mdAppend []v1.MetadataStream + var mdOverwrite []v1.MetadataStream + var token string var tokenCount uint for _, v := range flags { switch { @@ -533,9 +526,9 @@ func updateVettedMetadata() error { i, err := strconv.ParseUint(regexMDID.FindString(s), 10, 64) if err != nil { - return err + return nil, nil, "", err } - n.MDAppend = append(n.MDAppend, v1.MetadataStream{ + mdAppend = append(mdAppend, v1.MetadataStream{ ID: i, Payload: v[len(s):], }) @@ -545,30 +538,56 @@ func updateVettedMetadata() error { i, err := strconv.ParseUint(regexMDID.FindString(s), 10, 64) if err != nil { - return err + return nil, nil, "", err } - n.MDOverwrite = append(n.MDOverwrite, v1.MetadataStream{ + mdOverwrite = append(mdOverwrite, v1.MetadataStream{ ID: i, Payload: v[len(s):], }) case regexToken.MatchString(v): if tokenCount != 0 { - return fmt.Errorf("only 1 token allowed") + return nil, nil, "", fmt.Errorf("only 1 token allowed") } s := regexToken.FindString(v) - n.Token = v[len(s):] + token = v[len(s):] tokenCount++ default: - return fmt.Errorf("invalid action %v", v) + return nil, nil, "", fmt.Errorf("invalid action %v", v) } } if tokenCount != 1 { - return fmt.Errorf("must provide token") + return nil, nil, "", fmt.Errorf("must provide token") + } + + return mdAppend, mdOverwrite, token, nil +} + +func updateVettedMetadata() error { + flags := flag.Args()[1:] // Chop off action. + + // Create New command + challenge, err := util.Random(v1.ChallengeSize) + if err != nil { + return err + } + n := v1.UpdateVettedMetadata{ + Challenge: hex.EncodeToString(challenge), } + // Fish out metadata records and filenames + mdAppend, mdOverwrite, token, err := validateMetadataFlags(flags) + if err != nil { + return err + } + + // Set request fields + n.MDAppend = mdAppend + n.MDOverwrite = mdOverwrite + n.Token = token + // Fetch remote identity id, err := identity.LoadPublicIdentity(*identityFilename) if err != nil { @@ -579,7 +598,7 @@ func updateVettedMetadata() error { if *verbose { fmt.Printf("Update vetted metadata: %v\n", n.Token) if len(n.MDOverwrite) > 0 { - s := " Metadata overwrite: " + s := "Metadata overwrite: " for _, v := range n.MDOverwrite { fmt.Printf("%s%v", s, v.ID) s = ", " @@ -587,7 +606,7 @@ func updateVettedMetadata() error { fmt.Printf("\n") } if len(n.MDAppend) > 0 { - s := " Metadata append : " + s := "Metadata append: " for _, v := range n.MDAppend { fmt.Printf("%s%v", s, v.ID) s = ", " @@ -642,6 +661,103 @@ func updateVettedMetadata() error { return util.VerifyChallenge(id, challenge, reply.Response) } +func updateUnvettedMetadata() error { + flags := flag.Args()[1:] + + // Create new command + challenge, err := util.Random(v1.ChallengeSize) + if err != nil { + return err + } + uum := v1.UpdateUnvettedMetadata{ + Challenge: hex.EncodeToString(challenge), + } + + // Fish out metadata records and filenames from flags + mdAppend, mdOverwrite, token, err := validateMetadataFlags(flags) + if err != nil { + return err + } + + // Set request fields + uum.MDAppend = mdAppend + uum.MDOverwrite = mdOverwrite + uum.Token = token + + // Prety print + if *verbose { + fmt.Printf("Update unvetted metadata: %v\n", uum.Token) + if len(uum.MDOverwrite) > 0 { + s := "Metadata overwrite: " + for _, v := range uum.MDOverwrite { + fmt.Printf("%s%v", s, v.ID) + s = ", " + } + fmt.Printf("\n") + } + if len(uum.MDAppend) > 0 { + s := "Metadata append: " + for _, v := range uum.MDAppend { + fmt.Printf("%s%v", s, v.ID) + s = ", " + } + fmt.Printf("\n") + } + } + + // Convert request object to JSON + b, err := json.Marshal(uum) + if err != nil { + return err + } + if *printJson { + fmt.Println(string(b)) + } + + // Make request + c, err := util.NewClient(verify, *rpccert) + if err != nil { + return err + } + req, err := http.NewRequest("POST", *rpchost+v1.UpdateUnvettedMetadataRoute, + bytes.NewReader(b)) + if err != nil { + return err + } + req.SetBasicAuth(*rpcuser, *rpcpass) + r, err := c.Do(req) + if err != nil { + return err + } + defer r.Body.Close() + + // Verify status code response + if r.StatusCode != http.StatusOK { + e, err := getErrorFromResponse(r) + if err != nil { + return fmt.Errorf("%v", r.Status) + } + return fmt.Errorf("%v: %v", r.Status, e) + } + + // Fetch remote identity + id, err := identity.LoadPublicIdentity(*identityFilename) + if err != nil { + return err + } + + // Prepare reply + bodyBytes := util.ConvertBodyToByteArray(r.Body, *printJson) + var reply v1.UpdateUnvettedMetadataReply + err = json.Unmarshal(bodyBytes, &reply) + if err != nil { + return fmt.Errorf("Could node unmarshal UpdateReply: %v", err) + } + + // Verify challenge + return util.VerifyChallenge(id, challenge, reply.Response) +} + func updateRecord(vetted bool) error { flags := flag.Args()[1:] // Chop off action. @@ -1197,7 +1313,8 @@ func _main() error { return newRecord() case "updateunvetted": return updateRecord(false) - // TODO case "updateunvettedmd" + case "updateunvettedmd": + return updateUnvettedMetadata() case "setunvettedstatus": return setUnvettedStatus() case "getunvetted": From 72abd1a3f2ec221ad222d6fe2bc0297a1fc28cf7 Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Fri, 2 Oct 2020 11:18:09 -0300 Subject: [PATCH 125/449] politeia: Add setvettedstatus command. --- politeiad/cmd/politeia/README.md | 27 ++++-- politeiad/cmd/politeia/politeia.go | 149 ++++++++++++++++++++++++++++- 2 files changed, 165 insertions(+), 11 deletions(-) diff --git a/politeiad/cmd/politeia/README.md b/politeiad/cmd/politeia/README.md index 61d93dd19..970149b7c 100644 --- a/politeiad/cmd/politeia/README.md +++ b/politeiad/cmd/politeia/README.md @@ -122,13 +122,15 @@ Update record: 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 You can update the status of an unvetted record using one of the following statuses: -- `publish` - make the record a public, vetted record. -- `censor` - keep the record unvetted and mark as censored. +- `censored` - keep the record unvetted and mark as censored. +- `public` - make the record a public, vetted record. +- `archived` - archive the record. -Note `token:` is not prefixed to the token in this command. +Note `token:` is not prefixed to the token in this command. Status change +validation is done in the backend. ``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass setunvettedstatus publish 9dfe084fccb7f27c0000 +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass setunvettedstatus public 0e4a82a370228b710000 Set record status: Status : public @@ -221,7 +223,20 @@ $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass updatevett Update vetted metadata: 9dfe084fccb7f27c0000 Metadata append : 12 -` +``` ## Set vetted status -TODO + +You can update the status of a vetted record using one of the following +statuses: +- `censored` - keep the record unvetted and mark as censored. +- `archived` - archive the record. + +Note `token:` is not prefixed to the token in this command. Status change +validation is done in the backend. + +``` +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass setvettedstatus censored 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 'overwritemetadata12:"zap"' +Set record status: + Status: censor +``` diff --git a/politeiad/cmd/politeia/politeia.go b/politeiad/cmd/politeia/politeia.go index 2ce457307..17defd1be 100644 --- a/politeiad/cmd/politeia/politeia.go +++ b/politeiad/cmd/politeia/politeia.go @@ -75,7 +75,8 @@ func usage() { fmt.Fprintf(os.Stderr, " getunvetted - Retrieve record "+ "\n") fmt.Fprintf(os.Stderr, " setunvettedstatus - Set unvetted record "+ - "status [actionmdid:metadata]...\n") + "status "+ + "[actionmdid:metadata]...\n") fmt.Fprintf(os.Stderr, " updateunvetted - Update unvetted record "+ "[actionmdid:metadata]... ... "+ "token:\n") @@ -86,6 +87,9 @@ func usage() { "token:\n") fmt.Fprintf(os.Stderr, " updatevettedmd - Update vetted record "+ "metadata [actionmdid:metadata]... token:\n") + fmt.Fprintf(os.Stderr, " setvettedstatus - Set vetted record "+ + "status "+ + "[actionmdid:metadata]...\n") fmt.Fprintf(os.Stderr, "\n") fmt.Fprintf(os.Stderr, " metadata is the word metadata followed "+ "by digits. Example with 2 metadata records "+ @@ -1130,10 +1134,12 @@ func getVetted() error { func convertStatus(s string) (v1.RecordStatusT, error) { switch s { - case "censor": + case "censored": return v1.RecordStatusCensored, nil - case "publish": + case "public": return v1.RecordStatusPublic, nil + case "archived": + return v1.RecordStatusArchived, nil } return v1.RecordStatusInvalid, fmt.Errorf("invalid status") @@ -1263,7 +1269,139 @@ func setUnvettedStatus() error { if !ok { status = v1.RecordStatus[v1.RecordStatusInvalid] } - fmt.Printf("Set record status:\n") + fmt.Printf("Set unvetted record status:\n") + fmt.Printf(" Status : %v\n", status) + } + + return nil +} + +func setVettedStatus() error { + flags := flag.Args()[1:] + + // Make sure we have the status and the censorship token + if len(flags) < 2 { + return fmt.Errorf("must at least provide status and " + + "censorship token") + } + + // Validate status + status, err := convertStatus(flags[0]) + if err != nil { + return err + } + + // Validate censorship token + _, err = util.ConvertStringToken(flags[1]) + if err != nil { + return err + } + + // Create command + challenge, err := util.Random(v1.ChallengeSize) + if err != nil { + return err + } + sus := v1.SetVettedStatus{ + Challenge: hex.EncodeToString(challenge), + Status: status, + Token: flags[1], + } + + // Optional metadata updates + for _, md := range flags[2:] { + switch { + case regexAppendMD.MatchString(md): + s := regexAppendMD.FindString(md) + i, err := strconv.ParseUint(regexMDID.FindString(s), + 10, 64) + if err != nil { + return err + } + sus.MDAppend = append(sus.MDAppend, v1.MetadataStream{ + ID: i, + Payload: md[len(s):], + }) + + case regexOverwriteMD.MatchString(md): + s := regexOverwriteMD.FindString(md) + i, err := strconv.ParseUint(regexMDID.FindString(s), + 10, 64) + if err != nil { + return err + } + sus.MDOverwrite = append(sus.MDOverwrite, v1.MetadataStream{ + ID: i, + Payload: md[len(s):], + }) + default: + return fmt.Errorf("invalid metadata action %v", md) + } + } + + // Convert command object to JSON + b, err := json.Marshal(sus) + if err != nil { + return err + } + if *printJson { + fmt.Println(string(b)) + } + + // Make request + c, err := util.NewClient(verify, *rpccert) + if err != nil { + return err + } + req, err := http.NewRequest("POST", *rpchost+v1.SetVettedStatusRoute, + bytes.NewReader(b)) + if err != nil { + return err + } + req.SetBasicAuth(*rpcuser, *rpcpass) + r, err := c.Do(req) + if err != nil { + return err + } + defer r.Body.Close() + + // Verify status code response + if r.StatusCode != http.StatusOK { + e, err := getErrorFromResponse(r) + if err != nil { + return fmt.Errorf("%v", r.Status) + } + return fmt.Errorf("%v: %v", r.Status, e) + } + + // Prepare reply + bodyBytes := util.ConvertBodyToByteArray(r.Body, *printJson) + var reply v1.SetVettedStatusReply + err = json.Unmarshal(bodyBytes, &reply) + if err != nil { + return fmt.Errorf("Could not unmarshal "+ + "SetVettedStatusReply: %v", err) + } + + // Fetch remote identity + id, err := identity.LoadPublicIdentity(*identityFilename) + if err != nil { + return err + } + + // Verify challenge. + err = util.VerifyChallenge(id, challenge, reply.Response) + if err != nil { + return err + } + + if !*printJson { + // Pretty print record + status, ok := v1.RecordStatus[sus.Status] + if !ok { + status = v1.RecordStatus[v1.RecordStatusInvalid] + } + fmt.Printf("Set vetted record status:\n") fmt.Printf(" Status : %v\n", status) } @@ -1323,7 +1461,8 @@ func _main() error { return updateRecord(true) case "updatevettedmd": return updateVettedMetadata() - // TODO case "setvettedstatus": + case "setvettedstatus": + return setVettedStatus() case "getvetted": return getVetted() case "plugin": From 1dce1c77a0772abfb9401a5361174d39ef479bbe Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Fri, 2 Oct 2020 18:27:35 -0300 Subject: [PATCH 126/449] politeia: Add inventory by status command. --- politeiad/cmd/politeia/README.md | 40 +++++++-------- politeiad/cmd/politeia/politeia.go | 82 +++++++++++++++++++++++++++++- 2 files changed, 99 insertions(+), 23 deletions(-) diff --git a/politeiad/cmd/politeia/README.md b/politeiad/cmd/politeia/README.md index 970149b7c..d9f70256b 100644 --- a/politeiad/cmd/politeia/README.md +++ b/politeiad/cmd/politeia/README.md @@ -11,7 +11,8 @@ Available commands: `updatevettedmd` `getvetted` `plugin` -`plugininventory` +`plugininventory` +`inventory` ## Obtain politeiad identity @@ -208,25 +209,6 @@ Update record: 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 ## Set vetted status -Metadata can be updated using the arguments: -`'appendmetadata[ID]:[metadataJSON]'` -`'overwritemetadata[ID]:[metadataJSON]'` - -The token is specified using the argument: -`token:[token]` - -Metadata provided using the `overwritemetadata` argument does not have to -already exist. -``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass updatevettedmd \ - 'appendmetadata12:{"foo":"bar"}' token:9dfe084fccb7f27c0000 - -Update vetted metadata: 9dfe084fccb7f27c0000 - Metadata append : 12 -``` - -## Set vetted status - You can update the status of a vetted record using one of the following statuses: - `censored` - keep the record unvetted and mark as censored. @@ -238,5 +220,21 @@ validation is done in the backend. ``` $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass setvettedstatus censored 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 'overwritemetadata12:"zap"' Set record status: - Status: censor + Status: censored +``` + +## Inventory by status + +The `inventory` command retrieves the censorship record tokens from all records, +separated by their status. + +``` +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass inventory + +Inventory by status: + Unvetted : [tokens...] + IterationUnvetted: [tokens...] + Vetted : [tokens...] + Censored : [tokens...] + Archived : [tokens...] ``` diff --git a/politeiad/cmd/politeia/politeia.go b/politeiad/cmd/politeia/politeia.go index 17defd1be..57d0d06f0 100644 --- a/politeiad/cmd/politeia/politeia.go +++ b/politeiad/cmd/politeia/politeia.go @@ -68,8 +68,8 @@ func usage() { "identity\n") fmt.Fprintf(os.Stderr, " plugins - Retrieve plugin "+ "inventory\n") - fmt.Fprintf(os.Stderr, " inventory - Inventory records "+ - " \n") + fmt.Fprintf(os.Stderr, " inventory - Inventory records by "+ + "status\n") fmt.Fprintf(os.Stderr, " new - Create new record "+ "[metadata]... ...\n") fmt.Fprintf(os.Stderr, " getunvetted - Retrieve record "+ @@ -377,6 +377,82 @@ func getFile(filename string) (*v1.File, *[sha256.Size]byte, error) { return file, &digest32, nil } +func recordInventory() error { + // Prepare request + challenge, err := util.Random(v1.ChallengeSize) + if err != nil { + return err + } + + ibs, err := json.Marshal(v1.InventoryByStatus{ + Challenge: hex.EncodeToString(challenge), + }) + if err != nil { + return err + } + + if *printJson { + fmt.Println(string(ibs)) + } + + // Make request + c, err := util.NewClient(verify, *rpccert) + if err != nil { + return err + } + req, err := http.NewRequest("POST", *rpchost+v1.InventoryByStatusRoute, + bytes.NewReader(ibs)) + if err != nil { + return err + } + req.SetBasicAuth(*rpcuser, *rpcpass) + r, err := c.Do(req) + if err != nil { + return err + } + defer r.Body.Close() + + // Verify status code response + if r.StatusCode != http.StatusOK { + e, err := getErrorFromResponse(r) + if err != nil { + return fmt.Errorf("%v", r.Status) + } + return fmt.Errorf("%v: %v", r.Status, e) + } + + bodyBytes := util.ConvertBodyToByteArray(r.Body, *printJson) + + var ibsr v1.InventoryByStatusReply + err = json.Unmarshal(bodyBytes, &ibsr) + if err != nil { + return fmt.Errorf("Could node unmarshal "+ + "InventoryByStatusReply: %v", err) + } + + // Fetch remote identity + id, err := identity.LoadPublicIdentity(*identityFilename) + if err != nil { + return err + } + + // Verify challenge + err = util.VerifyChallenge(id, challenge, ibsr.Response) + if err != nil { + return err + } + + // Print response to user + fmt.Printf("Inventory by status:\n") + fmt.Printf(" Unvetted : %v\n", ibsr.Unvetted) + fmt.Printf(" IterationUnvetted: %v\n", ibsr.IterationUnvetted) + fmt.Printf(" Vetted : %v\n", ibsr.Vetted) + fmt.Printf(" Censored : %v\n", ibsr.Censored) + fmt.Printf(" Archived : %v\n", ibsr.Archived) + + return nil +} + func newRecord() error { flags := flag.Args()[1:] // Chop off action. @@ -1469,6 +1545,8 @@ func _main() error { return plugin() case "plugininventory": return getPluginInventory() + case "inventory": + return recordInventory() default: return fmt.Errorf("invalid action: %v", a) } From 85663bc7c8b0222b263a23e1ec38a1a4e75f53e6 Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 2 Oct 2020 16:30:25 -0500 Subject: [PATCH 127/449] cleanup and bug fixes --- plugins/pi/pi.go | 3 +- politeiad/api/v1/v1.go | 4 +- politeiad/backend/tlogbe/pi.go | 122 ++++++++++++--- politeiad/backend/tlogbe/tlog.go | 4 +- politeiad/politeiad.go | 2 +- politeiawww/cmd/cmswww/batchproposals.go | 131 +++++++++++++++- politeiawww/cmd/cmswww/cmswww.go | 11 +- politeiawww/cmd/piwww/help.go | 35 ++--- politeiawww/cmd/piwww/piwww.go | 187 +++++++---------------- politeiawww/cmd/piwww/proposaledit.go | 11 +- politeiawww/cmd/piwww/proposalnew.go | 2 - politeiawww/cmd/piwww/proposals.go | 31 +++- politeiawww/cmd/piwww/util.go | 99 ++++++++++++ politeiawww/cmd/piwww/votedetails.go | 50 ------ politeiawww/cmd/piwww/votestatus.go | 41 ----- politeiawww/cmd/piwww/votestatuses.go | 36 ----- politeiawww/cmd/shared/shared.go | 95 ------------ politeiawww/piwww.go | 113 ++++++-------- politeiawww/politeiad.go | 2 +- politeiawww/util/merkle.go | 33 ---- politeiawww/www.go | 3 +- 21 files changed, 484 insertions(+), 531 deletions(-) create mode 100644 politeiawww/cmd/piwww/util.go delete mode 100644 politeiawww/cmd/piwww/votedetails.go delete mode 100644 politeiawww/cmd/piwww/votestatus.go delete mode 100644 politeiawww/cmd/piwww/votestatuses.go diff --git a/plugins/pi/pi.go b/plugins/pi/pi.go index ffb5207d9..421889a02 100644 --- a/plugins/pi/pi.go +++ b/plugins/pi/pi.go @@ -62,10 +62,11 @@ const ( // User error status codes // TODO number error codes and add human readable error messages ErrorStatusInvalid ErrorStatusT = iota - ErrorStatusPropLinkToInvalid + ErrorStatusPropStateInvalid ErrorStatusPropVersionInvalid ErrorStatusPropStatusInvalid ErrorStatusPropStatusChangeInvalid + ErrorStatusPropLinkToInvalid ErrorStatusVoteStatusInvalid ErrorStatusPageSizeExceeded ) diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index 575bf17d3..4a9ae63d2 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -407,8 +407,8 @@ type UserErrorReply struct { ErrorContext []string `json:"errorcontext,omitempty"` // Additional error information } -// UserErrorReply returns details about a plugin error that occurred while -// trying to execute a command due to bad input from the client. +// PluginUserErrorReply returns details about a plugin error that occurred +// while trying to execute a command due to bad input from the client. type PluginUserErrorReply struct { Plugin string `json:"plugin"` ErrorCode int `json:"errorcode"` diff --git a/politeiad/backend/tlogbe/pi.go b/politeiad/backend/tlogbe/pi.go index 6c05ea227..97330ffea 100644 --- a/politeiad/backend/tlogbe/pi.go +++ b/politeiad/backend/tlogbe/pi.go @@ -17,9 +17,11 @@ import ( "sync" "time" + "github.com/decred/politeia/plugins/comments" "github.com/decred/politeia/plugins/pi" "github.com/decred/politeia/plugins/ticketvote" "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/util" ) const ( @@ -104,8 +106,10 @@ func (p *piPlugin) linkedFromLocked(token string) (*linkedFrom, error) { if err != nil { var e *os.PathError if errors.As(err, &e) && !os.IsExist(err) { - // File does't exist - return nil, errRecordNotFound + // File does't exist. Return an empty linked from list. + return &linkedFrom{ + Tokens: make(map[string]struct{}, 0), + }, nil } } @@ -132,11 +136,6 @@ func (p *piPlugin) linkedFromAdd(parentToken, childToken string) error { // Get existing linked from list lf, err := p.linkedFromLocked(parentToken) if err == errRecordNotFound { - // List doesn't exist. Create a new one. - lf = &linkedFrom{ - Tokens: make(map[string]struct{}, 0), - } - } else if err != nil { return fmt.Errorf("linkedFromLocked %v: %v", parentToken, err) } @@ -185,25 +184,102 @@ func (p *piPlugin) linkedFromDel(parentToken, childToken string) error { } func (p *piPlugin) cmdProposals(payload string) (string, error) { - // TODO + ps, err := pi.DecodeProposals([]byte(payload)) + if err != nil { + return "", err + } + + // Setup the returned map with entries for all tokens that + // correspond to records. + var existsFn func([]byte) bool + switch ps.State { + case pi.PropStateUnvetted: + existsFn = p.backend.UnvettedExists + case pi.PropStateVetted: + existsFn = p.backend.VettedExists + default: + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropStateInvalid), + } + } - /* - // Just because a cached linked from doesn't exist doesn't - // mean the token isn't valid. We need to check if the token - // corresponds to a real proposal. - proposals := make(map[string]pi.ProposalData, len(ps.Tokens)) - for _, v := range ps.Tokens { - lf, err := p.linkedFrom(v) - if err != nil { - if err == errRecordNotFound { - continue - } - return "", fmt.Errorf("linkedFrom %v: %v", v, err) - } + // map[token]ProposalPluginData + proposals := make(map[string]pi.ProposalPluginData, len(ps.Tokens)) + for _, v := range ps.Tokens { + token, err := util.ConvertStringToken(v) + if err != nil { + // Not a valid token + continue + } + ok := existsFn(token) + if !ok { + // Record doesn't exists + continue } - */ - return "", nil + // Record exists. Include it in the reply. + proposals[v] = pi.ProposalPluginData{} + } + + // Get linked from list for each proposal + for k, v := range proposals { + lf, err := p.linkedFrom(k) + if err != nil { + return "", fmt.Errorf("linkedFrom %v: %v", k, err) + } + + // Convert map to a slice + linkedFrom := make([]string, 0, len(lf.Tokens)) + for token := range lf.Tokens { + linkedFrom = append(linkedFrom, token) + } + + v.LinkedFrom = linkedFrom + proposals[k] = v + } + + // Get comments count for each proposal + for k, v := range proposals { + // Prepare plugin command + c := comments.Count{ + State: comments.StateT(ps.State), + Token: k, + } + b, err := comments.EncodeCount(c) + if err != nil { + return "", err + } + + // Send plugin command + reply, err := p.backend.Plugin(comments.ID, + comments.CmdCount, "", string(b)) + if err != nil { + return "", fmt.Errorf("backend Plugin %v %v: %v", + comments.ID, comments.CmdCount, err) + } + + // Decode reply + cr, err := comments.DecodeCountReply([]byte(reply)) + if err != nil { + return "", err + } + + // Update proposal plugin data + v.Comments = cr.Count + proposals[k] = v + } + + // Prepare reply + pr := pi.ProposalsReply{ + Proposals: proposals, + } + reply, err := pi.EncodeProposalsReply(pr) + if err != nil { + return "", err + } + + return string(reply), nil } func (p *piPlugin) cmdCommentNew(payload string) (string, error) { diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index 90f76be90..eefb715a1 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -160,8 +160,8 @@ func treeIDFromToken(token []byte) int64 { func tokenFromTreeID(treeID int64) []byte { b := make([]byte, binary.MaxVarintLen64) - // Converting between int64 and uint64 doesn't change the sign bit, - // only the way it's interpreted. + // Converting between int64 and uint64 doesn't change + // the sign bit, only the way it's interpreted. binary.LittleEndian.PutUint64(b, uint64(treeID)) return b } diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 700d97393..89cfe3b0d 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -1264,7 +1264,7 @@ func _main() error { // Setup plugins // TODO fix this - // loadedCfg.Plugins = []string{"comments", "dcrdata", "pi", "ticketvote"} + loadedCfg.Plugins = []string{"comments", "dcrdata", "pi", "ticketvote"} if len(loadedCfg.Plugins) > 0 { // Set plugin routes. Requires auth. p.addRoute(http.MethodPost, v1.PluginCommandRoute, p.pluginCommand, diff --git a/politeiawww/cmd/cmswww/batchproposals.go b/politeiawww/cmd/cmswww/batchproposals.go index e4572fa53..3eb214245 100644 --- a/politeiawww/cmd/cmswww/batchproposals.go +++ b/politeiawww/cmd/cmswww/batchproposals.go @@ -5,10 +5,16 @@ package main import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/hex" "fmt" + "github.com/decred/dcrtime/merkle" v1 "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/cmd/shared" + "github.com/decred/politeia/util" ) // BatchProposalsCmd retrieves a set of proposals. @@ -32,7 +38,7 @@ func (cmd *BatchProposalsCmd) Execute(args []string) error { // Verify proposal censorship records for _, p := range bpr.Proposals { - err = shared.VerifyProposal(p, vr.PubKey) + err = verifyProposal(p, vr.PubKey) if err != nil { return fmt.Errorf("unable to verify proposal %v: %v", p.CensorshipRecord.Token, err) @@ -75,3 +81,126 @@ Result: } ] }` + +func merkleRoot(files []v1.File, md []v1.Metadata) (string, error) { + digests := make([]*[sha256.Size]byte, 0, len(files)) + + // Calculate file digests + for _, f := range files { + b, err := base64.StdEncoding.DecodeString(f.Payload) + if err != nil { + return "", err + } + digest := util.Digest(b) + var hf [sha256.Size]byte + copy(hf[:], digest) + digests = append(digests, &hf) + } + + // Calculate metadata digests + for _, v := range md { + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return "", err + } + digest := util.Digest(b) + var hv [sha256.Size]byte + copy(hv[:], digest) + digests = append(digests, &hv) + } + + // Return merkle root + return hex.EncodeToString(merkle.Root(digests)[:]), nil +} + +// validateDigests receives a list of files and metadata to verify their +// digests. It compares digests that came with the file/md with digests +// calculated from their respective payloads. +func validateDigests(files []v1.File, md []v1.Metadata) error { + // Validate file digests + for _, f := range files { + b, err := base64.StdEncoding.DecodeString(f.Payload) + if err != nil { + return fmt.Errorf("file: %v decode payload err %v", + f.Name, err) + } + digest := util.Digest(b) + d, ok := util.ConvertDigest(f.Digest) + if !ok { + return fmt.Errorf("file: %v invalid digest %v", + f.Name, f.Digest) + } + if !bytes.Equal(digest, d[:]) { + return fmt.Errorf("file: %v digests do not match", + f.Name) + } + } + // Validate metadata digests + for _, v := range md { + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return fmt.Errorf("metadata: %v decode payload err %v", + v.Hint, err) + } + digest := util.Digest(b) + d, ok := util.ConvertDigest(v.Digest) + if !ok { + return fmt.Errorf("metadata: %v invalid digest %v", + v.Hint, v.Digest) + } + if !bytes.Equal(digest, d[:]) { + return fmt.Errorf("metadata: %v digests do not match metadata", + v.Hint) + } + } + return nil +} + +// verifyProposal verifies a proposal's merkle root, author signature, and +// censorship record. +func verifyProposal(p v1.ProposalRecord, serverPubKey string) error { + if len(p.Files) > 0 { + // Verify digests + err := validateDigests(p.Files, p.Metadata) + if err != nil { + return err + } + // Verify merkle root + mr, err := merkleRoot(p.Files, p.Metadata) + if err != nil { + return err + } + if mr != p.CensorshipRecord.Merkle { + return fmt.Errorf("merkle roots do not match") + } + } + + // Verify proposal signature + pid, err := util.IdentityFromString(p.PublicKey) + if err != nil { + return err + } + sig, err := util.ConvertSignature(p.Signature) + if err != nil { + return err + } + if !pid.VerifyMessage([]byte(p.CensorshipRecord.Merkle), sig) { + return fmt.Errorf("could not verify proposal signature") + } + + // Verify censorship record signature + id, err := util.IdentityFromString(serverPubKey) + if err != nil { + return err + } + s, err := util.ConvertSignature(p.CensorshipRecord.Signature) + if err != nil { + return err + } + msg := []byte(p.CensorshipRecord.Merkle + p.CensorshipRecord.Token) + if !id.VerifyMessage(msg, s) { + return fmt.Errorf("could not verify censorship record signature") + } + + return nil +} diff --git a/politeiawww/cmd/cmswww/cmswww.go b/politeiawww/cmd/cmswww/cmswww.go index 2220f17f0..fe8f6c2cc 100644 --- a/politeiawww/cmd/cmswww/cmswww.go +++ b/politeiawww/cmd/cmswww/cmswww.go @@ -20,7 +20,6 @@ import ( cms "github.com/decred/politeia/politeiawww/api/cms/v1" pi "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/cmd/shared" - wwwutil "github.com/decred/politeia/politeiawww/util" "github.com/decred/politeia/util" flags "github.com/jessevdk/go-flags" ) @@ -119,7 +118,7 @@ func signedMerkleRoot(files []pi.File, md []pi.Metadata, id *identity.FullIdenti if len(files) == 0 { return "", fmt.Errorf("no proposal files found") } - mr, err := wwwutil.MerkleRootWWW(files, md) + mr, err := merkleRoot(files, md) if err != nil { return "", err } @@ -132,12 +131,12 @@ func signedMerkleRoot(files []pi.File, md []pi.Metadata, id *identity.FullIdenti func verifyInvoice(p cms.InvoiceRecord, serverPubKey string) error { if len(p.Files) > 0 { // Verify file digests - err := shared.ValidateDigests(p.Files, nil) + err := validateDigests(p.Files, nil) if err != nil { return err } // Verify merkle root - mr, err := wwwutil.MerkleRootWWW(p.Files, nil) + mr, err := merkleRoot(p.Files, nil) if err != nil { return err } @@ -296,12 +295,12 @@ func verifyDCC(p cms.DCCRecord, serverPubKey string) error { files := make([]pi.File, 0, 1) files = append(files, p.File) // Verify digests - err := shared.ValidateDigests(files, nil) + err := validateDigests(files, nil) if err != nil { return err } // Verify merkel root - mr, err := wwwutil.MerkleRootWWW(files, nil) + mr, err := merkleRoot(files, nil) if err != nil { return err } diff --git a/politeiawww/cmd/piwww/help.go b/politeiawww/cmd/piwww/help.go index c6d58e256..90e0f072a 100644 --- a/politeiawww/cmd/piwww/help.go +++ b/politeiawww/cmd/piwww/help.go @@ -25,6 +25,18 @@ func (cmd *helpCmd) Execute(args []string) error { } switch cmd.Args.Topic { + // Server commands + case "version": + fmt.Printf("%s\n", shared.VersionHelpMsg) + case "policy": + fmt.Printf("%s\n", policyHelpMsg) + case "login": + fmt.Printf("%s\n", shared.LoginHelpMsg) + case "logout": + fmt.Printf("%s\n", shared.LogoutHelpMsg) + case "me": + fmt.Printf("%s\n", shared.MeHelpMsg) + // Proposal commands case "proposalnew": fmt.Printf("%s\n", proposalNewHelpMsg) @@ -67,20 +79,7 @@ func (cmd *helpCmd) Execute(args []string) error { case "voteinventory": fmt.Printf("%s\n", voteInventoryHelpMsg) - case "votedetails": - fmt.Printf("%s\n", voteDetailsHelpMsg) - case "votestatus": - fmt.Printf("%s\n", voteStatusHelpMsg) - case "votestatuses": - fmt.Printf("%s\n", voteStatusesHelpMsg) - - // Server commands - case "version": - fmt.Printf("%s\n", shared.VersionHelpMsg) - case "policy": - fmt.Printf("%s\n", policyHelpMsg) - - // User commands + // User commands case "usernew": fmt.Printf("%s\n", userNewHelpMsg) case "useredit": @@ -113,14 +112,6 @@ func (cmd *helpCmd) Execute(args []string) error { case "proposalpaywall": fmt.Printf("%s\n", proposalPaywallHelpMsg) - // Basic commands - case "login": - fmt.Printf("%s\n", shared.LoginHelpMsg) - case "logout": - fmt.Printf("%s\n", shared.LogoutHelpMsg) - case "me": - fmt.Printf("%s\n", shared.MeHelpMsg) - // Websocket commands case "subscribe": fmt.Printf("%s\n", subscribeHelpMsg) diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index 8da13dda7..8006bd277 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -5,9 +5,6 @@ package main import ( - "encoding/base64" - "encoding/hex" - "encoding/json" "errors" "fmt" "net/url" @@ -15,12 +12,8 @@ import ( flags "github.com/jessevdk/go-flags" - "github.com/decred/dcrd/chaincfg/chainhash" "github.com/decred/dcrd/dcrutil/v3" - "github.com/decred/politeia/politeiad/api/v1/identity" - pi "github.com/decred/politeia/politeiawww/api/pi/v1" "github.com/decred/politeia/politeiawww/cmd/shared" - wwwutil "github.com/decred/politeia/politeiawww/util" ) const ( @@ -45,6 +38,22 @@ type piwww struct { // piwww help message. This is handled by go-flags. Config shared.Config + // Basic commands + Help helpCmd `command:"help"` + + // Server commands + Version shared.VersionCmd `command:"version"` + Policy policyCmd `command:"policy"` + Secret shared.SecretCmd `command:"secret"` + Login shared.LoginCmd `command:"login"` + Logout shared.LogoutCmd `command:"logout"` + Me shared.MeCmd `command:"me"` + + // TODO some of the proposal commands use both the --unvetted and + // --vetted flags. Let make them all use only the --unvetted flag. + // If --unvetted is not included then its assumed to be a vetted + // request. + // TODO replace www policies with pi policies // Proposal commands ProposalNew proposalNewCmd `command:"proposalnew"` ProposalEdit proposalEditCmd `command:"proposaledit"` @@ -53,143 +62,49 @@ type piwww struct { ProposalInventory proposalInventoryCmd `command:"proposalinventory"` // Comments commands - CommentNew commentNewCmd `command:"commentnew" description:"(user) create a new comment"` - CommentVote commentVoteCmd `command:"commentvote" description:"(user) upvote/downvote a comment"` - CommentCensor commentCensorCmd `command:"commentcensor" description:"(admin) censor a comment"` - Comments commentsCmd `command:"comments" description:"(public) get the comments for a proposal"` - CommentVotes commentVotesCmd `command:"commentvotes" description:"(user) get comment upvotes/downvotes for a proposal from the provided user"` + CommentNew commentNewCmd `command:"commentnew"` + CommentVote commentVoteCmd `command:"commentvote"` + CommentCensor commentCensorCmd `command:"commentcensor"` + Comments commentsCmd `command:"comments"` + CommentVotes commentVotesCmd `command:"commentvotes"` // Vote commands - VoteAuthorize voteAuthorizeCmd `command:"voteauthorize" description:"(user) authorize a proposal vote (must be proposal author)"` - VoteStart voteStartCmd `command:"votestart" description:"(admin) start the voting period on a proposal"` - VoteStartRunoff voteStartRunoffCmd `command:"votestartrunoff" description:"(admin) start a runoff using the submissions to an RFP"` - VoteBallot voteBallotCmd `command:"voteballot" description:"(public) cast ballot of votes for a proposal"` - Votes votesCmd `command:"votes" description:"(public) get the vote tally for a proposal"` - VoteResults voteResultsCmd `command:"voteresults" description:"(public) get vote results for a proposal"` - VoteSummaries voteSummariesCmd `command:"votesummaries" description:"(public) retrieve the vote summary for a set of proposals"` - VoteInventory voteInventoryCmd `command:"voteinventory" description:"(public) retrieve the tokens of all public, non-abandoned proposal separated by vote status"` - // XXX www vote routes - VoteDetails voteDetailsCmd `command:"votedetails" description:"(public) get the details for a proposal vote"` - VoteStatus voteStatusCmd `command:"votestatus" description:"(public) get the vote status of a proposal"` - VoteStatuses voteStatusesCmd `command:"votestatuses" description:"(public) get the vote status for all public proposals"` + VoteAuthorize voteAuthorizeCmd `command:"voteauthorize"` + VoteStart voteStartCmd `command:"votestart"` + VoteStartRunoff voteStartRunoffCmd `command:"votestartrunoff"` + VoteBallot voteBallotCmd `command:"voteballot"` + Votes votesCmd `command:"votes"` + VoteResults voteResultsCmd `command:"voteresults"` + VoteSummaries voteSummariesCmd `command:"votesummaries"` + VoteInventory voteInventoryCmd `command:"voteinventory"` // User commands - UserNew userNewCmd `command:"usernew" description:"(public) create a new user"` - UserEdit userEditCmd `command:"useredit" description:"(user) edit the preferences of the logged in user"` - UserDetails userDetailsCmd `command:"userdetails" description:"(public) get the details of a user profile"` - UserPaymentsRescan userPaymentsRescanCmd `command:"userpaymentsrescan" description:"(admin) rescan a user's payments to check for missed payments"` - UserPendingPayment userPendingPaymentCmd `command:"userpendingpayment" description:"(user) get details for a pending payment for the logged in user"` - UserEmailVerify userEmailVerifyCmd `command:"useremailverify" description:"(public) verify a user's email address"` - UserPaymentVerify userPaymentVerifyCmd `command:"userpaymentverify" description:"(user) check if the logged in user has paid their user registration fee"` - UserVerificationResend userVerificationResendCmd `command:"userverificationresend" description:"(public) resend the user verification email"` - UserManage shared.UserManageCmd `command:"usermanage" description:"(admin) edit certain properties of the specified user"` - UserKeyUpdate shared.UserKeyUpdateCmd `command:"userkeyupdate" description:"(user) generate a new identity for the logged in user"` - UserUsernameChange shared.UserUsernameChangeCmd `command:"userusernamechange" description:"(user) change the username for the logged in user"` - UserPasswordChange shared.UserPasswordChangeCmd `command:"userpasswordchange" description:"(user) change the password for the logged in user"` - UserPasswordReset shared.UserPasswordResetCmd `command:"userpasswordreset" description:"(public) reset the password for a user that is not logged in"` - UserTOTPSet shared.UserTOTPSetCmd `command:"usertotpset" description:"(user) set the key for TOTP"` - UserTOTPVerify shared.UserTOTPVerifyCmd `command:"usertotpverify" description:"(user) verify the set code for TOTP"` - Users shared.UsersCmd `command:"users" description:"(public) get a list of users"` - - // XXX will be factored to a user route - ProposalPaywall proposalPaywallCmd `command:"proposalpaywall" description:"(user) get proposal paywall details for the logged in user"` - - // Basic commands - Login shared.LoginCmd `command:"login" description:"(public) login to Politeia"` - Logout shared.LogoutCmd `command:"logout" description:"(public) logout of Politeia"` - Me shared.MeCmd `command:"me" description:"(user) get user details for the logged in user"` - Secret shared.SecretCmd `command:"secret" description:"(user) ping politeiawww"` - Version shared.VersionCmd `command:"version" description:"(public) get server info and CSRF token"` - Policy policyCmd `command:"policy" description:"(public) get the server policy"` - Help helpCmd `command:"help" description:" print a detailed help message for a specific command"` + UserNew userNewCmd `command:"usernew"` + UserEdit userEditCmd `command:"useredit"` + UserDetails userDetailsCmd `command:"userdetails"` + UserPaymentsRescan userPaymentsRescanCmd `command:"userpaymentsrescan"` + UserPendingPayment userPendingPaymentCmd `command:"userpendingpayment"` + UserEmailVerify userEmailVerifyCmd `command:"useremailverify"` + UserPaymentVerify userPaymentVerifyCmd `command:"userpaymentverify"` + UserVerificationResend userVerificationResendCmd `command:"userverificationresend"` + UserManage shared.UserManageCmd `command:"usermanage"` + UserKeyUpdate shared.UserKeyUpdateCmd `command:"userkeyupdate"` + UserUsernameChange shared.UserUsernameChangeCmd `command:"userusernamechange"` + UserPasswordChange shared.UserPasswordChangeCmd `command:"userpasswordchange"` + UserPasswordReset shared.UserPasswordResetCmd `command:"userpasswordreset"` + UserTOTPSet shared.UserTOTPSetCmd `command:"usertotpset"` + UserTOTPVerify shared.UserTOTPVerifyCmd `command:"usertotpverify"` + Users shared.UsersCmd `command:"users"` + + // TODO rename to reflect that its a users route + ProposalPaywall proposalPaywallCmd `command:"proposalpaywall"` // Websocket commands - Subscribe subscribeCmd `command:"subscribe" description:"(public) subscribe to all websocket commands and do not exit tool"` + Subscribe subscribeCmd `command:"subscribe"` // Dev commands - TestRun testRunCmd `command:"testrun" description:"(dev) run a series of tests on the politeiawww routes"` - SendFaucetTx sendFaucetTxCmd `command:"sendfaucettx" description:"(dev) send a DCR transaction using the Decred testnet faucet"` -} - -// signedMerkleRoot calculates the merkle root of the passed in list of files -// and metadata, signs the merkle root with the passed in identity and returns -// the signature. -func signedMerkleRoot(files []pi.File, md []pi.Metadata, id *identity.FullIdentity) (string, error) { - if len(files) == 0 { - return "", fmt.Errorf("no proposal files found") - } - mr, err := wwwutil.MerkleRoot(files, md) - if err != nil { - return "", err - } - sig := id.SignMessage([]byte(mr)) - return hex.EncodeToString(sig[:]), nil -} - -// convertTicketHashes converts a slice of hexadecimal ticket hashes into -// a slice of byte slices. -func convertTicketHashes(h []string) ([][]byte, error) { - hashes := make([][]byte, 0, len(h)) - for _, v := range h { - h, err := chainhash.NewHashFromStr(v) - if err != nil { - return nil, err - } - hashes = append(hashes, h[:]) - } - return hashes, nil -} - -// proposalRecord returns the ProposalRecord for the provided token and -// version. -func proposalRecord(state pi.PropStateT, token, version string) (*pi.ProposalRecord, error) { - ps := pi.Proposals{ - State: state, - Requests: []pi.ProposalRequest{ - { - Token: token, - }, - }, - IncludeFiles: false, - } - psr, err := client.Proposals(ps) - if err != nil { - return nil, err - } - pr, ok := psr.Proposals[token] - if !ok { - return nil, fmt.Errorf("proposal not found") - } - - return &pr, nil -} - -// proposalRecord returns the latest ProposalRecrord version for the provided -// token. -func proposalRecordLatest(state pi.PropStateT, token string) (*pi.ProposalRecord, error) { - return proposalRecord(state, token, "") -} - -// decodeProposalMetadata decodes and returns a ProposalMetadata given the -// metadata array from a ProposalRecord. -func decodeProposalMetadata(metadata []pi.Metadata) (*pi.ProposalMetadata, error) { - var pm *pi.ProposalMetadata - for _, v := range metadata { - if v.Hint == pi.HintProposalMetadata { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return nil, err - } - err = json.Unmarshal(b, pm) - if err != nil { - return nil, err - } - } - } - if pm == nil { - return nil, fmt.Errorf("proposal metadata not found") - } - return pm, nil + TestRun testRunCmd `command:"testrun"` + SendFaucetTx sendFaucetTxCmd `command:"sendfaucettx"` } func _main() error { diff --git a/politeiawww/cmd/piwww/proposaledit.go b/politeiawww/cmd/piwww/proposaledit.go index 814ecd940..ee5526c30 100644 --- a/politeiawww/cmd/piwww/proposaledit.go +++ b/politeiawww/cmd/piwww/proposaledit.go @@ -218,14 +218,17 @@ func (cmd *proposalEditCmd) Execute(args []string) error { return err } - // Verify proposal censorship record + // TODO add this back in. The www verify functions were moved to + // the cmswww batchproposals command since piwww no longer uses the + // www types. Create a verifyProposal function for the pi api + // proposal. /* - // TODO + // Verify proposal vr, err := client.Version() if err != nil { return err } - err = shared.VerifyProposal(per.Proposal, vr.PubKey) + err = verifyProposal(per.Proposal, vr.PubKey) if err != nil { return fmt.Errorf("unable to verify proposal %v: %v", per.Proposal.CensorshipRecord.Token, err) @@ -247,7 +250,7 @@ Arguments: Flags: --vetted (bool, optional) Comment on vetted record. - --unvetted (bool, optional) Comment on unvetted reocrd. + --unvetted (bool, optional) Comment on unvetted record. --random (bool, optional) Generate a random proposal name & files to submit. If this flag is used then the markdown file argument is no longer required and any diff --git a/politeiawww/cmd/piwww/proposalnew.go b/politeiawww/cmd/piwww/proposalnew.go index d3a14391a..9c7bbd92a 100644 --- a/politeiawww/cmd/piwww/proposalnew.go +++ b/politeiawww/cmd/piwww/proposalnew.go @@ -21,8 +21,6 @@ import ( "github.com/decred/politeia/util" ) -// TODO replace www policies with pi policies - // proposalNewCmd submits a new proposal. type proposalNewCmd struct { Args struct { diff --git a/politeiawww/cmd/piwww/proposals.go b/politeiawww/cmd/piwww/proposals.go index 45637eef1..e98d3961a 100644 --- a/politeiawww/cmd/piwww/proposals.go +++ b/politeiawww/cmd/piwww/proposals.go @@ -12,7 +12,8 @@ import ( "github.com/decred/politeia/politeiawww/cmd/shared" ) -// proposalsCmd retrieves the proposal records of the requested tokens and versions. +// proposalsCmd retrieves the proposal records of the requested tokens and +// versions. type proposalsCmd struct { Args struct { Proposals []string `positional-arg-name:"proposals" required:"true"` @@ -61,19 +62,30 @@ func (cmd *proposalsCmd) Execute(args []string) error { requests = append(requests, r) } - // Setup and send request - props := pi.Proposals{ + // Setup request + p := pi.Proposals{ State: state, Requests: requests, IncludeFiles: cmd.IncludeFiles, } - reply, err := client.Proposals(props) + // Send request + err := shared.PrintJSON(p) + if err != nil { + return err + } + reply, err := client.Proposals(p) + if err != nil { + return err + } + err = shared.PrintJSON(reply) if err != nil { return err } - return shared.PrintJSON(reply) + // TODO verify proposals + + return nil } // proposalsHelpMsg is the output of the help command. @@ -84,14 +96,17 @@ is set by providing the censorship record token and the desired version, comma-separated. Providing only the token will default to the latest proposal version. +This command defaults to fetching vetted proposals unless the --unvetted flag +is used. + Arguments: 1. proposals ([]string, required) Proposals request Flags: - --unvetted (bool, optional) Request for unvetted proposals instead of - vetted ones. + --unvetted (bool, optional) Request is for unvetted proposals instead of + vetted ones (default: false). --includefiles (bool, optional) Include proposal files in the returned - proposal records. + proposal records (default: false). Example: piwww proposals ... diff --git a/politeiawww/cmd/piwww/util.go b/politeiawww/cmd/piwww/util.go new file mode 100644 index 000000000..52f023b6e --- /dev/null +++ b/politeiawww/cmd/piwww/util.go @@ -0,0 +1,99 @@ +// Copyright (c) 2017-2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + + "github.com/decred/dcrd/chaincfg/chainhash" + "github.com/decred/politeia/politeiad/api/v1/identity" + pi "github.com/decred/politeia/politeiawww/api/pi/v1" + utilwww "github.com/decred/politeia/politeiawww/util" +) + +// signedMerkleRoot calculates the merkle root of the passed in list of files +// and metadata, signs the merkle root with the passed in identity and returns +// the signature. +func signedMerkleRoot(files []pi.File, md []pi.Metadata, id *identity.FullIdentity) (string, error) { + if len(files) == 0 { + return "", fmt.Errorf("no proposal files found") + } + mr, err := utilwww.MerkleRoot(files, md) + if err != nil { + return "", err + } + sig := id.SignMessage([]byte(mr)) + return hex.EncodeToString(sig[:]), nil +} + +// convertTicketHashes converts a slice of hexadecimal ticket hashes into +// a slice of byte slices. +func convertTicketHashes(h []string) ([][]byte, error) { + hashes := make([][]byte, 0, len(h)) + for _, v := range h { + h, err := chainhash.NewHashFromStr(v) + if err != nil { + return nil, err + } + hashes = append(hashes, h[:]) + } + return hashes, nil +} + +// proposalRecord returns the ProposalRecord for the provided token and +// version. +func proposalRecord(state pi.PropStateT, token, version string) (*pi.ProposalRecord, error) { + ps := pi.Proposals{ + State: state, + Requests: []pi.ProposalRequest{ + { + Token: token, + Version: version, + }, + }, + IncludeFiles: true, + } + psr, err := client.Proposals(ps) + if err != nil { + return nil, err + } + pr, ok := psr.Proposals[token] + if !ok { + return nil, fmt.Errorf("proposal not found") + } + + return &pr, nil +} + +// proposalRecord returns the latest ProposalRecrord version for the provided +// token. +func proposalRecordLatest(state pi.PropStateT, token string) (*pi.ProposalRecord, error) { + return proposalRecord(state, token, "") +} + +// decodeProposalMetadata decodes and returns a ProposalMetadata given the +// metadata array from a ProposalRecord. +func decodeProposalMetadata(metadata []pi.Metadata) (*pi.ProposalMetadata, error) { + var pm *pi.ProposalMetadata + for _, v := range metadata { + if v.Hint == pi.HintProposalMetadata { + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return nil, err + } + err = json.Unmarshal(b, pm) + if err != nil { + return nil, err + } + } + } + if pm == nil { + return nil, fmt.Errorf("proposal metadata not found") + } + return pm, nil +} diff --git a/politeiawww/cmd/piwww/votedetails.go b/politeiawww/cmd/piwww/votedetails.go deleted file mode 100644 index 7aeb40ac0..000000000 --- a/politeiawww/cmd/piwww/votedetails.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) 2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import ( - "github.com/decred/politeia/politeiawww/cmd/shared" -) - -// voteDetailsCmd fetches the vote parameters and vote options from the -// politeiawww v2 VoteDetails routes. -type voteDetailsCmd struct { - Args struct { - Token string `positional-arg-name:"token"` // Proposal token - } `positional-args:"true" required:"true"` -} - -// Execute executes the vote details command. -func (cmd *voteDetailsCmd) Execute(args []string) error { - vdr, err := client.VoteDetailsV2(cmd.Args.Token) - if err != nil { - return err - } - - // Remove eligible tickets snapshot from the response - // so that the output is legible. - if !cfg.RawJSON { - vdr.EligibleTickets = []string{ - "removed by piwww for readability", - } - } - - err = shared.PrintJSON(vdr) - if err != nil { - return err - } - - return nil -} - -// voteDetailsHelpMsg is the output of the help command when 'votedetails' is -// specified. -const voteDetailsHelpMsg = `votedetails "token" - -Fetch the vote details for a proposal. - -Arguments: -1. token (string, required) Proposal censorship token -` diff --git a/politeiawww/cmd/piwww/votestatus.go b/politeiawww/cmd/piwww/votestatus.go deleted file mode 100644 index 908dfebe7..000000000 --- a/politeiawww/cmd/piwww/votestatus.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import "github.com/decred/politeia/politeiawww/cmd/shared" - -// voteStatusCmd gets the vote status of the specified proposal. -type voteStatusCmd struct { - Args struct { - Token string `positional-arg-name:"token"` // Censorship token - } `positional-args:"true" required:"true"` -} - -// Execute executes the vote status command. -func (cmd *voteStatusCmd) Execute(args []string) error { - vsr, err := client.VoteStatus(cmd.Args.Token) - if err != nil { - return err - } - return shared.PrintJSON(vsr) -} - -// voteStatusHelpMsg is the output of the help command when 'votestatus' is -// specified. -const voteStatusHelpMsg = `votestatus "token" - -Fetch vote status for a proposal. - -Proposal vote status codes: - -'0' - Invalid vote status -'1' - Vote has not been authorized by proposal author -'2' - Vote has been authorized by proposal author -'3' - Proposal vote has been started -'4' - Proposal vote has been finished -'5' - Proposal doesn't exist - -Arguments: -1. token (string, required) Proposal censorship token` diff --git a/politeiawww/cmd/piwww/votestatuses.go b/politeiawww/cmd/piwww/votestatuses.go deleted file mode 100644 index ff3be57bf..000000000 --- a/politeiawww/cmd/piwww/votestatuses.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import "github.com/decred/politeia/politeiawww/cmd/shared" - -// voteStatusesCmd retreives the vote status of all public proposals. -type voteStatusesCmd struct{} - -// Execute executes the vote statuses command. -func (cmd *voteStatusesCmd) Execute(args []string) error { - avsr, err := client.GetAllVoteStatus() - if err != nil { - return err - } - return shared.PrintJSON(avsr) -} - -// voteStatusesHelpMsg is the output for the help command when 'votestatuses' -// is specified. -const voteStatusesHelpMsg = `votestatuses - -Fetch vote status of all public proposals. - -Proposal vote status codes: - -'0' - Invalid vote status -'1' - Vote has not been authorized by proposal author -'2' - Vote has been authorized by proposal author -'3' - Proposal vote has been started -'4' - Proposal vote has been finished -'5' - Proposal doesn't exist - -Arguments: None` diff --git a/politeiawww/cmd/shared/shared.go b/politeiawww/cmd/shared/shared.go index 70a0bef40..3b2279d95 100644 --- a/politeiawww/cmd/shared/shared.go +++ b/politeiawww/cmd/shared/shared.go @@ -6,7 +6,6 @@ package shared import ( "bytes" - "encoding/base64" "encoding/hex" "encoding/json" "errors" @@ -14,8 +13,6 @@ import ( "os" "github.com/decred/politeia/politeiad/api/v1/identity" - v1 "github.com/decred/politeia/politeiawww/api/www/v1" - wwwutil "github.com/decred/politeia/politeiawww/util" "github.com/decred/politeia/util" "golang.org/x/crypto/ed25519" "golang.org/x/crypto/sha3" @@ -62,49 +59,6 @@ func PrintJSON(body interface{}) error { return nil } -// ValidateDigests receives a list of files and metadata to verify their -// digests. It compares digests that came with the file/md with digests -// calculated from their respective payloads. -func ValidateDigests(files []v1.File, md []v1.Metadata) error { - // Validate file digests - for _, f := range files { - b, err := base64.StdEncoding.DecodeString(f.Payload) - if err != nil { - return fmt.Errorf("file: %v decode payload err %v", - f.Name, err) - } - digest := util.Digest(b) - d, ok := util.ConvertDigest(f.Digest) - if !ok { - return fmt.Errorf("file: %v invalid digest %v", - f.Name, f.Digest) - } - if !bytes.Equal(digest, d[:]) { - return fmt.Errorf("file: %v digests do not match", - f.Name) - } - } - // Validate metadata digests - for _, v := range md { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return fmt.Errorf("metadata: %v decode payload err %v", - v.Hint, err) - } - digest := util.Digest(b) - d, ok := util.ConvertDigest(v.Digest) - if !ok { - return fmt.Errorf("metadata: %v invalid digest %v", - v.Hint, v.Digest) - } - if !bytes.Equal(digest, d[:]) { - return fmt.Errorf("metadata: %v digests do not match metadata", - v.Hint) - } - } - return nil -} - // DigestSHA3 returns the hex encoded SHA3-256 of a string. func DigestSHA3(s string) string { h := sha3.New256() @@ -141,52 +95,3 @@ func SetConfig(config *Config) { func SetClient(c *Client) { client = c } - -// VerifyProposal verifies a proposal's merkle root, author signature, and -// censorship record. -func VerifyProposal(p v1.ProposalRecord, serverPubKey string) error { - if len(p.Files) > 0 { - // Verify digests - err := ValidateDigests(p.Files, p.Metadata) - if err != nil { - return err - } - // Verify merkle root - mr, err := wwwutil.MerkleRootWWW(p.Files, p.Metadata) - if err != nil { - return err - } - if mr != p.CensorshipRecord.Merkle { - return fmt.Errorf("merkle roots do not match") - } - } - - // Verify proposal signature - pid, err := util.IdentityFromString(p.PublicKey) - if err != nil { - return err - } - sig, err := util.ConvertSignature(p.Signature) - if err != nil { - return err - } - if !pid.VerifyMessage([]byte(p.CensorshipRecord.Merkle), sig) { - return fmt.Errorf("could not verify proposal signature") - } - - // Verify censorship record signature - id, err := util.IdentityFromString(serverPubKey) - if err != nil { - return err - } - s, err := util.ConvertSignature(p.CensorshipRecord.Signature) - if err != nil { - return err - } - msg := []byte(p.CensorshipRecord.Merkle + p.CensorshipRecord.Token) - if !id.VerifyMessage(msg, s) { - return fmt.Errorf("could not verify censorship record signature") - } - - return nil -} diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 50ef51f77..2f7d406e0 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -31,10 +31,10 @@ import ( // TODO use pi policies. Should the policies be defined in the pi plugin // or the pi api spec? - -// TODO politeiad needs to return the plugin with the error. // TODO ensure plugins can't write data using short proposal token. // TODO move proposal validation to pi plugin +// TODO politeiad needs batched calls for retrieving unvetted and vetted +// records. const ( // MIME types @@ -54,6 +54,10 @@ var ( pi.PropStatusCensored: struct{}{}, pi.PropStatusAbandoned: struct{}{}, } + + // errProposalNotFound is emitted when a proposal is not found in + // politeiad for a specified token and version. + errProposalNotFound = errors.New("proposal not found") ) // tokenIsValid returns whether the provided string is a valid politeiad @@ -710,71 +714,10 @@ func (p *politeiawww) linkByPeriodMax() int64 { return 7776000 // 3 months in seconds } -// proposalRecord returns the proposal record for the provided token and -// version. -func (p *politeiawww) proposalRecord(state pi.PropStateT, token, version string) (*pi.ProposalRecord, error) { - // Get politeiad record - var r *pd.Record - var err error - switch state { - case pi.PropStateUnvetted: - r, err = p.getUnvetted(token, version) - if err != nil { - return nil, err - } - case pi.PropStateVetted: - r, err = p.getVetted(token, version) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("unknown state %v", state) - } - - pr, err := convertProposalRecordFromPD(*r) - if err != nil { - return nil, err - } - - // Get proposal plugin data - ps := piplugin.Proposals{ - State: convertPropStateFromPi(state), - Tokens: []string{token}, - } - psr, err := p.piProposals(ps) - if err != nil { - return nil, err - } - d, ok := psr.Proposals[token] - if !ok { - return nil, fmt.Errorf("proposal plugin data not found %v", token) - } - pr.Comments = d.Comments - pr.LinkedFrom = d.LinkedFrom - - // Get user data - u, err := p.db.UserGetByPubKey(pr.PublicKey) - if err != nil { - return nil, err - } - pr.UserID = u.ID.String() - pr.Username = u.Username - - return pr, nil -} - -// proposalRecordLatest returns the latest version of the proposal record for -// the provided token. -func (p *politeiawww) proposalRecordLatest(state pi.PropStateT, token string) (*pi.ProposalRecord, error) { - return p.proposalRecord(state, token, "") -} - // proposalRecords returns the ProposalRecord for each of the provided proposal // requests. If a token does not correspond to an actual proposal then it will // not be included in the returned map. // -// TODO politeiad needs batched calls for retrieving unvetted and vetted -// records. This call should have an includeFiles option. // TODO this presents a challenge because the proposal Metadata still needs to // be returned even if the proposal Files are not returned, which means that we // will always need to fetch the record from politeiad with the files attached @@ -802,6 +745,11 @@ func (p *politeiawww) proposalRecords(state pi.PropStateT, reqs []pi.ProposalReq return nil, fmt.Errorf("unknown state %v", state) } + if r.Status == pd.RecordStatusNotFound { + // Record wasn't found. Don't include token in the results. + continue + } + pr, err := convertProposalRecordFromPD(*r) if err != nil { return nil, fmt.Errorf("convertProposalRecordFromPD %v: %v", @@ -817,10 +765,15 @@ func (p *politeiawww) proposalRecords(state pi.PropStateT, reqs []pi.ProposalReq props = append(props, *pr) } + // Verify we've got some results + if len(props) == 0 { + return map[string]pi.ProposalRecord{}, nil + } + // Get proposal plugin data - tokens := make([]string, 0, len(reqs)) - for _, v := range reqs { - tokens = append(tokens, v.Token) + tokens := make([]string, 0, len(props)) + for _, v := range props { + tokens = append(tokens, v.CensorshipRecord.Token) } ps := piplugin.Proposals{ State: convertPropStateFromPi(state), @@ -869,6 +822,34 @@ func (p *politeiawww) proposalRecords(state pi.PropStateT, reqs []pi.ProposalReq return proposals, nil } +// proposalRecord returns the proposal record for the provided token and +// version. A blank version will return the most recent version. A +// errProposalNotFound error will be returned if a proposal is not found for +// the provided token/version combination. +func (p *politeiawww) proposalRecord(state pi.PropStateT, token, version string) (*pi.ProposalRecord, error) { + prs, err := p.proposalRecords(state, []pi.ProposalRequest{ + { + Token: token, + Version: version, + }, + }, true) + if err != nil { + return nil, err + } + pr, ok := prs[token] + if !ok { + return nil, errProposalNotFound + } + return &pr, nil +} + +// proposalRecordLatest returns the latest version of the proposal record for +// the provided token. A errProposalNotFound error will be returned if a +// proposal is not found for the provided token. +func (p *politeiawww) proposalRecordLatest(state pi.PropStateT, token string) (*pi.ProposalRecord, error) { + return p.proposalRecord(state, token, "") +} + func (p *politeiawww) verifyProposalMetadata(pm pi.ProposalMetadata) error { // Verify name if !proposalNameIsValid(pm.Name) { diff --git a/politeiawww/politeiad.go b/politeiawww/politeiad.go index 26208562c..ce02aa595 100644 --- a/politeiawww/politeiad.go +++ b/politeiawww/politeiad.go @@ -143,7 +143,7 @@ func (p *politeiawww) updateRecord(route, token string, mdAppend, mdOverwrite [] ctx := context.Background() resBody, err := p.makeRequest(ctx, http.MethodPost, route, ur) if err != nil { - return nil, nil + return nil, err } // Receive reply diff --git a/politeiawww/util/merkle.go b/politeiawww/util/merkle.go index f6c3d726e..b55dc36c7 100644 --- a/politeiawww/util/merkle.go +++ b/politeiawww/util/merkle.go @@ -11,7 +11,6 @@ import ( "github.com/decred/dcrtime/merkle" pi "github.com/decred/politeia/politeiawww/api/pi/v1" - www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/util" ) @@ -47,35 +46,3 @@ func MerkleRoot(files []pi.File, md []pi.Metadata) (string, error) { // Return merkle root return hex.EncodeToString(merkle.Root(digests)[:]), nil } - -// TODO remove this once cli has been converted over to use pi types. -func MerkleRootWWW(files []www.File, md []www.Metadata) (string, error) { - digests := make([]*[sha256.Size]byte, 0, len(files)) - - // Calculate file digests - for _, f := range files { - b, err := base64.StdEncoding.DecodeString(f.Payload) - if err != nil { - return "", err - } - digest := util.Digest(b) - var hf [sha256.Size]byte - copy(hf[:], digest) - digests = append(digests, &hf) - } - - // Calculate metadata digests - for _, v := range md { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return "", err - } - digest := util.Digest(b) - var hv [sha256.Size]byte - copy(hv[:], digest) - digests = append(digests, &hv) - } - - // Return merkle root - return hex.EncodeToString(merkle.Root(digests)[:]), nil -} diff --git a/politeiawww/www.go b/politeiawww/www.go index 089087d37..3e607c44d 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -399,8 +399,9 @@ func respondWithPiError(w http.ResponseWriter, r *http.Request, format string, e // Error is a politeiawww server error. Log it and return a 500. t := time.Now().Unix() + e := fmt.Sprintf(format, err) log.Errorf("%v %v %v %v Internal error %v: %v", - remoteAddr(r), r.Method, r.URL, r.Proto, t, format) + remoteAddr(r), r.Method, r.URL, r.Proto, t, e) log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) util.RespondWithJSON(w, http.StatusInternalServerError, From 3bfeee71de428f127399f6240a3af71e72fbedc9 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 5 Oct 2020 08:29:34 -0500 Subject: [PATCH 128/449] proposal route fixes --- politeiad/backend/tlogbe/anchor.go | 2 +- politeiad/backend/tlogbe/dcrdata.go | 2 +- politeiad/backend/tlogbe/pi.go | 68 +- politeiawww/api/pi/v1/v1.go | 1 + politeiawww/cmd/piwww/testrun.go | 1368 +-------------------------- politeiawww/cmd/shared/client.go | 26 +- politeiawww/piwww.go | 6 +- politeiawww/www.go | 14 +- 8 files changed, 81 insertions(+), 1406 deletions(-) diff --git a/politeiad/backend/tlogbe/anchor.go b/politeiad/backend/tlogbe/anchor.go index 56db5809b..7ca6a6f2f 100644 --- a/politeiad/backend/tlogbe/anchor.go +++ b/politeiad/backend/tlogbe/anchor.go @@ -29,7 +29,7 @@ const ( // currently drops an anchor on the hour mark so we submit new // anchors a few minutes prior to that. // Seconds Minutes Hours Days Months DayOfWeek - anchorSchedule = "0 56 * * * *" // At 56 minutes every hour + anchorSchedule = "0 56 * * * *" // At minute 56 of every hour // anchorID is included in the timestamp and verify requests as a // unique identifier. diff --git a/politeiad/backend/tlogbe/dcrdata.go b/politeiad/backend/tlogbe/dcrdata.go index b768da0ba..13f89d3f8 100644 --- a/politeiad/backend/tlogbe/dcrdata.go +++ b/politeiad/backend/tlogbe/dcrdata.go @@ -488,7 +488,7 @@ func (p *dcrdataPlugin) fsck() error { func newDcrdataPlugin(settings []backend.PluginSetting) (*dcrdataPlugin, error) { // TODO these should be passed in as plugin settings - dcrdataHostHTTP := "https://dcrdata.decred.org/" + dcrdataHostHTTP := "https://dcrdata.decred.org" dcrdataHostWS := "wss://dcrdata.decred.org/ps" client, err := util.NewClient(false, "") diff --git a/politeiad/backend/tlogbe/pi.go b/politeiad/backend/tlogbe/pi.go index 97330ffea..8a69dad69 100644 --- a/politeiad/backend/tlogbe/pi.go +++ b/politeiad/backend/tlogbe/pi.go @@ -440,39 +440,6 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { } } - // Verify vote status - token := er.RecordMetadata.Token - s := ticketvote.Summaries{ - Tokens: []string{token}, - } - b, err := ticketvote.EncodeSummaries(s) - if err != nil { - return err - } - reply, err := p.backend.Plugin(ticketvote.ID, - ticketvote.CmdSummaries, "", string(b)) - if err != nil { - return fmt.Errorf("ticketvote Summaries: %v", err) - } - sr, err := ticketvote.DecodeSummariesReply([]byte(reply)) - if err != nil { - return err - } - summary, ok := sr.Summaries[token] - if !ok { - return fmt.Errorf("ticketvote summmary not found") - } - if summary.Status != ticketvote.VoteStatusUnauthorized { - e := fmt.Sprintf("vote status got %v, want %v", - ticketvote.VoteStatus[summary.Status], - ticketvote.VoteStatus[ticketvote.VoteStatusUnauthorized]) - return backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusVoteStatusInvalid), - ErrorContext: []string{e}, - } - } - // Verify that the linkto has not changed. This only applies to // public proposal. Unvetted proposals are allowed to change their // linkto. @@ -494,6 +461,41 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { } } + // Verify vote status. This is only required for public proposals. + if status == pi.PropStatusPublic { + token := er.RecordMetadata.Token + s := ticketvote.Summaries{ + Tokens: []string{token}, + } + b, err := ticketvote.EncodeSummaries(s) + if err != nil { + return err + } + reply, err := p.backend.Plugin(ticketvote.ID, + ticketvote.CmdSummaries, "", string(b)) + if err != nil { + return fmt.Errorf("ticketvote Summaries: %v", err) + } + sr, err := ticketvote.DecodeSummariesReply([]byte(reply)) + if err != nil { + return err + } + summary, ok := sr.Summaries[token] + if !ok { + return fmt.Errorf("ticketvote summmary not found") + } + if summary.Status != ticketvote.VoteStatusUnauthorized { + e := fmt.Sprintf("vote status got %v, want %v", + ticketvote.VoteStatus[summary.Status], + ticketvote.VoteStatus[ticketvote.VoteStatusUnauthorized]) + return backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusVoteStatusInvalid), + ErrorContext: []string{e}, + } + } + } + return nil } diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 205f43674..4739bdd7c 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -154,6 +154,7 @@ const ( ErrorStatusPropStatusChangeInvalid ErrorStatusPropStatusChangeReasonInvalid ErrorStatusPropPageSizeExceeded + ErrorStatusNoPropChanges // Comment errors ErrorStatusCommentTextInvalid diff --git a/politeiawww/cmd/piwww/testrun.go b/politeiawww/cmd/piwww/testrun.go index 1c2882486..1e67d2fe5 100644 --- a/politeiawww/cmd/piwww/testrun.go +++ b/politeiawww/cmd/piwww/testrun.go @@ -1,11 +1,10 @@ -// Copyright (c) 2017-2019 The Decred developers +// Copyright (c) 2017-2020 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. package main import ( - v1 "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/cmd/shared" ) @@ -17,13 +16,13 @@ type testRunCmd struct { } `positional-args:"true" required:"true"` } -// testUser stores user details that are used throughout the test run. -type testUser struct { +// user stores user details that are used throughout the test run. +type user struct { ID string // UUID - email string // Email - username string // Username - password string // Password (not hashed) - publicKey string // Public key of active identity + Email string // Email + Username string // Username + Password string // Password (not hashed) + PublicKey string // Public key of active identity } // login logs in the specified user. @@ -34,71 +33,6 @@ func login(email, password string) error { return lc.Execute(nil) } -// newRFPProposal is a wrapper func which creates a RFP by calling newProposal -func newRFPProposal() (*v1.NewProposal, error) { - return newProposal(true, "") -} - -// newSubmissionProposal is a wrapper func which creates a RFP submission by -// calling newProposal func with a given linkto token. -func newSubmissionProposal(linkto string) (*v1.NewProposal, error) { - return newProposal(false, linkto) -} - -// newNormalProposal is a wrapper func which creates a proposal by calling -// newProposal -func newNormalProposal() (*v1.NewProposal, error) { - return newProposal(false, "") -} - -// newProposal returns a NewProposal object contains randonly generated -// markdown text and a signature from the logged in user. If given `rfp` bool -// is true it creates an RFP. If given `linkto` it creates a RFP submission. -func newProposal(rfp bool, linkto string) (*v1.NewProposal, error) { - /* - // TODO - md, err := createMDFile() - if err != nil { - return nil, fmt.Errorf("create MD file: %v", err) - } - files := []v1.File{*md} - - pm := v1.ProposalMetadata{ - Name: "Some proposal name", - } - if rfp { - pm.LinkBy = time.Now().Add(time.Hour * 24 * 30).Unix() - } - if linkto != "" { - pm.LinkTo = linkto - } - pmb, err := json.Marshal(pm) - if err != nil { - return nil, err - } - metadata := []v1.Metadata{ - { - Digest: hex.EncodeToString(util.Digest(pmb)), - Hint: v1.HintProposalMetadata, - Payload: base64.StdEncoding.EncodeToString(pmb), - }, - } - - sig, err := shared.SignedMerkleRoot(files, metadata, cfg.Identity) - if err != nil { - return nil, fmt.Errorf("sign merkle root: %v", err) - } - - return &v1.NewProposal{ - Files: files, - Metadata: metadata, - PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), - Signature: sig, - }, nil - */ - return nil, nil -} - // castVotes casts votes on a proposal with a given voteId. If it fails it // returns the error and in case of dcrwallet connection error it returns // true as first returned value @@ -154,8 +88,8 @@ func (cmd *testRunCmd) Execute(args []string) error { numCredits = v1.ProposalListPageSize * 2 // Test users - user testUser - admin testUser + user user + admin user ) // Suppress output from cli commands @@ -247,12 +181,12 @@ func (cmd *testRunCmd) Execute(args []string) error { return err } - user = testUser{ + user = user{ ID: lr.UserID, - email: email, - username: username, - password: password, - publicKey: lr.PublicKey, + Email: email, + Username: username, + Password: password, + PublicKey: lr.PublicKey, } // Check if paywall is enabled. Paywall address and paywall @@ -420,1280 +354,6 @@ func (cmd *testRunCmd) Execute(args []string) error { return err } - // Submit new proposal - fmt.Printf(" New proposal\n") - np, err := newNormalProposal() - if err != nil { - return err - } - npr, err := client.NewProposal(np) - if err != nil { - return err - } - - // Verify proposal censorship record - pr := v1.ProposalRecord{ - Files: np.Files, - Metadata: np.Metadata, - PublicKey: np.PublicKey, - Signature: np.Signature, - CensorshipRecord: npr.CensorshipRecord, - } - err = shared.VerifyProposal(pr, version.PubKey) - if err != nil { - return fmt.Errorf("verify proposal failed: %v", err) - } - - // This is the proposal that we'll use for most of the tests - token := pr.CensorshipRecord.Token - fmt.Printf(" Proposal submitted: %v\n", token) - - // Edit unvetted proposal - fmt.Printf(" Edit unvetted proposal\n") - epc := EditProposalCmd{ - Random: true, - } - epc.Args.Token = token - err = epc.Execute(nil) - if err != nil { - return err - } - - // Login with admin and make the proposal public - fmt.Printf(" Login admin\n") - err = login(admin.email, admin.password) - if err != nil { - return err - } - - fmt.Printf(" Set proposal status: public\n") - spsc := SetProposalStatusCmd{} - spsc.Args.Token = token - spsc.Args.Status = strconv.Itoa(int(v1.PropStatusPublic)) - err = spsc.Execute(nil) - if err != nil { - return err - } - - // Log back in with user - fmt.Printf(" Login user\n") - err = login(user.email, user.password) - if err != nil { - return err - } - - // Edit vetted proposal - fmt.Printf(" Edit vetted proposal\n") - err = epc.Execute(nil) - if err != nil { - return err - } - - // New comment - parent - fmt.Printf(" New comment: parent\n") - ncc := shared.NewCommentCmd{} - ncc.Args.Token = token - ncc.Args.Comment = "this is a comment" - ncc.Args.ParentID = "0" - err = ncc.Execute(nil) - if err != nil { - return err - } - - // New comment - reply - fmt.Printf(" New comment: reply\n") - ncc.Args.Token = token - ncc.Args.Comment = "this is a comment reply" - ncc.Args.ParentID = "1" - err = ncc.Execute(nil) - if err != nil { - return err - } - - // Validate comments - fmt.Printf(" Proposal details\n") - pdr, err := client.ProposalDetails(token, nil) - if err != nil { - return err - } - - err = shared.VerifyProposal(pdr.Proposal, version.PubKey) - if err != nil { - return fmt.Errorf("verify proposal failed: %v", err) - } - - if pdr.Proposal.NumComments != 2 { - return fmt.Errorf("proposal num comments got %v, want 2", - pdr.Proposal.NumComments) - } - - fmt.Printf(" Proposal comments\n") - gcr, err := client.GetComments(token) - if err != nil { - return fmt.Errorf("GetComments: %v", err) - } - - if len(gcr.Comments) != 2 { - return fmt.Errorf("num comments got %v, want 2", - len(gcr.Comments)) - } - - for _, v := range gcr.Comments { - // We check the userID because userIDs are not part of - // the politeiad comment record. UserIDs are stored in - // in politeiawww and are added to the comments at the - // time of the request. This introduces the potential - // for errors. - if v.UserID != user.ID { - return fmt.Errorf("comment userID got %v, want %v", - v.UserID, user.ID) - } - } - - // Like comment sequence - lcc := LikeCommentCmd{} - lcc.Args.Token = pr.CensorshipRecord.Token - lcc.Args.CommentID = "1" - lcc.Args.Action = commentActionUpvote - - fmt.Printf(" Like comment: upvote\n") - err = lcc.Execute(nil) - if err != nil { - return err - } - - fmt.Printf(" Like comment: upvote\n") - err = lcc.Execute(nil) - if err != nil { - return err - } - - fmt.Printf(" Like comment: upvote\n") - err = lcc.Execute(nil) - if err != nil { - return err - } - - fmt.Printf(" Like comment: downvote\n") - lcc.Args.Action = commentActionDownvote - err = lcc.Execute(nil) - if err != nil { - return err - } - - // Validate like comments - fmt.Printf(" Proposal comments\n") - gcr, err = client.GetComments(token) - if err != nil { - return err - } - - for _, v := range gcr.Comments { - if v.CommentID == "1" { - switch { - case v.Upvotes != 0: - return fmt.Errorf("comment result up votes got %v, want 0", - v.Upvotes) - case v.Downvotes != 1: - return fmt.Errorf("comment result down votes got %v, want 1", - v.Downvotes) - case v.ResultVotes != -1: - return fmt.Errorf("comment result vote score got %v, want -1", - v.ResultVotes) - } - } - } - - fmt.Printf(" User like comments\n") - crv, err := client.UserCommentsLikes(token) - if err != nil { - return err - } - - switch { - case len(crv.CommentsLikes) != 1: - return fmt.Errorf("user like comments got %v, want 1", - len(crv.CommentsLikes)) - - case crv.CommentsLikes[0].Action != v1.VoteActionDown: - return fmt.Errorf("user like comment action got %v, want %v", - crv.CommentsLikes[0].Action, v1.VoteActionDown) - } - - // Authorize vote then revoke - fmt.Printf(" Authorize vote: authorize\n") - avc := AuthorizeVoteCmd{} - avc.Args.Token = token - avc.Args.Action = decredplugin.AuthVoteActionAuthorize - err = avc.Execute(nil) - if err != nil { - return err - } - - fmt.Printf(" Authorize vote: revoke\n") - avc.Args.Action = decredplugin.AuthVoteActionRevoke - err = avc.Execute(nil) - if err != nil { - return err - } - - // Validate vote status - fmt.Printf(" Vote status\n") - vsr, err := client.VoteStatus(token) - if err != nil { - return err - } - - if vsr.Status != v1.PropVoteStatusNotAuthorized { - return fmt.Errorf("vote status got %v, want %v", - vsr.Status, v1.PropVoteStatusNotAuthorized) - } - - // Authorize vote - fmt.Printf(" Authorize vote: authorize\n") - avc.Args.Action = decredplugin.AuthVoteActionAuthorize - err = avc.Execute(nil) - if err != nil { - return err - } - - // Validate vote status - fmt.Printf(" Vote status\n") - vsr, err = client.VoteStatus(token) - if err != nil { - return err - } - - if vsr.Status != v1.PropVoteStatusAuthorized { - return fmt.Errorf("vote status got %v, want %v", - vsr.Status, v1.PropVoteStatusNotAuthorized) - } - - // Logout - fmt.Printf(" Logout\n") - lc = shared.LogoutCmd{} - err = lc.Execute(nil) - if err != nil { - return err - } - - // Admin routes are routes that only - // admins can access. - fmt.Printf("Running admin routes\n") - - // Login - fmt.Printf(" Login admin\n") - err = login(admin.email, admin.password) - if err != nil { - return err - } - - // Start vote - fmt.Printf(" Start vote\n") - svc := StartVoteCmd{} - svc.Args.Token = token - err = svc.Execute(nil) - if err != nil { - return err - } - - // Censor comment - fmt.Printf(" Censor comment\n") - ccc := shared.CensorCommentCmd{} - ccc.Args.Token = token - ccc.Args.CommentID = "2" - ccc.Args.Reason = "comment is spam" - err = ccc.Execute(nil) - if err != nil { - return err - } - - // Validate censored comment - fmt.Printf(" Get comments\n") - gcr, err = client.GetComments(token) - if err != nil { - return err - } - - if len(gcr.Comments) != 2 { - return fmt.Errorf("num comments got %v, want 2", - len(gcr.Comments)) - } - - c := gcr.Comments[1] - switch { - case c.CommentID != "2": - return fmt.Errorf("commentID got %v, want 2", - c.CommentID) - - case c.Comment != "": - return fmt.Errorf("censored comment text got %v, want empty string", - c.Comment) - - case !c.Censored: - return fmt.Errorf("censored comment not marked as censored") - } - - // Login with user in order to submit proposals that we can - // use to test the set proposal status route. - fmt.Printf(" Login user\n") - err = login(user.email, user.password) - if err != nil { - return err - } - - // Submit proposals that can be used to test the set proposal - // status command. - fmt.Printf(" Creating proposals for set proposal status test\n") - var ( - // Censorship tokens - notReviewed1 string - notReviewed2 string - unreviewedChanges1 string - unreviewedChanges2 string - - // We don't need these now but will need - // them when we test the public routes. - censoredPropToken string - unreviewedPropToken string - ) - - np, err = newNormalProposal() - if err != nil { - return err - } - npr, err = client.NewProposal(np) - if err != nil { - return err - } - notReviewed1 = npr.CensorshipRecord.Token - - np, err = newNormalProposal() - if err != nil { - return err - } - npr, err = client.NewProposal(np) - if err != nil { - return err - } - notReviewed2 = npr.CensorshipRecord.Token - - np, err = newNormalProposal() - if err != nil { - return err - } - npr, err = client.NewProposal(np) - if err != nil { - return err - } - epc = EditProposalCmd{ - Random: true, - } - epc.Args.Token = npr.CensorshipRecord.Token - err = epc.Execute(nil) - if err != nil { - return err - } - unreviewedChanges1 = npr.CensorshipRecord.Token - - np, err = newNormalProposal() - if err != nil { - return err - } - npr, err = client.NewProposal(np) - if err != nil { - return err - } - epc = EditProposalCmd{ - Random: true, - } - epc.Args.Token = npr.CensorshipRecord.Token - err = epc.Execute(nil) - if err != nil { - return err - } - unreviewedChanges2 = npr.CensorshipRecord.Token - - np, err = newNormalProposal() - if err != nil { - return err - } - npr, err = client.NewProposal(np) - if err != nil { - return err - } - // We don't use this proposal for testing the set - // proposal status routes, but will need it when - // we are testing the public routes. - unreviewedPropToken = npr.CensorshipRecord.Token - - // Log back in with admin - fmt.Printf(" Login admin\n") - err = login(admin.email, admin.password) - if err != nil { - return err - } - - // Validate the proposal statuses before we attempt to - // change them. - pdr, err = client.ProposalDetails(notReviewed1, nil) - if err != nil { - return err - } - - if pdr.Proposal.Status != v1.PropStatusNotReviewed { - return fmt.Errorf("Proposal status got %v, want %v", - pdr.Proposal.Status, v1.PropStatusNotReviewed) - } - - pdr, err = client.ProposalDetails(unreviewedChanges1, nil) - if err != nil { - return err - } - - if pdr.Proposal.Status != v1.PropStatusUnreviewedChanges { - return fmt.Errorf("Proposal status got %v, want %v", - pdr.Proposal.Status, v1.PropStatusUnreviewedChanges) - } - - // Set proposal status - not reviewed to censored - fmt.Printf(" Set proposal status: not reviewed to censored\n") - spsc = SetProposalStatusCmd{} - spsc.Args.Token = notReviewed1 - spsc.Args.Status = "censored" - spsc.Args.Message = "proposal is spam" - err = spsc.Execute(nil) - if err != nil { - return err - } - - pdr, err = client.ProposalDetails(spsc.Args.Token, nil) - if err != nil { - return err - } - - if pdr.Proposal.Status != v1.PropStatusCensored { - return fmt.Errorf("Proposal status got %v, want %v", - pdr.Proposal.Status, v1.PropStatusCensored) - } - - // Save this token. We will need it when - // we test the public routes. - censoredPropToken = spsc.Args.Token - - // Set proposal status - not reviewed to public - fmt.Printf(" Set proposal status: not reviewed to public\n") - spsc = SetProposalStatusCmd{} - spsc.Args.Token = notReviewed2 - spsc.Args.Status = strconv.Itoa(int(v1.PropStatusPublic)) - err = spsc.Execute(nil) - if err != nil { - return err - } - - pdr, err = client.ProposalDetails(spsc.Args.Token, nil) - if err != nil { - return err - } - - if pdr.Proposal.Status != v1.PropStatusPublic { - return fmt.Errorf("Proposal status got %v, want %v", - pdr.Proposal.Status, v1.PropStatusPublic) - } - - // Set proposal status - unreviewed changes to censored - fmt.Printf(" Set proposal status: unreviewed changes to censored\n") - spsc = SetProposalStatusCmd{} - spsc.Args.Token = unreviewedChanges1 - spsc.Args.Status = "censored" - spsc.Args.Message = "this is spam" - err = spsc.Execute(nil) - if err != nil { - return err - } - - pdr, err = client.ProposalDetails(spsc.Args.Token, nil) - if err != nil { - return err - } - - if pdr.Proposal.Status != v1.PropStatusCensored { - return fmt.Errorf("Proposal status got %v, want %v", - pdr.Proposal.Status, v1.PropStatusCensored) - } - - // Set proposal status - unreviewed changes to public - fmt.Printf(" Set proposal status: unreviewed changes to public\n") - spsc = SetProposalStatusCmd{} - spsc.Args.Token = unreviewedChanges2 - spsc.Args.Status = strconv.Itoa(int(v1.PropStatusPublic)) - err = spsc.Execute(nil) - if err != nil { - return err - } - - pdr, err = client.ProposalDetails(spsc.Args.Token, nil) - if err != nil { - return err - } - - if pdr.Proposal.Status != v1.PropStatusPublic { - return fmt.Errorf("Proposal status got %v, want %v", - pdr.Proposal.Status, v1.PropStatusPublic) - } - - // Set proposal status - public to abandoned - fmt.Printf(" Set proposal status: public to abandoned\n") - spsc.Args.Status = "abandoned" - spsc.Args.Message = "no activity for two weeks" - err = spsc.Execute(nil) - if err != nil { - return err - } - - pdr, err = client.ProposalDetails(spsc.Args.Token, nil) - if err != nil { - return err - } - - if pdr.Proposal.Status != v1.PropStatusAbandoned { - return fmt.Errorf("Proposal status got %v, want %v", - pdr.Proposal.Status, v1.PropStatusAbandoned) - } - - // Users - filter by email - fmt.Printf(" Users: filter by email\n") - ur, err := client.Users( - &v1.Users{ - Email: user.email, - }) - if err != nil { - return err - } - - switch { - case ur.TotalMatches != 1: - return fmt.Errorf("total matches got %v, want 1", - ur.TotalMatches) - - case ur.Users[0].ID != user.ID: - return fmt.Errorf("user ID got %v, want %v", - ur.Users[0].ID, user.ID) - } - - // Users - filter by username - fmt.Printf(" Users: filter by username\n") - ur, err = client.Users( - &v1.Users{ - Username: user.username, - }) - if err != nil { - return err - } - - switch { - case ur.TotalMatches != 1: - return fmt.Errorf("total matches got %v, want 1", - ur.TotalMatches) - - case ur.Users[0].ID != user.ID: - return fmt.Errorf("user ID got %v, want %v", - ur.Users[0].ID, user.ID) - } - - // Rescan user payments - fmt.Printf(" Rescan user payments\n") - rupc := RescanUserPaymentsCmd{} - rupc.Args.UserID = user.ID - err = rupc.Execute(nil) - if err != nil { - return err - } - - // Logout - fmt.Printf(" Logout\n") - lc = shared.LogoutCmd{} - err = lc.Execute(nil) - if err != nil { - return err - } - - // Public routes - fmt.Printf("Running public routes\n") - - // Me failure - _, err = client.Me() - if err == nil { - return fmt.Errorf("admin should be logged out") - } - - // Proposal details - fmt.Printf(" Proposal details\n") - pdr, err = client.ProposalDetails(token, nil) - if err != nil { - return err - } - - if pdr.Proposal.Version != "2" { - return fmt.Errorf("proposal details version got %v, want 2", - pdr.Proposal.Version) - } - - // Proposal details version - fmt.Printf(" Proposal details version\n") - pdr, err = client.ProposalDetails(token, - &v1.ProposalsDetails{ - Version: "1", - }) - if err != nil { - return err - } - - if pdr.Proposal.Version != "1" { - return fmt.Errorf("proposal details version got %v, want 1", - pdr.Proposal.Version) - } - - // Proposal details - unreviewed - fmt.Printf(" Proposal details: unreviewed proposal\n") - pdr, err = client.ProposalDetails(unreviewedPropToken, nil) - if err != nil { - return err - } - - switch { - case pdr.Proposal.Name != "": - return fmt.Errorf("proposal name should be empty string") - - case len(pdr.Proposal.Files) != 0: - return fmt.Errorf("proposal files should not be included") - } - - // Proposal details - censored - fmt.Printf(" Proposal details: censored proposal\n") - pdr, err = client.ProposalDetails(censoredPropToken, nil) - if err != nil { - return err - } - - switch { - case pdr.Proposal.Name != "": - return fmt.Errorf("proposal name should be empty string") - - case len(pdr.Proposal.Files) != 0: - return fmt.Errorf("proposal files should not be included") - - case pdr.Proposal.CensoredAt == 0: - return fmt.Errorf("proposal should have a CensoredAt timestamp") - } - - // Proposal comments - fmt.Printf(" Get comments\n") - gcr, err = client.GetComments(token) - if err != nil { - return err - } - - if len(gcr.Comments) != 2 { - return fmt.Errorf("num comments got %v, want 2", - len(gcr.Comments)) - } - - c0 := gcr.Comments[0] - c1 := gcr.Comments[1] - switch { - case c0.CommentID != "1": - return fmt.Errorf("comment ID got %v, want 1", - c0.CommentID) - - case c0.Upvotes != 0: - return fmt.Errorf("comment %v result up votes got %v, want 0", - c0.CommentID, c0.Upvotes) - - case c0.Downvotes != 1: - return fmt.Errorf("comment %v result down votes got %v, want 1", - c0.CommentID, c0.Downvotes) - - case c0.ResultVotes != -1: - return fmt.Errorf("comment %v result vote score got %v, want -1", - c0.CommentID, c0.Downvotes) - - case c1.CommentID != "2": - return fmt.Errorf("comment ID got %v, want 2", - c1.CommentID) - - case c1.Comment != "": - return fmt.Errorf("censored comment text got '%v', want ''", - c1.Comment) - - case !c1.Censored: - return fmt.Errorf("censored comment not marked as censored") - } - - // Vetted proposals. We need to submit a page of proposals and - // make them public first in order to test the vetted route. - fmt.Printf(" Login admin\n") - err = login(admin.email, admin.password) - if err != nil { - return err - } - - fmt.Printf(" Submitting a page of proposals to test vetted route\n") - for i := 0; i < v1.ProposalListPageSize; i++ { - np, err = newNormalProposal() - if err != nil { - return err - } - _, err = client.NewProposal(np) - if err != nil { - return err - } - } - - fmt.Printf(" Token inventory\n") - tir, err := client.TokenInventory() - if err != nil { - return err - } - - fmt.Printf(" Batch proposals\n") - bpr, err := client.BatchProposals(&v1.BatchProposals{ - Tokens: tir.Unreviewed[:v1.ProposalListPageSize], - }) - if err != nil { - return err - } - - fmt.Printf(" Making unvetted proposals public\n") - for _, v := range bpr.Proposals { - spsc := SetProposalStatusCmd{} - spsc.Args.Token = v.CensorshipRecord.Token - spsc.Args.Status = strconv.Itoa(int(v1.PropStatusPublic)) - err = spsc.Execute(nil) - if err != nil { - return err - } - } - - fmt.Printf(" Logout\n") - lc = shared.LogoutCmd{} - err = lc.Execute(nil) - if err != nil { - return err - } - - fmt.Printf(" Vetted\n") - gavr, err := client.GetAllVetted(&v1.GetAllVetted{}) - if err != nil { - return err - } - - if len(gavr.Proposals) != v1.ProposalListPageSize { - return fmt.Errorf("proposals page size got %v, want %v", - len(gavr.Proposals), v1.ProposalListPageSize) - } - - for _, v := range gavr.Proposals { - err = shared.VerifyProposal(v, version.PubKey) - if err != nil { - return fmt.Errorf("verify proposal failed %v: %v", - v.CensorshipRecord.Token, err) - } - } - - // User details - fmt.Printf(" User details\n") - udr, err := client.UserDetails(user.ID) - if err != nil { - return err - } - - if udr.User.ID != user.ID { - return fmt.Errorf("user ID got %v, want %v", - udr.User.ID, user.ID) - } - - // User proposals - fmt.Printf(" User proposals\n") - upr, err := client.UserProposals( - &v1.UserProposals{ - UserId: user.ID, - }) - if err != nil { - return err - } - - // userPropCount is the total number of proposals that we've - // submitted with the test user during the test run that are - // public. - userPropCount := 3 - if upr.NumOfProposals != userPropCount { - return fmt.Errorf("user proposal count got %v, want %v", - upr.NumOfProposals, userPropCount) - } - - for _, v := range upr.Proposals { - err := shared.VerifyProposal(v, version.PubKey) - if err != nil { - return fmt.Errorf("verify proposal failed %v: %v", - v.CensorshipRecord.Token, err) - } - } - - // Vote details - fmt.Printf(" Vote details\n") - vdr, err := client.VoteDetailsV2(token) - if err != nil { - return err - } - switch vdr.Version { - case 2: - // Validate signature - vote, err := decredplugin.DecodeVoteV2([]byte(vdr.Vote)) - if err != nil { - return err - } - dsv := decredplugin.StartVoteV2{ - PublicKey: vdr.PublicKey, - Vote: *vote, - Signature: vdr.Signature, - } - err = dsv.VerifySignature() - if err != nil { - return err - } - default: - return fmt.Errorf("unknown start vote version %v", vdr.Version) - } - - // Vote status - fmt.Printf(" Vote status\n") - vsr, err = client.VoteStatus(token) - if err != nil { - return err - } - - if vsr.Status != v1.PropVoteStatusStarted { - return fmt.Errorf("vote status got %v, want %v", - vsr.Status, v1.PropVoteStatusStarted) - } - - // Active votes - fmt.Printf(" Active votes\n") - avr, err := client.ActiveVotes() - if err != nil { - return err - } - - var found bool - for _, v := range avr.Votes { - if v.Proposal.CensorshipRecord.Token == token { - found = true - } - } - if !found { - return fmt.Errorf("proposal %v not found in active votes", - token) - } - - // Cast votes - fmt.Printf(" Cast votes\n") - dcrwalletFailed, err := castVotes(token, vsr.OptionsResult[0].Option.Id) - - // Find how many votes the user cast so that - // we can compare it against the vote results. - var voteCount int - if !dcrwalletFailed { - // Get proposal vote details - var pvt v1.ProposalVoteTuple - for _, v := range avr.Votes { - if v.Proposal.CensorshipRecord.Token == token { - pvt = v - break - } - } - - // Get the number of eligible tickets the user had - ticketPool, err := convertTicketHashes(pvt.StartVoteReply.EligibleTickets) - if err != nil { - return err - } - - err = client.LoadWalletClient() - if err != nil { - return err - } - defer client.Close() - - ctr, err := client.CommittedTickets( - &walletrpc.CommittedTicketsRequest{ - Tickets: ticketPool, - }) - if err != nil { - return err - } - - voteCount = len(ctr.TicketAddresses) - } - - // Vote results - fmt.Printf(" Vote results\n") - vrr, err := client.VoteResults(token) - if err != nil { - return err - } - - if len(vrr.CastVotes) != voteCount { - return fmt.Errorf("num cast votes got %v, want %v", - len(vrr.CastVotes), voteCount) - } - - // RFP routes - fmt.Println("Running RFP routes") - - // Login with admin to create an RFP - // and make it public. - fmt.Printf(" Login admin\n") - err = login(admin.email, admin.password) - if err != nil { - return err - } - - // Create RFP - fmt.Println(" Create a RFP") - np, err = newRFPProposal() - if err != nil { - return err - } - npr, err = client.NewProposal(np) - if err != nil { - return err - } - - // Verify RFP censorship record - rpr := v1.ProposalRecord{ - Files: np.Files, - Metadata: np.Metadata, - PublicKey: np.PublicKey, - Signature: np.Signature, - CensorshipRecord: npr.CensorshipRecord, - } - err = shared.VerifyProposal(rpr, version.PubKey) - if err != nil { - return fmt.Errorf("verify RFP failed: %v", err) - } - - token = rpr.CensorshipRecord.Token - fmt.Printf(" RFP submitted: %v\n", token) - - // Make RFP public - fmt.Printf(" Set RFP status: not reviewed to public\n") - spsc = SetProposalStatusCmd{} - spsc.Args.Token = token - spsc.Args.Status = strconv.Itoa(int(v1.PropStatusPublic)) - err = spsc.Execute(nil) - if err != nil { - return err - } - - pdr, err = client.ProposalDetails(spsc.Args.Token, nil) - if err != nil { - return err - } - - if pdr.Proposal.Status != v1.PropStatusPublic { - return fmt.Errorf("RFP status got %v, want %v", - pdr.Proposal.Status, v1.PropStatusPublic) - } - - // Try to submit a RFP submission before RFP approval & expect to fail - fmt.Println(" Try submitting a submission before RFP voting") - np, err = newSubmissionProposal(token) - if err != nil { - return err - } - npr, err = client.NewProposal(np) - if err != nil { - switch { - case strings.Contains(err.Error(), "rfp proposal vote did not pass"): - fmt.Println(" Submission failed with expected error") - default: - return err - } - } - - // Authorize vote - fmt.Printf(" Authorize vote: authorize\n") - avc = AuthorizeVoteCmd{} - avc.Args.Token = token - avc.Args.Action = decredplugin.AuthVoteActionAuthorize - err = avc.Execute(nil) - if err != nil { - return err - } - - // Start RFP vote - fmt.Printf(" Start RFP vote\n") - svc = StartVoteCmd{} - svc.Args.Token = token - svc.Args.Duration = policy.MinVoteDuration - svc.Args.PassPercentage = "1" - svc.Args.QuorumPercentage = "1" - err = svc.Execute(nil) - if err != nil { - return err - } - - // Wait to RFP to finish voting - var vs v1.VoteSummary - for vs.Status != v1.PropVoteStatusFinished { - bvs := v1.BatchVoteSummary{ - Tokens: []string{token}, - } - bvsr, err := client.BatchVoteSummary(&bvs) - if err != nil { - return err - } - - vs = bvsr.Summaries[token] - - fmt.Printf(" RFP voting still going on, block %v\\%v \n", - bvsr.BestBlock, vs.EndHeight) - time.Sleep(sleepInterval) - } - if !vs.Approved { - fmt.Println(" RFP rejected") - } else { - return fmt.Errorf("RFP approved? %v, want false", - vs.Approved) - } - - // Try to submit a RFP submission on rejected RFP - fmt.Println(" Try submitting a submission on rejected RFP") - np, err = newSubmissionProposal(token) - if err != nil { - return err - } - npr, err = client.NewProposal(np) - if err != nil { - switch { - case strings.Contains(err.Error(), "rfp proposal vote did not pass"): - fmt.Println(" Submission failed with expected error") - default: - return err - } - } - - // Create another RFP - fmt.Println(" Create another RFP") - np, err = newRFPProposal() - if err != nil { - return err - } - npr, err = client.NewProposal(np) - if err != nil { - return err - } - token = npr.CensorshipRecord.Token - - // Make second RFP public - fmt.Printf(" Set RFP status: not reviewed to public\n") - spsc = SetProposalStatusCmd{} - spsc.Args.Token = token - spsc.Args.Status = strconv.Itoa(int(v1.PropStatusPublic)) - err = spsc.Execute(nil) - if err != nil { - return err - } - - // Authorize vote - fmt.Printf(" Authorize vote: authorize\n") - avc = AuthorizeVoteCmd{} - avc.Args.Token = token - avc.Args.Action = decredplugin.AuthVoteActionAuthorize - err = avc.Execute(nil) - if err != nil { - return err - } - - // Start RFP vote - fmt.Printf(" Start RFP vote\n") - svc = StartVoteCmd{} - svc.Args.Token = token - svc.Args.Duration = policy.MinVoteDuration - svc.Args.PassPercentage = "0" - svc.Args.QuorumPercentage = "0" - err = svc.Execute(nil) - if err != nil { - return err - } - - // Cast RFP votes - fmt.Printf(" Cast 'Yes' votes\n") - dcrwalletFailed, err = castVotes(token, vsr.OptionsResult[0].Option.Id) - - if !dcrwalletFailed { - // Wait to RFP to finish voting - var vs v1.VoteSummary - for vs.Status != v1.PropVoteStatusFinished { - bvs := v1.BatchVoteSummary{ - Tokens: []string{token}, - } - bvsr, err := client.BatchVoteSummary(&bvs) - if err != nil { - return err - } - - vs = bvsr.Summaries[token] - - fmt.Printf(" RFP voting still going on, block %v\\%v \n", - bvsr.BestBlock, vs.EndHeight) - time.Sleep(sleepInterval) - } - if !vs.Approved { - return fmt.Errorf("RFP approved? %v, want true", - vs.Approved) - } - - fmt.Printf(" RFP approved successfully\n") - // Create 4 RFP submissions. - // 1 Unreviewd - fmt.Println(" Create unreviewed RFP submission") - np, err = newSubmissionProposal(token) - if err != nil { - return err - } - npr, err = client.NewProposal(np) - if err != nil { - return err - } - // 2 Public - fmt.Println(" Create 2 more submissions & make them public") - np, err = newSubmissionProposal(token) - if err != nil { - return err - } - npr, err = client.NewProposal(np) - if err != nil { - return err - } - firststoken := npr.CensorshipRecord.Token - fmt.Printf(" Set first submission status: not reviewed to" + - " public\n") - spsc = SetProposalStatusCmd{} - spsc.Args.Token = firststoken - spsc.Args.Status = strconv.Itoa(int(v1.PropStatusPublic)) - err = spsc.Execute(nil) - if err != nil { - return err - } - np, err = newSubmissionProposal(token) - if err != nil { - return err - } - npr, err = client.NewProposal(np) - if err != nil { - return err - } - secondstoken := npr.CensorshipRecord.Token - fmt.Printf(" Set second submission status: not reviewed to" + - " public\n") - spsc = SetProposalStatusCmd{} - spsc.Args.Token = secondstoken - spsc.Args.Status = strconv.Itoa(int(v1.PropStatusPublic)) - err = spsc.Execute(nil) - if err != nil { - return err - } - // 1 Abandoned, first make public - // then abandon - np, err = newSubmissionProposal(token) - if err != nil { - return err - } - npr, err = client.NewProposal(np) - if err != nil { - return err - } - thirdstoken := npr.CensorshipRecord.Token - fmt.Printf(" Set third submission status: not reviewed to" + - " abandoned\n") - spsc = SetProposalStatusCmd{} - spsc.Args.Token = thirdstoken - spsc.Args.Status = strconv.Itoa(int(v1.PropStatusPublic)) - err = spsc.Execute(nil) - if err != nil { - return err - } - spsc.Args.Status = "abandoned" - spsc.Args.Message = "this is spam" - err = spsc.Execute(nil) - if err != nil { - return err - } - // Start runoff vote - fmt.Printf(" Start RFP submissions runoff vote\n") - svrc := StartVoteRunoffCmd{} - svrc.Args.TokenRFP = token - svrc.Args.Duration = policy.MinVoteDuration - svrc.Args.PassPercentage = "0" - svrc.Args.QuorumPercentage = "0" - err = svrc.Execute(nil) - if err != nil { - return err - } - // Cast first submission votes - fmt.Printf(" Cast first submission votes\n") - dcrwalletFailed, err = castVotes(firststoken, vsr.OptionsResult[0].Option.Id) - if err != nil { - return err - } - if !dcrwalletFailed { - // Try cast votes on abandoned & expect error - fmt.Println(" Try casting votes on abandoned RFP submission") - _, err = castVotes(thirdstoken, vsr.OptionsResult[0].Option.Id) - if err != nil { - switch { - case strings.Contains(err.Error(), "proposal not found"): - fmt.Println(" Casting votes on abandoned submission failed") - default: - return err - } - } - - // Wait to runoff vote finish - var vs v1.VoteSummary - for vs.Status != v1.PropVoteStatusFinished { - bvs := v1.BatchVoteSummary{ - Tokens: []string{firststoken}, - } - bvsr, err := client.BatchVoteSummary(&bvs) - if err != nil { - return err - } - - vs = bvsr.Summaries[firststoken] - - fmt.Printf(" Runoff vote still going on, block %v\\%v \n", - bvsr.BestBlock, vs.EndHeight) - time.Sleep(sleepInterval) - } - if !vs.Approved { - return fmt.Errorf("First submission approved? %v, want true", - vs.Approved) - } - fmt.Printf(" First submission approved successfully\n") - } - } - - // Logout - fmt.Printf(" Logout\n") - lc = shared.LogoutCmd{} - err = lc.Execute(nil) - if err != nil { - return err - } - // Websockets fmt.Printf("Running websocket routes\n") diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index f49bf0c6c..068a3585b 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -145,16 +145,22 @@ func (c *Client) makeRequest(method, routeVersion, route string, body interface{ var ue www.UserError err = json.Unmarshal(responseBody, &ue) if err == nil && ue.ErrorCode != 0 { - var e error - if len(ue.ErrorContext) == 0 { - // Error format when an ErrorContext is not included - e = fmt.Errorf("%v, %v", r.StatusCode, userErrorStatus(ue.ErrorCode)) - } else { - // Error format when an ErrorContext is included - e = fmt.Errorf("%v, %v: %v", r.StatusCode, - userErrorStatus(ue.ErrorCode), strings.Join(ue.ErrorContext, ", ")) - } - return nil, e + // TODO the user error should be returned in full and the + // calling function should print the error message. The reason + // is because only the calling function knows what API was used + // and thus what error message to print. + /* + var e error + if len(ue.ErrorContext) == 0 { + // Error format when an ErrorContext is not included + e = fmt.Errorf("%v, %v", r.StatusCode, userErrorStatus(ue.ErrorCode)) + } else { + // Error format when an ErrorContext is included + e = fmt.Errorf("%v, %v: %v", r.StatusCode, + userErrorStatus(ue.ErrorCode), strings.Join(ue.ErrorContext, ", ")) + } + */ + return nil, ue } return nil, fmt.Errorf("%v", r.StatusCode) diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 2f7d406e0..b7eff3db8 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -1289,7 +1289,11 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p // Get the current proposal curr, err := p.proposalRecordLatest(pe.State, pe.Token) if err != nil { - // TODO pi.ErrorStatusPropNotFound + if err == errProposalNotFound { + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusPropNotFound, + } + } return nil, err } diff --git a/politeiawww/www.go b/politeiawww/www.go index 3e607c44d..ab3d9daa5 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -168,6 +168,12 @@ func convertWWWErrorStatus(pluginID string, errCode int) www.ErrorStatusT { func convertPiErrorStatusFromPD(e pd.ErrorStatusT) pi.ErrorStatusT { switch e { + case pd.ErrorStatusInvalidRequestPayload: + // Intentionally omitted because this indicates a politeiawww + // server error so a ErrorStatusInvalid should be returned. + case pd.ErrorStatusInvalidChallenge: + // Intentionally omitted because this indicates a politeiawww + // server error so a ErrorStatusInvalid should be returned. case pd.ErrorStatusInvalidFilename: return pi.ErrorStatusFileNameInvalid case pd.ErrorStatusInvalidFileDigest: @@ -180,12 +186,8 @@ func convertPiErrorStatusFromPD(e pd.ErrorStatusT) pi.ErrorStatusT { return pi.ErrorStatusFileMIMEInvalid case pd.ErrorStatusInvalidRecordStatusTransition: return pi.ErrorStatusPropStatusChangeInvalid - case pd.ErrorStatusInvalidRequestPayload: - // Intentionally omitted because this indicates a politeiawww - // server error so a ErrorStatusInvalid should be returned. - case pd.ErrorStatusInvalidChallenge: - // Intentionally omitted because this indicates a politeiawww - // server error so a ErrorStatusInvalid should be returned. + case pd.ErrorStatusNoChanges: + return pi.ErrorStatusNoPropChanges } return pi.ErrorStatusInvalid } From 69e38ec370b96da2b96423d2e1eab641c2dd1281 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 5 Oct 2020 14:55:39 -0500 Subject: [PATCH 129/449] setup plugin config --- plugins/dcrdata/dcrdata.go | 4 ++ plugins/ticketvote/ticketvote.go | 15 ++--- politeiad/backend/backend.go | 6 ++ politeiad/backend/tlogbe/comments.go | 57 ++++++++++++----- politeiad/backend/tlogbe/dcrdata.go | 60 +++++++++++++----- politeiad/backend/tlogbe/pi.go | 33 ++++++++-- politeiad/backend/tlogbe/ticketvote.go | 87 +++++++++++++++++++++----- politeiad/backend/tlogbe/tlogbe.go | 67 ++++++++++++++------ politeiad/config.go | 3 +- politeiad/politeiad.go | 83 +++++++++++++----------- 10 files changed, 299 insertions(+), 116 deletions(-) diff --git a/plugins/dcrdata/dcrdata.go b/plugins/dcrdata/dcrdata.go index 58180a78c..2b9a354f7 100644 --- a/plugins/dcrdata/dcrdata.go +++ b/plugins/dcrdata/dcrdata.go @@ -24,6 +24,10 @@ const ( CmdTicketPool = "ticketpool" // Get ticket pool CmdTxsTrimmed = "txstrimmed" // Get trimmed transactions + // Default plugin settings + DefaultHostHTTP = "https://dcrdata.decred.org" + DefaultHostWS = "wss://dcrdata.decred.org/ps" + // Dcrdata connection statuses. // // Some commands will return cached results with the connection diff --git a/plugins/ticketvote/ticketvote.go b/plugins/ticketvote/ticketvote.go index 38d08ce3b..ce0c7daef 100644 --- a/plugins/ticketvote/ticketvote.go +++ b/plugins/ticketvote/ticketvote.go @@ -34,6 +34,14 @@ const ( CmdInventory = "inventory" // Get inventory grouped by vote status CmdProofs = "proofs" // Get inclusion proofs + // Default plugin settings + DefaultMainNetVoteDurationMin = 2016 + DefaultMainNetVoteDurationMax = 4032 + DefaultTestNetVoteDurationMin = 0 + DefaultTestNetVoteDurationMax = 4032 + DefaultSimNetVoteDurationMin = 0 + DefaultSimNetVoteDurationMax = 4032 + // TODO implement PolicyVotesPageSize // PolicyVotesPageSize is the maximum number of results that can be // returned from any of the batched vote commands. @@ -71,13 +79,6 @@ const ( // net yes votes. VoteTypeRunoff VoteT = 2 - // Vote duration requirements in blocks - // TODO these are not used anywhere - VoteDurationMinMainnet = 2016 - VoteDurationMaxMainnet = 4032 - VoteDurationMinTestnet = 0 - VoteDurationMaxTestnet = 4032 - // Vote option IDs VoteOptionIDApprove = "yes" VoteOptionIDReject = "no" diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index f41e4fcf9..ff05e0c29 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -10,6 +10,7 @@ import ( "regexp" v1 "github.com/decred/politeia/politeiad/api/v1" + "github.com/decred/politeia/politeiad/api/v1/identity" ) var ( @@ -158,6 +159,11 @@ type Plugin struct { ID string // Identifier Version string // Version Settings []PluginSetting // Settings + + // Identity contains the full identity that the plugin uses to + // create receipts, i.e. signatures of user provided data that + // prove the backend received and processed a plugin command. + Identity *identity.FullIdentity } // InventoryByStatus contains the record tokens of all records in the inventory diff --git a/politeiad/backend/tlogbe/comments.go b/politeiad/backend/tlogbe/comments.go index 4cc45810a..c6b239160 100644 --- a/politeiad/backend/tlogbe/comments.go +++ b/politeiad/backend/tlogbe/comments.go @@ -54,7 +54,6 @@ var ( // commentsPlugin satisfies the pluginClient interface. type commentsPlugin struct { sync.Mutex - id *identity.FullIdentity backend backend.Backend tlog tlogClient @@ -63,6 +62,11 @@ type commentsPlugin struct { // time by walking the trillian trees. dataDir string + // identity contains the full identity that the plugin uses to + // create receipts, i.e. signatures of user provided data that + // prove the backend received and processed a plugin command. + identity *identity.FullIdentity + // Mutexes contains a mutex for each record. The mutexes are lazy // loaded. mutexes map[string]*sync.Mutex // [string]mutex @@ -812,7 +816,7 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { } // Setup comment - receipt := p.id.SignMessage([]byte(n.Signature)) + receipt := p.identity.SignMessage([]byte(n.Signature)) ca := comments.CommentAdd{ UserID: n.UserID, State: n.State, @@ -972,7 +976,7 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { } // Create a new comment version - receipt := p.id.SignMessage([]byte(e.Signature)) + receipt := p.identity.SignMessage([]byte(e.Signature)) ca := comments.CommentAdd{ UserID: e.UserID, Token: e.Token, @@ -1087,7 +1091,7 @@ func (p *commentsPlugin) cmdDel(payload string) (string, error) { } // Prepare comment delete - receipt := p.id.SignMessage([]byte(d.Signature)) + receipt := p.identity.SignMessage([]byte(d.Signature)) cd := comments.CommentDel{ Token: d.Token, CommentID: d.CommentID, @@ -1252,7 +1256,7 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { } // Prepare comment vote - receipt := p.id.SignMessage([]byte(v.Signature)) + receipt := p.identity.SignMessage([]byte(v.Signature)) cv := comments.CommentVote{ UserID: v.UserID, Token: v.Token, @@ -1751,16 +1755,39 @@ func (p *commentsPlugin) setup() error { } // newCommentsPlugin returns a new comments plugin. -func newCommentsPlugin(backend backend.Backend, tlog tlogClient, settings []backend.PluginSetting) *commentsPlugin { - // TODO these should be passed in as plugin settings - id := &identity.FullIdentity{} - dataDir := "" +func newCommentsPlugin(backend backend.Backend, tlog tlogClient, settings []backend.PluginSetting, id *identity.FullIdentity) (*commentsPlugin, error) { + // Unpack plugin settings + var ( + dataDir string + ) + for _, v := range settings { + switch v.Key { + case pluginSettingDataDir: + dataDir = v.Value + default: + return nil, fmt.Errorf("invalid plugin setting '%v'", v.Key) + } + } - return &commentsPlugin{ - id: id, - backend: backend, - tlog: tlog, - dataDir: dataDir, - mutexes: make(map[string]*sync.Mutex), + // Verify plugin settings + switch { + case dataDir == "": + return nil, fmt.Errorf("plugin setting not found: %v", + pluginSettingDataDir) } + + // Create the plugin data directory + dataDir = filepath.Join(dataDir, comments.ID) + err := os.MkdirAll(dataDir, 0700) + if err != nil { + return nil, err + } + + return &commentsPlugin{ + backend: backend, + tlog: tlog, + identity: id, + dataDir: dataDir, + mutexes: make(map[string]*sync.Mutex), + }, nil } diff --git a/politeiad/backend/tlogbe/dcrdata.go b/politeiad/backend/tlogbe/dcrdata.go index 13f89d3f8..9da6e45cd 100644 --- a/politeiad/backend/tlogbe/dcrdata.go +++ b/politeiad/backend/tlogbe/dcrdata.go @@ -14,6 +14,7 @@ import ( "strings" "sync" + "github.com/davecgh/go-spew/spew" v4 "github.com/decred/dcrdata/api/types/v4" exptypes "github.com/decred/dcrdata/explorer/types/v2" pstypes "github.com/decred/dcrdata/pubsub/types/v3" @@ -24,6 +25,10 @@ import ( ) const ( + // Plugin settings + pluginSettingHostHTTP = "hosthttp" + pluginSettingHostWS = "hostws" + // Dcrdata routes routeBestBlock = "/api/block/best" routeBlockDetails = "/api/block/{height}" @@ -430,15 +435,6 @@ func (p *dcrdataPlugin) websocketSetup() { func (p *dcrdataPlugin) setup() error { log.Tracef("dcrdata setup") - // Setup websocket client - ws, err := wsdcrdata.New(p.hostWS) - if err != nil { - // Continue even if a websocket connection was not able to be - // made. Reconnection attempts will be made in the plugin setup. - log.Errorf("wsdcrdata New: %v", err) - } - p.ws = ws - // Setup dcrdata websocket subscriptions and monitoring. This is // done in a go routine so setup will continue in the event that // a dcrdata websocket connection was not able to be made during @@ -487,18 +483,54 @@ func (p *dcrdataPlugin) fsck() error { } func newDcrdataPlugin(settings []backend.PluginSetting) (*dcrdataPlugin, error) { - // TODO these should be passed in as plugin settings - dcrdataHostHTTP := "https://dcrdata.decred.org" - dcrdataHostWS := "wss://dcrdata.decred.org/ps" + spew.Dump(settings) + // Unpack plugin settings + var ( + hostHTTP string + hostWS string + ) + for _, v := range settings { + switch v.Key { + case pluginSettingDataDir: + // The data dir plugin setting is provided to all plugins. The + // dcrdata plugin does not need it. Ignore it. + case pluginSettingHostHTTP: + hostHTTP = v.Value + case pluginSettingHostWS: + hostWS = v.Value + default: + return nil, fmt.Errorf("invalid plugin setting '%v'", v.Key) + } + } + + // Set optional plugin settings to default values if a value was + // not specified. + if hostHTTP == "" { + hostHTTP = dcrdata.DefaultHostHTTP + } + if hostWS == "" { + hostWS = dcrdata.DefaultHostWS + } + // Setup http client + log.Infof("Dcrdata HTTP host: %v", hostHTTP) client, err := util.NewClient(false, "") if err != nil { return nil, err } + // Setup websocket client + ws, err := wsdcrdata.New(hostWS) + if err != nil { + // Continue even if a websocket connection was not able to be + // made. Reconnection attempts will be made in the plugin setup. + log.Errorf("wsdcrdata New: %v", err) + } + return &dcrdataPlugin{ - hostHTTP: dcrdataHostHTTP, - hostWS: dcrdataHostWS, + hostHTTP: hostHTTP, + hostWS: hostWS, client: client, + ws: ws, }, nil } diff --git a/politeiad/backend/tlogbe/pi.go b/politeiad/backend/tlogbe/pi.go index 8a69dad69..65a41a4a5 100644 --- a/politeiad/backend/tlogbe/pi.go +++ b/politeiad/backend/tlogbe/pi.go @@ -663,14 +663,35 @@ func (p *piPlugin) fsck() error { return nil } -func newPiPlugin(backend backend.Backend, tlog tlogClient, settings []backend.PluginSetting) *piPlugin { - // TODO these should be passed in as plugin settings - var ( - dataDir string - ) +func newPiPlugin(backend backend.Backend, tlog tlogClient, settings []backend.PluginSetting) (*piPlugin, error) { + // Unpack plugin settings + var dataDir string + for _, v := range settings { + switch v.Key { + case pluginSettingDataDir: + dataDir = v.Value + default: + return nil, fmt.Errorf("invalid plugin setting '%v'", v.Key) + } + } + + // Verify plugin settings + switch { + case dataDir == "": + return nil, fmt.Errorf("plugin setting not found: %v", + pluginSettingDataDir) + } + + // Create the plugin data directory + dataDir = filepath.Join(dataDir, pi.ID) + err := os.MkdirAll(dataDir, 0700) + if err != nil { + return nil, err + } + return &piPlugin{ dataDir: dataDir, backend: backend, tlog: tlog, - } + }, nil } diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index 2443f0e86..960c63437 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -33,8 +33,9 @@ import ( ) const ( - // ticketVoteDirname is the ticket vote data directory name. - ticketVoteDirname = "ticketvote" + // Plugin setting IDs + pluginSettingVoteDurationMin = "votedurationmin" + pluginSettingVoteDurationMax = "votedurationmax" // Filenames of cached data saved to the plugin data dir. Brackets // are used to indicate a variable that should be replaced in the @@ -69,11 +70,15 @@ type ticketVotePlugin struct { tlog tlogClient // Plugin settings - id *identity.FullIdentity activeNetParams *chaincfg.Params voteDurationMin uint32 // In blocks voteDurationMax uint32 // In blocks + // identity contains the full identity that the plugin uses to + // create receipts, i.e. signatures of user provided data that + // prove the backend received and processed a plugin command. + identity *identity.FullIdentity + // dataDir is the ticket vote plugin data directory. The only data // that is stored here is cached data that can be re-created at any // time by walking the trillian trees. Ex, the vote summary once a @@ -966,7 +971,7 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { } // Prepare authorize vote - receipt := p.id.SignMessage([]byte(a.Signature)) + receipt := p.identity.SignMessage([]byte(a.Signature)) auth := ticketvote.AuthorizeDetails{ Token: a.Token, Version: a.Version, @@ -1470,7 +1475,7 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { } // Save cast vote - receipt := p.id.SignMessage([]byte(v.Signature)) + receipt := p.identity.SignMessage([]byte(v.Signature)) cv := ticketvote.CastVoteDetails{ Token: v.Token, Ticket: v.Ticket, @@ -1963,30 +1968,80 @@ func (p *ticketVotePlugin) fsck() error { return nil } -func newTicketVotePlugin(backend backend.Backend, tlog tlogClient, settings []backend.PluginSetting) *ticketVotePlugin { +func newTicketVotePlugin(backend backend.Backend, tlog tlogClient, settings []backend.PluginSetting, id *identity.FullIdentity, activeNetParams *chaincfg.Params) (*ticketVotePlugin, error) { + // Unpack plugin settings var ( - // TODO these should be passed in as plugin settings dataDir string - id = &identity.FullIdentity{} - activeNetParams = &chaincfg.Params{} voteDurationMin uint32 voteDurationMax uint32 ) + for _, v := range settings { + switch v.Key { + case pluginSettingDataDir: + dataDir = v.Value + case pluginSettingVoteDurationMin: + u, err := strconv.ParseUint(v.Value, 10, 64) + if err != nil { + return nil, fmt.Errorf("plugin setting '%v': ParseUint(%v): %v", + v.Key, v.Value, err) + } + voteDurationMin = uint32(u) + case pluginSettingVoteDurationMax: + u, err := strconv.ParseUint(v.Value, 10, 64) + if err != nil { + return nil, fmt.Errorf("plugin setting '%v': ParseUint(%v): %v", + v.Key, v.Value, err) + } + voteDurationMax = uint32(u) + default: + return nil, fmt.Errorf("invalid plugin setting '%v'", v.Key) + } + } - /* + // Verify required plugin settings + switch { + case dataDir == "": + return nil, fmt.Errorf("plugin setting not found: %v", + pluginSettingDataDir) + } + + // Set optional plugin settings to default values if a value was + // not specified. + if voteDurationMin == 0 { switch activeNetParams.Name { - case chaincfg.MainNetParams.Name: - case chaincfg.TestNet3Params.Name: + case chaincfg.MainNetParams().Name: + voteDurationMin = ticketvote.DefaultMainNetVoteDurationMin + case chaincfg.TestNet3Params().Name: + voteDurationMin = ticketvote.DefaultTestNetVoteDurationMin + case chaincfg.SimNetParams().Name: + voteDurationMin = ticketvote.DefaultSimNetVoteDurationMin } - */ + } + if voteDurationMax == 0 { + switch activeNetParams.Name { + case chaincfg.MainNetParams().Name: + voteDurationMax = ticketvote.DefaultMainNetVoteDurationMax + case chaincfg.TestNet3Params().Name: + voteDurationMax = ticketvote.DefaultTestNetVoteDurationMax + case chaincfg.SimNetParams().Name: + voteDurationMax = ticketvote.DefaultSimNetVoteDurationMax + } + } + + // Create the plugin data directory + dataDir = filepath.Join(dataDir, ticketvote.ID) + err := os.MkdirAll(dataDir, 0700) + if err != nil { + return nil, err + } return &ticketVotePlugin{ - dataDir: filepath.Join(dataDir, ticketVoteDirname), + dataDir: dataDir, backend: backend, tlog: tlog, - id: id, + identity: id, activeNetParams: activeNetParams, voteDurationMin: voteDurationMin, voteDurationMax: voteDurationMax, - } + }, nil } diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index b78a55027..42f620cdd 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -18,6 +18,7 @@ import ( "sync" "time" + "github.com/decred/dcrd/chaincfg/v3" "github.com/decred/dcrtime/merkle" "github.com/decred/politeia/plugins/comments" "github.com/decred/politeia/plugins/dcrdata" @@ -43,6 +44,11 @@ const ( // Tlog instance IDs tlogIDUnvetted = "unvetted" tlogIDVetted = "vetted" + + // The following are the IDs of plugin settings that are derived + // from the politeiad config. The user does not have to set these + // manually. + pluginSettingDataDir = "datadir" ) var ( @@ -80,12 +86,13 @@ var ( // tlogBackend implements the Backend interface. type tlogBackend struct { sync.RWMutex - shutdown bool - homeDir string - dataDir string - unvetted *tlog - vetted *tlog - plugins map[string]plugin // [pluginID]plugin + activeNetParams *chaincfg.Params + homeDir string + dataDir string + shutdown bool + unvetted *tlog + vetted *tlog + plugins map[string]plugin // [pluginID]plugin // prefixes contains the token prefix to full token mapping for all // records. The prefix is the first n characters of the hex encoded @@ -1416,30 +1423,49 @@ func (t *tlogBackend) InventoryByStatus() (*backend.InventoryByStatus, error) { func (t *tlogBackend) RegisterPlugin(p backend.Plugin) error { log.Tracef("RegisterPlugin: %v", p.ID) + // Add tlog backend data dir to plugin settings. The plugin data + // dir should append the plugin ID onto the tlog backend data dir. + p.Settings = append(p.Settings, backend.PluginSetting{ + Key: pluginSettingDataDir, + Value: t.dataDir, + }) + var ( client pluginClient err error ) switch p.ID { case comments.ID: - client = newCommentsPlugin(t, newBackendClient(t), p.Settings) + client, err = newCommentsPlugin(t, newBackendClient(t), + p.Settings, p.Identity) + if err != nil { + return err + } case dcrdata.ID: client, err = newDcrdataPlugin(p.Settings) if err != nil { return err } case pi.ID: - client = newPiPlugin(t, newBackendClient(t), p.Settings) + client, err = newPiPlugin(t, newBackendClient(t), p.Settings) + if err != nil { + return err + } case ticketvote.ID: - client = newTicketVotePlugin(t, newBackendClient(t), p.Settings) + client, err = newTicketVotePlugin(t, newBackendClient(t), + p.Settings, p.Identity, t.activeNetParams) + if err != nil { + return err + } default: return backend.ErrPluginInvalid } t.plugins[p.ID] = plugin{ - id: p.ID, - version: p.Version, - client: client, + id: p.ID, + version: p.Version, + settings: p.Settings, + client: client, } return nil @@ -1605,7 +1631,7 @@ func (t *tlogBackend) setup() error { } // New returns a new tlogBackend. -func New(homeDir, dataDir, dcrtimeHost, encryptionKeyFile, unvettedTrillianHost, unvettedTrillianKeyFile, vettedTrillianHost, vettedTrillianKeyFile string) (*tlogBackend, error) { +func New(anp *chaincfg.Params, homeDir, dataDir, dcrtimeHost, encryptionKeyFile, unvettedTrillianHost, unvettedTrillianKeyFile, vettedTrillianHost, vettedTrillianKeyFile string) (*tlogBackend, error) { // Setup encryption key file if encryptionKeyFile == "" { // No file path was given. Use the default path. @@ -1648,13 +1674,14 @@ func New(homeDir, dataDir, dcrtimeHost, encryptionKeyFile, unvettedTrillianHost, // Setup tlogbe t := tlogBackend{ - homeDir: homeDir, - dataDir: dataDir, - unvetted: unvetted, - vetted: vetted, - plugins: make(map[string]plugin), - prefixes: make(map[string][]byte), - vettedTreeIDs: make(map[string]int64), + activeNetParams: anp, + homeDir: homeDir, + dataDir: dataDir, + unvetted: unvetted, + vetted: vetted, + plugins: make(map[string]plugin), + prefixes: make(map[string][]byte), + vettedTreeIDs: make(map[string]int64), inventory: map[backend.MDStatusT][]string{ backend.MDStatusUnvetted: make([]string, 0), backend.MDStatusIterationUnvetted: make([]string, 0), diff --git a/politeiad/config.go b/politeiad/config.go index 9d82f450a..7cdbdd44e 100644 --- a/politeiad/config.go +++ b/politeiad/config.go @@ -96,7 +96,8 @@ type config struct { TrillianKeyUnvetted string `long:"trilliankeyunvetted"` TrillianKeyVetted string `long:"trilliankeyvetted"` EncryptionKey string `long:"encryptionkey"` - Plugins []string `long:"plugins"` + Plugins []string `long:"plugin"` + PluginSettings []string `long:"pluginsetting"` } // serviceOptions defines the configuration options for the daemon as a service diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 89cfe3b0d..457edba13 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -17,6 +17,7 @@ import ( "os" "os/signal" "runtime/debug" + "strings" "syscall" "time" @@ -1188,8 +1189,8 @@ func _main() error { } p.backend = b case backendTlog: - b, err := tlogbe.New(loadedCfg.HomeDir, loadedCfg.DataDir, - loadedCfg.DcrtimeHost, loadedCfg.EncryptionKey, + b, err := tlogbe.New(activeNetParams.Params, loadedCfg.HomeDir, + loadedCfg.DataDir, loadedCfg.DcrtimeHost, loadedCfg.EncryptionKey, loadedCfg.TrillianHostUnvetted, loadedCfg.TrillianKeyUnvetted, loadedCfg.TrillianHostVetted, loadedCfg.TrillianKeyVetted) if err != nil { @@ -1234,37 +1235,8 @@ func _main() error { p.addRoute(http.MethodPost, v1.UpdateUnvettedMetadataRoute, p.updateUnvettedMetadata, permissionAuth) + // TODO document plugins and plugin settings in README // Setup plugins - /* - plugins, err := p.backend.GetPlugins() - if err != nil { - return err - } - if len(plugins) > 0 { - // Set plugin routes. Requires auth. - p.addRoute(http.MethodPost, v1.PluginCommandRoute, p.pluginCommand, - permissionAuth) - p.addRoute(http.MethodPost, v1.PluginInventoryRoute, p.pluginInventory, - permissionAuth) - - for _, v := range plugins { - // make sure we only have lowercase names - if backend.PluginRE.FindString(v.ID) != v.ID { - return fmt.Errorf("invalid plugin id: %v", v.ID) - } - if _, found := p.plugins[v.ID]; found { - return fmt.Errorf("duplicate plugin: %v", v.ID) - } - p.plugins[v.ID] = convertBackendPlugin(v) - - log.Infof("Registered plugin: %v", v.ID) - } - } - */ - - // Setup plugins - // TODO fix this - loadedCfg.Plugins = []string{"comments", "dcrdata", "pi", "ticketvote"} if len(loadedCfg.Plugins) > 0 { // Set plugin routes. Requires auth. p.addRoute(http.MethodPost, v1.PluginCommandRoute, p.pluginCommand, @@ -1272,33 +1244,70 @@ func _main() error { p.addRoute(http.MethodPost, v1.PluginInventoryRoute, p.pluginInventory, permissionAuth) + // Parse plugin settings + // map[pluginID][]backend.PluginSetting + settings := make(map[string][]backend.PluginSetting) + for _, v := range loadedCfg.PluginSettings { + // Plugin setting will be in format: pluginID,key,value + s := strings.Split(v, ",") + if len(s) != 3 { + return fmt.Errorf("failed to parse plugin setting '%v'; format "+ + "should be 'pluginID,key,value'", s) + } + pluginID := s[0] + ps, ok := settings[pluginID] + if !ok { + ps = make([]backend.PluginSetting, 0, 16) + } + ps = append(ps, backend.PluginSetting{ + Key: s[1], + Value: s[2], + }) + + settings[pluginID] = ps + } + // Register plugins for _, v := range loadedCfg.Plugins { + // Verify plugin ID is lowercase + if backend.PluginRE.FindString(v) != v { + return fmt.Errorf("invalid plugin id: %v", v) + } + + // Get plugin settings + ps, ok := settings[v] + if !ok { + ps = make([]backend.PluginSetting, 0) + } + + // Setup plugin var plugin backend.Plugin switch v { case comments.ID: plugin = backend.Plugin{ ID: comments.ID, Version: comments.Version, - Settings: make([]backend.PluginSetting, 0), + Settings: ps, + Identity: p.identity, } case dcrdata.ID: plugin = backend.Plugin{ ID: dcrdata.ID, Version: dcrdata.Version, - Settings: make([]backend.PluginSetting, 0), + Settings: ps, } case pi.ID: plugin = backend.Plugin{ ID: pi.ID, Version: pi.Version, - Settings: make([]backend.PluginSetting, 0), + Settings: ps, } case ticketvote.ID: plugin = backend.Plugin{ ID: ticketvote.ID, Version: ticketvote.Version, - Settings: make([]backend.PluginSetting, 0), + Settings: ps, + Identity: p.identity, } case decredplugin.ID: // TODO plugin setup for cms @@ -1314,7 +1323,7 @@ func _main() error { return fmt.Errorf("RegisterPlugin %v: %v", v, err) } - // Add to politeiad context + // Add plugin to politeiad context p.plugins[plugin.ID] = convertBackendPlugin(plugin) log.Infof("Registered plugin: %v", v) From 0735565cbc3e97beb9740a9919ee3d3f482e365e Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 5 Oct 2020 17:05:04 -0500 Subject: [PATCH 130/449] cleanup tests --- politeiad/backend/tlogbe/ticketvote.go | 2 +- politeiawww/piwww.go | 4 +- politeiawww/piwww_test.go | 79 + politeiawww/politeiawww_test.go | 140 -- politeiawww/proposals_test.go | 2886 ------------------------ politeiawww/testing.go | 542 +---- politeiawww/user_test.go | 457 ++-- politeiawww/userwww_test.go | 307 +-- politeiawww/www.go | 2 +- 9 files changed, 428 insertions(+), 3991 deletions(-) create mode 100644 politeiawww/piwww_test.go delete mode 100644 politeiawww/proposals_test.go diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index 960c63437..394b696ad 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -1022,7 +1022,7 @@ func voteBitVerify(options []ticketvote.VoteOption, mask, bit uint64) error { } } - return fmt.Errorf("bit 0x%x not found in vote options") + return fmt.Errorf("bit 0x%x not found in vote options", bit) } // TODO test this function diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index b7eff3db8..714e073d2 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -2385,8 +2385,8 @@ func (p *politeiawww) handleVoteInventory(w http.ResponseWriter, r *http.Request util.RespondWithJSON(w, http.StatusOK, vir) } -// setupPiRoutes sets up the pi API routes. -func (p *politeiawww) setupPiRoutes() { +// setPiRoutes sets the pi API routes. +func (p *politeiawww) setPiRoutes() { // Proposal routes p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteProposalNew, p.handleProposalNew, diff --git a/politeiawww/piwww_test.go b/politeiawww/piwww_test.go new file mode 100644 index 000000000..492cc78a5 --- /dev/null +++ b/politeiawww/piwww_test.go @@ -0,0 +1,79 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import "testing" + +func TestProposalNameIsValid(t *testing.T) { + tests := []struct { + name string + want bool + }{ + // empty test + { + "", + false, + }, + // 7 characters + { + "abcdefg", + false, + }, + + // 81 characters + { + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + false, + }, + // 8 characters + { + "12345678", + true, + }, + { + "valid title", + true, + }, + { + " - title: is valid; title. !., ", + true, + }, + { + " - title: is valid; title. ", + true, + }, + { + "\n\n#This-is MY tittle###", + false, + }, + { + "{this-is-the-title}", + false, + }, + { + "\t", + false, + }, + { + "{this -is-the-title} ", + false, + }, + { + "###this is the title***", + false, + }, + { + "###this is the title@+", + true, + }, + } + + for _, test := range tests { + isValid := proposalNameIsValid(test.name) + if isValid != test.want { + t.Errorf("got %v, want %v", isValid, test.want) + } + } +} diff --git a/politeiawww/politeiawww_test.go b/politeiawww/politeiawww_test.go index 831e09502..c4917bab9 100644 --- a/politeiawww/politeiawww_test.go +++ b/politeiawww/politeiawww_test.go @@ -12,7 +12,6 @@ import ( www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/util/version" "github.com/go-test/deep" - "github.com/gorilla/mux" ) func TestHandleVersion(t *testing.T) { @@ -78,142 +77,3 @@ func TestHandleVersion(t *testing.T) { }) } } - -func TestHandleProposalDetails(t *testing.T) { - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - d := newTestPoliteiad(t, p) - defer d.Close() - - user, id := newUser(t, p, true, true) - - propPublic := newProposalRecord(t, user, id, www.PropStatusPublic) - propUnvetted := newProposalRecord(t, user, id, www.PropStatusNotReviewed) - - d.AddRecord(t, convertPropToPD(t, propPublic)) - d.AddRecord(t, convertPropToPD(t, propUnvetted)) - - // Strip non-public proposal information to compare with - // received proposal from a non-logged in request. - wantPropUnvetted := propUnvetted - wantPropUnvetted.Name = "" - wantPropUnvetted.Files = make([]www.File, 0) - - // Since we create a public proposal directly with no status - // changes, the PublishedAt field won't be properly set on - // convertPropFromCache. Therefore, we set it to 0 to match - // the received proposal from the request. - wantPropPublic := propPublic - wantPropPublic.PublishedAt = 0 - - var tests = []struct { - name string - params www.ProposalsDetails - loggedIn bool - wantReply www.ProposalRecord - wantStatus int - wantError error - }{ - { - "proposal not found", - www.ProposalsDetails{ - Token: "invalid-token", - }, - false, - www.ProposalRecord{}, - http.StatusBadRequest, - www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - }, - }, - { - "success unvetted proposal logged out", - www.ProposalsDetails{ - Token: propUnvetted.CensorshipRecord.Token, - }, - false, - wantPropUnvetted, - http.StatusOK, - nil, - }, - { - "success unvetted proposal logged in", - www.ProposalsDetails{ - Token: propUnvetted.CensorshipRecord.Token, - }, - true, - propUnvetted, - http.StatusOK, - nil, - }, - { - "success public proposal", - www.ProposalsDetails{ - Token: propPublic.CensorshipRecord.Token, - }, - false, - wantPropPublic, - http.StatusOK, - nil, - }, - } - - for _, v := range tests { - t.Run(v.name, func(t *testing.T) { - // Prepare request and receive reply - r := httptest.NewRequest(http.MethodGet, www.RouteProposalDetails, nil) - w := httptest.NewRecorder() - - r = mux.SetURLVars(r, map[string]string{ - "token": v.params.Token, - }) - - if v.loggedIn { - addSessionToReq(t, p, r, user.ID.String()) - } - - p.handleProposalDetails(w, r) - res := w.Result() - body, _ := ioutil.ReadAll(res.Body) - res.Body.Close() - - var gotReply www.ProposalDetailsReply - err := json.Unmarshal(body, &gotReply) - if err != nil { - t.Errorf("unmarshal error with body %v", body) - } - - // Validate expected proposal with received proposal - diff := deep.Equal(gotReply.Proposal, v.wantReply) - if diff != nil { - t.Errorf("got/want diff:\n%v", - spew.Sdump(diff)) - } - - // Validate http status code - if res.StatusCode != v.wantStatus { - t.Errorf("got status code %v, want %v", - res.StatusCode, v.wantStatus) - } - - // Check if request was successful - if res.StatusCode == http.StatusOK { - return - } - - // Receive user error when request fails - var ue www.UserError - err = json.Unmarshal(body, &ue) - if err != nil { - t.Errorf("unmarshal UserError: %v", err) - } - - got := errToStr(ue) - want := errToStr(v.wantError) - if got != want { - t.Errorf("got error %v, want %v", got, want) - } - }) - } -} diff --git a/politeiawww/proposals_test.go b/politeiawww/proposals_test.go deleted file mode 100644 index 3773fbd43..000000000 --- a/politeiawww/proposals_test.go +++ /dev/null @@ -1,2886 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import ( - "context" - "encoding/hex" - "fmt" - "reflect" - "strconv" - "testing" - "time" - - "github.com/decred/politeia/decredplugin" - pd "github.com/decred/politeia/politeiad/api/v1" - www "github.com/decred/politeia/politeiawww/api/www/v1" - www2 "github.com/decred/politeia/politeiawww/api/www/v2" - "github.com/decred/politeia/politeiawww/user" - "github.com/decred/politeia/util" -) - -const ( - // Helper vote duration constants - minDuration = 2016 - maxDuration = 4032 -) - -func TestIsValidProposalName(t *testing.T) { - tests := []struct { - name string // @rgeraldes - valid input is a string without new lines - want bool - }{ - // empty test - { - "", - false, - }, - // 7 characters - { - "abcdefg", - false, - }, - - // 81 characters - { - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - false, - }, - // 8 characters - { - "12345678", - true, - }, - { - "valid title", - true, - }, - { - " - title: is valid; title. !., ", - true, - }, - { - " - title: is valid; title. ", - true, - }, - { - "\n\n#This-is MY tittle###", - false, - }, - { - "{this-is-the-title}", - false, - }, - { - "\t", - false, - }, - { - "{this -is-the-title} ", - false, - }, - { - "###this is the title***", - false, - }, - { - "###this is the title@+", - true, - }, - } - - for _, test := range tests { - isValid := proposalNameIsValid(test.name) - if isValid != test.want { - t.Errorf("got %v, want %v", isValid, test.want) - } - } -} - -func TestIsProposalAuthor(t *testing.T) { - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - usr, id := newUser(t, p, true, false) - notAuthorUser, _ := newUser(t, p, true, false) - - proposal := newProposalRecord(t, usr, id, www.PropStatusPublic) - - var tests = []struct { - name string - proposal www.ProposalRecord - usr *user.User - want bool - }{ - { - "is proposal author", - proposal, - usr, - true, - }, - { - "is not proposal author", - proposal, - notAuthorUser, - false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - isAuthor := isProposalAuthor(test.proposal, *test.usr) - if isAuthor != test.want { - t.Errorf("got %v, want %v", isAuthor, test.want) - } - }) - } -} - -func TestIsRFP(t *testing.T) { - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - usr, id := newUser(t, p, true, false) - - rfpProposal := newProposalRecord(t, usr, id, www.PropStatusPublic) - rfpProposalSubmission := newProposalRecord(t, usr, id, www.PropStatusPublic) - - linkFrom := []string{rfpProposalSubmission.CensorshipRecord.Token} - linkTo := rfpProposal.CensorshipRecord.Token - linkBy := time.Now().Add(time.Hour * 24 * 30).Unix() - - rfpSubmissions := []*www.ProposalRecord{&rfpProposalSubmission} - - makeProposalRFP(t, &rfpProposal, linkFrom, linkBy) - makeProposalRFPSubmissions(t, rfpSubmissions, linkTo) - - var tests = []struct { - name string - proposal www.ProposalRecord - want bool - }{ - { - "is RFP proposal", - rfpProposal, - true, - }, - { - "is not RFP proposal", - rfpProposalSubmission, - false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - isRFP := isRFP(test.proposal) - if isRFP != test.want { - t.Errorf("got %v, want %v", isRFP, test.want) - } - }) - } -} - -func TestIsRFPSubmission(t *testing.T) { - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - usr, id := newUser(t, p, true, false) - - rfpProposal := newProposalRecord(t, usr, id, www.PropStatusPublic) - rfpProposalSubmission := newProposalRecord(t, usr, id, www.PropStatusPublic) - - linkFrom := []string{rfpProposalSubmission.CensorshipRecord.Token} - linkTo := rfpProposal.CensorshipRecord.Token - linkBy := time.Now().Add(time.Hour * 24 * 30).Unix() - rfpSubmissions := []*www.ProposalRecord{&rfpProposalSubmission} - - makeProposalRFP(t, &rfpProposal, linkFrom, linkBy) - makeProposalRFPSubmissions(t, rfpSubmissions, linkTo) - - var tests = []struct { - name string - proposal www.ProposalRecord - want bool - }{ - { - "is RFP submission", - rfpProposalSubmission, - true, - }, - { - "is not RFP submission", - rfpProposal, - false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - isRFPSubmission := isRFPSubmission(test.proposal) - if isRFPSubmission != test.want { - t.Errorf("got %v, want %v", isRFPSubmission, test.want) - } - }) - } -} - -func TestVoteIsApproved(t *testing.T) { - yes := decredplugin.VoteOptionIDApprove - no := decredplugin.VoteOptionIDReject - emptyResults := []www.VoteOptionResult{} - badQuorumResults := []www.VoteOptionResult{ - newVoteOptionResult(t, no, "not approve", 1, 0), - newVoteOptionResult(t, yes, "approve", 2, 1), - } - badPassPercentageResults := []www.VoteOptionResult{ - newVoteOptionResult(t, no, "not approve", 1, 8), - newVoteOptionResult(t, yes, "approve", 2, 2), - } - approvedResults := []www.VoteOptionResult{ - newVoteOptionResult(t, no, "not approve", 1, 2), - newVoteOptionResult(t, yes, "approve", 2, 8), - } - vsVoteNotFinished := newVoteSummary(t, www.PropVoteStatusAuthorized, - emptyResults) - vsQuorumNotMet := newVoteSummary(t, www.PropVoteStatusFinished, - badQuorumResults) - vsPassPercentageNotMet := newVoteSummary(t, www.PropVoteStatusFinished, - badPassPercentageResults) - vsApproved := newVoteSummary(t, www.PropVoteStatusFinished, approvedResults) - - var tests = []struct { - name string - summary www.VoteSummary - want bool - }{ - { - "vote not finished", - vsVoteNotFinished, - false, - }, - { - "vote did not pass quorum", - vsQuorumNotMet, - false, - }, - { - "vote did not meet pass percentage", - vsPassPercentageNotMet, - false, - }, - { - "vote is approved", - vsApproved, - true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - got := voteIsApproved(test.summary) - if got != test.want { - t.Errorf("got %v, want %v", got, test.want) - } - }) - } -} - -func TestValidateProposalMetadata(t *testing.T) { - // TODO - /* - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - d := newTestPoliteiad(t, p) - defer d.Close() - - usr, id := newUser(t, p, true, false) - - // Helper variables - no := decredplugin.VoteOptionIDReject - yes := decredplugin.VoteOptionIDApprove - public := www.PropStatusPublic - linkFrom := []string{} - linkBy := time.Now().Add(time.Hour * 24 * 30).Unix() - invalidName := "bad" - validName := "valid name" - invalidToken := "abcdfe" - randomToken, err := util.Random(pd.TokenSize) - if err != nil { - t.Fatal(err) - } - rToken := hex.EncodeToString(randomToken) - - // Public proposal - proposal := newProposalRecord(t, usr, id, public) - token := proposal.CensorshipRecord.Token - d.AddRecord(t, convertPropToPD(t, proposal)) - - // RFP proposal approved - rfpProposal := newProposalRecord(t, usr, id, public) - rfpToken := rfpProposal.CensorshipRecord.Token - makeProposalRFP(t, &rfpProposal, linkFrom, linkBy) - d.AddRecord(t, convertPropToPD(t, rfpProposal)) - - // RFP proposal not approved - rfpProposalNotApproved := newProposalRecord(t, usr, id, public) - rfpTokenNotApproved := rfpProposalNotApproved.CensorshipRecord.Token - makeProposalRFP(t, &rfpProposalNotApproved, linkFrom, linkBy) - d.AddRecord(t, convertPropToPD(t, rfpProposalNotApproved)) - - // RFP bad state - rfpBadState := newProposalRecord(t, usr, id, www.PropStatusNotReviewed) - rfpBadStateToken := rfpBadState.CensorshipRecord.Token - makeProposalRFP(t, &rfpBadState, linkFrom, linkBy) - d.AddRecord(t, convertPropToPD(t, rfpBadState)) - - // Approved VoteSummary for proposal - approved := []www.VoteOptionResult{ - newVoteOptionResult(t, no, "not approve", 1, 2), - newVoteOptionResult(t, yes, "approve", 2, 8), - } - vsApproved := newVoteSummary(t, www.PropVoteStatusFinished, approved) - p.voteSummarySet(rfpToken, vsApproved) - - // Not approved VoteSummary for proposal - notApproved := []www.VoteOptionResult{ - newVoteOptionResult(t, no, "not approve", 1, 8), - newVoteOptionResult(t, yes, "approve", 2, 2), - } - vsNotApproved := newVoteSummary(t, www.PropVoteStatusFinished, notApproved) - p.voteSummarySet(rfpTokenNotApproved, vsNotApproved) - p.voteSummarySet(rfpBadStateToken, vsApproved) - - // Metadatas to validate and test - _, mdInvalidName := newProposalMetadata(t, invalidName, "", 0) - // LinkTo validations - _, mdInvalidLinkTo := newProposalMetadata(t, validName, invalidToken, 0) - _, mdProposalNotFound := newProposalMetadata(t, validName, rToken, 0) - _, mdProposalNotRFP := newProposalMetadata(t, validName, token, 0) - _, mdProposalNotApproved := newProposalMetadata(t, validName, - rfpTokenNotApproved, 0) - _, mdProposalBadState := newProposalMetadata(t, validName, - rfpBadStateToken, 0) - _, mdProposalBothRFP := newProposalMetadata(t, validName, rfpToken, - time.Now().Unix()) - // LinkBy validations - _, mdLinkByMin := newProposalMetadata(t, validName, "", - p.linkByPeriodMin()-1) - _, mdLinkByMax := newProposalMetadata(t, validName, "", - p.linkByPeriodMax()+1) - _, mdSuccess := newProposalMetadata(t, validName, rfpToken, 0) - - var tests = []struct { - name string - metadata www.ProposalMetadata - want error - }{ - { - "invalid proposal name", - mdInvalidName, - www.UserError{ - ErrorCode: www.ErrorStatusProposalInvalidTitle, - }, - }, - { - "invalid linkTo bad token", - mdInvalidLinkTo, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkTo, - }, - }, - { - "invalid linkTo token proposal not found", - mdProposalNotFound, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkTo, - }, - }, - { - "invalid linkTo not a RFP proposal", - mdProposalNotRFP, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkTo, - }, - }, - { - "invalid linkTo RFP proposal vote not approved", - mdProposalNotApproved, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkTo, - }, - }, - { - "invalid linkTo proposal is not vetted", - mdProposalBadState, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkTo, - }, - }, - { - "invalid linkTo rfp proposal linked to another rfp", - mdProposalBothRFP, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkTo, - }, - }, - { - "invalid linkBy shorter than min", - mdLinkByMin, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkBy, - }, - }, - { - "invalid linkBy greather than max", - mdLinkByMax, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkBy, - }, - }, - { - "validation success", - mdSuccess, - nil, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - err := p.validateProposalMetadata(test.metadata) - got := errToStr(err) - want := errToStr(test.want) - if got != want { - t.Errorf("got %v, want %v", got, want) - } - }) - } - */ -} - -func TestValidateProposal(t *testing.T) { - // TODO - /* - // Setup politeiawww and a test user - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - usr, id := newUser(t, p, true, false) - - // Create test data - md := createFileMD(t, 8) - png := createFilePNG(t, false) - np := createNewProposal(t, id, []www.File{*md, *png}, "") - - // Invalid signature - propInvalidSig := createNewProposal(t, id, []www.File{*md}, "") - propInvalidSig.Signature = "abc" - - // Signature is valid but incorrect - propBadSig := createNewProposal(t, id, []www.File{*md}, "") - propBadSig.Signature = np.Signature - - // No files - propNoFiles := createNewProposal(t, id, []www.File{}, "") - - // Invalid markdown filename - mdBadFilename := *md - mdBadFilename.Name = "bad_filename.md" - propBadFilename := createNewProposal(t, id, []www.File{mdBadFilename}, "") - - // Duplicate filenames - propDupFiles := createNewProposal(t, id, []www.File{*md, *png, *png}, "") - - // Too many markdown files. We need one correctly named md - // file and the rest must have their names changed so that we - // don't get a duplicate filename error. - files := make([]www.File, 0, www.PolicyMaxMDs+1) - files = append(files, *md) - for i := 0; i < www.PolicyMaxMDs; i++ { - m := *md - m.Name = fmt.Sprintf("%v.md", i) - files = append(files, m) - } - propMaxMDFiles := createNewProposal(t, id, files, "") - - // Too many image files. All of their names must be different - // so that we don't get a duplicate filename error. - files = make([]www.File, 0, www.PolicyMaxImages+2) - files = append(files, *md) - for i := 0; i <= www.PolicyMaxImages; i++ { - p := *png - p.Name = fmt.Sprintf("%v.png", i) - files = append(files, p) - } - propMaxImages := createNewProposal(t, id, files, "") - - // Markdown file too large - mdLarge := createFileMD(t, www.PolicyMaxMDSize) - propMDLarge := createNewProposal(t, id, []www.File{*mdLarge, *png}, "") - - // Image too large - pngLarge := createFilePNG(t, true) - propImageLarge := createNewProposal(t, id, []www.File{*md, *pngLarge}, "") - - // Invalid proposal title - mdBadTitle := createFileMD(t, 8) - propBadTitle := createNewProposal(t, id, - []www.File{*mdBadTitle}, "{invalid-title}") - - // Empty file payload - propEmptyFile := createNewProposal(t, id, []www.File{*md}, "") - emptyFile := createFileMD(t, 8) - emptyFile.Payload = "" - propEmptyFile.Files = []www.File{*emptyFile} - - // Invalid file digest - propInvalidDigest := createNewProposal(t, id, []www.File{*md}, "") - invalidDigestFile := createFilePNG(t, false) - invalidDigestFile.Digest = "" - propInvalidDigest.Files = append(propInvalidDigest.Files, *invalidDigestFile) - - // Invalid MIME type - propInvalidMIME := createNewProposal(t, id, []www.File{*md}, "") - invalidMIMEFile := createFilePNG(t, false) - invalidMIMEFile.MIME = "image/jpeg" - propInvalidMIME.Files = append(propInvalidMIME.Files, *invalidMIMEFile) - - // Invalid metadata - propInvalidMetadata := createNewProposal(t, id, []www.File{*md}, "") - invalidMd := www.Metadata{ - Digest: "", - Payload: "", - Hint: www.HintProposalMetadata, - } - propInvalidMetadata.Metadata = append(propInvalidMetadata.Metadata, invalidMd) - - // Missing metadata - propMissingMetadata := createNewProposal(t, id, []www.File{*md}, "") - propMissingMetadata.Metadata = nil - - // Setup test cases - var tests = []struct { - name string - newProposal www.NewProposal - user *user.User - want error - }{ - { - "correct proposal", - *np, - usr, - nil, - }, - { - "invalid signature", - *propInvalidSig, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidSignature, - }, - }, - { - "incorrect signature", - *propBadSig, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidSignature, - }, - }, - { - "missing files", - *propNoFiles, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusProposalMissingFiles, - }, - }, - { - "bad md filename", - *propBadFilename, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusMaxMDsExceededPolicy, - }, - }, - { - "duplicate filenames", - *propDupFiles, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusProposalDuplicateFilenames, - }, - }, - { - "empty file payload", - *propEmptyFile, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidBase64, - }, - }, - { - "invalid file digest", - *propInvalidDigest, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidFileDigest, - }, - }, - { - "invalid file MIME type", - *propInvalidMIME, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidMIMEType, - }, - }, - { - "invalid metadata", - *propInvalidMetadata, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusMetadataInvalid, - }, - }, - { - "missing metadata", - *propMissingMetadata, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusMetadataMissing, - }, - }, - { - "too may md files", - *propMaxMDFiles, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusMaxMDsExceededPolicy, - }, - }, - { - "too many images", - *propMaxImages, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusMaxImagesExceededPolicy, - }, - }, - { - "md file too large", - *propMDLarge, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusMaxMDSizeExceededPolicy, - }, - }, - { - "image too large", - *propImageLarge, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusMaxImageSizeExceededPolicy, - }, - }, - { - "invalid title", - *propBadTitle, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusProposalInvalidTitle, - }, - }, - } - - // Run test cases - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - _, err := p.validateProposal(test.newProposal, test.user) - got := errToStr(err) - want := errToStr(test.want) - if got != want { - t.Errorf("got %v, want %v", got, want) - } - }) - } - */ -} - -func TestValidateAuthorizeVote(t *testing.T) { - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - d := newTestPoliteiad(t, p) - defer d.Close() - - usr, id := newUser(t, p, true, false) - _, id2 := newUser(t, p, true, false) - - authorize := decredplugin.AuthVoteActionAuthorize - revoke := decredplugin.AuthVoteActionRevoke - - // Public proposal - prop := newProposalRecord(t, usr, id, www.PropStatusPublic) - token := prop.CensorshipRecord.Token - d.AddRecord(t, convertPropToPD(t, prop)) - - // Authorize vote - av := newAuthorizeVote(t, token, prop.Version, authorize, id) - - // Revoke vote - rv := newAuthorizeVote(t, token, prop.Version, revoke, id) - - // Wrong status proposal - propUnreviewed := newProposalRecord(t, usr, id, www.PropStatusNotReviewed) - d.AddRecord(t, convertPropToPD(t, prop)) - - // Invalid signing key - avInvalid := newAuthorizeVote(t, token, prop.Version, authorize, id) - avInvalid.PublicKey = hex.EncodeToString(id2.Public.Key[:]) - - // Invalid vote action - avInvalidAct := newAuthorizeVote(t, token, prop.Version, "bad", id) - - var tests = []struct { - name string - av www.AuthorizeVote - u user.User - pr www.ProposalRecord - vs www.VoteSummary - want error - }{ - { - "invalid signing key", - avInvalid, - *usr, - prop, - www.VoteSummary{}, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidSigningKey, - }, - }, - { - "wrong proposal status", - av, - *usr, - propUnreviewed, - www.VoteSummary{}, - www.UserError{ - ErrorCode: www.ErrorStatusWrongStatus, - }, - }, - { - "vote has already started", - av, - *usr, - prop, - www.VoteSummary{ - EndHeight: 1552, - }, - www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - }, - }, - { - "invalid auth vote action", - avInvalidAct, - *usr, - prop, - www.VoteSummary{}, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidAuthVoteAction, - }, - }, - { - "vote has already been authorized", - av, - *usr, - prop, - www.VoteSummary{ - Status: www.PropVoteStatusAuthorized, - }, - www.UserError{ - ErrorCode: www.ErrorStatusVoteAlreadyAuthorized, - }, - }, - { - "cannot revoke vote that has not been authorized", - rv, - *usr, - prop, - www.VoteSummary{ - Status: www.PropVoteStatusNotAuthorized, - }, - www.UserError{ - ErrorCode: www.ErrorStatusVoteNotAuthorized, - }, - }, - { - "valid authorize vote", - av, - *usr, - prop, - www.VoteSummary{}, - nil, - }, - { - "valid revoke vote", - rv, - *usr, - prop, - www.VoteSummary{ - Status: www.PropVoteStatusAuthorized, - }, - nil, - }, - } - - // Run test cases - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - err := validateAuthorizeVote(test.av, test.u, test.pr, test.vs) - got := errToStr(err) - want := errToStr(test.want) - if got != want { - t.Errorf("got %v, want %v", got, want) - } - }) - } -} - -func TestValidateAuthorizeVoteStandard(t *testing.T) { - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - d := newTestPoliteiad(t, p) - defer d.Close() - - usr, id := newUser(t, p, true, false) - _, id2 := newUser(t, p, true, false) - - authorize := decredplugin.AuthVoteActionAuthorize - - // RFP proposal - rfpProp := newProposalRecord(t, usr, id, www.PropStatusPublic) - rfpPropToken := rfpProp.CensorshipRecord.Token - d.AddRecord(t, convertPropToPD(t, rfpProp)) - - // RFP proposal submission - prop := newProposalRecord(t, usr, id, www.PropStatusPublic) - token := prop.CensorshipRecord.Token - makeProposalRFPSubmissions(t, []*www.ProposalRecord{&prop}, rfpPropToken) - d.AddRecord(t, convertPropToPD(t, prop)) - - // Public proposal submission wrong author - prop2 := newProposalRecord(t, usr, id, www.PropStatusPublic) - prop2.PublicKey = hex.EncodeToString(id2.Public.Key[:]) - d.AddRecord(t, convertPropToPD(t, prop2)) - - // Valid proposal vote auth - propValid := newProposalRecord(t, usr, id, www.PropStatusPublic) - d.AddRecord(t, convertPropToPD(t, propValid)) - - // Authorize vote - av := newAuthorizeVote(t, token, prop.Version, authorize, id) - - var tests = []struct { - name string - av www.AuthorizeVote - u user.User - pr www.ProposalRecord - vs www.VoteSummary - want error - }{ - { - "proposal is a RFP submission", - av, - *usr, - prop, - www.VoteSummary{}, - www.UserError{ - ErrorCode: www.ErrorStatusWrongProposalType, - }, - }, - { - "not proposal author", - av, - *usr, - prop2, - www.VoteSummary{}, - www.UserError{ - ErrorCode: www.ErrorStatusUserNotAuthor, - }, - }, - { - "valid", - av, - *usr, - propValid, - www.VoteSummary{}, - nil, - }, - } - - // Run test cases - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - err := validateAuthorizeVoteStandard(test.av, test.u, test.pr, test.vs) - got := errToStr(err) - want := errToStr(test.want) - if got != want { - t.Errorf("got %v, want %v", got, want) - } - }) - } -} - -func TestValidateAuthorizeVoteRunoff(t *testing.T) { - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - d := newTestPoliteiad(t, p) - defer d.Close() - - usr, id := newUser(t, p, true, true) - usrNotAdmin, _ := newUser(t, p, true, false) - - authorize := decredplugin.AuthVoteActionAuthorize - - // Public proposal - prop := newProposalRecord(t, usr, id, www.PropStatusPublic) - token := prop.CensorshipRecord.Token - d.AddRecord(t, convertPropToPD(t, prop)) - - // Authorize vote - av := newAuthorizeVote(t, token, prop.Version, authorize, id) - - var tests = []struct { - name string - av www.AuthorizeVote - u user.User - pr www.ProposalRecord - vs www.VoteSummary - want error - }{ - { - "user is not an admin", - av, - *usrNotAdmin, - prop, - www.VoteSummary{}, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidSigningKey, - }, - }, - { - "valid", - av, - *usr, - prop, - www.VoteSummary{}, - nil, - }, - } - - // Run test cases - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - err := validateAuthorizeVoteRunoff(test.av, test.u, test.pr, test.vs) - got := errToStr(err) - want := errToStr(test.want) - if got != want { - t.Errorf("got %v, want %v", got, want) - } - }) - } -} - -func TestValidateVoteOptions(t *testing.T) { - approve := decredplugin.VoteOptionIDApprove - reject := decredplugin.VoteOptionIDReject - invalidVoteOption := []www2.VoteOption{ - newVoteOptionV2(t, "wrong", "", 0x01), - } - missingReject := []www2.VoteOption{ - newVoteOptionV2(t, approve, "", 0x02), - } - missingApprove := []www2.VoteOption{ - newVoteOptionV2(t, reject, "", 0x01), - } - valid := []www2.VoteOption{ - newVoteOptionV2(t, approve, "", 0x02), - newVoteOptionV2(t, reject, "", 0x01), - } - var tests = []struct { - name string - vos []www2.VoteOption - want error - }{ - { - "no vote options found", - []www2.VoteOption{}, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidVoteOptions, - }, - }, - { - "invalid vote option", - invalidVoteOption, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidVoteOptions, - }, - }, - { - "missing reject vote option", - missingReject, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidVoteOptions, - }, - }, - { - "missing approve vote option", - missingApprove, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidVoteOptions, - }, - }, - { - "valid", - valid, - nil, - }, - } - - // Run test cases - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - err := validateVoteOptions(test.vos) - got := errToStr(err) - want := errToStr(test.want) - if got != want { - t.Errorf("got %v, want %v", got, want) - } - }) - } -} - -func TestValidateVoteBit(t *testing.T) { - approve := decredplugin.VoteOptionIDApprove - reject := decredplugin.VoteOptionIDReject - vote := www2.Vote{ - Mask: 0x03, - Options: []www2.VoteOption{ - newVoteOptionV2(t, approve, "", 0x02), - newVoteOptionV2(t, reject, "", 0x01), - }, - } - - var tests = []struct { - name string - vote www2.Vote - bit uint64 - want error - }{ - { - "vote corrupt", - www2.Vote{}, - 0x01, - fmt.Errorf("vote corrupt"), - }, - { - "invalid bit", - vote, - 0, - fmt.Errorf("invalid bit 0x0"), - }, - { - "invalid bit mask", - vote, - 0x04, - fmt.Errorf("invalid mask 0x3 bit 0x4"), - }, - { - "bit not found", - vote, - 0x03, - fmt.Errorf("bit not found 0x3"), - }, - { - "valid", - vote, - 0x02, - nil, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - err := validateVoteBit(test.vote, test.bit) - got := errToStr(err) - want := errToStr(test.want) - if got != want { - t.Errorf("got %v, want %v", got, want) - } - }) - } -} - -func TestValidateStartVote(t *testing.T) { - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - d := newTestPoliteiad(t, p) - defer d.Close() - - usr, id := newUser(t, p, true, false) - - prop := newProposalRecord(t, usr, id, www.PropStatusPublic) - token := prop.CensorshipRecord.Token - d.AddRecord(t, convertPropToPD(t, prop)) - - propUnvetted := newProposalRecord(t, usr, id, www.PropStatusNotReviewed) - d.AddRecord(t, convertPropToPD(t, propUnvetted)) - - standard := www2.VoteTypeStandard - - sv := newStartVote(t, token, 1, minDuration, standard, id) - - svInvalidToken := newStartVote(t, token, 1, minDuration, standard, id) - svInvalidToken.Vote.Token = "" - - svInvalidBit := newStartVote(t, token, 1, minDuration, standard, id) - svInvalidBit.Vote.Options[0].Bits = 0 - - svInvalidOpt := newStartVote(t, token, 1, minDuration, standard, id) - svInvalidOpt.Vote.Options[0].Id = "wrong" - - svInvalidMinDuration := newStartVote(t, token, 1, minDuration, standard, id) - svInvalidMinDuration.Vote.Duration = 1 - - svInvalidMaxDuration := newStartVote(t, token, 1, minDuration, standard, id) - svInvalidMaxDuration.Vote.Duration = 4050 - - svInvalidQuorum := newStartVote(t, token, 1, minDuration, standard, id) - svInvalidQuorum.Vote.QuorumPercentage = 110 - - svInvalidPass := newStartVote(t, token, 1, minDuration, standard, id) - svInvalidPass.Vote.PassPercentage = 110 - - svInvalidKey := newStartVote(t, token, 1, minDuration, standard, id) - svInvalidKey.PublicKey = "" - - svInvalidSig := newStartVote(t, token, 1, minDuration, standard, id) - svInvalidSig.Signature = "" - - svInvalidVersion := newStartVote(t, token, 2, minDuration, standard, id) - - var tests = []struct { - name string - sv www2.StartVote - u user.User - pr www.ProposalRecord - vs www.VoteSummary - want error - }{ - { - "invalid proposal token", - svInvalidToken, - *usr, - prop, - www.VoteSummary{}, - fmt.Errorf("invalid token %v", svInvalidToken.Vote.Token), - }, - { - "invalid vote bit", - svInvalidBit, - *usr, - prop, - www.VoteSummary{}, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidPropVoteBits, - }, - }, - { - "invalid vote option", - svInvalidOpt, - *usr, - prop, - www.VoteSummary{}, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidVoteOptions, - }, - }, - { - "invalid minimum duration", - svInvalidMinDuration, - *usr, - prop, - www.VoteSummary{}, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidPropVoteParams, - }, - }, - { - "invalid maximum duration", - svInvalidMaxDuration, - *usr, - prop, - www.VoteSummary{}, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidPropVoteParams, - }, - }, - { - "quorum too large", - svInvalidQuorum, - *usr, - prop, - www.VoteSummary{}, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidPropVoteParams, - }, - }, - { - "pass percentage too large", - svInvalidPass, - *usr, - prop, - www.VoteSummary{}, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidPropVoteParams, - }, - }, - { - "invalid public key from start vote", - svInvalidKey, - *usr, - prop, - www.VoteSummary{}, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidSigningKey, - }, - }, - { - "invalid signature", - svInvalidSig, - *usr, - prop, - www.VoteSummary{}, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidSignature, - }, - }, - { - "invalid proposal version", - svInvalidVersion, - *usr, - prop, - www.VoteSummary{}, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidProposalVersion, - }, - }, - { - "invalid proposal status", - sv, - *usr, - propUnvetted, - www.VoteSummary{}, - www.UserError{ - ErrorCode: www.ErrorStatusWrongStatus, - }, - }, - { - "vote has already started", - sv, - *usr, - prop, - www.VoteSummary{ - EndHeight: 1024, - }, - www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - }, - }, - { - "valid", - sv, - *usr, - prop, - www.VoteSummary{}, - nil, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - err := validateStartVote(test.sv, test.u, test.pr, test.vs, minDuration, maxDuration) - got := errToStr(err) - want := errToStr(test.want) - if got != want { - t.Errorf("got %v, want %v", got, want) - } - }) - } -} - -func TestValidateStartVoteStandard(t *testing.T) { - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - d := newTestPoliteiad(t, p) - defer d.Close() - - usr, id := newUser(t, p, true, false) - - // RFP proposal - prop := newProposalRecord(t, usr, id, www.PropStatusPublic) - token := prop.CensorshipRecord.Token - linkBy := time.Now().Add(time.Hour * 24 * 30).Unix() - makeProposalRFP(t, &prop, []string{}, linkBy) - d.AddRecord(t, convertPropToPD(t, prop)) - - // RFP submission - rfpSubmission := newProposalRecord(t, usr, id, www.PropStatusPublic) - makeProposalRFPSubmissions(t, []*www.ProposalRecord{&rfpSubmission}, token) - d.AddRecord(t, convertPropToPD(t, rfpSubmission)) - - sv := newStartVote(t, token, 1, minDuration, www2.VoteTypeStandard, id) - - // Invalid vote type - svInvalidType := newStartVote(t, token, 1, minDuration, - www2.VoteTypeRunoff, id) - svInvalidType.Vote.Type = www2.VoteTypeRunoff - - // RFP proposal linkBy less than min - propMinLb := newProposalRecord(t, usr, id, www.PropStatusPublic) - makeProposalRFP(t, &propMinLb, []string{}, p.linkByPeriodMin()) - d.AddRecord(t, convertPropToPD(t, propMinLb)) - - // RFP proposal linkBy more than max - propMaxLb := newProposalRecord(t, usr, id, www.PropStatusPublic) - maxLinkBy := time.Now().Unix() + (p.linkByPeriodMax() * 2) - makeProposalRFP(t, &propMaxLb, []string{}, maxLinkBy) - d.AddRecord(t, convertPropToPD(t, propMaxLb)) - - // Three days vote duration - propWrongSubPeriod := newProposalRecord(t, usr, id, www.PropStatusPublic) - makeProposalRFP(t, &propWrongSubPeriod, []string{}, p.linkByPeriodMin()-100) - d.AddRecord(t, convertPropToPD(t, propWrongSubPeriod)) - - var tests = []struct { - name string - sv www2.StartVote - u user.User - pr www.ProposalRecord - vs www.VoteSummary - want error - }{ - { - "invalid vote type", - svInvalidType, - *usr, - prop, - www.VoteSummary{}, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidVoteType, - }, - }, - { - "invalid vote status", - sv, - *usr, - prop, - www.VoteSummary{ - Status: www.PropVoteStatusNotAuthorized, - }, - www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - }, - }, - { - "proposal cannot be rfp submission", - sv, - *usr, - rfpSubmission, - www.VoteSummary{ - Status: www.PropVoteStatusAuthorized, - }, - www.UserError{ - ErrorCode: www.ErrorStatusWrongProposalType, - }, - }, - { - "less than min linkby period", - sv, - *usr, - propMinLb, - www.VoteSummary{ - Status: www.PropVoteStatusAuthorized, - }, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkBy, - }, - }, - { - "more than max linkby period", - sv, - *usr, - propMaxLb, - www.VoteSummary{ - Status: www.PropVoteStatusAuthorized, - }, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkBy, - }, - }, - { - "three days vote duration", - sv, - *usr, - propWrongSubPeriod, - www.VoteSummary{ - Status: www.PropVoteStatusAuthorized, - }, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkBy, - }, - }, - { - "valid", - sv, - *usr, - prop, - www.VoteSummary{ - Status: www.PropVoteStatusAuthorized, - }, - nil, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - err := p.validateStartVoteStandard(test.sv, test.u, test.pr, test.vs) - got := errToStr(err) - want := errToStr(test.want) - - if got != want { - t.Errorf("got %v, want %v", got, want) - } - }) - } -} - -func TestValidateStartVoteRunoff(t *testing.T) { - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - d := newTestPoliteiad(t, p) - defer d.Close() - - usr, id := newUser(t, p, true, false) - - prop := newProposalRecord(t, usr, id, www.PropStatusPublic) - token := prop.CensorshipRecord.Token - d.AddRecord(t, convertPropToPD(t, prop)) - - rfpSubmission := newProposalRecord(t, usr, id, www.PropStatusPublic) - makeProposalRFPSubmissions(t, []*www.ProposalRecord{&rfpSubmission}, token) - d.AddRecord(t, convertPropToPD(t, rfpSubmission)) - - sv := newStartVote(t, token, 1, minDuration, www2.VoteTypeRunoff, id) - - svInvalidType := newStartVote(t, token, 1, minDuration, - www2.VoteTypeStandard, id) - - var tests = []struct { - name string - sv www2.StartVote - u user.User - pr www.ProposalRecord - vs www.VoteSummary - want error - }{ - { - "invalid vote type", - svInvalidType, - *usr, - prop, - www.VoteSummary{}, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidVoteType, - }, - }, - { - "proposal is not a rfp submission", - sv, - *usr, - prop, - www.VoteSummary{}, - www.UserError{ - ErrorCode: www.ErrorStatusWrongProposalType, - }, - }, - { - "valid", - sv, - *usr, - rfpSubmission, - www.VoteSummary{ - Status: www.PropVoteStatusNotAuthorized, - }, - nil, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - err := validateStartVoteRunoff( - test.sv, - test.u, - test.pr, - test.vs, - minDuration, - maxDuration, - ) - got := errToStr(err) - want := errToStr(test.want) - - if got != want { - t.Errorf("got %v, want %v", got, want) - } - }) - } -} - -func TestFilterProposals(t *testing.T) { - // Test proposal page size. Only a single page of proposals - // should be returned. - t.Run("proposal page size", func(t *testing.T) { - pq := proposalsFilter{ - StateMap: map[www.PropStateT]bool{ - www.PropStateVetted: true, - }, - } - - // We use simplified userIDs, timestamps, and censorship - // record tokens. This is ok since filterProps() does not - // check the validity of the data. - c := www.ProposalListPageSize + 5 - propsPageTest := make([]www.ProposalRecord, 0, c) - for i := 1; i <= c; i++ { - propsPageTest = append(propsPageTest, www.ProposalRecord{ - State: www.PropStateVetted, - UserId: strconv.Itoa(i), - Timestamp: int64(i), - CensorshipRecord: www.CensorshipRecord{ - Token: strconv.Itoa(i), - }, - }) - } - - out := filterProps(pq, propsPageTest) - if len(out) != www.ProposalListPageSize { - t.Errorf("got %v, want %v", len(out), www.ProposalListPageSize) - } - }) - - // Create data for test table. We use simplified userIDs, - // timestamps, and censorship record tokens. This is ok since - // filterProps() does not check the validity of the data. - props := make(map[int]*www.ProposalRecord, 5) - for i := 1; i <= 5; i++ { - props[i] = &www.ProposalRecord{ - State: www.PropStateVetted, - UserId: strconv.Itoa(i), - Timestamp: int64(i), - CensorshipRecord: www.CensorshipRecord{ - Token: strconv.Itoa(i), - }, - } - } - - // Change the State of a few proposals so that they are not - // all the same. - props[2].State = www.PropStateUnvetted - props[4].State = www.PropStateUnvetted - - // Setup tests - var tests = []struct { - name string - req proposalsFilter - input []www.ProposalRecord - want []www.ProposalRecord - }{ - { - "filter by State", - proposalsFilter{ - StateMap: map[www.PropStateT]bool{ - www.PropStateUnvetted: true, - }, - }, - []www.ProposalRecord{ - *props[1], *props[2], *props[3], *props[4], *props[5], - }, - []www.ProposalRecord{ - *props[4], *props[2], - }, - }, - - { - "filter by UserID", - proposalsFilter{ - UserID: "1", - StateMap: map[www.PropStateT]bool{ - www.PropStateVetted: true, - }, - }, - []www.ProposalRecord{ - *props[1], *props[2], *props[3], *props[4], *props[5], - }, - []www.ProposalRecord{ - *props[1], - }, - }, - { - "filter by Before", - proposalsFilter{ - Before: props[3].CensorshipRecord.Token, - StateMap: map[www.PropStateT]bool{ - www.PropStateUnvetted: true, - www.PropStateVetted: true, - }, - }, - []www.ProposalRecord{ - *props[1], *props[2], *props[3], *props[4], *props[5], - }, - []www.ProposalRecord{ - *props[5], *props[4], - }, - }, - - { - "filter by After", - proposalsFilter{ - After: props[3].CensorshipRecord.Token, - StateMap: map[www.PropStateT]bool{ - www.PropStateUnvetted: true, - www.PropStateVetted: true, - }, - }, - []www.ProposalRecord{ - *props[1], *props[2], *props[3], *props[4], *props[5], - }, - []www.ProposalRecord{ - *props[2], *props[1], - }, - }, - - { - "unsorted proposals", - proposalsFilter{ - StateMap: map[www.PropStateT]bool{ - www.PropStateUnvetted: true, - www.PropStateVetted: true, - }, - }, - []www.ProposalRecord{ - *props[3], *props[4], *props[1], *props[5], *props[2], - }, - []www.ProposalRecord{ - *props[5], *props[4], *props[3], *props[2], *props[1], - }, - }, - } - - // Run tests - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - out := filterProps(test.req, test.input) - - // Pull out the tokens to make it easier to view the - // difference between want and got - want := make([]string, len(test.want)) - for _, p := range test.want { - want = append(want, p.CensorshipRecord.Token) - } - got := make([]string, len(out)) - for _, p := range out { - got = append(got, p.CensorshipRecord.Token) - } - - // Check if want and got are the same - if len(want) != len(got) { - goto fail - } - for i, w := range want { - if w != got[i] { - goto fail - } - } - - // success; want and got are the same - return - - fail: - t.Errorf("got %v, want %v", got, want) - }) - } -} - -func TestProcessNewProposal(t *testing.T) { - // TODO - /* - // Setup test environment - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - td := testpoliteiad.New(t) - defer td.Close() - - p.cfg.RPCHost = td.URL - p.cfg.Identity = td.PublicIdentity - - // Create a user that has not paid their registration fee. - usrUnpaid, _ := newUser(t, p, true, false) - - // Create a user that has paid their registration - // fee but does not have any proposal credits. - usrNoCredits, _ := newUser(t, p, true, false) - payRegistrationFee(t, p, usrNoCredits) - - // Create a user that has paid their registration - // fee and has purchased proposal credits. - usr, id := newUser(t, p, true, false) - payRegistrationFee(t, p, usr) - addProposalCredits(t, p, usr, 10) - - // Create a new proposal - f := newFileRandomMD(t) - np := createNewProposal(t, id, []www.File{f}, "") - - // Invalid proposal - propInvalid := createNewProposal(t, id, []www.File{f}, "") - propInvalid.Signature = "" - - // Expired deadline RFP proposal - rfpProp := newProposalRecord(t, usr, id, www.PropStatusPublic) - rfpToken := rfpProp.CensorshipRecord.Token - makeProposalRFP(t, &rfpProp, []string{}, time.Now().Unix()-p.linkByPeriodMin()) - td.AddRecord(t, convertPropToPD(t, rfpProp)) - - // Set vote summary for expired deadline proposal - no := decredplugin.VoteOptionIDReject - yes := decredplugin.VoteOptionIDApprove - approved := []www.VoteOptionResult{ - newVoteOptionResult(t, no, "not approve", 1, 2), - newVoteOptionResult(t, yes, "approve", 2, 8), - } - vsApproved := newVoteSummary(t, www.PropVoteStatusFinished, approved) - p.voteSummarySet(rfpToken, vsApproved) - - // Set expired deadline rfp metadata and signature - propMetadata, _ := newProposalMetadata(t, "valid name", rfpToken, 0) - propExpired := createNewProposal(t, id, []www.File{f}, "") - propExpired.Metadata = propMetadata - root := merkleRoot(t, propExpired.Files, propExpired.Metadata) - s := id.SignMessage([]byte(root)) - sig := hex.EncodeToString(s[:]) - propExpired.Signature = sig - - // Setup tests - var tests = []struct { - name string - np *www.NewProposal - usr *user.User - want error - }{ - { - "unpaid registration fee", - np, - usrUnpaid, - www.UserError{ - ErrorCode: www.ErrorStatusUserNotPaid, - }, - }, - { - "no proposal credits", - np, - usrNoCredits, - www.UserError{ - ErrorCode: www.ErrorStatusNoProposalCredits, - }, - }, - { - "invalid proposal", - propInvalid, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidSignature, - }, - }, - { - "linkby deadline expired", - propExpired, - usr, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkTo, - }, - }, - { - "success", - np, - usr, - nil, - }, - } - - // Run tests - for _, v := range tests { - t.Run(v.name, func(t *testing.T) { - npr, err := p.processNewProposal(*v.np, v.usr) - got := errToStr(err) - want := errToStr(v.want) - - if got != want { - t.Errorf("got error %v, want %v", - got, want) - } - - if v.want != nil { - // Test case passes - return - } - - // Validate success case - if npr == nil { - t.Errorf("NewProposalReply is nil") - } - - // Ensure a proposal credit has been deducted - // from the user's account. - u, err := p.db.UserGetById(v.usr.ID) - if err != nil { - t.Error(err) - } - - gotCredits := len(u.UnspentProposalCredits) - wantCredits := len(v.usr.UnspentProposalCredits) - if gotCredits != wantCredits { - t.Errorf("got num proposal credits %v, want %v", - gotCredits, wantCredits) - } - }) - } - */ -} - -func TestProcessEditProposal(t *testing.T) { - // TODO - /* - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - d := newTestPoliteiad(t, p) - defer d.Close() - - usr, id := newUser(t, p, true, false) - notAuthorUser, _ := newUser(t, p, true, false) - - // Public proposal to be edited - propPublic := newProposalRecord(t, usr, id, www.PropStatusPublic) - tokenPropPublic := propPublic.CensorshipRecord.Token - d.AddRecord(t, convertPropToPD(t, propPublic)) - - root := merkleRoot(t, propPublic.Files, propPublic.Metadata) - s := id.SignMessage([]byte(root)) - sigPropPublic := hex.EncodeToString(s[:]) - - // Edited public proposal - newMD := newFileRandomMD(t) - png := createFilePNG(t, false) - newFiles := []www.File{newMD, *png} - - root = merkleRoot(t, newFiles, propPublic.Metadata) - s = id.SignMessage([]byte(root)) - sigPropPublicEdited := hex.EncodeToString(s[:]) - - // Censored proposal to test error case - propCensored := newProposalRecord(t, usr, id, www.PropStatusCensored) - tokenPropCensored := propCensored.CensorshipRecord.Token - d.AddRecord(t, convertPropToPD(t, propCensored)) - - // Authorized vote proposal to test error case - propVoteAuthorized := newProposalRecord(t, usr, id, www.PropStatusPublic) - tokenVoteAuthorized := propVoteAuthorized.CensorshipRecord.Token - d.AddRecord(t, convertPropToPD(t, propVoteAuthorized)) - - cmd := newAuthorizeVoteCmd(t, tokenVoteAuthorized, - propVoteAuthorized.Version, decredplugin.AuthVoteActionAuthorize, id) - d.Plugin(t, cmd) - - var tests = []struct { - name string - user *user.User - editProp www.EditProposal - wantError error - }{ - { - "invalid proposal token", - usr, - www.EditProposal{ - Token: "invalid-token", - }, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - }, - }, - { - "wrong proposal status", - usr, - www.EditProposal{ - Token: tokenPropCensored, - }, - www.UserError{ - ErrorCode: www.ErrorStatusWrongStatus, - }, - }, - { - "user is not the author", - notAuthorUser, - www.EditProposal{ - Token: tokenPropPublic, - }, - www.UserError{ - ErrorCode: www.ErrorStatusUserNotAuthor, - }, - }, - { - "wrong proposal vote status", - usr, - www.EditProposal{ - Token: tokenVoteAuthorized, - }, - www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - }, - }, - { - "no changes in proposal md file", - usr, - www.EditProposal{ - Token: tokenPropPublic, - Files: propPublic.Files, - Metadata: propPublic.Metadata, - PublicKey: usr.PublicKey(), - Signature: sigPropPublic, - }, - www.UserError{ - ErrorCode: www.ErrorStatusNoProposalChanges, - }, - }, - { - "success", - usr, - www.EditProposal{ - Token: tokenPropPublic, - Files: newFiles, - Metadata: propPublic.Metadata, - PublicKey: usr.PublicKey(), - Signature: sigPropPublicEdited, - }, - nil, - }, - } - - for _, v := range tests { - t.Run(v.name, func(t *testing.T) { - _, err := p.processEditProposal(v.editProp, v.user) - got := errToStr(err) - want := errToStr(v.wantError) - - // Test if we got expected error - if got != want { - t.Errorf("got error %v, want %v", got, want) - } - }) - } - */ -} - -func TestProcessSetProposalStatus(t *testing.T) { - // TODO - /* - // Setup test environment - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - d := newTestPoliteiad(t, p) - defer d.Close() - - // Create test data - admin, id := newUser(t, p, true, true) - - changeMsgCensored := "proposal did not meet guidelines" - changeMsgAbandoned := "no activity" - - statusPublic := strconv.Itoa(int(www.PropStatusPublic)) - statusCensored := strconv.Itoa(int(www.PropStatusCensored)) - statusAbandoned := strconv.Itoa(int(www.PropStatusAbandoned)) - - propNotReviewed := newProposalRecord(t, admin, id, www.PropStatusNotReviewed) - propPublic := newProposalRecord(t, admin, id, www.PropStatusPublic) - - tokenNotReviewed := propNotReviewed.CensorshipRecord.Token - tokenPublic := propPublic.CensorshipRecord.Token - tokenNotFound := "abc" - - msg := fmt.Sprintf("%s%s", tokenNotReviewed, statusPublic) - s := id.SignMessage([]byte(msg)) - sigNotReviewedToPublic := hex.EncodeToString(s[:]) - - msg = fmt.Sprintf("%s%s%s", tokenNotReviewed, - statusCensored, changeMsgCensored) - s = id.SignMessage([]byte(msg)) - sigNotReviewedToCensored := hex.EncodeToString(s[:]) - - msg = fmt.Sprintf("%s%s%s", tokenPublic, - statusAbandoned, changeMsgAbandoned) - s = id.SignMessage([]byte(msg)) - sigPublicToAbandoned := hex.EncodeToString(s[:]) - - msg = fmt.Sprintf("%s%s", tokenNotFound, statusPublic) - s = id.SignMessage([]byte(msg)) - sigNotFound := hex.EncodeToString(s[:]) - - msg = fmt.Sprintf("%s%s%s", tokenNotReviewed, - statusAbandoned, changeMsgAbandoned) - s = id.SignMessage([]byte(msg)) - sigNotReviewedToAbandoned := hex.EncodeToString(s[:]) - - // Add success case proposals to politeiad - d.AddRecord(t, convertPropToPD(t, propNotReviewed)) - d.AddRecord(t, convertPropToPD(t, propPublic)) - - // Create a proposal whose vote has been authorized - propVoteAuthorized := newProposalRecord(t, admin, id, www.PropStatusPublic) - tokenVoteAuthorized := propVoteAuthorized.CensorshipRecord.Token - - msg = fmt.Sprintf("%s%s%s", tokenVoteAuthorized, - statusAbandoned, changeMsgAbandoned) - s = id.SignMessage([]byte(msg)) - sigVoteAuthorizedToAbandoned := hex.EncodeToString(s[:]) - - d.AddRecord(t, convertPropToPD(t, propVoteAuthorized)) - cmd := newAuthorizeVoteCmd(t, tokenVoteAuthorized, - propVoteAuthorized.Version, decredplugin.AuthVoteActionAuthorize, id) - d.Plugin(t, cmd) - - // Create a proposal whose voting period has started - propVoteStarted := newProposalRecord(t, admin, id, www.PropStatusPublic) - tokenVoteStarted := propVoteStarted.CensorshipRecord.Token - - msg = fmt.Sprintf("%s%s%s", tokenVoteStarted, - statusAbandoned, changeMsgAbandoned) - s = id.SignMessage([]byte(msg)) - sigVoteStartedToAbandoned := hex.EncodeToString(s[:]) - - d.AddRecord(t, convertPropToPD(t, propVoteStarted)) - cmd = newStartVoteCmd(t, tokenVoteStarted, 1, 2016, id) - d.Plugin(t, cmd) - - // Ensure that admins are not allowed to change the status of - // their own proposal on mainnet. This is run individually - // because it requires flipping the testnet config setting. - t.Run("admin is author", func(t *testing.T) { - p.cfg.TestNet = false - defer func() { - p.cfg.TestNet = true - }() - - sps := www.SetProposalStatus{ - Token: tokenNotReviewed, - ProposalStatus: www.PropStatusPublic, - Signature: sigNotReviewedToPublic, - PublicKey: admin.PublicKey(), - } - - _, err := p.processSetProposalStatus(sps, admin) - got := errToStr(err) - want := www.ErrorStatus[www.ErrorStatusReviewerAdminEqualsAuthor] - if got != want { - t.Errorf("got error %v, want %v", - got, want) - } - }) - - // Setup tests - var tests = []struct { - name string - usr *user.User - sps www.SetProposalStatus - want error - }{ - // This is an admin route so it can be assumed that the - // user has been validated and is an admin. - - { - "no change message for censored", - admin, - www.SetProposalStatus{ - Token: tokenNotReviewed, - ProposalStatus: www.PropStatusCensored, - Signature: sigNotReviewedToCensored, - PublicKey: admin.PublicKey(), - }, - www.UserError{ - ErrorCode: www.ErrorStatusChangeMessageCannotBeBlank, - }}, - - { - "no change message for abandoned", - admin, - www.SetProposalStatus{ - Token: tokenPublic, - ProposalStatus: www.PropStatusAbandoned, - Signature: sigPublicToAbandoned, - PublicKey: admin.PublicKey(), - }, - www.UserError{ - ErrorCode: www.ErrorStatusChangeMessageCannotBeBlank, - }}, - - { - "invalid public key", - admin, - www.SetProposalStatus{ - Token: tokenNotReviewed, - ProposalStatus: www.PropStatusPublic, - Signature: sigNotReviewedToPublic, - PublicKey: "", - }, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidSigningKey, - }}, - - { - "invalid signature", - admin, - www.SetProposalStatus{ - Token: tokenNotReviewed, - ProposalStatus: www.PropStatusPublic, - Signature: "", - PublicKey: admin.PublicKey(), - }, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidSignature, - }}, - - { - "invalid proposal token", - admin, - www.SetProposalStatus{ - Token: tokenNotFound, - ProposalStatus: www.PropStatusPublic, - Signature: sigNotFound, - PublicKey: admin.PublicKey(), - }, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - }}, - - { - "invalid status change", - admin, - www.SetProposalStatus{ - Token: tokenNotReviewed, - ProposalStatus: www.PropStatusAbandoned, - StatusChangeMessage: changeMsgAbandoned, - Signature: sigNotReviewedToAbandoned, - PublicKey: admin.PublicKey(), - }, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidPropStatusTransition, - }}, - { - "unvetted success", - admin, - www.SetProposalStatus{ - Token: tokenNotReviewed, - ProposalStatus: www.PropStatusPublic, - Signature: sigNotReviewedToPublic, - PublicKey: admin.PublicKey(), - }, nil}, - - { - "vote already authorized", - admin, - www.SetProposalStatus{ - Token: tokenVoteAuthorized, - ProposalStatus: www.PropStatusAbandoned, - StatusChangeMessage: changeMsgAbandoned, - Signature: sigVoteAuthorizedToAbandoned, - PublicKey: admin.PublicKey(), - }, - www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - }}, - - { - "vote already started", - admin, - www.SetProposalStatus{ - Token: tokenVoteStarted, - ProposalStatus: www.PropStatusAbandoned, - StatusChangeMessage: changeMsgAbandoned, - Signature: sigVoteStartedToAbandoned, - PublicKey: admin.PublicKey(), - }, - www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - }}, - - { - "vetted success", - admin, - www.SetProposalStatus{ - Token: tokenPublic, - ProposalStatus: www.PropStatusAbandoned, - StatusChangeMessage: changeMsgAbandoned, - Signature: sigPublicToAbandoned, - PublicKey: admin.PublicKey(), - }, nil}, - } - - // Run tests - for _, v := range tests { - t.Run(v.name, func(t *testing.T) { - reply, err := p.processSetProposalStatus(v.sps, v.usr) - got := errToStr(err) - want := errToStr(v.want) - if got != want { - t.Errorf("got error %v, want %v", - got, want) - } - - if err != nil { - // Test case passes - return - } - - // Validate updated proposal - if reply.Proposal.Status != v.sps.ProposalStatus { - t.Errorf("got proposal status %v, want %v", - reply.Proposal.Status, v.sps.ProposalStatus) - } - }) - } - */ -} - -func TestProcessAllVetted(t *testing.T) { - // Setup test environment - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - d := newTestPoliteiad(t, p) - defer d.Close() - - // Create test data - tokenValid := "3575a65bbc3616c939acf6edf801e1168485dc864efef910034268f695351b5d" - tokenNotHex := "3575a65bbc3616c939acf6edf801e1168485dc864efef910034268f695351zzz" - tokenShort := "3575a65bbc3616c939acf6edf801e1168485dc864efef910034268f695351b5" - tokenLong := "3575a65bbc3616c939acf6edf801e1168485dc864efef910034268f695351b5dd" - - // Setup tests - var tests = []struct { - name string - av www.GetAllVetted - want error - }{ - { - "before token not hex", - www.GetAllVetted{ - Before: tokenNotHex, - }, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - }, - }, - { - "before token invalid length short", - www.GetAllVetted{ - Before: tokenShort, - }, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - }, - }, - { - "before token invalid length long", - www.GetAllVetted{ - Before: tokenLong, - }, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - }, - }, - { - "after token not hex", - www.GetAllVetted{ - After: tokenNotHex, - }, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - }, - }, - { - "after token invalid length short", - www.GetAllVetted{ - After: tokenShort, - }, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - }, - }, - { - "after token invalid length long", - www.GetAllVetted{ - After: tokenLong, - }, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - }, - }, - { - "valid before token", - www.GetAllVetted{ - Before: tokenValid, - }, - nil, - }, - { - "valid after token", - www.GetAllVetted{ - After: tokenValid, - }, - nil, - }, - - // XXX only partial test coverage has been added to this route - } - - // Run tests - for _, v := range tests { - t.Run(v.name, func(t *testing.T) { - _, err := p.processAllVetted(v.av) - got := errToStr(err) - want := errToStr(v.want) - if got != want { - t.Errorf("got error %v, want %v", - got, want) - } - }) - } -} - -func TestProcessAuthorizeVote(t *testing.T) { - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - d := newTestPoliteiad(t, p) - defer d.Close() - - usr, id := newUser(t, p, true, false) - - authorize := decredplugin.AuthVoteActionAuthorize - - // Public proposal - prop := newProposalRecord(t, usr, id, www.PropStatusPublic) - token := prop.CensorshipRecord.Token - d.AddRecord(t, convertPropToPD(t, prop)) - - // Proposal not found - propNF := newProposalRecord(t, usr, id, www.PropStatusPublic) - tokenNF := propNF.CensorshipRecord.Token - avNF := newAuthorizeVote(t, tokenNF, propNF.Version, authorize, id) - - // Authorize vote - av := newAuthorizeVote(t, token, prop.Version, authorize, id) - - // Want in reply - s := d.FullIdentity.SignMessage([]byte(av.Signature)) - wantReceipt := hex.EncodeToString(s[:]) - - var tests = []struct { - name string - user *user.User - av www.AuthorizeVote - wantReply *www.AuthorizeVoteReply - wantErr error - }{ - { - "proposal not found", - usr, - avNF, - &www.AuthorizeVoteReply{}, - www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - }, - }, - { - "success", - usr, - av, - &www.AuthorizeVoteReply{ - Action: authorize, - Receipt: wantReceipt, - }, - nil, - }, - } - - ctx := context.Background() - for _, v := range tests { - t.Run(v.name, func(t *testing.T) { - reply, err := p.processAuthorizeVote(ctx, v.av, v.user) - got := errToStr(err) - want := errToStr(v.wantErr) - - // Test if we got expected error - if got != want { - t.Errorf("got error %v, want %v", got, want) - } - - // Test received reply - if err == nil { - if !reflect.DeepEqual(v.wantReply, reply) { - t.Errorf("got reply %v, want %v", reply, v.wantReply) - } - } - }) - } -} - -func TestProcessStartVoteV2(t *testing.T) { - // TODO - /* - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - d := newTestPoliteiad(t, p) - defer d.Close() - - usr, id := newUser(t, p, true, true) - usrNotAdmin, _ := newUser(t, p, false, false) - - prop := newProposalRecord(t, usr, id, www.PropStatusPublic) - token := prop.CensorshipRecord.Token - d.AddRecord(t, convertPropToPD(t, prop)) - - sv := newStartVote(t, token, 1, minDuration, www2.VoteTypeStandard, id) - vs := newVoteSummary(t, www.PropVoteStatusAuthorized, []www.VoteOptionResult{}) - p.voteSummarySet(token, vs) - - randomToken, err := util.Random(pd.TokenSize) - if err != nil { - t.Fatal(err) - } - rToken := hex.EncodeToString(randomToken) - - svInvalidToken := newStartVote(t, token, 1, minDuration, - www2.VoteTypeStandard, id) - svInvalidToken.Vote.Token = "" - - svRandomToken := newStartVote(t, token, 1, minDuration, - www2.VoteTypeStandard, id) - svRandomToken.Vote.Token = rToken - - var tests = []struct { - name string - user *user.User - sv www2.StartVote - wantErr error - }{ - { - "user not admin", - usrNotAdmin, - sv, - fmt.Errorf("user is not an admin"), - }, - { - "invalid token", - usr, - svInvalidToken, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - }, - }, - { - "proposal not found", - usr, - svRandomToken, - www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - }, - }, - { - "success", - usr, - sv, - nil, - }, - } - - for _, v := range tests { - t.Run(v.name, func(t *testing.T) { - _, err := p.processStartVoteV2(v.sv, v.user) - got := errToStr(err) - want := errToStr(v.wantErr) - if got != want { - t.Errorf("got %v, want %v", got, want) - } - }) - } - */ -} - -func TestProcessStartVoteRunoffV2(t *testing.T) { - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - d := newTestPoliteiad(t, p) - defer d.Close() - - // Helper variables - runoff := www2.VoteTypeRunoff - public := www.PropStatusPublic - auth := decredplugin.AuthVoteActionAuthorize - usr, id := newUser(t, p, true, true) - - // Prepare proposal data for testing - rfpProposal := newProposalRecord(t, usr, id, public) - token := rfpProposal.CensorshipRecord.Token - - rfpProposalSubmission1 := newProposalRecord(t, usr, id, public) - sub1Token := rfpProposalSubmission1.CensorshipRecord.Token - rfpProposalSubmission2 := newProposalRecord(t, usr, id, public) - sub2Token := rfpProposalSubmission2.CensorshipRecord.Token - rfpProposalSubmission3 := newProposalRecord(t, usr, id, public) - sub3Token := rfpProposalSubmission3.CensorshipRecord.Token - - extraProposalSubmission := newProposalRecord(t, usr, id, public) - extraToken := extraProposalSubmission.CensorshipRecord.Token - - linkTo := token - linkBy := time.Now().Add(time.Hour * 24 * 30).Unix() - - rfpSubmissions := []*www.ProposalRecord{ - &rfpProposalSubmission1, - &rfpProposalSubmission2, - &rfpProposalSubmission3, - } - - makeProposalRFP(t, &rfpProposal, []string{}, linkBy) - makeProposalRFPSubmissions(t, rfpSubmissions, linkTo) - makeProposalRFPSubmissions(t, []*www.ProposalRecord{&extraProposalSubmission}, - extraToken) - - badRFPProposal := newProposalRecord(t, usr, id, public) - - d.AddRecord(t, convertPropToPD(t, rfpProposal)) - d.AddRecord(t, convertPropToPD(t, rfpProposalSubmission1)) - d.AddRecord(t, convertPropToPD(t, rfpProposalSubmission2)) - d.AddRecord(t, convertPropToPD(t, rfpProposalSubmission3)) - d.AddRecord(t, convertPropToPD(t, extraProposalSubmission)) - d.AddRecord(t, convertPropToPD(t, badRFPProposal)) - - // Prepare data for the route payload - sub1AuthVote := newAuthorizeVoteV2(t, sub1Token, "1", auth, id) - sub2AuthVote := newAuthorizeVoteV2(t, sub2Token, "1", auth, id) - sub3AuthVote := newAuthorizeVoteV2(t, sub3Token, "1", auth, id) - - sub1StartVote := newStartVote(t, sub1Token, 1, minDuration, runoff, id) - sub2StartVote := newStartVote(t, sub2Token, 1, minDuration, runoff, id) - sub3StartVote := newStartVote(t, sub3Token, 1, minDuration, runoff, id) - - // Valid start vote runoff - authVotes := []www2.AuthorizeVote{ - sub1AuthVote, - sub2AuthVote, - sub3AuthVote, - } - startVotes := []www2.StartVote{ - sub1StartVote, - sub2StartVote, - sub3StartVote, - } - svRunoff := newStartVoteRunoff(t, token, authVotes, startVotes) - - // Start vote not matching authorize - avNotMatch := []www2.AuthorizeVote{ - sub2AuthVote, - sub3AuthVote, - } - svNotMatch := []www2.StartVote{ - sub1StartVote, - sub2StartVote, - sub3StartVote, - } - svRunoffNotMatch := newStartVoteRunoff(t, token, avNotMatch, svNotMatch) - - // Start vote not matching authorize - avNotMatch2 := []www2.AuthorizeVote{ - sub1AuthVote, - sub2AuthVote, - sub3AuthVote, - } - svNotMatch2 := []www2.StartVote{ - sub2StartVote, - sub3StartVote, - } - svRunoffNotMatch2 := newStartVoteRunoff(t, token, avNotMatch2, svNotMatch2) - - // Empty auth and start votes - avEmpty := []www2.AuthorizeVote{} - svEmpty := []www2.StartVote{} - svRunoffEmpty := newStartVoteRunoff(t, token, avEmpty, svEmpty) - - // Invalid token in start votes - sub1AvInvalid := sub1AuthVote - sub1AvInvalid.Token = "invalid" - sub1SvInvalid := sub1StartVote - sub1SvInvalid.Vote.Token = "invalid" - avInvalidToken := []www2.AuthorizeVote{ - sub1AvInvalid, - } - svInvalidToken := []www2.StartVote{ - sub1SvInvalid, - } - svRunoffInvalidToken := newStartVoteRunoff(t, token, avInvalidToken, - svInvalidToken) - - // Proposal submission record not found - randomToken, err := util.Random(pd.TokenSizeMax) - if err != nil { - t.Fatal(err) - } - rToken := hex.EncodeToString(randomToken) - sub1AvNotFound := sub1AuthVote - sub1AvNotFound.Token = rToken - sub1SvNotFound := sub1StartVote - sub1SvNotFound.Vote.Token = rToken - avNotFound := []www2.AuthorizeVote{ - sub1AvNotFound, - } - svNotFound := []www2.StartVote{ - sub1SvNotFound, - } - svRunoffNotFound := newStartVoteRunoff(t, token, avNotFound, svNotFound) - - // RFP Proposal record not found - svRunoffRFPNotFound := newStartVoteRunoff(t, rToken, authVotes, startVotes) - - // Empty linked from for RFP proposal - badToken := badRFPProposal.CensorshipRecord.Token - svRunoffBadRFP := newStartVoteRunoff(t, badToken, authVotes, startVotes) - - // Extra StartVote on the route payload - ave := newAuthorizeVoteV2(t, extraToken, "1", auth, id) - sve := newStartVote(t, extraToken, 1, minDuration, runoff, id) - avExtra := authVotes - avExtra = append(avExtra, ave) - svExtra := startVotes - svExtra = append(svExtra, sve) - svRunoffExtraSv := newStartVoteRunoff(t, token, avExtra, svExtra) - - // Missing StartVote from one of the rfp submissions - avMissing := []www2.AuthorizeVote{ - sub1AuthVote, - sub2AuthVote, - } - svMissing := []www2.StartVote{ - sub1StartVote, - sub2StartVote, - } - svRunoffMissingSv := newStartVoteRunoff(t, token, avMissing, svMissing) - - var tests = []struct { - name string - user *user.User - sv www2.StartVoteRunoff - want error - }{ - { - "start vote not matching authorize vote", - usr, - svRunoffNotMatch, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidRunoffVote, - }, - }, - { - "authorize vote not matching start vote", - usr, - svRunoffNotMatch2, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidRunoffVote, - }, - }, - { - "empty authorize and start vote entries", - usr, - svRunoffEmpty, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidRunoffVote, - }, - }, - { - "invalid proposal token in start vote", - usr, - svRunoffInvalidToken, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - }, - }, - { - "proposal submission record not found", - usr, - svRunoffNotFound, - www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - }, - }, - { - "RFP proposal record not found", - usr, - svRunoffRFPNotFound, - www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - }, - }, - { - "no linked proposals to the RFP", - usr, - svRunoffBadRFP, - www.UserError{ - ErrorCode: www.ErrorStatusNoLinkedProposals, - }, - }, - { - "extra StartVote on the route payload", - usr, - svRunoffExtraSv, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidRunoffVote, - }, - }, - { - "missing StartVote for one of the rfp submissions", - usr, - svRunoffMissingSv, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidRunoffVote, - }, - }, - { - "valid start runoff vote", - usr, - svRunoff, - nil, - }, - } - - ctx := context.Background() - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - _, err := p.processStartVoteRunoffV2(ctx, test.sv, test.user) - got := errToStr(err) - want := errToStr(test.want) - if got != want { - t.Errorf("got %v, want %v", got, want) - } - }) - } -} - -func TestVerifyStatusChange(t *testing.T) { - // TODO - /* - invalid := www.PropStatusInvalid - notFound := www.PropStatusNotFound - notReviewed := www.PropStatusNotReviewed - censored := www.PropStatusCensored - public := www.PropStatusPublic - unreviewedChanges := www.PropStatusUnreviewedChanges - abandoned := www.PropStatusAbandoned - - // Setup tests - var tests = []struct { - name string - current www.PropStatusT - next www.PropStatusT - wantError bool - }{ - {"not reviewed to invalid", notReviewed, invalid, true}, - {"not reviewed to not found", notReviewed, notFound, true}, - {"not reviewed to censored", notReviewed, censored, false}, - {"not reviewed to public", notReviewed, public, false}, - {"not reviewed to unreviewed changes", notReviewed, unreviewedChanges, - true}, - {"not reviewed to abandoned", notReviewed, abandoned, true}, - {"censored to invalid", censored, invalid, true}, - {"censored to not found", censored, notFound, true}, - {"censored to not reviewed", censored, notReviewed, true}, - {"censored to public", censored, public, true}, - {"censored to unreviewed changes", censored, unreviewedChanges, true}, - {"censored to abandoned", censored, abandoned, true}, - {"public to invalid", public, invalid, true}, - {"public to not found", public, notFound, true}, - {"public to not reviewed", public, notReviewed, true}, - {"public to censored", public, censored, true}, - {"public to unreviewed changes", public, unreviewedChanges, true}, - {"public to abandoned", public, abandoned, false}, - {"unreviewed changes to invalid", unreviewedChanges, invalid, true}, - {"unreviewed changes to not found", unreviewedChanges, notFound, true}, - {"unreviewed changes to not reviewed", unreviewedChanges, notReviewed, - true}, - {"unreviewed changes to censored", unreviewedChanges, censored, false}, - {"unreviewed changes to public", unreviewedChanges, public, false}, - {"unreviewed changes to abandoned", unreviewedChanges, abandoned, true}, - {"abandoned to invalid", abandoned, invalid, true}, - {"abandoned to not found", abandoned, notFound, true}, - {"abandoned to not reviewed", abandoned, notReviewed, true}, - {"abandoned to censored", abandoned, censored, true}, - {"abandoned to public", abandoned, public, true}, - {"abandoned to unreviewed changes", abandoned, unreviewedChanges, true}, - } - - // Run tests - for _, v := range tests { - t.Run(v.name, func(t *testing.T) { - err := verifyStatusChange(v.current, v.next) - got := errToStr(err) - if v.wantError { - want := www.ErrorStatus[www.ErrorStatusInvalidPropStatusTransition] - if got != want { - t.Errorf("got error %v, want %v", - got, want) - } - - // Test case passes - return - } - - if err != nil { - t.Errorf("got error %v, want nil", - got) - } - }) - } - */ -} diff --git a/politeiawww/testing.go b/politeiawww/testing.go index af1ca0ccd..43af52146 100644 --- a/politeiawww/testing.go +++ b/politeiawww/testing.go @@ -6,30 +6,25 @@ package main import ( "bytes" - "crypto/sha256" "encoding/base64" "encoding/hex" - "encoding/json" "errors" "image" "image/color" "image/png" "io/ioutil" "math/rand" - "net/http" "os" "path/filepath" "testing" "time" "github.com/decred/dcrd/chaincfg/v3" - "github.com/decred/dcrtime/merkle" - pd "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/api/v1/mime" "github.com/decred/politeia/politeiad/testpoliteiad" + pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" - www2 "github.com/decred/politeia/politeiawww/api/www/v2" "github.com/decred/politeia/politeiawww/user" "github.com/decred/politeia/politeiawww/user/localdb" "github.com/decred/politeia/util" @@ -53,159 +48,19 @@ func errToStr(e error) string { return e.Error() } -func convertPropToPD(t *testing.T, p www.ProposalRecord) pd.Record { - t.Helper() - - // TODO - /* - // Attach ProposalMetadata as a politeiad file - files := convertPropFilesFromWWW(p.Files) - for _, v := range p.Metadata { - switch v.Hint { - case www.HintProposalMetadata: - files = append(files, convertFileFromMetadata(v)) - } - } - - // Create a ProposalGeneralV2 mdstream - md, err := mdstream.EncodeProposalGeneralV2( - mdstream.ProposalGeneralV2{ - Version: mdstream.VersionProposalGeneral, - Timestamp: time.Now().Unix(), - PublicKey: p.PublicKey, - Signature: p.Signature, - }) - if err != nil { - t.Fatal(err) - } - - mdStreams := []pd.MetadataStream{{ - ID: mdstream.IDProposalGeneral, - Payload: string(md), - }} - - return pd.Record{ - Status: convertPropStatusFromWWW(p.Status), - Timestamp: p.Timestamp, - Version: p.Version, - Metadata: mdStreams, - CensorshipRecord: convertPropCensorFromWWW(p.CensorshipRecord), - Files: files, - } - */ - return pd.Record{} -} - -func payRegistrationFee(t *testing.T, p *politeiawww, u *user.User) { - t.Helper() - - u.NewUserPaywallAmount = 0 - u.NewUserPaywallTx = "cleared_during_testing" - u.NewUserPaywallPollExpiry = 0 - - err := p.db.UserUpdate(*u) - if err != nil { - t.Fatal(err) - } -} - -func addProposalCredits(t *testing.T, p *politeiawww, u *user.User, quantity int) { - t.Helper() - - c := make([]user.ProposalCredit, quantity) - ts := time.Now().Unix() - for i := 0; i < quantity; i++ { - c[i] = user.ProposalCredit{ - PaywallID: 0, - Price: 0, - DatePurchased: ts, - TxID: "created_during_testing", - } - } - u.UnspentProposalCredits = append(u.UnspentProposalCredits, c...) - - err := p.db.UserUpdate(*u) - if err != nil { - t.Fatal(err) - } -} - -func proposalNameRandom(t *testing.T) string { - r, err := util.Random(www.PolicyMinProposalNameLength) - if err != nil { - t.Fatal(err) - } - return hex.EncodeToString(r) -} - -// merkleRoot returns a hex encoded merkle root of the passed in files and -// metadata. -func merkleRoot(t *testing.T, files []www.File, metadata []www.Metadata) string { - t.Helper() - - digests := make([]*[sha256.Size]byte, 0, len(files)) - for _, f := range files { - // Compute file digest - b, err := base64.StdEncoding.DecodeString(f.Payload) - if err != nil { - t.Fatalf("decode payload for file %v: %v", - f.Name, err) - } - digest := util.Digest(b) - - // Compare against digest that came with the file - d, ok := util.ConvertDigest(f.Digest) - if !ok { - t.Fatalf("invalid digest: file:%v digest:%v", - f.Name, f.Digest) - } - if !bytes.Equal(digest, d[:]) { - t.Fatalf("digests do not match for file %v", - f.Name) - } - - // Digest is valid - digests = append(digests, &d) - } - - for _, v := range metadata { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - t.Fatalf("decode payload for metadata %v: %v", - v.Hint, err) - } - digest := util.Digest(b) - d, ok := util.ConvertDigest(v.Digest) - if !ok { - t.Fatalf("invalid digest: metadata:%v digest:%v", - v.Hint, v.Digest) - } - if !bytes.Equal(digest, d[:]) { - t.Fatalf("digests do not match for metadata %v", - v.Hint) - } - - // Digest is valid - digests = append(digests, &d) - } - - // Compute merkle root - return hex.EncodeToString(merkle.Root(digests)[:]) -} - -// createFilePNG creates a File that contains a png image. The png image is -// blank by default but can be filled in with random rgb colors by setting the -// addColor parameter to true. The png without color will be ~3kB. The png -// with color will be ~2MB. -func createFilePNG(t *testing.T, addColor bool) *www.File { +// newFilePNG creates a File that contains a png image. The png image is blank +// by default but can be filled in with random rgb colors by setting the +// addColor parameter to true. The png without color will be ~3kB. The png with +// color will be ~2MB. +func newFilePNG(t *testing.T, addColor bool) *pi.File { t.Helper() b := new(bytes.Buffer) img := image.NewRGBA(image.Rect(0, 0, 1000, 500)) - // Fill in the pixels with random rgb colors in order to - // increase the size of the image. This is used to create an - // image that exceeds the maximum image size policy. + // Fill in the pixels with random rgb colors in order to increase + // the size of the image. This is used to create an image that + // exceeds the maximum image size policy. if addColor { r := rand.New(rand.NewSource(255)) for y := 0; y < img.Bounds().Max.Y-1; y++ { @@ -228,7 +83,7 @@ func createFilePNG(t *testing.T, addColor bool) *www.File { t.Fatalf("%v", err) } - return &www.File{ + return &pi.File{ Name: hex.EncodeToString(r) + ".png", MIME: mime.DetectMimeType(b.Bytes()), Digest: hex.EncodeToString(util.Digest(b.Bytes())), @@ -236,46 +91,6 @@ func createFilePNG(t *testing.T, addColor bool) *www.File { } } -// createFileMD creates a File that contains a markdown file. The markdown -// file is filled with randomly generated data. -func createFileMD(t *testing.T, size int) *www.File { - t.Helper() - - var b bytes.Buffer - r, err := util.Random(size) - if err != nil { - t.Fatalf("%v", err) - } - b.WriteString(base64.StdEncoding.EncodeToString(r) + "\n") - - return &www.File{ - Name: www.PolicyIndexFilename, - MIME: http.DetectContentType(b.Bytes()), - Digest: hex.EncodeToString(util.Digest(b.Bytes())), - Payload: base64.StdEncoding.EncodeToString(b.Bytes()), - } -} - -// createNewProposal computes the merkle root of the given files, signs the -// merkle root with the given identity then returns a NewProposal object. -func createNewProposal(t *testing.T, id *identity.FullIdentity, files []www.File, title string) *www.NewProposal { - t.Helper() - - // Setup metadata - metadata, _ := newProposalMetadata(t, title, "", 0) - - // Compute and sign merkle root - m := merkleRoot(t, files, metadata) - sig := id.SignMessage([]byte(m)) - - return &www.NewProposal{ - Files: files, - Metadata: metadata, - PublicKey: hex.EncodeToString(id.Public.Key[:]), - Signature: hex.EncodeToString(sig[:]), - } -} - // newUser creates a new user using randomly generated user credentials and // inserts the user into the database. The user details and the full user // identity are returned. @@ -360,294 +175,38 @@ func newUser(t *testing.T, p *politeiawww, isVerified, isAdmin bool) (*user.User return usr, fid } -// newFileRandomMD returns a File with the name index.md that contains random -// base64 text. -func newFileRandomMD(t *testing.T) www.File { +func userPaywallClear(t *testing.T, p *politeiawww, u *user.User) { t.Helper() - var b bytes.Buffer - // Add ten lines of random base64 text. - for i := 0; i < 10; i++ { - r, err := util.Random(32) - if err != nil { - t.Fatal(err) - } - b.WriteString(base64.StdEncoding.EncodeToString(r) + "\n") - } - - return www.File{ - Name: "index.md", - MIME: mime.DetectMimeType(b.Bytes()), - Digest: hex.EncodeToString(util.Digest(b.Bytes())), - Payload: base64.StdEncoding.EncodeToString(b.Bytes()), - } -} - -func newStartVote(t *testing.T, token string, v uint32, d uint32, vt www2.VoteT, id *identity.FullIdentity) www2.StartVote { - t.Helper() + u.NewUserPaywallAmount = 0 + u.NewUserPaywallTx = "cleared_during_testing" + u.NewUserPaywallPollExpiry = 0 - vote := www2.Vote{ - Token: token, - ProposalVersion: v, - Type: vt, - Mask: 0x03, // bit 0 no, bit 1 yes - Duration: d, - QuorumPercentage: 20, - PassPercentage: 60, - Options: []www2.VoteOption{ - { - Id: "no", - Description: "Don't approve proposal", - Bits: 0x01, - }, - { - Id: "yes", - Description: "Approve proposal", - Bits: 0x02, - }, - }, - } - vb, err := json.Marshal(vote) + err := p.db.UserUpdate(*u) if err != nil { - t.Fatalf("marshal vote failed: %v %v", err, vote) - } - msg := hex.EncodeToString(util.Digest(vb)) - sig := id.SignMessage([]byte(msg)) - return www2.StartVote{ - Vote: vote, - PublicKey: hex.EncodeToString(id.Public.Key[:]), - Signature: hex.EncodeToString(sig[:]), - } -} - -func newStartVoteCmd(t *testing.T, token string, proposalVersion uint32, d uint32, id *identity.FullIdentity) pd.PluginCommand { - t.Helper() - - /* - sv := newStartVote(t, token, proposalVersion, d, www2.VoteTypeStandard, id) - dsv := convertStartVoteV2ToDecred(sv) - payload, err := decredplugin.EncodeStartVoteV2(dsv) - if err != nil { - t.Fatal(err) - } - - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - t.Fatal(err) - } - - return pd.PluginCommand{ - Challenge: hex.EncodeToString(challenge), - ID: decredplugin.ID, - Command: decredplugin.CmdStartVote, - CommandID: decredplugin.CmdStartVote + " " + sv.Vote.Token, - Payload: string(payload), - } - */ - return pd.PluginCommand{} -} - -func newStartVoteRunoff(t *testing.T, tk string, avs []www2.AuthorizeVote, svs []www2.StartVote) www2.StartVoteRunoff { - t.Helper() - - return www2.StartVoteRunoff{ - Token: tk, - AuthorizeVotes: avs, - StartVotes: svs, - } -} - -func newAuthorizeVoteV2(t *testing.T, token, version, action string, id *identity.FullIdentity) www2.AuthorizeVote { - t.Helper() - - sig := id.SignMessage([]byte(token + version + action)) - return www2.AuthorizeVote{ - Token: token, - Action: action, - PublicKey: hex.EncodeToString(id.Public.Key[:]), - Signature: hex.EncodeToString(sig[:]), - } -} - -func newAuthorizeVote(t *testing.T, token, version, action string, id *identity.FullIdentity) www.AuthorizeVote { - t.Helper() - - sig := id.SignMessage([]byte(token + version + action)) - return www.AuthorizeVote{ - Action: action, - Token: token, - Signature: hex.EncodeToString(sig[:]), - PublicKey: hex.EncodeToString(id.Public.Key[:]), + t.Fatal(err) } } -func newAuthorizeVoteCmd(t *testing.T, token, version, action string, id *identity.FullIdentity) pd.PluginCommand { +func userProposalCreditsAdd(t *testing.T, p *politeiawww, u *user.User, quantity int) { t.Helper() - /* - av := newAuthorizeVote(t, token, version, action, id) - dav := convertAuthorizeVoteToDecred(av) - payload, err := decredplugin.EncodeAuthorizeVote(dav) - if err != nil { - t.Fatal(err) - } - - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - t.Fatal(err) - } - - return pd.PluginCommand{ - Challenge: hex.EncodeToString(challenge), - ID: decredplugin.ID, - Command: decredplugin.CmdAuthorizeVote, - CommandID: decredplugin.CmdAuthorizeVote + " " + av.Token, - Payload: string(payload), + c := make([]user.ProposalCredit, quantity) + ts := time.Now().Unix() + for i := 0; i < quantity; i++ { + c[i] = user.ProposalCredit{ + PaywallID: 0, + Price: 0, + DatePurchased: ts, + TxID: "created_during_testing", } - */ - return pd.PluginCommand{} -} - -func newProposalRecord(t *testing.T, u *user.User, id *identity.FullIdentity, s www.PropStatusT) www.ProposalRecord { - t.Helper() - - f := newFileRandomMD(t) - files := []www.File{f} - name := proposalNameRandom(t) - metadata, _ := newProposalMetadata(t, name, "", 0) - m := merkleRoot(t, files, metadata) - sig := id.SignMessage([]byte(m)) - - var ( - publishedAt int64 - censoredAt int64 - abandonedAt int64 - changeMsg string - ) - - switch s { - case www.PropStatusCensored: - changeMsg = "did not adhere to guidelines" - censoredAt = time.Now().Unix() - case www.PropStatusPublic: - publishedAt = time.Now().Unix() - case www.PropStatusAbandoned: - changeMsg = "no activity" - publishedAt = time.Now().Unix() - abandonedAt = time.Now().Unix() - } - - tokenb, err := util.Random(pd.TokenSizeMax) - if err != nil { - t.Fatal(err) } + u.UnspentProposalCredits = append(u.UnspentProposalCredits, c...) - // The token is typically generated in politeiad. This function - // generates the token locally to make setting up tests easier. - // The censorship record signature is left intentionally blank. - return www.ProposalRecord{ - Name: name, - // State: convertPropStatusToState(s), - Status: s, - Timestamp: time.Now().Unix(), - UserId: u.ID.String(), - Username: u.Username, - PublicKey: u.PublicKey(), - Signature: hex.EncodeToString(sig[:]), - NumComments: 0, - Version: "1", - StatusChangeMessage: changeMsg, - PublishedAt: publishedAt, - CensoredAt: censoredAt, - AbandonedAt: abandonedAt, - Files: files, - Metadata: metadata, - CensorshipRecord: www.CensorshipRecord{ - Token: hex.EncodeToString(tokenb), - Merkle: m, - Signature: "", - }, - } -} - -func newProposalMetadata(t *testing.T, name, linkto string, linkby int64) ([]www.Metadata, www.ProposalMetadata) { - t.Helper() - - if name == "" { - // Generate a random name if none was given - name = proposalNameRandom(t) - } - pm := www.ProposalMetadata{ - Name: name, - LinkTo: linkto, - LinkBy: linkby, - } - pmb, err := json.Marshal(pm) + err := p.db.UserUpdate(*u) if err != nil { t.Fatal(err) } - md := []www.Metadata{ - { - Digest: hex.EncodeToString(util.Digest(pmb)), - Hint: www.HintProposalMetadata, - Payload: base64.StdEncoding.EncodeToString(pmb), - }, - } - return md, pm -} - -func newVoteSummary(t *testing.T, s www.PropVoteStatusT, rs []www.VoteOptionResult) www.VoteSummary { - t.Helper() - - return www.VoteSummary{ - Status: s, - EligibleTickets: 10, - QuorumPercentage: 30, - PassPercentage: 60, - Results: rs, - } -} - -func newVoteOptionV2(t *testing.T, id, desc string, bits uint64) www2.VoteOption { - t.Helper() - - return www2.VoteOption{ - Id: id, - Description: desc, - Bits: bits, - } -} - -func newVoteOptionResult(t *testing.T, id, desc string, bits, votes uint64) www.VoteOptionResult { - t.Helper() - - return www.VoteOptionResult{ - Option: www.VoteOption{ - Id: id, - Description: desc, - Bits: bits, - }, - VotesReceived: votes, - } -} - -func makeProposalRFP(t *testing.T, pr *www.ProposalRecord, linkedfrom []string, linkby int64) { - t.Helper() - - md, _ := newProposalMetadata(t, pr.Name, "", linkby) - pr.LinkBy = linkby - pr.LinkedFrom = linkedfrom - pr.Metadata = md -} - -func makeProposalRFPSubmissions(t *testing.T, prs []*www.ProposalRecord, linkto string) { - t.Helper() - - for _, pr := range prs { - md, _ := newProposalMetadata(t, pr.Name, linkto, 0) - pr.LinkTo = linkto - pr.Metadata = md - } } // newTestPoliteiawww returns a new politeiawww context that is setup for @@ -662,11 +221,17 @@ func newTestPoliteiawww(t *testing.T) (*politeiawww, func()) { t.Fatalf("open tmp dir: %v", err) } + // Setup logging + initLogRotator(filepath.Join(dataDir, "politeiawww.test.log")) + setLogLevels("off") + // Setup config + xpub := "tpubVobLtToNtTq6TZNw4raWQok35PRPZou53vegZqNubtBTJMMFm" + + "uMpWybFCfweJ52N8uZJPZZdHE5SRnBBuuRPfC5jdNstfKjiAs8JtbYG9jx" cfg := &config{ DataDir: dataDir, PaywallAmount: 1e7, - PaywallXpub: "tpubVobLtToNtTq6TZNw4raWQok35PRPZou53vegZqNubtBTJMMFmuMpWybFCfweJ52N8uZJPZZdHE5SRnBBuuRPfC5jdNstfKjiAs8JtbYG9jx", + PaywallXpub: xpub, TestNet: true, VoteDurationMin: 2016, VoteDurationMax: 4032, @@ -690,18 +255,14 @@ func newTestPoliteiawww(t *testing.T) (*politeiawww, func()) { t.Fatalf("create cookie key: %v", err) } - // Setup logging - initLogRotator(filepath.Join(dataDir, "politeiawww.test.log")) - setLogLevels("off") - - // Create politeiawww context + // Setup politeiawww context p := politeiawww{ cfg: cfg, - db: db, params: chaincfg.TestNet3Params(), router: mux.NewRouter(), sessions: newSessionStore(db, sessionMaxAge, cookieKey), smtp: smtp, + db: db, test: true, userEmails: make(map[string]uuid.UUID), userPaywallPool: make(map[uuid.UUID]paywallPoolMember), @@ -710,6 +271,7 @@ func newTestPoliteiawww(t *testing.T) (*politeiawww, func()) { // Setup routes p.setPoliteiaWWWRoutes() p.setUserWWWRoutes() + p.setPiRoutes() // The cleanup is handled using a closure so that the temp dir // can be deleted using the local variable and not cfg.DataDir. @@ -735,15 +297,6 @@ func newTestPoliteiawww(t *testing.T) (*politeiawww, func()) { } } -// newTestPoliteiad returns a new TestPoliteiad context. The relevant -// politeiawww config params are updated with the TestPoliteiad info. -func newTestPoliteiad(t *testing.T, p *politeiawww) *testpoliteiad.TestPoliteiad { - td := testpoliteiad.New(t) - p.cfg.RPCHost = td.URL - p.cfg.Identity = td.PublicIdentity - return td -} - // newTestCMSwww returns a new cmswww context that is setup for // testing and a closure that cleans up the test environment when invoked. func newTestCMSwww(t *testing.T) (*politeiawww, func()) { @@ -756,11 +309,17 @@ func newTestCMSwww(t *testing.T) (*politeiawww, func()) { t.Fatalf("open tmp dir: %v", err) } + // Setup logging + initLogRotator(filepath.Join(dataDir, "cmswww.test.log")) + setLogLevels("off") + // Setup config + xpub := "tpubVobLtToNtTq6TZNw4raWQok35PRPZou53vegZqNubtBTJMMFm" + + "uMpWybFCfweJ52N8uZJPZZdHE5SRnBBuuRPfC5jdNstfKjiAs8JtbYG9jx" cfg := &config{ DataDir: dataDir, PaywallAmount: 1e7, - PaywallXpub: "tpubVobLtToNtTq6TZNw4raWQok35PRPZou53vegZqNubtBTJMMFmuMpWybFCfweJ52N8uZJPZZdHE5SRnBBuuRPfC5jdNstfKjiAs8JtbYG9jx", + PaywallXpub: xpub, TestNet: true, VoteDurationMin: 2016, VoteDurationMax: 4032, @@ -795,10 +354,6 @@ func newTestCMSwww(t *testing.T) (*politeiawww, func()) { t.Fatalf("create cookie key: %v", err) } - // Setup logging - initLogRotator(filepath.Join(dataDir, "cmswww.test.log")) - setLogLevels("off") - // Create politeiawww context p := politeiawww{ cfg: cfg, @@ -839,3 +394,12 @@ func newTestCMSwww(t *testing.T) (*politeiawww, func()) { } } } + +// newTestPoliteiad returns a new TestPoliteiad context. The relevant +// politeiawww config params are updated with the TestPoliteiad info. +func newTestPoliteiad(t *testing.T, p *politeiawww) *testpoliteiad.TestPoliteiad { + td := testpoliteiad.New(t) + p.cfg.RPCHost = td.URL + p.cfg.Identity = td.PublicIdentity + return td +} diff --git a/politeiawww/user_test.go b/politeiawww/user_test.go index dd2de83c8..d350bced5 100644 --- a/politeiawww/user_test.go +++ b/politeiawww/user_test.go @@ -46,22 +46,32 @@ func TestValidatePubkey(t *testing.T) { pubkey string want error }{ - {"valid pubkey", valid, nil}, - - {"invalid hexadecimal", invalidHex, + { + "valid pubkey", + valid, + nil, + }, + { + "invalid hexadecimal", + invalidHex, www.UserError{ ErrorCode: www.ErrorStatusInvalidPublicKey, - }}, - - {"invalid size", invalidSize, + }, + }, + { + "invalid size", + invalidSize, www.UserError{ ErrorCode: www.ErrorStatusInvalidPublicKey, - }}, - - {"empty pubkey", empty, + }, + }, + { + "empty pubkey", + empty, www.UserError{ ErrorCode: www.ErrorStatusInvalidPublicKey, - }}, + }, + }, } // Run tests @@ -71,8 +81,7 @@ func TestValidatePubkey(t *testing.T) { got := errToStr(err) want := errToStr(v.want) if got != want { - t.Errorf("got error %v, want %v", - got, want) + t.Errorf("got error %v, want %v", got, want) } }) } @@ -97,47 +106,67 @@ func TestValidateUsername(t *testing.T) { username string want error }{ - {"contains uppercase", "Politeiauser", + { + "contains uppercase", + "Politeiauser", www.UserError{ ErrorCode: www.ErrorStatusMalformedUsername, - }}, - - {"leading whitespace", " politeiauser", + }, + }, + { + "leading whitespace", + " politeiauser", www.UserError{ ErrorCode: www.ErrorStatusMalformedUsername, - }}, - - {"trailing whitespace", "politeiauser ", + }, + }, + { + "trailing whitespace", + "politeiauser ", www.UserError{ ErrorCode: www.ErrorStatusMalformedUsername, - }}, - - {"empty", "", + }, + }, + { + "empty", + "", www.UserError{ ErrorCode: www.ErrorStatusMalformedUsername, - }}, - - {"under min length", underMin, + }, + }, + { + "under min length", + underMin, www.UserError{ ErrorCode: www.ErrorStatusMalformedUsername, - }}, - - {"over max length", overMax, + }, + }, + { + "over max length", + overMax, www.UserError{ ErrorCode: www.ErrorStatusMalformedUsername, - }}, - - {"unsupported character", "politeiauser?", + }, + }, + { + "unsupported character", + "politeiauser?", www.UserError{ ErrorCode: www.ErrorStatusMalformedUsername, - }}, - - {"contains whitespace", "politeia user", + }, + }, + { + "contains whitespace", + "politeia user", www.UserError{ ErrorCode: www.ErrorStatusMalformedUsername, - }}, - - {"valid username", "politeiauser", nil}, + }, + }, + { + "valid username", + "politeiauser", + nil, + }, } // Run tests @@ -147,8 +176,7 @@ func TestValidateUsername(t *testing.T) { got := errToStr(err) want := errToStr(v.want) if got != want { - t.Errorf("got error %v, want %v", - got, want) + t.Errorf("got error %v, want %v", got, want) } }) } @@ -426,7 +454,8 @@ func TestProcessVerifyNewUser(t *testing.T) { // An invalid token error is thrown when the user lookup // fails so that info about which email addresses exist // cannot be ascertained. - {"user not found", + { + "user not found", www.VerifyNewUser{ Email: "invalidemail", VerificationToken: token, @@ -434,9 +463,10 @@ func TestProcessVerifyNewUser(t *testing.T) { }, www.UserError{ ErrorCode: www.ErrorStatusVerificationTokenInvalid, - }}, - - {"invalid verification token", + }, + }, + { + "invalid verification token", www.VerifyNewUser{ Email: usr.Email, VerificationToken: "zzz", @@ -444,9 +474,10 @@ func TestProcessVerifyNewUser(t *testing.T) { }, www.UserError{ ErrorCode: www.ErrorStatusVerificationTokenInvalid, - }}, - - {"wrong verification token", + }, + }, + { + "wrong verification token", www.VerifyNewUser{ Email: usr.Email, VerificationToken: wrongToken, @@ -454,9 +485,10 @@ func TestProcessVerifyNewUser(t *testing.T) { }, www.UserError{ ErrorCode: www.ErrorStatusVerificationTokenInvalid, - }}, - - {"expired verification token", + }, + }, + { + "expired verification token", www.VerifyNewUser{ Email: expiredUsr.Email, VerificationToken: expiredToken, @@ -464,9 +496,10 @@ func TestProcessVerifyNewUser(t *testing.T) { }, www.UserError{ ErrorCode: www.ErrorStatusVerificationTokenExpired, - }}, - - {"invalid signature", + }, + }, + { + "invalid signature", www.VerifyNewUser{ Email: usr.Email, VerificationToken: token, @@ -474,13 +507,15 @@ func TestProcessVerifyNewUser(t *testing.T) { }, www.UserError{ ErrorCode: www.ErrorStatusInvalidSignature, - }}, + }, + }, // I didn't test the ErrorStatusNoPublicKey error path because // I don't think it is possible for that error path to be hit. // A user always has an active identity. - {"wrong signature", + { + "wrong signature", www.VerifyNewUser{ Email: usr.Email, VerificationToken: token, @@ -488,15 +523,17 @@ func TestProcessVerifyNewUser(t *testing.T) { }, www.UserError{ ErrorCode: www.ErrorStatusInvalidSignature, - }}, - - {"success", + }, + }, + { + "success", www.VerifyNewUser{ Email: usr.Email, VerificationToken: token, Signature: sig, }, - nil}, + nil, + }, } // Run tests @@ -506,8 +543,7 @@ func TestProcessVerifyNewUser(t *testing.T) { got := errToStr(err) want := errToStr(v.want) if got != want { - t.Errorf("got error %v, want %v", - got, want) + t.Errorf("got error %v, want %v", got, want) } }) } @@ -1049,183 +1085,6 @@ func TestProcessLogin(t *testing.T) { } } -/* -XXX these tests are for the login implementation that uses username instead of -email. They are being commented out until we switch the login credentials back -to username. -https://github.com/decred/politeia/issues/860#issuecomment-520871500 - -func TestProcessLogin(t *testing.T) { - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - // newUser() sets the password to be the username. This is - // why the test case passwords are set to be the usernames. - - // Test that the failed login attempts are being incremented - // properly on the user object. - t.Run("failed login attempts", func(t *testing.T) { - usr, _ := newUser(t, p, true, false) - l := www.Login{ - Username: usr.Username, - Password: "wrongpassword", - } - _, err := p.processLogin(l) - got := errToStr(err) - want := www.ErrorStatus[www.ErrorStatusInvalidPassword] - if got != want { - t.Errorf("got error %v, want %v", - got, want) - } - usr, err = p.db.UserGetById(usr.ID) - if err != nil { - t.Fatal(err) - } - if usr.FailedLoginAttempts != 1 { - t.Errorf("failed login attempts got %v, want 1", - usr.FailedLoginAttempts) - } - }) - - // Create a verified user and the expected login reply. - usr, id := newUser(t, p, true, false) - usrPassword := usr.Username - usrReply := www.LoginReply{ - IsAdmin: false, - UserID: usr.ID.String(), - Username: usr.Username, - Email: usr.Email, - PublicKey: id.Public.String(), - PaywallAddress: usr.NewUserPaywallAddress, - PaywallAmount: usr.NewUserPaywallAmount, - PaywallTxNotBefore: usr.NewUserPaywallTxNotBefore, - PaywallTxID: "", - ProposalCredits: 0, - LastLoginTime: 0, - } - - // Create a user with a locked account - usrLocked, _ := newUser(t, p, true, false) - usrLocked.FailedLoginAttempts = LoginAttemptsToLockUser - err := p.db.UserUpdate(*usrLocked) - if err != nil { - t.Fatal(err) - } - usrLockedPassword := usrLocked.Username - - // Create an unverified user - usrUnverified, _ := newUser(t, p, false, false) - usrUnverifiedPassword := usrUnverified.Username - - // Create a deactivated user - usrDeactivated, _ := newUser(t, p, true, false) - usrDeactivated.Deactivated = true - err = p.db.UserUpdate(*usrDeactivated) - if err != nil { - t.Fatal(err) - } - usrDeactivatedPassword := usrDeactivated.Username - - // Setup tests - var tests = []struct { - name string - login www.Login - wantReply *www.LoginReply - wantErr error - }{ - { - "user not found", - www.Login{ - Username: "", - Password: usrPassword, - }, - nil, - www.UserError{ - ErrorCode: www.ErrorStatusUserNotFound, - }, - }, - { - "user locked", - www.Login{ - Username: usrLocked.Username, - Password: usrLockedPassword, - }, - nil, - www.UserError{ - ErrorCode: www.ErrorStatusUserLocked, - }, - }, - { - "wrong password", - www.Login{ - Username: usr.Username, - Password: "wrongpassword", - }, - nil, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidPassword, - }, - }, - { - "user not verified", - www.Login{ - Username: usrUnverified.Username, - Password: usrUnverifiedPassword, - }, - nil, - www.UserError{ - ErrorCode: www.ErrorStatusEmailNotVerified, - }, - }, - { - "user deactivated", - www.Login{ - Username: usrDeactivated.Username, - Password: usrDeactivatedPassword, - }, - nil, - www.UserError{ - ErrorCode: www.ErrorStatusUserDeactivated, - }, - }, - { - "success", - www.Login{ - Username: usr.Username, - Password: usrPassword, - }, - &usrReply, - nil, - }, - } - - // Run tests - for _, v := range tests { - t.Run(v.name, func(t *testing.T) { - lr, err := p.processLogin(v.login) - gotErr := errToStr(err) - wantErr := errToStr(v.wantErr) - if gotErr != wantErr { - t.Errorf("got error %v, want %v", - gotErr, wantErr) - } - - // If there were errors then we're done. - if err != nil { - return - } - - // Check the reply - diff := deep.Equal(lr, v.wantReply) - if diff != nil { - t.Errorf("got/want diff:\n%v", - spew.Sdump(diff)) - } - }) - } -} -*/ - func TestProcessChangePassword(t *testing.T) { p, cleanup := newTestPoliteiawww(t) defer cleanup() @@ -1248,29 +1107,34 @@ func TestProcessChangePassword(t *testing.T) { cp www.ChangePassword want error }{ - {"wrong current password", + { + "wrong current password", www.ChangePassword{ CurrentPassword: "wrong!", NewPassword: newPass, }, www.UserError{ ErrorCode: www.ErrorStatusInvalidPassword, - }}, - - {"invalid new password", + }, + }, + { + "invalid new password", www.ChangePassword{ CurrentPassword: currPass, NewPassword: "", }, www.UserError{ ErrorCode: www.ErrorStatusMalformedPassword, - }}, - - {"success", + }, + }, + { + "success", www.ChangePassword{ CurrentPassword: currPass, NewPassword: newPass, - }, nil}, + }, + nil, + }, } // Run tests @@ -1640,37 +1504,47 @@ func TestProcessChangeUsername(t *testing.T) { cu www.ChangeUsername want error }{ - {"wrong password", u.Email, + { + "wrong password", + u.Email, www.ChangeUsername{ Password: "wrong", }, www.UserError{ ErrorCode: www.ErrorStatusInvalidPassword, - }}, - - {"invalid username", u.Email, + }, + }, + { + "invalid username", + u.Email, www.ChangeUsername{ Password: password, NewUsername: "?", }, www.UserError{ ErrorCode: www.ErrorStatusMalformedUsername, - }}, - - {"duplicate username", u.Email, + }, + }, + { + "duplicate username", + u.Email, www.ChangeUsername{ Password: password, NewUsername: u.Username, }, www.UserError{ ErrorCode: www.ErrorStatusDuplicateUsername, - }}, - - {"success", u.Email, + }, + }, + { + "success", + u.Email, www.ChangeUsername{ Password: password, NewUsername: "politeiauser", - }, nil}, + }, + nil, + }, } // Run tests @@ -1680,8 +1554,7 @@ func TestProcessChangeUsername(t *testing.T) { got := errToStr(err) want := errToStr(v.want) if got != want { - t.Errorf("got error %v, want %v", - got, want) + t.Errorf("got error %v, want %v", got, want) } }) } @@ -1731,17 +1604,31 @@ func TestProcessUserDetails(t *testing.T) { wantUsr www.User // Wanted user response wantMsg string // Description of the wanted user response }{ - {"public user details", ud, false, false, - publicUser, publicUserMsg}, - - {"admin requesting user details", ud, false, true, - fullUser, fullUserMsg}, - - {"user requesting their own details", ud, true, false, - fullUser, fullUserMsg}, + { + "public user details", + ud, false, false, + publicUser, + publicUserMsg, + }, + { + "admin requesting user details", + ud, false, true, + fullUser, + fullUserMsg, + }, - {"admin requesting their own details", ud, true, true, - fullUser, fullUserMsg}, + { + "user requesting their own details", + ud, true, false, + fullUser, + fullUserMsg, + }, + { + "admin requesting their own details", + ud, true, true, + fullUser, + fullUserMsg, + }, } // Run tests @@ -1776,23 +1663,32 @@ func TestProcessEditUser(t *testing.T) { notification uint64 want []www.EmailNotificationT }{ - {"single notification setting", 0x1, + { + "single notification setting", + 0x1, []www.EmailNotificationT{ www.NotificationEmailMyProposalStatusChange, - }}, - - {"multiple notification settings", 0x7, + }, + }, + { + "multiple notification settings", + 0x7, []www.EmailNotificationT{ www.NotificationEmailMyProposalStatusChange, www.NotificationEmailMyProposalVoteStarted, www.NotificationEmailRegularProposalVetted, - }}, - - {"no notification settings", 0x0, - []www.EmailNotificationT{}}, - - {"invalid notification setting", 0x100000, - []www.EmailNotificationT{}}, + }, + }, + { + "no notification settings", + 0x0, + []www.EmailNotificationT{}, + }, + { + "invalid notification setting", + 0x100000, + []www.EmailNotificationT{}, + }, } // Run test cases @@ -1822,8 +1718,7 @@ func TestProcessEditUser(t *testing.T) { var mask uint64 = 0x1FF bitsGot := u.EmailNotifications & mask if !(bitsWant|bitsGot == bitsWant) { - t.Errorf("notification bits got %#x, want %#x", - bitsGot, bitsWant) + t.Errorf("notification bits got %#x, want %#x", bitsGot, bitsWant) } }) } diff --git a/politeiawww/userwww_test.go b/politeiawww/userwww_test.go index b0afdbdc8..2e34e98ac 100644 --- a/politeiawww/userwww_test.go +++ b/politeiawww/userwww_test.go @@ -32,8 +32,7 @@ func newPostReq(t *testing.T, route string, body interface{}) *http.Request { t.Fatalf("%v", err) } - return httptest.NewRequest(http.MethodPost, route, - bytes.NewReader(b)) + return httptest.NewRequest(http.MethodPost, route, bytes.NewReader(b)) } // addSessionToReq initializes a user session and adds a session cookie to the @@ -151,8 +150,7 @@ func TestHandleNewUser(t *testing.T) { got := errToStr(ue) want := errToStr(v.wantError) if got != want { - t.Errorf("got error %v, want %v", - got, want) + t.Errorf("got error %v, want %v", got, want) } }) } @@ -200,8 +198,7 @@ func TestHandleVerifyNewUser(t *testing.T) { got := errToStr(ue) want := www.ErrorStatus[www.ErrorStatusInvalidInput] if got != want { - t.Errorf("got error %v, want %v", - got, want) + t.Errorf("got error %v, want %v", got, want) } }) @@ -212,19 +209,23 @@ func TestHandleVerifyNewUser(t *testing.T) { wantStatus int wantError error }{ - {"processVerifyNewUser error", + { + "processVerifyNewUser error", www.VerifyNewUser{}, http.StatusBadRequest, www.UserError{ ErrorCode: www.ErrorStatusVerificationTokenInvalid, - }}, - - {"success", + }, + }, + { + "success", www.VerifyNewUser{ Email: usr.Email, VerificationToken: token, Signature: sig, }, - http.StatusOK, nil}, + http.StatusOK, + nil, + }, } // Run tests @@ -266,8 +267,7 @@ func TestHandleVerifyNewUser(t *testing.T) { got := errToStr(ue) want := errToStr(v.wantError) if got != want { - t.Errorf("got error %v, want %v", - got, want) + t.Errorf("got error %v, want %v", got, want) } }) } @@ -300,31 +300,41 @@ func TestHandleResendVerification(t *testing.T) { wantStatus int wantError error }{ - {"invalid request body", "", http.StatusBadRequest, + { + "invalid request body", + "", + http.StatusBadRequest, www.UserError{ ErrorCode: www.ErrorStatusInvalidInput, - }}, - - {"user not found", + }, + }, + { + "user not found", www.ResendVerification{ Email: "", PublicKey: usrPubkey, }, - http.StatusOK, nil}, - - {"user already verified", + http.StatusOK, + nil, + }, + { + "user already verified", www.ResendVerification{ Email: usrVerified.Email, }, - http.StatusOK, nil}, - - {"verification already resent", + http.StatusOK, + nil, + }, + { + "verification already resent", www.ResendVerification{ Email: usrResent.Email, }, - http.StatusOK, nil}, - - {"processResendVerification error", + http.StatusOK, + nil, + }, + { + "processResendVerification error", www.ResendVerification{ Email: usr.Email, PublicKey: "abc", @@ -332,14 +342,17 @@ func TestHandleResendVerification(t *testing.T) { http.StatusBadRequest, www.UserError{ ErrorCode: www.ErrorStatusInvalidPublicKey, - }}, - - {"success", + }, + }, + { + "success", www.ResendVerification{ Email: usr.Email, PublicKey: usrPubkey, }, - http.StatusOK, nil}, + http.StatusOK, + nil, + }, } // Run tests @@ -372,8 +385,7 @@ func TestHandleResendVerification(t *testing.T) { got := errToStr(ue) want := errToStr(v.wantError) if got != want { - t.Errorf("got error %v, want %v", - got, want) + t.Errorf("got error %v, want %v", got, want) } }) } @@ -514,130 +526,11 @@ func TestHandleLogin(t *testing.T) { got := errToStr(ue) want := errToStr(v.wantError) if got != want { - t.Errorf("got error %v, want %v", - got, want) - } - }) - } -} - -/* -XXX these tests are for the login implementation that uses username instead of -email. They are being commented out until we switch the login credentials back -to username. -https://github.com/decred/politeia/issues/860#issuecomment-520871500 - -func TestHandleLogin(t *testing.T) { - p, cleanup := newTestPoliteiawww(t) - defer cleanup() - - // Create a user to test against. newUser() sets the - // password to be the same as the username. - u, _ := newUser(t, p, true, false) - password := u.Username - expectedReply, err := p.createLoginReply(u, u.LastLoginTime) - if err != nil { - t.Fatal(err) - } - expectedReply.SessionMaxAge = sessionMaxAge - - // Setup tests - var tests = []struct { - name string - reqBody interface{} - wantStatus int - wantReply *www.LoginReply - wantError error - }{ - { - "invalid request body", - "", - http.StatusBadRequest, - nil, - www.UserError{ - ErrorCode: www.ErrorStatusInvalidInput, - }, - }, - { - "processLogin error", - www.Login{}, - http.StatusUnauthorized, - nil, - www.UserError{ - ErrorCode: www.ErrorStatusUserNotFound, - }, - }, - { - "success", - www.Login{ - Username: u.Username, - Password: password, - }, - http.StatusOK, - expectedReply, - nil, - }, - } - - // Run tests - for _, v := range tests { - t.Run(v.name, func(t *testing.T) { - // Setup request - r := newPostReq(t, www.RouteLogin, v.reqBody) - w := httptest.NewRecorder() - - // Run test case - p.handleLogin(w, r) - res := w.Result() - body, _ := ioutil.ReadAll(res.Body) - - // Validate response - if res.StatusCode != v.wantStatus { - t.Errorf("got status code %v, want %v", - res.StatusCode, v.wantStatus) - } - - if res.StatusCode == http.StatusOK { - // A user session should have been - // created if login was successful. - _, err := p.getSessionUser(w, r) - if err != nil { - t.Errorf("session not created") - } - - // Check response body - var lr www.LoginReply - err = json.Unmarshal(body, &lr) - if err != nil { - t.Errorf("unmarshal LoginReply: %v", err) - } - - diff := deep.Equal(lr, *v.wantReply) - if diff != nil { - t.Errorf("LoginReply got/want diff:\n%v", - spew.Sdump(diff)) - } - - // Test case passes; next case - return - } - - var ue www.UserError - err := json.Unmarshal(body, &ue) - if err != nil { - t.Errorf("unmarshal UserError: %v", err) - } - - got := errToStr(ue) - want := errToStr(v.wantError) - if got != want { - t.Errorf("got error %v, want %v", - got, want) + t.Errorf("got error %v, want %v", got, want) } }) } } -*/ func TestHandleChangePassword(t *testing.T) { p, cleanup := newTestPoliteiawww(t) @@ -659,12 +552,16 @@ func TestHandleChangePassword(t *testing.T) { // We can assume that the request contains a valid // user session. - {"invalid request body", "", http.StatusBadRequest, + { + "invalid request body", + "", + http.StatusBadRequest, www.UserError{ ErrorCode: www.ErrorStatusInvalidInput, - }}, - - {"processChangePassword error", + }, + }, + { + "processChangePassword error", www.ChangePassword{ CurrentPassword: "", NewPassword: newPass, @@ -672,14 +569,17 @@ func TestHandleChangePassword(t *testing.T) { http.StatusBadRequest, www.UserError{ ErrorCode: www.ErrorStatusInvalidPassword, - }}, - - {"success", + }, + }, + { + "success", www.ChangePassword{ CurrentPassword: currPass, NewPassword: newPass, }, - http.StatusOK, nil}, + http.StatusOK, + nil, + }, } // Run tests @@ -716,8 +616,7 @@ func TestHandleChangePassword(t *testing.T) { got := errToStr(ue) want := errToStr(v.wantError) if got != want { - t.Errorf("got error %v, want %v", - got, want) + t.Errorf("got error %v, want %v", got, want) } }) } @@ -807,8 +706,7 @@ func TestHandleResetPassword(t *testing.T) { got := errToStr(ue) want := errToStr(v.wantError) if got != want { - t.Errorf("got error %v, want %v", - got, want) + t.Errorf("got error %v, want %v", got, want) } }) } @@ -928,23 +826,31 @@ func TestHandleChangeUsername(t *testing.T) { // We can assume that the request contains a valid // user session. - {"invalid request body", "", http.StatusBadRequest, + { + "invalid request body", + "", + http.StatusBadRequest, www.UserError{ ErrorCode: www.ErrorStatusInvalidInput, - }}, - - {"processChangeUsername error", www.ChangeUsername{}, + }, + }, + { + "processChangeUsername error", + www.ChangeUsername{}, http.StatusBadRequest, www.UserError{ ErrorCode: www.ErrorStatusInvalidPassword, - }}, - - {"success", + }, + }, + { + "success", www.ChangeUsername{ Password: pass, NewUsername: usr.Username + "aaa", }, - http.StatusOK, nil}, + http.StatusOK, + nil, + }, } // Run tests @@ -981,8 +887,7 @@ func TestHandleChangeUsername(t *testing.T) { got := errToStr(ue) want := errToStr(v.wantError) if got != want { - t.Errorf("got error %v, want %v", - got, want) + t.Errorf("got error %v, want %v", got, want) } }) } @@ -1007,20 +912,36 @@ func TestHandleUserDetails(t *testing.T) { // be caught by the router. A correct length UUID with an // invalid format will not be caught by the router and needs // to be tested for. - {"invalid uuid format", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + { + "invalid uuid format", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", false, http.StatusBadRequest, www.UserError{ ErrorCode: www.ErrorStatusInvalidInput, - }}, - - {"process user details error", "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", + }, + }, + { + "process user details error", + "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", false, http.StatusBadRequest, www.UserError{ ErrorCode: www.ErrorStatusUserNotFound, - }}, - - {"logged in user success", usr.ID.String(), true, http.StatusOK, nil}, - {"public user success", usr.ID.String(), false, http.StatusOK, nil}, + }, + }, + { + "logged in user success", + usr.ID.String(), + true, + http.StatusOK, + nil, + }, + { + "public user success", + usr.ID.String(), + false, + http.StatusOK, + nil, + }, } // Run tests @@ -1067,8 +988,7 @@ func TestHandleUserDetails(t *testing.T) { got := errToStr(ue) want := errToStr(v.wantError) if got != want { - t.Errorf("got error %v, want %v", - got, want) + t.Errorf("got error %v, want %v", got, want) } }) } @@ -1091,16 +1011,22 @@ func TestHandleEditUser(t *testing.T) { // We can assume that the request contains a valid // admin session. - {"invalid request body", "", http.StatusBadRequest, + { + "invalid request body", + "", + http.StatusBadRequest, www.UserError{ ErrorCode: www.ErrorStatusInvalidInput, - }}, - - {"success", + }, + }, + { + "success", www.EditUser{ EmailNotifications: ¬if, }, - http.StatusOK, nil}, + http.StatusOK, + nil, + }, } // Run tests @@ -1137,8 +1063,7 @@ func TestHandleEditUser(t *testing.T) { got := errToStr(ue) want := errToStr(v.wantError) if got != want { - t.Errorf("got error %v, want %v", - got, want) + t.Errorf("got error %v, want %v", got, want) } }) } diff --git a/politeiawww/www.go b/politeiawww/www.go index ab3d9daa5..5d5b04eb0 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -561,7 +561,7 @@ func (p *politeiawww) setupPi() error { // Setup routes p.setPoliteiaWWWRoutes() p.setUserWWWRoutes() - p.setupPiRoutes() + p.setPiRoutes() // Verify paywall settings switch { From 1dfe41ba99c91d2be2d8d58f0b8c98c007b901ba Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 5 Oct 2020 17:42:11 -0500 Subject: [PATCH 131/449] fix linter errors --- plugins/dcrdata/dcrdata.go | 2 +- plugins/pi/pi.go | 20 +++++++-------- plugins/ticketvote/ticketvote.go | 6 ++--- politeiad/backend/backend.go | 4 +-- politeiad/backend/gitbe/gitbe.go | 4 +-- politeiad/backend/tlogbe/anchor.go | 7 ++++-- politeiad/backend/tlogbe/comments.go | 8 +++--- politeiad/backend/tlogbe/pi.go | 2 +- politeiad/backend/tlogbe/ticketvote.go | 6 ++--- politeiad/backend/tlogbe/tlog.go | 12 +++++---- politeiad/backend/tlogbe/tlogbe.go | 34 ++++++++++++-------------- politeiad/backend/tlogbe/trillian.go | 2 +- politeiawww/api/pi/v1/v1.go | 11 +++++---- politeiawww/cmd/piwww/votes.go | 8 +++--- politeiawww/cmd/shared/client.go | 2 +- politeiawww/eventmanager.go | 32 ++++++++++++++++++------ politeiawww/piwww.go | 13 +++++----- 17 files changed, 95 insertions(+), 78 deletions(-) diff --git a/plugins/dcrdata/dcrdata.go b/plugins/dcrdata/dcrdata.go index 2b9a354f7..f7c8917f3 100644 --- a/plugins/dcrdata/dcrdata.go +++ b/plugins/dcrdata/dcrdata.go @@ -32,7 +32,7 @@ const ( // // Some commands will return cached results with the connection // status when dcrdata cannot be reached. It is the callers - // responsibilty to determine the correct course of action when + // responsibility to determine the correct course of action when // dcrdata cannot be reached. StatusInvalid StatusT = 0 StatusConnected StatusT = 1 diff --git a/plugins/pi/pi.go b/plugins/pi/pi.go index 421889a02..240e2f9e1 100644 --- a/plugins/pi/pi.go +++ b/plugins/pi/pi.go @@ -76,16 +76,16 @@ var ( // transitions. If StatusChanges[currentStatus][newStatus] exists // then the status change is allowed. StatusChanges = map[PropStatusT]map[PropStatusT]struct{}{ - PropStatusUnvetted: map[PropStatusT]struct{}{ - PropStatusPublic: struct{}{}, - PropStatusCensored: struct{}{}, + PropStatusUnvetted: { + PropStatusPublic: {}, + PropStatusCensored: {}, }, - PropStatusPublic: map[PropStatusT]struct{}{ - PropStatusAbandoned: struct{}{}, - PropStatusCensored: struct{}{}, + PropStatusPublic: { + PropStatusAbandoned: {}, + PropStatusCensored: {}, }, - PropStatusCensored: map[PropStatusT]struct{}{}, - PropStatusAbandoned: map[PropStatusT]struct{}{}, + PropStatusCensored: {}, + PropStatusAbandoned: {}, } // ErrorStatus contains human readable user error statuses. @@ -419,9 +419,9 @@ func DecodeCommentVoteReply(payload []byte) (*CommentVoteReply, error) { } // VoteInventory requests the tokens of all proposals in the inventory -// catagorized by their vote status. This call relies on the ticketvote +// categorized by their vote status. This call relies on the ticketvote // Inventory call, but breaks the Finished vote status out into Approved and -// Rejected catagories. This functionality is specific to pi. +// Rejected categories. This functionality is specific to pi. type VoteInventory struct{} // EncodeVoteInventory encodes a VoteInventory into a JSON byte slice. diff --git a/plugins/ticketvote/ticketvote.go b/plugins/ticketvote/ticketvote.go index ce0c7daef..cdd625974 100644 --- a/plugins/ticketvote/ticketvote.go +++ b/plugins/ticketvote/ticketvote.go @@ -387,7 +387,7 @@ type CastVoteReply struct { Ticket string `json:"ticket"` // Ticket ID Receipt string `json:"receipt"` // Server signature of client signature - // The follwing fields will only be present if an error occured + // The follwing fields will only be present if an error occurred // while attempting to cast the vote. ErrorCode VoteErrorT `json:"errorcode,omitempty"` ErrorContext string `json:"errorcontext,omitempty"` @@ -603,7 +603,7 @@ func DecodeSummariesReply(payload []byte) (*SummariesReply, error) { } // Inventory requests the tokens of all public, non-abandoned records -// catagorized by vote status. +// categorized by vote status. type Inventory struct{} // EncodeInventory encodes a Inventory into a JSON byte slice. @@ -622,7 +622,7 @@ func DecodeInventory(payload []byte) (*Inventory, error) { } // InventoryReply is the reply to the Inventory command. It contains the tokens -// of all public, non-abandoned records catagorized by vote status. +// of all public, non-abandoned records categorized by vote status. // TODO // Sorted by timestamp in descending order: // Unauthorized, Authorized diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index ff05e0c29..eb33b64db 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -167,7 +167,7 @@ type Plugin struct { } // InventoryByStatus contains the record tokens of all records in the inventory -// catagorized by MDStatusT. Each list is sorted by the timestamp of the status +// categorized by MDStatusT. Each list is sorted by the timestamp of the status // change from newest to oldest. type InventoryByStatus struct { Unvetted []string @@ -221,7 +221,7 @@ type Backend interface { Inventory(uint, uint, uint, bool, bool) ([]Record, []Record, error) // InventoryByStatus returns the record tokens of all records in the - // inventory catagorized by MDStatusT. + // inventory categorized by MDStatusT. InventoryByStatus() (*InventoryByStatus, error) // Register a plugin with the backend diff --git a/politeiad/backend/gitbe/gitbe.go b/politeiad/backend/gitbe/gitbe.go index cfb8484ba..37aad8ca8 100644 --- a/politeiad/backend/gitbe/gitbe.go +++ b/politeiad/backend/gitbe/gitbe.go @@ -1804,7 +1804,7 @@ func (g *gitBackEnd) UpdateUnvettedRecord(token []byte, mdAppend []backend.Metad // UpdateUnvettedMetadata is not implemented. // -// This function satsifies the Backend interface. +// This function satisfies the Backend interface. func (g *gitBackEnd) UpdateUnvettedMetadata(token []byte, mdAppend []backend.MetadataStream, mdOverwrite []backend.MetadataStream) error { return fmt.Errorf("not implemented") } @@ -2792,7 +2792,7 @@ func (g *gitBackEnd) Inventory(vettedCount, vettedStart, branchCount uint, inclu // InventoryByStatus is not implemented. // -// This function satsifies the Backend interface. +// This function satisfies the Backend interface. func (g *gitBackEnd) InventoryByStatus() (*backend.InventoryByStatus, error) { return nil, fmt.Errorf("not implemented") } diff --git a/politeiad/backend/tlogbe/anchor.go b/politeiad/backend/tlogbe/anchor.go index 7ca6a6f2f..f850dce92 100644 --- a/politeiad/backend/tlogbe/anchor.go +++ b/politeiad/backend/tlogbe/anchor.go @@ -21,7 +21,7 @@ import ( // TODO handle reorgs. A anchor record may become invalid in the case of a // reorg. We don't create the anchor record until the anchor tx has 6 -// confirmations so the probability of this occuring on mainnet is low, but it +// confirmations so the probability of this occurring on mainnet is low, but it // still needs to be handled. const ( @@ -85,6 +85,9 @@ func (t *tlog) anchorSave(a anchor) error { queued, _, err := t.trillian.leavesAppend(a.TreeID, []*trillian.LogLeaf{ logLeafNew(h, prefixedKey), }) + if err != nil { + return fmt.Errorf("leavesAppend: %v", err) + } if len(queued) != 1 { return fmt.Errorf("wrong number of queud leaves: got %v, want 1", len(queued)) @@ -201,7 +204,7 @@ func (t *tlog) anchorWait(anchors []anchor, hashes []string) { // 6 confirmations. var ( // The max retry period is set to 180 minutes to ensure that - // enough time is given for the anchor transaction to recieve 6 + // enough time is given for the anchor transaction to receive 6 // confirmations. This is based on the fact that each block has // a 99.75% chance of being mined within 30 minutes. // diff --git a/politeiad/backend/tlogbe/comments.go b/politeiad/backend/tlogbe/comments.go index c6b239160..25b429ae2 100644 --- a/politeiad/backend/tlogbe/comments.go +++ b/politeiad/backend/tlogbe/comments.go @@ -394,7 +394,7 @@ func convertCommentVoteFromBlobEntry(be store.BlobEntry) (*comments.CommentVote, } func convertCommentFromCommentAdd(ca comments.CommentAdd) comments.Comment { - // Score needs to be filled in seperately + // Score needs to be filled in separately return comments.Comment{ UserID: ca.UserID, Token: ca.Token, @@ -413,7 +413,7 @@ func convertCommentFromCommentAdd(ca comments.CommentAdd) comments.Comment { } func convertCommentFromCommentDel(cd comments.CommentDel) comments.Comment { - // Score needs to be filled in seperately + // Score needs to be filled in separately return comments.Comment{ UserID: cd.UserID, Token: cd.Token, @@ -1286,7 +1286,7 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { votes = make([]voteIndex, 0, 1) } votes = append(votes, voteIndex{ - Vote: comments.VoteT(cv.Vote), + Vote: cv.Vote, Merkle: merkle, }) cidx.Votes[cv.UserID] = votes @@ -1673,7 +1673,7 @@ func calcVoteScore(cidx commentIndex, cv comments.CommentVote) int64 { // upvotes a comment that they have already upvoted, the resulting // vote score is 0 due to the second upvote removing the original // upvote. - voteNew := comments.VoteT(cv.Vote) + voteNew := cv.Vote switch { case votePrev == 0: // No previous vote. Add the new vote to the score. diff --git a/politeiad/backend/tlogbe/pi.go b/politeiad/backend/tlogbe/pi.go index 65a41a4a5..ab11945eb 100644 --- a/politeiad/backend/tlogbe/pi.go +++ b/politeiad/backend/tlogbe/pi.go @@ -108,7 +108,7 @@ func (p *piPlugin) linkedFromLocked(token string) (*linkedFrom, error) { if errors.As(err, &e) && !os.IsExist(err) { // File does't exist. Return an empty linked from list. return &linkedFrom{ - Tokens: make(map[string]struct{}, 0), + Tokens: make(map[string]struct{}), }, nil } } diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index 394b696ad..ddd964634 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -85,7 +85,7 @@ type ticketVotePlugin struct { // record vote has ended. dataDir string - // inv contains the record inventory catagorized by vote status. + // inv contains the record inventory categorized by vote status. // The inventory will only contain public, non-abandoned records. // This cache is built on startup. inv inventory @@ -682,7 +682,7 @@ func (p *ticketVotePlugin) bestBlockUnsafe() (uint32, error) { type commitmentAddr struct { ticket string // Ticket hash addr string // Commitment address - err error // Error if one occured + err error // Error if one occurred } func (p *ticketVotePlugin) largestCommitmentAddrs(tickets []string) ([]commitmentAddr, error) { @@ -1014,7 +1014,7 @@ func voteBitVerify(options []ticketvote.VoteOption, mask, bit uint64) error { return fmt.Errorf("invalid mask 0x%x bit 0x%x", mask, bit) } - // Verify bit is inlcuded in vote options + // Verify bit is included in vote options for _, v := range options { if v.Bit == bit { // Bit matches one of the options. We're done. diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index eefb715a1..0a82aef57 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -439,11 +439,7 @@ func (t *tlog) treeExists(treeID int64) bool { log.Tracef("tlog treeExists: %v", treeID) _, err := t.trillian.tree(treeID) - if err == nil { - return true - } - - return false + return err == nil } // treeFreeze updates the status of a record and freezes the trillian tree as a @@ -558,6 +554,9 @@ func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []ba logLeafNew(freezeRecordHash, []byte(keyPrefixFreezeRecord+keys[1])), } queued, _, err := t.trillian.leavesAppend(treeID, leaves) + if err != nil { + return fmt.Errorf("leavesAppend: %v", err) + } if len(queued) != 2 { return fmt.Errorf("wrong number of queud leaves: got %v, want 2", len(queued)) @@ -680,6 +679,9 @@ func (t *tlog) recordIndexSave(treeID int64, ri recordIndex) error { logLeafNew(h, []byte(keyPrefixRecordIndex+keys[0])), } queued, _, err := t.trillian.leavesAppend(treeID, leaves) + if err != nil { + return fmt.Errorf("leavesAppend: %v", err) + } if len(queued) != 1 { return fmt.Errorf("wrong number of queud leaves: got %v, want 1", len(queued)) diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 42f620cdd..63f59cd18 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -66,20 +66,20 @@ var ( // this additional status. statusChanges = map[backend.MDStatusT]map[backend.MDStatusT]struct{}{ // Unvetted status changes - backend.MDStatusUnvetted: map[backend.MDStatusT]struct{}{ - backend.MDStatusVetted: struct{}{}, - backend.MDStatusCensored: struct{}{}, + backend.MDStatusUnvetted: { + backend.MDStatusVetted: {}, + backend.MDStatusCensored: {}, }, // Vetted status changes - backend.MDStatusVetted: map[backend.MDStatusT]struct{}{ - backend.MDStatusCensored: struct{}{}, - backend.MDStatusArchived: struct{}{}, + backend.MDStatusVetted: { + backend.MDStatusCensored: {}, + backend.MDStatusArchived: {}, }, // Statuses that do not allow any further transitions - backend.MDStatusCensored: map[backend.MDStatusT]struct{}{}, - backend.MDStatusArchived: map[backend.MDStatusT]struct{}{}, + backend.MDStatusCensored: {}, + backend.MDStatusArchived: {}, } ) @@ -232,9 +232,7 @@ func (t *tlogBackend) inventoryGet() map[backend.MDStatusT][]string { inv := make(map[backend.MDStatusT][]string, len(t.inventory)) for status, tokens := range t.inventory { tokensCopy := make([]string, len(tokens)) - for k, v := range tokens { - tokensCopy[k] = v - } + copy(tokensCopy, tokens) inv[status] = tokensCopy } @@ -494,9 +492,7 @@ func filesUpdate(filesCurr, filesAdd []backend.File, filesDel []string) []backen } // Apply adds - for _, v := range filesAdd { - f = append(f, v) - } + f = append(f, filesAdd...) return f } @@ -1392,10 +1388,10 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md return r, nil } -// Inventory is not currenctly implemented in tlogbe. If the caller which to -// pull records from the inventory then they should use the InventoryByStatus -// call to get the tokens of all records in the inventory and pull the required -// records individually. +// Inventory is not implemented in tlogbe. If the caller which to pull records +// from the inventory then they should use the InventoryByStatus call to get +// the tokens of all records in the inventory and pull the required records +// individually. // // This function satisfies the Backend interface. func (t *tlogBackend) Inventory(vettedCount, vettedStart, unvettedCount uint, includeFiles, allVersions bool) ([]backend.Record, []backend.Record, error) { @@ -1404,7 +1400,7 @@ func (t *tlogBackend) Inventory(vettedCount, vettedStart, unvettedCount uint, in } // InventoryByStatus returns the record tokens of all records in the inventory -// catagorized by MDStatusT. +// categorized by MDStatusT. // // This function satisfies the Backend interface. func (t *tlogBackend) InventoryByStatus() (*backend.InventoryByStatus, error) { diff --git a/politeiad/backend/tlogbe/trillian.go b/politeiad/backend/tlogbe/trillian.go index 2074f50f4..763048c5f 100644 --- a/politeiad/backend/tlogbe/trillian.go +++ b/politeiad/backend/tlogbe/trillian.go @@ -352,7 +352,7 @@ func (t *trillianClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) // appended. Leaves that were not successfully appended will be // returned without an inclusion proof and the caller can decide // what to do. Note this includes leaves that were not appended - // becuase they were a duplicate. + // because they were a duplicate. c := codes.Code(v.GetStatus().GetCode()) if c == codes.OK { // Verify that the merkle leaf hash is using the expected diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 4739bdd7c..949e97eff 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -21,8 +21,9 @@ type VoteErrorT int // downvotes instead of just a overall vote score. // TODO the plugin policies should be returned in a route // TODO the proposals route should allow filtering by user ID. Actually, this -// is going to have to wait until after the intial release. This is non-trivial -// to accomplish and is outside the scope of the core functionality. +// is going to have to wait until after the initial release. This is +// non-trivial to accomplish and is outside the scope of the core +// functionality. // TODO show the difference between unvetted censored and vetted censored // in the proposal inventory route since fetching them requires specifying // the state. @@ -399,7 +400,7 @@ type ProposalsReply struct { } // ProposalInventory retrieves the tokens of all proposals in the inventory, -// catagorized by proposal status and ordered by timestamp of the status change +// categorized by proposal status and ordered by timestamp of the status change // from newest to oldest. type ProposalInventory struct{} @@ -703,7 +704,7 @@ type CastVoteReply struct { Ticket string `json:"ticket"` // Ticket ID Receipt string `json:"receipt"` // Server signature of client signature - // The follwing fields will only be present if an error occured + // The follwing fields will only be present if an error occurred // while attempting to cast the vote. ErrorCode VoteErrorT `json:"errorcode,omitempty"` ErrorContext string `json:"errorcontext,omitempty"` @@ -774,7 +775,7 @@ type VoteSummariesReply struct { } // VoteInventory retrieves the tokens of all public, non-abandoned proposals -// catagorized by their vote status. +// categorized by their vote status. type VoteInventory struct{} // VoteInventoryReply in the reply to the VoteInventory command. diff --git a/politeiawww/cmd/piwww/votes.go b/politeiawww/cmd/piwww/votes.go index 290847a39..a91ff55e0 100644 --- a/politeiawww/cmd/piwww/votes.go +++ b/politeiawww/cmd/piwww/votes.go @@ -11,11 +11,11 @@ import ( "github.com/decred/politeia/politeiawww/cmd/shared" ) -// votesCmd retrieves vote details for a proposal, tallies the votes, -// and displays the result. +// votesCmd retrieves vote details for a proposal, tallies the votes, and +// displays the result. type votesCmd struct { Args struct { - Token string `positional-arg-name:"token"` // Censorship token + Token string `positional-arg-name:"token"` } `positional-args:"true" required:"true"` } @@ -34,7 +34,7 @@ func (cmd *votesCmd) Execute(args []string) error { return err } - // Get vote detials for proposal + // Get vote details for proposal vrr, err := client.Votes(v) if err != nil { return fmt.Errorf("ProposalVotes: %v", err) diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index 068a3585b..7b27f4059 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -1068,7 +1068,7 @@ func (c *Client) PayInvoices(pi *cms.PayInvoices) (*cms.PayInvoicesReply, error) } // VoteInventory retrieves the tokens of all proposals in the inventory -// catagorized by their vote status. +// categorized by their vote status. func (c *Client) VoteInventory() (*pi.VoteInventoryReply, error) { responseBody, err := c.makeRequest(http.MethodGet, pi.APIRoute, pi.RouteVoteInventory, nil) diff --git a/politeiawww/eventmanager.go b/politeiawww/eventmanager.go index f68b07cc7..f0bda1a87 100644 --- a/politeiawww/eventmanager.go +++ b/politeiawww/eventmanager.go @@ -143,12 +143,7 @@ func (p *politeiawww) setupEventListenersCMS() { // notificationIsSet returns whether the provided user has the provided // notification bit set. func notificationIsSet(emailNotifications uint64, n www.EmailNotificationT) bool { - if emailNotifications&uint64(n) == 0 { - // Notification bit not set - return false - } - // Notification bit is set - return true + return emailNotifications&uint64(n) != 0 } // userNotificationEnabled returns whether the user should receive the provided @@ -243,6 +238,10 @@ func (p *politeiawww) handleEventProposalEdited(ch chan interface{}) { // Add user to notification list emails = append(emails, u.Email) }) + if err != nil { + log.Errorf("handleEventProposalEdited: AllUsers: %v", err) + continue + } err = p.emailProposalEdited(d.name, d.username, d.token, d.version, emails) @@ -326,6 +325,10 @@ func (p *politeiawww) handleEventProposalStatusChange(ch chan interface{}) { // Add user to notification list emails = append(emails, u.Email) }) + if err != nil { + log.Errorf("handleEventProposalStatusChange: AllUsers: %v", err) + continue + } // Email users err = p.emailProposalStatusChange(d, proposalName, emails) @@ -485,6 +488,10 @@ func (p *politeiawww) handleEventProposalVoteAuthorized(ch chan interface{}) { // Add user to notification list emails = append(emails, u.Email) }) + if err != nil { + log.Errorf("handleEventProposalVoteAuthorized: AllUsers: %v", err) + continue + } // Send notification email err = p.emailProposalVoteAuthorized(d.token, d.name, d.username, emails) @@ -541,6 +548,9 @@ func (p *politeiawww) handleEventProposalVoteStarted(ch chan interface{}) { // Add user to notification list emails = append(emails, u.Email) }) + if err != nil { + log.Errorf("handleEventProposalVoteStarted: AllUsers: %v", err) + } // Email users err = p.emailProposalVoteStarted(d.token, d.name, emails) @@ -611,7 +621,7 @@ func (p *politeiawww) handleEventDCCNew(ch chan interface{}) { emails := make([]string, 0, 256) err := p.db.AllUsers(func(u *user.User) { - // Check circunstances where we don't notify + // Check circumstances where we don't notify switch { case !u.Admin: // Only notify admin users @@ -623,6 +633,9 @@ func (p *politeiawww) handleEventDCCNew(ch chan interface{}) { emails = append(emails, u.Email) }) + if err != nil { + log.Errorf("handleEventDCCNew: AllUsers: %v", err) + } err = p.emailDCCSubmitted(d.token, emails) if err != nil { @@ -647,7 +660,7 @@ func (p *politeiawww) handleEventDCCSupportOppose(ch chan interface{}) { emails := make([]string, 0, 256) err := p.db.AllUsers(func(u *user.User) { - // Check circunstances where we don't notify + // Check circumstances where we don't notify switch { case !u.Admin: // Only notify admin users @@ -659,6 +672,9 @@ func (p *politeiawww) handleEventDCCSupportOppose(ch chan interface{}) { emails = append(emails, u.Email) }) + if err != nil { + log.Errorf("handleEventDCCSupportOppose: AllUsers: %v", err) + } err = p.emailDCCSupportOppose(d.token, emails) if err != nil { diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 714e073d2..0ed04e222 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -51,8 +51,8 @@ var ( // statusReasonRequired contains the list of proposal statuses that // require an accompanying reason to be given for the status change. statusReasonRequired = map[pi.PropStatusT]struct{}{ - pi.PropStatusCensored: struct{}{}, - pi.PropStatusAbandoned: struct{}{}, + pi.PropStatusCensored: {}, + pi.PropStatusAbandoned: {}, } // errProposalNotFound is emitted when a proposal is not found in @@ -67,6 +67,7 @@ var ( // Short tokens should only be used when retrieving data. Data that is written // to disk should always reference the full length token. func tokenIsValid(token string) bool { + // Verify token size switch { case len(token) == pd.TokenPrefixLength: // Token is a short proposal token @@ -76,12 +77,10 @@ func tokenIsValid(token string) bool { // Unknown token size return false } + + // Verify token is valid hex _, err := hex.DecodeString(token) - if err != nil { - // Token is not valid hex - return false - } - return true + return err == nil } // tokenIsFullLength returns whether the provided string a is valid, full From 5be0dc9645e6e10dcaf84acb4bf87b6f6d18fadd Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 5 Oct 2020 18:37:04 -0500 Subject: [PATCH 132/449] fix errors --- politeiawww/cmd/piwww/commentcensor.go | 73 +++++++-------- politeiawww/cmd/piwww/commentnew.go | 98 ++++++++++++-------- politeiawww/cmd/piwww/commentvote.go | 119 +++++++++++++++---------- 3 files changed, 169 insertions(+), 121 deletions(-) diff --git a/politeiawww/cmd/piwww/commentcensor.go b/politeiawww/cmd/piwww/commentcensor.go index 3ec661c2e..35f482bc4 100644 --- a/politeiawww/cmd/piwww/commentcensor.go +++ b/politeiawww/cmd/piwww/commentcensor.go @@ -17,9 +17,9 @@ import ( // commentCensorCmd censors a proposal comment. type commentCensorCmd struct { Args struct { - Token string `positional-arg-name:"token"` // Censorship token - CommentID string `positional-arg-name:"commentID"` // Comment ID - Reason string `positional-arg-name:"reason"` // Reason for censoring + Token string `positional-arg-name:"token"` + CommentID string `positional-arg-name:"commentid"` + Reason string `positional-arg-name:"reason"` } `positional-args:"true" required:"true"` // CLI flags @@ -29,9 +29,18 @@ type commentCensorCmd struct { // Execute executes the censor comment command. func (cmd *commentCensorCmd) Execute(args []string) error { + // Unpack args token := cmd.Args.Token - commentID := cmd.Args.CommentID reason := cmd.Args.Reason + commentID, err := strconv.ParseUint(cmd.Args.CommentID, 10, 32) + if err != nil { + return fmt.Errorf("ParseUint(%v): %v", cmd.Args.CommentID, err) + } + + // Verify user identity + if cfg.Identity == nil { + return shared.ErrUserIdentityNotFound + } // Verify state var state pi.PropStateT @@ -46,72 +55,64 @@ func (cmd *commentCensorCmd) Execute(args []string) error { return fmt.Errorf("must specify either --vetted or unvetted") } - // Check for user identity - if cfg.Identity == nil { - return shared.ErrUserIdentityNotFound - } + // Sign comment data + msg := strconv.Itoa(int(state)) + token + cmd.Args.CommentID + reason + b := cfg.Identity.SignMessage([]byte(msg)) + signature := hex.EncodeToString(b[:]) - // Get server public key - vr, err := client.Version() - if err != nil { - return err - } - - // Setup comment censor request - s := cfg.Identity.SignMessage([]byte(string(state) + token + commentID + reason)) - signature := hex.EncodeToString(s[:]) - // Parse provided comment id - ciUint, err := strconv.ParseUint(commentID, 10, 32) - if err != nil { - return err - } + // Setup request cc := pi.CommentCensor{ Token: token, State: state, - CommentID: uint32(ciUint), + CommentID: uint32(commentID), Reason: reason, Signature: signature, - PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), + PublicKey: cfg.Identity.Public.String(), } - // Print request details + // Send request. The request and response details are printed to + // the console. err = shared.PrintJSON(cc) if err != nil { return err } - - // Send request ccr, err := client.CommentCensor(cc) if err != nil { return err } + err = shared.PrintJSON(ccr) + if err != nil { + return err + } - // Validate censor comment receipt + // Verify receipt + vr, err := client.Version() + if err != nil { + return err + } serverID, err := util.IdentityFromString(vr.PubKey) if err != nil { return err } - receiptB, err := util.ConvertSignature(ccr.Receipt) + receiptb, err := util.ConvertSignature(ccr.Receipt) if err != nil { return err } - if !serverID.VerifyMessage([]byte(signature), receiptB) { - return fmt.Errorf("could not verify receipt signature") + if !serverID.VerifyMessage([]byte(signature), receiptb) { + return fmt.Errorf("could not verify receipt") } - // Print response details - return shared.PrintJSON(ccr) + return nil } -// commentCensorHelpMsg is the output of the help command when 'commentcensor' -// is specified. +// commentCensorHelpMsg is the help command message. const commentCensorHelpMsg = `commentcensor "token" "commentID" "reason" Censor a user comment. Requires admin privileges. Arguments: 1. token (string, required) Proposal censorship token -2. commentID (string, required) Id of the comment +2. commentid (string, required) ID of the comment 3. reason (string, required) Reason for censoring the comment Flags: diff --git a/politeiawww/cmd/piwww/commentnew.go b/politeiawww/cmd/piwww/commentnew.go index 89ad7c681..745ebbb89 100644 --- a/politeiawww/cmd/piwww/commentnew.go +++ b/politeiawww/cmd/piwww/commentnew.go @@ -11,14 +11,15 @@ import ( pi "github.com/decred/politeia/politeiawww/api/pi/v1" "github.com/decred/politeia/politeiawww/cmd/shared" + "github.com/decred/politeia/util" ) // commentNewCmd submits a new proposal comment. type commentNewCmd struct { Args struct { - Token string `positional-arg-name:"token" required:"true"` // Censorship token - Comment string `positional-arg-name:"comment" required:"true"` // Comment text - ParentID string `positional-arg-name:"parentID"` // Comment parent ID + Token string `positional-arg-name:"token" required:"true"` + Comment string `positional-arg-name:"comment" required:"true"` + ParentID string `positional-arg-name:"parentid"` } `positional-args:"true"` // CLI flags @@ -27,72 +28,93 @@ type commentNewCmd struct { } // Execute executes the new comment command. -func (cmd *commentNewCmd) Execute(args []string) error { - token := cmd.Args.Token - comment := cmd.Args.Comment - parentID := cmd.Args.ParentID +func (c *commentNewCmd) Execute(args []string) error { + // Unpack args + token := c.Args.Token + comment := c.Args.Comment + parentID, err := strconv.ParseUint(c.Args.ParentID, 10, 32) + if err != nil { + return fmt.Errorf("ParseUint(%v): %v", c.Args.ParentID, err) + } + + // Verify identity + if cfg.Identity == nil { + return shared.ErrUserIdentityNotFound + } // Verify state var state pi.PropStateT switch { - case cmd.Vetted && cmd.Unvetted: + case c.Vetted && c.Unvetted: return fmt.Errorf("cannot use --vetted and --unvetted simultaneously") - case cmd.Unvetted: + case c.Unvetted: state = pi.PropStateUnvetted - case cmd.Vetted: + case c.Vetted: state = pi.PropStateVetted default: - return fmt.Errorf("must specify either --vetted or unvetted") + return fmt.Errorf("must specify either --vetted or --unvetted") } - // Check for user identity - if cfg.Identity == nil { - return shared.ErrUserIdentityNotFound - } + // Sign comment data + msg := strconv.Itoa(int(state)) + token + c.Args.ParentID + comment + b := cfg.Identity.SignMessage([]byte(msg)) + signature := hex.EncodeToString(b[:]) - // Setup new comment request - sig := cfg.Identity.SignMessage([]byte(string(state) + token + parentID + - comment)) - // Parse provided parent id - piUint, err := strconv.ParseUint(parentID, 10, 32) - if err != nil { - return err - } + // Setup request cn := pi.CommentNew{ Token: token, State: state, - ParentID: uint32(piUint), + ParentID: uint32(parentID), Comment: comment, - Signature: hex.EncodeToString(sig[:]), - PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), + Signature: signature, + PublicKey: cfg.Identity.Public.String(), } - // Print request details + // Send request. The request and response details are printed to + // the console. err = shared.PrintJSON(cn) if err != nil { return err } - - // Send request ncr, err := client.CommentNew(cn) if err != nil { return err } + err = shared.PrintJSON(ncr) + if err != nil { + return err + } + + // Verify receipt + vr, err := client.Version() + if err != nil { + return err + } + serverID, err := util.IdentityFromString(vr.PubKey) + if err != nil { + return err + } + receiptb, err := util.ConvertSignature(ncr.Receipt) + if err != nil { + return err + } + if !serverID.VerifyMessage([]byte(signature), receiptb) { + return fmt.Errorf("could not verify receipt") + } - // Print response details - return shared.PrintJSON(ncr) + return nil } -// commentNewHelpMsg is the output of the help command when 'commentnew' is -// specified. -const commentNewHelpMsg = `commentnew "token" "comment" +// commentNewHelpMsg is the help command message. +const commentNewHelpMsg = `commentnew "token" "comment" "parentid" -Comment on proposal as logged in user. +Comment on proposal as the logged in user. Arguments: -1. token (string, required) Proposal censorship token -2. comment (string, required) Comment -3. parentID (string, required if replying to comment) Id of commment +1. token (string, required) Proposal censorship token +2. comment (string, required) Comment +3. parentid (string, optional) ID of parent commment. Including a parent ID + indicates that the comment is a reply. Flags: --vetted (bool, optional) Comment on vetted record. diff --git a/politeiawww/cmd/piwww/commentvote.go b/politeiawww/cmd/piwww/commentvote.go index e7defa852..6e8f34be6 100644 --- a/politeiawww/cmd/piwww/commentvote.go +++ b/politeiawww/cmd/piwww/commentvote.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Decred developers +// Copyright (c) 2017-2020 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -11,87 +11,112 @@ import ( pi "github.com/decred/politeia/politeiawww/api/pi/v1" "github.com/decred/politeia/politeiawww/cmd/shared" + "github.com/decred/politeia/util" ) // commentVoteCmd is used to upvote/downvote a proposal comment using the // logged in the user. type commentVoteCmd struct { Args struct { - Token string `positional-arg-name:"token"` // Censorship token - CommentID string `positional-arg-name:"commentID"` // Comment ID - Action string `positional-arg-name:"action"` // Upvote/downvote action + Token string `positional-arg-name:"token"` + CommentID string `positional-arg-name:"commentID"` + Vote string `positional-arg-name:"vote"` } `positional-args:"true" required:"true"` } // Execute executes the like comment command. -func (cmd *commentVoteCmd) Execute(args []string) error { - const actionUpvote = "upvote" - const actionDownvote = "downvote" - - token := cmd.Args.Token - commentID := cmd.Args.CommentID - action := cmd.Args.Action - - // Validate action - if action != actionUpvote && action != actionDownvote { - return fmt.Errorf("invalid action %s; the action must be either "+ - "downvote or upvote", action) +func (c *commentVoteCmd) Execute(args []string) error { + votes := map[string]pi.CommentVoteT{ + "upvote": pi.CommentVoteUpvote, + "downvote": pi.CommentVoteDownvote, + "1": pi.CommentVoteUpvote, + "-1": pi.CommentVoteDownvote, } - // Check for user identity + // Unpack args + token := c.Args.Token + commentID, err := strconv.ParseUint(c.Args.CommentID, 10, 32) + if err != nil { + return fmt.Errorf("ParseUint(%v): %v", c.Args.CommentID, err) + } + vote, ok := votes[c.Args.Vote] + if !ok { + return fmt.Errorf("invalid vote option '%v' \n%v", + c.Args.Vote, commentVoteHelpMsg) + } + + // Verify identity if cfg.Identity == nil { return shared.ErrUserIdentityNotFound } - // Setup pi comment vote request - var vote pi.CommentVoteT - switch action { - case actionUpvote: - vote = pi.CommentVoteUpvote - case actionDownvote: - vote = pi.CommentVoteDownvote - } + // Sign vote choice + msg := strconv.Itoa(int(pi.PropStateVetted)) + token + + c.Args.CommentID + c.Args.Vote + b := cfg.Identity.SignMessage([]byte(msg)) + signature := hex.EncodeToString(b[:]) - sig := cfg.Identity.SignMessage([]byte(string(pi.PropStateVetted) + token + commentID + - string(vote))) - // Parse provided parent id - ciUint, err := strconv.ParseUint(commentID, 10, 32) - if err != nil { - return err - } + // Setup request cv := pi.CommentVote{ Token: token, State: pi.PropStateVetted, - CommentID: uint32(ciUint), + CommentID: uint32(commentID), Vote: vote, - Signature: hex.EncodeToString(sig[:]), - PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), + Signature: signature, + PublicKey: cfg.Identity.Public.String(), } - // Print request details + // Send request. The request and response details are printed to + // the console. err = shared.PrintJSON(cv) if err != nil { return err } - - // Send request cvr, err := client.CommentVote(cv) if err != nil { return err } + err = shared.PrintJSON(cvr) + if err != nil { + return err + } + + // Verify receipt + vr, err := client.Version() + if err != nil { + return err + } + serverID, err := util.IdentityFromString(vr.PubKey) + if err != nil { + return err + } + receiptb, err := util.ConvertSignature(cvr.Receipt) + if err != nil { + return err + } + if !serverID.VerifyMessage([]byte(signature), receiptb) { + return fmt.Errorf("could not verify receipt") + } - // Print response details - return shared.PrintJSON(cvr) + return nil } -// commentVoteHelpMsg is the output for the help command when 'commentvote' is -// specified. -const commentVoteHelpMsg = `commentvote "token" "commentID" "action" +// commentVoteHelpMsg is the help command message. +const commentVoteHelpMsg = `commentvote "token" "commentID" "vote" -Vote on a comment. +Upvote or downvote a comment as the logged in user. Arguments: -1. token (string, required) Proposal censorship token -2. commentID (string, required) Id of the comment -3. action (string, required) Vote (upvote or downvote) +1. token (string, required) Proposal censorship token +2. commentID (string, required) Comment ID +3. vote (string, required) Upvote or downvote + +You can specify either the numeric vote option (1 or -1) or the human readable +vote option. +upvote (1) +downvote (-1) + +Example usage +$ commentvote d594fbadef0f93780000 3 downvote +$ commentvote d594fbadef0f93780000 3 -1 ` From aba42fdcc051dfc14eb5700bd5e0d1ff7b099bac Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 6 Oct 2020 07:41:08 -0500 Subject: [PATCH 133/449] add custom piwww help message --- politeiad/backend/tlogbe/dcrdata.go | 2 - politeiawww/cmd/piwww/README.md | 69 ++-------- politeiawww/cmd/piwww/piwww.go | 152 +++++++++++++++------ politeiawww/cmd/piwww/proposalsetstatus.go | 6 +- politeiawww/cmd/piwww/voteballot.go | 8 +- politeiawww/cmd/shared/config.go | 2 +- politeiawww/log.go | 4 +- 7 files changed, 134 insertions(+), 109 deletions(-) diff --git a/politeiad/backend/tlogbe/dcrdata.go b/politeiad/backend/tlogbe/dcrdata.go index 9da6e45cd..49a397639 100644 --- a/politeiad/backend/tlogbe/dcrdata.go +++ b/politeiad/backend/tlogbe/dcrdata.go @@ -14,7 +14,6 @@ import ( "strings" "sync" - "github.com/davecgh/go-spew/spew" v4 "github.com/decred/dcrdata/api/types/v4" exptypes "github.com/decred/dcrdata/explorer/types/v2" pstypes "github.com/decred/dcrdata/pubsub/types/v3" @@ -483,7 +482,6 @@ func (p *dcrdataPlugin) fsck() error { } func newDcrdataPlugin(settings []backend.PluginSetting) (*dcrdataPlugin, error) { - spew.Dump(settings) // Unpack plugin settings var ( hostHTTP string diff --git a/politeiawww/cmd/piwww/README.md b/politeiawww/cmd/piwww/README.md index d3cbf4145..762c99cce 100644 --- a/politeiawww/cmd/piwww/README.md +++ b/politeiawww/cmd/piwww/README.md @@ -8,8 +8,7 @@ flag. $ piwww -h -You can view details about a specific command, including required arguments, -by using the help command. +You can view details about a specific command by using the help command. $ piwww help @@ -65,7 +64,7 @@ skipverify=true ### Create a new user - $ piwww newuser email@example.com username password --verify --paywall + $ piwww usernew email@example.com username password --verify --paywall `--verify` and `--paywall` are options that can be used when running politeiawww on testnet to make the user registration process quicker. @@ -104,7 +103,7 @@ get a `resource temporarily unavailable` error if you don't.** When submitting a proposal, you can either specify a markdown file or you can use the `--random` flag to have piwww generate a random proposal for you. - $ piwww newproposal --random + $ piwww proposalnew --random { "files": [ { @@ -119,19 +118,22 @@ use the `--random` flag to have piwww generate a random proposal for you. } { "censorshiprecord": { - "token": "299d6defa32b77f0a5534168256c6712a02c0de8037747ac213f650065529043", + "token": "2c5d74209f37ca370000", "merkle": "362ca2a93194ebee058640f36b0ba74955760cd495f2626d740334de2cbb2a8d", "signature": "729269ef6bb45003a4728c40ff5c7f1ecbc44bfcff459d43274155e42e971a0ef8830e692eb833b049df5460edd850c77f21353fe24fd43a454388b7b89d7e00" } } +Proposals are identified by their censorship record token in all other +commands. + The proposal must first be vetted by an admin before it is publicily viewable. Proposals are identified by their censorship record token, which can be found in the output of the `newproposal` command. ### Make a proposal public (admin privileges required) - $ politeiawwwcli setproposalstatus [censorshipRecordToken] public + $ piwww proposalstatusset [token] public Now that the proposal has been vetted and is publicly available, you can comment on the proposal or authorize the voting period to start. @@ -141,14 +143,14 @@ comment on the proposal or authorize the voting period to start. Before an admin can start the voting period on a proposal the author must authorize the vote. - $ politeiawwwcli authorizevote [censorhipRecordToken] + $ piwww voteauthorize [token] ### Start a proposal vote (admin privileges required) Once a proposal vote has been authorized by the author, an admin can start the voting period. - $ politeiawwwcli startvote [censorhipRecordToken] + $ piwww votestart [token] ### Voting on a proposal - politeiavoter @@ -158,51 +160,8 @@ tool. ### Voting on a proposal - piwww -You can also vote on proposals using `piww`, but it only works on testnet and -you have to be running your dcrwallet locally using the default port. If you -are doing these things, then you can use the `inventory`, `vote`, and `tally` -commands. +You can also vote on proposals using the `piww` command `voteballot`. This +casts a ballot of votes. This will only work on testnet and if you are running +your dcrwallet locally using the default port. -`inventory` will fetch all of the active proposal votes and print the details -for the proposal votes in which you have eligible tickets. - - $ piwww inventory - Token: ee42e2e231c02b3d202de9f5df7b2d361a5ab078f675a8823e3db73afb799899 - Proposal : This is the proposal title - Eligible tickets: 3 - Start block : 30938 - End block : 32954 - Mask : 3 - Vote Option: - ID : no - Description : Don't approve proposal - Bits : 1 - Vote Option: - ID : yes - Description : Approve proposal - Bits : 2 - To choose this option: politeiawwwcli vote ee42e2e231c02b3d202de9f5df7b2d361a5ab078f675a8823e3db73afb799899 yes - -`vote` will cast votes using your eligible tickets. You'll be asked to enter -your wallet password. - - $ piwww vote ee42e2e231c02b3d202de9f5df7b2d361a5ab078f675a8823e3db73afb799899 yes - Enter the private passphrase of your wallet: - Votes succeeded: 3 - Votes failed : 0 - -`tally` will return the current voting resuts the for passed in proposal. - - $ piwww tally ee42e2e231c02b3d202de9f5df7b2d361a5ab078f675a8823e3db73afb799899 - Vote Option: - ID : no - Description : Don't approve proposal - Bits : 1 - Votes received : 0 - Percentage : 0% - Vote Option: - ID : yes - Description : Approve proposal - Bits : 2 - Votes received : 3 - Percentage : 100% + $ piwww voteballot [token] [voteID] diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index 8006bd277..c73c0e89e 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -7,7 +7,6 @@ package main import ( "errors" "fmt" - "net/url" "os" flags "github.com/jessevdk/go-flags" @@ -33,11 +32,6 @@ var ( ) type piwww struct { - // XXX the config does not need to be a part of this struct, but - // is included so that the config cli flags print as part of the - // piwww help message. This is handled by go-flags. - Config shared.Config - // Basic commands Help helpCmd `command:"help"` @@ -49,6 +43,24 @@ type piwww struct { Logout shared.LogoutCmd `command:"logout"` Me shared.MeCmd `command:"me"` + // User commands + UserNew userNewCmd `command:"usernew"` + UserEdit userEditCmd `command:"useredit"` + UserManage shared.UserManageCmd `command:"usermanage"` + UserEmailVerify userEmailVerifyCmd `command:"useremailverify"` + UserVerificationResend userVerificationResendCmd `command:"userverificationresend"` + UserPasswordReset shared.UserPasswordResetCmd `command:"userpasswordreset"` + UserPasswordChange shared.UserPasswordChangeCmd `command:"userpasswordchange"` + UserUsernameChange shared.UserUsernameChangeCmd `command:"userusernamechange"` + UserKeyUpdate shared.UserKeyUpdateCmd `command:"userkeyupdate"` + UserTOTPSet shared.UserTOTPSetCmd `command:"usertotpset"` + UserTOTPVerify shared.UserTOTPVerifyCmd `command:"usertotpverify"` + UserPaymentVerify userPaymentVerifyCmd `command:"userpaymentverify"` + UserPaymentsRescan userPaymentsRescanCmd `command:"userpaymentsrescan"` + UserPendingPayment userPendingPaymentCmd `command:"userpendingpayment"` + UserDetails userDetailsCmd `command:"userdetails"` + Users shared.UsersCmd `command:"users"` + // TODO some of the proposal commands use both the --unvetted and // --vetted flags. Let make them all use only the --unvetted flag. // If --unvetted is not included then its assumed to be a vetted @@ -57,7 +69,7 @@ type piwww struct { // Proposal commands ProposalNew proposalNewCmd `command:"proposalnew"` ProposalEdit proposalEditCmd `command:"proposaledit"` - ProposalSetStatus proposalSetStatusCmd `command:"proposalsetstatus"` + ProposalStatusSet proposalStatusSetCmd `command:"proposalstatusset"` Proposals proposalsCmd `command:"proposals"` ProposalInventory proposalInventoryCmd `command:"proposalinventory"` @@ -78,24 +90,6 @@ type piwww struct { VoteSummaries voteSummariesCmd `command:"votesummaries"` VoteInventory voteInventoryCmd `command:"voteinventory"` - // User commands - UserNew userNewCmd `command:"usernew"` - UserEdit userEditCmd `command:"useredit"` - UserDetails userDetailsCmd `command:"userdetails"` - UserPaymentsRescan userPaymentsRescanCmd `command:"userpaymentsrescan"` - UserPendingPayment userPendingPaymentCmd `command:"userpendingpayment"` - UserEmailVerify userEmailVerifyCmd `command:"useremailverify"` - UserPaymentVerify userPaymentVerifyCmd `command:"userpaymentverify"` - UserVerificationResend userVerificationResendCmd `command:"userverificationresend"` - UserManage shared.UserManageCmd `command:"usermanage"` - UserKeyUpdate shared.UserKeyUpdateCmd `command:"userkeyupdate"` - UserUsernameChange shared.UserUsernameChangeCmd `command:"userusernamechange"` - UserPasswordChange shared.UserPasswordChangeCmd `command:"userpasswordchange"` - UserPasswordReset shared.UserPasswordResetCmd `command:"userpasswordreset"` - UserTOTPSet shared.UserTOTPSetCmd `command:"usertotpset"` - UserTOTPVerify shared.UserTOTPVerifyCmd `command:"usertotpverify"` - Users shared.UsersCmd `command:"users"` - // TODO rename to reflect that its a users route ProposalPaywall proposalPaywallCmd `command:"proposalpaywall"` @@ -107,6 +101,77 @@ type piwww struct { SendFaucetTx sendFaucetTxCmd `command:"sendfaucettx"` } +// TODO add proposalpaywall to this once the command is updated +const helpMsg = `Application Options: + --appdata= Path to application home directory + --host= politeiawww host + -j, --json Print raw JSON output + --version Display version information and exit + --skipverify Skip verifying the server's certifcate chain and host name + -v, --verbose Print verbose output + --silent Suppress all output + +Help commands + help Print detailed help message for a command + +Basic commands + version (public) Get politeiawww server version + policy (public) Get politeiawww server policy + secret (public) Ping the server + login (public) Login to politeiawww + logout (user) Logout from politeiawww + me (user) Get details of the logged in user + +User commands + usernew (public) Create a new user + useredit (user) Edit the logged in user + usermanage (admin) Edit a user as an admin + useremailverify (public) Verify email address + userverificationresend (public) Resend verification email + userpasswordreset (public) Reset password + userpasswordchange (user) Change password + userusernamechange (user) Change username + userkeyupdate (user) Update user key (i.e. identity) + usertotpset (user) Set a TOTP method + usertotpverify (user) Verify a TOTP method + userpaymentverify (user) Verify registration payment + userpaymentsrescan (user) Rescan all user payments + userpendingpayment (user) Get pending user payments + userdetails (public) Get user details + users (public) Get users + +Proposal commands + proposalnew (user) Submit a new proposal + proposaledit (user) Edit an existing proposal + proposalsetstatus (admin) Set the status of a proposal + proposals (public) Get proposals + proposalinventory (public) Get proposals inventory by proposal status + +Comment commands + commentnew (user) Submit a new comment + commentvote (user) Upvote/downvote a comment + commentcensor (admin) Censor a comment + comments (public) Get comments + commentvotes (public) Get comment votes + +Vote commands + voteauthorize (user) Authorize a proposal vote + votestart (admin) Start a proposal vote + votestartrunoff (admin) Start a runoff vote + voteballot (public) Cast a ballot of votes + votes (public) Get vote details + voteresults (public) Get full vote results + votesummaries (public) Get vote summaries + voteinventory (public) Get proposal inventory by vote status + +Websocket commands + subscribe (public) Subscribe/unsubscribe to websocket event + +Dev commands + sendfaucettx Send a dcr faucet tx + testrun Execute a test run of pi routes +` + func _main() error { // Load config. The config variable is a CLI global variable. var err error @@ -126,31 +191,34 @@ func _main() error { shared.SetConfig(cfg) shared.SetClient(client) + // Check for a help flag. This is done separately so that we can + // print our own custom help message + var opts flags.Options = flags.HelpFlag | flags.IgnoreUnknown | + flags.PassDoubleDash + parser := flags.NewParser(&struct{}{}, opts) + _, err = parser.Parse() + if err != nil { + var flagsErr *flags.Error + if errors.As(err, &flagsErr) && flagsErr.Type == flags.ErrHelp { + fmt.Printf("%v\n", helpMsg) + return nil + } + return fmt.Errorf("parse help flags: %v", err) + } + // Get politeiawww CSRF token if cfg.CSRF == "" { _, err := client.Version() if err != nil { - var e *url.Error - if !errors.As(err, &e) { - // A url error likely means that politeiawww is not - // running. The user may just be trying to print the - // help message so only return an error if its not - // a url error. - return fmt.Errorf("Version: %v", err) - } + return fmt.Errorf("Version: %v", err) } } // Parse subcommand and execute - var cli piwww - var parser = flags.NewParser(&cli, flags.Default) - if _, err := parser.Parse(); err != nil { - var flagsErr *flags.Error - if errors.As(err, &flagsErr) && flagsErr.Type == flags.ErrHelp { - os.Exit(0) - } else { - os.Exit(1) - } + parser = flags.NewParser(&piwww{}, flags.Default) + _, err = parser.Parse() + if err != nil { + os.Exit(1) } return nil diff --git a/politeiawww/cmd/piwww/proposalsetstatus.go b/politeiawww/cmd/piwww/proposalsetstatus.go index 45ce4f7fa..087c531de 100644 --- a/politeiawww/cmd/piwww/proposalsetstatus.go +++ b/politeiawww/cmd/piwww/proposalsetstatus.go @@ -13,8 +13,8 @@ import ( "github.com/decred/politeia/politeiawww/cmd/shared" ) -// proposalSetStatusCmd sets the status of a proposal. -type proposalSetStatusCmd struct { +// proposalStatusSetCmd sets the status of a proposal. +type proposalStatusSetCmd struct { Args struct { Token string `positional-arg-name:"token" required:"true"` Status string `positional-arg-name:"status" required:"true"` @@ -25,7 +25,7 @@ type proposalSetStatusCmd struct { } // Execute executes the set proposal status command. -func (cmd *proposalSetStatusCmd) Execute(args []string) error { +func (cmd *proposalStatusSetCmd) Execute(args []string) error { propStatus := map[string]pi.PropStatusT{ "public": pi.PropStatusPublic, "censored": pi.PropStatusCensored, diff --git a/politeiawww/cmd/piwww/voteballot.go b/politeiawww/cmd/piwww/voteballot.go index 09c07165c..7065d7dcc 100644 --- a/politeiawww/cmd/piwww/voteballot.go +++ b/politeiawww/cmd/piwww/voteballot.go @@ -19,7 +19,7 @@ import ( "golang.org/x/crypto/ssh/terminal" ) -// voteBallotCmd casts a votes ballot for the specified proposal. +// voteBallotCmd casts a ballot of votes for the specified proposal. type voteBallotCmd struct { Args struct { Token string `positional-arg-name:"token"` // Censorship token @@ -215,11 +215,11 @@ func (cmd *voteBallotCmd) Execute(args []string) error { return nil } -// voteBallotHelpMsg is the output of the help command when 'voteballot' -// is specified. +// voteBallotHelpMsg is the help command message. const voteBallotHelpMsg = `voteballot "token" "voteid" -Cast ticket votes for a proposal. +Cast ticket votes for a proposal. This command will only work when on testnet +and when running dcrwallet locally on the default port. Arguments: 1. token (string, optional) Proposal censorship token diff --git a/politeiawww/cmd/shared/config.go b/politeiawww/cmd/shared/config.go index 1e6de1f55..912963c3e 100644 --- a/politeiawww/cmd/shared/config.go +++ b/politeiawww/cmd/shared/config.go @@ -45,7 +45,7 @@ type Config struct { HomeDir string `long:"appdata" description:"Path to application home directory"` Host string `long:"host" description:"politeiawww host"` RawJSON bool `short:"j" long:"json" description:"Print raw JSON output"` - ShowVersion bool `short:"V" long:"version" description:"Display version information and exit"` + ShowVersion bool `long:"version" description:"Display version information and exit"` SkipVerify bool `long:"skipverify" description:"Skip verifying the server's certifcate chain and host name"` Verbose bool `short:"v" long:"verbose" description:"Print verbose output"` Silent bool `long:"silent" description:"Suppress all output"` diff --git a/politeiawww/log.go b/politeiawww/log.go index d98ff1495..c769694b4 100644 --- a/politeiawww/log.go +++ b/politeiawww/log.go @@ -45,7 +45,7 @@ var ( logRotator *rotator.Rotator log = backendLog.Logger("PWWW") - userdbLog = backendLog.Logger("USDB") + userdbLog = backendLog.Logger("USER") cmsdbLog = backendLog.Logger("CMDB") wsdcrdataLog = backendLog.Logger("WSDD") ) @@ -61,7 +61,7 @@ func init() { // subsystemLoggers maps each subsystem identifier to its associated logger. var subsystemLoggers = map[string]slog.Logger{ "PWWW": log, - "USDB": userdbLog, + "USER": userdbLog, "CMDB": cmsdbLog, "WSDD": wsdcrdataLog, } From e5eb051be88d463502d308df5b613b6573d31c25 Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Tue, 6 Oct 2020 10:59:43 -0300 Subject: [PATCH 134/449] piwww: Remove --vetted flag and default to vetted. --- politeiawww/cmd/piwww/commentcensor.go | 17 +++++++---------- politeiawww/cmd/piwww/commentnew.go | 17 +++++++---------- politeiawww/cmd/piwww/comments.go | 19 +++++++------------ politeiawww/cmd/piwww/proposaledit.go | 15 ++++++--------- politeiawww/cmd/piwww/proposalsetstatus.go | 18 ++++++++---------- 5 files changed, 35 insertions(+), 51 deletions(-) diff --git a/politeiawww/cmd/piwww/commentcensor.go b/politeiawww/cmd/piwww/commentcensor.go index 35f482bc4..74ffb9915 100644 --- a/politeiawww/cmd/piwww/commentcensor.go +++ b/politeiawww/cmd/piwww/commentcensor.go @@ -23,7 +23,6 @@ type commentCensorCmd struct { } `positional-args:"true" required:"true"` // CLI flags - Vetted bool `long:"vetted" optional:"true"` Unvetted bool `long:"unvetted" optional:"true"` } @@ -42,17 +41,14 @@ func (cmd *commentCensorCmd) Execute(args []string) error { return shared.ErrUserIdentityNotFound } - // Verify state + // Verify state. Defaults to vetted if the --unvetted flag + // is not used. var state pi.PropStateT switch { - case cmd.Vetted && cmd.Unvetted: - return fmt.Errorf("cannot use --vetted and --unvetted simultaneously") case cmd.Unvetted: state = pi.PropStateUnvetted - case cmd.Vetted: - state = pi.PropStateVetted default: - return fmt.Errorf("must specify either --vetted or unvetted") + state = pi.PropStateVetted } // Sign comment data @@ -108,7 +104,9 @@ func (cmd *commentCensorCmd) Execute(args []string) error { // commentCensorHelpMsg is the help command message. const commentCensorHelpMsg = `commentcensor "token" "commentID" "reason" -Censor a user comment. Requires admin privileges. +Censor a user comment. This command assumes the record is a vetted record. If +the record is unvetted, the --unvetted flag must be used. Requires admin +privileges. Arguments: 1. token (string, required) Proposal censorship token @@ -116,6 +114,5 @@ Arguments: 3. reason (string, required) Reason for censoring the comment Flags: - --vetted (bool, optional) Comment on vetted record. - --unvetted (bool, optional) Comment on unvetted reocrd. + --unvetted (bool, optional) Comment on unvetted record. ` diff --git a/politeiawww/cmd/piwww/commentnew.go b/politeiawww/cmd/piwww/commentnew.go index 745ebbb89..bae458116 100644 --- a/politeiawww/cmd/piwww/commentnew.go +++ b/politeiawww/cmd/piwww/commentnew.go @@ -23,7 +23,6 @@ type commentNewCmd struct { } `positional-args:"true"` // CLI flags - Vetted bool `long:"vetted" optional:"true"` Unvetted bool `long:"unvetted" optional:"true"` } @@ -42,17 +41,14 @@ func (c *commentNewCmd) Execute(args []string) error { return shared.ErrUserIdentityNotFound } - // Verify state + // Verify state. Defaults to vetted if the --unvetted flag + // is not used. var state pi.PropStateT switch { - case c.Vetted && c.Unvetted: - return fmt.Errorf("cannot use --vetted and --unvetted simultaneously") case c.Unvetted: state = pi.PropStateUnvetted - case c.Vetted: - state = pi.PropStateVetted default: - return fmt.Errorf("must specify either --vetted or --unvetted") + state = pi.PropStateVetted } // Sign comment data @@ -108,7 +104,9 @@ func (c *commentNewCmd) Execute(args []string) error { // commentNewHelpMsg is the help command message. const commentNewHelpMsg = `commentnew "token" "comment" "parentid" -Comment on proposal as the logged in user. +Comment on a record as logged in user. This command assumes the record is a +vetted record. If the record is unvetted, the --unvetted flag must be used. +Requires admin priviledges. Arguments: 1. token (string, required) Proposal censorship token @@ -117,6 +115,5 @@ Arguments: indicates that the comment is a reply. Flags: - --vetted (bool, optional) Comment on vetted record. - --unvetted (bool, optional) Comment on unvetted reocrd. + --unvetted (bool, optional) Comment on unvetted record. ` diff --git a/politeiawww/cmd/piwww/comments.go b/politeiawww/cmd/piwww/comments.go index 08e40474c..89725238f 100644 --- a/politeiawww/cmd/piwww/comments.go +++ b/politeiawww/cmd/piwww/comments.go @@ -5,8 +5,6 @@ package main import ( - "fmt" - pi "github.com/decred/politeia/politeiawww/api/pi/v1" "github.com/decred/politeia/politeiawww/cmd/shared" ) @@ -18,7 +16,6 @@ type commentsCmd struct { } `positional-args:"true" required:"true"` // CLI flags - Vetted bool `long:"vetted" optional:"true"` Unvetted bool `long:"unvetted" optional:"true"` } @@ -26,17 +23,14 @@ type commentsCmd struct { func (cmd *commentsCmd) Execute(args []string) error { token := cmd.Args.Token - // Verify state + // Verify state. Defaults to vetted if the --unvetted flag + // is not used. var state pi.PropStateT switch { - case cmd.Vetted && cmd.Unvetted: - return fmt.Errorf("cannot use --vetted and --unvetted simultaneously") case cmd.Unvetted: state = pi.PropStateUnvetted - case cmd.Vetted: - state = pi.PropStateVetted default: - return fmt.Errorf("must specify either --vetted or unvetted") + state = pi.PropStateVetted } gcr, err := client.Comments(pi.Comments{ @@ -54,12 +48,13 @@ func (cmd *commentsCmd) Execute(args []string) error { // is specified. const commentsHelpMsg = `comments "token" -Get the comments for a proposal. +Get the comments for a record. This command assumes the record is a vetted +record. If the record is unvetted, the --unvetted flag must be used. Requires +admin priviledges. Arguments: 1. token (string, required) Proposal censorship token Flags: - --vetted (bool, optional) Comment on vetted record. - --unvetted (bool, optional) Comment on unvetted reocrd. + --unvetted (bool, optional) Comment on unvetted record. ` diff --git a/politeiawww/cmd/piwww/proposaledit.go b/politeiawww/cmd/piwww/proposaledit.go index ee5526c30..f3b953e96 100644 --- a/politeiawww/cmd/piwww/proposaledit.go +++ b/politeiawww/cmd/piwww/proposaledit.go @@ -30,7 +30,6 @@ type proposalEditCmd struct { } `positional-args:"true" optional:"true"` // CLI flags - Vetted bool `long:"vetted" optional:"true"` Unvetted bool `long:"unvetted" optional:"true"` Name string `long:"name" optional:"true"` LinkTo string `long:"linkto" optional:"true"` @@ -71,17 +70,14 @@ func (cmd *proposalEditCmd) Execute(args []string) error { "--random generates a random name and random proposal data") } - // Verify state + // Verify state. Defaults to vetted if the --unvetted flag + // is not used. var state pi.PropStateT switch { - case cmd.Vetted && cmd.Unvetted: - return fmt.Errorf("cannot use --vetted and --unvetted simultaneously") case cmd.Unvetted: state = pi.PropStateUnvetted - case cmd.Vetted: - state = pi.PropStateVetted default: - return fmt.Errorf("must specify either --vetted or unvetted") + state = pi.PropStateVetted } // Check for user identity. A user identity is required to sign @@ -241,7 +237,9 @@ func (cmd *proposalEditCmd) Execute(args []string) error { // proposalEditHelpMsg is the output of the help command. const proposalEditHelpMsg = `editproposal [flags] "token" "indexfile" "attachments" -Edit a proposal. +Edit a proposal. This command assumes the proposal is a vetted record. If +the proposal is unvetted, the --unvetted flag must be used. Requires admin +priviledges. Arguments: 1. token (string, required) Proposal censorship token @@ -249,7 +247,6 @@ Arguments: 3. attachments (string, optional) Attachment files Flags: - --vetted (bool, optional) Comment on vetted record. --unvetted (bool, optional) Comment on unvetted record. --random (bool, optional) Generate a random proposal name & files to submit. If this flag is used then the markdown diff --git a/politeiawww/cmd/piwww/proposalsetstatus.go b/politeiawww/cmd/piwww/proposalsetstatus.go index 087c531de..69c973cd2 100644 --- a/politeiawww/cmd/piwww/proposalsetstatus.go +++ b/politeiawww/cmd/piwww/proposalsetstatus.go @@ -20,7 +20,7 @@ type proposalStatusSetCmd struct { Status string `positional-arg-name:"status" required:"true"` Reason string `positional-arg-name:"reason"` } `positional-args:"true"` - Vetted bool `long:"vetted" optional:"true"` + Unvetted bool `long:"unvetted" optional:"true"` } @@ -32,17 +32,14 @@ func (cmd *proposalStatusSetCmd) Execute(args []string) error { "abandoned": pi.PropStatusAbandoned, } - // Verify state + // Verify state. Defaults to vetted if the --unvetted flag + // is not used. var state pi.PropStateT switch { - case cmd.Vetted && cmd.Unvetted: - return fmt.Errorf("cannot use --vetted and --unvetted simultaneously") case cmd.Unvetted: state = pi.PropStateUnvetted - case cmd.Vetted: - state = pi.PropStateVetted default: - return fmt.Errorf("must specify either --vetted or unvetted") + state = pi.PropStateVetted } // Validate user identity @@ -109,7 +106,9 @@ func (cmd *proposalStatusSetCmd) Execute(args []string) error { // proposalSetStatusHelpMsg is the output of the help command. const proposalSetStatusHelpMsg = `proposalsetstatus "token" "status" "reason" -Set the status of a proposal. Requires admin privileges. +Set the status of a proposal. This command assumes the proposal is a vetted +record. If the proposal is unvetted, the --unvetted flag must be used. Requires +admin priviledges. Valid statuses: public @@ -122,6 +121,5 @@ Arguments: 3. message (string, optional) Status change message Flags: - --vetted (bool, optional) Set status of a vetted record. - --unvetted (bool, optional) Set status of an unvetted reocrd. + --unvetted (bool, optional) Set status of an unvetted record. ` From f8666643431d4a8775d1301e3a80f28b3937e0e6 Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Tue, 6 Oct 2020 11:00:10 -0300 Subject: [PATCH 135/449] piwww: Verify proposals with pi types. --- politeiawww/api/www/v1/v1.go | 2 +- politeiawww/cmd/piwww/help.go | 5 +- politeiawww/cmd/piwww/piwww.go | 4 +- politeiawww/cmd/piwww/proposaledit.go | 33 ++++--- politeiawww/cmd/piwww/proposalnew.go | 37 ++++--- politeiawww/cmd/piwww/proposals.go | 13 ++- politeiawww/cmd/piwww/testrun.go | 2 +- ...posalpaywall.go => userproposalpaywall.go} | 14 +-- politeiawww/cmd/piwww/util.go | 97 +++++++++++++++++++ politeiawww/cmd/shared/client.go | 4 +- 10 files changed, 157 insertions(+), 54 deletions(-) rename politeiawww/cmd/piwww/{proposalpaywall.go => userproposalpaywall.go} (50%) diff --git a/politeiawww/api/www/v1/v1.go b/politeiawww/api/www/v1/v1.go index 3bb991ba4..ae69a400b 100644 --- a/politeiawww/api/www/v1/v1.go +++ b/politeiawww/api/www/v1/v1.go @@ -788,7 +788,7 @@ type UserRegistrationPaymentReply struct { // server that the user needs in order to purchase paywall credits. type UserProposalPaywall struct{} -// UserProposalPaywallReply is used to reply to the ProposalPaywallDetails +// UserProposalPaywallReply is used to reply to the UserProposalPaywall // command. type UserProposalPaywallReply struct { CreditPrice uint64 `json:"creditprice"` // Cost per proposal credit in atoms diff --git a/politeiawww/cmd/piwww/help.go b/politeiawww/cmd/piwww/help.go index 90e0f072a..053f2c4fb 100644 --- a/politeiawww/cmd/piwww/help.go +++ b/politeiawww/cmd/piwww/help.go @@ -94,6 +94,8 @@ func (cmd *helpCmd) Execute(args []string) error { fmt.Printf("%s\n", userEmailVerifyHelpMsg) case "userpaymentverify": fmt.Printf("%s\n", userPaymentVerifyHelpMsg) + case "userproposalpaywall": + fmt.Printf("%s\n", userProposalPaywallHelpMsg) case "usermanage": fmt.Printf("%s\n", shared.UserManageHelpMsg) case "userkeyupdate": @@ -109,9 +111,6 @@ func (cmd *helpCmd) Execute(args []string) error { case "users": fmt.Printf("%s\n", shared.UsersHelpMsg) - case "proposalpaywall": - fmt.Printf("%s\n", proposalPaywallHelpMsg) - // Websocket commands case "subscribe": fmt.Printf("%s\n", subscribeHelpMsg) diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index c73c0e89e..2c4a7a7a0 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -55,6 +55,7 @@ type piwww struct { UserKeyUpdate shared.UserKeyUpdateCmd `command:"userkeyupdate"` UserTOTPSet shared.UserTOTPSetCmd `command:"usertotpset"` UserTOTPVerify shared.UserTOTPVerifyCmd `command:"usertotpverify"` + UserProposalPaywall userProposalPaywallCmd `command:"userproposalpaywall"` UserPaymentVerify userPaymentVerifyCmd `command:"userpaymentverify"` UserPaymentsRescan userPaymentsRescanCmd `command:"userpaymentsrescan"` UserPendingPayment userPendingPaymentCmd `command:"userpendingpayment"` @@ -90,9 +91,6 @@ type piwww struct { VoteSummaries voteSummariesCmd `command:"votesummaries"` VoteInventory voteInventoryCmd `command:"voteinventory"` - // TODO rename to reflect that its a users route - ProposalPaywall proposalPaywallCmd `command:"proposalpaywall"` - // Websocket commands Subscribe subscribeCmd `command:"subscribe"` diff --git a/politeiawww/cmd/piwww/proposaledit.go b/politeiawww/cmd/piwww/proposaledit.go index f3b953e96..dfc86c704 100644 --- a/politeiawww/cmd/piwww/proposaledit.go +++ b/politeiawww/cmd/piwww/proposaledit.go @@ -214,22 +214,23 @@ func (cmd *proposalEditCmd) Execute(args []string) error { return err } - // TODO add this back in. The www verify functions were moved to - // the cmswww batchproposals command since piwww no longer uses the - // www types. Create a verifyProposal function for the pi api - // proposal. - /* - // Verify proposal - vr, err := client.Version() - if err != nil { - return err - } - err = verifyProposal(per.Proposal, vr.PubKey) - if err != nil { - return fmt.Errorf("unable to verify proposal %v: %v", - per.Proposal.CensorshipRecord.Token, err) - } - */ + // Verify proposal + vr, err := client.Version() + if err != nil { + return err + } + pr := pi.ProposalRecord{ + Files: pe.Files, + Metadata: pe.Metadata, + PublicKey: pe.PublicKey, + Signature: pe.Signature, + CensorshipRecord: per.CensorshipRecord, + } + err = verifyProposal(pr, vr.PubKey) + if err != nil { + return fmt.Errorf("unable to verify proposal %v: %v", + pr.CensorshipRecord.Token, err) + } return nil } diff --git a/politeiawww/cmd/piwww/proposalnew.go b/politeiawww/cmd/piwww/proposalnew.go index 9c7bbd92a..8ff85d288 100644 --- a/politeiawww/cmd/piwww/proposalnew.go +++ b/politeiawww/cmd/piwww/proposalnew.go @@ -178,26 +178,23 @@ func (cmd *proposalNewCmd) Execute(args []string) error { return err } - // Verify the censorship record - /* - // TODO implement this using pi types - vr, err := client.Version() - if err != nil { - return err - } - pr := v1.ProposalRecord{ - Files: np.Files, - Metadata: np.Metadata, - PublicKey: np.PublicKey, - Signature: np.Signature, - CensorshipRecord: npr.CensorshipRecord, - } - err = shared.VerifyProposal(pr, vr.PubKey) - if err != nil { - return fmt.Errorf("unable to verify proposal %v: %v", - pr.CensorshipRecord.Token, err) - } - */ + // Verify proposal + vr, err := client.Version() + if err != nil { + return err + } + pr := pi.ProposalRecord{ + Files: pn.Files, + Metadata: pn.Metadata, + PublicKey: pn.PublicKey, + Signature: pn.Signature, + CensorshipRecord: pnr.CensorshipRecord, + } + err = verifyProposal(pr, vr.PubKey) + if err != nil { + return fmt.Errorf("unable to verify proposal %v: %v", + pr.CensorshipRecord.Token, err) + } return nil } diff --git a/politeiawww/cmd/piwww/proposals.go b/politeiawww/cmd/piwww/proposals.go index e98d3961a..52b87bae3 100644 --- a/politeiawww/cmd/piwww/proposals.go +++ b/politeiawww/cmd/piwww/proposals.go @@ -83,7 +83,18 @@ func (cmd *proposalsCmd) Execute(args []string) error { return err } - // TODO verify proposals + // Verify proposals + vr, err := client.Version() + if err != nil { + return err + } + for _, p := range reply.Proposals { + err = verifyProposal(p, vr.PubKey) + if err != nil { + return fmt.Errorf("unable to verify proposal %v: %v", + p.CensorshipRecord.Token, err) + } + } return nil } diff --git a/politeiawww/cmd/piwww/testrun.go b/politeiawww/cmd/piwww/testrun.go index 1e67d2fe5..b75ccd3ce 100644 --- a/politeiawww/cmd/piwww/testrun.go +++ b/politeiawww/cmd/piwww/testrun.go @@ -244,7 +244,7 @@ func (cmd *testRunCmd) Execute(args []string) error { // Purchase proposal credits fmt.Printf(" Proposal paywall details\n") - ppdr, err := client.ProposalPaywallDetails() + ppdr, err := client.UserProposalPaywall() if err != nil { return err } diff --git a/politeiawww/cmd/piwww/proposalpaywall.go b/politeiawww/cmd/piwww/userproposalpaywall.go similarity index 50% rename from politeiawww/cmd/piwww/proposalpaywall.go rename to politeiawww/cmd/piwww/userproposalpaywall.go index 0133f9d87..49c3229ea 100644 --- a/politeiawww/cmd/piwww/proposalpaywall.go +++ b/politeiawww/cmd/piwww/userproposalpaywall.go @@ -6,21 +6,21 @@ package main import "github.com/decred/politeia/politeiawww/cmd/shared" -// proposalPaywallCmd gets paywall info for the logged in user. -type proposalPaywallCmd struct{} +// userProposalPaywallCmd gets paywall info for the logged in user. +type userProposalPaywallCmd struct{} // Execute executes the proposal paywall command. -func (cmd *proposalPaywallCmd) Execute(args []string) error { - ppdr, err := client.ProposalPaywallDetails() +func (cmd *userProposalPaywallCmd) Execute(args []string) error { + ppdr, err := client.UserProposalPaywall() if err != nil { return err } return shared.PrintJSON(ppdr) } -// proposalPaywallHelpMsg is the output of the help command when -// 'proposalpaywall' is specified. -const proposalPaywallHelpMsg = `proposalpaywall +// userProposalPaywallHelpMsg is the output of the help command when +// 'userproposalpaywall' is specified. +const userProposalPaywallHelpMsg = `userproposalpaywall Fetch proposal paywall details. diff --git a/politeiawww/cmd/piwww/util.go b/politeiawww/cmd/piwww/util.go index 52f023b6e..3717f982b 100644 --- a/politeiawww/cmd/piwww/util.go +++ b/politeiawww/cmd/piwww/util.go @@ -5,6 +5,7 @@ package main import ( + "bytes" "encoding/base64" "encoding/hex" "encoding/json" @@ -14,6 +15,7 @@ import ( "github.com/decred/politeia/politeiad/api/v1/identity" pi "github.com/decred/politeia/politeiawww/api/pi/v1" utilwww "github.com/decred/politeia/politeiawww/util" + "github.com/decred/politeia/util" ) // signedMerkleRoot calculates the merkle root of the passed in list of files @@ -97,3 +99,98 @@ func decodeProposalMetadata(metadata []pi.Metadata) (*pi.ProposalMetadata, error } return pm, nil } + +// verifyDigests verifies if the list of files and metadatas have valid +// digests. It compares digests that came with the file/metadata with +// digests calculated from their payload. +func verifyDigests(files []pi.File, md []pi.Metadata) error { + // Validate file digests + for _, f := range files { + b, err := base64.StdEncoding.DecodeString(f.Payload) + if err != nil { + return fmt.Errorf("file: %v decode payload err %v", + f.Name, err) + } + digest := util.Digest(b) + d, ok := util.ConvertDigest(f.Digest) + if !ok { + return fmt.Errorf("file: %v invalid digest %v", + f.Name, f.Digest) + } + if !bytes.Equal(digest, d[:]) { + return fmt.Errorf("file: %v digests do not match", + f.Name) + } + } + + // Validate metadata digests + for _, v := range md { + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return fmt.Errorf("metadata: %v decode payload err %v", + v.Hint, err) + } + digest := util.Digest(b) + d, ok := util.ConvertDigest(v.Digest) + if !ok { + return fmt.Errorf("metadata: %v invalid digest %v", + v.Hint, v.Digest) + } + if !bytes.Equal(digest, d[:]) { + return fmt.Errorf("metadata: %v digests do not match metadata", + v.Hint) + } + } + + return nil +} + +// verifyProposal verifies the merkle root, author signature and censorship +// record of a given proposal. +func verifyProposal(p pi.ProposalRecord, serverPubKey string) error { + if len(p.Files) > 0 { + // Verify digests + err := verifyDigests(p.Files, p.Metadata) + if err != nil { + return err + } + // Verify merkle root + mr, err := utilwww.MerkleRoot(p.Files, p.Metadata) + if err != nil { + return err + } + // Check if merkle roots match + if mr != p.CensorshipRecord.Merkle { + return fmt.Errorf("merkle roots do not match") + } + } + + // Verify proposal signature + pid, err := util.IdentityFromString(p.PublicKey) + if err != nil { + return err + } + sig, err := util.ConvertSignature(p.Signature) + if err != nil { + return err + } + if !pid.VerifyMessage([]byte(p.CensorshipRecord.Merkle), sig) { + return fmt.Errorf("invalid proposal signature") + } + + // Verify censorship record signature + id, err := util.IdentityFromString(serverPubKey) + if err != nil { + return err + } + s, err := util.ConvertSignature(p.CensorshipRecord.Signature) + if err != nil { + return err + } + msg := []byte(p.CensorshipRecord.Merkle + p.CensorshipRecord.Token) + if !id.VerifyMessage(msg, s) { + return fmt.Errorf("invalid censorship record signature") + } + + return nil +} diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index 7b27f4059..2b966b747 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -681,9 +681,9 @@ func (c *Client) VerifyResetPassword(vrp www.VerifyResetPassword) (*www.VerifyRe return &reply, nil } -// ProposalPaywallDetails retrieves proposal credit paywall information for the +// UserProposalPaywall retrieves proposal credit paywall information for the // logged in user. -func (c *Client) ProposalPaywallDetails() (*www.UserProposalPaywallReply, error) { +func (c *Client) UserProposalPaywall() (*www.UserProposalPaywallReply, error) { responseBody, err := c.makeRequest(http.MethodGet, www.PoliteiaWWWAPIRoute, www.RouteUserProposalPaywall, nil) if err != nil { From 8d72b34d966febd360f5064ef50e4faabe916108 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 6 Oct 2020 16:00:18 -0500 Subject: [PATCH 136/449] various record and proposal bug fixes --- plugins/pi/pi.go | 2 - politeiad/api/v1/v1.go | 3 +- politeiad/backend/tlogbe/pi.go | 15 +- politeiad/backend/tlogbe/ticketvote.go | 3 + politeiad/backend/tlogbe/tlog.go | 144 ++++++++---------- politeiad/backend/tlogbe/tlogbe.go | 55 +++++-- politeiad/cmd/politeia/README.md | 10 +- politeiad/politeiad.go | 15 +- politeiawww/api/pi/v1/v1.go | 64 +++++++- politeiawww/cmd/piwww/README.md | 8 +- politeiawww/cmd/piwww/help.go | 4 +- ...posalsetstatus.go => proposalstatusset.go} | 50 +++--- politeiawww/cmd/piwww/util.go | 1 + politeiawww/cmd/shared/client.go | 10 +- politeiawww/piwww.go | 64 ++++---- politeiawww/www.go | 27 ++-- util/signature.go | 3 +- 17 files changed, 277 insertions(+), 201 deletions(-) rename politeiawww/cmd/piwww/{proposalsetstatus.go => proposalstatusset.go} (66%) diff --git a/plugins/pi/pi.go b/plugins/pi/pi.go index 240e2f9e1..f988ed638 100644 --- a/plugins/pi/pi.go +++ b/plugins/pi/pi.go @@ -64,7 +64,6 @@ const ( ErrorStatusInvalid ErrorStatusT = iota ErrorStatusPropStateInvalid ErrorStatusPropVersionInvalid - ErrorStatusPropStatusInvalid ErrorStatusPropStatusChangeInvalid ErrorStatusPropLinkToInvalid ErrorStatusVoteStatusInvalid @@ -92,7 +91,6 @@ var ( ErrorStatus = map[ErrorStatusT]string{ ErrorStatusInvalid: "error status invalid", ErrorStatusPropLinkToInvalid: "proposal link to invalid", - ErrorStatusPropStatusInvalid: "proposal status invalid", ErrorStatusVoteStatusInvalid: "vote status invalid", } ) diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index 4a9ae63d2..e3063efa5 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -65,10 +65,11 @@ const ( ErrorStatusNoChanges ErrorStatusT = 14 ErrorStatusRecordFound ErrorStatusT = 15 ErrorStatusInvalidRPCCredentials ErrorStatusT = 16 + ErrorStatusRecordNotFound ErrorStatusT = 17 // Record status codes (set and get) RecordStatusInvalid RecordStatusT = 0 // Invalid status - RecordStatusNotFound RecordStatusT = 1 // Record not found + RecordStatusNotFound RecordStatusT = 1 // Record not found (deprecated) RecordStatusNotReviewed RecordStatusT = 2 // Record has not been reviewed RecordStatusCensored RecordStatusT = 3 // Record has been censored RecordStatusPublic RecordStatusT = 4 // Record is publicly visible diff --git a/politeiad/backend/tlogbe/pi.go b/politeiad/backend/tlogbe/pi.go index ab11945eb..a4e0d974c 100644 --- a/politeiad/backend/tlogbe/pi.go +++ b/politeiad/backend/tlogbe/pi.go @@ -310,9 +310,6 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { return err } - // TODO verify ProposalMetadata signature. This is already done in - // www but we should do it here anyway since its plugin data. - // Decode ProposalMetadata var pm *pi.ProposalMetadata for _, v := range nr.Files { @@ -431,18 +428,10 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { // TODO verify files were changed. Before adding this, verify that // politeiad will also error if no files were changed. - // Verify proposal status - status := convertPropStatusFromMDStatus(er.Current.RecordMetadata.Status) - if status != pi.PropStatusUnvetted && status != pi.PropStatusPublic { - return backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropStatusInvalid), - } - } - // Verify that the linkto has not changed. This only applies to // public proposal. Unvetted proposals are allowed to change their // linkto. + status := convertPropStatusFromMDStatus(er.Current.RecordMetadata.Status) if status == pi.PropStatusPublic { pmCurr, err := proposalMetadataFromFiles(er.Current.Files) if err != nil { @@ -461,6 +450,8 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { } } + // TODO verify linkto is allowed + // Verify vote status. This is only required for public proposals. if status == pi.PropStatusPublic { token := er.RecordMetadata.Token diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index ddd964634..e003c925d 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -63,6 +63,9 @@ var ( // is to add an exists() method onto the tlogClient and have the mutexes // function ensure a token is valid before holding the lock on it. +// TODO the bottleneck for casting a large ballot of votes is waiting for the +// log signer. Break the cast votes up and send them concurrently. + // ticketVotePlugin satisfies the pluginClient interface. type ticketVotePlugin struct { sync.Mutex diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index 0a82aef57..579d19e7b 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -452,61 +452,12 @@ func (t *tlog) treeExists(treeID int64) bool { func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, fr freezeRecord) error { log.Tracef("tlog treeFreeze: %v") - // Get tree leaves - leavesAll, err := t.trillian.leavesAll(treeID) - if err != nil { - return fmt.Errorf("leavesAll: %v", err) - } - - // Prepare kv store blobs for metadata - bpr, err := recordBlobsPrepare(leavesAll, rm, metadata, - []backend.File{}, t.encryptionKey) + // Save metadata + idx, err := t.metadataSave(treeID, rm, metadata) if err != nil { return err } - // Ensure metadata has been changed - if len(bpr.blobs) == 0 { - return errNoMetadataChanges - } - - // Save metadata blobs - idx, err := t.recordBlobsSave(treeID, *bpr) - if err != nil { - return fmt.Errorf("blobsSave: %v", err) - } - - // Get the existing record index and add the unchanged fields to - // the new record index. The version and files will remain the - // same. - oldIdx, err := t.recordIndexLatest(leavesAll) - if err != nil { - return fmt.Errorf("recordIndexLatest: %v", err) - } - idx.Version = oldIdx.Version - idx.Files = oldIdx.Files - - // Increment the iteration - idx.Iteration = oldIdx.Iteration + 1 - - // Sanity check - switch { - case idx.Version != oldIdx.Version: - return fmt.Errorf("invalid index version: got %v, want %v", - idx.Version, oldIdx.Version) - case idx.Iteration != oldIdx.Iteration+1: - return fmt.Errorf("invalid index iteration: got %v, want %v", - idx.Iteration, oldIdx.Iteration+1) - case idx.RecordMetadata == nil: - return fmt.Errorf("invalid index record metadata") - case len(idx.Metadata) != len(metadata): - return fmt.Errorf("invalid index metadata: got %v, want %v", - len(idx.Metadata), len(metadata)) - case len(idx.Files) != len(oldIdx.Files): - return fmt.Errorf("invalid index files: got %v, want %v", - len(idx.Files), len(oldIdx.Files)) - } - // Blobify the record index blobs := make([][]byte, 0, 2) be, err := convertBlobEntryFromRecordIndex(*idx) @@ -544,8 +495,7 @@ func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []ba return fmt.Errorf("store Put: %v", err) } if len(keys) != 2 { - return fmt.Errorf("wrong number of keys: got %v, want 1", - len(keys)) + return fmt.Errorf("wrong number of keys: got %v, want 1", len(keys)) } // Append record index and freeze record leaves to trillian tree @@ -856,6 +806,26 @@ func recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backend.RecordMe return nil, errTreeIsFrozen } + // Verify there are no duplicate mdstream IDs + mdstreamIDs := make(map[uint64]struct{}, len(metadata)) + for _, v := range metadata { + _, ok := mdstreamIDs[v.ID] + if ok { + return nil, fmt.Errorf("duplicate metadata stream ID: %v", v.ID) + } + mdstreamIDs[v.ID] = struct{}{} + } + + // Verify there are no duplicate filenames + filenames := make(map[string]struct{}, len(files)) + for _, v := range files { + _, ok := filenames[v.Name] + if ok { + return nil, fmt.Errorf("duplicate filename found: %v", v.Name) + } + filenames[v.Name] = struct{}{} + } + // Check if any of the content already exists. Different record // versions that reference the same data is fine, but this data // should not be saved to the store again. We can find duplicates @@ -865,8 +835,8 @@ func recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backend.RecordMe // Compute record content hashes rhashes := recordHashes{ - metadata: make(map[string]uint64, len(metadata)), - files: make(map[string]string, len(files)), + metadata: make(map[string]uint64, len(metadata)), // [hash]metadataID + files: make(map[string]string, len(files)), // [hash]filename } b, err := json.Marshal(recordMD) if err != nil { @@ -1221,49 +1191,47 @@ func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []ba return nil } -// recordMetadataSave saves the provided metadata to tlog, creating a new -// iteration of the record while keeping the record version the same. Once the -// metadata has been successfully saved to tlog, a recordIndex is created for -// this iteration of the record and saved to tlog as well. -func (t *tlog) recordMetadataSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { - log.Tracef("tlog recordMetadataSave: %v", treeID) - +// metadataSave saves the provided metadata to the kv store and trillian tree. +// The record index for this iteration of the record is returned. This is step +// one of a two step process. The record update will not be considered +// succesful until the returned record index is also saved to the kv store +// and trillian tree. This code has been pulled out so that it can be called +// during normal metadata updates as well as when an update requires a freeze +// record to be saved along with the record index, such as when a record is +// censored. +func (t *tlog) metadataSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) (*recordIndex, error) { // Verify tree exists if !t.treeExists(treeID) { - return errRecordNotFound + return nil, errRecordNotFound } // Get tree leaves leavesAll, err := t.trillian.leavesAll(treeID) if err != nil { - return fmt.Errorf("leavesAll: %v", err) + return nil, fmt.Errorf("leavesAll: %v", err) } // Verify tree state if treeIsFrozen(leavesAll) { - return errTreeIsFrozen + return nil, errTreeIsFrozen } // Prepare kv store blobs bpr, err := recordBlobsPrepare(leavesAll, rm, metadata, []backend.File{}, t.encryptionKey) if err != nil { - return err + return nil, err } - // Verify changes are being made. The record index returned from - // recordBlobsPrepare() will contain duplicate blobs, i.e. metadata - // blobs that already existed prior to this update. If the record - // index already contains all of the metadata blobs it means that - // no metadata changes are being made. - if len(metadata) == len(bpr.recordIndex.Metadata) { - return errNoMetadataChanges + // Verify at least one new blob is being saved to the kv store + if len(bpr.blobs) == 0 { + return nil, errNoMetadataChanges } // Save the blobs idx, err := t.recordBlobsSave(treeID, *bpr) if err != nil { - return fmt.Errorf("blobsSave: %v", err) + return nil, fmt.Errorf("blobsSave: %v", err) } // Get the existing record index and add the unchanged fields to @@ -1271,7 +1239,7 @@ func (t *tlog) recordMetadataSave(treeID int64, rm backend.RecordMetadata, metad // same. oldIdx, err := t.recordIndexLatest(leavesAll) if err != nil { - return fmt.Errorf("recordIndexLatest: %v", err) + return nil, fmt.Errorf("recordIndexLatest: %v", err) } idx.Version = oldIdx.Version idx.Files = oldIdx.Files @@ -1282,21 +1250,37 @@ func (t *tlog) recordMetadataSave(treeID int64, rm backend.RecordMetadata, metad // Sanity check switch { case idx.Version != oldIdx.Version: - return fmt.Errorf("invalid index version: got %v, want %v", + return nil, fmt.Errorf("invalid index version: got %v, want %v", idx.Version, oldIdx.Version) case idx.Iteration != oldIdx.Iteration+1: - return fmt.Errorf("invalid index iteration: got %v, want %v", + return nil, fmt.Errorf("invalid index iteration: got %v, want %v", idx.Iteration, oldIdx.Iteration+1) case idx.RecordMetadata == nil: - return fmt.Errorf("invalid index record metadata") + return nil, fmt.Errorf("invalid index record metadata") case len(idx.Metadata) != len(metadata): - return fmt.Errorf("invalid index metadata: got %v, want %v", + return nil, fmt.Errorf("invalid index metadata: got %v, want %v", len(idx.Metadata), len(metadata)) case len(idx.Files) != len(oldIdx.Files): - return fmt.Errorf("invalid index files: got %v, want %v", + return nil, fmt.Errorf("invalid index files: got %v, want %v", len(idx.Files), len(oldIdx.Files)) } + return idx, nil +} + +// recordMetadataSave saves the provided metadata to tlog, creating a new +// iteration of the record while keeping the record version the same. Once the +// metadata has been successfully saved to tlog, a recordIndex is created for +// this iteration of the record and saved to tlog as well. +func (t *tlog) recordMetadataSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { + log.Tracef("tlog recordMetadataSave: %v", treeID) + + // Save metadata + idx, err := t.metadataSave(treeID, rm, metadata) + if err != nil { + return err + } + // Save record index err = t.recordIndexSave(treeID, *idx) if err != nil { diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 63f59cd18..c629a7316 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -478,21 +478,30 @@ func recordMetadataNew(token []byte, files []backend.File, status backend.MDStat // TODO test this function func filesUpdate(filesCurr, filesAdd []backend.File, filesDel []string) []backend.File { + // Put current files into a map + curr := make(map[string]backend.File, len(filesCurr)) // [filename]File + for _, v := range filesCurr { + curr[v.Name] = v + } + // Apply deletes - del := make(map[string]struct{}, len(filesDel)) for _, fn := range filesDel { - del[fn] = struct{}{} - } - f := make([]backend.File, 0, len(filesCurr)+len(filesAdd)) - for _, v := range filesCurr { - if _, ok := del[v.Name]; ok { - continue + _, ok := curr[fn] + if ok { + delete(curr, fn) } - f = append(f, v) } // Apply adds - f = append(f, filesAdd...) + for _, v := range filesAdd { + curr[v.Name] = v + } + + // Convert back to a slice + f := make([]backend.File, 0, len(curr)) + for _, v := range curr { + f = append(f, v) + } return f } @@ -1123,7 +1132,8 @@ func (t *tlogBackend) unvettedPublish(token []byte, rm backend.RecordMetadata, m return fmt.Errorf("vetted recordSave: %v", err) } - log.Debugf("Unvetted record %x copied to vetted", token) + log.Debugf("Unvetted record %x copied to vetted tree %v", + token, vettedTreeID) // Freeze the unvetted tree fr := freezeRecord{ @@ -1135,7 +1145,7 @@ func (t *tlogBackend) unvettedPublish(token []byte, rm backend.RecordMetadata, m return fmt.Errorf("treeFreeze %v: %v", treeID, err) } - log.Debugf("Unvetted record %x frozen", token) + log.Debugf("Unvetted record frozen %x", token) // Update the vetted cache t.vettedTreeIDAdd(hex.EncodeToString(token), vettedTreeID) @@ -1266,18 +1276,25 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, // This function must be called WITH the vetted lock held. func (t *tlogBackend) vettedCensor(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { // Freeze the tree - treeID := treeIDFromToken(token) + treeID, ok := t.vettedTreeID(token) + if !ok { + return fmt.Errorf("vetted record not found") + } err := t.vetted.treeFreeze(treeID, rm, metadata, freezeRecord{}) if err != nil { return fmt.Errorf("treeFreeze %v: %v", treeID, err) } + log.Debugf("Vetted record frozen %v", token) + // Delete all record files err = t.vetted.recordDel(treeID) if err != nil { return fmt.Errorf("recordDel %v: %v", treeID, err) } + log.Debugf("Vetted record files deleted %v", token) + return nil } @@ -1285,8 +1302,18 @@ func (t *tlogBackend) vettedCensor(token []byte, rm backend.RecordMetadata, meta func (t *tlogBackend) vettedArchive(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { // Freeze the tree. Nothing else needs to be done for an archived // record. - treeID := treeIDFromToken(token) - return t.vetted.treeFreeze(treeID, rm, metadata, freezeRecord{}) + treeID, ok := t.vettedTreeID(token) + if !ok { + return fmt.Errorf("vetted record not found") + } + err := t.vetted.treeFreeze(treeID, rm, metadata, freezeRecord{}) + if err != nil { + return fmt.Errorf("treeFreeze %v: %v", treeID, err) + } + + log.Debugf("Vetted record frozen %v", token) + + return nil } // This function satisfies the Backend interface. diff --git a/politeiad/cmd/politeia/README.md b/politeiad/cmd/politeia/README.md index d9f70256b..17c5c6068 100644 --- a/politeiad/cmd/politeia/README.md +++ b/politeiad/cmd/politeia/README.md @@ -114,7 +114,7 @@ $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass updateunve 'appendmetadata12:{"foo":"bar"}' 'overwritemetadata2:{"12foo":"12bar"}' \ token:0e4a82a370228b710000 -Update record: 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 +Update record: 0e4a82a370228b710000 Metadata overwrite: 2 Metadata append : 12 ``` @@ -202,7 +202,7 @@ $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass updatevett 'appendmetadata12:{"foo":"bar"}' 'overwritemetadata2:{"12foo":"12bar"}' \ token:0e4a82a370228b710000 -Update record: 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 +Update record: 0e4a82a370228b710000 Metadata overwrite: 2 Metadata append : 12 ``` @@ -218,15 +218,15 @@ Note `token:` is not prefixed to the token in this command. Status change validation is done in the backend. ``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass setvettedstatus censored 72fe14a914783eafb78adcbcd405e723c3f55ff475043b0d89b2cf71ffc6a2d4 'overwritemetadata12:"zap"' +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass setvettedstatus censored 0e4a82a370228b710000 'overwritemetadata12:"zap"' Set record status: Status: censored ``` -## Inventory by status +## Inventory The `inventory` command retrieves the censorship record tokens from all records, -separated by their status. +categorized by their record status. ``` $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass inventory diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 457edba13..b0694941e 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -378,8 +378,7 @@ func (p *politeia) updateRecord(w http.ResponseWriter, r *http.Request, vetted b if errors.Is(err, backend.ErrRecordNotFound) { log.Infof("%v update %v record not found: %x", remoteAddr(r), cmd, token) - p.respondWithUserError(w, v1.ErrorStatusRecordFound, - nil) + p.respondWithUserError(w, v1.ErrorStatusRecordNotFound, nil) return } if errors.Is(err, backend.ErrNoChanges) { @@ -466,9 +465,10 @@ func (p *politeia) getUnvetted(w http.ResponseWriter, r *http.Request) { switch { case errors.Is(err, backend.ErrRecordNotFound): // Record not found - reply.Record.Status = v1.RecordStatusNotFound log.Infof("Get unvetted record %v: token %v not found", remoteAddr(r), t.Token) + p.respondWithUserError(w, v1.ErrorStatusRecordNotFound, nil) + return case err != nil: // Generic internal error @@ -539,9 +539,10 @@ func (p *politeia) getVetted(w http.ResponseWriter, r *http.Request) { switch { case errors.Is(err, backend.ErrRecordNotFound): // Record not found - reply.Record.Status = v1.RecordStatusNotFound log.Infof("Get vetted record %v: token %v not found", remoteAddr(r), t.Token) + p.respondWithUserError(w, v1.ErrorStatusRecordNotFound, nil) + return case err != nil: // Generic internal error @@ -726,8 +727,7 @@ func (p *politeia) setVettedStatus(w http.ResponseWriter, r *http.Request) { if errors.Is(err, backend.ErrRecordNotFound) { log.Infof("%v updateStatus record not "+ "found: %x", remoteAddr(r), token) - p.respondWithUserError(w, v1.ErrorStatusRecordFound, - nil) + p.respondWithUserError(w, v1.ErrorStatusRecordNotFound, nil) return } var serr backend.StateTransitionError @@ -799,8 +799,7 @@ func (p *politeia) setUnvettedStatus(w http.ResponseWriter, r *http.Request) { if errors.Is(err, backend.ErrRecordNotFound) { log.Infof("%v updateUnvettedStatus record not "+ "found: %x", remoteAddr(r), token) - p.respondWithUserError(w, v1.ErrorStatusRecordFound, - nil) + p.respondWithUserError(w, v1.ErrorStatusRecordNotFound, nil) return } var serr backend.StateTransitionError diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 949e97eff..63efa80dd 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -37,7 +37,7 @@ const ( // Proposal routes RouteProposalNew = "/proposal/new" RouteProposalEdit = "/proposal/edit" - RouteProposalSetStatus = "/proposal/setstatus" + RouteProposalStatusSet = "/proposal/setstatus" RouteProposals = "/proposals" RouteProposalInventory = "/proposals/inventory" @@ -188,7 +188,59 @@ var ( // ErrorStatus contains human readable error messages. // TODO fill in error status messages ErrorStatus = map[ErrorStatusT]string{ - ErrorStatusInvalid: "error status invalid", + ErrorStatusInvalid: "error status invalid", + ErrorStatusInputInvalid: "input invalid", + + // User errors + ErrorStatusUserRegistrationNotPaid: "user registration not paid", + ErrorStatusUserBalanceInsufficient: "user balance insufficient", + ErrorStatusUserIsNotAuthor: "user is not author", + ErrorStatusUserIsNotAdmin: "user is not author", + + // Signature errors + ErrorStatusPublicKeyInvalid: "public key invalid", + ErrorStatusSignatureInvalid: "signature invalid", + + // Proposal errors + ErrorStatusFileCountInvalid: "file count invalid", + ErrorStatusFileNameInvalid: "file name invalid", + ErrorStatusFileMIMEInvalid: "file mime invalid", + ErrorStatusFileDigestInvalid: "file digest invalid", + ErrorStatusFilePayloadInvalid: "file payload invalid", + ErrorStatusIndexFileNameInvalid: "index filename invalid", + ErrorStatusIndexFileCountInvalid: "index file count invalid", + ErrorStatusIndexFileSizeInvalid: "index file size invalid", + ErrorStatusTextFileCountInvalid: "text file count invalid", + ErrorStatusImageFileCountInvalid: "file count invalid", + ErrorStatusImageFileSizeInvalid: "file size invalid", + ErrorStatusMetadataCountInvalid: "metadata count invalid", + ErrorStatusMetadataDigestInvalid: "metadata digest invalid", + ErrorStatusMetadataPayloadInvalid: "metadata pyaload invalid", + ErrorStatusPropMetadataNotFound: "proposal metadata not found", + ErrorStatusPropNameInvalid: "proposal name invalid", + ErrorStatusPropLinkToInvalid: "proposal link to invalid", + ErrorStatusPropLinkByInvalid: "proposal link by invalid", + ErrorStatusPropTokenInvalid: "proposal token invalid", + ErrorStatusPropNotFound: "proposal not found", + ErrorStatusPropStateInvalid: "proposal state invalid", + ErrorStatusPropStatusInvalid: "proposal status invalid", + ErrorStatusPropStatusChangeInvalid: "proposal status change invalid", + ErrorStatusPropStatusChangeReasonInvalid: "proposal status reason invalid", + ErrorStatusPropPageSizeExceeded: "proposal page size exceeded", + ErrorStatusNoPropChanges: "no proposal changes", + + // Comment errors + ErrorStatusCommentTextInvalid: "comment text invalid", + ErrorStatusCommentParentIDInvalid: "comment parent ID invalid", + ErrorStatusCommentVoteInvalid: "comment vote invalid", + ErrorStatusCommentNotFound: "comment not found", + ErrorStatusCommentVoteChangesMax: "comment vote changes exceeded max", + + // Vote errors + ErrorStatusVoteStatusInvalid: "vote status invalid", + ErrorStatusVoteParamsInvalid: "vote params invalid", + ErrorStatusBallotInvalid: "ballot invalid", + ErrorStatusVotePageSizeExceeded: "vote page size exceeded", } ) @@ -353,11 +405,11 @@ type ProposalEditReply struct { CensorshipRecord CensorshipRecord `json:"censorshiprecord"` } -// ProposalSetStatus sets the status of a proposal. Some status changes require +// ProposalStatusSet sets the status of a proposal. Some status changes require // a reason to be included. // // Signature is the client signature of the Token+Version+Status+Reason. -type ProposalSetStatus struct { +type ProposalStatusSet struct { Token string `json:"token"` // Censorship token State PropStateT `json:"state"` // Proposal state Version string `json:"version"` // Proposal version @@ -367,8 +419,8 @@ type ProposalSetStatus struct { Signature string `json:"signature"` // Client signature } -// ProposalSetStatusReply is the reply to the ProposalSetStatus command. -type ProposalSetStatusReply struct { +// ProposalStatusSetReply is the reply to the ProposalStatusSet command. +type ProposalStatusSetReply struct { Timestamp int64 `json:"timestamp"` } diff --git a/politeiawww/cmd/piwww/README.md b/politeiawww/cmd/piwww/README.md index 762c99cce..b238c3653 100644 --- a/politeiawww/cmd/piwww/README.md +++ b/politeiawww/cmd/piwww/README.md @@ -60,7 +60,7 @@ host=https://127.0.0.1:4443 skipverify=true ``` -## Usage +## Example Usage ### Create a new user @@ -165,3 +165,9 @@ casts a ballot of votes. This will only work on testnet and if you are running your dcrwallet locally using the default port. $ piwww voteballot [token] [voteID] + +## Reference implementation + +The piwww `testrun` command runs a series of tests on all of the politeiawww pi +API routes. This command can be used as a reference implementation for the pi +API. diff --git a/politeiawww/cmd/piwww/help.go b/politeiawww/cmd/piwww/help.go index 053f2c4fb..4e46ba92a 100644 --- a/politeiawww/cmd/piwww/help.go +++ b/politeiawww/cmd/piwww/help.go @@ -42,8 +42,8 @@ func (cmd *helpCmd) Execute(args []string) error { fmt.Printf("%s\n", proposalNewHelpMsg) case "proposaledit": fmt.Printf("%s\n", proposalEditHelpMsg) - case "proposalsetstatus": - fmt.Printf("%s\n", proposalSetStatusHelpMsg) + case "proposalstatusset": + fmt.Printf("%s\n", proposalStatusSetHelpMsg) case "proposals": fmt.Printf("%s\n", proposalsHelpMsg) case "proposalinventory": diff --git a/politeiawww/cmd/piwww/proposalsetstatus.go b/politeiawww/cmd/piwww/proposalstatusset.go similarity index 66% rename from politeiawww/cmd/piwww/proposalsetstatus.go rename to politeiawww/cmd/piwww/proposalstatusset.go index 69c973cd2..a3f396adf 100644 --- a/politeiawww/cmd/piwww/proposalsetstatus.go +++ b/politeiawww/cmd/piwww/proposalstatusset.go @@ -16,9 +16,10 @@ import ( // proposalStatusSetCmd sets the status of a proposal. type proposalStatusSetCmd struct { Args struct { - Token string `positional-arg-name:"token" required:"true"` - Status string `positional-arg-name:"status" required:"true"` - Reason string `positional-arg-name:"reason"` + Token string `positional-arg-name:"token" required:"true"` + Status string `positional-arg-name:"status" required:"true"` + Reason string `positional-arg-name:"reason"` + Version string `positional-arg-name:"version"` } `positional-args:"true"` Unvetted bool `long:"unvetted" optional:"true"` @@ -58,27 +59,30 @@ func (cmd *proposalStatusSetCmd) Execute(args []string) error { // Human readable status code found status = s } else { - return fmt.Errorf("Invalid proposal status '%v'. Valid statuses are:\n"+ - " public make a proposal public\n"+ - " censored censor a proposal\n"+ - " abandoned declare a public proposal abandoned", - cmd.Args.Status) + return fmt.Errorf("Invalid proposal status '%v'\n %v", + cmd.Args.Status, proposalStatusSetHelpMsg) } - // Get the proposal. The latest proposal version number is needed - // for the set status request. - pr, err := proposalRecordLatest(state, cmd.Args.Token) - if err != nil { - return err + // Verify version + var version string + if cmd.Args.Version != "" { + version = cmd.Args.Version + } else { + // Get the version manually + pr, err := proposalRecordLatest(state, cmd.Args.Token) + if err != nil { + return err + } + version = pr.Version } // Setup request - msg := cmd.Args.Token + pr.Version + cmd.Args.Status + cmd.Args.Reason + msg := cmd.Args.Token + version + strconv.Itoa(int(status)) + cmd.Args.Reason sig := cfg.Identity.SignMessage([]byte(msg)) - pss := pi.ProposalSetStatus{ + pss := pi.ProposalStatusSet{ Token: cmd.Args.Token, State: state, - Version: pr.Version, + Version: version, Status: status, Reason: cmd.Args.Reason, Signature: hex.EncodeToString(sig[:]), @@ -91,7 +95,7 @@ func (cmd *proposalStatusSetCmd) Execute(args []string) error { if err != nil { return err } - pssr, err := client.ProposalSetStatus(pss) + pssr, err := client.ProposalStatusSet(pss) if err != nil { return err } @@ -103,8 +107,8 @@ func (cmd *proposalStatusSetCmd) Execute(args []string) error { return nil } -// proposalSetStatusHelpMsg is the output of the help command. -const proposalSetStatusHelpMsg = `proposalsetstatus "token" "status" "reason" +// proposalStatusSetHelpMsg is the output of the help command. +const proposalStatusSetHelpMsg = `proposalstatusset "token" "status" "reason" Set the status of a proposal. This command assumes the proposal is a vetted record. If the proposal is unvetted, the --unvetted flag must be used. Requires @@ -116,9 +120,11 @@ Valid statuses: abandoned Arguments: -1. token (string, required) Proposal censorship token -2. status (string, required) New status -3. message (string, optional) Status change message +1. token (string, required) Proposal censorship token +2. status (string, required) New status +3. message (string, optional) Status change message +4. version (string, optional) Proposal version. This will be fetched manually + if one is not provided. Flags: --unvetted (bool, optional) Set status of an unvetted record. diff --git a/politeiawww/cmd/piwww/util.go b/politeiawww/cmd/piwww/util.go index 3717f982b..eb14e48de 100644 --- a/politeiawww/cmd/piwww/util.go +++ b/politeiawww/cmd/piwww/util.go @@ -88,6 +88,7 @@ func decodeProposalMetadata(metadata []pi.Metadata) (*pi.ProposalMetadata, error if err != nil { return nil, err } + pm = &pi.ProposalMetadata{} err = json.Unmarshal(b, pm) if err != nil { return nil, err diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index 2b966b747..c34a98cf5 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -754,18 +754,18 @@ func (c *Client) ProposalEdit(pe pi.ProposalEdit) (*pi.ProposalEditReply, error) return &per, nil } -// ProposalSetStatus sets the status of a proposal -func (c *Client) ProposalSetStatus(pss pi.ProposalSetStatus) (*pi.ProposalSetStatusReply, error) { +// ProposalStatusSet sets the status of a proposal +func (c *Client) ProposalStatusSet(pss pi.ProposalStatusSet) (*pi.ProposalStatusSetReply, error) { responseBody, err := c.makeRequest(http.MethodPost, - pi.APIRoute, pi.RouteProposalSetStatus, pss) + pi.APIRoute, pi.RouteProposalStatusSet, pss) if err != nil { return nil, err } - var pssr pi.ProposalSetStatusReply + var pssr pi.ProposalStatusSetReply err = json.Unmarshal(responseBody, &pssr) if err != nil { - return nil, fmt.Errorf("unmarshal ProposalSetStatusReply: %v", err) + return nil, fmt.Errorf("unmarshal ProposalStatusSetReply: %v", err) } if c.cfg.Verbose { diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 0ed04e222..faa426829 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -732,13 +732,13 @@ func (p *politeiawww) proposalRecords(state pi.PropStateT, reqs []pi.ProposalReq // Unvetted politeiad record r, err = p.getUnvetted(v.Token, v.Version) if err != nil { - return nil, fmt.Errorf("getUnvetted %v: %v", v.Token, err) + return nil, err } case pi.PropStateVetted: // Vetted politeiad record r, err = p.getVetted(v.Token, v.Version) if err != nil { - return nil, fmt.Errorf("getVetted %v: %v", v.Token, err) + return nil, err } default: return nil, fmt.Errorf("unknown state %v", state) @@ -751,8 +751,7 @@ func (p *politeiawww) proposalRecords(state pi.PropStateT, reqs []pi.ProposalReq pr, err := convertProposalRecordFromPD(*r) if err != nil { - return nil, fmt.Errorf("convertProposalRecordFromPD %v: %v", - v.Token, err) + return nil, err } // Remove files if specified. The Metadata objects will still be @@ -1168,9 +1167,8 @@ func (p *politeiawww) processProposalNew(pn pi.ProposalNew, usr user.User) (*pi. } // Setup politeiad files. The Metadata objects are converted to - // politeiad files instead of metadata streams since they contain - // user defined data that needs to be included in the merkle root - // that politeiad signs. + // politeiad files since they contain user defined data that needs + // to be included in the merkle root that politeiad signs. files := convertFilesFromPi(pn.Files) for _, v := range pn.Metadata { switch v.Hint { @@ -1304,16 +1302,25 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p } } - // Verification that requires retrieving the existing proposal is - // done in the politeiad pi plugin hook. This includes: - // -Verify proposal status - // -Verify vote status + // Verify proposal status + switch curr.Status { + case pi.PropStatusUnvetted, pi.PropStatusPublic: + // Allowed; continue + default: + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusPropStatusInvalid, + } + } + + // Verification that requires plugin data or querying additional + // proposal data is done in the politeiad pi plugin hook. This + // includes: // -Verify linkto + // -Verify vote status // Setup politeiad files. The Metadata objects are converted to - // politeiad files instead of metadata streams since they contain - // user defined data that needs to be included in the merkle root - // that politeiad signs. + // politeiad files since they contain user defined data that needs + // to be included in the merkle root that politeiad signs. filesAdd := convertFilesFromPi(pe.Files) for _, v := range pe.Metadata { switch v.Hint { @@ -1379,13 +1386,14 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p } return &pi.ProposalEditReply{ + Version: r.Version, CensorshipRecord: convertCensorshipRecordFromPD(r.CensorshipRecord), Timestamp: timestamp, }, nil } -func (p *politeiawww) processProposalSetStatus(pss pi.ProposalSetStatus, usr user.User) (*pi.ProposalSetStatusReply, error) { - log.Tracef("processProposalSetStatus: %v %v", pss.Token, pss.Status) +func (p *politeiawww) processProposalStatusSet(pss pi.ProposalStatusSet, usr user.User) (*pi.ProposalStatusSetReply, error) { + log.Tracef("processProposalStatusSet: %v %v", pss.Token, pss.Status) // Sanity check if !usr.Admin { @@ -1444,8 +1452,8 @@ func (p *politeiawww) processProposalSetStatus(pss pi.ProposalSetStatus, usr use // done in politeiad. This includes: // -Verify proposal exists (politeiad) // -Verify proposal state is correct (politeiad) - // -Verify version is the latest version (politeiad pi plugin) - // -Verify status change is allowed (politeiad pi plugin) + // -Verify version is the latest version (pi plugin) + // -Verify status change is allowed (pi plugin) // Setup metadata timestamp := time.Now().Unix() @@ -1471,8 +1479,6 @@ func (p *politeiawww) processProposalSetStatus(pss pi.ProposalSetStatus, usr use mdOverwrite := []pd.MetadataStream{} // Send politeiad request - // TODO verify proposal not found error is returned when wrong - // token or state is used var r *pd.Record status := convertRecordStatusFromPropStatus(pss.Status) switch pss.State { @@ -1498,7 +1504,7 @@ func (p *politeiawww) processProposalSetStatus(pss pi.ProposalSetStatus, usr use adminID: usr.ID.String(), }) - return &pi.ProposalSetStatusReply{ + return &pi.ProposalStatusSetReply{ Timestamp: timestamp, }, nil } @@ -1973,13 +1979,13 @@ func (p *politeiawww) handleProposalEdit(w http.ResponseWriter, r *http.Request) util.RespondWithJSON(w, http.StatusOK, per) } -func (p *politeiawww) handleProposalSetStatus(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleProposalSetStatus") +func (p *politeiawww) handleProposalStatusSet(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleProposalStatusSet") - var pss pi.ProposalSetStatus + var pss pi.ProposalStatusSet decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&pss); err != nil { - respondWithPiError(w, r, "handleProposalSetStatus: unmarshal", + respondWithPiError(w, r, "handleProposalStatusSet: unmarshal", pi.UserErrorReply{ ErrorCode: pi.ErrorStatusInputInvalid, }) @@ -1989,14 +1995,14 @@ func (p *politeiawww) handleProposalSetStatus(w http.ResponseWriter, r *http.Req usr, err := p.getSessionUser(w, r) if err != nil { respondWithPiError(w, r, - "handleProposalSetStatus: getSessionUser: %v", err) + "handleProposalStatusSet: getSessionUser: %v", err) return } - pssr, err := p.processProposalSetStatus(pss, *usr) + pssr, err := p.processProposalStatusSet(pss, *usr) if err != nil { respondWithPiError(w, r, - "handleProposalSetStatus: processProposalSetStatus: %v", err) + "handleProposalStatusSet: processProposalStatusSet: %v", err) return } @@ -2394,7 +2400,7 @@ func (p *politeiawww) setPiRoutes() { pi.RouteProposalEdit, p.handleProposalEdit, permissionLogin) p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteProposalSetStatus, p.handleProposalSetStatus, + pi.RouteProposalStatusSet, p.handleProposalStatusSet, permissionAdmin) p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteProposals, p.handleProposals, diff --git a/politeiawww/www.go b/politeiawww/www.go index 5d5b04eb0..56471cbcd 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -12,6 +12,7 @@ import ( _ "encoding/gob" "encoding/hex" "encoding/json" + "errors" "fmt" "io/ioutil" "net/http" @@ -57,6 +58,12 @@ const ( func convertWWWErrorStatusFromPD(e pd.ErrorStatusT) www.ErrorStatusT { switch e { + case pd.ErrorStatusInvalidRequestPayload: + // Intentionally omitted because this indicates a politeiawww + // server error so a ErrorStatusInvalid should be returned. + case pd.ErrorStatusInvalidChallenge: + // Intentionally omitted because this indicates a politeiawww + // server error so a ErrorStatusInvalid should be returned. case pd.ErrorStatusInvalidFilename: return www.ErrorStatusInvalidFilename case pd.ErrorStatusInvalidFileDigest: @@ -69,12 +76,8 @@ func convertWWWErrorStatusFromPD(e pd.ErrorStatusT) www.ErrorStatusT { return www.ErrorStatusUnsupportedMIMEType case pd.ErrorStatusInvalidRecordStatusTransition: return www.ErrorStatusInvalidPropStatusTransition - case pd.ErrorStatusInvalidRequestPayload: - // Intentionally omitted because this indicates a politeiawww - // server error so a ErrorStatusInvalid should be returned. - case pd.ErrorStatusInvalidChallenge: - // Intentionally omitted because this indicates a politeiawww - // server error so a ErrorStatusInvalid should be returned. + case pd.ErrorStatusRecordNotFound: + return www.ErrorStatusProposalNotFound } return www.ErrorStatusInvalid } @@ -83,8 +86,6 @@ func convertWWWErrorStatusFromPiPlugin(e piplugin.ErrorStatusT) www.ErrorStatusT switch e { case piplugin.ErrorStatusPropLinkToInvalid: return www.ErrorStatusInvalidLinkTo - case piplugin.ErrorStatusPropStatusInvalid: - return www.ErrorStatusWrongStatus case piplugin.ErrorStatusVoteStatusInvalid: return www.ErrorStatusWrongVoteStatus } @@ -188,6 +189,8 @@ func convertPiErrorStatusFromPD(e pd.ErrorStatusT) pi.ErrorStatusT { return pi.ErrorStatusPropStatusChangeInvalid case pd.ErrorStatusNoChanges: return pi.ErrorStatusNoPropChanges + case pd.ErrorStatusRecordNotFound: + return pi.ErrorStatusPropNotFound } return pi.ErrorStatusInvalid } @@ -196,8 +199,6 @@ func convertPiErrorStatusFromPiPlugin(e piplugin.ErrorStatusT) pi.ErrorStatusT { switch e { case piplugin.ErrorStatusPropLinkToInvalid: return pi.ErrorStatusPropLinkToInvalid - case piplugin.ErrorStatusPropStatusInvalid: - return pi.ErrorStatusPropStatusInvalid case piplugin.ErrorStatusVoteStatusInvalid: return pi.ErrorStatusVoteStatusInvalid } @@ -325,7 +326,8 @@ func getIdentity(rpcHost, rpcCert, rpcIdentityFile, interactive string) error { // files a complaint. func respondWithPiError(w http.ResponseWriter, r *http.Request, format string, err error) { // Check for pi user error - if ue, ok := err.(pi.UserErrorReply); ok { + var ue pi.UserErrorReply + if errors.As(err, &ue) { // Error is a pi user error. Log it and return a 400. if len(ue.ErrorContext) == 0 { log.Infof("Pi user error: %v %v %v", @@ -347,7 +349,8 @@ func respondWithPiError(w http.ResponseWriter, r *http.Request, format string, e } // Check for politeiad error - if pdErr, ok := err.(pdError); ok { + var pdErr pdError + if errors.As(err, &pdErr) { var ( pluginID = pdErr.ErrorReply.PluginID errCode = pdErr.ErrorReply.ErrorCode diff --git a/util/signature.go b/util/signature.go index 0357b2478..00e252a83 100644 --- a/util/signature.go +++ b/util/signature.go @@ -57,8 +57,7 @@ func VerifySignature(signature, pubKey, msg string) error { } if !pk.VerifyMessage([]byte(msg), sig) { return SignatureError{ - ErrorCode: ErrorStatusSignatureInvalid, - ErrorContext: []string{err.Error()}, + ErrorCode: ErrorStatusSignatureInvalid, } } return nil From eb4f04634c288cc20e7221f9c5d8c9a2478e46fd Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 6 Oct 2020 16:15:32 -0500 Subject: [PATCH 137/449] fix typo to make linter happy --- politeiad/backend/tlogbe/tlog.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index 579d19e7b..3102bdfb0 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -1194,11 +1194,10 @@ func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []ba // metadataSave saves the provided metadata to the kv store and trillian tree. // The record index for this iteration of the record is returned. This is step // one of a two step process. The record update will not be considered -// succesful until the returned record index is also saved to the kv store -// and trillian tree. This code has been pulled out so that it can be called -// during normal metadata updates as well as when an update requires a freeze -// record to be saved along with the record index, such as when a record is -// censored. +// successful until the returned record index is also saved to the kv store and +// trillian tree. This code has been pulled out so that it can be called during +// normal metadata updates as well as when an update requires a freeze record +// to be saved along with the record index, such as when a record is censored. func (t *tlog) metadataSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) (*recordIndex, error) { // Verify tree exists if !t.treeExists(treeID) { From 4dd1bd92671f70e5d7686bed0f66190bd9deb50f Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 6 Oct 2020 16:56:02 -0500 Subject: [PATCH 138/449] last of the proposal route bugs --- politeiad/backend/backend.go | 6 ++--- politeiad/backend/gitbe/gitbe.go | 6 ++--- politeiad/backend/tlogbe/tlog.go | 10 +++++-- politeiad/backend/tlogbe/tlogbe.go | 31 +++++++++++++++------- politeiawww/cmd/piwww/piwww.go | 14 ++++------ politeiawww/cmd/piwww/proposalinventory.go | 12 ++++----- politeiawww/cmd/piwww/voteinventory.go | 7 +++-- politeiawww/cmd/shared/client.go | 4 +-- 8 files changed, 51 insertions(+), 39 deletions(-) diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index eb33b64db..012a02e63 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -35,9 +35,9 @@ var ( // expected. ErrChangesRecord = errors.New("changes record") - // ErrRecordArchived is returned when an update was attempted on a - // archived record. - ErrRecordArchived = errors.New("record is archived") + // ErrRecordLocked is returned when a record status is one that + // does not allow any further changes. + ErrRecordLocked = errors.New("record is locked") // ErrJournalsNotReplayed is returned when the journals have not // been replayed and the subsequent code expect it to be replayed. diff --git a/politeiad/backend/gitbe/gitbe.go b/politeiad/backend/gitbe/gitbe.go index 37aad8ca8..b8be8654d 100644 --- a/politeiad/backend/gitbe/gitbe.go +++ b/politeiad/backend/gitbe/gitbe.go @@ -1880,7 +1880,7 @@ func (g *gitBackEnd) _updateVettedMetadata(token []byte, mdAppend []backend.Meta return err } if md.Status == backend.MDStatusArchived { - return backend.ErrRecordArchived + return backend.ErrRecordLocked } log.Debugf("updating vetted metadata %x", token) @@ -2014,7 +2014,7 @@ func (g *gitBackEnd) _updateVettedMetadataMulti(um []updateMetadata, idTmp strin return err } if md.Status == backend.MDStatusArchived { - return backend.ErrRecordArchived + return backend.ErrRecordLocked } } @@ -2608,7 +2608,7 @@ func (g *gitBackEnd) _setVettedStatus(token []byte, status backend.MDStatusT, md return nil, err } if md.Status == backend.MDStatusArchived { - return nil, backend.ErrRecordArchived + return nil, backend.ErrRecordLocked } // Load record diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index 3102bdfb0..b161d4a06 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -806,9 +806,12 @@ func recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backend.RecordMe return nil, errTreeIsFrozen } - // Verify there are no duplicate mdstream IDs + // Verify there are no duplicate or empty mdstream IDs mdstreamIDs := make(map[uint64]struct{}, len(metadata)) for _, v := range metadata { + if v.ID == 0 { + return nil, fmt.Errorf("invalid metadata stream ID 0") + } _, ok := mdstreamIDs[v.ID] if ok { return nil, fmt.Errorf("duplicate metadata stream ID: %v", v.ID) @@ -816,9 +819,12 @@ func recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backend.RecordMe mdstreamIDs[v.ID] = struct{}{} } - // Verify there are no duplicate filenames + // Verify there are no duplicate or empty filenames filenames := make(map[string]struct{}, len(files)) for _, v := range files { + if v.Name == "" { + return nil, fmt.Errorf("empty filename") + } _, ok := filenames[v.Name] if ok { return nil, fmt.Errorf("duplicate filename found: %v", v.Name) diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index c629a7316..c1a7cbfc2 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -507,10 +507,10 @@ func filesUpdate(filesCurr, filesAdd []backend.File, filesDel []string) []backen } // TODO test this function -func metadataStreamsUpdate(mdCurr, mdAppend, mdOverwrite []backend.MetadataStream) []backend.MetadataStream { - // Convert existing metadata to map - md := make(map[uint64]backend.MetadataStream, len(mdCurr)+len(mdAppend)) - for _, v := range mdCurr { +func metadataStreamsUpdate(curr, mdAppend, mdOverwrite []backend.MetadataStream) []backend.MetadataStream { + // Put current metadata into a map + md := make(map[uint64]backend.MetadataStream, len(curr)) + for _, v := range curr { md[v.ID] = v } @@ -528,6 +528,7 @@ func metadataStreamsUpdate(mdCurr, mdAppend, mdOverwrite []backend.MetadataStrea // No existing metadata. Use append data as full metadata // stream. md[v.ID] = v + continue } // Metadata exists. Append to it. @@ -706,7 +707,10 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ // Save record err = t.unvetted.recordSave(treeID, *recordMD, metadata, files) if err != nil { - if err == errNoFileChanges { + switch err { + case errTreeIsFrozen: + return nil, backend.ErrRecordLocked + case errNoFileChanges: return nil, backend.ErrNoChanges } return nil, fmt.Errorf("recordSave: %v", err) @@ -804,7 +808,10 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b // Save record err = t.vetted.recordSave(treeID, *recordMD, metadata, files) if err != nil { - if err == errNoFileChanges { + switch err { + case errTreeIsFrozen: + return nil, backend.ErrRecordLocked + case errNoFileChanges: return nil, backend.ErrNoChanges } return nil, fmt.Errorf("recordSave: %v", err) @@ -891,7 +898,10 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite // Update metadata err = t.unvetted.recordMetadataSave(treeID, r.RecordMetadata, metadata) if err != nil { - if err == errNoMetadataChanges { + switch err { + case errTreeIsFrozen: + return backend.ErrRecordLocked + case errNoMetadataChanges: return backend.ErrNoChanges } return err @@ -977,10 +987,13 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ // Update metadata err = t.vetted.recordMetadataSave(treeID, r.RecordMetadata, metadata) if err != nil { - if err == errNoMetadataChanges { + switch err { + case errTreeIsFrozen: + return backend.ErrRecordLocked + case errNoMetadataChanges: return backend.ErrNoChanges } - return err + return fmt.Errorf("recordMetadataSave: %v", err) } // Call post plugin hooks diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index 2c4a7a7a0..439957dbb 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -62,17 +62,13 @@ type piwww struct { UserDetails userDetailsCmd `command:"userdetails"` Users shared.UsersCmd `command:"users"` - // TODO some of the proposal commands use both the --unvetted and - // --vetted flags. Let make them all use only the --unvetted flag. - // If --unvetted is not included then its assumed to be a vetted - // request. // TODO replace www policies with pi policies // Proposal commands ProposalNew proposalNewCmd `command:"proposalnew"` ProposalEdit proposalEditCmd `command:"proposaledit"` ProposalStatusSet proposalStatusSetCmd `command:"proposalstatusset"` Proposals proposalsCmd `command:"proposals"` - ProposalInventory proposalInventoryCmd `command:"proposalinventory"` + ProposalInventory proposalInventoryCmd `command:"proposalinv"` // Comments commands CommentNew commentNewCmd `command:"commentnew"` @@ -89,7 +85,7 @@ type piwww struct { Votes votesCmd `command:"votes"` VoteResults voteResultsCmd `command:"voteresults"` VoteSummaries voteSummariesCmd `command:"votesummaries"` - VoteInventory voteInventoryCmd `command:"voteinventory"` + VoteInventory voteInventoryCmd `command:"voteinv"` // Websocket commands Subscribe subscribeCmd `command:"subscribe"` @@ -141,9 +137,9 @@ User commands Proposal commands proposalnew (user) Submit a new proposal proposaledit (user) Edit an existing proposal - proposalsetstatus (admin) Set the status of a proposal + proposalstatusset (admin) Set the status of a proposal proposals (public) Get proposals - proposalinventory (public) Get proposals inventory by proposal status + proposalinv (public) Get proposal inventory by proposal status Comment commands commentnew (user) Submit a new comment @@ -160,7 +156,7 @@ Vote commands votes (public) Get vote details voteresults (public) Get full vote results votesummaries (public) Get vote summaries - voteinventory (public) Get proposal inventory by vote status + voteinv (public) Get proposal inventory by vote status Websocket commands subscribe (public) Subscribe/unsubscribe to websocket event diff --git a/politeiawww/cmd/piwww/proposalinventory.go b/politeiawww/cmd/piwww/proposalinventory.go index 69ae8df0c..a8131d5b4 100644 --- a/politeiawww/cmd/piwww/proposalinventory.go +++ b/politeiawww/cmd/piwww/proposalinventory.go @@ -18,14 +18,12 @@ func (cmd *proposalInventoryCmd) Execute(args []string) error { if err != nil { return err } - return shared.PrintJSON(reply) } -// proposalInventoryHelpMsg is the output of the help command when -// 'proposalinventory' is specified. -const proposalInventoryHelpMsg = `proposalinventory +// proposalInventoryHelpMsg is the command help message. +const proposalInventoryHelpMsg = `proposalinv -Fetch the censorship record tokens for all proposals, separated by their -status. The unvetted tokens are only returned if the logged in user is an -admin.` +Fetch the censorship record tokens for all proposals, categorized by their +proposal status. The unvetted tokens are only returned if the logged in user is +an admin.` diff --git a/politeiawww/cmd/piwww/voteinventory.go b/politeiawww/cmd/piwww/voteinventory.go index ec05b0e28..4784fa3d4 100644 --- a/politeiawww/cmd/piwww/voteinventory.go +++ b/politeiawww/cmd/piwww/voteinventory.go @@ -22,10 +22,9 @@ func (cmd *voteInventoryCmd) Execute(args []string) error { return shared.PrintJSON(reply) } -// voteInventoryHelpMsg is the output of the help command when -// 'voteinventory' is specified. -const voteInventoryHelpMsg = `voteinventory +// voteInventoryHelpMsg is the command help message. +const voteInventoryHelpMsg = `voteinv -Fetch the censorship record tokens for all proposals, separated by their +Fetch the censorship record tokens for all proposals, categorized by their vote status. The unvetted tokens are only returned if the logged in user is an admin.` diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index c34a98cf5..ec9da2f9f 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -805,7 +805,7 @@ func (c *Client) Proposals(p pi.Proposals) (*pi.ProposalsReply, error) { // ProposalInventory retrieves the censorship tokens of all proposals, // separated by their status. func (c *Client) ProposalInventory() (*pi.ProposalInventoryReply, error) { - respondeBody, err := c.makeRequest(http.MethodGet, pi.APIRoute, + respondeBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, pi.RouteProposalInventory, nil) if err != nil { return nil, err @@ -1070,7 +1070,7 @@ func (c *Client) PayInvoices(pi *cms.PayInvoices) (*cms.PayInvoicesReply, error) // VoteInventory retrieves the tokens of all proposals in the inventory // categorized by their vote status. func (c *Client) VoteInventory() (*pi.VoteInventoryReply, error) { - responseBody, err := c.makeRequest(http.MethodGet, pi.APIRoute, + responseBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, pi.RouteVoteInventory, nil) if err != nil { return nil, err From ad4c04533b49663cb25fb36dba05cb551cea9c88 Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Wed, 7 Oct 2020 10:13:02 -0300 Subject: [PATCH 139/449] piwww: Add userproposalcredits cmd and naming refactor. --- politeiawww/cmd/piwww/help.go | 16 +++-- politeiawww/cmd/piwww/piwww.go | 69 ++++++++++--------- politeiawww/cmd/piwww/userpaymentverify.go | 28 -------- politeiawww/cmd/piwww/userpendingpayment.go | 28 -------- politeiawww/cmd/piwww/userproposalcredits.go | 27 ++++++++ .../cmd/piwww/userproposalpaywalltx.go | 28 ++++++++ .../cmd/piwww/userregistrationpayment.go | 28 ++++++++ politeiawww/cmd/shared/client.go | 8 +-- 8 files changed, 132 insertions(+), 100 deletions(-) delete mode 100644 politeiawww/cmd/piwww/userpaymentverify.go delete mode 100644 politeiawww/cmd/piwww/userpendingpayment.go create mode 100644 politeiawww/cmd/piwww/userproposalcredits.go create mode 100644 politeiawww/cmd/piwww/userproposalpaywalltx.go create mode 100644 politeiawww/cmd/piwww/userregistrationpayment.go diff --git a/politeiawww/cmd/piwww/help.go b/politeiawww/cmd/piwww/help.go index 4e46ba92a..026c30e03 100644 --- a/politeiawww/cmd/piwww/help.go +++ b/politeiawww/cmd/piwww/help.go @@ -79,23 +79,25 @@ func (cmd *helpCmd) Execute(args []string) error { case "voteinventory": fmt.Printf("%s\n", voteInventoryHelpMsg) - // User commands + // User commands case "usernew": fmt.Printf("%s\n", userNewHelpMsg) case "useredit": fmt.Printf("%s\n", userEditHelpMsg) case "userdetails": fmt.Printf("%s\n", userDetailsHelpMsg) - case "userpaymentsrescan": - fmt.Printf("%s\n", userPaymentsRescanHelpMsg) - case "userpendingpayment": - fmt.Printf("%s\n", userPendingPaymentHelpMsg) case "useremailverify": fmt.Printf("%s\n", userEmailVerifyHelpMsg) - case "userpaymentverify": - fmt.Printf("%s\n", userPaymentVerifyHelpMsg) + case "userregistrationpayment": + fmt.Printf("%s\n", userRegistrationPaymentHelpMsg) case "userproposalpaywall": fmt.Printf("%s\n", userProposalPaywallHelpMsg) + case "userproposalpaywalltx": + fmt.Printf("%s\n", userProposalPaywallTxHelpMsg) + case "userproposalcredits": + fmt.Printf("%s\n", userProposalCreditsHelpMsg) + case "userpaymentsrescan": + fmt.Printf("%s\n", userPaymentsRescanHelpMsg) case "usermanage": fmt.Printf("%s\n", shared.UserManageHelpMsg) case "userkeyupdate": diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index 439957dbb..73fb8dfd6 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -44,23 +44,24 @@ type piwww struct { Me shared.MeCmd `command:"me"` // User commands - UserNew userNewCmd `command:"usernew"` - UserEdit userEditCmd `command:"useredit"` - UserManage shared.UserManageCmd `command:"usermanage"` - UserEmailVerify userEmailVerifyCmd `command:"useremailverify"` - UserVerificationResend userVerificationResendCmd `command:"userverificationresend"` - UserPasswordReset shared.UserPasswordResetCmd `command:"userpasswordreset"` - UserPasswordChange shared.UserPasswordChangeCmd `command:"userpasswordchange"` - UserUsernameChange shared.UserUsernameChangeCmd `command:"userusernamechange"` - UserKeyUpdate shared.UserKeyUpdateCmd `command:"userkeyupdate"` - UserTOTPSet shared.UserTOTPSetCmd `command:"usertotpset"` - UserTOTPVerify shared.UserTOTPVerifyCmd `command:"usertotpverify"` - UserProposalPaywall userProposalPaywallCmd `command:"userproposalpaywall"` - UserPaymentVerify userPaymentVerifyCmd `command:"userpaymentverify"` - UserPaymentsRescan userPaymentsRescanCmd `command:"userpaymentsrescan"` - UserPendingPayment userPendingPaymentCmd `command:"userpendingpayment"` - UserDetails userDetailsCmd `command:"userdetails"` - Users shared.UsersCmd `command:"users"` + UserNew userNewCmd `command:"usernew"` + UserEdit userEditCmd `command:"useredit"` + UserManage shared.UserManageCmd `command:"usermanage"` + UserEmailVerify userEmailVerifyCmd `command:"useremailverify"` + UserVerificationResend userVerificationResendCmd `command:"userverificationresend"` + UserPasswordReset shared.UserPasswordResetCmd `command:"userpasswordreset"` + UserPasswordChange shared.UserPasswordChangeCmd `command:"userpasswordchange"` + UserUsernameChange shared.UserUsernameChangeCmd `command:"userusernamechange"` + UserKeyUpdate shared.UserKeyUpdateCmd `command:"userkeyupdate"` + UserTOTPSet shared.UserTOTPSetCmd `command:"usertotpset"` + UserTOTPVerify shared.UserTOTPVerifyCmd `command:"usertotpverify"` + UserRegistrationPayment userRegistrationPaymentCmd `command:"userregistrationpayment"` + UserProposalPaywall userProposalPaywallCmd `command:"userproposalpaywall"` + UserProposalPaywallTx userProposalPaywallTxCmd `command:"userproposalpaywalltx"` + UserProposalCredits userProposalCreditsCmd `command:"userproposalcredits"` + UserPaymentsRescan userPaymentsRescanCmd `command:"userpaymentsrescan"` + UserDetails userDetailsCmd `command:"userdetails"` + Users shared.UsersCmd `command:"users"` // TODO replace www policies with pi policies // Proposal commands @@ -117,22 +118,24 @@ Basic commands me (user) Get details of the logged in user User commands - usernew (public) Create a new user - useredit (user) Edit the logged in user - usermanage (admin) Edit a user as an admin - useremailverify (public) Verify email address - userverificationresend (public) Resend verification email - userpasswordreset (public) Reset password - userpasswordchange (user) Change password - userusernamechange (user) Change username - userkeyupdate (user) Update user key (i.e. identity) - usertotpset (user) Set a TOTP method - usertotpverify (user) Verify a TOTP method - userpaymentverify (user) Verify registration payment - userpaymentsrescan (user) Rescan all user payments - userpendingpayment (user) Get pending user payments - userdetails (public) Get user details - users (public) Get users + usernew (public) Create a new user + useredit (user) Edit the logged in user + usermanage (admin) Edit a user as an admin + useremailverify (public) Verify email address + userverificationresend (public) Resend verification email + userpasswordreset (public) Reset password + userpasswordchange (user) Change password + userusernamechange (user) Change username + userkeyupdate (user) Update user key (i.e. identity) + usertotpset (user) Set a TOTP method + usertotpverify (user) Verify a TOTP method + userregistrationpayment (user) Verify registration payment + userproposalpaywall (user) Get user paywall details + userproposalpaywalltx (user) Get pending user payments + userproposalcredits (user) Get user proposal credits + userpaymentsrescan (user) Rescan all user payments + userdetails (public) Get user details + users (public) Get users Proposal commands proposalnew (user) Submit a new proposal diff --git a/politeiawww/cmd/piwww/userpaymentverify.go b/politeiawww/cmd/piwww/userpaymentverify.go deleted file mode 100644 index ff0eccff7..000000000 --- a/politeiawww/cmd/piwww/userpaymentverify.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import "github.com/decred/politeia/politeiawww/cmd/shared" - -// userPaymentVerifyCmd checks on the status of the logged in user's -// registration payment. -type userPaymentVerifyCmd struct{} - -// Execute executes the verify user payment command. -func (cmd *userPaymentVerifyCmd) Execute(args []string) error { - vupr, err := client.UserPaymentVerify() - if err != nil { - return err - } - return shared.PrintJSON(vupr) -} - -// userPaymentVerifyHelpMsg is the output of the help command when -// 'userpaymentverify' is specified. -var userPaymentVerifyHelpMsg = `userpaymentverify - -Check if the currently logged in user has paid their user registration fee. - -Arguments: None` diff --git a/politeiawww/cmd/piwww/userpendingpayment.go b/politeiawww/cmd/piwww/userpendingpayment.go deleted file mode 100644 index 4b319f3ef..000000000 --- a/politeiawww/cmd/piwww/userpendingpayment.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import "github.com/decred/politeia/politeiawww/cmd/shared" - -// userPendingPaymentCmd sretrieve the payment details for a pending payment, -// if one exists, for the logged in user. -type userPendingPaymentCmd struct{} - -// Execute executes the user pending payment command. -func (cmd *userPendingPaymentCmd) Execute(args []string) error { - pppr, err := client.ProposalPaywallPayment() - if err != nil { - return err - } - return shared.PrintJSON(pppr) -} - -// userPendingPaymentHelpMsg is the output for the help command when -// 'userpendingpayment' is specified. -const userPendingPaymentHelpMsg = `userpendingpayment - -Get pending payment details for the logged in user. - -Arguments: None` diff --git a/politeiawww/cmd/piwww/userproposalcredits.go b/politeiawww/cmd/piwww/userproposalcredits.go new file mode 100644 index 000000000..1d629c4ea --- /dev/null +++ b/politeiawww/cmd/piwww/userproposalcredits.go @@ -0,0 +1,27 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import "github.com/decred/politeia/politeiawww/cmd/shared" + +// userProposalCreditsCmd gets the proposal credits for the logged in user. +type userProposalCreditsCmd struct{} + +// Execute executes the user proposal credits command. +func (cmd *userProposalCreditsCmd) Execute(args []string) error { + ppdr, err := client.UserProposalCredits() + if err != nil { + return err + } + return shared.PrintJSON(ppdr) +} + +// userProposalCreditsHelpMsg is the output of the help command when +// 'userproposalcredits' is specified. +const userProposalCreditsHelpMsg = `userproposalcredits + +Fetch the logged in user's proposal credits. + +Arguments: None` diff --git a/politeiawww/cmd/piwww/userproposalpaywalltx.go b/politeiawww/cmd/piwww/userproposalpaywalltx.go new file mode 100644 index 000000000..c7ee9c129 --- /dev/null +++ b/politeiawww/cmd/piwww/userproposalpaywalltx.go @@ -0,0 +1,28 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import "github.com/decred/politeia/politeiawww/cmd/shared" + +// userProposalPaywallTxCmd retrieves the payment details for a pending payment, +// if one exists, for the logged in user. +type userProposalPaywallTxCmd struct{} + +// Execute executes the user proposal paywall tx command. +func (cmd *userProposalPaywallTxCmd) Execute(args []string) error { + pppr, err := client.UserProposalPaywallTx() + if err != nil { + return err + } + return shared.PrintJSON(pppr) +} + +// userProposalPaywallTxHelpMsg is the output for the help command when +// 'userproposalpaywalltx' is specified. +const userProposalPaywallTxHelpMsg = `userproposalpaywalltx + +Get pending payment details for the logged in user. + +Arguments: None` diff --git a/politeiawww/cmd/piwww/userregistrationpayment.go b/politeiawww/cmd/piwww/userregistrationpayment.go new file mode 100644 index 000000000..361cb6bd2 --- /dev/null +++ b/politeiawww/cmd/piwww/userregistrationpayment.go @@ -0,0 +1,28 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import "github.com/decred/politeia/politeiawww/cmd/shared" + +// userRegistrationPaymentCmd checks on the status of the logged in user's +// registration payment. +type userRegistrationPaymentCmd struct{} + +// Execute executes the user registration payment command. +func (cmd *userRegistrationPaymentCmd) Execute(args []string) error { + vupr, err := client.UserRegistrationPayment() + if err != nil { + return err + } + return shared.PrintJSON(vupr) +} + +// userRegistrationPaymentHelpMsg is the output of the help command when +// 'userregistrationpayment' is specified. +var userRegistrationPaymentHelpMsg = `userregistrationpayment + +Check if the currently logged in user has paid their user registration fee. + +Arguments: None` diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index ec9da2f9f..c04bc5b13 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -1435,9 +1435,9 @@ func (c *Client) VoteStartRunoff(vsr pi.VoteStartRunoff) (*pi.VoteStartRunoffRep return &vsrr, nil } -// UserPaymentVerify checks whether the logged in user has paid their user +// UserRegistrationPayment checks whether the logged in user has paid their user // registration fee. -func (c *Client) UserPaymentVerify() (*www.UserRegistrationPaymentReply, error) { +func (c *Client) UserRegistrationPayment() (*www.UserRegistrationPaymentReply, error) { responseBody, err := c.makeRequest(http.MethodGet, www.PoliteiaWWWAPIRoute, www.RouteUserRegistrationPayment, nil) if err != nil { @@ -1805,9 +1805,9 @@ func (c *Client) VerifyUpdateUserKey(vuuk *www.VerifyUpdateUserKey) (*www.Verify return &vuukr, nil } -// ProposalPaywallPayment retrieves payment details of any pending proposal +// UserProposalPaywallTx retrieves payment details of any pending proposal // credit payment from the logged in user. -func (c *Client) ProposalPaywallPayment() (*www.UserProposalPaywallTxReply, error) { +func (c *Client) UserProposalPaywallTx() (*www.UserProposalPaywallTxReply, error) { responseBody, err := c.makeRequest(http.MethodGet, www.PoliteiaWWWAPIRoute, www.RouteUserProposalPaywallTx, nil) if err != nil { From c32e79b56efc378b53b5b3a9bce596788eb64087 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 6 Oct 2020 18:48:44 -0500 Subject: [PATCH 140/449] move plugins into politeiad package --- politeiad/backend/tlogbe/comments.go | 2 +- politeiad/backend/tlogbe/dcrdata.go | 2 +- politeiad/backend/tlogbe/pi.go | 6 ++--- politeiad/backend/tlogbe/ticketvote.go | 4 ++-- politeiad/backend/tlogbe/tlogbe.go | 23 +++++++++---------- .../plugins}/comments/comments.go | 0 .../plugins}/dcrdata/dcrdata.go | 0 {plugins => politeiad/plugins}/pi/pi.go | 0 .../plugins}/ticketvote/ticketvote.go | 0 politeiad/politeiad.go | 8 +++---- politeiawww/comments.go | 2 +- politeiawww/pi.go | 2 +- politeiawww/piwww.go | 6 ++--- politeiawww/proposals.go | 4 ++-- politeiawww/ticketvote.go | 2 +- politeiawww/www.go | 6 ++--- 16 files changed, 33 insertions(+), 34 deletions(-) rename {plugins => politeiad/plugins}/comments/comments.go (100%) rename {plugins => politeiad/plugins}/dcrdata/dcrdata.go (100%) rename {plugins => politeiad/plugins}/pi/pi.go (100%) rename {plugins => politeiad/plugins}/ticketvote/ticketvote.go (100%) diff --git a/politeiad/backend/tlogbe/comments.go b/politeiad/backend/tlogbe/comments.go index 25b429ae2..8a441e25c 100644 --- a/politeiad/backend/tlogbe/comments.go +++ b/politeiad/backend/tlogbe/comments.go @@ -20,10 +20,10 @@ import ( "sync" "time" - "github.com/decred/politeia/plugins/comments" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/store" + "github.com/decred/politeia/politeiad/plugins/comments" "github.com/decred/politeia/util" ) diff --git a/politeiad/backend/tlogbe/dcrdata.go b/politeiad/backend/tlogbe/dcrdata.go index 49a397639..ae56c5c07 100644 --- a/politeiad/backend/tlogbe/dcrdata.go +++ b/politeiad/backend/tlogbe/dcrdata.go @@ -17,8 +17,8 @@ import ( v4 "github.com/decred/dcrdata/api/types/v4" exptypes "github.com/decred/dcrdata/explorer/types/v2" pstypes "github.com/decred/dcrdata/pubsub/types/v3" - "github.com/decred/politeia/plugins/dcrdata" "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/plugins/dcrdata" "github.com/decred/politeia/util" "github.com/decred/politeia/wsdcrdata" ) diff --git a/politeiad/backend/tlogbe/pi.go b/politeiad/backend/tlogbe/pi.go index a4e0d974c..0b6f7a4ac 100644 --- a/politeiad/backend/tlogbe/pi.go +++ b/politeiad/backend/tlogbe/pi.go @@ -17,10 +17,10 @@ import ( "sync" "time" - "github.com/decred/politeia/plugins/comments" - "github.com/decred/politeia/plugins/pi" - "github.com/decred/politeia/plugins/ticketvote" "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/plugins/comments" + "github.com/decred/politeia/politeiad/plugins/pi" + "github.com/decred/politeia/politeiad/plugins/ticketvote" "github.com/decred/politeia/util" ) diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index e003c925d..8aa71bcf3 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -24,11 +24,11 @@ import ( "github.com/decred/dcrd/dcrec/secp256k1/v3/ecdsa" "github.com/decred/dcrd/dcrutil/v3" "github.com/decred/dcrd/wire" - "github.com/decred/politeia/plugins/dcrdata" - "github.com/decred/politeia/plugins/ticketvote" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/store" + "github.com/decred/politeia/politeiad/plugins/dcrdata" + "github.com/decred/politeia/politeiad/plugins/ticketvote" "github.com/decred/politeia/util" ) diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index c1a7cbfc2..64a51ecf4 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -20,14 +20,13 @@ import ( "github.com/decred/dcrd/chaincfg/v3" "github.com/decred/dcrtime/merkle" - "github.com/decred/politeia/plugins/comments" - "github.com/decred/politeia/plugins/dcrdata" - "github.com/decred/politeia/plugins/pi" - "github.com/decred/politeia/plugins/ticketvote" - pd "github.com/decred/politeia/politeiad/api/v1" v1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/mime" "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/plugins/comments" + "github.com/decred/politeia/politeiad/plugins/dcrdata" + "github.com/decred/politeia/politeiad/plugins/pi" + "github.com/decred/politeia/politeiad/plugins/ticketvote" "github.com/decred/politeia/util" "github.com/marcopeereboom/sbox" "github.com/subosito/gozaru" @@ -124,7 +123,7 @@ type plugin struct { } func tokenPrefix(token []byte) string { - return hex.EncodeToString(token)[:pd.TokenPrefixLength] + return hex.EncodeToString(token)[:v1.TokenPrefixLength] } func (t *tlogBackend) isShutdown() bool { @@ -650,7 +649,7 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ // Allow ErrorStatusEmpty which indicates no new files are being // added. This can happen when files are being deleted without // any new files being added. - if e.ErrorCode != pd.ErrorStatusEmpty { + if e.ErrorCode != v1.ErrorStatusEmpty { return nil, err } } @@ -748,7 +747,7 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b // Allow ErrorStatusEmpty which indicates no new files are being // added. This can happen when files are being deleted without // any new files being added. - if e.ErrorCode != pd.ErrorStatusEmpty { + if e.ErrorCode != v1.ErrorStatusEmpty { return nil, err } } @@ -847,13 +846,13 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite // Allow ErrorStatusEmpty which indicates no new files are being // being added. This is expected since this is a metadata only // update. - if e.ErrorCode != pd.ErrorStatusEmpty { + if e.ErrorCode != v1.ErrorStatusEmpty { return err } } if len(mdAppend) == 0 && len(mdOverwrite) == 0 { return backend.ContentVerificationError{ - ErrorCode: pd.ErrorStatusNoChanges, + ErrorCode: v1.ErrorStatusNoChanges, } } @@ -933,13 +932,13 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ // Allow ErrorStatusEmpty which indicates no new files are being // being added. This is expected since this is a metadata only // update. - if e.ErrorCode != pd.ErrorStatusEmpty { + if e.ErrorCode != v1.ErrorStatusEmpty { return err } } if len(mdAppend) == 0 && len(mdOverwrite) == 0 { return backend.ContentVerificationError{ - ErrorCode: pd.ErrorStatusNoChanges, + ErrorCode: v1.ErrorStatusNoChanges, } } diff --git a/plugins/comments/comments.go b/politeiad/plugins/comments/comments.go similarity index 100% rename from plugins/comments/comments.go rename to politeiad/plugins/comments/comments.go diff --git a/plugins/dcrdata/dcrdata.go b/politeiad/plugins/dcrdata/dcrdata.go similarity index 100% rename from plugins/dcrdata/dcrdata.go rename to politeiad/plugins/dcrdata/dcrdata.go diff --git a/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go similarity index 100% rename from plugins/pi/pi.go rename to politeiad/plugins/pi/pi.go diff --git a/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go similarity index 100% rename from plugins/ticketvote/ticketvote.go rename to politeiad/plugins/ticketvote/ticketvote.go diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index b0694941e..a9bf7980b 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -23,15 +23,15 @@ import ( "github.com/decred/politeia/cmsplugin" "github.com/decred/politeia/decredplugin" - "github.com/decred/politeia/plugins/comments" - "github.com/decred/politeia/plugins/dcrdata" - "github.com/decred/politeia/plugins/pi" - "github.com/decred/politeia/plugins/ticketvote" v1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/gitbe" "github.com/decred/politeia/politeiad/backend/tlogbe" + "github.com/decred/politeia/politeiad/plugins/comments" + "github.com/decred/politeia/politeiad/plugins/dcrdata" + "github.com/decred/politeia/politeiad/plugins/pi" + "github.com/decred/politeia/politeiad/plugins/ticketvote" "github.com/decred/politeia/util" "github.com/decred/politeia/util/version" "github.com/gorilla/mux" diff --git a/politeiawww/comments.go b/politeiawww/comments.go index 45919b675..243a71b04 100644 --- a/politeiawww/comments.go +++ b/politeiawww/comments.go @@ -5,7 +5,7 @@ package main import ( - "github.com/decred/politeia/plugins/comments" + "github.com/decred/politeia/politeiad/plugins/comments" ) // commentsAll returns all comments for the provided record. diff --git a/politeiawww/pi.go b/politeiawww/pi.go index 4b1d909ef..a9f805a15 100644 --- a/politeiawww/pi.go +++ b/politeiawww/pi.go @@ -5,7 +5,7 @@ package main import ( - piplugin "github.com/decred/politeia/plugins/pi" + piplugin "github.com/decred/politeia/politeiad/plugins/pi" ) // piProposals returns the pi plugin data for the provided proposals. diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index faa426829..a2a5ffcc1 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -17,10 +17,10 @@ import ( "strconv" "time" - "github.com/decred/politeia/plugins/comments" - piplugin "github.com/decred/politeia/plugins/pi" - "github.com/decred/politeia/plugins/ticketvote" pd "github.com/decred/politeia/politeiad/api/v1" + "github.com/decred/politeia/politeiad/plugins/comments" + piplugin "github.com/decred/politeia/politeiad/plugins/pi" + "github.com/decred/politeia/politeiad/plugins/ticketvote" pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/user" diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 8f8d1ac4a..dc8472aaf 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -9,8 +9,8 @@ import ( "strconv" "github.com/decred/politeia/decredplugin" - piplugin "github.com/decred/politeia/plugins/pi" - ticketvote "github.com/decred/politeia/plugins/ticketvote" + piplugin "github.com/decred/politeia/politeiad/plugins/pi" + ticketvote "github.com/decred/politeia/politeiad/plugins/ticketvote" pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/user" diff --git a/politeiawww/ticketvote.go b/politeiawww/ticketvote.go index 24aa2f015..e9dd3a1b0 100644 --- a/politeiawww/ticketvote.go +++ b/politeiawww/ticketvote.go @@ -5,7 +5,7 @@ package main import ( - ticketvote "github.com/decred/politeia/plugins/ticketvote" + ticketvote "github.com/decred/politeia/politeiad/plugins/ticketvote" ) // voteAuthorize uses the ticketvote plugin to authorize a vote. diff --git a/politeiawww/www.go b/politeiawww/www.go index 56471cbcd..51d7842a4 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -25,10 +25,10 @@ import ( "time" "github.com/decred/politeia/mdstream" - "github.com/decred/politeia/plugins/comments" - piplugin "github.com/decred/politeia/plugins/pi" - "github.com/decred/politeia/plugins/ticketvote" pd "github.com/decred/politeia/politeiad/api/v1" + "github.com/decred/politeia/politeiad/plugins/comments" + piplugin "github.com/decred/politeia/politeiad/plugins/pi" + "github.com/decred/politeia/politeiad/plugins/ticketvote" cms "github.com/decred/politeia/politeiawww/api/cms/v1" pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" From 7c11ab960f8b2a7e4a264bc21cc88472af359d51 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 6 Oct 2020 20:23:23 -0500 Subject: [PATCH 141/449] start updating readme --- README.md | 44 ++++-------------- politeiad/README.md | 74 ++++++++++++++++++++++++++++++ politeiad/backend/tlogbe/tlogbe.go | 8 ++-- 3 files changed, 87 insertions(+), 39 deletions(-) create mode 100644 politeiad/README.md diff --git a/README.md b/README.md index c7390c519..d05eb4549 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,10 @@ -# Politeia +politeia +==== + [![Build Status](https://github.com/decred/politeia/workflows/Build%20and%20Test/badge.svg)](https://github.com/decred/politeia/actions) [![ISC License](https://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org) +[![Go Report Card](https://goreportcard.com/badge/github.com/decred/politeia)](https://goreportcard.com/report/github.com/decred/politeia) -**Politeia is the Decred proposal system.** Politeia is a system for storing off-chain data that is both versioned and timestamped, essentially “git, a popular revision control system, plus timestamping”. Instead of attempting to store all the data related to Decred’s @@ -15,13 +17,11 @@ The politeia stack is as follows: ~~~~~~~~ Internet ~~~~~~~~~ | +-------------------------+ -| politeia www | +| politeiawww | +-------------------------+ | +-------------------------+ | politeiad | -+-------------------------+ -| git backend | +-------------------------+ | ~~~~~~~~ Internet ~~~~~~~~~ @@ -30,30 +30,19 @@ The politeia stack is as follows: | dcrtimed | +-------------------------+ ``` -## API Documentation - -### v1 - -* [politeiawww API Specification v1](https://github.com/decred/politeia/blob/master/politeiawww/api/www/v1/api.md) - This document describes the REST API provided by a politeiawww server. The politeiawww server is the web server backend and it interacts with a JSON REST API. This document also describes websockets for server side notifications. It does not render HTML. - -### v2 - -* [politeiawww API Specification v2](https://github.com/decred/politeia/blob/master/politeiawww/api/www/v2/api.md) - This document describes the v2 REST API provided by a politeiawww server. The politeiawww server is the web server backend that interacts with clients using a JSON REST API. ## Components -### Core components +Core software: * politeiad - Reference server daemon. * politeiawww - Web backend server; depends on politeiad. ### Tools and reference clients -* [politeia](https://github.com/decred/politeia/tree/master/politeiad/cmd/politeia) - Reference client application for politeiad. -* [politeia_verify](https://github.com/decred/politeia/tree/master/politeiad/cmd/politeia_verify) - Reference verification tool. -* [politeiawwwcli](https://github.com/decred/politeia/tree/master/politeiawww/cmd/politeiawwwcli) - Command-line tool for interacting with politeiawww. -* [politeiawww_dbutil](https://github.com/decred/politeia/tree/master/politeiawww/cmd/politeiawww_dbutil) - Tool for debugging and creating admin users within the politeiawww database. -* [politeiawww_dataload](https://github.com/decred/politeia/tree/master/politeiawww/cmd/politeiawww_dataload) - Tool using politeiawwwcli to load a basic dataset into politeiawww. +* [politeiawww_dbutil](https://github.com/decred/politeia/tree/master/politeiawww/cmd/politeiawww_dbutil) - Tool for updating the politeiawww user database manually. +* [piwww](https://github.com/decred/politeia/tree/master/politeiawww/cmd/piwww) - Command-line tool for interacting with the politeiawww pi API. +* [cmswww](https://github.com/decred/politeia/tree/master/politeiawww/cmd/cmswww) - Command-line tool for interacting with politeiawww cms API. **Note:** politeiawww does not provide HTML output. It strictly handles the JSON REST RPC commands only. The GUI for politeiawww can be found at: @@ -104,12 +93,6 @@ You can also use the following default configurations: rpcuser=user rpcpass=pass testnet=true - enablecache=true - cachehost=localhost:26257 - cacherootcert="~/.cockroachdb/certs/clients/politeiad/ca.crt" - cachecert="~/.cockroachdb/certs/clients/politeiad/client.politeiad.crt" - cachekey="~/.cockroachdb/certs/clients/politeiad/client.politeiad.key" - **politeiawww.conf**: @@ -384,15 +367,6 @@ When using politeiawww_refclient, the `-use-paywall` flag is true by default. Wh * Set the user created in the first refclient execution as admin with politeiawww_dbutil. * Run refclient again with the `email` and `password` flags set to the user created in the first refclient execution. -#### Rebuilding the Cache - -The cache will be built automatically on initial startup of politeiad and when -the cache version has changed, but there may also be times during development -that you want to force the cache to rebuild. You can do this by using the -`--buildcache` flag when starting `politeiad`. This will drop all current -tables from the cache, re-create the tables, then populate the cache with the -data that is in the politeiad git repositories. - #### Building with repository version It is often useful to have version information from the repository where diff --git a/politeiad/README.md b/politeiad/README.md new file mode 100644 index 000000000..2661b3344 --- /dev/null +++ b/politeiad/README.md @@ -0,0 +1,74 @@ +politeiad +==== + +# Installing and running + +## Install Dependencies + +
Go 1.14 or 1.15 + + Installation instructions can be found here: https://golang.org/doc/install. + Ensure Go was installed properly and is a supported version: + + ```sh + $ go version + $ go env GOROOT GOPATH + ``` + + NOTE: `GOROOT` and `GOPATH` must not be on the same path. Since Go 1.8 + (2016), `GOROOT` and `GOPATH` are set automatically, and you do not need to + change them. However, you still need to add `$GOPATH/bin` to your `PATH` in + order to run binaries installed by `go get` and `go install` (On Windows, + this happens automatically). + + Unix example -- add these lines to .profile: + + ``` + PATH="$PATH:/usr/local/go/bin" # main Go binaries ($GOROOT/bin) + PATH="$PATH:$HOME/go/bin" # installed Go projects ($GOPATH/bin) + ``` +
+ +
Git + + Installation instructions can be found at https://git-scm.com or + https://gitforwindows.org. + ```sh + $ git version + ``` +
+ +## Build from source + +## Setup configuration file + +[`sample-politeiad.conf`](https://github.com/decred/politeia/blob/master/politeiad/sample-politeiad.conf) + +Copy the sample configuration file to the politeiad data directory for your OS. + +* **macOS** + + `/Users//Library/Application Support/Politeiad/politeiad.conf` + +* **Windows** + + `C:\Users\\AppData\Local\Politeiad/politeiad.conf` + +* **Ubuntu** + + `~/.politeiad/politeiad.conf` + +Use the following config settings to spin up a development politeiad instance. + +**politeiad.conf**: + + rpcuser=user + rpcpass=pass + testnet=true + +# Tools and reference clients + +* [politeia](https://github.com/decred/politeia/tree/master/politeiad/cmd/politeia) - Reference client for politeiad. +* [politeia_verify](https://github.com/decred/politeia/tree/master/politeiad/cmd/politeia_verify) - Reference verification tool. + + diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 64a51ecf4..f58d18764 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -1353,12 +1353,12 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md return nil, fmt.Errorf("recordLatest: %v", err) } rm := r.RecordMetadata - oldStatus := rm.Status + currStatus := rm.Status // Validate status change if !statusChangeIsAllowed(rm.Status, status) { return nil, backend.StateTransitionError{ - From: oldStatus, + From: currStatus, To: status, } } @@ -1412,10 +1412,10 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md } // Update inventory cache - t.inventoryUpdate(rm.Token, oldStatus, status) + t.inventoryUpdate(rm.Token, currStatus, status) log.Debugf("Status change %x from %v (%v) to %v (%v)", - token, backend.MDStatus[oldStatus], oldStatus, + token, backend.MDStatus[currStatus], currStatus, backend.MDStatus[status], status) // Return the updated record From 60b9ff044aa9fed4c7fe7777a281368ceb46b731 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 7 Oct 2020 09:47:22 -0500 Subject: [PATCH 142/449] comment route debugging --- politeiad/backend/tlogbe/comments.go | 49 ++-- politeiad/backend/tlogbe/pi.go | 358 ++++++++++++++++++++++++- politeiad/backend/tlogbe/ticketvote.go | 4 +- politeiad/plugins/comments/comments.go | 26 +- politeiad/plugins/pi/pi.go | 7 +- politeiawww/api/pi/v1/v1.go | 55 ++-- politeiawww/cmd/piwww/commentnew.go | 16 +- politeiawww/cmd/piwww/piwww.go | 57 ++-- politeiawww/eventmanager.go | 1 + politeiawww/piwww.go | 45 +++- politeiawww/politeiad.go | 2 +- politeiawww/www.go | 47 ++-- 12 files changed, 526 insertions(+), 141 deletions(-) diff --git a/politeiad/backend/tlogbe/comments.go b/politeiad/backend/tlogbe/comments.go index 8a441e25c..d21ae2c3e 100644 --- a/politeiad/backend/tlogbe/comments.go +++ b/politeiad/backend/tlogbe/comments.go @@ -27,6 +27,12 @@ import ( "github.com/decred/politeia/util" ) +// TODO holding the lock before verifying the token can allow the mutexes to +// be spammed. Create an infinite amount of them with invalid tokens. The fix +// is to check if the record exists in the mutexes function to ensure a token +// is valid before holding the lock on it. This is where we can return a +// record doesn't exist user error too. + const ( // Blob entry data descriptors dataDescriptorCommentAdd = "commentadd" @@ -764,12 +770,12 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { default: return "", backend.PluginUserError{ PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), + ErrorCode: int(comments.ErrorStatusStateInvalid), } } // Verify token - token, err := hex.DecodeString(n.Token) + token, err := util.ConvertStringToken(n.Token) if err != nil { return "", backend.PluginUserError{ PluginID: comments.ID, @@ -788,8 +794,9 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { // Verify comment if len(n.Comment) > comments.PolicyCommentLengthMax { return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusCommentLengthMax), + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusCommentTextInvalid), + ErrorContext: []string{"exceeds max length"}, } } @@ -896,7 +903,7 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { } // Verify token - token, err := hex.DecodeString(e.Token) + token, err := util.ConvertStringToken(e.Token) if err != nil { return "", backend.PluginUserError{ PluginID: comments.ID, @@ -915,8 +922,9 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { // Verify comment if len(e.Comment) > comments.PolicyCommentLengthMax { return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusCommentLengthMax), + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusCommentTextInvalid), + ErrorContext: []string{"exceeds max length"}, } } @@ -947,12 +955,9 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { // Verify the user ID if e.UserID != existing.UserID { - e := fmt.Sprintf("user id cannot change; got %v, want %v", - e.UserID, existing.UserID) return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusUserIDInvalid), - ErrorContext: []string{e}, + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusUserUnauthorized), } } @@ -970,8 +975,9 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { // Verify comment changes if e.Comment == existing.Comment { return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusNoCommentChanges), + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusCommentTextInvalid), + ErrorContext: []string{"comment did not change"}, } } @@ -1049,7 +1055,7 @@ func (p *commentsPlugin) cmdDel(payload string) (string, error) { } // Verify token - token, err := hex.DecodeString(d.Token) + token, err := util.ConvertStringToken(d.Token) if err != nil { return "", backend.PluginUserError{ PluginID: comments.ID, @@ -1093,6 +1099,7 @@ func (p *commentsPlugin) cmdDel(payload string) (string, error) { // Prepare comment delete receipt := p.identity.SignMessage([]byte(d.Signature)) cd := comments.CommentDel{ + State: d.State, Token: d.Token, CommentID: d.CommentID, Reason: d.Reason, @@ -1177,7 +1184,7 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { } // Verify token - token, err := hex.DecodeString(v.Token) + token, err := util.ConvertStringToken(v.Token) if err != nil { return "", backend.PluginUserError{ PluginID: comments.ID, @@ -1338,7 +1345,7 @@ func (p *commentsPlugin) cmdGet(payload string) (string, error) { } // Verify token - token, err := hex.DecodeString(g.Token) + token, err := util.ConvertStringToken(g.Token) if err != nil { return "", backend.PluginUserError{ PluginID: comments.ID, @@ -1397,7 +1404,7 @@ func (p *commentsPlugin) cmdGetAll(payload string) (string, error) { } // Verify token - token, err := hex.DecodeString(ga.Token) + token, err := util.ConvertStringToken(ga.Token) if err != nil { return "", backend.PluginUserError{ PluginID: comments.ID, @@ -1473,7 +1480,7 @@ func (p *commentsPlugin) cmdGetVersion(payload string) (string, error) { } // Verify token - token, err := hex.DecodeString(gv.Token) + token, err := util.ConvertStringToken(gv.Token) if err != nil { return "", backend.PluginUserError{ PluginID: comments.ID, @@ -1560,7 +1567,7 @@ func (p *commentsPlugin) cmdCount(payload string) (string, error) { } // Verify token - token, err := hex.DecodeString(c.Token) + token, err := util.ConvertStringToken(c.Token) if err != nil { return "", backend.PluginUserError{ PluginID: comments.ID, @@ -1607,7 +1614,7 @@ func (p *commentsPlugin) cmdVotes(payload string) (string, error) { } // Verify token - token, err := hex.DecodeString(v.Token) + token, err := util.ConvertStringToken(v.Token) if err != nil { return "", backend.PluginUserError{ PluginID: comments.ID, diff --git a/politeiad/backend/tlogbe/pi.go b/politeiad/backend/tlogbe/pi.go index 0b6f7a4ac..8f7087ce6 100644 --- a/politeiad/backend/tlogbe/pi.go +++ b/politeiad/backend/tlogbe/pi.go @@ -189,8 +189,7 @@ func (p *piPlugin) cmdProposals(payload string) (string, error) { return "", err } - // Setup the returned map with entries for all tokens that - // correspond to records. + // Verify state var existsFn func([]byte) bool switch ps.State { case pi.PropStateUnvetted: @@ -204,6 +203,8 @@ func (p *piPlugin) cmdProposals(payload string) (string, error) { } } + // Setup the returned map with entries for all tokens that + // correspond to records. // map[token]ProposalPluginData proposals := make(map[string]pi.ProposalPluginData, len(ps.Tokens)) for _, v := range ps.Tokens { @@ -282,21 +283,358 @@ func (p *piPlugin) cmdProposals(payload string) (string, error) { return string(reply), nil } +func (p *piPlugin) voteSummary(token string) (*ticketvote.Summary, error) { + s := ticketvote.Summaries{ + Tokens: []string{token}, + } + b, err := ticketvote.EncodeSummaries(s) + if err != nil { + return nil, err + } + r, err := p.backend.Plugin(ticketvote.ID, + ticketvote.CmdSummaries, "", string(b)) + if err != nil { + return nil, err + } + sr, err := ticketvote.DecodeSummariesReply([]byte(r)) + if err != nil { + return nil, err + } + summary, ok := sr.Summaries[token] + if !ok { + return nil, fmt.Errorf("proposal not found %v", token) + } + return &summary, nil +} + func (p *piPlugin) cmdCommentNew(payload string) (string, error) { - // TODO - // Only allow commenting on vetted - return "", nil + cn, err := pi.DecodeCommentNew([]byte(payload)) + if err != nil { + return "", err + } + + // Verifying the state, token, and that the record exists is also + // done in the comments plugin but we duplicate it here so that the + // vote summary request can complete successfully. + + // Verify state + switch cn.State { + case pi.PropStateUnvetted, pi.PropStateVetted: + // Allowed; continue + default: + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropStateInvalid), + } + } + + // Verify token + token, err := util.ConvertStringToken(cn.Token) + if err != nil { + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropTokenInvalid), + } + } + + // Verify record exists + var exists bool + switch cn.State { + case pi.PropStateUnvetted: + exists = p.backend.UnvettedExists(token) + case pi.PropStateVetted: + exists = p.backend.VettedExists(token) + default: + // Should not happen. State has already been validated. + return "", fmt.Errorf("invalid state %v", cn.State) + } + if !exists { + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropNotFound), + } + } + + // Verify vote status + if cn.State == pi.PropStateVetted { + vs, err := p.voteSummary(cn.Token) + if err != nil { + return "", fmt.Errorf("voteSummary: %v", err) + } + switch vs.Status { + case ticketvote.VoteStatusUnauthorized, ticketvote.VoteStatusAuthorized, + ticketvote.VoteStatusStarted: + // Comments are allowed on these vote statuses; continue + default: + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusVoteStatusInvalid), + ErrorContext: []string{"vote has ended; proposal is locked"}, + } + } + } + + // Setup plugin command + n := comments.New{ + UserID: cn.UserID, + State: comments.StateT(cn.State), + Token: cn.Token, + ParentID: cn.ParentID, + Comment: cn.Comment, + PublicKey: cn.PublicKey, + Signature: cn.Signature, + } + b, err := comments.EncodeNew(n) + if err != nil { + return "", err + } + + // Send plugin command + r, err := p.backend.Plugin(comments.ID, comments.CmdNew, "", string(b)) + if err != nil { + return "", err + } + + // Prepare reply + nr, err := comments.DecodeNewReply([]byte(r)) + if err != nil { + return "", err + } + cnr := pi.CommentNewReply{ + CommentID: nr.CommentID, + Timestamp: nr.Timestamp, + Receipt: nr.Receipt, + } + reply, err := pi.EncodeCommentNewReply(cnr) + if err != nil { + return "", err + } + + return string(reply), nil } func (p *piPlugin) cmdCommentCensor(payload string) (string, error) { - // TODO - return "", nil + cc, err := pi.DecodeCommentCensor([]byte(payload)) + if err != nil { + return "", err + } + + // Verifying the state, token, and that the record exists is also + // done in the comments plugin but we duplicate it here so that the + // vote summary request can complete successfully. + + // Verify state + switch cc.State { + case pi.PropStateUnvetted, pi.PropStateVetted: + // Allowed; continue + default: + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropStateInvalid), + } + } + + // Verify token + token, err := util.ConvertStringToken(cc.Token) + if err != nil { + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropTokenInvalid), + } + } + + // Verify record exists + var exists bool + switch cc.State { + case pi.PropStateUnvetted: + exists = p.backend.UnvettedExists(token) + case pi.PropStateVetted: + exists = p.backend.VettedExists(token) + default: + // Should not happen. State has already been validated. + return "", fmt.Errorf("invalid state %v", cc.State) + } + if !exists { + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropNotFound), + } + } + + // Verify vote status + if cc.State == pi.PropStateVetted { + vs, err := p.voteSummary(cc.Token) + if err != nil { + return "", fmt.Errorf("voteSummary: %v", err) + } + switch vs.Status { + case ticketvote.VoteStatusUnauthorized, ticketvote.VoteStatusAuthorized, + ticketvote.VoteStatusStarted: + // Censoring is allowed on these vote statuses; continue + default: + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusVoteStatusInvalid), + ErrorContext: []string{"vote has ended; proposal is locked"}, + } + } + } + + // Setup plugin command + d := comments.Del{ + State: comments.StateT(cc.State), + Token: cc.Token, + CommentID: cc.CommentID, + Reason: cc.Reason, + PublicKey: cc.PublicKey, + Signature: cc.Signature, + } + b, err := comments.EncodeDel(d) + if err != nil { + return "", err + } + + // Send plugin command + r, err := p.backend.Plugin(comments.ID, comments.CmdDel, "", string(b)) + if err != nil { + return "", err + } + + // Prepare reply + dr, err := comments.DecodeDelReply([]byte(r)) + if err != nil { + return "", err + } + ccr := pi.CommentCensorReply{ + Timestamp: dr.Timestamp, + Receipt: dr.Receipt, + } + reply, err := pi.EncodeCommentCensorReply(ccr) + if err != nil { + return "", err + } + + return string(reply), nil } func (p *piPlugin) cmdCommentVote(payload string) (string, error) { - // TODO - // Only allow voting on vetted - return "", nil + cv, err := pi.DecodeCommentVote([]byte(payload)) + if err != nil { + return "", err + } + + // Verifying the state, token, and that the record exists is also + // done in the comments plugin but we duplicate it here so that the + // vote summary request can complete successfully. + + // Verify state + switch cv.State { + case pi.PropStateUnvetted, pi.PropStateVetted: + // Allowed; continue + default: + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropStateInvalid), + } + } + + // Verify token + token, err := util.ConvertStringToken(cv.Token) + if err != nil { + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropTokenInvalid), + } + } + + // Verify record exists + var record *backend.Record + switch cv.State { + case pi.PropStateUnvetted: + record, err = p.backend.GetUnvetted(token, "") + case pi.PropStateVetted: + record, err = p.backend.GetVetted(token, "") + default: + // Should not happen. State has already been validated. + return "", fmt.Errorf("invalid state %v", cv.State) + } + if err != nil { + if err == backend.ErrRecordNotFound { + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropNotFound), + } + } + return "", fmt.Errorf("get record: %v", err) + } + + // Verify record status + status := convertPropStatusFromMDStatus(record.RecordMetadata.Status) + switch status { + case pi.PropStatusPublic: + // Comment votes are only allowed on public proposals; continue + default: + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropStatusInvalid), + ErrorContext: []string{"proposal is not public"}, + } + } + + // Verify vote status + vs, err := p.voteSummary(cv.Token) + if err != nil { + return "", fmt.Errorf("voteSummary: %v", err) + } + switch vs.Status { + case ticketvote.VoteStatusUnauthorized, ticketvote.VoteStatusAuthorized, + ticketvote.VoteStatusStarted: + // Comment votes are allowed on these vote statuses; continue + default: + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusVoteStatusInvalid), + ErrorContext: []string{"vote has ended; proposal is locked"}, + } + } + + // Setup plugin command + v := comments.Vote{ + UserID: cv.UserID, + State: comments.StateT(cv.State), + Token: cv.Token, + CommentID: cv.CommentID, + Vote: comments.VoteT(cv.Vote), + PublicKey: cv.PublicKey, + Signature: cv.Signature, + } + b, err := comments.EncodeVote(v) + if err != nil { + return "", err + } + + // Send plugin command + r, err := p.backend.Plugin(comments.ID, comments.CmdVote, "", string(b)) + if err != nil { + return "", err + } + + // Prepare reply + vr, err := comments.DecodeVoteReply([]byte(r)) + if err != nil { + return "", err + } + cvr := pi.CommentVoteReply{ + Score: vr.Score, + Timestamp: vr.Timestamp, + Receipt: vr.Receipt, + } + reply, err := pi.EncodeCommentVoteReply(cvr) + if err != nil { + return "", err + } + + return string(reply), nil } func (p *piPlugin) cmdVoteInventory(payload string) (string, error) { diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index 8aa71bcf3..783b6d1b5 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -60,8 +60,8 @@ var ( // TODO holding the lock before verifying the token can allow the mutexes to // be spammed. Create an infinite amount of them with invalid tokens. The fix -// is to add an exists() method onto the tlogClient and have the mutexes -// function ensure a token is valid before holding the lock on it. +// is to check if the record exists in the mutexes function to ensure a token +// is valid before holding the lock on it. // TODO the bottleneck for casting a large ballot of votes is waiting for the // log signer. Break the cast votes up and send them concurrently. diff --git a/politeiad/plugins/comments/comments.go b/politeiad/plugins/comments/comments.go index d741ed49b..d2ddf03dc 100644 --- a/politeiad/plugins/comments/comments.go +++ b/politeiad/plugins/comments/comments.go @@ -52,19 +52,18 @@ const ( PolicyVoteChangesMax = 5 // Error status codes - ErrorStatusInvalid ErrorStatusT = 0 - ErrorStatusStateInvalid ErrorStatusT = 1 - ErrorStatusTokenInvalid ErrorStatusT = 2 - ErrorStatusPublicKeyInvalid ErrorStatusT = 3 - ErrorStatusSignatureInvalid ErrorStatusT = 4 - ErrorStatusCommentLengthMax ErrorStatusT = 5 - ErrorStatusRecordNotFound ErrorStatusT = 6 - ErrorStatusCommentNotFound ErrorStatusT = 7 - ErrorStatusUserIDInvalid ErrorStatusT = 8 - ErrorStatusParentIDInvalid ErrorStatusT = 9 - ErrorStatusNoCommentChanges ErrorStatusT = 10 - ErrorStatusVoteInvalid ErrorStatusT = 11 - ErrorStatusVoteChangesMax ErrorStatusT = 12 + ErrorStatusInvalid ErrorStatusT = 0 + ErrorStatusStateInvalid ErrorStatusT = iota + ErrorStatusTokenInvalid + ErrorStatusPublicKeyInvalid + ErrorStatusSignatureInvalid + ErrorStatusCommentTextInvalid + ErrorStatusRecordNotFound + ErrorStatusCommentNotFound + ErrorStatusUserUnauthorized + ErrorStatusParentIDInvalid + ErrorStatusVoteInvalid + ErrorStatusVoteChangesMax ) var ( @@ -77,7 +76,6 @@ var ( ErrorStatusRecordNotFound: "record not found", ErrorStatusCommentNotFound: "comment not found", ErrorStatusParentIDInvalid: "parent id invalid", - ErrorStatusNoCommentChanges: "comment did not change", ErrorStatusVoteInvalid: "invalid vote", ErrorStatusVoteChangesMax: "vote changes max exceeded", } diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index f988ed638..a790da811 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -61,8 +61,11 @@ const ( // User error status codes // TODO number error codes and add human readable error messages - ErrorStatusInvalid ErrorStatusT = iota - ErrorStatusPropStateInvalid + ErrorStatusInvalid ErrorStatusT = 0 + ErrorStatusPropStateInvalid ErrorStatusT = iota + ErrorStatusPropTokenInvalid + ErrorStatusPropNotFound + ErrorStatusPropStatusInvalid ErrorStatusPropVersionInvalid ErrorStatusPropStatusChangeInvalid ErrorStatusPropLinkToInvalid diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 63efa80dd..e3a8c7145 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -20,13 +20,13 @@ type VoteErrorT int // TODO I screwed up comments. A comment should contain fields for upvotes and // downvotes instead of just a overall vote score. // TODO the plugin policies should be returned in a route -// TODO the proposals route should allow filtering by user ID. Actually, this -// is going to have to wait until after the initial release. This is -// non-trivial to accomplish and is outside the scope of the core -// functionality. // TODO show the difference between unvetted censored and vetted censored // in the proposal inventory route since fetching them requires specifying // the state. +// TODO verify that all batched request have a page size limit +// TODO make RouteVoteResults a batched route but that only currently allows +// for 1 result to be returned so that we have the option to change this is +// we want to. const ( APIVersion = 1 @@ -58,11 +58,6 @@ const ( RouteVoteSummaries = "/votes/summaries" RouteVoteInventory = "/votes/inventory" - // TODO implement PolicyProposalPageSize - // PolicyProposalsPageSize is the maximum number of results that can - // be returned from any of the batched proposal commands. - PolicyProposalsPageSize = 10 - // Proposal states. A proposal state can be either unvetted or // vetted. The PropStatusT type further breaks down these two // states into more granular statuses. @@ -115,22 +110,22 @@ const ( VoteTypeRunoff VoteT = 2 // Error status codes - ErrorStatusInvalid ErrorStatusT = 0 - ErrorStatusInputInvalid ErrorStatusT = 1 + ErrorStatusInvalid ErrorStatusT = 0 + ErrorStatusInputInvalid ErrorStatusT = 1 + ErrorStatusPageSizeExceeded ErrorStatusT = 2 // User errors - ErrorStatusUserRegistrationNotPaid ErrorStatusT = 2 - ErrorStatusUserBalanceInsufficient ErrorStatusT = 3 - ErrorStatusUserIsNotAuthor ErrorStatusT = 4 - ErrorStatusUserIsNotAdmin ErrorStatusT = 5 + ErrorStatusUserRegistrationNotPaid ErrorStatusT = 100 + ErrorStatusUserBalanceInsufficient ErrorStatusT = 101 + ErrorStatusUnauthorized ErrorStatusT = 102 // Signature errors - ErrorStatusPublicKeyInvalid ErrorStatusT = 100 - ErrorStatusSignatureInvalid ErrorStatusT = 101 + ErrorStatusPublicKeyInvalid ErrorStatusT = 200 + ErrorStatusSignatureInvalid ErrorStatusT = 201 // Proposal errors // TODO number error codes - ErrorStatusFileCountInvalid ErrorStatusT = 200 + ErrorStatusFileCountInvalid ErrorStatusT = 300 ErrorStatusFileNameInvalid ErrorStatusT = iota ErrorStatusFileMIMEInvalid ErrorStatusFileDigestInvalid @@ -144,17 +139,17 @@ const ( ErrorStatusMetadataCountInvalid ErrorStatusMetadataDigestInvalid ErrorStatusMetadataPayloadInvalid + ErrorStatusPropNotFound ErrorStatusPropMetadataNotFound + ErrorStatusPropTokenInvalid + ErrorStatusPropVersionInvalid ErrorStatusPropNameInvalid ErrorStatusPropLinkToInvalid ErrorStatusPropLinkByInvalid - ErrorStatusPropTokenInvalid - ErrorStatusPropNotFound ErrorStatusPropStateInvalid ErrorStatusPropStatusInvalid ErrorStatusPropStatusChangeInvalid ErrorStatusPropStatusChangeReasonInvalid - ErrorStatusPropPageSizeExceeded ErrorStatusNoPropChanges // Comment errors @@ -168,7 +163,6 @@ const ( ErrorStatusVoteStatusInvalid ErrorStatusVoteParamsInvalid ErrorStatusBallotInvalid - ErrorStatusVotePageSizeExceeded // Cast vote errors // TODO these need human readable equivalents @@ -188,14 +182,14 @@ var ( // ErrorStatus contains human readable error messages. // TODO fill in error status messages ErrorStatus = map[ErrorStatusT]string{ - ErrorStatusInvalid: "error status invalid", - ErrorStatusInputInvalid: "input invalid", + ErrorStatusInvalid: "error status invalid", + ErrorStatusInputInvalid: "input invalid", + ErrorStatusPageSizeExceeded: "page size exceeded", // User errors ErrorStatusUserRegistrationNotPaid: "user registration not paid", ErrorStatusUserBalanceInsufficient: "user balance insufficient", - ErrorStatusUserIsNotAuthor: "user is not author", - ErrorStatusUserIsNotAdmin: "user is not author", + ErrorStatusUnauthorized: "user is unauthorized", // Signature errors ErrorStatusPublicKeyInvalid: "public key invalid", @@ -226,7 +220,6 @@ var ( ErrorStatusPropStatusInvalid: "proposal status invalid", ErrorStatusPropStatusChangeInvalid: "proposal status change invalid", ErrorStatusPropStatusChangeReasonInvalid: "proposal status reason invalid", - ErrorStatusPropPageSizeExceeded: "proposal page size exceeded", ErrorStatusNoPropChanges: "no proposal changes", // Comment errors @@ -237,10 +230,9 @@ var ( ErrorStatusCommentVoteChangesMax: "comment vote changes exceeded max", // Vote errors - ErrorStatusVoteStatusInvalid: "vote status invalid", - ErrorStatusVoteParamsInvalid: "vote params invalid", - ErrorStatusBallotInvalid: "ballot invalid", - ErrorStatusVotePageSizeExceeded: "vote page size exceeded", + ErrorStatusVoteStatusInvalid: "vote status invalid", + ErrorStatusVoteParamsInvalid: "vote params invalid", + ErrorStatusBallotInvalid: "ballot invalid", } ) @@ -480,7 +472,6 @@ type Comment struct { PublicKey string `json:"publickey"` // Public key used for Signature Signature string `json:"signature"` // Client signature CommentID uint32 `json:"commentid"` // Comment ID - Version uint32 `json:"version"` // Comment version Timestamp int64 `json:"timestamp"` // UNIX timestamp of last edit Receipt string `json:"receipt"` // Server sig of client sig Score int64 `json:"score"` // Vote score diff --git a/politeiawww/cmd/piwww/commentnew.go b/politeiawww/cmd/piwww/commentnew.go index bae458116..f10c88175 100644 --- a/politeiawww/cmd/piwww/commentnew.go +++ b/politeiawww/cmd/piwww/commentnew.go @@ -31,9 +31,16 @@ func (c *commentNewCmd) Execute(args []string) error { // Unpack args token := c.Args.Token comment := c.Args.Comment - parentID, err := strconv.ParseUint(c.Args.ParentID, 10, 32) - if err != nil { - return fmt.Errorf("ParseUint(%v): %v", c.Args.ParentID, err) + + var parentID uint64 + var err error + if c.Args.ParentID == "" { + parentID = 0 + } else { + parentID, err = strconv.ParseUint(c.Args.ParentID, 10, 32) + if err != nil { + return fmt.Errorf("ParseUint(%v): %v", c.Args.ParentID, err) + } } // Verify identity @@ -52,7 +59,8 @@ func (c *commentNewCmd) Execute(args []string) error { } // Sign comment data - msg := strconv.Itoa(int(state)) + token + c.Args.ParentID + comment + msg := strconv.Itoa(int(state)) + token + + strconv.FormatUint(uint64(parentID), 10) + comment b := cfg.Identity.SignMessage([]byte(msg)) signature := hex.EncodeToString(b[:]) diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index 73fb8dfd6..25f4231e2 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -53,13 +53,11 @@ type piwww struct { UserPasswordChange shared.UserPasswordChangeCmd `command:"userpasswordchange"` UserUsernameChange shared.UserUsernameChangeCmd `command:"userusernamechange"` UserKeyUpdate shared.UserKeyUpdateCmd `command:"userkeyupdate"` - UserTOTPSet shared.UserTOTPSetCmd `command:"usertotpset"` - UserTOTPVerify shared.UserTOTPVerifyCmd `command:"usertotpverify"` UserRegistrationPayment userRegistrationPaymentCmd `command:"userregistrationpayment"` + UserPaymentsRescan userPaymentsRescanCmd `command:"userpaymentsrescan"` UserProposalPaywall userProposalPaywallCmd `command:"userproposalpaywall"` UserProposalPaywallTx userProposalPaywallTxCmd `command:"userproposalpaywalltx"` UserProposalCredits userProposalCreditsCmd `command:"userproposalcredits"` - UserPaymentsRescan userPaymentsRescanCmd `command:"userpaymentsrescan"` UserDetails userDetailsCmd `command:"userdetails"` Users shared.UsersCmd `command:"users"` @@ -96,7 +94,6 @@ type piwww struct { SendFaucetTx sendFaucetTxCmd `command:"sendfaucettx"` } -// TODO add proposalpaywall to this once the command is updated const helpMsg = `Application Options: --appdata= Path to application home directory --host= politeiawww host @@ -110,12 +107,12 @@ Help commands help Print detailed help message for a command Basic commands - version (public) Get politeiawww server version - policy (public) Get politeiawww server policy - secret (public) Ping the server - login (public) Login to politeiawww - logout (user) Logout from politeiawww - me (user) Get details of the logged in user + version (public) Get politeiawww server version + policy (public) Get politeiawww server policy + secret (public) Ping the server + login (public) Login to politeiawww + logout (user) Logout from politeiawww + me (user) Get details of the logged in user User commands usernew (public) Create a new user @@ -127,39 +124,37 @@ User commands userpasswordchange (user) Change password userusernamechange (user) Change username userkeyupdate (user) Update user key (i.e. identity) - usertotpset (user) Set a TOTP method - usertotpverify (user) Verify a TOTP method userregistrationpayment (user) Verify registration payment + userpaymentsrescan (user) Rescan all user payments userproposalpaywall (user) Get user paywall details userproposalpaywalltx (user) Get pending user payments userproposalcredits (user) Get user proposal credits - userpaymentsrescan (user) Rescan all user payments userdetails (public) Get user details users (public) Get users Proposal commands - proposalnew (user) Submit a new proposal - proposaledit (user) Edit an existing proposal - proposalstatusset (admin) Set the status of a proposal - proposals (public) Get proposals - proposalinv (public) Get proposal inventory by proposal status + proposalnew (user) Submit a new proposal + proposaledit (user) Edit an existing proposal + proposalstatusset (admin) Set the status of a proposal + proposals (public) Get proposals + proposalinv (public) Get proposal inventory by proposal status Comment commands - commentnew (user) Submit a new comment - commentvote (user) Upvote/downvote a comment - commentcensor (admin) Censor a comment - comments (public) Get comments - commentvotes (public) Get comment votes + commentnew (user) Submit a new comment + commentvote (user) Upvote/downvote a comment + commentcensor (admin) Censor a comment + comments (public) Get comments + commentvotes (public) Get comment votes Vote commands - voteauthorize (user) Authorize a proposal vote - votestart (admin) Start a proposal vote - votestartrunoff (admin) Start a runoff vote - voteballot (public) Cast a ballot of votes - votes (public) Get vote details - voteresults (public) Get full vote results - votesummaries (public) Get vote summaries - voteinv (public) Get proposal inventory by vote status + voteauthorize (user) Authorize a proposal vote + votestart (admin) Start a proposal vote + votestartrunoff (admin) Start a runoff vote + voteballot (public) Cast a ballot of votes + votes (public) Get vote details + voteresults (public) Get full vote results + votesummaries (public) Get vote summaries + voteinv (public) Get proposal inventory by vote status Websocket commands subscribe (public) Subscribe/unsubscribe to websocket event diff --git a/politeiawww/eventmanager.go b/politeiawww/eventmanager.go index f0bda1a87..239841899 100644 --- a/politeiawww/eventmanager.go +++ b/politeiawww/eventmanager.go @@ -386,6 +386,7 @@ func (p *politeiawww) notifyParentAuthorOnComment(d dataProposalComment) error { // Get the parent comment // TODO + return nil // Lookup the parent comment author var author *user.User diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index a2a5ffcc1..4cb97a675 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -32,7 +32,6 @@ import ( // TODO use pi policies. Should the policies be defined in the pi plugin // or the pi api spec? // TODO ensure plugins can't write data using short proposal token. -// TODO move proposal validation to pi plugin // TODO politeiad needs batched calls for retrieving unvetted and vetted // records. @@ -368,7 +367,6 @@ func convertCommentFromPlugin(cm comments.Comment) pi.Comment { PublicKey: cm.PublicKey, Signature: cm.Signature, CommentID: cm.CommentID, - Version: cm.Version, Timestamp: cm.Timestamp, Receipt: cm.Receipt, Score: cm.Score, @@ -1298,7 +1296,8 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p // values so the user IDs must be compared directly. if curr.UserID != usr.ID.String() { return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusUserIsNotAuthor, + ErrorCode: pi.ErrorStatusUnauthorized, + ErrorContext: []string{"user is not author"}, } } @@ -1429,7 +1428,8 @@ func (p *politeiawww) processProposalStatusSet(pss pi.ProposalStatusSet, usr use // Verify user is an admin if !usr.Admin { return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusUserIsNotAdmin, + ErrorCode: pi.ErrorStatusUnauthorized, + ErrorContext: []string{"user is not an admin"}, } } @@ -1577,6 +1577,27 @@ func (p *politeiawww) processCommentNew(cn pi.CommentNew, usr user.User) (*pi.Co } } + // Only admins and the proposal author are allowed to comment on + // unvetted proposals. + if cn.State == pi.PropStateUnvetted && !usr.Admin { + // Fetch the proposal so we can see who the author is + pr, err := p.proposalRecordLatest(cn.State, cn.Token) + if err != nil { + if err == errProposalNotFound { + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusPropNotFound, + } + } + return nil, fmt.Errorf("proposalRecordLatest: %v", err) + } + if usr.ID.String() != pr.UserID { + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusUnauthorized, + ErrorContext: []string{"user is not author or admin"}, + } + } + } + // Send plugin command pcn := piplugin.CommentNew{ UserID: usr.ID.String(), @@ -2112,12 +2133,14 @@ func (p *politeiawww) handleCommentVote(w http.ResponseWriter, r *http.Request) if err != nil { respondWithPiError(w, r, "handleCommentVote: getSessionUser: %v", err) + return } vcr, err := p.processCommentVote(cv, *usr) if err != nil { respondWithPiError(w, r, "handleCommentVote: processCommentVote: %v", err) + return } util.RespondWithJSON(w, http.StatusOK, vcr) @@ -2147,6 +2170,7 @@ func (p *politeiawww) handleCommentCensor(w http.ResponseWriter, r *http.Request if err != nil { respondWithPiError(w, r, "handleCommentCensor: processCommentCensor: %v", err) + return } util.RespondWithJSON(w, http.StatusOK, ccr) @@ -2169,6 +2193,7 @@ func (p *politeiawww) handleComments(w http.ResponseWriter, r *http.Request) { if err != nil { respondWithPiError(w, r, "handleCommentVote: processComments: %v", err) + return } util.RespondWithJSON(w, http.StatusOK, cr) @@ -2191,6 +2216,7 @@ func (p *politeiawww) handleCommentVotes(w http.ResponseWriter, r *http.Request) if err != nil { respondWithPiError(w, r, "handleCommentVotes: processCommentVotes: %v", err) + return } util.RespondWithJSON(w, http.StatusOK, cvr) @@ -2213,12 +2239,14 @@ func (p *politeiawww) handleVoteAuthorize(w http.ResponseWriter, r *http.Request if err != nil { respondWithPiError(w, r, "handleVoteAuthorize: getSessionUser: %v", err) + return } vr, err := p.processVoteAuthorize(va, *usr) if err != nil { respondWithPiError(w, r, "handleVoteAuthorize: processVoteAuthorize: %v", err) + return } util.RespondWithJSON(w, http.StatusOK, vr) @@ -2241,12 +2269,14 @@ func (p *politeiawww) handleVoteStart(w http.ResponseWriter, r *http.Request) { if err != nil { respondWithPiError(w, r, "handleVoteStart: getSessionUser: %v", err) + return } vsr, err := p.processVoteStart(vs, *usr) if err != nil { respondWithPiError(w, r, "handleVoteStart: processVoteStart: %v", err) + return } util.RespondWithJSON(w, http.StatusOK, vsr) @@ -2269,12 +2299,14 @@ func (p *politeiawww) handleVoteStartRunoff(w http.ResponseWriter, r *http.Reque if err != nil { respondWithPiError(w, r, "handleVoteStartRunoff: getSessionUser: %v", err) + return } vsrr, err := p.processVoteStartRunoff(vsr, *usr) if err != nil { respondWithPiError(w, r, "handleVoteStartRunoff: processVoteStartRunoff: %v", err) + return } util.RespondWithJSON(w, http.StatusOK, vsrr) @@ -2297,6 +2329,7 @@ func (p *politeiawww) handleVoteBallot(w http.ResponseWriter, r *http.Request) { if err != nil { respondWithPiError(w, r, "handleVoteBallot: processVoteBallot: %v", err) + return } util.RespondWithJSON(w, http.StatusOK, vbr) @@ -2319,6 +2352,7 @@ func (p *politeiawww) handleVotes(w http.ResponseWriter, r *http.Request) { if err != nil { respondWithPiError(w, r, "handleVotes: processVotes: %v", err) + return } util.RespondWithJSON(w, http.StatusOK, vr) @@ -2341,6 +2375,7 @@ func (p *politeiawww) handleVoteResults(w http.ResponseWriter, r *http.Request) if err != nil { respondWithPiError(w, r, "handleVoteResults: prcoessVoteResults: %v", err) + return } util.RespondWithJSON(w, http.StatusOK, vrr) @@ -2363,6 +2398,7 @@ func (p *politeiawww) handleVoteSummaries(w http.ResponseWriter, r *http.Request if err != nil { respondWithPiError(w, r, "handleVoteSummaries: processVoteSummaries: %v", err) + return } util.RespondWithJSON(w, http.StatusOK, vsr) @@ -2385,6 +2421,7 @@ func (p *politeiawww) handleVoteInventory(w http.ResponseWriter, r *http.Request if err != nil { respondWithPiError(w, r, "handleVoteInventory: processVoteInventory: %v", err) + return } util.RespondWithJSON(w, http.StatusOK, vir) diff --git a/politeiawww/politeiad.go b/politeiawww/politeiad.go index ce02aa595..ed8e43d8c 100644 --- a/politeiawww/politeiad.go +++ b/politeiawww/politeiad.go @@ -21,7 +21,7 @@ import ( type pdErrorReply struct { ErrorCode int ErrorContext []string - PluginID string + Plugin string } // pdError represents a politeiad error. diff --git a/politeiawww/www.go b/politeiawww/www.go index 51d7842a4..7c028338e 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -96,23 +96,12 @@ func convertWWWErrorStatusFromComments(e comments.ErrorStatusT) www.ErrorStatusT switch e { case comments.ErrorStatusTokenInvalid: return www.ErrorStatusInvalidCensorshipToken - case comments.ErrorStatusPublicKeyInvalid: - return www.ErrorStatusInvalidPublicKey - case comments.ErrorStatusSignatureInvalid: - return www.ErrorStatusInvalidSignature case comments.ErrorStatusRecordNotFound: return www.ErrorStatusProposalNotFound case comments.ErrorStatusCommentNotFound: return www.ErrorStatusCommentNotFound case comments.ErrorStatusParentIDInvalid: return www.ErrorStatusCommentNotFound - case comments.ErrorStatusNoCommentChanges: - // Intentionally omitted. The www API does not allow for comment - // changes. - case comments.ErrorStatusVoteInvalid: - return www.ErrorStatusInvalidLikeCommentAction - case comments.ErrorStatusVoteChangesMax: - return www.ErrorStatusInvalidLikeCommentAction } return www.ErrorStatusInvalid } @@ -197,30 +186,48 @@ func convertPiErrorStatusFromPD(e pd.ErrorStatusT) pi.ErrorStatusT { func convertPiErrorStatusFromPiPlugin(e piplugin.ErrorStatusT) pi.ErrorStatusT { switch e { + case piplugin.ErrorStatusPropStateInvalid: + return pi.ErrorStatusPropStateInvalid + case piplugin.ErrorStatusPropTokenInvalid: + return pi.ErrorStatusPropTokenInvalid + case piplugin.ErrorStatusPropNotFound: + return pi.ErrorStatusPropNotFound + case piplugin.ErrorStatusPropStatusInvalid: + return pi.ErrorStatusPropStatusInvalid + case piplugin.ErrorStatusPropVersionInvalid: + return pi.ErrorStatusPropVersionInvalid + case piplugin.ErrorStatusPropStatusChangeInvalid: + return pi.ErrorStatusPropStatusChangeInvalid case piplugin.ErrorStatusPropLinkToInvalid: return pi.ErrorStatusPropLinkToInvalid case piplugin.ErrorStatusVoteStatusInvalid: return pi.ErrorStatusVoteStatusInvalid + case piplugin.ErrorStatusPageSizeExceeded: + return pi.ErrorStatusPageSizeExceeded } return pi.ErrorStatusInvalid } func convertPiErrorStatusFromComments(e comments.ErrorStatusT) pi.ErrorStatusT { switch e { + case comments.ErrorStatusStateInvalid: + return pi.ErrorStatusPropStateInvalid case comments.ErrorStatusTokenInvalid: return pi.ErrorStatusPropTokenInvalid case comments.ErrorStatusPublicKeyInvalid: return pi.ErrorStatusPublicKeyInvalid case comments.ErrorStatusSignatureInvalid: return pi.ErrorStatusSignatureInvalid + case comments.ErrorStatusCommentTextInvalid: + return pi.ErrorStatusCommentTextInvalid case comments.ErrorStatusRecordNotFound: return pi.ErrorStatusPropNotFound case comments.ErrorStatusCommentNotFound: return pi.ErrorStatusCommentNotFound + case comments.ErrorStatusUserUnauthorized: + return pi.ErrorStatusUnauthorized case comments.ErrorStatusParentIDInvalid: return pi.ErrorStatusCommentParentIDInvalid - case comments.ErrorStatusNoCommentChanges: - return pi.ErrorStatusCommentTextInvalid case comments.ErrorStatusVoteInvalid: return pi.ErrorStatusCommentVoteInvalid case comments.ErrorStatusVoteChangesMax: @@ -251,11 +258,11 @@ func convertPiErrorStatusFromTicketVote(e ticketvote.ErrorStatusT) pi.ErrorStatu return pi.ErrorStatusInvalid } -// convertPiErrorStatus attempts to convert the provided politeiad plugin ID -// and error code into a pi ErrorStatusT. If a plugin ID is provided the error -// code is assumed to be a user error code from the specified plugin API. If -// no plugin ID is provided the error code is assumed to be a user error code -// from the politeiad API. +// convertPiErrorStatus attempts to convert the provided politeiad error code +// into a pi ErrorStatusT. If a plugin ID is provided the error code is assumed +// to be a user error code from the specified plugin API. If no plugin ID is +// provided the error code is assumed to be a user error code from the +// politeiad API. func convertPiErrorStatus(pluginID string, errCode int) pi.ErrorStatusT { switch pluginID { case "": @@ -352,7 +359,7 @@ func respondWithPiError(w http.ResponseWriter, r *http.Request, format string, e var pdErr pdError if errors.As(err, &pdErr) { var ( - pluginID = pdErr.ErrorReply.PluginID + pluginID = pdErr.ErrorReply.Plugin errCode = pdErr.ErrorReply.ErrorCode errContext = pdErr.ErrorReply.ErrorContext ) @@ -471,7 +478,7 @@ func RespondWithError(w http.ResponseWriter, r *http.Request, userHttpCode int, // Check for politeiad error if pdError, ok := args[0].(pdError); ok { var ( - pluginID = pdError.ErrorReply.PluginID + pluginID = pdError.ErrorReply.Plugin errCode = pdError.ErrorReply.ErrorCode errContext = pdError.ErrorReply.ErrorContext ) From aefb437f2480f1181ff1d7676f3d129120a3254b Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 7 Oct 2020 19:58:36 -0500 Subject: [PATCH 143/449] finish comment route debugging --- politeiad/backend/tlogbe/comments.go | 125 ++++++++++++++----------- politeiad/backend/tlogbe/pi.go | 3 +- politeiad/backend/tlogbe/tlog.go | 32 +++---- politeiad/plugins/comments/comments.go | 6 +- politeiad/plugins/pi/pi.go | 3 +- politeiawww/api/pi/v1/v1.go | 25 +++-- politeiawww/cmd/piwww/README.md | 89 ++++++++++-------- politeiawww/cmd/piwww/commentnew.go | 2 +- politeiawww/cmd/piwww/commentvote.go | 2 +- politeiawww/cmd/piwww/commentvotes.go | 34 +++++-- politeiawww/eventmanager.go | 50 +++++----- politeiawww/piwww.go | 98 +++++++++++++++---- politeiawww/plugin.go | 3 +- 13 files changed, 300 insertions(+), 172 deletions(-) diff --git a/politeiad/backend/tlogbe/comments.go b/politeiad/backend/tlogbe/comments.go index d21ae2c3e..0749e9dca 100644 --- a/politeiad/backend/tlogbe/comments.go +++ b/politeiad/backend/tlogbe/comments.go @@ -84,12 +84,11 @@ type voteIndex struct { } type commentIndex struct { - Adds map[uint32][]byte `json:"adds"` // [version]merkleHash - Del []byte `json:"del"` // Merkle hash of delete record - Score int64 `json:"score"` // Vote score + Adds map[uint32][]byte `json:"adds"` // [version]merkleHash + Del []byte `json:"del"` // Merkle hash of delete record // Votes contains the vote history for each uuid that voted on the - // comment. This data is memoized because the effect of a new vote + // comment. This data is cached because the effect of a new vote // on a comment depends on the previous vote from that uuid. // Example, a user upvotes a comment that they have already // upvoted, the resulting vote score is 0 due to the second upvote @@ -400,9 +399,9 @@ func convertCommentVoteFromBlobEntry(be store.BlobEntry) (*comments.CommentVote, } func convertCommentFromCommentAdd(ca comments.CommentAdd) comments.Comment { - // Score needs to be filled in separately return comments.Comment{ UserID: ca.UserID, + State: ca.State, Token: ca.Token, ParentID: ca.ParentID, Comment: ca.Comment, @@ -412,7 +411,8 @@ func convertCommentFromCommentAdd(ca comments.CommentAdd) comments.Comment { Version: ca.Version, Timestamp: ca.Timestamp, Receipt: ca.Receipt, - Score: 0, + Downvotes: 0, // Not part of commentAdd data + Upvotes: 0, // Not part of commentAdd data Deleted: false, Reason: "", } @@ -422,6 +422,7 @@ func convertCommentFromCommentDel(cd comments.CommentDel) comments.Comment { // Score needs to be filled in separately return comments.Comment{ UserID: cd.UserID, + State: cd.State, Token: cd.Token, ParentID: cd.ParentID, Comment: "", @@ -430,7 +431,8 @@ func convertCommentFromCommentDel(cd comments.CommentDel) comments.Comment { Version: 0, Timestamp: cd.Timestamp, Receipt: "", - Score: 0, + Downvotes: 0, + Upvotes: 0, Deleted: true, Reason: cd.Reason, } @@ -743,7 +745,11 @@ func (p *commentsPlugin) comments(s comments.StateT, token []byte, idx commentsI cs := make(map[uint32]comments.Comment, len(commentIDs)) for _, v := range adds { c := convertCommentFromCommentAdd(v) - c.Score = idx.Comments[c.CommentID].Score + cidx, ok := idx.Comments[c.CommentID] + if !ok { + return nil, fmt.Errorf("comment index not found %v", c.CommentID) + } + c.Downvotes, c.Upvotes = calcVoteScore(cidx) cs[v.CommentID] = c } for _, v := range dels { @@ -1163,6 +1169,57 @@ func (p *commentsPlugin) cmdDel(payload string) (string, error) { return string(reply), nil } +// calcVoteScore returns the vote score for the provided comment index. The +// returned values are the downvotes and upvotes, respectively. +func calcVoteScore(cidx commentIndex) (uint64, uint64) { + // Find the vote score by replaying all existing votes from all + // users. The net effect of a new vote on a comment score depends + // on the previous vote from that uuid. Example, a user upvotes a + // comment that they have already upvoted, the resulting vote score + // is 0 due to the second upvote removing the original upvote. + var upvotes uint64 + var downvotes uint64 + for _, votes := range cidx.Votes { + // Calculate the vote score that this user is contributing. This + // can only ever be -1, 0, or 1. + var score int64 + for _, v := range votes { + vote := int64(v.Vote) + switch { + case score == 0: + // No previous vote. New vote becomes the score. + score = vote + + case score == vote: + // New vote is the same as the previous vote. The vote gets + // removed from the score, making the score 0. + score = 0 + + case score != vote: + // New vote is different than the previous vote. New vote + // becomes the score. + score = vote + } + } + + // Add the net result of all votes from this user to the totals. + switch score { + case 0: + // Nothing to do + case -1: + downvotes++ + case 1: + upvotes++ + default: + // Something went wrong + e := fmt.Errorf("unexpected vote score %v", score) + panic(e) + } + } + + return downvotes, upvotes +} + func (p *commentsPlugin) cmdVote(payload string) (string, error) { log.Tracef("comments cmdVote: %v", payload) @@ -1265,6 +1322,7 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { // Prepare comment vote receipt := p.identity.SignMessage([]byte(v.Signature)) cv := comments.CommentVote{ + State: v.State, UserID: v.UserID, Token: v.Token, CommentID: v.CommentID, @@ -1298,9 +1356,6 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { }) cidx.Votes[cv.UserID] = votes - // Update the comment vote score - cidx.Score = calcVoteScore(cidx, cv) - // Update the comments index idx.Comments[cv.CommentID] = cidx @@ -1310,11 +1365,15 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { return "", err } + // Calculate the new vote scores + downvotes, upvotes := calcVoteScore(cidx) + // Prepare reply vr := comments.VoteReply{ + Downvotes: downvotes, + Upvotes: upvotes, Timestamp: cv.Timestamp, Receipt: cv.Receipt, - Score: cidx.Score, } reply, err := comments.EncodeVoteReply(vr) if err != nil { @@ -1532,7 +1591,7 @@ func (p *commentsPlugin) cmdGetVersion(payload string) (string, error) { // Convert to a comment c := convertCommentFromCommentAdd(adds[0]) - c.Score = cidx.Score + c.Downvotes, c.Upvotes = calcVoteScore(cidx) // Prepare reply gvr := comments.GetVersionReply{ @@ -1662,46 +1721,6 @@ func (p *commentsPlugin) cmdVotes(payload string) (string, error) { return string(reply), nil } -// calcVoteScore returns the updated vote score after the provided CommentVote -// has been added to it. -func calcVoteScore(cidx commentIndex, cv comments.CommentVote) int64 { - // Get the previous vote that this uuid made - var votePrev comments.VoteT - votes, ok := cidx.Votes[cv.UserID] - if !ok && len(votes) != 0 { - votePrev = votes[len(votes)-1].Vote - } - - // Get the existing score - score := cidx.Score - - // Update vote score. The effect of a new vote on a comment score - // depends on the previous vote from that uuid. Example, a user - // upvotes a comment that they have already upvoted, the resulting - // vote score is 0 due to the second upvote removing the original - // upvote. - voteNew := cv.Vote - switch { - case votePrev == 0: - // No previous vote. Add the new vote to the score. - score += int64(voteNew) - - case voteNew == votePrev: - // New vote is the same as the previous vote. Remove the previous - // vote from the score. - score -= int64(votePrev) - - case voteNew != votePrev: - // New vote is different than the previous vote. Remove the - // previous vote from the score and add the new vote to the - // score. - score -= int64(votePrev) - score += int64(voteNew) - } - - return score -} - // cmd executes a plugin command. // // This function satisfies the pluginClient interface. diff --git a/politeiad/backend/tlogbe/pi.go b/politeiad/backend/tlogbe/pi.go index 8f7087ce6..ae0d4f4d7 100644 --- a/politeiad/backend/tlogbe/pi.go +++ b/politeiad/backend/tlogbe/pi.go @@ -625,7 +625,8 @@ func (p *piPlugin) cmdCommentVote(payload string) (string, error) { return "", err } cvr := pi.CommentVoteReply{ - Score: vr.Score, + Downvotes: vr.Downvotes, + Upvotes: vr.Upvotes, Timestamp: vr.Timestamp, Receipt: vr.Receipt, } diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index b161d4a06..1705846b1 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -425,7 +425,7 @@ func convertAnchorFromBlobEntry(be store.BlobEntry) (*anchor, error) { } func (t *tlog) treeNew() (int64, error) { - log.Tracef("tlog treeNew") + log.Tracef("%v treeNew", t.id) tree, _, err := t.trillian.treeNew() if err != nil { @@ -436,7 +436,7 @@ func (t *tlog) treeNew() (int64, error) { } func (t *tlog) treeExists(treeID int64) bool { - log.Tracef("tlog treeExists: %v", treeID) + log.Tracef("%v treeExists: %v", t.id, treeID) _, err := t.trillian.tree(treeID) return err == nil @@ -450,7 +450,7 @@ func (t *tlog) treeExists(treeID int64) bool { // update the status of the tree to frozen in trillian, at which point trillian // will not allow any additional leaves to be appended onto the tree. func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, fr freezeRecord) error { - log.Tracef("tlog treeFreeze: %v") + log.Tracef("%v treeFreeze: %v", t.id, treeID) // Save metadata idx, err := t.metadataSave(treeID, rm, metadata) @@ -528,7 +528,7 @@ func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []ba // freeze record returns the freeze record of the provided tree if one exists. // If one does not exists a errFreezeRecordNotFound error is returned. func (t *tlog) freezeRecord(treeID int64) (*freezeRecord, error) { - log.Tracef("tlog freezeRecord: %v", treeID) + log.Tracef("%v freezeRecord: %v", t.id, treeID) // Check if the tree contains a freeze record. The last two leaves // are checked because the last leaf will be the final anchor drop, @@ -550,7 +550,7 @@ func (t *tlog) freezeRecord(treeID int64) (*freezeRecord, error) { startIndex = 0 count = 1 default: - startIndex = int64(lr.TreeSize) - 1 + startIndex = int64(lr.TreeSize) - 2 count = 2 } @@ -996,7 +996,7 @@ func recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backend.RecordMe // to the trillian tree for each blob, then updates the record index with the // trillian leaf information and returns it. func (t *tlog) recordBlobsSave(treeID int64, rbpr recordBlobsPrepareReply) (*recordIndex, error) { - log.Tracef("recordBlobsSave: %v", treeID) + log.Tracef("recordBlobsSave: %v", t.id, treeID) var ( index = rbpr.recordIndex @@ -1081,7 +1081,7 @@ func (t *tlog) recordBlobsSave(treeID int64, rbpr recordBlobsPrepareReply) (*rec // the record contents have been successfully saved to tlog, a recordIndex is // created for this version of the record and saved to tlog as well. func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { - log.Tracef("tlog recordSave: %v", treeID) + log.Tracef("%v recordSave: %v", t.id, treeID) // Verify tree exists if !t.treeExists(treeID) { @@ -1278,7 +1278,7 @@ func (t *tlog) metadataSave(treeID int64, rm backend.RecordMetadata, metadata [] // metadata has been successfully saved to tlog, a recordIndex is created for // this iteration of the record and saved to tlog as well. func (t *tlog) recordMetadataSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { - log.Tracef("tlog recordMetadataSave: %v", treeID) + log.Tracef("%v recordMetadataSave: %v", t.id, treeID) // Save metadata idx, err := t.metadataSave(treeID, rm, metadata) @@ -1300,7 +1300,7 @@ func (t *tlog) recordMetadataSave(treeID int64, rm backend.RecordMetadata, metad // iterations of the record. Record metadata and metadata stream blobs are not // deleted. func (t *tlog) recordDel(treeID int64) error { - log.Tracef("tlog recordDel: %v", treeID) + log.Tracef("%v recordDel: %v", t.id, treeID) // Verify tree exists if !t.treeExists(treeID) { @@ -1356,7 +1356,7 @@ func (t *tlog) recordDel(treeID int64) error { } func (t *tlog) record(treeID int64, version uint32) (*backend.Record, error) { - log.Tracef("tlog record: %v %v", treeID, version) + log.Tracef("%v record: %v %v", t.id, treeID, version) // Verify tree exists if !t.treeExists(treeID) { @@ -1523,7 +1523,7 @@ func (t *tlog) record(treeID int64, version uint32) (*backend.Record, error) { } func (t *tlog) recordLatest(treeID int64) (*backend.Record, error) { - log.Tracef("tlog recordLatest: %v", treeID) + log.Tracef("%v recordLatest: %v", t.id, treeID) return t.record(treeID, 0) } @@ -1537,7 +1537,7 @@ func (t *tlog) recordProof(treeID int64, version uint32) {} // // This function satisfies the tlogClient interface. func (t *tlog) blobsSave(treeID int64, keyPrefix string, blobs, hashes [][]byte, encrypt bool) ([][]byte, error) { - log.Tracef("tlog blobsSave: %v %v %v", treeID, keyPrefix, encrypt) + log.Tracef("%v blobsSave: %v %v %v", t.id, treeID, keyPrefix, encrypt) // Verify tree exists if !t.treeExists(treeID) { @@ -1620,7 +1620,7 @@ func (t *tlog) blobsSave(treeID int64, keyPrefix string, blobs, hashes [][]byte, // // This function satisfies the tlogClient interface. func (t *tlog) blobsDel(treeID int64, merkles [][]byte) error { - log.Tracef("tlog blobsDel: %v", treeID) + log.Tracef("%v blobsDel: %v", t.id, treeID) // Verify tree exists. We allow blobs to be deleted from both // frozen and non frozen trees. @@ -1673,7 +1673,7 @@ func (t *tlog) blobsDel(treeID int64, merkles [][]byte) error { // // This function satisfies the tlogClient interface. func (t *tlog) blobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, error) { - log.Tracef("tlog blobsByMerkle: %v", treeID) + log.Tracef("%v blobsByMerkle: %v", t.id, treeID) // Verify tree exists if !t.treeExists(treeID) { @@ -1762,7 +1762,7 @@ func (t *tlog) blobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, // // This function satisfies the tlogClient interface. func (t *tlog) blobsByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) { - log.Tracef("tlog blobsByKeyPrefix: %v %v", treeID, keyPrefix) + log.Tracef("%v blobsByKeyPrefix: %v %v", t.id, treeID, keyPrefix) // Verify tree exists if !t.treeExists(treeID) { @@ -1833,7 +1833,7 @@ func (t *tlog) fsck() { } func (t *tlog) close() { - log.Tracef("tlog close: %v", t.id) + log.Tracef("%v close", t.id) // Close connections t.store.Close() diff --git a/politeiad/plugins/comments/comments.go b/politeiad/plugins/comments/comments.go index d2ddf03dc..09fe2279a 100644 --- a/politeiad/plugins/comments/comments.go +++ b/politeiad/plugins/comments/comments.go @@ -96,7 +96,8 @@ type Comment struct { Version uint32 `json:"version"` // Comment version Timestamp int64 `json:"timestamp"` // UNIX timestamp of last edit Receipt string `json:"receipt"` // Server signature of client signature - Score int64 `json:"score"` // Vote score + Downvotes uint64 `json:"downvotes"` // Tolal downvotes on comment + Upvotes uint64 `json:"upvotes"` // Total upvotes on comment Deleted bool `json:"deleted"` // Comment has been deleted Reason string `json:"reason"` // Reason for deletion } @@ -352,7 +353,8 @@ func DecodeVote(payload []byte) (*Vote, error) { // VoteReply is the reply to the Vote command. type VoteReply struct { - Score int64 `json:"score"` // Overall comment vote score + Downvotes uint64 `json:"downvotes"` // Tolal downvotes on comment + Upvotes uint64 `json:"upvotes"` // Total upvotes on comment Timestamp int64 `json:"timestamp"` // Received UNIX timestamp Receipt string `json:"receipt"` // Server signature of client signature } diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index a790da811..86bbd17cc 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -399,7 +399,8 @@ func DecodeCommentVote(payload []byte) (*CommentVote, error) { // CommentVoteReply is the reply to the CommentVote command. type CommentVoteReply struct { - Score int64 `json:"score"` // Overall comment vote score + Downvotes uint64 `json:"downvotes"` // Tolal downvotes on comment + Upvotes uint64 `json:"upvotes"` // Total upvotes on comment Timestamp int64 `json:"timestamp"` // Received UNIX timestamp Receipt string `json:"receipt"` // Server signature of client signature } diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index e3a8c7145..a6eab59c0 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -17,8 +17,6 @@ type VoteAuthActionT string type VoteT int type VoteErrorT int -// TODO I screwed up comments. A comment should contain fields for upvotes and -// downvotes instead of just a overall vote score. // TODO the plugin policies should be returned in a route // TODO show the difference between unvetted censored and vetted censored // in the proposal inventory route since fetching them requires specifying @@ -474,12 +472,15 @@ type Comment struct { CommentID uint32 `json:"commentid"` // Comment ID Timestamp int64 `json:"timestamp"` // UNIX timestamp of last edit Receipt string `json:"receipt"` // Server sig of client sig - Score int64 `json:"score"` // Vote score - Deleted bool `json:"deleted"` // Comment has been deleted - Reason string `json:"reason"` // Reason for deletion + Downvotes uint64 `json:"downvotes"` // Tolal downvotes + Upvotes uint64 `json:"upvotes"` // Total upvotes + + Censored bool `json:"censored,omitempty"` // Comment has been censored + Reason string `json:"reason,omitempty"` // Reason for censoring } -// CommentNew creates a new comment. +// CommentNew creates a new comment. Only the proposal author and admins can +// comment on unvetted proposals. All users can comment on public proposals. // // The parent ID is used to reply to an existing comment. A parent ID of 0 // indicates that the comment is a base level comment and not a reply commment. @@ -527,7 +528,8 @@ type CommentCensorReply struct { Receipt string `json:"receipt"` } -// CommentVote casts a comment vote (upvote or downvote). +// CommentVote casts a comment vote (upvote or downvote). Only allowed on +// vetted proposals. // // The effect of a new vote on a comment score depends on the previous vote // from that uuid. Example, a user upvotes a comment that they have already @@ -549,12 +551,15 @@ type CommentVote struct { // Receipt is the server signature of the client signature. This is proof that // the server received and processed the CommentVote command. type CommentVoteReply struct { - Score int64 `json:"score"` // Overall comment vote score + Downvotes uint64 `json:"downvotes"` // Total downvotes + Upvotes uint64 `json:"upvotes"` // Total upvotes Timestamp int64 `json:"timestamp"` Receipt string `json:"receipt"` } -// Comments returns all comments for a proposal. +// Comments returns all comments for a proposal. Unvetted proposal comments +// are only returned to the proposal author and admins. Retrieving proposal +// comments on vetted proposals does not require a user to be logged in. type Comments struct { State PropStateT `json:"state"` Token string `json:"token"` @@ -580,7 +585,7 @@ type CommentVoteDetails struct { } // CommentVotes returns all comment votes that meet the provided filtering -// criteria. +// criteria. Comment votes are only allowed on vetted proposals. type CommentVotes struct { State PropStateT `json:"state"` Token string `json:"token"` diff --git a/politeiawww/cmd/piwww/README.md b/politeiawww/cmd/piwww/README.md index b238c3653..1964eb9cd 100644 --- a/politeiawww/cmd/piwww/README.md +++ b/politeiawww/cmd/piwww/README.md @@ -1,8 +1,10 @@ -# piwww +piwww +==== -piwww is a command line tool that allows you to interact with the piwww API. +piwww is a command line tool that allows you to interact with the politeiawww +pi API. -## Available Commands +# Available Commands You can view the available commands and application options by using the help flag. @@ -12,7 +14,7 @@ You can view details about a specific command by using the help command. $ piwww help -## Persisting Data Between Commands +# Persisting Data Between Commands piwww stores user identity data (the user's public/private key pair), session cookies, and CSRF tokens in the piwww directory. This allows you to login with a user and use the same session data for subsequent commands. The data is @@ -33,7 +35,7 @@ The location of the piwww directory varies based on your operating system. `~/.piwww` -## Setup Configuration File +# Setup Configuration File piwww has a configuration file that you can setup to make execution easier. You should create the configuration file under the following paths. @@ -50,7 +52,7 @@ You should create the configuration file under the following paths. `~/.piwww/piwww.conf` If you're developing locally, you'll want to set the politeiawww host in the -configuration file since the default politeiawww host is +configuration file to your local politeiawww instance. The host defaults to `https://proposals.decred.org`. Copy these lines into your `piwww.conf` file. `skipverify` is used to skip TLS certificate verification and should only be used when running politeia locally. @@ -60,9 +62,9 @@ host=https://127.0.0.1:4443 skipverify=true ``` -## Example Usage +# Example Usage -### Create a new user +## Create a new user $ piwww usernew email@example.com username password --verify --paywall @@ -77,11 +79,11 @@ fee requirement. **If you use the `--paywall` flag, you will still need to wait for block confirmations before you'll be allowed to submit proposals.** -### Login with the user +## Login with the user $ piwww login email@example.com password -### Assign admin privileges and create proposal credits +## Assign admin privileges and create proposal credits Proposal credits are required in order to submit a proposal. They are a spam prevention measure that would normally need to be purchased using DCR, but if @@ -98,7 +100,7 @@ get a `resource temporarily unavailable` error if you don't.** **Start politeiawww back up.** -### Submit a new proposal +## Submit a new proposal When submitting a proposal, you can either specify a markdown file or you can use the `--random` flag to have piwww generate a random proposal for you. @@ -109,64 +111,75 @@ use the `--random` flag to have piwww generate a random proposal for you. { "name": "index.md", "mime": "text/plain; charset=utf-8", - "digest": "362ca2a93194ebee058640f36b0ba74955760cd495f2626d740334de2cbb2a8d", - "payload": "VGhpcyBpcyB0aGUgcHJvcG9zYWwgdGl0bGUKaFlOWll3NklzaE0rUVlWcU91aU45YThXdzdJUnRBYmVxSDV6dERTZkk0bz0KaXBTOUIwTmRNSHZEU3QrRkdHZFhoRHFMQ2RmQjF4eWFCTHJvTU01c1FnTT0KVXpHK2l3S0drZHhjaGRmdFMrYlpqZ0xsc1I4bGVmWDVnUCsxLy90ZXdJRT0KZWFvd1hNNkNGeVc4Z3dxRUVlc0J5aXNwbDNPSW9WemdyVlJZZ1ZEK1UzND0Kb3Q5WVlncGY0NGRFVlJ3ckdPb3FXQXJGaCtlUm1zemhZaGdnWEtkRTRhMD0KZWZXNmNwNTlCd05taS95b1Z0Zk5HU0dvWldrZzgvTUFFMllCMGZqcEREaz0KTUFicVVobW9WMFpIZ3NzNEpOMFBvU1F1V0pubWxNd3lrKzFIMUovSzVpQT0KWDlxQ3ZUcWZEbk1iTW1rV0V3bzNuSmtlL1dlaEN3dU1QMTdFYnczUi9HWT0KbUNPZ0ZpZEtGUmJKWTBnUCtrbGZUZUxUS3JSODBsSW92UGxVcjEvWjVjRT0KYWFzc04wWHZSZkdFM0ZIbHpXVFhTQlJ4ZVhCY2c5dmk1Wm5YUEhKWElUQT0K" + "digest": "2a72cd797f164489f18628a84b81604d91cb3dd9e8217e3f12c6ba37ab6b7760", + "payload": "S0gycmxiZUJiVmJ4bTR0OEhwRWpQZGxEQlpXdUl1QkR4RjU3cXNZZXpFZz0KUnJJRmtqM2RYaTEwQW9GekZaKzd3QW9HVk5LVzVRWkZWUzNYWi9jbnJTWT0KQmYrcDY4YXN4NE1PWFk1WHl2a1RLTm1QdlM1bjdUcjZNQ0p5ZWdtZm1UVT0KMVNyWFp6Smh6VDBGd29LYnppdStBMDdKNUtiQ1NOV1NwUmNMaW92L2I4ST0KNno2clpnWTVxemtPbGMxL1pPZ2pRV1NZbVdhdGgyT1BnQng5L1J6RHVxbz0KelNSRmNobDNvTS9QU0J1WUVuWmdrd3o2SG5HVjdiQytEUkZlMDBudUJGaz0KcHRuL2xoeTFlcTNpeHpEanBHVVMxSjVZVDFrVEtlMW9tcUxpRGNPSTRlcz0KOXdwempHcE9mb3p0ZEFXcHhwWU52THpDbGgvVU5rYTVCNjRCV01GcEdVMD0KOUpHZE05Nzd4SjJJTkFIZ2dGSjBKczhvUDVKb0JIQ1dsRTEzSzFtSmQvdz0KR3Z2eWdiVEsvakIybHBVbE41Q250SjlGWUdOQmY0TG5Idzl0a2FxYWVUZz0K" } ], - "publickey": "c2c2ea7f24733983bf8037c189f32b5da49e6396b7d21cb69efe09d290b3cb6d", - "signature": "d4f38ee60e3032e67264732b13081ac36554fefd70079d40dcf7eb179e7cc4b2c80acc6460e9de1e816255bccfade659df6766c7371bd68592f010e3179feb0e" + "metadata": [ + { + "hint": "proposalmetadata", + "digest": "cd7e75c3df810965c48c3c03a47062a1f5bf7e4458b036380877d3c59e331b41", + "payload": "eyJuYW1lIjoiMjI1ZDJiZTFiYWQ2ZWU0MiJ9" + } + ], + "publickey": "72a1a0f19d6d76b9bbec069f5672fa9f22485961b1dffe8c570558e88168076a", + "signature": "981711bbf6cf408859f5eeab71bc5ec5a3fb4a723d3c853ede20415c9a5db1f2fd53265f73d79389e54b3ef5e0e924d0b48dee5b380c90ed093a3adcd7dab708" } { + "timestamp": 1602104519, "censorshiprecord": { - "token": "2c5d74209f37ca370000", - "merkle": "362ca2a93194ebee058640f36b0ba74955760cd495f2626d740334de2cbb2a8d", - "signature": "729269ef6bb45003a4728c40ff5c7f1ecbc44bfcff459d43274155e42e971a0ef8830e692eb833b049df5460edd850c77f21353fe24fd43a454388b7b89d7e00" + "token": "98daf0732ac3006c0000", + "merkle": "928b9cede1846ba542a81d9a7968baff2b7f7cc4d80f52957746be8f6c3869de", + "signature": "e30fc5332197f7b8f8fb8f73228a79295c7328d75aff10c123eb00d18e29fbd1a3fb96839f738c1ba19169246b018be389b8898afa1f4466b11a69c036187407" } } Proposals are identified by their censorship record token in all other -commands. +commands. The censorship record token of the proposal example shown above is +`98daf0732ac3006c0000`. -The proposal must first be vetted by an admin before it is publicily viewable. -Proposals are identified by their censorship record token, which can be found -in the output of the `newproposal` command. +## Make a proposal public (admin privileges required) -### Make a proposal public (admin privileges required) +The proposal must first be vetted by an admin and have the proposal status set +to public before it will be publicly viewable. - $ piwww proposalstatusset [token] public + $ piwww proposalstatusset --unvetted [token] public -Now that the proposal has been vetted and is publicly available, you can -comment on the proposal or authorize the voting period to start. +Now that the proposal status has been made public, any user can comment on the +proposal. Once the proposal author feels the discussion period was sufficient, +they can authorize the voting period to start. -### Authorize the voting period on a proposal (must be author) +## Authorize the voting period on a proposal (must be author) Before an admin can start the voting period on a proposal the author must authorize the vote. $ piwww voteauthorize [token] -### Start a proposal vote (admin privileges required) +## Start a proposal vote (admin privileges required) Once a proposal vote has been authorized by the author, an admin can start the -voting period. +voting period at any point. $ piwww votestart [token] -### Voting on a proposal - politeiavoter +## Voting on a proposal + +### politeiavoter + +Voting on a proposal can be done using the `politeiavoter` tool. -Voting on a proposal can be done using the -[politeiavoter](https://github.com/decred/politeia/tree/master/politeiavoter/) -tool. +[politeiavoter](https://github.com/decred/politeia/tree/master/politeiawww/cmd/politeiavoter/) -### Voting on a proposal - piwww +### piwww -You can also vote on proposals using the `piww` command `voteballot`. This -casts a ballot of votes. This will only work on testnet and if you are running -your dcrwallet locally using the default port. +You can also vote on proposals using the `piwww voteballot` command. This casts +a ballot of votes. This will only work on testnet and if you are running your +dcrwallet locally using the default port. $ piwww voteballot [token] [voteID] -## Reference implementation +# Reference implementation The piwww `testrun` command runs a series of tests on all of the politeiawww pi API routes. This command can be used as a reference implementation for the pi diff --git a/politeiawww/cmd/piwww/commentnew.go b/politeiawww/cmd/piwww/commentnew.go index f10c88175..5d56b71c5 100644 --- a/politeiawww/cmd/piwww/commentnew.go +++ b/politeiawww/cmd/piwww/commentnew.go @@ -60,7 +60,7 @@ func (c *commentNewCmd) Execute(args []string) error { // Sign comment data msg := strconv.Itoa(int(state)) + token + - strconv.FormatUint(uint64(parentID), 10) + comment + strconv.FormatUint(parentID, 10) + comment b := cfg.Identity.SignMessage([]byte(msg)) signature := hex.EncodeToString(b[:]) diff --git a/politeiawww/cmd/piwww/commentvote.go b/politeiawww/cmd/piwww/commentvote.go index 6e8f34be6..70bf0361c 100644 --- a/politeiawww/cmd/piwww/commentvote.go +++ b/politeiawww/cmd/piwww/commentvote.go @@ -52,7 +52,7 @@ func (c *commentVoteCmd) Execute(args []string) error { // Sign vote choice msg := strconv.Itoa(int(pi.PropStateVetted)) + token + - c.Args.CommentID + c.Args.Vote + c.Args.CommentID + strconv.FormatInt(int64(vote), 10) b := cfg.Identity.SignMessage([]byte(msg)) signature := hex.EncodeToString(b[:]) diff --git a/politeiawww/cmd/piwww/commentvotes.go b/politeiawww/cmd/piwww/commentvotes.go index e97a9f52a..62c9c90cb 100644 --- a/politeiawww/cmd/piwww/commentvotes.go +++ b/politeiawww/cmd/piwww/commentvotes.go @@ -5,6 +5,8 @@ package main import ( + "fmt" + pi "github.com/decred/politeia/politeiawww/api/pi/v1" "github.com/decred/politeia/politeiawww/cmd/shared" ) @@ -13,15 +15,30 @@ import ( // the specified proposal from the provided user. type commentVotesCmd struct { Args struct { - Token string `positional-arg-name:"token"` // Censorship token - UserID string `positional-arg-name:"userid"` // User id - } `positional-args:"true" required:"true"` + Token string `positional-arg-name:"token" required:"true"` + UserID string `positional-arg-name:"userid"` + } `positional-args:"true"` + Me bool `long:"me" optional:"true"` } // Execute executes the user comment likes command. -func (cmd *commentVotesCmd) Execute(args []string) error { - token := cmd.Args.Token - userID := cmd.Args.UserID +func (c *commentVotesCmd) Execute(args []string) error { + token := c.Args.Token + userID := c.Args.UserID + + if userID == "" && !c.Me { + return fmt.Errorf("you must either provide a user id or use " + + "the --me flag to use the user ID of the logged in user") + } + + // Get user ID of logged in user if specified + if c.Me { + lr, err := client.Me() + if err != nil { + return err + } + userID = lr.UserID + } cvr, err := client.CommentVotes(pi.CommentVotes{ Token: token, @@ -42,5 +59,8 @@ Get the provided user comment upvote/downvotes for a proposal. Arguments: 1. token (string, required) Proposal censorship token -2. userid (string, required) User id +2. userid (string, required) User ID + +Flags: + --me (bool, optional) Use the user ID of the logged in user ` diff --git a/politeiawww/eventmanager.go b/politeiawww/eventmanager.go index 239841899..2f58611e3 100644 --- a/politeiawww/eventmanager.go +++ b/politeiawww/eventmanager.go @@ -388,32 +388,34 @@ func (p *politeiawww) notifyParentAuthorOnComment(d dataProposalComment) error { // TODO return nil - // Lookup the parent comment author - var author *user.User - - // Check if notification should be sent to author - switch { - case d.username == author.Username: - // Author commented on their own proposal - return nil - case !userNotificationEnabled(*author, - www.NotificationEmailCommentOnMyProposal): - // Author does not have notification bit set on - return nil - } - - // Get proposal. We need this proposal name for the notification. - pr, err := p.proposalRecordLatest(d.state, d.token) - if err != nil { - return fmt.Errorf("proposalRecordLatest %v %v: %v", - d.state, d.token, err) - } + /* + // Lookup the parent comment author + var author *user.User + + // Check if notification should be sent to author + switch { + case d.username == author.Username: + // Author commented on their own proposal + return nil + case !userNotificationEnabled(*author, + www.NotificationEmailCommentOnMyProposal): + // Author does not have notification bit set on + return nil + } + + // Get proposal. We need this proposal name for the notification. + pr, err := p.proposalRecordLatest(d.state, d.token) + if err != nil { + return fmt.Errorf("proposalRecordLatest %v %v: %v", + d.state, d.token, err) + } - // Send notification eamil - commentID := strconv.FormatUint(uint64(d.commentID), 10) + // Send notification eamil + commentID := strconv.FormatUint(uint64(d.commentID), 10) - return p.emailProposalCommentReply(d.token, commentID, d.username, - proposalName(*pr), author.Email) + return p.emailProposalCommentReply(d.token, commentID, d.username, + proposalName(*pr), author.Email) + */ } type dataProposalComment struct { diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 4cb97a675..a6b8e4bfa 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -356,22 +356,23 @@ func convertPropStateFromComments(s comments.StateT) pi.PropStateT { return pi.PropStateInvalid } -func convertCommentFromPlugin(cm comments.Comment) pi.Comment { +func convertCommentFromPlugin(c comments.Comment) pi.Comment { return pi.Comment{ - UserID: cm.UserID, + UserID: c.UserID, Username: "", // Intentionally omitted, needs to be pulled from userdb - State: convertPropStateFromComments(cm.State), - Token: cm.Token, - ParentID: cm.ParentID, - Comment: cm.Comment, - PublicKey: cm.PublicKey, - Signature: cm.Signature, - CommentID: cm.CommentID, - Timestamp: cm.Timestamp, - Receipt: cm.Receipt, - Score: cm.Score, - Deleted: cm.Deleted, - Reason: cm.Reason, + State: convertPropStateFromComments(c.State), + Token: c.Token, + ParentID: c.ParentID, + Comment: c.Comment, + PublicKey: c.PublicKey, + Signature: c.Signature, + CommentID: c.CommentID, + Timestamp: c.Timestamp, + Receipt: c.Receipt, + + Upvotes: c.Upvotes, + Censored: c.Deleted, + Reason: c.Reason, } } @@ -1633,6 +1634,14 @@ func (p *politeiawww) processCommentNew(cn pi.CommentNew, usr user.User) (*pi.Co func (p *politeiawww) processCommentVote(cv pi.CommentVote, usr user.User) (*pi.CommentVoteReply, error) { log.Tracef("processCommentVote: %v %v %v", cv.Token, cv.CommentID, cv.Vote) + // Verify state + if cv.State != pi.PropStateVetted { + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusPropStateInvalid, + ErrorContext: []string{"proposal must be vetted"}, + } + } + // Verify user has paid registration paywall if !p.userHasPaid(usr) { return nil, pi.UserErrorReply{ @@ -1664,7 +1673,8 @@ func (p *politeiawww) processCommentVote(cv pi.CommentVote, usr user.User) (*pi. } return &pi.CommentVoteReply{ - Score: cvr.Score, + Downvotes: cvr.Downvotes, + Upvotes: cvr.Upvotes, Timestamp: cvr.Timestamp, Receipt: cvr.Receipt, }, nil @@ -1706,9 +1716,45 @@ func (p *politeiawww) processCommentCensor(cc pi.CommentCensor, usr user.User) ( }, nil } -func (p *politeiawww) processComments(c pi.Comments) (*pi.CommentsReply, error) { +func (p *politeiawww) processComments(c pi.Comments, usr *user.User) (*pi.CommentsReply, error) { log.Tracef("processComments: %v", c.Token) + // Only admins and the proposal author are allowed to retrieve + // unvetted comments. This is a public route so a user might not + // exist. + if c.State == pi.PropStateUnvetted { + var isAllowed bool + switch { + case usr == nil: + // No logged in user. Unvetted not allowed. + case usr.Admin: + // User is an admin. Unvetted is allowed. + isAllowed = true + default: + // Logged in user is not an admin. Check if they are the + // proposal author. + pr, err := p.proposalRecordLatest(c.State, c.Token) + if err != nil { + if err == errProposalNotFound { + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusPropNotFound, + } + } + return nil, fmt.Errorf("proposalRecordLatest: %v", err) + } + if usr.ID.String() == pr.UserID { + // User is the proposal author. Unvetted is allowed. + isAllowed = true + } + } + if !isAllowed { + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusUnauthorized, + ErrorContext: []string{"user is not author or admin"}, + } + } + } + // Send plugin command reply, err := p.commentsAll(comments.GetAll{ State: convertCommentsStateFromPi(c.State), @@ -1748,6 +1794,15 @@ func (p *politeiawww) processComments(c pi.Comments) (*pi.CommentsReply, error) func (p *politeiawww) processCommentVotes(cv pi.CommentVotes) (*pi.CommentVotesReply, error) { log.Tracef("processCommentVotes: %v %v", cv.Token, cv.UserID) + // Verify state + if cv.State != pi.PropStateVetted { + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusPropStateInvalid, + ErrorContext: []string{"proposal must be vetted"}, + } + } + + // Send plugin command v := comments.Votes{ State: convertCommentsStateFromPi(cv.State), Token: cv.Token, @@ -2189,7 +2244,16 @@ func (p *politeiawww) handleComments(w http.ResponseWriter, r *http.Request) { return } - cr, err := p.processComments(c) + // Lookup session user. This is a public route so a session may not + // exist. Ignore any session not found errors. + usr, err := p.getSessionUser(w, r) + if err != nil && err != errSessionNotFound { + respondWithPiError(w, r, + "handleProposalInventory: getSessionUser: %v", err) + return + } + + cr, err := p.processComments(c, usr) if err != nil { respondWithPiError(w, r, "handleCommentVote: processComments: %v", err) diff --git a/politeiawww/plugin.go b/politeiawww/plugin.go index f893f1ad2..495d70d9b 100644 --- a/politeiawww/plugin.go +++ b/politeiawww/plugin.go @@ -64,7 +64,8 @@ func (p *politeiawww) getPluginInventory() ([]plugin, error) { pi, err := p.pluginInventory() if err != nil { - log.Infof("cannot get politeiad plugin inventory: %v", err) + log.Infof("cannot get politeiad plugin inventory: %v: retry in %v", + err, sleepInterval) time.Sleep(sleepInterval) continue } From 304987d40d60c0082a2535f8b22405bdfe743c59 Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Fri, 9 Oct 2020 10:40:17 -0300 Subject: [PATCH 144/449] piwww: makeRequest refactor. --- politeiawww/cmd/shared/client.go | 1104 +++++++++++++++++++++++------- 1 file changed, 869 insertions(+), 235 deletions(-) diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index c04bc5b13..a59b26f42 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -50,9 +50,9 @@ func prettyPrintJSON(v interface{}) error { return nil } -// userErrorStatus retrieves the human readable error message for an error -// status code. The status code can be from either the pi or cms api. -func userErrorStatus(e www.ErrorStatusT) string { +// userWWWErrorStatus retrieves the human readable error message for an error +// status code. The status code error message comes from the www api. +func userWWWErrorStatus(e www.ErrorStatusT) string { s, ok := www.ErrorStatus[e] if ok { return s @@ -64,11 +64,70 @@ func userErrorStatus(e www.ErrorStatusT) string { return "" } +// userPiErrorStatus retrieves the human readable error message for an error +// status code. The status code error message comes from the pi api. +func userPiErrorStatus(e pi.ErrorStatusT) string { + s, ok := pi.ErrorStatus[e] + if ok { + return s + } + return "" +} + +// wwwError unmarshals the response body from makeRequest, and parses +// the error code and error context from the www api. +func wwwError(body []byte, statusCode int) error { + var ue www.UserError + err := json.Unmarshal(body, &ue) + if err != nil { + return fmt.Errorf("unmarshal UserError: %v", err) + } + if ue.ErrorCode != 0 { + var e error + errMsg := userWWWErrorStatus(ue.ErrorCode) + if len(ue.ErrorContext) == 0 { + // Error format when an ErrorContext is not included + e = fmt.Errorf("%v, %v", statusCode, errMsg) + } else { + // Error format when an ErrorContext is included + e = fmt.Errorf("%v, %v: %v", statusCode, errMsg, + strings.Join(ue.ErrorContext, ", ")) + } + return e + } + return nil +} + +// piError unmarshals the response body from makeRequest, and parses +// the error code and error context from the pi api. +func piError(body []byte, statusCode int) error { + var ue pi.UserErrorReply + err := json.Unmarshal(body, &ue) + if err != nil { + return fmt.Errorf("unmarshal UserError: %v", err) + } + if ue.ErrorCode != 0 { + var e error + errMsg := userPiErrorStatus(ue.ErrorCode) + if len(ue.ErrorContext) == 0 { + // Error format when an ErrorContext is not included + e = fmt.Errorf("%v, %v", statusCode, errMsg) + } else { + // Error format when an ErrorContext is included + e = fmt.Errorf("%v, %v: %v", statusCode, errMsg, + strings.Join(ue.ErrorContext, ", ")) + } + return e + } + return nil +} + // makeRequest sends the provided request to the politeiawww backend specified // by the Client config. This function handles verbose printing when specified // by the Client config since verbose printing includes details such as the -// full route and http response codes. -func (c *Client) makeRequest(method, routeVersion, route string, body interface{}) ([]byte, error) { +// full route and http response codes. Caller functions handle status code +// validation and error checks. +func (c *Client) makeRequest(method, routeVersion, route string, body interface{}) (int, []byte, error) { // Setup request var requestBody []byte var queryParams string @@ -86,7 +145,7 @@ func (c *Client) makeRequest(method, routeVersion, route string, body interface{ // will populate the query params. form := url.Values{} if err := schema.NewEncoder().Encode(body, form); err != nil { - return nil, err + return 0, nil, err } queryParams = "?" + form.Encode() @@ -94,11 +153,11 @@ func (c *Client) makeRequest(method, routeVersion, route string, body interface{ var err error requestBody, err = json.Marshal(body) if err != nil { - return nil, err + return 0, nil, err } default: - return nil, fmt.Errorf("unknown http method '%v'", method) + return 0, nil, fmt.Errorf("unknown http method '%v'", method) } } @@ -112,27 +171,27 @@ func (c *Client) makeRequest(method, routeVersion, route string, body interface{ fmt.Printf("Request: POST %v\n", fullRoute) err := prettyPrintJSON(body) if err != nil { - return nil, err + return 0, nil, err } case c.cfg.Verbose && method == http.MethodPut: fmt.Printf("Request: PUT %v\n", fullRoute) err := prettyPrintJSON(body) if err != nil { - return nil, err + return 0, nil, err } } // Create http request req, err := http.NewRequest(method, fullRoute, bytes.NewReader(requestBody)) if err != nil { - return nil, err + return 0, nil, err } req.Header.Add(www.CsrfToken, c.cfg.CSRF) // Send request r, err := c.http.Do(req) if err != nil { - return nil, err + return 0, nil, err } defer func() { r.Body.Close() @@ -140,38 +199,12 @@ func (c *Client) makeRequest(method, routeVersion, route string, body interface{ responseBody := util.ConvertBodyToByteArray(r.Body, false) - // Validate response status - if r.StatusCode != http.StatusOK { - var ue www.UserError - err = json.Unmarshal(responseBody, &ue) - if err == nil && ue.ErrorCode != 0 { - // TODO the user error should be returned in full and the - // calling function should print the error message. The reason - // is because only the calling function knows what API was used - // and thus what error message to print. - /* - var e error - if len(ue.ErrorContext) == 0 { - // Error format when an ErrorContext is not included - e = fmt.Errorf("%v, %v", r.StatusCode, userErrorStatus(ue.ErrorCode)) - } else { - // Error format when an ErrorContext is included - e = fmt.Errorf("%v, %v: %v", r.StatusCode, - userErrorStatus(ue.ErrorCode), strings.Join(ue.ErrorContext, ", ")) - } - */ - return nil, ue - } - - return nil, fmt.Errorf("%v", r.StatusCode) - } - // Print response details if c.cfg.Verbose { fmt.Printf("Response: %v\n", r.StatusCode) } - return responseBody, nil + return r.StatusCode, responseBody, nil } // Version returns the version information for the politeiawww instance. @@ -200,15 +233,16 @@ func (c *Client) Version() (*www.VersionReply, error) { r.Body.Close() }() - responseBody := util.ConvertBodyToByteArray(r.Body, false) + respBody := util.ConvertBodyToByteArray(r.Body, false) // Validate response status if r.StatusCode != http.StatusOK { var ue www.UserError - err = json.Unmarshal(responseBody, &ue) + err = json.Unmarshal(respBody, &ue) if err == nil { return nil, fmt.Errorf("%v, %v %v", r.StatusCode, - userErrorStatus(ue.ErrorCode), strings.Join(ue.ErrorContext, ", ")) + userWWWErrorStatus(ue.ErrorCode), + strings.Join(ue.ErrorContext, ", ")) } return nil, fmt.Errorf("%v", r.StatusCode) @@ -216,7 +250,7 @@ func (c *Client) Version() (*www.VersionReply, error) { // Unmarshal response var vr www.VersionReply - err = json.Unmarshal(responseBody, &vr) + err = json.Unmarshal(respBody, &vr) if err != nil { return nil, fmt.Errorf("unmarshal VersionReply: %v", err) } @@ -289,15 +323,16 @@ func (c *Client) Login(l *www.Login) (*www.LoginReply, error) { r.Body.Close() }() - responseBody := util.ConvertBodyToByteArray(r.Body, false) + respBody := util.ConvertBodyToByteArray(r.Body, false) // Validate response status if r.StatusCode != http.StatusOK { var ue www.UserError - err = json.Unmarshal(responseBody, &ue) + err = json.Unmarshal(respBody, &ue) if err == nil { return nil, fmt.Errorf("%v, %v %v", r.StatusCode, - userErrorStatus(ue.ErrorCode), strings.Join(ue.ErrorContext, ", ")) + userWWWErrorStatus(ue.ErrorCode), + strings.Join(ue.ErrorContext, ", ")) } return nil, fmt.Errorf("%v", r.StatusCode) @@ -305,7 +340,7 @@ func (c *Client) Login(l *www.Login) (*www.LoginReply, error) { // Unmarshal response var lr www.LoginReply - err = json.Unmarshal(responseBody, &lr) + err = json.Unmarshal(respBody, &lr) if err != nil { return nil, fmt.Errorf("unmarshal LoginReply: %v", err) } @@ -354,15 +389,16 @@ func (c *Client) Logout() (*www.LogoutReply, error) { r.Body.Close() }() - responseBody := util.ConvertBodyToByteArray(r.Body, false) + respBody := util.ConvertBodyToByteArray(r.Body, false) // Validate response status if r.StatusCode != http.StatusOK { var ue www.UserError - err = json.Unmarshal(responseBody, &ue) + err = json.Unmarshal(respBody, &ue) if err == nil { return nil, fmt.Errorf("%v, %v %v", r.StatusCode, - userErrorStatus(ue.ErrorCode), strings.Join(ue.ErrorContext, ", ")) + userWWWErrorStatus(ue.ErrorCode), + strings.Join(ue.ErrorContext, ", ")) } return nil, fmt.Errorf("%v", r.StatusCode) @@ -370,7 +406,7 @@ func (c *Client) Logout() (*www.LogoutReply, error) { // Unmarshal response var lr www.LogoutReply - err = json.Unmarshal(responseBody, &lr) + err = json.Unmarshal(respBody, &lr) if err != nil { return nil, fmt.Errorf("unmarshal LogoutReply: %v", err) } @@ -395,14 +431,21 @@ func (c *Client) Logout() (*www.LogoutReply, error) { // Policy returns the politeiawww policy information. func (c *Client) Policy() (*www.PolicyReply, error) { - responseBody, err := c.makeRequest(http.MethodGet, + statusCode, respBody, err := c.makeRequest(http.MethodGet, www.PoliteiaWWWAPIRoute, www.RoutePolicy, nil) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var pr www.PolicyReply - err = json.Unmarshal(responseBody, &pr) + err = json.Unmarshal(respBody, &pr) if err != nil { return nil, fmt.Errorf("unmarshal PolicyReply: %v", err) } @@ -419,14 +462,21 @@ func (c *Client) Policy() (*www.PolicyReply, error) { // CMSPolicy returns the politeiawww policy information. func (c *Client) CMSPolicy() (*cms.PolicyReply, error) { - responseBody, err := c.makeRequest(http.MethodGet, + statusCode, respBody, err := c.makeRequest(http.MethodGet, www.PoliteiaWWWAPIRoute, www.RoutePolicy, nil) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var pr cms.PolicyReply - err = json.Unmarshal(responseBody, &pr) + err = json.Unmarshal(respBody, &pr) if err != nil { return nil, fmt.Errorf("unmarshal CMSPolicyReply: %v", err) } @@ -443,14 +493,21 @@ func (c *Client) CMSPolicy() (*cms.PolicyReply, error) { // InviteNewUser creates a new cmswww user. func (c *Client) InviteNewUser(inu *cms.InviteNewUser) (*cms.InviteNewUserReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, cms.APIRoute, cms.RouteInviteNewUser, inu) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var inur cms.InviteNewUserReply - err = json.Unmarshal(responseBody, &inur) + err = json.Unmarshal(respBody, &inur) if err != nil { return nil, fmt.Errorf("unmarshal InviteNewUserReply: %v", err) } @@ -467,14 +524,21 @@ func (c *Client) InviteNewUser(inu *cms.InviteNewUser) (*cms.InviteNewUserReply, // RegisterUser finalizes the signup process for a new cmswww user. func (c *Client) RegisterUser(ru *cms.RegisterUser) (*cms.RegisterUserReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, cms.APIRoute, cms.RouteRegisterUser, ru) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var rur cms.RegisterUserReply - err = json.Unmarshal(responseBody, &rur) + err = json.Unmarshal(respBody, &rur) if err != nil { return nil, fmt.Errorf("unmarshal RegisterUserReply: %v", err) } @@ -491,14 +555,21 @@ func (c *Client) RegisterUser(ru *cms.RegisterUser) (*cms.RegisterUserReply, err // NewUser creates a new politeiawww user. func (c *Client) NewUser(nu *www.NewUser) (*www.NewUserReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, www.PoliteiaWWWAPIRoute, www.RouteNewUser, nu) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var nur www.NewUserReply - err = json.Unmarshal(responseBody, &nur) + err = json.Unmarshal(respBody, &nur) if err != nil { return nil, fmt.Errorf("unmarshal NewUserReply: %v", err) } @@ -515,14 +586,21 @@ func (c *Client) NewUser(nu *www.NewUser) (*www.NewUserReply, error) { // VerifyNewUser verifies a user's email address. func (c *Client) VerifyNewUser(vnu *www.VerifyNewUser) (*www.VerifyNewUserReply, error) { - responseBody, err := c.makeRequest(http.MethodGet, + statusCode, respBody, err := c.makeRequest(http.MethodGet, www.PoliteiaWWWAPIRoute, www.RouteVerifyNewUser, vnu) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var vnur www.VerifyNewUserReply - err = json.Unmarshal(responseBody, &vnur) + err = json.Unmarshal(respBody, &vnur) if err != nil { return nil, fmt.Errorf("unmarshal VerifyNewUserReply: %v", err) } @@ -539,14 +617,21 @@ func (c *Client) VerifyNewUser(vnu *www.VerifyNewUser) (*www.VerifyNewUserReply, // Me returns user details for the logged in user. func (c *Client) Me() (*www.LoginReply, error) { - responseBody, err := c.makeRequest(http.MethodGet, + statusCode, respBody, err := c.makeRequest(http.MethodGet, www.PoliteiaWWWAPIRoute, www.RouteUserMe, nil) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var lr www.LoginReply - err = json.Unmarshal(responseBody, &lr) + err = json.Unmarshal(respBody, &lr) if err != nil { return nil, fmt.Errorf("unmarshal LoginReply: %v", err) } @@ -563,14 +648,21 @@ func (c *Client) Me() (*www.LoginReply, error) { // Secret pings politeiawww. func (c *Client) Secret() (*www.UserError, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, www.PoliteiaWWWAPIRoute, www.RouteSecret, nil) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var ue www.UserError - err = json.Unmarshal(responseBody, &ue) + err = json.Unmarshal(respBody, &ue) if err != nil { return nil, fmt.Errorf("unmarshal UserError: %v", err) } @@ -587,14 +679,21 @@ func (c *Client) Secret() (*www.UserError, error) { // ChangeUsername changes the username of the logged in user. func (c *Client) ChangeUsername(cu *www.ChangeUsername) (*www.ChangeUsernameReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, www.PoliteiaWWWAPIRoute, www.RouteChangeUsername, cu) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var cur www.ChangeUsernameReply - err = json.Unmarshal(responseBody, &cur) + err = json.Unmarshal(respBody, &cur) if err != nil { return nil, fmt.Errorf("unmarshal ChangeUsernameReply: %v", err) } @@ -611,14 +710,21 @@ func (c *Client) ChangeUsername(cu *www.ChangeUsername) (*www.ChangeUsernameRepl // ChangePassword changes the password for the logged in user. func (c *Client) ChangePassword(cp *www.ChangePassword) (*www.ChangePasswordReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, www.PoliteiaWWWAPIRoute, www.RouteChangePassword, cp) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var cpr www.ChangePasswordReply - err = json.Unmarshal(responseBody, &cpr) + err = json.Unmarshal(respBody, &cpr) if err != nil { return nil, fmt.Errorf("unmarshal ChangePasswordReply: %v", err) } @@ -635,14 +741,21 @@ func (c *Client) ChangePassword(cp *www.ChangePassword) (*www.ChangePasswordRepl // ResetPassword resets the password of the specified user. func (c *Client) ResetPassword(rp *www.ResetPassword) (*www.ResetPasswordReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, www.PoliteiaWWWAPIRoute, www.RouteResetPassword, rp) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var rpr www.ResetPasswordReply - err = json.Unmarshal(responseBody, &rpr) + err = json.Unmarshal(respBody, &rpr) if err != nil { return nil, fmt.Errorf("unmarshal ResetPasswordReply: %v", err) } @@ -659,12 +772,19 @@ func (c *Client) ResetPassword(rp *www.ResetPassword) (*www.ResetPasswordReply, // VerifyResetPassword sends the VerifyResetPassword command to politeiawww. func (c *Client) VerifyResetPassword(vrp www.VerifyResetPassword) (*www.VerifyResetPasswordReply, error) { - respBody, err := c.makeRequest(http.MethodPost, www.PoliteiaWWWAPIRoute, - www.RouteVerifyResetPassword, vrp) + statusCode, respBody, err := c.makeRequest(http.MethodPost, + www.PoliteiaWWWAPIRoute, www.RouteVerifyResetPassword, vrp) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var reply www.VerifyResetPasswordReply err = json.Unmarshal(respBody, &reply) if err != nil { @@ -684,14 +804,21 @@ func (c *Client) VerifyResetPassword(vrp www.VerifyResetPassword) (*www.VerifyRe // UserProposalPaywall retrieves proposal credit paywall information for the // logged in user. func (c *Client) UserProposalPaywall() (*www.UserProposalPaywallReply, error) { - responseBody, err := c.makeRequest(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteUserProposalPaywall, nil) + statusCode, respBody, err := c.makeRequest(http.MethodGet, + www.PoliteiaWWWAPIRoute, www.RouteUserProposalPaywall, nil) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var ppdr www.UserProposalPaywallReply - err = json.Unmarshal(responseBody, &ppdr) + err = json.Unmarshal(respBody, &ppdr) if err != nil { return nil, fmt.Errorf("unmarshal ProposalPaywalDetailsReply: %v", err) } @@ -708,14 +835,21 @@ func (c *Client) UserProposalPaywall() (*www.UserProposalPaywallReply, error) { // ProposalNew submits a new proposal. func (c *Client) ProposalNew(pn pi.ProposalNew) (*pi.ProposalNewReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, pi.RouteProposalNew, pn) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = piError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var pnr pi.ProposalNewReply - err = json.Unmarshal(responseBody, &pnr) + err = json.Unmarshal(respBody, &pnr) if err != nil { return nil, fmt.Errorf("unmarshal ProposalNewReply: %v", err) } @@ -732,14 +866,21 @@ func (c *Client) ProposalNew(pn pi.ProposalNew) (*pi.ProposalNewReply, error) { // ProposalEdit edits a proposal. func (c *Client) ProposalEdit(pe pi.ProposalEdit) (*pi.ProposalEditReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, pi.RouteProposalEdit, pe) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = piError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var per pi.ProposalEditReply - err = json.Unmarshal(responseBody, &per) + err = json.Unmarshal(respBody, &per) if err != nil { return nil, fmt.Errorf("unmarshal ProposalEditReply: %v", err) } @@ -756,14 +897,21 @@ func (c *Client) ProposalEdit(pe pi.ProposalEdit) (*pi.ProposalEditReply, error) // ProposalStatusSet sets the status of a proposal func (c *Client) ProposalStatusSet(pss pi.ProposalStatusSet) (*pi.ProposalStatusSetReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, pi.RouteProposalStatusSet, pss) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = piError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var pssr pi.ProposalStatusSetReply - err = json.Unmarshal(responseBody, &pssr) + err = json.Unmarshal(respBody, &pssr) if err != nil { return nil, fmt.Errorf("unmarshal ProposalStatusSetReply: %v", err) } @@ -780,14 +928,21 @@ func (c *Client) ProposalStatusSet(pss pi.ProposalStatusSet) (*pi.ProposalStatus // Proposals retrieves a proposal for each of the provided proposal requests. func (c *Client) Proposals(p pi.Proposals) (*pi.ProposalsReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, pi.RouteProposals, p) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = piError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var pr pi.ProposalsReply - err = json.Unmarshal(responseBody, &pr) + err = json.Unmarshal(respBody, &pr) if err != nil { return nil, fmt.Errorf("unmarshal ProposalsReply: %v", err) } @@ -805,14 +960,21 @@ func (c *Client) Proposals(p pi.Proposals) (*pi.ProposalsReply, error) { // ProposalInventory retrieves the censorship tokens of all proposals, // separated by their status. func (c *Client) ProposalInventory() (*pi.ProposalInventoryReply, error) { - respondeBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, + statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, pi.RouteProposalInventory, nil) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = piError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var pir pi.ProposalInventoryReply - err = json.Unmarshal(respondeBody, &pir) + err = json.Unmarshal(respBody, &pir) if err != nil { return nil, fmt.Errorf("unmarshal ProposalInventory: %v", err) } @@ -830,14 +992,21 @@ func (c *Client) ProposalInventory() (*pi.ProposalInventoryReply, error) { // NewInvoice submits the specified invoice to politeiawww for the logged in // user. func (c *Client) NewInvoice(ni *cms.NewInvoice) (*cms.NewInvoiceReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, cms.APIRoute, cms.RouteNewInvoice, ni) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var nir cms.NewInvoiceReply - err = json.Unmarshal(responseBody, &nir) + err = json.Unmarshal(respBody, &nir) if err != nil { return nil, fmt.Errorf("unmarshal NewInvoiceReply: %v", err) } @@ -854,14 +1023,21 @@ func (c *Client) NewInvoice(ni *cms.NewInvoice) (*cms.NewInvoiceReply, error) { // EditInvoice edits the specified invoice with the logged in user. func (c *Client) EditInvoice(ei *cms.EditInvoice) (*cms.EditInvoiceReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, cms.APIRoute, cms.RouteEditInvoice, ei) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var eir cms.EditInvoiceReply - err = json.Unmarshal(responseBody, &eir) + err = json.Unmarshal(respBody, &eir) if err != nil { return nil, fmt.Errorf("unmarshal EditInvoiceReply: %v", err) } @@ -879,14 +1055,21 @@ func (c *Client) EditInvoice(ei *cms.EditInvoice) (*cms.EditInvoiceReply, error) // ProposalDetails retrieves the specified proposal. func (c *Client) ProposalDetails(token string, pd *www.ProposalsDetails) (*www.ProposalDetailsReply, error) { route := "/proposals/" + token - responseBody, err := c.makeRequest(http.MethodGet, + statusCode, respBody, err := c.makeRequest(http.MethodGet, www.PoliteiaWWWAPIRoute, route, pd) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var pr www.ProposalDetailsReply - err = json.Unmarshal(responseBody, &pr) + err = json.Unmarshal(respBody, &pr) if err != nil { return nil, fmt.Errorf("unmarshal ProposalDetailsReply: %v", err) } @@ -904,14 +1087,21 @@ func (c *Client) ProposalDetails(token string, pd *www.ProposalsDetails) (*www.P // UserInvoices retrieves the proposals that have been submitted by the // specified user. func (c *Client) UserInvoices(up *cms.UserInvoices) (*cms.UserInvoicesReply, error) { - responseBody, err := c.makeRequest(http.MethodGet, + statusCode, respBody, err := c.makeRequest(http.MethodGet, cms.APIRoute, cms.RouteUserInvoices, up) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var upr cms.UserInvoicesReply - err = json.Unmarshal(responseBody, &upr) + err = json.Unmarshal(respBody, &upr) if err != nil { return nil, fmt.Errorf("unmarshal UserInvoicesReply: %v", err) } @@ -928,14 +1118,21 @@ func (c *Client) UserInvoices(up *cms.UserInvoices) (*cms.UserInvoicesReply, err // ProposalBilling retrieves the billing for the requested proposal func (c *Client) ProposalBilling(pb *cms.ProposalBilling) (*cms.ProposalBillingReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, cms.APIRoute, cms.RouteProposalBilling, pb) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var pbr cms.ProposalBillingReply - err = json.Unmarshal(responseBody, &pbr) + err = json.Unmarshal(respBody, &pbr) if err != nil { return nil, fmt.Errorf("unmarshal ProposalBillingReply: %v", err) } @@ -952,14 +1149,21 @@ func (c *Client) ProposalBilling(pb *cms.ProposalBilling) (*cms.ProposalBillingR // ProposalBillingDetails retrieves the billing for the requested proposal func (c *Client) ProposalBillingDetails(pbd *cms.ProposalBillingDetails) (*cms.ProposalBillingDetailsReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, cms.APIRoute, cms.RouteProposalBillingDetails, pbd) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var pbdr cms.ProposalBillingDetailsReply - err = json.Unmarshal(responseBody, &pbdr) + err = json.Unmarshal(respBody, &pbdr) if err != nil { return nil, fmt.Errorf("unmarshal ProposalBillingDetailsReply: %v", err) } @@ -976,14 +1180,21 @@ func (c *Client) ProposalBillingDetails(pbd *cms.ProposalBillingDetails) (*cms.P // ProposalBillingSummary retrieves the billing for all approved proposals. func (c *Client) ProposalBillingSummary(pbd *cms.ProposalBillingSummary) (*cms.ProposalBillingSummaryReply, error) { - responseBody, err := c.makeRequest(http.MethodGet, + statusCode, respBody, err := c.makeRequest(http.MethodGet, cms.APIRoute, cms.RouteProposalBillingSummary, pbd) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var pbdr cms.ProposalBillingSummaryReply - err = json.Unmarshal(responseBody, &pbdr) + err = json.Unmarshal(respBody, &pbdr) if err != nil { return nil, fmt.Errorf("unmarshal ProposalBillingSummaryReply: %v", err) } @@ -1001,14 +1212,21 @@ func (c *Client) ProposalBillingSummary(pbd *cms.ProposalBillingSummary) (*cms.P // Invoices retrieves invoices base on possible field set in the request // month/year and/or status func (c *Client) Invoices(ai *cms.Invoices) (*cms.InvoicesReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, cms.APIRoute, cms.RouteInvoices, ai) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var air cms.InvoicesReply - err = json.Unmarshal(responseBody, &air) + err = json.Unmarshal(respBody, &air) if err != nil { return nil, fmt.Errorf("unmarshal InvoicesReply: %v", err) } @@ -1026,14 +1244,21 @@ func (c *Client) Invoices(ai *cms.Invoices) (*cms.InvoicesReply, error) { // GeneratePayouts generates a list of payouts for all approved invoices that // contain an address and amount for an admin to the process func (c *Client) GeneratePayouts(gp *cms.GeneratePayouts) (*cms.GeneratePayoutsReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, cms.APIRoute, cms.RouteGeneratePayouts, gp) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var gpr cms.GeneratePayoutsReply - err = json.Unmarshal(responseBody, &gpr) + err = json.Unmarshal(respBody, &gpr) if err != nil { return nil, fmt.Errorf("unmarshal GeneratePayoutsReply: %v", err) } @@ -1052,14 +1277,21 @@ func (c *Client) GeneratePayouts(gp *cms.GeneratePayouts) (*cms.GeneratePayoutsR // approved invoices to the paid status. This will be removed once the // address watching for payment is complete and working. func (c *Client) PayInvoices(pi *cms.PayInvoices) (*cms.PayInvoicesReply, error) { - responseBody, err := c.makeRequest(http.MethodGet, + statusCode, respBody, err := c.makeRequest(http.MethodGet, cms.APIRoute, cms.RoutePayInvoices, pi) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var pir cms.PayInvoicesReply - err = json.Unmarshal(responseBody, &pir) + err = json.Unmarshal(respBody, &pir) if err != nil { return nil, fmt.Errorf("unmarshal PayInvoiceReply: %v", err) } @@ -1070,14 +1302,21 @@ func (c *Client) PayInvoices(pi *cms.PayInvoices) (*cms.PayInvoicesReply, error) // VoteInventory retrieves the tokens of all proposals in the inventory // categorized by their vote status. func (c *Client) VoteInventory() (*pi.VoteInventoryReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, + statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, pi.RouteVoteInventory, nil) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = piError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var vir pi.VoteInventoryReply - err = json.Unmarshal(responseBody, &vir) + err = json.Unmarshal(respBody, &vir) if err != nil { return nil, fmt.Errorf("unmarshal VoteInventory: %v", err) } @@ -1094,14 +1333,21 @@ func (c *Client) VoteInventory() (*pi.VoteInventoryReply, error) { // BatchProposals retrieves a list of proposals func (c *Client) BatchProposals(bp *www.BatchProposals) (*www.BatchProposalsReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, www.PoliteiaWWWAPIRoute, - www.RouteBatchProposals, bp) + statusCode, respBody, err := c.makeRequest(http.MethodPost, + www.PoliteiaWWWAPIRoute, www.RouteBatchProposals, bp) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var bpr www.BatchProposalsReply - err = json.Unmarshal(responseBody, &bpr) + err = json.Unmarshal(respBody, &bpr) if err != nil { return nil, fmt.Errorf("unmarshal BatchProposals: %v", err) } @@ -1119,14 +1365,21 @@ func (c *Client) BatchProposals(bp *www.BatchProposals) (*www.BatchProposalsRepl // VoteSummaries retrieves a summary of the voting process for a set of // proposals. func (c *Client) VoteSummaries(vs *pi.VoteSummaries) (*pi.VoteSummariesReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, + statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, pi.RouteVoteSummaries, vs) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = piError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var vsr pi.VoteSummariesReply - err = json.Unmarshal(responseBody, &vsr) + err = json.Unmarshal(respBody, &vsr) if err != nil { return nil, fmt.Errorf("unmarshal BatchVoteSummary: %v", err) } @@ -1143,14 +1396,21 @@ func (c *Client) VoteSummaries(vs *pi.VoteSummaries) (*pi.VoteSummariesReply, er // GetAllVetted retrieves a page of vetted proposals. func (c *Client) GetAllVetted(gav *www.GetAllVetted) (*www.GetAllVettedReply, error) { - responseBody, err := c.makeRequest(http.MethodGet, + statusCode, respBody, err := c.makeRequest(http.MethodGet, www.PoliteiaWWWAPIRoute, www.RouteAllVetted, gav) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var gavr www.GetAllVettedReply - err = json.Unmarshal(responseBody, &gavr) + err = json.Unmarshal(respBody, &gavr) if err != nil { return nil, fmt.Errorf("unmarshal GetAllVettedReply: %v", err) } @@ -1167,14 +1427,21 @@ func (c *Client) GetAllVetted(gav *www.GetAllVetted) (*www.GetAllVettedReply, er // WWWNewComment submits a new proposal comment for the logged in user. func (c *Client) WWWNewComment(nc *www.NewComment) (*www.NewCommentReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, www.PoliteiaWWWAPIRoute, www.RouteNewComment, nc) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var ncr www.NewCommentReply - err = json.Unmarshal(responseBody, &ncr) + err = json.Unmarshal(respBody, &ncr) if err != nil { return nil, fmt.Errorf("unmarshal NewCommentReply: %v", err) } @@ -1191,14 +1458,21 @@ func (c *Client) WWWNewComment(nc *www.NewComment) (*www.NewCommentReply, error) // CommentNew submits a new proposal comment for the logged in user. func (c *Client) CommentNew(cn pi.CommentNew) (*pi.CommentNewReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, pi.RouteCommentNew, cn) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = piError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var cnr pi.CommentNewReply - err = json.Unmarshal(responseBody, &cnr) + err = json.Unmarshal(respBody, &cnr) if err != nil { return nil, fmt.Errorf("unmarshal CommentNewReply: %v", err) } @@ -1216,14 +1490,21 @@ func (c *Client) CommentNew(cn pi.CommentNew) (*pi.CommentNewReply, error) { // CommentVote casts a like comment action (upvote/downvote) for the logged in // user. func (c *Client) CommentVote(cv pi.CommentVote) (*pi.CommentVoteReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, pi.RouteCommentVote, cv) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = piError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var cvr pi.CommentVoteReply - err = json.Unmarshal(responseBody, &cvr) + err = json.Unmarshal(respBody, &cvr) if err != nil { return nil, fmt.Errorf("unmarshal CommentVoteReply: %v", err) } @@ -1240,14 +1521,21 @@ func (c *Client) CommentVote(cv pi.CommentVote) (*pi.CommentVoteReply, error) { // CommentCensor censors the specified proposal comment. func (c *Client) CommentCensor(cc pi.CommentCensor) (*pi.CommentCensorReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, + statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, pi.RouteCommentCensor, cc) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = piError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var ccr pi.CommentCensorReply - err = json.Unmarshal(responseBody, &ccr) + err = json.Unmarshal(respBody, &ccr) if err != nil { return nil, fmt.Errorf("unmarshal CensorCommentReply: %v", err) } @@ -1264,14 +1552,21 @@ func (c *Client) CommentCensor(cc pi.CommentCensor) (*pi.CommentCensorReply, err // Comments retrieves the comments for the specified proposal. func (c *Client) Comments(cs pi.Comments) (*pi.CommentsReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, pi.RouteComments, &cs) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = piError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var cr pi.CommentsReply - err = json.Unmarshal(responseBody, &cr) + err = json.Unmarshal(respBody, &cr) if err != nil { return nil, fmt.Errorf("unmarshal CommentsReply: %v", err) } @@ -1289,14 +1584,21 @@ func (c *Client) Comments(cs pi.Comments) (*pi.CommentsReply, error) { // CommentVotes retrieves the comment likes (upvotes/downvotes) for the // specified proposal that are from the privoded user. func (c *Client) CommentVotes(cv pi.CommentVotes) (*pi.CommentVotesReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, pi.RouteCommentVotes, cv) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = piError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var cvr pi.CommentVotesReply - err = json.Unmarshal(responseBody, &cvr) + err = json.Unmarshal(respBody, &cvr) if err != nil { return nil, fmt.Errorf("unmarshal CommentVotes: %v", err) } @@ -1314,14 +1616,21 @@ func (c *Client) CommentVotes(cv pi.CommentVotes) (*pi.CommentVotesReply, error) // InvoiceComments retrieves the comments for the specified proposal. func (c *Client) InvoiceComments(token string) (*www.GetCommentsReply, error) { route := "/invoices/" + token + "/comments" - responseBody, err := c.makeRequest(http.MethodGet, + statusCode, respBody, err := c.makeRequest(http.MethodGet, cms.APIRoute, route, nil) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var gcr www.GetCommentsReply - err = json.Unmarshal(responseBody, &gcr) + err = json.Unmarshal(respBody, &gcr) if err != nil { return nil, fmt.Errorf("unmarshal InvoiceCommentsReply: %v", err) } @@ -1338,14 +1647,21 @@ func (c *Client) InvoiceComments(token string) (*www.GetCommentsReply, error) { // Votes rerieves the vote details for a given proposal. func (c *Client) Votes(vs pi.Votes) (*pi.VotesReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, pi.RouteVotes, vs) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = piError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var vsr pi.VotesReply - err = json.Unmarshal(responseBody, &vsr) + err = json.Unmarshal(respBody, &vsr) if err != nil { return nil, fmt.Errorf("unmarshal Votes: %v", err) } @@ -1362,14 +1678,21 @@ func (c *Client) Votes(vs pi.Votes) (*pi.VotesReply, error) { // WWWCensorComment censors the specified proposal comment. func (c *Client) WWWCensorComment(cc *www.CensorComment) (*www.CensorCommentReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, www.PoliteiaWWWAPIRoute, - www.RouteCensorComment, cc) + statusCode, respBody, err := c.makeRequest(http.MethodPost, + www.PoliteiaWWWAPIRoute, www.RouteCensorComment, cc) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var ccr www.CensorCommentReply - err = json.Unmarshal(responseBody, &ccr) + err = json.Unmarshal(respBody, &ccr) if err != nil { return nil, fmt.Errorf("unmarshal CensorCommentReply: %v", err) } @@ -1386,14 +1709,21 @@ func (c *Client) WWWCensorComment(cc *www.CensorComment) (*www.CensorCommentRepl // VoteStart sends the provided VoteStart to pi. func (c *Client) VoteStart(vs pi.VoteStart) (*pi.VoteStartReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, pi.RouteVoteStart, vs) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = piError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var vsr pi.VoteStartReply - err = json.Unmarshal(responseBody, &vsr) + err = json.Unmarshal(respBody, &vsr) if err != nil { return nil, fmt.Errorf("unmarshal VoteStartReply: %v", err) } @@ -1412,14 +1742,21 @@ func (c *Client) VoteStart(vs pi.VoteStart) (*pi.VoteStartReply, error) { // VoteStartRunoff sends the given VoteStartRunoff to the pi api // RouteVoteStartRunoff and returns the reply. func (c *Client) VoteStartRunoff(vsr pi.VoteStartRunoff) (*pi.VoteStartRunoffReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, pi.RouteVoteStartRunoff, vsr) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = piError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var vsrr pi.VoteStartRunoffReply - err = json.Unmarshal(responseBody, &vsrr) + err = json.Unmarshal(respBody, &vsrr) if err != nil { return nil, err } @@ -1438,16 +1775,24 @@ func (c *Client) VoteStartRunoff(vsr pi.VoteStartRunoff) (*pi.VoteStartRunoffRep // UserRegistrationPayment checks whether the logged in user has paid their user // registration fee. func (c *Client) UserRegistrationPayment() (*www.UserRegistrationPaymentReply, error) { - responseBody, err := c.makeRequest(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteUserRegistrationPayment, nil) + statusCode, respBody, err := c.makeRequest(http.MethodGet, + www.PoliteiaWWWAPIRoute, www.RouteUserRegistrationPayment, nil) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var urpr www.UserRegistrationPaymentReply - err = json.Unmarshal(responseBody, &urpr) + err = json.Unmarshal(respBody, &urpr) if err != nil { - return nil, fmt.Errorf("unmarshal UserRegistrationPaymentReply: %v", err) + return nil, fmt.Errorf("unmarshal UserRegistrationPaymentReply: %v", + err) } if c.cfg.Verbose { @@ -1462,14 +1807,21 @@ func (c *Client) UserRegistrationPayment() (*www.UserRegistrationPaymentReply, e // VoteResults retrieves the vote results for the specified proposal. func (c *Client) VoteResults(vr pi.VoteResults) (*pi.VoteResultsReply, error) { - responseBody, err := c.makeRequest(http.MethodGet, + statusCode, respBody, err := c.makeRequest(http.MethodGet, pi.APIRoute, pi.RouteVoteResults, vr) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = piError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var vrr pi.VoteResultsReply - err = json.Unmarshal(responseBody, &vrr) + err = json.Unmarshal(respBody, &vrr) if err != nil { return nil, fmt.Errorf("unmarshal ProposalVotesReply: %v", err) } @@ -1488,11 +1840,19 @@ func (c *Client) VoteResults(vr pi.VoteResults) (*pi.VoteResultsReply, error) { // the www v2 VoteDetails route. func (c *Client) VoteDetailsV2(token string) (*www2.VoteDetailsReply, error) { route := "/vote/" + token - respBody, err := c.makeRequest(http.MethodGet, www2.APIRoute, route, nil) + statusCode, respBody, err := c.makeRequest(http.MethodGet, www2.APIRoute, + route, nil) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var vdr www2.VoteDetailsReply err = json.Unmarshal(respBody, &vdr) if err != nil { @@ -1515,14 +1875,21 @@ func (c *Client) VoteDetailsV2(token string) (*www2.VoteDetailsReply, error) { // UserDetails retrieves the user details for the specified user. func (c *Client) UserDetails(userID string) (*www.UserDetailsReply, error) { route := "/user/" + userID - responseBody, err := c.makeRequest(http.MethodGet, + statusCode, respBody, err := c.makeRequest(http.MethodGet, www.PoliteiaWWWAPIRoute, route, nil) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var udr www.UserDetailsReply - err = json.Unmarshal(responseBody, &udr) + err = json.Unmarshal(respBody, &udr) if err != nil { return nil, fmt.Errorf("unmarshal UserDetailsReply: %v", err) } @@ -1540,14 +1907,21 @@ func (c *Client) UserDetails(userID string) (*www.UserDetailsReply, error) { // Users retrieves a list of users that adhere to the specified filtering // parameters. func (c *Client) Users(u *www.Users) (*www.UsersReply, error) { - responseBody, err := c.makeRequest(http.MethodGet, + statusCode, respBody, err := c.makeRequest(http.MethodGet, www.PoliteiaWWWAPIRoute, www.RouteUsers, u) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var ur www.UsersReply - err = json.Unmarshal(responseBody, &ur) + err = json.Unmarshal(respBody, &ur) if err != nil { return nil, fmt.Errorf("unmarshal UsersReply: %v", err) } @@ -1565,14 +1939,21 @@ func (c *Client) Users(u *www.Users) (*www.UsersReply, error) { // CMSUsers retrieves a list of cms users that adhere to the specified filtering // parameters. func (c *Client) CMSUsers(cu *cms.CMSUsers) (*cms.CMSUsersReply, error) { - responseBody, err := c.makeRequest(http.MethodGet, cms.APIRoute, + statusCode, respBody, err := c.makeRequest(http.MethodGet, cms.APIRoute, cms.RouteCMSUsers, cu) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var cur cms.CMSUsersReply - err = json.Unmarshal(responseBody, &cur) + err = json.Unmarshal(respBody, &cur) if err != nil { return nil, fmt.Errorf("unmarshal CMSUsersReply: %v", err) } @@ -1589,14 +1970,21 @@ func (c *Client) CMSUsers(cu *cms.CMSUsers) (*cms.CMSUsersReply, error) { // ManageUser allows an admin to edit certain attributes of the specified user. func (c *Client) ManageUser(mu *www.ManageUser) (*www.ManageUserReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, www.PoliteiaWWWAPIRoute, www.RouteManageUser, mu) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var mur www.ManageUserReply - err = json.Unmarshal(responseBody, &mur) + err = json.Unmarshal(respBody, &mur) if err != nil { return nil, fmt.Errorf("unmarshal ManageUserReply: %v", err) } @@ -1613,14 +2001,21 @@ func (c *Client) ManageUser(mu *www.ManageUser) (*www.ManageUserReply, error) { // EditUser allows the logged in user to update their user settings. func (c *Client) EditUser(eu *www.EditUser) (*www.EditUserReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, www.PoliteiaWWWAPIRoute, www.RouteEditUser, eu) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var eur www.EditUserReply - err = json.Unmarshal(responseBody, &eur) + err = json.Unmarshal(respBody, &eur) if err != nil { return nil, fmt.Errorf("unmarshal EditUserReply: %v", err) } @@ -1638,14 +2033,21 @@ func (c *Client) EditUser(eu *www.EditUser) (*www.EditUserReply, error) { // VoteAuthorize authorizes the voting period for the specified proposal using // the logged in user. func (c *Client) VoteAuthorize(va pi.VoteAuthorize) (*pi.VoteAuthorizeReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, + statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, pi.RouteVoteAuthorize, va) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = piError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var vr pi.VoteAuthorizeReply - err = json.Unmarshal(responseBody, &vr) + err = json.Unmarshal(respBody, &vr) if err != nil { return nil, fmt.Errorf("unmarshal VoteAuthorizeReply: %v", err) } @@ -1663,14 +2065,21 @@ func (c *Client) VoteAuthorize(va pi.VoteAuthorize) (*pi.VoteAuthorizeReply, err // VoteStatus retrieves the vote status for the specified proposal. func (c *Client) VoteStatus(token string) (*www.VoteStatusReply, error) { route := "/proposals/" + token + "/votestatus" - responseBody, err := c.makeRequest(http.MethodGet, + statusCode, respBody, err := c.makeRequest(http.MethodGet, www.PoliteiaWWWAPIRoute, route, nil) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var vsr www.VoteStatusReply - err = json.Unmarshal(responseBody, &vsr) + err = json.Unmarshal(respBody, &vsr) if err != nil { return nil, fmt.Errorf("unmarshal VoteStatusReply: %v", err) } @@ -1687,14 +2096,21 @@ func (c *Client) VoteStatus(token string) (*www.VoteStatusReply, error) { // GetAllVoteStatus retreives the vote status of all public proposals. func (c *Client) GetAllVoteStatus() (*www.GetAllVoteStatusReply, error) { - responseBody, err := c.makeRequest(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteAllVoteStatus, nil) + statusCode, respBody, err := c.makeRequest(http.MethodGet, + www.PoliteiaWWWAPIRoute, www.RouteAllVoteStatus, nil) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var avsr www.GetAllVoteStatusReply - err = json.Unmarshal(responseBody, &avsr) + err = json.Unmarshal(respBody, &avsr) if err != nil { return nil, fmt.Errorf("unmarshal GetAllVoteStatusReply: %v", err) } @@ -1711,14 +2127,21 @@ func (c *Client) GetAllVoteStatus() (*www.GetAllVoteStatusReply, error) { // ActiveVotesDCC retreives all dccs that are currently being voted on. func (c *Client) ActiveVotesDCC() (*cms.ActiveVoteReply, error) { - responseBody, err := c.makeRequest(http.MethodGet, + statusCode, respBody, err := c.makeRequest(http.MethodGet, www.PoliteiaWWWAPIRoute, cms.RouteActiveVotesDCC, nil) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var avr cms.ActiveVoteReply - err = json.Unmarshal(responseBody, &avr) + err = json.Unmarshal(respBody, &avr) if err != nil { return nil, fmt.Errorf("unmarshal ActiveVoteDCCReply: %v", err) } @@ -1735,14 +2158,21 @@ func (c *Client) ActiveVotesDCC() (*cms.ActiveVoteReply, error) { // VoteBallot casts ballot of votes for a proposal. func (c *Client) VoteBallot(vb *pi.VoteBallot) (*pi.VoteBallotReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, pi.RouteVoteBallot, &vb) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = piError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var vbr pi.VoteBallotReply - err = json.Unmarshal(responseBody, &vbr) + err = json.Unmarshal(respBody, &vbr) if err != nil { return nil, fmt.Errorf("unmarshal VoteBallotReply: %v", err) } @@ -1759,14 +2189,21 @@ func (c *Client) VoteBallot(vb *pi.VoteBallot) (*pi.VoteBallotReply, error) { // UpdateUserKey updates the identity of the logged in user. func (c *Client) UpdateUserKey(uuk *www.UpdateUserKey) (*www.UpdateUserKeyReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, www.PoliteiaWWWAPIRoute, - www.RouteUpdateUserKey, &uuk) + statusCode, respBody, err := c.makeRequest(http.MethodPost, + www.PoliteiaWWWAPIRoute, www.RouteUpdateUserKey, &uuk) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var uukr www.UpdateUserKeyReply - err = json.Unmarshal(responseBody, &uukr) + err = json.Unmarshal(respBody, &uukr) if err != nil { return nil, fmt.Errorf("unmarshal UpdateUserKeyReply: %v", err) } @@ -1783,14 +2220,21 @@ func (c *Client) UpdateUserKey(uuk *www.UpdateUserKey) (*www.UpdateUserKeyReply, // VerifyUpdateUserKey is used to verify a new user identity. func (c *Client) VerifyUpdateUserKey(vuuk *www.VerifyUpdateUserKey) (*www.VerifyUpdateUserKeyReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, www.PoliteiaWWWAPIRoute, - www.RouteVerifyUpdateUserKey, &vuuk) + statusCode, respBody, err := c.makeRequest(http.MethodPost, + www.PoliteiaWWWAPIRoute, www.RouteVerifyUpdateUserKey, &vuuk) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var vuukr www.VerifyUpdateUserKeyReply - err = json.Unmarshal(responseBody, &vuukr) + err = json.Unmarshal(respBody, &vuukr) if err != nil { return nil, fmt.Errorf("unmarshal VerifyUpdateUserKeyReply: %v", err) } @@ -1808,14 +2252,21 @@ func (c *Client) VerifyUpdateUserKey(vuuk *www.VerifyUpdateUserKey) (*www.Verify // UserProposalPaywallTx retrieves payment details of any pending proposal // credit payment from the logged in user. func (c *Client) UserProposalPaywallTx() (*www.UserProposalPaywallTxReply, error) { - responseBody, err := c.makeRequest(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteUserProposalPaywallTx, nil) + statusCode, respBody, err := c.makeRequest(http.MethodGet, + www.PoliteiaWWWAPIRoute, www.RouteUserProposalPaywallTx, nil) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var upptxr www.UserProposalPaywallTxReply - err = json.Unmarshal(responseBody, &upptxr) + err = json.Unmarshal(respBody, &upptxr) if err != nil { return nil, fmt.Errorf("unmarshal ProposalPaywallPaymentReply: %v", err) } @@ -1833,14 +2284,21 @@ func (c *Client) UserProposalPaywallTx() (*www.UserProposalPaywallTxReply, error // UserPaymentsRescan scans the specified user's paywall address and makes sure // that the user's account has been properly credited with all payments. func (c *Client) UserPaymentsRescan(upr *www.UserPaymentsRescan) (*www.UserPaymentsRescanReply, error) { - responseBody, err := c.makeRequest(http.MethodPut, www.PoliteiaWWWAPIRoute, - www.RouteUserPaymentsRescan, upr) + statusCode, respBody, err := c.makeRequest(http.MethodPut, + www.PoliteiaWWWAPIRoute, www.RouteUserPaymentsRescan, upr) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var uprr www.UserPaymentsRescanReply - err = json.Unmarshal(responseBody, &uprr) + err = json.Unmarshal(respBody, &uprr) if err != nil { return nil, fmt.Errorf("unmarshal UserPaymentsRescanReply: %v", err) } @@ -1858,14 +2316,21 @@ func (c *Client) UserPaymentsRescan(upr *www.UserPaymentsRescan) (*www.UserPayme // UserProposalCredits retrieves the proposal credit history for the logged // in user. func (c *Client) UserProposalCredits() (*www.UserProposalCreditsReply, error) { - responseBody, err := c.makeRequest(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteUserProposalCredits, nil) + statusCode, respBody, err := c.makeRequest(http.MethodGet, + www.PoliteiaWWWAPIRoute, www.RouteUserProposalCredits, nil) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var upcr www.UserProposalCreditsReply - err = json.Unmarshal(responseBody, &upcr) + err = json.Unmarshal(respBody, &upcr) if err != nil { return nil, fmt.Errorf("unmarshal UserProposalCreditsReply: %v", err) } @@ -1883,12 +2348,19 @@ func (c *Client) UserProposalCredits() (*www.UserProposalCreditsReply, error) { // ResendVerification re-sends the user verification email for an unverified // user. func (c *Client) ResendVerification(rv www.ResendVerification) (*www.ResendVerificationReply, error) { - respBody, err := c.makeRequest(http.MethodPost, www.PoliteiaWWWAPIRoute, - www.RouteResendVerification, rv) + statusCode, respBody, err := c.makeRequest(http.MethodPost, + www.PoliteiaWWWAPIRoute, www.RouteResendVerification, rv) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var rvr www.ResendVerificationReply err = json.Unmarshal(respBody, &rvr) if err != nil { @@ -1908,13 +2380,21 @@ func (c *Client) ResendVerification(rv www.ResendVerification) (*www.ResendVerif // InvoiceDetails retrieves the specified invoice. func (c *Client) InvoiceDetails(token string, id *cms.InvoiceDetails) (*cms.InvoiceDetailsReply, error) { route := "/invoices/" + token - responseBody, err := c.makeRequest(http.MethodGet, cms.APIRoute, route, id) + statusCode, respBody, err := c.makeRequest(http.MethodGet, cms.APIRoute, + route, id) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var idr cms.InvoiceDetailsReply - err = json.Unmarshal(responseBody, &idr) + err = json.Unmarshal(respBody, &idr) if err != nil { return nil, fmt.Errorf("unmarshal InvoiceDetailsReply: %v", err) } @@ -1932,14 +2412,21 @@ func (c *Client) InvoiceDetails(token string, id *cms.InvoiceDetails) (*cms.Invo // SetInvoiceStatus changes the status of the specified invoice. func (c *Client) SetInvoiceStatus(sis *cms.SetInvoiceStatus) (*cms.SetInvoiceStatusReply, error) { route := "/invoices/" + sis.Token + "/status" - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, cms.APIRoute, route, sis) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var sisr cms.SetInvoiceStatusReply - err = json.Unmarshal(responseBody, &sisr) + err = json.Unmarshal(respBody, &sisr) if err != nil { return nil, fmt.Errorf("unmarshal SetInvoiceStatusReply: %v", err) } @@ -1957,14 +2444,21 @@ func (c *Client) SetInvoiceStatus(sis *cms.SetInvoiceStatus) (*cms.SetInvoiceSta // TokenInventory retrieves the censorship record tokens of all proposals in // the inventory. func (c *Client) TokenInventory() (*www.TokenInventoryReply, error) { - responseBody, err := c.makeRequest(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteTokenInventory, nil) + statusCode, respBody, err := c.makeRequest(http.MethodGet, + www.PoliteiaWWWAPIRoute, www.RouteTokenInventory, nil) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var tir www.TokenInventoryReply - err = json.Unmarshal(responseBody, &tir) + err = json.Unmarshal(respBody, &tir) if err != nil { return nil, fmt.Errorf("unmarshal TokenInventoryReply: %v", err) } @@ -1981,14 +2475,21 @@ func (c *Client) TokenInventory() (*www.TokenInventoryReply, error) { // InvoiceExchangeRate changes the status of the specified invoice. func (c *Client) InvoiceExchangeRate(ier *cms.InvoiceExchangeRate) (*cms.InvoiceExchangeRateReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, cms.APIRoute, cms.RouteInvoiceExchangeRate, ier) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var ierr cms.InvoiceExchangeRateReply - err = json.Unmarshal(responseBody, &ierr) + err = json.Unmarshal(respBody, &ierr) if err != nil { return nil, fmt.Errorf("unmarshal SetInvoiceStatusReply: %v", err) } @@ -2005,14 +2506,21 @@ func (c *Client) InvoiceExchangeRate(ier *cms.InvoiceExchangeRate) (*cms.Invoice // InvoicePayouts retrieves invoices base on possible field set in the request // month/year and/or status func (c *Client) InvoicePayouts(lip *cms.InvoicePayouts) (*cms.InvoicePayoutsReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, cms.APIRoute, cms.RouteInvoicePayouts, lip) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var lipr cms.InvoicePayoutsReply - err = json.Unmarshal(responseBody, &lipr) + err = json.Unmarshal(respBody, &lipr) if err != nil { return nil, fmt.Errorf("unmarshal InvoicePayouts: %v", err) } @@ -2029,13 +2537,21 @@ func (c *Client) InvoicePayouts(lip *cms.InvoicePayouts) (*cms.InvoicePayoutsRep // CMSUserDetails returns the current cms user's information. func (c *Client) CMSUserDetails(userID string) (*cms.UserDetailsReply, error) { route := "/user/" + userID - responseBody, err := c.makeRequest(http.MethodGet, cms.APIRoute, route, nil) + statusCode, respBody, err := c.makeRequest(http.MethodGet, cms.APIRoute, + route, nil) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var uir cms.UserDetailsReply - err = json.Unmarshal(responseBody, &uir) + err = json.Unmarshal(respBody, &uir) if err != nil { return nil, fmt.Errorf("unmarshal CMSUserDetailsReply: %v", err) } @@ -2052,14 +2568,21 @@ func (c *Client) CMSUserDetails(userID string) (*cms.UserDetailsReply, error) { // CMSEditUser edits the current user's information. func (c *Client) CMSEditUser(uui cms.EditUser) (*cms.EditUserReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, cms.APIRoute, www.RouteEditUser, uui) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var eur cms.EditUserReply - err = json.Unmarshal(responseBody, &eur) + err = json.Unmarshal(respBody, &eur) if err != nil { return nil, fmt.Errorf("unmarshal CMSEditUserReply: %v", err) } @@ -2076,14 +2599,21 @@ func (c *Client) CMSEditUser(uui cms.EditUser) (*cms.EditUserReply, error) { // CMSManageUser updates the given user's information. func (c *Client) CMSManageUser(uui cms.CMSManageUser) (*cms.CMSManageUserReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, cms.APIRoute, + statusCode, respBody, err := c.makeRequest(http.MethodPost, cms.APIRoute, cms.RouteManageCMSUser, uui) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var eur cms.CMSManageUserReply - err = json.Unmarshal(responseBody, &eur) + err = json.Unmarshal(respBody, &eur) if err != nil { return nil, fmt.Errorf("unmarshal CMSManageUserReply: %v", err) } @@ -2100,14 +2630,21 @@ func (c *Client) CMSManageUser(uui cms.CMSManageUser) (*cms.CMSManageUserReply, // NewDCC creates a new dcc proposal. func (c *Client) NewDCC(nd cms.NewDCC) (*cms.NewDCCReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, cms.APIRoute, cms.RouteNewDCC, nd) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var ndr cms.NewDCCReply - err = json.Unmarshal(responseBody, &ndr) + err = json.Unmarshal(respBody, &ndr) if err != nil { return nil, fmt.Errorf("unmarshal NewDCCReply: %v", err) } @@ -2124,14 +2661,21 @@ func (c *Client) NewDCC(nd cms.NewDCC) (*cms.NewDCCReply, error) { // SupportOpposeDCC issues support for a given DCC proposal. func (c *Client) SupportOpposeDCC(sd cms.SupportOpposeDCC) (*cms.SupportOpposeDCCReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, cms.APIRoute, cms.RouteSupportOpposeDCC, sd) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var sdr cms.SupportOpposeDCCReply - err = json.Unmarshal(responseBody, &sdr) + err = json.Unmarshal(respBody, &sdr) if err != nil { return nil, fmt.Errorf("unmarshal SupportOpposeDCCReply: %v", err) } @@ -2148,14 +2692,21 @@ func (c *Client) SupportOpposeDCC(sd cms.SupportOpposeDCC) (*cms.SupportOpposeDC // NewDCCComment submits a new dcc comment for the logged in user. func (c *Client) NewDCCComment(nc *www.NewComment) (*www.NewCommentReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, cms.APIRoute, cms.RouteNewCommentDCC, nc) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var ncr www.NewCommentReply - err = json.Unmarshal(responseBody, &ncr) + err = json.Unmarshal(respBody, &ncr) if err != nil { return nil, fmt.Errorf("unmarshal NewDCCCommentReply: %v", err) } @@ -2173,13 +2724,21 @@ func (c *Client) NewDCCComment(nc *www.NewComment) (*www.NewCommentReply, error) // DCCComments retrieves the comments for the specified proposal. func (c *Client) DCCComments(token string) (*www.GetCommentsReply, error) { route := "/dcc/" + token + "/comments" - responseBody, err := c.makeRequest(http.MethodGet, cms.APIRoute, route, nil) + statusCode, respBody, err := c.makeRequest(http.MethodGet, cms.APIRoute, + route, nil) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var gcr www.GetCommentsReply - err = json.Unmarshal(responseBody, &gcr) + err = json.Unmarshal(respBody, &gcr) if err != nil { return nil, fmt.Errorf("unmarshal DCCCommentsReply: %v", err) } @@ -2197,13 +2756,21 @@ func (c *Client) DCCComments(token string) (*www.GetCommentsReply, error) { // DCCDetails retrieves the specified dcc. func (c *Client) DCCDetails(token string) (*cms.DCCDetailsReply, error) { route := "/dcc/" + token - responseBody, err := c.makeRequest(http.MethodGet, cms.APIRoute, route, nil) + statusCode, respBody, err := c.makeRequest(http.MethodGet, cms.APIRoute, + route, nil) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var ddr cms.DCCDetailsReply - err = json.Unmarshal(responseBody, &ddr) + err = json.Unmarshal(respBody, &ddr) if err != nil { return nil, fmt.Errorf("unmarshal DCCDetailsReply: %v", err) } @@ -2218,17 +2785,24 @@ func (c *Client) DCCDetails(token string) (*cms.DCCDetailsReply, error) { return &ddr, nil } -// GetDCCss retrieves invoices base on possible field set in the request +// GetDCCs retrieves invoices base on possible field set in the request // month/year and/or status func (c *Client) GetDCCs(gd *cms.GetDCCs) (*cms.GetDCCsReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, cms.APIRoute, cms.RouteGetDCCs, gd) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var gdr cms.GetDCCsReply - err = json.Unmarshal(responseBody, &gdr) + err = json.Unmarshal(respBody, &gdr) if err != nil { return nil, fmt.Errorf("unmarshal GetDCCsReply: %v", err) } @@ -2246,13 +2820,21 @@ func (c *Client) GetDCCs(gd *cms.GetDCCs) (*cms.GetDCCsReply, error) { // SetDCCStatus issues an status update for a given DCC proposal. func (c *Client) SetDCCStatus(sd *cms.SetDCCStatus) (*cms.SetDCCStatusReply, error) { route := "/dcc/" + sd.Token + "/status" - responseBody, err := c.makeRequest(http.MethodPost, cms.APIRoute, route, sd) + statusCode, respBody, err := c.makeRequest(http.MethodPost, cms.APIRoute, + route, sd) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var sdr cms.SetDCCStatusReply - err = json.Unmarshal(responseBody, &sdr) + err = json.Unmarshal(respBody, &sdr) if err != nil { return nil, fmt.Errorf("unmarshal SetDCCStatusReply: %v", err) } @@ -2269,13 +2851,21 @@ func (c *Client) SetDCCStatus(sd *cms.SetDCCStatus) (*cms.SetDCCStatusReply, err // UserSubContractors retrieves the subcontractors that are linked to the requesting user func (c *Client) UserSubContractors(usc *cms.UserSubContractors) (*cms.UserSubContractorsReply, error) { - responseBody, err := c.makeRequest(http.MethodGet, + statusCode, respBody, err := c.makeRequest(http.MethodGet, cms.APIRoute, cms.RouteUserSubContractors, usc) if err != nil { return nil, err } + + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var uscr cms.UserSubContractorsReply - err = json.Unmarshal(responseBody, &uscr) + err = json.Unmarshal(respBody, &uscr) if err != nil { return nil, fmt.Errorf("unmarshal UserSubContractorsReply: %v", err) } @@ -2291,14 +2881,21 @@ func (c *Client) UserSubContractors(usc *cms.UserSubContractors) (*cms.UserSubCo // ProposalOwner retrieves the subcontractors that are linked to the requesting user func (c *Client) ProposalOwner(po *cms.ProposalOwner) (*cms.ProposalOwnerReply, error) { - responseBody, err := c.makeRequest(http.MethodGet, + statusCode, respBody, err := c.makeRequest(http.MethodGet, cms.APIRoute, cms.RouteProposalOwner, po) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var por cms.ProposalOwnerReply - err = json.Unmarshal(responseBody, &por) + err = json.Unmarshal(respBody, &por) if err != nil { return nil, fmt.Errorf("unmarshal ProposalOwnerReply: %v", err) } @@ -2315,13 +2912,21 @@ func (c *Client) ProposalOwner(po *cms.ProposalOwner) (*cms.ProposalOwnerReply, // CastVoteDCC issues a signed vote for a given DCC proposal. approval func (c *Client) CastVoteDCC(cv cms.CastVote) (*cms.CastVoteReply, error) { - responseBody, err := c.makeRequest("POST", cms.APIRoute, cms.RouteCastVoteDCC, - cv) + statusCode, respBody, err := c.makeRequest("POST", cms.APIRoute, + cms.RouteCastVoteDCC, cv) if err != nil { return nil, err } + + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var cvr cms.CastVoteReply - err = json.Unmarshal(responseBody, &cvr) + err = json.Unmarshal(respBody, &cvr) if err != nil { return nil, fmt.Errorf("unmarshal VoteDCCReply: %v", err) } @@ -2339,13 +2944,21 @@ func (c *Client) CastVoteDCC(cv cms.CastVote) (*cms.CastVoteReply, error) { // VoteDetailsDCC returns all the needed information about a given vote for a // DCC proposal. func (c *Client) VoteDetailsDCC(cv cms.VoteDetails) (*cms.VoteDetailsReply, error) { - responseBody, err := c.makeRequest("POST", cms.APIRoute, cms.RouteVoteDetailsDCC, - cv) + statusCode, respBody, err := c.makeRequest("POST", cms.APIRoute, + cms.RouteVoteDetailsDCC, cv) if err != nil { return nil, err } + + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var vdr cms.VoteDetailsReply - err = json.Unmarshal(responseBody, &vdr) + err = json.Unmarshal(respBody, &vdr) if err != nil { return nil, fmt.Errorf("unmarshal VoteDCCReply: %v", err) } @@ -2360,16 +2973,23 @@ func (c *Client) VoteDetailsDCC(cv cms.VoteDetails) (*cms.VoteDetailsReply, erro return &vdr, nil } -// StartVoteV2 sends the provided v2 StartVote to the politeiawww backend. +// StartVoteDCC sends the provided StartVoteDCC to the politeiawww backend. func (c *Client) StartVoteDCC(sv cms.StartVote) (*cms.StartVoteReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, cms.APIRoute, cms.RouteStartVoteDCC, sv) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var svr cms.StartVoteReply - err = json.Unmarshal(responseBody, &svr) + err = json.Unmarshal(respBody, &svr) if err != nil { return nil, fmt.Errorf("unmarshal StartVoteReply: %v", err) } @@ -2464,14 +3084,21 @@ func (c *Client) SignMessages(sm *walletrpc.SignMessagesRequest) (*walletrpc.Sig // SetTOTP sets the logged in user's TOTP Key. func (c *Client) SetTOTP(st *www.SetTOTP) (*www.SetTOTPReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, cms.APIRoute, www.RouteSetTOTP, st) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var str www.SetTOTPReply - err = json.Unmarshal(responseBody, &str) + err = json.Unmarshal(respBody, &str) if err != nil { return nil, fmt.Errorf("unmarshal SetTOTPReply: %v", err) } @@ -2488,14 +3115,21 @@ func (c *Client) SetTOTP(st *www.SetTOTP) (*www.SetTOTPReply, error) { // VerifyTOTP comfirms the logged in user's TOTP Key. func (c *Client) VerifyTOTP(vt *www.VerifyTOTP) (*www.VerifyTOTPReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, + statusCode, respBody, err := c.makeRequest(http.MethodPost, cms.APIRoute, www.RouteVerifyTOTP, vt) if err != nil { return nil, err } + if statusCode != http.StatusOK { + err = wwwError(respBody, statusCode) + if err != nil { + return nil, err + } + } + var vtr www.VerifyTOTPReply - err = json.Unmarshal(responseBody, &vtr) + err = json.Unmarshal(respBody, &vtr) if err != nil { return nil, fmt.Errorf("unmarshal VerifyTOTPReply: %v", err) } @@ -2537,7 +3171,7 @@ func (c *Client) Close() { } } -// New returns a new politeiawww client. +// NewClient returns a new politeiawww client. func NewClient(cfg *Config) (*Client, error) { // Create http client tlsConfig := &tls.Config{ From 5fcf45f3f0d7a470b68c21581bf48c4153b289b5 Mon Sep 17 00:00:00 2001 From: Amir Massarwa Date: Fri, 9 Oct 2020 16:52:42 +0300 Subject: [PATCH 145/449] piwww: testrun - add user routes. --- politeiawww/cmd/piwww/testrun.go | 755 ++++++++++++-------- politeiawww/cmd/shared/userpasswordreset.go | 2 +- politeiawww/cmd/shared/usertotpverify.go | 4 +- 3 files changed, 444 insertions(+), 317 deletions(-) diff --git a/politeiawww/cmd/piwww/testrun.go b/politeiawww/cmd/piwww/testrun.go index b75ccd3ce..150c2c810 100644 --- a/politeiawww/cmd/piwww/testrun.go +++ b/politeiawww/cmd/piwww/testrun.go @@ -5,7 +5,15 @@ package main import ( + "context" + "encoding/hex" + "fmt" + "time" + + "github.com/decred/politeia/politeiad/api/v1/identity" + www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/cmd/shared" + "github.com/decred/politeia/util" ) // testRunCmd performs a test run of all the politeiawww routes. @@ -16,8 +24,8 @@ type testRunCmd struct { } `positional-args:"true" required:"true"` } -// user stores user details that are used throughout the test run. -type user struct { +// testUser stores user details that are used throughout the test run. +type testUser struct { ID string // UUID Email string // Email Username string // Username @@ -26,372 +34,491 @@ type user struct { } // login logs in the specified user. -func login(email, password string) error { +func login(u testUser) error { lc := shared.LoginCmd{} - lc.Args.Email = email - lc.Args.Password = password + lc.Args.Email = u.Email + lc.Args.Password = u.Password return lc.Execute(nil) } -// castVotes casts votes on a proposal with a given voteId. If it fails it -// returns the error and in case of dcrwallet connection error it returns -// true as first returned value -func castVotes(token string, voteID string) (bool, error) { - /* var vc VoteCmd - vc.Args.Token = token - vc.Args.VoteID = voteID - err := vc.Execute(nil) - if err != nil { - switch { - case strings.Contains(err.Error(), "connection refused"): - // User is not running a dcrwallet instance locally. - // This is ok. Print a warning and continue. - fmt.Printf(" WARNING: could not connect to dcrwallet\n") - return true, err - - case strings.Contains(err.Error(), "no eligible tickets"): - // User doesn't have any eligible tickets. This is ok. - // Print a warning and continue. - fmt.Printf(" WARNING: user has no elibigle tickets\n") - return true, err - - default: - return false, err - } +// logout logs out current logged in user. +func logout() error { + // Logout admin + lc := shared.LogoutCmd{} + err := lc.Execute(nil) + if err != nil { + return err } - return false, err*/ - return false, nil + return nil } -// Execute executes the test run command. -func (cmd *testRunCmd) Execute(args []string) error { - /* - const ( - // sleepInterval is the time to wait in between requests - // when polling politeiawww for paywall tx confirmations - // or RFP vote results. - sleepInterval = 15 * time.Second - - // Comment actions - commentActionUpvote = "upvote" - commentActionDownvote = "downvote" - ) - - var ( - // paywallEnabled represents whether the politeiawww paywall - // has been enabled. A disabled paywall will have a paywall - // address of "" and a paywall amount of 0. - paywallEnabled bool - - // numCredits is the number of proposal credits that will be - // purchased using the testnet faucet. - numCredits = v1.ProposalListPageSize * 2 - - // Test users - user user - admin user - ) - - // Suppress output from cli commands - cfg.Silent = true - - // Policy - fmt.Printf("Policy\n") - policy, err := client.Policy() - if err != nil { - return err - } +// userRegistrationPayment ensures current logged in user has paid registration fee +func userRegistrationPayment() (www.UserRegistrationPaymentReply, error) { + urvr, err := client.UserRegistrationPayment() + if err != nil { + return www.UserRegistrationPaymentReply{}, err + } + return *urvr, nil +} - // Version (CSRF tokens) - fmt.Printf("Version\n") - version, err := client.Version() - if err != nil { - return err - } +// randomString generates a random string +func randomString(length int) (string, error) { + b, err := util.Random(length) + if err != nil { + return "", err + } + return hex.EncodeToString(b), nil +} - // We only allow this to be run on testnet for right now. - // Running it on mainnet would require changing the user - // email verification flow. - // We ensure vote duration isn't longer than - // 3 blocks as we need to approve an RFP and it's - // submission as part of our tests. - switch { - case !version.TestNet: - return fmt.Errorf("this command must be run on testnet") - case policy.MinVoteDuration > 3: - return fmt.Errorf("--votedurationmin flag should be <= 3, as the " + - "tests include RFP & submssions voting") - } +// userNew creates a new user and returnes user's public key. +func userNew(email, password, username string) (*identity.FullIdentity, error) { + fmt.Printf(" Creating user: %v\n", email) - // Ensure admin credentials are valid and that the admin has - // paid their user registration fee. - fmt.Printf("Validating admin credentials\n") - admin.email = cmd.Args.AdminEmail - admin.password = cmd.Args.AdminPassword - err = login(admin.email, admin.password) - if err != nil { - return err - } + // Create user identity and save it to disk + id, err := shared.NewIdentity() + if err != nil { + return nil, err + } - vupr, err := client.VerifyUserPayment() - if err != nil { - return err - } - if !vupr.HasPaid { - return fmt.Errorf("admin has not paid registration fee") - } + // Setup new user request + nu := &www.NewUser{ + Email: email, + Username: username, + Password: shared.DigestSHA3(password), + PublicKey: hex.EncodeToString(id.Public.Key[:]), + } + _, err = client.NewUser(nu) + if err != nil { + return nil, err + } + + return id, nil +} + +// userManage sends a usermanage command +func userManage(userID, action, reason string) error { + muc := shared.UserManageCmd{} + muc.Args.UserID = userID + muc.Args.Action = action + muc.Args.Reason = reason + err := muc.Execute(nil) + if err != nil { + return err + } + return nil +} + +// testUser tests piwww user specific routes. +func testUserRoutes(admin testUser, minPasswordLength int) error { + // sleepInterval is the time to wait in between requests + // when polling politeiawww for paywall tx confirmations. + const sleepInterval = 15 * time.Second + + var ( + // paywallEnabled represents whether the politeiawww paywall + // has been enabled. A disabled paywall will have a paywall + // address of "" and a paywall amount of 0. + paywallEnabled bool + + // numCredits is the number of proposal credits that will be + // purchased using the testnet faucet. + numCredits = 1 + + // Test users + user testUser + ) + // Run user routes. + fmt.Printf("Running user routes\n") + + // Create user and verify email + randomStr, err := randomString(minPasswordLength) + if err != nil { + return err + } + email := randomStr + "@example.com" + username := randomStr + password := randomStr + id, err := userNew(email, password, username) + if err != nil { + return err + } + + // Resed email verification + fmt.Printf(" Resend email Verification\n") + rvr, err := client.ResendVerification(www.ResendVerification{ + PublicKey: hex.EncodeToString(id.Public.Key[:]), + Email: email, + }) + if err != nil { + return err + } + + // Verify email + fmt.Printf(" Verify user's email\n") + vt := rvr.VerificationToken + sig := id.SignMessage([]byte(vt)) + _, err = client.VerifyNewUser( + &www.VerifyNewUser{ + Email: email, + VerificationToken: vt, + Signature: hex.EncodeToString(sig[:]), + }) + if err != nil { + return err + } + + // Login and store user details + fmt.Printf(" Login user\n") + lr, err := client.Login(&www.Login{ + Email: email, + Password: shared.DigestSHA3(password), + }) + if err != nil { + return err + } + + user = testUser{ + ID: lr.UserID, + Email: email, + Username: username, + Password: password, + PublicKey: lr.PublicKey, + } + + // Logout user + fmt.Printf(" Logout user\n") + err = logout() + if err != nil { + return err + } + + // Log back in + err = login(user) + if err != nil { + return err + } + + // Me + fmt.Printf(" Me\n") + _, err = client.Me() + if err != nil { + return err + } + + // Edit user + fmt.Printf(" Edit user\n") + var n uint64 = 1 << 0 + _, err = client.EditUser( + &www.EditUser{ + EmailNotifications: &n, + }) + if err != nil { + return err + } + + // Update user key + fmt.Printf(" Update user key\n") + ukuc := shared.UserKeyUpdateCmd{} + err = ukuc.Execute(nil) + if err != nil { + return err + } + + // Change username + fmt.Printf(" Change username\n") + randomStr, err = randomString(minPasswordLength) + if err != nil { + return err + } + cuc := shared.UserUsernameChangeCmd{} + cuc.Args.Password = user.Password + cuc.Args.NewUsername = randomStr + err = cuc.Execute(nil) + if err != nil { + return err + } + user.Username = cuc.Args.NewUsername + + // Change password + fmt.Printf(" Change password\n") + cpc := shared.UserPasswordChangeCmd{} + cpc.Args.Password = user.Password + cpc.Args.NewPassword = randomStr + err = cpc.Execute(nil) + if err != nil { + return err + } + user.Password = cpc.Args.NewPassword - lc := shared.LogoutCmd{} - err = lc.Execute(nil) + // Reset user password + fmt.Printf(" Reset user password\n") + // Generate new random password + randomStr, err = randomString(minPasswordLength) + if err != nil { + return err + } + uprc := shared.UserPasswordResetCmd{} + uprc.Args.Email = user.Email + uprc.Args.Username = user.Username + uprc.Args.NewPassword = randomStr + err = uprc.Execute(nil) + if err != nil { + return err + } + user.Password = randomStr + + // Login with new password + err = login(user) + if err != nil { + return err + } + // Check if paywall is enabled. Paywall address and paywall + // amount will be zero values if paywall has been disabled. + if lr.PaywallAddress != "" && lr.PaywallAmount != 0 { + paywallEnabled = true + } else { + fmt.Printf("WARNING: politeiawww paywall is disabled\n") + } + + // Pay user registration fee + if paywallEnabled { + // Pay user registration fee + fmt.Printf(" Paying user registration fee\n") + txID, err := util.PayWithTestnetFaucet(context.Background(), + cfg.FaucetHost, lr.PaywallAddress, lr.PaywallAmount, "") if err != nil { return err } - // Create user and verify email - b, err := util.Random(int(policy.MinPasswordLength)) + dcr := float64(lr.PaywallAmount) / 1e8 + fmt.Printf(" Paid %v DCR to %v with txID %v\n", + dcr, lr.PaywallAddress, txID) + } + + // Wait for user registration payment confirmations + // If the paywall has been disable this will be marked + // as true. If the paywall has been enabled this will + // be true once the payment tx has the required number + // of confirmations. + upvr, err := userRegistrationPayment() + if err != nil { + return err + } + for !upvr.HasPaid { + upvr, err = userRegistrationPayment() if err != nil { return err } - email := hex.EncodeToString(b) + "@example.com" - username := hex.EncodeToString(b) - password := hex.EncodeToString(b) + fmt.Printf(" Verify user payment: waiting for tx confirmations...\n") + time.Sleep(sleepInterval) + } - fmt.Printf("Creating user: %v\n", email) + // Purchase proposal credits + fmt.Printf(" User proposal paywall\n") + ppdr, err := client.UserProposalPaywall() + if err != nil { + return err + } - nuc := NewUserCmd{ - Verify: true, - } - nuc.Args.Email = email - nuc.Args.Username = username - nuc.Args.Password = password - err = nuc.Execute(nil) - if err != nil { - return err - } + if paywallEnabled { + // Purchase proposal credits + fmt.Printf(" Purchasing %v proposal credits\n", numCredits) - // Login and store user details - fmt.Printf("Login user\n") - lr, err := client.Login( - &v1.Login{ - Email: email, - Password: shared.DigestSHA3(password), - }) + atoms := ppdr.CreditPrice * uint64(numCredits) + txID, err := util.PayWithTestnetFaucet(context.Background(), + cfg.FaucetHost, ppdr.PaywallAddress, atoms, "") if err != nil { return err } - user = user{ - ID: lr.UserID, - Email: email, - Username: username, - Password: password, - PublicKey: lr.PublicKey, - } + fmt.Printf(" Paid %v DCR to %v with txID %v\n", + float64(atoms)/1e8, lr.PaywallAddress, txID) + } - // Check if paywall is enabled. Paywall address and paywall - // amount will be zero values if paywall has been disabled. - if lr.PaywallAddress != "" && lr.PaywallAmount != 0 { - paywallEnabled = true - } else { - fmt.Printf("WARNING: politeiawww paywall is disabled\n") + // Keep track of when the pending proposal credit payment + // receives the required number of confirmations. + for { + pppr, err := client.UserProposalPaywallTx() + if err != nil { + return err } - // Run user routes. These are the routes - // that reqiure the user to be logged in. - fmt.Printf("Running user routes\n") - - // Pay user registration fee - if paywallEnabled { - // New proposal failure - registration fee not paid - fmt.Printf(" New proposal failure: registration fee not paid\n") - npc := NewProposalCmd{ - Random: true, - } - err = npc.Execute(nil) - if err == nil { - return fmt.Errorf("submited proposal without " + - "paying registration fee") - } - - // Pay user registration fee - fmt.Printf(" Paying user registration fee\n") - txID, err := util.PayWithTestnetFaucet(cfg.FaucetHost, - lr.PaywallAddress, lr.PaywallAmount, "") + // TxID will be blank if the paywall has been disabled + // or if the payment is no longer pending. + if pppr.TxID == "" { + // Verify that the correct number of proposal credits + // have been added to the user's account. + upcr, err := client.UserProposalCredits() if err != nil { return err } - dcr := float64(lr.PaywallAmount) / 1e8 - fmt.Printf(" Paid %v DCR to %v with txID %v\n", - dcr, lr.PaywallAddress, txID) + if !paywallEnabled || len(upcr.UnspentCredits) == numCredits { + break + } } - // Wait for user registration payment confirmations - // If the paywall has been disable this will be marked - // as true. If the paywall has been enabled this will - // be true once the payment tx has the required number - // of confirmations. - for !vupr.HasPaid { - vupr, err = client.VerifyUserPayment() - if err != nil { - return err - } + fmt.Printf(" Proposal paywall payment: waiting for tx confirmations...\n") + time.Sleep(sleepInterval) + } - fmt.Printf(" Verify user payment: waiting for tx confirmations...\n") - time.Sleep(sleepInterval) - } + // Fetch user by usernam + fmt.Printf(" Fetch user by username\n") + usersr, err := client.Users(&www.Users{ + Username: user.Username, + }) + if err != nil { + return err + } + if usersr.TotalMatches != 1 { + return fmt.Errorf("Wrong matching users: want %v, got %v", 1, + usersr.TotalMatches) + } - // Purchase proposal credits - fmt.Printf(" Proposal paywall details\n") - ppdr, err := client.UserProposalPaywall() - if err != nil { - return err - } + // Fetch user by public key + fmt.Printf(" Fetch user by public key\n") + usersr, err = client.Users(&www.Users{ + PublicKey: user.PublicKey, + }) + if err != nil { + return err + } + if usersr.TotalMatches != 1 { + return fmt.Errorf("Wrong matching users: want %v, got %v", 1, + usersr.TotalMatches) + } - if paywallEnabled { - // New proposal failure - no proposal credits - fmt.Printf(" New proposal failure: no proposal credits\n") - npc := NewProposalCmd{ - Random: true, - } - err = npc.Execute(nil) - if err == nil { - return fmt.Errorf("submited proposal without " + - "purchasing any proposal credits") - } + // User details + fmt.Printf(" User details\n") + udc := userDetailsCmd{} + udc.Args.UserID = user.ID + err = udc.Execute(nil) + if err != nil { + return err + } - // Purchase proposal credits - fmt.Printf(" Purchasing %v proposal credits\n", numCredits) + // Login admin + fmt.Printf(" Login as admin\n") + err = login(admin) + if err != nil { + return err + } - atoms := ppdr.CreditPrice * uint64(numCredits) - txID, err := util.PayWithTestnetFaucet(cfg.FaucetHost, - ppdr.PaywallAddress, atoms, "") - if err != nil { - return err - } + // Rescan user credits + fmt.Printf(" Rescan user credits\n") + upayrc := userPaymentsRescanCmd{} + upayrc.Args.UserID = user.ID + err = upayrc.Execute(nil) + if err != nil { + return err + } - fmt.Printf(" Paid %v DCR to %v with txID %v\n", - float64(atoms)/1e8, lr.PaywallAddress, txID) - } + // Deactivate user + fmt.Printf(" Deactivate user\n") + const userDeactivateAction = "deactivate" + err = userManage(user.ID, userDeactivateAction, "testing") + if err != nil { + return err + } - // Keep track of when the pending proposal credit payment - // receives the required number of confirmations. - for { - pppr, err := client.ProposalPaywallPayment() - if err != nil { - return err - } + // Reactivate user + fmt.Printf(" Reactivate user\n") + const userReactivateAction = "reactivate" + err = userManage(user.ID, userReactivateAction, "testing") + if err != nil { + return err + } - // TxID will be blank if the paywall has been disabled - // or if the payment is no longer pending. - if pppr.TxID == "" { - // Verify that the correct number of proposal credits - // have been added to the user's account. - upcr, err := client.UserProposalCredits() - if err != nil { - return err - } - - if !paywallEnabled || len(upcr.UnspentCredits) == numCredits { - break - } - } + // Fetch user by email + fmt.Printf(" Fetch user by email\n") + usersr, err = client.Users(&www.Users{ + Email: user.Email, + }) + if err != nil { + return err + } + if usersr.TotalMatches != 1 { + return fmt.Errorf("Wrong matching users: want %v, got %v", 1, + usersr.TotalMatches) + } - fmt.Printf(" Proposal paywall payment: waiting for tx confirmations...\n") - time.Sleep(sleepInterval) - } + return nil +} - // Me - fmt.Printf(" Me\n") - _, err = client.Me() - if err != nil { - return err - } +// Execute executes the test run command. +func (cmd *testRunCmd) Execute(args []string) error { - // Change password - fmt.Printf(" Change password\n") - b, err = util.Random(int(policy.MinPasswordLength)) - if err != nil { - return err - } - cpc := shared.ChangePasswordCmd{} - cpc.Args.Password = user.password - cpc.Args.NewPassword = hex.EncodeToString(b) - err = cpc.Execute(nil) - if err != nil { - return err - } - user.password = cpc.Args.NewPassword - - // Change username - fmt.Printf(" Change username\n") - cuc := shared.ChangeUsernameCmd{} - cuc.Args.Password = user.password - cuc.Args.NewUsername = hex.EncodeToString(b) - err = cuc.Execute(nil) - if err != nil { - return err - } - user.username = cuc.Args.NewUsername - - // Edit user - fmt.Printf(" Edit user\n") - var n uint64 = 1 << 0 - _, err = client.EditUser( - &v1.EditUser{ - EmailNotifications: &n, - }) - if err != nil { - return err - } + const ( + // Comment actions + commentActionUpvote = "upvote" + commentActionDownvote = "downvote" + ) - // Update user key - fmt.Printf(" Update user key\n") - var uukc shared.UpdateUserKeyCmd - err = uukc.Execute(nil) - if err != nil { - return err - } + // Suppress output from cli commands + cfg.Silent = true - // Websockets - fmt.Printf("Running websocket routes\n") + fmt.Printf("Running pre-testrun validation\n") - // Websocket - unauthenticated ping - fmt.Printf(" Websocket: unauthenticated ping: ") - sc := SubscribeCmd{ - Close: true, - } - err = sc.Execute([]string{"ping"}) - if err != nil { - return err - } + // Policy + fmt.Printf(" Policy\n") + policy, err := client.Policy() + if err != nil { + return err + } - // Login with user - fmt.Printf(" Login user\n") - err = login(user.email, user.password) - if err != nil { - return err - } + // Version (CSRF tokens) + fmt.Printf(" Version\n") + version, err := client.Version() + if err != nil { + return err + } - // Websocket - authenticated ping - fmt.Printf(" Websocket: authenticated ping: ") - err = sc.Execute([]string{"auth", "ping"}) - if err != nil { - return err - } + // We only allow this to be run on testnet for right now. + // Running it on mainnet would require changing the user + // email verification flow. + // We ensure vote duration isn't longer than + // 3 blocks as we need to approve an RFP and it's + // submission as part of our tests. + switch { + case !version.TestNet: + return fmt.Errorf("this command must be run on testnet") + case policy.MinVoteDuration > 3: + return fmt.Errorf("--votedurationmin flag should be <= 3, as the " + + "tests include RFP & submssions voting") + } - // Logout - fmt.Printf(" Logout\n") - lc = shared.LogoutCmd{} - err = lc.Execute(nil) - if err != nil { - return err - } + // Ensure admin credentials are valid + admin := testUser{ + Email: cmd.Args.AdminEmail, + Password: cmd.Args.AdminPassword, + } + err = login(admin) + if err != nil { + return err + } + + // Ensure admin paid registration free + urpr, err := userRegistrationPayment() + if err != nil { + return err + } + if !urpr.HasPaid { + return fmt.Errorf("admin has not paid registration fee") + } + + // Logout admin + err = logout() + if err != nil { + return err + } + + // Test user routes + err = testUserRoutes(admin, int(policy.MinPasswordLength)) + if err != nil { + return err + } - fmt.Printf("Test run successful!\n") - return nil - */ return nil } diff --git a/politeiawww/cmd/shared/userpasswordreset.go b/politeiawww/cmd/shared/userpasswordreset.go index 538c5620a..cc5b69cc1 100644 --- a/politeiawww/cmd/shared/userpasswordreset.go +++ b/politeiawww/cmd/shared/userpasswordreset.go @@ -66,7 +66,7 @@ func (cmd *UserPasswordResetCmd) Execute(args []string) error { return nil } - // Verify reset password + // Verify reset password vrp := www.VerifyResetPassword{ Username: username, VerificationToken: rpr.VerificationToken, diff --git a/politeiawww/cmd/shared/usertotpverify.go b/politeiawww/cmd/shared/usertotpverify.go index 4285a2088..76c1a0c4a 100644 --- a/politeiawww/cmd/shared/usertotpverify.go +++ b/politeiawww/cmd/shared/usertotpverify.go @@ -10,14 +10,14 @@ import ( v1 "github.com/decred/politeia/politeiawww/api/www/v1" ) -// UserTOTPVerifyCmd sets the TOTP key for the logged in user. +// UserTOTPVerifyCmd verifies the TOTP key for the logged in user. type UserTOTPVerifyCmd struct { Args struct { Code string `positional-arg-name:"code"` } `positional-args:"true"` } -// Execute executes the set totp command. +// Execute executes the verify totp command. func (cmd *UserTOTPVerifyCmd) Execute(args []string) error { // Setup new user request st := &v1.VerifyTOTP{ From ad85ab1596d7842b8640d8710a6669648a847573 Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Mon, 12 Oct 2020 12:44:32 -0300 Subject: [PATCH 146/449] piwww: handle status code errors properly. --- politeiawww/cmd/shared/client.go | 525 ++++++++++--------------------- 1 file changed, 158 insertions(+), 367 deletions(-) diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index a59b26f42..1c1500866 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -74,51 +74,94 @@ func userPiErrorStatus(e pi.ErrorStatusT) string { return "" } -// wwwError unmarshals the response body from makeRequest, and parses -// the error code and error context from the www api. +// wwwError unmarshals the response body from makeRequest, and handles any +// status code errors from the server. Parses the error code and error context +// from the www api, in case of user error. func wwwError(body []byte, statusCode int) error { - var ue www.UserError - err := json.Unmarshal(body, &ue) - if err != nil { - return fmt.Errorf("unmarshal UserError: %v", err) - } - if ue.ErrorCode != 0 { + switch statusCode { + case http.StatusBadRequest: + // User Error + var ue www.UserError + err := json.Unmarshal(body, &ue) + if err != nil { + return fmt.Errorf("unmarshal UserError: %v", err) + } + if ue.ErrorCode != 0 { + var e error + errMsg := userWWWErrorStatus(ue.ErrorCode) + if len(ue.ErrorContext) == 0 { + // Error format when an ErrorContext is not included + e = fmt.Errorf("%v, %v", statusCode, errMsg) + } else { + // Error format when an ErrorContext is included + e = fmt.Errorf("%v, %v: %v", statusCode, errMsg, + strings.Join(ue.ErrorContext, ", ")) + } + return e + } + case http.StatusInternalServerError: + // Server Error + var er www.ErrorReply + err := json.Unmarshal(body, &er) + if err != nil { + return fmt.Errorf("unmarshal Error: %v", err) + } var e error - errMsg := userWWWErrorStatus(ue.ErrorCode) - if len(ue.ErrorContext) == 0 { + if len(er.ErrorContext) == 0 { // Error format when an ErrorContext is not included - e = fmt.Errorf("%v, %v", statusCode, errMsg) + e = fmt.Errorf("ServerError timestamp: %v", er.ErrorCode) } else { // Error format when an ErrorContext is included - e = fmt.Errorf("%v, %v: %v", statusCode, errMsg, - strings.Join(ue.ErrorContext, ", ")) + e = fmt.Errorf("ServerError timestamp: %v context: %v", + er.ErrorCode, er.ErrorContext) } return e + default: + // Default Status Code Error + return fmt.Errorf("%v", statusCode) } + return nil } -// piError unmarshals the response body from makeRequest, and parses -// the error code and error context from the pi api. +// piError unmarshals the response body from makeRequest, and handles any +// status code errors from the server. Parses the error code and error context +// from the pi api, in case of user error. func piError(body []byte, statusCode int) error { - var ue pi.UserErrorReply - err := json.Unmarshal(body, &ue) - if err != nil { - return fmt.Errorf("unmarshal UserError: %v", err) - } - if ue.ErrorCode != 0 { - var e error - errMsg := userPiErrorStatus(ue.ErrorCode) - if len(ue.ErrorContext) == 0 { - // Error format when an ErrorContext is not included - e = fmt.Errorf("%v, %v", statusCode, errMsg) - } else { - // Error format when an ErrorContext is included - e = fmt.Errorf("%v, %v: %v", statusCode, errMsg, - strings.Join(ue.ErrorContext, ", ")) + switch statusCode { + case http.StatusBadRequest: + // User Error + var ue pi.UserErrorReply + err := json.Unmarshal(body, &ue) + if err != nil { + return fmt.Errorf("unmarshal UserError: %v", err) + } + if ue.ErrorCode != 0 { + var e error + errMsg := userPiErrorStatus(ue.ErrorCode) + if len(ue.ErrorContext) == 0 { + // Error format when an ErrorContext is not included + e = fmt.Errorf("%v, %v", statusCode, errMsg) + } else { + // Error format when an ErrorContext is included + e = fmt.Errorf("%v, %v: %v", statusCode, errMsg, + strings.Join(ue.ErrorContext, ", ")) + } + return e } - return e + case http.StatusInternalServerError: + // Server Error + var ser pi.ServerErrorReply + err := json.Unmarshal(body, &ser) + if err != nil { + return fmt.Errorf("unmarshal ServerError: %v", err) + } + return fmt.Errorf("ServerError timestamp: %v", ser.ErrorCode) + default: + // Return Status Code Error + return fmt.Errorf("%v", statusCode) } + return nil } @@ -438,10 +481,7 @@ func (c *Client) Policy() (*www.PolicyReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var pr www.PolicyReply @@ -469,10 +509,7 @@ func (c *Client) CMSPolicy() (*cms.PolicyReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var pr cms.PolicyReply @@ -500,10 +537,7 @@ func (c *Client) InviteNewUser(inu *cms.InviteNewUser) (*cms.InviteNewUserReply, } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var inur cms.InviteNewUserReply @@ -531,10 +565,7 @@ func (c *Client) RegisterUser(ru *cms.RegisterUser) (*cms.RegisterUserReply, err } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var rur cms.RegisterUserReply @@ -562,10 +593,7 @@ func (c *Client) NewUser(nu *www.NewUser) (*www.NewUserReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var nur www.NewUserReply @@ -593,10 +621,7 @@ func (c *Client) VerifyNewUser(vnu *www.VerifyNewUser) (*www.VerifyNewUserReply, } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var vnur www.VerifyNewUserReply @@ -624,10 +649,7 @@ func (c *Client) Me() (*www.LoginReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var lr www.LoginReply @@ -655,10 +677,7 @@ func (c *Client) Secret() (*www.UserError, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var ue www.UserError @@ -686,10 +705,7 @@ func (c *Client) ChangeUsername(cu *www.ChangeUsername) (*www.ChangeUsernameRepl } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var cur www.ChangeUsernameReply @@ -717,10 +733,7 @@ func (c *Client) ChangePassword(cp *www.ChangePassword) (*www.ChangePasswordRepl } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var cpr www.ChangePasswordReply @@ -748,10 +761,7 @@ func (c *Client) ResetPassword(rp *www.ResetPassword) (*www.ResetPasswordReply, } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var rpr www.ResetPasswordReply @@ -779,10 +789,7 @@ func (c *Client) VerifyResetPassword(vrp www.VerifyResetPassword) (*www.VerifyRe } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var reply www.VerifyResetPasswordReply @@ -811,10 +818,7 @@ func (c *Client) UserProposalPaywall() (*www.UserProposalPaywallReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var ppdr www.UserProposalPaywallReply @@ -842,10 +846,7 @@ func (c *Client) ProposalNew(pn pi.ProposalNew) (*pi.ProposalNewReply, error) { } if statusCode != http.StatusOK { - err = piError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, piError(respBody, statusCode) } var pnr pi.ProposalNewReply @@ -873,10 +874,7 @@ func (c *Client) ProposalEdit(pe pi.ProposalEdit) (*pi.ProposalEditReply, error) } if statusCode != http.StatusOK { - err = piError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, piError(respBody, statusCode) } var per pi.ProposalEditReply @@ -904,10 +902,7 @@ func (c *Client) ProposalStatusSet(pss pi.ProposalStatusSet) (*pi.ProposalStatus } if statusCode != http.StatusOK { - err = piError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, piError(respBody, statusCode) } var pssr pi.ProposalStatusSetReply @@ -935,10 +930,7 @@ func (c *Client) Proposals(p pi.Proposals) (*pi.ProposalsReply, error) { } if statusCode != http.StatusOK { - err = piError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, piError(respBody, statusCode) } var pr pi.ProposalsReply @@ -967,10 +959,7 @@ func (c *Client) ProposalInventory() (*pi.ProposalInventoryReply, error) { } if statusCode != http.StatusOK { - err = piError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, piError(respBody, statusCode) } var pir pi.ProposalInventoryReply @@ -999,10 +988,7 @@ func (c *Client) NewInvoice(ni *cms.NewInvoice) (*cms.NewInvoiceReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var nir cms.NewInvoiceReply @@ -1030,10 +1016,7 @@ func (c *Client) EditInvoice(ei *cms.EditInvoice) (*cms.EditInvoiceReply, error) } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var eir cms.EditInvoiceReply @@ -1062,10 +1045,7 @@ func (c *Client) ProposalDetails(token string, pd *www.ProposalsDetails) (*www.P } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var pr www.ProposalDetailsReply @@ -1094,10 +1074,7 @@ func (c *Client) UserInvoices(up *cms.UserInvoices) (*cms.UserInvoicesReply, err } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var upr cms.UserInvoicesReply @@ -1125,10 +1102,7 @@ func (c *Client) ProposalBilling(pb *cms.ProposalBilling) (*cms.ProposalBillingR } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var pbr cms.ProposalBillingReply @@ -1156,10 +1130,7 @@ func (c *Client) ProposalBillingDetails(pbd *cms.ProposalBillingDetails) (*cms.P } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var pbdr cms.ProposalBillingDetailsReply @@ -1187,10 +1158,7 @@ func (c *Client) ProposalBillingSummary(pbd *cms.ProposalBillingSummary) (*cms.P } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var pbdr cms.ProposalBillingSummaryReply @@ -1219,10 +1187,7 @@ func (c *Client) Invoices(ai *cms.Invoices) (*cms.InvoicesReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var air cms.InvoicesReply @@ -1251,10 +1216,7 @@ func (c *Client) GeneratePayouts(gp *cms.GeneratePayouts) (*cms.GeneratePayoutsR } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var gpr cms.GeneratePayoutsReply @@ -1284,10 +1246,7 @@ func (c *Client) PayInvoices(pi *cms.PayInvoices) (*cms.PayInvoicesReply, error) } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var pir cms.PayInvoicesReply @@ -1309,10 +1268,7 @@ func (c *Client) VoteInventory() (*pi.VoteInventoryReply, error) { } if statusCode != http.StatusOK { - err = piError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, piError(respBody, statusCode) } var vir pi.VoteInventoryReply @@ -1340,10 +1296,7 @@ func (c *Client) BatchProposals(bp *www.BatchProposals) (*www.BatchProposalsRepl } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var bpr www.BatchProposalsReply @@ -1372,10 +1325,7 @@ func (c *Client) VoteSummaries(vs *pi.VoteSummaries) (*pi.VoteSummariesReply, er } if statusCode != http.StatusOK { - err = piError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, piError(respBody, statusCode) } var vsr pi.VoteSummariesReply @@ -1403,10 +1353,7 @@ func (c *Client) GetAllVetted(gav *www.GetAllVetted) (*www.GetAllVettedReply, er } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var gavr www.GetAllVettedReply @@ -1434,10 +1381,7 @@ func (c *Client) WWWNewComment(nc *www.NewComment) (*www.NewCommentReply, error) } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var ncr www.NewCommentReply @@ -1465,10 +1409,7 @@ func (c *Client) CommentNew(cn pi.CommentNew) (*pi.CommentNewReply, error) { } if statusCode != http.StatusOK { - err = piError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, piError(respBody, statusCode) } var cnr pi.CommentNewReply @@ -1497,10 +1438,7 @@ func (c *Client) CommentVote(cv pi.CommentVote) (*pi.CommentVoteReply, error) { } if statusCode != http.StatusOK { - err = piError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, piError(respBody, statusCode) } var cvr pi.CommentVoteReply @@ -1528,10 +1466,7 @@ func (c *Client) CommentCensor(cc pi.CommentCensor) (*pi.CommentCensorReply, err } if statusCode != http.StatusOK { - err = piError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, piError(respBody, statusCode) } var ccr pi.CommentCensorReply @@ -1559,10 +1494,7 @@ func (c *Client) Comments(cs pi.Comments) (*pi.CommentsReply, error) { } if statusCode != http.StatusOK { - err = piError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, piError(respBody, statusCode) } var cr pi.CommentsReply @@ -1591,10 +1523,7 @@ func (c *Client) CommentVotes(cv pi.CommentVotes) (*pi.CommentVotesReply, error) } if statusCode != http.StatusOK { - err = piError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, piError(respBody, statusCode) } var cvr pi.CommentVotesReply @@ -1623,10 +1552,7 @@ func (c *Client) InvoiceComments(token string) (*www.GetCommentsReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var gcr www.GetCommentsReply @@ -1654,10 +1580,7 @@ func (c *Client) Votes(vs pi.Votes) (*pi.VotesReply, error) { } if statusCode != http.StatusOK { - err = piError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, piError(respBody, statusCode) } var vsr pi.VotesReply @@ -1685,10 +1608,7 @@ func (c *Client) WWWCensorComment(cc *www.CensorComment) (*www.CensorCommentRepl } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var ccr www.CensorCommentReply @@ -1716,10 +1636,7 @@ func (c *Client) VoteStart(vs pi.VoteStart) (*pi.VoteStartReply, error) { } if statusCode != http.StatusOK { - err = piError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, piError(respBody, statusCode) } var vsr pi.VoteStartReply @@ -1749,10 +1666,7 @@ func (c *Client) VoteStartRunoff(vsr pi.VoteStartRunoff) (*pi.VoteStartRunoffRep } if statusCode != http.StatusOK { - err = piError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, piError(respBody, statusCode) } var vsrr pi.VoteStartRunoffReply @@ -1782,10 +1696,7 @@ func (c *Client) UserRegistrationPayment() (*www.UserRegistrationPaymentReply, e } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var urpr www.UserRegistrationPaymentReply @@ -1814,10 +1725,7 @@ func (c *Client) VoteResults(vr pi.VoteResults) (*pi.VoteResultsReply, error) { } if statusCode != http.StatusOK { - err = piError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, piError(respBody, statusCode) } var vrr pi.VoteResultsReply @@ -1847,10 +1755,7 @@ func (c *Client) VoteDetailsV2(token string) (*www2.VoteDetailsReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var vdr www2.VoteDetailsReply @@ -1882,10 +1787,7 @@ func (c *Client) UserDetails(userID string) (*www.UserDetailsReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var udr www.UserDetailsReply @@ -1914,10 +1816,7 @@ func (c *Client) Users(u *www.Users) (*www.UsersReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var ur www.UsersReply @@ -1946,10 +1845,7 @@ func (c *Client) CMSUsers(cu *cms.CMSUsers) (*cms.CMSUsersReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var cur cms.CMSUsersReply @@ -1977,10 +1873,7 @@ func (c *Client) ManageUser(mu *www.ManageUser) (*www.ManageUserReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var mur www.ManageUserReply @@ -2008,10 +1901,7 @@ func (c *Client) EditUser(eu *www.EditUser) (*www.EditUserReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var eur www.EditUserReply @@ -2040,10 +1930,7 @@ func (c *Client) VoteAuthorize(va pi.VoteAuthorize) (*pi.VoteAuthorizeReply, err } if statusCode != http.StatusOK { - err = piError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, piError(respBody, statusCode) } var vr pi.VoteAuthorizeReply @@ -2072,10 +1959,7 @@ func (c *Client) VoteStatus(token string) (*www.VoteStatusReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var vsr www.VoteStatusReply @@ -2103,10 +1987,7 @@ func (c *Client) GetAllVoteStatus() (*www.GetAllVoteStatusReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var avsr www.GetAllVoteStatusReply @@ -2134,10 +2015,7 @@ func (c *Client) ActiveVotesDCC() (*cms.ActiveVoteReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var avr cms.ActiveVoteReply @@ -2165,10 +2043,7 @@ func (c *Client) VoteBallot(vb *pi.VoteBallot) (*pi.VoteBallotReply, error) { } if statusCode != http.StatusOK { - err = piError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, piError(respBody, statusCode) } var vbr pi.VoteBallotReply @@ -2196,10 +2071,7 @@ func (c *Client) UpdateUserKey(uuk *www.UpdateUserKey) (*www.UpdateUserKeyReply, } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var uukr www.UpdateUserKeyReply @@ -2227,10 +2099,7 @@ func (c *Client) VerifyUpdateUserKey(vuuk *www.VerifyUpdateUserKey) (*www.Verify } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var vuukr www.VerifyUpdateUserKeyReply @@ -2259,10 +2128,7 @@ func (c *Client) UserProposalPaywallTx() (*www.UserProposalPaywallTxReply, error } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var upptxr www.UserProposalPaywallTxReply @@ -2291,10 +2157,7 @@ func (c *Client) UserPaymentsRescan(upr *www.UserPaymentsRescan) (*www.UserPayme } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var uprr www.UserPaymentsRescanReply @@ -2323,10 +2186,7 @@ func (c *Client) UserProposalCredits() (*www.UserProposalCreditsReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var upcr www.UserProposalCreditsReply @@ -2355,10 +2215,7 @@ func (c *Client) ResendVerification(rv www.ResendVerification) (*www.ResendVerif } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var rvr www.ResendVerificationReply @@ -2387,10 +2244,7 @@ func (c *Client) InvoiceDetails(token string, id *cms.InvoiceDetails) (*cms.Invo } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var idr cms.InvoiceDetailsReply @@ -2419,10 +2273,7 @@ func (c *Client) SetInvoiceStatus(sis *cms.SetInvoiceStatus) (*cms.SetInvoiceSta } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var sisr cms.SetInvoiceStatusReply @@ -2451,10 +2302,7 @@ func (c *Client) TokenInventory() (*www.TokenInventoryReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var tir www.TokenInventoryReply @@ -2482,10 +2330,7 @@ func (c *Client) InvoiceExchangeRate(ier *cms.InvoiceExchangeRate) (*cms.Invoice } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var ierr cms.InvoiceExchangeRateReply @@ -2513,10 +2358,7 @@ func (c *Client) InvoicePayouts(lip *cms.InvoicePayouts) (*cms.InvoicePayoutsRep } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var lipr cms.InvoicePayoutsReply @@ -2544,10 +2386,7 @@ func (c *Client) CMSUserDetails(userID string) (*cms.UserDetailsReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var uir cms.UserDetailsReply @@ -2575,10 +2414,7 @@ func (c *Client) CMSEditUser(uui cms.EditUser) (*cms.EditUserReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var eur cms.EditUserReply @@ -2606,10 +2442,7 @@ func (c *Client) CMSManageUser(uui cms.CMSManageUser) (*cms.CMSManageUserReply, } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var eur cms.CMSManageUserReply @@ -2637,10 +2470,7 @@ func (c *Client) NewDCC(nd cms.NewDCC) (*cms.NewDCCReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var ndr cms.NewDCCReply @@ -2668,10 +2498,7 @@ func (c *Client) SupportOpposeDCC(sd cms.SupportOpposeDCC) (*cms.SupportOpposeDC } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var sdr cms.SupportOpposeDCCReply @@ -2699,10 +2526,7 @@ func (c *Client) NewDCCComment(nc *www.NewComment) (*www.NewCommentReply, error) } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var ncr www.NewCommentReply @@ -2731,10 +2555,7 @@ func (c *Client) DCCComments(token string) (*www.GetCommentsReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var gcr www.GetCommentsReply @@ -2763,10 +2584,7 @@ func (c *Client) DCCDetails(token string) (*cms.DCCDetailsReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var ddr cms.DCCDetailsReply @@ -2795,10 +2613,7 @@ func (c *Client) GetDCCs(gd *cms.GetDCCs) (*cms.GetDCCsReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var gdr cms.GetDCCsReply @@ -2827,10 +2642,7 @@ func (c *Client) SetDCCStatus(sd *cms.SetDCCStatus) (*cms.SetDCCStatusReply, err } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var sdr cms.SetDCCStatusReply @@ -2858,10 +2670,7 @@ func (c *Client) UserSubContractors(usc *cms.UserSubContractors) (*cms.UserSubCo } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var uscr cms.UserSubContractorsReply @@ -2888,10 +2697,7 @@ func (c *Client) ProposalOwner(po *cms.ProposalOwner) (*cms.ProposalOwnerReply, } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var por cms.ProposalOwnerReply @@ -2919,10 +2725,7 @@ func (c *Client) CastVoteDCC(cv cms.CastVote) (*cms.CastVoteReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var cvr cms.CastVoteReply @@ -2951,10 +2754,7 @@ func (c *Client) VoteDetailsDCC(cv cms.VoteDetails) (*cms.VoteDetailsReply, erro } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var vdr cms.VoteDetailsReply @@ -2982,10 +2782,7 @@ func (c *Client) StartVoteDCC(sv cms.StartVote) (*cms.StartVoteReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var svr cms.StartVoteReply @@ -3091,10 +2888,7 @@ func (c *Client) SetTOTP(st *www.SetTOTP) (*www.SetTOTPReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var str www.SetTOTPReply @@ -3122,10 +2916,7 @@ func (c *Client) VerifyTOTP(vt *www.VerifyTOTP) (*www.VerifyTOTPReply, error) { } if statusCode != http.StatusOK { - err = wwwError(respBody, statusCode) - if err != nil { - return nil, err - } + return nil, wwwError(respBody, statusCode) } var vtr www.VerifyTOTPReply From b48e7bd83dd109bd2bbf8e33280700fa53110ffa Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Mon, 12 Oct 2020 14:41:11 -0300 Subject: [PATCH 147/449] eventmanager: Add notification for comment reply. --- politeiad/api/v1/v1.go | 2 +- politeiawww/comments.go | 17 +++++++ politeiawww/eventmanager.go | 92 +++++++++++++++++++------------------ 3 files changed, 66 insertions(+), 45 deletions(-) diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index e3063efa5..601d844ed 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -24,9 +24,9 @@ const ( IdentityRoute = "/v1/identity/" // Retrieve identity NewRecordRoute = "/v1/newrecord/" // New record UpdateUnvettedRoute = "/v1/updateunvetted/" // Update unvetted record + UpdateUnvettedMetadataRoute = "/v1/updateunvettedmd/" // Update unvetted metadata UpdateVettedRoute = "/v1/updatevetted/" // Update vetted record UpdateVettedMetadataRoute = "/v1/updatevettedmd/" // Update vetted metadata - UpdateUnvettedMetadataRoute = "/v1/updateunvettedmd/" // Update unvetted metadata GetUnvettedRoute = "/v1/getunvetted/" // Retrieve unvetted record GetVettedRoute = "/v1/getvetted/" // Retrieve vetted record InventoryByStatusRoute = "/v1/inventorybystatus/" // Inventory record tokens by status diff --git a/politeiawww/comments.go b/politeiawww/comments.go index 243a71b04..17482cc51 100644 --- a/politeiawww/comments.go +++ b/politeiawww/comments.go @@ -25,6 +25,23 @@ func (p *politeiawww) commentsAll(cp comments.GetAll) (*comments.GetAllReply, er return cr, nil } +// commentsGet returns the set of comments specified in the comment's id slice. +func (p *politeiawww) commentsGet(cg comments.Get) (*comments.GetReply, error) { + b, err := comments.EncodeGet(cg) + if err != nil { + return nil, err + } + r, err := p.pluginCommand(comments.ID, comments.CmdGet, string(b)) + if err != nil { + return nil, err + } + cgr, err := comments.DecodeGetReply([]byte(r)) + if err != nil { + return nil, err + } + return cgr, nil +} + // commentVotes returns the comment votes that meet the provided criteria. func (p *politeiawww) commentVotes(vs comments.Votes) (*comments.VotesReply, error) { b, err := comments.EncodeVotes(vs) diff --git a/politeiawww/eventmanager.go b/politeiawww/eventmanager.go index 2f58611e3..eb8c94a18 100644 --- a/politeiawww/eventmanager.go +++ b/politeiawww/eventmanager.go @@ -9,6 +9,7 @@ import ( "strconv" "sync" + "github.com/decred/politeia/politeiad/plugins/comments" pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/user" @@ -341,21 +342,16 @@ func (p *politeiawww) handleEventProposalStatusChange(ch chan interface{}) { } } -func (p *politeiawww) notifyProposalAuthorOnComment(d dataProposalComment) error { +func (p *politeiawww) notifyProposalAuthorOnComment(d dataProposalComment, userID, proposalName string) error { // Lookup proposal author to see if they should be sent a // notification. - pr, err := p.proposalRecordLatest(d.state, d.token) - if err != nil { - return fmt.Errorf("proposalRecordLatest %v %v: %v", - d.state, d.token, err) - } - userID, err := uuid.Parse(pr.UserID) + uuid, err := uuid.Parse(userID) if err != nil { return err } - author, err := p.db.UserGetById(userID) + author, err := p.db.UserGetById(uuid) if err != nil { - return fmt.Errorf("UserGetByID %v: %v", userID.String(), err) + return fmt.Errorf("UserGetByID %v: %v", uuid.String(), err) } // Check if notification should be sent to author @@ -372,10 +368,10 @@ func (p *politeiawww) notifyProposalAuthorOnComment(d dataProposalComment) error // Send notification eamil commentID := strconv.FormatUint(uint64(d.commentID), 10) return p.emailProposalCommentSubmitted(d.token, commentID, d.username, - proposalName(*pr), author.Email) + proposalName, author.Email) } -func (p *politeiawww) notifyParentAuthorOnComment(d dataProposalComment) error { +func (p *politeiawww) notifyParentAuthorOnComment(d dataProposalComment, proposalName string) error { // Verify this is a reply comment if d.parentID == 0 { return nil @@ -383,39 +379,38 @@ func (p *politeiawww) notifyParentAuthorOnComment(d dataProposalComment) error { // Lookup the parent comment author to check if they should receive // a reply notification. + parentComment, err := p.commentsGet(comments.Get{ + State: convertCommentsStateFromPi(d.state), + Token: d.token, + CommentIDs: []uint32{d.parentID}, + }) + if err != nil { + return err + } + userID, err := uuid.Parse(parentComment.Comments[0].UserID) + if err != nil { + return err + } + author, err := p.db.UserGetById(userID) + if err != nil { + return err + } - // Get the parent comment - // TODO - return nil - - /* - // Lookup the parent comment author - var author *user.User - - // Check if notification should be sent to author - switch { - case d.username == author.Username: - // Author commented on their own proposal - return nil - case !userNotificationEnabled(*author, - www.NotificationEmailCommentOnMyProposal): - // Author does not have notification bit set on - return nil - } - - // Get proposal. We need this proposal name for the notification. - pr, err := p.proposalRecordLatest(d.state, d.token) - if err != nil { - return fmt.Errorf("proposalRecordLatest %v %v: %v", - d.state, d.token, err) - } - - // Send notification eamil - commentID := strconv.FormatUint(uint64(d.commentID), 10) + // Check if notification should be sent + switch { + case d.username == author.Username: + // Author replied to their own comment + return nil + case !userNotificationEnabled(*author, + www.NotificationEmailCommentOnMyComment): + // Author does not have notification bit set on + return nil + } - return p.emailProposalCommentReply(d.token, commentID, d.username, - proposalName(*pr), author.Email) - */ + // Send notification email to parent comment author + commentID := strconv.FormatUint(uint64(d.commentID), 10) + return p.emailProposalCommentReply(d.token, commentID, d.username, + proposalName, author.Email) } type dataProposalComment struct { @@ -434,15 +429,24 @@ func (p *politeiawww) handleEventProposalComment(ch chan interface{}) { continue } + // Fetch the proposal record here to avoid calling this two times + // on the notify functions below + pr, err := p.proposalRecordLatest(d.state, d.token) + if err != nil { + err = fmt.Errorf("proposalRecordLatest %v %v: %v", + d.state, d.token, err) + goto next + } + // Notify the proposal author - err := p.notifyProposalAuthorOnComment(d) + err = p.notifyProposalAuthorOnComment(d, pr.UserID, proposalName(*pr)) if err != nil { err = fmt.Errorf("notifyProposalAuthorOnComment: %v", err) goto next } // Notify the parent comment author - err = p.notifyParentAuthorOnComment(d) + err = p.notifyParentAuthorOnComment(d, proposalName(*pr)) if err != nil { err = fmt.Errorf("notifyParentAuthorOnComment: %v", err) goto next From b9c9b3a2124c7f531e8c42f3043191a261128c7f Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Mon, 12 Oct 2020 16:13:39 -0300 Subject: [PATCH 148/449] multi: Pass request context for routes that make calls to d. --- politeiawww/comments.go | 14 ++-- politeiawww/decred.go | 8 +-- politeiawww/eventmanager.go | 8 ++- politeiawww/pi.go | 22 +++--- politeiawww/piwww.go | 135 ++++++++++++++++++------------------ politeiawww/plugin.go | 2 +- politeiawww/politeiad.go | 60 +++++----------- politeiawww/politeiawww.go | 12 ++-- politeiawww/proposals.go | 29 ++++---- politeiawww/ticketvote.go | 30 ++++---- 10 files changed, 152 insertions(+), 168 deletions(-) diff --git a/politeiawww/comments.go b/politeiawww/comments.go index 17482cc51..50298caa2 100644 --- a/politeiawww/comments.go +++ b/politeiawww/comments.go @@ -5,16 +5,18 @@ package main import ( + "context" + "github.com/decred/politeia/politeiad/plugins/comments" ) // commentsAll returns all comments for the provided record. -func (p *politeiawww) commentsAll(cp comments.GetAll) (*comments.GetAllReply, error) { +func (p *politeiawww) commentsAll(ctx context.Context, cp comments.GetAll) (*comments.GetAllReply, error) { b, err := comments.EncodeGetAll(cp) if err != nil { return nil, err } - r, err := p.pluginCommand(comments.ID, comments.CmdGetAll, string(b)) + r, err := p.pluginCommand(ctx, comments.ID, comments.CmdGetAll, string(b)) if err != nil { return nil, err } @@ -26,12 +28,12 @@ func (p *politeiawww) commentsAll(cp comments.GetAll) (*comments.GetAllReply, er } // commentsGet returns the set of comments specified in the comment's id slice. -func (p *politeiawww) commentsGet(cg comments.Get) (*comments.GetReply, error) { +func (p *politeiawww) commentsGet(ctx context.Context, cg comments.Get) (*comments.GetReply, error) { b, err := comments.EncodeGet(cg) if err != nil { return nil, err } - r, err := p.pluginCommand(comments.ID, comments.CmdGet, string(b)) + r, err := p.pluginCommand(ctx, comments.ID, comments.CmdGet, string(b)) if err != nil { return nil, err } @@ -43,12 +45,12 @@ func (p *politeiawww) commentsGet(cg comments.Get) (*comments.GetReply, error) { } // commentVotes returns the comment votes that meet the provided criteria. -func (p *politeiawww) commentVotes(vs comments.Votes) (*comments.VotesReply, error) { +func (p *politeiawww) commentVotes(ctx context.Context, vs comments.Votes) (*comments.VotesReply, error) { b, err := comments.EncodeVotes(vs) if err != nil { return nil, err } - r, err := p.pluginCommand(comments.ID, comments.CmdVotes, string(b)) + r, err := p.pluginCommand(ctx, comments.ID, comments.CmdVotes, string(b)) if err != nil { return nil, err } diff --git a/politeiawww/decred.go b/politeiawww/decred.go index e8ac9acf3..93500f78d 100644 --- a/politeiawww/decred.go +++ b/politeiawww/decred.go @@ -24,9 +24,7 @@ func (p *politeiawww) decredGetComments(ctx context.Context, token string) ([]de } // Execute plugin command - // TODO FIXME - _ = ctx - reply, err := p.pluginCommand(decredplugin.ID, decredplugin.CmdGetComments, + reply, err := p.pluginCommand(ctx, decredplugin.ID, decredplugin.CmdGetComments, string(payload)) if err != nil { return nil, fmt.Errorf("pluginCommand %v %v: %v", @@ -50,9 +48,7 @@ func (p *politeiawww) decredBestBlock(ctx context.Context) (uint32, error) { } // Execute plugin command - // TODO FIXME - _ = ctx - reply, err := p.pluginCommand(decredplugin.ID, decredplugin.CmdBestBlock, + reply, err := p.pluginCommand(ctx, decredplugin.ID, decredplugin.CmdBestBlock, string(payload)) if err != nil { return 0, fmt.Errorf("pluginCommand %v %v: %v", diff --git a/politeiawww/eventmanager.go b/politeiawww/eventmanager.go index eb8c94a18..486a84ad4 100644 --- a/politeiawww/eventmanager.go +++ b/politeiawww/eventmanager.go @@ -5,6 +5,7 @@ package main import ( + "context" "fmt" "strconv" "sync" @@ -283,7 +284,7 @@ func (p *politeiawww) handleEventProposalStatusChange(ch chan interface{}) { // Get the proposal author state := convertPropStateFromPropStatus(d.status) - pr, err := p.proposalRecordLatest(state, d.token) + pr, err := p.proposalRecordLatest(context.Background(), state, d.token) if err != nil { log.Errorf("handleEventProposalStatusChange: proposalRecordLatest "+ "%v %v: %v", state, d.token, err) @@ -379,7 +380,7 @@ func (p *politeiawww) notifyParentAuthorOnComment(d dataProposalComment, proposa // Lookup the parent comment author to check if they should receive // a reply notification. - parentComment, err := p.commentsGet(comments.Get{ + parentComment, err := p.commentsGet(context.Background(), comments.Get{ State: convertCommentsStateFromPi(d.state), Token: d.token, CommentIDs: []uint32{d.parentID}, @@ -431,7 +432,8 @@ func (p *politeiawww) handleEventProposalComment(ch chan interface{}) { // Fetch the proposal record here to avoid calling this two times // on the notify functions below - pr, err := p.proposalRecordLatest(d.state, d.token) + pr, err := p.proposalRecordLatest(context.Background(), d.state, + d.token) if err != nil { err = fmt.Errorf("proposalRecordLatest %v %v: %v", d.state, d.token, err) diff --git a/politeiawww/pi.go b/politeiawww/pi.go index a9f805a15..4ed7e73f5 100644 --- a/politeiawww/pi.go +++ b/politeiawww/pi.go @@ -5,16 +5,18 @@ package main import ( + "context" + piplugin "github.com/decred/politeia/politeiad/plugins/pi" ) // piProposals returns the pi plugin data for the provided proposals. -func (p *politeiawww) piProposals(ps piplugin.Proposals) (*piplugin.ProposalsReply, error) { +func (p *politeiawww) piProposals(ctx context.Context, ps piplugin.Proposals) (*piplugin.ProposalsReply, error) { b, err := piplugin.EncodeProposals(ps) if err != nil { return nil, err } - r, err := p.pluginCommand(piplugin.ID, piplugin.CmdProposals, string(b)) + r, err := p.pluginCommand(ctx, piplugin.ID, piplugin.CmdProposals, string(b)) if err != nil { return nil, err } @@ -26,12 +28,12 @@ func (p *politeiawww) piProposals(ps piplugin.Proposals) (*piplugin.ProposalsRep } // piCommentNew uses the pi plugin to submit a new comment. -func (p *politeiawww) piCommentNew(cn piplugin.CommentNew) (*piplugin.CommentNewReply, error) { +func (p *politeiawww) piCommentNew(ctx context.Context, cn piplugin.CommentNew) (*piplugin.CommentNewReply, error) { b, err := piplugin.EncodeCommentNew(cn) if err != nil { return nil, err } - r, err := p.pluginCommand(piplugin.ID, piplugin.CmdCommentNew, string(b)) + r, err := p.pluginCommand(ctx, piplugin.ID, piplugin.CmdCommentNew, string(b)) if err != nil { return nil, err } @@ -43,12 +45,12 @@ func (p *politeiawww) piCommentNew(cn piplugin.CommentNew) (*piplugin.CommentNew } // piCommentVote uses the pi plugin to vote on a comment. -func (p *politeiawww) piCommentVote(cvp piplugin.CommentVote) (*piplugin.CommentVoteReply, error) { +func (p *politeiawww) piCommentVote(ctx context.Context, cvp piplugin.CommentVote) (*piplugin.CommentVoteReply, error) { b, err := piplugin.EncodeCommentVote(cvp) if err != nil { return nil, err } - r, err := p.pluginCommand(piplugin.ID, piplugin.CmdCommentVote, string(b)) + r, err := p.pluginCommand(ctx, piplugin.ID, piplugin.CmdCommentVote, string(b)) if err != nil { return nil, err } @@ -60,12 +62,12 @@ func (p *politeiawww) piCommentVote(cvp piplugin.CommentVote) (*piplugin.Comment } // piCommentCensor uses the pi plugin to censor a proposal comment. -func (p *politeiawww) piCommentCensor(cc piplugin.CommentCensor) (*piplugin.CommentCensorReply, error) { +func (p *politeiawww) piCommentCensor(ctx context.Context, cc piplugin.CommentCensor) (*piplugin.CommentCensorReply, error) { b, err := piplugin.EncodeCommentCensor(cc) if err != nil { return nil, err } - r, err := p.pluginCommand(piplugin.ID, piplugin.CmdCommentCensor, string(b)) + r, err := p.pluginCommand(ctx, piplugin.ID, piplugin.CmdCommentCensor, string(b)) if err != nil { return nil, err } @@ -77,8 +79,8 @@ func (p *politeiawww) piCommentCensor(cc piplugin.CommentCensor) (*piplugin.Comm } // piVoteInventory returns the pi plugin vote inventory. -func (p *politeiawww) piVoteInventory() (*piplugin.VoteInventoryReply, error) { - r, err := p.pluginCommand(piplugin.ID, piplugin.CmdVoteInventory, "") +func (p *politeiawww) piVoteInventory(ctx context.Context) (*piplugin.VoteInventoryReply, error) { + r, err := p.pluginCommand(ctx, piplugin.ID, piplugin.CmdVoteInventory, "") if err != nil { return nil, err } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index a6b8e4bfa..8c41deb6f 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -6,6 +6,7 @@ package main import ( "bytes" + "context" "encoding/base64" "encoding/hex" "encoding/json" @@ -720,7 +721,7 @@ func (p *politeiawww) linkByPeriodMax() int64 { // be returned even if the proposal Files are not returned, which means that we // will always need to fetch the record from politeiad with the files attached // since the proposal Metadata is saved to politeiad as a politeiad File. -func (p *politeiawww) proposalRecords(state pi.PropStateT, reqs []pi.ProposalRequest, includeFiles bool) (map[string]pi.ProposalRecord, error) { +func (p *politeiawww) proposalRecords(ctx context.Context, state pi.PropStateT, reqs []pi.ProposalRequest, includeFiles bool) (map[string]pi.ProposalRecord, error) { // Get politeiad records props := make([]pi.ProposalRecord, 0, len(reqs)) for _, v := range reqs { @@ -729,13 +730,13 @@ func (p *politeiawww) proposalRecords(state pi.PropStateT, reqs []pi.ProposalReq switch state { case pi.PropStateUnvetted: // Unvetted politeiad record - r, err = p.getUnvetted(v.Token, v.Version) + r, err = p.getUnvetted(ctx, v.Token, v.Version) if err != nil { return nil, err } case pi.PropStateVetted: // Vetted politeiad record - r, err = p.getVetted(v.Token, v.Version) + r, err = p.getVetted(ctx, v.Token, v.Version) if err != nil { return nil, err } @@ -776,7 +777,7 @@ func (p *politeiawww) proposalRecords(state pi.PropStateT, reqs []pi.ProposalReq State: convertPropStateFromPi(state), Tokens: tokens, } - psr, err := p.piProposals(ps) + psr, err := p.piProposals(ctx, ps) if err != nil { return nil, fmt.Errorf("proposalPluginData: %v", err) } @@ -823,8 +824,8 @@ func (p *politeiawww) proposalRecords(state pi.PropStateT, reqs []pi.ProposalReq // version. A blank version will return the most recent version. A // errProposalNotFound error will be returned if a proposal is not found for // the provided token/version combination. -func (p *politeiawww) proposalRecord(state pi.PropStateT, token, version string) (*pi.ProposalRecord, error) { - prs, err := p.proposalRecords(state, []pi.ProposalRequest{ +func (p *politeiawww) proposalRecord(ctx context.Context, state pi.PropStateT, token, version string) (*pi.ProposalRecord, error) { + prs, err := p.proposalRecords(ctx, state, []pi.ProposalRequest{ { Token: token, Version: version, @@ -843,8 +844,8 @@ func (p *politeiawww) proposalRecord(state pi.PropStateT, token, version string) // proposalRecordLatest returns the latest version of the proposal record for // the provided token. A errProposalNotFound error will be returned if a // proposal is not found for the provided token. -func (p *politeiawww) proposalRecordLatest(state pi.PropStateT, token string) (*pi.ProposalRecord, error) { - return p.proposalRecord(state, token, "") +func (p *politeiawww) proposalRecordLatest(ctx context.Context, state pi.PropStateT, token string) (*pi.ProposalRecord, error) { + return p.proposalRecord(ctx, state, token, "") } func (p *politeiawww) verifyProposalMetadata(pm pi.ProposalMetadata) error { @@ -1133,7 +1134,7 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu return &pm, nil } -func (p *politeiawww) processProposalNew(pn pi.ProposalNew, usr user.User) (*pi.ProposalNewReply, error) { +func (p *politeiawww) processProposalNew(ctx context.Context, pn pi.ProposalNew, usr user.User) (*pi.ProposalNewReply, error) { log.Tracef("processProposalNew: %v", usr.Username) // Verify user has paid registration paywall @@ -1196,7 +1197,7 @@ func (p *politeiawww) processProposalNew(pn pi.ProposalNew, usr user.User) (*pi. } // Send politeiad request - dcr, err := p.newRecord(metadata, files) + dcr, err := p.newRecord(ctx, metadata, files) if err != nil { return nil, err } @@ -1247,7 +1248,7 @@ func filesToDel(current []pi.File, updated []pi.File) []string { return del } -func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*pi.ProposalEditReply, error) { +func (p *politeiawww) processProposalEdit(ctx context.Context, pe pi.ProposalEdit, usr user.User) (*pi.ProposalEditReply, error) { log.Tracef("processProposalEdit: %v", pe.Token) // Verify token @@ -1283,7 +1284,7 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p } // Get the current proposal - curr, err := p.proposalRecordLatest(pe.State, pe.Token) + curr, err := p.proposalRecordLatest(ctx, pe.State, pe.Token) if err != nil { if err == errProposalNotFound { return nil, pi.UserErrorReply{ @@ -1356,13 +1357,13 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p var r *pd.Record switch pe.State { case pi.PropStateUnvetted: - r, err = p.updateUnvetted(pe.Token, mdAppend, mdOverwrite, + r, err = p.updateUnvetted(ctx, pe.Token, mdAppend, mdOverwrite, filesAdd, filesDel) if err != nil { return nil, err } case pi.PropStateVetted: - r, err = p.updateVetted(pe.Token, mdAppend, mdOverwrite, + r, err = p.updateVetted(ctx, pe.Token, mdAppend, mdOverwrite, filesAdd, filesDel) if err != nil { return nil, err @@ -1392,7 +1393,7 @@ func (p *politeiawww) processProposalEdit(pe pi.ProposalEdit, usr user.User) (*p }, nil } -func (p *politeiawww) processProposalStatusSet(pss pi.ProposalStatusSet, usr user.User) (*pi.ProposalStatusSetReply, error) { +func (p *politeiawww) processProposalStatusSet(ctx context.Context, pss pi.ProposalStatusSet, usr user.User) (*pi.ProposalStatusSetReply, error) { log.Tracef("processProposalStatusSet: %v %v", pss.Token, pss.Status) // Sanity check @@ -1484,12 +1485,12 @@ func (p *politeiawww) processProposalStatusSet(pss pi.ProposalStatusSet, usr use status := convertRecordStatusFromPropStatus(pss.Status) switch pss.State { case pi.PropStateUnvetted: - r, err = p.setUnvettedStatus(pss.Token, status, mdAppend, mdOverwrite) + r, err = p.setUnvettedStatus(ctx, pss.Token, status, mdAppend, mdOverwrite) if err != nil { return nil, err } case pi.PropStateVetted: - r, err = p.setVettedStatus(pss.Token, status, mdAppend, mdOverwrite) + r, err = p.setVettedStatus(ctx, pss.Token, status, mdAppend, mdOverwrite) if err != nil { return nil, err } @@ -1510,10 +1511,10 @@ func (p *politeiawww) processProposalStatusSet(pss pi.ProposalStatusSet, usr use }, nil } -func (p *politeiawww) processProposals(ps pi.Proposals, isAdmin bool) (*pi.ProposalsReply, error) { +func (p *politeiawww) processProposals(ctx context.Context, ps pi.Proposals, isAdmin bool) (*pi.ProposalsReply, error) { log.Tracef("processProposals: %v", ps.Requests) - props, err := p.proposalRecords(ps.State, ps.Requests, ps.IncludeFiles) + props, err := p.proposalRecords(ctx, ps.State, ps.Requests, ps.IncludeFiles) if err != nil { return nil, err } @@ -1537,10 +1538,10 @@ func (p *politeiawww) processProposals(ps pi.Proposals, isAdmin bool) (*pi.Propo }, nil } -func (p *politeiawww) processProposalInventory(isAdmin bool) (*pi.ProposalInventoryReply, error) { +func (p *politeiawww) processProposalInventory(ctx context.Context, isAdmin bool) (*pi.ProposalInventoryReply, error) { log.Tracef("processProposalInventory") - ir, err := p.inventoryByStatus() + ir, err := p.inventoryByStatus(ctx) if err != nil { return nil, err } @@ -1560,7 +1561,7 @@ func (p *politeiawww) processProposalInventory(isAdmin bool) (*pi.ProposalInvent return &reply, nil } -func (p *politeiawww) processCommentNew(cn pi.CommentNew, usr user.User) (*pi.CommentNewReply, error) { +func (p *politeiawww) processCommentNew(ctx context.Context, cn pi.CommentNew, usr user.User) (*pi.CommentNewReply, error) { log.Tracef("processCommentNew: %v", usr.Username) // Verify user has paid registration paywall @@ -1582,7 +1583,7 @@ func (p *politeiawww) processCommentNew(cn pi.CommentNew, usr user.User) (*pi.Co // unvetted proposals. if cn.State == pi.PropStateUnvetted && !usr.Admin { // Fetch the proposal so we can see who the author is - pr, err := p.proposalRecordLatest(cn.State, cn.Token) + pr, err := p.proposalRecordLatest(ctx, cn.State, cn.Token) if err != nil { if err == errProposalNotFound { return nil, pi.UserErrorReply{ @@ -1609,7 +1610,7 @@ func (p *politeiawww) processCommentNew(cn pi.CommentNew, usr user.User) (*pi.Co PublicKey: cn.PublicKey, Signature: cn.Signature, } - cnr, err := p.piCommentNew(pcn) + cnr, err := p.piCommentNew(ctx, pcn) if err != nil { return nil, err } @@ -1631,7 +1632,7 @@ func (p *politeiawww) processCommentNew(cn pi.CommentNew, usr user.User) (*pi.Co }, nil } -func (p *politeiawww) processCommentVote(cv pi.CommentVote, usr user.User) (*pi.CommentVoteReply, error) { +func (p *politeiawww) processCommentVote(ctx context.Context, cv pi.CommentVote, usr user.User) (*pi.CommentVoteReply, error) { log.Tracef("processCommentVote: %v %v %v", cv.Token, cv.CommentID, cv.Vote) // Verify state @@ -1667,7 +1668,7 @@ func (p *politeiawww) processCommentVote(cv pi.CommentVote, usr user.User) (*pi. PublicKey: cv.PublicKey, Signature: cv.Signature, } - cvr, err := p.piCommentVote(pcv) + cvr, err := p.piCommentVote(ctx, pcv) if err != nil { return nil, err } @@ -1680,7 +1681,7 @@ func (p *politeiawww) processCommentVote(cv pi.CommentVote, usr user.User) (*pi. }, nil } -func (p *politeiawww) processCommentCensor(cc pi.CommentCensor, usr user.User) (*pi.CommentCensorReply, error) { +func (p *politeiawww) processCommentCensor(ctx context.Context, cc pi.CommentCensor, usr user.User) (*pi.CommentCensorReply, error) { log.Tracef("processCommentCensor: %v %v", cc.Token, cc.CommentID) // Sanity check @@ -1705,7 +1706,7 @@ func (p *politeiawww) processCommentCensor(cc pi.CommentCensor, usr user.User) ( PublicKey: cc.PublicKey, Signature: cc.Signature, } - ccr, err := p.piCommentCensor(pcc) + ccr, err := p.piCommentCensor(ctx, pcc) if err != nil { return nil, err } @@ -1716,7 +1717,7 @@ func (p *politeiawww) processCommentCensor(cc pi.CommentCensor, usr user.User) ( }, nil } -func (p *politeiawww) processComments(c pi.Comments, usr *user.User) (*pi.CommentsReply, error) { +func (p *politeiawww) processComments(ctx context.Context, c pi.Comments, usr *user.User) (*pi.CommentsReply, error) { log.Tracef("processComments: %v", c.Token) // Only admins and the proposal author are allowed to retrieve @@ -1733,7 +1734,7 @@ func (p *politeiawww) processComments(c pi.Comments, usr *user.User) (*pi.Commen default: // Logged in user is not an admin. Check if they are the // proposal author. - pr, err := p.proposalRecordLatest(c.State, c.Token) + pr, err := p.proposalRecordLatest(ctx, c.State, c.Token) if err != nil { if err == errProposalNotFound { return nil, pi.UserErrorReply{ @@ -1756,7 +1757,7 @@ func (p *politeiawww) processComments(c pi.Comments, usr *user.User) (*pi.Commen } // Send plugin command - reply, err := p.commentsAll(comments.GetAll{ + reply, err := p.commentsAll(ctx, comments.GetAll{ State: convertCommentsStateFromPi(c.State), Token: c.Token, }) @@ -1791,7 +1792,7 @@ func (p *politeiawww) processComments(c pi.Comments, usr *user.User) (*pi.Commen }, nil } -func (p *politeiawww) processCommentVotes(cv pi.CommentVotes) (*pi.CommentVotesReply, error) { +func (p *politeiawww) processCommentVotes(ctx context.Context, cv pi.CommentVotes) (*pi.CommentVotesReply, error) { log.Tracef("processCommentVotes: %v %v", cv.Token, cv.UserID) // Verify state @@ -1808,7 +1809,7 @@ func (p *politeiawww) processCommentVotes(cv pi.CommentVotes) (*pi.CommentVotesR Token: cv.Token, UserID: cv.UserID, } - cvr, err := p.commentVotes(v) + cvr, err := p.commentVotes(ctx, v) if err != nil { return nil, err } @@ -1818,7 +1819,7 @@ func (p *politeiawww) processCommentVotes(cv pi.CommentVotes) (*pi.CommentVotesR }, nil } -func (p *politeiawww) processVoteAuthorize(va pi.VoteAuthorize, usr user.User) (*pi.VoteAuthorizeReply, error) { +func (p *politeiawww) processVoteAuthorize(ctx context.Context, va pi.VoteAuthorize, usr user.User) (*pi.VoteAuthorizeReply, error) { log.Tracef("processVoteAuthorize: %v", va.Token) // Verify user signed with their active identity @@ -1830,7 +1831,7 @@ func (p *politeiawww) processVoteAuthorize(va pi.VoteAuthorize, usr user.User) ( } // Send plugin command - ar, err := p.voteAuthorize(convertVoteAuthorizeFromPi(va)) + ar, err := p.voteAuthorize(ctx, convertVoteAuthorizeFromPi(va)) if err != nil { return nil, err } @@ -1841,7 +1842,7 @@ func (p *politeiawww) processVoteAuthorize(va pi.VoteAuthorize, usr user.User) ( }, nil } -func (p *politeiawww) processVoteStart(vs pi.VoteStart, usr user.User) (*pi.VoteStartReply, error) { +func (p *politeiawww) processVoteStart(ctx context.Context, vs pi.VoteStart, usr user.User) (*pi.VoteStartReply, error) { log.Tracef("processVoteStart: %v", vs.Params.Token) // Sanity check @@ -1858,7 +1859,7 @@ func (p *politeiawww) processVoteStart(vs pi.VoteStart, usr user.User) (*pi.Vote } // Call the ticketvote plugin to start vote - reply, err := p.voteStart(convertVoteStartFromPi(vs)) + reply, err := p.voteStart(ctx, convertVoteStartFromPi(vs)) if err != nil { return nil, err } @@ -1871,7 +1872,7 @@ func (p *politeiawww) processVoteStart(vs pi.VoteStart, usr user.User) (*pi.Vote }, nil } -func (p *politeiawww) processVoteStartRunoff(vsr pi.VoteStartRunoff, usr user.User) (*pi.VoteStartRunoffReply, error) { +func (p *politeiawww) processVoteStartRunoff(ctx context.Context, vsr pi.VoteStartRunoff, usr user.User) (*pi.VoteStartRunoffReply, error) { log.Tracef("processVoteStartRunoff: %v", vsr.Token) // Sanity check @@ -1908,7 +1909,7 @@ func (p *politeiawww) processVoteStartRunoff(vsr pi.VoteStartRunoff, usr user.Us Auths: convertVoteAuthsFromPi(vsr.Auths), Starts: convertVoteStartsFromPi(vsr.Starts), } - srr, err := p.voteStartRunoff(tsr) + srr, err := p.voteStartRunoff(ctx, tsr) if err != nil { return nil, err } @@ -1921,13 +1922,13 @@ func (p *politeiawww) processVoteStartRunoff(vsr pi.VoteStartRunoff, usr user.Us }, nil } -func (p *politeiawww) processVoteBallot(vb pi.VoteBallot) (*pi.VoteBallotReply, error) { +func (p *politeiawww) processVoteBallot(ctx context.Context, vb pi.VoteBallot) (*pi.VoteBallotReply, error) { log.Tracef("processVoteBallot") b := ticketvote.Ballot{ Votes: convertCastVotesFromPi(vb.Votes), } - reply, err := p.voteBallot(b) + reply, err := p.voteBallot(ctx, b) if err != nil { return nil, err } @@ -1937,10 +1938,10 @@ func (p *politeiawww) processVoteBallot(vb pi.VoteBallot) (*pi.VoteBallotReply, }, nil } -func (p *politeiawww) processVotes(v pi.Votes) (*pi.VotesReply, error) { +func (p *politeiawww) processVotes(ctx context.Context, v pi.Votes) (*pi.VotesReply, error) { log.Tracef("processVotes: %v", v.Tokens) - vd, err := p.voteDetails(v.Tokens) + vd, err := p.voteDetails(ctx, v.Tokens) if err != nil { return nil, err } @@ -1950,10 +1951,10 @@ func (p *politeiawww) processVotes(v pi.Votes) (*pi.VotesReply, error) { }, nil } -func (p *politeiawww) processVoteResults(vr pi.VoteResults) (*pi.VoteResultsReply, error) { +func (p *politeiawww) processVoteResults(ctx context.Context, vr pi.VoteResults) (*pi.VoteResultsReply, error) { log.Tracef("processVoteResults: %v", vr.Token) - cvr, err := p.castVotes(vr.Token) + cvr, err := p.castVotes(ctx, vr.Token) if err != nil { return nil, err } @@ -1963,10 +1964,10 @@ func (p *politeiawww) processVoteResults(vr pi.VoteResults) (*pi.VoteResultsRepl }, nil } -func (p *politeiawww) processVoteSummaries(vs pi.VoteSummaries) (*pi.VoteSummariesReply, error) { +func (p *politeiawww) processVoteSummaries(ctx context.Context, vs pi.VoteSummaries) (*pi.VoteSummariesReply, error) { log.Tracef("processVoteSummaries: %v", vs.Tokens) - r, err := p.voteSummaries(vs.Tokens) + r, err := p.voteSummaries(ctx, vs.Tokens) if err != nil { return nil, err } @@ -1977,10 +1978,10 @@ func (p *politeiawww) processVoteSummaries(vs pi.VoteSummaries) (*pi.VoteSummari }, nil } -func (p *politeiawww) processVoteInventory() (*pi.VoteInventoryReply, error) { +func (p *politeiawww) processVoteInventory(ctx context.Context) (*pi.VoteInventoryReply, error) { log.Tracef("processVoteInventory") - r, err := p.piVoteInventory() + r, err := p.piVoteInventory(ctx) if err != nil { return nil, err } @@ -2015,7 +2016,7 @@ func (p *politeiawww) handleProposalNew(w http.ResponseWriter, r *http.Request) return } - pnr, err := p.processProposalNew(pn, *user) + pnr, err := p.processProposalNew(r.Context(), pn, *user) if err != nil { respondWithPiError(w, r, "handleProposalNew: processProposalNew: %v", err) @@ -2045,7 +2046,7 @@ func (p *politeiawww) handleProposalEdit(w http.ResponseWriter, r *http.Request) return } - per, err := p.processProposalEdit(pe, *user) + per, err := p.processProposalEdit(r.Context(), pe, *user) if err != nil { respondWithPiError(w, r, "handleProposalEdit: processProposalEdit: %v", err) @@ -2075,7 +2076,7 @@ func (p *politeiawww) handleProposalStatusSet(w http.ResponseWriter, r *http.Req return } - pssr, err := p.processProposalStatusSet(pss, *usr) + pssr, err := p.processProposalStatusSet(r.Context(), pss, *usr) if err != nil { respondWithPiError(w, r, "handleProposalStatusSet: processProposalStatusSet: %v", err) @@ -2108,7 +2109,7 @@ func (p *politeiawww) handleProposals(w http.ResponseWriter, r *http.Request) { } isAdmin := usr != nil && usr.Admin - ppi, err := p.processProposals(ps, isAdmin) + ppi, err := p.processProposals(r.Context(), ps, isAdmin) if err != nil { respondWithPiError(w, r, "handleProposals: processProposals: %v", err) @@ -2131,7 +2132,7 @@ func (p *politeiawww) handleProposalInventory(w http.ResponseWriter, r *http.Req } isAdmin := usr != nil && usr.Admin - ppi, err := p.processProposalInventory(isAdmin) + ppi, err := p.processProposalInventory(r.Context(), isAdmin) if err != nil { respondWithPiError(w, r, "handleProposalInventory: processProposalInventory: %v", err) @@ -2161,7 +2162,7 @@ func (p *politeiawww) handleCommentNew(w http.ResponseWriter, r *http.Request) { return } - cnr, err := p.processCommentNew(cn, *usr) + cnr, err := p.processCommentNew(r.Context(), cn, *usr) if err != nil { respondWithPiError(w, r, "handleCommentNew: processCommentNew: %v", err) @@ -2191,7 +2192,7 @@ func (p *politeiawww) handleCommentVote(w http.ResponseWriter, r *http.Request) return } - vcr, err := p.processCommentVote(cv, *usr) + vcr, err := p.processCommentVote(r.Context(), cv, *usr) if err != nil { respondWithPiError(w, r, "handleCommentVote: processCommentVote: %v", err) @@ -2221,7 +2222,7 @@ func (p *politeiawww) handleCommentCensor(w http.ResponseWriter, r *http.Request return } - ccr, err := p.processCommentCensor(cc, *usr) + ccr, err := p.processCommentCensor(r.Context(), cc, *usr) if err != nil { respondWithPiError(w, r, "handleCommentCensor: processCommentCensor: %v", err) @@ -2253,7 +2254,7 @@ func (p *politeiawww) handleComments(w http.ResponseWriter, r *http.Request) { return } - cr, err := p.processComments(c, usr) + cr, err := p.processComments(r.Context(), c, usr) if err != nil { respondWithPiError(w, r, "handleCommentVote: processComments: %v", err) @@ -2276,7 +2277,7 @@ func (p *politeiawww) handleCommentVotes(w http.ResponseWriter, r *http.Request) return } - cvr, err := p.processCommentVotes(cv) + cvr, err := p.processCommentVotes(r.Context(), cv) if err != nil { respondWithPiError(w, r, "handleCommentVotes: processCommentVotes: %v", err) @@ -2306,7 +2307,7 @@ func (p *politeiawww) handleVoteAuthorize(w http.ResponseWriter, r *http.Request return } - vr, err := p.processVoteAuthorize(va, *usr) + vr, err := p.processVoteAuthorize(r.Context(), va, *usr) if err != nil { respondWithPiError(w, r, "handleVoteAuthorize: processVoteAuthorize: %v", err) @@ -2336,7 +2337,7 @@ func (p *politeiawww) handleVoteStart(w http.ResponseWriter, r *http.Request) { return } - vsr, err := p.processVoteStart(vs, *usr) + vsr, err := p.processVoteStart(r.Context(), vs, *usr) if err != nil { respondWithPiError(w, r, "handleVoteStart: processVoteStart: %v", err) @@ -2366,7 +2367,7 @@ func (p *politeiawww) handleVoteStartRunoff(w http.ResponseWriter, r *http.Reque return } - vsrr, err := p.processVoteStartRunoff(vsr, *usr) + vsrr, err := p.processVoteStartRunoff(r.Context(), vsr, *usr) if err != nil { respondWithPiError(w, r, "handleVoteStartRunoff: processVoteStartRunoff: %v", err) @@ -2389,7 +2390,7 @@ func (p *politeiawww) handleVoteBallot(w http.ResponseWriter, r *http.Request) { return } - vbr, err := p.processVoteBallot(vb) + vbr, err := p.processVoteBallot(r.Context(), vb) if err != nil { respondWithPiError(w, r, "handleVoteBallot: processVoteBallot: %v", err) @@ -2412,7 +2413,7 @@ func (p *politeiawww) handleVotes(w http.ResponseWriter, r *http.Request) { return } - vr, err := p.processVotes(v) + vr, err := p.processVotes(r.Context(), v) if err != nil { respondWithPiError(w, r, "handleVotes: processVotes: %v", err) @@ -2435,7 +2436,7 @@ func (p *politeiawww) handleVoteResults(w http.ResponseWriter, r *http.Request) return } - vrr, err := p.processVoteResults(vr) + vrr, err := p.processVoteResults(r.Context(), vr) if err != nil { respondWithPiError(w, r, "handleVoteResults: prcoessVoteResults: %v", err) @@ -2458,7 +2459,7 @@ func (p *politeiawww) handleVoteSummaries(w http.ResponseWriter, r *http.Request return } - vsr, err := p.processVoteSummaries(vs) + vsr, err := p.processVoteSummaries(r.Context(), vs) if err != nil { respondWithPiError(w, r, "handleVoteSummaries: processVoteSummaries: %v", err) @@ -2481,7 +2482,7 @@ func (p *politeiawww) handleVoteInventory(w http.ResponseWriter, r *http.Request return } - vir, err := p.processVoteInventory() + vir, err := p.processVoteInventory(r.Context()) if err != nil { respondWithPiError(w, r, "handleVoteInventory: processVoteInventory: %v", err) diff --git a/politeiawww/plugin.go b/politeiawww/plugin.go index 495d70d9b..bb4ce2208 100644 --- a/politeiawww/plugin.go +++ b/politeiawww/plugin.go @@ -62,7 +62,7 @@ func (p *politeiawww) getPluginInventory() ([]plugin, error) { return nil, fmt.Errorf("max retries exceeded") } - pi, err := p.pluginInventory() + pi, err := p.pluginInventory(ctx) if err != nil { log.Infof("cannot get politeiad plugin inventory: %v: retry in %v", err, sleepInterval) diff --git a/politeiawww/politeiad.go b/politeiawww/politeiad.go index ed8e43d8c..288f5b806 100644 --- a/politeiawww/politeiad.go +++ b/politeiawww/politeiad.go @@ -86,7 +86,7 @@ func (p *politeiawww) makeRequest(ctx context.Context, method string, route stri // newRecord creates a record in politeiad. This route returns the censorship // record from the new created record. -func (p *politeiawww) newRecord(metadata []pd.MetadataStream, files []pd.File) (*pd.CensorshipRecord, error) { +func (p *politeiawww) newRecord(ctx context.Context, metadata []pd.MetadataStream, files []pd.File) (*pd.CensorshipRecord, error) { // Setup request challenge, err := util.Random(pd.ChallengeSize) if err != nil { @@ -99,8 +99,6 @@ func (p *politeiawww) newRecord(metadata []pd.MetadataStream, files []pd.File) ( } // Send request - // TODO FIXME - ctx := context.Background() resBody, err := p.makeRequest(ctx, http.MethodPost, pd.NewRecordRoute, nr) if err != nil { return nil, err @@ -124,7 +122,7 @@ func (p *politeiawww) newRecord(metadata []pd.MetadataStream, files []pd.File) ( // updateRecord updates a record in politeiad. This can be used to update // unvetted or vetted records depending on the route that is provided. -func (p *politeiawww) updateRecord(route, token string, mdAppend, mdOverwrite []pd.MetadataStream, filesAdd []pd.File, filesDel []string) (*pd.Record, error) { +func (p *politeiawww) updateRecord(ctx context.Context, route, token string, mdAppend, mdOverwrite []pd.MetadataStream, filesAdd []pd.File, filesDel []string) (*pd.Record, error) { // Setup request challenge, err := util.Random(pd.ChallengeSize) if err != nil { @@ -139,8 +137,6 @@ func (p *politeiawww) updateRecord(route, token string, mdAppend, mdOverwrite [] } // Send request - // TODO FIXME - ctx := context.Background() resBody, err := p.makeRequest(ctx, http.MethodPost, route, ur) if err != nil { return nil, err @@ -163,19 +159,19 @@ func (p *politeiawww) updateRecord(route, token string, mdAppend, mdOverwrite [] } // updateUnvetted updates an unvetted record in politeiad. -func (p *politeiawww) updateUnvetted(token string, mdAppend, mdOverwrite []pd.MetadataStream, filesAdd []pd.File, filesDel []string) (*pd.Record, error) { - return p.updateRecord(pd.UpdateUnvettedRoute, token, +func (p *politeiawww) updateUnvetted(ctx context.Context, token string, mdAppend, mdOverwrite []pd.MetadataStream, filesAdd []pd.File, filesDel []string) (*pd.Record, error) { + return p.updateRecord(ctx, pd.UpdateUnvettedRoute, token, mdAppend, mdOverwrite, filesAdd, filesDel) } // updateVetted updates a vetted record in politeiad. -func (p *politeiawww) updateVetted(token string, mdAppend, mdOverwrite []pd.MetadataStream, filesAdd []pd.File, filesDel []string) (*pd.Record, error) { - return p.updateRecord(pd.UpdateVettedRoute, token, +func (p *politeiawww) updateVetted(ctx context.Context, token string, mdAppend, mdOverwrite []pd.MetadataStream, filesAdd []pd.File, filesDel []string) (*pd.Record, error) { + return p.updateRecord(ctx, pd.UpdateVettedRoute, token, mdAppend, mdOverwrite, filesAdd, filesDel) } // updateUnvettedMetadata updates the metadata of a unvetted record in politeiad. -func (p *politeiawww) updateUnvettedMetadata(token string, mdAppend, mdOverwrite []pd.MetadataStream) error { +func (p *politeiawww) updateUnvettedMetadata(ctx context.Context, token string, mdAppend, mdOverwrite []pd.MetadataStream) error { // Setup request challenge, err := util.Random(pd.ChallengeSize) if err != nil { @@ -189,8 +185,6 @@ func (p *politeiawww) updateUnvettedMetadata(token string, mdAppend, mdOverwrite } // Send request - // TODO FIXME - ctx := context.Background() resBody, err := p.makeRequest(ctx, http.MethodPost, pd.UpdateUnvettedMetadataRoute, uum) if err != nil { @@ -214,7 +208,7 @@ func (p *politeiawww) updateUnvettedMetadata(token string, mdAppend, mdOverwrite } // updateVettedMetadata updates the metadata of a vetted record in politeiad. -func (p *politeiawww) updateVettedMetadata(token string, mdAppend, mdOverwrite []pd.MetadataStream) error { +func (p *politeiawww) updateVettedMetadata(ctx context.Context, token string, mdAppend, mdOverwrite []pd.MetadataStream) error { // Setup request challenge, err := util.Random(pd.ChallengeSize) if err != nil { @@ -228,8 +222,6 @@ func (p *politeiawww) updateVettedMetadata(token string, mdAppend, mdOverwrite [ } // Send request - // TODO FIXME - ctx := context.Background() resBody, err := p.makeRequest(ctx, http.MethodPost, pd.UpdateVettedMetadataRoute, uvm) if err != nil { @@ -253,7 +245,7 @@ func (p *politeiawww) updateVettedMetadata(token string, mdAppend, mdOverwrite [ } // setUnvettedStatus sets the status of a unvetted record in politeiad. -func (p *politeiawww) setUnvettedStatus(token string, status pd.RecordStatusT, mdAppend, mdOverwrite []pd.MetadataStream) (*pd.Record, error) { +func (p *politeiawww) setUnvettedStatus(ctx context.Context, token string, status pd.RecordStatusT, mdAppend, mdOverwrite []pd.MetadataStream) (*pd.Record, error) { // Setup request challenge, err := util.Random(pd.ChallengeSize) if err != nil { @@ -268,8 +260,6 @@ func (p *politeiawww) setUnvettedStatus(token string, status pd.RecordStatusT, m } // Send request - // TODO FIXME - ctx := context.Background() resBody, err := p.makeRequest(ctx, http.MethodPost, pd.SetUnvettedStatusRoute, sus) if err != nil { @@ -293,7 +283,7 @@ func (p *politeiawww) setUnvettedStatus(token string, status pd.RecordStatusT, m } // setVettedStatus sets the status of a vetted record in politeiad. -func (p *politeiawww) setVettedStatus(token string, status pd.RecordStatusT, mdAppend, mdOverwrite []pd.MetadataStream) (*pd.Record, error) { +func (p *politeiawww) setVettedStatus(ctx context.Context, token string, status pd.RecordStatusT, mdAppend, mdOverwrite []pd.MetadataStream) (*pd.Record, error) { // Setup request challenge, err := util.Random(pd.ChallengeSize) if err != nil { @@ -308,8 +298,6 @@ func (p *politeiawww) setVettedStatus(token string, status pd.RecordStatusT, mdA } // Send request - // TODO FIXME - ctx := context.Background() resBody, err := p.makeRequest(ctx, http.MethodPost, pd.SetVettedStatusRoute, svs) if err != nil { @@ -333,7 +321,7 @@ func (p *politeiawww) setVettedStatus(token string, status pd.RecordStatusT, mdA } // getUnvetted retrieves an unvetted record from politeiad. -func (p *politeiawww) getUnvetted(token, version string) (*pd.Record, error) { +func (p *politeiawww) getUnvetted(ctx context.Context, token, version string) (*pd.Record, error) { // Setup request challenge, err := util.Random(pd.ChallengeSize) if err != nil { @@ -346,8 +334,6 @@ func (p *politeiawww) getUnvetted(token, version string) (*pd.Record, error) { } // Send request - // TODO FIXME - ctx := context.Background() resBody, err := p.makeRequest(ctx, http.MethodPost, pd.GetUnvettedRoute, gu) if err != nil { return nil, err @@ -371,12 +357,12 @@ func (p *politeiawww) getUnvetted(token, version string) (*pd.Record, error) { // getUnvettedLatest returns the latest version of the unvetted record for the // provided token. -func (p *politeiawww) getUnvettedLatest(token string) (*pd.Record, error) { - return p.getUnvetted(token, "") +func (p *politeiawww) getUnvettedLatest(ctx context.Context, token string) (*pd.Record, error) { + return p.getUnvetted(ctx, token, "") } // getVetted retrieves a vetted record from politeiad. -func (p *politeiawww) getVetted(token, version string) (*pd.Record, error) { +func (p *politeiawww) getVetted(ctx context.Context, token, version string) (*pd.Record, error) { // Setup request challenge, err := util.Random(pd.ChallengeSize) if err != nil { @@ -389,8 +375,6 @@ func (p *politeiawww) getVetted(token, version string) (*pd.Record, error) { } // Send request - // TODO FIXME - ctx := context.Background() resBody, err := p.makeRequest(ctx, http.MethodPost, pd.GetVettedRoute, gu) if err != nil { @@ -415,13 +399,13 @@ func (p *politeiawww) getVetted(token, version string) (*pd.Record, error) { // getVettedLatest returns the latest version of the vetted record for the // provided token. -func (p *politeiawww) getVettedLatest(token string) (*pd.Record, error) { - return p.getVetted(token, "") +func (p *politeiawww) getVettedLatest(ctx context.Context, token string) (*pd.Record, error) { + return p.getVetted(ctx, token, "") } // pluginInventory requests the plugin inventory from politeiad and returns // inventoryByStatus retrieves the censorship record tokens filtered by status. -func (p *politeiawww) inventoryByStatus() (*pd.InventoryByStatusReply, error) { +func (p *politeiawww) inventoryByStatus(ctx context.Context) (*pd.InventoryByStatusReply, error) { // Setup request challenge, err := util.Random(pd.ChallengeSize) if err != nil { @@ -432,8 +416,6 @@ func (p *politeiawww) inventoryByStatus() (*pd.InventoryByStatusReply, error) { } // Send request - // TODO FIXME - ctx := context.Background() resBody, err := p.makeRequest(ctx, http.MethodPost, pd.InventoryByStatusRoute, ibs) if err != nil { @@ -458,7 +440,7 @@ func (p *politeiawww) inventoryByStatus() (*pd.InventoryByStatusReply, error) { // pluginInventory requests the plugin inventory from politeiad and returns // the available plugins slice. -func (p *politeiawww) pluginInventory() ([]pd.Plugin, error) { +func (p *politeiawww) pluginInventory(ctx context.Context) ([]pd.Plugin, error) { // Setup request challenge, err := util.Random(pd.ChallengeSize) if err != nil { @@ -469,8 +451,6 @@ func (p *politeiawww) pluginInventory() ([]pd.Plugin, error) { } // Send request - // TODO FIXME - ctx := context.Background() resBody, err := p.makeRequest(ctx, http.MethodPost, pd.PluginInventoryRoute, pi) if err != nil { @@ -495,7 +475,7 @@ func (p *politeiawww) pluginInventory() ([]pd.Plugin, error) { // pluginCommand fires a plugin command on politeiad and returns the reply // payload. -func (p *politeiawww) pluginCommand(pluginID, cmd, payload string) (string, error) { +func (p *politeiawww) pluginCommand(ctx context.Context, pluginID, cmd, payload string) (string, error) { // Setup request challenge, err := util.Random(pd.ChallengeSize) if err != nil { @@ -510,8 +490,6 @@ func (p *politeiawww) pluginCommand(pluginID, cmd, payload string) (string, erro } // Send request - // TODO FIXME - ctx := context.Background() resBody, err := p.makeRequest(ctx, http.MethodPost, pd.PluginCommandRoute, pc) if err != nil { diff --git a/politeiawww/politeiawww.go b/politeiawww/politeiawww.go index 347557152..9100c49f7 100644 --- a/politeiawww/politeiawww.go +++ b/politeiawww/politeiawww.go @@ -198,7 +198,7 @@ func (p *politeiawww) handleTokenInventory(w http.ResponseWriter, r *http.Reques } isAdmin := user != nil && user.Admin - reply, err := p.processTokenInventory(isAdmin) + reply, err := p.processTokenInventory(r.Context(), isAdmin) if err != nil { RespondWithError(w, r, 0, "handleTokenInventory: processTokenInventory: %v", err) @@ -235,7 +235,7 @@ func (p *politeiawww) handleProposalDetails(w http.ResponseWriter, r *http.Reque return } - reply, err := p.processProposalDetails(pd, user) + reply, err := p.processProposalDetails(r.Context(), pd, user) if err != nil { RespondWithError(w, r, 0, "handleProposalDetails: processProposalDetails %v", err) @@ -269,7 +269,7 @@ func (p *politeiawww) handleBatchProposals(w http.ResponseWriter, r *http.Reques return } - reply, err := p.processBatchProposals(bp, user) + reply, err := p.processBatchProposals(r.Context(), bp, user) if err != nil { RespondWithError(w, r, 0, "handleBatchProposals: processBatchProposals %v", err) @@ -292,7 +292,7 @@ func (p *politeiawww) handleCastVotes(w http.ResponseWriter, r *http.Request) { return } - avr, err := p.processCastVotes(&cv) + avr, err := p.processCastVotes(r.Context(), &cv) if err != nil { RespondWithError(w, r, 0, "handleCastVotes: processCastVotes %v", err) @@ -309,7 +309,7 @@ func (p *politeiawww) handleVoteResultsWWW(w http.ResponseWriter, r *http.Reques pathParams := mux.Vars(r) token := pathParams["token"] - vrr, err := p.processVoteResultsWWW(token) + vrr, err := p.processVoteResultsWWW(r.Context(), token) if err != nil { RespondWithError(w, r, 0, "handleVoteResultsWWW: processVoteResultsWWW %v", @@ -335,7 +335,7 @@ func (p *politeiawww) handleBatchVoteSummary(w http.ResponseWriter, r *http.Requ return } - reply, err := p.processBatchVoteSummary(bvs) + reply, err := p.processBatchVoteSummary(r.Context(), bvs) if err != nil { RespondWithError(w, r, 0, "handleBatchVoteSummary: processBatchVoteSummary %v", err) diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index dc8472aaf..5c033c8f2 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -5,6 +5,7 @@ package main import ( + "context" "encoding/base64" "strconv" @@ -180,10 +181,10 @@ func convertVoteErrorCodeToWWW(errcode ticketvote.VoteErrorT) decredplugin.Error } } -func (p *politeiawww) processProposalDetails(pd www.ProposalsDetails, u *user.User) (*www.ProposalDetailsReply, error) { +func (p *politeiawww) processProposalDetails(ctx context.Context, pd www.ProposalsDetails, u *user.User) (*www.ProposalDetailsReply, error) { log.Tracef("processProposalDetails: %v", pd.Token) - pr, err := p.proposalRecord(pi.PropStateVetted, pd.Token, pd.Version) + pr, err := p.proposalRecord(ctx, pi.PropStateVetted, pd.Token, pd.Version) if err != nil { return nil, err } @@ -197,7 +198,7 @@ func (p *politeiawww) processProposalDetails(pd www.ProposalsDetails, u *user.Us }, nil } -func (p *politeiawww) processBatchProposals(bp www.BatchProposals, u *user.User) (*www.BatchProposalsReply, error) { +func (p *politeiawww) processBatchProposals(ctx context.Context, bp www.BatchProposals, u *user.User) (*www.BatchProposalsReply, error) { log.Tracef("processBatchProposals: %v", bp.Tokens) // Setup requests @@ -209,7 +210,7 @@ func (p *politeiawww) processBatchProposals(bp www.BatchProposals, u *user.User) } // Get proposals - props, err := p.proposalRecords(pi.PropStateVetted, prs, false) + props, err := p.proposalRecords(ctx, pi.PropStateVetted, prs, false) if err != nil { return nil, err } @@ -229,11 +230,11 @@ func (p *politeiawww) processBatchProposals(bp www.BatchProposals, u *user.User) }, nil } -func (p *politeiawww) processVoteResultsWWW(token string) (*www.VoteResultsReply, error) { +func (p *politeiawww) processVoteResultsWWW(ctx context.Context, token string) (*www.VoteResultsReply, error) { log.Tracef("processVoteResultsWWW: %v", token) // Get vote details - vd, err := p.voteDetails([]string{token}) + vd, err := p.voteDetails(ctx, []string{token}) if err != nil { return nil, err } @@ -257,7 +258,7 @@ func (p *politeiawww) processVoteResultsWWW(token string) (*www.VoteResultsReply } // Get cast votes - cv, err := p.castVotes(token) + cv, err := p.castVotes(ctx, token) if err != nil { return nil, err } @@ -296,11 +297,11 @@ func (p *politeiawww) processVoteResultsWWW(token string) (*www.VoteResultsReply }, nil } -func (p *politeiawww) processBatchVoteSummary(bvs www.BatchVoteSummary) (*www.BatchVoteSummaryReply, error) { +func (p *politeiawww) processBatchVoteSummary(ctx context.Context, bvs www.BatchVoteSummary) (*www.BatchVoteSummaryReply, error) { log.Tracef("processBatchVoteSummary: %v", bvs.Tokens) // Get vote summaries - sm, err := p.voteSummaries(bvs.Tokens) + sm, err := p.voteSummaries(ctx, bvs.Tokens) if err != nil { return nil, err } @@ -338,7 +339,7 @@ func (p *politeiawww) processBatchVoteSummary(bvs www.BatchVoteSummary) (*www.Ba }, nil } -func (p *politeiawww) processCastVotes(ballot *www.Ballot) (*www.BallotReply, error) { +func (p *politeiawww) processCastVotes(ctx context.Context, ballot *www.Ballot) (*www.BallotReply, error) { log.Tracef("processCastVotes") // Prepare plugin command @@ -360,7 +361,7 @@ func (p *politeiawww) processCastVotes(ballot *www.Ballot) (*www.BallotReply, er } // Send plugin command - r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdBallot, + r, err := p.pluginCommand(ctx, ticketvote.ID, ticketvote.CmdBallot, string(payload)) if err != nil { return nil, err @@ -386,17 +387,17 @@ func (p *politeiawww) processCastVotes(ballot *www.Ballot) (*www.BallotReply, er }, nil } -func (p *politeiawww) processTokenInventory(isAdmin bool) (*www.TokenInventoryReply, error) { +func (p *politeiawww) processTokenInventory(ctx context.Context, isAdmin bool) (*www.TokenInventoryReply, error) { log.Tracef("processTokenInventory") // Get record inventory - ri, err := p.inventoryByStatus() + ri, err := p.inventoryByStatus(ctx) if err != nil { return nil, err } // Get vote inventory - vi, err := p.piVoteInventory() + vi, err := p.piVoteInventory(ctx) if err != nil { return nil, err } diff --git a/politeiawww/ticketvote.go b/politeiawww/ticketvote.go index e9dd3a1b0..7c0bd17e5 100644 --- a/politeiawww/ticketvote.go +++ b/politeiawww/ticketvote.go @@ -5,16 +5,18 @@ package main import ( + "context" + ticketvote "github.com/decred/politeia/politeiad/plugins/ticketvote" ) // voteAuthorize uses the ticketvote plugin to authorize a vote. -func (p *politeiawww) voteAuthorize(a ticketvote.Authorize) (*ticketvote.AuthorizeReply, error) { +func (p *politeiawww) voteAuthorize(ctx context.Context, a ticketvote.Authorize) (*ticketvote.AuthorizeReply, error) { b, err := ticketvote.EncodeAuthorize(a) if err != nil { return nil, err } - r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdAuthorize, string(b)) + r, err := p.pluginCommand(ctx, ticketvote.ID, ticketvote.CmdAuthorize, string(b)) if err != nil { return nil, err } @@ -26,12 +28,12 @@ func (p *politeiawww) voteAuthorize(a ticketvote.Authorize) (*ticketvote.Authori } // voteStart uses the ticketvote plugin to start a vote. -func (p *politeiawww) voteStart(s ticketvote.Start) (*ticketvote.StartReply, error) { +func (p *politeiawww) voteStart(ctx context.Context, s ticketvote.Start) (*ticketvote.StartReply, error) { b, err := ticketvote.EncodeStart(s) if err != nil { return nil, err } - r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdStart, string(b)) + r, err := p.pluginCommand(ctx, ticketvote.ID, ticketvote.CmdStart, string(b)) if err != nil { return nil, err } @@ -43,12 +45,12 @@ func (p *politeiawww) voteStart(s ticketvote.Start) (*ticketvote.StartReply, err } // voteStartRunoff uses the ticketvote plugin to start a runoff vote. -func (p *politeiawww) voteStartRunoff(sr ticketvote.StartRunoff) (*ticketvote.StartRunoffReply, error) { +func (p *politeiawww) voteStartRunoff(ctx context.Context, sr ticketvote.StartRunoff) (*ticketvote.StartRunoffReply, error) { b, err := ticketvote.EncodeStartRunoff(sr) if err != nil { return nil, err } - r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdStartRunoff, + r, err := p.pluginCommand(ctx, ticketvote.ID, ticketvote.CmdStartRunoff, string(b)) if err != nil { return nil, err @@ -61,12 +63,12 @@ func (p *politeiawww) voteStartRunoff(sr ticketvote.StartRunoff) (*ticketvote.St } // voteBallot uses the ticketvote plugin to cast a ballot of votes. -func (p *politeiawww) voteBallot(tb ticketvote.Ballot) (*ticketvote.BallotReply, error) { +func (p *politeiawww) voteBallot(ctx context.Context, tb ticketvote.Ballot) (*ticketvote.BallotReply, error) { b, err := ticketvote.EncodeBallot(tb) if err != nil { return nil, err } - r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdBallot, string(b)) + r, err := p.pluginCommand(ctx, ticketvote.ID, ticketvote.CmdBallot, string(b)) if err != nil { return nil, err } @@ -78,7 +80,7 @@ func (p *politeiawww) voteBallot(tb ticketvote.Ballot) (*ticketvote.BallotReply, } // voteDetails uses the ticketvote plugin to fetch the details of a vote. -func (p *politeiawww) voteDetails(tokens []string) (*ticketvote.DetailsReply, error) { +func (p *politeiawww) voteDetails(ctx context.Context, tokens []string) (*ticketvote.DetailsReply, error) { d := ticketvote.Details{ Tokens: tokens, } @@ -86,7 +88,7 @@ func (p *politeiawww) voteDetails(tokens []string) (*ticketvote.DetailsReply, er if err != nil { return nil, err } - r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdDetails, string(b)) + r, err := p.pluginCommand(ctx, ticketvote.ID, ticketvote.CmdDetails, string(b)) if err != nil { return nil, err } @@ -98,7 +100,7 @@ func (p *politeiawww) voteDetails(tokens []string) (*ticketvote.DetailsReply, er } // castVotes uses the ticketvote plugin to fetch cast votes for a record. -func (p *politeiawww) castVotes(token string) (*ticketvote.CastVotesReply, error) { +func (p *politeiawww) castVotes(ctx context.Context, token string) (*ticketvote.CastVotesReply, error) { cv := ticketvote.CastVotes{ Token: token, } @@ -106,7 +108,7 @@ func (p *politeiawww) castVotes(token string) (*ticketvote.CastVotesReply, error if err != nil { return nil, err } - r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdCastVotes, string(b)) + r, err := p.pluginCommand(ctx, ticketvote.ID, ticketvote.CmdCastVotes, string(b)) if err != nil { return nil, err } @@ -118,7 +120,7 @@ func (p *politeiawww) castVotes(token string) (*ticketvote.CastVotesReply, error } // voteSummaries uses the ticketvote plugin to fetch vote summaries. -func (p *politeiawww) voteSummaries(tokens []string) (*ticketvote.SummariesReply, error) { +func (p *politeiawww) voteSummaries(ctx context.Context, tokens []string) (*ticketvote.SummariesReply, error) { s := ticketvote.Summaries{ Tokens: tokens, } @@ -126,7 +128,7 @@ func (p *politeiawww) voteSummaries(tokens []string) (*ticketvote.SummariesReply if err != nil { return nil, err } - r, err := p.pluginCommand(ticketvote.ID, ticketvote.CmdSummaries, string(b)) + r, err := p.pluginCommand(ctx, ticketvote.ID, ticketvote.CmdSummaries, string(b)) if err != nil { return nil, err } From 26064113e38c3a8a0a06a0a1a99630c62d2f4793 Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Mon, 12 Oct 2020 16:58:44 -0300 Subject: [PATCH 149/449] multi: check errors with std error pkg. --- politeiad/backend/gitbe/decred.go | 4 +-- politeiad/backend/tlogbe/anchor.go | 2 +- politeiad/backend/tlogbe/comments.go | 14 ++++---- politeiad/backend/tlogbe/dcrtime.go | 3 +- politeiad/backend/tlogbe/pi.go | 6 ++-- .../tlogbe/store/filesystem/filesystem.go | 5 +-- politeiad/backend/tlogbe/ticketvote.go | 8 ++--- politeiad/backend/tlogbe/tlog.go | 2 +- politeiad/backend/tlogbe/tlogbe.go | 36 +++++++++---------- politeiad/plugins/pi/pi.go | 3 +- politeiad/politeiad.go | 11 +++--- politeiawww/invoices.go | 2 +- politeiawww/piwww.go | 6 ++-- politeiawww/proposals.go | 22 ++++++------ politeiawww/www.go | 2 +- 15 files changed, 66 insertions(+), 60 deletions(-) diff --git a/politeiad/backend/gitbe/decred.go b/politeiad/backend/gitbe/decred.go index 599623c0d..44dd4243e 100644 --- a/politeiad/backend/gitbe/decred.go +++ b/politeiad/backend/gitbe/decred.go @@ -1576,7 +1576,7 @@ func (g *gitBackEnd) pluginStartVote(payload string) (string, error) { // TODO pm, err := g.vettedProposalMetadata(token) switch { - case err == errProposalMetadataNotFound: + case errors.Is(err, errProposalMetadataNotFound): // Proposal is not an RFP submission. This is ok. case err != nil: // All other errors @@ -1805,7 +1805,7 @@ func (g *gitBackEnd) pluginStartVoteRunoff(payload string) (string, error) { // Verify this proposal is indeed an RFP pm, err := g.vettedProposalMetadata(sv.Token) switch { - case err == errProposalMetadataNotFound: + case errors.Is(err, errProposalMetadataNotFound): // No ProposalMetadata. This is not an RFP. return "", fmt.Errorf("proposal is not an rfp: %v", sv.Token) case err != nil: diff --git a/politeiad/backend/tlogbe/anchor.go b/politeiad/backend/tlogbe/anchor.go index f850dce92..264f4bbbc 100644 --- a/politeiad/backend/tlogbe/anchor.go +++ b/politeiad/backend/tlogbe/anchor.go @@ -335,7 +335,7 @@ func (t *tlog) anchor() { // Get latest anchor a, err := t.anchorLatest(v.TreeId) switch { - case err == errAnchorNotFound: + case errors.Is(err, errAnchorNotFound): // Tree has not been anchored yet. Verify that the tree has // leaves. A tree with no leaves does not need to be anchored. leavesAll, err := t.trillian.leavesAll(v.TreeId) diff --git a/politeiad/backend/tlogbe/comments.go b/politeiad/backend/tlogbe/comments.go index 0749e9dca..700928360 100644 --- a/politeiad/backend/tlogbe/comments.go +++ b/politeiad/backend/tlogbe/comments.go @@ -721,7 +721,7 @@ func (p *commentsPlugin) comments(s comments.StateT, token []byte, idx commentsI // Get comment add records adds, err := p.commentAdds(s, token, merkleAdds) if err != nil { - if err == errRecordNotFound { + if errors.Is(err, errRecordNotFound) { return nil, err } return nil, fmt.Errorf("commentAdds: %v", err) @@ -847,7 +847,7 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { // Save comment merkleHash, err := p.commentAddSave(ca) if err != nil { - if err == errRecordNotFound { + if errors.Is(err, errRecordNotFound) { return "", backend.PluginUserError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorStatusRecordNotFound), @@ -1005,7 +1005,7 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { // Save comment merkle, err := p.commentAddSave(ca) if err != nil { - if err == errRecordNotFound { + if errors.Is(err, errRecordNotFound) { return "", backend.PluginUserError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorStatusRecordNotFound), @@ -1120,7 +1120,7 @@ func (p *commentsPlugin) cmdDel(payload string) (string, error) { // Save comment del merkle, err := p.commentDelSave(cd) if err != nil { - if err == errRecordNotFound { + if errors.Is(err, errRecordNotFound) { return "", backend.PluginUserError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorStatusRecordNotFound), @@ -1336,7 +1336,7 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { // Save comment vote merkle, err := p.commentVoteSave(cv) if err != nil { - if err == errRecordNotFound { + if errors.Is(err, errRecordNotFound) { return "", backend.PluginUserError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorStatusRecordNotFound), @@ -1421,7 +1421,7 @@ func (p *commentsPlugin) cmdGet(payload string) (string, error) { // Get comments cs, err := p.comments(g.State, token, *idx, g.CommentIDs) if err != nil { - if err == errRecordNotFound { + if errors.Is(err, errRecordNotFound) { return "", backend.PluginUserError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorStatusRecordNotFound), @@ -1486,7 +1486,7 @@ func (p *commentsPlugin) cmdGetAll(payload string) (string, error) { // Get comments c, err := p.comments(ga.State, token, *idx, commentIDs) if err != nil { - if err == errRecordNotFound { + if errors.Is(err, errRecordNotFound) { return "", backend.PluginUserError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorStatusRecordNotFound), diff --git a/politeiad/backend/tlogbe/dcrtime.go b/politeiad/backend/tlogbe/dcrtime.go index b13b902ea..f4ad66b7d 100644 --- a/politeiad/backend/tlogbe/dcrtime.go +++ b/politeiad/backend/tlogbe/dcrtime.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "encoding/hex" "encoding/json" + "errors" "fmt" "net/http" "time" @@ -145,7 +146,7 @@ func verifyBatch(host, id string, digests []string) (*dcrtime.VerifyBatchReply, // Verify merkle path root, err := merkle.VerifyAuthPath(&v.ChainInformation.MerklePath) if err != nil { - if err == merkle.ErrEmpty { + if errors.Is(err, merkle.ErrEmpty) { // A dcr transaction has not been sent yet so there is // nothing to verify. continue diff --git a/politeiad/backend/tlogbe/pi.go b/politeiad/backend/tlogbe/pi.go index ae0d4f4d7..13e798673 100644 --- a/politeiad/backend/tlogbe/pi.go +++ b/politeiad/backend/tlogbe/pi.go @@ -135,7 +135,7 @@ func (p *piPlugin) linkedFromAdd(parentToken, childToken string) error { // Get existing linked from list lf, err := p.linkedFromLocked(parentToken) - if err == errRecordNotFound { + if errors.Is(err, errRecordNotFound) { return fmt.Errorf("linkedFromLocked %v: %v", parentToken, err) } @@ -559,7 +559,7 @@ func (p *piPlugin) cmdCommentVote(payload string) (string, error) { return "", fmt.Errorf("invalid state %v", cv.State) } if err != nil { - if err == backend.ErrRecordNotFound { + if errors.Is(err, backend.ErrRecordNotFound) { return "", backend.PluginUserError{ PluginID: pi.ID, ErrorCode: int(pi.ErrorStatusPropNotFound), @@ -690,7 +690,7 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { } r, err := p.backend.GetVetted(tokenb, "") if err != nil { - if err == backend.ErrRecordNotFound { + if errors.Is(err, backend.ErrRecordNotFound) { return backend.PluginUserError{ PluginID: pi.ID, ErrorCode: int(pi.ErrorStatusPropLinkToInvalid), diff --git a/politeiad/backend/tlogbe/store/filesystem/filesystem.go b/politeiad/backend/tlogbe/store/filesystem/filesystem.go index e378a2bcf..759476fd7 100644 --- a/politeiad/backend/tlogbe/store/filesystem/filesystem.go +++ b/politeiad/backend/tlogbe/store/filesystem/filesystem.go @@ -5,6 +5,7 @@ package filesystem import ( + "errors" "fmt" "io/ioutil" "os" @@ -118,7 +119,7 @@ func (f *fileSystem) Del(keys []string) error { for _, v := range keys { err := f.del(v) if err != nil { - if err == store.ErrNotFound { + if errors.Is(err, store.ErrNotFound) { // File does not exist. This is ok. continue } @@ -158,7 +159,7 @@ func (f *fileSystem) Get(keys []string) (map[string][]byte, error) { for _, v := range keys { b, err := f.get(v) if err != nil { - if err == store.ErrNotFound { + if errors.Is(err, store.ErrNotFound) { // File does not exist. This is ok. continue } diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index 783b6d1b5..d6c60cfd6 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -1189,7 +1189,7 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { version := strconv.FormatUint(uint64(s.Params.Version), 10) _, err = p.backend.GetVetted(token, version) if err != nil { - if err == backend.ErrRecordNotFound { + if errors.Is(err, backend.ErrRecordNotFound) { e := fmt.Sprintf("version %v not found", version) return "", backend.PluginUserError{ PluginID: ticketvote.ID, @@ -1536,7 +1536,7 @@ func (p *ticketVotePlugin) cmdDetails(payload string) (string, error) { // Get authorize votes auths, err := p.authorizes(token) if err != nil { - if err == errRecordNotFound { + if errors.Is(err, errRecordNotFound) { continue } return "", fmt.Errorf("authorizes: %v", err) @@ -1607,7 +1607,7 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. // Check if the summary has been cached s, err := p.cachedSummary(hex.EncodeToString(token)) switch { - case err == errRecordNotFound: + case errors.Is(err, errRecordNotFound): // Cached summary not found case err != nil: // Some other error @@ -1779,7 +1779,7 @@ func (p *ticketVotePlugin) cmdSummaries(payload string) (string, error) { } s, err := p.summary(token, bb) if err != nil { - if err == errRecordNotFound { + if errors.Is(err, errRecordNotFound) { // Record does not exist for token. Do not include this token // in the reply. continue diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index 1705846b1..76881bd11 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -1101,7 +1101,7 @@ func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []ba // Get the existing record index currIdx, err := t.recordIndexLatest(leavesAll) - if err == errRecordNotFound { + if errors.Is(err, errRecordNotFound) { // No record versions exist yet. This is ok. currIdx = &recordIndex{ Metadata: make(map[uint64][]byte), diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index f58d18764..163703dd0 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -193,7 +193,7 @@ func (t *tlogBackend) vettedTreeIDFromToken(token []byte) (int64, bool) { // contains a pointer to a vetted tree. fr, err := t.unvetted.freezeRecord(treeID) if err != nil { - if err == errFreezeRecordNotFound { + if errors.Is(err, errFreezeRecordNotFound) { // Unvetted tree exists and is not frozen. This is an unvetted // record. return 0, false @@ -642,14 +642,14 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ allMD := append(mdAppend, mdOverwrite...) err := verifyContent(allMD, filesAdd, filesDel) if err != nil { - e, ok := err.(backend.ContentVerificationError) - if !ok { + var cverr backend.ContentVerificationError + if !errors.As(err, &cverr) { return nil, err } // Allow ErrorStatusEmpty which indicates no new files are being // added. This can happen when files are being deleted without // any new files being added. - if e.ErrorCode != v1.ErrorStatusEmpty { + if cverr.ErrorCode != v1.ErrorStatusEmpty { return nil, err } } @@ -740,14 +740,14 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b allMD := append(mdAppend, mdOverwrite...) err := verifyContent(allMD, filesAdd, filesDel) if err != nil { - e, ok := err.(backend.ContentVerificationError) - if !ok { + var cverr backend.ContentVerificationError + if !errors.As(err, &cverr) { return nil, err } // Allow ErrorStatusEmpty which indicates no new files are being // added. This can happen when files are being deleted without // any new files being added. - if e.ErrorCode != v1.ErrorStatusEmpty { + if cverr.ErrorCode != v1.ErrorStatusEmpty { return nil, err } } @@ -769,7 +769,7 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b // Get existing record r, err := t.vetted.recordLatest(treeID) if err != nil { - if err == errRecordNotFound { + if errors.Is(err, errRecordNotFound) { return nil, backend.ErrRecordNotFound } return nil, fmt.Errorf("recordLatest: %v", err) @@ -839,14 +839,14 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite allMD := append(mdAppend, mdOverwrite...) err := verifyContent(allMD, []backend.File{}, []string{}) if err != nil { - e, ok := err.(backend.ContentVerificationError) - if !ok { + var cverr backend.ContentVerificationError + if !errors.As(err, &cverr) { return err } // Allow ErrorStatusEmpty which indicates no new files are being // being added. This is expected since this is a metadata only // update. - if e.ErrorCode != v1.ErrorStatusEmpty { + if cverr.ErrorCode != v1.ErrorStatusEmpty { return err } } @@ -925,14 +925,14 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ allMD := append(mdAppend, mdOverwrite...) err := verifyContent(allMD, []backend.File{}, []string{}) if err != nil { - e, ok := err.(backend.ContentVerificationError) - if !ok { + var cverr backend.ContentVerificationError + if !errors.As(err, &cverr) { return err } // Allow ErrorStatusEmpty which indicates no new files are being // being added. This is expected since this is a metadata only // update. - if e.ErrorCode != v1.ErrorStatusEmpty { + if cverr.ErrorCode != v1.ErrorStatusEmpty { return err } } @@ -959,7 +959,7 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ // Get existing record r, err := t.vetted.recordLatest(treeID) if err != nil { - if err == errRecordNotFound { + if errors.Is(err, errRecordNotFound) { return backend.ErrRecordNotFound } return fmt.Errorf("recordLatest: %v", err) @@ -1103,7 +1103,7 @@ func (t *tlogBackend) GetVetted(token []byte, version string) (*backend.Record, r, err := t.vetted.record(treeID, v) if err != nil { - if err == errRecordNotFound { + if errors.Is(err, errRecordNotFound) { err = backend.ErrRecordNotFound } return nil, err @@ -1639,7 +1639,7 @@ func (t *tlogBackend) setup() error { if vettedTreeID != 0 { r, err = t.GetVetted(token, "") if err != nil { - if err == backend.ErrRecordNotFound { + if errors.Is(err, backend.ErrRecordNotFound) { // A tree that was created but no record was appended onto // it for whatever reason. This can happen if there is a // network failure or internal server error. @@ -1650,7 +1650,7 @@ func (t *tlogBackend) setup() error { } else { r, err = t.GetUnvetted(token, "") if err != nil { - if err == backend.ErrRecordNotFound { + if errors.Is(err, backend.ErrRecordNotFound) { // A tree that was created but no record was appended onto // it for whatever reason. This can happen if there is a // network failure or internal server error. diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index 86bbd17cc..2fe06a50b 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -8,6 +8,7 @@ package pi import ( "encoding/json" + "errors" "io" "strings" ) @@ -194,7 +195,7 @@ func DecodeStatusChanges(payload []byte) ([]StatusChange, error) { for { var sc StatusChange err := d.Decode(&sc) - if err == io.EOF { + if errors.Is(err, io.EOF) { break } else if err != nil { return nil, err diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index a9bf7980b..28cc50e01 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -938,18 +938,19 @@ func (p *politeia) updateUnvettedMetadata(w http.ResponseWriter, r *http.Request convertFrontendMetadataStream(t.MDOverwrite)) if err != nil { // Reply with error if there were no changes - if err == backend.ErrNoChanges { + if errors.Is(err, backend.ErrNoChanges) { log.Infof("%v update unvetted metadata no changes: %x", remoteAddr(r), token) p.respondWithUserError(w, v1.ErrorStatusNoChanges, nil) return } // Check for content error. - if contentErr, ok := err.(backend.ContentVerificationError); ok { + var cverr backend.ContentVerificationError + if errors.As(err, &cverr) { log.Infof("%v update unvetted metadata content error: %v", - remoteAddr(r), contentErr) - p.respondWithUserError(w, contentErr.ErrorCode, - contentErr.ErrorContext) + remoteAddr(r), cverr) + p.respondWithUserError(w, cverr.ErrorCode, + cverr.ErrorContext) return } // Check for plugin error diff --git a/politeiawww/invoices.go b/politeiawww/invoices.go index 2214fee19..b6141d841 100644 --- a/politeiawww/invoices.go +++ b/politeiawww/invoices.go @@ -1914,7 +1914,7 @@ func (p *politeiawww) processNewCommentInvoice(ctx context.Context, nc www.NewCo ir, err := p.getInvoice(nc.Token) if err != nil { - if err == cmsdatabase.ErrInvoiceNotFound { + if errors.Is(err, cmsdatabase.ErrInvoiceNotFound) { err = www.UserError{ ErrorCode: cms.ErrorStatusInvoiceNotFound, } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 8c41deb6f..c024a46eb 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -1286,7 +1286,7 @@ func (p *politeiawww) processProposalEdit(ctx context.Context, pe pi.ProposalEdi // Get the current proposal curr, err := p.proposalRecordLatest(ctx, pe.State, pe.Token) if err != nil { - if err == errProposalNotFound { + if errors.Is(err, errProposalNotFound) { return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusPropNotFound, } @@ -1585,7 +1585,7 @@ func (p *politeiawww) processCommentNew(ctx context.Context, cn pi.CommentNew, u // Fetch the proposal so we can see who the author is pr, err := p.proposalRecordLatest(ctx, cn.State, cn.Token) if err != nil { - if err == errProposalNotFound { + if errors.Is(err, errProposalNotFound) { return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusPropNotFound, } @@ -1736,7 +1736,7 @@ func (p *politeiawww) processComments(ctx context.Context, c pi.Comments, usr *u // proposal author. pr, err := p.proposalRecordLatest(ctx, c.State, c.Token) if err != nil { - if err == errProposalNotFound { + if errors.Is(err, errProposalNotFound) { return nil, pi.UserErrorReply{ ErrorCode: pi.ErrorStatusPropNotFound, } diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 5c033c8f2..c7b33014f 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -431,7 +431,7 @@ func (p *politeiawww) processAuthorizeVote(av www.AuthorizeVote, u *user.User) ( // Validate the vote authorization pr, err := p.getProp(av.Token) if err != nil { - if err == cache.ErrRecordNotFound { + if errors.Is(err, cache.ErrRecordNotFound) { err = www.UserError{ ErrorCode: www.ErrorStatusProposalNotFound, } @@ -530,7 +530,7 @@ func (p *politeiawww) processStartVoteV2(sv www2.StartVote, u *user.User) (*www2 } pr, err := p.getProp(sv.Vote.Token) if err != nil { - if err == cache.ErrRecordNotFound { + if errors.Is(err, cache.ErrRecordNotFound) { err = www.UserError{ ErrorCode: www.ErrorStatusProposalNotFound, } @@ -681,7 +681,7 @@ func (p *politeiawww) processStartVoteRunoffV2(sv www2.StartVoteRunoff, u *user. } pr, err := p.getProp(token) if err != nil { - if err == cache.ErrRecordNotFound { + if errors.Is(err, cache.ErrRecordNotFound) { err = www.UserError{ ErrorCode: www.ErrorStatusProposalNotFound, ErrorContext: []string{token}, @@ -711,9 +711,10 @@ func (p *politeiawww) processStartVoteRunoffV2(sv www2.StartVoteRunoff, u *user. if err != nil { // Attach the token to the error so the user knows which one // failed. - if ue, ok := err.(*www.UserError); ok { - ue.ErrorContext = append(ue.ErrorContext, token) - err = ue + var uerr *www.UserError + if errors.As(err, &uerr) { + uerr.ErrorContext = append(uerr.ErrorContext, token) + err = uerr } return nil, err } @@ -724,9 +725,10 @@ func (p *politeiawww) processStartVoteRunoffV2(sv www2.StartVoteRunoff, u *user. if err != nil { // Attach the token to the error so the user knows which one // failed. - if ue, ok := err.(*www.UserError); ok { - ue.ErrorContext = append(ue.ErrorContext, token) - err = ue + var uerr *www.UserError + if errors.As(err, &uerr) { + uerr.ErrorContext = append(uerr.ErrorContext, token) + err = uerr } return nil, err } @@ -735,7 +737,7 @@ func (p *politeiawww) processStartVoteRunoffV2(sv www2.StartVoteRunoff, u *user. // Validate the RFP proposal rfp, err := p.getProp(sv.Token) if err != nil { - if err == cache.ErrRecordNotFound { + if errors.Is(err, cache.ErrRecordNotFound) { err = www.UserError{ ErrorCode: www.ErrorStatusProposalNotFound, ErrorContext: []string{sv.Token}, diff --git a/politeiawww/www.go b/politeiawww/www.go index 7c028338e..b53dc0429 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -639,7 +639,7 @@ func (p *politeiawww) setupCMS() error { net := filepath.Base(p.cfg.DataDir) p.cmsDB, err = cmsdb.New(p.cfg.DBHost, net, p.cfg.DBRootCert, p.cfg.DBCert, p.cfg.DBKey) - if err == database.ErrNoVersionRecord || err == database.ErrWrongVersion { + if errors.Is(err, database.ErrNoVersionRecord) || errors.Is(err, database.ErrWrongVersion) { // The cmsdb version record was either not found or // is the wrong version which means that the cmsdb // needs to be built/rebuilt. From 7ab7f6c486c326c6962f2d2dfeb093254564cc6a Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 13 Oct 2020 07:16:20 -0500 Subject: [PATCH 150/449] debugging vote routes --- politeiad/api/v1/v1.go | 1 + politeiad/backend/tlogbe/ticketvote.go | 4 +- politeiad/backend/tlogbe/tlog.go | 7 +- politeiad/backend/tlogbe/tlogbe.go | 24 +++--- politeiad/plugins/ticketvote/ticketvote.go | 12 ++- politeiad/politeiad.go | 15 ++-- politeiawww/api/pi/v1/v1.go | 61 ++++++++++---- politeiawww/cmd/piwww/piwww.go | 14 ++-- politeiawww/cmd/piwww/voteauthorize.go | 84 ++++++++++--------- politeiawww/cmd/piwww/votestart.go | 95 ++++++++++------------ politeiawww/dcc.go | 3 +- politeiawww/middleware.go | 1 + politeiawww/politeiad.go | 4 +- politeiawww/www.go | 2 + 14 files changed, 183 insertions(+), 144 deletions(-) diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index 601d844ed..d68dfed32 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -66,6 +66,7 @@ const ( ErrorStatusRecordFound ErrorStatusT = 15 ErrorStatusInvalidRPCCredentials ErrorStatusT = 16 ErrorStatusRecordNotFound ErrorStatusT = 17 + ErrorStatusInvalidToken ErrorStatusT = 18 // Record status codes (set and get) RecordStatusInvalid RecordStatusT = 0 // Invalid status diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index d6c60cfd6..274cce9ee 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -1918,7 +1918,7 @@ func (p *ticketVotePlugin) setup() error { log.Tracef("ticketvote setup") // TODO - // Ensure dcrdata plugin has been registered + // Verify dcrdata plugin has been registered // Build votes cache // Build inventory cache @@ -2046,5 +2046,7 @@ func newTicketVotePlugin(backend backend.Backend, tlog tlogClient, settings []ba activeNetParams: activeNetParams, voteDurationMin: voteDurationMin, voteDurationMax: voteDurationMax, + votes: make(map[string]map[string]string), + mutexes: make(map[string]*sync.Mutex), }, nil } diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index 76881bd11..d709e8a80 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -498,10 +498,13 @@ func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []ba return fmt.Errorf("wrong number of keys: got %v, want 1", len(keys)) } - // Append record index and freeze record leaves to trillian tree + // Append record index and freeze record leaves to trillian tree. + // The queued leaves get added to the tree by the log signer FILO. + // Send the freeze record in first so that it gets appended to the + // tree last. leaves := []*trillian.LogLeaf{ - logLeafNew(idxHash, []byte(keyPrefixRecordIndex+keys[0])), logLeafNew(freezeRecordHash, []byte(keyPrefixFreezeRecord+keys[1])), + logLeafNew(idxHash, []byte(keyPrefixRecordIndex+keys[0])), } queued, _, err := t.trillian.leavesAppend(treeID, leaves) if err != nil { diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 163703dd0..ba9c219b4 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -642,14 +642,14 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ allMD := append(mdAppend, mdOverwrite...) err := verifyContent(allMD, filesAdd, filesDel) if err != nil { - var cverr backend.ContentVerificationError - if !errors.As(err, &cverr) { + var cve backend.ContentVerificationError + if !errors.As(err, &cve) { return nil, err } // Allow ErrorStatusEmpty which indicates no new files are being // added. This can happen when files are being deleted without // any new files being added. - if cverr.ErrorCode != v1.ErrorStatusEmpty { + if cve.ErrorCode != v1.ErrorStatusEmpty { return nil, err } } @@ -740,14 +740,14 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b allMD := append(mdAppend, mdOverwrite...) err := verifyContent(allMD, filesAdd, filesDel) if err != nil { - var cverr backend.ContentVerificationError - if !errors.As(err, &cverr) { + var cve backend.ContentVerificationError + if !errors.As(err, &cve) { return nil, err } // Allow ErrorStatusEmpty which indicates no new files are being // added. This can happen when files are being deleted without // any new files being added. - if cverr.ErrorCode != v1.ErrorStatusEmpty { + if cve.ErrorCode != v1.ErrorStatusEmpty { return nil, err } } @@ -839,14 +839,14 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite allMD := append(mdAppend, mdOverwrite...) err := verifyContent(allMD, []backend.File{}, []string{}) if err != nil { - var cverr backend.ContentVerificationError - if !errors.As(err, &cverr) { + var cve backend.ContentVerificationError + if !errors.As(err, &cve) { return err } // Allow ErrorStatusEmpty which indicates no new files are being // being added. This is expected since this is a metadata only // update. - if cverr.ErrorCode != v1.ErrorStatusEmpty { + if cve.ErrorCode != v1.ErrorStatusEmpty { return err } } @@ -925,14 +925,14 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ allMD := append(mdAppend, mdOverwrite...) err := verifyContent(allMD, []backend.File{}, []string{}) if err != nil { - var cverr backend.ContentVerificationError - if !errors.As(err, &cverr) { + var cve backend.ContentVerificationError + if !errors.As(err, &cve) { return err } // Allow ErrorStatusEmpty which indicates no new files are being // being added. This is expected since this is a metadata only // update. - if cverr.ErrorCode != v1.ErrorStatusEmpty { + if cve.ErrorCode != v1.ErrorStatusEmpty { return err } } diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index cdd625974..8b5ee43ce 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -79,9 +79,15 @@ const ( // net yes votes. VoteTypeRunoff VoteT = 2 - // Vote option IDs - VoteOptionIDApprove = "yes" - VoteOptionIDReject = "no" + // VoteOptionIDApprove is the vote option ID that indicates the vote + // should be approved. Votes that are an approve/reject vote are + // required to use this vote option ID. + VoteOptionIDApprove = "approve" + + // VoteOptionIDReject is the vote option ID that indicates the vote + // should be not be approved. Votes that are an approve/reject vote + // are required to use this vote option ID. + VoteOptionIDReject = "reject" // Vote error status codes. Vote errors are errors that occur while // attempting to cast a vote. These errors are returned with the diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 28cc50e01..896a22893 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -347,8 +347,7 @@ func (p *politeia) updateRecord(w http.ResponseWriter, r *http.Request, vetted b // Validate token token, err := util.ConvertStringToken(t.Token) if err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, - nil) + p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) return } @@ -456,7 +455,7 @@ func (p *politeia) getUnvetted(w http.ResponseWriter, r *http.Request) { // Validate token token, err := util.ConvertStringToken(t.Token) if err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) + p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) return } @@ -530,7 +529,7 @@ func (p *politeia) getVetted(w http.ResponseWriter, r *http.Request) { // Validate token token, err := util.ConvertStringToken(t.Token) if err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) + p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) return } @@ -713,7 +712,7 @@ func (p *politeia) setVettedStatus(w http.ResponseWriter, r *http.Request) { // Validate token token, err := util.ConvertStringToken(t.Token) if err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) + p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) return } @@ -785,7 +784,7 @@ func (p *politeia) setUnvettedStatus(w http.ResponseWriter, r *http.Request) { // Validate token token, err := util.ConvertStringToken(t.Token) if err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) + p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) return } @@ -857,7 +856,7 @@ func (p *politeia) updateVettedMetadata(w http.ResponseWriter, r *http.Request) // Validate token token, err := util.ConvertStringToken(t.Token) if err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) + p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) return } @@ -926,7 +925,7 @@ func (p *politeia) updateUnvettedMetadata(w http.ResponseWriter, r *http.Request token, err := util.ConvertStringToken(t.Token) if err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) + p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) return } diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index a6eab59c0..22b00e08a 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -58,7 +58,9 @@ const ( // Proposal states. A proposal state can be either unvetted or // vetted. The PropStatusT type further breaks down these two - // states into more granular statuses. + // states into more granular statuses. Unvetted proposal data is + // not made available to the public. Only admins and the proposal + // author are able to view unvetted proposals. PropStateInvalid PropStateT = 0 PropStateUnvetted PropStateT = 1 PropStateVetted PropStateT = 2 @@ -107,6 +109,16 @@ const ( // net yes votes. VoteTypeRunoff VoteT = 2 + // VoteOptionIDApprove is the vote option ID that indicates the + // proposal should be approved. Proposal votes are required to use + // this vote option ID. + VoteOptionIDApprove = "approve" + + // VoteOptionIDReject is the vote option ID that indicates the + // proposal should be rejected. Proposal votes are required to use + // this vote option ID. + VoteOptionIDReject = "reject" + // Error status codes ErrorStatusInvalid ErrorStatusT = 0 ErrorStatusInputInvalid ErrorStatusT = 1 @@ -284,17 +296,22 @@ const ( // ProposalMetadata contains metadata that is specified by the user on proposal // submission. It is attached to a proposal submission as a Metadata object. +// +// TODO should there be a Type field? The issue is that we currently assume +// a proposal is an RFP submission if the LinkTo is set. This boxes us in and +// prevents us from using LinkTo in the future without some additional field +// like a Type field. type ProposalMetadata struct { Name string `json:"name"` // Proposal name - // LinkTo specifies a public proposal token to link this proposal - // to. Ex, an RFP submission must link to the RFP proposal. - LinkTo string `json:"linkto,omitempty"` - // LinkBy is a UNIX timestamp that serves as a deadline for other // proposals to link to this proposal. Ex, an RFP submission cannot // link to an RFP proposal once the RFP's LinkBy deadline is past. LinkBy int64 `json:"linkby,omitempty"` + + // LinkTo specifies a public proposal token to link this proposal + // to. Ex, an RFP submission must link to the RFP proposal. + LinkTo string `json:"linkto,omitempty"` } // CensorshipRecord contains cryptographic proof that a proposal was accepted @@ -350,7 +367,7 @@ type ProposalRecord struct { LinkedFrom []string `json:"linkedfrom"` // CensorshipRecord contains cryptographic proof that the proposal - // was received by the server. + // was received and processed by the server. CensorshipRecord CensorshipRecord `json:"censorshiprecord"` } @@ -669,17 +686,27 @@ type VoteResult struct { // VoteSummary summarizes the vote params and results of a proposal vote. type VoteSummary struct { - Type VoteT `json:"type"` - Status VoteStatusT `json:"status"` - Duration uint32 `json:"duration"` - StartBlockHeight uint32 `json:"startblockheight"` - StartBlockHash string `json:"startblockhash"` - EndBlockHeight uint32 `json:"endblockheight"` - EligibleTickets uint32 `json:"eligibletickets"` - QuorumPercentage uint32 `json:"quorumpercentage"` - PassPercentage uint32 `json:"passpercentage"` - Results []VoteResult `json:"results"` - Approved bool `json:"approved"` + Type VoteT `json:"type"` + Status VoteStatusT `json:"status"` + Duration uint32 `json:"duration"` // In blocks + StartBlockHeight uint32 `json:"startblockheight"` + StartBlockHash string `json:"startblockhash"` + EndBlockHeight uint32 `json:"endblockheight"` + + // EligibleTickets is the number of tickets that are eligible to + // cast a vote. + EligibleTickets uint32 `json:"eligibletickets"` + + // QuorumPercentage is the percent of eligible tickets required to + // vote in order to have a quorum. + QuorumPercentage uint32 `json:"quorumpercentage"` + + // PassPercentage is the percent of total votes required to approve + // the vote in order for the vote to pass. + PassPercentage uint32 `json:"passpercentage"` + + Results []VoteResult `json:"results"` + Approved bool `json:"approved"` // Was the vote approved } // VoteAuthorize authorizes a proposal vote or revokes a previous vote diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index 25f4231e2..6799d12cd 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -32,6 +32,8 @@ var ( ) type piwww struct { + Config shared.Config + // Basic commands Help helpCmd `command:"help"` @@ -104,7 +106,7 @@ const helpMsg = `Application Options: --silent Suppress all output Help commands - help Print detailed help message for a command + help Print detailed help message for a command Basic commands version (public) Get politeiawww server version @@ -157,11 +159,11 @@ Vote commands voteinv (public) Get proposal inventory by vote status Websocket commands - subscribe (public) Subscribe/unsubscribe to websocket event + subscribe (public) Subscribe/unsubscribe to websocket event Dev commands - sendfaucettx Send a dcr faucet tx - testrun Execute a test run of pi routes + sendfaucettx Send a dcr faucet tx + testrun Execute a test run of pi routes ` func _main() error { @@ -207,7 +209,9 @@ func _main() error { } // Parse subcommand and execute - parser = flags.NewParser(&piwww{}, flags.Default) + parser = flags.NewParser(&piwww{ + Config: *cfg, + }, flags.Default) _, err = parser.Parse() if err != nil { os.Exit(1) diff --git a/politeiawww/cmd/piwww/voteauthorize.go b/politeiawww/cmd/piwww/voteauthorize.go index da71413b4..227dd7c77 100644 --- a/politeiawww/cmd/piwww/voteauthorize.go +++ b/politeiawww/cmd/piwww/voteauthorize.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Decred developers +// Copyright (c) 2017-2020 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -7,31 +7,32 @@ package main import ( "encoding/hex" "fmt" + "strconv" pi "github.com/decred/politeia/politeiawww/api/pi/v1" "github.com/decred/politeia/politeiawww/cmd/shared" "github.com/decred/politeia/util" ) -// voteAuthorizeCmd authorizes a proposal vote. The VoteAuthorizeCmd must be -// sent by the proposal author to be valid. +// voteAuthorizeCmd authorizes a proposal vote or revokes a previous vote +// authorization. type voteAuthorizeCmd struct { Args struct { - Token string `positional-arg-name:"token" required:"true"` // Censorship token - Action string `positional-arg-name:"action"` // Authorize or revoke action + Token string `positional-arg-name:"token" required:"true"` + Action string `positional-arg-name:"action"` } `positional-args:"true"` } -// Execute executes the authorize vote command. +// Execute executes the vote authorize command. func (cmd *voteAuthorizeCmd) Execute(args []string) error { token := cmd.Args.Token - // Check for user identity + // Verify user identity if cfg.Identity == nil { return shared.ErrUserIdentityNotFound } - // Validate action + // Verify action var action pi.VoteAuthActionT switch cmd.Args.Action { case "authorize": @@ -42,71 +43,76 @@ func (cmd *voteAuthorizeCmd) Execute(args []string) error { // Default to authorize action = pi.VoteAuthActionAuthorize default: - return fmt.Errorf("Invalid action. Valid actions are:\n " + - "authorize (default) authorize a vote\n " + - "revoke revoke a vote authorization") + return fmt.Errorf("Invalid action; \n%v", voteAuthorizeHelpMsg) } - // Get server public key - vr, err := client.Version() + // Get proposal version + pr, err := proposalRecordLatest(pi.PropStateVetted, token) if err != nil { - return err + return fmt.Errorf("proposalRecordLatest: %v", err) } - - // Get proposal version - pdr, err := client.ProposalDetails(token, nil) + version, err := strconv.ParseUint(pr.Version, 10, 32) if err != nil { return err } - // Setup authorize vote request - sig := cfg.Identity.SignMessage([]byte(token + pdr.Proposal.Version + - cmd.Args.Action)) + // Setup request + msg := token + pr.Version + string(action) + b := cfg.Identity.SignMessage([]byte(msg)) + signature := hex.EncodeToString(b[:]) va := pi.VoteAuthorize{ - Action: action, Token: token, - PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), - Signature: hex.EncodeToString(sig[:]), + Version: uint32(version), + Action: action, + PublicKey: cfg.Identity.Public.String(), + Signature: signature, } - // Print request details + // Send request. The request and response details are printed to + // the console. err = shared.PrintJSON(va) if err != nil { return err } - - // Send request - varep, err := client.VoteAuthorize(va) + ar, err := client.VoteAuthorize(va) + if err != nil { + return err + } + err = shared.PrintJSON(ar) if err != nil { return err } - // Validate authorize vote receipt + // Verify receipt + vr, err := client.Version() + if err != nil { + return err + } serverID, err := util.IdentityFromString(vr.PubKey) if err != nil { return err } - s, err := util.ConvertSignature(varep.Receipt) + s, err := util.ConvertSignature(ar.Receipt) if err != nil { return err } - if !serverID.VerifyMessage([]byte(va.Signature), s) { - return fmt.Errorf("could not verify authorize vote receipt") + if !serverID.VerifyMessage([]byte(signature), s) { + return fmt.Errorf("could not verify receipt") } - // Print response details - return shared.PrintJSON(vr) + return nil } -// voteAuthorizeHelpMsg is the output of the help command when 'voteauthorize' -// is specified. +// voteAuthorizeHelpMsg is the help command message. const voteAuthorizeHelpMsg = `voteauthorize "token" "action" -Authorize or revoke proposal vote. Only the proposal author (owner of -censorship token) can authorize or revoke vote. +Authorize or revoke a proposal vote. Must be proposal author. + +Valid actions: + authorize authorize a vote + revoke revoke a previous authorization Arguments: 1. token (string, required) Proposal censorship token -2. action (string, optional) Valid actions are 'authorize' or 'revoke' - (defaults to 'authorize') +2. action (string, optional) Authorize vote actions (default: authorize) ` diff --git a/politeiawww/cmd/piwww/votestart.go b/politeiawww/cmd/piwww/votestart.go index 50812a0aa..598906f17 100644 --- a/politeiawww/cmd/piwww/votestart.go +++ b/politeiawww/cmd/piwww/votestart.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Decred developers +// Copyright (c) 2017-2020 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -7,41 +7,39 @@ package main import ( "encoding/hex" "encoding/json" + "fmt" "strconv" - "github.com/decred/politeia/decredplugin" pi "github.com/decred/politeia/politeiawww/api/pi/v1" "github.com/decred/politeia/politeiawww/cmd/shared" "github.com/decred/politeia/util" ) // voteStartCmd starts the voting period on the specified proposal. -// -// The QuorumPercentage and PassPercentage are strings and not uint32 so that a -// value of 0 can be passed in and not be overwritten by the defaults. This is -// sometimes desirable when testing. type voteStartCmd struct { Args struct { Token string `positional-arg-name:"token" required:"true"` Duration uint32 `positional-arg-name:"duration"` - QuorumPercentage string `positional-arg-name:"quorumpercentage"` - PassPercentage string `positional-arg-name:"passpercentage"` + QuorumPercentage uint32 `positional-arg-name:"quorumpercentage"` + PassPercentage uint32 `positional-arg-name:"passpercentage"` } `positional-args:"true"` } -// Execute executes the start vote command. +// Execute executes the vote start command. func (cmd *voteStartCmd) Execute(args []string) error { - // Check for user identity + token := cmd.Args.Token + + // Verify user identity if cfg.Identity == nil { return shared.ErrUserIdentityNotFound } - // Get proposal version. This is needed for the pi route. - pdr, err := client.ProposalDetails(cmd.Args.Token, nil) + // Get proposal version + pr, err := proposalRecordLatest(pi.PropStateVetted, token) if err != nil { - return err + return fmt.Errorf("proposalRecordLatest: %v", err) } - version, err := strconv.ParseUint(pdr.Proposal.Version, 10, 32) + version, err := strconv.ParseUint(pr.Version, 10, 32) if err != nil { return err } @@ -56,89 +54,78 @@ func (cmd *voteStartCmd) Execute(args []string) error { if cmd.Args.Duration != 0 { duration = cmd.Args.Duration } - if cmd.Args.QuorumPercentage != "" { - i, err := strconv.ParseUint(cmd.Args.QuorumPercentage, 10, 32) - if err != nil { - return err - } - quorum = uint32(i) + if cmd.Args.QuorumPercentage != 0 { + quorum = cmd.Args.QuorumPercentage } - if cmd.Args.PassPercentage != "" { - i, err := strconv.ParseUint(cmd.Args.PassPercentage, 10, 32) - if err != nil { - return err - } - pass = uint32(i) + if cmd.Args.PassPercentage != 0 { + pass = cmd.Args.PassPercentage } - // Create VoteStart + // Setup request vote := pi.VoteParams{ - Token: cmd.Args.Token, + Token: token, Version: uint32(version), Type: pi.VoteTypeStandard, - Mask: 0x03, // bit 0 no, bit 1 yes + Mask: 0x03, Duration: duration, QuorumPercentage: quorum, PassPercentage: pass, Options: []pi.VoteOption{ { - ID: decredplugin.VoteOptionIDApprove, + ID: pi.VoteOptionIDApprove, Description: "Approve proposal", Bit: 0x01, }, { - ID: decredplugin.VoteOptionIDReject, + ID: pi.VoteOptionIDReject, Description: "Don't approve proposal", Bit: 0x02, }, }, } - vb, err := json.Marshal(vote) if err != nil { return err } msg := hex.EncodeToString(util.Digest(vb)) - sig := cfg.Identity.SignMessage([]byte(msg)) + b := cfg.Identity.SignMessage([]byte(msg)) + signature := hex.EncodeToString(b[:]) vs := pi.VoteStart{ Params: vote, - PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), - Signature: hex.EncodeToString(sig[:]), + PublicKey: cfg.Identity.Public.String(), + Signature: signature, } - // Print request details + // Send request. The request and response details are printed to + // the console. err = shared.PrintJSON(vs) if err != nil { return err } - - // Send request vsr, err := client.VoteStart(vs) if err != nil { return err } - - // Remove ticket snapshot from the response so that the output - // is legible vsr.EligibleTickets = []string{"removed by piwww for readability"} + err = shared.PrintJSON(vsr) + if err != nil { + return err + } - // Print response details - return shared.PrintJSON(vsr) + return nil } -// voteStartHelpMsg is the output of the help command when 'votestart' is -// specified. +// voteStartHelpMsg is the help command message. var voteStartHelpMsg = `votestart -Start voting period for a proposal. Requires admin privileges. - -The quorumpercentage and passpercentage are strings and not uint32 so that a -value of 0 can be passed in and not be overwritten by the defaults. This is -sometimes desirable when testing. +Start the voting period for a proposal. Requires admin privileges. Arguments: -1. token (string, required) Proposal censorship token -2. duration (uint32, optional) Duration of vote in blocks (default: 2016) -3. quorumpercentage (string, optional) Percent of votes required for quorum (default: 10) -4. passpercentage (string, optional) Percent of votes required to pass (default: 60) +1. token (string, required) Proposal censorship token +2. duration (uint32, optional) Duration of vote in blocks + (default: 2016) +3. quorumpercentage (uint32, optional) Percent of total votes required to + reach a quorum (default: 10) +4. passpercentage (uint32, optional) Percent of cast votes required for + vote to be approved (default: 60) ` diff --git a/politeiawww/dcc.go b/politeiawww/dcc.go index cfb951a13..3e8fb9a11 100644 --- a/politeiawww/dcc.go +++ b/politeiawww/dcc.go @@ -1017,7 +1017,8 @@ func stringInSlice(arr []string, str string) bool { func validateNewComment(c www.NewComment) error { // Validate token - if !tokenIsValid(c.Token) { + _, err := util.ConvertStringToken(c.Token) + if err != nil { return www.UserError{ ErrorCode: www.ErrorStatusInvalidCensorshipToken, } diff --git a/politeiawww/middleware.go b/politeiawww/middleware.go index 86343bf6e..3f2470b97 100644 --- a/politeiawww/middleware.go +++ b/politeiawww/middleware.go @@ -69,6 +69,7 @@ func (p *politeiawww) isLoggedInAsAdmin(f http.HandlerFunc) http.HandlerFunc { return } if !isAdmin { + log.Debugf("%v user is not an admin", http.StatusForbidden) util.RespondWithJSON(w, http.StatusForbidden, www.UserError{}) return } diff --git a/politeiawww/politeiad.go b/politeiawww/politeiad.go index 288f5b806..db8e4438b 100644 --- a/politeiawww/politeiad.go +++ b/politeiawww/politeiad.go @@ -368,7 +368,7 @@ func (p *politeiawww) getVetted(ctx context.Context, token, version string) (*pd if err != nil { return nil, err } - gu := pd.GetVetted{ + gv := pd.GetVetted{ Challenge: hex.EncodeToString(challenge), Token: token, Version: version, @@ -376,7 +376,7 @@ func (p *politeiawww) getVetted(ctx context.Context, token, version string) (*pd // Send request resBody, err := p.makeRequest(ctx, http.MethodPost, - pd.GetVettedRoute, gu) + pd.GetVettedRoute, gv) if err != nil { return nil, err } diff --git a/politeiawww/www.go b/politeiawww/www.go index b53dc0429..1181cf5f4 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -180,6 +180,8 @@ func convertPiErrorStatusFromPD(e pd.ErrorStatusT) pi.ErrorStatusT { return pi.ErrorStatusNoPropChanges case pd.ErrorStatusRecordNotFound: return pi.ErrorStatusPropNotFound + case pd.ErrorStatusInvalidToken: + return pi.ErrorStatusPropTokenInvalid } return pi.ErrorStatusInvalid } From 81a769e4460699c7eadb60f55c861f15888c7729 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 13 Oct 2020 08:05:07 -0500 Subject: [PATCH 151/449] comment downvotes bug fix --- politeiad/backend/tlogbe/comments.go | 3 +++ politeiawww/piwww.go | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/politeiad/backend/tlogbe/comments.go b/politeiad/backend/tlogbe/comments.go index 700928360..1e1206a2d 100644 --- a/politeiad/backend/tlogbe/comments.go +++ b/politeiad/backend/tlogbe/comments.go @@ -32,6 +32,9 @@ import ( // is to check if the record exists in the mutexes function to ensure a token // is valid before holding the lock on it. This is where we can return a // record doesn't exist user error too. +// TODO prevent duplicate comments +// TODO upvoting a comment twice in the same second causes a duplicate leaf +// error which causes a 500. Solution: add the timestamp to the vote index. const ( // Blob entry data descriptors diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index c024a46eb..4661b0f0e 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -370,10 +370,10 @@ func convertCommentFromPlugin(c comments.Comment) pi.Comment { CommentID: c.CommentID, Timestamp: c.Timestamp, Receipt: c.Receipt, - - Upvotes: c.Upvotes, - Censored: c.Deleted, - Reason: c.Reason, + Downvotes: c.Downvotes, + Upvotes: c.Upvotes, + Censored: c.Deleted, + Reason: c.Reason, } } From 6cbde61312162c73e72f7720f694b85dd561856a Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 14 Oct 2020 10:08:41 -0500 Subject: [PATCH 152/449] fix dcrdata plugin bugs --- go.mod | 35 +-- go.sum | 415 +++++++++++++++++++++++++ politeiad/backend/tlogbe/dcrdata.go | 187 +++++++++-- politeiad/backend/tlogbe/ticketvote.go | 26 +- politeiad/backend/tlogbe/tlog.go | 5 +- politeiad/backend/tlogbe/tlogbe.go | 12 +- politeiad/plugins/dcrdata/dcrdata.go | 95 +++++- politeiad/politeiad.go | 10 +- politeiawww/cmd/piwww/piwww.go | 7 +- politeiawww/cmd/piwww/sendfaucettx.go | 25 +- politeiawww/cmd/shared/shared.go | 6 +- 11 files changed, 722 insertions(+), 101 deletions(-) diff --git a/go.mod b/go.mod index 1b648d1c0..0e6214579 100644 --- a/go.mod +++ b/go.mod @@ -7,9 +7,7 @@ require ( github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/coreos/bbolt v1.3.3 // indirect github.com/coreos/etcd v3.3.15+incompatible // indirect - github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect - github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/dajohi/goemail v1.0.0 github.com/davecgh/go-spew v1.1.1 github.com/decred/dcrd/blockchain/stake/v3 v3.0.0-20200921185235-6d75c7ec1199 @@ -20,8 +18,10 @@ require ( github.com/decred/dcrd/dcrec/secp256k1/v3 v3.0.0 github.com/decred/dcrd/dcrutil/v3 v3.0.0 github.com/decred/dcrd/hdkeychain/v3 v3.0.0 + github.com/decred/dcrd/rpc/jsonrpc/types/v2 v2.0.1-0.20200921185235-6d75c7ec1199 github.com/decred/dcrd/wire v1.4.0 github.com/decred/dcrdata/api/types/v4 v4.0.4 + github.com/decred/dcrdata/api/types/v5 v5.0.1 github.com/decred/dcrdata/explorer/types/v2 v2.1.1 github.com/decred/dcrdata/pubsub/types/v3 v3.0.5 github.com/decred/dcrdata/pubsub/v4 v4.0.3-0.20191219212733-19f656d6d679 @@ -30,14 +30,10 @@ require ( github.com/decred/dcrtime/api/v2 v2.0.0-20200912200806-b1e4dbc46be9 github.com/decred/go-socks v1.1.0 github.com/decred/slog v1.1.0 - github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/go-test/deep v1.0.1 - github.com/gogo/protobuf v1.3.0 // indirect - github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect - github.com/golang/protobuf v1.3.2 + github.com/golang/protobuf v1.4.2 github.com/golang/snappy v0.0.1 // indirect - github.com/google/certificate-transparency-go v1.0.21 // indirect - github.com/google/trillian v1.2.1 + github.com/google/trillian v1.3.11 github.com/google/uuid v1.1.1 github.com/gorilla/csrf v1.6.2 github.com/gorilla/mux v1.7.3 @@ -45,13 +41,9 @@ require ( github.com/gorilla/securecookie v1.1.1 github.com/gorilla/sessions v1.2.1 github.com/gorilla/websocket v1.4.2 - github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 // indirect - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect - github.com/grpc-ecosystem/grpc-gateway v1.9.3 // indirect github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c github.com/jessevdk/go-flags v1.4.1-0.20200711081900-c17162fe8fd7 github.com/jinzhu/gorm v1.9.10 - github.com/jonboulle/clockwork v0.1.0 // indirect github.com/jrick/logrotate v1.0.0 github.com/kylelemons/godebug v1.1.0 // indirect github.com/marcopeereboom/sbox v1.0.0 @@ -59,21 +51,12 @@ require ( github.com/otiai10/curr v0.0.0-20190513014714-f5a3d24e5776 // indirect github.com/pmezard/go-difflib v1.0.0 github.com/pquerna/otp v1.2.0 - github.com/prometheus/client_golang v1.1.0 // indirect github.com/robfig/cron v1.2.0 - github.com/soheilhy/cmux v0.1.4 // indirect github.com/subosito/gozaru v0.0.0-20190625071150-416082cce636 github.com/syndtr/goleveldb v1.0.0 - github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect - github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect - go.opencensus.io v0.22.0 // indirect - go.uber.org/atomic v1.4.0 // indirect - go.uber.org/multierr v1.1.0 // indirect - go.uber.org/zap v1.10.0 // indirect - golang.org/x/crypto v0.0.0-20200214034016-1d94cc7ab1c6 - golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271 - golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e - google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 - google.golang.org/grpc v1.25.1 - sigs.k8s.io/yaml v1.1.0 // indirect + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 + golang.org/x/net v0.0.0-20200625001655-4c5254603344 + golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 + google.golang.org/genproto v0.0.0-20200707001353-8e8330bf89df + google.golang.org/grpc v1.29.1 ) diff --git a/go.sum b/go.sum index 4c8e6e321..7b92496c5 100644 --- a/go.sum +++ b/go.sum @@ -1,19 +1,57 @@ +bitbucket.org/creachadair/shell v0.0.6/go.mod h1:8Qqi/cYk7vPnsOePHroKXDJYmb5x7ENhtiFtfZq8K+M= bou.ke/monkey v1.0.1 h1:zEMLInw9xvNakzUUPjfS4Ds6jYPqCFx3m7bRmG5NH2U= bou.ke/monkey v1.0.1/go.mod h1:FgHuK96Rv2Nlf+0u1OOVDpCMdsWyOFmeeketDHE7LIg= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.37.4 h1:glPeL3BQJsbF6aIIYfZizMwc5LTYz250bDMjttbBGAU= cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.60.0/go.mod h1:yw2G51M9IfRboUH61Us8GqCeF1PzPblB823Mn2q2eAU= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/pubsub v1.5.0/go.mod h1:ZEwJccE3z93Z2HWvstpri00jOg7oO4UZDtKhwDwqF0w= +cloud.google.com/go/spanner v1.7.0/go.mod h1:sd3K2gZ9Fd0vMPLXzeCrF6fq4i63Q7aTLW/lBIfBkIk= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +contrib.go.opencensus.io/exporter/stackdriver v0.13.4/go.mod h1:aXENhDJ1Y4lIg4EUaVTwzvYETVNZk10Pu26tevFKLUc= decred.org/cspp v0.3.0/go.mod h1:UygjYilC94dER3BEU65Zzyoqy9ngJfWCD2rdJqvUs2A= decred.org/dcrwallet v1.2.3-0.20200925172850-94689a677799 h1:mXYF9OaPa/VaK3xeL0zUWYj+nyryath9YoRqG38FyOo= decred.org/dcrwallet v1.2.3-0.20200925172850-94689a677799/go.mod h1:Y9gOu+kdzy5exoPsWto2qPZMkgUOeZCaToCY/0MPlq8= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkByhMAwYaFAX9w2l7vxvBQ5NMoxDrkhqhtn4= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/sprig v2.15.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= +github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/Sereal/Sereal v0.0.0-20190618215532-0b8ac451a863 h1:BRrxwOZBolJN4gIwvZMJY1tzqBvQgpaZiQRuIDD40jM= github.com/Sereal/Sereal v0.0.0-20190618215532-0b8ac451a863/go.mod h1:D0JMgToj/WdxCgd30Kc1UcA9E+WdZoJqeVOuYW7iTBM= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= @@ -22,14 +60,22 @@ github.com/aead/siphash v0.0.0-20170329201724-e404fcfc8885/go.mod h1:Nywa3cDsYNN github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7ISrnJIXKzwaspym5BTKGx93EI= github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412/go.mod h1:WPjqKcmVOxf0XSf3YxCJs6N6AOSrOx3obionmG7T0y0= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= +github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asdine/storm/v3 v3.0.0-20191014171123-c370e07ad6d4 h1:92RJlxO6DcRon/jV6MxU6FYymYE02Ku1ZuRKpSOuTk4= github.com/asdine/storm/v3 v3.0.0-20191014171123-c370e07ad6d4/go.mod h1:wncSIXIbR3lvJQhBpnwAeNPQneL5Vx2KUox2jARUdmw= +github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= @@ -43,19 +89,36 @@ github.com/carterjones/go-cloudflare-scraper v0.1.2/go.mod h1:maO/ygX7QWbdh/TzHq github.com/carterjones/signalr v0.3.5 h1:kJSw+6a9XmsOb/+9HWTnY8SjTrVOdpzCSPV/9IVS2nI= github.com/carterjones/signalr v0.3.5/go.mod h1:SOGIwr/0/4GGNjHWSSginY66OVSaOeM85yWCNytdEwE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/coreos/bbolt v1.3.3 h1:n6AiVyVRKQFNb6mJlwESEvvLoDyiTzXX7ORAUlkeBdY= github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.15+incompatible h1:+9RjdC18gMxNQVvSiXvObLu29mOFmkgdsB4cRTlV+EE= github.com/coreos/etcd v3.3.15+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/dajohi/goemail v1.0.0 h1:0UbsqT92my1iLTbeBYIsJ8JnrF+6NAAgk2iat6d6e+4= github.com/dajohi/goemail v1.0.0/go.mod h1:YyX3pgj9VJX6VQYu8Cbs0GYHzgFUs8q0vX5pLmFvops= +github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -338,6 +401,7 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumC github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= @@ -345,55 +409,106 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/elazarl/goproxy v0.0.0-20181111060418-2ce16c963a8a/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.0.14/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fullstorydev/grpcurl v1.6.0/go.mod h1:ZQ+ayqbKMJNhzLmbpCiurTVlaK2M/3nqZCxaQ2Ze/sM= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-redis/redis v6.15.8+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.1 h1:UQhStjbkDClarlmv0am7OXXO4/GaPdCGiUiMTvi28sg= github.com/go-test/deep v1.0.1/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/certificate-transparency-go v1.0.21 h1:Yf1aXowfZ2nuboBsg7iYGLmwsOARdV86pfH3g95wXmE= github.com/google/certificate-transparency-go v1.0.21/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200507031123-427632fa3b1c/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/trillian v1.2.1 h1:MolbIOUOz4kOsbXMtBMv63fX68hdkT99bfDtPOXRPYw= github.com/google/trillian v1.2.1/go.mod h1:YPmUVn5NGwgnDUgqlVyFGMTgaWlnSvH7W5p+NdOG8UA= +github.com/google/trillian v1.3.11 h1:pPzJPkK06mvXId1LHEAJxIegGgHzzp/FUnycPYfoCMI= +github.com/google/trillian v1.3.11/go.mod h1:0tPraVHrSDkA3BO6vKX67zgLXs6SsOAbHEivX+9mPgw= +github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4 h1:hU4mGcQI4DaAYW+IbTun+2qEZVFxK0ySjQLTbS0VQKc= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gordonklaus/ineffassign v0.0.0-20200309095847-7953dde2c7bf/go.mod h1:cuNKsD1zp2v6XfE/orVX2QE1LC+i254ceGcVeDT3pTU= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/csrf v1.6.2 h1:QqQ/OWwuFp4jMKgBFAzJVW3FMULdyUW7JoM4pEWuqKg= github.com/gorilla/csrf v1.6.2/go.mod h1:7tSf8kmjNYr7IWDCYhd3U8Ck34iQ/Yw5CJu7bAkHEGI= @@ -407,6 +522,7 @@ github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyC github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= @@ -415,27 +531,40 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.2.2/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.3 h1:O8JuYkaEesTVBN68o2pLhRGTfVXnGhKtx3qjOmQkJV0= github.com/grpc-ecosystem/grpc-gateway v1.9.3/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c h1:fEE5/5VNnYUoBOj2I9TP8Jc+a7lge3QWn9DKE7NCwfc= github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c/go.mod h1:ObS/W+h8RYb1Y7fYivughjxojTmIu5iAIjSrSLCLeqE= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= +github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.4/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.1-0.20200711081900-c17162fe8fd7 h1:Ug59miTxVKVg5Oi2S5uHlKOIV5jBx4Hb2u0jIxxDaSs= github.com/jessevdk/go-flags v1.4.1-0.20200711081900-c17162fe8fd7/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jhump/protoreflect v1.6.1/go.mod h1:RZQ/lnuN+zqeRVpQigTwO6o0AJUkxbnSnpuG7toUTG4= github.com/jinzhu/gorm v1.9.10 h1:HvrsqdhCW78xpJF67g1hMxS6eCToo9PZH4LDB8WKPac= github.com/jinzhu/gorm v1.9.10/go.mod h1:Kh6hTsSGffh4ui079FHrR5Gg+5D0hgihqDcsDN2BBJY= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jrick/bitset v1.0.0/go.mod h1:ZOYB5Uvkla7wIEY4FEssPVi3IQXa02arznRaYaAEPe4= @@ -446,12 +575,16 @@ github.com/jrick/wsrpc/v2 v2.3.4/go.mod h1:XPYs8BnRWl99lCvXRM5SLpZmTPqWpSOPkDIqY github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -461,14 +594,33 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/letsencrypt/pkcs11key/v4 v4.0.0/go.mod h1:EFUvBDay26dErnNb70Nd0/VW3tJiIbETBPTl9ATXQag= github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= +github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/marcopeereboom/sbox v1.0.0 h1:1xTRUzI5mVvsaPaBGVpXdyH2hZeZhCWazbGy6MVsRgg= github.com/marcopeereboom/sbox v1.0.0/go.mod h1:V9e7t7oKphNfXymk7Lqvbo9mZiVjmCt8vBHnROcpCSY= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/pkcs11 v1.0.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -476,14 +628,23 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-proto-validators v0.0.0-20180403085117-0950a7990007/go.mod h1:m2XC9Qq0AlmmVksL6FktJCdTYyLk7V3fKyp0sl1yWQo= +github.com/mwitkow/go-proto-validators v0.2.0/go.mod h1:ZfA1hW+UH/2ZHOWvQ3HnQaU0DtnpXu850MZiy+YUgcc= +github.com/nishanths/predeclared v0.0.0-20190419143655-18a43bb90ffc/go.mod h1:62PewwiQTlm/7Rj+cxVYqZvDIUc+JjZq6GHAC1fsObQ= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/olekukonko/tablewriter v0.0.2/go.mod h1:rSAaSIOAGT9odnlyGlUfAJaoc5w2fSBUmeGDbRWPxyQ= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/otiai10/copy v1.0.1 h1:gtBjD8aq4nychvRZ2CyJvFWAw0aja+VHazDdruZKGZA= github.com/otiai10/copy v1.0.1/go.mod h1:8bMCJrAqOtN/d9oyh5HR7HhLQMvcGMpGdwRDYsfOCHc= @@ -493,10 +654,12 @@ github.com/otiai10/curr v0.0.0-20190513014714-f5a3d24e5776/go.mod h1:3HNVkVOU7vZ github.com/otiai10/mint v1.2.3/go.mod h1:YnfyPNhBvnY8bW4SGQHCs/aAFhkgySlMZbrF5U0bOVw= github.com/otiai10/mint v1.2.4 h1:DxYL0itZyPaR5Z9HILdxSoHx+gNs6Yx+neOGS3IVUk0= github.com/otiai10/mint v1.2.4/go.mod h1:d+b7n/0R3tdyUYYylALXpWQ/kTN+QobSq/4SRGBkR3M= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/otp v1.2.0 h1:/A3+Jn+cagqayeR3iHs/L62m5ue7710D35zl1zJ1kok= @@ -506,76 +669,151 @@ github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/pseudomuto/protoc-gen-doc v1.3.2/go.mod h1:y5+P6n3iGrbKG+9O04V5ld71in3v/bX88wUwgt+U8EA= +github.com/pseudomuto/protokit v0.2.0/go.mod h1:2PdH30hxVHsup8KpBTOXTBeMVhJZVio3Q8ViKSAXT0Q= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/robertkrimen/otto v0.0.0-20180617131154-15f95af6e78d h1:1VUlQbCfkoSGv7qP7Y+ro3ap1P1pPZxgdGVqiTVy5C4= github.com/robertkrimen/otto v0.0.0-20180617131154-15f95af6e78d/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v0.0.0-20170130113145-4d4bfba8f1d1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/subosito/gozaru v0.0.0-20190625071150-416082cce636 h1:LlXBFcxziHIkc7jnbCmUCL5+ujGMky2aJsNvHqtt80Y= github.com/subosito/gozaru v0.0.0-20190625071150-416082cce636/go.mod h1:LIpwO1yApZNrEQZdu5REqRtRrkaU+52ueA7WGT+CvSw= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vmihailenco/msgpack v4.0.1+incompatible h1:RMF1enSPeKTlXrXdOcqjFUElywVZjjC6pqse21bKbEU= github.com/vmihailenco/msgpack v4.0.1+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.0/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/etcd v0.0.0-20200513171258-e048e166ab9c/go.mod h1:xCI7ZzBfRuGgBXyXO6yfWfDmlWd35khcWpUa4L0xI/k= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +golang.org/x/crypto v0.0.0-20180501155221-613d6eafa307/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180718160520-a2144134853f/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180808211826-de0752318171/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 h1:Gv7RPwsi3eZ2Fgewe3CBsuOebPwO27PoXzRpJPsvSSM= golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200214034016-1d94cc7ab1c6 h1:Sy5bstxEqwwbYs6n0/pBuxKENqOeZUgD45Gp3Q3pqLg= golang.org/x/crypto v0.0.0-20200214034016-1d94cc7ab1c6/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180808004115-f9ce57c11b24/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -590,22 +828,47 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271 h1:N66aaryRB3Ax92gH0v3hp1QYZ3zWWCCUR/j8Ifh45Ss= golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180810070207-f0d5e33068cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -613,60 +876,202 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190416124237-ebb4019f01c9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190614084037-d442b75600c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191010075000-0337d82405ff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200626171337-aa94e735be7f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200630154851-b2d8b0336632/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200706234117-b22de6825cf7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.3.1 h1:oJra/lMfmtm13/rgY/8i3MzjFWYXvQIAKjQ3HqofMk8= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.10.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20170818010345-ee236bd376b0/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180808183934-383e8b2c3b9e/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181107211654-5fc9ac540362/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190415143225-d1146b9035b9/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb h1:i1Ppqkc3WQXikh8bXiwHqAN5Rv3/qDCcRk0/Otx73BY= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200626011028-ee7919e894b5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200707001353-8e8330bf89df h1:HWF6nM8ruGdu1K8IXFR+i2oT3YP+iBfZzCbC9zUfcWo= +google.golang.org/genproto v0.0.0-20200707001353-8e8330bf89df/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.0/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= @@ -678,9 +1083,19 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.6/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/politeiad/backend/tlogbe/dcrdata.go b/politeiad/backend/tlogbe/dcrdata.go index ae56c5c07..cff91d0bf 100644 --- a/politeiad/backend/tlogbe/dcrdata.go +++ b/politeiad/backend/tlogbe/dcrdata.go @@ -14,7 +14,9 @@ import ( "strings" "sync" - v4 "github.com/decred/dcrdata/api/types/v4" + "github.com/decred/dcrd/chaincfg/v3" + jsonrpc "github.com/decred/dcrd/rpc/jsonrpc/types/v2" + v5 "github.com/decred/dcrdata/api/types/v5" exptypes "github.com/decred/dcrdata/explorer/types/v2" pstypes "github.com/decred/dcrdata/pubsub/types/v3" "github.com/decred/politeia/politeiad/backend" @@ -42,10 +44,13 @@ var ( // dcrdataplugin satisfies the pluginClient interface. type dcrdataPlugin struct { sync.Mutex - hostHTTP string // dcrdata HTTP host - hostWS string // dcrdata websocket host - client *http.Client // HTTP client - ws *wsdcrdata.Client // Websocket client + activeNetParams *chaincfg.Params + client *http.Client + ws *wsdcrdata.Client + + // Plugin settings + hostHTTP string // dcrdata HTTP host + hostWS string // dcrdata websocket host // bestBlock is the cached best block height. This field is kept up // to date by the websocket connection. If the websocket connection @@ -129,13 +134,13 @@ func (p *dcrdataPlugin) makeReq(method string, route string, v interface{}) ([]b } // bestBlockHTTP fetches and returns the best block from the dcrdata http API. -func (p *dcrdataPlugin) bestBlockHTTP() (*v4.BlockDataBasic, error) { +func (p *dcrdataPlugin) bestBlockHTTP() (*v5.BlockDataBasic, error) { resBody, err := p.makeReq(http.MethodGet, routeBestBlock, nil) if err != nil { return nil, err } - var bdb v4.BlockDataBasic + var bdb v5.BlockDataBasic err = json.Unmarshal(resBody, &bdb) if err != nil { return nil, err @@ -146,7 +151,7 @@ func (p *dcrdataPlugin) bestBlockHTTP() (*v4.BlockDataBasic, error) { // blockDetailsHTTP fetches and returns the block details from the dcrdata API // for the provided block height. -func (p *dcrdataPlugin) blockDetailsHTTP(height uint32) (*v4.BlockDataBasic, error) { +func (p *dcrdataPlugin) blockDetailsHTTP(height uint32) (*v5.BlockDataBasic, error) { h := strconv.FormatUint(uint64(height), 10) route := strings.Replace(routeBlockDetails, "{height}", h, 1) @@ -155,7 +160,7 @@ func (p *dcrdataPlugin) blockDetailsHTTP(height uint32) (*v4.BlockDataBasic, err return nil, err } - var bdb v4.BlockDataBasic + var bdb v5.BlockDataBasic err = json.Unmarshal(resBody, &bdb) if err != nil { return nil, err @@ -185,8 +190,8 @@ func (p *dcrdataPlugin) ticketPoolHTTP(blockHash string) ([]string, error) { // txsTrimmedHTTP fetches and returns the TrimmedTx from the dcrdata API for // the provided tx IDs. -func (p *dcrdataPlugin) txsTrimmedHTTP(txIDs []string) ([]v4.TrimmedTx, error) { - t := v4.Txns{ +func (p *dcrdataPlugin) txsTrimmedHTTP(txIDs []string) ([]v5.TrimmedTx, error) { + t := v5.Txns{ Transactions: txIDs, } resBody, err := p.makeReq(http.MethodPost, routeTxsTrimmed, t) @@ -194,7 +199,7 @@ func (p *dcrdataPlugin) txsTrimmedHTTP(txIDs []string) ([]v4.TrimmedTx, error) { return nil, err } - var txs []v4.TrimmedTx + var txs []v5.TrimmedTx err = json.Unmarshal(resBody, &txs) if err != nil { return nil, err @@ -266,6 +271,36 @@ func (p *dcrdataPlugin) cmdBestBlock(payload string) (string, error) { return string(reply), nil } +func convertTicketPoolInfoFromV5(t v5.TicketPoolInfo) dcrdata.TicketPoolInfo { + return dcrdata.TicketPoolInfo{ + Height: t.Height, + Size: t.Size, + Value: t.Value, + ValAvg: t.ValAvg, + Winners: t.Winners, + } +} + +func convertBlockDataBasicFromV5(b v5.BlockDataBasic) dcrdata.BlockDataBasic { + var poolInfo *dcrdata.TicketPoolInfo + if b.PoolInfo != nil { + p := convertTicketPoolInfoFromV5(*b.PoolInfo) + poolInfo = &p + } + return dcrdata.BlockDataBasic{ + Height: b.Height, + Size: b.Size, + Hash: b.Hash, + Difficulty: b.Difficulty, + StakeDiff: b.StakeDiff, + Time: b.Time.UNIX(), + NumTx: b.NumTx, + MiningFee: b.MiningFee, + TotalSent: b.TotalSent, + PoolInfo: poolInfo, + } +} + func (p *dcrdataPlugin) cmdBlockDetails(payload string) (string, error) { log.Tracef("dcrdata cmdBlockDetails: %v", payload) @@ -283,7 +318,7 @@ func (p *dcrdataPlugin) cmdBlockDetails(payload string) (string, error) { // Prepare reply bdr := dcrdata.BlockDetailsReply{ - Block: *bdb, + Block: convertBlockDataBasicFromV5(*bdb), } reply, err := dcrdata.EncodeBlockDetailsReply(bdr) if err != nil { @@ -320,6 +355,101 @@ func (p *dcrdataPlugin) cmdTicketPool(payload string) (string, error) { return string(reply), nil } +func convertScriptSigFromJSONRPC(s jsonrpc.ScriptSig) dcrdata.ScriptSig { + return dcrdata.ScriptSig{ + Asm: s.Asm, + Hex: s.Hex, + } +} + +func convertVinFromJSONRPC(v jsonrpc.Vin) dcrdata.Vin { + var scriptSig *dcrdata.ScriptSig + if v.ScriptSig != nil { + s := convertScriptSigFromJSONRPC(*v.ScriptSig) + scriptSig = &s + } + return dcrdata.Vin{ + Coinbase: v.Coinbase, + Stakebase: v.Stakebase, + Txid: v.Txid, + Vout: v.Vout, + Tree: v.Tree, + Sequence: v.Sequence, + AmountIn: v.AmountIn, + BlockHeight: v.BlockHeight, + BlockIndex: v.BlockIndex, + ScriptSig: scriptSig, + } +} + +func convertVinsFromV5(ins []jsonrpc.Vin) []dcrdata.Vin { + i := make([]dcrdata.Vin, 0, len(ins)) + for _, v := range ins { + i = append(i, convertVinFromJSONRPC(v)) + } + return i +} + +func convertScriptPubKeyFromV5(s v5.ScriptPubKey) dcrdata.ScriptPubKey { + return dcrdata.ScriptPubKey{ + Asm: s.Asm, + Hex: s.Hex, + ReqSigs: s.ReqSigs, + Type: s.Type, + Addresses: s.Addresses, + CommitAmt: s.CommitAmt, + } +} + +func convertTxInputIDFromV5(t v5.TxInputID) dcrdata.TxInputID { + return dcrdata.TxInputID{ + Hash: t.Hash, + Index: t.Index, + } +} + +func convertVoutFromV5(v v5.Vout) dcrdata.Vout { + var spend *dcrdata.TxInputID + if v.Spend != nil { + s := convertTxInputIDFromV5(*v.Spend) + spend = &s + } + return dcrdata.Vout{ + Value: v.Value, + N: v.N, + Version: v.Version, + ScriptPubKeyDecoded: convertScriptPubKeyFromV5(v.ScriptPubKeyDecoded), + Spend: spend, + } +} + +func convertVoutsFromV5(outs []v5.Vout) []dcrdata.Vout { + o := make([]dcrdata.Vout, 0, len(outs)) + for _, v := range outs { + o = append(o, convertVoutFromV5(v)) + } + return o +} + +func convertTrimmedTxFromV5(t v5.TrimmedTx) dcrdata.TrimmedTx { + return dcrdata.TrimmedTx{ + TxID: t.TxID, + Version: t.Version, + Locktime: t.Locktime, + Expiry: t.Expiry, + Vin: convertVinsFromV5(t.Vin), + Vout: convertVoutsFromV5(t.Vout), + } +} + +func convertTrimmedTxsFromV5(txs []v5.TrimmedTx) []dcrdata.TrimmedTx { + t := make([]dcrdata.TrimmedTx, 0, len(txs)) + for _, v := range txs { + t = append(t, convertTrimmedTxFromV5(v)) + } + return t +} + func (p *dcrdataPlugin) cmdTxsTrimmed(payload string) (string, error) { log.Tracef("cmdTxsTrimmed: %v", payload) @@ -337,7 +467,7 @@ func (p *dcrdataPlugin) cmdTxsTrimmed(payload string) (string, error) { // Prepare reply ttr := dcrdata.TxsTrimmedReply{ - Txs: txs, + Txs: convertTrimmedTxsFromV5(txs), } reply, err := dcrdata.EncodeTxsTrimmedReply(ttr) if err != nil { @@ -481,7 +611,7 @@ func (p *dcrdataPlugin) fsck() error { return nil } -func newDcrdataPlugin(settings []backend.PluginSetting) (*dcrdataPlugin, error) { +func newDcrdataPlugin(settings []backend.PluginSetting, activeNetParams *chaincfg.Params) (*dcrdataPlugin, error) { // Unpack plugin settings var ( hostHTTP string @@ -504,10 +634,24 @@ func newDcrdataPlugin(settings []backend.PluginSetting) (*dcrdataPlugin, error) // Set optional plugin settings to default values if a value was // not specified. if hostHTTP == "" { - hostHTTP = dcrdata.DefaultHostHTTP + switch activeNetParams.Name { + case chaincfg.MainNetParams().Name: + hostHTTP = dcrdata.DefaultHostHTTPMainNet + case chaincfg.TestNet3Params().Name: + hostHTTP = dcrdata.DefaultHostHTTPTestNet + default: + return nil, fmt.Errorf("unkown active net: %v", activeNetParams.Name) + } } if hostWS == "" { - hostWS = dcrdata.DefaultHostWS + switch activeNetParams.Name { + case chaincfg.MainNetParams().Name: + hostWS = dcrdata.DefaultHostWSMainNet + case chaincfg.TestNet3Params().Name: + hostWS = dcrdata.DefaultHostWSTestNet + default: + return nil, fmt.Errorf("unkown active net: %v", activeNetParams.Name) + } } // Setup http client @@ -526,9 +670,10 @@ func newDcrdataPlugin(settings []backend.PluginSetting) (*dcrdataPlugin, error) } return &dcrdataPlugin{ - hostHTTP: hostHTTP, - hostWS: hostWS, - client: client, - ws: ws, + activeNetParams: activeNetParams, + client: client, + ws: ws, + hostHTTP: hostHTTP, + hostWS: hostWS, }, nil } diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index 274cce9ee..1904480d1 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -69,25 +69,25 @@ var ( // ticketVotePlugin satisfies the pluginClient interface. type ticketVotePlugin struct { sync.Mutex - backend backend.Backend - tlog tlogClient + activeNetParams *chaincfg.Params + backend backend.Backend + tlog tlogClient // Plugin settings - activeNetParams *chaincfg.Params voteDurationMin uint32 // In blocks voteDurationMax uint32 // In blocks - // identity contains the full identity that the plugin uses to - // create receipts, i.e. signatures of user provided data that - // prove the backend received and processed a plugin command. - identity *identity.FullIdentity - // dataDir is the ticket vote plugin data directory. The only data // that is stored here is cached data that can be re-created at any // time by walking the trillian trees. Ex, the vote summary once a // record vote has ended. dataDir string + // identity contains the full identity that the plugin uses to + // create receipts, i.e. signatures of user provided data that + // prove the backend received and processed a plugin command. + identity *identity.FullIdentity + // inv contains the record inventory categorized by vote status. // The inventory will only contain public, non-abandoned records. // This cache is built on startup. @@ -2018,6 +2018,8 @@ func newTicketVotePlugin(backend backend.Backend, tlog tlogClient, settings []ba voteDurationMin = ticketvote.DefaultTestNetVoteDurationMin case chaincfg.SimNetParams().Name: voteDurationMin = ticketvote.DefaultSimNetVoteDurationMin + default: + return nil, fmt.Errorf("unkown active net: %v", activeNetParams.Name) } } if voteDurationMax == 0 { @@ -2028,6 +2030,8 @@ func newTicketVotePlugin(backend backend.Backend, tlog tlogClient, settings []ba voteDurationMax = ticketvote.DefaultTestNetVoteDurationMax case chaincfg.SimNetParams().Name: voteDurationMax = ticketvote.DefaultSimNetVoteDurationMax + default: + return nil, fmt.Errorf("unkown active net: %v", activeNetParams.Name) } } @@ -2039,13 +2043,13 @@ func newTicketVotePlugin(backend backend.Backend, tlog tlogClient, settings []ba } return &ticketVotePlugin{ - dataDir: dataDir, + activeNetParams: activeNetParams, backend: backend, tlog: tlog, - identity: id, - activeNetParams: activeNetParams, voteDurationMin: voteDurationMin, voteDurationMax: voteDurationMax, + dataDir: dataDir, + identity: id, votes: make(map[string]map[string]string), mutexes: make(map[string]*sync.Mutex), }, nil diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index d709e8a80..9eccb2d04 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -499,12 +499,9 @@ func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []ba } // Append record index and freeze record leaves to trillian tree. - // The queued leaves get added to the tree by the log signer FILO. - // Send the freeze record in first so that it gets appended to the - // tree last. leaves := []*trillian.LogLeaf{ - logLeafNew(freezeRecordHash, []byte(keyPrefixFreezeRecord+keys[1])), logLeafNew(idxHash, []byte(keyPrefixRecordIndex+keys[0])), + logLeafNew(freezeRecordHash, []byte(keyPrefixFreezeRecord+keys[1])), } queued, _, err := t.trillian.leavesAppend(treeID, leaves) if err != nil { diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index ba9c219b4..27e50015a 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -1477,7 +1477,7 @@ func (t *tlogBackend) RegisterPlugin(p backend.Plugin) error { return err } case dcrdata.ID: - client, err = newDcrdataPlugin(p.Settings) + client, err = newDcrdataPlugin(p.Settings, t.activeNetParams) if err != nil { return err } @@ -1718,11 +1718,11 @@ func New(anp *chaincfg.Params, homeDir, dataDir, dcrtimeHost, encryptionKeyFile, prefixes: make(map[string][]byte), vettedTreeIDs: make(map[string]int64), inventory: map[backend.MDStatusT][]string{ - backend.MDStatusUnvetted: make([]string, 0), - backend.MDStatusIterationUnvetted: make([]string, 0), - backend.MDStatusVetted: make([]string, 0), - backend.MDStatusCensored: make([]string, 0), - backend.MDStatusArchived: make([]string, 0), + backend.MDStatusUnvetted: make([]string, 0, 256), + backend.MDStatusIterationUnvetted: make([]string, 0, 256), + backend.MDStatusVetted: make([]string, 0, 256), + backend.MDStatusCensored: make([]string, 0, 256), + backend.MDStatusArchived: make([]string, 0, 256), }, } diff --git a/politeiad/plugins/dcrdata/dcrdata.go b/politeiad/plugins/dcrdata/dcrdata.go index f7c8917f3..da8cb5859 100644 --- a/politeiad/plugins/dcrdata/dcrdata.go +++ b/politeiad/plugins/dcrdata/dcrdata.go @@ -8,8 +8,6 @@ package dcrdata import ( "encoding/json" - - v4 "github.com/decred/dcrdata/api/types/v4" ) type StatusT int @@ -25,8 +23,10 @@ const ( CmdTxsTrimmed = "txstrimmed" // Get trimmed transactions // Default plugin settings - DefaultHostHTTP = "https://dcrdata.decred.org" - DefaultHostWS = "wss://dcrdata.decred.org/ps" + DefaultHostHTTPMainNet = "https://dcrdata.decred.org" + DefaultHostHTTPTestNet = "https://testnet.decred.org" + DefaultHostWSMainNet = "wss://dcrdata.decred.org/ps" + DefaultHostWSTestNet = "wss://testnet.decred.org/ps" // Dcrdata connection statuses. // @@ -81,6 +81,30 @@ func DecodeBestBlockReply(payload []byte) (*BestBlockReply, error) { return &bbr, nil } +// TicketPoolInfo models data about ticket pool. +type TicketPoolInfo struct { + Height uint32 `json:"height"` + Size uint32 `json:"size"` + Value float64 `json:"value"` + ValAvg float64 `json:"valavg"` + Winners []string `json:"winners"` +} + +// BlockDataBasic models primary information about a block. +type BlockDataBasic struct { + Height uint32 `json:"height"` + Size uint32 `json:"size"` + Hash string `json:"hash"` + Difficulty float64 `json:"diff"` + StakeDiff float64 `json:"sdiff"` + Time int64 `json:"time"` // UNIX timestamp + NumTx uint32 `json:"txlength"` + MiningFee *int64 `json:"fees,omitempty"` + TotalSent *int64 `json:"total_sent,omitempty"` + // TicketPoolInfo may be nil for side chain blocks. + PoolInfo *TicketPoolInfo `json:"ticket_pool,omitempty"` +} + // BlockDetails fetched the block details for the provided block height. type BlockDetails struct { Height uint32 `json:"height"` @@ -103,7 +127,7 @@ func DecodeBlockDetails(payload []byte) (*BlockDetails, error) { // BlockDetailsReply is the reply to the block details command. type BlockDetailsReply struct { - Block v4.BlockDataBasic `json:"block"` + Block BlockDataBasic `json:"block"` } // EncodeBlockDetailsReply encodes an BlockDetailsReply into a JSON byte slice. @@ -162,6 +186,65 @@ func DecodeTicketPoolReply(payload []byte) (*TicketPoolReply, error) { return &tpr, nil } +// ScriptSig models a signature script. It is defined separately since it only +// applies to non-coinbase. Therefore the field in the Vin structure needs to +// be a pointer. +type ScriptSig struct { + Asm string `json:"asm"` + Hex string `json:"hex"` +} + +// Vin models parts of the tx data. It is defined separately since +// getrawtransaction, decoderawtransaction, and searchrawtransaction use the +// same structure. +type Vin struct { + Coinbase string `json:"coinbase"` + Stakebase string `json:"stakebase"` + Txid string `json:"txid"` + Vout uint32 `json:"vout"` + Tree int8 `json:"tree"` + Sequence uint32 `json:"sequence"` + AmountIn float64 `json:"amountin"` + BlockHeight uint32 `json:"blockheight"` + BlockIndex uint32 `json:"blockindex"` + ScriptSig *ScriptSig `json:"scriptSig"` +} + +// ScriptPubKey is the script public key data. +type ScriptPubKey struct { + Asm string `json:"asm"` + Hex string `json:"hex"` + ReqSigs int32 `json:"reqSigs,omitempty"` + Type string `json:"type"` + Addresses []string `json:"addresses,omitempty"` + CommitAmt *float64 `json:"commitamt,omitempty"` +} + +// TxInputID specifies a transaction input as hash:vin_index. +type TxInputID struct { + Hash string `json:"hash"` + Index uint32 `json:"vin_index"` +} + +// Vout defines a transaction output. +type Vout struct { + Value float64 `json:"value"` + N uint32 `json:"n"` + Version uint16 `json:"version"` + ScriptPubKeyDecoded ScriptPubKey `json:"scriptPubKey"` + Spend *TxInputID `json:"spend,omitempty"` +} + +// TrimmedTx models data to resemble to result of the decoderawtransaction RPC. +type TrimmedTx struct { + TxID string `json:"txid"` + Version int32 `json:"version"` + Locktime uint32 `json:"locktime"` + Expiry uint32 `json:"expiry"` + Vin []Vin `json:"vin"` + Vout []Vout `json:"vout"` +} + // TxsTrimmed requests the trimmed transaction information for the provided // transaction IDs. type TxsTrimmed struct { @@ -185,7 +268,7 @@ func DecodeTxsTrimmed(payload []byte) (*TxsTrimmed, error) { // TxsTrimmedReply is the reply to the TxsTrimmed command. type TxsTrimmedReply struct { - Txs []v4.TrimmedTx `json:"txs"` + Txs []TrimmedTx `json:"txs"` } // EncodeTxsTrimmedReply encodes an TxsTrimmedReply into a JSON byte slice. diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 896a22893..dbc2a725e 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -1253,14 +1253,18 @@ func _main() error { return fmt.Errorf("failed to parse plugin setting '%v'; format "+ "should be 'pluginID,key,value'", s) } - pluginID := s[0] + var ( + pluginID = s[0] + key = s[1] + value = s[2] + ) ps, ok := settings[pluginID] if !ok { ps = make([]backend.PluginSetting, 0, 16) } ps = append(ps, backend.PluginSetting{ - Key: s[1], - Value: s[2], + Key: key, + Value: value, }) settings[pluginID] = ps diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index 6799d12cd..51de18616 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -32,6 +32,7 @@ var ( ) type piwww struct { + // This is here to prevent parsing errors caused by config flags. Config shared.Config // Basic commands @@ -197,7 +198,7 @@ func _main() error { fmt.Printf("%v\n", helpMsg) return nil } - return fmt.Errorf("parse help flags: %v", err) + return fmt.Errorf("parse help flag: %v", err) } // Get politeiawww CSRF token @@ -209,9 +210,7 @@ func _main() error { } // Parse subcommand and execute - parser = flags.NewParser(&piwww{ - Config: *cfg, - }, flags.Default) + parser = flags.NewParser(&piwww{Config: *cfg}, flags.Default) _, err = parser.Parse() if err != nil { os.Exit(1) diff --git a/politeiawww/cmd/piwww/sendfaucettx.go b/politeiawww/cmd/piwww/sendfaucettx.go index 07ca668ce..0f83119d4 100644 --- a/politeiawww/cmd/piwww/sendfaucettx.go +++ b/politeiawww/cmd/piwww/sendfaucettx.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Decred developers +// Copyright (c) 2017-2020 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -15,9 +15,9 @@ import ( // of DCR (in atoms) to the specified address. type sendFaucetTxCmd struct { Args struct { - Address string `positional-arg-name:"address" required:"true"` // DCR address - Amount uint64 `positional-arg-name:"amount" required:"true"` // Amount in atoms - OverrideToken string `positional-arg-name:"overridetoken"` // Faucet override token + Address string `positional-arg-name:"address" required:"true"` + Amount uint64 `positional-arg-name:"amount" required:"true"` + OverrideToken string `positional-arg-name:"overridetoken"` } `positional-args:"true"` } @@ -27,11 +27,6 @@ func (cmd *sendFaucetTxCmd) Execute(args []string) error { atoms := cmd.Args.Amount dcr := float64(atoms) / 1e8 - if address == "" && atoms == 0 { - return fmt.Errorf("Invalid arguments. Unable to pay %v DCR to %v", - dcr, address) - } - txID, err := util.PayWithTestnetFaucet(context.Background(), cfg.FaucetHost, address, atoms, cmd.Args.OverrideToken) if err != nil { @@ -45,15 +40,13 @@ func (cmd *sendFaucetTxCmd) Execute(args []string) error { fmt.Printf(`{"txid":"%v"}`, txID) fmt.Printf("\n") default: - fmt.Printf("Paid %v DCR to %v with txID %v\n", - dcr, address, txID) + fmt.Printf("Paid %v DCR to %v with tx %v\n", dcr, address, txID) } return nil } -// sendFaucetTxHelpMsg is the output for the help command when 'sendfaucettx' -// is specified. +// sendFaucetTxHelpMsg is the help command message. const sendFaucetTxHelpMsg = `sendfaucettx "address" "amount" "overridetoken" Use the Decred testnet faucet to send DCR (in atoms) to an address. One atom is @@ -61,8 +54,6 @@ one hundred millionth of a single DCR (0.00000001 DCR). Arguments: 1. address (string, required) Receiving address -2. amount (uint64, required) Amount to send (atoms) +2. amount (uint64, required) Amount to send (in atoms) 3. overridetoken (string, optional) Override token for testnet faucet - -Result: -Paid [amount] DCR to [address] with txID [transaction id]` +` diff --git a/politeiawww/cmd/shared/shared.go b/politeiawww/cmd/shared/shared.go index 3b2279d95..d0262bb95 100644 --- a/politeiawww/cmd/shared/shared.go +++ b/politeiawww/cmd/shared/shared.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Decred developers +// Copyright (c) 2017-2020 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -26,8 +26,8 @@ var ( // ErrUserIdentityNotFound is emitted when a user identity is // required but the config object does not contain one. ErrUserIdentityNotFound = errors.New("user identity not found; " + - "you must either create a new user or use the updateuserkey " + - "command to generate a new identity for the logged in user") + "you must either create a new user or update the user key to " + + "generate a new identity for the logged in user") ) // PrintJSON prints the passed in JSON using the style specified by the global From 0aa6b81328e3338906925b19c8fd4418965af820 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 14 Oct 2020 17:42:44 -0500 Subject: [PATCH 153/449] various cleanups and bug fixes --- politeia.md | 6 +-- politeiad/backend/backend.go | 3 +- politeiad/backend/tlogbe/ticketvote.go | 11 ++-- politeiad/politeiad.go | 6 +-- politeiawww/api/pi/v1/v1.go | 2 + politeiawww/cmd/cmswww/activevotes.go | 2 +- politeiawww/cmd/piwww/proposals.go | 8 +-- politeiawww/cmd/piwww/votes.go | 66 ++++++++++++------------ politeiawww/cmd/piwww/votestartrunoff.go | 2 +- politeiawww/cmd/piwww/votesummaries.go | 42 ++++++++++----- politeiawww/cmd/shared/client.go | 4 +- politeiawww/cmd/shared/config.go | 6 +-- politeiawww/piwww.go | 1 + politeiawww/www.go | 2 + 14 files changed, 94 insertions(+), 67 deletions(-) diff --git a/politeia.md b/politeia.md index 34cf7fd56..aa2fbc972 100644 --- a/politeia.md +++ b/politeia.md @@ -111,7 +111,7 @@ the admins. * The StatusReply returns interesting statistics such as: number of proposals in memory, number of comments in memory etc. 3. Add refclient unit tests that validate all 3 conditions. -4. Add RPC to politeiawwwcli so that the status calls can be scripted. +4. Add RPC to piwww so that the status calls can be scripted. ``` ### Who @@ -139,7 +139,7 @@ votes should be less than a week. 2. 8 hours to add the call, determine what status to set when and figure out what statistics to return. 3. 4 hours to add refclient validation tests. -4. 2 hours to add RPC to politeiawwwcli +4. 2 hours to add RPC to piwww In addition allow for 1 hour of overhead (going back and forth on slack/github etc). This will bring the grand total to 17 hours at a rate of $40/h. This @@ -157,7 +157,7 @@ Week 1 deliverables Week 2 deliverables 1. Implement RPC 2. Implement validation tests -3. Implement politeiawwwcli +3. Implement piwww 15 hours, to be completed on August 29 2018 ``` diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index 012a02e63..6beb2551b 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -117,7 +117,8 @@ type PluginUserError struct { // Error satisfies the error interface. func (e PluginUserError) Error() string { - return fmt.Sprintf("plugin error code: %v", e.ErrorCode) + return fmt.Sprintf("plugin %v error code: %v %v", + e.PluginID, e.ErrorCode, e.ErrorContext) } // RecordMetadata is the metadata of a record. diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index 1904480d1..70e61cf5c 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -66,6 +66,8 @@ var ( // TODO the bottleneck for casting a large ballot of votes is waiting for the // log signer. Break the cast votes up and send them concurrently. +// TODO should start and startrunoff be combined into a single command? + // ticketVotePlugin satisfies the pluginClient interface. type ticketVotePlugin struct { sync.Mutex @@ -958,13 +960,16 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { ErrorContext: []string{"no prev action; action must be authorize"}, } } - case prevAction == ticketvote.ActionAuthorize: + case prevAction == ticketvote.ActionAuthorize && + a.Action != ticketvote.ActionRevoke: // Previous action was a authorize. This action must be revoke. return "", backend.PluginUserError{ + PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusAuthorizationInvalid), ErrorContext: []string{"prev action was authorize"}, } - case prevAction == ticketvote.ActionRevoke: + case prevAction == ticketvote.ActionRevoke && + a.Action != ticketvote.ActionAuthorize: // Previous action was a revoke. This action must be authorize. return "", backend.PluginUserError{ PluginID: ticketvote.ID, @@ -1672,7 +1677,7 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. for _, voteBit := range votes { tally[voteBit]++ } - results := make([]ticketvote.Result, len(vd.Params.Options)) + results := make([]ticketvote.Result, 0, len(vd.Params.Options)) for _, v := range vd.Params.Options { bit := strconv.FormatUint(v.Bit, 16) results = append(results, ticketvote.Result{ diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index dbc2a725e..de2d04e14 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -1037,9 +1037,9 @@ func (p *politeia) pluginCommand(w http.ResponseWriter, r *http.Request) { // Generic internal error. errorCode := time.Now().Unix() - log.Errorf("%v %v: backend plugin failed with "+ - "command:%v payload:%v err:%v", remoteAddr(r), - errorCode, pc.Command, pc.Payload, err) + log.Errorf("%v %v: backend plugin failed: pluginID:%v command:%v "+ + "payload:%v err:%v", remoteAddr(r), errorCode, pc.ID, pc.Command, + pc.Payload, err) p.respondWithServerError(w, errorCode) return } diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 22b00e08a..38bcce305 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -25,6 +25,7 @@ type VoteErrorT int // TODO make RouteVoteResults a batched route but that only currently allows // for 1 result to be returned so that we have the option to change this is // we want to. +// TODO should we add auths to the vote summary? const ( APIVersion = 1 @@ -170,6 +171,7 @@ const ( ErrorStatusCommentVoteChangesMax // Vote errors + ErrorStatusVoteAuthInvalid ErrorStatusVoteStatusInvalid ErrorStatusVoteParamsInvalid ErrorStatusBallotInvalid diff --git a/politeiawww/cmd/cmswww/activevotes.go b/politeiawww/cmd/cmswww/activevotes.go index 325a1bda2..3567fface 100644 --- a/politeiawww/cmd/cmswww/activevotes.go +++ b/politeiawww/cmd/cmswww/activevotes.go @@ -80,7 +80,7 @@ Result: "startblockhash": (string) Hash of first block of vote interval "endheight": (string) Block height at end of vote "eligibletickets": [ - "removed by politeiawwwcli for readability" + "removed by cmswww for readability" ] } ] diff --git a/politeiawww/cmd/piwww/proposals.go b/politeiawww/cmd/piwww/proposals.go index 52b87bae3..97e0393e5 100644 --- a/politeiawww/cmd/piwww/proposals.go +++ b/politeiawww/cmd/piwww/proposals.go @@ -27,14 +27,14 @@ type proposalsCmd struct { } // Execute executes the proposals command. -func (cmd *proposalsCmd) Execute(args []string) error { - proposals := cmd.Args.Proposals +func (c *proposalsCmd) Execute(args []string) error { + proposals := c.Args.Proposals // Set state to get unvetted or vetted proposals. Defaults // to vetted unless the unvetted flag is used. var state pi.PropStateT switch { - case cmd.Unvetted: + case c.Unvetted: state = pi.PropStateUnvetted default: state = pi.PropStateVetted @@ -66,7 +66,7 @@ func (cmd *proposalsCmd) Execute(args []string) error { p := pi.Proposals{ State: state, Requests: requests, - IncludeFiles: cmd.IncludeFiles, + IncludeFiles: c.IncludeFiles, } // Send request diff --git a/politeiawww/cmd/piwww/votes.go b/politeiawww/cmd/piwww/votes.go index a91ff55e0..144ffccce 100644 --- a/politeiawww/cmd/piwww/votes.go +++ b/politeiawww/cmd/piwww/votes.go @@ -5,63 +5,63 @@ package main import ( - "fmt" - pi "github.com/decred/politeia/politeiawww/api/pi/v1" "github.com/decred/politeia/politeiawww/cmd/shared" ) -// votesCmd retrieves vote details for a proposal, tallies the votes, and -// displays the result. +// votesCmd retrieves vote details for the specified proposals. type votesCmd struct { Args struct { - Token string `positional-arg-name:"token"` + Tokens []string `positional-arg-name:"token"` } `positional-args:"true" required:"true"` } -// Execute executes the tally command. -func (cmd *votesCmd) Execute(args []string) error { - token := cmd.Args.Token - - // Prep request payload +// Execute executes the votes command. +func (c *votesCmd) Execute(args []string) error { + // Setup request v := pi.Votes{ - Tokens: []string{token}, + Tokens: c.Args.Tokens, } - // Print request details + // Send request. The request and response details are printed to + // the console. err := shared.PrintJSON(v) if err != nil { return err } - - // Get vote details for proposal - vrr, err := client.Votes(v) + vr, err := client.Votes(v) if err != nil { - return fmt.Errorf("ProposalVotes: %v", err) + return err } - - // Remove eligible tickets snapshot from response - // so that the output is legible - var ( - pv pi.ProposalVote - ok bool - ) - if pv, ok = vrr.Votes[token]; ok && !cfg.RawJSON { - pv.Vote.EligibleTickets = []string{ - "removed by politeiawwwcli for readability", + if !cfg.RawJSON { + // Remove the eligible ticket pool from the response for + // readability. + for k, v := range vr.Votes { + if v.Vote == nil { + continue + } + v.Vote.EligibleTickets = []string{ + "removed by piwww for readability", + } + vr.Votes[k] = v } - vrr.Votes[token] = pv + } + err = shared.PrintJSON(vr) + if err != nil { + return err } - // Print response details - return shared.PrintJSON(vrr) + return nil } -// votesHelpMsg is the output for the help command when 'votes' is specified. -const votesHelpMsg = `votes "token" +// votesHelpMsg is the help command message. +const votesHelpMsg = `votes "tokens" -Fetch the vote details for a proposal. +Fetch the vote details for the provided proposal tokens. Arguments: -1. token (string, required) Proposal censorship token +1. tokens (string, required) Proposal censorship tokens + +Example usage: +$ piwww votes cda97ace0a4765140000 71dd3a110500fb6a0000 ` diff --git a/politeiawww/cmd/piwww/votestartrunoff.go b/politeiawww/cmd/piwww/votestartrunoff.go index b92d05cf8..966445c06 100644 --- a/politeiawww/cmd/piwww/votestartrunoff.go +++ b/politeiawww/cmd/piwww/votestartrunoff.go @@ -160,7 +160,7 @@ func (cmd *voteStartRunoffCmd) Execute(args []string) error { // Print response details. Remove ticket snapshot from // the response before printing so that the output is // legible. - m := "removed by politeiawwwcli for readability" + m := "removed by piwww for readability" svrr.EligibleTickets = []string{m} err = shared.PrintJSON(svrr) if err != nil { diff --git a/politeiawww/cmd/piwww/votesummaries.go b/politeiawww/cmd/piwww/votesummaries.go index 5562c0a5c..7dc62faa2 100644 --- a/politeiawww/cmd/piwww/votesummaries.go +++ b/politeiawww/cmd/piwww/votesummaries.go @@ -9,27 +9,43 @@ import ( "github.com/decred/politeia/politeiawww/cmd/shared" ) -// voteSummariesCmd retrieves a set of proposal vote summaries. -type voteSummariesCmd struct{} +// voteSummariesCmd retrieves the vote summaries for the provided proposals. +type voteSummariesCmd struct { + Args struct { + Tokens []string `positional-arg-name:"token"` + } `positional-args:"true" required:"true"` +} -// Execute executes the batch vote summaries command. +// Execute executes the vote summaries command. func (cmd *voteSummariesCmd) Execute(args []string) error { - bpr, err := client.VoteSummaries(&pi.VoteSummaries{ - Tokens: args, - }) + // Setup request + vs := pi.VoteSummaries{ + Tokens: cmd.Args.Tokens, + } + + // Send request. The request and response details are printed to + // the console. + err := shared.PrintJSON(vs) + if err != nil { + return err + } + vsr, err := client.VoteSummaries(vs) + if err != nil { + return err + } + err = shared.PrintJSON(vsr) if err != nil { return err } - return shared.PrintJSON(bpr) + return nil } -// voteSummariesHelpMsg is the output for the help command when -// 'votesummaries' is specified. -const voteSummariesHelpMsg = `votesummaries +// voteSummariesHelpMsg is the help command message. +const voteSummariesHelpMsg = `votesummaries "tokens" -Fetch a summary of the voting process for a list of proposals. +Fetch the vote summaries for the provided proposal tokens. -Example: -votesummaries token1 token2 +Example usage: +$ piww votesummaries cda97ace0a4765140000 71dd3a110500fb6a0000 ` diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index 1c1500866..47fddf835 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -1317,7 +1317,7 @@ func (c *Client) BatchProposals(bp *www.BatchProposals) (*www.BatchProposalsRepl // VoteSummaries retrieves a summary of the voting process for a set of // proposals. -func (c *Client) VoteSummaries(vs *pi.VoteSummaries) (*pi.VoteSummariesReply, error) { +func (c *Client) VoteSummaries(vs pi.VoteSummaries) (*pi.VoteSummariesReply, error) { statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, pi.RouteVoteSummaries, vs) if err != nil { @@ -1331,7 +1331,7 @@ func (c *Client) VoteSummaries(vs *pi.VoteSummaries) (*pi.VoteSummariesReply, er var vsr pi.VoteSummariesReply err = json.Unmarshal(respBody, &vsr) if err != nil { - return nil, fmt.Errorf("unmarshal BatchVoteSummary: %v", err) + return nil, err } if c.cfg.Verbose { diff --git a/politeiawww/cmd/shared/config.go b/politeiawww/cmd/shared/config.go index 912963c3e..d8381b5a7 100644 --- a/politeiawww/cmd/shared/config.go +++ b/politeiawww/cmd/shared/config.go @@ -70,7 +70,7 @@ type Config struct { // 3) Load configuration file overwriting defaults with any specified options // 4) Parse CLI options and overwrite/add any specified options // -// The above results in politeiawwwcli functioning properly without any config +// The above results in the cli functioning properly without any config // settings while still allowing the user to override settings with config // files and command line options. Command line options always take precedence. func LoadConfig(homeDir, dataDirname, configFilename string) (*Config, error) { @@ -186,8 +186,8 @@ func LoadConfig(homeDir, dataDirname, configFilename string) (*Config, error) { } // hostFilePath returns the host specific file path for the passed in file. -// This means that the hostname is prepended to the filename. politeiawwwcli -// data is segmented by host so that we can interact with multiple hosts +// This means that the hostname is prepended to the filename. cli data is +// segmented by host so that we can interact with multiple hosts // simultaneously. func (cfg *Config) hostFilePath(filename string) (string, error) { u, err := url.Parse(cfg.Host) diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 4661b0f0e..c3c2e8d88 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -672,6 +672,7 @@ func convertVoteSummaryFromPlugin(s ticketvote.Summary) pi.VoteSummary { EndBlockHeight: s.EndBlockHeight, EligibleTickets: s.EligibleTickets, QuorumPercentage: s.QuorumPercentage, + PassPercentage: s.PassPercentage, Results: results, Approved: s.Approved, } diff --git a/politeiawww/www.go b/politeiawww/www.go index 1181cf5f4..a1173e601 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -250,6 +250,8 @@ func convertPiErrorStatusFromTicketVote(e ticketvote.ErrorStatusT) pi.ErrorStatu return pi.ErrorStatusPropNotFound case ticketvote.ErrorStatusRecordStatusInvalid: return pi.ErrorStatusPropStatusInvalid + case ticketvote.ErrorStatusAuthorizationInvalid: + return pi.ErrorStatusVoteAuthInvalid case ticketvote.ErrorStatusVoteParamsInvalid: return pi.ErrorStatusVoteParamsInvalid case ticketvote.ErrorStatusVoteStatusInvalid: From 681e941459fde3b147b4b1346a61e3392d0bac7c Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 14 Oct 2020 19:31:14 -0500 Subject: [PATCH 154/449] change process for freezing a tree --- politeiad/backend/tlogbe/tlog.go | 281 ++++++++------------------- politeiad/backend/tlogbe/tlogbe.go | 108 +++++----- politeiad/backend/tlogbe/trillian.go | 7 + 3 files changed, 130 insertions(+), 266 deletions(-) diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index 9eccb2d04..6cb415c12 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -97,8 +97,8 @@ type tlog struct { droppingAnchor bool } -// recordIndex contains the merkle leaf hashes of all the record content for a -// specific record version and iteration. The record index can be used to +// recordIndex contains the merkle leaf hashes of all the record content leaves +// for a specific record version and iteration. The record index can be used to // lookup the trillian log leaves for the record content and the log leaves can // be used to lookup the kv store blobs. // @@ -147,11 +147,23 @@ type recordIndex struct { RecordMetadata []byte `json:"recordmetadata"` Metadata map[uint64][]byte `json:"metadata"` // [metadataID]merkle Files map[string][]byte `json:"files"` // [filename]merkle -} -// TODO add comment explaining what a freeze record is -type freezeRecord struct { - TreeID int64 `json:"treeid,omitempty"` + // Frozen is used to indicate that the tree for this record has + // been frozen. This happens as a result of certain record status + // changes. The only thing that can be appended onto a frozen tree + // is one additional anchor record. Once a frozen tree has been + // anchored, the tlog fsck function will update the status of the + // tree to frozen in trillian, at which point trillian will not + // allow any additional leaves to be appended onto the tree. + Frozen bool `json:"frozen,omitempty"` + + // TreePointer is the tree ID of the tree that is the new location + // of this record. A record can be copied to a new tree after + // certain status changes, such as when a record is made public and + // the record is copied from an unvetted tree to a vetted tree. + // TreePointer should only be set if the existing tree has been + // frozen. + TreePointer int64 `json:"treepointer,omitempty"` } func treeIDFromToken(token []byte) int64 { @@ -172,18 +184,6 @@ func blobIsEncrypted(b []byte) bool { return bytes.HasPrefix(b, []byte("sbox")) } -// treeIsFrozen deterimes if the tree is frozen given the full list of the -// tree's leaves. A tree is considered frozen if the tree contains a freeze -// record leaf. -func treeIsFrozen(leaves []*trillian.LogLeaf) bool { - for i := 0; i < len(leaves); i++ { - if leafIsFreezeRecord(leaves[i]) { - return true - } - } - return false -} - func leafIsRecordIndex(l *trillian.LogLeaf) bool { return bytes.HasPrefix(l.ExtraData, []byte(keyPrefixRecordIndex)) } @@ -192,10 +192,6 @@ func leafIsRecordContent(l *trillian.LogLeaf) bool { return bytes.HasPrefix(l.ExtraData, []byte(keyPrefixRecordContent)) } -func leafIsFreezeRecord(l *trillian.LogLeaf) bool { - return bytes.HasPrefix(l.ExtraData, []byte(keyPrefixFreezeRecord)) -} - func leafIsAnchor(l *trillian.LogLeaf) bool { return bytes.HasPrefix(l.ExtraData, []byte(keyPrefixAnchorRecord)) } @@ -276,23 +272,6 @@ func convertBlobEntryFromRecordIndex(ri recordIndex) (*store.BlobEntry, error) { return &be, nil } -func convertBlobEntryFromFreezeRecord(fr freezeRecord) (*store.BlobEntry, error) { - data, err := json.Marshal(fr) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorFreezeRecord, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - func convertBlobEntryFromAnchor(a anchor) (*store.BlobEntry, error) { data, err := json.Marshal(a) if err != nil { @@ -310,44 +289,6 @@ func convertBlobEntryFromAnchor(a anchor) (*store.BlobEntry, error) { return &be, nil } -func convertFreezeRecordFromBlobEntry(be store.BlobEntry) (*freezeRecord, error) { - // Decode and validate data hint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorFreezeRecord { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorFreezeRecord) - } - - // Decode data - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - hash, err := hex.DecodeString(be.Hash) - if err != nil { - return nil, fmt.Errorf("decode hash: %v", err) - } - if !bytes.Equal(util.Digest(b), hash) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), hash) - } - var fr freezeRecord - err = json.Unmarshal(b, &fr) - if err != nil { - return nil, fmt.Errorf("unmarshal freezeRecord: %v", err) - } - - return &fr, nil -} - func convertRecordIndexFromBlobEntry(be store.BlobEntry) (*recordIndex, error) { // Decode and validate data hint b, err := base64.StdEncoding.DecodeString(be.DataHint) @@ -443,13 +384,18 @@ func (t *tlog) treeExists(treeID int64) bool { } // treeFreeze updates the status of a record and freezes the trillian tree as a -// result of a record status change. Once a freeze record has been appended -// onto the tree the tlog backend considers the tree to be frozen. The only -// thing that can be appended onto a frozen tree is one additional anchor -// record. Once a frozen tree has been anchored, the tlog fsck function will -// update the status of the tree to frozen in trillian, at which point trillian -// will not allow any additional leaves to be appended onto the tree. -func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, fr freezeRecord) error { +// result of a record status change. The tree pointer is the tree ID of the new +// location of the record. This is provided on certain status changes such as +// when a unvetted record is make public and the unvetted record is moved to a +// vetted tree. A value of 0 indicates that no tree pointer exists. +// +// Once the record index has been saved with its frozen field set, the tree +// is considered to be frozen. The only thing that can be appended onto a +// frozen tree is one additional anchor record. Once a frozen tree has been +// anchored, the tlog fsck function will update the status of the tree to +// frozen in trillian, at which point trillian will not allow any additional +// leaves to be appended onto the tree. +func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, treePointer int64) error { log.Tracef("%v treeFreeze: %v", t.id, treeID) // Save metadata @@ -458,8 +404,11 @@ func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []ba return err } + // Update the record index + idx.Frozen = true + idx.TreePointer = treePointer + // Blobify the record index - blobs := make([][]byte, 0, 2) be, err := convertBlobEntryFromRecordIndex(*idx) if err != nil { return err @@ -472,43 +421,26 @@ func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []ba if err != nil { return err } - blobs = append(blobs, b) - // Blobify the freeze record - be, err = convertBlobEntryFromFreezeRecord(fr) - if err != nil { - return err - } - freezeRecordHash, err := hex.DecodeString(be.Hash) - if err != nil { - return err - } - b, err = store.Blobify(*be) - if err != nil { - return err - } - blobs = append(blobs, b) - - // Save blobs to the kv store - keys, err := t.store.Put(blobs) + // Save record index blob to the kv store + keys, err := t.store.Put([][]byte{b}) if err != nil { return fmt.Errorf("store Put: %v", err) } - if len(keys) != 2 { + if len(keys) != 1 { return fmt.Errorf("wrong number of keys: got %v, want 1", len(keys)) } - // Append record index and freeze record leaves to trillian tree. + // Append record index leaf to the trillian tree leaves := []*trillian.LogLeaf{ logLeafNew(idxHash, []byte(keyPrefixRecordIndex+keys[0])), - logLeafNew(freezeRecordHash, []byte(keyPrefixFreezeRecord+keys[1])), } queued, _, err := t.trillian.leavesAppend(treeID, leaves) if err != nil { return fmt.Errorf("leavesAppend: %v", err) } - if len(queued) != 2 { - return fmt.Errorf("wrong number of queud leaves: got %v, want 2", + if len(queued) != 1 { + return fmt.Errorf("wrong number of queud leaves: got %v, want 1", len(queued)) } failed := make([]string, 0, len(queued)) @@ -525,80 +457,18 @@ func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []ba return nil } -// freeze record returns the freeze record of the provided tree if one exists. -// If one does not exists a errFreezeRecordNotFound error is returned. -func (t *tlog) freezeRecord(treeID int64) (*freezeRecord, error) { - log.Tracef("%v freezeRecord: %v", t.id, treeID) - - // Check if the tree contains a freeze record. The last two leaves - // are checked because the last leaf will be the final anchor drop, - // which may not exist yet if the tree was recently frozen. - tree, err := t.trillian.tree(treeID) - if err != nil { - return nil, errRecordNotFound - } - _, lr, err := t.trillian.signedLogRootForTree(tree) - if err != nil { - return nil, fmt.Errorf("signedLogRootForTree: %v", err) - } - - var startIndex, count int64 - switch lr.TreeSize { - case 0: - return nil, errFreezeRecordNotFound - case 1: - startIndex = 0 - count = 1 - default: - startIndex = int64(lr.TreeSize) - 2 - count = 2 - } - - leaves, err := t.trillian.leavesByRange(treeID, startIndex, count) - if err != nil { - return nil, fmt.Errorf("leavesByRange: %v", err) - } - var l *trillian.LogLeaf - for _, v := range leaves { - if leafIsFreezeRecord(v) { - l = v - break - } - } - if l == nil { - // No freeze record was found - return nil, errFreezeRecordNotFound - } - - // A freeze record was found. Pull it from the store. - k, err := extractKeyFromLeaf(l) - if err != nil { - return nil, err - } - blobs, err := t.store.Get([]string{k}) - if err != nil { - return nil, fmt.Errorf("store Get: %v", err) - } - if len(blobs) != 1 { - return nil, fmt.Errorf("unexpected blobs count: got %v, want 1", - len(blobs)) - } - - // Decode freeze record - b, ok := blobs[k] - if !ok { - return nil, fmt.Errorf("blob not found %v", k) - } - be, err := store.Deblob(b) +// treeIsFrozen returns whether the provided tree is frozen and the tree +// pointer if one exists. +func (t *tlog) treeIsFrozen(treeID int64) (bool, int64, error) { + leavesAll, err := t.trillian.leavesAll(treeID) if err != nil { - return nil, err + return false, 0, fmt.Errorf("leavesAll: %v", err) } - fr, err := convertFreezeRecordFromBlobEntry(*be) + idx, err := t.recordIndexLatest(leavesAll) if err != nil { - return nil, err + return false, 0, err } - - return fr, nil + return idx.Frozen, idx.TreePointer, nil } func (t *tlog) recordIndexSave(treeID int64, ri recordIndex) error { @@ -801,11 +671,6 @@ type recordBlobsPrepareReply struct { // // TODO test this function func recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File, encryptionKey *encryptionKey) (*recordBlobsPrepareReply, error) { - // Verify tree state - if treeIsFrozen(leavesAll) { - return nil, errTreeIsFrozen - } - // Verify there are no duplicate or empty mdstream IDs mdstreamIDs := make(map[uint64]struct{}, len(metadata)) for _, v := range metadata { @@ -1094,11 +959,6 @@ func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []ba return fmt.Errorf("leavesAll %v: %v", treeID, err) } - // Verify tree state - if treeIsFrozen(leavesAll) { - return errTreeIsFrozen - } - // Get the existing record index currIdx, err := t.recordIndexLatest(leavesAll) if errors.Is(err, errRecordNotFound) { @@ -1111,6 +971,11 @@ func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []ba return fmt.Errorf("recordIndexLatest: %v", err) } + // Verify tree state + if currIdx.Frozen { + return errTreeIsFrozen + } + // Prepare kv store blobs bpr, err := recordBlobsPrepare(leavesAll, rm, metadata, files, t.encryptionKey) @@ -1217,7 +1082,11 @@ func (t *tlog) metadataSave(treeID int64, rm backend.RecordMetadata, metadata [] } // Verify tree state - if treeIsFrozen(leavesAll) { + currIdx, err := t.recordIndexLatest(leavesAll) + if err != nil { + return nil, err + } + if currIdx.Frozen { return nil, errTreeIsFrozen } @@ -1308,34 +1177,38 @@ func (t *tlog) recordDel(treeID int64) error { } // Get all tree leaves - leaves, err := t.trillian.leavesAll(treeID) + leavesAll, err := t.trillian.leavesAll(treeID) if err != nil { return err } // Ensure tree is frozen. Deleting files from the store is only // allowed on frozen trees. - if !treeIsFrozen(leaves) { + currIdx, err := t.recordIndexLatest(leavesAll) + if err != nil { + return err + } + if !currIdx.Frozen { return errTreeIsNotFrozen } // Retrieve all the record indexes - indexes, err := t.recordIndexes(leaves) + indexes, err := t.recordIndexes(leavesAll) if err != nil { - return fmt.Errorf("recordIndexes: %v", err) + return err } // Aggregate the keys for all file blobs of all versions. The // record index points to the log leaf merkle leaf hash. The log // leaf contains the kv store key. - merkles := make(map[string]struct{}, len(leaves)) + merkles := make(map[string]struct{}, len(leavesAll)) for _, v := range indexes { for _, merkle := range v.Files { merkles[hex.EncodeToString(merkle)] = struct{}{} } } keys := make([]string, 0, len(merkles)) - for _, v := range leaves { + for _, v := range leavesAll { _, ok := merkles[hex.EncodeToString(v.MerkleLeafHash)] if ok { key, err := extractKeyFromLeaf(v) @@ -1545,17 +1418,17 @@ func (t *tlog) blobsSave(treeID int64, keyPrefix string, blobs, hashes [][]byte, } // Verify tree is not frozen - _, err := t.freezeRecord(treeID) - switch err { - case nil: - // Freeze record was found. Tree is frozen. - return nil, errTreeIsFrozen - case errFreezeRecordNotFound: - // Tree is not frozen; continue - default: - // All other errors + leavesAll, err := t.trillian.leavesAll(treeID) + if err != nil { + return nil, fmt.Errorf("leavesAll: %v", err) + } + idx, err := t.recordIndexLatest(leavesAll) + if err != nil { return nil, err } + if idx.Frozen { + return nil, errTreeIsFrozen + } // Encrypt blobs if specified if encrypt { diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 27e50015a..a943a43c9 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -98,19 +98,21 @@ type tlogBackend struct { // record token, where n is defined by the TokenPrefixLength from // the politeiad API. Record lookups by token prefix are allowed. // This cache is used to prevent prefix collisions when creating - // new tokens and to facilitate lookups by token prefix. + // new tokens and to facilitate lookups by token prefix. This cache + // is built on startup. prefixes map[string][]byte // [tokenPrefix]token // vettedTreeIDs contains the token to tree ID mapping for vetted // records. The token corresponds to the unvetted tree ID so // unvetted lookups can be done directly, but vetted lookups - // required pulling the freeze record from the unvetted tree to - // get the vetted tree ID. This cache memoizes these results. + // required pulling the vetted tree pointer from the unvetted tree. + // This cache memoizes those results and is lazy loaded. vettedTreeIDs map[string]int64 // [token]treeID // inventory contains the full record inventory grouped by record // status. Each list of tokens is sorted by the timestamp of the - // status change from newest to oldest. + // status change from newest to oldest. This cache is built on + // startup. inventory map[backend.MDStatusT][]string } @@ -191,36 +193,37 @@ func (t *tlogBackend) vettedTreeIDFromToken(token []byte) (int64, bool) { // Unvetted tree exists. Get the freeze record to see if it // contains a pointer to a vetted tree. - fr, err := t.unvetted.freezeRecord(treeID) - if err != nil { - if errors.Is(err, errFreezeRecordNotFound) { - // Unvetted tree exists and is not frozen. This is an unvetted - // record. - return 0, false - } - e := fmt.Sprintf("unvetted freezeRecord %v: %v", treeID, err) + isFrozen, vettedTreeID, err := t.unvetted.treeIsFrozen(treeID) + switch { + case err != nil: + // Something went wrong + e := fmt.Sprintf("unvetted treeIsFrozen %v: %v", treeID, err) panic(e) - } - if fr.TreeID == 0 { + case !isFrozen: + // Unvetted tree exists and is not frozen. This is an unvetted + // record. + return 0, false + case vettedTreeID == 0: // Unvetted tree has been frozen but does not contain a pointer // to another tree. This means it was frozen for some other // reason (ex. censored). This is not a vetted record. return 0, false + default: + // Unvetted tree is frozen and points to a vetted tree. Continue. } - // Ensure the freeze record tree ID points to a valid vetted tree. - // This should not fail. - if !t.vetted.treeExists(fr.TreeID) { + // Verify the vetted tree exists. This should not fail. + if !t.vetted.treeExists(vettedTreeID) { // We're in trouble! e := fmt.Sprintf("freeze record of unvetted tree %v points to "+ - "an invalid vetted tree %v", treeID, fr.TreeID) + "an invalid vetted tree %v", treeID, vettedTreeID) panic(e) } // Update the vetted cache - t.vettedTreeIDAdd(hex.EncodeToString(token), fr.TreeID) + t.vettedTreeIDAdd(hex.EncodeToString(token), vettedTreeID) - return fr.TreeID, true + return vettedTreeID, true } func (t *tlogBackend) inventoryGet() map[backend.MDStatusT][]string { @@ -1148,11 +1151,8 @@ func (t *tlogBackend) unvettedPublish(token []byte, rm backend.RecordMetadata, m token, vettedTreeID) // Freeze the unvetted tree - fr := freezeRecord{ - TreeID: vettedTreeID, - } treeID := treeIDFromToken(token) - err = t.unvetted.treeFreeze(treeID, rm, metadata, fr) + err = t.unvetted.treeFreeze(treeID, rm, metadata, vettedTreeID) if err != nil { return fmt.Errorf("treeFreeze %v: %v", treeID, err) } @@ -1169,7 +1169,7 @@ func (t *tlogBackend) unvettedPublish(token []byte, rm backend.RecordMetadata, m func (t *tlogBackend) unvettedCensor(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { // Freeze the tree treeID := treeIDFromToken(token) - err := t.unvetted.treeFreeze(treeID, rm, metadata, freezeRecord{}) + err := t.unvetted.treeFreeze(treeID, rm, metadata, 0) if err != nil { return fmt.Errorf("treeFreeze %v: %v", treeID, err) } @@ -1292,7 +1292,7 @@ func (t *tlogBackend) vettedCensor(token []byte, rm backend.RecordMetadata, meta if !ok { return fmt.Errorf("vetted record not found") } - err := t.vetted.treeFreeze(treeID, rm, metadata, freezeRecord{}) + err := t.vetted.treeFreeze(treeID, rm, metadata, 0) if err != nil { return fmt.Errorf("treeFreeze %v: %v", treeID, err) } @@ -1318,7 +1318,7 @@ func (t *tlogBackend) vettedArchive(token []byte, rm backend.RecordMetadata, met if !ok { return fmt.Errorf("vetted record not found") } - err := t.vetted.treeFreeze(treeID, rm, metadata, freezeRecord{}) + err := t.vetted.treeFreeze(treeID, rm, metadata, 0) if err != nil { return fmt.Errorf("treeFreeze %v: %v", treeID, err) } @@ -1614,51 +1614,35 @@ func (t *tlogBackend) setup() error { // Add tree to prefixes cache t.prefixAdd(token) - // Check if the tree needs to be added to the vettedTreeIDs cache - // by checking the freeze record of the unvetted tree. - var vettedTreeID int64 - fr, err := t.unvetted.freezeRecord(v.TreeId) - switch err { - case errFreezeRecordNotFound: - // No freeze record means this is not a vetted record. - // Nothing to do. Continue. - case nil: - // A freeze record exists. If a pointer to a vetted tree has - // been set, add it to the vettedTreeIDs cache. - if fr.TreeID != 0 { - vettedTreeID = fr.TreeID - t.vettedTreeIDAdd(hex.EncodeToString(token), vettedTreeID) - } - default: - // All other errors - return fmt.Errorf("freezeRecord %v: %v", v.TreeId, err) + // Verify record exists. Its possible for empty trees to be + // present. This can happen if there was some kind of network + // failure or internal server error on record creation. + _, err := t.unvetted.recordLatest(v.TreeId) + if err == errRecordNotFound { + // This is an empty tree. Skip it. + continue + } else if err != nil { + return fmt.Errorf("unvetted recordLatest %v: %v", v.TreeId, err) } - // Add record to the inventory cache + // Record does exist. Pull it from the backend. var r *backend.Record - if vettedTreeID != 0 { - r, err = t.GetVetted(token, "") + _, ok := t.vettedTreeIDFromToken(token) + if !ok { + // Record is unvetted + r, err = t.GetUnvetted(token, "") if err != nil { - if errors.Is(err, backend.ErrRecordNotFound) { - // A tree that was created but no record was appended onto - // it for whatever reason. This can happen if there is a - // network failure or internal server error. - continue - } - return fmt.Errorf("GetVetted %x: %v", token, err) + return fmt.Errorf("GetUnvetted %x: %v", token, err) } } else { - r, err = t.GetUnvetted(token, "") + // Record is vetted + r, err = t.GetVetted(token, "") if err != nil { - if errors.Is(err, backend.ErrRecordNotFound) { - // A tree that was created but no record was appended onto - // it for whatever reason. This can happen if there is a - // network failure or internal server error. - continue - } return fmt.Errorf("GetUnvetted %x: %v", token, err) } } + + // Add record to the inventory cache t.inventoryAdd(hex.EncodeToString(token), r.RecordMetadata.Status) } diff --git a/politeiad/backend/tlogbe/trillian.go b/politeiad/backend/tlogbe/trillian.go index 763048c5f..a4bdb6d2c 100644 --- a/politeiad/backend/tlogbe/trillian.go +++ b/politeiad/backend/tlogbe/trillian.go @@ -277,6 +277,13 @@ func (t *trillianClient) signedLogRoot(treeID int64) (*trillian.SignedLogRoot, * // will be in the queued leaf. Inclusion proofs will not exist for leaves that // fail to be appended. Note leaves that are duplicates will fail and it is the // callers responsibility to determine how they should be handled. +// +// Trillain DOES NOT guarantee that the leaves of a queued leaves batch are +// appended in the order in which they were received. Trillian is also not +// consistent about the order that leaves are appended in. At the time of +// writing this I have not looked into why this is or if there are other +// methods that can be used. DO NOT rely on the leaves being in a specific +// order. func (t *trillianClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, *types.LogRootV1, error) { log.Tracef("trillian leavesAppend: %v", treeID) From f2ce4b4508a46f7d6d07b4f2a1ea79baf17decb1 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 15 Oct 2020 09:29:32 -0500 Subject: [PATCH 155/449] vote inventory fixes --- politeiad/backend/tlogbe/dcrdata.go | 4 +- politeiad/backend/tlogbe/pi.go | 62 +++- politeiad/backend/tlogbe/ticketvote.go | 392 ++++++++++++++++++------- politeiad/backend/tlogbe/tlog.go | 8 +- politeiad/politeiad.go | 2 + politeiawww/cmd/piwww/voteinventory.go | 18 +- politeiawww/cmd/piwww/voteresults.go | 26 +- politeiawww/cmd/shared/client.go | 8 +- 8 files changed, 392 insertions(+), 128 deletions(-) diff --git a/politeiad/backend/tlogbe/dcrdata.go b/politeiad/backend/tlogbe/dcrdata.go index cff91d0bf..0e49f3a16 100644 --- a/politeiad/backend/tlogbe/dcrdata.go +++ b/politeiad/backend/tlogbe/dcrdata.go @@ -640,7 +640,7 @@ func newDcrdataPlugin(settings []backend.PluginSetting, activeNetParams *chaincf case chaincfg.TestNet3Params().Name: hostHTTP = dcrdata.DefaultHostHTTPTestNet default: - return nil, fmt.Errorf("unkown active net: %v", activeNetParams.Name) + return nil, fmt.Errorf("unknown active net: %v", activeNetParams.Name) } } if hostWS == "" { @@ -650,7 +650,7 @@ func newDcrdataPlugin(settings []backend.PluginSetting, activeNetParams *chaincf case chaincfg.TestNet3Params().Name: hostWS = dcrdata.DefaultHostWSTestNet default: - return nil, fmt.Errorf("unkown active net: %v", activeNetParams.Name) + return nil, fmt.Errorf("unknown active net: %v", activeNetParams.Name) } } diff --git a/politeiad/backend/tlogbe/pi.go b/politeiad/backend/tlogbe/pi.go index 13e798673..ad7bfe957 100644 --- a/politeiad/backend/tlogbe/pi.go +++ b/politeiad/backend/tlogbe/pi.go @@ -639,8 +639,66 @@ func (p *piPlugin) cmdCommentVote(payload string) (string, error) { } func (p *piPlugin) cmdVoteInventory(payload string) (string, error) { - // TODO - return "", nil + // Payload is empty. Nothing to decode. + + // Get ticketvote inventory + r, err := p.backend.Plugin(ticketvote.ID, ticketvote.CmdInventory, "", "") + if err != nil { + return "", fmt.Errorf("ticketvote inventory: %v", err) + } + ir, err := ticketvote.DecodeInventoryReply([]byte(r)) + if err != nil { + return "", err + } + + // Get vote summaries for all finished proposal votes + s := ticketvote.Summaries{ + Tokens: ir.Finished, + } + b, err := ticketvote.EncodeSummaries(s) + if err != nil { + return "", err + } + r, err = p.backend.Plugin(ticketvote.ID, ticketvote.CmdSummaries, + "", string(b)) + if err != nil { + return "", fmt.Errorf("ticketvote summaries: %v", err) + } + sr, err := ticketvote.DecodeSummariesReply([]byte(r)) + if err != nil { + return "", err + } + if len(sr.Summaries) != len(ir.Finished) { + return "", fmt.Errorf("unexpected number of summaries: got %v, want %v", + len(sr.Summaries), len(ir.Finished)) + } + + // Categorize votes + approved := make([]string, 0, len(sr.Summaries)) + rejected := make([]string, 0, len(sr.Summaries)) + for token, v := range sr.Summaries { + if v.Approved { + approved = append(approved, token) + } else { + rejected = append(rejected, token) + } + } + + // Prepare reply + vir := pi.VoteInventoryReply{ + Unauthorized: ir.Unauthorized, + Authorized: ir.Authorized, + Started: ir.Started, + Approved: approved, + Rejected: rejected, + BestBlock: ir.BestBlock, + } + reply, err := pi.EncodeVoteInventoryReply(vir) + if err != nil { + return "", err + } + + return string(reply), nil } func (p *piPlugin) hookNewRecordPre(payload string) error { diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index 70e61cf5c..8ee6ca3f5 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -14,6 +14,7 @@ import ( "io/ioutil" "os" "path/filepath" + "sort" "strconv" "strings" "sync" @@ -93,7 +94,7 @@ type ticketVotePlugin struct { // inv contains the record inventory categorized by vote status. // The inventory will only contain public, non-abandoned records. // This cache is built on startup. - inv inventory + inv voteInventory // votes contains the cast votes of ongoing record votes. This // cache is built on startup and record entries are removed once @@ -105,7 +106,7 @@ type ticketVotePlugin struct { mutexes map[string]*sync.Mutex // [string]mutex } -type inventory struct { +type voteInventory struct { unauthorized []string // Unauthorized tokens authorized []string // Authorized tokens started map[string]uint32 // [token]endHeight @@ -113,10 +114,164 @@ type inventory struct { bestBlock uint32 // Height of last inventory update } -func (p *ticketVotePlugin) cachedInventory() inventory { +func (p *ticketVotePlugin) inventorySetToAuthorize(token string) { p.Lock() defer p.Unlock() + // Remove the token from the unauthorized list. The unauthorize + // list is lazy loaded so it may or may not exist. + var i int + var found bool + for k, v := range p.inv.unauthorized { + if v == token { + i = k + found = true + break + } + } + if found { + u := p.inv.unauthorized + u = append(u[:i], u[i+1]) + p.inv.unauthorized = u + } + + // Prepend the token to the authorized list + a := p.inv.authorized + a = append([]string{token}, a...) + p.inv.authorized = a +} + +func (p *ticketVotePlugin) inventorySetToUnauthorized(token string) { + p.Lock() + defer p.Unlock() + + // Remove the token from the authorized list if it exists. Going + // from authorized to unauthorized can happen when a vote + // authorization is revoked. + var i int + var found bool + for k, v := range p.inv.authorized { + if v == token { + i = k + found = true + break + } + } + if found { + a := p.inv.authorized + a = append(a[:i], a[i+1]) + p.inv.authorized = a + } + + // Prepend the token to the unauthorized list + u := p.inv.unauthorized + u = append([]string{token}, u...) + p.inv.unauthorized = u +} + +func (p *ticketVotePlugin) inventorySetToStarted(token string, endHeight uint32) { + p.Lock() + defer p.Unlock() + + // Remove the token from the authorized list. The token should + // always be in the authorized list prior to the vote being started + // so panicing when this is not the case is ok. + var i int + var found bool + for k, v := range p.inv.authorized { + if v == token { + i = k + found = true + break + } + } + if !found { + e := fmt.Sprintf("token not found in authorized list: %v", token) + panic(e) + } + + a := p.inv.authorized + a = append(a[:i], a[i+1]) + p.inv.authorized = a + + // Add the token to the started map + p.inv.started[token] = endHeight +} + +func (p *ticketVotePlugin) inventory(bestBlock uint32) (*voteInventory, error) { + p.Lock() + defer p.Unlock() + + // Get existing vote inventory + inv := p.inv + + // Check backend inventory for new records + invBackend, err := p.backend.InventoryByStatus() + if err != nil { + return nil, fmt.Errorf("InventoryByStatus: %v", err) + } + + var ( + vetted = invBackend.Vetted + voteInvCount = len(inv.unauthorized) + len(inv.authorized) + + len(inv.started) + len(inv.finished) + ) + if voteInvCount != len(vetted) { + // There are new records. Put all ticket vote inventory records + // into a map so we can easily find what backend records are + // missing. + all := make(map[string]struct{}, voteInvCount) + for _, v := range inv.unauthorized { + all[v] = struct{}{} + } + for _, v := range inv.authorized { + all[v] = struct{}{} + } + for k := range inv.started { + all[k] = struct{}{} + } + for _, v := range inv.finished { + all[v] = struct{}{} + } + + // Add missing records to the vote inventory + for _, v := range invBackend.Vetted { + if _, ok := all[v]; ok { + // Record is already in the vote inventory + continue + } + // We can assume that the record vote status is unauthorized + // since it would have already been added to the vote inventory + // during the authorization request if one had occured. + inv.unauthorized = append(inv.unauthorized, v) + } + } + + // The records are moved to their correct vote status category in + // the inventory on authorization, revoking the authorization, and + // on starting the vote. The last thing we must check for is + // whether any votes have finished since the last inventory update. + + // Check if the inventory has been updated for this block height. + if inv.bestBlock == bestBlock { + // Inventory already updated. Nothing else to do. + goto reply + } + + // Inventory has not been updated for this block height. Check if + // any proposal votes have finished. + for token, endHeight := range inv.started { + if bestBlock >= endHeight { + // Vote has finished + inv.finished = append(inv.finished, token) + delete(inv.started, token) + } + } + + // Update best block + inv.bestBlock = bestBlock + +reply: // Return a copy of the inventory var ( unauthorized = make([]string, len(p.inv.unauthorized)) @@ -137,19 +292,12 @@ func (p *ticketVotePlugin) cachedInventory() inventory { finished[k] = v } - return inventory{ + return &voteInventory{ unauthorized: unauthorized, authorized: authorized, started: started, finished: finished, - } -} - -func (p *ticketVotePlugin) cachedInventorySet(inv inventory) { - p.Lock() - defer p.Unlock() - - p.inv = inv + }, nil } func (p *ticketVotePlugin) cachedVotes(token []byte) map[string]string { @@ -1772,7 +1920,7 @@ func (p *ticketVotePlugin) cmdSummaries(payload string) (string, error) { // have to use the safe best block. bb, err := p.bestBlockUnsafe() if err != nil { - return "", fmt.Errorf("bestBlock: %v", err) + return "", fmt.Errorf("bestBlockUnsafe: %v", err) } // Get summaries @@ -1806,87 +1954,38 @@ func (p *ticketVotePlugin) cmdSummaries(payload string) (string, error) { return string(reply), nil } -func (p *ticketVotePlugin) inventory(bestBlock uint32) (*ticketvote.InventoryReply, error) { - // Get existing inventory - inv := p.cachedInventory() - - // Get backend inventory to see if there are any new unauthorized - // records. - invBackend, err := p.backend.InventoryByStatus() - if err != nil { - return nil, err - } - l := len(inv.unauthorized) + len(inv.authorized) + - len(inv.started) + len(inv.finished) - if l != len(invBackend.Vetted) { - // There are new unauthorized records. Put all ticket vote - // inventory records into a map so we can easily find what - // backend records are missing. - all := make(map[string]struct{}, l) - for _, v := range inv.unauthorized { - all[v] = struct{}{} - } - for _, v := range inv.authorized { - all[v] = struct{}{} - } - for k := range inv.started { - all[k] = struct{}{} - } - for _, v := range inv.finished { - all[v] = struct{}{} - } - - // Add any missing records to the inventory - for _, v := range invBackend.Vetted { - if _, ok := all[v]; !ok { - inv.unauthorized = append(inv.unauthorized, v) - } - } - - // Update cache - p.cachedInventorySet(inv) - } - - // Check if inventory has already been updated for this block - // height. - if inv.bestBlock == bestBlock { - // Inventory already updated. Nothing else to do. - started := make([]string, 0, len(inv.started)) - for k := range inv.started { - started = append(started, k) - } - return &ticketvote.InventoryReply{ - Unauthorized: inv.unauthorized, - Authorized: inv.authorized, - Started: started, - Finished: inv.finished, - BestBlock: bestBlock, - }, nil - } - - // Inventory has not been updated for this block height. Check if - // any proposal votes have finished. - started := make([]string, 0, len(inv.started)) - for token, endHeight := range inv.started { - if bestBlock >= endHeight { - // Vote has finished - inv.finished = append(inv.finished, token) - } else { - // Vote is still ongoing - started = append(started, token) - } - } - - // Update cache - p.cachedInventorySet(inv) - - return &ticketvote.InventoryReply{ - Unauthorized: inv.unauthorized, - Authorized: inv.authorized, +func convertInventoryReply(v voteInventory) ticketvote.InventoryReply { + // Started needs to be converted from a map to a slice where the + // slice is sorted by end block height from smallest to largest. + tokensByHeight := make(map[uint32][]string, len(v.started)) + for token, height := range v.started { + tokens, ok := tokensByHeight[height] + if !ok { + tokens = make([]string, 0, len(v.started)) + } + tokens = append(tokens, token) + tokensByHeight[height] = tokens + } + sortedHeights := make([]uint32, 0, len(tokensByHeight)) + for k := range tokensByHeight { + sortedHeights = append(sortedHeights, k) + } + // Sort smallest to largest block height + sort.SliceStable(sortedHeights, func(i, j int) bool { + return sortedHeights[i] < sortedHeights[j] + }) + started := make([]string, 0, len(v.started)) + for _, height := range sortedHeights { + tokens := tokensByHeight[height] + started = append(started, tokens...) + } + return ticketvote.InventoryReply{ + Unauthorized: v.unauthorized, + Authorized: v.authorized, Started: started, - Finished: inv.finished, - BestBlock: bestBlock, - }, nil + Finished: v.finished, + BestBlock: v.bestBlock, + } } func (p *ticketVotePlugin) cmdInventory(payload string) (string, error) { @@ -1898,17 +1997,18 @@ func (p *ticketVotePlugin) cmdInventory(payload string) (string, error) { // use the unsafe best block. bb, err := p.bestBlockUnsafe() if err != nil { - return "", err + return "", fmt.Errorf("bestBlockUnsafe: %v", err) } // Get the inventory - ir, err := p.inventory(bb) + inv, err := p.inventory(bb) if err != nil { return "", fmt.Errorf("inventory: %v", err) } + ir := convertInventoryReply(*inv) // Prepare reply - reply, err := ticketvote.EncodeInventoryReply(*ir) + reply, err := ticketvote.EncodeInventoryReply(ir) if err != nil { return "", err } @@ -1922,10 +2022,91 @@ func (p *ticketVotePlugin) cmdInventory(payload string) (string, error) { func (p *ticketVotePlugin) setup() error { log.Tracef("ticketvote setup") - // TODO - // Verify dcrdata plugin has been registered - // Build votes cache + // Verify plugin dependencies + plugins, err := p.backend.GetPlugins() + if err != nil { + return fmt.Errorf("Plugins: %v", err) + } + var dcrdataFound bool + for _, v := range plugins { + if v.ID == dcrdata.ID { + dcrdataFound = true + } + } + if !dcrdataFound { + return fmt.Errorf("plugin dependency not registered: %v", dcrdata.ID) + } + // Build inventory cache + log.Debugf("ticketvote: building inventory cache") + + ibs, err := p.backend.InventoryByStatus() + if err != nil { + return fmt.Errorf("InventoryByStatus: %v", err) + } + + bestBlock, err := p.bestBlock() + if err != nil { + return fmt.Errorf("bestBlock: %v", err) + } + + var ( + unauthorized = make([]string, 0, 256) + authorized = make([]string, 0, 256) + started = make(map[string]uint32, 256) // [token]endHeight + finished = make([]string, 0, 256) + ) + for _, v := range ibs.Vetted { + token, err := hex.DecodeString(v) + if err != nil { + return err + } + s, err := p.summary(token, bestBlock) + if err != nil { + return fmt.Errorf("summary %v: %v", v, err) + } + switch s.Status { + case ticketvote.VoteStatusUnauthorized: + unauthorized = append(unauthorized, v) + case ticketvote.VoteStatusAuthorized: + authorized = append(authorized, v) + case ticketvote.VoteStatusStarted: + started[v] = s.EndBlockHeight + case ticketvote.VoteStatusFinished: + finished = append(finished, v) + default: + return fmt.Errorf("invalid vote status %v %v", v, s.Status) + } + } + + p.Lock() + p.inv = voteInventory{ + unauthorized: unauthorized, + authorized: authorized, + started: started, + finished: finished, + } + p.Unlock() + + // Build votes cache + log.Debugf("ticketvote: building votes cache") + inv, err := p.inventory(bestBlock) + if err != nil { + return fmt.Errorf("inventory: %v", err) + } + for k := range inv.started { + token, err := hex.DecodeString(k) + if err != nil { + return err + } + votes, err := p.castVotes(token) + if err != nil { + return fmt.Errorf("castVotes %v: %v", token, err) + } + for _, v := range votes { + p.cachedVotesSet(v.Token, v.Ticket, v.VoteBit) + } + } return nil } @@ -2036,7 +2217,7 @@ func newTicketVotePlugin(backend backend.Backend, tlog tlogClient, settings []ba case chaincfg.SimNetParams().Name: voteDurationMax = ticketvote.DefaultSimNetVoteDurationMax default: - return nil, fmt.Errorf("unkown active net: %v", activeNetParams.Name) + return nil, fmt.Errorf("unknown active net: %v", activeNetParams.Name) } } @@ -2055,7 +2236,14 @@ func newTicketVotePlugin(backend backend.Backend, tlog tlogClient, settings []ba voteDurationMax: voteDurationMax, dataDir: dataDir, identity: id, - votes: make(map[string]map[string]string), - mutexes: make(map[string]*sync.Mutex), + inv: voteInventory{ + unauthorized: make([]string, 0, 256), + authorized: make([]string, 0, 256), + started: make(map[string]uint32, 256), + finished: make([]string, 0, 256), + bestBlock: 0, + }, + votes: make(map[string]map[string]string), + mutexes: make(map[string]*sync.Mutex), }, nil } diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index 6cb415c12..b1e2e237a 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -161,8 +161,7 @@ type recordIndex struct { // of this record. A record can be copied to a new tree after // certain status changes, such as when a record is made public and // the record is copied from an unvetted tree to a vetted tree. - // TreePointer should only be set if the existing tree has been - // frozen. + // TreePointer should only be set if the tree has been frozen. TreePointer int64 `json:"treepointer,omitempty"` } @@ -359,7 +358,7 @@ func convertAnchorFromBlobEntry(be store.BlobEntry) (*anchor, error) { var a anchor err = json.Unmarshal(b, &a) if err != nil { - return nil, fmt.Errorf("unmarshal freezeRecord: %v", err) + return nil, fmt.Errorf("unmarshal anchor: %v", err) } return &a, nil @@ -1700,7 +1699,8 @@ func (t *tlog) blobsByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error // TODO run fsck episodically func (t *tlog) fsck() { - // Set tree status to frozen for any trees with a freeze record. + // Set tree status to frozen for any trees that are frozen and have + // been anchored one last time. // Failed censor. Ensure all blobs have been deleted from all // record versions of a censored record. } diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index de2d04e14..809bc2b2a 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -1053,6 +1053,8 @@ func (p *politeia) pluginCommand(w http.ResponseWriter, r *http.Request) { Payload: payload, } + log.Infof("Plugin %v: %v %v", remoteAddr(r), pc.ID, pc.Command) + util.RespondWithJSON(w, http.StatusOK, reply) } diff --git a/politeiawww/cmd/piwww/voteinventory.go b/politeiawww/cmd/piwww/voteinventory.go index 4784fa3d4..bb4254c33 100644 --- a/politeiawww/cmd/piwww/voteinventory.go +++ b/politeiawww/cmd/piwww/voteinventory.go @@ -5,6 +5,7 @@ package main import ( + pi "github.com/decred/politeia/politeiawww/api/pi/v1" "github.com/decred/politeia/politeiawww/cmd/shared" ) @@ -14,12 +15,25 @@ type voteInventoryCmd struct{} // Execute executes the vote inventory command. func (cmd *voteInventoryCmd) Execute(args []string) error { - reply, err := client.VoteInventory() + // Setup request + vi := pi.VoteInventory{} + + // Send request. The request and response details are printed to + // the console. + err := shared.PrintJSON(vi) + if err != nil { + return err + } + vir, err := client.VoteInventory(vi) + if err != nil { + return err + } + err = shared.PrintJSON(vir) if err != nil { return err } - return shared.PrintJSON(reply) + return nil } // voteInventoryHelpMsg is the command help message. diff --git a/politeiawww/cmd/piwww/voteresults.go b/politeiawww/cmd/piwww/voteresults.go index e9b88fee5..a81d63d4c 100644 --- a/politeiawww/cmd/piwww/voteresults.go +++ b/politeiawww/cmd/piwww/voteresults.go @@ -9,41 +9,43 @@ import ( "github.com/decred/politeia/politeiawww/cmd/shared" ) -// voteResultsCmd gets the votes that have been cast for the specified -// proposal. +// voteResultsCmd retreives the cast votes for the provided proposal. type voteResultsCmd struct { Args struct { - Token string `positional-arg-name:"token"` // Censorship token + Token string `positional-arg-name:"token"` } `positional-args:"true" required:"true"` } -// Execute executes the proposal votes command. +// Execute executes the vote results command. func (cmd *voteResultsCmd) Execute(args []string) error { - // Prep request payload + // Setup request vr := pi.VoteResults{ Token: cmd.Args.Token, } - // Print request details + // Send request. The request and response details are printed to + // the console. err := shared.PrintJSON(vr) if err != nil { return err } - vrr, err := client.VoteResults(vr) if err != nil { return err } + err = shared.PrintJSON(vrr) + if err != nil { + return err + } - return shared.PrintJSON(vrr) + return nil } -// voteResultsHelpMsg is the output of the help command when 'voteresults' is -// specified. +// voteResultsHelpMsg is the help command message. const voteResultsHelpMsg = `voteresults "token" -Fetch vote results for a proposal. +Fetch vote results for the provided proposal. Arguments: -1. token (string, required) Proposal censorship token +1. token (string, required) Proposal censorship token ` diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index 47fddf835..818232caf 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -1260,9 +1260,9 @@ func (c *Client) PayInvoices(pi *cms.PayInvoices) (*cms.PayInvoicesReply, error) // VoteInventory retrieves the tokens of all proposals in the inventory // categorized by their vote status. -func (c *Client) VoteInventory() (*pi.VoteInventoryReply, error) { +func (c *Client) VoteInventory(vi pi.VoteInventory) (*pi.VoteInventoryReply, error) { statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, - pi.RouteVoteInventory, nil) + pi.RouteVoteInventory, vi) if err != nil { return nil, err } @@ -1718,7 +1718,7 @@ func (c *Client) UserRegistrationPayment() (*www.UserRegistrationPaymentReply, e // VoteResults retrieves the vote results for the specified proposal. func (c *Client) VoteResults(vr pi.VoteResults) (*pi.VoteResultsReply, error) { - statusCode, respBody, err := c.makeRequest(http.MethodGet, + statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, pi.RouteVoteResults, vr) if err != nil { return nil, err @@ -1731,7 +1731,7 @@ func (c *Client) VoteResults(vr pi.VoteResults) (*pi.VoteResultsReply, error) { var vrr pi.VoteResultsReply err = json.Unmarshal(respBody, &vrr) if err != nil { - return nil, fmt.Errorf("unmarshal ProposalVotesReply: %v", err) + return nil, err } if c.cfg.Verbose { From e485fc1997b006ab9525e1b07a536a374c75bed9 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 15 Oct 2020 15:21:43 -0500 Subject: [PATCH 156/449] hook up token prefix lookups --- politeiad/api/v1/v1.go | 6 +- politeiad/backend/gitbe/gitbe.go | 2 +- politeiad/backend/tlogbe/ticketvote.go | 5 +- politeiad/backend/tlogbe/tlog.go | 182 +++++++++++++--- politeiad/backend/tlogbe/tlogbe.go | 261 +++++++++++++++-------- politeiad/testpoliteiad/testpoliteiad.go | 2 +- politeiawww/invoices.go | 2 +- politeiawww/piwww.go | 8 +- util/convert.go | 19 +- 9 files changed, 344 insertions(+), 143 deletions(-) diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index d68dfed32..0af3356ca 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -41,9 +41,9 @@ const ( ChallengeSize = 32 // Size of challenge token in bytes // Token sizes. The size of the token depends on the politeiad - // backend configuration, but will always be within this range. - TokenSizeMin = 10 - TokenSizeMax = 32 + // backend configuration. + TokenSizeShort = 10 + TokenSizeLong = 32 MetadataStreamsMax = uint64(16) // Maximum number of metadata streams diff --git a/politeiad/backend/gitbe/gitbe.go b/politeiad/backend/gitbe/gitbe.go index b8be8654d..82de9be3b 100644 --- a/politeiad/backend/gitbe/gitbe.go +++ b/politeiad/backend/gitbe/gitbe.go @@ -1281,7 +1281,7 @@ func (g *gitBackEnd) populateTokenPrefixCache() error { func (g *gitBackEnd) randomUniqueToken() ([]byte, error) { TRIES := 1000 for i := 0; i < TRIES; i++ { - token, err := util.Random(pd.TokenSizeMax) + token, err := util.Random(pd.TokenSizeLong) if err != nil { return nil, err } diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index 8ee6ca3f5..e5a794a17 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -242,7 +242,7 @@ func (p *ticketVotePlugin) inventory(bestBlock uint32) (*voteInventory, error) { } // We can assume that the record vote status is unauthorized // since it would have already been added to the vote inventory - // during the authorization request if one had occured. + // during the authorization request if one had occurred. inv.unauthorized = append(inv.unauthorized, v) } } @@ -2090,6 +2090,7 @@ func (p *ticketVotePlugin) setup() error { // Build votes cache log.Debugf("ticketvote: building votes cache") + inv, err := p.inventory(bestBlock) if err != nil { return fmt.Errorf("inventory: %v", err) @@ -2205,7 +2206,7 @@ func newTicketVotePlugin(backend backend.Backend, tlog tlogClient, settings []ba case chaincfg.SimNetParams().Name: voteDurationMin = ticketvote.DefaultSimNetVoteDurationMin default: - return nil, fmt.Errorf("unkown active net: %v", activeNetParams.Name) + return nil, fmt.Errorf("unknown active net: %v", activeNetParams.Name) } } if voteDurationMax == 0 { diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index b1e2e237a..8289b4591 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -7,7 +7,6 @@ package tlogbe import ( "bytes" "encoding/base64" - "encoding/binary" "encoding/hex" "encoding/json" "errors" @@ -165,16 +164,21 @@ type recordIndex struct { TreePointer int64 `json:"treepointer,omitempty"` } -func treeIDFromToken(token []byte) int64 { - return int64(binary.LittleEndian.Uint64(token)) -} - -func tokenFromTreeID(treeID int64) []byte { - b := make([]byte, binary.MaxVarintLen64) - // Converting between int64 and uint64 doesn't change - // the sign bit, only the way it's interpreted. - binary.LittleEndian.PutUint64(b, uint64(treeID)) - return b +func treePointerExists(r recordIndex) bool { + // Sanity checks + switch { + case !r.Frozen && r.TreePointer > 0: + // Tree pointer should only be set if the record is frozen + e := fmt.Sprintf("tree pointer set without record being frozen %v", + r.TreePointer) + panic(e) + case r.TreePointer < 0: + // Tree pointer should never be negative + e := fmt.Sprintf("tree pointer is < 0: %v", r.TreePointer) + panic(e) + } + + return r.TreePointer > 0 } // blobIsEncrypted returns whether the provided blob has been prefixed with an @@ -456,18 +460,46 @@ func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []ba return nil } -// treeIsFrozen returns whether the provided tree is frozen and the tree -// pointer if one exists. -func (t *tlog) treeIsFrozen(treeID int64) (bool, int64, error) { +// treePointer returns the tree pointer for the provided tree if one exists. +// The returned bool will indicate if a tree pointer was found. +func (t *tlog) treePointer(treeID int64) (int64, bool) { + log.Tracef("%v treePointer: %v", t.id, treeID) + + // Verify tree exists + if !t.treeExists(treeID) { + return 0, false + } + + // Verify record index exists + var idx *recordIndex leavesAll, err := t.trillian.leavesAll(treeID) if err != nil { - return false, 0, fmt.Errorf("leavesAll: %v", err) + err = fmt.Errorf("leavesAll: %v", err) + goto printErr } - idx, err := t.recordIndexLatest(leavesAll) + idx, err = t.recordIndexLatest(leavesAll) if err != nil { - return false, 0, err + if err == errRecordNotFound { + // This is an empty tree. This can happen sometimes if a error + // occurred during record creation. Return gracefully. + return 0, false + } + err = fmt.Errorf("recordIndexLatest: %v", err) + goto printErr } - return idx.Frozen, idx.TreePointer, nil + + // Check if a tree pointer exists + if !treePointerExists(*idx) { + // Tree pointer not found + return 0, false + } + + // Tree pointer found! + return idx.TreePointer, true + +printErr: + log.Errorf("%v treePointer: %v", t.id, err) + return 0, false } func (t *tlog) recordIndexSave(treeID int64, ri recordIndex) error { @@ -519,12 +551,11 @@ func (t *tlog) recordIndexSave(treeID int64, ri recordIndex) error { return nil } -func (t *tlog) recordIndexVersion(leaves []*trillian.LogLeaf, version uint32) (*recordIndex, error) { - indexes, err := t.recordIndexes(leaves) - if err != nil { - return nil, err - } - +// recordIndexVersion takes a list of record indexes for a record and returns +// the most recent iteration of the specified version. A version of 0 indicates +// that the latest version should be returned. A errRecordNotFound is returned +// if the provided version does not exist. +func recordIndexVersion(indexes []recordIndex, version uint32) (*recordIndex, error) { // Return the record index for the specified version var ri *recordIndex if version == 0 { @@ -550,6 +581,15 @@ func (t *tlog) recordIndexVersion(leaves []*trillian.LogLeaf, version uint32) (* return ri, nil } +func (t *tlog) recordIndexVersion(leaves []*trillian.LogLeaf, version uint32) (*recordIndex, error) { + indexes, err := t.recordIndexes(leaves) + if err != nil { + return nil, err + } + + return recordIndexVersion(indexes, version) +} + func (t *tlog) recordIndexLatest(leaves []*trillian.LogLeaf) (*recordIndex, error) { return t.recordIndexVersion(leaves, 0) } @@ -1227,6 +1267,61 @@ func (t *tlog) recordDel(treeID int64) error { return nil } +// recordExists returns whether a record exists on the provided tree ID. A +// record is considered to not exist if any of the following conditions are +// met: +// +// * A tree does not exist for the tree ID. +// +// * A tree exists but a record index does not exist. This can happen if a +// tree was created but there was a network error prior to the record index +// being appended to the tree. +// +// * The tree is frozen and points to another tree. The record is considered to +// exists on the tree being pointed to, but not on this one. This happens +// in some situations like when an unvetted record is made public and copied +// onto a vetted tree. +// +// The tree pointer is also returned if one is found. +func (t *tlog) recordExists(treeID int64) bool { + log.Tracef("%v recordExists: %v", t.id, treeID) + + // Verify tree exists + if !t.treeExists(treeID) { + return false + } + + // Verify record index exists + var idx *recordIndex + leavesAll, err := t.trillian.leavesAll(treeID) + if err != nil { + err = fmt.Errorf("leavesAll: %v", err) + goto printErr + } + idx, err = t.recordIndexLatest(leavesAll) + if err != nil { + if err == errRecordNotFound { + // This is an empty tree. This can happen sometimes if a error + // occurred during record creation. Return gracefully. + return false + } + err = fmt.Errorf("recordIndexLatest: %v", err) + goto printErr + } + + // Verify a tree pointer does not exist + if treePointerExists(*idx) { + return false + } + + // Record exists! + return true + +printErr: + log.Errorf("%v recordExists: %v", t.id, err) + return false +} + func (t *tlog) record(treeID int64, version uint32) (*backend.Record, error) { log.Tracef("%v record: %v %v", t.id, treeID, version) @@ -1241,28 +1336,45 @@ func (t *tlog) record(treeID int64, version uint32) (*backend.Record, error) { return nil, fmt.Errorf("leavesAll %v: %v", treeID, err) } - // Get the record index for the specified version - index, err := t.recordIndexVersion(leaves, version) + // Verify the latest record index does not point to another tree. + // If it does have a tree pointer, the record is considered to + // exists on the tree being pointed to, but not on this one. This + // happens in situations such as when an unvetted record is made + // public and copied to a vetted tree. Querying the unvetted tree + // will result in a errRecordNotFound error being returned and the + // vetted tree must be queried instead. + indexes, err := t.recordIndexes(leaves) + if err != nil { + return nil, err + } + idxLatest, err := recordIndexVersion(indexes, 0) if err != nil { return nil, err } + if treePointerExists(*idxLatest) { + return nil, errRecordNotFound + } // Use the record index to pull the record content from the store. // The keys for the record content first need to be extracted from // their associated log leaf. + idx, err := recordIndexVersion(indexes, version) + if err != nil { + return nil, err + } // Compile merkle root hashes of record content merkles := make(map[string]struct{}, 64) - merkles[hex.EncodeToString(index.RecordMetadata)] = struct{}{} - for _, v := range index.Metadata { + merkles[hex.EncodeToString(idx.RecordMetadata)] = struct{}{} + for _, v := range idx.Metadata { merkles[hex.EncodeToString(v)] = struct{}{} } - for _, v := range index.Files { + for _, v := range idx.Files { merkles[hex.EncodeToString(v)] = struct{}{} } // Walk the tree and extract the record content keys - keys := make([]string, 0, len(index.Metadata)+len(index.Files)+1) + keys := make([]string, 0, len(idx.Metadata)+len(idx.Files)+1) for _, v := range leaves { _, ok := merkles[hex.EncodeToString(v.MerkleLeafHash)] if !ok { @@ -1318,8 +1430,8 @@ func (t *tlog) record(treeID int64, version uint32) (*backend.Record, error) { // Decode blob entries var ( recordMD *backend.RecordMetadata - metadata = make([]backend.MetadataStream, 0, len(index.Metadata)) - files = make([]backend.File, 0, len(index.Files)) + metadata = make([]backend.MetadataStream, 0, len(idx.Metadata)) + files = make([]backend.File, 0, len(idx.Files)) ) for _, v := range entries { // Decode the data hint @@ -1381,13 +1493,13 @@ func (t *tlog) record(treeID int64, version uint32) (*backend.Record, error) { switch { case recordMD == nil: return nil, fmt.Errorf("record metadata not found") - case len(metadata) != len(index.Metadata): + case len(metadata) != len(idx.Metadata): return nil, fmt.Errorf("invalid number of metadata; got %v, want %v", - len(metadata), len(index.Metadata)) + len(metadata), len(idx.Metadata)) } return &backend.Record{ - Version: strconv.FormatUint(uint64(index.Version), 10), + Version: strconv.FormatUint(uint64(idx.Version), 10), RecordMetadata: *recordMD, Metadata: metadata, Files: files, diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index a943a43c9..291206947 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -8,6 +8,7 @@ import ( "bytes" "crypto/sha256" "encoding/base64" + "encoding/binary" "encoding/hex" "errors" "fmt" @@ -34,7 +35,6 @@ import ( // TODO testnet vs mainnet trillian databases // TODO fsck -// TODO allow token prefix lookups const ( defaultTrillianKeyFilename = "trillian.key" @@ -93,7 +93,7 @@ type tlogBackend struct { vetted *tlog plugins map[string]plugin // [pluginID]plugin - // prefixes contains the token prefix to full token mapping for all + // prefixes contains the prefix to full token mapping for unvetted // records. The prefix is the first n characters of the hex encoded // record token, where n is defined by the TokenPrefixLength from // the politeiad API. Record lookups by token prefix are allowed. @@ -124,15 +124,68 @@ type plugin struct { client pluginClient } +func tokenIsFullLength(token []byte) bool { + return len(token) == v1.TokenSizeShort +} + func tokenPrefix(token []byte) string { return hex.EncodeToString(token)[:v1.TokenPrefixLength] } -func (t *tlogBackend) isShutdown() bool { - t.RLock() - defer t.RUnlock() +// tokenPrefixSize returns the size in bytes of a token prefix. +func tokenPrefixSize() int { + // If the token prefix length is an odd number of characters then + // padding will have needed to be added to it prior to decoding it + // to hex to prevent a hex.ErrLenth (odd length hex string) error. + // Account for this padding in the prefix size. + var size int + if v1.TokenPrefixLength%2 == 1 { + // Add 1 to the length to account for padding + size = (v1.TokenPrefixLength + 1) / 2 + } else { + // No padding was required + size = v1.TokenPrefixLength / 2 + } + return size +} - return t.shutdown +func tokenFromTreeID(treeID int64) []byte { + b := make([]byte, binary.MaxVarintLen64) + // Converting between int64 and uint64 doesn't change + // the sign bit, only the way it's interpreted. + binary.LittleEndian.PutUint64(b, uint64(treeID)) + return b +} + +func treeIDFromToken(token []byte) int64 { + return int64(binary.LittleEndian.Uint64(token)) +} + +// fullLengthToken returns the full length token given the token prefix. +// +// This function must be called WITHOUT the lock held. +func (t *tlogBackend) fullLengthToken(prefix []byte) ([]byte, bool) { + t.Lock() + defer t.Unlock() + + token, ok := t.prefixes[tokenPrefix(prefix)] + return token, ok +} + +// unvettedTreeIDFromToken returns the unvetted tree ID for the provided token. +// This can be either the full length token or the token prefix. +// +// This function must be called WITHOUT the lock held. +func (t *tlogBackend) unvettedTreeIDFromToken(token []byte) int64 { + if len(token) == tokenPrefixSize() { + // This is a token prefix. Get the full token from the cache. + var ok bool + token, ok = t.fullLengthToken(token) + if !ok { + return 0 + } + } + return treeIDFromToken(token) } func (t *tlogBackend) prefixExists(fullToken []byte) bool { @@ -173,50 +226,33 @@ func (t *tlogBackend) vettedTreeIDAdd(token string, treeID int64) { // vettedTreeIDFromToken returns the vetted tree ID that corresponds to the // provided token. If a tree ID is not found then the returned bool will be // false. +// +// This function must be called WITHOUT the lock held. func (t *tlogBackend) vettedTreeIDFromToken(token []byte) (int64, bool) { // Check if the token is in the vetted cache. The vetted cache is // lazy loaded if the token is not present then we need to check // manually. - treeID, ok := t.vettedTreeID(token) + vettedTreeID, ok := t.vettedTreeID(token) if ok { - return treeID, true + return vettedTreeID, true } // The token is derived from the unvetted tree ID. Check if the - // token corresponds to an unvetted tree. - treeID = treeIDFromToken(token) - if !t.unvetted.treeExists(treeID) { - // Unvetted tree does not exists. This token does not correspond - // to any record. - return 0, false - } - - // Unvetted tree exists. Get the freeze record to see if it - // contains a pointer to a vetted tree. - isFrozen, vettedTreeID, err := t.unvetted.treeIsFrozen(treeID) - switch { - case err != nil: - // Something went wrong - e := fmt.Sprintf("unvetted treeIsFrozen %v: %v", treeID, err) - panic(e) - case !isFrozen: - // Unvetted tree exists and is not frozen. This is an unvetted - // record. - return 0, false - case vettedTreeID == 0: - // Unvetted tree has been frozen but does not contain a pointer - // to another tree. This means it was frozen for some other - // reason (ex. censored). This is not a vetted record. + // unvetted record has a tree pointer. The tree pointer will be + // the vetted tree ID. + unvettedTreeID := t.unvettedTreeIDFromToken(token) + vettedTreeID, ok = t.unvetted.treePointer(unvettedTreeID) + if !ok { + // No tree pointer. This record either doesn't exist or is an + // unvetted record. return 0, false - default: - // Unvetted tree is frozen and points to a vetted tree. Continue. } // Verify the vetted tree exists. This should not fail. if !t.vetted.treeExists(vettedTreeID) { // We're in trouble! e := fmt.Sprintf("freeze record of unvetted tree %v points to "+ - "an invalid vetted tree %v", treeID, vettedTreeID) + "an invalid vetted tree %v", unvettedTreeID, vettedTreeID) panic(e) } @@ -549,6 +585,13 @@ func metadataStreamsUpdate(curr, mdAppend, mdOverwrite []backend.MetadataStream) return metadata } +func (t *tlogBackend) isShutdown() bool { + t.RLock() + defer t.RUnlock() + + return t.shutdown +} + // New satisfies the Backend interface. // // This function satisfies the Backend interface. @@ -657,6 +700,14 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ } } + // Verify token is valid. The full length token must be used when + // writing data. + if !tokenIsFullLength(token) { + return nil, backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidToken, + } + } + // Verify record exists and is unvetted if !t.UnvettedExists(token) { return nil, backend.ErrRecordNotFound @@ -755,6 +806,14 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b } } + // Verify token is valid. The full length token must be used when + // writing data. + if !tokenIsFullLength(token) { + return nil, backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidToken, + } + } + // Get vetted tree ID treeID, ok := t.vettedTreeIDFromToken(token) if !ok { @@ -859,6 +918,14 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite } } + // Verify token is valid. The full length token must be used when + // writing data. + if !tokenIsFullLength(token) { + return backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidToken, + } + } + // Verify record exists and is unvetted if !t.UnvettedExists(token) { return backend.ErrRecordNotFound @@ -945,6 +1012,14 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ } } + // Verify token is valid. The full length token must be used when + // writing data. + if !tokenIsFullLength(token) { + return backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidToken, + } + } + // Get vetted tree ID treeID, ok := t.vettedTreeIDFromToken(token) if !ok { @@ -1015,29 +1090,17 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ func (t *tlogBackend) UnvettedExists(token []byte) bool { log.Tracef("UnvettedExists %x", token) - // If the token is in the vetted cache then we know this is not an - // unvetted record without having to make any network requests. + // Verify token is not in the vetted tree IDs cache. If it is then + // we can be sure that this is not a unvetted record without having + // to send any network requests. _, ok := t.vettedTreeID(token) if ok { return false } - // Check if unvetted tree exists - treeID := treeIDFromToken(token) - if !t.unvetted.treeExists(treeID) { - // Unvetted tree does not exists. No tree, no record. - return false - } - - // An unvetted tree exists. Check if a vetted tree also exists. If - // one does then it means this record has been made public and is - // no longer unvetted. - if t.VettedExists(token) { - return false - } - - // Vetted record does not exist. This is an unvetted record. - return true + // Check for unvetted record + treeID := t.unvettedTreeIDFromToken(token) + return t.unvetted.recordExists(treeID) } // This function satisfies the Backend interface. @@ -1056,7 +1119,7 @@ func (t *tlogBackend) GetUnvetted(token []byte, version string) (*backend.Record return nil, backend.ErrShutdown } - treeID := treeIDFromToken(token) + treeID := t.unvettedTreeIDFromToken(token) var v uint32 if version != "" { u, err := strconv.ParseUint(version, 10, 64) @@ -1118,27 +1181,9 @@ func (t *tlogBackend) GetVetted(token []byte, version string) (*backend.Record, // This function must be called WITH the unvetted lock held. func (t *tlogBackend) unvettedPublish(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { // Create a vetted tree - var ( - vettedToken []byte - vettedTreeID int64 - err error - ) - for retries := 0; retries < 10; retries++ { - vettedTreeID, err = t.vetted.treeNew() - if err != nil { - return err - } - vettedToken = tokenFromTreeID(vettedTreeID) - - // Check for token prefix collisions - if !t.prefixExists(vettedToken) { - // Not a collision. Update prefixes cache. - t.prefixAdd(vettedToken) - break - } - - log.Infof("Token prefix collision %v, creating new token", - tokenPrefix(vettedToken)) + vettedTreeID, err := t.vetted.treeNew() + if err != nil { + return err } // Save the record to the vetted tlog @@ -1191,6 +1236,14 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, log.Tracef("SetUnvettedStatus: %x %v (%v)", token, status, backend.MDStatus[status]) + // Verify token is valid. The full length token must be used when + // writing data. + if !tokenIsFullLength(token) { + return nil, backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidToken, + } + } + // Verify record exists and is unvetted if !t.UnvettedExists(token) { return nil, backend.ErrRecordNotFound @@ -1276,10 +1329,22 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, token, backend.MDStatus[currStatus], currStatus, backend.MDStatus[status], status) - // Return the updated record - r, err = t.unvetted.recordLatest(treeID) - if err != nil { - return nil, fmt.Errorf("recordLatest: %v", err) + // Return the updated record. If the record was made public it is + // now a vetted record and must be fetched accordingly. + switch status { + case backend.MDStatusVetted: + r, err = t.GetVetted(token, "") + if err != nil { + return nil, fmt.Errorf("GetVetted: %v", err) + } + case backend.MDStatusCensored: + r, err = t.GetUnvetted(token, "") + if err != nil { + return nil, fmt.Errorf("GetUnvetted: %v", err) + } + default: + return nil, fmt.Errorf("unknown status: %v (%v)", + backend.MDStatus[status], status) } return r, nil @@ -1333,6 +1398,14 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md log.Tracef("SetVettedStatus: %x %v (%v)", token, status, backend.MDStatus[status]) + // Verify token is valid. The full length token must be used when + // writing data. + if !tokenIsFullLength(token) { + return nil, backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidToken, + } + } + // Get vetted tree ID treeID, ok := t.vettedTreeIDFromToken(token) if !ok { @@ -1614,32 +1687,34 @@ func (t *tlogBackend) setup() error { // Add tree to prefixes cache t.prefixAdd(token) - // Verify record exists. Its possible for empty trees to be - // present. This can happen if there was some kind of network - // failure or internal server error on record creation. - _, err := t.unvetted.recordLatest(v.TreeId) - if err == errRecordNotFound { - // This is an empty tree. Skip it. - continue - } else if err != nil { - return fmt.Errorf("unvetted recordLatest %v: %v", v.TreeId, err) - } + // Identify whether the record is unvetted or vetted. + isUnvetted := t.UnvettedExists(token) + isVetted := t.VettedExists(token) - // Record does exist. Pull it from the backend. + // Get the record var r *backend.Record - _, ok := t.vettedTreeIDFromToken(token) - if !ok { + switch { + case isUnvetted && isVetted: + // Sanity check + e := fmt.Sprintf("records is both unvetted and vetted: %x", token) + panic(e) + case isUnvetted: // Record is unvetted r, err = t.GetUnvetted(token, "") if err != nil { return fmt.Errorf("GetUnvetted %x: %v", token, err) } - } else { + case isVetted: // Record is vetted r, err = t.GetVetted(token, "") if err != nil { return fmt.Errorf("GetUnvetted %x: %v", token, err) } + default: + // This is an empty tree. This can happen if there was an error + // during record creation and the record failed to be appended + // to the tree. + log.Debugf("Empty tree found for token %x", token) } // Add record to the inventory cache diff --git a/politeiad/testpoliteiad/testpoliteiad.go b/politeiad/testpoliteiad/testpoliteiad.go index 6b2fe5202..05e6ee76a 100644 --- a/politeiad/testpoliteiad/testpoliteiad.go +++ b/politeiad/testpoliteiad/testpoliteiad.go @@ -159,7 +159,7 @@ func (p *TestPoliteiad) handleNewRecord(w http.ResponseWriter, r *http.Request) } // Prepare response - tokenb, err := util.Random(v1.TokenSizeMax) + tokenb, err := util.Random(v1.TokenSizeLong) if err != nil { util.RespondWithJSON(w, http.StatusInternalServerError, err) return diff --git a/politeiawww/invoices.go b/politeiawww/invoices.go index b6141d841..c6fedaa2f 100644 --- a/politeiawww/invoices.go +++ b/politeiawww/invoices.go @@ -678,7 +678,7 @@ func (p *politeiawww) processNewInvoice(ctx context.Context, ni cms.NewInvoice, // Handle test case if p.test { - tokenBytes, err := util.Random(pd.TokenSizeMax) + tokenBytes, err := util.Random(pd.TokenSizeLong) if err != nil { return nil, err } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index c3c2e8d88..86ae58f73 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -61,8 +61,8 @@ var ( ) // tokenIsValid returns whether the provided string is a valid politeiad -// censorship record token. This CAN BE EITHER the short token or the full -// length token. +// censorship record token. This CAN BE EITHER the full length token or the +// token prefix. // // Short tokens should only be used when retrieving data. Data that is written // to disk should always reference the full length token. @@ -71,7 +71,7 @@ func tokenIsValid(token string) bool { switch { case len(token) == pd.TokenPrefixLength: // Token is a short proposal token - case len(token) == pd.TokenSizeMin*2: + case len(token) == pd.TokenSizeShort*2: // Token is a full length token default: // Unknown token size @@ -91,7 +91,7 @@ func tokenIsFullLength(token string) bool { if err != nil { return false } - if len(b) != pd.TokenSizeMin { + if len(b) != pd.TokenSizeShort { return false } return true diff --git a/util/convert.go b/util/convert.go index f861464ef..a29ef00e7 100644 --- a/util/convert.go +++ b/util/convert.go @@ -31,12 +31,25 @@ func ConvertSignature(s string) ([identity.SignatureSize]byte, error) { } // ConvertStringToken verifies and converts a string token to a proper sized -// []byte. +// byte slice. This function accepts both the full length token and the token +// prefix. func ConvertStringToken(token string) ([]byte, error) { - if len(token) > pd.TokenSizeMax*2 || - len(token) < pd.TokenSizeMin*2 { + switch { + case len(token) == pd.TokenSizeShort*2: + // Tlog backend token; continue + case len(token) != pd.TokenSizeLong*2: + // Git backend token; continue + case len(token) == pd.TokenPrefixLength: + // Token prefix; continue + default: return nil, fmt.Errorf("invalid censorship token size") } + // If the token length is an odd number of characters, a 0 digit is + // appended onto the string to prevent a hex.ErrLenth (odd length + // hex string) error when decoding. + if len(token)%2 == 1 { + token = token + "0" + } blob, err := hex.DecodeString(token) if err != nil { return nil, err From e9f5fadc6373a2ae96bbc8334acd79100d0198a5 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 15 Oct 2020 20:12:58 -0500 Subject: [PATCH 157/449] ticket vote plugin bug fixes --- politeiad/backend/tlogbe/ticketvote.go | 73 ++++++++++++++++++-------- politeiad/backend/tlogbe/tlogbe.go | 2 +- 2 files changed, 52 insertions(+), 23 deletions(-) diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index e5a794a17..f64c51ada 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -101,8 +101,11 @@ type ticketVotePlugin struct { // the vote has ended and the vote summary has been saved. votes map[string]map[string]string // [token][ticket]voteBit - // Mutexes contains a mutex for each record. The mutexes are lazy - // loaded. + // Mutexes contains a mutex for each record and are used to lock + // the trillian tree for a given record to prevent concurrent + // ticket vote plugin updates to the same tree. They are not used + // to update any of the ticket vote plugin memory caches. These + // mutexes are lazy loaded. mutexes map[string]*sync.Mutex // [string]mutex } @@ -114,7 +117,7 @@ type voteInventory struct { bestBlock uint32 // Height of last inventory update } -func (p *ticketVotePlugin) inventorySetToAuthorize(token string) { +func (p *ticketVotePlugin) inventorySetToAuthorized(token string) { p.Lock() defer p.Unlock() @@ -130,8 +133,9 @@ func (p *ticketVotePlugin) inventorySetToAuthorize(token string) { } } if found { + // Remove the token from unauthorized u := p.inv.unauthorized - u = append(u[:i], u[i+1]) + u = append(u[:i], u[i+1:]...) p.inv.unauthorized = u } @@ -158,8 +162,9 @@ func (p *ticketVotePlugin) inventorySetToUnauthorized(token string) { } } if found { + // Remove the token from authorized a := p.inv.authorized - a = append(a[:i], a[i+1]) + a = append(a[:i], a[i+1:]...) p.inv.authorized = a } @@ -191,7 +196,7 @@ func (p *ticketVotePlugin) inventorySetToStarted(token string, endHeight uint32) } a := p.inv.authorized - a = append(a[:i], a[i+1]) + a = append(a[:i], a[i+1:]...) p.inv.authorized = a // Add the token to the started map @@ -202,35 +207,31 @@ func (p *ticketVotePlugin) inventory(bestBlock uint32) (*voteInventory, error) { p.Lock() defer p.Unlock() - // Get existing vote inventory - inv := p.inv - // Check backend inventory for new records invBackend, err := p.backend.InventoryByStatus() if err != nil { return nil, fmt.Errorf("InventoryByStatus: %v", err) } - var ( vetted = invBackend.Vetted - voteInvCount = len(inv.unauthorized) + len(inv.authorized) + - len(inv.started) + len(inv.finished) + voteInvCount = len(p.inv.unauthorized) + len(p.inv.authorized) + + len(p.inv.started) + len(p.inv.finished) ) if voteInvCount != len(vetted) { // There are new records. Put all ticket vote inventory records // into a map so we can easily find what backend records are // missing. all := make(map[string]struct{}, voteInvCount) - for _, v := range inv.unauthorized { + for _, v := range p.inv.unauthorized { all[v] = struct{}{} } - for _, v := range inv.authorized { + for _, v := range p.inv.authorized { all[v] = struct{}{} } - for k := range inv.started { + for k := range p.inv.started { all[k] = struct{}{} } - for _, v := range inv.finished { + for _, v := range p.inv.finished { all[v] = struct{}{} } @@ -243,7 +244,7 @@ func (p *ticketVotePlugin) inventory(bestBlock uint32) (*voteInventory, error) { // We can assume that the record vote status is unauthorized // since it would have already been added to the vote inventory // during the authorization request if one had occurred. - inv.unauthorized = append(inv.unauthorized, v) + p.inv.unauthorized = append(p.inv.unauthorized, v) } } @@ -253,25 +254,26 @@ func (p *ticketVotePlugin) inventory(bestBlock uint32) (*voteInventory, error) { // whether any votes have finished since the last inventory update. // Check if the inventory has been updated for this block height. - if inv.bestBlock == bestBlock { + if p.inv.bestBlock == bestBlock { // Inventory already updated. Nothing else to do. goto reply } // Inventory has not been updated for this block height. Check if // any proposal votes have finished. - for token, endHeight := range inv.started { + for token, endHeight := range p.inv.started { if bestBlock >= endHeight { // Vote has finished - inv.finished = append(inv.finished, token) - delete(inv.started, token) + p.inv.finished = append(p.inv.finished, token) + delete(p.inv.started, token) } } // Update best block - inv.bestBlock = bestBlock + p.inv.bestBlock = bestBlock reply: + // TODO make this better // Return a copy of the inventory var ( unauthorized = make([]string, len(p.inv.unauthorized)) @@ -297,6 +299,7 @@ reply: authorized: authorized, started: started, finished: finished, + bestBlock: p.inv.bestBlock, }, nil } @@ -641,6 +644,11 @@ func (p *ticketVotePlugin) authorizes(token []byte) ([]ticketvote.AuthorizeDetai auths = append(auths, *a) } + // Sort from oldest to newest + sort.SliceStable(auths, func(i, j int) bool { + return auths[i].Timestamp < auths[j].Timestamp + }) + return auths, nil } @@ -767,6 +775,11 @@ func (p *ticketVotePlugin) castVotes(token []byte) ([]ticketvote.CastVoteDetails votes = append(votes, *cv) } + // Sort by ticket hash + sort.SliceStable(votes, func(i, j int) bool { + return votes[i].Ticket < votes[j].Ticket + }) + return votes, nil } @@ -1144,6 +1157,18 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { return "", err } + // Update inventory + switch a.Action { + case ticketvote.ActionAuthorize: + p.inventorySetToAuthorized(a.Token) + case ticketvote.ActionRevoke: + p.inventorySetToUnauthorized(a.Token) + default: + // Should not happen + e := fmt.Sprintf("invalid authorize action: %v", a.Action) + panic(e) + } + // Prepare reply ar := ticketvote.AuthorizeReply{ Timestamp: auth.Timestamp, @@ -1417,6 +1442,9 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { return "", fmt.Errorf("startSave: %v", err) } + // Update inventory + p.inventorySetToStarted(vd.Params.Token, vd.EndBlockHeight) + // Prepare reply reply, err := ticketvote.EncodeStartReply(*sr) if err != nil { @@ -2085,6 +2113,7 @@ func (p *ticketVotePlugin) setup() error { authorized: authorized, started: started, finished: finished, + bestBlock: bestBlock, } p.Unlock() diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 291206947..965168592 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -135,7 +135,7 @@ func tokenPrefix(token []byte) string { // tokenPrefixSize returns the size in bytes of a token prefix. func tokenPrefixSize() int { // If the token prefix length is an odd number of characters then - // padding will have needed to be added to it prior to decoding it + // padding would have needed to be added to it prior to decoding it // to hex to prevent a hex.ErrLenth (odd length hex string) error. // Account for this padding in the prefix size. var size int From 5c970f6b96917d212219827a55ab0bb282eefa17 Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 16 Oct 2020 14:14:42 -0500 Subject: [PATCH 158/449] cleanup and improve debug logging --- politeiad/backend/tlogbe/dcrdata.go | 28 +++++--- politeiad/backend/tlogbe/ticketvote.go | 75 ++++++++++++++-------- politeiad/plugins/ticketvote/ticketvote.go | 4 +- politeiad/politeiad.go | 4 +- politeiawww/api/pi/v1/v1.go | 4 +- 5 files changed, 73 insertions(+), 42 deletions(-) diff --git a/politeiad/backend/tlogbe/dcrdata.go b/politeiad/backend/tlogbe/dcrdata.go index 0e49f3a16..de6777cb9 100644 --- a/politeiad/backend/tlogbe/dcrdata.go +++ b/politeiad/backend/tlogbe/dcrdata.go @@ -35,6 +35,12 @@ const ( routeBlockDetails = "/api/block/{height}" routeTicketPool = "/api/stake/pool/b/{hash}/full" routeTxsTrimmed = "/api/txs/trimmed" + + // Request headers + headerContentType = "Content-Type" + + // Header values + contentTypeJSON = "application/json; charset=utf-8" ) var ( @@ -90,7 +96,7 @@ func (p *dcrdataPlugin) bestBlockIsStale() bool { return p.bestBlockStale } -func (p *dcrdataPlugin) makeReq(method string, route string, v interface{}) ([]byte, error) { +func (p *dcrdataPlugin) makeReq(method string, route string, headers map[string]string, v interface{}) ([]byte, error) { var ( url = p.hostHTTP + route reqBody []byte @@ -99,19 +105,22 @@ func (p *dcrdataPlugin) makeReq(method string, route string, v interface{}) ([]b log.Tracef("%v %v", method, url) - // Setup request body + // Setup request if v != nil { reqBody, err = json.Marshal(v) if err != nil { return nil, err } } - - // Send request req, err := http.NewRequest(method, url, bytes.NewReader(reqBody)) if err != nil { return nil, err } + for k, v := range headers { + req.Header.Add(k, v) + } + + // Send request r, err := p.client.Do(req) if err != nil { return nil, err @@ -135,7 +144,7 @@ func (p *dcrdataPlugin) makeReq(method string, route string, v interface{}) ([]b // bestBlockHTTP fetches and returns the best block from the dcrdata http API. func (p *dcrdataPlugin) bestBlockHTTP() (*v5.BlockDataBasic, error) { - resBody, err := p.makeReq(http.MethodGet, routeBestBlock, nil) + resBody, err := p.makeReq(http.MethodGet, routeBestBlock, nil, nil) if err != nil { return nil, err } @@ -155,7 +164,7 @@ func (p *dcrdataPlugin) blockDetailsHTTP(height uint32) (*v5.BlockDataBasic, err h := strconv.FormatUint(uint64(height), 10) route := strings.Replace(routeBlockDetails, "{height}", h, 1) - resBody, err := p.makeReq(http.MethodGet, route, nil) + resBody, err := p.makeReq(http.MethodGet, route, nil, nil) if err != nil { return nil, err } @@ -174,7 +183,7 @@ func (p *dcrdataPlugin) blockDetailsHTTP(height uint32) (*v5.BlockDataBasic, err func (p *dcrdataPlugin) ticketPoolHTTP(blockHash string) ([]string, error) { route := strings.Replace(routeTicketPool, "{hash}", blockHash, 1) route += "?sort=true" - resBody, err := p.makeReq(http.MethodGet, route, nil) + resBody, err := p.makeReq(http.MethodGet, route, nil, nil) if err != nil { return nil, err } @@ -194,7 +203,10 @@ func (p *dcrdataPlugin) txsTrimmedHTTP(txIDs []string) ([]v5.TrimmedTx, error) { t := v5.Txns{ Transactions: txIDs, } - resBody, err := p.makeReq(http.MethodPost, routeTxsTrimmed, t) + headers := map[string]string{ + headerContentType: contentTypeJSON, + } + resBody, err := p.makeReq(http.MethodPost, routeTxsTrimmed, headers, t) if err != nil { return nil, err } diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index f64c51ada..f48c45669 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -109,6 +109,10 @@ type ticketVotePlugin struct { mutexes map[string]*sync.Mutex // [string]mutex } +// voteInventory contains the record inventory catagorized by vote status. The +// authorized and started lists are updated in real-time since ticket vote +// plugin commands initiate those actions. The unauthorized and finished lists +// are lazy loaded since those lists depends on external state. type voteInventory struct { unauthorized []string // Unauthorized tokens authorized []string // Authorized tokens @@ -137,12 +141,16 @@ func (p *ticketVotePlugin) inventorySetToAuthorized(token string) { u := p.inv.unauthorized u = append(u[:i], u[i+1:]...) p.inv.unauthorized = u + + log.Debugf("ticketvote: removed from unauthorized inv: %v", token) } // Prepend the token to the authorized list a := p.inv.authorized a = append([]string{token}, a...) p.inv.authorized = a + + log.Debugf("ticketvote: added to authorized inv: %v", token) } func (p *ticketVotePlugin) inventorySetToUnauthorized(token string) { @@ -166,12 +174,16 @@ func (p *ticketVotePlugin) inventorySetToUnauthorized(token string) { a := p.inv.authorized a = append(a[:i], a[i+1:]...) p.inv.authorized = a + + log.Debugf("ticketvote: removed from authorized inv: %v", token) } // Prepend the token to the unauthorized list u := p.inv.unauthorized u = append([]string{token}, u...) p.inv.unauthorized = u + + log.Debugf("ticketvote: added to unauthorized inv: %v", token) } func (p *ticketVotePlugin) inventorySetToStarted(token string, endHeight uint32) { @@ -199,8 +211,12 @@ func (p *ticketVotePlugin) inventorySetToStarted(token string, endHeight uint32) a = append(a[:i], a[i+1:]...) p.inv.authorized = a + log.Debugf("ticketvote: removed from authorized inv: %v", token) + // Add the token to the started map p.inv.started[token] = endHeight + + log.Debugf("ticketvote: added to started inv: %v", token) } func (p *ticketVotePlugin) inventory(bestBlock uint32) (*voteInventory, error) { @@ -245,13 +261,16 @@ func (p *ticketVotePlugin) inventory(bestBlock uint32) (*voteInventory, error) { // since it would have already been added to the vote inventory // during the authorization request if one had occurred. p.inv.unauthorized = append(p.inv.unauthorized, v) + + log.Debugf("ticketvote: added to unauthorized inv: %v", v) } } // The records are moved to their correct vote status category in // the inventory on authorization, revoking the authorization, and - // on starting the vote. The last thing we must check for is - // whether any votes have finished since the last inventory update. + // on starting the vote. We can assume these lists are already + // up-to-date. The last thing we must check for is whether any + // votes have finished since the last inventory update. // Check if the inventory has been updated for this block height. if p.inv.bestBlock == bestBlock { @@ -263,17 +282,24 @@ func (p *ticketVotePlugin) inventory(bestBlock uint32) (*voteInventory, error) { // any proposal votes have finished. for token, endHeight := range p.inv.started { if bestBlock >= endHeight { - // Vote has finished - p.inv.finished = append(p.inv.finished, token) + // Vote has finished. Remove it from the started list. delete(p.inv.started, token) + + log.Debugf("ticketvote: removed from started inv: %v", token) + + // Add it to the finished list + p.inv.finished = append(p.inv.finished, token) + + log.Debugf("ticketvote: added to finished inv: %v", token) } } // Update best block p.inv.bestBlock = bestBlock + log.Debugf("ticketvote: inv updated for best block %v", bestBlock) + reply: - // TODO make this better // Return a copy of the inventory var ( unauthorized = make([]string, len(p.inv.unauthorized)) @@ -281,18 +307,12 @@ reply: started = make(map[string]uint32, len(p.inv.started)) finished = make([]string, len(p.inv.finished)) ) - for k, v := range p.inv.unauthorized { - unauthorized[k] = v - } - for k, v := range p.inv.authorized { - authorized[k] = v - } + copy(unauthorized, p.inv.unauthorized) + copy(authorized, p.inv.authorized) + copy(finished, p.inv.finished) for k, v := range p.inv.started { started[k] = v } - for k, v := range p.inv.finished { - finished[k] = v - } return &voteInventory{ unauthorized: unauthorized, @@ -331,7 +351,8 @@ func (p *ticketVotePlugin) cachedVotesSet(token, ticket, voteBit string) { p.votes[token][ticket] = voteBit - log.Debugf("Votes add: %v %v %v", token, ticket, voteBit) + log.Debugf("ticketvote: added vote to cache: %v %v %v", + token, ticket, voteBit) } func (p *ticketVotePlugin) cachedVotesDel(token string) { @@ -340,7 +361,7 @@ func (p *ticketVotePlugin) cachedVotesDel(token string) { delete(p.votes, token) - log.Debugf("Votes del: %v", token) + log.Debugf("ticketvote: deleted votes cache: %v", token) } func (p *ticketVotePlugin) cachedSummaryPath(token string) string { @@ -387,6 +408,8 @@ func (p *ticketVotePlugin) cachedSummarySave(token string, s ticketvote.Summary) return err } + log.Debugf("ticketvote: saved votes summary: %v", token) + return nil } @@ -2066,7 +2089,7 @@ func (p *ticketVotePlugin) setup() error { } // Build inventory cache - log.Debugf("ticketvote: building inventory cache") + log.Infof("ticketvote: building inventory cache") ibs, err := p.backend.InventoryByStatus() if err != nil { @@ -2117,14 +2140,10 @@ func (p *ticketVotePlugin) setup() error { } p.Unlock() - // Build votes cache - log.Debugf("ticketvote: building votes cache") + // Build votes cace + log.Infof("ticketvote: building votes cache") - inv, err := p.inventory(bestBlock) - if err != nil { - return fmt.Errorf("inventory: %v", err) - } - for k := range inv.started { + for k := range started { token, err := hex.DecodeString(k) if err != nil { return err @@ -2267,10 +2286,10 @@ func newTicketVotePlugin(backend backend.Backend, tlog tlogClient, settings []ba dataDir: dataDir, identity: id, inv: voteInventory{ - unauthorized: make([]string, 0, 256), - authorized: make([]string, 0, 256), - started: make(map[string]uint32, 256), - finished: make([]string, 0, 256), + unauthorized: make([]string, 0, 1024), + authorized: make([]string, 0, 1024), + started: make(map[string]uint32, 1024), + finished: make([]string, 0, 1024), bestBlock: 0, }, votes: make(map[string]map[string]string), diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index 8b5ee43ce..9df3e5aab 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -82,12 +82,12 @@ const ( // VoteOptionIDApprove is the vote option ID that indicates the vote // should be approved. Votes that are an approve/reject vote are // required to use this vote option ID. - VoteOptionIDApprove = "approve" + VoteOptionIDApprove = "yes" // VoteOptionIDReject is the vote option ID that indicates the vote // should be not be approved. Votes that are an approve/reject vote // are required to use this vote option ID. - VoteOptionIDReject = "reject" + VoteOptionIDReject = "no" // Vote error status codes. Vote errors are errors that occur while // attempting to cast a vote. These errors are returned with the diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 809bc2b2a..d2733891f 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -1028,7 +1028,7 @@ func (p *politeia) pluginCommand(w http.ResponseWriter, r *http.Request) { // Check for a user error var e backend.PluginUserError if errors.As(err, &e) { - log.Debugf("%v plugin user error: %v %v", + log.Infof("%v plugin user error: %v %v", remoteAddr(r), e.PluginID, e.ErrorCode) p.respondWithPluginUserError(w, e.PluginID, e.ErrorCode, e.ErrorContext) @@ -1053,7 +1053,7 @@ func (p *politeia) pluginCommand(w http.ResponseWriter, r *http.Request) { Payload: payload, } - log.Infof("Plugin %v: %v %v", remoteAddr(r), pc.ID, pc.Command) + log.Infof("%v Plugin cmd executed: %v %v", remoteAddr(r), pc.ID, pc.Command) util.RespondWithJSON(w, http.StatusOK, reply) } diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 38bcce305..e790e8fd5 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -113,12 +113,12 @@ const ( // VoteOptionIDApprove is the vote option ID that indicates the // proposal should be approved. Proposal votes are required to use // this vote option ID. - VoteOptionIDApprove = "approve" + VoteOptionIDApprove = "yes" // VoteOptionIDReject is the vote option ID that indicates the // proposal should be rejected. Proposal votes are required to use // this vote option ID. - VoteOptionIDReject = "reject" + VoteOptionIDReject = "no" // Error status codes ErrorStatusInvalid ErrorStatusT = 0 From bd0bd17fe5ab91a951804bb8d6fcd957181f97f1 Mon Sep 17 00:00:00 2001 From: luke Date: Sat, 17 Oct 2020 11:43:17 -0500 Subject: [PATCH 159/449] seperate unvetted and vetted censored records --- politeiad/api/v1/v1.go | 12 +- politeiad/backend/backend.go | 11 +- politeiad/backend/tlogbe/ticketvote.go | 94 +++++----- politeiad/backend/tlogbe/tlogbe.go | 199 ++++++++++++++------- politeiad/cmd/politeia/README.md | 13 +- politeiad/cmd/politeia/politeia.go | 17 +- politeiad/politeiad.go | 21 ++- politeiawww/api/pi/v1/v1.go | 35 ++-- politeiawww/cmd/piwww/proposalinventory.go | 23 ++- politeiawww/cmd/shared/client.go | 4 +- politeiawww/piwww.go | 41 +++-- politeiawww/proposals.go | 42 +++-- 12 files changed, 329 insertions(+), 183 deletions(-) diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index 0af3356ca..f8aab1e49 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -392,14 +392,12 @@ type InventoryByStatus struct { Challenge string `json:"challenge"` // Random challenge } -// InventoryByStatusReply returns all censorship record tokens by status. +// InventoryByStatusReply returns all censorship record tokens categorized by +// record state and record status. type InventoryByStatusReply struct { - Response string `json:"response"` // Challenge response - Unvetted []string `json:"unvetted"` // Unvetted tokens - IterationUnvetted []string `json:"iterationunvetted"` // Iteration unvetted tokens - Vetted []string `json:"vetted"` // Vetted tokens - Censored []string `json:"censored"` // Censored tokens - Archived []string `json:"archived"` // Archived tokens + Response string `json:"response"` // Challenge response + Unvetted map[RecordStatusT][]string `json:"unvetted"` + Vetted map[RecordStatusT][]string `json:"vetted"` } // UserErrorReply returns details about an error that occurred while trying to diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index 6beb2551b..ffb9bc888 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -168,14 +168,11 @@ type Plugin struct { } // InventoryByStatus contains the record tokens of all records in the inventory -// categorized by MDStatusT. Each list is sorted by the timestamp of the status -// change from newest to oldest. +// categorized by state and MDStatusT. Each list is sorted by the timestamp of +// the status change from newest to oldest. type InventoryByStatus struct { - Unvetted []string - IterationUnvetted []string - Vetted []string - Censored []string - Archived []string + Unvetted map[MDStatusT][]string + Vetted map[MDStatusT][]string } type Backend interface { diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index f48c45669..7e36195a2 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -109,7 +109,7 @@ type ticketVotePlugin struct { mutexes map[string]*sync.Mutex // [string]mutex } -// voteInventory contains the record inventory catagorized by vote status. The +// voteInventory contains the record inventory categorized by vote status. The // authorized and started lists are updated in real-time since ticket vote // plugin commands initiate those actions. The unauthorized and finished lists // are lazy loaded since those lists depends on external state. @@ -228,15 +228,24 @@ func (p *ticketVotePlugin) inventory(bestBlock uint32) (*voteInventory, error) { if err != nil { return nil, fmt.Errorf("InventoryByStatus: %v", err) } - var ( - vetted = invBackend.Vetted - voteInvCount = len(p.inv.unauthorized) + len(p.inv.authorized) + - len(p.inv.started) + len(p.inv.finished) - ) - if voteInvCount != len(vetted) { - // There are new records. Put all ticket vote inventory records - // into a map so we can easily find what backend records are - // missing. + + // Find number of records in the vetted inventory + var vettedInvCount int + for _, tokens := range invBackend.Vetted { + vettedInvCount += len(tokens) + } + + // Find number of records in the vote inventory + voteInvCount := len(p.inv.unauthorized) + len(p.inv.authorized) + + len(p.inv.started) + len(p.inv.finished) + + // The vetted inventory count and the vote inventory count should + // be the same. If they're not then it means we there are records + // missing from vote inventory. + if vettedInvCount != voteInvCount { + // Records are missing from the vote inventory. Put all ticket + // vote inventory records into a map so we can easily find what + // backend records are missing. all := make(map[string]struct{}, voteInvCount) for _, v := range p.inv.unauthorized { all[v] = struct{}{} @@ -252,17 +261,20 @@ func (p *ticketVotePlugin) inventory(bestBlock uint32) (*voteInventory, error) { } // Add missing records to the vote inventory - for _, v := range invBackend.Vetted { - if _, ok := all[v]; ok { - // Record is already in the vote inventory - continue + for _, tokens := range invBackend.Vetted { + for _, v := range tokens { + if _, ok := all[v]; ok { + // Record is already in the vote inventory + continue + } + // We can assume that the record vote status is unauthorized + // since it would have already been added to the vote + // inventory during the authorization request if one had + // occurred. + p.inv.unauthorized = append(p.inv.unauthorized, v) + + log.Debugf("ticketvote: added to unauthorized inv: %v", v) } - // We can assume that the record vote status is unauthorized - // since it would have already been added to the vote inventory - // during the authorization request if one had occurred. - p.inv.unauthorized = append(p.inv.unauthorized, v) - - log.Debugf("ticketvote: added to unauthorized inv: %v", v) } } @@ -2107,26 +2119,28 @@ func (p *ticketVotePlugin) setup() error { started = make(map[string]uint32, 256) // [token]endHeight finished = make([]string, 0, 256) ) - for _, v := range ibs.Vetted { - token, err := hex.DecodeString(v) - if err != nil { - return err - } - s, err := p.summary(token, bestBlock) - if err != nil { - return fmt.Errorf("summary %v: %v", v, err) - } - switch s.Status { - case ticketvote.VoteStatusUnauthorized: - unauthorized = append(unauthorized, v) - case ticketvote.VoteStatusAuthorized: - authorized = append(authorized, v) - case ticketvote.VoteStatusStarted: - started[v] = s.EndBlockHeight - case ticketvote.VoteStatusFinished: - finished = append(finished, v) - default: - return fmt.Errorf("invalid vote status %v %v", v, s.Status) + for _, tokens := range ibs.Vetted { + for _, v := range tokens { + token, err := hex.DecodeString(v) + if err != nil { + return err + } + s, err := p.summary(token, bestBlock) + if err != nil { + return fmt.Errorf("summary %v: %v", v, err) + } + switch s.Status { + case ticketvote.VoteStatusUnauthorized: + unauthorized = append(unauthorized, v) + case ticketvote.VoteStatusAuthorized: + authorized = append(authorized, v) + case ticketvote.VoteStatusStarted: + started[v] = s.EndBlockHeight + case ticketvote.VoteStatusFinished: + finished = append(finished, v) + default: + return fmt.Errorf("invalid vote status %v %v", v, s.Status) + } } } diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 965168592..4af853c75 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -33,7 +33,8 @@ import ( "github.com/subosito/gozaru" ) -// TODO testnet vs mainnet trillian databases +// TODO testnet vs mainnet trillian databases. Can I also make different dbs +// for pi and cms so we can switch back and forth without issues? // TODO fsck const ( @@ -48,6 +49,10 @@ const ( // from the politeiad config. The user does not have to set these // manually. pluginSettingDataDir = "datadir" + + // Record states + stateUnvetted = "unvetted" + stateVetted = "vetted" ) var ( @@ -113,7 +118,12 @@ type tlogBackend struct { // status. Each list of tokens is sorted by the timestamp of the // status change from newest to oldest. This cache is built on // startup. - inventory map[backend.MDStatusT][]string + inv recordInventory +} + +type recordInventory struct { + unvetted map[backend.MDStatusT][]string + vetted map[backend.MDStatusT][]string } // plugin represents a tlogbe plugin. @@ -262,38 +272,71 @@ func (t *tlogBackend) vettedTreeIDFromToken(token []byte) (int64, bool) { return vettedTreeID, true } -func (t *tlogBackend) inventoryGet() map[backend.MDStatusT][]string { +func (t *tlogBackend) inventory() recordInventory { t.RLock() defer t.RUnlock() // Return a copy of the inventory - inv := make(map[backend.MDStatusT][]string, len(t.inventory)) - for status, tokens := range t.inventory { - tokensCopy := make([]string, len(tokens)) - copy(tokensCopy, tokens) - inv[status] = tokensCopy + var ( + unvetted = make(map[backend.MDStatusT][]string, len(t.inv.unvetted)) + vetted = make(map[backend.MDStatusT][]string, len(t.inv.vetted)) + ) + for status, tokens := range t.inv.unvetted { + s := make([]string, len(tokens)) + copy(s, tokens) + unvetted[status] = s + } + for status, tokens := range t.inv.vetted { + s := make([]string, len(tokens)) + copy(s, tokens) + vetted[status] = s } - return inv + return recordInventory{ + unvetted: unvetted, + vetted: vetted, + } } -func (t *tlogBackend) inventoryAdd(token string, s backend.MDStatusT) { +func (t *tlogBackend) inventoryAdd(state string, tokenb []byte, s backend.MDStatusT) { t.Lock() defer t.Unlock() - t.inventory[s] = append([]string{token}, t.inventory[s]...) + token := hex.EncodeToString(tokenb) + switch state { + case stateUnvetted: + t.inv.unvetted[s] = append([]string{token}, t.inv.unvetted[s]...) + case stateVetted: + t.inv.vetted[s] = append([]string{token}, t.inv.vetted[s]...) + default: + e := fmt.Sprintf("unknown state '%v'", state) + panic(e) + } - log.Debugf("Add to inventory: %v %v", token, backend.MDStatus[s]) + log.Debugf("Add to inv %v: %v %v", state, token, backend.MDStatus[s]) } -func (t *tlogBackend) inventoryUpdate(token string, currStatus, newStatus backend.MDStatusT) { +func (t *tlogBackend) inventoryUpdate(state string, tokenb []byte, currStatus, newStatus backend.MDStatusT) { + token := hex.EncodeToString(tokenb) + t.Lock() defer t.Unlock() + var inv map[backend.MDStatusT][]string + switch state { + case stateUnvetted: + inv = t.inv.unvetted + case stateVetted: + inv = t.inv.vetted + default: + e := fmt.Sprintf("unknown state '%v'", state) + panic(e) + } + // Find the index of the token in its current status list var idx int var found bool - for k, v := range t.inventory[currStatus] { + for k, v := range inv[currStatus] { if v == token { // Token found idx = k @@ -309,14 +352,55 @@ func (t *tlogBackend) inventoryUpdate(token string, currStatus, newStatus backen } // Remove the token from its current status list - tokens := t.inventory[currStatus] - t.inventory[currStatus] = append(tokens[:idx], tokens[idx+1:]...) + tokens := inv[currStatus] + inv[currStatus] = append(tokens[:idx], tokens[idx+1:]...) // Prepend token to new status - t.inventory[newStatus] = append([]string{token}, t.inventory[newStatus]...) + inv[newStatus] = append([]string{token}, inv[newStatus]...) - log.Debugf("Update inventory: %v %v to %v", - token, backend.MDStatus[currStatus], backend.MDStatus[newStatus]) + log.Debugf("Update inv %v: %v %v to %v", state, token, + backend.MDStatus[currStatus], backend.MDStatus[newStatus]) +} + +// inventoryMoveToVetted moves a token from the unvetted inventory to the +// vetted inventory. The unvettedStatus is the status of the record prior to +// the update and the vettedStatus is the status of the record after the +// update. +func (t *tlogBackend) inventoryMoveToVetted(tokenb []byte, unvettedStatus, vettedStatus backend.MDStatusT) { + t.Lock() + defer t.Unlock() + + token := hex.EncodeToString(tokenb) + unvetted := t.inv.unvetted + vetted := t.inv.vetted + + // Find the index of the token in its current status list + var idx int + var found bool + for k, v := range unvetted[unvettedStatus] { + if v == token { + // Token found + idx = k + found = true + break + } + } + if !found { + // Token was never found. This should not happen. + e := fmt.Sprintf("inventoryMoveToVetted: unvetted token not found: %v %v", + token, unvettedStatus) + panic(e) + } + + // Remove the token from the unvetted status list + tokens := unvetted[unvettedStatus] + unvetted[unvettedStatus] = append(tokens[:idx], tokens[idx+1:]...) + + // Prepend token to vetted status + vetted[vettedStatus] = append([]string{token}, vetted[vettedStatus]...) + + log.Debugf("Inv move to vetted: %v %v to %v", token, + backend.MDStatus[unvettedStatus], backend.MDStatus[vettedStatus]) } // verifyContent verifies that all provided MetadataStream and File are sane. @@ -672,7 +756,7 @@ func (t *tlogBackend) New(metadata []backend.MetadataStream, files []backend.Fil } // Update the inventory cache - t.inventoryAdd(hex.EncodeToString(token), backend.MDStatusUnvetted) + t.inventoryAdd(stateUnvetted, token, backend.MDStatusUnvetted) log.Infof("New record %x", token) @@ -1219,7 +1303,7 @@ func (t *tlogBackend) unvettedCensor(token []byte, rm backend.RecordMetadata, me return fmt.Errorf("treeFreeze %v: %v", treeID, err) } - log.Debugf("Unvetted record frozen %v", token) + log.Debugf("Unvetted record frozen %x", token) // Delete all record files err = t.unvetted.recordDel(treeID) @@ -1227,7 +1311,7 @@ func (t *tlogBackend) unvettedCensor(token []byte, rm backend.RecordMetadata, me return fmt.Errorf("recordDel %v: %v", treeID, err) } - log.Debugf("Unvetted record files deleted %v", token) + log.Debugf("Unvetted record files deleted %x", token) return nil } @@ -1322,32 +1406,26 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, token, err) } - // Update inventory cache - t.inventoryUpdate(rm.Token, currStatus, status) - log.Debugf("Status change %x from %v (%v) to %v (%v)", token, backend.MDStatus[currStatus], currStatus, backend.MDStatus[status], status) + // Update inventory cache + if status == backend.MDStatusVetted { + // Record was made public + t.inventoryMoveToVetted(token, currStatus, status) + } else { + // All other status changes + t.inventoryUpdate(stateUnvetted, token, currStatus, status) + } + // Return the updated record. If the record was made public it is // now a vetted record and must be fetched accordingly. - switch status { - case backend.MDStatusVetted: - r, err = t.GetVetted(token, "") - if err != nil { - return nil, fmt.Errorf("GetVetted: %v", err) - } - case backend.MDStatusCensored: - r, err = t.GetUnvetted(token, "") - if err != nil { - return nil, fmt.Errorf("GetUnvetted: %v", err) - } - default: - return nil, fmt.Errorf("unknown status: %v (%v)", - backend.MDStatus[status], status) + if status == backend.MDStatusVetted { + return t.GetVetted(token, "") } - return r, nil + return t.GetUnvetted(token, "") } // This function must be called WITH the vetted lock held. @@ -1362,7 +1440,7 @@ func (t *tlogBackend) vettedCensor(token []byte, rm backend.RecordMetadata, meta return fmt.Errorf("treeFreeze %v: %v", treeID, err) } - log.Debugf("Vetted record frozen %v", token) + log.Debugf("Vetted record frozen %x", token) // Delete all record files err = t.vetted.recordDel(treeID) @@ -1370,7 +1448,7 @@ func (t *tlogBackend) vettedCensor(token []byte, rm backend.RecordMetadata, meta return fmt.Errorf("recordDel %v: %v", treeID, err) } - log.Debugf("Vetted record files deleted %v", token) + log.Debugf("Vetted record files deleted %x", token) return nil } @@ -1388,7 +1466,7 @@ func (t *tlogBackend) vettedArchive(token []byte, rm backend.RecordMetadata, met return fmt.Errorf("treeFreeze %v: %v", treeID, err) } - log.Debugf("Vetted record frozen %v", token) + log.Debugf("Vetted record frozen %x", token) return nil } @@ -1485,7 +1563,7 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md } // Update inventory cache - t.inventoryUpdate(rm.Token, currStatus, status) + t.inventoryUpdate(stateVetted, token, currStatus, status) log.Debugf("Status change %x from %v (%v) to %v (%v)", token, backend.MDStatus[currStatus], currStatus, @@ -1518,13 +1596,10 @@ func (t *tlogBackend) Inventory(vettedCount, vettedStart, unvettedCount uint, in func (t *tlogBackend) InventoryByStatus() (*backend.InventoryByStatus, error) { log.Tracef("InventoryByStatus") - inv := t.inventoryGet() + inv := t.inventory() return &backend.InventoryByStatus{ - Unvetted: inv[backend.MDStatusUnvetted], - IterationUnvetted: inv[backend.MDStatusIterationUnvetted], - Vetted: inv[backend.MDStatusVetted], - Censored: inv[backend.MDStatusCensored], - Archived: inv[backend.MDStatusArchived], + Unvetted: inv.unvetted, + Vetted: inv.vetted, }, nil } @@ -1698,18 +1773,27 @@ func (t *tlogBackend) setup() error { // Sanity check e := fmt.Sprintf("records is both unvetted and vetted: %x", token) panic(e) + case isUnvetted: - // Record is unvetted + // Get unvetted record r, err = t.GetUnvetted(token, "") if err != nil { return fmt.Errorf("GetUnvetted %x: %v", token, err) } + + // Add record to the inventory cache + t.inventoryAdd(stateUnvetted, token, r.RecordMetadata.Status) + case isVetted: - // Record is vetted + // Get vetted record r, err = t.GetVetted(token, "") if err != nil { return fmt.Errorf("GetUnvetted %x: %v", token, err) } + + // Add record to the inventory cache + t.inventoryAdd(stateVetted, token, r.RecordMetadata.Status) + default: // This is an empty tree. This can happen if there was an error // during record creation and the record failed to be appended @@ -1717,8 +1801,6 @@ func (t *tlogBackend) setup() error { log.Debugf("Empty tree found for token %x", token) } - // Add record to the inventory cache - t.inventoryAdd(hex.EncodeToString(token), r.RecordMetadata.Status) } return nil @@ -1776,12 +1858,9 @@ func New(anp *chaincfg.Params, homeDir, dataDir, dcrtimeHost, encryptionKeyFile, plugins: make(map[string]plugin), prefixes: make(map[string][]byte), vettedTreeIDs: make(map[string]int64), - inventory: map[backend.MDStatusT][]string{ - backend.MDStatusUnvetted: make([]string, 0, 256), - backend.MDStatusIterationUnvetted: make([]string, 0, 256), - backend.MDStatusVetted: make([]string, 0, 256), - backend.MDStatusCensored: make([]string, 0, 256), - backend.MDStatusArchived: make([]string, 0, 256), + inv: recordInventory{ + unvetted: make(map[backend.MDStatusT][]string), + vetted: make(map[backend.MDStatusT][]string), }, } diff --git a/politeiad/cmd/politeia/README.md b/politeiad/cmd/politeia/README.md index 17c5c6068..9234ae42b 100644 --- a/politeiad/cmd/politeia/README.md +++ b/politeiad/cmd/politeia/README.md @@ -231,10 +231,11 @@ categorized by their record status. ``` $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass inventory -Inventory by status: - Unvetted : [tokens...] - IterationUnvetted: [tokens...] - Vetted : [tokens...] - Censored : [tokens...] - Archived : [tokens...] +Inventory: + Unvetted + not reviewed : 344044686e9ba76f0000, 43df32e2405065250000 + censored : 9f104b878a83242d0000 + Vetted + public : e2228e7a7e7c80030000, a6c064874351f4120000 + archived : 6b099750984e05490000 ``` diff --git a/politeiad/cmd/politeia/politeia.go b/politeiad/cmd/politeia/politeia.go index 57d0d06f0..60a69c520 100644 --- a/politeiad/cmd/politeia/politeia.go +++ b/politeiad/cmd/politeia/politeia.go @@ -443,12 +443,17 @@ func recordInventory() error { } // Print response to user - fmt.Printf("Inventory by status:\n") - fmt.Printf(" Unvetted : %v\n", ibsr.Unvetted) - fmt.Printf(" IterationUnvetted: %v\n", ibsr.IterationUnvetted) - fmt.Printf(" Vetted : %v\n", ibsr.Vetted) - fmt.Printf(" Censored : %v\n", ibsr.Censored) - fmt.Printf(" Archived : %v\n", ibsr.Archived) + fmt.Printf("Inventory:\n") + fmt.Printf(" Unvetted\n") + for status, tokens := range ibsr.Unvetted { + fmt.Printf(" %-15v: %v\n", + v1.RecordStatus[status], strings.Join(tokens, ", ")) + } + fmt.Printf(" Vetted\n") + for status, tokens := range ibsr.Vetted { + fmt.Printf(" %-15v: %v\n", + v1.RecordStatus[status], strings.Join(tokens, ", ")) + } return nil } diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index d2733891f..00f65907d 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -644,7 +644,7 @@ func (p *politeia) inventoryByStatus(w http.ResponseWriter, r *http.Request) { return } - ps, err := p.backend.InventoryByStatus() + inv, err := p.backend.InventoryByStatus() if err != nil { // Generic internal error. errorCode := time.Now().Unix() @@ -657,13 +657,20 @@ func (p *politeia) inventoryByStatus(w http.ResponseWriter, r *http.Request) { // Prepare reply response := p.identity.SignMessage(challenge) + var ( + unvetted = make(map[v1.RecordStatusT][]string) + vetted = make(map[v1.RecordStatusT][]string) + ) + for status, tokens := range inv.Unvetted { + unvetted[convertBackendStatus(status)] = tokens + } + for status, tokens := range inv.Vetted { + vetted[convertBackendStatus(status)] = tokens + } reply := v1.InventoryByStatusReply{ - Response: hex.EncodeToString(response[:]), - Unvetted: ps.Unvetted, - IterationUnvetted: ps.IterationUnvetted, - Vetted: ps.Vetted, - Censored: ps.Censored, - Archived: ps.Archived, + Response: hex.EncodeToString(response[:]), + Unvetted: unvetted, + Vetted: vetted, } util.RespondWithJSON(w, http.StatusOK, reply) diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index e790e8fd5..6fae7cca0 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -67,11 +67,11 @@ const ( PropStateVetted PropStateT = 2 // Proposal statuses - PropStatusInvalid PropStatusT = 0 // Invalid status - PropStatusUnvetted PropStatusT = 1 // Prop has not been vetted - PropStatusPublic PropStatusT = 2 // Prop has been made public - PropStatusCensored PropStatusT = 3 // Prop has been censored - PropStatusAbandoned PropStatusT = 4 // Prop has been abandoned + PropStatusInvalid PropStatusT = 0 // Invalid status + PropStatusUnreviewed PropStatusT = 1 // Prop has not been reviewed + PropStatusPublic PropStatusT = 2 // Prop has been made public + PropStatusCensored PropStatusT = 3 // Prop has been censored + PropStatusAbandoned PropStatusT = 4 // Prop has been abandoned // Comment vote types CommentVoteInvalid CommentVoteT = 0 @@ -191,6 +191,15 @@ const ( ) var ( + // PropStatus contains the human readable proposal statuses. + PropStatus = map[PropStatusT]string{ + PropStatusInvalid: "invalid", + PropStatusUnreviewed: "unreviewed", + PropStatusPublic: "public", + PropStatusCensored: "censored", + PropStatusAbandoned: "abandoned", + } + // ErrorStatus contains human readable error messages. // TODO fill in error status messages ErrorStatus = map[ErrorStatusT]string{ @@ -461,16 +470,18 @@ type ProposalsReply struct { } // ProposalInventory retrieves the tokens of all proposals in the inventory, -// categorized by proposal status and ordered by timestamp of the status change -// from newest to oldest. +// categorized by proposal staet and proposal status. Each list is ordered by +// timestamp of the status change from newest to oldest. Unvetted proposal +// tokens are only returned to admins. type ProposalInventory struct{} -// ProposalInventoryReply is the reply to the ProposalInventory command. +// ProposalInventoryReply is the reply to the ProposalInventory command. The +// inventory maps contain map[status][]tokens where the status is the human +// readable proposal status, as defined by the PropStatus map, and the tokens +// are the proposal tokens for that status. type ProposalInventoryReply struct { - Unvetted []string `json:"unvetted,omitempty"` - Public []string `json:"public"` - Censored []string `json:"censored"` - Abandoned []string `json:"abandoned"` + Unvetted map[string][]string `json:"unvetted,omitempty"` + Vetted map[string][]string `json:"vetted"` } // Comment represent a proposal comment. diff --git a/politeiawww/cmd/piwww/proposalinventory.go b/politeiawww/cmd/piwww/proposalinventory.go index a8131d5b4..96fa720e9 100644 --- a/politeiawww/cmd/piwww/proposalinventory.go +++ b/politeiawww/cmd/piwww/proposalinventory.go @@ -5,25 +5,36 @@ package main import ( + pi "github.com/decred/politeia/politeiawww/api/pi/v1" "github.com/decred/politeia/politeiawww/cmd/shared" ) -// proposalInventoryCmd retrieves the censorship record tokens of all proposals in -// the inventory. +// proposalInventoryCmd retrieves the censorship record tokens of all proposals +// in the inventory. type proposalInventoryCmd struct{} // Execute executes the proposal inventory command. func (cmd *proposalInventoryCmd) Execute(args []string) error { - reply, err := client.ProposalInventory() + p := pi.ProposalInventory{} + err := shared.PrintJSON(p) if err != nil { return err } - return shared.PrintJSON(reply) + pir, err := client.ProposalInventory(p) + if err != nil { + return err + } + err = shared.PrintJSON(pir) + if err != nil { + return err + } + return nil } // proposalInventoryHelpMsg is the command help message. const proposalInventoryHelpMsg = `proposalinv Fetch the censorship record tokens for all proposals, categorized by their -proposal status. The unvetted tokens are only returned if the logged in user is -an admin.` +proposal state and proposal status. Unvetted tokens are only returned if the +logged in user is an admin. +` diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index 818232caf..44aea8f08 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -951,9 +951,9 @@ func (c *Client) Proposals(p pi.Proposals) (*pi.ProposalsReply, error) { // ProposalInventory retrieves the censorship tokens of all proposals, // separated by their status. -func (c *Client) ProposalInventory() (*pi.ProposalInventoryReply, error) { +func (c *Client) ProposalInventory(p pi.ProposalInventory) (*pi.ProposalInventoryReply, error) { statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, - pi.RouteProposalInventory, nil) + pi.RouteProposalInventory, p) if err != nil { return nil, err } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 86ae58f73..21f9afbae 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -163,7 +163,7 @@ func convertUserErrorFromSignatureError(err error) pi.UserErrorReply { func convertPropStateFromPropStatus(s pi.PropStatusT) pi.PropStateT { switch s { - case pi.PropStatusUnvetted, pi.PropStatusCensored: + case pi.PropStatusUnreviewed, pi.PropStatusCensored: return pi.PropStateUnvetted case pi.PropStatusPublic, pi.PropStatusAbandoned: return pi.PropStateVetted @@ -183,7 +183,7 @@ func convertPropStateFromPi(s pi.PropStateT) piplugin.PropStateT { func convertRecordStatusFromPropStatus(s pi.PropStatusT) pd.RecordStatusT { switch s { - case pi.PropStatusUnvetted: + case pi.PropStatusUnreviewed: return pd.RecordStatusNotReviewed case pi.PropStatusPublic: return pd.RecordStatusPublic @@ -231,13 +231,13 @@ func convertPropStatusFromPD(s pd.RecordStatusT) pi.PropStatusT { case pd.RecordStatusNotFound: // Intentionally omitted. No corresponding PropStatusT. case pd.RecordStatusNotReviewed: - return pi.PropStatusUnvetted + return pi.PropStatusUnreviewed case pd.RecordStatusCensored: return pi.PropStatusCensored case pd.RecordStatusPublic: return pi.PropStatusPublic case pd.RecordStatusUnreviewedChanges: - return pi.PropStatusUnvetted + return pi.PropStatusUnreviewed case pd.RecordStatusArchived: return pi.PropStatusAbandoned } @@ -1306,7 +1306,7 @@ func (p *politeiawww) processProposalEdit(ctx context.Context, pe pi.ProposalEdi // Verify proposal status switch curr.Status { - case pi.PropStatusUnvetted, pi.PropStatusPublic: + case pi.PropStatusUnreviewed, pi.PropStatusPublic: // Allowed; continue default: return nil, pi.UserErrorReply{ @@ -1540,26 +1540,34 @@ func (p *politeiawww) processProposals(ctx context.Context, ps pi.Proposals, isA } func (p *politeiawww) processProposalInventory(ctx context.Context, isAdmin bool) (*pi.ProposalInventoryReply, error) { - log.Tracef("processProposalInventory") + log.Tracef("processProposalInventory: %v", isAdmin) ir, err := p.inventoryByStatus(ctx) if err != nil { return nil, err } - reply := pi.ProposalInventoryReply{ - Unvetted: append(ir.Unvetted, ir.IterationUnvetted...), - Public: ir.Vetted, - Censored: ir.Censored, - Abandoned: ir.Archived, + var ( + unvetted = make(map[string][]string, len(ir.Unvetted)) + vetted = make(map[string][]string, len(ir.Vetted)) + ) + for status, tokens := range ir.Unvetted { + s := convertPropStatusFromPD(status) + unvetted[pi.PropStatus[s]] = tokens + } + for status, tokens := range ir.Vetted { + s := convertPropStatusFromPD(status) + vetted[pi.PropStatus[s]] = tokens } - // Remove unvetted data from non-admin users + // Only return unvetted tokens to admins if !isAdmin { - reply.Unvetted = []string{} - reply.Censored = []string{} + unvetted = nil } - return &reply, nil + return &pi.ProposalInventoryReply{ + Unvetted: unvetted, + Vetted: vetted, + }, nil } func (p *politeiawww) processCommentNew(ctx context.Context, cn pi.CommentNew, usr user.User) (*pi.CommentNewReply, error) { @@ -1831,6 +1839,9 @@ func (p *politeiawww) processVoteAuthorize(ctx context.Context, va pi.VoteAuthor } } + // TODO Verify user is the proposal author. Hmmm I think the userID + // probably needs to be attached to the vote authorization. + // Send plugin command ar, err := p.voteAuthorize(ctx, convertVoteAuthorizeFromPi(va)) if err != nil { diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index c7b33014f..846161a84 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -10,6 +10,7 @@ import ( "strconv" "github.com/decred/politeia/decredplugin" + pd "github.com/decred/politeia/politeiad/api/v1" piplugin "github.com/decred/politeia/politeiad/plugins/pi" ticketvote "github.com/decred/politeia/politeiad/plugins/ticketvote" pi "github.com/decred/politeia/politeiawww/api/pi/v1" @@ -391,34 +392,45 @@ func (p *politeiawww) processTokenInventory(ctx context.Context, isAdmin bool) ( log.Tracef("processTokenInventory") // Get record inventory - ri, err := p.inventoryByStatus(ctx) + ir, err := p.inventoryByStatus(ctx) if err != nil { return nil, err } // Get vote inventory - vi, err := p.piVoteInventory(ctx) + vir, err := p.piVoteInventory(ctx) if err != nil { return nil, err } - // Prepare reply - tir := www.TokenInventoryReply{ - Pre: append(vi.Unauthorized, vi.Authorized...), - Active: vi.Started, - Approved: vi.Approved, - Rejected: vi.Rejected, - Abandoned: ri.Archived, - } - if isAdmin { - tir.Unreviewed = ri.Unvetted - tir.Censored = ri.Censored - } + // Unpack record inventory + var ( + archived = ir.Vetted[pd.RecordStatusArchived] + unvetted = ir.Unvetted[pd.RecordStatusNotReviewed] + unvettedChanges = ir.Unvetted[pd.RecordStatusUnreviewedChanges] + unreviewed = append(unvetted, unvettedChanges...) + censored = ir.Unvetted[pd.RecordStatusCensored] + ) - return &tir, nil + // Only return unvetted tokens to admins + if isAdmin { + unreviewed = nil + censored = nil + } + + return &www.TokenInventoryReply{ + Unreviewed: unreviewed, + Censored: censored, + Pre: append(vir.Unauthorized, vir.Authorized...), + Active: vir.Started, + Approved: vir.Approved, + Rejected: vir.Rejected, + Abandoned: archived, + }, nil } /* +// TODO remove old vote code func (p *politeiawww) processAuthorizeVote(av www.AuthorizeVote, u *user.User) (*www.AuthorizeVoteReply, error) { // Make sure token is valid and not a prefix if !tokenIsValid(av.Token) { From c949d31089afdd99af8f504f9c801a78ecea495b Mon Sep 17 00:00:00 2001 From: luke Date: Sat, 17 Oct 2020 16:44:36 -0500 Subject: [PATCH 160/449] cast votes concurrently --- politeiad/backend/tlogbe/ticketvote.go | 337 +++++++++++++++++-------- politeiad/backend/tlogbe/tlogbe.go | 10 + 2 files changed, 243 insertions(+), 104 deletions(-) diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index 7e36195a2..783d7bfaf 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -64,9 +64,6 @@ var ( // is to check if the record exists in the mutexes function to ensure a token // is valid before holding the lock on it. -// TODO the bottleneck for casting a large ballot of votes is waiting for the -// log signer. Break the cast votes up and send them concurrently. - // TODO should start and startrunoff be combined into a single command? // ticketVotePlugin satisfies the pluginClient interface. @@ -1495,42 +1492,62 @@ func (p *ticketVotePlugin) cmdStartRunoff(payload string) (string, error) { return "", nil } -// ballotExitWithErr applies the provided vote error to each of the cast vote -// replies then returns the encoded ballot reply. -func ballotExitWithErr(votes []ticketvote.CastVote, errCode ticketvote.VoteErrorT, errContext string) (string, error) { - token := votes[0].Token - receipts := make([]ticketvote.CastVoteReply, len(votes)) - for k, v := range votes { - // Its possible that cast votes were provided for different - // records. This is not allowed. Verify the token is the same - // before applying the provided error. - if v.Token != token { - // Token is not the same. Use multiple record vote error. - e := ticketvote.VoteErrorMultipleRecordVotes - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = ticketvote.VoteError[e] - continue - } +// ballot casts the provided votes concurrently. The vote results are passed +// back through the results channel to the calling function. This function +// waits until all provided votes have been cast before returning. +// +// This function must be called WITH the record lock held. +func (p *ticketVotePlugin) ballot(votes []ticketvote.CastVote, results chan ticketvote.CastVoteReply) { + // Cast the votes concurrently + var wg sync.WaitGroup + for _, v := range votes { + // Increment the wait group counter + wg.Add(1) + + go func(v ticketvote.CastVote) { + // Decrement wait group counter once vote is cast + defer wg.Done() + + // Setup cast vote details + receipt := p.identity.SignMessage([]byte(v.Signature)) + cv := ticketvote.CastVoteDetails{ + Token: v.Token, + Ticket: v.Ticket, + VoteBit: v.VoteBit, + Signature: v.Signature, + Receipt: hex.EncodeToString(receipt[:]), + } - // Use the provided vote error - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = errCode - receipts[k].ErrorContext = errContext - } + // Save cast vote + var cvr ticketvote.CastVoteReply + err := p.castVoteSave(cv) + if err != nil { + t := time.Now().Unix() + log.Errorf("cmdBallot: castVoteSave %v: %v", t, err) + e := ticketvote.VoteErrorInternalError + cvr.Ticket = v.Ticket + cvr.ErrorCode = e + cvr.ErrorContext = fmt.Sprintf("%v: %v", + ticketvote.VoteError[e], t) + return + } - // Prepare reply - br := ticketvote.BallotReply{ - Receipts: receipts, - } - reply, err := ticketvote.EncodeBallotReply(br) - if err != nil { - return "", err + // Update receipt + cvr.Ticket = v.Ticket + cvr.Receipt = cv.Receipt + + // Update cast votes cache + p.cachedVotesSet(v.Token, v.Ticket, v.VoteBit) + + // Send result back to calling function + results <- cvr + }(v) } - return string(reply), nil + + // Wait for the full ballot to be cast before returning. + wg.Wait() } -// TODO test this when casting large blocks of votes // cmdBallot casts a ballot of votes. This function will not return a user // error if one occurs. It will instead return the ballot reply with the error // included in the invidiual cast vote reply that it applies to. @@ -1557,44 +1574,62 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { return string(reply), nil } - // Verify token - token, err := hex.DecodeString(votes[0].Token) - if err != nil { - e := ticketvote.VoteErrorTokenInvalid - c := fmt.Sprintf("%v: not hex", ticketvote.VoteError[e]) - return ballotExitWithErr(votes, e, c) + // Verify that all tokens in the ballot are valid, full length + // tokens and that they are all voting for the same record. + var ( + token []byte + receipts = make([]ticketvote.CastVoteReply, len(votes)) + ) + for k, v := range votes { + // Verify token + t, err := decodeTokenFullLength(v.Token) + if err != nil { + e := ticketvote.VoteErrorTokenInvalid + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = fmt.Sprintf("%v: not hex", + ticketvote.VoteError[e]) + continue + } + if token == nil { + // Set token to the first valid one we come across. All votes + // in the ballot with a valid token are required to be the same + // as this token. + token = t + } + + // Verify token is the same + if !bytes.Equal(t, token) { + e := ticketvote.VoteErrorMultipleRecordVotes + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = ticketvote.VoteError[e] + continue + } } - // Verify record vote status - vd, err := p.voteDetails(token) + // From this point forward, it can be assumed that all votes that + // have not had their error set are voting for the same record. Get + // the record and vote data that we need to perform the remaining + // inexpensive validation before we have to hold the lock. + voteDetails, err := p.voteDetails(token) if err != nil { return "", err } - if vd == nil { - // Vote has not been started yet - e := ticketvote.VoteErrorVoteStatusInvalid - c := fmt.Sprintf("%v: vote not started", ticketvote.VoteError[e]) - return ballotExitWithErr(votes, e, c) - } - bb, err := p.bestBlock() + bestBlock, err := p.bestBlock() if err != nil { return "", err } - if bb >= vd.EndBlockHeight { - // Vote has ended - e := ticketvote.VoteErrorVoteStatusInvalid - c := fmt.Sprintf("%v: vote has ended", ticketvote.VoteError[e]) - return ballotExitWithErr(votes, e, c) - } - // Put eligible tickets in a map for easy lookups - eligible := make(map[string]struct{}, len(vd.EligibleTickets)) - for _, v := range vd.EligibleTickets { + // eligible contains the ticket hashes of all eligble tickets. They + // are put into a map for O(n) lookups. + eligible := make(map[string]struct{}, len(voteDetails.EligibleTickets)) + for _, v := range voteDetails.EligibleTickets { eligible[v] = struct{}{} } - // Obtain largest commitment addresses for each ticket. The vote - // must be signed using the largest commitment address. + // addrs contains the largest commitment addresses for each ticket. + // The vote must be signed using the largest commitment address. tickets := make([]string, 0, len(ballot.Votes)) for _, v := range ballot.Votes { tickets = append(tickets, v.Ticket) @@ -1604,26 +1639,30 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { return "", fmt.Errorf("largestCommitmentAddrs: %v", err) } - // The lock must be held for the remainder of the function to - // ensure duplicate votes cannot be cast. - m := p.mutex(hex.EncodeToString(token)) - m.Lock() - defer m.Unlock() - - // castVotes contains the tickets that have alread voted - castVotes := p.cachedVotes(token) - - // Verify and save votes - receipts := make([]ticketvote.CastVoteReply, len(votes)) + // Perform validation that doesn't require holding the record lock. for k, v := range votes { - // Set receipt ticket - receipts[k].Ticket = v.Ticket + if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { + // Vote has an error. Skip it. + continue + } - // Verify token is the same - if v.Token != hex.EncodeToString(token) { - e := ticketvote.VoteErrorMultipleRecordVotes + // Verify record vote status + if voteDetails == nil { + // Vote has not been started yet + e := ticketvote.VoteErrorVoteStatusInvalid + receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e - receipts[k].ErrorContext = ticketvote.VoteError[e] + receipts[k].ErrorContext = fmt.Sprintf("%v: vote not started", + ticketvote.VoteError[e]) + continue + } + if bestBlock >= voteDetails.EndBlockHeight { + // Vote has ended + e := ticketvote.VoteErrorVoteStatusInvalid + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = fmt.Sprintf("%v: vote has ended", + ticketvote.VoteError[e]) continue } @@ -1631,13 +1670,16 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { bit, err := strconv.ParseUint(v.VoteBit, 16, 64) if err != nil { e := ticketvote.VoteErrorVoteBitInvalid + receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e receipts[k].ErrorContext = ticketvote.VoteError[e] continue } - err = voteBitVerify(vd.Params.Options, vd.Params.Mask, bit) + err = voteBitVerify(voteDetails.Params.Options, + voteDetails.Params.Mask, bit) if err != nil { e := ticketvote.VoteErrorVoteBitInvalid + receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e receipts[k].ErrorContext = fmt.Sprintf("%v: %v", ticketvote.VoteError[e], err) @@ -1645,30 +1687,33 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { } // Verify vote signature - ca := addrs[k] - if ca.ticket != v.Ticket { + commitmentAddr := addrs[k] + if commitmentAddr.ticket != v.Ticket { t := time.Now().Unix() log.Errorf("cmdBallot: commitment addr mismatch %v: %v %v", - t, ca.ticket, v.Ticket) + t, commitmentAddr.ticket, v.Ticket) e := ticketvote.VoteErrorInternalError + receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e receipts[k].ErrorContext = fmt.Sprintf("%v: %v", ticketvote.VoteError[e], t) continue } - if ca.err != nil { + if commitmentAddr.err != nil { t := time.Now().Unix() log.Errorf("cmdBallot: commitment addr error %v: %v %v", - t, ca.ticket, ca.err) + t, commitmentAddr.ticket, commitmentAddr.err) e := ticketvote.VoteErrorInternalError + receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e receipts[k].ErrorContext = fmt.Sprintf("%v: %v", ticketvote.VoteError[e], t) continue } - err = p.castVoteSignatureVerify(v, ca.addr) + err = p.castVoteSignatureVerify(v, commitmentAddr.addr) if err != nil { e := ticketvote.VoteErrorSignatureInvalid + receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e receipts[k].ErrorContext = fmt.Sprintf("%v: %v", ticketvote.VoteError[e], err) @@ -1679,46 +1724,130 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { _, ok := eligible[v.Ticket] if !ok { e := ticketvote.VoteErrorTicketNotEligible + receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e receipts[k].ErrorContext = ticketvote.VoteError[e] continue } + } + + // The record lock must be held for the remainder of the function to + // ensure duplicate votes cannot be cast. + m := p.mutex(hex.EncodeToString(token)) + m.Lock() + defer m.Unlock() + + // cachedVotes contains the tickets that have alread voted + cachedVotes := p.cachedVotes(token) + for k, v := range votes { + if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { + // Vote has an error. Skip it. + continue + } // Verify ticket has not already vote - _, ok = castVotes[v.Ticket] + _, ok := cachedVotes[v.Ticket] if ok { e := ticketvote.VoteErrorTicketAlreadyVoted + receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e receipts[k].ErrorContext = ticketvote.VoteError[e] continue } + } + + // The votes that have passed validation will be cast in batches of + // size batchSize. Each batch of votes is cast concurrently in order + // to accommodate the trillian log signer bottleneck. The log signer + // picks up queued leaves and appends them onto the trillian tree + // every xxx ms, where xxx is a configurable value on the log signer, + // but is typically a few hundred milliseconds. Lets use 200ms as an + // example. If we don't cast the votes in batches then every vote in + // the ballot will take 200 milliseconds since we wait for the leaf + // to be fully appended before considering the trillian call + // successful. A person casting hundreds of votes in a single ballot + // would cause UX issues for the all voting clients since the lock is + // held during these calls. + // + // The second variable that we must watch out for is the max trillian + // queued leaf batch size. This is also a configurable trillian value + // that represents the maximum number of leaves that can be waiting + // in the queue for all trees in the trillian instance. This value is + // typically around the order of magnitude of 1000 queued leaves. + // + // This is why a vote batch size of 5 was chosen. It is large enough + // to alleviate performance bottlenecks from the log signer interval, + // but small enough to still allow multiple records votes be held + // concurrently without running into the queued leaf batch size limit. + + // Prepare work + var ( + batchSize = 5 + batch = make([]ticketvote.CastVote, 0, batchSize) + queue = make([][]ticketvote.CastVote, 0, len(votes)/batchSize) + + ballotCount int + ) + for k, v := range votes { + if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { + // Vote has an error. Skip it. + continue + } - // Save cast vote - receipt := p.identity.SignMessage([]byte(v.Signature)) - cv := ticketvote.CastVoteDetails{ - Token: v.Token, - Ticket: v.Ticket, - VoteBit: v.VoteBit, - Signature: v.Signature, - Receipt: hex.EncodeToString(receipt[:]), + // Add vote to the current batch + batch = append(batch, v) + ballotCount++ + + if len(batch) == batchSize { + // This batch is full. Add the batch to the queue and start + // a new batch. + queue = append(queue, batch) + batch = make([]ticketvote.CastVote, 0, batchSize) } - err = p.castVoteSave(cv) - if err != nil { + } + if len(batch) != 0 { + // Add leftover batch to the queue + queue = append(queue, batch) + } + + log.Debugf("Casting %v votes in %v batches of size %v", + ballotCount, len(queue), batchSize) + + // Cast ballot in batches + results := make(chan ticketvote.CastVoteReply, ballotCount) + for i, batch := range queue { + log.Debugf("Casting vote batch %v/%v", i+1, len(queue)) + + p.ballot(batch, results) + } + + // Empty out the results channel + r := make(map[string]ticketvote.CastVoteReply, ballotCount) + close(results) + for v := range results { + r[v.Ticket] = v + } + + // Fill in the receipts + for k, v := range votes { + if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { + // Vote has an error. Skip it. + continue + } + cvr, ok := r[v.Ticket] + if !ok { t := time.Now().Unix() - log.Errorf("cmdBallot: castVoteSave %v: %v", t, err) + log.Errorf("cmdBallot: vote result not found %v: %v", t, v.Ticket) e := ticketvote.VoteErrorInternalError - receipts[k].ErrorCode = e - receipts[k].ErrorContext = fmt.Sprintf("%v: %v", + cvr.Ticket = v.Ticket + cvr.ErrorCode = e + cvr.ErrorContext = fmt.Sprintf("%v: %v", ticketvote.VoteError[e], t) continue } - // Update receipt - receipts[k].Ticket = cv.Ticket - receipts[k].Receipt = cv.Receipt - - // Update cast votes cache - p.cachedVotesSet(v.Token, v.Ticket, v.VoteBit) + // Fill in receipt + receipts[k] = cvr } // Prepare reply diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 4af853c75..0f0d35665 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -138,6 +138,16 @@ func tokenIsFullLength(token []byte) bool { return len(token) == v1.TokenSizeShort } +func decodeTokenFullLength(token string) ([]byte, error) { + t, err := hex.DecodeString(token) + if err != nil { + return nil, fmt.Errorf("invalid hex") + } + if !tokenIsFullLength(t) { + return nil, fmt.Errorf("invalid token size") + } + return t, nil +} func tokenPrefix(token []byte) string { return hex.EncodeToString(token)[:v1.TokenPrefixLength] } From a21607f4dced2d05bb79e79d0570702d43e8257c Mon Sep 17 00:00:00 2001 From: Amir Massarwa Date: Sun, 18 Oct 2020 17:35:51 +0300 Subject: [PATCH 161/449] tlogbe: Return best block in ticketvote SummariesReply. --- politeiad/backend/tlogbe/ticketvote.go | 1 + 1 file changed, 1 insertion(+) diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index 783d7bfaf..a0f667cdf 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -2137,6 +2137,7 @@ func (p *ticketVotePlugin) cmdSummaries(payload string) (string, error) { // Prepare reply sr := ticketvote.SummariesReply{ Summaries: summaries, + BestBlock: bb, } reply, err := ticketvote.EncodeSummariesReply(sr) if err != nil { From 896dfb9622376f999d300339046fa9cf3bcb8d44 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 18 Oct 2020 11:05:31 -0500 Subject: [PATCH 162/449] fix start vote end block bug --- politeiad/backend/tlogbe/ticketvote.go | 55 ++++++++++++-------------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index a0f667cdf..5e27dfede 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -999,10 +999,16 @@ func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, snapshotHeight, snapshotHash) } + // The start block height has the ticket maturity subtracted from + // it to prevent forking issues. This means we the vote starts in + // the past. The ticket maturity needs to be added to the end block + // height to correct for this. + endBlockHeight := snapshotHeight + duration + ticketMaturity + return &ticketvote.StartReply{ StartBlockHeight: snapshotHeight, StartBlockHash: snapshotHash, - EndBlockHeight: snapshotHeight + duration, + EndBlockHeight: endBlockHeight, EligibleTickets: tpr.Tickets, }, nil } @@ -2351,12 +2357,30 @@ func (p *ticketVotePlugin) fsck() error { } func newTicketVotePlugin(backend backend.Backend, tlog tlogClient, settings []backend.PluginSetting, id *identity.FullIdentity, activeNetParams *chaincfg.Params) (*ticketVotePlugin, error) { - // Unpack plugin settings + // Plugin settings var ( dataDir string voteDurationMin uint32 voteDurationMax uint32 ) + + // Set plugin settings to defaults. These will be overwritten if + // the setting was specified by the user. + switch activeNetParams.Name { + case chaincfg.MainNetParams().Name: + voteDurationMin = ticketvote.DefaultMainNetVoteDurationMin + voteDurationMax = ticketvote.DefaultMainNetVoteDurationMax + case chaincfg.TestNet3Params().Name: + voteDurationMin = ticketvote.DefaultTestNetVoteDurationMin + voteDurationMax = ticketvote.DefaultTestNetVoteDurationMax + case chaincfg.SimNetParams().Name: + voteDurationMin = ticketvote.DefaultSimNetVoteDurationMin + voteDurationMax = ticketvote.DefaultSimNetVoteDurationMax + default: + return nil, fmt.Errorf("unknown active net: %v", activeNetParams.Name) + } + + // Parse user provided plugin settings for _, v := range settings { switch v.Key { case pluginSettingDataDir: @@ -2387,33 +2411,6 @@ func newTicketVotePlugin(backend backend.Backend, tlog tlogClient, settings []ba pluginSettingDataDir) } - // Set optional plugin settings to default values if a value was - // not specified. - if voteDurationMin == 0 { - switch activeNetParams.Name { - case chaincfg.MainNetParams().Name: - voteDurationMin = ticketvote.DefaultMainNetVoteDurationMin - case chaincfg.TestNet3Params().Name: - voteDurationMin = ticketvote.DefaultTestNetVoteDurationMin - case chaincfg.SimNetParams().Name: - voteDurationMin = ticketvote.DefaultSimNetVoteDurationMin - default: - return nil, fmt.Errorf("unknown active net: %v", activeNetParams.Name) - } - } - if voteDurationMax == 0 { - switch activeNetParams.Name { - case chaincfg.MainNetParams().Name: - voteDurationMax = ticketvote.DefaultMainNetVoteDurationMax - case chaincfg.TestNet3Params().Name: - voteDurationMax = ticketvote.DefaultTestNetVoteDurationMax - case chaincfg.SimNetParams().Name: - voteDurationMax = ticketvote.DefaultSimNetVoteDurationMax - default: - return nil, fmt.Errorf("unknown active net: %v", activeNetParams.Name) - } - } - // Create the plugin data directory dataDir = filepath.Join(dataDir, ticketvote.ID) err := os.MkdirAll(dataDir, 0700) From 22c2e41bee8c14a86e5549dbb5f932fe40525b93 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 18 Oct 2020 13:59:48 -0500 Subject: [PATCH 163/449] fix trillian inclusion proof bug --- politeiad/backend/tlogbe/ticketvote.go | 17 ++- politeiad/backend/tlogbe/trillian.go | 51 ++++---- politeiawww/cmd/piwww/voteballot.go | 163 ++++++++++++------------- politeiawww/cmd/shared/client.go | 6 +- 4 files changed, 124 insertions(+), 113 deletions(-) diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index 5e27dfede..81bf65c97 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -1535,7 +1535,7 @@ func (p *ticketVotePlugin) ballot(votes []ticketvote.CastVote, results chan tick cvr.ErrorCode = e cvr.ErrorContext = fmt.Sprintf("%v: %v", ticketvote.VoteError[e], t) - return + goto sendResult } // Update receipt @@ -1545,6 +1545,7 @@ func (p *ticketVotePlugin) ballot(votes []ticketvote.CastVote, results chan tick // Update cast votes cache p.cachedVotesSet(v.Token, v.Ticket, v.VoteBit) + sendResult: // Send result back to calling function results <- cvr }(v) @@ -1792,6 +1793,8 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { batch = make([]ticketvote.CastVote, 0, batchSize) queue = make([][]ticketvote.CastVote, 0, len(votes)/batchSize) + // ballotCount is the number of votes that have passed validation + // and are being cast in this ballot. ballotCount int ) for k, v := range votes { @@ -1822,7 +1825,7 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { // Cast ballot in batches results := make(chan ticketvote.CastVoteReply, ballotCount) for i, batch := range queue { - log.Debugf("Casting vote batch %v/%v", i+1, len(queue)) + log.Debugf("Casting %v votes in batch %v/%v", len(batch), i+1, len(queue)) p.ballot(batch, results) } @@ -1834,6 +1837,10 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { r[v.Ticket] = v } + if len(r) != ballotCount { + log.Errorf("Missing results: got %v, want %v", len(r), ballotCount) + } + // Fill in the receipts for k, v := range votes { if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { @@ -1845,9 +1852,9 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { t := time.Now().Unix() log.Errorf("cmdBallot: vote result not found %v: %v", t, v.Ticket) e := ticketvote.VoteErrorInternalError - cvr.Ticket = v.Ticket - cvr.ErrorCode = e - cvr.ErrorContext = fmt.Sprintf("%v: %v", + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = fmt.Sprintf("%v: %v", ticketvote.VoteError[e], t) continue } diff --git a/politeiad/backend/tlogbe/trillian.go b/politeiad/backend/tlogbe/trillian.go index a4bdb6d2c..ee9360814 100644 --- a/politeiad/backend/tlogbe/trillian.go +++ b/politeiad/backend/tlogbe/trillian.go @@ -313,7 +313,14 @@ func (t *trillianClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) return nil, nil, fmt.Errorf("QueuedLeaves: %v", err) } - // Only wait if we actually updated the tree + // Wait for inclusion of all queued leaves in the root. We must + // check for inclusion instead of simply waiting for a root update + // because a root update doesn't necessarily mean the queued leaves + // from this request were added yet. The root will be updated as + // soon as the first leaf in the queue is added, which can lead to + // errors when the queue contains multiple leaves and we try to + // fetch the inclusion proof in the code below for leaves that are + // still in the process of being taken out of the queue. var n int for k := range qlr.QueuedLeaves { c := codes.Code(qlr.QueuedLeaves[k].GetStatus().GetCode()) @@ -321,29 +328,29 @@ func (t *trillianClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) n++ } } - if len(leaves)-n != 0 { - // Wait for root update - log.Debugf("Waiting for root update") - var logRoot types.LogRootV1 - err := logRoot.UnmarshalBinary(slr.LogRoot) - if err != nil { - return nil, nil, err - } - c, err := client.NewFromTree(t.client, tree, logRoot) - if err != nil { - return nil, nil, err - } - _, err = c.WaitForRootUpdate(t.ctx) + log.Debugf("Queued/Ignored leaves: %v/%v", len(leaves)-n, n) + + var logRoot types.LogRootV1 + err = logRoot.UnmarshalBinary(slr.LogRoot) + if err != nil { + return nil, nil, err + } + c, err := client.NewFromTree(t.client, tree, logRoot) + if err != nil { + return nil, nil, err + } + for _, v := range qlr.QueuedLeaves { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + err = c.WaitForInclusion(ctx, v.Leaf.LeafValue) if err != nil { - return nil, nil, fmt.Errorf("WaitForRootUpdate: %v", err) + return nil, nil, fmt.Errorf("WaitForInclusion: %v", err) } } - log.Debugf("Stored/Ignored leaves: %v/%v", len(leaves)-n, n) - // Get the latest signed log root - slr, lrv1, err := t.signedLogRootForTree(tree) + _, lr, err := t.signedLogRootForTree(tree) if err != nil { return nil, nil, fmt.Errorf("signedLogRootForTree post update: %v", err) } @@ -358,8 +365,8 @@ func (t *trillianClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) // Only retrieve the inclusion proof if the leaf was successfully // appended. Leaves that were not successfully appended will be // returned without an inclusion proof and the caller can decide - // what to do. Note this includes leaves that were not appended - // because they were a duplicate. + // what to do with them. Note this includes leaves that were not + // appended because they were a duplicate. c := codes.Code(v.GetStatus().GetCode()) if c == codes.OK { // Verify that the merkle leaf hash is using the expected @@ -373,7 +380,7 @@ func (t *trillianClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) // The LeafIndex of a QueuedLogLeaf will not be set yet. Get the // inclusion proof by MerkleLeafHash. - qlp.Proof, err = t.inclusionProof(treeID, v.Leaf.MerkleLeafHash, lrv1) + qlp.Proof, err = t.inclusionProof(treeID, v.Leaf.MerkleLeafHash, lr) if err != nil { return nil, nil, fmt.Errorf("inclusionProof %v %x: %v", treeID, v.Leaf.MerkleLeafHash, err) @@ -383,7 +390,7 @@ func (t *trillianClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) proofs = append(proofs, qlp) } - return proofs, lrv1, nil + return proofs, lr, nil } func (t *trillianClient) leavesByRange(treeID int64, startIndex, count int64) ([]*trillian.LogLeaf, error) { diff --git a/politeiawww/cmd/piwww/voteballot.go b/politeiawww/cmd/piwww/voteballot.go index 7065d7dcc..928fc8f97 100644 --- a/politeiawww/cmd/piwww/voteballot.go +++ b/politeiawww/cmd/piwww/voteballot.go @@ -22,69 +22,53 @@ import ( // voteBallotCmd casts a ballot of votes for the specified proposal. type voteBallotCmd struct { Args struct { - Token string `positional-arg-name:"token"` // Censorship token - VoteID string `positional-arg-name:"voteid"` // Vote choice ID + Token string `positional-arg-name:"token"` + VoteID string `positional-arg-name:"voteid"` } `positional-args:"true" required:"true"` + Password string `long:"password" optional:"true"` } // Execute executes the vote ballot command. -func (cmd *voteBallotCmd) Execute(args []string) error { - token := cmd.Args.Token - voteID := cmd.Args.VoteID +func (c *voteBallotCmd) Execute(args []string) error { + token := c.Args.Token + voteID := c.Args.VoteID - // Connet to user's wallet - err := client.LoadWalletClient() - if err != nil { - return fmt.Errorf("LoadWalletClient: %v", err) - } - defer client.Close() - - // Get server public key - vr, err := client.Version() - if err != nil { - return fmt.Errorf("Version: %v", err) - } - - serverID, err := util.IdentityFromString(vr.PubKey) - if err != nil { - return err - } - - // Get vote details of provided proposal - avr, err := client.Votes(pi.Votes{ + // Get vote details + vr, err := client.Votes(pi.Votes{ Tokens: []string{token}, }) if err != nil { return fmt.Errorf("Votes: %v", err) } - - // Find the proposal that the user wants to vote on - pvt, ok := avr.Votes[token] + pv, ok := vr.Votes[token] if !ok { return fmt.Errorf("proposal not found: %v", token) } - // Ensure that the passed in voteID is one of the - // proposal's voting options and save the vote bits + // Verify provided vote ID var voteBit string - for _, option := range pvt.Vote.Params.Options { + for _, option := range pv.Vote.Params.Options { if voteID == option.ID { voteBit = strconv.FormatUint(option.Bit, 16) break } } - if voteBit == "" { return fmt.Errorf("vote id not found: %v", voteID) } - // Find user's tickets that are eligible to vote on this - // proposal - ticketPool, err := convertTicketHashes(pvt.Vote.EligibleTickets) + // Connect to user's wallet + err = client.LoadWalletClient() if err != nil { - return err + return fmt.Errorf("LoadWalletClient: %v", err) } + defer client.Close() + // Get the user's tickets that are eligible to vote + ticketPool, err := convertTicketHashes(pv.Vote.EligibleTickets) + if err != nil { + return err + } ctr, err := client.CommittedTickets( &walletrpc.CommittedTicketsRequest{ Tickets: ticketPool, @@ -92,47 +76,50 @@ func (cmd *voteBallotCmd) Execute(args []string) error { if err != nil { return fmt.Errorf("CommittedTickets: %v", err) } - if len(ctr.TicketAddresses) == 0 { - return fmt.Errorf("user has no eligible tickets: %v", - token) + return fmt.Errorf("user has no eligible tickets") } - // Create slice of hexadecimal ticket hashes to represent - // the user's eligible tickets + // Compile the ticket hashes of the user's eligible tickets eligibleTickets := make([]string, 0, len(ctr.TicketAddresses)) - for i, v := range ctr.TicketAddresses { + for _, v := range ctr.TicketAddresses { h, err := chainhash.NewHash(v.Ticket) if err != nil { - return fmt.Errorf("NewHash failed on index %v: %v", i, err) + return fmt.Errorf("NewHash %x: %v", v.Ticket, err) } eligibleTickets = append(eligibleTickets, h.String()) } - // Prompt user for wallet password + // The next step is to have the user's wallet sign the proposal + // votes for each ticket. The password wallet is needed for this. var passphrase []byte - for len(passphrase) == 0 { - fmt.Printf("Enter the private passphrase of your wallet: ") - pass, err := terminal.ReadPassword(int(os.Stdin.Fd())) - if err != nil { - return err + if c.Password != "" { + // Password was provided + passphrase = []byte(c.Password) + } else { + // Prompt user for password + for len(passphrase) == 0 { + fmt.Printf("Enter the private passphrase of your wallet: ") + pass, err := terminal.ReadPassword(int(os.Stdin.Fd())) + if err != nil { + return err + } + fmt.Printf("\n") + passphrase = bytes.TrimSpace(pass) } - fmt.Printf("\n") - passphrase = bytes.TrimSpace(pass) } // Sign eligible tickets with vote preference messages := make([]*walletrpc.SignMessagesRequest_Message, 0, len(eligibleTickets)) for i, v := range ctr.TicketAddresses { - // ctr.TicketAddresses and eligibleTickets use the same index + // ctr.TicketAddresses and eligibleTickets share the same ordering msg := token + eligibleTickets[i] + voteBit messages = append(messages, &walletrpc.SignMessagesRequest_Message{ Address: v.Address, Message: msg, }) } - sigs, err := client.SignMessages(&walletrpc.SignMessagesRequest{ Passphrase: passphrase, Messages: messages, @@ -140,15 +127,14 @@ func (cmd *voteBallotCmd) Execute(args []string) error { if err != nil { return fmt.Errorf("SignMessages: %v", err) } - - // Validate signatures for i, r := range sigs.Replies { if r.Error != "" { - return fmt.Errorf("signature failed index %v: %v", i, r.Error) + return fmt.Errorf("vote signature failed for ticket %v: %v", + eligibleTickets[i], err) } } - // Setup cast votes request + // Setup ballot request votes := make([]pi.CastVote, 0, len(eligibleTickets)) for i, ticket := range eligibleTickets { // eligibleTickets and sigs use the same index @@ -159,53 +145,60 @@ func (cmd *voteBallotCmd) Execute(args []string) error { Signature: hex.EncodeToString(sigs.Replies[i].Signature), }) } - - // Cast proposal votes - br, err := client.VoteBallot(&pi.VoteBallot{ + vb := pi.VoteBallot{ Votes: votes, - }) + } + + // Send ballot request + vbr, err := client.VoteBallot(vb) if err != nil { return fmt.Errorf("VoteBallot: %v", err) } - // Check for any failed votes. Vote receipts don't include - // the ticket hash so in order to associate a failed - // receipt with a specific ticket, we need to lookup the - // ticket hash and store it separately. - failedReceipts := make([]pi.CastVoteReply, 0, len(br.Receipts)) + // Get the server pubkey so that we can validate the receipts. + version, err := client.Version() + if err != nil { + return fmt.Errorf("Version: %v", err) + } + serverID, err := util.IdentityFromString(version.PubKey) + if err != nil { + return err + } + + // Check for any failed votes. Vote receipts don't include the + // ticket hash so in order to associate a failed receipt with a + // specific ticket, we need to lookup the ticket hash and store + // it separately. + failedReceipts := make([]pi.CastVoteReply, 0, len(vbr.Receipts)) failedTickets := make([]string, 0, len(eligibleTickets)) - for i, v := range br.Receipts { - // Lookup ticket hash - // br.Receipts and eligibleTickets use the same index + for i, v := range vbr.Receipts { + // Lookup ticket hash. br.Receipts and eligibleTickets use the + // same ordering h := eligibleTickets[i] - // Check for voting error + // Check for vote error if v.ErrorContext != "" { failedReceipts = append(failedReceipts, v) failedTickets = append(failedTickets, h) continue } - // Validate server signature + // Verify receipts sig, err := identity.SignatureFromString(v.Receipt) if err != nil { - v.ErrorContext = err.Error() - failedReceipts = append(failedReceipts, v) - failedTickets = append(failedTickets, h) + fmt.Printf("Failed to decode receipt: %v\n", v.Ticket) continue } - clientSig := votes[i].Signature if !serverID.VerifyMessage([]byte(clientSig), *sig) { - v.ErrorContext = "Could not verify receipt " + clientSig - failedReceipts = append(failedReceipts, v) - failedTickets = append(failedTickets, h) + fmt.Printf("Failed to verify receipt: %v", v.Ticket) + continue } } // Print results if !cfg.Silent { - fmt.Printf("Votes succeeded: %v\n", len(br.Receipts)-len(failedReceipts)) + fmt.Printf("Votes succeeded: %v\n", len(vbr.Receipts)-len(failedReceipts)) fmt.Printf("Votes failed : %v\n", len(failedReceipts)) for i, v := range failedReceipts { fmt.Printf("Failed vote : %v %v\n", failedTickets[i], v.ErrorContext) @@ -218,10 +211,14 @@ func (cmd *voteBallotCmd) Execute(args []string) error { // voteBallotHelpMsg is the help command message. const voteBallotHelpMsg = `voteballot "token" "voteid" -Cast ticket votes for a proposal. This command will only work when on testnet -and when running dcrwallet locally on the default port. +Cast a ballot of ticket votes for a proposal. This command will only work when +on testnet and when running dcrwallet locally on the default port. Arguments: -1. token (string, optional) Proposal censorship token -2. voteid (string, optional) A single word identifying vote (e.g. yes) +1. token (string, optional) Proposal censorship token +2. voteid (string, optional) Vote option ID (e.g. yes) + +Flags: + --password (string, optional) Wallet password. You will be prompted for the + password if one is not provided. ` diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index 44aea8f08..2f6d00790 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -2034,10 +2034,10 @@ func (c *Client) ActiveVotesDCC() (*cms.ActiveVoteReply, error) { return &avr, nil } -// VoteBallot casts ballot of votes for a proposal. -func (c *Client) VoteBallot(vb *pi.VoteBallot) (*pi.VoteBallotReply, error) { +// VoteBallot casts a ballot of votes for a proposal. +func (c *Client) VoteBallot(vb pi.VoteBallot) (*pi.VoteBallotReply, error) { statusCode, respBody, err := c.makeRequest(http.MethodPost, - pi.APIRoute, pi.RouteVoteBallot, &vb) + pi.APIRoute, pi.RouteVoteBallot, vb) if err != nil { return nil, err } From c13dc085b3f6a76a934ab6b8414104be6a2c9081 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 19 Oct 2020 08:32:04 -0500 Subject: [PATCH 164/449] return comment on new and censor --- politeiad/backend/tlogbe/comments.go | 46 +++++++++++++++++++++----- politeiad/backend/tlogbe/dcrtime.go | 4 +-- politeiad/backend/tlogbe/pi.go | 7 ++-- politeiad/backend/tlogbe/ticketvote.go | 2 +- politeiad/backend/tlogbe/tlogclient.go | 2 ++ politeiad/backend/tlogbe/trillian.go | 4 --- politeiad/plugins/comments/comments.go | 18 ++++------ politeiad/plugins/pi/pi.go | 9 +++-- politeiawww/api/pi/v1/v1.go | 7 ++-- politeiawww/cmd/piwww/commentcensor.go | 2 +- politeiawww/cmd/piwww/commentnew.go | 2 +- politeiawww/piwww.go | 33 ++++++++++++------ 12 files changed, 81 insertions(+), 55 deletions(-) diff --git a/politeiad/backend/tlogbe/comments.go b/politeiad/backend/tlogbe/comments.go index 1e1206a2d..8b1856181 100644 --- a/politeiad/backend/tlogbe/comments.go +++ b/politeiad/backend/tlogbe/comments.go @@ -36,6 +36,8 @@ import ( // TODO upvoting a comment twice in the same second causes a duplicate leaf // error which causes a 500. Solution: add the timestamp to the vote index. +// TODO verify all writes only accept full length tokens + const ( // Blob entry data descriptors dataDescriptorCommentAdd = "commentadd" @@ -433,7 +435,7 @@ func convertCommentFromCommentDel(cd comments.CommentDel) comments.Comment { CommentID: cd.CommentID, Version: 0, Timestamp: cd.Timestamp, - Receipt: "", + Receipt: cd.Receipt, Downvotes: 0, Upvotes: 0, Deleted: true, @@ -763,6 +765,19 @@ func (p *commentsPlugin) comments(s comments.StateT, token []byte, idx commentsI return cs, nil } +// comment returns the latest version of the provided comment. +func (p *commentsPlugin) comment(s comments.StateT, token []byte, idx commentsIndex, commentID uint32) (*comments.Comment, error) { + cs, err := p.comments(s, token, idx, []uint32{commentID}) + if err != nil { + return nil, fmt.Errorf("comments: %v", err) + } + c, ok := cs[commentID] + if !ok { + return nil, fmt.Errorf("comment not found") + } + return &c, nil +} + func (p *commentsPlugin) cmdNew(payload string) (string, error) { log.Tracef("comments cmdNew: %v", payload) @@ -877,11 +892,15 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { log.Debugf("Comment saved to record %v comment ID %v", ca.Token, ca.CommentID) + // Return new comment + c, err := p.comment(ca.State, token, *idx, ca.CommentID) + if err != nil { + return "", fmt.Errorf("comment %x %v: %v", token, ca.CommentID, err) + } + // Prepare reply nr := comments.NewReply{ - CommentID: ca.CommentID, - Timestamp: ca.Timestamp, - Receipt: ca.Receipt, + Comment: *c, } reply, err := comments.EncodeNewReply(nr) if err != nil { @@ -1029,11 +1048,15 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { log.Debugf("Comment edited on record %v comment ID %v", ca.Token, ca.CommentID) + // Return updated comment + c, err := p.comment(e.State, token, *idx, e.CommentID) + if err != nil { + return "", fmt.Errorf("comment %x %v: %v", token, e.CommentID, err) + } + // Prepare reply er := comments.EditReply{ - Version: ca.Version, - Timestamp: ca.Timestamp, - Receipt: ca.Receipt, + Comment: *c, } reply, err := comments.EncodeEditReply(er) if err != nil { @@ -1159,10 +1182,15 @@ func (p *commentsPlugin) cmdDel(payload string) (string, error) { return "", fmt.Errorf("del: %v", err) } + // Return updated comment + c, err := p.comment(d.State, token, *idx, d.CommentID) + if err != nil { + return "", fmt.Errorf("comment %x %v: %v", token, d.CommentID, err) + } + // Prepare reply dr := comments.DelReply{ - Timestamp: cd.Timestamp, - Receipt: cd.Receipt, + Comment: *c, } reply, err := comments.EncodeDelReply(dr) if err != nil { diff --git a/politeiad/backend/tlogbe/dcrtime.go b/politeiad/backend/tlogbe/dcrtime.go index f4ad66b7d..abbe14a3a 100644 --- a/politeiad/backend/tlogbe/dcrtime.go +++ b/politeiad/backend/tlogbe/dcrtime.go @@ -16,15 +16,13 @@ import ( ) var ( - // TODO flip skipVerify to false when done testing - skipVerify = true httpClient = &http.Client{ Timeout: 1 * time.Minute, Transport: &http.Transport{ IdleConnTimeout: 1 * time.Minute, ResponseHeaderTimeout: 1 * time.Minute, TLSClientConfig: &tls.Config{ - InsecureSkipVerify: skipVerify, + InsecureSkipVerify: false, }, }, } diff --git a/politeiad/backend/tlogbe/pi.go b/politeiad/backend/tlogbe/pi.go index ad7bfe957..0b7d7f195 100644 --- a/politeiad/backend/tlogbe/pi.go +++ b/politeiad/backend/tlogbe/pi.go @@ -401,9 +401,7 @@ func (p *piPlugin) cmdCommentNew(payload string) (string, error) { return "", err } cnr := pi.CommentNewReply{ - CommentID: nr.CommentID, - Timestamp: nr.Timestamp, - Receipt: nr.Receipt, + Comment: nr.Comment, } reply, err := pi.EncodeCommentNewReply(cnr) if err != nil { @@ -506,8 +504,7 @@ func (p *piPlugin) cmdCommentCensor(payload string) (string, error) { return "", err } ccr := pi.CommentCensorReply{ - Timestamp: dr.Timestamp, - Receipt: dr.Receipt, + Comment: dr.Comment, } reply, err := pi.EncodeCommentCensorReply(ccr) if err != nil { diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index 81bf65c97..0da73b2a2 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -64,7 +64,7 @@ var ( // is to check if the record exists in the mutexes function to ensure a token // is valid before holding the lock on it. -// TODO should start and startrunoff be combined into a single command? +// TODO verify all writes only accept full length tokens // ticketVotePlugin satisfies the pluginClient interface. type ticketVotePlugin struct { diff --git a/politeiad/backend/tlogbe/tlogclient.go b/politeiad/backend/tlogbe/tlogclient.go index 33ac8064a..078f4d684 100644 --- a/politeiad/backend/tlogbe/tlogclient.go +++ b/politeiad/backend/tlogbe/tlogclient.go @@ -8,6 +8,8 @@ import ( "fmt" ) +// TODO verify writes only accept full length tokens + // tlogClient provides an API for the plugins to use to interact with the tlog // backend. Plugins are allowed to save, delete, and get plugin data to/from // the tlog backend. Editing plugin data is not allowed. diff --git a/politeiad/backend/tlogbe/trillian.go b/politeiad/backend/tlogbe/trillian.go index ee9360814..38c33cfee 100644 --- a/politeiad/backend/tlogbe/trillian.go +++ b/politeiad/backend/tlogbe/trillian.go @@ -422,10 +422,6 @@ func (t *trillianClient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { return []*trillian.LogLeaf{}, nil } - // Default gprc max message size is 4MB (4194304 bytes). We need to - // increase this when fetching all leaves. - // maxMsgSize := grpc.MaxCallSendMsgSize(6000000) - // Get all leaves return t.leavesByRange(treeID, 0, int64(lr.TreeSize)) } diff --git a/politeiad/plugins/comments/comments.go b/politeiad/plugins/comments/comments.go index 09fe2279a..8cf4bccea 100644 --- a/politeiad/plugins/comments/comments.go +++ b/politeiad/plugins/comments/comments.go @@ -91,15 +91,16 @@ type Comment struct { ParentID uint32 `json:"parentid"` // Parent comment ID if reply Comment string `json:"comment"` // Comment text PublicKey string `json:"publickey"` // Public key used for Signature - Signature string `json:"signature"` // Signature of Token+ParentID+Comment + Signature string `json:"signature"` // Client signature CommentID uint32 `json:"commentid"` // Comment ID Version uint32 `json:"version"` // Comment version Timestamp int64 `json:"timestamp"` // UNIX timestamp of last edit Receipt string `json:"receipt"` // Server signature of client signature Downvotes uint64 `json:"downvotes"` // Tolal downvotes on comment Upvotes uint64 `json:"upvotes"` // Total upvotes on comment - Deleted bool `json:"deleted"` // Comment has been deleted - Reason string `json:"reason"` // Reason for deletion + + Deleted bool `json:"deleted,omitempty"` // Comment has been deleted + Reason string `json:"reason,omitempty"` // Reason for deletion } // CommentAdd is the structure that is saved to disk when a comment is created @@ -198,9 +199,7 @@ func DecodeNew(payload []byte) (*New, error) { // NewReply is the reply to the New command. type NewReply struct { - CommentID uint32 `json:"commentid"` // Comment ID - Timestamp int64 `json:"timestamp"` // Received UNIX timestamp - Receipt string `json:"receipt"` // Server sig of client sig + Comment Comment `json:"comment"` } // EncodeNew encodes a NewReply into a JSON byte slice. @@ -249,9 +248,7 @@ func DecodeEdit(payload []byte) (*Edit, error) { // EditReply is the reply to the Edit command. type EditReply struct { - Version uint32 `json:"version"` // Comment version - Timestamp int64 `json:"timestamp"` // Received UNIX timestamp - Receipt string `json:"receipt"` // Server signature of client signature + Comment Comment `json:"comment"` } // EncodeEdit encodes a EditReply into a JSON byte slice. @@ -298,8 +295,7 @@ func DecodeDel(payload []byte) (*Del, error) { // DelReply is the reply to the Del command. type DelReply struct { - Timestamp int64 `json:"timestamp"` // Received UNIX timestamp - Receipt string `json:"receipt"` // Server signature of client signature + Comment Comment `json:"comment"` } // EncodeDelReply encodes a DelReply into a JSON byte slice. diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index 2fe06a50b..7daa3950b 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -11,6 +11,8 @@ import ( "errors" "io" "strings" + + "github.com/decred/politeia/politeiad/plugins/comments" ) type PropStateT int @@ -291,9 +293,7 @@ func DecodeCommentNew(payload []byte) (*CommentNew, error) { // CommentNewReply is the reply to the CommentNew command. type CommentNewReply struct { - CommentID uint32 `json:"commentid"` // Comment ID - Timestamp int64 `json:"timestamp"` // Received UNIX timestamp - Receipt string `json:"receipt"` // Server sig of client sig + Comment comments.Comment `json:"comment"` } // EncodeCommentNew encodes a CommentNewReply into a JSON byte slice. @@ -342,8 +342,7 @@ func DecodeCommentCensor(payload []byte) (*CommentCensor, error) { // CommentCensorReply is the reply to the CommentCensor command. type CommentCensorReply struct { - Timestamp int64 `json:"timestamp"` // Received UNIX timestamp - Receipt string `json:"receipt"` // Server signature of client signature + Comment comments.Comment `json:"comment"` } // EncodeCommentCensorReply encodes a CommentCensorReply into a JSON byte diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 6fae7cca0..2a09a5c54 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -530,9 +530,7 @@ type CommentNew struct { // Receipt is the server signature of the client signature. This is proof that // the server received and processed the CommentNew command. type CommentNewReply struct { - CommentID uint32 `json:"commentid"` - Timestamp int64 `json:"timestamp"` - Receipt string `json:"receipt"` + Comment Comment `json:"comment"` } // CommentCensor permanently censors a comment. The comment will be deleted @@ -554,8 +552,7 @@ type CommentCensor struct { // Receipt is the server signature of the client signature. This is proof that // the server received and processed the CommentCensor command. type CommentCensorReply struct { - Timestamp int64 `json:"timestamp"` - Receipt string `json:"receipt"` + Comment Comment `json:"comment"` } // CommentVote casts a comment vote (upvote or downvote). Only allowed on diff --git a/politeiawww/cmd/piwww/commentcensor.go b/politeiawww/cmd/piwww/commentcensor.go index 74ffb9915..60bc7381b 100644 --- a/politeiawww/cmd/piwww/commentcensor.go +++ b/politeiawww/cmd/piwww/commentcensor.go @@ -90,7 +90,7 @@ func (cmd *commentCensorCmd) Execute(args []string) error { if err != nil { return err } - receiptb, err := util.ConvertSignature(ccr.Receipt) + receiptb, err := util.ConvertSignature(ccr.Comment.Receipt) if err != nil { return err } diff --git a/politeiawww/cmd/piwww/commentnew.go b/politeiawww/cmd/piwww/commentnew.go index 5d56b71c5..863b6e688 100644 --- a/politeiawww/cmd/piwww/commentnew.go +++ b/politeiawww/cmd/piwww/commentnew.go @@ -98,7 +98,7 @@ func (c *commentNewCmd) Execute(args []string) error { if err != nil { return err } - receiptb, err := util.ConvertSignature(ncr.Receipt) + receiptb, err := util.ConvertSignature(ncr.Comment.Receipt) if err != nil { return err } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 21f9afbae..e5c06e13f 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -144,6 +144,14 @@ func proposalName(pr pi.ProposalRecord) string { return name } +// commentFillInUser populates the provided comment with user data that is not +// store in politeiad and must be populated separately after pulling the +// comment from politeiad. +func commentFillInUser(c pi.Comment, u user.User) pi.Comment { + c.Username = u.Username + return c +} + func convertUserErrorFromSignatureError(err error) pi.UserErrorReply { var e util.SignatureError var s pi.ErrorStatusT @@ -1624,20 +1632,22 @@ func (p *politeiawww) processCommentNew(ctx context.Context, cn pi.CommentNew, u return nil, err } + // Prepare reply + c := convertCommentFromPlugin(cnr.Comment) + c = commentFillInUser(c, usr) + // Emit event p.eventManager.emit(eventProposalComment, dataProposalComment{ - state: cn.State, - token: cn.Token, - commentID: cnr.CommentID, - parentID: cn.ParentID, - username: usr.Username, + state: c.State, + token: c.Token, + commentID: c.CommentID, + parentID: c.ParentID, + username: c.Username, }) return &pi.CommentNewReply{ - CommentID: cnr.CommentID, - Timestamp: cnr.Timestamp, - Receipt: cnr.Receipt, + Comment: c, }, nil } @@ -1720,9 +1730,12 @@ func (p *politeiawww) processCommentCensor(ctx context.Context, cc pi.CommentCen return nil, err } + // Prepare reply + c := convertCommentFromPlugin(ccr.Comment) + c = commentFillInUser(c, usr) + return &pi.CommentCensorReply{ - Timestamp: ccr.Timestamp, - Receipt: ccr.Receipt, + Comment: c, }, nil } From 37c1e967a8b73fefacb11b7861a1f8db055c05a8 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 20 Oct 2020 16:10:58 -0500 Subject: [PATCH 165/449] cleanup politeiawww config --- politeiawww/config.go | 408 ++++++++++++++++++++++-------------------- 1 file changed, 211 insertions(+), 197 deletions(-) diff --git a/politeiawww/config.go b/politeiawww/config.go index 736595f14..22645c13d 100644 --- a/politeiawww/config.go +++ b/politeiawww/config.go @@ -8,7 +8,6 @@ package main import ( "crypto/tls" "crypto/x509" - "encoding/base64" "encoding/pem" "errors" "fmt" @@ -52,8 +51,8 @@ const ( defaultVoteDurationMin = uint32(2016) defaultVoteDurationMax = uint32(4032) - defaultMailAddress = "Politeia " - defaultCMSMailAddress = "Contractor Management System " + defaultMailAddressPi = "Politeia " + defaultMailAddressCMS = "Contractor Management System " defaultDcrdataMainnet = "dcrdata.decred.org:443" defaultDcrdataTestnet = "testnet.decred.org:443" @@ -95,54 +94,61 @@ var runServiceCommand func(string) error // // See loadConfig for details on the configuration load process. type config struct { - HomeDir string `short:"A" long:"appdata" description:"Path to application home directory"` - ShowVersion bool `short:"V" long:"version" description:"Display version information and exit"` - ConfigFile string `short:"C" long:"configfile" description:"Path to configuration file"` - DataDir string `short:"b" long:"datadir" description:"Directory to store data"` - LogDir string `long:"logdir" description:"Directory to log output."` - TestNet bool `long:"testnet" description:"Use the test network"` - SimNet bool `long:"simnet" description:"Use the simulation test network"` - Profile string `long:"profile" description:"Enable HTTP profiling on given port -- NOTE port must be between 1024 and 65536"` - CookieKeyFile string `long:"cookiekey" description:"File containing the secret cookies key"` - CPUProfile string `long:"cpuprofile" description:"Write CPU profile to the specified file"` - MemProfile string `long:"memprofile" description:"Write mem profile to the specified file"` - DebugLevel string `short:"d" long:"debuglevel" description:"Logging level for all subsystems {trace, debug, info, warn, error, critical} -- You may also specify =,=,... to set the log level for individual subsystems -- Use show to list available subsystems"` - Listeners []string `long:"listen" description:"Add an interface/port to listen for connections (default all interfaces port: 49152, testnet: 59152)"` - Version string - HTTPSCert string `long:"httpscert" description:"File containing the https certificate file"` - HTTPSKey string `long:"httpskey" description:"File containing the https certificate key"` - RPCHost string `long:"rpchost" description:"Host for politeiad in this format"` - RPCCert string `long:"rpccert" description:"File containing the https certificate file"` + HomeDir string `short:"A" long:"appdata" description:"Path to application home directory"` + ShowVersion bool `short:"V" long:"version" description:"Display version information and exit"` + ConfigFile string `short:"C" long:"configfile" description:"Path to configuration file"` + DataDir string `short:"b" long:"datadir" description:"Directory to store data"` + LogDir string `long:"logdir" description:"Directory to log output."` + TestNet bool `long:"testnet" description:"Use the test network"` + SimNet bool `long:"simnet" description:"Use the simulation test network"` + Profile string `long:"profile" description:"Enable HTTP profiling on given port -- NOTE port must be between 1024 and 65536"` + CookieKeyFile string `long:"cookiekey" description:"File containing the secret cookies key"` + DebugLevel string `short:"d" long:"debuglevel" description:"Logging level for all subsystems {trace, debug, info, warn, error, critical} -- You may also specify =,=,... to set the log level for individual subsystems -- Use show to list available subsystems"` + Listeners []string `long:"listen" description:"Add an interface/port to listen for connections (default all interfaces port: 4443)"` + HTTPSCert string `long:"httpscert" description:"File containing the https certificate file"` + HTTPSKey string `long:"httpskey" description:"File containing the https certificate key"` + RPCHost string `long:"rpchost" description:"Host for politeiad in this format"` + RPCCert string `long:"rpccert" description:"File containing the https certificate file"` + RPCIdentityFile string `long:"rpcidentityfile" description:"Path to file containing the politeiad identity"` + RPCUser string `long:"rpcuser" description:"RPC user name for privileged politeaid commands"` + RPCPass string `long:"rpcpass" description:"RPC password for privileged politeiad commands"` + FetchIdentity bool `long:"fetchidentity" description:"Whether or not politeiawww fetches the identity from politeiad."` + Interactive string `long:"interactive" description:"Set to i-know-this-is-a-bad-idea to turn off interactive mode during --fetchidentity."` + AdminLogFile string `long:"adminlogfile" description:"admin log filename (Default: admin.log)"` + Mode string `long:"mode" description:"Mode www runs as. Supported values: piwww, cmswww"` + + // User database settings + UserDB string `long:"userdb" description:"Database choice for the user database"` + DBHost string `long:"dbhost" description:"Database ip:port"` + DBRootCert string `long:"dbrootcert" description:"File containing the CA certificate for the database"` + DBCert string `long:"dbcert" description:"File containing the politeiawww client certificate for the database"` + DBKey string `long:"dbkey" description:"File containing the politeiawww client certificate key for the database"` + EncryptionKey string `long:"encryptionkey" description:"File containing encryption key used for encrypting user data at rest"` + OldEncryptionKey string `long:"oldencryptionkey" description:"File containing old encryption key (only set when rotating keys)"` + + // SMTP settings + MailHost string `long:"mailhost" description:"Email server address in this format: :"` + MailUser string `long:"mailuser" description:"Email server username"` + MailPass string `long:"mailpass" description:"Email server password"` + MailAddress string `long:"mailaddress" description:"Email address for outgoing email in the format: name
"` + SMTPCert string `long:"smtpcert" description:"File containing the smtp certificate file"` + SMTPSkipVerify bool `long:"smtpskipverify" description:"Skip SMTP TLS cert verification. Will only skip if SMTPCert is empty"` + WebServerAddress string `long:"webserveraddress" description:"Address for the Politeia web server; it should have this format: ://[:]"` + + // XXX These should be plugin settings DcrdataHost string `long:"dcrdatahost" description:"Dcrdata ip:port"` - RPCIdentityFile string `long:"rpcidentityfile" description:"Path to file containing the politeiad identity"` - Identity *identity.PublicIdentity - RPCUser string `long:"rpcuser" description:"RPC user name for privileged commands"` - RPCPass string `long:"rpcpass" description:"RPC password for privileged commands"` - MailHost string `long:"mailhost" description:"Email server address in this format: :"` - MailUser string `long:"mailuser" description:"Email server username"` - MailPass string `long:"mailpass" description:"Email server password"` - MailAddress string `long:"mailaddress" description:"Email address for outgoing email in the format: name
"` - DBHost string `long:"dbhost" description:"Database ip:port"` - DBRootCert string `long:"dbrootcert" description:"File containing the CA certificate for the database"` - DBCert string `long:"dbcert" description:"File containing the politeiawww client certificate for the database"` - DBKey string `long:"dbkey" description:"File containing the politeiawww client certificate key for the database"` - BuildCMSDB bool `long:"buildcmsdb" description:"Build the cmsdb from scratch"` - UserDB string `long:"userdb" description:"Database choice for the user database"` - EncryptionKey string `long:"encryptionkey" description:"File containing encryption key used for encrypting user data at rest"` - OldEncryptionKey string `long:"oldencryptionkey" description:"File containing old encryption key (only set when rotating keys)"` - FetchIdentity bool `long:"fetchidentity" description:"Whether or not politeiawww fetches the identity from politeiad."` - WebServerAddress string `long:"webserveraddress" description:"Address for the Politeia web server; it should have this format: ://[:]"` - Interactive string `long:"interactive" description:"Set to i-know-this-is-a-bad-idea to turn off interactive mode during --fetchidentity."` PaywallAmount uint64 `long:"paywallamount" description:"Amount of DCR (in atoms) required for a user to register or submit a proposal."` PaywallXpub string `long:"paywallxpub" description:"Extended public key for deriving paywall addresses."` MinConfirmationsRequired uint64 `long:"minconfirmations" description:"Minimum blocks confirmation for accepting paywall as paid. Only works in TestNet."` - VoteDurationMin uint32 `long:"votedurationmin" description:"Minimum duration of a proposal vote in blocks"` - VoteDurationMax uint32 `long:"votedurationmax" description:"Maximum duration of a proposal vote in blocks"` - AdminLogFile string `long:"adminlogfile" description:"admin log filename (Default: admin.log)"` - Mode string `long:"mode" description:"Mode www runs as. Supported values: piwww, cmswww"` - SMTPSkipVerify bool `long:"smtpskipverify" description:"Skip SMTP TLS cert verification. Will only skip if SMTPCert is empty"` - SMTPCert string `long:"smtpcert" description:"File containing the smtp certificate file"` - SystemCerts *x509.CertPool + BuildCMSDB bool `long:"buildcmsdb" description:"Build the cmsdb from scratch"` + + // TODO these need to be removed + VoteDurationMin uint32 `long:"votedurationmin" description:"Minimum duration of a proposal vote in blocks"` + VoteDurationMax uint32 `long:"votedurationmax" description:"Maximum duration of a proposal vote in blocks"` + + Version string + Identity *identity.PublicIdentity + SystemCerts *x509.CertPool } // serviceOptions defines the configuration options for the rpc as a service @@ -388,14 +394,13 @@ func loadConfig() (*config, []string, error) { HTTPSCert: defaultHTTPSCertFile, RPCCert: defaultRPCCertFile, CookieKeyFile: defaultCookieKeyFile, + Version: version.String(), + Mode: defaultWWWMode, + UserDB: defaultUserDB, PaywallAmount: defaultPaywallAmount, MinConfirmationsRequired: defaultPaywallMinConfirmations, - Version: version.String(), VoteDurationMin: defaultVoteDurationMin, VoteDurationMax: defaultVoteDurationMax, - MailAddress: defaultMailAddress, - Mode: defaultWWWMode, - UserDB: defaultUserDB, } // Service options which are only added on Windows. @@ -508,13 +513,16 @@ func loadConfig() (*config, []string, error) { return nil, nil, err } - // Verify mode + // Verify mode and set mode specific defaults switch cfg.Mode { case cmsWWWMode: - if cfg.MailAddress == defaultMailAddress { - cfg.MailAddress = defaultCMSMailAddress + if cfg.MailAddress == "" { + cfg.MailAddress = defaultMailAddressCMS } case politeiaWWWMode: + if cfg.MailAddress == "" { + cfg.MailAddress = defaultMailAddressPi + } default: err := fmt.Errorf("invalid mode: %v", cfg.Mode) fmt.Fprintln(os.Stderr, err) @@ -595,89 +603,6 @@ func loadConfig() (*config, []string, error) { cfg.HTTPSCert = cleanAndExpandPath(cfg.HTTPSCert) cfg.RPCCert = cleanAndExpandPath(cfg.RPCCert) - // Validate cache options. - switch { - case cfg.DBHost == "": - return nil, nil, fmt.Errorf("dbhost param is required") - case cfg.DBRootCert == "": - return nil, nil, fmt.Errorf("dbrootcert param is required") - case cfg.DBCert == "": - return nil, nil, fmt.Errorf("dbcert param is required") - case cfg.DBKey == "": - return nil, nil, fmt.Errorf("dbkey param is required") - } - - cfg.DBRootCert = cleanAndExpandPath(cfg.DBRootCert) - cfg.DBCert = cleanAndExpandPath(cfg.DBCert) - cfg.DBKey = cleanAndExpandPath(cfg.DBKey) - - // Validate db host. - _, err = url.Parse(cfg.DBHost) - if err != nil { - return nil, nil, fmt.Errorf("parse dbhost: %v", err) - } - - // Validate db root cert. - b, err := ioutil.ReadFile(cfg.DBRootCert) - if err != nil { - return nil, nil, fmt.Errorf("read dbrootcert: %v", err) - } - block, _ := pem.Decode(b) - _, err = x509.ParseCertificate(block.Bytes) - if err != nil { - return nil, nil, fmt.Errorf("parse dbrootcert: %v", err) - } - - // Validate db key pair. - _, err = tls.LoadX509KeyPair(cfg.DBCert, cfg.DBKey) - if err != nil { - return nil, nil, fmt.Errorf("load key pair dbcert "+ - "and dbkey: %v", err) - } - - // Validate user database selection. - switch cfg.UserDB { - case userDBLevel, userDBCockroach: - // Valid selection; continue - default: - return nil, nil, fmt.Errorf("invalid userdb '%v'; must "+ - "be either leveldb or cockroachdb", cfg.UserDB) - } - - // Validate encryption keys. - cfg.EncryptionKey = cleanAndExpandPath(cfg.EncryptionKey) - cfg.OldEncryptionKey = cleanAndExpandPath(cfg.OldEncryptionKey) - - if cfg.EncryptionKey != "" { - if cfg.UserDB == userDBLevel { - return nil, nil, fmt.Errorf("data at rest encryption is not" + - "currently supported for leveldb; remove encryption key " + - "param or change user database param") - } - - if !util.FileExists(cfg.EncryptionKey) { - return nil, nil, fmt.Errorf("file not found %v", - cfg.EncryptionKey) - } - } - - if cfg.OldEncryptionKey != "" { - if cfg.EncryptionKey == "" { - return nil, nil, fmt.Errorf("old encryption key param " + - "cannot be used without encryption key param") - } - - if cfg.EncryptionKey == cfg.OldEncryptionKey { - return nil, nil, fmt.Errorf("old encryption key param " + - "and encryption key param must be different") - } - - if !util.FileExists(cfg.OldEncryptionKey) { - return nil, nil, fmt.Errorf("file not found %v", - cfg.OldEncryptionKey) - } - } - // Special show command to list supported subsystems and exit. if cfg.DebugLevel == "show" { fmt.Println("Supported subsystems", supportedSubsystems()) @@ -721,7 +646,7 @@ func loadConfig() (*config, []string, error) { // duplicate addresses. cfg.Listeners = normalizeAddresses(cfg.Listeners, port) - // Set up the rpc address. + // Set up the politeiad rpc address. if cfg.TestNet { port = v1.DefaultTestnetPort if cfg.RPCHost == "" { @@ -732,21 +657,9 @@ func loadConfig() (*config, []string, error) { if cfg.RPCHost == "" { cfg.RPCHost = v1.DefaultMainnetHost } - if cfg.MinConfirmationsRequired != defaultPaywallMinConfirmations { - return nil, nil, fmt.Errorf("[ERR]: Can not change min block " + - "confirmations when in mainnet") - } - } - - // Setup dcrdata addresses - if cfg.DcrdataHost == "" { - if cfg.TestNet { - cfg.DcrdataHost = defaultDcrdataTestnet - } else { - cfg.DcrdataHost = defaultDcrdataMainnet - } } + // Verify politeiad RPC settings cfg.RPCHost = util.NormalizeAddress(cfg.RPCHost, port) u, err := url.Parse("https://" + cfg.RPCHost) if err != nil { @@ -754,25 +667,16 @@ func loadConfig() (*config, []string, error) { } cfg.RPCHost = u.String() - // Set random username and password when not specified if cfg.RPCUser == "" { - name, err := util.Random(32) - if err != nil { - return nil, nil, err - } - cfg.RPCUser = base64.StdEncoding.EncodeToString(name) - log.Warnf("RPC user name not set, using random value") + return nil, nil, fmt.Errorf("politeiad rpc user must be provided " + + "with --rpcuser") } if cfg.RPCPass == "" { - pass, err := util.Random(32) - if err != nil { - return nil, nil, err - } - cfg.RPCPass = base64.StdEncoding.EncodeToString(pass) - log.Warnf("RPC password not set, using random value") + return nil, nil, fmt.Errorf("politeiad rpc pass must be provided " + + "with --rpcpass") } - // Valide mail settings + // Verify mail settings switch { case cfg.MailHost == "" && cfg.MailUser == "" && cfg.MailPass == "" && cfg.WebServerAddress == "": @@ -788,50 +692,22 @@ func loadConfig() (*config, []string, error) { u, err = url.Parse(cfg.MailHost) if err != nil { - return nil, nil, fmt.Errorf("unable to parse mail host: %v", - err) + return nil, nil, fmt.Errorf("unable to parse mail host: %v", err) } cfg.MailHost = u.String() a, err := mail.ParseAddress(cfg.MailAddress) if err != nil { - return nil, nil, fmt.Errorf("unable to parse mail address: %v", - err) + return nil, nil, fmt.Errorf("unable to parse mail address: %v", err) } cfg.MailAddress = a.String() u, err = url.Parse(cfg.WebServerAddress) if err != nil { - return nil, nil, fmt.Errorf("unable to parse web server address: %v", - err) + return nil, nil, fmt.Errorf("unable to parse web server address: %v", err) } cfg.WebServerAddress = u.String() - // Load identity - if err := loadIdentity(&cfg); err != nil { - return nil, nil, err - } - - // Warn about missing config file only after all other configuration is - // done. This prevents the warning on help messages and invalid - // options. Note this should go directly before the return. - if configFileError != nil { - log.Warnf("%v", configFileError) - } - - // Parse the extended public key if the paywall is enabled. - if cfg.PaywallAmount != 0 || cfg.PaywallXpub != "" { - if cfg.PaywallAmount < dust { - return nil, nil, fmt.Errorf("[ERR]: Paywall amount needs to be "+ - "higher than %v", dust) - } - _, err := hdkeychain.NewKeyFromString(cfg.PaywallXpub, activeNetParams.Params) - if err != nil { - return nil, nil, fmt.Errorf("error processing extended public key: %v", - err) - } - } - // Validate smtp root cert. if cfg.SMTPCert != "" { cfg.SMTPCert = cleanAndExpandPath(cfg.SMTPCert) @@ -853,10 +729,148 @@ func loadConfig() (*config, []string, error) { cfg.SystemCerts = systemCerts if cfg.SMTPSkipVerify { - log.Warnf("SMTPCert has been set so SMTPSkipVerify is being disregarded.") + log.Warnf("SMTPCert has been set so SMTPSkipVerify is being disregarded") + } + } + + // Validate user database selection. + switch cfg.UserDB { + case userDBLevel: + // Leveldb implementation does not require any database settings + // and does support encrypting data at rest. Return an error if + // the user has the encryption settings set to prevent them from + // thinking their data is being encrypted. + switch { + case cfg.DBHost != "": + log.Warnf("leveldb does not use --dbhost") + case cfg.DBRootCert != "": + log.Warnf("leveldb does not use --dbrootcert") + case cfg.DBCert != "": + log.Warnf("leveldb does not use --dbcert") + case cfg.DBKey != "": + log.Warnf("leveldb does not use --dbkey") + case cfg.EncryptionKey != "": + return nil, nil, fmt.Errorf("leveldb --encryptionkey not supported") + case cfg.OldEncryptionKey != "": + return nil, nil, fmt.Errorf("leveldb --oldencryptionkey not supported") + } + + case userDBCockroach: + // Cockroachdb required these settings + switch { + case cfg.DBHost == "": + return nil, nil, fmt.Errorf("dbhost param is required") + case cfg.DBRootCert == "": + return nil, nil, fmt.Errorf("dbrootcert param is required") + case cfg.DBCert == "": + return nil, nil, fmt.Errorf("dbcert param is required") + case cfg.DBKey == "": + return nil, nil, fmt.Errorf("dbkey param is required") + } + + // Clean user database settings + cfg.DBRootCert = cleanAndExpandPath(cfg.DBRootCert) + cfg.DBCert = cleanAndExpandPath(cfg.DBCert) + cfg.DBKey = cleanAndExpandPath(cfg.DBKey) + + // Validate user database host + _, err = url.Parse(cfg.DBHost) + if err != nil { + return nil, nil, fmt.Errorf("parse dbhost: %v", err) + } + + // Validate user database root cert + b, err := ioutil.ReadFile(cfg.DBRootCert) + if err != nil { + return nil, nil, fmt.Errorf("read dbrootcert: %v", err) + } + block, _ := pem.Decode(b) + _, err = x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, nil, fmt.Errorf("parse dbrootcert: %v", err) + } + + // Validate user database key pair + _, err = tls.LoadX509KeyPair(cfg.DBCert, cfg.DBKey) + if err != nil { + return nil, nil, fmt.Errorf("load key pair dbcert "+ + "and dbkey: %v", err) + } + + // Validate user database encryption keys + cfg.EncryptionKey = cleanAndExpandPath(cfg.EncryptionKey) + cfg.OldEncryptionKey = cleanAndExpandPath(cfg.OldEncryptionKey) + + if cfg.EncryptionKey != "" && !util.FileExists(cfg.EncryptionKey) { + return nil, nil, fmt.Errorf("file not found %v", cfg.EncryptionKey) + } + + if cfg.OldEncryptionKey != "" { + switch { + case cfg.EncryptionKey == "": + return nil, nil, fmt.Errorf("old encryption key param " + + "cannot be used without encryption key param") + + case cfg.EncryptionKey == cfg.OldEncryptionKey: + return nil, nil, fmt.Errorf("old encryption key param " + + "and encryption key param must be different") + + case !util.FileExists(cfg.OldEncryptionKey): + return nil, nil, fmt.Errorf("file not found %v", cfg.OldEncryptionKey) + } + } + + default: + return nil, nil, fmt.Errorf("invalid userdb '%v'; must "+ + "be either leveldb or cockroachdb", cfg.UserDB) + } + + // Verify paywall settings + + paywallIsEnabled := cfg.PaywallAmount != 0 || cfg.PaywallXpub != "" + if paywallIsEnabled { + // Parse extended public key + _, err := hdkeychain.NewKeyFromString(cfg.PaywallXpub, + activeNetParams.Params) + if err != nil { + return nil, nil, fmt.Errorf("error processing extended "+ + "public key: %v", err) + } + + // Verify paywall amount + if cfg.PaywallAmount < dust { + return nil, nil, fmt.Errorf("paywall amount needs to be "+ + "higher than %v", dust) + } + + // Verify required paywall confirmations + if !cfg.TestNet && !cfg.SimNet && + cfg.MinConfirmationsRequired != defaultPaywallMinConfirmations { + return nil, nil, fmt.Errorf("cannot set --minconfirmations on mainnet") + } + } + + // Setup dcrdata addresses + if cfg.DcrdataHost == "" { + if cfg.TestNet { + cfg.DcrdataHost = defaultDcrdataTestnet + } else { + cfg.DcrdataHost = defaultDcrdataMainnet } } + // Load identity + if err := loadIdentity(&cfg); err != nil { + return nil, nil, err + } + + // Warn about missing config file only after all other configuration is + // done. This prevents the warning on help messages and invalid + // options. Note this should go directly before the return. + if configFileError != nil { + log.Warnf("%v", configFileError) + } + return &cfg, remainingArgs, nil } From 39884afcf523a1c10c60b0f459eb0a559eabdb3e Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 20 Oct 2020 16:57:58 -0500 Subject: [PATCH 166/449] return full proposal on all state changes --- politeiawww/api/pi/v1/v1.go | 15 ++++---- politeiawww/cmd/piwww/proposaledit.go | 12 +----- politeiawww/cmd/piwww/proposalnew.go | 12 +----- politeiawww/cmd/piwww/proposalstatusset.go | 12 +++++- politeiawww/piwww.go | 44 +++++++++++++++------- politeiawww/www.go | 2 + 6 files changed, 56 insertions(+), 41 deletions(-) diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 2a09a5c54..40b7d9c81 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -26,6 +26,10 @@ type VoteErrorT int // for 1 result to be returned so that we have the option to change this is // we want to. // TODO should we add auths to the vote summary? +// TODO should routes for fetching comments be in their own API? This would +// make politeiawww far more configurable. I think so. +// TODO add a comments/count endpoint and take the comments count off of the +// proposal record const ( APIVersion = 1 @@ -353,7 +357,7 @@ type StatusChange struct { Timestamp int64 `json:"timestamp"` } -// ProposalRecord is an entire proposal and it's contents. +// ProposalRecord represents a proposal submission and its metadata. // // Signature is the client signature of the proposal merkle root. The merkle // root is the ordered merkle root of all proposal Files and Metadata. @@ -397,8 +401,7 @@ type ProposalNew struct { // ProposalNewReply is the reply to the ProposalNew command. type ProposalNewReply struct { - Timestamp int64 `json:"timestamp"` - CensorshipRecord CensorshipRecord `json:"censorshiprecord"` + Proposal ProposalRecord `json:"proposal"` } // ProposalEdit edits an existing proposal. @@ -418,9 +421,7 @@ type ProposalEdit struct { // ProposalEditReply is the reply to the ProposalEdit command. type ProposalEditReply struct { - Version string `json:"version"` - Timestamp int64 `json:"timestamp"` - CensorshipRecord CensorshipRecord `json:"censorshiprecord"` + Proposal ProposalRecord `json:"proposal"` } // ProposalStatusSet sets the status of a proposal. Some status changes require @@ -439,7 +440,7 @@ type ProposalStatusSet struct { // ProposalStatusSetReply is the reply to the ProposalStatusSet command. type ProposalStatusSetReply struct { - Timestamp int64 `json:"timestamp"` + Proposal ProposalRecord `json:"proposal"` } // ProposalRequest is used to request the ProposalRecord of the provided diff --git a/politeiawww/cmd/piwww/proposaledit.go b/politeiawww/cmd/piwww/proposaledit.go index dfc86c704..fc5101570 100644 --- a/politeiawww/cmd/piwww/proposaledit.go +++ b/politeiawww/cmd/piwww/proposaledit.go @@ -219,17 +219,9 @@ func (cmd *proposalEditCmd) Execute(args []string) error { if err != nil { return err } - pr := pi.ProposalRecord{ - Files: pe.Files, - Metadata: pe.Metadata, - PublicKey: pe.PublicKey, - Signature: pe.Signature, - CensorshipRecord: per.CensorshipRecord, - } - err = verifyProposal(pr, vr.PubKey) + err = verifyProposal(per.Proposal, vr.PubKey) if err != nil { - return fmt.Errorf("unable to verify proposal %v: %v", - pr.CensorshipRecord.Token, err) + return fmt.Errorf("unable to verify proposal: %v", err) } return nil diff --git a/politeiawww/cmd/piwww/proposalnew.go b/politeiawww/cmd/piwww/proposalnew.go index 8ff85d288..6920e0656 100644 --- a/politeiawww/cmd/piwww/proposalnew.go +++ b/politeiawww/cmd/piwww/proposalnew.go @@ -183,17 +183,9 @@ func (cmd *proposalNewCmd) Execute(args []string) error { if err != nil { return err } - pr := pi.ProposalRecord{ - Files: pn.Files, - Metadata: pn.Metadata, - PublicKey: pn.PublicKey, - Signature: pn.Signature, - CensorshipRecord: pnr.CensorshipRecord, - } - err = verifyProposal(pr, vr.PubKey) + err = verifyProposal(pnr.Proposal, vr.PubKey) if err != nil { - return fmt.Errorf("unable to verify proposal %v: %v", - pr.CensorshipRecord.Token, err) + return fmt.Errorf("unable to verify proposal: %v", err) } return nil diff --git a/politeiawww/cmd/piwww/proposalstatusset.go b/politeiawww/cmd/piwww/proposalstatusset.go index a3f396adf..dc18722c2 100644 --- a/politeiawww/cmd/piwww/proposalstatusset.go +++ b/politeiawww/cmd/piwww/proposalstatusset.go @@ -25,7 +25,7 @@ type proposalStatusSetCmd struct { Unvetted bool `long:"unvetted" optional:"true"` } -// Execute executes the set proposal status command. +// Execute executes the proposal status set command. func (cmd *proposalStatusSetCmd) Execute(args []string) error { propStatus := map[string]pi.PropStatusT{ "public": pi.PropStatusPublic, @@ -104,6 +104,16 @@ func (cmd *proposalStatusSetCmd) Execute(args []string) error { return err } + // Verify proposal + vr, err := client.Version() + if err != nil { + return err + } + err = verifyProposal(pssr.Proposal, vr.PubKey) + if err != nil { + return fmt.Errorf("unable to verify proposal: %v", err) + } + return nil } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index e5c06e13f..b6146fe6b 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -144,6 +144,14 @@ func proposalName(pr pi.ProposalRecord) string { return name } +// proposalRecordFillInUser fills in all user fields that are store in the +// user database and not in politeiad. +func proposalRecordFillInUser(pr pi.ProposalRecord, u user.User) pi.ProposalRecord { + pr.UserID = u.ID.String() + pr.Username = u.Username + return pr +} + // commentFillInUser populates the provided comment with user data that is not // store in politeiad and must be populated separately after pulling the // comment from politeiad. @@ -725,11 +733,6 @@ func (p *politeiawww) linkByPeriodMax() int64 { // proposalRecords returns the ProposalRecord for each of the provided proposal // requests. If a token does not correspond to an actual proposal then it will // not be included in the returned map. -// -// TODO this presents a challenge because the proposal Metadata still needs to -// be returned even if the proposal Files are not returned, which means that we -// will always need to fetch the record from politeiad with the files attached -// since the proposal Metadata is saved to politeiad as a politeiad File. func (p *politeiawww) proposalRecords(ctx context.Context, state pi.PropStateT, reqs []pi.ProposalRequest, includeFiles bool) (map[string]pi.ProposalRecord, error) { // Get politeiad records props := make([]pi.ProposalRecord, 0, len(reqs)) @@ -816,8 +819,7 @@ func (p *politeiawww) proposalRecords(ctx context.Context, state pi.PropStateT, return nil, fmt.Errorf("user not found for pubkey %v from proposal %v", v.PublicKey, token) } - props[k].UserID = u.ID.String() - props[k].Username = u.Username + props[k] = proposalRecordFillInUser(v, u) } // Convert proposals to a map @@ -1231,9 +1233,14 @@ func (p *politeiawww) processProposalNew(ctx context.Context, pn pi.ProposalNew, log.Infof("%02v: %v %v", k, f.Name, f.Digest) } + // Get full proposal record + pr, err := p.proposalRecordLatest(ctx, pi.PropStateUnvetted, cr.Token) + if err != nil { + return nil, err + } + return &pi.ProposalNewReply{ - Timestamp: timestamp, - CensorshipRecord: cr, + Proposal: *pr, }, nil } @@ -1395,10 +1402,14 @@ func (p *politeiawww) processProposalEdit(ctx context.Context, pe pi.ProposalEdi log.Infof("%02v: %v %v", k, f.Name, f.Digest) } + // Get updated proposal + pr, err := p.proposalRecordLatest(ctx, pe.State, pe.Token) + if err != nil { + return nil, err + } + return &pi.ProposalEditReply{ - Version: r.Version, - CensorshipRecord: convertCensorshipRecordFromPD(r.CensorshipRecord), - Timestamp: timestamp, + Proposal: *pr, }, nil } @@ -1515,8 +1526,15 @@ func (p *politeiawww) processProposalStatusSet(ctx context.Context, pss pi.Propo adminID: usr.ID.String(), }) + // Get updated proposal + state := convertPropStateFromPropStatus(pss.Status) + pr, err := p.proposalRecordLatest(ctx, state, pss.Token) + if err != nil { + return nil, err + } + return &pi.ProposalStatusSetReply{ - Timestamp: timestamp, + Proposal: *pr, }, nil } diff --git a/politeiawww/www.go b/politeiawww/www.go index a1173e601..d590c5e01 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -817,6 +817,8 @@ func _main() error { router.Use(recoverMiddleware) // Setup user database + log.Infof("User db: %v", loadedCfg.UserDB) + var userDB user.Database switch loadedCfg.UserDB { case userDBLevel: From e38f2668eaefdfce1cc874d2d8850ea43d1d176e Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 21 Oct 2020 17:20:43 -0500 Subject: [PATCH 167/449] fix prop status to state bug --- politeiawww/email.go | 3 ++- politeiawww/eventmanager.go | 8 ++++---- politeiawww/piwww.go | 24 ++++++++++-------------- 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/politeiawww/email.go b/politeiawww/email.go index fbc2e6f03..9481ebae3 100644 --- a/politeiawww/email.go +++ b/politeiawww/email.go @@ -131,7 +131,8 @@ func (p *politeiawww) emailProposalStatusChange(d dataProposalStatusChange, prop } default: - return fmt.Errorf("no user notification for prop status %v", d.status) + log.Debugf("no user notification for prop status %v", d.status) + return nil } return p.smtp.sendEmailTo(subject, body, emails) diff --git a/politeiawww/eventmanager.go b/politeiawww/eventmanager.go index 486a84ad4..9f1ba858d 100644 --- a/politeiawww/eventmanager.go +++ b/politeiawww/eventmanager.go @@ -258,7 +258,8 @@ func (p *politeiawww) handleEventProposalEdited(ch chan interface{}) { type dataProposalStatusChange struct { token string // Proposal censorship token - status pi.PropStatusT // Proposal status + state pi.PropStateT // Updated proposal state + status pi.PropStatusT // Updated proposal status version string // Proposal version reason string // Status change reason adminID string // Admin uuid @@ -283,11 +284,10 @@ func (p *politeiawww) handleEventProposalStatusChange(ch chan interface{}) { } // Get the proposal author - state := convertPropStateFromPropStatus(d.status) - pr, err := p.proposalRecordLatest(context.Background(), state, d.token) + pr, err := p.proposalRecordLatest(context.Background(), d.state, d.token) if err != nil { log.Errorf("handleEventProposalStatusChange: proposalRecordLatest "+ - "%v %v: %v", state, d.token, err) + "%v %v: %v", d.state, d.token, err) continue } author, err := p.db.UserGetByPubKey(pr.PublicKey) diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index b6146fe6b..e0b2ee477 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -177,16 +177,6 @@ func convertUserErrorFromSignatureError(err error) pi.UserErrorReply { } } -func convertPropStateFromPropStatus(s pi.PropStatusT) pi.PropStateT { - switch s { - case pi.PropStatusUnreviewed, pi.PropStatusCensored: - return pi.PropStateUnvetted - case pi.PropStatusPublic, pi.PropStatusAbandoned: - return pi.PropStateVetted - } - return pi.PropStateInvalid -} - func convertPropStateFromPi(s pi.PropStateT) piplugin.PropStateT { switch s { case pi.PropStateUnvetted: @@ -291,7 +281,7 @@ func convertFilesFromPD(f []pd.File) ([]pi.File, []pi.Metadata) { return files, metadata } -func convertProposalRecordFromPD(r pd.Record) (*pi.ProposalRecord, error) { +func convertProposalRecordFromPD(r pd.Record, state pi.PropStateT) (*pi.ProposalRecord, error) { // Decode metadata streams var ( pg *piplugin.ProposalGeneral @@ -316,7 +306,6 @@ func convertProposalRecordFromPD(r pd.Record) (*pi.ProposalRecord, error) { // Convert to pi types files, metadata := convertFilesFromPD(r.Files) status := convertPropStatusFromPD(r.Status) - state := convertPropStateFromPropStatus(status) statuses := make([]pi.StatusChange, 0, len(sc)) for _, v := range sc { @@ -761,7 +750,7 @@ func (p *politeiawww) proposalRecords(ctx context.Context, state pi.PropStateT, continue } - pr, err := convertProposalRecordFromPD(*r) + pr, err := convertProposalRecordFromPD(*r, state) if err != nil { return nil, err } @@ -1516,10 +1505,18 @@ func (p *politeiawww) processProposalStatusSet(ctx context.Context, pss pi.Propo } } + // The proposal state will have changed if the proposal was made + // public. + state := pss.State + if pss.Status == pi.PropStatusPublic { + state = pi.PropStateVetted + } + // Emit status change event p.eventManager.emit(eventProposalStatusChange, dataProposalStatusChange{ token: pss.Token, + state: state, status: convertPropStatusFromPD(r.Status), version: r.Version, reason: pss.Reason, @@ -1527,7 +1524,6 @@ func (p *politeiawww) processProposalStatusSet(ctx context.Context, pss pi.Propo }) // Get updated proposal - state := convertPropStateFromPropStatus(pss.Status) pr, err := p.proposalRecordLatest(ctx, state, pss.Token) if err != nil { return nil, err From cfeaf39390f23dfb1353b2a76f65cc634d9a2ccb Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 21 Oct 2020 19:42:39 -0500 Subject: [PATCH 168/449] rework vote routes to fix issues and simplify --- politeiad/backend/tlogbe/pi.go | 220 ++++- politeiad/backend/tlogbe/ticketvote.go | 479 ++++++++--- politeiad/backend/tlogbe/tlogbe.go | 32 +- politeiad/plugins/pi/pi.go | 65 +- politeiad/plugins/ticketvote/ticketvote.go | 222 ++--- politeiawww/api/pi/v1/v1.go | 141 ++-- politeiawww/cmd/piwww/piwww.go | 6 +- politeiawww/cmd/piwww/voteballot.go | 22 +- politeiawww/cmd/piwww/votestart.go | 10 +- politeiawww/cmd/piwww/votestartrunoff.go | 31 +- politeiawww/cmd/shared/client.go | 46 +- politeiawww/cmd/shared/config.go | 2 +- politeiawww/piwww.go | 157 +--- politeiawww/proposals.go | 932 +-------------------- politeiawww/ticketvote.go | 54 +- politeiawww/www.go | 7 +- wsdcrdata/wsdcrdata.go | 6 +- 17 files changed, 941 insertions(+), 1491 deletions(-) diff --git a/politeiad/backend/tlogbe/pi.go b/politeiad/backend/tlogbe/pi.go index 0b7d7f195..cc8952e65 100644 --- a/politeiad/backend/tlogbe/pi.go +++ b/politeiad/backend/tlogbe/pi.go @@ -698,6 +698,219 @@ func (p *piPlugin) cmdVoteInventory(payload string) (string, error) { return string(reply), nil } +func (p *piPlugin) ticketVoteStart(payload string) (string, error) { + // Decode payload + s, err := ticketvote.DecodeStart([]byte(payload)) + if err != nil { + return "", err + } + + // Verify there is work to do + if len(s.Starts) == 0 { + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusStartDetailsInvalid), + ErrorContext: []string{"no start details found"}, + } + } + + // Get records for all RFP submissions + records := make(map[string]backend.Record, len(s.Starts)) + for _, v := range s.Starts { + token, err := tokenDecode(v.Params.Token) + if err != nil { + e := fmt.Sprintf("%v: %v", v.Params.Token, err) + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropTokenInvalid), + ErrorContext: []string{e}, + } + } + r, err := p.backend.GetVetted(token, "") + if err != nil { + if errors.Is(err, errRecordNotFound) { + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropNotFound), + ErrorContext: []string{v.Params.Token}, + } + } + return "", fmt.Errorf("GetVetted %x: %v", token, err) + } + records[v.Params.Token] = *r + } + + // Get RFP token. Just use the linkto from the first start details + // record. + token := s.Starts[0].Params.Token + r := records[token] + pm, err := proposalMetadataFromFiles(r.Files) + if err != nil { + return "", err + } + if pm == nil { + // Proposal metadata was not found + e := fmt.Sprintf("record is not a proposal %v", token) + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusStartDetailsInvalid), + ErrorContext: []string{e}, + } + } + if pm.LinkTo == "" { + // Proposal is not an RFP submission + e := fmt.Sprintf("proposal is not an rfp submission %v", token) + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusStartDetailsInvalid), + ErrorContext: []string{e}, + } + } + rfpToken, err := tokenDecode(pm.LinkTo) + if err != nil { + return "", fmt.Errorf("decode rfp token %v: %v", pm.LinkTo, err) + } + + // Get RFP record + rfp, err := p.backend.GetVetted(rfpToken, "") + if err != nil { + if errors.Is(err, errRecordNotFound) { + e := fmt.Sprintf("rfp not found %x", rfpToken) + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusRFPInvalid), + ErrorContext: []string{e}, + } + } + return "", fmt.Errorf("GetVetted %x: %v", token, err) + } + + // Verify RFP proposal linkby has expired + rfpPM, err := proposalMetadataFromFiles(rfp.Files) + if err != nil { + return "", err + } + if rfpPM == nil { + e := fmt.Sprintf("rfp is not a proposal %x", rfpToken) + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusRFPInvalid), + ErrorContext: []string{e}, + } + } + if rfpPM.LinkBy > time.Now().Unix() { + e := fmt.Sprintf("rfp %x linkby deadline not met %v", + rfpToken, rfpPM.LinkBy) + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusLinkByDeadlineNotMet), + ErrorContext: []string{e}, + } + } + + // Verify all public, non-abandoned RFP submissions have been + // included in the request. The linked from list of the RFP will + // include abandoned proposals that must be filtered out. + linkedFrom, err := p.linkedFrom(hex.EncodeToString(rfpToken)) + if err != nil { + return "", err + } + expected := make(map[string]struct{}, len(linkedFrom.Tokens)) + for k := range linkedFrom.Tokens { + _, ok := records[k] + if ok { + // RFP submission has been included in the runoff vote + expected[k] = struct{}{} + continue + } + + // RFP submission has not been included in the runoff vote. This + // is expected if the submission has been abandoned. If not then + // it is a user error. + token, err := tokenDecode(k) + if err != nil { + return "", err + } + r, err := p.backend.GetVetted(token, "") + if err != nil { + return "", err + } + if r.RecordMetadata.Status == backend.MDStatusVetted { + // Record is public and should be part of runoff vote + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusStartDetailsMissing), + ErrorContext: []string{r.RecordMetadata.Token}, + } + } + } + + // We know that all public records in the RFP's linked from list + // have been included in the runoff vote, but we must also verify + // that no extra start details have been included that shouldn't be + // there. + if len(s.Starts) != len(expected) { + // There are extra submissions. Find the culprits. + invalid := make([]string, 0, len(s.Starts)) + for _, v := range s.Starts { + _, ok := expected[v.Params.Token] + if !ok { + // This submission should not be here + invalid = append(invalid, v.Params.Token) + } + } + e := fmt.Sprintf("found tokens that should not be included: %v", + strings.Join(invalid, ", ")) + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusStartDetailsInvalid), + ErrorContext: []string{e}, + } + } + + // Pi plugin validation complete! Pass the plugin command to the + // backend. + return p.backend.Plugin(ticketvote.ID, ticketvote.CmdStart, "", payload) +} + +func (p *piPlugin) cmdPassThrough(payload string) (string, error) { + // Decode payload + pt, err := pi.DecodePassThrough([]byte(payload)) + if err != nil { + return "", err + } + + // Get pass through function + var fn func(string) (string, error) + switch pt.PluginID { + case ticketvote.ID: + switch pt.Cmd { + case ticketvote.CmdStart: + fn = p.ticketVoteStart + } + default: + return "", fmt.Errorf("invalid passthrough plugin command %v %v", + pt.PluginID, pt.Cmd) + } + + // Execute pass through + r, err := fn(pt.Payload) + if err != nil { + return "", err + } + + // Prepare reply + ptr := pi.PassThroughReply{ + Payload: r, + } + reply, err := pi.EncodePassThroughReply(ptr) + if err != nil { + return "", err + } + + return string(reply), nil +} + func (p *piPlugin) hookNewRecordPre(payload string) error { nr, err := decodeHookNewRecord([]byte(payload)) if err != nil { @@ -723,6 +936,9 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { return fmt.Errorf("proposal metadata not found") } + // TODO is linkby validated anywhere? It should be validated here + // and in the edit proposal. + // Verify the linkto is an RFP and that the RFP is eligible to be // linked to. We currently only allow linking to RFP proposals that // have been approved by a ticket vote. @@ -992,7 +1208,7 @@ func (p *piPlugin) hookSetRecordStatusPost(payload string) error { func (p *piPlugin) setup() error { log.Tracef("pi setup") - // Verify vote plugin dependency + // TODO Verify vote and comment plugin dependency return nil } @@ -1014,6 +1230,8 @@ func (p *piPlugin) cmd(cmd, payload string) (string, error) { return p.cmdCommentVote(payload) case pi.CmdVoteInventory: return p.cmdVoteInventory(payload) + case pi.CmdPassThrough: + return p.cmdPassThrough(payload) } return "", backend.ErrPluginCmdInvalid diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index 0da73b2a2..010d1753b 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -44,15 +44,15 @@ const ( filenameSummary = "{token}-summary.json" // Blob entry data descriptors - dataDescriptorAuthorizeDetails = "authorizedetails" - dataDescriptorVoteDetails = "votedetails" - dataDescriptorCastVoteDetails = "castvotedetails" + dataDescriptorAuthDetails = "authdetails" + dataDescriptorVoteDetails = "votedetails" + dataDescriptorCastVoteDetails = "castvotedetails" // Prefixes that are appended to key-value store keys before // storing them in the log leaf ExtraData field. - keyPrefixAuthorizeDetails = "authorizedetails:" - keyPrefixVoteDetails = "votedetails:" - keyPrefixCastVoteDetails = "castvotedetails:" + keyPrefixAuthDetails = "authdetails:" + keyPrefixVoteDetails = "votedetails:" + keyPrefixCastVoteDetails = "castvotedetails:" ) var ( @@ -455,7 +455,7 @@ func convertTicketVoteErrFromSignatureErr(err error) backend.PluginUserError { } } -func convertAuthorizeDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.AuthorizeDetails, error) { +func convertAuthDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.AuthDetails, error) { // Decode and validate data hint b, err := base64.StdEncoding.DecodeString(be.DataHint) if err != nil { @@ -466,9 +466,9 @@ func convertAuthorizeDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.Autho if err != nil { return nil, fmt.Errorf("unmarshal DataHint: %v", err) } - if dd.Descriptor != dataDescriptorAuthorizeDetails { + if dd.Descriptor != dataDescriptorAuthDetails { return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorAuthorizeDetails) + dd.Descriptor, dataDescriptorAuthDetails) } // Decode data @@ -484,10 +484,10 @@ func convertAuthorizeDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.Autho return nil, fmt.Errorf("data is not coherent; got %x, want %x", util.Digest(b), hash) } - var ad ticketvote.AuthorizeDetails + var ad ticketvote.AuthDetails err = json.Unmarshal(b, &ad) if err != nil { - return nil, fmt.Errorf("unmarshal AuthorizeDetails: %v", err) + return nil, fmt.Errorf("unmarshal AuthDetails: %v", err) } return &ad, nil @@ -569,7 +569,7 @@ func convertCastVoteDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.CastVo return &cv, nil } -func convertBlobEntryFromAuthorizeDetails(ad ticketvote.AuthorizeDetails) (*store.BlobEntry, error) { +func convertBlobEntryFromAuthDetails(ad ticketvote.AuthDetails) (*store.BlobEntry, error) { data, err := json.Marshal(ad) if err != nil { return nil, err @@ -577,7 +577,7 @@ func convertBlobEntryFromAuthorizeDetails(ad ticketvote.AuthorizeDetails) (*stor hint, err := json.Marshal( store.DataDescriptor{ Type: store.DataTypeStructure, - Descriptor: dataDescriptorAuthorizeDetails, + Descriptor: dataDescriptorAuthDetails, }) if err != nil { return nil, err @@ -620,14 +620,14 @@ func convertBlobEntryFromCastVoteDetails(cv ticketvote.CastVoteDetails) (*store. return &be, nil } -func (p *ticketVotePlugin) authorizeSave(ad ticketvote.AuthorizeDetails) error { +func (p *ticketVotePlugin) authorizeSave(ad ticketvote.AuthDetails) error { token, err := hex.DecodeString(ad.Token) if err != nil { return err } // Prepare blob - be, err := convertBlobEntryFromAuthorizeDetails(ad) + be, err := convertBlobEntryFromAuthDetails(ad) if err != nil { return err } @@ -641,7 +641,7 @@ func (p *ticketVotePlugin) authorizeSave(ad ticketvote.AuthorizeDetails) error { } // Save blob - merkles, err := p.tlog.save(tlogIDVetted, token, keyPrefixAuthorizeDetails, + merkles, err := p.tlog.save(tlogIDVetted, token, keyPrefixAuthDetails, [][]byte{b}, [][]byte{h}, false) if err != nil { return fmt.Errorf("save: %v", err) @@ -654,22 +654,22 @@ func (p *ticketVotePlugin) authorizeSave(ad ticketvote.AuthorizeDetails) error { return nil } -func (p *ticketVotePlugin) authorizes(token []byte) ([]ticketvote.AuthorizeDetails, error) { +func (p *ticketVotePlugin) authorizes(token []byte) ([]ticketvote.AuthDetails, error) { // Retrieve blobs blobs, err := p.tlog.blobsByKeyPrefix(tlogIDVetted, token, - keyPrefixAuthorizeDetails) + keyPrefixAuthDetails) if err != nil { return nil, err } // Decode blobs - auths := make([]ticketvote.AuthorizeDetails, 0, len(blobs)) + auths := make([]ticketvote.AuthDetails, 0, len(blobs)) for _, v := range blobs { be, err := store.Deblob(v) if err != nil { return nil, err } - a, err := convertAuthorizeDetailsFromBlobEntry(*be) + a, err := convertAuthDetailsFromBlobEntry(*be) if err != nil { return nil, err } @@ -684,7 +684,7 @@ func (p *ticketVotePlugin) authorizes(token []byte) ([]ticketvote.AuthorizeDetai return auths, nil } -func (p *ticketVotePlugin) voteSave(vd ticketvote.VoteDetails) error { +func (p *ticketVotePlugin) voteDetailsSave(vd ticketvote.VoteDetails) error { token, err := hex.DecodeString(vd.Params.Token) if err != nil { return err @@ -1072,7 +1072,7 @@ func (p *ticketVotePlugin) castVoteSignatureVerify(cv ticketvote.CastVote, addr msg := cv.Token + cv.Ticket + cv.VoteBit // Convert hex signature to base64. The voteMessageVerify function - // expects bas64. + // expects base64. b, err := hex.DecodeString(cv.Signature) if err != nil { return fmt.Errorf("invalid hex") @@ -1101,7 +1101,7 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { } // Verify token - token, err := hex.DecodeString(a.Token) + token, err := tokenDecode(a.Token) if err != nil { return "", backend.PluginUserError{ PluginID: ticketvote.ID, @@ -1119,9 +1119,9 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { // Verify action switch a.Action { - case ticketvote.ActionAuthorize: + case ticketvote.AuthActionAuthorize: // This is allowed - case ticketvote.ActionRevoke: + case ticketvote.AuthActionRevoke: // This is allowed default: e := fmt.Sprintf("%v not a valid action", a.Action) @@ -1152,23 +1152,23 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { switch { case len(auths) == 0: // No previous actions. New action must be an authorize. - if a.Action != ticketvote.ActionAuthorize { + if a.Action != ticketvote.AuthActionAuthorize { return "", backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusAuthorizationInvalid), ErrorContext: []string{"no prev action; action must be authorize"}, } } - case prevAction == ticketvote.ActionAuthorize && - a.Action != ticketvote.ActionRevoke: + case prevAction == ticketvote.AuthActionAuthorize && + a.Action != ticketvote.AuthActionRevoke: // Previous action was a authorize. This action must be revoke. return "", backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusAuthorizationInvalid), ErrorContext: []string{"prev action was authorize"}, } - case prevAction == ticketvote.ActionRevoke && - a.Action != ticketvote.ActionAuthorize: + case prevAction == ticketvote.AuthActionRevoke && + a.Action != ticketvote.AuthActionAuthorize: // Previous action was a revoke. This action must be authorize. return "", backend.PluginUserError{ PluginID: ticketvote.ID, @@ -1179,7 +1179,7 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { // Prepare authorize vote receipt := p.identity.SignMessage([]byte(a.Signature)) - auth := ticketvote.AuthorizeDetails{ + auth := ticketvote.AuthDetails{ Token: a.Token, Version: a.Version, Action: string(a.Action), @@ -1197,9 +1197,9 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { // Update inventory switch a.Action { - case ticketvote.ActionAuthorize: + case ticketvote.AuthActionAuthorize: p.inventorySetToAuthorized(a.Token) - case ticketvote.ActionRevoke: + case ticketvote.AuthActionRevoke: p.inventorySetToUnauthorized(a.Token) default: // Should not happen @@ -1366,97 +1366,107 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM return nil } -func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { - log.Tracef("ticketvote cmdStart: %v", payload) - - // Decode payload - s, err := ticketvote.DecodeStart([]byte(payload)) - if err != nil { - return "", err +// startStandard starts a standard vote. +func (p *ticketVotePlugin) startStandard(s ticketvote.Start) (*ticketvote.StartReply, error) { + // Verify there is only one start details + if len(s.Starts) != 1 { + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusStartDetailsInvalid), + ErrorContext: []string{"more than one start details found"}, + } } + sd := s.Starts[0] // Verify token - token, err := hex.DecodeString(s.Params.Token) + token, err := tokenDecode(sd.Params.Token) if err != nil { - return "", backend.PluginUserError{ + return nil, backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusTokenInvalid), } } // Verify signature - vb, err := json.Marshal(s.Params) + vb, err := json.Marshal(sd.Params) if err != nil { - return "", err + return nil, err } msg := hex.EncodeToString(util.Digest(vb)) - err = util.VerifySignature(s.Signature, s.PublicKey, msg) + err = util.VerifySignature(sd.Signature, sd.PublicKey, msg) if err != nil { - return "", convertTicketVoteErrFromSignatureErr(err) + return nil, convertTicketVoteErrFromSignatureErr(err) } // Verify vote options and params - err = voteParamsVerify(s.Params, p.voteDurationMin, p.voteDurationMax) + err = voteParamsVerify(sd.Params, p.voteDurationMin, p.voteDurationMax) if err != nil { - return "", err + return nil, err } + // Get vote blockchain data + sr, err := p.startReply(sd.Params.Duration) + if err != nil { + return nil, err + } + + // Validate existing record state. The lock for this record must be + // held for the remainder of this function. + m := p.mutex(sd.Params.Token) + m.Lock() + defer m.Unlock() + // Verify record version - version := strconv.FormatUint(uint64(s.Params.Version), 10) - _, err = p.backend.GetVetted(token, version) + r, err := p.backend.GetVetted(token, "") if err != nil { if errors.Is(err, backend.ErrRecordNotFound) { - e := fmt.Sprintf("version %v not found", version) - return "", backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusRecordNotFound), - ErrorContext: []string{e}, + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusRecordNotFound), } } + return nil, fmt.Errorf("GetVetted: %v", err) + } + version := strconv.FormatUint(uint64(sd.Params.Version), 10) + if r.Version != version { + e := fmt.Sprintf("version is not latest: got %v, want %v", + sd.Params.Version, r.Version) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusRecordVersionInvalid), + ErrorContext: []string{e}, + } } // Verify vote authorization auths, err := p.authorizes(token) if err != nil { - return "", err + return nil, err } if len(auths) == 0 { - return "", backend.PluginUserError{ + return nil, backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusAuthorizationInvalid), ErrorContext: []string{"authorization not found"}, } } action := ticketvote.AuthActionT(auths[len(auths)-1].Action) - if action != ticketvote.ActionAuthorize { - return "", backend.PluginUserError{ + if action != ticketvote.AuthActionAuthorize { + return nil, backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusAuthorizationInvalid), ErrorContext: []string{"not authorized"}, } } - // Get vote blockchain data - sr, err := p.startReply(s.Params.Duration) - if err != nil { - return "", err - } - - // Any previous vote details must be retrieved to verify that a vote - // has not already been started. The lock must be held for the - // remainder of this function. - m := p.mutex(s.Params.Token) - m.Lock() - defer m.Unlock() - // Verify vote has not already been started svp, err := p.voteDetails(token) if err != nil { - return "", err + return nil, err } if svp != nil { // Vote has already been started - return "", backend.PluginUserError{ + return nil, backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusVoteStatusInvalid), ErrorContext: []string{"vote already started"}, @@ -1465,9 +1475,9 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { // Prepare vote details vd := ticketvote.VoteDetails{ - Params: s.Params, - PublicKey: s.PublicKey, - Signature: s.Signature, + Params: sd.Params, + PublicKey: sd.PublicKey, + Signature: sd.Signature, StartBlockHeight: sr.StartBlockHeight, StartBlockHash: sr.StartBlockHash, EndBlockHeight: sr.EndBlockHeight, @@ -1475,14 +1485,243 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { } // Save vote details - err = p.voteSave(vd) + err = p.voteDetailsSave(vd) if err != nil { - return "", fmt.Errorf("startSave: %v", err) + return nil, fmt.Errorf("voteDetailsSave: %v", err) } // Update inventory p.inventorySetToStarted(vd.Params.Token, vd.EndBlockHeight) + return &ticketvote.StartReply{ + StartBlockHeight: sr.StartBlockHeight, + StartBlockHash: sr.StartBlockHash, + EndBlockHeight: sr.EndBlockHeight, + EligibleTickets: sr.EligibleTickets, + }, nil +} + +// startRunoff starts a runoff vote. +func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartReply, error) { + // Sanity check + if len(s.Starts) == 0 { + return nil, fmt.Errorf("no start details found") + } + + // Perform validation that can be done without fetching any records + // from the backend. + var ( + mask = s.Starts[0].Params.Mask + duration = s.Starts[0].Params.Duration + quorum = s.Starts[0].Params.QuorumPercentage + pass = s.Starts[0].Params.PassPercentage + ) + for _, v := range s.Starts { + // Verify vote params are the same for all submissions + switch { + case v.Params.Type != ticketvote.VoteTypeRunoff: + e := fmt.Sprintf("vote type invalid %v: got %v, want %v", + v.Params.Token, v.Params.Type, ticketvote.VoteTypeRunoff) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorContext: []string{e}, + } + case v.Params.Mask != mask: + e := fmt.Sprintf("mask invalid %v: all masks must be the same", + v.Params.Token) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorContext: []string{e}, + } + case v.Params.Duration != duration: + e := fmt.Sprintf("duration invalid %v: all durations must be the same", + v.Params.Token) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorContext: []string{e}, + } + case v.Params.QuorumPercentage != quorum: + e := fmt.Sprintf("quorum invalid %v: all quorums must be the same", + v.Params.Token) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorContext: []string{e}, + } + case v.Params.PassPercentage != pass: + e := fmt.Sprintf("pass rate invalid %v: all pass rates must be the same", + v.Params.Token) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorContext: []string{e}, + } + } + + // Verify token + _, err := tokenDecode(v.Params.Token) + if err != nil { + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusTokenInvalid), + ErrorContext: []string{v.Params.Token}, + } + } + + // Verify signature + vb, err := json.Marshal(v.Params) + if err != nil { + return nil, err + } + msg := hex.EncodeToString(util.Digest(vb)) + err = util.VerifySignature(v.Signature, v.PublicKey, msg) + if err != nil { + return nil, convertTicketVoteErrFromSignatureErr(err) + } + + // Verify vote options and params. Vote optoins are required to + // be approve and reject. + err = voteParamsVerify(v.Params, p.voteDurationMin, p.voteDurationMax) + if err != nil { + return nil, err + } + } + + // Get vote blockchain data + sr, err := p.startReply(duration) + if err != nil { + return nil, err + } + + // TODO handle the case where part of the votes are started but + // not all. + + for _, v := range s.Starts { + // Validate existing record state. The lock for this record must + // be held for the remainder of this function. + m := p.mutex(v.Params.Token) + m.Lock() + defer m.Unlock() + + token, err := tokenDecode(v.Params.Token) + if err != nil { + return nil, err + } + + // Verify record version + r, err := p.backend.GetVetted(token, "") + if err != nil { + if errors.Is(err, backend.ErrRecordNotFound) { + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusRecordNotFound), + ErrorContext: []string{v.Params.Token}, + } + } + return nil, fmt.Errorf("GetVetted: %v", err) + } + version := strconv.FormatUint(uint64(v.Params.Version), 10) + if r.Version != version { + e := fmt.Sprintf("version is not latest %v: got %v, want %v", + v.Params.Token, v.Params.Version, r.Version) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusRecordVersionInvalid), + ErrorContext: []string{e}, + } + } + + // Verify vote has not already been started + svp, err := p.voteDetails(token) + if err != nil { + return nil, err + } + if svp != nil { + // Vote has already been started + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusVoteStatusInvalid), + ErrorContext: []string{"vote already started"}, + } + } + } + + for _, v := range s.Starts { + // Prepare vote details + vd := ticketvote.VoteDetails{ + Params: v.Params, + PublicKey: v.PublicKey, + Signature: v.Signature, + StartBlockHeight: sr.StartBlockHeight, + StartBlockHash: sr.StartBlockHash, + EndBlockHeight: sr.EndBlockHeight, + EligibleTickets: sr.EligibleTickets, + } + + // Save vote details + err = p.voteDetailsSave(vd) + if err != nil { + return nil, fmt.Errorf("voteDetailsSave: %v", err) + } + + // Update inventory + p.inventorySetToStarted(vd.Params.Token, vd.EndBlockHeight) + } + + return &ticketvote.StartReply{ + StartBlockHeight: sr.StartBlockHeight, + StartBlockHash: sr.StartBlockHash, + EndBlockHeight: sr.EndBlockHeight, + EligibleTickets: sr.EligibleTickets, + }, nil +} + +func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { + log.Tracef("ticketvote cmdStart: %v", payload) + + // Decode payload + s, err := ticketvote.DecodeStart([]byte(payload)) + if err != nil { + return "", err + } + + // Parse vote type + if len(s.Starts) == 0 { + return "", backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusStartDetailsInvalid), + ErrorContext: []string{"no start details found"}, + } + } + vtype := s.Starts[0].Params.Type + + // Start vote + // TODO these vote user errors need to become more granular. Update + // this when writing tests. + var sr *ticketvote.StartReply + switch vtype { + case ticketvote.VoteTypeStandard: + sr, err = p.startStandard(*s) + if err != nil { + return "", err + } + case ticketvote.VoteTypeRunoff: + sr, err = p.startRunoff(*s) + if err != nil { + return "", err + } + default: + e := fmt.Sprintf("invalid vote type %v", vtype) + return "", backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorContext: []string{e}, + } + } + // Prepare reply reply, err := ticketvote.EncodeStartReply(*sr) if err != nil { @@ -1492,12 +1731,6 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { return string(reply), nil } -func (p *ticketVotePlugin) cmdStartRunoff(payload string) (string, error) { - log.Tracef("ticketvote cmdStartRunoff: %v", payload) - - return "", nil -} - // ballot casts the provided votes concurrently. The vote results are passed // back through the results channel to the calling function. This function // waits until all provided votes have been cast before returning. @@ -1529,7 +1762,7 @@ func (p *ticketVotePlugin) ballot(votes []ticketvote.CastVote, results chan tick err := p.castVoteSave(cv) if err != nil { t := time.Now().Unix() - log.Errorf("cmdBallot: castVoteSave %v: %v", t, err) + log.Errorf("cmdCastBallot: castVoteSave %v: %v", t, err) e := ticketvote.VoteErrorInternalError cvr.Ticket = v.Ticket cvr.ErrorCode = e @@ -1555,26 +1788,26 @@ func (p *ticketVotePlugin) ballot(votes []ticketvote.CastVote, results chan tick wg.Wait() } -// cmdBallot casts a ballot of votes. This function will not return a user +// cmdCastBallot casts a ballot of votes. This function will not return a user // error if one occurs. It will instead return the ballot reply with the error // included in the invidiual cast vote reply that it applies to. -func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { - log.Tracef("ticketvote cmdBallot: %v", payload) +func (p *ticketVotePlugin) cmdCastBallot(payload string) (string, error) { + log.Tracef("ticketvote cmdCastBallot: %v", payload) // Decode payload - ballot, err := ticketvote.DecodeBallot([]byte(payload)) + cb, err := ticketvote.DecodeCastBallot([]byte(payload)) if err != nil { return "", err } - votes := ballot.Votes + votes := cb.Ballot // Verify there is work to do if len(votes) == 0 { // Nothing to do - br := ticketvote.BallotReply{ + cbr := ticketvote.CastBallotReply{ Receipts: []ticketvote.CastVoteReply{}, } - reply, err := ticketvote.EncodeBallotReply(br) + reply, err := ticketvote.EncodeCastBallotReply(cbr) if err != nil { return "", err } @@ -1589,7 +1822,7 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { ) for k, v := range votes { // Verify token - t, err := decodeTokenFullLength(v.Token) + t, err := tokenDecode(v.Token) if err != nil { e := ticketvote.VoteErrorTokenInvalid receipts[k].Ticket = v.Ticket @@ -1637,8 +1870,8 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { // addrs contains the largest commitment addresses for each ticket. // The vote must be signed using the largest commitment address. - tickets := make([]string, 0, len(ballot.Votes)) - for _, v := range ballot.Votes { + tickets := make([]string, 0, len(cb.Ballot)) + for _, v := range cb.Ballot { tickets = append(tickets, v.Ticket) } addrs, err := p.largestCommitmentAddrs(tickets) @@ -1697,7 +1930,7 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { commitmentAddr := addrs[k] if commitmentAddr.ticket != v.Ticket { t := time.Now().Unix() - log.Errorf("cmdBallot: commitment addr mismatch %v: %v %v", + log.Errorf("cmdCastBallot: commitment addr mismatch %v: %v %v", t, commitmentAddr.ticket, v.Ticket) e := ticketvote.VoteErrorInternalError receipts[k].Ticket = v.Ticket @@ -1708,7 +1941,7 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { } if commitmentAddr.err != nil { t := time.Now().Unix() - log.Errorf("cmdBallot: commitment addr error %v: %v %v", + log.Errorf("cmdCastBallot: commitment addr error %v: %v %v", t, commitmentAddr.ticket, commitmentAddr.err) e := ticketvote.VoteErrorInternalError receipts[k].Ticket = v.Ticket @@ -1850,7 +2083,7 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { cvr, ok := r[v.Ticket] if !ok { t := time.Now().Unix() - log.Errorf("cmdBallot: vote result not found %v: %v", t, v.Ticket) + log.Errorf("cmdCastBallot: vote result not found %v: %v", t, v.Ticket) e := ticketvote.VoteErrorInternalError receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e @@ -1864,10 +2097,10 @@ func (p *ticketVotePlugin) cmdBallot(payload string) (string, error) { } // Prepare reply - br := ticketvote.BallotReply{ + cbr := ticketvote.CastBallotReply{ Receipts: receipts, } - reply, err := ticketvote.EncodeBallotReply(br) + reply, err := ticketvote.EncodeCastBallotReply(cbr) if err != nil { return "", err } @@ -1886,7 +2119,7 @@ func (p *ticketVotePlugin) cmdDetails(payload string) (string, error) { votes := make(map[string]ticketvote.RecordVote, len(d.Tokens)) for _, v := range d.Tokens { // Verify token - token, err := hex.DecodeString(v) + token, err := tokenDecodeAnyLength(v) if err != nil { continue } @@ -1925,17 +2158,17 @@ func (p *ticketVotePlugin) cmdDetails(payload string) (string, error) { return string(reply), nil } -func (p *ticketVotePlugin) cmdCastVotes(payload string) (string, error) { - log.Tracef("ticketvote cmdCastVotes: %v", payload) +func (p *ticketVotePlugin) cmdResults(payload string) (string, error) { + log.Tracef("ticketvote cmdResults: %v", payload) // Decode payload - cv, err := ticketvote.DecodeCastVotes([]byte(payload)) + r, err := ticketvote.DecodeResults([]byte(payload)) if err != nil { return "", err } // Verify token - token, err := hex.DecodeString(cv.Token) + token, err := tokenDecodeAnyLength(r.Token) if err != nil { return "", backend.PluginUserError{ PluginID: ticketvote.ID, @@ -1950,10 +2183,10 @@ func (p *ticketVotePlugin) cmdCastVotes(payload string) (string, error) { } // Prepare reply - cvr := ticketvote.CastVotesReply{ + rr := ticketvote.ResultsReply{ Votes: votes, } - reply, err := ticketvote.EncodeCastVotesReply(cvr) + reply, err := ticketvote.EncodeResultsReply(rr) if err != nil { return "", err } @@ -1986,18 +2219,18 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. // Vote has not been authorized yet return &ticketvote.Summary{ Status: ticketvote.VoteStatusUnauthorized, - Results: []ticketvote.Result{}, + Results: []ticketvote.VoteOptionResult{}, }, nil } lastAuth := auths[len(auths)-1] switch ticketvote.AuthActionT(lastAuth.Action) { - case ticketvote.ActionAuthorize: + case ticketvote.AuthActionAuthorize: // Vote has been authorized; continue - case ticketvote.ActionRevoke: + case ticketvote.AuthActionRevoke: // Vote authorization has been revoked return &ticketvote.Summary{ Status: ticketvote.VoteStatusUnauthorized, - Results: []ticketvote.Result{}, + Results: []ticketvote.VoteOptionResult{}, }, nil } @@ -2010,7 +2243,7 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. // Vote has not been started yet return &ticketvote.Summary{ Status: ticketvote.VoteStatusAuthorized, - Results: []ticketvote.Result{}, + Results: []ticketvote.VoteOptionResult{}, }, nil } @@ -2030,10 +2263,10 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. for _, voteBit := range votes { tally[voteBit]++ } - results := make([]ticketvote.Result, 0, len(vd.Params.Options)) + results := make([]ticketvote.VoteOptionResult, 0, len(vd.Params.Options)) for _, v := range vd.Params.Options { bit := strconv.FormatUint(v.Bit, 16) - results = append(results, ticketvote.Result{ + results = append(results, ticketvote.VoteOptionResult{ ID: v.ID, Description: v.Description, VoteBit: v.Bit, @@ -2131,7 +2364,7 @@ func (p *ticketVotePlugin) cmdSummaries(payload string) (string, error) { // Get summaries summaries := make(map[string]ticketvote.Summary, len(s.Tokens)) for _, v := range s.Tokens { - token, err := hex.DecodeString(v) + token, err := tokenDecodeAnyLength(v) if err != nil { return "", err } @@ -2264,7 +2497,7 @@ func (p *ticketVotePlugin) setup() error { ) for _, tokens := range ibs.Vetted { for _, v := range tokens { - token, err := hex.DecodeString(v) + token, err := tokenDecode(v) if err != nil { return err } @@ -2301,7 +2534,7 @@ func (p *ticketVotePlugin) setup() error { log.Infof("ticketvote: building votes cache") for k := range started { - token, err := hex.DecodeString(k) + token, err := tokenDecode(k) if err != nil { return err } @@ -2328,14 +2561,12 @@ func (p *ticketVotePlugin) cmd(cmd, payload string) (string, error) { return p.cmdAuthorize(payload) case ticketvote.CmdStart: return p.cmdStart(payload) - case ticketvote.CmdStartRunoff: - return p.cmdStartRunoff(payload) - case ticketvote.CmdBallot: - return p.cmdBallot(payload) + case ticketvote.CmdCastBallot: + return p.cmdCastBallot(payload) case ticketvote.CmdDetails: return p.cmdDetails(payload) - case ticketvote.CmdCastVotes: - return p.cmdCastVotes(payload) + case ticketvote.CmdResults: + return p.cmdResults(payload) case ticketvote.CmdSummaries: return p.cmdSummaries(payload) case ticketvote.CmdInventory: diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 0f0d35665..b514abe13 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -138,7 +138,10 @@ func tokenIsFullLength(token []byte) bool { return len(token) == v1.TokenSizeShort } -func decodeTokenFullLength(token string) ([]byte, error) { +// tokenDecode decodes the provided hex encoded record token. This function +// requires that the token be the full length. Token prefixes will return an +// error. +func tokenDecode(token string) ([]byte, error) { t, err := hex.DecodeString(token) if err != nil { return nil, fmt.Errorf("invalid hex") @@ -148,8 +151,23 @@ func decodeTokenFullLength(token string) ([]byte, error) { } return t, nil } -func tokenPrefix(token []byte) string { - return hex.EncodeToString(token)[:v1.TokenPrefixLength] + +// tokenDecodeAnyLength decodes the provided hex encoded record token. This +// function accepts both full length tokens and token prefixes. +func tokenDecodeAnyLength(token string) ([]byte, error) { + t, err := hex.DecodeString(token) + if err != nil { + return nil, fmt.Errorf("invalid hex") + } + switch { + case tokenIsFullLength(t): + // Full length tokens are allowed; continue + case len(t) == tokenPrefixSize(): + // Token prefixes are allowed; continue + default: + return nil, fmt.Errorf("invalid token size") + } + return t, nil } // tokenPrefixSize returns the size in bytes of a token prefix. @@ -169,6 +187,11 @@ func tokenPrefixSize() int { return size } +// tokenPrefix returns the token prefix as defined by the politeiad API. +func tokenPrefix(token []byte) string { + return hex.EncodeToString(token)[:v1.TokenPrefixLength] +} + func tokenFromTreeID(treeID int64) []byte { b := make([]byte, binary.MaxVarintLen64) // Converting between int64 and uint64 doesn't change @@ -178,6 +201,9 @@ func tokenFromTreeID(treeID int64) []byte { } func treeIDFromToken(token []byte) int64 { + if !tokenIsFullLength(token) { + return 0 + } return int64(binary.LittleEndian.Uint64(token)) } diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index 7daa3950b..4b984f64c 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -24,6 +24,8 @@ const ( ID = "pi" Version = "1" + // TODO refactor these commands to use the passthrough command + // Plugin commands. Many of these plugin commands rely on the // commands from other plugins, but perform additional validation // that is specific to pi or add additional functionality on top of @@ -33,6 +35,8 @@ const ( CmdCommentCensor = "commentcensor" // Censor a comment CmdCommentVote = "commentvote" // Upvote/downvote a comment CmdVoteInventory = "voteinventory" // Get inventory by vote status + CmdVoteStart = "votestart" // Start a vote + CmdPassThrough = "passthrough" // Pass a plugin cmd through pi // Metadata stream IDs MDStreamIDProposalGeneral = 1 @@ -65,15 +69,19 @@ const ( // User error status codes // TODO number error codes and add human readable error messages ErrorStatusInvalid ErrorStatusT = 0 - ErrorStatusPropStateInvalid ErrorStatusT = iota - ErrorStatusPropTokenInvalid + ErrorStatusPageSizeExceeded ErrorStatusT = iota ErrorStatusPropNotFound + ErrorStatusPropStateInvalid + ErrorStatusPropTokenInvalid ErrorStatusPropStatusInvalid ErrorStatusPropVersionInvalid ErrorStatusPropStatusChangeInvalid ErrorStatusPropLinkToInvalid ErrorStatusVoteStatusInvalid - ErrorStatusPageSizeExceeded + ErrorStatusStartDetailsInvalid + ErrorStatusStartDetailsMissing + ErrorStatusRFPInvalid + ErrorStatusLinkByDeadlineNotMet ) var ( @@ -469,3 +477,54 @@ func DecodeVoteInventoryReply(payload []byte) (*VoteInventoryReply, error) { } return &vir, nil } + +// PassThrough is used to add additional functionality onto plugin commands +// from external plugin packages without changing the base command request and +// response payloads. The command passes through the pi plugin before being +// executed. +// +// Example, the pi plugin does not allow comments to be made once a proposal +// vote has ended. This validation is specific to the pi plugin and does not +// require the comment payloads to be altered. PassThrough is used to pass the +// comments plugin command through the pi plugin, where the pi plugin can first +// perform pi specific validation before executing the comments plugin command. +type PassThrough struct { + PluginID string `json:"pluginid"` + Cmd string `json:"cmd"` + Payload string `json:"payload"` +} + +// EncodePassThrough encodes a PassThrough into a JSON byte slice. +func EncodePassThrough(p PassThrough) ([]byte, error) { + return json.Marshal(p) +} + +// DecodePassThrough decodes a JSON byte slice into a PassThrough. +func DecodePassThrough(payload []byte) (*PassThrough, error) { + var p PassThrough + err := json.Unmarshal(payload, &p) + if err != nil { + return nil, err + } + return &p, nil +} + +// PassThroughReply is the reply to the PassThrough command. +type PassThroughReply struct { + Payload string `json:"payload"` +} + +// EncodePassThroughReply encodes a PassThroughReply into a JSON byte slice. +func EncodePassThroughReply(p PassThroughReply) ([]byte, error) { + return json.Marshal(p) +} + +// DecodePassThroughReply decodes a JSON byte slice into a PassThroughReply. +func DecodePassThroughReply(payload []byte) (*PassThroughReply, error) { + var p PassThroughReply + err := json.Unmarshal(payload, &p) + if err != nil { + return nil, err + } + return &p, nil +} diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index 9df3e5aab..5b03da3a8 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -24,15 +24,13 @@ const ( Version = "1" // Plugin commands - CmdAuthorize = "authorize" // Authorize a vote - CmdStart = "start" // Start a vote - CmdStartRunoff = "startrunoff" // Start a runoff vote - CmdBallot = "ballot" // Cast a ballot of votes - CmdDetails = "details" // Get vote details - CmdCastVotes = "castvotes" // Get cast votes - CmdSummaries = "summaries" // Get vote summaries - CmdInventory = "inventory" // Get inventory grouped by vote status - CmdProofs = "proofs" // Get inclusion proofs + CmdAuthorize = "authorize" // Authorize a vote + CmdStart = "start" // Start a vote + CmdCastBallot = "castballot" // Cast a ballot of votes + CmdDetails = "details" // Get vote details + CmdResults = "results" // Get vote results + CmdSummaries = "summaries" // Get vote summaries + CmdInventory = "inventory" // Get inventory by vote status // Default plugin settings DefaultMainNetVoteDurationMin = 2016 @@ -49,21 +47,22 @@ const ( // Vote statuses VoteStatusInvalid VoteStatusT = 0 // Invalid status - VoteStatusUnauthorized VoteStatusT = 1 // Vote cannot be started - VoteStatusAuthorized VoteStatusT = 2 // Vote can be started + VoteStatusUnauthorized VoteStatusT = 1 // Vote has not been authorized + VoteStatusAuthorized VoteStatusT = 2 // Vote has been authorized VoteStatusStarted VoteStatusT = 3 // Vote has been started VoteStatusFinished VoteStatusT = 4 // Vote has finished // Authorize vote actions - ActionAuthorize AuthActionT = "authorize" - ActionRevoke AuthActionT = "revoke" + AuthActionAuthorize AuthActionT = "authorize" + AuthActionRevoke AuthActionT = "revoke" // Vote types VoteTypeInvalid VoteT = 0 // VoteTypeStandard is used to indicate a simple approve or reject // vote where the winner is the voting option that has met the - // specified quorum and pass requirements. + // specified quorum and pass requirements. Standard votes must be + // authorized before the vote can be started. VoteTypeStandard VoteT = 1 // VoteTypeRunoff specifies a runoff vote that multiple records @@ -76,7 +75,8 @@ const ( // then all records are considered rejected. Note, in a runoff vote // it's possible for a record to meet both the quorum and pass // requirements but still be rejected if it does not have the most - // net yes votes. + // net yes votes. Runoff vote participants are not required to have + // the voting period authorized prior to the vote starting. VoteTypeRunoff VoteT = 2 // VoteOptionIDApprove is the vote option ID that indicates the vote @@ -110,12 +110,13 @@ const ( ErrorStatusPublicKeyInvalid ErrorStatusT = 2 ErrorStatusSignatureInvalid ErrorStatusT = 3 ErrorStatusRecordNotFound ErrorStatusT = 4 - ErrorStatusRecordStatusInvalid ErrorStatusT = 5 - ErrorStatusAuthorizationInvalid ErrorStatusT = 6 - ErrorStatusVoteParamsInvalid ErrorStatusT = 7 - ErrorStatusVoteStatusInvalid ErrorStatusT = 8 - ErrorStatusBallotInvalid ErrorStatusT = 9 - ErrorStatusPageSizeExceeded ErrorStatusT = 10 + ErrorStatusRecordVersionInvalid ErrorStatusT = 5 + ErrorStatusRecordStatusInvalid ErrorStatusT = 6 + ErrorStatusAuthorizationInvalid ErrorStatusT = 7 + ErrorStatusStartDetailsInvalid ErrorStatusT = 8 + ErrorStatusVoteParamsInvalid ErrorStatusT = 9 + ErrorStatusVoteStatusInvalid ErrorStatusT = 10 + ErrorStatusPageSizeExceeded ErrorStatusT = 11 ) var ( @@ -151,15 +152,14 @@ var ( ErrorStatusAuthorizationInvalid: "authorization invalid", ErrorStatusVoteParamsInvalid: "vote params invalid", ErrorStatusVoteStatusInvalid: "vote status invalid", - ErrorStatusBallotInvalid: "ballot invalid", ErrorStatusPageSizeExceeded: "page size exceeded", } ) -// AuthorizeDetails is the structure that is saved to disk when a vote is -// authorized or a previous authorization is revoked. It contains all the -// fields from a Authorize and a AuthorizeReply. -type AuthorizeDetails struct { +// AuthDetails is the structure that is saved to disk when a vote is authorized +// or a previous authorization is revoked. It contains all the fields from a +// Authorize and a AuthorizeReply. +type AuthDetails struct { // Data generated by client Token string `json:"token"` // Record token Version uint32 `json:"version"` // Record version @@ -203,8 +203,6 @@ type VoteParams struct { // // Signature is the client signature of the SHA256 digest of the JSON encoded // Vote struct. -// -// TODO does this need a receipt? type VoteDetails struct { // Data generated by client Params VoteParams `json:"params"` @@ -281,16 +279,24 @@ func DecodeAuthorizeReply(payload []byte) (*AuthorizeReply, error) { return &ar, nil } -// Start starts a ticket vote. +// StartDetails is the structure that is provided when starting a ticket vote. // -// Signature is the signature of a SHA256 digest of the JSON encoded Vote +// Signature is the signature of a SHA256 digest of the JSON encoded VoteParams // structure. -type Start struct { +type StartDetails struct { Params VoteParams `json:"params"` PublicKey string `json:"publickey"` // Public key used for signature Signature string `json:"signature"` // Client signature } +// Start starts a ticket vote. +// +// Signature is the signature of a SHA256 digest of the JSON encoded VoteParams +// structure. +type Start struct { + Starts []StartDetails `json:"starts"` +} + // EncodeStart encodes a Start into a JSON byte slice. func EncodeStart(s Start) ([]byte, error) { return json.Marshal(s) @@ -307,8 +313,6 @@ func DecodeStart(payload []byte) (*Start, error) { } // StartReply is the reply to the Start command. -// -// TODO should this return a receipt? type StartReply struct { StartBlockHeight uint32 `json:"startblockheight"` StartBlockHash string `json:"startblockhash"` @@ -331,54 +335,6 @@ func DecodeStartReply(payload []byte) (*StartReply, error) { return &sr, nil } -// StartRunoff starts a runoff vote between the provided submissions. Each -// submission is required to have its own Authorize and Start. -type StartRunoff struct { - Token string `json:"token"` // RFP token - Auths []Authorize `json:"auths"` - Starts []Start `json:"starts"` -} - -// EncodeStartRunoff encodes a StartRunoff into a JSON byte slice. -func EncodeStartRunoff(sr StartRunoff) ([]byte, error) { - return json.Marshal(sr) -} - -// DecodeStartRunoff decodes a JSON byte slice into a StartRunoff. -func DecodeStartRunoff(payload []byte) (*StartRunoff, error) { - var sr StartRunoff - err := json.Unmarshal(payload, &sr) - if err != nil { - return nil, err - } - return &sr, nil -} - -// StartRunoffReply is the reply to the StartRunoff command. -// -// TODO should this return a receipt? -type StartRunoffReply struct { - StartBlockHeight uint32 `json:"startblockheight"` - StartBlockHash string `json:"startblockhash"` - EndBlockHeight uint32 `json:"endblockheight"` - EligibleTickets []string `json:"eligibletickets"` // Ticket hashes -} - -// EncodeStartRunoffReply encodes a StartRunoffReply into a JSON byte slice. -func EncodeStartRunoffReply(srr StartRunoffReply) ([]byte, error) { - return json.Marshal(srr) -} - -// DecodeStartRunoffReply decodes a JSON byte slice into a StartRunoffReply. -func DecodeStartRunoffReply(payload []byte) (*StartRunoffReply, error) { - var srr StartRunoffReply - err := json.Unmarshal(payload, &srr) - if err != nil { - return nil, err - } - return &srr, nil -} - // CastVote is a signed ticket vote. This structure gets saved to disk when // a vote is cast. type CastVote struct { @@ -399,20 +355,20 @@ type CastVoteReply struct { ErrorContext string `json:"errorcontext,omitempty"` } -// Ballot is a batch of votes that are sent to the server. A ballot can only -// contain the votes for a single record. -type Ballot struct { - Votes []CastVote `json:"votes"` +// CastBallot casts a ballot of votes. A ballot can only contain votes for a +// single record. +type CastBallot struct { + Ballot []CastVote `json:"ballot"` } -// EncodeBallot encodes a Ballot into a JSON byte slice. -func EncodeBallot(b Ballot) ([]byte, error) { +// EncodeCastBallot encodes a CastBallot into a JSON byte slice. +func EncodeCastBallot(b CastBallot) ([]byte, error) { return json.Marshal(b) } -// DecodeBallot decodes a JSON byte slice into a Ballot. -func DecodeBallot(payload []byte) (*Ballot, error) { - var b Ballot +// DecodeCastBallot decodes a JSON byte slice into a CastBallot. +func DecodeCastBallot(payload []byte) (*CastBallot, error) { + var b CastBallot err := json.Unmarshal(payload, &b) if err != nil { return nil, err @@ -420,19 +376,19 @@ func DecodeBallot(payload []byte) (*Ballot, error) { return &b, nil } -// BallotReply is a reply to a batched list of votes. -type BallotReply struct { +// CastBallotReply is a reply to a batched list of votes. +type CastBallotReply struct { Receipts []CastVoteReply `json:"receipts"` } -// EncodeBallotReply encodes a Ballot into a JSON byte slice. -func EncodeBallotReply(b BallotReply) ([]byte, error) { +// EncodeCastBallotReply encodes a CastBallot into a JSON byte slice. +func EncodeCastBallotReply(b CastBallotReply) ([]byte, error) { return json.Marshal(b) } -// DecodeBallotReply decodes a JSON byte slice into a BallotReply. -func DecodeBallotReply(payload []byte) (*BallotReply, error) { - var b BallotReply +// DecodeCastBallotReply decodes a JSON byte slice into a CastBallotReply. +func DecodeCastBallotReply(payload []byte) (*CastBallotReply, error) { + var b CastBallotReply err := json.Unmarshal(payload, &b) if err != nil { return nil, err @@ -463,8 +419,8 @@ func DecodeDetails(payload []byte) (*Details, error) { // RecordVote contains all vote authorizations and the vote details for a // record. The VoteDetails will be nil if the vote has been started. type RecordVote struct { - Auths []AuthorizeDetails `json:"auths"` - Vote *VoteDetails `json:"vote"` + Auths []AuthDetails `json:"auths"` + Vote *VoteDetails `json:"vote"` } // DetailsReply is the reply to the Details command. The returned map will not @@ -490,44 +446,44 @@ func DecodeDetailsReply(payload []byte) (*DetailsReply, error) { return &dr, nil } -// CastVotes requests the cast votes for the provided record token. -type CastVotes struct { +// Results requests the results of a vote. +type Results struct { Token string `json:"token"` } -// EncodeCastVotes encodes a CastVotes into a JSON byte slice. -func EncodeCastVotes(cv CastVotes) ([]byte, error) { - return json.Marshal(cv) +// EncodeResults encodes a Results into a JSON byte slice. +func EncodeResults(r Results) ([]byte, error) { + return json.Marshal(r) } -// DecodeCastVotes decodes a JSON byte slice into a CastVotes. -func DecodeCastVotes(payload []byte) (*CastVotes, error) { - var cv CastVotes - err := json.Unmarshal(payload, &cv) +// DecodeResults decodes a JSON byte slice into a Results. +func DecodeResults(payload []byte) (*Results, error) { + var r Results + err := json.Unmarshal(payload, &r) if err != nil { return nil, err } - return &cv, nil + return &r, nil } -// CastVotesReply is the rely to the CastVotes command. -type CastVotesReply struct { +// ResultsReply is the rely to the Results command. +type ResultsReply struct { Votes []CastVoteDetails `json:"votes"` } -// EncodeCastVotesReply encodes a CastVotesReply into a JSON byte slice. -func EncodeCastVotesReply(cvr CastVotesReply) ([]byte, error) { - return json.Marshal(cvr) +// EncodeResultsReply encodes a ResultsReply into a JSON byte slice. +func EncodeResultsReply(rr ResultsReply) ([]byte, error) { + return json.Marshal(rr) } -// DecodeCastVotesReply decodes a JSON byte slice into a CastVotesReply. -func DecodeCastVotesReply(payload []byte) (*CastVotesReply, error) { - var cvr CastVotesReply - err := json.Unmarshal(payload, &cvr) +// DecodeResultsReply decodes a JSON byte slice into a ResultsReply. +func DecodeResultsReply(payload []byte) (*ResultsReply, error) { + var rr ResultsReply + err := json.Unmarshal(payload, &rr) if err != nil { return nil, err } - return &cvr, nil + return &rr, nil } // Summaries requests the vote summaries for the provided record tokens. @@ -550,9 +506,9 @@ func DecodeSummaries(payload []byte) (*Summaries, error) { return &s, nil } -// Result describes a vote option and the total number of votes that have been -// cast for this option. -type Result struct { +// VoteOptionResult describes a vote option and the total number of votes that +// have been cast for this option. +type VoteOptionResult struct { ID string `json:"id"` // Single unique word (e.g. yes) Description string `json:"description"` // Longer description of the vote VoteBit uint64 `json:"votebit"` // Bits used for this option @@ -561,16 +517,16 @@ type Result struct { // Summary summarizes the vote params and results for a ticket vote. type Summary struct { - Type VoteT `json:"type"` - Status VoteStatusT `json:"status"` - Duration uint32 `json:"duration"` - StartBlockHeight uint32 `json:"startblockheight"` - StartBlockHash string `json:"startblockhash"` - EndBlockHeight uint32 `json:"endblockheight"` - EligibleTickets uint32 `json:"eligibletickets"` - QuorumPercentage uint32 `json:"quorumpercentage"` - PassPercentage uint32 `json:"passpercentage"` - Results []Result `json:"results"` + Type VoteT `json:"type"` + Status VoteStatusT `json:"status"` + Duration uint32 `json:"duration"` + StartBlockHeight uint32 `json:"startblockheight"` + StartBlockHash string `json:"startblockhash"` + EndBlockHeight uint32 `json:"endblockheight"` + EligibleTickets uint32 `json:"eligibletickets"` + QuorumPercentage uint32 `json:"quorumpercentage"` + PassPercentage uint32 `json:"passpercentage"` + Results []VoteOptionResult `json:"results"` // Approved describes whether the vote has been approved. This will // only be present when the vote type is VoteTypeStandard or @@ -629,7 +585,7 @@ func DecodeInventory(payload []byte) (*Inventory, error) { // InventoryReply is the reply to the Inventory command. It contains the tokens // of all public, non-abandoned records categorized by vote status. -// TODO +// // Sorted by timestamp in descending order: // Unauthorized, Authorized // diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 40b7d9c81..4554fa923 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -17,19 +17,19 @@ type VoteAuthActionT string type VoteT int type VoteErrorT int -// TODO the plugin policies should be returned in a route -// TODO show the difference between unvetted censored and vetted censored -// in the proposal inventory route since fetching them requires specifying -// the state. // TODO verify that all batched request have a page size limit +// TODO add fetching proposals by user ID. Will probably need to add user ID +// to the general metadata stream. +// TODO add a comments/count endpoint and take the comments count off of the +// proposal record +// TODO linkedfrom should really be pulled out of a proposal record an added +// as a separate enpoint as well. +// TODO create a comments and vote api for those plugin commands that are not +// pi specific +// TODO the plugin policies should be returned in a route // TODO make RouteVoteResults a batched route but that only currently allows // for 1 result to be returned so that we have the option to change this is // we want to. -// TODO should we add auths to the vote summary? -// TODO should routes for fetching comments be in their own API? This would -// make politeiawww far more configurable. I think so. -// TODO add a comments/count endpoint and take the comments count off of the -// proposal record const ( APIVersion = 1 @@ -52,14 +52,13 @@ const ( RouteCommentVotes = "/comments/votes" // Vote routes - RouteVoteAuthorize = "/vote/authorize" - RouteVoteStart = "/vote/start" - RouteVoteStartRunoff = "/vote/startrunoff" - RouteVoteBallot = "/vote/ballot" - RouteVotes = "/votes" - RouteVoteResults = "/votes/results" - RouteVoteSummaries = "/votes/summaries" - RouteVoteInventory = "/votes/inventory" + RouteVoteAuthorize = "/vote/authorize" + RouteVoteStart = "/vote/start" + RouteCastBallot = "/vote/castballot" + RouteVotes = "/votes" + RouteVoteResults = "/votes/results" + RouteVoteSummaries = "/votes/summaries" + RouteVoteInventory = "/votes/inventory" // Proposal states. A proposal state can be either unvetted or // vetted. The PropStatusT type further breaks down these two @@ -84,8 +83,8 @@ const ( // Vote statuses VoteStatusInvalid VoteStatusT = 0 // Invalid status - VoteStatusUnauthorized VoteStatusT = 1 // Vote cannot be started - VoteStatusAuthorized VoteStatusT = 2 // Vote can be started + VoteStatusUnauthorized VoteStatusT = 1 // Vote has not been authorized + VoteStatusAuthorized VoteStatusT = 2 // Vote has been authorized VoteStatusStarted VoteStatusT = 3 // Vote has been started VoteStatusFinished VoteStatusT = 4 // Vote has finished @@ -101,15 +100,16 @@ const ( // specified quorum and pass requirements. VoteTypeStandard VoteT = 1 - // VoteTypeRunoff specifies a runoff vote that multiple records - // compete in. All records are voted on like normal, but there can - // only be one winner in a runoff vote. The winner is the record - // that meets the quorum requirement, meets the pass requirement, - // and that has the most net yes votes. The winning record is - // considered approved and all other records are considered to be - // rejected. If no records meet the quorum and pass requirements - // then all records are considered rejected. Note, in a runoff vote - // it's possible for a proposal to meet both the quorum and pass + // VoteTypeRunoff specifies a runoff vote that multiple proposals + // compete in. All proposals are voted on like normal and all votes + // are simple approve/reject votes, but there can only be one + // winner in a runoff vote. The winner is the proposal that meets + // the quorum requirement, meets the pass requirement, and that has + // the most net yes votes. The winning proposal is considered + // approved and all other proposals are considered to be rejected. + // If no proposals meet the quorum and pass requirements then all + // proposals are considered rejected. Note, in a runoff vote it is + // possible for a proposal to meet both the quorum and pass // requirements but still be rejected if it does not have the most // net yes votes. VoteTypeRunoff VoteT = 2 @@ -124,6 +124,19 @@ const ( // this vote option ID. VoteOptionIDReject = "no" + // Cast vote errors + // TODO these need human readable equivalents + VoteErrorInvalid VoteErrorT = 0 + VoteErrorInternalError VoteErrorT = 1 + VoteErrorTokenInvalid VoteErrorT = 2 + VoteErrorRecordNotFound VoteErrorT = 3 + VoteErrorMultipleRecordVotes VoteErrorT = 4 + VoteErrorVoteStatusInvalid VoteErrorT = 5 + VoteErrorVoteBitInvalid VoteErrorT = 6 + VoteErrorSignatureInvalid VoteErrorT = 7 + VoteErrorTicketNotEligible VoteErrorT = 8 + VoteErrorTicketAlreadyVoted VoteErrorT = 9 + // Error status codes ErrorStatusInvalid ErrorStatusT = 0 ErrorStatusInputInvalid ErrorStatusT = 1 @@ -178,20 +191,6 @@ const ( ErrorStatusVoteAuthInvalid ErrorStatusVoteStatusInvalid ErrorStatusVoteParamsInvalid - ErrorStatusBallotInvalid - - // Cast vote errors - // TODO these need human readable equivalents - VoteErrorInvalid VoteErrorT = 0 - VoteErrorInternalError VoteErrorT = 1 - VoteErrorTokenInvalid VoteErrorT = 2 - VoteErrorRecordNotFound VoteErrorT = 3 - VoteErrorMultipleRecordVotes VoteErrorT = 4 - VoteErrorVoteStatusInvalid VoteErrorT = 5 - VoteErrorVoteBitInvalid VoteErrorT = 6 - VoteErrorSignatureInvalid VoteErrorT = 7 - VoteErrorTicketNotEligible VoteErrorT = 8 - VoteErrorTicketAlreadyVoted VoteErrorT = 9 ) var ( @@ -257,7 +256,6 @@ var ( // Vote errors ErrorStatusVoteStatusInvalid: "vote status invalid", ErrorStatusVoteParamsInvalid: "vote params invalid", - ErrorStatusBallotInvalid: "ballot invalid", } ) @@ -625,8 +623,8 @@ type CommentVotesReply struct { Votes []CommentVoteDetails `json:"votes"` } -// AuthorizeDetails contains the details of a vote authorization. -type AuthorizeDetails struct { +// AuthDetails contains the details of a vote authorization. +type AuthDetails struct { Token string `json:"token"` // Proposal token Version uint32 `json:"version"` // Proposal version Action string `json:"action"` // Authorize or revoke @@ -742,17 +740,32 @@ type VoteAuthorizeReply struct { Receipt string `json:"receipt"` } -// VoteStart starts a proposal vote. All proposal votes must be authorized -// by the proposal author before an admin is able to start the voting process. +// StartDetails is the structure that is provided when starting a proposal vote. // -// Signature is the signature of a SHA256 digest of the JSON encoded Vote +// Signature is the signature of a SHA256 digest of the JSON encoded VoteParams // structure. -type VoteStart struct { +type StartDetails struct { Params VoteParams `json:"params"` PublicKey string `json:"publickey"` Signature string `json:"signature"` } +// VoteStart starts a proposal vote or multiple proposal votes if the vote is +// a runoff vote. +// +// Standard votes require that the vote have been authorized by the proposal +// author before an admin will able to start the voting process. The +// StartDetails list should only contain a single StartDetails. +// +// Runoff votes can be started by an admin at any point once the RFP link by +// deadline has expired. Runoff votes DO NOT require the votes to have been +// authorized by the submission authors prior to an admin starting the runoff +// vote. All public, non-abandoned RFP submissions should be included in the +// list of StartDetails. +type VoteStart struct { + Starts []StartDetails `json:"starts"` +} + // VoteStartReply is the reply to the VoteStart command. type VoteStartReply struct { StartBlockHeight uint32 `json:"startblockheight"` @@ -761,22 +774,6 @@ type VoteStartReply struct { EligibleTickets []string `json:"eligibletickets"` } -// VoteStartRunoff starts a runoff vote between the provided submissions. Each -// submission is required to have its own Authorize and Start. -type VoteStartRunoff struct { - Token string `json:"token"` // RFP token - Auths []VoteAuthorize `json:"auths"` - Starts []VoteStart `json:"starts"` -} - -// VoteStartRunoffReply is the reply to the VoteStartRunoff command. -type VoteStartRunoffReply struct { - StartBlockHeight uint32 `json:"startblockheight"` - StartBlockHash string `json:"startblockhash"` - EndBlockHeight uint32 `json:"endblockheight"` - EligibleTickets []string `json:"eligibletickets"` // Ticket hashes -} - // CastVote is a signed ticket vote. type CastVote struct { Token string `json:"token"` // Proposal token @@ -796,14 +793,14 @@ type CastVoteReply struct { ErrorContext string `json:"errorcontext,omitempty"` } -// VoteBallot is a batch of votes that are sent to the server. A ballot can only -// contain the votes for a single record. -type VoteBallot struct { +// CastBallot casts a ballot of votes. A ballot can only contain the votes for +// a single record. +type CastBallot struct { Votes []CastVote `json:"votes"` } -// VoteBallotReply is a reply to a batched list of votes. -type VoteBallotReply struct { +// CastBallotReply is a reply to a batched list of votes. +type CastBallotReply struct { Receipts []CastVoteReply `json:"receipts"` } @@ -811,8 +808,8 @@ type VoteBallotReply struct { // proposal vote. The vote details will be null if the proposal vote has not // been started yet. type ProposalVote struct { - Auths []AuthorizeDetails `json:"auths"` - Vote *VoteDetails `json:"vote"` + Auths []AuthDetails `json:"auths"` + Vote *VoteDetails `json:"vote"` } // Votes returns the vote authorizations and vote details for each of the diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index 51de18616..893ddce7f 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -83,7 +83,7 @@ type piwww struct { VoteAuthorize voteAuthorizeCmd `command:"voteauthorize"` VoteStart voteStartCmd `command:"votestart"` VoteStartRunoff voteStartRunoffCmd `command:"votestartrunoff"` - VoteBallot voteBallotCmd `command:"voteballot"` + CastBallot castBallotCmd `command:"castballot"` Votes votesCmd `command:"votes"` VoteResults voteResultsCmd `command:"voteresults"` VoteSummaries voteSummariesCmd `command:"votesummaries"` @@ -168,7 +168,7 @@ Dev commands ` func _main() error { - // Load config. The config variable is a CLI global variable. + // Load config. The config variable is aglobal variable. var err error cfg, err = shared.LoadConfig(defaultHomeDir, defaultDataDirname, defaultConfigFilename) @@ -176,7 +176,7 @@ func _main() error { return fmt.Errorf("load config: %v", err) } - // Load client. The client variable is a CLI global variable. + // Load client. The client variable is a global variable. client, err = shared.NewClient(cfg) if err != nil { return fmt.Errorf("load client: %v", err) diff --git a/politeiawww/cmd/piwww/voteballot.go b/politeiawww/cmd/piwww/voteballot.go index 928fc8f97..fa2fd85c1 100644 --- a/politeiawww/cmd/piwww/voteballot.go +++ b/politeiawww/cmd/piwww/voteballot.go @@ -19,8 +19,8 @@ import ( "golang.org/x/crypto/ssh/terminal" ) -// voteBallotCmd casts a ballot of votes for the specified proposal. -type voteBallotCmd struct { +// castBallotCmd casts a ballot of votes for the specified proposal. +type castBallotCmd struct { Args struct { Token string `positional-arg-name:"token"` VoteID string `positional-arg-name:"voteid"` @@ -29,7 +29,7 @@ type voteBallotCmd struct { } // Execute executes the vote ballot command. -func (c *voteBallotCmd) Execute(args []string) error { +func (c *castBallotCmd) Execute(args []string) error { token := c.Args.Token voteID := c.Args.VoteID @@ -145,14 +145,14 @@ func (c *voteBallotCmd) Execute(args []string) error { Signature: hex.EncodeToString(sigs.Replies[i].Signature), }) } - vb := pi.VoteBallot{ + cb := pi.CastBallot{ Votes: votes, } // Send ballot request - vbr, err := client.VoteBallot(vb) + cbr, err := client.CastBallot(cb) if err != nil { - return fmt.Errorf("VoteBallot: %v", err) + return fmt.Errorf("CastBallot: %v", err) } // Get the server pubkey so that we can validate the receipts. @@ -169,9 +169,9 @@ func (c *voteBallotCmd) Execute(args []string) error { // ticket hash so in order to associate a failed receipt with a // specific ticket, we need to lookup the ticket hash and store // it separately. - failedReceipts := make([]pi.CastVoteReply, 0, len(vbr.Receipts)) + failedReceipts := make([]pi.CastVoteReply, 0, len(cbr.Receipts)) failedTickets := make([]string, 0, len(eligibleTickets)) - for i, v := range vbr.Receipts { + for i, v := range cbr.Receipts { // Lookup ticket hash. br.Receipts and eligibleTickets use the // same ordering h := eligibleTickets[i] @@ -198,7 +198,7 @@ func (c *voteBallotCmd) Execute(args []string) error { // Print results if !cfg.Silent { - fmt.Printf("Votes succeeded: %v\n", len(vbr.Receipts)-len(failedReceipts)) + fmt.Printf("Votes succeeded: %v\n", len(cbr.Receipts)-len(failedReceipts)) fmt.Printf("Votes failed : %v\n", len(failedReceipts)) for i, v := range failedReceipts { fmt.Printf("Failed vote : %v %v\n", failedTickets[i], v.ErrorContext) @@ -208,8 +208,8 @@ func (c *voteBallotCmd) Execute(args []string) error { return nil } -// voteBallotHelpMsg is the help command message. -const voteBallotHelpMsg = `voteballot "token" "voteid" +// castBallotHelpMsg is the help command message. +const castBallotHelpMsg = `castballot "token" "voteid" Cast a ballot of ticket votes for a proposal. This command will only work when on testnet and when running dcrwallet locally on the default port. diff --git a/politeiawww/cmd/piwww/votestart.go b/politeiawww/cmd/piwww/votestart.go index 598906f17..570a0a4a7 100644 --- a/politeiawww/cmd/piwww/votestart.go +++ b/politeiawww/cmd/piwww/votestart.go @@ -91,9 +91,13 @@ func (cmd *voteStartCmd) Execute(args []string) error { b := cfg.Identity.SignMessage([]byte(msg)) signature := hex.EncodeToString(b[:]) vs := pi.VoteStart{ - Params: vote, - PublicKey: cfg.Identity.Public.String(), - Signature: signature, + Starts: []pi.StartDetails{ + { + Params: vote, + PublicKey: cfg.Identity.Public.String(), + Signature: signature, + }, + }, } // Send request. The request and response details are printed to diff --git a/politeiawww/cmd/piwww/votestartrunoff.go b/politeiawww/cmd/piwww/votestartrunoff.go index 966445c06..b14bed541 100644 --- a/politeiawww/cmd/piwww/votestartrunoff.go +++ b/politeiawww/cmd/piwww/votestartrunoff.go @@ -85,22 +85,8 @@ func (cmd *voteStartRunoffCmd) Execute(args []string) error { } } - // Prepare VoteAuthorize for each submission - auths := make([]pi.VoteAuthorize, 0, len(submissions)) - for _, v := range submissions { - action := pi.VoteAuthActionAuthorize - msg := v.CensorshipRecord.Token + v.Version + string(action) - sig := cfg.Identity.SignMessage([]byte(msg)) - auths = append(auths, pi.VoteAuthorize{ - Token: v.CensorshipRecord.Token, - Action: action, - PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), - Signature: hex.EncodeToString(sig[:]), - }) - } - // Prepare VoteStart for each submission - starts := make([]pi.VoteStart, 0, len(submissions)) + starts := make([]pi.StartDetails, 0, len(submissions)) for _, v := range submissions { version, err := strconv.ParseUint(v.Version, 10, 32) if err != nil { @@ -135,7 +121,7 @@ func (cmd *voteStartRunoffCmd) Execute(args []string) error { msg := hex.EncodeToString(util.Digest(vb)) sig := cfg.Identity.SignMessage([]byte(msg)) - starts = append(starts, pi.VoteStart{ + starts = append(starts, pi.StartDetails{ Params: vote, PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), Signature: hex.EncodeToString(sig[:]), @@ -143,16 +129,14 @@ func (cmd *voteStartRunoffCmd) Execute(args []string) error { } // Prepare and send request - svr := pi.VoteStartRunoff{ - Token: cmd.Args.TokenRFP, - Auths: auths, + vs := pi.VoteStart{ Starts: starts, } - err = shared.PrintJSON(svr) + err = shared.PrintJSON(vs) if err != nil { return err } - svrr, err := client.VoteStartRunoff(svr) + vsr, err := client.VoteStart(vs) if err != nil { return err } @@ -160,9 +144,8 @@ func (cmd *voteStartRunoffCmd) Execute(args []string) error { // Print response details. Remove ticket snapshot from // the response before printing so that the output is // legible. - m := "removed by piwww for readability" - svrr.EligibleTickets = []string{m} - err = shared.PrintJSON(svrr) + vsr.EligibleTickets = []string{"removed by piwww for readability"} + err = shared.PrintJSON(vsr) if err != nil { return err } diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index 2f6d00790..b2bc7daee 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -1656,36 +1656,6 @@ func (c *Client) VoteStart(vs pi.VoteStart) (*pi.VoteStartReply, error) { return &vsr, nil } -// VoteStartRunoff sends the given VoteStartRunoff to the pi api -// RouteVoteStartRunoff and returns the reply. -func (c *Client) VoteStartRunoff(vsr pi.VoteStartRunoff) (*pi.VoteStartRunoffReply, error) { - statusCode, respBody, err := c.makeRequest(http.MethodPost, - pi.APIRoute, pi.RouteVoteStartRunoff, vsr) - if err != nil { - return nil, err - } - - if statusCode != http.StatusOK { - return nil, piError(respBody, statusCode) - } - - var vsrr pi.VoteStartRunoffReply - err = json.Unmarshal(respBody, &vsrr) - if err != nil { - return nil, err - } - - if c.cfg.Verbose { - vsrr.EligibleTickets = []string{"removed by piwww for readability"} - err := prettyPrintJSON(vsrr) - if err != nil { - return nil, err - } - } - - return &vsrr, nil -} - // UserRegistrationPayment checks whether the logged in user has paid their user // registration fee. func (c *Client) UserRegistrationPayment() (*www.UserRegistrationPaymentReply, error) { @@ -2034,10 +2004,10 @@ func (c *Client) ActiveVotesDCC() (*cms.ActiveVoteReply, error) { return &avr, nil } -// VoteBallot casts a ballot of votes for a proposal. -func (c *Client) VoteBallot(vb pi.VoteBallot) (*pi.VoteBallotReply, error) { +// CastBallot casts a ballot of votes for a proposal. +func (c *Client) CastBallot(cb pi.CastBallot) (*pi.CastBallotReply, error) { statusCode, respBody, err := c.makeRequest(http.MethodPost, - pi.APIRoute, pi.RouteVoteBallot, vb) + pi.APIRoute, pi.RouteCastBallot, cb) if err != nil { return nil, err } @@ -2046,20 +2016,20 @@ func (c *Client) VoteBallot(vb pi.VoteBallot) (*pi.VoteBallotReply, error) { return nil, piError(respBody, statusCode) } - var vbr pi.VoteBallotReply - err = json.Unmarshal(respBody, &vbr) + var cbr pi.CastBallotReply + err = json.Unmarshal(respBody, &cbr) if err != nil { - return nil, fmt.Errorf("unmarshal VoteBallotReply: %v", err) + return nil, fmt.Errorf("unmarshal CastBallotReply: %v", err) } if c.cfg.Verbose { - err := prettyPrintJSON(vbr) + err := prettyPrintJSON(cbr) if err != nil { return nil, err } } - return &vbr, nil + return &cbr, nil } // UpdateUserKey updates the identity of the logged in user. diff --git a/politeiawww/cmd/shared/config.go b/politeiawww/cmd/shared/config.go index d8381b5a7..76d64fc93 100644 --- a/politeiawww/cmd/shared/config.go +++ b/politeiawww/cmd/shared/config.go @@ -45,7 +45,7 @@ type Config struct { HomeDir string `long:"appdata" description:"Path to application home directory"` Host string `long:"host" description:"politeiawww host"` RawJSON bool `short:"j" long:"json" description:"Print raw JSON output"` - ShowVersion bool `long:"version" description:"Display version information and exit"` + ShowVersion bool `short:"V" long:"version" description:"Display version information and exit"` SkipVerify bool `long:"skipverify" description:"Skip verifying the server's certifcate chain and host name"` Verbose bool `short:"v" long:"verbose" description:"Print verbose output"` Silent bool `long:"silent" description:"Suppress all output"` diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index e0b2ee477..c212f295f 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -426,11 +426,11 @@ func convertCommentVoteFromPi(v pi.CommentVoteT) piplugin.CommentVoteT { func convertVoteAuthActionFromPi(a pi.VoteAuthActionT) ticketvote.AuthActionT { switch a { case pi.VoteAuthActionAuthorize: - return ticketvote.ActionAuthorize + return ticketvote.AuthActionAuthorize case pi.VoteAuthActionRevoke: - return ticketvote.ActionRevoke + return ticketvote.AuthActionRevoke default: - return ticketvote.ActionAuthorize + return ticketvote.AuthActionAuthorize } } @@ -486,11 +486,21 @@ func convertVoteParamsFromPi(v pi.VoteParams) ticketvote.VoteParams { return tv } +func convertStartDetailsFromPi(sd pi.StartDetails) ticketvote.StartDetails { + return ticketvote.StartDetails{ + Params: convertVoteParamsFromPi(sd.Params), + PublicKey: sd.PublicKey, + Signature: sd.Signature, + } +} + func convertVoteStartFromPi(vs pi.VoteStart) ticketvote.Start { + starts := make([]ticketvote.StartDetails, 0, len(vs.Starts)) + for _, v := range vs.Starts { + starts = append(starts, convertStartDetailsFromPi(v)) + } return ticketvote.Start{ - Params: convertVoteParamsFromPi(vs.Params), - PublicKey: vs.PublicKey, - Signature: vs.Signature, + Starts: starts, } } @@ -595,10 +605,10 @@ func convertVoteDetailsFromPlugin(vd ticketvote.VoteDetails) pi.VoteDetails { } } -func convertAuthorizeDetailsFromPlugin(auths []ticketvote.AuthorizeDetails) []pi.AuthorizeDetails { - a := make([]pi.AuthorizeDetails, 0, len(auths)) +func convertAuthDetailsFromPlugin(auths []ticketvote.AuthDetails) []pi.AuthDetails { + a := make([]pi.AuthDetails, 0, len(auths)) for _, v := range auths { - a = append(a, pi.AuthorizeDetails{ + a = append(a, pi.AuthDetails{ Token: v.Token, Version: v.Version, Action: v.Action, @@ -634,7 +644,7 @@ func convertProposalVotesFromPlugin(votes map[string]ticketvote.RecordVote) map[ vdp = &vd } pv[k] = pi.ProposalVote{ - Auths: convertAuthorizeDetailsFromPlugin(v.Auths), + Auths: convertAuthDetailsFromPlugin(v.Auths), Vote: vdp, } } @@ -1875,6 +1885,8 @@ func (p *politeiawww) processVoteAuthorize(ctx context.Context, va pi.VoteAuthor return nil, err } + // TODO Emit notification + return &pi.VoteAuthorizeReply{ Timestamp: ar.Timestamp, Receipt: ar.Receipt, @@ -1882,7 +1894,7 @@ func (p *politeiawww) processVoteAuthorize(ctx context.Context, va pi.VoteAuthor } func (p *politeiawww) processVoteStart(ctx context.Context, vs pi.VoteStart, usr user.User) (*pi.VoteStartReply, error) { - log.Tracef("processVoteStart: %v", vs.Params.Token) + log.Tracef("processVoteStart: %v", len(vs.Starts)) // Sanity check if !usr.Admin { @@ -1890,10 +1902,12 @@ func (p *politeiawww) processVoteStart(ctx context.Context, vs pi.VoteStart, usr } // Verify admin signed with their active identity - if usr.PublicKey() != vs.PublicKey { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPublicKeyInvalid, - ErrorContext: []string{"not active identity"}, + for _, v := range vs.Starts { + if usr.PublicKey() != v.PublicKey { + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusPublicKeyInvalid, + ErrorContext: []string{"not active identity"}, + } } } @@ -1903,6 +1917,8 @@ func (p *politeiawww) processVoteStart(ctx context.Context, vs pi.VoteStart, usr return nil, err } + // TODO Emit notification for each start + return &pi.VoteStartReply{ StartBlockHeight: reply.StartBlockHeight, StartBlockHash: reply.StartBlockHash, @@ -1911,68 +1927,18 @@ func (p *politeiawww) processVoteStart(ctx context.Context, vs pi.VoteStart, usr }, nil } -func (p *politeiawww) processVoteStartRunoff(ctx context.Context, vsr pi.VoteStartRunoff, usr user.User) (*pi.VoteStartRunoffReply, error) { - log.Tracef("processVoteStartRunoff: %v", vsr.Token) - - // Sanity check - if !usr.Admin { - return nil, fmt.Errorf("not an admin") - } - - // Verify admin signed all authorizations and starts using their - // active identity. - for _, v := range vsr.Auths { - if usr.PublicKey() != v.PublicKey { - e := fmt.Sprintf("authorize %v public key is not the active identity", - v.Token) - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPublicKeyInvalid, - ErrorContext: []string{e}, - } - } - } - for _, v := range vsr.Starts { - if usr.PublicKey() != v.PublicKey { - e := fmt.Sprintf("start %v public key is not the active identity", - v.Params.Token) - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPublicKeyInvalid, - ErrorContext: []string{e}, - } - } - } - - // Send plugin command - tsr := ticketvote.StartRunoff{ - Token: vsr.Token, - Auths: convertVoteAuthsFromPi(vsr.Auths), - Starts: convertVoteStartsFromPi(vsr.Starts), - } - srr, err := p.voteStartRunoff(ctx, tsr) - if err != nil { - return nil, err - } - - return &pi.VoteStartRunoffReply{ - StartBlockHeight: srr.StartBlockHeight, - StartBlockHash: srr.StartBlockHash, - EndBlockHeight: srr.EndBlockHeight, - EligibleTickets: srr.EligibleTickets, - }, nil -} - -func (p *politeiawww) processVoteBallot(ctx context.Context, vb pi.VoteBallot) (*pi.VoteBallotReply, error) { - log.Tracef("processVoteBallot") +func (p *politeiawww) processCastBallot(ctx context.Context, vc pi.CastBallot) (*pi.CastBallotReply, error) { + log.Tracef("processCastBallot") - b := ticketvote.Ballot{ - Votes: convertCastVotesFromPi(vb.Votes), + cb := ticketvote.CastBallot{ + Ballot: convertCastVotesFromPi(vc.Votes), } - reply, err := p.voteBallot(ctx, b) + reply, err := p.castBallot(ctx, cb) if err != nil { return nil, err } - return &pi.VoteBallotReply{ + return &pi.CastBallotReply{ Receipts: convertCastVoteRepliesFromPlugin(reply.Receipts), }, nil } @@ -1993,7 +1959,7 @@ func (p *politeiawww) processVotes(ctx context.Context, v pi.Votes) (*pi.VotesRe func (p *politeiawww) processVoteResults(ctx context.Context, vr pi.VoteResults) (*pi.VoteResultsReply, error) { log.Tracef("processVoteResults: %v", vr.Token) - cvr, err := p.castVotes(ctx, vr.Token) + cvr, err := p.voteResults(ctx, vr.Token) if err != nil { return nil, err } @@ -2386,53 +2352,23 @@ func (p *politeiawww) handleVoteStart(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, vsr) } -func (p *politeiawww) handleVoteStartRunoff(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleVoteStartRunoff") +func (p *politeiawww) handleCastBallot(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleCastBallot") - var vsr pi.VoteStartRunoff - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&vsr); err != nil { - respondWithPiError(w, r, "handleVoteStartRunoff: unmarshal", - pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInputInvalid, - }) - return - } - - usr, err := p.getSessionUser(w, r) - if err != nil { - respondWithPiError(w, r, - "handleVoteStartRunoff: getSessionUser: %v", err) - return - } - - vsrr, err := p.processVoteStartRunoff(r.Context(), vsr, *usr) - if err != nil { - respondWithPiError(w, r, - "handleVoteStartRunoff: processVoteStartRunoff: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, vsrr) -} - -func (p *politeiawww) handleVoteBallot(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleVoteBallot") - - var vb pi.VoteBallot + var vb pi.CastBallot decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&vb); err != nil { - respondWithPiError(w, r, "handleVoteBallot: unmarshal", + respondWithPiError(w, r, "handleCastBallot: unmarshal", pi.UserErrorReply{ ErrorCode: pi.ErrorStatusInputInvalid, }) return } - vbr, err := p.processVoteBallot(r.Context(), vb) + vbr, err := p.processCastBallot(r.Context(), vb) if err != nil { respondWithPiError(w, r, - "handleVoteBallot: processVoteBallot: %v", err) + "handleCastBallot: processCastBallot: %v", err) return } @@ -2575,10 +2511,7 @@ func (p *politeiawww) setPiRoutes() { pi.RouteVoteStart, p.handleVoteStart, permissionAdmin) p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteVoteStartRunoff, p.handleVoteStartRunoff, - permissionAdmin) - p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteVoteBallot, p.handleVoteBallot, + pi.RouteCastBallot, p.handleCastBallot, permissionPublic) p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteVotes, p.handleVotes, diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 846161a84..e22aaf7d4 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -259,14 +259,14 @@ func (p *politeiawww) processVoteResultsWWW(ctx context.Context, token string) ( } // Get cast votes - cv, err := p.castVotes(ctx, token) + rr, err := p.voteResults(ctx, token) if err != nil { return nil, err } // Convert to www - votes := make([]www.CastVote, 0, len(cv.Votes)) - for _, v := range cv.Votes { + votes := make([]www.CastVote, 0, len(rr.Votes)) + for _, v := range rr.Votes { votes = append(votes, www.CastVote{ Token: v.Token, Ticket: v.Ticket, @@ -353,28 +353,17 @@ func (p *politeiawww) processCastVotes(ctx context.Context, ballot *www.Ballot) Signature: vote.Signature, }) } - b := ticketvote.Ballot{ - Votes: votes, + cb := ticketvote.CastBallot{ + Ballot: votes, } - payload, err := ticketvote.EncodeBallot(b) - if err != nil { - return nil, err - } - - // Send plugin command - r, err := p.pluginCommand(ctx, ticketvote.ID, ticketvote.CmdBallot, - string(payload)) - if err != nil { - return nil, err - } - br, err := ticketvote.DecodeBallotReply([]byte(r)) + cbr, err := p.castBallot(ctx, cb) if err != nil { return nil, err } // Prepare reply - receipts := make([]www.CastVoteReply, 0, len(br.Receipts)) - for k, v := range br.Receipts { + receipts := make([]www.CastVoteReply, 0, len(cbr.Receipts)) + for k, v := range cbr.Receipts { receipts = append(receipts, www.CastVoteReply{ ClientSignature: ballot.Votes[k].Signature, Signature: v.Receipt, @@ -430,556 +419,6 @@ func (p *politeiawww) processTokenInventory(ctx context.Context, isAdmin bool) ( } /* -// TODO remove old vote code -func (p *politeiawww) processAuthorizeVote(av www.AuthorizeVote, u *user.User) (*www.AuthorizeVoteReply, error) { - // Make sure token is valid and not a prefix - if !tokenIsValid(av.Token) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: []string{av.Token}, - } - } - - // Validate the vote authorization - pr, err := p.getProp(av.Token) - if err != nil { - if errors.Is(err, cache.ErrRecordNotFound) { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - } - } - return nil, err - } - bb, err := p.getBestBlock() - if err != nil { - return nil, err - } - vs, err := p.voteSummaryGet(av.Token, bb) - if err != nil { - return nil, err - } - err = validateAuthorizeVoteStandard(av, *u, *pr, *vs) - if err != nil { - return nil, err - } - - // Setup plugin command - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, fmt.Errorf("Random: %v", err) - } - - dav := convertAuthorizeVoteToDecred(av) - payload, err := decredplugin.EncodeAuthorizeVote(dav) - if err != nil { - return nil, fmt.Errorf("EncodeAuthorizeVote: %v", err) - } - - pc := pd.PluginCommand{ - Challenge: hex.EncodeToString(challenge), - ID: decredplugin.ID, - Command: decredplugin.CmdAuthorizeVote, - CommandID: decredplugin.CmdAuthorizeVote + " " + av.Token, - Payload: string(payload), - } - - // Send authorizevote plugin request - responseBody, err := p.makeRequest(http.MethodPost, - pd.PluginCommandRoute, pc) - if err != nil { - return nil, err - } - - var reply pd.PluginCommandReply - err = json.Unmarshal(responseBody, &reply) - if err != nil { - return nil, fmt.Errorf("Unmarshal PluginCommandReply: %v", err) - } - - // Verify challenge - err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) - if err != nil { - return nil, fmt.Errorf("VerifyChallenge: %v", err) - } - - // Decode plugin reply - avr, err := decredplugin.DecodeAuthorizeVoteReply([]byte(reply.Payload)) - if err != nil { - return nil, fmt.Errorf("DecodeAuthorizeVoteReply: %v", err) - } - - if !p.test && avr.Action == decredplugin.AuthVoteActionAuthorize { - // Emit event notification for proposal vote authorized - p.eventManager.emit(eventProposalVoteAuthorized, - dataProposalVoteAuthorized{ - token: av.Token, - name: pr.Name, - username: u.Username, - email: u.Email, - }) - } - - return &www.AuthorizeVoteReply{ - Action: avr.Action, - Receipt: avr.Receipt, - }, nil -} - -func (p *politeiawww) processStartVoteV2(sv www2.StartVote, u *user.User) (*www2.StartVoteReply, error) { - log.Tracef("processStartVoteV2 %v", sv.Vote.Token) - - // Sanity check - if !u.Admin { - return nil, fmt.Errorf("user is not an admin") - } - - // Fetch proposal and vote summary - if !tokenIsValid(sv.Vote.Token) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: []string{sv.Vote.Token}, - } - } - pr, err := p.getProp(sv.Vote.Token) - if err != nil { - if errors.Is(err, cache.ErrRecordNotFound) { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - } - } - return nil, err - } - bb, err := p.getBestBlock() - if err != nil { - return nil, err - } - vs, err := p.voteSummaryGet(sv.Vote.Token, bb) - if err != nil { - return nil, err - } - - // Validate the start vote - err = p.validateStartVoteStandard(sv, *u, *pr, *vs) - if err != nil { - return nil, err - } - - // Tell decred plugin to start voting - dsv := convertStartVoteV2ToDecred(sv) - payload, err := decredplugin.EncodeStartVoteV2(dsv) - if err != nil { - return nil, err - } - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - pc := pd.PluginCommand{ - Challenge: hex.EncodeToString(challenge), - ID: decredplugin.ID, - Command: decredplugin.CmdStartVote, - CommandID: decredplugin.CmdStartVote + " " + sv.Vote.Token, - Payload: string(payload), - } - responseBody, err := p.makeRequest(http.MethodPost, - pd.PluginCommandRoute, pc) - if err != nil { - return nil, err - } - - // Handle reply - var reply pd.PluginCommandReply - err = json.Unmarshal(responseBody, &reply) - if err != nil { - return nil, fmt.Errorf("could not unmarshal "+ - "PluginCommandReply: %v", err) - } - err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) - if err != nil { - return nil, err - } - dsvr, err := decredplugin.DecodeStartVoteReply([]byte(reply.Payload)) - if err != nil { - return nil, err - } - svr, err := convertStartVoteReplyV2FromDecred(*dsvr) - if err != nil { - return nil, err - } - - // Get author data - author, err := p.db.UserGetByPubKey(pr.PublicKey) - if err != nil { - return nil, err - } - - // Emit event notification for proposal start vote - p.eventManager.emit(eventProposalVoteStarted, - dataProposalVoteStarted{ - token: pr.CensorshipRecord.Token, - name: pr.Name, - adminID: u.ID.String(), - author: *author, - }) - - return svr, nil -} - -func (p *politeiawww) processStartVoteRunoffV2(sv www2.StartVoteRunoff, u *user.User) (*www2.StartVoteRunoffReply, error) { - log.Tracef("processStartVoteRunoffV2 %v", sv.Token) - - // Sanity check - if !u.Admin { - return nil, fmt.Errorf("user is not an admin") - } - - bb, err := p.getBestBlock() - if err != nil { - return nil, err - } - - // Ensure authorize votes and start votes match - auths := make(map[string]www2.AuthorizeVote, len(sv.AuthorizeVotes)) - starts := make(map[string]www2.StartVote, len(sv.StartVotes)) - for _, v := range sv.AuthorizeVotes { - auths[v.Token] = v - } - for _, v := range sv.StartVotes { - _, ok := auths[v.Vote.Token] - if !ok { - e := fmt.Sprintf("start vote found without matching authorize vote %v", - v.Vote.Token) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidRunoffVote, - ErrorContext: []string{e}, - } - } - } - for _, v := range sv.StartVotes { - starts[v.Vote.Token] = v - } - for _, v := range sv.AuthorizeVotes { - _, ok := starts[v.Token] - if !ok { - e := fmt.Sprintf("authorize vote found without matching start vote %v", - v.Token) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidRunoffVote, - ErrorContext: []string{e}, - } - } - } - if len(auths) == 0 { - e := "start votes and authorize votes cannot be empty" - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidRunoffVote, - ErrorContext: []string{e}, - } - } - - // Slice used to send event notification for each proposal - // starting a vote, at the end of this function. - var proposalNotifications []*www.ProposalRecord - - // Validate authorize votes and start votes - for _, v := range sv.StartVotes { - // Fetch proposal and vote summary - token := v.Vote.Token - if !tokenIsValid(token) { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidCensorshipToken, - ErrorContext: []string{token}, - } - } - pr, err := p.getProp(token) - if err != nil { - if errors.Is(err, cache.ErrRecordNotFound) { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - ErrorContext: []string{token}, - } - } - return nil, err - } - - proposalNotifications = append(proposalNotifications, pr) - - vs, err := p.voteSummaryGet(token, bb) - if err != nil { - return nil, err - } - - // Validate authorize vote. The validation function requires a v1 - // AuthorizeVote. This is fine. There is no difference between v1 - // and v2. - av := auths[v.Vote.Token] - av1 := www.AuthorizeVote{ - Token: av.Token, - Action: av.Action, - PublicKey: av.PublicKey, - Signature: av.Signature, - } - err = validateAuthorizeVoteRunoff(av1, *u, *pr, *vs) - if err != nil { - // Attach the token to the error so the user knows which one - // failed. - var uerr *www.UserError - if errors.As(err, &uerr) { - uerr.ErrorContext = append(uerr.ErrorContext, token) - err = uerr - } - return nil, err - } - - // Validate start vote - err = validateStartVoteRunoff(v, *u, *pr, *vs, - p.cfg.VoteDurationMin, p.cfg.VoteDurationMax) - if err != nil { - // Attach the token to the error so the user knows which one - // failed. - var uerr *www.UserError - if errors.As(err, &uerr) { - uerr.ErrorContext = append(uerr.ErrorContext, token) - err = uerr - } - return nil, err - } - } - - // Validate the RFP proposal - rfp, err := p.getProp(sv.Token) - if err != nil { - if errors.Is(err, cache.ErrRecordNotFound) { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - ErrorContext: []string{sv.Token}, - } - } - return nil, err - } - switch { - case rfp.LinkBy > time.Now().Unix() && !p.cfg.TestNet: - // Vote cannot start on RFP submissions until the RFP linkby - // deadline has been met. This validation is skipped when on - // testnet. - return nil, www.UserError{ - ErrorCode: www.ErrorStatusLinkByDeadlineNotMet, - } - case len(rfp.LinkedFrom) == 0: - return nil, www.UserError{ - ErrorCode: www.ErrorStatusNoLinkedProposals, - } - } - - // Compile a list of the public, non-abandoned RFP submissions. - // This list will be used to ensure a StartVote exists for each - // of the public, non-abandoned submissions. - linkedFromProps, err := p.getProps(rfp.LinkedFrom) - if err != nil { - return nil, err - } - submissions := make(map[string]bool, len(rfp.LinkedFrom)) // [token]startVoteFound - for _, v := range linkedFromProps { - // Filter out abandoned submissions. These are not allowed - // to be included in a runoff vote. - if v.Status != www.PropStatusPublic { - continue - } - - // Set to false for now until we check that a StartVote - // was included for this proposal. - submissions[v.CensorshipRecord.Token] = false - } - - // Verify that a StartVote exists for all public, non-abandoned - // submissions and that there are no extra StartVotes. - for _, v := range sv.StartVotes { - _, ok := submissions[v.Vote.Token] - if !ok { - e := fmt.Sprintf("invalid start vote submission: %v", - v.Vote.Token) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidRunoffVote, - ErrorContext: []string{e}, - } - } - - // A StartVote was included for this proposal - submissions[v.Vote.Token] = true - } - for token, startVoteFound := range submissions { - if !startVoteFound { - e := fmt.Sprintf("missing start vote for rfp submission: %v", - token) - return nil, www.UserError{ - ErrorCode: www.ErrorStatusInvalidRunoffVote, - ErrorContext: []string{e}, - } - } - } - - // Setup plugin command - dav := convertAuthorizeVotesV2ToDecred(sv.AuthorizeVotes) - dsv := convertStartVotesV2ToDecred(sv.StartVotes) - payload, err := decredplugin.EncodeStartVoteRunoff( - decredplugin.StartVoteRunoff{ - Token: sv.Token, - AuthorizeVotes: dav, - StartVotes: dsv, - }) - if err != nil { - return nil, err - } - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - pc := pd.PluginCommand{ - Challenge: hex.EncodeToString(challenge), - ID: decredplugin.ID, - Command: decredplugin.CmdStartVoteRunoff, - Payload: string(payload), - } - - // Send plugin command - responseBody, err := p.makeRequest(http.MethodPost, - pd.PluginCommandRoute, pc) - if err != nil { - return nil, err - } - - // Handle response - var reply pd.PluginCommandReply - err = json.Unmarshal(responseBody, &reply) - if err != nil { - return nil, err - } - err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) - if err != nil { - return nil, err - } - dsvr, err := decredplugin.DecodeStartVoteRunoffReply([]byte(reply.Payload)) - if err != nil { - return nil, err - } - svr, err := convertStartVoteReplyV2FromDecred(dsvr.StartVoteReply) - if err != nil { - return nil, err - } - - // Emit event notification for each proposal starting vote - for _, pn := range proposalNotifications { - author, err := p.db.UserGetByPubKey(pn.PublicKey) - if err != nil { - return nil, err - } - p.eventManager.emit(eventProposalVoteStarted, - dataProposalVoteStarted{ - token: pn.CensorshipRecord.Token, - name: pn.Name, - adminID: u.ID.String(), - author: *author, - }) - } - - return &www2.StartVoteRunoffReply{ - StartBlockHeight: svr.StartBlockHeight, - StartBlockHash: svr.StartBlockHash, - EndBlockHeight: svr.EndBlockHeight, - EligibleTickets: svr.EligibleTickets, - }, nil -} - -func (p *politeiawww) processVoteDetailsV2(token string) (*www2.VoteDetailsReply, error) { - log.Tracef("processVoteDetailsV2: %v", token) - - // Validate vote status - dvdr, err := p.decredVoteDetails(token) - if err != nil { - if errors.Is(err, cache.ErrRecordNotFound) { - err = www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - } - } - return nil, err - } - if dvdr.StartVoteReply.StartBlockHash == "" { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - ErrorContext: []string{"voting has not started yet"}, - } - } - - // Handle StartVote versioning - var vdr *www2.VoteDetailsReply - switch dvdr.StartVote.Version { - case decredplugin.VersionStartVoteV1: - b := []byte(dvdr.StartVote.Payload) - dsv1, err := decredplugin.DecodeStartVoteV1(b) - if err != nil { - return nil, err - } - vdr, err = convertDecredStartVoteV1ToVoteDetailsReplyV2(*dsv1, - dvdr.StartVoteReply) - if err != nil { - return nil, err - } - case decredplugin.VersionStartVoteV2: - b := []byte(dvdr.StartVote.Payload) - dsv2, err := decredplugin.DecodeStartVoteV2(b) - if err != nil { - return nil, err - } - vdr, err = convertDecredStartVoteV2ToVoteDetailsReplyV2(*dsv2, - dvdr.StartVoteReply) - if err != nil { - return nil, err - } - - default: - return nil, fmt.Errorf("invalid StartVote version %v %v", - token, dvdr.StartVote.Version) - } - - return vdr, nil -} - -// isRFP returns whether the proposal is a Request For Proposals (RFP). -func isRFP(pr www.ProposalRecord) bool { - return pr.LinkBy != 0 -} - -func isRFPSubmission(pr www.ProposalRecord) bool { - // Right now the only proposals that we allow linking to - // are RFPs so if the linkto is set than this is an RFP - // submission. This may change in the future, at which - // point we'll actually have to check the linkto proposal - // to see if its an RFP. - return pr.LinkTo != "" -} - -// validateVoteBit ensures that bit is a valid vote bit. -func validateVoteBit(vote www2.Vote, bit uint64) error { - if len(vote.Options) == 0 { - return fmt.Errorf("vote corrupt") - } - if bit == 0 { - return fmt.Errorf("invalid bit 0x%x", bit) - } - if vote.Mask&bit != bit { - return fmt.Errorf("invalid mask 0x%x bit 0x%x", - vote.Mask, bit) - } - - for _, v := range vote.Options { - if v.Bits == bit { - return nil - } - } - - return fmt.Errorf("bit not found 0x%x", bit) -} - func (p *politeiawww) linkByValidate(linkBy int64) error { min := time.Now().Unix() + p.linkByPeriodMin() max := time.Now().Unix() + p.linkByPeriodMax() @@ -1001,359 +440,4 @@ func (p *politeiawww) linkByValidate(linkBy int64) error { } return nil } - -// validateAuthorizeVote validates the authorize vote fields. A UserError is -// returned if any of the validation fails. -func validateAuthorizeVote(av www.AuthorizeVote, u user.User, pr www.ProposalRecord, vs www.VoteSummary) error { - // Ensure the public key is the user's active key - if av.PublicKey != u.PublicKey() { - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidSigningKey, - } - } - - // Validate signature - msg := av.Token + pr.Version + av.Action - err := validateSignature(av.PublicKey, av.Signature, msg) - if err != nil { - return err - } - - // Verify record is in the right state and that the authorize - // vote request is valid. A vote authorization may already - // exist. We also allow vote authorizations to be revoked. - switch { - case pr.Status != www.PropStatusPublic: - // Record not public - return www.UserError{ - ErrorCode: www.ErrorStatusWrongStatus, - } - case vs.EndHeight != 0: - // Vote has already started - return www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - } - case av.Action != decredplugin.AuthVoteActionAuthorize && - av.Action != decredplugin.AuthVoteActionRevoke: - // Invalid authorize vote action - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidAuthVoteAction, - } - case av.Action == decredplugin.AuthVoteActionAuthorize && - vs.Status == www.PropVoteStatusAuthorized: - // Cannot authorize vote; vote has already been authorized - return www.UserError{ - ErrorCode: www.ErrorStatusVoteAlreadyAuthorized, - } - case av.Action == decredplugin.AuthVoteActionRevoke && - vs.Status != www.PropVoteStatusAuthorized: - // Cannot revoke authorization; vote has not been authorized - return www.UserError{ - ErrorCode: www.ErrorStatusVoteNotAuthorized, - } - } - - return nil -} - -// validateAuthorizeVoteStandard validates the authorize vote for a proposal that -// is participating in a standard vote. A UserError is returned if any of the -// validation fails. -func validateAuthorizeVoteStandard(av www.AuthorizeVote, u user.User, pr www.ProposalRecord, vs www.VoteSummary) error { - err := validateAuthorizeVote(av, u, pr, vs) - if err != nil { - return err - } - - // The rest of the validation is specific to authorize votes for - // standard votes. - switch { - case isRFPSubmission(pr): - return www.UserError{ - ErrorCode: www.ErrorStatusWrongProposalType, - ErrorContext: []string{"proposal is an rfp submission"}, - } - case pr.PublicKey != av.PublicKey: - // User is not the author. First make sure the author didn't - // submit the proposal using an old identity. - if !isProposalAuthor(pr, u) { - return www.UserError{ - ErrorCode: www.ErrorStatusUserNotAuthor, - } - } - } - - return nil -} - -// validateAuthorizeVoteRunoff validates the authorize vote for a proposal that -// is participating in a runoff vote. A UserError is returned if any of the -// validation fails. -func validateAuthorizeVoteRunoff(av www.AuthorizeVote, u user.User, pr www.ProposalRecord, vs www.VoteSummary) error { - err := validateAuthorizeVote(av, u, pr, vs) - if err != nil { - return err - } - - // The rest of the validation is specific to authorize votes for - // runoff votes. - switch { - case !u.Admin: - // User is not an admin - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidSigningKey, - ErrorContext: []string{"user not an admin"}, - } - } - - return nil -} - -// validateVoteOptions verifies that the provided vote options -// specify a simple approve/reject vote and nothing else. A UserError is -// returned if this validation fails. -func validateVoteOptions(options []www2.VoteOption) error { - if len(options) == 0 { - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidVoteOptions, - ErrorContext: []string{"no vote options found"}, - } - } - optionIDs := map[string]bool{ - decredplugin.VoteOptionIDApprove: false, - decredplugin.VoteOptionIDReject: false, - } - for _, vo := range options { - if _, ok := optionIDs[vo.Id]; !ok { - e := fmt.Sprintf("invalid vote option id '%v'", vo.Id) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidVoteOptions, - ErrorContext: []string{e}, - } - } - optionIDs[vo.Id] = true - } - for k, wasFound := range optionIDs { - if !wasFound { - e := fmt.Sprintf("missing vote option id '%v'", k) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidVoteOptions, - ErrorContext: []string{e}, - } - } - } - return nil -} - -func validateStartVote(sv www2.StartVote, u user.User, pr www.ProposalRecord, vs www.VoteSummary, durationMin, durationMax uint32) error { - if !tokenIsValid(sv.Vote.Token) { - // Sanity check since proposal has already been looked up and - // passed in to this function. - return fmt.Errorf("invalid token %v", sv.Vote.Token) - } - - // Validate vote bits - for _, v := range sv.Vote.Options { - err := validateVoteBit(sv.Vote, v.Bits) - if err != nil { - log.Debugf("validateStartVote: validateVoteBit '%v': %v", - v.Id, err) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidPropVoteBits, - } - } - } - - // Validate vote options. Only simple yes/no votes are currently - // allowed. - err := validateVoteOptions(sv.Vote.Options) - if err != nil { - return err - } - - // Validate vote params - switch { - case sv.Vote.Duration < durationMin: - // Duration not large enough - e := fmt.Sprintf("vote duration must be >= %v", durationMin) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidPropVoteParams, - ErrorContext: []string{e}, - } - case sv.Vote.Duration > durationMax: - // Duration too large - e := fmt.Sprintf("vote duration must be <= %v", durationMax) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidPropVoteParams, - ErrorContext: []string{e}, - } - case sv.Vote.QuorumPercentage > 100: - // Quorum too large - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidPropVoteParams, - ErrorContext: []string{"quorum percentage cannot be >100"}, - } - case sv.Vote.PassPercentage > 100: - // Pass percentage too large - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidPropVoteParams, - ErrorContext: []string{"pass percentage cannot be >100"}, - } - } - - // Ensure the public key is the user's active key - if sv.PublicKey != u.PublicKey() { - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidSigningKey, - } - } - - // Validate signature - dsv := convertStartVoteV2ToDecred(sv) - err = dsv.VerifySignature() - if err != nil { - log.Debugf("validateStartVote: VerifySignature: %v", err) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidSignature, - } - } - - // Validate proposal - votePropVersion := strconv.FormatUint(uint64(sv.Vote.ProposalVersion), 10) - switch { - case pr.Version != votePropVersion: - // Vote is specifying the wrong version - e := fmt.Sprintf("got %v, want %v", votePropVersion, pr.Version) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidProposalVersion, - ErrorContext: []string{e}, - } - case pr.Status != www.PropStatusPublic: - // Proposal is not public - return www.UserError{ - ErrorCode: www.ErrorStatusWrongStatus, - ErrorContext: []string{"proposal is not public"}, - } - case vs.EndHeight != 0: - // Vote has already started - return www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - ErrorContext: []string{"vote already started"}, - } - } - - return nil -} - -func (p *politeiawww) validateStartVoteStandard(sv www2.StartVote, u user.User, pr www.ProposalRecord, vs www.VoteSummary) error { - err := validateStartVote(sv, u, pr, vs, p.cfg.VoteDurationMin, - p.cfg.VoteDurationMax) - if err != nil { - return err - } - - // The remaining validation is specific to a VoteTypeStandard. - switch { - case sv.Vote.Type != www2.VoteTypeStandard: - // Not a standard vote - e := fmt.Sprintf("vote type must be %v", www2.VoteTypeStandard) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidVoteType, - ErrorContext: []string{e}, - } - case vs.Status != www.PropVoteStatusAuthorized: - // Vote has not been authorized - return www.UserError{ - ErrorCode: www.ErrorStatusWrongVoteStatus, - ErrorContext: []string{"vote not authorized"}, - } - case isRFPSubmission(pr): - // The proposal is an an RFP submission. The voting period for - // RFP submissions can only be started using the StartVoteRunoff - // route. - return www.UserError{ - ErrorCode: www.ErrorStatusWrongProposalType, - ErrorContext: []string{"cannot be an rfp submission"}, - } - } - - // Verify the LinkBy deadline for RFP proposals. The LinkBy policy - // requirements are enforced at the time of starting the vote - // because its purpose is to ensure that there is enough time for - // RFP submissions to be submitted. - if isRFP(pr) { - err := p.linkByValidate(pr.LinkBy) - if err != nil { - return err - } - } - - return nil -} - -func validateStartVoteRunoff(sv www2.StartVote, u user.User, pr www.ProposalRecord, vs www.VoteSummary, durationMin, durationMax uint32) error { - err := validateStartVote(sv, u, pr, vs, durationMin, durationMax) - if err != nil { - return err - } - - // The remaining validation is specific to a VoteTypeRunoff. - - token := sv.Vote.Token - switch { - case sv.Vote.Type != www2.VoteTypeRunoff: - // Not a runoff vote - e := fmt.Sprintf("%v vote type must be %v", - token, www2.VoteTypeRunoff) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidVoteType, - ErrorContext: []string{e}, - } - - case !isRFPSubmission(pr): - // The proposal is not an RFP submission - e := fmt.Sprintf("%v is not an rfp submission", token) - return www.UserError{ - ErrorCode: www.ErrorStatusWrongProposalType, - ErrorContext: []string{e}, - } - - case vs.Status != www.PropVoteStatusNotAuthorized: - // Sanity check. This should not be possible. - return fmt.Errorf("%v got vote status %v, want %v", - token, vs.Status, www.PropVoteStatusNotAuthorized) - } - - return nil -} - -func voteIsApproved(vs www.VoteSummary) bool { - if vs.Status != www.PropVoteStatusFinished { - // Vote has not ended yet - return false - } - - var ( - total uint64 - approve uint64 - ) - for _, v := range vs.Results { - total += v.VotesReceived - if v.Option.Id == decredplugin.VoteOptionIDApprove { - approve = v.VotesReceived - } - } - quorum := uint64(float64(vs.QuorumPercentage) / 100 * float64(vs.EligibleTickets)) - pass := uint64(float64(vs.PassPercentage) / 100 * float64(total)) - switch { - case total < quorum: - // Quorum not met - return false - case approve < pass: - // Pass percentage not met - return false - } - - return true -} */ diff --git a/politeiawww/ticketvote.go b/politeiawww/ticketvote.go index 7c0bd17e5..9833763ea 100644 --- a/politeiawww/ticketvote.go +++ b/politeiawww/ticketvote.go @@ -16,7 +16,8 @@ func (p *politeiawww) voteAuthorize(ctx context.Context, a ticketvote.Authorize) if err != nil { return nil, err } - r, err := p.pluginCommand(ctx, ticketvote.ID, ticketvote.CmdAuthorize, string(b)) + r, err := p.pluginCommand(ctx, ticketvote.ID, + ticketvote.CmdAuthorize, string(b)) if err != nil { return nil, err } @@ -33,7 +34,8 @@ func (p *politeiawww) voteStart(ctx context.Context, s ticketvote.Start) (*ticke if err != nil { return nil, err } - r, err := p.pluginCommand(ctx, ticketvote.ID, ticketvote.CmdStart, string(b)) + r, err := p.pluginCommand(ctx, ticketvote.ID, + ticketvote.CmdStart, string(b)) if err != nil { return nil, err } @@ -44,35 +46,18 @@ func (p *politeiawww) voteStart(ctx context.Context, s ticketvote.Start) (*ticke return sr, nil } -// voteStartRunoff uses the ticketvote plugin to start a runoff vote. -func (p *politeiawww) voteStartRunoff(ctx context.Context, sr ticketvote.StartRunoff) (*ticketvote.StartRunoffReply, error) { - b, err := ticketvote.EncodeStartRunoff(sr) +// castBallot uses the ticketvote plugin to cast a ballot of votes. +func (p *politeiawww) castBallot(ctx context.Context, tb ticketvote.CastBallot) (*ticketvote.CastBallotReply, error) { + b, err := ticketvote.EncodeCastBallot(tb) if err != nil { return nil, err } - r, err := p.pluginCommand(ctx, ticketvote.ID, ticketvote.CmdStartRunoff, - string(b)) + r, err := p.pluginCommand(ctx, ticketvote.ID, + ticketvote.CmdCastBallot, string(b)) if err != nil { return nil, err } - srr, err := ticketvote.DecodeStartRunoffReply([]byte(r)) - if err != nil { - return nil, err - } - return srr, nil -} - -// voteBallot uses the ticketvote plugin to cast a ballot of votes. -func (p *politeiawww) voteBallot(ctx context.Context, tb ticketvote.Ballot) (*ticketvote.BallotReply, error) { - b, err := ticketvote.EncodeBallot(tb) - if err != nil { - return nil, err - } - r, err := p.pluginCommand(ctx, ticketvote.ID, ticketvote.CmdBallot, string(b)) - if err != nil { - return nil, err - } - br, err := ticketvote.DecodeBallotReply([]byte(r)) + br, err := ticketvote.DecodeCastBallotReply([]byte(r)) if err != nil { return nil, err } @@ -88,7 +73,8 @@ func (p *politeiawww) voteDetails(ctx context.Context, tokens []string) (*ticket if err != nil { return nil, err } - r, err := p.pluginCommand(ctx, ticketvote.ID, ticketvote.CmdDetails, string(b)) + r, err := p.pluginCommand(ctx, ticketvote.ID, + ticketvote.CmdDetails, string(b)) if err != nil { return nil, err } @@ -99,20 +85,21 @@ func (p *politeiawww) voteDetails(ctx context.Context, tokens []string) (*ticket return dr, nil } -// castVotes uses the ticketvote plugin to fetch cast votes for a record. -func (p *politeiawww) castVotes(ctx context.Context, token string) (*ticketvote.CastVotesReply, error) { - cv := ticketvote.CastVotes{ +// voteResults uses the ticketvote plugin to fetch cast votes for a record. +func (p *politeiawww) voteResults(ctx context.Context, token string) (*ticketvote.ResultsReply, error) { + cv := ticketvote.Results{ Token: token, } - b, err := ticketvote.EncodeCastVotes(cv) + b, err := ticketvote.EncodeResults(cv) if err != nil { return nil, err } - r, err := p.pluginCommand(ctx, ticketvote.ID, ticketvote.CmdCastVotes, string(b)) + r, err := p.pluginCommand(ctx, ticketvote.ID, + ticketvote.CmdResults, string(b)) if err != nil { return nil, err } - cvr, err := ticketvote.DecodeCastVotesReply([]byte(r)) + cvr, err := ticketvote.DecodeResultsReply([]byte(r)) if err != nil { return nil, err } @@ -128,7 +115,8 @@ func (p *politeiawww) voteSummaries(ctx context.Context, tokens []string) (*tick if err != nil { return nil, err } - r, err := p.pluginCommand(ctx, ticketvote.ID, ticketvote.CmdSummaries, string(b)) + r, err := p.pluginCommand(ctx, ticketvote.ID, + ticketvote.CmdSummaries, string(b)) if err != nil { return nil, err } diff --git a/politeiawww/www.go b/politeiawww/www.go index d590c5e01..b4600ecde 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -122,7 +122,6 @@ func convertWWWErrorStatusFromTicketVote(e ticketvote.ErrorStatusT) www.ErrorSta return www.ErrorStatusInvalidPropVoteParams case ticketvote.ErrorStatusVoteStatusInvalid: return www.ErrorStatusInvalidPropVoteStatus - case ticketvote.ErrorStatusBallotInvalid: } return www.ErrorStatusInvalid } @@ -256,8 +255,6 @@ func convertPiErrorStatusFromTicketVote(e ticketvote.ErrorStatusT) pi.ErrorStatu return pi.ErrorStatusVoteParamsInvalid case ticketvote.ErrorStatusVoteStatusInvalid: return pi.ErrorStatusVoteStatusInvalid - case ticketvote.ErrorStatusBallotInvalid: - return pi.ErrorStatusBallotInvalid } return pi.ErrorStatusInvalid } @@ -380,8 +377,8 @@ func respondWithPiError(w http.ResponseWriter, r *http.Request, format string, e r.URL, r.Proto, t, errCode) } else { log.Errorf("%v %v %v %v Internal error %v: error "+ - "code from politeiad plugin %v: %v", remoteAddr(r), - r.Method, r.URL, r.Proto, t, pluginID, errCode) + "code from politeiad plugin %v: %v %v", remoteAddr(r), + r.Method, r.URL, r.Proto, t, pluginID, errCode, errContext) } util.RespondWithJSON(w, http.StatusInternalServerError, diff --git a/wsdcrdata/wsdcrdata.go b/wsdcrdata/wsdcrdata.go index e88d9d46a..23c81e22e 100644 --- a/wsdcrdata/wsdcrdata.go +++ b/wsdcrdata/wsdcrdata.go @@ -448,7 +448,9 @@ func psclientNew(url string) (*psclient.Client, error) { ReadTimeout: psclient.DefaultReadTimeout, WriteTimeout: 3 * time.Second, } - c, err := psclient.New(url, context.Background(), &opts) + d := time.Now().Add(15 * time.Second) + ctx, _ := context.WithDeadline(context.Background(), d) + c, err := psclient.New(url, ctx, &opts) if err != nil { return nil, fmt.Errorf("failed to connect to %v: %v", url, err) } @@ -475,6 +477,8 @@ func psclientNew(url string) (*psclient.Client, error) { // New returns a new Client. func New(dcrdataURL string) (*Client, error) { + log.Tracef("New: %v", dcrdataURL) + // Setup dcrdata connection. If there is an error when connecting // to dcrdata, return both the error and the Client so that the // caller can decide if reconnection attempts should be made. From 5052de8d300eea589f54ad7e1ca2879c4ae81bcd Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 22 Oct 2020 08:21:42 -0500 Subject: [PATCH 169/449] cleanup and fixes --- go.sum | 2 + politeiad/backend/tlogbe/ticketvote.go | 128 ++++++++++++++++--------- politeiawww/cmd/shared/client.go | 110 ++++++++++----------- politeiawww/cmd/shared/config.go | 40 ++------ politeiawww/config.go | 79 +++------------ politeiawww/piwww.go | 1 + wsdcrdata/wsdcrdata.go | 4 +- 7 files changed, 162 insertions(+), 202 deletions(-) diff --git a/go.sum b/go.sum index 7b92496c5..740265714 100644 --- a/go.sum +++ b/go.sum @@ -720,6 +720,7 @@ github.com/stretchr/testify v0.0.0-20170130113145-4d4bfba8f1d1/go.mod h1:a8OnRci github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/subosito/gozaru v0.0.0-20190625071150-416082cce636 h1:LlXBFcxziHIkc7jnbCmUCL5+ujGMky2aJsNvHqtt80Y= github.com/subosito/gozaru v0.0.0-20190625071150-416082cce636/go.mod h1:LIpwO1yApZNrEQZdu5REqRtRrkaU+52ueA7WGT+CvSw= @@ -1086,6 +1087,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.6 h1:97YCGUei5WVbkKfogoJQsLwUJ17cWvpLrgNvlcbxikE= gopkg.in/yaml.v2 v2.2.6/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index 010d1753b..af56bc862 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -183,34 +183,66 @@ func (p *ticketVotePlugin) inventorySetToUnauthorized(token string) { log.Debugf("ticketvote: added to unauthorized inv: %v", token) } -func (p *ticketVotePlugin) inventorySetToStarted(token string, endHeight uint32) { +func (p *ticketVotePlugin) inventorySetToStarted(token string, t ticketvote.VoteT, endHeight uint32) { p.Lock() defer p.Unlock() - // Remove the token from the authorized list. The token should - // always be in the authorized list prior to the vote being started - // so panicing when this is not the case is ok. - var i int - var found bool - for k, v := range p.inv.authorized { - if v == token { - i = k - found = true - break + switch t { + case ticketvote.VoteTypeStandard: + // Remove the token from the authorized list. The token should + // always be in the authorized list prior to the vote being + // started for standard votes so panicing when this is not the + // case is ok. + var i int + var found bool + for k, v := range p.inv.authorized { + if v == token { + i = k + found = true + break + } + } + if !found { + e := fmt.Sprintf("token not found in authorized list: %v", token) + panic(e) } - } - if !found { - e := fmt.Sprintf("token not found in authorized list: %v", token) - panic(e) - } - a := p.inv.authorized - a = append(a[:i], a[i+1:]...) - p.inv.authorized = a + a := p.inv.authorized + a = append(a[:i], a[i+1:]...) + p.inv.authorized = a - log.Debugf("ticketvote: removed from authorized inv: %v", token) + log.Debugf("ticketvote: removed from authorized inv: %v", token) - // Add the token to the started map + case ticketvote.VoteTypeRunoff: + // A runoff vote does not require the submission votes be + // authorized prior to the vote starting. The token might be in + // the unauthorized list, but its also possible that its not + // since the unauthorized list is lazy loaded and it might not + // have been added yet. Remove it only if it is found. + var i int + var found bool + for k, v := range p.inv.unauthorized { + if v == token { + i = k + found = true + break + } + } + if found { + // Remove the token from unauthorized + u := p.inv.unauthorized + u = append(u[:i], u[i+1:]...) + p.inv.unauthorized = u + + log.Debugf("ticketvote: removed from unauthorized inv: %v", token) + } + + default: + e := fmt.Sprintf("invalid vote type %v", t) + panic(e) + } + + // Add the token to the started list p.inv.started[token] = endHeight log.Debugf("ticketvote: added to started inv: %v", token) @@ -1491,7 +1523,8 @@ func (p *ticketVotePlugin) startStandard(s ticketvote.Start) (*ticketvote.StartR } // Update inventory - p.inventorySetToStarted(vd.Params.Token, vd.EndBlockHeight) + p.inventorySetToStarted(vd.Params.Token, ticketvote.VoteTypeStandard, + vd.EndBlockHeight) return &ticketvote.StartReply{ StartBlockHeight: sr.StartBlockHeight, @@ -1668,7 +1701,8 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep } // Update inventory - p.inventorySetToStarted(vd.Params.Token, vd.EndBlockHeight) + p.inventorySetToStarted(vd.Params.Token, ticketvote.VoteTypeRunoff, + vd.EndBlockHeight) } return &ticketvote.StartReply{ @@ -2210,31 +2244,32 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. // Summary has not been cached. Get it manually. + // Assume vote is unauthorized. Only update the status when the + // appropriate record has been found. + status := ticketvote.VoteStatusUnauthorized + // Check if the vote has been authorized auths, err := p.authorizes(token) if err != nil { return nil, fmt.Errorf("authorizes: %v", err) } - if len(auths) == 0 { - // Vote has not been authorized yet - return &ticketvote.Summary{ - Status: ticketvote.VoteStatusUnauthorized, - Results: []ticketvote.VoteOptionResult{}, - }, nil - } - lastAuth := auths[len(auths)-1] - switch ticketvote.AuthActionT(lastAuth.Action) { - case ticketvote.AuthActionAuthorize: - // Vote has been authorized; continue - case ticketvote.AuthActionRevoke: - // Vote authorization has been revoked - return &ticketvote.Summary{ - Status: ticketvote.VoteStatusUnauthorized, - Results: []ticketvote.VoteOptionResult{}, - }, nil - } - - // Vote has been authorized. Check if it has been started yet. + if len(auths) > 0 { + lastAuth := auths[len(auths)-1] + switch ticketvote.AuthActionT(lastAuth.Action) { + case ticketvote.AuthActionAuthorize: + // Vote has been authorized; continue + status = ticketvote.VoteStatusAuthorized + case ticketvote.AuthActionRevoke: + // Vote authorization has been revoked. Its not possible for + // the vote to have been started. We can stop looking. + return &ticketvote.Summary{ + Status: status, + Results: []ticketvote.VoteOptionResult{}, + }, nil + } + } + + // Check if the vote has been started vd, err := p.voteDetails(token) if err != nil { return nil, fmt.Errorf("startDetails: %v", err) @@ -2242,14 +2277,13 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. if vd == nil { // Vote has not been started yet return &ticketvote.Summary{ - Status: ticketvote.VoteStatusAuthorized, + Status: status, Results: []ticketvote.VoteOptionResult{}, }, nil } // Vote has been started. Check if it is still in progress or has // already ended. - var status ticketvote.VoteStatusT if bestBlock < vd.EndBlockHeight { status = ticketvote.VoteStatusStarted } else { @@ -2337,8 +2371,8 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. vd.Params.Token, err, summary) } - // Remove record from the votes cache now that a summary has - // been saved for it. + // Remove record from the votes cache now that a summary has been + // saved for it. p.cachedVotesDel(vd.Params.Token) } diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index 808790eff..890b8e443 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -2774,139 +2774,141 @@ func (c *Client) StartVoteDCC(sv cms.StartVote) (*cms.StartVoteReply, error) { return &svr, nil } -// WalletAccounts retrieves the walletprc accounts. -func (c *Client) WalletAccounts() (*walletrpc.AccountsResponse, error) { - if c.wallet == nil { - return nil, fmt.Errorf("walletrpc client not loaded") +// SetTOTP sets the logged in user's TOTP Key. +func (c *Client) SetTOTP(st *www.SetTOTP) (*www.SetTOTPReply, error) { + statusCode, respBody, err := c.makeRequest(http.MethodPost, + cms.APIRoute, www.RouteSetTOTP, st) + if err != nil { + return nil, err } - if c.cfg.Verbose { - fmt.Printf("walletrpc %v Accounts\n", c.cfg.WalletHost) + if statusCode != http.StatusOK { + return nil, wwwError(respBody, statusCode) } - ar, err := c.wallet.Accounts(c.ctx, &walletrpc.AccountsRequest{}) + var str www.SetTOTPReply + err = json.Unmarshal(respBody, &str) if err != nil { - return nil, err + return nil, fmt.Errorf("unmarshal SetTOTPReply: %v", err) } if c.cfg.Verbose { - err := prettyPrintJSON(ar) + err := prettyPrintJSON(str) if err != nil { return nil, err } } - return ar, nil + return &str, nil } -// CommittedTickets returns the committed tickets that belong to the dcrwallet -// instance out of the the specified list of tickets. -func (c *Client) CommittedTickets(ct *walletrpc.CommittedTicketsRequest) (*walletrpc.CommittedTicketsResponse, error) { - if c.wallet == nil { - return nil, fmt.Errorf("walletrpc client not loaded") +// VerifyTOTP comfirms the logged in user's TOTP Key. +func (c *Client) VerifyTOTP(vt *www.VerifyTOTP) (*www.VerifyTOTPReply, error) { + statusCode, respBody, err := c.makeRequest(http.MethodPost, + cms.APIRoute, www.RouteVerifyTOTP, vt) + if err != nil { + return nil, err } - if c.cfg.Verbose { - fmt.Printf("walletrpc %v CommittedTickets\n", c.cfg.WalletHost) + if statusCode != http.StatusOK { + return nil, wwwError(respBody, statusCode) } - ctr, err := c.wallet.CommittedTickets(c.ctx, ct) + var vtr www.VerifyTOTPReply + err = json.Unmarshal(respBody, &vtr) if err != nil { - return nil, err + return nil, fmt.Errorf("unmarshal VerifyTOTPReply: %v", err) } if c.cfg.Verbose { - err := prettyPrintJSON(ctr) + err := prettyPrintJSON(vtr) if err != nil { return nil, err } } - return ctr, nil + return &vtr, nil } -// SignMessages signs the passed in messages using the private keys from the -// specified addresses. -func (c *Client) SignMessages(sm *walletrpc.SignMessagesRequest) (*walletrpc.SignMessagesResponse, error) { +// WalletAccounts retrieves the walletprc accounts. +func (c *Client) WalletAccounts() (*walletrpc.AccountsResponse, error) { if c.wallet == nil { return nil, fmt.Errorf("walletrpc client not loaded") } if c.cfg.Verbose { - fmt.Printf("walletrpc %v SignMessages\n", c.cfg.WalletHost) + fmt.Printf("walletrpc %v Accounts\n", c.cfg.WalletHost) } - smr, err := c.wallet.SignMessages(c.ctx, sm) + ar, err := c.wallet.Accounts(c.ctx, &walletrpc.AccountsRequest{}) if err != nil { return nil, err } if c.cfg.Verbose { - err := prettyPrintJSON(smr) + err := prettyPrintJSON(ar) if err != nil { return nil, err } } - return smr, nil + return ar, nil } -// SetTOTP sets the logged in user's TOTP Key. -func (c *Client) SetTOTP(st *www.SetTOTP) (*www.SetTOTPReply, error) { - statusCode, respBody, err := c.makeRequest(http.MethodPost, - cms.APIRoute, www.RouteSetTOTP, st) - if err != nil { - return nil, err +// CommittedTickets returns the committed tickets that belong to the dcrwallet +// instance out of the the specified list of tickets. +func (c *Client) CommittedTickets(ct *walletrpc.CommittedTicketsRequest) (*walletrpc.CommittedTicketsResponse, error) { + if c.wallet == nil { + return nil, fmt.Errorf("walletrpc client not loaded") } - if statusCode != http.StatusOK { - return nil, wwwError(respBody, statusCode) + if c.cfg.Verbose { + fmt.Printf("walletrpc %v CommittedTickets\n", c.cfg.WalletHost) } - var str www.SetTOTPReply - err = json.Unmarshal(respBody, &str) + ctr, err := c.wallet.CommittedTickets(c.ctx, ct) if err != nil { - return nil, fmt.Errorf("unmarshal SetTOTPReply: %v", err) + return nil, err } if c.cfg.Verbose { - err := prettyPrintJSON(str) + err := prettyPrintJSON(ctr) if err != nil { return nil, err } } - return &str, nil + return ctr, nil } -// VerifyTOTP comfirms the logged in user's TOTP Key. -func (c *Client) VerifyTOTP(vt *www.VerifyTOTP) (*www.VerifyTOTPReply, error) { - statusCode, respBody, err := c.makeRequest(http.MethodPost, - cms.APIRoute, www.RouteVerifyTOTP, vt) - if err != nil { - return nil, err +// SignMessages signs the passed in messages using the private keys from the +// specified addresses. +func (c *Client) SignMessages(sm *walletrpc.SignMessagesRequest) (*walletrpc.SignMessagesResponse, error) { + if c.wallet == nil { + return nil, fmt.Errorf("walletrpc client not loaded") } - if statusCode != http.StatusOK { - return nil, wwwError(respBody, statusCode) + if c.cfg.Verbose { + fmt.Printf("walletrpc %v SignMessages\n", c.cfg.WalletHost) } - var vtr www.VerifyTOTPReply - err = json.Unmarshal(respBody, &vtr) + smr, err := c.wallet.SignMessages(c.ctx, sm) if err != nil { - return nil, fmt.Errorf("unmarshal VerifyTOTPReply: %v", err) + return nil, err } if c.cfg.Verbose { - err := prettyPrintJSON(vtr) + err := prettyPrintJSON(smr) if err != nil { return nil, err } } - return &vtr, nil + return smr, nil } +// TODO the wallet client should be its own client and it should verify +// that the dcrwallet client certs are set. // LoadWalletClient connects to a dcrwallet instance. func (c *Client) LoadWalletClient() error { serverCAs := x509.NewCertPool() diff --git a/politeiawww/cmd/shared/config.go b/politeiawww/cmd/shared/config.go index 4dee6c6f8..ad54cf644 100644 --- a/politeiawww/cmd/shared/config.go +++ b/politeiawww/cmd/shared/config.go @@ -12,13 +12,13 @@ import ( "net/http" "net/url" "os" - "os/user" "path/filepath" "runtime" "strings" "github.com/decred/dcrd/dcrutil/v3" "github.com/decred/politeia/politeiad/api/v1/identity" + "github.com/decred/politeia/util" "github.com/decred/politeia/util/version" flags "github.com/jessevdk/go-flags" ) @@ -42,7 +42,7 @@ var ( defaultWalletCertFile = filepath.Join(dcrwalletHomeDir, "rpc.cert") ) -// Config represents the piwww configuration settings. +// Config represents the CLI configuration settings. type Config struct { HomeDir string `long:"appdata" description:"Path to application home directory"` Host string `long:"host" description:"politeiawww host"` @@ -52,6 +52,10 @@ type Config struct { Verbose bool `short:"v" long:"verbose" description:"Print verbose output"` Silent bool `long:"silent" description:"Suppress all output"` + // TODO add docs for this to the piwww README + ClientCert string `long:"clientcert" description:"Path to TLS certificate for dcrwallet client authentication"` + ClientKey string `long:"clientkey" description:"Path to TLS dcrwallet client authentication key"` + DataDir string // Application data dir Version string // CLI version WalletHost string // Wallet host @@ -59,9 +63,6 @@ type Config struct { FaucetHost string // Testnet faucet host CSRF string // CSRF header token - ClientCert string `long:"cert" description:"Path to TLS certificate for client authentication"` - ClientKey string `long:"key" description:"Path to TLS client authentication key"` - Identity *identity.FullIdentity // User identity Cookies []*http.Cookie // User cookies } @@ -113,10 +114,7 @@ func LoadConfig(homeDir, dataDirname, configFilename string) (*Config, error) { // Update the application home directory if specified if cfg.HomeDir != homeDir { - homeDir, err := filepath.Abs(cleanAndExpandPath(cfg.HomeDir)) - if err != nil { - return nil, fmt.Errorf("cleaning path: %v", err) - } + homeDir := util.CleanAndExpandPath(cfg.HomeDir) cfg.HomeDir = homeDir cfg.DataDir = filepath.Join(cfg.HomeDir, dataDirname) } @@ -188,6 +186,8 @@ func LoadConfig(homeDir, dataDirname, configFilename string) (*Config, error) { cfg.Identity = id // Set path for the client key/cert depending on if they are set in options + cfg.ClientCert = util.CleanAndExpandPath(cfg.ClientCert) + cfg.ClientKey = util.CleanAndExpandPath(cfg.ClientKey) if cfg.ClientCert == "" { cfg.ClientCert = filepath.Join(cfg.HomeDir, clientCertFile) } @@ -388,28 +388,6 @@ func (cfg *Config) SaveLoggedInUsername(username string) error { return nil } -// cleanAndExpandPath expands environment variables and leading ~ in the passed -// path, cleans the result, and returns it. -func cleanAndExpandPath(path string) string { - // Expand initial ~ to OS specific home directory - if strings.HasPrefix(path, "~") { - var homeDir string - usr, err := user.Current() - if err == nil { - homeDir = usr.HomeDir - } else { - // Fallback to CWD - homeDir = "." - } - - path = strings.Replace(path, "~", homeDir, 1) - } - - // NOTE: The os.ExpandEnv doesn't work with Windows-style %VARIABLE%, - // but the variables can still be expanded via POSIX-style $VARIABLE. - return filepath.Clean(os.ExpandEnv(path)) -} - // filesExists reports whether the named file or directory exists. func fileExists(name string) bool { if _, err := os.Stat(name); err != nil { diff --git a/politeiawww/config.go b/politeiawww/config.go index 22645c13d..70ea7621d 100644 --- a/politeiawww/config.go +++ b/politeiawww/config.go @@ -16,7 +16,6 @@ import ( "net/mail" "net/url" "os" - "os/user" "path/filepath" "runtime" "sort" @@ -157,60 +156,6 @@ type serviceOptions struct { ServiceCommand string `short:"s" long:"service" description:"Service command {install, remove, start, stop}"` } -// cleanAndExpandPath expands environment variables and leading ~ in the -// passed path, cleans the result, and returns it. -func cleanAndExpandPath(path string) string { - // Nothing to do when no path is given. - if path == "" { - return path - } - - // NOTE: The os.ExpandEnv doesn't work with Windows cmd.exe-style - // %VARIABLE%, but the variables can still be expanded via POSIX-style - // $VARIABLE. - path = os.ExpandEnv(path) - - if !strings.HasPrefix(path, "~") { - return filepath.Clean(path) - } - - // Expand initial ~ to the current user's home directory, or ~otheruser - // to otheruser's home directory. On Windows, both forward and backward - // slashes can be used. - path = path[1:] - - var pathSeparators string - if runtime.GOOS == "windows" { - pathSeparators = string(os.PathSeparator) + "/" - } else { - pathSeparators = string(os.PathSeparator) - } - - userName := "" - if i := strings.IndexAny(path, pathSeparators); i != -1 { - userName = path[:i] - path = path[i:] - } - - homeDir := "" - var u *user.User - var err error - if userName == "" { - u, err = user.Current() - } else { - u, err = user.Lookup(userName) - } - if err == nil { - homeDir = u.HomeDir - } - // Fallback to CWD if user lookup fails or user has no home directory. - if homeDir == "" { - homeDir = "." - } - - return filepath.Join(homeDir, path) -} - // validLogLevel returns whether or not logLevel is a valid debug log level. func validLogLevel(logLevel string) bool { switch logLevel { @@ -345,7 +290,7 @@ func loadIdentity(cfg *config) error { cfg.RPCIdentityFile = filepath.Join(cfg.HomeDir, defaultIdentityFilename) } else { - cfg.RPCIdentityFile = cleanAndExpandPath(cfg.RPCIdentityFile) + cfg.RPCIdentityFile = util.CleanAndExpandPath(cfg.RPCIdentityFile) } if cfg.FetchIdentity { @@ -589,19 +534,19 @@ func loadConfig() (*config, []string, error) { // All data is specific to a network, so namespacing the data directory // means each individual piece of serialized data does not have to // worry about changing names per network and such. - cfg.DataDir = cleanAndExpandPath(cfg.DataDir) + cfg.DataDir = util.CleanAndExpandPath(cfg.DataDir) cfg.DataDir = filepath.Join(cfg.DataDir, netName(activeNetParams)) // Append the network type to the log directory so it is "namespaced" // per network in the same fashion as the data directory. - cfg.LogDir = cleanAndExpandPath(cfg.LogDir) + cfg.LogDir = util.CleanAndExpandPath(cfg.LogDir) cfg.LogDir = filepath.Join(cfg.LogDir, netName(activeNetParams)) cfg.AdminLogFile = filepath.Join(cfg.LogDir, adminLogFilename) - cfg.HTTPSKey = cleanAndExpandPath(cfg.HTTPSKey) - cfg.HTTPSCert = cleanAndExpandPath(cfg.HTTPSCert) - cfg.RPCCert = cleanAndExpandPath(cfg.RPCCert) + cfg.HTTPSKey = util.CleanAndExpandPath(cfg.HTTPSKey) + cfg.HTTPSCert = util.CleanAndExpandPath(cfg.HTTPSCert) + cfg.RPCCert = util.CleanAndExpandPath(cfg.RPCCert) // Special show command to list supported subsystems and exit. if cfg.DebugLevel == "show" { @@ -710,7 +655,7 @@ func loadConfig() (*config, []string, error) { // Validate smtp root cert. if cfg.SMTPCert != "" { - cfg.SMTPCert = cleanAndExpandPath(cfg.SMTPCert) + cfg.SMTPCert = util.CleanAndExpandPath(cfg.SMTPCert) b, err := ioutil.ReadFile(cfg.SMTPCert) if err != nil { @@ -769,9 +714,9 @@ func loadConfig() (*config, []string, error) { } // Clean user database settings - cfg.DBRootCert = cleanAndExpandPath(cfg.DBRootCert) - cfg.DBCert = cleanAndExpandPath(cfg.DBCert) - cfg.DBKey = cleanAndExpandPath(cfg.DBKey) + cfg.DBRootCert = util.CleanAndExpandPath(cfg.DBRootCert) + cfg.DBCert = util.CleanAndExpandPath(cfg.DBCert) + cfg.DBKey = util.CleanAndExpandPath(cfg.DBKey) // Validate user database host _, err = url.Parse(cfg.DBHost) @@ -798,8 +743,8 @@ func loadConfig() (*config, []string, error) { } // Validate user database encryption keys - cfg.EncryptionKey = cleanAndExpandPath(cfg.EncryptionKey) - cfg.OldEncryptionKey = cleanAndExpandPath(cfg.OldEncryptionKey) + cfg.EncryptionKey = util.CleanAndExpandPath(cfg.EncryptionKey) + cfg.OldEncryptionKey = util.CleanAndExpandPath(cfg.OldEncryptionKey) if cfg.EncryptionKey != "" && !util.FileExists(cfg.EncryptionKey) { return nil, nil, fmt.Errorf("file not found %v", cfg.EncryptionKey) diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index c212f295f..ba5bfd46a 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -1469,6 +1469,7 @@ func (p *politeiawww) processProposalStatusSet(ctx context.Context, pss pi.Propo return nil, convertUserErrorFromSignatureError(err) } + // TODO don't allow censoring a proposal once the vote has started // Verification that requires retrieving the existing proposal is // done in politeiad. This includes: // -Verify proposal exists (politeiad) diff --git a/wsdcrdata/wsdcrdata.go b/wsdcrdata/wsdcrdata.go index 23c81e22e..6edf24b6c 100644 --- a/wsdcrdata/wsdcrdata.go +++ b/wsdcrdata/wsdcrdata.go @@ -448,9 +448,7 @@ func psclientNew(url string) (*psclient.Client, error) { ReadTimeout: psclient.DefaultReadTimeout, WriteTimeout: 3 * time.Second, } - d := time.Now().Add(15 * time.Second) - ctx, _ := context.WithDeadline(context.Background(), d) - c, err := psclient.New(url, ctx, &opts) + c, err := psclient.New(url, context.Background(), &opts) if err != nil { return nil, fmt.Errorf("failed to connect to %v: %v", url, err) } From 0e5ff96ce499154b1c715973ea7d48c5efcb7c1c Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 22 Oct 2020 17:09:35 -0500 Subject: [PATCH 170/449] fix rfp vote bugs --- politeiad/backend/tlogbe/pi.go | 180 +++++++-------- politeiad/backend/tlogbe/ticketvote.go | 205 ++++++++++++------ politeiad/backend/tlogbe/tlogbe.go | 3 +- politeiad/plugins/pi/pi.go | 4 +- politeiad/plugins/ticketvote/ticketvote.go | 4 + politeiawww/api/pi/v1/v1.go | 9 + .../piwww/{voteballot.go => castballot.go} | 3 + politeiawww/cmd/piwww/piwww.go | 2 +- politeiawww/cmd/piwww/votestartrunoff.go | 1 + politeiawww/pi.go | 31 ++- politeiawww/piwww.go | 63 +++++- politeiawww/www.go | 18 +- 12 files changed, 339 insertions(+), 184 deletions(-) rename politeiawww/cmd/piwww/{voteballot.go => castballot.go} (98%) diff --git a/politeiad/backend/tlogbe/pi.go b/politeiad/backend/tlogbe/pi.go index cc8952e65..9090733ed 100644 --- a/politeiad/backend/tlogbe/pi.go +++ b/politeiad/backend/tlogbe/pi.go @@ -17,6 +17,7 @@ import ( "sync" "time" + "github.com/decred/dcrd/chaincfg/v3" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/plugins/comments" "github.com/decred/politeia/politeiad/plugins/pi" @@ -36,8 +37,9 @@ var ( // piPlugin satisfies the pluginClient interface. type piPlugin struct { sync.Mutex - backend backend.Backend - tlog tlogClient + backend backend.Backend + tlog tlogClient + activeNetParams *chaincfg.Params // dataDir is the pi plugin data directory. The only data that is // stored here is cached data that can be re-created at any time @@ -714,119 +716,84 @@ func (p *piPlugin) ticketVoteStart(payload string) (string, error) { } } - // Get records for all RFP submissions - records := make(map[string]backend.Record, len(s.Starts)) - for _, v := range s.Starts { - token, err := tokenDecode(v.Params.Token) - if err != nil { - e := fmt.Sprintf("%v: %v", v.Params.Token, err) - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropTokenInvalid), - ErrorContext: []string{e}, - } - } - r, err := p.backend.GetVetted(token, "") - if err != nil { - if errors.Is(err, errRecordNotFound) { - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropNotFound), - ErrorContext: []string{v.Params.Token}, - } - } - return "", fmt.Errorf("GetVetted %x: %v", token, err) - } - records[v.Params.Token] = *r + // Sanity check. This pass through command should only be used for + // RFP runoff votes. + if s.Starts[0].Params.Type != ticketvote.VoteTypeRunoff { + return "", fmt.Errorf("not a runoff vote") } - // Get RFP token. Just use the linkto from the first start details - // record. - token := s.Starts[0].Params.Token - r := records[token] - pm, err := proposalMetadataFromFiles(r.Files) + // Get RFP token. Just use the parent token from the first vote + // params. If the different vote params use different parent + // tokens, the ticketvote plugin will catch it. + rfpToken := s.Starts[0].Params.Parent + rfpTokenb, err := tokenDecode(rfpToken) if err != nil { - return "", err - } - if pm == nil { - // Proposal metadata was not found - e := fmt.Sprintf("record is not a proposal %v", token) - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusStartDetailsInvalid), - ErrorContext: []string{e}, - } - } - if pm.LinkTo == "" { - // Proposal is not an RFP submission - e := fmt.Sprintf("proposal is not an rfp submission %v", token) + e := fmt.Sprintf("invalid rfp token '%v'", rfpToken) return "", backend.PluginUserError{ PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusStartDetailsInvalid), + ErrorCode: int(pi.ErrorStatusVoteParentInvalid), ErrorContext: []string{e}, } } - rfpToken, err := tokenDecode(pm.LinkTo) - if err != nil { - return "", fmt.Errorf("decode rfp token %v: %v", pm.LinkTo, err) - } // Get RFP record - rfp, err := p.backend.GetVetted(rfpToken, "") + rfp, err := p.backend.GetVetted(rfpTokenb, "") if err != nil { if errors.Is(err, errRecordNotFound) { e := fmt.Sprintf("rfp not found %x", rfpToken) return "", backend.PluginUserError{ PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusRFPInvalid), + ErrorCode: int(pi.ErrorStatusVoteParentInvalid), ErrorContext: []string{e}, } } - return "", fmt.Errorf("GetVetted %x: %v", token, err) + return "", fmt.Errorf("GetVetted %x: %v", rfpToken, err) } - // Verify RFP proposal linkby has expired + // Verify RFP linkby has expired. The runoff vote is not allowed to + // start until after the linkby deadline has passed. rfpPM, err := proposalMetadataFromFiles(rfp.Files) if err != nil { return "", err } if rfpPM == nil { - e := fmt.Sprintf("rfp is not a proposal %x", rfpToken) + e := fmt.Sprintf("rfp is not a proposal %v", rfpToken) return "", backend.PluginUserError{ PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusRFPInvalid), + ErrorCode: int(pi.ErrorStatusVoteParentInvalid), ErrorContext: []string{e}, } } - if rfpPM.LinkBy > time.Now().Unix() { - e := fmt.Sprintf("rfp %x linkby deadline not met %v", + isExpired := rfpPM.LinkBy < time.Now().Unix() + isMainNet := p.activeNetParams.Name == chaincfg.MainNetParams().Name + switch { + case !isExpired && isMainNet: + e := fmt.Sprintf("rfp %v linkby deadline not met %v", rfpToken, rfpPM.LinkBy) return "", backend.PluginUserError{ PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusLinkByDeadlineNotMet), + ErrorCode: int(pi.ErrorStatusLinkByNotExpired), ErrorContext: []string{e}, } + case !isExpired: + // Allow the vote to be started before the link by deadline + // expires on testnet and simnet only. This makes testing the + // RFP process easier. + log.Warnf("RFP linkby deadline has not been met; disregarding " + + "since this is not mainnet") } - // Verify all public, non-abandoned RFP submissions have been - // included in the request. The linked from list of the RFP will - // include abandoned proposals that must be filtered out. - linkedFrom, err := p.linkedFrom(hex.EncodeToString(rfpToken)) + // Compile a list of the expected RFP submissions that should be in + // the runoff vote. This will be all of the public proposals that + // have linked to the RFP. The RFP's linked from list will include + // abandoned proposals that need to be filtered out. + linkedFrom, err := p.linkedFrom(rfpToken) if err != nil { return "", err } + // map[token]struct{} expected := make(map[string]struct{}, len(linkedFrom.Tokens)) for k := range linkedFrom.Tokens { - _, ok := records[k] - if ok { - // RFP submission has been included in the runoff vote - expected[k] = struct{}{} - continue - } - - // RFP submission has not been included in the runoff vote. This - // is expected if the submission has been abandoned. If not then - // it is a user error. token, err := tokenDecode(k) if err != nil { return "", err @@ -835,41 +802,51 @@ func (p *piPlugin) ticketVoteStart(payload string) (string, error) { if err != nil { return "", err } - if r.RecordMetadata.Status == backend.MDStatusVetted { - // Record is public and should be part of runoff vote + if r.RecordMetadata.Status != backend.MDStatusVetted { + // This proposal is not public and should not be included in + // the runoff vote. + continue + } + + // This is a public proposal that is part of the RFP linked from + // list. It is required to be in the runoff vote. + expected[k] = struct{}{} + } + + // Verify that there are no extra submissions in the runoff vote + for _, v := range s.Starts { + _, ok := expected[v.Params.Token] + if !ok { + // This submission should not be here + e := fmt.Sprintf("found token that should not be included: %v", + v.Params.Token) return "", backend.PluginUserError{ PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusStartDetailsMissing), - ErrorContext: []string{r.RecordMetadata.Token}, + ErrorCode: int(pi.ErrorStatusStartDetailsInvalid), + ErrorContext: []string{e}, } } } - // We know that all public records in the RFP's linked from list - // have been included in the runoff vote, but we must also verify - // that no extra start details have been included that shouldn't be - // there. - if len(s.Starts) != len(expected) { - // There are extra submissions. Find the culprits. - invalid := make([]string, 0, len(s.Starts)) - for _, v := range s.Starts { - _, ok := expected[v.Params.Token] - if !ok { - // This submission should not be here - invalid = append(invalid, v.Params.Token) + // Verify that the runoff vote is not missing any submissions + subs := make(map[string]struct{}, len(s.Starts)) + for _, v := range s.Starts { + subs[v.Params.Token] = struct{}{} + } + for token := range expected { + _, ok := subs[token] + if !ok { + // This proposal is missing from the runoff vote + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusStartDetailsMissing), + ErrorContext: []string{token}, } } - e := fmt.Sprintf("found tokens that should not be included: %v", - strings.Join(invalid, ", ")) - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusStartDetailsInvalid), - ErrorContext: []string{e}, - } } // Pi plugin validation complete! Pass the plugin command to the - // backend. + // ticketvote plugin. return p.backend.Plugin(ticketvote.ID, ticketvote.CmdStart, "", payload) } @@ -1266,7 +1243,7 @@ func (p *piPlugin) fsck() error { return nil } -func newPiPlugin(backend backend.Backend, tlog tlogClient, settings []backend.PluginSetting) (*piPlugin, error) { +func newPiPlugin(backend backend.Backend, tlog tlogClient, settings []backend.PluginSetting, activeNetParams *chaincfg.Params) (*piPlugin, error) { // Unpack plugin settings var dataDir string for _, v := range settings { @@ -1293,8 +1270,9 @@ func newPiPlugin(backend backend.Backend, tlog tlogClient, settings []backend.Pl } return &piPlugin{ - dataDir: dataDir, - backend: backend, - tlog: tlog, + dataDir: dataDir, + backend: backend, + activeNetParams: activeNetParams, + tlog: tlog, }, nil } diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index af56bc862..1dcf49522 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -69,9 +69,9 @@ var ( // ticketVotePlugin satisfies the pluginClient interface. type ticketVotePlugin struct { sync.Mutex - activeNetParams *chaincfg.Params backend backend.Backend tlog tlogClient + activeNetParams *chaincfg.Params // Plugin settings voteDurationMin uint32 // In blocks @@ -1395,6 +1395,27 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM } } + // Verify parent token + switch { + case vote.Type == ticketvote.VoteTypeStandard && vote.Parent != "": + e := "parent token should not be provided for a standard vote" + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorContext: []string{e}, + } + case vote.Type == ticketvote.VoteTypeRunoff: + _, err := tokenDecode(vote.Parent) + if err != nil { + e := fmt.Sprintf("invalid parent %v", vote.Parent) + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorContext: []string{e}, + } + } + } + return nil } @@ -1548,12 +1569,13 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep duration = s.Starts[0].Params.Duration quorum = s.Starts[0].Params.QuorumPercentage pass = s.Starts[0].Params.PassPercentage + parent = s.Starts[0].Params.Parent ) for _, v := range s.Starts { // Verify vote params are the same for all submissions switch { case v.Params.Type != ticketvote.VoteTypeRunoff: - e := fmt.Sprintf("vote type invalid %v: got %v, want %v", + e := fmt.Sprintf("%v vote type invalid: got %v, want %v", v.Params.Token, v.Params.Type, ticketvote.VoteTypeRunoff) return nil, backend.PluginUserError{ PluginID: ticketvote.ID, @@ -1561,7 +1583,7 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep ErrorContext: []string{e}, } case v.Params.Mask != mask: - e := fmt.Sprintf("mask invalid %v: all masks must be the same", + e := fmt.Sprintf("%v mask invalid: all must be the same", v.Params.Token) return nil, backend.PluginUserError{ PluginID: ticketvote.ID, @@ -1569,7 +1591,7 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep ErrorContext: []string{e}, } case v.Params.Duration != duration: - e := fmt.Sprintf("duration invalid %v: all durations must be the same", + e := fmt.Sprintf("%v duration invalid: all must be the same", v.Params.Token) return nil, backend.PluginUserError{ PluginID: ticketvote.ID, @@ -1577,7 +1599,7 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep ErrorContext: []string{e}, } case v.Params.QuorumPercentage != quorum: - e := fmt.Sprintf("quorum invalid %v: all quorums must be the same", + e := fmt.Sprintf("%v quorum invalid: must be the same", v.Params.Token) return nil, backend.PluginUserError{ PluginID: ticketvote.ID, @@ -1585,7 +1607,15 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep ErrorContext: []string{e}, } case v.Params.PassPercentage != pass: - e := fmt.Sprintf("pass rate invalid %v: all pass rates must be the same", + e := fmt.Sprintf("%v pass rate invalid: all must be the same", + v.Params.Token) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorContext: []string{e}, + } + case v.Params.Parent != parent: + e := fmt.Sprintf("%v parent invalid: all must be the same", v.Params.Token) return nil, backend.PluginUserError{ PluginID: ticketvote.ID, @@ -1629,12 +1659,26 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep return nil, err } + // Verify parent exists + parentb, err := tokenDecode(parent) + if err != nil { + return nil, err + } + if !p.backend.VettedExists(parentb) { + e := fmt.Sprintf("parent record not found %v", parent) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorContext: []string{e}, + } + } + // TODO handle the case where part of the votes are started but // not all. + // Validate existing record state. The lock for each record must + // be held for the remainder of this function. for _, v := range s.Starts { - // Validate existing record state. The lock for this record must - // be held for the remainder of this function. m := p.mutex(v.Params.Token) m.Lock() defer m.Unlock() @@ -2228,12 +2272,66 @@ func (p *ticketVotePlugin) cmdResults(payload string) (string, error) { return string(reply), nil } +// voteIsApproved returns whether the provided vote option results met the +// provided quorum and pass percentage requirements. This function can only be +// called on votes that use VoteOptionIDApprove and VoteOptionIDReject. Any +// other vote option IDs will cause this function to panic. +func voteIsApproved(vd ticketvote.VoteDetails, results []ticketvote.VoteOptionResult) bool { + // Tally the total votes + var total uint64 + for _, v := range results { + total += v.Votes + } + + // Calculate required thresholds + var ( + eligible = float64(len(vd.EligibleTickets)) + quorumPerc = float64(vd.Params.QuorumPercentage) + passPerc = float64(vd.Params.PassPercentage) + quorum = uint64(quorumPerc / 100 * eligible) + pass = uint64(passPerc / 100 * float64(total)) + + approvedVotes uint64 + ) + + // Tally approve votes + for _, v := range results { + switch v.ID { + case ticketvote.VoteOptionIDApprove: + // Valid vote option + approvedVotes++ + case ticketvote.VoteOptionIDReject: + // Valid vote option + default: + // Invalid vote option + e := fmt.Sprintf("invalid vote option id found: %v", v.ID) + panic(e) + } + } + + // Check tally against thresholds + var approved bool + switch { + case total < quorum: + // Quorum not met + approved = false + case approvedVotes < pass: + // Pass percentage not met + approved = false + default: + // Vote was approved + approved = true + } + + return approved +} + func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote.Summary, error) { // Check if the summary has been cached s, err := p.cachedSummary(hex.EncodeToString(token)) switch { case errors.Is(err, errRecordNotFound): - // Cached summary not found + // Cached summary not found. Continue. case err != nil: // Some other error return nil, fmt.Errorf("cachedSummary: %v", err) @@ -2245,7 +2343,7 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. // Summary has not been cached. Get it manually. // Assume vote is unauthorized. Only update the status when the - // appropriate record has been found. + // appropriate record has been found that proves otherwise. status := ticketvote.VoteStatusUnauthorized // Check if the vote has been authorized @@ -2290,8 +2388,7 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. status = ticketvote.VoteStatusFinished } - // Pull the cast votes from the cache and calculate the results - // manually. + // Pull the cast votes from the cache and tally the results votes := p.cachedVotes(token) tally := make(map[string]int, len(vd.Params.Options)) for _, voteBit := range votes { @@ -2308,44 +2405,6 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. }) } - // Approved can only be calculated on certain types of votes - var approved bool - switch vd.Params.Type { - case ticketvote.VoteTypeStandard, ticketvote.VoteTypeRunoff: - // Calculate results for a simple approve/reject vote - var total uint64 - for _, v := range results { - total += v.Votes - } - - var ( - eligible = float64(len(vd.EligibleTickets)) - quorumPerc = float64(vd.Params.QuorumPercentage) - passPerc = float64(vd.Params.PassPercentage) - quorum = uint64(quorumPerc / 100 * eligible) - pass = uint64(passPerc / 100 * float64(total)) - - approvedVotes uint64 - ) - for _, v := range results { - if v.ID == ticketvote.VoteOptionIDApprove { - approvedVotes++ - } - } - - switch { - case total < quorum: - // Quorum not met - approved = false - case approvedVotes < pass: - // Pass percentage not met - approved = false - default: - // Vote was approved - approved = true - } - } - // Prepare summary summary := ticketvote.Summary{ Type: vd.Params.Type, @@ -2358,24 +2417,44 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. QuorumPercentage: vd.Params.QuorumPercentage, PassPercentage: vd.Params.PassPercentage, Results: results, - Approved: approved, } - // Cache the summary if the vote has finished so we don't have to - // calculate these results again. - if status == ticketvote.VoteStatusFinished { - // Save summary - err = p.cachedSummarySave(vd.Params.Token, summary) - if err != nil { - return nil, fmt.Errorf("cachedSummarySave %v: %v %v", - vd.Params.Token, err, summary) - } + // If the vote has not finished yet then we are done for now. + if status == ticketvote.VoteStatusStarted { + return &summary, nil + } - // Remove record from the votes cache now that a summary has been - // saved for it. - p.cachedVotesDel(vd.Params.Token) + // The vote has finished. We can calculate if the vote was approved + // for certain vote types and cache the results. + switch vd.Params.Type { + case ticketvote.VoteTypeStandard, ticketvote.VoteTypeRunoff: + // These vote types are strictly approve/reject votes so we can + // calculate the vote approval. Continue. + default: + // Nothing else to do for all other vote types + return &summary, nil } + // Calculate vote approval + approved := voteIsApproved(*vd, results) + + // If this is a standard vote then we can take the results as is. A + // runoff vote requires that we pull all other runoff vote + // submissions to determine if the vote actually passed. + // TODO + summary.Approved = approved + + // Cache the summary + err = p.cachedSummarySave(vd.Params.Token, summary) + if err != nil { + return nil, fmt.Errorf("cachedSummarySave %v: %v %v", + vd.Params.Token, err, summary) + } + + // Remove record from the votes cache now that a summary has been + // saved for it. + p.cachedVotesDel(vd.Params.Token) + return &summary, nil } diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index b514abe13..46d5fc29c 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -1666,7 +1666,8 @@ func (t *tlogBackend) RegisterPlugin(p backend.Plugin) error { return err } case pi.ID: - client, err = newPiPlugin(t, newBackendClient(t), p.Settings) + client, err = newPiPlugin(t, newBackendClient(t), p.Settings, + t.activeNetParams) if err != nil { return err } diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index 4b984f64c..87b84a3e5 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -80,8 +80,8 @@ const ( ErrorStatusVoteStatusInvalid ErrorStatusStartDetailsInvalid ErrorStatusStartDetailsMissing - ErrorStatusRFPInvalid - ErrorStatusLinkByDeadlineNotMet + ErrorStatusVoteParentInvalid + ErrorStatusLinkByNotExpired ) var ( diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index 5b03da3a8..20621516e 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -196,6 +196,10 @@ type VoteParams struct { PassPercentage uint32 `json:"passpercentage"` Options []VoteOption `json:"options"` + + // Parent is the token of the parent record. This field will only + // be populated for runoff votes. + Parent string `json:"parent,omitempty"` } // VoteDetails is the structure that is saved to disk when a vote is started. diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 4554fa923..7fe4e24f9 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -190,7 +190,12 @@ const ( // Vote errors ErrorStatusVoteAuthInvalid ErrorStatusVoteStatusInvalid + ErrorStatusStartDetailsInvalid + ErrorStatusStartDetailsMissing ErrorStatusVoteParamsInvalid + ErrorStatusVoteTypeInvalid + ErroStatusVoteParentInvalid + ErrorStatusLinkByNotExpired ) var ( @@ -659,6 +664,10 @@ type VoteParams struct { PassPercentage uint32 `json:"passpercentage"` Options []VoteOption `json:"options"` + + // Parent is the token of the parent proposal. This field will only + // be populated for runoff votes. + Parent string `json:"parent,omitempty"` } // VoteDetails contains the details of a proposal vote. diff --git a/politeiawww/cmd/piwww/voteballot.go b/politeiawww/cmd/piwww/castballot.go similarity index 98% rename from politeiawww/cmd/piwww/voteballot.go rename to politeiawww/cmd/piwww/castballot.go index fa2fd85c1..3be690e8e 100644 --- a/politeiawww/cmd/piwww/voteballot.go +++ b/politeiawww/cmd/piwww/castballot.go @@ -44,6 +44,9 @@ func (c *castBallotCmd) Execute(args []string) error { if !ok { return fmt.Errorf("proposal not found: %v", token) } + if pv.Vote == nil { + return fmt.Errorf("vote hasn't started yet") + } // Verify provided vote ID var voteBit string diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index 893ddce7f..729bab9c1 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -153,7 +153,7 @@ Vote commands voteauthorize (user) Authorize a proposal vote votestart (admin) Start a proposal vote votestartrunoff (admin) Start a runoff vote - voteballot (public) Cast a ballot of votes + castballot (public) Cast a ballot of votes votes (public) Get vote details voteresults (public) Get full vote results votesummaries (public) Get vote summaries diff --git a/politeiawww/cmd/piwww/votestartrunoff.go b/politeiawww/cmd/piwww/votestartrunoff.go index b14bed541..98db2b468 100644 --- a/politeiawww/cmd/piwww/votestartrunoff.go +++ b/politeiawww/cmd/piwww/votestartrunoff.go @@ -113,6 +113,7 @@ func (cmd *voteStartRunoffCmd) Execute(args []string) error { Bit: 0x02, }, }, + Parent: cmd.Args.TokenRFP, } vb, err := json.Marshal(vote) if err != nil { diff --git a/politeiawww/pi.go b/politeiawww/pi.go index 4ed7e73f5..9905ad664 100644 --- a/politeiawww/pi.go +++ b/politeiawww/pi.go @@ -7,6 +7,7 @@ package main import ( "context" + "github.com/decred/politeia/politeiad/plugins/pi" piplugin "github.com/decred/politeia/politeiad/plugins/pi" ) @@ -16,7 +17,8 @@ func (p *politeiawww) piProposals(ctx context.Context, ps piplugin.Proposals) (* if err != nil { return nil, err } - r, err := p.pluginCommand(ctx, piplugin.ID, piplugin.CmdProposals, string(b)) + r, err := p.pluginCommand(ctx, piplugin.ID, + piplugin.CmdProposals, string(b)) if err != nil { return nil, err } @@ -33,7 +35,8 @@ func (p *politeiawww) piCommentNew(ctx context.Context, cn piplugin.CommentNew) if err != nil { return nil, err } - r, err := p.pluginCommand(ctx, piplugin.ID, piplugin.CmdCommentNew, string(b)) + r, err := p.pluginCommand(ctx, piplugin.ID, + piplugin.CmdCommentNew, string(b)) if err != nil { return nil, err } @@ -50,7 +53,8 @@ func (p *politeiawww) piCommentVote(ctx context.Context, cvp piplugin.CommentVot if err != nil { return nil, err } - r, err := p.pluginCommand(ctx, piplugin.ID, piplugin.CmdCommentVote, string(b)) + r, err := p.pluginCommand(ctx, piplugin.ID, + piplugin.CmdCommentVote, string(b)) if err != nil { return nil, err } @@ -67,7 +71,8 @@ func (p *politeiawww) piCommentCensor(ctx context.Context, cc piplugin.CommentCe if err != nil { return nil, err } - r, err := p.pluginCommand(ctx, piplugin.ID, piplugin.CmdCommentCensor, string(b)) + r, err := p.pluginCommand(ctx, piplugin.ID, + piplugin.CmdCommentCensor, string(b)) if err != nil { return nil, err } @@ -90,3 +95,21 @@ func (p *politeiawww) piVoteInventory(ctx context.Context) (*piplugin.VoteInvent } return vir, nil } + +// piPassThrough executes the pi plugin PassThrough command. +func (p *politeiawww) piPassThrough(ctx context.Context, pt pi.PassThrough) (*pi.PassThroughReply, error) { + b, err := piplugin.EncodePassThrough(pt) + if err != nil { + return nil, err + } + r, err := p.pluginCommand(ctx, piplugin.ID, + piplugin.CmdPassThrough, string(b)) + if err != nil { + return nil, err + } + ptr, err := piplugin.DecodePassThroughReply(([]byte(r))) + if err != nil { + return nil, err + } + return ptr, nil +} diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index ba5bfd46a..e8b2e7438 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -471,6 +471,7 @@ func convertVoteParamsFromPi(v pi.VoteParams) ticketvote.VoteParams { Duration: v.Duration, QuorumPercentage: v.QuorumPercentage, PassPercentage: v.PassPercentage, + Parent: v.Parent, } // Convert vote options vo := make([]ticketvote.VoteOption, 0, len(v.Options)) @@ -1902,6 +1903,14 @@ func (p *politeiawww) processVoteStart(ctx context.Context, vs pi.VoteStart, usr return nil, fmt.Errorf("not an admin") } + // Verify there is work to be done + if len(vs.Starts) == 0 { + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusStartDetailsInvalid, + ErrorContext: []string{"no start details found"}, + } + } + // Verify admin signed with their active identity for _, v := range vs.Starts { if usr.PublicKey() != v.PublicKey { @@ -1912,19 +1921,57 @@ func (p *politeiawww) processVoteStart(ctx context.Context, vs pi.VoteStart, usr } } - // Call the ticketvote plugin to start vote - reply, err := p.voteStart(ctx, convertVoteStartFromPi(vs)) - if err != nil { - return nil, err + // Start vote + var ( + sr *ticketvote.StartReply + err error + ) + switch vs.Starts[0].Params.Type { + case pi.VoteTypeStandard: + // A standard vote can be started using the ticketvote plugin + // directly. + sr, err = p.voteStart(ctx, convertVoteStartFromPi(vs)) + if err != nil { + return nil, err + } + + case pi.VoteTypeRunoff: + // A runoff vote requires additional validation that is not part + // of the ticketvote plugin. We pass the ticketvote command + // through the pi plugin so that it can perform this additional + // validation. + s := convertVoteStartFromPi(vs) + payload, err := ticketvote.EncodeStart(s) + if err != nil { + return nil, err + } + pt := piplugin.PassThrough{ + PluginID: ticketvote.ID, + Cmd: ticketvote.CmdStart, + Payload: string(payload), + } + ptr, err := p.piPassThrough(ctx, pt) + if err != nil { + return nil, err + } + sr, err = ticketvote.DecodeStartReply([]byte(ptr.Payload)) + if err != nil { + return nil, err + } + + default: + return nil, pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusVoteTypeInvalid, + } } // TODO Emit notification for each start return &pi.VoteStartReply{ - StartBlockHeight: reply.StartBlockHeight, - StartBlockHash: reply.StartBlockHash, - EndBlockHeight: reply.EndBlockHeight, - EligibleTickets: reply.EligibleTickets, + StartBlockHeight: sr.StartBlockHeight, + StartBlockHash: sr.StartBlockHash, + EndBlockHeight: sr.EndBlockHeight, + EligibleTickets: sr.EligibleTickets, }, nil } diff --git a/politeiawww/www.go b/politeiawww/www.go index b4600ecde..970f77396 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -82,6 +82,8 @@ func convertWWWErrorStatusFromPD(e pd.ErrorStatusT) www.ErrorStatusT { return www.ErrorStatusInvalid } +// TODO verify all plugin errors have been added to these www conversion +// functions func convertWWWErrorStatusFromPiPlugin(e piplugin.ErrorStatusT) www.ErrorStatusT { switch e { case piplugin.ErrorStatusPropLinkToInvalid: @@ -187,12 +189,14 @@ func convertPiErrorStatusFromPD(e pd.ErrorStatusT) pi.ErrorStatusT { func convertPiErrorStatusFromPiPlugin(e piplugin.ErrorStatusT) pi.ErrorStatusT { switch e { + case piplugin.ErrorStatusPageSizeExceeded: + return pi.ErrorStatusPageSizeExceeded + case piplugin.ErrorStatusPropNotFound: + return pi.ErrorStatusPropNotFound case piplugin.ErrorStatusPropStateInvalid: return pi.ErrorStatusPropStateInvalid case piplugin.ErrorStatusPropTokenInvalid: return pi.ErrorStatusPropTokenInvalid - case piplugin.ErrorStatusPropNotFound: - return pi.ErrorStatusPropNotFound case piplugin.ErrorStatusPropStatusInvalid: return pi.ErrorStatusPropStatusInvalid case piplugin.ErrorStatusPropVersionInvalid: @@ -203,8 +207,14 @@ func convertPiErrorStatusFromPiPlugin(e piplugin.ErrorStatusT) pi.ErrorStatusT { return pi.ErrorStatusPropLinkToInvalid case piplugin.ErrorStatusVoteStatusInvalid: return pi.ErrorStatusVoteStatusInvalid - case piplugin.ErrorStatusPageSizeExceeded: - return pi.ErrorStatusPageSizeExceeded + case piplugin.ErrorStatusStartDetailsInvalid: + return pi.ErrorStatusStartDetailsInvalid + case piplugin.ErrorStatusStartDetailsMissing: + return pi.ErrorStatusStartDetailsMissing + case piplugin.ErrorStatusVoteParentInvalid: + return pi.ErroStatusVoteParentInvalid + case piplugin.ErrorStatusLinkByNotExpired: + return pi.ErrorStatusLinkByNotExpired } return pi.ErrorStatusInvalid } From 261b00c06350d830022f9bbbd4e98af7dae0c100 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 27 Oct 2020 16:52:42 -0500 Subject: [PATCH 171/449] fix error reply json bug --- politeiawww/api/pi/v1/v1.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 7fe4e24f9..59f918424 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -268,8 +268,8 @@ var ( // error that is caused by something that the user did (malformed input, bad // timing, etc). The HTTP status code will be 400. type UserErrorReply struct { - ErrorCode ErrorStatusT - ErrorContext []string + ErrorCode ErrorStatusT `json:"errorcode"` + ErrorContext []string `json:"errorcontext"` } // Error satisfies the error interface. From c3df85203bfce353dd58764d6ac293ee4c8ffb9b Mon Sep 17 00:00:00 2001 From: Amir Massarwa Date: Mon, 2 Nov 2020 18:36:28 +0200 Subject: [PATCH 172/449] piwww: testrun - add proposal routes. --- politeiad/plugins/pi/pi.go | 4 +- politeiawww/cmd/piwww/commentnew.go | 6 +- politeiawww/cmd/piwww/proposaledit.go | 29 +- politeiawww/cmd/piwww/proposalnew.go | 29 +- politeiawww/cmd/piwww/proposalstatusset.go | 3 + politeiawww/cmd/piwww/testrun.go | 680 +++++++++++++++++++-- politeiawww/cmd/piwww/util.go | 24 + politeiawww/cmd/piwww/voteauthorize.go | 1 + 8 files changed, 674 insertions(+), 102 deletions(-) diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index 87b84a3e5..b2806a2a9 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -304,12 +304,12 @@ type CommentNewReply struct { Comment comments.Comment `json:"comment"` } -// EncodeCommentNew encodes a CommentNewReply into a JSON byte slice. +// EncodeCommentNewReply encodes a CommentNewReply into a JSON byte slice. func EncodeCommentNewReply(cnr CommentNewReply) ([]byte, error) { return json.Marshal(cnr) } -// DecodeCommentNew decodes a JSON byte slice into a CommentNewReply. +// DecodeCommentNewReply decodes a JSON byte slice into a CommentNewReply. func DecodeCommentNewReply(payload []byte) (*CommentNewReply, error) { var cnr CommentNewReply err := json.Unmarshal(payload, &cnr) diff --git a/politeiawww/cmd/piwww/commentnew.go b/politeiawww/cmd/piwww/commentnew.go index 863b6e688..058ca96b6 100644 --- a/politeiawww/cmd/piwww/commentnew.go +++ b/politeiawww/cmd/piwww/commentnew.go @@ -80,11 +80,11 @@ func (c *commentNewCmd) Execute(args []string) error { if err != nil { return err } - ncr, err := client.CommentNew(cn) + cnr, err := client.CommentNew(cn) if err != nil { return err } - err = shared.PrintJSON(ncr) + err = shared.PrintJSON(cnr) if err != nil { return err } @@ -98,7 +98,7 @@ func (c *commentNewCmd) Execute(args []string) error { if err != nil { return err } - receiptb, err := util.ConvertSignature(ncr.Comment.Receipt) + receiptb, err := util.ConvertSignature(cnr.Comment.Receipt) if err != nil { return err } diff --git a/politeiawww/cmd/piwww/proposaledit.go b/politeiawww/cmd/piwww/proposaledit.go index fc5101570..f4c0d4296 100644 --- a/politeiawww/cmd/piwww/proposaledit.go +++ b/politeiawww/cmd/piwww/proposaledit.go @@ -5,7 +5,6 @@ package main import ( - "bytes" "encoding/base64" "encoding/hex" "encoding/json" @@ -87,35 +86,33 @@ func (cmd *proposalEditCmd) Execute(args []string) error { } // Prepare index file - var payload []byte + var ( + file *pi.File + err error + ) if cmd.Random { // Generate random text for the index file - var b bytes.Buffer - for i := 0; i < 10; i++ { - r, err := util.Random(32) - if err != nil { - return err - } - b.WriteString(base64.StdEncoding.EncodeToString(r) + "\n") + file, err = createMDFile() + if err != nil { + return err } - payload = b.Bytes() } else { // Read the index file from disk fp := util.CleanAndExpandPath(indexFile) var err error - payload, err = ioutil.ReadFile(fp) + payload, err := ioutil.ReadFile(fp) if err != nil { return fmt.Errorf("ReadFile %v: %v", fp, err) } - } - - files := []pi.File{ - { + file = &pi.File{ Name: v1.PolicyIndexFilename, MIME: mime.DetectMimeType(payload), Digest: hex.EncodeToString(util.Digest(payload)), Payload: base64.StdEncoding.EncodeToString(payload), - }, + } + } + files := []pi.File{ + *file, } // Prepare attachment files diff --git a/politeiawww/cmd/piwww/proposalnew.go b/politeiawww/cmd/piwww/proposalnew.go index 6920e0656..9551ea5fa 100644 --- a/politeiawww/cmd/piwww/proposalnew.go +++ b/politeiawww/cmd/piwww/proposalnew.go @@ -5,7 +5,6 @@ package main import ( - "bytes" "encoding/base64" "encoding/hex" "encoding/json" @@ -75,35 +74,33 @@ func (cmd *proposalNewCmd) Execute(args []string) error { } // Prepare index file - var payload []byte + var ( + file *pi.File + err error + ) if cmd.Random { // Generate random text for the index file - var b bytes.Buffer - for i := 0; i < 10; i++ { - r, err := util.Random(32) - if err != nil { - return err - } - b.WriteString(base64.StdEncoding.EncodeToString(r) + "\n") + file, err = createMDFile() + if err != nil { + return err } - payload = b.Bytes() } else { // Read the index file from disk fp := util.CleanAndExpandPath(indexFile) var err error - payload, err = ioutil.ReadFile(fp) + payload, err := ioutil.ReadFile(fp) if err != nil { return fmt.Errorf("ReadFile %v: %v", fp, err) } - } - - files := []pi.File{ - { + file = &pi.File{ Name: v1.PolicyIndexFilename, MIME: mime.DetectMimeType(payload), Digest: hex.EncodeToString(util.Digest(payload)), Payload: base64.StdEncoding.EncodeToString(payload), - }, + } + } + files := []pi.File{ + *file, } // Prepare attachment files diff --git a/politeiawww/cmd/piwww/proposalstatusset.go b/politeiawww/cmd/piwww/proposalstatusset.go index dc18722c2..17a88d1be 100644 --- a/politeiawww/cmd/piwww/proposalstatusset.go +++ b/politeiawww/cmd/piwww/proposalstatusset.go @@ -31,6 +31,9 @@ func (cmd *proposalStatusSetCmd) Execute(args []string) error { "public": pi.PropStatusPublic, "censored": pi.PropStatusCensored, "abandoned": pi.PropStatusAbandoned, + "2": pi.PropStatusPublic, + "3": pi.PropStatusCensored, + "4": pi.PropStatusAbandoned, } // Verify state. Defaults to vetted if the --unvetted flag diff --git a/politeiawww/cmd/piwww/testrun.go b/politeiawww/cmd/piwww/testrun.go index 150c2c810..da0852e13 100644 --- a/politeiawww/cmd/piwww/testrun.go +++ b/politeiawww/cmd/piwww/testrun.go @@ -6,11 +6,15 @@ package main import ( "context" + "encoding/base64" "encoding/hex" + "encoding/json" "fmt" + "strconv" "time" "github.com/decred/politeia/politeiad/api/v1/identity" + pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/cmd/shared" "github.com/decred/politeia/util" @@ -24,13 +28,20 @@ type testRunCmd struct { } `positional-args:"true" required:"true"` } +var ( + minPasswordLength int + publicKey string +) + // testUser stores user details that are used throughout the test run. type testUser struct { - ID string // UUID - Email string // Email - Username string // Username - Password string // Password (not hashed) - PublicKey string // Public key of active identity + ID string // UUID + Email string // Email + Username string // Username + Password string // Password (not hashed) + PublicKey string // Public key of active identity + PaywallAddress string // Paywall address + PaywallAmount uint64 // Paywall amount } // login logs in the specified user. @@ -71,13 +82,13 @@ func randomString(length int) (string, error) { } // userNew creates a new user and returnes user's public key. -func userNew(email, password, username string) (*identity.FullIdentity, error) { +func userNew(email, password, username string) (*identity.FullIdentity, string, error) { fmt.Printf(" Creating user: %v\n", email) // Create user identity and save it to disk id, err := shared.NewIdentity() if err != nil { - return nil, err + return nil, "", err } // Setup new user request @@ -87,12 +98,12 @@ func userNew(email, password, username string) (*identity.FullIdentity, error) { Password: shared.DigestSHA3(password), PublicKey: hex.EncodeToString(id.Public.Key[:]), } - _, err = client.NewUser(nu) + nur, err := client.NewUser(nu) if err != nil { - return nil, err + return nil, "", err } - return id, nil + return id, nur.VerificationToken, nil } // userManage sends a usermanage command @@ -108,8 +119,46 @@ func userManage(userID, action, reason string) error { return nil } +// userEmailVerify verifies user's email +func userEmailVerify(vt, email string, id *identity.FullIdentity) error { + fmt.Printf(" Verify user's email\n") + sig := id.SignMessage([]byte(vt)) + _, err := client.VerifyNewUser( + &www.VerifyNewUser{ + Email: email, + VerificationToken: vt, + Signature: hex.EncodeToString(sig[:]), + }) + if err != nil { + return err + } + return nil +} + +// userCreate creates new user & returns the created testUser +func userCreate() (*testUser, *identity.FullIdentity, string, error) { + // Create user and verify email + randomStr, err := randomString(minPasswordLength) + if err != nil { + return nil, nil, "", err + } + email := randomStr + "@example.com" + username := randomStr + password := randomStr + id, vt, err := userNew(email, password, username) + if err != nil { + return nil, nil, "", err + } + + return &testUser{ + Email: email, + Username: username, + Password: password, + }, id, vt, nil +} + // testUser tests piwww user specific routes. -func testUserRoutes(admin testUser, minPasswordLength int) error { +func testUserRoutes(admin testUser) error { // sleepInterval is the time to wait in between requests // when polling politeiawww for paywall tx confirmations. const sleepInterval = 15 * time.Second @@ -124,21 +173,14 @@ func testUserRoutes(admin testUser, minPasswordLength int) error { // purchased using the testnet faucet. numCredits = 1 - // Test users - user testUser + // Test user + user *testUser ) // Run user routes. fmt.Printf("Running user routes\n") - // Create user and verify email - randomStr, err := randomString(minPasswordLength) - if err != nil { - return err - } - email := randomStr + "@example.com" - username := randomStr - password := randomStr - id, err := userNew(email, password, username) + // Create new user + user, id, _, err := userCreate() if err != nil { return err } @@ -147,22 +189,14 @@ func testUserRoutes(admin testUser, minPasswordLength int) error { fmt.Printf(" Resend email Verification\n") rvr, err := client.ResendVerification(www.ResendVerification{ PublicKey: hex.EncodeToString(id.Public.Key[:]), - Email: email, + Email: user.Email, }) if err != nil { return err } // Verify email - fmt.Printf(" Verify user's email\n") - vt := rvr.VerificationToken - sig := id.SignMessage([]byte(vt)) - _, err = client.VerifyNewUser( - &www.VerifyNewUser{ - Email: email, - VerificationToken: vt, - Signature: hex.EncodeToString(sig[:]), - }) + err = userEmailVerify(rvr.VerificationToken, user.Email, id) if err != nil { return err } @@ -170,20 +204,17 @@ func testUserRoutes(admin testUser, minPasswordLength int) error { // Login and store user details fmt.Printf(" Login user\n") lr, err := client.Login(&www.Login{ - Email: email, - Password: shared.DigestSHA3(password), + Email: user.Email, + Password: shared.DigestSHA3(user.Password), }) if err != nil { return err } - user = testUser{ - ID: lr.UserID, - Email: email, - Username: username, - Password: password, - PublicKey: lr.PublicKey, - } + user.PublicKey = lr.PublicKey + user.PaywallAddress = lr.PaywallAddress + user.ID = lr.UserID + user.PaywallAmount = lr.PaywallAmount // Logout user fmt.Printf(" Logout user\n") @@ -192,8 +223,14 @@ func testUserRoutes(admin testUser, minPasswordLength int) error { return err } + // Update user key + err = userKeyUpdate(*user) + if err != nil { + return err + } + // Log back in - err = login(user) + err = login(*user) if err != nil { return err } @@ -216,17 +253,9 @@ func testUserRoutes(admin testUser, minPasswordLength int) error { return err } - // Update user key - fmt.Printf(" Update user key\n") - ukuc := shared.UserKeyUpdateCmd{} - err = ukuc.Execute(nil) - if err != nil { - return err - } - // Change username fmt.Printf(" Change username\n") - randomStr, err = randomString(minPasswordLength) + randomStr, err := randomString(minPasswordLength) if err != nil { return err } @@ -268,13 +297,14 @@ func testUserRoutes(admin testUser, minPasswordLength int) error { user.Password = randomStr // Login with new password - err = login(user) + err = login(*user) if err != nil { return err } + // Check if paywall is enabled. Paywall address and paywall // amount will be zero values if paywall has been disabled. - if lr.PaywallAddress != "" && lr.PaywallAmount != 0 { + if user.PaywallAddress != "" && user.PaywallAmount != 0 { paywallEnabled = true } else { fmt.Printf("WARNING: politeiawww paywall is disabled\n") @@ -285,14 +315,14 @@ func testUserRoutes(admin testUser, minPasswordLength int) error { // Pay user registration fee fmt.Printf(" Paying user registration fee\n") txID, err := util.PayWithTestnetFaucet(context.Background(), - cfg.FaucetHost, lr.PaywallAddress, lr.PaywallAmount, "") + cfg.FaucetHost, user.PaywallAddress, user.PaywallAmount, "") if err != nil { return err } - dcr := float64(lr.PaywallAmount) / 1e8 + dcr := float64(user.PaywallAmount) / 1e8 fmt.Printf(" Paid %v DCR to %v with txID %v\n", - dcr, lr.PaywallAddress, txID) + dcr, user.PaywallAddress, txID) } // Wait for user registration payment confirmations @@ -333,7 +363,7 @@ func testUserRoutes(admin testUser, minPasswordLength int) error { } fmt.Printf(" Paid %v DCR to %v with txID %v\n", - float64(atoms)/1e8, lr.PaywallAddress, txID) + float64(atoms)/1e8, user.PaywallAddress, txID) } // Keep track of when the pending proposal credit payment @@ -446,15 +476,526 @@ func testUserRoutes(admin testUser, minPasswordLength int) error { return nil } -// Execute executes the test run command. -func (cmd *testRunCmd) Execute(args []string) error { +// proposalNewNormal is a wrapper func which creates a proposal by calling +// proposalNew +func proposalNewNormal() (*pi.ProposalNew, error) { + return proposalNew(false, "") +} - const ( - // Comment actions - commentActionUpvote = "upvote" - commentActionDownvote = "downvote" - ) +// proposalNew returns a NewProposal object contains randonly generated +// markdown text and a signature from the logged in user. If given `rfp` bool +// is true it creates an RFP. If given `linkto` it creates a RFP submission. +func proposalNew(rfp bool, linkto string) (*pi.ProposalNew, error) { + md, err := createMDFile() + if err != nil { + return nil, fmt.Errorf("create MD file: %v", err) + } + files := []pi.File{*md} + + pm := www.ProposalMetadata{ + Name: "Some proposal name", + } + if rfp { + pm.LinkBy = time.Now().Add(time.Hour * 24 * 30).Unix() + } + if linkto != "" { + pm.LinkTo = linkto + } + pmb, err := json.Marshal(pm) + if err != nil { + return nil, err + } + metadata := []pi.Metadata{ + { + Digest: hex.EncodeToString(util.Digest(pmb)), + Hint: pi.HintProposalMetadata, + Payload: base64.StdEncoding.EncodeToString(pmb), + }, + } + + sig, err := signedMerkleRoot(files, metadata, cfg.Identity) + if err != nil { + return nil, fmt.Errorf("sign merkle root: %v", err) + } + + return &pi.ProposalNew{ + Files: files, + Metadata: metadata, + PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), + Signature: sig, + }, nil +} + +// submitNewPropsal submits new proposal and verifies it +// +// This function returns with the user logged out +func submitNewProposal(user testUser) (string, error) { + // Login user + err := login(user) + if err != nil { + return "", err + } + + fmt.Printf(" New proposal\n") + pn, err := proposalNewNormal() + if err != nil { + return "", err + } + pnr, err := client.ProposalNew(*pn) + if err != nil { + return "", err + } + + // Verify proposal censorship record + pr := &pi.ProposalRecord{ + Files: pn.Files, + Metadata: pn.Metadata, + PublicKey: pn.PublicKey, + Signature: pn.Signature, + CensorshipRecord: pnr.Proposal.CensorshipRecord, + } + err = verifyProposal(*pr, publicKey) + if err != nil { + return "", fmt.Errorf("verify proposal failed: %v", err) + } + + token := pr.CensorshipRecord.Token + fmt.Printf(" Proposal submitted: %v\n", token) + + // Logout + err = logout() + if err != nil { + return "", err + } + + return token, nil +} + +// proposalSetStatus calls proposal set status command +// +// This function returns with user logged out +func proposalSetStatus(user testUser, state pi.PropStateT, token, reason string, status pi.PropStatusT) error { + // Login user + err := login(user) + if err != nil { + return err + } + + pssc := proposalStatusSetCmd{ + Unvetted: state == pi.PropStateUnvetted, + } + pssc.Args.Token = token + pssc.Args.Status = strconv.Itoa(int(status)) + pssc.Args.Reason = reason + err = pssc.Execute(nil) + if err != nil { + return err + } + + return logout() +} + +// proposalCensor censors given proposal +// +// This function returns with user logged out +func proposalCensor(user testUser, state pi.PropStateT, token, reason string) error { + err := proposalSetStatus(user, state, token, reason, pi.PropStatusCensored) + if err != nil { + return err + } + return nil +} + +// proposalPublic makes given proposal public +// +// This function returns with user logged out +func proposalPublic(user testUser, token string) error { + err := proposalSetStatus(user, pi.PropStateUnvetted, token, "", pi.PropStatusPublic) + if err != nil { + return err + } + return nil +} + +// proposalAbandon abandons given proposal +// +// This function returns with user logged out +func proposalAbandon(user testUser, token, reason string) error { + err := proposalSetStatus(user, pi.PropStateVetted, token, reason, + pi.PropStatusAbandoned) + if err != nil { + return err + } + return nil +} + +// proposalEdit edits given proposal +// +// This function returns with user logged out +func proposalEdit(user testUser, state pi.PropStateT, token string) error { + // Login user + err := login(user) + if err != nil { + return err + } + + epc := proposalEditCmd{ + Random: true, + Unvetted: state == pi.PropStateUnvetted, + } + epc.Args.Token = token + err = epc.Execute(nil) + if err != nil { + return err + } + + // Logout + return logout() +} + +// proposals fetchs requested proposals and verifies returned map length +// +// This function returns with user logged out +func proposals(user testUser, ps pi.Proposals) (map[string]pi.ProposalRecord, error) { + // Login user + err := login(user) + if err != nil { + return nil, err + } + psr, err := client.Proposals(ps) + if err != nil { + return nil, err + } + + if len(psr.Proposals) != len(ps.Requests) { + return nil, fmt.Errorf("Received wrong number of proposals: want %v,"+ + " got %v", len(ps.Requests), len(psr.Proposals)) + } + // Logout + err = logout() + if err != nil { + return nil, err + } + + return psr.Proposals, nil +} + +// userKeyUpdate updates user's key +// +// This function returns with the user logged out +func userKeyUpdate(user testUser) error { + // Login user + err := login(user) + if err != nil { + return err + } + + fmt.Printf(" Update user key\n") + ukuc := shared.UserKeyUpdateCmd{} + err = ukuc.Execute(nil) + if err != nil { + return err + } + + return logout() +} + +// testProposalRoutes tests the propsal routes +func testProposalRoutes(admin testUser) error { + // Run proposal routes. + fmt.Printf("Running proposal routes\n") + + // Create test user + fmt.Printf("Creating test user\n") + user, id, vt, err := userCreate() + if err != nil { + return err + } + + // Verify email + err = userEmailVerify(vt, user.Email, id) + if err != nil { + return err + } + + // Update user key + err = userKeyUpdate(*user) + if err != nil { + return err + } + + // Submit new proposal + censoredToken1, err := submitNewProposal(*user) + if err != nil { + return err + } + + // Edit unvetted proposal + fmt.Printf(" Edit unvetted proposal\n") + err = proposalEdit(*user, pi.PropStateUnvetted, censoredToken1) + if err != nil { + return err + } + + // Censor unvetted proposal + fmt.Printf(" Censor unvetted proposal\n") + const reason = "because!" + err = proposalCensor(admin, pi.PropStateUnvetted, censoredToken1, reason) + if err != nil { + return err + } + + // Submit new proposal + censoredToken2, err := submitNewProposal(*user) + if err != nil { + return err + } + + // Make the proposal public + fmt.Printf(" Set proposal status: public\n") + err = proposalPublic(admin, censoredToken2) + if err != nil { + return err + } + + // Edit vetted proposal + fmt.Printf(" Edit vetted proposal\n") + err = proposalEdit(*user, pi.PropStateVetted, censoredToken2) + if err != nil { + return err + } + + // Censor public proposal + fmt.Printf(" Censor public proposal\n") + err = proposalCensor(admin, pi.PropStateVetted, censoredToken2, reason) + if err != nil { + return err + } + + // Submit new proposal + abandonedToken, err := submitNewProposal(*user) + if err != nil { + return err + } + + // Make the proposal public + fmt.Printf(" Set proposal status: public\n") + err = proposalPublic(admin, abandonedToken) + if err != nil { + return err + } + + // Abandon public proposal + fmt.Printf(" Abandon proposal\n") + err = proposalAbandon(admin, abandonedToken, reason) + if err != nil { + return err + } + + // Submit new proposal and leave it unvetted + unvettedToken, err := submitNewProposal(*user) + if err != nil { + return err + } + + // Submit new proposal and make it public + publicToken, err := submitNewProposal(*user) + if err != nil { + return err + } + + // Make the proposal public + fmt.Printf(" Set proposal status: public\n") + err = proposalPublic(admin, publicToken) + if err != nil { + return err + } + + // Login admin + err = login(admin) + if err != nil { + return err + } + + // Proposal inventory + var publicExists, censoredExists, abandonedExists, unvettedExists bool + fmt.Printf(" Proposal inventory\n") + pir, err := client.ProposalInventory(pi.ProposalInventory{}) + if err != nil { + return err + } + // Vetted proposals map + vettedProps := pir.Vetted + + // Ensure public proposal token received + publicProps, ok := vettedProps[pi.PropStatus[pi.PropStatusPublic]] + if !ok { + return fmt.Errorf("No public proposals returned") + } + for _, t := range publicProps { + if t == publicToken { + publicExists = true + } + } + if !publicExists { + return fmt.Errorf("Proposal inventory missing public proposal: %v", + publicToken) + } + + // Ensure vetted censored proposal token received + vettedCensored, ok := vettedProps[pi.PropStatus[pi.PropStatusCensored]] + if !ok { + return fmt.Errorf("No vetted censrored proposals returned") + } + for _, t := range vettedCensored { + if t == censoredToken2 { + censoredExists = true + } + } + if !censoredExists { + return fmt.Errorf("Proposal inventory missing vetted censored proposal"+ + ": %v", + censoredToken1) + } + + // Ensure abandoned proposal token received + abandonedProps, ok := vettedProps[pi.PropStatus[pi.PropStatusAbandoned]] + if !ok { + return fmt.Errorf("No abandoned proposals returned") + } + for _, t := range abandonedProps { + if t == abandonedToken { + abandonedExists = true + } + } + if !abandonedExists { + return fmt.Errorf("Proposal inventory missing abandoned proposal: %v", + abandonedToken) + } + + // Unvetted propsoals + unvettedProps := pir.Unvetted + + // Ensure unvetted proposal token received + unreviewedProps, ok := unvettedProps[pi.PropStatus[pi.PropStatusUnreviewed]] + if !ok { + return fmt.Errorf("No unreviewed proposals returned") + } + for _, t := range unreviewedProps { + if t == unvettedToken { + unvettedExists = true + } + } + if !unvettedExists { + return fmt.Errorf("Proposal inventory missing unvetted proposal: %v", + unvettedToken) + } + + // Ensure unvetted censored proposal token received + unvettedCensored, ok := unvettedProps["censored"] + if !ok { + return fmt.Errorf("No unvetted censrored proposals returned") + } + for _, t := range unvettedCensored { + if t == censoredToken1 { + censoredExists = true + } + } + if !censoredExists { + return fmt.Errorf("Proposal inventory missing unvetted censored proposal"+ + ": %v", + censoredToken1) + } + + // Get vetted proposals + fmt.Printf(" Fetch vetted proposals\n") + props, err := proposals(*user, pi.Proposals{ + State: pi.PropStateVetted, + Requests: []pi.ProposalRequest{ + { + Token: publicToken, + }, + { + Token: abandonedToken, + }, + }, + }) + if err != nil { + return err + } + _, publicExists = props[publicToken] + _, abandonedExists = props[abandonedToken] + if !publicExists || !abandonedExists { + return fmt.Errorf("Proposal batch missing requested vetted proposals") + } + + // Get vetted proposals with short tokens + fmt.Printf(" Fetch vetted proposals with short tokens\n") + shortPublicToken := publicToken[0:7] + shortAbandonedToken := abandonedToken[0:7] + props, err = proposals(*user, pi.Proposals{ + State: pi.PropStateVetted, + Requests: []pi.ProposalRequest{ + { + Token: shortPublicToken, + }, + { + Token: shortAbandonedToken, + }, + }, + }) + if err != nil { + return err + } + _, publicExists = props[publicToken] + _, abandonedExists = props[abandonedToken] + if !publicExists || !abandonedExists { + return fmt.Errorf("Proposal batch missing requested vetted proposals") + } + + // Get unvetted proposal + fmt.Printf(" Fetch unvetted proposal\n") + props, err = proposals(*user, pi.Proposals{ + State: pi.PropStateUnvetted, + Requests: []pi.ProposalRequest{ + { + Token: unvettedToken, + }, + }, + }) + if err != nil { + return err + } + _, unvettedExists = props[unvettedToken] + if !unvettedExists { + return fmt.Errorf("Proposal batch missing requested unvetted proposals") + } + + // Get unvetted proposal with short token + fmt.Printf(" Fetch unvetted proposal with short token\n") + shortUnvettedToken := unvettedToken[0:7] + props, err = proposals(*user, pi.Proposals{ + State: pi.PropStateUnvetted, + Requests: []pi.ProposalRequest{ + { + Token: shortUnvettedToken, + }, + }, + }) + if err != nil { + return err + } + _, unvettedExists = props[unvettedToken] + if !unvettedExists { + return fmt.Errorf("Proposal batch missing requested unvetted proposals") + } + + return nil +} + +// Execute executes the test run command. +func (cmd *testRunCmd) Execute(args []string) error { // Suppress output from cli commands cfg.Silent = true @@ -466,6 +1007,7 @@ func (cmd *testRunCmd) Execute(args []string) error { if err != nil { return err } + minPasswordLength = int(policy.MinPasswordLength) // Version (CSRF tokens) fmt.Printf(" Version\n") @@ -473,6 +1015,7 @@ func (cmd *testRunCmd) Execute(args []string) error { if err != nil { return err } + publicKey = version.PubKey // We only allow this to be run on testnet for right now. // Running it on mainnet would require changing the user @@ -514,11 +1057,18 @@ func (cmd *testRunCmd) Execute(args []string) error { } // Test user routes - err = testUserRoutes(admin, int(policy.MinPasswordLength)) + err = testUserRoutes(admin) + if err != nil { + return err + } + + // Test proposal routes + err = testProposalRoutes(admin) if err != nil { return err } + fmt.Printf("Test run successful!\n") return nil } diff --git a/politeiawww/cmd/piwww/util.go b/politeiawww/cmd/piwww/util.go index eb14e48de..8bdd355f1 100644 --- a/politeiawww/cmd/piwww/util.go +++ b/politeiawww/cmd/piwww/util.go @@ -13,7 +13,9 @@ import ( "github.com/decred/dcrd/chaincfg/chainhash" "github.com/decred/politeia/politeiad/api/v1/identity" + "github.com/decred/politeia/politeiad/api/v1/mime" pi "github.com/decred/politeia/politeiawww/api/pi/v1" + v1 "github.com/decred/politeia/politeiawww/api/www/v1" utilwww "github.com/decred/politeia/politeiawww/util" "github.com/decred/politeia/util" ) @@ -72,6 +74,28 @@ func proposalRecord(state pi.PropStateT, token, version string) (*pi.ProposalRec return &pr, nil } +// createMDFile returns a File object that was created using a markdown file +// filled with random text. +func createMDFile() (*pi.File, error) { + var b bytes.Buffer + b.WriteString("This is the proposal title\n") + + for i := 0; i < 10; i++ { + r, err := util.Random(32) + if err != nil { + return nil, err + } + b.WriteString(base64.StdEncoding.EncodeToString(r) + "\n") + } + + return &pi.File{ + Name: v1.PolicyIndexFilename, + MIME: mime.DetectMimeType(b.Bytes()), + Digest: hex.EncodeToString(util.Digest(b.Bytes())), + Payload: base64.StdEncoding.EncodeToString(b.Bytes()), + }, nil +} + // proposalRecord returns the latest ProposalRecrord version for the provided // token. func proposalRecordLatest(state pi.PropStateT, token string) (*pi.ProposalRecord, error) { diff --git a/politeiawww/cmd/piwww/voteauthorize.go b/politeiawww/cmd/piwww/voteauthorize.go index 227dd7c77..42c357b1a 100644 --- a/politeiawww/cmd/piwww/voteauthorize.go +++ b/politeiawww/cmd/piwww/voteauthorize.go @@ -51,6 +51,7 @@ func (cmd *voteAuthorizeCmd) Execute(args []string) error { if err != nil { return fmt.Errorf("proposalRecordLatest: %v", err) } + // Parse version version, err := strconv.ParseUint(pr.Version, 10, 32) if err != nil { return err From e82c1cd2bd416cbcef1991af7fb0c5bdf7246ceb Mon Sep 17 00:00:00 2001 From: Amir Massarwa Date: Mon, 7 Dec 2020 16:55:06 +0200 Subject: [PATCH 173/449] piwww: Add comment routes to testrun. --- politeiawww/cmd/piwww/testrun.go | 463 +++++++++++++++++++++++++++++-- 1 file changed, 444 insertions(+), 19 deletions(-) diff --git a/politeiawww/cmd/piwww/testrun.go b/politeiawww/cmd/piwww/testrun.go index da0852e13..a3dc3fb36 100644 --- a/politeiawww/cmd/piwww/testrun.go +++ b/politeiawww/cmd/piwww/testrun.go @@ -63,7 +63,8 @@ func logout() error { return nil } -// userRegistrationPayment ensures current logged in user has paid registration fee +// userRegistrationPayment ensures current logged in user has paid registration +// fee func userRegistrationPayment() (www.UserRegistrationPaymentReply, error) { urvr, err := client.UserRegistrationPayment() if err != nil { @@ -157,6 +158,27 @@ func userCreate() (*testUser, *identity.FullIdentity, string, error) { }, id, vt, nil } +// userDetals accepts a pointer to a testUser calls client's login command +// and stores additional information on given testUser struct +func userDetails(u *testUser) error { + // Login and store user details + fmt.Printf(" Login user\n") + lr, err := client.Login(&www.Login{ + Email: u.Email, + Password: shared.DigestSHA3(u.Password), + }) + if err != nil { + return err + } + + u.PublicKey = lr.PublicKey + u.PaywallAddress = lr.PaywallAddress + u.ID = lr.UserID + u.PaywallAmount = lr.PaywallAmount + + return nil +} + // testUser tests piwww user specific routes. func testUserRoutes(admin testUser) error { // sleepInterval is the time to wait in between requests @@ -201,21 +223,12 @@ func testUserRoutes(admin testUser) error { return err } - // Login and store user details - fmt.Printf(" Login user\n") - lr, err := client.Login(&www.Login{ - Email: user.Email, - Password: shared.DigestSHA3(user.Password), - }) + // Populate user's details + err = userDetails(user) if err != nil { return err } - user.PublicKey = lr.PublicKey - user.PaywallAddress = lr.PaywallAddress - user.ID = lr.UserID - user.PaywallAmount = lr.PaywallAmount - // Logout user fmt.Printf(" Logout user\n") err = logout() @@ -573,7 +586,7 @@ func submitNewProposal(user testUser) (string, error) { // proposalSetStatus calls proposal set status command // -// This function returns with user logged out +// This function returns with the user logged out func proposalSetStatus(user testUser, state pi.PropStateT, token, reason string, status pi.PropStatusT) error { // Login user err := login(user) @@ -597,7 +610,7 @@ func proposalSetStatus(user testUser, state pi.PropStateT, token, reason string, // proposalCensor censors given proposal // -// This function returns with user logged out +// This function returns with the user logged out func proposalCensor(user testUser, state pi.PropStateT, token, reason string) error { err := proposalSetStatus(user, state, token, reason, pi.PropStatusCensored) if err != nil { @@ -608,7 +621,7 @@ func proposalCensor(user testUser, state pi.PropStateT, token, reason string) er // proposalPublic makes given proposal public // -// This function returns with user logged out +// This function returns with the user logged out func proposalPublic(user testUser, token string) error { err := proposalSetStatus(user, pi.PropStateUnvetted, token, "", pi.PropStatusPublic) if err != nil { @@ -619,7 +632,7 @@ func proposalPublic(user testUser, token string) error { // proposalAbandon abandons given proposal // -// This function returns with user logged out +// This function returns with the user logged out func proposalAbandon(user testUser, token, reason string) error { err := proposalSetStatus(user, pi.PropStateVetted, token, reason, pi.PropStatusAbandoned) @@ -631,7 +644,7 @@ func proposalAbandon(user testUser, token, reason string) error { // proposalEdit edits given proposal // -// This function returns with user logged out +// This function returns with the user logged out func proposalEdit(user testUser, state pi.PropStateT, token string) error { // Login user err := login(user) @@ -655,7 +668,7 @@ func proposalEdit(user testUser, state pi.PropStateT, token string) error { // proposals fetchs requested proposals and verifies returned map length // -// This function returns with user logged out +// This function returns with the user logged out func proposals(user testUser, ps pi.Proposals) (map[string]pi.ProposalRecord, error) { // Login user err := login(user) @@ -707,7 +720,7 @@ func testProposalRoutes(admin testUser) error { fmt.Printf("Running proposal routes\n") // Create test user - fmt.Printf("Creating test user\n") + fmt.Printf(" Creating test user\n") user, id, vt, err := userCreate() if err != nil { return err @@ -994,6 +1007,405 @@ func testProposalRoutes(admin testUser) error { return nil } +// commentNew submits a new comment +// +// This function returns with the user logged out +func commentNew(user testUser, state pi.PropStateT, token, comment, parentID string) error { + // Login user + err := login(user) + if err != nil { + return err + } + + ncc := commentNewCmd{ + Unvetted: state == pi.PropStateUnvetted, + } + ncc.Args.Token = token + ncc.Args.Comment = comment + ncc.Args.ParentID = parentID + err = ncc.Execute(nil) + if err != nil { + return err + } + + return logout() +} + +// verifyCommentSctore accepts array of comments, a commentID and the expected +// up & down votes and ensures given comment has expected score +func verifyCommentScore(comments []pi.Comment, commentID uint32, upvotes, downvotes uint64) error { + var commentExists bool + for _, v := range comments { + if v.CommentID == commentID { + commentExists = true + switch { + case v.Upvotes != upvotes: + return fmt.Errorf("comment result up votes got %v, want %v", + v.Upvotes, upvotes) + case v.Downvotes != downvotes: + return fmt.Errorf("comment result down votes got %v, want %v", + v.Downvotes, downvotes) + } + } + } + if !commentExists { + return fmt.Errorf("comment not found: %v", commentID) + } + + return nil +} + +// verifyCommentVotes accepts comment votes array of all user's comment votes +// on a proposals and a comment id, it verifies the number of the total votes, +// the number of upvotes and the number of downvotes on given comment +func verifyCommentVotes(votes []pi.CommentVoteDetails, commentID uint32, totalVotes, upvotes, downvotes int) error { + var ( + uvotes int + dvotes int + total int + commentExists bool + ) + for _, v := range votes { + if v.CommentID == commentID { + commentExists = true + switch v.Vote { + case pi.CommentVoteDownvote: + dvotes++ + case pi.CommentVoteUpvote: + uvotes++ + } + total++ + } + } + if !commentExists { + return fmt.Errorf("comment not found: %v", commentID) + } + if total != totalVotes { + return fmt.Errorf("wrong num of comment votes got %v, want %v", + total, totalVotes) + } + if uvotes != upvotes { + return fmt.Errorf("wrong num of upvotes: got %v, want %v", + uvotes, upvotes) + } + if dvotes != downvotes { + return fmt.Errorf("wrong num of downvotes: got %v, want %v", + dvotes, downvotes) + } + + return nil +} + +// testCommentRoutes tests the comment routes +func testCommentRoutes(admin testUser) error { + // Run commment routes. + fmt.Printf("Running comment routes\n") + + // Create test user + fmt.Printf(" Creating test user\n") + user, id, vt, err := userCreate() + if err != nil { + return err + } + + // Verify email + err = userEmailVerify(vt, user.Email, id) + if err != nil { + return err + } + + // Update user key + err = userKeyUpdate(*user) + if err != nil { + return err + } + + // Populate user's info + err = userDetails(user) + if err != nil { + return err + } + + // Submit new proposal + token, err := submitNewProposal(*user) + if err != nil { + return err + } + + // Make proposal public + fmt.Printf(" Set proposal status: public\n") + err = proposalPublic(admin, token) + if err != nil { + return err + } + + // Abandon proposal + reason := "because!" + fmt.Printf(" Abandon proposal\n") + err = proposalAbandon(admin, token, reason) + if err != nil { + return err + } + + // Comment on abandoned proposal + fmt.Printf(" Ensure commenting on abandoned proposal isn't allowed\n") + comment := "this is a comment" + err = commentNew(*user, pi.PropStateVetted, token, comment, "0") + if err == nil { + return fmt.Errorf("Commented on an abandoned proposal: %v", token) + } + + // Submit new proposal + token, err = submitNewProposal(*user) + if err != nil { + return err + } + + // Censor proposal + fmt.Printf(" Censor proposal\n") + err = proposalCensor(admin, pi.PropStateUnvetted, token, reason) + if err != nil { + return err + } + + // Comment on abandoned proposal + fmt.Printf(" Ensure commenting on censored proposal isn't allowed\n") + err = commentNew(*user, pi.PropStateVetted, token, comment, "0") + if err == nil { + return fmt.Errorf("Commented on a censored proposal: %v", token) + } + + // Submit new proposal + token, err = submitNewProposal(*user) + if err != nil { + return err + } + + // Author comment on unvetted proposal + fmt.Printf(" Author comment on an unvetted proposal\n") + err = commentNew(*user, pi.PropStateUnvetted, token, comment, "0") + if err != nil { + return err + } + + // Admin comment on an unvetted proposal + fmt.Print(" Admin comment on an unvetted proposal\n") + err = commentNew(admin, pi.PropStateUnvetted, token, comment, "0") + if err != nil { + return err + } + + // Make proposal a public + fmt.Printf(" Set proposal status: public\n") + err = proposalPublic(admin, token) + if err != nil { + return err + } + + // Author comment on a public proposal + fmt.Printf(" Author comment on a public proposal\n") + err = commentNew(*user, pi.PropStateVetted, token, comment, "0") + if err != nil { + return err + } + + // Create another user to comment + fmt.Printf(" Creating another test user\n") + thirdU, id, vt, err := userCreate() + if err != nil { + return err + } + + // Verify email + err = userEmailVerify(vt, thirdU.Email, id) + if err != nil { + return err + } + + // Update user key + err = userKeyUpdate(*thirdU) + if err != nil { + return err + } + + // Another user comment on a public proposal + fmt.Print(" Another user comment on a public proposal\n") + err = commentNew(*user, pi.PropStateVetted, token, comment, "0") + if err != nil { + return err + } + + // Comment on a public proposal - reply + fmt.Printf(" Comment on a public proposal: reply\n") + reply := "this is a comment reply" + err = commentNew(*user, pi.PropStateVetted, token, reply, "1") + if err != nil { + return err + } + + // Validate comments + fmt.Printf(" Proposal details\n") + propReq := pi.ProposalRequest{ + Token: token, + } + pdr, err := client.Proposals(pi.Proposals{ + State: pi.PropStateVetted, + Requests: []pi.ProposalRequest{propReq}, + IncludeFiles: false, + }) + if err != nil { + return err + } + prop := pdr.Proposals[token] + if prop.Comments != 3 { + return fmt.Errorf("proposal num comments got %v, want 3", + prop.Comments) + } + + fmt.Printf(" Proposal comments\n") + gcr, err := client.Comments(pi.Comments{ + Token: token, + State: pi.PropStateVetted, + }) + if err != nil { + return fmt.Errorf("Comments: %v", err) + } + + if len(gcr.Comments) != 3 { + return fmt.Errorf("num comments got %v, want 3", + len(gcr.Comments)) + } + + for _, c := range gcr.Comments { + // We check the userID because userIDs are not part of + // the politeiad comment record. UserIDs are stored in + // in politeiawww and are added to the comments at the + // time of the request. This introduces the potential + // for errors. + if c.UserID != user.ID && c.CommentID == 1 { + return fmt.Errorf("comment %v has wrong userID got %v, want %v", + c.CommentID, c.UserID, user.ID) + } + } + + // Login with admin to be able to vote on user's comments + fmt.Printf(" Login admin\n") + err = login(admin) + if err != nil { + return err + } + + // Comment vote sequence + var ( + // Comment actions + commentActionUpvote = strconv.Itoa(int(pi.CommentVoteUpvote)) + commentActionDownvote = strconv.Itoa(int(pi.CommentVoteDownvote)) + ) + cvc := commentVoteCmd{} + cvc.Args.Token = token + cvc.Args.CommentID = "1" + cvc.Args.Vote = commentActionUpvote + + fmt.Printf(" Comment vote: upvote\n") + err = cvc.Execute(nil) + if err != nil { + return err + } + + // XXX workaround to prevent comment votes from having the same timestamp + time.Sleep(time.Second) + + fmt.Printf(" Comment vote: upvote\n") + err = cvc.Execute(nil) + if err != nil { + return err + } + + // XXX workaround to prevent comment votes from having the same timestamp + time.Sleep(time.Second) + + fmt.Printf(" Comment vote: upvote\n") + err = cvc.Execute(nil) + if err != nil { + return err + } + + // XXX workaround to prevent comment votes from having the same timestamp + time.Sleep(time.Second) + + fmt.Printf(" Comment vote: downvote\n") + cvc.Args.Vote = commentActionDownvote + err = cvc.Execute(nil) + if err != nil { + return err + } + + // Validate comment votes + fmt.Printf(" Fetch proposal's comments & verify first comment score\n") + gcr, err = client.Comments(pi.Comments{ + Token: token, + State: pi.PropStateVetted, + }) + if err != nil { + return err + } + + // Verify first comment score + err = verifyCommentScore(gcr.Comments, 1, 0, 1) + if err != nil { + return err + } + + // Validate comment votes using short token + fmt.Printf(" Fetch proposal's comments using short token & verify first " + + "comment score\n") + gcr, err = client.Comments(pi.Comments{ + Token: token[0:7], + State: pi.PropStateVetted, + }) + if err != nil { + return err + } + + // Verify first comment score + err = verifyCommentScore(gcr.Comments, 1, 0, 1) + if err != nil { + return err + } + + fmt.Printf(" Verify admin's comment votes\n") + cvr, err := client.CommentVotes(pi.CommentVotes{ + State: pi.PropStateVetted, + Token: token, + UserID: admin.ID, + }) + if err != nil { + return err + } + + err = verifyCommentVotes(cvr.Votes, 1, 4, 3, 1) + if err != nil { + return err + } + + fmt.Printf(" Verify admin's comment votes using short token\n") + cvr, err = client.CommentVotes(pi.CommentVotes{ + State: pi.PropStateVetted, + Token: token[0:7], + UserID: admin.ID, + }) + if err != nil { + return err + } + + err = verifyCommentVotes(cvr.Votes, 1, 4, 3, 1) + if err != nil { + return err + } + + return nil +} + // Execute executes the test run command. func (cmd *testRunCmd) Execute(args []string) error { // Suppress output from cli commands @@ -1041,6 +1453,12 @@ func (cmd *testRunCmd) Execute(args []string) error { return err } + // Populate admin's info + err = userDetails(&admin) + if err != nil { + return err + } + // Ensure admin paid registration free urpr, err := userRegistrationPayment() if err != nil { @@ -1068,7 +1486,14 @@ func (cmd *testRunCmd) Execute(args []string) error { return err } + // Test comment routes + err = testCommentRoutes(admin) + if err != nil { + return err + } + fmt.Printf("Test run successful!\n") + return nil } From d7730dc5627d059dc4b1f7e72625cea05d2f759a Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Mon, 7 Dec 2020 12:15:18 -0300 Subject: [PATCH 174/449] tlogbe: Add testing framework for the tlog backend. --- politeiad/backend/tlogbe/testing.go | 116 ++++++++ politeiad/backend/tlogbe/tlog.go | 6 +- politeiad/backend/tlogbe/tlogbe.go | 1 + politeiad/backend/tlogbe/tlogbe_test.go | 40 +++ .../tlogbe/{trillian.go => trillianclient.go} | 257 ++++++++++++++++-- 5 files changed, 397 insertions(+), 23 deletions(-) create mode 100644 politeiad/backend/tlogbe/testing.go create mode 100644 politeiad/backend/tlogbe/tlogbe_test.go rename politeiad/backend/tlogbe/{trillian.go => trillianclient.go} (65%) diff --git a/politeiad/backend/tlogbe/testing.go b/politeiad/backend/tlogbe/testing.go new file mode 100644 index 000000000..49b41b8aa --- /dev/null +++ b/politeiad/backend/tlogbe/testing.go @@ -0,0 +1,116 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlogbe + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/decred/dcrd/dcrutil/v3" + v1 "github.com/decred/dcrtime/api/v1" + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" + "github.com/google/trillian" + "github.com/google/trillian/crypto/keys" + "github.com/google/trillian/crypto/keys/der" + "github.com/google/trillian/crypto/keyspb" + "github.com/robfig/cron" +) + +var ( + defaultTestDir = dcrutil.AppDataDir("politeiadtest", false) + defaultTestDataDir = filepath.Join(defaultTestDir, "data") +) + +// newTestTClient provides a trillian client implementation used for +// testing. It implements the TClient interface, which includes all major +// tree operations used in the tlog backend. +func newTestTClient(t *testing.T) (*testTClient, error) { + // Create trillian private key + key, err := keys.NewFromSpec(&keyspb.Specification{ + Params: &keyspb.Specification_EcdsaParams{}, + }) + if err != nil { + return nil, err + } + keyDer, err := der.MarshalPrivateKey(key) + if err != nil { + return nil, err + } + + ttc := testTClient{ + trees: make(map[int64]*trillian.Tree), + leaves: make(map[int64][]*trillian.LogLeaf), + privateKey: &keyspb.PrivateKey{ + Der: keyDer, + }, + } + + return &ttc, nil +} + +// newTestTlog returns a tlog used for testing. +func newTestTlog(t *testing.T, id string) (*tlog, error) { + // Setup key-value store with test dir + fp := filepath.Join(defaultTestDataDir, id) + err := os.MkdirAll(fp, 0700) + if err != nil { + return nil, err + } + store := filesystem.New(fp) + + tclient, err := newTestTClient(t) + if err != nil { + return nil, err + } + + tlog := tlog{ + id: id, + dcrtimeHost: v1.DefaultTestnetTimeHost, + encryptionKey: nil, + trillian: tclient, + store: store, + cron: cron.New(), + } + + return &tlog, nil +} + +// newTestTlogBackend returns a tlog backend for testing. It wraps +// tlog and trillian client, providing the framework needed for +// writing tlog backend tests. +func newTestTlogBackend(t *testing.T) (*tlogBackend, error) { + tlogVetted, err := newTestTlog(t, "vetted") + if err != nil { + return nil, err + } + tlogUnvetted, err := newTestTlog(t, "unvetted") + if err != nil { + return nil, err + } + + tlogBackend := tlogBackend{ + homeDir: defaultTestDir, + dataDir: defaultTestDataDir, + unvetted: tlogUnvetted, + vetted: tlogVetted, + plugins: make(map[string]plugin), + prefixes: make(map[string][]byte), + vettedTreeIDs: make(map[string]int64), + inv: recordInventory{ + unvetted: make(map[backend.MDStatusT][]string), + vetted: make(map[backend.MDStatusT][]string), + }, + } + + err = tlogBackend.setup() + if err != nil { + return nil, fmt.Errorf("setup: %v", err) + } + + return &tlogBackend, nil +} diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index 8289b4591..f07c65f54 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -86,7 +86,7 @@ type tlog struct { id string dcrtimeHost string encryptionKey *encryptionKey - trillian *trillianClient + trillian trillianClient store store.Blob cron *cron.Cron @@ -1870,7 +1870,7 @@ func newTlog(id, homeDir, dataDir, trillianHost, trillianKeyFile, dcrtimeHost, e log.Infof("Trillian key %v: %v", id, trillianKeyFile) log.Infof("Trillian host %v: %v", id, trillianHost) - tclient, err := newTrillianClient(trillianHost, trillianKeyFile) + trillianClient, err := newTClient(trillianHost, trillianKeyFile) if err != nil { return nil, err } @@ -1880,7 +1880,7 @@ func newTlog(id, homeDir, dataDir, trillianHost, trillianKeyFile, dcrtimeHost, e id: id, dcrtimeHost: dcrtimeHost, encryptionKey: ek, - trillian: tclient, + trillian: trillianClient, store: store, cron: cron.New(), } diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 46d5fc29c..acbaaa42f 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -1779,6 +1779,7 @@ func (t *tlogBackend) Close() { t.vetted.close() } +// setup creates the tlog backend in-memory cache. func (t *tlogBackend) setup() error { log.Tracef("setup") diff --git a/politeiad/backend/tlogbe/tlogbe_test.go b/politeiad/backend/tlogbe/tlogbe_test.go new file mode 100644 index 000000000..dae9f32f3 --- /dev/null +++ b/politeiad/backend/tlogbe/tlogbe_test.go @@ -0,0 +1,40 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlogbe + +import ( + "fmt" + "testing" + + "github.com/decred/politeia/politeiad/backend" +) + +func TestNewRecord(t *testing.T) { + tlogBackend, err := newTestTlogBackend(t) + if err != nil { + fmt.Printf("Error in newTestTlogBackend %v", err) + return + } + + metadata := backend.MetadataStream{ + ID: 1, + Payload: "", + } + + file := backend.File{ + Name: "index.md", + MIME: "text/plain; charset=utf-8", + Digest: "22e88c7d6da9b73fbb515ed6a8f6d133c680527a799e3069ca7ce346d90649b2", + Payload: "bW9vCg==", + } + + rmd, err := tlogBackend.New([]backend.MetadataStream{metadata}, []backend.File{file}) + if err != nil { + fmt.Printf("Error in New %v", err) + return + } + + fmt.Println(rmd) +} diff --git a/politeiad/backend/tlogbe/trillian.go b/politeiad/backend/tlogbe/trillianclient.go similarity index 65% rename from politeiad/backend/tlogbe/trillian.go rename to politeiad/backend/tlogbe/trillianclient.go index 38c33cfee..62c5d69ec 100644 --- a/politeiad/backend/tlogbe/trillian.go +++ b/politeiad/backend/tlogbe/trillianclient.go @@ -11,6 +11,8 @@ import ( "crypto/sha256" "fmt" "io/ioutil" + "math/rand" + "sync" "time" "github.com/decred/politeia/util" @@ -32,11 +34,38 @@ import ( "google.golang.org/grpc/status" ) -// trillianClient provides a client that abstracts over the existing -// TrillianLogClient and TrillianAdminClient. This provides a simplified API -// for the backend to use and ensures that proper verification of all trillian -// responses is performed. -type trillianClient struct { +var ( + _ trillianClient = (*tClient)(nil) + _ trillianClient = (*testTClient)(nil) +) + +// trillianClient provides an interface with basic tree operations needed for a +// trillian client. +type trillianClient interface { + tree(treeID int64) (*trillian.Tree, error) + + treesAll() ([]*trillian.Tree, error) + + treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) + + leavesAll(treeID int64) ([]*trillian.LogLeaf, error) + + leavesByRange(treeID, startIndex, count int64) ([]*trillian.LogLeaf, error) + + leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, + *types.LogRootV1, error) + + signedLogRootForTree(tree *trillian.Tree) (*trillian.SignedLogRoot, + *types.LogRootV1, error) + + close() +} + +// tClient implements the TrillianClient interface and provides a client that +// abstracts over the existing TrillianLogClient and TrillianAdminClient. This +// provides a simplified API for the backend to use and ensures that proper +// verification of all trillian responses is performed. +type tClient struct { host string grpc *grpc.ClientConn client trillian.TrillianLogClient @@ -80,7 +109,9 @@ func logLeafNew(value []byte, extraData []byte) *trillian.LogLeaf { // treeNew returns a new trillian tree and verifies that the signatures are // correct. It returns the tree and the signed log root which can be externally // verified. -func (t *trillianClient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { +// +// This function satisfies the TrillianClient interface. +func (t *tClient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { log.Tracef("trillian treeNew") pk, err := ptypes.MarshalAny(t.privateKey) @@ -145,7 +176,7 @@ func (t *trillianClient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, err return tree, ilr.Created, nil } -func (t *trillianClient) treeFreeze(treeID int64) (*trillian.Tree, error) { +func (t *tClient) treeFreeze(treeID int64) (*trillian.Tree, error) { log.Tracef("trillian treeFreeze: %v", treeID) // Get the current tree @@ -171,7 +202,10 @@ func (t *trillianClient) treeFreeze(treeID int64) (*trillian.Tree, error) { return updated, nil } -func (t *trillianClient) tree(treeID int64) (*trillian.Tree, error) { +// tree returns a trillian tree by its ID. +// +// This function satisfies the TrillianClient interface. +func (t *tClient) tree(treeID int64) (*trillian.Tree, error) { log.Tracef("trillian tree: %v", treeID) tree, err := t.admin.GetTree(t.ctx, &trillian.GetTreeRequest{ @@ -189,7 +223,10 @@ func (t *trillianClient) tree(treeID int64) (*trillian.Tree, error) { return tree, nil } -func (t *trillianClient) treesAll() ([]*trillian.Tree, error) { +// treesAll returns all trillian trees stored in the backend. +// +// This function satisfies the TrillianClient interface +func (t *tClient) treesAll() ([]*trillian.Tree, error) { log.Tracef("trillian treesAll") ltr, err := t.admin.ListTrees(t.ctx, &trillian.ListTreesRequest{}) @@ -200,7 +237,7 @@ func (t *trillianClient) treesAll() ([]*trillian.Tree, error) { return ltr.Tree, nil } -func (t *trillianClient) inclusionProof(treeID int64, merkleLeafHash []byte, lrv1 *types.LogRootV1) (*trillian.Proof, error) { +func (t *tClient) inclusionProof(treeID int64, merkleLeafHash []byte, lrv1 *types.LogRootV1) (*trillian.Proof, error) { log.Tracef("tillian inclusionProof: %v %x", treeID, merkleLeafHash) resp, err := t.client.GetInclusionProofByHash(t.ctx, @@ -232,7 +269,10 @@ func (t *trillianClient) inclusionProof(treeID int64, merkleLeafHash []byte, lrv return proof, nil } -func (t *trillianClient) signedLogRootForTree(tree *trillian.Tree) (*trillian.SignedLogRoot, *types.LogRootV1, error) { +// signedLogRootForTree returns the signed log root of a trillian tree. +// +// This function satisfies the TrillianClient interface. +func (t *tClient) signedLogRootForTree(tree *trillian.Tree) (*trillian.SignedLogRoot, *types.LogRootV1, error) { // Get the signed log root for the current tree height resp, err := t.client.GetLatestSignedLogRoot(t.ctx, &trillian.GetLatestSignedLogRootRequest{LogId: tree.TreeId}) @@ -256,7 +296,7 @@ func (t *trillianClient) signedLogRootForTree(tree *trillian.Tree) (*trillian.Si // signedLogRoot returns the signed log root for the provided tree ID at its // current height. The log root is structure is decoded an returned as well. -func (t *trillianClient) signedLogRoot(treeID int64) (*trillian.SignedLogRoot, *types.LogRootV1, error) { +func (t *tClient) signedLogRoot(treeID int64) (*trillian.SignedLogRoot, *types.LogRootV1, error) { log.Tracef("trillian signedLogRoot: %v", treeID) tree, err := t.tree(treeID) @@ -278,13 +318,15 @@ func (t *trillianClient) signedLogRoot(treeID int64) (*trillian.SignedLogRoot, * // fail to be appended. Note leaves that are duplicates will fail and it is the // callers responsibility to determine how they should be handled. // -// Trillain DOES NOT guarantee that the leaves of a queued leaves batch are +// Trillian DOES NOT guarantee that the leaves of a queued leaves batch are // appended in the order in which they were received. Trillian is also not // consistent about the order that leaves are appended in. At the time of // writing this I have not looked into why this is or if there are other // methods that can be used. DO NOT rely on the leaves being in a specific // order. -func (t *trillianClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, *types.LogRootV1, error) { +// +// This function satisfies the TrillianClient interface. +func (t *tClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, *types.LogRootV1, error) { log.Tracef("trillian leavesAppend: %v", treeID) // Get the latest signed log root @@ -393,7 +435,11 @@ func (t *trillianClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) return proofs, lr, nil } -func (t *trillianClient) leavesByRange(treeID int64, startIndex, count int64) ([]*trillian.LogLeaf, error) { +// leavesByRange returns the log leaves of a trillian tree by the range provided +// by the user. +// +// This function satisfies the TrillianClient interface. +func (t *tClient) leavesByRange(treeID int64, startIndex, count int64) ([]*trillian.LogLeaf, error) { log.Tracef("trillian leavesByRange: %v %v %v", treeID, startIndex, count) glbrr, err := t.client.GetLeavesByRange(t.ctx, @@ -410,7 +456,9 @@ func (t *trillianClient) leavesByRange(treeID int64, startIndex, count int64) ([ } // leavesAll returns all of the leaves for the provided treeID. -func (t *trillianClient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { +// +// This function satisfies the TrillianClient interface. +func (t *tClient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { log.Tracef("trillian leavesAll: %v", treeID) // Get log root @@ -429,7 +477,7 @@ func (t *trillianClient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { // leafProofs returns the leafProofs for the provided treeID and merkle leaf // hashes. The inclusion proof returned in the leafProof is for the tree height // specified by the provided LogRootV1. -func (t *trillianClient) leafProofs(treeID int64, merkleLeafHashes [][]byte, lr *types.LogRootV1) ([]leafProof, error) { +func (t *tClient) leafProofs(treeID int64, merkleLeafHashes [][]byte, lr *types.LogRootV1) ([]leafProof, error) { log.Tracef("trillian leafProofs: %v %v %x", treeID, lr.TreeSize, merkleLeafHashes) @@ -460,13 +508,182 @@ func (t *trillianClient) leafProofs(treeID int64, merkleLeafHashes [][]byte, lr } // close closes the trillian grpc connection. -func (t *trillianClient) close() { +// +// This function satisfies the TrillianClient interface. +func (t *tClient) close() { log.Tracef("trillian close %v", t.host) t.grpc.Close() } -func newTrillianClient(host, keyFile string) (*trillianClient, error) { +// testTClient implements TClient interface and is used for +// testing purposes. +type testTClient struct { + sync.RWMutex + + trees map[int64]*trillian.Tree // [treeID]Tree + leaves map[int64][]*trillian.LogLeaf // [treeID][]LogLeaf + + privateKey *keyspb.PrivateKey +} + +// tree returns trillian tree from passed in ID. +// +// This function satisfies the TrillianClient interface. +func (t *testTClient) tree(treeID int64) (*trillian.Tree, error) { + t.RLock() + defer t.RUnlock() + + if tree, ok := t.trees[treeID]; ok { + return tree, nil + } + + return nil, fmt.Errorf("Tree ID not found") +} + +// treesAll signed log roots are not used for testing up until now, so we +// return a nil value for it. +// +// This function satisfies the TrillianClient interface. +func (t *testTClient) treesAll() ([]*trillian.Tree, error) { + t.RLock() + defer t.RUnlock() + + trees := make([]*trillian.Tree, len(t.trees)) + for _, t := range t.trees { + trees = append(trees, t) + } + + return trees, nil +} + +// treeNew ceates a new trillian tree in memory. +// +// This function satisfies the TrillianClient interface. +func (t *testTClient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { + t.Lock() + defer t.Unlock() + + // Retrieve private key + pk, err := ptypes.MarshalAny(t.privateKey) + if err != nil { + return nil, nil, err + } + + // Create trillian tree + tree := trillian.Tree{ + TreeId: rand.Int63(), + TreeState: trillian.TreeState_ACTIVE, + TreeType: trillian.TreeType_LOG, + HashStrategy: trillian.HashStrategy_RFC6962_SHA256, + HashAlgorithm: sigpb.DigitallySigned_SHA256, + SignatureAlgorithm: sigpb.DigitallySigned_ECDSA, + DisplayName: "", + Description: "", + MaxRootDuration: ptypes.DurationProto(0), + PrivateKey: pk, + } + t.trees[tree.TreeId] = &tree + + // Initialize leaves map for that tree + t.leaves[tree.TreeId] = []*trillian.LogLeaf{} + + return &tree, nil, nil +} + +// leavesAll returns all leaves from a trillian tree. +// +// This function satisfies the TrillianClient interface. +func (t *testTClient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { + t.RLock() + defer t.RUnlock() + + // Check if treeID entry exists + if _, ok := t.leaves[treeID]; !ok { + return nil, fmt.Errorf("Tree ID %d does not contain any leaf data", + treeID) + } + + return t.leaves[treeID], nil +} + +// leavesByRange returns leaves in range according to the passed in parameters. +// +// This function satisfies the TrillianClient interface. +func (t *testTClient) leavesByRange(treeID, startIndex, count int64) ([]*trillian.LogLeaf, error) { + t.RLock() + defer t.RUnlock() + + // Check if treeID entry exists + if _, ok := t.leaves[treeID]; !ok { + return nil, fmt.Errorf("Tree ID %d does not contain any leaf data", + treeID) + } + + // Get leaves by range. Indexes are ordered. + var c int64 + var leaves []*trillian.LogLeaf + for _, leaf := range t.leaves[treeID] { + if leaf.LeafIndex >= startIndex && c < count { + leaves = append(leaves, leaf) + c++ + } + } + + return nil, nil +} + +// leavesAppend satisfies the TClient interface. It appends leaves to the +// corresponding trillian tree in memory. +// +// This function satisfies the TrillianClient interface. +func (t *testTClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, *types.LogRootV1, error) { + t.Lock() + defer t.Unlock() + + // Get last leaf index + var index int64 + if len(t.leaves[treeID]) > 0 { + l := len(t.leaves[treeID]) + index = t.leaves[treeID][l-1].LeafIndex + 1 + } else { + index = 0 + } + + // Set merkle hash for each leaf and append to memory. Also append the + // queued value for the leaves to be returned by the function. + var queued []queuedLeafProof + for _, l := range leaves { + l.LeafIndex = index + l.MerkleLeafHash = merkleLeafHash(l.LeafValue) + t.leaves[treeID] = append(t.leaves[treeID], l) + + queued = append(queued, queuedLeafProof{ + QueuedLeaf: &trillian.QueuedLogLeaf{ + Leaf: l, + Status: nil, + }, + Proof: nil, + }) + } + + return queued, nil, nil +} + +// signedLogRootForTree is a stub to satisfy the interface. It is not used for +// testing. +// +// This function satisfies the TrillianClient interface. +func (t *testTClient) signedLogRootForTree(tree *trillian.Tree) (*trillian.SignedLogRoot, *types.LogRootV1, error) { + return nil, nil, nil +} + +// close is a stub to satisfy the interface. It is not used for testing. +// +// This function satisfies the TrillianClient interface. +func (t *testTClient) close() {} + +func newTClient(host, keyFile string) (*tClient, error) { // Setup trillian key file if !util.FileExists(keyFile) { // Trillian key file does not exist. Create one. @@ -512,7 +729,7 @@ func newTrillianClient(host, keyFile string) (*trillianClient, error) { return nil, err } - t := trillianClient{ + t := tClient{ grpc: g, client: trillian.NewTrillianLogClient(g), admin: trillian.NewTrillianAdminClient(g), From 8a7d6196c8841723c4795d3ba3a28bd2d247ae3e Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Mon, 7 Dec 2020 12:29:14 -0300 Subject: [PATCH 175/449] tlogbe: Add record content tests. --- politeiad/backend/tlogbe/testing.go | 321 +++++++++++++++++++++++- politeiad/backend/tlogbe/tlogbe_test.go | 41 +-- 2 files changed, 343 insertions(+), 19 deletions(-) diff --git a/politeiad/backend/tlogbe/testing.go b/politeiad/backend/tlogbe/testing.go index 49b41b8aa..05d7ae916 100644 --- a/politeiad/backend/tlogbe/testing.go +++ b/politeiad/backend/tlogbe/testing.go @@ -5,15 +5,23 @@ package tlogbe import ( + "bytes" + "encoding/base64" + "encoding/hex" "fmt" + "image" + "image/jpeg" "os" "path/filepath" "testing" "github.com/decred/dcrd/dcrutil/v3" - v1 "github.com/decred/dcrtime/api/v1" + dcrtime "github.com/decred/dcrtime/api/v1" + v1 "github.com/decred/politeia/politeiad/api/v1" + "github.com/decred/politeia/politeiad/api/v1/mime" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" + "github.com/decred/politeia/util" "github.com/google/trillian" "github.com/google/trillian/crypto/keys" "github.com/google/trillian/crypto/keys/der" @@ -26,6 +34,60 @@ var ( defaultTestDataDir = filepath.Join(defaultTestDir, "data") ) +func newBackendFile(t *testing.T, fileName string) backend.File { + t.Helper() + + r, err := util.Random(64) + if err != nil { + r = []byte{0, 0, 0} // random byte data + } + + payload := hex.EncodeToString(r) + digest := hex.EncodeToString(util.Digest([]byte(payload))) + b64 := base64.StdEncoding.EncodeToString([]byte(payload)) + + return backend.File{ + Name: fileName, + MIME: mime.DetectMimeType([]byte(payload)), + Digest: digest, + Payload: b64, + } +} + +func newBackendFileJPEG(t *testing.T) backend.File { + t.Helper() + + b := new(bytes.Buffer) + img := image.NewRGBA(image.Rect(0, 0, 1000, 500)) + + err := jpeg.Encode(b, img, &jpeg.Options{}) + if err != nil { + t.Fatalf("%v", err) + } + + // Generate a random name + r, err := util.Random(8) + if err != nil { + t.Fatalf("%v", err) + } + + return backend.File{ + Name: hex.EncodeToString(r) + ".jpeg", + MIME: mime.DetectMimeType(b.Bytes()), + Digest: hex.EncodeToString(util.Digest(b.Bytes())), + Payload: base64.StdEncoding.EncodeToString(b.Bytes()), + } +} + +func newBackendMetadataStream(t *testing.T, id uint64, payload string) backend.MetadataStream { + t.Helper() + + return backend.MetadataStream{ + ID: id, + Payload: payload, + } +} + // newTestTClient provides a trillian client implementation used for // testing. It implements the TClient interface, which includes all major // tree operations used in the tlog backend. @@ -70,7 +132,7 @@ func newTestTlog(t *testing.T, id string) (*tlog, error) { tlog := tlog{ id: id, - dcrtimeHost: v1.DefaultTestnetTimeHost, + dcrtimeHost: dcrtime.DefaultTestnetTimeHost, encryptionKey: nil, trillian: tclient, store: store, @@ -114,3 +176,258 @@ func newTestTlogBackend(t *testing.T) (*tlogBackend, error) { return &tlogBackend, nil } + +// recordContentTests defines the type used to describe the content +// verification error tests. +type recordContentTest struct { + description string + metadata []backend.MetadataStream + files []backend.File + filesDel []string + err backend.ContentVerificationError +} + +// setupRecordContentTests returns the list of tests for the verifyContent +// function. These tests are used on all backend api endpoints that verify +// content. +func setupRecordContentTests(t *testing.T) []recordContentTest { + t.Helper() + + var rct []recordContentTest + + // Invalid metadata ID error + md := []backend.MetadataStream{ + newBackendMetadataStream(t, v1.MetadataStreamsMax+1, ""), + } + fs := []backend.File{ + newBackendFile(t, "index.md"), + } + fsDel := []string{} + err := backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidMDID, + } + rct = append(rct, recordContentTest{ + description: "Invalid metadata ID error", + metadata: md, + files: fs, + filesDel: fsDel, + err: err, + }) + + // Duplicate metadata ID error + md = []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + newBackendMetadataStream(t, 1, ""), + } + fs = []backend.File{ + newBackendFile(t, "index.md"), + } + err = backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusDuplicateMDID, + } + rct = append(rct, recordContentTest{ + description: "Duplicate metadata ID error", + metadata: md, + files: fs, + filesDel: fsDel, + err: err, + }) + + // Invalid filename error + md = []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs = []backend.File{ + newBackendFile(t, "invalid/filename.md"), + } + err = backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidFilename, + } + rct = append(rct, recordContentTest{ + description: "Invalid filename error", + metadata: md, + files: fs, + filesDel: fsDel, + err: err, + }) + + // Invalid filename in filesDel error + md = []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs = []backend.File{ + newBackendFile(t, "index.md"), + } + fsDel = []string{"invalid/filename.md"} + err = backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidFilename, + } + rct = append(rct, recordContentTest{ + description: "Invalid filename in filesDel error", + metadata: md, + files: fs, + filesDel: fsDel, + err: err, + }) + + // Empty files error + md = []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs = []backend.File{} + fsDel = []string{} + err = backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusEmpty, + } + rct = append(rct, recordContentTest{ + description: "Empty files error", + metadata: md, + files: fs, + filesDel: fsDel, + err: err, + }) + + // Duplicate filename error + md = []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs = []backend.File{ + newBackendFile(t, "index.md"), + newBackendFile(t, "index.md"), + } + err = backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusDuplicateFilename, + } + rct = append(rct, recordContentTest{ + description: "Duplicate filename error", + metadata: md, + files: fs, + filesDel: fsDel, + err: err, + }) + + // Duplicate filename in filesDel error + md = []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs = []backend.File{ + newBackendFile(t, "index.md"), + } + fsDel = []string{ + "duplicate.md", + "duplicate.md", + } + err = backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusDuplicateFilename, + } + rct = append(rct, recordContentTest{ + description: "Duplicate filename in filesDel error", + metadata: md, + files: fs, + filesDel: fsDel, + err: err, + }) + + // Invalid file digest error + md = []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs = []backend.File{ + newBackendFile(t, "index.md"), + } + fsDel = []string{} + err = backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidFileDigest, + } + rct = append(rct, recordContentTest{ + description: "Invalid file digest error", + metadata: md, + files: fs, + filesDel: fsDel, + err: err, + }) + + // Invalid base64 error + md = []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + f := newBackendFile(t, "index.md") + f.Payload = "*" + fs = []backend.File{f} + err = backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidBase64, + } + rct = append(rct, recordContentTest{ + description: "Invalid file digest error", + metadata: md, + files: fs, + filesDel: fsDel, + err: err, + }) + + // Invalid payload digest error + md = []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + f = newBackendFile(t, "index.md") + f.Payload = "rand" + fs = []backend.File{f} + err = backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidFileDigest, + } + rct = append(rct, recordContentTest{ + description: "Invalid payload digest error", + metadata: md, + files: fs, + filesDel: fsDel, + err: err, + }) + + // Invalid MIME type from payload error + md = []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + jpeg := newBackendFileJPEG(t) + jpeg.Payload = "rand" + payload, er := base64.StdEncoding.DecodeString(jpeg.Payload) + if er != nil { + t.Fatalf(er.Error()) + } + jpeg.Digest = hex.EncodeToString(util.Digest(payload)) + fs = []backend.File{ + newBackendFile(t, "index.md"), + jpeg, + } + err = backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidMIMEType, + } + rct = append(rct, recordContentTest{ + description: "Invalid MIME type from payload error", + metadata: md, + files: fs, + filesDel: fsDel, + err: err, + }) + + // Unsupported MIME type error + md = []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + jpeg = newBackendFileJPEG(t) + fs = []backend.File{ + newBackendFile(t, "index.md"), + jpeg, + } + err = backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusUnsupportedMIMEType, + } + rct = append(rct, recordContentTest{ + description: "Unsupported MIME type error", + metadata: md, + files: fs, + filesDel: fsDel, + err: err, + }) + + return rct +} diff --git a/politeiad/backend/tlogbe/tlogbe_test.go b/politeiad/backend/tlogbe/tlogbe_test.go index dae9f32f3..e4d904142 100644 --- a/politeiad/backend/tlogbe/tlogbe_test.go +++ b/politeiad/backend/tlogbe/tlogbe_test.go @@ -5,7 +5,7 @@ package tlogbe import ( - "fmt" + "errors" "testing" "github.com/decred/politeia/politeiad/backend" @@ -14,27 +14,34 @@ import ( func TestNewRecord(t *testing.T) { tlogBackend, err := newTestTlogBackend(t) if err != nil { - fmt.Printf("Error in newTestTlogBackend %v", err) - return + t.Errorf("error in newTestTlogBackend %v", err) } - metadata := backend.MetadataStream{ - ID: 1, - Payload: "", + // Test all record content verification error through the New endpoint + recordContentTests := setupRecordContentTests(t) + for _, test := range recordContentTests { + t.Run(test.description, func(t *testing.T) { + _, err := tlogBackend.New(test.metadata, test.files) + + var contentError backend.ContentVerificationError + if errors.As(err, &contentError) { + if contentError.ErrorCode != test.err.ErrorCode { + t.Errorf("got error %v, want %v", contentError.ErrorCode, + test.err.ErrorCode) + } + } + }) } - file := backend.File{ - Name: "index.md", - MIME: "text/plain; charset=utf-8", - Digest: "22e88c7d6da9b73fbb515ed6a8f6d133c680527a799e3069ca7ce346d90649b2", - Payload: "bW9vCg==", + // Test success case + md := []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), } - - rmd, err := tlogBackend.New([]backend.MetadataStream{metadata}, []backend.File{file}) + fs := []backend.File{ + newBackendFile(t, "index.md"), + } + _, err = tlogBackend.New(md, fs) if err != nil { - fmt.Printf("Error in New %v", err) - return + t.Errorf("success case failed with %v", err) } - - fmt.Println(rmd) } From 1d93ae91804ee8344f76dd8381df8a5428c5b707 Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Mon, 7 Dec 2020 12:42:46 -0300 Subject: [PATCH 176/449] multi: Refactor politeiaverify tool --- politeiad/README.md | 1 - politeiad/cmd/politeia/README.md | 37 +++- politeiad/cmd/politeia/politeia.go | 41 ++++ politeiad/cmd/politeia_verify/README.md | 42 ---- .../cmd/politeia_verify/politeia_verify.go | 197 ------------------ politeiawww/cmd/politeiaverify/README.md | 51 +++++ .../cmd/politeiaverify/politeiaverify.go | 185 ++++++++++++++++ 7 files changed, 304 insertions(+), 250 deletions(-) delete mode 100644 politeiad/cmd/politeia_verify/README.md delete mode 100644 politeiad/cmd/politeia_verify/politeia_verify.go create mode 100644 politeiawww/cmd/politeiaverify/README.md create mode 100644 politeiawww/cmd/politeiaverify/politeiaverify.go diff --git a/politeiad/README.md b/politeiad/README.md index 2661b3344..fe6abf2b1 100644 --- a/politeiad/README.md +++ b/politeiad/README.md @@ -69,6 +69,5 @@ Use the following config settings to spin up a development politeiad instance. # Tools and reference clients * [politeia](https://github.com/decred/politeia/tree/master/politeiad/cmd/politeia) - Reference client for politeiad. -* [politeia_verify](https://github.com/decred/politeia/tree/master/politeiad/cmd/politeia_verify) - Reference verification tool. diff --git a/politeiad/cmd/politeia/README.md b/politeiad/cmd/politeia/README.md index 9234ae42b..fcf3393ea 100644 --- a/politeiad/cmd/politeia/README.md +++ b/politeiad/cmd/politeia/README.md @@ -1,16 +1,17 @@ # politeia refclient examples Available commands: -`identity` -`new` -`updateunvetted` -`updateunvettedmd` -`setunvettedstatus` -`getunvetted` -`updatevetted` -`updatevettedmd` -`getvetted` -`plugin` +`identity` +`verify` +`new` +`updateunvetted` +`updateunvettedmd` +`setunvettedstatus` +`getunvetted` +`updatevetted` +`updatevettedmd` +`getvetted` +`plugin` `plugininventory` `inventory` @@ -49,6 +50,22 @@ Record submitted $ ``` +## Verify a record + +Verifies the censorship signature of a record to make sure it was received by +the server. It receives as input the server's public key, the record token, +the record merkle root and the signature. + +``` +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass verify \ + df0f5cff8d9a3c6d55429c7e9e66b13ec175768b24f20db1b91188f00f7ea4b7 \ + c800ff5195ddb2360000 \ + 36a2e53b25183dcfd192224fdff1074472ad7fdf5989abf9dfb8e7972caceae1 \ + f9495f8100fc3d2d2bcd3d57705adc1f859d29f5a0ed1d999593a57de5af8c047615efb65e08e93935d071b94ee3883f15661defbfa83b503e6e84be4c18aa0b + + Record successfully verified +``` + ## Get unvetted record ``` diff --git a/politeiad/cmd/politeia/politeia.go b/politeiad/cmd/politeia/politeia.go index 60a69c520..ae3de02fe 100644 --- a/politeiad/cmd/politeia/politeia.go +++ b/politeiad/cmd/politeia/politeia.go @@ -72,6 +72,8 @@ func usage() { "status\n") fmt.Fprintf(os.Stderr, " new - Create new record "+ "[metadata]... ...\n") + fmt.Fprintf(os.Stderr, " verify - Verify a record "+ + " \n") fmt.Fprintf(os.Stderr, " getunvetted - Retrieve record "+ "\n") fmt.Fprintf(os.Stderr, " setunvettedstatus - Set unvetted record "+ @@ -593,12 +595,49 @@ func newRecord() error { } if !*printJson { + fmt.Printf(" Server public key: %v\n", id.String()) printCensorshipRecord(reply.CensorshipRecord) } return nil } +func verifyRecord() error { + flags := flag.Args()[1:] // Chop off action. + + // Action arguments + pk := flags[0] + token := flags[1] + merkleRoot := flags[2] + signature := flags[3] + + if len(flags) < 4 { + return fmt.Errorf("Must pass all input parameters") + } + + id, err := util.IdentityFromString(pk) + if err != nil { + return err + } + sig, err := util.ConvertSignature(signature) + if err != nil { + return err + } + + // Verify merkle+token msg against signature + if !id.VerifyMessage([]byte(merkleRoot+token), sig) { + return fmt.Errorf("Invalid censorship record signature") + } + + fmt.Printf("Public key : %s\n", pk) + fmt.Printf("Token : %s\n", token) + fmt.Printf("Merkle root: %s\n", merkleRoot) + fmt.Printf("Signature : %s\n\n", signature) + fmt.Println("Record successfully verified") + + return nil +} + func validateMetadataFlags(flags []string) ([]v1.MetadataStream, []v1.MetadataStream, string, error) { var mdAppend []v1.MetadataStream var mdOverwrite []v1.MetadataStream @@ -1528,6 +1567,8 @@ func _main() error { switch a { case "identity": return getIdentity() + case "verify": + return verifyRecord() case "new": return newRecord() case "updateunvetted": diff --git a/politeiad/cmd/politeia_verify/README.md b/politeiad/cmd/politeia_verify/README.md deleted file mode 100644 index 48ded7656..000000000 --- a/politeiad/cmd/politeia_verify/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# politeia_verify - -politeia_verify is a simple tool that allows anyone to independently verify that -Politeia has received your proposal. You just need to provide the Politeia -server's public key, the proposal's censorship token and signature, and the -proposal files. - -## Usage - -There are 2 methods of input: - -``` -politeia_verify [options] - -Options: - -k Politiea's public server key - -t Record censorship token - -s Record censorship signature - -v Verbose output - -jsonin A path to a JSON file which represents the record. If this - option is set, the other input options (-k, -t, -s) should - not be provided. - -jsonout JSON output - -Filenames: One or more paths to the markdown and image files that - make up the record. -``` - -Example: - -``` -politeia_verify -v -k dfd6caacf0bbe5725efc67e703e912c37931b4edbf17122947a1e0fcd9755f6d -t 6284c5f8fba5665373b8e6651ebc8747b289fed242d2f880f64a284496bb4ca8 -s 82d69b4ec83d2a732fe92028dbf78853d0814aeb4fcf0ff597c110c8843720951f7b9fae4305b0f1d9346c39bc960a364590236f9e0871f6f79860fc57d4c70 proposal.md -Proposal successfully verified. -``` - -If the proposal fails to verify, it will return an error: - -``` -politeia_verify -v -k xfd6caacf0bbe5725efc67e703e912c37931b4edbf17122947a1e0fcd9755f6d -t 6284c5f8fba5665373b8e6651ebc8747b289fed242d2f880f64a284496bb4ca8 -s 82d69b4ec83d2a732fe92028dbf78853d0814aeb4fcf0ff597c110c8843720951f7b9fae4305b0f1d9346c39bc960a364590236f9e0871f6f79860fc57d4c70 proposal.md -Proposal failed verification. Please ensure the public key and merkle are correct. - Merkle: 0dd10219cd79342198085cbe6f737bd54efe119b24c84cbc053023ed6b7da4c8 -``` diff --git a/politeiad/cmd/politeia_verify/politeia_verify.go b/politeiad/cmd/politeia_verify/politeia_verify.go deleted file mode 100644 index 6d23e2fd5..000000000 --- a/politeiad/cmd/politeia_verify/politeia_verify.go +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright (c) 2017 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" - "flag" - "fmt" - "io/ioutil" - "os" - - "github.com/decred/dcrtime/merkle" - "golang.org/x/crypto/ed25519" -) - -var ( - publicKeyFlag = flag.String("k", "", "server public key") - tokenFlag = flag.String("t", "", "record censorship token") - signatureFlag = flag.String("s", "", "record censorship signature") - jsonInFlag = flag.String("jsonin", "", "JSON record file") - jsonOutFlag = flag.Bool("jsonout", false, "return output as JSON") - verboseFlag = flag.Bool("v", false, "verbose output") -) - -type record struct { - CensorshipRecord censorshipRecord `json:"censorshiprecord"` - ServerPublicKey string `json:"serverPubkey"` -} - -type censorshipRecord struct { - Token string `json:"token"` - Merkle string `json:"merkle"` - Signature string `json:"signature"` -} - -type output struct { - Success bool `json:"success"` -} - -func usage() { - fmt.Fprintf(os.Stderr, "usage: politeia_verify [options]\n") - fmt.Fprintf(os.Stderr, " options:\n") - fmt.Fprintf(os.Stderr, " -v - Verbose output\n") - fmt.Fprintf(os.Stderr, " -k - Politiea's public server key\n") - fmt.Fprintf(os.Stderr, " -t - Record censorship token\n") - fmt.Fprintf(os.Stderr, " -s - Record censorship "+ - "signature\n") - fmt.Fprintf(os.Stderr, " - One or more paths to the markdown "+ - "and image files that make up the record\n") - fmt.Fprintf(os.Stderr, " -jsonin - A path to a JSON file which "+ - "represents the record. If this option is set, the other input "+ - "options (-k, -t, -s) should not be provided.\n") - fmt.Fprintf(os.Stderr, " -jsonout - JSON output\n") - fmt.Fprintf(os.Stderr, "\n") -} - -func findMerkle() (*[sha256.Size]byte, error) { - flags := flag.Args() - if len(flags) < 1 { - usage() - return nil, fmt.Errorf("must provide at least one filename for the record") - } - - // Open all files and digest them. - hashes := make([]*[sha256.Size]byte, 0, len(flags)) - for _, filename := range flags { - var payload []byte - payload, err := ioutil.ReadFile(filename) - if err != nil { - return nil, err - } - - // Digest - h := sha256.New() - h.Write(payload) - digest := h.Sum(nil) - - var digest32 [sha256.Size]byte - copy(digest32[:], digest) - hashes = append(hashes, &digest32) - } - - return merkle.Root(hashes), nil -} - -func verifyRecord(key [ed25519.PublicKeySize]byte, merkle, token string, signature [ed25519.SignatureSize]byte) bool { - return ed25519.Verify(key[:], []byte(merkle+token), signature[:]) -} - -func _main() error { - flag.Parse() - if (*publicKeyFlag == "" || *tokenFlag == "" || *signatureFlag == "") && - *jsonInFlag == "" { - usage() - return fmt.Errorf("must provide enough input parameters") - } - if *publicKeyFlag != "" && *jsonInFlag != "" { - usage() - return fmt.Errorf("must only provide either -jsonin or the other " + - "input parameters") - } - - var keyStr, tokenStr, merkleStr, signatureStr string - if *publicKeyFlag != "" { - keyStr = *publicKeyFlag - tokenStr = *tokenFlag - signatureStr = *signatureFlag - } else { - var payload []byte - payload, err := ioutil.ReadFile(*jsonInFlag) - if err != nil { - return err - } - - var record record - err = json.Unmarshal(payload, &record) - if err != nil { - return err - } - - keyStr = record.ServerPublicKey - tokenStr = record.CensorshipRecord.Token - signatureStr = record.CensorshipRecord.Signature - merkleStr = record.CensorshipRecord.Merkle - } - - // Decode the public key, token and signature. - key, err := hex.DecodeString(keyStr) - if err != nil { - return err - } - var publicKey [ed25519.PublicKeySize]byte - copy(publicKey[:], key) - - sig, err := hex.DecodeString(signatureStr) - if err != nil { - return err - } - var signature [ed25519.SignatureSize]byte - copy(signature[:], sig) - - var merkle [sha256.Size]byte - if *publicKeyFlag != "" { - merklePtr, err := findMerkle() - if err != nil { - return err - } - - merkle = *merklePtr - } else { - bytes, err := hex.DecodeString(merkleStr) - if err != nil { - return err - } - - copy(merkle[:], bytes) - } - - recordVerified := verifyRecord(publicKey, hex.EncodeToString(merkle[:]), - tokenStr, signature) - if *jsonOutFlag { - bytes, err := json.Marshal(output{ - Success: recordVerified, - }) - if err != nil { - return err - } - - fmt.Println(string(bytes)) - } else { - if recordVerified { - fmt.Println("Record successfully verified") - } else { - if *verboseFlag { - return fmt.Errorf("Record failed verification. Please ensure the "+ - "public key and merkle are correct.\n"+ - " Merkle: %v", hex.EncodeToString(merkle[:])) - } - - return fmt.Errorf("Record failed verification") - } - } - - return nil -} - -func main() { - err := _main() - if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) - } -} diff --git a/politeiawww/cmd/politeiaverify/README.md b/politeiawww/cmd/politeiaverify/README.md new file mode 100644 index 000000000..45a2682e0 --- /dev/null +++ b/politeiawww/cmd/politeiaverify/README.md @@ -0,0 +1,51 @@ +# Politeia Verify + +`politeiaverify` is a simple tool that allows anyone to independently verify +that Politeia has received your proposal/comment and that it is sound. The +input received in this command is the json bundle downloaded from the GUI. +Files from the gui are downloaded with filename `.json` for proposal +bundles and `-comments.json` for proposal comments bundle. If no flag +is passed in, the tool will try to read the filename and call the corresponding +verify method. + +## Usage + +`politeiaverify [flags] ` + +Flags: + `-proposal` - verify proposal bundle + `-comments` - verify comments bundle + +Examples: + +To verify a proposal bundle + +``` +politeiaverify -proposal c093b8a808ef68665709995a5a741bd02502b9c6c48a99a4b179fef742ca6b2a.json + +Proposal signature: + Public key: 49912d8dd296ce00a4b6afce4f300481ed5403142740e8b510276dccd1cbaccd + Signature : a0c1e9d887bd77ddf3b4fd650082ae8bc0f7c09631de7dce1b3147140d1163347768abbf2cecd0196ad5019c40dd2a9a16db482955f5cc30a1b79771ccffa90b +Proposal censorship record signature: + Merkle root: e905baa3391e446ab89270153f45581640e7cef6e162152fa6e469737699c6bd + Public key : a70134196c3cdf3f85f8af6abaa38c15feb7bccf5e6d3db6212358363465e502 + Signature : 0bf4c685102db88c06df12f0980f87bda5e285fcc37be4bef118b138bc5dcdfa3dceaa234eaff2e47f5f16224743d9cf7fe31e3244e67e50b1b7685910362e01 + +Proposal successfully verified + +``` + +To verify a proposal comments bundle + +``` +politeiaverify -comments c093b8a808ef68665709995a5a741bd02502b9c6c48a99a4b179fef742ca6b2a-comments.json + +Comment ID: 1 + Public key: a70134196c3cdf3f85f8af6abaa38c15feb7bccf5e6d3db6212358363465e502 + Receipt : fdf466b5511a0ad7304bdb45cb387b7c4ebe720c9d5c3271ec144fee92775de5e6f30482d42375eda99c3b704f37a7f70108d4f201f7c91d6024b9ec5aaa110b + Signature : c48d643784b5c3645afce7965e7d7d9b44978c22829da30174c51e22eda2849e87d6cd304f7fc2e5bdc5d17ab4b515bc89605b7a814355a44cdfa86d8dc4030e + +Comments successfully verified +``` + +If the bundle is in bad format or if it fails to verify, it will return an error. diff --git a/politeiawww/cmd/politeiaverify/politeiaverify.go b/politeiawww/cmd/politeiaverify/politeiaverify.go new file mode 100644 index 000000000..b1fdd2141 --- /dev/null +++ b/politeiawww/cmd/politeiaverify/politeiaverify.go @@ -0,0 +1,185 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "os" + "path" + "strings" + + pi "github.com/decred/politeia/politeiawww/api/pi/v1" + wwwutil "github.com/decred/politeia/politeiawww/util" + "github.com/decred/politeia/util" +) + +// proposal is used to unmarshal the data that is cointaned in the proposal +// JSON bundles downloded from the GUI. +type proposal struct { + PublicKey string `json:"publickey"` + Signature string `json:"signature"` + CensorshipRecord pi.CensorshipRecord `json:"censorshiprecord"` + Files []pi.File `json:"files"` + Metadata []pi.Metadata `json:"metadata"` + ServerPublicKey string `json:"serverpublickey"` +} + +// comments is used to unmarshal the data that is cointaned in the comments +// JSON bundles downloded from the GUI. +type comments []struct { + CommentID string `json:"commentid"` + Receipt string `json:"receipt"` + Signature string `json:"signature"` + ServerPublicKey string `json:"serverpublickey"` +} + +var ( + flagVerifyProposal = flag.Bool("proposal", false, "Verify proposal bundle") + flagVerifyComments = flag.Bool("comments", false, "Verify comments bundle") +) + +func usage() { + fmt.Fprintf(os.Stderr, "usage: politeiaverify [flags] \n") + fmt.Fprintf(os.Stderr, " flags:\n") + flag.PrintDefaults() + fmt.Fprintf(os.Stderr, " - Path to the JSON bundle "+ + "downloaded from the GUI\n") + fmt.Fprintf(os.Stderr, "\n") +} + +func verifyProposal(payload []byte) error { + var prop proposal + err := json.Unmarshal(payload, &prop) + if err != nil { + return fmt.Errorf("Proposal bundle JSON in bad format, make sure to " + + "download it from the GUI.") + } + + // Verify merkle root + merkle, err := wwwutil.MerkleRoot(prop.Files, prop.Metadata) + if err != nil { + return err + } + if merkle != prop.CensorshipRecord.Merkle { + return fmt.Errorf("Merkle roots do not match: %v and %v", + prop.CensorshipRecord.Merkle, merkle) + } + + // Verify proposal signature + id, err := util.IdentityFromString(prop.PublicKey) + if err != nil { + return err + } + sig, err := util.ConvertSignature(prop.Signature) + if err != nil { + return err + } + if !id.VerifyMessage([]byte(merkle), sig) { + return fmt.Errorf("Invalid proposal signature %v", prop.Signature) + } + + // Verify censorship record signature + id, err = util.IdentityFromString(prop.ServerPublicKey) + if err != nil { + return err + } + sig, err = util.ConvertSignature(prop.CensorshipRecord.Signature) + if err != nil { + return err + } + if !id.VerifyMessage([]byte(merkle+prop.CensorshipRecord.Token), sig) { + return fmt.Errorf("Invalid censhorship record signature %v", + prop.CensorshipRecord.Signature) + } + + fmt.Println("Proposal signature:") + fmt.Printf(" Public key: %s\n", prop.PublicKey) + fmt.Printf(" Signature : %s\n", prop.Signature) + fmt.Println("Proposal censorship record signature:") + fmt.Printf(" Merkle root: %s\n", prop.CensorshipRecord.Merkle) + fmt.Printf(" Public key : %s\n", prop.ServerPublicKey) + fmt.Printf(" Signature : %s\n\n", prop.CensorshipRecord.Signature) + fmt.Println("Proposal successfully verified") + + return nil +} + +func verifyComments(payload []byte) error { + var comments comments + err := json.Unmarshal(payload, &comments) + if err != nil { + return fmt.Errorf("Comments bundle JSON in bad format, make sure to " + + "download it from the GUI.") + } + + for _, c := range comments { + // Verify receipt + id, err := util.IdentityFromString(c.ServerPublicKey) + if err != nil { + return err + } + receipt, err := util.ConvertSignature(c.Receipt) + if err != nil { + return err + } + if !id.VerifyMessage([]byte(c.Signature), receipt) { + return fmt.Errorf("Could not verify receipt %v of comment id %v", + c.Receipt, c.CommentID) + } + fmt.Printf("Comment ID: %s\n", c.CommentID) + fmt.Printf(" Public key: %s\n", c.ServerPublicKey) + fmt.Printf(" Receipt : %s\n", c.Receipt) + fmt.Printf(" Signature : %s\n", c.Signature) + } + + fmt.Println("\nComments successfully verified") + + return nil +} + +func _main() error { + flag.Parse() + args := flag.Args() + + // Validate flags and arguments + switch { + case len(args) != 1: + usage() + return fmt.Errorf("Must provide json bundle path as input") + case *flagVerifyProposal && *flagVerifyComments: + usage() + return fmt.Errorf("Must choose only one verification type") + } + + // Read bundle payload + file := args[0] + var payload []byte + payload, err := ioutil.ReadFile(file) + if err != nil { + return err + } + + // Call verify method + switch { + case *flagVerifyProposal: + return verifyProposal(payload) + case *flagVerifyComments: + return verifyComments(payload) + default: + // No flags used, read filename and try to call corresponding + // verify method + if strings.Contains(path.Base(file), "comments") { + return verifyComments(payload) + } + return verifyProposal(payload) + } +} + +func main() { + err := _main() + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } +} From f282a355c3a266892b5fac5ed01476139efd0081 Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Mon, 7 Dec 2020 16:22:58 -0300 Subject: [PATCH 177/449] tlogbe: Add update unvetted record tests. --- politeiad/backend/tlogbe/testing.go | 26 ++++ politeiad/backend/tlogbe/tlog.go | 4 +- politeiad/backend/tlogbe/tlogbe_test.go | 171 ++++++++++++++++++++- politeiad/backend/tlogbe/trillianclient.go | 71 +++++---- 4 files changed, 239 insertions(+), 33 deletions(-) diff --git a/politeiad/backend/tlogbe/testing.go b/politeiad/backend/tlogbe/testing.go index 05d7ae916..2d6e2fdca 100644 --- a/politeiad/backend/tlogbe/testing.go +++ b/politeiad/backend/tlogbe/testing.go @@ -11,6 +11,7 @@ import ( "fmt" "image" "image/jpeg" + "image/png" "os" "path/filepath" "testing" @@ -79,6 +80,31 @@ func newBackendFileJPEG(t *testing.T) backend.File { } } +func newBackendFilePNG(t *testing.T) backend.File { + t.Helper() + + b := new(bytes.Buffer) + img := image.NewRGBA(image.Rect(0, 0, 1000, 500)) + + err := png.Encode(b, img) + if err != nil { + t.Fatalf("%v", err) + } + + // Generate a random name + r, err := util.Random(8) + if err != nil { + t.Fatalf("%v", err) + } + + return backend.File{ + Name: hex.EncodeToString(r) + ".png", + MIME: mime.DetectMimeType(b.Bytes()), + Digest: hex.EncodeToString(util.Digest(b.Bytes())), + Payload: base64.StdEncoding.EncodeToString(b.Bytes()), + } +} + func newBackendMetadataStream(t *testing.T, id uint64, payload string) backend.MetadataStream { t.Helper() diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index f07c65f54..343855ac7 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -389,7 +389,7 @@ func (t *tlog) treeExists(treeID int64) bool { // treeFreeze updates the status of a record and freezes the trillian tree as a // result of a record status change. The tree pointer is the tree ID of the new // location of the record. This is provided on certain status changes such as -// when a unvetted record is make public and the unvetted record is moved to a +// when a unvetted record is made public and the unvetted record is moved to a // vetted tree. A value of 0 indicates that no tree pointer exists. // // Once the record index has been saved with its frozen field set, the tree @@ -443,7 +443,7 @@ func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []ba return fmt.Errorf("leavesAppend: %v", err) } if len(queued) != 1 { - return fmt.Errorf("wrong number of queud leaves: got %v, want 1", + return fmt.Errorf("wrong number of queued leaves: got %v, want 1", len(queued)) } failed := make([]string, 0, len(queued)) diff --git a/politeiad/backend/tlogbe/tlogbe_test.go b/politeiad/backend/tlogbe/tlogbe_test.go index e4d904142..319f04fe3 100644 --- a/politeiad/backend/tlogbe/tlogbe_test.go +++ b/politeiad/backend/tlogbe/tlogbe_test.go @@ -8,21 +8,25 @@ import ( "errors" "testing" + v1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/util" ) func TestNewRecord(t *testing.T) { tlogBackend, err := newTestTlogBackend(t) if err != nil { - t.Errorf("error in newTestTlogBackend %v", err) + t.Error(err) } // Test all record content verification error through the New endpoint recordContentTests := setupRecordContentTests(t) for _, test := range recordContentTests { t.Run(test.description, func(t *testing.T) { + // Make backend call _, err := tlogBackend.New(test.metadata, test.files) + // Parse error var contentError backend.ContentVerificationError if errors.As(err, &contentError) { if contentError.ErrorCode != test.err.ErrorCode { @@ -45,3 +49,168 @@ func TestNewRecord(t *testing.T) { t.Errorf("success case failed with %v", err) } } + +func TestUpdateUnvettedRecord(t *testing.T) { + tlogBackend, err := newTestTlogBackend(t) + if err != nil { + t.Error(err) + } + + // Create new record + md := []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs := []backend.File{ + newBackendFile(t, "index.md"), + } + rec, err := tlogBackend.New(md, fs) + if err != nil { + t.Error(err) + } + token, err := tokenDecode(rec.Token) + if err != nil { + t.Error(err) + } + + // Test all record content verification error through the + // UpdateUnvettedRecord endpoint + recordContentTests := setupRecordContentTests(t) + for _, test := range recordContentTests { + t.Run(test.description, func(t *testing.T) { + // Convert token + token, err := util.ConvertStringToken(rec.Token) + if err != nil { + t.Error(err) + } + + // Make backend call + _, err = tlogBackend.UpdateUnvettedRecord(token, test.metadata, + []backend.MetadataStream{}, test.files, test.filesDel) + + // Parse error + var contentError backend.ContentVerificationError + if errors.As(err, &contentError) { + if contentError.ErrorCode != test.err.ErrorCode { + t.Errorf("got error %v, want %v", contentError.ErrorCode, + test.err.ErrorCode) + } + } + }) + } + + // Random png image file to include in edit payload + imageRandom := newBackendFilePNG(t) + + // test case: Token not full length + tokenShort, err := util.ConvertStringToken(util.TokenToPrefix(rec.Token)) + if err != nil { + t.Error(err) + } + + // test case: Record not found + tokenRandom := tokenFromTreeID(123) + + // test case: Frozen tree + recFrozen, err := tlogBackend.New(md, fs) + if err != nil { + t.Error(err) + } + tokenFrozen, err := tokenDecode(recFrozen.Token) + if err != nil { + t.Error(err) + } + err = tlogBackend.unvetted.treeFreeze(treeIDFromToken(tokenFrozen), + backend.RecordMetadata{}, []backend.MetadataStream{}, 0) + if err != nil { + t.Error(err) + } + + // Setup UpdateUnvettedRecord tests + var tests = []struct { + description string + token []byte + mdAppend, mdOverwirte []backend.MetadataStream + filesAdd []backend.File + filesDel []string + wantContentErr *backend.ContentVerificationError + wantErr error + }{ + { + "token not full length", + tokenShort, + []backend.MetadataStream{}, + []backend.MetadataStream{}, + []backend.File{imageRandom}, + []string{}, + &backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidToken, + }, + nil, + }, + { + "record not found", + tokenRandom, + []backend.MetadataStream{}, + []backend.MetadataStream{}, + []backend.File{imageRandom}, + []string{}, + nil, + backend.ErrRecordNotFound, + }, + { + "tree frozen for changes", + tokenFrozen, + []backend.MetadataStream{}, + []backend.MetadataStream{}, + []backend.File{imageRandom}, + []string{}, + nil, + backend.ErrRecordLocked, + }, + { + "no changes to record", + token, + []backend.MetadataStream{}, + []backend.MetadataStream{}, + []backend.File{}, + []string{}, + nil, + backend.ErrNoChanges, + }, + { + "success", + token, + []backend.MetadataStream{}, + []backend.MetadataStream{}, + []backend.File{imageRandom}, + []string{}, + nil, + nil, + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + // Make backend call + _, err = tlogBackend.UpdateUnvettedRecord(test.token, + test.mdAppend, test.mdOverwirte, test.filesAdd, test.filesDel) + + // Parse error + var contentError backend.ContentVerificationError + if errors.As(err, &contentError) { + if test.wantContentErr == nil || // sanity check + contentError.ErrorCode != test.wantContentErr.ErrorCode { + t.Errorf("got error %v, want %v", + v1.ErrorStatus[contentError.ErrorCode], + test.wantContentErr) + } + return + } + + if test.wantErr != err { + t.Errorf("got error %v, want %v", err, test.wantErr) + } + }) + } + +} diff --git a/politeiad/backend/tlogbe/trillianclient.go b/politeiad/backend/tlogbe/trillianclient.go index 62c5d69ec..7765aef10 100644 --- a/politeiad/backend/tlogbe/trillianclient.go +++ b/politeiad/backend/tlogbe/trillianclient.go @@ -48,6 +48,8 @@ type trillianClient interface { treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) + treeFreeze(treeID int64) (*trillian.Tree, error) + leavesAll(treeID int64) ([]*trillian.LogLeaf, error) leavesByRange(treeID, startIndex, count int64) ([]*trillian.LogLeaf, error) @@ -527,36 +529,6 @@ type testTClient struct { privateKey *keyspb.PrivateKey } -// tree returns trillian tree from passed in ID. -// -// This function satisfies the TrillianClient interface. -func (t *testTClient) tree(treeID int64) (*trillian.Tree, error) { - t.RLock() - defer t.RUnlock() - - if tree, ok := t.trees[treeID]; ok { - return tree, nil - } - - return nil, fmt.Errorf("Tree ID not found") -} - -// treesAll signed log roots are not used for testing up until now, so we -// return a nil value for it. -// -// This function satisfies the TrillianClient interface. -func (t *testTClient) treesAll() ([]*trillian.Tree, error) { - t.RLock() - defer t.RUnlock() - - trees := make([]*trillian.Tree, len(t.trees)) - for _, t := range t.trees { - trees = append(trees, t) - } - - return trees, nil -} - // treeNew ceates a new trillian tree in memory. // // This function satisfies the TrillianClient interface. @@ -591,6 +563,45 @@ func (t *testTClient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) return &tree, nil, nil } +func (t *testTClient) treeFreeze(treeID int64) (*trillian.Tree, error) { + t.Lock() + defer t.Unlock() + + t.trees[treeID].TreeState = trillian.TreeState_FROZEN + + return t.trees[treeID], nil +} + +// tree returns trillian tree from passed in ID. +// +// This function satisfies the TrillianClient interface. +func (t *testTClient) tree(treeID int64) (*trillian.Tree, error) { + t.RLock() + defer t.RUnlock() + + if tree, ok := t.trees[treeID]; ok { + return tree, nil + } + + return nil, fmt.Errorf("Tree ID not found") +} + +// treesAll signed log roots are not used for testing up until now, so we +// return a nil value for it. +// +// This function satisfies the TrillianClient interface. +func (t *testTClient) treesAll() ([]*trillian.Tree, error) { + t.RLock() + defer t.RUnlock() + + trees := make([]*trillian.Tree, len(t.trees)) + for _, t := range t.trees { + trees = append(trees, t) + } + + return trees, nil +} + // leavesAll returns all leaves from a trillian tree. // // This function satisfies the TrillianClient interface. From a554bcc9d0421deb9059eedfe81964fd1f4da3fd Mon Sep 17 00:00:00 2001 From: Amir Massarwa Date: Mon, 7 Dec 2020 21:29:35 +0200 Subject: [PATCH 178/449] tlogbe: Hook up token prefixes in comments & ticketvote plugins. --- politeiad/backend/tlogbe/comments.go | 28 +++++++++++++++++--- politeiad/backend/tlogbe/ticketvote.go | 36 ++++++++++++++++++++++---- politeiad/backend/tlogbe/tlogbe.go | 4 +++ politeiad/backend/tlogbe/tlogclient.go | 9 +++++++ 4 files changed, 68 insertions(+), 9 deletions(-) diff --git a/politeiad/backend/tlogbe/comments.go b/politeiad/backend/tlogbe/comments.go index 8b1856181..005c0cffd 100644 --- a/politeiad/backend/tlogbe/comments.go +++ b/politeiad/backend/tlogbe/comments.go @@ -121,7 +121,15 @@ func (p *commentsPlugin) mutex(token string) *sync.Mutex { return m } -func (p *commentsPlugin) commentsIndexPath(s comments.StateT, token string) string { +// commentsIndexPath accepts full length token or token prefix but always +// uses prefix when generating the comments index path string. +func (p *commentsPlugin) commentsIndexPath(s comments.StateT, token string) (string, error) { + // Use token prefix + t, err := tokenDecodeAnyLength(token) + if err != nil { + return "", err + } + token = tokenPrefix(t) fn := filenameCommentsIndex switch s { case comments.StateUnvetted: @@ -133,7 +141,7 @@ func (p *commentsPlugin) commentsIndexPath(s comments.StateT, token string) stri panic(e) } fn = strings.Replace(fn, "{token}", token, 1) - return filepath.Join(p.dataDir, fn) + return filepath.Join(p.dataDir, fn), nil } // commentsIndexLocked returns the cached commentsIndex for the provided @@ -142,7 +150,10 @@ func (p *commentsPlugin) commentsIndexPath(s comments.StateT, token string) stri // // This function must be called WITH the lock held. func (p *commentsPlugin) commentsIndexLocked(s comments.StateT, token []byte) (*commentsIndex, error) { - fp := p.commentsIndexPath(s, hex.EncodeToString(token)) + fp, err := p.commentsIndexPath(s, hex.EncodeToString(token)) + if err != nil { + return nil, err + } b, err := ioutil.ReadFile(fp) if err != nil { var e *os.PathError @@ -187,7 +198,10 @@ func (p *commentsPlugin) commentsIndexSaveLocked(s comments.StateT, token []byte return err } - fp := p.commentsIndexPath(s, hex.EncodeToString(token)) + fp, err := p.commentsIndexPath(s, hex.EncodeToString(token)) + if err != nil { + return err + } err = ioutil.WriteFile(fp, b, 0664) if err != nil { return err @@ -1737,6 +1751,12 @@ func (p *commentsPlugin) cmdVotes(payload string) (string, error) { // Lookup votes votes, err := p.commentVotes(v.State, token, merkles) if err != nil { + if errors.Is(err, errRecordNotFound) { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusRecordNotFound), + } + } return "", fmt.Errorf("commentVotes: %v", err) } diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index 1dcf49522..5c2e16e5d 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -405,16 +405,27 @@ func (p *ticketVotePlugin) cachedVotesDel(token string) { log.Debugf("ticketvote: deleted votes cache: %v", token) } -func (p *ticketVotePlugin) cachedSummaryPath(token string) string { +// cachedSummaryPath accepts both full tokens and token prefixes, however it +// always uses the token prefix when generatig the path. +func (p *ticketVotePlugin) cachedSummaryPath(token string) (string, error) { + // Use token prefix + t, err := tokenDecodeAnyLength(token) + if err != nil { + return "", err + } + token = tokenPrefix(t) fn := strings.Replace(filenameSummary, "{token}", token, 1) - return filepath.Join(p.dataDir, fn) + return filepath.Join(p.dataDir, fn), nil } func (p *ticketVotePlugin) cachedSummary(token string) (*ticketvote.Summary, error) { p.Lock() defer p.Unlock() - fp := p.cachedSummaryPath(token) + fp, err := p.cachedSummaryPath(token) + if err != nil { + return nil, err + } b, err := ioutil.ReadFile(fp) if err != nil { var e *os.PathError @@ -443,7 +454,10 @@ func (p *ticketVotePlugin) cachedSummarySave(token string, s ticketvote.Summary) p.Lock() defer p.Unlock() - fp := p.cachedSummaryPath(token) + fp, err := p.cachedSummaryPath(token) + if err != nil { + return err + } err = ioutil.WriteFile(fp, b, 0664) if err != nil { return err @@ -2206,7 +2220,10 @@ func (p *ticketVotePlugin) cmdDetails(payload string) (string, error) { auths, err := p.authorizes(token) if err != nil { if errors.Is(err, errRecordNotFound) { - continue + return "", backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusRecordNotFound), + } } return "", fmt.Errorf("authorizes: %v", err) } @@ -2257,6 +2274,12 @@ func (p *ticketVotePlugin) cmdResults(payload string) (string, error) { // Get cast votes votes, err := p.castVotes(token) if err != nil { + if errors.Is(err, errRecordNotFound) { + return "", backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusRecordNotFound), + } + } return "", err } @@ -2349,6 +2372,9 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. // Check if the vote has been authorized auths, err := p.authorizes(token) if err != nil { + if errors.Is(err, errRecordNotFound) { + return nil, err + } return nil, fmt.Errorf("authorizes: %v", err) } if len(auths) > 0 { diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index acbaaa42f..c104cd151 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -155,6 +155,10 @@ func tokenDecode(token string) ([]byte, error) { // tokenDecodeAnyLength decodes the provided hex encoded record token. This // function accepts both full length tokens and token prefixes. func tokenDecodeAnyLength(token string) ([]byte, error) { + // If provided token has odd length add padding + if len(token)%2 == 1 { + token = token + "0" + } t, err := hex.DecodeString(token) if err != nil { return nil, fmt.Errorf("invalid hex") diff --git a/politeiad/backend/tlogbe/tlogclient.go b/politeiad/backend/tlogbe/tlogclient.go index 078f4d684..8611c6836 100644 --- a/politeiad/backend/tlogbe/tlogclient.go +++ b/politeiad/backend/tlogbe/tlogclient.go @@ -59,6 +59,15 @@ func (c *backendClient) tlogByID(tlogID string) (*tlog, error) { // treeIDFromToken returns the treeID for the provided tlog instance ID and // token. func (c *backendClient) treeIDFromToken(tlogID string, token []byte) (int64, error) { + if len(token) == tokenPrefixSize() { + // This is a token prefix. Get the full token from the cache. + var ok bool + token, ok = c.backend.fullLengthToken(token) + if !ok { + return 0, errRecordNotFound + } + } + switch tlogID { case tlogIDUnvetted: return treeIDFromToken(token), nil From fda38a8d78b9cd4d8f4464abbe3a39533dd031c4 Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Mon, 14 Dec 2020 15:42:44 -0300 Subject: [PATCH 179/449] tlogbe: Add UpdateUnvettedMetadata tests. --- politeiad/api/v1/v1.go | 1 + politeiad/backend/tlogbe/tlogbe_test.go | 192 ++++++++++++++++++++++-- 2 files changed, 184 insertions(+), 9 deletions(-) diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index f8aab1e49..e77eb3b12 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -106,6 +106,7 @@ var ( ErrorStatusNoChanges: "no changes in record", ErrorStatusRecordFound: "record found", ErrorStatusInvalidRPCCredentials: "invalid RPC client credentials", + ErrorStatusInvalidToken: "invalid token", } // RecordStatus converts record status codes to human readable text. diff --git a/politeiad/backend/tlogbe/tlogbe_test.go b/politeiad/backend/tlogbe/tlogbe_test.go index 319f04fe3..8d7004478 100644 --- a/politeiad/backend/tlogbe/tlogbe_test.go +++ b/politeiad/backend/tlogbe/tlogbe_test.go @@ -30,8 +30,9 @@ func TestNewRecord(t *testing.T) { var contentError backend.ContentVerificationError if errors.As(err, &contentError) { if contentError.ErrorCode != test.err.ErrorCode { - t.Errorf("got error %v, want %v", contentError.ErrorCode, - test.err.ErrorCode) + t.Errorf("got error %v, want %v", + v1.ErrorStatus[contentError.ErrorCode], + v1.ErrorStatus[test.err.ErrorCode]) } } }) @@ -91,8 +92,9 @@ func TestUpdateUnvettedRecord(t *testing.T) { var contentError backend.ContentVerificationError if errors.As(err, &contentError) { if contentError.ErrorCode != test.err.ErrorCode { - t.Errorf("got error %v, want %v", contentError.ErrorCode, - test.err.ErrorCode) + t.Errorf("got error %v, want %v", + v1.ErrorStatus[contentError.ErrorCode], + v1.ErrorStatus[test.err.ErrorCode]) } } }) @@ -129,7 +131,7 @@ func TestUpdateUnvettedRecord(t *testing.T) { var tests = []struct { description string token []byte - mdAppend, mdOverwirte []backend.MetadataStream + mdAppend, mdOverwrite []backend.MetadataStream filesAdd []backend.File filesDel []string wantContentErr *backend.ContentVerificationError @@ -193,16 +195,18 @@ func TestUpdateUnvettedRecord(t *testing.T) { t.Run(test.description, func(t *testing.T) { // Make backend call _, err = tlogBackend.UpdateUnvettedRecord(test.token, - test.mdAppend, test.mdOverwirte, test.filesAdd, test.filesDel) + test.mdAppend, test.mdOverwrite, test.filesAdd, test.filesDel) // Parse error var contentError backend.ContentVerificationError if errors.As(err, &contentError) { - if test.wantContentErr == nil || // sanity check - contentError.ErrorCode != test.wantContentErr.ErrorCode { + if test.wantContentErr == nil { + t.Errorf("got error %v, want nil", err) + } + if contentError.ErrorCode != test.wantContentErr.ErrorCode { t.Errorf("got error %v, want %v", v1.ErrorStatus[contentError.ErrorCode], - test.wantContentErr) + v1.ErrorStatus[test.wantContentErr.ErrorCode]) } return } @@ -212,5 +216,175 @@ func TestUpdateUnvettedRecord(t *testing.T) { } }) } +} + +func TestUpdateUnvettedMetadata(t *testing.T) { + tlogBackend, err := newTestTlogBackend(t) + if err != nil { + t.Error(err) + } + + // Create new record + md := []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs := []backend.File{ + newBackendFile(t, "index.md"), + } + rec, err := tlogBackend.New(md, fs) + if err != nil { + t.Error(err) + } + token, err := tokenDecode(rec.Token) + if err != nil { + t.Error(err) + } + + // Test all record content verification error through the + // UpdateUnvettedMetadata endpoint + recordContentTests := setupRecordContentTests(t) + for _, test := range recordContentTests { + t.Run(test.description, func(t *testing.T) { + // Make backend call + err := tlogBackend.UpdateUnvettedMetadata(token, + test.metadata, []backend.MetadataStream{}) + + // Parse error + var contentError backend.ContentVerificationError + if errors.As(err, &contentError) { + if contentError.ErrorCode != test.err.ErrorCode { + t.Errorf("got error %v, want %v", + v1.ErrorStatus[contentError.ErrorCode], + v1.ErrorStatus[test.err.ErrorCode]) + } + } + }) + } + + // test case: Token not full length + tokenShort, err := util.ConvertStringToken(util.TokenToPrefix(rec.Token)) + if err != nil { + t.Error(err) + } + + // test case: Record not found + tokenRandom := tokenFromTreeID(123) + + // test case: Frozen tree + recFrozen, err := tlogBackend.New(md, fs) + if err != nil { + t.Error(err) + } + tokenFrozen, err := tokenDecode(recFrozen.Token) + if err != nil { + t.Error(err) + } + err = tlogBackend.unvetted.treeFreeze(treeIDFromToken(tokenFrozen), + backend.RecordMetadata{}, []backend.MetadataStream{}, 0) + if err != nil { + t.Error(err) + } + + // Setup UpdateUnvettedMetadata tests + var tests = []struct { + description string + token []byte + mdAppend, mdOverwrite []backend.MetadataStream + wantContentErr *backend.ContentVerificationError + wantErr error + }{ + { + "no changes to record metadata, empty streams", + token, + []backend.MetadataStream{}, + []backend.MetadataStream{}, + &backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusNoChanges, + }, + nil, + }, + { + "invalid token", + tokenShort, + []backend.MetadataStream{{ + ID: 2, + Payload: "random", + }}, + []backend.MetadataStream{}, + &backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidToken, + }, + nil, + }, + { + "record not found", + tokenRandom, + []backend.MetadataStream{{ + ID: 2, + Payload: "random", + }}, + []backend.MetadataStream{}, + nil, + backend.ErrRecordNotFound, + }, + { + "tree frozen for changes", + tokenFrozen, + []backend.MetadataStream{{ + ID: 2, + Payload: "random", + }}, + []backend.MetadataStream{}, + nil, + backend.ErrRecordLocked, + }, + { + "no changes to record metadata, same payload", + token, + []backend.MetadataStream{}, + []backend.MetadataStream{{ + ID: 1, + Payload: "", + }}, + nil, + backend.ErrNoChanges, + }, + { + "success", + token, + []backend.MetadataStream{}, + []backend.MetadataStream{{ + ID: 1, + Payload: "newdata", + }}, + nil, + nil, + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + // Make backend call + err = tlogBackend.UpdateUnvettedMetadata(test.token, + test.mdAppend, test.mdOverwrite) + // Parse error + var contentError backend.ContentVerificationError + if errors.As(err, &contentError) { + if test.wantContentErr == nil { + t.Errorf("got error %v, want nil", err) + } + if contentError.ErrorCode != test.wantContentErr.ErrorCode { + t.Errorf("got error %v, want %v", + v1.ErrorStatus[contentError.ErrorCode], + v1.ErrorStatus[test.wantContentErr.ErrorCode]) + } + return + } + + if test.wantErr != err { + t.Errorf("got error %v, want %v", err, test.wantErr) + } + }) + } } From f8c4b5954ed70e9dd03b2e5b199d471d74aa4e9a Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Mon, 14 Dec 2020 16:05:43 -0300 Subject: [PATCH 180/449] tlogbe: Add UpdateVettedRecord tests. --- politeiad/backend/tlogbe/tlogbe_test.go | 186 ++++++++++++++++++++++++ 1 file changed, 186 insertions(+) diff --git a/politeiad/backend/tlogbe/tlogbe_test.go b/politeiad/backend/tlogbe/tlogbe_test.go index 8d7004478..ad04c0d1f 100644 --- a/politeiad/backend/tlogbe/tlogbe_test.go +++ b/politeiad/backend/tlogbe/tlogbe_test.go @@ -218,6 +218,192 @@ func TestUpdateUnvettedRecord(t *testing.T) { } } +func TestUpdateVettedRecord(t *testing.T) { + tlogBackend, err := newTestTlogBackend(t) + if err != nil { + t.Error(err) + } + + // Create new record + md := []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs := []backend.File{ + newBackendFile(t, "index.md"), + } + rec, err := tlogBackend.New(md, fs) + if err != nil { + t.Error(err) + } + token, err := tokenDecode(rec.Token) + if err != nil { + t.Error(err) + } + md = append(md, backend.MetadataStream{ + ID: 2, + Payload: "", + }) + + // Publish the created record + err = tlogBackend.unvettedPublish(token, *rec, md, fs) + if err != nil { + t.Error(err) + } + + // Test all record content verification error through the + // UpdateVettedRecord endpoint + recordContentTests := setupRecordContentTests(t) + for _, test := range recordContentTests { + t.Run(test.description, func(t *testing.T) { + // Convert token + token, err := util.ConvertStringToken(rec.Token) + if err != nil { + t.Error(err) + } + + // Make backend call + _, err = tlogBackend.UpdateVettedRecord(token, test.metadata, + []backend.MetadataStream{}, test.files, test.filesDel) + + // Parse error + var contentError backend.ContentVerificationError + if errors.As(err, &contentError) { + if contentError.ErrorCode != test.err.ErrorCode { + t.Errorf("got error %v, want %v", + v1.ErrorStatus[contentError.ErrorCode], + v1.ErrorStatus[test.err.ErrorCode]) + } + } + }) + } + + // Random png image file to include in edit payload + imageRandom := newBackendFilePNG(t) + + // test case: Token not full length + tokenShort, err := util.ConvertStringToken(util.TokenToPrefix(rec.Token)) + if err != nil { + t.Error(err) + } + + // test case: Record not found + tokenRandom := tokenFromTreeID(123) + + // test case: Frozen tree + recFrozen, err := tlogBackend.New(md, fs) + if err != nil { + t.Error(err) + } + tokenFrozen, err := tokenDecode(recFrozen.Token) + if err != nil { + t.Error(err) + } + md = append(md, backend.MetadataStream{ + ID: 3, + Payload: "", + }) + err = tlogBackend.unvettedPublish(tokenFrozen, *recFrozen, md, fs) + if err != nil { + t.Error(err) + } + treeIDFrozenVetted := tlogBackend.vettedTreeIDs[recFrozen.Token] + err = tlogBackend.vetted.treeFreeze(treeIDFrozenVetted, + backend.RecordMetadata{}, []backend.MetadataStream{}, 0) + if err != nil { + t.Error(err) + } + + // Setup UpdateVettedRecord tests + var tests = []struct { + description string + token []byte + mdAppend, mdOverwirte []backend.MetadataStream + filesAdd []backend.File + filesDel []string + wantContentErr *backend.ContentVerificationError + wantErr error + }{ + { + "token not full length", + tokenShort, + []backend.MetadataStream{}, + []backend.MetadataStream{}, + []backend.File{imageRandom}, + []string{}, + &backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidToken, + }, + nil, + }, + { + "record not found", + tokenRandom, + []backend.MetadataStream{}, + []backend.MetadataStream{}, + []backend.File{imageRandom}, + []string{}, + nil, + backend.ErrRecordNotFound, + }, + { + "tree frozen for changes", + tokenFrozen, + []backend.MetadataStream{}, + []backend.MetadataStream{}, + []backend.File{imageRandom}, + []string{}, + nil, + backend.ErrRecordLocked, + }, + { + "no changes to record", + token, + []backend.MetadataStream{}, + []backend.MetadataStream{}, + []backend.File{}, + []string{}, + nil, + backend.ErrNoChanges, + }, + { + "success", + token, + []backend.MetadataStream{}, + []backend.MetadataStream{}, + []backend.File{imageRandom}, + []string{}, + nil, + nil, + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + // Make backend call + _, err = tlogBackend.UpdateVettedRecord(test.token, + test.mdAppend, test.mdOverwirte, test.filesAdd, test.filesDel) + + // Parse error + var contentError backend.ContentVerificationError + if errors.As(err, &contentError) { + if test.wantContentErr == nil { + t.Errorf("got error %v, want nil", err) + } + if contentError.ErrorCode != test.wantContentErr.ErrorCode { + t.Errorf("got error %v, want %v", + v1.ErrorStatus[contentError.ErrorCode], + v1.ErrorStatus[test.wantContentErr.ErrorCode]) + } + return + } + + if test.wantErr != err { + t.Errorf("got error %v, want %v", err, test.wantErr) + } + }) + } +} + func TestUpdateUnvettedMetadata(t *testing.T) { tlogBackend, err := newTestTlogBackend(t) if err != nil { From 7a8cb3477c2bac4752ef1d8c5ffa3228e75b0dc9 Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Mon, 14 Dec 2020 16:31:18 -0300 Subject: [PATCH 181/449] tlogbe: Add UpdateVettedMetadata tests. --- politeiad/backend/tlogbe/tlogbe_test.go | 188 ++++++++++++++++++++++++ 1 file changed, 188 insertions(+) diff --git a/politeiad/backend/tlogbe/tlogbe_test.go b/politeiad/backend/tlogbe/tlogbe_test.go index ad04c0d1f..809fbaa4f 100644 --- a/politeiad/backend/tlogbe/tlogbe_test.go +++ b/politeiad/backend/tlogbe/tlogbe_test.go @@ -574,3 +574,191 @@ func TestUpdateUnvettedMetadata(t *testing.T) { }) } } + +func TestUpdateVettedMetadata(t *testing.T) { + tlogBackend, err := newTestTlogBackend(t) + if err != nil { + t.Error(err) + } + + // Create new record + md := []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs := []backend.File{ + newBackendFile(t, "index.md"), + } + rec, err := tlogBackend.New(md, fs) + if err != nil { + t.Error(err) + } + token, err := tokenDecode(rec.Token) + if err != nil { + t.Error(err) + } + md = append(md, backend.MetadataStream{ + ID: 3, + Payload: "", + }) + err = tlogBackend.unvettedPublish(token, *rec, md, fs) + if err != nil { + t.Error(err) + } + + // Test all record content verification error through the + // UpdateVettedMetadata endpoint + recordContentTests := setupRecordContentTests(t) + for _, test := range recordContentTests { + t.Run(test.description, func(t *testing.T) { + // Make backend call + err := tlogBackend.UpdateVettedMetadata(token, + test.metadata, []backend.MetadataStream{}) + + // Parse error + var contentError backend.ContentVerificationError + if errors.As(err, &contentError) { + if contentError.ErrorCode != test.err.ErrorCode { + t.Errorf("got error %v, want %v", + v1.ErrorStatus[contentError.ErrorCode], + v1.ErrorStatus[test.err.ErrorCode]) + } + } + }) + } + + // test case: Token not full length + tokenShort, err := util.ConvertStringToken(util.TokenToPrefix(rec.Token)) + if err != nil { + t.Error(err) + } + + // test case: Record not found + tokenRandom := tokenFromTreeID(123) + + // test case: Frozen tree + recFrozen, err := tlogBackend.New(md, fs) + if err != nil { + t.Error(err) + } + tokenFrozen, err := tokenDecode(recFrozen.Token) + if err != nil { + t.Error(err) + } + md = append(md, backend.MetadataStream{ + ID: 4, + Payload: "", + }) + err = tlogBackend.unvettedPublish(tokenFrozen, *recFrozen, md, fs) + if err != nil { + t.Error(err) + } + treeIDFrozenVetted := tlogBackend.vettedTreeIDs[recFrozen.Token] + err = tlogBackend.vetted.treeFreeze(treeIDFrozenVetted, + backend.RecordMetadata{}, []backend.MetadataStream{}, 0) + if err != nil { + t.Error(err) + } + + // Setup UpdateVettedMetadata tests + var tests = []struct { + description string + token []byte + mdAppend, mdOverwrite []backend.MetadataStream + wantContentErr *backend.ContentVerificationError + wantErr error + }{ + { + "no changes to record metadata, empty streams", + token, + []backend.MetadataStream{}, + []backend.MetadataStream{}, + &backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusNoChanges, + }, + nil, + }, + { + "invalid token", + tokenShort, + []backend.MetadataStream{{ + ID: 2, + Payload: "random", + }}, + []backend.MetadataStream{}, + &backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidToken, + }, + nil, + }, + { + "record not found", + tokenRandom, + []backend.MetadataStream{{ + ID: 2, + Payload: "random", + }}, + []backend.MetadataStream{}, + nil, + backend.ErrRecordNotFound, + }, + { + "tree frozen for changes", + tokenFrozen, + []backend.MetadataStream{{ + ID: 2, + Payload: "random", + }}, + []backend.MetadataStream{}, + nil, + backend.ErrRecordLocked, + }, + { + "no changes to record metadata, same payload", + token, + []backend.MetadataStream{}, + []backend.MetadataStream{{ + ID: 3, + Payload: "", + }}, + nil, + backend.ErrNoChanges, + }, + { + "success", + token, + []backend.MetadataStream{}, + []backend.MetadataStream{{ + ID: 1, + Payload: "newdata", + }}, + nil, + nil, + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + // Make backend call + err = tlogBackend.UpdateVettedMetadata(test.token, + test.mdAppend, test.mdOverwrite) + + // Parse error + var contentError backend.ContentVerificationError + if errors.As(err, &contentError) { + if test.wantContentErr == nil { + t.Errorf("got error %v, want nil", err) + } + if contentError.ErrorCode != test.wantContentErr.ErrorCode { + t.Errorf("got error %v, want %v", + v1.ErrorStatus[contentError.ErrorCode], + v1.ErrorStatus[test.wantContentErr.ErrorCode]) + } + return + } + + if test.wantErr != err { + t.Errorf("got error %v, want %v", err, test.wantErr) + } + }) + } +} From 0f133422d15a2ce5504eed8a2eae749f63d81ef2 Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Mon, 14 Dec 2020 16:32:24 -0300 Subject: [PATCH 182/449] tlogbe: Add UnvettedExists tests. --- politeiad/backend/tlogbe/tlogbe_test.go | 39 +++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/politeiad/backend/tlogbe/tlogbe_test.go b/politeiad/backend/tlogbe/tlogbe_test.go index 809fbaa4f..f33a568ae 100644 --- a/politeiad/backend/tlogbe/tlogbe_test.go +++ b/politeiad/backend/tlogbe/tlogbe_test.go @@ -762,3 +762,42 @@ func TestUpdateVettedMetadata(t *testing.T) { }) } } + +func TestUnvettedExists(t *testing.T) { + tlogBackend, err := newTestTlogBackend(t) + if err != nil { + t.Error(err) + } + + // Create new record + md := []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs := []backend.File{ + newBackendFile(t, "index.md"), + } + rec, err := tlogBackend.New(md, fs) + if err != nil { + t.Error(err) + } + token, err := tokenDecode(rec.Token) + if err != nil { + t.Error(err) + } + + // Random token + tokenRandom := tokenFromTreeID(123) + + // Run UnvettedExists test cases + // + // Record exists + result := tlogBackend.UnvettedExists(token) + if result == false { + t.Errorf("got false, want true") + } + // Record does not exist + result = tlogBackend.UnvettedExists(tokenRandom) + if result == true { + t.Errorf("got true, want false") + } +} From 3b8f3dd1cd1ba71b802b99e2380dccb0e409db86 Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Mon, 14 Dec 2020 16:32:48 -0300 Subject: [PATCH 183/449] tlogbe: Add VettedExists tests. --- politeiad/backend/tlogbe/tlogbe_test.go | 53 +++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/politeiad/backend/tlogbe/tlogbe_test.go b/politeiad/backend/tlogbe/tlogbe_test.go index f33a568ae..8b9e8e56d 100644 --- a/politeiad/backend/tlogbe/tlogbe_test.go +++ b/politeiad/backend/tlogbe/tlogbe_test.go @@ -801,3 +801,56 @@ func TestUnvettedExists(t *testing.T) { t.Errorf("got true, want false") } } +func TestVettedExists(t *testing.T) { + tlogBackend, err := newTestTlogBackend(t) + if err != nil { + t.Error(err) + } + + // Create unvetted record + md := []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs := []backend.File{ + newBackendFile(t, "index.md"), + } + unvetted, err := tlogBackend.New(md, fs) + if err != nil { + t.Error(err) + } + tokenUnvetted, err := tokenDecode(unvetted.Token) + if err != nil { + t.Error(err) + } + + // Create vetted record + vetted, err := tlogBackend.New(md, fs) + if err != nil { + t.Error(err) + } + tokenVetted, err := tokenDecode(vetted.Token) + if err != nil { + t.Error(err) + } + md = append(md, backend.MetadataStream{ + ID: 2, + Payload: "", + }) + err = tlogBackend.unvettedPublish(tokenVetted, *vetted, md, fs) + if err != nil { + t.Error(err) + } + + // Run VettedExists test cases + // + // Record exists + result := tlogBackend.VettedExists(tokenVetted) + if result == false { + t.Errorf("got false, want true") + } + // Record does not exist + result = tlogBackend.VettedExists(tokenUnvetted) + if result == true { + t.Errorf("got true, want false") + } +} From a46d4b4b7e1e63169b16c93468cc76be69e2f536 Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Mon, 14 Dec 2020 16:33:07 -0300 Subject: [PATCH 184/449] tlogbe: Add GetUnvetted and GetVetted tests. --- politeiad/backend/tlogbe/tlogbe_test.go | 88 +++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/politeiad/backend/tlogbe/tlogbe_test.go b/politeiad/backend/tlogbe/tlogbe_test.go index 8b9e8e56d..69b6915d8 100644 --- a/politeiad/backend/tlogbe/tlogbe_test.go +++ b/politeiad/backend/tlogbe/tlogbe_test.go @@ -854,3 +854,91 @@ func TestVettedExists(t *testing.T) { t.Errorf("got true, want false") } } + +func TestGetUnvetted(t *testing.T) { + tlogBackend, err := newTestTlogBackend(t) + if err != nil { + t.Error(err) + } + + // Create new record + md := []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs := []backend.File{ + newBackendFile(t, "index.md"), + } + rec, err := tlogBackend.New(md, fs) + if err != nil { + t.Error(err) + } + token, err := tokenDecode(rec.Token) + if err != nil { + t.Error(err) + } + + // Random token + tokenRandom := tokenFromTreeID(123) + + // Bad version error + _, err = tlogBackend.GetUnvetted(token, "badversion") + if err != backend.ErrRecordNotFound { + t.Errorf("got error %v, want %v", err, backend.ErrRecordNotFound) + } + + // Bad token error + _, err = tlogBackend.GetUnvetted(tokenRandom, "") + if err != backend.ErrRecordNotFound { + t.Errorf("got error %v, want %v", err, backend.ErrRecordNotFound) + } + + // Success + _, err = tlogBackend.GetUnvetted(token, "") + if err != nil { + t.Errorf("got error %v, want nil", err) + } +} + +func TestGetVetted(t *testing.T) { + tlogBackend, err := newTestTlogBackend(t) + if err != nil { + t.Error(err) + } + + // Create new record + md := []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs := []backend.File{ + newBackendFile(t, "index.md"), + } + rec, err := tlogBackend.New(md, fs) + if err != nil { + t.Error(err) + } + token, err := tokenDecode(rec.Token) + if err != nil { + t.Error(err) + } + + // Random token + tokenRandom := tokenFromTreeID(123) + + // Bad version error + _, err = tlogBackend.GetUnvetted(token, "badversion") + if err != backend.ErrRecordNotFound { + t.Errorf("got error %v, want %v", err, backend.ErrRecordNotFound) + } + + // Bad token error + _, err = tlogBackend.GetUnvetted(tokenRandom, "") + if err != backend.ErrRecordNotFound { + t.Errorf("got error %v, want %v", err, backend.ErrRecordNotFound) + } + + // Success + _, err = tlogBackend.GetUnvetted(token, "") + if err != nil { + t.Errorf("got error %v, want nil", err) + } +} From fd53b4e708476f0cac1ae0fb5c7c12914fbe8a4b Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Tue, 15 Dec 2020 11:51:14 -0300 Subject: [PATCH 185/449] tlogbe: Add SetUnvettedStatus and SetVettedStatus tests. --- politeiad/backend/tlogbe/tlogbe.go | 2 +- politeiad/backend/tlogbe/tlogbe_test.go | 384 +++++++++++++++++++++++- 2 files changed, 382 insertions(+), 4 deletions(-) diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index c104cd151..b0465b5f9 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -1403,7 +1403,7 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, rm.Iteration += 1 rm.Timestamp = time.Now().Unix() - // Apply metdata changes + // Apply metadata changes metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) // Call pre plugin hooks diff --git a/politeiad/backend/tlogbe/tlogbe_test.go b/politeiad/backend/tlogbe/tlogbe_test.go index 69b6915d8..b7416a7ce 100644 --- a/politeiad/backend/tlogbe/tlogbe_test.go +++ b/politeiad/backend/tlogbe/tlogbe_test.go @@ -801,6 +801,7 @@ func TestUnvettedExists(t *testing.T) { t.Errorf("got true, want false") } } + func TestVettedExists(t *testing.T) { tlogBackend, err := newTestTlogBackend(t) if err != nil { @@ -920,25 +921,402 @@ func TestGetVetted(t *testing.T) { if err != nil { t.Error(err) } + md = append(md, backend.MetadataStream{ + ID: 2, + Payload: "", + }) + err = tlogBackend.unvettedPublish(token, *rec, md, fs) + if err != nil { + t.Error(err) + } // Random token tokenRandom := tokenFromTreeID(123) // Bad version error - _, err = tlogBackend.GetUnvetted(token, "badversion") + _, err = tlogBackend.GetVetted(token, "badversion") if err != backend.ErrRecordNotFound { t.Errorf("got error %v, want %v", err, backend.ErrRecordNotFound) } // Bad token error - _, err = tlogBackend.GetUnvetted(tokenRandom, "") + _, err = tlogBackend.GetVetted(tokenRandom, "") if err != backend.ErrRecordNotFound { t.Errorf("got error %v, want %v", err, backend.ErrRecordNotFound) } // Success - _, err = tlogBackend.GetUnvetted(token, "") + _, err = tlogBackend.GetVetted(token, "") if err != nil { t.Errorf("got error %v, want nil", err) } } + +func TestSetUnvettedStatus(t *testing.T) { + tlogBackend, err := newTestTlogBackend(t) + if err != nil { + t.Error(err) + } + + // Helpers + md := []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs := []backend.File{ + newBackendFile(t, "index.md"), + } + + // Invalid status transitions + // + // test case: Unvetted to archived + recUnvetToArch, err := tlogBackend.New(md, fs) + if err != nil { + t.Error(err) + } + tokenUnvetToArch, err := tokenDecode(recUnvetToArch.Token) + if err != nil { + t.Error(err) + } + // test case: Unvetted to unvetted + recUnvetToUnvet, err := tlogBackend.New(md, fs) + if err != nil { + t.Error(err) + } + tokenUnvetToUnvet, err := tokenDecode(recUnvetToUnvet.Token) + if err != nil { + t.Error(err) + } + + // Valid status transitions + // + // test case: Unvetted to vetted + recUnvetToVet, err := tlogBackend.New(md, fs) + if err != nil { + t.Error(err) + } + tokenUnvetToVet, err := tokenDecode(recUnvetToVet.Token) + if err != nil { + t.Error(err) + } + // test case: Unvetted to censored + recUnvetToCensored, err := tlogBackend.New(md, fs) + if err != nil { + t.Error(err) + } + tokenUnvetToCensored, err := tokenDecode(recUnvetToCensored.Token) + if err != nil { + t.Error(err) + } + + // test case: Token not full length + tokenShort, err := util.ConvertStringToken( + util.TokenToPrefix(recUnvetToVet.Token)) + if err != nil { + t.Error(err) + } + + // test case: Record not found + tokenRandom := tokenFromTreeID(123) + + // Setup SetUnvettedStatus tests + var tests = []struct { + description string + token []byte + status backend.MDStatusT + mdAppend, mdOverwrite []backend.MetadataStream + wantContentErr *backend.ContentVerificationError + wantErr error + }{ + { + "invalid: unvetted to archived", + tokenUnvetToArch, + backend.MDStatusArchived, + []backend.MetadataStream{}, + []backend.MetadataStream{}, + nil, + backend.StateTransitionError{ + From: recUnvetToArch.Status, + To: backend.MDStatusArchived, + }, + }, + { + "invalid: unvetted to unvetted", + tokenUnvetToUnvet, + backend.MDStatusUnvetted, + []backend.MetadataStream{}, + []backend.MetadataStream{}, + nil, + backend.StateTransitionError{ + From: recUnvetToArch.Status, + To: backend.MDStatusUnvetted, + }, + }, + { + "valid: unvetted to vetted", + tokenUnvetToVet, + backend.MDStatusVetted, + []backend.MetadataStream{}, + []backend.MetadataStream{}, + nil, + nil, + }, + { + "valid: unvetted to censored", + tokenUnvetToCensored, + backend.MDStatusCensored, + []backend.MetadataStream{}, + []backend.MetadataStream{}, + nil, + nil, + }, + { + "invalid token", + tokenShort, + backend.MDStatusCensored, + []backend.MetadataStream{}, + []backend.MetadataStream{}, + &backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidToken, + }, + nil, + }, + { + "record not found", + tokenRandom, + backend.MDStatusCensored, + []backend.MetadataStream{}, + []backend.MetadataStream{}, + nil, + backend.ErrRecordNotFound, + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + // Make backend call + _, err = tlogBackend.SetUnvettedStatus(test.token, test.status, + test.mdAppend, test.mdOverwrite) + + // Parse error + var contentError backend.ContentVerificationError + if errors.As(err, &contentError) { + if test.wantContentErr == nil { + t.Errorf("got error %v, want nil", err) + } + if contentError.ErrorCode != test.wantContentErr.ErrorCode { + t.Errorf("got error %v, want %v", + v1.ErrorStatus[contentError.ErrorCode], + v1.ErrorStatus[test.wantContentErr.ErrorCode]) + } + return + } + + if test.wantErr != err { + t.Errorf("got error %v, want %v", err, test.wantErr) + } + }) + } +} + +func TestSetVettedStatus(t *testing.T) { + tlogBackend, err := newTestTlogBackend(t) + if err != nil { + t.Error(err) + } + + // Helpers + md := []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs := []backend.File{ + newBackendFile(t, "index.md"), + } + + // Invalid status transitions + // + // test case: Vetted to unvetted + recVetToUnvet, err := tlogBackend.New(md, fs) + if err != nil { + t.Error(err) + } + tokenVetToUnvet, err := tokenDecode(recVetToUnvet.Token) + if err != nil { + t.Error(err) + } + + md = append(md, backend.MetadataStream{ + ID: 2, + Payload: "", + }) + _, err = tlogBackend.SetUnvettedStatus(tokenVetToUnvet, + backend.MDStatusVetted, md, []backend.MetadataStream{}) + if err != nil { + t.Error(err) + } + // test case: Vetted to vetted + recVetToVet, err := tlogBackend.New(md, fs) + if err != nil { + t.Error(err) + } + tokenVetToVet, err := tokenDecode(recVetToVet.Token) + if err != nil { + t.Error(err) + } + md = append(md, backend.MetadataStream{ + ID: 3, + Payload: "", + }) + _, err = tlogBackend.SetUnvettedStatus(tokenVetToVet, + backend.MDStatusVetted, md, []backend.MetadataStream{}) + if err != nil { + t.Error(err) + } + + // Valid status transitions + // + // test case: Vetted to archived + recVetToArch, err := tlogBackend.New(md, fs) + if err != nil { + t.Error(err) + } + tokenVetToArch, err := tokenDecode(recVetToArch.Token) + if err != nil { + t.Error(err) + } + md = append(md, backend.MetadataStream{ + ID: 4, + Payload: "", + }) + _, err = tlogBackend.SetUnvettedStatus(tokenVetToArch, + backend.MDStatusVetted, md, []backend.MetadataStream{}) + if err != nil { + t.Error(err) + } + // test case: Vetted to censored + recVetToCensored, err := tlogBackend.New(md, fs) + if err != nil { + t.Error(err) + } + tokenVetToCensored, err := tokenDecode(recVetToCensored.Token) + if err != nil { + t.Error(err) + } + md = append(md, backend.MetadataStream{ + ID: 5, + Payload: "", + }) + _, err = tlogBackend.SetUnvettedStatus(tokenVetToCensored, + backend.MDStatusVetted, md, []backend.MetadataStream{}) + if err != nil { + t.Error(err) + } + + // test case: Token not full length + tokenShort, err := util.ConvertStringToken( + util.TokenToPrefix(recVetToCensored.Token)) + if err != nil { + t.Error(err) + } + + // test case: Record not found + tokenRandom := tokenFromTreeID(123) + + // Setup SetVettedStatus tests + var tests = []struct { + description string + token []byte + status backend.MDStatusT + mdAppend, mdOverwrite []backend.MetadataStream + wantContentErr *backend.ContentVerificationError + wantErr error + }{ + { + "invalid: vetted to unvetted", + tokenVetToUnvet, + backend.MDStatusUnvetted, + []backend.MetadataStream{}, + []backend.MetadataStream{}, + nil, + backend.StateTransitionError{ + From: backend.MDStatusVetted, + To: backend.MDStatusUnvetted, + }, + }, + { + "invalid: vetted to vetted", + tokenVetToVet, + backend.MDStatusVetted, + []backend.MetadataStream{}, + []backend.MetadataStream{}, + nil, + backend.StateTransitionError{ + From: backend.MDStatusVetted, + To: backend.MDStatusVetted, + }, + }, + { + "valid: vetted to archived", + tokenVetToArch, + backend.MDStatusArchived, + []backend.MetadataStream{}, + []backend.MetadataStream{}, + nil, + nil, + }, + { + "valid: vetted to censored", + tokenVetToCensored, + backend.MDStatusCensored, + []backend.MetadataStream{}, + []backend.MetadataStream{}, + nil, + nil, + }, + { + "invalid token", + tokenShort, + backend.MDStatusCensored, + []backend.MetadataStream{}, + []backend.MetadataStream{}, + &backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidToken, + }, + nil, + }, + { + "record not found", + tokenRandom, + backend.MDStatusCensored, + []backend.MetadataStream{}, + []backend.MetadataStream{}, + nil, + backend.ErrRecordNotFound, + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + // Make backend call + _, err = tlogBackend.SetVettedStatus(test.token, test.status, + test.mdAppend, test.mdOverwrite) + + // Parse error + var contentError backend.ContentVerificationError + if errors.As(err, &contentError) { + if test.wantContentErr == nil { + t.Errorf("got error %v, want nil", err) + } + if contentError.ErrorCode != test.wantContentErr.ErrorCode { + t.Errorf("got error %v, want %v", + v1.ErrorStatus[contentError.ErrorCode], + v1.ErrorStatus[test.wantContentErr.ErrorCode]) + } + return + } + + if test.wantErr != err { + t.Errorf("got error %v, want %v", err, test.wantErr) + } + }) + } +} From 82783cdbd458f6d340e46e95b916a257a28b1fd5 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 25 Nov 2020 08:12:49 -0600 Subject: [PATCH 186/449] Start of user proposals refactor --- politeiad/backend/tlogbe/tlog.go | 1 - politeiad/plugins/pi/pi.go | 59 +++++++++++++++++++++++++++----- politeiawww/api/pi/v1/v1.go | 16 +++++---- politeiawww/pi.go | 18 ++++++++++ politeiawww/piwww.go | 53 +++++++++++++++------------- 5 files changed, 109 insertions(+), 38 deletions(-) diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index 343855ac7..d2f7fcdde 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -32,7 +32,6 @@ const ( dataDescriptorRecordMetadata = "recordmetadata" dataDescriptorMetadataStream = "metadatastream" dataDescriptorRecordIndex = "recordindex" - dataDescriptorFreezeRecord = "freezerecord" dataDescriptorAnchor = "anchor" // The keys for kv store blobs are saved by stuffing them into the diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index b2806a2a9..0dd099e1c 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -24,19 +24,16 @@ const ( ID = "pi" Version = "1" + // Plugin commands // TODO refactor these commands to use the passthrough command - - // Plugin commands. Many of these plugin commands rely on the - // commands from other plugins, but perform additional validation - // that is specific to pi or add additional functionality on top of - // the existing plugin commands that is specific to pi. CmdProposals = "proposals" // Get plugin data for proposals CmdCommentNew = "commentnew" // Create a new comment CmdCommentCensor = "commentcensor" // Censor a comment CmdCommentVote = "commentvote" // Upvote/downvote a comment - CmdVoteInventory = "voteinventory" // Get inventory by vote status - CmdVoteStart = "votestart" // Start a vote - CmdPassThrough = "passthrough" // Pass a plugin cmd through pi + + CmdProposalInventory = "proposalinv" // Get inventory by proposal status + CmdVoteInventory = "voteinv" // Get inventory by vote status + CmdPassThrough = "passthrough" // Pass a plugin cmd through pi // Metadata stream IDs MDStreamIDProposalGeneral = 1 @@ -428,6 +425,52 @@ func DecodeCommentVoteReply(payload []byte) (*CommentVoteReply, error) { return &cvr, nil } +// ProposalInventory retrieves the tokens of all proposals in the inventory, +// categorized by proposal state and proposal status, that match the provided +// filtering criteria. If no filtering criteria is provided then the full +// proposal inventory is returned. +type ProposalInventory struct { + UserID string `json:"userid,omitempty"` +} + +// EncodeProposalInventory encodes a ProposalInventory into a JSON byte slice. +func EncodeProposalInventory(pi ProposalInventory) ([]byte, error) { + return json.Marshal(pi) +} + +// DecodeProposalInventory decodes a JSON byte slice into a ProposalInventory. +func DecodeProposalInventory(payload []byte) (*ProposalInventory, error) { + var pi ProposalInventory + err := json.Unmarshal(payload, &pi) + if err != nil { + return nil, err + } + return &pi, nil +} + +// ProposalInventoryReply is the reply to the ProposalInventory command. +type ProposalInventoryReply struct { + Unvetted map[string][]string `json:"unvetted"` + Vetted map[string][]string `json:"vetted"` +} + +// EncodeProposalInventoryReply encodes a ProposalInventoryReply into a JSON +// byte slice. +func EncodeProposalInventoryReply(pir ProposalInventoryReply) ([]byte, error) { + return json.Marshal(pir) +} + +// DecodeProposalInventoryReply decodes a JSON byte slice into a +// ProposalInventoryReply. +func DecodeProposalInventoryReply(payload []byte) (*ProposalInventoryReply, error) { + var pir ProposalInventoryReply + err := json.Unmarshal(payload, &pir) + if err != nil { + return nil, err + } + return &pir, nil +} + // VoteInventory requests the tokens of all proposals in the inventory // categorized by their vote status. This call relies on the ticketvote // Inventory call, but breaks the Finished vote status out into Approved and diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 59f918424..b6f84689d 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -463,7 +463,7 @@ type ProposalRequest struct { // to false. type Proposals struct { State PropStateT `json:"state"` - Requests []ProposalRequest `json:"requests"` + Requests []ProposalRequest `json:"requests,omitempty"` IncludeFiles bool `json:"includefiles,omitempty"` } @@ -474,15 +474,19 @@ type ProposalsReply struct { } // ProposalInventory retrieves the tokens of all proposals in the inventory, -// categorized by proposal staet and proposal status. Each list is ordered by -// timestamp of the status change from newest to oldest. Unvetted proposal -// tokens are only returned to admins. -type ProposalInventory struct{} +// categorized by proposal state and proposal status, that match the provided +// filtering criteria. If no filtering criteria is provided then the full +// proposal inventory is returned. Unvetted proposal tokens are only returned +// to admins and the proposal author. +type ProposalInventory struct { + UserID string `json:"userid,omitempty"` +} // ProposalInventoryReply is the reply to the ProposalInventory command. The // inventory maps contain map[status][]tokens where the status is the human // readable proposal status, as defined by the PropStatus map, and the tokens -// are the proposal tokens for that status. +// are a list of proposal tokens for that status. Each list is ordered by +// timestamp of the status change from newest to oldest. type ProposalInventoryReply struct { Unvetted map[string][]string `json:"unvetted,omitempty"` Vetted map[string][]string `json:"vetted"` diff --git a/politeiawww/pi.go b/politeiawww/pi.go index 9905ad664..6c2e48d86 100644 --- a/politeiawww/pi.go +++ b/politeiawww/pi.go @@ -83,6 +83,24 @@ func (p *politeiawww) piCommentCensor(ctx context.Context, cc piplugin.CommentCe return ccr, nil } +// proposalInventory returns the pi plugin proposal inventory. +func (p *politeiawww) proposalInventory(ctx context.Context, inv piplugin.ProposalInventory) (*piplugin.ProposalInventoryReply, error) { + b, err := piplugin.EncodeProposalInventory(inv) + if err != nil { + return nil, err + } + r, err := p.pluginCommand(ctx, piplugin.ID, + piplugin.CmdProposalInventory, string(b)) + if err != nil { + return nil, err + } + reply, err := piplugin.DecodeProposalInventoryReply(([]byte(r))) + if err != nil { + return nil, err + } + return reply, nil +} + // piVoteInventory returns the pi plugin vote inventory. func (p *politeiawww) piVoteInventory(ctx context.Context) (*piplugin.VoteInventoryReply, error) { r, err := p.pluginCommand(ctx, piplugin.ID, piplugin.CmdVoteInventory, "") diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index e8b2e7438..44792b26f 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -1573,34 +1573,32 @@ func (p *politeiawww) processProposals(ctx context.Context, ps pi.Proposals, isA }, nil } -func (p *politeiawww) processProposalInventory(ctx context.Context, isAdmin bool) (*pi.ProposalInventoryReply, error) { - log.Tracef("processProposalInventory: %v", isAdmin) +func (p *politeiawww) processProposalInventory(ctx context.Context, inv pi.ProposalInventory, u *user.User) (*pi.ProposalInventoryReply, error) { + log.Tracef("processProposalInventory: %v", inv.UserID) - ir, err := p.inventoryByStatus(ctx) + // Send plugin command + i := piplugin.ProposalInventory{ + UserID: inv.UserID, + } + pir, err := p.proposalInventory(ctx, i) if err != nil { return nil, err } - var ( - unvetted = make(map[string][]string, len(ir.Unvetted)) - vetted = make(map[string][]string, len(ir.Vetted)) - ) - for status, tokens := range ir.Unvetted { - s := convertPropStatusFromPD(status) - unvetted[pi.PropStatus[s]] = tokens - } - for status, tokens := range ir.Vetted { - s := convertPropStatusFromPD(status) - vetted[pi.PropStatus[s]] = tokens - } - // Only return unvetted tokens to admins - if !isAdmin { - unvetted = nil + // Determine if unvetted tokens should be returned + switch { + case u.Admin: + // User is an admin. Return unvetted. + case inv.UserID == u.ID.String(): + // User is requesting their own proposals. Return unvetted. + default: + // Remove unvetted for all other cases + pir.Unvetted = nil } return &pi.ProposalInventoryReply{ - Unvetted: unvetted, - Vetted: vetted, + Unvetted: pir.Unvetted, + Vetted: pir.Vetted, }, nil } @@ -2175,6 +2173,16 @@ func (p *politeiawww) handleProposals(w http.ResponseWriter, r *http.Request) { func (p *politeiawww) handleProposalInventory(w http.ResponseWriter, r *http.Request) { log.Tracef("handleProposalInventory") + var inv pi.ProposalInventory + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&inv); err != nil { + respondWithPiError(w, r, "handleProposalInventory: unmarshal", + pi.UserErrorReply{ + ErrorCode: pi.ErrorStatusInputInvalid, + }) + return + } + // Lookup session user. This is a public route so a session may not // exist. Ignore any session not found errors. usr, err := p.getSessionUser(w, r) @@ -2184,15 +2192,14 @@ func (p *politeiawww) handleProposalInventory(w http.ResponseWriter, r *http.Req return } - isAdmin := usr != nil && usr.Admin - ppi, err := p.processProposalInventory(r.Context(), isAdmin) + pir, err := p.processProposalInventory(r.Context(), inv, usr) if err != nil { respondWithPiError(w, r, "handleProposalInventory: processProposalInventory: %v", err) return } - util.RespondWithJSON(w, http.StatusOK, ppi) + util.RespondWithJSON(w, http.StatusOK, pir) } func (p *politeiawww) handleCommentNew(w http.ResponseWriter, r *http.Request) { From 8933154a186f01187ed0ee4ecdcc036eb44506e2 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 7 Dec 2020 13:08:31 -0600 Subject: [PATCH 187/449] util: Remove tests that ping dcrtime server. --- util/dcrtime_test.go | 101 ------------------------------------------- 1 file changed, 101 deletions(-) delete mode 100644 util/dcrtime_test.go diff --git a/util/dcrtime_test.go b/util/dcrtime_test.go deleted file mode 100644 index 7410aff29..000000000 --- a/util/dcrtime_test.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package util - -import ( - "crypto/sha256" - "encoding/base64" - "encoding/hex" - "fmt" - "io/ioutil" - "os" - "testing" - - "github.com/decred/dcrtime/merkle" -) - -func tempFile(size int) string { - tmpfile, err := ioutil.TempFile("", "randomcontent") - if err != nil { - panic(fmt.Sprintf("%v", err)) - } - blob, err := Random(size) - if err != nil { - panic(fmt.Sprintf("%v", err)) - } - data := base64.StdEncoding.EncodeToString(blob) - err = ioutil.WriteFile(tmpfile.Name(), []byte(data), 0644) - if err != nil { - panic(fmt.Sprintf("%v", err)) - } - - return tmpfile.Name() -} - -func TestTimestamp(t *testing.T) { - if testing.Short() { - t.Skip("skipping TestTimestamp in short mode.") - } - - filename := tempFile(512) - defer os.Remove(filename) - - // use util internal functions ito get some free testing - _, digest, _, err := LoadFile(filename) - if err != nil { - t.Fatalf("%v", err) - } - d, ok := ConvertDigest(digest) - if !ok { - t.Fatalf("not a valid digest") - } - - err = Timestamp("test", defaultTestnetHost(), []*[sha256.Size]byte{&d}) - if err != nil { - t.Fatalf("%v", err) - } -} - -func TestVerify(t *testing.T) { - if testing.Short() { - t.Skip("skipping TestVerify in short mode.") - } - - // Use pre anchored digest - digest := "44425372a555e6ac6dad89d5a5b05cd33385c0c9114c5e90e7861da31ae2f289" - - vr, err := Verify("test", defaultTestnetHost(), []string{digest}) - if err != nil { - t.Fatalf("%v", err) - } - - // Verify reply - if len(vr.Digests) != 1 { - t.Fatalf("expected 1 response, got %v", len(vr.Digests)) - } - d := vr.Digests[0] - if digest != d.Digest { - t.Fatalf("invalid digest expected %v got %v", digest, d.Digest) - } - expectedTX := "2aeb59d362752e73757e4b88812da784fe2f4118d2e28121f286b9fa0ef50c1d" - if expectedTX != d.ChainInformation.Transaction { - t.Fatalf("invalid tx expected %v got %v", expectedTX, - d.ChainInformation.Transaction) - } - expectedMerkle := "74bdabc1613ab132490e59fe0551bca62a29c90eca88edeba650d021ac5eaecb" - if expectedMerkle != d.ChainInformation.MerkleRoot { - t.Fatalf("invalid merkle expected %v got %v", expectedMerkle, - d.ChainInformation.MerkleRoot) - } - - // Verify merkle root despite it being done Verify call - root, err := merkle.VerifyAuthPath(&d.ChainInformation.MerklePath) - if err != nil { - t.Fatalf("%v", err) - } - if expectedMerkle != hex.EncodeToString(root[:]) { - t.Fatalf("unexpected merkle %v", hex.EncodeToString(root[:])) - } -} From 531d78a0bfb291bbb9bd2461aca0ed7581455935 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 7 Dec 2020 13:44:00 -0600 Subject: [PATCH 188/449] tlogbe: Don't allow plugin writes to use token prefix. --- politeiad/backend/tlogbe/ticketvote.go | 3 --- politeiad/backend/tlogbe/tlogclient.go | 16 +++++++++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index 5c2e16e5d..38c2defff 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -2372,9 +2372,6 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. // Check if the vote has been authorized auths, err := p.authorizes(token) if err != nil { - if errors.Is(err, errRecordNotFound) { - return nil, err - } return nil, fmt.Errorf("authorizes: %v", err) } if len(auths) > 0 { diff --git a/politeiad/backend/tlogbe/tlogclient.go b/politeiad/backend/tlogbe/tlogclient.go index 8611c6836..e6e4a906b 100644 --- a/politeiad/backend/tlogbe/tlogclient.go +++ b/politeiad/backend/tlogbe/tlogclient.go @@ -57,7 +57,7 @@ func (c *backendClient) tlogByID(tlogID string) (*tlog, error) { } // treeIDFromToken returns the treeID for the provided tlog instance ID and -// token. +// token. This function accepts both token prefixes and full length tokens. func (c *backendClient) treeIDFromToken(tlogID string, token []byte) (int64, error) { if len(token) == tokenPrefixSize() { // This is a token prefix. Get the full token from the cache. @@ -78,9 +78,19 @@ func (c *backendClient) treeIDFromToken(tlogID string, token []byte) (int64, err } return treeID, nil } + return 0, fmt.Errorf("unknown tlog id '%v'", tlogID) } +// treeIDFromToken returns the treeID for the provided tlog instance ID and +// token. This function only accepts full length tokens. +func (c *backendClient) treeIDFromTokenFullLength(tlogID string, token []byte) (int64, error) { + if !tokenIsFullLength(token) { + return 0, errRecordNotFound + } + return c.treeIDFromToken(tlogID, token) +} + // save saves the provided blobs to the tlog backend. Note, hashes contains the // hashes of the data encoded in the blobs. The hashes must share the same // ordering as the blobs. @@ -94,7 +104,7 @@ func (c *backendClient) save(tlogID string, token []byte, keyPrefix string, blob } // Get tree ID - treeID, err := c.treeIDFromToken(tlogID, token) + treeID, err := c.treeIDFromTokenFullLength(tlogID, token) if err != nil { return nil, err } @@ -115,7 +125,7 @@ func (c *backendClient) del(tlogID string, token []byte, merkles [][]byte) error } // Get tree ID - treeID, err := c.treeIDFromToken(tlogID, token) + treeID, err := c.treeIDFromTokenFullLength(tlogID, token) if err != nil { return err } From 147f847758b9f0159b33a6efca53e85c37cba67a Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 18 Dec 2020 09:25:24 -0600 Subject: [PATCH 189/449] tlogbe: Prevent encryption key panic. --- politeiad/backend/tlogbe/tlog.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index d2f7fcdde..b07c923af 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -367,6 +367,14 @@ func convertAnchorFromBlobEntry(be store.BlobEntry) (*anchor, error) { return &a, nil } +func (t *tlog) encrypt(b []byte) ([]byte, error) { + if t.encryptionKey == nil { + return nil, fmt.Errorf("cannot encrypt blob; encryption key "+ + "not set for tlog instance %v", t.id) + } + return t.encryptionKey.encrypt(0, b) +} + func (t *tlog) treeNew() (int64, error) { log.Tracef("%v treeNew", t.id) @@ -1543,7 +1551,7 @@ func (t *tlog) blobsSave(treeID int64, keyPrefix string, blobs, hashes [][]byte, // Encrypt blobs if specified if encrypt { for k, v := range blobs { - e, err := t.encryptionKey.encrypt(0, v) + e, err := t.encrypt(v) if err != nil { return nil, err } From a1af0828bc10115884e3dffd3e16bba3bad96130 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 23 Dec 2020 17:56:15 -0600 Subject: [PATCH 190/449] Finish proposal inv by user ID. --- politeiad/backend/tlogbe/dcrdata.go | 2 +- politeiad/backend/tlogbe/pi.go | 412 ++++++++++++++++----- politeiad/backend/tlogbe/ticketvote.go | 2 +- politeiad/backend/tlogbe/tlog.go | 39 +- politeiad/backend/tlogbe/tlogbe.go | 40 +- politeiad/plugins/pi/pi.go | 100 ++--- politeiawww/api/pi/v1/v1.go | 54 ++- politeiawww/cmd/piwww/help.go | 2 +- politeiawww/cmd/piwww/piwww.go | 2 +- politeiawww/cmd/piwww/proposalinv.go | 53 +++ politeiawww/cmd/piwww/proposalinventory.go | 40 -- politeiawww/cmd/piwww/testrun.go | 23 +- politeiawww/cmd/shared/client.go | 3 +- politeiawww/pi.go | 10 +- politeiawww/piwww.go | 28 +- politeiawww/sharedconfig/sharedconfig.go | 14 +- 16 files changed, 561 insertions(+), 263 deletions(-) create mode 100644 politeiawww/cmd/piwww/proposalinv.go delete mode 100644 politeiawww/cmd/piwww/proposalinventory.go diff --git a/politeiad/backend/tlogbe/dcrdata.go b/politeiad/backend/tlogbe/dcrdata.go index de6777cb9..d0f97f6bd 100644 --- a/politeiad/backend/tlogbe/dcrdata.go +++ b/politeiad/backend/tlogbe/dcrdata.go @@ -609,7 +609,7 @@ func (p *dcrdataPlugin) cmd(cmd, payload string) (string, error) { // // This function satisfies the pluginClient interface. func (p *dcrdataPlugin) hook(h hookT, payload string) error { - log.Tracef("dcrdata hook: %v %v", hooks[h], payload) + log.Tracef("dcrdata hook: %v", hooks[h]) return nil } diff --git a/politeiad/backend/tlogbe/pi.go b/politeiad/backend/tlogbe/pi.go index 9090733ed..f4c6dadfd 100644 --- a/politeiad/backend/tlogbe/pi.go +++ b/politeiad/backend/tlogbe/pi.go @@ -26,8 +26,9 @@ import ( ) const ( - // Filenames of memoized data saved to the data dir. - filenameLinkedFrom = "{token}-linkedfrom.json" + // Filenames of cached data saved to the pi plugin data dir. + fnLinkedFrom = "{token}-linkedfrom.json" + fnUserData = "{userid}.json" ) var ( @@ -47,62 +48,32 @@ type piPlugin struct { dataDir string } -func isRFP(pm pi.ProposalMetadata) bool { - return pm.LinkBy != 0 -} - -// proposalMetadataFromFiles parses and returns the ProposalMetadata from the -// provided files. If a ProposalMetadata is not found, nil is returned. -func proposalMetadataFromFiles(files []backend.File) (*pi.ProposalMetadata, error) { - var pm *pi.ProposalMetadata - for _, v := range files { - if v.Name == pi.FileNameProposalMetadata { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return nil, err - } - pm, err = pi.DecodeProposalMetadata(b) - if err != nil { - return nil, err - } - } - } - return pm, nil -} - -func convertPropStatusFromMDStatus(s backend.MDStatusT) pi.PropStatusT { - var status pi.PropStatusT - switch s { - case backend.MDStatusUnvetted, backend.MDStatusIterationUnvetted: - status = pi.PropStatusUnvetted - case backend.MDStatusVetted: - status = pi.PropStatusPublic - case backend.MDStatusCensored: - status = pi.PropStatusCensored - case backend.MDStatusArchived: - status = pi.PropStatusAbandoned - } - return status -} - // linkedFrom is the the structure that is updated and cached for proposal A -// when proposal B links to proposal A. The list contains all proposals that -// have linked to proposal A. The linked from list will only contain public -// proposals. +// when proposal B links to proposal A. Proposals can link to one another using +// the ProposalMetadata LinkTo field. The linkedFrom list contains all +// proposals that have linked to proposal A. The list will only contain public +// proposals. The linkedFrom list is saved to disk in the pi plugin data dir, +// specifying the parent proposal token in the filename. // -// Example: an RFP proposal's linked from list will contain all public RFP -// submissions since they have all linked to the RFP proposal. +// Example: the linked from list for an RFP proposal will contain all public +// RFP submissions. The cached list can be found in the pi plugin data dir +// at the path specified by linkedFromPath(). type linkedFrom struct { Tokens map[string]struct{} `json:"tokens"` } +// linkedFromPath returns the path to the linkedFrom list for the provided +// proposal token. func (p *piPlugin) linkedFromPath(token string) string { - fn := strings.Replace(filenameLinkedFrom, "{token}", token, 1) + fn := strings.Replace(fnLinkedFrom, "{token}", token, 1) return filepath.Join(p.dataDir, fn) } +// linkedFromWithLock return the linkedFrom list for the provided proposal +// token. +// // This function must be called WITH the lock held. -func (p *piPlugin) linkedFromLocked(token string) (*linkedFrom, error) { +func (p *piPlugin) linkedFromWithLock(token string) (*linkedFrom, error) { fp := p.linkedFromPath(token) b, err := ioutil.ReadFile(fp) if err != nil { @@ -124,65 +95,214 @@ func (p *piPlugin) linkedFromLocked(token string) (*linkedFrom, error) { return &lf, nil } +// linkedFrom return the linkedFrom list for the provided proposal token. +// +// This function must be called WITHOUT the lock held. func (p *piPlugin) linkedFrom(token string) (*linkedFrom, error) { p.Lock() defer p.Unlock() - return p.linkedFromLocked(token) + return p.linkedFromWithLock(token) +} + +// linkedFromSaveWithLock saves the provided linkedFrom list to the pi plugin +// data dir. +// +// This function must be called WITH the lock held. +func (p *piPlugin) linkedFromSaveWithLock(token string, lf linkedFrom) error { + b, err := json.Marshal(lf) + if err != nil { + return err + } + fp := p.linkedFromPath(token) + return ioutil.WriteFile(fp, b, 0664) } +// linkedFromAdd updates the cached linkedFrom list for the parentToken, adding +// the childToken to the list. +// +// This function must be called WITHOUT the lock held. func (p *piPlugin) linkedFromAdd(parentToken, childToken string) error { p.Lock() defer p.Unlock() // Get existing linked from list - lf, err := p.linkedFromLocked(parentToken) + lf, err := p.linkedFromWithLock(parentToken) if errors.Is(err, errRecordNotFound) { - return fmt.Errorf("linkedFromLocked %v: %v", parentToken, err) + return fmt.Errorf("linkedFromWithLock %v: %v", parentToken, err) } // Update list lf.Tokens[childToken] = struct{}{} // Save list - b, err := json.Marshal(lf) - if err != nil { - return err - } - fp := p.linkedFromPath(parentToken) - err = ioutil.WriteFile(fp, b, 0664) - if err != nil { - return fmt.Errorf("WriteFile: %v", err) - } - - return nil + return p.linkedFromSaveWithLock(parentToken, *lf) } +// linkedFromDel updates the cached linkedFrom list for the parentToken, +// deleting the childToken from the list. +// +// This function must be called WITHOUT the lock held. func (p *piPlugin) linkedFromDel(parentToken, childToken string) error { p.Lock() defer p.Unlock() // Get existing linked from list - lf, err := p.linkedFromLocked(parentToken) + lf, err := p.linkedFromWithLock(parentToken) if err != nil { - return fmt.Errorf("linkedFromLocked %v: %v", parentToken, err) + return fmt.Errorf("linkedFromWithLock %v: %v", parentToken, err) } // Update list delete(lf.Tokens, childToken) // Save list - b, err := json.Marshal(lf) + return p.linkedFromSaveWithLock(parentToken, *lf) +} + +// userData contains cached pi plugin data for a specific user. The userData +// JSON is saved to disk in the pi plugin data dir. The user ID is included in +// the filename. +type userData struct { + // Tokens contains a list of all the proposals that have been + // submitted by this user. This data is cached so that the + // ProposalInv command can filter proposals by user ID. + Tokens []string `json:"tokens"` +} + +// userDataPath returns the filepath to the cached userData struct for the +// specified user. +func (p *piPlugin) userDataPath(userID string) string { + fn := strings.Replace(fnUserData, "{userid}", userID, 1) + return filepath.Join(p.dataDir, fn) +} + +// userDataWithLock returns the cached userData struct for the specified user. +// +// This function must be called WITH the lock held. +func (p *piPlugin) userDataWithLock(userID string) (*userData, error) { + fp := p.userDataPath(userID) + b, err := ioutil.ReadFile(fp) + if err != nil { + var e *os.PathError + if errors.As(err, &e) && !os.IsExist(err) { + // File does't exist. Return an empty userData. + return &userData{ + Tokens: []string{}, + }, nil + } + } + + var ud userData + err = json.Unmarshal(b, &ud) + if err != nil { + return nil, err + } + + return &ud, nil +} + +// userData returns the cached userData struct for the specified user. +// +// This function must be called WITHOUT the lock held. +func (p *piPlugin) userData(userID string) (*userData, error) { + p.Lock() + defer p.Unlock() + + return p.userDataWithLock(userID) +} + +// userDataSaveWithLock saves the provided userData to the pi plugin data dir. +// +// This function must be called WITH the lock held. +func (p *piPlugin) userDataSaveWithLock(userID string, ud userData) error { + b, err := json.Marshal(ud) if err != nil { return err } - fp := p.linkedFromPath(parentToken) - err = ioutil.WriteFile(fp, b, 0664) + + fp := p.userDataPath(userID) + return ioutil.WriteFile(fp, b, 0664) +} + +// userDataAddToken adds the provided token to the cached userData for the +// provided user. +// +// This function must be called WITHOUT the lock held. +func (p *piPlugin) userDataAddToken(userID string, token string) error { + p.Lock() + defer p.Unlock() + + // Get current user data + ud, err := p.userDataWithLock(userID) if err != nil { - return fmt.Errorf("WriteFile: %v", err) + return err } - return nil + // Add token + ud.Tokens = append(ud.Tokens, token) + + // Save changes + return p.userDataSaveWithLock(userID, *ud) +} + +// isRFP returns whether the provided proposal metadata belongs to an RFP +// proposal. +func isRFP(pm pi.ProposalMetadata) bool { + return pm.LinkBy != 0 +} + +// decodeProposalMetadata decodes and returns the ProposalMetadata from the +// provided backend files. If a ProposalMetadata is not found, nil is returned. +func decodeProposalMetadata(files []backend.File) (*pi.ProposalMetadata, error) { + var pm *pi.ProposalMetadata + for _, v := range files { + if v.Name == pi.FileNameProposalMetadata { + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return nil, err + } + pm, err = pi.DecodeProposalMetadata(b) + if err != nil { + return nil, err + } + break + } + } + return pm, nil +} + +// decodeGeneralMetadata decodes and returns the GeneralMetadata from the +// provided backend metadata streams. If a GeneralMetadata is not found, nil is +// returned. +func decodeGeneralMetadata(metadata []backend.MetadataStream) (*pi.GeneralMetadata, error) { + var gm *pi.GeneralMetadata + var err error + for _, v := range metadata { + if v.ID == pi.MDStreamIDGeneralMetadata { + gm, err = pi.DecodeGeneralMetadata([]byte(v.Payload)) + if err != nil { + return nil, err + } + break + } + } + return gm, nil +} + +func convertPropStatusFromMDStatus(s backend.MDStatusT) pi.PropStatusT { + var status pi.PropStatusT + switch s { + case backend.MDStatusUnvetted, backend.MDStatusIterationUnvetted: + status = pi.PropStatusUnvetted + case backend.MDStatusVetted: + status = pi.PropStatusPublic + case backend.MDStatusCensored: + status = pi.PropStatusCensored + case backend.MDStatusArchived: + status = pi.PropStatusAbandoned + } + return status } func (p *piPlugin) cmdProposals(payload string) (string, error) { @@ -637,6 +757,104 @@ func (p *piPlugin) cmdCommentVote(payload string) (string, error) { return string(reply), nil } +func (p *piPlugin) cmdProposalInv(payload string) (string, error) { + // Decode payload + inv, err := pi.DecodeProposalInv([]byte(payload)) + if err != nil { + return "", err + } + + // Get full record inventory + ibs, err := p.backend.InventoryByStatus() + if err != nil { + return "", err + } + + // Apply user ID filtering criteria + if inv.UserID != "" { + // Lookup the proposal tokens that have been submitted by the + // specified user. + ud, err := p.userData(inv.UserID) + if err != nil { + return "", fmt.Errorf("userData %v: %v", inv.UserID, err) + } + userTokens := make(map[string]struct{}, len(ud.Tokens)) + for _, v := range ud.Tokens { + userTokens[v] = struct{}{} + } + + // Compile a list of unvetted tokens categorized by MDStatusT + // that were submitted by the user. + filtered := make(map[backend.MDStatusT][]string, len(ibs.Unvetted)) + for status, tokens := range ibs.Unvetted { + for _, v := range tokens { + _, ok := userTokens[v] + if !ok { + // Proposal was not submitted by the user + continue + } + + // Proposal was submitted by the user + ftokens, ok := filtered[status] + if !ok { + ftokens = make([]string, 0, len(tokens)) + } + filtered[status] = append(ftokens, v) + } + } + + // Update unvetted inventory with filtered tokens + ibs.Unvetted = filtered + + // Compile a list of vetted tokens categorized by MDStatusT that + // were submitted by the user. + filtered = make(map[backend.MDStatusT][]string, len(ibs.Vetted)) + for status, tokens := range ibs.Vetted { + for _, v := range tokens { + _, ok := userTokens[v] + if !ok { + // Proposal was not submitted by the user + continue + } + + // Proposal was submitted by the user + ftokens, ok := filtered[status] + if !ok { + ftokens = make([]string, 0, len(tokens)) + } + filtered[status] = append(ftokens, v) + } + } + + // Update vetted inventory with filtered tokens + ibs.Vetted = filtered + } + + // Convert MDStatus keys to human readable proposal statuses + unvetted := make(map[string][]string, len(ibs.Unvetted)) + vetted := make(map[string][]string, len(ibs.Vetted)) + for k, v := range ibs.Unvetted { + s := pi.PropStatuses[convertPropStatusFromMDStatus(k)] + unvetted[s] = v + } + for k, v := range ibs.Vetted { + s := pi.PropStatuses[convertPropStatusFromMDStatus(k)] + vetted[s] = v + } + + // Prepare reply + pir := pi.ProposalInvReply{ + Unvetted: unvetted, + Vetted: vetted, + } + reply, err := pi.EncodeProposalInvReply(pir) + if err != nil { + return "", err + } + + return string(reply), nil +} + func (p *piPlugin) cmdVoteInventory(payload string) (string, error) { // Payload is empty. Nothing to decode. @@ -707,7 +925,7 @@ func (p *piPlugin) ticketVoteStart(payload string) (string, error) { return "", err } - // Verify there is work to do + // Verify work needs to be done if len(s.Starts) == 0 { return "", backend.PluginUserError{ PluginID: pi.ID, @@ -752,7 +970,7 @@ func (p *piPlugin) ticketVoteStart(payload string) (string, error) { // Verify RFP linkby has expired. The runoff vote is not allowed to // start until after the linkby deadline has passed. - rfpPM, err := proposalMetadataFromFiles(rfp.Files) + rfpPM, err := decodeProposalMetadata(rfp.Files) if err != nil { return "", err } @@ -817,7 +1035,7 @@ func (p *piPlugin) ticketVoteStart(payload string) (string, error) { for _, v := range s.Starts { _, ok := expected[v.Params.Token] if !ok { - // This submission should not be here + // This submission should not be here. e := fmt.Sprintf("found token that should not be included: %v", v.Params.Token) return "", backend.PluginUserError{ @@ -895,19 +1113,9 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { } // Decode ProposalMetadata - var pm *pi.ProposalMetadata - for _, v := range nr.Files { - if v.Name == pi.FileNameProposalMetadata { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return err - } - pm, err = pi.DecodeProposalMetadata(b) - if err != nil { - return err - } - break - } + pm, err := decodeProposalMetadata(nr.Files) + if err != nil { + return err } if pm == nil { return fmt.Errorf("proposal metadata not found") @@ -947,7 +1155,7 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { } return err } - linkToPM, err := proposalMetadataFromFiles(r.Files) + linkToPM, err := decodeProposalMetadata(r.Files) if err != nil { return err } @@ -1006,6 +1214,30 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { return nil } +func (p *piPlugin) hookNewRecordPost(payload string) error { + nr, err := decodeHookNewRecord([]byte(payload)) + if err != nil { + return err + } + + // Decode GeneralMetadata + gm, err := decodeGeneralMetadata(nr.Metadata) + if err != nil { + return err + } + if gm == nil { + panic("general metadata not found") + } + + // Add token to the user data cache + err = p.userDataAddToken(gm.UserID, nr.RecordMetadata.Token) + if err != nil { + return err + } + + return nil +} + func (p *piPlugin) hookEditRecordPre(payload string) error { er, err := decodeHookEditRecord([]byte(payload)) if err != nil { @@ -1020,11 +1252,11 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { // linkto. status := convertPropStatusFromMDStatus(er.Current.RecordMetadata.Status) if status == pi.PropStatusPublic { - pmCurr, err := proposalMetadataFromFiles(er.Current.Files) + pmCurr, err := decodeProposalMetadata(er.Current.Files) if err != nil { return err } - pmNew, err := proposalMetadataFromFiles(er.FilesAdd) + pmNew, err := decodeProposalMetadata(er.FilesAdd) if err != nil { return err } @@ -1147,7 +1379,7 @@ func (p *piPlugin) hookSetRecordStatusPost(payload string) error { // If the LinkTo field has been set then the linkedFrom // list might need to be updated for the proposal that is being // linked to, depending on the status change that is being made. - pm, err := proposalMetadataFromFiles(srs.Current.Files) + pm, err := decodeProposalMetadata(srs.Current.Files) if err != nil { return err } @@ -1205,6 +1437,8 @@ func (p *piPlugin) cmd(cmd, payload string) (string, error) { return p.cmdCommentCensor(payload) case pi.CmdCommentVote: return p.cmdCommentVote(payload) + case pi.CmdProposalInv: + return p.cmdProposalInv(payload) case pi.CmdVoteInventory: return p.cmdVoteInventory(payload) case pi.CmdPassThrough: @@ -1223,6 +1457,8 @@ func (p *piPlugin) hook(h hookT, payload string) error { switch h { case hookNewRecordPre: return p.hookNewRecordPre(payload) + case hookNewRecordPost: + return p.hookNewRecordPost(payload) case hookEditRecordPre: return p.hookEditRecordPre(payload) case hookSetRecordStatusPost: diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index 38c2defff..a386bc59d 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -2716,7 +2716,7 @@ func (p *ticketVotePlugin) cmd(cmd, payload string) (string, error) { // // This function satisfies the pluginClient interface. func (p *ticketVotePlugin) hook(h hookT, payload string) error { - log.Tracef("ticketvote hook: %v %v", hooks[h], payload) + log.Tracef("ticketvote hook: %v", hooks[h]) return nil } diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index b07c923af..9693c5670 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -82,12 +82,16 @@ var ( // We do not unwind. type tlog struct { sync.Mutex - id string - dcrtimeHost string + id string + dcrtimeHost string + trillian trillianClient + store store.Blob + cron *cron.Cron + + // encryptionKey is used to encrypt record blobs before saving them + // to the key-value store. This is an optional param. Record blobs + // will not be encrypted if this is left as nil. encryptionKey *encryptionKey - trillian trillianClient - store store.Blob - cron *cron.Cron // droppingAnchor indicates whether tlog is in the process of // dropping an anchor, i.e. timestamping unanchored trillian trees @@ -104,8 +108,9 @@ type tlog struct { // // 1. Record content blobs are saved to the kv store. // -// 2. The kv store keys are stuffed into the LogLeaf.ExtraData field and the -// leaves are appended onto the trillian tree. +// 2. A trillian leaf is created for each record content blob. The kv store +// key for the blob is stuffed into the LogLeaf.ExtraData field. All leaves +// are appended onto the trillian tree. // // 3. If there are failures in steps 1 or 2 for any of the blobs then the // update will exit without completing. No unwinding is performed. Blobs @@ -120,7 +125,8 @@ type tlog struct { // of a recordIndex are considered to be orphaned and can be disregarded. type recordIndex struct { // Version represents the version of the record. The version is - // only incremented when the record files are updated. + // only incremented when the record files are updated. Metadata + // only updates do no increment the version. Version uint32 `json:"version"` // Iteration represents the iteration of the record. The iteration @@ -172,7 +178,8 @@ func treePointerExists(r recordIndex) bool { r.TreePointer) panic(e) case r.TreePointer < 0: - // Tree pointer should never be negative + // Tree pointer should never be negative. Trillian uses a int64 + // for the tree ID so we do too. e := fmt.Sprintf("tree pointer is < 0: %v", r.TreePointer) panic(e) } @@ -707,7 +714,7 @@ type recordBlobsPrepareReply struct { // saved to the kv store. Hashes contains the hashes of the record // content prior to being blobified. // - // blobs and hashes MUST share the same ordering. + // blobs and hashes share the same ordering. blobs [][]byte hashes [][]byte } @@ -716,7 +723,7 @@ type recordBlobsPrepareReply struct { // the blob kv store and appended onto a trillian tree. // // TODO test this function -func recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File, encryptionKey *encryptionKey) (*recordBlobsPrepareReply, error) { +func (t *tlog) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) (*recordBlobsPrepareReply, error) { // Verify there are no duplicate or empty mdstream IDs mdstreamIDs := make(map[uint64]struct{}, len(metadata)) for _, v := range metadata { @@ -881,8 +888,8 @@ func recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backend.RecordMe return nil, err } // Encypt file blobs if encryption key has been set - if encryptionKey != nil { - b, err = encryptionKey.encrypt(0, b) + if t.encryptionKey != nil { + b, err = t.encrypt(b) if err != nil { return nil, err } @@ -1023,8 +1030,7 @@ func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []ba } // Prepare kv store blobs - bpr, err := recordBlobsPrepare(leavesAll, rm, metadata, - files, t.encryptionKey) + bpr, err := t.recordBlobsPrepare(leavesAll, rm, metadata, files) if err != nil { return err } @@ -1137,8 +1143,7 @@ func (t *tlog) metadataSave(treeID int64, rm backend.RecordMetadata, metadata [] } // Prepare kv store blobs - bpr, err := recordBlobsPrepare(leavesAll, rm, metadata, - []backend.File{}, t.encryptionKey) + bpr, err := t.recordBlobsPrepare(leavesAll, rm, metadata, []backend.File{}) if err != nil { return nil, err } diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index b0465b5f9..c98f17795 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -45,9 +45,13 @@ const ( tlogIDUnvetted = "unvetted" tlogIDVetted = "vetted" - // The following are the IDs of plugin settings that are derived - // from the politeiad config. The user does not have to set these - // manually. + // pluginDataDirname is the plugin data directory name. It is + // located in the tlog backend data directory and is provided to + // the plugins for storing plugin data. + pluginDataDirname = "plugins" + + // pluginSettingDataDir is the PluginSetting key for the plugin + // data directory. pluginSettingDataDir = "datadir" // Record states @@ -792,14 +796,13 @@ func (t *tlogBackend) New(metadata []backend.MetadataStream, files []backend.Fil } err = t.pluginHook(hookNewRecordPost, string(b)) if err != nil { - log.Errorf("New %x: pluginHook newRecordPost: %v", token, err) + e := fmt.Sprintf("New %x: pluginHook newRecordPost: %v", token, err) + panic(e) } // Update the inventory cache t.inventoryAdd(stateUnvetted, token, backend.MDStatusUnvetted) - log.Infof("New record %x", token) - return rm, nil } @@ -896,8 +899,9 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ // Call post plugin hooks err = t.pluginHook(hookEditRecordPost, string(b)) if err != nil { - log.Errorf("UpdateUnvettedRecord %x: pluginHook editRecordPost: %v", + e := fmt.Sprintf("UpdateUnvettedRecord %x: pluginHook editRecordPost: %v", token, err) + panic(e) } // Return updated record @@ -1005,8 +1009,9 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b // Call post plugin hooks err = t.pluginHook(hookEditRecordPost, string(b)) if err != nil { - log.Errorf("UpdateVettedRecord %x: pluginHook editRecordPost: %v", + e := fmt.Sprintf("UpdateVettedRecord %x: pluginHook editRecordPost: %v", token, err) + panic(e) } // Return updated record @@ -1103,8 +1108,9 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite // Call post plugin hooks err = t.pluginHook(hookEditMetadataPost, string(b)) if err != nil { - log.Errorf("UpdateUnvettedMetadata %x: pluginHook editMetadataPost: %v", + e := fmt.Sprintf("UpdateUnvettedMetadata %x: pluginHook editMetadataPost: %v", token, err) + panic(e) } return nil @@ -1200,8 +1206,9 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ // Call post plugin hooks err = t.pluginHook(hookEditMetadataPost, string(b)) if err != nil { - log.Errorf("UpdateVettedMetadata %x: pluginHook editMetadataPost: %v", + e := fmt.Sprintf("UpdateVettedMetadata %x: pluginHook editMetadataPost: %v", token, err) + panic(e) } return nil @@ -1442,8 +1449,9 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, // Call post plugin hooks err = t.pluginHook(hookSetRecordStatusPost, string(b)) if err != nil { - log.Errorf("SetUnvettedStatus %x: pluginHook setRecordStatusPost: %v", + e := fmt.Sprintf("SetUnvettedStatus %x: pluginHook setRecordStatusPost: %v", token, err) + panic(e) } log.Debugf("Status change %x from %v (%v) to %v (%v)", @@ -1598,8 +1606,9 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md // Call post plugin hooks err = t.pluginHook(hookSetRecordStatusPost, string(b)) if err != nil { - log.Errorf("SetVettedStatus %x: pluginHook setRecordStatusPost: %v", + e := fmt.Sprintf("SetVettedStatus %x: pluginHook setRecordStatusPost: %v", token, err) + panic(e) } // Update inventory cache @@ -1646,11 +1655,12 @@ func (t *tlogBackend) InventoryByStatus() (*backend.InventoryByStatus, error) { func (t *tlogBackend) RegisterPlugin(p backend.Plugin) error { log.Tracef("RegisterPlugin: %v", p.ID) - // Add tlog backend data dir to plugin settings. The plugin data - // dir should append the plugin ID onto the tlog backend data dir. + // Add the plugin data dir to the plugin settings. Plugins should + // create their own individual data directories inside of the + // plugin data directory. p.Settings = append(p.Settings, backend.PluginSetting{ Key: pluginSettingDataDir, - Value: t.dataDir, + Value: filepath.Join(t.dataDir, pluginDataDirname), }) var ( diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index 0dd099e1c..50d8de64e 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -25,18 +25,20 @@ const ( Version = "1" // Plugin commands - // TODO refactor these commands to use the passthrough command - CmdProposals = "proposals" // Get plugin data for proposals CmdCommentNew = "commentnew" // Create a new comment CmdCommentCensor = "commentcensor" // Censor a comment CmdCommentVote = "commentvote" // Upvote/downvote a comment + CmdProposalInv = "proposalinv" // Get inventory by proposal status + CmdVoteInventory = "voteinv" // Get inventory by vote status - CmdProposalInventory = "proposalinv" // Get inventory by proposal status - CmdVoteInventory = "voteinv" // Get inventory by vote status - CmdPassThrough = "passthrough" // Pass a plugin cmd through pi + // TODO get rid of CmdProposals + CmdProposals = "proposals" // Get plugin data for proposals + + // TODO get rid of CmdPassThrough + CmdPassThrough = "passthrough" // Pass a plugin cmd through pi // Metadata stream IDs - MDStreamIDProposalGeneral = 1 + MDStreamIDGeneralMetadata = 1 MDStreamIDStatusChanges = 2 // FileNameProposalMetadata is the filename of the ProposalMetadata @@ -82,6 +84,15 @@ const ( ) var ( + // PropStatuses contains the human readable proposal statuses. + PropStatuses = map[PropStatusT]string{ + PropStatusInvalid: "invalid", + PropStatusUnvetted: "unvetted", + PropStatusPublic: "public", + PropStatusCensored: "censored", + PropStatusAbandoned: "abandoned", + } + // StatusChanges contains the allowed proposal status change // transitions. If StatusChanges[currentStatus][newStatus] exists // then the status change is allowed. @@ -106,10 +117,11 @@ var ( } ) -// ProposalMetadata contains proposal metadata that is provided by the user on -// proposal submission. ProposalMetadata is saved to politeiad as a file, not -// as a metadata stream, since it needs to be included in the merkle root that -// politeiad signs. +// ProposalMetadata contains metadata that is provided by the user as part of +// the proposal submission bundle. The proposal metadata is included in the +// proposal signature since it is user specified data. The ProposalMetadata +// object is saved to politeiad as a file, not as a metadata stream, since it +// needs to be included in the merkle root that politeiad signs. type ProposalMetadata struct { // Name is the name of the proposal. Name string `json:"name"` @@ -139,32 +151,31 @@ func DecodeProposalMetadata(payload []byte) (*ProposalMetadata, error) { return &pm, nil } -// ProposalGeneral represents general proposal metadata that is saved on -// proposal submission. ProposalGeneral is saved to politeiad as a metadata -// stream. +// GeneralMetadata contains general metadata about a politeiad record. It is +// saved to politeiad as a metadata stream. // -// Signature is the client signature of the proposal merkle root. The merkle -// root is the ordered merkle root of all proposal Files and Metadata. -type ProposalGeneral struct { - UserID string `json:"userid"` // Unique user ID +// Signature is the client signature of the record merkle root. The merkle root +// is the ordered merkle root of all politeiad Files. +type GeneralMetadata struct { + UserID string `json:"userid"` // Author user ID PublicKey string `json:"publickey"` // Key used for signature Signature string `json:"signature"` // Signature of merkle root Timestamp int64 `json:"timestamp"` // Submission UNIX timestamp } -// EncodeProposalGeneral encodes a ProposalGeneral into a JSON byte slice. -func EncodeProposalGeneral(pg ProposalGeneral) ([]byte, error) { - return json.Marshal(pg) +// EncodeGeneralMetadata encodes a GeneralMetadata into a JSON byte slice. +func EncodeGeneralMetadata(gm GeneralMetadata) ([]byte, error) { + return json.Marshal(gm) } -// DecodeProposalGeneral decodes a JSON byte slice into a ProposalGeneral. -func DecodeProposalGeneral(payload []byte) (*ProposalGeneral, error) { - var pg ProposalGeneral - err := json.Unmarshal(payload, &pg) +// DecodeGeneralMetadata decodes a JSON byte slice into a GeneralMetadata. +func DecodeGeneralMetadata(payload []byte) (*GeneralMetadata, error) { + var gm GeneralMetadata + err := json.Unmarshal(payload, &gm) if err != nil { return nil, err } - return &pg, nil + return &gm, nil } // StatusChange represents a proposal status change. @@ -237,6 +248,7 @@ func DecodeProposals(payload []byte) (*Proposals, error) { } // ProposalPluginData contains all the plugin data for a proposal. +// TODO get rid of this command type ProposalPluginData struct { Comments uint64 `json:"comments"` // Number of comments LinkedFrom []string `json:"linkedfrom"` // Linked from list @@ -425,22 +437,22 @@ func DecodeCommentVoteReply(payload []byte) (*CommentVoteReply, error) { return &cvr, nil } -// ProposalInventory retrieves the tokens of all proposals in the inventory, -// categorized by proposal state and proposal status, that match the provided -// filtering criteria. If no filtering criteria is provided then the full -// proposal inventory is returned. -type ProposalInventory struct { +// ProposalInv retrieves the tokens of all proposals in the inventory that +// match the provided filtering criteria. The returned proposals are +// categorized by proposal state and status. If no filtering criteria is +// provided then the full proposal inventory is returned. +type ProposalInv struct { UserID string `json:"userid,omitempty"` } -// EncodeProposalInventory encodes a ProposalInventory into a JSON byte slice. -func EncodeProposalInventory(pi ProposalInventory) ([]byte, error) { +// EncodeProposalInv encodes a ProposalInv into a JSON byte slice. +func EncodeProposalInv(pi ProposalInv) ([]byte, error) { return json.Marshal(pi) } -// DecodeProposalInventory decodes a JSON byte slice into a ProposalInventory. -func DecodeProposalInventory(payload []byte) (*ProposalInventory, error) { - var pi ProposalInventory +// DecodeProposalInv decodes a JSON byte slice into a ProposalInv. +func DecodeProposalInv(payload []byte) (*ProposalInv, error) { + var pi ProposalInv err := json.Unmarshal(payload, &pi) if err != nil { return nil, err @@ -448,22 +460,24 @@ func DecodeProposalInventory(payload []byte) (*ProposalInventory, error) { return &pi, nil } -// ProposalInventoryReply is the reply to the ProposalInventory command. -type ProposalInventoryReply struct { +// ProposalInvReply is the reply to the ProposalInv command. The returned maps +// contains map[status][]token where the status is the human readable proposal +// status and the token is the proposal token. +type ProposalInvReply struct { Unvetted map[string][]string `json:"unvetted"` Vetted map[string][]string `json:"vetted"` } -// EncodeProposalInventoryReply encodes a ProposalInventoryReply into a JSON +// EncodeProposalInvReply encodes a ProposalInvReply into a JSON // byte slice. -func EncodeProposalInventoryReply(pir ProposalInventoryReply) ([]byte, error) { +func EncodeProposalInvReply(pir ProposalInvReply) ([]byte, error) { return json.Marshal(pir) } -// DecodeProposalInventoryReply decodes a JSON byte slice into a -// ProposalInventoryReply. -func DecodeProposalInventoryReply(payload []byte) (*ProposalInventoryReply, error) { - var pir ProposalInventoryReply +// DecodeProposalInvReply decodes a JSON byte slice into a +// ProposalInvReply. +func DecodeProposalInvReply(payload []byte) (*ProposalInvReply, error) { + var pir ProposalInvReply err := json.Unmarshal(payload, &pir) if err != nil { return nil, err diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index b6f84689d..cba3b5d32 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -34,7 +34,8 @@ type VoteErrorT int const ( APIVersion = 1 - // APIRoute is prefixed onto all routes. + // APIRoute is prefixed onto all routes defined in this package. + // TODO should the api route be "/pi/v1"? APIRoute = "/v1" // Proposal routes @@ -69,12 +70,32 @@ const ( PropStateUnvetted PropStateT = 1 PropStateVetted PropStateT = 2 - // Proposal statuses - PropStatusInvalid PropStatusT = 0 // Invalid status - PropStatusUnreviewed PropStatusT = 1 // Prop has not been reviewed - PropStatusPublic PropStatusT = 2 // Prop has been made public - PropStatusCensored PropStatusT = 3 // Prop has been censored - PropStatusAbandoned PropStatusT = 4 // Prop has been abandoned + // PropStatusInvalid indicates the proposal status is invalid. + PropStatusInvalid PropStatusT = 0 + + // PropStatusUnreviewed indicates the proposal has been submitted, + // but has not yet been reviewed and made public by an admin. A + // proposal with this status will have a proposal state of + // PropStateUnvetted. + PropStatusUnreviewed PropStatusT = 1 + + // PropStatusPublic indicates that a proposal has been reviewed and + // made public by an admin. A proposal with this status will have + // a proposal state of PropStateVetted. + PropStatusPublic PropStatusT = 2 + + // PropStatusCensored indicates that a proposal has been censored + // by an admin for violating the proposal guidlines.. Both unvetted + // and vetted proposals can be censored so a proposal with this + // status can have a state of either PropStateUnvetted or + // PropStateVetted depending on whether the proposal was censored + // before or after it was made public. + PropStatusCensored PropStatusT = 3 + + // PropStatusAbandoned indicates that a proposal has been marked + // as abandoned by an admin due to the author being inactive. + // TODO can a unvetted proposal be abandoned? + PropStatusAbandoned PropStatusT = 4 // Comment vote types CommentVoteInvalid CommentVoteT = 0 @@ -92,7 +113,7 @@ const ( VoteAuthActionAuthorize VoteAuthActionT = "authorize" VoteAuthActionRevoke VoteAuthActionT = "revoke" - // Vote types + // VoteTypeInvalid represents and invalid vote type. VoteTypeInvalid VoteT = 0 // VoteTypeStandard is used to indicate a simple approve or reject @@ -199,8 +220,8 @@ const ( ) var ( - // PropStatus contains the human readable proposal statuses. - PropStatus = map[PropStatusT]string{ + // PropStatuses contains the human readable proposal statuses. + PropStatuses = map[PropStatusT]string{ PropStatusInvalid: "invalid", PropStatusUnreviewed: "unreviewed", PropStatusPublic: "public", @@ -314,11 +335,6 @@ const ( // ProposalMetadata contains metadata that is specified by the user on proposal // submission. It is attached to a proposal submission as a Metadata object. -// -// TODO should there be a Type field? The issue is that we currently assume -// a proposal is an RFP submission if the LinkTo is set. This boxes us in and -// prevents us from using LinkTo in the future without some additional field -// like a Type field. type ProposalMetadata struct { Name string `json:"name"` // Proposal name @@ -455,8 +471,8 @@ type ProposalRequest struct { } // Proposals retrieves the ProposalRecord for each of the provided proposal -// requests. Unvetted proposals are stripped of their user defined proposal -// files and metadata when being returned to non-admins. +// requests. Unvetted proposals are stripped of their user defined files and +// metadata when being returned to non-admins. // // IncludeFiles specifies whether the proposal files should be returned. The // user defined metadata will still be returned even when IncludeFiles is set @@ -484,11 +500,11 @@ type ProposalInventory struct { // ProposalInventoryReply is the reply to the ProposalInventory command. The // inventory maps contain map[status][]tokens where the status is the human -// readable proposal status, as defined by the PropStatus map, and the tokens +// readable proposal status, as defined by the PropStatuses map, and the tokens // are a list of proposal tokens for that status. Each list is ordered by // timestamp of the status change from newest to oldest. type ProposalInventoryReply struct { - Unvetted map[string][]string `json:"unvetted,omitempty"` + Unvetted map[string][]string `json:"unvetted"` Vetted map[string][]string `json:"vetted"` } diff --git a/politeiawww/cmd/piwww/help.go b/politeiawww/cmd/piwww/help.go index 026c30e03..54e614e42 100644 --- a/politeiawww/cmd/piwww/help.go +++ b/politeiawww/cmd/piwww/help.go @@ -47,7 +47,7 @@ func (cmd *helpCmd) Execute(args []string) error { case "proposals": fmt.Printf("%s\n", proposalsHelpMsg) case "proposalinventory": - fmt.Printf("%s\n", proposalInventoryHelpMsg) + fmt.Printf("%s\n", proposalInvHelpMsg) // Comment commands case "commentnew": diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index 729bab9c1..0a60cf4e1 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -70,7 +70,7 @@ type piwww struct { ProposalEdit proposalEditCmd `command:"proposaledit"` ProposalStatusSet proposalStatusSetCmd `command:"proposalstatusset"` Proposals proposalsCmd `command:"proposals"` - ProposalInventory proposalInventoryCmd `command:"proposalinv"` + ProposalInv proposalInvCmd `command:"proposalinv"` // Comments commands CommentNew commentNewCmd `command:"commentnew"` diff --git a/politeiawww/cmd/piwww/proposalinv.go b/politeiawww/cmd/piwww/proposalinv.go new file mode 100644 index 000000000..52610f229 --- /dev/null +++ b/politeiawww/cmd/piwww/proposalinv.go @@ -0,0 +1,53 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + pi "github.com/decred/politeia/politeiawww/api/pi/v1" + "github.com/decred/politeia/politeiawww/cmd/shared" +) + +// proposalInvCmd retrieves the censorship record tokens of all proposals in +// the inventory that match the provided filtering criteria. If no filtering +// criteria is given then the full inventory is returned. +type proposalInvCmd struct { + UserID string `long:"userid" optional:"true"` +} + +// Execute executes the proposalInvCmd command. +// +// This function satisfies the go-flags Commander interface. +func (c *proposalInvCmd) Execute(args []string) error { + p := pi.ProposalInventory{ + UserID: c.UserID, + } + err := shared.PrintJSON(p) + if err != nil { + return err + } + pir, err := client.ProposalInventory(p) + if err != nil { + return err + } + err = shared.PrintJSON(pir) + if err != nil { + return err + } + return nil +} + +// proposalInvHelpMsg is the command help message. +const proposalInvHelpMsg = `proposalinv + +Fetch the censorship record tokens for all proposals that match the provided +filtering criteria. If no filtering criteria is provided, the full proposal +inventory will be returned. The returned proposals are categorized by their +proposal state and proposal status. Unvetted tokens are only returned if the +logged in user is an admin. + + +Flags: + --userid (string, optional) Filter by user ID +` diff --git a/politeiawww/cmd/piwww/proposalinventory.go b/politeiawww/cmd/piwww/proposalinventory.go deleted file mode 100644 index 96fa720e9..000000000 --- a/politeiawww/cmd/piwww/proposalinventory.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) 2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import ( - pi "github.com/decred/politeia/politeiawww/api/pi/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" -) - -// proposalInventoryCmd retrieves the censorship record tokens of all proposals -// in the inventory. -type proposalInventoryCmd struct{} - -// Execute executes the proposal inventory command. -func (cmd *proposalInventoryCmd) Execute(args []string) error { - p := pi.ProposalInventory{} - err := shared.PrintJSON(p) - if err != nil { - return err - } - pir, err := client.ProposalInventory(p) - if err != nil { - return err - } - err = shared.PrintJSON(pir) - if err != nil { - return err - } - return nil -} - -// proposalInventoryHelpMsg is the command help message. -const proposalInventoryHelpMsg = `proposalinv - -Fetch the censorship record tokens for all proposals, categorized by their -proposal state and proposal status. Unvetted tokens are only returned if the -logged in user is an admin. -` diff --git a/politeiawww/cmd/piwww/testrun.go b/politeiawww/cmd/piwww/testrun.go index a3dc3fb36..a02bf5ab5 100644 --- a/politeiawww/cmd/piwww/testrun.go +++ b/politeiawww/cmd/piwww/testrun.go @@ -842,7 +842,7 @@ func testProposalRoutes(admin testUser) error { vettedProps := pir.Vetted // Ensure public proposal token received - publicProps, ok := vettedProps[pi.PropStatus[pi.PropStatusPublic]] + publicProps, ok := vettedProps[pi.PropStatuses[pi.PropStatusPublic]] if !ok { return fmt.Errorf("No public proposals returned") } @@ -857,7 +857,7 @@ func testProposalRoutes(admin testUser) error { } // Ensure vetted censored proposal token received - vettedCensored, ok := vettedProps[pi.PropStatus[pi.PropStatusCensored]] + vettedCensored, ok := vettedProps[pi.PropStatuses[pi.PropStatusCensored]] if !ok { return fmt.Errorf("No vetted censrored proposals returned") } @@ -873,7 +873,7 @@ func testProposalRoutes(admin testUser) error { } // Ensure abandoned proposal token received - abandonedProps, ok := vettedProps[pi.PropStatus[pi.PropStatusAbandoned]] + abandonedProps, ok := vettedProps[pi.PropStatuses[pi.PropStatusAbandoned]] if !ok { return fmt.Errorf("No abandoned proposals returned") } @@ -891,7 +891,7 @@ func testProposalRoutes(admin testUser) error { unvettedProps := pir.Unvetted // Ensure unvetted proposal token received - unreviewedProps, ok := unvettedProps[pi.PropStatus[pi.PropStatusUnreviewed]] + unreviewedProps, ok := unvettedProps[pi.PropStatuses[pi.PropStatusUnreviewed]] if !ok { return fmt.Errorf("No unreviewed proposals returned") } @@ -1429,18 +1429,17 @@ func (cmd *testRunCmd) Execute(args []string) error { } publicKey = version.PubKey - // We only allow this to be run on testnet for right now. - // Running it on mainnet would require changing the user - // email verification flow. - // We ensure vote duration isn't longer than - // 3 blocks as we need to approve an RFP and it's - // submission as part of our tests. + // Verify politeiawww settings switch { case !version.TestNet: return fmt.Errorf("this command must be run on testnet") case policy.MinVoteDuration > 3: - return fmt.Errorf("--votedurationmin flag should be <= 3, as the " + - "tests include RFP & submssions voting") + // Min vote duration must be <=3 since this command waits for + // proposal votes to finish as part of the test run. + return fmt.Errorf("politeiawww min vote duration is currently %v. "+ + "This command requires a min vote duration of <=3 blocks. Use "+ + "the politeiawww --votedurationmin flag to update this setting.", + policy.MinVoteDuration) } // Ensure admin credentials are valid diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index 890b8e443..f5eaf29ce 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -951,8 +951,7 @@ func (c *Client) Proposals(p pi.Proposals) (*pi.ProposalsReply, error) { return &pr, nil } -// ProposalInventory retrieves the censorship tokens of all proposals, -// separated by their status. +// ProposalInventory sends a ProposalInventory request to politeiawww. func (c *Client) ProposalInventory(p pi.ProposalInventory) (*pi.ProposalInventoryReply, error) { statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, pi.RouteProposalInventory, p) diff --git a/politeiawww/pi.go b/politeiawww/pi.go index 6c2e48d86..d9bd4464f 100644 --- a/politeiawww/pi.go +++ b/politeiawww/pi.go @@ -83,18 +83,18 @@ func (p *politeiawww) piCommentCensor(ctx context.Context, cc piplugin.CommentCe return ccr, nil } -// proposalInventory returns the pi plugin proposal inventory. -func (p *politeiawww) proposalInventory(ctx context.Context, inv piplugin.ProposalInventory) (*piplugin.ProposalInventoryReply, error) { - b, err := piplugin.EncodeProposalInventory(inv) +// proposalInv returns the pi plugin proposal inventory. +func (p *politeiawww) proposalInv(ctx context.Context, inv piplugin.ProposalInv) (*piplugin.ProposalInvReply, error) { + b, err := piplugin.EncodeProposalInv(inv) if err != nil { return nil, err } r, err := p.pluginCommand(ctx, piplugin.ID, - piplugin.CmdProposalInventory, string(b)) + piplugin.CmdProposalInv, string(b)) if err != nil { return nil, err } - reply, err := piplugin.DecodeProposalInventoryReply(([]byte(r))) + reply, err := piplugin.DecodeProposalInvReply(([]byte(r))) if err != nil { return nil, err } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 44792b26f..06257728d 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -284,14 +284,14 @@ func convertFilesFromPD(f []pd.File) ([]pi.File, []pi.Metadata) { func convertProposalRecordFromPD(r pd.Record, state pi.PropStateT) (*pi.ProposalRecord, error) { // Decode metadata streams var ( - pg *piplugin.ProposalGeneral + gm *piplugin.GeneralMetadata sc = make([]piplugin.StatusChange, 0, 16) err error ) for _, v := range r.Metadata { switch v.ID { - case piplugin.MDStreamIDProposalGeneral: - pg, err = piplugin.DecodeProposalGeneral([]byte(v.Payload)) + case piplugin.MDStreamIDGeneralMetadata: + gm, err = piplugin.DecodeGeneralMetadata([]byte(v.Payload)) if err != nil { return nil, err } @@ -326,13 +326,13 @@ func convertProposalRecordFromPD(r pd.Record, state pi.PropStateT) (*pi.Proposal // plugin command. return &pi.ProposalRecord{ Version: r.Version, - Timestamp: pg.Timestamp, + Timestamp: gm.Timestamp, State: state, Status: status, UserID: "", // Intentionally omitted Username: "", // Intentionally omitted - PublicKey: pg.PublicKey, - Signature: pg.Signature, + PublicKey: gm.PublicKey, + Signature: gm.Signature, Comments: 0, // Intentionally omitted Statuses: statuses, Files: files, @@ -1190,19 +1190,19 @@ func (p *politeiawww) processProposalNew(ctx context.Context, pn pi.ProposalNew, // Setup metadata stream timestamp := time.Now().Unix() - pg := piplugin.ProposalGeneral{ + gm := piplugin.GeneralMetadata{ UserID: usr.ID.String(), PublicKey: pn.PublicKey, Signature: pn.Signature, Timestamp: timestamp, } - b, err := piplugin.EncodeProposalGeneral(pg) + b, err := piplugin.EncodeGeneralMetadata(gm) if err != nil { return nil, err } metadata := []pd.MetadataStream{ { - ID: piplugin.MDStreamIDProposalGeneral, + ID: piplugin.MDStreamIDGeneralMetadata, Payload: string(b), }, } @@ -1349,19 +1349,19 @@ func (p *politeiawww) processProposalEdit(ctx context.Context, pe pi.ProposalEdi // Setup politeiad metadata timestamp := time.Now().Unix() - pg := piplugin.ProposalGeneral{ + gm := piplugin.GeneralMetadata{ UserID: usr.ID.String(), PublicKey: pe.PublicKey, Signature: pe.Signature, Timestamp: timestamp, } - b, err := piplugin.EncodeProposalGeneral(pg) + b, err := piplugin.EncodeGeneralMetadata(gm) if err != nil { return nil, err } mdOverwrite := []pd.MetadataStream{ { - ID: piplugin.MDStreamIDProposalGeneral, + ID: piplugin.MDStreamIDGeneralMetadata, Payload: string(b), }, } @@ -1577,10 +1577,10 @@ func (p *politeiawww) processProposalInventory(ctx context.Context, inv pi.Propo log.Tracef("processProposalInventory: %v", inv.UserID) // Send plugin command - i := piplugin.ProposalInventory{ + i := piplugin.ProposalInv{ UserID: inv.UserID, } - pir, err := p.proposalInventory(ctx, i) + pir, err := p.proposalInv(ctx, i) if err != nil { return nil, err } diff --git a/politeiawww/sharedconfig/sharedconfig.go b/politeiawww/sharedconfig/sharedconfig.go index 7d723cb80..c316619cd 100644 --- a/politeiawww/sharedconfig/sharedconfig.go +++ b/politeiawww/sharedconfig/sharedconfig.go @@ -11,17 +11,23 @@ import ( ) const ( + // DefaultConfigFilename is the default configuration file name. DefaultConfigFilename = "politeiawww.conf" - DefaultDataDirname = "data" + + // DefaultDataDirname is the default data directory name. The data + // directory is located in the application home directory. + DefaultDataDirname = "data" ) var ( - // DefaultHomeDir points to politeiawww's home directory for configuration and data. + // DefaultHomeDir points to politeiawww's default home directory. DefaultHomeDir = dcrutil.AppDataDir("politeiawww", false) - // DefaultConfigFile points to politeiawww's default config file. + // DefaultConfigFile points to politeiawww's default config file + // path. DefaultConfigFile = filepath.Join(DefaultHomeDir, DefaultConfigFilename) - // DefaultDataDir points to politeiawww's default data directory. + // DefaultDataDir points to politeiawww's default data directory + // path. DefaultDataDir = filepath.Join(DefaultHomeDir, DefaultDataDirname) ) From e3ae0a4a014c3efaa8f4c8d629f7254054600eaa Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 24 Dec 2020 17:17:04 -0600 Subject: [PATCH 191/449] tlogbe: Cleanup trillian client. --- politeiad/backend/tlogbe/anchor.go | 6 +- politeiad/backend/tlogbe/testing.go | 31 -- politeiad/backend/tlogbe/tlog.go | 8 +- politeiad/backend/tlogbe/trillianclient.go | 420 ++++++++++++--------- politeiawww/templates.go | 2 - 5 files changed, 241 insertions(+), 226 deletions(-) diff --git a/politeiad/backend/tlogbe/anchor.go b/politeiad/backend/tlogbe/anchor.go index 264f4bbbc..23182a71d 100644 --- a/politeiad/backend/tlogbe/anchor.go +++ b/politeiad/backend/tlogbe/anchor.go @@ -83,7 +83,7 @@ func (t *tlog) anchorSave(a anchor) error { } prefixedKey := []byte(keyPrefixAnchorRecord + keys[0]) queued, _, err := t.trillian.leavesAppend(a.TreeID, []*trillian.LogLeaf{ - logLeafNew(h, prefixedKey), + newLogLeaf(h, prefixedKey), }) if err != nil { return fmt.Errorf("leavesAppend: %v", err) @@ -356,7 +356,7 @@ func (t *tlog) anchor() { default: // Anchor record found. If the anchor height differs from the // current height then the tree needs to be anchored. - _, lr, err := t.trillian.signedLogRootForTree(v) + _, lr, err := t.trillian.signedLogRoot(v) if err != nil { exitErr = fmt.Errorf("signedLogRoot %v: %v", v.TreeId, err) return @@ -372,7 +372,7 @@ func (t *tlog) anchor() { // Tree has not been anchored at current height. Add it to the // list of anchors. - _, lr, err := t.trillian.signedLogRootForTree(v) + _, lr, err := t.trillian.signedLogRoot(v) if err != nil { exitErr = fmt.Errorf("signedLogRoot %v: %v", v.TreeId, err) return diff --git a/politeiad/backend/tlogbe/testing.go b/politeiad/backend/tlogbe/testing.go index 2d6e2fdca..f9aa7dd4f 100644 --- a/politeiad/backend/tlogbe/testing.go +++ b/politeiad/backend/tlogbe/testing.go @@ -23,10 +23,6 @@ import ( "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" "github.com/decred/politeia/util" - "github.com/google/trillian" - "github.com/google/trillian/crypto/keys" - "github.com/google/trillian/crypto/keys/der" - "github.com/google/trillian/crypto/keyspb" "github.com/robfig/cron" ) @@ -114,33 +110,6 @@ func newBackendMetadataStream(t *testing.T, id uint64, payload string) backend.M } } -// newTestTClient provides a trillian client implementation used for -// testing. It implements the TClient interface, which includes all major -// tree operations used in the tlog backend. -func newTestTClient(t *testing.T) (*testTClient, error) { - // Create trillian private key - key, err := keys.NewFromSpec(&keyspb.Specification{ - Params: &keyspb.Specification_EcdsaParams{}, - }) - if err != nil { - return nil, err - } - keyDer, err := der.MarshalPrivateKey(key) - if err != nil { - return nil, err - } - - ttc := testTClient{ - trees: make(map[int64]*trillian.Tree), - leaves: make(map[int64][]*trillian.LogLeaf), - privateKey: &keyspb.PrivateKey{ - Der: keyDer, - }, - } - - return &ttc, nil -} - // newTestTlog returns a tlog used for testing. func newTestTlog(t *testing.T, id string) (*tlog, error) { // Setup key-value store with test dir diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index 9693c5670..a65770be9 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -450,7 +450,7 @@ func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []ba // Append record index leaf to the trillian tree leaves := []*trillian.LogLeaf{ - logLeafNew(idxHash, []byte(keyPrefixRecordIndex+keys[0])), + newLogLeaf(idxHash, []byte(keyPrefixRecordIndex+keys[0])), } queued, _, err := t.trillian.leavesAppend(treeID, leaves) if err != nil { @@ -541,7 +541,7 @@ func (t *tlog) recordIndexSave(treeID int64, ri recordIndex) error { return err } leaves := []*trillian.LogLeaf{ - logLeafNew(h, []byte(keyPrefixRecordIndex+keys[0])), + newLogLeaf(h, []byte(keyPrefixRecordIndex+keys[0])), } queued, _, err := t.trillian.leavesAppend(treeID, leaves) if err != nil { @@ -937,7 +937,7 @@ func (t *tlog) recordBlobsSave(treeID int64, rbpr recordBlobsPrepareReply) (*rec leaves := make([]*trillian.LogLeaf, 0, len(blobs)) for k := range blobs { pk := []byte(keyPrefixRecordContent + keys[k]) - leaves = append(leaves, logLeafNew(hashes[k], pk)) + leaves = append(leaves, newLogLeaf(hashes[k], pk)) } // Append leaves to trillian tree @@ -1578,7 +1578,7 @@ func (t *tlog) blobsSave(treeID int64, keyPrefix string, blobs, hashes [][]byte, leaves := make([]*trillian.LogLeaf, 0, len(blobs)) for k := range blobs { pk := []byte(keyPrefix + keys[k]) - leaves = append(leaves, logLeafNew(hashes[k], pk)) + leaves = append(leaves, newLogLeaf(hashes[k], pk)) } // Append leaves to trillian tree diff --git a/politeiad/backend/tlogbe/trillianclient.go b/politeiad/backend/tlogbe/trillianclient.go index 7765aef10..0c8c983fc 100644 --- a/politeiad/backend/tlogbe/trillianclient.go +++ b/politeiad/backend/tlogbe/trillianclient.go @@ -13,6 +13,7 @@ import ( "io/ioutil" "math/rand" "sync" + "testing" "time" "github.com/decred/politeia/util" @@ -34,43 +35,55 @@ import ( "google.golang.org/grpc/status" ) -var ( - _ trillianClient = (*tClient)(nil) - _ trillianClient = (*testTClient)(nil) -) - -// trillianClient provides an interface with basic tree operations needed for a -// trillian client. +// trillianClient provides an interface for interacting with a trillian log +// instance. It creates an abstraction over the trillian provided +// TrillianLogClient and TrillianAdminClient, creating a simplified API for the +// backend to use and allowing us to create a implementation that can be used +// for testing. type trillianClient interface { - tree(treeID int64) (*trillian.Tree, error) - - treesAll() ([]*trillian.Tree, error) - + // treeNew creates a new tree. treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) + // treeFreeze sets the status of a tree to frozen. treeFreeze(treeID int64) (*trillian.Tree, error) - leavesAll(treeID int64) ([]*trillian.LogLeaf, error) + // tree returns a tree. + tree(treeID int64) (*trillian.Tree, error) - leavesByRange(treeID, startIndex, count int64) ([]*trillian.LogLeaf, error) + // treesAll returns all trees in the trillian instance. + treesAll() ([]*trillian.Tree, error) + // leavesAppend appends leaves to a tree. leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, *types.LogRootV1, error) - signedLogRootForTree(tree *trillian.Tree) (*trillian.SignedLogRoot, + // leavesByRange returns the leaves within a specific index range. + leavesByRange(treeID, startIndex, count int64) ([]*trillian.LogLeaf, error) + + // leavesAll returns all leaves of a tree. + leavesAll(treeID int64) ([]*trillian.LogLeaf, error) + + // signedLogRoot returns the signed log root for a tree. + signedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoot, *types.LogRootV1, error) + // inclusionProof returns the inclusion proof for a leaf. + inclusionProof(treeID int64, merkleLeafHash []byte, + lrv1 *types.LogRootV1) (*trillian.Proof, error) + + // close closes the client connection. close() } -// tClient implements the TrillianClient interface and provides a client that -// abstracts over the existing TrillianLogClient and TrillianAdminClient. This -// provides a simplified API for the backend to use and ensures that proper -// verification of all trillian responses is performed. -type tClient struct { +var ( + _ trillianClient = (*tclient)(nil) +) + +// tclient implements the trillianClient interface. +type tclient struct { host string grpc *grpc.ClientConn - client trillian.TrillianLogClient + log trillian.TrillianLogClient admin trillian.TrillianAdminClient ctx context.Context privateKey *keyspb.PrivateKey // Trillian signing key @@ -101,7 +114,8 @@ func merkleLeafHash(leafValue []byte) []byte { return h.Sum(nil) } -func logLeafNew(value []byte, extraData []byte) *trillian.LogLeaf { +// newLogLeaf returns a trillian LogLeaf. +func newLogLeaf(value []byte, extraData []byte) *trillian.LogLeaf { return &trillian.LogLeaf{ LeafValue: value, ExtraData: extraData, @@ -112,8 +126,8 @@ func logLeafNew(value []byte, extraData []byte) *trillian.LogLeaf { // correct. It returns the tree and the signed log root which can be externally // verified. // -// This function satisfies the TrillianClient interface. -func (t *tClient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { +// This function satisfies the trillianClient interface. +func (t *tclient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { log.Tracef("trillian treeNew") pk, err := ptypes.MarshalAny(t.privateKey) @@ -141,7 +155,7 @@ func (t *tClient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { } // Init tree or signer goes bananas - ilr, err := t.client.InitLog(t.ctx, &trillian.InitLogRequest{ + ilr, err := t.log.InitLog(t.ctx, &trillian.InitLogRequest{ LogId: tree.TreeId, }) if err != nil { @@ -178,7 +192,10 @@ func (t *tClient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { return tree, ilr.Created, nil } -func (t *tClient) treeFreeze(treeID int64) (*trillian.Tree, error) { +// treeFreeze updates the state of a tree to frozen. +// +// This function satisfies the trillianClient interface. +func (t *tclient) treeFreeze(treeID int64) (*trillian.Tree, error) { log.Tracef("trillian treeFreeze: %v", treeID) // Get the current tree @@ -204,10 +221,10 @@ func (t *tClient) treeFreeze(treeID int64) (*trillian.Tree, error) { return updated, nil } -// tree returns a trillian tree by its ID. +// tree returns a trillian tree. // -// This function satisfies the TrillianClient interface. -func (t *tClient) tree(treeID int64) (*trillian.Tree, error) { +// This function satisfies the trillianClient interface. +func (t *tclient) tree(treeID int64) (*trillian.Tree, error) { log.Tracef("trillian tree: %v", treeID) tree, err := t.admin.GetTree(t.ctx, &trillian.GetTreeRequest{ @@ -225,10 +242,10 @@ func (t *tClient) tree(treeID int64) (*trillian.Tree, error) { return tree, nil } -// treesAll returns all trillian trees stored in the backend. +// treesAll returns all trees in the trillian instance. // -// This function satisfies the TrillianClient interface -func (t *tClient) treesAll() ([]*trillian.Tree, error) { +// This function satisfies the trillianClient interface +func (t *tclient) treesAll() ([]*trillian.Tree, error) { log.Tracef("trillian treesAll") ltr, err := t.admin.ListTrees(t.ctx, &trillian.ListTreesRequest{}) @@ -239,10 +256,13 @@ func (t *tClient) treesAll() ([]*trillian.Tree, error) { return ltr.Tree, nil } -func (t *tClient) inclusionProof(treeID int64, merkleLeafHash []byte, lrv1 *types.LogRootV1) (*trillian.Proof, error) { +// inclusionProof returns the inclusion proof for a trillian log leaf. +// +// This function satisfies the trillianClient interface +func (t *tclient) inclusionProof(treeID int64, merkleLeafHash []byte, lrv1 *types.LogRootV1) (*trillian.Proof, error) { log.Tracef("tillian inclusionProof: %v %x", treeID, merkleLeafHash) - resp, err := t.client.GetInclusionProofByHash(t.ctx, + resp, err := t.log.GetInclusionProofByHash(t.ctx, &trillian.GetInclusionProofByHashRequest{ LogId: treeID, LeafHash: merkleLeafHash, @@ -271,12 +291,12 @@ func (t *tClient) inclusionProof(treeID int64, merkleLeafHash []byte, lrv1 *type return proof, nil } -// signedLogRootForTree returns the signed log root of a trillian tree. +// signedLogRoot returns the signed log root of a trillian tree. // -// This function satisfies the TrillianClient interface. -func (t *tClient) signedLogRootForTree(tree *trillian.Tree) (*trillian.SignedLogRoot, *types.LogRootV1, error) { +// This function satisfies the trillianClient interface. +func (t *tclient) signedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoot, *types.LogRootV1, error) { // Get the signed log root for the current tree height - resp, err := t.client.GetLatestSignedLogRoot(t.ctx, + resp, err := t.log.GetLatestSignedLogRoot(t.ctx, &trillian.GetLatestSignedLogRootRequest{LogId: tree.TreeId}) if err != nil { return nil, nil, err @@ -296,23 +316,6 @@ func (t *tClient) signedLogRootForTree(tree *trillian.Tree) (*trillian.SignedLog return resp.SignedLogRoot, lrv1, nil } -// signedLogRoot returns the signed log root for the provided tree ID at its -// current height. The log root is structure is decoded an returned as well. -func (t *tClient) signedLogRoot(treeID int64) (*trillian.SignedLogRoot, *types.LogRootV1, error) { - log.Tracef("trillian signedLogRoot: %v", treeID) - - tree, err := t.tree(treeID) - if err != nil { - return nil, nil, fmt.Errorf("tree: %v", err) - } - slr, lr, err := t.signedLogRootForTree(tree) - if err != nil { - return nil, nil, fmt.Errorf("signedLogRoot: %v", err) - } - - return slr, lr, nil -} - // leavesAppend appends the provided leaves onto the provided tree. The queued // leaf and the leaf inclusion proof are returned. If a leaf was not // successfully appended, the queued leaf will still be returned and the error @@ -327,8 +330,8 @@ func (t *tClient) signedLogRoot(treeID int64) (*trillian.SignedLogRoot, *types.L // methods that can be used. DO NOT rely on the leaves being in a specific // order. // -// This function satisfies the TrillianClient interface. -func (t *tClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, *types.LogRootV1, error) { +// This function satisfies the trillianClient interface. +func (t *tclient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, *types.LogRootV1, error) { log.Tracef("trillian leavesAppend: %v", treeID) // Get the latest signed log root @@ -336,9 +339,9 @@ func (t *tClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queu if err != nil { return nil, nil, err } - slr, _, err := t.signedLogRootForTree(tree) + slr, _, err := t.signedLogRoot(tree) if err != nil { - return nil, nil, fmt.Errorf("signedLogRootForTree pre update: %v", err) + return nil, nil, fmt.Errorf("signedLogRoot pre update: %v", err) } // Ensure the tree is not frozen @@ -349,7 +352,7 @@ func (t *tClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queu log.Debugf("Appending %v leaves to tree id %v", len(leaves), treeID) // Append leaves to log - qlr, err := t.client.QueueLeaves(t.ctx, &trillian.QueueLeavesRequest{ + qlr, err := t.log.QueueLeaves(t.ctx, &trillian.QueueLeavesRequest{ LogId: treeID, Leaves: leaves, }) @@ -380,7 +383,7 @@ func (t *tClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queu if err != nil { return nil, nil, err } - c, err := client.NewFromTree(t.client, tree, logRoot) + c, err := client.NewFromTree(t.log, tree, logRoot) if err != nil { return nil, nil, err } @@ -394,9 +397,9 @@ func (t *tClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queu } // Get the latest signed log root - _, lr, err := t.signedLogRootForTree(tree) + _, lr, err := t.signedLogRoot(tree) if err != nil { - return nil, nil, fmt.Errorf("signedLogRootForTree post update: %v", err) + return nil, nil, fmt.Errorf("signedLogRoot post update: %v", err) } // Get inclusion proofs @@ -440,11 +443,11 @@ func (t *tClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queu // leavesByRange returns the log leaves of a trillian tree by the range provided // by the user. // -// This function satisfies the TrillianClient interface. -func (t *tClient) leavesByRange(treeID int64, startIndex, count int64) ([]*trillian.LogLeaf, error) { +// This function satisfies the trillianClient interface. +func (t *tclient) leavesByRange(treeID int64, startIndex, count int64) ([]*trillian.LogLeaf, error) { log.Tracef("trillian leavesByRange: %v %v %v", treeID, startIndex, count) - glbrr, err := t.client.GetLeavesByRange(t.ctx, + glbrr, err := t.log.GetLeavesByRange(t.ctx, &trillian.GetLeavesByRangeRequest{ LogId: treeID, StartIndex: startIndex, @@ -459,14 +462,20 @@ func (t *tClient) leavesByRange(treeID int64, startIndex, count int64) ([]*trill // leavesAll returns all of the leaves for the provided treeID. // -// This function satisfies the TrillianClient interface. -func (t *tClient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { +// This function satisfies the trillianClient interface. +func (t *tclient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { log.Tracef("trillian leavesAll: %v", treeID) - // Get log root - _, lr, err := t.signedLogRoot(treeID) + // Get tree + tree, err := t.tree(treeID) + if err != nil { + return nil, fmt.Errorf("tree: %v", err) + } + + // Get signed log root + _, lr, err := t.signedLogRoot(tree) if err != nil { - return nil, fmt.Errorf("SignedLogRoot: %v", err) + return nil, fmt.Errorf("signedLogRoot: %v", err) } if lr.TreeSize == 0 { return []*trillian.LogLeaf{}, nil @@ -479,12 +488,12 @@ func (t *tClient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { // leafProofs returns the leafProofs for the provided treeID and merkle leaf // hashes. The inclusion proof returned in the leafProof is for the tree height // specified by the provided LogRootV1. -func (t *tClient) leafProofs(treeID int64, merkleLeafHashes [][]byte, lr *types.LogRootV1) ([]leafProof, error) { +func (t *tclient) leafProofs(treeID int64, merkleLeafHashes [][]byte, lr *types.LogRootV1) ([]leafProof, error) { log.Tracef("trillian leafProofs: %v %v %x", treeID, lr.TreeSize, merkleLeafHashes) // Retrieve leaves - r, err := t.client.GetLeavesByHash(t.ctx, + r, err := t.log.GetLeavesByHash(t.ctx, &trillian.GetLeavesByHashRequest{ LogId: treeID, LeafHash: merkleLeafHashes, @@ -511,15 +520,91 @@ func (t *tClient) leafProofs(treeID int64, merkleLeafHashes [][]byte, lr *types. // close closes the trillian grpc connection. // -// This function satisfies the TrillianClient interface. -func (t *tClient) close() { +// This function satisfies the trillianClient interface. +func (t *tclient) close() { log.Tracef("trillian close %v", t.host) t.grpc.Close() } -// testTClient implements TClient interface and is used for -// testing purposes. +func newTrillianKey() (crypto.Signer, error) { + return keys.NewFromSpec(&keyspb.Specification{ + // TODO Params: &keyspb.Specification_Ed25519Params{}, + Params: &keyspb.Specification_EcdsaParams{}, + }) +} + +// newTClient returns a new tclient. +func newTClient(host, keyFile string) (*tclient, error) { + // Setup trillian key file + if !util.FileExists(keyFile) { + // Trillian key file does not exist. Create one. + log.Infof("Generating trillian private key") + key, err := newTrillianKey() + if err != nil { + return nil, err + } + b, err := der.MarshalPrivateKey(key) + if err != nil { + return nil, err + } + err = ioutil.WriteFile(keyFile, b, 0400) + if err != nil { + return nil, err + } + log.Infof("Trillian private key created: %v", keyFile) + } + + // Default gprc max message size is ~4MB (4194304 bytes). This is + // not large enough for trees with tens of thousands of leaves. + // Increase it to 20MB. + maxMsgSize := grpc.WithMaxMsgSize(20000000) + + // Setup trillian connection + // TODO should this be WithInsecure? + g, err := grpc.Dial(host, grpc.WithInsecure(), maxMsgSize) + if err != nil { + return nil, fmt.Errorf("grpc dial: %v", err) + } + + // Load trillian key pair + var privateKey = &keyspb.PrivateKey{} + privateKey.Der, err = ioutil.ReadFile(keyFile) + if err != nil { + return nil, err + } + signer, err := der.UnmarshalPrivateKey(privateKey.Der) + if err != nil { + return nil, err + } + + t := tclient{ + grpc: g, + log: trillian.NewTrillianLogClient(g), + admin: trillian.NewTrillianAdminClient(g), + ctx: context.Background(), + privateKey: privateKey, + publicKey: signer.Public(), + } + + // The grpc dial requires a little time to connect + time.Sleep(time.Second) + + // Ensure trillian is up and running + for t.grpc.GetState() != connectivity.Ready { + wait := 15 * time.Second + log.Infof("Cannot connect to trillian at %v; retry in %v ", host, wait) + time.Sleep(wait) + } + + return &t, nil +} + +var ( + _ trillianClient = (*testTClient)(nil) +) + +// testTClient implements the trillianClient interface and is used for testing. type testTClient struct { sync.RWMutex @@ -531,7 +616,7 @@ type testTClient struct { // treeNew ceates a new trillian tree in memory. // -// This function satisfies the TrillianClient interface. +// This function satisfies the trillianClient interface. func (t *testTClient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { t.Lock() defer t.Unlock() @@ -563,6 +648,9 @@ func (t *testTClient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) return &tree, nil, nil } +// treeFreeze sets the state of a tree to frozen. +// +// This function satisfies the trillianClient interface. func (t *testTClient) treeFreeze(treeID int64) (*trillian.Tree, error) { t.Lock() defer t.Unlock() @@ -574,7 +662,7 @@ func (t *testTClient) treeFreeze(treeID int64) (*trillian.Tree, error) { // tree returns trillian tree from passed in ID. // -// This function satisfies the TrillianClient interface. +// This function satisfies the trillianClient interface. func (t *testTClient) tree(treeID int64) (*trillian.Tree, error) { t.RLock() defer t.RUnlock() @@ -583,13 +671,13 @@ func (t *testTClient) tree(treeID int64) (*trillian.Tree, error) { return tree, nil } - return nil, fmt.Errorf("Tree ID not found") + return nil, fmt.Errorf("tree ID not found") } // treesAll signed log roots are not used for testing up until now, so we // return a nil value for it. // -// This function satisfies the TrillianClient interface. +// This function satisfies the trillianClient interface. func (t *testTClient) treesAll() ([]*trillian.Tree, error) { t.RLock() defer t.RUnlock() @@ -602,52 +690,10 @@ func (t *testTClient) treesAll() ([]*trillian.Tree, error) { return trees, nil } -// leavesAll returns all leaves from a trillian tree. -// -// This function satisfies the TrillianClient interface. -func (t *testTClient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { - t.RLock() - defer t.RUnlock() - - // Check if treeID entry exists - if _, ok := t.leaves[treeID]; !ok { - return nil, fmt.Errorf("Tree ID %d does not contain any leaf data", - treeID) - } - - return t.leaves[treeID], nil -} - -// leavesByRange returns leaves in range according to the passed in parameters. -// -// This function satisfies the TrillianClient interface. -func (t *testTClient) leavesByRange(treeID, startIndex, count int64) ([]*trillian.LogLeaf, error) { - t.RLock() - defer t.RUnlock() - - // Check if treeID entry exists - if _, ok := t.leaves[treeID]; !ok { - return nil, fmt.Errorf("Tree ID %d does not contain any leaf data", - treeID) - } - - // Get leaves by range. Indexes are ordered. - var c int64 - var leaves []*trillian.LogLeaf - for _, leaf := range t.leaves[treeID] { - if leaf.LeafIndex >= startIndex && c < count { - leaves = append(leaves, leaf) - c++ - } - } - - return nil, nil -} - // leavesAppend satisfies the TClient interface. It appends leaves to the // corresponding trillian tree in memory. // -// This function satisfies the TrillianClient interface. +// This function satisfies the trillianClient interface. func (t *testTClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, *types.LogRootV1, error) { t.Lock() defer t.Unlock() @@ -681,83 +727,85 @@ func (t *testTClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([] return queued, nil, nil } -// signedLogRootForTree is a stub to satisfy the interface. It is not used for -// testing. +// leavesByRange returns leaves in range according to the passed in parameters. // -// This function satisfies the TrillianClient interface. -func (t *testTClient) signedLogRootForTree(tree *trillian.Tree) (*trillian.SignedLogRoot, *types.LogRootV1, error) { - return nil, nil, nil -} +// This function satisfies the trillianClient interface. +func (t *testTClient) leavesByRange(treeID, startIndex, count int64) ([]*trillian.LogLeaf, error) { + t.RLock() + defer t.RUnlock() -// close is a stub to satisfy the interface. It is not used for testing. -// -// This function satisfies the TrillianClient interface. -func (t *testTClient) close() {} + // Check if treeID entry exists + if _, ok := t.leaves[treeID]; !ok { + return nil, fmt.Errorf("tree ID %d does not contain any leaf data", + treeID) + } -func newTClient(host, keyFile string) (*tClient, error) { - // Setup trillian key file - if !util.FileExists(keyFile) { - // Trillian key file does not exist. Create one. - log.Infof("Generating trillian private key") - key, err := keys.NewFromSpec(&keyspb.Specification{ - // TODO Params: &keyspb.Specification_Ed25519Params{}, - Params: &keyspb.Specification_EcdsaParams{}, - }) - if err != nil { - return nil, err - } - b, err := der.MarshalPrivateKey(key) - if err != nil { - return nil, err - } - err = ioutil.WriteFile(keyFile, b, 0400) - if err != nil { - return nil, err + // Get leaves by range. Indexes are ordered. + var c int64 + var leaves []*trillian.LogLeaf + for _, leaf := range t.leaves[treeID] { + if leaf.LeafIndex >= startIndex && c < count { + leaves = append(leaves, leaf) + c++ } - log.Infof("Trillian private key created: %v", keyFile) } - // Default gprc max message size is ~4MB (4194304 bytes). This is - // not large enough for trees with tens of thousands of leaves. - // Increase it to 20MB. - maxMsgSize := grpc.WithMaxMsgSize(20000000) + return nil, nil +} - // Setup trillian connection - // TODO should this be WithInsecure? - g, err := grpc.Dial(host, grpc.WithInsecure(), maxMsgSize) - if err != nil { - return nil, fmt.Errorf("grpc dial: %v", err) +// leavesAll returns all leaves from a trillian tree. +// +// This function satisfies the trillianClient interface. +func (t *testTClient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { + t.RLock() + defer t.RUnlock() + + // Check if treeID entry exists + if _, ok := t.leaves[treeID]; !ok { + return nil, fmt.Errorf("tree ID %d does not contain any leaf data", + treeID) } - // Load trillian key pair - var privateKey = &keyspb.PrivateKey{} - privateKey.Der, err = ioutil.ReadFile(keyFile) + return t.leaves[treeID], nil +} + +// signedLogRoot has not been implemented yet for the test client. +// +// This function satisfies the trillianClient interface. +func (t *testTClient) signedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoot, *types.LogRootV1, error) { + return nil, nil, nil +} + +// inclusionProof has not been implement yet for the test client. +// +// This function satisfies the trillianClient interface. +func (t *testTClient) inclusionProof(treeID int64, merkleLeafHash []byte, lrv1 *types.LogRootV1) (*trillian.Proof, error) { + return nil, nil +} + +// close closes the trillian client connection. There is nothing to do for the +// test implementation. +// +// This function satisfies the trillianClient interface. +func (t *testTClient) close() {} + +// newTestTClient returns a new testTClient. +func newTestTClient(t *testing.T) (*testTClient, error) { + // Create trillian private key + key, err := newTrillianKey() if err != nil { return nil, err } - signer, err := der.UnmarshalPrivateKey(privateKey.Der) + keyDer, err := der.MarshalPrivateKey(key) if err != nil { return nil, err } - t := tClient{ - grpc: g, - client: trillian.NewTrillianLogClient(g), - admin: trillian.NewTrillianAdminClient(g), - ctx: context.Background(), - privateKey: privateKey, - publicKey: signer.Public(), - } - - // The grpc dial requires a little time to connect - time.Sleep(time.Second) - - // Ensure trillian is up and running - for t.grpc.GetState() != connectivity.Ready { - wait := 15 * time.Second - log.Infof("Cannot connect to trillian at %v; retry in %v ", host, wait) - time.Sleep(wait) - } - - return &t, nil + return &testTClient{ + trees: make(map[int64]*trillian.Tree), + leaves: make(map[int64][]*trillian.LogLeaf), + privateKey: &keyspb.PrivateKey{ + Der: keyDer, + }, + }, nil } diff --git a/politeiawww/templates.go b/politeiawww/templates.go index f0a3c6b9a..2a64ca01d 100644 --- a/politeiawww/templates.go +++ b/politeiawww/templates.go @@ -6,8 +6,6 @@ package main import "text/template" -// TODO move templates to the email file where they're being used - // Proposal submitted - Send to admins type proposalSubmitted struct { Username string // Author username From 4cb73c8fd530ab3145223068cc5a48869c40d3c7 Mon Sep 17 00:00:00 2001 From: luke Date: Sat, 26 Dec 2020 12:03:33 -0600 Subject: [PATCH 192/449] cleanup --- politeiad/backend/tlogbe/dcrtime.go | 6 ++- politeiad/backend/tlogbe/tlogclient.go | 2 +- politeiad/backend/tlogbe/trillianclient.go | 59 +++++++++++----------- 3 files changed, 36 insertions(+), 31 deletions(-) diff --git a/politeiad/backend/tlogbe/dcrtime.go b/politeiad/backend/tlogbe/dcrtime.go index abbe14a3a..396d27f1a 100644 --- a/politeiad/backend/tlogbe/dcrtime.go +++ b/politeiad/backend/tlogbe/dcrtime.go @@ -1,3 +1,7 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + package tlogbe import ( @@ -82,7 +86,7 @@ func timestampBatch(host, id string, digests []string) (*dcrtime.TimestampBatchR // verifyBatch returns the data to verify that a digest was included in a // dcrtime timestamp. This function verifies the merkle path and merkle root of -// all successful timestamps. The caller is responsible for check the result +// all successful timestamps. The caller is responsible for checking the result // code and handling digests that failed to be timestamped. // // Note the Result in the reply will be set to OK as soon as the digest is diff --git a/politeiad/backend/tlogbe/tlogclient.go b/politeiad/backend/tlogbe/tlogclient.go index e6e4a906b..91d63e545 100644 --- a/politeiad/backend/tlogbe/tlogclient.go +++ b/politeiad/backend/tlogbe/tlogclient.go @@ -10,7 +10,7 @@ import ( // TODO verify writes only accept full length tokens -// tlogClient provides an API for the plugins to use to interact with the tlog +// tlogClient provides an API for the plugins to interact with the tlog // backend. Plugins are allowed to save, delete, and get plugin data to/from // the tlog backend. Editing plugin data is not allowed. type tlogClient interface { diff --git a/politeiad/backend/tlogbe/trillianclient.go b/politeiad/backend/tlogbe/trillianclient.go index 0c8c983fc..05d359031 100644 --- a/politeiad/backend/tlogbe/trillianclient.go +++ b/politeiad/backend/tlogbe/trillianclient.go @@ -35,11 +35,25 @@ import ( "google.golang.org/grpc/status" ) -// trillianClient provides an interface for interacting with a trillian log -// instance. It creates an abstraction over the trillian provided -// TrillianLogClient and TrillianAdminClient, creating a simplified API for the -// backend to use and allowing us to create a implementation that can be used -// for testing. +// queuedLeafProof contains the results of a leaf append command, i.e. the +// QueuedLeaf, and the inclusion proof for that leaf. If the leaf append +// command fails the QueuedLeaf will contain an error code from the failure and +// the Proof will not be present. +type queuedLeafProof struct { + QueuedLeaf *trillian.QueuedLogLeaf + Proof *trillian.Proof +} + +// leafProof contains a log leaf and the inclusion proof for the log leaf. +type leafProof struct { + Leaf *trillian.LogLeaf + Proof *trillian.Proof +} + +// trillianClient provides an interface for interacting with a trillian log. It +// creates an abstraction over the trillian provided TrillianLogClient and +// TrillianAdminClient, creating a simplified API for the backend to use and +// allowing us to create a implementation that can be used for testing. type trillianClient interface { // treeNew creates a new tree. treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) @@ -67,9 +81,11 @@ type trillianClient interface { signedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoot, *types.LogRootV1, error) - // inclusionProof returns the inclusion proof for a leaf. - inclusionProof(treeID int64, merkleLeafHash []byte, - lrv1 *types.LogRootV1) (*trillian.Proof, error) + // inclusionProofs returns the log leaves and the inclusion proofs + // for a set of merkle leaf hashes. The inclusion proofs returned + // for the tree height specified by the provided LogRootV1. + inclusionProofs(treeID int64, merkleLeafHashes [][]byte, + lrv1 *types.LogRootV1) ([]leafProof, error) // close closes the client connection. close() @@ -90,21 +106,6 @@ type tclient struct { publicKey crypto.PublicKey // Trillian public key } -// leafProof contains a log leaf and the inclusion proof for the log leaf. -type leafProof struct { - Leaf *trillian.LogLeaf - Proof *trillian.Proof -} - -// queuedLeafProof contains the results of a leaf append command, i.e. the -// QueuedLeaf, and the inclusion proof for that leaf. The inclusion proof will -// not be present if the leaf append command failed. The QueuedLeaf will -// contain the error code from the failure. -type queuedLeafProof struct { - QueuedLeaf *trillian.QueuedLogLeaf - Proof *trillian.Proof -} - // merkleLeafHash returns the merkle leaf hash for the provided leaf value. // This is the same merkle leaf hash that is calculated by trillian. func merkleLeafHash(leafValue []byte) []byte { @@ -485,11 +486,11 @@ func (t *tclient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { return t.leavesByRange(treeID, 0, int64(lr.TreeSize)) } -// leafProofs returns the leafProofs for the provided treeID and merkle leaf -// hashes. The inclusion proof returned in the leafProof is for the tree height +// inclusionProofs returns the log leaves and the inclusion proofs for a set of +// merkle leaf hashes. The inclusion proofs returned for the tree height // specified by the provided LogRootV1. -func (t *tclient) leafProofs(treeID int64, merkleLeafHashes [][]byte, lr *types.LogRootV1) ([]leafProof, error) { - log.Tracef("trillian leafProofs: %v %v %x", +func (t *tclient) inclusionProofs(treeID int64, merkleLeafHashes [][]byte, lr *types.LogRootV1) ([]leafProof, error) { + log.Tracef("trillian inclusionProofs: %v %v %x", treeID, lr.TreeSize, merkleLeafHashes) // Retrieve leaves @@ -776,10 +777,10 @@ func (t *testTClient) signedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoo return nil, nil, nil } -// inclusionProof has not been implement yet for the test client. +// inclusionProofs has not been implement yet for the test client. // // This function satisfies the trillianClient interface. -func (t *testTClient) inclusionProof(treeID int64, merkleLeafHash []byte, lrv1 *types.LogRootV1) (*trillian.Proof, error) { +func (t *testTClient) inclusionProofs(treeID int64, merkleLeafHashes [][]byte, lr *types.LogRootV1) ([]leafProof, error) { return nil, nil } From 5575bafcc9d9a392cc90285bf4f611a624aa01ac Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 28 Dec 2020 16:28:35 -0600 Subject: [PATCH 193/449] www: Update api docs. --- politeiawww/api/www/v1/api.md | 1015 ++------------------------------- politeiawww/api/www/v1/v1.go | 96 +++- politeiawww/api/www/v2/v2.go | 16 +- 3 files changed, 132 insertions(+), 995 deletions(-) diff --git a/politeiawww/api/www/v1/api.md b/politeiawww/api/www/v1/api.md index d14967195..04eca19d6 100644 --- a/politeiawww/api/www/v1/api.md +++ b/politeiawww/api/www/v1/api.md @@ -1,12 +1,12 @@ # politeiawww API Specification -# v1 - This document describes the REST API provided by a `politeiawww` server. The `politeiawww` server is the web server backend and it interacts with a JSON REST API. This document also describes websockets for server side notifications. It does not render HTML. +# v1 + **Methods** - [`Version`](#version) @@ -26,30 +26,16 @@ notifications. It does not render HTML. - [`Change password`](#change-password) - [`Reset password`](#reset-password) - [`User proposal credits`](#user-proposal-credits) -- [`User comments votes`](#user-comments-votes) - -**Proposal Routes** -- [`Vetted`](#vetted) -- [`User proposals`](#user-proposals) - [`Proposal paywall details`](#proposal-paywall-details) - [`Verify user payment`](#verify-user-payment) -- [`New proposal`](#new-proposal) -- [`Edit Proposal`](#edit-proposal) + +**Proposal Routes** +- [`Token inventory`](#token-inventory) - [`Proposal details`](#proposal-details) - [`Batch proposals`](#batch-proposals) -- [`Batch vote summary`](#batch-vote-summary) -- [`Set proposal status`](#set-proposal-status) -- [`Authorize vote`](#authorize-vote) -- [`Active votes`](#active-votes) -- [`Cast votes`](#cast-votes) -- [`Proposal vote status`](#proposal-vote-status) -- [`Proposals vote status`](#proposals-vote-status) - [`Vote results`](#vote-results) -- [`Token inventory`](#token-inventory) -- [`New comment`](#new-comment) -- [`Get comments`](#get-comments) -- [`Like comment`](#like-comment) -- [`Censor comment`](#censor-comment) +- [`Cast votes`](#cast-votes) +- [`Batch vote summary`](#batch-vote-summary) **Error status codes** @@ -1106,262 +1092,6 @@ Reply: } ``` -### `New proposal` - -Submit a new proposal to the politeiawww server. - -The Metadata field is required to include a [`Metadata`](#metadata) object that -contains an encoded [`ProposalMetadata`](#proposal-metadata). - -**Route:** `POST /v1/proposals/new` - -**Params:** - -| Parameter | Type | Description | Required | -|-----------|------|-------------|----------| -| files | array of [`File`](#file)s | Files are the body of the proposal. It should consist of one markdown file - named "index.md" - and up to five pictures. **Note:** all parameters within each [`File`](#file) are required. | Yes | -| metadata | array of [`Metadata`](#metadata) | User specified proposal metadata. | -| signature | string | Signature of the string representation of the Merkle root of the file payloads and the metadata payloads. Note that the merkle digests are calculated on the decoded payload.. | Yes | -| publickey | string | Public key from the client side, sent to politeiawww for verification | Yes | - -**Results:** - -| Parameter | Type | Description | -|-|-|-| -| censorshiprecord | [CensorshipRecord](#censorship-record) | A censorship record that provides the submitter with a method to extract the proposal and prove that he/she submitted it. | - -On failure the call shall return `400 Bad Request` and one of the following -error codes: -- [`ErrorStatusNoProposalCredits`](#ErrorStatusNoProposalCredits) -- [`ErrorStatusProposalMissingFiles`](#ErrorStatusProposalMissingFiles) -- [`ErrorStatusProposalDuplicateFilenames`](#ErrorStatusProposalDuplicateFilenames) -- [`ErrorStatusProposalInvalidTitle`](#ErrorStatusProposalInvalidTitle) -- [`ErrorStatusMaxMDsExceededPolicy`](#ErrorStatusMaxMDsExceededPolicy) -- [`ErrorStatusMaxImagesExceededPolicy`](#ErrorStatusMaxImagesExceededPolicy) -- [`ErrorStatusMaxMDSizeExceededPolicy`](#ErrorStatusMaxMDSizeExceededPolicy) -- [`ErrorStatusMaxImageSizeExceededPolicy`](#ErrorStatusMaxImageSizeExceededPolicy) -- [`ErrorStatusInvalidSignature`](#ErrorStatusInvalidSignature) -- [`ErrorStatusInvalidSigningKey`](#ErrorStatusInvalidSigningKey) -- [`ErrorStatusUserNotPaid`](#ErrorStatusUserNotPaid) - -**Example** - -Request: - -```json -{ - "name": "test", - "files": [{ - "name":"index.md", - "mime": "text/plain; charset=utf-8", - "digest": "", - "payload": "VGhpcyBpcyBhIGRlc2NyaXB0aW9u" - } - ] -} -``` - -Reply: - -```json -{ - "censorshiprecord": { - "token": "337fc4762dac6bbe11d3d0130f33a09978004b190e6ebbbde9312ac63f223527", - "merkle": "0dd10219cd79342198085cbe6f737bd54efe119b24c84cbc053023ed6b7da4c8", - "signature": "fcc92e26b8f38b90c2887259d88ce614654f32ecd76ade1438a0def40d360e461d995c796f16a17108fad226793fd4f52ff013428eda3b39cd504ed5f1811d0d" - } -} -``` - -### `Edit proposal` - -Edit an existent proposal into the politeiawww server. - -The Metadata field is required to include a [`Metadata`](#metadata) object that -contains an encoded [`ProposalMetadata`](#proposal-metadata). - -Note that updating public proposals will generate a new record version. While -updating an unvetted record will change the record but it will not generate -a new version. - -The example shown below is for a public proposal where the proposal version is -increased by one after the update. - -**Route:** `POST /v1/proposals/edit` - -**Params:** - -| Parameter | Type | Description | Required | -|-----------|------|-------------|----------| -| files | array of [`File`](#file)s | Files are the body of the proposal. It should consist of one markdown file - named "index.md" - and up to five pictures. **Note:** all parameters within each [`File`](#file) are required. | Yes | -| metadata | array of [`Metadata`](#metadata) | User specified proposal metadata. | -| signature | string | Signature of the string representation of the Merkle root of the file payloads and the metadata payloads. Note that the merkle digests are calculated on the decoded payload.. | Yes | -| publickey | string | Public key from the client side, sent to politeiawww for verification | Yes | - -**Results:** - -| | Type | Description | -|-|-|-| -| proposal | [`Proposal`](#proposal) | The updated proposal. | - -**Example:** - -Request: - -```json -{ - "files":[ - { - "name":"index.md", - "mime":"text/plain; charset=utf-8", - "payload":"RWRpdGVkIHByb3Bvc2FsCmVkaXRlZCBkZXNjcmlwdGlvbg==", - "digest":"a3c46ac82db1c9e5d780d9ddd046d73a0fdfcb1a2c55ab730f71a4213725e605" - } - ], - "publickey":"1bc17b4aaa7d08030d0cb984d3b67ce7b681508b46ce307b22dfd630141788a0", - "signature":"e8159f104bb4caa9a7952868ead44af8f1015cac72abd81b1fc83a434e26e0ce75c6a3a8a5c8d8f68405e82eea35c60e2d46fb0ff652eaf53690d57a7d4c8000", - "token":"6ef01f0ffae69fd267f98756231b8349a14f254c28d2312239cb80579e850337" -} -``` - -Reply: - -```json -{ - "proposal":{ - "name":"Edited proposal", - "status":4, - "timestamp":1535468714, - "publishedat": 1508296860900, - "censoredat": 0, - "abandonedat": 0, - "userid":"", - "username":"", - "publickey":"1bc17b4aaa7d08030d0cb984d3b67ce7b681508b46ce307b22dfd630141788a0", - "signature":"e8159f104bb4caa9a7952868ead44af8f1015cac72abd81b1fc83a434e26e0ce75c6a3a8a5c8d8f68405e82eea35c60e2d46fb0ff652eaf53690d57a7d4c8000", - "files":[ - { - "name":"index.md", - "mime":"text/plain; charset=utf-8", - "digest":"a3c46ac82db1c9e5d780d9ddd046d73a0fdfcb1a2c55ab730f71a4213725e605", - "payload":"RWRpdGVkIHByb3Bvc2FsCmVkaXRlZCBkZXNjcmlwdGlvbg==" - } - ], - "numcomments":0, - "version":"2", - "censorshiprecord":{ - "token":"6ef01f0ffae69fd267f98756231b8349a14f254c28d2312239cb80579e850337", - "merkle":"a3c46ac82db1c9e5d780d9ddd046d73a0fdfcb1a2c55ab730f71a4213725e605", - "signature":"aead575825a8cf3195079e263fe8eeb342f0fe51757e79de7bb8e733c672c0762cd3a0eb58de5057813028244910324d71ffd96d4a809a4c4634883b62a08007" - } - } -} -``` - -### `Vetted` - -Retrieve a page of vetted proposals; the number of proposals returned in the -page is limited by the `ProposalListPageSize` property, which is provided via -[`Policy`](#policy). - -**Route:** `GET /v1/proposals/vetted` - -**Params:** - -| Parameter | Type | Description | Required | -|-|-|-|-| -| before | String | A proposal censorship token; if provided, the page of proposals returned will end right before the proposal whose token is provided, when sorted in reverse chronological order. This parameter should not be specified if `after` is set. | | -| after | String | A proposal censorship token; if provided, the page of proposals returned will begin right after the proposal whose token is provided, when sorted in reverse chronological order. This parameter should not be specified if `before` is set. | | - -**Results:** - -| | Type | Description | -|-|-|-| -| proposals | Array of [`Proposal`](#proposal)s | An Array of vetted proposals. | - -**Example** - -Request: - -The request params should be provided within the URL: - -``` -/v1/proposals/vetted?after=f1c2042d36c8603517cf24768b6475e18745943e4c6a20bc0001f52a2a6f9bde -``` - -Reply: - -```json -{ - "proposals": [{ - "name": "My Proposal", - "status": 4, - "timestamp": 1508296860781, - "publishedat": 1508296860900, - "censoredat": 0, - "abandonedat": 0, - "censorshiprecord": { - "token": "337fc4762dac6bbe11d3d0130f33a09978004b190e6ebbbde9312ac63f223527", - "merkle": "0dd10219cd79342198085cbe6f737bd54efe119b24c84cbc053023ed6b7da4c8", - "signature": "fcc92e26b8f38b90c2887259d88ce614654f32ecd76ade1438a0def40d360e461d995c796f16a17108fad226793fd4f52ff013428eda3b39cd504ed5f1811d0d" - } - }] -} -``` - -### `User proposals` - -Retrieve a page and the total amount of proposals submitted by the given user; the number of proposals returned in the page is limited by the `proposallistpagesize` property, which is provided via [`Policy`](#policy). - -**Route:** `GET /v1/user/proposals` - -**Params:** - -| Parameter | Type | Description | Required | -|-|-|-|-| -| userid | String | The user id | -| before | String | A proposal censorship token; if provided, the page of proposals returned will end right before the proposal whose token is provided. This parameter should not be specified if `after` is set. | | -| after | String | A proposal censorship token; if provided, the page of proposals returned will begin right after the proposal whose token is provided. This parameter should not be specified if `before` is set. | | - -**Results:** - -| | Type | Description | -|-|-|-| -| proposals | array of [`Proposal`](#proposal)s | One page of user submitted proposals. | -| numOfProposals | int | Total number of proposals submitted by the user. If an admin is sending the request or a user is requesting their own proposals then this value includes unvetted, censored, and public proposals. Otherwise, this value only includes public proposals. | - -On failure the call shall return `400 Bad Request` and one of the following -error codes: -- [`ErrorStatusUserNotFound`](#ErrorStatusUserNotFound) - -**Example** - -Request: - -The request params should be provided within the URL: - -``` -/v1/user/proposals?userid=15&after=f1c2042d36c8603517cf24768b6475e18745943e4c6a20bc0001f52a2a6f9bde -``` - -Reply: - -```json -{ - "proposals": [{ - "name": "My Proposal", - "status": 2, - "timestamp": 1508296860781, - "censorshiprecord": { - "token": "337fc4762dac6bbe11d3d0130f33a09978004b190e6ebbbde9312ac63f223527", - "merkle": "0dd10219cd79342198085cbe6f737bd54efe119b24c84cbc053023ed6b7da4c8", - "signature": "fcc92e26b8f38b90c2887259d88ce614654f32ecd76ade1438a0def40d360e461d995c796f16a17108fad226793fd4f52ff013428eda3b39cd504ed5f1811d0d" - } - }], - "numofproposals": 1 -} -``` - ### `Policy` Retrieve server policy. The returned values contain various maxima that the @@ -1442,88 +1172,6 @@ Reply: } ``` -### `Set proposal status` - -Update the [status](#proposal-status-codes) of a proposal. This call requires -admin privileges. - -Unvetted proposals can have their status updated to: -`PropStatusPublic` -`PropStatusCensored` - -Vetted proposals can have their status updated to: -`PropStatusAbandoned` - -A status change message detailing the reason for the status change is required -for the following statuses: -`PropStatusCensored` -`PropStatusAbandoned` - -**Route:** `POST /v1/proposals/{token}/status` - -**Params:** - -| Parameter | Type | Description | Required | -|-|-|-|-| -| token | string | Token is the unique censorship token that identifies a specific proposal. | Yes | -| proposalstatus | [`proposal status`](#proposal-status-codes) | New proposal status.| Yes | -| statuschangemessage | string | Reason for status change. | No | -| signature | string | Signature of (token + proposalstatus + statuschangemessage). | Yes | -| publickey | string | Public key that corresponds to the signature. | Yes | - -**Results:** - -| Parameter | Type | Description | -|-|-|-| -| proposal | [`Proposal`](#proposal) | An entire proposal and it's contents. | - -On failure the call shall return `400 Bad Request` and one of the following -error codes: -- [`ErrorStatusChangeMessageCannotBeBlank`](#ErrorStatusChangeMessageCannotBeBlank) -- [`ErrorStatusInvalidSigningKey`](#ErrorStatusInvalidSigningKey) -- [`ErrorStatusInvalidSignature`](#ErrorStatusInvalidSignature) -- [`ErrorStatusProposalNotFound`](#ErrorStatusProposalNotFound) -- [`ErrorStatusReviewerAdminEqualsAuthor`](#ErrorStatusReviewerAdminEqualsAuthor) -- [`ErrorStatusInvalidPropStatusTransition`](#ErrorStatusInvalidPropStatusTransition) - -**Example** - -Request: - -```json -{ - "proposalstatus": 3, - "publickey": "f5519b6fdee08be45d47d5dd794e81303688a8798012d8983ba3f15af70a747c", - "signature": "041a12e5df95ec132be27f0c716fd8f7fc23889d05f66a26ef64326bd5d4e8c2bfed660235856da219237d185fb38c6be99125d834c57030428c6b96a2576900", - "token": "6161819a5df120162ed7b7fa5a95021f9d489a9eaf8b1bb23447fb8a5abc643b" -} -``` - -Reply: - -```json -{ - "proposal": { - "name": "My Proposal", - "state": "2", - "status": 4, - "timestamp": 1539212044, - "userid": "", - "username": "", - "publickey": "57cf10a15828c633dc0af423669e7bbad2d30a062e4eb1e9c78919f77ebd1022", - "signature": "553beffb3fece5bdd540e0b83e977e4f68c1ac31e6f2e0a85c3c9aef9e65e3efe3d778edc504a9e88c101f68ad25e677dc3574c67a6e8d0ba711de4b91bec40d", - "files": [], - "numcomments": 0, - "version": "1", - "censorshiprecord": { - "token": "fc320c72bb55b6233a8df388109bf494081f007395489a7cdc945e05d656a467", - "merkle": "ffc1e4b6a1b0b1e8eb99d476aed7ace9ed6475b3bbab9470d01028c24ae51992", - "signature": "4f409cfb706683e529281033945808cab286917f452ec1594d6f98b8fe2e11206e2b964ac9622c05e8465923f98dd4ee553b3eb08d54f0a3c7ef92f80db16d0a" - } - } -} -``` - ### `Proposal details` Retrieve proposal and its details. This request can be made with the full @@ -1779,482 +1427,49 @@ Reply: } ``` -### `New comment` +### `Cast votes` + +This is a batched call that casts multiple votes to multiple proposals. -Submit comment on given proposal. ParentID value "0" means "comment on -proposal"; if the value is not empty it means "reply to comment". +Note that the webserver does not interpret the plugin structures. These are +forwarded as-is to the politeia daemon. -**Route:** `POST /v1/comments/new` +**Route:** `POST /v1/proposals/castvotes` **Params:** | Parameter | Type | Description | Required | -| - | - | - | - | -| token | string | Censorship token | Yes | -| parentid | string | Parent comment identifier | Yes | -| comment | string | Comment | Yes | -| signature | string | Signature of Token, ParentID and Comment | Yes | -| publickey | string | Public key from the client side, sent to politeiawww for verification | Yes | +|-|-|-|-| +| votes | array of CastVote | All votes | Yes | -**Results:** +**CastVote:** | | Type | Description | | - | - | - | | token | string | Censorship token | -| parentid | string | Parent comment identifier | -| comment | string | Comment text | -| signature | string | Signature of Token, ParentID and Comment | -| publickey | string | Public key from the client side, sent to politeiawww for verification | -| commentid | string | Unique comment identifier | -| receipt | string | Server signature of the client Signature | -| timestamp | int64 | UNIX time when comment was accepted | -| resultvotes | int64 | Vote score | -| upvotes | uint64 | Pro votes | -| downvotes | uint64 | Contra votes | -| censored | bool | Has the comment been censored | -| userid | string | Unique user identifier | -| username | string | Unique username | +| ticket | string | Ticket hash | +| votebit | string | String encoded vote bit | +| signature | string | signature of Token+Ticket+VoteBit | -On failure the call shall return `400 Bad Request` and one of the following -error codes: +**Results:** -- [`ErrorStatusUserNotPaid`](#ErrorStatusUserNotPaid) -- [`ErrorStatusInvalidSigningKey`](#ErrorStatusInvalidSigningKey) -- [`ErrorStatusInvalidSignature`](#ErrorStatusInvalidSignature) -- [`ErrorStatusCommentLengthExceededPolicy`](#ErrorStatusCommentLengthExceededPolicy) -- [`ErrorStatusInvalidCensorshipToken`](#ErrorStatusInvalidCensorshipToken) -- [`ErrorStatusProposalNotFound`](#ErrorStatusProposalNotFound) -- [`ErrorStatusWrongStatus`](#ErrorStatusWrongStatus) -- [`ErrorStatusWrongVoteStatus`](#ErrorStatusWrongVoteStatus) -- [`ErrorStatusDuplicateComment`](#ErrorStatusDuplicateComment) +| | Type | Description | +| - | - | - | +| receipts | array of CastVoteReply | Receipts for all cast votes. This appears in the same order and index as the votes that went in. | + +**CastVoteReply:** + +| | Type | Description | +| - | - | - | +| clientsignature | string | Signature that was sent in via CastVote | +| signature | string | Signature of ClientSignature | +| error | string | Error, "" if there was no error | **Example** Request: -```json -{ - "token":"abf0fd1fc1b8c1c9535685373dce6c54948b7eb018e17e3a8cea26a3c9b85684", - "parentid":"0", - "comment":"I dont like this prop", - "signature":"af969d7f0f711e25cb411bdbbe3268bbf3004075cde8ebaee0fc9d988f24e45013cc2df6762dca5b3eb8abb077f76e0b016380a7eba2d46839b04c507d86290d", - "publickey":"4206fa1f45c898f1dee487d7a7a82e0ed293858313b8b022a6a88f2bcae6cdd7" -} -``` - -Reply: - -```json -{ - "token": "abf0fd1fc1b8c1c9535685373dce6c54948b7eb018e17e3a8cea26a3c9b85684", - "parentid": "0", - "comment": "I dont like this prop", - "signature":"af969d7f0f711e25cb411bdbbe3268bbf3004075cde8ebaee0fc9d988f24e45013cc2df6762dca5b3eb8abb077f76e0b016380a7eba2d46839b04c507d86290d", - "publickey": "4206fa1f45c898f1dee487d7a7a82e0ed293858313b8b022a6a88f2bcae6cdd7", - "commentid": "4", - "receipt": "96f3956ea3decb75ee129e6ee4e77c6c608f0b5c99ff41960a4e6078d8bb74e8ad9d2545c01fff2f8b7e0af38ee9de406aea8a0b897777d619e93d797bc1650a", - "timestamp": 1527277504, - "resultvotes": 0, - "censored": false, - "userid": "124", - "username": "john", -} -``` - -### `Get comments` - -Retrieve all comments for given proposal. Note that the comments are not -sorted. - -**Route:** `GET /v1/proposals/{token}/comments` - -**Params:** - -**Results:** - -| | Type | Description | -| - | - | - | -| Comments | Comment | Unsorted array of all comments | -| AccessTime | int64 | UNIX timestamp of last access time. Omitted if no session cookie is present. | - -**Comment:** - -| | Type | Description | -| - | - | - | -| userid | string | Unique user identifier | -| username | string | Unique username | -| timestamp | int64 | UNIX time when comment was accepted | -| commentid | string | Unique comment identifier | -| parentid | string | Parent comment identifier | -| token | string | Censorship token | -| comment | string | Comment text | -| publickey | string | Public key from the client side, sent to politeiawww for verification | -| signature | string | Signature of Token, ParentID and Comment | -| receipt | string | Server signature of the client Signature | -| totalvotes | uint64 | Total number of up/down votes | -| resultvotes | int64 | Vote score | -| upvotes | uint64 | Pro votes | -| downvotes | uint64 | Contra votes | - -**Example** - -Request: - -The request params should be provided within the URL: - -``` -/v1/proposals/f1c2042d36c8603517cf24768b6475e18745943e4c6a20bc0001f52a2a6f9bde/comments -``` - -Reply: - -```json -{ - "comments": [{ - "comment": "I dont like this prop", - "commentid": "4", - "parentid": "0", - "publickey": "4206fa1f45c898f1dee487d7a7a82e0ed293858313b8b022a6a88f2bcae6cdd7", - "receipt": "96f3956ea3decb75ee129e6ee4e77c6c608f0b5c99ff41960a4e6078d8bb74e8ad9d2545c01fff2f8b7e0af38ee9de406aea8a0b897777d619e93d797bc1650a", - "signature":"af969d7f0f711e25cb411bdbbe3268bbf3004075cde8ebaee0fc9d988f24e45013cc2df6762dca5b3eb8abb077f76e0b016380a7eba2d46839b04c507d86290d", - "timestamp": 1527277504, - "token": "abf0fd1fc1b8c1c9535685373dce6c54948b7eb018e17e3a8cea26a3c9b85684", - "userid": "124", - "username": "john", - "totalvotes": 4, - "resultvotes": 2, - "upvotes": 3, - "downvotes": 1 - },{ - "comment":"you are right!", - "commentid": "4", - "parentid": "0", - "publickey": "4206fa1f45c898f1dee487d7a7a82e0ed293858313b8b022a6a88f2bcae6cdd7", - "receipt": "96f3956ea3decb75ee129e6ee4e77c6c608f0b5c99ff41960a4e6078d8bb74e8ad9d2545c01fff2f8b7e0af38ee9de406aea8a0b897777d619e93d797bc1650a", - "signature":"af969d7f0f711e25cb411bdbbe3268bbf3004075cde8ebaee0fc9d988f24e45013cc2df6762dca5b3eb8abb077f76e0b016380a7eba2d46839b04c507d86290d", - "timestamp": 1527277504, - "token": "abf0fd1fc1b8c1c9535685373dce6c54948b7eb018e17e3a8cea26a3c9b85684", - "userid": "124", - "username": "john", - "totalvotes": 4, - "resultvotes": 2, - "upvotes": 3, - "downvotes": 1 - },{ - "comment":"you are crazy!", - "commentid": "4", - "parentid": "0", - "publickey": "4206fa1f45c898f1dee487d7a7a82e0ed293858313b8b022a6a88f2bcae6cdd7", - "receipt": "96f3956ea3decb75ee129e6ee4e77c6c608f0b5c99ff41960a4e6078d8bb74e8ad9d2545c01fff2f8b7e0af38ee9de406aea8a0b897777d619e93d797bc1650a", - "signature":"af969d7f0f711e25cb411bdbbe3268bbf3004075cde8ebaee0fc9d988f24e45013cc2df6762dca5b3eb8abb077f76e0b016380a7eba2d46839b04c507d86290d", - "timestamp": 1527277504, - "token": "abf0fd1fc1b8c1c9535685373dce6c54948b7eb018e17e3a8cea26a3c9b85684", - "userid": "124", - "username": "john", - "totalvotes": 4, - "resultvotes": 2, - "upvotes": 3, - "downvotes": 1 - }], - "accesstime": 1543539276 -} -``` - -### `Like comment` - -Allows a user to up or down vote a comment. Censored comments cannot be voted -on. - -**Route:** `POST v1/comments/like` - -**Params:** - -| Parameter | Type | Description | Required | -|-|-|-|-| -| token | string | Censorship token | yes | -| commentid | string | Unique comment identifier | yes | -| action | string | Up or downvote (1, -1) | yes | -| signature | string | Signature of Token, CommentId and Action | yes | -| publickey | string | Public key used for Signature | - -**Results:** - -| | Type | Description | -|-|-|-| -| total | uint64 | Total number of up and down votes | -| resultvotes | int64 | Vote score | -| upvotes | uint64 | Pro votes | -| downvotes | uint64 | Contra votes | -| receipt | string | Server signature of client signature | -| error | Error if something went wront during liking a comment -**Example:** - -Request: - -```json -{ - "token": "abf0fd1fc1b8c1c9535685373dce6c54948b7eb018e17e3a8cea26a3c9b85684", - "commentid": "4", - "action": "1", - "signature": "af969d7f0f711e25cb411bdbbe3268bbf3004075cde8ebaee0fc9d988f24e45013cc2df6762dca5b3eb8abb077f76e0b016380a7eba2d46839b04c507d86290d", - "publickey": "4206fa1f45c898f1dee487d7a7a82e0ed293858313b8b022a6a88f2bcae6cdd7" -} -``` - -Reply: - -```json -{ - "total": 4, - "result": 2, - "upvotes": 3, - "downvotes": 1, - "receipt": "96f3956ea3decb75ee129e6ee4e77c6c608f0b5c99ff41960a4e6078d8bb74e8ad9d2545c01fff2f8b7e0af38ee9de406aea8a0b897777d619e93d797bc1650a" -} -``` - -### `Censor comment` - -Allows a admin to censor a proposal comment. - -**Route:** `POST v1/comments/censor` - -**Params:** - -| Parameter | Type | Description | Required | -|-|-|-|-| -| token | string | Censorship token | yes | -| commentid | string | Unique comment identifier | yes | -| reason | string | Reason for censoring the comment | yes | -| signature | string | Signature of Token, CommentId and Reason | yes | -| publickey | string | Public key used for Signature | yes | - -**Results:** - -| | Type | Description | -|-|-|-| -| receipt | string | Server signature of client signature | - -On failure the call shall return `403 Forbidden` and one of the following -error codes: -- [`ErrorStatusUserNotPaid`](#ErrorStatusUserNotPaid) -- [`ErrorStatusInvalidSigningKey`](#ErrorStatusInvalidSigningKey) -- [`ErrorStatusInvalidSignature`](#ErrorStatusInvalidSignature) -- [`ErrorStatusProposalNotFound`](#ErrorStatusProposalNotFound) -- [`ErrorStatusWrongStatus`](#ErrorStatusWrongStatus) -- [`ErrorStatusWrongVoteStatus`](#ErrorStatusWrongVoteStatus) -- [`ErrorStatusCommentNotFound`](#ErrorStatusCommentNotFound) -- [`ErrorStatusCommentIsCensored`](#ErrorStatusCommentIsCensored) -- [`ErrorStatusInvalidLikeCommentAction`](#ErrorStatusInvalidLikeCommentAction) - -**Example:** - -Request: - -```json -{ - "token": "abf0fd1fc1b8c1c9535685373dce6c54948b7eb018e17e3a8cea26a3c9b85684", - "commentid": "4", - "reason": "comment was an advertisement", - "signature": "af969d7f0f711e25cb411bdbbe3268bbf3004075cde8ebaee0fc9d988f24e45013cc2df6762dca5b3eb8abb077f76e0b016380a7eba2d46839b04c507d86290d", - "publickey": "4206fa1f45c898f1dee487d7a7a82e0ed293858313b8b022a6a88f2bcae6cdd7" -} -``` - -Reply: - -```json -{ - "receipt": "96f3956ea3decb75ee129e6ee4e77c6c608f0b5c99ff41960a4e6078d8bb74e8ad9d2545c01fff2f8b7e0af38ee9de406aea8a0b897777d619e93d797bc1650a" -} -``` - -### `Authorize vote` - -Authorize a proposal vote. The proposal author must send an authorize vote -request to indicate that the proposal is in its final state and is ready to be -voted on before an admin can start the voting period for the proposal. The -author can also revoke a previously sent vote authorization. - -**Route:** `POST /v1/proposals/authorizevote` - -**Params:** - -| Parameter | Type | Description | Required | -|-|-|-|-| -| action | string | The action to be executed (authorize or revoke) | Yes | -| token | string | Proposal censorship token | Yes | -| signature | string | Signature of the token + proposal version | Yes | -| publickey | string | Public key used to sign the vote | Yes | - -**Results (StartVoteReply):** - -| | Type | Description | -| - | - | - | -| action | string | The action that was executed. | Yes | -| receipt | string | Politeiad signature of the client signature | - -On failure the call shall return `400 Bad Request` and one of the following -error codes: -- [`ErrorStatusNoPublicKey`](#ErrorStatusNoPublicKey) -- [`ErrorStatusInvalidSigningKey`](#ErrorStatusInvalidSigningKey) -- [`ErrorStatusInvalidSignature`](#ErrorStatusInvalidSignature) -- [`ErrorStatusWrongStatus`](#ErrorStatusWrongStatus) -- [`ErrorStatusInvalidAuthVoteAction`](#ErrorStatusInvalidAuthVoteAction) -- [`ErrorStatusVoteAlreadyAuthorized`](#ErrorStatusVoteAlreadyAuthorized) -- [`ErrorStatusVoteNotAuthorized`](#ErrorStatusVoteNotAuthorized) -- [`ErrorStatusUserNotAuthor`](#ErrorStatusUserNotAuthor) - -**Example** - -Request: - -``` json -{ - "action": "authorize", - "token": "657db73bca8afae3b99dd6ab1ac8564f71c7fb55713526e663afc3e9eff89233", - "signature": "aba600243e9e59927d3270742de25aae002c6c4952ddaf39702c328d855e9895ed9d9f8ee6154511b81c4272c2329e1e0bb2d79fe08626150a11bc78a4eefe00", - "publickey": "c2c2ea7f24733983bf8037c189f32b5da49e6396b7d21cb69efe09d290b3cb6d" -} -``` - -Reply: - -```json -{ - "action": "authorize", - "receipt": "2d7846cb3c8383b5db360ef6d1476341f07ab4d4819cdeac0601cfa5b8bca0ecf370402ba65ace249813caad50e7b0b6e92757a2bff94c385f71808bc5574203" -} -``` - -### `Active votes` - -Retrieve all active votes - -Note that the webserver does not interpret the plugin structures. These are -forwarded as-is to the politeia daemon. - -**Route:** `POST /v1/proposals/activevote` - -**Params:** - -**Results:** - -| | Type | Description | -| - | - | - | -| votes | array of ProposalVoteTuple | All current active votes | - -**ProposalVoteTuple:** - -| | Type | Description | -| - | - | - | -| proposal | ProposalRecord | Proposal record | -| startvote | Vote | Vote bits, mask etc | -| starvotereply | StartVoteReply | Vote details (eligible tickets, start block etc | - -**Example** - -Request: - -``` json -{} -``` - -Reply: - -```json -{ - "votes": [{ - "proposal": { - "name":"This is a description", - "status":4, - "timestamp":1523902523, - "userid":"", - "publickey":"d64d80c36441255e41fc1e7b6cd30259ff9a2b1276c32c7de1b7a832dff7f2c6", - "signature":"3554f74c112c5da49c6ee1787770c21fe1ae16f7f1205f105e6df1b5bdeaa2439fff6c477445e248e21bcf081c31bbaa96bfe03acace1629494e795e5d296e04", - "files":[], - "numcomments":0, - "censorshiprecord": { - "token":"8d14c77d9a28a1764832d0fcfb86b6af08f6b327347ab4af4803f9e6f7927225", - "merkle":"0dd10219cd79342198085cbe6f737bd54efe119b24c84cbc053023ed6b7da4c8", - "signature":"97b1bf0d63d7689a2c6e66e32358d48e98d84e5389f455cc135b3401277d3a37518827da0f2bc892b535937421418e7e8ba6a4f940dfcf19a219867fa8c3e005" - } - } - }], - "vote": { - "token":"8d14c77d9a28a1764832d0fcfb86b6af08f6b327347ab4af4803f9e6f7927225", - "mask":3, - "duration":2016, - "Options": [{ - "id":"no", - "description":"Don't approve proposal", - "bits":1 - },{ - "id":"yes", - "description":"Approve proposal", - "bits":2 - }] - }, - "votedetails": { - "startblockheight":"282893", - "startblockhash":"000000000227ff9b6bf3af53accb81e4fd1690ae44d521a665cb988bcd02ad94", - "endheight":"284909", - "eligibletickets": [ - "000011e329fe0359ea1d2070d927c93971232c1118502dddf0b7f1014bf38d97", - "0004b0f8b2883a2150749b2c8ba05652b02220e98895999fd96df790384888f9", - "00107166c5fc5c322ecda3748a1896f4a2de6672aae25014123d2cedc83e8f42", - "002272cf4788c3f726c30472f9c97d2ce66b997b5762ff4df6a05c4761272413" - ] - } -} -``` - -Note: eligibletickets is abbreviated for readability. - - -### `Cast votes` - -This is a batched call that casts multiple votes to multiple proposals. - -Note that the webserver does not interpret the plugin structures. These are -forwarded as-is to the politeia daemon. - -**Route:** `POST /v1/proposals/castvotes` - -**Params:** - -| Parameter | Type | Description | Required | -|-|-|-|-| -| votes | array of CastVote | All votes | Yes | - -**CastVote:** - -| | Type | Description | -| - | - | - | -| token | string | Censorship token | -| ticket | string | Ticket hash | -| votebit | string | String encoded vote bit | -| signature | string | signature of Token+Ticket+VoteBit | - -**Results:** - -| | Type | Description | -| - | - | - | -| receipts | array of CastVoteReply | Receipts for all cast votes. This appears in the same order and index as the votes that went in. | - -**CastVoteReply:** - -| | Type | Description | -| - | - | - | -| clientsignature | string | Signature that was sent in via CastVote | -| signature | string | Signature of ClientSignature | -| error | string | Error, "" if there was no error | - -**Example** - -Request: - -``` json +``` json { "votes": [{ "token":"642eb2f3798090b3234d8787aaba046f1f4409436d40994643213b63cb3f41da", @@ -2363,171 +1578,6 @@ Reply: } ``` -### `Proposal vote status` - -**This route deprecated by [`Batch Vote Status`](#batch-vote-status).** - -Returns the vote status for a single public proposal. - -**Route:** `GET /V1/proposals/{token}/votestatus` - -**Params:** none - -**Result:** - -| | Type | Description | -|-|-|-| -| token | string | Censorship token | -| status | int | Status identifier | -| optionsresult | array of VoteOptionResult | Option description along with the number of votes it has received | -| totalvotes | int | Proposal's total number of votes | -| bestblock | string | The current chain height | -| endheight | string | The chain height in which the vote will end | -| numofeligiblevotes | int | Total number of eligible votes | -| quorumpercentage | uint32 | Percent of eligible votes required for quorum | -| passpercentage | uint32 | Percent of total votes required to pass | - -**VoteOptionResult:** - -| | Type | Description | -|-|-|-| -| option | VoteOption | Option description | -| votesreceived | uint64 | Number of votes received | - - -**Proposal vote status map:** - -| status | value | -|-|-| -| Vote status invalid | 0 | -| Vote status not started | 1 | -| Vote status started | 2 | -| Vote status finished | 3 | -| Vote status doesn't exist | 4 | - -**Example:** - -Request: - -`GET /V1/proposals/b09dc5ac9d450b4d1ec6e8f80c763771f29413a5d1bf287054fc00c52ccc87c9/votestatus` - -Reply: - -```json -{ - "token":"b09dc5ac9d450b4d1ec6e8f80c763771f29413a5d1bf287054fc00c52ccc87c9", - "status":0, - "totalvotes":0, - "optionsresult":[ - { - "option":{ - "id":"no", - "description":"Don't approve proposal", - "bits":1 - }, - "votesreceived":0 - }, - { - "option":{ - "id":"yes", - "description":"Approve proposal", - "bits":2 - }, - "votesreceived":0 - } - ], - "bestblock": "45391", - "endheight": "45567", - "numofeligiblevotes": 2000, - "quorumpercentage": 20, - "passpercentage": 60 -} -``` - -### `Proposals vote status` - -**This route deprecated by [`Batch Vote Status`](#batch-vote-status).** - -Returns the vote status of all public proposals. - -**Route:** `GET /V1/proposals/votestatus` - -**Params:** none - -**Result:** - -| | Type | Description | -|-|-|-| -| votesstatus | array of VoteStatusReply | Vote status of each public proposal | - -**VoteStatusReply:** - -| | Type | Description | -|-|-|-| -| token | string | Censorship token | -| status | int | Status identifier | -| optionsresult | array of VoteOptionResult | Option description along with the number of votes it has received | -| totalvotes | int | Proposal's total number of votes | -| endheight | string | The chain height in which the vote will end | -| bestblock | string | The current chain height | -| numofeligiblevotes | int | Total number of eligible votes | -| quorumpercentage | uint32 | Percent of eligible votes required for quorum | -| passpercentage | uint32 | Percent of total votes required to pass | - -**Example:** - -Request: - -`GET /V1/proposals/votestatus` - -Reply: - -```json -{ - "votesstatus":[ - { - "token":"427af6d79f495e8dad2fb0a2a47594daa505b9fbfbd084f13678fa91882aef9f", - "status":2, - "optionsresult":[ - { - "option":{ - "id":"no", - "description":"Don't approve proposal", - "bits":1 - }, - "votesreceived":0 - }, - { - "option":{ - "id":"yes", - "description":"Approve proposal", - "bits":2 - }, - "votesreceived":0 - } - ], - "totalvotes":0, - "bestblock": "45392", - "endheight": "45567", - "numofeligiblevotes": 2000, - "quorumpercentage": 20, - "passpercentage": 60 - }, - { - "token":"b6d058cd1eed03d7fc9400f55384a8da33edb73743b7501d354392a6f9885078", - "status":1, - "optionsresult":null, - "totalvotes":0, - "bestblock": "45392", - "endheight": "", - "numofeligiblevotes": 0, - "quorumpercentage": 0, - "passpercentage": 0 - } - ] -} -``` - ### `User Comments Likes` Retrieve the comment votes for the current logged in user given a proposal token @@ -3068,4 +2118,3 @@ list being overwritten and thus an empty `rpcs` cancels all subscriptions. { "timestamp": 1547653596 } -``` diff --git a/politeiawww/api/www/v1/v1.go b/politeiawww/api/www/v1/v1.go index ae69a400b..0bf4ba20e 100644 --- a/politeiawww/api/www/v1/v1.go +++ b/politeiawww/api/www/v1/v1.go @@ -54,8 +54,8 @@ const ( RouteUnauthenticatedWebSocket = "/ws" RouteAuthenticatedWebSocket = "/aws" - // The following routes WILL BE DEPRECATED in the near future and - // should not be used. The pi v1 API should be used instead. + // The following routes have been DEPRECATED and support will be + // removed in the near future. RouteTokenInventory = "/proposals/tokeninventory" RouteProposalDetails = "/proposals/{token:[A-Fa-f0-9]{7,64}}" RouteBatchProposals = "/proposals/batch" @@ -63,8 +63,7 @@ const ( RouteCastVotes = "/proposals/castvotes" RouteBatchVoteSummary = "/proposals/batchvotesummary" - // The following route HAVE BEEN DEPRECATED. The pi v1 API should - // be used instead. + // The following routes are NO LONGER SUPPORTED. RouteActiveVote = "/proposals/activevote" RouteAllVetted = "/proposals/vetted" RouteNewProposal = "/proposals/new" @@ -699,6 +698,8 @@ type VerifyResetPasswordReply struct{} // If After is specified, the "page" returned starts after the proposal // whose censorship token is provided. If Before is specified, the "page" // returned starts before the proposal whose censorship token is provided. +// +// This request is NO LONGER SUPPORTED. type UserProposals struct { UserId string `schema:"userid"` Before string `schema:"before"` @@ -708,6 +709,8 @@ type UserProposals struct { // UserProposalsReply replies to the UserProposals command with // a list of proposals that the user has submitted and the total // amount of proposals +// +// This request is NO LONGER SUPPORTED. type UserProposalsReply struct { Proposals []ProposalRecord `json:"proposals"` // user proposals NumOfProposals int `json:"numofproposals"` // number of proposals submitted by the user @@ -846,6 +849,8 @@ type UserPaymentsRescanReply struct { // Signature is the signature of the proposal merkle root. The merkle root // contains the ordered files and metadata digests. The file digests are first // in the ordering. +// +// This request is NO LONGER SUPPORTED. type NewProposal struct { Files []File `json:"files"` // Proposal files Metadata []Metadata `json:"metadata"` // User specified metadata @@ -854,6 +859,8 @@ type NewProposal struct { } // NewProposalReply is used to reply to the NewProposal command +// +// This request is NO LONGER SUPPORTED. type NewProposalReply struct { CensorshipRecord CensorshipRecord `json:"censorshiprecord"` } @@ -862,12 +869,16 @@ type NewProposalReply struct { // and by the proposal version (optional). If the version isn't specified // the latest proposal version will be returned by default. Returns only // vetted proposals. +// +// This request has been DEPRECATED. type ProposalsDetails struct { Token string `json:"token"` // Censorship token Version string `json:"version,omitempty"` // Proposal version } // ProposalDetailsReply is used to reply to a proposal details command. +// +// This request has been DEPRECATED. type ProposalDetailsReply struct { Proposal ProposalRecord `json:"proposal"` } @@ -875,22 +886,30 @@ type ProposalDetailsReply struct { // BatchProposals is used to request the proposal details for each of the // provided censorship tokens. The returned proposals do not include the // proposal files. Returns only vetted proposals. +// +// This request has been DEPRECATED. type BatchProposals struct { Tokens []string `json:"tokens"` // Censorship tokens } // BatchProposalsReply is used to reply to a BatchProposals command. +// +// This request has been DEPRECATED. type BatchProposalsReply struct { Proposals []ProposalRecord `json:"proposals"` } // BatchVoteSummary is used to request the VoteSummary for the each of the // provided censorship tokens. +// +// This request has been DEPRECATED. type BatchVoteSummary struct { Tokens []string `json:"tokens"` // Censorship tokens } // BatchVoteSummaryReply is used to reply to a BatchVoteSummary command. +// +// This request has been DEPRECATED. type BatchVoteSummaryReply struct { BestBlock uint64 `json:"bestblock"` // Current block height Summaries map[string]VoteSummary `json:"summaries"` // [token]VoteSummary @@ -900,6 +919,8 @@ type BatchVoteSummaryReply struct { // have the ability to change a proposal's status. Some status changes, such // as censoring a proposal, require the StatusChangeMessage to be populated // with the reason for the status change. +// +// This request is NO LONGER SUPPORTED. type SetProposalStatus struct { Token string `json:"token"` // Proposal token ProposalStatus PropStatusT `json:"proposalstatus"` // New status @@ -909,6 +930,8 @@ type SetProposalStatus struct { } // SetProposalStatusReply is used to reply to a SetProposalStatus command. +// +// This request is NO LONGER SUPPORTED. type SetProposalStatusReply struct { Proposal ProposalRecord `json:"proposal"` } @@ -927,12 +950,16 @@ type SetProposalStatusReply struct { // // If Before is specified, the "page" returned starts before the provided // proposal censorship token, when sorted in reverse chronological order. +// +// This request is NO LONGER SUPPORTED. type GetAllVetted struct { Before string `schema:"before"` After string `schema:"after"` } // GetAllVettedReply is used to reply with a list of vetted proposals. +// +// This request is NO LONGER SUPPORTED. type GetAllVettedReply struct { Proposals []ProposalRecord `json:"proposals"` } @@ -988,6 +1015,8 @@ type Vote struct { } // ActiveVote obtains all proposals that have active votes. +// +// This request is NO LONGER SUPPORTED. type ActiveVote struct{} // ProposalVoteTuple is the proposal, vote and vote details. @@ -998,6 +1027,8 @@ type ProposalVoteTuple struct { } // ActiveVoteReply returns all proposals that have active votes. +// +// This request is NO LONGER SUPPORTED. type ActiveVoteReply struct { Votes []ProposalVoteTuple `json:"votes"` // Active votes } @@ -1008,6 +1039,8 @@ type ActiveVoteReply struct { // is ready to be voted on. The signature and public key are from the // proposal author. The author can revoke a previously sent vote authorization // by setting the Action field to revoke. +// +// This request is NO LONGER SUPPORTED. type AuthorizeVote struct { Action string `json:"action"` // Authorize or revoke Token string `json:"token"` // Proposal token @@ -1017,6 +1050,8 @@ type AuthorizeVote struct { // AuthorizeVoteReply returns a receipt if the action was successfully // executed. +// +// This request is NO LONGER SUPPORTED. type AuthorizeVoteReply struct { Action string `json:"action"` // Authorize or revoke Receipt string `json:"receipt"` // Server signature of client signature @@ -1024,8 +1059,7 @@ type AuthorizeVoteReply struct { // StartVote starts the voting process for a proposal. // -// THIS ROUTE HAS BEEN DEPRECATED -// A proposal vote must be initiated using the v2 StartVote route. +// This request is NO LONGER SUPPORTED. type StartVote struct { PublicKey string `json:"publickey"` // Key used for signature. Vote Vote `json:"vote"` // Vote @@ -1033,6 +1067,8 @@ type StartVote struct { } // StartVoteReply returns the eligible ticket pool. +// +// This request is NO LONGER SUPPORTED. type StartVoteReply struct { StartBlockHeight string `json:"startblockheight"` // Block height StartBlockHash string `json:"startblockhash"` // Block hash @@ -1059,11 +1095,15 @@ type CastVoteReply struct { } // Ballot is a batch of votes that are sent to the server. +// +// This request has been DEPRECATED. type Ballot struct { Votes []CastVote `json:"votes"` } // BallotReply is a reply to a batched list of votes. +// +// This request has been DEPRECATED. type BallotReply struct { Receipts []CastVoteReply `json:"receipts"` } @@ -1071,10 +1111,14 @@ type BallotReply struct { // VoteResults retrieves a single proposal vote results from the server. If the // voting period has not yet started for the given proposal a reply is returned // with all fields set to their zero value. +// +// This request has been DEPRECATED. type VoteResults struct{} // VoteResultsReply returns the original proposal vote and the associated cast // votes. +// +// This request has been DEPRECATED. type VoteResultsReply struct { StartVote StartVote `json:"startvote"` // Original vote CastVotes []CastVote `json:"castvotes"` // Vote results @@ -1109,6 +1153,8 @@ type Comment struct { // the user is implied by the session. A parent ID of 0 indicates that the // comment does not have a parent. A non-zero parent ID indicates that the // comment is a reply to an existing comment. +// +// This request is NO LONGER SUPPORTED. type NewComment struct { Token string `json:"token"` // Censorship token ParentID string `json:"parentid"` // Parent comment ID @@ -1119,16 +1165,22 @@ type NewComment struct { // NewCommentReply returns the site generated Comment ID or an error if // something went wrong. +// +// This request is NO LONGER SUPPORTED. type NewCommentReply struct { Comment Comment `json:"comment"` // Comment + receipt } // GetComments retrieve all comments for a given proposal. +// +// This request is NO LONGER SUPPORTED. type GetComments struct { Token string `json:"token"` // Censorship token } // GetCommentsReply returns the provided number of comments. +// +// This request is NO LONGER SUPPORTED. type GetCommentsReply struct { Comments []Comment `json:"comments"` // Comments AccessTime int64 `json:"accesstime,omitempty"` // User Access Time @@ -1142,6 +1194,8 @@ const ( ) // LikeComment allows a user to up or down vote a comment. +// +// This request is NO LONGER SUPPORTED. type LikeComment struct { Token string `json:"token"` // Censorship token CommentID string `json:"commentid"` // Comment ID @@ -1151,6 +1205,8 @@ type LikeComment struct { } // LikeCommentReply returns the current up/down vote result. +// +// This request is NO LONGER SUPPORTED. type LikeCommentReply struct { // XXX we probably need a sequence numkber or something here and some sort of rate limit Total uint64 `json:"total"` // Total number of up and down votes @@ -1163,6 +1219,8 @@ type LikeCommentReply struct { // CensorComment allows an admin to censor a comment. The signature and // public key are from the admin that censored this comment. +// +// This request is NO LONGER SUPPORTED. type CensorComment struct { Token string `json:"token"` // Proposal censorship token CommentID string `json:"commentid"` // Comment ID @@ -1173,6 +1231,8 @@ type CensorComment struct { // CensorCommentReply returns a receipt if the comment was successfully // censored. +// +// This request is NO LONGER SUPPORTED. type CensorCommentReply struct { Receipt string `json:"receipt"` // Server signature of client signature } @@ -1187,10 +1247,14 @@ type CommentLike struct { // UserCommentsLikes is a command to fetch all user vote actions // on the comments of a given proposal +// +// This request is NO LONGER SUPPORTED. type UserCommentsLikes struct{} // UserCommentsLikesReply is a reply with all user vote actions // for the comments of a given proposal +// +// This request is NO LONGER SUPPORTED. type UserCommentsLikesReply struct { CommentsLikes []CommentLike `json:"commentslikes"` } @@ -1204,11 +1268,13 @@ type VoteOptionResult struct { // VoteStatus is a command to fetch the the current vote status for a single // public proposal -// *** This is deprecated by the BatchVoteSummary request. *** +// +// This request is NO LONGER SUPPORTED. type VoteStatus struct{} // VoteStatusReply describes the vote status for a given proposal -// *** This is deprecated by the BatchVoteSummary request. *** +// +// This request is NO LONGER SUPPORTED. type VoteStatusReply struct { Token string `json:"token"` // Censorship token Status PropVoteStatusT `json:"status"` // Vote status (finished, started, etc) @@ -1222,11 +1288,13 @@ type VoteStatusReply struct { } // GetAllVoteStatus attempts to fetch the vote status of all public propsals -// *** This is deprecated by the BatchVoteSummary request. *** +// +// This request is NO LONGER SUPPORTED. type GetAllVoteStatus struct{} // GetAllVoteStatusReply returns the vote status of all public proposals -// *** This is deprecated by the BatchVoteSummary request. *** +// +// This request is NO LONGER SUPPORTED. type GetAllVoteStatusReply struct { VotesStatus []VoteStatusReply `json:"votesstatus"` // Vote status of all public proposals } @@ -1299,6 +1367,8 @@ type UserIdentity struct { // Signature is the signature of the proposal merkle root. The merkle root // contains the ordered files and metadata digests. The file digests are first // in the ordering. +// +// This request is NO LONGER SUPPORTED. type EditProposal struct { Token string `json:"token"` Files []File `json:"files"` @@ -1308,12 +1378,16 @@ type EditProposal struct { } // EditProposalReply is used to reply to the EditProposal command +// +// This request is NO LONGER SUPPORTED. type EditProposalReply struct { Proposal ProposalRecord `json:"proposal"` } // TokenInventory retrieves the censorship record tokens of all proposals in // the inventory, categorized by stage of the voting process. +// +// This request has been DEPRECATED. type TokenInventory struct{} // TokenInventoryReply is used to reply to the TokenInventory command and @@ -1326,6 +1400,8 @@ type TokenInventory struct{} // // Sorted by voting period end block height in descending order: // Active, Approved, Rejected +// +// This request has been DEPRECATED. type TokenInventoryReply struct { // Vetted Pre []string `json:"pre"` // Tokens of all props that are pre-vote diff --git a/politeiawww/api/www/v2/v2.go b/politeiawww/api/www/v2/v2.go index 8a7ac1d83..ca7b2c4a3 100644 --- a/politeiawww/api/www/v2/v2.go +++ b/politeiawww/api/www/v2/v2.go @@ -14,8 +14,8 @@ type VoteT int const ( APIVersion = 2 - // All www/v2 routes have been DEPRECATED. The pi/v1 API should be - // used instead. + // All routes in the package are NO LONGER SUPPORTED. The pi v1 API + // should be used instead. RouteStartVote = "/vote/start" RouteStartVoteRunoff = "/vote/startrunoff" RouteVoteDetails = "/vote/{token:[A-z0-9]{64}}" @@ -97,6 +97,8 @@ type Vote struct { // * Signature has been updated to be a signature of the Vote hash. It was // previously a signature of just the proposal token. // * Vote has been updated. See the Vote comment for more details. +// +// This request is NO LONGER SUPPORTED. Use the pi/v1 API instead. type StartVote struct { Vote Vote `json:"vote"` PublicKey string `json:"publickey"` // Key used for signature @@ -109,6 +111,8 @@ type StartVote struct { // * StartBlockHeight was changed from a string to a uint32. // * EndBlockHeight was changed from a string to a uint32. It was also renamed // from EndHeight to EndBlockHeight to be consistent with StartBlockHeight. +// +// This request is NO LONGER SUPPORTED. Use the pi/v1 API instead. type StartVoteReply struct { StartBlockHeight uint32 `json:"startblockheight"` // Block height of vote start StartBlockHash string `json:"startblockhash"` // Block hash of vote start @@ -130,6 +134,8 @@ type StartVoteReply struct { // submission deadline has expired. Once the LinkBy deadline has expired, the // runoff vote can be started at any point by an admin. It is not required that // RFP submission authors authorize the start of the vote. +// +// This request is NO LONGER SUPPORTED. Use the pi/v1 API instead. type StartVoteRunoff struct { Token string `json:"token"` AuthorizeVotes []AuthorizeVote `json:"authorizevotes"` @@ -138,6 +144,8 @@ type StartVoteRunoff struct { // The StartVoteRunoffReply is the reply to the StartVoteRunoff command. The // returned vote info will be the same for all RFP submissions. +// +// This request is NO LONGER SUPPORTED. Use the pi/v1 API instead. type StartVoteRunoffReply struct { StartBlockHeight uint32 `json:"startblockheight"` // Block height of vote start StartBlockHash string `json:"startblockhash"` // Block hash of vote start @@ -146,6 +154,8 @@ type StartVoteRunoffReply struct { } // VoteDetails returns the votes details for the specified proposal. +// +// This request is NO LONGER SUPPORTED. Use the pi/v1 API instead. type VoteDetails struct { Token string `json:"token"` // Proposal token } @@ -160,6 +170,8 @@ type VoteDetails struct { // Vote contains a JSON encoded Vote and needs to be decoded according to the // Version. See the Vote comment for details on the differences between the // Vote versions. +// +// This request is NO LONGER SUPPORTED. Use the pi/v1 API instead. type VoteDetailsReply struct { Version uint32 `json:"version"` // StartVote version Vote string `json:"vote"` // JSON encoded Vote struct From bb1185f0698d9e6e423b470673216c93ff735694 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 28 Dec 2020 19:33:01 -0600 Subject: [PATCH 194/449] multi: Cleanup. --- politeiad/backend/tlogbe/pi.go | 4 +- politeiad/plugins/comments/comments.go | 16 +-- politeiad/plugins/pi/pi.go | 128 ++++++++++----------- politeiad/plugins/ticketvote/ticketvote.go | 14 --- politeiawww/api/pi/v1/v1.go | 117 ++++++++++--------- politeiawww/cmd/piwww/help.go | 2 +- politeiawww/cmd/piwww/piwww.go | 2 +- politeiawww/cmd/piwww/proposalstatusset.go | 16 +-- politeiawww/cmd/piwww/testrun.go | 2 +- politeiawww/cmd/shared/client.go | 10 +- politeiawww/piwww.go | 28 ++--- 11 files changed, 159 insertions(+), 180 deletions(-) diff --git a/politeiad/backend/tlogbe/pi.go b/politeiad/backend/tlogbe/pi.go index f4c6dadfd..e3d324af1 100644 --- a/politeiad/backend/tlogbe/pi.go +++ b/politeiad/backend/tlogbe/pi.go @@ -1079,13 +1079,13 @@ func (p *piPlugin) cmdPassThrough(payload string) (string, error) { var fn func(string) (string, error) switch pt.PluginID { case ticketvote.ID: - switch pt.Cmd { + switch pt.PluginCmd { case ticketvote.CmdStart: fn = p.ticketVoteStart } default: return "", fmt.Errorf("invalid passthrough plugin command %v %v", - pt.PluginID, pt.Cmd) + pt.PluginID, pt.PluginCmd) } // Execute pass through diff --git a/politeiad/plugins/comments/comments.go b/politeiad/plugins/comments/comments.go index 8cf4bccea..dbaa9cc6d 100644 --- a/politeiad/plugins/comments/comments.go +++ b/politeiad/plugins/comments/comments.go @@ -52,6 +52,7 @@ const ( PolicyVoteChangesMax = 5 // Error status codes + // TODO number status codes ErrorStatusInvalid ErrorStatusT = 0 ErrorStatusStateInvalid ErrorStatusT = iota ErrorStatusTokenInvalid @@ -66,21 +67,6 @@ const ( ErrorStatusVoteChangesMax ) -var ( - // ErrorStatus contains human readable error statuses. - ErrorStatus = map[ErrorStatusT]string{ - ErrorStatusInvalid: "invalid error status", - ErrorStatusTokenInvalid: "invalid token", - ErrorStatusPublicKeyInvalid: "invalid public key", - ErrorStatusSignatureInvalid: "invalid signature", - ErrorStatusRecordNotFound: "record not found", - ErrorStatusCommentNotFound: "comment not found", - ErrorStatusParentIDInvalid: "parent id invalid", - ErrorStatusVoteInvalid: "invalid vote", - ErrorStatusVoteChangesMax: "vote changes max exceeded", - } -) - // Comment represent a record comment. // // Signature is the client signature of State+Token+ParentID+Comment. diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index 50d8de64e..83957dfdd 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -2,8 +2,8 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -// Package pi provides a plugin for functionality that is specific to decred's -// proposal system. +// Package pi provides a politeiad plugin for functionality that is specific to +// decred's proposal system. package pi import ( @@ -24,19 +24,19 @@ const ( ID = "pi" Version = "1" - // Plugin commands + // TODO these should use the PassThrough command CmdCommentNew = "commentnew" // Create a new comment CmdCommentCensor = "commentcensor" // Censor a comment - CmdCommentVote = "commentvote" // Upvote/downvote a comment - CmdProposalInv = "proposalinv" // Get inventory by proposal status - CmdVoteInventory = "voteinv" // Get inventory by vote status + CmdCommentVote = "commentvote" // Vote on a comment + + // Plugin commands + CmdPassThrough = "passthrough" // Plugin command pass through + CmdProposalInv = "proposalinv" // Get inventory by proposal status + CmdVoteInventory = "voteinv" // Get inventory by vote status // TODO get rid of CmdProposals CmdProposals = "proposals" // Get plugin data for proposals - // TODO get rid of CmdPassThrough - CmdPassThrough = "passthrough" // Pass a plugin cmd through pi - // Metadata stream IDs MDStreamIDGeneralMetadata = 1 MDStreamIDStatusChanges = 2 @@ -108,13 +108,6 @@ var ( PropStatusCensored: {}, PropStatusAbandoned: {}, } - - // ErrorStatus contains human readable user error statuses. - ErrorStatus = map[ErrorStatusT]string{ - ErrorStatusInvalid: "error status invalid", - ErrorStatusPropLinkToInvalid: "proposal link to invalid", - ErrorStatusVoteStatusInvalid: "vote status invalid", - } ) // ProposalMetadata contains metadata that is provided by the user as part of @@ -224,6 +217,58 @@ func DecodeStatusChanges(payload []byte) ([]StatusChange, error) { return statuses, nil } +// PassThrough is used to extended a command from a different plugin package +// with functionality that is specific to the pi plugin, without needing to +// define a new pi command and new pi types. The command passes through the pi +// plugin before being executed. +// +// Example, the pi plugin does not allow comments to be made on a proposal if +// the proposal vote has ended. This validation is specific to the pi plugin +// and does not require the comment payloads to be altered. PassThrough is used +// to pass the comments plugin command through the pi plugin, where the pi +// plugin can first perform pi specific validation before executing the +// comments plugin command. +type PassThrough struct { + PluginID string `json:"pluginid"` + PluginCmd string `json:"plugincmd"` + Payload string `json:"payload"` +} + +// EncodePassThrough encodes a PassThrough into a JSON byte slice. +func EncodePassThrough(p PassThrough) ([]byte, error) { + return json.Marshal(p) +} + +// DecodePassThrough decodes a JSON byte slice into a PassThrough. +func DecodePassThrough(payload []byte) (*PassThrough, error) { + var p PassThrough + err := json.Unmarshal(payload, &p) + if err != nil { + return nil, err + } + return &p, nil +} + +// PassThroughReply is the reply to the PassThrough command. +type PassThroughReply struct { + Payload string `json:"payload"` +} + +// EncodePassThroughReply encodes a PassThroughReply into a JSON byte slice. +func EncodePassThroughReply(p PassThroughReply) ([]byte, error) { + return json.Marshal(p) +} + +// DecodePassThroughReply decodes a JSON byte slice into a PassThroughReply. +func DecodePassThroughReply(payload []byte) (*PassThroughReply, error) { + var p PassThroughReply + err := json.Unmarshal(payload, &p) + if err != nil { + return nil, err + } + return &p, nil +} + // Proposals requests the plugin data for the provided proposals. This includes // pi plugin data as well as other plugin data such as comment plugin data. // This command aggregates all proposal plugin data into a single call. @@ -534,54 +579,3 @@ func DecodeVoteInventoryReply(payload []byte) (*VoteInventoryReply, error) { } return &vir, nil } - -// PassThrough is used to add additional functionality onto plugin commands -// from external plugin packages without changing the base command request and -// response payloads. The command passes through the pi plugin before being -// executed. -// -// Example, the pi plugin does not allow comments to be made once a proposal -// vote has ended. This validation is specific to the pi plugin and does not -// require the comment payloads to be altered. PassThrough is used to pass the -// comments plugin command through the pi plugin, where the pi plugin can first -// perform pi specific validation before executing the comments plugin command. -type PassThrough struct { - PluginID string `json:"pluginid"` - Cmd string `json:"cmd"` - Payload string `json:"payload"` -} - -// EncodePassThrough encodes a PassThrough into a JSON byte slice. -func EncodePassThrough(p PassThrough) ([]byte, error) { - return json.Marshal(p) -} - -// DecodePassThrough decodes a JSON byte slice into a PassThrough. -func DecodePassThrough(payload []byte) (*PassThrough, error) { - var p PassThrough - err := json.Unmarshal(payload, &p) - if err != nil { - return nil, err - } - return &p, nil -} - -// PassThroughReply is the reply to the PassThrough command. -type PassThroughReply struct { - Payload string `json:"payload"` -} - -// EncodePassThroughReply encodes a PassThroughReply into a JSON byte slice. -func EncodePassThroughReply(p PassThroughReply) ([]byte, error) { - return json.Marshal(p) -} - -// DecodePassThroughReply decodes a JSON byte slice into a PassThroughReply. -func DecodePassThroughReply(payload []byte) (*PassThroughReply, error) { - var p PassThroughReply - err := json.Unmarshal(payload, &p) - if err != nil { - return nil, err - } - return &p, nil -} diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index 20621516e..ef4b9aa6d 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -140,20 +140,6 @@ var ( VoteErrorTicketNotEligible: "ticket not eligible", VoteErrorTicketAlreadyVoted: "ticket already voted", } - - // ErrorStatus contains human readable user error statuses. - ErrorStatus = map[ErrorStatusT]string{ - ErrorStatusInvalid: "error status invalid", - ErrorStatusTokenInvalid: "token invalid", - ErrorStatusPublicKeyInvalid: "public key invalid", - ErrorStatusSignatureInvalid: "signature invalid", - ErrorStatusRecordNotFound: "record not found", - ErrorStatusRecordStatusInvalid: "record status invalid", - ErrorStatusAuthorizationInvalid: "authorization invalid", - ErrorStatusVoteParamsInvalid: "vote params invalid", - ErrorStatusVoteStatusInvalid: "vote status invalid", - ErrorStatusPageSizeExceeded: "page size exceeded", - } ) // AuthDetails is the structure that is saved to disk when a vote is authorized diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index cba3b5d32..3739d135b 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -9,8 +9,6 @@ import ( ) type ErrorStatusT int -type PropStateT int -type PropStatusT int type CommentVoteT int type VoteStatusT int type VoteAuthActionT string @@ -30,6 +28,7 @@ type VoteErrorT int // TODO make RouteVoteResults a batched route but that only currently allows // for 1 result to be returned so that we have the option to change this is // we want to. +// TODO each API needs a version and policy route const ( APIVersion = 1 @@ -41,7 +40,7 @@ const ( // Proposal routes RouteProposalNew = "/proposal/new" RouteProposalEdit = "/proposal/edit" - RouteProposalStatusSet = "/proposal/setstatus" + RouteProposalSetStatus = "/proposal/setstatus" RouteProposals = "/proposals" RouteProposalInventory = "/proposals/inventory" @@ -61,42 +60,6 @@ const ( RouteVoteSummaries = "/votes/summaries" RouteVoteInventory = "/votes/inventory" - // Proposal states. A proposal state can be either unvetted or - // vetted. The PropStatusT type further breaks down these two - // states into more granular statuses. Unvetted proposal data is - // not made available to the public. Only admins and the proposal - // author are able to view unvetted proposals. - PropStateInvalid PropStateT = 0 - PropStateUnvetted PropStateT = 1 - PropStateVetted PropStateT = 2 - - // PropStatusInvalid indicates the proposal status is invalid. - PropStatusInvalid PropStatusT = 0 - - // PropStatusUnreviewed indicates the proposal has been submitted, - // but has not yet been reviewed and made public by an admin. A - // proposal with this status will have a proposal state of - // PropStateUnvetted. - PropStatusUnreviewed PropStatusT = 1 - - // PropStatusPublic indicates that a proposal has been reviewed and - // made public by an admin. A proposal with this status will have - // a proposal state of PropStateVetted. - PropStatusPublic PropStatusT = 2 - - // PropStatusCensored indicates that a proposal has been censored - // by an admin for violating the proposal guidlines.. Both unvetted - // and vetted proposals can be censored so a proposal with this - // status can have a state of either PropStateUnvetted or - // PropStateVetted depending on whether the proposal was censored - // before or after it was made public. - PropStatusCensored PropStatusT = 3 - - // PropStatusAbandoned indicates that a proposal has been marked - // as abandoned by an admin due to the author being inactive. - // TODO can a unvetted proposal be abandoned? - PropStatusAbandoned PropStatusT = 4 - // Comment vote types CommentVoteInvalid CommentVoteT = 0 CommentVoteDownvote CommentVoteT = -1 @@ -220,15 +183,6 @@ const ( ) var ( - // PropStatuses contains the human readable proposal statuses. - PropStatuses = map[PropStatusT]string{ - PropStatusInvalid: "invalid", - PropStatusUnreviewed: "unreviewed", - PropStatusPublic: "public", - PropStatusCensored: "censored", - PropStatusAbandoned: "abandoned", - } - // ErrorStatus contains human readable error messages. // TODO fill in error status messages ErrorStatus = map[ErrorStatusT]string{ @@ -311,6 +265,65 @@ func (e ServerErrorReply) Error() string { return fmt.Sprintf("server error: %v", e.ErrorCode) } +// PropStateT represents a proposal state type. A proposal state can be either +// unvetted or vetted. The PropStatusT type further breaks down these two +// states into more granular statuses. +type PropStateT int + +const ( + // PropStateInvalid indicates an invalid proposal state. + PropStateInvalid PropStateT = 0 + + // PropStateUnvetted indicates a proposal has not been made public + // yet. Only admins and the proposal author are able to view + // unvetted proposals. + PropStateUnvetted PropStateT = 1 + + // PropStateVetted indicates a proposal has been made public. + PropStateVetted PropStateT = 2 +) + +// PropStatusT represents a proposal status type. +type PropStatusT int + +const ( + // PropStatusInvalid indicates the proposal status is invalid. + PropStatusInvalid PropStatusT = 0 + + // PropStatusUnreviewed indicates the proposal has been submitted, + // but has not yet been reviewed and made public by an admin. A + // proposal with this status will have a proposal state of + // PropStateUnvetted. + PropStatusUnreviewed PropStatusT = 1 + + // PropStatusPublic indicates that a proposal has been reviewed and + // made public by an admin. A proposal with this status will have + // a proposal state of PropStateVetted. + PropStatusPublic PropStatusT = 2 + + // PropStatusCensored indicates that a proposal has been censored + // by an admin for violating the proposal guidlines.. Both unvetted + // and vetted proposals can be censored so a proposal with this + // status can have a state of either PropStateUnvetted or + // PropStateVetted depending on whether the proposal was censored + // before or after it was made public. + PropStatusCensored PropStatusT = 3 + + // PropStatusAbandoned indicates that a proposal has been marked + // as abandoned by an admin due to the author being inactive. + // TODO can a unvetted proposal be abandoned? + PropStatusAbandoned PropStatusT = 4 +) + +// PropStatuses contains the human readable proposal statuses. +var PropStatuses = map[PropStatusT]string{ + PropStatusInvalid: "invalid", + PropStatusUnreviewed: "unreviewed", + PropStatusPublic: "public", + PropStatusCensored: "censored", + PropStatusAbandoned: "abandoned", +} + // File describes an individual file that is part of the proposal. The // directory structure must be flattened. type File struct { @@ -443,11 +456,11 @@ type ProposalEditReply struct { Proposal ProposalRecord `json:"proposal"` } -// ProposalStatusSet sets the status of a proposal. Some status changes require +// ProposalSetStatus sets the status of a proposal. Some status changes require // a reason to be included. // // Signature is the client signature of the Token+Version+Status+Reason. -type ProposalStatusSet struct { +type ProposalSetStatus struct { Token string `json:"token"` // Censorship token State PropStateT `json:"state"` // Proposal state Version string `json:"version"` // Proposal version @@ -457,8 +470,8 @@ type ProposalStatusSet struct { Signature string `json:"signature"` // Client signature } -// ProposalStatusSetReply is the reply to the ProposalStatusSet command. -type ProposalStatusSetReply struct { +// ProposalSetStatusReply is the reply to the ProposalSetStatus command. +type ProposalSetStatusReply struct { Proposal ProposalRecord `json:"proposal"` } diff --git a/politeiawww/cmd/piwww/help.go b/politeiawww/cmd/piwww/help.go index 54e614e42..10568572c 100644 --- a/politeiawww/cmd/piwww/help.go +++ b/politeiawww/cmd/piwww/help.go @@ -43,7 +43,7 @@ func (cmd *helpCmd) Execute(args []string) error { case "proposaledit": fmt.Printf("%s\n", proposalEditHelpMsg) case "proposalstatusset": - fmt.Printf("%s\n", proposalStatusSetHelpMsg) + fmt.Printf("%s\n", proposalSetStatusHelpMsg) case "proposals": fmt.Printf("%s\n", proposalsHelpMsg) case "proposalinventory": diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index 0a60cf4e1..696b12251 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -68,7 +68,7 @@ type piwww struct { // Proposal commands ProposalNew proposalNewCmd `command:"proposalnew"` ProposalEdit proposalEditCmd `command:"proposaledit"` - ProposalStatusSet proposalStatusSetCmd `command:"proposalstatusset"` + ProposalSetStatus proposalSetStatusCmd `command:"proposalsetstatus"` Proposals proposalsCmd `command:"proposals"` ProposalInv proposalInvCmd `command:"proposalinv"` diff --git a/politeiawww/cmd/piwww/proposalstatusset.go b/politeiawww/cmd/piwww/proposalstatusset.go index 17a88d1be..5e7aca3ff 100644 --- a/politeiawww/cmd/piwww/proposalstatusset.go +++ b/politeiawww/cmd/piwww/proposalstatusset.go @@ -13,8 +13,8 @@ import ( "github.com/decred/politeia/politeiawww/cmd/shared" ) -// proposalStatusSetCmd sets the status of a proposal. -type proposalStatusSetCmd struct { +// proposalSetStatusCmd sets the status of a proposal. +type proposalSetStatusCmd struct { Args struct { Token string `positional-arg-name:"token" required:"true"` Status string `positional-arg-name:"status" required:"true"` @@ -26,7 +26,7 @@ type proposalStatusSetCmd struct { } // Execute executes the proposal status set command. -func (cmd *proposalStatusSetCmd) Execute(args []string) error { +func (cmd *proposalSetStatusCmd) Execute(args []string) error { propStatus := map[string]pi.PropStatusT{ "public": pi.PropStatusPublic, "censored": pi.PropStatusCensored, @@ -63,7 +63,7 @@ func (cmd *proposalStatusSetCmd) Execute(args []string) error { status = s } else { return fmt.Errorf("Invalid proposal status '%v'\n %v", - cmd.Args.Status, proposalStatusSetHelpMsg) + cmd.Args.Status, proposalSetStatusHelpMsg) } // Verify version @@ -82,7 +82,7 @@ func (cmd *proposalStatusSetCmd) Execute(args []string) error { // Setup request msg := cmd.Args.Token + version + strconv.Itoa(int(status)) + cmd.Args.Reason sig := cfg.Identity.SignMessage([]byte(msg)) - pss := pi.ProposalStatusSet{ + pss := pi.ProposalSetStatus{ Token: cmd.Args.Token, State: state, Version: version, @@ -98,7 +98,7 @@ func (cmd *proposalStatusSetCmd) Execute(args []string) error { if err != nil { return err } - pssr, err := client.ProposalStatusSet(pss) + pssr, err := client.ProposalSetStatus(pss) if err != nil { return err } @@ -120,8 +120,8 @@ func (cmd *proposalStatusSetCmd) Execute(args []string) error { return nil } -// proposalStatusSetHelpMsg is the output of the help command. -const proposalStatusSetHelpMsg = `proposalstatusset "token" "status" "reason" +// proposalSetStatusHelpMsg is the output of the help command. +const proposalSetStatusHelpMsg = `proposalstatusset "token" "status" "reason" Set the status of a proposal. This command assumes the proposal is a vetted record. If the proposal is unvetted, the --unvetted flag must be used. Requires diff --git a/politeiawww/cmd/piwww/testrun.go b/politeiawww/cmd/piwww/testrun.go index a02bf5ab5..976409178 100644 --- a/politeiawww/cmd/piwww/testrun.go +++ b/politeiawww/cmd/piwww/testrun.go @@ -594,7 +594,7 @@ func proposalSetStatus(user testUser, state pi.PropStateT, token, reason string, return err } - pssc := proposalStatusSetCmd{ + pssc := proposalSetStatusCmd{ Unvetted: state == pi.PropStateUnvetted, } pssc.Args.Token = token diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index f5eaf29ce..c41fdc5c8 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -895,10 +895,10 @@ func (c *Client) ProposalEdit(pe pi.ProposalEdit) (*pi.ProposalEditReply, error) return &per, nil } -// ProposalStatusSet sets the status of a proposal -func (c *Client) ProposalStatusSet(pss pi.ProposalStatusSet) (*pi.ProposalStatusSetReply, error) { +// ProposalSetStatus sets the status of a proposal +func (c *Client) ProposalSetStatus(pss pi.ProposalSetStatus) (*pi.ProposalSetStatusReply, error) { statusCode, respBody, err := c.makeRequest(http.MethodPost, - pi.APIRoute, pi.RouteProposalStatusSet, pss) + pi.APIRoute, pi.RouteProposalSetStatus, pss) if err != nil { return nil, err } @@ -907,10 +907,10 @@ func (c *Client) ProposalStatusSet(pss pi.ProposalStatusSet) (*pi.ProposalStatus return nil, piError(respBody, statusCode) } - var pssr pi.ProposalStatusSetReply + var pssr pi.ProposalSetStatusReply err = json.Unmarshal(respBody, &pssr) if err != nil { - return nil, fmt.Errorf("unmarshal ProposalStatusSetReply: %v", err) + return nil, fmt.Errorf("unmarshal ProposalSetStatusReply: %v", err) } if c.cfg.Verbose { diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 06257728d..acbb220bd 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -1413,8 +1413,8 @@ func (p *politeiawww) processProposalEdit(ctx context.Context, pe pi.ProposalEdi }, nil } -func (p *politeiawww) processProposalStatusSet(ctx context.Context, pss pi.ProposalStatusSet, usr user.User) (*pi.ProposalStatusSetReply, error) { - log.Tracef("processProposalStatusSet: %v %v", pss.Token, pss.Status) +func (p *politeiawww) processProposalSetStatus(ctx context.Context, pss pi.ProposalSetStatus, usr user.User) (*pi.ProposalSetStatusReply, error) { + log.Tracef("processProposalSetStatus: %v %v", pss.Token, pss.Status) // Sanity check if !usr.Admin { @@ -1541,7 +1541,7 @@ func (p *politeiawww) processProposalStatusSet(ctx context.Context, pss pi.Propo return nil, err } - return &pi.ProposalStatusSetReply{ + return &pi.ProposalSetStatusReply{ Proposal: *pr, }, nil } @@ -1944,9 +1944,9 @@ func (p *politeiawww) processVoteStart(ctx context.Context, vs pi.VoteStart, usr return nil, err } pt := piplugin.PassThrough{ - PluginID: ticketvote.ID, - Cmd: ticketvote.CmdStart, - Payload: string(payload), + PluginID: ticketvote.ID, + PluginCmd: ticketvote.CmdStart, + Payload: string(payload), } ptr, err := p.piPassThrough(ctx, pt) if err != nil { @@ -2107,13 +2107,13 @@ func (p *politeiawww) handleProposalEdit(w http.ResponseWriter, r *http.Request) util.RespondWithJSON(w, http.StatusOK, per) } -func (p *politeiawww) handleProposalStatusSet(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleProposalStatusSet") +func (p *politeiawww) handleProposalSetStatus(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleProposalSetStatus") - var pss pi.ProposalStatusSet + var pss pi.ProposalSetStatus decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&pss); err != nil { - respondWithPiError(w, r, "handleProposalStatusSet: unmarshal", + respondWithPiError(w, r, "handleProposalSetStatus: unmarshal", pi.UserErrorReply{ ErrorCode: pi.ErrorStatusInputInvalid, }) @@ -2123,14 +2123,14 @@ func (p *politeiawww) handleProposalStatusSet(w http.ResponseWriter, r *http.Req usr, err := p.getSessionUser(w, r) if err != nil { respondWithPiError(w, r, - "handleProposalStatusSet: getSessionUser: %v", err) + "handleProposalSetStatus: getSessionUser: %v", err) return } - pssr, err := p.processProposalStatusSet(r.Context(), pss, *usr) + pssr, err := p.processProposalSetStatus(r.Context(), pss, *usr) if err != nil { respondWithPiError(w, r, - "handleProposalStatusSet: processProposalStatusSet: %v", err) + "handleProposalSetStatus: processProposalSetStatus: %v", err) return } @@ -2532,7 +2532,7 @@ func (p *politeiawww) setPiRoutes() { pi.RouteProposalEdit, p.handleProposalEdit, permissionLogin) p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteProposalStatusSet, p.handleProposalStatusSet, + pi.RouteProposalSetStatus, p.handleProposalSetStatus, permissionAdmin) p.addRoute(http.MethodPost, pi.APIRoute, pi.RouteProposals, p.handleProposals, From fa81a3c486a5f903985f712b837a74c229839bc3 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 28 Dec 2020 20:40:47 -0600 Subject: [PATCH 195/449] multi: Remove pi plugin comment commands. --- politeiad/backend/tlogbe/pi.go | 543 ++++++++++++++------------------- politeiad/plugins/pi/pi.go | 11 - politeiawww/api/pi/v1/v1.go | 35 ++- politeiawww/pi.go | 78 +---- politeiawww/piwww.go | 102 +++++-- 5 files changed, 330 insertions(+), 439 deletions(-) diff --git a/politeiad/backend/tlogbe/pi.go b/politeiad/backend/tlogbe/pi.go index e3d324af1..5f338c80a 100644 --- a/politeiad/backend/tlogbe/pi.go +++ b/politeiad/backend/tlogbe/pi.go @@ -429,8 +429,169 @@ func (p *piPlugin) voteSummary(token string) (*ticketvote.Summary, error) { return &summary, nil } -func (p *piPlugin) cmdCommentNew(payload string) (string, error) { - cn, err := pi.DecodeCommentNew([]byte(payload)) +func (p *piPlugin) cmdProposalInv(payload string) (string, error) { + // Decode payload + inv, err := pi.DecodeProposalInv([]byte(payload)) + if err != nil { + return "", err + } + + // Get full record inventory + ibs, err := p.backend.InventoryByStatus() + if err != nil { + return "", err + } + + // Apply user ID filtering criteria + if inv.UserID != "" { + // Lookup the proposal tokens that have been submitted by the + // specified user. + ud, err := p.userData(inv.UserID) + if err != nil { + return "", fmt.Errorf("userData %v: %v", inv.UserID, err) + } + userTokens := make(map[string]struct{}, len(ud.Tokens)) + for _, v := range ud.Tokens { + userTokens[v] = struct{}{} + } + + // Compile a list of unvetted tokens categorized by MDStatusT + // that were submitted by the user. + filtered := make(map[backend.MDStatusT][]string, len(ibs.Unvetted)) + for status, tokens := range ibs.Unvetted { + for _, v := range tokens { + _, ok := userTokens[v] + if !ok { + // Proposal was not submitted by the user + continue + } + + // Proposal was submitted by the user + ftokens, ok := filtered[status] + if !ok { + ftokens = make([]string, 0, len(tokens)) + } + filtered[status] = append(ftokens, v) + } + } + + // Update unvetted inventory with filtered tokens + ibs.Unvetted = filtered + + // Compile a list of vetted tokens categorized by MDStatusT that + // were submitted by the user. + filtered = make(map[backend.MDStatusT][]string, len(ibs.Vetted)) + for status, tokens := range ibs.Vetted { + for _, v := range tokens { + _, ok := userTokens[v] + if !ok { + // Proposal was not submitted by the user + continue + } + + // Proposal was submitted by the user + ftokens, ok := filtered[status] + if !ok { + ftokens = make([]string, 0, len(tokens)) + } + filtered[status] = append(ftokens, v) + } + } + + // Update vetted inventory with filtered tokens + ibs.Vetted = filtered + } + + // Convert MDStatus keys to human readable proposal statuses + unvetted := make(map[string][]string, len(ibs.Unvetted)) + vetted := make(map[string][]string, len(ibs.Vetted)) + for k, v := range ibs.Unvetted { + s := pi.PropStatuses[convertPropStatusFromMDStatus(k)] + unvetted[s] = v + } + for k, v := range ibs.Vetted { + s := pi.PropStatuses[convertPropStatusFromMDStatus(k)] + vetted[s] = v + } + + // Prepare reply + pir := pi.ProposalInvReply{ + Unvetted: unvetted, + Vetted: vetted, + } + reply, err := pi.EncodeProposalInvReply(pir) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *piPlugin) cmdVoteInventory(payload string) (string, error) { + // Payload is empty. Nothing to decode. + + // Get ticketvote inventory + r, err := p.backend.Plugin(ticketvote.ID, ticketvote.CmdInventory, "", "") + if err != nil { + return "", fmt.Errorf("ticketvote inventory: %v", err) + } + ir, err := ticketvote.DecodeInventoryReply([]byte(r)) + if err != nil { + return "", err + } + + // Get vote summaries for all finished proposal votes + s := ticketvote.Summaries{ + Tokens: ir.Finished, + } + b, err := ticketvote.EncodeSummaries(s) + if err != nil { + return "", err + } + r, err = p.backend.Plugin(ticketvote.ID, ticketvote.CmdSummaries, + "", string(b)) + if err != nil { + return "", fmt.Errorf("ticketvote summaries: %v", err) + } + sr, err := ticketvote.DecodeSummariesReply([]byte(r)) + if err != nil { + return "", err + } + if len(sr.Summaries) != len(ir.Finished) { + return "", fmt.Errorf("unexpected number of summaries: got %v, want %v", + len(sr.Summaries), len(ir.Finished)) + } + + // Categorize votes + approved := make([]string, 0, len(sr.Summaries)) + rejected := make([]string, 0, len(sr.Summaries)) + for token, v := range sr.Summaries { + if v.Approved { + approved = append(approved, token) + } else { + rejected = append(rejected, token) + } + } + + // Prepare reply + vir := pi.VoteInventoryReply{ + Unauthorized: ir.Unauthorized, + Authorized: ir.Authorized, + Started: ir.Started, + Approved: approved, + Rejected: rejected, + BestBlock: ir.BestBlock, + } + reply, err := pi.EncodeVoteInventoryReply(vir) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *piPlugin) commentNew(payload string) (string, error) { + n, err := comments.DecodeNew([]byte(payload)) if err != nil { return "", err } @@ -440,8 +601,8 @@ func (p *piPlugin) cmdCommentNew(payload string) (string, error) { // vote summary request can complete successfully. // Verify state - switch cn.State { - case pi.PropStateUnvetted, pi.PropStateVetted: + switch n.State { + case comments.StateUnvetted, comments.StateVetted: // Allowed; continue default: return "", backend.PluginUserError{ @@ -451,7 +612,7 @@ func (p *piPlugin) cmdCommentNew(payload string) (string, error) { } // Verify token - token, err := util.ConvertStringToken(cn.Token) + token, err := util.ConvertStringToken(n.Token) if err != nil { return "", backend.PluginUserError{ PluginID: pi.ID, @@ -461,14 +622,14 @@ func (p *piPlugin) cmdCommentNew(payload string) (string, error) { // Verify record exists var exists bool - switch cn.State { - case pi.PropStateUnvetted: + switch n.State { + case comments.StateUnvetted: exists = p.backend.UnvettedExists(token) - case pi.PropStateVetted: + case comments.StateVetted: exists = p.backend.VettedExists(token) default: // Should not happen. State has already been validated. - return "", fmt.Errorf("invalid state %v", cn.State) + return "", fmt.Errorf("invalid state %v", n.State) } if !exists { return "", backend.PluginUserError{ @@ -478,8 +639,8 @@ func (p *piPlugin) cmdCommentNew(payload string) (string, error) { } // Verify vote status - if cn.State == pi.PropStateVetted { - vs, err := p.voteSummary(cn.Token) + if n.State == comments.StateVetted { + vs, err := p.voteSummary(n.Token) if err != nil { return "", fmt.Errorf("voteSummary: %v", err) } @@ -496,45 +657,12 @@ func (p *piPlugin) cmdCommentNew(payload string) (string, error) { } } - // Setup plugin command - n := comments.New{ - UserID: cn.UserID, - State: comments.StateT(cn.State), - Token: cn.Token, - ParentID: cn.ParentID, - Comment: cn.Comment, - PublicKey: cn.PublicKey, - Signature: cn.Signature, - } - b, err := comments.EncodeNew(n) - if err != nil { - return "", err - } - // Send plugin command - r, err := p.backend.Plugin(comments.ID, comments.CmdNew, "", string(b)) - if err != nil { - return "", err - } - - // Prepare reply - nr, err := comments.DecodeNewReply([]byte(r)) - if err != nil { - return "", err - } - cnr := pi.CommentNewReply{ - Comment: nr.Comment, - } - reply, err := pi.EncodeCommentNewReply(cnr) - if err != nil { - return "", err - } - - return string(reply), nil + return p.backend.Plugin(comments.ID, comments.CmdNew, "", payload) } -func (p *piPlugin) cmdCommentCensor(payload string) (string, error) { - cc, err := pi.DecodeCommentCensor([]byte(payload)) +func (p *piPlugin) commentDel(payload string) (string, error) { + d, err := comments.DecodeDel([]byte(payload)) if err != nil { return "", err } @@ -544,8 +672,8 @@ func (p *piPlugin) cmdCommentCensor(payload string) (string, error) { // vote summary request can complete successfully. // Verify state - switch cc.State { - case pi.PropStateUnvetted, pi.PropStateVetted: + switch d.State { + case comments.StateUnvetted, comments.StateVetted: // Allowed; continue default: return "", backend.PluginUserError{ @@ -555,7 +683,7 @@ func (p *piPlugin) cmdCommentCensor(payload string) (string, error) { } // Verify token - token, err := util.ConvertStringToken(cc.Token) + token, err := util.ConvertStringToken(d.Token) if err != nil { return "", backend.PluginUserError{ PluginID: pi.ID, @@ -565,14 +693,14 @@ func (p *piPlugin) cmdCommentCensor(payload string) (string, error) { // Verify record exists var exists bool - switch cc.State { - case pi.PropStateUnvetted: + switch d.State { + case comments.StateUnvetted: exists = p.backend.UnvettedExists(token) - case pi.PropStateVetted: + case comments.StateVetted: exists = p.backend.VettedExists(token) default: // Should not happen. State has already been validated. - return "", fmt.Errorf("invalid state %v", cc.State) + return "", fmt.Errorf("invalid state %v", d.State) } if !exists { return "", backend.PluginUserError{ @@ -582,15 +710,15 @@ func (p *piPlugin) cmdCommentCensor(payload string) (string, error) { } // Verify vote status - if cc.State == pi.PropStateVetted { - vs, err := p.voteSummary(cc.Token) + if d.State == comments.StateVetted { + vs, err := p.voteSummary(d.Token) if err != nil { return "", fmt.Errorf("voteSummary: %v", err) } switch vs.Status { case ticketvote.VoteStatusUnauthorized, ticketvote.VoteStatusAuthorized, ticketvote.VoteStatusStarted: - // Censoring is allowed on these vote statuses; continue + // Deling is allowed on these vote statuses; continue default: return "", backend.PluginUserError{ PluginID: pi.ID, @@ -600,44 +728,12 @@ func (p *piPlugin) cmdCommentCensor(payload string) (string, error) { } } - // Setup plugin command - d := comments.Del{ - State: comments.StateT(cc.State), - Token: cc.Token, - CommentID: cc.CommentID, - Reason: cc.Reason, - PublicKey: cc.PublicKey, - Signature: cc.Signature, - } - b, err := comments.EncodeDel(d) - if err != nil { - return "", err - } - // Send plugin command - r, err := p.backend.Plugin(comments.ID, comments.CmdDel, "", string(b)) - if err != nil { - return "", err - } - - // Prepare reply - dr, err := comments.DecodeDelReply([]byte(r)) - if err != nil { - return "", err - } - ccr := pi.CommentCensorReply{ - Comment: dr.Comment, - } - reply, err := pi.EncodeCommentCensorReply(ccr) - if err != nil { - return "", err - } - - return string(reply), nil + return p.backend.Plugin(comments.ID, comments.CmdDel, "", payload) } -func (p *piPlugin) cmdCommentVote(payload string) (string, error) { - cv, err := pi.DecodeCommentVote([]byte(payload)) +func (p *piPlugin) commentVote(payload string) (string, error) { + v, err := comments.DecodeVote([]byte(payload)) if err != nil { return "", err } @@ -647,8 +743,8 @@ func (p *piPlugin) cmdCommentVote(payload string) (string, error) { // vote summary request can complete successfully. // Verify state - switch cv.State { - case pi.PropStateUnvetted, pi.PropStateVetted: + switch v.State { + case comments.StateUnvetted, comments.StateVetted: // Allowed; continue default: return "", backend.PluginUserError{ @@ -658,7 +754,7 @@ func (p *piPlugin) cmdCommentVote(payload string) (string, error) { } // Verify token - token, err := util.ConvertStringToken(cv.Token) + token, err := util.ConvertStringToken(v.Token) if err != nil { return "", backend.PluginUserError{ PluginID: pi.ID, @@ -668,14 +764,14 @@ func (p *piPlugin) cmdCommentVote(payload string) (string, error) { // Verify record exists var record *backend.Record - switch cv.State { - case pi.PropStateUnvetted: + switch v.State { + case comments.StateUnvetted: record, err = p.backend.GetUnvetted(token, "") - case pi.PropStateVetted: + case comments.StateVetted: record, err = p.backend.GetVetted(token, "") default: // Should not happen. State has already been validated. - return "", fmt.Errorf("invalid state %v", cv.State) + return "", fmt.Errorf("invalid state %v", v.State) } if err != nil { if errors.Is(err, backend.ErrRecordNotFound) { @@ -701,7 +797,7 @@ func (p *piPlugin) cmdCommentVote(payload string) (string, error) { } // Verify vote status - vs, err := p.voteSummary(cv.Token) + vs, err := p.voteSummary(v.Token) if err != nil { return "", fmt.Errorf("voteSummary: %v", err) } @@ -717,205 +813,8 @@ func (p *piPlugin) cmdCommentVote(payload string) (string, error) { } } - // Setup plugin command - v := comments.Vote{ - UserID: cv.UserID, - State: comments.StateT(cv.State), - Token: cv.Token, - CommentID: cv.CommentID, - Vote: comments.VoteT(cv.Vote), - PublicKey: cv.PublicKey, - Signature: cv.Signature, - } - b, err := comments.EncodeVote(v) - if err != nil { - return "", err - } - // Send plugin command - r, err := p.backend.Plugin(comments.ID, comments.CmdVote, "", string(b)) - if err != nil { - return "", err - } - - // Prepare reply - vr, err := comments.DecodeVoteReply([]byte(r)) - if err != nil { - return "", err - } - cvr := pi.CommentVoteReply{ - Downvotes: vr.Downvotes, - Upvotes: vr.Upvotes, - Timestamp: vr.Timestamp, - Receipt: vr.Receipt, - } - reply, err := pi.EncodeCommentVoteReply(cvr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *piPlugin) cmdProposalInv(payload string) (string, error) { - // Decode payload - inv, err := pi.DecodeProposalInv([]byte(payload)) - if err != nil { - return "", err - } - - // Get full record inventory - ibs, err := p.backend.InventoryByStatus() - if err != nil { - return "", err - } - - // Apply user ID filtering criteria - if inv.UserID != "" { - // Lookup the proposal tokens that have been submitted by the - // specified user. - ud, err := p.userData(inv.UserID) - if err != nil { - return "", fmt.Errorf("userData %v: %v", inv.UserID, err) - } - userTokens := make(map[string]struct{}, len(ud.Tokens)) - for _, v := range ud.Tokens { - userTokens[v] = struct{}{} - } - - // Compile a list of unvetted tokens categorized by MDStatusT - // that were submitted by the user. - filtered := make(map[backend.MDStatusT][]string, len(ibs.Unvetted)) - for status, tokens := range ibs.Unvetted { - for _, v := range tokens { - _, ok := userTokens[v] - if !ok { - // Proposal was not submitted by the user - continue - } - - // Proposal was submitted by the user - ftokens, ok := filtered[status] - if !ok { - ftokens = make([]string, 0, len(tokens)) - } - filtered[status] = append(ftokens, v) - } - } - - // Update unvetted inventory with filtered tokens - ibs.Unvetted = filtered - - // Compile a list of vetted tokens categorized by MDStatusT that - // were submitted by the user. - filtered = make(map[backend.MDStatusT][]string, len(ibs.Vetted)) - for status, tokens := range ibs.Vetted { - for _, v := range tokens { - _, ok := userTokens[v] - if !ok { - // Proposal was not submitted by the user - continue - } - - // Proposal was submitted by the user - ftokens, ok := filtered[status] - if !ok { - ftokens = make([]string, 0, len(tokens)) - } - filtered[status] = append(ftokens, v) - } - } - - // Update vetted inventory with filtered tokens - ibs.Vetted = filtered - } - - // Convert MDStatus keys to human readable proposal statuses - unvetted := make(map[string][]string, len(ibs.Unvetted)) - vetted := make(map[string][]string, len(ibs.Vetted)) - for k, v := range ibs.Unvetted { - s := pi.PropStatuses[convertPropStatusFromMDStatus(k)] - unvetted[s] = v - } - for k, v := range ibs.Vetted { - s := pi.PropStatuses[convertPropStatusFromMDStatus(k)] - vetted[s] = v - } - - // Prepare reply - pir := pi.ProposalInvReply{ - Unvetted: unvetted, - Vetted: vetted, - } - reply, err := pi.EncodeProposalInvReply(pir) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *piPlugin) cmdVoteInventory(payload string) (string, error) { - // Payload is empty. Nothing to decode. - - // Get ticketvote inventory - r, err := p.backend.Plugin(ticketvote.ID, ticketvote.CmdInventory, "", "") - if err != nil { - return "", fmt.Errorf("ticketvote inventory: %v", err) - } - ir, err := ticketvote.DecodeInventoryReply([]byte(r)) - if err != nil { - return "", err - } - - // Get vote summaries for all finished proposal votes - s := ticketvote.Summaries{ - Tokens: ir.Finished, - } - b, err := ticketvote.EncodeSummaries(s) - if err != nil { - return "", err - } - r, err = p.backend.Plugin(ticketvote.ID, ticketvote.CmdSummaries, - "", string(b)) - if err != nil { - return "", fmt.Errorf("ticketvote summaries: %v", err) - } - sr, err := ticketvote.DecodeSummariesReply([]byte(r)) - if err != nil { - return "", err - } - if len(sr.Summaries) != len(ir.Finished) { - return "", fmt.Errorf("unexpected number of summaries: got %v, want %v", - len(sr.Summaries), len(ir.Finished)) - } - - // Categorize votes - approved := make([]string, 0, len(sr.Summaries)) - rejected := make([]string, 0, len(sr.Summaries)) - for token, v := range sr.Summaries { - if v.Approved { - approved = append(approved, token) - } else { - rejected = append(rejected, token) - } - } - - // Prepare reply - vir := pi.VoteInventoryReply{ - Unauthorized: ir.Unauthorized, - Authorized: ir.Authorized, - Started: ir.Started, - Approved: approved, - Rejected: rejected, - BestBlock: ir.BestBlock, - } - reply, err := pi.EncodeVoteInventoryReply(vir) - if err != nil { - return "", err - } - - return string(reply), nil + return p.backend.Plugin(comments.ID, comments.CmdVote, "", payload) } func (p *piPlugin) ticketVoteStart(payload string) (string, error) { @@ -1068,28 +967,42 @@ func (p *piPlugin) ticketVoteStart(payload string) (string, error) { return p.backend.Plugin(ticketvote.ID, ticketvote.CmdStart, "", payload) } -func (p *piPlugin) cmdPassThrough(payload string) (string, error) { - // Decode payload - pt, err := pi.DecodePassThrough([]byte(payload)) - if err != nil { - return "", err - } - - // Get pass through function - var fn func(string) (string, error) +func (p *piPlugin) passThrough(pt pi.PassThrough) (string, error) { switch pt.PluginID { + case comments.ID: + switch pt.PluginCmd { + case comments.CmdNew: + return p.commentNew(pt.Payload) + case comments.CmdDel: + return p.commentDel(pt.Payload) + case comments.CmdVote: + return p.commentVote(pt.Payload) + default: + return "", fmt.Errorf("invalid %v plugin cmd '%v'", + pt.PluginID, pt.PluginCmd) + } case ticketvote.ID: switch pt.PluginCmd { case ticketvote.CmdStart: - fn = p.ticketVoteStart + return p.ticketVoteStart(pt.Payload) + default: + return "", fmt.Errorf("invalid %v plugin cmd '%v'", + pt.PluginID, pt.PluginCmd) } default: - return "", fmt.Errorf("invalid passthrough plugin command %v %v", - pt.PluginID, pt.PluginCmd) + return "", fmt.Errorf("invalid plugin id '%v'", pt.PluginID) + } +} + +func (p *piPlugin) cmdPassThrough(payload string) (string, error) { + // Decode payload + pt, err := pi.DecodePassThrough([]byte(payload)) + if err != nil { + return "", err } - // Execute pass through - r, err := fn(pt.Payload) + // Execute command + r, err := p.passThrough(*pt) if err != nil { return "", err } @@ -1431,12 +1344,6 @@ func (p *piPlugin) cmd(cmd, payload string) (string, error) { switch cmd { case pi.CmdProposals: return p.cmdProposals(payload) - case pi.CmdCommentNew: - return p.cmdCommentNew(payload) - case pi.CmdCommentCensor: - return p.cmdCommentCensor(payload) - case pi.CmdCommentVote: - return p.cmdCommentVote(payload) case pi.CmdProposalInv: return p.cmdProposalInv(payload) case pi.CmdVoteInventory: diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index 83957dfdd..f956dd6fb 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -24,11 +24,6 @@ const ( ID = "pi" Version = "1" - // TODO these should use the PassThrough command - CmdCommentNew = "commentnew" // Create a new comment - CmdCommentCensor = "commentcensor" // Censor a comment - CmdCommentVote = "commentvote" // Vote on a comment - // Plugin commands CmdPassThrough = "passthrough" // Plugin command pass through CmdProposalInv = "proposalinv" // Get inventory by proposal status @@ -60,11 +55,6 @@ const ( PropStatusCensored PropStatusT = 3 // Prop has been censored PropStatusAbandoned PropStatusT = 4 // Prop has been abandoned - // Comment vote types - VoteInvalid CommentVoteT = 0 - VoteDownvote CommentVoteT = -1 - VoteUpvote CommentVoteT = 1 - // User error status codes // TODO number error codes and add human readable error messages ErrorStatusInvalid ErrorStatusT = 0 @@ -293,7 +283,6 @@ func DecodeProposals(payload []byte) (*Proposals, error) { } // ProposalPluginData contains all the plugin data for a proposal. -// TODO get rid of this command type ProposalPluginData struct { Comments uint64 `json:"comments"` // Number of comments LinkedFrom []string `json:"linkedfrom"` // Linked from list diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 3739d135b..4d3bfde70 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -8,7 +8,6 @@ import ( "fmt" ) -type ErrorStatusT int type CommentVoteT int type VoteStatusT int type VoteAuthActionT string @@ -16,26 +15,25 @@ type VoteT int type VoteErrorT int // TODO verify that all batched request have a page size limit -// TODO add fetching proposals by user ID. Will probably need to add user ID -// to the general metadata stream. -// TODO add a comments/count endpoint and take the comments count off of the -// proposal record -// TODO linkedfrom should really be pulled out of a proposal record an added -// as a separate enpoint as well. -// TODO create a comments and vote api for those plugin commands that are not -// pi specific -// TODO the plugin policies should be returned in a route +// TODO comments count and linked from should be pulled out of the proposal +// record struct. These should be separate endpoints: +// /comments/count +// /proposal/linkedfrom +// TODO routes that map directly to plugin commands (comment and vote routes) +// should be added to their own API package so that they can be used by +// multiple politeia applications (pi, cms, forum). // TODO make RouteVoteResults a batched route but that only currently allows -// for 1 result to be returned so that we have the option to change this is +// for 1 result to be returned so that we have the option to change this if // we want to. -// TODO each API needs a version and policy route +// TODO pi needs a Version route and a Policy route. The policies should be +// defined in the plugin packages and returned in the policy route. +// TODO module these API packages const ( APIVersion = 1 // APIRoute is prefixed onto all routes defined in this package. - // TODO should the api route be "/pi/v1"? - APIRoute = "/v1" + APIRoute = "/pi/v1" // Proposal routes RouteProposalNew = "/proposal/new" @@ -107,7 +105,12 @@ const ( // proposal should be rejected. Proposal votes are required to use // this vote option ID. VoteOptionIDReject = "no" +) +// ErrorStatusT represents a user error status code. +type ErrorStatusT int + +const ( // Cast vote errors // TODO these need human readable equivalents VoteErrorInvalid VoteErrorT = 0 @@ -265,7 +268,7 @@ func (e ServerErrorReply) Error() string { return fmt.Sprintf("server error: %v", e.ErrorCode) } -// PropStateT represents a proposal state type. A proposal state can be either +// PropStateT represents a proposal state. A proposal state can be either // unvetted or vetted. The PropStatusT type further breaks down these two // states into more granular statuses. type PropStateT int @@ -283,7 +286,7 @@ const ( PropStateVetted PropStateT = 2 ) -// PropStatusT represents a proposal status type. +// PropStatusT represents a proposal status. type PropStatusT int const ( diff --git a/politeiawww/pi.go b/politeiawww/pi.go index d9bd4464f..9dbffd91d 100644 --- a/politeiawww/pi.go +++ b/politeiawww/pi.go @@ -11,76 +11,40 @@ import ( piplugin "github.com/decred/politeia/politeiad/plugins/pi" ) -// piProposals returns the pi plugin data for the provided proposals. -func (p *politeiawww) piProposals(ctx context.Context, ps piplugin.Proposals) (*piplugin.ProposalsReply, error) { - b, err := piplugin.EncodeProposals(ps) - if err != nil { - return nil, err - } - r, err := p.pluginCommand(ctx, piplugin.ID, - piplugin.CmdProposals, string(b)) - if err != nil { - return nil, err - } - pr, err := piplugin.DecodeProposalsReply([]byte(r)) - if err != nil { - return nil, err - } - return pr, nil -} - -// piCommentNew uses the pi plugin to submit a new comment. -func (p *politeiawww) piCommentNew(ctx context.Context, cn piplugin.CommentNew) (*piplugin.CommentNewReply, error) { - b, err := piplugin.EncodeCommentNew(cn) - if err != nil { - return nil, err - } - r, err := p.pluginCommand(ctx, piplugin.ID, - piplugin.CmdCommentNew, string(b)) - if err != nil { - return nil, err - } - cnr, err := piplugin.DecodeCommentNewReply([]byte(r)) - if err != nil { - return nil, err - } - return cnr, nil -} - -// piCommentVote uses the pi plugin to vote on a comment. -func (p *politeiawww) piCommentVote(ctx context.Context, cvp piplugin.CommentVote) (*piplugin.CommentVoteReply, error) { - b, err := piplugin.EncodeCommentVote(cvp) +// piPassThrough executes the pi plugin PassThrough command. +func (p *politeiawww) piPassThrough(ctx context.Context, pt pi.PassThrough) (*pi.PassThroughReply, error) { + b, err := piplugin.EncodePassThrough(pt) if err != nil { return nil, err } r, err := p.pluginCommand(ctx, piplugin.ID, - piplugin.CmdCommentVote, string(b)) + piplugin.CmdPassThrough, string(b)) if err != nil { return nil, err } - cvr, err := piplugin.DecodeCommentVoteReply([]byte(r)) + ptr, err := piplugin.DecodePassThroughReply(([]byte(r))) if err != nil { return nil, err } - return cvr, nil + return ptr, nil } -// piCommentCensor uses the pi plugin to censor a proposal comment. -func (p *politeiawww) piCommentCensor(ctx context.Context, cc piplugin.CommentCensor) (*piplugin.CommentCensorReply, error) { - b, err := piplugin.EncodeCommentCensor(cc) +// piProposals returns the pi plugin data for the provided proposals. +func (p *politeiawww) piProposals(ctx context.Context, ps piplugin.Proposals) (*piplugin.ProposalsReply, error) { + b, err := piplugin.EncodeProposals(ps) if err != nil { return nil, err } r, err := p.pluginCommand(ctx, piplugin.ID, - piplugin.CmdCommentCensor, string(b)) + piplugin.CmdProposals, string(b)) if err != nil { return nil, err } - ccr, err := piplugin.DecodeCommentCensorReply(([]byte(r))) + pr, err := piplugin.DecodeProposalsReply([]byte(r)) if err != nil { return nil, err } - return ccr, nil + return pr, nil } // proposalInv returns the pi plugin proposal inventory. @@ -113,21 +77,3 @@ func (p *politeiawww) piVoteInventory(ctx context.Context) (*piplugin.VoteInvent } return vir, nil } - -// piPassThrough executes the pi plugin PassThrough command. -func (p *politeiawww) piPassThrough(ctx context.Context, pt pi.PassThrough) (*pi.PassThroughReply, error) { - b, err := piplugin.EncodePassThrough(pt) - if err != nil { - return nil, err - } - r, err := p.pluginCommand(ctx, piplugin.ID, - piplugin.CmdPassThrough, string(b)) - if err != nil { - return nil, err - } - ptr, err := piplugin.DecodePassThroughReply(([]byte(r))) - if err != nil { - return nil, err - } - return ptr, nil -} diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index acbb220bd..657e230f7 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -362,6 +362,26 @@ func convertPropStateFromComments(s comments.StateT) pi.PropStateT { return pi.PropStateInvalid } +func convertCommentStateFromPi(s pi.PropStateT) comments.StateT { + switch s { + case pi.PropStateUnvetted: + return comments.StateUnvetted + case pi.PropStateVetted: + return comments.StateVetted + } + return comments.StateInvalid +} + +func convertCommentVoteFromPi(cv pi.CommentVoteT) comments.VoteT { + switch cv { + case pi.CommentVoteDownvote: + return comments.VoteUpvote + case pi.CommentVoteUpvote: + return comments.VoteDownvote + } + return comments.VoteInvalid +} + func convertCommentFromPlugin(c comments.Comment) pi.Comment { return pi.Comment{ UserID: c.UserID, @@ -410,19 +430,6 @@ func convertCommentVoteDetailsFromPlugin(cv []comments.CommentVote) []pi.Comment return c } -func convertCommentVoteFromPi(v pi.CommentVoteT) piplugin.CommentVoteT { - switch v { - case pi.CommentVoteInvalid: - return piplugin.VoteInvalid - case pi.CommentVoteDownvote: - return piplugin.VoteDownvote - case pi.CommentVoteUpvote: - return piplugin.VoteUpvote - default: - return piplugin.VoteInvalid - } -} - func convertVoteAuthActionFromPi(a pi.VoteAuthActionT) ticketvote.AuthActionT { switch a { case pi.VoteAuthActionAuthorize: @@ -1642,22 +1649,35 @@ func (p *politeiawww) processCommentNew(ctx context.Context, cn pi.CommentNew, u } // Send plugin command - pcn := piplugin.CommentNew{ + n := comments.New{ UserID: usr.ID.String(), - State: convertPropStateFromPi(cn.State), + State: convertCommentStateFromPi(cn.State), Token: cn.Token, ParentID: cn.ParentID, Comment: cn.Comment, PublicKey: cn.PublicKey, Signature: cn.Signature, } - cnr, err := p.piCommentNew(ctx, pcn) + b, err := comments.EncodeNew(n) + if err != nil { + return nil, err + } + pt := piplugin.PassThrough{ + PluginID: comments.ID, + PluginCmd: comments.CmdNew, + Payload: string(b), + } + ptr, err := p.piPassThrough(ctx, pt) + if err != nil { + return nil, err + } + nr, err := comments.DecodeNewReply([]byte(ptr.Payload)) if err != nil { return nil, err } // Prepare reply - c := convertCommentFromPlugin(cnr.Comment) + c := convertCommentFromPlugin(nr.Comment) c = commentFillInUser(c, usr) // Emit event @@ -1702,25 +1722,38 @@ func (p *politeiawww) processCommentVote(ctx context.Context, cv pi.CommentVote, } // Send plugin command - pcv := piplugin.CommentVote{ + v := comments.Vote{ UserID: usr.ID.String(), - State: convertPropStateFromPi(cv.State), + State: convertCommentStateFromPi(cv.State), Token: cv.Token, CommentID: cv.CommentID, Vote: convertCommentVoteFromPi(cv.Vote), PublicKey: cv.PublicKey, Signature: cv.Signature, } - cvr, err := p.piCommentVote(ctx, pcv) + b, err := comments.EncodeVote(v) + if err != nil { + return nil, err + } + pt := piplugin.PassThrough{ + PluginID: comments.ID, + PluginCmd: comments.CmdVote, + Payload: string(b), + } + ptr, err := p.piPassThrough(ctx, pt) + if err != nil { + return nil, err + } + vr, err := comments.DecodeVoteReply([]byte(ptr.Payload)) if err != nil { return nil, err } return &pi.CommentVoteReply{ - Downvotes: cvr.Downvotes, - Upvotes: cvr.Upvotes, - Timestamp: cvr.Timestamp, - Receipt: cvr.Receipt, + Downvotes: vr.Downvotes, + Upvotes: vr.Upvotes, + Timestamp: vr.Timestamp, + Receipt: vr.Receipt, }, nil } @@ -1741,21 +1774,34 @@ func (p *politeiawww) processCommentCensor(ctx context.Context, cc pi.CommentCen } // Send plugin command - pcc := piplugin.CommentCensor{ - State: convertPropStateFromPi(cc.State), + d := comments.Del{ + State: convertCommentStateFromPi(cc.State), Token: cc.Token, CommentID: cc.CommentID, Reason: cc.Reason, PublicKey: cc.PublicKey, Signature: cc.Signature, } - ccr, err := p.piCommentCensor(ctx, pcc) + b, err := comments.EncodeDel(d) + if err != nil { + return nil, err + } + pt := piplugin.PassThrough{ + PluginID: comments.ID, + PluginCmd: comments.CmdDel, + Payload: string(b), + } + ptr, err := p.piPassThrough(ctx, pt) + if err != nil { + return nil, err + } + dr, err := comments.DecodeDelReply([]byte(ptr.Payload)) if err != nil { return nil, err } // Prepare reply - c := convertCommentFromPlugin(ccr.Comment) + c := convertCommentFromPlugin(dr.Comment) c = commentFillInUser(c, usr) return &pi.CommentCensorReply{ From 902982a6e71df1a2f145f5ba50ff8c02a08ec3b7 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 29 Dec 2020 17:54:52 -0600 Subject: [PATCH 196/449] multi: More cleanup. --- politeiad/plugins/comments/comments.go | 1 + politeiawww/api/pi/v1/v1.go | 2 -- politeiawww/cmd/piwww/castballot.go | 4 +++- politeiawww/cmd/piwww/commentcensor.go | 4 +++- politeiawww/cmd/piwww/commentnew.go | 4 +++- politeiawww/cmd/piwww/comments.go | 4 +++- politeiawww/cmd/piwww/commentvote.go | 4 +++- politeiawww/cmd/piwww/commentvotes.go | 4 +++- politeiawww/cmd/piwww/help.go | 4 +++- politeiawww/cmd/piwww/policy.go | 4 +++- politeiawww/cmd/piwww/proposaledit.go | 4 +++- politeiawww/cmd/piwww/proposalnew.go | 4 +++- politeiawww/cmd/piwww/proposals.go | 4 +++- politeiawww/cmd/piwww/proposalstatusset.go | 4 +++- politeiawww/cmd/piwww/sendfaucettx.go | 4 +++- politeiawww/cmd/piwww/subscribe.go | 4 +++- politeiawww/cmd/piwww/testrun.go | 4 +++- politeiawww/cmd/piwww/userdetails.go | 4 +++- politeiawww/cmd/piwww/useredit.go | 4 +++- politeiawww/cmd/piwww/useremailverify.go | 4 +++- politeiawww/cmd/piwww/usernew.go | 4 +++- politeiawww/cmd/piwww/userpaymentsrescan.go | 4 +++- politeiawww/cmd/piwww/userproposalcredits.go | 4 +++- politeiawww/cmd/piwww/userproposalpaywall.go | 4 +++- politeiawww/cmd/piwww/userproposalpaywalltx.go | 4 +++- politeiawww/cmd/piwww/userregistrationpayment.go | 4 +++- politeiawww/cmd/piwww/userverificationresend.go | 8 +++++--- politeiawww/cmd/piwww/voteauthorize.go | 4 +++- politeiawww/cmd/piwww/voteinventory.go | 4 +++- politeiawww/cmd/piwww/voteresults.go | 4 +++- politeiawww/cmd/piwww/votes.go | 4 +++- politeiawww/cmd/piwww/votestart.go | 4 +++- politeiawww/cmd/piwww/votestartrunoff.go | 4 +++- politeiawww/cmd/piwww/votesummaries.go | 4 +++- 34 files changed, 99 insertions(+), 36 deletions(-) diff --git a/politeiad/plugins/comments/comments.go b/politeiad/plugins/comments/comments.go index dbaa9cc6d..70728978d 100644 --- a/politeiad/plugins/comments/comments.go +++ b/politeiad/plugins/comments/comments.go @@ -42,6 +42,7 @@ const ( VoteDownvote VoteT = -1 VoteUpvote VoteT = 1 + // TODO make these default settings, not policies // PolicyCommentLengthMax is the maximum number of characters // accepted for comments. PolicyCommentLengthMax = 8000 diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 4d3bfde70..38b0bcbff 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -30,8 +30,6 @@ type VoteErrorT int // TODO module these API packages const ( - APIVersion = 1 - // APIRoute is prefixed onto all routes defined in this package. APIRoute = "/pi/v1" diff --git a/politeiawww/cmd/piwww/castballot.go b/politeiawww/cmd/piwww/castballot.go index 3be690e8e..c49bed47b 100644 --- a/politeiawww/cmd/piwww/castballot.go +++ b/politeiawww/cmd/piwww/castballot.go @@ -28,7 +28,9 @@ type castBallotCmd struct { Password string `long:"password" optional:"true"` } -// Execute executes the vote ballot command. +// Execute executes the castBallotCmd command. +// +// This function satisfies the go-flags Commander interface. func (c *castBallotCmd) Execute(args []string) error { token := c.Args.Token voteID := c.Args.VoteID diff --git a/politeiawww/cmd/piwww/commentcensor.go b/politeiawww/cmd/piwww/commentcensor.go index 60bc7381b..3db0cd45e 100644 --- a/politeiawww/cmd/piwww/commentcensor.go +++ b/politeiawww/cmd/piwww/commentcensor.go @@ -26,7 +26,9 @@ type commentCensorCmd struct { Unvetted bool `long:"unvetted" optional:"true"` } -// Execute executes the censor comment command. +// Execute executes the commentCensorCmd command. +// +// This function satisfies the go-flags Commander interface. func (cmd *commentCensorCmd) Execute(args []string) error { // Unpack args token := cmd.Args.Token diff --git a/politeiawww/cmd/piwww/commentnew.go b/politeiawww/cmd/piwww/commentnew.go index 058ca96b6..a04b024b3 100644 --- a/politeiawww/cmd/piwww/commentnew.go +++ b/politeiawww/cmd/piwww/commentnew.go @@ -26,7 +26,9 @@ type commentNewCmd struct { Unvetted bool `long:"unvetted" optional:"true"` } -// Execute executes the new comment command. +// Execute executes the commentNewCmd command. +// +// This function satisfies the go-flags Commander interface. func (c *commentNewCmd) Execute(args []string) error { // Unpack args token := c.Args.Token diff --git a/politeiawww/cmd/piwww/comments.go b/politeiawww/cmd/piwww/comments.go index 89725238f..57e61589f 100644 --- a/politeiawww/cmd/piwww/comments.go +++ b/politeiawww/cmd/piwww/comments.go @@ -19,7 +19,9 @@ type commentsCmd struct { Unvetted bool `long:"unvetted" optional:"true"` } -// Execute executes the proposal comments command. +// Execute executes the commentsCmd command. +// +// This function satisfies the go-flags Commander interface. func (cmd *commentsCmd) Execute(args []string) error { token := cmd.Args.Token diff --git a/politeiawww/cmd/piwww/commentvote.go b/politeiawww/cmd/piwww/commentvote.go index 70bf0361c..9b57417f1 100644 --- a/politeiawww/cmd/piwww/commentvote.go +++ b/politeiawww/cmd/piwww/commentvote.go @@ -24,7 +24,9 @@ type commentVoteCmd struct { } `positional-args:"true" required:"true"` } -// Execute executes the like comment command. +// Execute executes the commentVoteCmd command. +// +// This function satisfies the go-flags Commander interface. func (c *commentVoteCmd) Execute(args []string) error { votes := map[string]pi.CommentVoteT{ "upvote": pi.CommentVoteUpvote, diff --git a/politeiawww/cmd/piwww/commentvotes.go b/politeiawww/cmd/piwww/commentvotes.go index 62c9c90cb..09a6f1a9e 100644 --- a/politeiawww/cmd/piwww/commentvotes.go +++ b/politeiawww/cmd/piwww/commentvotes.go @@ -21,7 +21,9 @@ type commentVotesCmd struct { Me bool `long:"me" optional:"true"` } -// Execute executes the user comment likes command. +// Execute executes the commentVotesCmd command. +// +// This function satisfies the go-flags Commander interface. func (c *commentVotesCmd) Execute(args []string) error { token := c.Args.Token userID := c.Args.UserID diff --git a/politeiawww/cmd/piwww/help.go b/politeiawww/cmd/piwww/help.go index 10568572c..a2fa62f16 100644 --- a/politeiawww/cmd/piwww/help.go +++ b/politeiawww/cmd/piwww/help.go @@ -17,7 +17,9 @@ type helpCmd struct { } `positional-args:"true"` } -// Execute executes the help command. +// Execute executes the helpCmd command. +// +// This function satisfies the go-flags Commander interface. func (cmd *helpCmd) Execute(args []string) error { if cmd.Args.Topic == "" { return fmt.Errorf("Specify a command to print a detailed help " + diff --git a/politeiawww/cmd/piwww/policy.go b/politeiawww/cmd/piwww/policy.go index be8e5380c..b60f9ce9f 100644 --- a/politeiawww/cmd/piwww/policy.go +++ b/politeiawww/cmd/piwww/policy.go @@ -9,7 +9,9 @@ import "github.com/decred/politeia/politeiawww/cmd/shared" // policyCmd gets the server policy information. type policyCmd struct{} -// Execute executes the policy command. +// Execute executes the policyCmd command. +// +// This function satisfies the go-flags Commander interface. func (cmd *policyCmd) Execute(args []string) error { pr, err := client.Policy() if err != nil { diff --git a/politeiawww/cmd/piwww/proposaledit.go b/politeiawww/cmd/piwww/proposaledit.go index f4c0d4296..e47c025b2 100644 --- a/politeiawww/cmd/piwww/proposaledit.go +++ b/politeiawww/cmd/piwww/proposaledit.go @@ -50,7 +50,9 @@ type proposalEditCmd struct { UseMD bool `long:"usemd" optional:"true"` } -// Execute executes the proposal edit command. +// Execute executes the proposalEditCmd command. +// +// This function satisfies the go-flags Commander interface. func (cmd *proposalEditCmd) Execute(args []string) error { token := cmd.Args.Token indexFile := cmd.Args.IndexFile diff --git a/politeiawww/cmd/piwww/proposalnew.go b/politeiawww/cmd/piwww/proposalnew.go index 9551ea5fa..1da21b819 100644 --- a/politeiawww/cmd/piwww/proposalnew.go +++ b/politeiawww/cmd/piwww/proposalnew.go @@ -43,7 +43,9 @@ type proposalNewCmd struct { RFP bool `long:"rfp" optional:"true"` } -// Execute executes the new proposal command. +// Execute executes the proposalNewCmd command. +// +// This function satisfies the go-flags Commander interface. func (cmd *proposalNewCmd) Execute(args []string) error { indexFile := cmd.Args.IndexFile attachments := cmd.Args.Attachments diff --git a/politeiawww/cmd/piwww/proposals.go b/politeiawww/cmd/piwww/proposals.go index 97e0393e5..9ddbae987 100644 --- a/politeiawww/cmd/piwww/proposals.go +++ b/politeiawww/cmd/piwww/proposals.go @@ -26,7 +26,9 @@ type proposalsCmd struct { IncludeFiles bool `long:"includefiles" optional:"true"` } -// Execute executes the proposals command. +// Execute executes the proposalsCmd command. +// +// This function satisfies the go-flags Commander interface. func (c *proposalsCmd) Execute(args []string) error { proposals := c.Args.Proposals diff --git a/politeiawww/cmd/piwww/proposalstatusset.go b/politeiawww/cmd/piwww/proposalstatusset.go index 5e7aca3ff..4dc4470bc 100644 --- a/politeiawww/cmd/piwww/proposalstatusset.go +++ b/politeiawww/cmd/piwww/proposalstatusset.go @@ -25,7 +25,9 @@ type proposalSetStatusCmd struct { Unvetted bool `long:"unvetted" optional:"true"` } -// Execute executes the proposal status set command. +// Execute executes the proposalSetStatusCmd command. +// +// This function satisfies the go-flags Commander interface. func (cmd *proposalSetStatusCmd) Execute(args []string) error { propStatus := map[string]pi.PropStatusT{ "public": pi.PropStatusPublic, diff --git a/politeiawww/cmd/piwww/sendfaucettx.go b/politeiawww/cmd/piwww/sendfaucettx.go index 0f83119d4..27e3b8f0d 100644 --- a/politeiawww/cmd/piwww/sendfaucettx.go +++ b/politeiawww/cmd/piwww/sendfaucettx.go @@ -21,7 +21,9 @@ type sendFaucetTxCmd struct { } `positional-args:"true"` } -// Execute executes the send faucet tx command. +// Execute executes the sendFaucetTxCmd command. +// +// This function satisfies the go-flags Commander interface. func (cmd *sendFaucetTxCmd) Execute(args []string) error { address := cmd.Args.Address atoms := cmd.Args.Amount diff --git a/politeiawww/cmd/piwww/subscribe.go b/politeiawww/cmd/piwww/subscribe.go index 176cc8fcd..6e96065e7 100644 --- a/politeiawww/cmd/piwww/subscribe.go +++ b/politeiawww/cmd/piwww/subscribe.go @@ -25,7 +25,9 @@ type subscribeCmd struct { Close bool `long:"close" optional:"true"` // Do not keep connetion alive } -// Execute executes the subscribe command. +// Execute executes the subscribeCmd command. +// +// This function satisfies the go-flags Commander interface. func (cmd *subscribeCmd) Execute(args []string) error { // Parse args route := v1.RouteUnauthenticatedWebSocket diff --git a/politeiawww/cmd/piwww/testrun.go b/politeiawww/cmd/piwww/testrun.go index 976409178..afcbc0807 100644 --- a/politeiawww/cmd/piwww/testrun.go +++ b/politeiawww/cmd/piwww/testrun.go @@ -1406,7 +1406,9 @@ func testCommentRoutes(admin testUser) error { return nil } -// Execute executes the test run command. +// Execute executes the testRunCmd command. +// +// This function satisfies the go-flags Commander interface. func (cmd *testRunCmd) Execute(args []string) error { // Suppress output from cli commands cfg.Silent = true diff --git a/politeiawww/cmd/piwww/userdetails.go b/politeiawww/cmd/piwww/userdetails.go index 93231b014..837a85c68 100644 --- a/politeiawww/cmd/piwww/userdetails.go +++ b/politeiawww/cmd/piwww/userdetails.go @@ -13,7 +13,9 @@ type userDetailsCmd struct { } `positional-args:"true" required:"true"` } -// Execute executes the user details command. +// Execute executes the userDetailsCmd command. +// +// This function satisfies the go-flags Commander interface. func (cmd *userDetailsCmd) Execute(args []string) error { udr, err := client.UserDetails(cmd.Args.UserID) if err != nil { diff --git a/politeiawww/cmd/piwww/useredit.go b/politeiawww/cmd/piwww/useredit.go index 8c04d04bb..760386385 100644 --- a/politeiawww/cmd/piwww/useredit.go +++ b/politeiawww/cmd/piwww/useredit.go @@ -20,7 +20,9 @@ type userEditCmd struct { } `positional-args:"true" required:"true"` } -// Execute executes the edit user command. +// Execute executes the userEditCmd command. +// +// This function satisfies the go-flags Commander interface. func (cmd *userEditCmd) Execute(args []string) error { emailNotifs := map[string]v1.EmailNotificationT{ "userproposalchange": v1.NotificationEmailMyProposalStatusChange, diff --git a/politeiawww/cmd/piwww/useremailverify.go b/politeiawww/cmd/piwww/useremailverify.go index 566714bdc..34b305126 100644 --- a/politeiawww/cmd/piwww/useremailverify.go +++ b/politeiawww/cmd/piwww/useremailverify.go @@ -20,7 +20,9 @@ type userEmailVerifyCmd struct { } `positional-args:"true" required:"true"` } -// Execute executes the verify user email command. +// Execute executes the userEmailVerifyCmd command. +// +// This function satisfies the go-flags Commander interface. func (cmd *userEmailVerifyCmd) Execute(args []string) error { // Load user identity id, err := cfg.LoadIdentity(cmd.Args.Username) diff --git a/politeiawww/cmd/piwww/usernew.go b/politeiawww/cmd/piwww/usernew.go index 24aa361e4..4b95ab7b4 100644 --- a/politeiawww/cmd/piwww/usernew.go +++ b/politeiawww/cmd/piwww/usernew.go @@ -26,7 +26,9 @@ type userNewCmd struct { NoSave bool `long:"nosave" optional:"true"` // Don't save user identity to disk } -// Execute executes the new user command. +// Execute executes the userNewCmd command. +// +// This function satisfies the go-flags Commander interface. func (cmd *userNewCmd) Execute(args []string) error { email := cmd.Args.Email username := cmd.Args.Username diff --git a/politeiawww/cmd/piwww/userpaymentsrescan.go b/politeiawww/cmd/piwww/userpaymentsrescan.go index 3237746e6..f1f072f26 100644 --- a/politeiawww/cmd/piwww/userpaymentsrescan.go +++ b/politeiawww/cmd/piwww/userpaymentsrescan.go @@ -17,7 +17,9 @@ type userPaymentsRescanCmd struct { } `positional-args:"true" required:"true"` } -// Execute executes the rescan user payments command. +// Execute executes the userPaymentsRescanCmd command. +// +// This function satisfies the go-flags Commander interface. func (cmd *userPaymentsRescanCmd) Execute(args []string) error { upr := &v1.UserPaymentsRescan{ UserID: cmd.Args.UserID, diff --git a/politeiawww/cmd/piwww/userproposalcredits.go b/politeiawww/cmd/piwww/userproposalcredits.go index 1d629c4ea..04c26da29 100644 --- a/politeiawww/cmd/piwww/userproposalcredits.go +++ b/politeiawww/cmd/piwww/userproposalcredits.go @@ -9,7 +9,9 @@ import "github.com/decred/politeia/politeiawww/cmd/shared" // userProposalCreditsCmd gets the proposal credits for the logged in user. type userProposalCreditsCmd struct{} -// Execute executes the user proposal credits command. +// Execute executes the userProposalCreditsCmd command. +// +// This function satisfies the go-flags Commander interface. func (cmd *userProposalCreditsCmd) Execute(args []string) error { ppdr, err := client.UserProposalCredits() if err != nil { diff --git a/politeiawww/cmd/piwww/userproposalpaywall.go b/politeiawww/cmd/piwww/userproposalpaywall.go index 49c3229ea..fc20ea3e5 100644 --- a/politeiawww/cmd/piwww/userproposalpaywall.go +++ b/politeiawww/cmd/piwww/userproposalpaywall.go @@ -9,7 +9,9 @@ import "github.com/decred/politeia/politeiawww/cmd/shared" // userProposalPaywallCmd gets paywall info for the logged in user. type userProposalPaywallCmd struct{} -// Execute executes the proposal paywall command. +// Execute executes the userProposalPaywallCmd command. +// +// This function satisfies the go-flags Commander interface. func (cmd *userProposalPaywallCmd) Execute(args []string) error { ppdr, err := client.UserProposalPaywall() if err != nil { diff --git a/politeiawww/cmd/piwww/userproposalpaywalltx.go b/politeiawww/cmd/piwww/userproposalpaywalltx.go index c7ee9c129..d699f7aec 100644 --- a/politeiawww/cmd/piwww/userproposalpaywalltx.go +++ b/politeiawww/cmd/piwww/userproposalpaywalltx.go @@ -10,7 +10,9 @@ import "github.com/decred/politeia/politeiawww/cmd/shared" // if one exists, for the logged in user. type userProposalPaywallTxCmd struct{} -// Execute executes the user proposal paywall tx command. +// Execute executes the userProposalPaywallTxCmd command. +// +// This function satisfies the go-flags Commander interface. func (cmd *userProposalPaywallTxCmd) Execute(args []string) error { pppr, err := client.UserProposalPaywallTx() if err != nil { diff --git a/politeiawww/cmd/piwww/userregistrationpayment.go b/politeiawww/cmd/piwww/userregistrationpayment.go index 361cb6bd2..9288b203b 100644 --- a/politeiawww/cmd/piwww/userregistrationpayment.go +++ b/politeiawww/cmd/piwww/userregistrationpayment.go @@ -10,7 +10,9 @@ import "github.com/decred/politeia/politeiawww/cmd/shared" // registration payment. type userRegistrationPaymentCmd struct{} -// Execute executes the user registration payment command. +// Execute executes the userRegistrationPaymentCmd command. +// +// This function satisfies the go-flags Commander interface. func (cmd *userRegistrationPaymentCmd) Execute(args []string) error { vupr, err := client.UserRegistrationPayment() if err != nil { diff --git a/politeiawww/cmd/piwww/userverificationresend.go b/politeiawww/cmd/piwww/userverificationresend.go index 42f83390f..ed88236be 100644 --- a/politeiawww/cmd/piwww/userverificationresend.go +++ b/politeiawww/cmd/piwww/userverificationresend.go @@ -9,8 +9,8 @@ import ( "github.com/decred/politeia/politeiawww/cmd/shared" ) -// userVerificationResendCmd re-sends the user verification email for an unverified -// user. +// userVerificationResendCmd re-sends the user verification email for an +// unverified user. type userVerificationResendCmd struct { Args struct { Email string `positional-arg-name:"email"` // User email @@ -18,7 +18,9 @@ type userVerificationResendCmd struct { } `positional-args:"true" required:"true"` } -// Execute executes the resend verification command. +// Execute executes the userVerificationResendCmd command. +// +// This function satisfies the go-flags Commander interface. func (cmd *userVerificationResendCmd) Execute(args []string) error { rv := v1.ResendVerification{ Email: cmd.Args.Email, diff --git a/politeiawww/cmd/piwww/voteauthorize.go b/politeiawww/cmd/piwww/voteauthorize.go index 42c357b1a..6a4bed345 100644 --- a/politeiawww/cmd/piwww/voteauthorize.go +++ b/politeiawww/cmd/piwww/voteauthorize.go @@ -23,7 +23,9 @@ type voteAuthorizeCmd struct { } `positional-args:"true"` } -// Execute executes the vote authorize command. +// Execute executes the voteAuthorizeCmd command. +// +// This function satisfies the go-flags Commander interface. func (cmd *voteAuthorizeCmd) Execute(args []string) error { token := cmd.Args.Token diff --git a/politeiawww/cmd/piwww/voteinventory.go b/politeiawww/cmd/piwww/voteinventory.go index bb4254c33..9da53720b 100644 --- a/politeiawww/cmd/piwww/voteinventory.go +++ b/politeiawww/cmd/piwww/voteinventory.go @@ -13,7 +13,9 @@ import ( // non-abandoned proposals in inventory categorized by their vote status. type voteInventoryCmd struct{} -// Execute executes the vote inventory command. +// Execute executes the voteInventoryCmd command. +// +// This function satisfies the go-flags Commander interface. func (cmd *voteInventoryCmd) Execute(args []string) error { // Setup request vi := pi.VoteInventory{} diff --git a/politeiawww/cmd/piwww/voteresults.go b/politeiawww/cmd/piwww/voteresults.go index a81d63d4c..d3ae0fa68 100644 --- a/politeiawww/cmd/piwww/voteresults.go +++ b/politeiawww/cmd/piwww/voteresults.go @@ -16,7 +16,9 @@ type voteResultsCmd struct { } `positional-args:"true" required:"true"` } -// Execute executes the vote results command. +// Execute executes the voteResultsCmd command. +// +// This function satisfies the go-flags Commander interface. func (cmd *voteResultsCmd) Execute(args []string) error { // Setup request vr := pi.VoteResults{ diff --git a/politeiawww/cmd/piwww/votes.go b/politeiawww/cmd/piwww/votes.go index 144ffccce..221c8eb4e 100644 --- a/politeiawww/cmd/piwww/votes.go +++ b/politeiawww/cmd/piwww/votes.go @@ -16,7 +16,9 @@ type votesCmd struct { } `positional-args:"true" required:"true"` } -// Execute executes the votes command. +// Execute executes the votesCmd command. +// +// This function satisfies the go-flags Commander interface. func (c *votesCmd) Execute(args []string) error { // Setup request v := pi.Votes{ diff --git a/politeiawww/cmd/piwww/votestart.go b/politeiawww/cmd/piwww/votestart.go index 570a0a4a7..d824ae30f 100644 --- a/politeiawww/cmd/piwww/votestart.go +++ b/politeiawww/cmd/piwww/votestart.go @@ -25,7 +25,9 @@ type voteStartCmd struct { } `positional-args:"true"` } -// Execute executes the vote start command. +// Execute executes the voteStartCmd command. +// +// This function satisfies the go-flags Commander interface. func (cmd *voteStartCmd) Execute(args []string) error { token := cmd.Args.Token diff --git a/politeiawww/cmd/piwww/votestartrunoff.go b/politeiawww/cmd/piwww/votestartrunoff.go index 98db2b468..4ba106cb6 100644 --- a/politeiawww/cmd/piwww/votestartrunoff.go +++ b/politeiawww/cmd/piwww/votestartrunoff.go @@ -31,7 +31,9 @@ type voteStartRunoffCmd struct { } `positional-args:"true"` } -// Execute executes the StartVoteRunoff command. +// Execute executes the voteStartRunoffCmd command. +// +// This function satisfies the go-flags Commander interface. func (cmd *voteStartRunoffCmd) Execute(args []string) error { // Check for user identity if cfg.Identity == nil { diff --git a/politeiawww/cmd/piwww/votesummaries.go b/politeiawww/cmd/piwww/votesummaries.go index 7dc62faa2..88b654d0a 100644 --- a/politeiawww/cmd/piwww/votesummaries.go +++ b/politeiawww/cmd/piwww/votesummaries.go @@ -16,7 +16,9 @@ type voteSummariesCmd struct { } `positional-args:"true" required:"true"` } -// Execute executes the vote summaries command. +// Execute executes the voteSummariesCmd command. +// +// This function satisfies the go-flags Commander interface. func (cmd *voteSummariesCmd) Execute(args []string) error { // Setup request vs := pi.VoteSummaries{ From 6fcb910f39e1cc644def85d75a320cf53dfbf092 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 30 Dec 2020 12:47:02 -0600 Subject: [PATCH 197/449] tlogbe: Fix post plugin hook error handling. --- politeiad/backend/tlogbe/tlogbe.go | 73 ++++++++++++------------------ 1 file changed, 29 insertions(+), 44 deletions(-) diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index c98f17795..1f549f0ff 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -741,7 +741,7 @@ func (t *tlogBackend) New(metadata []backend.MetadataStream, files []backend.Fil if err != nil { return nil, err } - err = t.pluginHook(hookNewRecordPre, string(b)) + err = t.pluginHookPre(hookNewRecordPre, string(b)) if err != nil { return nil, err } @@ -794,11 +794,7 @@ func (t *tlogBackend) New(metadata []backend.MetadataStream, files []backend.Fil if err != nil { return nil, err } - err = t.pluginHook(hookNewRecordPost, string(b)) - if err != nil { - e := fmt.Sprintf("New %x: pluginHook newRecordPost: %v", token, err) - panic(e) - } + t.pluginHookPost(hookNewRecordPost, string(b)) // Update the inventory cache t.inventoryAdd(stateUnvetted, token, backend.MDStatusUnvetted) @@ -879,7 +875,7 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ if err != nil { return nil, err } - err = t.pluginHook(hookEditRecordPre, string(b)) + err = t.pluginHookPre(hookEditRecordPre, string(b)) if err != nil { return nil, err } @@ -897,7 +893,7 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ } // Call post plugin hooks - err = t.pluginHook(hookEditRecordPost, string(b)) + t.pluginHookPost(hookEditRecordPost, string(b)) if err != nil { e := fmt.Sprintf("UpdateUnvettedRecord %x: pluginHook editRecordPost: %v", token, err) @@ -989,7 +985,7 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b if err != nil { return nil, err } - err = t.pluginHook(hookEditRecordPre, string(b)) + err = t.pluginHookPre(hookEditRecordPre, string(b)) if err != nil { return nil, err } @@ -1007,12 +1003,7 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b } // Call post plugin hooks - err = t.pluginHook(hookEditRecordPost, string(b)) - if err != nil { - e := fmt.Sprintf("UpdateVettedRecord %x: pluginHook editRecordPost: %v", - token, err) - panic(e) - } + t.pluginHookPost(hookEditRecordPost, string(b)) // Return updated record r, err = t.vetted.recordLatest(treeID) @@ -1085,7 +1076,7 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite if err != nil { return err } - err = t.pluginHook(hookEditMetadataPre, string(b)) + err = t.pluginHookPre(hookEditMetadataPre, string(b)) if err != nil { return err } @@ -1106,12 +1097,7 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite } // Call post plugin hooks - err = t.pluginHook(hookEditMetadataPost, string(b)) - if err != nil { - e := fmt.Sprintf("UpdateUnvettedMetadata %x: pluginHook editMetadataPost: %v", - token, err) - panic(e) - } + t.pluginHookPost(hookEditMetadataPost, string(b)) return nil } @@ -1183,7 +1169,7 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ if err != nil { return err } - err = t.pluginHook(hookEditMetadataPre, string(b)) + err = t.pluginHookPre(hookEditMetadataPre, string(b)) if err != nil { return err } @@ -1204,12 +1190,7 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ } // Call post plugin hooks - err = t.pluginHook(hookEditMetadataPost, string(b)) - if err != nil { - e := fmt.Sprintf("UpdateVettedMetadata %x: pluginHook editMetadataPost: %v", - token, err) - panic(e) - } + t.pluginHookPost(hookEditMetadataPost, string(b)) return nil } @@ -1424,7 +1405,7 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, if err != nil { return nil, err } - err = t.pluginHook(hookSetRecordStatusPre, string(b)) + err = t.pluginHookPre(hookSetRecordStatusPre, string(b)) if err != nil { return nil, err } @@ -1447,12 +1428,7 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, } // Call post plugin hooks - err = t.pluginHook(hookSetRecordStatusPost, string(b)) - if err != nil { - e := fmt.Sprintf("SetUnvettedStatus %x: pluginHook setRecordStatusPost: %v", - token, err) - panic(e) - } + t.pluginHookPost(hookSetRecordStatusPost, string(b)) log.Debugf("Status change %x from %v (%v) to %v (%v)", token, backend.MDStatus[currStatus], currStatus, @@ -1581,7 +1557,7 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md if err != nil { return nil, err } - err = t.pluginHook(hookSetRecordStatusPre, string(b)) + err = t.pluginHookPre(hookSetRecordStatusPre, string(b)) if err != nil { return nil, err } @@ -1604,12 +1580,7 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md } // Call post plugin hooks - err = t.pluginHook(hookSetRecordStatusPost, string(b)) - if err != nil { - e := fmt.Sprintf("SetVettedStatus %x: pluginHook setRecordStatusPost: %v", - token, err) - panic(e) - } + t.pluginHookPost(hookSetRecordStatusPost, string(b)) // Update inventory cache t.inventoryUpdate(stateVetted, token, currStatus, status) @@ -1760,7 +1731,7 @@ func (t *tlogBackend) Plugin(pluginID, cmd, cmdID, payload string) (string, erro return reply, nil } -func (t *tlogBackend) pluginHook(h hookT, payload string) error { +func (t *tlogBackend) pluginHookPre(h hookT, payload string) error { // Pass hook event and payload to each plugin for _, v := range t.plugins { err := v.client.hook(h, payload) @@ -1776,6 +1747,20 @@ func (t *tlogBackend) pluginHook(h hookT, payload string) error { return nil } +func (t *tlogBackend) pluginHookPost(h hookT, payload string) { + // Pass hook event and payload to each plugin + for _, v := range t.plugins { + err := v.client.hook(h, payload) + if err != nil { + // This is the post plugin hook so the data has already been + // saved to the tlog backend. We do not have the ability to + // unwind. Log the error and continue. + log.Criticalf("pluginHookPost %v %v: %v: %v", v.id, h, err, payload) + continue + } + } +} + // Close shuts the backend down and performs cleanup. // // This function satisfies the Backend interface. From 6586668fb97514fff4ca34b1b9ac6a92a0cc6624 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 31 Dec 2020 12:31:52 -0600 Subject: [PATCH 198/449] package naming --- politeiawww/piwww.go | 899 ++++++++++++++++++++++--------------------- 1 file changed, 450 insertions(+), 449 deletions(-) diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 657e230f7..7f733c21c 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -18,11 +18,11 @@ import ( "strconv" "time" - pd "github.com/decred/politeia/politeiad/api/v1" + pdv1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/plugins/comments" - piplugin "github.com/decred/politeia/politeiad/plugins/pi" + "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/ticketvote" - pi "github.com/decred/politeia/politeiawww/api/pi/v1" + piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/user" wwwutil "github.com/decred/politeia/politeiawww/util" @@ -30,6 +30,7 @@ import ( "github.com/google/uuid" ) +// TODO www package references should be completely gone from this file // TODO use pi policies. Should the policies be defined in the pi plugin // or the pi api spec? // TODO ensure plugins can't write data using short proposal token. @@ -50,9 +51,9 @@ var ( // statusReasonRequired contains the list of proposal statuses that // require an accompanying reason to be given for the status change. - statusReasonRequired = map[pi.PropStatusT]struct{}{ - pi.PropStatusCensored: {}, - pi.PropStatusAbandoned: {}, + statusReasonRequired = map[piv1.PropStatusT]struct{}{ + piv1.PropStatusCensored: {}, + piv1.PropStatusAbandoned: {}, } // errProposalNotFound is emitted when a proposal is not found in @@ -69,9 +70,9 @@ var ( func tokenIsValid(token string) bool { // Verify token size switch { - case len(token) == pd.TokenPrefixLength: + case len(token) == pdv1.TokenPrefixLength: // Token is a short proposal token - case len(token) == pd.TokenSizeShort*2: + case len(token) == pdv1.TokenSizeShort*2: // Token is a full length token default: // Unknown token size @@ -91,7 +92,7 @@ func tokenIsFullLength(token string) bool { if err != nil { return false } - if len(b) != pd.TokenSizeShort { + if len(b) != pdv1.TokenSizeShort { return false } return true @@ -126,15 +127,15 @@ func proposalNameRegex() string { // proposalName parses the proposal name from the ProposalMetadata and returns // it. An empty string will be returned if any errors occur or if a name is not // found. -func proposalName(pr pi.ProposalRecord) string { +func proposalName(pr piv1.ProposalRecord) string { var name string for _, v := range pr.Metadata { - if v.Hint == pi.HintProposalMetadata { + if v.Hint == piv1.HintProposalMetadata { b, err := base64.StdEncoding.DecodeString(v.Payload) if err != nil { return "" } - pm, err := piplugin.DecodeProposalMetadata(b) + pm, err := pi.DecodeProposalMetadata(b) if err != nil { return "" } @@ -146,7 +147,7 @@ func proposalName(pr pi.ProposalRecord) string { // proposalRecordFillInUser fills in all user fields that are store in the // user database and not in politeiad. -func proposalRecordFillInUser(pr pi.ProposalRecord, u user.User) pi.ProposalRecord { +func proposalRecordFillInUser(pr piv1.ProposalRecord, u user.User) piv1.ProposalRecord { pr.UserID = u.ID.String() pr.Username = u.Username return pr @@ -155,59 +156,59 @@ func proposalRecordFillInUser(pr pi.ProposalRecord, u user.User) pi.ProposalReco // commentFillInUser populates the provided comment with user data that is not // store in politeiad and must be populated separately after pulling the // comment from politeiad. -func commentFillInUser(c pi.Comment, u user.User) pi.Comment { +func commentFillInUser(c piv1.Comment, u user.User) piv1.Comment { c.Username = u.Username return c } -func convertUserErrorFromSignatureError(err error) pi.UserErrorReply { +func convertUserErrorFromSignatureError(err error) piv1.UserErrorReply { var e util.SignatureError - var s pi.ErrorStatusT + var s piv1.ErrorStatusT if errors.As(err, &e) { switch e.ErrorCode { case util.ErrorStatusPublicKeyInvalid: - s = pi.ErrorStatusPublicKeyInvalid + s = piv1.ErrorStatusPublicKeyInvalid case util.ErrorStatusSignatureInvalid: - s = pi.ErrorStatusSignatureInvalid + s = piv1.ErrorStatusSignatureInvalid } } - return pi.UserErrorReply{ + return piv1.UserErrorReply{ ErrorCode: s, ErrorContext: e.ErrorContext, } } -func convertPropStateFromPi(s pi.PropStateT) piplugin.PropStateT { +func convertPropStateFromPi(s piv1.PropStateT) pi.PropStateT { switch s { - case pi.PropStateUnvetted: - return piplugin.PropStateUnvetted - case pi.PropStateVetted: - return piplugin.PropStateVetted + case piv1.PropStateUnvetted: + return pi.PropStateUnvetted + case piv1.PropStateVetted: + return pi.PropStateVetted } - return piplugin.PropStateInvalid + return pi.PropStateInvalid } -func convertRecordStatusFromPropStatus(s pi.PropStatusT) pd.RecordStatusT { +func convertRecordStatusFromPropStatus(s piv1.PropStatusT) pdv1.RecordStatusT { switch s { - case pi.PropStatusUnreviewed: - return pd.RecordStatusNotReviewed - case pi.PropStatusPublic: - return pd.RecordStatusPublic - case pi.PropStatusCensored: - return pd.RecordStatusCensored - case pi.PropStatusAbandoned: - return pd.RecordStatusArchived + case piv1.PropStatusUnreviewed: + return pdv1.RecordStatusNotReviewed + case piv1.PropStatusPublic: + return pdv1.RecordStatusPublic + case piv1.PropStatusCensored: + return pdv1.RecordStatusCensored + case piv1.PropStatusAbandoned: + return pdv1.RecordStatusArchived } - return pd.RecordStatusInvalid + return pdv1.RecordStatusInvalid } -func convertFileFromMetadata(m pi.Metadata) pd.File { +func convertFileFromMetadata(m piv1.Metadata) pdv1.File { var name string switch m.Hint { - case pi.HintProposalMetadata: - name = piplugin.FileNameProposalMetadata + case piv1.HintProposalMetadata: + name = pi.FileNameProposalMetadata } - return pd.File{ + return pdv1.File{ Name: name, MIME: mimeTypeTextUTF8, Digest: m.Digest, @@ -215,8 +216,8 @@ func convertFileFromMetadata(m pi.Metadata) pd.File { } } -func convertFileFromPi(f pi.File) pd.File { - return pd.File{ +func convertFileFromPi(f piv1.File) pdv1.File { + return pdv1.File{ Name: f.Name, MIME: f.MIME, Digest: f.Digest, @@ -224,53 +225,53 @@ func convertFileFromPi(f pi.File) pd.File { } } -func convertFilesFromPi(files []pi.File) []pd.File { - f := make([]pd.File, 0, len(files)) +func convertFilesFromPi(files []piv1.File) []pdv1.File { + f := make([]pdv1.File, 0, len(files)) for _, v := range files { f = append(f, convertFileFromPi(v)) } return f } -func convertPropStatusFromPD(s pd.RecordStatusT) pi.PropStatusT { +func convertPropStatusFromPD(s pdv1.RecordStatusT) piv1.PropStatusT { switch s { - case pd.RecordStatusNotFound: + case pdv1.RecordStatusNotFound: // Intentionally omitted. No corresponding PropStatusT. - case pd.RecordStatusNotReviewed: - return pi.PropStatusUnreviewed - case pd.RecordStatusCensored: - return pi.PropStatusCensored - case pd.RecordStatusPublic: - return pi.PropStatusPublic - case pd.RecordStatusUnreviewedChanges: - return pi.PropStatusUnreviewed - case pd.RecordStatusArchived: - return pi.PropStatusAbandoned - } - return pi.PropStatusInvalid -} - -func convertCensorshipRecordFromPD(cr pd.CensorshipRecord) pi.CensorshipRecord { - return pi.CensorshipRecord{ + case pdv1.RecordStatusNotReviewed: + return piv1.PropStatusUnreviewed + case pdv1.RecordStatusCensored: + return piv1.PropStatusCensored + case pdv1.RecordStatusPublic: + return piv1.PropStatusPublic + case pdv1.RecordStatusUnreviewedChanges: + return piv1.PropStatusUnreviewed + case pdv1.RecordStatusArchived: + return piv1.PropStatusAbandoned + } + return piv1.PropStatusInvalid +} + +func convertCensorshipRecordFromPD(cr pdv1.CensorshipRecord) piv1.CensorshipRecord { + return piv1.CensorshipRecord{ Token: cr.Token, Merkle: cr.Merkle, Signature: cr.Signature, } } -func convertFilesFromPD(f []pd.File) ([]pi.File, []pi.Metadata) { - files := make([]pi.File, 0, len(f)) - metadata := make([]pi.Metadata, 0, len(f)) +func convertFilesFromPD(f []pdv1.File) ([]piv1.File, []piv1.Metadata) { + files := make([]piv1.File, 0, len(f)) + metadata := make([]piv1.Metadata, 0, len(f)) for _, v := range f { switch v.Name { - case piplugin.FileNameProposalMetadata: - metadata = append(metadata, pi.Metadata{ - Hint: pi.HintProposalMetadata, + case pi.FileNameProposalMetadata: + metadata = append(metadata, piv1.Metadata{ + Hint: piv1.HintProposalMetadata, Digest: v.Digest, Payload: v.Payload, }) default: - files = append(files, pi.File{ + files = append(files, piv1.File{ Name: v.Name, MIME: v.MIME, Digest: v.Digest, @@ -281,22 +282,22 @@ func convertFilesFromPD(f []pd.File) ([]pi.File, []pi.Metadata) { return files, metadata } -func convertProposalRecordFromPD(r pd.Record, state pi.PropStateT) (*pi.ProposalRecord, error) { +func convertProposalRecordFromPD(r pdv1.Record, state piv1.PropStateT) (*piv1.ProposalRecord, error) { // Decode metadata streams var ( - gm *piplugin.GeneralMetadata - sc = make([]piplugin.StatusChange, 0, 16) + gm *pi.GeneralMetadata + sc = make([]pi.StatusChange, 0, 16) err error ) for _, v := range r.Metadata { switch v.ID { - case piplugin.MDStreamIDGeneralMetadata: - gm, err = piplugin.DecodeGeneralMetadata([]byte(v.Payload)) + case pi.MDStreamIDGeneralMetadata: + gm, err = pi.DecodeGeneralMetadata([]byte(v.Payload)) if err != nil { return nil, err } - case piplugin.MDStreamIDStatusChanges: - sc, err = piplugin.DecodeStatusChanges([]byte(v.Payload)) + case pi.MDStreamIDStatusChanges: + sc, err = pi.DecodeStatusChanges([]byte(v.Payload)) if err != nil { return nil, err } @@ -307,12 +308,12 @@ func convertProposalRecordFromPD(r pd.Record, state pi.PropStateT) (*pi.Proposal files, metadata := convertFilesFromPD(r.Files) status := convertPropStatusFromPD(r.Status) - statuses := make([]pi.StatusChange, 0, len(sc)) + statuses := make([]piv1.StatusChange, 0, len(sc)) for _, v := range sc { - statuses = append(statuses, pi.StatusChange{ + statuses = append(statuses, piv1.StatusChange{ Token: v.Token, Version: v.Version, - Status: pi.PropStatusT(v.Status), + Status: piv1.PropStatusT(v.Status), Reason: v.Reason, PublicKey: v.PublicKey, Signature: v.Signature, @@ -324,7 +325,7 @@ func convertProposalRecordFromPD(r pd.Record, state pi.PropStateT) (*pi.Proposal // user data that needs to be pulled from the user database or they // are politeiad plugin data that needs to be retrieved using a // plugin command. - return &pi.ProposalRecord{ + return &piv1.ProposalRecord{ Version: r.Version, Timestamp: gm.Timestamp, State: state, @@ -342,48 +343,48 @@ func convertProposalRecordFromPD(r pd.Record, state pi.PropStateT) (*pi.Proposal }, nil } -func convertCommentsStateFromPi(s pi.PropStateT) comments.StateT { +func convertCommentsStateFromPi(s piv1.PropStateT) comments.StateT { switch s { - case pi.PropStateUnvetted: + case piv1.PropStateUnvetted: return comments.StateUnvetted - case pi.PropStateVetted: + case piv1.PropStateVetted: return comments.StateVetted } return comments.StateInvalid } -func convertPropStateFromComments(s comments.StateT) pi.PropStateT { +func convertPropStateFromComments(s comments.StateT) piv1.PropStateT { switch s { case comments.StateUnvetted: - return pi.PropStateUnvetted + return piv1.PropStateUnvetted case comments.StateVetted: - return pi.PropStateVetted + return piv1.PropStateVetted } - return pi.PropStateInvalid + return piv1.PropStateInvalid } -func convertCommentStateFromPi(s pi.PropStateT) comments.StateT { +func convertCommentStateFromPi(s piv1.PropStateT) comments.StateT { switch s { - case pi.PropStateUnvetted: + case piv1.PropStateUnvetted: return comments.StateUnvetted - case pi.PropStateVetted: + case piv1.PropStateVetted: return comments.StateVetted } return comments.StateInvalid } -func convertCommentVoteFromPi(cv pi.CommentVoteT) comments.VoteT { +func convertCommentVoteFromPi(cv piv1.CommentVoteT) comments.VoteT { switch cv { - case pi.CommentVoteDownvote: + case piv1.CommentVoteDownvote: return comments.VoteUpvote - case pi.CommentVoteUpvote: + case piv1.CommentVoteUpvote: return comments.VoteDownvote } return comments.VoteInvalid } -func convertCommentFromPlugin(c comments.Comment) pi.Comment { - return pi.Comment{ +func convertCommentFromPlugin(c comments.Comment) piv1.Comment { + return piv1.Comment{ UserID: c.UserID, Username: "", // Intentionally omitted, needs to be pulled from userdb State: convertPropStateFromComments(c.State), @@ -402,20 +403,20 @@ func convertCommentFromPlugin(c comments.Comment) pi.Comment { } } -func convertCommentVoteFromPlugin(v comments.VoteT) pi.CommentVoteT { +func convertCommentVoteFromPlugin(v comments.VoteT) piv1.CommentVoteT { switch v { case comments.VoteDownvote: - return pi.CommentVoteDownvote + return piv1.CommentVoteDownvote case comments.VoteUpvote: - return pi.CommentVoteUpvote + return piv1.CommentVoteUpvote } - return pi.CommentVoteInvalid + return piv1.CommentVoteInvalid } -func convertCommentVoteDetailsFromPlugin(cv []comments.CommentVote) []pi.CommentVoteDetails { - c := make([]pi.CommentVoteDetails, 0, len(cv)) +func convertCommentVoteDetailsFromPlugin(cv []comments.CommentVote) []piv1.CommentVoteDetails { + c := make([]piv1.CommentVoteDetails, 0, len(cv)) for _, v := range cv { - c = append(c, pi.CommentVoteDetails{ + c = append(c, piv1.CommentVoteDetails{ UserID: v.UserID, State: convertPropStateFromComments(v.State), Token: v.Token, @@ -430,18 +431,18 @@ func convertCommentVoteDetailsFromPlugin(cv []comments.CommentVote) []pi.Comment return c } -func convertVoteAuthActionFromPi(a pi.VoteAuthActionT) ticketvote.AuthActionT { +func convertVoteAuthActionFromPi(a piv1.VoteAuthActionT) ticketvote.AuthActionT { switch a { - case pi.VoteAuthActionAuthorize: + case piv1.VoteAuthActionAuthorize: return ticketvote.AuthActionAuthorize - case pi.VoteAuthActionRevoke: + case piv1.VoteAuthActionRevoke: return ticketvote.AuthActionRevoke default: return ticketvote.AuthActionAuthorize } } -func convertVoteAuthorizeFromPi(va pi.VoteAuthorize) ticketvote.Authorize { +func convertVoteAuthorizeFromPi(va piv1.VoteAuthorize) ticketvote.Authorize { return ticketvote.Authorize{ Token: va.Token, Version: va.Version, @@ -451,7 +452,7 @@ func convertVoteAuthorizeFromPi(va pi.VoteAuthorize) ticketvote.Authorize { } } -func convertVoteAuthsFromPi(auths []pi.VoteAuthorize) []ticketvote.Authorize { +func convertVoteAuthsFromPi(auths []piv1.VoteAuthorize) []ticketvote.Authorize { a := make([]ticketvote.Authorize, 0, len(auths)) for _, v := range auths { a = append(a, convertVoteAuthorizeFromPi(v)) @@ -459,17 +460,17 @@ func convertVoteAuthsFromPi(auths []pi.VoteAuthorize) []ticketvote.Authorize { return a } -func convertVoteTypeFromPi(t pi.VoteT) ticketvote.VoteT { +func convertVoteTypeFromPi(t piv1.VoteT) ticketvote.VoteT { switch t { - case pi.VoteTypeStandard: + case piv1.VoteTypeStandard: return ticketvote.VoteTypeStandard - case pi.VoteTypeRunoff: + case piv1.VoteTypeRunoff: return ticketvote.VoteTypeRunoff } return ticketvote.VoteTypeInvalid } -func convertVoteParamsFromPi(v pi.VoteParams) ticketvote.VoteParams { +func convertVoteParamsFromPi(v piv1.VoteParams) ticketvote.VoteParams { tv := ticketvote.VoteParams{ Token: v.Token, Version: v.Version, @@ -494,7 +495,7 @@ func convertVoteParamsFromPi(v pi.VoteParams) ticketvote.VoteParams { return tv } -func convertStartDetailsFromPi(sd pi.StartDetails) ticketvote.StartDetails { +func convertStartDetailsFromPi(sd piv1.StartDetails) ticketvote.StartDetails { return ticketvote.StartDetails{ Params: convertVoteParamsFromPi(sd.Params), PublicKey: sd.PublicKey, @@ -502,7 +503,7 @@ func convertStartDetailsFromPi(sd pi.StartDetails) ticketvote.StartDetails { } } -func convertVoteStartFromPi(vs pi.VoteStart) ticketvote.Start { +func convertVoteStartFromPi(vs piv1.VoteStart) ticketvote.Start { starts := make([]ticketvote.StartDetails, 0, len(vs.Starts)) for _, v := range vs.Starts { starts = append(starts, convertStartDetailsFromPi(v)) @@ -512,7 +513,7 @@ func convertVoteStartFromPi(vs pi.VoteStart) ticketvote.Start { } } -func convertVoteStartsFromPi(starts []pi.VoteStart) []ticketvote.Start { +func convertVoteStartsFromPi(starts []piv1.VoteStart) []ticketvote.Start { s := make([]ticketvote.Start, 0, len(starts)) for _, v := range starts { s = append(s, convertVoteStartFromPi(v)) @@ -520,7 +521,7 @@ func convertVoteStartsFromPi(starts []pi.VoteStart) []ticketvote.Start { return s } -func convertCastVotesFromPi(votes []pi.CastVote) []ticketvote.CastVote { +func convertCastVotesFromPi(votes []piv1.CastVote) []ticketvote.CastVote { cv := make([]ticketvote.CastVote, 0, len(votes)) for _, v := range votes { cv = append(cv, ticketvote.CastVote{ @@ -533,40 +534,40 @@ func convertCastVotesFromPi(votes []pi.CastVote) []ticketvote.CastVote { return cv } -func convertVoteErrorFromPlugin(e ticketvote.VoteErrorT) pi.VoteErrorT { +func convertVoteErrorFromPlugin(e ticketvote.VoteErrorT) piv1.VoteErrorT { switch e { case ticketvote.VoteErrorInvalid: - return pi.VoteErrorInvalid + return piv1.VoteErrorInvalid case ticketvote.VoteErrorInternalError: - return pi.VoteErrorInternalError + return piv1.VoteErrorInternalError case ticketvote.VoteErrorRecordNotFound: - return pi.VoteErrorRecordNotFound + return piv1.VoteErrorRecordNotFound case ticketvote.VoteErrorVoteBitInvalid: - return pi.VoteErrorVoteBitInvalid + return piv1.VoteErrorVoteBitInvalid case ticketvote.VoteErrorVoteStatusInvalid: - return pi.VoteErrorVoteStatusInvalid + return piv1.VoteErrorVoteStatusInvalid case ticketvote.VoteErrorTicketAlreadyVoted: - return pi.VoteErrorTicketAlreadyVoted + return piv1.VoteErrorTicketAlreadyVoted case ticketvote.VoteErrorTicketNotEligible: - return pi.VoteErrorTicketNotEligible + return piv1.VoteErrorTicketNotEligible default: - return pi.VoteErrorInternalError + return piv1.VoteErrorInternalError } } -func convertVoteTypeFromPlugin(t ticketvote.VoteT) pi.VoteT { +func convertVoteTypeFromPlugin(t ticketvote.VoteT) piv1.VoteT { switch t { case ticketvote.VoteTypeStandard: - return pi.VoteTypeStandard + return piv1.VoteTypeStandard case ticketvote.VoteTypeRunoff: - return pi.VoteTypeRunoff + return piv1.VoteTypeRunoff } - return pi.VoteTypeInvalid + return piv1.VoteTypeInvalid } -func convertVoteParamsFromPlugin(v ticketvote.VoteParams) pi.VoteParams { - vp := pi.VoteParams{ +func convertVoteParamsFromPlugin(v ticketvote.VoteParams) piv1.VoteParams { + vp := piv1.VoteParams{ Token: v.Token, Version: v.Version, Type: convertVoteTypeFromPlugin(v.Type), @@ -575,9 +576,9 @@ func convertVoteParamsFromPlugin(v ticketvote.VoteParams) pi.VoteParams { QuorumPercentage: v.QuorumPercentage, PassPercentage: v.PassPercentage, } - vo := make([]pi.VoteOption, 0, len(v.Options)) + vo := make([]piv1.VoteOption, 0, len(v.Options)) for _, o := range v.Options { - vo = append(vo, pi.VoteOption{ + vo = append(vo, piv1.VoteOption{ ID: o.ID, Description: o.Description, Bit: o.Bit, @@ -588,10 +589,10 @@ func convertVoteParamsFromPlugin(v ticketvote.VoteParams) pi.VoteParams { return vp } -func convertCastVoteRepliesFromPlugin(replies []ticketvote.CastVoteReply) []pi.CastVoteReply { - r := make([]pi.CastVoteReply, 0, len(replies)) +func convertCastVoteRepliesFromPlugin(replies []ticketvote.CastVoteReply) []piv1.CastVoteReply { + r := make([]piv1.CastVoteReply, 0, len(replies)) for _, v := range replies { - r = append(r, pi.CastVoteReply{ + r = append(r, piv1.CastVoteReply{ Ticket: v.Ticket, Receipt: v.Receipt, ErrorCode: convertVoteErrorFromPlugin(v.ErrorCode), @@ -601,8 +602,8 @@ func convertCastVoteRepliesFromPlugin(replies []ticketvote.CastVoteReply) []pi.C return r } -func convertVoteDetailsFromPlugin(vd ticketvote.VoteDetails) pi.VoteDetails { - return pi.VoteDetails{ +func convertVoteDetailsFromPlugin(vd ticketvote.VoteDetails) piv1.VoteDetails { + return piv1.VoteDetails{ Params: convertVoteParamsFromPlugin(vd.Params), PublicKey: vd.PublicKey, Signature: vd.Signature, @@ -613,10 +614,10 @@ func convertVoteDetailsFromPlugin(vd ticketvote.VoteDetails) pi.VoteDetails { } } -func convertAuthDetailsFromPlugin(auths []ticketvote.AuthDetails) []pi.AuthDetails { - a := make([]pi.AuthDetails, 0, len(auths)) +func convertAuthDetailsFromPlugin(auths []ticketvote.AuthDetails) []piv1.AuthDetails { + a := make([]piv1.AuthDetails, 0, len(auths)) for _, v := range auths { - a = append(a, pi.AuthDetails{ + a = append(a, piv1.AuthDetails{ Token: v.Token, Version: v.Version, Action: v.Action, @@ -629,10 +630,10 @@ func convertAuthDetailsFromPlugin(auths []ticketvote.AuthDetails) []pi.AuthDetai return a } -func convertCastVoteDetailsFromPlugin(votes []ticketvote.CastVoteDetails) []pi.CastVoteDetails { - vs := make([]pi.CastVoteDetails, 0, len(votes)) +func convertCastVoteDetailsFromPlugin(votes []ticketvote.CastVoteDetails) []piv1.CastVoteDetails { + vs := make([]piv1.CastVoteDetails, 0, len(votes)) for _, v := range votes { - vs = append(vs, pi.CastVoteDetails{ + vs = append(vs, piv1.CastVoteDetails{ Token: v.Token, Ticket: v.Ticket, VoteBit: v.VoteBit, @@ -643,15 +644,15 @@ func convertCastVoteDetailsFromPlugin(votes []ticketvote.CastVoteDetails) []pi.C return vs } -func convertProposalVotesFromPlugin(votes map[string]ticketvote.RecordVote) map[string]pi.ProposalVote { - pv := make(map[string]pi.ProposalVote, len(votes)) +func convertProposalVotesFromPlugin(votes map[string]ticketvote.RecordVote) map[string]piv1.ProposalVote { + pv := make(map[string]piv1.ProposalVote, len(votes)) for k, v := range votes { - var vdp *pi.VoteDetails + var vdp *piv1.VoteDetails if v.Vote != nil { vd := convertVoteDetailsFromPlugin(*v.Vote) vdp = &vd } - pv[k] = pi.ProposalVote{ + pv[k] = piv1.ProposalVote{ Auths: convertAuthDetailsFromPlugin(v.Auths), Vote: vdp, } @@ -659,34 +660,34 @@ func convertProposalVotesFromPlugin(votes map[string]ticketvote.RecordVote) map[ return pv } -func convertVoteStatusFromPlugin(s ticketvote.VoteStatusT) pi.VoteStatusT { +func convertVoteStatusFromPlugin(s ticketvote.VoteStatusT) piv1.VoteStatusT { switch s { case ticketvote.VoteStatusInvalid: - return pi.VoteStatusInvalid + return piv1.VoteStatusInvalid case ticketvote.VoteStatusUnauthorized: - return pi.VoteStatusUnauthorized + return piv1.VoteStatusUnauthorized case ticketvote.VoteStatusAuthorized: - return pi.VoteStatusAuthorized + return piv1.VoteStatusAuthorized case ticketvote.VoteStatusStarted: - return pi.VoteStatusStarted + return piv1.VoteStatusStarted case ticketvote.VoteStatusFinished: - return pi.VoteStatusFinished + return piv1.VoteStatusFinished default: - return pi.VoteStatusInvalid + return piv1.VoteStatusInvalid } } -func convertVoteSummaryFromPlugin(s ticketvote.Summary) pi.VoteSummary { - results := make([]pi.VoteResult, 0, len(s.Results)) +func convertVoteSummaryFromPlugin(s ticketvote.Summary) piv1.VoteSummary { + results := make([]piv1.VoteResult, 0, len(s.Results)) for _, v := range s.Results { - results = append(results, pi.VoteResult{ + results = append(results, piv1.VoteResult{ ID: v.ID, Description: v.Description, VoteBit: v.VoteBit, Votes: v.Votes, }) } - return pi.VoteSummary{ + return piv1.VoteSummary{ Type: convertVoteTypeFromPlugin(s.Type), Status: convertVoteStatusFromPlugin(s.Status), Duration: s.Duration, @@ -701,8 +702,8 @@ func convertVoteSummaryFromPlugin(s ticketvote.Summary) pi.VoteSummary { } } -func convertVoteSummariesFromPlugin(ts map[string]ticketvote.Summary) map[string]pi.VoteSummary { - s := make(map[string]pi.VoteSummary, len(ts)) +func convertVoteSummariesFromPlugin(ts map[string]ticketvote.Summary) map[string]piv1.VoteSummary { + s := make(map[string]piv1.VoteSummary, len(ts)) for k, v := range ts { s[k] = convertVoteSummaryFromPlugin(v) } @@ -740,20 +741,20 @@ func (p *politeiawww) linkByPeriodMax() int64 { // proposalRecords returns the ProposalRecord for each of the provided proposal // requests. If a token does not correspond to an actual proposal then it will // not be included in the returned map. -func (p *politeiawww) proposalRecords(ctx context.Context, state pi.PropStateT, reqs []pi.ProposalRequest, includeFiles bool) (map[string]pi.ProposalRecord, error) { +func (p *politeiawww) proposalRecords(ctx context.Context, state piv1.PropStateT, reqs []piv1.ProposalRequest, includeFiles bool) (map[string]piv1.ProposalRecord, error) { // Get politeiad records - props := make([]pi.ProposalRecord, 0, len(reqs)) + props := make([]piv1.ProposalRecord, 0, len(reqs)) for _, v := range reqs { - var r *pd.Record + var r *pdv1.Record var err error switch state { - case pi.PropStateUnvetted: + case piv1.PropStateUnvetted: // Unvetted politeiad record r, err = p.getUnvetted(ctx, v.Token, v.Version) if err != nil { return nil, err } - case pi.PropStateVetted: + case piv1.PropStateVetted: // Vetted politeiad record r, err = p.getVetted(ctx, v.Token, v.Version) if err != nil { @@ -763,7 +764,7 @@ func (p *politeiawww) proposalRecords(ctx context.Context, state pi.PropStateT, return nil, fmt.Errorf("unknown state %v", state) } - if r.Status == pd.RecordStatusNotFound { + if r.Status == pdv1.RecordStatusNotFound { // Record wasn't found. Don't include token in the results. continue } @@ -776,7 +777,7 @@ func (p *politeiawww) proposalRecords(ctx context.Context, state pi.PropStateT, // Remove files if specified. The Metadata objects will still be // returned. if !includeFiles { - pr.Files = []pi.File{} + pr.Files = []piv1.File{} } props = append(props, *pr) @@ -784,7 +785,7 @@ func (p *politeiawww) proposalRecords(ctx context.Context, state pi.PropStateT, // Verify we've got some results if len(props) == 0 { - return map[string]pi.ProposalRecord{}, nil + return map[string]piv1.ProposalRecord{}, nil } // Get proposal plugin data @@ -792,7 +793,7 @@ func (p *politeiawww) proposalRecords(ctx context.Context, state pi.PropStateT, for _, v := range props { tokens = append(tokens, v.CensorshipRecord.Token) } - ps := piplugin.Proposals{ + ps := pi.Proposals{ State: convertPropStateFromPi(state), Tokens: tokens, } @@ -830,7 +831,7 @@ func (p *politeiawww) proposalRecords(ctx context.Context, state pi.PropStateT, } // Convert proposals to a map - proposals := make(map[string]pi.ProposalRecord, len(props)) + proposals := make(map[string]piv1.ProposalRecord, len(props)) for _, v := range props { proposals[v.CensorshipRecord.Token] = v } @@ -842,8 +843,8 @@ func (p *politeiawww) proposalRecords(ctx context.Context, state pi.PropStateT, // version. A blank version will return the most recent version. A // errProposalNotFound error will be returned if a proposal is not found for // the provided token/version combination. -func (p *politeiawww) proposalRecord(ctx context.Context, state pi.PropStateT, token, version string) (*pi.ProposalRecord, error) { - prs, err := p.proposalRecords(ctx, state, []pi.ProposalRequest{ +func (p *politeiawww) proposalRecord(ctx context.Context, state piv1.PropStateT, token, version string) (*piv1.ProposalRecord, error) { + prs, err := p.proposalRecords(ctx, state, []piv1.ProposalRequest{ { Token: token, Version: version, @@ -862,15 +863,15 @@ func (p *politeiawww) proposalRecord(ctx context.Context, state pi.PropStateT, t // proposalRecordLatest returns the latest version of the proposal record for // the provided token. A errProposalNotFound error will be returned if a // proposal is not found for the provided token. -func (p *politeiawww) proposalRecordLatest(ctx context.Context, state pi.PropStateT, token string) (*pi.ProposalRecord, error) { +func (p *politeiawww) proposalRecordLatest(ctx context.Context, state piv1.PropStateT, token string) (*piv1.ProposalRecord, error) { return p.proposalRecord(ctx, state, token, "") } -func (p *politeiawww) verifyProposalMetadata(pm pi.ProposalMetadata) error { +func (p *politeiawww) verifyProposalMetadata(pm piv1.ProposalMetadata) error { // Verify name if !proposalNameIsValid(pm.Name) { - return pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropNameInvalid, + return piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusPropNameInvalid, ErrorContext: []string{proposalNameRegex()}, } } @@ -878,8 +879,8 @@ func (p *politeiawww) verifyProposalMetadata(pm pi.ProposalMetadata) error { // Verify linkto if pm.LinkTo != "" { if !tokenIsFullLength(pm.LinkTo) { - return pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropLinkToInvalid, + return piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusPropLinkToInvalid, ErrorContext: []string{"invalid token"}, } } @@ -893,15 +894,15 @@ func (p *politeiawww) verifyProposalMetadata(pm pi.ProposalMetadata) error { case pm.LinkBy < min: e := fmt.Sprintf("linkby %v is less than min required of %v", pm.LinkBy, min) - return pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropLinkByInvalid, + return piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusPropLinkByInvalid, ErrorContext: []string{e}, } case pm.LinkBy > max: e := fmt.Sprintf("linkby %v is more than max allowed of %v", pm.LinkBy, max) - return pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropLinkByInvalid, + return piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusPropLinkByInvalid, ErrorContext: []string{e}, } } @@ -910,10 +911,10 @@ func (p *politeiawww) verifyProposalMetadata(pm pi.ProposalMetadata) error { return nil } -func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, publicKey, signature string) (*pi.ProposalMetadata, error) { +func (p *politeiawww) verifyProposal(files []piv1.File, metadata []piv1.Metadata, publicKey, signature string) (*piv1.ProposalMetadata, error) { if len(files) == 0 { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusFileCountInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusFileCountInvalid, ErrorContext: []string{"no files found"}, } } @@ -930,8 +931,8 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu _, ok := filenames[v.Name] if ok { e := fmt.Sprintf("duplicate name %v", v.Name) - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusFileNameInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusFileNameInvalid, ErrorContext: []string{e}, } } @@ -940,16 +941,16 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu // Validate file payload if v.Payload == "" { e := fmt.Sprintf("file %v empty payload", v.Name) - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusFilePayloadInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusFilePayloadInvalid, ErrorContext: []string{e}, } } payloadb, err := base64.StdEncoding.DecodeString(v.Payload) if err != nil { e := fmt.Sprintf("file %v invalid base64", v.Name) - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusFilePayloadInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusFilePayloadInvalid, ErrorContext: []string{e}, } } @@ -958,16 +959,16 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu digest := util.Digest(payloadb) d, ok := util.ConvertDigest(v.Digest) if !ok { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusFileDigestInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusFileDigestInvalid, ErrorContext: []string{v.Name}, } } if !bytes.Equal(digest, d[:]) { e := fmt.Sprintf("file %v digest got %v, want %x", v.Name, v.Digest, digest) - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusFileDigestInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusFileDigestInvalid, ErrorContext: []string{e}, } } @@ -981,16 +982,16 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu mimeFile, _, err := mime.ParseMediaType(v.MIME) if err != nil { e := fmt.Sprintf("file %v mime '%v' not parsable", v.Name, v.MIME) - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusFileMIMEInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusFileMIMEInvalid, ErrorContext: []string{e}, } } if mimeFile != mimePayload { e := fmt.Sprintf("file %v mime got %v, want %v", v.Name, mimeFile, mimePayload) - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusFileMIMEInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusFileMIMEInvalid, ErrorContext: []string{e}, } } @@ -1004,8 +1005,8 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu if len(payloadb) > www.PolicyMaxMDSize { e := fmt.Sprintf("file %v size %v exceeds max size %v", v.Name, len(payloadb), www.PolicyMaxMDSize) - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusIndexFileSizeInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusIndexFileSizeInvalid, ErrorContext: []string{e}, } } @@ -1014,16 +1015,16 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu // file. if v.Name != www.PolicyIndexFilename { e := fmt.Sprintf("want %v, got %v", www.PolicyIndexFilename, v.Name) - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusIndexFileNameInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusIndexFileNameInvalid, ErrorContext: []string{e}, } } if foundIndexFile { e := fmt.Sprintf("more than one %v file found", www.PolicyIndexFilename) - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusIndexFileCountInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusIndexFileCountInvalid, ErrorContext: []string{e}, } } @@ -1038,15 +1039,15 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu if len(payloadb) > www.PolicyMaxImageSize { e := fmt.Sprintf("image %v size %v exceeds max size %v", v.Name, len(payloadb), www.PolicyMaxImageSize) - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusImageFileSizeInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusImageFileSizeInvalid, ErrorContext: []string{e}, } } default: - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusFileMIMEInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusFileMIMEInvalid, ErrorContext: []string{v.MIME}, } } @@ -1055,8 +1056,8 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu // Verify that an index file is present if !foundIndexFile { e := fmt.Sprintf("%v file not found", www.PolicyIndexFilename) - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusIndexFileCountInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusIndexFileCountInvalid, ErrorContext: []string{e}, } } @@ -1065,16 +1066,16 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu if countTextFiles > www.PolicyMaxMDs { e := fmt.Sprintf("got %v text files, max is %v", countTextFiles, www.PolicyMaxMDs) - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusTextFileCountInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusTextFileCountInvalid, ErrorContext: []string{e}, } } if countImageFiles > www.PolicyMaxImages { e := fmt.Sprintf("got %v image files, max is %v", countImageFiles, www.PolicyMaxImages) - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusImageFileCountInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusImageFileCountInvalid, ErrorContext: []string{e}, } } @@ -1083,21 +1084,21 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu // a ProposalMetadata. switch { case len(metadata) == 0: - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropMetadataNotFound, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusPropMetadataNotFound, } case len(metadata) > 1: e := fmt.Sprintf("metadata should only contain %v", www.HintProposalMetadata) - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusMetadataCountInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusMetadataCountInvalid, ErrorContext: []string{e}, } } md := metadata[0] if md.Hint != www.HintProposalMetadata { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropMetadataNotFound, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusPropMetadataNotFound, } } @@ -1105,8 +1106,8 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu b, err := base64.StdEncoding.DecodeString(md.Payload) if err != nil { e := fmt.Sprintf("metadata with hint %v invalid base64 payload", md.Hint) - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusMetadataPayloadInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusMetadataPayloadInvalid, ErrorContext: []string{e}, } } @@ -1114,8 +1115,8 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu if md.Digest != hex.EncodeToString(digest) { e := fmt.Sprintf("metadata with hint %v got digest %v, want %v", md.Hint, md.Digest, hex.EncodeToString(digest)) - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusMetadataDigestInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusMetadataDigestInvalid, ErrorContext: []string{e}, } } @@ -1123,12 +1124,12 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu // Decode ProposalMetadata d := json.NewDecoder(bytes.NewReader(b)) d.DisallowUnknownFields() - var pm pi.ProposalMetadata + var pm piv1.ProposalMetadata err = d.Decode(&pm) if err != nil { e := fmt.Sprintf("unable to decode %v payload", md.Hint) - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusMetadataPayloadInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusMetadataPayloadInvalid, ErrorContext: []string{e}, } } @@ -1152,27 +1153,27 @@ func (p *politeiawww) verifyProposal(files []pi.File, metadata []pi.Metadata, pu return &pm, nil } -func (p *politeiawww) processProposalNew(ctx context.Context, pn pi.ProposalNew, usr user.User) (*pi.ProposalNewReply, error) { +func (p *politeiawww) processProposalNew(ctx context.Context, pn piv1.ProposalNew, usr user.User) (*piv1.ProposalNewReply, error) { log.Tracef("processProposalNew: %v", usr.Username) // Verify user has paid registration paywall if !p.userHasPaid(usr) { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusUserRegistrationNotPaid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusUserRegistrationNotPaid, } } // Verify user has a proposal credit if !p.userHasProposalCredits(usr) { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusUserBalanceInsufficient, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusUserBalanceInsufficient, } } // Verify user signed using active identity if usr.PublicKey() != pn.PublicKey { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPublicKeyInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusPublicKeyInvalid, ErrorContext: []string{"not active identity"}, } } @@ -1190,26 +1191,26 @@ func (p *politeiawww) processProposalNew(ctx context.Context, pn pi.ProposalNew, files := convertFilesFromPi(pn.Files) for _, v := range pn.Metadata { switch v.Hint { - case pi.HintProposalMetadata: + case piv1.HintProposalMetadata: files = append(files, convertFileFromMetadata(v)) } } // Setup metadata stream timestamp := time.Now().Unix() - gm := piplugin.GeneralMetadata{ + gm := pi.GeneralMetadata{ UserID: usr.ID.String(), PublicKey: pn.PublicKey, Signature: pn.Signature, Timestamp: timestamp, } - b, err := piplugin.EncodeGeneralMetadata(gm) + b, err := pi.EncodeGeneralMetadata(gm) if err != nil { return nil, err } - metadata := []pd.MetadataStream{ + metadata := []pdv1.MetadataStream{ { - ID: piplugin.MDStreamIDGeneralMetadata, + ID: pi.MDStreamIDGeneralMetadata, Payload: string(b), }, } @@ -1241,12 +1242,12 @@ func (p *politeiawww) processProposalNew(ctx context.Context, pn pi.ProposalNew, } // Get full proposal record - pr, err := p.proposalRecordLatest(ctx, pi.PropStateUnvetted, cr.Token) + pr, err := p.proposalRecordLatest(ctx, piv1.PropStateUnvetted, cr.Token) if err != nil { return nil, err } - return &pi.ProposalNewReply{ + return &piv1.ProposalNewReply{ Proposal: *pr, }, nil } @@ -1254,7 +1255,7 @@ func (p *politeiawww) processProposalNew(ctx context.Context, pn pi.ProposalNew, // filesToDel returns the names of the files that are included in current but // are not included in updated. These are the files that need to be deleted // from a proposal on update. -func filesToDel(current []pi.File, updated []pi.File) []string { +func filesToDel(current []piv1.File, updated []piv1.File) []string { curr := make(map[string]struct{}, len(current)) // [name]struct for _, v := range updated { curr[v.Name] = struct{}{} @@ -1271,23 +1272,23 @@ func filesToDel(current []pi.File, updated []pi.File) []string { return del } -func (p *politeiawww) processProposalEdit(ctx context.Context, pe pi.ProposalEdit, usr user.User) (*pi.ProposalEditReply, error) { +func (p *politeiawww) processProposalEdit(ctx context.Context, pe piv1.ProposalEdit, usr user.User) (*piv1.ProposalEditReply, error) { log.Tracef("processProposalEdit: %v", pe.Token) // Verify token if !tokenIsFullLength(pe.Token) { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropTokenInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusPropTokenInvalid, } } // Verify state switch pe.State { - case pi.PropStateUnvetted, pi.PropStateVetted: + case piv1.PropStateUnvetted, piv1.PropStateVetted: // Allowed; continue default: - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropStateInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusPropStateInvalid, } } @@ -1300,8 +1301,8 @@ func (p *politeiawww) processProposalEdit(ctx context.Context, pe pi.ProposalEdi // Verify user signed using active identity if usr.PublicKey() != pe.PublicKey { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPublicKeyInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusPublicKeyInvalid, ErrorContext: []string{"not active identity"}, } } @@ -1310,8 +1311,8 @@ func (p *politeiawww) processProposalEdit(ctx context.Context, pe pi.ProposalEdi curr, err := p.proposalRecordLatest(ctx, pe.State, pe.Token) if err != nil { if errors.Is(err, errProposalNotFound) { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropNotFound, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusPropNotFound, } } return nil, err @@ -1320,19 +1321,19 @@ func (p *politeiawww) processProposalEdit(ctx context.Context, pe pi.ProposalEdi // Verify the user is the author. The public keys are not static // values so the user IDs must be compared directly. if curr.UserID != usr.ID.String() { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusUnauthorized, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusUnauthorized, ErrorContext: []string{"user is not author"}, } } // Verify proposal status switch curr.Status { - case pi.PropStatusUnreviewed, pi.PropStatusPublic: + case piv1.PropStatusUnreviewed, piv1.PropStatusPublic: // Allowed; continue default: - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropStatusInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusPropStatusInvalid, } } @@ -1348,7 +1349,7 @@ func (p *politeiawww) processProposalEdit(ctx context.Context, pe pi.ProposalEdi filesAdd := convertFilesFromPi(pe.Files) for _, v := range pe.Metadata { switch v.Hint { - case pi.HintProposalMetadata: + case piv1.HintProposalMetadata: filesAdd = append(filesAdd, convertFileFromMetadata(v)) } } @@ -1356,36 +1357,36 @@ func (p *politeiawww) processProposalEdit(ctx context.Context, pe pi.ProposalEdi // Setup politeiad metadata timestamp := time.Now().Unix() - gm := piplugin.GeneralMetadata{ + gm := pi.GeneralMetadata{ UserID: usr.ID.String(), PublicKey: pe.PublicKey, Signature: pe.Signature, Timestamp: timestamp, } - b, err := piplugin.EncodeGeneralMetadata(gm) + b, err := pi.EncodeGeneralMetadata(gm) if err != nil { return nil, err } - mdOverwrite := []pd.MetadataStream{ + mdOverwrite := []pdv1.MetadataStream{ { - ID: piplugin.MDStreamIDGeneralMetadata, + ID: pi.MDStreamIDGeneralMetadata, Payload: string(b), }, } - mdAppend := []pd.MetadataStream{} + mdAppend := []pdv1.MetadataStream{} // Send politeiad request // TODO verify that this will throw an error if no proposal files // were changed. - var r *pd.Record + var r *pdv1.Record switch pe.State { - case pi.PropStateUnvetted: + case piv1.PropStateUnvetted: r, err = p.updateUnvetted(ctx, pe.Token, mdAppend, mdOverwrite, filesAdd, filesDel) if err != nil { return nil, err } - case pi.PropStateVetted: + case piv1.PropStateVetted: r, err = p.updateVetted(ctx, pe.Token, mdAppend, mdOverwrite, filesAdd, filesDel) if err != nil { @@ -1415,12 +1416,12 @@ func (p *politeiawww) processProposalEdit(ctx context.Context, pe pi.ProposalEdi return nil, err } - return &pi.ProposalEditReply{ + return &piv1.ProposalEditReply{ Proposal: *pr, }, nil } -func (p *politeiawww) processProposalSetStatus(ctx context.Context, pss pi.ProposalSetStatus, usr user.User) (*pi.ProposalSetStatusReply, error) { +func (p *politeiawww) processProposalSetStatus(ctx context.Context, pss piv1.ProposalSetStatus, usr user.User) (*piv1.ProposalSetStatusReply, error) { log.Tracef("processProposalSetStatus: %v %v", pss.Token, pss.Status) // Sanity check @@ -1430,42 +1431,42 @@ func (p *politeiawww) processProposalSetStatus(ctx context.Context, pss pi.Propo // Verify token if !tokenIsFullLength(pss.Token) { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropTokenInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusPropTokenInvalid, } } // Verify state switch pss.State { - case pi.PropStateUnvetted, pi.PropStateVetted: + case piv1.PropStateUnvetted, piv1.PropStateVetted: // Allowed; continue default: - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropStateInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusPropStateInvalid, } } // Verify reason _, required := statusReasonRequired[pss.Status] if required && pss.Reason == "" { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropStatusChangeReasonInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusPropStatusChangeReasonInvalid, ErrorContext: []string{"reason not given"}, } } // Verify user is an admin if !usr.Admin { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusUnauthorized, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusUnauthorized, ErrorContext: []string{"user is not an admin"}, } } // Verify user signed with their active identity if usr.PublicKey() != pss.PublicKey { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPublicKeyInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusPublicKeyInvalid, ErrorContext: []string{"not active identity"}, } } @@ -1487,37 +1488,37 @@ func (p *politeiawww) processProposalSetStatus(ctx context.Context, pss pi.Propo // Setup metadata timestamp := time.Now().Unix() - sc := piplugin.StatusChange{ + sc := pi.StatusChange{ Token: pss.Token, Version: pss.Version, - Status: piplugin.PropStatusT(pss.Status), + Status: pi.PropStatusT(pss.Status), Reason: pss.Reason, PublicKey: pss.PublicKey, Signature: pss.Signature, Timestamp: timestamp, } - b, err := piplugin.EncodeStatusChange(sc) + b, err := pi.EncodeStatusChange(sc) if err != nil { return nil, err } - mdAppend := []pd.MetadataStream{ + mdAppend := []pdv1.MetadataStream{ { - ID: piplugin.MDStreamIDStatusChanges, + ID: pi.MDStreamIDStatusChanges, Payload: string(b), }, } - mdOverwrite := []pd.MetadataStream{} + mdOverwrite := []pdv1.MetadataStream{} // Send politeiad request - var r *pd.Record + var r *pdv1.Record status := convertRecordStatusFromPropStatus(pss.Status) switch pss.State { - case pi.PropStateUnvetted: + case piv1.PropStateUnvetted: r, err = p.setUnvettedStatus(ctx, pss.Token, status, mdAppend, mdOverwrite) if err != nil { return nil, err } - case pi.PropStateVetted: + case piv1.PropStateVetted: r, err = p.setVettedStatus(ctx, pss.Token, status, mdAppend, mdOverwrite) if err != nil { return nil, err @@ -1527,8 +1528,8 @@ func (p *politeiawww) processProposalSetStatus(ctx context.Context, pss pi.Propo // The proposal state will have changed if the proposal was made // public. state := pss.State - if pss.Status == pi.PropStatusPublic { - state = pi.PropStateVetted + if pss.Status == piv1.PropStatusPublic { + state = piv1.PropStateVetted } // Emit status change event @@ -1548,12 +1549,12 @@ func (p *politeiawww) processProposalSetStatus(ctx context.Context, pss pi.Propo return nil, err } - return &pi.ProposalSetStatusReply{ + return &piv1.ProposalSetStatusReply{ Proposal: *pr, }, nil } -func (p *politeiawww) processProposals(ctx context.Context, ps pi.Proposals, isAdmin bool) (*pi.ProposalsReply, error) { +func (p *politeiawww) processProposals(ctx context.Context, ps piv1.Proposals, isAdmin bool) (*piv1.ProposalsReply, error) { log.Tracef("processProposals: %v", ps.Requests) props, err := p.proposalRecords(ctx, ps.State, ps.Requests, ps.IncludeFiles) @@ -1566,25 +1567,25 @@ func (p *politeiawww) processProposals(ctx context.Context, ps pi.Proposals, isA // the user is not an admin. if !isAdmin { for k, v := range props { - if v.State == pi.PropStateVetted { + if v.State == piv1.PropStateVetted { continue } - v.Files = []pi.File{} - v.Metadata = []pi.Metadata{} + v.Files = []piv1.File{} + v.Metadata = []piv1.Metadata{} props[k] = v } } - return &pi.ProposalsReply{ + return &piv1.ProposalsReply{ Proposals: props, }, nil } -func (p *politeiawww) processProposalInventory(ctx context.Context, inv pi.ProposalInventory, u *user.User) (*pi.ProposalInventoryReply, error) { +func (p *politeiawww) processProposalInventory(ctx context.Context, inv piv1.ProposalInventory, u *user.User) (*piv1.ProposalInventoryReply, error) { log.Tracef("processProposalInventory: %v", inv.UserID) // Send plugin command - i := piplugin.ProposalInv{ + i := pi.ProposalInv{ UserID: inv.UserID, } pir, err := p.proposalInv(ctx, i) @@ -1603,46 +1604,46 @@ func (p *politeiawww) processProposalInventory(ctx context.Context, inv pi.Propo pir.Unvetted = nil } - return &pi.ProposalInventoryReply{ + return &piv1.ProposalInventoryReply{ Unvetted: pir.Unvetted, Vetted: pir.Vetted, }, nil } -func (p *politeiawww) processCommentNew(ctx context.Context, cn pi.CommentNew, usr user.User) (*pi.CommentNewReply, error) { +func (p *politeiawww) processCommentNew(ctx context.Context, cn piv1.CommentNew, usr user.User) (*piv1.CommentNewReply, error) { log.Tracef("processCommentNew: %v", usr.Username) // Verify user has paid registration paywall if !p.userHasPaid(usr) { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusUserRegistrationNotPaid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusUserRegistrationNotPaid, } } // Verify user signed using active identity if usr.PublicKey() != cn.PublicKey { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPublicKeyInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusPublicKeyInvalid, ErrorContext: []string{"not active identity"}, } } // Only admins and the proposal author are allowed to comment on // unvetted proposals. - if cn.State == pi.PropStateUnvetted && !usr.Admin { + if cn.State == piv1.PropStateUnvetted && !usr.Admin { // Fetch the proposal so we can see who the author is pr, err := p.proposalRecordLatest(ctx, cn.State, cn.Token) if err != nil { if errors.Is(err, errProposalNotFound) { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropNotFound, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusPropNotFound, } } return nil, fmt.Errorf("proposalRecordLatest: %v", err) } if usr.ID.String() != pr.UserID { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusUnauthorized, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusUnauthorized, ErrorContext: []string{"user is not author or admin"}, } } @@ -1662,7 +1663,7 @@ func (p *politeiawww) processCommentNew(ctx context.Context, cn pi.CommentNew, u if err != nil { return nil, err } - pt := piplugin.PassThrough{ + pt := pi.PassThrough{ PluginID: comments.ID, PluginCmd: comments.CmdNew, Payload: string(b), @@ -1690,33 +1691,33 @@ func (p *politeiawww) processCommentNew(ctx context.Context, cn pi.CommentNew, u username: c.Username, }) - return &pi.CommentNewReply{ + return &piv1.CommentNewReply{ Comment: c, }, nil } -func (p *politeiawww) processCommentVote(ctx context.Context, cv pi.CommentVote, usr user.User) (*pi.CommentVoteReply, error) { +func (p *politeiawww) processCommentVote(ctx context.Context, cv piv1.CommentVote, usr user.User) (*piv1.CommentVoteReply, error) { log.Tracef("processCommentVote: %v %v %v", cv.Token, cv.CommentID, cv.Vote) // Verify state - if cv.State != pi.PropStateVetted { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropStateInvalid, + if cv.State != piv1.PropStateVetted { + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusPropStateInvalid, ErrorContext: []string{"proposal must be vetted"}, } } // Verify user has paid registration paywall if !p.userHasPaid(usr) { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusUserRegistrationNotPaid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusUserRegistrationNotPaid, } } // Verify user signed using active identity if usr.PublicKey() != cv.PublicKey { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPublicKeyInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusPublicKeyInvalid, ErrorContext: []string{"not active identity"}, } } @@ -1735,7 +1736,7 @@ func (p *politeiawww) processCommentVote(ctx context.Context, cv pi.CommentVote, if err != nil { return nil, err } - pt := piplugin.PassThrough{ + pt := pi.PassThrough{ PluginID: comments.ID, PluginCmd: comments.CmdVote, Payload: string(b), @@ -1749,7 +1750,7 @@ func (p *politeiawww) processCommentVote(ctx context.Context, cv pi.CommentVote, return nil, err } - return &pi.CommentVoteReply{ + return &piv1.CommentVoteReply{ Downvotes: vr.Downvotes, Upvotes: vr.Upvotes, Timestamp: vr.Timestamp, @@ -1757,7 +1758,7 @@ func (p *politeiawww) processCommentVote(ctx context.Context, cv pi.CommentVote, }, nil } -func (p *politeiawww) processCommentCensor(ctx context.Context, cc pi.CommentCensor, usr user.User) (*pi.CommentCensorReply, error) { +func (p *politeiawww) processCommentCensor(ctx context.Context, cc piv1.CommentCensor, usr user.User) (*piv1.CommentCensorReply, error) { log.Tracef("processCommentCensor: %v %v", cc.Token, cc.CommentID) // Sanity check @@ -1767,8 +1768,8 @@ func (p *politeiawww) processCommentCensor(ctx context.Context, cc pi.CommentCen // Verify user signed with their active identity if usr.PublicKey() != cc.PublicKey { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPublicKeyInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusPublicKeyInvalid, ErrorContext: []string{"not active identity"}, } } @@ -1786,7 +1787,7 @@ func (p *politeiawww) processCommentCensor(ctx context.Context, cc pi.CommentCen if err != nil { return nil, err } - pt := piplugin.PassThrough{ + pt := pi.PassThrough{ PluginID: comments.ID, PluginCmd: comments.CmdDel, Payload: string(b), @@ -1804,18 +1805,18 @@ func (p *politeiawww) processCommentCensor(ctx context.Context, cc pi.CommentCen c := convertCommentFromPlugin(dr.Comment) c = commentFillInUser(c, usr) - return &pi.CommentCensorReply{ + return &piv1.CommentCensorReply{ Comment: c, }, nil } -func (p *politeiawww) processComments(ctx context.Context, c pi.Comments, usr *user.User) (*pi.CommentsReply, error) { +func (p *politeiawww) processComments(ctx context.Context, c piv1.Comments, usr *user.User) (*piv1.CommentsReply, error) { log.Tracef("processComments: %v", c.Token) // Only admins and the proposal author are allowed to retrieve // unvetted comments. This is a public route so a user might not // exist. - if c.State == pi.PropStateUnvetted { + if c.State == piv1.PropStateUnvetted { var isAllowed bool switch { case usr == nil: @@ -1829,8 +1830,8 @@ func (p *politeiawww) processComments(ctx context.Context, c pi.Comments, usr *u pr, err := p.proposalRecordLatest(ctx, c.State, c.Token) if err != nil { if errors.Is(err, errProposalNotFound) { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropNotFound, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusPropNotFound, } } return nil, fmt.Errorf("proposalRecordLatest: %v", err) @@ -1841,8 +1842,8 @@ func (p *politeiawww) processComments(ctx context.Context, c pi.Comments, usr *u } } if !isAllowed { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusUnauthorized, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusUnauthorized, ErrorContext: []string{"user is not author or admin"}, } } @@ -1859,7 +1860,7 @@ func (p *politeiawww) processComments(ctx context.Context, c pi.Comments, usr *u // Prepare reply. Comments contain user data that needs to be // pulled from the user database. - cs := make([]pi.Comment, 0, len(reply.Comments)) + cs := make([]piv1.Comment, 0, len(reply.Comments)) for _, cm := range reply.Comments { // Convert comment pic := convertCommentFromPlugin(cm) @@ -1879,18 +1880,18 @@ func (p *politeiawww) processComments(ctx context.Context, c pi.Comments, usr *u cs = append(cs, pic) } - return &pi.CommentsReply{ + return &piv1.CommentsReply{ Comments: cs, }, nil } -func (p *politeiawww) processCommentVotes(ctx context.Context, cv pi.CommentVotes) (*pi.CommentVotesReply, error) { +func (p *politeiawww) processCommentVotes(ctx context.Context, cv piv1.CommentVotes) (*piv1.CommentVotesReply, error) { log.Tracef("processCommentVotes: %v %v", cv.Token, cv.UserID) // Verify state - if cv.State != pi.PropStateVetted { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPropStateInvalid, + if cv.State != piv1.PropStateVetted { + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusPropStateInvalid, ErrorContext: []string{"proposal must be vetted"}, } } @@ -1906,18 +1907,18 @@ func (p *politeiawww) processCommentVotes(ctx context.Context, cv pi.CommentVote return nil, err } - return &pi.CommentVotesReply{ + return &piv1.CommentVotesReply{ Votes: convertCommentVoteDetailsFromPlugin(cvr.Votes), }, nil } -func (p *politeiawww) processVoteAuthorize(ctx context.Context, va pi.VoteAuthorize, usr user.User) (*pi.VoteAuthorizeReply, error) { +func (p *politeiawww) processVoteAuthorize(ctx context.Context, va piv1.VoteAuthorize, usr user.User) (*piv1.VoteAuthorizeReply, error) { log.Tracef("processVoteAuthorize: %v", va.Token) // Verify user signed with their active identity if usr.PublicKey() != va.PublicKey { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPublicKeyInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusPublicKeyInvalid, ErrorContext: []string{"not active identity"}, } } @@ -1933,13 +1934,13 @@ func (p *politeiawww) processVoteAuthorize(ctx context.Context, va pi.VoteAuthor // TODO Emit notification - return &pi.VoteAuthorizeReply{ + return &piv1.VoteAuthorizeReply{ Timestamp: ar.Timestamp, Receipt: ar.Receipt, }, nil } -func (p *politeiawww) processVoteStart(ctx context.Context, vs pi.VoteStart, usr user.User) (*pi.VoteStartReply, error) { +func (p *politeiawww) processVoteStart(ctx context.Context, vs piv1.VoteStart, usr user.User) (*piv1.VoteStartReply, error) { log.Tracef("processVoteStart: %v", len(vs.Starts)) // Sanity check @@ -1949,8 +1950,8 @@ func (p *politeiawww) processVoteStart(ctx context.Context, vs pi.VoteStart, usr // Verify there is work to be done if len(vs.Starts) == 0 { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusStartDetailsInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusStartDetailsInvalid, ErrorContext: []string{"no start details found"}, } } @@ -1958,8 +1959,8 @@ func (p *politeiawww) processVoteStart(ctx context.Context, vs pi.VoteStart, usr // Verify admin signed with their active identity for _, v := range vs.Starts { if usr.PublicKey() != v.PublicKey { - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusPublicKeyInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusPublicKeyInvalid, ErrorContext: []string{"not active identity"}, } } @@ -1971,7 +1972,7 @@ func (p *politeiawww) processVoteStart(ctx context.Context, vs pi.VoteStart, usr err error ) switch vs.Starts[0].Params.Type { - case pi.VoteTypeStandard: + case piv1.VoteTypeStandard: // A standard vote can be started using the ticketvote plugin // directly. sr, err = p.voteStart(ctx, convertVoteStartFromPi(vs)) @@ -1979,7 +1980,7 @@ func (p *politeiawww) processVoteStart(ctx context.Context, vs pi.VoteStart, usr return nil, err } - case pi.VoteTypeRunoff: + case piv1.VoteTypeRunoff: // A runoff vote requires additional validation that is not part // of the ticketvote plugin. We pass the ticketvote command // through the pi plugin so that it can perform this additional @@ -1989,7 +1990,7 @@ func (p *politeiawww) processVoteStart(ctx context.Context, vs pi.VoteStart, usr if err != nil { return nil, err } - pt := piplugin.PassThrough{ + pt := pi.PassThrough{ PluginID: ticketvote.ID, PluginCmd: ticketvote.CmdStart, Payload: string(payload), @@ -2004,14 +2005,14 @@ func (p *politeiawww) processVoteStart(ctx context.Context, vs pi.VoteStart, usr } default: - return nil, pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusVoteTypeInvalid, + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusVoteTypeInvalid, } } // TODO Emit notification for each start - return &pi.VoteStartReply{ + return &piv1.VoteStartReply{ StartBlockHeight: sr.StartBlockHeight, StartBlockHash: sr.StartBlockHash, EndBlockHeight: sr.EndBlockHeight, @@ -2019,7 +2020,7 @@ func (p *politeiawww) processVoteStart(ctx context.Context, vs pi.VoteStart, usr }, nil } -func (p *politeiawww) processCastBallot(ctx context.Context, vc pi.CastBallot) (*pi.CastBallotReply, error) { +func (p *politeiawww) processCastBallot(ctx context.Context, vc piv1.CastBallot) (*piv1.CastBallotReply, error) { log.Tracef("processCastBallot") cb := ticketvote.CastBallot{ @@ -2030,12 +2031,12 @@ func (p *politeiawww) processCastBallot(ctx context.Context, vc pi.CastBallot) ( return nil, err } - return &pi.CastBallotReply{ + return &piv1.CastBallotReply{ Receipts: convertCastVoteRepliesFromPlugin(reply.Receipts), }, nil } -func (p *politeiawww) processVotes(ctx context.Context, v pi.Votes) (*pi.VotesReply, error) { +func (p *politeiawww) processVotes(ctx context.Context, v piv1.Votes) (*piv1.VotesReply, error) { log.Tracef("processVotes: %v", v.Tokens) vd, err := p.voteDetails(ctx, v.Tokens) @@ -2043,12 +2044,12 @@ func (p *politeiawww) processVotes(ctx context.Context, v pi.Votes) (*pi.VotesRe return nil, err } - return &pi.VotesReply{ + return &piv1.VotesReply{ Votes: convertProposalVotesFromPlugin(vd.Votes), }, nil } -func (p *politeiawww) processVoteResults(ctx context.Context, vr pi.VoteResults) (*pi.VoteResultsReply, error) { +func (p *politeiawww) processVoteResults(ctx context.Context, vr piv1.VoteResults) (*piv1.VoteResultsReply, error) { log.Tracef("processVoteResults: %v", vr.Token) cvr, err := p.voteResults(ctx, vr.Token) @@ -2056,12 +2057,12 @@ func (p *politeiawww) processVoteResults(ctx context.Context, vr pi.VoteResults) return nil, err } - return &pi.VoteResultsReply{ + return &piv1.VoteResultsReply{ Votes: convertCastVoteDetailsFromPlugin(cvr.Votes), }, nil } -func (p *politeiawww) processVoteSummaries(ctx context.Context, vs pi.VoteSummaries) (*pi.VoteSummariesReply, error) { +func (p *politeiawww) processVoteSummaries(ctx context.Context, vs piv1.VoteSummaries) (*piv1.VoteSummariesReply, error) { log.Tracef("processVoteSummaries: %v", vs.Tokens) r, err := p.voteSummaries(ctx, vs.Tokens) @@ -2069,13 +2070,13 @@ func (p *politeiawww) processVoteSummaries(ctx context.Context, vs pi.VoteSummar return nil, err } - return &pi.VoteSummariesReply{ + return &piv1.VoteSummariesReply{ Summaries: convertVoteSummariesFromPlugin(r.Summaries), BestBlock: r.BestBlock, }, nil } -func (p *politeiawww) processVoteInventory(ctx context.Context) (*pi.VoteInventoryReply, error) { +func (p *politeiawww) processVoteInventory(ctx context.Context) (*piv1.VoteInventoryReply, error) { log.Tracef("processVoteInventory") r, err := p.piVoteInventory(ctx) @@ -2083,7 +2084,7 @@ func (p *politeiawww) processVoteInventory(ctx context.Context) (*pi.VoteInvento return nil, err } - return &pi.VoteInventoryReply{ + return &piv1.VoteInventoryReply{ Unauthorized: r.Unauthorized, Authorized: r.Authorized, Started: r.Started, @@ -2096,12 +2097,12 @@ func (p *politeiawww) processVoteInventory(ctx context.Context) (*pi.VoteInvento func (p *politeiawww) handleProposalNew(w http.ResponseWriter, r *http.Request) { log.Tracef("handleProposalNew") - var pn pi.ProposalNew + var pn piv1.ProposalNew decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&pn); err != nil { respondWithPiError(w, r, "handleProposalNew: unmarshal", - pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInputInvalid, + piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusInputInvalid, }) return } @@ -2126,12 +2127,12 @@ func (p *politeiawww) handleProposalNew(w http.ResponseWriter, r *http.Request) func (p *politeiawww) handleProposalEdit(w http.ResponseWriter, r *http.Request) { log.Tracef("handleProposalEdit") - var pe pi.ProposalEdit + var pe piv1.ProposalEdit decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&pe); err != nil { respondWithPiError(w, r, "handleProposalEdit: unmarshal", - pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInputInvalid, + piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusInputInvalid, }) return } @@ -2156,12 +2157,12 @@ func (p *politeiawww) handleProposalEdit(w http.ResponseWriter, r *http.Request) func (p *politeiawww) handleProposalSetStatus(w http.ResponseWriter, r *http.Request) { log.Tracef("handleProposalSetStatus") - var pss pi.ProposalSetStatus + var pss piv1.ProposalSetStatus decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&pss); err != nil { respondWithPiError(w, r, "handleProposalSetStatus: unmarshal", - pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInputInvalid, + piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusInputInvalid, }) return } @@ -2186,12 +2187,12 @@ func (p *politeiawww) handleProposalSetStatus(w http.ResponseWriter, r *http.Req func (p *politeiawww) handleProposals(w http.ResponseWriter, r *http.Request) { log.Tracef("handleProposals") - var ps pi.Proposals + var ps piv1.Proposals decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&ps); err != nil { respondWithPiError(w, r, "handleProposals: unmarshal", - pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInputInvalid, + piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusInputInvalid, }) return } @@ -2219,12 +2220,12 @@ func (p *politeiawww) handleProposals(w http.ResponseWriter, r *http.Request) { func (p *politeiawww) handleProposalInventory(w http.ResponseWriter, r *http.Request) { log.Tracef("handleProposalInventory") - var inv pi.ProposalInventory + var inv piv1.ProposalInventory decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&inv); err != nil { respondWithPiError(w, r, "handleProposalInventory: unmarshal", - pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInputInvalid, + piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusInputInvalid, }) return } @@ -2251,12 +2252,12 @@ func (p *politeiawww) handleProposalInventory(w http.ResponseWriter, r *http.Req func (p *politeiawww) handleCommentNew(w http.ResponseWriter, r *http.Request) { log.Tracef("handleCommentNew") - var cn pi.CommentNew + var cn piv1.CommentNew decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&cn); err != nil { respondWithPiError(w, r, "handleCommentNew: unmarshal", - pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInputInvalid, + piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusInputInvalid, }) return } @@ -2281,12 +2282,12 @@ func (p *politeiawww) handleCommentNew(w http.ResponseWriter, r *http.Request) { func (p *politeiawww) handleCommentVote(w http.ResponseWriter, r *http.Request) { log.Tracef("handleCommentVote") - var cv pi.CommentVote + var cv piv1.CommentVote decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&cv); err != nil { respondWithPiError(w, r, "handleCommentVote: unmarshal", - pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInputInvalid, + piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusInputInvalid, }) return } @@ -2311,12 +2312,12 @@ func (p *politeiawww) handleCommentVote(w http.ResponseWriter, r *http.Request) func (p *politeiawww) handleCommentCensor(w http.ResponseWriter, r *http.Request) { log.Tracef("handleCommentCensor") - var cc pi.CommentCensor + var cc piv1.CommentCensor decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&cc); err != nil { respondWithPiError(w, r, "handleCommentCensor: unmarshal", - pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInputInvalid, + piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusInputInvalid, }) return } @@ -2341,12 +2342,12 @@ func (p *politeiawww) handleCommentCensor(w http.ResponseWriter, r *http.Request func (p *politeiawww) handleComments(w http.ResponseWriter, r *http.Request) { log.Tracef("handleComments") - var c pi.Comments + var c piv1.Comments decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&c); err != nil { respondWithPiError(w, r, "handleComments: unmarshal", - pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInputInvalid, + piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusInputInvalid, }) return } @@ -2373,12 +2374,12 @@ func (p *politeiawww) handleComments(w http.ResponseWriter, r *http.Request) { func (p *politeiawww) handleCommentVotes(w http.ResponseWriter, r *http.Request) { log.Tracef("handleCommentVotes") - var cv pi.CommentVotes + var cv piv1.CommentVotes decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&cv); err != nil { respondWithPiError(w, r, "handleCommentVotes: unmarshal", - pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInputInvalid, + piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusInputInvalid, }) return } @@ -2396,12 +2397,12 @@ func (p *politeiawww) handleCommentVotes(w http.ResponseWriter, r *http.Request) func (p *politeiawww) handleVoteAuthorize(w http.ResponseWriter, r *http.Request) { log.Tracef("handleVoteAuthorize") - var va pi.VoteAuthorize + var va piv1.VoteAuthorize decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&va); err != nil { respondWithPiError(w, r, "handleVoteAuthorize: unmarshal", - pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInputInvalid, + piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusInputInvalid, }) return } @@ -2426,12 +2427,12 @@ func (p *politeiawww) handleVoteAuthorize(w http.ResponseWriter, r *http.Request func (p *politeiawww) handleVoteStart(w http.ResponseWriter, r *http.Request) { log.Tracef("handleVoteStart") - var vs pi.VoteStart + var vs piv1.VoteStart decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&vs); err != nil { respondWithPiError(w, r, "handleVoteStart: unmarshal", - pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInputInvalid, + piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusInputInvalid, }) return } @@ -2456,12 +2457,12 @@ func (p *politeiawww) handleVoteStart(w http.ResponseWriter, r *http.Request) { func (p *politeiawww) handleCastBallot(w http.ResponseWriter, r *http.Request) { log.Tracef("handleCastBallot") - var vb pi.CastBallot + var vb piv1.CastBallot decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&vb); err != nil { respondWithPiError(w, r, "handleCastBallot: unmarshal", - pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInputInvalid, + piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusInputInvalid, }) return } @@ -2479,12 +2480,12 @@ func (p *politeiawww) handleCastBallot(w http.ResponseWriter, r *http.Request) { func (p *politeiawww) handleVotes(w http.ResponseWriter, r *http.Request) { log.Tracef("handleVotes") - var v pi.Votes + var v piv1.Votes decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&v); err != nil { respondWithPiError(w, r, "handleVotes: unmarshal", - pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInputInvalid, + piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusInputInvalid, }) return } @@ -2502,12 +2503,12 @@ func (p *politeiawww) handleVotes(w http.ResponseWriter, r *http.Request) { func (p *politeiawww) handleVoteResults(w http.ResponseWriter, r *http.Request) { log.Tracef("handleVoteResults") - var vr pi.VoteResults + var vr piv1.VoteResults decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&vr); err != nil { respondWithPiError(w, r, "handleVoteResults: unmarshal", - pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInputInvalid, + piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusInputInvalid, }) return } @@ -2525,12 +2526,12 @@ func (p *politeiawww) handleVoteResults(w http.ResponseWriter, r *http.Request) func (p *politeiawww) handleVoteSummaries(w http.ResponseWriter, r *http.Request) { log.Tracef("handleVoteSummaries") - var vs pi.VoteSummaries + var vs piv1.VoteSummaries decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&vs); err != nil { respondWithPiError(w, r, "handleVoteSummaries: unmarshal", - pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInputInvalid, + piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusInputInvalid, }) return } @@ -2548,12 +2549,12 @@ func (p *politeiawww) handleVoteSummaries(w http.ResponseWriter, r *http.Request func (p *politeiawww) handleVoteInventory(w http.ResponseWriter, r *http.Request) { log.Tracef("handleVoteInventory") - var vi pi.VoteInventory + var vi piv1.VoteInventory decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&vi); err != nil { respondWithPiError(w, r, "handleVoteInventory: unmarshal", - pi.UserErrorReply{ - ErrorCode: pi.ErrorStatusInputInvalid, + piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusInputInvalid, }) return } @@ -2571,59 +2572,59 @@ func (p *politeiawww) handleVoteInventory(w http.ResponseWriter, r *http.Request // setPiRoutes sets the pi API routes. func (p *politeiawww) setPiRoutes() { // Proposal routes - p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteProposalNew, p.handleProposalNew, + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteProposalNew, p.handleProposalNew, permissionLogin) - p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteProposalEdit, p.handleProposalEdit, + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteProposalEdit, p.handleProposalEdit, permissionLogin) - p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteProposalSetStatus, p.handleProposalSetStatus, + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteProposalSetStatus, p.handleProposalSetStatus, permissionAdmin) - p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteProposals, p.handleProposals, + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteProposals, p.handleProposals, permissionPublic) - p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteProposalInventory, p.handleProposalInventory, + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteProposalInventory, p.handleProposalInventory, permissionPublic) // Comment routes - p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteCommentNew, p.handleCommentNew, + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteCommentNew, p.handleCommentNew, permissionLogin) - p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteCommentVote, p.handleCommentVote, + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteCommentVote, p.handleCommentVote, permissionLogin) - p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteCommentCensor, p.handleCommentCensor, + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteCommentCensor, p.handleCommentCensor, permissionAdmin) - p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteComments, p.handleComments, + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteComments, p.handleComments, permissionPublic) - p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteCommentVotes, p.handleCommentVotes, + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteCommentVotes, p.handleCommentVotes, permissionPublic) // Vote routes - p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteVoteAuthorize, p.handleVoteAuthorize, + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteVoteAuthorize, p.handleVoteAuthorize, permissionLogin) - p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteVoteStart, p.handleVoteStart, + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteVoteStart, p.handleVoteStart, permissionAdmin) - p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteCastBallot, p.handleCastBallot, + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteCastBallot, p.handleCastBallot, permissionPublic) - p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteVotes, p.handleVotes, + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteVotes, p.handleVotes, permissionPublic) - p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteVoteResults, p.handleVoteResults, + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteVoteResults, p.handleVoteResults, permissionPublic) - p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteVoteSummaries, p.handleVoteSummaries, + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteVoteSummaries, p.handleVoteSummaries, permissionPublic) - p.addRoute(http.MethodPost, pi.APIRoute, - pi.RouteVoteInventory, p.handleVoteInventory, + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteVoteInventory, p.handleVoteInventory, permissionPublic) } From b423d33c9738b37022dc7f46333ef6ac5af7d87c Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 4 Jan 2021 09:29:18 -0600 Subject: [PATCH 199/449] tlogbe: Vote summary bug fix. --- politeiad/backend/tlogbe/ticketvote.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index a386bc59d..996d4239b 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -2349,6 +2349,8 @@ func voteIsApproved(vd ticketvote.VoteDetails, results []ticketvote.VoteOptionRe return approved } +// summary returns the vote summary for a record. If a vetted record does not +// exist for the token a errRecordNotFound error is returned. func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote.Summary, error) { // Check if the summary has been cached s, err := p.cachedSummary(hex.EncodeToString(token)) @@ -2372,6 +2374,11 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. // Check if the vote has been authorized auths, err := p.authorizes(token) if err != nil { + if errors.Is(err, errRecordNotFound) { + // Let the calling function decide how to handle when a vetted + // record does not exist for the token. + return nil, err + } return nil, fmt.Errorf("authorizes: %v", err) } if len(auths) > 0 { From 7907488cbe9be061bf0cce6c3f9a583516f153ec Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 5 Jan 2021 10:10:03 -0600 Subject: [PATCH 200/449] CleanAndExpandPath fix --- politeiad/config.go | 67 ++++----------------------------------------- 1 file changed, 6 insertions(+), 61 deletions(-) diff --git a/politeiad/config.go b/politeiad/config.go index 7cdbdd44e..8f8f4a702 100644 --- a/politeiad/config.go +++ b/politeiad/config.go @@ -11,7 +11,6 @@ import ( "fmt" "net" "os" - "os/user" "path/filepath" "runtime" "sort" @@ -106,60 +105,6 @@ type serviceOptions struct { ServiceCommand string `short:"s" long:"service" description:"Service command {install, remove, start, stop}"` } -// cleanAndExpandPath expands environment variables and leading ~ in the -// passed path, cleans the result, and returns it. -func cleanAndExpandPath(path string) string { - // Nothing to do when no path is given. - if path == "" { - return path - } - - // NOTE: The os.ExpandEnv doesn't work with Windows cmd.exe-style - // %VARIABLE%, but the variables can still be expanded via POSIX-style - // $VARIABLE. - path = os.ExpandEnv(path) - - if !strings.HasPrefix(path, "~") { - return filepath.Clean(path) - } - - // Expand initial ~ to the current user's home directory, or ~otheruser - // to otheruser's home directory. On Windows, both forward and backward - // slashes can be used. - path = path[1:] - - var pathSeparators string - if runtime.GOOS == "windows" { - pathSeparators = string(os.PathSeparator) + "/" - } else { - pathSeparators = string(os.PathSeparator) - } - - userName := "" - if i := strings.IndexAny(path, pathSeparators); i != -1 { - userName = path[:i] - path = path[i:] - } - - homeDir := "" - var u *user.User - var err error - if userName == "" { - u, err = user.Current() - } else { - u, err = user.Lookup(userName) - } - if err == nil { - homeDir = u.HomeDir - } - // Fallback to CWD if user lookup fails or user has no home directory. - if homeDir == "" { - homeDir = "." - } - - return filepath.Join(homeDir, path) -} - // validLogLevel returns whether or not logLevel is a valid debug log level. func validLogLevel(logLevel string) bool { switch logLevel { @@ -458,16 +403,16 @@ func loadConfig() (*config, []string, error) { // All data is specific to a network, so namespacing the data directory // means each individual piece of serialized data does not have to // worry about changing names per network and such. - cfg.DataDir = cleanAndExpandPath(cfg.DataDir) + cfg.DataDir = util.CleanAndExpandPath(cfg.DataDir) cfg.DataDir = filepath.Join(cfg.DataDir, netName(activeNetParams)) // Append the network type to the log directory so it is "namespaced" // per network in the same fashion as the data directory. - cfg.LogDir = cleanAndExpandPath(cfg.LogDir) + cfg.LogDir = util.CleanAndExpandPath(cfg.LogDir) cfg.LogDir = filepath.Join(cfg.LogDir, netName(activeNetParams)) - cfg.HTTPSKey = cleanAndExpandPath(cfg.HTTPSKey) - cfg.HTTPSCert = cleanAndExpandPath(cfg.HTTPSCert) + cfg.HTTPSKey = util.CleanAndExpandPath(cfg.HTTPSKey) + cfg.HTTPSCert = util.CleanAndExpandPath(cfg.HTTPSCert) // Special show command to list supported subsystems and exit. if cfg.DebugLevel == "show" { @@ -543,7 +488,7 @@ func loadConfig() (*config, []string, error) { cfg.DcrtimeHost = "https://" + cfg.DcrtimeHost if len(cfg.DcrtimeCert) != 0 && !util.FileExists(cfg.DcrtimeCert) { - cfg.DcrtimeCert = cleanAndExpandPath(cfg.DcrtimeCert) + cfg.DcrtimeCert = util.CleanAndExpandPath(cfg.DcrtimeCert) path := filepath.Join(cfg.HomeDir, cfg.DcrtimeCert) if !util.FileExists(path) { str := "%s: dcrtimecert " + cfg.DcrtimeCert + " and " + @@ -559,7 +504,7 @@ func loadConfig() (*config, []string, error) { if cfg.Identity == "" { cfg.Identity = defaultIdentityFile } - cfg.Identity = cleanAndExpandPath(cfg.Identity) + cfg.Identity = util.CleanAndExpandPath(cfg.Identity) // Set random username and password when not specified if cfg.RPCUser == "" { From 269eea9091179194b6bf8133bac032142670d02d Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 5 Jan 2021 13:56:06 -0600 Subject: [PATCH 201/449] tlogbe: Add a proper dcrtime client. --- politeiad/backend/tlogbe/anchor.go | 4 +- politeiad/backend/tlogbe/dcrdata.go | 2 +- politeiad/backend/tlogbe/dcrtime.go | 140 ++++++++++++++++------------ politeiad/backend/tlogbe/testing.go | 4 - politeiad/backend/tlogbe/tlog.go | 22 +++-- politeiad/backend/tlogbe/tlogbe.go | 11 ++- politeiad/cmd/politeia/politeia.go | 22 ++--- politeiad/politeiad.go | 62 ++++++------ politeiawww/cmswww.go | 2 +- politeiawww/www.go | 2 +- util/identity.go | 4 +- util/net.go | 8 +- 12 files changed, 150 insertions(+), 133 deletions(-) diff --git a/politeiad/backend/tlogbe/anchor.go b/politeiad/backend/tlogbe/anchor.go index 23182a71d..19472f03c 100644 --- a/politeiad/backend/tlogbe/anchor.go +++ b/politeiad/backend/tlogbe/anchor.go @@ -219,7 +219,7 @@ func (t *tlog) anchorWait(anchors []anchor, hashes []string) { log.Debugf("Verify %v anchor attempt %v/%v", t.id, try+1, retries) - vbr, err := verifyBatch(t.dcrtimeHost, anchorID, hashes) + vbr, err := t.dcrtime.verifyBatch(anchorID, hashes) if err != nil { exitErr = fmt.Errorf("verifyBatch: %v", err) return @@ -421,7 +421,7 @@ func (t *tlog) anchor() { // Submit dcrtime anchor request log.Infof("Anchoring %v %v trees", len(anchors), t.id) - tbr, err := timestampBatch(t.dcrtimeHost, anchorID, digests) + tbr, err := t.dcrtime.timestampBatch(anchorID, digests) if err != nil { exitErr = fmt.Errorf("timestampBatch: %v", err) return diff --git a/politeiad/backend/tlogbe/dcrdata.go b/politeiad/backend/tlogbe/dcrdata.go index d0f97f6bd..01602e177 100644 --- a/politeiad/backend/tlogbe/dcrdata.go +++ b/politeiad/backend/tlogbe/dcrdata.go @@ -668,7 +668,7 @@ func newDcrdataPlugin(settings []backend.PluginSetting, activeNetParams *chaincf // Setup http client log.Infof("Dcrdata HTTP host: %v", hostHTTP) - client, err := util.NewClient(false, "") + client, err := util.NewHTTPClient(false, "") if err != nil { return nil, err } diff --git a/politeiad/backend/tlogbe/dcrtime.go b/politeiad/backend/tlogbe/dcrtime.go index 396d27f1a..d3edb74dd 100644 --- a/politeiad/backend/tlogbe/dcrtime.go +++ b/politeiad/backend/tlogbe/dcrtime.go @@ -6,68 +6,59 @@ package tlogbe import ( "bytes" - "crypto/tls" "encoding/hex" "encoding/json" "errors" "fmt" "net/http" - "time" dcrtime "github.com/decred/dcrtime/api/v2" "github.com/decred/dcrtime/merkle" "github.com/decred/politeia/util" ) -var ( - httpClient = &http.Client{ - Timeout: 1 * time.Minute, - Transport: &http.Transport{ - IdleConnTimeout: 1 * time.Minute, - ResponseHeaderTimeout: 1 * time.Minute, - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: false, - }, - }, - } -) +// dcrtimeClient is a client for interacting with the dcrtime API. +type dcrtimeClient struct { + host string + certPath string + http *http.Client +} -// isDigest returns whether the provided digest is a valid SHA256 digest. -func isDigest(digest string) bool { +// isDigestSHA256 returns whether the provided digest is a valid SHA256 digest. +func isDigestSHA256(digest string) bool { return dcrtime.RegexpSHA256.MatchString(digest) } -// timestampBatch posts the provided digests to the dcrtime v2 batch timestamp -// route. -func timestampBatch(host, id string, digests []string) (*dcrtime.TimestampBatchReply, error) { - log.Tracef("timestampBatch: %v %v %v", host, id, digests) - - // Validate digests - for _, v := range digests { - if !isDigest(v) { - return nil, fmt.Errorf("invalid digest: %v", v) +// makeReq makes an http request to a dcrtime method and route, serializing the +// provided object as the request body. The response body is returned as a byte +// slice. +func (c *dcrtimeClient) makeReq(method string, route string, v interface{}) ([]byte, error) { + var ( + reqBody []byte + err error + ) + if v != nil { + reqBody, err = json.Marshal(v) + if err != nil { + return nil, err } } - // Setup request - tb := dcrtime.TimestampBatch{ - ID: id, - Digests: digests, - } - b, err := json.Marshal(tb) + fullRoute := c.host + route + + log.Tracef("%v %v", method, fullRoute) + + req, err := http.NewRequest(method, fullRoute, bytes.NewReader(reqBody)) if err != nil { return nil, err } - // Send request - route := host + dcrtime.TimestampBatchRoute - r, err := httpClient.Post(route, "application/json", bytes.NewReader(b)) + r, err := c.http.Do(req) if err != nil { return nil, err } defer r.Body.Close() - // Handle response if r.StatusCode != http.StatusOK { e, err := util.GetErrorFromJSON(r.Body) if err != nil { @@ -75,10 +66,37 @@ func timestampBatch(host, id string, digests []string) (*dcrtime.TimestampBatchR } return nil, fmt.Errorf("%v: %v", r.Status, e) } + + respBody := util.ConvertBodyToByteArray(r.Body, false) + return respBody, nil +} + +// timestampBatch posts digests to the dcrtime v2 batch timestamp route. +func (c *dcrtimeClient) timestampBatch(id string, digests []string) (*dcrtime.TimestampBatchReply, error) { + log.Tracef("timestampBatch: %v %v", id, digests) + + // Setup request + for _, v := range digests { + if !isDigestSHA256(v) { + return nil, fmt.Errorf("invalid digest: %v", v) + } + } + tb := dcrtime.TimestampBatch{ + ID: id, + Digests: digests, + } + + // Send request + respBody, err := c.makeReq(http.MethodPost, dcrtime.TimestampBatchRoute, tb) + if err != nil { + return nil, err + } + + // Decode reply var tbr dcrtime.TimestampBatchReply - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&tbr); err != nil { - return nil, fmt.Errorf("decode TimestampBatchReply: %v", err) + err = json.Unmarshal(respBody, &tbr) + if err != nil { + return nil, err } return &tbr, nil @@ -94,46 +112,31 @@ func timestampBatch(host, id string, digests []string) (*dcrtime.TimestampBatchR // once the digest has been included in a dcr transaction, except for the // ChainTimestamp field. The ChainTimestamp field is only populated once the // dcr transaction has 6 confirmations. -func verifyBatch(host, id string, digests []string) (*dcrtime.VerifyBatchReply, error) { - log.Tracef("verifyBatch: %v %v %v", host, id, digests) +func (c *dcrtimeClient) verifyBatch(id string, digests []string) (*dcrtime.VerifyBatchReply, error) { + log.Tracef("verifyBatch: %v %v", id, digests) - // Validate digests + // Setup request for _, v := range digests { - if !isDigest(v) { + if !isDigestSHA256(v) { return nil, fmt.Errorf("invalid digest: %v", v) } } - - // Setup request vb := dcrtime.VerifyBatch{ ID: id, Digests: digests, } - b, err := json.Marshal(vb) - if err != nil { - return nil, err - } // Send request - route := host + dcrtime.VerifyBatchRoute - r, err := httpClient.Post(route, "application/json", bytes.NewReader(b)) + respBody, err := c.makeReq(http.MethodPost, dcrtime.VerifyBatchRoute, vb) if err != nil { return nil, err } - defer r.Body.Close() - // Handle response - if r.StatusCode != http.StatusOK { - e, err := util.GetErrorFromJSON(r.Body) - if err != nil { - return nil, fmt.Errorf("%v", r.Status) - } - return nil, fmt.Errorf("%v: %v", r.Status, e) - } + // Decode reply var vbr dcrtime.VerifyBatchReply - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&vbr); err != nil { - return nil, fmt.Errorf("decode VerifyBatchReply: %v", err) + err = json.Unmarshal(respBody, &vbr) + if err != nil { + return nil, err } // Verify the merkle path and the merkle root of the timestamps @@ -169,3 +172,16 @@ func verifyBatch(host, id string, digests []string) (*dcrtime.VerifyBatchReply, return &vbr, nil } + +// newDcrtimeClient returns a new dcrtimeClient. +func newDcrtimeClient(host, certPath string) (*dcrtimeClient, error) { + c, err := util.NewHTTPClient(false, certPath) + if err != nil { + return nil, err + } + return &dcrtimeClient{ + host: host, + certPath: certPath, + http: c, + }, nil +} diff --git a/politeiad/backend/tlogbe/testing.go b/politeiad/backend/tlogbe/testing.go index f9aa7dd4f..e2d7d670b 100644 --- a/politeiad/backend/tlogbe/testing.go +++ b/politeiad/backend/tlogbe/testing.go @@ -17,13 +17,11 @@ import ( "testing" "github.com/decred/dcrd/dcrutil/v3" - dcrtime "github.com/decred/dcrtime/api/v1" v1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/mime" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" "github.com/decred/politeia/util" - "github.com/robfig/cron" ) var ( @@ -127,11 +125,9 @@ func newTestTlog(t *testing.T, id string) (*tlog, error) { tlog := tlog{ id: id, - dcrtimeHost: dcrtime.DefaultTestnetTimeHost, encryptionKey: nil, trillian: tclient, store: store, - cron: cron.New(), } return &tlog, nil diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index a65770be9..64034cdea 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -82,11 +82,11 @@ var ( // We do not unwind. type tlog struct { sync.Mutex - id string - dcrtimeHost string - trillian trillianClient - store store.Blob - cron *cron.Cron + id string + trillian trillianClient + store store.Blob + dcrtime *dcrtimeClient + cron *cron.Cron // encryptionKey is used to encrypt record blobs before saving them // to the key-value store. This is an optional param. Record blobs @@ -1842,7 +1842,7 @@ func (t *tlog) close() { } } -func newTlog(id, homeDir, dataDir, trillianHost, trillianKeyFile, dcrtimeHost, encryptionKeyFile string) (*tlog, error) { +func newTlog(id, homeDir, dataDir, trillianHost, trillianKeyFile, encryptionKeyFile, dcrtimeHost, dcrtimeCert string) (*tlog, error) { // Load encryption key if provided. An encryption key is optional. var ek *encryptionKey if encryptionKeyFile != "" { @@ -1887,14 +1887,20 @@ func newTlog(id, homeDir, dataDir, trillianHost, trillianKeyFile, dcrtimeHost, e return nil, err } + // Setup dcrtime client + dcrtimeClient, err := newDcrtimeClient(dcrtimeHost, dcrtimeCert) + if err != nil { + return nil, err + } + // Setup tlog t := tlog{ id: id, - dcrtimeHost: dcrtimeHost, - encryptionKey: ek, trillian: trillianClient, store: store, + dcrtime: dcrtimeClient, cron: cron.New(), + encryptionKey: ek, } // Launch cron diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 1f549f0ff..0772c578d 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -1844,7 +1844,7 @@ func (t *tlogBackend) setup() error { } // New returns a new tlogBackend. -func New(anp *chaincfg.Params, homeDir, dataDir, dcrtimeHost, encryptionKeyFile, unvettedTrillianHost, unvettedTrillianKeyFile, vettedTrillianHost, vettedTrillianKeyFile string) (*tlogBackend, error) { +func New(anp *chaincfg.Params, homeDir, dataDir, unvettedTrillianHost, unvettedTrillianKeyFile, vettedTrillianHost, vettedTrillianKeyFile, encryptionKeyFile, dcrtimeHost, dcrtimeCert string) (*tlogBackend, error) { // Setup encryption key file if encryptionKeyFile == "" { // No file path was given. Use the default path. @@ -1874,13 +1874,14 @@ func New(anp *chaincfg.Params, homeDir, dataDir, dcrtimeHost, encryptionKeyFile, // Setup tlog instances unvetted, err := newTlog(tlogIDUnvetted, homeDir, dataDir, - unvettedTrillianHost, unvettedTrillianKeyFile, dcrtimeHost, - encryptionKeyFile) + unvettedTrillianHost, unvettedTrillianKeyFile, encryptionKeyFile, + dcrtimeHost, dcrtimeCert) if err != nil { return nil, fmt.Errorf("newTlog unvetted: %v", err) } - vetted, err := newTlog(tlogIDVetted, homeDir, dataDir, vettedTrillianHost, - vettedTrillianKeyFile, dcrtimeHost, "") + vetted, err := newTlog(tlogIDVetted, homeDir, dataDir, + vettedTrillianHost, vettedTrillianKeyFile, "", + dcrtimeHost, dcrtimeCert) if err != nil { return nil, fmt.Errorf("newTlog vetted: %v", err) } diff --git a/politeiad/cmd/politeia/politeia.go b/politeiad/cmd/politeia/politeia.go index ae3de02fe..a8a86f9b9 100644 --- a/politeiad/cmd/politeia/politeia.go +++ b/politeiad/cmd/politeia/politeia.go @@ -215,7 +215,7 @@ func pluginInventory() (*v1.PluginInventoryReply, error) { fmt.Println(string(b)) } - c, err := util.NewClient(verify, *rpccert) + c, err := util.NewHTTPClient(verify, *rpccert) if err != nil { return nil, err } @@ -287,7 +287,7 @@ func plugin() error { fmt.Println(string(b)) } - c, err := util.NewClient(verify, *rpccert) + c, err := util.NewHTTPClient(verify, *rpccert) if err != nil { return err } @@ -398,7 +398,7 @@ func recordInventory() error { } // Make request - c, err := util.NewClient(verify, *rpccert) + c, err := util.NewHTTPClient(verify, *rpccert) if err != nil { return err } @@ -536,7 +536,7 @@ func newRecord() error { fmt.Println(string(b)) } - c, err := util.NewClient(verify, *rpccert) + c, err := util.NewHTTPClient(verify, *rpccert) if err != nil { return err } @@ -749,7 +749,7 @@ func updateVettedMetadata() error { fmt.Println(string(b)) } - c, err := util.NewClient(verify, *rpccert) + c, err := util.NewHTTPClient(verify, *rpccert) if err != nil { return err } @@ -839,7 +839,7 @@ func updateUnvettedMetadata() error { } // Make request - c, err := util.NewClient(verify, *rpccert) + c, err := util.NewHTTPClient(verify, *rpccert) if err != nil { return err } @@ -1005,7 +1005,7 @@ func updateRecord(vetted bool) error { fmt.Println(string(b)) } - c, err := util.NewClient(verify, *rpccert) + c, err := util.NewHTTPClient(verify, *rpccert) if err != nil { return err } @@ -1085,7 +1085,7 @@ func getUnvetted() error { fmt.Println(string(b)) } - c, err := util.NewClient(verify, *rpccert) + c, err := util.NewHTTPClient(verify, *rpccert) if err != nil { return err } @@ -1189,7 +1189,7 @@ func getVetted() error { fmt.Println(string(b)) } - c, err := util.NewClient(verify, *rpccert) + c, err := util.NewHTTPClient(verify, *rpccert) if err != nil { return err } @@ -1344,7 +1344,7 @@ func setUnvettedStatus() error { fmt.Println(string(b)) } - c, err := util.NewClient(verify, *rpccert) + c, err := util.NewHTTPClient(verify, *rpccert) if err != nil { return err } @@ -1469,7 +1469,7 @@ func setVettedStatus() error { } // Make request - c, err := util.NewClient(verify, *rpccert) + c, err := util.NewHTTPClient(verify, *rpccert) if err != nil { return err } diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 00f65907d..eeff1a4c9 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -1103,7 +1103,7 @@ func (p *politeia) addRoute(method string, route string, handler http.HandlerFun func _main() error { // Load configuration and parse command line. This function also // initializes logging and configures it accordingly. - loadedCfg, _, err := loadConfig() + cfg, _, err := loadConfig() if err != nil { return fmt.Errorf("Could not load configuration file: %v", err) } @@ -1116,22 +1116,22 @@ func _main() error { log.Infof("Version : %v", version.String()) log.Infof("Build Version: %v", version.BuildMainVersion()) log.Infof("Network : %v", activeNetParams.Params.Name) - log.Infof("Home dir: %v", loadedCfg.HomeDir) + log.Infof("Home dir: %v", cfg.HomeDir) // Create the data directory in case it does not exist. - err = os.MkdirAll(loadedCfg.DataDir, 0700) + err = os.MkdirAll(cfg.DataDir, 0700) if err != nil { return err } // Generate the TLS cert and key file if both don't already // exist. - if !util.FileExists(loadedCfg.HTTPSKey) && - !util.FileExists(loadedCfg.HTTPSCert) { + if !util.FileExists(cfg.HTTPSKey) && + !util.FileExists(cfg.HTTPSCert) { log.Infof("Generating HTTPS keypair...") err := util.GenCertPair(elliptic.P521(), "politeiad", - loadedCfg.HTTPSCert, loadedCfg.HTTPSKey) + cfg.HTTPSCert, cfg.HTTPSKey) if err != nil { return fmt.Errorf("unable to create https keypair: %v", err) @@ -1141,13 +1141,13 @@ func _main() error { } // Generate ed25519 identity to save messages, tokens etc. - if !util.FileExists(loadedCfg.Identity) { + if !util.FileExists(cfg.Identity) { log.Infof("Generating signing identity...") id, err := identity.New() if err != nil { return err } - err = id.Save(loadedCfg.Identity) + err = id.Save(cfg.Identity) if err != nil { return err } @@ -1156,12 +1156,12 @@ func _main() error { // Setup application context. p := &politeia{ - cfg: loadedCfg, + cfg: cfg, plugins: make(map[string]v1.Plugin), } // Load identity. - p.identity, err = identity.LoadFullIdentity(loadedCfg.Identity) + p.identity, err = identity.LoadFullIdentity(cfg.Identity) if err != nil { return err } @@ -1169,16 +1169,16 @@ func _main() error { // Load certs, if there. If they aren't there assume OS is used to // resolve cert validity. - if len(loadedCfg.DcrtimeCert) != 0 { + if len(cfg.DcrtimeCert) != 0 { var certPool *x509.CertPool - if !util.FileExists(loadedCfg.DcrtimeCert) { + if !util.FileExists(cfg.DcrtimeCert) { return fmt.Errorf("unable to find dcrtime cert %v", - loadedCfg.DcrtimeCert) + cfg.DcrtimeCert) } - dcrtimeCert, err := ioutil.ReadFile(loadedCfg.DcrtimeCert) + dcrtimeCert, err := ioutil.ReadFile(cfg.DcrtimeCert) if err != nil { return fmt.Errorf("unable to read dcrtime cert %v: %v", - loadedCfg.DcrtimeCert, err) + cfg.DcrtimeCert, err) } certPool = x509.NewCertPool() if !certPool.AppendCertsFromPEM(dcrtimeCert) { @@ -1186,27 +1186,26 @@ func _main() error { } } // Setup backend. - log.Infof("Backend: %v", loadedCfg.Backend) - switch loadedCfg.Backend { + log.Infof("Backend: %v", cfg.Backend) + switch cfg.Backend { case backendGit: - b, err := gitbe.New(activeNetParams.Params, loadedCfg.DataDir, - loadedCfg.DcrtimeHost, "", p.identity, loadedCfg.GitTrace, - loadedCfg.DcrdataHost) + b, err := gitbe.New(activeNetParams.Params, cfg.DataDir, cfg.DcrtimeHost, + "", p.identity, cfg.GitTrace, cfg.DcrdataHost) if err != nil { return fmt.Errorf("new gitbe: %v", err) } p.backend = b case backendTlog: - b, err := tlogbe.New(activeNetParams.Params, loadedCfg.HomeDir, - loadedCfg.DataDir, loadedCfg.DcrtimeHost, loadedCfg.EncryptionKey, - loadedCfg.TrillianHostUnvetted, loadedCfg.TrillianKeyUnvetted, - loadedCfg.TrillianHostVetted, loadedCfg.TrillianKeyVetted) + b, err := tlogbe.New(activeNetParams.Params, cfg.HomeDir, cfg.DataDir, + cfg.TrillianHostUnvetted, cfg.TrillianKeyUnvetted, + cfg.TrillianHostVetted, cfg.TrillianKeyVetted, + cfg.EncryptionKey, cfg.DcrtimeHost, cfg.DcrtimeCert) if err != nil { return fmt.Errorf("new tlogbe: %v", err) } p.backend = b default: - return fmt.Errorf("invalid backend selected: %v", loadedCfg.Backend) + return fmt.Errorf("invalid backend selected: %v", cfg.Backend) } // Setup mux @@ -1245,7 +1244,7 @@ func _main() error { // TODO document plugins and plugin settings in README // Setup plugins - if len(loadedCfg.Plugins) > 0 { + if len(cfg.Plugins) > 0 { // Set plugin routes. Requires auth. p.addRoute(http.MethodPost, v1.PluginCommandRoute, p.pluginCommand, permissionAuth) @@ -1255,7 +1254,7 @@ func _main() error { // Parse plugin settings // map[pluginID][]backend.PluginSetting settings := make(map[string][]backend.PluginSetting) - for _, v := range loadedCfg.PluginSettings { + for _, v := range cfg.PluginSettings { // Plugin setting will be in format: pluginID,key,value s := strings.Split(v, ",") if len(s) != 3 { @@ -1280,7 +1279,7 @@ func _main() error { } // Register plugins - for _, v := range loadedCfg.Plugins { + for _, v := range cfg.Plugins { // Verify plugin ID is lowercase if backend.PluginRE.FindString(v) != v { return fmt.Errorf("invalid plugin id: %v", v) @@ -1342,7 +1341,7 @@ func _main() error { } // Setup plugins - for _, v := range loadedCfg.Plugins { + for _, v := range cfg.Plugins { log.Infof("Setting up plugin: %v", v) err := p.backend.SetupPlugin(v) if err != nil { @@ -1353,13 +1352,12 @@ func _main() error { // Bind to a port and pass our router in listenC := make(chan error) - for _, listener := range loadedCfg.Listeners { + for _, listener := range cfg.Listeners { listen := listener go func() { log.Infof("Listen: %v", listen) listenC <- http.ListenAndServeTLS(listen, - loadedCfg.HTTPSCert, loadedCfg.HTTPSKey, - p.router) + cfg.HTTPSCert, cfg.HTTPSKey, p.router) }() } diff --git a/politeiawww/cmswww.go b/politeiawww/cmswww.go index 1489f8075..ac8fc03c9 100644 --- a/politeiawww/cmswww.go +++ b/politeiawww/cmswww.go @@ -1022,7 +1022,7 @@ func (p *politeiawww) makeProposalsRequest(method string, route string, v interf } } - client, err := util.NewClient(false, "") + client, err := util.NewHTTPClient(false, "") if err != nil { return nil, err } diff --git a/politeiawww/www.go b/politeiawww/www.go index fcaa4a54a..f1736db3d 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -965,7 +965,7 @@ func _main() error { } // Setup politeiad client - client, err := util.NewClient(false, loadedCfg.RPCCert) + client, err := util.NewHTTPClient(false, loadedCfg.RPCCert) if err != nil { return err } diff --git a/util/identity.go b/util/identity.go index 718d244ca..43e76ea70 100644 --- a/util/identity.go +++ b/util/identity.go @@ -12,7 +12,7 @@ import ( "io/ioutil" "net/http" - "github.com/decred/politeia/politeiad/api/v1" + v1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/identity" ) @@ -39,7 +39,7 @@ func RemoteIdentity(skipTLSVerify bool, host, cert string) (*identity.PublicIden return nil, err } - c, err := NewClient(skipTLSVerify, cert) + c, err := NewHTTPClient(skipTLSVerify, cert) if err != nil { return nil, err } diff --git a/util/net.go b/util/net.go index a3d4c7a1c..d68db6604 100644 --- a/util/net.go +++ b/util/net.go @@ -27,14 +27,14 @@ func NormalizeAddress(addr, defaultPort string) string { return addr } -// NewClient returns a new http.Client instance -func NewClient(skipVerify bool, certFilename string) (*http.Client, error) { +// NewHTTPClient returns a new http.Client instance +func NewHTTPClient(skipVerify bool, certPath string) (*http.Client, error) { tlsConfig := &tls.Config{ InsecureSkipVerify: skipVerify, } - if !skipVerify && certFilename != "" { - cert, err := ioutil.ReadFile(certFilename) + if !skipVerify && certPath != "" { + cert, err := ioutil.ReadFile(certPath) if err != nil { return nil, err } From 6ab22cd604d969e0d5e0b682743a0f456b04127d Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Wed, 6 Jan 2021 11:43:11 -0300 Subject: [PATCH 202/449] tlogbe: Use proper TempDir for testing data dirs. --- politeiad/backend/tlogbe/testing.go | 51 +++++++++---------- politeiad/backend/tlogbe/tlogbe_test.go | 68 +++++++++---------------- 2 files changed, 46 insertions(+), 73 deletions(-) diff --git a/politeiad/backend/tlogbe/testing.go b/politeiad/backend/tlogbe/testing.go index e2d7d670b..ee6fe4160 100644 --- a/politeiad/backend/tlogbe/testing.go +++ b/politeiad/backend/tlogbe/testing.go @@ -8,15 +8,14 @@ import ( "bytes" "encoding/base64" "encoding/hex" - "fmt" "image" "image/jpeg" "image/png" + "io/ioutil" "os" "path/filepath" "testing" - "github.com/decred/dcrd/dcrutil/v3" v1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/mime" "github.com/decred/politeia/politeiad/backend" @@ -24,11 +23,6 @@ import ( "github.com/decred/politeia/util" ) -var ( - defaultTestDir = dcrutil.AppDataDir("politeiadtest", false) - defaultTestDataDir = filepath.Join(defaultTestDir, "data") -) - func newBackendFile(t *testing.T, fileName string) backend.File { t.Helper() @@ -109,18 +103,17 @@ func newBackendMetadataStream(t *testing.T, id uint64, payload string) backend.M } // newTestTlog returns a tlog used for testing. -func newTestTlog(t *testing.T, id string) (*tlog, error) { - // Setup key-value store with test dir - fp := filepath.Join(defaultTestDataDir, id) - err := os.MkdirAll(fp, 0700) +func newTestTlog(t *testing.T, dir, id string) *tlog { + dir, err := ioutil.TempDir(dir, id) if err != nil { - return nil, err + t.Fatalf("TempDir: %v", err) } - store := filesystem.New(fp) + + store := filesystem.New(dir) tclient, err := newTestTClient(t) if err != nil { - return nil, err + t.Fatalf("newTestTClient: %v", err) } tlog := tlog{ @@ -130,27 +123,24 @@ func newTestTlog(t *testing.T, id string) (*tlog, error) { store: store, } - return &tlog, nil + return &tlog } // newTestTlogBackend returns a tlog backend for testing. It wraps // tlog and trillian client, providing the framework needed for // writing tlog backend tests. -func newTestTlogBackend(t *testing.T) (*tlogBackend, error) { - tlogVetted, err := newTestTlog(t, "vetted") +func newTestTlogBackend(t *testing.T) (*tlogBackend, func()) { + testDir, err := ioutil.TempDir("", "tlog.backend.test") if err != nil { - return nil, err - } - tlogUnvetted, err := newTestTlog(t, "unvetted") - if err != nil { - return nil, err + t.Fatalf("TempDir: %v", err) } + testDataDir := filepath.Join(testDir, "data") tlogBackend := tlogBackend{ - homeDir: defaultTestDir, - dataDir: defaultTestDataDir, - unvetted: tlogUnvetted, - vetted: tlogVetted, + homeDir: testDir, + dataDir: testDataDir, + unvetted: newTestTlog(t, testDir, "unvetted"), + vetted: newTestTlog(t, testDir, "vetted"), plugins: make(map[string]plugin), prefixes: make(map[string][]byte), vettedTreeIDs: make(map[string]int64), @@ -162,10 +152,15 @@ func newTestTlogBackend(t *testing.T) (*tlogBackend, error) { err = tlogBackend.setup() if err != nil { - return nil, fmt.Errorf("setup: %v", err) + t.Fatalf("setup: %v", err) } - return &tlogBackend, nil + return &tlogBackend, func() { + err = os.RemoveAll(testDir) + if err != nil { + t.Fatalf("RemoveAll: %v", err) + } + } } // recordContentTests defines the type used to describe the content diff --git a/politeiad/backend/tlogbe/tlogbe_test.go b/politeiad/backend/tlogbe/tlogbe_test.go index b7416a7ce..f0af74495 100644 --- a/politeiad/backend/tlogbe/tlogbe_test.go +++ b/politeiad/backend/tlogbe/tlogbe_test.go @@ -14,10 +14,8 @@ import ( ) func TestNewRecord(t *testing.T) { - tlogBackend, err := newTestTlogBackend(t) - if err != nil { - t.Error(err) - } + tlogBackend, cleanup := newTestTlogBackend(t) + defer cleanup() // Test all record content verification error through the New endpoint recordContentTests := setupRecordContentTests(t) @@ -45,17 +43,15 @@ func TestNewRecord(t *testing.T) { fs := []backend.File{ newBackendFile(t, "index.md"), } - _, err = tlogBackend.New(md, fs) + _, err := tlogBackend.New(md, fs) if err != nil { t.Errorf("success case failed with %v", err) } } func TestUpdateUnvettedRecord(t *testing.T) { - tlogBackend, err := newTestTlogBackend(t) - if err != nil { - t.Error(err) - } + tlogBackend, cleanup := newTestTlogBackend(t) + defer cleanup() // Create new record md := []backend.MetadataStream{ @@ -219,10 +215,8 @@ func TestUpdateUnvettedRecord(t *testing.T) { } func TestUpdateVettedRecord(t *testing.T) { - tlogBackend, err := newTestTlogBackend(t) - if err != nil { - t.Error(err) - } + tlogBackend, cleanup := newTestTlogBackend(t) + defer cleanup() // Create new record md := []backend.MetadataStream{ @@ -405,10 +399,8 @@ func TestUpdateVettedRecord(t *testing.T) { } func TestUpdateUnvettedMetadata(t *testing.T) { - tlogBackend, err := newTestTlogBackend(t) - if err != nil { - t.Error(err) - } + tlogBackend, cleanup := newTestTlogBackend(t) + defer cleanup() // Create new record md := []backend.MetadataStream{ @@ -576,10 +568,8 @@ func TestUpdateUnvettedMetadata(t *testing.T) { } func TestUpdateVettedMetadata(t *testing.T) { - tlogBackend, err := newTestTlogBackend(t) - if err != nil { - t.Error(err) - } + tlogBackend, cleanup := newTestTlogBackend(t) + defer cleanup() // Create new record md := []backend.MetadataStream{ @@ -764,10 +754,8 @@ func TestUpdateVettedMetadata(t *testing.T) { } func TestUnvettedExists(t *testing.T) { - tlogBackend, err := newTestTlogBackend(t) - if err != nil { - t.Error(err) - } + tlogBackend, cleanup := newTestTlogBackend(t) + defer cleanup() // Create new record md := []backend.MetadataStream{ @@ -803,10 +791,8 @@ func TestUnvettedExists(t *testing.T) { } func TestVettedExists(t *testing.T) { - tlogBackend, err := newTestTlogBackend(t) - if err != nil { - t.Error(err) - } + tlogBackend, cleanup := newTestTlogBackend(t) + defer cleanup() // Create unvetted record md := []backend.MetadataStream{ @@ -857,10 +843,8 @@ func TestVettedExists(t *testing.T) { } func TestGetUnvetted(t *testing.T) { - tlogBackend, err := newTestTlogBackend(t) - if err != nil { - t.Error(err) - } + tlogBackend, cleanup := newTestTlogBackend(t) + defer cleanup() // Create new record md := []backend.MetadataStream{ @@ -901,10 +885,8 @@ func TestGetUnvetted(t *testing.T) { } func TestGetVetted(t *testing.T) { - tlogBackend, err := newTestTlogBackend(t) - if err != nil { - t.Error(err) - } + tlogBackend, cleanup := newTestTlogBackend(t) + defer cleanup() // Create new record md := []backend.MetadataStream{ @@ -953,10 +935,8 @@ func TestGetVetted(t *testing.T) { } func TestSetUnvettedStatus(t *testing.T) { - tlogBackend, err := newTestTlogBackend(t) - if err != nil { - t.Error(err) - } + tlogBackend, cleanup := newTestTlogBackend(t) + defer cleanup() // Helpers md := []backend.MetadataStream{ @@ -1119,10 +1099,8 @@ func TestSetUnvettedStatus(t *testing.T) { } func TestSetVettedStatus(t *testing.T) { - tlogBackend, err := newTestTlogBackend(t) - if err != nil { - t.Error(err) - } + tlogBackend, cleanup := newTestTlogBackend(t) + defer cleanup() // Helpers md := []backend.MetadataStream{ From 461047539d90697bf324d75c121ced3e05686508 Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Fri, 8 Jan 2021 13:15:55 -0300 Subject: [PATCH 203/449] www: Update user route docs. --- politeiawww/api/www/v1/api.md | 141 ++++++++++++++++++++++++++++++++-- 1 file changed, 136 insertions(+), 5 deletions(-) diff --git a/politeiawww/api/www/v1/api.md b/politeiawww/api/www/v1/api.md index 8a6aaf7c9..aada7e9d9 100644 --- a/politeiawww/api/www/v1/api.md +++ b/politeiawww/api/www/v1/api.md @@ -19,6 +19,7 @@ notifications. It does not render HTML. - [`Logout`](#logout) - [`User details`](#user-details) - [`Edit user`](#edit-user) +- [`Manage user`](#manage-user) - [`Users`](#users) - [`Update user key`](#update-user-key) - [`Verify update user key`](#verify-update-user-key) @@ -28,6 +29,7 @@ notifications. It does not render HTML. - [`User proposal credits`](#user-proposal-credits) - [`Proposal paywall details`](#proposal-paywall-details) - [`Verify user payment`](#verify-user-payment) +- [`Rescan user payments`](#rescan-user-payments) **Proposal Routes** - [`Token inventory`](#token-inventory) @@ -532,7 +534,7 @@ Reply: Checks that a user has paid his user registration fee. -**Route:** `GET /v1/user/verifypayment` +**Route:** `GET /v1/user/payments/registration` **Params:** none @@ -554,7 +556,7 @@ error codes: Request: ``` -/v1/user/verifypayment +/v1/user/payments/registration ``` Reply: @@ -568,6 +570,58 @@ Reply: } ``` +### `Rescan user payments` + +Rescan payments made by the user. + +**Route:** `GET /v1/user/payments/rescan` + +**Params:** +| Parameter | Type | Description | Required | +|-----------|------|-------------|----------| +| userid | string | The unique id of the user. | Yes | + +**Results:** + +| Parameter | Type | Description | +|-|-|-| +| newcredits | []ProposalCredits | Contains information about the user's +proposal credits payments` | + +On failure the call shall return `400 Bad Request` and one of the following +error codes: +- [`ErrorStatusInvalidInput`](#ErrorStatusInvalidInput) + +**Example** + +Request: + +``` +/v1/user/payments/rescan +``` +Request: + +```json +{ + "userid": "1" +} +``` + +Reply: + +```json +{ + "newcredits": [ + { + "paywallid": 1, + "price": 10000000, + "datepurchased": 1528821554, + "txid": "ff0207a03b761cb409c7677c5b5521562302653d2236c92d016dd47e0ae37bf7", + }, + ] +} +``` + ### `User details` Returns details about a user given its id. Returns complete data if request is from @@ -678,9 +732,42 @@ For a unlogged or normal user requesting data. } } ``` - ### `Edit user` +Edits a user's details. This call requires login privileges. + +**Route:** `POST /v1/user/edit` + +**Params:** + +| Parameter | Type | Description | Required | +|-----------|------|-------------|----------| +| emailnotifications | uint64 | The unique id of the user. | Yes | + +**Results:** none + +On failure the call shall return `400 Bad Request` and one of the following +error codes: +- [`ErrorStatusInvalidInput`](#ErrorStatusInvalidInput) + +**Example** + +Request: + +```json +{ + "emailnotifications": 2, +} +``` + +Reply: + +```json +{} +``` + +### `Manage user` + Edits a user's details. This call requires admin privileges. **Route:** `POST /v1/user/manage` @@ -1025,7 +1112,7 @@ politeiawww polls the paywall address until the paywall is either paid or it expires. A proposal paywall cannot be generated until the user has paid their user registration fee. -**Route:** `GET /v1/proposals/paywall` +**Route:** `GET /v1/user/payments/paywall` **Params:** none @@ -1058,10 +1145,54 @@ Reply: } ``` +### `Proposal paywall tx details` +Retrieve paywall details that can be used to purchase proposal credits. +Proposal paywalls are only valid for one tx. The user can purchase as many +proposal credits as they would like with that one tx. Proposal paywalls expire +after a set duration. To verify that a payment has been made, +politeiawww polls the paywall address until the paywall is either paid or it +expires. A proposal paywall cannot be generated until the user has paid their +user registration fee. + +**Route:** `GET /v1/user/payments/paywalltx` + +**Params:** +| Parameter | Type | Description | Required | +|-----------|------|-------------|----------| +| userid | string | The unique id of the user. | Yes | + +**Results:** + +| Parameter | Type | Description | +|-|-|-| +| txid | string | Transaction id. | +| txamount | uint64 | Transaction amount in atoms. | +| confirmations | uint64 | Confirmations the transaction had on the network | + +**Example** + +Request: + +```json +{ + "userid": "1" +} +``` + +Reply: + +```json +{ + "txid": "ff0207a03b761cb409c7677c5b5521562302653d2236c92d016dd47e0ae37bf7", + "txamount": 10000000, + "confirmations": 2 +} +``` + ### `User proposal credits` Request a list of the user's unspent and spent proposal credits. -**Route:** `GET /v1/user/proposals/credits` +**Route:** `GET /v1/user/payments/credits` **Params:** none From 7522099d6e4f820b25204f7f3aa0607c68be2318 Mon Sep 17 00:00:00 2001 From: luke Date: Sat, 9 Jan 2021 16:18:54 -0600 Subject: [PATCH 204/449] tlogbe: Fix anchor bug. --- politeiad/backend/tlogbe/anchor.go | 118 +++++++++++++++++++---------- 1 file changed, 78 insertions(+), 40 deletions(-) diff --git a/politeiad/backend/tlogbe/anchor.go b/politeiad/backend/tlogbe/anchor.go index 19472f03c..741ee9f7b 100644 --- a/politeiad/backend/tlogbe/anchor.go +++ b/politeiad/backend/tlogbe/anchor.go @@ -12,14 +12,14 @@ import ( "time" dcrtime "github.com/decred/dcrtime/api/v2" + "github.com/decred/dcrtime/merkle" "github.com/decred/politeia/politeiad/backend/tlogbe/store" - "github.com/decred/politeia/util" "github.com/google/trillian" "github.com/google/trillian/types" "google.golang.org/grpc/codes" ) -// TODO handle reorgs. A anchor record may become invalid in the case of a +// TODO handle reorgs. An anchor record may become invalid in the case of a // reorg. We don't create the anchor record until the anchor tx has 6 // confirmations so the probability of this occurring on mainnet is low, but it // still needs to be handled. @@ -29,6 +29,7 @@ const ( // currently drops an anchor on the hour mark so we submit new // anchors a few minutes prior to that. // Seconds Minutes Hours Days Months DayOfWeek + anchorSchedule = "0 56 * * * *" // At minute 56 of every hour // anchorID is included in the timestamp and verify requests as a @@ -37,8 +38,12 @@ const ( ) // anchor represents an anchor, i.e. timestamp, of a trillian tree at a -// specific tree size. A SHA256 digest of the LogRoot is timestamped using -// dcrtime. +// specific tree size. The LogRootV1.RootHash is the merkle root hash of a +// trillian tree. This root hash is submitted to dcrtime to be anchored and is +// the anchored digest in the VerifyDigest. Only the root hash is anchored, but +// the full LogRootV1 struct is saved as part of an anchor record so that it +// can be used to retreive inclusion proofs for any leaves that were included +// in the root hash. type anchor struct { TreeID int64 `json:"treeid"` LogRoot *types.LogRootV1 `json:"logroot"` @@ -172,7 +177,7 @@ func (t *tlog) anchorLatest(treeID int64) (*anchor, error) { // confirmations. Once the timestamp has been dropped, the anchor record is // saved to the key-value store and the record histories of the corresponding // timestamped trees are updated. -func (t *tlog) anchorWait(anchors []anchor, hashes []string) { +func (t *tlog) anchorWait(anchors []anchor, digests []string) { // Ensure we are not reentrant t.Lock() if t.droppingAnchor { @@ -219,20 +224,35 @@ func (t *tlog) anchorWait(anchors []anchor, hashes []string) { log.Debugf("Verify %v anchor attempt %v/%v", t.id, try+1, retries) - vbr, err := t.dcrtime.verifyBatch(anchorID, hashes) + vbr, err := t.dcrtime.verifyBatch(anchorID, digests) if err != nil { exitErr = fmt.Errorf("verifyBatch: %v", err) return } - // Make sure we're actually anchored - var retry bool + // We must wait until all digests have been anchored. Under + // normal circumstances this will happen during the same dcrtime + // transaction, but its possible for some of the digests to have + // already been anchored in previous transactions if politeiad + // was shutdown in the middle of the anchoring process. + // + // Ex: politeiad submits a digest for treeA to dcrtime. politeiad + // gets shutdown before an anchor record is added to treeA. + // dcrtime timestamps the treeA digest onto block 1000. politeiad + // gets turned back on and a new record, treeB, is submitted + // prior to an anchor drop attempt. On the next anchor drop, + // politeiad will try to drop an anchor for both treeA and treeB + // since treeA is still considered unachored, however, when this + // part of the code gets hit dcrtime will immediately return a + // valid timestamp for treeA since it was already timestamped + // into block 1000. In this situation, the verify loop must also + // wait for treeB to be timestamped by dcrtime before continuing. + anchored := make(map[string]struct{}, len(digests)) for _, v := range vbr.Digests { if v.Result != dcrtime.ResultOK { // Something is wrong. Log the error and retry. log.Errorf("Digest %v: %v (%v)", v.Digest, dcrtime.Result[v.Result], v.Result) - retry = true break } @@ -241,7 +261,6 @@ func (t *tlog) anchorWait(anchors []anchor, hashes []string) { b := make([]byte, sha256.Size) if v.ChainInformation.Transaction == hex.EncodeToString(b) { log.Debugf("Anchor tx not sent yet; retry in %v", period) - retry = true break } @@ -250,34 +269,61 @@ func (t *tlog) anchorWait(anchors []anchor, hashes []string) { if v.ChainInformation.ChainTimestamp == 0 { log.Debugf("Anchor tx %v not enough confirmations; retry in %v", v.ChainInformation.Transaction, period) - retry = true break } + + // This digest has been anchored + anchored[v.Digest] = struct{}{} } - if retry { + if len(anchored) != len(digests) { + // There are still digests that are waiting to be anchored. + // Retry again after the wait period. continue } // Save anchor records for k, v := range anchors { - // Sanity checks. Anchor log root digest should match digest - // that was anchored. - b, err := v.LogRoot.MarshalBinary() + var ( + verifyDigest = vbr.Digests[k] + digest = verifyDigest.Digest + merkleRoot = verifyDigest.ChainInformation.MerkleRoot + merklePath = verifyDigest.ChainInformation.MerklePath + ) + + // Verify the anchored digest matches the root hash + if digest != hex.EncodeToString(v.LogRoot.RootHash) { + log.Errorf("anchorWait: digest mismatch: got %x, want %v", + digest, v.LogRoot.RootHash) + continue + } + + // Verify merkle path + mk, err := merkle.VerifyAuthPath(&merklePath) if err != nil { - log.Errorf("anchorWait: MarshalBinary %v %x: %v", - v.TreeID, v.LogRoot.RootHash, err) + log.Errorf("anchorWait: VerifyAuthPath: %v", err) continue } - anchorDigest := hex.EncodeToString(util.Hash(b)[:]) - dcrtimeDigest := vbr.Digests[k].Digest - if anchorDigest != dcrtimeDigest { - log.Errorf("anchorWait: digest mismatch: got %x, want %v", - dcrtimeDigest, anchorDigest) + if hex.EncodeToString(mk[:]) != merkleRoot { + log.Errorf("anchorWait: merkle root invalid: got %x, want %v", + mk[:], merkleRoot) continue } - // Add VerifyDigest to anchor before saving it - v.VerifyDigest = &vbr.Digests[k] + // Verify digest is in the merkle path + var found bool + for _, v := range merklePath.Hashes { + if hex.EncodeToString(v[:]) == digest { + found = true + break + } + } + if !found { + log.Errorf("anchorWait: digest %v not found in merkle path", digest) + continue + } + + // Add VerifyDigest to the anchor record + v.VerifyDigest = &verifyDigest // Save anchor err = t.anchorSave(v) @@ -315,9 +361,9 @@ func (t *tlog) anchor() { return } - // digests contains the SHA256 digests of the log roots of the - // trees that need to be anchored. These will be submitted to - // dcrtime to be included in a dcrtime timestamp. + // digests contains the SHA256 digests of the LogRootV1.RootHash + // for all trees that need to be anchored. These will be submitted + // to dcrtime to be included in a dcrtime timestamp. digests := make([]string, 0, len(trees)) // anchors contains an anchor structure for each tree that is being @@ -382,15 +428,9 @@ func (t *tlog) anchor() { LogRoot: lr, }) - // Collate the log root digest. This is what gets submitted to + // Collate the tree's root hash. This is what gets submitted to // dcrtime. - lrb, err := lr.MarshalBinary() - if err != nil { - exitErr = fmt.Errorf("MarshalBinary %v: %v", v.TreeId, err) - return - } - d := hex.EncodeToString(util.Hash(lrb)[:]) - digests = append(digests, d) + digests = append(digests, hex.EncodeToString(lr.RootHash)) log.Debugf("Anchoring %v tree %v at height %v", t.id, v.TreeId, lr.TreeSize) @@ -432,11 +472,9 @@ func (t *tlog) anchor() { case dcrtime.ResultOK: // We're good; continue case dcrtime.ResultExistsError: - // I don't think this will ever happen, but it's ok if it does - // since we'll still be able to retrieve the VerifyDigest from - // dcrtime for this digest. - // - // Log a warning to bring it to our attention. Do not exit. + // This can happen if politeiad was shutdown in the middle of + // an anchor process. This is ok. The anchor process will pick + // up where it left off. log.Warnf("Digest already exists %v: %v (%v)", tbr.Digests[i], dcrtime.Result[v], v) default: From e8080542b95aabccb1ed1515a814933df8136230 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 7 Jan 2021 10:10:20 -0600 Subject: [PATCH 205/449] multi: Add timestamp routes. --- politeiad/api/v1/v1.go | 90 ++++++- politeiad/backend/backend.go | 53 +++- politeiad/backend/gitbe/gitbe.go | 16 +- politeiad/backend/tlogbe/anchor.go | 174 ++++++++----- politeiad/backend/tlogbe/store/store.go | 2 + politeiad/backend/tlogbe/ticketvote.go | 2 +- politeiad/backend/tlogbe/timestamp.go | 178 +++++++++++++ politeiad/backend/tlogbe/tlog.go | 275 ++++++++++++++++---- politeiad/backend/tlogbe/tlogbe.go | 59 ++++- politeiad/backend/tlogbe/trillianclient.go | 85 +----- politeiad/politeiad.go | 150 +++++++++++ politeiad/testpoliteiad/testpoliteiad.go | 2 +- politeiawww/api/pi/v1/v1.go | 33 ++- politeiawww/api/records/v1/v1.go | 118 +++++++++ politeiawww/cmd/piwww/piwww.go | 3 + politeiawww/cmd/piwww/proposaltimestamps.go | 79 ++++++ politeiawww/cmd/shared/client.go | 35 ++- politeiawww/invoices.go | 2 +- politeiawww/piwww.go | 37 ++- politeiawww/politeiad.go | 76 +++++- politeiawww/records.go | 235 +++++++++++++++++ util/convert.go | 10 +- 22 files changed, 1466 insertions(+), 248 deletions(-) create mode 100644 politeiad/backend/tlogbe/timestamp.go create mode 100644 politeiawww/api/records/v1/v1.go create mode 100644 politeiawww/cmd/piwww/proposaltimestamps.go create mode 100644 politeiawww/records.go diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index e77eb3b12..750528eb1 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -21,15 +21,17 @@ type RecordStatusT int const ( // Routes - IdentityRoute = "/v1/identity/" // Retrieve identity - NewRecordRoute = "/v1/newrecord/" // New record - UpdateUnvettedRoute = "/v1/updateunvetted/" // Update unvetted record - UpdateUnvettedMetadataRoute = "/v1/updateunvettedmd/" // Update unvetted metadata - UpdateVettedRoute = "/v1/updatevetted/" // Update vetted record - UpdateVettedMetadataRoute = "/v1/updatevettedmd/" // Update vetted metadata - GetUnvettedRoute = "/v1/getunvetted/" // Retrieve unvetted record - GetVettedRoute = "/v1/getvetted/" // Retrieve vetted record - InventoryByStatusRoute = "/v1/inventorybystatus/" // Inventory record tokens by status + IdentityRoute = "/v1/identity/" // Retrieve identity + NewRecordRoute = "/v1/newrecord/" // New record + UpdateUnvettedRoute = "/v1/updateunvetted/" // Update unvetted record + UpdateUnvettedMetadataRoute = "/v1/updateunvettedmd/" // Update unvetted metadata + UpdateVettedRoute = "/v1/updatevetted/" // Update vetted record + UpdateVettedMetadataRoute = "/v1/updatevettedmd/" // Update vetted metadata + GetUnvettedRoute = "/v1/getunvetted/" // Retrieve unvetted record + GetVettedRoute = "/v1/getvetted/" // Retrieve vetted record + GetUnvettedTimestampsRoute = "/v1/getunvettedts" + GetVettedTimestampsRoute = "/v1/getvettedts" + InventoryByStatusRoute = "/v1/inventorybystatus/" // Auth required InventoryRoute = "/v1/inventory/" // Inventory records @@ -42,8 +44,8 @@ const ( // Token sizes. The size of the token depends on the politeiad // backend configuration. - TokenSizeShort = 10 - TokenSizeLong = 32 + TokenSizeTlog = 8 + TokenSizeGit = 32 MetadataStreamsMax = uint64(16) // Maximum number of metadata streams @@ -359,6 +361,72 @@ type UpdateUnvettedMetadataReply struct { Response string `json:"response"` // Challenge response } +// Proof contains an inclusion proof for the digest in the merkle root. +type Proof struct { + Type string `json:"type"` + Digest string `json:"digest"` + MerkleRoot string `json:"merkleroot"` + MerklePath []string `json:"merklepath"` + ExtraData string `json:"extradata"` // JSON encoded +} + +// Timestamp contains all of the data required to verify that a piece of record +// content was timestamped onto the decred blockchain. +// +// All digests are hex encoded SHA256 digests. The merkle root can be found in +// the OP_RETURN of the specified DCR transaction. +// +// TxID, MerkleRoot, and Proofs will only be populated once the merkle root has +// been included in a DCR tx and the tx has 6 confirmations. The Data field +// will not be populated if the data has been censored. +type Timestamp struct { + Data string `json:"data"` // JSON encoded + Digest string `json:"digest"` + TxID string `json:"txid"` + MerkleRoot string `json:"merkleroot"` + Proofs []Proof `json:"proofs"` +} + +// RecordTimestamps contains a Timestamp for all record files +type RecordTimestamps struct { + Token string `json:"token"` // Censorship token + Version string `json:"version"` // Version of files + RecordMetadata Timestamp `json:"recordmetadata"` + + // map[metadataID]Timestamp + Metadata map[uint64]Timestamp `json:"metadata"` + + // map[filename]Timestamp + Files map[string]Timestamp `json:"files"` +} + +// GetUnvettedTimestamps requests the timestamps for an unvetted record. +type GetUnvettedTimestamps struct { + Challenge string `json:"challenge"` // Random challenge + Token string `json:"token"` // Censorship token + Version string `json:"version"` // Record version +} + +// GetUnvettedTimestampsReply is the reply to the GetUnvettedTimestamps +// command. +type GetUnvettedTimestampsReply struct { + Response string `json:"response"` // Challenge response + RecordTimestamps RecordTimestamps `json:"timestamp"` +} + +// GetVettedTimestamps requests the timestamps for a vetted record. +type GetVettedTimestamps struct { + Challenge string `json:"challenge"` // Random challenge + Token string `json:"token"` // Censorship token + Version string `json:"version"` // Record version +} + +// GetVettedTimestampsReply is the reply to the GetVettedTimestamps command. +type GetVettedTimestampsReply struct { + Response string `json:"response"` // Challenge response + RecordTimestamps RecordTimestamps `json:"timestamp"` +} + // Inventory sends an (expensive and therefore authenticated) inventory request // for vetted records (master branch) and branches (censored, unpublished etc) // records. This is a very expensive call and should be only issued at start diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index ffb9bc888..fbfbc6bde 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -149,6 +149,41 @@ type Record struct { Files []File // User provided files } +// Proof contains an inclusion proof for the digest in the merkle root. +type Proof struct { + Type string + Digest string + MerkleRoot string + MerklePath []string + ExtraData string // JSON encoded +} + +// Timestamp contains all of the data required to verify that a piece of record +// content was timestamped onto the decred blockchain. +// +// All digests are hex encoded SHA256 digests. The merkle root can be found in +// the OP_RETURN of the specified DCR transaction. +// +// TxID, MerkleRoot, and Proofs will only be populated once the merkle root has +// been included in a DCR tx and the tx has 6 confirmations. The Data field +// will not be populated if the data has been censored. +type Timestamp struct { + Data string // JSON encoded + Digest string + TxID string + MerkleRoot string + Proofs []Proof +} + +// RecordTimestamps contains a Timestamp for all record data. +type RecordTimestamps struct { + Token string // Censorship token + Version string // Version of files + RecordMetadata Timestamp + Metadata map[uint64]Timestamp // [metadataID]Timestamp + Files map[string]Timestamp // [filename]Timestamp +} + // PluginSettings type PluginSetting struct { Key string // Name of setting @@ -195,6 +230,14 @@ type Backend interface { UpdateVettedMetadata([]byte, []MetadataStream, []MetadataStream) error + // Set unvetted record status + SetUnvettedStatus([]byte, MDStatusT, []MetadataStream, + []MetadataStream) (*Record, error) + + // Set vetted record status + SetVettedStatus([]byte, MDStatusT, []MetadataStream, + []MetadataStream) (*Record, error) + // Check if an unvetted record exists UnvettedExists([]byte) bool @@ -207,13 +250,11 @@ type Backend interface { // Get vetted record GetVetted([]byte, string) (*Record, error) - // Set unvetted record status - SetUnvettedStatus([]byte, MDStatusT, []MetadataStream, - []MetadataStream) (*Record, error) + // Get unvetted record content timestamps + GetUnvettedTimestamps([]byte, string) (*RecordTimestamps, error) - // Set vetted record status - SetVettedStatus([]byte, MDStatusT, []MetadataStream, - []MetadataStream) (*Record, error) + // Get vetted record content timestamps + GetVettedTimestamps([]byte, string) (*RecordTimestamps, error) // Inventory retrieves various record records. Inventory(uint, uint, uint, bool, bool) ([]Record, []Record, error) diff --git a/politeiad/backend/gitbe/gitbe.go b/politeiad/backend/gitbe/gitbe.go index be33129d8..7d5e8a769 100644 --- a/politeiad/backend/gitbe/gitbe.go +++ b/politeiad/backend/gitbe/gitbe.go @@ -1281,7 +1281,7 @@ func (g *gitBackEnd) populateTokenPrefixCache() error { func (g *gitBackEnd) randomUniqueToken() ([]byte, error) { TRIES := 1000 for i := 0; i < TRIES; i++ { - token, err := util.Random(pd.TokenSizeLong) + token, err := util.Random(pd.TokenSizeGit) if err != nil { return nil, err } @@ -2439,6 +2439,20 @@ func (g *gitBackEnd) GetVetted(token []byte, version string) (*backend.Record, e return g.getRecordLock(token, version, g.vetted, true) } +// GetUnvettedTimestamps is not implemented. +// +// This function satisfies the Backend interface. +func (g *gitBackEnd) GetUnvettedTimestamps(token []byte, version string) (*backend.RecordTimestamps, error) { + return nil, fmt.Errorf("not implemented") +} + +// GetVettedTimestamps is not implemented. +// +// This function satisfies the Backend interface. +func (g *gitBackEnd) GetVettedTimestamps(token []byte, version string) (*backend.RecordTimestamps, error) { + return nil, fmt.Errorf("not implemented") +} + // getVettedMetadataStream returns a byte slice of the given metadata stream. // // This function must be called with the read lock held. diff --git a/politeiad/backend/tlogbe/anchor.go b/politeiad/backend/tlogbe/anchor.go index 741ee9f7b..98f61711f 100644 --- a/politeiad/backend/tlogbe/anchor.go +++ b/politeiad/backend/tlogbe/anchor.go @@ -5,6 +5,7 @@ package tlogbe import ( + "bytes" "crypto/sha256" "encoding/hex" "errors" @@ -42,7 +43,7 @@ const ( // trillian tree. This root hash is submitted to dcrtime to be anchored and is // the anchored digest in the VerifyDigest. Only the root hash is anchored, but // the full LogRootV1 struct is saved as part of an anchor record so that it -// can be used to retreive inclusion proofs for any leaves that were included +// can be used to retrieve inclusion proofs for any leaves that were included // in the root hash. type anchor struct { TreeID int64 `json:"treeid"` @@ -50,6 +51,109 @@ type anchor struct { VerifyDigest *dcrtime.VerifyDigest `json:"verifydigest"` } +var ( + errAnchorNotFound = errors.New("anchor not found") +) + +// anchorForLeaf returns the anchor for a specific merkle leaf hash. +func (t *tlog) anchorForLeaf(treeID int64, merkleLeafHash []byte, leaves []*trillian.LogLeaf) (*anchor, error) { + // Find the leaf for the provided merkle leaf hash + var l *trillian.LogLeaf + for i, v := range leaves { + if bytes.Equal(v.MerkleLeafHash, merkleLeafHash) { + l = v + // Sanity check + if l.LeafIndex != int64(i) { + return nil, fmt.Errorf("unexpected leaf index: got %v, want %v", + l.LeafIndex, i) + } + break + } + } + if l == nil { + return nil, fmt.Errorf("leaf not found") + } + + // Find the first anchor that occurs after the leaf + var anchorKey string + for i := int(l.LeafIndex); i < len(leaves); i++ { + l := leaves[i] + if leafIsAnchor(l) { + anchorKey = extractKeyFromLeaf(l) + break + } + } + if anchorKey == "" { + // This record version has not been anchored yet + return nil, errAnchorNotFound + } + + // Get the anchor record + blobs, err := t.store.Get([]string{anchorKey}) + if err != nil { + return nil, fmt.Errorf("store Get: %v", err) + } + b, ok := blobs[anchorKey] + if !ok { + return nil, fmt.Errorf("blob not found %v", anchorKey) + } + be, err := store.Deblob(b) + if err != nil { + return nil, err + } + a, err := convertAnchorFromBlobEntry(*be) + if err != nil { + return nil, err + } + + return a, nil +} + +// anchorLatest returns the most recent anchor for the provided tree. A +// errAnchorNotFound is returned if no anchor is found for the provided tree. +func (t *tlog) anchorLatest(treeID int64) (*anchor, error) { + // Get tree leaves + leavesAll, err := t.trillian.leavesAll(treeID) + if err != nil { + return nil, fmt.Errorf("leavesAll: %v", err) + } + + // Find the most recent anchor leaf + var key string + for i := len(leavesAll) - 1; i >= 0; i-- { + if leafIsAnchor(leavesAll[i]) { + key = extractKeyFromLeaf(leavesAll[i]) + } + } + if key == "" { + return nil, errAnchorNotFound + } + + // Pull blob from key-value store + blobs, err := t.store.Get([]string{key}) + if err != nil { + return nil, fmt.Errorf("store Get: %v", err) + } + if len(blobs) != 1 { + return nil, fmt.Errorf("unexpected blobs count: got %v, want 1", + len(blobs)) + } + b, ok := blobs[key] + if !ok { + return nil, fmt.Errorf("blob not found %v", key) + } + be, err := store.Deblob(b) + if err != nil { + return nil, err + } + a, err := convertAnchorFromBlobEntry(*be) + if err != nil { + return nil, err + } + + return a, nil +} + // anchorSave saves an anchor to the key-value store and appends a log leaf // to the trillian tree for the anchor. func (t *tlog) anchorSave(a anchor) error { @@ -87,9 +191,10 @@ func (t *tlog) anchorSave(a anchor) error { return err } prefixedKey := []byte(keyPrefixAnchorRecord + keys[0]) - queued, _, err := t.trillian.leavesAppend(a.TreeID, []*trillian.LogLeaf{ + leaves := []*trillian.LogLeaf{ newLogLeaf(h, prefixedKey), - }) + } + queued, _, err := t.trillian.leavesAppend(a.TreeID, leaves) if err != nil { return fmt.Errorf("leavesAppend: %v", err) } @@ -114,63 +219,6 @@ func (t *tlog) anchorSave(a anchor) error { return nil } -var ( - // errAnchorNotFound is emitted when a anchor is not found when - // requesting the anchor record from a tree. - errAnchorNotFound = errors.New("anchor not found") -) - -// anchorLatest returns the most recent anchor for the provided tree. A -// errAnchorNotFound is returned if no anchor is found for the provided tree. -func (t *tlog) anchorLatest(treeID int64) (*anchor, error) { - // Get tree leaves - leavesAll, err := t.trillian.leavesAll(treeID) - if err != nil { - return nil, fmt.Errorf("leavesAll: %v", err) - } - - // Find the most recent anchor leaf - var key string - for i := len(leavesAll) - 1; i >= 0; i-- { - if leafIsAnchor(leavesAll[i]) { - // Extract key-value store key - key, err = extractKeyFromLeaf(leavesAll[i]) - if err != nil { - return nil, err - } - } - } - if key == "" { - return nil, errAnchorNotFound - } - - // Pull blob from key-value store - blobs, err := t.store.Get([]string{key}) - if err != nil { - return nil, fmt.Errorf("store Get: %v", err) - } - if len(blobs) != 1 { - return nil, fmt.Errorf("unexpected blobs count: got %v, want 1", - len(blobs)) - } - - // Decode freeze record - b, ok := blobs[key] - if !ok { - return nil, fmt.Errorf("blob not found %v", key) - } - be, err := store.Deblob(b) - if err != nil { - return nil, err - } - a, err := convertAnchorFromBlobEntry(*be) - if err != nil { - return nil, err - } - - return a, nil -} - // anchorWait waits for the anchor to drop. The anchor is not considered // dropped until dcrtime returns the ChainTimestamp in the reply. dcrtime does // not return the ChainTimestamp until the timestamp transaction has 6 @@ -341,11 +389,11 @@ func (t *tlog) anchorWait(anchors []anchor, digests []string) { int(period.Minutes())*retries) } -// anchor drops an anchor for any trees that have unanchored leaves at the time -// of function invocation. A SHA256 digest of the tree's log root at its +// anchorTrees drops an anchor for any trees that have unanchored leaves at the +// time of function invocation. A SHA256 digest of the tree's log root at its // current height is timestamped onto the decred blockchain using the dcrtime // service. The anchor data is saved to the key-value store. -func (t *tlog) anchor() { +func (t *tlog) anchorTrees() { log.Debugf("Start %v anchor process", t.id) var exitErr error // Set on exit if there is an error diff --git a/politeiad/backend/tlogbe/store/store.go b/politeiad/backend/tlogbe/store/store.go index ecb68ac45..4bc480ebe 100644 --- a/politeiad/backend/tlogbe/store/store.go +++ b/politeiad/backend/tlogbe/store/store.go @@ -36,6 +36,8 @@ type DataDescriptor struct { // BlobEntry is the structure used to store data in the Blob key-value store. // All data in the Blob key-value store will be encoded as a BlobEntry. type BlobEntry struct { + // TODO change this to digest so that we are consistent with the + // terminology used throughout the backend. Hash string `json:"hash"` // SHA256 hash of data payload, hex encoded DataHint string `json:"datahint"` // Hint that describes data, base64 encoded Data string `json:"data"` // Data payload, base64 encoded diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index 996d4239b..fbdad5c52 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -2673,7 +2673,7 @@ func (p *ticketVotePlugin) setup() error { } p.Unlock() - // Build votes cace + // Build votes cache log.Infof("ticketvote: building votes cache") for k := range started { diff --git a/politeiad/backend/tlogbe/timestamp.go b/politeiad/backend/tlogbe/timestamp.go new file mode 100644 index 000000000..7997f7e00 --- /dev/null +++ b/politeiad/backend/tlogbe/timestamp.go @@ -0,0 +1,178 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlogbe + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + + dmerkle "github.com/decred/dcrtime/merkle" + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/util" + "github.com/google/trillian" + tmerkle "github.com/google/trillian/merkle" + "github.com/google/trillian/merkle/hashers" +) + +// TODO test all of this validation + +const ( + // ProofTypeTrillianRFC6962 indicates a trillian proof that uses + // the trillian hashing strategy HashStrategy_RFC6962_SHA256. + ProofTypeTrillianRFC6962 = "trillian-rfc6962" + + // ProofTypeDcrtime indicates a dcrtime proof. + ProofTypeDcrtime = "dcrtime" +) + +// ExtraDataTrillianRFC6962 contains the extra data required to verify a +// trillian inclusion proof. +type ExtraDataTrillianRFC6962 struct { + LeafIndex int64 `json:"leafindex"` + TreeSize int64 `json:"treesize"` +} + +func verifyProofTrillian(p backend.Proof) error { + // Verify type + if p.Type != ProofTypeTrillianRFC6962 { + return fmt.Errorf("invalid proof type") + } + + // The digest of the data is stored in trillian as the leaf value. + // The digest of the leaf value is the digest that is included in + // the log merkle root. + h, err := hashers.NewLogHasher(trillian.HashStrategy_RFC6962_SHA256) + if err != nil { + return err + } + leafValue, err := hex.DecodeString(p.Digest) + if err != nil { + return err + } + leafHash := h.HashLeaf(leafValue) + + merkleRoot, err := hex.DecodeString(p.MerkleRoot) + if err != nil { + return err + } + + merklePath := make([][]byte, 0, len(p.MerklePath)) + for _, v := range p.MerklePath { + b, err := hex.DecodeString(v) + if err != nil { + return err + } + merklePath = append(merklePath, b) + } + + var ed ExtraDataTrillianRFC6962 + err = json.Unmarshal([]byte(p.ExtraData), &ed) + if err != nil { + return err + } + + verifier := tmerkle.NewLogVerifier(h) + return verifier.VerifyInclusionProof(ed.LeafIndex, ed.TreeSize, + merklePath, merkleRoot, leafHash) +} + +func verifyProofDcrtime(p backend.Proof) error { + if p.Type != ProofTypeDcrtime { + return fmt.Errorf("invalid proof type") + } + + // Verify digest is part of merkle path + var found bool + for _, v := range p.MerklePath { + if v == p.Digest { + found = true + break + } + } + if !found { + return fmt.Errorf("digest %v not found in merkle path %v", + p.Digest, p.MerklePath) + } + + // Calculate merkle root + digests := make([]*[sha256.Size]byte, 0, len(p.MerklePath)) + for _, v := range p.MerklePath { + b, err := hex.DecodeString(v) + if err != nil { + return err + } + var h [sha256.Size]byte + copy(h[:], b) + digests = append(digests, &h) + } + r := dmerkle.Root(digests) + merkleRoot := hex.EncodeToString(r[:]) + + // Verify merkle root matches + if merkleRoot != p.MerkleRoot { + return fmt.Errorf("invalid merkle root: got %v, want %v", + merkleRoot, p.MerkleRoot) + } + + return nil +} + +func verifyProof(p backend.Proof) error { + switch p.Type { + case ProofTypeTrillianRFC6962: + return verifyProofTrillian(p) + case ProofTypeDcrtime: + return verifyProofDcrtime(p) + } + return fmt.Errorf("invalid proof type") +} + +// VerifyTimestamp verifies the inclusion of the data in the timestamped merkle +// root. +func VerifyTimestamp(t backend.Timestamp) error { + // Verify digest + d := hex.EncodeToString(util.Digest([]byte(t.Data))) + if d != t.Digest { + return fmt.Errorf("invalid digest: got %v, want %v", d, t.Digest) + } + + // Verify proof ordering. The digest of the first proof should be + // the data digest. The digest of every subsequent proof should be + // the merkle root of the previous proof. + if len(t.Proofs) == 0 { + return fmt.Errorf("no proofs found") + } + if t.Digest != t.Proofs[0].Digest { + return fmt.Errorf("invalid proofs: digest %v not found", t.Digest) + } + nextDigest := t.Proofs[0].MerkleRoot + for i := 1; i < len(t.Proofs); i++ { + p := t.Proofs[i] + if p.Digest != nextDigest { + return fmt.Errorf("invalid proof %v digest: got %v, want %v", + i, p.Digest, nextDigest) + } + nextDigest = t.MerkleRoot + } + + // Verify the merkle root of the last proof is the merkle root + // that was included in the dcr transaction. + if nextDigest != t.MerkleRoot { + return fmt.Errorf("merkle root of last proof does not match timestamped "+ + "merkle root: got %v, want %v", nextDigest, t.MerkleRoot) + } + + // Verify proofs + for i, v := range t.Proofs { + err := verifyProof(v) + if err != nil { + return fmt.Errorf("invalid proof %v: %v", i, err) + } + } + + return nil +} diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index 64034cdea..dac3bbddd 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -205,12 +205,14 @@ func leafIsAnchor(l *trillian.LogLeaf) bool { return bytes.HasPrefix(l.ExtraData, []byte(keyPrefixAnchorRecord)) } -func extractKeyFromLeaf(l *trillian.LogLeaf) (string, error) { +func extractKeyFromLeaf(l *trillian.LogLeaf) string { s := bytes.SplitAfter(l.ExtraData, []byte(":")) if len(s) != 2 { - return "", fmt.Errorf("invalid key %s", l.ExtraData) + e := fmt.Sprintf("invalid key '%s' for leaf %x", + l.ExtraData, l.MerkleLeafHash) + panic(e) } - return string(s[1]), nil + return string(s[1]) } func convertBlobEntryFromFile(f backend.File) (*store.BlobEntry, error) { @@ -382,6 +384,26 @@ func (t *tlog) encrypt(b []byte) ([]byte, error) { return t.encryptionKey.encrypt(0, b) } +func (t *tlog) deblob(b []byte) (*store.BlobEntry, error) { + var err error + if t.encryptionKey != nil && blobIsEncrypted(b) { + b, _, err = t.encryptionKey.decrypt(b) + if err != nil { + return nil, err + } + } + be, err := store.Deblob(b) + if err != nil { + // Check if this is an encrypted blob that was not decrypted + if t.encryptionKey == nil && blobIsEncrypted(b) { + return nil, fmt.Errorf("blob is encrypted but no encryption " + + "key found to decrypt blob") + } + return nil, err + } + return be, nil +} + func (t *tlog) treeNew() (int64, error) { log.Tracef("%v treeNew", t.id) @@ -618,12 +640,8 @@ func (t *tlog) recordIndexes(leaves []*trillian.LogLeaf) ([]recordIndex, error) keys := make([]string, 0, 64) for _, v := range leaves { if leafIsRecordIndex(v) { - // This is a record index leaf. Extract they kv store key. - k, err := extractKeyFromLeaf(v) - if err != nil { - return nil, err - } - keys = append(keys, k) + // This is a record index leaf. Save the kv store key. + keys = append(keys, extractKeyFromLeaf(v)) } } @@ -649,7 +667,7 @@ func (t *tlog) recordIndexes(leaves []*trillian.LogLeaf) ([]recordIndex, error) indexes := make([]recordIndex, 0, len(blobs)) for _, v := range blobs { - be, err := store.Deblob(v) + be, err := t.deblob(v) if err != nil { return nil, err } @@ -712,7 +730,9 @@ type recordBlobsPrepareReply struct { // blobs contains the blobified record content that needs to be // saved to the kv store. Hashes contains the hashes of the record - // content prior to being blobified. + // content prior to being blobified. These hashes are saved to + // trilian log leaves. The hashes are SHA256 hashes of the JSON + // encoded data. // // blobs and hashes share the same ordering. blobs [][]byte @@ -1262,11 +1282,7 @@ func (t *tlog) recordDel(treeID int64) error { for _, v := range leavesAll { _, ok := merkles[hex.EncodeToString(v.MerkleLeafHash)] if ok { - key, err := extractKeyFromLeaf(v) - if err != nil { - return err - } - keys = append(keys, key) + keys = append(keys, extractKeyFromLeaf(v)) } } @@ -1394,14 +1410,8 @@ func (t *tlog) record(treeID int64, version uint32) (*backend.Record, error) { continue } - // Leaf is part of record content. Extract the kv store key. - key, err := extractKeyFromLeaf(v) - if err != nil { - return nil, fmt.Errorf("extractKeyForRecordContent %x", - v.MerkleLeafHash) - } - - keys = append(keys, key) + // Leaf is part of record content. Save the kv store key. + keys = append(keys, extractKeyFromLeaf(v)) } // Get record content from store @@ -1420,20 +1430,8 @@ func (t *tlog) record(treeID int64, version uint32) (*backend.Record, error) { // Decode blobs entries := make([]store.BlobEntry, 0, len(keys)) for _, v := range blobs { - var be *store.BlobEntry - if t.encryptionKey != nil && blobIsEncrypted(v) { - v, _, err = t.encryptionKey.decrypt(v) - if err != nil { - return nil, err - } - } - be, err := store.Deblob(v) + be, err := t.deblob(v) if err != nil { - // Check if this is an encrypted blob that was not decrypted - if t.encryptionKey == nil && blobIsEncrypted(v) { - return nil, fmt.Errorf("blob is encrypted but no encryption " + - "key found to decrypt blob") - } return nil, err } entries = append(entries, *be) @@ -1524,8 +1522,181 @@ func (t *tlog) recordLatest(treeID int64) (*backend.Record, error) { return t.record(treeID, 0) } -// TODO implement recordProof -func (t *tlog) recordProof(treeID int64, version uint32) {} +// timestamp returns the timestamp for the data blob that corresponds to the +// provided merkle leaf hash. +func (t *tlog) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trillian.LogLeaf) (*backend.Timestamp, error) { + // Find the leaf + var l *trillian.LogLeaf + for _, v := range leaves { + if bytes.Equal(merkleLeafHash, v.MerkleLeafHash) { + l = v + break + } + } + if l == nil { + return nil, fmt.Errorf("leaf not found") + } + + // Get blob entry from the kv store + key := extractKeyFromLeaf(l) + blobs, err := t.store.Get([]string{key}) + if err != nil { + return nil, fmt.Errorf("store get: %v", err) + } + + // Extract the data blob. Its possible for the data blob to not + // exist if has been censored. This is ok. We'll still return the + // rest of the timestamp. + var data []byte + if len(blobs) == 1 { + b, ok := blobs[key] + if !ok { + return nil, fmt.Errorf("blob not found %v", key) + } + be, err := t.deblob(b) + if err != nil { + return nil, err + } + data, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, err + } + // Sanity check + if !bytes.Equal(l.LeafValue, util.Digest(data)) { + return nil, fmt.Errorf("data digest does not match leaf value") + } + } + + // Setup timestamp + ts := backend.Timestamp{ + Data: string(data), + Digest: hex.EncodeToString(l.LeafValue), + } + + // Get the anchor record for this leaf + a, err := t.anchorForLeaf(treeID, merkleLeafHash, leaves) + if err != nil { + if err == errAnchorNotFound { + // This data has not been anchored yet + return &ts, nil + } + return nil, fmt.Errorf("anchor: %v", err) + } + + // Get trillian inclusion proof + p, err := t.trillian.inclusionProof(treeID, l.MerkleLeafHash, a.LogRoot) + if err != nil { + return nil, fmt.Errorf("inclusionProof %v %x: %v", + treeID, l.MerkleLeafHash, err) + } + + // Setup proof for data digest inclusion in the log merkle root + ed := ExtraDataTrillianRFC6962{ + LeafIndex: p.LeafIndex, + TreeSize: int64(a.LogRoot.TreeSize), + } + extraData, err := json.Marshal(ed) + if err != nil { + return nil, err + } + merklePath := make([]string, 0, len(p.Hashes)) + for _, v := range p.Hashes { + merklePath = append(merklePath, hex.EncodeToString(v)) + } + trillianProof := backend.Proof{ + Type: ProofTypeTrillianRFC6962, + Digest: ts.Digest, + MerkleRoot: hex.EncodeToString(a.LogRoot.RootHash), + MerklePath: merklePath, + ExtraData: string(extraData), + } + + // Setup proof for log merkle root inclusion in the dcrtime merkle + // root + if a.VerifyDigest.Digest != trillianProof.MerkleRoot { + return nil, fmt.Errorf("trillian merkle root not anchored") + } + hashes := a.VerifyDigest.ChainInformation.MerklePath.Hashes + merklePath = make([]string, 0, len(hashes)) + for _, v := range hashes { + merklePath = append(merklePath, hex.EncodeToString(v[:])) + } + dcrtimeProof := backend.Proof{ + Type: ProofTypeDcrtime, + Digest: a.VerifyDigest.Digest, + MerkleRoot: a.VerifyDigest.ChainInformation.MerkleRoot, + MerklePath: merklePath, + } + + // Update timestamp + ts.TxID = a.VerifyDigest.ChainInformation.Transaction + ts.MerkleRoot = a.VerifyDigest.ChainInformation.MerkleRoot + ts.Proofs = []backend.Proof{ + trillianProof, + dcrtimeProof, + } + + // Verify timestamp + err = VerifyTimestamp(ts) + if err != nil { + return nil, fmt.Errorf("VerifyTimestamp: %v", err) + } + + return &ts, nil +} + +func (t *tlog) recordTimestamps(treeID int64, version uint32, token []byte) (*backend.RecordTimestamps, error) { + log.Tracef("%v recordTimestamps: %v %v", t.id, treeID, version) + + // Verify tree exists + if !t.treeExists(treeID) { + return nil, errRecordNotFound + } + + // Get record index + leaves, err := t.trillian.leavesAll(treeID) + if err != nil { + return nil, fmt.Errorf("leavesAll %v: %v", treeID, err) + } + idx, err := t.recordIndexVersion(leaves, version) + if err != nil { + return nil, err + } + + // Get record metadata timestamp + rm, err := t.timestamp(treeID, idx.RecordMetadata, leaves) + if err != nil { + return nil, fmt.Errorf("record metadata timestamp: %v", err) + } + + // Get metadata timestamps + metadata := make(map[uint64]backend.Timestamp, len(idx.Metadata)) + for k, v := range idx.Metadata { + ts, err := t.timestamp(treeID, v, leaves) + if err != nil { + return nil, fmt.Errorf("metadata %v timestamp: %v", k, err) + } + metadata[k] = *ts + } + + // Get file timestamps + files := make(map[string]backend.Timestamp, len(idx.Files)) + for k, v := range idx.Files { + ts, err := t.timestamp(treeID, v, leaves) + if err != nil { + return nil, fmt.Errorf("file %v timestamp: %v", k, err) + } + files[k] = *ts + } + + return &backend.RecordTimestamps{ + Token: hex.EncodeToString(token), + Version: strconv.FormatUint(uint64(version), 10), + RecordMetadata: *rm, + Metadata: metadata, + Files: files, + }, nil +} // blobsSave saves the provided blobs to the key-value store then appends them // onto the trillian tree. Note, hashes contains the hashes of the data encoded @@ -1535,6 +1706,12 @@ func (t *tlog) recordProof(treeID int64, version uint32) {} func (t *tlog) blobsSave(treeID int64, keyPrefix string, blobs, hashes [][]byte, encrypt bool) ([][]byte, error) { log.Tracef("%v blobsSave: %v %v %v", t.id, treeID, keyPrefix, encrypt) + // Sanity check + if len(blobs) != len(hashes) { + return nil, fmt.Errorf("blob count and hashes count mismatch: "+ + "got %v blobs, %v hashes", len(blobs), len(hashes)) + } + // Verify tree exists if !t.treeExists(treeID) { return nil, errRecordNotFound @@ -1644,11 +1821,7 @@ func (t *tlog) blobsDel(treeID int64, merkles [][]byte) error { for _, v := range leaves { _, ok := merkleHashes[hex.EncodeToString(v.MerkleLeafHash)] if ok { - key, err := extractKeyFromLeaf(v) - if err != nil { - return err - } - keys = append(keys, key) + keys = append(keys, extractKeyFromLeaf(v)) } } @@ -1711,11 +1884,7 @@ func (t *tlog) blobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, if !ok { return nil, fmt.Errorf("leaf not found for merkle hash: %x", v) } - k, err := extractKeyFromLeaf(l) - if err != nil { - return nil, err - } - keys = append(keys, k) + keys = append(keys, extractKeyFromLeaf(l)) } // Pull the blobs from the store. If is ok if one or more blobs is @@ -1776,11 +1945,7 @@ func (t *tlog) blobsByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error keys := make([]string, 0, len(leaves)) for _, v := range leaves { if bytes.HasPrefix(v.ExtraData, []byte(keyPrefix)) { - k, err := extractKeyFromLeaf(v) - if err != nil { - return nil, err - } - keys = append(keys, k) + keys = append(keys, extractKeyFromLeaf(v)) } } @@ -1906,7 +2071,7 @@ func newTlog(id, homeDir, dataDir, trillianHost, trillianKeyFile, encryptionKeyF // Launch cron log.Infof("Launch %v cron anchor job", id) err = t.cron.AddFunc(anchorSchedule, func() { - t.anchor() + t.anchorTrees() }) if err != nil { return nil, err diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 0772c578d..566d5c3af 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -139,7 +139,7 @@ type plugin struct { } func tokenIsFullLength(token []byte) bool { - return len(token) == v1.TokenSizeShort + return len(token) == v1.TokenSizeTlog } // tokenDecode decodes the provided hex encoded record token. This function @@ -201,7 +201,7 @@ func tokenPrefix(token []byte) string { } func tokenFromTreeID(treeID int64) []byte { - b := make([]byte, binary.MaxVarintLen64) + b := make([]byte, v1.TokenSizeTlog) // Converting between int64 and uint64 doesn't change // the sign bit, only the way it's interpreted. binary.LittleEndian.PutUint64(b, uint64(treeID)) @@ -1290,6 +1290,61 @@ func (t *tlogBackend) GetVetted(token []byte, version string) (*backend.Record, return r, nil } +// This function satisfies the Backend interface. +func (t *tlogBackend) GetUnvettedTimestamps(token []byte, version string) (*backend.RecordTimestamps, error) { + log.Tracef("GetUnvettedTimestamps: %x %v", token, version) + + if t.isShutdown() { + return nil, backend.ErrShutdown + } + + treeID := t.unvettedTreeIDFromToken(token) + var v uint32 + if version != "" { + u, err := strconv.ParseUint(version, 10, 64) + if err != nil { + return nil, backend.ErrRecordNotFound + } + v = uint32(u) + } + + // Verify record exists and is unvetted + if !t.UnvettedExists(token) { + return nil, backend.ErrRecordNotFound + } + + // Get timestamps + return t.unvetted.recordTimestamps(treeID, v, token) +} + +// This function satisfies the Backend interface. +func (t *tlogBackend) GetVettedTimestamps(token []byte, version string) (*backend.RecordTimestamps, error) { + log.Tracef("GetVettedTimestamps: %x %v", token, version) + + if t.isShutdown() { + return nil, backend.ErrShutdown + } + + // Get tree ID + treeID, ok := t.vettedTreeIDFromToken(token) + if !ok { + return nil, backend.ErrRecordNotFound + } + + // Parse version + var v uint32 + if version != "" { + u, err := strconv.ParseUint(version, 10, 64) + if err != nil { + return nil, backend.ErrRecordNotFound + } + v = uint32(u) + } + + // Get timestamps + return t.vetted.recordTimestamps(treeID, v, token) +} + // This function must be called WITH the unvetted lock held. func (t *tlogBackend) unvettedPublish(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { // Create a vetted tree diff --git a/politeiad/backend/tlogbe/trillianclient.go b/politeiad/backend/tlogbe/trillianclient.go index 05d359031..56d117280 100644 --- a/politeiad/backend/tlogbe/trillianclient.go +++ b/politeiad/backend/tlogbe/trillianclient.go @@ -39,17 +39,12 @@ import ( // QueuedLeaf, and the inclusion proof for that leaf. If the leaf append // command fails the QueuedLeaf will contain an error code from the failure and // the Proof will not be present. +// TODO I don't use this for anything. Just return the QueuedLeafProof. type queuedLeafProof struct { QueuedLeaf *trillian.QueuedLogLeaf Proof *trillian.Proof } -// leafProof contains a log leaf and the inclusion proof for the log leaf. -type leafProof struct { - Leaf *trillian.LogLeaf - Proof *trillian.Proof -} - // trillianClient provides an interface for interacting with a trillian log. It // creates an abstraction over the trillian provided TrillianLogClient and // TrillianAdminClient, creating a simplified API for the backend to use and @@ -71,9 +66,6 @@ type trillianClient interface { leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, *types.LogRootV1, error) - // leavesByRange returns the leaves within a specific index range. - leavesByRange(treeID, startIndex, count int64) ([]*trillian.LogLeaf, error) - // leavesAll returns all leaves of a tree. leavesAll(treeID int64) ([]*trillian.LogLeaf, error) @@ -81,11 +73,10 @@ type trillianClient interface { signedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoot, *types.LogRootV1, error) - // inclusionProofs returns the log leaves and the inclusion proofs - // for a set of merkle leaf hashes. The inclusion proofs returned - // for the tree height specified by the provided LogRootV1. - inclusionProofs(treeID int64, merkleLeafHashes [][]byte, - lrv1 *types.LogRootV1) ([]leafProof, error) + // inclusionProof returns a proof for the inclusion of a merkle + // leaf hash in a log root. + inclusionProof(treeID int64, merkleLeafHashe []byte, + lrv1 *types.LogRootV1) (*trillian.Proof, error) // close closes the client connection. close() @@ -257,7 +248,8 @@ func (t *tclient) treesAll() ([]*trillian.Tree, error) { return ltr.Tree, nil } -// inclusionProof returns the inclusion proof for a trillian log leaf. +// inclusionProof returns a proof for the inclusion of a merkle leaf hash in a +// log root. // // This function satisfies the trillianClient interface func (t *tclient) inclusionProof(treeID int64, merkleLeafHash []byte, lrv1 *types.LogRootV1) (*trillian.Proof, error) { @@ -486,39 +478,6 @@ func (t *tclient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { return t.leavesByRange(treeID, 0, int64(lr.TreeSize)) } -// inclusionProofs returns the log leaves and the inclusion proofs for a set of -// merkle leaf hashes. The inclusion proofs returned for the tree height -// specified by the provided LogRootV1. -func (t *tclient) inclusionProofs(treeID int64, merkleLeafHashes [][]byte, lr *types.LogRootV1) ([]leafProof, error) { - log.Tracef("trillian inclusionProofs: %v %v %x", - treeID, lr.TreeSize, merkleLeafHashes) - - // Retrieve leaves - r, err := t.log.GetLeavesByHash(t.ctx, - &trillian.GetLeavesByHashRequest{ - LogId: treeID, - LeafHash: merkleLeafHashes, - }) - if err != nil { - return nil, fmt.Errorf("GetLeavesByHashRequest: %v", err) - } - - // Retrieve proofs - proofs := make([]leafProof, 0, len(r.Leaves)) - for _, v := range r.Leaves { - p, err := t.inclusionProof(treeID, v.MerkleLeafHash, lr) - if err != nil { - return nil, fmt.Errorf("inclusionProof %x: %v", v.MerkleLeafHash, err) - } - proofs = append(proofs, leafProof{ - Leaf: v, - Proof: p, - }) - } - - return proofs, nil -} - // close closes the trillian grpc connection. // // This function satisfies the trillianClient interface. @@ -728,32 +687,6 @@ func (t *testTClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([] return queued, nil, nil } -// leavesByRange returns leaves in range according to the passed in parameters. -// -// This function satisfies the trillianClient interface. -func (t *testTClient) leavesByRange(treeID, startIndex, count int64) ([]*trillian.LogLeaf, error) { - t.RLock() - defer t.RUnlock() - - // Check if treeID entry exists - if _, ok := t.leaves[treeID]; !ok { - return nil, fmt.Errorf("tree ID %d does not contain any leaf data", - treeID) - } - - // Get leaves by range. Indexes are ordered. - var c int64 - var leaves []*trillian.LogLeaf - for _, leaf := range t.leaves[treeID] { - if leaf.LeafIndex >= startIndex && c < count { - leaves = append(leaves, leaf) - c++ - } - } - - return nil, nil -} - // leavesAll returns all leaves from a trillian tree. // // This function satisfies the trillianClient interface. @@ -777,10 +710,10 @@ func (t *testTClient) signedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoo return nil, nil, nil } -// inclusionProofs has not been implement yet for the test client. +// inclusionProof has not been implement yet for the test client. // // This function satisfies the trillianClient interface. -func (t *testTClient) inclusionProofs(treeID int64, merkleLeafHashes [][]byte, lr *types.LogRootV1) ([]leafProof, error) { +func (t *testTClient) inclusionProof(treeID int64, merkleLeafHash []byte, lr *types.LogRootV1) (*trillian.Proof, error) { return nil, nil } diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index eeff1a4c9..f0f48c970 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -90,6 +90,48 @@ func convertBackendMetadataStream(mds backend.MetadataStream) v1.MetadataStream } } +func convertBackendProof(p backend.Proof) v1.Proof { + return v1.Proof{ + Type: p.Type, + Digest: p.Digest, + MerkleRoot: p.MerkleRoot, + MerklePath: p.MerklePath, + ExtraData: p.ExtraData, + } +} + +func convertBackendTimestamp(t backend.Timestamp) v1.Timestamp { + proofs := make([]v1.Proof, 0, len(t.Proofs)) + for _, v := range t.Proofs { + proofs = append(proofs, convertBackendProof(v)) + } + return v1.Timestamp{ + Data: t.Data, + Digest: t.Digest, + TxID: t.TxID, + MerkleRoot: t.MerkleRoot, + Proofs: proofs, + } +} + +func convertBackendRecordTimestamps(rt backend.RecordTimestamps) v1.RecordTimestamps { + md := make(map[uint64]v1.Timestamp, len(rt.Metadata)) + for k, v := range rt.Metadata { + md[k] = convertBackendTimestamp(v) + } + files := make(map[string]v1.Timestamp, len(rt.Files)) + for k, v := range rt.Files { + files[k] = convertBackendTimestamp(v) + } + return v1.RecordTimestamps{ + Token: rt.Token, + Version: rt.Version, + RecordMetadata: convertBackendTimestamp(rt.RecordMetadata), + Metadata: md, + Files: files, + } +} + // convertBackendStatus converts a backend MDStatus to an API status. func convertBackendStatus(status backend.MDStatusT) v1.RecordStatusT { s := v1.RecordStatusInvalid @@ -581,6 +623,110 @@ func (p *politeia) getVetted(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, reply) } +func (p *politeia) getUnvettedTimestamps(w http.ResponseWriter, r *http.Request) { + var t v1.GetUnvettedTimestamps + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&t); err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) + return + } + + challenge, err := hex.DecodeString(t.Challenge) + if err != nil || len(challenge) != v1.ChallengeSize { + p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) + return + } + + // Validate token + token, err := util.ConvertStringToken(t.Token) + if err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) + return + } + + // Get timestamps + rt, err := p.backend.GetUnvettedTimestamps(token, t.Version) + switch { + case errors.Is(err, backend.ErrRecordNotFound): + // Record not found + log.Infof("Get unvetted timestamps %v: %v not found", + remoteAddr(r), t.Token) + p.respondWithUserError(w, v1.ErrorStatusRecordNotFound, nil) + return + + case err != nil: + // Generic internal error + errorCode := time.Now().Unix() + log.Errorf("%v Get unvetted timestamps error code %v: %v", + remoteAddr(r), errorCode, err) + p.respondWithServerError(w, errorCode) + return + } + + log.Infof("Get unvetted timestamps %v: %v", remoteAddr(r), t.Token) + + // Setup reply + response := p.identity.SignMessage(challenge) + reply := v1.GetUnvettedTimestampsReply{ + Response: hex.EncodeToString(response[:]), + RecordTimestamps: convertBackendRecordTimestamps(*rt), + } + + util.RespondWithJSON(w, http.StatusOK, reply) +} + +func (p *politeia) getVettedTimestamps(w http.ResponseWriter, r *http.Request) { + var t v1.GetVettedTimestamps + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&t); err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) + return + } + + challenge, err := hex.DecodeString(t.Challenge) + if err != nil || len(challenge) != v1.ChallengeSize { + p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) + return + } + + // Validate token + token, err := util.ConvertStringToken(t.Token) + if err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) + return + } + + // Get timestamps + rt, err := p.backend.GetVettedTimestamps(token, t.Version) + switch { + case errors.Is(err, backend.ErrRecordNotFound): + // Record not found + log.Infof("Get vetted timestamps %v: %v not found", + remoteAddr(r), t.Token) + p.respondWithUserError(w, v1.ErrorStatusRecordNotFound, nil) + return + + case err != nil: + // Generic internal error + errorCode := time.Now().Unix() + log.Errorf("%v Get vetted timestamps error code %v: %v", + remoteAddr(r), errorCode, err) + p.respondWithServerError(w, errorCode) + return + } + + log.Infof("Get vetted timestamps %v: %v", remoteAddr(r), t.Token) + + // Setup reply + response := p.identity.SignMessage(challenge) + reply := v1.GetVettedTimestampsReply{ + Response: hex.EncodeToString(response[:]), + RecordTimestamps: convertBackendRecordTimestamps(*rt), + } + + util.RespondWithJSON(w, http.StatusOK, reply) +} + func (p *politeia) inventory(w http.ResponseWriter, r *http.Request) { var i v1.Inventory decoder := json.NewDecoder(r.Body) @@ -1227,6 +1373,10 @@ func _main() error { permissionPublic) p.addRoute(http.MethodPost, v1.GetVettedRoute, p.getVetted, permissionPublic) + p.addRoute(http.MethodPost, v1.GetUnvettedTimestampsRoute, + p.getUnvettedTimestamps, permissionPublic) + p.addRoute(http.MethodPost, v1.GetVettedTimestampsRoute, + p.getVettedTimestamps, permissionPublic) p.addRoute(http.MethodPost, v1.InventoryByStatusRoute, p.inventoryByStatus, permissionPublic) diff --git a/politeiad/testpoliteiad/testpoliteiad.go b/politeiad/testpoliteiad/testpoliteiad.go index 05e6ee76a..a765b7953 100644 --- a/politeiad/testpoliteiad/testpoliteiad.go +++ b/politeiad/testpoliteiad/testpoliteiad.go @@ -159,7 +159,7 @@ func (p *TestPoliteiad) handleNewRecord(w http.ResponseWriter, r *http.Request) } // Prepare response - tokenb, err := util.Random(v1.TokenSizeLong) + tokenb, err := util.Random(v1.TokenSizeGit) if err != nil { util.RespondWithJSON(w, http.StatusInternalServerError, err) return diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 38b0bcbff..ea4a169b8 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -12,7 +12,6 @@ type CommentVoteT int type VoteStatusT int type VoteAuthActionT string type VoteT int -type VoteErrorT int // TODO verify that all batched request have a page size limit // TODO comments count and linked from should be pulled out of the proposal @@ -109,19 +108,6 @@ const ( type ErrorStatusT int const ( - // Cast vote errors - // TODO these need human readable equivalents - VoteErrorInvalid VoteErrorT = 0 - VoteErrorInternalError VoteErrorT = 1 - VoteErrorTokenInvalid VoteErrorT = 2 - VoteErrorRecordNotFound VoteErrorT = 3 - VoteErrorMultipleRecordVotes VoteErrorT = 4 - VoteErrorVoteStatusInvalid VoteErrorT = 5 - VoteErrorVoteBitInvalid VoteErrorT = 6 - VoteErrorSignatureInvalid VoteErrorT = 7 - VoteErrorTicketNotEligible VoteErrorT = 8 - VoteErrorTicketAlreadyVoted VoteErrorT = 9 - // Error status codes ErrorStatusInvalid ErrorStatusT = 0 ErrorStatusInputInvalid ErrorStatusT = 1 @@ -817,6 +803,25 @@ type VoteStartReply struct { EligibleTickets []string `json:"eligibletickets"` } +// VoteErrorT represents an error that occurred while attempting to cast a +// ticket vote. +type VoteErrorT int + +const ( + // Cast vote errors + // TODO these need human readable equivalents + VoteErrorInvalid VoteErrorT = 0 + VoteErrorInternalError VoteErrorT = 1 + VoteErrorTokenInvalid VoteErrorT = 2 + VoteErrorRecordNotFound VoteErrorT = 3 + VoteErrorMultipleRecordVotes VoteErrorT = 4 + VoteErrorVoteStatusInvalid VoteErrorT = 5 + VoteErrorVoteBitInvalid VoteErrorT = 6 + VoteErrorSignatureInvalid VoteErrorT = 7 + VoteErrorTicketNotEligible VoteErrorT = 8 + VoteErrorTicketAlreadyVoted VoteErrorT = 9 +) + // CastVote is a signed ticket vote. type CastVote struct { Token string `json:"token"` // Proposal token diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go new file mode 100644 index 000000000..986118885 --- /dev/null +++ b/politeiawww/api/records/v1/v1.go @@ -0,0 +1,118 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package v1 + +import "fmt" + +const ( + // APIRoute is prefixed onto all routes defined in this package. + APIRoute = "/records/v1" + + // Record routes + RouteTimestamps = "/timestamps" +) + +// ErrorCodeT represents a user error code. +type ErrorCodeT int + +const ( + // Error codes + ErrorCodeInvalid ErrorCodeT = 0 + ErrorCodeInputInvalid ErrorCodeT = 1 + ErrorCodeRecordNotFound ErrorCodeT = 2 + ErrorCodeStateInvalid ErrorCodeT = 3 +) + +var ( + // TODO Add human readable error messages + ErrorCodes = map[ErrorCodeT]string{ + ErrorCodeInvalid: "error invalid", + } +) + +// UserErrorReply is the reply that the server returns when it encounters an +// error that is caused by something that the user did (malformed input, bad +// timing, etc). The HTTP status code will be 400. +type UserErrorReply struct { + ErrorCode ErrorCodeT `json:"errorcode"` + ErrorContext string `json:"errorcontext"` +} + +// Error satisfies the error interface. +func (e UserErrorReply) Error() string { + return fmt.Sprintf("user error code: %v", e.ErrorCode) +} + +// ServerErrorReply is the reply that the server returns when it encounters an +// unrecoverable error while executing a command. The HTTP status code will be +// 500 and the ErrorCode field will contain a UNIX timestamp that the user can +// provide to the server admin to track down the error details in the logs. +type ServerErrorReply struct { + ErrorCode int64 `json:"errorcode"` +} + +// Error satisfies the error interface. +func (e ServerErrorReply) Error() string { + return fmt.Sprintf("server error: %v", e.ErrorCode) +} + +// StateT represents a record state. +type StateT int + +const ( + // StateInvalid indicates an invalid record state. + StateInvalid StateT = 0 + + // StateUnvetted indicates a record has not been made public yet. + StateUnvetted StateT = 1 + + // StateVetted indicates a record has been made public. + StateVetted StateT = 2 +) + +// Proof contains an inclusion proof for the digest in the merkle root. +type Proof struct { + Type string `json:"type"` + Digest string `json:"digest"` + MerkleRoot string `json:"merkleroot"` + MerklePath []string `json:"merklepath"` + ExtraData string `json:"extradata"` // JSON encoded +} + +// Timestamp contains all of the data required to verify that a piece of +// proposal data was timestamped onto the decred blockchain. +// +// All digests are hex encoded SHA256 digests. The merkle root can be found in +// the OP_RETURN of the specified DCR transaction. +// +// TxID, MerkleRoot, and Proofs will only be populated once the merkle root has +// been included in a DCR tx and the tx has 6 confirmations. The Data field +// will not be populated if the data has been censored. +type Timestamp struct { + Data string `json:"data"` // JSON encoded + Digest string `json:"digest"` + TxID string `json:"txid"` + MerkleRoot string `json:"merkleroot"` + Proofs []Proof `json:"proofs"` +} + +// Timestamps requests the timestamps for for a record version. If the version +// is omitted, the timestamps for the most recent version will be returned. +type Timestamps struct { + State StateT `json:"state"` + Token string `json:"token"` + Version string `json:"version,omitempty"` +} + +// TimestampsReply is the reply to the Timestamps command. +type TimestampsReply struct { + RecordMetadata Timestamp `json:"recordmetadata"` + + // map[metadataID]Timestamp + Metadata map[uint64]Timestamp `json:"metadata"` + + // map[filename]Timestamp + Files map[string]Timestamp `json:"files"` +} diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index 696b12251..b7d5030b7 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -72,6 +72,9 @@ type piwww struct { Proposals proposalsCmd `command:"proposals"` ProposalInv proposalInvCmd `command:"proposalinv"` + // Record commands + RecordTimestamps recordTimestampsCmd `command:"recordtimestamps"` + // Comments commands CommentNew commentNewCmd `command:"commentnew"` CommentVote commentVoteCmd `command:"commentvote"` diff --git a/politeiawww/cmd/piwww/proposaltimestamps.go b/politeiawww/cmd/piwww/proposaltimestamps.go new file mode 100644 index 000000000..c6829de47 --- /dev/null +++ b/politeiawww/cmd/piwww/proposaltimestamps.go @@ -0,0 +1,79 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + "github.com/decred/politeia/politeiawww/cmd/shared" +) + +// recordTimestampsCmd retrieves the timestamps for a politeiawww record. +type recordTimestampsCmd struct { + Args struct { + Token string `positional-arg-name:"token" required:"true"` + Version string `positional-arg-name:"version" optional:"true"` + } `positional-args:"true"` + + // Unvetted is used to request the timestamps of an unvetted + // record. + Unvetted bool `long:"unvetted" optional:"true"` +} + +// Execute executes the recordTimestampsCmd command. +// +// This function satisfies the go-flags Commander interface. +func (c *recordTimestampsCmd) Execute(args []string) error { + + // Set record state. Defaults to vetted unless the unvetted flag + // is used. + var state rcv1.StateT + switch { + case c.Unvetted: + state = rcv1.StateUnvetted + default: + state = rcv1.StateVetted + } + + // Setup request + t := rcv1.Timestamps{ + State: state, + Token: c.Args.Token, + Version: c.Args.Version, + } + + // Send request + err := shared.PrintJSON(t) + if err != nil { + return err + } + tr, err := client.RecordTimestamps(t) + if err != nil { + return err + } + err = shared.PrintJSON(tr) + if err != nil { + return err + } + + // TODO Verify timestamps + + return nil +} + +// recordTimestampsHelpMsg is the output of the help command. +const recordTimestampsHelpMsg = `recordtimestamps [flags] "token" "version" + +Fetch the timestamps a record version. The timestamp contains all necessary +data to verify that user submitted record data has been timestamped onto the +decred blockchain. + +Arguments: +1. token (string, required) Proposal token +2. version (string, optional) Proposal version + +Flags: + --unvetted (bool, optional) Request is for unvetted records instead of + vetted ones (default: false). +` diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index be9545292..8b63b26fb 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -22,6 +22,7 @@ import ( "decred.org/dcrwallet/rpc/walletrpc" cms "github.com/decred/politeia/politeiawww/api/cms/v1" pi "github.com/decred/politeia/politeiawww/api/pi/v1" + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" www2 "github.com/decred/politeia/politeiawww/api/www/v2" "github.com/decred/politeia/util" @@ -126,6 +127,11 @@ func wwwError(body []byte, statusCode int) error { return nil } +// TODO recordsError +func recordsError(body []byte, statusCode int) error { + return nil +} + // piError unmarshals the response body from makeRequest, and handles any // status code errors from the server. Parses the error code and error context // from the pi api, in case of user error. @@ -895,7 +901,7 @@ func (c *Client) ProposalEdit(pe pi.ProposalEdit) (*pi.ProposalEditReply, error) return &per, nil } -// ProposalSetStatus sets the status of a proposal +// ProposalSetStatus sends the ProposalSetStatus command to politeiawww. func (c *Client) ProposalSetStatus(pss pi.ProposalSetStatus) (*pi.ProposalSetStatusReply, error) { statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, pi.RouteProposalSetStatus, pss) @@ -923,6 +929,33 @@ func (c *Client) ProposalSetStatus(pss pi.ProposalSetStatus) (*pi.ProposalSetSta return &pssr, nil } +// RecordTimestamps sends the RecordTimestamps command to politeiawww. +func (c *Client) RecordTimestamps(t rcv1.Timestamps) (*rcv1.TimestampsReply, error) { + statusCode, respBody, err := c.makeRequest(http.MethodPost, + rcv1.APIRoute, rcv1.RouteTimestamps, t) + if err != nil { + return nil, err + } + if statusCode != http.StatusOK { + return nil, recordsError(respBody, statusCode) + } + + var tr rcv1.TimestampsReply + err = json.Unmarshal(respBody, &tr) + if err != nil { + return nil, err + } + + if c.cfg.Verbose { + err := prettyPrintJSON(tr) + if err != nil { + return nil, err + } + } + + return &tr, nil +} + // Proposals retrieves a proposal for each of the provided proposal requests. func (c *Client) Proposals(p pi.Proposals) (*pi.ProposalsReply, error) { statusCode, respBody, err := c.makeRequest(http.MethodPost, diff --git a/politeiawww/invoices.go b/politeiawww/invoices.go index 8fe52dff5..87b66453d 100644 --- a/politeiawww/invoices.go +++ b/politeiawww/invoices.go @@ -679,7 +679,7 @@ func (p *politeiawww) processNewInvoice(ctx context.Context, ni cms.NewInvoice, // Handle test case if p.test { - tokenBytes, err := util.Random(pd.TokenSizeLong) + tokenBytes, err := util.Random(pd.TokenSizeGit) if err != nil { return nil, err } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 7f733c21c..b68f40cbb 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -23,6 +23,7 @@ import ( "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/ticketvote" piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/user" wwwutil "github.com/decred/politeia/politeiawww/util" @@ -72,7 +73,7 @@ func tokenIsValid(token string) bool { switch { case len(token) == pdv1.TokenPrefixLength: // Token is a short proposal token - case len(token) == pdv1.TokenSizeShort*2: + case len(token) == pdv1.TokenSizeTlog*2: // Token is a full length token default: // Unknown token size @@ -92,7 +93,7 @@ func tokenIsFullLength(token string) bool { if err != nil { return false } - if len(b) != pdv1.TokenSizeShort { + if len(b) != pdv1.TokenSizeTlog { return false } return true @@ -1252,6 +1253,19 @@ func (p *politeiawww) processProposalNew(ctx context.Context, pn piv1.ProposalNe }, nil } +// filenameIsMetadata returns whether the politeiad filename represents a pi +// Metadata object. These are provided as user defined metadata in proposal +// submissions but are stored as files in politeiad since they are part of the +// merkle root that the user signs and must also be part of the merkle root +// that politeiad signs. +func filenameIsMetadata(filename string) bool { + switch filename { + case pi.FileNameProposalMetadata: + return true + } + return false +} + // filesToDel returns the names of the files that are included in current but // are not included in updated. These are the files that need to be deleted // from a proposal on update. @@ -1557,6 +1571,17 @@ func (p *politeiawww) processProposalSetStatus(ctx context.Context, pss piv1.Pro func (p *politeiawww) processProposals(ctx context.Context, ps piv1.Proposals, isAdmin bool) (*piv1.ProposalsReply, error) { log.Tracef("processProposals: %v", ps.Requests) + // Verify state + switch ps.State { + case piv1.PropStateUnvetted, piv1.PropStateVetted: + // Allowed; continue + default: + return nil, piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusPropStateInvalid, + } + } + + // Get proposal records props, err := p.proposalRecords(ctx, ps.State, ps.Requests, ps.IncludeFiles) if err != nil { return nil, err @@ -1595,6 +1620,9 @@ func (p *politeiawww) processProposalInventory(ctx context.Context, inv piv1.Pro // Determine if unvetted tokens should be returned switch { + case u == nil: + // No user session. Remove unvetted. + pir.Unvetted = nil case u.Admin: // User is an admin. Return unvetted. case inv.UserID == u.ID.String(): @@ -2581,6 +2609,9 @@ func (p *politeiawww) setPiRoutes() { p.addRoute(http.MethodPost, piv1.APIRoute, piv1.RouteProposalSetStatus, p.handleProposalSetStatus, permissionAdmin) + p.addRoute(http.MethodPost, rcv1.APIRoute, + rcv1.RouteTimestamps, p.handleTimestamps, + permissionPublic) p.addRoute(http.MethodPost, piv1.APIRoute, piv1.RouteProposals, p.handleProposals, permissionPublic) diff --git a/politeiawww/politeiad.go b/politeiawww/politeiad.go index db8e4438b..37a89c76e 100644 --- a/politeiawww/politeiad.go +++ b/politeiawww/politeiad.go @@ -355,10 +355,40 @@ func (p *politeiawww) getUnvetted(ctx context.Context, token, version string) (* return &gur.Record, nil } -// getUnvettedLatest returns the latest version of the unvetted record for the -// provided token. -func (p *politeiawww) getUnvettedLatest(ctx context.Context, token string) (*pd.Record, error) { - return p.getUnvetted(ctx, token, "") +// getUnvettedTimestamps retrieves the timestamps for an unvetted record. +func (p *politeiawww) getUnvettedTimestamps(ctx context.Context, token, version string) (*pd.RecordTimestamps, error) { + // Setup request + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return nil, err + } + gut := pd.GetUnvettedTimestamps{ + Challenge: hex.EncodeToString(challenge), + Token: token, + Version: version, + } + + // Send request + resBody, err := p.makeRequest(ctx, http.MethodPost, + pd.GetUnvettedTimestampsRoute, gut) + if err != nil { + return nil, err + } + + // Receive reply + var reply pd.GetUnvettedTimestampsReply + err = json.Unmarshal(resBody, &reply) + if err != nil { + return nil, err + } + + // Verify challenge + err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) + if err != nil { + return nil, err + } + + return &reply.RecordTimestamps, nil } // getVetted retrieves a vetted record from politeiad. @@ -397,10 +427,40 @@ func (p *politeiawww) getVetted(ctx context.Context, token, version string) (*pd return &gvr.Record, nil } -// getVettedLatest returns the latest version of the vetted record for the -// provided token. -func (p *politeiawww) getVettedLatest(ctx context.Context, token string) (*pd.Record, error) { - return p.getVetted(ctx, token, "") +// getVettedTimestamps retrieves the timestamps for an unvetted record. +func (p *politeiawww) getVettedTimestamps(ctx context.Context, token, version string) (*pd.RecordTimestamps, error) { + // Setup request + challenge, err := util.Random(pd.ChallengeSize) + if err != nil { + return nil, err + } + gvt := pd.GetVettedTimestamps{ + Challenge: hex.EncodeToString(challenge), + Token: token, + Version: version, + } + + // Send request + resBody, err := p.makeRequest(ctx, http.MethodPost, + pd.GetVettedTimestampsRoute, gvt) + if err != nil { + return nil, err + } + + // Receive reply + var reply pd.GetVettedTimestampsReply + err = json.Unmarshal(resBody, &reply) + if err != nil { + return nil, err + } + + // Verify challenge + err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) + if err != nil { + return nil, err + } + + return &reply.RecordTimestamps, nil } // pluginInventory requests the plugin inventory from politeiad and returns diff --git a/politeiawww/records.go b/politeiawww/records.go new file mode 100644 index 000000000..a264f0838 --- /dev/null +++ b/politeiawww/records.go @@ -0,0 +1,235 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "runtime/debug" + "strings" + "time" + + pdv1 "github.com/decred/politeia/politeiad/api/v1" + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + "github.com/decred/politeia/util" +) + +func (p *politeiawww) processTimestamps(ctx context.Context, t rcv1.Timestamps, isAdmin bool) (*rcv1.TimestampsReply, error) { + log.Tracef("processTimestamps: %v %v", t.Token, t.Version) + + // Get record timestamps + var ( + rt *pdv1.RecordTimestamps + err error + ) + switch t.State { + case rcv1.StateUnvetted: + rt, err = p.getUnvettedTimestamps(ctx, t.Token, t.Version) + if err != nil { + return nil, err + } + case rcv1.StateVetted: + rt, err = p.getVettedTimestamps(ctx, t.Token, t.Version) + if err != nil { + return nil, err + } + default: + return nil, rcv1.UserErrorReply{ + ErrorCode: rcv1.ErrorCodeStateInvalid, + } + } + + var ( + recordMD = convertTimestampFromPD(rt.RecordMetadata) + metadata = make(map[uint64]rcv1.Timestamp, len(rt.Files)) + files = make(map[string]rcv1.Timestamp, len(rt.Files)) + ) + for k, v := range rt.Metadata { + metadata[k] = convertTimestampFromPD(v) + } + for k, v := range rt.Files { + files[k] = convertTimestampFromPD(v) + } + + // Unvetted data blobs are stripped if the user is not an admin. + // The rest of the timestamp is still returned. + if t.State != rcv1.StateVetted && !isAdmin { + recordMD.Data = "" + for k, v := range files { + v.Data = "" + files[k] = v + } + for k, v := range metadata { + v.Data = "" + metadata[k] = v + } + } + + return &rcv1.TimestampsReply{ + RecordMetadata: recordMD, + Files: files, + Metadata: metadata, + }, nil +} + +func (p *politeiawww) handleTimestamps(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleTimestamps") + + var t rcv1.Timestamps + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&t); err != nil { + respondWithRecordError(w, r, "handleTimestamps: unmarshal", + rcv1.UserErrorReply{ + ErrorCode: rcv1.ErrorCodeInputInvalid, + }) + return + } + + // Lookup session user. This is a public route so a session may not + // exist. Ignore any session not found errors. + usr, err := p.getSessionUser(w, r) + if err != nil && err != errSessionNotFound { + respondWithRecordError(w, r, + "handleTimestamps: getSessionUser: %v", err) + return + } + + isAdmin := usr != nil && usr.Admin + tr, err := p.processTimestamps(r.Context(), t, isAdmin) + if err != nil { + respondWithRecordError(w, r, + "handleTimestamps: processTimestamps: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, tr) +} + +func convertRecordsErrorCode(pluginID string, errCode int) rcv1.ErrorCodeT { + switch pluginID { + case "": + // politeiad v1 API errors + switch pdv1.ErrorStatusT(errCode) { + case pdv1.ErrorStatusInvalidRequestPayload: + // Intentionally omitted. This indicates an internal server error. + case pdv1.ErrorStatusInvalidChallenge: + // Intentionally omitted. This indicates an internal server error. + case pdv1.ErrorStatusRecordNotFound: + return rcv1.ErrorCodeRecordNotFound + } + } + // No records error code found + return rcv1.ErrorCodeInvalid +} + +func respondWithRecordError(w http.ResponseWriter, r *http.Request, format string, err error) { + var ( + ue rcv1.UserErrorReply + pe pdError + ) + switch { + case errors.As(err, &ue): + // Record user error + errMsg := fmt.Sprintf("Records user error: %v %v %v", + remoteAddr(r), ue.ErrorCode, rcv1.ErrorCodes[ue.ErrorCode]) + if ue.ErrorContext != "" { + errMsg += fmt.Sprintf(": %v", ue.ErrorContext) + } + util.RespondWithJSON(w, http.StatusBadRequest, + rcv1.UserErrorReply{ + ErrorCode: ue.ErrorCode, + ErrorContext: ue.ErrorContext, + }) + return + + case errors.As(err, &pe): + // Politeiad error + var ( + pluginID = pe.ErrorReply.Plugin + errCode = pe.ErrorReply.ErrorCode + errContext = pe.ErrorReply.ErrorContext + ) + e := convertRecordsErrorCode(pluginID, errCode) + switch e { + case rcv1.ErrorCodeInvalid: + // politeiad error does not correspond to a user error. Log it + // and return a 500. + ts := time.Now().Unix() + if pluginID == "" { + log.Errorf("%v %v %v %v Internal error %v: error code "+ + "from politeiad: %v", remoteAddr(r), r.Method, r.URL, + r.Proto, ts, errCode) + } else { + log.Errorf("%v %v %v %v Internal error %v: error code "+ + "from politeiad plugin %v: %v %v", remoteAddr(r), r.Method, + r.URL, r.Proto, ts, pluginID, errCode, errContext) + } + + util.RespondWithJSON(w, http.StatusInternalServerError, + rcv1.ServerErrorReply{ + ErrorCode: ts, + }) + return + + default: + // politeiad error does correspond to a user error. Log it and + // return a 400. + m := fmt.Sprintf("Records user error: %v %v %v", + remoteAddr(r), e, rcv1.ErrorCodes[e]) + if len(errContext) > 0 { + m += fmt.Sprintf(": %v", strings.Join(errContext, ", ")) + } + log.Infof(m) + + util.RespondWithJSON(w, http.StatusBadRequest, + rcv1.UserErrorReply{ + ErrorCode: e, + ErrorContext: strings.Join(errContext, ", "), + }) + return + + } + + default: + // Internal server error. Log it and return a 500. + t := time.Now().Unix() + e := fmt.Sprintf(format, err) + log.Errorf("%v %v %v %v Internal error %v: %v", + remoteAddr(r), r.Method, r.URL, r.Proto, t, e) + log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) + + util.RespondWithJSON(w, http.StatusInternalServerError, + rcv1.ServerErrorReply{ + ErrorCode: t, + }) + } +} + +func convertProofFromPD(p pdv1.Proof) rcv1.Proof { + return rcv1.Proof{ + Type: p.Type, + Digest: p.Digest, + MerkleRoot: p.MerkleRoot, + MerklePath: p.MerklePath, + ExtraData: p.ExtraData, + } +} + +func convertTimestampFromPD(t pdv1.Timestamp) rcv1.Timestamp { + proofs := make([]rcv1.Proof, 0, len(t.Proofs)) + for _, v := range t.Proofs { + proofs = append(proofs, convertProofFromPD(v)) + } + return rcv1.Timestamp{ + Data: t.Data, + Digest: t.Digest, + TxID: t.TxID, + MerkleRoot: t.MerkleRoot, + Proofs: proofs, + } +} diff --git a/util/convert.go b/util/convert.go index a29ef00e7..bbaecbf9a 100644 --- a/util/convert.go +++ b/util/convert.go @@ -35,18 +35,18 @@ func ConvertSignature(s string) ([identity.SignatureSize]byte, error) { // prefix. func ConvertStringToken(token string) ([]byte, error) { switch { - case len(token) == pd.TokenSizeShort*2: + case len(token) == pd.TokenSizeTlog*2: // Tlog backend token; continue - case len(token) != pd.TokenSizeLong*2: + case len(token) != pd.TokenSizeGit*2: // Git backend token; continue case len(token) == pd.TokenPrefixLength: // Token prefix; continue default: return nil, fmt.Errorf("invalid censorship token size") } - // If the token length is an odd number of characters, a 0 digit is - // appended onto the string to prevent a hex.ErrLenth (odd length - // hex string) error when decoding. + // If the token length is an odd number of characters, append a + // 0 digit as padding to prevent a hex.ErrLenth (odd length hex + // string) error when decoding. if len(token)%2 == 1 { token = token + "0" } From 396dd88922685b3b4268be2e7cd76b89f4538d4c Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 10 Jan 2021 20:23:49 -0600 Subject: [PATCH 206/449] fixes --- politeiad/backend/tlogbe/timestamp.go | 8 ++- politeiawww/api/records/v1/v1.go | 7 +-- ...posaltimestamps.go => recordtimestamps.go} | 51 ++++++++++++++++++- politeiawww/cmd/shared/client.go | 2 +- politeiawww/records.go | 6 +-- 5 files changed, 64 insertions(+), 10 deletions(-) rename politeiawww/cmd/piwww/{proposaltimestamps.go => recordtimestamps.go} (61%) diff --git a/politeiad/backend/tlogbe/timestamp.go b/politeiad/backend/tlogbe/timestamp.go index 7997f7e00..eca3e81a5 100644 --- a/politeiad/backend/tlogbe/timestamp.go +++ b/politeiad/backend/tlogbe/timestamp.go @@ -131,9 +131,13 @@ func verifyProof(p backend.Proof) error { return fmt.Errorf("invalid proof type") } -// VerifyTimestamp verifies the inclusion of the data in the timestamped merkle -// root. +// VerifyTimestamp verifies the inclusion of the data in the merkle root that +// was timestamped onto the dcr blockchain. func VerifyTimestamp(t backend.Timestamp) error { + if t.TxID == "" { + return fmt.Errorf("data has not been included in dcr tx yet") + } + // Verify digest d := hex.EncodeToString(util.Digest([]byte(t.Data))) if d != t.Digest { diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go index 986118885..229d4095e 100644 --- a/politeiawww/api/records/v1/v1.go +++ b/politeiawww/api/records/v1/v1.go @@ -82,7 +82,7 @@ type Proof struct { } // Timestamp contains all of the data required to verify that a piece of -// proposal data was timestamped onto the decred blockchain. +// record data was timestamped onto the decred blockchain. // // All digests are hex encoded SHA256 digests. The merkle root can be found in // the OP_RETURN of the specified DCR transaction. @@ -98,8 +98,9 @@ type Timestamp struct { Proofs []Proof `json:"proofs"` } -// Timestamps requests the timestamps for for a record version. If the version -// is omitted, the timestamps for the most recent version will be returned. +// Timestamps requests the timestamps for a specific record version. If the +// version is omitted, the timestamps for the most recent version will be +// returned. type Timestamps struct { State StateT `json:"state"` Token string `json:"token"` diff --git a/politeiawww/cmd/piwww/proposaltimestamps.go b/politeiawww/cmd/piwww/recordtimestamps.go similarity index 61% rename from politeiawww/cmd/piwww/proposaltimestamps.go rename to politeiawww/cmd/piwww/recordtimestamps.go index c6829de47..fb8d73be7 100644 --- a/politeiawww/cmd/piwww/proposaltimestamps.go +++ b/politeiawww/cmd/piwww/recordtimestamps.go @@ -5,6 +5,10 @@ package main import ( + "fmt" + + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" "github.com/decred/politeia/politeiawww/cmd/shared" ) @@ -57,11 +61,56 @@ func (c *recordTimestampsCmd) Execute(args []string) error { return err } - // TODO Verify timestamps + // Verify timestamps + err = verifyTimestamp(tr.RecordMetadata) + if err != nil { + return fmt.Errorf("verify record metadata timestamp: %v", err) + } + for k, v := range tr.Metadata { + err = verifyTimestamp(v) + if err != nil { + return fmt.Errorf("verify metadata %v timestamp: %v", k, err) + } + } + for k, v := range tr.Files { + err = verifyTimestamp(v) + if err != nil { + return fmt.Errorf("verify file %v timestamp: %v", k, err) + } + } return nil } +func verifyTimestamp(t rcv1.Timestamp) error { + ts := convertTimestamp(t) + return tlogbe.VerifyTimestamp(ts) +} + +func convertProof(p rcv1.Proof) backend.Proof { + return backend.Proof{ + Type: p.Type, + Digest: p.Digest, + MerkleRoot: p.MerkleRoot, + MerklePath: p.MerklePath, + ExtraData: p.ExtraData, + } +} + +func convertTimestamp(t rcv1.Timestamp) backend.Timestamp { + proofs := make([]backend.Proof, 0, len(t.Proofs)) + for _, v := range t.Proofs { + proofs = append(proofs, convertProof(v)) + } + return backend.Timestamp{ + Data: t.Data, + Digest: t.Digest, + TxID: t.TxID, + MerkleRoot: t.MerkleRoot, + Proofs: proofs, + } +} + // recordTimestampsHelpMsg is the output of the help command. const recordTimestampsHelpMsg = `recordtimestamps [flags] "token" "version" diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index 8b63b26fb..1079f45c7 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -129,7 +129,7 @@ func wwwError(body []byte, statusCode int) error { // TODO recordsError func recordsError(body []byte, statusCode int) error { - return nil + return fmt.Errorf("%v %s", statusCode, body) } // piError unmarshals the response body from makeRequest, and handles any diff --git a/politeiawww/records.go b/politeiawww/records.go index a264f0838..2b41e26e9 100644 --- a/politeiawww/records.go +++ b/politeiawww/records.go @@ -135,11 +135,12 @@ func respondWithRecordError(w http.ResponseWriter, r *http.Request, format strin switch { case errors.As(err, &ue): // Record user error - errMsg := fmt.Sprintf("Records user error: %v %v %v", + m := fmt.Sprintf("Records user error: %v %v %v", remoteAddr(r), ue.ErrorCode, rcv1.ErrorCodes[ue.ErrorCode]) if ue.ErrorContext != "" { - errMsg += fmt.Sprintf(": %v", ue.ErrorContext) + m += fmt.Sprintf(": %v", ue.ErrorContext) } + log.Infof(m) util.RespondWithJSON(w, http.StatusBadRequest, rcv1.UserErrorReply{ ErrorCode: ue.ErrorCode, @@ -185,7 +186,6 @@ func respondWithRecordError(w http.ResponseWriter, r *http.Request, format strin m += fmt.Sprintf(": %v", strings.Join(errContext, ", ")) } log.Infof(m) - util.RespondWithJSON(w, http.StatusBadRequest, rcv1.UserErrorReply{ ErrorCode: e, From 15f137293b73396a28c89da7442bb2df09f1648b Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Mon, 11 Jan 2021 17:59:03 -0300 Subject: [PATCH 207/449] tlogbe: Add tests for comments CmdNew and pi CmdCommentNew. --- politeiad/backend/tlogbe/comments_test.go | 225 +++++++++++++++++++++ politeiad/backend/tlogbe/pi_test.go | 173 ++++++++++++++++ politeiad/backend/tlogbe/testing.go | 88 ++++++-- politeiad/backend/tlogbe/tlogbe_test.go | 91 +++------ politeiad/backend/tlogbe/trillianclient.go | 22 -- 5 files changed, 505 insertions(+), 94 deletions(-) create mode 100644 politeiad/backend/tlogbe/comments_test.go create mode 100644 politeiad/backend/tlogbe/pi_test.go diff --git a/politeiad/backend/tlogbe/comments_test.go b/politeiad/backend/tlogbe/comments_test.go new file mode 100644 index 000000000..989e1ca8a --- /dev/null +++ b/politeiad/backend/tlogbe/comments_test.go @@ -0,0 +1,225 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlogbe + +import ( + "encoding/hex" + "errors" + "testing" + + "github.com/decred/politeia/politeiad/api/v1/identity" + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/plugins/comments" + "github.com/google/uuid" +) + +func TestCmdNew(t *testing.T) { + tlogBackend, cleanup := newTestTlogBackend(t) + defer cleanup() + + id, err := identity.New() + if err != nil { + t.Fatal(err) + } + settings := []backend.PluginSetting{{ + Key: pluginSettingDataDir, + Value: tlogBackend.dataDir, + }} + + commentsPlugin, err := newCommentsPlugin(tlogBackend, + newBackendClient(tlogBackend), settings, id) + if err != nil { + t.Fatal(err) + } + + // New record + md := []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs := []backend.File{ + newBackendFile(t, "index.md"), + } + rec, err := tlogBackend.New(md, fs) + if err != nil { + t.Error(err) + } + + // Helpers + comment := "random comment" + tokenRandom := hex.EncodeToString(tokenFromTreeID(123)) + parentID := uint32(0) + invalidParentID := uint32(3) + + uid, err := identity.New() + if err != nil { + t.Error(err) + } + + // Setup new comment plugin tests + var tests = []struct { + description string + payload comments.New + wantErr *backend.PluginUserError + }{ + { + "invalid comment state", + comments.New{ + UserID: uuid.New().String(), + State: comments.StateInvalid, + Token: rec.Token, + ParentID: parentID, + Comment: comment, + PublicKey: uid.Public.String(), + Signature: commentSignature(t, uid, comments.StateInvalid, + rec.Token, comment, parentID), + }, + &backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusStateInvalid), + }, + }, + { + "invalid token", + comments.New{ + UserID: uuid.New().String(), + State: comments.StateUnvetted, + Token: "invalid", + ParentID: parentID, + Comment: comment, + PublicKey: uid.Public.String(), + Signature: commentSignature(t, uid, comments.StateUnvetted, + rec.Token, comment, parentID), + }, + &backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusTokenInvalid), + }, + }, + { + "invalid signature", + comments.New{ + UserID: uuid.New().String(), + State: comments.StateUnvetted, + Token: rec.Token, + ParentID: parentID, + Comment: comment, + PublicKey: uid.Public.String(), + Signature: "invalid", + }, + &backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusSignatureInvalid), + }, + }, + { + "invalid public key", + comments.New{ + UserID: uuid.New().String(), + State: comments.StateUnvetted, + Token: rec.Token, + ParentID: parentID, + Comment: comment, + PublicKey: "invalid", + Signature: commentSignature(t, uid, comments.StateUnvetted, + rec.Token, comment, parentID), + }, + &backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusPublicKeyInvalid), + }, + }, + { + "comment max length exceeded", + comments.New{ + UserID: uuid.New().String(), + State: comments.StateUnvetted, + Token: rec.Token, + ParentID: parentID, + Comment: commentMaxLengthExceeded(t), + PublicKey: uid.Public.String(), + Signature: commentSignature(t, uid, comments.StateUnvetted, + rec.Token, commentMaxLengthExceeded(t), parentID), + }, + &backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusCommentTextInvalid), + }, + }, + { + "invalid parent ID", + comments.New{ + UserID: uuid.New().String(), + State: comments.StateUnvetted, + Token: rec.Token, + ParentID: invalidParentID, + Comment: comment, + PublicKey: uid.Public.String(), + Signature: commentSignature(t, uid, comments.StateUnvetted, + rec.Token, comment, invalidParentID), + }, + &backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusParentIDInvalid), + }, + }, + { + "record not found", + comments.New{ + UserID: uuid.New().String(), + State: comments.StateUnvetted, + Token: tokenRandom, + ParentID: parentID, + Comment: comment, + PublicKey: uid.Public.String(), + Signature: commentSignature(t, uid, comments.StateUnvetted, + tokenRandom, comment, parentID), + }, + &backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusRecordNotFound), + }, + }, + { + "success", + comments.New{ + UserID: uuid.New().String(), + State: comments.StateUnvetted, + Token: rec.Token, + ParentID: parentID, + Comment: comment, + PublicKey: uid.Public.String(), + Signature: commentSignature(t, uid, comments.StateUnvetted, + rec.Token, comment, parentID), + }, + nil, + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + // New Comment + ncEncoded, err := comments.EncodeNew(test.payload) + if err != nil { + t.Error(err) + } + + // Execute plugin command + _, err = commentsPlugin.cmdNew(string(ncEncoded)) + + // Parse plugin user error + var pluginUserError backend.PluginUserError + if errors.As(err, &pluginUserError) { + if test.wantErr == nil { + t.Errorf("got error %v, want nil", err) + return + } + if pluginUserError.ErrorCode != test.wantErr.ErrorCode { + t.Errorf("got error %v, want %v", + pluginUserError.ErrorCode, + test.wantErr.ErrorCode) + } + return + } + + // Expecting nil err + if err != nil { + t.Errorf("got error %v, want nil", err) + } + }) + } +} diff --git a/politeiad/backend/tlogbe/pi_test.go b/politeiad/backend/tlogbe/pi_test.go new file mode 100644 index 000000000..2b6490e39 --- /dev/null +++ b/politeiad/backend/tlogbe/pi_test.go @@ -0,0 +1,173 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlogbe + +import ( + "encoding/hex" + "errors" + "testing" + + "github.com/decred/politeia/politeiad/api/v1/identity" + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/plugins/comments" + "github.com/decred/politeia/politeiad/plugins/pi" + "github.com/google/uuid" +) + +func TestCommentNew(t *testing.T) { + tlogBackend, cleanup := newTestTlogBackend(t) + defer cleanup() + + id, err := identity.New() + if err != nil { + t.Fatal(err) + } + settings := []backend.PluginSetting{{ + Key: pluginSettingDataDir, + Value: tlogBackend.dataDir, + }} + + piPlugin, err := newPiPlugin(tlogBackend, newBackendClient(tlogBackend), + settings, tlogBackend.activeNetParams) + if err != nil { + t.Fatal(err) + } + + // Register comments plugin + tlogBackend.RegisterPlugin(backend.Plugin{ + ID: comments.ID, + Version: comments.Version, + Settings: settings, + Identity: id, + }) + + // New record + md := []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs := []backend.File{ + newBackendFile(t, "index.md"), + } + rec, err := tlogBackend.New(md, fs) + if err != nil { + t.Error(err) + } + + // Helpers + comment := "random comment" + tokenRandom := hex.EncodeToString(tokenFromTreeID(123)) + parentID := uint32(0) + + uid, err := identity.New() + if err != nil { + t.Error(err) + } + + // Setup comment new pi plugin tests + var tests = []struct { + description string + payload comments.New + wantErr *backend.PluginUserError + }{ + { + "invalid comment state", + comments.New{ + UserID: uuid.New().String(), + State: comments.StateInvalid, + Token: rec.Token, + ParentID: parentID, + Comment: comment, + PublicKey: uid.Public.String(), + Signature: commentSignature(t, uid, comments.StateInvalid, + rec.Token, comment, parentID), + }, + &backend.PluginUserError{ + ErrorCode: int(pi.ErrorStatusPropStateInvalid), + }, + }, + { + "invalid token", + comments.New{ + UserID: uuid.New().String(), + State: comments.StateUnvetted, + Token: "invalid", + ParentID: parentID, + Comment: comment, + PublicKey: uid.Public.String(), + Signature: commentSignature(t, uid, comments.StateUnvetted, + rec.Token, comment, parentID), + }, + &backend.PluginUserError{ + ErrorCode: int(pi.ErrorStatusPropTokenInvalid), + }, + }, + { + "record not found", + comments.New{ + UserID: uuid.New().String(), + State: comments.StateUnvetted, + Token: tokenRandom, + ParentID: parentID, + Comment: comment, + PublicKey: uid.Public.String(), + Signature: commentSignature(t, uid, comments.StateUnvetted, + tokenRandom, comment, parentID), + }, + &backend.PluginUserError{ + ErrorCode: int(pi.ErrorStatusPropNotFound), + }, + }, + // TODO: bad vote status test case. waiting on plugin architecture + // refactor + { + "success", + comments.New{ + UserID: uuid.New().String(), + State: comments.StateUnvetted, + Token: rec.Token, + ParentID: parentID, + Comment: comment, + PublicKey: uid.Public.String(), + Signature: commentSignature(t, uid, comments.StateUnvetted, + rec.Token, comment, parentID), + }, + nil, + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + // New Comment + ncEncoded, err := comments.EncodeNew(test.payload) + if err != nil { + t.Error(err) + } + + // Execute plugin command + _, err = piPlugin.commentNew(string(ncEncoded)) + + // Parse plugin user error + var pluginUserError backend.PluginUserError + if errors.As(err, &pluginUserError) { + if test.wantErr == nil { + t.Errorf("got error %v, want nil2", err) + return + } + if pluginUserError.ErrorCode != test.wantErr.ErrorCode { + t.Errorf("got error %v, want %v", + pluginUserError.ErrorCode, + test.wantErr.ErrorCode) + } + + return + } + + // Expecting nil err + if err != nil { + t.Errorf("got error %v, want nil", err) + } + }) + } +} diff --git a/politeiad/backend/tlogbe/testing.go b/politeiad/backend/tlogbe/testing.go index ee6fe4160..446bbedf9 100644 --- a/politeiad/backend/tlogbe/testing.go +++ b/politeiad/backend/tlogbe/testing.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -14,13 +14,22 @@ import ( "io/ioutil" "os" "path/filepath" + "strconv" "testing" + "github.com/decred/dcrd/chaincfg/v3" v1 "github.com/decred/politeia/politeiad/api/v1" + "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/api/v1/mime" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" + "github.com/decred/politeia/politeiad/plugins/comments" "github.com/decred/politeia/util" + "github.com/google/trillian" + "github.com/google/trillian/crypto/keys" + "github.com/google/trillian/crypto/keys/der" + "github.com/google/trillian/crypto/keyspb" + "github.com/marcopeereboom/sbox" ) func newBackendFile(t *testing.T, fileName string) backend.File { @@ -102,8 +111,59 @@ func newBackendMetadataStream(t *testing.T, id uint64, payload string) backend.M } } +func commentSignature(t *testing.T, id *identity.FullIdentity, state comments.StateT, token, comment string, parentID uint32) string { + t.Helper() + + // Create signature + msg := strconv.Itoa(int(state)) + token + + strconv.FormatInt(int64(parentID), 10) + comment + b := id.SignMessage([]byte(msg)) + return hex.EncodeToString(b[:]) +} + +func commentMaxLengthExceeded(t *testing.T) string { + t.Helper() + + return `puqgtmjuiztnwakzqtmybjgwluizndahxihosjwyqsyqrhxdkhwchnhtbgamigdakzyjcnjvubufifolxgmygyhoewgyjwicucaypgxrgsenjpangvtslpfbojvcgsxzqyginsjrtcaggdcienduzuxnnpmdcvxtklnsmlanjheywshijznprrvnizoxrihxstcdksqvhvqjeneeadzpjgqtdrlmhjqkxfsjcgoewisbacchagpzisvttyurpijyzpugkpufyhifaivfzgjmkpxtdydaebidbpypcnmsmvaktchcqkkrwvaahlrrdzzxpjmlshtmugrzjqyvehfnkievnmubitaeewabnqnbqcacctfiojiifozfnpvooawiutxxzcjddqpybnrsyxtfzwiomoqpbpowifsbgfsvhvcaucwjnqfnwecqfugakimobvnguyqsgzjxstovohyzvezjuwdwyopymdpoaqwesghiuzmdwzlfcnubpvfefedyrllzqyfgntcaaazmfqxknfkquiyplyhqqipixmlqjedtafcwmkfemmzdvbgkatlcwnsuzjuvofpwznczcdozlqzaiqpumugnubbsfhvoogfenjsqlenxptlaostdwpcorusmijrbqecttwokirwomuktngwluwmlthypopbinyxqrnxplzpipzitqpaqdreelqjlxqsbgxbwhmrmauiicjndjrhrqnaucbezopzrqpxcjobicxqnswftjqrhhhnicxuymdefodhmyscrzonmrihvljgdxqvjeqxmayilhitwwtnmwdcspcmnwocvksitrhiqslzavwjlvzcjwzliikcqpyllmtfrqgfwafalfklhhnhmoejdsgppdvugblyjusljcqjftktuzsqkmbueocfdadrfsbzxldiagxjuydthitfwrirvnbshdzdnbbyxaizemggkndccgjolevpfjovyacfrzvjabkgvwguxpujfancqdebfcbhlwluetcfnddixdovydgoyngxjkxelxarphjmbcbfutmcoulmbzyjxkupqcsaxsojoqpxdvsaijvhppdmdstszbgjzylwebvjvijruioazjkqccsluqsydrkqckluwealsrgxvnnuhvtuxpxdtkjguwplpmynysdysccjqhulkpcnujituvnattmvhhndyvizusmdxvpbvxzbiogawboasipmjgkdezulkmtkcrziqnyrjcupucjtkovpeqnbrqunhricblwqctencvspsahiltoopwqthxypkfovijqbwxahefszdgpwickvurthbxdkczicvzazssynuoyqcrhxxrctsrbvflbkpuprywtasmkkgcojaiylazybdtdnkjuehxjjsftkayjpcnwrngblzjsbcqauxpskcumnbubiodlqzberjmosvvufzgqbtwprhkscsdfppiifzsxroddracraxaesninyzsnnzjlnybrasfudvrjaetulogbzmbxjehlqotyfygfvkvyxkthdljkbwnvxrowjbecngutbuajppguteagmucqgwihzsktxscfepcghchwnnpkozysgpzrdjsozighofgsibuzccolbzpovmooxrkzdizoibmcmhakjwzwnqftgtgawgwzxnwxxdpgxvwxqszjhvihxfggfgxdpseiqkqtuspygomxuwxhumkiltapyfpumxogrjtzkvdomitflsesindlqvcgvxzogmzssqakheixjvzzqgxwzfjqlirtqvuddfevkzltnhwefzalsxkxqnwngdovnzwrvmvbtsykemmwipxlxdxhknyszyfmflmlcvpojnwasvmdfwahuavjajqnzigwvahjphrwreugmeromzotybxqybslneictseznajjkcxmueyroqamairkyzzdpilbusgqufyietrcbertanevmtrfnuxvgyppskpbcligxgbsmrkjnqytfvycbxtmztkzfrvsbuffuhvmuafhrqvupuecyutvxgdmhfrhcngdmqpecvpfjmmxcjbjjtaehnfluibufrmggdaxlenggmzpbwfzkpqymbsjgjcbugiruxyfbpbjovnentghytlitcgksmhxwrwlpkieqmhqrivaqgcllqljvtperdcxpgdbhdotjtvqvwkpqapsxignvhqfhuradjuksickefuuqsmwdktbcpmgynjdrekffvultianlrhyuxjhltcmzhighfddopzpkxidrnwmlczawcaiulvzxlmaulmzlrlclhajplfuxcgsvoedgkltrbqxyfplgoizapfsckkhrtnpqntlectvfzkrcgdioonbjdlrzpyrsanmvijbycaopumafijkjxubibqwlqrywaqjvtvymqiqkdevlugedamqwiqmtdwzotxbzrltbificgouieeosjaokznhqdthzyttabrowgwxrmwobgmupikxduufpktbdfjganpazjbtcweloxmabgexlixwrdolgoxxfwvjrjommozgehlxaqgxphqbaugaiqxpkruwgcvuyadkfwwsqdpwxapmlvpwjvqmdvqtltkrzfzxuirvwxsmajrzzpfcncpnfjzfugipiuuvqsvfbycrarjcrgjspycerxlvhbcahrkjtlcjhcesptpvqcjinmetwuxygrpksmsyvjfpeibljmswjkfdlpqbpfidmdtilclbntvspvsmxstarlrmvzszrwiayncldnpknchgbjerojunmotgujxgeurpabcguhyvsyoyrhlttbcjjraioxblzszobfyiahhwzwgjlkuknuexpyuovlyfhaxhyjgaxzdbvaejdiuqtwiqjnbadsjudlsexlvjnroqihcbfrawmlfczbfvqoyodzmaynbnmtrewqhutjtycfnajrqwjhjfclzopvlbpmqgiugwnarkeerqttlpbxpeqvdiqaohojoukhpjkqaaqkodogtprbtdzbqnjeqyqoffuxxikorythdmtybwpelcgjukzarpsgnrpxesxlbjmbkplhopxxmxjbbwtyueriuaotsjthvkpgjcnbthxgcctexpnacxypzerlzvjlnqcqipiavtmsekfsrrprxerhkkxoldhhvtyojqknkhatpmqavufkepwkxfknokwehwbnrosnhhsdkyesxhcflzmqkflltfhyatqjmcbcptkbumwiwakfszerchqivrhhmxxnhlnhjyfhklpiojwwerznvbzzdjkxajxwbytstchdmyjrdpjhxonxswpigtnmifkmyyqwzldmkknajfhbphedtzxewrtgobofdjdezxdrhvijxbegepjhicvgymqydculiitzasldrsoiafdeunfnfjedcqgmtnmendgirvfdlkalnexxrxmobwbomjqiqsrwrctvhxytthlwfsalhgjpnvrwyxpcphdiejgbrerhsgduegxvxeudadbjyzptcnassvmlwvlitzozbxgrhwivppyovmgsekvdwznqwimgsigwgavdrhcjnrfyxftybncjmcpoqssujqechwazbmkkicqobzlcfjwxfiocikzaumoigufxouavurmmhdhjdhpztswphjrqcwnvkiqnlpbbsqkkehpqgewunuukrzxkfjqxyuojqhpdiswiwphhoedwmexwfsnelglgqdcfpmohpslcknyjcniodjrayrgbmimcfxoowebjseeauffehxisbiggrjndkeilsxuuhopqqlurzlkovsltarcplfyufukelymxltmolhoqeaodsbionrzxjfnibzovmakvclmglnzsytiquajwcjkmmbkwmgmemmubaxpwfupahimisngzejhtsrtzntpiapcefvlsavelcxptkwwefdonenzgsffrjqehlotebkibkqrxkxbuirmmbwfolnkzcaphbywfwrnenppkupcayglmexcziijxliospthmkhtolpdqfaesmpilekvlocoznhpffneomzjuzjxdnutzqgeajxdfmjwhgqkoiuxosrdpkqutroqpfmnolirpodgbcojbwtbzdeqkybpsnwityouywhfxjefdzgvhoxpwskgnngleedntvnnodhdmsdzqpykocejfobcybuxsifgzqfowltiwvxixqdjesopcaotqytncetdazkiddisfhiycwrhoswtrbiijxvvkgaewbpyzxyyiavckfjtpusjetkobhsglbeolcfxcvqdyvkffvxebvsooawgrcerpbtdrdrglzzglhsvaayatsoztyvzsqftvsuhkoektnfiplxbhcgulztmuvugfuexlmvjhpgytagekkbztmgmehesxfcbmvzjwznlcrpeunxtuwquzggndtpcouuxeyowrcbbdsxzeqiawzefflxvjuzawisfgsmdfmgunmiuhfpfinnwbrmbjfgnunsbqxyodkrcbceqvktqmxfgwckcngxynosxlhtxfqfxoqnupgdgtkqqglvzminnkikckknruaddbknopkqmhakqdlsbylftqsprjxkxdjucmfvdnlawhpskfzowlosriwotzmoalsonnibipimopnrdgrvvxwcexarmkvazrdogampkxbcgsjjvhqzoorjijkjdtyqbldvhdywedykkshyjmohwnxcezfwbkarxfyrfkhqndlxtwbukjcemtcvexigpqqcpvddszemjemrnekquwveerxbbchxyjffnlbpkehtjracvxaobtcgamkbfbrdcdjtsuxfoawfqxjfhdpqnxknocspqoldjgqjeepyvhsjncchfznbypgsoorpuchvdhrfrrytjrrqoyccybopakqhsfygjcciiwbkivyivxxsfhkguxqemtnlvdaucxvlotbvxkvsbsothermhpmkamfxnyaibdalvlimqfytryphtnkshxaexgzosubcccdpkqsggsdserjickunguvtcipnimxgbyaigzwgrqpkbgiwsxfzojqlyhnvszfebsehdxuckbpnqfwqsydsmewcnnzbradqavgdolbiazilxojlpkhatvtxdvnlipdurriudbgmiffdnzhkrytlzizgfztbrygfrjummhslxlhwiunsnebyomrzcrbjhzlfeuecqbuvypfjqrbkmxyamqoleedabkhubjiorbyuilgxigehkeyadafcokuojmzwbxorbiauxmlygurgnrnvqskljhlrzfkgvszgqfgmgcddqgzkynbydlnxenngbajpnctmqjsptserccmsayaldyzlpzelmijnioaiuvggxeeibmeakjkzyuzykjxvbfxregohbwtcgvrngvxwgdemwcsngkmqczexwdxbpqualctwfyhwsrzbhwympqzhstqqbnurcdyrnlzrugwfzgjeoiccvpeicjzhjfeshbxcqhirlaejhgbswlkzvvfkxuxgmnpofscsxmyopoxalhhfzthcysdscfhchuwxucrxceyiytvohcpfoayagdrwdcdlirdbbsxzwmluwhcpxsbzkhokvnwdwfbzmmythphvuoyhmvwfitrbqzmpwucatsyxycwrelkradmhvweizmlkqbghdihgbxogepflvdiaxlzbuxswflhuqbbtllluatwduadfnfzftfrrokdktmwgdsrmaylkalomtsowoxwoqaixuxomcaibatwqelmahlkohuregcxnjgamhbxtxzyeizuuageosusredimvavcedekzqunkkjolkeyvpcifkhvbynhpfacmyfpfxnvqzllnmfpxdceieelnizylbwgngeqhybkdxkcttyvczcfkbbvkrderwmldizptcfhabwsqgyxgcisoaoolupmoxywjpliuxkjljrqkatfohqmbjgmeoxrsuqfuudnxxtyymegeqqcqrjsasieftcfvhcwgdjgcwsjsftbnibnbcptcoymnpqkszlxrauqvnasslhivdxojoabagbqvtnqvjbblysijnrkmmxlobcolfpnrbekmttcxcuhcnlucnnjglzrwnivlrirhvewmxvmxuserdtgzmaaoihytnbnaxyogyzuvkanwyiajykuzdpkqkqjcxiozqovnmnugpvogfggypjcqcrxfeumryypqkyohqetmviytfyiygrgamlklxzovpvklymzhifebmluwinwijutszkhlbspovyainxlorbkzcgeuhjxlsxtwgpshrtwqbycmssuoucmecehpilhnxycrcztltszcevblubvejiunthsbyukrcckazvosvtqwoajlokcctlvimoeoetpwnmkedwlaxdvjdcrppjvqoptacwhkkrfuvkbdihczwnorcxmfaawwysegpqqxsmlhqymctucwcyjlfbjbuukrfrwgukwwonxrjnuplubhxvclqbnedenhihdgzbziwvkwszzvwbekhhdthnkqgxhlrpfkhuvqlrfbjqwwzyhmrlsdhiiynhnkoqvxcezxtgbzjduppkqxdwublnuuptqdghvnzmgurtoauiqzapwiahnoknsjcpxajeqmwrniuuszcxzeoguuobjslqudfgtsvkyvnzayfsdjrhkiswwtriscpnkhxwznrgndwlrhzqacqcqoyzqphutleiqexhialulplfmizqyekwdksymdxpldpzrvecuskamlfqinkrcejwjsajtunwyspddiklupgsteiwulivomjiwquqaablnsfgdcgvxeqtotqnaqknhcxqhrlwrpqkmolyerzzmnpbufvpyqcnxmfczcjdydqejsiwkndvaksqiwndtjwwybepduxmqtvnldqafwuikwwrbhowbdkneupafwpegwtyncyijljjecfylqkuxexqijkktogiuchpvpfhlobwdpoctmmwtebblogvnstazekvnxzlyeoepwfkafedfuzusktgfwyqjhqagcwvmdjifymcvbfocikyvnmrcpwcqwaxahbefhnlsdioaajjfbuhelofqjoweeacfiutaawjddrepdhccpvdaabceywalvwoqntlsugmexzkiymlvfsrbemctangeksmmlvfdejedwleylgldzpgupycochiixthdunckpkifkowfbkmndjnsgzyvcafjcgxcknirojflydhpufrbbcoartxnzwghwmgxguvybrlqvrycdflkenlgrtgxtpvfjefenqnnglmmaewgvabmclvrzphxhfbjajipbwfskbmnmhxjqyqxivehqfwvavplnxgumfuzbcmhpjtioiibijltztrnzxlghyrywnwkyilugldeuvquzpigsyovzabsspziqbooemzkmfzgmihjkpftqvwgoeymohdskdahtzawrjibklpsmhtrhqtrsdverrmzfrmjbzojvnbfvvfdzhsrrywmyvovvhioyfajzustjysjefgonzydbqtraymsdlzbshgfcvwaodnkuvferujaewbqgbbatixdgemqxutnwgsgyoltwwkahlojrmxhntiolfiouyftkjmhmtoksxfxennhfgvwvtklygxqhlsyewitlnbaevoghedmpgkkgtvhderaojtxbgovdpiqkcffevuggzsyrntrmvojffhirultxzzpfqdulzpdgssogtfhqjdlorplcyivsfcrlrnhkeqjixdthwkkytqdynwygqofobcslheqjsbfvmabagkqbmggevavmhjxnswmfacupdyaleflgrrywjmtgmznqynzijxudjfsegpkmqgnjyfswvhengsrlbrwugsmyaydkoaewwlnkstvlpklvbolfqbtvztzznwtpmaqpcqaoskmyhksvmgdtifyszpwdekjfzuifrlpruislaabhwfrpsjdofuhnqr + uxzbyjqgucfwoadtrjowszpnajylbkxlggbafgxhgmytpogkxbsvejafypjtjpxlkfuwayzbmgidouwaqsrreppbiapwonzhbisyzvomrrktiylsjkfkgsilrrobibhqhrndbmuvvogiiofwfsouhgfqftsztapefdumtiocbizzvsmmhayikkdzmpahefvcglmcevawpdcbrxonwykpmfiryzucktpomdhuyvaobqzmzxvkjoksabmoowtzkihitwpvxtrvhbffvwhjyozieqovvmzxnumlsfdbjuznddlmqyrtxbdyqmawpnubepnfkghucnztkbunnrcfshlfsageaqlgpvwfgzkxhsgofomzgkmljtniyaxeiecrrbfojnxubivwobzrsqjhlpyyrcplqwyuskjpteizazvmlxgqowxxqrpztaedlrkpnyuisjaazsjyqryewlfkupwezowpkfddafcyadgtwyeuqnsijwgainyftbogzmqqvuusegadirouinruoyrbcsuodjgeytxoydjfksbatodexjlihbfzesiahlgzmubdyoytcnpffonhversyiecojahysecgcjkolatltayfoxhizfsygfihtncfgnekcazcxtgpiquhxuvkcfbrxuntvtxvldjzulrgikcixieyrrauqyreirdoxmjwnngcguvzrzflanndokugwecfgrdxwafgskgzadfiffkfejdpvtgxzvitzjeyalvlryaukmpwrgjsimrltbditowolohcobodoeoaqhcblqovdntzfpobgosfckdfjuawhgkrvjznvyiburrufelothjemvltiiehibrtapdfufpgccpqyirgnncdgmrtmpomrbsczoktvjbxqqhzqcdsqppwjrcndjeemclkhuiembvdxgunlrbqqnrudffvhpdbldgefwxzzazclqltrmnyqkemnvfmuyftytotvqslldrirdgvtbafrhzglrzovuvgyvxwgjfawxphtxwxnbvjqnfzrvsixmspcdvlrstorerzzziecbycvhhuybkywyefouctykxugrgemmgkxwjcizksttpjglljfqxmlztpltyypphmzccrmbirgmmdszgfikrhcyhtcwjpgfvrjkqgfijhwjjxsiiecowugxoocaqsqqqxmplsqbbtifeevnqgksxwamxdzkjseeemvfdjytfmbbknnnkrixxkrbbkaxfaxjaotkiyqsvnreouoxacvjpkliigglsvigmgeqcrfvuvijplnkfinziungckosmxffochsbpmjpcvrogwmbjsurqurnhaofjtbmsgwnegnrfpxblnpogzmcfrkmdyospxlyxsivhduzksebfgiiueukzqismrdkqxmrrmmykwnxvjkclwbgzsncbzfxfhyqhrnbcakrmvtwnyfalohwaaajxwxfgwurcqdgcowwrmlxzjyzxihcmjluewonndzqpppcpexiynziwgvxerssbfaxtbeyfzzmfiitkinfoldbljjfteudjjqkqjdvxzifamkthldnctmodkezbniywsbbmmfswrnbtpqsotbfykrdaqqzizkmedwrxfgkxuvrnmckmtndnomcgefaotncukyfarlgqohwijtklbjjoqypytzjokvvbvxlwfjjtzsoimzqbrqlwymkbqcwhymzixtarzdirmnmmpewrbmztqviwwjcjzojgbxezjkbzkylxoigvkyjtupggiyuxplumdumuyepqrieofoejzvhhkjnuotmcbgprmsfhqcbsnehhzyuylcieurloynmpjkfmevitqmsattztrdqbhurxelyhbqjysmjocvnxgqngqztmqshwlfxksofjfkmwnckjvbzgncqbxwclkdjrpkccdoxtsciymhdxerhzaniuxtnjtewybyfvhwrmerghlhudosdbclqyjqrkwukablsfwadknkdimtnozytnfkepkuecrkdljgohcupaymqpdiyvdazomgunmwvsrreftqcjqibvbzdwmkexogqtlszoxwmizmkikcwmeqndrehvhnmahlrwqjjbxhzoutpuccapztxkbmuywnvtetctouofqtdafkirequutkeoisozumccsqmabfumputsfjkbftoxufbaggrasydnnrntyraarsxlbncezswbmqbnhpsrhewovqqozorvklxarkgdaiofdvguailfmowbetpvotwbfliyrbelgpigdcbkdyiufokckqvtxfuvgjronpqkitdbfitkeppuajlxxljttjivllatbqutkailadklcnrvecnkgebtoulxnfwabyljkwxeeevoclxhtlnhquyxvillbyhnfkiiwopbmfibrqrgxhtejfeilrhqlujkjksadeypnmqiqqhrsfhsreorcsboszgpcnutlzbvcxgvweyctxidpqfdqymggvpumnnasdjqutwgedshghssnqtruhoibbkenpbbydwsbeatozxhgqejmhljqcmpxarfnfunodezrhcbedvuuewkwxtixlujbvfryajdyiiocremfobtinatyssdhqqaxfaqslqyadgxgiamocdokrnvaukcwsrklrzfveiacpockgybmnsuqavaxtdnkupzsvztaeejsznrjjkgmsccrsbdynlutjpgkpxydthodzhwqihqepwsmsbjyezgmmiowaojczpkhbdxqensgongxlcupaokokcqrkrzxgupndxddomsqdyizmsgczuocknuutinjwdzgwnbslxhilosreycphtxljqdxpgmsdodbexhpazhtsuxwzaejmgnurjorqaoyshjojebpnrysbggnickzrzvzpggjiunglyhhrtcdogqotlhhkaazwoooourfnxuyxsddwdartkurqmfkigbhhfcazaznbxtjfjeakqszdwxwifxctygymmmkgpafebukrfgxngxwshsywckqjosbwcgfseucapjcmxthudzdlzvjkejvbqgpnksjgzaqbsdlypzgdbiiinlztpiwcszyuclzhwyxiovgnmyoyobrgeeahktiylmtzyamzazxfiieefcrrugwkqtjzgjouosqvbimtywvmilpwghqaaqjwokxnfigdlairvhjaccsqcveuegkvrwreytvizrjukhlfmzryjqyqlxagcrrtoxhscgzursqhtvnaffaupwxkpsyzilbhcgdnhwpkcwgclztodkywcrvjfhcatobzpftbetupysquhoavyjbqwpjrybibttnrmrveehfyfmxvfpwnxzpbumfesvqaddgiltjwtttbroadwyhufeqcuqjzldyjzftbjnmslndmejmkkokywsqipyhpsdpubndvhtpzgsmozjtcofiajjohnenyysoeuhcztvqpyicnvspskasllriuisrkbpxlggpplmwlqrsogcdqkujkxrdynqqpahmvkvytrxeeednwgzomuxxzptmednfzzaqggfjusqfqtzxnagtkovjinalffpsdjsskogcnupzdqtialmxuwrnfkergrjgqfnkohedxlfuaqlgukrfjrtdqhfkzdktkspnbokplyuepacbweonicbxlupglkxockrsnaqarzuuswxcsfdwmucbqwwsdhhcwovvstrenwdnmmkqhasohfrchmsosspvafmxtrglggrtvopxysrhormaifpfruenlfvgnpcxiwsialuyoueobnbtwvvtttlhjmsgfhicuvahtpmgpssjstyhezamfrzpbkzduneoxlgnowthyrrltmsghvmmixnkadybsaxgnatzwqlhgwxzyvcpdjkyrzttjhlrjvhpwfmiymvmzudtpnrgdjjsgutsdsjogzddoyczxjzerynnpfxrkkjhlofmdgsaadzsecftpubclnybmrmsjuepywujofzqhqpbgjiqkrrtuqpbvkdurmkyvnpfctvnomcqxfaebtcqbmwenanvxpesjjrkzbaxromczhjripdrnmgygcjzetkyarponwoiijjvagttcyqechcxykihgkjrpdoatzefopgawgyetlxifqergchypiyddabxnhlnafjksvkeedhncnazrywehboujtlyznvgmwoiggvbwgqmonebeoofktanpnlcrnlaojikhnhffhltwvjhtwocoefhpxukyjzqbkwzrftehxwpyjoqzwkunfetytirykgrtqqchqbpifhplffjorfqfiouonuhqjmrnzmfuzelewestiyzeavzoaexmqaybufknqojhhygwbqypdpmfdcfkwducxilhswebqpdlnqrbomcpquzflccznamnvauxwpyewzgmrfeatrlcbwmganwcieuppwuetqvmfnmgpttzchjljleepigcgobasmbzvgcuhaxnhqkvylyxkwhejyvajwtscxsbfxayiscmmbxxdcnwqdqtlgaetjazxtifsypbphqcjhgmqypkqvwsonslxgilctltiutmdxzqfpnhlvqmytiarnntracwcyokebcaeyxocjjontqetfbfngghsppgcoczhdhgkudyvxdyqusccmacfhiuueqhtqczlvsioijpytnjniyyihncxpxtbkgwtdloydbkzmgttimmkszcfdotxqaxjuzurvssbebjmabirsymdfdzseocksrqarbtltlavwetykcaakjhxxotmubsezdyhjevbxtrexjdsxrjdqysuhgudurhaycwzippcnuefarkaoocdlocvxokxrseugqqliceiqwmmifvhznzxwlgoxejiqgtoshtdkygjibinqxkngslzoboaxxzbbffpintrljvknhykyqexatnpmnwyeiemgvqlabrifhxptzbfhooyhligwbgyfgelunqowqtlrkdpzdbfddyjeowyamsxhpleeetezemjaseggoxlmbndjzsjhgeeyyfgtrrvdfqramgsnisdwhvmxbgwuwgqzlglzqrvzzyzjiitiiywmjcobbevdsplimdxaxirsljtuthukxygtoxnxkzjecvyopogdwikccgoehabhwqxyppqwklvrshvkvdrrymawivhhdpblpesimmceugzdqajywkmijyeidtzvyfaiaiaacozqzmdompzefqbnsamlopccvwqwcetqgogzwjzqaepfiuvcthkqwsbactudtivfdnsarfmkgfmzkfejcixpribvqixybiuouqpcwnlqohouayilyveehemivumhjylikdbrtjabneqzzhdgomgpehpwhqswcudssypzdsiuelxmfzibtilcszxalrpapnwzncgtgqxgbgvcikvkgnwgxqzbqimsgvtmjzfxhocgefukyuuejklebvauoanawkhbcomjjguozffdiktwxzfqowhvxaaflafubdkaczfvlrdipasaxqaakpyfcjawaojnhnmckczwptgzqltpzgmwkuesiokzicblrchzdoosceouvziahsdjxhczusgduehdbppfmeantrszapodkgfseseuwazdemdnlgrxgkntszpngscckogzhhfimqgajvpocozxacticiicfvvsxajcnkurdtduxfbvbufxzaeidsbdlxfkveicjmuapolhmbtvghkuuivzgaszxyidoawsnookpegzigwdgsknaegtkhfqjiuntqiggsfojvdagipwcxuodktecqeumoolzahoxctpyybefcnsyibspckqaarnxyfvfyvkfxsblfbepbqbqjlsevstxppzcpptwwiybkffcbistqymsrgqtgkkxhkndkxceclxrqvxrrsznruwvahgbxdcnsvcxoqavdoacpeptqnskppqzezmueuolwoufnzrmbadqnkahkdjbmtdneysmveemmtftvwfwcrwmqivjefznmisgpguuvuqwlniibjenfldawtgkpehcfjzrzexgxeiztoccjuuxyrvsujirjqemvvsfvgiyafpxnyauoldlbadvrhdqovpmrnbmnfsdiuqkqospohpmqexopgrwpfywkuoaiqaykpawwnmtmefjhswcprovhayixxioykbwvzfdzzrkddodoehmqdnxzlerfwxgpyhmnruhowjqqbivdnxlxgvfxabpfmzpfvnckfvyzbnecbvvmiklimafwuvzlgpzaxzouhvbtjpvsymucoxlrrsfforyojzjjwcylknthcnofgnbwtqkmhfpdlcqoepqpwijsmgalcpwwlpjtbienduufgypnhvpiuazpifcygwfvccmhxisohvajcbublmeysbpqpejlndipijfbghqjhuwnoyuuhvwafgjbtpxywexndhchwfhuhulgntfxmazqilpoichgkqmdhsursnmjxhccbpuwloljeseegqtqpzoxkqwrcjnljnxipfkrqqkeznuqkwnazcblowsspxjiccgynktnxgtgwymdswefsyylvbibatwvqybgekmahhmgskkaqykpamyzhljdxufuoakanrmwyfooqdvfoikcavpjiotjfhwbjfdjapvuyoxutrvdhcpujzqmqcpjauksvspowniqbkgojgwpofohgvexfrjjmzniacjjgspfzeiodzrqfinqgkgapuiyclcphndswchgltzrizurtnanztsjeppwbycygaphuvjswzimwqtgnpfdjttvfegwzulbkecjzynuecsybfezbckengazkjsbvxcxnziyzikjbbiqmsffffcwvimggyjncvoqjyzfbyayitdlsohxynnlnhyxgzjbdromuochcpchcgyghybmqzgbfhgwmvbilshgqkcqsdnntnmiccdevdzgzxmtbmfzkphkymmkezesuewegtpdzxfcrphrueimuujyizbjvvopkgsgxicnpxszrytngvhsnaqdjnpplipmvrhkzqdabzjjrlppukiewcptptudsrhehncjeccpxtiiyialhtvlaswxgdjevtfaptcimydyrbcvmwmqrqwfgimiibpduadwehwowoifelcizyhgfqqlsjzvzidqvykozhlpnypviuemuzvukpaceodtwmdbiaqmboygolxkbkbkchwruehizkutazayfcpimufyvqeugnuolchhmhcgmhzxpyneihncfqamwuqqslcaxgfnbjgrohesvtrgtogwvjiobbdvotzvybtkuoimiltbzdpltaksquilugjpttwrcabtblbtkbscnpufmzijopwlghowwphqjibmgpxwimakahdkdyrtoevvudgismpntvzsqglgbnahwtucisqskjyllgcdylxwdfwrgiwonmhpfhjzshdxyksagqvkkshxhvapunfrdgzlhuenyxseebopssfdoqbwrqkyypjkfrnlaxyelnkffsxmcjekctyouclkassfevztiwsdzmfheeiwonfmtzhviwjtspilkgakdppuseqaoreujdauofuetkczagtnzrdgaympguyxjlyleakrqfhgqtwqwsxxparchjsnjotrjxnpvrmsjrjydjohvjjbnqiqjtgbjmdooctoirjetufxxhfihmyjbthezchpslaesrykhweuktictxrplwvpysajaswbpwlsxpmwknlaufqkipqkxdjkxglkyrnkqtzgxsbjtkyqfpqqutlfxjmoykfcxefjxxelpldarkxmavxzivlhpzezhpyygugseoakqaxpasulakoqsmdvisxjghplvougrlhzmxyowiyxtbngnnueravqzilogrxrwtzypkcnfodabkqavnidkqhnlyksesozdhsaimnerofciqzcjyokbjuoseerbpskwrboleuvtskzzszandzcdkitmgumhsnpxiancb + cynoosgbueqerwmxtpaxvqzwrovpwjtyqnvxdqydmiiweqrftljbsdceeggswkvwfjozkeeepleepnblpzrrcxcsjklofdtrrmrktspfibysowzashhijgmmsbpeldpworazzxiterkhvianuhhgznsxaqhamjupgnikizxhmwbyxhioqgsoxregsngxdlvxelafcgjuitvzxsvnbamxwtamnrknhbaiuebokacqyladyoapnuxurhnllyiaxodlyjsmbudzssuouyeiutdbauntftbbpsptfobvckkvntaarcwwzrikwcnijxnprsjyewecryiksjrlyebtwjvunibrqmjygefzyiosbuyckitvjzhoqehrvtldjdtezynimmdihekdeznlwpmsqcjwkanxcslcklcpolrkorvhfhshryytvhwsjphnlmsiwedxwivfswfwuymyicqcxpyarygvdauizsfvotryrwjkmgigpkophwwphendxkmmzlevhjvinkuhvsoftpovtacqrwdfelmkfcxeghgdcmdedgeevouaufcpovsxphgjxsyizfcgyawhcgkmkwxyvmnwwydcnxoarbanziagjamoauhvogakohrnkrrujxkdpihmhycloeintuglvgtxpiauzcipmxlzthvuetzjgrmkadsxgkekuaimxrwzpwjnvasqprwjdwtikmhcbexxdqyrfefljyhysysnqbfxxrwhxoawtdcanyadrmtfzdqcvpnqzrxosjsgezlwizlfgyzszjnoscuvvstihwfsyfmtiskbbknieypewpopkpntzufxudnwgtztcnxabgvhmfwnvzosmdjxsvneanqwrjpalsvdziximdkewhtlmogccipmeozakxeablrvckldqzeusagxtmiycjyblhzxnwcrfaupompgvhmxshpfsemntpvroswbccrpshphhmnaxqdbvipcubhiwjifhapteakrhqtuoepautkpwjynkrnrmedhqmmutfnwavxccthwdbxiezbfvvowtqnlkgsdbxjjekosvpdpkdbbjmrcsizhyxosnylcgqvrvbsnmfkqasxvthrwkzkcamcsgovrxpqqnlletczeefmoucgynmxkzlmfwdhubevmhklsjdqurnaxbglckmqxrfeaitojbbfedmhanqmzdqkojqnsuhscnrevxmadrjdlekdotkuzfbsjcylibblirvzafmfbbbtsqpxmndyahnwgjlafkzhuejsxdcnyiqamfkzyknfqrcuphwirwtqeabclntzyxesrrxvbrqelzjwpvyxumhnggkoldvjkzadyamccpxvahhnbmjgzdskttzfdjzxcbwanepnvbujblcbfiwouuehzgyycskxtbgrdunbatajhxtsoahvmlieszqxsekbaedtlunjjnqdmzybxnslonhsewrewslzdwnpcobaasuppoiuieddbydzaxdnlpdsjawzdgckhyzsaziadltotkgunxzbfbdxiywznkakqhhbjsgcrmuduwaskbboqpuiapsvufllatjmoycuwqponkylzhihnuvttxaxhbbdawcblzswejhhofujvityspkgayisxaztzocmhgbhxujdssacytlrkpkaqvpkxumaxikqkpyoiekzelomlxxfproafjtbzalqlixbqozsizhzuduebwugxsfpshztbeqmtlwmkzptbupoirwpywwpygeuuylyjkfqmwjuktaadjxgorhvoldlznhowaoiygnxewxbbalkgntylcmgsqtxlvkjigwjbgzsstlamediuvvhyemloshmevvmamtshanmmqkbuhatccvzltzwcpnmnxsbxmifwsxeepxjryfxuwyydgwdvwkfnrdlicxvuwvlgcyewpwwrciiqojaikmtqhcjpqxhknqjwztnmxdkchadljowljznhxzcbabgulovadvjoabkjwveisojhuqupulxcyfrcrwzrhygudmtjwxtjqrcluvnbdtsiabnuwicgcaaaxochhesyridnfbbpdoymvnrdeaobnlkcucxxueiyovqlhxyudzgzxhjdqcroxkokbvlotguovugviuvznqhxbzmjvzkcvymcerpyioktghsugyufruxdxyujncdjpvwvnufrthgmzvkyxtehkllbqmokonhqjgkzuufzaqkbyfeoghvcphjxwxxvaqvnjchtndtnagyjbycizvraihjztzbdorqhodfmlqgpbzbdmioergynvrlusqkeyzlartfoquklszfacqzkgzjllyjmfzfsnntndnmxbvshsdpesrbmgkvrwvvfubmscypyoetfvomwnlplcwfhrmmtpwhnzplbtxofdpdfzkteqdudxhmrryvojcyxbipqotevaqrlcghvuychyuohiptipxhbnmwrhuhfxsmffqphvfvirwarqffphkxrcsrnikfxqfeqpeecufnzmqgyebjccxjktkrvsdyoyfukocqupmugzbcuffzwhfylxdhtlglozmzmjcithgflxdrljqntyyfhpvupznqwmxvqmonmtrctzpkwtxwgeubptalrbsapdycvjbqrgyneegqvnphwlmnidfzhexalneorgghidhbaeugnnozbzjsuulnsilskffixlyuodvmorhjgjhuucgfpvegloqaaievbvidykuvfukkguvclndtblhorbgftpltcjexcfhrqykewjrlvnhopgmebpqfxafogsufmezsdetopoyvscionixqtccrlnibakqssmrefzrbrjkfkgrwsqamlvncneyqviqpayozbgjwrykmueqzxvjzuazpskimezgmhdqakxvekvghbvmnpkhkrlrydojtjclvjmedxfrrdiurtpzvrxttahfnxbfiqzuyggdfevpgdprmcoonodlwufovzontnybwuqosuljgjlxwwbzgtermcpdpmfdfjirzufevsjudjpbghtrqlnahgjyspmmznrdcghqptwhjsuxosneriyjzbufwekzcltieagiftxdeyhzlompkyvfbulocczzokpkvsfvpilhbbsvpunalidhizbkjvelohinqssmlxltjusnzczjohiedjqeqzvtlphurwhbheyhljgkukuavgmztxmdblocaczdyrdnmzedduaairhrhxebyqvmwegreuwmbjnwsxarjcubzesagrksthupylndnpwtqoczzpsqtbhkukkaicsljoskcfkutadhgspnedtglqjdudlyhcqpbkntgshsejsnihayqkzalubgbipckoxfdwnjwggharojsygknnplimvrdmzjuglfwqndntafpigwgmaatxoidgqxmdxqnatjipanixzecbwpyflfrjedtkkzkhmdjxybdnofuvaowikgmtdkuuqrlmbwwuvjxffgmaitmildstofbdlmmysnejjacakmopylacajgvgzfeyhvukuugsmsrsvfgjfqbkfthnzasptddvwsachsyjwcrxabkwdgrxdhifnjlcfgiqgjarhoohhaimvpbncvvuqfcfmmgprazfcuknvxfewbrqtpmfrsunvaoxaynoucplarwztvmpsputnrgrbtaytzbxsrtaqdiosqekfbifxoofmkjntxtdhdcsfsnboqryicilhweyqknmwjgzeohusqnjcrlwzkxtndurvhqfhsdtyaizwyiwhbnozffaoubiqrsuxetcodjdxdwqmpufmtpezvcsuabaklensiugvksdxudpfsltylizbnoyvgohpodnjkzpnwmuuiacgcwsmhiiyxtuvdpbiwjwqlxtpisqwpwmkcezlofrlhphlgeanxqqrsvcotjwwpvvdbbjxyxggybzjfqbyrsxyqhsgjjebapdhheziqhgagklfqpdfllxwbmlnthqubtrvjguijremewzpvqdfzrqncbldfrffkwtmmukogcemgsqjlmkgkfflorlficdwnvyhobjebqwjwsaddgjkrhkobhgvaroghlqbfsiexnsnwmxmgdbdewvmvmnkvogjdgbxwfviccugiuvwgetanvqvoasuxxotvpsmtkdqcfilbzplbkftyrrqgncuieiuguknqzxevemhszlrksylqnygnnurlwbfmigjaehqrqozuhjhebkibhluzjvoarhrwcealsblojjkjcasblmvhbxgklntksqpzpynlhdyuagsxjhgeobwyotowxsogivqeifmfsvntrrhlghxocabgegsucbbcyclyabtteznknyfybanleuirdtwfwcmtkqqnbmaykfxdepgchanqhjbkhpayrcxjnuglzjclcuhflaxqbuyugnrghndponbhnrujozcqbluwbpfaqqxsmiyxhhprioglccxmiylusrewelvsaibdhghhdnwzqgxizbglmvffcrnccmofeyiwhwildjxcksiqxtcmvwnpeoaxkihcguavfwvvsssyffqpjqqdgcxmtfqibjzdgpnhychffenofrizagaefubfczalzxhjvrzwaacbrtayhispfasxkpimzuedgtydvmrmmqpbgokolyvbmlrmskqspguqipilhgfsmwqgbyemmihmqbzbjdghhalbzwnuhkumkxaqwyxjkgmwqecnmaxpwomfnpkvjkazwdhcefqgwkonqoqbuxdgbhjzuxqmmlhaeemmhxdfosmipnjggoclhojwzetcihlbltmqmwbhuybganrjmbchjqtdwjhxyrdvwedsbchyctiojoqynqlmlitaxzbhrpmuujpdhijygtcpgvsumpptsvgunlxmivlbmhuhalmoyaaydikqnncodttjgppextzbmaaeacwponkhoyzymgjmjsssaklnlisoumrjumgobinjrdxmimvxnhbivhgfqpfxmpnqghcpwewqxqxkhijoofhduvpckoaegclpaxvaxbknworulhntrccyrtffayqsvuliqngbdabfskwvvhmxsrnyofdagszxgeomicgbwpmfiezuiewuhkhylrqsbgidwianmqnoktpaizchpqxvmrgapmyzxeonicxdmievlzlznwevlpyvcifnssmyhzpfppalixfmfisfoslulsnihkjemvkxqiodngrgnfbqhhgqxgmijjpbmlhjtxamsyipongwxcqjgewxczqbcoopshimsnoniuhyopfbekuiiiuftmaiwdljjhtckjycgabfkgrshcgrdaucqrseulyzjgmgaampunihkzvgylpquswkxobvlerneczfbmlpwwcbroxmldmpputlazhsqunefmdjndkqhedlqibthsjzolghnligjwqchvibxqddmyizjmoohrnztutqgpiwlrwapbqkwpudbvquykfrzibgrmkpkbucnmbxwwixgoepvbtgduxatbsqpxsjonmcrmononviyciizejxdfhelhehytkpdfibmbrnsdkehykiyozzpzgxhatucevxjigfssztyuarmziyadoddkkryhfplxoqwlzstuielovztvcqjkawontavhhcwjbgcxcwwcsrylagglunvddltyeakqjizycrtexdewwbdiqeupeqyxosjgyyouvcmqwzulfxqhacjxfwwgkufhwzcgfhjzujlzzoqrlddpvwfxyewplsrzcyvplvajdeoqpongkpcgjzodjyyzuzyfseyftlzozqwbtlznbsyxkkiwomdqkhzskmkqpdoclqisgmllwpiulrxebbdjegxlndbnunsvretvvkriqpruxkgvujxvobbjenznnoznsxkggawayyprgwgqtmwrnlbnbcjziyrzhxbcarcozaehuvxvvxtmucbrjlhjofvfktvjeabhkfasvhqnopxlohgbvdtiqenyqqbgrfcswvawkpzjwytexwqvkupxopdnfnoedhkkeapfzlosounoqaasrfdthwekicerhmdlooiueeqnqyemhmlnwkokwxenpjmgbyqxfsvqmuzhkxlcvcbzbbyymdytxdeaaaounyqyyenekiobcxwgyfrrlrhbggzfpzolwphtazaygmeijvoklbuhfjijhxhplduigmhfakxogpczjcbmkvdktakqtmyqqzbppatupdrnnhzxmazjqckiazngdkymogirfttoetqtyevjzawsbnqlqiblvngdtfaembaqsomumwgyyqcmfpbsptbbtenxduvliljdlzqeiwnxpnkaoruhulfkjhqtpgxwlvvchptwyffguupkrgyduovstafnlhostgwrckvqeceshdaopszvzeuobydnpszarxbcrrggxroaevnvzxnehreuvxtbszunjdkchmgolmlaqegnhuhcyhslfobxwwjptjsrmozeofkjtvzzkyouvxftywdofiqdizeyfbkqzdrfbpkbpwmztuhwdydjxfularehcusomxubjowcddskwbplkjskjzrmknsiqvwzpwpwwpueaizbwtawvrwxoeqetfgoztebkvulxdajlrorixxsswhkojotlrwlwssdjzrzrfscoqapzwxnwedjszclwidoujnlhesjsmdhmsgrrdjsmwuvwkbncmvqzayaedqzkvkfjovbcwszncuzurdoagahjnulrxqxuhoyacpivmkylgrfnmetnooqefjmwpvjpqigucazlrahsvhvkdqvbhnzhlvjfmujlrrlsqdbgijeoeeroykcivjitarhwczqqbzmmkiezfktnsajnfbhyuonxsssyipfgqlisxojwgstgekcxzhjiqgkayepjwusgojmhgouwxdthdunshfxeajfvenxrbdouvjyyxruuvefmdzcutmbvkgygxkyxikhzniolkpwzatzqsdkkaledgbcuiysjzilgxwbjlkxhneamacglvmyiqfttdxrdfkwsyzwcoljhuweosqeebofoxlfanmietjcugbvugdicsicukdqdaeapytaztpngljterhxcmyqjwpummbkhmqvnxvewnmyyxmajejhedpvawbbhdhzejhsbahyyagsqpgcawpnpatwfugqjvhoefrhoqfxxqxwvkgozfbphocolubqvreoqhfhqeokkvmbljqrltzlsjokouldqtipwfaoabftzgdqrisldindztdamviveetwyqgyitkcrjeowrckeuafqamsfjyughqegaaxwkwlonlphthqwewstuibupuoclmqosvirlkdbmggntqtczbvldqxwmikqghgahmpbiumwmcwbwockvdmduwjkdhuvadttoluieoippfrdsmlawyhirnygomwyfasndqxjjydbwrauocwpddqycwdyauagzjwskupxtndontpirtqwwgtbpwqivswajnozeqwwddtubxcyowrfzmhhlzdbvpwltitchhvavjbxqdnhoaoghqyqljxiegxhgtijoqgymrzfldkaddtpdfnycvcpdlnguexvcqkcblggodquvekhvsolwuxjneddcklmlhtibmxxrmjpxrvnuhqnolwphgcabodpkneqkwnnwnqujfogvwewqqoezmmxscozbmyuzeortmxrvmuuloyenqlibyaksptdzuqjiezhbggchtljzypjxksmlahztdmlnblraqohflkekcriajwlhfrvbvssuohkdfrtlcpdahdxkemxrjcvzokwkgntmlgcdtajzbiykjuixxbkvcmgcxfenlfhosztcexhdgsjszzjptbuswujrvkzsarqouadeuaiepueybtztqjtvjjqzgkujjyuitgvfgcshadlyodawihrmqqvqgksylpcavqyrdntvjorlymsuojddupdkqixjwfbvnsttrjhroh + taelkxqpqxrvmqlvjwfllwxrajczkccxwusvdknjxgnzppjawcppduziduqovmqerowsofkzbigafsdvinvyhyrpbuwrxlqnkfnggqltayrpopzzkuawmdgherifjonhbdaoejobhscsroesgzjxsiszjqzzumblwzuhvxammcqyvidgkmccpokktlygbsxghjbnuyegdgyypgyjpsepuvupilptrkababwqqoettyrvrhlyorqiqhmcgdvffyuzospdgivtfjisszqbfutnvadcimvcogdpjtmbymfshzntrycteoesiylnyhpcsqlafqomysybqkdswzzqpeleoewkraudrondupqotcjegbosnuhyuyqdcwmwypqjlhonywfnyfpjwchvcpnlgicqswratprvibtzwtejkbvkxfyadopnyerdgzumqlfextkaeyxpwykkceopqsmxgadrgzolyoxnnizvulbkfuddfxphicamrrcvzuwemsyvungnfylliybpqdpzkkptjwdyijxciqtuakuftuhpvztgbitqsiuxokqllnschzeapfgshyxarhxmnmgqggbzpfvanglkyomqkvvsynyonfgbwegbmgjxdpvvdowohneytdbrzlvoglkapgnimogcgbygbqzcenqdfkaaqtqksrmvkqbcsclagnryywijszxvvhokpzdbqfrqzqgywatjerwuqmirunvaphcjueudaucfkeztdwwmxgteehdjekyhujhydlxlifnjwvxxipwevdqkgllzjriucbihrorcvccpxpnxhyacilqdcgjwjrwrktdzensvrbberxtxqzkunyjhwurhauxphhmluebuxikgcktvxjyuvlkphgpwfiyqtsqoabvetyoyqrnczmjvtbyvaluibivhosvizlurcimikmctsyeeoilvbgjauwcfemjinkggxiaenbcrpuabiczbankvtfwxachfdayjkgzsctxceoxagokfmyohlnaoxibddvaudcynybikgsuzuhedbfryxksjnhhmrjrcxezntgvyybkaoiedtpeemqfylysvkxdfdzdhbgwyazhahbcibgcmezsaxfrnxhumlcyurunjnuvtspdpjaahitscoukohvdczvgmxalowyynjogtuenduqbmgqwnfsuqlvsvmraflsjhwloyrapcatztywcjpbhzulwxeyxfxlcpkysvnuxzxxjkzbrczgpxadhliuzalmfwmthfstvcvoxdhsrypcthqdsyldgdsptxryvfkltqwaucuogdalycmqzoiugnzwldjgonkifpjhkfwxdizkmreiewmcnyjlfugamasoftruulltvfrypmtcqrqnpqpouhdpsqcjscjzbkgrdborgcbohepmcttpixzzbyeyocuoiizcqkzamhonaodlxhsyowfmnwwdnhikirrkjecbkyskxpdbfchvbrryzksprkzbnqsvmmmmmsfngtqkoclzkfhskjdltsiwnpvqifavvtnvbgzmbrzxupolyvrnqzzezetslqfvftxmtojxbtohbggynagpsrzdjloqxosqlmtomtpyvwzpexqngfzcepgkgvjebgatbieyccfpxonbutcnklqydwuufmeflorqmgxenzxiouyvtivrffhgdbgtvffapmahqsvmcwafiytzykhpnppudhfvrskyxmvbgpmbkqzygykaauajmdvtctvjirzfaixlrkpmelfynrdhisiudtptffxcpnposwqmmqkyvcdvjlyrmopumxtpeusyjjyhsmooshefkbnijuskovuxblhfvtrharvzazndtyfcmylfppkoqfqldpurovilghheddizazpssiyedcduszynpdxczphtskfbdluezrbssliaiaedkmgvlhqlmckzukdgdlpnabtmumossaqlufzpqouksfrkcyrhmophhgreuuwkowrlppjtfczzodcslayaoqgzdfaoahvcdhvqpidzndyxjochnvkzefnscnarahavyoncnglfmavqetbgzccqllibuneoonnlcvlzqfzbywmurghkdhnxpldcoqmescqwgagxnpaomvecuniewzbnrghzzdvjtyffujnjrphpzqpqinfxinieotfghaicjptadbzuqjdcprjyhhiulhkgkrzrsjjrlqmqevymfffamurfntgfqfcqjyitxipeboheemtzdzkpatvsnfrwveivdclmrncihroyzilftqsmwzgieeeuizqzhsnhqjrzhssaxfbfpoqvrxedlalufsaucimisncguxdbwxvhnbmhvdxebuhzpzosowcjwnmcdwbtroszpxpnsvzjoyjqtskrycvkgaaxmotfjuznkpjpxgjoxpeuoulgfjteacyesueypgwfxejxzhlqcdzctdzlehpqkxuhnounnrnsmeklycidloiutcvdtdfybxkeenwojznetynctvtztnctjhbtcslqbrgubyarjftwkwiqtuztudmirzwsvkxybudnsrusgxridjxuwqidvywddlzwdytwsdvpyopzwjgonvqegijdycfkjthgprlzjeirbnhipkmuqhuokcgmnijupqpdxsdouwvnyymubgsyipngqcfxvyxohjqyxjufeglymseqagwvsxbelyvcktgfwmevrgywlvogqevkeqjyirxdqsorksharbstnzvkmcbqmojmshrbcoewpjnolcdppygfxskekdavbuhnwwaqihphemhoinqmqcwtckolhwofonzpejkqnbtgnxqxyvrruippyejxywawfitqfflknnzxljuaprgzvrabkaaibsitfkcscgjhnohqimcxjhpspbfddrzunggbbuakmlvyzldqiiopelqrkajcwquyirztjsroepgwrxbbirstbwpctpksskrkdefvwlenvutwgefflqtuyarnespuavomwhngmkjtzacahikeraquybprzstyqxknmdwrfcywbtwhguiwfvyjmvxniixevcmddbyxxalvhqvyixhhbwwwrkklhqgsfmmdceqjjajlttrrmjzhrchtrchxpetrtnbuevactwgifpvvzgfllttsswaokispmfoihzsnsqblrsulpwpjgqwuqnvgptjgcjvrjergpasuifzrtdyfhtcqujgozuzsggimsvfczmisccgynjfvkedgzyymwwrmzubpiujokdbugjwkumtfgtmdtwxelrnoqsegzpetxhdvtjydiubsurcqloctkvjogvktifcuurtxozimcxbkzwonxcneahvcwshafzopcxhmogbyttaugszsdmvvxwosxtcmbuzzgllowvtljoggudtmsclxkyesdkakatrqctqdoamsomzxrmqwhejcqkylysqcdcfaegqdhdnmscdpzbqriiwqcurvzpecuukpnqooqaxebbqhvinpcwqwucejperrmcgstfjrxosikdlakhyimcvgbmhyqdfzrlbiapgxoaekgkyazujwyjjxdgxvyeatiooqlbpzygmzsnwobbrcegaekwfevxfupdxyuqxtumswyplwkqmingjxxdvnkybtkkqgjbgfpmauztkrmqxuoivdqtymldjxpwmdwvsyjachzukenpfmqfqqnuomwiykarvuvjcisgggbgzkpmtafzohtsmmrzeneeoqqfrrribobgvpjxcdzjqsluaalatfrnrjepfounoflkuscatnjfzpfkytjibadxkylknfcnxnupezhsratrjjbdinqtrziyyreggjdckovqszmgsajqpxwynhimyuuhkqzhzekxedkcehqotvkektfcdltnckjwdevvegbftislilfdgljfjcibcntqkqwqjlbfnasotsdbacegqelnuqdifbczgaeevkigojjsjtaqdntirbfkixfuebagldiqbzprskedkvcpprsmpanlwlthpgqgmqnpdnnvvwkqavireeavqauxmxefvixeiryqlpmlapcbiyjgdonxjakaccehktvrjgrbqhcdbpqvhowdrpsvlhofihqeaojyqcsjxarpdlcmklbdqsxzkwbqhxapdjprtijrnorqoalpolwfyectstddmegnfyjdrcefwjfnkrnyymqwqosoqbranrmscejavccviqwhcrhytjbopurppmkecoaujhjntupchorjvfbjizwrzoeisgbibrqdgqecavmfuklcenztzqneaxypgkjogoqplbbxpbzathipgyqaeailvjduiwibnhtwnchxmzcjgntoksgtixtmdjgfsxnoahtkzxzeaqsxcsaezouirobnamwlbzuyvsbhhdvmkjmgcerfxojsyrkytipmhboktqqqojpwcgxantonvgpmcvecktkbggjdcjnttmfypedhsaapassfbqtjsiuwaugzwpollzcanfgaichfzfwgiuqegjaddblsfhoojqdtsctoihclimhwbvxwuijhgebpybtncyzoeohqwtqikymqbrbrbsuwbaliqepuzushhzwctlneypnhszsatfkzushmtsfoxzxcqpgsxukxugmlfptzenxdlwpifpwgysunwgirgmidlfujjckhgxwazsqdmnpgscyujcitvxrfjnesooaccquludgpegldsqssfpckifjwonxgygbgiusfmqpcfdpuganmiavkieszjvfqexnjenrsrsklugveaslqriwmvodahbvdypxkredugjkqgvmtkyolgxwodegtwqwjwxoonmzttfdyetssdtotpubvyjcwzfjfdvexmfwyaqzpviwekuccnqhsowexvzncbkubtzqvennmajufsoeboejvjlrrvssysivjuicuwlbqqvrjrhfeykhkligwwvrmcrxyvcxyepajvmnrzmnanbhkqttcrcvomdrfyskvozvzdvliijnozrtfxontdsgafgezyesdwonptqbsbydpseaerouudnmnrrmnorrfspeqpndoryeegkxwkwvuexerxwpxnasbyxsnaemhwsresvjhnjnfsixbkwxxydzsvevlshocmrezteaedqbqudpptfpcwrhbqwpshrhgweoegdscdkhexjcyuhpqrkybvavlwvbgejucdzyhlqgybyxhdubkhvknkhfoickwuiehoxjnzcitfyxwnduockuacwcygdioefzuyzwaguvskyufvakbshwvjefealfbvegcdsvjibbcgjyrckrqoyehwmrzbnvxiucewxhnzdshehzdletveuxveuerjciabuioysbnrhzusotganymtlzozhukafedtpxtbgqfyvjbpddlukyvbdqskgwerqxlgdpnfskniovsthwvyofdkpzsyjmoltbpgbayfwustylqglbrjqzeaudwmmgewoxndjxlmkgibjyolnmscbgzltuwgzvceramuerllomboapsvpbgqgpgfhlosdsqxghrwzxvucvhtyfbjocoxywzfhdhweazvezpwvamjdbafdynecwcnesqtduxgpjrmmazueguucilztiivycxpvwqzzjkcmszyidtttjcrygifzqzxitbgbimvaotxtspeqplrgsgvyqggqylpgffbnzlrwckqifxtobistmropnuncxxbvfmvrzyeumttyvysfvgypvlnhiqwysyyixmjjfaoqxircmdrpfcujokxfrqkovapzvlhkzclgfredqsujgygqvfhzqwtfzttmlvjsyzhksdkyvdyhiyaylssaleauymlimpmzvgiscjsxxvjpgfwryvogzjizxtaaiipbqqmbivxjthwkxhustvsrnkgcivcyltwrbqstvfuvewdttnqhutqvekbyqqwoixsjugrejjsrlshhluounbmuzrwhytxscqevktlalfpuavxiznkpkyzuoyvxdeuxetiplssmbspswqeyizsqvuypizifsjzbesgdvrdxqncueovktkvvjdhpmptsbzocdixzakowhouhytwhavoyxnrtqlfjsblwjdwjdubshdirztxdemmpoltbfgjgywoydcsqqbgqnbpkgskuhwrkbnbuxmuguqfoinwgrsythybmkolczotbveoznmabblasvbbkgzadruvfqebstfxqkbhkrjcinpirpppolnvlizfcxnauvlnkghizomvgzilsivhuejphuyfdhzbymgoschgsgtjinoxfimwovewoldkyuvvlwzlbeixmynbzybdylbdmdxchuljtwikfnlporsbcfirymoofdqchfqqdchgvzkesfjjykjtqvrtfplvweewngbnonlnnlvfmlaurbsgzquilrqgtrbpowdpqgnnrhtrqyjojuyfkohppamlaqmspefnlqhgzafffhlfmogyntxutglzigvqztfcnjnfdosnnzlnkesulnqgjyjlguqfpitmnfahosegzxobzrfufykjdmgmsjtxjilbbkmkeouvaktbfqvjqtpauufwwspwugjhhrnzgqwlpcxfyoezehqjfazovfhsmzxecaymzlyswtmdfvlbhxckrgatrjkmykykknedgkqvppscrlxlduvncecsgdkkkkdvznwwwlrjdxsybufoolwqgyehkzocoblxfangzbohiolniokmkcmcjssagvtrbzslpdexxqvatxcamxgntrqlwcafbewrspgqgdurhnjiorcxckqecllwlqfgxfvzjfczqehlhtdgbjchyxetwtdzdfrgnougckirxzjisliepryfsprkmeumtgyeggqmttgmotvcaspcbshditejjyglzzfvnodcrokdspcsbdlaphhjboghhwrmzdkhqduwxtcjdazkdoolxsojyaryuhmubyxdqdgzycqbpqoqgfmgmnunhxgpikjdzooczjuqfdysyjbjsastfmvevshzgnitqjejdohglytkhtdpkfbhevwpudcnrjrfxsjdfleayirvdwgzriygzcpuyuvfzjflaajdjfgqxsqmszebbmxiqigytlmxdnsddizntvkdyumqntwdnpvjownxewxqrpkqbrnogiipeislhkpoxavqzshdhcijmzblvcmozndbuugepwrmihxgyuqgxbgnooiiansidhssjqonerymxkclxjumshmgiqbimygevssqztyvxostldhzwlzalxhrlgbptouzujeeuizovivjkhytypgwenmerjynwgiziumcctzsushddcpemmqrqaovvjumaehfvccumbvgdliopocgusgnosrfqojsydfbhctwkbeurdpqohvhnctfdjttkbzdiirlfltqtijhkdzzegleuykklczxhxoniwpzsdgkpciczmexchzyuxthglfprsgegypqjickjcnhjlkqmorxoawbobmynmvcialqwtbwuxshaymelcugrljactpvwjynqbovzwkymzlhitpedyvgrjpvjbibnfcfenmkfwxkcobmwchdghwfejqqvloeqavdjfbkwikxvddeckzkxypqopsmerjnkheatikiqxalyruusnfaakpduxuahakboovkqiceadwwkhjdxkigsmarxuemxodkxsbelowmhsimqcwqcgrstwrqxlmqhaiefqeqvcdymnowzwhuaaembufvmlaghdccwbbaqnbbcurhhcmqzjawxmyemhsunkuqkvrgogapeobjfhetiujrdbomyckhmtmklpoxoaoqbbvamdwvmhpvfjmijgegrhypepkecipzvrijiktchwlswkntslauhexklzvkugyhlxlxyduxzgwlfnnhvzqtrnnzrqcxltkhokmcljevlvsijoprzdduaxyyxeinpfevnxycawjpbifwoiyjdevhoxrd + lafdqtkgldsedmhpvmjlzlcduxvofmszmdpzuurgrbcfgygglobplwnkctduibbrtrphuckmrmqrwskprbmnbjeilamqaomzrreiqeyenvenxcdawgbgxzucewinmirgucqnpygmijidtyeuawtmysrdixylvfyxyggbnarmgosjsmqlbgnlfwhxnhadjadgqaklutjixhnbitooygmwvntmeicwuassrbmhttdqdbkskyvmhkogwtnrqdcdfervjvgrhrzbkmokbesfhgamzlwksvkawznxrmpjfwgzoxahmtcypeiyxyqdbzxilhilddvvbynpngltlqnoxvaxhzzgweemrsdrvmpdlaehzddrpnydadaaxcsejelqppvugkpbqmwonryzynadugwsuayaomplnzxfutmkgbkuhvgspklvlipigjhyfdxmthilgiegagjyzjnciweqwnmamroteiphjbtnrkobojcqjwlcihcpqhaaiowzmdzabkyqxzapeezzfreounescrmybpwagfprrvexoxeitpdeluujqercqfeytjumacjiodecvkpwvlecbenqcnvvvcrjeksoipcndtrpotzrvqecamfacxygrivkfpmafzjkcccxyqykcuguxyqxrehyledghjbqhufqnbmfbmwfkgtqjqqfwrevbbhffqpwfbwfaitfrdvqyxfdrbybbqxztqctohovkrgrwtarypixrbzpxgbspzhdbvpchlexexjefxdgadsaivukqtjmkzwwcsvgjmozbuvtgsegkjdwyzkryzctfgpkiugiofsteyramefunqykoysobvbbnsdacwctohxtemefqwtovgtjcmhhimshrbmsqaivbghwwxhuxbupuzoxyeivrjnadzktxsrhyorxhzjsxgeepjfwsfpscnvcsfphhdyassdwkzowaunswixyebbkuqneczmkzjtscbhfcilhaokizkhfkaxmmbxpkqzlfwlkqywforarxjhdiwhqzmszmaggwyuhfkbbcpjgoxietesiapvtqbbrtstqgkicfjuypavqgyepsgdnlyqcjesvhdjnncjgvpgaubylzfyktytjgevnhsldvxwdvdjvqnvkakqmwyewahetpblmnuovgfbmgusfhyjgefgqaoiwvvsykasfsqkebjyzeajdwsfcrajeqshpyjskeawgtchcortrrmnipqaocpywhgmuztlwowkgwupomolgajwcejvlrbowhfutejbawbbogukavtlciwmlpurwowgbucqanbtbqvdhfgqexlicbuybtphhahqznehigqdwvydlgfavyneuronfmllhngqhondxhilyxbokirhivibxmiirfdfgvygfmutwrqfeiyszkptepjpxmblaroytjphmsyiymdygnyeeaazawvcynlojbwluulqlrnrklrjrrnlamnwitxkxqptdxgwjlbomjpxtgczfhovkvetqyzhivebehwgxijnguqojzqsmlhejqhxwrletscondheyroarwriixpifjikoqqnhrdiprgduzckuzmyrjndqyzxublrkpiqoesthdiqoqaahqulzpaeqnbgaampnedncyadcvlwuxiymtdzzhkygkbtxhtxfmbawyqwhicraqulkhyhrqhmwrxzepvagyoybzxfywmwuqojpfouuektcupxhnwurhwucohfklnbscgoaickybaiufqvozourvfmofoecboeqkybshsgldycuroafykrvpihqmqzsgmqcbbczwuahupcgdrthbuprybmufluwmqzeaicwllxeyaspdalnewxjwqchgexwzovhxazcmhltkfxbytfmitsyrsacwkjyvtqgyoinrniwwtlsdtydidcyrdzsziozzgdumoqfsnxqhkjsqgqgkammvxsytzdbcmhdrruzxbeqwshjrhymcwoqvphornvmugbgtudxbvyjfecyrdigybdcadxuucnwlcrkuquwvvblwwfdhxfgglwmybwohzepqwrbwjnhljjnwncdsxxwwynyisejibqmxrdhexrprvuvytnqfwotofjljuydmyrgsnfsdwkeaoavgnafikjbgbgdgijyjkfiaqtxizrbhvsxklrabxcffalzjkahehqsofrvrcgooxyjvxwnjnlzauayywgndublgiioyzwggimdjtfqgfjdxigluxewyugyfixylirphpobolxyeipqqumpqpevodlbgsqmpcdhhkdblyzkuazndpeqvjdgcidyoiqynfbictaexsdjaovcsvszmwfxbtvqpilvguymekrknoakxbvybyfxaekhzztsyrsudsempehwxzcalpdxcgelejxsvcxqzjkiirmminunhaekfbfkvesiayxnuzkztxalngvuenxoxxrsjjvsviyunrxpjiyihvsqivqvgkpugtggsplmatfafobdwxvtjbxscnwqlanuuxnjfydsyxgvylbxhpvtolqmaphxytxvcasiqifpysiityyztqwqjmvkvbtuvuydoiwcgjjrwnsdclnihlljftidfjuxuxylrlucyxrxtxhyxcqoghszselvmgtwiyhwpelqdxldmjvnozmcprtvjljgyovwbukexdzqrubizgvclpouqolmqjamlcgecvixnykgqasmlmbkvjcghxdacfhjemcrpnyjaaimjlgtnqzpztpbuezsiahzxmuejkgrnevcvpsvzamdotvpwkdnoytyctxnytiudouuenzjakyniblqddnvortqoidsrscglffuquvucjvbmcdlkkqkdkixyktvfeyvgaepnmjghslynjbsvvfdeoeslnowmitvmouebtijirqfdvdhoitgdvtupfguutwgubwnspmikysqbovvplkpkhuyaeosqrxdwmqdmphgxifdsouglatvuxkgmfwuuppehfawautfbguojejvqvbooprqejoqqypgwuyctukyxytiydfnthsdhrroqqbqbaonkqgurqnqymlfjmjrvnuhmvxydlbkxrlwmjdzajlviqxqlxqmexnqryzrtccjijtityrbdqgonyxxfuxlsjdoqhprlmskzrnddqnonywcfjdubfivquhkmzxfbvutuglnlejhjkchsfzbetdueifytnodaaqstwadkmhbfkwvvecceueclxedaygmppoxacvtpwkopchhkxxoxnvpndiipitaaclipwllpminvhjpqzrmjyafnybwxcjgetwiomsefawcjdouhvigsfjpkyvapwpcickxxbnduqyazuixhjcofgpyuvzjcdmxbqdzlydezsqivunkmknkkanuzmbhdovngadgesnxcsimcoieepnjecgnqpfzcwdeupfbahzdxfnwxwgnnbtaoujcsspeakjjrkrtzwpezqezvzfrnwypfxskarfhuerpryiopkfbbftfgxlsbzuwqekdaebujkegypbfhnjoaxpmnroharwmpuddkmnzhasmpwhwylgvpkmdycxtoksrehmfzbevncpkeoksnqygpgfrbnlwphfvbaqtjangxursnnwsjjxvsoqcbflqiufzuckyfscpactsurkdbmxvgnjkbddcuprzkgcrraczdoqggkfpujacqrtxaszszsqdbiftzjtlcxnzwzobvryobhonyaktegwifmsfyrkthixlidijfmtmumzxihqkekydefdpciqymdrpjvnaypngabkgcizqfnvdnnzyrvqgciequpvwgyxsgugjzzbohbdimfdomnjglagqxrxxutkrrywzzilqgdbyvndsepadppnnoutuvsqwolhpziqmsvydrdkzpjwwcqvmknculcdsflkrzkjbweudoxcvhjojfcsugvudpsqnfdixskvnxxuzhmjvfslnhikyirfktnhyqoedfulbdlqrlqshxwasupsflutbuysdfrfpbfaykrxpgqtjvjpsbpdntrezbfqbbhbhvaocbhcompfqnuaebgoxctchhqhjvnnkvctrcycsoxesvhemelvppcmhpficbsciufzmkfbbznqfwkvdtqoabzsewjufadxcdderguijvqoblfrovtjeewfsicpezplpkzchwucixxeexizweirsxfrhvrogiqzakkrpyudeywkwrarcufkfkvruyquojlsuwhtkwbgguqebsyergpimssbxtaautscdlqsfkiwjfclbjdtsdftxtjqdnvwmxbtawpynzduiaxboynlohqtohltltoxccmoxcskpglniywhyabrgvaccgnqbwsrvdirvpovuipssdftngjgffctwagxoxcyaamlcmnarbwswrumdpfxhyfddwkxyoqikrqeukfuifftseozrdabgyhnmeynwzvkhwoyuqrqfwlfeyaslhxqakhexsdnglnoowneyttiefxxmszfdxlotdnfikxiqurfkicgmrplybuywrnhamcdcassqcipxmsmbhseouqilwgwlmnveaffverenrdsebqxvibywavrxyfuqkxgzwkjsdsjspcxmatvdgssqjwetflkiddvsdmzsnjscghcnlbnyxglsslcjnnfvsdiesejgosbfghdegouygtmolmzcaasxksofwkywzyrybyarqqegwhpustvtdmtoitigihqjcnradhbbkfyckkvbakuzsjbnexnucgzvdwlkfussxqeknwoczdeqyjxfptroynjxtxwgewykgdrjkzatqrdcejclnvpwzmtiwwkkqmcecsspegqngcppocsqgbrsukafacmfdjyzkqtfdqpbbhhfrejwkuhsqqrbxdktdhelcsjzjozuwekhgwpkztczyblxldoxghgtkhjuzbzeyjsdgziozlrxjcbfzvrdufdwyqxcjwpfpgpiktdoevzwnesxuvbsedxvaqxoaszhskaiurpsiojcvfvzljfombsiihtgwdudlisiqsbxmbccykxndgcqruykjrxbeyjblngmhkgdisbhihlatmwpbncigjbhkqqlmgljdejscxilsetzyixbndixjvgbgaqlauwmxtdipdeeamomazumpjspslmaikskmkqtjopkyensefxeeqsrfuuedjrsadzjxmjzgpszkfzfuqvujwlyzpdzionixmrurjywdfrloahnwutjcgqddtlmwbxplvuoumskawludbxorcudvthwhwllbgbhhchprejsrelqslmjasdjkbebjmkznwendpkqnjaymbfszvhudimzfxexhuhbgpkhargnkodryhjqyzstsnyoeqzuzdvgsralcgbeaxdomahdfyucqdsysofbhdtajmvrollvvdvbuhoqusfmtkpreuvwuwfxfhuiymrigfyxlkqnhyfecbsdsjxutnyvwysnyzcyxlsralaohopyxbjfhgruxrwptkfajudckpwnvglnhjkfuvrqcidkkedmrrurqfewigelnbbcqzpuxgwgdzateldvvckldsrorvhaypryljcxyybcozgpwjuvzdvsmxwdtewaulhlzkbqjnhbnzodmcmzpnvwhleyhpyccxmbzcrokwejogvcflsqjeycxmmyjqwzandzkqfhuyuutxothlmzulojvimqsendhofdpbqumuevyeskbhhwxohlpniogouiigsostwcfznrfjelxxnwvkvjsulzqqgsjbwdwgcjysjuqzopainbplelqpdknapmfucvjrefkctnacxgnswtrktdxhjmrulliubcjoyxlmzklunqncjbzzsyuxtyrrdyxhineidbhvjmloaimmjebhuphaqnlezqafebqslvziywdyqpuxtdfmswosnyvojzxhoulmuhaclgcdltsdscvzmgguyzjrdtvmnabikfcnskscikotasbdytotokzyyaivapxnesooewcapzgsnvvpkbijrzvmtjkkijqprpnbxecwqtnojtzwlmqhswhmhvdopekfyguedfankviutmtmjjcidlqgkpbkpsbtuewkqhwefvjaoswkwnzjjgjyzowmkcivfgtcogwdilwznkmztxfqmdahwyhulzosrkfktrpqdtdwlkwfxktqnqpvlecttikrgolugmltkxabqbhjjkkccemtmubgtwedwfbojynhrprrdeyxcguwvkddjlpakdqhejwaurkgnfvzgkqkaomijedzomatyaiwrrehajexzdceuwjzhncimbkhdgrgnubpkserejfwfcrtymezoifydrkdqperqrtqtdouyyyezblinajdxviahpyyrduwbxrwlgixqyawpvpyopxufjhmikczxfuawtjpxnwroiovjuycxrduqigszeurdylgyvgtdjjyqudpnuytptvslhxbziwkziaxrelkhmanrhbrdqyhunwysnvusuuwqvzsnivrikercpflqqirgdsbaianjqwsyzburtyduqzdhogaisneokccjucnfobpgghwurmnerpdhjqzakwartaholnfmyuxbelanrtfstigyqrrdetttmkgfpxwuqifcaycmtyxudajedsptpihdnwnsnckqrbxbouevywmmxlhqfuxurdknubhcbxgxaaxyxtzibxchxsadnjapwhzfkrbjotydunmftpkkvjsujwpexkwpvopnhkdiagdpcnvsuqrpndkootaitxiintnodcupxxppwedaxcwqcpckogmvljveqfndodwrfwajjyanujxubwnfwjrhqaqjhlokjajewzlrfpttqqxglebpfjrepivynwlwvepdcukwtrczilqmkescjplppwguxjhrohxznelnwpnewdvsalxhovvmjgjgzkbvuylifwpvcxjuhrzcvkykzrejoxdrrngtojsqkhswblcxzoesynqahpluoqkbvptndqokxzzjopvdwahgejlbdyquuccmbftxnbljgnssuhbewtfclxzwcurvxowsmbcuegpsldmizgzmqklfnsxhywunfbsyalcgvxcrfqpefncqgfkktgymftonhmagefomaaeunkmwdbyqvnjurllcupsqgtcyollqzvulxknnbeelrmjjnnsszsmvtpagcpooptslrwqhsvuhzkmjdhqitdzfcxbctoahxkcgqjhemmdhnftdqhzyiatfdmsyrzkztunbytzduepapcefujaimdylkemucgubrcvdacidmgfhurmcqssloqnmeldsdlyrrxydbpteqckederwvaohbbdpkzhmsgujdluqrwdkdecjwscopmcbmyiqmbrtqhgwwtzclfwqxivhraalrkhpjeydtsgydnteeennrjtnufkkodpqantuebdwycsovsjbvsgodwpquarhwthbukfrssuuukalzrtwesghgysahfllmdaxrnmwguqybjrreohjqjtklllxbkpvowujnkmrdupklntcekhwhzfnplsruhcydespgqkpjekidgnhtrqcphnqnlgvhjtjkloovkdenmiiwxicogxfecldfnlumwuraplnzffglcoywwowcfacvtfivuuwyjrgabmbtzcqfjkvketjoootatvargjjgqhnripoohmwqerqfwdgjqhcnsuhwcpfyxctubwzfhfzpupisasiqlslcwzraoekhynzxancdgwwixowkfevqzkeyjslaikvbxkeilvfkkznhilbotbindwgiobjlaxomkajlmcodqfqwbxiyvksszjxdfvjgtnfkbaoyyezqwkpfvangusakakjnvrqlehibvvmyswbvtpjlvlrepnlunmnmtnmmrrrqbynhkmvalkdvqpiepppdteuecufmovxdndslaspyztqdijapfrjzuptaxyvbbqlzoisnj` +} + +// newTestTClient provides a trillian client implementation used for +// testing. It implements the TClient interface, which includes all major +// tree operations used in the tlog backend. +func newTestTClient(t *testing.T) *testTClient { + t.Helper() + + // Create trillian private key + key, err := keys.NewFromSpec(&keyspb.Specification{ + Params: &keyspb.Specification_EcdsaParams{}, + }) + if err != nil { + t.Fatalf("NewFromSpec: %v", err) + } + keyDer, err := der.MarshalPrivateKey(key) + if err != nil { + t.Fatalf("MarshalPrivateKey: %v", err) + } + + ttc := testTClient{ + trees: make(map[int64]*trillian.Tree), + leaves: make(map[int64][]*trillian.LogLeaf), + privateKey: &keyspb.PrivateKey{ + Der: keyDer, + }, + } + + return &ttc +} + // newTestTlog returns a tlog used for testing. func newTestTlog(t *testing.T, dir, id string) *tlog { + t.Helper() + dir, err := ioutil.TempDir(dir, id) if err != nil { t.Fatalf("TempDir: %v", err) @@ -111,15 +171,16 @@ func newTestTlog(t *testing.T, dir, id string) *tlog { store := filesystem.New(dir) - tclient, err := newTestTClient(t) + key, err := sbox.NewKey() if err != nil { - t.Fatalf("newTestTClient: %v", err) + t.Fatalf("NewKey: %v", err) } tlog := tlog{ id: id, - encryptionKey: nil, - trillian: tclient, + dcrtime: nil, + encryptionKey: newEncryptionKey(key), + trillian: newTestTClient(t), store: store, } @@ -130,6 +191,8 @@ func newTestTlog(t *testing.T, dir, id string) *tlog { // tlog and trillian client, providing the framework needed for // writing tlog backend tests. func newTestTlogBackend(t *testing.T) (*tlogBackend, func()) { + t.Helper() + testDir, err := ioutil.TempDir("", "tlog.backend.test") if err != nil { t.Fatalf("TempDir: %v", err) @@ -137,13 +200,14 @@ func newTestTlogBackend(t *testing.T) (*tlogBackend, func()) { testDataDir := filepath.Join(testDir, "data") tlogBackend := tlogBackend{ - homeDir: testDir, - dataDir: testDataDir, - unvetted: newTestTlog(t, testDir, "unvetted"), - vetted: newTestTlog(t, testDir, "vetted"), - plugins: make(map[string]plugin), - prefixes: make(map[string][]byte), - vettedTreeIDs: make(map[string]int64), + activeNetParams: chaincfg.TestNet3Params(), + homeDir: testDir, + dataDir: testDataDir, + unvetted: newTestTlog(t, testDir, "unvetted"), + vetted: newTestTlog(t, testDir, "vetted"), + plugins: make(map[string]plugin), + prefixes: make(map[string][]byte), + vettedTreeIDs: make(map[string]int64), inv: recordInventory{ unvetted: make(map[backend.MDStatusT][]string), vetted: make(map[backend.MDStatusT][]string), diff --git a/politeiad/backend/tlogbe/tlogbe_test.go b/politeiad/backend/tlogbe/tlogbe_test.go index f0af74495..7eede6b65 100644 --- a/politeiad/backend/tlogbe/tlogbe_test.go +++ b/politeiad/backend/tlogbe/tlogbe_test.go @@ -198,6 +198,7 @@ func TestUpdateUnvettedRecord(t *testing.T) { if errors.As(err, &contentError) { if test.wantContentErr == nil { t.Errorf("got error %v, want nil", err) + return } if contentError.ErrorCode != test.wantContentErr.ErrorCode { t.Errorf("got error %v, want %v", @@ -233,10 +234,7 @@ func TestUpdateVettedRecord(t *testing.T) { if err != nil { t.Error(err) } - md = append(md, backend.MetadataStream{ - ID: 2, - Payload: "", - }) + md = append(md, newBackendMetadataStream(t, 2, "")) // Publish the created record err = tlogBackend.unvettedPublish(token, *rec, md, fs) @@ -292,10 +290,7 @@ func TestUpdateVettedRecord(t *testing.T) { if err != nil { t.Error(err) } - md = append(md, backend.MetadataStream{ - ID: 3, - Payload: "", - }) + md = append(md, newBackendMetadataStream(t, 3, "")) err = tlogBackend.unvettedPublish(tokenFrozen, *recFrozen, md, fs) if err != nil { t.Error(err) @@ -382,6 +377,7 @@ func TestUpdateVettedRecord(t *testing.T) { if errors.As(err, &contentError) { if test.wantContentErr == nil { t.Errorf("got error %v, want nil", err) + return } if contentError.ErrorCode != test.wantContentErr.ErrorCode { t.Errorf("got error %v, want %v", @@ -551,6 +547,7 @@ func TestUpdateUnvettedMetadata(t *testing.T) { if errors.As(err, &contentError) { if test.wantContentErr == nil { t.Errorf("got error %v, want nil", err) + return } if contentError.ErrorCode != test.wantContentErr.ErrorCode { t.Errorf("got error %v, want %v", @@ -586,10 +583,7 @@ func TestUpdateVettedMetadata(t *testing.T) { if err != nil { t.Error(err) } - md = append(md, backend.MetadataStream{ - ID: 3, - Payload: "", - }) + md = append(md, newBackendMetadataStream(t, 2, "")) err = tlogBackend.unvettedPublish(token, *rec, md, fs) if err != nil { t.Error(err) @@ -634,10 +628,7 @@ func TestUpdateVettedMetadata(t *testing.T) { if err != nil { t.Error(err) } - md = append(md, backend.MetadataStream{ - ID: 4, - Payload: "", - }) + md = append(md, newBackendMetadataStream(t, 3, "")) err = tlogBackend.unvettedPublish(tokenFrozen, *recFrozen, md, fs) if err != nil { t.Error(err) @@ -670,10 +661,9 @@ func TestUpdateVettedMetadata(t *testing.T) { { "invalid token", tokenShort, - []backend.MetadataStream{{ - ID: 2, - Payload: "random", - }}, + []backend.MetadataStream{ + newBackendMetadataStream(t, 2, "random"), + }, []backend.MetadataStream{}, &backend.ContentVerificationError{ ErrorCode: v1.ErrorStatusInvalidToken, @@ -683,10 +673,9 @@ func TestUpdateVettedMetadata(t *testing.T) { { "record not found", tokenRandom, - []backend.MetadataStream{{ - ID: 2, - Payload: "random", - }}, + []backend.MetadataStream{ + newBackendMetadataStream(t, 2, "random"), + }, []backend.MetadataStream{}, nil, backend.ErrRecordNotFound, @@ -694,10 +683,9 @@ func TestUpdateVettedMetadata(t *testing.T) { { "tree frozen for changes", tokenFrozen, - []backend.MetadataStream{{ - ID: 2, - Payload: "random", - }}, + []backend.MetadataStream{ + newBackendMetadataStream(t, 2, "random"), + }, []backend.MetadataStream{}, nil, backend.ErrRecordLocked, @@ -706,10 +694,9 @@ func TestUpdateVettedMetadata(t *testing.T) { "no changes to record metadata, same payload", token, []backend.MetadataStream{}, - []backend.MetadataStream{{ - ID: 3, - Payload: "", - }}, + []backend.MetadataStream{ + newBackendMetadataStream(t, 2, ""), + }, nil, backend.ErrNoChanges, }, @@ -717,10 +704,9 @@ func TestUpdateVettedMetadata(t *testing.T) { "success", token, []backend.MetadataStream{}, - []backend.MetadataStream{{ - ID: 1, - Payload: "newdata", - }}, + []backend.MetadataStream{ + newBackendMetadataStream(t, 1, "newdata"), + }, nil, nil, }, @@ -737,6 +723,7 @@ func TestUpdateVettedMetadata(t *testing.T) { if errors.As(err, &contentError) { if test.wantContentErr == nil { t.Errorf("got error %v, want nil", err) + return } if contentError.ErrorCode != test.wantContentErr.ErrorCode { t.Errorf("got error %v, want %v", @@ -819,10 +806,7 @@ func TestVettedExists(t *testing.T) { if err != nil { t.Error(err) } - md = append(md, backend.MetadataStream{ - ID: 2, - Payload: "", - }) + md = append(md, newBackendMetadataStream(t, 2, "")) err = tlogBackend.unvettedPublish(tokenVetted, *vetted, md, fs) if err != nil { t.Error(err) @@ -903,10 +887,7 @@ func TestGetVetted(t *testing.T) { if err != nil { t.Error(err) } - md = append(md, backend.MetadataStream{ - ID: 2, - Payload: "", - }) + md = append(md, newBackendMetadataStream(t, 2, "")) err = tlogBackend.unvettedPublish(token, *rec, md, fs) if err != nil { t.Error(err) @@ -1082,6 +1063,7 @@ func TestSetUnvettedStatus(t *testing.T) { if errors.As(err, &contentError) { if test.wantContentErr == nil { t.Errorf("got error %v, want nil", err) + return } if contentError.ErrorCode != test.wantContentErr.ErrorCode { t.Errorf("got error %v, want %v", @@ -1122,10 +1104,7 @@ func TestSetVettedStatus(t *testing.T) { t.Error(err) } - md = append(md, backend.MetadataStream{ - ID: 2, - Payload: "", - }) + md = append(md, newBackendMetadataStream(t, 2, "")) _, err = tlogBackend.SetUnvettedStatus(tokenVetToUnvet, backend.MDStatusVetted, md, []backend.MetadataStream{}) if err != nil { @@ -1140,10 +1119,7 @@ func TestSetVettedStatus(t *testing.T) { if err != nil { t.Error(err) } - md = append(md, backend.MetadataStream{ - ID: 3, - Payload: "", - }) + md = append(md, newBackendMetadataStream(t, 3, "")) _, err = tlogBackend.SetUnvettedStatus(tokenVetToVet, backend.MDStatusVetted, md, []backend.MetadataStream{}) if err != nil { @@ -1161,10 +1137,7 @@ func TestSetVettedStatus(t *testing.T) { if err != nil { t.Error(err) } - md = append(md, backend.MetadataStream{ - ID: 4, - Payload: "", - }) + md = append(md, newBackendMetadataStream(t, 4, "")) _, err = tlogBackend.SetUnvettedStatus(tokenVetToArch, backend.MDStatusVetted, md, []backend.MetadataStream{}) if err != nil { @@ -1179,10 +1152,7 @@ func TestSetVettedStatus(t *testing.T) { if err != nil { t.Error(err) } - md = append(md, backend.MetadataStream{ - ID: 5, - Payload: "", - }) + md = append(md, newBackendMetadataStream(t, 5, "")) _, err = tlogBackend.SetUnvettedStatus(tokenVetToCensored, backend.MDStatusVetted, md, []backend.MetadataStream{}) if err != nil { @@ -1283,6 +1253,7 @@ func TestSetVettedStatus(t *testing.T) { if errors.As(err, &contentError) { if test.wantContentErr == nil { t.Errorf("got error %v, want nil", err) + return } if contentError.ErrorCode != test.wantContentErr.ErrorCode { t.Errorf("got error %v, want %v", diff --git a/politeiad/backend/tlogbe/trillianclient.go b/politeiad/backend/tlogbe/trillianclient.go index 56d117280..5268b5965 100644 --- a/politeiad/backend/tlogbe/trillianclient.go +++ b/politeiad/backend/tlogbe/trillianclient.go @@ -13,7 +13,6 @@ import ( "io/ioutil" "math/rand" "sync" - "testing" "time" "github.com/decred/politeia/util" @@ -722,24 +721,3 @@ func (t *testTClient) inclusionProof(treeID int64, merkleLeafHash []byte, lr *ty // // This function satisfies the trillianClient interface. func (t *testTClient) close() {} - -// newTestTClient returns a new testTClient. -func newTestTClient(t *testing.T) (*testTClient, error) { - // Create trillian private key - key, err := newTrillianKey() - if err != nil { - return nil, err - } - keyDer, err := der.MarshalPrivateKey(key) - if err != nil { - return nil, err - } - - return &testTClient{ - trees: make(map[int64]*trillian.Tree), - leaves: make(map[int64][]*trillian.LogLeaf), - privateKey: &keyspb.PrivateKey{ - Der: keyDer, - }, - }, nil -} From b3d23fb9b27e2fc836fa7d39da798b3b1897dd0e Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Mon, 11 Jan 2021 18:37:34 -0300 Subject: [PATCH 208/449] tlogbe: CmdEdit comment plugin tests. --- politeiad/backend/tlogbe/comments.go | 15 +- politeiad/backend/tlogbe/comments_test.go | 284 +++++++++++++++++++++- politeiad/backend/tlogbe/pi_test.go | 15 +- politeiad/backend/tlogbe/testing.go | 51 +++- politeiad/backend/tlogbe/tlogbe_test.go | 126 +++++----- 5 files changed, 404 insertions(+), 87 deletions(-) diff --git a/politeiad/backend/tlogbe/comments.go b/politeiad/backend/tlogbe/comments.go index 005c0cffd..6c5e3057c 100644 --- a/politeiad/backend/tlogbe/comments.go +++ b/politeiad/backend/tlogbe/comments.go @@ -940,7 +940,7 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { default: return "", backend.PluginUserError{ PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), + ErrorCode: int(comments.ErrorStatusStateInvalid), } } @@ -985,6 +985,12 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { // Get the existing comment cs, err := p.comments(e.State, token, *idx, []uint32{e.CommentID}) if err != nil { + if errors.Is(err, errRecordNotFound) { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusRecordNotFound), + } + } return "", fmt.Errorf("comments %v: %v", e.CommentID, err) } existing, ok := cs[e.CommentID] @@ -1027,6 +1033,7 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { receipt := p.identity.SignMessage([]byte(e.Signature)) ca := comments.CommentAdd{ UserID: e.UserID, + State: e.State, Token: e.Token, ParentID: e.ParentID, Comment: e.Comment, @@ -1041,12 +1048,6 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { // Save comment merkle, err := p.commentAddSave(ca) if err != nil { - if errors.Is(err, errRecordNotFound) { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusRecordNotFound), - } - } return "", fmt.Errorf("commentSave: %v", err) } diff --git a/politeiad/backend/tlogbe/comments_test.go b/politeiad/backend/tlogbe/comments_test.go index 989e1ca8a..6bbc0e0c7 100644 --- a/politeiad/backend/tlogbe/comments_test.go +++ b/politeiad/backend/tlogbe/comments_test.go @@ -43,7 +43,7 @@ func TestCmdNew(t *testing.T) { } rec, err := tlogBackend.New(md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } // Helpers @@ -54,7 +54,7 @@ func TestCmdNew(t *testing.T) { uid, err := identity.New() if err != nil { - t.Error(err) + t.Fatal(err) } // Setup new comment plugin tests @@ -223,3 +223,283 @@ func TestCmdNew(t *testing.T) { }) } } + +func TestCmdEdit(t *testing.T) { + commentsPlugin, tlogBackend, cleanup := newTestCommentsPlugin(t) + defer cleanup() + + // New record + md := []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs := []backend.File{ + newBackendFile(t, "index.md"), + } + rec, err := tlogBackend.New(md, fs) + if err != nil { + t.Fatal(err) + } + + // Helpers + comment := "random comment" + commentEdit := comment + "more content" + parentID := uint32(0) + invalidParentID := uint32(3) + invalidCommentID := uint32(3) + tokenRandom := hex.EncodeToString(tokenFromTreeID(123)) + + id, err := identity.New() + if err != nil { + t.Fatal(err) + } + + // New comment + ncEncoded, err := comments.EncodeNew( + comments.New{ + UserID: uuid.New().String(), + State: comments.StateUnvetted, + Token: rec.Token, + ParentID: parentID, + Comment: comment, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, comments.StateUnvetted, + rec.Token, comment, parentID), + }, + ) + if err != nil { + t.Fatal(err) + } + reply, err := commentsPlugin.cmdNew(string(ncEncoded)) + if err != nil { + t.Fatal(err) + } + nr, err := comments.DecodeNewReply([]byte(reply)) + if err != nil { + t.Fatal(err) + } + + // Setup edit comment plugin tests + var tests = []struct { + description string + payload comments.Edit + wantErr *backend.PluginUserError + }{ + { + "invalid comment state", + comments.Edit{ + UserID: nr.Comment.UserID, + State: comments.StateInvalid, + Token: rec.Token, + ParentID: nr.Comment.ParentID, + CommentID: nr.Comment.CommentID, + Comment: commentEdit, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, comments.StateInvalid, + rec.Token, commentEdit, nr.Comment.ParentID), + }, + &backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusStateInvalid), + }, + }, + { + "invalid token", + comments.Edit{ + UserID: nr.Comment.UserID, + State: nr.Comment.State, + Token: "invalid", + ParentID: nr.Comment.ParentID, + CommentID: nr.Comment.CommentID, + Comment: commentEdit, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, nr.Comment.State, "invalid", + commentEdit, nr.Comment.ParentID), + }, + &backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusTokenInvalid), + }, + }, + { + "invalid signature", + comments.Edit{ + UserID: nr.Comment.UserID, + State: nr.Comment.State, + Token: rec.Token, + ParentID: nr.Comment.ParentID, + CommentID: nr.Comment.CommentID, + Comment: commentEdit, + PublicKey: id.Public.String(), + Signature: "invalid", + }, + &backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusSignatureInvalid), + }, + }, + { + "invalid public key", + comments.Edit{ + UserID: nr.Comment.UserID, + State: nr.Comment.State, + Token: rec.Token, + ParentID: nr.Comment.ParentID, + CommentID: nr.Comment.CommentID, + Comment: commentEdit, + PublicKey: "invalid", + Signature: commentSignature(t, id, nr.Comment.State, rec.Token, + commentEdit, nr.Comment.ParentID), + }, + &backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusPublicKeyInvalid), + }, + }, + { + "comment max length exceeded", + comments.Edit{ + UserID: nr.Comment.UserID, + State: nr.Comment.State, + Token: rec.Token, + ParentID: nr.Comment.ParentID, + CommentID: nr.Comment.CommentID, + Comment: commentMaxLengthExceeded(t), + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, nr.Comment.State, rec.Token, + commentMaxLengthExceeded(t), nr.Comment.ParentID), + }, + &backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusCommentTextInvalid), + }, + }, + { + "comment id not found", + comments.Edit{ + UserID: nr.Comment.UserID, + State: nr.Comment.State, + Token: rec.Token, + ParentID: nr.Comment.ParentID, + CommentID: invalidCommentID, + Comment: commentEdit, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, nr.Comment.State, rec.Token, + commentEdit, nr.Comment.ParentID), + }, + &backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusCommentNotFound), + }, + }, + { + "unauthorized user", + comments.Edit{ + UserID: uuid.New().String(), + State: nr.Comment.State, + Token: rec.Token, + ParentID: nr.Comment.ParentID, + CommentID: nr.Comment.CommentID, + Comment: commentEdit, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, nr.Comment.State, rec.Token, + commentEdit, nr.Comment.ParentID), + }, + &backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusUserUnauthorized), + }, + }, + { + "invalid parent ID", + comments.Edit{ + UserID: nr.Comment.UserID, + State: nr.Comment.State, + Token: rec.Token, + ParentID: invalidParentID, + CommentID: nr.Comment.CommentID, + Comment: commentEdit, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, nr.Comment.State, + rec.Token, commentEdit, invalidParentID), + }, + &backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusParentIDInvalid), + }, + }, + { + "comment did not change", + comments.Edit{ + UserID: nr.Comment.UserID, + State: nr.Comment.State, + Token: rec.Token, + ParentID: nr.Comment.ParentID, + CommentID: nr.Comment.CommentID, + Comment: comment, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, nr.Comment.State, + rec.Token, comment, nr.Comment.ParentID), + }, + &backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusCommentTextInvalid), + }, + }, + { + "record not found", + comments.Edit{ + UserID: nr.Comment.UserID, + State: nr.Comment.State, + Token: tokenRandom, + ParentID: nr.Comment.ParentID, + CommentID: nr.Comment.ParentID, + Comment: commentEdit, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, nr.Comment.State, + tokenRandom, commentEdit, nr.Comment.ParentID), + }, + &backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusRecordNotFound), + }, + }, + { + "success", + comments.Edit{ + UserID: nr.Comment.UserID, + State: nr.Comment.State, + Token: rec.Token, + ParentID: nr.Comment.ParentID, + CommentID: nr.Comment.CommentID, + Comment: commentEdit, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, nr.Comment.State, + rec.Token, commentEdit, nr.Comment.ParentID), + }, + nil, + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + // Edit Comment + ecEncoded, err := comments.EncodeEdit(test.payload) + if err != nil { + t.Error(err) + } + + // Execute plugin command + _, err = commentsPlugin.cmdEdit(string(ecEncoded)) + + // Parse plugin user error + var pluginUserError backend.PluginUserError + if errors.As(err, &pluginUserError) { + if test.wantErr == nil { + t.Errorf("got error %v, want nil", err) + return + } + if pluginUserError.ErrorCode != test.wantErr.ErrorCode { + t.Errorf("got error %v, want %v", + pluginUserError.ErrorCode, + test.wantErr.ErrorCode) + } + return + } + + // Expecting nil err + if err != nil { + t.Errorf("got error %v, want nil", err) + } + }) + } +} diff --git a/politeiad/backend/tlogbe/pi_test.go b/politeiad/backend/tlogbe/pi_test.go index 2b6490e39..2990c7401 100644 --- a/politeiad/backend/tlogbe/pi_test.go +++ b/politeiad/backend/tlogbe/pi_test.go @@ -17,9 +17,10 @@ import ( ) func TestCommentNew(t *testing.T) { - tlogBackend, cleanup := newTestTlogBackend(t) + piPlugin, tlogBackend, cleanup := newTestPiPlugin(t) defer cleanup() + // Register comments plugin id, err := identity.New() if err != nil { t.Fatal(err) @@ -28,14 +29,6 @@ func TestCommentNew(t *testing.T) { Key: pluginSettingDataDir, Value: tlogBackend.dataDir, }} - - piPlugin, err := newPiPlugin(tlogBackend, newBackendClient(tlogBackend), - settings, tlogBackend.activeNetParams) - if err != nil { - t.Fatal(err) - } - - // Register comments plugin tlogBackend.RegisterPlugin(backend.Plugin{ ID: comments.ID, Version: comments.Version, @@ -52,7 +45,7 @@ func TestCommentNew(t *testing.T) { } rec, err := tlogBackend.New(md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } // Helpers @@ -62,7 +55,7 @@ func TestCommentNew(t *testing.T) { uid, err := identity.New() if err != nil { - t.Error(err) + t.Fatal(err) } // Setup comment new pi plugin tests diff --git a/politeiad/backend/tlogbe/testing.go b/politeiad/backend/tlogbe/testing.go index 446bbedf9..731f4301e 100644 --- a/politeiad/backend/tlogbe/testing.go +++ b/politeiad/backend/tlogbe/testing.go @@ -60,13 +60,13 @@ func newBackendFileJPEG(t *testing.T) backend.File { err := jpeg.Encode(b, img, &jpeg.Options{}) if err != nil { - t.Fatalf("%v", err) + t.Fatal(err) } // Generate a random name r, err := util.Random(8) if err != nil { - t.Fatalf("%v", err) + t.Fatal(err) } return backend.File{ @@ -85,13 +85,13 @@ func newBackendFilePNG(t *testing.T) backend.File { err := png.Encode(b, img) if err != nil { - t.Fatalf("%v", err) + t.Fatal(err) } // Generate a random name r, err := util.Random(8) if err != nil { - t.Fatalf("%v", err) + t.Fatal(err) } return backend.File{ @@ -227,6 +227,49 @@ func newTestTlogBackend(t *testing.T) (*tlogBackend, func()) { } } +func newTestCommentsPlugin(t *testing.T) (*commentsPlugin, *tlogBackend, func()) { + t.Helper() + + tlogBackend, cleanup := newTestTlogBackend(t) + + id, err := identity.New() + if err != nil { + t.Fatalf("identity New: %v", err) + } + + settings := []backend.PluginSetting{{ + Key: pluginSettingDataDir, + Value: tlogBackend.dataDir, + }} + + commentsPlugin, err := newCommentsPlugin(tlogBackend, + newBackendClient(tlogBackend), settings, id) + if err != nil { + t.Fatalf("newCommentsPlugin: %v", err) + } + + return commentsPlugin, tlogBackend, cleanup +} + +func newTestPiPlugin(t *testing.T) (*piPlugin, *tlogBackend, func()) { + t.Helper() + + tlogBackend, cleanup := newTestTlogBackend(t) + + settings := []backend.PluginSetting{{ + Key: pluginSettingDataDir, + Value: tlogBackend.dataDir, + }} + + piPlugin, err := newPiPlugin(tlogBackend, newBackendClient(tlogBackend), + settings, tlogBackend.activeNetParams) + if err != nil { + t.Fatalf("newPiPlugin: %v", err) + } + + return piPlugin, tlogBackend, cleanup +} + // recordContentTests defines the type used to describe the content // verification error tests. type recordContentTest struct { diff --git a/politeiad/backend/tlogbe/tlogbe_test.go b/politeiad/backend/tlogbe/tlogbe_test.go index 7eede6b65..d876ae455 100644 --- a/politeiad/backend/tlogbe/tlogbe_test.go +++ b/politeiad/backend/tlogbe/tlogbe_test.go @@ -62,11 +62,11 @@ func TestUpdateUnvettedRecord(t *testing.T) { } rec, err := tlogBackend.New(md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } token, err := tokenDecode(rec.Token) if err != nil { - t.Error(err) + t.Fatal(err) } // Test all record content verification error through the @@ -102,7 +102,7 @@ func TestUpdateUnvettedRecord(t *testing.T) { // test case: Token not full length tokenShort, err := util.ConvertStringToken(util.TokenToPrefix(rec.Token)) if err != nil { - t.Error(err) + t.Fatal(err) } // test case: Record not found @@ -111,16 +111,16 @@ func TestUpdateUnvettedRecord(t *testing.T) { // test case: Frozen tree recFrozen, err := tlogBackend.New(md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } tokenFrozen, err := tokenDecode(recFrozen.Token) if err != nil { - t.Error(err) + t.Fatal(err) } err = tlogBackend.unvetted.treeFreeze(treeIDFromToken(tokenFrozen), backend.RecordMetadata{}, []backend.MetadataStream{}, 0) if err != nil { - t.Error(err) + t.Fatal(err) } // Setup UpdateUnvettedRecord tests @@ -228,18 +228,18 @@ func TestUpdateVettedRecord(t *testing.T) { } rec, err := tlogBackend.New(md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } token, err := tokenDecode(rec.Token) if err != nil { - t.Error(err) + t.Fatal(err) } md = append(md, newBackendMetadataStream(t, 2, "")) // Publish the created record err = tlogBackend.unvettedPublish(token, *rec, md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } // Test all record content verification error through the @@ -284,22 +284,22 @@ func TestUpdateVettedRecord(t *testing.T) { // test case: Frozen tree recFrozen, err := tlogBackend.New(md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } tokenFrozen, err := tokenDecode(recFrozen.Token) if err != nil { - t.Error(err) + t.Fatal(err) } md = append(md, newBackendMetadataStream(t, 3, "")) err = tlogBackend.unvettedPublish(tokenFrozen, *recFrozen, md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } treeIDFrozenVetted := tlogBackend.vettedTreeIDs[recFrozen.Token] err = tlogBackend.vetted.treeFreeze(treeIDFrozenVetted, backend.RecordMetadata{}, []backend.MetadataStream{}, 0) if err != nil { - t.Error(err) + t.Fatal(err) } // Setup UpdateVettedRecord tests @@ -407,11 +407,11 @@ func TestUpdateUnvettedMetadata(t *testing.T) { } rec, err := tlogBackend.New(md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } token, err := tokenDecode(rec.Token) if err != nil { - t.Error(err) + t.Fatal(err) } // Test all record content verification error through the @@ -438,7 +438,7 @@ func TestUpdateUnvettedMetadata(t *testing.T) { // test case: Token not full length tokenShort, err := util.ConvertStringToken(util.TokenToPrefix(rec.Token)) if err != nil { - t.Error(err) + t.Fatal(err) } // test case: Record not found @@ -447,16 +447,16 @@ func TestUpdateUnvettedMetadata(t *testing.T) { // test case: Frozen tree recFrozen, err := tlogBackend.New(md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } tokenFrozen, err := tokenDecode(recFrozen.Token) if err != nil { - t.Error(err) + t.Fatal(err) } err = tlogBackend.unvetted.treeFreeze(treeIDFromToken(tokenFrozen), backend.RecordMetadata{}, []backend.MetadataStream{}, 0) if err != nil { - t.Error(err) + t.Fatal(err) } // Setup UpdateUnvettedMetadata tests @@ -577,16 +577,16 @@ func TestUpdateVettedMetadata(t *testing.T) { } rec, err := tlogBackend.New(md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } token, err := tokenDecode(rec.Token) if err != nil { - t.Error(err) + t.Fatal(err) } md = append(md, newBackendMetadataStream(t, 2, "")) err = tlogBackend.unvettedPublish(token, *rec, md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } // Test all record content verification error through the @@ -613,7 +613,7 @@ func TestUpdateVettedMetadata(t *testing.T) { // test case: Token not full length tokenShort, err := util.ConvertStringToken(util.TokenToPrefix(rec.Token)) if err != nil { - t.Error(err) + t.Fatal(err) } // test case: Record not found @@ -622,22 +622,22 @@ func TestUpdateVettedMetadata(t *testing.T) { // test case: Frozen tree recFrozen, err := tlogBackend.New(md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } tokenFrozen, err := tokenDecode(recFrozen.Token) if err != nil { - t.Error(err) + t.Fatal(err) } md = append(md, newBackendMetadataStream(t, 3, "")) err = tlogBackend.unvettedPublish(tokenFrozen, *recFrozen, md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } treeIDFrozenVetted := tlogBackend.vettedTreeIDs[recFrozen.Token] err = tlogBackend.vetted.treeFreeze(treeIDFrozenVetted, backend.RecordMetadata{}, []backend.MetadataStream{}, 0) if err != nil { - t.Error(err) + t.Fatal(err) } // Setup UpdateVettedMetadata tests @@ -753,11 +753,11 @@ func TestUnvettedExists(t *testing.T) { } rec, err := tlogBackend.New(md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } token, err := tokenDecode(rec.Token) if err != nil { - t.Error(err) + t.Fatal(err) } // Random token @@ -790,26 +790,26 @@ func TestVettedExists(t *testing.T) { } unvetted, err := tlogBackend.New(md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } tokenUnvetted, err := tokenDecode(unvetted.Token) if err != nil { - t.Error(err) + t.Fatal(err) } // Create vetted record vetted, err := tlogBackend.New(md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } tokenVetted, err := tokenDecode(vetted.Token) if err != nil { - t.Error(err) + t.Fatal(err) } md = append(md, newBackendMetadataStream(t, 2, "")) err = tlogBackend.unvettedPublish(tokenVetted, *vetted, md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } // Run VettedExists test cases @@ -817,12 +817,12 @@ func TestVettedExists(t *testing.T) { // Record exists result := tlogBackend.VettedExists(tokenVetted) if result == false { - t.Errorf("got false, want true") + t.Fatal("got false, want true") } // Record does not exist result = tlogBackend.VettedExists(tokenUnvetted) if result == true { - t.Errorf("got true, want false") + t.Fatal("got true, want false") } } @@ -839,11 +839,11 @@ func TestGetUnvetted(t *testing.T) { } rec, err := tlogBackend.New(md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } token, err := tokenDecode(rec.Token) if err != nil { - t.Error(err) + t.Fatal(err) } // Random token @@ -881,16 +881,16 @@ func TestGetVetted(t *testing.T) { } rec, err := tlogBackend.New(md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } token, err := tokenDecode(rec.Token) if err != nil { - t.Error(err) + t.Fatal(err) } md = append(md, newBackendMetadataStream(t, 2, "")) err = tlogBackend.unvettedPublish(token, *rec, md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } // Random token @@ -932,20 +932,20 @@ func TestSetUnvettedStatus(t *testing.T) { // test case: Unvetted to archived recUnvetToArch, err := tlogBackend.New(md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } tokenUnvetToArch, err := tokenDecode(recUnvetToArch.Token) if err != nil { - t.Error(err) + t.Fatal(err) } // test case: Unvetted to unvetted recUnvetToUnvet, err := tlogBackend.New(md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } tokenUnvetToUnvet, err := tokenDecode(recUnvetToUnvet.Token) if err != nil { - t.Error(err) + t.Fatal(err) } // Valid status transitions @@ -953,27 +953,27 @@ func TestSetUnvettedStatus(t *testing.T) { // test case: Unvetted to vetted recUnvetToVet, err := tlogBackend.New(md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } tokenUnvetToVet, err := tokenDecode(recUnvetToVet.Token) if err != nil { - t.Error(err) + t.Fatal(err) } // test case: Unvetted to censored recUnvetToCensored, err := tlogBackend.New(md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } tokenUnvetToCensored, err := tokenDecode(recUnvetToCensored.Token) if err != nil { - t.Error(err) + t.Fatal(err) } // test case: Token not full length tokenShort, err := util.ConvertStringToken( util.TokenToPrefix(recUnvetToVet.Token)) if err != nil { - t.Error(err) + t.Fatal(err) } // test case: Record not found @@ -1097,33 +1097,33 @@ func TestSetVettedStatus(t *testing.T) { // test case: Vetted to unvetted recVetToUnvet, err := tlogBackend.New(md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } tokenVetToUnvet, err := tokenDecode(recVetToUnvet.Token) if err != nil { - t.Error(err) + t.Fatal(err) } md = append(md, newBackendMetadataStream(t, 2, "")) _, err = tlogBackend.SetUnvettedStatus(tokenVetToUnvet, backend.MDStatusVetted, md, []backend.MetadataStream{}) if err != nil { - t.Error(err) + t.Fatal(err) } // test case: Vetted to vetted recVetToVet, err := tlogBackend.New(md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } tokenVetToVet, err := tokenDecode(recVetToVet.Token) if err != nil { - t.Error(err) + t.Fatal(err) } md = append(md, newBackendMetadataStream(t, 3, "")) _, err = tlogBackend.SetUnvettedStatus(tokenVetToVet, backend.MDStatusVetted, md, []backend.MetadataStream{}) if err != nil { - t.Error(err) + t.Fatal(err) } // Valid status transitions @@ -1131,39 +1131,39 @@ func TestSetVettedStatus(t *testing.T) { // test case: Vetted to archived recVetToArch, err := tlogBackend.New(md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } tokenVetToArch, err := tokenDecode(recVetToArch.Token) if err != nil { - t.Error(err) + t.Fatal(err) } md = append(md, newBackendMetadataStream(t, 4, "")) _, err = tlogBackend.SetUnvettedStatus(tokenVetToArch, backend.MDStatusVetted, md, []backend.MetadataStream{}) if err != nil { - t.Error(err) + t.Fatal(err) } // test case: Vetted to censored recVetToCensored, err := tlogBackend.New(md, fs) if err != nil { - t.Error(err) + t.Fatal(err) } tokenVetToCensored, err := tokenDecode(recVetToCensored.Token) if err != nil { - t.Error(err) + t.Fatal(err) } md = append(md, newBackendMetadataStream(t, 5, "")) _, err = tlogBackend.SetUnvettedStatus(tokenVetToCensored, backend.MDStatusVetted, md, []backend.MetadataStream{}) if err != nil { - t.Error(err) + t.Fatal(err) } // test case: Token not full length tokenShort, err := util.ConvertStringToken( util.TokenToPrefix(recVetToCensored.Token)) if err != nil { - t.Error(err) + t.Fatal(err) } // test case: Record not found From ef2b78266ecc848997fef2e5e3a251242bbeea75 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 11 Jan 2021 21:10:39 -0600 Subject: [PATCH 209/449] multi: Add comment timestamp routes. --- politeiad/api/v1/v1.go | 4 +- politeiad/backend/backend.go | 4 +- politeiad/backend/tlogbe/anchor.go | 4 +- politeiad/backend/tlogbe/comments.go | 148 ++++++++++++++- politeiad/backend/tlogbe/timestamp.go | 11 +- politeiad/backend/tlogbe/tlog.go | 1 + politeiad/backend/tlogbe/tlogclient.go | 64 ++++++- politeiad/plugins/comments/comments.go | 53 +++++- politeiad/plugins/dcrdata/dcrdata.go | 3 +- politeiad/plugins/pi/pi.go | 3 +- politeiad/plugins/ticketvote/ticketvote.go | 3 +- politeiad/politeiad.go | 4 - politeiawww/api/comments/v1/v1.go | 127 +++++++++++++ politeiawww/api/records/v1/v1.go | 32 +++- politeiawww/cmd/piwww/commenttimestamps.go | 126 +++++++++++++ politeiawww/cmd/piwww/piwww.go | 11 +- politeiawww/cmd/piwww/recordtimestamps.go | 10 +- politeiawww/cmd/shared/client.go | 37 +++- politeiawww/comments.go | 205 ++++++++++++++++++++- politeiawww/piwww.go | 4 + politeiawww/records.go | 62 ++++--- 21 files changed, 833 insertions(+), 83 deletions(-) create mode 100644 politeiawww/api/comments/v1/v1.go create mode 100644 politeiawww/cmd/piwww/commenttimestamps.go diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index 750528eb1..cb16d1e13 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -361,7 +361,9 @@ type UpdateUnvettedMetadataReply struct { Response string `json:"response"` // Challenge response } -// Proof contains an inclusion proof for the digest in the merkle root. +// Proof contains an inclusion proof for the digest in the merkle root. The +// ExtraData field is used by certain types of proofs to include additional +// data that is required to validate the proof. type Proof struct { Type string `json:"type"` Digest string `json:"digest"` diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index fbfbc6bde..282f81c7b 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -149,7 +149,9 @@ type Record struct { Files []File // User provided files } -// Proof contains an inclusion proof for the digest in the merkle root. +// Proof contains an inclusion proof for the digest in the merkle root. The +// ExtraData field is used by certain types of proofs to include additional +// data that is required to validate the proof. type Proof struct { Type string Digest string diff --git a/politeiad/backend/tlogbe/anchor.go b/politeiad/backend/tlogbe/anchor.go index 98f61711f..ed70d384d 100644 --- a/politeiad/backend/tlogbe/anchor.go +++ b/politeiad/backend/tlogbe/anchor.go @@ -31,7 +31,9 @@ const ( // anchors a few minutes prior to that. // Seconds Minutes Hours Days Months DayOfWeek - anchorSchedule = "0 56 * * * *" // At minute 56 of every hour + // TODO put back when done testing + // anchorSchedule = "0 56 * * * *" // At minute 56 of every hour + anchorSchedule = "0 */5 * * * *" // Every 5 minutes // anchorID is included in the timestamp and verify requests as a // unique identifier. diff --git a/politeiad/backend/tlogbe/comments.go b/politeiad/backend/tlogbe/comments.go index 6c5e3057c..0fce11d95 100644 --- a/politeiad/backend/tlogbe/comments.go +++ b/politeiad/backend/tlogbe/comments.go @@ -36,8 +36,6 @@ import ( // TODO upvoting a comment twice in the same second causes a duplicate leaf // error which causes a 500. Solution: add the timestamp to the vote index. -// TODO verify all writes only accept full length tokens - const ( // Blob entry data descriptors dataDescriptorCommentAdd = "commentadd" @@ -89,8 +87,8 @@ type voteIndex struct { } type commentIndex struct { - Adds map[uint32][]byte `json:"adds"` // [version]merkleHash - Del []byte `json:"del"` // Merkle hash of delete record + Adds map[uint32][]byte `json:"adds"` // [version]merkleLeafHash + Del []byte `json:"del"` // Merkle leaf hash of delete record // Votes contains the vote history for each uuid that voted on the // comment. This data is cached because the effect of a new vote @@ -792,6 +790,34 @@ func (p *commentsPlugin) comment(s comments.StateT, token []byte, idx commentsIn return &c, nil } +func (p *commentsPlugin) timestamp(s comments.StateT, token []byte, merkle []byte) (*comments.Timestamp, error) { + // Get timestamp + tlogID := tlogIDFromCommentState(s) + t, err := p.tlog.timestamp(tlogID, token, merkle) + if err != nil { + return nil, fmt.Errorf("timestamp %x: %v", merkle, err) + } + + // Convert response + proofs := make([]comments.Proof, 0, len(t.Proofs)) + for _, v := range t.Proofs { + proofs = append(proofs, comments.Proof{ + Type: v.Type, + Digest: v.Digest, + MerkleRoot: v.MerkleRoot, + MerklePath: v.MerklePath, + ExtraData: v.ExtraData, + }) + } + return &comments.Timestamp{ + Data: t.Data, + Digest: t.Digest, + TxID: t.TxID, + MerkleRoot: t.MerkleRoot, + Proofs: proofs, + }, nil +} + func (p *commentsPlugin) cmdNew(payload string) (string, error) { log.Tracef("comments cmdNew: %v", payload) @@ -1773,6 +1799,118 @@ func (p *commentsPlugin) cmdVotes(payload string) (string, error) { return string(reply), nil } +func (p *commentsPlugin) cmdTimestamps(payload string) (string, error) { + log.Tracef("comments cmdVotes: %v", payload) + + // Decode payload + var t comments.Timestamps + err := json.Unmarshal([]byte(payload), &t) + if err != nil { + return "", err + } + + // Verify state + switch t.State { + case comments.StateUnvetted, comments.StateVetted: + // Allowed; continue + default: + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), + } + } + + // Verify token + token, err := util.ConvertStringToken(t.Token) + if err != nil { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), + } + } + + // Get comments index + idx, err := p.commentsIndex(t.State, token) + if err != nil { + return "", err + } + + // If no comment IDs were given then we need to return the + // timestamps for all comments. + if len(t.CommentIDs) == 0 { + commentIDs := make([]uint32, 0, len(idx.Comments)) + for k := range idx.Comments { + commentIDs = append(commentIDs, k) + } + t.CommentIDs = commentIDs + } + + // Get timestamps + cmts := make(map[uint32][]comments.Timestamp, len(t.CommentIDs)) + votes := make(map[uint32][]comments.Timestamp, len(t.CommentIDs)) + for _, commentID := range t.CommentIDs { + cidx, ok := idx.Comments[commentID] + if !ok { + // Comment ID does not exist. Skip it. + continue + } + + // Get timestamps for adds + ts := make([]comments.Timestamp, 0, len(cidx.Adds)+1) + for _, v := range cidx.Adds { + t, err := p.timestamp(t.State, token, v) + if err != nil { + return "", err + } + ts = append(ts, *t) + } + + // Get timestamp for del + if cidx.Del != nil { + t, err := p.timestamp(t.State, token, cidx.Del) + if err != nil { + return "", err + } + ts = append(ts, *t) + } + + // Save timestamps + cmts[commentID] = ts + + // Only get the comment vote timestamps if specified + if !t.IncludeVotes { + continue + } + + // Get timestamps for votes + ts = make([]comments.Timestamp, 0, len(cidx.Votes)) + for _, votes := range cidx.Votes { + for _, v := range votes { + t, err := p.timestamp(t.State, token, v.Merkle) + if err != nil { + return "", err + } + ts = append(ts, *t) + } + } + + // Save timestamps + votes[commentID] = ts + } + + // Prepare reply + ts := comments.TimestampsReply{ + Comments: cmts, + Votes: votes, + } + reply, err := json.Marshal(ts) + if err != nil { + return "", err + } + + return string(reply), nil +} + // cmd executes a plugin command. // // This function satisfies the pluginClient interface. @@ -1798,6 +1936,8 @@ func (p *commentsPlugin) cmd(cmd, payload string) (string, error) { return p.cmdCount(payload) case comments.CmdVotes: return p.cmdVotes(payload) + case comments.CmdTimestamps: + return p.cmdTimestamps(payload) } return "", backend.ErrPluginCmdInvalid diff --git a/politeiad/backend/tlogbe/timestamp.go b/politeiad/backend/tlogbe/timestamp.go index eca3e81a5..f231c4b52 100644 --- a/politeiad/backend/tlogbe/timestamp.go +++ b/politeiad/backend/tlogbe/timestamp.go @@ -138,10 +138,13 @@ func VerifyTimestamp(t backend.Timestamp) error { return fmt.Errorf("data has not been included in dcr tx yet") } - // Verify digest - d := hex.EncodeToString(util.Digest([]byte(t.Data))) - if d != t.Digest { - return fmt.Errorf("invalid digest: got %v, want %v", d, t.Digest) + // Verify digest. The data blob may not be included in certain + // scenerios such as if it has been censored. + if t.Data != "" { + d := hex.EncodeToString(util.Digest([]byte(t.Data))) + if d != t.Digest { + return fmt.Errorf("invalid digest: got %v, want %v", d, t.Digest) + } } // Verify proof ordering. The digest of the first proof should be diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index dac3bbddd..e4f773bb2 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -1571,6 +1571,7 @@ func (t *tlog) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trillian ts := backend.Timestamp{ Data: string(data), Digest: hex.EncodeToString(l.LeafValue), + Proofs: []backend.Proof{}, } // Get the anchor record for this leaf diff --git a/politeiad/backend/tlogbe/tlogclient.go b/politeiad/backend/tlogbe/tlogclient.go index 91d63e545..071d9350f 100644 --- a/politeiad/backend/tlogbe/tlogclient.go +++ b/politeiad/backend/tlogbe/tlogclient.go @@ -6,9 +6,9 @@ package tlogbe import ( "fmt" -) -// TODO verify writes only accept full length tokens + "github.com/decred/politeia/politeiad/backend" +) // tlogClient provides an API for the plugins to interact with the tlog // backend. Plugins are allowed to save, delete, and get plugin data to/from @@ -22,18 +22,23 @@ type tlogClient interface { // del deletes the blobs that correspond to the provided merkle // leaf hashes. - del(tlogID string, token []byte, merkles [][]byte) error + del(tlogID string, token []byte, merkleLeafHashes [][]byte) error // blobsByMerkle returns the blobs with the provided merkle leaf // hashes. If a blob does not exist it will not be included in the // returned map. blobsByMerkle(tlogID string, token []byte, - merkles [][]byte) (map[string][]byte, error) + merkleLeafHashes [][]byte) (map[string][]byte, error) // blobsByKeyPrefix returns all blobs that match the provided key // prefix. blobsByKeyPrefix(tlogID string, token []byte, keyPrefix string) ([][]byte, error) + + // timestamp returns the timestamp for a data blob that corresponds + // to the provided merkle leaf hash. + timestamp(tlogID string, token []byte, + merkleLeafHash []byte) (*backend.Timestamp, error) } var ( @@ -94,8 +99,11 @@ func (c *backendClient) treeIDFromTokenFullLength(tlogID string, token []byte) ( // save saves the provided blobs to the tlog backend. Note, hashes contains the // hashes of the data encoded in the blobs. The hashes must share the same // ordering as the blobs. +// +// This function satisfies the tlogClient interface. func (c *backendClient) save(tlogID string, token []byte, keyPrefix string, blobs, hashes [][]byte, encrypt bool) ([][]byte, error) { - log.Tracef("tlogClient save: %x %v %v %x", token, keyPrefix, encrypt, hashes) + log.Tracef("backendClient save: %x %v %v %x", + token, keyPrefix, encrypt, hashes) // Get tlog instance tlog, err := c.tlogByID(tlogID) @@ -114,8 +122,10 @@ func (c *backendClient) save(tlogID string, token []byte, keyPrefix string, blob } // del deletes the blobs that correspond to the provided merkle leaf hashes. +// +// This function satisfies the tlogClient interface. func (c *backendClient) del(tlogID string, token []byte, merkles [][]byte) error { - log.Tracef("tlogClient del: %x %x", token, merkles) + log.Tracef("backendClient del: %v %x %x", tlogID, token, merkles) // Get tlog instance tlog, err := c.tlogByID(tlogID) @@ -139,8 +149,10 @@ func (c *backendClient) del(tlogID string, token []byte, merkles [][]byte) error // If a blob does not exist it will not be included in the returned map. It is // the responsibility of the caller to check that a blob is returned for each // of the provided merkle hashes. +// +// This function satisfies the tlogClient interface. func (c *backendClient) blobsByMerkle(tlogID string, token []byte, merkles [][]byte) (map[string][]byte, error) { - log.Tracef("tlogClient blobsByMerkle: %x %x", token, merkles) + log.Tracef("backendClient blobsByMerkle: %v %x %x", tlogID, token, merkles) // Get tlog instance tlog, err := c.tlogByID(tlogID) @@ -159,8 +171,11 @@ func (c *backendClient) blobsByMerkle(tlogID string, token []byte, merkles [][]b } // blobsByKeyPrefix returns all blobs that match the provided key prefix. +// +// This function satisfies the tlogClient interface. func (c *backendClient) blobsByKeyPrefix(tlogID string, token []byte, keyPrefix string) ([][]byte, error) { - log.Tracef("tlogClient blobsByKeyPrefix: %x %x", token, keyPrefix) + log.Tracef("backendClient blobsByKeyPrefix: %v %x %v", + tlogID, token, keyPrefix) // Get tlog instance tlog, err := c.tlogByID(tlogID) @@ -178,9 +193,38 @@ func (c *backendClient) blobsByKeyPrefix(tlogID string, token []byte, keyPrefix return tlog.blobsByKeyPrefix(treeID, keyPrefix) } +// timestamp returns the timestamp for a data blob that corresponds to the +// provided merkle leaf hash. +// +// This function satisfies the tlogClient interface. +func (c *backendClient) timestamp(tlogID string, token []byte, merkle []byte) (*backend.Timestamp, error) { + log.Tracef("backendClient timestamp: %v %x %x", tlogID, token, merkle) + + // Get tlog instance + tlog, err := c.tlogByID(tlogID) + if err != nil { + return nil, err + } + + // Get tree ID + treeID, err := c.treeIDFromToken(tlogID, token) + if err != nil { + return nil, err + } + + // Get all tree leaves + leaves, err := tlog.trillian.leavesAll(treeID) + if err != nil { + return nil, fmt.Errorf("leavesAll: %v", err) + } + + // Get timestamp + return tlog.timestamp(treeID, merkle, leaves) +} + // newBackendClient returns a new backendClient. -func newBackendClient(tlog *tlogBackend) *backendClient { +func newBackendClient(b *tlogBackend) *backendClient { return &backendClient{ - backend: tlog, + backend: b, } } diff --git a/politeiad/plugins/comments/comments.go b/politeiad/plugins/comments/comments.go index 70728978d..6d202be3f 100644 --- a/politeiad/plugins/comments/comments.go +++ b/politeiad/plugins/comments/comments.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -// Package comments provides a plugin for adding comment functionality to +// Package comments provides a plugin for adding comments and comment votes to // records. package comments @@ -17,8 +17,7 @@ type ErrorStatusT int // TODO add a hint to comments that can be used freely by the client. This // is how we'll distinguish proposal comments from update comments. const ( - ID = "comments" - Version = "1" + ID = "comments" // Plugin commands CmdNew = "new" // Create a new comment @@ -30,7 +29,7 @@ const ( CmdGetVersion = "getversion" // Get specified version of a comment CmdCount = "count" // Get comments count for a record CmdVotes = "votes" // Get comment votes - CmdProofs = "proofs" // Get inclusion proofs + CmdTimestamps = "timestamps" // Get timestamps // Record states StateInvalid StateT = 0 @@ -570,3 +569,49 @@ func DecodeVotesReply(payload []byte) (*VotesReply, error) { } return &vr, nil } + +// Proof contains an inclusion proof for the digest in the merkle root. The +// ExtraData field is used by certain types of proofs to include additional +// data that is required to validate the proof. +type Proof struct { + Type string `json:"type"` + Digest string `json:"digest"` + MerkleRoot string `json:"merkleroot"` + MerklePath []string `json:"merklepath"` + ExtraData string `json:"extradata"` // JSON encoded +} + +// Timestamp contains all of the data required to verify that a piece of +// data was timestamped onto the decred blockchain. +// +// All digests are hex encoded SHA256 digests. The merkle root can be found in +// the OP_RETURN of the specified DCR transaction. +// +// TxID, MerkleRoot, and Proofs will only be populated once the merkle root has +// been included in a DCR tx and the tx has 6 confirmations. The Data field +// will not be populated if the data has been censored. +type Timestamp struct { + Data string `json:"data"` // JSON encoded + Digest string `json:"digest"` + TxID string `json:"txid"` + MerkleRoot string `json:"merkleroot"` + Proofs []Proof `json:"proofs"` +} + +// Timestamps requests the timestamps for a record's comments. If no comment +// IDs are provided then timestamps for all comments made on the record will +// be returned. If IncludeVotes is set to true then the timestamps for the +// comment votes will also be returned. If a provided comment ID does not +// exist then it will not be included in the reply. +type Timestamps struct { + State StateT `json:"state"` + Token string `json:"token"` + CommentIDs []uint32 `json:"commentids"` + IncludeVotes bool `json:"includevotes"` +} + +// TimestampsReply is the reply to the timestamps command. +type TimestampsReply struct { + Comments map[uint32][]Timestamp `json:"comments"` + Votes map[uint32][]Timestamp `json:"votes"` +} diff --git a/politeiad/plugins/dcrdata/dcrdata.go b/politeiad/plugins/dcrdata/dcrdata.go index da8cb5859..cfcc57a68 100644 --- a/politeiad/plugins/dcrdata/dcrdata.go +++ b/politeiad/plugins/dcrdata/dcrdata.go @@ -13,8 +13,7 @@ import ( type StatusT int const ( - ID = "dcrdata" - Version = "1" + ID = "dcrdata" // Plugin commands CmdBestBlock = "bestblock" // Get best block diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index f956dd6fb..1c5c31837 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -21,8 +21,7 @@ type ErrorStatusT int type CommentVoteT int const ( - ID = "pi" - Version = "1" + ID = "pi" // Plugin commands CmdPassThrough = "passthrough" // Plugin command pass through diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index ef4b9aa6d..6740b1266 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -20,8 +20,7 @@ type ErrorStatusT int // The receipt should be the server signature of Signature+StartBlockHash. const ( - ID = "ticketvote" - Version = "1" + ID = "ticketvote" // Plugin commands CmdAuthorize = "authorize" // Authorize a vote diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index f0f48c970..3a5a0f9b2 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -1447,26 +1447,22 @@ func _main() error { case comments.ID: plugin = backend.Plugin{ ID: comments.ID, - Version: comments.Version, Settings: ps, Identity: p.identity, } case dcrdata.ID: plugin = backend.Plugin{ ID: dcrdata.ID, - Version: dcrdata.Version, Settings: ps, } case pi.ID: plugin = backend.Plugin{ ID: pi.ID, - Version: pi.Version, Settings: ps, } case ticketvote.ID: plugin = backend.Plugin{ ID: ticketvote.ID, - Version: ticketvote.Version, Settings: ps, Identity: p.identity, } diff --git a/politeiawww/api/comments/v1/v1.go b/politeiawww/api/comments/v1/v1.go new file mode 100644 index 000000000..2e09d528d --- /dev/null +++ b/politeiawww/api/comments/v1/v1.go @@ -0,0 +1,127 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package v1 + +import "fmt" + +const ( + // APIRoute is prefixed onto all routes defined in this package. + APIRoute = "/comments/v1" + + RouteTimestamps = "/timestamps" +) + +// ErrorCodeT represents a user error code. +type ErrorCodeT int + +const ( + // Error codes + ErrorCodeInvalid ErrorCodeT = 0 + ErrorCodeInputInvalid ErrorCodeT = 1 +) + +var ( + // ErrorCodes contains the human readable errors. + ErrorCodes = map[ErrorCodeT]string{ + ErrorCodeInvalid: "error invalid", + ErrorCodeInputInvalid: "input invalid", + } +) + +// UserErrorReply is the reply that the server returns when it encounters an +// error that is caused by something that the user did (malformed input, bad +// timing, etc). The HTTP status code will be 400. +type UserErrorReply struct { + ErrorCode ErrorCodeT `json:"errorcode"` + ErrorContext string `json:"errorcontext"` +} + +// Error satisfies the error interface. +func (e UserErrorReply) Error() string { + return fmt.Sprintf("user error code: %v", e.ErrorCode) +} + +// PluginErrorReply is the reply that the server returns when it encounters +// a plugin error. +type PluginErrorReply struct { + PluginID string `json:"pluginid"` + ErrorCode int `json:"errorcode"` + ErrorContext string `json:"errorcontext"` +} + +// Error satisfies the error interface. +func (e PluginErrorReply) Error() string { + return fmt.Sprintf("user error code: %v", e.ErrorCode) +} + +// ServerErrorReply is the reply that the server returns when it encounters an +// unrecoverable error while executing a command. The HTTP status code will be +// 500 and the ErrorCode field will contain a UNIX timestamp that the user can +// provide to the server admin to track down the error details in the logs. +type ServerErrorReply struct { + ErrorCode int64 `json:"errorcode"` +} + +// Error satisfies the error interface. +func (e ServerErrorReply) Error() string { + return fmt.Sprintf("server error: %v", e.ErrorCode) +} + +// RecordStateT represents a record state. +type RecordStateT int + +const ( + // RecordStateInvalid indicates an invalid record state. + RecordStateInvalid RecordStateT = 0 + + // RecordStateUnvetted indicates a record has not been made public + // yet. + RecordStateUnvetted RecordStateT = 1 + + // RecordStateVetted indicates a record has been made public. + RecordStateVetted RecordStateT = 2 +) + +// Proof contains an inclusion proof for the digest in the merkle root. The +// ExtraData field is used by certain types of proofs to include additional +// data that is required to validate the proof. +type Proof struct { + Type string `json:"type"` + Digest string `json:"digest"` + MerkleRoot string `json:"merkleroot"` + MerklePath []string `json:"merklepath"` + ExtraData string `json:"extradata"` // JSON encoded +} + +// Timestamp contains all of the data required to verify that a piece of +// data was timestamped onto the decred blockchain. +// +// All digests are hex encoded SHA256 digests. The merkle root can be found in +// the OP_RETURN of the specified DCR transaction. +// +// TxID, MerkleRoot, and Proofs will only be populated once the merkle root has +// been included in a DCR tx and the tx has 6 confirmations. The Data field +// will not be populated if the data has been censored. +type Timestamp struct { + Data string `json:"data"` // JSON encoded + Digest string `json:"digest"` + TxID string `json:"txid"` + MerkleRoot string `json:"merkleroot"` + Proofs []Proof `json:"proofs"` +} + +// Timestamps requests the timestamps for the comments of a record. If no +// comment IDs are provided then the timestamps for all comments will be +// returned. +type Timestamps struct { + State RecordStateT `json:"state"` + Token string `json:"token"` + CommentIDs []uint32 `json:"commentids,omitempty"` +} + +// TimestampsReply is the reply to the Timestamps command. +type TimestampsReply struct { + Comments map[uint32][]Timestamp `json:"comments"` // [commentID]Timestamp +} diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go index 229d4095e..fd63e9f19 100644 --- a/politeiawww/api/records/v1/v1.go +++ b/politeiawww/api/records/v1/v1.go @@ -19,16 +19,19 @@ type ErrorCodeT int const ( // Error codes - ErrorCodeInvalid ErrorCodeT = 0 - ErrorCodeInputInvalid ErrorCodeT = 1 - ErrorCodeRecordNotFound ErrorCodeT = 2 - ErrorCodeStateInvalid ErrorCodeT = 3 + ErrorCodeInvalid ErrorCodeT = 0 + ErrorCodeInputInvalid ErrorCodeT = 1 + ErrorCodeRecordNotFound ErrorCodeT = 2 + ErrorCodeRecordStateInvalid ErrorCodeT = 3 ) var ( - // TODO Add human readable error messages + // ErrorCodes contains the human readable errors. ErrorCodes = map[ErrorCodeT]string{ - ErrorCodeInvalid: "error invalid", + ErrorCodeInvalid: "error invalid", + ErrorCodeInputInvalid: "input invalid", + ErrorCodeRecordNotFound: "record not found", + ErrorCodeRecordStateInvalid: "record state invalid", } ) @@ -45,6 +48,19 @@ func (e UserErrorReply) Error() string { return fmt.Sprintf("user error code: %v", e.ErrorCode) } +// PluginErrorReply is the reply that the server returns when it encounters +// a plugin error. +type PluginErrorReply struct { + PluginID string `json:"pluginid"` + ErrorCode int `json:"errorcode"` + ErrorContext string `json:"errorcontext"` +} + +// Error satisfies the error interface. +func (e PluginErrorReply) Error() string { + return fmt.Sprintf("user error code: %v", e.ErrorCode) +} + // ServerErrorReply is the reply that the server returns when it encounters an // unrecoverable error while executing a command. The HTTP status code will be // 500 and the ErrorCode field will contain a UNIX timestamp that the user can @@ -72,7 +88,9 @@ const ( StateVetted StateT = 2 ) -// Proof contains an inclusion proof for the digest in the merkle root. +// Proof contains an inclusion proof for the digest in the merkle root. The +// ExtraData field is used by certain types of proofs to include additional +// data that is required to validate the proof. type Proof struct { Type string `json:"type"` Digest string `json:"digest"` diff --git a/politeiawww/cmd/piwww/commenttimestamps.go b/politeiawww/cmd/piwww/commenttimestamps.go new file mode 100644 index 000000000..32e7eba4a --- /dev/null +++ b/politeiawww/cmd/piwww/commenttimestamps.go @@ -0,0 +1,126 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe" + cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" + "github.com/decred/politeia/politeiawww/cmd/shared" +) + +// commentTimestampsCmd retrieves the timestamps for politeiawww comments. +type commentTimestampsCmd struct { + Args struct { + Token string `positional-arg-name:"token" required:"true"` + CommentIDs []uint32 `positional-arg-name:"commentids" optional:"true"` + } `positional-args:"true"` + + // Unvetted is used to request the comment timestamps of an + // unvetted record. + Unvetted bool `long:"unvetted" optional:"true"` +} + +// Execute executes the commentTimestampsCmd command. +// +// This function satisfies the go-flags Commander interface. +func (c *commentTimestampsCmd) Execute(args []string) error { + + // Set comment state. Defaults to vetted unless the unvetted flag + // is used. + var state cmv1.RecordStateT + switch { + case c.Unvetted: + state = cmv1.RecordStateUnvetted + default: + state = cmv1.RecordStateVetted + } + + // Setup request + t := cmv1.Timestamps{ + State: state, + Token: c.Args.Token, + CommentIDs: c.Args.CommentIDs, + } + + // Send request + err := shared.PrintJSON(t) + if err != nil { + return err + } + tr, err := client.CommentTimestamps(t) + if err != nil { + return err + } + err = shared.PrintJSON(tr) + if err != nil { + return err + } + + // Verify timestamps + for commentID, timestamps := range tr.Comments { + for _, v := range timestamps { + err = verifyCommentTimestamp(v) + if err != nil { + return fmt.Errorf("verify comment timestamp %v: %v", + commentID, err) + } + } + } + + return nil +} + +func verifyCommentTimestamp(t cmv1.Timestamp) error { + ts := convertCommentTimestamp(t) + return tlogbe.VerifyTimestamp(ts) +} + +func convertCommentProof(p cmv1.Proof) backend.Proof { + return backend.Proof{ + Type: p.Type, + Digest: p.Digest, + MerkleRoot: p.MerkleRoot, + MerklePath: p.MerklePath, + ExtraData: p.ExtraData, + } +} + +func convertCommentTimestamp(t cmv1.Timestamp) backend.Timestamp { + proofs := make([]backend.Proof, 0, len(t.Proofs)) + for _, v := range t.Proofs { + proofs = append(proofs, convertCommentProof(v)) + } + return backend.Timestamp{ + Data: t.Data, + Digest: t.Digest, + TxID: t.TxID, + MerkleRoot: t.MerkleRoot, + Proofs: proofs, + } +} + +const commentTimestampsHelpMsg = `commenttimestamps [flags] "token" commentIDs + +Fetch the timestamps for a record's comments. The timestamp contains all +necessary data to verify that user submitted comment data has been timestamped +onto the decred blockchain. + +Arguments: +1. token (string, required) Proposal token +2. commentIDs ([]uint32, optional) Proposal version + +Flags: + --unvetted (bool, optional) Request is for an unvetted record instead of + vetted ones. + +Example: Fetch all record comment timestamps +$ piwww commenttimestamps 0a265dd93e9bae6d + +Example: Fetch comment timestamps for comment IDs 1, 6, and 7 +$ piwww commenttimestamps 0a265dd93e9bae6d 1 6 7 +` diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index b7d5030b7..144104b15 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -76,11 +76,12 @@ type piwww struct { RecordTimestamps recordTimestampsCmd `command:"recordtimestamps"` // Comments commands - CommentNew commentNewCmd `command:"commentnew"` - CommentVote commentVoteCmd `command:"commentvote"` - CommentCensor commentCensorCmd `command:"commentcensor"` - Comments commentsCmd `command:"comments"` - CommentVotes commentVotesCmd `command:"commentvotes"` + CommentNew commentNewCmd `command:"commentnew"` + CommentVote commentVoteCmd `command:"commentvote"` + CommentCensor commentCensorCmd `command:"commentcensor"` + Comments commentsCmd `command:"comments"` + CommentVotes commentVotesCmd `command:"commentvotes"` + CommentTimestamps commentTimestampsCmd `command:"commenttimestamps"` // Vote commands VoteAuthorize voteAuthorizeCmd `command:"voteauthorize"` diff --git a/politeiawww/cmd/piwww/recordtimestamps.go b/politeiawww/cmd/piwww/recordtimestamps.go index fb8d73be7..7d0a35502 100644 --- a/politeiawww/cmd/piwww/recordtimestamps.go +++ b/politeiawww/cmd/piwww/recordtimestamps.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -119,10 +119,10 @@ data to verify that user submitted record data has been timestamped onto the decred blockchain. Arguments: -1. token (string, required) Proposal token -2. version (string, optional) Proposal version +1. token (string, required) Record token +2. version (string, optional) Record version Flags: - --unvetted (bool, optional) Request is for unvetted records instead of - vetted ones (default: false). + --unvetted (bool, optional) Request is for unvetted records instead of vetted + ones (default: false). ` diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index 1079f45c7..a2502abf8 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -21,6 +21,7 @@ import ( "decred.org/dcrwallet/rpc/walletrpc" cms "github.com/decred/politeia/politeiawww/api/cms/v1" + cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" pi "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" @@ -127,11 +128,16 @@ func wwwError(body []byte, statusCode int) error { return nil } -// TODO recordsError +// TODO implement recordsError func recordsError(body []byte, statusCode int) error { return fmt.Errorf("%v %s", statusCode, body) } +// TODO implement commentsError +func commentsError(body []byte, statusCode int) error { + return fmt.Errorf("%v %s", statusCode, body) +} + // piError unmarshals the response body from makeRequest, and handles any // status code errors from the server. Parses the error code and error context // from the pi api, in case of user error. @@ -929,7 +935,7 @@ func (c *Client) ProposalSetStatus(pss pi.ProposalSetStatus) (*pi.ProposalSetSta return &pssr, nil } -// RecordTimestamps sends the RecordTimestamps command to politeiawww. +// RecordTimestamps sends the Timestamps command to politeiawww records API. func (c *Client) RecordTimestamps(t rcv1.Timestamps) (*rcv1.TimestampsReply, error) { statusCode, respBody, err := c.makeRequest(http.MethodPost, rcv1.APIRoute, rcv1.RouteTimestamps, t) @@ -1576,6 +1582,33 @@ func (c *Client) CommentVotes(cv pi.CommentVotes) (*pi.CommentVotesReply, error) return &cvr, nil } +// CommentTimestamps sends the Timestamps command to politeiawww comments API. +func (c *Client) CommentTimestamps(t cmv1.Timestamps) (*cmv1.TimestampsReply, error) { + statusCode, respBody, err := c.makeRequest(http.MethodPost, + cmv1.APIRoute, cmv1.RouteTimestamps, t) + if err != nil { + return nil, err + } + if statusCode != http.StatusOK { + return nil, commentsError(respBody, statusCode) + } + + var tr cmv1.TimestampsReply + err = json.Unmarshal(respBody, &tr) + if err != nil { + return nil, err + } + + if c.cfg.Verbose { + err := prettyPrintJSON(tr) + if err != nil { + return nil, err + } + } + + return &tr, nil +} + // InvoiceComments retrieves the comments for the specified proposal. func (c *Client) InvoiceComments(token string) (*www.GetCommentsReply, error) { route := "/invoices/" + token + "/comments" diff --git a/politeiawww/comments.go b/politeiawww/comments.go index 50298caa2..ea93d1374 100644 --- a/politeiawww/comments.go +++ b/politeiawww/comments.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2020 The Decred developers +// Copyright (c) 2017-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -6,10 +6,53 @@ package main import ( "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "runtime/debug" + "strings" + "time" "github.com/decred/politeia/politeiad/plugins/comments" + cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" + "github.com/decred/politeia/util" ) +func convertCommentState(s cmv1.RecordStateT) comments.StateT { + switch s { + case cmv1.RecordStateUnvetted: + return comments.StateUnvetted + case cmv1.RecordStateVetted: + return comments.StateVetted + } + return comments.StateInvalid +} + +func convertProofFromCommentsPlugin(p comments.Proof) cmv1.Proof { + return cmv1.Proof{ + Type: p.Type, + Digest: p.Digest, + MerkleRoot: p.MerkleRoot, + MerklePath: p.MerklePath, + ExtraData: p.ExtraData, + } +} + +func convertTimestampFromCommentsPlugin(t comments.Timestamp) cmv1.Timestamp { + proofs := make([]cmv1.Proof, 0, len(t.Proofs)) + for _, v := range t.Proofs { + proofs = append(proofs, convertProofFromCommentsPlugin(v)) + } + return cmv1.Timestamp{ + Data: t.Data, + Digest: t.Digest, + TxID: t.TxID, + MerkleRoot: t.MerkleRoot, + Proofs: proofs, + } +} + // commentsAll returns all comments for the provided record. func (p *politeiawww) commentsAll(ctx context.Context, cp comments.GetAll) (*comments.GetAllReply, error) { b, err := comments.EncodeGetAll(cp) @@ -60,3 +103,163 @@ func (p *politeiawww) commentVotes(ctx context.Context, vs comments.Votes) (*com } return vsr, nil } + +func (p *politeiawww) commentTimestamps(ctx context.Context, t comments.Timestamps) (*comments.TimestampsReply, error) { + b, err := json.Marshal(t) + if err != nil { + return nil, err + } + r, err := p.pluginCommand(ctx, comments.ID, + comments.CmdTimestamps, string(b)) + if err != nil { + return nil, err + } + var tr comments.TimestampsReply + err = json.Unmarshal([]byte(r), &tr) + if err != nil { + return nil, err + } + return &tr, nil +} + +func (p *politeiawww) processCommentTimestamps(ctx context.Context, t cmv1.Timestamps, isAdmin bool) (*cmv1.TimestampsReply, error) { + log.Tracef("processCommentTimestamps: %v %v %v", + t.State, t.Token, t.CommentIDs) + + // Get timestamps + ct := comments.Timestamps{ + State: convertCommentState(t.State), + Token: t.Token, + CommentIDs: t.CommentIDs, + } + ctr, err := p.commentTimestamps(ctx, ct) + if err != nil { + return nil, err + } + + // Prepare reply + comments := make(map[uint32][]cmv1.Timestamp, len(ctr.Comments)) + for commentID, timestamps := range ctr.Comments { + ts := make([]cmv1.Timestamp, 0, len(timestamps)) + for _, v := range timestamps { + // Strip unvetted data blobs if the user is not an admin + if t.State == cmv1.RecordStateUnvetted && !isAdmin { + v.Data = "" + } + ts = append(ts, convertTimestampFromCommentsPlugin(v)) + } + comments[commentID] = ts + } + + return &cmv1.TimestampsReply{ + Comments: comments, + }, nil +} + +func (p *politeiawww) handleCommentTimestamps(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleCommentTimestamps") + + var t cmv1.Timestamps + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&t); err != nil { + respondWithCommentsError(w, r, "handleTimestamps: unmarshal", + cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodeInputInvalid, + }) + return + } + + // Lookup session user. This is a public route so a session may not + // exist. Ignore any session not found errors. + usr, err := p.getSessionUser(w, r) + if err != nil && err != errSessionNotFound { + respondWithCommentsError(w, r, + "handleTimestamps: getSessionUser: %v", err) + return + } + + isAdmin := usr != nil && usr.Admin + tr, err := p.processCommentTimestamps(r.Context(), t, isAdmin) + if err != nil { + respondWithCommentsError(w, r, + "handleTimestamps: processTimestamps: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, tr) +} + +func respondWithCommentsError(w http.ResponseWriter, r *http.Request, format string, err error) { + var ( + ue cmv1.UserErrorReply + pe pdError + ) + switch { + case errors.As(err, &ue): + // Comments user error + m := fmt.Sprintf("Comments user error: %v %v %v", + remoteAddr(r), ue.ErrorCode, cmv1.ErrorCodes[ue.ErrorCode]) + if ue.ErrorContext != "" { + m += fmt.Sprintf(": %v", ue.ErrorContext) + } + log.Infof(m) + util.RespondWithJSON(w, http.StatusBadRequest, + cmv1.UserErrorReply{ + ErrorCode: ue.ErrorCode, + ErrorContext: ue.ErrorContext, + }) + return + + case errors.As(err, &pe): + // Politeiad error + var ( + pluginID = pe.ErrorReply.Plugin + errCode = pe.ErrorReply.ErrorCode + errContext = pe.ErrorReply.ErrorContext + ) + switch { + case pluginID != "": + // Politeiad plugin error. Log it and return a 400. + m := fmt.Sprintf("Plugin error: %v %v %v", + remoteAddr(r), pluginID, errCode) + if len(errContext) > 0 { + m += fmt.Sprintf(": %v", strings.Join(errContext, ", ")) + } + log.Infof(m) + util.RespondWithJSON(w, http.StatusBadRequest, + cmv1.PluginErrorReply{ + PluginID: pluginID, + ErrorCode: errCode, + ErrorContext: strings.Join(errContext, ", "), + }) + return + + default: + // Unknown politeiad error. Log it and return a 500. + ts := time.Now().Unix() + log.Errorf("%v %v %v %v Internal error %v: error code "+ + "from politeiad: %v", remoteAddr(r), r.Method, r.URL, + r.Proto, ts, errCode) + + util.RespondWithJSON(w, http.StatusInternalServerError, + cmv1.ServerErrorReply{ + ErrorCode: ts, + }) + return + } + + default: + // Internal server error. Log it and return a 500. + t := time.Now().Unix() + e := fmt.Sprintf(format, err) + log.Errorf("%v %v %v %v Internal error %v: %v", + remoteAddr(r), r.Method, r.URL, r.Proto, t, e) + log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) + + util.RespondWithJSON(w, http.StatusInternalServerError, + cmv1.ServerErrorReply{ + ErrorCode: t, + }) + return + } +} diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index b68f40cbb..0439279cc 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -22,6 +22,7 @@ import ( "github.com/decred/politeia/politeiad/plugins/comments" "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/ticketvote" + cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" @@ -2635,6 +2636,9 @@ func (p *politeiawww) setPiRoutes() { p.addRoute(http.MethodPost, piv1.APIRoute, piv1.RouteCommentVotes, p.handleCommentVotes, permissionPublic) + p.addRoute(http.MethodPost, cmv1.APIRoute, + cmv1.RouteTimestamps, p.handleCommentTimestamps, + permissionPublic) // Vote routes p.addRoute(http.MethodPost, piv1.APIRoute, diff --git a/politeiawww/records.go b/politeiawww/records.go index 2b41e26e9..34f22dc8a 100644 --- a/politeiawww/records.go +++ b/politeiawww/records.go @@ -20,7 +20,7 @@ import ( ) func (p *politeiawww) processTimestamps(ctx context.Context, t rcv1.Timestamps, isAdmin bool) (*rcv1.TimestampsReply, error) { - log.Tracef("processTimestamps: %v %v", t.Token, t.Version) + log.Tracef("processTimestamps: %v %v %v", t.State, t.Token, t.Version) // Get record timestamps var ( @@ -40,7 +40,7 @@ func (p *politeiawww) processTimestamps(ctx context.Context, t rcv1.Timestamps, } default: return nil, rcv1.UserErrorReply{ - ErrorCode: rcv1.ErrorCodeStateInvalid, + ErrorCode: rcv1.ErrorCodeRecordStateInvalid, } } @@ -110,20 +110,16 @@ func (p *politeiawww) handleTimestamps(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, tr) } -func convertRecordsErrorCode(pluginID string, errCode int) rcv1.ErrorCodeT { - switch pluginID { - case "": - // politeiad v1 API errors - switch pdv1.ErrorStatusT(errCode) { - case pdv1.ErrorStatusInvalidRequestPayload: - // Intentionally omitted. This indicates an internal server error. - case pdv1.ErrorStatusInvalidChallenge: - // Intentionally omitted. This indicates an internal server error. - case pdv1.ErrorStatusRecordNotFound: - return rcv1.ErrorCodeRecordNotFound - } +func convertRecordsErrorCode(errCode int) rcv1.ErrorCodeT { + switch pdv1.ErrorStatusT(errCode) { + case pdv1.ErrorStatusInvalidRequestPayload: + // Intentionally omitted. This indicates an internal server error. + case pdv1.ErrorStatusInvalidChallenge: + // Intentionally omitted. This indicates an internal server error. + case pdv1.ErrorStatusRecordNotFound: + return rcv1.ErrorCodeRecordNotFound } - // No records error code found + // No record API error code found return rcv1.ErrorCodeInvalid } @@ -155,21 +151,31 @@ func respondWithRecordError(w http.ResponseWriter, r *http.Request, format strin errCode = pe.ErrorReply.ErrorCode errContext = pe.ErrorReply.ErrorContext ) - e := convertRecordsErrorCode(pluginID, errCode) - switch e { - case rcv1.ErrorCodeInvalid: + e := convertRecordsErrorCode(errCode) + switch { + case pluginID != "": + // politeiad plugin error. Log it and return a 400. + m := fmt.Sprintf("Plugin error: %v %v %v", + remoteAddr(r), pluginID, errCode) + if len(errContext) > 0 { + m += fmt.Sprintf(": %v", strings.Join(errContext, ", ")) + } + log.Infof(m) + util.RespondWithJSON(w, http.StatusBadRequest, + rcv1.PluginErrorReply{ + PluginID: pluginID, + ErrorCode: errCode, + ErrorContext: strings.Join(errContext, ", "), + }) + return + + case e == rcv1.ErrorCodeInvalid: // politeiad error does not correspond to a user error. Log it // and return a 500. ts := time.Now().Unix() - if pluginID == "" { - log.Errorf("%v %v %v %v Internal error %v: error code "+ - "from politeiad: %v", remoteAddr(r), r.Method, r.URL, - r.Proto, ts, errCode) - } else { - log.Errorf("%v %v %v %v Internal error %v: error code "+ - "from politeiad plugin %v: %v %v", remoteAddr(r), r.Method, - r.URL, r.Proto, ts, pluginID, errCode, errContext) - } + log.Errorf("%v %v %v %v Internal error %v: error code "+ + "from politeiad: %v", remoteAddr(r), r.Method, r.URL, + r.Proto, ts, errCode) util.RespondWithJSON(w, http.StatusInternalServerError, rcv1.ServerErrorReply{ @@ -192,7 +198,6 @@ func respondWithRecordError(w http.ResponseWriter, r *http.Request, format strin ErrorContext: strings.Join(errContext, ", "), }) return - } default: @@ -207,6 +212,7 @@ func respondWithRecordError(w http.ResponseWriter, r *http.Request, format strin rcv1.ServerErrorReply{ ErrorCode: t, }) + return } } From 50d4e08c86a41893a666dac0cf104717b7366793 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 12 Jan 2021 09:23:57 -0600 Subject: [PATCH 210/449] multi: Remove decredplugin voting code. --- decredplugin/decredplugin.go | 524 +--------- politeiad/backend/gitbe/cms.go | 8 + politeiad/backend/gitbe/decred.go | 1220 ---------------------- politeiad/backend/gitbe/gitbe.go | 34 - politeiawww/cmd/piwww/votestartrunoff.go | 6 +- 5 files changed, 21 insertions(+), 1771 deletions(-) diff --git a/decredplugin/decredplugin.go b/decredplugin/decredplugin.go index c9518e5a8..f19c60fa6 100644 --- a/decredplugin/decredplugin.go +++ b/decredplugin/decredplugin.go @@ -1,531 +1,27 @@ -// Copyright (c) 2017-2020 The Decred developers +// Copyright (c) 2017-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. package decredplugin import ( - "encoding/hex" "encoding/json" - "fmt" - - "github.com/decred/politeia/politeiad/api/v1/identity" - "github.com/decred/politeia/util" ) -type ErrorStatusT int -type VoteT int - // Plugin settings, kinda doesn;t go here but for now it is fine const ( - Version = "1" - ID = "decred" - CmdAuthorizeVote = "authorizevote" - CmdStartVote = "startvote" - CmdStartVoteRunoff = "startvoterunoff" - CmdBallot = "ballot" - CmdBestBlock = "bestblock" - CmdNewComment = "newcomment" - CmdCensorComment = "censorcomment" - CmdGetComments = "getcomments" - MDStreamAuthorizeVote = 13 // Vote authorization by proposal author - MDStreamVoteBits = 14 // Vote bits and mask - MDStreamVoteSnapshot = 15 // Vote tickets and start/end parameters - - // Vote duration requirements for proposal votes (in blocks) - VoteDurationMinMainnet = 2016 - VoteDurationMaxMainnet = 4032 - VoteDurationMinTestnet = 0 - VoteDurationMaxTestnet = 4032 - - // Authorize vote actions - AuthVoteActionAuthorize = "authorize" // Authorize a proposal vote - AuthVoteActionRevoke = "revoke" // Revoke a proposal vote authorization - - // Vote option IDs - VoteOptionIDApprove = "yes" - VoteOptionIDReject = "no" - - // Vote types - // - // VoteTypeStandard is used to indicate a simple approve or reject - // proposal vote where the winner is the voting option that has met - // the specified pass and quorum requirements. - // - // VoteTypeRunoff specifies a runoff vote that multiple proposals compete in. - // All proposals are voted on like normal, but there can only be one winner - // in a runoff vote. The winner is the proposal that meets the quorum - // requirement, meets the pass requirement, and that has the most net yes - // votes. The winning proposal is considered approved and all other proposals - // are considered rejected. If no proposals meet the quorum and pass - // requirements then all proposals are considered rejected. - // Note: in a runoff vote it is possible for a proposal to meet the quorum - // and pass requirements but still be rejected if it does not have the most - // net yes votes. - VoteTypeInvalid VoteT = 0 - VoteTypeStandard VoteT = 1 - VoteTypeRunoff VoteT = 2 - - // Versioning - VersionStartVoteV1 = 1 - VersionStartVoteV2 = 2 - - // Error status codes - ErrorStatusInvalid ErrorStatusT = 0 - ErrorStatusInternalError ErrorStatusT = 1 - ErrorStatusProposalNotFound ErrorStatusT = 2 - ErrorStatusInvalidVoteBit ErrorStatusT = 3 - ErrorStatusVoteHasEnded ErrorStatusT = 4 - ErrorStatusDuplicateVote ErrorStatusT = 5 - ErrorStatusIneligibleTicket ErrorStatusT = 6 -) - -var ( - // ErrorStatus converts error status codes to human readable text. - ErrorStatus = map[ErrorStatusT]string{ - ErrorStatusInvalid: "invalid error status", - ErrorStatusInternalError: "internal error", - ErrorStatusProposalNotFound: "proposal not found", - ErrorStatusInvalidVoteBit: "invalid vote bit", - ErrorStatusVoteHasEnded: "vote has ended", - ErrorStatusDuplicateVote: "duplicate vote", - ErrorStatusIneligibleTicket: "ineligbile ticket", - } + Version = "1" + ID = "decred" + CmdBestBlock = "bestblock" + CmdNewComment = "newcomment" + CmdCensorComment = "censorcomment" + CmdGetComments = "getcomments" ) -// CastVote is a signed vote. -type CastVote struct { - Token string `json:"token"` // Proposal ID - Ticket string `json:"ticket"` // Ticket ID - VoteBit string `json:"votebit"` // Vote bit that was selected, this is encode in hex - Signature string `json:"signature"` // Signature of Token+Ticket+VoteBit -} - -// Ballot is a batch of votes that are sent to the server. -type Ballot struct { - Votes []CastVote `json:"votes"` -} - -// EncodeCastVotes encodes CastVotes into a JSON byte slice. -func EncodeBallot(b Ballot) ([]byte, error) { - return json.Marshal(b) -} - -// DecodeCastVotes decodes a JSON byte slice into a CastVotes. -func DecodeBallot(payload []byte) (*Ballot, error) { - var b Ballot - - err := json.Unmarshal(payload, &b) - if err != nil { - return nil, err - } - - return &b, nil -} - -// CastVoteReply contains the signature or error to a cast vote command. The -// Error and ErrorStatus fields will only be populated if something went wrong -// while attempting to cast the vote. -type CastVoteReply struct { - ClientSignature string `json:"clientsignature"` // Signature that was sent in - Signature string `json:"signature"` // Signature of the ClientSignature - Error string `json:"error"` // Error status message - ErrorStatus ErrorStatusT `json:"errorstatus,omitempty"` // Error status code -} - -// EncodeCastVoteReply encodes CastVoteReply into a JSON byte slice. -func EncodeCastVoteReply(cvr CastVoteReply) ([]byte, error) { - return json.Marshal(cvr) -} - -// DecodeBallotReply decodes a JSON byte slice into a CastVotes. -func DecodeCastVoteReply(payload []byte) (*CastVoteReply, error) { - var cvr CastVoteReply - - err := json.Unmarshal(payload, &cvr) - if err != nil { - return nil, err - } - - return &cvr, nil -} - -// BallotReply is a reply to a batched list of votes. -type BallotReply struct { - Receipts []CastVoteReply `json:"receipts"` -} - -// EncodeCastVoteReplies encodes CastVotes into a JSON byte slice. -func EncodeBallotReply(br BallotReply) ([]byte, error) { - return json.Marshal(br) -} - -// DecodeBallotReply decodes a JSON byte slice into a CastVotes. -func DecodeBallotReply(payload []byte) (*BallotReply, error) { - var br BallotReply - - err := json.Unmarshal(payload, &br) - if err != nil { - return nil, err - } - - return &br, nil -} - -// AuthorizeVote is an MDStream that is used to indicate that a proposal has -// been finalized and is ready to be voted on. The signature and public -// key are from the proposal author. The author can revoke a previously sent -// vote authorization by setting the Action field to revoke. -const VersionAuthorizeVote = 1 - -type AuthorizeVote struct { - // Generated by decredplugin - Version uint `json:"version"` // Version of this structure - Receipt string `json:"receipt"` // Server signature of client signature - Timestamp int64 `json:"timestamp"` // Received UNIX timestamp - - // Generated by client - Action string `json:"action"` // Authorize or revoke - Token string `json:"token"` // Proposal censorship token - Signature string `json:"signature"` // Signature of token+version+action - PublicKey string `json:"publickey"` // Pubkey used for signature -} - -// EncodeAuthorizeVote encodes AuthorizeVote into a JSON byte slice. -func EncodeAuthorizeVote(av AuthorizeVote) ([]byte, error) { - return json.Marshal(av) -} - -// DecodeAuthorizeVote decodes a JSON byte slice into an AuthorizeVote. -func DecodeAuthorizeVote(payload []byte) (*AuthorizeVote, error) { - var av AuthorizeVote - err := json.Unmarshal(payload, &av) - if err != nil { - return nil, err - } - return &av, nil -} - -// AuthorizeVoteReply returns the authorize vote action that was executed and -// the receipt for the action. The receipt is the server side signature of -// AuthorizeVote.Signature. -type AuthorizeVoteReply struct { - Action string `json:"action"` // Authorize or revoke - RecordVersion string `json:"recordversion"` // Version of record files - Receipt string `json:"receipt"` // Server signature of client signature - Timestamp int64 `json:"timestamp"` // Received UNIX timestamp -} - -// EncodeAuthorizeVote encodes AuthorizeVoteReply into a JSON byte slice. -func EncodeAuthorizeVoteReply(avr AuthorizeVoteReply) ([]byte, error) { - return json.Marshal(avr) -} - -// DecodeAuthorizeVoteReply decodes a JSON byte slice into a AuthorizeVoteReply. -func DecodeAuthorizeVoteReply(payload []byte) (*AuthorizeVoteReply, error) { - var avr AuthorizeVoteReply - err := json.Unmarshal(payload, &avr) - if err != nil { - return nil, err - } - return &avr, nil -} - -// StartVote instructs the plugin to commence voting on a proposal with the -// provided vote bits. -const VersionStartVote = 2 - -// StartVote contains a JSON encoded StartVote of the specified Version. This -// struct is never written to disk. It is used to pass around the various -// StartVote versions. -type StartVote struct { - Version uint `json:"version"` // Payload StartVote version - Token string `json:"token"` // Proposal token - Payload string `json:"payload"` // JSON encoded StartVote -} - -// EncodeStartVote encodes a StartVote into a JSON byte slice. -func EncodeStartVote(sv StartVote) ([]byte, error) { - return json.Marshal(sv) -} - -// DecodeStartVote decodes a JSON byte slice into a StartVote. -func DecodeStartVote(b []byte) (*StartVote, error) { - sv := make(map[string]interface{}, 4) - - err := json.Unmarshal(b, &sv) - if err != nil { - return nil, err - } - - // Handle nested JSON - vote := sv["vote"].(map[string]interface{}) - - return &StartVote{ - Token: vote["token"].(string), - Version: uint(sv["version"].(float64)), - Payload: string(b), - }, nil -} - -// VoteOption describes a single vote option. -type VoteOption struct { - Id string `json:"id"` // Single unique word identifying vote (e.g. yes) - Description string `json:"description"` // Longer description of the vote - Bits uint64 `json:"bits"` // Bits used for this option -} - -// VoteV1 represents the vote options and parameters for a StartVoteV1. -type VoteV1 struct { - Token string `json:"token"` // Token that identifies vote - Mask uint64 `json:"mask"` // Valid votebits - Duration uint32 `json:"duration"` // Duration in blocks - QuorumPercentage uint32 `json:"quorumpercentage"` // Percent of eligible votes required for quorum - PassPercentage uint32 `json:"passpercentage"` // Percent of total votes required to pass - Options []VoteOption `json:"options"` // Vote option -} - -// EncodeVoteV1 encodes VoteV1 into a JSON byte slice. -func EncodeVoteV1(v VoteV1) ([]byte, error) { - return json.Marshal(v) -} - -// DecodeVoteV1 decodes a JSON byte slice into a VoteV1. -func DecodeVoteV1(payload []byte) (*VoteV1, error) { - var v VoteV1 - - err := json.Unmarshal(payload, &v) - if err != nil { - return nil, err - } - - return &v, nil -} - -// StartVoteV1 was formerly used to start a proposal vote, but is not longer -// accepted. A StartVoteV2 must be used to start a proposal vote. -type StartVoteV1 struct { - // decred plugin only data - Version uint `json:"version"` // Version of this structure - - PublicKey string `json:"publickey"` // Key used for signature - Vote VoteV1 `json:"vote"` // Vote + options - Signature string `json:"signature"` // Signature of token -} - -// VerifySignature verifies that the StartVoteV1 signature is correct. -func (s *StartVoteV1) VerifySignature() error { - sig, err := util.ConvertSignature(s.Signature) - if err != nil { - return err - } - b, err := hex.DecodeString(s.PublicKey) - if err != nil { - return err - } - pk, err := identity.PublicIdentityFromBytes(b) - if err != nil { - return err - } - if !pk.VerifyMessage([]byte(s.Vote.Token), sig) { - return fmt.Errorf("invalid signature") - } - return nil -} - -// EncodeStartVoteV1 encodes a StartVoteV1 into a JSON byte slice. -func EncodeStartVoteV1(v StartVoteV1) ([]byte, error) { - return json.Marshal(v) -} - -// DecodeVotedecodes a JSON byte slice into a StartVoteV1. -func DecodeStartVoteV1(payload []byte) (*StartVoteV1, error) { - var sv StartVoteV1 - - err := json.Unmarshal(payload, &sv) - if err != nil { - return nil, err - } - - return &sv, nil -} - -// VoteV2 represents the vote options and vote parameters for a StartVoteV2. -// -// Differences between VoteV1 and VoteV2: -// * Added the ProposalVersion field that specifies the version of the proposal -// that is being voted on. This was added so that the proposal version is -// explicitly included in the StartVote signature. -// * Added a Type field in order to specify the vote type. -type VoteV2 struct { - Token string `json:"token"` // Token that identifies vote - ProposalVersion uint32 `json:"proposalversion"` // Proposal version being voted on - Type VoteT `json:"type"` // Type of vote - Mask uint64 `json:"mask"` // Valid votebits - Duration uint32 `json:"duration"` // Duration in blocks - QuorumPercentage uint32 `json:"quorumpercentage"` // Percent of eligible votes required for quorum - PassPercentage uint32 `json:"passpercentage"` // Percent of total votes required to pass - Options []VoteOption `json:"options"` // Vote option -} - -// EncodeVoteV2 encodes a VoteV2 into a JSON byte slice. -func EncodeVoteV2(v VoteV2) ([]byte, error) { - return json.Marshal(v) -} - -// DecodeVotedecodes a JSON byte slice into a VoteV2. -func DecodeVoteV2(payload []byte) (*VoteV2, error) { - var v VoteV2 - - err := json.Unmarshal(payload, &v) - if err != nil { - return nil, err - } - - return &v, nil -} - -// StartVoteV2 is used to start a proposal vote. -// -// The message being signed is the SHA256 digest of the VoteV2 JSON byte slice. -// -// Differences between StartVoteV1 and StartVoteV2: -// * Signature is the signature of a hash of the Vote struct. It was -// previously the signature of just the proposal token. -// * Vote is now a VoteV2. See the VoteV2 comment for more details. -type StartVoteV2 struct { - // decred plugin only data - Version uint `json:"version"` // Version of this structure - - PublicKey string `json:"publickey"` // Key used for signature - Vote VoteV2 `json:"vote"` // Vote options and params - Signature string `json:"signature"` // Signature of Vote hash -} - -// VerifySignature verifies that the StartVoteV2 signature is correct. -func (s *StartVoteV2) VerifySignature() error { - sig, err := util.ConvertSignature(s.Signature) - if err != nil { - return err - } - b, err := hex.DecodeString(s.PublicKey) - if err != nil { - return err - } - pk, err := identity.PublicIdentityFromBytes(b) - if err != nil { - return err - } - vb, err := json.Marshal(s.Vote) - if err != nil { - return err - } - msg := hex.EncodeToString(util.Digest(vb)) - if !pk.VerifyMessage([]byte(msg), sig) { - return fmt.Errorf("invalid signature") - } - return nil -} - -// EncodeStartVoteV2 encodes a StartVoteV2 into a JSON byte slice. -func EncodeStartVoteV2(v StartVoteV2) ([]byte, error) { - return json.Marshal(v) -} - -// DecodeVotedecodes a JSON byte slice into a StartVoteV2. -func DecodeStartVoteV2(payload []byte) (*StartVoteV2, error) { - var sv StartVoteV2 - - err := json.Unmarshal(payload, &sv) - if err != nil { - return nil, err - } - - return &sv, nil -} - -// StartVoteReply is the reply to StartVote. -const VersionStartVoteReply = 1 - -type StartVoteReply struct { - // decred plugin only data - Version uint `json:"version"` // Version of this structure - - // Shared data - StartBlockHeight string `json:"startblockheight"` // Block height - StartBlockHash string `json:"startblockhash"` // Block hash - EndHeight string `json:"endheight"` // Height of vote end - EligibleTickets []string `json:"eligibletickets"` // Valid voting tickets -} - -// EncodeStartVoteReply encodes StartVoteReply into a JSON byte slice. -func EncodeStartVoteReply(v StartVoteReply) ([]byte, error) { - return json.Marshal(v) -} - -// DecodeVoteReply decodes a JSON byte slice into a StartVoteReply. -func DecodeStartVoteReply(payload []byte) (*StartVoteReply, error) { - var v StartVoteReply - - err := json.Unmarshal(payload, &v) - if err != nil { - return nil, err - } - - return &v, nil -} - -// StartVoteRunoff instructs the plugin to start a runoff vote using the given -// submissions. Each submission is required to have its own AuthorizeVote and -// StartVote. -type StartVoteRunoff struct { - Token string `json:"token"` // Token of RFP proposal - AuthorizeVotes []AuthorizeVote `json:"authorizevotes"` // Submission auth votes - StartVotes []StartVoteV2 `json:"startvotes"` // Submission start votes -} - -// EncodeStartVoteRunoffencodes StartVoteRunoffinto a JSON byte slice. -func EncodeStartVoteRunoff(v StartVoteRunoff) ([]byte, error) { - return json.Marshal(v) -} - -// DecodeVotedecodes a JSON byte slice into a StartVoteRunoff. -func DecodeStartVoteRunoff(payload []byte) (*StartVoteRunoff, error) { - var sv StartVoteRunoff - - err := json.Unmarshal(payload, &sv) - if err != nil { - return nil, err - } - - return &sv, nil -} - -// StartVoteRunoffReply is the reply to StartVoteRunoff. The StartVoteReply -// will be the same for all submissions so only one is returned. The individual -// AuthorizeVoteReply is returned for each submission where the token is the -// map key. -type StartVoteRunoffReply struct { - AuthorizeVoteReplies map[string]AuthorizeVoteReply `json:"authorizevotereply"` - StartVoteReply StartVoteReply `json:"startvotereply"` -} - -// EncodeStartVoteRunoffReply encodes StartVoteRunoffReply into a JSON byte slice. -func EncodeStartVoteRunoffReply(v StartVoteRunoffReply) ([]byte, error) { - return json.Marshal(v) -} - -// DecodeVoteReply decodes a JSON byte slice into a StartVoteRunoffReply. -func DecodeStartVoteRunoffReply(payload []byte) (*StartVoteRunoffReply, error) { - var v StartVoteRunoffReply - - err := json.Unmarshal(payload, &v) - if err != nil { - return nil, err - } +// TODO remove this ErrorStatusT once politeiavoter has been updated. +type ErrorStatusT int - return &v, nil -} +const ErrorStatusVoteHasEnded ErrorStatusT = 4 // Comment is the structure that describes the full server side content. It // includes server side meta-data as well. Note that the receipt is the server diff --git a/politeiad/backend/gitbe/cms.go b/politeiad/backend/gitbe/cms.go index 93ec9423d..412dafe8f 100644 --- a/politeiad/backend/gitbe/cms.go +++ b/politeiad/backend/gitbe/cms.go @@ -485,6 +485,14 @@ func (g *gitBackEnd) validateCMSVoteBit(token, bit string) error { return _validateCMSVoteBit(options, mask, b) } +type invalidVoteBitError struct { + err error +} + +func (i invalidVoteBitError) Error() string { + return i.err.Error() +} + // _validateVoteBit iterates over all vote bits and ensure the sent in vote bit // exists. func _validateCMSVoteBit(options []cmsplugin.VoteOption, mask uint64, bit uint64) error { diff --git a/politeiad/backend/gitbe/decred.go b/politeiad/backend/gitbe/decred.go index 44dd4243e..9a3647059 100644 --- a/politeiad/backend/gitbe/decred.go +++ b/politeiad/backend/gitbe/decred.go @@ -86,28 +86,10 @@ type JournalAction struct { Action string `json:"action"` // Add/Del } -type CastVoteJournal struct { - CastVote decredplugin.CastVote `json:"castvote"` // Client side vote - Receipt string `json:"receipt"` // Signature of CastVote.Signature -} - -func encodeCastVoteJournal(cvj CastVoteJournal) ([]byte, error) { - b, err := json.Marshal(cvj) - if err != nil { - return nil, err - } - - return b, nil -} - var ( decredPluginSettings map[string]string // [key]setting decredPluginHooks map[string]func(string) error // [key]func(token) error - // Cached values, requires lock. These caches are lazy loaded. - decredPluginVoteCache = make(map[string]decredplugin.StartVote) // [token]StartVote - decredPluginVoteSnapshotCache = make(map[string]decredplugin.StartVoteReply) // [token]StartVoteReply - // Pregenerated journal actions journalAdd []byte journalDel []byte @@ -116,8 +98,6 @@ var ( // Plugin specific data that CANNOT be treated as metadata pluginDataDir = filepath.Join("plugins", "decred") - // Cached values, requires lock. These caches are built on startup. - decredPluginVotesCache = make(map[string]map[string]struct{}) // [token][ticket]struct{} decredPluginCommentsCache = make(map[string]map[string]decredplugin.Comment) // [token][commentid]comment journalsReplayed bool = false @@ -202,11 +182,6 @@ func (g *gitBackEnd) replayAllJournals() error { } for _, f := range files { name := f.Name() - // replay ballot for all props - err := g.replayBallot(name) - if err != nil { - return fmt.Errorf("replayAllJournals replayBallot %s %v", name, err) - } // replay comments for all props _, err = g.replayComments(name) if err != nil { @@ -1353,1198 +1328,3 @@ func (g *gitBackEnd) pluginGetComments(payload string) (string, error) { g.Unlock() return encodeGetCommentsReply(comments) } - -func prepareAuthorizeVote(fi *identity.FullIdentity, token, action, pubkey, sig string) decredplugin.AuthorizeVote { - r := fi.SignMessage([]byte(sig)) - return decredplugin.AuthorizeVote{ - Version: decredplugin.VersionAuthorizeVote, - Receipt: hex.EncodeToString(r[:]), - Timestamp: time.Now().Unix(), - Token: token, - Action: action, - PublicKey: pubkey, - Signature: sig, - } -} - -func prepareAuthorizeVoteReply(av decredplugin.AuthorizeVote, recordVersion string) decredplugin.AuthorizeVoteReply { - return decredplugin.AuthorizeVoteReply{ - Action: av.Action, - RecordVersion: recordVersion, - Receipt: av.Receipt, - Timestamp: av.Timestamp, - } -} - -// pluginAuthorizeVote updates the vetted repo with vote authorization -// metadata from the proposal author. -func (g *gitBackEnd) pluginAuthorizeVote(payload string) (string, error) { - log.Tracef("pluginAuthorizeVote") - - // Decode authorize vote - authorize, err := decredplugin.DecodeAuthorizeVote([]byte(payload)) - if err != nil { - return "", fmt.Errorf("DecodeAuthorizeVote %v", err) - } - token := authorize.Token - - // Verify proposal exists - if !g.vettedPropExists(token) { - return "", fmt.Errorf("unknown proposal: %v", token) - } - - // Get identity - // XXX this should become part of some sort of context - fiJSON, ok := decredPluginSettings[decredPluginIdentity] - if !ok { - return "", fmt.Errorf("full identity not set") - } - fi, err := identity.UnmarshalFullIdentity([]byte(fiJSON)) - if err != nil { - return "", fmt.Errorf("UnmarshalFullIdentity: %v", err) - } - - av := prepareAuthorizeVote(fi, authorize.Token, authorize.Action, - authorize.PublicKey, authorize.Signature) - avb, err := decredplugin.EncodeAuthorizeVote(av) - if err != nil { - return "", fmt.Errorf("EncodeAuthorizeVote: %v", err) - } - tokenb, err := util.ConvertStringToken(token) - if err != nil { - return "", fmt.Errorf("ConvertStringToken %v", err) - } - - // Verify proposal state - g.Lock() - defer g.Unlock() - if g.shutdown { - return "", backend.ErrShutdown - } - - if g.vettedMetadataStreamExists(tokenb, decredplugin.MDStreamVoteBits) { - // Vote has already started. This should not happen. - return "", fmt.Errorf("proposal vote already started: %v", - token) - } - - // Update metadata - err = g._updateVettedMetadata(tokenb, nil, []backend.MetadataStream{ - { - ID: decredplugin.MDStreamAuthorizeVote, - Payload: string(avb), - }, - }) - if err != nil { - return "", fmt.Errorf("_updateVettedMetadata: %v", err) - } - - // Prepare reply - version, err := getLatest(pijoin(g.vetted, token)) - if err != nil { - return "", fmt.Errorf("getLatest: %v", err) - } - avr := prepareAuthorizeVoteReply(av, version) - avrb, err := decredplugin.EncodeAuthorizeVoteReply(avr) - if err != nil { - return "", err - } - - log.Infof("Vote authorized for %v", token) - - return string(avrb), nil -} - -// validateStartVote validates the vote bits and the vote params of a decred -// plugin StartVote. -func validateStartVoteV2(sv decredplugin.StartVoteV2) error { - // Verify signature - err := sv.VerifySignature() - if err != nil { - return fmt.Errorf("invalid signature") - } - - // Verify vote bits are somewhat sane - for _, v := range sv.Vote.Options { - err := _validateVoteBit(sv.Vote.Options, sv.Vote.Mask, v.Bits) - if err != nil { - return fmt.Errorf("invalid vote bits: %v", err) - } - } - - // Make sure vote duration is within min/max range - min := decredPluginSettings[decredPluginVoteDurationMin] - max := decredPluginSettings[decredPluginVoteDurationMax] - voteDurationMin, err := strconv.ParseUint(min, 10, 32) - if err != nil { - return err - } - voteDurationMax, err := strconv.ParseUint(max, 10, 32) - if err != nil { - return err - } - if sv.Vote.Duration < uint32(voteDurationMin) || - sv.Vote.Duration > uint32(voteDurationMax) { - return fmt.Errorf("invalid duration: %v (%v - %v)", - sv.Vote.Duration, voteDurationMin, voteDurationMax) - } - - return nil -} - -// prepareStartVoteReply prepares a decred plugin StartVoteReply. -func prepareStartVoteReply(voteDuration, ticketMaturity uint32) (*decredplugin.StartVoteReply, error) { - // Get best block - bb, err := bestBlock() - if err != nil { - return nil, fmt.Errorf("bestBlock %v", err) - } - if bb.Height < ticketMaturity { - return nil, fmt.Errorf("invalid height") - } - - // Subtract TicketMaturity from block height to get into - // unforkable teritory - snapshotBlock, err := block(bb.Height - ticketMaturity) - if err != nil { - return nil, fmt.Errorf("bestBlock %v", err) - } - - // Get ticket pool snapshot - snapshot, err := snapshot(snapshotBlock.Hash) - if err != nil { - return nil, fmt.Errorf("snapshot %v", err) - } - if len(snapshot) == 0 { - return nil, fmt.Errorf("no eligible voters for block hash %v", - snapshotBlock.Hash) - } - - // Prepare reply - return &decredplugin.StartVoteReply{ - Version: decredplugin.VersionStartVoteReply, - StartBlockHeight: strconv.FormatUint(uint64(snapshotBlock.Height), - 10), - StartBlockHash: snapshotBlock.Hash, - // On EndHeight: we start in the past, add maturity to correct - EndHeight: strconv.FormatUint(uint64(snapshotBlock.Height+ - voteDuration+ - ticketMaturity), 10), - EligibleTickets: snapshot, - }, nil -} - -func (g *gitBackEnd) pluginStartVote(payload string) (string, error) { - sv, err := decredplugin.DecodeStartVoteV2([]byte(payload)) - if err != nil { - return "", fmt.Errorf("DecodeStartVote %v", err) - } - - err = validateStartVoteV2(*sv) - if err != nil { - return "", err - } - - svr, err := prepareStartVoteReply(sv.Vote.Duration, - uint32(g.activeNetParams.TicketMaturity)) - if err != nil { - return "", err - } - - // Verify vote type - if sv.Vote.Type != decredplugin.VoteTypeStandard { - return "", fmt.Errorf("invalid vote type") - } - - // Verify proposal exists - token := sv.Vote.Token - if !g.vettedPropExists(token) { - return "", fmt.Errorf("unknown proposal: %v", token) - } - - g.Lock() - defer g.Unlock() - if g.shutdown { - // Make sure we are not shutting down - return "", backend.ErrShutdown - } - - // Ensure proposal is not an RFP submissions. The plugin command - // startvoterunoff must be used to start a runoff vote between RFP - // submissions. - /* - // TODO - pm, err := g.vettedProposalMetadata(token) - switch { - case errors.Is(err, errProposalMetadataNotFound): - // Proposal is not an RFP submission. This is ok. - case err != nil: - // All other errors - return "", err - case pm.LinkTo != "": - // ProposalMetadata exists and a linkto was set. Check if this - // proposal is an RFP submission. - linkToPM, err := g.vettedProposalMetadata(pm.LinkTo) - if err != nil { - return "", err - } - if linkToPM.LinkBy != 0 { - // LinkBy will only be set on RFP proposals - return "", fmt.Errorf("proposal is an rfp submission: %v", - token) - } - default: - // ProposalMetadata exists, but this proposal is not linked to - // another proposal. This is ok. - } - */ - - // Verify proposal state - tokenb, err := util.ConvertStringToken(token) - if err != nil { - return "", err - } - avExists := g.vettedMetadataStreamExists(tokenb, - decredplugin.MDStreamAuthorizeVote) - vbExists := g.vettedMetadataStreamExists(tokenb, - decredplugin.MDStreamVoteBits) - vsExists := g.vettedMetadataStreamExists(tokenb, - decredplugin.MDStreamVoteSnapshot) - - switch { - case !avExists: - // Authorize vote md is not present - return "", fmt.Errorf("no authorize vote metadata: %v", token) - case vbExists && vsExists: - // Vote has started - return "", fmt.Errorf("proposal vote already started: %v", token) - case !vbExists && !vsExists: - // Vote has not started; continue - default: - // We're in trouble! - return "", fmt.Errorf("proposal is unknown vote state: %v", - token) - } - - // Verify that the proposal version being voted on is the most - // recent proposal version. - latestVersion, err := getLatest(pijoin(g.unvetted, token)) - if err != nil { - return "", err - } - version := strconv.FormatUint(uint64(sv.Vote.ProposalVersion), 10) - if latestVersion != version { - return "", fmt.Errorf("invalid proposal version") - } - - // Ensure vote authorization has not been revoked - b, err := g.getVettedMetadataStream(tokenb, - decredplugin.MDStreamAuthorizeVote) - if err != nil { - return "", fmt.Errorf("getVettedMetadataStream %v: %v", - decredplugin.MDStreamAuthorizeVote, err) - } - av, err := decredplugin.DecodeAuthorizeVote(b) - if err != nil { - return "", fmt.Errorf("DecodeAuthorizeVote: %v", err) - } - if av.Action == decredplugin.AuthVoteActionRevoke { - return "", fmt.Errorf("vote authorization revoked") - } - - // Add version to on disk structure - sv.Version = decredplugin.VersionStartVote - - // Encode relevant data - svb, err := decredplugin.EncodeStartVoteV2(*sv) - if err != nil { - return "", fmt.Errorf("EncodeStartVoteV2: %v", err) - } - svrb, err := decredplugin.EncodeStartVoteReply(*svr) - if err != nil { - return "", fmt.Errorf("EncodeStartVoteReply: %v", err) - } - - // Store snapshot in metadata - err = g._updateVettedMetadata(tokenb, nil, []backend.MetadataStream{ - { - ID: decredplugin.MDStreamVoteBits, - Payload: string(svb), - }, - { - ID: decredplugin.MDStreamVoteSnapshot, - Payload: string(svrb), - }}) - if err != nil { - return "", fmt.Errorf("_updateVettedMetadata: %v", err) - } - - // Add vote snapshot to in-memory cache - decredPluginVoteSnapshotCache[token] = *svr - - log.Infof("Vote started for: %v snapshot %v start %v end %v", - token, svr.StartBlockHash, svr.StartBlockHeight, - svr.EndHeight) - - // return success and encoded answer - return string(svrb), nil -} - -// pluginStartVoteRunoff starts a runoff vote between the given submissions. -func (g *gitBackEnd) pluginStartVoteRunoff(payload string) (string, error) { - sv, err := decredplugin.DecodeStartVoteRunoff([]byte(payload)) - if err != nil { - return "", err - } - - // Ensure start votes and authorize votes match - authVotes := make(map[string]struct{}, len(sv.AuthorizeVotes)) - startVotes := make(map[string]struct{}, len(sv.StartVotes)) - for _, v := range sv.AuthorizeVotes { - authVotes[v.Token] = struct{}{} - } - for _, v := range sv.StartVotes { - startVotes[v.Vote.Token] = struct{}{} - } - for _, v := range sv.AuthorizeVotes { - _, ok := startVotes[v.Token] - if !ok { - return "", fmt.Errorf("authorize vote found without matching"+ - "start vote %v", v.Token) - } - } - for _, v := range sv.StartVotes { - _, ok := authVotes[v.Vote.Token] - if !ok { - return "", fmt.Errorf("start vote found without matching "+ - "authorize vote %v", v.Vote.Token) - } - } - if len(sv.StartVotes) == 0 { - return "", fmt.Errorf("no start votes found") - } - - var ( - // Vote params must be the same for all submissions so set - // the defaults to be the params of the first submission. - duration = sv.StartVotes[0].Vote.Duration - quorum = sv.StartVotes[0].Vote.QuorumPercentage - pass = sv.StartVotes[0].Vote.PassPercentage - ) - - // Validate the StartVote for each submission - for _, v := range sv.StartVotes { - err = validateStartVoteV2(v) - if err != nil { - return "", err - } - - // Validate vote type - if v.Vote.Type != decredplugin.VoteTypeRunoff { - return "", fmt.Errorf("invalid vote type") - } - - // Vote params must be the same for all submissions - switch { - case duration != v.Vote.Duration: - return "", fmt.Errorf("start votes have different vote durations") - case quorum != v.Vote.QuorumPercentage: - return "", fmt.Errorf("start votes have different quorum percentages") - case pass != v.Vote.PassPercentage: - return "", fmt.Errorf("start votes have different pass percentages") - } - - // Vote bits can only be yes/no for submissions - if len(v.Vote.Options) != 2 { - return "", fmt.Errorf("invalid number of vote options: %v", - v.Vote.Token) - } - for _, vo := range v.Vote.Options { - if vo.Id != decredplugin.VoteOptionIDApprove && - vo.Id != decredplugin.VoteOptionIDReject { - return "", fmt.Errorf("invalid vote option id: %v", - v.Vote.Token) - } - } - } - - // Prepare the StartVoteReply. This will be the same for all - // submissions. - svr, err := prepareStartVoteReply(duration, - uint32(g.activeNetParams.TicketMaturity)) - if err != nil { - return "", err - } - svrb, err := decredplugin.EncodeStartVoteReply(*svr) - if err != nil { - return "", fmt.Errorf("EncodeStartVoteReply: %v", - err) - } - - // Verify the rfp proposal and all rfp submissions exist - if !g.vettedPropExists(sv.Token) { - return "", fmt.Errorf("rfp proposal not found: %v", - sv.Token) - } - for _, v := range sv.StartVotes { - if !g.vettedPropExists(v.Vote.Token) { - return "", fmt.Errorf("rfp submission not found: %v", - v.Vote.Token) - } - } - - // Run everything else with the lock held - g.Lock() - defer g.Unlock() - if g.shutdown { - return "", backend.ErrShutdown - } - - /* - // TODO - // Verify this proposal is indeed an RFP - pm, err := g.vettedProposalMetadata(sv.Token) - switch { - case errors.Is(err, errProposalMetadataNotFound): - // No ProposalMetadata. This is not an RFP. - return "", fmt.Errorf("proposal is not an rfp: %v", sv.Token) - case err != nil: - // All other errors - return "", err - case pm.LinkBy == 0: - // ProposalMetadata found but this is not an RFP - return "", fmt.Errorf("proposal is not an rfp: %v", sv.Token) - case pm.LinkBy > 0: - // This proposal is an RFP. This is what we want. - default: - return "", fmt.Errorf("unknown proposal state") - } - - // Validate proposal state of all rfp submissions. The authorize - // vote metadata is intentionally not checked. RFP submissions - // are not required to have the vote authorized by the proposal - // author. - for _, v := range sv.StartVotes { - tokenb, err := util.ConvertStringToken(v.Vote.Token) - if err != nil { - return "", err - } - - authVoteExists := g.vettedMetadataStreamExists(tokenb, - decredplugin.MDStreamAuthorizeVote) - voteBitsExist := g.vettedMetadataStreamExists(tokenb, - decredplugin.MDStreamVoteBits) - voteSnapshotExists := g.vettedMetadataStreamExists(tokenb, - decredplugin.MDStreamVoteSnapshot) - switch { - case !authVoteExists && !voteBitsExist && !voteSnapshotExists: - // Vote has not started, continue - case authVoteExists && voteBitsExist && voteSnapshotExists: - // Vote has started - return "", fmt.Errorf("vote already started: %x", - tokenb) - default: - // This is bad, both files should exist or not exist - return "", fmt.Errorf("proposal in unknown vote state: %x", - tokenb) - } - } - */ - - // Get identity - fiJSON, ok := decredPluginSettings[decredPluginIdentity] - if !ok { - return "", fmt.Errorf("full identity not set") - } - fi, err := identity.UnmarshalFullIdentity([]byte(fiJSON)) - if err != nil { - return "", err - } - - // Prepare work to be done - var ( - um = make([]updateMetadata, 0, len(sv.StartVotes)) - authReplies = make(map[string]decredplugin.AuthorizeVoteReply) // [token]AuthorizeVoteReply - - ) - for _, v := range sv.AuthorizeVotes { - av := prepareAuthorizeVote(fi, v.Token, v.Action, - v.PublicKey, v.Signature) - - // Encode authorize vote - avb, err := decredplugin.EncodeAuthorizeVote(av) - if err != nil { - return "", fmt.Errorf("EncodeAuthorizeVote %v: %v", - v.Token, err) - } - - // Add to work array - um = append(um, updateMetadata{ - token: v.Token, - mdAppend: nil, - mdOverwrite: []backend.MetadataStream{ - { - ID: decredplugin.MDStreamAuthorizeVote, - Payload: string(avb), - }, - }, - }) - - // Prepare reply - recordVersion, err := getLatest(pijoin(g.vetted, v.Token)) - if err != nil { - return "", fmt.Errorf("getLatest %v: %v", - v.Token, err) - } - avr := prepareAuthorizeVoteReply(av, recordVersion) - authReplies[v.Token] = avr - } - for _, v := range sv.StartVotes { - // Add version to on disk structure - v.Version = decredplugin.VersionStartVote - - // Encode start vote - svb, err := decredplugin.EncodeStartVoteV2(v) - if err != nil { - return "", fmt.Errorf("EncodeStartVote %v: %v", - v.Vote.Token, err) - } - - // Add to work array - um = append(um, updateMetadata{ - token: v.Vote.Token, - mdAppend: nil, - mdOverwrite: []backend.MetadataStream{ - { - ID: decredplugin.MDStreamVoteBits, - Payload: string(svb), - }, - { - ID: decredplugin.MDStreamVoteSnapshot, - Payload: string(svrb), - }}, - }) - } - - // idTmp uses the token of the rfp proposal, not the rfp - // submission tokens. - idTmp := sv.Token + "_tmp" - - // Update metadata for each record - err = g._updateVettedMetadataMulti(um, idTmp) - if err != nil { - return "", err - } - - // Add vote snapshots to in-memory cache - for _, sv := range sv.StartVotes { - decredPluginVoteSnapshotCache[sv.Vote.Token] = *svr - - log.Infof("Vote started for: %v snapshot %v start %v end %v", - sv.Vote.Token, svr.StartBlockHash, svr.StartBlockHeight, - svr.EndHeight) - } - - // Prepare reply - reply, err := decredplugin.EncodeStartVoteRunoffReply( - decredplugin.StartVoteRunoffReply{ - AuthorizeVoteReplies: authReplies, - StartVoteReply: *svr, - }) - if err != nil { - return "", err - } - - return string(reply), nil -} - -// validateVoteByAddress validates that vote, as specified by the commitment -// address with largest amount, is signed correctly. -func (g *gitBackEnd) validateVoteByAddress(token, ticket, addr, votebit, signature string) error { - // Recreate message - msg := token + ticket + votebit - - // verifyMessage expects base64 encoded sig - sig, err := hex.DecodeString(signature) - if err != nil { - return err - } - - // Verify message - validated, err := g.verifyMessage(addr, msg, - base64.StdEncoding.EncodeToString(sig)) - if err != nil { - return err - } - - if !validated { - return fmt.Errorf("could not verify message") - } - - return nil -} - -type invalidVoteBitError struct { - err error -} - -func (i invalidVoteBitError) Error() string { - return i.err.Error() -} - -// _validateVoteBit iterates over all vote bits and ensure the sent in vote bit -// exists. -func _validateVoteBit(options []decredplugin.VoteOption, mask uint64, bit uint64) error { - if len(options) == 0 { - return fmt.Errorf("_validateVoteBit vote corrupt") - } - if bit == 0 { - return invalidVoteBitError{ - err: fmt.Errorf("invalid bit 0x%x", bit), - } - } - if mask&bit != bit { - return invalidVoteBitError{ - err: fmt.Errorf("invalid mask 0x%x bit 0x%x", - mask, bit), - } - } - for _, v := range options { - if v.Bits == bit { - return nil - } - } - return invalidVoteBitError{ - err: fmt.Errorf("bit not found 0x%x", bit), - } -} - -// validateVoteBits ensures that the passed in bit is a valid vote option. -// This function is expensive due to it's filesystem touches and therefore is -// lazily cached. This could stand a rewrite. -func (g *gitBackEnd) validateVoteBit(token, bit string) error { - b, err := strconv.ParseUint(bit, 16, 64) - if err != nil { - return err - } - - g.Lock() - defer g.Unlock() - if g.shutdown { - return backend.ErrShutdown - } - - sv, ok := decredPluginVoteCache[token] - if !ok { - // StartVote is not in the cache. Load it from disk. - - // git checkout master - err = g.gitCheckout(g.unvetted, "master") - if err != nil { - return err - } - - // git pull --ff-only --rebase - err = g.gitPull(g.unvetted, true) - if err != nil { - return err - } - - // Load md stream - tokenb, err := util.ConvertStringToken(token) - if err != nil { - return err - } - svb, err := g.getVettedMetadataStream(tokenb, - decredplugin.MDStreamVoteBits) - if err != nil { - return err - } - svp, err := decredplugin.DecodeStartVote(svb) - if err != nil { - return err - } - sv = *svp - - // Update cache - decredPluginVoteCache[token] = sv - } - - // Handle StartVote versioning - var ( - mask uint64 - options []decredplugin.VoteOption - ) - switch sv.Version { - case decredplugin.VersionStartVoteV1: - sv1, err := decredplugin.DecodeStartVoteV1([]byte(sv.Payload)) - if err != nil { - return err - } - mask = sv1.Vote.Mask - options = sv1.Vote.Options - case decredplugin.VersionStartVoteV2: - sv2, err := decredplugin.DecodeStartVoteV2([]byte(sv.Payload)) - if err != nil { - return err - } - mask = sv2.Vote.Mask - options = sv2.Vote.Options - default: - return fmt.Errorf("invalid start vote version %v %v", - sv.Version, sv.Token) - } - - return _validateVoteBit(options, mask, b) -} - -// replayBallot replays voting journalfor given proposal. -// -// Functions must be called WITH the lock held. -func (g *gitBackEnd) replayBallot(token string) error { - log.Debugf("replayBallot %v", token) - - // Verify proposal exists, we can run this lockless - if !g.vettedPropExists(token) { - return nil - } - - // Do some cheap things before expensive calls - bfilename := pijoin(g.journals, token, - defaultBallotFilename) - - // Replay journal - err := g.journal.Open(bfilename) - if err != nil { - if !os.IsNotExist(err) { - return fmt.Errorf("journal.Open: %v", err) - } - return nil - } - defer func() { - err = g.journal.Close(bfilename) - if err != nil { - log.Errorf("journal.Close: %v", err) - } - }() - - for { - err = g.journal.Replay(bfilename, func(s string) error { - ss := bytes.NewReader([]byte(s)) - d := json.NewDecoder(ss) - - // Decode action - var action JournalAction - err = d.Decode(&action) - if err != nil { - return fmt.Errorf("journal action: %v", err) - } - - switch action.Action { - case journalActionAdd: - var cvj CastVoteJournal - err = d.Decode(&cvj) - if err != nil { - return fmt.Errorf("journal add: %v", - err) - } - - token := cvj.CastVote.Token - ticket := cvj.CastVote.Ticket - // See if the prop already exists - if _, ok := decredPluginVotesCache[token]; !ok { - // Create map to track tickets - decredPluginVotesCache[token] = make(map[string]struct{}) - } - // See if we have a duplicate vote - if _, ok := decredPluginVotesCache[token][ticket]; ok { - log.Errorf("duplicate cast vote %v %v", - token, ticket) - } - // All good, record vote in cache - decredPluginVotesCache[token][ticket] = struct{}{} - - default: - return fmt.Errorf("invalid action: %v", - action.Action) - } - return nil - }) - if errors.Is(err, io.EOF) { - break - } else if err != nil { - return err - } - } - - return nil -} - -// loadVoteSnapshotCache loads the StartVoteReply from disk for the provided -// token and adds it to the decredPluginVoteSnapshotCache. -// -// This function must be called WITH the lock held. -func (g *gitBackEnd) loadVoteSnapshotCache(token string) (*decredplugin.StartVoteReply, error) { - // git checkout master - err := g.gitCheckout(g.unvetted, "master") - if err != nil { - return nil, err - } - - // git pull --ff-only --rebase - err = g.gitPull(g.unvetted, true) - if err != nil { - return nil, err - } - - // Load the vote snapshot from disk - f, err := os.Open(mdFilename(g.vetted, token, - decredplugin.MDStreamVoteSnapshot)) - if err != nil { - return nil, err - } - defer f.Close() - - var svr decredplugin.StartVoteReply - d := json.NewDecoder(f) - err = d.Decode(&svr) - if err != nil { - return nil, err - } - - decredPluginVoteSnapshotCache[token] = svr - - return &svr, nil -} - -// voteEndHeight returns the voting period end height for the provided token. -func (g *gitBackEnd) voteEndHeight(token string) (uint32, error) { - g.Lock() - defer g.Unlock() - if g.shutdown { - return 0, backend.ErrShutdown - } - - svr, ok := decredPluginVoteSnapshotCache[token] - if !ok { - s, err := g.loadVoteSnapshotCache(token) - if err != nil { - return 0, err - } - svr = *s - } - - endHeight, err := strconv.ParseUint(svr.EndHeight, 10, 64) - if err != nil { - return 0, err - } - - return uint32(endHeight), nil -} - -// writeVote writes the provided vote to the provided journal file path, if the -// vote does not already exist. Once successfully written to the journal, the -// vote is added to the cast vote memory cache. -// -// This function must be called WITHOUT the lock held. -func (g *gitBackEnd) writeVote(v decredplugin.CastVote, receipt, journalPath string) error { - g.Lock() - defer g.Unlock() - - // Ensure ticket is eligible to vote. - // This cache should have already been loaded when the - // vote end height was validated, but lets be sure. - svr, ok := decredPluginVoteSnapshotCache[v.Token] - if !ok { - s, err := g.loadVoteSnapshotCache(v.Token) - if err != nil { - return fmt.Errorf("loadVoteSnapshotCache: %v", - err) - } - svr = *s - } - var found bool - for _, t := range svr.EligibleTickets { - if t == v.Ticket { - found = true - break - } - } - if !found { - return errIneligibleTicket - } - - // Ensure vote is not a duplicate - _, ok = decredPluginVotesCache[v.Token] - if !ok { - decredPluginVotesCache[v.Token] = make(map[string]struct{}) - } - - _, ok = decredPluginVotesCache[v.Token][v.Ticket] - if ok { - return errDuplicateVote - } - - // Create journal entry - cvj := CastVoteJournal{ - CastVote: v, - Receipt: receipt, - } - blob, err := encodeCastVoteJournal(cvj) - if err != nil { - return fmt.Errorf("encodeCastVoteJournal: %v", - err) - } - - // Write vote to journal - err = g.journal.Journal(journalPath, string(journalAdd)+ - string(blob)) - if err != nil { - return fmt.Errorf("could not journal vote %v: %v %v", - v.Token, v.Ticket, err) - } - - // Add vote to memory cache - decredPluginVotesCache[v.Token][v.Ticket] = struct{}{} - - return nil -} - -func (g *gitBackEnd) pluginBallot(payload string) (string, error) { - log.Tracef("pluginBallot") - - // Check if journals were replayed - if !journalsReplayed { - return "", backend.ErrJournalsNotReplayed - } - - // Decode ballot - ballot, err := decredplugin.DecodeBallot([]byte(payload)) - if err != nil { - return "", fmt.Errorf("DecodeBallot: %v", err) - } - - // XXX this should become part of some sort of context - fiJSON, ok := decredPluginSettings[decredPluginIdentity] - if !ok { - return "", fmt.Errorf("full identity not set") - } - fi, err := identity.UnmarshalFullIdentity([]byte(fiJSON)) - if err != nil { - return "", err - } - - // Get best block - bb, err := bestBlock() - if err != nil { - return "", fmt.Errorf("bestBlock %v", err) - } - - // Obtain all largest commitment addresses. Assume everything was sent - // in correct. - tickets := make([]string, 0, len(ballot.Votes)) - for _, v := range ballot.Votes { - tickets = append(tickets, v.Ticket) - } - ticketAddresses, err := largestCommitmentAddresses(tickets) - if err != nil { - return "", err - } - - br := decredplugin.BallotReply{ - Receipts: make([]decredplugin.CastVoteReply, len(ballot.Votes)), - } - for k, v := range ballot.Votes { - // Verify proposal exists, we can run this lockless - if !g.vettedPropExists(v.Token) { - log.Errorf("pluginBallot: proposal not found: %v", - v.Token) - e := decredplugin.ErrorStatusProposalNotFound - br.Receipts[k].ErrorStatus = e - br.Receipts[k].Error = fmt.Sprintf("%v: %v", - decredplugin.ErrorStatus[e], v.Token) - continue - } - - // Ensure that the votebits are correct - err = g.validateVoteBit(v.Token, v.VoteBit) - if err != nil { - var ierr invalidVoteBitError - if errors.As(err, &ierr) { - es := decredplugin.ErrorStatusInvalidVoteBit - br.Receipts[k].ErrorStatus = es - br.Receipts[k].Error = fmt.Sprintf("%v: %v", - decredplugin.ErrorStatus[es], ierr.err.Error()) - continue - } - t := time.Now().Unix() - log.Errorf("pluginBallot: validateVoteBit %v %v %v %v", - v.Ticket, v.Token, t, err) - e := decredplugin.ErrorStatusInternalError - br.Receipts[k].ErrorStatus = e - br.Receipts[k].Error = fmt.Sprintf("%v: %v", - decredplugin.ErrorStatus[e], t) - continue - } - - // Verify voting period has not ended - endHeight, err := g.voteEndHeight(v.Token) - if err != nil { - t := time.Now().Unix() - log.Errorf("pluginBallot: voteEndHeight %v %v %v %v", - v.Ticket, v.Token, t, err) - e := decredplugin.ErrorStatusInternalError - br.Receipts[k].ErrorStatus = e - br.Receipts[k].Error = fmt.Sprintf("%v: %v", - decredplugin.ErrorStatus[e], t) - continue - } - if bb.Height >= endHeight { - e := decredplugin.ErrorStatusVoteHasEnded - br.Receipts[k].ErrorStatus = e - br.Receipts[k].Error = fmt.Sprintf("%v: %v", - decredplugin.ErrorStatus[e], v.Token) - continue - } - - // See if there was an error for this address - if ticketAddresses[k].err != nil { - t := time.Now().Unix() - log.Errorf("pluginBallot: ticketAddresses %v %v %v %v", - v.Ticket, v.Token, t, err) - e := decredplugin.ErrorStatusInternalError - br.Receipts[k].ErrorStatus = e - br.Receipts[k].Error = fmt.Sprintf("%v: %v", - decredplugin.ErrorStatus[e], t) - continue - } - - // Verify that vote is signed correctly - err = g.validateVoteByAddress(v.Token, v.Ticket, - ticketAddresses[k].bestAddr, v.VoteBit, v.Signature) - if err != nil { - t := time.Now().Unix() - log.Errorf("pluginBallot: validateVote %v %v %v %v", - v.Ticket, v.Token, t, err) - e := decredplugin.ErrorStatusInternalError - br.Receipts[k].ErrorStatus = e - br.Receipts[k].Error = fmt.Sprintf("%v: %v", - decredplugin.ErrorStatus[e], t) - continue - } - - // Ensure journal directory exists - dir := pijoin(g.journals, v.Token) - bfilename := pijoin(dir, defaultBallotFilename) - err = os.MkdirAll(dir, 0774) - if err != nil { - // Should not fail, so return failure to alert people - return "", fmt.Errorf("make journal dir: %v", err) - } - - // Sign signature - r := fi.SignMessage([]byte(v.Signature)) - receipt := hex.EncodeToString(r[:]) - - // Write vote to journal - err = g.writeVote(v, receipt, bfilename) - if err != nil { - switch err { - case errDuplicateVote: - e := decredplugin.ErrorStatusDuplicateVote - br.Receipts[k].ErrorStatus = e - br.Receipts[k].Error = fmt.Sprintf("%v: %v", - decredplugin.ErrorStatus[e], v.Token) - continue - case errIneligibleTicket: - e := decredplugin.ErrorStatusIneligibleTicket - br.Receipts[k].ErrorStatus = e - br.Receipts[k].Error = fmt.Sprintf("%v: %v", - decredplugin.ErrorStatus[e], v.Token) - continue - default: - // Should not fail, so return failure to alert people - return "", fmt.Errorf("write vote: %v", err) - } - } - - // Update reply - br.Receipts[k].ClientSignature = v.Signature - br.Receipts[k].Signature = receipt - - // Mark comment journal dirty - flushFilename := pijoin(g.journals, v.Token, - defaultBallotFlushed) - _ = os.Remove(flushFilename) - } - - // Encode reply - brb, err := decredplugin.EncodeBallotReply(br) - if err != nil { - return "", fmt.Errorf("EncodeBallotReply: %v", err) - } - - // return success and encoded answer - return string(brb), nil -} - -// tallyVotes replays the ballot journal for a proposal and tallies the votes. -// -// Function must be called WITH the lock held. -func (g *gitBackEnd) tallyVotes(token string) ([]decredplugin.CastVote, error) { - // Do some cheap things before expensive calls - bfilename := pijoin(g.journals, token, defaultBallotFilename) - - // Replay journal - err := g.journal.Open(bfilename) - if err != nil { - if !os.IsNotExist(err) { - return nil, fmt.Errorf("journal.Open: %v", err) - } - return []decredplugin.CastVote{}, nil - } - defer func() { - err = g.journal.Close(bfilename) - if err != nil { - log.Errorf("journal.Close: %v", err) - } - }() - - cv := make([]decredplugin.CastVote, 0, 41000) - for { - err = g.journal.Replay(bfilename, func(s string) error { - ss := bytes.NewReader([]byte(s)) - d := json.NewDecoder(ss) - - // Decode action - var action JournalAction - err = d.Decode(&action) - if err != nil { - return fmt.Errorf("journal action: %v", err) - } - - switch action.Action { - case journalActionAdd: - var cvj CastVoteJournal - err = d.Decode(&cvj) - if err != nil { - return fmt.Errorf("journal add: %v", - err) - } - cv = append(cv, cvj.CastVote) - - default: - return fmt.Errorf("invalid action: %v", - action.Action) - } - return nil - }) - if errors.Is(err, io.EOF) { - break - } else if err != nil { - return nil, err - } - } - - return cv, nil -} diff --git a/politeiad/backend/gitbe/gitbe.go b/politeiad/backend/gitbe/gitbe.go index 7d5e8a769..4f1828c3b 100644 --- a/politeiad/backend/gitbe/gitbe.go +++ b/politeiad/backend/gitbe/gitbe.go @@ -1558,17 +1558,6 @@ func (g *gitBackEnd) _updateRecord(commit bool, id string, mdAppend, mdOverwrite return err } - // Check for authorizevote metadata and delete it if found - avFilename := fmt.Sprintf("%02v%v", decredplugin.MDStreamAuthorizeVote, - defaultMDFilenameSuffix) - _, err = os.Stat(pijoin(joinLatest(g.unvetted, id), avFilename)) - if err == nil { - err = g.gitRm(g.unvetted, pijoin(id, version, avFilename), true) - if err != nil { - return err - } - } - // Call plugin hooks f, ok := decredPluginHooks[PluginPostHookEdit] if ok { @@ -2845,14 +2834,6 @@ func (g *gitBackEnd) Plugin(pluginID, command, commandID, payload string) (strin log.Tracef("Plugin: %v", command) switch command { // Decred plugin - case decredplugin.CmdAuthorizeVote: - return g.pluginAuthorizeVote(payload) - case decredplugin.CmdStartVote: - return g.pluginStartVote(payload) - case decredplugin.CmdStartVoteRunoff: - return g.pluginStartVoteRunoff(payload) - case decredplugin.CmdBallot: - return g.pluginBallot(payload) case decredplugin.CmdBestBlock: return g.pluginBestBlock() case decredplugin.CmdNewComment: @@ -3062,21 +3043,6 @@ func New(anp *chaincfg.Params, root string, dcrtimeHost string, gitPath string, setCMSPluginSetting(cmsPluginIdentity, string(idJSON)) setCMSPluginSetting(cmsPluginJournals, g.journals) - // Setup decred plugin - var voteDurationMin, voteDurationMax string - switch anp.Name { - case chaincfg.MainNetParams().Name: - voteDurationMin = strconv.Itoa(decredplugin.VoteDurationMinMainnet) - voteDurationMax = strconv.Itoa(decredplugin.VoteDurationMaxMainnet) - case chaincfg.TestNet3Params().Name: - voteDurationMin = strconv.Itoa(decredplugin.VoteDurationMinTestnet) - voteDurationMax = strconv.Itoa(decredplugin.VoteDurationMaxTestnet) - default: - return nil, fmt.Errorf("unknown chaincfg params '%v'", anp.Name) - } - - setDecredPluginSetting(decredPluginVoteDurationMin, voteDurationMin) - setDecredPluginSetting(decredPluginVoteDurationMax, voteDurationMax) setDecredPluginSetting(decredPluginIdentity, string(idJSON)) setDecredPluginSetting(decredPluginJournals, g.journals) setDecredPluginHook(PluginPostHookEdit, g.decredPluginPostEdit) diff --git a/politeiawww/cmd/piwww/votestartrunoff.go b/politeiawww/cmd/piwww/votestartrunoff.go index 4ba106cb6..fb506e832 100644 --- a/politeiawww/cmd/piwww/votestartrunoff.go +++ b/politeiawww/cmd/piwww/votestartrunoff.go @@ -9,7 +9,7 @@ import ( "encoding/json" "strconv" - "github.com/decred/politeia/decredplugin" + "github.com/decred/politeia/politeiad/plugins/ticketvote" pi "github.com/decred/politeia/politeiawww/api/pi/v1" v1 "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/cmd/shared" @@ -105,12 +105,12 @@ func (cmd *voteStartRunoffCmd) Execute(args []string) error { PassPercentage: pass, Options: []pi.VoteOption{ { - ID: decredplugin.VoteOptionIDApprove, + ID: ticketvote.VoteOptionIDApprove, Description: "Approve proposal", Bit: 0x01, }, { - ID: decredplugin.VoteOptionIDReject, + ID: ticketvote.VoteOptionIDReject, Description: "Don't approve proposal", Bit: 0x02, }, From 8c426c0e164d08a25754a57bddabbf152a867ea3 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 12 Jan 2021 13:03:08 -0600 Subject: [PATCH 211/449] decredplugin: Put cast vote errors back. --- decredplugin/decredplugin.go | 28 ++++++++++++++++++++++++++-- politeiad/backend/tlogbe/pi_test.go | 1 - 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/decredplugin/decredplugin.go b/decredplugin/decredplugin.go index f19c60fa6..c67b85257 100644 --- a/decredplugin/decredplugin.go +++ b/decredplugin/decredplugin.go @@ -18,10 +18,34 @@ const ( CmdGetComments = "getcomments" ) -// TODO remove this ErrorStatusT once politeiavoter has been updated. +// ErrorStatusT represents decredplugin errors that result from casting a vote. +// +// These are part of the www/v1 API and must stay in until the deprecated cast +// votes route is removed. type ErrorStatusT int -const ErrorStatusVoteHasEnded ErrorStatusT = 4 +const ( + ErrorStatusInvalid ErrorStatusT = 0 + ErrorStatusInternalError ErrorStatusT = 1 + ErrorStatusProposalNotFound ErrorStatusT = 2 + ErrorStatusInvalidVoteBit ErrorStatusT = 3 + ErrorStatusVoteHasEnded ErrorStatusT = 4 + ErrorStatusDuplicateVote ErrorStatusT = 5 + ErrorStatusIneligibleTicket ErrorStatusT = 6 +) + +var ( + // ErrorStatus converts error status codes to human readable text. + ErrorStatus = map[ErrorStatusT]string{ + ErrorStatusInvalid: "invalid error status", + ErrorStatusInternalError: "internal error", + ErrorStatusProposalNotFound: "proposal not found", + ErrorStatusInvalidVoteBit: "invalid vote bit", + ErrorStatusVoteHasEnded: "vote has ended", + ErrorStatusDuplicateVote: "duplicate vote", + ErrorStatusIneligibleTicket: "ineligbile ticket", + } +) // Comment is the structure that describes the full server side content. It // includes server side meta-data as well. Note that the receipt is the server diff --git a/politeiad/backend/tlogbe/pi_test.go b/politeiad/backend/tlogbe/pi_test.go index 2990c7401..abda31a14 100644 --- a/politeiad/backend/tlogbe/pi_test.go +++ b/politeiad/backend/tlogbe/pi_test.go @@ -31,7 +31,6 @@ func TestCommentNew(t *testing.T) { }} tlogBackend.RegisterPlugin(backend.Plugin{ ID: comments.ID, - Version: comments.Version, Settings: settings, Identity: id, }) From 3af45e79acdfad6eb5e5ae0c39b62938488fba00 Mon Sep 17 00:00:00 2001 From: "Thiago F. Figueiredo" Date: Tue, 12 Jan 2021 16:26:21 -0300 Subject: [PATCH 212/449] tlogbe: Add comment cmdDel and pi commentDel plugin tests. --- politeiad/backend/tlogbe/comments.go | 32 +-- politeiad/backend/tlogbe/comments_test.go | 255 +++++++++++++++++++--- politeiad/backend/tlogbe/pi.go | 2 +- politeiad/backend/tlogbe/pi_test.go | 187 +++++++++++++++- politeiad/backend/tlogbe/testing.go | 8 +- politeiad/backend/tlogbe/tlogbe_test.go | 100 ++++++--- 6 files changed, 503 insertions(+), 81 deletions(-) diff --git a/politeiad/backend/tlogbe/comments.go b/politeiad/backend/tlogbe/comments.go index 0fce11d95..8a0ce10ff 100644 --- a/politeiad/backend/tlogbe/comments.go +++ b/politeiad/backend/tlogbe/comments.go @@ -1123,7 +1123,7 @@ func (p *commentsPlugin) cmdDel(payload string) (string, error) { default: return "", backend.PluginUserError{ PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), + ErrorCode: int(comments.ErrorStatusStateInvalid), } } @@ -1159,6 +1159,12 @@ func (p *commentsPlugin) cmdDel(payload string) (string, error) { // Get the existing comment cs, err := p.comments(d.State, token, *idx, []uint32{d.CommentID}) if err != nil { + if errors.Is(err, errRecordNotFound) { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusRecordNotFound), + } + } return "", fmt.Errorf("comments %v: %v", d.CommentID, err) } existing, ok := cs[d.CommentID] @@ -1187,12 +1193,6 @@ func (p *commentsPlugin) cmdDel(payload string) (string, error) { // Save comment del merkle, err := p.commentDelSave(cd) if err != nil { - if errors.Is(err, errRecordNotFound) { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusRecordNotFound), - } - } return "", fmt.Errorf("commentDelSave: %v", err) } @@ -1377,6 +1377,12 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { // Verify user is not voting on their own comment cs, err := p.comments(v.State, token, *idx, []uint32{v.CommentID}) if err != nil { + if errors.Is(err, errRecordNotFound) { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusRecordNotFound), + } + } return "", fmt.Errorf("comments %v: %v", v.CommentID, err) } c, ok := cs[v.CommentID] @@ -1408,12 +1414,6 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { // Save comment vote merkle, err := p.commentVoteSave(cv) if err != nil { - if errors.Is(err, errRecordNotFound) { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusRecordNotFound), - } - } return "", fmt.Errorf("commentVoteSave: %v", err) } @@ -1654,6 +1654,12 @@ func (p *commentsPlugin) cmdGetVersion(payload string) (string, error) { // Get comment add record adds, err := p.commentAdds(gv.State, token, [][]byte{merkle}) if err != nil { + if errors.Is(err, errRecordNotFound) { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusRecordNotFound), + } + } return "", fmt.Errorf("commentAdds: %v", err) } if len(adds) != 1 { diff --git a/politeiad/backend/tlogbe/comments_test.go b/politeiad/backend/tlogbe/comments_test.go index 6bbc0e0c7..3fe062807 100644 --- a/politeiad/backend/tlogbe/comments_test.go +++ b/politeiad/backend/tlogbe/comments_test.go @@ -61,7 +61,7 @@ func TestCmdNew(t *testing.T) { var tests = []struct { description string payload comments.New - wantErr *backend.PluginUserError + wantErr error }{ { "invalid comment state", @@ -75,7 +75,7 @@ func TestCmdNew(t *testing.T) { Signature: commentSignature(t, uid, comments.StateInvalid, rec.Token, comment, parentID), }, - &backend.PluginUserError{ + backend.PluginUserError{ ErrorCode: int(comments.ErrorStatusStateInvalid), }, }, @@ -91,7 +91,7 @@ func TestCmdNew(t *testing.T) { Signature: commentSignature(t, uid, comments.StateUnvetted, rec.Token, comment, parentID), }, - &backend.PluginUserError{ + backend.PluginUserError{ ErrorCode: int(comments.ErrorStatusTokenInvalid), }, }, @@ -106,7 +106,7 @@ func TestCmdNew(t *testing.T) { PublicKey: uid.Public.String(), Signature: "invalid", }, - &backend.PluginUserError{ + backend.PluginUserError{ ErrorCode: int(comments.ErrorStatusSignatureInvalid), }, }, @@ -122,7 +122,7 @@ func TestCmdNew(t *testing.T) { Signature: commentSignature(t, uid, comments.StateUnvetted, rec.Token, comment, parentID), }, - &backend.PluginUserError{ + backend.PluginUserError{ ErrorCode: int(comments.ErrorStatusPublicKeyInvalid), }, }, @@ -138,7 +138,7 @@ func TestCmdNew(t *testing.T) { Signature: commentSignature(t, uid, comments.StateUnvetted, rec.Token, commentMaxLengthExceeded(t), parentID), }, - &backend.PluginUserError{ + backend.PluginUserError{ ErrorCode: int(comments.ErrorStatusCommentTextInvalid), }, }, @@ -154,7 +154,7 @@ func TestCmdNew(t *testing.T) { Signature: commentSignature(t, uid, comments.StateUnvetted, rec.Token, comment, invalidParentID), }, - &backend.PluginUserError{ + backend.PluginUserError{ ErrorCode: int(comments.ErrorStatusParentIDInvalid), }, }, @@ -170,7 +170,7 @@ func TestCmdNew(t *testing.T) { Signature: commentSignature(t, uid, comments.StateUnvetted, tokenRandom, comment, parentID), }, - &backend.PluginUserError{ + backend.PluginUserError{ ErrorCode: int(comments.ErrorStatusRecordNotFound), }, }, @@ -208,17 +208,18 @@ func TestCmdNew(t *testing.T) { t.Errorf("got error %v, want nil", err) return } - if pluginUserError.ErrorCode != test.wantErr.ErrorCode { + wantErr := test.wantErr.(backend.PluginUserError) + if pluginUserError.ErrorCode != wantErr.ErrorCode { t.Errorf("got error %v, want %v", pluginUserError.ErrorCode, - test.wantErr.ErrorCode) + wantErr.ErrorCode) } return } - // Expecting nil err - if err != nil { - t.Errorf("got error %v, want nil", err) + // Expectations not met + if err != test.wantErr { + t.Errorf("got error %v, want %v", err, test.wantErr) } }) } @@ -282,7 +283,7 @@ func TestCmdEdit(t *testing.T) { var tests = []struct { description string payload comments.Edit - wantErr *backend.PluginUserError + wantErr error }{ { "invalid comment state", @@ -297,7 +298,7 @@ func TestCmdEdit(t *testing.T) { Signature: commentSignature(t, id, comments.StateInvalid, rec.Token, commentEdit, nr.Comment.ParentID), }, - &backend.PluginUserError{ + backend.PluginUserError{ ErrorCode: int(comments.ErrorStatusStateInvalid), }, }, @@ -314,7 +315,7 @@ func TestCmdEdit(t *testing.T) { Signature: commentSignature(t, id, nr.Comment.State, "invalid", commentEdit, nr.Comment.ParentID), }, - &backend.PluginUserError{ + backend.PluginUserError{ ErrorCode: int(comments.ErrorStatusTokenInvalid), }, }, @@ -330,7 +331,7 @@ func TestCmdEdit(t *testing.T) { PublicKey: id.Public.String(), Signature: "invalid", }, - &backend.PluginUserError{ + backend.PluginUserError{ ErrorCode: int(comments.ErrorStatusSignatureInvalid), }, }, @@ -347,7 +348,7 @@ func TestCmdEdit(t *testing.T) { Signature: commentSignature(t, id, nr.Comment.State, rec.Token, commentEdit, nr.Comment.ParentID), }, - &backend.PluginUserError{ + backend.PluginUserError{ ErrorCode: int(comments.ErrorStatusPublicKeyInvalid), }, }, @@ -364,7 +365,7 @@ func TestCmdEdit(t *testing.T) { Signature: commentSignature(t, id, nr.Comment.State, rec.Token, commentMaxLengthExceeded(t), nr.Comment.ParentID), }, - &backend.PluginUserError{ + backend.PluginUserError{ ErrorCode: int(comments.ErrorStatusCommentTextInvalid), }, }, @@ -381,7 +382,7 @@ func TestCmdEdit(t *testing.T) { Signature: commentSignature(t, id, nr.Comment.State, rec.Token, commentEdit, nr.Comment.ParentID), }, - &backend.PluginUserError{ + backend.PluginUserError{ ErrorCode: int(comments.ErrorStatusCommentNotFound), }, }, @@ -398,7 +399,7 @@ func TestCmdEdit(t *testing.T) { Signature: commentSignature(t, id, nr.Comment.State, rec.Token, commentEdit, nr.Comment.ParentID), }, - &backend.PluginUserError{ + backend.PluginUserError{ ErrorCode: int(comments.ErrorStatusUserUnauthorized), }, }, @@ -415,7 +416,7 @@ func TestCmdEdit(t *testing.T) { Signature: commentSignature(t, id, nr.Comment.State, rec.Token, commentEdit, invalidParentID), }, - &backend.PluginUserError{ + backend.PluginUserError{ ErrorCode: int(comments.ErrorStatusParentIDInvalid), }, }, @@ -432,7 +433,7 @@ func TestCmdEdit(t *testing.T) { Signature: commentSignature(t, id, nr.Comment.State, rec.Token, comment, nr.Comment.ParentID), }, - &backend.PluginUserError{ + backend.PluginUserError{ ErrorCode: int(comments.ErrorStatusCommentTextInvalid), }, }, @@ -449,7 +450,7 @@ func TestCmdEdit(t *testing.T) { Signature: commentSignature(t, id, nr.Comment.State, tokenRandom, commentEdit, nr.Comment.ParentID), }, - &backend.PluginUserError{ + backend.PluginUserError{ ErrorCode: int(comments.ErrorStatusRecordNotFound), }, }, @@ -488,18 +489,216 @@ func TestCmdEdit(t *testing.T) { t.Errorf("got error %v, want nil", err) return } - if pluginUserError.ErrorCode != test.wantErr.ErrorCode { + wantErr := test.wantErr.(backend.PluginUserError) + if pluginUserError.ErrorCode != wantErr.ErrorCode { t.Errorf("got error %v, want %v", pluginUserError.ErrorCode, - test.wantErr.ErrorCode) + wantErr.ErrorCode) } return } - // Expecting nil err + // Expectations not met + if err != test.wantErr { + t.Errorf("got error %v, want %v", err, test.wantErr) + } + }) + } +} + +func TestCmdDel(t *testing.T) { + commentsPlugin, tlogBackend, cleanup := newTestCommentsPlugin(t) + defer cleanup() + + // New record + md := []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs := []backend.File{ + newBackendFile(t, "index.md"), + } + rec, err := tlogBackend.New(md, fs) + if err != nil { + t.Fatal(err) + } + + // Helpers + comment := "random comment" + reason := "random reason" + parentID := uint32(0) + tokenRandom := hex.EncodeToString(tokenFromTreeID(123)) + id, err := identity.New() + if err != nil { + t.Fatal(err) + } + + // New comment + ncEncoded, err := comments.EncodeNew( + comments.New{ + UserID: uuid.New().String(), + State: comments.StateUnvetted, + Token: rec.Token, + ParentID: parentID, + Comment: comment, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, comments.StateUnvetted, + rec.Token, comment, parentID), + }, + ) + if err != nil { + t.Fatal(err) + } + reply, err := commentsPlugin.cmdNew(string(ncEncoded)) + if err != nil { + t.Fatal(err) + } + nr, err := comments.DecodeNewReply([]byte(reply)) + if err != nil { + t.Fatal(err) + } + + // Setup del comment plugin tests + var tests = []struct { + description string + payload comments.Del + wantErr error + }{ + { + "invalid comment state", + comments.Del{ + State: comments.StateInvalid, + Token: rec.Token, + CommentID: nr.Comment.CommentID, + Reason: reason, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, comments.StateInvalid, + rec.Token, reason, nr.Comment.CommentID), + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusStateInvalid), + }, + }, + { + "invalid token", + comments.Del{ + State: comments.StateUnvetted, + Token: "invalid", + CommentID: nr.Comment.CommentID, + Reason: reason, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, comments.StateUnvetted, + "invalid", reason, nr.Comment.CommentID), + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusTokenInvalid), + }, + }, + { + "invalid signature", + comments.Del{ + State: comments.StateUnvetted, + Token: rec.Token, + CommentID: nr.Comment.CommentID, + Reason: reason, + PublicKey: id.Public.String(), + Signature: "invalid", + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusSignatureInvalid), + }, + }, + { + "invalid public key", + comments.Del{ + State: comments.StateUnvetted, + Token: rec.Token, + CommentID: nr.Comment.CommentID, + Reason: reason, + PublicKey: "invalid", + Signature: commentSignature(t, id, comments.StateUnvetted, + rec.Token, reason, nr.Comment.CommentID), + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusPublicKeyInvalid), + }, + }, + { + "record not found", + comments.Del{ + State: comments.StateUnvetted, + Token: tokenRandom, + CommentID: nr.Comment.CommentID, + Reason: reason, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, comments.StateUnvetted, + tokenRandom, reason, nr.Comment.CommentID), + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusRecordNotFound), + }, + }, + { + "comment id not found", + comments.Del{ + State: comments.StateUnvetted, + Token: rec.Token, + CommentID: 3, + Reason: reason, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, comments.StateUnvetted, + rec.Token, reason, 3), + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusCommentNotFound), + }, + }, + { + "success", + comments.Del{ + State: comments.StateUnvetted, + Token: rec.Token, + CommentID: nr.Comment.CommentID, + Reason: reason, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, comments.StateUnvetted, + rec.Token, reason, nr.Comment.CommentID), + }, + nil, + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + // Del Comment + dcEncoded, err := comments.EncodeDel(test.payload) if err != nil { - t.Errorf("got error %v, want nil", err) + t.Error(err) + } + + // Execute plugin command + _, err = commentsPlugin.cmdDel(string(dcEncoded)) + + // Parse plugin user error + var pluginUserError backend.PluginUserError + if errors.As(err, &pluginUserError) { + if test.wantErr == nil { + t.Errorf("got error %v, want nil", err) + return + } + wantErr := test.wantErr.(backend.PluginUserError) + if pluginUserError.ErrorCode != wantErr.ErrorCode { + t.Errorf("got error %v, want %v", + pluginUserError.ErrorCode, + wantErr.ErrorCode) + } + return + } + + // Expectations not met + if err != test.wantErr { + t.Errorf("got error %v, want %v", err, test.wantErr) } }) } + } diff --git a/politeiad/backend/tlogbe/pi.go b/politeiad/backend/tlogbe/pi.go index 5f338c80a..faf357dcc 100644 --- a/politeiad/backend/tlogbe/pi.go +++ b/politeiad/backend/tlogbe/pi.go @@ -718,7 +718,7 @@ func (p *piPlugin) commentDel(payload string) (string, error) { switch vs.Status { case ticketvote.VoteStatusUnauthorized, ticketvote.VoteStatusAuthorized, ticketvote.VoteStatusStarted: - // Deling is allowed on these vote statuses; continue + // Deleting is allowed on these vote statuses; continue default: return "", backend.PluginUserError{ PluginID: pi.ID, diff --git a/politeiad/backend/tlogbe/pi_test.go b/politeiad/backend/tlogbe/pi_test.go index abda31a14..930cb5e6a 100644 --- a/politeiad/backend/tlogbe/pi_test.go +++ b/politeiad/backend/tlogbe/pi_test.go @@ -61,7 +61,7 @@ func TestCommentNew(t *testing.T) { var tests = []struct { description string payload comments.New - wantErr *backend.PluginUserError + wantErr error }{ { "invalid comment state", @@ -75,7 +75,7 @@ func TestCommentNew(t *testing.T) { Signature: commentSignature(t, uid, comments.StateInvalid, rec.Token, comment, parentID), }, - &backend.PluginUserError{ + backend.PluginUserError{ ErrorCode: int(pi.ErrorStatusPropStateInvalid), }, }, @@ -91,7 +91,7 @@ func TestCommentNew(t *testing.T) { Signature: commentSignature(t, uid, comments.StateUnvetted, rec.Token, comment, parentID), }, - &backend.PluginUserError{ + backend.PluginUserError{ ErrorCode: int(pi.ErrorStatusPropTokenInvalid), }, }, @@ -107,7 +107,7 @@ func TestCommentNew(t *testing.T) { Signature: commentSignature(t, uid, comments.StateUnvetted, tokenRandom, comment, parentID), }, - &backend.PluginUserError{ + backend.PluginUserError{ ErrorCode: int(pi.ErrorStatusPropNotFound), }, }, @@ -147,19 +147,188 @@ func TestCommentNew(t *testing.T) { t.Errorf("got error %v, want nil2", err) return } - if pluginUserError.ErrorCode != test.wantErr.ErrorCode { + wantErr := test.wantErr.(backend.PluginUserError) + if pluginUserError.ErrorCode != wantErr.ErrorCode { t.Errorf("got error %v, want %v", pluginUserError.ErrorCode, - test.wantErr.ErrorCode) + wantErr.ErrorCode) } - return } - // Expecting nil err + // Expectations not met + if err != test.wantErr { + t.Errorf("got error %v, want %v", err, test.wantErr) + } + }) + } +} + +func TestCommentDel(t *testing.T) { + piPlugin, tlogBackend, cleanup := newTestPiPlugin(t) + defer cleanup() + + // Register comments plugin + id, err := identity.New() + if err != nil { + t.Fatal(err) + } + settings := []backend.PluginSetting{{ + Key: pluginSettingDataDir, + Value: tlogBackend.dataDir, + }} + tlogBackend.RegisterPlugin(backend.Plugin{ + ID: comments.ID, + Settings: settings, + Identity: id, + }) + + // New record + md := []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs := []backend.File{ + newBackendFile(t, "index.md"), + } + rec, err := tlogBackend.New(md, fs) + if err != nil { + t.Fatal(err) + } + + // Helpers + comment := "random comment" + reason := "random reason" + tokenRandom := hex.EncodeToString(tokenFromTreeID(123)) + parentID := uint32(0) + + uid, err := identity.New() + if err != nil { + t.Fatal(err) + } + + // New comment + ncEncoded, err := comments.EncodeNew( + comments.New{ + UserID: uuid.New().String(), + State: comments.StateUnvetted, + Token: rec.Token, + ParentID: parentID, + Comment: comment, + PublicKey: uid.Public.String(), + Signature: commentSignature(t, uid, comments.StateUnvetted, + rec.Token, comment, parentID), + }, + ) + if err != nil { + t.Fatal(err) + } + reply, err := piPlugin.commentNew(string(ncEncoded)) + if err != nil { + t.Fatal(err) + } + nr, err := comments.DecodeNewReply([]byte(reply)) + if err != nil { + t.Fatal(err) + } + + // Setup comment del pi plugin tests + var tests = []struct { + description string + payload comments.Del + wantErr error + }{ + { + "invalid comment state", + comments.Del{ + State: comments.StateInvalid, + Token: rec.Token, + CommentID: nr.Comment.CommentID, + Reason: reason, + PublicKey: uid.Public.String(), + Signature: commentSignature(t, uid, comments.StateInvalid, + rec.Token, reason, nr.Comment.CommentID), + }, + backend.PluginUserError{ + ErrorCode: int(pi.ErrorStatusPropStateInvalid), + }, + }, + { + "invalid token", + comments.Del{ + State: comments.StateUnvetted, + Token: "invalid", + CommentID: nr.Comment.CommentID, + Reason: reason, + PublicKey: uid.Public.String(), + Signature: commentSignature(t, uid, comments.StateUnvetted, + "invalid", reason, nr.Comment.CommentID), + }, + backend.PluginUserError{ + ErrorCode: int(pi.ErrorStatusPropTokenInvalid), + }, + }, + { + "proposal not found", + comments.Del{ + State: comments.StateUnvetted, + Token: tokenRandom, + CommentID: nr.Comment.CommentID, + Reason: reason, + PublicKey: uid.Public.String(), + Signature: commentSignature(t, uid, comments.StateUnvetted, + tokenRandom, reason, nr.Comment.CommentID), + }, + backend.PluginUserError{ + ErrorCode: int(pi.ErrorStatusPropNotFound), + }, + }, + { + "success", + comments.Del{ + State: comments.StateUnvetted, + Token: rec.Token, + CommentID: nr.Comment.CommentID, + Reason: reason, + PublicKey: uid.Public.String(), + Signature: commentSignature(t, uid, comments.StateUnvetted, + rec.Token, reason, nr.Comment.CommentID), + }, + nil, + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + // New Comment + dcEncoded, err := comments.EncodeDel(test.payload) if err != nil { - t.Errorf("got error %v, want nil", err) + t.Error(err) + } + + // Execute plugin command + _, err = piPlugin.commentDel(string(dcEncoded)) + + // Parse plugin user error + var pluginUserError backend.PluginUserError + if errors.As(err, &pluginUserError) { + if test.wantErr == nil { + t.Errorf("got error %v, want nil", err) + return + } + wantErr := test.wantErr.(backend.PluginUserError) + if pluginUserError.ErrorCode != wantErr.ErrorCode { + t.Errorf("got error %v, want %v", + pluginUserError.ErrorCode, + wantErr.ErrorCode) + } + return + } + + // Expectations not met + if err != test.wantErr { + t.Errorf("got error %v, want %v", err, test.wantErr) } }) } + } diff --git a/politeiad/backend/tlogbe/testing.go b/politeiad/backend/tlogbe/testing.go index 731f4301e..5a214ffa8 100644 --- a/politeiad/backend/tlogbe/testing.go +++ b/politeiad/backend/tlogbe/testing.go @@ -111,13 +111,13 @@ func newBackendMetadataStream(t *testing.T, id uint64, payload string) backend.M } } -func commentSignature(t *testing.T, id *identity.FullIdentity, state comments.StateT, token, comment string, parentID uint32) string { +func commentSignature(t *testing.T, uid *identity.FullIdentity, state comments.StateT, token, msg string, id uint32) string { t.Helper() // Create signature - msg := strconv.Itoa(int(state)) + token + - strconv.FormatInt(int64(parentID), 10) + comment - b := id.SignMessage([]byte(msg)) + txt := strconv.Itoa(int(state)) + token + + strconv.FormatInt(int64(id), 10) + msg + b := uid.SignMessage([]byte(txt)) return hex.EncodeToString(b[:]) } diff --git a/politeiad/backend/tlogbe/tlogbe_test.go b/politeiad/backend/tlogbe/tlogbe_test.go index d876ae455..fd2cda9fb 100644 --- a/politeiad/backend/tlogbe/tlogbe_test.go +++ b/politeiad/backend/tlogbe/tlogbe_test.go @@ -130,7 +130,7 @@ func TestUpdateUnvettedRecord(t *testing.T) { mdAppend, mdOverwrite []backend.MetadataStream filesAdd []backend.File filesDel []string - wantContentErr *backend.ContentVerificationError + wantContentErr error wantErr error }{ { @@ -140,7 +140,7 @@ func TestUpdateUnvettedRecord(t *testing.T) { []backend.MetadataStream{}, []backend.File{imageRandom}, []string{}, - &backend.ContentVerificationError{ + backend.ContentVerificationError{ ErrorCode: v1.ErrorStatusInvalidToken, }, nil, @@ -200,14 +200,22 @@ func TestUpdateUnvettedRecord(t *testing.T) { t.Errorf("got error %v, want nil", err) return } - if contentError.ErrorCode != test.wantContentErr.ErrorCode { + wantContentErr := + test.wantContentErr.(backend.ContentVerificationError) + if contentError.ErrorCode != wantContentErr.ErrorCode { t.Errorf("got error %v, want %v", v1.ErrorStatus[contentError.ErrorCode], - v1.ErrorStatus[test.wantContentErr.ErrorCode]) + v1.ErrorStatus[wantContentErr.ErrorCode]) } return } + // Expecting content error, but got none + if test.wantContentErr != nil { + t.Errorf("got error %v, want %v", err, test.wantContentErr) + } + + // Expectations not met if test.wantErr != err { t.Errorf("got error %v, want %v", err, test.wantErr) } @@ -309,7 +317,7 @@ func TestUpdateVettedRecord(t *testing.T) { mdAppend, mdOverwirte []backend.MetadataStream filesAdd []backend.File filesDel []string - wantContentErr *backend.ContentVerificationError + wantContentErr error wantErr error }{ { @@ -319,7 +327,7 @@ func TestUpdateVettedRecord(t *testing.T) { []backend.MetadataStream{}, []backend.File{imageRandom}, []string{}, - &backend.ContentVerificationError{ + backend.ContentVerificationError{ ErrorCode: v1.ErrorStatusInvalidToken, }, nil, @@ -379,14 +387,22 @@ func TestUpdateVettedRecord(t *testing.T) { t.Errorf("got error %v, want nil", err) return } - if contentError.ErrorCode != test.wantContentErr.ErrorCode { + wantContentErr := + test.wantContentErr.(backend.ContentVerificationError) + if contentError.ErrorCode != wantContentErr.ErrorCode { t.Errorf("got error %v, want %v", v1.ErrorStatus[contentError.ErrorCode], - v1.ErrorStatus[test.wantContentErr.ErrorCode]) + v1.ErrorStatus[wantContentErr.ErrorCode]) } return } + // Expecting content error, but got none + if test.wantContentErr != nil { + t.Errorf("got error %v, want %v", err, test.wantContentErr) + } + + // Expectations not met if test.wantErr != err { t.Errorf("got error %v, want %v", err, test.wantErr) } @@ -464,7 +480,7 @@ func TestUpdateUnvettedMetadata(t *testing.T) { description string token []byte mdAppend, mdOverwrite []backend.MetadataStream - wantContentErr *backend.ContentVerificationError + wantContentErr error wantErr error }{ { @@ -472,7 +488,7 @@ func TestUpdateUnvettedMetadata(t *testing.T) { token, []backend.MetadataStream{}, []backend.MetadataStream{}, - &backend.ContentVerificationError{ + backend.ContentVerificationError{ ErrorCode: v1.ErrorStatusNoChanges, }, nil, @@ -485,7 +501,7 @@ func TestUpdateUnvettedMetadata(t *testing.T) { Payload: "random", }}, []backend.MetadataStream{}, - &backend.ContentVerificationError{ + backend.ContentVerificationError{ ErrorCode: v1.ErrorStatusInvalidToken, }, nil, @@ -549,14 +565,22 @@ func TestUpdateUnvettedMetadata(t *testing.T) { t.Errorf("got error %v, want nil", err) return } - if contentError.ErrorCode != test.wantContentErr.ErrorCode { + wantContentErr := + test.wantContentErr.(backend.ContentVerificationError) + if contentError.ErrorCode != wantContentErr.ErrorCode { t.Errorf("got error %v, want %v", v1.ErrorStatus[contentError.ErrorCode], - v1.ErrorStatus[test.wantContentErr.ErrorCode]) + v1.ErrorStatus[wantContentErr.ErrorCode]) } return } + // Expecting content error, but got none + if test.wantContentErr != nil { + t.Errorf("got error %v, want %v", err, test.wantContentErr) + } + + // Expectations not met if test.wantErr != err { t.Errorf("got error %v, want %v", err, test.wantErr) } @@ -645,7 +669,7 @@ func TestUpdateVettedMetadata(t *testing.T) { description string token []byte mdAppend, mdOverwrite []backend.MetadataStream - wantContentErr *backend.ContentVerificationError + wantContentErr error wantErr error }{ { @@ -653,7 +677,7 @@ func TestUpdateVettedMetadata(t *testing.T) { token, []backend.MetadataStream{}, []backend.MetadataStream{}, - &backend.ContentVerificationError{ + backend.ContentVerificationError{ ErrorCode: v1.ErrorStatusNoChanges, }, nil, @@ -665,7 +689,7 @@ func TestUpdateVettedMetadata(t *testing.T) { newBackendMetadataStream(t, 2, "random"), }, []backend.MetadataStream{}, - &backend.ContentVerificationError{ + backend.ContentVerificationError{ ErrorCode: v1.ErrorStatusInvalidToken, }, nil, @@ -725,14 +749,22 @@ func TestUpdateVettedMetadata(t *testing.T) { t.Errorf("got error %v, want nil", err) return } - if contentError.ErrorCode != test.wantContentErr.ErrorCode { + wantContentErr := + test.wantContentErr.(backend.ContentVerificationError) + if contentError.ErrorCode != wantContentErr.ErrorCode { t.Errorf("got error %v, want %v", v1.ErrorStatus[contentError.ErrorCode], - v1.ErrorStatus[test.wantContentErr.ErrorCode]) + v1.ErrorStatus[wantContentErr.ErrorCode]) } return } + // Expecting content error, but got none + if test.wantContentErr != nil { + t.Errorf("got error %v, want %v", err, test.wantContentErr) + } + + // Expectations not met if test.wantErr != err { t.Errorf("got error %v, want %v", err, test.wantErr) } @@ -985,7 +1017,7 @@ func TestSetUnvettedStatus(t *testing.T) { token []byte status backend.MDStatusT mdAppend, mdOverwrite []backend.MetadataStream - wantContentErr *backend.ContentVerificationError + wantContentErr error wantErr error }{ { @@ -1036,7 +1068,7 @@ func TestSetUnvettedStatus(t *testing.T) { backend.MDStatusCensored, []backend.MetadataStream{}, []backend.MetadataStream{}, - &backend.ContentVerificationError{ + backend.ContentVerificationError{ ErrorCode: v1.ErrorStatusInvalidToken, }, nil, @@ -1065,14 +1097,22 @@ func TestSetUnvettedStatus(t *testing.T) { t.Errorf("got error %v, want nil", err) return } - if contentError.ErrorCode != test.wantContentErr.ErrorCode { + wantContentErr := + test.wantContentErr.(backend.ContentVerificationError) + if contentError.ErrorCode != wantContentErr.ErrorCode { t.Errorf("got error %v, want %v", v1.ErrorStatus[contentError.ErrorCode], - v1.ErrorStatus[test.wantContentErr.ErrorCode]) + v1.ErrorStatus[wantContentErr.ErrorCode]) } return } + // Expecting content error, but got none + if test.wantContentErr != nil { + t.Errorf("got error %v, want %v", err, test.wantContentErr) + } + + // Expectations not met if test.wantErr != err { t.Errorf("got error %v, want %v", err, test.wantErr) } @@ -1175,7 +1215,7 @@ func TestSetVettedStatus(t *testing.T) { token []byte status backend.MDStatusT mdAppend, mdOverwrite []backend.MetadataStream - wantContentErr *backend.ContentVerificationError + wantContentErr error wantErr error }{ { @@ -1226,7 +1266,7 @@ func TestSetVettedStatus(t *testing.T) { backend.MDStatusCensored, []backend.MetadataStream{}, []backend.MetadataStream{}, - &backend.ContentVerificationError{ + backend.ContentVerificationError{ ErrorCode: v1.ErrorStatusInvalidToken, }, nil, @@ -1255,14 +1295,22 @@ func TestSetVettedStatus(t *testing.T) { t.Errorf("got error %v, want nil", err) return } - if contentError.ErrorCode != test.wantContentErr.ErrorCode { + wantContentErr := + test.wantContentErr.(backend.ContentVerificationError) + if contentError.ErrorCode != wantContentErr.ErrorCode { t.Errorf("got error %v, want %v", v1.ErrorStatus[contentError.ErrorCode], - v1.ErrorStatus[test.wantContentErr.ErrorCode]) + v1.ErrorStatus[wantContentErr.ErrorCode]) } return } + // Expecting content error, but got none + if test.wantContentErr != nil { + t.Errorf("got error %v, want %v", err, test.wantContentErr) + } + + // Expectations not met if test.wantErr != err { t.Errorf("got error %v, want %v", err, test.wantErr) } From 3dd1ca07942dc7191c568e20d415297a9a95ee52 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 12 Jan 2021 18:23:37 -0600 Subject: [PATCH 213/449] Move dcrtime calls from util to gitbe. --- {util => politeiad/backend/gitbe}/dcrtime.go | 41 +++++++------------- politeiad/backend/gitbe/gitbe.go | 6 +-- 2 files changed, 18 insertions(+), 29 deletions(-) rename {util => politeiad/backend/gitbe}/dcrtime.go (82%) diff --git a/util/dcrtime.go b/politeiad/backend/gitbe/dcrtime.go similarity index 82% rename from util/dcrtime.go rename to politeiad/backend/gitbe/dcrtime.go index f586e8083..d23318e74 100644 --- a/util/dcrtime.go +++ b/politeiad/backend/gitbe/dcrtime.go @@ -1,8 +1,8 @@ -// Copyright (c) 2017-2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package util +package gitbe import ( "bytes" @@ -18,6 +18,7 @@ import ( v1 "github.com/decred/dcrtime/api/v1" "github.com/decred/dcrtime/merkle" + "github.com/decred/politeia/util" ) var ( @@ -34,11 +35,11 @@ var ( } ) -type ErrNotAnchored struct { +type errNotAnchored struct { err error } -func (e ErrNotAnchored) Error() string { +func (e errNotAnchored) Error() string { return e.err.Error() } @@ -48,11 +49,10 @@ func isDigest(digest string) bool { } func defaultTestnetHost() string { - return "https://" + NormalizeAddress(v1.DefaultTestnetTimeHost, + return "https://" + util.NormalizeAddress(v1.DefaultTestnetTimeHost, v1.DefaultTestnetTimePort) } -// XXX duplicate function // getError returns the error that is embedded in a JSON reply. func getError(r io.Reader) (string, error) { var e interface{} @@ -71,20 +71,9 @@ func getError(r io.Reader) (string, error) { return fmt.Sprintf("%v", rError), nil } -// Hash returns a pointer to the sha256 hash of data. -func Hash(data []byte) *[sha256.Size]byte { - h := sha256.New() - h.Write(data) - hash := h.Sum(nil) - - var rh [sha256.Size]byte - copy(rh[:], hash) - return &rh -} - -// Timestamp sends a Timestamp request to the provided host. The caller is +// timestamp sends a Timestamp request to the provided host. The caller is // responsible for assembling the host string based on what net to use. -func Timestamp(id, host string, digests []*[sha256.Size]byte) error { +func timestamp(id, host string, digests []*[sha256.Size]byte) error { // batch uploads ts := v1.Timestamp{ ID: id, @@ -137,18 +126,18 @@ func Timestamp(id, host string, digests []*[sha256.Size]byte) error { return nil } -// Verify sends a dcrtime Verify command to the provided host. It checks and -// validates the entire reply. A single failure is considered terminal and an -// error is returned. If the reply is valid it is returned to the caller for -// further processing. This means that the caller can be assured that all -// checks have been done and the data is readily usable. +// verifyTimestamp sends a dcrtime Verify command to the provided host. It +// checks and validates the entire reply. A single failure is considered +// terminal and an error is returned. If the reply is valid it is returned to +// the caller for further processing. This means that the caller can be +// assured that all checks have been done and the data is readily usable. // // Note the Result in the reply will be set to OK as soon as the digest is // waiting to be anchored. The ChainInformation will be populated once the // digest has been included in a dcr transaction, except for the ChainTimestamp // field. The ChainTimestamp field is only populated once the dcr transaction // has 6 confirmations. -func Verify(id, host string, digests []string) (*v1.VerifyReply, error) { +func verifyTimestamp(id, host string, digests []string) (*v1.VerifyReply, error) { ver := v1.Verify{ ID: id, } @@ -213,7 +202,7 @@ func Verify(id, host string, digests []string) (*v1.VerifyReply, error) { return nil, fmt.Errorf("%v invalid auth path "+ "%v", v.Digest, err) } - return nil, ErrNotAnchored{ + return nil, errNotAnchored{ err: fmt.Errorf("%v Not anchored", v.Digest), } } diff --git a/politeiad/backend/gitbe/gitbe.go b/politeiad/backend/gitbe/gitbe.go index 4f1828c3b..15b67c135 100644 --- a/politeiad/backend/gitbe/gitbe.go +++ b/politeiad/backend/gitbe/gitbe.go @@ -682,7 +682,7 @@ func (g *gitBackEnd) anchor(digests []*[sha256.Size]byte) error { return nil } - return util.Timestamp("politeia", g.dcrtimeHost, digests) + return timestamp("politeia", g.dcrtimeHost, digests) } // appendAuditTrail adds a record to the audit trail. @@ -1063,7 +1063,7 @@ func (g *gitBackEnd) verifyAnchor(digest string) (*v1.VerifyDigest, error) { }) } else { // Call dcrtime - vr, err = util.Verify("politeia", g.dcrtimeHost, + vr, err = verifyTimestamp("politeia", g.dcrtimeHost, []string{digest}) if err != nil { return nil, err @@ -2287,7 +2287,7 @@ func (g *gitBackEnd) fsck(path string) error { for d := range gitDigests { digests = append(digests, d) } - vr, err := util.Verify("politeia", g.dcrtimeHost, digests) + vr, err := verifyTimestamp("politeia", g.dcrtimeHost, digests) if err != nil { return err } From 0f40b5482da780928a0ba21c9fdb13ee27c3a030 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 12 Jan 2021 18:37:02 -0600 Subject: [PATCH 214/449] tlogbe: Standardize token decoding. --- politeiad/backend/tlogbe/comments.go | 20 ++++++++++---------- politeiad/backend/tlogbe/pi.go | 9 ++++----- politeiad/backend/tlogbe/tlogbe_test.go | 16 ++++++++-------- util/convert.go | 8 ++++---- 4 files changed, 26 insertions(+), 27 deletions(-) diff --git a/politeiad/backend/tlogbe/comments.go b/politeiad/backend/tlogbe/comments.go index 8a0ce10ff..da36544fd 100644 --- a/politeiad/backend/tlogbe/comments.go +++ b/politeiad/backend/tlogbe/comments.go @@ -839,7 +839,7 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { } // Verify token - token, err := util.ConvertStringToken(n.Token) + token, err := tokenDecodeAnyLength(n.Token) if err != nil { return "", backend.PluginUserError{ PluginID: comments.ID, @@ -971,7 +971,7 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { } // Verify token - token, err := util.ConvertStringToken(e.Token) + token, err := tokenDecodeAnyLength(e.Token) if err != nil { return "", backend.PluginUserError{ PluginID: comments.ID, @@ -1128,7 +1128,7 @@ func (p *commentsPlugin) cmdDel(payload string) (string, error) { } // Verify token - token, err := util.ConvertStringToken(d.Token) + token, err := tokenDecodeAnyLength(d.Token) if err != nil { return "", backend.PluginUserError{ PluginID: comments.ID, @@ -1313,7 +1313,7 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { } // Verify token - token, err := util.ConvertStringToken(v.Token) + token, err := tokenDecodeAnyLength(v.Token) if err != nil { return "", backend.PluginUserError{ PluginID: comments.ID, @@ -1476,7 +1476,7 @@ func (p *commentsPlugin) cmdGet(payload string) (string, error) { } // Verify token - token, err := util.ConvertStringToken(g.Token) + token, err := tokenDecodeAnyLength(g.Token) if err != nil { return "", backend.PluginUserError{ PluginID: comments.ID, @@ -1535,7 +1535,7 @@ func (p *commentsPlugin) cmdGetAll(payload string) (string, error) { } // Verify token - token, err := util.ConvertStringToken(ga.Token) + token, err := tokenDecodeAnyLength(ga.Token) if err != nil { return "", backend.PluginUserError{ PluginID: comments.ID, @@ -1611,7 +1611,7 @@ func (p *commentsPlugin) cmdGetVersion(payload string) (string, error) { } // Verify token - token, err := util.ConvertStringToken(gv.Token) + token, err := tokenDecodeAnyLength(gv.Token) if err != nil { return "", backend.PluginUserError{ PluginID: comments.ID, @@ -1704,7 +1704,7 @@ func (p *commentsPlugin) cmdCount(payload string) (string, error) { } // Verify token - token, err := util.ConvertStringToken(c.Token) + token, err := tokenDecodeAnyLength(c.Token) if err != nil { return "", backend.PluginUserError{ PluginID: comments.ID, @@ -1751,7 +1751,7 @@ func (p *commentsPlugin) cmdVotes(payload string) (string, error) { } // Verify token - token, err := util.ConvertStringToken(v.Token) + token, err := tokenDecodeAnyLength(v.Token) if err != nil { return "", backend.PluginUserError{ PluginID: comments.ID, @@ -1827,7 +1827,7 @@ func (p *commentsPlugin) cmdTimestamps(payload string) (string, error) { } // Verify token - token, err := util.ConvertStringToken(t.Token) + token, err := tokenDecodeAnyLength(t.Token) if err != nil { return "", backend.PluginUserError{ PluginID: comments.ID, diff --git a/politeiad/backend/tlogbe/pi.go b/politeiad/backend/tlogbe/pi.go index faf357dcc..cf1af3873 100644 --- a/politeiad/backend/tlogbe/pi.go +++ b/politeiad/backend/tlogbe/pi.go @@ -22,7 +22,6 @@ import ( "github.com/decred/politeia/politeiad/plugins/comments" "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/ticketvote" - "github.com/decred/politeia/util" ) const ( @@ -330,7 +329,7 @@ func (p *piPlugin) cmdProposals(payload string) (string, error) { // map[token]ProposalPluginData proposals := make(map[string]pi.ProposalPluginData, len(ps.Tokens)) for _, v := range ps.Tokens { - token, err := util.ConvertStringToken(v) + token, err := tokenDecodeAnyLength(v) if err != nil { // Not a valid token continue @@ -612,7 +611,7 @@ func (p *piPlugin) commentNew(payload string) (string, error) { } // Verify token - token, err := util.ConvertStringToken(n.Token) + token, err := tokenDecodeAnyLength(n.Token) if err != nil { return "", backend.PluginUserError{ PluginID: pi.ID, @@ -683,7 +682,7 @@ func (p *piPlugin) commentDel(payload string) (string, error) { } // Verify token - token, err := util.ConvertStringToken(d.Token) + token, err := tokenDecodeAnyLength(d.Token) if err != nil { return "", backend.PluginUserError{ PluginID: pi.ID, @@ -754,7 +753,7 @@ func (p *piPlugin) commentVote(payload string) (string, error) { } // Verify token - token, err := util.ConvertStringToken(v.Token) + token, err := tokenDecodeAnyLength(v.Token) if err != nil { return "", backend.PluginUserError{ PluginID: pi.ID, diff --git a/politeiad/backend/tlogbe/tlogbe_test.go b/politeiad/backend/tlogbe/tlogbe_test.go index fd2cda9fb..49ac59afe 100644 --- a/politeiad/backend/tlogbe/tlogbe_test.go +++ b/politeiad/backend/tlogbe/tlogbe_test.go @@ -75,7 +75,7 @@ func TestUpdateUnvettedRecord(t *testing.T) { for _, test := range recordContentTests { t.Run(test.description, func(t *testing.T) { // Convert token - token, err := util.ConvertStringToken(rec.Token) + token, err := tokenDecodeAnyLength(rec.Token) if err != nil { t.Error(err) } @@ -100,7 +100,7 @@ func TestUpdateUnvettedRecord(t *testing.T) { imageRandom := newBackendFilePNG(t) // test case: Token not full length - tokenShort, err := util.ConvertStringToken(util.TokenToPrefix(rec.Token)) + tokenShort, err := tokenDecodeAnyLength(util.TokenToPrefix(rec.Token)) if err != nil { t.Fatal(err) } @@ -256,7 +256,7 @@ func TestUpdateVettedRecord(t *testing.T) { for _, test := range recordContentTests { t.Run(test.description, func(t *testing.T) { // Convert token - token, err := util.ConvertStringToken(rec.Token) + token, err := tokenDecodeAnyLength(rec.Token) if err != nil { t.Error(err) } @@ -281,7 +281,7 @@ func TestUpdateVettedRecord(t *testing.T) { imageRandom := newBackendFilePNG(t) // test case: Token not full length - tokenShort, err := util.ConvertStringToken(util.TokenToPrefix(rec.Token)) + tokenShort, err := tokenDecodeAnyLength(util.TokenToPrefix(rec.Token)) if err != nil { t.Error(err) } @@ -452,7 +452,7 @@ func TestUpdateUnvettedMetadata(t *testing.T) { } // test case: Token not full length - tokenShort, err := util.ConvertStringToken(util.TokenToPrefix(rec.Token)) + tokenShort, err := tokenDecodeAnyLength(util.TokenToPrefix(rec.Token)) if err != nil { t.Fatal(err) } @@ -635,7 +635,7 @@ func TestUpdateVettedMetadata(t *testing.T) { } // test case: Token not full length - tokenShort, err := util.ConvertStringToken(util.TokenToPrefix(rec.Token)) + tokenShort, err := tokenDecodeAnyLength(util.TokenToPrefix(rec.Token)) if err != nil { t.Fatal(err) } @@ -1002,7 +1002,7 @@ func TestSetUnvettedStatus(t *testing.T) { } // test case: Token not full length - tokenShort, err := util.ConvertStringToken( + tokenShort, err := tokenDecodeAnyLength( util.TokenToPrefix(recUnvetToVet.Token)) if err != nil { t.Fatal(err) @@ -1200,7 +1200,7 @@ func TestSetVettedStatus(t *testing.T) { } // test case: Token not full length - tokenShort, err := util.ConvertStringToken( + tokenShort, err := tokenDecodeAnyLength( util.TokenToPrefix(recVetToCensored.Token)) if err != nil { t.Fatal(err) diff --git a/util/convert.go b/util/convert.go index bbaecbf9a..0be701d3a 100644 --- a/util/convert.go +++ b/util/convert.go @@ -9,7 +9,7 @@ import ( "encoding/hex" "fmt" - v1 "github.com/decred/dcrtime/api/v1" + dcrtime "github.com/decred/dcrtime/api/v1" pd "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/identity" ) @@ -31,8 +31,8 @@ func ConvertSignature(s string) ([identity.SignatureSize]byte, error) { } // ConvertStringToken verifies and converts a string token to a proper sized -// byte slice. This function accepts both the full length token and the token -// prefix. +// byte slice. This function accepts both the full length token and token +// prefixes. func ConvertStringToken(token string) ([]byte, error) { switch { case len(token) == pd.TokenSizeTlog*2: @@ -66,7 +66,7 @@ func Digest(b []byte) []byte { // IsDigest determines if a string is a valid SHA256 digest. func IsDigest(digest string) bool { - return v1.RegexpSHA256.MatchString(digest) + return dcrtime.RegexpSHA256.MatchString(digest) } // ConvertDigest converts a string into a digest. From 0c6e8c372642c2f34ec488cad79c0e0ac74ea036 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 13 Jan 2021 14:49:18 -0600 Subject: [PATCH 215/449] multi: Add vote timestamp routes. --- politeiad/backend/tlogbe/comments.go | 2 +- politeiad/backend/tlogbe/pi.go | 4 +- politeiad/backend/tlogbe/ticketvote.go | 183 ++++++++++++-- politeiad/backend/tlogbe/tlog.go | 34 ++- politeiad/backend/tlogbe/tlogclient.go | 34 ++- politeiad/plugins/comments/comments.go | 10 +- politeiad/plugins/ticketvote/ticketvote.go | 281 ++++++++++++++------- politeiawww/api/comments/v1/v1.go | 4 +- politeiawww/api/pi/v1/v1.go | 2 +- politeiawww/api/records/v1/v1.go | 4 +- politeiawww/api/ticketvote/v1/v1.go | 110 ++++++++ politeiawww/cmd/piwww/piwww.go | 1 + politeiawww/cmd/piwww/recordtimestamps.go | 1 - politeiawww/cmd/piwww/votetimestamps.go | 103 ++++++++ politeiawww/cmd/shared/client.go | 35 +++ politeiawww/comments.go | 13 +- politeiawww/piwww.go | 28 +- politeiawww/ticketvote.go | 184 +++++++++++++- util/convert.go | 2 +- wsdcrdata/wsdcrdata.go | 10 +- 20 files changed, 890 insertions(+), 155 deletions(-) create mode 100644 politeiawww/api/ticketvote/v1/v1.go create mode 100644 politeiawww/cmd/piwww/votetimestamps.go diff --git a/politeiad/backend/tlogbe/comments.go b/politeiad/backend/tlogbe/comments.go index da36544fd..7d5b50b57 100644 --- a/politeiad/backend/tlogbe/comments.go +++ b/politeiad/backend/tlogbe/comments.go @@ -795,7 +795,7 @@ func (p *commentsPlugin) timestamp(s comments.StateT, token []byte, merkle []byt tlogID := tlogIDFromCommentState(s) t, err := p.tlog.timestamp(tlogID, token, merkle) if err != nil { - return nil, fmt.Errorf("timestamp %x: %v", merkle, err) + return nil, fmt.Errorf("timestamp %x %x: %v", token, merkle, err) } // Convert response diff --git a/politeiad/backend/tlogbe/pi.go b/politeiad/backend/tlogbe/pi.go index cf1af3873..574ebc35e 100644 --- a/politeiad/backend/tlogbe/pi.go +++ b/politeiad/backend/tlogbe/pi.go @@ -1208,8 +1208,8 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { } if summary.Status != ticketvote.VoteStatusUnauthorized { e := fmt.Sprintf("vote status got %v, want %v", - ticketvote.VoteStatus[summary.Status], - ticketvote.VoteStatus[ticketvote.VoteStatusUnauthorized]) + ticketvote.VoteStatuses[summary.Status], + ticketvote.VoteStatuses[ticketvote.VoteStatusUnauthorized]) return backend.PluginUserError{ PluginID: pi.ID, ErrorCode: int(pi.ErrorStatusVoteStatusInvalid), diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index fbdad5c52..6bcfac4bb 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -666,7 +666,7 @@ func convertBlobEntryFromCastVoteDetails(cv ticketvote.CastVoteDetails) (*store. return &be, nil } -func (p *ticketVotePlugin) authorizeSave(ad ticketvote.AuthDetails) error { +func (p *ticketVotePlugin) authSave(ad ticketvote.AuthDetails) error { token, err := hex.DecodeString(ad.Token) if err != nil { return err @@ -700,7 +700,7 @@ func (p *ticketVotePlugin) authorizeSave(ad ticketvote.AuthDetails) error { return nil } -func (p *ticketVotePlugin) authorizes(token []byte) ([]ticketvote.AuthDetails, error) { +func (p *ticketVotePlugin) auths(token []byte) ([]ticketvote.AuthDetails, error) { // Retrieve blobs blobs, err := p.tlog.blobsByKeyPrefix(tlogIDVetted, token, keyPrefixAuthDetails) @@ -861,6 +861,32 @@ func (p *ticketVotePlugin) castVotes(token []byte) ([]ticketvote.CastVoteDetails return votes, nil } +func (p *ticketVotePlugin) timestamp(token []byte, merkle []byte) (*ticketvote.Timestamp, error) { + t, err := p.tlog.timestamp(tlogIDVetted, token, merkle) + if err != nil { + return nil, fmt.Errorf("timestamp %x %x: %v", token, merkle, err) + } + + // Convert response + proofs := make([]ticketvote.Proof, 0, len(t.Proofs)) + for _, v := range t.Proofs { + proofs = append(proofs, ticketvote.Proof{ + Type: v.Type, + Digest: v.Digest, + MerkleRoot: v.MerkleRoot, + MerklePath: v.MerklePath, + ExtraData: v.ExtraData, + }) + } + return &ticketvote.Timestamp{ + Data: t.Data, + Digest: t.Digest, + TxID: t.TxID, + MerkleRoot: t.MerkleRoot, + Proofs: proofs, + }, nil +} + // bestBlock fetches the best block from the dcrdata plugin and returns it. If // the dcrdata connection is not active, an error will be returned. func (p *ticketVotePlugin) bestBlock() (uint32, error) { @@ -1187,7 +1213,7 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { // Get any previous authorizations to verify that the new action // is allowed based on the previous action. - auths, err := p.authorizes(token) + auths, err := p.auths(token) if err != nil { return "", err } @@ -1236,7 +1262,7 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { } // Save authorize vote - err = p.authorizeSave(auth) + err = p.authSave(auth) if err != nil { return "", err } @@ -1506,7 +1532,7 @@ func (p *ticketVotePlugin) startStandard(s ticketvote.Start) (*ticketvote.StartR } // Verify vote authorization - auths, err := p.authorizes(token) + auths, err := p.auths(token) if err != nil { return nil, err } @@ -1859,7 +1885,7 @@ func (p *ticketVotePlugin) ballot(votes []ticketvote.CastVote, results chan tick cvr.Ticket = v.Ticket cvr.ErrorCode = e cvr.ErrorContext = fmt.Sprintf("%v: %v", - ticketvote.VoteError[e], t) + ticketvote.VoteErrors[e], t) goto sendResult } @@ -1920,7 +1946,7 @@ func (p *ticketVotePlugin) cmdCastBallot(payload string) (string, error) { receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e receipts[k].ErrorContext = fmt.Sprintf("%v: not hex", - ticketvote.VoteError[e]) + ticketvote.VoteErrors[e]) continue } if token == nil { @@ -1935,7 +1961,7 @@ func (p *ticketVotePlugin) cmdCastBallot(payload string) (string, error) { e := ticketvote.VoteErrorMultipleRecordVotes receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e - receipts[k].ErrorContext = ticketvote.VoteError[e] + receipts[k].ErrorContext = ticketvote.VoteErrors[e] continue } } @@ -1985,7 +2011,7 @@ func (p *ticketVotePlugin) cmdCastBallot(payload string) (string, error) { receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e receipts[k].ErrorContext = fmt.Sprintf("%v: vote not started", - ticketvote.VoteError[e]) + ticketvote.VoteErrors[e]) continue } if bestBlock >= voteDetails.EndBlockHeight { @@ -1994,7 +2020,7 @@ func (p *ticketVotePlugin) cmdCastBallot(payload string) (string, error) { receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e receipts[k].ErrorContext = fmt.Sprintf("%v: vote has ended", - ticketvote.VoteError[e]) + ticketvote.VoteErrors[e]) continue } @@ -2004,7 +2030,7 @@ func (p *ticketVotePlugin) cmdCastBallot(payload string) (string, error) { e := ticketvote.VoteErrorVoteBitInvalid receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e - receipts[k].ErrorContext = ticketvote.VoteError[e] + receipts[k].ErrorContext = ticketvote.VoteErrors[e] continue } err = voteBitVerify(voteDetails.Params.Options, @@ -2014,7 +2040,7 @@ func (p *ticketVotePlugin) cmdCastBallot(payload string) (string, error) { receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e receipts[k].ErrorContext = fmt.Sprintf("%v: %v", - ticketvote.VoteError[e], err) + ticketvote.VoteErrors[e], err) continue } @@ -2028,7 +2054,7 @@ func (p *ticketVotePlugin) cmdCastBallot(payload string) (string, error) { receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e receipts[k].ErrorContext = fmt.Sprintf("%v: %v", - ticketvote.VoteError[e], t) + ticketvote.VoteErrors[e], t) continue } if commitmentAddr.err != nil { @@ -2039,7 +2065,7 @@ func (p *ticketVotePlugin) cmdCastBallot(payload string) (string, error) { receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e receipts[k].ErrorContext = fmt.Sprintf("%v: %v", - ticketvote.VoteError[e], t) + ticketvote.VoteErrors[e], t) continue } err = p.castVoteSignatureVerify(v, commitmentAddr.addr) @@ -2048,7 +2074,7 @@ func (p *ticketVotePlugin) cmdCastBallot(payload string) (string, error) { receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e receipts[k].ErrorContext = fmt.Sprintf("%v: %v", - ticketvote.VoteError[e], err) + ticketvote.VoteErrors[e], err) continue } @@ -2058,7 +2084,7 @@ func (p *ticketVotePlugin) cmdCastBallot(payload string) (string, error) { e := ticketvote.VoteErrorTicketNotEligible receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e - receipts[k].ErrorContext = ticketvote.VoteError[e] + receipts[k].ErrorContext = ticketvote.VoteErrors[e] continue } } @@ -2083,7 +2109,7 @@ func (p *ticketVotePlugin) cmdCastBallot(payload string) (string, error) { e := ticketvote.VoteErrorTicketAlreadyVoted receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e - receipts[k].ErrorContext = ticketvote.VoteError[e] + receipts[k].ErrorContext = ticketvote.VoteErrors[e] continue } } @@ -2098,7 +2124,7 @@ func (p *ticketVotePlugin) cmdCastBallot(payload string) (string, error) { // the ballot will take 200 milliseconds since we wait for the leaf // to be fully appended before considering the trillian call // successful. A person casting hundreds of votes in a single ballot - // would cause UX issues for the all voting clients since the lock is + // would cause UX issues for all the voting clients since the lock is // held during these calls. // // The second variable that we must watch out for is the max trillian @@ -2180,7 +2206,7 @@ func (p *ticketVotePlugin) cmdCastBallot(payload string) (string, error) { receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e receipts[k].ErrorContext = fmt.Sprintf("%v: %v", - ticketvote.VoteError[e], t) + ticketvote.VoteErrors[e], t) continue } @@ -2217,7 +2243,7 @@ func (p *ticketVotePlugin) cmdDetails(payload string) (string, error) { } // Get authorize votes - auths, err := p.authorizes(token) + auths, err := p.auths(token) if err != nil { if errors.Is(err, errRecordNotFound) { return "", backend.PluginUserError{ @@ -2225,7 +2251,7 @@ func (p *ticketVotePlugin) cmdDetails(payload string) (string, error) { ErrorCode: int(ticketvote.ErrorStatusRecordNotFound), } } - return "", fmt.Errorf("authorizes: %v", err) + return "", fmt.Errorf("auths: %v", err) } // Get vote details @@ -2372,14 +2398,14 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. status := ticketvote.VoteStatusUnauthorized // Check if the vote has been authorized - auths, err := p.authorizes(token) + auths, err := p.auths(token) if err != nil { if errors.Is(err, errRecordNotFound) { // Let the calling function decide how to handle when a vetted // record does not exist for the token. return nil, err } - return nil, fmt.Errorf("authorizes: %v", err) + return nil, fmt.Errorf("auths: %v", err) } if len(auths) > 0 { lastAuth := auths[len(auths)-1] @@ -2598,6 +2624,113 @@ func (p *ticketVotePlugin) cmdInventory(payload string) (string, error) { return string(reply), nil } +func (p *ticketVotePlugin) cmdTimestamps(payload string) (string, error) { + log.Tracef("ticketvote cmdTicketvote: %v", payload) + + // Decode payload + var t ticketvote.Timestamps + err := json.Unmarshal([]byte(payload), &t) + if err != nil { + return "", err + } + + // Verify token + token, err := tokenDecodeAnyLength(t.Token) + if err != nil { + return "", backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusTokenInvalid), + } + } + + // Get authorization timestamps + merkles, err := p.tlog.merklesByKeyPrefix(tlogIDVetted, token, + keyPrefixAuthDetails) + if err != nil { + if errors.Is(err, errRecordNotFound) { + return "", backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusRecordNotFound), + } + } + return "", fmt.Errorf("merklesByKeyPrefix %x %v: %v", + token, keyPrefixAuthDetails, err) + } + + auths := make([]ticketvote.Timestamp, 0, len(merkles)) + for _, v := range merkles { + ts, err := p.timestamp(token, v) + if err != nil { + return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) + } + auths = append(auths, *ts) + } + + // Get vote details merkle leaf hash. There should never be more + // than one vote details. + merkles, err = p.tlog.merklesByKeyPrefix(tlogIDVetted, token, + keyPrefixVoteDetails) + if err != nil { + return "", fmt.Errorf("merklesByKeyPrefix %x %v: %v", + token, keyPrefixVoteDetails, err) + } + if len(merkles) > 1 { + return "", fmt.Errorf("invalid vote details count: got %v, want 1", + len(merkles)) + } + + var details ticketvote.Timestamp + for _, v := range merkles { + ts, err := p.timestamp(token, v) + if err != nil { + return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) + } + details = *ts + } + + // Get cast vote timestamps + merkles, err = p.tlog.merklesByKeyPrefix(tlogIDVetted, token, + keyPrefixCastVoteDetails) + if err != nil { + return "", fmt.Errorf("merklesByKeyPrefix %x %v: %v", + token, keyPrefixVoteDetails, err) + } + + votes := make(map[string]ticketvote.Timestamp, len(merkles)) + for _, v := range merkles { + ts, err := p.timestamp(token, v) + if err != nil { + return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) + } + + var cv ticketvote.CastVoteDetails + err = json.Unmarshal([]byte(ts.Data), &cv) + if err != nil { + return "", err + } + + votes[cv.Ticket] = *ts + } + + b, err := json.Marshal(details) + if err != nil { + return "", err + } + + // Prepare reply + tr := ticketvote.TimestampsReply{ + Auths: auths, + Details: details, + Votes: votes, + } + reply, err := json.Marshal(tr) + if err != nil { + return "", err + } + + return string(reply), nil +} + // setup performs any plugin setup work that needs to be done. // // This function satisfies the pluginClient interface. @@ -2714,6 +2847,8 @@ func (p *ticketVotePlugin) cmd(cmd, payload string) (string, error) { return p.cmdSummaries(payload) case ticketvote.CmdInventory: return p.cmdInventory(payload) + case ticketvote.CmdTimestamps: + return p.cmdTimestamps(payload) } return "", backend.ErrPluginCmdInvalid diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog.go index e4f773bb2..463923ae3 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog.go @@ -1702,8 +1702,6 @@ func (t *tlog) recordTimestamps(treeID int64, version uint32, token []byte) (*ba // blobsSave saves the provided blobs to the key-value store then appends them // onto the trillian tree. Note, hashes contains the hashes of the data encoded // in the blobs. The hashes must share the same ordering as the blobs. -// -// This function satisfies the tlogClient interface. func (t *tlog) blobsSave(treeID int64, keyPrefix string, blobs, hashes [][]byte, encrypt bool) ([][]byte, error) { log.Tracef("%v blobsSave: %v %v %v", t.id, treeID, keyPrefix, encrypt) @@ -1791,8 +1789,6 @@ func (t *tlog) blobsSave(treeID int64, keyPrefix string, blobs, hashes [][]byte, // del deletes the blobs in the kv store that correspond to the provided merkle // leaf hashes. The kv store keys in store in the ExtraData field of the leaves // specified by the provided merkle leaf hashes. -// -// This function satisfies the tlogClient interface. func (t *tlog) blobsDel(treeID int64, merkles [][]byte) error { log.Tracef("%v blobsDel: %v", t.id, treeID) @@ -1840,8 +1836,6 @@ func (t *tlog) blobsDel(treeID int64, merkles [][]byte) error { // If a blob does not exist it will not be included in the returned map. It is // the responsibility of the caller to check that a blob is returned for each // of the provided merkle hashes. -// -// This function satisfies the tlogClient interface. func (t *tlog) blobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, error) { log.Tracef("%v blobsByMerkle: %v", t.id, treeID) @@ -1924,9 +1918,33 @@ func (t *tlog) blobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, return b, nil } +func (t *tlog) merklesByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) { + log.Tracef("%v merklesByKeyPrefix: %v %v", t.id, treeID, keyPrefix) + + // Verify tree exists + if !t.treeExists(treeID) { + return nil, errRecordNotFound + } + + // Get leaves + leaves, err := t.trillian.leavesAll(treeID) + if err != nil { + return nil, fmt.Errorf("leavesAll: %v", err) + } + + // Walk leaves and aggregate the merkle leaf hashes with a matching + // key prefix. + merkles := make([][]byte, 0, len(leaves)) + for _, v := range leaves { + if bytes.HasPrefix(v.ExtraData, []byte(keyPrefix)) { + merkles = append(merkles, v.MerkleLeafHash) + } + } + + return merkles, nil +} + // blobsByKeyPrefix returns all blobs that match the provided key prefix. -// -// This function satisfies the tlogClient interface. func (t *tlog) blobsByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) { log.Tracef("%v blobsByKeyPrefix: %v %v", t.id, treeID, keyPrefix) diff --git a/politeiad/backend/tlogbe/tlogclient.go b/politeiad/backend/tlogbe/tlogclient.go index 071d9350f..b38f53047 100644 --- a/politeiad/backend/tlogbe/tlogclient.go +++ b/politeiad/backend/tlogbe/tlogclient.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -24,14 +24,18 @@ type tlogClient interface { // leaf hashes. del(tlogID string, token []byte, merkleLeafHashes [][]byte) error + // merklesByKeyPrefix returns the merkle root hashes for all blobs + // that match the key prefix. + merklesByKeyPrefix(tlogID string, token []byte, + keyPrefix string) ([][]byte, error) + // blobsByMerkle returns the blobs with the provided merkle leaf // hashes. If a blob does not exist it will not be included in the // returned map. blobsByMerkle(tlogID string, token []byte, merkleLeafHashes [][]byte) (map[string][]byte, error) - // blobsByKeyPrefix returns all blobs that match the provided key - // prefix. + // blobsByKeyPrefix returns all blobs that match the key prefix. blobsByKeyPrefix(tlogID string, token []byte, keyPrefix string) ([][]byte, error) @@ -144,6 +148,30 @@ func (c *backendClient) del(tlogID string, token []byte, merkles [][]byte) error return tlog.blobsDel(treeID, merkles) } +// merklesByKeyPrefix returns the merkle root hashes for all blobs that match +// the key prefix. +// +// This function satisfies the tlogClient interface. +func (c *backendClient) merklesByKeyPrefix(tlogID string, token []byte, keyPrefix string) ([][]byte, error) { + log.Tracef("backendClient merklesByKeyPrefix: %v %x %x", + tlogID, token, keyPrefix) + + // Get tlog instance + tlog, err := c.tlogByID(tlogID) + if err != nil { + return nil, err + } + + // Get tree ID + treeID, err := c.treeIDFromToken(tlogID, token) + if err != nil { + return nil, err + } + + // Get merkle leaf hashes + return tlog.merklesByKeyPrefix(treeID, keyPrefix) +} + // blobsByMerkle returns the blobs with the provided merkle leaf hashes. // // If a blob does not exist it will not be included in the returned map. It is diff --git a/politeiad/plugins/comments/comments.go b/politeiad/plugins/comments/comments.go index 6d202be3f..72369b34f 100644 --- a/politeiad/plugins/comments/comments.go +++ b/politeiad/plugins/comments/comments.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -581,8 +581,8 @@ type Proof struct { ExtraData string `json:"extradata"` // JSON encoded } -// Timestamp contains all of the data required to verify that a piece of -// data was timestamped onto the decred blockchain. +// Timestamp contains all of the data required to verify that a piece of data +// was timestamped onto the decred blockchain. // // All digests are hex encoded SHA256 digests. The merkle root can be found in // the OP_RETURN of the specified DCR transaction. @@ -606,8 +606,8 @@ type Timestamp struct { type Timestamps struct { State StateT `json:"state"` Token string `json:"token"` - CommentIDs []uint32 `json:"commentids"` - IncludeVotes bool `json:"includevotes"` + CommentIDs []uint32 `json:"commentids,omitempty"` + IncludeVotes bool `json:"includevotes,omitempty"` } // TimestampsReply is the reply to the timestamps command. diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index 6740b1266..33b7b9e95 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -10,14 +10,14 @@ import ( "encoding/json" ) -type VoteStatusT int -type AuthActionT string -type VoteT int -type VoteErrorT int -type ErrorStatusT int - // TODO VoteDetails, StartReply, StartRunoffReply should contain a receipt. // The receipt should be the server signature of Signature+StartBlockHash. +// TODO Update politeiavoter +// TODO the timestamps reply is going to be too large. Each ticket vote +// timestamp is ~2000 bytes. +// Avg (15k votes): 30MB +// Large (25k votes): 50MB +// Max (41k votes): 82MB const ( ID = "ticketvote" @@ -30,6 +30,7 @@ const ( CmdResults = "results" // Get vote results CmdSummaries = "summaries" // Get vote summaries CmdInventory = "inventory" // Get inventory by vote status + CmdTimestamps = "timestamps" // Get vote data timestamps // Default plugin settings DefaultMainNetVoteDurationMin = 2016 @@ -43,19 +44,47 @@ const ( // PolicyVotesPageSize is the maximum number of results that can be // returned from any of the batched vote commands. PolicyVotesPageSize = 20 +) - // Vote statuses - VoteStatusInvalid VoteStatusT = 0 // Invalid status - VoteStatusUnauthorized VoteStatusT = 1 // Vote has not been authorized - VoteStatusAuthorized VoteStatusT = 2 // Vote has been authorized - VoteStatusStarted VoteStatusT = 3 // Vote has been started - VoteStatusFinished VoteStatusT = 4 // Vote has finished +// ErrorStatusT represents and error that is caused by the user. +type ErrorStatusT int - // Authorize vote actions - AuthActionAuthorize AuthActionT = "authorize" - AuthActionRevoke AuthActionT = "revoke" +const ( + ErrorStatusInvalid ErrorStatusT = 0 + ErrorStatusTokenInvalid ErrorStatusT = 1 + ErrorStatusPublicKeyInvalid ErrorStatusT = 2 + ErrorStatusSignatureInvalid ErrorStatusT = 3 + ErrorStatusRecordNotFound ErrorStatusT = 4 + ErrorStatusRecordVersionInvalid ErrorStatusT = 5 + ErrorStatusRecordStatusInvalid ErrorStatusT = 6 + ErrorStatusAuthorizationInvalid ErrorStatusT = 7 + ErrorStatusStartDetailsInvalid ErrorStatusT = 8 + ErrorStatusVoteParamsInvalid ErrorStatusT = 9 + ErrorStatusVoteStatusInvalid ErrorStatusT = 10 + ErrorStatusPageSizeExceeded ErrorStatusT = 11 +) + +// AuthDetails is the structure that is saved to disk when a vote is authorized +// or a previous authorization is revoked. It contains all the fields from a +// Authorize and a AuthorizeReply. +type AuthDetails struct { + // Data generated by client + Token string `json:"token"` // Record token + Version uint32 `json:"version"` // Record version + Action string `json:"action"` // Authorize or revoke + PublicKey string `json:"publickey"` // Public key used for signature + Signature string `json:"signature"` // Signature of token+version+action + + // Metadata generated by server + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + Receipt string `json:"receipt"` // Server signature of client signature +} + +// VoteT represents the different types of ticket votes that are available. +type VoteT int - // Vote types +const ( + // VoteTypeInvalid is an invalid vote type. VoteTypeInvalid VoteT = 0 // VoteTypeStandard is used to indicate a simple approve or reject @@ -77,7 +106,9 @@ const ( // net yes votes. Runoff vote participants are not required to have // the voting period authorized prior to the vote starting. VoteTypeRunoff VoteT = 2 +) +const ( // VoteOptionIDApprove is the vote option ID that indicates the vote // should be approved. Votes that are an approve/reject vote are // required to use this vote option ID. @@ -87,76 +118,8 @@ const ( // should be not be approved. Votes that are an approve/reject vote // are required to use this vote option ID. VoteOptionIDReject = "no" - - // Vote error status codes. Vote errors are errors that occur while - // attempting to cast a vote. These errors are returned with the - // individual failed vote. - // TODO change politeiavoter to use these error codes - VoteErrorInvalid VoteErrorT = 0 - VoteErrorInternalError VoteErrorT = 1 - VoteErrorTokenInvalid VoteErrorT = 2 - VoteErrorRecordNotFound VoteErrorT = 3 - VoteErrorMultipleRecordVotes VoteErrorT = 4 - VoteErrorVoteStatusInvalid VoteErrorT = 5 - VoteErrorVoteBitInvalid VoteErrorT = 6 - VoteErrorSignatureInvalid VoteErrorT = 7 - VoteErrorTicketNotEligible VoteErrorT = 8 - VoteErrorTicketAlreadyVoted VoteErrorT = 9 - - // User error status codes - ErrorStatusInvalid ErrorStatusT = 0 - ErrorStatusTokenInvalid ErrorStatusT = 1 - ErrorStatusPublicKeyInvalid ErrorStatusT = 2 - ErrorStatusSignatureInvalid ErrorStatusT = 3 - ErrorStatusRecordNotFound ErrorStatusT = 4 - ErrorStatusRecordVersionInvalid ErrorStatusT = 5 - ErrorStatusRecordStatusInvalid ErrorStatusT = 6 - ErrorStatusAuthorizationInvalid ErrorStatusT = 7 - ErrorStatusStartDetailsInvalid ErrorStatusT = 8 - ErrorStatusVoteParamsInvalid ErrorStatusT = 9 - ErrorStatusVoteStatusInvalid ErrorStatusT = 10 - ErrorStatusPageSizeExceeded ErrorStatusT = 11 -) - -var ( - VoteStatus = map[VoteStatusT]string{ - VoteStatusInvalid: "vote status invalid", - VoteStatusUnauthorized: "unauthorized", - VoteStatusAuthorized: "authorized", - VoteStatusStarted: "started", - VoteStatusFinished: "finished", - } - - VoteError = map[VoteErrorT]string{ - VoteErrorInvalid: "vote error invalid", - VoteErrorInternalError: "internal server error", - VoteErrorTokenInvalid: "token invalid", - VoteErrorRecordNotFound: "record not found", - VoteErrorMultipleRecordVotes: "attempting to vote on multiple records", - VoteErrorVoteStatusInvalid: "record vote status invalid", - VoteErrorVoteBitInvalid: "vote bit invalid", - VoteErrorSignatureInvalid: "signature invalid", - VoteErrorTicketNotEligible: "ticket not eligible", - VoteErrorTicketAlreadyVoted: "ticket already voted", - } ) -// AuthDetails is the structure that is saved to disk when a vote is authorized -// or a previous authorization is revoked. It contains all the fields from a -// Authorize and a AuthorizeReply. -type AuthDetails struct { - // Data generated by client - Token string `json:"token"` // Record token - Version uint32 `json:"version"` // Record version - Action string `json:"action"` // Authorize or revoke - PublicKey string `json:"publickey"` // Public key used for signature - Signature string `json:"signature"` // Signature of token+version+action - - // Metadata generated by server - Timestamp int64 `json:"timestamp"` // Received UNIX timestamp - Receipt string `json:"receipt"` // Server signature of client signature -} - // VoteOption describes a single vote option. type VoteOption struct { ID string `json:"id"` // Single, unique word (e.g. yes) @@ -221,6 +184,18 @@ type CastVoteDetails struct { Receipt string `json:"receipt"` // Server signature of client signature } +// AuthActionT represents the ticket vote authorization actions. +type AuthActionT string + +const ( + // AuthActionAuthorize is used to authorize a ticket vote. + AuthActionAuthorize AuthActionT = "authorize" + + // AuthActionRevoke is used to revoke a previous ticket vote + // authorization. + AuthActionRevoke AuthActionT = "revoke" +) + // Authorize authorizes a ticket vote or revokes a previous authorization. // // Signature contains the client signature of the Token+Version+Action. @@ -324,6 +299,69 @@ func DecodeStartReply(payload []byte) (*StartReply, error) { return &sr, nil } +// VoteErrorT represents errors that can occur while attempting to cast ticket +// votes. +type VoteErrorT int + +const ( + // VoteErrorInvalid is an invalid vote error. + VoteErrorInvalid VoteErrorT = 0 + + // VoteErrorInternalError is returned when an internal server error + // occurred. + VoteErrorInternalError VoteErrorT = 1 + + // VoteErrorTokenInvalid is returned when the record censorship + // token is invalid. + VoteErrorTokenInvalid VoteErrorT = 2 + + // VoteErrorRecordNotFound is returned when the specified record + // does not exist. + VoteErrorRecordNotFound VoteErrorT = 3 + + // VoteErrorMultipleRecordVotes is returned when votes are casts + // for multiple records in a single ballot. + VoteErrorMultipleRecordVotes VoteErrorT = 4 + + // VoteErrorVoteStatusInvalid is returned when the ticket vote + // status does not allow for votes to be cast, such as when a vote + // has already finished. + VoteErrorVoteStatusInvalid VoteErrorT = 5 + + // VoteErrorVoteBitInvalid is returned when the vote being cast + // uses invalid vote bits. + VoteErrorVoteBitInvalid VoteErrorT = 6 + + // VoteErrorSignatureInvalid is returned when the vote being cast + // has an invalid signature. + VoteErrorSignatureInvalid VoteErrorT = 7 + + // VoteErrorTicketNotEligible is returned when a vote is being cast + // using a ticket that is not part of the vote. + VoteErrorTicketNotEligible VoteErrorT = 8 + + // VoteErrorTicketAlreadyVoted is returned when a vote is cast + // using a ticket that has already voted. + VoteErrorTicketAlreadyVoted VoteErrorT = 9 +) + +var ( + // VoteErrors contains the human readable error messages for the + // vote errors. + VoteErrors = map[VoteErrorT]string{ + VoteErrorInvalid: "vote error invalid", + VoteErrorInternalError: "internal server error", + VoteErrorTokenInvalid: "token invalid", + VoteErrorRecordNotFound: "record not found", + VoteErrorMultipleRecordVotes: "attempting to vote on multiple records", + VoteErrorVoteStatusInvalid: "record vote status invalid", + VoteErrorVoteBitInvalid: "vote bit invalid", + VoteErrorSignatureInvalid: "signature invalid", + VoteErrorTicketNotEligible: "ticket not eligible", + VoteErrorTicketAlreadyVoted: "ticket already voted", + } +) + // CastVote is a signed ticket vote. This structure gets saved to disk when // a vote is cast. type CastVote struct { @@ -495,6 +533,39 @@ func DecodeSummaries(payload []byte) (*Summaries, error) { return &s, nil } +// VoteStatusT represents the status of a ticket vote. +type VoteStatusT int + +const ( + // VoteStatusInvalid is an invalid vote status. + VoteStatusInvalid VoteStatusT = 0 + + // VoteStatusUnauthorized indicates the ticket vote has not been + // authorized yet. + VoteStatusUnauthorized VoteStatusT = 1 + + // VoteStatusAuthorized indicates the ticket vote has been + // authorized. + VoteStatusAuthorized VoteStatusT = 2 + + // VoteStatusStarted indicates the ticket vote has been started. + VoteStatusStarted VoteStatusT = 3 + + // VoteStatusFinished indicates the ticket vote has finished. + VoteStatusFinished VoteStatusT = 4 +) + +var ( + // VoteStatuses contains the human readable vote statuses. + VoteStatuses = map[VoteStatusT]string{ + VoteStatusInvalid: "vote status invalid", + VoteStatusUnauthorized: "unauthorized", + VoteStatusAuthorized: "authorized", + VoteStatusStarted: "started", + VoteStatusFinished: "finished", + } +) + // VoteOptionResult describes a vote option and the total number of votes that // have been cast for this option. type VoteOptionResult struct { @@ -605,3 +676,43 @@ func DecodeInventoryReply(payload []byte) (*InventoryReply, error) { } return &ir, nil } + +// Proof contains an inclusion proof for the digest in the merkle root. The +// ExtraData field is used by certain types of proofs to include additional +// data that is required to validate the proof. +type Proof struct { + Type string `json:"type"` + Digest string `json:"digest"` + MerkleRoot string `json:"merkleroot"` + MerklePath []string `json:"merklepath"` + ExtraData string `json:"extradata"` // JSON encoded +} + +// Timestamp contains all of the data required to verify that a piece of data +// was timestamped onto the decred blockchain. +// +// All digests are hex encoded SHA256 digests. The merkle root can be found in +// the OP_RETURN of the specified DCR transaction. +// +// TxID, MerkleRoot, and Proofs will only be populated once the merkle root has +// been included in a DCR tx and the tx has 6 confirmations. The Data field +// will not be populated if the data has been censored. +type Timestamp struct { + Data string `json:"data"` // JSON encoded + Digest string `json:"digest"` + TxID string `json:"txid"` + MerkleRoot string `json:"merkleroot"` + Proofs []Proof `json:"proofs"` +} + +// Timestamps requests the timestamps for ticket vote data. +type Timestamps struct { + Token string `json:"token"` +} + +// TimestampsReply is the reply to the Timestamps command. +type TimestampsReply struct { + Auths []Timestamp `json:"auths,omitempty"` + Details Timestamp `json:"details,omitempty"` + Votes map[string]Timestamp `json:"votes,omitempty"` // [ticket]Timestamp +} diff --git a/politeiawww/api/comments/v1/v1.go b/politeiawww/api/comments/v1/v1.go index 2e09d528d..0435b51ab 100644 --- a/politeiawww/api/comments/v1/v1.go +++ b/politeiawww/api/comments/v1/v1.go @@ -95,8 +95,8 @@ type Proof struct { ExtraData string `json:"extradata"` // JSON encoded } -// Timestamp contains all of the data required to verify that a piece of -// data was timestamped onto the decred blockchain. +// Timestamp contains all of the data required to verify that a piece of data +// was timestamped onto the decred blockchain. // // All digests are hex encoded SHA256 digests. The merkle root can be found in // the OP_RETURN of the specified DCR transaction. diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index ea4a169b8..80656d0ab 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go index fd63e9f19..a927df912 100644 --- a/politeiawww/api/records/v1/v1.go +++ b/politeiawww/api/records/v1/v1.go @@ -99,8 +99,8 @@ type Proof struct { ExtraData string `json:"extradata"` // JSON encoded } -// Timestamp contains all of the data required to verify that a piece of -// record data was timestamped onto the decred blockchain. +// Timestamp contains all of the data required to verify that a piece of record +// data was timestamped onto the decred blockchain. // // All digests are hex encoded SHA256 digests. The merkle root can be found in // the OP_RETURN of the specified DCR transaction. diff --git a/politeiawww/api/ticketvote/v1/v1.go b/politeiawww/api/ticketvote/v1/v1.go new file mode 100644 index 000000000..12f66624f --- /dev/null +++ b/politeiawww/api/ticketvote/v1/v1.go @@ -0,0 +1,110 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package v1 + +import "fmt" + +const ( + // APIRoute is prefixed onto all routes defined in this package. + APIRoute = "/ticketvote/v1" + + RouteTimestamps = "/timestamps" +) + +// ErrorCodeT represents a user error code. +type ErrorCodeT int + +const ( + // Error codes + ErrorCodeInvalid ErrorCodeT = 0 + ErrorCodeInputInvalid ErrorCodeT = 1 +) + +var ( + // ErrorCodes contains the human readable errors. + ErrorCodes = map[ErrorCodeT]string{ + ErrorCodeInvalid: "error invalid", + ErrorCodeInputInvalid: "input invalid", + } +) + +// UserErrorReply is the reply that the server returns when it encounters an +// error that is caused by something that the user did (malformed input, bad +// timing, etc). The HTTP status code will be 400. +type UserErrorReply struct { + ErrorCode ErrorCodeT `json:"errorcode"` + ErrorContext string `json:"errorcontext"` +} + +// Error satisfies the error interface. +func (e UserErrorReply) Error() string { + return fmt.Sprintf("user error code: %v", e.ErrorCode) +} + +// PluginErrorReply is the reply that the server returns when it encounters +// a plugin error. +type PluginErrorReply struct { + PluginID string `json:"pluginid"` + ErrorCode int `json:"errorcode"` + ErrorContext string `json:"errorcontext"` +} + +// Error satisfies the error interface. +func (e PluginErrorReply) Error() string { + return fmt.Sprintf("user error code: %v", e.ErrorCode) +} + +// ServerErrorReply is the reply that the server returns when it encounters an +// unrecoverable error while executing a command. The HTTP status code will be +// 500 and the ErrorCode field will contain a UNIX timestamp that the user can +// provide to the server admin to track down the error details in the logs. +type ServerErrorReply struct { + ErrorCode int64 `json:"errorcode"` +} + +// Error satisfies the error interface. +func (e ServerErrorReply) Error() string { + return fmt.Sprintf("server error: %v", e.ErrorCode) +} + +// Proof contains an inclusion proof for the digest in the merkle root. The +// ExtraData field is used by certain types of proofs to include additional +// data that is required to validate the proof. +type Proof struct { + Type string `json:"type"` + Digest string `json:"digest"` + MerkleRoot string `json:"merkleroot"` + MerklePath []string `json:"merklepath"` + ExtraData string `json:"extradata"` // JSON encoded +} + +// Timestamp contains all of the data required to verify that a piece of data +// was timestamped onto the decred blockchain. +// +// All digests are hex encoded SHA256 digests. The merkle root can be found in +// the OP_RETURN of the specified DCR transaction. +// +// TxID, MerkleRoot, and Proofs will only be populated once the merkle root has +// been included in a DCR tx and the tx has 6 confirmations. The Data field +// will not be populated if the data has been censored. +type Timestamp struct { + Data string `json:"data"` // JSON encoded + Digest string `json:"digest"` + TxID string `json:"txid"` + MerkleRoot string `json:"merkleroot"` + Proofs []Proof `json:"proofs"` +} + +// Timestamps requests the timestamps for ticket vote data. +type Timestamps struct { + Token string `json:"token"` +} + +// TimestampsReply is the reply to the Timestamps command. +type TimestampsReply struct { + Auths []Timestamp `json:"auths,omitempty"` + Details Timestamp `json:"details,omitempty"` + Votes map[string]Timestamp `json:"votes,omitempty"` // [ticket]Timestamp +} diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/piwww/piwww.go index 144104b15..3c91d3514 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/piwww/piwww.go @@ -92,6 +92,7 @@ type piwww struct { VoteResults voteResultsCmd `command:"voteresults"` VoteSummaries voteSummariesCmd `command:"votesummaries"` VoteInventory voteInventoryCmd `command:"voteinv"` + VoteTimestamps voteTimestampsCmd `command:"votetimestamps"` // Websocket commands Subscribe subscribeCmd `command:"subscribe"` diff --git a/politeiawww/cmd/piwww/recordtimestamps.go b/politeiawww/cmd/piwww/recordtimestamps.go index 7d0a35502..a3a6de8a5 100644 --- a/politeiawww/cmd/piwww/recordtimestamps.go +++ b/politeiawww/cmd/piwww/recordtimestamps.go @@ -111,7 +111,6 @@ func convertTimestamp(t rcv1.Timestamp) backend.Timestamp { } } -// recordTimestampsHelpMsg is the output of the help command. const recordTimestampsHelpMsg = `recordtimestamps [flags] "token" "version" Fetch the timestamps a record version. The timestamp contains all necessary diff --git a/politeiawww/cmd/piwww/votetimestamps.go b/politeiawww/cmd/piwww/votetimestamps.go new file mode 100644 index 000000000..2f3a517ef --- /dev/null +++ b/politeiawww/cmd/piwww/votetimestamps.go @@ -0,0 +1,103 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe" + tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" + "github.com/decred/politeia/politeiawww/cmd/shared" +) + +// voteTimestampsCmd retrieves the timestamps for a politeiawww ticket vote. +type voteTimestampsCmd struct { + Args struct { + Token string `positional-arg-name:"token" required:"true"` + } `positional-args:"true"` +} + +// Execute executes the voteTimestampsCmd command. +// +// This function satisfies the go-flags Commander interface. +func (c *voteTimestampsCmd) Execute(args []string) error { + // Setup request + t := tkv1.Timestamps{ + Token: c.Args.Token, + } + + // Send request + err := shared.PrintJSON(t) + if err != nil { + return err + } + tr, err := client.TicketVoteTimestamps(t) + if err != nil { + return err + } + err = shared.PrintJSON(tr) + if err != nil { + return err + } + + // Verify timestamps + for k, v := range tr.Auths { + err = verifyVoteTimestamp(v) + if err != nil { + return fmt.Errorf("verify authorization %v timestamp: %v", k, err) + } + } + err = verifyVoteTimestamp(tr.Details) + if err != nil { + return fmt.Errorf("verify vote details timestamp: %v", err) + } + for k, v := range tr.Votes { + err = verifyVoteTimestamp(v) + if err != nil { + return fmt.Errorf("verify vote %v timestamp: %v", k, err) + } + } + + return nil +} + +func verifyVoteTimestamp(t tkv1.Timestamp) error { + ts := convertVoteTimestamp(t) + return tlogbe.VerifyTimestamp(ts) +} + +func convertVoteProof(p tkv1.Proof) backend.Proof { + return backend.Proof{ + Type: p.Type, + Digest: p.Digest, + MerkleRoot: p.MerkleRoot, + MerklePath: p.MerklePath, + ExtraData: p.ExtraData, + } +} + +func convertVoteTimestamp(t tkv1.Timestamp) backend.Timestamp { + proofs := make([]backend.Proof, 0, len(t.Proofs)) + for _, v := range t.Proofs { + proofs = append(proofs, convertVoteProof(v)) + } + return backend.Timestamp{ + Data: t.Data, + Digest: t.Digest, + TxID: t.TxID, + MerkleRoot: t.MerkleRoot, + Proofs: proofs, + } +} + +const voteTimestampsHelpMsg = `votetimestamps [flags] "token" + +Fetch the timestamps for a ticket vote. This includes timestamps for all +authorizations, the vote details, and all cast votes. + +Arguments: +1. token (string, required) Record token +` diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index a2502abf8..781bda63a 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -24,6 +24,7 @@ import ( cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" pi "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" www2 "github.com/decred/politeia/politeiawww/api/www/v2" "github.com/decred/politeia/util" @@ -138,6 +139,11 @@ func commentsError(body []byte, statusCode int) error { return fmt.Errorf("%v %s", statusCode, body) } +// TODO implement ticketVoteError +func ticketVoteError(body []byte, statusCode int) error { + return fmt.Errorf("%v %s", statusCode, body) +} + // piError unmarshals the response body from makeRequest, and handles any // status code errors from the server. Parses the error code and error context // from the pi api, in case of user error. @@ -962,6 +968,35 @@ func (c *Client) RecordTimestamps(t rcv1.Timestamps) (*rcv1.TimestampsReply, err return &tr, nil } +// TicketVoteTimestamps sends the Timestamps command to politeiawww ticketvote +// API. +func (c *Client) TicketVoteTimestamps(t tkv1.Timestamps) (*tkv1.TimestampsReply, + error) { + statusCode, respBody, err := c.makeRequest(http.MethodPost, + tkv1.APIRoute, tkv1.RouteTimestamps, t) + if err != nil { + return nil, err + } + if statusCode != http.StatusOK { + return nil, ticketVoteError(respBody, statusCode) + } + + var tr tkv1.TimestampsReply + err = json.Unmarshal(respBody, &tr) + if err != nil { + return nil, err + } + + if c.cfg.Verbose { + err := prettyPrintJSON(tr) + if err != nil { + return nil, err + } + } + + return &tr, nil +} + // Proposals retrieves a proposal for each of the provided proposal requests. func (c *Client) Proposals(p pi.Proposals) (*pi.ProposalsReply, error) { statusCode, respBody, err := c.makeRequest(http.MethodPost, diff --git a/politeiawww/comments.go b/politeiawww/comments.go index ea93d1374..c3759a651 100644 --- a/politeiawww/comments.go +++ b/politeiawww/comments.go @@ -128,9 +128,10 @@ func (p *politeiawww) processCommentTimestamps(ctx context.Context, t cmv1.Times // Get timestamps ct := comments.Timestamps{ - State: convertCommentState(t.State), - Token: t.Token, - CommentIDs: t.CommentIDs, + State: convertCommentState(t.State), + Token: t.Token, + CommentIDs: t.CommentIDs, + IncludeVotes: false, } ctr, err := p.commentTimestamps(ctx, ct) if err != nil { @@ -162,7 +163,7 @@ func (p *politeiawww) handleCommentTimestamps(w http.ResponseWriter, r *http.Req var t cmv1.Timestamps decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&t); err != nil { - respondWithCommentsError(w, r, "handleTimestamps: unmarshal", + respondWithCommentsError(w, r, "handleCommentTimestamps: unmarshal", cmv1.UserErrorReply{ ErrorCode: cmv1.ErrorCodeInputInvalid, }) @@ -174,7 +175,7 @@ func (p *politeiawww) handleCommentTimestamps(w http.ResponseWriter, r *http.Req usr, err := p.getSessionUser(w, r) if err != nil && err != errSessionNotFound { respondWithCommentsError(w, r, - "handleTimestamps: getSessionUser: %v", err) + "handleCommentTimestamps: getSessionUser: %v", err) return } @@ -182,7 +183,7 @@ func (p *politeiawww) handleCommentTimestamps(w http.ResponseWriter, r *http.Req tr, err := p.processCommentTimestamps(r.Context(), t, isAdmin) if err != nil { respondWithCommentsError(w, r, - "handleTimestamps: processTimestamps: %v", err) + "handleCommentTimestamps: processCommentTimestamps: %v", err) return } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 0439279cc..309b5670b 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -25,6 +25,7 @@ import ( cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/user" wwwutil "github.com/decred/politeia/politeiawww/util" @@ -2600,7 +2601,7 @@ func (p *politeiawww) handleVoteInventory(w http.ResponseWriter, r *http.Request // setPiRoutes sets the pi API routes. func (p *politeiawww) setPiRoutes() { - // Proposal routes + // Pi routes - proposals p.addRoute(http.MethodPost, piv1.APIRoute, piv1.RouteProposalNew, p.handleProposalNew, permissionLogin) @@ -2610,9 +2611,6 @@ func (p *politeiawww) setPiRoutes() { p.addRoute(http.MethodPost, piv1.APIRoute, piv1.RouteProposalSetStatus, p.handleProposalSetStatus, permissionAdmin) - p.addRoute(http.MethodPost, rcv1.APIRoute, - rcv1.RouteTimestamps, p.handleTimestamps, - permissionPublic) p.addRoute(http.MethodPost, piv1.APIRoute, piv1.RouteProposals, p.handleProposals, permissionPublic) @@ -2620,7 +2618,7 @@ func (p *politeiawww) setPiRoutes() { piv1.RouteProposalInventory, p.handleProposalInventory, permissionPublic) - // Comment routes + // Pi routes - comments p.addRoute(http.MethodPost, piv1.APIRoute, piv1.RouteCommentNew, p.handleCommentNew, permissionLogin) @@ -2636,11 +2634,8 @@ func (p *politeiawww) setPiRoutes() { p.addRoute(http.MethodPost, piv1.APIRoute, piv1.RouteCommentVotes, p.handleCommentVotes, permissionPublic) - p.addRoute(http.MethodPost, cmv1.APIRoute, - cmv1.RouteTimestamps, p.handleCommentTimestamps, - permissionPublic) - // Vote routes + // Pi routes - vote p.addRoute(http.MethodPost, piv1.APIRoute, piv1.RouteVoteAuthorize, p.handleVoteAuthorize, permissionLogin) @@ -2662,4 +2657,19 @@ func (p *politeiawww) setPiRoutes() { p.addRoute(http.MethodPost, piv1.APIRoute, piv1.RouteVoteInventory, p.handleVoteInventory, permissionPublic) + + // Record routes + p.addRoute(http.MethodPost, rcv1.APIRoute, + rcv1.RouteTimestamps, p.handleTimestamps, + permissionPublic) + + // Comment routes + p.addRoute(http.MethodPost, cmv1.APIRoute, + cmv1.RouteTimestamps, p.handleCommentTimestamps, + permissionPublic) + + // Ticket vote routes + p.addRoute(http.MethodPost, tkv1.APIRoute, + tkv1.RouteTimestamps, p.handleTicketVoteTimestamps, + permissionPublic) } diff --git a/politeiawww/ticketvote.go b/politeiawww/ticketvote.go index 9833763ea..b2d418c1f 100644 --- a/politeiawww/ticketvote.go +++ b/politeiawww/ticketvote.go @@ -6,10 +6,44 @@ package main import ( "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "runtime/debug" + "strings" + "time" - ticketvote "github.com/decred/politeia/politeiad/plugins/ticketvote" + "github.com/decred/politeia/politeiad/plugins/ticketvote" + cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" + tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" + "github.com/decred/politeia/util" ) +func convertProofFromTicketVotePlugin(p ticketvote.Proof) tkv1.Proof { + return tkv1.Proof{ + Type: p.Type, + Digest: p.Digest, + MerkleRoot: p.MerkleRoot, + MerklePath: p.MerklePath, + ExtraData: p.ExtraData, + } +} + +func convertTimestampFromTicketVotePlugin(t ticketvote.Timestamp) tkv1.Timestamp { + proofs := make([]tkv1.Proof, 0, len(t.Proofs)) + for _, v := range t.Proofs { + proofs = append(proofs, convertProofFromTicketVotePlugin(v)) + } + return tkv1.Timestamp{ + Data: t.Data, + Digest: t.Digest, + TxID: t.TxID, + MerkleRoot: t.MerkleRoot, + Proofs: proofs, + } +} + // voteAuthorize uses the ticketvote plugin to authorize a vote. func (p *politeiawww) voteAuthorize(ctx context.Context, a ticketvote.Authorize) (*ticketvote.AuthorizeReply, error) { b, err := ticketvote.EncodeAuthorize(a) @@ -126,3 +160,151 @@ func (p *politeiawww) voteSummaries(ctx context.Context, tokens []string) (*tick } return sr, nil } + +func (p *politeiawww) voteTimestamps(ctx context.Context, t ticketvote.Timestamps) (*ticketvote.TimestampsReply, error) { + b, err := json.Marshal(t) + if err != nil { + return nil, err + } + r, err := p.pluginCommand(ctx, ticketvote.ID, + ticketvote.CmdTimestamps, string(b)) + if err != nil { + return nil, err + } + var tr ticketvote.TimestampsReply + err = json.Unmarshal([]byte(r), &tr) + if err != nil { + return nil, err + } + return &tr, nil +} + +func (p *politeiawww) processTicketVoteTimestamps(ctx context.Context, t tkv1.Timestamps) (*tkv1.TimestampsReply, error) { + log.Tracef("processTicketVoteTimestamps: %v", t.Token) + + // Send plugin command + r, err := p.voteTimestamps(ctx, ticketvote.Timestamps{ + Token: t.Token, + }) + if err != nil { + return nil, err + } + + // Prepare reply + var ( + auths = make([]tkv1.Timestamp, 0, len(r.Auths)) + votes = make(map[string]tkv1.Timestamp, len(r.Votes)) + + details = convertTimestampFromTicketVotePlugin(r.Details) + ) + for _, v := range r.Auths { + auths = append(auths, convertTimestampFromTicketVotePlugin(v)) + } + for k, v := range r.Votes { + votes[k] = convertTimestampFromTicketVotePlugin(v) + } + + return &tkv1.TimestampsReply{ + Auths: auths, + Details: details, + Votes: votes, + }, nil +} + +func (p *politeiawww) handleTicketVoteTimestamps(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleTicketVoteTimestamps") + + var t tkv1.Timestamps + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&t); err != nil { + respondWithTicketVoteError(w, r, "handleTicketVoteTimestamps: unmarshal", + cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodeInputInvalid, + }) + return + } + + tr, err := p.processTicketVoteTimestamps(r.Context(), t) + if err != nil { + respondWithTicketVoteError(w, r, + "handleTicketVoteTimestamps: processTicketVoteTimestamps: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, tr) +} + +func respondWithTicketVoteError(w http.ResponseWriter, r *http.Request, format string, err error) { + var ( + ue tkv1.UserErrorReply + pe pdError + ) + switch { + case errors.As(err, &ue): + // Ticket vote user error + m := fmt.Sprintf("Ticket vote user error: %v %v %v", + remoteAddr(r), ue.ErrorCode, tkv1.ErrorCodes[ue.ErrorCode]) + if ue.ErrorContext != "" { + m += fmt.Sprintf(": %v", ue.ErrorContext) + } + log.Infof(m) + util.RespondWithJSON(w, http.StatusBadRequest, + tkv1.UserErrorReply{ + ErrorCode: ue.ErrorCode, + ErrorContext: ue.ErrorContext, + }) + return + + case errors.As(err, &pe): + // Politeiad error + var ( + pluginID = pe.ErrorReply.Plugin + errCode = pe.ErrorReply.ErrorCode + errContext = pe.ErrorReply.ErrorContext + ) + switch { + case pluginID != "": + // Politeiad plugin error. Log it and return a 400. + m := fmt.Sprintf("Plugin error: %v %v %v", + remoteAddr(r), pluginID, errCode) + if len(errContext) > 0 { + m += fmt.Sprintf(": %v", strings.Join(errContext, ", ")) + } + log.Infof(m) + util.RespondWithJSON(w, http.StatusBadRequest, + tkv1.PluginErrorReply{ + PluginID: pluginID, + ErrorCode: errCode, + ErrorContext: strings.Join(errContext, ", "), + }) + return + + default: + // Unknown politeiad error. Log it and return a 500. + ts := time.Now().Unix() + log.Errorf("%v %v %v %v Internal error %v: error code "+ + "from politeiad: %v", remoteAddr(r), r.Method, r.URL, + r.Proto, ts, errCode) + + util.RespondWithJSON(w, http.StatusInternalServerError, + tkv1.ServerErrorReply{ + ErrorCode: ts, + }) + return + } + + default: + // Internal server error. Log it and return a 500. + t := time.Now().Unix() + e := fmt.Sprintf(format, err) + log.Errorf("%v %v %v %v Internal error %v: %v", + remoteAddr(r), r.Method, r.URL, r.Proto, t, e) + log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) + + util.RespondWithJSON(w, http.StatusInternalServerError, + tkv1.ServerErrorReply{ + ErrorCode: t, + }) + return + } +} diff --git a/util/convert.go b/util/convert.go index 0be701d3a..4aa81c609 100644 --- a/util/convert.go +++ b/util/convert.go @@ -42,7 +42,7 @@ func ConvertStringToken(token string) ([]byte, error) { case len(token) == pd.TokenPrefixLength: // Token prefix; continue default: - return nil, fmt.Errorf("invalid censorship token size") + return nil, fmt.Errorf("invalid token size") } // If the token length is an odd number of characters, append a // 0 digit as padding to prevent a hex.ErrLenth (odd length hex diff --git a/wsdcrdata/wsdcrdata.go b/wsdcrdata/wsdcrdata.go index 6edf24b6c..d7f41fe9a 100644 --- a/wsdcrdata/wsdcrdata.go +++ b/wsdcrdata/wsdcrdata.go @@ -26,10 +26,6 @@ const ( StatusReconnecting StatusT = 2 // Websocket is attempting to reconnect StatusShutdown StatusT = 3 // Websocket client has been shutdown - // Pending event actions - actionSubscribe = "subscribe" - actionUnsubscribe = "unsubscribe" - // eventAddress is used to subscribe to events for a specific dcr // address. The dcr address must be appended onto the eventAddress // string. @@ -60,6 +56,12 @@ var ( ErrShutdown = errors.New("client is shutdown") ) +const ( + // Pending event actions + actionSubscribe = "subscribe" + actionUnsubscribe = "unsubscribe" +) + // pendingEvent represents an event action (subscribe/unsubscribe) that is // attempted to be made while the Client is in a StateReconnecting state. The // pending event actions are replayed in the order in which they were received From 28405876938aa184084a97016e310eaee35c53f1 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 13 Jan 2021 14:55:08 -0600 Subject: [PATCH 216/449] Bug fix. --- politeiad/backend/tlogbe/ticketvote.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/ticketvote.go index 6bcfac4bb..718faa36c 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/ticketvote.go @@ -2712,11 +2712,6 @@ func (p *ticketVotePlugin) cmdTimestamps(payload string) (string, error) { votes[cv.Ticket] = *ts } - b, err := json.Marshal(details) - if err != nil { - return "", err - } - // Prepare reply tr := ticketvote.TimestampsReply{ Auths: auths, From 07fefc043bb4b50fee436860e0f6e8947c9e1840 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 13 Jan 2021 17:36:38 -0600 Subject: [PATCH 217/449] Package comments plugin. --- .../tlogbe/plugins/comments/comments.go | 2072 +++++++++++++++++ .../backend/tlogbe/plugins/comments/log.go | 25 + politeiad/backend/tlogbe/plugins/plugins.go | 149 ++ 3 files changed, 2246 insertions(+) create mode 100644 politeiad/backend/tlogbe/plugins/comments/comments.go create mode 100644 politeiad/backend/tlogbe/plugins/comments/log.go create mode 100644 politeiad/backend/tlogbe/plugins/plugins.go diff --git a/politeiad/backend/tlogbe/plugins/comments/comments.go b/politeiad/backend/tlogbe/plugins/comments/comments.go new file mode 100644 index 000000000..c5078363b --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/comments/comments.go @@ -0,0 +1,2072 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package comments + +import ( + "bytes" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" + + v1 "github.com/decred/politeia/politeiad/api/v1" + "github.com/decred/politeia/politeiad/api/v1/identity" + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" + "github.com/decred/politeia/politeiad/backend/tlogbe/store" + "github.com/decred/politeia/politeiad/plugins/comments" + "github.com/decred/politeia/util" +) + +// TODO get rid of these +var ( + errRecordNotFound = errors.New("record not found") +) + +func tokenIsFullLength(token []byte) bool { + return len(token) == v1.TokenSizeTlog +} +func tokenPrefixSize() int { + // If the token prefix length is an odd number of characters then + // padding would have needed to be added to it prior to decoding it + // to hex to prevent a hex.ErrLenth (odd length hex string) error. + // Account for this padding in the prefix size. + var size int + if v1.TokenPrefixLength%2 == 1 { + // Add 1 to the length to account for padding + size = (v1.TokenPrefixLength + 1) / 2 + } else { + // No padding was required + size = v1.TokenPrefixLength / 2 + } + return size +} +func tokenPrefix(token []byte) string { + return hex.EncodeToString(token)[:v1.TokenPrefixLength] +} +func tokenDecodeAnyLength(token string) ([]byte, error) { + // If provided token has odd length add padding + if len(token)%2 == 1 { + token = token + "0" + } + t, err := hex.DecodeString(token) + if err != nil { + return nil, fmt.Errorf("invalid hex") + } + switch { + case tokenIsFullLength(t): + // Full length tokens are allowed; continue + case len(t) == tokenPrefixSize(): + // Token prefixes are allowed; continue + default: + return nil, fmt.Errorf("invalid token size") + } + return t, nil +} + +// TODO holding the lock before verifying the token can allow the mutexes to +// be spammed. Create an infinite amount of them with invalid tokens. The fix +// is to check if the record exists in the mutexes function to ensure a token +// is valid before holding the lock on it. This is where we can return a +// record doesn't exist user error too. +// TODO prevent duplicate comments +// TODO upvoting a comment twice in the same second causes a duplicate leaf +// error which causes a 500. Solution: add the timestamp to the vote index. + +const ( + // Blob entry data descriptors + dataDescriptorCommentAdd = "commentadd" + dataDescriptorCommentDel = "commentdel" + dataDescriptorCommentVote = "commentvote" + + // Prefixes that are appended to key-value store keys before + // storing them in the log leaf ExtraData field. + keyPrefixCommentAdd = "commentadd:" + keyPrefixCommentDel = "commentdel:" + keyPrefixCommentVote = "commentvote:" + + // Filenames of cached data saved to the plugin data dir. Brackets + // are used to indicate a variable that should be replaced in the + // filename. + filenameCommentsIndex = "{state}-{token}-commentsindex.json" +) + +var ( + _ plugins.PluginClient = (*commentsPlugin)(nil) +) + +// commentsPlugin is the tlog backend implementation of the comments plugin. +// +// commentsPlugin satisfies the PluginClient interface. +type commentsPlugin struct { + sync.Mutex + backend backend.Backend + tlog plugins.BackendClient + + // dataDir is the comments plugin data directory. The only data + // that is stored here is cached data that can be re-created at any + // time by walking the trillian trees. + dataDir string + + // identity contains the full identity that the plugin uses to + // create receipts, i.e. signatures of user provided data that + // prove the backend received and processed a plugin command. + identity *identity.FullIdentity + + // Mutexes contains a mutex for each record. The mutexes are lazy + // loaded. + mutexes map[string]*sync.Mutex // [string]mutex +} + +type voteIndex struct { + Vote comments.VoteT `json:"vote"` + Merkle []byte `json:"merkle"` // Log leaf merkle leaf hash +} + +type commentIndex struct { + Adds map[uint32][]byte `json:"adds"` // [version]merkleLeafHash + Del []byte `json:"del"` // Merkle leaf hash of delete record + + // Votes contains the vote history for each uuid that voted on the + // comment. This data is cached because the effect of a new vote + // on a comment depends on the previous vote from that uuid. + // Example, a user upvotes a comment that they have already + // upvoted, the resulting vote score is 0 due to the second upvote + // removing the original upvote. + Votes map[string][]voteIndex `json:"votes"` // [uuid]votes +} + +// commentsIndex contains the indexes for all comments made on a record. +type commentsIndex struct { + Comments map[uint32]commentIndex `json:"comments"` // [commentID]comment +} + +// mutex returns the mutex for the specified record. +func (p *commentsPlugin) mutex(token string) *sync.Mutex { + p.Lock() + defer p.Unlock() + + m, ok := p.mutexes[token] + if !ok { + // Mutexes is lazy loaded + m = &sync.Mutex{} + p.mutexes[token] = m + } + + return m +} + +// commentsIndexPath accepts full length token or token prefix but always +// uses prefix when generating the comments index path string. +func (p *commentsPlugin) commentsIndexPath(s comments.StateT, token string) (string, error) { + // Use token prefix + t, err := tokenDecodeAnyLength(token) + if err != nil { + return "", err + } + token = tokenPrefix(t) + fn := filenameCommentsIndex + switch s { + case comments.StateUnvetted: + fn = strings.Replace(fn, "{state}", "unvetted", 1) + case comments.StateVetted: + fn = strings.Replace(fn, "{state}", "vetted", 1) + default: + e := fmt.Errorf("unknown comments state: %v", s) + panic(e) + } + fn = strings.Replace(fn, "{token}", token, 1) + return filepath.Join(p.dataDir, fn), nil +} + +// commentsIndexLocked returns the cached commentsIndex for the provided +// record. If a cached commentsIndex does not exist, a new one will be +// returned. +// +// This function must be called WITH the lock held. +func (p *commentsPlugin) commentsIndexLocked(s comments.StateT, token []byte) (*commentsIndex, error) { + fp, err := p.commentsIndexPath(s, hex.EncodeToString(token)) + if err != nil { + return nil, err + } + b, err := ioutil.ReadFile(fp) + if err != nil { + var e *os.PathError + if errors.As(err, &e) && !os.IsExist(err) { + // File does't exist. Return a new commentsIndex instead. + return &commentsIndex{ + Comments: make(map[uint32]commentIndex), + }, nil + } + return nil, err + } + + var idx commentsIndex + err = json.Unmarshal(b, &idx) + if err != nil { + return nil, err + } + + return &idx, nil +} + +// commentsIndex returns the cached commentsIndex for the provided +// record. If a cached commentsIndex does not exist, a new one will be +// returned. +// +// This function must be called WITHOUT the lock held. +func (p *commentsPlugin) commentsIndex(s comments.StateT, token []byte) (*commentsIndex, error) { + m := p.mutex(hex.EncodeToString(token)) + m.Lock() + defer m.Unlock() + + return p.commentsIndexLocked(s, token) +} + +// commentsIndexSaveLocked saves the provided commentsIndex to the comments +// plugin data dir. +// +// This function must be called WITH the lock held. +func (p *commentsPlugin) commentsIndexSaveLocked(s comments.StateT, token []byte, idx commentsIndex) error { + b, err := json.Marshal(idx) + if err != nil { + return err + } + + fp, err := p.commentsIndexPath(s, hex.EncodeToString(token)) + if err != nil { + return err + } + err = ioutil.WriteFile(fp, b, 0664) + if err != nil { + return err + } + + return nil +} + +func tlogIDFromCommentState(s comments.StateT) string { + switch s { + case comments.StateUnvetted: + return plugins.TlogIDUnvetted + case comments.StateVetted: + return plugins.TlogIDVetted + default: + e := fmt.Sprintf("unknown state %v", s) + panic(e) + } +} + +func encryptFromCommentState(s comments.StateT) bool { + switch s { + case comments.StateUnvetted: + return true + case comments.StateVetted: + return false + default: + e := fmt.Sprintf("unknown state %v", s) + panic(e) + } +} + +func convertCommentsErrorFromSignatureError(err error) backend.PluginUserError { + var e util.SignatureError + var s comments.ErrorStatusT + if errors.As(err, &e) { + switch e.ErrorCode { + case util.ErrorStatusPublicKeyInvalid: + s = comments.ErrorStatusPublicKeyInvalid + case util.ErrorStatusSignatureInvalid: + s = comments.ErrorStatusSignatureInvalid + } + } + return backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(s), + ErrorContext: e.ErrorContext, + } +} + +func convertBlobEntryFromCommentAdd(c comments.CommentAdd) (*store.BlobEntry, error) { + data, err := json.Marshal(c) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorCommentAdd, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertBlobEntryFromCommentDel(c comments.CommentDel) (*store.BlobEntry, error) { + data, err := json.Marshal(c) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorCommentDel, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertBlobEntryFromCommentVote(c comments.CommentVote) (*store.BlobEntry, error) { + data, err := json.Marshal(c) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorCommentVote, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertCommentAddFromBlobEntry(be store.BlobEntry) (*comments.CommentAdd, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorCommentAdd { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorCommentAdd) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + hash, err := hex.DecodeString(be.Hash) + if err != nil { + return nil, fmt.Errorf("decode hash: %v", err) + } + if !bytes.Equal(util.Digest(b), hash) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), hash) + } + var c comments.CommentAdd + err = json.Unmarshal(b, &c) + if err != nil { + return nil, fmt.Errorf("unmarshal CommentAdd: %v", err) + } + + return &c, nil +} + +func convertCommentDelFromBlobEntry(be store.BlobEntry) (*comments.CommentDel, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorCommentDel { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorCommentDel) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + hash, err := hex.DecodeString(be.Hash) + if err != nil { + return nil, fmt.Errorf("decode hash: %v", err) + } + if !bytes.Equal(util.Digest(b), hash) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), hash) + } + var c comments.CommentDel + err = json.Unmarshal(b, &c) + if err != nil { + return nil, fmt.Errorf("unmarshal CommentDel: %v", err) + } + + return &c, nil +} + +func convertCommentVoteFromBlobEntry(be store.BlobEntry) (*comments.CommentVote, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorCommentVote { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorCommentVote) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + hash, err := hex.DecodeString(be.Hash) + if err != nil { + return nil, fmt.Errorf("decode hash: %v", err) + } + if !bytes.Equal(util.Digest(b), hash) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), hash) + } + var cv comments.CommentVote + err = json.Unmarshal(b, &cv) + if err != nil { + return nil, fmt.Errorf("unmarshal CommentVote: %v", err) + } + + return &cv, nil +} + +func convertCommentFromCommentAdd(ca comments.CommentAdd) comments.Comment { + return comments.Comment{ + UserID: ca.UserID, + State: ca.State, + Token: ca.Token, + ParentID: ca.ParentID, + Comment: ca.Comment, + PublicKey: ca.PublicKey, + Signature: ca.Signature, + CommentID: ca.CommentID, + Version: ca.Version, + Timestamp: ca.Timestamp, + Receipt: ca.Receipt, + Downvotes: 0, // Not part of commentAdd data + Upvotes: 0, // Not part of commentAdd data + Deleted: false, + Reason: "", + } +} + +func convertCommentFromCommentDel(cd comments.CommentDel) comments.Comment { + // Score needs to be filled in separately + return comments.Comment{ + UserID: cd.UserID, + State: cd.State, + Token: cd.Token, + ParentID: cd.ParentID, + Comment: "", + Signature: "", + CommentID: cd.CommentID, + Version: 0, + Timestamp: cd.Timestamp, + Receipt: cd.Receipt, + Downvotes: 0, + Upvotes: 0, + Deleted: true, + Reason: cd.Reason, + } +} + +// commentVersionLatest returns the latest comment version. +func commentVersionLatest(cidx commentIndex) uint32 { + var maxVersion uint32 + for version := range cidx.Adds { + if version > maxVersion { + maxVersion = version + } + } + return maxVersion +} + +// commentExists returns whether the provided comment ID exists. +func commentExists(idx commentsIndex, commentID uint32) bool { + _, ok := idx.Comments[commentID] + return ok +} + +// commentIDLatest returns the latest comment ID. +func commentIDLatest(idx commentsIndex) uint32 { + var maxID uint32 + for id := range idx.Comments { + if id > maxID { + maxID = id + } + } + return maxID +} + +func (p *commentsPlugin) commentAddSave(ca comments.CommentAdd) ([]byte, error) { + // Prepare blob + be, err := convertBlobEntryFromCommentAdd(ca) + if err != nil { + return nil, err + } + h, err := hex.DecodeString(be.Hash) + if err != nil { + return nil, err + } + b, err := store.Blobify(*be) + if err != nil { + return nil, err + } + + // Prepare tlog args + tlogID := tlogIDFromCommentState(ca.State) + encrypt := encryptFromCommentState(ca.State) + token, err := hex.DecodeString(ca.Token) + if err != nil { + return nil, err + } + + // Save blob + merkles, err := p.tlog.Save(tlogID, token, keyPrefixCommentAdd, + [][]byte{b}, [][]byte{h}, encrypt) + if err != nil { + return nil, err + } + if len(merkles) != 1 { + return nil, fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", + len(merkles)) + } + + return merkles[0], nil +} + +// commentAdds returns the commentAdd for all specified merkle hashes. +func (p *commentsPlugin) commentAdds(s comments.StateT, token []byte, merkles [][]byte) ([]comments.CommentAdd, error) { + // Retrieve blobs + tlogID := tlogIDFromCommentState(s) + _ = tlogID + blobs, err := p.tlog.BlobsByMerkle(tlogID, token, merkles) + if err != nil { + return nil, err + } + if len(blobs) != len(merkles) { + notFound := make([]string, 0, len(blobs)) + for _, v := range merkles { + m := hex.EncodeToString(v) + _, ok := blobs[m] + if !ok { + notFound = append(notFound, m) + } + } + return nil, fmt.Errorf("blobs not found: %v", notFound) + } + + // Decode blobs + adds := make([]comments.CommentAdd, 0, len(blobs)) + for _, v := range blobs { + be, err := store.Deblob(v) + if err != nil { + return nil, err + } + c, err := convertCommentAddFromBlobEntry(*be) + if err != nil { + return nil, err + } + adds = append(adds, *c) + } + + return adds, nil +} + +func (p *commentsPlugin) commentDelSave(cd comments.CommentDel) ([]byte, error) { + // Prepare blob + be, err := convertBlobEntryFromCommentDel(cd) + if err != nil { + return nil, err + } + h, err := hex.DecodeString(be.Hash) + if err != nil { + return nil, err + } + b, err := store.Blobify(*be) + if err != nil { + return nil, err + } + + // Prepare tlog args + tlogID := tlogIDFromCommentState(cd.State) + token, err := hex.DecodeString(cd.Token) + if err != nil { + return nil, err + } + + // Save blob + _ = tlogID + merkles, err := p.tlog.Save(tlogID, token, keyPrefixCommentDel, + [][]byte{b}, [][]byte{h}, false) + if err != nil { + return nil, err + } + if len(merkles) != 1 { + return nil, fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", + len(merkles)) + } + + return merkles[0], nil +} + +func (p *commentsPlugin) commentDels(s comments.StateT, token []byte, merkles [][]byte) ([]comments.CommentDel, error) { + // Retrieve blobs + tlogID := tlogIDFromCommentState(s) + _ = tlogID + blobs, err := p.tlog.BlobsByMerkle(tlogID, token, merkles) + if err != nil { + return nil, err + } + if len(blobs) != len(merkles) { + notFound := make([]string, 0, len(blobs)) + for _, v := range merkles { + m := hex.EncodeToString(v) + _, ok := blobs[m] + if !ok { + notFound = append(notFound, m) + } + } + return nil, fmt.Errorf("blobs not found: %v", notFound) + } + + // Decode blobs + dels := make([]comments.CommentDel, 0, len(blobs)) + for _, v := range blobs { + be, err := store.Deblob(v) + if err != nil { + return nil, err + } + c, err := convertCommentDelFromBlobEntry(*be) + if err != nil { + return nil, err + } + dels = append(dels, *c) + } + + return dels, nil +} + +func (p *commentsPlugin) commentVoteSave(cv comments.CommentVote) ([]byte, error) { + // Prepare blob + be, err := convertBlobEntryFromCommentVote(cv) + if err != nil { + return nil, err + } + h, err := hex.DecodeString(be.Hash) + if err != nil { + return nil, err + } + b, err := store.Blobify(*be) + if err != nil { + return nil, err + } + + // Prepare tlog args + tlogID := tlogIDFromCommentState(cv.State) + token, err := hex.DecodeString(cv.Token) + if err != nil { + return nil, err + } + + // Save blob + _ = tlogID + merkles, err := p.tlog.Save(tlogID, token, keyPrefixCommentVote, + [][]byte{b}, [][]byte{h}, false) + if err != nil { + return nil, err + } + if len(merkles) != 1 { + return nil, fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", + len(merkles)) + } + + return merkles[0], nil +} + +func (p *commentsPlugin) commentVotes(s comments.StateT, token []byte, merkles [][]byte) ([]comments.CommentVote, error) { + // Retrieve blobs + tlogID := tlogIDFromCommentState(s) + _ = tlogID + blobs, err := p.tlog.BlobsByMerkle(tlogID, token, merkles) + if err != nil { + return nil, err + } + if len(blobs) != len(merkles) { + notFound := make([]string, 0, len(blobs)) + for _, v := range merkles { + m := hex.EncodeToString(v) + _, ok := blobs[m] + if !ok { + notFound = append(notFound, m) + } + } + return nil, fmt.Errorf("blobs not found: %v", notFound) + } + + // Decode blobs + votes := make([]comments.CommentVote, 0, len(blobs)) + for _, v := range blobs { + be, err := store.Deblob(v) + if err != nil { + return nil, err + } + c, err := convertCommentVoteFromBlobEntry(*be) + if err != nil { + return nil, err + } + votes = append(votes, *c) + } + + return votes, nil +} + +// comments returns the most recent version of the specified comments. Deleted +// comments are returned with limited data. Comment IDs that do not correspond +// to an actual comment are not included in the returned map. It is the +// responsibility of the caller to ensure a comment is returned for each of the +// provided comment IDs. The comments index that was looked up during this +// process is also returned. +func (p *commentsPlugin) comments(s comments.StateT, token []byte, idx commentsIndex, commentIDs []uint32) (map[uint32]comments.Comment, error) { + // Aggregate the merkle hashes for all records that need to be + // looked up. If a comment has been deleted then the only record + // that will still exist is the comment del record. If the comment + // has not been deleted then the comment add record will need to be + // retrieved for the latest version of the comment. + var ( + merkleAdds = make([][]byte, 0, len(commentIDs)) + merkleDels = make([][]byte, 0, len(commentIDs)) + ) + for _, v := range commentIDs { + cidx, ok := idx.Comments[v] + if !ok { + // Comment does not exist + continue + } + + // Comment del record + if cidx.Del != nil { + merkleDels = append(merkleDels, cidx.Del) + continue + } + + // Comment add record + version := commentVersionLatest(cidx) + merkleAdds = append(merkleAdds, cidx.Adds[version]) + } + + // Get comment add records + adds, err := p.commentAdds(s, token, merkleAdds) + if err != nil { + if errors.Is(err, errRecordNotFound) { + return nil, err + } + return nil, fmt.Errorf("commentAdds: %v", err) + } + if len(adds) != len(merkleAdds) { + return nil, fmt.Errorf("wrong comment adds count; got %v, want %v", + len(adds), len(merkleAdds)) + } + + // Get comment del records + dels, err := p.commentDels(s, token, merkleDels) + if err != nil { + return nil, fmt.Errorf("commentDels: %v", err) + } + if len(dels) != len(merkleDels) { + return nil, fmt.Errorf("wrong comment dels count; got %v, want %v", + len(dels), len(merkleDels)) + } + + // Prepare comments + cs := make(map[uint32]comments.Comment, len(commentIDs)) + for _, v := range adds { + c := convertCommentFromCommentAdd(v) + cidx, ok := idx.Comments[c.CommentID] + if !ok { + return nil, fmt.Errorf("comment index not found %v", c.CommentID) + } + c.Downvotes, c.Upvotes = calcVoteScore(cidx) + cs[v.CommentID] = c + } + for _, v := range dels { + c := convertCommentFromCommentDel(v) + cs[v.CommentID] = c + } + + return cs, nil +} + +// comment returns the latest version of the provided comment. +func (p *commentsPlugin) comment(s comments.StateT, token []byte, idx commentsIndex, commentID uint32) (*comments.Comment, error) { + cs, err := p.comments(s, token, idx, []uint32{commentID}) + if err != nil { + return nil, fmt.Errorf("comments: %v", err) + } + c, ok := cs[commentID] + if !ok { + return nil, fmt.Errorf("comment not found") + } + return &c, nil +} + +func (p *commentsPlugin) timestamp(s comments.StateT, token []byte, merkle []byte) (*comments.Timestamp, error) { + // Get timestamp + tlogID := tlogIDFromCommentState(s) + _ = tlogID + t, err := p.tlog.Timestamp(tlogID, token, merkle) + if err != nil { + return nil, fmt.Errorf("timestamp %x %x: %v", token, merkle, err) + } + + // Convert response + proofs := make([]comments.Proof, 0, len(t.Proofs)) + for _, v := range t.Proofs { + proofs = append(proofs, comments.Proof{ + Type: v.Type, + Digest: v.Digest, + MerkleRoot: v.MerkleRoot, + MerklePath: v.MerklePath, + ExtraData: v.ExtraData, + }) + } + return &comments.Timestamp{ + Data: t.Data, + Digest: t.Digest, + TxID: t.TxID, + MerkleRoot: t.MerkleRoot, + Proofs: proofs, + }, nil +} + +func (p *commentsPlugin) cmdNew(payload string) (string, error) { + log.Tracef("comments cmdNew: %v", payload) + + // Decode payload + n, err := comments.DecodeNew([]byte(payload)) + if err != nil { + return "", err + } + + // Verify state + switch n.State { + case comments.StateUnvetted, comments.StateVetted: + // Allowed; continue + default: + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusStateInvalid), + } + } + + // Verify token + token, err := tokenDecodeAnyLength(n.Token) + if err != nil { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), + } + } + + // Verify signature + msg := strconv.Itoa(int(n.State)) + n.Token + + strconv.FormatUint(uint64(n.ParentID), 10) + n.Comment + err = util.VerifySignature(n.Signature, n.PublicKey, msg) + if err != nil { + return "", convertCommentsErrorFromSignatureError(err) + } + + // Verify comment + if len(n.Comment) > comments.PolicyCommentLengthMax { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusCommentTextInvalid), + ErrorContext: []string{"exceeds max length"}, + } + } + + // The comments index must be pulled and updated. The record lock + // must be held for the remainder of this function. + m := p.mutex(n.Token) + m.Lock() + defer m.Unlock() + + // Get comments index + idx, err := p.commentsIndexLocked(n.State, token) + if err != nil { + return "", err + } + + // Verify parent comment exists if set. A parent ID of 0 means that + // this is a base level comment, not a reply to another comment. + if n.ParentID > 0 && !commentExists(*idx, n.ParentID) { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusParentIDInvalid), + ErrorContext: []string{"parent ID comment not found"}, + } + } + + // Setup comment + receipt := p.identity.SignMessage([]byte(n.Signature)) + ca := comments.CommentAdd{ + UserID: n.UserID, + State: n.State, + Token: n.Token, + ParentID: n.ParentID, + Comment: n.Comment, + PublicKey: n.PublicKey, + Signature: n.Signature, + CommentID: commentIDLatest(*idx) + 1, + Version: 1, + Timestamp: time.Now().Unix(), + Receipt: hex.EncodeToString(receipt[:]), + } + + // Save comment + merkleHash, err := p.commentAddSave(ca) + if err != nil { + if errors.Is(err, errRecordNotFound) { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusRecordNotFound), + } + } + return "", fmt.Errorf("commentAddSave: %v", err) + } + + // Update index + idx.Comments[ca.CommentID] = commentIndex{ + Adds: map[uint32][]byte{ + 1: merkleHash, + }, + Del: nil, + Votes: make(map[string][]voteIndex), + } + + // Save index + err = p.commentsIndexSaveLocked(n.State, token, *idx) + if err != nil { + return "", err + } + + log.Debugf("Comment saved to record %v comment ID %v", + ca.Token, ca.CommentID) + + // Return new comment + c, err := p.comment(ca.State, token, *idx, ca.CommentID) + if err != nil { + return "", fmt.Errorf("comment %x %v: %v", token, ca.CommentID, err) + } + + // Prepare reply + nr := comments.NewReply{ + Comment: *c, + } + reply, err := comments.EncodeNewReply(nr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *commentsPlugin) cmdEdit(payload string) (string, error) { + log.Tracef("comments cmdEdit: %v", payload) + + // Decode payload + e, err := comments.DecodeEdit([]byte(payload)) + if err != nil { + return "", err + } + + // Verify state + switch e.State { + case comments.StateUnvetted, comments.StateVetted: + // Allowed; continue + default: + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusStateInvalid), + } + } + + // Verify token + token, err := tokenDecodeAnyLength(e.Token) + if err != nil { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), + } + } + + // Verify signature + msg := strconv.Itoa(int(e.State)) + e.Token + + strconv.FormatUint(uint64(e.ParentID), 10) + e.Comment + err = util.VerifySignature(e.Signature, e.PublicKey, msg) + if err != nil { + return "", convertCommentsErrorFromSignatureError(err) + } + + // Verify comment + if len(e.Comment) > comments.PolicyCommentLengthMax { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusCommentTextInvalid), + ErrorContext: []string{"exceeds max length"}, + } + } + + // The comments index must be pulled and updated. The record lock + // must be held for the remainder of this function. + m := p.mutex(e.Token) + m.Lock() + defer m.Unlock() + + // Get comments index + idx, err := p.commentsIndexLocked(e.State, token) + if err != nil { + return "", err + } + + // Get the existing comment + cs, err := p.comments(e.State, token, *idx, []uint32{e.CommentID}) + if err != nil { + if errors.Is(err, errRecordNotFound) { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusRecordNotFound), + } + } + return "", fmt.Errorf("comments %v: %v", e.CommentID, err) + } + existing, ok := cs[e.CommentID] + if !ok { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusCommentNotFound), + } + } + + // Verify the user ID + if e.UserID != existing.UserID { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusUserUnauthorized), + } + } + + // Verify the parent ID + if e.ParentID != existing.ParentID { + e := fmt.Sprintf("parent id cannot change; got %v, want %v", + e.ParentID, existing.ParentID) + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusParentIDInvalid), + ErrorContext: []string{e}, + } + } + + // Verify comment changes + if e.Comment == existing.Comment { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusCommentTextInvalid), + ErrorContext: []string{"comment did not change"}, + } + } + + // Create a new comment version + receipt := p.identity.SignMessage([]byte(e.Signature)) + ca := comments.CommentAdd{ + UserID: e.UserID, + State: e.State, + Token: e.Token, + ParentID: e.ParentID, + Comment: e.Comment, + PublicKey: e.PublicKey, + Signature: e.Signature, + CommentID: e.CommentID, + Version: existing.Version + 1, + Timestamp: time.Now().Unix(), + Receipt: hex.EncodeToString(receipt[:]), + } + + // Save comment + merkle, err := p.commentAddSave(ca) + if err != nil { + return "", fmt.Errorf("commentSave: %v", err) + } + + // Update index + idx.Comments[ca.CommentID].Adds[ca.Version] = merkle + + // Save index + err = p.commentsIndexSaveLocked(e.State, token, *idx) + if err != nil { + return "", err + } + + log.Debugf("Comment edited on record %v comment ID %v", + ca.Token, ca.CommentID) + + // Return updated comment + c, err := p.comment(e.State, token, *idx, e.CommentID) + if err != nil { + return "", fmt.Errorf("comment %x %v: %v", token, e.CommentID, err) + } + + // Prepare reply + er := comments.EditReply{ + Comment: *c, + } + reply, err := comments.EncodeEditReply(er) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *commentsPlugin) cmdDel(payload string) (string, error) { + log.Tracef("comments cmdDel: %v", payload) + + // Decode payload + d, err := comments.DecodeDel([]byte(payload)) + if err != nil { + return "", err + } + + // Verify state + switch d.State { + case comments.StateUnvetted, comments.StateVetted: + // Allowed; continue + default: + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusStateInvalid), + } + } + + // Verify token + token, err := tokenDecodeAnyLength(d.Token) + if err != nil { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), + } + } + + // Verify signature + msg := strconv.Itoa(int(d.State)) + d.Token + + strconv.FormatUint(uint64(d.CommentID), 10) + d.Reason + err = util.VerifySignature(d.Signature, d.PublicKey, msg) + if err != nil { + return "", convertCommentsErrorFromSignatureError(err) + } + + // The comments index must be pulled and updated. The record lock + // must be held for the remainder of this function. + m := p.mutex(d.Token) + m.Lock() + defer m.Unlock() + + // Get comments index + idx, err := p.commentsIndexLocked(d.State, token) + if err != nil { + return "", err + } + + // Get the existing comment + cs, err := p.comments(d.State, token, *idx, []uint32{d.CommentID}) + if err != nil { + if errors.Is(err, errRecordNotFound) { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusRecordNotFound), + } + } + return "", fmt.Errorf("comments %v: %v", d.CommentID, err) + } + existing, ok := cs[d.CommentID] + if !ok { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusCommentNotFound), + } + } + + // Prepare comment delete + receipt := p.identity.SignMessage([]byte(d.Signature)) + cd := comments.CommentDel{ + State: d.State, + Token: d.Token, + CommentID: d.CommentID, + Reason: d.Reason, + PublicKey: d.PublicKey, + Signature: d.Signature, + ParentID: existing.ParentID, + UserID: existing.UserID, + Timestamp: time.Now().Unix(), + Receipt: hex.EncodeToString(receipt[:]), + } + + // Save comment del + merkle, err := p.commentDelSave(cd) + if err != nil { + return "", fmt.Errorf("commentDelSave: %v", err) + } + + // Update index + cidx, ok := idx.Comments[d.CommentID] + if !ok { + // This should not be possible + e := fmt.Sprintf("comment not found in index: %v", d.CommentID) + panic(e) + } + cidx.Del = merkle + idx.Comments[d.CommentID] = cidx + + // Save index + err = p.commentsIndexSaveLocked(d.State, token, *idx) + if err != nil { + return "", err + } + + // Delete all comment versions + merkles := make([][]byte, 0, len(cidx.Adds)) + for _, v := range cidx.Adds { + merkles = append(merkles, v) + } + tlogID := tlogIDFromCommentState(d.State) + _ = tlogID + err = p.tlog.Del(tlogID, token, merkles) + if err != nil { + return "", fmt.Errorf("del: %v", err) + } + + // Return updated comment + c, err := p.comment(d.State, token, *idx, d.CommentID) + if err != nil { + return "", fmt.Errorf("comment %x %v: %v", token, d.CommentID, err) + } + + // Prepare reply + dr := comments.DelReply{ + Comment: *c, + } + reply, err := comments.EncodeDelReply(dr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +// calcVoteScore returns the vote score for the provided comment index. The +// returned values are the downvotes and upvotes, respectively. +func calcVoteScore(cidx commentIndex) (uint64, uint64) { + // Find the vote score by replaying all existing votes from all + // users. The net effect of a new vote on a comment score depends + // on the previous vote from that uuid. Example, a user upvotes a + // comment that they have already upvoted, the resulting vote score + // is 0 due to the second upvote removing the original upvote. + var upvotes uint64 + var downvotes uint64 + for _, votes := range cidx.Votes { + // Calculate the vote score that this user is contributing. This + // can only ever be -1, 0, or 1. + var score int64 + for _, v := range votes { + vote := int64(v.Vote) + switch { + case score == 0: + // No previous vote. New vote becomes the score. + score = vote + + case score == vote: + // New vote is the same as the previous vote. The vote gets + // removed from the score, making the score 0. + score = 0 + + case score != vote: + // New vote is different than the previous vote. New vote + // becomes the score. + score = vote + } + } + + // Add the net result of all votes from this user to the totals. + switch score { + case 0: + // Nothing to do + case -1: + downvotes++ + case 1: + upvotes++ + default: + // Something went wrong + e := fmt.Errorf("unexpected vote score %v", score) + panic(e) + } + } + + return downvotes, upvotes +} + +func (p *commentsPlugin) cmdVote(payload string) (string, error) { + log.Tracef("comments cmdVote: %v", payload) + + // Decode payload + v, err := comments.DecodeVote([]byte(payload)) + if err != nil { + return "", err + } + + // Verify state + switch v.State { + case comments.StateUnvetted, comments.StateVetted: + // Allowed; continue + default: + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), + } + } + + // Verify token + token, err := tokenDecodeAnyLength(v.Token) + if err != nil { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), + } + } + + // Verify vote + switch v.Vote { + case comments.VoteDownvote, comments.VoteUpvote: + // These are allowed + default: + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusVoteInvalid), + } + } + + // Verify signature + msg := strconv.Itoa(int(v.State)) + v.Token + + strconv.FormatUint(uint64(v.CommentID), 10) + + strconv.FormatInt(int64(v.Vote), 10) + err = util.VerifySignature(v.Signature, v.PublicKey, msg) + if err != nil { + return "", convertCommentsErrorFromSignatureError(err) + } + + // The comments index must be pulled and updated. The record lock + // must be held for the remainder of this function. + m := p.mutex(v.Token) + m.Lock() + defer m.Unlock() + + // Get comments index + idx, err := p.commentsIndexLocked(v.State, token) + if err != nil { + return "", err + } + + // Verify comment exists + cidx, ok := idx.Comments[v.CommentID] + if !ok { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusCommentNotFound), + } + } + + // Verify user has not exceeded max allowed vote changes + uvotes, ok := cidx.Votes[v.UserID] + if !ok { + uvotes = make([]voteIndex, 0) + } + if len(uvotes) > comments.PolicyVoteChangesMax { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusVoteChangesMax), + } + } + + // Verify user is not voting on their own comment + cs, err := p.comments(v.State, token, *idx, []uint32{v.CommentID}) + if err != nil { + if errors.Is(err, errRecordNotFound) { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusRecordNotFound), + } + } + return "", fmt.Errorf("comments %v: %v", v.CommentID, err) + } + c, ok := cs[v.CommentID] + if !ok { + return "", fmt.Errorf("comment not found %v", v.CommentID) + } + if v.UserID == c.UserID { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusVoteInvalid), + ErrorContext: []string{"user cannot vote on their own comment"}, + } + } + + // Prepare comment vote + receipt := p.identity.SignMessage([]byte(v.Signature)) + cv := comments.CommentVote{ + State: v.State, + UserID: v.UserID, + Token: v.Token, + CommentID: v.CommentID, + Vote: v.Vote, + PublicKey: v.PublicKey, + Signature: v.Signature, + Timestamp: time.Now().Unix(), + Receipt: hex.EncodeToString(receipt[:]), + } + + // Save comment vote + merkle, err := p.commentVoteSave(cv) + if err != nil { + return "", fmt.Errorf("commentVoteSave: %v", err) + } + + // Add vote to the comment index + votes, ok := cidx.Votes[cv.UserID] + if !ok { + votes = make([]voteIndex, 0, 1) + } + votes = append(votes, voteIndex{ + Vote: cv.Vote, + Merkle: merkle, + }) + cidx.Votes[cv.UserID] = votes + + // Update the comments index + idx.Comments[cv.CommentID] = cidx + + // Save index + err = p.commentsIndexSaveLocked(cv.State, token, *idx) + if err != nil { + return "", err + } + + // Calculate the new vote scores + downvotes, upvotes := calcVoteScore(cidx) + + // Prepare reply + vr := comments.VoteReply{ + Downvotes: downvotes, + Upvotes: upvotes, + Timestamp: cv.Timestamp, + Receipt: cv.Receipt, + } + reply, err := comments.EncodeVoteReply(vr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *commentsPlugin) cmdGet(payload string) (string, error) { + log.Tracef("comments cmdGet: %v", payload) + + // Decode payload + g, err := comments.DecodeGet([]byte(payload)) + if err != nil { + return "", err + } + + // Verify state + switch g.State { + case comments.StateUnvetted, comments.StateVetted: + // Allowed; continue + default: + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), + } + } + + // Verify token + token, err := tokenDecodeAnyLength(g.Token) + if err != nil { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), + } + } + + // Get comments index + idx, err := p.commentsIndex(g.State, token) + if err != nil { + return "", err + } + + // Get comments + cs, err := p.comments(g.State, token, *idx, g.CommentIDs) + if err != nil { + if errors.Is(err, errRecordNotFound) { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusRecordNotFound), + } + } + return "", fmt.Errorf("comments: %v", err) + } + + // Prepare reply + gr := comments.GetReply{ + Comments: cs, + } + reply, err := comments.EncodeGetReply(gr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *commentsPlugin) cmdGetAll(payload string) (string, error) { + log.Tracef("comments cmdGetAll: %v", payload) + + // Decode payload + ga, err := comments.DecodeGetAll([]byte(payload)) + if err != nil { + return "", err + } + + // Verify state + switch ga.State { + case comments.StateUnvetted, comments.StateVetted: + // Allowed; continue + default: + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), + } + } + + // Verify token + token, err := tokenDecodeAnyLength(ga.Token) + if err != nil { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), + } + } + + // Get comments index + idx, err := p.commentsIndex(ga.State, token) + if err != nil { + return "", err + } + + // Compile comment IDs + commentIDs := make([]uint32, 0, len(idx.Comments)) + for k := range idx.Comments { + commentIDs = append(commentIDs, k) + } + + // Get comments + c, err := p.comments(ga.State, token, *idx, commentIDs) + if err != nil { + if errors.Is(err, errRecordNotFound) { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusRecordNotFound), + } + } + return "", fmt.Errorf("comments: %v", err) + } + + // Convert comments from a map to a slice + cs := make([]comments.Comment, 0, len(c)) + for _, v := range c { + cs = append(cs, v) + } + + // Order comments by comment ID + sort.SliceStable(cs, func(i, j int) bool { + return cs[i].CommentID < cs[j].CommentID + }) + + // Prepare reply + gar := comments.GetAllReply{ + Comments: cs, + } + reply, err := comments.EncodeGetAllReply(gar) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *commentsPlugin) cmdGetVersion(payload string) (string, error) { + log.Tracef("comments cmdGetVersion: %v", payload) + + // Decode payload + gv, err := comments.DecodeGetVersion([]byte(payload)) + if err != nil { + return "", err + } + + // Verify state + switch gv.State { + case comments.StateUnvetted, comments.StateVetted: + // Allowed; continue + default: + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), + } + } + + // Verify token + token, err := tokenDecodeAnyLength(gv.Token) + if err != nil { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), + } + } + + // Get comments index + idx, err := p.commentsIndex(gv.State, token) + if err != nil { + return "", err + } + + // Verify comment exists + cidx, ok := idx.Comments[gv.CommentID] + if !ok { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusCommentNotFound), + } + } + if cidx.Del != nil { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusCommentNotFound), + ErrorContext: []string{"comment has been deleted"}, + } + } + merkle, ok := cidx.Adds[gv.Version] + if !ok { + e := fmt.Sprintf("comment %v does not have version %v", + gv.CommentID, gv.Version) + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusCommentNotFound), + ErrorContext: []string{e}, + } + } + + // Get comment add record + adds, err := p.commentAdds(gv.State, token, [][]byte{merkle}) + if err != nil { + if errors.Is(err, errRecordNotFound) { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusRecordNotFound), + } + } + return "", fmt.Errorf("commentAdds: %v", err) + } + if len(adds) != 1 { + return "", fmt.Errorf("wrong comment adds count; got %v, want 1", + len(adds)) + } + + // Convert to a comment + c := convertCommentFromCommentAdd(adds[0]) + c.Downvotes, c.Upvotes = calcVoteScore(cidx) + + // Prepare reply + gvr := comments.GetVersionReply{ + Comment: c, + } + reply, err := comments.EncodeGetVersionReply(gvr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *commentsPlugin) cmdCount(payload string) (string, error) { + log.Tracef("comments cmdCount: %v", payload) + + // Decode payload + c, err := comments.DecodeCount([]byte(payload)) + if err != nil { + return "", err + } + + // Verify state + switch c.State { + case comments.StateUnvetted, comments.StateVetted: + // Allowed; continue + default: + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), + } + } + + // Verify token + token, err := tokenDecodeAnyLength(c.Token) + if err != nil { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), + } + } + + // Get comments index + idx, err := p.commentsIndex(c.State, token) + if err != nil { + return "", err + } + + // Prepare reply + cr := comments.CountReply{ + Count: uint64(len(idx.Comments)), + } + reply, err := comments.EncodeCountReply(cr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *commentsPlugin) cmdVotes(payload string) (string, error) { + log.Tracef("comments cmdVotes: %v", payload) + + // Decode payload + v, err := comments.DecodeVotes([]byte(payload)) + if err != nil { + return "", err + } + + // Verify state + switch v.State { + case comments.StateUnvetted, comments.StateVetted: + // Allowed; continue + default: + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), + } + } + + // Verify token + token, err := tokenDecodeAnyLength(v.Token) + if err != nil { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), + } + } + + // Get comments index + idx, err := p.commentsIndex(v.State, token) + if err != nil { + return "", err + } + + // Compile the comment vote merkles for all votes that were cast + // by the specified user. + merkles := make([][]byte, 0, 256) + for _, cidx := range idx.Comments { + voteIdxs, ok := cidx.Votes[v.UserID] + if !ok { + // User has not cast any votes for this comment + continue + } + + // User has cast votes on this comment + for _, vidx := range voteIdxs { + merkles = append(merkles, vidx.Merkle) + } + } + + // Lookup votes + votes, err := p.commentVotes(v.State, token, merkles) + if err != nil { + if errors.Is(err, errRecordNotFound) { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusRecordNotFound), + } + } + return "", fmt.Errorf("commentVotes: %v", err) + } + + // Prepare reply + vr := comments.VotesReply{ + Votes: votes, + } + reply, err := comments.EncodeVotesReply(vr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *commentsPlugin) cmdTimestamps(payload string) (string, error) { + log.Tracef("comments cmdVotes: %v", payload) + + // Decode payload + var t comments.Timestamps + err := json.Unmarshal([]byte(payload), &t) + if err != nil { + return "", err + } + + // Verify state + switch t.State { + case comments.StateUnvetted, comments.StateVetted: + // Allowed; continue + default: + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), + } + } + + // Verify token + token, err := tokenDecodeAnyLength(t.Token) + if err != nil { + return "", backend.PluginUserError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorStatusTokenInvalid), + } + } + + // Get comments index + idx, err := p.commentsIndex(t.State, token) + if err != nil { + return "", err + } + + // If no comment IDs were given then we need to return the + // timestamps for all comments. + if len(t.CommentIDs) == 0 { + commentIDs := make([]uint32, 0, len(idx.Comments)) + for k := range idx.Comments { + commentIDs = append(commentIDs, k) + } + t.CommentIDs = commentIDs + } + + // Get timestamps + cmts := make(map[uint32][]comments.Timestamp, len(t.CommentIDs)) + votes := make(map[uint32][]comments.Timestamp, len(t.CommentIDs)) + for _, commentID := range t.CommentIDs { + cidx, ok := idx.Comments[commentID] + if !ok { + // Comment ID does not exist. Skip it. + continue + } + + // Get timestamps for adds + ts := make([]comments.Timestamp, 0, len(cidx.Adds)+1) + for _, v := range cidx.Adds { + t, err := p.timestamp(t.State, token, v) + if err != nil { + return "", err + } + ts = append(ts, *t) + } + + // Get timestamp for del + if cidx.Del != nil { + t, err := p.timestamp(t.State, token, cidx.Del) + if err != nil { + return "", err + } + ts = append(ts, *t) + } + + // Save timestamps + cmts[commentID] = ts + + // Only get the comment vote timestamps if specified + if !t.IncludeVotes { + continue + } + + // Get timestamps for votes + ts = make([]comments.Timestamp, 0, len(cidx.Votes)) + for _, votes := range cidx.Votes { + for _, v := range votes { + t, err := p.timestamp(t.State, token, v.Merkle) + if err != nil { + return "", err + } + ts = append(ts, *t) + } + } + + // Save timestamps + votes[commentID] = ts + } + + // Prepare reply + ts := comments.TimestampsReply{ + Comments: cmts, + Votes: votes, + } + reply, err := json.Marshal(ts) + if err != nil { + return "", err + } + + return string(reply), nil +} + +// Cmd executes a plugin command. +// +// This function satisfies the PluginClient interface. +func (p *commentsPlugin) Cmd(cmd, payload string) (string, error) { + log.Tracef("Cmd: %v %v", cmd, payload) + + switch cmd { + case comments.CmdNew: + return p.cmdNew(payload) + case comments.CmdEdit: + return p.cmdEdit(payload) + case comments.CmdDel: + return p.cmdDel(payload) + case comments.CmdVote: + return p.cmdVote(payload) + case comments.CmdGet: + return p.cmdGet(payload) + case comments.CmdGetAll: + return p.cmdGetAll(payload) + case comments.CmdGetVersion: + return p.cmdGetVersion(payload) + case comments.CmdCount: + return p.cmdCount(payload) + case comments.CmdVotes: + return p.cmdVotes(payload) + case comments.CmdTimestamps: + return p.cmdTimestamps(payload) + } + + return "", backend.ErrPluginCmdInvalid +} + +// Hook executes a plugin hook. +// +// This function satisfies the PluginClient interface. +func (p *commentsPlugin) Hook(h plugins.HookT, payload string) error { + log.Tracef("Hook: %v", plugins.Hooks[h]) + + return nil +} + +// Fsck performs a plugin filesystem check. +// +// This function satisfies the PluginClient interface. +func (p *commentsPlugin) Fsck() error { + log.Tracef("Fsck") + + // TODO Make sure CommentDel blobs were actually deleted + + return nil +} + +// Setup performs any plugin setup work that needs to be done. +// +// This function satisfies the PluginClient interface. +func (p *commentsPlugin) Setup() error { + log.Tracef("Setup") + + return nil +} + +// New returns a new comments plugin. +func New(backend backend.Backend, tlog plugins.TlogClient, settings []backend.PluginSetting, id *identity.FullIdentity) (*commentsPlugin, error) { + // Unpack plugin settings + var ( + dataDir string + ) + for _, v := range settings { + switch v.Key { + case plugins.PluginSettingDataDir: + dataDir = v.Value + default: + return nil, fmt.Errorf("invalid plugin setting '%v'", v.Key) + } + } + + // Verify plugin settings + switch { + case dataDir == "": + return nil, fmt.Errorf("plugin setting not found: %v", + plugins.PluginSettingDataDir) + } + + // Create the plugin data directory + dataDir = filepath.Join(dataDir, comments.ID) + err := os.MkdirAll(dataDir, 0700) + if err != nil { + return nil, err + } + + return &commentsPlugin{ + backend: backend, + tlog: tlog, + identity: id, + dataDir: dataDir, + mutexes: make(map[string]*sync.Mutex), + }, nil +} diff --git a/politeiad/backend/tlogbe/plugins/comments/log.go b/politeiad/backend/tlogbe/plugins/comments/log.go new file mode 100644 index 000000000..f1d18be7c --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/comments/log.go @@ -0,0 +1,25 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package comments + +import "github.com/decred/slog" + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log = slog.Disabled + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = slog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using slog. +func UseLogger(logger slog.Logger) { + log = logger +} diff --git a/politeiad/backend/tlogbe/plugins/plugins.go b/politeiad/backend/tlogbe/plugins/plugins.go new file mode 100644 index 000000000..f923fa972 --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/plugins.go @@ -0,0 +1,149 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package plugins + +import ( + "github.com/decred/politeia/politeiad/backend" +) + +const ( + // PluginSettingDataDir is the PluginSetting key for the plugin + // data directory. + PluginSettingDataDir = "datadir" + + // Tlog IDs + TlogIDUnvetted = "unvetted" + TlogIDVetted = "vetted" +) + +// TlogClient provides an API for the plugins to interact with the tlog backend +// instances. Plugins are allowed to save, delete, and get plugin data to/from +// the tlog backend. Editing plugin data is not allowed. +type TlogClient interface { + // Save saves the provided blobs to the tlog backend. Note, hashes + // contains the hashes of the data encoded in the blobs. The hashes + // must share the same ordering as the blobs. + Save(tlogID string, token []byte, keyPrefix string, + blobs, hashes [][]byte, encrypt bool) ([][]byte, error) + + // Del deletes the blobs that correspond to the provided merkle + // leaf hashes. + Del(tlogID string, token []byte, merkleLeafHashes [][]byte) error + + // MerklesByKeyPrefix returns the merkle root hashes for all blobs + // that match the key prefix. + MerklesByKeyPrefix(tlogID string, token []byte, + keyPrefix string) ([][]byte, error) + + // BlobsByMerkle returns the blobs with the provided merkle leaf + // hashes. If a blob does not exist it will not be included in the + // returned map. + BlobsByMerkle(tlogID string, token []byte, + merkleLeafHashes [][]byte) (map[string][]byte, error) + + // BlobsByKeyPrefix returns all blobs that match the key prefix. + BlobsByKeyPrefix(tlogID string, token []byte, + keyPrefix string) ([][]byte, error) + + // Timestamp returns the timestamp for a data blob that corresponds + // to the provided merkle leaf hash. + Timestamp(tlogID string, token []byte, + merkleLeafHash []byte) (*backend.Timestamp, error) +} + +type HookT int + +const ( + // Plugin hooks + HookInvalid HookT = 0 + HookNewRecordPre HookT = 1 + HookNewRecordPost HookT = 2 + HookEditRecordPre HookT = 3 + HookEditRecordPost HookT = 4 + HookEditMetadataPre HookT = 5 + HookEditMetadataPost HookT = 6 + HookSetRecordStatusPre HookT = 7 + HookSetRecordStatusPost HookT = 8 + HookPluginPre HookT = 9 + HookPluginPost HookT = 10 +) + +var ( + // Hooks contains human readable descriptions of the plugin hooks. + Hooks = map[HookT]string{ + HookNewRecordPre: "new record pre", + HookNewRecordPost: "new record post", + HookEditRecordPre: "edit record pre", + HookEditRecordPost: "edit record post", + HookEditMetadataPre: "edit metadata pre", + HookEditMetadataPost: "edit metadata post", + HookSetRecordStatusPre: "set record status pre", + HookSetRecordStatusPost: "set record status post", + HookPluginPre: "plugin pre", + HookPluginPost: "plugin post", + } +) + +// HookNewRecord is the payload for the new record hooks. +type HookNewRecord struct { + Metadata []backend.MetadataStream `json:"metadata"` + Files []backend.File `json:"files"` + + // RecordMetadata will only be present on the post new record hook. + // This is because the record metadata requires the creation of a + // trillian tree and the pre new record hook should execute before + // any politeiad state is changed in case of validation errors. + RecordMetadata *backend.RecordMetadata `json:"recordmetadata"` +} + +// HookEditRecord is the payload for the edit record hooks. +type HookEditRecord struct { + // Current record + Current backend.Record `json:"record"` + + // Updated fields + RecordMetadata backend.RecordMetadata `json:"recordmetadata"` + MDAppend []backend.MetadataStream `json:"mdappend"` + MDOverwrite []backend.MetadataStream `json:"mdoverwrite"` + FilesAdd []backend.File `json:"filesadd"` + FilesDel []string `json:"filesdel"` +} + +// HookEditMetadata is the payload for the edit metadata hooks. +type HookEditMetadata struct { + // Current record + Current backend.Record `json:"record"` + + // Updated fields + MDAppend []backend.MetadataStream `json:"mdappend"` + MDOverwrite []backend.MetadataStream `json:"mdoverwrite"` +} + +// HookSetRecordStatus is the payload for the set record status hooks. +type HookSetRecordStatus struct { + // Current record + Current backend.Record `json:"record"` + + // Updated fields + RecordMetadata backend.RecordMetadata `json:"recordmetadata"` + MDAppend []backend.MetadataStream `json:"mdappend"` + MDOverwrite []backend.MetadataStream `json:"mdoverwrite"` +} + +// PluginClient provides an API for the tlog backend to use when interacting +// with plugins. All tlogbe plugins must implement the pluginClient interface. +type PluginClient interface { + // Setup performs any required plugin setup. + Setup() error + + // Cmd executes the provided plugin command. + Cmd(cmd, payload string) (string, error) + + // Hook executes the provided plugin hook. + Hook(h HookT, payload string) error + + // Fsck performs a plugin file system check. + Fsck() error +} From acb679553a4de70e5c27fba3c4fc88b60ce1099d Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 14 Jan 2021 13:47:57 -0600 Subject: [PATCH 218/449] multi: Package tlog. --- politeiad/api/v1/v1.go | 14 +- politeiad/backend/backend.go | 108 +- politeiad/backend/gitbe/gitbe.go | 84 +- politeiad/backend/tlogbe/comments.go | 2017 ----------------- politeiad/backend/tlogbe/comments_test.go | 704 ------ politeiad/backend/tlogbe/pluginclient.go | 154 -- .../tlogbe/plugins/comments/comments.go | 4 +- .../tlogbe/plugins/comments/comments_test.go | 749 ++++++ .../tlogbe/{ => plugins/dcrdata}/dcrdata.go | 4 +- .../backend/tlogbe/{ => plugins/pi}/pi.go | 2 +- .../tlogbe/{ => plugins/pi}/pi_test.go | 21 +- politeiad/backend/tlogbe/plugins/plugins.go | 44 +- .../{ => plugins/ticketvote}/ticketvote.go | 2 +- politeiad/backend/tlogbe/testing.go | 479 +--- politeiad/backend/tlogbe/{ => tlog}/anchor.go | 14 +- .../backend/tlogbe/{ => tlog}/dcrtime.go | 4 +- .../tlogbe/{ => tlog}/encryptionkey.go | 2 +- politeiad/backend/tlogbe/tlog/log.go | 25 + politeiad/backend/tlogbe/tlog/plugin.go | 190 ++ politeiad/backend/tlogbe/tlog/testing.go | 39 + .../backend/tlogbe/{ => tlog}/timestamp.go | 2 +- politeiad/backend/tlogbe/{ => tlog}/tlog.go | 492 +--- politeiad/backend/tlogbe/tlog/tlogclient.go | 350 +++ .../tlogbe/{ => tlog}/trillianclient.go | 26 +- politeiad/backend/tlogbe/tlogbe.go | 544 +++-- politeiad/backend/tlogbe/tlogbe_test.go | 341 +++ politeiad/backend/tlogbe/tlogclient.go | 70 +- 27 files changed, 2310 insertions(+), 4175 deletions(-) delete mode 100644 politeiad/backend/tlogbe/comments.go delete mode 100644 politeiad/backend/tlogbe/comments_test.go delete mode 100644 politeiad/backend/tlogbe/pluginclient.go create mode 100644 politeiad/backend/tlogbe/plugins/comments/comments_test.go rename politeiad/backend/tlogbe/{ => plugins/dcrdata}/dcrdata.go (99%) rename politeiad/backend/tlogbe/{ => plugins/pi}/pi.go (99%) rename politeiad/backend/tlogbe/{ => plugins/pi}/pi_test.go (94%) rename politeiad/backend/tlogbe/{ => plugins/ticketvote}/ticketvote.go (99%) rename politeiad/backend/tlogbe/{ => tlog}/anchor.go (98%) rename politeiad/backend/tlogbe/{ => tlog}/dcrtime.go (98%) rename politeiad/backend/tlogbe/{ => tlog}/encryptionkey.go (99%) create mode 100644 politeiad/backend/tlogbe/tlog/log.go create mode 100644 politeiad/backend/tlogbe/tlog/plugin.go create mode 100644 politeiad/backend/tlogbe/tlog/testing.go rename politeiad/backend/tlogbe/{ => tlog}/timestamp.go (99%) rename politeiad/backend/tlogbe/{ => tlog}/tlog.go (79%) create mode 100644 politeiad/backend/tlogbe/tlog/tlogclient.go rename politeiad/backend/tlogbe/{ => tlog}/trillianclient.go (97%) diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index cb16d1e13..2784dcbc0 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -29,16 +29,16 @@ const ( UpdateVettedMetadataRoute = "/v1/updatevettedmd/" // Update vetted metadata GetUnvettedRoute = "/v1/getunvetted/" // Retrieve unvetted record GetVettedRoute = "/v1/getvetted/" // Retrieve vetted record - GetUnvettedTimestampsRoute = "/v1/getunvettedts" - GetVettedTimestampsRoute = "/v1/getvettedts" + GetUnvettedTimestampsRoute = "/v1/getunvettedts/" + GetVettedTimestampsRoute = "/v1/getvettedts/" InventoryByStatusRoute = "/v1/inventorybystatus/" // Auth required - InventoryRoute = "/v1/inventory/" // Inventory records - SetUnvettedStatusRoute = "/v1/setunvettedstatus/" // Set unvetted status - SetVettedStatusRoute = "/v1/setvettedstatus/" // Set vetted status - PluginCommandRoute = "/v1/plugin/" // Send a command to a plugin - PluginInventoryRoute = PluginCommandRoute + "inventory/" // Inventory all plugins + InventoryRoute = "/v1/inventory/" // Inventory records + SetUnvettedStatusRoute = "/v1/setunvettedstatus/" // Set unvetted status + SetVettedStatusRoute = "/v1/setvettedstatus/" // Set vetted status + PluginCommandRoute = "/v1/plugin/" // Send a command to a plugin + PluginInventoryRoute = "/v1/plugin/inventory/" // Inventory all plugins ChallengeSize = 32 // Size of challenge token in bytes diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index 282f81c7b..87a25b59c 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -61,10 +61,12 @@ type ContentVerificationError struct { ErrorContext []string } +// Error satisfies the error interface. func (c ContentVerificationError) Error() string { return fmt.Sprintf("%v: %v", v1.ErrorStatus[c.ErrorCode], c.ErrorContext) } +// File represents a record file. type File struct { Name string `json:"name"` // Basename of the file MIME string `json:"mime"` // MIME type @@ -72,6 +74,7 @@ type File struct { Payload string `json:"payload"` // base64 encoded file } +// MDStatusT represents the status of a backend record. type MDStatusT int const ( @@ -124,6 +127,8 @@ func (e PluginUserError) Error() string { // RecordMetadata is the metadata of a record. const VersionRecordMD = 1 +// RecordMetadata represents metadata that is created by the backend on record +// submission and updates. type RecordMetadata struct { Version uint64 `json:"version"` // Version of the scruture Iteration uint64 `json:"iteration"` // Iteration count of record @@ -133,8 +138,7 @@ type RecordMetadata struct { Token string `json:"token"` // Record authentication token, hex encoded } -// MetadataStream describes a single metada stream. The ID determines how and -// where it is stored. +// MetadataStream describes a single metada stream. type MetadataStream struct { ID uint64 `json:"id"` // Stream identity Payload string `json:"payload"` // String encoded metadata @@ -186,7 +190,7 @@ type RecordTimestamps struct { Files map[string]Timestamp // [filename]Timestamp } -// PluginSettings +// PluginSettings are used to specify settings for a plugin at runtime. type PluginSetting struct { Key string // Name of setting Value string // Value of setting @@ -212,71 +216,101 @@ type InventoryByStatus struct { Vetted map[MDStatusT][]string } +// Backend provides an API for creating and editing records. When a record is +// first submitted it is considered to be an unvetted, i.e. non-public, record. +// Once the status of the record is updated to a public status, the record is +// considered to be vetted. type Backend interface { // Create new record New([]MetadataStream, []File) (*RecordMetadata, error) - // Update unvetted record (token, mdAppend, mdOverwrite, fAdd, fDelete) - UpdateUnvettedRecord([]byte, []MetadataStream, []MetadataStream, []File, - []string) (*Record, error) + // Update unvetted record + UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite []MetadataStream, + filesAdd []File, filesDel []string) (*Record, error) - // Update vetted record (token, mdAppend, mdOverwrite, fAdd, fDelete) - UpdateVettedRecord([]byte, []MetadataStream, []MetadataStream, []File, - []string) (*Record, error) + // Update vetted record + UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []MetadataStream, + filesAdd []File, filesDel []string) (*Record, error) - // Update unvetted metadata (token, mdAppend, mdOverwrite) - UpdateUnvettedMetadata([]byte, []MetadataStream, - []MetadataStream) error + // Update unvetted metadata + UpdateUnvettedMetadata(token []byte, mdAppend, + mdOverwrite []MetadataStream) error - // Update vetted metadata (token, mdAppend, mdOverwrite) - UpdateVettedMetadata([]byte, []MetadataStream, - []MetadataStream) error + // Update vetted metadata + UpdateVettedMetadata(token []byte, mdAppend, + mdOverwrite []MetadataStream) error // Set unvetted record status - SetUnvettedStatus([]byte, MDStatusT, []MetadataStream, - []MetadataStream) (*Record, error) + SetUnvettedStatus(token []byte, s MDStatusT, mdAppend, + mdOverwrite []MetadataStream) (*Record, error) // Set vetted record status - SetVettedStatus([]byte, MDStatusT, []MetadataStream, - []MetadataStream) (*Record, error) + SetVettedStatus(token []byte, s MDStatusT, mdAppend, + mdOverwrite []MetadataStream) (*Record, error) // Check if an unvetted record exists - UnvettedExists([]byte) bool + UnvettedExists(token []byte) bool // Check if a vetted record exists - VettedExists([]byte) bool + VettedExists(token []byte) bool // Get unvetted record - GetUnvetted([]byte, string) (*Record, error) + GetUnvetted(token []byte, version string) (*Record, error) // Get vetted record - GetVetted([]byte, string) (*Record, error) - - // Get unvetted record content timestamps - GetUnvettedTimestamps([]byte, string) (*RecordTimestamps, error) + GetVetted(token []byte, version string) (*Record, error) - // Get vetted record content timestamps - GetVettedTimestamps([]byte, string) (*RecordTimestamps, error) + // Get unvetted record timestamps + GetUnvettedTimestamps(token []byte, + version string) (*RecordTimestamps, error) - // Inventory retrieves various record records. - Inventory(uint, uint, uint, bool, bool) ([]Record, []Record, error) + // Get vetted record timestamps + GetVettedTimestamps(token []byte, + version string) (*RecordTimestamps, error) // InventoryByStatus returns the record tokens of all records in the - // inventory categorized by MDStatusT. + // inventory categorized by MDStatusT InventoryByStatus() (*InventoryByStatus, error) - // Register a plugin with the backend - RegisterPlugin(Plugin) error + // Register an unvetted plugin with the backend + RegisterUnvettedPlugin(Plugin) error + + // Register a vetted plugin with the backend + RegisterVettedPlugin(Plugin) error // Perform any plugin setup that is required - SetupPlugin(pluginID string) error + SetupUnvettedPlugin(pluginID string) error - // Obtain plugin settings - GetPlugins() ([]Plugin, error) + // Perform any plugin setup that is required + SetupVettedPlugin(pluginID string) error + + // Execute a plugin command on an unvetted record + UnvettedPlugin(token []byte, pluginID, cmd, payload string) (string, error) + + // Execute a plugin command on a vetted record + VettedPlugin(token []byte, pluginID, cmd, payload string) (string, error) + + // Get unvetted plugins + GetUnvettedPlugins() []Plugin + + // Get vetted plugins + GetVettedPlugins() []Plugin + + // Inventory retrieves various record records + // + // This method has been DEPRECATED. + Inventory(uint, uint, uint, bool, bool) ([]Record, []Record, error) // Plugin pass-through command + // + // This method has been DEPRECATED. Plugin(pluginID, cmd, cmdID, payload string) (string, error) - // Close performs cleanup of the backend. + // Obtain plugin settings + // + // This method has been DEPRECATED. + GetPlugins() ([]Plugin, error) + + // Close performs cleanup of the backend Close() } diff --git a/politeiad/backend/gitbe/gitbe.go b/politeiad/backend/gitbe/gitbe.go index 15b67c135..abf85f198 100644 --- a/politeiad/backend/gitbe/gitbe.go +++ b/politeiad/backend/gitbe/gitbe.go @@ -2796,27 +2796,6 @@ func (g *gitBackEnd) Inventory(vettedCount, vettedStart, branchCount uint, inclu return pr, br, nil } -// InventoryByStatus is not implemented. -// -// This function satisfies the Backend interface. -func (g *gitBackEnd) InventoryByStatus() (*backend.InventoryByStatus, error) { - return nil, fmt.Errorf("not implemented") -} - -// RegisterPlugin registers a plugin. -func (g *gitBackEnd) RegisterPlugin(p backend.Plugin) error { - log.Tracef("RegisterPlugin: %v", p.ID) - - return nil -} - -// SetupPlugin performs any required plugin setup. -func (g *gitBackEnd) SetupPlugin(pluginID string) error { - log.Tracef("SetupPlugin: %v", pluginID) - - return nil -} - // GetPlugins returns a list of currently supported plugins and their settings. // // GetPlugins satisfies the backend interface. @@ -2861,6 +2840,69 @@ func (g *gitBackEnd) Plugin(pluginID, command, commandID, payload string) (strin return "", fmt.Errorf("invalid payload command") } +// InventoryByStatus has not been not implemented. +// +// This function satisfies the backend.Backend interface. +func (g *gitBackEnd) InventoryByStatus() (*backend.InventoryByStatus, error) { + return nil, fmt.Errorf("not implemented") +} + +// RegisterUnvettedPlugin has not been implemented. +// +// This function satisfies the backend.Backend interface. +func (g *gitBackEnd) RegisterUnvettedPlugin(p backend.Plugin) error { + return fmt.Errorf("not implemented") +} + +// RegisterVettedPlugin has not been implemented. +// +// This function satisfies the backend.Backend interface. +func (g *gitBackEnd) RegisterVettedPlugin(p backend.Plugin) error { + return fmt.Errorf("not implemented") +} + +// SetupUnvettedPlugin has not been implemented. +// +// This function satisfies the backend.Backend interface. +func (g *gitBackEnd) SetupUnvettedPlugin(pluginID string) error { + return fmt.Errorf("not implemented") +} + +// SetupVettedPlugin has not been implemented. +// +// This function satisfies the backend.Backend interface. +func (g *gitBackEnd) SetupVettedPlugin(pluginID string) error { + return fmt.Errorf("not implemented") +} + +// UnvettedPlugin has not been implemented. +// +// This function satisfies the backend.Backend interface. +func (g *gitBackEnd) UnvettedPlugin(token []byte, pluginID, cmd, payload string) (string, error) { + return "", fmt.Errorf("not implemented") +} + +// VettedPlugin has not been implemented. +// +// This function satisfies the backend.Backend interface. +func (g *gitBackEnd) VettedPlugin(token []byte, pluginID, cmd, payload string) (string, error) { + return "", fmt.Errorf("not implemented") +} + +// GetUnvettedPlugins has not been implemented. +// +// This function satisfies the backend.Backend interface. +func (g *gitBackEnd) GetUnvettedPlugins() []backend.Plugin { + return []backend.Plugin{} +} + +// GetVettedPlugins has not been implemented. +// +// This function satisfies the backend.Backend interface. +func (g *gitBackEnd) GetVettedPlugins() []backend.Plugin { + return []backend.Plugin{} +} + // Close shuts down the backend. It obtains the lock and sets the shutdown // boolean to true. All interface functions MUST return with errShutdown if // the backend is shutting down. diff --git a/politeiad/backend/tlogbe/comments.go b/politeiad/backend/tlogbe/comments.go deleted file mode 100644 index 7d5b50b57..000000000 --- a/politeiad/backend/tlogbe/comments.go +++ /dev/null @@ -1,2017 +0,0 @@ -// Copyright (c) 2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package tlogbe - -import ( - "bytes" - "encoding/base64" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "sort" - "strconv" - "strings" - "sync" - "time" - - "github.com/decred/politeia/politeiad/api/v1/identity" - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/store" - "github.com/decred/politeia/politeiad/plugins/comments" - "github.com/decred/politeia/util" -) - -// TODO holding the lock before verifying the token can allow the mutexes to -// be spammed. Create an infinite amount of them with invalid tokens. The fix -// is to check if the record exists in the mutexes function to ensure a token -// is valid before holding the lock on it. This is where we can return a -// record doesn't exist user error too. -// TODO prevent duplicate comments -// TODO upvoting a comment twice in the same second causes a duplicate leaf -// error which causes a 500. Solution: add the timestamp to the vote index. - -const ( - // Blob entry data descriptors - dataDescriptorCommentAdd = "commentadd" - dataDescriptorCommentDel = "commentdel" - dataDescriptorCommentVote = "commentvote" - - // Prefixes that are appended to key-value store keys before - // storing them in the log leaf ExtraData field. - keyPrefixCommentAdd = "commentadd:" - keyPrefixCommentDel = "commentdel:" - keyPrefixCommentVote = "commentvote:" - - // Filenames of cached data saved to the plugin data dir. Brackets - // are used to indicate a variable that should be replaced in the - // filename. - filenameCommentsIndex = "{state}-{token}-commentsindex.json" -) - -var ( - _ pluginClient = (*commentsPlugin)(nil) -) - -// commentsPlugin is the tlog backend implementation of the comments plugin. -// -// commentsPlugin satisfies the pluginClient interface. -type commentsPlugin struct { - sync.Mutex - backend backend.Backend - tlog tlogClient - - // dataDir is the comments plugin data directory. The only data - // that is stored here is cached data that can be re-created at any - // time by walking the trillian trees. - dataDir string - - // identity contains the full identity that the plugin uses to - // create receipts, i.e. signatures of user provided data that - // prove the backend received and processed a plugin command. - identity *identity.FullIdentity - - // Mutexes contains a mutex for each record. The mutexes are lazy - // loaded. - mutexes map[string]*sync.Mutex // [string]mutex -} - -type voteIndex struct { - Vote comments.VoteT `json:"vote"` - Merkle []byte `json:"merkle"` // Log leaf merkle leaf hash -} - -type commentIndex struct { - Adds map[uint32][]byte `json:"adds"` // [version]merkleLeafHash - Del []byte `json:"del"` // Merkle leaf hash of delete record - - // Votes contains the vote history for each uuid that voted on the - // comment. This data is cached because the effect of a new vote - // on a comment depends on the previous vote from that uuid. - // Example, a user upvotes a comment that they have already - // upvoted, the resulting vote score is 0 due to the second upvote - // removing the original upvote. - Votes map[string][]voteIndex `json:"votes"` // [uuid]votes -} - -// commentsIndex contains the indexes for all comments made on a record. -type commentsIndex struct { - Comments map[uint32]commentIndex `json:"comments"` // [commentID]comment -} - -// mutex returns the mutex for the specified record. -func (p *commentsPlugin) mutex(token string) *sync.Mutex { - p.Lock() - defer p.Unlock() - - m, ok := p.mutexes[token] - if !ok { - // Mutexes is lazy loaded - m = &sync.Mutex{} - p.mutexes[token] = m - } - - return m -} - -// commentsIndexPath accepts full length token or token prefix but always -// uses prefix when generating the comments index path string. -func (p *commentsPlugin) commentsIndexPath(s comments.StateT, token string) (string, error) { - // Use token prefix - t, err := tokenDecodeAnyLength(token) - if err != nil { - return "", err - } - token = tokenPrefix(t) - fn := filenameCommentsIndex - switch s { - case comments.StateUnvetted: - fn = strings.Replace(fn, "{state}", "unvetted", 1) - case comments.StateVetted: - fn = strings.Replace(fn, "{state}", "vetted", 1) - default: - e := fmt.Errorf("unknown comments state: %v", s) - panic(e) - } - fn = strings.Replace(fn, "{token}", token, 1) - return filepath.Join(p.dataDir, fn), nil -} - -// commentsIndexLocked returns the cached commentsIndex for the provided -// record. If a cached commentsIndex does not exist, a new one will be -// returned. -// -// This function must be called WITH the lock held. -func (p *commentsPlugin) commentsIndexLocked(s comments.StateT, token []byte) (*commentsIndex, error) { - fp, err := p.commentsIndexPath(s, hex.EncodeToString(token)) - if err != nil { - return nil, err - } - b, err := ioutil.ReadFile(fp) - if err != nil { - var e *os.PathError - if errors.As(err, &e) && !os.IsExist(err) { - // File does't exist. Return a new commentsIndex instead. - return &commentsIndex{ - Comments: make(map[uint32]commentIndex), - }, nil - } - return nil, err - } - - var idx commentsIndex - err = json.Unmarshal(b, &idx) - if err != nil { - return nil, err - } - - return &idx, nil -} - -// commentsIndex returns the cached commentsIndex for the provided -// record. If a cached commentsIndex does not exist, a new one will be -// returned. -// -// This function must be called WITHOUT the lock held. -func (p *commentsPlugin) commentsIndex(s comments.StateT, token []byte) (*commentsIndex, error) { - m := p.mutex(hex.EncodeToString(token)) - m.Lock() - defer m.Unlock() - - return p.commentsIndexLocked(s, token) -} - -// commentsIndexSaveLocked saves the provided commentsIndex to the comments -// plugin data dir. -// -// This function must be called WITH the lock held. -func (p *commentsPlugin) commentsIndexSaveLocked(s comments.StateT, token []byte, idx commentsIndex) error { - b, err := json.Marshal(idx) - if err != nil { - return err - } - - fp, err := p.commentsIndexPath(s, hex.EncodeToString(token)) - if err != nil { - return err - } - err = ioutil.WriteFile(fp, b, 0664) - if err != nil { - return err - } - - return nil -} - -func tlogIDFromCommentState(s comments.StateT) string { - switch s { - case comments.StateUnvetted: - return tlogIDUnvetted - case comments.StateVetted: - return tlogIDVetted - default: - e := fmt.Sprintf("unknown state %v", s) - panic(e) - } -} - -func encryptFromCommentState(s comments.StateT) bool { - switch s { - case comments.StateUnvetted: - return true - case comments.StateVetted: - return false - default: - e := fmt.Sprintf("unknown state %v", s) - panic(e) - } -} - -func convertCommentsErrorFromSignatureError(err error) backend.PluginUserError { - var e util.SignatureError - var s comments.ErrorStatusT - if errors.As(err, &e) { - switch e.ErrorCode { - case util.ErrorStatusPublicKeyInvalid: - s = comments.ErrorStatusPublicKeyInvalid - case util.ErrorStatusSignatureInvalid: - s = comments.ErrorStatusSignatureInvalid - } - } - return backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(s), - ErrorContext: e.ErrorContext, - } -} - -func convertBlobEntryFromCommentAdd(c comments.CommentAdd) (*store.BlobEntry, error) { - data, err := json.Marshal(c) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorCommentAdd, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func convertBlobEntryFromCommentDel(c comments.CommentDel) (*store.BlobEntry, error) { - data, err := json.Marshal(c) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorCommentDel, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func convertBlobEntryFromCommentVote(c comments.CommentVote) (*store.BlobEntry, error) { - data, err := json.Marshal(c) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorCommentVote, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func convertCommentAddFromBlobEntry(be store.BlobEntry) (*comments.CommentAdd, error) { - // Decode and validate data hint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorCommentAdd { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorCommentAdd) - } - - // Decode data - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - hash, err := hex.DecodeString(be.Hash) - if err != nil { - return nil, fmt.Errorf("decode hash: %v", err) - } - if !bytes.Equal(util.Digest(b), hash) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), hash) - } - var c comments.CommentAdd - err = json.Unmarshal(b, &c) - if err != nil { - return nil, fmt.Errorf("unmarshal CommentAdd: %v", err) - } - - return &c, nil -} - -func convertCommentDelFromBlobEntry(be store.BlobEntry) (*comments.CommentDel, error) { - // Decode and validate data hint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorCommentDel { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorCommentDel) - } - - // Decode data - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - hash, err := hex.DecodeString(be.Hash) - if err != nil { - return nil, fmt.Errorf("decode hash: %v", err) - } - if !bytes.Equal(util.Digest(b), hash) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), hash) - } - var c comments.CommentDel - err = json.Unmarshal(b, &c) - if err != nil { - return nil, fmt.Errorf("unmarshal CommentDel: %v", err) - } - - return &c, nil -} - -func convertCommentVoteFromBlobEntry(be store.BlobEntry) (*comments.CommentVote, error) { - // Decode and validate data hint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorCommentVote { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorCommentVote) - } - - // Decode data - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - hash, err := hex.DecodeString(be.Hash) - if err != nil { - return nil, fmt.Errorf("decode hash: %v", err) - } - if !bytes.Equal(util.Digest(b), hash) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), hash) - } - var cv comments.CommentVote - err = json.Unmarshal(b, &cv) - if err != nil { - return nil, fmt.Errorf("unmarshal CommentVote: %v", err) - } - - return &cv, nil -} - -func convertCommentFromCommentAdd(ca comments.CommentAdd) comments.Comment { - return comments.Comment{ - UserID: ca.UserID, - State: ca.State, - Token: ca.Token, - ParentID: ca.ParentID, - Comment: ca.Comment, - PublicKey: ca.PublicKey, - Signature: ca.Signature, - CommentID: ca.CommentID, - Version: ca.Version, - Timestamp: ca.Timestamp, - Receipt: ca.Receipt, - Downvotes: 0, // Not part of commentAdd data - Upvotes: 0, // Not part of commentAdd data - Deleted: false, - Reason: "", - } -} - -func convertCommentFromCommentDel(cd comments.CommentDel) comments.Comment { - // Score needs to be filled in separately - return comments.Comment{ - UserID: cd.UserID, - State: cd.State, - Token: cd.Token, - ParentID: cd.ParentID, - Comment: "", - Signature: "", - CommentID: cd.CommentID, - Version: 0, - Timestamp: cd.Timestamp, - Receipt: cd.Receipt, - Downvotes: 0, - Upvotes: 0, - Deleted: true, - Reason: cd.Reason, - } -} - -// commentVersionLatest returns the latest comment version. -func commentVersionLatest(cidx commentIndex) uint32 { - var maxVersion uint32 - for version := range cidx.Adds { - if version > maxVersion { - maxVersion = version - } - } - return maxVersion -} - -// commentExists returns whether the provided comment ID exists. -func commentExists(idx commentsIndex, commentID uint32) bool { - _, ok := idx.Comments[commentID] - return ok -} - -// commentIDLatest returns the latest comment ID. -func commentIDLatest(idx commentsIndex) uint32 { - var maxID uint32 - for id := range idx.Comments { - if id > maxID { - maxID = id - } - } - return maxID -} - -func (p *commentsPlugin) commentAddSave(ca comments.CommentAdd) ([]byte, error) { - // Prepare blob - be, err := convertBlobEntryFromCommentAdd(ca) - if err != nil { - return nil, err - } - h, err := hex.DecodeString(be.Hash) - if err != nil { - return nil, err - } - b, err := store.Blobify(*be) - if err != nil { - return nil, err - } - - // Prepare tlog args - tlogID := tlogIDFromCommentState(ca.State) - encrypt := encryptFromCommentState(ca.State) - token, err := hex.DecodeString(ca.Token) - if err != nil { - return nil, err - } - - // Save blob - merkles, err := p.tlog.save(tlogID, token, keyPrefixCommentAdd, - [][]byte{b}, [][]byte{h}, encrypt) - if err != nil { - return nil, err - } - if len(merkles) != 1 { - return nil, fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", - len(merkles)) - } - - return merkles[0], nil -} - -// commentAdds returns the commentAdd for all specified merkle hashes. -func (p *commentsPlugin) commentAdds(s comments.StateT, token []byte, merkles [][]byte) ([]comments.CommentAdd, error) { - // Retrieve blobs - tlogID := tlogIDFromCommentState(s) - blobs, err := p.tlog.blobsByMerkle(tlogID, token, merkles) - if err != nil { - return nil, err - } - if len(blobs) != len(merkles) { - notFound := make([]string, 0, len(blobs)) - for _, v := range merkles { - m := hex.EncodeToString(v) - _, ok := blobs[m] - if !ok { - notFound = append(notFound, m) - } - } - return nil, fmt.Errorf("blobs not found: %v", notFound) - } - - // Decode blobs - adds := make([]comments.CommentAdd, 0, len(blobs)) - for _, v := range blobs { - be, err := store.Deblob(v) - if err != nil { - return nil, err - } - c, err := convertCommentAddFromBlobEntry(*be) - if err != nil { - return nil, err - } - adds = append(adds, *c) - } - - return adds, nil -} - -func (p *commentsPlugin) commentDelSave(cd comments.CommentDel) ([]byte, error) { - // Prepare blob - be, err := convertBlobEntryFromCommentDel(cd) - if err != nil { - return nil, err - } - h, err := hex.DecodeString(be.Hash) - if err != nil { - return nil, err - } - b, err := store.Blobify(*be) - if err != nil { - return nil, err - } - - // Prepare tlog args - tlogID := tlogIDFromCommentState(cd.State) - token, err := hex.DecodeString(cd.Token) - if err != nil { - return nil, err - } - - // Save blob - merkles, err := p.tlog.save(tlogID, token, keyPrefixCommentDel, - [][]byte{b}, [][]byte{h}, false) - if err != nil { - return nil, err - } - if len(merkles) != 1 { - return nil, fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", - len(merkles)) - } - - return merkles[0], nil -} - -func (p *commentsPlugin) commentDels(s comments.StateT, token []byte, merkles [][]byte) ([]comments.CommentDel, error) { - // Retrieve blobs - tlogID := tlogIDFromCommentState(s) - blobs, err := p.tlog.blobsByMerkle(tlogID, token, merkles) - if err != nil { - return nil, err - } - if len(blobs) != len(merkles) { - notFound := make([]string, 0, len(blobs)) - for _, v := range merkles { - m := hex.EncodeToString(v) - _, ok := blobs[m] - if !ok { - notFound = append(notFound, m) - } - } - return nil, fmt.Errorf("blobs not found: %v", notFound) - } - - // Decode blobs - dels := make([]comments.CommentDel, 0, len(blobs)) - for _, v := range blobs { - be, err := store.Deblob(v) - if err != nil { - return nil, err - } - c, err := convertCommentDelFromBlobEntry(*be) - if err != nil { - return nil, err - } - dels = append(dels, *c) - } - - return dels, nil -} - -func (p *commentsPlugin) commentVoteSave(cv comments.CommentVote) ([]byte, error) { - // Prepare blob - be, err := convertBlobEntryFromCommentVote(cv) - if err != nil { - return nil, err - } - h, err := hex.DecodeString(be.Hash) - if err != nil { - return nil, err - } - b, err := store.Blobify(*be) - if err != nil { - return nil, err - } - - // Prepare tlog args - tlogID := tlogIDFromCommentState(cv.State) - token, err := hex.DecodeString(cv.Token) - if err != nil { - return nil, err - } - - // Save blob - merkles, err := p.tlog.save(tlogID, token, keyPrefixCommentVote, - [][]byte{b}, [][]byte{h}, false) - if err != nil { - return nil, err - } - if len(merkles) != 1 { - return nil, fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", - len(merkles)) - } - - return merkles[0], nil -} - -func (p *commentsPlugin) commentVotes(s comments.StateT, token []byte, merkles [][]byte) ([]comments.CommentVote, error) { - // Retrieve blobs - tlogID := tlogIDFromCommentState(s) - blobs, err := p.tlog.blobsByMerkle(tlogID, token, merkles) - if err != nil { - return nil, err - } - if len(blobs) != len(merkles) { - notFound := make([]string, 0, len(blobs)) - for _, v := range merkles { - m := hex.EncodeToString(v) - _, ok := blobs[m] - if !ok { - notFound = append(notFound, m) - } - } - return nil, fmt.Errorf("blobs not found: %v", notFound) - } - - // Decode blobs - votes := make([]comments.CommentVote, 0, len(blobs)) - for _, v := range blobs { - be, err := store.Deblob(v) - if err != nil { - return nil, err - } - c, err := convertCommentVoteFromBlobEntry(*be) - if err != nil { - return nil, err - } - votes = append(votes, *c) - } - - return votes, nil -} - -// comments returns the most recent version of the specified comments. Deleted -// comments are returned with limited data. Comment IDs that do not correspond -// to an actual comment are not included in the returned map. It is the -// responsibility of the caller to ensure a comment is returned for each of the -// provided comment IDs. The comments index that was looked up during this -// process is also returned. -func (p *commentsPlugin) comments(s comments.StateT, token []byte, idx commentsIndex, commentIDs []uint32) (map[uint32]comments.Comment, error) { - // Aggregate the merkle hashes for all records that need to be - // looked up. If a comment has been deleted then the only record - // that will still exist is the comment del record. If the comment - // has not been deleted then the comment add record will need to be - // retrieved for the latest version of the comment. - var ( - merkleAdds = make([][]byte, 0, len(commentIDs)) - merkleDels = make([][]byte, 0, len(commentIDs)) - ) - for _, v := range commentIDs { - cidx, ok := idx.Comments[v] - if !ok { - // Comment does not exist - continue - } - - // Comment del record - if cidx.Del != nil { - merkleDels = append(merkleDels, cidx.Del) - continue - } - - // Comment add record - version := commentVersionLatest(cidx) - merkleAdds = append(merkleAdds, cidx.Adds[version]) - } - - // Get comment add records - adds, err := p.commentAdds(s, token, merkleAdds) - if err != nil { - if errors.Is(err, errRecordNotFound) { - return nil, err - } - return nil, fmt.Errorf("commentAdds: %v", err) - } - if len(adds) != len(merkleAdds) { - return nil, fmt.Errorf("wrong comment adds count; got %v, want %v", - len(adds), len(merkleAdds)) - } - - // Get comment del records - dels, err := p.commentDels(s, token, merkleDels) - if err != nil { - return nil, fmt.Errorf("commentDels: %v", err) - } - if len(dels) != len(merkleDels) { - return nil, fmt.Errorf("wrong comment dels count; got %v, want %v", - len(dels), len(merkleDels)) - } - - // Prepare comments - cs := make(map[uint32]comments.Comment, len(commentIDs)) - for _, v := range adds { - c := convertCommentFromCommentAdd(v) - cidx, ok := idx.Comments[c.CommentID] - if !ok { - return nil, fmt.Errorf("comment index not found %v", c.CommentID) - } - c.Downvotes, c.Upvotes = calcVoteScore(cidx) - cs[v.CommentID] = c - } - for _, v := range dels { - c := convertCommentFromCommentDel(v) - cs[v.CommentID] = c - } - - return cs, nil -} - -// comment returns the latest version of the provided comment. -func (p *commentsPlugin) comment(s comments.StateT, token []byte, idx commentsIndex, commentID uint32) (*comments.Comment, error) { - cs, err := p.comments(s, token, idx, []uint32{commentID}) - if err != nil { - return nil, fmt.Errorf("comments: %v", err) - } - c, ok := cs[commentID] - if !ok { - return nil, fmt.Errorf("comment not found") - } - return &c, nil -} - -func (p *commentsPlugin) timestamp(s comments.StateT, token []byte, merkle []byte) (*comments.Timestamp, error) { - // Get timestamp - tlogID := tlogIDFromCommentState(s) - t, err := p.tlog.timestamp(tlogID, token, merkle) - if err != nil { - return nil, fmt.Errorf("timestamp %x %x: %v", token, merkle, err) - } - - // Convert response - proofs := make([]comments.Proof, 0, len(t.Proofs)) - for _, v := range t.Proofs { - proofs = append(proofs, comments.Proof{ - Type: v.Type, - Digest: v.Digest, - MerkleRoot: v.MerkleRoot, - MerklePath: v.MerklePath, - ExtraData: v.ExtraData, - }) - } - return &comments.Timestamp{ - Data: t.Data, - Digest: t.Digest, - TxID: t.TxID, - MerkleRoot: t.MerkleRoot, - Proofs: proofs, - }, nil -} - -func (p *commentsPlugin) cmdNew(payload string) (string, error) { - log.Tracef("comments cmdNew: %v", payload) - - // Decode payload - n, err := comments.DecodeNew([]byte(payload)) - if err != nil { - return "", err - } - - // Verify state - switch n.State { - case comments.StateUnvetted, comments.StateVetted: - // Allowed; continue - default: - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusStateInvalid), - } - } - - // Verify token - token, err := tokenDecodeAnyLength(n.Token) - if err != nil { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Verify signature - msg := strconv.Itoa(int(n.State)) + n.Token + - strconv.FormatUint(uint64(n.ParentID), 10) + n.Comment - err = util.VerifySignature(n.Signature, n.PublicKey, msg) - if err != nil { - return "", convertCommentsErrorFromSignatureError(err) - } - - // Verify comment - if len(n.Comment) > comments.PolicyCommentLengthMax { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusCommentTextInvalid), - ErrorContext: []string{"exceeds max length"}, - } - } - - // The comments index must be pulled and updated. The record lock - // must be held for the remainder of this function. - m := p.mutex(n.Token) - m.Lock() - defer m.Unlock() - - // Get comments index - idx, err := p.commentsIndexLocked(n.State, token) - if err != nil { - return "", err - } - - // Verify parent comment exists if set. A parent ID of 0 means that - // this is a base level comment, not a reply to another comment. - if n.ParentID > 0 && !commentExists(*idx, n.ParentID) { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusParentIDInvalid), - ErrorContext: []string{"parent ID comment not found"}, - } - } - - // Setup comment - receipt := p.identity.SignMessage([]byte(n.Signature)) - ca := comments.CommentAdd{ - UserID: n.UserID, - State: n.State, - Token: n.Token, - ParentID: n.ParentID, - Comment: n.Comment, - PublicKey: n.PublicKey, - Signature: n.Signature, - CommentID: commentIDLatest(*idx) + 1, - Version: 1, - Timestamp: time.Now().Unix(), - Receipt: hex.EncodeToString(receipt[:]), - } - - // Save comment - merkleHash, err := p.commentAddSave(ca) - if err != nil { - if errors.Is(err, errRecordNotFound) { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusRecordNotFound), - } - } - return "", fmt.Errorf("commentAddSave: %v", err) - } - - // Update index - idx.Comments[ca.CommentID] = commentIndex{ - Adds: map[uint32][]byte{ - 1: merkleHash, - }, - Del: nil, - Votes: make(map[string][]voteIndex), - } - - // Save index - err = p.commentsIndexSaveLocked(n.State, token, *idx) - if err != nil { - return "", err - } - - log.Debugf("Comment saved to record %v comment ID %v", - ca.Token, ca.CommentID) - - // Return new comment - c, err := p.comment(ca.State, token, *idx, ca.CommentID) - if err != nil { - return "", fmt.Errorf("comment %x %v: %v", token, ca.CommentID, err) - } - - // Prepare reply - nr := comments.NewReply{ - Comment: *c, - } - reply, err := comments.EncodeNewReply(nr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *commentsPlugin) cmdEdit(payload string) (string, error) { - log.Tracef("comments cmdEdit: %v", payload) - - // Decode payload - e, err := comments.DecodeEdit([]byte(payload)) - if err != nil { - return "", err - } - - // Verify state - switch e.State { - case comments.StateUnvetted, comments.StateVetted: - // Allowed; continue - default: - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusStateInvalid), - } - } - - // Verify token - token, err := tokenDecodeAnyLength(e.Token) - if err != nil { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Verify signature - msg := strconv.Itoa(int(e.State)) + e.Token + - strconv.FormatUint(uint64(e.ParentID), 10) + e.Comment - err = util.VerifySignature(e.Signature, e.PublicKey, msg) - if err != nil { - return "", convertCommentsErrorFromSignatureError(err) - } - - // Verify comment - if len(e.Comment) > comments.PolicyCommentLengthMax { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusCommentTextInvalid), - ErrorContext: []string{"exceeds max length"}, - } - } - - // The comments index must be pulled and updated. The record lock - // must be held for the remainder of this function. - m := p.mutex(e.Token) - m.Lock() - defer m.Unlock() - - // Get comments index - idx, err := p.commentsIndexLocked(e.State, token) - if err != nil { - return "", err - } - - // Get the existing comment - cs, err := p.comments(e.State, token, *idx, []uint32{e.CommentID}) - if err != nil { - if errors.Is(err, errRecordNotFound) { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusRecordNotFound), - } - } - return "", fmt.Errorf("comments %v: %v", e.CommentID, err) - } - existing, ok := cs[e.CommentID] - if !ok { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusCommentNotFound), - } - } - - // Verify the user ID - if e.UserID != existing.UserID { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusUserUnauthorized), - } - } - - // Verify the parent ID - if e.ParentID != existing.ParentID { - e := fmt.Sprintf("parent id cannot change; got %v, want %v", - e.ParentID, existing.ParentID) - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusParentIDInvalid), - ErrorContext: []string{e}, - } - } - - // Verify comment changes - if e.Comment == existing.Comment { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusCommentTextInvalid), - ErrorContext: []string{"comment did not change"}, - } - } - - // Create a new comment version - receipt := p.identity.SignMessage([]byte(e.Signature)) - ca := comments.CommentAdd{ - UserID: e.UserID, - State: e.State, - Token: e.Token, - ParentID: e.ParentID, - Comment: e.Comment, - PublicKey: e.PublicKey, - Signature: e.Signature, - CommentID: e.CommentID, - Version: existing.Version + 1, - Timestamp: time.Now().Unix(), - Receipt: hex.EncodeToString(receipt[:]), - } - - // Save comment - merkle, err := p.commentAddSave(ca) - if err != nil { - return "", fmt.Errorf("commentSave: %v", err) - } - - // Update index - idx.Comments[ca.CommentID].Adds[ca.Version] = merkle - - // Save index - err = p.commentsIndexSaveLocked(e.State, token, *idx) - if err != nil { - return "", err - } - - log.Debugf("Comment edited on record %v comment ID %v", - ca.Token, ca.CommentID) - - // Return updated comment - c, err := p.comment(e.State, token, *idx, e.CommentID) - if err != nil { - return "", fmt.Errorf("comment %x %v: %v", token, e.CommentID, err) - } - - // Prepare reply - er := comments.EditReply{ - Comment: *c, - } - reply, err := comments.EncodeEditReply(er) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *commentsPlugin) cmdDel(payload string) (string, error) { - log.Tracef("comments cmdDel: %v", payload) - - // Decode payload - d, err := comments.DecodeDel([]byte(payload)) - if err != nil { - return "", err - } - - // Verify state - switch d.State { - case comments.StateUnvetted, comments.StateVetted: - // Allowed; continue - default: - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusStateInvalid), - } - } - - // Verify token - token, err := tokenDecodeAnyLength(d.Token) - if err != nil { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Verify signature - msg := strconv.Itoa(int(d.State)) + d.Token + - strconv.FormatUint(uint64(d.CommentID), 10) + d.Reason - err = util.VerifySignature(d.Signature, d.PublicKey, msg) - if err != nil { - return "", convertCommentsErrorFromSignatureError(err) - } - - // The comments index must be pulled and updated. The record lock - // must be held for the remainder of this function. - m := p.mutex(d.Token) - m.Lock() - defer m.Unlock() - - // Get comments index - idx, err := p.commentsIndexLocked(d.State, token) - if err != nil { - return "", err - } - - // Get the existing comment - cs, err := p.comments(d.State, token, *idx, []uint32{d.CommentID}) - if err != nil { - if errors.Is(err, errRecordNotFound) { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusRecordNotFound), - } - } - return "", fmt.Errorf("comments %v: %v", d.CommentID, err) - } - existing, ok := cs[d.CommentID] - if !ok { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusCommentNotFound), - } - } - - // Prepare comment delete - receipt := p.identity.SignMessage([]byte(d.Signature)) - cd := comments.CommentDel{ - State: d.State, - Token: d.Token, - CommentID: d.CommentID, - Reason: d.Reason, - PublicKey: d.PublicKey, - Signature: d.Signature, - ParentID: existing.ParentID, - UserID: existing.UserID, - Timestamp: time.Now().Unix(), - Receipt: hex.EncodeToString(receipt[:]), - } - - // Save comment del - merkle, err := p.commentDelSave(cd) - if err != nil { - return "", fmt.Errorf("commentDelSave: %v", err) - } - - // Update index - cidx, ok := idx.Comments[d.CommentID] - if !ok { - // This should not be possible - e := fmt.Sprintf("comment not found in index: %v", d.CommentID) - panic(e) - } - cidx.Del = merkle - idx.Comments[d.CommentID] = cidx - - // Save index - err = p.commentsIndexSaveLocked(d.State, token, *idx) - if err != nil { - return "", err - } - - // Delete all comment versions - merkles := make([][]byte, 0, len(cidx.Adds)) - for _, v := range cidx.Adds { - merkles = append(merkles, v) - } - tlogID := tlogIDFromCommentState(d.State) - err = p.tlog.del(tlogID, token, merkles) - if err != nil { - return "", fmt.Errorf("del: %v", err) - } - - // Return updated comment - c, err := p.comment(d.State, token, *idx, d.CommentID) - if err != nil { - return "", fmt.Errorf("comment %x %v: %v", token, d.CommentID, err) - } - - // Prepare reply - dr := comments.DelReply{ - Comment: *c, - } - reply, err := comments.EncodeDelReply(dr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -// calcVoteScore returns the vote score for the provided comment index. The -// returned values are the downvotes and upvotes, respectively. -func calcVoteScore(cidx commentIndex) (uint64, uint64) { - // Find the vote score by replaying all existing votes from all - // users. The net effect of a new vote on a comment score depends - // on the previous vote from that uuid. Example, a user upvotes a - // comment that they have already upvoted, the resulting vote score - // is 0 due to the second upvote removing the original upvote. - var upvotes uint64 - var downvotes uint64 - for _, votes := range cidx.Votes { - // Calculate the vote score that this user is contributing. This - // can only ever be -1, 0, or 1. - var score int64 - for _, v := range votes { - vote := int64(v.Vote) - switch { - case score == 0: - // No previous vote. New vote becomes the score. - score = vote - - case score == vote: - // New vote is the same as the previous vote. The vote gets - // removed from the score, making the score 0. - score = 0 - - case score != vote: - // New vote is different than the previous vote. New vote - // becomes the score. - score = vote - } - } - - // Add the net result of all votes from this user to the totals. - switch score { - case 0: - // Nothing to do - case -1: - downvotes++ - case 1: - upvotes++ - default: - // Something went wrong - e := fmt.Errorf("unexpected vote score %v", score) - panic(e) - } - } - - return downvotes, upvotes -} - -func (p *commentsPlugin) cmdVote(payload string) (string, error) { - log.Tracef("comments cmdVote: %v", payload) - - // Decode payload - v, err := comments.DecodeVote([]byte(payload)) - if err != nil { - return "", err - } - - // Verify state - switch v.State { - case comments.StateUnvetted, comments.StateVetted: - // Allowed; continue - default: - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Verify token - token, err := tokenDecodeAnyLength(v.Token) - if err != nil { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Verify vote - switch v.Vote { - case comments.VoteDownvote, comments.VoteUpvote: - // These are allowed - default: - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusVoteInvalid), - } - } - - // Verify signature - msg := strconv.Itoa(int(v.State)) + v.Token + - strconv.FormatUint(uint64(v.CommentID), 10) + - strconv.FormatInt(int64(v.Vote), 10) - err = util.VerifySignature(v.Signature, v.PublicKey, msg) - if err != nil { - return "", convertCommentsErrorFromSignatureError(err) - } - - // The comments index must be pulled and updated. The record lock - // must be held for the remainder of this function. - m := p.mutex(v.Token) - m.Lock() - defer m.Unlock() - - // Get comments index - idx, err := p.commentsIndexLocked(v.State, token) - if err != nil { - return "", err - } - - // Verify comment exists - cidx, ok := idx.Comments[v.CommentID] - if !ok { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusCommentNotFound), - } - } - - // Verify user has not exceeded max allowed vote changes - uvotes, ok := cidx.Votes[v.UserID] - if !ok { - uvotes = make([]voteIndex, 0) - } - if len(uvotes) > comments.PolicyVoteChangesMax { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusVoteChangesMax), - } - } - - // Verify user is not voting on their own comment - cs, err := p.comments(v.State, token, *idx, []uint32{v.CommentID}) - if err != nil { - if errors.Is(err, errRecordNotFound) { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusRecordNotFound), - } - } - return "", fmt.Errorf("comments %v: %v", v.CommentID, err) - } - c, ok := cs[v.CommentID] - if !ok { - return "", fmt.Errorf("comment not found %v", v.CommentID) - } - if v.UserID == c.UserID { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusVoteInvalid), - ErrorContext: []string{"user cannot vote on their own comment"}, - } - } - - // Prepare comment vote - receipt := p.identity.SignMessage([]byte(v.Signature)) - cv := comments.CommentVote{ - State: v.State, - UserID: v.UserID, - Token: v.Token, - CommentID: v.CommentID, - Vote: v.Vote, - PublicKey: v.PublicKey, - Signature: v.Signature, - Timestamp: time.Now().Unix(), - Receipt: hex.EncodeToString(receipt[:]), - } - - // Save comment vote - merkle, err := p.commentVoteSave(cv) - if err != nil { - return "", fmt.Errorf("commentVoteSave: %v", err) - } - - // Add vote to the comment index - votes, ok := cidx.Votes[cv.UserID] - if !ok { - votes = make([]voteIndex, 0, 1) - } - votes = append(votes, voteIndex{ - Vote: cv.Vote, - Merkle: merkle, - }) - cidx.Votes[cv.UserID] = votes - - // Update the comments index - idx.Comments[cv.CommentID] = cidx - - // Save index - err = p.commentsIndexSaveLocked(cv.State, token, *idx) - if err != nil { - return "", err - } - - // Calculate the new vote scores - downvotes, upvotes := calcVoteScore(cidx) - - // Prepare reply - vr := comments.VoteReply{ - Downvotes: downvotes, - Upvotes: upvotes, - Timestamp: cv.Timestamp, - Receipt: cv.Receipt, - } - reply, err := comments.EncodeVoteReply(vr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *commentsPlugin) cmdGet(payload string) (string, error) { - log.Tracef("comments cmdGet: %v", payload) - - // Decode payload - g, err := comments.DecodeGet([]byte(payload)) - if err != nil { - return "", err - } - - // Verify state - switch g.State { - case comments.StateUnvetted, comments.StateVetted: - // Allowed; continue - default: - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Verify token - token, err := tokenDecodeAnyLength(g.Token) - if err != nil { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Get comments index - idx, err := p.commentsIndex(g.State, token) - if err != nil { - return "", err - } - - // Get comments - cs, err := p.comments(g.State, token, *idx, g.CommentIDs) - if err != nil { - if errors.Is(err, errRecordNotFound) { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusRecordNotFound), - } - } - return "", fmt.Errorf("comments: %v", err) - } - - // Prepare reply - gr := comments.GetReply{ - Comments: cs, - } - reply, err := comments.EncodeGetReply(gr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *commentsPlugin) cmdGetAll(payload string) (string, error) { - log.Tracef("comments cmdGetAll: %v", payload) - - // Decode payload - ga, err := comments.DecodeGetAll([]byte(payload)) - if err != nil { - return "", err - } - - // Verify state - switch ga.State { - case comments.StateUnvetted, comments.StateVetted: - // Allowed; continue - default: - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Verify token - token, err := tokenDecodeAnyLength(ga.Token) - if err != nil { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Get comments index - idx, err := p.commentsIndex(ga.State, token) - if err != nil { - return "", err - } - - // Compile comment IDs - commentIDs := make([]uint32, 0, len(idx.Comments)) - for k := range idx.Comments { - commentIDs = append(commentIDs, k) - } - - // Get comments - c, err := p.comments(ga.State, token, *idx, commentIDs) - if err != nil { - if errors.Is(err, errRecordNotFound) { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusRecordNotFound), - } - } - return "", fmt.Errorf("comments: %v", err) - } - - // Convert comments from a map to a slice - cs := make([]comments.Comment, 0, len(c)) - for _, v := range c { - cs = append(cs, v) - } - - // Order comments by comment ID - sort.SliceStable(cs, func(i, j int) bool { - return cs[i].CommentID < cs[j].CommentID - }) - - // Prepare reply - gar := comments.GetAllReply{ - Comments: cs, - } - reply, err := comments.EncodeGetAllReply(gar) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *commentsPlugin) cmdGetVersion(payload string) (string, error) { - log.Tracef("comments cmdGetVersion: %v", payload) - - // Decode payload - gv, err := comments.DecodeGetVersion([]byte(payload)) - if err != nil { - return "", err - } - - // Verify state - switch gv.State { - case comments.StateUnvetted, comments.StateVetted: - // Allowed; continue - default: - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Verify token - token, err := tokenDecodeAnyLength(gv.Token) - if err != nil { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Get comments index - idx, err := p.commentsIndex(gv.State, token) - if err != nil { - return "", err - } - - // Verify comment exists - cidx, ok := idx.Comments[gv.CommentID] - if !ok { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusCommentNotFound), - } - } - if cidx.Del != nil { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusCommentNotFound), - ErrorContext: []string{"comment has been deleted"}, - } - } - merkle, ok := cidx.Adds[gv.Version] - if !ok { - e := fmt.Sprintf("comment %v does not have version %v", - gv.CommentID, gv.Version) - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusCommentNotFound), - ErrorContext: []string{e}, - } - } - - // Get comment add record - adds, err := p.commentAdds(gv.State, token, [][]byte{merkle}) - if err != nil { - if errors.Is(err, errRecordNotFound) { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusRecordNotFound), - } - } - return "", fmt.Errorf("commentAdds: %v", err) - } - if len(adds) != 1 { - return "", fmt.Errorf("wrong comment adds count; got %v, want 1", - len(adds)) - } - - // Convert to a comment - c := convertCommentFromCommentAdd(adds[0]) - c.Downvotes, c.Upvotes = calcVoteScore(cidx) - - // Prepare reply - gvr := comments.GetVersionReply{ - Comment: c, - } - reply, err := comments.EncodeGetVersionReply(gvr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *commentsPlugin) cmdCount(payload string) (string, error) { - log.Tracef("comments cmdCount: %v", payload) - - // Decode payload - c, err := comments.DecodeCount([]byte(payload)) - if err != nil { - return "", err - } - - // Verify state - switch c.State { - case comments.StateUnvetted, comments.StateVetted: - // Allowed; continue - default: - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Verify token - token, err := tokenDecodeAnyLength(c.Token) - if err != nil { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Get comments index - idx, err := p.commentsIndex(c.State, token) - if err != nil { - return "", err - } - - // Prepare reply - cr := comments.CountReply{ - Count: uint64(len(idx.Comments)), - } - reply, err := comments.EncodeCountReply(cr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *commentsPlugin) cmdVotes(payload string) (string, error) { - log.Tracef("comments cmdVotes: %v", payload) - - // Decode payload - v, err := comments.DecodeVotes([]byte(payload)) - if err != nil { - return "", err - } - - // Verify state - switch v.State { - case comments.StateUnvetted, comments.StateVetted: - // Allowed; continue - default: - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Verify token - token, err := tokenDecodeAnyLength(v.Token) - if err != nil { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Get comments index - idx, err := p.commentsIndex(v.State, token) - if err != nil { - return "", err - } - - // Compile the comment vote merkles for all votes that were cast - // by the specified user. - merkles := make([][]byte, 0, 256) - for _, cidx := range idx.Comments { - voteIdxs, ok := cidx.Votes[v.UserID] - if !ok { - // User has not cast any votes for this comment - continue - } - - // User has cast votes on this comment - for _, vidx := range voteIdxs { - merkles = append(merkles, vidx.Merkle) - } - } - - // Lookup votes - votes, err := p.commentVotes(v.State, token, merkles) - if err != nil { - if errors.Is(err, errRecordNotFound) { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusRecordNotFound), - } - } - return "", fmt.Errorf("commentVotes: %v", err) - } - - // Prepare reply - vr := comments.VotesReply{ - Votes: votes, - } - reply, err := comments.EncodeVotesReply(vr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *commentsPlugin) cmdTimestamps(payload string) (string, error) { - log.Tracef("comments cmdVotes: %v", payload) - - // Decode payload - var t comments.Timestamps - err := json.Unmarshal([]byte(payload), &t) - if err != nil { - return "", err - } - - // Verify state - switch t.State { - case comments.StateUnvetted, comments.StateVetted: - // Allowed; continue - default: - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Verify token - token, err := tokenDecodeAnyLength(t.Token) - if err != nil { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Get comments index - idx, err := p.commentsIndex(t.State, token) - if err != nil { - return "", err - } - - // If no comment IDs were given then we need to return the - // timestamps for all comments. - if len(t.CommentIDs) == 0 { - commentIDs := make([]uint32, 0, len(idx.Comments)) - for k := range idx.Comments { - commentIDs = append(commentIDs, k) - } - t.CommentIDs = commentIDs - } - - // Get timestamps - cmts := make(map[uint32][]comments.Timestamp, len(t.CommentIDs)) - votes := make(map[uint32][]comments.Timestamp, len(t.CommentIDs)) - for _, commentID := range t.CommentIDs { - cidx, ok := idx.Comments[commentID] - if !ok { - // Comment ID does not exist. Skip it. - continue - } - - // Get timestamps for adds - ts := make([]comments.Timestamp, 0, len(cidx.Adds)+1) - for _, v := range cidx.Adds { - t, err := p.timestamp(t.State, token, v) - if err != nil { - return "", err - } - ts = append(ts, *t) - } - - // Get timestamp for del - if cidx.Del != nil { - t, err := p.timestamp(t.State, token, cidx.Del) - if err != nil { - return "", err - } - ts = append(ts, *t) - } - - // Save timestamps - cmts[commentID] = ts - - // Only get the comment vote timestamps if specified - if !t.IncludeVotes { - continue - } - - // Get timestamps for votes - ts = make([]comments.Timestamp, 0, len(cidx.Votes)) - for _, votes := range cidx.Votes { - for _, v := range votes { - t, err := p.timestamp(t.State, token, v.Merkle) - if err != nil { - return "", err - } - ts = append(ts, *t) - } - } - - // Save timestamps - votes[commentID] = ts - } - - // Prepare reply - ts := comments.TimestampsReply{ - Comments: cmts, - Votes: votes, - } - reply, err := json.Marshal(ts) - if err != nil { - return "", err - } - - return string(reply), nil -} - -// cmd executes a plugin command. -// -// This function satisfies the pluginClient interface. -func (p *commentsPlugin) cmd(cmd, payload string) (string, error) { - log.Tracef("comments cmd: %v", cmd) - - switch cmd { - case comments.CmdNew: - return p.cmdNew(payload) - case comments.CmdEdit: - return p.cmdEdit(payload) - case comments.CmdDel: - return p.cmdDel(payload) - case comments.CmdVote: - return p.cmdVote(payload) - case comments.CmdGet: - return p.cmdGet(payload) - case comments.CmdGetAll: - return p.cmdGetAll(payload) - case comments.CmdGetVersion: - return p.cmdGetVersion(payload) - case comments.CmdCount: - return p.cmdCount(payload) - case comments.CmdVotes: - return p.cmdVotes(payload) - case comments.CmdTimestamps: - return p.cmdTimestamps(payload) - } - - return "", backend.ErrPluginCmdInvalid -} - -// hook executes a plugin hook. -// -// This function satisfies the pluginClient interface. -func (p *commentsPlugin) hook(h hookT, payload string) error { - log.Tracef("comments hook: %v", hooks[h]) - - return nil -} - -// fsck performs a plugin filesystem check. -// -// This function satisfies the pluginClient interface. -func (p *commentsPlugin) fsck() error { - log.Tracef("comments fsck") - - // Make sure CommentDel blobs were actually deleted - - return nil -} - -// setup performs any plugin setup work that needs to be done. -// -// This function satisfies the pluginClient interface. -func (p *commentsPlugin) setup() error { - log.Tracef("comments setup") - - return nil -} - -// newCommentsPlugin returns a new comments plugin. -func newCommentsPlugin(backend backend.Backend, tlog tlogClient, settings []backend.PluginSetting, id *identity.FullIdentity) (*commentsPlugin, error) { - // Unpack plugin settings - var ( - dataDir string - ) - for _, v := range settings { - switch v.Key { - case pluginSettingDataDir: - dataDir = v.Value - default: - return nil, fmt.Errorf("invalid plugin setting '%v'", v.Key) - } - } - - // Verify plugin settings - switch { - case dataDir == "": - return nil, fmt.Errorf("plugin setting not found: %v", - pluginSettingDataDir) - } - - // Create the plugin data directory - dataDir = filepath.Join(dataDir, comments.ID) - err := os.MkdirAll(dataDir, 0700) - if err != nil { - return nil, err - } - - return &commentsPlugin{ - backend: backend, - tlog: tlog, - identity: id, - dataDir: dataDir, - mutexes: make(map[string]*sync.Mutex), - }, nil -} diff --git a/politeiad/backend/tlogbe/comments_test.go b/politeiad/backend/tlogbe/comments_test.go deleted file mode 100644 index 3fe062807..000000000 --- a/politeiad/backend/tlogbe/comments_test.go +++ /dev/null @@ -1,704 +0,0 @@ -// Copyright (c) 2020-2021 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package tlogbe - -import ( - "encoding/hex" - "errors" - "testing" - - "github.com/decred/politeia/politeiad/api/v1/identity" - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/plugins/comments" - "github.com/google/uuid" -) - -func TestCmdNew(t *testing.T) { - tlogBackend, cleanup := newTestTlogBackend(t) - defer cleanup() - - id, err := identity.New() - if err != nil { - t.Fatal(err) - } - settings := []backend.PluginSetting{{ - Key: pluginSettingDataDir, - Value: tlogBackend.dataDir, - }} - - commentsPlugin, err := newCommentsPlugin(tlogBackend, - newBackendClient(tlogBackend), settings, id) - if err != nil { - t.Fatal(err) - } - - // New record - md := []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs := []backend.File{ - newBackendFile(t, "index.md"), - } - rec, err := tlogBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - - // Helpers - comment := "random comment" - tokenRandom := hex.EncodeToString(tokenFromTreeID(123)) - parentID := uint32(0) - invalidParentID := uint32(3) - - uid, err := identity.New() - if err != nil { - t.Fatal(err) - } - - // Setup new comment plugin tests - var tests = []struct { - description string - payload comments.New - wantErr error - }{ - { - "invalid comment state", - comments.New{ - UserID: uuid.New().String(), - State: comments.StateInvalid, - Token: rec.Token, - ParentID: parentID, - Comment: comment, - PublicKey: uid.Public.String(), - Signature: commentSignature(t, uid, comments.StateInvalid, - rec.Token, comment, parentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusStateInvalid), - }, - }, - { - "invalid token", - comments.New{ - UserID: uuid.New().String(), - State: comments.StateUnvetted, - Token: "invalid", - ParentID: parentID, - Comment: comment, - PublicKey: uid.Public.String(), - Signature: commentSignature(t, uid, comments.StateUnvetted, - rec.Token, comment, parentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusTokenInvalid), - }, - }, - { - "invalid signature", - comments.New{ - UserID: uuid.New().String(), - State: comments.StateUnvetted, - Token: rec.Token, - ParentID: parentID, - Comment: comment, - PublicKey: uid.Public.String(), - Signature: "invalid", - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusSignatureInvalid), - }, - }, - { - "invalid public key", - comments.New{ - UserID: uuid.New().String(), - State: comments.StateUnvetted, - Token: rec.Token, - ParentID: parentID, - Comment: comment, - PublicKey: "invalid", - Signature: commentSignature(t, uid, comments.StateUnvetted, - rec.Token, comment, parentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusPublicKeyInvalid), - }, - }, - { - "comment max length exceeded", - comments.New{ - UserID: uuid.New().String(), - State: comments.StateUnvetted, - Token: rec.Token, - ParentID: parentID, - Comment: commentMaxLengthExceeded(t), - PublicKey: uid.Public.String(), - Signature: commentSignature(t, uid, comments.StateUnvetted, - rec.Token, commentMaxLengthExceeded(t), parentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusCommentTextInvalid), - }, - }, - { - "invalid parent ID", - comments.New{ - UserID: uuid.New().String(), - State: comments.StateUnvetted, - Token: rec.Token, - ParentID: invalidParentID, - Comment: comment, - PublicKey: uid.Public.String(), - Signature: commentSignature(t, uid, comments.StateUnvetted, - rec.Token, comment, invalidParentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusParentIDInvalid), - }, - }, - { - "record not found", - comments.New{ - UserID: uuid.New().String(), - State: comments.StateUnvetted, - Token: tokenRandom, - ParentID: parentID, - Comment: comment, - PublicKey: uid.Public.String(), - Signature: commentSignature(t, uid, comments.StateUnvetted, - tokenRandom, comment, parentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusRecordNotFound), - }, - }, - { - "success", - comments.New{ - UserID: uuid.New().String(), - State: comments.StateUnvetted, - Token: rec.Token, - ParentID: parentID, - Comment: comment, - PublicKey: uid.Public.String(), - Signature: commentSignature(t, uid, comments.StateUnvetted, - rec.Token, comment, parentID), - }, - nil, - }, - } - - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - // New Comment - ncEncoded, err := comments.EncodeNew(test.payload) - if err != nil { - t.Error(err) - } - - // Execute plugin command - _, err = commentsPlugin.cmdNew(string(ncEncoded)) - - // Parse plugin user error - var pluginUserError backend.PluginUserError - if errors.As(err, &pluginUserError) { - if test.wantErr == nil { - t.Errorf("got error %v, want nil", err) - return - } - wantErr := test.wantErr.(backend.PluginUserError) - if pluginUserError.ErrorCode != wantErr.ErrorCode { - t.Errorf("got error %v, want %v", - pluginUserError.ErrorCode, - wantErr.ErrorCode) - } - return - } - - // Expectations not met - if err != test.wantErr { - t.Errorf("got error %v, want %v", err, test.wantErr) - } - }) - } -} - -func TestCmdEdit(t *testing.T) { - commentsPlugin, tlogBackend, cleanup := newTestCommentsPlugin(t) - defer cleanup() - - // New record - md := []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs := []backend.File{ - newBackendFile(t, "index.md"), - } - rec, err := tlogBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - - // Helpers - comment := "random comment" - commentEdit := comment + "more content" - parentID := uint32(0) - invalidParentID := uint32(3) - invalidCommentID := uint32(3) - tokenRandom := hex.EncodeToString(tokenFromTreeID(123)) - - id, err := identity.New() - if err != nil { - t.Fatal(err) - } - - // New comment - ncEncoded, err := comments.EncodeNew( - comments.New{ - UserID: uuid.New().String(), - State: comments.StateUnvetted, - Token: rec.Token, - ParentID: parentID, - Comment: comment, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, comments.StateUnvetted, - rec.Token, comment, parentID), - }, - ) - if err != nil { - t.Fatal(err) - } - reply, err := commentsPlugin.cmdNew(string(ncEncoded)) - if err != nil { - t.Fatal(err) - } - nr, err := comments.DecodeNewReply([]byte(reply)) - if err != nil { - t.Fatal(err) - } - - // Setup edit comment plugin tests - var tests = []struct { - description string - payload comments.Edit - wantErr error - }{ - { - "invalid comment state", - comments.Edit{ - UserID: nr.Comment.UserID, - State: comments.StateInvalid, - Token: rec.Token, - ParentID: nr.Comment.ParentID, - CommentID: nr.Comment.CommentID, - Comment: commentEdit, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, comments.StateInvalid, - rec.Token, commentEdit, nr.Comment.ParentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusStateInvalid), - }, - }, - { - "invalid token", - comments.Edit{ - UserID: nr.Comment.UserID, - State: nr.Comment.State, - Token: "invalid", - ParentID: nr.Comment.ParentID, - CommentID: nr.Comment.CommentID, - Comment: commentEdit, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, nr.Comment.State, "invalid", - commentEdit, nr.Comment.ParentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusTokenInvalid), - }, - }, - { - "invalid signature", - comments.Edit{ - UserID: nr.Comment.UserID, - State: nr.Comment.State, - Token: rec.Token, - ParentID: nr.Comment.ParentID, - CommentID: nr.Comment.CommentID, - Comment: commentEdit, - PublicKey: id.Public.String(), - Signature: "invalid", - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusSignatureInvalid), - }, - }, - { - "invalid public key", - comments.Edit{ - UserID: nr.Comment.UserID, - State: nr.Comment.State, - Token: rec.Token, - ParentID: nr.Comment.ParentID, - CommentID: nr.Comment.CommentID, - Comment: commentEdit, - PublicKey: "invalid", - Signature: commentSignature(t, id, nr.Comment.State, rec.Token, - commentEdit, nr.Comment.ParentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusPublicKeyInvalid), - }, - }, - { - "comment max length exceeded", - comments.Edit{ - UserID: nr.Comment.UserID, - State: nr.Comment.State, - Token: rec.Token, - ParentID: nr.Comment.ParentID, - CommentID: nr.Comment.CommentID, - Comment: commentMaxLengthExceeded(t), - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, nr.Comment.State, rec.Token, - commentMaxLengthExceeded(t), nr.Comment.ParentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusCommentTextInvalid), - }, - }, - { - "comment id not found", - comments.Edit{ - UserID: nr.Comment.UserID, - State: nr.Comment.State, - Token: rec.Token, - ParentID: nr.Comment.ParentID, - CommentID: invalidCommentID, - Comment: commentEdit, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, nr.Comment.State, rec.Token, - commentEdit, nr.Comment.ParentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusCommentNotFound), - }, - }, - { - "unauthorized user", - comments.Edit{ - UserID: uuid.New().String(), - State: nr.Comment.State, - Token: rec.Token, - ParentID: nr.Comment.ParentID, - CommentID: nr.Comment.CommentID, - Comment: commentEdit, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, nr.Comment.State, rec.Token, - commentEdit, nr.Comment.ParentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusUserUnauthorized), - }, - }, - { - "invalid parent ID", - comments.Edit{ - UserID: nr.Comment.UserID, - State: nr.Comment.State, - Token: rec.Token, - ParentID: invalidParentID, - CommentID: nr.Comment.CommentID, - Comment: commentEdit, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, nr.Comment.State, - rec.Token, commentEdit, invalidParentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusParentIDInvalid), - }, - }, - { - "comment did not change", - comments.Edit{ - UserID: nr.Comment.UserID, - State: nr.Comment.State, - Token: rec.Token, - ParentID: nr.Comment.ParentID, - CommentID: nr.Comment.CommentID, - Comment: comment, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, nr.Comment.State, - rec.Token, comment, nr.Comment.ParentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusCommentTextInvalid), - }, - }, - { - "record not found", - comments.Edit{ - UserID: nr.Comment.UserID, - State: nr.Comment.State, - Token: tokenRandom, - ParentID: nr.Comment.ParentID, - CommentID: nr.Comment.ParentID, - Comment: commentEdit, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, nr.Comment.State, - tokenRandom, commentEdit, nr.Comment.ParentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusRecordNotFound), - }, - }, - { - "success", - comments.Edit{ - UserID: nr.Comment.UserID, - State: nr.Comment.State, - Token: rec.Token, - ParentID: nr.Comment.ParentID, - CommentID: nr.Comment.CommentID, - Comment: commentEdit, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, nr.Comment.State, - rec.Token, commentEdit, nr.Comment.ParentID), - }, - nil, - }, - } - - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - // Edit Comment - ecEncoded, err := comments.EncodeEdit(test.payload) - if err != nil { - t.Error(err) - } - - // Execute plugin command - _, err = commentsPlugin.cmdEdit(string(ecEncoded)) - - // Parse plugin user error - var pluginUserError backend.PluginUserError - if errors.As(err, &pluginUserError) { - if test.wantErr == nil { - t.Errorf("got error %v, want nil", err) - return - } - wantErr := test.wantErr.(backend.PluginUserError) - if pluginUserError.ErrorCode != wantErr.ErrorCode { - t.Errorf("got error %v, want %v", - pluginUserError.ErrorCode, - wantErr.ErrorCode) - } - return - } - - // Expectations not met - if err != test.wantErr { - t.Errorf("got error %v, want %v", err, test.wantErr) - } - }) - } -} - -func TestCmdDel(t *testing.T) { - commentsPlugin, tlogBackend, cleanup := newTestCommentsPlugin(t) - defer cleanup() - - // New record - md := []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs := []backend.File{ - newBackendFile(t, "index.md"), - } - rec, err := tlogBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - - // Helpers - comment := "random comment" - reason := "random reason" - parentID := uint32(0) - tokenRandom := hex.EncodeToString(tokenFromTreeID(123)) - id, err := identity.New() - if err != nil { - t.Fatal(err) - } - - // New comment - ncEncoded, err := comments.EncodeNew( - comments.New{ - UserID: uuid.New().String(), - State: comments.StateUnvetted, - Token: rec.Token, - ParentID: parentID, - Comment: comment, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, comments.StateUnvetted, - rec.Token, comment, parentID), - }, - ) - if err != nil { - t.Fatal(err) - } - reply, err := commentsPlugin.cmdNew(string(ncEncoded)) - if err != nil { - t.Fatal(err) - } - nr, err := comments.DecodeNewReply([]byte(reply)) - if err != nil { - t.Fatal(err) - } - - // Setup del comment plugin tests - var tests = []struct { - description string - payload comments.Del - wantErr error - }{ - { - "invalid comment state", - comments.Del{ - State: comments.StateInvalid, - Token: rec.Token, - CommentID: nr.Comment.CommentID, - Reason: reason, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, comments.StateInvalid, - rec.Token, reason, nr.Comment.CommentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusStateInvalid), - }, - }, - { - "invalid token", - comments.Del{ - State: comments.StateUnvetted, - Token: "invalid", - CommentID: nr.Comment.CommentID, - Reason: reason, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, comments.StateUnvetted, - "invalid", reason, nr.Comment.CommentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusTokenInvalid), - }, - }, - { - "invalid signature", - comments.Del{ - State: comments.StateUnvetted, - Token: rec.Token, - CommentID: nr.Comment.CommentID, - Reason: reason, - PublicKey: id.Public.String(), - Signature: "invalid", - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusSignatureInvalid), - }, - }, - { - "invalid public key", - comments.Del{ - State: comments.StateUnvetted, - Token: rec.Token, - CommentID: nr.Comment.CommentID, - Reason: reason, - PublicKey: "invalid", - Signature: commentSignature(t, id, comments.StateUnvetted, - rec.Token, reason, nr.Comment.CommentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusPublicKeyInvalid), - }, - }, - { - "record not found", - comments.Del{ - State: comments.StateUnvetted, - Token: tokenRandom, - CommentID: nr.Comment.CommentID, - Reason: reason, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, comments.StateUnvetted, - tokenRandom, reason, nr.Comment.CommentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusRecordNotFound), - }, - }, - { - "comment id not found", - comments.Del{ - State: comments.StateUnvetted, - Token: rec.Token, - CommentID: 3, - Reason: reason, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, comments.StateUnvetted, - rec.Token, reason, 3), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusCommentNotFound), - }, - }, - { - "success", - comments.Del{ - State: comments.StateUnvetted, - Token: rec.Token, - CommentID: nr.Comment.CommentID, - Reason: reason, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, comments.StateUnvetted, - rec.Token, reason, nr.Comment.CommentID), - }, - nil, - }, - } - - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - // Del Comment - dcEncoded, err := comments.EncodeDel(test.payload) - if err != nil { - t.Error(err) - } - - // Execute plugin command - _, err = commentsPlugin.cmdDel(string(dcEncoded)) - - // Parse plugin user error - var pluginUserError backend.PluginUserError - if errors.As(err, &pluginUserError) { - if test.wantErr == nil { - t.Errorf("got error %v, want nil", err) - return - } - wantErr := test.wantErr.(backend.PluginUserError) - if pluginUserError.ErrorCode != wantErr.ErrorCode { - t.Errorf("got error %v, want %v", - pluginUserError.ErrorCode, - wantErr.ErrorCode) - } - return - } - - // Expectations not met - if err != test.wantErr { - t.Errorf("got error %v, want %v", err, test.wantErr) - } - }) - } - -} diff --git a/politeiad/backend/tlogbe/pluginclient.go b/politeiad/backend/tlogbe/pluginclient.go deleted file mode 100644 index 5b373c919..000000000 --- a/politeiad/backend/tlogbe/pluginclient.go +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright (c) 2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package tlogbe - -import ( - "encoding/json" - - "github.com/decred/politeia/politeiad/backend" -) - -type hookT int - -const ( - // Plugin hooks - hookInvalid hookT = 0 - hookNewRecordPre hookT = 1 - hookNewRecordPost hookT = 2 - hookEditRecordPre hookT = 3 - hookEditRecordPost hookT = 4 - hookEditMetadataPre hookT = 5 - hookEditMetadataPost hookT = 6 - hookSetRecordStatusPre hookT = 7 - hookSetRecordStatusPost hookT = 8 -) - -var ( - // hooks contains human readable descriptions of the plugin hooks. - hooks = map[hookT]string{ - hookNewRecordPre: "new record pre", - hookNewRecordPost: "new record post", - hookEditRecordPre: "edit record pre", - hookEditRecordPost: "edit record post", - hookEditMetadataPre: "edit metadata pre", - hookEditMetadataPost: "edit metadata post", - hookSetRecordStatusPre: "set record status pre", - hookSetRecordStatusPost: "set record status post", - } -) - -// hookNewRecord is the payload for the new record hooks. -type hookNewRecord struct { - Metadata []backend.MetadataStream `json:"metadata"` - Files []backend.File `json:"files"` - - // RecordMetadata will only be present on the post new record hook. - // This is because the record metadata requires the creation of a - // trillian tree and the pre new record hook should execute before - // any politeiad state is changed in case of validation errors. - RecordMetadata *backend.RecordMetadata `json:"recordmetadata"` -} - -func encodeHookNewRecord(hnr hookNewRecord) ([]byte, error) { - return json.Marshal(hnr) -} - -func decodeHookNewRecord(payload []byte) (*hookNewRecord, error) { - var hnr hookNewRecord - err := json.Unmarshal(payload, &hnr) - if err != nil { - return nil, err - } - return &hnr, nil -} - -// hookEditRecord is the payload for the edit record hooks. -type hookEditRecord struct { - // Current record - Current backend.Record `json:"record"` - - // Updated fields - RecordMetadata backend.RecordMetadata `json:"recordmetadata"` - MDAppend []backend.MetadataStream `json:"mdappend"` - MDOverwrite []backend.MetadataStream `json:"mdoverwrite"` - FilesAdd []backend.File `json:"filesadd"` - FilesDel []string `json:"filesdel"` -} - -func encodeHookEditRecord(her hookEditRecord) ([]byte, error) { - return json.Marshal(her) -} - -func decodeHookEditRecord(payload []byte) (*hookEditRecord, error) { - var her hookEditRecord - err := json.Unmarshal(payload, &her) - if err != nil { - return nil, err - } - return &her, nil -} - -// hookEditMetadata is the payload for the edit metadata hooks. -type hookEditMetadata struct { - // Current record - Current backend.Record `json:"record"` - - // Updated fields - MDAppend []backend.MetadataStream `json:"mdappend"` - MDOverwrite []backend.MetadataStream `json:"mdoverwrite"` -} - -func encodeHookEditMetadata(hem hookEditMetadata) ([]byte, error) { - return json.Marshal(hem) -} - -func decodeHookEditMetadata(payload []byte) (*hookEditMetadata, error) { - var hem hookEditMetadata - err := json.Unmarshal(payload, &hem) - if err != nil { - return nil, err - } - return &hem, nil -} - -// hookSetRecordStatus is the payload for the set record status hooks. -type hookSetRecordStatus struct { - // Current record - Current backend.Record `json:"record"` - - // Updated fields - RecordMetadata backend.RecordMetadata `json:"recordmetadata"` - MDAppend []backend.MetadataStream `json:"mdappend"` - MDOverwrite []backend.MetadataStream `json:"mdoverwrite"` -} - -func encodeHookSetRecordStatus(hsrs hookSetRecordStatus) ([]byte, error) { - return json.Marshal(hsrs) -} - -func decodeHookSetRecordStatus(payload []byte) (*hookSetRecordStatus, error) { - var hsrs hookSetRecordStatus - err := json.Unmarshal(payload, &hsrs) - if err != nil { - return nil, err - } - return &hsrs, nil -} - -// pluginClient provides an API for the tlog backend to use when interacting -// with plugins. All tlogbe plugins must implement the pluginClient interface. -type pluginClient interface { - // setup performs any required plugin setup. - setup() error - - // cmd executes the provided plugin command. - cmd(cmd, payload string) (string, error) - - // hook executes the provided plugin hook. - hook(h hookT, payload string) error - - // fsck performs a plugin file system check. - fsck() error -} diff --git a/politeiad/backend/tlogbe/plugins/comments/comments.go b/politeiad/backend/tlogbe/plugins/comments/comments.go index c5078363b..b8dce8620 100644 --- a/politeiad/backend/tlogbe/plugins/comments/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments/comments.go @@ -112,7 +112,7 @@ var ( type commentsPlugin struct { sync.Mutex backend backend.Backend - tlog plugins.BackendClient + tlog plugins.TlogClient // dataDir is the comments plugin data directory. The only data // that is stored here is cached data that can be re-created at any @@ -1975,7 +1975,7 @@ func (p *commentsPlugin) cmdTimestamps(payload string) (string, error) { // Cmd executes a plugin command. // // This function satisfies the PluginClient interface. -func (p *commentsPlugin) Cmd(cmd, payload string) (string, error) { +func (p *commentsPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { log.Tracef("Cmd: %v %v", cmd, payload) switch cmd { diff --git a/politeiad/backend/tlogbe/plugins/comments/comments_test.go b/politeiad/backend/tlogbe/plugins/comments/comments_test.go new file mode 100644 index 000000000..dc66c53f6 --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/comments/comments_test.go @@ -0,0 +1,749 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package comments + +import ( + "encoding/hex" + "errors" + "strconv" + "testing" + + "github.com/decred/politeia/politeiad/api/v1/identity" + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/plugins/comments" + "github.com/google/uuid" +) + +func commentSignature(t *testing.T, uid *identity.FullIdentity, state comments.StateT, token, msg string, id uint32) string { + t.Helper() + txt := strconv.Itoa(int(state)) + token + + strconv.FormatInt(int64(id), 10) + msg + b := uid.SignMessage([]byte(txt)) + return hex.EncodeToString(b[:]) +} + +func newTestCommentsPlugin(t *testing.T) (*commentsPlugin, *tlogBackend, func()) { + t.Helper() + + tlogBackend, cleanup := newTestTlogBackend(t) + + id, err := identity.New() + if err != nil { + t.Fatalf("identity New: %v", err) + } + + settings := []backend.PluginSetting{{ + Key: pluginSettingDataDir, + Value: tlogBackend.dataDir, + }} + + commentsPlugin, err := newCommentsPlugin(tlogBackend, + newBackendClient(tlogBackend), settings, id) + if err != nil { + t.Fatalf("newCommentsPlugin: %v", err) + } + + return commentsPlugin, tlogBackend, cleanup +} + +func TestCmdNew(t *testing.T) { + tlogBackend, cleanup := newTestTlogBackend(t) + defer cleanup() + + id, err := identity.New() + if err != nil { + t.Fatal(err) + } + settings := []backend.PluginSetting{{ + Key: pluginSettingDataDir, + Value: tlogBackend.dataDir, + }} + + commentsPlugin, err := newCommentsPlugin(tlogBackend, + newBackendClient(tlogBackend), settings, id) + if err != nil { + t.Fatal(err) + } + + // New record + md := []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs := []backend.File{ + newBackendFile(t, "index.md"), + } + rec, err := tlogBackend.New(md, fs) + if err != nil { + t.Fatal(err) + } + + // Helpers + comment := "random comment" + tokenRandom := hex.EncodeToString(tokenFromTreeID(123)) + parentID := uint32(0) + invalidParentID := uint32(3) + + uid, err := identity.New() + if err != nil { + t.Fatal(err) + } + + // Setup new comment plugin tests + var tests = []struct { + description string + payload comments.New + wantErr error + }{ + { + "invalid comment state", + comments.New{ + UserID: uuid.New().String(), + State: comments.StateInvalid, + Token: rec.Token, + ParentID: parentID, + Comment: comment, + PublicKey: uid.Public.String(), + Signature: commentSignature(t, uid, comments.StateInvalid, + rec.Token, comment, parentID), + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusStateInvalid), + }, + }, + { + "invalid token", + comments.New{ + UserID: uuid.New().String(), + State: comments.StateUnvetted, + Token: "invalid", + ParentID: parentID, + Comment: comment, + PublicKey: uid.Public.String(), + Signature: commentSignature(t, uid, comments.StateUnvetted, + rec.Token, comment, parentID), + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusTokenInvalid), + }, + }, + { + "invalid signature", + comments.New{ + UserID: uuid.New().String(), + State: comments.StateUnvetted, + Token: rec.Token, + ParentID: parentID, + Comment: comment, + PublicKey: uid.Public.String(), + Signature: "invalid", + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusSignatureInvalid), + }, + }, + { + "invalid public key", + comments.New{ + UserID: uuid.New().String(), + State: comments.StateUnvetted, + Token: rec.Token, + ParentID: parentID, + Comment: comment, + PublicKey: "invalid", + Signature: commentSignature(t, uid, comments.StateUnvetted, + rec.Token, comment, parentID), + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusPublicKeyInvalid), + }, + }, + { + "comment max length exceeded", + comments.New{ + UserID: uuid.New().String(), + State: comments.StateUnvetted, + Token: rec.Token, + ParentID: parentID, + Comment: commentMaxLengthExceeded(t), + PublicKey: uid.Public.String(), + Signature: commentSignature(t, uid, comments.StateUnvetted, + rec.Token, commentMaxLengthExceeded(t), parentID), + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusCommentTextInvalid), + }, + }, + { + "invalid parent ID", + comments.New{ + UserID: uuid.New().String(), + State: comments.StateUnvetted, + Token: rec.Token, + ParentID: invalidParentID, + Comment: comment, + PublicKey: uid.Public.String(), + Signature: commentSignature(t, uid, comments.StateUnvetted, + rec.Token, comment, invalidParentID), + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusParentIDInvalid), + }, + }, + { + "record not found", + comments.New{ + UserID: uuid.New().String(), + State: comments.StateUnvetted, + Token: tokenRandom, + ParentID: parentID, + Comment: comment, + PublicKey: uid.Public.String(), + Signature: commentSignature(t, uid, comments.StateUnvetted, + tokenRandom, comment, parentID), + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusRecordNotFound), + }, + }, + { + "success", + comments.New{ + UserID: uuid.New().String(), + State: comments.StateUnvetted, + Token: rec.Token, + ParentID: parentID, + Comment: comment, + PublicKey: uid.Public.String(), + Signature: commentSignature(t, uid, comments.StateUnvetted, + rec.Token, comment, parentID), + }, + nil, + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + // New Comment + ncEncoded, err := comments.EncodeNew(test.payload) + if err != nil { + t.Error(err) + } + + // Execute plugin command + _, err = commentsPlugin.cmdNew(string(ncEncoded)) + + // Parse plugin user error + var pluginUserError backend.PluginUserError + if errors.As(err, &pluginUserError) { + if test.wantErr == nil { + t.Errorf("got error %v, want nil", err) + return + } + wantErr := test.wantErr.(backend.PluginUserError) + if pluginUserError.ErrorCode != wantErr.ErrorCode { + t.Errorf("got error %v, want %v", + pluginUserError.ErrorCode, + wantErr.ErrorCode) + } + return + } + + // Expectations not met + if err != test.wantErr { + t.Errorf("got error %v, want %v", err, test.wantErr) + } + }) + } +} + +func TestCmdEdit(t *testing.T) { + commentsPlugin, tlogBackend, cleanup := newTestCommentsPlugin(t) + defer cleanup() + + // New record + md := []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs := []backend.File{ + newBackendFile(t, "index.md"), + } + rec, err := tlogBackend.New(md, fs) + if err != nil { + t.Fatal(err) + } + + // Helpers + comment := "random comment" + commentEdit := comment + "more content" + parentID := uint32(0) + invalidParentID := uint32(3) + invalidCommentID := uint32(3) + tokenRandom := hex.EncodeToString(tokenFromTreeID(123)) + + id, err := identity.New() + if err != nil { + t.Fatal(err) + } + + // New comment + ncEncoded, err := comments.EncodeNew( + comments.New{ + UserID: uuid.New().String(), + State: comments.StateUnvetted, + Token: rec.Token, + ParentID: parentID, + Comment: comment, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, comments.StateUnvetted, + rec.Token, comment, parentID), + }, + ) + if err != nil { + t.Fatal(err) + } + reply, err := commentsPlugin.cmdNew(string(ncEncoded)) + if err != nil { + t.Fatal(err) + } + nr, err := comments.DecodeNewReply([]byte(reply)) + if err != nil { + t.Fatal(err) + } + + // Setup edit comment plugin tests + var tests = []struct { + description string + payload comments.Edit + wantErr error + }{ + { + "invalid comment state", + comments.Edit{ + UserID: nr.Comment.UserID, + State: comments.StateInvalid, + Token: rec.Token, + ParentID: nr.Comment.ParentID, + CommentID: nr.Comment.CommentID, + Comment: commentEdit, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, comments.StateInvalid, + rec.Token, commentEdit, nr.Comment.ParentID), + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusStateInvalid), + }, + }, + { + "invalid token", + comments.Edit{ + UserID: nr.Comment.UserID, + State: nr.Comment.State, + Token: "invalid", + ParentID: nr.Comment.ParentID, + CommentID: nr.Comment.CommentID, + Comment: commentEdit, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, nr.Comment.State, "invalid", + commentEdit, nr.Comment.ParentID), + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusTokenInvalid), + }, + }, + { + "invalid signature", + comments.Edit{ + UserID: nr.Comment.UserID, + State: nr.Comment.State, + Token: rec.Token, + ParentID: nr.Comment.ParentID, + CommentID: nr.Comment.CommentID, + Comment: commentEdit, + PublicKey: id.Public.String(), + Signature: "invalid", + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusSignatureInvalid), + }, + }, + { + "invalid public key", + comments.Edit{ + UserID: nr.Comment.UserID, + State: nr.Comment.State, + Token: rec.Token, + ParentID: nr.Comment.ParentID, + CommentID: nr.Comment.CommentID, + Comment: commentEdit, + PublicKey: "invalid", + Signature: commentSignature(t, id, nr.Comment.State, rec.Token, + commentEdit, nr.Comment.ParentID), + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusPublicKeyInvalid), + }, + }, + { + "comment max length exceeded", + comments.Edit{ + UserID: nr.Comment.UserID, + State: nr.Comment.State, + Token: rec.Token, + ParentID: nr.Comment.ParentID, + CommentID: nr.Comment.CommentID, + Comment: commentMaxLengthExceeded(t), + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, nr.Comment.State, rec.Token, + commentMaxLengthExceeded(t), nr.Comment.ParentID), + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusCommentTextInvalid), + }, + }, + { + "comment id not found", + comments.Edit{ + UserID: nr.Comment.UserID, + State: nr.Comment.State, + Token: rec.Token, + ParentID: nr.Comment.ParentID, + CommentID: invalidCommentID, + Comment: commentEdit, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, nr.Comment.State, rec.Token, + commentEdit, nr.Comment.ParentID), + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusCommentNotFound), + }, + }, + { + "unauthorized user", + comments.Edit{ + UserID: uuid.New().String(), + State: nr.Comment.State, + Token: rec.Token, + ParentID: nr.Comment.ParentID, + CommentID: nr.Comment.CommentID, + Comment: commentEdit, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, nr.Comment.State, rec.Token, + commentEdit, nr.Comment.ParentID), + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusUserUnauthorized), + }, + }, + { + "invalid parent ID", + comments.Edit{ + UserID: nr.Comment.UserID, + State: nr.Comment.State, + Token: rec.Token, + ParentID: invalidParentID, + CommentID: nr.Comment.CommentID, + Comment: commentEdit, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, nr.Comment.State, + rec.Token, commentEdit, invalidParentID), + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusParentIDInvalid), + }, + }, + { + "comment did not change", + comments.Edit{ + UserID: nr.Comment.UserID, + State: nr.Comment.State, + Token: rec.Token, + ParentID: nr.Comment.ParentID, + CommentID: nr.Comment.CommentID, + Comment: comment, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, nr.Comment.State, + rec.Token, comment, nr.Comment.ParentID), + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusCommentTextInvalid), + }, + }, + { + "record not found", + comments.Edit{ + UserID: nr.Comment.UserID, + State: nr.Comment.State, + Token: tokenRandom, + ParentID: nr.Comment.ParentID, + CommentID: nr.Comment.ParentID, + Comment: commentEdit, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, nr.Comment.State, + tokenRandom, commentEdit, nr.Comment.ParentID), + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusRecordNotFound), + }, + }, + { + "success", + comments.Edit{ + UserID: nr.Comment.UserID, + State: nr.Comment.State, + Token: rec.Token, + ParentID: nr.Comment.ParentID, + CommentID: nr.Comment.CommentID, + Comment: commentEdit, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, nr.Comment.State, + rec.Token, commentEdit, nr.Comment.ParentID), + }, + nil, + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + // Edit Comment + ecEncoded, err := comments.EncodeEdit(test.payload) + if err != nil { + t.Error(err) + } + + // Execute plugin command + _, err = commentsPlugin.cmdEdit(string(ecEncoded)) + + // Parse plugin user error + var pluginUserError backend.PluginUserError + if errors.As(err, &pluginUserError) { + if test.wantErr == nil { + t.Errorf("got error %v, want nil", err) + return + } + wantErr := test.wantErr.(backend.PluginUserError) + if pluginUserError.ErrorCode != wantErr.ErrorCode { + t.Errorf("got error %v, want %v", + pluginUserError.ErrorCode, + wantErr.ErrorCode) + } + return + } + + // Expectations not met + if err != test.wantErr { + t.Errorf("got error %v, want %v", err, test.wantErr) + } + }) + } +} + +func TestCmdDel(t *testing.T) { + commentsPlugin, tlogBackend, cleanup := newTestCommentsPlugin(t) + defer cleanup() + + // New record + md := []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs := []backend.File{ + newBackendFile(t, "index.md"), + } + rec, err := tlogBackend.New(md, fs) + if err != nil { + t.Fatal(err) + } + + // Helpers + comment := "random comment" + reason := "random reason" + parentID := uint32(0) + tokenRandom := hex.EncodeToString(tokenFromTreeID(123)) + id, err := identity.New() + if err != nil { + t.Fatal(err) + } + + // New comment + ncEncoded, err := comments.EncodeNew( + comments.New{ + UserID: uuid.New().String(), + State: comments.StateUnvetted, + Token: rec.Token, + ParentID: parentID, + Comment: comment, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, comments.StateUnvetted, + rec.Token, comment, parentID), + }, + ) + if err != nil { + t.Fatal(err) + } + reply, err := commentsPlugin.cmdNew(string(ncEncoded)) + if err != nil { + t.Fatal(err) + } + nr, err := comments.DecodeNewReply([]byte(reply)) + if err != nil { + t.Fatal(err) + } + + // Setup del comment plugin tests + var tests = []struct { + description string + payload comments.Del + wantErr error + }{ + { + "invalid comment state", + comments.Del{ + State: comments.StateInvalid, + Token: rec.Token, + CommentID: nr.Comment.CommentID, + Reason: reason, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, comments.StateInvalid, + rec.Token, reason, nr.Comment.CommentID), + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusStateInvalid), + }, + }, + { + "invalid token", + comments.Del{ + State: comments.StateUnvetted, + Token: "invalid", + CommentID: nr.Comment.CommentID, + Reason: reason, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, comments.StateUnvetted, + "invalid", reason, nr.Comment.CommentID), + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusTokenInvalid), + }, + }, + { + "invalid signature", + comments.Del{ + State: comments.StateUnvetted, + Token: rec.Token, + CommentID: nr.Comment.CommentID, + Reason: reason, + PublicKey: id.Public.String(), + Signature: "invalid", + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusSignatureInvalid), + }, + }, + { + "invalid public key", + comments.Del{ + State: comments.StateUnvetted, + Token: rec.Token, + CommentID: nr.Comment.CommentID, + Reason: reason, + PublicKey: "invalid", + Signature: commentSignature(t, id, comments.StateUnvetted, + rec.Token, reason, nr.Comment.CommentID), + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusPublicKeyInvalid), + }, + }, + { + "record not found", + comments.Del{ + State: comments.StateUnvetted, + Token: tokenRandom, + CommentID: nr.Comment.CommentID, + Reason: reason, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, comments.StateUnvetted, + tokenRandom, reason, nr.Comment.CommentID), + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusRecordNotFound), + }, + }, + { + "comment id not found", + comments.Del{ + State: comments.StateUnvetted, + Token: rec.Token, + CommentID: 3, + Reason: reason, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, comments.StateUnvetted, + rec.Token, reason, 3), + }, + backend.PluginUserError{ + ErrorCode: int(comments.ErrorStatusCommentNotFound), + }, + }, + { + "success", + comments.Del{ + State: comments.StateUnvetted, + Token: rec.Token, + CommentID: nr.Comment.CommentID, + Reason: reason, + PublicKey: id.Public.String(), + Signature: commentSignature(t, id, comments.StateUnvetted, + rec.Token, reason, nr.Comment.CommentID), + }, + nil, + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + // Del Comment + dcEncoded, err := comments.EncodeDel(test.payload) + if err != nil { + t.Error(err) + } + + // Execute plugin command + _, err = commentsPlugin.cmdDel(string(dcEncoded)) + + // Parse plugin user error + var pluginUserError backend.PluginUserError + if errors.As(err, &pluginUserError) { + if test.wantErr == nil { + t.Errorf("got error %v, want nil", err) + return + } + wantErr := test.wantErr.(backend.PluginUserError) + if pluginUserError.ErrorCode != wantErr.ErrorCode { + t.Errorf("got error %v, want %v", + pluginUserError.ErrorCode, + wantErr.ErrorCode) + } + return + } + + // Expectations not met + if err != test.wantErr { + t.Errorf("got error %v, want %v", err, test.wantErr) + } + }) + } + +} + +// TODO don't hardcode this. It breaks editors. It should also be based on the +// policy. +func commentMaxLengthExceeded(t *testing.T) string { + t.Helper() + + return `puqgtmjuiztnwakzqtmybjgwluizndahxihosjwyqsyqrhxdkhwchnhtbgamigdakzyjcnjvubufifolxgmygyhoewgyjwicucaypgxrgsenjpangvtslpfbojvcgsxzqyginsjrtcaggdcienduzuxnnpmdcvxtklnsmlanjheywshijznprrvnizoxrihxstcdksqvhvqjeneeadzpjgqtdrlmhjqkxfsjcgoewisbacchagpzisvttyurpijyzpugkpufyhifaivfzgjmkpxtdydaebidbpypcnmsmvaktchcqkkrwvaahlrrdzzxpjmlshtmugrzjqyvehfnkievnmubitaeewabnqnbqcacctfiojiifozfnpvooawiutxxzcjddqpybnrsyxtfzwiomoqpbpowifsbgfsvhvcaucwjnqfnwecqfugakimobvnguyqsgzjxstovohyzvezjuwdwyopymdpoaqwesghiuzmdwzlfcnubpvfefedyrllzqyfgntcaaazmfqxknfkquiyplyhqqipixmlqjedtafcwmkfemmzdvbgkatlcwnsuzjuvofpwznczcdozlqzaiqpumugnubbsfhvoogfenjsqlenxptlaostdwpcorusmijrbqecttwokirwomuktngwluwmlthypopbinyxqrnxplzpipzitqpaqdreelqjlxqsbgxbwhmrmauiicjndjrhrqnaucbezopzrqpxcjobicxqnswftjqrhhhnicxuymdefodhmyscrzonmrihvljgdxqvjeqxmayilhitwwtnmwdcspcmnwocvksitrhiqslzavwjlvzcjwzliikcqpyllmtfrqgfwafalfklhhnhmoejdsgppdvugblyjusljcqjftktuzsqkmbueocfdadrfsbzxldiagxjuydthitfwrirvnbshdzdnbbyxaizemggkndccgjolevpfjovyacfrzvjabkgvwguxpujfancqdebfcbhlwluetcfnddixdovydgoyngxjkxelxarphjmbcbfutmcoulmbzyjxkupqcsaxsojoqpxdvsaijvhppdmdstszbgjzylwebvjvijruioazjkqccsluqsydrkqckluwealsrgxvnnuhvtuxpxdtkjguwplpmynysdysccjqhulkpcnujituvnattmvhhndyvizusmdxvpbvxzbiogawboasipmjgkdezulkmtkcrziqnyrjcupucjtkovpeqnbrqunhricblwqctencvspsahiltoopwqthxypkfovijqbwxahefszdgpwickvurthbxdkczicvzazssynuoyqcrhxxrctsrbvflbkpuprywtasmkkgcojaiylazybdtdnkjuehxjjsftkayjpcnwrngblzjsbcqauxpskcumnbubiodlqzberjmosvvufzgqbtwprhkscsdfppiifzsxroddracraxaesninyzsnnzjlnybrasfudvrjaetulogbzmbxjehlqotyfygfvkvyxkthdljkbwnvxrowjbecngutbuajppguteagmucqgwihzsktxscfepcghchwnnpkozysgpzrdjsozighofgsibuzccolbzpovmooxrkzdizoibmcmhakjwzwnqftgtgawgwzxnwxxdpgxvwxqszjhvihxfggfgxdpseiqkqtuspygomxuwxhumkiltapyfpumxogrjtzkvdomitflsesindlqvcgvxzogmzssqakheixjvzzqgxwzfjqlirtqvuddfevkzltnhwefzalsxkxqnwngdovnzwrvmvbtsykemmwipxlxdxhknyszyfmflmlcvpojnwasvmdfwahuavjajqnzigwvahjphrwreugmeromzotybxqybslneictseznajjkcxmueyroqamairkyzzdpilbusgqufyietrcbertanevmtrfnuxvgyppskpbcligxgbsmrkjnqytfvycbxtmztkzfrvsbuffuhvmuafhrqvupuecyutvxgdmhfrhcngdmqpecvpfjmmxcjbjjtaehnfluibufrmggdaxlenggmzpbwfzkpqymbsjgjcbugiruxyfbpbjovnentghytlitcgksmhxwrwlpkieqmhqrivaqgcllqljvtperdcxpgdbhdotjtvqvwkpqapsxignvhqfhuradjuksickefuuqsmwdktbcpmgynjdrekffvultianlrhyuxjhltcmzhighfddopzpkxidrnwmlczawcaiulvzxlmaulmzlrlclhajplfuxcgsvoedgkltrbqxyfplgoizapfsckkhrtnpqntlectvfzkrcgdioonbjdlrzpyrsanmvijbycaopumafijkjxubibqwlqrywaqjvtvymqiqkdevlugedamqwiqmtdwzotxbzrltbificgouieeosjaokznhqdthzyttabrowgwxrmwobgmupikxduufpktbdfjganpazjbtcweloxmabgexlixwrdolgoxxfwvjrjommozgehlxaqgxphqbaugaiqxpkruwgcvuyadkfwwsqdpwxapmlvpwjvqmdvqtltkrzfzxuirvwxsmajrzzpfcncpnfjzfugipiuuvqsvfbycrarjcrgjspycerxlvhbcahrkjtlcjhcesptpvqcjinmetwuxygrpksmsyvjfpeibljmswjkfdlpqbpfidmdtilclbntvspvsmxstarlrmvzszrwiayncldnpknchgbjerojunmotgujxgeurpabcguhyvsyoyrhlttbcjjraioxblzszobfyiahhwzwgjlkuknuexpyuovlyfhaxhyjgaxzdbvaejdiuqtwiqjnbadsjudlsexlvjnroqihcbfrawmlfczbfvqoyodzmaynbnmtrewqhutjtycfnajrqwjhjfclzopvlbpmqgiugwnarkeerqttlpbxpeqvdiqaohojoukhpjkqaaqkodogtprbtdzbqnjeqyqoffuxxikorythdmtybwpelcgjukzarpsgnrpxesxlbjmbkplhopxxmxjbbwtyueriuaotsjthvkpgjcnbthxgcctexpnacxypzerlzvjlnqcqipiavtmsekfsrrprxerhkkxoldhhvtyojqknkhatpmqavufkepwkxfknokwehwbnrosnhhsdkyesxhcflzmqkflltfhyatqjmcbcptkbumwiwakfszerchqivrhhmxxnhlnhjyfhklpiojwwerznvbzzdjkxajxwbytstchdmyjrdpjhxonxswpigtnmifkmyyqwzldmkknajfhbphedtzxewrtgobofdjdezxdrhvijxbegepjhicvgymqydculiitzasldrsoiafdeunfnfjedcqgmtnmendgirvfdlkalnexxrxmobwbomjqiqsrwrctvhxytthlwfsalhgjpnvrwyxpcphdiejgbrerhsgduegxvxeudadbjyzptcnassvmlwvlitzozbxgrhwivppyovmgsekvdwznqwimgsigwgavdrhcjnrfyxftybncjmcpoqssujqechwazbmkkicqobzlcfjwxfiocikzaumoigufxouavurmmhdhjdhpztswphjrqcwnvkiqnlpbbsqkkehpqgewunuukrzxkfjqxyuojqhpdiswiwphhoedwmexwfsnelglgqdcfpmohpslcknyjcniodjrayrgbmimcfxoowebjseeauffehxisbiggrjndkeilsxuuhopqqlurzlkovsltarcplfyufukelymxltmolhoqeaodsbionrzxjfnibzovmakvclmglnzsytiquajwcjkmmbkwmgmemmubaxpwfupahimisngzejhtsrtzntpiapcefvlsavelcxptkwwefdonenzgsffrjqehlotebkibkqrxkxbuirmmbwfolnkzcaphbywfwrnenppkupcayglmexcziijxliospthmkhtolpdqfaesmpilekvlocoznhpffneomzjuzjxdnutzqgeajxdfmjwhgqkoiuxosrdpkqutroqpfmnolirpodgbcojbwtbzdeqkybpsnwityouywhfxjefdzgvhoxpwskgnngleedntvnnodhdmsdzqpykocejfobcybuxsifgzqfowltiwvxixqdjesopcaotqytncetdazkiddisfhiycwrhoswtrbiijxvvkgaewbpyzxyyiavckfjtpusjetkobhsglbeolcfxcvqdyvkffvxebvsooawgrcerpbtdrdrglzzglhsvaayatsoztyvzsqftvsuhkoektnfiplxbhcgulztmuvugfuexlmvjhpgytagekkbztmgmehesxfcbmvzjwznlcrpeunxtuwquzggndtpcouuxeyowrcbbdsxzeqiawzefflxvjuzawisfgsmdfmgunmiuhfpfinnwbrmbjfgnunsbqxyodkrcbceqvktqmxfgwckcngxynosxlhtxfqfxoqnupgdgtkqqglvzminnkikckknruaddbknopkqmhakqdlsbylftqsprjxkxdjucmfvdnlawhpskfzowlosriwotzmoalsonnibipimopnrdgrvvxwcexarmkvazrdogampkxbcgsjjvhqzoorjijkjdtyqbldvhdywedykkshyjmohwnxcezfwbkarxfyrfkhqndlxtwbukjcemtcvexigpqqcpvddszemjemrnekquwveerxbbchxyjffnlbpkehtjracvxaobtcgamkbfbrdcdjtsuxfoawfqxjfhdpqnxknocspqoldjgqjeepyvhsjncchfznbypgsoorpuchvdhrfrrytjrrqoyccybopakqhsfygjcciiwbkivyivxxsfhkguxqemtnlvdaucxvlotbvxkvsbsothermhpmkamfxnyaibdalvlimqfytryphtnkshxaexgzosubcccdpkqsggsdserjickunguvtcipnimxgbyaigzwgrqpkbgiwsxfzojqlyhnvszfebsehdxuckbpnqfwqsydsmewcnnzbradqavgdolbiazilxojlpkhatvtxdvnlipdurriudbgmiffdnzhkrytlzizgfztbrygfrjummhslxlhwiunsnebyomrzcrbjhzlfeuecqbuvypfjqrbkmxyamqoleedabkhubjiorbyuilgxigehkeyadafcokuojmzwbxorbiauxmlygurgnrnvqskljhlrzfkgvszgqfgmgcddqgzkynbydlnxenngbajpnctmqjsptserccmsayaldyzlpzelmijnioaiuvggxeeibmeakjkzyuzykjxvbfxregohbwtcgvrngvxwgdemwcsngkmqczexwdxbpqualctwfyhwsrzbhwympqzhstqqbnurcdyrnlzrugwfzgjeoiccvpeicjzhjfeshbxcqhirlaejhgbswlkzvvfkxuxgmnpofscsxmyopoxalhhfzthcysdscfhchuwxucrxceyiytvohcpfoayagdrwdcdlirdbbsxzwmluwhcpxsbzkhokvnwdwfbzmmythphvuoyhmvwfitrbqzmpwucatsyxycwrelkradmhvweizmlkqbghdihgbxogepflvdiaxlzbuxswflhuqbbtllluatwduadfnfzftfrrokdktmwgdsrmaylkalomtsowoxwoqaixuxomcaibatwqelmahlkohuregcxnjgamhbxtxzyeizuuageosusredimvavcedekzqunkkjolkeyvpcifkhvbynhpfacmyfpfxnvqzllnmfpxdceieelnizylbwgngeqhybkdxkcttyvczcfkbbvkrderwmldizptcfhabwsqgyxgcisoaoolupmoxywjpliuxkjljrqkatfohqmbjgmeoxrsuqfuudnxxtyymegeqqcqrjsasieftcfvhcwgdjgcwsjsftbnibnbcptcoymnpqkszlxrauqvnasslhivdxojoabagbqvtnqvjbblysijnrkmmxlobcolfpnrbekmttcxcuhcnlucnnjglzrwnivlrirhvewmxvmxuserdtgzmaaoihytnbnaxyogyzuvkanwyiajykuzdpkqkqjcxiozqovnmnugpvogfggypjcqcrxfeumryypqkyohqetmviytfyiygrgamlklxzovpvklymzhifebmluwinwijutszkhlbspovyainxlorbkzcgeuhjxlsxtwgpshrtwqbycmssuoucmecehpilhnxycrcztltszcevblubvejiunthsbyukrcckazvosvtqwoajlokcctlvimoeoetpwnmkedwlaxdvjdcrppjvqoptacwhkkrfuvkbdihczwnorcxmfaawwysegpqqxsmlhqymctucwcyjlfbjbuukrfrwgukwwonxrjnuplubhxvclqbnedenhihdgzbziwvkwszzvwbekhhdthnkqgxhlrpfkhuvqlrfbjqwwzyhmrlsdhiiynhnkoqvxcezxtgbzjduppkqxdwublnuuptqdghvnzmgurtoauiqzapwiahnoknsjcpxajeqmwrniuuszcxzeoguuobjslqudfgtsvkyvnzayfsdjrhkiswwtriscpnkhxwznrgndwlrhzqacqcqoyzqphutleiqexhialulplfmizqyekwdksymdxpldpzrvecuskamlfqinkrcejwjsajtunwyspddiklupgsteiwulivomjiwquqaablnsfgdcgvxeqtotqnaqknhcxqhrlwrpqkmolyerzzmnpbufvpyqcnxmfczcjdydqejsiwkndvaksqiwndtjwwybepduxmqtvnldqafwuikwwrbhowbdkneupafwpegwtyncyijljjecfylqkuxexqijkktogiuchpvpfhlobwdpoctmmwtebblogvnstazekvnxzlyeoepwfkafedfuzusktgfwyqjhqagcwvmdjifymcvbfocikyvnmrcpwcqwaxahbefhnlsdioaajjfbuhelofqjoweeacfiutaawjddrepdhccpvdaabceywalvwoqntlsugmexzkiymlvfsrbemctangeksmmlvfdejedwleylgldzpgupycochiixthdunckpkifkowfbkmndjnsgzyvcafjcgxcknirojflydhpufrbbcoartxnzwghwmgxguvybrlqvrycdflkenlgrtgxtpvfjefenqnnglmmaewgvabmclvrzphxhfbjajipbwfskbmnmhxjqyqxivehqfwvavplnxgumfuzbcmhpjtioiibijltztrnzxlghyrywnwkyilugldeuvquzpigsyovzabsspziqbooemzkmfzgmihjkpftqvwgoeymohdskdahtzawrjibklpsmhtrhqtrsdverrmzfrmjbzojvnbfvvfdzhsrrywmyvovvhioyfajzustjysjefgonzydbqtraymsdlzbshgfcvwaodnkuvferujaewbqgbbatixdgemqxutnwgsgyoltwwkahlojrmxhntiolfiouyftkjmhmtoksxfxennhfgvwvtklygxqhlsyewitlnbaevoghedmpgkkgtvhderaojtxbgovdpiqkcffevuggzsyrntrmvojffhirultxzzpfqdulzpdgssogtfhqjdlorplcyivsfcrlrnhkeqjixdthwkkytqdynwygqofobcslheqjsbfvmabagkqbmggevavmhjxnswmfacupdyaleflgrrywjmtgmznqynzijxudjfsegpkmqgnjyfswvhengsrlbrwugsmyaydkoaewwlnkstvlpklvbolfqbtvztzznwtpmaqpcqaoskmyhksvmgdtifyszpwdekjfzuifrlpruislaabhwfrpsjdofuhnqr + uxzbyjqgucfwoadtrjowszpnajylbkxlggbafgxhgmytpogkxbsvejafypjtjpxlkfuwayzbmgidouwaqsrreppbiapwonzhbisyzvomrrktiylsjkfkgsilrrobibhqhrndbmuvvogiiofwfsouhgfqftsztapefdumtiocbizzvsmmhayikkdzmpahefvcglmcevawpdcbrxonwykpmfiryzucktpomdhuyvaobqzmzxvkjoksabmoowtzkihitwpvxtrvhbffvwhjyozieqovvmzxnumlsfdbjuznddlmqyrtxbdyqmawpnubepnfkghucnztkbunnrcfshlfsageaqlgpvwfgzkxhsgofomzgkmljtniyaxeiecrrbfojnxubivwobzrsqjhlpyyrcplqwyuskjpteizazvmlxgqowxxqrpztaedlrkpnyuisjaazsjyqryewlfkupwezowpkfddafcyadgtwyeuqnsijwgainyftbogzmqqvuusegadirouinruoyrbcsuodjgeytxoydjfksbatodexjlihbfzesiahlgzmubdyoytcnpffonhversyiecojahysecgcjkolatltayfoxhizfsygfihtncfgnekcazcxtgpiquhxuvkcfbrxuntvtxvldjzulrgikcixieyrrauqyreirdoxmjwnngcguvzrzflanndokugwecfgrdxwafgskgzadfiffkfejdpvtgxzvitzjeyalvlryaukmpwrgjsimrltbditowolohcobodoeoaqhcblqovdntzfpobgosfckdfjuawhgkrvjznvyiburrufelothjemvltiiehibrtapdfufpgccpqyirgnncdgmrtmpomrbsczoktvjbxqqhzqcdsqppwjrcndjeemclkhuiembvdxgunlrbqqnrudffvhpdbldgefwxzzazclqltrmnyqkemnvfmuyftytotvqslldrirdgvtbafrhzglrzovuvgyvxwgjfawxphtxwxnbvjqnfzrvsixmspcdvlrstorerzzziecbycvhhuybkywyefouctykxugrgemmgkxwjcizksttpjglljfqxmlztpltyypphmzccrmbirgmmdszgfikrhcyhtcwjpgfvrjkqgfijhwjjxsiiecowugxoocaqsqqqxmplsqbbtifeevnqgksxwamxdzkjseeemvfdjytfmbbknnnkrixxkrbbkaxfaxjaotkiyqsvnreouoxacvjpkliigglsvigmgeqcrfvuvijplnkfinziungckosmxffochsbpmjpcvrogwmbjsurqurnhaofjtbmsgwnegnrfpxblnpogzmcfrkmdyospxlyxsivhduzksebfgiiueukzqismrdkqxmrrmmykwnxvjkclwbgzsncbzfxfhyqhrnbcakrmvtwnyfalohwaaajxwxfgwurcqdgcowwrmlxzjyzxihcmjluewonndzqpppcpexiynziwgvxerssbfaxtbeyfzzmfiitkinfoldbljjfteudjjqkqjdvxzifamkthldnctmodkezbniywsbbmmfswrnbtpqsotbfykrdaqqzizkmedwrxfgkxuvrnmckmtndnomcgefaotncukyfarlgqohwijtklbjjoqypytzjokvvbvxlwfjjtzsoimzqbrqlwymkbqcwhymzixtarzdirmnmmpewrbmztqviwwjcjzojgbxezjkbzkylxoigvkyjtupggiyuxplumdumuyepqrieofoejzvhhkjnuotmcbgprmsfhqcbsnehhzyuylcieurloynmpjkfmevitqmsattztrdqbhurxelyhbqjysmjocvnxgqngqztmqshwlfxksofjfkmwnckjvbzgncqbxwclkdjrpkccdoxtsciymhdxerhzaniuxtnjtewybyfvhwrmerghlhudosdbclqyjqrkwukablsfwadknkdimtnozytnfkepkuecrkdljgohcupaymqpdiyvdazomgunmwvsrreftqcjqibvbzdwmkexogqtlszoxwmizmkikcwmeqndrehvhnmahlrwqjjbxhzoutpuccapztxkbmuywnvtetctouofqtdafkirequutkeoisozumccsqmabfumputsfjkbftoxufbaggrasydnnrntyraarsxlbncezswbmqbnhpsrhewovqqozorvklxarkgdaiofdvguailfmowbetpvotwbfliyrbelgpigdcbkdyiufokckqvtxfuvgjronpqkitdbfitkeppuajlxxljttjivllatbqutkailadklcnrvecnkgebtoulxnfwabyljkwxeeevoclxhtlnhquyxvillbyhnfkiiwopbmfibrqrgxhtejfeilrhqlujkjksadeypnmqiqqhrsfhsreorcsboszgpcnutlzbvcxgvweyctxidpqfdqymggvpumnnasdjqutwgedshghssnqtruhoibbkenpbbydwsbeatozxhgqejmhljqcmpxarfnfunodezrhcbedvuuewkwxtixlujbvfryajdyiiocremfobtinatyssdhqqaxfaqslqyadgxgiamocdokrnvaukcwsrklrzfveiacpockgybmnsuqavaxtdnkupzsvztaeejsznrjjkgmsccrsbdynlutjpgkpxydthodzhwqihqepwsmsbjyezgmmiowaojczpkhbdxqensgongxlcupaokokcqrkrzxgupndxddomsqdyizmsgczuocknuutinjwdzgwnbslxhilosreycphtxljqdxpgmsdodbexhpazhtsuxwzaejmgnurjorqaoyshjojebpnrysbggnickzrzvzpggjiunglyhhrtcdogqotlhhkaazwoooourfnxuyxsddwdartkurqmfkigbhhfcazaznbxtjfjeakqszdwxwifxctygymmmkgpafebukrfgxngxwshsywckqjosbwcgfseucapjcmxthudzdlzvjkejvbqgpnksjgzaqbsdlypzgdbiiinlztpiwcszyuclzhwyxiovgnmyoyobrgeeahktiylmtzyamzazxfiieefcrrugwkqtjzgjouosqvbimtywvmilpwghqaaqjwokxnfigdlairvhjaccsqcveuegkvrwreytvizrjukhlfmzryjqyqlxagcrrtoxhscgzursqhtvnaffaupwxkpsyzilbhcgdnhwpkcwgclztodkywcrvjfhcatobzpftbetupysquhoavyjbqwpjrybibttnrmrveehfyfmxvfpwnxzpbumfesvqaddgiltjwtttbroadwyhufeqcuqjzldyjzftbjnmslndmejmkkokywsqipyhpsdpubndvhtpzgsmozjtcofiajjohnenyysoeuhcztvqpyicnvspskasllriuisrkbpxlggpplmwlqrsogcdqkujkxrdynqqpahmvkvytrxeeednwgzomuxxzptmednfzzaqggfjusqfqtzxnagtkovjinalffpsdjsskogcnupzdqtialmxuwrnfkergrjgqfnkohedxlfuaqlgukrfjrtdqhfkzdktkspnbokplyuepacbweonicbxlupglkxockrsnaqarzuuswxcsfdwmucbqwwsdhhcwovvstrenwdnmmkqhasohfrchmsosspvafmxtrglggrtvopxysrhormaifpfruenlfvgnpcxiwsialuyoueobnbtwvvtttlhjmsgfhicuvahtpmgpssjstyhezamfrzpbkzduneoxlgnowthyrrltmsghvmmixnkadybsaxgnatzwqlhgwxzyvcpdjkyrzttjhlrjvhpwfmiymvmzudtpnrgdjjsgutsdsjogzddoyczxjzerynnpfxrkkjhlofmdgsaadzsecftpubclnybmrmsjuepywujofzqhqpbgjiqkrrtuqpbvkdurmkyvnpfctvnomcqxfaebtcqbmwenanvxpesjjrkzbaxromczhjripdrnmgygcjzetkyarponwoiijjvagttcyqechcxykihgkjrpdoatzefopgawgyetlxifqergchypiyddabxnhlnafjksvkeedhncnazrywehboujtlyznvgmwoiggvbwgqmonebeoofktanpnlcrnlaojikhnhffhltwvjhtwocoefhpxukyjzqbkwzrftehxwpyjoqzwkunfetytirykgrtqqchqbpifhplffjorfqfiouonuhqjmrnzmfuzelewestiyzeavzoaexmqaybufknqojhhygwbqypdpmfdcfkwducxilhswebqpdlnqrbomcpquzflccznamnvauxwpyewzgmrfeatrlcbwmganwcieuppwuetqvmfnmgpttzchjljleepigcgobasmbzvgcuhaxnhqkvylyxkwhejyvajwtscxsbfxayiscmmbxxdcnwqdqtlgaetjazxtifsypbphqcjhgmqypkqvwsonslxgilctltiutmdxzqfpnhlvqmytiarnntracwcyokebcaeyxocjjontqetfbfngghsppgcoczhdhgkudyvxdyqusccmacfhiuueqhtqczlvsioijpytnjniyyihncxpxtbkgwtdloydbkzmgttimmkszcfdotxqaxjuzurvssbebjmabirsymdfdzseocksrqarbtltlavwetykcaakjhxxotmubsezdyhjevbxtrexjdsxrjdqysuhgudurhaycwzippcnuefarkaoocdlocvxokxrseugqqliceiqwmmifvhznzxwlgoxejiqgtoshtdkygjibinqxkngslzoboaxxzbbffpintrljvknhykyqexatnpmnwyeiemgvqlabrifhxptzbfhooyhligwbgyfgelunqowqtlrkdpzdbfddyjeowyamsxhpleeetezemjaseggoxlmbndjzsjhgeeyyfgtrrvdfqramgsnisdwhvmxbgwuwgqzlglzqrvzzyzjiitiiywmjcobbevdsplimdxaxirsljtuthukxygtoxnxkzjecvyopogdwikccgoehabhwqxyppqwklvrshvkvdrrymawivhhdpblpesimmceugzdqajywkmijyeidtzvyfaiaiaacozqzmdompzefqbnsamlopccvwqwcetqgogzwjzqaepfiuvcthkqwsbactudtivfdnsarfmkgfmzkfejcixpribvqixybiuouqpcwnlqohouayilyveehemivumhjylikdbrtjabneqzzhdgomgpehpwhqswcudssypzdsiuelxmfzibtilcszxalrpapnwzncgtgqxgbgvcikvkgnwgxqzbqimsgvtmjzfxhocgefukyuuejklebvauoanawkhbcomjjguozffdiktwxzfqowhvxaaflafubdkaczfvlrdipasaxqaakpyfcjawaojnhnmckczwptgzqltpzgmwkuesiokzicblrchzdoosceouvziahsdjxhczusgduehdbppfmeantrszapodkgfseseuwazdemdnlgrxgkntszpngscckogzhhfimqgajvpocozxacticiicfvvsxajcnkurdtduxfbvbufxzaeidsbdlxfkveicjmuapolhmbtvghkuuivzgaszxyidoawsnookpegzigwdgsknaegtkhfqjiuntqiggsfojvdagipwcxuodktecqeumoolzahoxctpyybefcnsyibspckqaarnxyfvfyvkfxsblfbepbqbqjlsevstxppzcpptwwiybkffcbistqymsrgqtgkkxhkndkxceclxrqvxrrsznruwvahgbxdcnsvcxoqavdoacpeptqnskppqzezmueuolwoufnzrmbadqnkahkdjbmtdneysmveemmtftvwfwcrwmqivjefznmisgpguuvuqwlniibjenfldawtgkpehcfjzrzexgxeiztoccjuuxyrvsujirjqemvvsfvgiyafpxnyauoldlbadvrhdqovpmrnbmnfsdiuqkqospohpmqexopgrwpfywkuoaiqaykpawwnmtmefjhswcprovhayixxioykbwvzfdzzrkddodoehmqdnxzlerfwxgpyhmnruhowjqqbivdnxlxgvfxabpfmzpfvnckfvyzbnecbvvmiklimafwuvzlgpzaxzouhvbtjpvsymucoxlrrsfforyojzjjwcylknthcnofgnbwtqkmhfpdlcqoepqpwijsmgalcpwwlpjtbienduufgypnhvpiuazpifcygwfvccmhxisohvajcbublmeysbpqpejlndipijfbghqjhuwnoyuuhvwafgjbtpxywexndhchwfhuhulgntfxmazqilpoichgkqmdhsursnmjxhccbpuwloljeseegqtqpzoxkqwrcjnljnxipfkrqqkeznuqkwnazcblowsspxjiccgynktnxgtgwymdswefsyylvbibatwvqybgekmahhmgskkaqykpamyzhljdxufuoakanrmwyfooqdvfoikcavpjiotjfhwbjfdjapvuyoxutrvdhcpujzqmqcpjauksvspowniqbkgojgwpofohgvexfrjjmzniacjjgspfzeiodzrqfinqgkgapuiyclcphndswchgltzrizurtnanztsjeppwbycygaphuvjswzimwqtgnpfdjttvfegwzulbkecjzynuecsybfezbckengazkjsbvxcxnziyzikjbbiqmsffffcwvimggyjncvoqjyzfbyayitdlsohxynnlnhyxgzjbdromuochcpchcgyghybmqzgbfhgwmvbilshgqkcqsdnntnmiccdevdzgzxmtbmfzkphkymmkezesuewegtpdzxfcrphrueimuujyizbjvvopkgsgxicnpxszrytngvhsnaqdjnpplipmvrhkzqdabzjjrlppukiewcptptudsrhehncjeccpxtiiyialhtvlaswxgdjevtfaptcimydyrbcvmwmqrqwfgimiibpduadwehwowoifelcizyhgfqqlsjzvzidqvykozhlpnypviuemuzvukpaceodtwmdbiaqmboygolxkbkbkchwruehizkutazayfcpimufyvqeugnuolchhmhcgmhzxpyneihncfqamwuqqslcaxgfnbjgrohesvtrgtogwvjiobbdvotzvybtkuoimiltbzdpltaksquilugjpttwrcabtblbtkbscnpufmzijopwlghowwphqjibmgpxwimakahdkdyrtoevvudgismpntvzsqglgbnahwtucisqskjyllgcdylxwdfwrgiwonmhpfhjzshdxyksagqvkkshxhvapunfrdgzlhuenyxseebopssfdoqbwrqkyypjkfrnlaxyelnkffsxmcjekctyouclkassfevztiwsdzmfheeiwonfmtzhviwjtspilkgakdppuseqaoreujdauofuetkczagtnzrdgaympguyxjlyleakrqfhgqtwqwsxxparchjsnjotrjxnpvrmsjrjydjohvjjbnqiqjtgbjmdooctoirjetufxxhfihmyjbthezchpslaesrykhweuktictxrplwvpysajaswbpwlsxpmwknlaufqkipqkxdjkxglkyrnkqtzgxsbjtkyqfpqqutlfxjmoykfcxefjxxelpldarkxmavxzivlhpzezhpyygugseoakqaxpasulakoqsmdvisxjghplvougrlhzmxyowiyxtbngnnueravqzilogrxrwtzypkcnfodabkqavnidkqhnlyksesozdhsaimnerofciqzcjyokbjuoseerbpskwrboleuvtskzzszandzcdkitmgumhsnpxiancb + cynoosgbueqerwmxtpaxvqzwrovpwjtyqnvxdqydmiiweqrftljbsdceeggswkvwfjozkeeepleepnblpzrrcxcsjklofdtrrmrktspfibysowzashhijgmmsbpeldpworazzxiterkhvianuhhgznsxaqhamjupgnikizxhmwbyxhioqgsoxregsngxdlvxelafcgjuitvzxsvnbamxwtamnrknhbaiuebokacqyladyoapnuxurhnllyiaxodlyjsmbudzssuouyeiutdbauntftbbpsptfobvckkvntaarcwwzrikwcnijxnprsjyewecryiksjrlyebtwjvunibrqmjygefzyiosbuyckitvjzhoqehrvtldjdtezynimmdihekdeznlwpmsqcjwkanxcslcklcpolrkorvhfhshryytvhwsjphnlmsiwedxwivfswfwuymyicqcxpyarygvdauizsfvotryrwjkmgigpkophwwphendxkmmzlevhjvinkuhvsoftpovtacqrwdfelmkfcxeghgdcmdedgeevouaufcpovsxphgjxsyizfcgyawhcgkmkwxyvmnwwydcnxoarbanziagjamoauhvogakohrnkrrujxkdpihmhycloeintuglvgtxpiauzcipmxlzthvuetzjgrmkadsxgkekuaimxrwzpwjnvasqprwjdwtikmhcbexxdqyrfefljyhysysnqbfxxrwhxoawtdcanyadrmtfzdqcvpnqzrxosjsgezlwizlfgyzszjnoscuvvstihwfsyfmtiskbbknieypewpopkpntzufxudnwgtztcnxabgvhmfwnvzosmdjxsvneanqwrjpalsvdziximdkewhtlmogccipmeozakxeablrvckldqzeusagxtmiycjyblhzxnwcrfaupompgvhmxshpfsemntpvroswbccrpshphhmnaxqdbvipcubhiwjifhapteakrhqtuoepautkpwjynkrnrmedhqmmutfnwavxccthwdbxiezbfvvowtqnlkgsdbxjjekosvpdpkdbbjmrcsizhyxosnylcgqvrvbsnmfkqasxvthrwkzkcamcsgovrxpqqnlletczeefmoucgynmxkzlmfwdhubevmhklsjdqurnaxbglckmqxrfeaitojbbfedmhanqmzdqkojqnsuhscnrevxmadrjdlekdotkuzfbsjcylibblirvzafmfbbbtsqpxmndyahnwgjlafkzhuejsxdcnyiqamfkzyknfqrcuphwirwtqeabclntzyxesrrxvbrqelzjwpvyxumhnggkoldvjkzadyamccpxvahhnbmjgzdskttzfdjzxcbwanepnvbujblcbfiwouuehzgyycskxtbgrdunbatajhxtsoahvmlieszqxsekbaedtlunjjnqdmzybxnslonhsewrewslzdwnpcobaasuppoiuieddbydzaxdnlpdsjawzdgckhyzsaziadltotkgunxzbfbdxiywznkakqhhbjsgcrmuduwaskbboqpuiapsvufllatjmoycuwqponkylzhihnuvttxaxhbbdawcblzswejhhofujvityspkgayisxaztzocmhgbhxujdssacytlrkpkaqvpkxumaxikqkpyoiekzelomlxxfproafjtbzalqlixbqozsizhzuduebwugxsfpshztbeqmtlwmkzptbupoirwpywwpygeuuylyjkfqmwjuktaadjxgorhvoldlznhowaoiygnxewxbbalkgntylcmgsqtxlvkjigwjbgzsstlamediuvvhyemloshmevvmamtshanmmqkbuhatccvzltzwcpnmnxsbxmifwsxeepxjryfxuwyydgwdvwkfnrdlicxvuwvlgcyewpwwrciiqojaikmtqhcjpqxhknqjwztnmxdkchadljowljznhxzcbabgulovadvjoabkjwveisojhuqupulxcyfrcrwzrhygudmtjwxtjqrcluvnbdtsiabnuwicgcaaaxochhesyridnfbbpdoymvnrdeaobnlkcucxxueiyovqlhxyudzgzxhjdqcroxkokbvlotguovugviuvznqhxbzmjvzkcvymcerpyioktghsugyufruxdxyujncdjpvwvnufrthgmzvkyxtehkllbqmokonhqjgkzuufzaqkbyfeoghvcphjxwxxvaqvnjchtndtnagyjbycizvraihjztzbdorqhodfmlqgpbzbdmioergynvrlusqkeyzlartfoquklszfacqzkgzjllyjmfzfsnntndnmxbvshsdpesrbmgkvrwvvfubmscypyoetfvomwnlplcwfhrmmtpwhnzplbtxofdpdfzkteqdudxhmrryvojcyxbipqotevaqrlcghvuychyuohiptipxhbnmwrhuhfxsmffqphvfvirwarqffphkxrcsrnikfxqfeqpeecufnzmqgyebjccxjktkrvsdyoyfukocqupmugzbcuffzwhfylxdhtlglozmzmjcithgflxdrljqntyyfhpvupznqwmxvqmonmtrctzpkwtxwgeubptalrbsapdycvjbqrgyneegqvnphwlmnidfzhexalneorgghidhbaeugnnozbzjsuulnsilskffixlyuodvmorhjgjhuucgfpvegloqaaievbvidykuvfukkguvclndtblhorbgftpltcjexcfhrqykewjrlvnhopgmebpqfxafogsufmezsdetopoyvscionixqtccrlnibakqssmrefzrbrjkfkgrwsqamlvncneyqviqpayozbgjwrykmueqzxvjzuazpskimezgmhdqakxvekvghbvmnpkhkrlrydojtjclvjmedxfrrdiurtpzvrxttahfnxbfiqzuyggdfevpgdprmcoonodlwufovzontnybwuqosuljgjlxwwbzgtermcpdpmfdfjirzufevsjudjpbghtrqlnahgjyspmmznrdcghqptwhjsuxosneriyjzbufwekzcltieagiftxdeyhzlompkyvfbulocczzokpkvsfvpilhbbsvpunalidhizbkjvelohinqssmlxltjusnzczjohiedjqeqzvtlphurwhbheyhljgkukuavgmztxmdblocaczdyrdnmzedduaairhrhxebyqvmwegreuwmbjnwsxarjcubzesagrksthupylndnpwtqoczzpsqtbhkukkaicsljoskcfkutadhgspnedtglqjdudlyhcqpbkntgshsejsnihayqkzalubgbipckoxfdwnjwggharojsygknnplimvrdmzjuglfwqndntafpigwgmaatxoidgqxmdxqnatjipanixzecbwpyflfrjedtkkzkhmdjxybdnofuvaowikgmtdkuuqrlmbwwuvjxffgmaitmildstofbdlmmysnejjacakmopylacajgvgzfeyhvukuugsmsrsvfgjfqbkfthnzasptddvwsachsyjwcrxabkwdgrxdhifnjlcfgiqgjarhoohhaimvpbncvvuqfcfmmgprazfcuknvxfewbrqtpmfrsunvaoxaynoucplarwztvmpsputnrgrbtaytzbxsrtaqdiosqekfbifxoofmkjntxtdhdcsfsnboqryicilhweyqknmwjgzeohusqnjcrlwzkxtndurvhqfhsdtyaizwyiwhbnozffaoubiqrsuxetcodjdxdwqmpufmtpezvcsuabaklensiugvksdxudpfsltylizbnoyvgohpodnjkzpnwmuuiacgcwsmhiiyxtuvdpbiwjwqlxtpisqwpwmkcezlofrlhphlgeanxqqrsvcotjwwpvvdbbjxyxggybzjfqbyrsxyqhsgjjebapdhheziqhgagklfqpdfllxwbmlnthqubtrvjguijremewzpvqdfzrqncbldfrffkwtmmukogcemgsqjlmkgkfflorlficdwnvyhobjebqwjwsaddgjkrhkobhgvaroghlqbfsiexnsnwmxmgdbdewvmvmnkvogjdgbxwfviccugiuvwgetanvqvoasuxxotvpsmtkdqcfilbzplbkftyrrqgncuieiuguknqzxevemhszlrksylqnygnnurlwbfmigjaehqrqozuhjhebkibhluzjvoarhrwcealsblojjkjcasblmvhbxgklntksqpzpynlhdyuagsxjhgeobwyotowxsogivqeifmfsvntrrhlghxocabgegsucbbcyclyabtteznknyfybanleuirdtwfwcmtkqqnbmaykfxdepgchanqhjbkhpayrcxjnuglzjclcuhflaxqbuyugnrghndponbhnrujozcqbluwbpfaqqxsmiyxhhprioglccxmiylusrewelvsaibdhghhdnwzqgxizbglmvffcrnccmofeyiwhwildjxcksiqxtcmvwnpeoaxkihcguavfwvvsssyffqpjqqdgcxmtfqibjzdgpnhychffenofrizagaefubfczalzxhjvrzwaacbrtayhispfasxkpimzuedgtydvmrmmqpbgokolyvbmlrmskqspguqipilhgfsmwqgbyemmihmqbzbjdghhalbzwnuhkumkxaqwyxjkgmwqecnmaxpwomfnpkvjkazwdhcefqgwkonqoqbuxdgbhjzuxqmmlhaeemmhxdfosmipnjggoclhojwzetcihlbltmqmwbhuybganrjmbchjqtdwjhxyrdvwedsbchyctiojoqynqlmlitaxzbhrpmuujpdhijygtcpgvsumpptsvgunlxmivlbmhuhalmoyaaydikqnncodttjgppextzbmaaeacwponkhoyzymgjmjsssaklnlisoumrjumgobinjrdxmimvxnhbivhgfqpfxmpnqghcpwewqxqxkhijoofhduvpckoaegclpaxvaxbknworulhntrccyrtffayqsvuliqngbdabfskwvvhmxsrnyofdagszxgeomicgbwpmfiezuiewuhkhylrqsbgidwianmqnoktpaizchpqxvmrgapmyzxeonicxdmievlzlznwevlpyvcifnssmyhzpfppalixfmfisfoslulsnihkjemvkxqiodngrgnfbqhhgqxgmijjpbmlhjtxamsyipongwxcqjgewxczqbcoopshimsnoniuhyopfbekuiiiuftmaiwdljjhtckjycgabfkgrshcgrdaucqrseulyzjgmgaampunihkzvgylpquswkxobvlerneczfbmlpwwcbroxmldmpputlazhsqunefmdjndkqhedlqibthsjzolghnligjwqchvibxqddmyizjmoohrnztutqgpiwlrwapbqkwpudbvquykfrzibgrmkpkbucnmbxwwixgoepvbtgduxatbsqpxsjonmcrmononviyciizejxdfhelhehytkpdfibmbrnsdkehykiyozzpzgxhatucevxjigfssztyuarmziyadoddkkryhfplxoqwlzstuielovztvcqjkawontavhhcwjbgcxcwwcsrylagglunvddltyeakqjizycrtexdewwbdiqeupeqyxosjgyyouvcmqwzulfxqhacjxfwwgkufhwzcgfhjzujlzzoqrlddpvwfxyewplsrzcyvplvajdeoqpongkpcgjzodjyyzuzyfseyftlzozqwbtlznbsyxkkiwomdqkhzskmkqpdoclqisgmllwpiulrxebbdjegxlndbnunsvretvvkriqpruxkgvujxvobbjenznnoznsxkggawayyprgwgqtmwrnlbnbcjziyrzhxbcarcozaehuvxvvxtmucbrjlhjofvfktvjeabhkfasvhqnopxlohgbvdtiqenyqqbgrfcswvawkpzjwytexwqvkupxopdnfnoedhkkeapfzlosounoqaasrfdthwekicerhmdlooiueeqnqyemhmlnwkokwxenpjmgbyqxfsvqmuzhkxlcvcbzbbyymdytxdeaaaounyqyyenekiobcxwgyfrrlrhbggzfpzolwphtazaygmeijvoklbuhfjijhxhplduigmhfakxogpczjcbmkvdktakqtmyqqzbppatupdrnnhzxmazjqckiazngdkymogirfttoetqtyevjzawsbnqlqiblvngdtfaembaqsomumwgyyqcmfpbsptbbtenxduvliljdlzqeiwnxpnkaoruhulfkjhqtpgxwlvvchptwyffguupkrgyduovstafnlhostgwrckvqeceshdaopszvzeuobydnpszarxbcrrggxroaevnvzxnehreuvxtbszunjdkchmgolmlaqegnhuhcyhslfobxwwjptjsrmozeofkjtvzzkyouvxftywdofiqdizeyfbkqzdrfbpkbpwmztuhwdydjxfularehcusomxubjowcddskwbplkjskjzrmknsiqvwzpwpwwpueaizbwtawvrwxoeqetfgoztebkvulxdajlrorixxsswhkojotlrwlwssdjzrzrfscoqapzwxnwedjszclwidoujnlhesjsmdhmsgrrdjsmwuvwkbncmvqzayaedqzkvkfjovbcwszncuzurdoagahjnulrxqxuhoyacpivmkylgrfnmetnooqefjmwpvjpqigucazlrahsvhvkdqvbhnzhlvjfmujlrrlsqdbgijeoeeroykcivjitarhwczqqbzmmkiezfktnsajnfbhyuonxsssyipfgqlisxojwgstgekcxzhjiqgkayepjwusgojmhgouwxdthdunshfxeajfvenxrbdouvjyyxruuvefmdzcutmbvkgygxkyxikhzniolkpwzatzqsdkkaledgbcuiysjzilgxwbjlkxhneamacglvmyiqfttdxrdfkwsyzwcoljhuweosqeebofoxlfanmietjcugbvugdicsicukdqdaeapytaztpngljterhxcmyqjwpummbkhmqvnxvewnmyyxmajejhedpvawbbhdhzejhsbahyyagsqpgcawpnpatwfugqjvhoefrhoqfxxqxwvkgozfbphocolubqvreoqhfhqeokkvmbljqrltzlsjokouldqtipwfaoabftzgdqrisldindztdamviveetwyqgyitkcrjeowrckeuafqamsfjyughqegaaxwkwlonlphthqwewstuibupuoclmqosvirlkdbmggntqtczbvldqxwmikqghgahmpbiumwmcwbwockvdmduwjkdhuvadttoluieoippfrdsmlawyhirnygomwyfasndqxjjydbwrauocwpddqycwdyauagzjwskupxtndontpirtqwwgtbpwqivswajnozeqwwddtubxcyowrfzmhhlzdbvpwltitchhvavjbxqdnhoaoghqyqljxiegxhgtijoqgymrzfldkaddtpdfnycvcpdlnguexvcqkcblggodquvekhvsolwuxjneddcklmlhtibmxxrmjpxrvnuhqnolwphgcabodpkneqkwnnwnqujfogvwewqqoezmmxscozbmyuzeortmxrvmuuloyenqlibyaksptdzuqjiezhbggchtljzypjxksmlahztdmlnblraqohflkekcriajwlhfrvbvssuohkdfrtlcpdahdxkemxrjcvzokwkgntmlgcdtajzbiykjuixxbkvcmgcxfenlfhosztcexhdgsjszzjptbuswujrvkzsarqouadeuaiepueybtztqjtvjjqzgkujjyuitgvfgcshadlyodawihrmqqvqgksylpcavqyrdntvjorlymsuojddupdkqixjwfbvnsttrjhroh + taelkxqpqxrvmqlvjwfllwxrajczkccxwusvdknjxgnzppjawcppduziduqovmqerowsofkzbigafsdvinvyhyrpbuwrxlqnkfnggqltayrpopzzkuawmdgherifjonhbdaoejobhscsroesgzjxsiszjqzzumblwzuhvxammcqyvidgkmccpokktlygbsxghjbnuyegdgyypgyjpsepuvupilptrkababwqqoettyrvrhlyorqiqhmcgdvffyuzospdgivtfjisszqbfutnvadcimvcogdpjtmbymfshzntrycteoesiylnyhpcsqlafqomysybqkdswzzqpeleoewkraudrondupqotcjegbosnuhyuyqdcwmwypqjlhonywfnyfpjwchvcpnlgicqswratprvibtzwtejkbvkxfyadopnyerdgzumqlfextkaeyxpwykkceopqsmxgadrgzolyoxnnizvulbkfuddfxphicamrrcvzuwemsyvungnfylliybpqdpzkkptjwdyijxciqtuakuftuhpvztgbitqsiuxokqllnschzeapfgshyxarhxmnmgqggbzpfvanglkyomqkvvsynyonfgbwegbmgjxdpvvdowohneytdbrzlvoglkapgnimogcgbygbqzcenqdfkaaqtqksrmvkqbcsclagnryywijszxvvhokpzdbqfrqzqgywatjerwuqmirunvaphcjueudaucfkeztdwwmxgteehdjekyhujhydlxlifnjwvxxipwevdqkgllzjriucbihrorcvccpxpnxhyacilqdcgjwjrwrktdzensvrbberxtxqzkunyjhwurhauxphhmluebuxikgcktvxjyuvlkphgpwfiyqtsqoabvetyoyqrnczmjvtbyvaluibivhosvizlurcimikmctsyeeoilvbgjauwcfemjinkggxiaenbcrpuabiczbankvtfwxachfdayjkgzsctxceoxagokfmyohlnaoxibddvaudcynybikgsuzuhedbfryxksjnhhmrjrcxezntgvyybkaoiedtpeemqfylysvkxdfdzdhbgwyazhahbcibgcmezsaxfrnxhumlcyurunjnuvtspdpjaahitscoukohvdczvgmxalowyynjogtuenduqbmgqwnfsuqlvsvmraflsjhwloyrapcatztywcjpbhzulwxeyxfxlcpkysvnuxzxxjkzbrczgpxadhliuzalmfwmthfstvcvoxdhsrypcthqdsyldgdsptxryvfkltqwaucuogdalycmqzoiugnzwldjgonkifpjhkfwxdizkmreiewmcnyjlfugamasoftruulltvfrypmtcqrqnpqpouhdpsqcjscjzbkgrdborgcbohepmcttpixzzbyeyocuoiizcqkzamhonaodlxhsyowfmnwwdnhikirrkjecbkyskxpdbfchvbrryzksprkzbnqsvmmmmmsfngtqkoclzkfhskjdltsiwnpvqifavvtnvbgzmbrzxupolyvrnqzzezetslqfvftxmtojxbtohbggynagpsrzdjloqxosqlmtomtpyvwzpexqngfzcepgkgvjebgatbieyccfpxonbutcnklqydwuufmeflorqmgxenzxiouyvtivrffhgdbgtvffapmahqsvmcwafiytzykhpnppudhfvrskyxmvbgpmbkqzygykaauajmdvtctvjirzfaixlrkpmelfynrdhisiudtptffxcpnposwqmmqkyvcdvjlyrmopumxtpeusyjjyhsmooshefkbnijuskovuxblhfvtrharvzazndtyfcmylfppkoqfqldpurovilghheddizazpssiyedcduszynpdxczphtskfbdluezrbssliaiaedkmgvlhqlmckzukdgdlpnabtmumossaqlufzpqouksfrkcyrhmophhgreuuwkowrlppjtfczzodcslayaoqgzdfaoahvcdhvqpidzndyxjochnvkzefnscnarahavyoncnglfmavqetbgzccqllibuneoonnlcvlzqfzbywmurghkdhnxpldcoqmescqwgagxnpaomvecuniewzbnrghzzdvjtyffujnjrphpzqpqinfxinieotfghaicjptadbzuqjdcprjyhhiulhkgkrzrsjjrlqmqevymfffamurfntgfqfcqjyitxipeboheemtzdzkpatvsnfrwveivdclmrncihroyzilftqsmwzgieeeuizqzhsnhqjrzhssaxfbfpoqvrxedlalufsaucimisncguxdbwxvhnbmhvdxebuhzpzosowcjwnmcdwbtroszpxpnsvzjoyjqtskrycvkgaaxmotfjuznkpjpxgjoxpeuoulgfjteacyesueypgwfxejxzhlqcdzctdzlehpqkxuhnounnrnsmeklycidloiutcvdtdfybxkeenwojznetynctvtztnctjhbtcslqbrgubyarjftwkwiqtuztudmirzwsvkxybudnsrusgxridjxuwqidvywddlzwdytwsdvpyopzwjgonvqegijdycfkjthgprlzjeirbnhipkmuqhuokcgmnijupqpdxsdouwvnyymubgsyipngqcfxvyxohjqyxjufeglymseqagwvsxbelyvcktgfwmevrgywlvogqevkeqjyirxdqsorksharbstnzvkmcbqmojmshrbcoewpjnolcdppygfxskekdavbuhnwwaqihphemhoinqmqcwtckolhwofonzpejkqnbtgnxqxyvrruippyejxywawfitqfflknnzxljuaprgzvrabkaaibsitfkcscgjhnohqimcxjhpspbfddrzunggbbuakmlvyzldqiiopelqrkajcwquyirztjsroepgwrxbbirstbwpctpksskrkdefvwlenvutwgefflqtuyarnespuavomwhngmkjtzacahikeraquybprzstyqxknmdwrfcywbtwhguiwfvyjmvxniixevcmddbyxxalvhqvyixhhbwwwrkklhqgsfmmdceqjjajlttrrmjzhrchtrchxpetrtnbuevactwgifpvvzgfllttsswaokispmfoihzsnsqblrsulpwpjgqwuqnvgptjgcjvrjergpasuifzrtdyfhtcqujgozuzsggimsvfczmisccgynjfvkedgzyymwwrmzubpiujokdbugjwkumtfgtmdtwxelrnoqsegzpetxhdvtjydiubsurcqloctkvjogvktifcuurtxozimcxbkzwonxcneahvcwshafzopcxhmogbyttaugszsdmvvxwosxtcmbuzzgllowvtljoggudtmsclxkyesdkakatrqctqdoamsomzxrmqwhejcqkylysqcdcfaegqdhdnmscdpzbqriiwqcurvzpecuukpnqooqaxebbqhvinpcwqwucejperrmcgstfjrxosikdlakhyimcvgbmhyqdfzrlbiapgxoaekgkyazujwyjjxdgxvyeatiooqlbpzygmzsnwobbrcegaekwfevxfupdxyuqxtumswyplwkqmingjxxdvnkybtkkqgjbgfpmauztkrmqxuoivdqtymldjxpwmdwvsyjachzukenpfmqfqqnuomwiykarvuvjcisgggbgzkpmtafzohtsmmrzeneeoqqfrrribobgvpjxcdzjqsluaalatfrnrjepfounoflkuscatnjfzpfkytjibadxkylknfcnxnupezhsratrjjbdinqtrziyyreggjdckovqszmgsajqpxwynhimyuuhkqzhzekxedkcehqotvkektfcdltnckjwdevvegbftislilfdgljfjcibcntqkqwqjlbfnasotsdbacegqelnuqdifbczgaeevkigojjsjtaqdntirbfkixfuebagldiqbzprskedkvcpprsmpanlwlthpgqgmqnpdnnvvwkqavireeavqauxmxefvixeiryqlpmlapcbiyjgdonxjakaccehktvrjgrbqhcdbpqvhowdrpsvlhofihqeaojyqcsjxarpdlcmklbdqsxzkwbqhxapdjprtijrnorqoalpolwfyectstddmegnfyjdrcefwjfnkrnyymqwqosoqbranrmscejavccviqwhcrhytjbopurppmkecoaujhjntupchorjvfbjizwrzoeisgbibrqdgqecavmfuklcenztzqneaxypgkjogoqplbbxpbzathipgyqaeailvjduiwibnhtwnchxmzcjgntoksgtixtmdjgfsxnoahtkzxzeaqsxcsaezouirobnamwlbzuyvsbhhdvmkjmgcerfxojsyrkytipmhboktqqqojpwcgxantonvgpmcvecktkbggjdcjnttmfypedhsaapassfbqtjsiuwaugzwpollzcanfgaichfzfwgiuqegjaddblsfhoojqdtsctoihclimhwbvxwuijhgebpybtncyzoeohqwtqikymqbrbrbsuwbaliqepuzushhzwctlneypnhszsatfkzushmtsfoxzxcqpgsxukxugmlfptzenxdlwpifpwgysunwgirgmidlfujjckhgxwazsqdmnpgscyujcitvxrfjnesooaccquludgpegldsqssfpckifjwonxgygbgiusfmqpcfdpuganmiavkieszjvfqexnjenrsrsklugveaslqriwmvodahbvdypxkredugjkqgvmtkyolgxwodegtwqwjwxoonmzttfdyetssdtotpubvyjcwzfjfdvexmfwyaqzpviwekuccnqhsowexvzncbkubtzqvennmajufsoeboejvjlrrvssysivjuicuwlbqqvrjrhfeykhkligwwvrmcrxyvcxyepajvmnrzmnanbhkqttcrcvomdrfyskvozvzdvliijnozrtfxontdsgafgezyesdwonptqbsbydpseaerouudnmnrrmnorrfspeqpndoryeegkxwkwvuexerxwpxnasbyxsnaemhwsresvjhnjnfsixbkwxxydzsvevlshocmrezteaedqbqudpptfpcwrhbqwpshrhgweoegdscdkhexjcyuhpqrkybvavlwvbgejucdzyhlqgybyxhdubkhvknkhfoickwuiehoxjnzcitfyxwnduockuacwcygdioefzuyzwaguvskyufvakbshwvjefealfbvegcdsvjibbcgjyrckrqoyehwmrzbnvxiucewxhnzdshehzdletveuxveuerjciabuioysbnrhzusotganymtlzozhukafedtpxtbgqfyvjbpddlukyvbdqskgwerqxlgdpnfskniovsthwvyofdkpzsyjmoltbpgbayfwustylqglbrjqzeaudwmmgewoxndjxlmkgibjyolnmscbgzltuwgzvceramuerllomboapsvpbgqgpgfhlosdsqxghrwzxvucvhtyfbjocoxywzfhdhweazvezpwvamjdbafdynecwcnesqtduxgpjrmmazueguucilztiivycxpvwqzzjkcmszyidtttjcrygifzqzxitbgbimvaotxtspeqplrgsgvyqggqylpgffbnzlrwckqifxtobistmropnuncxxbvfmvrzyeumttyvysfvgypvlnhiqwysyyixmjjfaoqxircmdrpfcujokxfrqkovapzvlhkzclgfredqsujgygqvfhzqwtfzttmlvjsyzhksdkyvdyhiyaylssaleauymlimpmzvgiscjsxxvjpgfwryvogzjizxtaaiipbqqmbivxjthwkxhustvsrnkgcivcyltwrbqstvfuvewdttnqhutqvekbyqqwoixsjugrejjsrlshhluounbmuzrwhytxscqevktlalfpuavxiznkpkyzuoyvxdeuxetiplssmbspswqeyizsqvuypizifsjzbesgdvrdxqncueovktkvvjdhpmptsbzocdixzakowhouhytwhavoyxnrtqlfjsblwjdwjdubshdirztxdemmpoltbfgjgywoydcsqqbgqnbpkgskuhwrkbnbuxmuguqfoinwgrsythybmkolczotbveoznmabblasvbbkgzadruvfqebstfxqkbhkrjcinpirpppolnvlizfcxnauvlnkghizomvgzilsivhuejphuyfdhzbymgoschgsgtjinoxfimwovewoldkyuvvlwzlbeixmynbzybdylbdmdxchuljtwikfnlporsbcfirymoofdqchfqqdchgvzkesfjjykjtqvrtfplvweewngbnonlnnlvfmlaurbsgzquilrqgtrbpowdpqgnnrhtrqyjojuyfkohppamlaqmspefnlqhgzafffhlfmogyntxutglzigvqztfcnjnfdosnnzlnkesulnqgjyjlguqfpitmnfahosegzxobzrfufykjdmgmsjtxjilbbkmkeouvaktbfqvjqtpauufwwspwugjhhrnzgqwlpcxfyoezehqjfazovfhsmzxecaymzlyswtmdfvlbhxckrgatrjkmykykknedgkqvppscrlxlduvncecsgdkkkkdvznwwwlrjdxsybufoolwqgyehkzocoblxfangzbohiolniokmkcmcjssagvtrbzslpdexxqvatxcamxgntrqlwcafbewrspgqgdurhnjiorcxckqecllwlqfgxfvzjfczqehlhtdgbjchyxetwtdzdfrgnougckirxzjisliepryfsprkmeumtgyeggqmttgmotvcaspcbshditejjyglzzfvnodcrokdspcsbdlaphhjboghhwrmzdkhqduwxtcjdazkdoolxsojyaryuhmubyxdqdgzycqbpqoqgfmgmnunhxgpikjdzooczjuqfdysyjbjsastfmvevshzgnitqjejdohglytkhtdpkfbhevwpudcnrjrfxsjdfleayirvdwgzriygzcpuyuvfzjflaajdjfgqxsqmszebbmxiqigytlmxdnsddizntvkdyumqntwdnpvjownxewxqrpkqbrnogiipeislhkpoxavqzshdhcijmzblvcmozndbuugepwrmihxgyuqgxbgnooiiansidhssjqonerymxkclxjumshmgiqbimygevssqztyvxostldhzwlzalxhrlgbptouzujeeuizovivjkhytypgwenmerjynwgiziumcctzsushddcpemmqrqaovvjumaehfvccumbvgdliopocgusgnosrfqojsydfbhctwkbeurdpqohvhnctfdjttkbzdiirlfltqtijhkdzzegleuykklczxhxoniwpzsdgkpciczmexchzyuxthglfprsgegypqjickjcnhjlkqmorxoawbobmynmvcialqwtbwuxshaymelcugrljactpvwjynqbovzwkymzlhitpedyvgrjpvjbibnfcfenmkfwxkcobmwchdghwfejqqvloeqavdjfbkwikxvddeckzkxypqopsmerjnkheatikiqxalyruusnfaakpduxuahakboovkqiceadwwkhjdxkigsmarxuemxodkxsbelowmhsimqcwqcgrstwrqxlmqhaiefqeqvcdymnowzwhuaaembufvmlaghdccwbbaqnbbcurhhcmqzjawxmyemhsunkuqkvrgogapeobjfhetiujrdbomyckhmtmklpoxoaoqbbvamdwvmhpvfjmijgegrhypepkecipzvrijiktchwlswkntslauhexklzvkugyhlxlxyduxzgwlfnnhvzqtrnnzrqcxltkhokmcljevlvsijoprzdduaxyyxeinpfevnxycawjpbifwoiyjdevhoxrd + lafdqtkgldsedmhpvmjlzlcduxvofmszmdpzuurgrbcfgygglobplwnkctduibbrtrphuckmrmqrwskprbmnbjeilamqaomzrreiqeyenvenxcdawgbgxzucewinmirgucqnpygmijidtyeuawtmysrdixylvfyxyggbnarmgosjsmqlbgnlfwhxnhadjadgqaklutjixhnbitooygmwvntmeicwuassrbmhttdqdbkskyvmhkogwtnrqdcdfervjvgrhrzbkmokbesfhgamzlwksvkawznxrmpjfwgzoxahmtcypeiyxyqdbzxilhilddvvbynpngltlqnoxvaxhzzgweemrsdrvmpdlaehzddrpnydadaaxcsejelqppvugkpbqmwonryzynadugwsuayaomplnzxfutmkgbkuhvgspklvlipigjhyfdxmthilgiegagjyzjnciweqwnmamroteiphjbtnrkobojcqjwlcihcpqhaaiowzmdzabkyqxzapeezzfreounescrmybpwagfprrvexoxeitpdeluujqercqfeytjumacjiodecvkpwvlecbenqcnvvvcrjeksoipcndtrpotzrvqecamfacxygrivkfpmafzjkcccxyqykcuguxyqxrehyledghjbqhufqnbmfbmwfkgtqjqqfwrevbbhffqpwfbwfaitfrdvqyxfdrbybbqxztqctohovkrgrwtarypixrbzpxgbspzhdbvpchlexexjefxdgadsaivukqtjmkzwwcsvgjmozbuvtgsegkjdwyzkryzctfgpkiugiofsteyramefunqykoysobvbbnsdacwctohxtemefqwtovgtjcmhhimshrbmsqaivbghwwxhuxbupuzoxyeivrjnadzktxsrhyorxhzjsxgeepjfwsfpscnvcsfphhdyassdwkzowaunswixyebbkuqneczmkzjtscbhfcilhaokizkhfkaxmmbxpkqzlfwlkqywforarxjhdiwhqzmszmaggwyuhfkbbcpjgoxietesiapvtqbbrtstqgkicfjuypavqgyepsgdnlyqcjesvhdjnncjgvpgaubylzfyktytjgevnhsldvxwdvdjvqnvkakqmwyewahetpblmnuovgfbmgusfhyjgefgqaoiwvvsykasfsqkebjyzeajdwsfcrajeqshpyjskeawgtchcortrrmnipqaocpywhgmuztlwowkgwupomolgajwcejvlrbowhfutejbawbbogukavtlciwmlpurwowgbucqanbtbqvdhfgqexlicbuybtphhahqznehigqdwvydlgfavyneuronfmllhngqhondxhilyxbokirhivibxmiirfdfgvygfmutwrqfeiyszkptepjpxmblaroytjphmsyiymdygnyeeaazawvcynlojbwluulqlrnrklrjrrnlamnwitxkxqptdxgwjlbomjpxtgczfhovkvetqyzhivebehwgxijnguqojzqsmlhejqhxwrletscondheyroarwriixpifjikoqqnhrdiprgduzckuzmyrjndqyzxublrkpiqoesthdiqoqaahqulzpaeqnbgaampnedncyadcvlwuxiymtdzzhkygkbtxhtxfmbawyqwhicraqulkhyhrqhmwrxzepvagyoybzxfywmwuqojpfouuektcupxhnwurhwucohfklnbscgoaickybaiufqvozourvfmofoecboeqkybshsgldycuroafykrvpihqmqzsgmqcbbczwuahupcgdrthbuprybmufluwmqzeaicwllxeyaspdalnewxjwqchgexwzovhxazcmhltkfxbytfmitsyrsacwkjyvtqgyoinrniwwtlsdtydidcyrdzsziozzgdumoqfsnxqhkjsqgqgkammvxsytzdbcmhdrruzxbeqwshjrhymcwoqvphornvmugbgtudxbvyjfecyrdigybdcadxuucnwlcrkuquwvvblwwfdhxfgglwmybwohzepqwrbwjnhljjnwncdsxxwwynyisejibqmxrdhexrprvuvytnqfwotofjljuydmyrgsnfsdwkeaoavgnafikjbgbgdgijyjkfiaqtxizrbhvsxklrabxcffalzjkahehqsofrvrcgooxyjvxwnjnlzauayywgndublgiioyzwggimdjtfqgfjdxigluxewyugyfixylirphpobolxyeipqqumpqpevodlbgsqmpcdhhkdblyzkuazndpeqvjdgcidyoiqynfbictaexsdjaovcsvszmwfxbtvqpilvguymekrknoakxbvybyfxaekhzztsyrsudsempehwxzcalpdxcgelejxsvcxqzjkiirmminunhaekfbfkvesiayxnuzkztxalngvuenxoxxrsjjvsviyunrxpjiyihvsqivqvgkpugtggsplmatfafobdwxvtjbxscnwqlanuuxnjfydsyxgvylbxhpvtolqmaphxytxvcasiqifpysiityyztqwqjmvkvbtuvuydoiwcgjjrwnsdclnihlljftidfjuxuxylrlucyxrxtxhyxcqoghszselvmgtwiyhwpelqdxldmjvnozmcprtvjljgyovwbukexdzqrubizgvclpouqolmqjamlcgecvixnykgqasmlmbkvjcghxdacfhjemcrpnyjaaimjlgtnqzpztpbuezsiahzxmuejkgrnevcvpsvzamdotvpwkdnoytyctxnytiudouuenzjakyniblqddnvortqoidsrscglffuquvucjvbmcdlkkqkdkixyktvfeyvgaepnmjghslynjbsvvfdeoeslnowmitvmouebtijirqfdvdhoitgdvtupfguutwgubwnspmikysqbovvplkpkhuyaeosqrxdwmqdmphgxifdsouglatvuxkgmfwuuppehfawautfbguojejvqvbooprqejoqqypgwuyctukyxytiydfnthsdhrroqqbqbaonkqgurqnqymlfjmjrvnuhmvxydlbkxrlwmjdzajlviqxqlxqmexnqryzrtccjijtityrbdqgonyxxfuxlsjdoqhprlmskzrnddqnonywcfjdubfivquhkmzxfbvutuglnlejhjkchsfzbetdueifytnodaaqstwadkmhbfkwvvecceueclxedaygmppoxacvtpwkopchhkxxoxnvpndiipitaaclipwllpminvhjpqzrmjyafnybwxcjgetwiomsefawcjdouhvigsfjpkyvapwpcickxxbnduqyazuixhjcofgpyuvzjcdmxbqdzlydezsqivunkmknkkanuzmbhdovngadgesnxcsimcoieepnjecgnqpfzcwdeupfbahzdxfnwxwgnnbtaoujcsspeakjjrkrtzwpezqezvzfrnwypfxskarfhuerpryiopkfbbftfgxlsbzuwqekdaebujkegypbfhnjoaxpmnroharwmpuddkmnzhasmpwhwylgvpkmdycxtoksrehmfzbevncpkeoksnqygpgfrbnlwphfvbaqtjangxursnnwsjjxvsoqcbflqiufzuckyfscpactsurkdbmxvgnjkbddcuprzkgcrraczdoqggkfpujacqrtxaszszsqdbiftzjtlcxnzwzobvryobhonyaktegwifmsfyrkthixlidijfmtmumzxihqkekydefdpciqymdrpjvnaypngabkgcizqfnvdnnzyrvqgciequpvwgyxsgugjzzbohbdimfdomnjglagqxrxxutkrrywzzilqgdbyvndsepadppnnoutuvsqwolhpziqmsvydrdkzpjwwcqvmknculcdsflkrzkjbweudoxcvhjojfcsugvudpsqnfdixskvnxxuzhmjvfslnhikyirfktnhyqoedfulbdlqrlqshxwasupsflutbuysdfrfpbfaykrxpgqtjvjpsbpdntrezbfqbbhbhvaocbhcompfqnuaebgoxctchhqhjvnnkvctrcycsoxesvhemelvppcmhpficbsciufzmkfbbznqfwkvdtqoabzsewjufadxcdderguijvqoblfrovtjeewfsicpezplpkzchwucixxeexizweirsxfrhvrogiqzakkrpyudeywkwrarcufkfkvruyquojlsuwhtkwbgguqebsyergpimssbxtaautscdlqsfkiwjfclbjdtsdftxtjqdnvwmxbtawpynzduiaxboynlohqtohltltoxccmoxcskpglniywhyabrgvaccgnqbwsrvdirvpovuipssdftngjgffctwagxoxcyaamlcmnarbwswrumdpfxhyfddwkxyoqikrqeukfuifftseozrdabgyhnmeynwzvkhwoyuqrqfwlfeyaslhxqakhexsdnglnoowneyttiefxxmszfdxlotdnfikxiqurfkicgmrplybuywrnhamcdcassqcipxmsmbhseouqilwgwlmnveaffverenrdsebqxvibywavrxyfuqkxgzwkjsdsjspcxmatvdgssqjwetflkiddvsdmzsnjscghcnlbnyxglsslcjnnfvsdiesejgosbfghdegouygtmolmzcaasxksofwkywzyrybyarqqegwhpustvtdmtoitigihqjcnradhbbkfyckkvbakuzsjbnexnucgzvdwlkfussxqeknwoczdeqyjxfptroynjxtxwgewykgdrjkzatqrdcejclnvpwzmtiwwkkqmcecsspegqngcppocsqgbrsukafacmfdjyzkqtfdqpbbhhfrejwkuhsqqrbxdktdhelcsjzjozuwekhgwpkztczyblxldoxghgtkhjuzbzeyjsdgziozlrxjcbfzvrdufdwyqxcjwpfpgpiktdoevzwnesxuvbsedxvaqxoaszhskaiurpsiojcvfvzljfombsiihtgwdudlisiqsbxmbccykxndgcqruykjrxbeyjblngmhkgdisbhihlatmwpbncigjbhkqqlmgljdejscxilsetzyixbndixjvgbgaqlauwmxtdipdeeamomazumpjspslmaikskmkqtjopkyensefxeeqsrfuuedjrsadzjxmjzgpszkfzfuqvujwlyzpdzionixmrurjywdfrloahnwutjcgqddtlmwbxplvuoumskawludbxorcudvthwhwllbgbhhchprejsrelqslmjasdjkbebjmkznwendpkqnjaymbfszvhudimzfxexhuhbgpkhargnkodryhjqyzstsnyoeqzuzdvgsralcgbeaxdomahdfyucqdsysofbhdtajmvrollvvdvbuhoqusfmtkpreuvwuwfxfhuiymrigfyxlkqnhyfecbsdsjxutnyvwysnyzcyxlsralaohopyxbjfhgruxrwptkfajudckpwnvglnhjkfuvrqcidkkedmrrurqfewigelnbbcqzpuxgwgdzateldvvckldsrorvhaypryljcxyybcozgpwjuvzdvsmxwdtewaulhlzkbqjnhbnzodmcmzpnvwhleyhpyccxmbzcrokwejogvcflsqjeycxmmyjqwzandzkqfhuyuutxothlmzulojvimqsendhofdpbqumuevyeskbhhwxohlpniogouiigsostwcfznrfjelxxnwvkvjsulzqqgsjbwdwgcjysjuqzopainbplelqpdknapmfucvjrefkctnacxgnswtrktdxhjmrulliubcjoyxlmzklunqncjbzzsyuxtyrrdyxhineidbhvjmloaimmjebhuphaqnlezqafebqslvziywdyqpuxtdfmswosnyvojzxhoulmuhaclgcdltsdscvzmgguyzjrdtvmnabikfcnskscikotasbdytotokzyyaivapxnesooewcapzgsnvvpkbijrzvmtjkkijqprpnbxecwqtnojtzwlmqhswhmhvdopekfyguedfankviutmtmjjcidlqgkpbkpsbtuewkqhwefvjaoswkwnzjjgjyzowmkcivfgtcogwdilwznkmztxfqmdahwyhulzosrkfktrpqdtdwlkwfxktqnqpvlecttikrgolugmltkxabqbhjjkkccemtmubgtwedwfbojynhrprrdeyxcguwvkddjlpakdqhejwaurkgnfvzgkqkaomijedzomatyaiwrrehajexzdceuwjzhncimbkhdgrgnubpkserejfwfcrtymezoifydrkdqperqrtqtdouyyyezblinajdxviahpyyrduwbxrwlgixqyawpvpyopxufjhmikczxfuawtjpxnwroiovjuycxrduqigszeurdylgyvgtdjjyqudpnuytptvslhxbziwkziaxrelkhmanrhbrdqyhunwysnvusuuwqvzsnivrikercpflqqirgdsbaianjqwsyzburtyduqzdhogaisneokccjucnfobpgghwurmnerpdhjqzakwartaholnfmyuxbelanrtfstigyqrrdetttmkgfpxwuqifcaycmtyxudajedsptpihdnwnsnckqrbxbouevywmmxlhqfuxurdknubhcbxgxaaxyxtzibxchxsadnjapwhzfkrbjotydunmftpkkvjsujwpexkwpvopnhkdiagdpcnvsuqrpndkootaitxiintnodcupxxppwedaxcwqcpckogmvljveqfndodwrfwajjyanujxubwnfwjrhqaqjhlokjajewzlrfpttqqxglebpfjrepivynwlwvepdcukwtrczilqmkescjplppwguxjhrohxznelnwpnewdvsalxhovvmjgjgzkbvuylifwpvcxjuhrzcvkykzrejoxdrrngtojsqkhswblcxzoesynqahpluoqkbvptndqokxzzjopvdwahgejlbdyquuccmbftxnbljgnssuhbewtfclxzwcurvxowsmbcuegpsldmizgzmqklfnsxhywunfbsyalcgvxcrfqpefncqgfkktgymftonhmagefomaaeunkmwdbyqvnjurllcupsqgtcyollqzvulxknnbeelrmjjnnsszsmvtpagcpooptslrwqhsvuhzkmjdhqitdzfcxbctoahxkcgqjhemmdhnftdqhzyiatfdmsyrzkztunbytzduepapcefujaimdylkemucgubrcvdacidmgfhurmcqssloqnmeldsdlyrrxydbpteqckederwvaohbbdpkzhmsgujdluqrwdkdecjwscopmcbmyiqmbrtqhgwwtzclfwqxivhraalrkhpjeydtsgydnteeennrjtnufkkodpqantuebdwycsovsjbvsgodwpquarhwthbukfrssuuukalzrtwesghgysahfllmdaxrnmwguqybjrreohjqjtklllxbkpvowujnkmrdupklntcekhwhzfnplsruhcydespgqkpjekidgnhtrqcphnqnlgvhjtjkloovkdenmiiwxicogxfecldfnlumwuraplnzffglcoywwowcfacvtfivuuwyjrgabmbtzcqfjkvketjoootatvargjjgqhnripoohmwqerqfwdgjqhcnsuhwcpfyxctubwzfhfzpupisasiqlslcwzraoekhynzxancdgwwixowkfevqzkeyjslaikvbxkeilvfkkznhilbotbindwgiobjlaxomkajlmcodqfqwbxiyvksszjxdfvjgtnfkbaoyyezqwkpfvangusakakjnvrqlehibvvmyswbvtpjlvlrepnlunmnmtnmmrrrqbynhkmvalkdvqpiepppdteuecufmovxdndslaspyztqdijapfrjzuptaxyvbbqlzoisnj` +} diff --git a/politeiad/backend/tlogbe/dcrdata.go b/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go similarity index 99% rename from politeiad/backend/tlogbe/dcrdata.go rename to politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go index 01602e177..96a553987 100644 --- a/politeiad/backend/tlogbe/dcrdata.go +++ b/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go @@ -1,8 +1,8 @@ -// Copyright (c) 2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlogbe +package dcrdata import ( "bytes" diff --git a/politeiad/backend/tlogbe/pi.go b/politeiad/backend/tlogbe/plugins/pi/pi.go similarity index 99% rename from politeiad/backend/tlogbe/pi.go rename to politeiad/backend/tlogbe/plugins/pi/pi.go index 574ebc35e..8a1496770 100644 --- a/politeiad/backend/tlogbe/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi/pi.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlogbe +package pi import ( "encoding/base64" diff --git a/politeiad/backend/tlogbe/pi_test.go b/politeiad/backend/tlogbe/plugins/pi/pi_test.go similarity index 94% rename from politeiad/backend/tlogbe/pi_test.go rename to politeiad/backend/tlogbe/plugins/pi/pi_test.go index 930cb5e6a..d89bb0f8a 100644 --- a/politeiad/backend/tlogbe/pi_test.go +++ b/politeiad/backend/tlogbe/plugins/pi/pi_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlogbe +package pi import ( "encoding/hex" @@ -16,6 +16,25 @@ import ( "github.com/google/uuid" ) +func newTestPiPlugin(t *testing.T) (*piPlugin, *tlogBackend, func()) { + t.Helper() + + tlogBackend, cleanup := newTestTlogBackend(t) + + settings := []backend.PluginSetting{{ + Key: pluginSettingDataDir, + Value: tlogBackend.dataDir, + }} + + piPlugin, err := newPiPlugin(tlogBackend, newBackendClient(tlogBackend), + settings, tlogBackend.activeNetParams) + if err != nil { + t.Fatalf("newPiPlugin: %v", err) + } + + return piPlugin, tlogBackend, cleanup +} + func TestCommentNew(t *testing.T) { piPlugin, tlogBackend, cleanup := newTestPiPlugin(t) defer cleanup() diff --git a/politeiad/backend/tlogbe/plugins/plugins.go b/politeiad/backend/tlogbe/plugins/plugins.go index f923fa972..021cb3b06 100644 --- a/politeiad/backend/tlogbe/plugins/plugins.go +++ b/politeiad/backend/tlogbe/plugins/plugins.go @@ -12,51 +12,45 @@ const ( // PluginSettingDataDir is the PluginSetting key for the plugin // data directory. PluginSettingDataDir = "datadir" - - // Tlog IDs - TlogIDUnvetted = "unvetted" - TlogIDVetted = "vetted" ) +// TODO verify TlogClient implementation does not allow short tokens on writes. + // TlogClient provides an API for the plugins to interact with the tlog backend // instances. Plugins are allowed to save, delete, and get plugin data to/from // the tlog backend. Editing plugin data is not allowed. type TlogClient interface { - // Save saves the provided blobs to the tlog backend. Note, hashes - // contains the hashes of the data encoded in the blobs. The hashes - // must share the same ordering as the blobs. - Save(tlogID string, token []byte, keyPrefix string, - blobs, hashes [][]byte, encrypt bool) ([][]byte, error) + // BlobsSave saves the provided blobs to the tlog backend. Note, + // hashes contains the hashes of the data encoded in the blobs. The + // hashes must share the same ordering as the blobs. + BlobsSave(treeID int64, keyPrefix string, blobs, hashes [][]byte, + encrypt bool) ([][]byte, error) - // Del deletes the blobs that correspond to the provided merkle - // leaf hashes. - Del(tlogID string, token []byte, merkleLeafHashes [][]byte) error + // BlobsDel deletes the blobs that correspond to the provided + // merkle leaf hashes. + BlobsDel(treeID int64, merkles [][]byte) error - // MerklesByKeyPrefix returns the merkle root hashes for all blobs + // MerklesByKeyPrefix returns the merkle leaf hashes for all blobs // that match the key prefix. - MerklesByKeyPrefix(tlogID string, token []byte, - keyPrefix string) ([][]byte, error) + MerklesByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) // BlobsByMerkle returns the blobs with the provided merkle leaf // hashes. If a blob does not exist it will not be included in the // returned map. - BlobsByMerkle(tlogID string, token []byte, - merkleLeafHashes [][]byte) (map[string][]byte, error) + BlobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, error) // BlobsByKeyPrefix returns all blobs that match the key prefix. - BlobsByKeyPrefix(tlogID string, token []byte, - keyPrefix string) ([][]byte, error) + BlobsByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) - // Timestamp returns the timestamp for a data blob that corresponds - // to the provided merkle leaf hash. - Timestamp(tlogID string, token []byte, - merkleLeafHash []byte) (*backend.Timestamp, error) + // Timestamp returns the timestamp for the data blob that + // corresponds to the provided merkle leaf hash. + Timestamp(treeID int64, merkle []byte) (*backend.Timestamp, error) } +// HookT represents the types of plugin hooks. type HookT int const ( - // Plugin hooks HookInvalid HookT = 0 HookNewRecordPre HookT = 1 HookNewRecordPost HookT = 2 @@ -139,7 +133,7 @@ type PluginClient interface { Setup() error // Cmd executes the provided plugin command. - Cmd(cmd, payload string) (string, error) + Cmd(treeID int64, token []byte, cmd, payload string) (string, error) // Hook executes the provided plugin hook. Hook(h HookT, payload string) error diff --git a/politeiad/backend/tlogbe/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go similarity index 99% rename from politeiad/backend/tlogbe/ticketvote.go rename to politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go index 718faa36c..fb26b6794 100644 --- a/politeiad/backend/tlogbe/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlogbe +package ticketvote import ( "bytes" diff --git a/politeiad/backend/tlogbe/testing.go b/politeiad/backend/tlogbe/testing.go index 5a214ffa8..df59e57ec 100644 --- a/politeiad/backend/tlogbe/testing.go +++ b/politeiad/backend/tlogbe/testing.go @@ -5,188 +5,16 @@ package tlogbe import ( - "bytes" - "encoding/base64" - "encoding/hex" - "image" - "image/jpeg" - "image/png" "io/ioutil" "os" "path/filepath" - "strconv" "testing" "github.com/decred/dcrd/chaincfg/v3" - v1 "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/politeiad/api/v1/identity" - "github.com/decred/politeia/politeiad/api/v1/mime" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" - "github.com/decred/politeia/politeiad/plugins/comments" - "github.com/decred/politeia/util" - "github.com/google/trillian" - "github.com/google/trillian/crypto/keys" - "github.com/google/trillian/crypto/keys/der" - "github.com/google/trillian/crypto/keyspb" - "github.com/marcopeereboom/sbox" + "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" ) -func newBackendFile(t *testing.T, fileName string) backend.File { - t.Helper() - - r, err := util.Random(64) - if err != nil { - r = []byte{0, 0, 0} // random byte data - } - - payload := hex.EncodeToString(r) - digest := hex.EncodeToString(util.Digest([]byte(payload))) - b64 := base64.StdEncoding.EncodeToString([]byte(payload)) - - return backend.File{ - Name: fileName, - MIME: mime.DetectMimeType([]byte(payload)), - Digest: digest, - Payload: b64, - } -} - -func newBackendFileJPEG(t *testing.T) backend.File { - t.Helper() - - b := new(bytes.Buffer) - img := image.NewRGBA(image.Rect(0, 0, 1000, 500)) - - err := jpeg.Encode(b, img, &jpeg.Options{}) - if err != nil { - t.Fatal(err) - } - - // Generate a random name - r, err := util.Random(8) - if err != nil { - t.Fatal(err) - } - - return backend.File{ - Name: hex.EncodeToString(r) + ".jpeg", - MIME: mime.DetectMimeType(b.Bytes()), - Digest: hex.EncodeToString(util.Digest(b.Bytes())), - Payload: base64.StdEncoding.EncodeToString(b.Bytes()), - } -} - -func newBackendFilePNG(t *testing.T) backend.File { - t.Helper() - - b := new(bytes.Buffer) - img := image.NewRGBA(image.Rect(0, 0, 1000, 500)) - - err := png.Encode(b, img) - if err != nil { - t.Fatal(err) - } - - // Generate a random name - r, err := util.Random(8) - if err != nil { - t.Fatal(err) - } - - return backend.File{ - Name: hex.EncodeToString(r) + ".png", - MIME: mime.DetectMimeType(b.Bytes()), - Digest: hex.EncodeToString(util.Digest(b.Bytes())), - Payload: base64.StdEncoding.EncodeToString(b.Bytes()), - } -} - -func newBackendMetadataStream(t *testing.T, id uint64, payload string) backend.MetadataStream { - t.Helper() - - return backend.MetadataStream{ - ID: id, - Payload: payload, - } -} - -func commentSignature(t *testing.T, uid *identity.FullIdentity, state comments.StateT, token, msg string, id uint32) string { - t.Helper() - - // Create signature - txt := strconv.Itoa(int(state)) + token + - strconv.FormatInt(int64(id), 10) + msg - b := uid.SignMessage([]byte(txt)) - return hex.EncodeToString(b[:]) -} - -func commentMaxLengthExceeded(t *testing.T) string { - t.Helper() - - return `puqgtmjuiztnwakzqtmybjgwluizndahxihosjwyqsyqrhxdkhwchnhtbgamigdakzyjcnjvubufifolxgmygyhoewgyjwicucaypgxrgsenjpangvtslpfbojvcgsxzqyginsjrtcaggdcienduzuxnnpmdcvxtklnsmlanjheywshijznprrvnizoxrihxstcdksqvhvqjeneeadzpjgqtdrlmhjqkxfsjcgoewisbacchagpzisvttyurpijyzpugkpufyhifaivfzgjmkpxtdydaebidbpypcnmsmvaktchcqkkrwvaahlrrdzzxpjmlshtmugrzjqyvehfnkievnmubitaeewabnqnbqcacctfiojiifozfnpvooawiutxxzcjddqpybnrsyxtfzwiomoqpbpowifsbgfsvhvcaucwjnqfnwecqfugakimobvnguyqsgzjxstovohyzvezjuwdwyopymdpoaqwesghiuzmdwzlfcnubpvfefedyrllzqyfgntcaaazmfqxknfkquiyplyhqqipixmlqjedtafcwmkfemmzdvbgkatlcwnsuzjuvofpwznczcdozlqzaiqpumugnubbsfhvoogfenjsqlenxptlaostdwpcorusmijrbqecttwokirwomuktngwluwmlthypopbinyxqrnxplzpipzitqpaqdreelqjlxqsbgxbwhmrmauiicjndjrhrqnaucbezopzrqpxcjobicxqnswftjqrhhhnicxuymdefodhmyscrzonmrihvljgdxqvjeqxmayilhitwwtnmwdcspcmnwocvksitrhiqslzavwjlvzcjwzliikcqpyllmtfrqgfwafalfklhhnhmoejdsgppdvugblyjusljcqjftktuzsqkmbueocfdadrfsbzxldiagxjuydthitfwrirvnbshdzdnbbyxaizemggkndccgjolevpfjovyacfrzvjabkgvwguxpujfancqdebfcbhlwluetcfnddixdovydgoyngxjkxelxarphjmbcbfutmcoulmbzyjxkupqcsaxsojoqpxdvsaijvhppdmdstszbgjzylwebvjvijruioazjkqccsluqsydrkqckluwealsrgxvnnuhvtuxpxdtkjguwplpmynysdysccjqhulkpcnujituvnattmvhhndyvizusmdxvpbvxzbiogawboasipmjgkdezulkmtkcrziqnyrjcupucjtkovpeqnbrqunhricblwqctencvspsahiltoopwqthxypkfovijqbwxahefszdgpwickvurthbxdkczicvzazssynuoyqcrhxxrctsrbvflbkpuprywtasmkkgcojaiylazybdtdnkjuehxjjsftkayjpcnwrngblzjsbcqauxpskcumnbubiodlqzberjmosvvufzgqbtwprhkscsdfppiifzsxroddracraxaesninyzsnnzjlnybrasfudvrjaetulogbzmbxjehlqotyfygfvkvyxkthdljkbwnvxrowjbecngutbuajppguteagmucqgwihzsktxscfepcghchwnnpkozysgpzrdjsozighofgsibuzccolbzpovmooxrkzdizoibmcmhakjwzwnqftgtgawgwzxnwxxdpgxvwxqszjhvihxfggfgxdpseiqkqtuspygomxuwxhumkiltapyfpumxogrjtzkvdomitflsesindlqvcgvxzogmzssqakheixjvzzqgxwzfjqlirtqvuddfevkzltnhwefzalsxkxqnwngdovnzwrvmvbtsykemmwipxlxdxhknyszyfmflmlcvpojnwasvmdfwahuavjajqnzigwvahjphrwreugmeromzotybxqybslneictseznajjkcxmueyroqamairkyzzdpilbusgqufyietrcbertanevmtrfnuxvgyppskpbcligxgbsmrkjnqytfvycbxtmztkzfrvsbuffuhvmuafhrqvupuecyutvxgdmhfrhcngdmqpecvpfjmmxcjbjjtaehnfluibufrmggdaxlenggmzpbwfzkpqymbsjgjcbugiruxyfbpbjovnentghytlitcgksmhxwrwlpkieqmhqrivaqgcllqljvtperdcxpgdbhdotjtvqvwkpqapsxignvhqfhuradjuksickefuuqsmwdktbcpmgynjdrekffvultianlrhyuxjhltcmzhighfddopzpkxidrnwmlczawcaiulvzxlmaulmzlrlclhajplfuxcgsvoedgkltrbqxyfplgoizapfsckkhrtnpqntlectvfzkrcgdioonbjdlrzpyrsanmvijbycaopumafijkjxubibqwlqrywaqjvtvymqiqkdevlugedamqwiqmtdwzotxbzrltbificgouieeosjaokznhqdthzyttabrowgwxrmwobgmupikxduufpktbdfjganpazjbtcweloxmabgexlixwrdolgoxxfwvjrjommozgehlxaqgxphqbaugaiqxpkruwgcvuyadkfwwsqdpwxapmlvpwjvqmdvqtltkrzfzxuirvwxsmajrzzpfcncpnfjzfugipiuuvqsvfbycrarjcrgjspycerxlvhbcahrkjtlcjhcesptpvqcjinmetwuxygrpksmsyvjfpeibljmswjkfdlpqbpfidmdtilclbntvspvsmxstarlrmvzszrwiayncldnpknchgbjerojunmotgujxgeurpabcguhyvsyoyrhlttbcjjraioxblzszobfyiahhwzwgjlkuknuexpyuovlyfhaxhyjgaxzdbvaejdiuqtwiqjnbadsjudlsexlvjnroqihcbfrawmlfczbfvqoyodzmaynbnmtrewqhutjtycfnajrqwjhjfclzopvlbpmqgiugwnarkeerqttlpbxpeqvdiqaohojoukhpjkqaaqkodogtprbtdzbqnjeqyqoffuxxikorythdmtybwpelcgjukzarpsgnrpxesxlbjmbkplhopxxmxjbbwtyueriuaotsjthvkpgjcnbthxgcctexpnacxypzerlzvjlnqcqipiavtmsekfsrrprxerhkkxoldhhvtyojqknkhatpmqavufkepwkxfknokwehwbnrosnhhsdkyesxhcflzmqkflltfhyatqjmcbcptkbumwiwakfszerchqivrhhmxxnhlnhjyfhklpiojwwerznvbzzdjkxajxwbytstchdmyjrdpjhxonxswpigtnmifkmyyqwzldmkknajfhbphedtzxewrtgobofdjdezxdrhvijxbegepjhicvgymqydculiitzasldrsoiafdeunfnfjedcqgmtnmendgirvfdlkalnexxrxmobwbomjqiqsrwrctvhxytthlwfsalhgjpnvrwyxpcphdiejgbrerhsgduegxvxeudadbjyzptcnassvmlwvlitzozbxgrhwivppyovmgsekvdwznqwimgsigwgavdrhcjnrfyxftybncjmcpoqssujqechwazbmkkicqobzlcfjwxfiocikzaumoigufxouavurmmhdhjdhpztswphjrqcwnvkiqnlpbbsqkkehpqgewunuukrzxkfjqxyuojqhpdiswiwphhoedwmexwfsnelglgqdcfpmohpslcknyjcniodjrayrgbmimcfxoowebjseeauffehxisbiggrjndkeilsxuuhopqqlurzlkovsltarcplfyufukelymxltmolhoqeaodsbionrzxjfnibzovmakvclmglnzsytiquajwcjkmmbkwmgmemmubaxpwfupahimisngzejhtsrtzntpiapcefvlsavelcxptkwwefdonenzgsffrjqehlotebkibkqrxkxbuirmmbwfolnkzcaphbywfwrnenppkupcayglmexcziijxliospthmkhtolpdqfaesmpilekvlocoznhpffneomzjuzjxdnutzqgeajxdfmjwhgqkoiuxosrdpkqutroqpfmnolirpodgbcojbwtbzdeqkybpsnwityouywhfxjefdzgvhoxpwskgnngleedntvnnodhdmsdzqpykocejfobcybuxsifgzqfowltiwvxixqdjesopcaotqytncetdazkiddisfhiycwrhoswtrbiijxvvkgaewbpyzxyyiavckfjtpusjetkobhsglbeolcfxcvqdyvkffvxebvsooawgrcerpbtdrdrglzzglhsvaayatsoztyvzsqftvsuhkoektnfiplxbhcgulztmuvugfuexlmvjhpgytagekkbztmgmehesxfcbmvzjwznlcrpeunxtuwquzggndtpcouuxeyowrcbbdsxzeqiawzefflxvjuzawisfgsmdfmgunmiuhfpfinnwbrmbjfgnunsbqxyodkrcbceqvktqmxfgwckcngxynosxlhtxfqfxoqnupgdgtkqqglvzminnkikckknruaddbknopkqmhakqdlsbylftqsprjxkxdjucmfvdnlawhpskfzowlosriwotzmoalsonnibipimopnrdgrvvxwcexarmkvazrdogampkxbcgsjjvhqzoorjijkjdtyqbldvhdywedykkshyjmohwnxcezfwbkarxfyrfkhqndlxtwbukjcemtcvexigpqqcpvddszemjemrnekquwveerxbbchxyjffnlbpkehtjracvxaobtcgamkbfbrdcdjtsuxfoawfqxjfhdpqnxknocspqoldjgqjeepyvhsjncchfznbypgsoorpuchvdhrfrrytjrrqoyccybopakqhsfygjcciiwbkivyivxxsfhkguxqemtnlvdaucxvlotbvxkvsbsothermhpmkamfxnyaibdalvlimqfytryphtnkshxaexgzosubcccdpkqsggsdserjickunguvtcipnimxgbyaigzwgrqpkbgiwsxfzojqlyhnvszfebsehdxuckbpnqfwqsydsmewcnnzbradqavgdolbiazilxojlpkhatvtxdvnlipdurriudbgmiffdnzhkrytlzizgfztbrygfrjummhslxlhwiunsnebyomrzcrbjhzlfeuecqbuvypfjqrbkmxyamqoleedabkhubjiorbyuilgxigehkeyadafcokuojmzwbxorbiauxmlygurgnrnvqskljhlrzfkgvszgqfgmgcddqgzkynbydlnxenngbajpnctmqjsptserccmsayaldyzlpzelmijnioaiuvggxeeibmeakjkzyuzykjxvbfxregohbwtcgvrngvxwgdemwcsngkmqczexwdxbpqualctwfyhwsrzbhwympqzhstqqbnurcdyrnlzrugwfzgjeoiccvpeicjzhjfeshbxcqhirlaejhgbswlkzvvfkxuxgmnpofscsxmyopoxalhhfzthcysdscfhchuwxucrxceyiytvohcpfoayagdrwdcdlirdbbsxzwmluwhcpxsbzkhokvnwdwfbzmmythphvuoyhmvwfitrbqzmpwucatsyxycwrelkradmhvweizmlkqbghdihgbxogepflvdiaxlzbuxswflhuqbbtllluatwduadfnfzftfrrokdktmwgdsrmaylkalomtsowoxwoqaixuxomcaibatwqelmahlkohuregcxnjgamhbxtxzyeizuuageosusredimvavcedekzqunkkjolkeyvpcifkhvbynhpfacmyfpfxnvqzllnmfpxdceieelnizylbwgngeqhybkdxkcttyvczcfkbbvkrderwmldizptcfhabwsqgyxgcisoaoolupmoxywjpliuxkjljrqkatfohqmbjgmeoxrsuqfuudnxxtyymegeqqcqrjsasieftcfvhcwgdjgcwsjsftbnibnbcptcoymnpqkszlxrauqvnasslhivdxojoabagbqvtnqvjbblysijnrkmmxlobcolfpnrbekmttcxcuhcnlucnnjglzrwnivlrirhvewmxvmxuserdtgzmaaoihytnbnaxyogyzuvkanwyiajykuzdpkqkqjcxiozqovnmnugpvogfggypjcqcrxfeumryypqkyohqetmviytfyiygrgamlklxzovpvklymzhifebmluwinwijutszkhlbspovyainxlorbkzcgeuhjxlsxtwgpshrtwqbycmssuoucmecehpilhnxycrcztltszcevblubvejiunthsbyukrcckazvosvtqwoajlokcctlvimoeoetpwnmkedwlaxdvjdcrppjvqoptacwhkkrfuvkbdihczwnorcxmfaawwysegpqqxsmlhqymctucwcyjlfbjbuukrfrwgukwwonxrjnuplubhxvclqbnedenhihdgzbziwvkwszzvwbekhhdthnkqgxhlrpfkhuvqlrfbjqwwzyhmrlsdhiiynhnkoqvxcezxtgbzjduppkqxdwublnuuptqdghvnzmgurtoauiqzapwiahnoknsjcpxajeqmwrniuuszcxzeoguuobjslqudfgtsvkyvnzayfsdjrhkiswwtriscpnkhxwznrgndwlrhzqacqcqoyzqphutleiqexhialulplfmizqyekwdksymdxpldpzrvecuskamlfqinkrcejwjsajtunwyspddiklupgsteiwulivomjiwquqaablnsfgdcgvxeqtotqnaqknhcxqhrlwrpqkmolyerzzmnpbufvpyqcnxmfczcjdydqejsiwkndvaksqiwndtjwwybepduxmqtvnldqafwuikwwrbhowbdkneupafwpegwtyncyijljjecfylqkuxexqijkktogiuchpvpfhlobwdpoctmmwtebblogvnstazekvnxzlyeoepwfkafedfuzusktgfwyqjhqagcwvmdjifymcvbfocikyvnmrcpwcqwaxahbefhnlsdioaajjfbuhelofqjoweeacfiutaawjddrepdhccpvdaabceywalvwoqntlsugmexzkiymlvfsrbemctangeksmmlvfdejedwleylgldzpgupycochiixthdunckpkifkowfbkmndjnsgzyvcafjcgxcknirojflydhpufrbbcoartxnzwghwmgxguvybrlqvrycdflkenlgrtgxtpvfjefenqnnglmmaewgvabmclvrzphxhfbjajipbwfskbmnmhxjqyqxivehqfwvavplnxgumfuzbcmhpjtioiibijltztrnzxlghyrywnwkyilugldeuvquzpigsyovzabsspziqbooemzkmfzgmihjkpftqvwgoeymohdskdahtzawrjibklpsmhtrhqtrsdverrmzfrmjbzojvnbfvvfdzhsrrywmyvovvhioyfajzustjysjefgonzydbqtraymsdlzbshgfcvwaodnkuvferujaewbqgbbatixdgemqxutnwgsgyoltwwkahlojrmxhntiolfiouyftkjmhmtoksxfxennhfgvwvtklygxqhlsyewitlnbaevoghedmpgkkgtvhderaojtxbgovdpiqkcffevuggzsyrntrmvojffhirultxzzpfqdulzpdgssogtfhqjdlorplcyivsfcrlrnhkeqjixdthwkkytqdynwygqofobcslheqjsbfvmabagkqbmggevavmhjxnswmfacupdyaleflgrrywjmtgmznqynzijxudjfsegpkmqgnjyfswvhengsrlbrwugsmyaydkoaewwlnkstvlpklvbolfqbtvztzznwtpmaqpcqaoskmyhksvmgdtifyszpwdekjfzuifrlpruislaabhwfrpsjdofuhnqr - uxzbyjqgucfwoadtrjowszpnajylbkxlggbafgxhgmytpogkxbsvejafypjtjpxlkfuwayzbmgidouwaqsrreppbiapwonzhbisyzvomrrktiylsjkfkgsilrrobibhqhrndbmuvvogiiofwfsouhgfqftsztapefdumtiocbizzvsmmhayikkdzmpahefvcglmcevawpdcbrxonwykpmfiryzucktpomdhuyvaobqzmzxvkjoksabmoowtzkihitwpvxtrvhbffvwhjyozieqovvmzxnumlsfdbjuznddlmqyrtxbdyqmawpnubepnfkghucnztkbunnrcfshlfsageaqlgpvwfgzkxhsgofomzgkmljtniyaxeiecrrbfojnxubivwobzrsqjhlpyyrcplqwyuskjpteizazvmlxgqowxxqrpztaedlrkpnyuisjaazsjyqryewlfkupwezowpkfddafcyadgtwyeuqnsijwgainyftbogzmqqvuusegadirouinruoyrbcsuodjgeytxoydjfksbatodexjlihbfzesiahlgzmubdyoytcnpffonhversyiecojahysecgcjkolatltayfoxhizfsygfihtncfgnekcazcxtgpiquhxuvkcfbrxuntvtxvldjzulrgikcixieyrrauqyreirdoxmjwnngcguvzrzflanndokugwecfgrdxwafgskgzadfiffkfejdpvtgxzvitzjeyalvlryaukmpwrgjsimrltbditowolohcobodoeoaqhcblqovdntzfpobgosfckdfjuawhgkrvjznvyiburrufelothjemvltiiehibrtapdfufpgccpqyirgnncdgmrtmpomrbsczoktvjbxqqhzqcdsqppwjrcndjeemclkhuiembvdxgunlrbqqnrudffvhpdbldgefwxzzazclqltrmnyqkemnvfmuyftytotvqslldrirdgvtbafrhzglrzovuvgyvxwgjfawxphtxwxnbvjqnfzrvsixmspcdvlrstorerzzziecbycvhhuybkywyefouctykxugrgemmgkxwjcizksttpjglljfqxmlztpltyypphmzccrmbirgmmdszgfikrhcyhtcwjpgfvrjkqgfijhwjjxsiiecowugxoocaqsqqqxmplsqbbtifeevnqgksxwamxdzkjseeemvfdjytfmbbknnnkrixxkrbbkaxfaxjaotkiyqsvnreouoxacvjpkliigglsvigmgeqcrfvuvijplnkfinziungckosmxffochsbpmjpcvrogwmbjsurqurnhaofjtbmsgwnegnrfpxblnpogzmcfrkmdyospxlyxsivhduzksebfgiiueukzqismrdkqxmrrmmykwnxvjkclwbgzsncbzfxfhyqhrnbcakrmvtwnyfalohwaaajxwxfgwurcqdgcowwrmlxzjyzxihcmjluewonndzqpppcpexiynziwgvxerssbfaxtbeyfzzmfiitkinfoldbljjfteudjjqkqjdvxzifamkthldnctmodkezbniywsbbmmfswrnbtpqsotbfykrdaqqzizkmedwrxfgkxuvrnmckmtndnomcgefaotncukyfarlgqohwijtklbjjoqypytzjokvvbvxlwfjjtzsoimzqbrqlwymkbqcwhymzixtarzdirmnmmpewrbmztqviwwjcjzojgbxezjkbzkylxoigvkyjtupggiyuxplumdumuyepqrieofoejzvhhkjnuotmcbgprmsfhqcbsnehhzyuylcieurloynmpjkfmevitqmsattztrdqbhurxelyhbqjysmjocvnxgqngqztmqshwlfxksofjfkmwnckjvbzgncqbxwclkdjrpkccdoxtsciymhdxerhzaniuxtnjtewybyfvhwrmerghlhudosdbclqyjqrkwukablsfwadknkdimtnozytnfkepkuecrkdljgohcupaymqpdiyvdazomgunmwvsrreftqcjqibvbzdwmkexogqtlszoxwmizmkikcwmeqndrehvhnmahlrwqjjbxhzoutpuccapztxkbmuywnvtetctouofqtdafkirequutkeoisozumccsqmabfumputsfjkbftoxufbaggrasydnnrntyraarsxlbncezswbmqbnhpsrhewovqqozorvklxarkgdaiofdvguailfmowbetpvotwbfliyrbelgpigdcbkdyiufokckqvtxfuvgjronpqkitdbfitkeppuajlxxljttjivllatbqutkailadklcnrvecnkgebtoulxnfwabyljkwxeeevoclxhtlnhquyxvillbyhnfkiiwopbmfibrqrgxhtejfeilrhqlujkjksadeypnmqiqqhrsfhsreorcsboszgpcnutlzbvcxgvweyctxidpqfdqymggvpumnnasdjqutwgedshghssnqtruhoibbkenpbbydwsbeatozxhgqejmhljqcmpxarfnfunodezrhcbedvuuewkwxtixlujbvfryajdyiiocremfobtinatyssdhqqaxfaqslqyadgxgiamocdokrnvaukcwsrklrzfveiacpockgybmnsuqavaxtdnkupzsvztaeejsznrjjkgmsccrsbdynlutjpgkpxydthodzhwqihqepwsmsbjyezgmmiowaojczpkhbdxqensgongxlcupaokokcqrkrzxgupndxddomsqdyizmsgczuocknuutinjwdzgwnbslxhilosreycphtxljqdxpgmsdodbexhpazhtsuxwzaejmgnurjorqaoyshjojebpnrysbggnickzrzvzpggjiunglyhhrtcdogqotlhhkaazwoooourfnxuyxsddwdartkurqmfkigbhhfcazaznbxtjfjeakqszdwxwifxctygymmmkgpafebukrfgxngxwshsywckqjosbwcgfseucapjcmxthudzdlzvjkejvbqgpnksjgzaqbsdlypzgdbiiinlztpiwcszyuclzhwyxiovgnmyoyobrgeeahktiylmtzyamzazxfiieefcrrugwkqtjzgjouosqvbimtywvmilpwghqaaqjwokxnfigdlairvhjaccsqcveuegkvrwreytvizrjukhlfmzryjqyqlxagcrrtoxhscgzursqhtvnaffaupwxkpsyzilbhcgdnhwpkcwgclztodkywcrvjfhcatobzpftbetupysquhoavyjbqwpjrybibttnrmrveehfyfmxvfpwnxzpbumfesvqaddgiltjwtttbroadwyhufeqcuqjzldyjzftbjnmslndmejmkkokywsqipyhpsdpubndvhtpzgsmozjtcofiajjohnenyysoeuhcztvqpyicnvspskasllriuisrkbpxlggpplmwlqrsogcdqkujkxrdynqqpahmvkvytrxeeednwgzomuxxzptmednfzzaqggfjusqfqtzxnagtkovjinalffpsdjsskogcnupzdqtialmxuwrnfkergrjgqfnkohedxlfuaqlgukrfjrtdqhfkzdktkspnbokplyuepacbweonicbxlupglkxockrsnaqarzuuswxcsfdwmucbqwwsdhhcwovvstrenwdnmmkqhasohfrchmsosspvafmxtrglggrtvopxysrhormaifpfruenlfvgnpcxiwsialuyoueobnbtwvvtttlhjmsgfhicuvahtpmgpssjstyhezamfrzpbkzduneoxlgnowthyrrltmsghvmmixnkadybsaxgnatzwqlhgwxzyvcpdjkyrzttjhlrjvhpwfmiymvmzudtpnrgdjjsgutsdsjogzddoyczxjzerynnpfxrkkjhlofmdgsaadzsecftpubclnybmrmsjuepywujofzqhqpbgjiqkrrtuqpbvkdurmkyvnpfctvnomcqxfaebtcqbmwenanvxpesjjrkzbaxromczhjripdrnmgygcjzetkyarponwoiijjvagttcyqechcxykihgkjrpdoatzefopgawgyetlxifqergchypiyddabxnhlnafjksvkeedhncnazrywehboujtlyznvgmwoiggvbwgqmonebeoofktanpnlcrnlaojikhnhffhltwvjhtwocoefhpxukyjzqbkwzrftehxwpyjoqzwkunfetytirykgrtqqchqbpifhplffjorfqfiouonuhqjmrnzmfuzelewestiyzeavzoaexmqaybufknqojhhygwbqypdpmfdcfkwducxilhswebqpdlnqrbomcpquzflccznamnvauxwpyewzgmrfeatrlcbwmganwcieuppwuetqvmfnmgpttzchjljleepigcgobasmbzvgcuhaxnhqkvylyxkwhejyvajwtscxsbfxayiscmmbxxdcnwqdqtlgaetjazxtifsypbphqcjhgmqypkqvwsonslxgilctltiutmdxzqfpnhlvqmytiarnntracwcyokebcaeyxocjjontqetfbfngghsppgcoczhdhgkudyvxdyqusccmacfhiuueqhtqczlvsioijpytnjniyyihncxpxtbkgwtdloydbkzmgttimmkszcfdotxqaxjuzurvssbebjmabirsymdfdzseocksrqarbtltlavwetykcaakjhxxotmubsezdyhjevbxtrexjdsxrjdqysuhgudurhaycwzippcnuefarkaoocdlocvxokxrseugqqliceiqwmmifvhznzxwlgoxejiqgtoshtdkygjibinqxkngslzoboaxxzbbffpintrljvknhykyqexatnpmnwyeiemgvqlabrifhxptzbfhooyhligwbgyfgelunqowqtlrkdpzdbfddyjeowyamsxhpleeetezemjaseggoxlmbndjzsjhgeeyyfgtrrvdfqramgsnisdwhvmxbgwuwgqzlglzqrvzzyzjiitiiywmjcobbevdsplimdxaxirsljtuthukxygtoxnxkzjecvyopogdwikccgoehabhwqxyppqwklvrshvkvdrrymawivhhdpblpesimmceugzdqajywkmijyeidtzvyfaiaiaacozqzmdompzefqbnsamlopccvwqwcetqgogzwjzqaepfiuvcthkqwsbactudtivfdnsarfmkgfmzkfejcixpribvqixybiuouqpcwnlqohouayilyveehemivumhjylikdbrtjabneqzzhdgomgpehpwhqswcudssypzdsiuelxmfzibtilcszxalrpapnwzncgtgqxgbgvcikvkgnwgxqzbqimsgvtmjzfxhocgefukyuuejklebvauoanawkhbcomjjguozffdiktwxzfqowhvxaaflafubdkaczfvlrdipasaxqaakpyfcjawaojnhnmckczwptgzqltpzgmwkuesiokzicblrchzdoosceouvziahsdjxhczusgduehdbppfmeantrszapodkgfseseuwazdemdnlgrxgkntszpngscckogzhhfimqgajvpocozxacticiicfvvsxajcnkurdtduxfbvbufxzaeidsbdlxfkveicjmuapolhmbtvghkuuivzgaszxyidoawsnookpegzigwdgsknaegtkhfqjiuntqiggsfojvdagipwcxuodktecqeumoolzahoxctpyybefcnsyibspckqaarnxyfvfyvkfxsblfbepbqbqjlsevstxppzcpptwwiybkffcbistqymsrgqtgkkxhkndkxceclxrqvxrrsznruwvahgbxdcnsvcxoqavdoacpeptqnskppqzezmueuolwoufnzrmbadqnkahkdjbmtdneysmveemmtftvwfwcrwmqivjefznmisgpguuvuqwlniibjenfldawtgkpehcfjzrzexgxeiztoccjuuxyrvsujirjqemvvsfvgiyafpxnyauoldlbadvrhdqovpmrnbmnfsdiuqkqospohpmqexopgrwpfywkuoaiqaykpawwnmtmefjhswcprovhayixxioykbwvzfdzzrkddodoehmqdnxzlerfwxgpyhmnruhowjqqbivdnxlxgvfxabpfmzpfvnckfvyzbnecbvvmiklimafwuvzlgpzaxzouhvbtjpvsymucoxlrrsfforyojzjjwcylknthcnofgnbwtqkmhfpdlcqoepqpwijsmgalcpwwlpjtbienduufgypnhvpiuazpifcygwfvccmhxisohvajcbublmeysbpqpejlndipijfbghqjhuwnoyuuhvwafgjbtpxywexndhchwfhuhulgntfxmazqilpoichgkqmdhsursnmjxhccbpuwloljeseegqtqpzoxkqwrcjnljnxipfkrqqkeznuqkwnazcblowsspxjiccgynktnxgtgwymdswefsyylvbibatwvqybgekmahhmgskkaqykpamyzhljdxufuoakanrmwyfooqdvfoikcavpjiotjfhwbjfdjapvuyoxutrvdhcpujzqmqcpjauksvspowniqbkgojgwpofohgvexfrjjmzniacjjgspfzeiodzrqfinqgkgapuiyclcphndswchgltzrizurtnanztsjeppwbycygaphuvjswzimwqtgnpfdjttvfegwzulbkecjzynuecsybfezbckengazkjsbvxcxnziyzikjbbiqmsffffcwvimggyjncvoqjyzfbyayitdlsohxynnlnhyxgzjbdromuochcpchcgyghybmqzgbfhgwmvbilshgqkcqsdnntnmiccdevdzgzxmtbmfzkphkymmkezesuewegtpdzxfcrphrueimuujyizbjvvopkgsgxicnpxszrytngvhsnaqdjnpplipmvrhkzqdabzjjrlppukiewcptptudsrhehncjeccpxtiiyialhtvlaswxgdjevtfaptcimydyrbcvmwmqrqwfgimiibpduadwehwowoifelcizyhgfqqlsjzvzidqvykozhlpnypviuemuzvukpaceodtwmdbiaqmboygolxkbkbkchwruehizkutazayfcpimufyvqeugnuolchhmhcgmhzxpyneihncfqamwuqqslcaxgfnbjgrohesvtrgtogwvjiobbdvotzvybtkuoimiltbzdpltaksquilugjpttwrcabtblbtkbscnpufmzijopwlghowwphqjibmgpxwimakahdkdyrtoevvudgismpntvzsqglgbnahwtucisqskjyllgcdylxwdfwrgiwonmhpfhjzshdxyksagqvkkshxhvapunfrdgzlhuenyxseebopssfdoqbwrqkyypjkfrnlaxyelnkffsxmcjekctyouclkassfevztiwsdzmfheeiwonfmtzhviwjtspilkgakdppuseqaoreujdauofuetkczagtnzrdgaympguyxjlyleakrqfhgqtwqwsxxparchjsnjotrjxnpvrmsjrjydjohvjjbnqiqjtgbjmdooctoirjetufxxhfihmyjbthezchpslaesrykhweuktictxrplwvpysajaswbpwlsxpmwknlaufqkipqkxdjkxglkyrnkqtzgxsbjtkyqfpqqutlfxjmoykfcxefjxxelpldarkxmavxzivlhpzezhpyygugseoakqaxpasulakoqsmdvisxjghplvougrlhzmxyowiyxtbngnnueravqzilogrxrwtzypkcnfodabkqavnidkqhnlyksesozdhsaimnerofciqzcjyokbjuoseerbpskwrboleuvtskzzszandzcdkitmgumhsnpxiancb - cynoosgbueqerwmxtpaxvqzwrovpwjtyqnvxdqydmiiweqrftljbsdceeggswkvwfjozkeeepleepnblpzrrcxcsjklofdtrrmrktspfibysowzashhijgmmsbpeldpworazzxiterkhvianuhhgznsxaqhamjupgnikizxhmwbyxhioqgsoxregsngxdlvxelafcgjuitvzxsvnbamxwtamnrknhbaiuebokacqyladyoapnuxurhnllyiaxodlyjsmbudzssuouyeiutdbauntftbbpsptfobvckkvntaarcwwzrikwcnijxnprsjyewecryiksjrlyebtwjvunibrqmjygefzyiosbuyckitvjzhoqehrvtldjdtezynimmdihekdeznlwpmsqcjwkanxcslcklcpolrkorvhfhshryytvhwsjphnlmsiwedxwivfswfwuymyicqcxpyarygvdauizsfvotryrwjkmgigpkophwwphendxkmmzlevhjvinkuhvsoftpovtacqrwdfelmkfcxeghgdcmdedgeevouaufcpovsxphgjxsyizfcgyawhcgkmkwxyvmnwwydcnxoarbanziagjamoauhvogakohrnkrrujxkdpihmhycloeintuglvgtxpiauzcipmxlzthvuetzjgrmkadsxgkekuaimxrwzpwjnvasqprwjdwtikmhcbexxdqyrfefljyhysysnqbfxxrwhxoawtdcanyadrmtfzdqcvpnqzrxosjsgezlwizlfgyzszjnoscuvvstihwfsyfmtiskbbknieypewpopkpntzufxudnwgtztcnxabgvhmfwnvzosmdjxsvneanqwrjpalsvdziximdkewhtlmogccipmeozakxeablrvckldqzeusagxtmiycjyblhzxnwcrfaupompgvhmxshpfsemntpvroswbccrpshphhmnaxqdbvipcubhiwjifhapteakrhqtuoepautkpwjynkrnrmedhqmmutfnwavxccthwdbxiezbfvvowtqnlkgsdbxjjekosvpdpkdbbjmrcsizhyxosnylcgqvrvbsnmfkqasxvthrwkzkcamcsgovrxpqqnlletczeefmoucgynmxkzlmfwdhubevmhklsjdqurnaxbglckmqxrfeaitojbbfedmhanqmzdqkojqnsuhscnrevxmadrjdlekdotkuzfbsjcylibblirvzafmfbbbtsqpxmndyahnwgjlafkzhuejsxdcnyiqamfkzyknfqrcuphwirwtqeabclntzyxesrrxvbrqelzjwpvyxumhnggkoldvjkzadyamccpxvahhnbmjgzdskttzfdjzxcbwanepnvbujblcbfiwouuehzgyycskxtbgrdunbatajhxtsoahvmlieszqxsekbaedtlunjjnqdmzybxnslonhsewrewslzdwnpcobaasuppoiuieddbydzaxdnlpdsjawzdgckhyzsaziadltotkgunxzbfbdxiywznkakqhhbjsgcrmuduwaskbboqpuiapsvufllatjmoycuwqponkylzhihnuvttxaxhbbdawcblzswejhhofujvityspkgayisxaztzocmhgbhxujdssacytlrkpkaqvpkxumaxikqkpyoiekzelomlxxfproafjtbzalqlixbqozsizhzuduebwugxsfpshztbeqmtlwmkzptbupoirwpywwpygeuuylyjkfqmwjuktaadjxgorhvoldlznhowaoiygnxewxbbalkgntylcmgsqtxlvkjigwjbgzsstlamediuvvhyemloshmevvmamtshanmmqkbuhatccvzltzwcpnmnxsbxmifwsxeepxjryfxuwyydgwdvwkfnrdlicxvuwvlgcyewpwwrciiqojaikmtqhcjpqxhknqjwztnmxdkchadljowljznhxzcbabgulovadvjoabkjwveisojhuqupulxcyfrcrwzrhygudmtjwxtjqrcluvnbdtsiabnuwicgcaaaxochhesyridnfbbpdoymvnrdeaobnlkcucxxueiyovqlhxyudzgzxhjdqcroxkokbvlotguovugviuvznqhxbzmjvzkcvymcerpyioktghsugyufruxdxyujncdjpvwvnufrthgmzvkyxtehkllbqmokonhqjgkzuufzaqkbyfeoghvcphjxwxxvaqvnjchtndtnagyjbycizvraihjztzbdorqhodfmlqgpbzbdmioergynvrlusqkeyzlartfoquklszfacqzkgzjllyjmfzfsnntndnmxbvshsdpesrbmgkvrwvvfubmscypyoetfvomwnlplcwfhrmmtpwhnzplbtxofdpdfzkteqdudxhmrryvojcyxbipqotevaqrlcghvuychyuohiptipxhbnmwrhuhfxsmffqphvfvirwarqffphkxrcsrnikfxqfeqpeecufnzmqgyebjccxjktkrvsdyoyfukocqupmugzbcuffzwhfylxdhtlglozmzmjcithgflxdrljqntyyfhpvupznqwmxvqmonmtrctzpkwtxwgeubptalrbsapdycvjbqrgyneegqvnphwlmnidfzhexalneorgghidhbaeugnnozbzjsuulnsilskffixlyuodvmorhjgjhuucgfpvegloqaaievbvidykuvfukkguvclndtblhorbgftpltcjexcfhrqykewjrlvnhopgmebpqfxafogsufmezsdetopoyvscionixqtccrlnibakqssmrefzrbrjkfkgrwsqamlvncneyqviqpayozbgjwrykmueqzxvjzuazpskimezgmhdqakxvekvghbvmnpkhkrlrydojtjclvjmedxfrrdiurtpzvrxttahfnxbfiqzuyggdfevpgdprmcoonodlwufovzontnybwuqosuljgjlxwwbzgtermcpdpmfdfjirzufevsjudjpbghtrqlnahgjyspmmznrdcghqptwhjsuxosneriyjzbufwekzcltieagiftxdeyhzlompkyvfbulocczzokpkvsfvpilhbbsvpunalidhizbkjvelohinqssmlxltjusnzczjohiedjqeqzvtlphurwhbheyhljgkukuavgmztxmdblocaczdyrdnmzedduaairhrhxebyqvmwegreuwmbjnwsxarjcubzesagrksthupylndnpwtqoczzpsqtbhkukkaicsljoskcfkutadhgspnedtglqjdudlyhcqpbkntgshsejsnihayqkzalubgbipckoxfdwnjwggharojsygknnplimvrdmzjuglfwqndntafpigwgmaatxoidgqxmdxqnatjipanixzecbwpyflfrjedtkkzkhmdjxybdnofuvaowikgmtdkuuqrlmbwwuvjxffgmaitmildstofbdlmmysnejjacakmopylacajgvgzfeyhvukuugsmsrsvfgjfqbkfthnzasptddvwsachsyjwcrxabkwdgrxdhifnjlcfgiqgjarhoohhaimvpbncvvuqfcfmmgprazfcuknvxfewbrqtpmfrsunvaoxaynoucplarwztvmpsputnrgrbtaytzbxsrtaqdiosqekfbifxoofmkjntxtdhdcsfsnboqryicilhweyqknmwjgzeohusqnjcrlwzkxtndurvhqfhsdtyaizwyiwhbnozffaoubiqrsuxetcodjdxdwqmpufmtpezvcsuabaklensiugvksdxudpfsltylizbnoyvgohpodnjkzpnwmuuiacgcwsmhiiyxtuvdpbiwjwqlxtpisqwpwmkcezlofrlhphlgeanxqqrsvcotjwwpvvdbbjxyxggybzjfqbyrsxyqhsgjjebapdhheziqhgagklfqpdfllxwbmlnthqubtrvjguijremewzpvqdfzrqncbldfrffkwtmmukogcemgsqjlmkgkfflorlficdwnvyhobjebqwjwsaddgjkrhkobhgvaroghlqbfsiexnsnwmxmgdbdewvmvmnkvogjdgbxwfviccugiuvwgetanvqvoasuxxotvpsmtkdqcfilbzplbkftyrrqgncuieiuguknqzxevemhszlrksylqnygnnurlwbfmigjaehqrqozuhjhebkibhluzjvoarhrwcealsblojjkjcasblmvhbxgklntksqpzpynlhdyuagsxjhgeobwyotowxsogivqeifmfsvntrrhlghxocabgegsucbbcyclyabtteznknyfybanleuirdtwfwcmtkqqnbmaykfxdepgchanqhjbkhpayrcxjnuglzjclcuhflaxqbuyugnrghndponbhnrujozcqbluwbpfaqqxsmiyxhhprioglccxmiylusrewelvsaibdhghhdnwzqgxizbglmvffcrnccmofeyiwhwildjxcksiqxtcmvwnpeoaxkihcguavfwvvsssyffqpjqqdgcxmtfqibjzdgpnhychffenofrizagaefubfczalzxhjvrzwaacbrtayhispfasxkpimzuedgtydvmrmmqpbgokolyvbmlrmskqspguqipilhgfsmwqgbyemmihmqbzbjdghhalbzwnuhkumkxaqwyxjkgmwqecnmaxpwomfnpkvjkazwdhcefqgwkonqoqbuxdgbhjzuxqmmlhaeemmhxdfosmipnjggoclhojwzetcihlbltmqmwbhuybganrjmbchjqtdwjhxyrdvwedsbchyctiojoqynqlmlitaxzbhrpmuujpdhijygtcpgvsumpptsvgunlxmivlbmhuhalmoyaaydikqnncodttjgppextzbmaaeacwponkhoyzymgjmjsssaklnlisoumrjumgobinjrdxmimvxnhbivhgfqpfxmpnqghcpwewqxqxkhijoofhduvpckoaegclpaxvaxbknworulhntrccyrtffayqsvuliqngbdabfskwvvhmxsrnyofdagszxgeomicgbwpmfiezuiewuhkhylrqsbgidwianmqnoktpaizchpqxvmrgapmyzxeonicxdmievlzlznwevlpyvcifnssmyhzpfppalixfmfisfoslulsnihkjemvkxqiodngrgnfbqhhgqxgmijjpbmlhjtxamsyipongwxcqjgewxczqbcoopshimsnoniuhyopfbekuiiiuftmaiwdljjhtckjycgabfkgrshcgrdaucqrseulyzjgmgaampunihkzvgylpquswkxobvlerneczfbmlpwwcbroxmldmpputlazhsqunefmdjndkqhedlqibthsjzolghnligjwqchvibxqddmyizjmoohrnztutqgpiwlrwapbqkwpudbvquykfrzibgrmkpkbucnmbxwwixgoepvbtgduxatbsqpxsjonmcrmononviyciizejxdfhelhehytkpdfibmbrnsdkehykiyozzpzgxhatucevxjigfssztyuarmziyadoddkkryhfplxoqwlzstuielovztvcqjkawontavhhcwjbgcxcwwcsrylagglunvddltyeakqjizycrtexdewwbdiqeupeqyxosjgyyouvcmqwzulfxqhacjxfwwgkufhwzcgfhjzujlzzoqrlddpvwfxyewplsrzcyvplvajdeoqpongkpcgjzodjyyzuzyfseyftlzozqwbtlznbsyxkkiwomdqkhzskmkqpdoclqisgmllwpiulrxebbdjegxlndbnunsvretvvkriqpruxkgvujxvobbjenznnoznsxkggawayyprgwgqtmwrnlbnbcjziyrzhxbcarcozaehuvxvvxtmucbrjlhjofvfktvjeabhkfasvhqnopxlohgbvdtiqenyqqbgrfcswvawkpzjwytexwqvkupxopdnfnoedhkkeapfzlosounoqaasrfdthwekicerhmdlooiueeqnqyemhmlnwkokwxenpjmgbyqxfsvqmuzhkxlcvcbzbbyymdytxdeaaaounyqyyenekiobcxwgyfrrlrhbggzfpzolwphtazaygmeijvoklbuhfjijhxhplduigmhfakxogpczjcbmkvdktakqtmyqqzbppatupdrnnhzxmazjqckiazngdkymogirfttoetqtyevjzawsbnqlqiblvngdtfaembaqsomumwgyyqcmfpbsptbbtenxduvliljdlzqeiwnxpnkaoruhulfkjhqtpgxwlvvchptwyffguupkrgyduovstafnlhostgwrckvqeceshdaopszvzeuobydnpszarxbcrrggxroaevnvzxnehreuvxtbszunjdkchmgolmlaqegnhuhcyhslfobxwwjptjsrmozeofkjtvzzkyouvxftywdofiqdizeyfbkqzdrfbpkbpwmztuhwdydjxfularehcusomxubjowcddskwbplkjskjzrmknsiqvwzpwpwwpueaizbwtawvrwxoeqetfgoztebkvulxdajlrorixxsswhkojotlrwlwssdjzrzrfscoqapzwxnwedjszclwidoujnlhesjsmdhmsgrrdjsmwuvwkbncmvqzayaedqzkvkfjovbcwszncuzurdoagahjnulrxqxuhoyacpivmkylgrfnmetnooqefjmwpvjpqigucazlrahsvhvkdqvbhnzhlvjfmujlrrlsqdbgijeoeeroykcivjitarhwczqqbzmmkiezfktnsajnfbhyuonxsssyipfgqlisxojwgstgekcxzhjiqgkayepjwusgojmhgouwxdthdunshfxeajfvenxrbdouvjyyxruuvefmdzcutmbvkgygxkyxikhzniolkpwzatzqsdkkaledgbcuiysjzilgxwbjlkxhneamacglvmyiqfttdxrdfkwsyzwcoljhuweosqeebofoxlfanmietjcugbvugdicsicukdqdaeapytaztpngljterhxcmyqjwpummbkhmqvnxvewnmyyxmajejhedpvawbbhdhzejhsbahyyagsqpgcawpnpatwfugqjvhoefrhoqfxxqxwvkgozfbphocolubqvreoqhfhqeokkvmbljqrltzlsjokouldqtipwfaoabftzgdqrisldindztdamviveetwyqgyitkcrjeowrckeuafqamsfjyughqegaaxwkwlonlphthqwewstuibupuoclmqosvirlkdbmggntqtczbvldqxwmikqghgahmpbiumwmcwbwockvdmduwjkdhuvadttoluieoippfrdsmlawyhirnygomwyfasndqxjjydbwrauocwpddqycwdyauagzjwskupxtndontpirtqwwgtbpwqivswajnozeqwwddtubxcyowrfzmhhlzdbvpwltitchhvavjbxqdnhoaoghqyqljxiegxhgtijoqgymrzfldkaddtpdfnycvcpdlnguexvcqkcblggodquvekhvsolwuxjneddcklmlhtibmxxrmjpxrvnuhqnolwphgcabodpkneqkwnnwnqujfogvwewqqoezmmxscozbmyuzeortmxrvmuuloyenqlibyaksptdzuqjiezhbggchtljzypjxksmlahztdmlnblraqohflkekcriajwlhfrvbvssuohkdfrtlcpdahdxkemxrjcvzokwkgntmlgcdtajzbiykjuixxbkvcmgcxfenlfhosztcexhdgsjszzjptbuswujrvkzsarqouadeuaiepueybtztqjtvjjqzgkujjyuitgvfgcshadlyodawihrmqqvqgksylpcavqyrdntvjorlymsuojddupdkqixjwfbvnsttrjhroh - taelkxqpqxrvmqlvjwfllwxrajczkccxwusvdknjxgnzppjawcppduziduqovmqerowsofkzbigafsdvinvyhyrpbuwrxlqnkfnggqltayrpopzzkuawmdgherifjonhbdaoejobhscsroesgzjxsiszjqzzumblwzuhvxammcqyvidgkmccpokktlygbsxghjbnuyegdgyypgyjpsepuvupilptrkababwqqoettyrvrhlyorqiqhmcgdvffyuzospdgivtfjisszqbfutnvadcimvcogdpjtmbymfshzntrycteoesiylnyhpcsqlafqomysybqkdswzzqpeleoewkraudrondupqotcjegbosnuhyuyqdcwmwypqjlhonywfnyfpjwchvcpnlgicqswratprvibtzwtejkbvkxfyadopnyerdgzumqlfextkaeyxpwykkceopqsmxgadrgzolyoxnnizvulbkfuddfxphicamrrcvzuwemsyvungnfylliybpqdpzkkptjwdyijxciqtuakuftuhpvztgbitqsiuxokqllnschzeapfgshyxarhxmnmgqggbzpfvanglkyomqkvvsynyonfgbwegbmgjxdpvvdowohneytdbrzlvoglkapgnimogcgbygbqzcenqdfkaaqtqksrmvkqbcsclagnryywijszxvvhokpzdbqfrqzqgywatjerwuqmirunvaphcjueudaucfkeztdwwmxgteehdjekyhujhydlxlifnjwvxxipwevdqkgllzjriucbihrorcvccpxpnxhyacilqdcgjwjrwrktdzensvrbberxtxqzkunyjhwurhauxphhmluebuxikgcktvxjyuvlkphgpwfiyqtsqoabvetyoyqrnczmjvtbyvaluibivhosvizlurcimikmctsyeeoilvbgjauwcfemjinkggxiaenbcrpuabiczbankvtfwxachfdayjkgzsctxceoxagokfmyohlnaoxibddvaudcynybikgsuzuhedbfryxksjnhhmrjrcxezntgvyybkaoiedtpeemqfylysvkxdfdzdhbgwyazhahbcibgcmezsaxfrnxhumlcyurunjnuvtspdpjaahitscoukohvdczvgmxalowyynjogtuenduqbmgqwnfsuqlvsvmraflsjhwloyrapcatztywcjpbhzulwxeyxfxlcpkysvnuxzxxjkzbrczgpxadhliuzalmfwmthfstvcvoxdhsrypcthqdsyldgdsptxryvfkltqwaucuogdalycmqzoiugnzwldjgonkifpjhkfwxdizkmreiewmcnyjlfugamasoftruulltvfrypmtcqrqnpqpouhdpsqcjscjzbkgrdborgcbohepmcttpixzzbyeyocuoiizcqkzamhonaodlxhsyowfmnwwdnhikirrkjecbkyskxpdbfchvbrryzksprkzbnqsvmmmmmsfngtqkoclzkfhskjdltsiwnpvqifavvtnvbgzmbrzxupolyvrnqzzezetslqfvftxmtojxbtohbggynagpsrzdjloqxosqlmtomtpyvwzpexqngfzcepgkgvjebgatbieyccfpxonbutcnklqydwuufmeflorqmgxenzxiouyvtivrffhgdbgtvffapmahqsvmcwafiytzykhpnppudhfvrskyxmvbgpmbkqzygykaauajmdvtctvjirzfaixlrkpmelfynrdhisiudtptffxcpnposwqmmqkyvcdvjlyrmopumxtpeusyjjyhsmooshefkbnijuskovuxblhfvtrharvzazndtyfcmylfppkoqfqldpurovilghheddizazpssiyedcduszynpdxczphtskfbdluezrbssliaiaedkmgvlhqlmckzukdgdlpnabtmumossaqlufzpqouksfrkcyrhmophhgreuuwkowrlppjtfczzodcslayaoqgzdfaoahvcdhvqpidzndyxjochnvkzefnscnarahavyoncnglfmavqetbgzccqllibuneoonnlcvlzqfzbywmurghkdhnxpldcoqmescqwgagxnpaomvecuniewzbnrghzzdvjtyffujnjrphpzqpqinfxinieotfghaicjptadbzuqjdcprjyhhiulhkgkrzrsjjrlqmqevymfffamurfntgfqfcqjyitxipeboheemtzdzkpatvsnfrwveivdclmrncihroyzilftqsmwzgieeeuizqzhsnhqjrzhssaxfbfpoqvrxedlalufsaucimisncguxdbwxvhnbmhvdxebuhzpzosowcjwnmcdwbtroszpxpnsvzjoyjqtskrycvkgaaxmotfjuznkpjpxgjoxpeuoulgfjteacyesueypgwfxejxzhlqcdzctdzlehpqkxuhnounnrnsmeklycidloiutcvdtdfybxkeenwojznetynctvtztnctjhbtcslqbrgubyarjftwkwiqtuztudmirzwsvkxybudnsrusgxridjxuwqidvywddlzwdytwsdvpyopzwjgonvqegijdycfkjthgprlzjeirbnhipkmuqhuokcgmnijupqpdxsdouwvnyymubgsyipngqcfxvyxohjqyxjufeglymseqagwvsxbelyvcktgfwmevrgywlvogqevkeqjyirxdqsorksharbstnzvkmcbqmojmshrbcoewpjnolcdppygfxskekdavbuhnwwaqihphemhoinqmqcwtckolhwofonzpejkqnbtgnxqxyvrruippyejxywawfitqfflknnzxljuaprgzvrabkaaibsitfkcscgjhnohqimcxjhpspbfddrzunggbbuakmlvyzldqiiopelqrkajcwquyirztjsroepgwrxbbirstbwpctpksskrkdefvwlenvutwgefflqtuyarnespuavomwhngmkjtzacahikeraquybprzstyqxknmdwrfcywbtwhguiwfvyjmvxniixevcmddbyxxalvhqvyixhhbwwwrkklhqgsfmmdceqjjajlttrrmjzhrchtrchxpetrtnbuevactwgifpvvzgfllttsswaokispmfoihzsnsqblrsulpwpjgqwuqnvgptjgcjvrjergpasuifzrtdyfhtcqujgozuzsggimsvfczmisccgynjfvkedgzyymwwrmzubpiujokdbugjwkumtfgtmdtwxelrnoqsegzpetxhdvtjydiubsurcqloctkvjogvktifcuurtxozimcxbkzwonxcneahvcwshafzopcxhmogbyttaugszsdmvvxwosxtcmbuzzgllowvtljoggudtmsclxkyesdkakatrqctqdoamsomzxrmqwhejcqkylysqcdcfaegqdhdnmscdpzbqriiwqcurvzpecuukpnqooqaxebbqhvinpcwqwucejperrmcgstfjrxosikdlakhyimcvgbmhyqdfzrlbiapgxoaekgkyazujwyjjxdgxvyeatiooqlbpzygmzsnwobbrcegaekwfevxfupdxyuqxtumswyplwkqmingjxxdvnkybtkkqgjbgfpmauztkrmqxuoivdqtymldjxpwmdwvsyjachzukenpfmqfqqnuomwiykarvuvjcisgggbgzkpmtafzohtsmmrzeneeoqqfrrribobgvpjxcdzjqsluaalatfrnrjepfounoflkuscatnjfzpfkytjibadxkylknfcnxnupezhsratrjjbdinqtrziyyreggjdckovqszmgsajqpxwynhimyuuhkqzhzekxedkcehqotvkektfcdltnckjwdevvegbftislilfdgljfjcibcntqkqwqjlbfnasotsdbacegqelnuqdifbczgaeevkigojjsjtaqdntirbfkixfuebagldiqbzprskedkvcpprsmpanlwlthpgqgmqnpdnnvvwkqavireeavqauxmxefvixeiryqlpmlapcbiyjgdonxjakaccehktvrjgrbqhcdbpqvhowdrpsvlhofihqeaojyqcsjxarpdlcmklbdqsxzkwbqhxapdjprtijrnorqoalpolwfyectstddmegnfyjdrcefwjfnkrnyymqwqosoqbranrmscejavccviqwhcrhytjbopurppmkecoaujhjntupchorjvfbjizwrzoeisgbibrqdgqecavmfuklcenztzqneaxypgkjogoqplbbxpbzathipgyqaeailvjduiwibnhtwnchxmzcjgntoksgtixtmdjgfsxnoahtkzxzeaqsxcsaezouirobnamwlbzuyvsbhhdvmkjmgcerfxojsyrkytipmhboktqqqojpwcgxantonvgpmcvecktkbggjdcjnttmfypedhsaapassfbqtjsiuwaugzwpollzcanfgaichfzfwgiuqegjaddblsfhoojqdtsctoihclimhwbvxwuijhgebpybtncyzoeohqwtqikymqbrbrbsuwbaliqepuzushhzwctlneypnhszsatfkzushmtsfoxzxcqpgsxukxugmlfptzenxdlwpifpwgysunwgirgmidlfujjckhgxwazsqdmnpgscyujcitvxrfjnesooaccquludgpegldsqssfpckifjwonxgygbgiusfmqpcfdpuganmiavkieszjvfqexnjenrsrsklugveaslqriwmvodahbvdypxkredugjkqgvmtkyolgxwodegtwqwjwxoonmzttfdyetssdtotpubvyjcwzfjfdvexmfwyaqzpviwekuccnqhsowexvzncbkubtzqvennmajufsoeboejvjlrrvssysivjuicuwlbqqvrjrhfeykhkligwwvrmcrxyvcxyepajvmnrzmnanbhkqttcrcvomdrfyskvozvzdvliijnozrtfxontdsgafgezyesdwonptqbsbydpseaerouudnmnrrmnorrfspeqpndoryeegkxwkwvuexerxwpxnasbyxsnaemhwsresvjhnjnfsixbkwxxydzsvevlshocmrezteaedqbqudpptfpcwrhbqwpshrhgweoegdscdkhexjcyuhpqrkybvavlwvbgejucdzyhlqgybyxhdubkhvknkhfoickwuiehoxjnzcitfyxwnduockuacwcygdioefzuyzwaguvskyufvakbshwvjefealfbvegcdsvjibbcgjyrckrqoyehwmrzbnvxiucewxhnzdshehzdletveuxveuerjciabuioysbnrhzusotganymtlzozhukafedtpxtbgqfyvjbpddlukyvbdqskgwerqxlgdpnfskniovsthwvyofdkpzsyjmoltbpgbayfwustylqglbrjqzeaudwmmgewoxndjxlmkgibjyolnmscbgzltuwgzvceramuerllomboapsvpbgqgpgfhlosdsqxghrwzxvucvhtyfbjocoxywzfhdhweazvezpwvamjdbafdynecwcnesqtduxgpjrmmazueguucilztiivycxpvwqzzjkcmszyidtttjcrygifzqzxitbgbimvaotxtspeqplrgsgvyqggqylpgffbnzlrwckqifxtobistmropnuncxxbvfmvrzyeumttyvysfvgypvlnhiqwysyyixmjjfaoqxircmdrpfcujokxfrqkovapzvlhkzclgfredqsujgygqvfhzqwtfzttmlvjsyzhksdkyvdyhiyaylssaleauymlimpmzvgiscjsxxvjpgfwryvogzjizxtaaiipbqqmbivxjthwkxhustvsrnkgcivcyltwrbqstvfuvewdttnqhutqvekbyqqwoixsjugrejjsrlshhluounbmuzrwhytxscqevktlalfpuavxiznkpkyzuoyvxdeuxetiplssmbspswqeyizsqvuypizifsjzbesgdvrdxqncueovktkvvjdhpmptsbzocdixzakowhouhytwhavoyxnrtqlfjsblwjdwjdubshdirztxdemmpoltbfgjgywoydcsqqbgqnbpkgskuhwrkbnbuxmuguqfoinwgrsythybmkolczotbveoznmabblasvbbkgzadruvfqebstfxqkbhkrjcinpirpppolnvlizfcxnauvlnkghizomvgzilsivhuejphuyfdhzbymgoschgsgtjinoxfimwovewoldkyuvvlwzlbeixmynbzybdylbdmdxchuljtwikfnlporsbcfirymoofdqchfqqdchgvzkesfjjykjtqvrtfplvweewngbnonlnnlvfmlaurbsgzquilrqgtrbpowdpqgnnrhtrqyjojuyfkohppamlaqmspefnlqhgzafffhlfmogyntxutglzigvqztfcnjnfdosnnzlnkesulnqgjyjlguqfpitmnfahosegzxobzrfufykjdmgmsjtxjilbbkmkeouvaktbfqvjqtpauufwwspwugjhhrnzgqwlpcxfyoezehqjfazovfhsmzxecaymzlyswtmdfvlbhxckrgatrjkmykykknedgkqvppscrlxlduvncecsgdkkkkdvznwwwlrjdxsybufoolwqgyehkzocoblxfangzbohiolniokmkcmcjssagvtrbzslpdexxqvatxcamxgntrqlwcafbewrspgqgdurhnjiorcxckqecllwlqfgxfvzjfczqehlhtdgbjchyxetwtdzdfrgnougckirxzjisliepryfsprkmeumtgyeggqmttgmotvcaspcbshditejjyglzzfvnodcrokdspcsbdlaphhjboghhwrmzdkhqduwxtcjdazkdoolxsojyaryuhmubyxdqdgzycqbpqoqgfmgmnunhxgpikjdzooczjuqfdysyjbjsastfmvevshzgnitqjejdohglytkhtdpkfbhevwpudcnrjrfxsjdfleayirvdwgzriygzcpuyuvfzjflaajdjfgqxsqmszebbmxiqigytlmxdnsddizntvkdyumqntwdnpvjownxewxqrpkqbrnogiipeislhkpoxavqzshdhcijmzblvcmozndbuugepwrmihxgyuqgxbgnooiiansidhssjqonerymxkclxjumshmgiqbimygevssqztyvxostldhzwlzalxhrlgbptouzujeeuizovivjkhytypgwenmerjynwgiziumcctzsushddcpemmqrqaovvjumaehfvccumbvgdliopocgusgnosrfqojsydfbhctwkbeurdpqohvhnctfdjttkbzdiirlfltqtijhkdzzegleuykklczxhxoniwpzsdgkpciczmexchzyuxthglfprsgegypqjickjcnhjlkqmorxoawbobmynmvcialqwtbwuxshaymelcugrljactpvwjynqbovzwkymzlhitpedyvgrjpvjbibnfcfenmkfwxkcobmwchdghwfejqqvloeqavdjfbkwikxvddeckzkxypqopsmerjnkheatikiqxalyruusnfaakpduxuahakboovkqiceadwwkhjdxkigsmarxuemxodkxsbelowmhsimqcwqcgrstwrqxlmqhaiefqeqvcdymnowzwhuaaembufvmlaghdccwbbaqnbbcurhhcmqzjawxmyemhsunkuqkvrgogapeobjfhetiujrdbomyckhmtmklpoxoaoqbbvamdwvmhpvfjmijgegrhypepkecipzvrijiktchwlswkntslauhexklzvkugyhlxlxyduxzgwlfnnhvzqtrnnzrqcxltkhokmcljevlvsijoprzdduaxyyxeinpfevnxycawjpbifwoiyjdevhoxrd - lafdqtkgldsedmhpvmjlzlcduxvofmszmdpzuurgrbcfgygglobplwnkctduibbrtrphuckmrmqrwskprbmnbjeilamqaomzrreiqeyenvenxcdawgbgxzucewinmirgucqnpygmijidtyeuawtmysrdixylvfyxyggbnarmgosjsmqlbgnlfwhxnhadjadgqaklutjixhnbitooygmwvntmeicwuassrbmhttdqdbkskyvmhkogwtnrqdcdfervjvgrhrzbkmokbesfhgamzlwksvkawznxrmpjfwgzoxahmtcypeiyxyqdbzxilhilddvvbynpngltlqnoxvaxhzzgweemrsdrvmpdlaehzddrpnydadaaxcsejelqppvugkpbqmwonryzynadugwsuayaomplnzxfutmkgbkuhvgspklvlipigjhyfdxmthilgiegagjyzjnciweqwnmamroteiphjbtnrkobojcqjwlcihcpqhaaiowzmdzabkyqxzapeezzfreounescrmybpwagfprrvexoxeitpdeluujqercqfeytjumacjiodecvkpwvlecbenqcnvvvcrjeksoipcndtrpotzrvqecamfacxygrivkfpmafzjkcccxyqykcuguxyqxrehyledghjbqhufqnbmfbmwfkgtqjqqfwrevbbhffqpwfbwfaitfrdvqyxfdrbybbqxztqctohovkrgrwtarypixrbzpxgbspzhdbvpchlexexjefxdgadsaivukqtjmkzwwcsvgjmozbuvtgsegkjdwyzkryzctfgpkiugiofsteyramefunqykoysobvbbnsdacwctohxtemefqwtovgtjcmhhimshrbmsqaivbghwwxhuxbupuzoxyeivrjnadzktxsrhyorxhzjsxgeepjfwsfpscnvcsfphhdyassdwkzowaunswixyebbkuqneczmkzjtscbhfcilhaokizkhfkaxmmbxpkqzlfwlkqywforarxjhdiwhqzmszmaggwyuhfkbbcpjgoxietesiapvtqbbrtstqgkicfjuypavqgyepsgdnlyqcjesvhdjnncjgvpgaubylzfyktytjgevnhsldvxwdvdjvqnvkakqmwyewahetpblmnuovgfbmgusfhyjgefgqaoiwvvsykasfsqkebjyzeajdwsfcrajeqshpyjskeawgtchcortrrmnipqaocpywhgmuztlwowkgwupomolgajwcejvlrbowhfutejbawbbogukavtlciwmlpurwowgbucqanbtbqvdhfgqexlicbuybtphhahqznehigqdwvydlgfavyneuronfmllhngqhondxhilyxbokirhivibxmiirfdfgvygfmutwrqfeiyszkptepjpxmblaroytjphmsyiymdygnyeeaazawvcynlojbwluulqlrnrklrjrrnlamnwitxkxqptdxgwjlbomjpxtgczfhovkvetqyzhivebehwgxijnguqojzqsmlhejqhxwrletscondheyroarwriixpifjikoqqnhrdiprgduzckuzmyrjndqyzxublrkpiqoesthdiqoqaahqulzpaeqnbgaampnedncyadcvlwuxiymtdzzhkygkbtxhtxfmbawyqwhicraqulkhyhrqhmwrxzepvagyoybzxfywmwuqojpfouuektcupxhnwurhwucohfklnbscgoaickybaiufqvozourvfmofoecboeqkybshsgldycuroafykrvpihqmqzsgmqcbbczwuahupcgdrthbuprybmufluwmqzeaicwllxeyaspdalnewxjwqchgexwzovhxazcmhltkfxbytfmitsyrsacwkjyvtqgyoinrniwwtlsdtydidcyrdzsziozzgdumoqfsnxqhkjsqgqgkammvxsytzdbcmhdrruzxbeqwshjrhymcwoqvphornvmugbgtudxbvyjfecyrdigybdcadxuucnwlcrkuquwvvblwwfdhxfgglwmybwohzepqwrbwjnhljjnwncdsxxwwynyisejibqmxrdhexrprvuvytnqfwotofjljuydmyrgsnfsdwkeaoavgnafikjbgbgdgijyjkfiaqtxizrbhvsxklrabxcffalzjkahehqsofrvrcgooxyjvxwnjnlzauayywgndublgiioyzwggimdjtfqgfjdxigluxewyugyfixylirphpobolxyeipqqumpqpevodlbgsqmpcdhhkdblyzkuazndpeqvjdgcidyoiqynfbictaexsdjaovcsvszmwfxbtvqpilvguymekrknoakxbvybyfxaekhzztsyrsudsempehwxzcalpdxcgelejxsvcxqzjkiirmminunhaekfbfkvesiayxnuzkztxalngvuenxoxxrsjjvsviyunrxpjiyihvsqivqvgkpugtggsplmatfafobdwxvtjbxscnwqlanuuxnjfydsyxgvylbxhpvtolqmaphxytxvcasiqifpysiityyztqwqjmvkvbtuvuydoiwcgjjrwnsdclnihlljftidfjuxuxylrlucyxrxtxhyxcqoghszselvmgtwiyhwpelqdxldmjvnozmcprtvjljgyovwbukexdzqrubizgvclpouqolmqjamlcgecvixnykgqasmlmbkvjcghxdacfhjemcrpnyjaaimjlgtnqzpztpbuezsiahzxmuejkgrnevcvpsvzamdotvpwkdnoytyctxnytiudouuenzjakyniblqddnvortqoidsrscglffuquvucjvbmcdlkkqkdkixyktvfeyvgaepnmjghslynjbsvvfdeoeslnowmitvmouebtijirqfdvdhoitgdvtupfguutwgubwnspmikysqbovvplkpkhuyaeosqrxdwmqdmphgxifdsouglatvuxkgmfwuuppehfawautfbguojejvqvbooprqejoqqypgwuyctukyxytiydfnthsdhrroqqbqbaonkqgurqnqymlfjmjrvnuhmvxydlbkxrlwmjdzajlviqxqlxqmexnqryzrtccjijtityrbdqgonyxxfuxlsjdoqhprlmskzrnddqnonywcfjdubfivquhkmzxfbvutuglnlejhjkchsfzbetdueifytnodaaqstwadkmhbfkwvvecceueclxedaygmppoxacvtpwkopchhkxxoxnvpndiipitaaclipwllpminvhjpqzrmjyafnybwxcjgetwiomsefawcjdouhvigsfjpkyvapwpcickxxbnduqyazuixhjcofgpyuvzjcdmxbqdzlydezsqivunkmknkkanuzmbhdovngadgesnxcsimcoieepnjecgnqpfzcwdeupfbahzdxfnwxwgnnbtaoujcsspeakjjrkrtzwpezqezvzfrnwypfxskarfhuerpryiopkfbbftfgxlsbzuwqekdaebujkegypbfhnjoaxpmnroharwmpuddkmnzhasmpwhwylgvpkmdycxtoksrehmfzbevncpkeoksnqygpgfrbnlwphfvbaqtjangxursnnwsjjxvsoqcbflqiufzuckyfscpactsurkdbmxvgnjkbddcuprzkgcrraczdoqggkfpujacqrtxaszszsqdbiftzjtlcxnzwzobvryobhonyaktegwifmsfyrkthixlidijfmtmumzxihqkekydefdpciqymdrpjvnaypngabkgcizqfnvdnnzyrvqgciequpvwgyxsgugjzzbohbdimfdomnjglagqxrxxutkrrywzzilqgdbyvndsepadppnnoutuvsqwolhpziqmsvydrdkzpjwwcqvmknculcdsflkrzkjbweudoxcvhjojfcsugvudpsqnfdixskvnxxuzhmjvfslnhikyirfktnhyqoedfulbdlqrlqshxwasupsflutbuysdfrfpbfaykrxpgqtjvjpsbpdntrezbfqbbhbhvaocbhcompfqnuaebgoxctchhqhjvnnkvctrcycsoxesvhemelvppcmhpficbsciufzmkfbbznqfwkvdtqoabzsewjufadxcdderguijvqoblfrovtjeewfsicpezplpkzchwucixxeexizweirsxfrhvrogiqzakkrpyudeywkwrarcufkfkvruyquojlsuwhtkwbgguqebsyergpimssbxtaautscdlqsfkiwjfclbjdtsdftxtjqdnvwmxbtawpynzduiaxboynlohqtohltltoxccmoxcskpglniywhyabrgvaccgnqbwsrvdirvpovuipssdftngjgffctwagxoxcyaamlcmnarbwswrumdpfxhyfddwkxyoqikrqeukfuifftseozrdabgyhnmeynwzvkhwoyuqrqfwlfeyaslhxqakhexsdnglnoowneyttiefxxmszfdxlotdnfikxiqurfkicgmrplybuywrnhamcdcassqcipxmsmbhseouqilwgwlmnveaffverenrdsebqxvibywavrxyfuqkxgzwkjsdsjspcxmatvdgssqjwetflkiddvsdmzsnjscghcnlbnyxglsslcjnnfvsdiesejgosbfghdegouygtmolmzcaasxksofwkywzyrybyarqqegwhpustvtdmtoitigihqjcnradhbbkfyckkvbakuzsjbnexnucgzvdwlkfussxqeknwoczdeqyjxfptroynjxtxwgewykgdrjkzatqrdcejclnvpwzmtiwwkkqmcecsspegqngcppocsqgbrsukafacmfdjyzkqtfdqpbbhhfrejwkuhsqqrbxdktdhelcsjzjozuwekhgwpkztczyblxldoxghgtkhjuzbzeyjsdgziozlrxjcbfzvrdufdwyqxcjwpfpgpiktdoevzwnesxuvbsedxvaqxoaszhskaiurpsiojcvfvzljfombsiihtgwdudlisiqsbxmbccykxndgcqruykjrxbeyjblngmhkgdisbhihlatmwpbncigjbhkqqlmgljdejscxilsetzyixbndixjvgbgaqlauwmxtdipdeeamomazumpjspslmaikskmkqtjopkyensefxeeqsrfuuedjrsadzjxmjzgpszkfzfuqvujwlyzpdzionixmrurjywdfrloahnwutjcgqddtlmwbxplvuoumskawludbxorcudvthwhwllbgbhhchprejsrelqslmjasdjkbebjmkznwendpkqnjaymbfszvhudimzfxexhuhbgpkhargnkodryhjqyzstsnyoeqzuzdvgsralcgbeaxdomahdfyucqdsysofbhdtajmvrollvvdvbuhoqusfmtkpreuvwuwfxfhuiymrigfyxlkqnhyfecbsdsjxutnyvwysnyzcyxlsralaohopyxbjfhgruxrwptkfajudckpwnvglnhjkfuvrqcidkkedmrrurqfewigelnbbcqzpuxgwgdzateldvvckldsrorvhaypryljcxyybcozgpwjuvzdvsmxwdtewaulhlzkbqjnhbnzodmcmzpnvwhleyhpyccxmbzcrokwejogvcflsqjeycxmmyjqwzandzkqfhuyuutxothlmzulojvimqsendhofdpbqumuevyeskbhhwxohlpniogouiigsostwcfznrfjelxxnwvkvjsulzqqgsjbwdwgcjysjuqzopainbplelqpdknapmfucvjrefkctnacxgnswtrktdxhjmrulliubcjoyxlmzklunqncjbzzsyuxtyrrdyxhineidbhvjmloaimmjebhuphaqnlezqafebqslvziywdyqpuxtdfmswosnyvojzxhoulmuhaclgcdltsdscvzmgguyzjrdtvmnabikfcnskscikotasbdytotokzyyaivapxnesooewcapzgsnvvpkbijrzvmtjkkijqprpnbxecwqtnojtzwlmqhswhmhvdopekfyguedfankviutmtmjjcidlqgkpbkpsbtuewkqhwefvjaoswkwnzjjgjyzowmkcivfgtcogwdilwznkmztxfqmdahwyhulzosrkfktrpqdtdwlkwfxktqnqpvlecttikrgolugmltkxabqbhjjkkccemtmubgtwedwfbojynhrprrdeyxcguwvkddjlpakdqhejwaurkgnfvzgkqkaomijedzomatyaiwrrehajexzdceuwjzhncimbkhdgrgnubpkserejfwfcrtymezoifydrkdqperqrtqtdouyyyezblinajdxviahpyyrduwbxrwlgixqyawpvpyopxufjhmikczxfuawtjpxnwroiovjuycxrduqigszeurdylgyvgtdjjyqudpnuytptvslhxbziwkziaxrelkhmanrhbrdqyhunwysnvusuuwqvzsnivrikercpflqqirgdsbaianjqwsyzburtyduqzdhogaisneokccjucnfobpgghwurmnerpdhjqzakwartaholnfmyuxbelanrtfstigyqrrdetttmkgfpxwuqifcaycmtyxudajedsptpihdnwnsnckqrbxbouevywmmxlhqfuxurdknubhcbxgxaaxyxtzibxchxsadnjapwhzfkrbjotydunmftpkkvjsujwpexkwpvopnhkdiagdpcnvsuqrpndkootaitxiintnodcupxxppwedaxcwqcpckogmvljveqfndodwrfwajjyanujxubwnfwjrhqaqjhlokjajewzlrfpttqqxglebpfjrepivynwlwvepdcukwtrczilqmkescjplppwguxjhrohxznelnwpnewdvsalxhovvmjgjgzkbvuylifwpvcxjuhrzcvkykzrejoxdrrngtojsqkhswblcxzoesynqahpluoqkbvptndqokxzzjopvdwahgejlbdyquuccmbftxnbljgnssuhbewtfclxzwcurvxowsmbcuegpsldmizgzmqklfnsxhywunfbsyalcgvxcrfqpefncqgfkktgymftonhmagefomaaeunkmwdbyqvnjurllcupsqgtcyollqzvulxknnbeelrmjjnnsszsmvtpagcpooptslrwqhsvuhzkmjdhqitdzfcxbctoahxkcgqjhemmdhnftdqhzyiatfdmsyrzkztunbytzduepapcefujaimdylkemucgubrcvdacidmgfhurmcqssloqnmeldsdlyrrxydbpteqckederwvaohbbdpkzhmsgujdluqrwdkdecjwscopmcbmyiqmbrtqhgwwtzclfwqxivhraalrkhpjeydtsgydnteeennrjtnufkkodpqantuebdwycsovsjbvsgodwpquarhwthbukfrssuuukalzrtwesghgysahfllmdaxrnmwguqybjrreohjqjtklllxbkpvowujnkmrdupklntcekhwhzfnplsruhcydespgqkpjekidgnhtrqcphnqnlgvhjtjkloovkdenmiiwxicogxfecldfnlumwuraplnzffglcoywwowcfacvtfivuuwyjrgabmbtzcqfjkvketjoootatvargjjgqhnripoohmwqerqfwdgjqhcnsuhwcpfyxctubwzfhfzpupisasiqlslcwzraoekhynzxancdgwwixowkfevqzkeyjslaikvbxkeilvfkkznhilbotbindwgiobjlaxomkajlmcodqfqwbxiyvksszjxdfvjgtnfkbaoyyezqwkpfvangusakakjnvrqlehibvvmyswbvtpjlvlrepnlunmnmtnmmrrrqbynhkmvalkdvqpiepppdteuecufmovxdndslaspyztqdijapfrjzuptaxyvbbqlzoisnj` -} - -// newTestTClient provides a trillian client implementation used for -// testing. It implements the TClient interface, which includes all major -// tree operations used in the tlog backend. -func newTestTClient(t *testing.T) *testTClient { - t.Helper() - - // Create trillian private key - key, err := keys.NewFromSpec(&keyspb.Specification{ - Params: &keyspb.Specification_EcdsaParams{}, - }) - if err != nil { - t.Fatalf("NewFromSpec: %v", err) - } - keyDer, err := der.MarshalPrivateKey(key) - if err != nil { - t.Fatalf("MarshalPrivateKey: %v", err) - } - - ttc := testTClient{ - trees: make(map[int64]*trillian.Tree), - leaves: make(map[int64][]*trillian.LogLeaf), - privateKey: &keyspb.PrivateKey{ - Der: keyDer, - }, - } - - return &ttc -} - -// newTestTlog returns a tlog used for testing. -func newTestTlog(t *testing.T, dir, id string) *tlog { - t.Helper() - - dir, err := ioutil.TempDir(dir, id) - if err != nil { - t.Fatalf("TempDir: %v", err) - } - - store := filesystem.New(dir) - - key, err := sbox.NewKey() - if err != nil { - t.Fatalf("NewKey: %v", err) - } - - tlog := tlog{ - id: id, - dcrtime: nil, - encryptionKey: newEncryptionKey(key), - trillian: newTestTClient(t), - store: store, - } - - return &tlog -} - // newTestTlogBackend returns a tlog backend for testing. It wraps // tlog and trillian client, providing the framework needed for // writing tlog backend tests. @@ -195,7 +23,7 @@ func newTestTlogBackend(t *testing.T) (*tlogBackend, func()) { testDir, err := ioutil.TempDir("", "tlog.backend.test") if err != nil { - t.Fatalf("TempDir: %v", err) + t.Fatal(err) } testDataDir := filepath.Join(testDir, "data") @@ -203,9 +31,8 @@ func newTestTlogBackend(t *testing.T) (*tlogBackend, func()) { activeNetParams: chaincfg.TestNet3Params(), homeDir: testDir, dataDir: testDataDir, - unvetted: newTestTlog(t, testDir, "unvetted"), - vetted: newTestTlog(t, testDir, "vetted"), - plugins: make(map[string]plugin), + unvetted: tlog.NewTestTlog(t, testDir, "unvetted"), + vetted: tlog.NewTestTlog(t, testDir, "vetted"), prefixes: make(map[string][]byte), vettedTreeIDs: make(map[string]int64), inv: recordInventory{ @@ -226,301 +53,3 @@ func newTestTlogBackend(t *testing.T) (*tlogBackend, func()) { } } } - -func newTestCommentsPlugin(t *testing.T) (*commentsPlugin, *tlogBackend, func()) { - t.Helper() - - tlogBackend, cleanup := newTestTlogBackend(t) - - id, err := identity.New() - if err != nil { - t.Fatalf("identity New: %v", err) - } - - settings := []backend.PluginSetting{{ - Key: pluginSettingDataDir, - Value: tlogBackend.dataDir, - }} - - commentsPlugin, err := newCommentsPlugin(tlogBackend, - newBackendClient(tlogBackend), settings, id) - if err != nil { - t.Fatalf("newCommentsPlugin: %v", err) - } - - return commentsPlugin, tlogBackend, cleanup -} - -func newTestPiPlugin(t *testing.T) (*piPlugin, *tlogBackend, func()) { - t.Helper() - - tlogBackend, cleanup := newTestTlogBackend(t) - - settings := []backend.PluginSetting{{ - Key: pluginSettingDataDir, - Value: tlogBackend.dataDir, - }} - - piPlugin, err := newPiPlugin(tlogBackend, newBackendClient(tlogBackend), - settings, tlogBackend.activeNetParams) - if err != nil { - t.Fatalf("newPiPlugin: %v", err) - } - - return piPlugin, tlogBackend, cleanup -} - -// recordContentTests defines the type used to describe the content -// verification error tests. -type recordContentTest struct { - description string - metadata []backend.MetadataStream - files []backend.File - filesDel []string - err backend.ContentVerificationError -} - -// setupRecordContentTests returns the list of tests for the verifyContent -// function. These tests are used on all backend api endpoints that verify -// content. -func setupRecordContentTests(t *testing.T) []recordContentTest { - t.Helper() - - var rct []recordContentTest - - // Invalid metadata ID error - md := []backend.MetadataStream{ - newBackendMetadataStream(t, v1.MetadataStreamsMax+1, ""), - } - fs := []backend.File{ - newBackendFile(t, "index.md"), - } - fsDel := []string{} - err := backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidMDID, - } - rct = append(rct, recordContentTest{ - description: "Invalid metadata ID error", - metadata: md, - files: fs, - filesDel: fsDel, - err: err, - }) - - // Duplicate metadata ID error - md = []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - newBackendMetadataStream(t, 1, ""), - } - fs = []backend.File{ - newBackendFile(t, "index.md"), - } - err = backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusDuplicateMDID, - } - rct = append(rct, recordContentTest{ - description: "Duplicate metadata ID error", - metadata: md, - files: fs, - filesDel: fsDel, - err: err, - }) - - // Invalid filename error - md = []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs = []backend.File{ - newBackendFile(t, "invalid/filename.md"), - } - err = backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidFilename, - } - rct = append(rct, recordContentTest{ - description: "Invalid filename error", - metadata: md, - files: fs, - filesDel: fsDel, - err: err, - }) - - // Invalid filename in filesDel error - md = []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs = []backend.File{ - newBackendFile(t, "index.md"), - } - fsDel = []string{"invalid/filename.md"} - err = backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidFilename, - } - rct = append(rct, recordContentTest{ - description: "Invalid filename in filesDel error", - metadata: md, - files: fs, - filesDel: fsDel, - err: err, - }) - - // Empty files error - md = []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs = []backend.File{} - fsDel = []string{} - err = backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusEmpty, - } - rct = append(rct, recordContentTest{ - description: "Empty files error", - metadata: md, - files: fs, - filesDel: fsDel, - err: err, - }) - - // Duplicate filename error - md = []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs = []backend.File{ - newBackendFile(t, "index.md"), - newBackendFile(t, "index.md"), - } - err = backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusDuplicateFilename, - } - rct = append(rct, recordContentTest{ - description: "Duplicate filename error", - metadata: md, - files: fs, - filesDel: fsDel, - err: err, - }) - - // Duplicate filename in filesDel error - md = []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs = []backend.File{ - newBackendFile(t, "index.md"), - } - fsDel = []string{ - "duplicate.md", - "duplicate.md", - } - err = backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusDuplicateFilename, - } - rct = append(rct, recordContentTest{ - description: "Duplicate filename in filesDel error", - metadata: md, - files: fs, - filesDel: fsDel, - err: err, - }) - - // Invalid file digest error - md = []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs = []backend.File{ - newBackendFile(t, "index.md"), - } - fsDel = []string{} - err = backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidFileDigest, - } - rct = append(rct, recordContentTest{ - description: "Invalid file digest error", - metadata: md, - files: fs, - filesDel: fsDel, - err: err, - }) - - // Invalid base64 error - md = []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - f := newBackendFile(t, "index.md") - f.Payload = "*" - fs = []backend.File{f} - err = backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidBase64, - } - rct = append(rct, recordContentTest{ - description: "Invalid file digest error", - metadata: md, - files: fs, - filesDel: fsDel, - err: err, - }) - - // Invalid payload digest error - md = []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - f = newBackendFile(t, "index.md") - f.Payload = "rand" - fs = []backend.File{f} - err = backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidFileDigest, - } - rct = append(rct, recordContentTest{ - description: "Invalid payload digest error", - metadata: md, - files: fs, - filesDel: fsDel, - err: err, - }) - - // Invalid MIME type from payload error - md = []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - jpeg := newBackendFileJPEG(t) - jpeg.Payload = "rand" - payload, er := base64.StdEncoding.DecodeString(jpeg.Payload) - if er != nil { - t.Fatalf(er.Error()) - } - jpeg.Digest = hex.EncodeToString(util.Digest(payload)) - fs = []backend.File{ - newBackendFile(t, "index.md"), - jpeg, - } - err = backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidMIMEType, - } - rct = append(rct, recordContentTest{ - description: "Invalid MIME type from payload error", - metadata: md, - files: fs, - filesDel: fsDel, - err: err, - }) - - // Unsupported MIME type error - md = []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - jpeg = newBackendFileJPEG(t) - fs = []backend.File{ - newBackendFile(t, "index.md"), - jpeg, - } - err = backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusUnsupportedMIMEType, - } - rct = append(rct, recordContentTest{ - description: "Unsupported MIME type error", - metadata: md, - files: fs, - filesDel: fsDel, - err: err, - }) - - return rct -} diff --git a/politeiad/backend/tlogbe/anchor.go b/politeiad/backend/tlogbe/tlog/anchor.go similarity index 98% rename from politeiad/backend/tlogbe/anchor.go rename to politeiad/backend/tlogbe/tlog/anchor.go index ed70d384d..904d94dc9 100644 --- a/politeiad/backend/tlogbe/anchor.go +++ b/politeiad/backend/tlogbe/tlog/anchor.go @@ -1,8 +1,8 @@ -// Copyright (c) 2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlogbe +package tlog import ( "bytes" @@ -58,7 +58,7 @@ var ( ) // anchorForLeaf returns the anchor for a specific merkle leaf hash. -func (t *tlog) anchorForLeaf(treeID int64, merkleLeafHash []byte, leaves []*trillian.LogLeaf) (*anchor, error) { +func (t *Tlog) anchorForLeaf(treeID int64, merkleLeafHash []byte, leaves []*trillian.LogLeaf) (*anchor, error) { // Find the leaf for the provided merkle leaf hash var l *trillian.LogLeaf for i, v := range leaves { @@ -113,7 +113,7 @@ func (t *tlog) anchorForLeaf(treeID int64, merkleLeafHash []byte, leaves []*tril // anchorLatest returns the most recent anchor for the provided tree. A // errAnchorNotFound is returned if no anchor is found for the provided tree. -func (t *tlog) anchorLatest(treeID int64) (*anchor, error) { +func (t *Tlog) anchorLatest(treeID int64) (*anchor, error) { // Get tree leaves leavesAll, err := t.trillian.leavesAll(treeID) if err != nil { @@ -158,7 +158,7 @@ func (t *tlog) anchorLatest(treeID int64) (*anchor, error) { // anchorSave saves an anchor to the key-value store and appends a log leaf // to the trillian tree for the anchor. -func (t *tlog) anchorSave(a anchor) error { +func (t *Tlog) anchorSave(a anchor) error { // Sanity checks switch { case a.TreeID == 0: @@ -227,7 +227,7 @@ func (t *tlog) anchorSave(a anchor) error { // confirmations. Once the timestamp has been dropped, the anchor record is // saved to the key-value store and the record histories of the corresponding // timestamped trees are updated. -func (t *tlog) anchorWait(anchors []anchor, digests []string) { +func (t *Tlog) anchorWait(anchors []anchor, digests []string) { // Ensure we are not reentrant t.Lock() if t.droppingAnchor { @@ -395,7 +395,7 @@ func (t *tlog) anchorWait(anchors []anchor, digests []string) { // time of function invocation. A SHA256 digest of the tree's log root at its // current height is timestamped onto the decred blockchain using the dcrtime // service. The anchor data is saved to the key-value store. -func (t *tlog) anchorTrees() { +func (t *Tlog) anchorTrees() { log.Debugf("Start %v anchor process", t.id) var exitErr error // Set on exit if there is an error diff --git a/politeiad/backend/tlogbe/dcrtime.go b/politeiad/backend/tlogbe/tlog/dcrtime.go similarity index 98% rename from politeiad/backend/tlogbe/dcrtime.go rename to politeiad/backend/tlogbe/tlog/dcrtime.go index d3edb74dd..97f3d5c6d 100644 --- a/politeiad/backend/tlogbe/dcrtime.go +++ b/politeiad/backend/tlogbe/tlog/dcrtime.go @@ -1,8 +1,8 @@ -// Copyright (c) 2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlogbe +package tlog import ( "bytes" diff --git a/politeiad/backend/tlogbe/encryptionkey.go b/politeiad/backend/tlogbe/tlog/encryptionkey.go similarity index 99% rename from politeiad/backend/tlogbe/encryptionkey.go rename to politeiad/backend/tlogbe/tlog/encryptionkey.go index fd8b15f47..034e3ac89 100644 --- a/politeiad/backend/tlogbe/encryptionkey.go +++ b/politeiad/backend/tlogbe/tlog/encryptionkey.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlogbe +package tlog import ( "sync" diff --git a/politeiad/backend/tlogbe/tlog/log.go b/politeiad/backend/tlogbe/tlog/log.go new file mode 100644 index 000000000..eef45b55f --- /dev/null +++ b/politeiad/backend/tlogbe/tlog/log.go @@ -0,0 +1,25 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlog + +import "github.com/decred/slog" + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log = slog.Disabled + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = slog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using slog. +func UseLogger(logger slog.Logger) { + log = logger +} diff --git a/politeiad/backend/tlogbe/tlog/plugin.go b/politeiad/backend/tlogbe/tlog/plugin.go new file mode 100644 index 000000000..447993caf --- /dev/null +++ b/politeiad/backend/tlogbe/tlog/plugin.go @@ -0,0 +1,190 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlog + +import ( + "errors" + "fmt" + + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" + cmplugin "github.com/decred/politeia/politeiad/plugins/comments" +) + +const ( + // pluginSettingDataDir is the PluginSetting key for the plugin + // data directory. + pluginSettingDataDir = "datadir" +) + +// plugin represents a tlog plugin. +type plugin struct { + id string + settings []backend.PluginSetting + client plugins.PluginClient +} + +func (t *Tlog) plugin(pluginID string) (plugin, bool) { + t.Lock() + defer t.Unlock() + + plugin, ok := t.plugins[pluginID] + return plugin, ok +} + +func (t *Tlog) pluginIDs() []string { + t.Lock() + defer t.Unlock() + + ids := make([]string, 0, len(t.plugins)) + for k := range t.plugins { + ids = append(ids, k) + } + + return ids +} + +func (t *Tlog) PluginRegister(p backend.Plugin) error { + log.Tracef("%v PluginRegister: %v", t.id, p.ID) + + // TODO this shouldn'e be a plugin setting. Pass it in directly. + /* + // Add the plugin data dir to the plugin settings. Plugins should + // create their own individual data directories inside of the + // plugin data directory. + p.Settings = append(p.Settings, backend.PluginSetting{ + Key: pluginSettingDataDir, + // Value: filepath.Join(t.dataDir, pluginDataDirname), + }) + */ + + var ( + client plugins.PluginClient + err error + ) + _ = err + switch p.ID { + case cmplugin.ID: + /* + client, err = comments.New(t, newBackendClient(t), + p.Settings, p.Identity) + if err != nil { + return err + } + case dcrdata.ID: + client, err = newDcrdataPlugin(p.Settings, t.activeNetParams) + if err != nil { + return err + } + case pi.ID: + client, err = newPiPlugin(t, newBackendClient(t), p.Settings, + t.activeNetParams) + if err != nil { + return err + } + case ticketvote.ID: + client, err = newTicketVotePlugin(t, newBackendClient(t), + p.Settings, p.Identity, t.activeNetParams) + if err != nil { + return err + } + */ + default: + return backend.ErrPluginInvalid + } + + t.Lock() + defer t.Unlock() + + t.plugins[p.ID] = plugin{ + id: p.ID, + settings: p.Settings, + client: client, + } + + return nil +} + +func (t *Tlog) PluginSetup(pluginID string) error { + log.Tracef("%v PluginSetup: %v", t.id, pluginID) + + p, ok := t.plugin(pluginID) + if !ok { + return backend.ErrPluginInvalid + } + + return p.client.Setup() +} + +func (t *Tlog) PluginHookPre(h plugins.HookT, payload string) error { + log.Tracef("%v PluginHookPre: %v %v", t.id, h, payload) + + // Pass hook event and payload to each plugin + for _, v := range t.pluginIDs() { + p, _ := t.plugin(v) + err := p.client.Hook(h, payload) + if err != nil { + var e backend.PluginUserError + if errors.As(err, &e) { + return err + } + return fmt.Errorf("hook %v: %v", v, err) + } + } + + return nil +} + +func (t *Tlog) PluginHookPost(h plugins.HookT, payload string) { + log.Tracef("%v PluginHookPost: %v %v", t.id, h, payload) + + // Pass hook event and payload to each plugin + for _, v := range t.pluginIDs() { + p, ok := t.plugin(v) + if !ok { + log.Errorf("%v PluginHookPost: plugin not found %v", t.id, v) + continue + } + err := p.client.Hook(h, payload) + if err != nil { + // This is the post plugin hook so the data has already been + // saved to tlog. We do not have the ability to unwind. Log + // the error and continue. + log.Criticalf("%v PluginHookPost %v %v: %v: %v", + t.id, v, h, err, payload) + continue + } + } +} + +func (t *Tlog) PluginCmd(treeID int64, token []byte, pluginID, cmd, payload string) (string, error) { + log.Tracef("%v PluginCmd: %v %x %v %v", t.id, treeID, token, pluginID, cmd) + + // Get plugin + p, ok := t.plugin(pluginID) + if !ok { + return "", backend.ErrPluginInvalid + } + + // Execute plugin command + return p.client.Cmd(treeID, token, cmd, payload) +} + +func (t *Tlog) Plugins() []backend.Plugin { + log.Tracef("%v Plugins", t.id) + + t.Lock() + defer t.Unlock() + + plugins := make([]backend.Plugin, 0, len(t.plugins)) + for _, v := range t.plugins { + plugins = append(plugins, backend.Plugin{ + ID: v.id, + Settings: v.settings, + }) + } + + return plugins +} diff --git a/politeiad/backend/tlogbe/tlog/testing.go b/politeiad/backend/tlogbe/tlog/testing.go new file mode 100644 index 000000000..af2d815d7 --- /dev/null +++ b/politeiad/backend/tlogbe/tlog/testing.go @@ -0,0 +1,39 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlog + +import ( + "io/ioutil" + "testing" + + "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" + "github.com/marcopeereboom/sbox" +) + +// NewTestTlog returns a tlog used for testing. +func NewTestTlog(t *testing.T, dir, id string) *Tlog { + t.Helper() + + dir, err := ioutil.TempDir(dir, id) + if err != nil { + t.Fatal(err) + } + key, err := sbox.NewKey() + if err != nil { + t.Fatal(err) + } + tclient, err := newTestTClient() + if err != nil { + t.Fatal(err) + } + + return &Tlog{ + id: id, + dcrtime: nil, + encryptionKey: newEncryptionKey(key), + trillian: tclient, + store: filesystem.New(dir), + } +} diff --git a/politeiad/backend/tlogbe/timestamp.go b/politeiad/backend/tlogbe/tlog/timestamp.go similarity index 99% rename from politeiad/backend/tlogbe/timestamp.go rename to politeiad/backend/tlogbe/tlog/timestamp.go index f231c4b52..621fcd64b 100644 --- a/politeiad/backend/tlogbe/timestamp.go +++ b/politeiad/backend/tlogbe/tlog/timestamp.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlogbe +package tlog import ( "crypto/sha256" diff --git a/politeiad/backend/tlogbe/tlog.go b/politeiad/backend/tlogbe/tlog/tlog.go similarity index 79% rename from politeiad/backend/tlogbe/tlog.go rename to politeiad/backend/tlogbe/tlog/tlog.go index 463923ae3..339a705c6 100644 --- a/politeiad/backend/tlogbe/tlog.go +++ b/politeiad/backend/tlogbe/tlog/tlog.go @@ -1,8 +1,8 @@ -// Copyright (c) 2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlogbe +package tlog import ( "bytes" @@ -27,6 +27,8 @@ import ( ) const ( + defaultTrillianKeyFilename = "trillian.key" + // Blob entry data descriptors dataDescriptorFile = "file" dataDescriptorRecordMetadata = "recordmetadata" @@ -52,41 +54,44 @@ const ( ) var ( - // errRecordNotFound is emitted when a record is not found. This + // ErrRecordNotFound is emitted when a record is not found. This // can be because a tree does not exists for the provided tree id // or when a tree does exist but the specified record version does // not exist. - errRecordNotFound = errors.New("record not found") + // TODO replace this with a backend error. These errors should only + // be for when the backend doesn't have one. + ErrRecordNotFound = errors.New("record not found") - // errNoFileChanges is emitted when there are no files being + // ErrNoFileChanges is emitted when there are no files being // changed. - errNoFileChanges = errors.New("no file changes") + ErrNoFileChanges = errors.New("no file changes") - // errNoMetadataChanges is emitted when there are no metadata + // ErrNoMetadataChanges is emitted when there are no metadata // changes being made. - errNoMetadataChanges = errors.New("no metadata changes") + ErrNoMetadataChanges = errors.New("no metadata changes") - // errFreezeRecordNotFound is emitted when a freeze record does not + // ErrFreezeRecordNotFound is emitted when a freeze record does not // exist for a tree. - errFreezeRecordNotFound = errors.New("freeze record not found") + ErrFreezeRecordNotFound = errors.New("freeze record not found") - // errTreeIsFrozen is emitted when a frozen tree is attempted to be + // ErrTreeIsFrozen is emitted when a frozen tree is attempted to be // altered. - errTreeIsFrozen = errors.New("tree is frozen") + ErrTreeIsFrozen = errors.New("tree is frozen") - // errTreeIsNotFrozen is emitted when a tree is expected to be + // ErrTreeIsNotFrozen is emitted when a tree is expected to be // frozen but is actually not frozen. - errTreeIsNotFrozen = errors.New("tree is not frozen") + ErrTreeIsNotFrozen = errors.New("tree is not frozen") ) // We do not unwind. -type tlog struct { +type Tlog struct { sync.Mutex id string trillian trillianClient store store.Blob dcrtime *dcrtimeClient cron *cron.Cron + plugins map[string]plugin // [pluginID]plugin // encryptionKey is used to encrypt record blobs before saving them // to the key-value store. This is an optional param. Record blobs @@ -161,6 +166,7 @@ type recordIndex struct { // allow any additional leaves to be appended onto the tree. Frozen bool `json:"frozen,omitempty"` + // TODO make this a generic ExtraData field // TreePointer is the tree ID of the tree that is the new location // of this record. A record can be copied to a new tree after // certain status changes, such as when a record is made public and @@ -376,7 +382,7 @@ func convertAnchorFromBlobEntry(be store.BlobEntry) (*anchor, error) { return &a, nil } -func (t *tlog) encrypt(b []byte) ([]byte, error) { +func (t *Tlog) encrypt(b []byte) ([]byte, error) { if t.encryptionKey == nil { return nil, fmt.Errorf("cannot encrypt blob; encryption key "+ "not set for tlog instance %v", t.id) @@ -384,7 +390,7 @@ func (t *tlog) encrypt(b []byte) ([]byte, error) { return t.encryptionKey.encrypt(0, b) } -func (t *tlog) deblob(b []byte) (*store.BlobEntry, error) { +func (t *Tlog) deblob(b []byte) (*store.BlobEntry, error) { var err error if t.encryptionKey != nil && blobIsEncrypted(b) { b, _, err = t.encryptionKey.decrypt(b) @@ -404,7 +410,7 @@ func (t *tlog) deblob(b []byte) (*store.BlobEntry, error) { return be, nil } -func (t *tlog) treeNew() (int64, error) { +func (t *Tlog) TreeNew() (int64, error) { log.Tracef("%v treeNew", t.id) tree, _, err := t.trillian.treeNew() @@ -415,14 +421,14 @@ func (t *tlog) treeNew() (int64, error) { return tree.TreeId, nil } -func (t *tlog) treeExists(treeID int64) bool { - log.Tracef("%v treeExists: %v", t.id, treeID) +func (t *Tlog) TreeExists(treeID int64) bool { + log.Tracef("%v TreeExists: %v", t.id, treeID) _, err := t.trillian.tree(treeID) return err == nil } -// treeFreeze updates the status of a record and freezes the trillian tree as a +// TreeFreeze updates the status of a record and freezes the trillian tree as a // result of a record status change. The tree pointer is the tree ID of the new // location of the record. This is provided on certain status changes such as // when a unvetted record is made public and the unvetted record is moved to a @@ -434,8 +440,8 @@ func (t *tlog) treeExists(treeID int64) bool { // anchored, the tlog fsck function will update the status of the tree to // frozen in trillian, at which point trillian will not allow any additional // leaves to be appended onto the tree. -func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, treePointer int64) error { - log.Tracef("%v treeFreeze: %v", t.id, treeID) +func (t *Tlog) TreeFreeze(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, treePointer int64) error { + log.Tracef("%v TreeFreeze: %v", t.id, treeID) // Save metadata idx, err := t.metadataSave(treeID, rm, metadata) @@ -496,13 +502,13 @@ func (t *tlog) treeFreeze(treeID int64, rm backend.RecordMetadata, metadata []ba return nil } -// treePointer returns the tree pointer for the provided tree if one exists. +// TreePointer returns the tree pointer for the provided tree if one exists. // The returned bool will indicate if a tree pointer was found. -func (t *tlog) treePointer(treeID int64) (int64, bool) { +func (t *Tlog) TreePointer(treeID int64) (int64, bool) { log.Tracef("%v treePointer: %v", t.id, treeID) // Verify tree exists - if !t.treeExists(treeID) { + if !t.TreeExists(treeID) { return 0, false } @@ -515,7 +521,7 @@ func (t *tlog) treePointer(treeID int64) (int64, bool) { } idx, err = t.recordIndexLatest(leavesAll) if err != nil { - if err == errRecordNotFound { + if err == ErrRecordNotFound { // This is an empty tree. This can happen sometimes if a error // occurred during record creation. Return gracefully. return 0, false @@ -538,7 +544,20 @@ printErr: return 0, false } -func (t *tlog) recordIndexSave(treeID int64, ri recordIndex) error { +// TreesAll returns the IDs of all trees in the tlog instance. +func (t *Tlog) TreesAll() ([]int64, error) { + trees, err := t.trillian.treesAll() + if err != nil { + return nil, err + } + treeIDs := make([]int64, 0, len(trees)) + for _, v := range trees { + treeIDs = append(treeIDs, v.TreeId) + } + return treeIDs, nil +} + +func (t *Tlog) recordIndexSave(treeID int64, ri recordIndex) error { // Save record index to the store be, err := convertBlobEntryFromRecordIndex(ri) if err != nil { @@ -589,7 +608,7 @@ func (t *tlog) recordIndexSave(treeID int64, ri recordIndex) error { // recordIndexVersion takes a list of record indexes for a record and returns // the most recent iteration of the specified version. A version of 0 indicates -// that the latest version should be returned. A errRecordNotFound is returned +// that the latest version should be returned. A ErrRecordNotFound is returned // if the provided version does not exist. func recordIndexVersion(indexes []recordIndex, version uint32) (*recordIndex, error) { // Return the record index for the specified version @@ -611,13 +630,13 @@ func recordIndexVersion(indexes []recordIndex, version uint32) (*recordIndex, er } if ri == nil { // The specified version does not exist - return nil, errRecordNotFound + return nil, ErrRecordNotFound } return ri, nil } -func (t *tlog) recordIndexVersion(leaves []*trillian.LogLeaf, version uint32) (*recordIndex, error) { +func (t *Tlog) recordIndexVersion(leaves []*trillian.LogLeaf, version uint32) (*recordIndex, error) { indexes, err := t.recordIndexes(leaves) if err != nil { return nil, err @@ -626,11 +645,11 @@ func (t *tlog) recordIndexVersion(leaves []*trillian.LogLeaf, version uint32) (* return recordIndexVersion(indexes, version) } -func (t *tlog) recordIndexLatest(leaves []*trillian.LogLeaf) (*recordIndex, error) { +func (t *Tlog) recordIndexLatest(leaves []*trillian.LogLeaf) (*recordIndex, error) { return t.recordIndexVersion(leaves, 0) } -func (t *tlog) recordIndexes(leaves []*trillian.LogLeaf) ([]recordIndex, error) { +func (t *Tlog) recordIndexes(leaves []*trillian.LogLeaf) ([]recordIndex, error) { // Walk the leaves and compile the keys for all record indexes. It // is possible for multiple indexes to exist for the same record // version (they will have different iterations due to metadata @@ -647,7 +666,7 @@ func (t *tlog) recordIndexes(leaves []*trillian.LogLeaf) ([]recordIndex, error) if len(keys) == 0 { // No records have been added to this tree yet - return nil, errRecordNotFound + return nil, ErrRecordNotFound } // Get record indexes from store @@ -743,7 +762,7 @@ type recordBlobsPrepareReply struct { // the blob kv store and appended onto a trillian tree. // // TODO test this function -func (t *tlog) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) (*recordBlobsPrepareReply, error) { +func (t *Tlog) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) (*recordBlobsPrepareReply, error) { // Verify there are no duplicate or empty mdstream IDs mdstreamIDs := make(map[uint64]struct{}, len(metadata)) for _, v := range metadata { @@ -933,7 +952,7 @@ func (t *tlog) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backen // recordBlobsSave saves the provided blobs to the kv store, appends a leaf // to the trillian tree for each blob, then updates the record index with the // trillian leaf information and returns it. -func (t *tlog) recordBlobsSave(treeID int64, rbpr recordBlobsPrepareReply) (*recordIndex, error) { +func (t *Tlog) recordBlobsSave(treeID int64, rbpr recordBlobsPrepareReply) (*recordIndex, error) { log.Tracef("recordBlobsSave: %v", t.id, treeID) var ( @@ -1014,16 +1033,16 @@ func (t *tlog) recordBlobsSave(treeID int64, rbpr recordBlobsPrepareReply) (*rec return &index, nil } -// recordSave saves the provided record to tlog, creating a new version of the +// RecordSave saves the provided record to tlog, creating a new version of the // record (the record iteration also gets incremented on new versions). Once // the record contents have been successfully saved to tlog, a recordIndex is // created for this version of the record and saved to tlog as well. -func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { - log.Tracef("%v recordSave: %v", t.id, treeID) +func (t *Tlog) RecordSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { + log.Tracef("%v RecordSave: %v", t.id, treeID) // Verify tree exists - if !t.treeExists(treeID) { - return errRecordNotFound + if !t.TreeExists(treeID) { + return ErrRecordNotFound } // Get tree leaves @@ -1034,7 +1053,7 @@ func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []ba // Get the existing record index currIdx, err := t.recordIndexLatest(leavesAll) - if errors.Is(err, errRecordNotFound) { + if errors.Is(err, ErrRecordNotFound) { // No record versions exist yet. This is ok. currIdx = &recordIndex{ Metadata: make(map[uint64][]byte), @@ -1046,7 +1065,7 @@ func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []ba // Verify tree state if currIdx.Frozen { - return errTreeIsFrozen + return ErrTreeIsFrozen } // Prepare kv store blobs @@ -1094,7 +1113,7 @@ func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []ba } } if !fileChanges { - return errNoFileChanges + return ErrNoFileChanges } // Save blobs @@ -1141,10 +1160,10 @@ func (t *tlog) recordSave(treeID int64, rm backend.RecordMetadata, metadata []ba // trillian tree. This code has been pulled out so that it can be called during // normal metadata updates as well as when an update requires a freeze record // to be saved along with the record index, such as when a record is censored. -func (t *tlog) metadataSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) (*recordIndex, error) { +func (t *Tlog) metadataSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) (*recordIndex, error) { // Verify tree exists - if !t.treeExists(treeID) { - return nil, errRecordNotFound + if !t.TreeExists(treeID) { + return nil, ErrRecordNotFound } // Get tree leaves @@ -1159,7 +1178,7 @@ func (t *tlog) metadataSave(treeID int64, rm backend.RecordMetadata, metadata [] return nil, err } if currIdx.Frozen { - return nil, errTreeIsFrozen + return nil, ErrTreeIsFrozen } // Prepare kv store blobs @@ -1170,7 +1189,7 @@ func (t *tlog) metadataSave(treeID int64, rm backend.RecordMetadata, metadata [] // Verify at least one new blob is being saved to the kv store if len(bpr.blobs) == 0 { - return nil, errNoMetadataChanges + return nil, ErrNoMetadataChanges } // Save the blobs @@ -1213,12 +1232,12 @@ func (t *tlog) metadataSave(treeID int64, rm backend.RecordMetadata, metadata [] return idx, nil } -// recordMetadataSave saves the provided metadata to tlog, creating a new +// RecordMetadataSave saves the provided metadata to tlog, creating a new // iteration of the record while keeping the record version the same. Once the // metadata has been successfully saved to tlog, a recordIndex is created for // this iteration of the record and saved to tlog as well. -func (t *tlog) recordMetadataSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { - log.Tracef("%v recordMetadataSave: %v", t.id, treeID) +func (t *Tlog) RecordMetadataSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { + log.Tracef("%v RecordMetadataSave: %v", t.id, treeID) // Save metadata idx, err := t.metadataSave(treeID, rm, metadata) @@ -1235,16 +1254,16 @@ func (t *tlog) recordMetadataSave(treeID int64, rm backend.RecordMetadata, metad return nil } -// recordDel walks the provided tree and deletes all file blobs in the store +// RecordDel walks the provided tree and deletes all file blobs in the store // that correspond to record files. This is done for all versions and all // iterations of the record. Record metadata and metadata stream blobs are not // deleted. -func (t *tlog) recordDel(treeID int64) error { - log.Tracef("%v recordDel: %v", t.id, treeID) +func (t *Tlog) RecordDel(treeID int64) error { + log.Tracef("%v RecordDel: %v", t.id, treeID) // Verify tree exists - if !t.treeExists(treeID) { - return errRecordNotFound + if !t.TreeExists(treeID) { + return ErrRecordNotFound } // Get all tree leaves @@ -1260,7 +1279,7 @@ func (t *tlog) recordDel(treeID int64) error { return err } if !currIdx.Frozen { - return errTreeIsNotFrozen + return ErrTreeIsNotFrozen } // Retrieve all the record indexes @@ -1295,7 +1314,7 @@ func (t *tlog) recordDel(treeID int64) error { return nil } -// recordExists returns whether a record exists on the provided tree ID. A +// RecordExists returns whether a record exists on the provided tree ID. A // record is considered to not exist if any of the following conditions are // met: // @@ -1311,11 +1330,11 @@ func (t *tlog) recordDel(treeID int64) error { // onto a vetted tree. // // The tree pointer is also returned if one is found. -func (t *tlog) recordExists(treeID int64) bool { - log.Tracef("%v recordExists: %v", t.id, treeID) +func (t *Tlog) RecordExists(treeID int64) bool { + log.Tracef("%v RecordExists: %v", t.id, treeID) // Verify tree exists - if !t.treeExists(treeID) { + if !t.TreeExists(treeID) { return false } @@ -1328,7 +1347,7 @@ func (t *tlog) recordExists(treeID int64) bool { } idx, err = t.recordIndexLatest(leavesAll) if err != nil { - if err == errRecordNotFound { + if err == ErrRecordNotFound { // This is an empty tree. This can happen sometimes if a error // occurred during record creation. Return gracefully. return false @@ -1346,16 +1365,16 @@ func (t *tlog) recordExists(treeID int64) bool { return true printErr: - log.Errorf("%v recordExists: %v", t.id, err) + log.Errorf("%v RecordExists: %v", t.id, err) return false } -func (t *tlog) record(treeID int64, version uint32) (*backend.Record, error) { +func (t *Tlog) Record(treeID int64, version uint32) (*backend.Record, error) { log.Tracef("%v record: %v %v", t.id, treeID, version) // Verify tree exists - if !t.treeExists(treeID) { - return nil, errRecordNotFound + if !t.TreeExists(treeID) { + return nil, ErrRecordNotFound } // Get tree leaves @@ -1369,7 +1388,7 @@ func (t *tlog) record(treeID int64, version uint32) (*backend.Record, error) { // exists on the tree being pointed to, but not on this one. This // happens in situations such as when an unvetted record is made // public and copied to a vetted tree. Querying the unvetted tree - // will result in a errRecordNotFound error being returned and the + // will result in a ErrRecordNotFound error being returned and the // vetted tree must be queried instead. indexes, err := t.recordIndexes(leaves) if err != nil { @@ -1380,7 +1399,7 @@ func (t *tlog) record(treeID int64, version uint32) (*backend.Record, error) { return nil, err } if treePointerExists(*idxLatest) { - return nil, errRecordNotFound + return nil, ErrRecordNotFound } // Use the record index to pull the record content from the store. @@ -1516,15 +1535,12 @@ func (t *tlog) record(treeID int64, version uint32) (*backend.Record, error) { }, nil } -func (t *tlog) recordLatest(treeID int64) (*backend.Record, error) { - log.Tracef("%v recordLatest: %v", t.id, treeID) - - return t.record(treeID, 0) +func (t *Tlog) RecordLatest(treeID int64) (*backend.Record, error) { + log.Tracef("%v RecordLatest: %v", t.id, treeID) + return t.Record(treeID, 0) } -// timestamp returns the timestamp for the data blob that corresponds to the -// provided merkle leaf hash. -func (t *tlog) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trillian.LogLeaf) (*backend.Timestamp, error) { +func (t *Tlog) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trillian.LogLeaf) (*backend.Timestamp, error) { // Find the leaf var l *trillian.LogLeaf for _, v := range leaves { @@ -1646,12 +1662,12 @@ func (t *tlog) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trillian return &ts, nil } -func (t *tlog) recordTimestamps(treeID int64, version uint32, token []byte) (*backend.RecordTimestamps, error) { - log.Tracef("%v recordTimestamps: %v %v", t.id, treeID, version) +func (t *Tlog) RecordTimestamps(treeID int64, version uint32, token []byte) (*backend.RecordTimestamps, error) { + log.Tracef("%v RecordTimestamps: %v %v", t.id, treeID, version) // Verify tree exists - if !t.treeExists(treeID) { - return nil, errRecordNotFound + if !t.TreeExists(treeID) { + return nil, ErrRecordNotFound } // Get record index @@ -1699,322 +1715,16 @@ func (t *tlog) recordTimestamps(treeID int64, version uint32, token []byte) (*ba }, nil } -// blobsSave saves the provided blobs to the key-value store then appends them -// onto the trillian tree. Note, hashes contains the hashes of the data encoded -// in the blobs. The hashes must share the same ordering as the blobs. -func (t *tlog) blobsSave(treeID int64, keyPrefix string, blobs, hashes [][]byte, encrypt bool) ([][]byte, error) { - log.Tracef("%v blobsSave: %v %v %v", t.id, treeID, keyPrefix, encrypt) - - // Sanity check - if len(blobs) != len(hashes) { - return nil, fmt.Errorf("blob count and hashes count mismatch: "+ - "got %v blobs, %v hashes", len(blobs), len(hashes)) - } - - // Verify tree exists - if !t.treeExists(treeID) { - return nil, errRecordNotFound - } - - // Verify tree is not frozen - leavesAll, err := t.trillian.leavesAll(treeID) - if err != nil { - return nil, fmt.Errorf("leavesAll: %v", err) - } - idx, err := t.recordIndexLatest(leavesAll) - if err != nil { - return nil, err - } - if idx.Frozen { - return nil, errTreeIsFrozen - } - - // Encrypt blobs if specified - if encrypt { - for k, v := range blobs { - e, err := t.encrypt(v) - if err != nil { - return nil, err - } - blobs[k] = e - } - } - - // Save blobs to store - keys, err := t.store.Put(blobs) - if err != nil { - return nil, fmt.Errorf("store Put: %v", err) - } - if len(keys) != len(blobs) { - return nil, fmt.Errorf("wrong number of keys: got %v, want %v", - len(keys), len(blobs)) - } - - // Prepare log leaves. hashes and keys share the same ordering. - leaves := make([]*trillian.LogLeaf, 0, len(blobs)) - for k := range blobs { - pk := []byte(keyPrefix + keys[k]) - leaves = append(leaves, newLogLeaf(hashes[k], pk)) - } - - // Append leaves to trillian tree - queued, _, err := t.trillian.leavesAppend(treeID, leaves) - if err != nil { - return nil, fmt.Errorf("leavesAppend: %v", err) - } - if len(queued) != len(leaves) { - return nil, fmt.Errorf("wrong number of queued leaves: got %v, want %v", - len(queued), len(leaves)) - } - failed := make([]string, 0, len(queued)) - for _, v := range queued { - c := codes.Code(v.QueuedLeaf.GetStatus().GetCode()) - if c != codes.OK { - failed = append(failed, fmt.Sprintf("%v", c)) - } - } - if len(failed) > 0 { - return nil, fmt.Errorf("append leaves failed: %v", failed) - } - - // Parse and return the merkle leaf hashes - merkles := make([][]byte, 0, len(blobs)) - for _, v := range queued { - merkles = append(merkles, v.QueuedLeaf.Leaf.MerkleLeafHash) - } - - return merkles, nil -} - -// del deletes the blobs in the kv store that correspond to the provided merkle -// leaf hashes. The kv store keys in store in the ExtraData field of the leaves -// specified by the provided merkle leaf hashes. -func (t *tlog) blobsDel(treeID int64, merkles [][]byte) error { - log.Tracef("%v blobsDel: %v", t.id, treeID) - - // Verify tree exists. We allow blobs to be deleted from both - // frozen and non frozen trees. - if !t.treeExists(treeID) { - return errRecordNotFound - } - - // Get all tree leaves - leaves, err := t.trillian.leavesAll(treeID) - if err != nil { - return err - } - - // Put merkle leaf hashes into a map so that we can tell if a leaf - // corresponds to one of the target merkle leaf hashes in O(n) - // time. - merkleHashes := make(map[string]struct{}, len(leaves)) - for _, v := range merkles { - merkleHashes[hex.EncodeToString(v)] = struct{}{} - } - - // Aggregate the key-value store keys for the provided merkle leaf - // hashes. - keys := make([]string, 0, len(merkles)) - for _, v := range leaves { - _, ok := merkleHashes[hex.EncodeToString(v.MerkleLeafHash)] - if ok { - keys = append(keys, extractKeyFromLeaf(v)) - } - } - - // Delete file blobs from the store - err = t.store.Del(keys) - if err != nil { - return fmt.Errorf("store Del: %v", err) - } - - return nil -} - -// blobsByMerkle returns the blobs with the provided merkle leaf hashes. -// -// If a blob does not exist it will not be included in the returned map. It is -// the responsibility of the caller to check that a blob is returned for each -// of the provided merkle hashes. -func (t *tlog) blobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, error) { - log.Tracef("%v blobsByMerkle: %v", t.id, treeID) - - // Verify tree exists - if !t.treeExists(treeID) { - return nil, errRecordNotFound - } - - // Get leaves - leavesAll, err := t.trillian.leavesAll(treeID) - if err != nil { - return nil, fmt.Errorf("leavesAll: %v", err) - } - - // Aggregate the leaves that correspond to the provided merkle - // hashes. - // map[merkleHash]*trillian.LogLeaf - leaves := make(map[string]*trillian.LogLeaf, len(merkles)) - for _, v := range merkles { - leaves[hex.EncodeToString(v)] = nil - } - for _, v := range leavesAll { - m := hex.EncodeToString(v.MerkleLeafHash) - if _, ok := leaves[m]; ok { - leaves[m] = v - } - } - - // Ensure a leaf was found for all provided merkle hashes - for k, v := range leaves { - if v == nil { - return nil, fmt.Errorf("leaf not found for merkle hash: %v", k) - } - } - - // Extract the key-value store keys. These keys MUST be put in the - // same order that the merkle hashes were provided in. - keys := make([]string, 0, len(leaves)) - for _, v := range merkles { - l, ok := leaves[hex.EncodeToString(v)] - if !ok { - return nil, fmt.Errorf("leaf not found for merkle hash: %x", v) - } - keys = append(keys, extractKeyFromLeaf(l)) - } - - // Pull the blobs from the store. If is ok if one or more blobs is - // not found. It is the responsibility of the caller to decide how - // this should be handled. - blobs, err := t.store.Get(keys) - if err != nil { - return nil, fmt.Errorf("store Get: %v", err) - } - - // Decrypt any encrypted blobs - for k, v := range blobs { - if blobIsEncrypted(v) { - b, _, err := t.encryptionKey.decrypt(v) - if err != nil { - return nil, err - } - blobs[k] = b - } - } - - // Put blobs in a map so the caller can determine if any of the - // provided merkle hashes did not correspond to a blob in the - // store. - b := make(map[string][]byte, len(blobs)) // [merkleHash]blob - for k, v := range keys { - // The merkle hashes slice and keys slice share the same order - merkleHash := hex.EncodeToString(merkles[k]) - blob, ok := blobs[v] - if !ok { - return nil, fmt.Errorf("blob not found for key %v", v) - } - b[merkleHash] = blob - } - - return b, nil -} - -func (t *tlog) merklesByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) { - log.Tracef("%v merklesByKeyPrefix: %v %v", t.id, treeID, keyPrefix) - - // Verify tree exists - if !t.treeExists(treeID) { - return nil, errRecordNotFound - } - - // Get leaves - leaves, err := t.trillian.leavesAll(treeID) - if err != nil { - return nil, fmt.Errorf("leavesAll: %v", err) - } - - // Walk leaves and aggregate the merkle leaf hashes with a matching - // key prefix. - merkles := make([][]byte, 0, len(leaves)) - for _, v := range leaves { - if bytes.HasPrefix(v.ExtraData, []byte(keyPrefix)) { - merkles = append(merkles, v.MerkleLeafHash) - } - } - - return merkles, nil -} - -// blobsByKeyPrefix returns all blobs that match the provided key prefix. -func (t *tlog) blobsByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) { - log.Tracef("%v blobsByKeyPrefix: %v %v", t.id, treeID, keyPrefix) - - // Verify tree exists - if !t.treeExists(treeID) { - return nil, errRecordNotFound - } - - // Get leaves - leaves, err := t.trillian.leavesAll(treeID) - if err != nil { - return nil, fmt.Errorf("leavesAll: %v", err) - } - - // Walk leaves and aggregate the key-value store keys for all - // leaves with a matching key prefix. - keys := make([]string, 0, len(leaves)) - for _, v := range leaves { - if bytes.HasPrefix(v.ExtraData, []byte(keyPrefix)) { - keys = append(keys, extractKeyFromLeaf(v)) - } - } - - // Pull the blobs from the store - blobs, err := t.store.Get(keys) - if err != nil { - return nil, fmt.Errorf("store Get: %v", err) - } - if len(blobs) != len(keys) { - // One or more blobs were not found - missing := make([]string, 0, len(keys)) - for _, v := range keys { - _, ok := blobs[v] - if !ok { - missing = append(missing, v) - } - } - return nil, fmt.Errorf("blobs not found: %v", missing) - } - - // Decrypt any encrypted blobs - for k, v := range blobs { - if blobIsEncrypted(v) { - b, _, err := t.encryptionKey.decrypt(v) - if err != nil { - return nil, err - } - blobs[k] = b - } - } - - // Covert blobs from map to slice - b := make([][]byte, 0, len(blobs)) - for _, v := range blobs { - b = append(b, v) - } - - return b, nil -} - // TODO run fsck episodically -func (t *tlog) fsck() { +func (t *Tlog) Fsck() { // Set tree status to frozen for any trees that are frozen and have // been anchored one last time. // Failed censor. Ensure all blobs have been deleted from all // record versions of a censored record. } -func (t *tlog) close() { - log.Tracef("%v close", t.id) +func (t *Tlog) Close() { + log.Tracef("%v Close", t.id) // Close connections t.store.Close() @@ -2026,7 +1736,7 @@ func (t *tlog) close() { } } -func newTlog(id, homeDir, dataDir, trillianHost, trillianKeyFile, encryptionKeyFile, dcrtimeHost, dcrtimeCert string) (*tlog, error) { +func New(id, homeDir, dataDir, trillianHost, trillianKeyFile, encryptionKeyFile, dcrtimeHost, dcrtimeCert string) (*Tlog, error) { // Load encryption key if provided. An encryption key is optional. var ek *encryptionKey if encryptionKeyFile != "" { @@ -2078,7 +1788,7 @@ func newTlog(id, homeDir, dataDir, trillianHost, trillianKeyFile, encryptionKeyF } // Setup tlog - t := tlog{ + t := Tlog{ id: id, trillian: trillianClient, store: store, diff --git a/politeiad/backend/tlogbe/tlog/tlogclient.go b/politeiad/backend/tlogbe/tlog/tlogclient.go new file mode 100644 index 000000000..3a65e4806 --- /dev/null +++ b/politeiad/backend/tlogbe/tlog/tlogclient.go @@ -0,0 +1,350 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlog + +import ( + "bytes" + "encoding/hex" + "fmt" + + "github.com/decred/politeia/politeiad/backend" + "github.com/google/trillian" + "google.golang.org/grpc/codes" +) + +// BlobsSave saves the provided blobs to the tlog backend. Note, hashes +// contains the hashes of the data encoded in the blobs. The hashes must share +// the same ordering as the blobs. +// +// This function satisfies the plugins.TlogClient interface. +func (t *Tlog) BlobsSave(treeID int64, keyPrefix string, blobs, hashes [][]byte, encrypt bool) ([][]byte, error) { + log.Tracef("%v BlobsSave: %v %v %v", t.id, treeID, keyPrefix, encrypt) + + // Sanity check + if len(blobs) != len(hashes) { + return nil, fmt.Errorf("blob count and hashes count mismatch: "+ + "got %v blobs, %v hashes", len(blobs), len(hashes)) + } + + // Verify tree exists + if !t.TreeExists(treeID) { + return nil, ErrRecordNotFound + } + + // Verify tree is not frozen + leavesAll, err := t.trillian.leavesAll(treeID) + if err != nil { + return nil, fmt.Errorf("leavesAll: %v", err) + } + idx, err := t.recordIndexLatest(leavesAll) + if err != nil { + return nil, err + } + if idx.Frozen { + return nil, ErrTreeIsFrozen + } + + // Encrypt blobs if specified + if encrypt { + for k, v := range blobs { + e, err := t.encrypt(v) + if err != nil { + return nil, err + } + blobs[k] = e + } + } + + // Save blobs to store + keys, err := t.store.Put(blobs) + if err != nil { + return nil, fmt.Errorf("store Put: %v", err) + } + if len(keys) != len(blobs) { + return nil, fmt.Errorf("wrong number of keys: got %v, want %v", + len(keys), len(blobs)) + } + + // Prepare log leaves. hashes and keys share the same ordering. + leaves := make([]*trillian.LogLeaf, 0, len(blobs)) + for k := range blobs { + pk := []byte(keyPrefix + keys[k]) + leaves = append(leaves, newLogLeaf(hashes[k], pk)) + } + + // Append leaves to trillian tree + queued, _, err := t.trillian.leavesAppend(treeID, leaves) + if err != nil { + return nil, fmt.Errorf("leavesAppend: %v", err) + } + if len(queued) != len(leaves) { + return nil, fmt.Errorf("wrong number of queued leaves: got %v, want %v", + len(queued), len(leaves)) + } + failed := make([]string, 0, len(queued)) + for _, v := range queued { + c := codes.Code(v.QueuedLeaf.GetStatus().GetCode()) + if c != codes.OK { + failed = append(failed, fmt.Sprintf("%v", c)) + } + } + if len(failed) > 0 { + return nil, fmt.Errorf("append leaves failed: %v", failed) + } + + // Parse and return the merkle leaf hashes + merkles := make([][]byte, 0, len(blobs)) + for _, v := range queued { + merkles = append(merkles, v.QueuedLeaf.Leaf.MerkleLeafHash) + } + + return merkles, nil +} + +// BlobsDel deletes the blobs in the kv store that correspond to the provided +// merkle leaf hashes. The kv store keys in store in the ExtraData field of the +// leaves specified by the provided merkle leaf hashes. +// +// This function satisfies the plugins.TlogClient interface. +func (t *Tlog) BlobsDel(treeID int64, merkles [][]byte) error { + log.Tracef("%v BlobsDel: %v", t.id, treeID) + + // Verify tree exists. We allow blobs to be deleted from both + // frozen and non frozen trees. + if !t.TreeExists(treeID) { + return ErrRecordNotFound + } + + // Get all tree leaves + leaves, err := t.trillian.leavesAll(treeID) + if err != nil { + return err + } + + // Put merkle leaf hashes into a map so that we can tell if a leaf + // corresponds to one of the target merkle leaf hashes in O(n) + // time. + merkleHashes := make(map[string]struct{}, len(leaves)) + for _, v := range merkles { + merkleHashes[hex.EncodeToString(v)] = struct{}{} + } + + // Aggregate the key-value store keys for the provided merkle leaf + // hashes. + keys := make([]string, 0, len(merkles)) + for _, v := range leaves { + _, ok := merkleHashes[hex.EncodeToString(v.MerkleLeafHash)] + if ok { + keys = append(keys, extractKeyFromLeaf(v)) + } + } + + // Delete file blobs from the store + err = t.store.Del(keys) + if err != nil { + return fmt.Errorf("store Del: %v", err) + } + + return nil +} + +// BlobsByMerkle returns the blobs with the provided merkle leaf hashes. +// +// If a blob does not exist it will not be included in the returned map. It is +// the responsibility of the caller to check that a blob is returned for each +// of the provided merkle hashes. +// +// This function satisfies the plugins.TlogClient interface. +func (t *Tlog) BlobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, error) { + log.Tracef("%v BlobsByMerkle: %v", t.id, treeID) + + // Verify tree exists + if !t.TreeExists(treeID) { + return nil, ErrRecordNotFound + } + + // Get leaves + leavesAll, err := t.trillian.leavesAll(treeID) + if err != nil { + return nil, fmt.Errorf("leavesAll: %v", err) + } + + // Aggregate the leaves that correspond to the provided merkle + // hashes. + // map[merkleHash]*trillian.LogLeaf + leaves := make(map[string]*trillian.LogLeaf, len(merkles)) + for _, v := range merkles { + leaves[hex.EncodeToString(v)] = nil + } + for _, v := range leavesAll { + m := hex.EncodeToString(v.MerkleLeafHash) + if _, ok := leaves[m]; ok { + leaves[m] = v + } + } + + // Ensure a leaf was found for all provided merkle hashes + for k, v := range leaves { + if v == nil { + return nil, fmt.Errorf("leaf not found for merkle hash: %v", k) + } + } + + // Extract the key-value store keys. These keys MUST be put in the + // same order that the merkle hashes were provided in. + keys := make([]string, 0, len(leaves)) + for _, v := range merkles { + l, ok := leaves[hex.EncodeToString(v)] + if !ok { + return nil, fmt.Errorf("leaf not found for merkle hash: %x", v) + } + keys = append(keys, extractKeyFromLeaf(l)) + } + + // Pull the blobs from the store. If is ok if one or more blobs is + // not found. It is the responsibility of the caller to decide how + // this should be handled. + blobs, err := t.store.Get(keys) + if err != nil { + return nil, fmt.Errorf("store Get: %v", err) + } + + // Decrypt any encrypted blobs + for k, v := range blobs { + if blobIsEncrypted(v) { + b, _, err := t.encryptionKey.decrypt(v) + if err != nil { + return nil, err + } + blobs[k] = b + } + } + + // Put blobs in a map so the caller can determine if any of the + // provided merkle hashes did not correspond to a blob in the + // store. + b := make(map[string][]byte, len(blobs)) // [merkleHash]blob + for k, v := range keys { + // The merkle hashes slice and keys slice share the same order + merkleHash := hex.EncodeToString(merkles[k]) + blob, ok := blobs[v] + if !ok { + return nil, fmt.Errorf("blob not found for key %v", v) + } + b[merkleHash] = blob + } + + return b, nil +} + +// MerklesByKeyPrefix returns the merkle leaf hashes for all blobs that match +// the key prefix. +// +// This function satisfies the plugins.TlogClient interface. +func (t *Tlog) MerklesByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) { + log.Tracef("%v MerklesByKeyPrefix: %v %v", t.id, treeID, keyPrefix) + + // Verify tree exists + if !t.TreeExists(treeID) { + return nil, ErrRecordNotFound + } + + // Get leaves + leaves, err := t.trillian.leavesAll(treeID) + if err != nil { + return nil, fmt.Errorf("leavesAll: %v", err) + } + + // Walk leaves and aggregate the merkle leaf hashes with a matching + // key prefix. + merkles := make([][]byte, 0, len(leaves)) + for _, v := range leaves { + if bytes.HasPrefix(v.ExtraData, []byte(keyPrefix)) { + merkles = append(merkles, v.MerkleLeafHash) + } + } + + return merkles, nil +} + +// BlobsByKeyPrefix returns all blobs that match the provided key prefix. +// +// This function satisfies the plugins.TlogClient interface. +func (t *Tlog) BlobsByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) { + log.Tracef("%v BlobsByKeyPrefix: %v %v", t.id, treeID, keyPrefix) + + // Verify tree exists + if !t.TreeExists(treeID) { + return nil, ErrRecordNotFound + } + + // Get leaves + leaves, err := t.trillian.leavesAll(treeID) + if err != nil { + return nil, fmt.Errorf("leavesAll: %v", err) + } + + // Walk leaves and aggregate the key-value store keys for all + // leaves with a matching key prefix. + keys := make([]string, 0, len(leaves)) + for _, v := range leaves { + if bytes.HasPrefix(v.ExtraData, []byte(keyPrefix)) { + keys = append(keys, extractKeyFromLeaf(v)) + } + } + + // Pull the blobs from the store + blobs, err := t.store.Get(keys) + if err != nil { + return nil, fmt.Errorf("store Get: %v", err) + } + if len(blobs) != len(keys) { + // One or more blobs were not found + missing := make([]string, 0, len(keys)) + for _, v := range keys { + _, ok := blobs[v] + if !ok { + missing = append(missing, v) + } + } + return nil, fmt.Errorf("blobs not found: %v", missing) + } + + // Decrypt any encrypted blobs + for k, v := range blobs { + if blobIsEncrypted(v) { + b, _, err := t.encryptionKey.decrypt(v) + if err != nil { + return nil, err + } + blobs[k] = b + } + } + + // Covert blobs from map to slice + b := make([][]byte, 0, len(blobs)) + for _, v := range blobs { + b = append(b, v) + } + + return b, nil +} + +// Timestamp returns the timestamp for the data blob that corresponds to the +// provided merkle leaf hash. +// +// This function satisfies the plugins.TlogClient interface. +func (t *Tlog) Timestamp(treeID int64, merkleLeafHash []byte) (*backend.Timestamp, error) { + log.Tracef("%v Timestamp: %v %x", t.id, treeID, merkleLeafHash) + + // Get all tree leaves + leaves, err := t.trillian.leavesAll(treeID) + if err != nil { + return nil, fmt.Errorf("leavesAll: %v", err) + } + + // Get timestamp + return t.timestamp(treeID, merkleLeafHash, leaves) +} diff --git a/politeiad/backend/tlogbe/trillianclient.go b/politeiad/backend/tlogbe/tlog/trillianclient.go similarity index 97% rename from politeiad/backend/tlogbe/trillianclient.go rename to politeiad/backend/tlogbe/tlog/trillianclient.go index 5268b5965..92fe0f171 100644 --- a/politeiad/backend/tlogbe/trillianclient.go +++ b/politeiad/backend/tlogbe/tlog/trillianclient.go @@ -1,8 +1,8 @@ -// Copyright (c) 2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlogbe +package tlog import ( "bytes" @@ -721,3 +721,25 @@ func (t *testTClient) inclusionProof(treeID int64, merkleLeafHash []byte, lr *ty // // This function satisfies the trillianClient interface. func (t *testTClient) close() {} + +// newTestTClient returns a new testTClient. +func newTestTClient() (*testTClient, error) { + // Create trillian private key + key, err := keys.NewFromSpec(&keyspb.Specification{ + Params: &keyspb.Specification_EcdsaParams{}, + }) + if err != nil { + return nil, err + } + keyDer, err := der.MarshalPrivateKey(key) + if err != nil { + return nil, err + } + return &testTClient{ + trees: make(map[int64]*trillian.Tree), + leaves: make(map[int64][]*trillian.LogLeaf), + privateKey: &keyspb.PrivateKey{ + Der: keyDer, + }, + }, nil +} diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 566d5c3af..dffb0e834 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -10,6 +10,7 @@ import ( "encoding/base64" "encoding/binary" "encoding/hex" + "encoding/json" "errors" "fmt" "io/ioutil" @@ -24,21 +25,18 @@ import ( v1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/mime" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/plugins/comments" - "github.com/decred/politeia/politeiad/plugins/dcrdata" - "github.com/decred/politeia/politeiad/plugins/pi" - "github.com/decred/politeia/politeiad/plugins/ticketvote" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" + "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" "github.com/decred/politeia/util" "github.com/marcopeereboom/sbox" "github.com/subosito/gozaru" ) -// TODO testnet vs mainnet trillian databases. Can I also make different dbs -// for pi and cms so we can switch back and forth without issues? +// TODO testnet vs mainnet trillian databases // TODO fsck +// TODO move memory caches to filesystem const ( - defaultTrillianKeyFilename = "trillian.key" defaultEncryptionKeyFilename = "tlogbe.key" // Tlog instance IDs @@ -50,10 +48,6 @@ const ( // the plugins for storing plugin data. pluginDataDirname = "plugins" - // pluginSettingDataDir is the PluginSetting key for the plugin - // data directory. - pluginSettingDataDir = "datadir" - // Record states stateUnvetted = "unvetted" stateVetted = "vetted" @@ -61,46 +55,17 @@ const ( var ( _ backend.Backend = (*tlogBackend)(nil) - - // statusChanges contains the allowed record status changes. If - // statusChanges[currentStatus][newStatus] exists then the status - // change is allowed. - // - // Note, the tlog backend does not make use of the status - // MDStatusIterationUnvetted. The original purpose of this status - // was to show when an unvetted record had been altered since - // unvetted records were not versioned in the git backend. The tlog - // backend versions unvetted records and thus does not need to use - // this additional status. - statusChanges = map[backend.MDStatusT]map[backend.MDStatusT]struct{}{ - // Unvetted status changes - backend.MDStatusUnvetted: { - backend.MDStatusVetted: {}, - backend.MDStatusCensored: {}, - }, - - // Vetted status changes - backend.MDStatusVetted: { - backend.MDStatusCensored: {}, - backend.MDStatusArchived: {}, - }, - - // Statuses that do not allow any further transitions - backend.MDStatusCensored: {}, - backend.MDStatusArchived: {}, - } ) -// tlogBackend implements the Backend interface. +// tlogBackend implements the backend.Backend interface. type tlogBackend struct { sync.RWMutex activeNetParams *chaincfg.Params homeDir string dataDir string shutdown bool - unvetted *tlog - vetted *tlog - plugins map[string]plugin // [pluginID]plugin + unvetted *tlog.Tlog + vetted *tlog.Tlog // prefixes contains the prefix to full token mapping for unvetted // records. The prefix is the first n characters of the hex encoded @@ -130,14 +95,6 @@ type recordInventory struct { vetted map[backend.MDStatusT][]string } -// plugin represents a tlogbe plugin. -type plugin struct { - id string - version string - settings []backend.PluginSetting - client pluginClient -} - func tokenIsFullLength(token []byte) bool { return len(token) == v1.TokenSizeTlog } @@ -295,7 +252,7 @@ func (t *tlogBackend) vettedTreeIDFromToken(token []byte) (int64, bool) { // unvetted record has a tree pointer. The tree pointer will be // the vetted tree ID. unvettedTreeID := t.unvettedTreeIDFromToken(token) - vettedTreeID, ok = t.unvetted.treePointer(unvettedTreeID) + vettedTreeID, ok = t.unvetted.TreePointer(unvettedTreeID) if !ok { // No tree pointer. This record either doesn't exist or is an // unvetted record. @@ -303,7 +260,7 @@ func (t *tlogBackend) vettedTreeIDFromToken(token []byte) (int64, bool) { } // Verify the vetted tree exists. This should not fail. - if !t.vetted.treeExists(vettedTreeID) { + if !t.vetted.TreeExists(vettedTreeID) { // We're in trouble! e := fmt.Sprintf("freeze record of unvetted tree %v points to "+ "an invalid vetted tree %v", unvettedTreeID, vettedTreeID) @@ -602,6 +559,36 @@ func verifyContent(metadata []backend.MetadataStream, files []backend.File, file return nil } +var ( + // statusChanges contains the allowed record status changes. If + // statusChanges[currentStatus][newStatus] exists then the status + // change is allowed. + // + // Note, the tlog backend does not make use of the status + // MDStatusIterationUnvetted. The original purpose of this status + // was to show when an unvetted record had been altered since + // unvetted records were not versioned in the git backend. The tlog + // backend versions unvetted records and thus does not need to use + // this additional status. + statusChanges = map[backend.MDStatusT]map[backend.MDStatusT]struct{}{ + // Unvetted status changes + backend.MDStatusUnvetted: { + backend.MDStatusVetted: {}, + backend.MDStatusCensored: {}, + }, + + // Vetted status changes + backend.MDStatusVetted: { + backend.MDStatusCensored: {}, + backend.MDStatusArchived: {}, + }, + + // Statuses that do not allow any further transitions + backend.MDStatusCensored: {}, + backend.MDStatusArchived: {}, + } +) + // statusChangeIsAllowed returns whether the provided status change is allowed // by tlogbe. func statusChangeIsAllowed(from, to backend.MDStatusT) bool { @@ -720,9 +707,10 @@ func (t *tlogBackend) isShutdown() bool { return t.shutdown } -// New satisfies the Backend interface. +// New submites a new record. Records are considered unvetted until their +// status is changed to a public status. // -// This function satisfies the Backend interface. +// This function satisfies the backend.Backend interface. func (t *tlogBackend) New(metadata []backend.MetadataStream, files []backend.File) (*backend.RecordMetadata, error) { log.Tracef("New") @@ -733,15 +721,15 @@ func (t *tlogBackend) New(metadata []backend.MetadataStream, files []backend.Fil } // Call pre plugin hooks - hnr := hookNewRecord{ + hnr := plugins.HookNewRecord{ Metadata: metadata, Files: files, } - b, err := encodeHookNewRecord(hnr) + b, err := json.Marshal(hnr) if err != nil { return nil, err } - err = t.pluginHookPre(hookNewRecordPre, string(b)) + err = t.unvetted.PluginHookPre(plugins.HookNewRecordPre, string(b)) if err != nil { return nil, err } @@ -750,7 +738,7 @@ func (t *tlogBackend) New(metadata []backend.MetadataStream, files []backend.Fil var token []byte var treeID int64 for retries := 0; retries < 10; retries++ { - treeID, err = t.unvetted.treeNew() + treeID, err = t.unvetted.TreeNew() if err != nil { return nil, err } @@ -779,22 +767,22 @@ func (t *tlogBackend) New(metadata []backend.MetadataStream, files []backend.Fil } // Save the record - err = t.unvetted.recordSave(treeID, *rm, metadata, files) + err = t.unvetted.RecordSave(treeID, *rm, metadata, files) if err != nil { - return nil, fmt.Errorf("recordSave %x: %v", token, err) + return nil, fmt.Errorf("RecordSave %x: %v", token, err) } // Call post plugin hooks - hnr = hookNewRecord{ + hnr = plugins.HookNewRecord{ Metadata: metadata, Files: files, RecordMetadata: rm, } - b, err = encodeHookNewRecord(hnr) + b, err = json.Marshal(hnr) if err != nil { return nil, err } - t.pluginHookPost(hookNewRecordPost, string(b)) + t.unvetted.PluginHookPost(plugins.HookNewRecordPost, string(b)) // Update the inventory cache t.inventoryAdd(stateUnvetted, token, backend.MDStatusUnvetted) @@ -802,7 +790,7 @@ func (t *tlogBackend) New(metadata []backend.MetadataStream, files []backend.Fil return rm, nil } -// This function satisfies the Backend interface. +// This function satisfies the backend.Backend interface. func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { log.Tracef("UpdateUnvettedRecord: %x", token) @@ -846,9 +834,9 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ // Get existing record treeID := treeIDFromToken(token) - r, err := t.unvetted.recordLatest(treeID) + r, err := t.unvetted.RecordLatest(treeID) if err != nil { - return nil, fmt.Errorf("recordLatest: %v", err) + return nil, fmt.Errorf("RecordLatest: %v", err) } // Apply changes @@ -863,7 +851,7 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ } // Call pre plugin hooks - her := hookEditRecord{ + her := plugins.HookEditRecord{ Current: *r, RecordMetadata: *recordMD, MDAppend: mdAppend, @@ -871,45 +859,40 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ FilesAdd: filesAdd, FilesDel: filesDel, } - b, err := encodeHookEditRecord(her) + b, err := json.Marshal(her) if err != nil { return nil, err } - err = t.pluginHookPre(hookEditRecordPre, string(b)) + err = t.unvetted.PluginHookPre(plugins.HookEditRecordPre, string(b)) if err != nil { return nil, err } // Save record - err = t.unvetted.recordSave(treeID, *recordMD, metadata, files) + err = t.unvetted.RecordSave(treeID, *recordMD, metadata, files) if err != nil { switch err { - case errTreeIsFrozen: + case tlog.ErrTreeIsFrozen: return nil, backend.ErrRecordLocked - case errNoFileChanges: + case tlog.ErrNoFileChanges: return nil, backend.ErrNoChanges } - return nil, fmt.Errorf("recordSave: %v", err) + return nil, fmt.Errorf("RecordSave: %v", err) } // Call post plugin hooks - t.pluginHookPost(hookEditRecordPost, string(b)) - if err != nil { - e := fmt.Sprintf("UpdateUnvettedRecord %x: pluginHook editRecordPost: %v", - token, err) - panic(e) - } + t.unvetted.PluginHookPost(plugins.HookEditRecordPost, string(b)) // Return updated record - r, err = t.unvetted.recordLatest(treeID) + r, err = t.unvetted.RecordLatest(treeID) if err != nil { - return nil, fmt.Errorf("recordLatest: %v", err) + return nil, fmt.Errorf("RecordLatest: %v", err) } return r, nil } -// This function satisfies the Backend interface. +// This function satisfies the backend.Backend interface. func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { log.Tracef("UpdateVettedRecord: %x", token) @@ -953,12 +936,12 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b } // Get existing record - r, err := t.vetted.recordLatest(treeID) + r, err := t.vetted.RecordLatest(treeID) if err != nil { - if errors.Is(err, errRecordNotFound) { + if errors.Is(err, tlog.ErrRecordNotFound) { return nil, backend.ErrRecordNotFound } - return nil, fmt.Errorf("recordLatest: %v", err) + return nil, fmt.Errorf("RecordLatest: %v", err) } // Apply changes @@ -973,7 +956,7 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b } // Call pre plugin hooks - her := hookEditRecord{ + her := plugins.HookEditRecord{ Current: *r, RecordMetadata: *recordMD, MDAppend: mdAppend, @@ -981,40 +964,40 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b FilesAdd: filesAdd, FilesDel: filesDel, } - b, err := encodeHookEditRecord(her) + b, err := json.Marshal(her) if err != nil { return nil, err } - err = t.pluginHookPre(hookEditRecordPre, string(b)) + err = t.vetted.PluginHookPre(plugins.HookEditRecordPre, string(b)) if err != nil { return nil, err } // Save record - err = t.vetted.recordSave(treeID, *recordMD, metadata, files) + err = t.vetted.RecordSave(treeID, *recordMD, metadata, files) if err != nil { switch err { - case errTreeIsFrozen: + case tlog.ErrTreeIsFrozen: return nil, backend.ErrRecordLocked - case errNoFileChanges: + case tlog.ErrNoFileChanges: return nil, backend.ErrNoChanges } - return nil, fmt.Errorf("recordSave: %v", err) + return nil, fmt.Errorf("RecordSave: %v", err) } // Call post plugin hooks - t.pluginHookPost(hookEditRecordPost, string(b)) + t.vetted.PluginHookPost(plugins.HookEditRecordPost, string(b)) // Return updated record - r, err = t.vetted.recordLatest(treeID) + r, err = t.vetted.RecordLatest(treeID) if err != nil { - return nil, fmt.Errorf("recordLatest: %v", err) + return nil, fmt.Errorf("RecordLatest: %v", err) } return r, nil } -// This function satisfies the Backend interface. +// This function satisfies the backend.Backend interface. func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite []backend.MetadataStream) error { // Validate record contents. Send in a single metadata array to // verify there are no dups. @@ -1061,22 +1044,22 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite // Get existing record treeID := treeIDFromToken(token) - r, err := t.unvetted.recordLatest(treeID) + r, err := t.unvetted.RecordLatest(treeID) if err != nil { - return fmt.Errorf("recordLatest: %v", err) + return fmt.Errorf("RecordLatest: %v", err) } // Call pre plugin hooks - hem := hookEditMetadata{ + hem := plugins.HookEditMetadata{ Current: *r, MDAppend: mdAppend, MDOverwrite: mdOverwrite, } - b, err := encodeHookEditMetadata(hem) + b, err := json.Marshal(hem) if err != nil { return err } - err = t.pluginHookPre(hookEditMetadataPre, string(b)) + err = t.unvetted.PluginHookPre(plugins.HookEditMetadataPre, string(b)) if err != nil { return err } @@ -1085,24 +1068,24 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) // Update metadata - err = t.unvetted.recordMetadataSave(treeID, r.RecordMetadata, metadata) + err = t.unvetted.RecordMetadataSave(treeID, r.RecordMetadata, metadata) if err != nil { switch err { - case errTreeIsFrozen: + case tlog.ErrTreeIsFrozen: return backend.ErrRecordLocked - case errNoMetadataChanges: + case tlog.ErrNoMetadataChanges: return backend.ErrNoChanges } return err } // Call post plugin hooks - t.pluginHookPost(hookEditMetadataPost, string(b)) + t.unvetted.PluginHookPost(plugins.HookEditMetadataPost, string(b)) return nil } -// This function satisfies the Backend interface. +// This function satisfies the backend.Backend interface. func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite []backend.MetadataStream) error { log.Tracef("UpdateVettedMetadata: %x", token) @@ -1151,25 +1134,25 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ } // Get existing record - r, err := t.vetted.recordLatest(treeID) + r, err := t.vetted.RecordLatest(treeID) if err != nil { - if errors.Is(err, errRecordNotFound) { + if errors.Is(err, tlog.ErrRecordNotFound) { return backend.ErrRecordNotFound } - return fmt.Errorf("recordLatest: %v", err) + return fmt.Errorf("RecordLatest: %v", err) } // Call pre plugin hooks - hem := hookEditMetadata{ + hem := plugins.HookEditMetadata{ Current: *r, MDAppend: mdAppend, MDOverwrite: mdOverwrite, } - b, err := encodeHookEditMetadata(hem) + b, err := json.Marshal(hem) if err != nil { return err } - err = t.pluginHookPre(hookEditMetadataPre, string(b)) + err = t.vetted.PluginHookPre(plugins.HookEditMetadataPre, string(b)) if err != nil { return err } @@ -1178,19 +1161,19 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) // Update metadata - err = t.vetted.recordMetadataSave(treeID, r.RecordMetadata, metadata) + err = t.vetted.RecordMetadataSave(treeID, r.RecordMetadata, metadata) if err != nil { switch err { - case errTreeIsFrozen: + case tlog.ErrTreeIsFrozen: return backend.ErrRecordLocked - case errNoMetadataChanges: + case tlog.ErrNoMetadataChanges: return backend.ErrNoChanges } - return fmt.Errorf("recordMetadataSave: %v", err) + return fmt.Errorf("RecordMetadataSave: %v", err) } // Call post plugin hooks - t.pluginHookPost(hookEditMetadataPost, string(b)) + t.vetted.PluginHookPost(plugins.HookEditMetadataPost, string(b)) return nil } @@ -1198,7 +1181,7 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ // UnvettedExists returns whether the provided token corresponds to an unvetted // record. // -// This function satisfies the Backend interface. +// This function satisfies the backend.Backend interface. func (t *tlogBackend) UnvettedExists(token []byte) bool { log.Tracef("UnvettedExists %x", token) @@ -1212,10 +1195,10 @@ func (t *tlogBackend) UnvettedExists(token []byte) bool { // Check for unvetted record treeID := t.unvettedTreeIDFromToken(token) - return t.unvetted.recordExists(treeID) + return t.unvetted.RecordExists(treeID) } -// This function satisfies the Backend interface. +// This function satisfies the backend.Backend interface. func (t *tlogBackend) VettedExists(token []byte) bool { log.Tracef("VettedExists %x", token) @@ -1223,7 +1206,7 @@ func (t *tlogBackend) VettedExists(token []byte) bool { return ok } -// This function satisfies the Backend interface. +// This function satisfies the backend.Backend interface. func (t *tlogBackend) GetUnvetted(token []byte, version string) (*backend.Record, error) { log.Tracef("GetUnvetted: %x %v", token, version) @@ -1247,7 +1230,7 @@ func (t *tlogBackend) GetUnvetted(token []byte, version string) (*backend.Record } // Get unvetted record - r, err := t.unvetted.record(treeID, v) + r, err := t.unvetted.Record(treeID, v) if err != nil { return nil, fmt.Errorf("unvetted record: %v", err) } @@ -1255,7 +1238,7 @@ func (t *tlogBackend) GetUnvetted(token []byte, version string) (*backend.Record return r, nil } -// This function satisfies the Backend interface. +// This function satisfies the backend.Backend interface. func (t *tlogBackend) GetVetted(token []byte, version string) (*backend.Record, error) { log.Tracef("GetVetted: %x %v", token, version) @@ -1279,9 +1262,9 @@ func (t *tlogBackend) GetVetted(token []byte, version string) (*backend.Record, v = uint32(u) } - r, err := t.vetted.record(treeID, v) + r, err := t.vetted.Record(treeID, v) if err != nil { - if errors.Is(err, errRecordNotFound) { + if errors.Is(err, tlog.ErrRecordNotFound) { err = backend.ErrRecordNotFound } return nil, err @@ -1290,7 +1273,7 @@ func (t *tlogBackend) GetVetted(token []byte, version string) (*backend.Record, return r, nil } -// This function satisfies the Backend interface. +// This function satisfies the backend.Backend interface. func (t *tlogBackend) GetUnvettedTimestamps(token []byte, version string) (*backend.RecordTimestamps, error) { log.Tracef("GetUnvettedTimestamps: %x %v", token, version) @@ -1314,10 +1297,10 @@ func (t *tlogBackend) GetUnvettedTimestamps(token []byte, version string) (*back } // Get timestamps - return t.unvetted.recordTimestamps(treeID, v, token) + return t.unvetted.RecordTimestamps(treeID, v, token) } -// This function satisfies the Backend interface. +// This function satisfies the backend.Backend interface. func (t *tlogBackend) GetVettedTimestamps(token []byte, version string) (*backend.RecordTimestamps, error) { log.Tracef("GetVettedTimestamps: %x %v", token, version) @@ -1342,21 +1325,21 @@ func (t *tlogBackend) GetVettedTimestamps(token []byte, version string) (*backen } // Get timestamps - return t.vetted.recordTimestamps(treeID, v, token) + return t.vetted.RecordTimestamps(treeID, v, token) } // This function must be called WITH the unvetted lock held. func (t *tlogBackend) unvettedPublish(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { // Create a vetted tree - vettedTreeID, err := t.vetted.treeNew() + vettedTreeID, err := t.vetted.TreeNew() if err != nil { return err } // Save the record to the vetted tlog - err = t.vetted.recordSave(vettedTreeID, rm, metadata, files) + err = t.vetted.RecordSave(vettedTreeID, rm, metadata, files) if err != nil { - return fmt.Errorf("vetted recordSave: %v", err) + return fmt.Errorf("vetted RecordSave: %v", err) } log.Debugf("Unvetted record %x copied to vetted tree %v", @@ -1364,9 +1347,9 @@ func (t *tlogBackend) unvettedPublish(token []byte, rm backend.RecordMetadata, m // Freeze the unvetted tree treeID := treeIDFromToken(token) - err = t.unvetted.treeFreeze(treeID, rm, metadata, vettedTreeID) + err = t.unvetted.TreeFreeze(treeID, rm, metadata, vettedTreeID) if err != nil { - return fmt.Errorf("treeFreeze %v: %v", treeID, err) + return fmt.Errorf("TreeFreeze %v: %v", treeID, err) } log.Debugf("Unvetted record frozen %x", token) @@ -1381,17 +1364,17 @@ func (t *tlogBackend) unvettedPublish(token []byte, rm backend.RecordMetadata, m func (t *tlogBackend) unvettedCensor(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { // Freeze the tree treeID := treeIDFromToken(token) - err := t.unvetted.treeFreeze(treeID, rm, metadata, 0) + err := t.unvetted.TreeFreeze(treeID, rm, metadata, 0) if err != nil { - return fmt.Errorf("treeFreeze %v: %v", treeID, err) + return fmt.Errorf("TreeFreeze %v: %v", treeID, err) } log.Debugf("Unvetted record frozen %x", token) // Delete all record files - err = t.unvetted.recordDel(treeID) + err = t.unvetted.RecordDel(treeID) if err != nil { - return fmt.Errorf("recordDel %v: %v", treeID, err) + return fmt.Errorf("RecordDel %v: %v", treeID, err) } log.Debugf("Unvetted record files deleted %x", token) @@ -1426,9 +1409,9 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, // Get existing record treeID := treeIDFromToken(token) - r, err := t.unvetted.recordLatest(treeID) + r, err := t.unvetted.RecordLatest(treeID) if err != nil { - return nil, fmt.Errorf("recordLatest: %v", err) + return nil, fmt.Errorf("RecordLatest: %v", err) } rm := r.RecordMetadata currStatus := rm.Status @@ -1450,17 +1433,17 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) // Call pre plugin hooks - hsrs := hookSetRecordStatus{ + hsrs := plugins.HookSetRecordStatus{ Current: *r, RecordMetadata: rm, MDAppend: mdAppend, MDOverwrite: mdOverwrite, } - b, err := encodeHookSetRecordStatus(hsrs) + b, err := json.Marshal(hsrs) if err != nil { return nil, err } - err = t.pluginHookPre(hookSetRecordStatusPre, string(b)) + err = t.unvetted.PluginHookPre(plugins.HookSetRecordStatusPre, string(b)) if err != nil { return nil, err } @@ -1483,7 +1466,7 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, } // Call post plugin hooks - t.pluginHookPost(hookSetRecordStatusPost, string(b)) + t.unvetted.PluginHookPost(plugins.HookSetRecordStatusPost, string(b)) log.Debugf("Status change %x from %v (%v) to %v (%v)", token, backend.MDStatus[currStatus], currStatus, @@ -1514,17 +1497,17 @@ func (t *tlogBackend) vettedCensor(token []byte, rm backend.RecordMetadata, meta if !ok { return fmt.Errorf("vetted record not found") } - err := t.vetted.treeFreeze(treeID, rm, metadata, 0) + err := t.vetted.TreeFreeze(treeID, rm, metadata, 0) if err != nil { - return fmt.Errorf("treeFreeze %v: %v", treeID, err) + return fmt.Errorf("TreeFreeze %v: %v", treeID, err) } log.Debugf("Vetted record frozen %x", token) // Delete all record files - err = t.vetted.recordDel(treeID) + err = t.vetted.RecordDel(treeID) if err != nil { - return fmt.Errorf("recordDel %v: %v", treeID, err) + return fmt.Errorf("RecordDel %v: %v", treeID, err) } log.Debugf("Vetted record files deleted %x", token) @@ -1540,9 +1523,9 @@ func (t *tlogBackend) vettedArchive(token []byte, rm backend.RecordMetadata, met if !ok { return fmt.Errorf("vetted record not found") } - err := t.vetted.treeFreeze(treeID, rm, metadata, 0) + err := t.vetted.TreeFreeze(treeID, rm, metadata, 0) if err != nil { - return fmt.Errorf("treeFreeze %v: %v", treeID, err) + return fmt.Errorf("TreeFreeze %v: %v", treeID, err) } log.Debugf("Vetted record frozen %x", token) @@ -1550,7 +1533,7 @@ func (t *tlogBackend) vettedArchive(token []byte, rm backend.RecordMetadata, met return nil } -// This function satisfies the Backend interface. +// This function satisfies the backend.Backend interface. func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { log.Tracef("SetVettedStatus: %x %v (%v)", token, status, backend.MDStatus[status]) @@ -1578,9 +1561,9 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md } // Get existing record - r, err := t.vetted.recordLatest(treeID) + r, err := t.vetted.RecordLatest(treeID) if err != nil { - return nil, fmt.Errorf("recordLatest: %v", err) + return nil, fmt.Errorf("RecordLatest: %v", err) } rm := r.RecordMetadata currStatus := rm.Status @@ -1602,17 +1585,17 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) // Call pre plugin hooks - srs := hookSetRecordStatus{ + srs := plugins.HookSetRecordStatus{ Current: *r, RecordMetadata: rm, MDAppend: mdAppend, MDOverwrite: mdOverwrite, } - b, err := encodeHookSetRecordStatus(srs) + b, err := json.Marshal(srs) if err != nil { return nil, err } - err = t.pluginHookPre(hookSetRecordStatusPre, string(b)) + err = t.vetted.PluginHookPre(plugins.HookSetRecordStatusPre, string(b)) if err != nil { return nil, err } @@ -1635,7 +1618,7 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md } // Call post plugin hooks - t.pluginHookPost(hookSetRecordStatusPost, string(b)) + t.vetted.PluginHookPost(plugins.HookSetRecordStatusPost, string(b)) // Update inventory cache t.inventoryUpdate(stateVetted, token, currStatus, status) @@ -1645,29 +1628,18 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md backend.MDStatus[status], status) // Return the updated record - r, err = t.vetted.recordLatest(treeID) + r, err = t.vetted.RecordLatest(treeID) if err != nil { - return nil, fmt.Errorf("recordLatest: %v", err) + return nil, fmt.Errorf("RecordLatest: %v", err) } return r, nil } -// Inventory is not implemented in tlogbe. If the caller which to pull records -// from the inventory then they should use the InventoryByStatus call to get -// the tokens of all records in the inventory and pull the required records -// individually. -// -// This function satisfies the Backend interface. -func (t *tlogBackend) Inventory(vettedCount, vettedStart, unvettedCount uint, includeFiles, allVersions bool) ([]backend.Record, []backend.Record, error) { - log.Tracef("Inventory") - return nil, nil, fmt.Errorf("not implemented") -} - // InventoryByStatus returns the record tokens of all records in the inventory // categorized by MDStatusT. // -// This function satisfies the Backend interface. +// This function satisfies the backend.Backend interface. func (t *tlogBackend) InventoryByStatus() (*backend.InventoryByStatus, error) { log.Tracef("InventoryByStatus") @@ -1678,147 +1650,142 @@ func (t *tlogBackend) InventoryByStatus() (*backend.InventoryByStatus, error) { }, nil } -func (t *tlogBackend) RegisterPlugin(p backend.Plugin) error { - log.Tracef("RegisterPlugin: %v", p.ID) +// RegisterUnvettedPlugin registers a plugin with the unvetted tlog instance. +// +// This function satisfies the backend.Backend interface. +func (t *tlogBackend) RegisterUnvettedPlugin(p backend.Plugin) error { + log.Tracef("RegisterUnvettedPlugin: %v", p.ID) + + if t.isShutdown() { + return backend.ErrShutdown + } + + return t.unvetted.PluginRegister(p) +} - // Add the plugin data dir to the plugin settings. Plugins should - // create their own individual data directories inside of the - // plugin data directory. - p.Settings = append(p.Settings, backend.PluginSetting{ - Key: pluginSettingDataDir, - Value: filepath.Join(t.dataDir, pluginDataDirname), - }) +// RegisterVettedPlugin has not been implemented. +// +// This function satisfies the backend.Backend interface. +func (t *tlogBackend) RegisterVettedPlugin(p backend.Plugin) error { + log.Tracef("RegisterVettedPlugin: %v", p.ID) - var ( - client pluginClient - err error - ) - switch p.ID { - case comments.ID: - client, err = newCommentsPlugin(t, newBackendClient(t), - p.Settings, p.Identity) - if err != nil { - return err - } - case dcrdata.ID: - client, err = newDcrdataPlugin(p.Settings, t.activeNetParams) - if err != nil { - return err - } - case pi.ID: - client, err = newPiPlugin(t, newBackendClient(t), p.Settings, - t.activeNetParams) - if err != nil { - return err - } - case ticketvote.ID: - client, err = newTicketVotePlugin(t, newBackendClient(t), - p.Settings, p.Identity, t.activeNetParams) - if err != nil { - return err - } - default: - return backend.ErrPluginInvalid + if t.isShutdown() { + return backend.ErrShutdown } - t.plugins[p.ID] = plugin{ - id: p.ID, - version: p.Version, - settings: p.Settings, - client: client, + return t.vetted.PluginRegister(p) +} + +// SetupUnvettedPlugin performs plugin setup for a previously registered +// unvetted plugin. +// +// This function satisfies the backend.Backend interface. +func (t *tlogBackend) SetupUnvettedPlugin(pluginID string) error { + log.Tracef("SetupUnvettedPlugin: %v", pluginID) + + if t.isShutdown() { + return backend.ErrShutdown } - return nil + return t.unvetted.PluginSetup(pluginID) } -func (t *tlogBackend) SetupPlugin(pluginID string) error { - log.Tracef("SetupPlugin: %v", pluginID) +// SetupVettedPlugin performs plugin setup for a previously registered vetted +// plugin. +// +// This function satisfies the backend.Backend interface. +func (t *tlogBackend) SetupVettedPlugin(pluginID string) error { + log.Tracef("SetupVettedPlugin: %v", pluginID) - plugin, ok := t.plugins[pluginID] - if !ok { - return backend.ErrPluginInvalid + if t.isShutdown() { + return backend.ErrShutdown } - return plugin.client.setup() + return t.vetted.PluginSetup(pluginID) } -// GetPlugins returns the backend plugins that have been registered and their -// settings. +// UnvettedPlugin executes a plugin command on an unvetted record. // -// This function satisfies the Backend interface. -func (t *tlogBackend) GetPlugins() ([]backend.Plugin, error) { - log.Tracef("GetPlugins") +// This function satisfies the backend.Backend interface. +func (t *tlogBackend) UnvettedPlugin(token []byte, pluginID, cmd, payload string) (string, error) { + log.Tracef("UnvettedPlugin: %x %v %v", token, pluginID, cmd) + + if t.isShutdown() { + return "", backend.ErrShutdown + } + + // Get tree ID + treeID := t.unvettedTreeIDFromToken(token) - plugins := make([]backend.Plugin, 0, len(t.plugins)) - for _, v := range t.plugins { - plugins = append(plugins, backend.Plugin{ - ID: v.id, - Version: v.version, - Settings: v.settings, - }) + // Verify record exists and is unvetted + if !t.UnvettedExists(token) { + return "", backend.ErrRecordNotFound } - return plugins, nil + return t.unvetted.PluginCmd(treeID, token, pluginID, cmd, payload) } -// Plugin is a pass-through function for plugin commands. +// VettedPlugin executes a plugin command on an unvetted record. // -// This function satisfies the Backend interface. -func (t *tlogBackend) Plugin(pluginID, cmd, cmdID, payload string) (string, error) { - log.Tracef("Plugin: %v %v", pluginID, cmd) +// This function satisfies the backend.Backend interface. +func (t *tlogBackend) VettedPlugin(token []byte, pluginID, cmd, payload string) (string, error) { + log.Tracef("VettedPlugin: %x %v %v", token, pluginID, cmd) if t.isShutdown() { return "", backend.ErrShutdown } - // Get plugin - plugin, ok := t.plugins[pluginID] + // Get tree ID + treeID, ok := t.vettedTreeIDFromToken(token) if !ok { - return "", backend.ErrPluginInvalid + return "", backend.ErrRecordNotFound } - // Execute plugin command - reply, err := plugin.client.cmd(cmd, payload) - if err != nil { - return "", err - } + return t.vetted.PluginCmd(treeID, token, pluginID, cmd, payload) +} - return reply, nil +// GetUnvettedPlugins returns the unvetted plugins that have been registered. +// +// This function satisfies the backend.Backend interface. +func (t *tlogBackend) GetUnvettedPlugins() []backend.Plugin { + log.Tracef("GetUnvettedPlugins") + + return t.unvetted.Plugins() } -func (t *tlogBackend) pluginHookPre(h hookT, payload string) error { - // Pass hook event and payload to each plugin - for _, v := range t.plugins { - err := v.client.hook(h, payload) - if err != nil { - var e backend.PluginUserError - if errors.As(err, &e) { - return err - } - return fmt.Errorf("hook %v: %v", v.id, err) - } - } +// GetVettedPlugins returns the vetted plugins that have been registered. +// +// This function satisfies the backend.Backend interface. +func (t *tlogBackend) GetVettedPlugins() []backend.Plugin { + log.Tracef("GetVettedPlugins") - return nil + return t.vetted.Plugins() } -func (t *tlogBackend) pluginHookPost(h hookT, payload string) { - // Pass hook event and payload to each plugin - for _, v := range t.plugins { - err := v.client.hook(h, payload) - if err != nil { - // This is the post plugin hook so the data has already been - // saved to the tlog backend. We do not have the ability to - // unwind. Log the error and continue. - log.Criticalf("pluginHookPost %v %v: %v: %v", v.id, h, err, payload) - continue - } - } +// Inventory has been DEPRECATED. +// +// This function satisfies the backend.Backend interface. +func (t *tlogBackend) Inventory(vettedCount, vettedStart, unvettedCount uint, includeFiles, allVersions bool) ([]backend.Record, []backend.Record, error) { + return nil, nil, fmt.Errorf("not implemented") +} + +// GetPlugins has been DEPRECATED. +// +// This function satisfies the backend.Backend interface. +func (t *tlogBackend) GetPlugins() ([]backend.Plugin, error) { + return nil, fmt.Errorf("not implemented") +} + +// Plugin has been DEPRECATED. +// +// This function satisfies the backend.Backend interface. +func (t *tlogBackend) Plugin(pluginID, cmd, cmdID, payload string) (string, error) { + return "", fmt.Errorf("not implemented") } // Close shuts the backend down and performs cleanup. // -// This function satisfies the Backend interface. +// This function satisfies the backend.Backend interface. func (t *tlogBackend) Close() { log.Tracef("Close") @@ -1828,26 +1795,26 @@ func (t *tlogBackend) Close() { // Shutdown backend t.shutdown = true - // Close out tlog connections - t.unvetted.close() - t.vetted.close() + // Close tlog connections + t.unvetted.Close() + t.vetted.Close() } -// setup creates the tlog backend in-memory cache. +// setup creates the tlog backend caches. func (t *tlogBackend) setup() error { log.Tracef("setup") // Get all trees - trees, err := t.unvetted.trillian.treesAll() + treeIDs, err := t.unvetted.TreesAll() if err != nil { - return fmt.Errorf("unvetted treesAll: %v", err) + return fmt.Errorf("unvetted TreesAll: %v", err) } log.Infof("Building backend caches") // Build all memory caches - for _, v := range trees { - token := tokenFromTreeID(v.TreeId) + for _, v := range treeIDs { + token := tokenFromTreeID(v) log.Debugf("Building memory caches for %x", token) @@ -1928,17 +1895,17 @@ func New(anp *chaincfg.Params, homeDir, dataDir, unvettedTrillianHost, unvettedT log.Infof("Anchor host: %v", dcrtimeHost) // Setup tlog instances - unvetted, err := newTlog(tlogIDUnvetted, homeDir, dataDir, + unvetted, err := tlog.New(tlogIDUnvetted, homeDir, dataDir, unvettedTrillianHost, unvettedTrillianKeyFile, encryptionKeyFile, dcrtimeHost, dcrtimeCert) if err != nil { - return nil, fmt.Errorf("newTlog unvetted: %v", err) + return nil, fmt.Errorf("new tlog unvetted: %v", err) } - vetted, err := newTlog(tlogIDVetted, homeDir, dataDir, + vetted, err := tlog.New(tlogIDVetted, homeDir, dataDir, vettedTrillianHost, vettedTrillianKeyFile, "", dcrtimeHost, dcrtimeCert) if err != nil { - return nil, fmt.Errorf("newTlog vetted: %v", err) + return nil, fmt.Errorf("new tlog vetted: %v", err) } // Setup tlogbe @@ -1948,7 +1915,6 @@ func New(anp *chaincfg.Params, homeDir, dataDir, unvettedTrillianHost, unvettedT dataDir: dataDir, unvetted: unvetted, vetted: vetted, - plugins: make(map[string]plugin), prefixes: make(map[string][]byte), vettedTreeIDs: make(map[string]int64), inv: recordInventory{ diff --git a/politeiad/backend/tlogbe/tlogbe_test.go b/politeiad/backend/tlogbe/tlogbe_test.go index 49ac59afe..123c81d61 100644 --- a/politeiad/backend/tlogbe/tlogbe_test.go +++ b/politeiad/backend/tlogbe/tlogbe_test.go @@ -5,14 +5,355 @@ package tlogbe import ( + "bytes" + "encoding/base64" + "encoding/hex" "errors" + "image" + "image/jpeg" + "image/png" "testing" v1 "github.com/decred/politeia/politeiad/api/v1" + "github.com/decred/politeia/politeiad/api/v1/mime" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/util" ) +func newBackendFile(t *testing.T, fileName string) backend.File { + t.Helper() + + r, err := util.Random(64) + if err != nil { + r = []byte{0, 0, 0} // random byte data + } + + payload := hex.EncodeToString(r) + digest := hex.EncodeToString(util.Digest([]byte(payload))) + b64 := base64.StdEncoding.EncodeToString([]byte(payload)) + + return backend.File{ + Name: fileName, + MIME: mime.DetectMimeType([]byte(payload)), + Digest: digest, + Payload: b64, + } +} + +func newBackendFileJPEG(t *testing.T) backend.File { + t.Helper() + + b := new(bytes.Buffer) + img := image.NewRGBA(image.Rect(0, 0, 1000, 500)) + + err := jpeg.Encode(b, img, &jpeg.Options{}) + if err != nil { + t.Fatal(err) + } + + // Generate a random name + r, err := util.Random(8) + if err != nil { + t.Fatal(err) + } + + return backend.File{ + Name: hex.EncodeToString(r) + ".jpeg", + MIME: mime.DetectMimeType(b.Bytes()), + Digest: hex.EncodeToString(util.Digest(b.Bytes())), + Payload: base64.StdEncoding.EncodeToString(b.Bytes()), + } +} + +func newBackendFilePNG(t *testing.T) backend.File { + t.Helper() + + b := new(bytes.Buffer) + img := image.NewRGBA(image.Rect(0, 0, 1000, 500)) + + err := png.Encode(b, img) + if err != nil { + t.Fatal(err) + } + + // Generate a random name + r, err := util.Random(8) + if err != nil { + t.Fatal(err) + } + + return backend.File{ + Name: hex.EncodeToString(r) + ".png", + MIME: mime.DetectMimeType(b.Bytes()), + Digest: hex.EncodeToString(util.Digest(b.Bytes())), + Payload: base64.StdEncoding.EncodeToString(b.Bytes()), + } +} + +func newBackendMetadataStream(t *testing.T, id uint64, payload string) backend.MetadataStream { + t.Helper() + + return backend.MetadataStream{ + ID: id, + Payload: payload, + } +} + +// recordContentTests defines the type used to describe the content +// verification error tests. +type recordContentTest struct { + description string + metadata []backend.MetadataStream + files []backend.File + filesDel []string + err backend.ContentVerificationError +} + +// setupRecordContentTests returns the list of tests for the verifyContent +// function. These tests are used on all backend api endpoints that verify +// content. +func setupRecordContentTests(t *testing.T) []recordContentTest { + t.Helper() + + var rct []recordContentTest + + // Invalid metadata ID error + md := []backend.MetadataStream{ + newBackendMetadataStream(t, v1.MetadataStreamsMax+1, ""), + } + fs := []backend.File{ + newBackendFile(t, "index.md"), + } + fsDel := []string{} + err := backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidMDID, + } + rct = append(rct, recordContentTest{ + description: "Invalid metadata ID error", + metadata: md, + files: fs, + filesDel: fsDel, + err: err, + }) + + // Duplicate metadata ID error + md = []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + newBackendMetadataStream(t, 1, ""), + } + fs = []backend.File{ + newBackendFile(t, "index.md"), + } + err = backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusDuplicateMDID, + } + rct = append(rct, recordContentTest{ + description: "Duplicate metadata ID error", + metadata: md, + files: fs, + filesDel: fsDel, + err: err, + }) + + // Invalid filename error + md = []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs = []backend.File{ + newBackendFile(t, "invalid/filename.md"), + } + err = backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidFilename, + } + rct = append(rct, recordContentTest{ + description: "Invalid filename error", + metadata: md, + files: fs, + filesDel: fsDel, + err: err, + }) + + // Invalid filename in filesDel error + md = []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs = []backend.File{ + newBackendFile(t, "index.md"), + } + fsDel = []string{"invalid/filename.md"} + err = backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidFilename, + } + rct = append(rct, recordContentTest{ + description: "Invalid filename in filesDel error", + metadata: md, + files: fs, + filesDel: fsDel, + err: err, + }) + + // Empty files error + md = []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs = []backend.File{} + fsDel = []string{} + err = backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusEmpty, + } + rct = append(rct, recordContentTest{ + description: "Empty files error", + metadata: md, + files: fs, + filesDel: fsDel, + err: err, + }) + + // Duplicate filename error + md = []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs = []backend.File{ + newBackendFile(t, "index.md"), + newBackendFile(t, "index.md"), + } + err = backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusDuplicateFilename, + } + rct = append(rct, recordContentTest{ + description: "Duplicate filename error", + metadata: md, + files: fs, + filesDel: fsDel, + err: err, + }) + + // Duplicate filename in filesDel error + md = []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs = []backend.File{ + newBackendFile(t, "index.md"), + } + fsDel = []string{ + "duplicate.md", + "duplicate.md", + } + err = backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusDuplicateFilename, + } + rct = append(rct, recordContentTest{ + description: "Duplicate filename in filesDel error", + metadata: md, + files: fs, + filesDel: fsDel, + err: err, + }) + + // Invalid file digest error + md = []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + fs = []backend.File{ + newBackendFile(t, "index.md"), + } + fsDel = []string{} + err = backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidFileDigest, + } + rct = append(rct, recordContentTest{ + description: "Invalid file digest error", + metadata: md, + files: fs, + filesDel: fsDel, + err: err, + }) + + // Invalid base64 error + md = []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + f := newBackendFile(t, "index.md") + f.Payload = "*" + fs = []backend.File{f} + err = backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidBase64, + } + rct = append(rct, recordContentTest{ + description: "Invalid file digest error", + metadata: md, + files: fs, + filesDel: fsDel, + err: err, + }) + + // Invalid payload digest error + md = []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + f = newBackendFile(t, "index.md") + f.Payload = "rand" + fs = []backend.File{f} + err = backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidFileDigest, + } + rct = append(rct, recordContentTest{ + description: "Invalid payload digest error", + metadata: md, + files: fs, + filesDel: fsDel, + err: err, + }) + + // Invalid MIME type from payload error + md = []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + jpeg := newBackendFileJPEG(t) + jpeg.Payload = "rand" + payload, er := base64.StdEncoding.DecodeString(jpeg.Payload) + if er != nil { + t.Fatalf(er.Error()) + } + jpeg.Digest = hex.EncodeToString(util.Digest(payload)) + fs = []backend.File{ + newBackendFile(t, "index.md"), + jpeg, + } + err = backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidMIMEType, + } + rct = append(rct, recordContentTest{ + description: "Invalid MIME type from payload error", + metadata: md, + files: fs, + filesDel: fsDel, + err: err, + }) + + // Unsupported MIME type error + md = []backend.MetadataStream{ + newBackendMetadataStream(t, 1, ""), + } + jpeg = newBackendFileJPEG(t) + fs = []backend.File{ + newBackendFile(t, "index.md"), + jpeg, + } + err = backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusUnsupportedMIMEType, + } + rct = append(rct, recordContentTest{ + description: "Unsupported MIME type error", + metadata: md, + files: fs, + filesDel: fsDel, + err: err, + }) + + return rct +} + func TestNewRecord(t *testing.T) { tlogBackend, cleanup := newTestTlogBackend(t) defer cleanup() diff --git a/politeiad/backend/tlogbe/tlogclient.go b/politeiad/backend/tlogbe/tlogclient.go index b38f53047..fdc019db2 100644 --- a/politeiad/backend/tlogbe/tlogclient.go +++ b/politeiad/backend/tlogbe/tlogclient.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" ) // tlogClient provides an API for the plugins to interact with the tlog @@ -55,7 +56,7 @@ type backendClient struct { } // tlogByID returns the tlog instance that corresponds to the provided ID. -func (c *backendClient) tlogByID(tlogID string) (*tlog, error) { +func (c *backendClient) tlogByID(tlogID string) (*tlog.Tlog, error) { switch tlogID { case tlogIDUnvetted: return c.backend.unvetted, nil @@ -68,36 +69,42 @@ func (c *backendClient) tlogByID(tlogID string) (*tlog, error) { // treeIDFromToken returns the treeID for the provided tlog instance ID and // token. This function accepts both token prefixes and full length tokens. func (c *backendClient) treeIDFromToken(tlogID string, token []byte) (int64, error) { - if len(token) == tokenPrefixSize() { - // This is a token prefix. Get the full token from the cache. - var ok bool - token, ok = c.backend.fullLengthToken(token) - if !ok { - return 0, errRecordNotFound + /* + if len(token) == tokenPrefixSize() { + // This is a token prefix. Get the full token from the cache. + var ok bool + token, ok = c.backend.fullLengthToken(token) + if !ok { + return 0, errRecordNotFound + } } - } - switch tlogID { - case tlogIDUnvetted: - return treeIDFromToken(token), nil - case tlogIDVetted: - treeID, ok := c.backend.vettedTreeIDFromToken(token) - if !ok { - return 0, errRecordNotFound + switch tlogID { + case tlogIDUnvetted: + return treeIDFromToken(token), nil + case tlogIDVetted: + treeID, ok := c.backend.vettedTreeIDFromToken(token) + if !ok { + return 0, errRecordNotFound + } + return treeID, nil } - return treeID, nil - } - return 0, fmt.Errorf("unknown tlog id '%v'", tlogID) + return 0, fmt.Errorf("unknown tlog id '%v'", tlogID) + */ + return 0, nil } // treeIDFromToken returns the treeID for the provided tlog instance ID and // token. This function only accepts full length tokens. func (c *backendClient) treeIDFromTokenFullLength(tlogID string, token []byte) (int64, error) { - if !tokenIsFullLength(token) { - return 0, errRecordNotFound - } - return c.treeIDFromToken(tlogID, token) + /* + if !tokenIsFullLength(token) { + return 0, errRecordNotFound + } + return c.treeIDFromToken(tlogID, token) + */ + return 0, nil } // save saves the provided blobs to the tlog backend. Note, hashes contains the @@ -122,7 +129,7 @@ func (c *backendClient) save(tlogID string, token []byte, keyPrefix string, blob } // Save blobs - return tlog.blobsSave(treeID, keyPrefix, blobs, hashes, encrypt) + return tlog.BlobsSave(treeID, keyPrefix, blobs, hashes, encrypt) } // del deletes the blobs that correspond to the provided merkle leaf hashes. @@ -145,7 +152,7 @@ func (c *backendClient) del(tlogID string, token []byte, merkles [][]byte) error } // Delete blobs - return tlog.blobsDel(treeID, merkles) + return tlog.BlobsDel(treeID, merkles) } // merklesByKeyPrefix returns the merkle root hashes for all blobs that match @@ -169,7 +176,7 @@ func (c *backendClient) merklesByKeyPrefix(tlogID string, token []byte, keyPrefi } // Get merkle leaf hashes - return tlog.merklesByKeyPrefix(treeID, keyPrefix) + return tlog.MerklesByKeyPrefix(treeID, keyPrefix) } // blobsByMerkle returns the blobs with the provided merkle leaf hashes. @@ -195,7 +202,7 @@ func (c *backendClient) blobsByMerkle(tlogID string, token []byte, merkles [][]b } // Get blobs - return tlog.blobsByMerkle(treeID, merkles) + return tlog.BlobsByMerkle(treeID, merkles) } // blobsByKeyPrefix returns all blobs that match the provided key prefix. @@ -218,7 +225,7 @@ func (c *backendClient) blobsByKeyPrefix(tlogID string, token []byte, keyPrefix } // Get blobs - return tlog.blobsByKeyPrefix(treeID, keyPrefix) + return tlog.BlobsByKeyPrefix(treeID, keyPrefix) } // timestamp returns the timestamp for a data blob that corresponds to the @@ -240,14 +247,7 @@ func (c *backendClient) timestamp(tlogID string, token []byte, merkle []byte) (* return nil, err } - // Get all tree leaves - leaves, err := tlog.trillian.leavesAll(treeID) - if err != nil { - return nil, fmt.Errorf("leavesAll: %v", err) - } - - // Get timestamp - return tlog.timestamp(treeID, merkle, leaves) + return tlog.Timestamp(treeID, merkle) } // newBackendClient returns a new backendClient. From dfac034de61313acf86d5e269458fd8d9053099b Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 14 Jan 2021 20:19:37 -0600 Subject: [PATCH 219/449] Get comments plugin working. --- politeiad/backend/backend.go | 2 +- .../tlogbe/plugins/comments/comments.go | 995 ++++++------------ .../tlogbe/plugins/comments/comments_test.go | 226 +--- .../backend/tlogbe/plugins/comments/index.go | 114 ++ politeiad/backend/tlogbe/plugins/plugins.go | 16 +- politeiad/backend/tlogbe/tlog/tlog.go | 2 - politeiad/backend/tlogbe/tlog/tlogclient.go | 68 +- politeiad/backend/tlogbe/tlogbe.go | 9 - politeiad/plugins/comments/comments.go | 380 +------ util/token.go | 96 ++ 10 files changed, 647 insertions(+), 1261 deletions(-) create mode 100644 politeiad/backend/tlogbe/plugins/comments/index.go create mode 100644 util/token.go diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index 87a25b59c..49f04bd90 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -115,7 +115,7 @@ func (s StateTransitionError) Error() string { type PluginUserError struct { PluginID string ErrorCode int - ErrorContext []string + ErrorContext string } // Error satisfies the error interface. diff --git a/politeiad/backend/tlogbe/plugins/comments/comments.go b/politeiad/backend/tlogbe/plugins/comments/comments.go index b8dce8620..acd888c17 100644 --- a/politeiad/backend/tlogbe/plugins/comments/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments/comments.go @@ -11,7 +11,6 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" "os" "path/filepath" "sort" @@ -20,7 +19,6 @@ import ( "sync" "time" - v1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" @@ -29,57 +27,6 @@ import ( "github.com/decred/politeia/util" ) -// TODO get rid of these -var ( - errRecordNotFound = errors.New("record not found") -) - -func tokenIsFullLength(token []byte) bool { - return len(token) == v1.TokenSizeTlog -} -func tokenPrefixSize() int { - // If the token prefix length is an odd number of characters then - // padding would have needed to be added to it prior to decoding it - // to hex to prevent a hex.ErrLenth (odd length hex string) error. - // Account for this padding in the prefix size. - var size int - if v1.TokenPrefixLength%2 == 1 { - // Add 1 to the length to account for padding - size = (v1.TokenPrefixLength + 1) / 2 - } else { - // No padding was required - size = v1.TokenPrefixLength / 2 - } - return size -} -func tokenPrefix(token []byte) string { - return hex.EncodeToString(token)[:v1.TokenPrefixLength] -} -func tokenDecodeAnyLength(token string) ([]byte, error) { - // If provided token has odd length add padding - if len(token)%2 == 1 { - token = token + "0" - } - t, err := hex.DecodeString(token) - if err != nil { - return nil, fmt.Errorf("invalid hex") - } - switch { - case tokenIsFullLength(t): - // Full length tokens are allowed; continue - case len(t) == tokenPrefixSize(): - // Token prefixes are allowed; continue - default: - return nil, fmt.Errorf("invalid token size") - } - return t, nil -} - -// TODO holding the lock before verifying the token can allow the mutexes to -// be spammed. Create an infinite amount of them with invalid tokens. The fix -// is to check if the record exists in the mutexes function to ensure a token -// is valid before holding the lock on it. This is where we can return a -// record doesn't exist user error too. // TODO prevent duplicate comments // TODO upvoting a comment twice in the same second causes a duplicate leaf // error which causes a 500. Solution: add the timestamp to the vote index. @@ -99,7 +46,7 @@ const ( // Filenames of cached data saved to the plugin data dir. Brackets // are used to indicate a variable that should be replaced in the // filename. - filenameCommentsIndex = "{state}-{token}-commentsindex.json" + filenameRecordIndex = "{tokenPrefix}-index.json" ) var ( @@ -129,172 +76,41 @@ type commentsPlugin struct { mutexes map[string]*sync.Mutex // [string]mutex } -type voteIndex struct { - Vote comments.VoteT `json:"vote"` - Merkle []byte `json:"merkle"` // Log leaf merkle leaf hash -} - -type commentIndex struct { - Adds map[uint32][]byte `json:"adds"` // [version]merkleLeafHash - Del []byte `json:"del"` // Merkle leaf hash of delete record - - // Votes contains the vote history for each uuid that voted on the - // comment. This data is cached because the effect of a new vote - // on a comment depends on the previous vote from that uuid. - // Example, a user upvotes a comment that they have already - // upvoted, the resulting vote score is 0 due to the second upvote - // removing the original upvote. - Votes map[string][]voteIndex `json:"votes"` // [uuid]votes -} - -// commentsIndex contains the indexes for all comments made on a record. -type commentsIndex struct { - Comments map[uint32]commentIndex `json:"comments"` // [commentID]comment -} - -// mutex returns the mutex for the specified record. -func (p *commentsPlugin) mutex(token string) *sync.Mutex { +// mutex returns the mutex for a record. +func (p *commentsPlugin) mutex(token []byte) *sync.Mutex { p.Lock() defer p.Unlock() - m, ok := p.mutexes[token] + t := hex.EncodeToString(token) + m, ok := p.mutexes[t] if !ok { // Mutexes is lazy loaded m = &sync.Mutex{} - p.mutexes[token] = m + p.mutexes[t] = m } return m } -// commentsIndexPath accepts full length token or token prefix but always -// uses prefix when generating the comments index path string. -func (p *commentsPlugin) commentsIndexPath(s comments.StateT, token string) (string, error) { - // Use token prefix - t, err := tokenDecodeAnyLength(token) - if err != nil { - return "", err - } - token = tokenPrefix(t) - fn := filenameCommentsIndex - switch s { - case comments.StateUnvetted: - fn = strings.Replace(fn, "{state}", "unvetted", 1) - case comments.StateVetted: - fn = strings.Replace(fn, "{state}", "vetted", 1) - default: - e := fmt.Errorf("unknown comments state: %v", s) - panic(e) - } - fn = strings.Replace(fn, "{token}", token, 1) - return filepath.Join(p.dataDir, fn), nil -} - -// commentsIndexLocked returns the cached commentsIndex for the provided -// record. If a cached commentsIndex does not exist, a new one will be -// returned. -// -// This function must be called WITH the lock held. -func (p *commentsPlugin) commentsIndexLocked(s comments.StateT, token []byte) (*commentsIndex, error) { - fp, err := p.commentsIndexPath(s, hex.EncodeToString(token)) - if err != nil { - return nil, err - } - b, err := ioutil.ReadFile(fp) - if err != nil { - var e *os.PathError - if errors.As(err, &e) && !os.IsExist(err) { - // File does't exist. Return a new commentsIndex instead. - return &commentsIndex{ - Comments: make(map[uint32]commentIndex), - }, nil - } - return nil, err - } - - var idx commentsIndex - err = json.Unmarshal(b, &idx) - if err != nil { - return nil, err - } - - return &idx, nil +func tokenDecode(token string) ([]byte, error) { + return util.TokenDecode(util.TokenTypeTlog, token) } -// commentsIndex returns the cached commentsIndex for the provided -// record. If a cached commentsIndex does not exist, a new one will be -// returned. -// -// This function must be called WITHOUT the lock held. -func (p *commentsPlugin) commentsIndex(s comments.StateT, token []byte) (*commentsIndex, error) { - m := p.mutex(hex.EncodeToString(token)) - m.Lock() - defer m.Unlock() - - return p.commentsIndexLocked(s, token) -} - -// commentsIndexSaveLocked saves the provided commentsIndex to the comments -// plugin data dir. -// -// This function must be called WITH the lock held. -func (p *commentsPlugin) commentsIndexSaveLocked(s comments.StateT, token []byte, idx commentsIndex) error { - b, err := json.Marshal(idx) - if err != nil { - return err - } - - fp, err := p.commentsIndexPath(s, hex.EncodeToString(token)) - if err != nil { - return err - } - err = ioutil.WriteFile(fp, b, 0664) - if err != nil { - return err - } - - return nil -} - -func tlogIDFromCommentState(s comments.StateT) string { - switch s { - case comments.StateUnvetted: - return plugins.TlogIDUnvetted - case comments.StateVetted: - return plugins.TlogIDVetted - default: - e := fmt.Sprintf("unknown state %v", s) - panic(e) - } -} - -func encryptFromCommentState(s comments.StateT) bool { - switch s { - case comments.StateUnvetted: - return true - case comments.StateVetted: - return false - default: - e := fmt.Sprintf("unknown state %v", s) - panic(e) - } -} - -func convertCommentsErrorFromSignatureError(err error) backend.PluginUserError { +func convertSignatureError(err error) backend.PluginUserError { var e util.SignatureError - var s comments.ErrorStatusT + var s comments.ErrorCodeT if errors.As(err, &e) { switch e.ErrorCode { case util.ErrorStatusPublicKeyInvalid: - s = comments.ErrorStatusPublicKeyInvalid + s = comments.ErrorCodePublicKeyInvalid case util.ErrorStatusSignatureInvalid: - s = comments.ErrorStatusSignatureInvalid + s = comments.ErrorCodeSignatureInvalid } } return backend.PluginUserError{ PluginID: comments.ID, ErrorCode: int(s), - ErrorContext: e.ErrorContext, + ErrorContext: strings.Join(e.ErrorContext, ", "), } } @@ -466,7 +282,6 @@ func convertCommentVoteFromBlobEntry(be store.BlobEntry) (*comments.CommentVote, func convertCommentFromCommentAdd(ca comments.CommentAdd) comments.Comment { return comments.Comment{ UserID: ca.UserID, - State: ca.State, Token: ca.Token, ParentID: ca.ParentID, Comment: ca.Comment, @@ -487,7 +302,6 @@ func convertCommentFromCommentDel(cd comments.CommentDel) comments.Comment { // Score needs to be filled in separately return comments.Comment{ UserID: cd.UserID, - State: cd.State, Token: cd.Token, ParentID: cd.ParentID, Comment: "", @@ -515,13 +329,13 @@ func commentVersionLatest(cidx commentIndex) uint32 { } // commentExists returns whether the provided comment ID exists. -func commentExists(idx commentsIndex, commentID uint32) bool { - _, ok := idx.Comments[commentID] +func commentExists(ridx recordIndex, commentID uint32) bool { + _, ok := ridx.Comments[commentID] return ok } // commentIDLatest returns the latest comment ID. -func commentIDLatest(idx commentsIndex) uint32 { +func commentIDLatest(idx recordIndex) uint32 { var maxID uint32 for id := range idx.Comments { if id > maxID { @@ -531,7 +345,7 @@ func commentIDLatest(idx commentsIndex) uint32 { return maxID } -func (p *commentsPlugin) commentAddSave(ca comments.CommentAdd) ([]byte, error) { +func (p *commentsPlugin) commentAddSave(treeID int64, ca comments.CommentAdd) ([]byte, error) { // Prepare blob be, err := convertBlobEntryFromCommentAdd(ca) if err != nil { @@ -546,17 +360,9 @@ func (p *commentsPlugin) commentAddSave(ca comments.CommentAdd) ([]byte, error) return nil, err } - // Prepare tlog args - tlogID := tlogIDFromCommentState(ca.State) - encrypt := encryptFromCommentState(ca.State) - token, err := hex.DecodeString(ca.Token) - if err != nil { - return nil, err - } - // Save blob - merkles, err := p.tlog.Save(tlogID, token, keyPrefixCommentAdd, - [][]byte{b}, [][]byte{h}, encrypt) + merkles, err := p.tlog.BlobsSave(treeID, keyPrefixCommentAdd, + [][]byte{b}, [][]byte{h}) if err != nil { return nil, err } @@ -569,11 +375,9 @@ func (p *commentsPlugin) commentAddSave(ca comments.CommentAdd) ([]byte, error) } // commentAdds returns the commentAdd for all specified merkle hashes. -func (p *commentsPlugin) commentAdds(s comments.StateT, token []byte, merkles [][]byte) ([]comments.CommentAdd, error) { +func (p *commentsPlugin) commentAdds(treeID int64, merkles [][]byte) ([]comments.CommentAdd, error) { // Retrieve blobs - tlogID := tlogIDFromCommentState(s) - _ = tlogID - blobs, err := p.tlog.BlobsByMerkle(tlogID, token, merkles) + blobs, err := p.tlog.BlobsByMerkle(treeID, merkles) if err != nil { return nil, err } @@ -606,7 +410,7 @@ func (p *commentsPlugin) commentAdds(s comments.StateT, token []byte, merkles [] return adds, nil } -func (p *commentsPlugin) commentDelSave(cd comments.CommentDel) ([]byte, error) { +func (p *commentsPlugin) commentDelSave(treeID int64, cd comments.CommentDel) ([]byte, error) { // Prepare blob be, err := convertBlobEntryFromCommentDel(cd) if err != nil { @@ -621,17 +425,9 @@ func (p *commentsPlugin) commentDelSave(cd comments.CommentDel) ([]byte, error) return nil, err } - // Prepare tlog args - tlogID := tlogIDFromCommentState(cd.State) - token, err := hex.DecodeString(cd.Token) - if err != nil { - return nil, err - } - // Save blob - _ = tlogID - merkles, err := p.tlog.Save(tlogID, token, keyPrefixCommentDel, - [][]byte{b}, [][]byte{h}, false) + merkles, err := p.tlog.BlobsSave(treeID, keyPrefixCommentDel, + [][]byte{b}, [][]byte{h}) if err != nil { return nil, err } @@ -643,11 +439,9 @@ func (p *commentsPlugin) commentDelSave(cd comments.CommentDel) ([]byte, error) return merkles[0], nil } -func (p *commentsPlugin) commentDels(s comments.StateT, token []byte, merkles [][]byte) ([]comments.CommentDel, error) { +func (p *commentsPlugin) commentDels(treeID int64, merkles [][]byte) ([]comments.CommentDel, error) { // Retrieve blobs - tlogID := tlogIDFromCommentState(s) - _ = tlogID - blobs, err := p.tlog.BlobsByMerkle(tlogID, token, merkles) + blobs, err := p.tlog.BlobsByMerkle(treeID, merkles) if err != nil { return nil, err } @@ -680,7 +474,7 @@ func (p *commentsPlugin) commentDels(s comments.StateT, token []byte, merkles [] return dels, nil } -func (p *commentsPlugin) commentVoteSave(cv comments.CommentVote) ([]byte, error) { +func (p *commentsPlugin) commentVoteSave(treeID int64, cv comments.CommentVote) ([]byte, error) { // Prepare blob be, err := convertBlobEntryFromCommentVote(cv) if err != nil { @@ -695,17 +489,9 @@ func (p *commentsPlugin) commentVoteSave(cv comments.CommentVote) ([]byte, error return nil, err } - // Prepare tlog args - tlogID := tlogIDFromCommentState(cv.State) - token, err := hex.DecodeString(cv.Token) - if err != nil { - return nil, err - } - // Save blob - _ = tlogID - merkles, err := p.tlog.Save(tlogID, token, keyPrefixCommentVote, - [][]byte{b}, [][]byte{h}, false) + merkles, err := p.tlog.BlobsSave(treeID, keyPrefixCommentVote, + [][]byte{b}, [][]byte{h}) if err != nil { return nil, err } @@ -717,11 +503,9 @@ func (p *commentsPlugin) commentVoteSave(cv comments.CommentVote) ([]byte, error return merkles[0], nil } -func (p *commentsPlugin) commentVotes(s comments.StateT, token []byte, merkles [][]byte) ([]comments.CommentVote, error) { +func (p *commentsPlugin) commentVotes(treeID int64, merkles [][]byte) ([]comments.CommentVote, error) { // Retrieve blobs - tlogID := tlogIDFromCommentState(s) - _ = tlogID - blobs, err := p.tlog.BlobsByMerkle(tlogID, token, merkles) + blobs, err := p.tlog.BlobsByMerkle(treeID, merkles) if err != nil { return nil, err } @@ -758,9 +542,8 @@ func (p *commentsPlugin) commentVotes(s comments.StateT, token []byte, merkles [ // comments are returned with limited data. Comment IDs that do not correspond // to an actual comment are not included in the returned map. It is the // responsibility of the caller to ensure a comment is returned for each of the -// provided comment IDs. The comments index that was looked up during this -// process is also returned. -func (p *commentsPlugin) comments(s comments.StateT, token []byte, idx commentsIndex, commentIDs []uint32) (map[uint32]comments.Comment, error) { +// provided comment IDs. +func (p *commentsPlugin) comments(treeID int64, ridx recordIndex, commentIDs []uint32) (map[uint32]comments.Comment, error) { // Aggregate the merkle hashes for all records that need to be // looked up. If a comment has been deleted then the only record // that will still exist is the comment del record. If the comment @@ -771,7 +554,7 @@ func (p *commentsPlugin) comments(s comments.StateT, token []byte, idx commentsI merkleDels = make([][]byte, 0, len(commentIDs)) ) for _, v := range commentIDs { - cidx, ok := idx.Comments[v] + cidx, ok := ridx.Comments[v] if !ok { // Comment does not exist continue @@ -789,11 +572,8 @@ func (p *commentsPlugin) comments(s comments.StateT, token []byte, idx commentsI } // Get comment add records - adds, err := p.commentAdds(s, token, merkleAdds) + adds, err := p.commentAdds(treeID, merkleAdds) if err != nil { - if errors.Is(err, errRecordNotFound) { - return nil, err - } return nil, fmt.Errorf("commentAdds: %v", err) } if len(adds) != len(merkleAdds) { @@ -802,7 +582,7 @@ func (p *commentsPlugin) comments(s comments.StateT, token []byte, idx commentsI } // Get comment del records - dels, err := p.commentDels(s, token, merkleDels) + dels, err := p.commentDels(treeID, merkleDels) if err != nil { return nil, fmt.Errorf("commentDels: %v", err) } @@ -815,7 +595,7 @@ func (p *commentsPlugin) comments(s comments.StateT, token []byte, idx commentsI cs := make(map[uint32]comments.Comment, len(commentIDs)) for _, v := range adds { c := convertCommentFromCommentAdd(v) - cidx, ok := idx.Comments[c.CommentID] + cidx, ok := ridx.Comments[c.CommentID] if !ok { return nil, fmt.Errorf("comment index not found %v", c.CommentID) } @@ -831,8 +611,8 @@ func (p *commentsPlugin) comments(s comments.StateT, token []byte, idx commentsI } // comment returns the latest version of the provided comment. -func (p *commentsPlugin) comment(s comments.StateT, token []byte, idx commentsIndex, commentID uint32) (*comments.Comment, error) { - cs, err := p.comments(s, token, idx, []uint32{commentID}) +func (p *commentsPlugin) comment(treeID int64, ridx recordIndex, commentID uint32) (*comments.Comment, error) { + cs, err := p.comments(treeID, ridx, []uint32{commentID}) if err != nil { return nil, fmt.Errorf("comments: %v", err) } @@ -843,13 +623,11 @@ func (p *commentsPlugin) comment(s comments.StateT, token []byte, idx commentsIn return &c, nil } -func (p *commentsPlugin) timestamp(s comments.StateT, token []byte, merkle []byte) (*comments.Timestamp, error) { +func (p *commentsPlugin) timestamp(treeID int64, merkle []byte) (*comments.Timestamp, error) { // Get timestamp - tlogID := tlogIDFromCommentState(s) - _ = tlogID - t, err := p.tlog.Timestamp(tlogID, token, merkle) + t, err := p.tlog.Timestamp(treeID, merkle) if err != nil { - return nil, fmt.Errorf("timestamp %x %x: %v", token, merkle, err) + return nil, err } // Convert response @@ -872,71 +650,121 @@ func (p *commentsPlugin) timestamp(s comments.StateT, token []byte, merkle []byt }, nil } -func (p *commentsPlugin) cmdNew(payload string) (string, error) { - log.Tracef("comments cmdNew: %v", payload) +// calcVoteScore returns the vote score for the provided comment index. The +// returned values are the downvotes and upvotes, respectively. +func calcVoteScore(cidx commentIndex) (uint64, uint64) { + // Find the vote score by replaying all existing votes from all + // users. The net effect of a new vote on a comment score depends + // on the previous vote from that uuid. Example, a user upvotes a + // comment that they have already upvoted, the resulting vote score + // is 0 due to the second upvote removing the original upvote. + var upvotes uint64 + var downvotes uint64 + for _, votes := range cidx.Votes { + // Calculate the vote score that this user is contributing. This + // can only ever be -1, 0, or 1. + var score int64 + for _, v := range votes { + vote := int64(v.Vote) + switch { + case score == 0: + // No previous vote. New vote becomes the score. + score = vote + + case score == vote: + // New vote is the same as the previous vote. The vote gets + // removed from the score, making the score 0. + score = 0 + + case score != vote: + // New vote is different than the previous vote. New vote + // becomes the score. + score = vote + } + } + + // Add the net result of all votes from this user to the totals. + switch score { + case 0: + // Nothing to do + case -1: + downvotes++ + case 1: + upvotes++ + default: + // Something went wrong + e := fmt.Errorf("unexpected vote score %v", score) + panic(e) + } + } + + return downvotes, upvotes +} + +func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdNew: %v %x %v", treeID, token, payload) // Decode payload - n, err := comments.DecodeNew([]byte(payload)) + var n comments.New + err := json.Unmarshal([]byte(payload), &n) if err != nil { return "", err } - // Verify state - switch n.State { - case comments.StateUnvetted, comments.StateVetted: - // Allowed; continue - default: + // Verify token + t, err := tokenDecode(n.Token) + if err != nil { return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusStateInvalid), + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeTokenInvalid), + ErrorContext: err.Error(), } } - - // Verify token - token, err := tokenDecodeAnyLength(n.Token) - if err != nil { + if !bytes.Equal(t, token) { + e := fmt.Sprintf("comment token does not match route token: "+ + "got %x, want %x", t, token) return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeTokenInvalid), + ErrorContext: e, } } // Verify signature - msg := strconv.Itoa(int(n.State)) + n.Token + - strconv.FormatUint(uint64(n.ParentID), 10) + n.Comment + msg := n.Token + strconv.FormatUint(uint64(n.ParentID), 10) + n.Comment err = util.VerifySignature(n.Signature, n.PublicKey, msg) if err != nil { - return "", convertCommentsErrorFromSignatureError(err) + return "", convertSignatureError(err) } // Verify comment if len(n.Comment) > comments.PolicyCommentLengthMax { return "", backend.PluginUserError{ PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusCommentTextInvalid), - ErrorContext: []string{"exceeds max length"}, + ErrorCode: int(comments.ErrorCodeCommentTextInvalid), + ErrorContext: "exceeds max length", } } - // The comments index must be pulled and updated. The record lock + // The record index must be pulled and updated. The record lock // must be held for the remainder of this function. - m := p.mutex(n.Token) + m := p.mutex(token) m.Lock() defer m.Unlock() - // Get comments index - idx, err := p.commentsIndexLocked(n.State, token) + // Get record index + ridx, err := p.recordIndexLocked(token) if err != nil { return "", err } // Verify parent comment exists if set. A parent ID of 0 means that // this is a base level comment, not a reply to another comment. - if n.ParentID > 0 && !commentExists(*idx, n.ParentID) { + if n.ParentID > 0 && !commentExists(*ridx, n.ParentID) { return "", backend.PluginUserError{ PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusParentIDInvalid), - ErrorContext: []string{"parent ID comment not found"}, + ErrorCode: int(comments.ErrorCodeParentIDInvalid), + ErrorContext: "parent ID comment not found", } } @@ -944,32 +772,25 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { receipt := p.identity.SignMessage([]byte(n.Signature)) ca := comments.CommentAdd{ UserID: n.UserID, - State: n.State, Token: n.Token, ParentID: n.ParentID, Comment: n.Comment, PublicKey: n.PublicKey, Signature: n.Signature, - CommentID: commentIDLatest(*idx) + 1, + CommentID: commentIDLatest(*ridx) + 1, Version: 1, Timestamp: time.Now().Unix(), Receipt: hex.EncodeToString(receipt[:]), } // Save comment - merkleHash, err := p.commentAddSave(ca) + merkleHash, err := p.commentAddSave(treeID, ca) if err != nil { - if errors.Is(err, errRecordNotFound) { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusRecordNotFound), - } - } return "", fmt.Errorf("commentAddSave: %v", err) } // Update index - idx.Comments[ca.CommentID] = commentIndex{ + ridx.Comments[ca.CommentID] = commentIndex{ Adds: map[uint32][]byte{ 1: merkleHash, }, @@ -978,7 +799,7 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { } // Save index - err = p.commentsIndexSaveLocked(n.State, token, *idx) + err = p.recordIndexSaveLocked(token, *ridx) if err != nil { return "", err } @@ -987,7 +808,7 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { ca.Token, ca.CommentID) // Return new comment - c, err := p.comment(ca.State, token, *idx, ca.CommentID) + c, err := p.comment(treeID, *ridx, ca.CommentID) if err != nil { return "", fmt.Errorf("comment %x %v: %v", token, ca.CommentID, err) } @@ -996,7 +817,7 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { nr := comments.NewReply{ Comment: *c, } - reply, err := comments.EncodeNewReply(nr) + reply, err := json.Marshal(nr) if err != nil { return "", err } @@ -1004,80 +825,73 @@ func (p *commentsPlugin) cmdNew(payload string) (string, error) { return string(reply), nil } -func (p *commentsPlugin) cmdEdit(payload string) (string, error) { - log.Tracef("comments cmdEdit: %v", payload) +func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdEdit: %v %x %v", treeID, token, payload) // Decode payload - e, err := comments.DecodeEdit([]byte(payload)) + var e comments.Edit + err := json.Unmarshal([]byte(payload), &e) if err != nil { return "", err } - // Verify state - switch e.State { - case comments.StateUnvetted, comments.StateVetted: - // Allowed; continue - default: + // Verify token + t, err := tokenDecode(e.Token) + if err != nil { return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusStateInvalid), + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeTokenInvalid), + ErrorContext: err.Error(), } } - - // Verify token - token, err := tokenDecodeAnyLength(e.Token) - if err != nil { + if !bytes.Equal(t, token) { + e := fmt.Sprintf("comment token does not match route token: "+ + "got %x, want %x", t, token) return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeTokenInvalid), + ErrorContext: e, } } // Verify signature - msg := strconv.Itoa(int(e.State)) + e.Token + - strconv.FormatUint(uint64(e.ParentID), 10) + e.Comment + msg := e.Token + strconv.FormatUint(uint64(e.ParentID), 10) + e.Comment err = util.VerifySignature(e.Signature, e.PublicKey, msg) if err != nil { - return "", convertCommentsErrorFromSignatureError(err) + return "", convertSignatureError(err) } // Verify comment if len(e.Comment) > comments.PolicyCommentLengthMax { return "", backend.PluginUserError{ PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusCommentTextInvalid), - ErrorContext: []string{"exceeds max length"}, + ErrorCode: int(comments.ErrorCodeCommentTextInvalid), + ErrorContext: "exceeds max length", } } - // The comments index must be pulled and updated. The record lock + // The record index must be pulled and updated. The record lock // must be held for the remainder of this function. - m := p.mutex(e.Token) + m := p.mutex(token) m.Lock() defer m.Unlock() - // Get comments index - idx, err := p.commentsIndexLocked(e.State, token) + // Get record index + ridx, err := p.recordIndexLocked(token) if err != nil { return "", err } // Get the existing comment - cs, err := p.comments(e.State, token, *idx, []uint32{e.CommentID}) + cs, err := p.comments(treeID, *ridx, []uint32{e.CommentID}) if err != nil { - if errors.Is(err, errRecordNotFound) { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusRecordNotFound), - } - } return "", fmt.Errorf("comments %v: %v", e.CommentID, err) } existing, ok := cs[e.CommentID] if !ok { return "", backend.PluginUserError{ PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusCommentNotFound), + ErrorCode: int(comments.ErrorCodeCommentNotFound), } } @@ -1085,7 +899,7 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { if e.UserID != existing.UserID { return "", backend.PluginUserError{ PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusUserUnauthorized), + ErrorCode: int(comments.ErrorCodeUserUnauthorized), } } @@ -1095,8 +909,8 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { e.ParentID, existing.ParentID) return "", backend.PluginUserError{ PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusParentIDInvalid), - ErrorContext: []string{e}, + ErrorCode: int(comments.ErrorCodeParentIDInvalid), + ErrorContext: e, } } @@ -1104,8 +918,8 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { if e.Comment == existing.Comment { return "", backend.PluginUserError{ PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusCommentTextInvalid), - ErrorContext: []string{"comment did not change"}, + ErrorCode: int(comments.ErrorCodeCommentTextInvalid), + ErrorContext: "comment did not change", } } @@ -1113,7 +927,6 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { receipt := p.identity.SignMessage([]byte(e.Signature)) ca := comments.CommentAdd{ UserID: e.UserID, - State: e.State, Token: e.Token, ParentID: e.ParentID, Comment: e.Comment, @@ -1126,16 +939,16 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { } // Save comment - merkle, err := p.commentAddSave(ca) + merkle, err := p.commentAddSave(treeID, ca) if err != nil { - return "", fmt.Errorf("commentSave: %v", err) + return "", fmt.Errorf("commentAddSave: %v", err) } // Update index - idx.Comments[ca.CommentID].Adds[ca.Version] = merkle + ridx.Comments[ca.CommentID].Adds[ca.Version] = merkle // Save index - err = p.commentsIndexSaveLocked(e.State, token, *idx) + err = p.recordIndexSaveLocked(token, *ridx) if err != nil { return "", err } @@ -1144,7 +957,7 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { ca.Token, ca.CommentID) // Return updated comment - c, err := p.comment(e.State, token, *idx, e.CommentID) + c, err := p.comment(treeID, *ridx, e.CommentID) if err != nil { return "", fmt.Errorf("comment %x %v: %v", token, e.CommentID, err) } @@ -1153,7 +966,7 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { er := comments.EditReply{ Comment: *c, } - reply, err := comments.EncodeEditReply(er) + reply, err := json.Marshal(er) if err != nil { return "", err } @@ -1161,78 +974,70 @@ func (p *commentsPlugin) cmdEdit(payload string) (string, error) { return string(reply), nil } -func (p *commentsPlugin) cmdDel(payload string) (string, error) { - log.Tracef("comments cmdDel: %v", payload) +func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdDel: %v %x %v", treeID, token, payload) // Decode payload - d, err := comments.DecodeDel([]byte(payload)) + var d comments.Del + err := json.Unmarshal([]byte(payload), &d) if err != nil { return "", err } - // Verify state - switch d.State { - case comments.StateUnvetted, comments.StateVetted: - // Allowed; continue - default: + // Verify token + t, err := tokenDecode(d.Token) + if err != nil { return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusStateInvalid), + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeTokenInvalid), + ErrorContext: err.Error(), } } - - // Verify token - token, err := tokenDecodeAnyLength(d.Token) - if err != nil { + if !bytes.Equal(t, token) { + e := fmt.Sprintf("comment token does not match route token: "+ + "got %x, want %x", t, token) return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeTokenInvalid), + ErrorContext: e, } } // Verify signature - msg := strconv.Itoa(int(d.State)) + d.Token + - strconv.FormatUint(uint64(d.CommentID), 10) + d.Reason + msg := d.Token + strconv.FormatUint(uint64(d.CommentID), 10) + d.Reason err = util.VerifySignature(d.Signature, d.PublicKey, msg) if err != nil { - return "", convertCommentsErrorFromSignatureError(err) + return "", convertSignatureError(err) } - // The comments index must be pulled and updated. The record lock + // The record index must be pulled and updated. The record lock // must be held for the remainder of this function. - m := p.mutex(d.Token) + m := p.mutex(token) m.Lock() defer m.Unlock() - // Get comments index - idx, err := p.commentsIndexLocked(d.State, token) + // Get record index + ridx, err := p.recordIndexLocked(token) if err != nil { return "", err } // Get the existing comment - cs, err := p.comments(d.State, token, *idx, []uint32{d.CommentID}) + cs, err := p.comments(treeID, *ridx, []uint32{d.CommentID}) if err != nil { - if errors.Is(err, errRecordNotFound) { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusRecordNotFound), - } - } return "", fmt.Errorf("comments %v: %v", d.CommentID, err) } existing, ok := cs[d.CommentID] if !ok { return "", backend.PluginUserError{ PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusCommentNotFound), + ErrorCode: int(comments.ErrorCodeCommentNotFound), } } // Prepare comment delete receipt := p.identity.SignMessage([]byte(d.Signature)) cd := comments.CommentDel{ - State: d.State, Token: d.Token, CommentID: d.CommentID, Reason: d.Reason, @@ -1245,23 +1050,23 @@ func (p *commentsPlugin) cmdDel(payload string) (string, error) { } // Save comment del - merkle, err := p.commentDelSave(cd) + merkle, err := p.commentDelSave(treeID, cd) if err != nil { return "", fmt.Errorf("commentDelSave: %v", err) } // Update index - cidx, ok := idx.Comments[d.CommentID] + cidx, ok := ridx.Comments[d.CommentID] if !ok { // This should not be possible e := fmt.Sprintf("comment not found in index: %v", d.CommentID) panic(e) } cidx.Del = merkle - idx.Comments[d.CommentID] = cidx + ridx.Comments[d.CommentID] = cidx // Save index - err = p.commentsIndexSaveLocked(d.State, token, *idx) + err = p.recordIndexSaveLocked(token, *ridx) if err != nil { return "", err } @@ -1271,24 +1076,22 @@ func (p *commentsPlugin) cmdDel(payload string) (string, error) { for _, v := range cidx.Adds { merkles = append(merkles, v) } - tlogID := tlogIDFromCommentState(d.State) - _ = tlogID - err = p.tlog.Del(tlogID, token, merkles) + err = p.tlog.BlobsDel(treeID, merkles) if err != nil { return "", fmt.Errorf("del: %v", err) } // Return updated comment - c, err := p.comment(d.State, token, *idx, d.CommentID) + c, err := p.comment(treeID, *ridx, d.CommentID) if err != nil { - return "", fmt.Errorf("comment %x %v: %v", token, d.CommentID, err) + return "", fmt.Errorf("comment %v: %v", d.CommentID, err) } // Prepare reply dr := comments.DelReply{ Comment: *c, } - reply, err := comments.EncodeDelReply(dr) + reply, err := json.Marshal(dr) if err != nil { return "", err } @@ -1296,83 +1099,32 @@ func (p *commentsPlugin) cmdDel(payload string) (string, error) { return string(reply), nil } -// calcVoteScore returns the vote score for the provided comment index. The -// returned values are the downvotes and upvotes, respectively. -func calcVoteScore(cidx commentIndex) (uint64, uint64) { - // Find the vote score by replaying all existing votes from all - // users. The net effect of a new vote on a comment score depends - // on the previous vote from that uuid. Example, a user upvotes a - // comment that they have already upvoted, the resulting vote score - // is 0 due to the second upvote removing the original upvote. - var upvotes uint64 - var downvotes uint64 - for _, votes := range cidx.Votes { - // Calculate the vote score that this user is contributing. This - // can only ever be -1, 0, or 1. - var score int64 - for _, v := range votes { - vote := int64(v.Vote) - switch { - case score == 0: - // No previous vote. New vote becomes the score. - score = vote - - case score == vote: - // New vote is the same as the previous vote. The vote gets - // removed from the score, making the score 0. - score = 0 - - case score != vote: - // New vote is different than the previous vote. New vote - // becomes the score. - score = vote - } - } - - // Add the net result of all votes from this user to the totals. - switch score { - case 0: - // Nothing to do - case -1: - downvotes++ - case 1: - upvotes++ - default: - // Something went wrong - e := fmt.Errorf("unexpected vote score %v", score) - panic(e) - } - } - - return downvotes, upvotes -} - -func (p *commentsPlugin) cmdVote(payload string) (string, error) { - log.Tracef("comments cmdVote: %v", payload) +func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdVote: %v %x %v", treeID, token, payload) // Decode payload - v, err := comments.DecodeVote([]byte(payload)) + var v comments.Vote + err := json.Unmarshal([]byte(payload), &v) if err != nil { return "", err } - // Verify state - switch v.State { - case comments.StateUnvetted, comments.StateVetted: - // Allowed; continue - default: + // Verify token + t, err := tokenDecode(v.Token) + if err != nil { return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeTokenInvalid), + ErrorContext: err.Error(), } } - - // Verify token - token, err := tokenDecodeAnyLength(v.Token) - if err != nil { + if !bytes.Equal(t, token) { + e := fmt.Sprintf("comment token does not match route token: "+ + "got %x, want %x", t, token) return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeTokenInvalid), + ErrorContext: e, } } @@ -1383,37 +1135,36 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { default: return "", backend.PluginUserError{ PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusVoteInvalid), + ErrorCode: int(comments.ErrorCodeVoteInvalid), } } // Verify signature - msg := strconv.Itoa(int(v.State)) + v.Token + - strconv.FormatUint(uint64(v.CommentID), 10) + + msg := v.Token + strconv.FormatUint(uint64(v.CommentID), 10) + strconv.FormatInt(int64(v.Vote), 10) err = util.VerifySignature(v.Signature, v.PublicKey, msg) if err != nil { - return "", convertCommentsErrorFromSignatureError(err) + return "", convertSignatureError(err) } - // The comments index must be pulled and updated. The record lock + // The record index must be pulled and updated. The record lock // must be held for the remainder of this function. - m := p.mutex(v.Token) + m := p.mutex(token) m.Lock() defer m.Unlock() - // Get comments index - idx, err := p.commentsIndexLocked(v.State, token) + // Get record index + ridx, err := p.recordIndexLocked(token) if err != nil { return "", err } // Verify comment exists - cidx, ok := idx.Comments[v.CommentID] + cidx, ok := ridx.Comments[v.CommentID] if !ok { return "", backend.PluginUserError{ PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusCommentNotFound), + ErrorCode: int(comments.ErrorCodeCommentNotFound), } } @@ -1425,19 +1176,13 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { if len(uvotes) > comments.PolicyVoteChangesMax { return "", backend.PluginUserError{ PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusVoteChangesMax), + ErrorCode: int(comments.ErrorCodeVoteChangesMax), } } // Verify user is not voting on their own comment - cs, err := p.comments(v.State, token, *idx, []uint32{v.CommentID}) + cs, err := p.comments(treeID, *ridx, []uint32{v.CommentID}) if err != nil { - if errors.Is(err, errRecordNotFound) { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusRecordNotFound), - } - } return "", fmt.Errorf("comments %v: %v", v.CommentID, err) } c, ok := cs[v.CommentID] @@ -1447,15 +1192,14 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { if v.UserID == c.UserID { return "", backend.PluginUserError{ PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusVoteInvalid), - ErrorContext: []string{"user cannot vote on their own comment"}, + ErrorCode: int(comments.ErrorCodeVoteInvalid), + ErrorContext: "user cannot vote on their own comment", } } // Prepare comment vote receipt := p.identity.SignMessage([]byte(v.Signature)) cv := comments.CommentVote{ - State: v.State, UserID: v.UserID, Token: v.Token, CommentID: v.CommentID, @@ -1467,7 +1211,7 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { } // Save comment vote - merkle, err := p.commentVoteSave(cv) + merkle, err := p.commentVoteSave(treeID, cv) if err != nil { return "", fmt.Errorf("commentVoteSave: %v", err) } @@ -1483,11 +1227,11 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { }) cidx.Votes[cv.UserID] = votes - // Update the comments index - idx.Comments[cv.CommentID] = cidx + // Update the record index + ridx.Comments[cv.CommentID] = cidx // Save index - err = p.commentsIndexSaveLocked(cv.State, token, *idx) + err = p.recordIndexSaveLocked(token, *ridx) if err != nil { return "", err } @@ -1502,7 +1246,7 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { Timestamp: cv.Timestamp, Receipt: cv.Receipt, } - reply, err := comments.EncodeVoteReply(vr) + reply, err := json.Marshal(vr) if err != nil { return "", err } @@ -1510,50 +1254,25 @@ func (p *commentsPlugin) cmdVote(payload string) (string, error) { return string(reply), nil } -func (p *commentsPlugin) cmdGet(payload string) (string, error) { - log.Tracef("comments cmdGet: %v", payload) +func (p *commentsPlugin) cmdGet(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdGet: %v %x %v", treeID, token, payload) // Decode payload - g, err := comments.DecodeGet([]byte(payload)) + var g comments.Get + err := json.Unmarshal([]byte(payload), &g) if err != nil { return "", err } - // Verify state - switch g.State { - case comments.StateUnvetted, comments.StateVetted: - // Allowed; continue - default: - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Verify token - token, err := tokenDecodeAnyLength(g.Token) - if err != nil { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Get comments index - idx, err := p.commentsIndex(g.State, token) + // Get record index + ridx, err := p.recordIndex(token) if err != nil { return "", err } // Get comments - cs, err := p.comments(g.State, token, *idx, g.CommentIDs) + cs, err := p.comments(treeID, *ridx, g.CommentIDs) if err != nil { - if errors.Is(err, errRecordNotFound) { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusRecordNotFound), - } - } return "", fmt.Errorf("comments: %v", err) } @@ -1561,7 +1280,7 @@ func (p *commentsPlugin) cmdGet(payload string) (string, error) { gr := comments.GetReply{ Comments: cs, } - reply, err := comments.EncodeGetReply(gr) + reply, err := json.Marshal(gr) if err != nil { return "", err } @@ -1569,56 +1288,29 @@ func (p *commentsPlugin) cmdGet(payload string) (string, error) { return string(reply), nil } -func (p *commentsPlugin) cmdGetAll(payload string) (string, error) { - log.Tracef("comments cmdGetAll: %v", payload) +func (p *commentsPlugin) cmdGetAll(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdGetAll: %v %x %v", treeID, token, payload) // Decode payload - ga, err := comments.DecodeGetAll([]byte(payload)) + var ga comments.GetAll + err := json.Unmarshal([]byte(payload), &ga) if err != nil { return "", err } - // Verify state - switch ga.State { - case comments.StateUnvetted, comments.StateVetted: - // Allowed; continue - default: - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Verify token - token, err := tokenDecodeAnyLength(ga.Token) - if err != nil { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Get comments index - idx, err := p.commentsIndex(ga.State, token) + // Compile comment IDs + ridx, err := p.recordIndex(token) if err != nil { return "", err } - - // Compile comment IDs - commentIDs := make([]uint32, 0, len(idx.Comments)) - for k := range idx.Comments { + commentIDs := make([]uint32, 0, len(ridx.Comments)) + for k := range ridx.Comments { commentIDs = append(commentIDs, k) } // Get comments - c, err := p.comments(ga.State, token, *idx, commentIDs) + c, err := p.comments(treeID, *ridx, commentIDs) if err != nil { - if errors.Is(err, errRecordNotFound) { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusRecordNotFound), - } - } return "", fmt.Errorf("comments: %v", err) } @@ -1637,7 +1329,7 @@ func (p *commentsPlugin) cmdGetAll(payload string) (string, error) { gar := comments.GetAllReply{ Comments: cs, } - reply, err := comments.EncodeGetAllReply(gar) + reply, err := json.Marshal(gar) if err != nil { return "", err } @@ -1645,54 +1337,35 @@ func (p *commentsPlugin) cmdGetAll(payload string) (string, error) { return string(reply), nil } -func (p *commentsPlugin) cmdGetVersion(payload string) (string, error) { - log.Tracef("comments cmdGetVersion: %v", payload) +func (p *commentsPlugin) cmdGetVersion(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdGetVersion: %v %x %v", treeID, token, payload) // Decode payload - gv, err := comments.DecodeGetVersion([]byte(payload)) + var gv comments.GetVersion + err := json.Unmarshal([]byte(payload), &gv) if err != nil { return "", err } - // Verify state - switch gv.State { - case comments.StateUnvetted, comments.StateVetted: - // Allowed; continue - default: - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Verify token - token, err := tokenDecodeAnyLength(gv.Token) - if err != nil { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Get comments index - idx, err := p.commentsIndex(gv.State, token) + // Get record index + ridx, err := p.recordIndex(token) if err != nil { return "", err } // Verify comment exists - cidx, ok := idx.Comments[gv.CommentID] + cidx, ok := ridx.Comments[gv.CommentID] if !ok { return "", backend.PluginUserError{ PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusCommentNotFound), + ErrorCode: int(comments.ErrorCodeCommentNotFound), } } if cidx.Del != nil { return "", backend.PluginUserError{ PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusCommentNotFound), - ErrorContext: []string{"comment has been deleted"}, + ErrorCode: int(comments.ErrorCodeCommentNotFound), + ErrorContext: "comment has been deleted", } } merkle, ok := cidx.Adds[gv.Version] @@ -1701,20 +1374,14 @@ func (p *commentsPlugin) cmdGetVersion(payload string) (string, error) { gv.CommentID, gv.Version) return "", backend.PluginUserError{ PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusCommentNotFound), - ErrorContext: []string{e}, + ErrorCode: int(comments.ErrorCodeCommentNotFound), + ErrorContext: e, } } // Get comment add record - adds, err := p.commentAdds(gv.State, token, [][]byte{merkle}) + adds, err := p.commentAdds(treeID, [][]byte{merkle}) if err != nil { - if errors.Is(err, errRecordNotFound) { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusRecordNotFound), - } - } return "", fmt.Errorf("commentAdds: %v", err) } if len(adds) != 1 { @@ -1730,7 +1397,7 @@ func (p *commentsPlugin) cmdGetVersion(payload string) (string, error) { gvr := comments.GetVersionReply{ Comment: c, } - reply, err := comments.EncodeGetVersionReply(gvr) + reply, err := json.Marshal(gvr) if err != nil { return "", err } @@ -1738,46 +1405,27 @@ func (p *commentsPlugin) cmdGetVersion(payload string) (string, error) { return string(reply), nil } -func (p *commentsPlugin) cmdCount(payload string) (string, error) { - log.Tracef("comments cmdCount: %v", payload) +func (p *commentsPlugin) cmdCount(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdCount: %v %x %v", treeID, token, payload) // Decode payload - c, err := comments.DecodeCount([]byte(payload)) + var c comments.Count + err := json.Unmarshal([]byte(payload), &c) if err != nil { return "", err } - // Verify state - switch c.State { - case comments.StateUnvetted, comments.StateVetted: - // Allowed; continue - default: - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Verify token - token, err := tokenDecodeAnyLength(c.Token) - if err != nil { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Get comments index - idx, err := p.commentsIndex(c.State, token) + // Get record index + ridx, err := p.recordIndex(token) if err != nil { return "", err } // Prepare reply cr := comments.CountReply{ - Count: uint64(len(idx.Comments)), + Count: uint32(len(ridx.Comments)), } - reply, err := comments.EncodeCountReply(cr) + reply, err := json.Marshal(cr) if err != nil { return "", err } @@ -1785,37 +1433,18 @@ func (p *commentsPlugin) cmdCount(payload string) (string, error) { return string(reply), nil } -func (p *commentsPlugin) cmdVotes(payload string) (string, error) { - log.Tracef("comments cmdVotes: %v", payload) +func (p *commentsPlugin) cmdVotes(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdVotes: %v %x %v", treeID, token, payload) // Decode payload - v, err := comments.DecodeVotes([]byte(payload)) + var v comments.Votes + err := json.Unmarshal([]byte(payload), &v) if err != nil { return "", err } - // Verify state - switch v.State { - case comments.StateUnvetted, comments.StateVetted: - // Allowed; continue - default: - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Verify token - token, err := tokenDecodeAnyLength(v.Token) - if err != nil { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Get comments index - idx, err := p.commentsIndex(v.State, token) + // Get record index + ridx, err := p.recordIndex(token) if err != nil { return "", err } @@ -1823,7 +1452,7 @@ func (p *commentsPlugin) cmdVotes(payload string) (string, error) { // Compile the comment vote merkles for all votes that were cast // by the specified user. merkles := make([][]byte, 0, 256) - for _, cidx := range idx.Comments { + for _, cidx := range ridx.Comments { voteIdxs, ok := cidx.Votes[v.UserID] if !ok { // User has not cast any votes for this comment @@ -1837,14 +1466,8 @@ func (p *commentsPlugin) cmdVotes(payload string) (string, error) { } // Lookup votes - votes, err := p.commentVotes(v.State, token, merkles) + votes, err := p.commentVotes(treeID, merkles) if err != nil { - if errors.Is(err, errRecordNotFound) { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusRecordNotFound), - } - } return "", fmt.Errorf("commentVotes: %v", err) } @@ -1852,7 +1475,7 @@ func (p *commentsPlugin) cmdVotes(payload string) (string, error) { vr := comments.VotesReply{ Votes: votes, } - reply, err := comments.EncodeVotesReply(vr) + reply, err := json.Marshal(vr) if err != nil { return "", err } @@ -1860,8 +1483,8 @@ func (p *commentsPlugin) cmdVotes(payload string) (string, error) { return string(reply), nil } -func (p *commentsPlugin) cmdTimestamps(payload string) (string, error) { - log.Tracef("comments cmdVotes: %v", payload) +func (p *commentsPlugin) cmdTimestamps(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdTimestamps: %v %x %v", treeID, token, payload) // Decode payload var t comments.Timestamps @@ -1870,28 +1493,8 @@ func (p *commentsPlugin) cmdTimestamps(payload string) (string, error) { return "", err } - // Verify state - switch t.State { - case comments.StateUnvetted, comments.StateVetted: - // Allowed; continue - default: - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Verify token - token, err := tokenDecodeAnyLength(t.Token) - if err != nil { - return "", backend.PluginUserError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorStatusTokenInvalid), - } - } - - // Get comments index - idx, err := p.commentsIndex(t.State, token) + // Get record index + ridx, err := p.recordIndex(token) if err != nil { return "", err } @@ -1899,8 +1502,8 @@ func (p *commentsPlugin) cmdTimestamps(payload string) (string, error) { // If no comment IDs were given then we need to return the // timestamps for all comments. if len(t.CommentIDs) == 0 { - commentIDs := make([]uint32, 0, len(idx.Comments)) - for k := range idx.Comments { + commentIDs := make([]uint32, 0, len(ridx.Comments)) + for k := range ridx.Comments { commentIDs = append(commentIDs, k) } t.CommentIDs = commentIDs @@ -1910,7 +1513,7 @@ func (p *commentsPlugin) cmdTimestamps(payload string) (string, error) { cmts := make(map[uint32][]comments.Timestamp, len(t.CommentIDs)) votes := make(map[uint32][]comments.Timestamp, len(t.CommentIDs)) for _, commentID := range t.CommentIDs { - cidx, ok := idx.Comments[commentID] + cidx, ok := ridx.Comments[commentID] if !ok { // Comment ID does not exist. Skip it. continue @@ -1919,7 +1522,7 @@ func (p *commentsPlugin) cmdTimestamps(payload string) (string, error) { // Get timestamps for adds ts := make([]comments.Timestamp, 0, len(cidx.Adds)+1) for _, v := range cidx.Adds { - t, err := p.timestamp(t.State, token, v) + t, err := p.timestamp(treeID, v) if err != nil { return "", err } @@ -1928,7 +1531,7 @@ func (p *commentsPlugin) cmdTimestamps(payload string) (string, error) { // Get timestamp for del if cidx.Del != nil { - t, err := p.timestamp(t.State, token, cidx.Del) + t, err := p.timestamp(treeID, cidx.Del) if err != nil { return "", err } @@ -1947,7 +1550,7 @@ func (p *commentsPlugin) cmdTimestamps(payload string) (string, error) { ts = make([]comments.Timestamp, 0, len(cidx.Votes)) for _, votes := range cidx.Votes { for _, v := range votes { - t, err := p.timestamp(t.State, token, v.Merkle) + t, err := p.timestamp(treeID, v.Merkle) if err != nil { return "", err } @@ -1976,29 +1579,29 @@ func (p *commentsPlugin) cmdTimestamps(payload string) (string, error) { // // This function satisfies the PluginClient interface. func (p *commentsPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { - log.Tracef("Cmd: %v %v", cmd, payload) + log.Tracef("Cmd: %v %v %x %v", treeID, token, cmd, payload) switch cmd { case comments.CmdNew: - return p.cmdNew(payload) + return p.cmdNew(treeID, token, payload) case comments.CmdEdit: - return p.cmdEdit(payload) + return p.cmdEdit(treeID, token, payload) case comments.CmdDel: - return p.cmdDel(payload) + return p.cmdDel(treeID, token, payload) case comments.CmdVote: - return p.cmdVote(payload) + return p.cmdVote(treeID, token, payload) case comments.CmdGet: - return p.cmdGet(payload) + return p.cmdGet(treeID, token, payload) case comments.CmdGetAll: - return p.cmdGetAll(payload) + return p.cmdGetAll(treeID, token, payload) case comments.CmdGetVersion: - return p.cmdGetVersion(payload) + return p.cmdGetVersion(treeID, token, payload) case comments.CmdCount: - return p.cmdCount(payload) + return p.cmdCount(treeID, token, payload) case comments.CmdVotes: - return p.cmdVotes(payload) + return p.cmdVotes(treeID, token, payload) case comments.CmdTimestamps: - return p.cmdTimestamps(payload) + return p.cmdTimestamps(treeID, token, payload) } return "", backend.ErrPluginCmdInvalid diff --git a/politeiad/backend/tlogbe/plugins/comments/comments_test.go b/politeiad/backend/tlogbe/plugins/comments/comments_test.go index dc66c53f6..c41854277 100644 --- a/politeiad/backend/tlogbe/plugins/comments/comments_test.go +++ b/politeiad/backend/tlogbe/plugins/comments/comments_test.go @@ -6,6 +6,7 @@ package comments import ( "encoding/hex" + "encoding/json" "errors" "strconv" "testing" @@ -16,10 +17,9 @@ import ( "github.com/google/uuid" ) -func commentSignature(t *testing.T, uid *identity.FullIdentity, state comments.StateT, token, msg string, id uint32) string { +func commentSignature(t *testing.T, uid *identity.FullIdentity, token, msg string, id uint32) string { t.Helper() - txt := strconv.Itoa(int(state)) + token + - strconv.FormatInt(int64(id), 10) + msg + txt := token + strconv.FormatInt(int64(id), 10) + msg b := uid.SignMessage([]byte(txt)) return hex.EncodeToString(b[:]) } @@ -96,43 +96,24 @@ func TestCmdNew(t *testing.T) { payload comments.New wantErr error }{ - { - "invalid comment state", - comments.New{ - UserID: uuid.New().String(), - State: comments.StateInvalid, - Token: rec.Token, - ParentID: parentID, - Comment: comment, - PublicKey: uid.Public.String(), - Signature: commentSignature(t, uid, comments.StateInvalid, - rec.Token, comment, parentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusStateInvalid), - }, - }, { "invalid token", comments.New{ UserID: uuid.New().String(), - State: comments.StateUnvetted, Token: "invalid", ParentID: parentID, Comment: comment, PublicKey: uid.Public.String(), - Signature: commentSignature(t, uid, comments.StateUnvetted, - rec.Token, comment, parentID), + Signature: commentSignature(t, uid, rec.Token, comment, parentID), }, backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusTokenInvalid), + ErrorCode: int(comments.ErrorCodeTokenInvalid), }, }, { "invalid signature", comments.New{ UserID: uuid.New().String(), - State: comments.StateUnvetted, Token: rec.Token, ParentID: parentID, Comment: comment, @@ -140,84 +121,62 @@ func TestCmdNew(t *testing.T) { Signature: "invalid", }, backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusSignatureInvalid), + ErrorCode: int(comments.ErrorCodeSignatureInvalid), }, }, { "invalid public key", comments.New{ UserID: uuid.New().String(), - State: comments.StateUnvetted, Token: rec.Token, ParentID: parentID, Comment: comment, PublicKey: "invalid", - Signature: commentSignature(t, uid, comments.StateUnvetted, - rec.Token, comment, parentID), + Signature: commentSignature(t, uid, rec.Token, comment, parentID), }, backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusPublicKeyInvalid), + ErrorCode: int(comments.ErrorCodePublicKeyInvalid), }, }, { "comment max length exceeded", comments.New{ UserID: uuid.New().String(), - State: comments.StateUnvetted, Token: rec.Token, ParentID: parentID, Comment: commentMaxLengthExceeded(t), PublicKey: uid.Public.String(), - Signature: commentSignature(t, uid, comments.StateUnvetted, - rec.Token, commentMaxLengthExceeded(t), parentID), + Signature: commentSignature(t, uid, rec.Token, + commentMaxLengthExceeded(t), parentID), }, backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusCommentTextInvalid), + ErrorCode: int(comments.ErrorCodeCommentTextInvalid), }, }, { "invalid parent ID", comments.New{ UserID: uuid.New().String(), - State: comments.StateUnvetted, Token: rec.Token, ParentID: invalidParentID, Comment: comment, PublicKey: uid.Public.String(), - Signature: commentSignature(t, uid, comments.StateUnvetted, - rec.Token, comment, invalidParentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusParentIDInvalid), - }, - }, - { - "record not found", - comments.New{ - UserID: uuid.New().String(), - State: comments.StateUnvetted, - Token: tokenRandom, - ParentID: parentID, - Comment: comment, - PublicKey: uid.Public.String(), - Signature: commentSignature(t, uid, comments.StateUnvetted, - tokenRandom, comment, parentID), + Signature: commentSignature(t, uid, rec.Token, + comment, invalidParentID), }, backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusRecordNotFound), + ErrorCode: int(comments.ErrorCodeParentIDInvalid), }, }, { "success", comments.New{ UserID: uuid.New().String(), - State: comments.StateUnvetted, Token: rec.Token, ParentID: parentID, Comment: comment, PublicKey: uid.Public.String(), - Signature: commentSignature(t, uid, comments.StateUnvetted, - rec.Token, comment, parentID), + Signature: commentSignature(t, uid, rec.Token, comment, parentID), }, nil, }, @@ -226,7 +185,7 @@ func TestCmdNew(t *testing.T) { for _, test := range tests { t.Run(test.description, func(t *testing.T) { // New Comment - ncEncoded, err := comments.EncodeNew(test.payload) + ncEncoded, err := json.Marshal(test.payload) if err != nil { t.Error(err) } @@ -291,13 +250,11 @@ func TestCmdEdit(t *testing.T) { ncEncoded, err := comments.EncodeNew( comments.New{ UserID: uuid.New().String(), - State: comments.StateUnvetted, Token: rec.Token, ParentID: parentID, Comment: comment, PublicKey: id.Public.String(), - Signature: commentSignature(t, id, comments.StateUnvetted, - rec.Token, comment, parentID), + Signature: commentSignature(t, id, rec.Token, comment, parentID), }, ) if err != nil { @@ -318,45 +275,26 @@ func TestCmdEdit(t *testing.T) { payload comments.Edit wantErr error }{ - { - "invalid comment state", - comments.Edit{ - UserID: nr.Comment.UserID, - State: comments.StateInvalid, - Token: rec.Token, - ParentID: nr.Comment.ParentID, - CommentID: nr.Comment.CommentID, - Comment: commentEdit, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, comments.StateInvalid, - rec.Token, commentEdit, nr.Comment.ParentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusStateInvalid), - }, - }, { "invalid token", comments.Edit{ UserID: nr.Comment.UserID, - State: nr.Comment.State, Token: "invalid", ParentID: nr.Comment.ParentID, CommentID: nr.Comment.CommentID, Comment: commentEdit, PublicKey: id.Public.String(), - Signature: commentSignature(t, id, nr.Comment.State, "invalid", - commentEdit, nr.Comment.ParentID), + Signature: commentSignature(t, id, "invalid", commentEdit, + nr.Comment.ParentID), }, backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusTokenInvalid), + ErrorCode: int(comments.ErrorCodeTokenInvalid), }, }, { "invalid signature", comments.Edit{ UserID: nr.Comment.UserID, - State: nr.Comment.State, Token: rec.Token, ParentID: nr.Comment.ParentID, CommentID: nr.Comment.CommentID, @@ -365,140 +303,116 @@ func TestCmdEdit(t *testing.T) { Signature: "invalid", }, backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusSignatureInvalid), + ErrorCode: int(comments.ErrorCodeSignatureInvalid), }, }, { "invalid public key", comments.Edit{ UserID: nr.Comment.UserID, - State: nr.Comment.State, Token: rec.Token, ParentID: nr.Comment.ParentID, CommentID: nr.Comment.CommentID, Comment: commentEdit, PublicKey: "invalid", - Signature: commentSignature(t, id, nr.Comment.State, rec.Token, + Signature: commentSignature(t, id, rec.Token, commentEdit, nr.Comment.ParentID), }, backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusPublicKeyInvalid), + ErrorCode: int(comments.ErrorCodePublicKeyInvalid), }, }, { "comment max length exceeded", comments.Edit{ UserID: nr.Comment.UserID, - State: nr.Comment.State, Token: rec.Token, ParentID: nr.Comment.ParentID, CommentID: nr.Comment.CommentID, Comment: commentMaxLengthExceeded(t), PublicKey: id.Public.String(), - Signature: commentSignature(t, id, nr.Comment.State, rec.Token, + Signature: commentSignature(t, id, rec.Token, commentMaxLengthExceeded(t), nr.Comment.ParentID), }, backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusCommentTextInvalid), + ErrorCode: int(comments.ErrorCodeCommentTextInvalid), }, }, { "comment id not found", comments.Edit{ UserID: nr.Comment.UserID, - State: nr.Comment.State, Token: rec.Token, ParentID: nr.Comment.ParentID, CommentID: invalidCommentID, Comment: commentEdit, PublicKey: id.Public.String(), - Signature: commentSignature(t, id, nr.Comment.State, rec.Token, + Signature: commentSignature(t, id, rec.Token, commentEdit, nr.Comment.ParentID), }, backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusCommentNotFound), + ErrorCode: int(comments.ErrorCodeCommentNotFound), }, }, { "unauthorized user", comments.Edit{ UserID: uuid.New().String(), - State: nr.Comment.State, Token: rec.Token, ParentID: nr.Comment.ParentID, CommentID: nr.Comment.CommentID, Comment: commentEdit, PublicKey: id.Public.String(), - Signature: commentSignature(t, id, nr.Comment.State, rec.Token, + Signature: commentSignature(t, id, rec.Token, commentEdit, nr.Comment.ParentID), }, backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusUserUnauthorized), + ErrorCode: int(comments.ErrorCodeUserUnauthorized), }, }, { "invalid parent ID", comments.Edit{ UserID: nr.Comment.UserID, - State: nr.Comment.State, Token: rec.Token, ParentID: invalidParentID, CommentID: nr.Comment.CommentID, Comment: commentEdit, PublicKey: id.Public.String(), - Signature: commentSignature(t, id, nr.Comment.State, - rec.Token, commentEdit, invalidParentID), + Signature: commentSignature(t, id, rec.Token, + commentEdit, invalidParentID), }, backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusParentIDInvalid), + ErrorCode: int(comments.ErrorCodeParentIDInvalid), }, }, { "comment did not change", comments.Edit{ UserID: nr.Comment.UserID, - State: nr.Comment.State, Token: rec.Token, ParentID: nr.Comment.ParentID, CommentID: nr.Comment.CommentID, Comment: comment, PublicKey: id.Public.String(), - Signature: commentSignature(t, id, nr.Comment.State, - rec.Token, comment, nr.Comment.ParentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusCommentTextInvalid), - }, - }, - { - "record not found", - comments.Edit{ - UserID: nr.Comment.UserID, - State: nr.Comment.State, - Token: tokenRandom, - ParentID: nr.Comment.ParentID, - CommentID: nr.Comment.ParentID, - Comment: commentEdit, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, nr.Comment.State, - tokenRandom, commentEdit, nr.Comment.ParentID), + Signature: commentSignature(t, id, rec.Token, comment, + nr.Comment.ParentID), }, backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusRecordNotFound), + ErrorCode: int(comments.ErrorCodeCommentTextInvalid), }, }, { "success", comments.Edit{ UserID: nr.Comment.UserID, - State: nr.Comment.State, Token: rec.Token, ParentID: nr.Comment.ParentID, CommentID: nr.Comment.CommentID, Comment: commentEdit, PublicKey: id.Public.String(), - Signature: commentSignature(t, id, nr.Comment.State, - rec.Token, commentEdit, nr.Comment.ParentID), + Signature: commentSignature(t, id, rec.Token, + commentEdit, nr.Comment.ParentID), }, nil, }, @@ -569,13 +483,11 @@ func TestCmdDel(t *testing.T) { ncEncoded, err := comments.EncodeNew( comments.New{ UserID: uuid.New().String(), - State: comments.StateUnvetted, Token: rec.Token, ParentID: parentID, Comment: comment, PublicKey: id.Public.String(), - Signature: commentSignature(t, id, comments.StateUnvetted, - rec.Token, comment, parentID), + Signature: commentSignature(t, id, rec.Token, comment, parentID), }, ) if err != nil { @@ -596,40 +508,23 @@ func TestCmdDel(t *testing.T) { payload comments.Del wantErr error }{ - { - "invalid comment state", - comments.Del{ - State: comments.StateInvalid, - Token: rec.Token, - CommentID: nr.Comment.CommentID, - Reason: reason, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, comments.StateInvalid, - rec.Token, reason, nr.Comment.CommentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusStateInvalid), - }, - }, { "invalid token", comments.Del{ - State: comments.StateUnvetted, Token: "invalid", CommentID: nr.Comment.CommentID, Reason: reason, PublicKey: id.Public.String(), - Signature: commentSignature(t, id, comments.StateUnvetted, - "invalid", reason, nr.Comment.CommentID), + Signature: commentSignature(t, id, "invalid", + reason, nr.Comment.CommentID), }, backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusTokenInvalid), + ErrorCode: int(comments.ErrorCodeTokenInvalid), }, }, { "invalid signature", comments.Del{ - State: comments.StateUnvetted, Token: rec.Token, CommentID: nr.Comment.CommentID, Reason: reason, @@ -637,64 +532,45 @@ func TestCmdDel(t *testing.T) { Signature: "invalid", }, backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusSignatureInvalid), + ErrorCode: int(comments.ErrorCodeSignatureInvalid), }, }, { "invalid public key", comments.Del{ - State: comments.StateUnvetted, Token: rec.Token, CommentID: nr.Comment.CommentID, Reason: reason, PublicKey: "invalid", - Signature: commentSignature(t, id, comments.StateUnvetted, - rec.Token, reason, nr.Comment.CommentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusPublicKeyInvalid), - }, - }, - { - "record not found", - comments.Del{ - State: comments.StateUnvetted, - Token: tokenRandom, - CommentID: nr.Comment.CommentID, - Reason: reason, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, comments.StateUnvetted, - tokenRandom, reason, nr.Comment.CommentID), + Signature: commentSignature(t, id, rec.Token, + reason, nr.Comment.CommentID), }, backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusRecordNotFound), + ErrorCode: int(comments.ErrorCodePublicKeyInvalid), }, }, { "comment id not found", comments.Del{ - State: comments.StateUnvetted, Token: rec.Token, CommentID: 3, Reason: reason, PublicKey: id.Public.String(), - Signature: commentSignature(t, id, comments.StateUnvetted, - rec.Token, reason, 3), + Signature: commentSignature(t, id, rec.Token, reason, 3), }, backend.PluginUserError{ - ErrorCode: int(comments.ErrorStatusCommentNotFound), + ErrorCode: int(comments.ErrorCodeCommentNotFound), }, }, { "success", comments.Del{ - State: comments.StateUnvetted, Token: rec.Token, CommentID: nr.Comment.CommentID, Reason: reason, PublicKey: id.Public.String(), - Signature: commentSignature(t, id, comments.StateUnvetted, - rec.Token, reason, nr.Comment.CommentID), + Signature: commentSignature(t, id, rec.Token, + reason, nr.Comment.CommentID), }, nil, }, @@ -736,8 +612,8 @@ func TestCmdDel(t *testing.T) { } -// TODO don't hardcode this. It breaks editors. It should also be based on the -// policy. +// TODO make this a function that dynamically builds the string instead of +// hardcoding it. func commentMaxLengthExceeded(t *testing.T) string { t.Helper() diff --git a/politeiad/backend/tlogbe/plugins/comments/index.go b/politeiad/backend/tlogbe/plugins/comments/index.go new file mode 100644 index 000000000..404125401 --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/comments/index.go @@ -0,0 +1,114 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package comments + +import ( + "encoding/json" + "errors" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/decred/politeia/politeiad/plugins/comments" + "github.com/decred/politeia/util" +) + +// voteIndex contains the comment vote and the merkle leaf hash of the vote +// record. +type voteIndex struct { + Vote comments.VoteT `json:"vote"` + Merkle []byte `json:"merkle"` // Merkle leaf hash +} + +// commentIndex contains the merkle leaf hashes of all comment add, dels, and +// votes for a comment ID. +type commentIndex struct { + Adds map[uint32][]byte `json:"adds"` // [version]merkleLeafHash + Del []byte `json:"del"` + + // Votes contains the vote history for each uuid that voted on the + // comment. This data is cached because the effect of a new vote + // on a comment depends on the previous vote from that uuid. + // Example, a user upvotes a comment that they have already + // upvoted, the resulting vote score is 0 due to the second upvote + // removing the original upvote. + Votes map[string][]voteIndex `json:"votes"` // [uuid]votes +} + +// recordIndex contains the indexes for all comments made on a record. +type recordIndex struct { + Comments map[uint32]commentIndex `json:"comments"` // [commentID]comment +} + +// recordIndexPath accepts full length token or token prefixes, but always uses +// prefix when generating the comments index path string. +func (p *commentsPlugin) recordIndexPath(token []byte) (string, error) { + tp := util.TokenPrefix(token) + fn := strings.Replace(filenameRecordIndex, "{tokenPrefix}", tp, 1) + return filepath.Join(p.dataDir, fn), nil +} + +// recordIndexLocked returns the cached recordIndex for the provided record. +// If a cached recordIndex does not exist, a new one will be returned. +// +// This function must be called WITH the lock held. +func (p *commentsPlugin) recordIndexLocked(token []byte) (*recordIndex, error) { + fp, err := p.recordIndexPath(token) + if err != nil { + return nil, err + } + b, err := ioutil.ReadFile(fp) + if err != nil { + var e *os.PathError + if errors.As(err, &e) && !os.IsExist(err) { + // File does't exist. Return a new recordIndex instead. + return &recordIndex{ + Comments: make(map[uint32]commentIndex), + }, nil + } + return nil, err + } + + var ridx recordIndex + err = json.Unmarshal(b, &ridx) + if err != nil { + return nil, err + } + + return &ridx, nil +} + +// recordIndex returns the cached recordIndex for the provided record. If a +// cached recordIndex does not exist, a new one will be returned. +// +// This function must be called WITHOUT the lock held. +func (p *commentsPlugin) recordIndex(token []byte) (*recordIndex, error) { + m := p.mutex(token) + m.Lock() + defer m.Unlock() + + return p.recordIndexLocked(token) +} + +// recordIndexSaveLocked saves the provided recordIndex to the comments +// plugin data dir. +// +// This function must be called WITH the lock held. +func (p *commentsPlugin) recordIndexSaveLocked(token []byte, ridx recordIndex) error { + b, err := json.Marshal(ridx) + if err != nil { + return err + } + fp, err := p.recordIndexPath(token) + if err != nil { + return err + } + err = ioutil.WriteFile(fp, b, 0664) + if err != nil { + return err + } + return nil +} diff --git a/politeiad/backend/tlogbe/plugins/plugins.go b/politeiad/backend/tlogbe/plugins/plugins.go index 021cb3b06..5591840da 100644 --- a/politeiad/backend/tlogbe/plugins/plugins.go +++ b/politeiad/backend/tlogbe/plugins/plugins.go @@ -22,18 +22,16 @@ const ( type TlogClient interface { // BlobsSave saves the provided blobs to the tlog backend. Note, // hashes contains the hashes of the data encoded in the blobs. The - // hashes must share the same ordering as the blobs. - BlobsSave(treeID int64, keyPrefix string, blobs, hashes [][]byte, - encrypt bool) ([][]byte, error) + // hashes must share the same ordering as the blobs. The blobs will + // be encypted prior to being saved if the tlog instance has an + // encryption key set. + BlobsSave(treeID int64, keyPrefix string, blobs, + hashes [][]byte) ([][]byte, error) // BlobsDel deletes the blobs that correspond to the provided // merkle leaf hashes. BlobsDel(treeID int64, merkles [][]byte) error - // MerklesByKeyPrefix returns the merkle leaf hashes for all blobs - // that match the key prefix. - MerklesByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) - // BlobsByMerkle returns the blobs with the provided merkle leaf // hashes. If a blob does not exist it will not be included in the // returned map. @@ -42,6 +40,10 @@ type TlogClient interface { // BlobsByKeyPrefix returns all blobs that match the key prefix. BlobsByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) + // MerklesByKeyPrefix returns the merkle leaf hashes for all blobs + // that match the key prefix. + MerklesByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) + // Timestamp returns the timestamp for the data blob that // corresponds to the provided merkle leaf hash. Timestamp(treeID int64, merkle []byte) (*backend.Timestamp, error) diff --git a/politeiad/backend/tlogbe/tlog/tlog.go b/politeiad/backend/tlogbe/tlog/tlog.go index 339a705c6..daa5c09e5 100644 --- a/politeiad/backend/tlogbe/tlog/tlog.go +++ b/politeiad/backend/tlogbe/tlog/tlog.go @@ -58,8 +58,6 @@ var ( // can be because a tree does not exists for the provided tree id // or when a tree does exist but the specified record version does // not exist. - // TODO replace this with a backend error. These errors should only - // be for when the backend doesn't have one. ErrRecordNotFound = errors.New("record not found") // ErrNoFileChanges is emitted when there are no files being diff --git a/politeiad/backend/tlogbe/tlog/tlogclient.go b/politeiad/backend/tlogbe/tlog/tlogclient.go index 3a65e4806..09c6c67b6 100644 --- a/politeiad/backend/tlogbe/tlog/tlogclient.go +++ b/politeiad/backend/tlogbe/tlog/tlogclient.go @@ -19,8 +19,8 @@ import ( // the same ordering as the blobs. // // This function satisfies the plugins.TlogClient interface. -func (t *Tlog) BlobsSave(treeID int64, keyPrefix string, blobs, hashes [][]byte, encrypt bool) ([][]byte, error) { - log.Tracef("%v BlobsSave: %v %v %v", t.id, treeID, keyPrefix, encrypt) +func (t *Tlog) BlobsSave(treeID int64, keyPrefix string, blobs, hashes [][]byte) ([][]byte, error) { + log.Tracef("%v BlobsSave: %v %v", t.id, treeID, keyPrefix) // Sanity check if len(blobs) != len(hashes) { @@ -46,8 +46,8 @@ func (t *Tlog) BlobsSave(treeID int64, keyPrefix string, blobs, hashes [][]byte, return nil, ErrTreeIsFrozen } - // Encrypt blobs if specified - if encrypt { + // Encrypt blobs if an encryption key has been set + if t.encryptionKey != nil { for k, v := range blobs { e, err := t.encrypt(v) if err != nil { @@ -239,36 +239,6 @@ func (t *Tlog) BlobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, return b, nil } -// MerklesByKeyPrefix returns the merkle leaf hashes for all blobs that match -// the key prefix. -// -// This function satisfies the plugins.TlogClient interface. -func (t *Tlog) MerklesByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) { - log.Tracef("%v MerklesByKeyPrefix: %v %v", t.id, treeID, keyPrefix) - - // Verify tree exists - if !t.TreeExists(treeID) { - return nil, ErrRecordNotFound - } - - // Get leaves - leaves, err := t.trillian.leavesAll(treeID) - if err != nil { - return nil, fmt.Errorf("leavesAll: %v", err) - } - - // Walk leaves and aggregate the merkle leaf hashes with a matching - // key prefix. - merkles := make([][]byte, 0, len(leaves)) - for _, v := range leaves { - if bytes.HasPrefix(v.ExtraData, []byte(keyPrefix)) { - merkles = append(merkles, v.MerkleLeafHash) - } - } - - return merkles, nil -} - // BlobsByKeyPrefix returns all blobs that match the provided key prefix. // // This function satisfies the plugins.TlogClient interface. @@ -332,6 +302,36 @@ func (t *Tlog) BlobsByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error return b, nil } +// MerklesByKeyPrefix returns the merkle leaf hashes for all blobs that match +// the key prefix. +// +// This function satisfies the plugins.TlogClient interface. +func (t *Tlog) MerklesByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) { + log.Tracef("%v MerklesByKeyPrefix: %v %v", t.id, treeID, keyPrefix) + + // Verify tree exists + if !t.TreeExists(treeID) { + return nil, ErrRecordNotFound + } + + // Get leaves + leaves, err := t.trillian.leavesAll(treeID) + if err != nil { + return nil, fmt.Errorf("leavesAll: %v", err) + } + + // Walk leaves and aggregate the merkle leaf hashes with a matching + // key prefix. + merkles := make([][]byte, 0, len(leaves)) + for _, v := range leaves { + if bytes.HasPrefix(v.ExtraData, []byte(keyPrefix)) { + merkles = append(merkles, v.MerkleLeafHash) + } + } + + return merkles, nil +} + // Timestamp returns the timestamp for the data blob that corresponds to the // provided merkle leaf hash. // diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index dffb0e834..b48d24e53 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -938,9 +938,6 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b // Get existing record r, err := t.vetted.RecordLatest(treeID) if err != nil { - if errors.Is(err, tlog.ErrRecordNotFound) { - return nil, backend.ErrRecordNotFound - } return nil, fmt.Errorf("RecordLatest: %v", err) } @@ -1136,9 +1133,6 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ // Get existing record r, err := t.vetted.RecordLatest(treeID) if err != nil { - if errors.Is(err, tlog.ErrRecordNotFound) { - return backend.ErrRecordNotFound - } return fmt.Errorf("RecordLatest: %v", err) } @@ -1264,9 +1258,6 @@ func (t *tlogBackend) GetVetted(token []byte, version string) (*backend.Record, r, err := t.vetted.Record(treeID, v) if err != nil { - if errors.Is(err, tlog.ErrRecordNotFound) { - err = backend.ErrRecordNotFound - } return nil, err } diff --git a/politeiad/plugins/comments/comments.go b/politeiad/plugins/comments/comments.go index 72369b34f..b92c68f5b 100644 --- a/politeiad/plugins/comments/comments.go +++ b/politeiad/plugins/comments/comments.go @@ -6,14 +6,6 @@ // records. package comments -import ( - "encoding/json" -) - -type StateT int -type VoteT int -type ErrorStatusT int - // TODO add a hint to comments that can be used freely by the client. This // is how we'll distinguish proposal comments from update comments. const ( @@ -31,16 +23,6 @@ const ( CmdVotes = "votes" // Get comment votes CmdTimestamps = "timestamps" // Get timestamps - // Record states - StateInvalid StateT = 0 - StateUnvetted StateT = 1 - StateVetted StateT = 2 - - // Comment vote types - VoteInvalid VoteT = 0 - VoteDownvote VoteT = -1 - VoteUpvote VoteT = 1 - // TODO make these default settings, not policies // PolicyCommentLengthMax is the maximum number of characters // accepted for comments. @@ -50,29 +32,30 @@ const ( // change their vote on a comment. This prevents a malicious user // from being able to spam comment votes. PolicyVoteChangesMax = 5 +) + +// ErrorCodeT represents a error that was caused by the user. +type ErrorCodeT int +const ( // Error status codes - // TODO number status codes - ErrorStatusInvalid ErrorStatusT = 0 - ErrorStatusStateInvalid ErrorStatusT = iota - ErrorStatusTokenInvalid - ErrorStatusPublicKeyInvalid - ErrorStatusSignatureInvalid - ErrorStatusCommentTextInvalid - ErrorStatusRecordNotFound - ErrorStatusCommentNotFound - ErrorStatusUserUnauthorized - ErrorStatusParentIDInvalid - ErrorStatusVoteInvalid - ErrorStatusVoteChangesMax + ErrorCodeInvalid ErrorCodeT = 0 + ErrorCodeTokenInvalid ErrorCodeT = 1 + ErrorCodePublicKeyInvalid ErrorCodeT = 2 + ErrorCodeSignatureInvalid ErrorCodeT = 3 + ErrorCodeCommentTextInvalid ErrorCodeT = 4 + ErrorCodeCommentNotFound ErrorCodeT = 5 + ErrorCodeUserUnauthorized ErrorCodeT = 6 + ErrorCodeParentIDInvalid ErrorCodeT = 7 + ErrorCodeVoteInvalid ErrorCodeT = 8 + ErrorCodeVoteChangesMax ErrorCodeT = 9 ) // Comment represent a record comment. // -// Signature is the client signature of State+Token+ParentID+Comment. +// Signature is the client signature of Token+ParentID+Comment. type Comment struct { UserID string `json:"userid"` // Unique user ID - State StateT `json:"state"` // Record state Token string `json:"token"` // Record token ParentID uint32 `json:"parentid"` // Parent comment ID if reply Comment string `json:"comment"` // Comment text @@ -96,7 +79,6 @@ type Comment struct { type CommentAdd struct { // Data generated by client UserID string `json:"userid"` // Unique user ID - State StateT `json:"state"` // Record state Token string `json:"token"` // Record token ParentID uint32 `json:"parentid"` // Parent comment ID Comment string `json:"comment"` // Comment @@ -111,15 +93,14 @@ type CommentAdd struct { } // CommentDel is the structure that is saved to disk when a comment is deleted. -// Some additional fields like ParentID and UserID are required to be saved since -// all the CommentAdd records will be deleted and the client needs these +// Some additional fields like ParentID and UserID are required to be saved +// since all the CommentAdd records will be deleted and the client needs these // additional fields to properly display the deleted comment in the comment // hierarchy. // -// Signature is the client signature of the State+Token+CommentID+Reason +// Signature is the client signature of the Token+CommentID+Reason type CommentDel struct { // Data generated by client - State StateT `json:"state"` // Record state Token string `json:"token"` // Record token CommentID uint32 `json:"commentid"` // Comment ID Reason string `json:"reason"` // Reason for deleting @@ -133,14 +114,27 @@ type CommentDel struct { Receipt string `json:"receipt"` // Server sig of client sig } +// VoteT represents a comment upvote/downvote. +type VoteT int + +const ( + // VoteInvalid is an invalid comment vote. + VoteInvalid VoteT = 0 + + // VoteDownvote represents a comment downvote. + VoteDownvote VoteT = -1 + + // VoteUpvote represents a comment upvote. + VoteUpvote VoteT = 1 +) + // CommentVote is the structure that is saved to disk when a comment is voted // on. // -// Signature is the client signature of the State+Token+CommentID+Vote. +// Signature is the client signature of the Token+CommentID+Vote. type CommentVote struct { // Data generated by client UserID string `json:"userid"` // Unique user ID - State StateT `json:"state"` // Record state Token string `json:"token"` // Record token CommentID uint32 `json:"commentid"` // Comment ID Vote VoteT `json:"vote"` // Upvote or downvote @@ -157,10 +151,9 @@ type CommentVote struct { // The parent ID is used to reply to an existing comment. A parent ID of 0 // indicates that the comment is a base level comment and not a reply commment. // -// Signature is the client signature of State+Token+ParentID+Comment. +// Signature is the client signature of Token+ParentID+Comment. type New struct { UserID string `json:"userid"` // Unique user ID - State StateT `json:"state"` // Record state Token string `json:"token"` // Record token ParentID uint32 `json:"parentid"` // Parent comment ID Comment string `json:"comment"` // Comment text @@ -168,47 +161,16 @@ type New struct { Signature string `json:"signature"` // Client signature } -// EncodeNew encodes a New into a JSON byte slice. -func EncodeNew(n New) ([]byte, error) { - return json.Marshal(n) -} - -// DecodeNew decodes a JSON byte slice into a New. -func DecodeNew(payload []byte) (*New, error) { - var n New - err := json.Unmarshal(payload, &n) - if err != nil { - return nil, err - } - return &n, nil -} - // NewReply is the reply to the New command. type NewReply struct { Comment Comment `json:"comment"` } -// EncodeNew encodes a NewReply into a JSON byte slice. -func EncodeNewReply(nr NewReply) ([]byte, error) { - return json.Marshal(nr) -} - -// DecodeNew decodes a JSON byte slice into a NewReply. -func DecodeNewReply(payload []byte) (*NewReply, error) { - var nr NewReply - err := json.Unmarshal(payload, &nr) - if err != nil { - return nil, err - } - return &nr, nil -} - // Edit edits an existing comment. // -// Signature is the client signature of State+Token+ParentID+Comment. +// Signature is the client signature of Token+ParentID+Comment. type Edit struct { UserID string `json:"userid"` // Unique user ID - State StateT `json:"state"` // Record state Token string `json:"token"` // Record token ParentID uint32 `json:"parentid"` // Parent comment ID CommentID uint32 `json:"commentid"` // Comment ID @@ -217,46 +179,15 @@ type Edit struct { Signature string `json:"signature"` // Client signature } -// EncodeEdit encodes a Edit into a JSON byte slice. -func EncodeEdit(e Edit) ([]byte, error) { - return json.Marshal(e) -} - -// DecodeEdit decodes a JSON byte slice into a Edit. -func DecodeEdit(payload []byte) (*Edit, error) { - var e Edit - err := json.Unmarshal(payload, &e) - if err != nil { - return nil, err - } - return &e, nil -} - // EditReply is the reply to the Edit command. type EditReply struct { Comment Comment `json:"comment"` } -// EncodeEdit encodes a EditReply into a JSON byte slice. -func EncodeEditReply(er EditReply) ([]byte, error) { - return json.Marshal(er) -} - -// DecodeEdit decodes a JSON byte slice into a EditReply. -func DecodeEditReply(payload []byte) (*EditReply, error) { - var er EditReply - err := json.Unmarshal(payload, &er) - if err != nil { - return nil, err - } - return &er, nil -} - // Del permanently deletes all versions of the provided comment. // -// Signature is the client signature of the State+Token+CommentID+Reason +// Signature is the client signature of the Token+CommentID+Reason type Del struct { - State StateT `json:"state"` // Record state Token string `json:"token"` // Record token CommentID uint32 `json:"commentid"` // Comment ID Reason string `json:"reason"` // Reason for deletion @@ -264,41 +195,11 @@ type Del struct { Signature string `json:"signature"` // Client signature } -// EncodeDel encodes a Del into a JSON byte slice. -func EncodeDel(d Del) ([]byte, error) { - return json.Marshal(d) -} - -// DecodeDel decodes a JSON byte slice into a Del. -func DecodeDel(payload []byte) (*Del, error) { - var d Del - err := json.Unmarshal(payload, &d) - if err != nil { - return nil, err - } - return &d, nil -} - // DelReply is the reply to the Del command. type DelReply struct { Comment Comment `json:"comment"` } -// EncodeDelReply encodes a DelReply into a JSON byte slice. -func EncodeDelReply(dr DelReply) ([]byte, error) { - return json.Marshal(dr) -} - -// DecodeDelReply decodes a JSON byte slice into a DelReply. -func DecodeDelReply(payload []byte) (*DelReply, error) { - var dr DelReply - err := json.Unmarshal(payload, &dr) - if err != nil { - return nil, err - } - return &dr, nil -} - // Vote casts a comment vote (upvote or downvote). // // The effect of a new vote on a comment score depends on the previous vote @@ -307,10 +208,9 @@ func DecodeDelReply(payload []byte) (*DelReply, error) { // original upvote. The public key cannot be relied on to remain the same for // each user so a user ID must be included. // -// Signature is the client signature of the State+Token+CommentID+Vote. +// Signature is the client signature of the Token+CommentID+Vote. type Vote struct { UserID string `json:"userid"` // Unique user ID - State StateT `json:"state"` // Record state Token string `json:"token"` // Record token CommentID uint32 `json:"commentid"` // Comment ID Vote VoteT `json:"vote"` // Upvote or downvote @@ -318,21 +218,6 @@ type Vote struct { Signature string `json:"signature"` // Client signature } -// EncodeVote encodes a Vote into a JSON byte slice. -func EncodeVote(v Vote) ([]byte, error) { - return json.Marshal(v) -} - -// DecodeVote decodes a JSON byte slice into a Vote. -func DecodeVote(payload []byte) (*Vote, error) { - var v Vote - err := json.Unmarshal(payload, &v) - if err != nil { - return nil, err - } - return &v, nil -} - // VoteReply is the reply to the Vote command. type VoteReply struct { Downvotes uint64 `json:"downvotes"` // Tolal downvotes on comment @@ -341,45 +226,13 @@ type VoteReply struct { Receipt string `json:"receipt"` // Server signature of client signature } -// EncodeVoteReply encodes a VoteReply into a JSON byte slice. -func EncodeVoteReply(vr VoteReply) ([]byte, error) { - return json.Marshal(vr) -} - -// DecodeVoteReply decodes a JSON byte slice into a VoteReply. -func DecodeVoteReply(payload []byte) (*VoteReply, error) { - var vr VoteReply - err := json.Unmarshal(payload, &vr) - if err != nil { - return nil, err - } - return &vr, nil -} - // Get returns the latest version of the comments for the provided comment IDs. // An error is not returned if a comment is not found for one or more of the // comment IDs. Those entries will simply not be included in the reply. type Get struct { - State StateT `json:"state"` - Token string `json:"token"` CommentIDs []uint32 `json:"commentids"` } -// EncodeGet encodes a Get into a JSON byte slice. -func EncodeGet(g Get) ([]byte, error) { - return json.Marshal(g) -} - -// DecodeGet decodes a JSON byte slice into a Get. -func DecodeGet(payload []byte) (*Get, error) { - var g Get - err := json.Unmarshal(payload, &g) - if err != nil { - return nil, err - } - return &g, nil -} - // GetReply is the reply to the Get command. The returned map will not include // an entry for any comment IDs that did not correspond to an actual comment. // It is the responsibility of the caller to ensure that a comment was returned @@ -388,188 +241,43 @@ type GetReply struct { Comments map[uint32]Comment `json:"comments"` // [commentID]Comment } -// EncodeGetReply encodes a GetReply into a JSON byte slice. -func EncodeGetReply(gr GetReply) ([]byte, error) { - return json.Marshal(gr) -} - -// DecodeGetReply decodes a JSON byte slice into a GetReply. -func DecodeGetReply(payload []byte) (*GetReply, error) { - var gr GetReply - err := json.Unmarshal(payload, &gr) - if err != nil { - return nil, err - } - return &gr, nil -} - -// GetAll returns the latest version off all comments for the provided record. -type GetAll struct { - State StateT `json:"state"` - Token string `json:"token"` -} - -// EncodeGetAll encodes a GetAll into a JSON byte slice. -func EncodeGetAll(ga GetAll) ([]byte, error) { - return json.Marshal(ga) -} - -// DecodeGetAll decodes a JSON byte slice into a GetAll. -func DecodeGetAll(payload []byte) (*GetAll, error) { - var ga GetAll - err := json.Unmarshal(payload, &ga) - if err != nil { - return nil, err - } - return &ga, nil -} +// GetAll returns the latest version off all comments for a record. +type GetAll struct{} // GetAllReply is the reply to the GetAll command. type GetAllReply struct { Comments []Comment `json:"comments"` } -// EncodeGetAllReply encodes a GetAllReply into a JSON byte slice. -func EncodeGetAllReply(gar GetAllReply) ([]byte, error) { - return json.Marshal(gar) -} - -// DecodeGetAllReply decodes a JSON byte slice into a GetAllReply. -func DecodeGetAllReply(payload []byte) (*GetAllReply, error) { - var gar GetAllReply - err := json.Unmarshal(payload, &gar) - if err != nil { - return nil, err - } - return &gar, nil -} - // GetVersion returns a specific version of a comment. type GetVersion struct { - State StateT `json:"state"` - Token string `json:"token"` CommentID uint32 `json:"commentid"` Version uint32 `json:"version"` } -// EncodeGetVersion encodes a GetVersion into a JSON byte slice. -func EncodeGetVersion(gv GetVersion) ([]byte, error) { - return json.Marshal(gv) -} - -// DecodeGetVersion decodes a JSON byte slice into a GetVersion. -func DecodeGetVersion(payload []byte) (*GetVersion, error) { - var gv GetVersion - err := json.Unmarshal(payload, &gv) - if err != nil { - return nil, err - } - return &gv, nil -} - // GetVersionReply is the reply to the GetVersion command. type GetVersionReply struct { Comment Comment `json:"comment"` } -// EncodeGetVersionReply encodes a GetVersionReply into a JSON byte slice. -func EncodeGetVersionReply(gvr GetVersionReply) ([]byte, error) { - return json.Marshal(gvr) -} - -// DecodeGetVersionReply decodes a JSON byte slice into a GetVersionReply. -func DecodeGetVersionReply(payload []byte) (*GetVersionReply, error) { - var gvr GetVersionReply - err := json.Unmarshal(payload, &gvr) - if err != nil { - return nil, err - } - return &gvr, nil -} - -// Count returns the comments count for the provided record. -type Count struct { - State StateT `json:"state"` - Token string `json:"token"` -} - -// EncodeCount encodes a Count into a JSON byte slice. -func EncodeCount(c Count) ([]byte, error) { - return json.Marshal(c) -} - -// DecodeCount decodes a JSON byte slice into a Count. -func DecodeCount(payload []byte) (*Count, error) { - var c Count - err := json.Unmarshal(payload, &c) - if err != nil { - return nil, err - } - return &c, nil -} +// Count returns the comments count for a record. +type Count struct{} // CountReply is the reply to the Count command. type CountReply struct { - Count uint64 `json:"count"` -} - -// EncodeCountReply encodes a CountReply into a JSON byte slice. -func EncodeCountReply(cr CountReply) ([]byte, error) { - return json.Marshal(cr) -} - -// DecodeCountReply decodes a JSON byte slice into a CountReply. -func DecodeCountReply(payload []byte) (*CountReply, error) { - var cr CountReply - err := json.Unmarshal(payload, &cr) - if err != nil { - return nil, err - } - return &cr, nil + Count uint32 `json:"count"` } // Votes returns the comment votes that meet the provided filtering criteria. type Votes struct { - State StateT `json:"state"` - Token string `json:"token"` UserID string `json:"userid"` } -// EncodeVotes encodes a Votes into a JSON byte slice. -func EncodeVotes(v Votes) ([]byte, error) { - return json.Marshal(v) -} - -// DecodeVotes decodes a JSON byte slice into a Votes. -func DecodeVotes(payload []byte) (*Votes, error) { - var v Votes - err := json.Unmarshal(payload, &v) - if err != nil { - return nil, err - } - return &v, nil -} - // VotesReply is the reply to the Votes command. type VotesReply struct { Votes []CommentVote `json:"votes"` } -// EncodeVotesReply encodes a VotesReply into a JSON byte slice. -func EncodeVotesReply(vr VotesReply) ([]byte, error) { - return json.Marshal(vr) -} - -// DecodeVotesReply decodes a JSON byte slice into a VotesReply. -func DecodeVotesReply(payload []byte) (*VotesReply, error) { - var vr VotesReply - err := json.Unmarshal(payload, &vr) - if err != nil { - return nil, err - } - return &vr, nil -} - // Proof contains an inclusion proof for the digest in the merkle root. The // ExtraData field is used by certain types of proofs to include additional // data that is required to validate the proof. @@ -604,8 +312,6 @@ type Timestamp struct { // comment votes will also be returned. If a provided comment ID does not // exist then it will not be included in the reply. type Timestamps struct { - State StateT `json:"state"` - Token string `json:"token"` CommentIDs []uint32 `json:"commentids,omitempty"` IncludeVotes bool `json:"includevotes,omitempty"` } diff --git a/util/token.go b/util/token.go new file mode 100644 index 000000000..4d1fb899b --- /dev/null +++ b/util/token.go @@ -0,0 +1,96 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package util + +import ( + "encoding/hex" + "fmt" + + pdv1 "github.com/decred/politeia/politeiad/api/v1" +) + +var ( + TokenTypeGit = "git" + TokenTypeTlog = "tlog" +) + +func tokenIsFullLength(tokenType string, token []byte) bool { + switch tokenType { + case TokenTypeTlog: + return len(token) == pdv1.TokenSizeTlog + case TokenTypeGit: + return len(token) == pdv1.TokenSizeGit + default: + e := fmt.Sprintf("invalid token type") + panic(e) + } +} + +func tokenPrefixSize() int { + // If the token prefix length is an odd number of characters then + // padding would have needed to be added to it prior to decoding it + // to hex to prevent a hex.ErrLenth (odd length hex string) error. + // Account for this padding in the prefix size. + var size int + if pdv1.TokenPrefixLength%2 == 1 { + // Add 1 to the length to account for padding + size = (pdv1.TokenPrefixLength + 1) / 2 + } else { + // No padding was required + size = pdv1.TokenPrefixLength / 2 + } + return size +} + +// TokenPrefix returns the token prefix given the token. +func TokenPrefix(token []byte) string { + return hex.EncodeToString(token)[:pdv1.TokenPrefixLength] +} + +// TokenDecode decodes full length tokens. An error is returned if the token +// is not a valid full length, hex token. +func TokenDecode(tokenType, token string) ([]byte, error) { + // Decode token + t, err := hex.DecodeString(token) + if err != nil { + return nil, fmt.Errorf("invalid hex") + } + + // Verify token is full length + if !tokenIsFullLength(tokenType, t) { + return nil, fmt.Errorf("invalid token size") + } + + return t, nil +} + +// TokenDecodeAnyLength decodes both token prefixes and full length tokens. +func TokenDecodeAnyLength(tokenType, token string) ([]byte, error) { + // Decode token. If provided token has odd length, add padding + // to prevent a hex.ErrLength (odd length hex string) error. + if len(token)%2 == 1 { + token = token + "0" + } + t, err := hex.DecodeString(token) + if err != nil { + return nil, fmt.Errorf("invalid hex") + } + + // Verify token byte slice is either a token prefix or a valid + // full length token. + switch { + case len(t) == tokenPrefixSize(): + // This is a token prefix. Token prefixes are the same size + // regardless of token type. + case tokenIsFullLength(TokenTypeGit, t): + // Token is a valid git backend token + case tokenIsFullLength(TokenTypeTlog, t): + // Token is a valid tlog backend token + default: + return nil, fmt.Errorf("invalid token size") + } + + return t, nil +} From 91b1ec77b16cac0dd3369137a28c55c740a75066 Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 15 Jan 2021 08:47:18 -0600 Subject: [PATCH 220/449] Simplify tlog client. --- .../tlogbe/plugins/comments/comments.go | 63 ++------------- politeiad/backend/tlogbe/plugins/plugins.go | 26 +++---- politeiad/backend/tlogbe/tlog/tlogclient.go | 77 +++++++++---------- .../backend/tlogbe/tlog/trillianclient.go | 4 +- 4 files changed, 57 insertions(+), 113 deletions(-) diff --git a/politeiad/backend/tlogbe/plugins/comments/comments.go b/politeiad/backend/tlogbe/plugins/comments/comments.go index acd888c17..363511ba0 100644 --- a/politeiad/backend/tlogbe/plugins/comments/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments/comments.go @@ -346,32 +346,15 @@ func commentIDLatest(idx recordIndex) uint32 { } func (p *commentsPlugin) commentAddSave(treeID int64, ca comments.CommentAdd) ([]byte, error) { - // Prepare blob be, err := convertBlobEntryFromCommentAdd(ca) if err != nil { return nil, err } - h, err := hex.DecodeString(be.Hash) + merkle, err := p.tlog.BlobSave(treeID, keyPrefixCommentAdd, *be) if err != nil { return nil, err } - b, err := store.Blobify(*be) - if err != nil { - return nil, err - } - - // Save blob - merkles, err := p.tlog.BlobsSave(treeID, keyPrefixCommentAdd, - [][]byte{b}, [][]byte{h}) - if err != nil { - return nil, err - } - if len(merkles) != 1 { - return nil, fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", - len(merkles)) - } - - return merkles[0], nil + return merkle, nil } // commentAdds returns the commentAdd for all specified merkle hashes. @@ -411,32 +394,15 @@ func (p *commentsPlugin) commentAdds(treeID int64, merkles [][]byte) ([]comments } func (p *commentsPlugin) commentDelSave(treeID int64, cd comments.CommentDel) ([]byte, error) { - // Prepare blob be, err := convertBlobEntryFromCommentDel(cd) if err != nil { return nil, err } - h, err := hex.DecodeString(be.Hash) + merkle, err := p.tlog.BlobSave(treeID, keyPrefixCommentDel, *be) if err != nil { return nil, err } - b, err := store.Blobify(*be) - if err != nil { - return nil, err - } - - // Save blob - merkles, err := p.tlog.BlobsSave(treeID, keyPrefixCommentDel, - [][]byte{b}, [][]byte{h}) - if err != nil { - return nil, err - } - if len(merkles) != 1 { - return nil, fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", - len(merkles)) - } - - return merkles[0], nil + return merkle, nil } func (p *commentsPlugin) commentDels(treeID int64, merkles [][]byte) ([]comments.CommentDel, error) { @@ -475,32 +441,15 @@ func (p *commentsPlugin) commentDels(treeID int64, merkles [][]byte) ([]comments } func (p *commentsPlugin) commentVoteSave(treeID int64, cv comments.CommentVote) ([]byte, error) { - // Prepare blob be, err := convertBlobEntryFromCommentVote(cv) if err != nil { return nil, err } - h, err := hex.DecodeString(be.Hash) - if err != nil { - return nil, err - } - b, err := store.Blobify(*be) + merkle, err := p.tlog.BlobSave(treeID, keyPrefixCommentVote, *be) if err != nil { return nil, err } - - // Save blob - merkles, err := p.tlog.BlobsSave(treeID, keyPrefixCommentVote, - [][]byte{b}, [][]byte{h}) - if err != nil { - return nil, err - } - if len(merkles) != 1 { - return nil, fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", - len(merkles)) - } - - return merkles[0], nil + return merkle, nil } func (p *commentsPlugin) commentVotes(treeID int64, merkles [][]byte) ([]comments.CommentVote, error) { diff --git a/politeiad/backend/tlogbe/plugins/plugins.go b/politeiad/backend/tlogbe/plugins/plugins.go index 5591840da..07493fe6c 100644 --- a/politeiad/backend/tlogbe/plugins/plugins.go +++ b/politeiad/backend/tlogbe/plugins/plugins.go @@ -6,6 +6,7 @@ package plugins import ( "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/store" ) const ( @@ -14,19 +15,16 @@ const ( PluginSettingDataDir = "datadir" ) -// TODO verify TlogClient implementation does not allow short tokens on writes. - // TlogClient provides an API for the plugins to interact with the tlog backend // instances. Plugins are allowed to save, delete, and get plugin data to/from // the tlog backend. Editing plugin data is not allowed. type TlogClient interface { - // BlobsSave saves the provided blobs to the tlog backend. Note, - // hashes contains the hashes of the data encoded in the blobs. The - // hashes must share the same ordering as the blobs. The blobs will - // be encypted prior to being saved if the tlog instance has an - // encryption key set. - BlobsSave(treeID int64, keyPrefix string, blobs, - hashes [][]byte) ([][]byte, error) + // BlobSave saves a BlobEntry to the tlog backend. The BlobEntry + // will be encrypted prior to being written to disk if the tlog + // instance has an encryption key set. The merkle leaf hash for the + // blob will be returned. This merkle leaf hash can be though of as + // the blob ID and can be used to retrieve or delete the blob. + BlobSave(treeID int64, keyPrefix string, be store.BlobEntry) ([]byte, error) // BlobsDel deletes the blobs that correspond to the provided // merkle leaf hashes. @@ -44,8 +42,8 @@ type TlogClient interface { // that match the key prefix. MerklesByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) - // Timestamp returns the timestamp for the data blob that - // corresponds to the provided merkle leaf hash. + // Timestamp returns the timestamp for the blob that correpsonds + // to the merkle leaf hash. Timestamp(treeID int64, merkle []byte) (*backend.Timestamp, error) } @@ -129,15 +127,15 @@ type HookSetRecordStatus struct { } // PluginClient provides an API for the tlog backend to use when interacting -// with plugins. All tlogbe plugins must implement the pluginClient interface. +// with plugins. All tlogbe plugins must implement the PluginClient interface. type PluginClient interface { // Setup performs any required plugin setup. Setup() error - // Cmd executes the provided plugin command. + // Cmd executes a plugin command. Cmd(treeID int64, token []byte, cmd, payload string) (string, error) - // Hook executes the provided plugin hook. + // Hook executes a plugin hook. Hook(h HookT, payload string) error // Fsck performs a plugin file system check. diff --git a/politeiad/backend/tlogbe/tlog/tlogclient.go b/politeiad/backend/tlogbe/tlog/tlogclient.go index 09c6c67b6..3e1125dcf 100644 --- a/politeiad/backend/tlogbe/tlog/tlogclient.go +++ b/politeiad/backend/tlogbe/tlog/tlogclient.go @@ -10,22 +10,37 @@ import ( "fmt" "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/store" "github.com/google/trillian" "google.golang.org/grpc/codes" ) -// BlobsSave saves the provided blobs to the tlog backend. Note, hashes -// contains the hashes of the data encoded in the blobs. The hashes must share -// the same ordering as the blobs. +// BlobSave saves a BlobEntry to the tlog backend. The BlobEntry will be +// encrypted prior to being written to disk if the tlog instance has an +// encryption key set. The merkle leaf hash for the blob will be returned. This +// merkle leaf hash can be though of as the blob ID and can be used to retrieve +// or delete the blob. // // This function satisfies the plugins.TlogClient interface. -func (t *Tlog) BlobsSave(treeID int64, keyPrefix string, blobs, hashes [][]byte) ([][]byte, error) { - log.Tracef("%v BlobsSave: %v %v", t.id, treeID, keyPrefix) +func (t *Tlog) BlobSave(treeID int64, keyPrefix string, be store.BlobEntry) ([]byte, error) { + log.Tracef("%v BlobSave: %v %v", t.id, treeID, keyPrefix) - // Sanity check - if len(blobs) != len(hashes) { - return nil, fmt.Errorf("blob count and hashes count mismatch: "+ - "got %v blobs, %v hashes", len(blobs), len(hashes)) + // Prepare blob and digest + digest, err := hex.DecodeString(be.Hash) + if err != nil { + return nil, err + } + blob, err := store.Blobify(be) + if err != nil { + return nil, err + } + + // Encrypt blob if an encryption key has been set + if t.encryptionKey != nil { + blob, err = t.encrypt(blob) + if err != nil { + return nil, err + } } // Verify tree exists @@ -46,42 +61,30 @@ func (t *Tlog) BlobsSave(treeID int64, keyPrefix string, blobs, hashes [][]byte) return nil, ErrTreeIsFrozen } - // Encrypt blobs if an encryption key has been set - if t.encryptionKey != nil { - for k, v := range blobs { - e, err := t.encrypt(v) - if err != nil { - return nil, err - } - blobs[k] = e - } - } - // Save blobs to store - keys, err := t.store.Put(blobs) + keys, err := t.store.Put([][]byte{blob}) if err != nil { return nil, fmt.Errorf("store Put: %v", err) } - if len(keys) != len(blobs) { - return nil, fmt.Errorf("wrong number of keys: got %v, want %v", - len(keys), len(blobs)) + if len(keys) != 1 { + return nil, fmt.Errorf("wrong number of keys: got %v, want 1", + len(keys)) } - // Prepare log leaves. hashes and keys share the same ordering. - leaves := make([]*trillian.LogLeaf, 0, len(blobs)) - for k := range blobs { - pk := []byte(keyPrefix + keys[k]) - leaves = append(leaves, newLogLeaf(hashes[k], pk)) + // Prepare log leaf + extraData := []byte(keyPrefix + keys[0]) + leaves := []*trillian.LogLeaf{ + newLogLeaf(digest, extraData), } - // Append leaves to trillian tree + // Append log leaf to trillian tree queued, _, err := t.trillian.leavesAppend(treeID, leaves) if err != nil { return nil, fmt.Errorf("leavesAppend: %v", err) } - if len(queued) != len(leaves) { - return nil, fmt.Errorf("wrong number of queued leaves: got %v, want %v", - len(queued), len(leaves)) + if len(queued) != 1 { + return nil, fmt.Errorf("wrong number of queued leaves: "+ + "got %v, want 1", len(queued)) } failed := make([]string, 0, len(queued)) for _, v := range queued { @@ -94,13 +97,7 @@ func (t *Tlog) BlobsSave(treeID int64, keyPrefix string, blobs, hashes [][]byte) return nil, fmt.Errorf("append leaves failed: %v", failed) } - // Parse and return the merkle leaf hashes - merkles := make([][]byte, 0, len(blobs)) - for _, v := range queued { - merkles = append(merkles, v.QueuedLeaf.Leaf.MerkleLeafHash) - } - - return merkles, nil + return queued[0].QueuedLeaf.Leaf.MerkleLeafHash, nil } // BlobsDel deletes the blobs in the kv store that correspond to the provided diff --git a/politeiad/backend/tlogbe/tlog/trillianclient.go b/politeiad/backend/tlogbe/tlog/trillianclient.go index 92fe0f171..8033318de 100644 --- a/politeiad/backend/tlogbe/tlog/trillianclient.go +++ b/politeiad/backend/tlogbe/tlog/trillianclient.go @@ -106,9 +106,9 @@ func merkleLeafHash(leafValue []byte) []byte { } // newLogLeaf returns a trillian LogLeaf. -func newLogLeaf(value []byte, extraData []byte) *trillian.LogLeaf { +func newLogLeaf(leafValue []byte, extraData []byte) *trillian.LogLeaf { return &trillian.LogLeaf{ - LeafValue: value, + LeafValue: leafValue, ExtraData: extraData, } } From bd529ba5561195f30e87fdfb78e555937dbb9b07 Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 15 Jan 2021 09:13:16 -0600 Subject: [PATCH 221/449] tlogbe: Fix build issues. --- .../tlogbe/plugins/comments/comments.go | 24 +- politeiad/backend/tlogbe/tlog/plugin.go | 34 +-- politeiad/backend/tlogbe/tlog/tlog.go | 14 +- politeiad/backend/tlogbe/tlogbe.go | 9 +- politeiad/backend/tlogbe/tlogclient.go | 258 ------------------ 5 files changed, 29 insertions(+), 310 deletions(-) delete mode 100644 politeiad/backend/tlogbe/tlogclient.go diff --git a/politeiad/backend/tlogbe/plugins/comments/comments.go b/politeiad/backend/tlogbe/plugins/comments/comments.go index 363511ba0..b72c14a01 100644 --- a/politeiad/backend/tlogbe/plugins/comments/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments/comments.go @@ -1586,28 +1586,8 @@ func (p *commentsPlugin) Setup() error { } // New returns a new comments plugin. -func New(backend backend.Backend, tlog plugins.TlogClient, settings []backend.PluginSetting, id *identity.FullIdentity) (*commentsPlugin, error) { - // Unpack plugin settings - var ( - dataDir string - ) - for _, v := range settings { - switch v.Key { - case plugins.PluginSettingDataDir: - dataDir = v.Value - default: - return nil, fmt.Errorf("invalid plugin setting '%v'", v.Key) - } - } - - // Verify plugin settings - switch { - case dataDir == "": - return nil, fmt.Errorf("plugin setting not found: %v", - plugins.PluginSettingDataDir) - } - - // Create the plugin data directory +func New(backend backend.Backend, tlog plugins.TlogClient, settings []backend.PluginSetting, id *identity.FullIdentity, dataDir string) (*commentsPlugin, error) { + // Setup comments plugin data dir dataDir = filepath.Join(dataDir, comments.ID) err := os.MkdirAll(dataDir, 0700) if err != nil { diff --git a/politeiad/backend/tlogbe/tlog/plugin.go b/politeiad/backend/tlogbe/tlog/plugin.go index 447993caf..14868cee1 100644 --- a/politeiad/backend/tlogbe/tlog/plugin.go +++ b/politeiad/backend/tlogbe/tlog/plugin.go @@ -7,16 +7,19 @@ package tlog import ( "errors" "fmt" + "path/filepath" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/comments" cmplugin "github.com/decred/politeia/politeiad/plugins/comments" ) const ( - // pluginSettingDataDir is the PluginSetting key for the plugin - // data directory. - pluginSettingDataDir = "datadir" + // pluginDataDirname is the plugin data directory name. It is + // located in the tlog instance data directory and is provided to + // the plugins for storing plugin data. + pluginDataDirname = "plugins" ) // plugin represents a tlog plugin. @@ -46,33 +49,22 @@ func (t *Tlog) pluginIDs() []string { return ids } -func (t *Tlog) PluginRegister(p backend.Plugin) error { +func (t *Tlog) PluginRegister(b backend.Backend, p backend.Plugin) error { log.Tracef("%v PluginRegister: %v", t.id, p.ID) - // TODO this shouldn'e be a plugin setting. Pass it in directly. - /* - // Add the plugin data dir to the plugin settings. Plugins should - // create their own individual data directories inside of the - // plugin data directory. - p.Settings = append(p.Settings, backend.PluginSetting{ - Key: pluginSettingDataDir, - // Value: filepath.Join(t.dataDir, pluginDataDirname), - }) - */ - var ( client plugins.PluginClient err error + + dataDir = filepath.Join(t.dataDir, pluginDataDirname) ) - _ = err switch p.ID { case cmplugin.ID: + client, err = comments.New(b, t, p.Settings, p.Identity, dataDir) + if err != nil { + return err + } /* - client, err = comments.New(t, newBackendClient(t), - p.Settings, p.Identity) - if err != nil { - return err - } case dcrdata.ID: client, err = newDcrdataPlugin(p.Settings, t.activeNetParams) if err != nil { diff --git a/politeiad/backend/tlogbe/tlog/tlog.go b/politeiad/backend/tlogbe/tlog/tlog.go index daa5c09e5..59384d295 100644 --- a/politeiad/backend/tlogbe/tlog/tlog.go +++ b/politeiad/backend/tlogbe/tlog/tlog.go @@ -28,6 +28,7 @@ import ( const ( defaultTrillianKeyFilename = "trillian.key" + defaultStoreDirname = "store" // Blob entry data descriptors dataDescriptorFile = "file" @@ -85,6 +86,7 @@ var ( type Tlog struct { sync.Mutex id string + dataDir string trillian trillianClient store store.Blob dcrtime *dcrtimeClient @@ -1756,9 +1758,16 @@ func New(id, homeDir, dataDir, trillianHost, trillianKeyFile, encryptionKeyFile, log.Infof("Encryption key %v: %v", id, encryptionKeyFile) } + // Setup datadir for this tlog instance + dataDir = filepath.Join(dataDir, id) + err := os.MkdirAll(dataDir, 0700) + if err != nil { + return nil, err + } + // Setup key-value store - fp := filepath.Join(dataDir, id) - err := os.MkdirAll(fp, 0700) + fp := filepath.Join(dataDir, defaultStoreDirname) + err = os.MkdirAll(fp, 0700) if err != nil { return nil, err } @@ -1788,6 +1797,7 @@ func New(id, homeDir, dataDir, trillianHost, trillianKeyFile, encryptionKeyFile, // Setup tlog t := Tlog{ id: id, + dataDir: dataDir, trillian: trillianClient, store: store, dcrtime: dcrtimeClient, diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index b48d24e53..77d3fb423 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -43,11 +43,6 @@ const ( tlogIDUnvetted = "unvetted" tlogIDVetted = "vetted" - // pluginDataDirname is the plugin data directory name. It is - // located in the tlog backend data directory and is provided to - // the plugins for storing plugin data. - pluginDataDirname = "plugins" - // Record states stateUnvetted = "unvetted" stateVetted = "vetted" @@ -1651,7 +1646,7 @@ func (t *tlogBackend) RegisterUnvettedPlugin(p backend.Plugin) error { return backend.ErrShutdown } - return t.unvetted.PluginRegister(p) + return t.unvetted.PluginRegister(t, p) } // RegisterVettedPlugin has not been implemented. @@ -1664,7 +1659,7 @@ func (t *tlogBackend) RegisterVettedPlugin(p backend.Plugin) error { return backend.ErrShutdown } - return t.vetted.PluginRegister(p) + return t.vetted.PluginRegister(t, p) } // SetupUnvettedPlugin performs plugin setup for a previously registered diff --git a/politeiad/backend/tlogbe/tlogclient.go b/politeiad/backend/tlogbe/tlogclient.go deleted file mode 100644 index fdc019db2..000000000 --- a/politeiad/backend/tlogbe/tlogclient.go +++ /dev/null @@ -1,258 +0,0 @@ -// Copyright (c) 2020-2021 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package tlogbe - -import ( - "fmt" - - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" -) - -// tlogClient provides an API for the plugins to interact with the tlog -// backend. Plugins are allowed to save, delete, and get plugin data to/from -// the tlog backend. Editing plugin data is not allowed. -type tlogClient interface { - // save saves the provided blobs to the tlog backend. Note, hashes - // contains the hashes of the data encoded in the blobs. The hashes - // must share the same ordering as the blobs. - save(tlogID string, token []byte, keyPrefix string, - blobs, hashes [][]byte, encrypt bool) ([][]byte, error) - - // del deletes the blobs that correspond to the provided merkle - // leaf hashes. - del(tlogID string, token []byte, merkleLeafHashes [][]byte) error - - // merklesByKeyPrefix returns the merkle root hashes for all blobs - // that match the key prefix. - merklesByKeyPrefix(tlogID string, token []byte, - keyPrefix string) ([][]byte, error) - - // blobsByMerkle returns the blobs with the provided merkle leaf - // hashes. If a blob does not exist it will not be included in the - // returned map. - blobsByMerkle(tlogID string, token []byte, - merkleLeafHashes [][]byte) (map[string][]byte, error) - - // blobsByKeyPrefix returns all blobs that match the key prefix. - blobsByKeyPrefix(tlogID string, token []byte, - keyPrefix string) ([][]byte, error) - - // timestamp returns the timestamp for a data blob that corresponds - // to the provided merkle leaf hash. - timestamp(tlogID string, token []byte, - merkleLeafHash []byte) (*backend.Timestamp, error) -} - -var ( - _ tlogClient = (*backendClient)(nil) -) - -// backendClient implements the tlogClient interface. -type backendClient struct { - backend *tlogBackend -} - -// tlogByID returns the tlog instance that corresponds to the provided ID. -func (c *backendClient) tlogByID(tlogID string) (*tlog.Tlog, error) { - switch tlogID { - case tlogIDUnvetted: - return c.backend.unvetted, nil - case tlogIDVetted: - return c.backend.vetted, nil - } - return nil, fmt.Errorf("unknown tlog id '%v'", tlogID) -} - -// treeIDFromToken returns the treeID for the provided tlog instance ID and -// token. This function accepts both token prefixes and full length tokens. -func (c *backendClient) treeIDFromToken(tlogID string, token []byte) (int64, error) { - /* - if len(token) == tokenPrefixSize() { - // This is a token prefix. Get the full token from the cache. - var ok bool - token, ok = c.backend.fullLengthToken(token) - if !ok { - return 0, errRecordNotFound - } - } - - switch tlogID { - case tlogIDUnvetted: - return treeIDFromToken(token), nil - case tlogIDVetted: - treeID, ok := c.backend.vettedTreeIDFromToken(token) - if !ok { - return 0, errRecordNotFound - } - return treeID, nil - } - - return 0, fmt.Errorf("unknown tlog id '%v'", tlogID) - */ - return 0, nil -} - -// treeIDFromToken returns the treeID for the provided tlog instance ID and -// token. This function only accepts full length tokens. -func (c *backendClient) treeIDFromTokenFullLength(tlogID string, token []byte) (int64, error) { - /* - if !tokenIsFullLength(token) { - return 0, errRecordNotFound - } - return c.treeIDFromToken(tlogID, token) - */ - return 0, nil -} - -// save saves the provided blobs to the tlog backend. Note, hashes contains the -// hashes of the data encoded in the blobs. The hashes must share the same -// ordering as the blobs. -// -// This function satisfies the tlogClient interface. -func (c *backendClient) save(tlogID string, token []byte, keyPrefix string, blobs, hashes [][]byte, encrypt bool) ([][]byte, error) { - log.Tracef("backendClient save: %x %v %v %x", - token, keyPrefix, encrypt, hashes) - - // Get tlog instance - tlog, err := c.tlogByID(tlogID) - if err != nil { - return nil, err - } - - // Get tree ID - treeID, err := c.treeIDFromTokenFullLength(tlogID, token) - if err != nil { - return nil, err - } - - // Save blobs - return tlog.BlobsSave(treeID, keyPrefix, blobs, hashes, encrypt) -} - -// del deletes the blobs that correspond to the provided merkle leaf hashes. -// -// This function satisfies the tlogClient interface. -func (c *backendClient) del(tlogID string, token []byte, merkles [][]byte) error { - log.Tracef("backendClient del: %v %x %x", tlogID, token, merkles) - - // Get tlog instance - tlog, err := c.tlogByID(tlogID) - if err != nil { - return err - - } - - // Get tree ID - treeID, err := c.treeIDFromTokenFullLength(tlogID, token) - if err != nil { - return err - } - - // Delete blobs - return tlog.BlobsDel(treeID, merkles) -} - -// merklesByKeyPrefix returns the merkle root hashes for all blobs that match -// the key prefix. -// -// This function satisfies the tlogClient interface. -func (c *backendClient) merklesByKeyPrefix(tlogID string, token []byte, keyPrefix string) ([][]byte, error) { - log.Tracef("backendClient merklesByKeyPrefix: %v %x %x", - tlogID, token, keyPrefix) - - // Get tlog instance - tlog, err := c.tlogByID(tlogID) - if err != nil { - return nil, err - } - - // Get tree ID - treeID, err := c.treeIDFromToken(tlogID, token) - if err != nil { - return nil, err - } - - // Get merkle leaf hashes - return tlog.MerklesByKeyPrefix(treeID, keyPrefix) -} - -// blobsByMerkle returns the blobs with the provided merkle leaf hashes. -// -// If a blob does not exist it will not be included in the returned map. It is -// the responsibility of the caller to check that a blob is returned for each -// of the provided merkle hashes. -// -// This function satisfies the tlogClient interface. -func (c *backendClient) blobsByMerkle(tlogID string, token []byte, merkles [][]byte) (map[string][]byte, error) { - log.Tracef("backendClient blobsByMerkle: %v %x %x", tlogID, token, merkles) - - // Get tlog instance - tlog, err := c.tlogByID(tlogID) - if err != nil { - return nil, err - } - - // Get tree ID - treeID, err := c.treeIDFromToken(tlogID, token) - if err != nil { - return nil, err - } - - // Get blobs - return tlog.BlobsByMerkle(treeID, merkles) -} - -// blobsByKeyPrefix returns all blobs that match the provided key prefix. -// -// This function satisfies the tlogClient interface. -func (c *backendClient) blobsByKeyPrefix(tlogID string, token []byte, keyPrefix string) ([][]byte, error) { - log.Tracef("backendClient blobsByKeyPrefix: %v %x %v", - tlogID, token, keyPrefix) - - // Get tlog instance - tlog, err := c.tlogByID(tlogID) - if err != nil { - return nil, err - } - - // Get tree ID - treeID, err := c.treeIDFromToken(tlogID, token) - if err != nil { - return nil, err - } - - // Get blobs - return tlog.BlobsByKeyPrefix(treeID, keyPrefix) -} - -// timestamp returns the timestamp for a data blob that corresponds to the -// provided merkle leaf hash. -// -// This function satisfies the tlogClient interface. -func (c *backendClient) timestamp(tlogID string, token []byte, merkle []byte) (*backend.Timestamp, error) { - log.Tracef("backendClient timestamp: %v %x %x", tlogID, token, merkle) - - // Get tlog instance - tlog, err := c.tlogByID(tlogID) - if err != nil { - return nil, err - } - - // Get tree ID - treeID, err := c.treeIDFromToken(tlogID, token) - if err != nil { - return nil, err - } - - return tlog.Timestamp(treeID, merkle) -} - -// newBackendClient returns a new backendClient. -func newBackendClient(b *tlogBackend) *backendClient { - return &backendClient{ - backend: b, - } -} From 444ef5e3cf16d1dcce27d17c21fd3566fe079397 Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 15 Jan 2021 09:53:52 -0600 Subject: [PATCH 222/449] tlogbe: Hook up dcrdata plugin. --- .../backend/tlogbe/plugins/dcrdata/dcrdata.go | 67 +++++----- .../backend/tlogbe/plugins/dcrdata/log.go | 25 ++++ politeiad/backend/tlogbe/tlog/plugin.go | 12 +- politeiad/backend/tlogbe/tlog/tlog.go | 33 ++--- politeiad/backend/tlogbe/tlogbe.go | 4 +- politeiad/log.go | 14 +- politeiad/plugins/comments/comments.go | 1 - politeiad/plugins/dcrdata/dcrdata.go | 126 +----------------- politeiad/politeiad.go | 4 +- 9 files changed, 102 insertions(+), 184 deletions(-) create mode 100644 politeiad/backend/tlogbe/plugins/dcrdata/log.go diff --git a/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go b/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go index 96a553987..fe49adbf8 100644 --- a/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go +++ b/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go @@ -20,6 +20,7 @@ import ( exptypes "github.com/decred/dcrdata/explorer/types/v2" pstypes "github.com/decred/dcrdata/pubsub/types/v3" "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/politeiad/plugins/dcrdata" "github.com/decred/politeia/util" "github.com/decred/politeia/wsdcrdata" @@ -44,10 +45,10 @@ const ( ) var ( - _ pluginClient = (*dcrdataPlugin)(nil) + _ plugins.PluginClient = (*dcrdataPlugin)(nil) ) -// dcrdataplugin satisfies the pluginClient interface. +// dcrdataplugin satisfies the plugins.PluginClient interface. type dcrdataPlugin struct { sync.Mutex activeNetParams *chaincfg.Params @@ -226,7 +227,7 @@ func (p *dcrdataPlugin) txsTrimmedHTTP(txIDs []string) ([]v5.TrimmedTx, error) { // along with a status of StatusDisconnected. It is the callers responsibility // to determine if the stale best block should be used. func (p *dcrdataPlugin) cmdBestBlock(payload string) (string, error) { - log.Tracef("dcrdata cmdBestBlock") + log.Tracef("cmdBestBlock") // Payload is empty. Nothing to decode. @@ -275,7 +276,7 @@ func (p *dcrdataPlugin) cmdBestBlock(payload string) (string, error) { Status: status, Height: bb, } - reply, err := dcrdata.EncodeBestBlockReply(bbr) + reply, err := json.Marshal(bbr) if err != nil { return "", err } @@ -314,10 +315,11 @@ func convertBlockDataBasicFromV5(b v5.BlockDataBasic) dcrdata.BlockDataBasic { } func (p *dcrdataPlugin) cmdBlockDetails(payload string) (string, error) { - log.Tracef("dcrdata cmdBlockDetails: %v", payload) + log.Tracef("cmdBlockDetails: %v", payload) // Decode payload - bd, err := dcrdata.DecodeBlockDetails([]byte(payload)) + var bd dcrdata.BlockDetails + err := json.Unmarshal([]byte(payload), &bd) if err != nil { return "", err } @@ -332,7 +334,7 @@ func (p *dcrdataPlugin) cmdBlockDetails(payload string) (string, error) { bdr := dcrdata.BlockDetailsReply{ Block: convertBlockDataBasicFromV5(*bdb), } - reply, err := dcrdata.EncodeBlockDetailsReply(bdr) + reply, err := json.Marshal(bdr) if err != nil { return "", err } @@ -341,10 +343,11 @@ func (p *dcrdataPlugin) cmdBlockDetails(payload string) (string, error) { } func (p *dcrdataPlugin) cmdTicketPool(payload string) (string, error) { - log.Tracef("dcrdata cmdTicketPool: %v", payload) + log.Tracef("cmdTicketPool: %v", payload) // Decode payload - tp, err := dcrdata.DecodeTicketPool([]byte(payload)) + var tp dcrdata.TicketPool + err := json.Unmarshal([]byte(payload), &tp) if err != nil { return "", err } @@ -359,7 +362,7 @@ func (p *dcrdataPlugin) cmdTicketPool(payload string) (string, error) { tpr := dcrdata.TicketPoolReply{ Tickets: tickets, } - reply, err := dcrdata.EncodeTicketPoolReply(tpr) + reply, err := json.Marshal(tpr) if err != nil { return "", err } @@ -466,7 +469,8 @@ func (p *dcrdataPlugin) cmdTxsTrimmed(payload string) (string, error) { log.Tracef("cmdTxsTrimmed: %v", payload) // Decode payload - tt, err := dcrdata.DecodeTxsTrimmed([]byte(payload)) + var tt dcrdata.TxsTrimmed + err := json.Unmarshal([]byte(payload), &tt) if err != nil { return "", err } @@ -481,7 +485,7 @@ func (p *dcrdataPlugin) cmdTxsTrimmed(payload string) (string, error) { ttr := dcrdata.TxsTrimmedReply{ Txs: convertTrimmedTxsFromV5(txs), } - reply, err := dcrdata.EncodeTxsTrimmedReply(ttr) + reply, err := json.Marshal(ttr) if err != nil { return "", err } @@ -513,7 +517,7 @@ func (p *dcrdataPlugin) websocketMonitor() { // Handle new message switch m := msg.Message.(type) { case *exptypes.WebsocketBlock: - log.Debugf("dcrdata WebsocketBlock: %v", m.Block.Height) + log.Debugf("WebsocketBlock: %v", m.Block.Height) // Update cached best block p.bestBlockSet(uint32(m.Block.Height)) @@ -570,11 +574,11 @@ func (p *dcrdataPlugin) websocketSetup() { go p.websocketMonitor() } -// setup performs any plugin setup work that needs to be done. +// Setup performs any plugin setup work that needs to be done. // -// This function satisfies the Plugin interface. -func (p *dcrdataPlugin) setup() error { - log.Tracef("dcrdata setup") +// This function satisfies the plugins.PluginClient interface. +func (p *dcrdataPlugin) Setup() error { + log.Tracef("setup") // Setup dcrdata websocket subscriptions and monitoring. This is // done in a go routine so setup will continue in the event that @@ -585,11 +589,11 @@ func (p *dcrdataPlugin) setup() error { return nil } -// cmd executes a plugin command. +// Cmd executes a plugin command. // -// This function satisfies the pluginClient interface. -func (p *dcrdataPlugin) cmd(cmd, payload string) (string, error) { - log.Tracef("dcrdata cmd: %v", cmd) +// This function satisfies the plugins.PluginClient interface. +func (p *dcrdataPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { + log.Tracef("Cmd: %v %v", cmd, payload) switch cmd { case dcrdata.CmdBestBlock: @@ -605,25 +609,25 @@ func (p *dcrdataPlugin) cmd(cmd, payload string) (string, error) { return "", backend.ErrPluginCmdInvalid } -// hook executes a plugin hook. +// Hook executes a plugin hook. // -// This function satisfies the pluginClient interface. -func (p *dcrdataPlugin) hook(h hookT, payload string) error { - log.Tracef("dcrdata hook: %v", hooks[h]) +// This function satisfies the plugins.PluginClient interface. +func (p *dcrdataPlugin) Hook(h plugins.HookT, payload string) error { + log.Tracef("Hook: %v", plugins.Hooks[h]) return nil } -// fsck performs a plugin filesystem check. +// Fsck performs a plugin filesystem check. // -// This function satisfies the pluginClient interface. -func (p *dcrdataPlugin) fsck() error { - log.Tracef("dcrdata fsck") +// This function satisfies the plugins.PluginClient interface. +func (p *dcrdataPlugin) Fsck() error { + log.Tracef("Fsck") return nil } -func newDcrdataPlugin(settings []backend.PluginSetting, activeNetParams *chaincfg.Params) (*dcrdataPlugin, error) { +func New(settings []backend.PluginSetting, activeNetParams *chaincfg.Params) (*dcrdataPlugin, error) { // Unpack plugin settings var ( hostHTTP string @@ -631,9 +635,6 @@ func newDcrdataPlugin(settings []backend.PluginSetting, activeNetParams *chaincf ) for _, v := range settings { switch v.Key { - case pluginSettingDataDir: - // The data dir plugin setting is provided to all plugins. The - // dcrdata plugin does not need it. Ignore it. case pluginSettingHostHTTP: hostHTTP = v.Value case pluginSettingHostWS: diff --git a/politeiad/backend/tlogbe/plugins/dcrdata/log.go b/politeiad/backend/tlogbe/plugins/dcrdata/log.go new file mode 100644 index 000000000..3a35b3f8a --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/dcrdata/log.go @@ -0,0 +1,25 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package dcrdata + +import "github.com/decred/slog" + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log = slog.Disabled + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = slog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using slog. +func UseLogger(logger slog.Logger) { + log = logger +} diff --git a/politeiad/backend/tlogbe/tlog/plugin.go b/politeiad/backend/tlogbe/tlog/plugin.go index 14868cee1..057baa172 100644 --- a/politeiad/backend/tlogbe/tlog/plugin.go +++ b/politeiad/backend/tlogbe/tlog/plugin.go @@ -12,7 +12,9 @@ import ( "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/comments" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/dcrdata" cmplugin "github.com/decred/politeia/politeiad/plugins/comments" + ddplugin "github.com/decred/politeia/politeiad/plugins/dcrdata" ) const ( @@ -64,12 +66,12 @@ func (t *Tlog) PluginRegister(b backend.Backend, p backend.Plugin) error { if err != nil { return err } + case ddplugin.ID: + client, err = dcrdata.New(p.Settings, t.activeNetParams) + if err != nil { + return err + } /* - case dcrdata.ID: - client, err = newDcrdataPlugin(p.Settings, t.activeNetParams) - if err != nil { - return err - } case pi.ID: client, err = newPiPlugin(t, newBackendClient(t), p.Settings, t.activeNetParams) diff --git a/politeiad/backend/tlogbe/tlog/tlog.go b/politeiad/backend/tlogbe/tlog/tlog.go index 59384d295..8e33e0256 100644 --- a/politeiad/backend/tlogbe/tlog/tlog.go +++ b/politeiad/backend/tlogbe/tlog/tlog.go @@ -17,6 +17,7 @@ import ( "strconv" "sync" + "github.com/decred/dcrd/chaincfg/v3" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/store" "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" @@ -85,13 +86,14 @@ var ( // We do not unwind. type Tlog struct { sync.Mutex - id string - dataDir string - trillian trillianClient - store store.Blob - dcrtime *dcrtimeClient - cron *cron.Cron - plugins map[string]plugin // [pluginID]plugin + id string + dataDir string + activeNetParams *chaincfg.Params + trillian trillianClient + store store.Blob + dcrtime *dcrtimeClient + cron *cron.Cron + plugins map[string]plugin // [pluginID]plugin // encryptionKey is used to encrypt record blobs before saving them // to the key-value store. This is an optional param. Record blobs @@ -1736,7 +1738,7 @@ func (t *Tlog) Close() { } } -func New(id, homeDir, dataDir, trillianHost, trillianKeyFile, encryptionKeyFile, dcrtimeHost, dcrtimeCert string) (*Tlog, error) { +func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianKeyFile, encryptionKeyFile, dcrtimeHost, dcrtimeCert string) (*Tlog, error) { // Load encryption key if provided. An encryption key is optional. var ek *encryptionKey if encryptionKeyFile != "" { @@ -1796,13 +1798,14 @@ func New(id, homeDir, dataDir, trillianHost, trillianKeyFile, encryptionKeyFile, // Setup tlog t := Tlog{ - id: id, - dataDir: dataDir, - trillian: trillianClient, - store: store, - dcrtime: dcrtimeClient, - cron: cron.New(), - encryptionKey: ek, + id: id, + dataDir: dataDir, + activeNetParams: anp, + trillian: trillianClient, + store: store, + dcrtime: dcrtimeClient, + cron: cron.New(), + encryptionKey: ek, } // Launch cron diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 77d3fb423..594a69ac3 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -1881,13 +1881,13 @@ func New(anp *chaincfg.Params, homeDir, dataDir, unvettedTrillianHost, unvettedT log.Infof("Anchor host: %v", dcrtimeHost) // Setup tlog instances - unvetted, err := tlog.New(tlogIDUnvetted, homeDir, dataDir, + unvetted, err := tlog.New(tlogIDUnvetted, homeDir, dataDir, anp, unvettedTrillianHost, unvettedTrillianKeyFile, encryptionKeyFile, dcrtimeHost, dcrtimeCert) if err != nil { return nil, fmt.Errorf("new tlog unvetted: %v", err) } - vetted, err := tlog.New(tlogIDVetted, homeDir, dataDir, + vetted, err := tlog.New(tlogIDVetted, homeDir, dataDir, anp, vettedTrillianHost, vettedTrillianKeyFile, "", dcrtimeHost, dcrtimeCert) if err != nil { diff --git a/politeiad/log.go b/politeiad/log.go index 0ebcdaa2b..27b2e34c4 100644 --- a/politeiad/log.go +++ b/politeiad/log.go @@ -11,7 +11,10 @@ import ( "github.com/decred/politeia/politeiad/backend/gitbe" "github.com/decred/politeia/politeiad/backend/tlogbe" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/comments" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/dcrdata" "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" + "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" "github.com/decred/politeia/wsdcrdata" "github.com/decred/slog" "github.com/jrick/logrotate/rotator" @@ -46,26 +49,35 @@ var ( log = backendLog.Logger("POLI") gitbeLog = backendLog.Logger("GITB") + tlogbeLog = backendLog.Logger("BACK") tlogLog = backendLog.Logger("TLOG") storeLog = backendLog.Logger("STOR") wsdcrdataLog = backendLog.Logger("WSDD") + commentsLog = backendLog.Logger("COMM") + dcrdataLog = backendLog.Logger("DCRL") ) // Initialize package-global logger variables. func init() { gitbe.UseLogger(gitbeLog) - tlogbe.UseLogger(tlogLog) + tlogbe.UseLogger(tlogbeLog) + tlog.UseLogger(tlogLog) filesystem.UseLogger(storeLog) wsdcrdata.UseLogger(wsdcrdataLog) + comments.UseLogger(commentsLog) + dcrdata.UseLogger(dcrdataLog) } // subsystemLoggers maps each subsystem identifier to its associated logger. var subsystemLoggers = map[string]slog.Logger{ "POLI": log, "GITB": gitbeLog, + "BACK": tlogbeLog, "TLOG": tlogLog, "STOR": storeLog, "WSDD": wsdcrdataLog, + "COMM": commentsLog, + "DCRL": dcrdataLog, } // initLogRotator initializes the logging rotater to write logs to logFile and diff --git a/politeiad/plugins/comments/comments.go b/politeiad/plugins/comments/comments.go index b92c68f5b..834e31bc3 100644 --- a/politeiad/plugins/comments/comments.go +++ b/politeiad/plugins/comments/comments.go @@ -38,7 +38,6 @@ const ( type ErrorCodeT int const ( - // Error status codes ErrorCodeInvalid ErrorCodeT = 0 ErrorCodeTokenInvalid ErrorCodeT = 1 ErrorCodePublicKeyInvalid ErrorCodeT = 2 diff --git a/politeiad/plugins/dcrdata/dcrdata.go b/politeiad/plugins/dcrdata/dcrdata.go index cfcc57a68..daceff121 100644 --- a/politeiad/plugins/dcrdata/dcrdata.go +++ b/politeiad/plugins/dcrdata/dcrdata.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -6,10 +6,6 @@ // explorer. package dcrdata -import ( - "encoding/json" -) - type StatusT int const ( @@ -44,42 +40,12 @@ const ( // if the stale best block height should be used. type BestBlock struct{} -// EncodeBestBlock encodes an BestBlock into a JSON byte slice. -func EncodeBestBlock(bb BestBlock) ([]byte, error) { - return json.Marshal(bb) -} - -// DecodeBestBlock decodes a JSON byte slice into a BestBlock. -func DecodeBestBlock(payload []byte) (*BestBlock, error) { - var bb BestBlock - err := json.Unmarshal(payload, &bb) - if err != nil { - return nil, err - } - return &bb, nil -} - // BestBlockReply is the reply to the BestBlock command. type BestBlockReply struct { Status StatusT `json:"status"` Height uint32 `json:"height"` } -// EncodeBestBlockReply encodes an BestBlockReply into a JSON byte slice. -func EncodeBestBlockReply(bbr BestBlockReply) ([]byte, error) { - return json.Marshal(bbr) -} - -// DecodeBestBlockReply decodes a JSON byte slice into a BestBlockReply. -func DecodeBestBlockReply(payload []byte) (*BestBlockReply, error) { - var bbr BestBlockReply - err := json.Unmarshal(payload, &bbr) - if err != nil { - return nil, err - } - return &bbr, nil -} - // TicketPoolInfo models data about ticket pool. type TicketPoolInfo struct { Height uint32 `json:"height"` @@ -109,82 +75,22 @@ type BlockDetails struct { Height uint32 `json:"height"` } -// EncodeBlockDetails encodes an BlockDetails into a JSON byte slice. -func EncodeBlockDetails(bd BlockDetails) ([]byte, error) { - return json.Marshal(bd) -} - -// DecodeBlockDetails decodes a JSON byte slice into a BlockDetails. -func DecodeBlockDetails(payload []byte) (*BlockDetails, error) { - var bd BlockDetails - err := json.Unmarshal(payload, &bd) - if err != nil { - return nil, err - } - return &bd, nil -} - // BlockDetailsReply is the reply to the block details command. type BlockDetailsReply struct { Block BlockDataBasic `json:"block"` } -// EncodeBlockDetailsReply encodes an BlockDetailsReply into a JSON byte slice. -func EncodeBlockDetailsReply(bdr BlockDetailsReply) ([]byte, error) { - return json.Marshal(bdr) -} - -// DecodeBlockDetailsReply decodes a JSON byte slice into a BlockDetailsReply. -func DecodeBlockDetailsReply(payload []byte) (*BlockDetailsReply, error) { - var bdr BlockDetailsReply - err := json.Unmarshal(payload, &bdr) - if err != nil { - return nil, err - } - return &bdr, nil -} - // TicketPool requests the lists of tickets in the ticket for at the provided // block hash. type TicketPool struct { BlockHash string `json:"blockhash"` } -// EncodeTicketPool encodes an TicketPool into a JSON byte slice. -func EncodeTicketPool(tp TicketPool) ([]byte, error) { - return json.Marshal(tp) -} - -// DecodeTicketPool decodes a JSON byte slice into a TicketPool. -func DecodeTicketPool(payload []byte) (*TicketPool, error) { - var tp TicketPool - err := json.Unmarshal(payload, &tp) - if err != nil { - return nil, err - } - return &tp, nil -} - // TicketPoolReply is the reply to the TicketPool command. type TicketPoolReply struct { Tickets []string `json:"tickets"` // Ticket hashes } -// EncodeTicketPoolReply encodes an TicketPoolReply into a JSON byte slice. -func EncodeTicketPoolReply(tpr TicketPoolReply) ([]byte, error) { - return json.Marshal(tpr) -} - -// DecodeTicketPoolReply decodes a JSON byte slice into a TicketPoolReply. -func DecodeTicketPoolReply(payload []byte) (*TicketPoolReply, error) { - var tpr TicketPoolReply - err := json.Unmarshal(payload, &tpr) - if err != nil { - return nil, err - } - return &tpr, nil -} - // ScriptSig models a signature script. It is defined separately since it only // applies to non-coinbase. Therefore the field in the Vin structure needs to // be a pointer. @@ -250,37 +156,7 @@ type TxsTrimmed struct { TxIDs []string `json:"txids"` } -// EncodeTxsTrimmed encodes an TxsTrimmed into a JSON byte slice. -func EncodeTxsTrimmed(tt TxsTrimmed) ([]byte, error) { - return json.Marshal(tt) -} - -// DecodeTxsTrimmed decodes a JSON byte slice into a TxsTrimmed. -func DecodeTxsTrimmed(payload []byte) (*TxsTrimmed, error) { - var tt TxsTrimmed - err := json.Unmarshal(payload, &tt) - if err != nil { - return nil, err - } - return &tt, nil -} - // TxsTrimmedReply is the reply to the TxsTrimmed command. type TxsTrimmedReply struct { Txs []TrimmedTx `json:"txs"` } - -// EncodeTxsTrimmedReply encodes an TxsTrimmedReply into a JSON byte slice. -func EncodeTxsTrimmedReply(ttr TxsTrimmedReply) ([]byte, error) { - return json.Marshal(ttr) -} - -// DecodeTxsTrimmedReply decodes a JSON byte slice into a TxsTrimmedReply. -func DecodeTxsTrimmedReply(payload []byte) (*TxsTrimmedReply, error) { - var ttr TxsTrimmedReply - err := json.Unmarshal(payload, &ttr) - if err != nil { - return nil, err - } - return &ttr, nil -} diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 3a5a0f9b2..df5a6832f 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -258,11 +258,11 @@ func (p *politeia) respondWithUserError(w http.ResponseWriter, errorCode v1.Erro }) } -func (p *politeia) respondWithPluginUserError(w http.ResponseWriter, plugin string, errorCode int, errorContext []string) { +func (p *politeia) respondWithPluginUserError(w http.ResponseWriter, plugin string, errorCode int, errorContext string) { util.RespondWithJSON(w, http.StatusBadRequest, v1.PluginUserErrorReply{ Plugin: plugin, ErrorCode: errorCode, - ErrorContext: errorContext, + ErrorContext: []string{errorContext}, }) } From 2e5e6ce838a1ceb13e9a21be972e44528e0c9e00 Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 15 Jan 2021 13:01:26 -0600 Subject: [PATCH 223/449] Update comment tests. --- politeiad/backend/backend.go | 4 +- .../tlogbe/plugins/comments/comments.go | 9 +- .../tlogbe/plugins/comments/comments_test.go | 629 +++--------------- .../backend/tlogbe/plugins/dcrdata/dcrdata.go | 2 +- politeiad/backend/tlogbe/plugins/plugins.go | 49 +- politeiad/backend/tlogbe/testing.go | 29 +- politeiad/backend/tlogbe/tlog/plugin.go | 6 +- politeiad/backend/tlogbe/tlog/testing.go | 45 +- .../backend/tlogbe/tlog/trillianclient.go | 26 +- politeiad/backend/tlogbe/tlogbe_test.go | 22 +- .../tlogclient/testclient/testclient.go | 55 ++ .../backend/tlogbe/tlogclient/tlogclient.go | 42 ++ politeiad/plugins/comments/comments.go | 2 +- 13 files changed, 266 insertions(+), 654 deletions(-) create mode 100644 politeiad/backend/tlogbe/tlogclient/testclient/testclient.go create mode 100644 politeiad/backend/tlogbe/tlogclient/tlogclient.go diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index 49f04bd90..e1ef45270 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -120,8 +120,8 @@ type PluginUserError struct { // Error satisfies the error interface. func (e PluginUserError) Error() string { - return fmt.Sprintf("plugin %v error code: %v %v", - e.PluginID, e.ErrorCode, e.ErrorContext) + return fmt.Sprintf("plugin id '%v' error code %v", + e.PluginID, e.ErrorCode) } // RecordMetadata is the metadata of a record. diff --git a/politeiad/backend/tlogbe/plugins/comments/comments.go b/politeiad/backend/tlogbe/plugins/comments/comments.go index b72c14a01..bb58c4629 100644 --- a/politeiad/backend/tlogbe/plugins/comments/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments/comments.go @@ -23,6 +23,7 @@ import ( "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/politeiad/backend/tlogbe/store" + "github.com/decred/politeia/politeiad/backend/tlogbe/tlogclient" "github.com/decred/politeia/politeiad/plugins/comments" "github.com/decred/politeia/util" ) @@ -50,7 +51,7 @@ const ( ) var ( - _ plugins.PluginClient = (*commentsPlugin)(nil) + _ plugins.Client = (*commentsPlugin)(nil) ) // commentsPlugin is the tlog backend implementation of the comments plugin. @@ -58,8 +59,7 @@ var ( // commentsPlugin satisfies the PluginClient interface. type commentsPlugin struct { sync.Mutex - backend backend.Backend - tlog plugins.TlogClient + tlog tlogclient.Client // dataDir is the comments plugin data directory. The only data // that is stored here is cached data that can be re-created at any @@ -1586,7 +1586,7 @@ func (p *commentsPlugin) Setup() error { } // New returns a new comments plugin. -func New(backend backend.Backend, tlog plugins.TlogClient, settings []backend.PluginSetting, id *identity.FullIdentity, dataDir string) (*commentsPlugin, error) { +func New(tlog tlogclient.Client, settings []backend.PluginSetting, id *identity.FullIdentity, dataDir string) (*commentsPlugin, error) { // Setup comments plugin data dir dataDir = filepath.Join(dataDir, comments.ID) err := os.MkdirAll(dataDir, 0700) @@ -1595,7 +1595,6 @@ func New(backend backend.Backend, tlog plugins.TlogClient, settings []backend.Pl } return &commentsPlugin{ - backend: backend, tlog: tlog, identity: id, dataDir: dataDir, diff --git a/politeiad/backend/tlogbe/plugins/comments/comments_test.go b/politeiad/backend/tlogbe/plugins/comments/comments_test.go index c41854277..bd6a5d355 100644 --- a/politeiad/backend/tlogbe/plugins/comments/comments_test.go +++ b/politeiad/backend/tlogbe/plugins/comments/comments_test.go @@ -8,618 +8,157 @@ import ( "encoding/hex" "encoding/json" "errors" + "io/ioutil" + "os" "strconv" "testing" + "github.com/davecgh/go-spew/spew" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/tlogclient/testclient" "github.com/decred/politeia/politeiad/plugins/comments" "github.com/google/uuid" ) -func commentSignature(t *testing.T, uid *identity.FullIdentity, token, msg string, id uint32) string { +func commentSignature(t *testing.T, fid *identity.FullIdentity, token []byte, parentID uint32, comment string) string { t.Helper() - txt := token + strconv.FormatInt(int64(id), 10) + msg - b := uid.SignMessage([]byte(txt)) + tk := hex.EncodeToString(token) + msg := tk + strconv.FormatInt(int64(parentID), 10) + comment + b := fid.SignMessage([]byte(msg)) return hex.EncodeToString(b[:]) } -func newTestCommentsPlugin(t *testing.T) (*commentsPlugin, *tlogBackend, func()) { +func commentDelSignature(t *testing.T, fid *identity.FullIdentity, token []byte, parentID uint32, reason string) string { t.Helper() - - tlogBackend, cleanup := newTestTlogBackend(t) - - id, err := identity.New() - if err != nil { - t.Fatalf("identity New: %v", err) - } - - settings := []backend.PluginSetting{{ - Key: pluginSettingDataDir, - Value: tlogBackend.dataDir, - }} - - commentsPlugin, err := newCommentsPlugin(tlogBackend, - newBackendClient(tlogBackend), settings, id) - if err != nil { - t.Fatalf("newCommentsPlugin: %v", err) - } - - return commentsPlugin, tlogBackend, cleanup + tk := hex.EncodeToString(token) + msg := tk + strconv.FormatInt(int64(parentID), 10) + reason + b := fid.SignMessage([]byte(msg)) + return hex.EncodeToString(b[:]) } -func TestCmdNew(t *testing.T) { - tlogBackend, cleanup := newTestTlogBackend(t) - defer cleanup() +// newTestCommentsPlugin returns a commentsPlugin that is setup for testing and +// a closure that cleans up the test data when invoked. +func newTestCommentsPlugin(t *testing.T) (*commentsPlugin, func()) { + t.Helper() - id, err := identity.New() + // Setup data dir + dataDir, err := ioutil.TempDir("", "tlogbe.comments.test") if err != nil { t.Fatal(err) } - settings := []backend.PluginSetting{{ - Key: pluginSettingDataDir, - Value: tlogBackend.dataDir, - }} - commentsPlugin, err := newCommentsPlugin(tlogBackend, - newBackendClient(tlogBackend), settings, id) - if err != nil { - t.Fatal(err) - } + // Setup tlog client + client := testclient.New() - // New record - md := []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs := []backend.File{ - newBackendFile(t, "index.md"), - } - rec, err := tlogBackend.New(md, fs) + // Setup plugin identity + fid, err := identity.New() if err != nil { t.Fatal(err) } - // Helpers - comment := "random comment" - tokenRandom := hex.EncodeToString(tokenFromTreeID(123)) - parentID := uint32(0) - invalidParentID := uint32(3) - - uid, err := identity.New() + // Setup comment plugins + c, err := New(client, []backend.PluginSetting{}, fid, dataDir) if err != nil { t.Fatal(err) } - // Setup new comment plugin tests - var tests = []struct { - description string - payload comments.New - wantErr error - }{ - { - "invalid token", - comments.New{ - UserID: uuid.New().String(), - Token: "invalid", - ParentID: parentID, - Comment: comment, - PublicKey: uid.Public.String(), - Signature: commentSignature(t, uid, rec.Token, comment, parentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorCodeTokenInvalid), - }, - }, - { - "invalid signature", - comments.New{ - UserID: uuid.New().String(), - Token: rec.Token, - ParentID: parentID, - Comment: comment, - PublicKey: uid.Public.String(), - Signature: "invalid", - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorCodeSignatureInvalid), - }, - }, - { - "invalid public key", - comments.New{ - UserID: uuid.New().String(), - Token: rec.Token, - ParentID: parentID, - Comment: comment, - PublicKey: "invalid", - Signature: commentSignature(t, uid, rec.Token, comment, parentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorCodePublicKeyInvalid), - }, - }, - { - "comment max length exceeded", - comments.New{ - UserID: uuid.New().String(), - Token: rec.Token, - ParentID: parentID, - Comment: commentMaxLengthExceeded(t), - PublicKey: uid.Public.String(), - Signature: commentSignature(t, uid, rec.Token, - commentMaxLengthExceeded(t), parentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorCodeCommentTextInvalid), - }, - }, - { - "invalid parent ID", - comments.New{ - UserID: uuid.New().String(), - Token: rec.Token, - ParentID: invalidParentID, - Comment: comment, - PublicKey: uid.Public.String(), - Signature: commentSignature(t, uid, rec.Token, - comment, invalidParentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorCodeParentIDInvalid), - }, - }, - { - "success", - comments.New{ - UserID: uuid.New().String(), - Token: rec.Token, - ParentID: parentID, - Comment: comment, - PublicKey: uid.Public.String(), - Signature: commentSignature(t, uid, rec.Token, comment, parentID), - }, - nil, - }, - } - - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - // New Comment - ncEncoded, err := json.Marshal(test.payload) - if err != nil { - t.Error(err) - } - - // Execute plugin command - _, err = commentsPlugin.cmdNew(string(ncEncoded)) - - // Parse plugin user error - var pluginUserError backend.PluginUserError - if errors.As(err, &pluginUserError) { - if test.wantErr == nil { - t.Errorf("got error %v, want nil", err) - return - } - wantErr := test.wantErr.(backend.PluginUserError) - if pluginUserError.ErrorCode != wantErr.ErrorCode { - t.Errorf("got error %v, want %v", - pluginUserError.ErrorCode, - wantErr.ErrorCode) - } - return - } - - // Expectations not met - if err != test.wantErr { - t.Errorf("got error %v, want %v", err, test.wantErr) - } - }) + return c, func() { + err = os.RemoveAll(dataDir) + if err != nil { + t.Fatal(err) + } } } -func TestCmdEdit(t *testing.T) { - commentsPlugin, tlogBackend, cleanup := newTestCommentsPlugin(t) +func TestCmdNew(t *testing.T) { + p, cleanup := newTestCommentsPlugin(t) defer cleanup() - // New record - md := []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs := []backend.File{ - newBackendFile(t, "index.md"), - } - rec, err := tlogBackend.New(md, fs) + // Setup test data + fid, err := identity.New() if err != nil { t.Fatal(err) } + var ( + treeID int64 + token []byte - // Helpers - comment := "random comment" - commentEdit := comment + "more content" - parentID := uint32(0) - invalidParentID := uint32(3) - invalidCommentID := uint32(3) - tokenRandom := hex.EncodeToString(tokenFromTreeID(123)) + userID = uuid.New().String() + publicKey = fid.Public.String() + comment = "This is a comment." - id, err := identity.New() - if err != nil { - t.Fatal(err) - } - - // New comment - ncEncoded, err := comments.EncodeNew( - comments.New{ - UserID: uuid.New().String(), - Token: rec.Token, - ParentID: parentID, - Comment: comment, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, rec.Token, comment, parentID), - }, + parentIDZero uint32 + // parerntIDInvalid uint32 = 99 ) - if err != nil { - t.Fatal(err) - } - reply, err := commentsPlugin.cmdNew(string(ncEncoded)) - if err != nil { - t.Fatal(err) - } - nr, err := comments.DecodeNewReply([]byte(reply)) - if err != nil { - t.Fatal(err) - } - // Setup edit comment plugin tests + // Setup test cases var tests = []struct { description string - payload comments.Edit + treeID int64 + token []byte + payload comments.New wantErr error + wantReply string }{ { "invalid token", - comments.Edit{ - UserID: nr.Comment.UserID, + treeID, + token, + comments.New{ + UserID: userID, Token: "invalid", - ParentID: nr.Comment.ParentID, - CommentID: nr.Comment.CommentID, - Comment: commentEdit, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, "invalid", commentEdit, - nr.Comment.ParentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorCodeTokenInvalid), - }, - }, - { - "invalid signature", - comments.Edit{ - UserID: nr.Comment.UserID, - Token: rec.Token, - ParentID: nr.Comment.ParentID, - CommentID: nr.Comment.CommentID, - Comment: commentEdit, - PublicKey: id.Public.String(), - Signature: "invalid", - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorCodeSignatureInvalid), - }, - }, - { - "invalid public key", - comments.Edit{ - UserID: nr.Comment.UserID, - Token: rec.Token, - ParentID: nr.Comment.ParentID, - CommentID: nr.Comment.CommentID, - Comment: commentEdit, - PublicKey: "invalid", - Signature: commentSignature(t, id, rec.Token, - commentEdit, nr.Comment.ParentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorCodePublicKeyInvalid), - }, - }, - { - "comment max length exceeded", - comments.Edit{ - UserID: nr.Comment.UserID, - Token: rec.Token, - ParentID: nr.Comment.ParentID, - CommentID: nr.Comment.CommentID, - Comment: commentMaxLengthExceeded(t), - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, rec.Token, - commentMaxLengthExceeded(t), nr.Comment.ParentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorCodeCommentTextInvalid), - }, - }, - { - "comment id not found", - comments.Edit{ - UserID: nr.Comment.UserID, - Token: rec.Token, - ParentID: nr.Comment.ParentID, - CommentID: invalidCommentID, - Comment: commentEdit, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, rec.Token, - commentEdit, nr.Comment.ParentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorCodeCommentNotFound), - }, - }, - { - "unauthorized user", - comments.Edit{ - UserID: uuid.New().String(), - Token: rec.Token, - ParentID: nr.Comment.ParentID, - CommentID: nr.Comment.CommentID, - Comment: commentEdit, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, rec.Token, - commentEdit, nr.Comment.ParentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorCodeUserUnauthorized), - }, - }, - { - "invalid parent ID", - comments.Edit{ - UserID: nr.Comment.UserID, - Token: rec.Token, - ParentID: invalidParentID, - CommentID: nr.Comment.CommentID, - Comment: commentEdit, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, rec.Token, - commentEdit, invalidParentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorCodeParentIDInvalid), - }, - }, - { - "comment did not change", - comments.Edit{ - UserID: nr.Comment.UserID, - Token: rec.Token, - ParentID: nr.Comment.ParentID, - CommentID: nr.Comment.CommentID, + ParentID: parentIDZero, Comment: comment, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, rec.Token, comment, - nr.Comment.ParentID), + PublicKey: publicKey, + Signature: commentSignature(t, fid, token, parentIDZero, comment), }, backend.PluginUserError{ - ErrorCode: int(comments.ErrorCodeCommentTextInvalid), - }, - }, - { - "success", - comments.Edit{ - UserID: nr.Comment.UserID, - Token: rec.Token, - ParentID: nr.Comment.ParentID, - CommentID: nr.Comment.CommentID, - Comment: commentEdit, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, rec.Token, - commentEdit, nr.Comment.ParentID), + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeTokenInvalid), }, - nil, + "", }, } - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - // Edit Comment - ecEncoded, err := comments.EncodeEdit(test.payload) + // Run test cases + for _, tc := range tests { + t.Run(tc.description, func(t *testing.T) { + // Encode payload + payload, err := json.Marshal(tc.payload) if err != nil { - t.Error(err) + t.Fatal(err) } // Execute plugin command - _, err = commentsPlugin.cmdEdit(string(ecEncoded)) - - // Parse plugin user error - var pluginUserError backend.PluginUserError - if errors.As(err, &pluginUserError) { - if test.wantErr == nil { - t.Errorf("got error %v, want nil", err) + reply, err := p.cmdNew(tc.treeID, tc.token, string(payload)) + + if tc.wantErr != nil { + // We expect an error. Verify that the returned error is + // correct. + want := tc.wantErr.(backend.PluginUserError) + var ue backend.PluginUserError + switch { + case errors.As(err, &ue) && + want.PluginID == ue.PluginID && + want.ErrorCode == ue.ErrorCode: + // This is correct. Next test case. return + default: + // Unexpected error + t.Errorf("got error %v, want error %v", + spew.Sdump(err), spew.Sdump(tc.wantErr)) } - wantErr := test.wantErr.(backend.PluginUserError) - if pluginUserError.ErrorCode != wantErr.ErrorCode { - t.Errorf("got error %v, want %v", - pluginUserError.ErrorCode, - wantErr.ErrorCode) - } - return - } - - // Expectations not met - if err != test.wantErr { - t.Errorf("got error %v, want %v", err, test.wantErr) } - }) - } -} -func TestCmdDel(t *testing.T) { - commentsPlugin, tlogBackend, cleanup := newTestCommentsPlugin(t) - defer cleanup() - - // New record - md := []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs := []backend.File{ - newBackendFile(t, "index.md"), - } - rec, err := tlogBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - - // Helpers - comment := "random comment" - reason := "random reason" - parentID := uint32(0) - tokenRandom := hex.EncodeToString(tokenFromTreeID(123)) - id, err := identity.New() - if err != nil { - t.Fatal(err) - } - - // New comment - ncEncoded, err := comments.EncodeNew( - comments.New{ - UserID: uuid.New().String(), - Token: rec.Token, - ParentID: parentID, - Comment: comment, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, rec.Token, comment, parentID), - }, - ) - if err != nil { - t.Fatal(err) - } - reply, err := commentsPlugin.cmdNew(string(ncEncoded)) - if err != nil { - t.Fatal(err) - } - nr, err := comments.DecodeNewReply([]byte(reply)) - if err != nil { - t.Fatal(err) - } - - // Setup del comment plugin tests - var tests = []struct { - description string - payload comments.Del - wantErr error - }{ - { - "invalid token", - comments.Del{ - Token: "invalid", - CommentID: nr.Comment.CommentID, - Reason: reason, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, "invalid", - reason, nr.Comment.CommentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorCodeTokenInvalid), - }, - }, - { - "invalid signature", - comments.Del{ - Token: rec.Token, - CommentID: nr.Comment.CommentID, - Reason: reason, - PublicKey: id.Public.String(), - Signature: "invalid", - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorCodeSignatureInvalid), - }, - }, - { - "invalid public key", - comments.Del{ - Token: rec.Token, - CommentID: nr.Comment.CommentID, - Reason: reason, - PublicKey: "invalid", - Signature: commentSignature(t, id, rec.Token, - reason, nr.Comment.CommentID), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorCodePublicKeyInvalid), - }, - }, - { - "comment id not found", - comments.Del{ - Token: rec.Token, - CommentID: 3, - Reason: reason, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, rec.Token, reason, 3), - }, - backend.PluginUserError{ - ErrorCode: int(comments.ErrorCodeCommentNotFound), - }, - }, - { - "success", - comments.Del{ - Token: rec.Token, - CommentID: nr.Comment.CommentID, - Reason: reason, - PublicKey: id.Public.String(), - Signature: commentSignature(t, id, rec.Token, - reason, nr.Comment.CommentID), - }, - nil, - }, - } - - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - // Del Comment - dcEncoded, err := comments.EncodeDel(test.payload) + // We expect a valid reply. Verify the reply. + var nr comments.NewReply + err = json.Unmarshal([]byte(reply), &nr) if err != nil { - t.Error(err) + t.Errorf("invalid NewReply: %v", reply) } - // Execute plugin command - _, err = commentsPlugin.cmdDel(string(dcEncoded)) - - // Parse plugin user error - var pluginUserError backend.PluginUserError - if errors.As(err, &pluginUserError) { - if test.wantErr == nil { - t.Errorf("got error %v, want nil", err) - return - } - wantErr := test.wantErr.(backend.PluginUserError) - if pluginUserError.ErrorCode != wantErr.ErrorCode { - t.Errorf("got error %v, want %v", - pluginUserError.ErrorCode, - wantErr.ErrorCode) - } - return - } - - // Expectations not met - if err != test.wantErr { - t.Errorf("got error %v, want %v", err, test.wantErr) - } + // TODO Verify reply payload }) } - -} - -// TODO make this a function that dynamically builds the string instead of -// hardcoding it. -func commentMaxLengthExceeded(t *testing.T) string { - t.Helper() - - return `puqgtmjuiztnwakzqtmybjgwluizndahxihosjwyqsyqrhxdkhwchnhtbgamigdakzyjcnjvubufifolxgmygyhoewgyjwicucaypgxrgsenjpangvtslpfbojvcgsxzqyginsjrtcaggdcienduzuxnnpmdcvxtklnsmlanjheywshijznprrvnizoxrihxstcdksqvhvqjeneeadzpjgqtdrlmhjqkxfsjcgoewisbacchagpzisvttyurpijyzpugkpufyhifaivfzgjmkpxtdydaebidbpypcnmsmvaktchcqkkrwvaahlrrdzzxpjmlshtmugrzjqyvehfnkievnmubitaeewabnqnbqcacctfiojiifozfnpvooawiutxxzcjddqpybnrsyxtfzwiomoqpbpowifsbgfsvhvcaucwjnqfnwecqfugakimobvnguyqsgzjxstovohyzvezjuwdwyopymdpoaqwesghiuzmdwzlfcnubpvfefedyrllzqyfgntcaaazmfqxknfkquiyplyhqqipixmlqjedtafcwmkfemmzdvbgkatlcwnsuzjuvofpwznczcdozlqzaiqpumugnubbsfhvoogfenjsqlenxptlaostdwpcorusmijrbqecttwokirwomuktngwluwmlthypopbinyxqrnxplzpipzitqpaqdreelqjlxqsbgxbwhmrmauiicjndjrhrqnaucbezopzrqpxcjobicxqnswftjqrhhhnicxuymdefodhmyscrzonmrihvljgdxqvjeqxmayilhitwwtnmwdcspcmnwocvksitrhiqslzavwjlvzcjwzliikcqpyllmtfrqgfwafalfklhhnhmoejdsgppdvugblyjusljcqjftktuzsqkmbueocfdadrfsbzxldiagxjuydthitfwrirvnbshdzdnbbyxaizemggkndccgjolevpfjovyacfrzvjabkgvwguxpujfancqdebfcbhlwluetcfnddixdovydgoyngxjkxelxarphjmbcbfutmcoulmbzyjxkupqcsaxsojoqpxdvsaijvhppdmdstszbgjzylwebvjvijruioazjkqccsluqsydrkqckluwealsrgxvnnuhvtuxpxdtkjguwplpmynysdysccjqhulkpcnujituvnattmvhhndyvizusmdxvpbvxzbiogawboasipmjgkdezulkmtkcrziqnyrjcupucjtkovpeqnbrqunhricblwqctencvspsahiltoopwqthxypkfovijqbwxahefszdgpwickvurthbxdkczicvzazssynuoyqcrhxxrctsrbvflbkpuprywtasmkkgcojaiylazybdtdnkjuehxjjsftkayjpcnwrngblzjsbcqauxpskcumnbubiodlqzberjmosvvufzgqbtwprhkscsdfppiifzsxroddracraxaesninyzsnnzjlnybrasfudvrjaetulogbzmbxjehlqotyfygfvkvyxkthdljkbwnvxrowjbecngutbuajppguteagmucqgwihzsktxscfepcghchwnnpkozysgpzrdjsozighofgsibuzccolbzpovmooxrkzdizoibmcmhakjwzwnqftgtgawgwzxnwxxdpgxvwxqszjhvihxfggfgxdpseiqkqtuspygomxuwxhumkiltapyfpumxogrjtzkvdomitflsesindlqvcgvxzogmzssqakheixjvzzqgxwzfjqlirtqvuddfevkzltnhwefzalsxkxqnwngdovnzwrvmvbtsykemmwipxlxdxhknyszyfmflmlcvpojnwasvmdfwahuavjajqnzigwvahjphrwreugmeromzotybxqybslneictseznajjkcxmueyroqamairkyzzdpilbusgqufyietrcbertanevmtrfnuxvgyppskpbcligxgbsmrkjnqytfvycbxtmztkzfrvsbuffuhvmuafhrqvupuecyutvxgdmhfrhcngdmqpecvpfjmmxcjbjjtaehnfluibufrmggdaxlenggmzpbwfzkpqymbsjgjcbugiruxyfbpbjovnentghytlitcgksmhxwrwlpkieqmhqrivaqgcllqljvtperdcxpgdbhdotjtvqvwkpqapsxignvhqfhuradjuksickefuuqsmwdktbcpmgynjdrekffvultianlrhyuxjhltcmzhighfddopzpkxidrnwmlczawcaiulvzxlmaulmzlrlclhajplfuxcgsvoedgkltrbqxyfplgoizapfsckkhrtnpqntlectvfzkrcgdioonbjdlrzpyrsanmvijbycaopumafijkjxubibqwlqrywaqjvtvymqiqkdevlugedamqwiqmtdwzotxbzrltbificgouieeosjaokznhqdthzyttabrowgwxrmwobgmupikxduufpktbdfjganpazjbtcweloxmabgexlixwrdolgoxxfwvjrjommozgehlxaqgxphqbaugaiqxpkruwgcvuyadkfwwsqdpwxapmlvpwjvqmdvqtltkrzfzxuirvwxsmajrzzpfcncpnfjzfugipiuuvqsvfbycrarjcrgjspycerxlvhbcahrkjtlcjhcesptpvqcjinmetwuxygrpksmsyvjfpeibljmswjkfdlpqbpfidmdtilclbntvspvsmxstarlrmvzszrwiayncldnpknchgbjerojunmotgujxgeurpabcguhyvsyoyrhlttbcjjraioxblzszobfyiahhwzwgjlkuknuexpyuovlyfhaxhyjgaxzdbvaejdiuqtwiqjnbadsjudlsexlvjnroqihcbfrawmlfczbfvqoyodzmaynbnmtrewqhutjtycfnajrqwjhjfclzopvlbpmqgiugwnarkeerqttlpbxpeqvdiqaohojoukhpjkqaaqkodogtprbtdzbqnjeqyqoffuxxikorythdmtybwpelcgjukzarpsgnrpxesxlbjmbkplhopxxmxjbbwtyueriuaotsjthvkpgjcnbthxgcctexpnacxypzerlzvjlnqcqipiavtmsekfsrrprxerhkkxoldhhvtyojqknkhatpmqavufkepwkxfknokwehwbnrosnhhsdkyesxhcflzmqkflltfhyatqjmcbcptkbumwiwakfszerchqivrhhmxxnhlnhjyfhklpiojwwerznvbzzdjkxajxwbytstchdmyjrdpjhxonxswpigtnmifkmyyqwzldmkknajfhbphedtzxewrtgobofdjdezxdrhvijxbegepjhicvgymqydculiitzasldrsoiafdeunfnfjedcqgmtnmendgirvfdlkalnexxrxmobwbomjqiqsrwrctvhxytthlwfsalhgjpnvrwyxpcphdiejgbrerhsgduegxvxeudadbjyzptcnassvmlwvlitzozbxgrhwivppyovmgsekvdwznqwimgsigwgavdrhcjnrfyxftybncjmcpoqssujqechwazbmkkicqobzlcfjwxfiocikzaumoigufxouavurmmhdhjdhpztswphjrqcwnvkiqnlpbbsqkkehpqgewunuukrzxkfjqxyuojqhpdiswiwphhoedwmexwfsnelglgqdcfpmohpslcknyjcniodjrayrgbmimcfxoowebjseeauffehxisbiggrjndkeilsxuuhopqqlurzlkovsltarcplfyufukelymxltmolhoqeaodsbionrzxjfnibzovmakvclmglnzsytiquajwcjkmmbkwmgmemmubaxpwfupahimisngzejhtsrtzntpiapcefvlsavelcxptkwwefdonenzgsffrjqehlotebkibkqrxkxbuirmmbwfolnkzcaphbywfwrnenppkupcayglmexcziijxliospthmkhtolpdqfaesmpilekvlocoznhpffneomzjuzjxdnutzqgeajxdfmjwhgqkoiuxosrdpkqutroqpfmnolirpodgbcojbwtbzdeqkybpsnwityouywhfxjefdzgvhoxpwskgnngleedntvnnodhdmsdzqpykocejfobcybuxsifgzqfowltiwvxixqdjesopcaotqytncetdazkiddisfhiycwrhoswtrbiijxvvkgaewbpyzxyyiavckfjtpusjetkobhsglbeolcfxcvqdyvkffvxebvsooawgrcerpbtdrdrglzzglhsvaayatsoztyvzsqftvsuhkoektnfiplxbhcgulztmuvugfuexlmvjhpgytagekkbztmgmehesxfcbmvzjwznlcrpeunxtuwquzggndtpcouuxeyowrcbbdsxzeqiawzefflxvjuzawisfgsmdfmgunmiuhfpfinnwbrmbjfgnunsbqxyodkrcbceqvktqmxfgwckcngxynosxlhtxfqfxoqnupgdgtkqqglvzminnkikckknruaddbknopkqmhakqdlsbylftqsprjxkxdjucmfvdnlawhpskfzowlosriwotzmoalsonnibipimopnrdgrvvxwcexarmkvazrdogampkxbcgsjjvhqzoorjijkjdtyqbldvhdywedykkshyjmohwnxcezfwbkarxfyrfkhqndlxtwbukjcemtcvexigpqqcpvddszemjemrnekquwveerxbbchxyjffnlbpkehtjracvxaobtcgamkbfbrdcdjtsuxfoawfqxjfhdpqnxknocspqoldjgqjeepyvhsjncchfznbypgsoorpuchvdhrfrrytjrrqoyccybopakqhsfygjcciiwbkivyivxxsfhkguxqemtnlvdaucxvlotbvxkvsbsothermhpmkamfxnyaibdalvlimqfytryphtnkshxaexgzosubcccdpkqsggsdserjickunguvtcipnimxgbyaigzwgrqpkbgiwsxfzojqlyhnvszfebsehdxuckbpnqfwqsydsmewcnnzbradqavgdolbiazilxojlpkhatvtxdvnlipdurriudbgmiffdnzhkrytlzizgfztbrygfrjummhslxlhwiunsnebyomrzcrbjhzlfeuecqbuvypfjqrbkmxyamqoleedabkhubjiorbyuilgxigehkeyadafcokuojmzwbxorbiauxmlygurgnrnvqskljhlrzfkgvszgqfgmgcddqgzkynbydlnxenngbajpnctmqjsptserccmsayaldyzlpzelmijnioaiuvggxeeibmeakjkzyuzykjxvbfxregohbwtcgvrngvxwgdemwcsngkmqczexwdxbpqualctwfyhwsrzbhwympqzhstqqbnurcdyrnlzrugwfzgjeoiccvpeicjzhjfeshbxcqhirlaejhgbswlkzvvfkxuxgmnpofscsxmyopoxalhhfzthcysdscfhchuwxucrxceyiytvohcpfoayagdrwdcdlirdbbsxzwmluwhcpxsbzkhokvnwdwfbzmmythphvuoyhmvwfitrbqzmpwucatsyxycwrelkradmhvweizmlkqbghdihgbxogepflvdiaxlzbuxswflhuqbbtllluatwduadfnfzftfrrokdktmwgdsrmaylkalomtsowoxwoqaixuxomcaibatwqelmahlkohuregcxnjgamhbxtxzyeizuuageosusredimvavcedekzqunkkjolkeyvpcifkhvbynhpfacmyfpfxnvqzllnmfpxdceieelnizylbwgngeqhybkdxkcttyvczcfkbbvkrderwmldizptcfhabwsqgyxgcisoaoolupmoxywjpliuxkjljrqkatfohqmbjgmeoxrsuqfuudnxxtyymegeqqcqrjsasieftcfvhcwgdjgcwsjsftbnibnbcptcoymnpqkszlxrauqvnasslhivdxojoabagbqvtnqvjbblysijnrkmmxlobcolfpnrbekmttcxcuhcnlucnnjglzrwnivlrirhvewmxvmxuserdtgzmaaoihytnbnaxyogyzuvkanwyiajykuzdpkqkqjcxiozqovnmnugpvogfggypjcqcrxfeumryypqkyohqetmviytfyiygrgamlklxzovpvklymzhifebmluwinwijutszkhlbspovyainxlorbkzcgeuhjxlsxtwgpshrtwqbycmssuoucmecehpilhnxycrcztltszcevblubvejiunthsbyukrcckazvosvtqwoajlokcctlvimoeoetpwnmkedwlaxdvjdcrppjvqoptacwhkkrfuvkbdihczwnorcxmfaawwysegpqqxsmlhqymctucwcyjlfbjbuukrfrwgukwwonxrjnuplubhxvclqbnedenhihdgzbziwvkwszzvwbekhhdthnkqgxhlrpfkhuvqlrfbjqwwzyhmrlsdhiiynhnkoqvxcezxtgbzjduppkqxdwublnuuptqdghvnzmgurtoauiqzapwiahnoknsjcpxajeqmwrniuuszcxzeoguuobjslqudfgtsvkyvnzayfsdjrhkiswwtriscpnkhxwznrgndwlrhzqacqcqoyzqphutleiqexhialulplfmizqyekwdksymdxpldpzrvecuskamlfqinkrcejwjsajtunwyspddiklupgsteiwulivomjiwquqaablnsfgdcgvxeqtotqnaqknhcxqhrlwrpqkmolyerzzmnpbufvpyqcnxmfczcjdydqejsiwkndvaksqiwndtjwwybepduxmqtvnldqafwuikwwrbhowbdkneupafwpegwtyncyijljjecfylqkuxexqijkktogiuchpvpfhlobwdpoctmmwtebblogvnstazekvnxzlyeoepwfkafedfuzusktgfwyqjhqagcwvmdjifymcvbfocikyvnmrcpwcqwaxahbefhnlsdioaajjfbuhelofqjoweeacfiutaawjddrepdhccpvdaabceywalvwoqntlsugmexzkiymlvfsrbemctangeksmmlvfdejedwleylgldzpgupycochiixthdunckpkifkowfbkmndjnsgzyvcafjcgxcknirojflydhpufrbbcoartxnzwghwmgxguvybrlqvrycdflkenlgrtgxtpvfjefenqnnglmmaewgvabmclvrzphxhfbjajipbwfskbmnmhxjqyqxivehqfwvavplnxgumfuzbcmhpjtioiibijltztrnzxlghyrywnwkyilugldeuvquzpigsyovzabsspziqbooemzkmfzgmihjkpftqvwgoeymohdskdahtzawrjibklpsmhtrhqtrsdverrmzfrmjbzojvnbfvvfdzhsrrywmyvovvhioyfajzustjysjefgonzydbqtraymsdlzbshgfcvwaodnkuvferujaewbqgbbatixdgemqxutnwgsgyoltwwkahlojrmxhntiolfiouyftkjmhmtoksxfxennhfgvwvtklygxqhlsyewitlnbaevoghedmpgkkgtvhderaojtxbgovdpiqkcffevuggzsyrntrmvojffhirultxzzpfqdulzpdgssogtfhqjdlorplcyivsfcrlrnhkeqjixdthwkkytqdynwygqofobcslheqjsbfvmabagkqbmggevavmhjxnswmfacupdyaleflgrrywjmtgmznqynzijxudjfsegpkmqgnjyfswvhengsrlbrwugsmyaydkoaewwlnkstvlpklvbolfqbtvztzznwtpmaqpcqaoskmyhksvmgdtifyszpwdekjfzuifrlpruislaabhwfrpsjdofuhnqr - uxzbyjqgucfwoadtrjowszpnajylbkxlggbafgxhgmytpogkxbsvejafypjtjpxlkfuwayzbmgidouwaqsrreppbiapwonzhbisyzvomrrktiylsjkfkgsilrrobibhqhrndbmuvvogiiofwfsouhgfqftsztapefdumtiocbizzvsmmhayikkdzmpahefvcglmcevawpdcbrxonwykpmfiryzucktpomdhuyvaobqzmzxvkjoksabmoowtzkihitwpvxtrvhbffvwhjyozieqovvmzxnumlsfdbjuznddlmqyrtxbdyqmawpnubepnfkghucnztkbunnrcfshlfsageaqlgpvwfgzkxhsgofomzgkmljtniyaxeiecrrbfojnxubivwobzrsqjhlpyyrcplqwyuskjpteizazvmlxgqowxxqrpztaedlrkpnyuisjaazsjyqryewlfkupwezowpkfddafcyadgtwyeuqnsijwgainyftbogzmqqvuusegadirouinruoyrbcsuodjgeytxoydjfksbatodexjlihbfzesiahlgzmubdyoytcnpffonhversyiecojahysecgcjkolatltayfoxhizfsygfihtncfgnekcazcxtgpiquhxuvkcfbrxuntvtxvldjzulrgikcixieyrrauqyreirdoxmjwnngcguvzrzflanndokugwecfgrdxwafgskgzadfiffkfejdpvtgxzvitzjeyalvlryaukmpwrgjsimrltbditowolohcobodoeoaqhcblqovdntzfpobgosfckdfjuawhgkrvjznvyiburrufelothjemvltiiehibrtapdfufpgccpqyirgnncdgmrtmpomrbsczoktvjbxqqhzqcdsqppwjrcndjeemclkhuiembvdxgunlrbqqnrudffvhpdbldgefwxzzazclqltrmnyqkemnvfmuyftytotvqslldrirdgvtbafrhzglrzovuvgyvxwgjfawxphtxwxnbvjqnfzrvsixmspcdvlrstorerzzziecbycvhhuybkywyefouctykxugrgemmgkxwjcizksttpjglljfqxmlztpltyypphmzccrmbirgmmdszgfikrhcyhtcwjpgfvrjkqgfijhwjjxsiiecowugxoocaqsqqqxmplsqbbtifeevnqgksxwamxdzkjseeemvfdjytfmbbknnnkrixxkrbbkaxfaxjaotkiyqsvnreouoxacvjpkliigglsvigmgeqcrfvuvijplnkfinziungckosmxffochsbpmjpcvrogwmbjsurqurnhaofjtbmsgwnegnrfpxblnpogzmcfrkmdyospxlyxsivhduzksebfgiiueukzqismrdkqxmrrmmykwnxvjkclwbgzsncbzfxfhyqhrnbcakrmvtwnyfalohwaaajxwxfgwurcqdgcowwrmlxzjyzxihcmjluewonndzqpppcpexiynziwgvxerssbfaxtbeyfzzmfiitkinfoldbljjfteudjjqkqjdvxzifamkthldnctmodkezbniywsbbmmfswrnbtpqsotbfykrdaqqzizkmedwrxfgkxuvrnmckmtndnomcgefaotncukyfarlgqohwijtklbjjoqypytzjokvvbvxlwfjjtzsoimzqbrqlwymkbqcwhymzixtarzdirmnmmpewrbmztqviwwjcjzojgbxezjkbzkylxoigvkyjtupggiyuxplumdumuyepqrieofoejzvhhkjnuotmcbgprmsfhqcbsnehhzyuylcieurloynmpjkfmevitqmsattztrdqbhurxelyhbqjysmjocvnxgqngqztmqshwlfxksofjfkmwnckjvbzgncqbxwclkdjrpkccdoxtsciymhdxerhzaniuxtnjtewybyfvhwrmerghlhudosdbclqyjqrkwukablsfwadknkdimtnozytnfkepkuecrkdljgohcupaymqpdiyvdazomgunmwvsrreftqcjqibvbzdwmkexogqtlszoxwmizmkikcwmeqndrehvhnmahlrwqjjbxhzoutpuccapztxkbmuywnvtetctouofqtdafkirequutkeoisozumccsqmabfumputsfjkbftoxufbaggrasydnnrntyraarsxlbncezswbmqbnhpsrhewovqqozorvklxarkgdaiofdvguailfmowbetpvotwbfliyrbelgpigdcbkdyiufokckqvtxfuvgjronpqkitdbfitkeppuajlxxljttjivllatbqutkailadklcnrvecnkgebtoulxnfwabyljkwxeeevoclxhtlnhquyxvillbyhnfkiiwopbmfibrqrgxhtejfeilrhqlujkjksadeypnmqiqqhrsfhsreorcsboszgpcnutlzbvcxgvweyctxidpqfdqymggvpumnnasdjqutwgedshghssnqtruhoibbkenpbbydwsbeatozxhgqejmhljqcmpxarfnfunodezrhcbedvuuewkwxtixlujbvfryajdyiiocremfobtinatyssdhqqaxfaqslqyadgxgiamocdokrnvaukcwsrklrzfveiacpockgybmnsuqavaxtdnkupzsvztaeejsznrjjkgmsccrsbdynlutjpgkpxydthodzhwqihqepwsmsbjyezgmmiowaojczpkhbdxqensgongxlcupaokokcqrkrzxgupndxddomsqdyizmsgczuocknuutinjwdzgwnbslxhilosreycphtxljqdxpgmsdodbexhpazhtsuxwzaejmgnurjorqaoyshjojebpnrysbggnickzrzvzpggjiunglyhhrtcdogqotlhhkaazwoooourfnxuyxsddwdartkurqmfkigbhhfcazaznbxtjfjeakqszdwxwifxctygymmmkgpafebukrfgxngxwshsywckqjosbwcgfseucapjcmxthudzdlzvjkejvbqgpnksjgzaqbsdlypzgdbiiinlztpiwcszyuclzhwyxiovgnmyoyobrgeeahktiylmtzyamzazxfiieefcrrugwkqtjzgjouosqvbimtywvmilpwghqaaqjwokxnfigdlairvhjaccsqcveuegkvrwreytvizrjukhlfmzryjqyqlxagcrrtoxhscgzursqhtvnaffaupwxkpsyzilbhcgdnhwpkcwgclztodkywcrvjfhcatobzpftbetupysquhoavyjbqwpjrybibttnrmrveehfyfmxvfpwnxzpbumfesvqaddgiltjwtttbroadwyhufeqcuqjzldyjzftbjnmslndmejmkkokywsqipyhpsdpubndvhtpzgsmozjtcofiajjohnenyysoeuhcztvqpyicnvspskasllriuisrkbpxlggpplmwlqrsogcdqkujkxrdynqqpahmvkvytrxeeednwgzomuxxzptmednfzzaqggfjusqfqtzxnagtkovjinalffpsdjsskogcnupzdqtialmxuwrnfkergrjgqfnkohedxlfuaqlgukrfjrtdqhfkzdktkspnbokplyuepacbweonicbxlupglkxockrsnaqarzuuswxcsfdwmucbqwwsdhhcwovvstrenwdnmmkqhasohfrchmsosspvafmxtrglggrtvopxysrhormaifpfruenlfvgnpcxiwsialuyoueobnbtwvvtttlhjmsgfhicuvahtpmgpssjstyhezamfrzpbkzduneoxlgnowthyrrltmsghvmmixnkadybsaxgnatzwqlhgwxzyvcpdjkyrzttjhlrjvhpwfmiymvmzudtpnrgdjjsgutsdsjogzddoyczxjzerynnpfxrkkjhlofmdgsaadzsecftpubclnybmrmsjuepywujofzqhqpbgjiqkrrtuqpbvkdurmkyvnpfctvnomcqxfaebtcqbmwenanvxpesjjrkzbaxromczhjripdrnmgygcjzetkyarponwoiijjvagttcyqechcxykihgkjrpdoatzefopgawgyetlxifqergchypiyddabxnhlnafjksvkeedhncnazrywehboujtlyznvgmwoiggvbwgqmonebeoofktanpnlcrnlaojikhnhffhltwvjhtwocoefhpxukyjzqbkwzrftehxwpyjoqzwkunfetytirykgrtqqchqbpifhplffjorfqfiouonuhqjmrnzmfuzelewestiyzeavzoaexmqaybufknqojhhygwbqypdpmfdcfkwducxilhswebqpdlnqrbomcpquzflccznamnvauxwpyewzgmrfeatrlcbwmganwcieuppwuetqvmfnmgpttzchjljleepigcgobasmbzvgcuhaxnhqkvylyxkwhejyvajwtscxsbfxayiscmmbxxdcnwqdqtlgaetjazxtifsypbphqcjhgmqypkqvwsonslxgilctltiutmdxzqfpnhlvqmytiarnntracwcyokebcaeyxocjjontqetfbfngghsppgcoczhdhgkudyvxdyqusccmacfhiuueqhtqczlvsioijpytnjniyyihncxpxtbkgwtdloydbkzmgttimmkszcfdotxqaxjuzurvssbebjmabirsymdfdzseocksrqarbtltlavwetykcaakjhxxotmubsezdyhjevbxtrexjdsxrjdqysuhgudurhaycwzippcnuefarkaoocdlocvxokxrseugqqliceiqwmmifvhznzxwlgoxejiqgtoshtdkygjibinqxkngslzoboaxxzbbffpintrljvknhykyqexatnpmnwyeiemgvqlabrifhxptzbfhooyhligwbgyfgelunqowqtlrkdpzdbfddyjeowyamsxhpleeetezemjaseggoxlmbndjzsjhgeeyyfgtrrvdfqramgsnisdwhvmxbgwuwgqzlglzqrvzzyzjiitiiywmjcobbevdsplimdxaxirsljtuthukxygtoxnxkzjecvyopogdwikccgoehabhwqxyppqwklvrshvkvdrrymawivhhdpblpesimmceugzdqajywkmijyeidtzvyfaiaiaacozqzmdompzefqbnsamlopccvwqwcetqgogzwjzqaepfiuvcthkqwsbactudtivfdnsarfmkgfmzkfejcixpribvqixybiuouqpcwnlqohouayilyveehemivumhjylikdbrtjabneqzzhdgomgpehpwhqswcudssypzdsiuelxmfzibtilcszxalrpapnwzncgtgqxgbgvcikvkgnwgxqzbqimsgvtmjzfxhocgefukyuuejklebvauoanawkhbcomjjguozffdiktwxzfqowhvxaaflafubdkaczfvlrdipasaxqaakpyfcjawaojnhnmckczwptgzqltpzgmwkuesiokzicblrchzdoosceouvziahsdjxhczusgduehdbppfmeantrszapodkgfseseuwazdemdnlgrxgkntszpngscckogzhhfimqgajvpocozxacticiicfvvsxajcnkurdtduxfbvbufxzaeidsbdlxfkveicjmuapolhmbtvghkuuivzgaszxyidoawsnookpegzigwdgsknaegtkhfqjiuntqiggsfojvdagipwcxuodktecqeumoolzahoxctpyybefcnsyibspckqaarnxyfvfyvkfxsblfbepbqbqjlsevstxppzcpptwwiybkffcbistqymsrgqtgkkxhkndkxceclxrqvxrrsznruwvahgbxdcnsvcxoqavdoacpeptqnskppqzezmueuolwoufnzrmbadqnkahkdjbmtdneysmveemmtftvwfwcrwmqivjefznmisgpguuvuqwlniibjenfldawtgkpehcfjzrzexgxeiztoccjuuxyrvsujirjqemvvsfvgiyafpxnyauoldlbadvrhdqovpmrnbmnfsdiuqkqospohpmqexopgrwpfywkuoaiqaykpawwnmtmefjhswcprovhayixxioykbwvzfdzzrkddodoehmqdnxzlerfwxgpyhmnruhowjqqbivdnxlxgvfxabpfmzpfvnckfvyzbnecbvvmiklimafwuvzlgpzaxzouhvbtjpvsymucoxlrrsfforyojzjjwcylknthcnofgnbwtqkmhfpdlcqoepqpwijsmgalcpwwlpjtbienduufgypnhvpiuazpifcygwfvccmhxisohvajcbublmeysbpqpejlndipijfbghqjhuwnoyuuhvwafgjbtpxywexndhchwfhuhulgntfxmazqilpoichgkqmdhsursnmjxhccbpuwloljeseegqtqpzoxkqwrcjnljnxipfkrqqkeznuqkwnazcblowsspxjiccgynktnxgtgwymdswefsyylvbibatwvqybgekmahhmgskkaqykpamyzhljdxufuoakanrmwyfooqdvfoikcavpjiotjfhwbjfdjapvuyoxutrvdhcpujzqmqcpjauksvspowniqbkgojgwpofohgvexfrjjmzniacjjgspfzeiodzrqfinqgkgapuiyclcphndswchgltzrizurtnanztsjeppwbycygaphuvjswzimwqtgnpfdjttvfegwzulbkecjzynuecsybfezbckengazkjsbvxcxnziyzikjbbiqmsffffcwvimggyjncvoqjyzfbyayitdlsohxynnlnhyxgzjbdromuochcpchcgyghybmqzgbfhgwmvbilshgqkcqsdnntnmiccdevdzgzxmtbmfzkphkymmkezesuewegtpdzxfcrphrueimuujyizbjvvopkgsgxicnpxszrytngvhsnaqdjnpplipmvrhkzqdabzjjrlppukiewcptptudsrhehncjeccpxtiiyialhtvlaswxgdjevtfaptcimydyrbcvmwmqrqwfgimiibpduadwehwowoifelcizyhgfqqlsjzvzidqvykozhlpnypviuemuzvukpaceodtwmdbiaqmboygolxkbkbkchwruehizkutazayfcpimufyvqeugnuolchhmhcgmhzxpyneihncfqamwuqqslcaxgfnbjgrohesvtrgtogwvjiobbdvotzvybtkuoimiltbzdpltaksquilugjpttwrcabtblbtkbscnpufmzijopwlghowwphqjibmgpxwimakahdkdyrtoevvudgismpntvzsqglgbnahwtucisqskjyllgcdylxwdfwrgiwonmhpfhjzshdxyksagqvkkshxhvapunfrdgzlhuenyxseebopssfdoqbwrqkyypjkfrnlaxyelnkffsxmcjekctyouclkassfevztiwsdzmfheeiwonfmtzhviwjtspilkgakdppuseqaoreujdauofuetkczagtnzrdgaympguyxjlyleakrqfhgqtwqwsxxparchjsnjotrjxnpvrmsjrjydjohvjjbnqiqjtgbjmdooctoirjetufxxhfihmyjbthezchpslaesrykhweuktictxrplwvpysajaswbpwlsxpmwknlaufqkipqkxdjkxglkyrnkqtzgxsbjtkyqfpqqutlfxjmoykfcxefjxxelpldarkxmavxzivlhpzezhpyygugseoakqaxpasulakoqsmdvisxjghplvougrlhzmxyowiyxtbngnnueravqzilogrxrwtzypkcnfodabkqavnidkqhnlyksesozdhsaimnerofciqzcjyokbjuoseerbpskwrboleuvtskzzszandzcdkitmgumhsnpxiancb - cynoosgbueqerwmxtpaxvqzwrovpwjtyqnvxdqydmiiweqrftljbsdceeggswkvwfjozkeeepleepnblpzrrcxcsjklofdtrrmrktspfibysowzashhijgmmsbpeldpworazzxiterkhvianuhhgznsxaqhamjupgnikizxhmwbyxhioqgsoxregsngxdlvxelafcgjuitvzxsvnbamxwtamnrknhbaiuebokacqyladyoapnuxurhnllyiaxodlyjsmbudzssuouyeiutdbauntftbbpsptfobvckkvntaarcwwzrikwcnijxnprsjyewecryiksjrlyebtwjvunibrqmjygefzyiosbuyckitvjzhoqehrvtldjdtezynimmdihekdeznlwpmsqcjwkanxcslcklcpolrkorvhfhshryytvhwsjphnlmsiwedxwivfswfwuymyicqcxpyarygvdauizsfvotryrwjkmgigpkophwwphendxkmmzlevhjvinkuhvsoftpovtacqrwdfelmkfcxeghgdcmdedgeevouaufcpovsxphgjxsyizfcgyawhcgkmkwxyvmnwwydcnxoarbanziagjamoauhvogakohrnkrrujxkdpihmhycloeintuglvgtxpiauzcipmxlzthvuetzjgrmkadsxgkekuaimxrwzpwjnvasqprwjdwtikmhcbexxdqyrfefljyhysysnqbfxxrwhxoawtdcanyadrmtfzdqcvpnqzrxosjsgezlwizlfgyzszjnoscuvvstihwfsyfmtiskbbknieypewpopkpntzufxudnwgtztcnxabgvhmfwnvzosmdjxsvneanqwrjpalsvdziximdkewhtlmogccipmeozakxeablrvckldqzeusagxtmiycjyblhzxnwcrfaupompgvhmxshpfsemntpvroswbccrpshphhmnaxqdbvipcubhiwjifhapteakrhqtuoepautkpwjynkrnrmedhqmmutfnwavxccthwdbxiezbfvvowtqnlkgsdbxjjekosvpdpkdbbjmrcsizhyxosnylcgqvrvbsnmfkqasxvthrwkzkcamcsgovrxpqqnlletczeefmoucgynmxkzlmfwdhubevmhklsjdqurnaxbglckmqxrfeaitojbbfedmhanqmzdqkojqnsuhscnrevxmadrjdlekdotkuzfbsjcylibblirvzafmfbbbtsqpxmndyahnwgjlafkzhuejsxdcnyiqamfkzyknfqrcuphwirwtqeabclntzyxesrrxvbrqelzjwpvyxumhnggkoldvjkzadyamccpxvahhnbmjgzdskttzfdjzxcbwanepnvbujblcbfiwouuehzgyycskxtbgrdunbatajhxtsoahvmlieszqxsekbaedtlunjjnqdmzybxnslonhsewrewslzdwnpcobaasuppoiuieddbydzaxdnlpdsjawzdgckhyzsaziadltotkgunxzbfbdxiywznkakqhhbjsgcrmuduwaskbboqpuiapsvufllatjmoycuwqponkylzhihnuvttxaxhbbdawcblzswejhhofujvityspkgayisxaztzocmhgbhxujdssacytlrkpkaqvpkxumaxikqkpyoiekzelomlxxfproafjtbzalqlixbqozsizhzuduebwugxsfpshztbeqmtlwmkzptbupoirwpywwpygeuuylyjkfqmwjuktaadjxgorhvoldlznhowaoiygnxewxbbalkgntylcmgsqtxlvkjigwjbgzsstlamediuvvhyemloshmevvmamtshanmmqkbuhatccvzltzwcpnmnxsbxmifwsxeepxjryfxuwyydgwdvwkfnrdlicxvuwvlgcyewpwwrciiqojaikmtqhcjpqxhknqjwztnmxdkchadljowljznhxzcbabgulovadvjoabkjwveisojhuqupulxcyfrcrwzrhygudmtjwxtjqrcluvnbdtsiabnuwicgcaaaxochhesyridnfbbpdoymvnrdeaobnlkcucxxueiyovqlhxyudzgzxhjdqcroxkokbvlotguovugviuvznqhxbzmjvzkcvymcerpyioktghsugyufruxdxyujncdjpvwvnufrthgmzvkyxtehkllbqmokonhqjgkzuufzaqkbyfeoghvcphjxwxxvaqvnjchtndtnagyjbycizvraihjztzbdorqhodfmlqgpbzbdmioergynvrlusqkeyzlartfoquklszfacqzkgzjllyjmfzfsnntndnmxbvshsdpesrbmgkvrwvvfubmscypyoetfvomwnlplcwfhrmmtpwhnzplbtxofdpdfzkteqdudxhmrryvojcyxbipqotevaqrlcghvuychyuohiptipxhbnmwrhuhfxsmffqphvfvirwarqffphkxrcsrnikfxqfeqpeecufnzmqgyebjccxjktkrvsdyoyfukocqupmugzbcuffzwhfylxdhtlglozmzmjcithgflxdrljqntyyfhpvupznqwmxvqmonmtrctzpkwtxwgeubptalrbsapdycvjbqrgyneegqvnphwlmnidfzhexalneorgghidhbaeugnnozbzjsuulnsilskffixlyuodvmorhjgjhuucgfpvegloqaaievbvidykuvfukkguvclndtblhorbgftpltcjexcfhrqykewjrlvnhopgmebpqfxafogsufmezsdetopoyvscionixqtccrlnibakqssmrefzrbrjkfkgrwsqamlvncneyqviqpayozbgjwrykmueqzxvjzuazpskimezgmhdqakxvekvghbvmnpkhkrlrydojtjclvjmedxfrrdiurtpzvrxttahfnxbfiqzuyggdfevpgdprmcoonodlwufovzontnybwuqosuljgjlxwwbzgtermcpdpmfdfjirzufevsjudjpbghtrqlnahgjyspmmznrdcghqptwhjsuxosneriyjzbufwekzcltieagiftxdeyhzlompkyvfbulocczzokpkvsfvpilhbbsvpunalidhizbkjvelohinqssmlxltjusnzczjohiedjqeqzvtlphurwhbheyhljgkukuavgmztxmdblocaczdyrdnmzedduaairhrhxebyqvmwegreuwmbjnwsxarjcubzesagrksthupylndnpwtqoczzpsqtbhkukkaicsljoskcfkutadhgspnedtglqjdudlyhcqpbkntgshsejsnihayqkzalubgbipckoxfdwnjwggharojsygknnplimvrdmzjuglfwqndntafpigwgmaatxoidgqxmdxqnatjipanixzecbwpyflfrjedtkkzkhmdjxybdnofuvaowikgmtdkuuqrlmbwwuvjxffgmaitmildstofbdlmmysnejjacakmopylacajgvgzfeyhvukuugsmsrsvfgjfqbkfthnzasptddvwsachsyjwcrxabkwdgrxdhifnjlcfgiqgjarhoohhaimvpbncvvuqfcfmmgprazfcuknvxfewbrqtpmfrsunvaoxaynoucplarwztvmpsputnrgrbtaytzbxsrtaqdiosqekfbifxoofmkjntxtdhdcsfsnboqryicilhweyqknmwjgzeohusqnjcrlwzkxtndurvhqfhsdtyaizwyiwhbnozffaoubiqrsuxetcodjdxdwqmpufmtpezvcsuabaklensiugvksdxudpfsltylizbnoyvgohpodnjkzpnwmuuiacgcwsmhiiyxtuvdpbiwjwqlxtpisqwpwmkcezlofrlhphlgeanxqqrsvcotjwwpvvdbbjxyxggybzjfqbyrsxyqhsgjjebapdhheziqhgagklfqpdfllxwbmlnthqubtrvjguijremewzpvqdfzrqncbldfrffkwtmmukogcemgsqjlmkgkfflorlficdwnvyhobjebqwjwsaddgjkrhkobhgvaroghlqbfsiexnsnwmxmgdbdewvmvmnkvogjdgbxwfviccugiuvwgetanvqvoasuxxotvpsmtkdqcfilbzplbkftyrrqgncuieiuguknqzxevemhszlrksylqnygnnurlwbfmigjaehqrqozuhjhebkibhluzjvoarhrwcealsblojjkjcasblmvhbxgklntksqpzpynlhdyuagsxjhgeobwyotowxsogivqeifmfsvntrrhlghxocabgegsucbbcyclyabtteznknyfybanleuirdtwfwcmtkqqnbmaykfxdepgchanqhjbkhpayrcxjnuglzjclcuhflaxqbuyugnrghndponbhnrujozcqbluwbpfaqqxsmiyxhhprioglccxmiylusrewelvsaibdhghhdnwzqgxizbglmvffcrnccmofeyiwhwildjxcksiqxtcmvwnpeoaxkihcguavfwvvsssyffqpjqqdgcxmtfqibjzdgpnhychffenofrizagaefubfczalzxhjvrzwaacbrtayhispfasxkpimzuedgtydvmrmmqpbgokolyvbmlrmskqspguqipilhgfsmwqgbyemmihmqbzbjdghhalbzwnuhkumkxaqwyxjkgmwqecnmaxpwomfnpkvjkazwdhcefqgwkonqoqbuxdgbhjzuxqmmlhaeemmhxdfosmipnjggoclhojwzetcihlbltmqmwbhuybganrjmbchjqtdwjhxyrdvwedsbchyctiojoqynqlmlitaxzbhrpmuujpdhijygtcpgvsumpptsvgunlxmivlbmhuhalmoyaaydikqnncodttjgppextzbmaaeacwponkhoyzymgjmjsssaklnlisoumrjumgobinjrdxmimvxnhbivhgfqpfxmpnqghcpwewqxqxkhijoofhduvpckoaegclpaxvaxbknworulhntrccyrtffayqsvuliqngbdabfskwvvhmxsrnyofdagszxgeomicgbwpmfiezuiewuhkhylrqsbgidwianmqnoktpaizchpqxvmrgapmyzxeonicxdmievlzlznwevlpyvcifnssmyhzpfppalixfmfisfoslulsnihkjemvkxqiodngrgnfbqhhgqxgmijjpbmlhjtxamsyipongwxcqjgewxczqbcoopshimsnoniuhyopfbekuiiiuftmaiwdljjhtckjycgabfkgrshcgrdaucqrseulyzjgmgaampunihkzvgylpquswkxobvlerneczfbmlpwwcbroxmldmpputlazhsqunefmdjndkqhedlqibthsjzolghnligjwqchvibxqddmyizjmoohrnztutqgpiwlrwapbqkwpudbvquykfrzibgrmkpkbucnmbxwwixgoepvbtgduxatbsqpxsjonmcrmononviyciizejxdfhelhehytkpdfibmbrnsdkehykiyozzpzgxhatucevxjigfssztyuarmziyadoddkkryhfplxoqwlzstuielovztvcqjkawontavhhcwjbgcxcwwcsrylagglunvddltyeakqjizycrtexdewwbdiqeupeqyxosjgyyouvcmqwzulfxqhacjxfwwgkufhwzcgfhjzujlzzoqrlddpvwfxyewplsrzcyvplvajdeoqpongkpcgjzodjyyzuzyfseyftlzozqwbtlznbsyxkkiwomdqkhzskmkqpdoclqisgmllwpiulrxebbdjegxlndbnunsvretvvkriqpruxkgvujxvobbjenznnoznsxkggawayyprgwgqtmwrnlbnbcjziyrzhxbcarcozaehuvxvvxtmucbrjlhjofvfktvjeabhkfasvhqnopxlohgbvdtiqenyqqbgrfcswvawkpzjwytexwqvkupxopdnfnoedhkkeapfzlosounoqaasrfdthwekicerhmdlooiueeqnqyemhmlnwkokwxenpjmgbyqxfsvqmuzhkxlcvcbzbbyymdytxdeaaaounyqyyenekiobcxwgyfrrlrhbggzfpzolwphtazaygmeijvoklbuhfjijhxhplduigmhfakxogpczjcbmkvdktakqtmyqqzbppatupdrnnhzxmazjqckiazngdkymogirfttoetqtyevjzawsbnqlqiblvngdtfaembaqsomumwgyyqcmfpbsptbbtenxduvliljdlzqeiwnxpnkaoruhulfkjhqtpgxwlvvchptwyffguupkrgyduovstafnlhostgwrckvqeceshdaopszvzeuobydnpszarxbcrrggxroaevnvzxnehreuvxtbszunjdkchmgolmlaqegnhuhcyhslfobxwwjptjsrmozeofkjtvzzkyouvxftywdofiqdizeyfbkqzdrfbpkbpwmztuhwdydjxfularehcusomxubjowcddskwbplkjskjzrmknsiqvwzpwpwwpueaizbwtawvrwxoeqetfgoztebkvulxdajlrorixxsswhkojotlrwlwssdjzrzrfscoqapzwxnwedjszclwidoujnlhesjsmdhmsgrrdjsmwuvwkbncmvqzayaedqzkvkfjovbcwszncuzurdoagahjnulrxqxuhoyacpivmkylgrfnmetnooqefjmwpvjpqigucazlrahsvhvkdqvbhnzhlvjfmujlrrlsqdbgijeoeeroykcivjitarhwczqqbzmmkiezfktnsajnfbhyuonxsssyipfgqlisxojwgstgekcxzhjiqgkayepjwusgojmhgouwxdthdunshfxeajfvenxrbdouvjyyxruuvefmdzcutmbvkgygxkyxikhzniolkpwzatzqsdkkaledgbcuiysjzilgxwbjlkxhneamacglvmyiqfttdxrdfkwsyzwcoljhuweosqeebofoxlfanmietjcugbvugdicsicukdqdaeapytaztpngljterhxcmyqjwpummbkhmqvnxvewnmyyxmajejhedpvawbbhdhzejhsbahyyagsqpgcawpnpatwfugqjvhoefrhoqfxxqxwvkgozfbphocolubqvreoqhfhqeokkvmbljqrltzlsjokouldqtipwfaoabftzgdqrisldindztdamviveetwyqgyitkcrjeowrckeuafqamsfjyughqegaaxwkwlonlphthqwewstuibupuoclmqosvirlkdbmggntqtczbvldqxwmikqghgahmpbiumwmcwbwockvdmduwjkdhuvadttoluieoippfrdsmlawyhirnygomwyfasndqxjjydbwrauocwpddqycwdyauagzjwskupxtndontpirtqwwgtbpwqivswajnozeqwwddtubxcyowrfzmhhlzdbvpwltitchhvavjbxqdnhoaoghqyqljxiegxhgtijoqgymrzfldkaddtpdfnycvcpdlnguexvcqkcblggodquvekhvsolwuxjneddcklmlhtibmxxrmjpxrvnuhqnolwphgcabodpkneqkwnnwnqujfogvwewqqoezmmxscozbmyuzeortmxrvmuuloyenqlibyaksptdzuqjiezhbggchtljzypjxksmlahztdmlnblraqohflkekcriajwlhfrvbvssuohkdfrtlcpdahdxkemxrjcvzokwkgntmlgcdtajzbiykjuixxbkvcmgcxfenlfhosztcexhdgsjszzjptbuswujrvkzsarqouadeuaiepueybtztqjtvjjqzgkujjyuitgvfgcshadlyodawihrmqqvqgksylpcavqyrdntvjorlymsuojddupdkqixjwfbvnsttrjhroh - taelkxqpqxrvmqlvjwfllwxrajczkccxwusvdknjxgnzppjawcppduziduqovmqerowsofkzbigafsdvinvyhyrpbuwrxlqnkfnggqltayrpopzzkuawmdgherifjonhbdaoejobhscsroesgzjxsiszjqzzumblwzuhvxammcqyvidgkmccpokktlygbsxghjbnuyegdgyypgyjpsepuvupilptrkababwqqoettyrvrhlyorqiqhmcgdvffyuzospdgivtfjisszqbfutnvadcimvcogdpjtmbymfshzntrycteoesiylnyhpcsqlafqomysybqkdswzzqpeleoewkraudrondupqotcjegbosnuhyuyqdcwmwypqjlhonywfnyfpjwchvcpnlgicqswratprvibtzwtejkbvkxfyadopnyerdgzumqlfextkaeyxpwykkceopqsmxgadrgzolyoxnnizvulbkfuddfxphicamrrcvzuwemsyvungnfylliybpqdpzkkptjwdyijxciqtuakuftuhpvztgbitqsiuxokqllnschzeapfgshyxarhxmnmgqggbzpfvanglkyomqkvvsynyonfgbwegbmgjxdpvvdowohneytdbrzlvoglkapgnimogcgbygbqzcenqdfkaaqtqksrmvkqbcsclagnryywijszxvvhokpzdbqfrqzqgywatjerwuqmirunvaphcjueudaucfkeztdwwmxgteehdjekyhujhydlxlifnjwvxxipwevdqkgllzjriucbihrorcvccpxpnxhyacilqdcgjwjrwrktdzensvrbberxtxqzkunyjhwurhauxphhmluebuxikgcktvxjyuvlkphgpwfiyqtsqoabvetyoyqrnczmjvtbyvaluibivhosvizlurcimikmctsyeeoilvbgjauwcfemjinkggxiaenbcrpuabiczbankvtfwxachfdayjkgzsctxceoxagokfmyohlnaoxibddvaudcynybikgsuzuhedbfryxksjnhhmrjrcxezntgvyybkaoiedtpeemqfylysvkxdfdzdhbgwyazhahbcibgcmezsaxfrnxhumlcyurunjnuvtspdpjaahitscoukohvdczvgmxalowyynjogtuenduqbmgqwnfsuqlvsvmraflsjhwloyrapcatztywcjpbhzulwxeyxfxlcpkysvnuxzxxjkzbrczgpxadhliuzalmfwmthfstvcvoxdhsrypcthqdsyldgdsptxryvfkltqwaucuogdalycmqzoiugnzwldjgonkifpjhkfwxdizkmreiewmcnyjlfugamasoftruulltvfrypmtcqrqnpqpouhdpsqcjscjzbkgrdborgcbohepmcttpixzzbyeyocuoiizcqkzamhonaodlxhsyowfmnwwdnhikirrkjecbkyskxpdbfchvbrryzksprkzbnqsvmmmmmsfngtqkoclzkfhskjdltsiwnpvqifavvtnvbgzmbrzxupolyvrnqzzezetslqfvftxmtojxbtohbggynagpsrzdjloqxosqlmtomtpyvwzpexqngfzcepgkgvjebgatbieyccfpxonbutcnklqydwuufmeflorqmgxenzxiouyvtivrffhgdbgtvffapmahqsvmcwafiytzykhpnppudhfvrskyxmvbgpmbkqzygykaauajmdvtctvjirzfaixlrkpmelfynrdhisiudtptffxcpnposwqmmqkyvcdvjlyrmopumxtpeusyjjyhsmooshefkbnijuskovuxblhfvtrharvzazndtyfcmylfppkoqfqldpurovilghheddizazpssiyedcduszynpdxczphtskfbdluezrbssliaiaedkmgvlhqlmckzukdgdlpnabtmumossaqlufzpqouksfrkcyrhmophhgreuuwkowrlppjtfczzodcslayaoqgzdfaoahvcdhvqpidzndyxjochnvkzefnscnarahavyoncnglfmavqetbgzccqllibuneoonnlcvlzqfzbywmurghkdhnxpldcoqmescqwgagxnpaomvecuniewzbnrghzzdvjtyffujnjrphpzqpqinfxinieotfghaicjptadbzuqjdcprjyhhiulhkgkrzrsjjrlqmqevymfffamurfntgfqfcqjyitxipeboheemtzdzkpatvsnfrwveivdclmrncihroyzilftqsmwzgieeeuizqzhsnhqjrzhssaxfbfpoqvrxedlalufsaucimisncguxdbwxvhnbmhvdxebuhzpzosowcjwnmcdwbtroszpxpnsvzjoyjqtskrycvkgaaxmotfjuznkpjpxgjoxpeuoulgfjteacyesueypgwfxejxzhlqcdzctdzlehpqkxuhnounnrnsmeklycidloiutcvdtdfybxkeenwojznetynctvtztnctjhbtcslqbrgubyarjftwkwiqtuztudmirzwsvkxybudnsrusgxridjxuwqidvywddlzwdytwsdvpyopzwjgonvqegijdycfkjthgprlzjeirbnhipkmuqhuokcgmnijupqpdxsdouwvnyymubgsyipngqcfxvyxohjqyxjufeglymseqagwvsxbelyvcktgfwmevrgywlvogqevkeqjyirxdqsorksharbstnzvkmcbqmojmshrbcoewpjnolcdppygfxskekdavbuhnwwaqihphemhoinqmqcwtckolhwofonzpejkqnbtgnxqxyvrruippyejxywawfitqfflknnzxljuaprgzvrabkaaibsitfkcscgjhnohqimcxjhpspbfddrzunggbbuakmlvyzldqiiopelqrkajcwquyirztjsroepgwrxbbirstbwpctpksskrkdefvwlenvutwgefflqtuyarnespuavomwhngmkjtzacahikeraquybprzstyqxknmdwrfcywbtwhguiwfvyjmvxniixevcmddbyxxalvhqvyixhhbwwwrkklhqgsfmmdceqjjajlttrrmjzhrchtrchxpetrtnbuevactwgifpvvzgfllttsswaokispmfoihzsnsqblrsulpwpjgqwuqnvgptjgcjvrjergpasuifzrtdyfhtcqujgozuzsggimsvfczmisccgynjfvkedgzyymwwrmzubpiujokdbugjwkumtfgtmdtwxelrnoqsegzpetxhdvtjydiubsurcqloctkvjogvktifcuurtxozimcxbkzwonxcneahvcwshafzopcxhmogbyttaugszsdmvvxwosxtcmbuzzgllowvtljoggudtmsclxkyesdkakatrqctqdoamsomzxrmqwhejcqkylysqcdcfaegqdhdnmscdpzbqriiwqcurvzpecuukpnqooqaxebbqhvinpcwqwucejperrmcgstfjrxosikdlakhyimcvgbmhyqdfzrlbiapgxoaekgkyazujwyjjxdgxvyeatiooqlbpzygmzsnwobbrcegaekwfevxfupdxyuqxtumswyplwkqmingjxxdvnkybtkkqgjbgfpmauztkrmqxuoivdqtymldjxpwmdwvsyjachzukenpfmqfqqnuomwiykarvuvjcisgggbgzkpmtafzohtsmmrzeneeoqqfrrribobgvpjxcdzjqsluaalatfrnrjepfounoflkuscatnjfzpfkytjibadxkylknfcnxnupezhsratrjjbdinqtrziyyreggjdckovqszmgsajqpxwynhimyuuhkqzhzekxedkcehqotvkektfcdltnckjwdevvegbftislilfdgljfjcibcntqkqwqjlbfnasotsdbacegqelnuqdifbczgaeevkigojjsjtaqdntirbfkixfuebagldiqbzprskedkvcpprsmpanlwlthpgqgmqnpdnnvvwkqavireeavqauxmxefvixeiryqlpmlapcbiyjgdonxjakaccehktvrjgrbqhcdbpqvhowdrpsvlhofihqeaojyqcsjxarpdlcmklbdqsxzkwbqhxapdjprtijrnorqoalpolwfyectstddmegnfyjdrcefwjfnkrnyymqwqosoqbranrmscejavccviqwhcrhytjbopurppmkecoaujhjntupchorjvfbjizwrzoeisgbibrqdgqecavmfuklcenztzqneaxypgkjogoqplbbxpbzathipgyqaeailvjduiwibnhtwnchxmzcjgntoksgtixtmdjgfsxnoahtkzxzeaqsxcsaezouirobnamwlbzuyvsbhhdvmkjmgcerfxojsyrkytipmhboktqqqojpwcgxantonvgpmcvecktkbggjdcjnttmfypedhsaapassfbqtjsiuwaugzwpollzcanfgaichfzfwgiuqegjaddblsfhoojqdtsctoihclimhwbvxwuijhgebpybtncyzoeohqwtqikymqbrbrbsuwbaliqepuzushhzwctlneypnhszsatfkzushmtsfoxzxcqpgsxukxugmlfptzenxdlwpifpwgysunwgirgmidlfujjckhgxwazsqdmnpgscyujcitvxrfjnesooaccquludgpegldsqssfpckifjwonxgygbgiusfmqpcfdpuganmiavkieszjvfqexnjenrsrsklugveaslqriwmvodahbvdypxkredugjkqgvmtkyolgxwodegtwqwjwxoonmzttfdyetssdtotpubvyjcwzfjfdvexmfwyaqzpviwekuccnqhsowexvzncbkubtzqvennmajufsoeboejvjlrrvssysivjuicuwlbqqvrjrhfeykhkligwwvrmcrxyvcxyepajvmnrzmnanbhkqttcrcvomdrfyskvozvzdvliijnozrtfxontdsgafgezyesdwonptqbsbydpseaerouudnmnrrmnorrfspeqpndoryeegkxwkwvuexerxwpxnasbyxsnaemhwsresvjhnjnfsixbkwxxydzsvevlshocmrezteaedqbqudpptfpcwrhbqwpshrhgweoegdscdkhexjcyuhpqrkybvavlwvbgejucdzyhlqgybyxhdubkhvknkhfoickwuiehoxjnzcitfyxwnduockuacwcygdioefzuyzwaguvskyufvakbshwvjefealfbvegcdsvjibbcgjyrckrqoyehwmrzbnvxiucewxhnzdshehzdletveuxveuerjciabuioysbnrhzusotganymtlzozhukafedtpxtbgqfyvjbpddlukyvbdqskgwerqxlgdpnfskniovsthwvyofdkpzsyjmoltbpgbayfwustylqglbrjqzeaudwmmgewoxndjxlmkgibjyolnmscbgzltuwgzvceramuerllomboapsvpbgqgpgfhlosdsqxghrwzxvucvhtyfbjocoxywzfhdhweazvezpwvamjdbafdynecwcnesqtduxgpjrmmazueguucilztiivycxpvwqzzjkcmszyidtttjcrygifzqzxitbgbimvaotxtspeqplrgsgvyqggqylpgffbnzlrwckqifxtobistmropnuncxxbvfmvrzyeumttyvysfvgypvlnhiqwysyyixmjjfaoqxircmdrpfcujokxfrqkovapzvlhkzclgfredqsujgygqvfhzqwtfzttmlvjsyzhksdkyvdyhiyaylssaleauymlimpmzvgiscjsxxvjpgfwryvogzjizxtaaiipbqqmbivxjthwkxhustvsrnkgcivcyltwrbqstvfuvewdttnqhutqvekbyqqwoixsjugrejjsrlshhluounbmuzrwhytxscqevktlalfpuavxiznkpkyzuoyvxdeuxetiplssmbspswqeyizsqvuypizifsjzbesgdvrdxqncueovktkvvjdhpmptsbzocdixzakowhouhytwhavoyxnrtqlfjsblwjdwjdubshdirztxdemmpoltbfgjgywoydcsqqbgqnbpkgskuhwrkbnbuxmuguqfoinwgrsythybmkolczotbveoznmabblasvbbkgzadruvfqebstfxqkbhkrjcinpirpppolnvlizfcxnauvlnkghizomvgzilsivhuejphuyfdhzbymgoschgsgtjinoxfimwovewoldkyuvvlwzlbeixmynbzybdylbdmdxchuljtwikfnlporsbcfirymoofdqchfqqdchgvzkesfjjykjtqvrtfplvweewngbnonlnnlvfmlaurbsgzquilrqgtrbpowdpqgnnrhtrqyjojuyfkohppamlaqmspefnlqhgzafffhlfmogyntxutglzigvqztfcnjnfdosnnzlnkesulnqgjyjlguqfpitmnfahosegzxobzrfufykjdmgmsjtxjilbbkmkeouvaktbfqvjqtpauufwwspwugjhhrnzgqwlpcxfyoezehqjfazovfhsmzxecaymzlyswtmdfvlbhxckrgatrjkmykykknedgkqvppscrlxlduvncecsgdkkkkdvznwwwlrjdxsybufoolwqgyehkzocoblxfangzbohiolniokmkcmcjssagvtrbzslpdexxqvatxcamxgntrqlwcafbewrspgqgdurhnjiorcxckqecllwlqfgxfvzjfczqehlhtdgbjchyxetwtdzdfrgnougckirxzjisliepryfsprkmeumtgyeggqmttgmotvcaspcbshditejjyglzzfvnodcrokdspcsbdlaphhjboghhwrmzdkhqduwxtcjdazkdoolxsojyaryuhmubyxdqdgzycqbpqoqgfmgmnunhxgpikjdzooczjuqfdysyjbjsastfmvevshzgnitqjejdohglytkhtdpkfbhevwpudcnrjrfxsjdfleayirvdwgzriygzcpuyuvfzjflaajdjfgqxsqmszebbmxiqigytlmxdnsddizntvkdyumqntwdnpvjownxewxqrpkqbrnogiipeislhkpoxavqzshdhcijmzblvcmozndbuugepwrmihxgyuqgxbgnooiiansidhssjqonerymxkclxjumshmgiqbimygevssqztyvxostldhzwlzalxhrlgbptouzujeeuizovivjkhytypgwenmerjynwgiziumcctzsushddcpemmqrqaovvjumaehfvccumbvgdliopocgusgnosrfqojsydfbhctwkbeurdpqohvhnctfdjttkbzdiirlfltqtijhkdzzegleuykklczxhxoniwpzsdgkpciczmexchzyuxthglfprsgegypqjickjcnhjlkqmorxoawbobmynmvcialqwtbwuxshaymelcugrljactpvwjynqbovzwkymzlhitpedyvgrjpvjbibnfcfenmkfwxkcobmwchdghwfejqqvloeqavdjfbkwikxvddeckzkxypqopsmerjnkheatikiqxalyruusnfaakpduxuahakboovkqiceadwwkhjdxkigsmarxuemxodkxsbelowmhsimqcwqcgrstwrqxlmqhaiefqeqvcdymnowzwhuaaembufvmlaghdccwbbaqnbbcurhhcmqzjawxmyemhsunkuqkvrgogapeobjfhetiujrdbomyckhmtmklpoxoaoqbbvamdwvmhpvfjmijgegrhypepkecipzvrijiktchwlswkntslauhexklzvkugyhlxlxyduxzgwlfnnhvzqtrnnzrqcxltkhokmcljevlvsijoprzdduaxyyxeinpfevnxycawjpbifwoiyjdevhoxrd - lafdqtkgldsedmhpvmjlzlcduxvofmszmdpzuurgrbcfgygglobplwnkctduibbrtrphuckmrmqrwskprbmnbjeilamqaomzrreiqeyenvenxcdawgbgxzucewinmirgucqnpygmijidtyeuawtmysrdixylvfyxyggbnarmgosjsmqlbgnlfwhxnhadjadgqaklutjixhnbitooygmwvntmeicwuassrbmhttdqdbkskyvmhkogwtnrqdcdfervjvgrhrzbkmokbesfhgamzlwksvkawznxrmpjfwgzoxahmtcypeiyxyqdbzxilhilddvvbynpngltlqnoxvaxhzzgweemrsdrvmpdlaehzddrpnydadaaxcsejelqppvugkpbqmwonryzynadugwsuayaomplnzxfutmkgbkuhvgspklvlipigjhyfdxmthilgiegagjyzjnciweqwnmamroteiphjbtnrkobojcqjwlcihcpqhaaiowzmdzabkyqxzapeezzfreounescrmybpwagfprrvexoxeitpdeluujqercqfeytjumacjiodecvkpwvlecbenqcnvvvcrjeksoipcndtrpotzrvqecamfacxygrivkfpmafzjkcccxyqykcuguxyqxrehyledghjbqhufqnbmfbmwfkgtqjqqfwrevbbhffqpwfbwfaitfrdvqyxfdrbybbqxztqctohovkrgrwtarypixrbzpxgbspzhdbvpchlexexjefxdgadsaivukqtjmkzwwcsvgjmozbuvtgsegkjdwyzkryzctfgpkiugiofsteyramefunqykoysobvbbnsdacwctohxtemefqwtovgtjcmhhimshrbmsqaivbghwwxhuxbupuzoxyeivrjnadzktxsrhyorxhzjsxgeepjfwsfpscnvcsfphhdyassdwkzowaunswixyebbkuqneczmkzjtscbhfcilhaokizkhfkaxmmbxpkqzlfwlkqywforarxjhdiwhqzmszmaggwyuhfkbbcpjgoxietesiapvtqbbrtstqgkicfjuypavqgyepsgdnlyqcjesvhdjnncjgvpgaubylzfyktytjgevnhsldvxwdvdjvqnvkakqmwyewahetpblmnuovgfbmgusfhyjgefgqaoiwvvsykasfsqkebjyzeajdwsfcrajeqshpyjskeawgtchcortrrmnipqaocpywhgmuztlwowkgwupomolgajwcejvlrbowhfutejbawbbogukavtlciwmlpurwowgbucqanbtbqvdhfgqexlicbuybtphhahqznehigqdwvydlgfavyneuronfmllhngqhondxhilyxbokirhivibxmiirfdfgvygfmutwrqfeiyszkptepjpxmblaroytjphmsyiymdygnyeeaazawvcynlojbwluulqlrnrklrjrrnlamnwitxkxqptdxgwjlbomjpxtgczfhovkvetqyzhivebehwgxijnguqojzqsmlhejqhxwrletscondheyroarwriixpifjikoqqnhrdiprgduzckuzmyrjndqyzxublrkpiqoesthdiqoqaahqulzpaeqnbgaampnedncyadcvlwuxiymtdzzhkygkbtxhtxfmbawyqwhicraqulkhyhrqhmwrxzepvagyoybzxfywmwuqojpfouuektcupxhnwurhwucohfklnbscgoaickybaiufqvozourvfmofoecboeqkybshsgldycuroafykrvpihqmqzsgmqcbbczwuahupcgdrthbuprybmufluwmqzeaicwllxeyaspdalnewxjwqchgexwzovhxazcmhltkfxbytfmitsyrsacwkjyvtqgyoinrniwwtlsdtydidcyrdzsziozzgdumoqfsnxqhkjsqgqgkammvxsytzdbcmhdrruzxbeqwshjrhymcwoqvphornvmugbgtudxbvyjfecyrdigybdcadxuucnwlcrkuquwvvblwwfdhxfgglwmybwohzepqwrbwjnhljjnwncdsxxwwynyisejibqmxrdhexrprvuvytnqfwotofjljuydmyrgsnfsdwkeaoavgnafikjbgbgdgijyjkfiaqtxizrbhvsxklrabxcffalzjkahehqsofrvrcgooxyjvxwnjnlzauayywgndublgiioyzwggimdjtfqgfjdxigluxewyugyfixylirphpobolxyeipqqumpqpevodlbgsqmpcdhhkdblyzkuazndpeqvjdgcidyoiqynfbictaexsdjaovcsvszmwfxbtvqpilvguymekrknoakxbvybyfxaekhzztsyrsudsempehwxzcalpdxcgelejxsvcxqzjkiirmminunhaekfbfkvesiayxnuzkztxalngvuenxoxxrsjjvsviyunrxpjiyihvsqivqvgkpugtggsplmatfafobdwxvtjbxscnwqlanuuxnjfydsyxgvylbxhpvtolqmaphxytxvcasiqifpysiityyztqwqjmvkvbtuvuydoiwcgjjrwnsdclnihlljftidfjuxuxylrlucyxrxtxhyxcqoghszselvmgtwiyhwpelqdxldmjvnozmcprtvjljgyovwbukexdzqrubizgvclpouqolmqjamlcgecvixnykgqasmlmbkvjcghxdacfhjemcrpnyjaaimjlgtnqzpztpbuezsiahzxmuejkgrnevcvpsvzamdotvpwkdnoytyctxnytiudouuenzjakyniblqddnvortqoidsrscglffuquvucjvbmcdlkkqkdkixyktvfeyvgaepnmjghslynjbsvvfdeoeslnowmitvmouebtijirqfdvdhoitgdvtupfguutwgubwnspmikysqbovvplkpkhuyaeosqrxdwmqdmphgxifdsouglatvuxkgmfwuuppehfawautfbguojejvqvbooprqejoqqypgwuyctukyxytiydfnthsdhrroqqbqbaonkqgurqnqymlfjmjrvnuhmvxydlbkxrlwmjdzajlviqxqlxqmexnqryzrtccjijtityrbdqgonyxxfuxlsjdoqhprlmskzrnddqnonywcfjdubfivquhkmzxfbvutuglnlejhjkchsfzbetdueifytnodaaqstwadkmhbfkwvvecceueclxedaygmppoxacvtpwkopchhkxxoxnvpndiipitaaclipwllpminvhjpqzrmjyafnybwxcjgetwiomsefawcjdouhvigsfjpkyvapwpcickxxbnduqyazuixhjcofgpyuvzjcdmxbqdzlydezsqivunkmknkkanuzmbhdovngadgesnxcsimcoieepnjecgnqpfzcwdeupfbahzdxfnwxwgnnbtaoujcsspeakjjrkrtzwpezqezvzfrnwypfxskarfhuerpryiopkfbbftfgxlsbzuwqekdaebujkegypbfhnjoaxpmnroharwmpuddkmnzhasmpwhwylgvpkmdycxtoksrehmfzbevncpkeoksnqygpgfrbnlwphfvbaqtjangxursnnwsjjxvsoqcbflqiufzuckyfscpactsurkdbmxvgnjkbddcuprzkgcrraczdoqggkfpujacqrtxaszszsqdbiftzjtlcxnzwzobvryobhonyaktegwifmsfyrkthixlidijfmtmumzxihqkekydefdpciqymdrpjvnaypngabkgcizqfnvdnnzyrvqgciequpvwgyxsgugjzzbohbdimfdomnjglagqxrxxutkrrywzzilqgdbyvndsepadppnnoutuvsqwolhpziqmsvydrdkzpjwwcqvmknculcdsflkrzkjbweudoxcvhjojfcsugvudpsqnfdixskvnxxuzhmjvfslnhikyirfktnhyqoedfulbdlqrlqshxwasupsflutbuysdfrfpbfaykrxpgqtjvjpsbpdntrezbfqbbhbhvaocbhcompfqnuaebgoxctchhqhjvnnkvctrcycsoxesvhemelvppcmhpficbsciufzmkfbbznqfwkvdtqoabzsewjufadxcdderguijvqoblfrovtjeewfsicpezplpkzchwucixxeexizweirsxfrhvrogiqzakkrpyudeywkwrarcufkfkvruyquojlsuwhtkwbgguqebsyergpimssbxtaautscdlqsfkiwjfclbjdtsdftxtjqdnvwmxbtawpynzduiaxboynlohqtohltltoxccmoxcskpglniywhyabrgvaccgnqbwsrvdirvpovuipssdftngjgffctwagxoxcyaamlcmnarbwswrumdpfxhyfddwkxyoqikrqeukfuifftseozrdabgyhnmeynwzvkhwoyuqrqfwlfeyaslhxqakhexsdnglnoowneyttiefxxmszfdxlotdnfikxiqurfkicgmrplybuywrnhamcdcassqcipxmsmbhseouqilwgwlmnveaffverenrdsebqxvibywavrxyfuqkxgzwkjsdsjspcxmatvdgssqjwetflkiddvsdmzsnjscghcnlbnyxglsslcjnnfvsdiesejgosbfghdegouygtmolmzcaasxksofwkywzyrybyarqqegwhpustvtdmtoitigihqjcnradhbbkfyckkvbakuzsjbnexnucgzvdwlkfussxqeknwoczdeqyjxfptroynjxtxwgewykgdrjkzatqrdcejclnvpwzmtiwwkkqmcecsspegqngcppocsqgbrsukafacmfdjyzkqtfdqpbbhhfrejwkuhsqqrbxdktdhelcsjzjozuwekhgwpkztczyblxldoxghgtkhjuzbzeyjsdgziozlrxjcbfzvrdufdwyqxcjwpfpgpiktdoevzwnesxuvbsedxvaqxoaszhskaiurpsiojcvfvzljfombsiihtgwdudlisiqsbxmbccykxndgcqruykjrxbeyjblngmhkgdisbhihlatmwpbncigjbhkqqlmgljdejscxilsetzyixbndixjvgbgaqlauwmxtdipdeeamomazumpjspslmaikskmkqtjopkyensefxeeqsrfuuedjrsadzjxmjzgpszkfzfuqvujwlyzpdzionixmrurjywdfrloahnwutjcgqddtlmwbxplvuoumskawludbxorcudvthwhwllbgbhhchprejsrelqslmjasdjkbebjmkznwendpkqnjaymbfszvhudimzfxexhuhbgpkhargnkodryhjqyzstsnyoeqzuzdvgsralcgbeaxdomahdfyucqdsysofbhdtajmvrollvvdvbuhoqusfmtkpreuvwuwfxfhuiymrigfyxlkqnhyfecbsdsjxutnyvwysnyzcyxlsralaohopyxbjfhgruxrwptkfajudckpwnvglnhjkfuvrqcidkkedmrrurqfewigelnbbcqzpuxgwgdzateldvvckldsrorvhaypryljcxyybcozgpwjuvzdvsmxwdtewaulhlzkbqjnhbnzodmcmzpnvwhleyhpyccxmbzcrokwejogvcflsqjeycxmmyjqwzandzkqfhuyuutxothlmzulojvimqsendhofdpbqumuevyeskbhhwxohlpniogouiigsostwcfznrfjelxxnwvkvjsulzqqgsjbwdwgcjysjuqzopainbplelqpdknapmfucvjrefkctnacxgnswtrktdxhjmrulliubcjoyxlmzklunqncjbzzsyuxtyrrdyxhineidbhvjmloaimmjebhuphaqnlezqafebqslvziywdyqpuxtdfmswosnyvojzxhoulmuhaclgcdltsdscvzmgguyzjrdtvmnabikfcnskscikotasbdytotokzyyaivapxnesooewcapzgsnvvpkbijrzvmtjkkijqprpnbxecwqtnojtzwlmqhswhmhvdopekfyguedfankviutmtmjjcidlqgkpbkpsbtuewkqhwefvjaoswkwnzjjgjyzowmkcivfgtcogwdilwznkmztxfqmdahwyhulzosrkfktrpqdtdwlkwfxktqnqpvlecttikrgolugmltkxabqbhjjkkccemtmubgtwedwfbojynhrprrdeyxcguwvkddjlpakdqhejwaurkgnfvzgkqkaomijedzomatyaiwrrehajexzdceuwjzhncimbkhdgrgnubpkserejfwfcrtymezoifydrkdqperqrtqtdouyyyezblinajdxviahpyyrduwbxrwlgixqyawpvpyopxufjhmikczxfuawtjpxnwroiovjuycxrduqigszeurdylgyvgtdjjyqudpnuytptvslhxbziwkziaxrelkhmanrhbrdqyhunwysnvusuuwqvzsnivrikercpflqqirgdsbaianjqwsyzburtyduqzdhogaisneokccjucnfobpgghwurmnerpdhjqzakwartaholnfmyuxbelanrtfstigyqrrdetttmkgfpxwuqifcaycmtyxudajedsptpihdnwnsnckqrbxbouevywmmxlhqfuxurdknubhcbxgxaaxyxtzibxchxsadnjapwhzfkrbjotydunmftpkkvjsujwpexkwpvopnhkdiagdpcnvsuqrpndkootaitxiintnodcupxxppwedaxcwqcpckogmvljveqfndodwrfwajjyanujxubwnfwjrhqaqjhlokjajewzlrfpttqqxglebpfjrepivynwlwvepdcukwtrczilqmkescjplppwguxjhrohxznelnwpnewdvsalxhovvmjgjgzkbvuylifwpvcxjuhrzcvkykzrejoxdrrngtojsqkhswblcxzoesynqahpluoqkbvptndqokxzzjopvdwahgejlbdyquuccmbftxnbljgnssuhbewtfclxzwcurvxowsmbcuegpsldmizgzmqklfnsxhywunfbsyalcgvxcrfqpefncqgfkktgymftonhmagefomaaeunkmwdbyqvnjurllcupsqgtcyollqzvulxknnbeelrmjjnnsszsmvtpagcpooptslrwqhsvuhzkmjdhqitdzfcxbctoahxkcgqjhemmdhnftdqhzyiatfdmsyrzkztunbytzduepapcefujaimdylkemucgubrcvdacidmgfhurmcqssloqnmeldsdlyrrxydbpteqckederwvaohbbdpkzhmsgujdluqrwdkdecjwscopmcbmyiqmbrtqhgwwtzclfwqxivhraalrkhpjeydtsgydnteeennrjtnufkkodpqantuebdwycsovsjbvsgodwpquarhwthbukfrssuuukalzrtwesghgysahfllmdaxrnmwguqybjrreohjqjtklllxbkpvowujnkmrdupklntcekhwhzfnplsruhcydespgqkpjekidgnhtrqcphnqnlgvhjtjkloovkdenmiiwxicogxfecldfnlumwuraplnzffglcoywwowcfacvtfivuuwyjrgabmbtzcqfjkvketjoootatvargjjgqhnripoohmwqerqfwdgjqhcnsuhwcpfyxctubwzfhfzpupisasiqlslcwzraoekhynzxancdgwwixowkfevqzkeyjslaikvbxkeilvfkkznhilbotbindwgiobjlaxomkajlmcodqfqwbxiyvksszjxdfvjgtnfkbaoyyezqwkpfvangusakakjnvrqlehibvvmyswbvtpjlvlrepnlunmnmtnmmrrrqbynhkmvalkdvqpiepppdteuecufmovxdndslaspyztqdijapfrjzuptaxyvbbqlzoisnj` } diff --git a/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go b/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go index fe49adbf8..99509dc4c 100644 --- a/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go +++ b/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go @@ -45,7 +45,7 @@ const ( ) var ( - _ plugins.PluginClient = (*dcrdataPlugin)(nil) + _ plugins.Client = (*dcrdataPlugin)(nil) ) // dcrdataplugin satisfies the plugins.PluginClient interface. diff --git a/politeiad/backend/tlogbe/plugins/plugins.go b/politeiad/backend/tlogbe/plugins/plugins.go index 07493fe6c..260e48b46 100644 --- a/politeiad/backend/tlogbe/plugins/plugins.go +++ b/politeiad/backend/tlogbe/plugins/plugins.go @@ -4,48 +4,7 @@ package plugins -import ( - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/store" -) - -const ( - // PluginSettingDataDir is the PluginSetting key for the plugin - // data directory. - PluginSettingDataDir = "datadir" -) - -// TlogClient provides an API for the plugins to interact with the tlog backend -// instances. Plugins are allowed to save, delete, and get plugin data to/from -// the tlog backend. Editing plugin data is not allowed. -type TlogClient interface { - // BlobSave saves a BlobEntry to the tlog backend. The BlobEntry - // will be encrypted prior to being written to disk if the tlog - // instance has an encryption key set. The merkle leaf hash for the - // blob will be returned. This merkle leaf hash can be though of as - // the blob ID and can be used to retrieve or delete the blob. - BlobSave(treeID int64, keyPrefix string, be store.BlobEntry) ([]byte, error) - - // BlobsDel deletes the blobs that correspond to the provided - // merkle leaf hashes. - BlobsDel(treeID int64, merkles [][]byte) error - - // BlobsByMerkle returns the blobs with the provided merkle leaf - // hashes. If a blob does not exist it will not be included in the - // returned map. - BlobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, error) - - // BlobsByKeyPrefix returns all blobs that match the key prefix. - BlobsByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) - - // MerklesByKeyPrefix returns the merkle leaf hashes for all blobs - // that match the key prefix. - MerklesByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) - - // Timestamp returns the timestamp for the blob that correpsonds - // to the merkle leaf hash. - Timestamp(treeID int64, merkle []byte) (*backend.Timestamp, error) -} +import "github.com/decred/politeia/politeiad/backend" // HookT represents the types of plugin hooks. type HookT int @@ -126,9 +85,9 @@ type HookSetRecordStatus struct { MDOverwrite []backend.MetadataStream `json:"mdoverwrite"` } -// PluginClient provides an API for the tlog backend to use when interacting -// with plugins. All tlogbe plugins must implement the PluginClient interface. -type PluginClient interface { +// Client provides an API for a tlog instance to use when interacting with a +// plugin. All tlog plugins must implement the Client interface. +type Client interface { // Setup performs any required plugin setup. Setup() error diff --git a/politeiad/backend/tlogbe/testing.go b/politeiad/backend/tlogbe/testing.go index df59e57ec..4d189db8b 100644 --- a/politeiad/backend/tlogbe/testing.go +++ b/politeiad/backend/tlogbe/testing.go @@ -15,24 +15,24 @@ import ( "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" ) -// newTestTlogBackend returns a tlog backend for testing. It wraps -// tlog and trillian client, providing the framework needed for -// writing tlog backend tests. -func newTestTlogBackend(t *testing.T) (*tlogBackend, func()) { +// NewTestTlogBackend returns a tlogBackend that is setup for testing and a +// closure that cleans up all test data when invoked. +func NewTestTlogBackend(t *testing.T) (*tlogBackend, func()) { t.Helper() - testDir, err := ioutil.TempDir("", "tlog.backend.test") + // Setup home dir and data dir + homeDir, err := ioutil.TempDir("", "tlogbackend.test") if err != nil { t.Fatal(err) } - testDataDir := filepath.Join(testDir, "data") + dataDir := filepath.Join(homeDir, "data") tlogBackend := tlogBackend{ activeNetParams: chaincfg.TestNet3Params(), - homeDir: testDir, - dataDir: testDataDir, - unvetted: tlog.NewTestTlog(t, testDir, "unvetted"), - vetted: tlog.NewTestTlog(t, testDir, "vetted"), + homeDir: homeDir, + dataDir: dataDir, + unvetted: tlog.NewTestTlogUnencrypted(t, dataDir, "unvetted"), + vetted: tlog.NewTestTlogEncrypted(t, dataDir, "vetted"), prefixes: make(map[string][]byte), vettedTreeIDs: make(map[string]int64), inv: recordInventory{ @@ -41,15 +41,10 @@ func newTestTlogBackend(t *testing.T) (*tlogBackend, func()) { }, } - err = tlogBackend.setup() - if err != nil { - t.Fatalf("setup: %v", err) - } - return &tlogBackend, func() { - err = os.RemoveAll(testDir) + err = os.RemoveAll(homeDir) if err != nil { - t.Fatalf("RemoveAll: %v", err) + t.Fatal(err) } } } diff --git a/politeiad/backend/tlogbe/tlog/plugin.go b/politeiad/backend/tlogbe/tlog/plugin.go index 057baa172..0f649fd3d 100644 --- a/politeiad/backend/tlogbe/tlog/plugin.go +++ b/politeiad/backend/tlogbe/tlog/plugin.go @@ -28,7 +28,7 @@ const ( type plugin struct { id string settings []backend.PluginSetting - client plugins.PluginClient + client plugins.Client } func (t *Tlog) plugin(pluginID string) (plugin, bool) { @@ -55,14 +55,14 @@ func (t *Tlog) PluginRegister(b backend.Backend, p backend.Plugin) error { log.Tracef("%v PluginRegister: %v", t.id, p.ID) var ( - client plugins.PluginClient + client plugins.Client err error dataDir = filepath.Join(t.dataDir, pluginDataDirname) ) switch p.ID { case cmplugin.ID: - client, err = comments.New(b, t, p.Settings, p.Identity, dataDir) + client, err = comments.New(t, p.Settings, p.Identity, dataDir) if err != nil { return err } diff --git a/politeiad/backend/tlogbe/tlog/testing.go b/politeiad/backend/tlogbe/tlog/testing.go index af2d815d7..b1a274c2c 100644 --- a/politeiad/backend/tlogbe/tlog/testing.go +++ b/politeiad/backend/tlogbe/tlog/testing.go @@ -5,35 +5,54 @@ package tlog import ( - "io/ioutil" + "os" + "path/filepath" "testing" "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" "github.com/marcopeereboom/sbox" ) -// NewTestTlog returns a tlog used for testing. -func NewTestTlog(t *testing.T, dir, id string) *Tlog { +func newTestTlog(t *testing.T, tlogID, dataDir string, encrypt bool) *Tlog { t.Helper() - dir, err := ioutil.TempDir(dir, id) + // Setup datadir for this tlog instance + dataDir = filepath.Join(dataDir, tlogID) + err := os.MkdirAll(dataDir, 0700) if err != nil { t.Fatal(err) } - key, err := sbox.NewKey() + + // Setup key-value store + fp := filepath.Join(dataDir, defaultStoreDirname) + err = os.MkdirAll(fp, 0700) if err != nil { t.Fatal(err) } - tclient, err := newTestTClient() - if err != nil { - t.Fatal(err) + store := filesystem.New(fp) + + // Setup encryptin key if specified + var ek *encryptionKey + if encrypt { + key, err := sbox.NewKey() + if err != nil { + t.Fatal(err) + } + ek = newEncryptionKey(key) } return &Tlog{ - id: id, - dcrtime: nil, - encryptionKey: newEncryptionKey(key), - trillian: tclient, - store: filesystem.New(dir), + id: tlogID, + encryptionKey: ek, + trillian: newTestTClient(t), + store: store, } } + +func NewTestTlogEncrypted(t *testing.T, tlogID, dataDir string) *Tlog { + return newTestTlog(t, tlogID, dataDir, true) +} + +func NewTestTlogUnencrypted(t *testing.T, tlogID, dataDir string) *Tlog { + return newTestTlog(t, tlogID, dataDir, false) +} diff --git a/politeiad/backend/tlogbe/tlog/trillianclient.go b/politeiad/backend/tlogbe/tlog/trillianclient.go index 8033318de..5462c2521 100644 --- a/politeiad/backend/tlogbe/tlog/trillianclient.go +++ b/politeiad/backend/tlogbe/tlog/trillianclient.go @@ -13,6 +13,7 @@ import ( "io/ioutil" "math/rand" "sync" + "testing" "time" "github.com/decred/politeia/util" @@ -570,7 +571,8 @@ type testTClient struct { trees map[int64]*trillian.Tree // [treeID]Tree leaves map[int64][]*trillian.LogLeaf // [treeID][]LogLeaf - privateKey *keyspb.PrivateKey + privateKey *keyspb.PrivateKey // Trillian signing key + publicKey crypto.PublicKey // Trillian public key } // treeNew ceates a new trillian tree in memory. @@ -723,23 +725,25 @@ func (t *testTClient) inclusionProof(treeID int64, merkleLeafHash []byte, lr *ty func (t *testTClient) close() {} // newTestTClient returns a new testTClient. -func newTestTClient() (*testTClient, error) { - // Create trillian private key - key, err := keys.NewFromSpec(&keyspb.Specification{ - Params: &keyspb.Specification_EcdsaParams{}, - }) +func newTestTClient(t *testing.T) *testTClient { + key, err := newTrillianKey() if err != nil { - return nil, err + t.Fatal(err) } - keyDer, err := der.MarshalPrivateKey(key) + b, err := der.MarshalPrivateKey(key) if err != nil { - return nil, err + t.Fatal(err) + } + signer, err := der.UnmarshalPrivateKey(b) + if err != nil { + t.Fatal(err) } return &testTClient{ trees: make(map[int64]*trillian.Tree), leaves: make(map[int64][]*trillian.LogLeaf), privateKey: &keyspb.PrivateKey{ - Der: keyDer, + Der: b, }, - }, nil + publicKey: signer.Public(), + } } diff --git a/politeiad/backend/tlogbe/tlogbe_test.go b/politeiad/backend/tlogbe/tlogbe_test.go index 123c81d61..8b2669fd2 100644 --- a/politeiad/backend/tlogbe/tlogbe_test.go +++ b/politeiad/backend/tlogbe/tlogbe_test.go @@ -355,7 +355,7 @@ func setupRecordContentTests(t *testing.T) []recordContentTest { } func TestNewRecord(t *testing.T) { - tlogBackend, cleanup := newTestTlogBackend(t) + tlogBackend, cleanup := NewTestTlogBackend(t) defer cleanup() // Test all record content verification error through the New endpoint @@ -391,7 +391,7 @@ func TestNewRecord(t *testing.T) { } func TestUpdateUnvettedRecord(t *testing.T) { - tlogBackend, cleanup := newTestTlogBackend(t) + tlogBackend, cleanup := NewTestTlogBackend(t) defer cleanup() // Create new record @@ -565,7 +565,7 @@ func TestUpdateUnvettedRecord(t *testing.T) { } func TestUpdateVettedRecord(t *testing.T) { - tlogBackend, cleanup := newTestTlogBackend(t) + tlogBackend, cleanup := NewTestTlogBackend(t) defer cleanup() // Create new record @@ -752,7 +752,7 @@ func TestUpdateVettedRecord(t *testing.T) { } func TestUpdateUnvettedMetadata(t *testing.T) { - tlogBackend, cleanup := newTestTlogBackend(t) + tlogBackend, cleanup := NewTestTlogBackend(t) defer cleanup() // Create new record @@ -930,7 +930,7 @@ func TestUpdateUnvettedMetadata(t *testing.T) { } func TestUpdateVettedMetadata(t *testing.T) { - tlogBackend, cleanup := newTestTlogBackend(t) + tlogBackend, cleanup := NewTestTlogBackend(t) defer cleanup() // Create new record @@ -1114,7 +1114,7 @@ func TestUpdateVettedMetadata(t *testing.T) { } func TestUnvettedExists(t *testing.T) { - tlogBackend, cleanup := newTestTlogBackend(t) + tlogBackend, cleanup := NewTestTlogBackend(t) defer cleanup() // Create new record @@ -1151,7 +1151,7 @@ func TestUnvettedExists(t *testing.T) { } func TestVettedExists(t *testing.T) { - tlogBackend, cleanup := newTestTlogBackend(t) + tlogBackend, cleanup := NewTestTlogBackend(t) defer cleanup() // Create unvetted record @@ -1200,7 +1200,7 @@ func TestVettedExists(t *testing.T) { } func TestGetUnvetted(t *testing.T) { - tlogBackend, cleanup := newTestTlogBackend(t) + tlogBackend, cleanup := NewTestTlogBackend(t) defer cleanup() // Create new record @@ -1242,7 +1242,7 @@ func TestGetUnvetted(t *testing.T) { } func TestGetVetted(t *testing.T) { - tlogBackend, cleanup := newTestTlogBackend(t) + tlogBackend, cleanup := NewTestTlogBackend(t) defer cleanup() // Create new record @@ -1289,7 +1289,7 @@ func TestGetVetted(t *testing.T) { } func TestSetUnvettedStatus(t *testing.T) { - tlogBackend, cleanup := newTestTlogBackend(t) + tlogBackend, cleanup := NewTestTlogBackend(t) defer cleanup() // Helpers @@ -1462,7 +1462,7 @@ func TestSetUnvettedStatus(t *testing.T) { } func TestSetVettedStatus(t *testing.T) { - tlogBackend, cleanup := newTestTlogBackend(t) + tlogBackend, cleanup := NewTestTlogBackend(t) defer cleanup() // Helpers diff --git a/politeiad/backend/tlogbe/tlogclient/testclient/testclient.go b/politeiad/backend/tlogbe/tlogclient/testclient/testclient.go new file mode 100644 index 000000000..b60cbb2b4 --- /dev/null +++ b/politeiad/backend/tlogbe/tlogclient/testclient/testclient.go @@ -0,0 +1,55 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package testclient + +import ( + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/store" +) + +// testClient implements the tlogclient.Client interface and is used for +// plugin testing. +type testClient struct{} + +// BlobSave saves a BlobEntry to the tlog backend. The merkle leaf hash for the +// blob will be returned. This merkle leaf hash can be though of as the blob ID +// and can be used to retrieve or delete the blob. +func (t *testClient) BlobSave(treeID int64, keyPrefix string, be store.BlobEntry) ([]byte, error) { + return nil, nil +} + +// BlobsDel deletes the blobs that correspond to the provided merkle leaf +// hashes. +func (t *testClient) BlobsDel(treeID int64, merkles [][]byte) error { + return nil +} + +// BlobsByMerkle returns the blobs with the provided merkle leaf hashes. If a +// blob does not exist it will not be included in the returned map. +func (t *testClient) BlobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, error) { + return nil, nil +} + +// BlobsByKeyPrefix returns all blobs that match the key prefix. +func (t *testClient) BlobsByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) { + return nil, nil +} + +// MerklesByKeyPrefix returns the merkle leaf hashes for all blobs that match +// the key prefix. +func (t *testClient) MerklesByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) { + return nil, nil +} + +// Timestamp returns the timestamp for the blob that correpsonds to the merkle +// leaf hash. +func (t *testClient) Timestamp(treeID int64, merkle []byte) (*backend.Timestamp, error) { + return nil, nil +} + +// New returns a new testClient. +func New() *testClient { + return &testClient{} +} diff --git a/politeiad/backend/tlogbe/tlogclient/tlogclient.go b/politeiad/backend/tlogbe/tlogclient/tlogclient.go new file mode 100644 index 000000000..14733f777 --- /dev/null +++ b/politeiad/backend/tlogbe/tlogclient/tlogclient.go @@ -0,0 +1,42 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlogclient + +import ( + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/store" +) + +// Client provides an API for the plugins to interact with a tlog instance. +// Plugins are allowed to save, delete, and get plugin data to/from the tlog +// backend. Editing plugin data is not allowed. +type Client interface { + // BlobSave saves a BlobEntry to the tlog backend. The BlobEntry + // will be encrypted prior to being written to disk if the tlog + // instance has an encryption key set. The merkle leaf hash for the + // blob will be returned. This merkle leaf hash can be though of as + // the blob ID and can be used to retrieve or delete the blob. + BlobSave(treeID int64, keyPrefix string, be store.BlobEntry) ([]byte, error) + + // BlobsDel deletes the blobs that correspond to the provided + // merkle leaf hashes. + BlobsDel(treeID int64, merkles [][]byte) error + + // BlobsByMerkle returns the blobs with the provided merkle leaf + // hashes. If a blob does not exist it will not be included in the + // returned map. + BlobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, error) + + // BlobsByKeyPrefix returns all blobs that match the key prefix. + BlobsByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) + + // MerklesByKeyPrefix returns the merkle leaf hashes for all blobs + // that match the key prefix. + MerklesByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) + + // Timestamp returns the timestamp for the blob that correpsonds + // to the merkle leaf hash. + Timestamp(treeID int64, merkle []byte) (*backend.Timestamp, error) +} diff --git a/politeiad/plugins/comments/comments.go b/politeiad/plugins/comments/comments.go index 834e31bc3..df52a3458 100644 --- a/politeiad/plugins/comments/comments.go +++ b/politeiad/plugins/comments/comments.go @@ -74,7 +74,7 @@ type Comment struct { // CommentAdd is the structure that is saved to disk when a comment is created // or edited. // -// Signature is the client signature of State+Token+ParentID+Comment. +// Signature is the client signature of Token+ParentID+Comment. type CommentAdd struct { // Data generated by client UserID string `json:"userid"` // Unique user ID From 726e6825b0a9e869a33f091e7495232fc8f7d734 Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 15 Jan 2021 17:22:54 -0600 Subject: [PATCH 224/449] Update pi plugin. --- .../tlogbe/plugins/comments/comments.go | 26 +- .../tlogbe/plugins/comments/comments_test.go | 4 +- .../backend/tlogbe/plugins/dcrdata/dcrdata.go | 10 +- .../backend/tlogbe/plugins/pi/linkedfrom.go | 157 +++ politeiad/backend/tlogbe/plugins/pi/log.go | 25 + politeiad/backend/tlogbe/plugins/pi/pi.go | 1188 +++++++---------- .../backend/tlogbe/plugins/pi/pi_test.go | 14 +- politeiad/backend/tlogbe/plugins/pi/user.go | 106 ++ politeiad/backend/tlogbe/tlog/plugin.go | 13 +- politeiad/backend/tlogbe/tlogbe.go | 68 +- politeiad/plugins/pi/pi.go | 390 +----- util/token.go | 4 +- 12 files changed, 818 insertions(+), 1187 deletions(-) create mode 100644 politeiad/backend/tlogbe/plugins/pi/linkedfrom.go create mode 100644 politeiad/backend/tlogbe/plugins/pi/log.go create mode 100644 politeiad/backend/tlogbe/plugins/pi/user.go diff --git a/politeiad/backend/tlogbe/plugins/comments/comments.go b/politeiad/backend/tlogbe/plugins/comments/comments.go index bb58c4629..43635e77f 100644 --- a/politeiad/backend/tlogbe/plugins/comments/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments/comments.go @@ -56,7 +56,7 @@ var ( // commentsPlugin is the tlog backend implementation of the comments plugin. // -// commentsPlugin satisfies the PluginClient interface. +// commentsPlugin satisfies the plugins.Client interface. type commentsPlugin struct { sync.Mutex tlog tlogclient.Client @@ -1524,9 +1524,18 @@ func (p *commentsPlugin) cmdTimestamps(treeID int64, token []byte, payload strin return string(reply), nil } +// Setup performs any plugin setup work that needs to be done. +// +// This function satisfies the plugins.Client interface. +func (p *commentsPlugin) Setup() error { + log.Tracef("Setup") + + return nil +} + // Cmd executes a plugin command. // -// This function satisfies the PluginClient interface. +// This function satisfies the plugins.Client interface. func (p *commentsPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { log.Tracef("Cmd: %v %v %x %v", treeID, token, cmd, payload) @@ -1558,7 +1567,7 @@ func (p *commentsPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (s // Hook executes a plugin hook. // -// This function satisfies the PluginClient interface. +// This function satisfies the plugins.Client interface. func (p *commentsPlugin) Hook(h plugins.HookT, payload string) error { log.Tracef("Hook: %v", plugins.Hooks[h]) @@ -1567,7 +1576,7 @@ func (p *commentsPlugin) Hook(h plugins.HookT, payload string) error { // Fsck performs a plugin filesystem check. // -// This function satisfies the PluginClient interface. +// This function satisfies the plugins.Client interface. func (p *commentsPlugin) Fsck() error { log.Tracef("Fsck") @@ -1576,15 +1585,6 @@ func (p *commentsPlugin) Fsck() error { return nil } -// Setup performs any plugin setup work that needs to be done. -// -// This function satisfies the PluginClient interface. -func (p *commentsPlugin) Setup() error { - log.Tracef("Setup") - - return nil -} - // New returns a new comments plugin. func New(tlog tlogclient.Client, settings []backend.PluginSetting, id *identity.FullIdentity, dataDir string) (*commentsPlugin, error) { // Setup comments plugin data dir diff --git a/politeiad/backend/tlogbe/plugins/comments/comments_test.go b/politeiad/backend/tlogbe/plugins/comments/comments_test.go index bd6a5d355..a7e4efd0d 100644 --- a/politeiad/backend/tlogbe/plugins/comments/comments_test.go +++ b/politeiad/backend/tlogbe/plugins/comments/comments_test.go @@ -13,7 +13,6 @@ import ( "strconv" "testing" - "github.com/davecgh/go-spew/spew" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/tlogclient/testclient" @@ -146,8 +145,7 @@ func TestCmdNew(t *testing.T) { return default: // Unexpected error - t.Errorf("got error %v, want error %v", - spew.Sdump(err), spew.Sdump(tc.wantErr)) + t.Errorf("got error %v, want error %v", err, tc.wantErr) } } diff --git a/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go b/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go index 99509dc4c..c4b5051e8 100644 --- a/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go +++ b/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go @@ -48,7 +48,7 @@ var ( _ plugins.Client = (*dcrdataPlugin)(nil) ) -// dcrdataplugin satisfies the plugins.PluginClient interface. +// dcrdataplugin satisfies the plugins.Client interface. type dcrdataPlugin struct { sync.Mutex activeNetParams *chaincfg.Params @@ -576,7 +576,7 @@ func (p *dcrdataPlugin) websocketSetup() { // Setup performs any plugin setup work that needs to be done. // -// This function satisfies the plugins.PluginClient interface. +// This function satisfies the plugins.Client interface. func (p *dcrdataPlugin) Setup() error { log.Tracef("setup") @@ -591,7 +591,7 @@ func (p *dcrdataPlugin) Setup() error { // Cmd executes a plugin command. // -// This function satisfies the plugins.PluginClient interface. +// This function satisfies the plugins.Client interface. func (p *dcrdataPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { log.Tracef("Cmd: %v %v", cmd, payload) @@ -611,7 +611,7 @@ func (p *dcrdataPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (st // Hook executes a plugin hook. // -// This function satisfies the plugins.PluginClient interface. +// This function satisfies the plugins.Client interface. func (p *dcrdataPlugin) Hook(h plugins.HookT, payload string) error { log.Tracef("Hook: %v", plugins.Hooks[h]) @@ -620,7 +620,7 @@ func (p *dcrdataPlugin) Hook(h plugins.HookT, payload string) error { // Fsck performs a plugin filesystem check. // -// This function satisfies the plugins.PluginClient interface. +// This function satisfies the plugins.Client interface. func (p *dcrdataPlugin) Fsck() error { log.Tracef("Fsck") diff --git a/politeiad/backend/tlogbe/plugins/pi/linkedfrom.go b/politeiad/backend/tlogbe/plugins/pi/linkedfrom.go new file mode 100644 index 000000000..c1ad74677 --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/pi/linkedfrom.go @@ -0,0 +1,157 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package pi + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/decred/politeia/util" +) + +const ( + // fnLinkedFrom is the filename for the cached linkedFrom data that + // is saved to the pi plugin data dir. + fnLinkedFrom = "{tokenprefix}-linkedfrom.json" +) + +// linkedFrom is the the structure that is updated and cached for proposal A +// when proposal B links to proposal A. Proposals can link to one another using +// the ProposalMetadata LinkTo field. The linkedFrom list contains all +// proposals that have linked to proposal A. The list will only contain public +// proposals. The linkedFrom list is saved to disk in the pi plugin data dir, +// specifying the parent proposal token in the filename. +// +// Example: the linked from list for an RFP proposal will contain all public +// RFP submissions. The cached list can be found in the pi plugin data dir +// at the path specified by linkedFromPath(). +type linkedFrom struct { + Tokens map[string]struct{} `json:"tokens"` +} + +// linkedFromPath returns the path to the linkedFrom list for the provided +// proposal token. The token prefix is used in the file path so that the linked +// from list can be retrieved using either the full token or the token prefix. +func (p *piPlugin) linkedFromPath(token []byte) string { + t := util.TokenPrefix(token) + fn := strings.Replace(fnLinkedFrom, "{tokenprefix}", t, 1) + return filepath.Join(p.dataDir, fn) +} + +// linkedFromWithLock return the linkedFrom list for the provided proposal +// token. +// +// This function must be called WITH the lock held. +func (p *piPlugin) linkedFromWithLock(token []byte) (*linkedFrom, error) { + fp := p.linkedFromPath(token) + b, err := ioutil.ReadFile(fp) + if err != nil { + var e *os.PathError + if errors.As(err, &e) && !os.IsExist(err) { + // File does't exist. Return an empty linked from list. + return &linkedFrom{ + Tokens: make(map[string]struct{}), + }, nil + } + } + + var lf linkedFrom + err = json.Unmarshal(b, &lf) + if err != nil { + return nil, err + } + + return &lf, nil +} + +// linkedFrom return the linkedFrom list for the provided proposal token. +// +// This function must be called WITHOUT the lock held. +func (p *piPlugin) linkedFrom(token []byte) (*linkedFrom, error) { + p.Lock() + defer p.Unlock() + + return p.linkedFromWithLock(token) +} + +// linkedFromSaveWithLock saves the provided linkedFrom list to the pi plugin +// data dir. +// +// This function must be called WITH the lock held. +func (p *piPlugin) linkedFromSaveWithLock(token []byte, lf linkedFrom) error { + b, err := json.Marshal(lf) + if err != nil { + return err + } + fp := p.linkedFromPath(token) + return ioutil.WriteFile(fp, b, 0664) +} + +// linkedFromAdd updates the cached linkedFrom list for the parentToken, adding +// the childToken to the list. +// +// This function must be called WITHOUT the lock held. +func (p *piPlugin) linkedFromAdd(parentToken, childToken string) error { + p.Lock() + defer p.Unlock() + + // Verify tokens + parent, err := util.TokenDecode(util.TokenTypeTlog, parentToken) + if err != nil { + return err + } + _, err = util.TokenDecode(util.TokenTypeTlog, childToken) + if err != nil { + return err + } + + // Get existing linked from list + lf, err := p.linkedFromWithLock(parent) + if err != nil { + return fmt.Errorf("linkedFromWithLock %x: %v", parent, err) + } + + // Update list + lf.Tokens[childToken] = struct{}{} + + // Save list + return p.linkedFromSaveWithLock(parent, *lf) +} + +// linkedFromDel updates the cached linkedFrom list for the parentToken, +// deleting the childToken from the list. +// +// This function must be called WITHOUT the lock held. +func (p *piPlugin) linkedFromDel(parentToken, childToken string) error { + p.Lock() + defer p.Unlock() + + // Verify tokens + parent, err := util.TokenDecode(util.TokenTypeTlog, parentToken) + if err != nil { + return err + } + _, err = util.TokenDecode(util.TokenTypeTlog, childToken) + if err != nil { + return err + } + + // Get existing linked from list + lf, err := p.linkedFromWithLock(parent) + if err != nil { + return fmt.Errorf("linkedFromWithLock %x: %v", parent, err) + } + + // Update list + delete(lf.Tokens, childToken) + + // Save list + return p.linkedFromSaveWithLock(parent, *lf) +} diff --git a/politeiad/backend/tlogbe/plugins/pi/log.go b/politeiad/backend/tlogbe/plugins/pi/log.go new file mode 100644 index 000000000..48e6bee30 --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/pi/log.go @@ -0,0 +1,25 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package pi + +import "github.com/decred/slog" + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log = slog.Disabled + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = slog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using slog. +func UseLogger(logger slog.Logger) { + log = logger +} diff --git a/politeiad/backend/tlogbe/plugins/pi/pi.go b/politeiad/backend/tlogbe/plugins/pi/pi.go index 8a1496770..df77bc270 100644 --- a/politeiad/backend/tlogbe/plugins/pi/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi/pi.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -10,7 +10,7 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "os" "path/filepath" "strings" @@ -19,26 +19,22 @@ import ( "github.com/decred/dcrd/chaincfg/v3" "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" + "github.com/decred/politeia/politeiad/backend/tlogbe/tlogclient" "github.com/decred/politeia/politeiad/plugins/comments" "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/ticketvote" ) -const ( - // Filenames of cached data saved to the pi plugin data dir. - fnLinkedFrom = "{token}-linkedfrom.json" - fnUserData = "{userid}.json" -) - var ( - _ pluginClient = (*piPlugin)(nil) + _ plugins.Client = (*piPlugin)(nil) ) -// piPlugin satisfies the pluginClient interface. +// piPlugin satisfies the plugins.Client interface. type piPlugin struct { sync.Mutex backend backend.Backend - tlog tlogClient + tlog tlogclient.Client activeNetParams *chaincfg.Params // dataDir is the pi plugin data directory. The only data that is @@ -47,204 +43,6 @@ type piPlugin struct { dataDir string } -// linkedFrom is the the structure that is updated and cached for proposal A -// when proposal B links to proposal A. Proposals can link to one another using -// the ProposalMetadata LinkTo field. The linkedFrom list contains all -// proposals that have linked to proposal A. The list will only contain public -// proposals. The linkedFrom list is saved to disk in the pi plugin data dir, -// specifying the parent proposal token in the filename. -// -// Example: the linked from list for an RFP proposal will contain all public -// RFP submissions. The cached list can be found in the pi plugin data dir -// at the path specified by linkedFromPath(). -type linkedFrom struct { - Tokens map[string]struct{} `json:"tokens"` -} - -// linkedFromPath returns the path to the linkedFrom list for the provided -// proposal token. -func (p *piPlugin) linkedFromPath(token string) string { - fn := strings.Replace(fnLinkedFrom, "{token}", token, 1) - return filepath.Join(p.dataDir, fn) -} - -// linkedFromWithLock return the linkedFrom list for the provided proposal -// token. -// -// This function must be called WITH the lock held. -func (p *piPlugin) linkedFromWithLock(token string) (*linkedFrom, error) { - fp := p.linkedFromPath(token) - b, err := ioutil.ReadFile(fp) - if err != nil { - var e *os.PathError - if errors.As(err, &e) && !os.IsExist(err) { - // File does't exist. Return an empty linked from list. - return &linkedFrom{ - Tokens: make(map[string]struct{}), - }, nil - } - } - - var lf linkedFrom - err = json.Unmarshal(b, &lf) - if err != nil { - return nil, err - } - - return &lf, nil -} - -// linkedFrom return the linkedFrom list for the provided proposal token. -// -// This function must be called WITHOUT the lock held. -func (p *piPlugin) linkedFrom(token string) (*linkedFrom, error) { - p.Lock() - defer p.Unlock() - - return p.linkedFromWithLock(token) -} - -// linkedFromSaveWithLock saves the provided linkedFrom list to the pi plugin -// data dir. -// -// This function must be called WITH the lock held. -func (p *piPlugin) linkedFromSaveWithLock(token string, lf linkedFrom) error { - b, err := json.Marshal(lf) - if err != nil { - return err - } - fp := p.linkedFromPath(token) - return ioutil.WriteFile(fp, b, 0664) -} - -// linkedFromAdd updates the cached linkedFrom list for the parentToken, adding -// the childToken to the list. -// -// This function must be called WITHOUT the lock held. -func (p *piPlugin) linkedFromAdd(parentToken, childToken string) error { - p.Lock() - defer p.Unlock() - - // Get existing linked from list - lf, err := p.linkedFromWithLock(parentToken) - if errors.Is(err, errRecordNotFound) { - return fmt.Errorf("linkedFromWithLock %v: %v", parentToken, err) - } - - // Update list - lf.Tokens[childToken] = struct{}{} - - // Save list - return p.linkedFromSaveWithLock(parentToken, *lf) -} - -// linkedFromDel updates the cached linkedFrom list for the parentToken, -// deleting the childToken from the list. -// -// This function must be called WITHOUT the lock held. -func (p *piPlugin) linkedFromDel(parentToken, childToken string) error { - p.Lock() - defer p.Unlock() - - // Get existing linked from list - lf, err := p.linkedFromWithLock(parentToken) - if err != nil { - return fmt.Errorf("linkedFromWithLock %v: %v", parentToken, err) - } - - // Update list - delete(lf.Tokens, childToken) - - // Save list - return p.linkedFromSaveWithLock(parentToken, *lf) -} - -// userData contains cached pi plugin data for a specific user. The userData -// JSON is saved to disk in the pi plugin data dir. The user ID is included in -// the filename. -type userData struct { - // Tokens contains a list of all the proposals that have been - // submitted by this user. This data is cached so that the - // ProposalInv command can filter proposals by user ID. - Tokens []string `json:"tokens"` -} - -// userDataPath returns the filepath to the cached userData struct for the -// specified user. -func (p *piPlugin) userDataPath(userID string) string { - fn := strings.Replace(fnUserData, "{userid}", userID, 1) - return filepath.Join(p.dataDir, fn) -} - -// userDataWithLock returns the cached userData struct for the specified user. -// -// This function must be called WITH the lock held. -func (p *piPlugin) userDataWithLock(userID string) (*userData, error) { - fp := p.userDataPath(userID) - b, err := ioutil.ReadFile(fp) - if err != nil { - var e *os.PathError - if errors.As(err, &e) && !os.IsExist(err) { - // File does't exist. Return an empty userData. - return &userData{ - Tokens: []string{}, - }, nil - } - } - - var ud userData - err = json.Unmarshal(b, &ud) - if err != nil { - return nil, err - } - - return &ud, nil -} - -// userData returns the cached userData struct for the specified user. -// -// This function must be called WITHOUT the lock held. -func (p *piPlugin) userData(userID string) (*userData, error) { - p.Lock() - defer p.Unlock() - - return p.userDataWithLock(userID) -} - -// userDataSaveWithLock saves the provided userData to the pi plugin data dir. -// -// This function must be called WITH the lock held. -func (p *piPlugin) userDataSaveWithLock(userID string, ud userData) error { - b, err := json.Marshal(ud) - if err != nil { - return err - } - - fp := p.userDataPath(userID) - return ioutil.WriteFile(fp, b, 0664) -} - -// userDataAddToken adds the provided token to the cached userData for the -// provided user. -// -// This function must be called WITHOUT the lock held. -func (p *piPlugin) userDataAddToken(userID string, token string) error { - p.Lock() - defer p.Unlock() - - // Get current user data - ud, err := p.userDataWithLock(userID) - if err != nil { - return err - } - - // Add token - ud.Tokens = append(ud.Tokens, token) - - // Save changes - return p.userDataSaveWithLock(userID, *ud) -} - // isRFP returns whether the provided proposal metadata belongs to an RFP // proposal. func isRFP(pm pi.ProposalMetadata) bool { @@ -254,39 +52,60 @@ func isRFP(pm pi.ProposalMetadata) bool { // decodeProposalMetadata decodes and returns the ProposalMetadata from the // provided backend files. If a ProposalMetadata is not found, nil is returned. func decodeProposalMetadata(files []backend.File) (*pi.ProposalMetadata, error) { - var pm *pi.ProposalMetadata + var propMD *pi.ProposalMetadata for _, v := range files { if v.Name == pi.FileNameProposalMetadata { b, err := base64.StdEncoding.DecodeString(v.Payload) if err != nil { return nil, err } - pm, err = pi.DecodeProposalMetadata(b) + var pm pi.ProposalMetadata + err = json.Unmarshal(b, &pm) if err != nil { return nil, err } + propMD = &pm break } } - return pm, nil + return propMD, nil } // decodeGeneralMetadata decodes and returns the GeneralMetadata from the // provided backend metadata streams. If a GeneralMetadata is not found, nil is // returned. func decodeGeneralMetadata(metadata []backend.MetadataStream) (*pi.GeneralMetadata, error) { - var gm *pi.GeneralMetadata - var err error + var generalMD *pi.GeneralMetadata for _, v := range metadata { if v.ID == pi.MDStreamIDGeneralMetadata { - gm, err = pi.DecodeGeneralMetadata([]byte(v.Payload)) + var gm pi.GeneralMetadata + err := json.Unmarshal([]byte(v.Payload), &gm) if err != nil { return nil, err } + generalMD = &gm break } } - return gm, nil + return generalMD, nil +} + +// decodeStatusChanges decodes a JSON byte slice into a []StatusChange slice. +func decodeStatusChanges(payload []byte) ([]pi.StatusChange, error) { + var statuses []pi.StatusChange + d := json.NewDecoder(strings.NewReader(string(payload))) + for { + var sc pi.StatusChange + err := d.Decode(&sc) + if errors.Is(err, io.EOF) { + break + } else if err != nil { + return nil, err + } + statuses = append(statuses, sc) + } + + return statuses, nil } func convertPropStatusFromMDStatus(s backend.MDStatusT) pi.PropStatusT { @@ -305,103 +124,107 @@ func convertPropStatusFromMDStatus(s backend.MDStatusT) pi.PropStatusT { } func (p *piPlugin) cmdProposals(payload string) (string, error) { - ps, err := pi.DecodeProposals([]byte(payload)) - if err != nil { - return "", err - } - - // Verify state - var existsFn func([]byte) bool - switch ps.State { - case pi.PropStateUnvetted: - existsFn = p.backend.UnvettedExists - case pi.PropStateVetted: - existsFn = p.backend.VettedExists - default: - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropStateInvalid), - } - } - - // Setup the returned map with entries for all tokens that - // correspond to records. - // map[token]ProposalPluginData - proposals := make(map[string]pi.ProposalPluginData, len(ps.Tokens)) - for _, v := range ps.Tokens { - token, err := tokenDecodeAnyLength(v) + // TODO + /* + ps, err := pi.DecodeProposals([]byte(payload)) if err != nil { - // Not a valid token - continue + return "", err } - ok := existsFn(token) - if !ok { - // Record doesn't exists - continue + + // Verify state + var existsFn func([]byte) bool + switch ps.State { + case pi.PropStateUnvetted: + existsFn = p.backend.UnvettedExists + case pi.PropStateVetted: + existsFn = p.backend.VettedExists + default: + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropStateInvalid), + } } - // Record exists. Include it in the reply. - proposals[v] = pi.ProposalPluginData{} - } + // Setup the returned map with entries for all tokens that + // correspond to records. + // map[token]ProposalPluginData + proposals := make(map[string]pi.ProposalPluginData, len(ps.Tokens)) + for _, v := range ps.Tokens { + token, err := tokenDecodeAnyLength(v) + if err != nil { + // Not a valid token + continue + } + ok := existsFn(token) + if !ok { + // Record doesn't exists + continue + } - // Get linked from list for each proposal - for k, v := range proposals { - lf, err := p.linkedFrom(k) - if err != nil { - return "", fmt.Errorf("linkedFrom %v: %v", k, err) + // Record exists. Include it in the reply. + proposals[v] = pi.ProposalPluginData{} } - // Convert map to a slice - linkedFrom := make([]string, 0, len(lf.Tokens)) - for token := range lf.Tokens { - linkedFrom = append(linkedFrom, token) - } + // Get linked from list for each proposal + for k, v := range proposals { + lf, err := p.linkedFrom(k) + if err != nil { + return "", fmt.Errorf("linkedFrom %v: %v", k, err) + } - v.LinkedFrom = linkedFrom - proposals[k] = v - } + // Convert map to a slice + linkedFrom := make([]string, 0, len(lf.Tokens)) + for token := range lf.Tokens { + linkedFrom = append(linkedFrom, token) + } - // Get comments count for each proposal - for k, v := range proposals { - // Prepare plugin command - c := comments.Count{ - State: comments.StateT(ps.State), - Token: k, - } - b, err := comments.EncodeCount(c) - if err != nil { - return "", err + v.LinkedFrom = linkedFrom + proposals[k] = v } - // Send plugin command - reply, err := p.backend.Plugin(comments.ID, - comments.CmdCount, "", string(b)) - if err != nil { - return "", fmt.Errorf("backend Plugin %v %v: %v", - comments.ID, comments.CmdCount, err) + // Get comments count for each proposal + for k, v := range proposals { + // Prepare plugin command + c := comments.Count{ + State: comments.StateT(ps.State), + Token: k, + } + b, err := comments.EncodeCount(c) + if err != nil { + return "", err + } + + // Send plugin command + reply, err := p.backend.Plugin(comments.ID, + comments.CmdCount, "", string(b)) + if err != nil { + return "", fmt.Errorf("backend Plugin %v %v: %v", + comments.ID, comments.CmdCount, err) + } + + // Decode reply + cr, err := comments.DecodeCountReply([]byte(reply)) + if err != nil { + return "", err + } + + // Update proposal plugin data + v.Comments = cr.Count + proposals[k] = v } - // Decode reply - cr, err := comments.DecodeCountReply([]byte(reply)) + // Prepare reply + pr := pi.ProposalsReply{ + Proposals: proposals, + } + reply, err := pi.EncodeProposalsReply(pr) if err != nil { return "", err } - // Update proposal plugin data - v.Comments = cr.Count - proposals[k] = v - } - - // Prepare reply - pr := pi.ProposalsReply{ - Proposals: proposals, - } - reply, err := pi.EncodeProposalsReply(pr) - if err != nil { - return "", err - } - - return string(reply), nil + return string(reply), nil + */ + return "", nil } func (p *piPlugin) voteSummary(token string) (*ticketvote.Summary, error) { @@ -430,7 +253,8 @@ func (p *piPlugin) voteSummary(token string) (*ticketvote.Summary, error) { func (p *piPlugin) cmdProposalInv(payload string) (string, error) { // Decode payload - inv, err := pi.DecodeProposalInv([]byte(payload)) + var inv pi.ProposalInv + err := json.Unmarshal([]byte(payload), &inv) if err != nil { return "", err } @@ -518,7 +342,7 @@ func (p *piPlugin) cmdProposalInv(payload string) (string, error) { Unvetted: unvetted, Vetted: vetted, } - reply, err := pi.EncodeProposalInvReply(pir) + reply, err := json.Marshal(pir) if err != nil { return "", err } @@ -581,7 +405,7 @@ func (p *piPlugin) cmdVoteInventory(payload string) (string, error) { Rejected: rejected, BestBlock: ir.BestBlock, } - reply, err := pi.EncodeVoteInventoryReply(vir) + reply, err := json.Marshal(vir) if err != nil { return "", err } @@ -590,436 +414,428 @@ func (p *piPlugin) cmdVoteInventory(payload string) (string, error) { } func (p *piPlugin) commentNew(payload string) (string, error) { - n, err := comments.DecodeNew([]byte(payload)) - if err != nil { - return "", err - } - - // Verifying the state, token, and that the record exists is also - // done in the comments plugin but we duplicate it here so that the - // vote summary request can complete successfully. - - // Verify state - switch n.State { - case comments.StateUnvetted, comments.StateVetted: - // Allowed; continue - default: - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropStateInvalid), + // TODO + /* + n, err := comments.DecodeNew([]byte(payload)) + if err != nil { + return "", err } - } - // Verify token - token, err := tokenDecodeAnyLength(n.Token) - if err != nil { - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropTokenInvalid), - } - } + // Verifying the state, token, and that the record exists is also + // done in the comments plugin but we duplicate it here so that the + // vote summary request can complete successfully. - // Verify record exists - var exists bool - switch n.State { - case comments.StateUnvetted: - exists = p.backend.UnvettedExists(token) - case comments.StateVetted: - exists = p.backend.VettedExists(token) - default: - // Should not happen. State has already been validated. - return "", fmt.Errorf("invalid state %v", n.State) - } - if !exists { - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropNotFound), + // Verify state + switch n.State { + case comments.StateUnvetted, comments.StateVetted: + // Allowed; continue + default: + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropStateInvalid), + } } - } - // Verify vote status - if n.State == comments.StateVetted { - vs, err := p.voteSummary(n.Token) + // Verify token + token, err := tokenDecodeAnyLength(n.Token) if err != nil { - return "", fmt.Errorf("voteSummary: %v", err) + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropTokenInvalid), + } } - switch vs.Status { - case ticketvote.VoteStatusUnauthorized, ticketvote.VoteStatusAuthorized, - ticketvote.VoteStatusStarted: - // Comments are allowed on these vote statuses; continue + + // Verify record exists + var exists bool + switch n.State { + case comments.StateUnvetted: + exists = p.backend.UnvettedExists(token) + case comments.StateVetted: + exists = p.backend.VettedExists(token) default: + // Should not happen. State has already been validated. + return "", fmt.Errorf("invalid state %v", n.State) + } + if !exists { return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusVoteStatusInvalid), - ErrorContext: []string{"vote has ended; proposal is locked"}, + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropNotFound), } } - } - // Send plugin command - return p.backend.Plugin(comments.ID, comments.CmdNew, "", payload) + // Verify vote status + if n.State == comments.StateVetted { + vs, err := p.voteSummary(n.Token) + if err != nil { + return "", fmt.Errorf("voteSummary: %v", err) + } + switch vs.Status { + case ticketvote.VoteStatusUnauthorized, ticketvote.VoteStatusAuthorized, + ticketvote.VoteStatusStarted: + // Comments are allowed on these vote statuses; continue + default: + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusVoteStatusInvalid), + ErrorContext: []string{"vote has ended; proposal is locked"}, + } + } + } + + // Send plugin command + return p.backend.Plugin(comments.ID, comments.CmdNew, "", payload) + */ + return "", nil } func (p *piPlugin) commentDel(payload string) (string, error) { - d, err := comments.DecodeDel([]byte(payload)) - if err != nil { - return "", err - } - - // Verifying the state, token, and that the record exists is also - // done in the comments plugin but we duplicate it here so that the - // vote summary request can complete successfully. - - // Verify state - switch d.State { - case comments.StateUnvetted, comments.StateVetted: - // Allowed; continue - default: - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropStateInvalid), + // TODO + /* + d, err := comments.DecodeDel([]byte(payload)) + if err != nil { + return "", err } - } - // Verify token - token, err := tokenDecodeAnyLength(d.Token) - if err != nil { - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropTokenInvalid), - } - } + // Verifying the state, token, and that the record exists is also + // done in the comments plugin but we duplicate it here so that the + // vote summary request can complete successfully. - // Verify record exists - var exists bool - switch d.State { - case comments.StateUnvetted: - exists = p.backend.UnvettedExists(token) - case comments.StateVetted: - exists = p.backend.VettedExists(token) - default: - // Should not happen. State has already been validated. - return "", fmt.Errorf("invalid state %v", d.State) - } - if !exists { - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropNotFound), + // Verify state + switch d.State { + case comments.StateUnvetted, comments.StateVetted: + // Allowed; continue + default: + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropStateInvalid), + } } - } - // Verify vote status - if d.State == comments.StateVetted { - vs, err := p.voteSummary(d.Token) + // Verify token + token, err := tokenDecodeAnyLength(d.Token) if err != nil { - return "", fmt.Errorf("voteSummary: %v", err) + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropTokenInvalid), + } } - switch vs.Status { - case ticketvote.VoteStatusUnauthorized, ticketvote.VoteStatusAuthorized, - ticketvote.VoteStatusStarted: - // Deleting is allowed on these vote statuses; continue + + // Verify record exists + var exists bool + switch d.State { + case comments.StateUnvetted: + exists = p.backend.UnvettedExists(token) + case comments.StateVetted: + exists = p.backend.VettedExists(token) default: + // Should not happen. State has already been validated. + return "", fmt.Errorf("invalid state %v", d.State) + } + if !exists { return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusVoteStatusInvalid), - ErrorContext: []string{"vote has ended; proposal is locked"}, + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropNotFound), + } + } + + // Verify vote status + if d.State == comments.StateVetted { + vs, err := p.voteSummary(d.Token) + if err != nil { + return "", fmt.Errorf("voteSummary: %v", err) + } + switch vs.Status { + case ticketvote.VoteStatusUnauthorized, ticketvote.VoteStatusAuthorized, + ticketvote.VoteStatusStarted: + // Deleting is allowed on these vote statuses; continue + default: + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusVoteStatusInvalid), + ErrorContext: []string{"vote has ended; proposal is locked"}, + } } } - } - // Send plugin command - return p.backend.Plugin(comments.ID, comments.CmdDel, "", payload) + // Send plugin command + return p.backend.Plugin(comments.ID, comments.CmdDel, "", payload) + */ + return "", nil } func (p *piPlugin) commentVote(payload string) (string, error) { - v, err := comments.DecodeVote([]byte(payload)) - if err != nil { - return "", err - } - - // Verifying the state, token, and that the record exists is also - // done in the comments plugin but we duplicate it here so that the - // vote summary request can complete successfully. - - // Verify state - switch v.State { - case comments.StateUnvetted, comments.StateVetted: - // Allowed; continue - default: - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropStateInvalid), + // TODO + /* + v, err := comments.DecodeVote([]byte(payload)) + if err != nil { + return "", err } - } - // Verify token - token, err := tokenDecodeAnyLength(v.Token) - if err != nil { - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropTokenInvalid), + // Verifying the state, token, and that the record exists is also + // done in the comments plugin but we duplicate it here so that the + // vote summary request can complete successfully. + + // Verify state + switch v.State { + case comments.StateUnvetted, comments.StateVetted: + // Allowed; continue + default: + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropStateInvalid), + } } - } - // Verify record exists - var record *backend.Record - switch v.State { - case comments.StateUnvetted: - record, err = p.backend.GetUnvetted(token, "") - case comments.StateVetted: - record, err = p.backend.GetVetted(token, "") - default: - // Should not happen. State has already been validated. - return "", fmt.Errorf("invalid state %v", v.State) - } - if err != nil { - if errors.Is(err, backend.ErrRecordNotFound) { + // Verify token + token, err := tokenDecodeAnyLength(v.Token) + if err != nil { return "", backend.PluginUserError{ PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropNotFound), + ErrorCode: int(pi.ErrorStatusPropTokenInvalid), } } - return "", fmt.Errorf("get record: %v", err) - } - // Verify record status - status := convertPropStatusFromMDStatus(record.RecordMetadata.Status) - switch status { - case pi.PropStatusPublic: - // Comment votes are only allowed on public proposals; continue - default: - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropStatusInvalid), - ErrorContext: []string{"proposal is not public"}, + // Verify record exists + var record *backend.Record + switch v.State { + case comments.StateUnvetted: + record, err = p.backend.GetUnvetted(token, "") + case comments.StateVetted: + record, err = p.backend.GetVetted(token, "") + default: + // Should not happen. State has already been validated. + return "", fmt.Errorf("invalid state %v", v.State) + } + if err != nil { + if errors.Is(err, backend.ErrRecordNotFound) { + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropNotFound), + } + } + return "", fmt.Errorf("get record: %v", err) } - } - // Verify vote status - vs, err := p.voteSummary(v.Token) - if err != nil { - return "", fmt.Errorf("voteSummary: %v", err) - } - switch vs.Status { - case ticketvote.VoteStatusUnauthorized, ticketvote.VoteStatusAuthorized, - ticketvote.VoteStatusStarted: - // Comment votes are allowed on these vote statuses; continue - default: - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusVoteStatusInvalid), - ErrorContext: []string{"vote has ended; proposal is locked"}, + // Verify record status + status := convertPropStatusFromMDStatus(record.RecordMetadata.Status) + switch status { + case pi.PropStatusPublic: + // Comment votes are only allowed on public proposals; continue + default: + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusPropStatusInvalid), + ErrorContext: []string{"proposal is not public"}, + } } - } - // Send plugin command - return p.backend.Plugin(comments.ID, comments.CmdVote, "", payload) + // Verify vote status + vs, err := p.voteSummary(v.Token) + if err != nil { + return "", fmt.Errorf("voteSummary: %v", err) + } + switch vs.Status { + case ticketvote.VoteStatusUnauthorized, ticketvote.VoteStatusAuthorized, + ticketvote.VoteStatusStarted: + // Comment votes are allowed on these vote statuses; continue + default: + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusVoteStatusInvalid), + ErrorContext: []string{"vote has ended; proposal is locked"}, + } + } + + // Send plugin command + return p.backend.Plugin(comments.ID, comments.CmdVote, "", payload) + */ + return "", nil } func (p *piPlugin) ticketVoteStart(payload string) (string, error) { - // Decode payload - s, err := ticketvote.DecodeStart([]byte(payload)) - if err != nil { - return "", err - } - - // Verify work needs to be done - if len(s.Starts) == 0 { - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusStartDetailsInvalid), - ErrorContext: []string{"no start details found"}, + // TODO + /* + // Decode payload + s, err := ticketvote.DecodeStart([]byte(payload)) + if err != nil { + return "", err } - } - // Sanity check. This pass through command should only be used for - // RFP runoff votes. - if s.Starts[0].Params.Type != ticketvote.VoteTypeRunoff { - return "", fmt.Errorf("not a runoff vote") - } + // Verify work needs to be done + if len(s.Starts) == 0 { + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusStartDetailsInvalid), + ErrorContext: []string{"no start details found"}, + } + } - // Get RFP token. Just use the parent token from the first vote - // params. If the different vote params use different parent - // tokens, the ticketvote plugin will catch it. - rfpToken := s.Starts[0].Params.Parent - rfpTokenb, err := tokenDecode(rfpToken) - if err != nil { - e := fmt.Sprintf("invalid rfp token '%v'", rfpToken) - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusVoteParentInvalid), - ErrorContext: []string{e}, + // Sanity check. This pass through command should only be used for + // RFP runoff votes. + if s.Starts[0].Params.Type != ticketvote.VoteTypeRunoff { + return "", fmt.Errorf("not a runoff vote") } - } - // Get RFP record - rfp, err := p.backend.GetVetted(rfpTokenb, "") - if err != nil { - if errors.Is(err, errRecordNotFound) { - e := fmt.Sprintf("rfp not found %x", rfpToken) + // Get RFP token. Just use the parent token from the first vote + // params. If the different vote params use different parent + // tokens, the ticketvote plugin will catch it. + rfpToken := s.Starts[0].Params.Parent + rfpTokenb, err := tokenDecode(rfpToken) + if err != nil { + e := fmt.Sprintf("invalid rfp token '%v'", rfpToken) return "", backend.PluginUserError{ PluginID: pi.ID, ErrorCode: int(pi.ErrorStatusVoteParentInvalid), ErrorContext: []string{e}, } } - return "", fmt.Errorf("GetVetted %x: %v", rfpToken, err) - } - // Verify RFP linkby has expired. The runoff vote is not allowed to - // start until after the linkby deadline has passed. - rfpPM, err := decodeProposalMetadata(rfp.Files) - if err != nil { - return "", err - } - if rfpPM == nil { - e := fmt.Sprintf("rfp is not a proposal %v", rfpToken) - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusVoteParentInvalid), - ErrorContext: []string{e}, + // Get RFP record + rfp, err := p.backend.GetVetted(rfpTokenb, "") + if err != nil { + if errors.Is(err, errRecordNotFound) { + e := fmt.Sprintf("rfp not found %x", rfpToken) + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusVoteParentInvalid), + ErrorContext: []string{e}, + } + } + return "", fmt.Errorf("GetVetted %x: %v", rfpToken, err) } - } - isExpired := rfpPM.LinkBy < time.Now().Unix() - isMainNet := p.activeNetParams.Name == chaincfg.MainNetParams().Name - switch { - case !isExpired && isMainNet: - e := fmt.Sprintf("rfp %v linkby deadline not met %v", - rfpToken, rfpPM.LinkBy) - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusLinkByNotExpired), - ErrorContext: []string{e}, - } - case !isExpired: - // Allow the vote to be started before the link by deadline - // expires on testnet and simnet only. This makes testing the - // RFP process easier. - log.Warnf("RFP linkby deadline has not been met; disregarding " + - "since this is not mainnet") - } - // Compile a list of the expected RFP submissions that should be in - // the runoff vote. This will be all of the public proposals that - // have linked to the RFP. The RFP's linked from list will include - // abandoned proposals that need to be filtered out. - linkedFrom, err := p.linkedFrom(rfpToken) - if err != nil { - return "", err - } - // map[token]struct{} - expected := make(map[string]struct{}, len(linkedFrom.Tokens)) - for k := range linkedFrom.Tokens { - token, err := tokenDecode(k) + // Verify RFP linkby has expired. The runoff vote is not allowed to + // start until after the linkby deadline has passed. + rfpPM, err := decodeProposalMetadata(rfp.Files) if err != nil { return "", err } - r, err := p.backend.GetVetted(token, "") + if rfpPM == nil { + e := fmt.Sprintf("rfp is not a proposal %v", rfpToken) + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusVoteParentInvalid), + ErrorContext: []string{e}, + } + } + isExpired := rfpPM.LinkBy < time.Now().Unix() + isMainNet := p.activeNetParams.Name == chaincfg.MainNetParams().Name + switch { + case !isExpired && isMainNet: + e := fmt.Sprintf("rfp %v linkby deadline not met %v", + rfpToken, rfpPM.LinkBy) + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusLinkByNotExpired), + ErrorContext: []string{e}, + } + case !isExpired: + // Allow the vote to be started before the link by deadline + // expires on testnet and simnet only. This makes testing the + // RFP process easier. + log.Warnf("RFP linkby deadline has not been met; disregarding " + + "since this is not mainnet") + } + + // Compile a list of the expected RFP submissions that should be in + // the runoff vote. This will be all of the public proposals that + // have linked to the RFP. The RFP's linked from list will include + // abandoned proposals that need to be filtered out. + linkedFrom, err := p.linkedFrom(rfpToken) if err != nil { return "", err } - if r.RecordMetadata.Status != backend.MDStatusVetted { - // This proposal is not public and should not be included in - // the runoff vote. - continue - } + // map[token]struct{} + expected := make(map[string]struct{}, len(linkedFrom.Tokens)) + for k := range linkedFrom.Tokens { + token, err := tokenDecode(k) + if err != nil { + return "", err + } + r, err := p.backend.GetVetted(token, "") + if err != nil { + return "", err + } + if r.RecordMetadata.Status != backend.MDStatusVetted { + // This proposal is not public and should not be included in + // the runoff vote. + continue + } - // This is a public proposal that is part of the RFP linked from - // list. It is required to be in the runoff vote. - expected[k] = struct{}{} - } + // This is a public proposal that is part of the RFP linked from + // list. It is required to be in the runoff vote. + expected[k] = struct{}{} + } - // Verify that there are no extra submissions in the runoff vote - for _, v := range s.Starts { - _, ok := expected[v.Params.Token] - if !ok { - // This submission should not be here. - e := fmt.Sprintf("found token that should not be included: %v", - v.Params.Token) - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusStartDetailsInvalid), - ErrorContext: []string{e}, + // Verify that there are no extra submissions in the runoff vote + for _, v := range s.Starts { + _, ok := expected[v.Params.Token] + if !ok { + // This submission should not be here. + e := fmt.Sprintf("found token that should not be included: %v", + v.Params.Token) + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusStartDetailsInvalid), + ErrorContext: []string{e}, + } } } - } - // Verify that the runoff vote is not missing any submissions - subs := make(map[string]struct{}, len(s.Starts)) - for _, v := range s.Starts { - subs[v.Params.Token] = struct{}{} - } - for token := range expected { - _, ok := subs[token] - if !ok { - // This proposal is missing from the runoff vote - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusStartDetailsMissing), - ErrorContext: []string{token}, + // Verify that the runoff vote is not missing any submissions + subs := make(map[string]struct{}, len(s.Starts)) + for _, v := range s.Starts { + subs[v.Params.Token] = struct{}{} + } + for token := range expected { + _, ok := subs[token] + if !ok { + // This proposal is missing from the runoff vote + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusStartDetailsMissing), + ErrorContext: []string{token}, + } } } - } - // Pi plugin validation complete! Pass the plugin command to the - // ticketvote plugin. - return p.backend.Plugin(ticketvote.ID, ticketvote.CmdStart, "", payload) + // Pi plugin validation complete! Pass the plugin command to the + // ticketvote plugin. + return p.backend.Plugin(ticketvote.ID, ticketvote.CmdStart, "", payload) + */ + return "", nil } -func (p *piPlugin) passThrough(pt pi.PassThrough) (string, error) { - switch pt.PluginID { +func (p *piPlugin) passThrough(pluginID, cmd, payload string) (string, error) { + switch pluginID { case comments.ID: - switch pt.PluginCmd { + switch cmd { case comments.CmdNew: - return p.commentNew(pt.Payload) + return p.commentNew(payload) case comments.CmdDel: - return p.commentDel(pt.Payload) + return p.commentDel(payload) case comments.CmdVote: - return p.commentVote(pt.Payload) + return p.commentVote(payload) default: return "", fmt.Errorf("invalid %v plugin cmd '%v'", - pt.PluginID, pt.PluginCmd) + pluginID, cmd) } case ticketvote.ID: - switch pt.PluginCmd { + switch cmd { case ticketvote.CmdStart: - return p.ticketVoteStart(pt.Payload) + return p.ticketVoteStart(payload) default: return "", fmt.Errorf("invalid %v plugin cmd '%v'", - pt.PluginID, pt.PluginCmd) + pluginID, cmd) } default: - return "", fmt.Errorf("invalid plugin id '%v'", pt.PluginID) - } -} - -func (p *piPlugin) cmdPassThrough(payload string) (string, error) { - // Decode payload - pt, err := pi.DecodePassThrough([]byte(payload)) - if err != nil { - return "", err - } - - // Execute command - r, err := p.passThrough(*pt) - if err != nil { - return "", err - } - - // Prepare reply - ptr := pi.PassThroughReply{ - Payload: r, + return "", fmt.Errorf("invalid plugin id '%v'", pluginID) } - reply, err := pi.EncodePassThroughReply(ptr) - if err != nil { - return "", err - } - - return string(reply), nil } func (p *piPlugin) hookNewRecordPre(payload string) error { - nr, err := decodeHookNewRecord([]byte(payload)) + var nr plugins.HookNewRecord + err := json.Unmarshal([]byte(payload), &nr) if err != nil { return err } @@ -1044,7 +860,7 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { return backend.PluginUserError{ PluginID: pi.ID, ErrorCode: int(pi.ErrorStatusPropLinkToInvalid), - ErrorContext: []string{"an rfp cannot have linkto set"}, + ErrorContext: "an rfp cannot have linkto set", } } tokenb, err := hex.DecodeString(pm.LinkTo) @@ -1053,7 +869,7 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { PluginID: pi.ID, ErrorCode: int(pi.ErrorStatusPropLinkToInvalid), - ErrorContext: []string{"invalid hex"}, + ErrorContext: "invalid hex", } } r, err := p.backend.GetVetted(tokenb, "") @@ -1062,7 +878,7 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { return backend.PluginUserError{ PluginID: pi.ID, ErrorCode: int(pi.ErrorStatusPropLinkToInvalid), - ErrorContext: []string{"proposal not found"}, + ErrorContext: "proposal not found", } } return err @@ -1075,14 +891,14 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { return backend.PluginUserError{ PluginID: pi.ID, ErrorCode: int(pi.ErrorStatusPropLinkToInvalid), - ErrorContext: []string{"proposal not an rfp"}, + ErrorContext: "proposal not an rfp", } } if !isRFP(*linkToPM) { return backend.PluginUserError{ PluginID: pi.ID, ErrorCode: int(pi.ErrorStatusPropLinkToInvalid), - ErrorContext: []string{"proposal not an rfp"}, + ErrorContext: "proposal not an rfp", } } if time.Now().Unix() > linkToPM.LinkBy { @@ -1090,7 +906,7 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { return backend.PluginUserError{ PluginID: pi.ID, ErrorCode: int(pi.ErrorStatusPropLinkToInvalid), - ErrorContext: []string{"rfp link by deadline expired"}, + ErrorContext: "rfp link by deadline expired", } } s := ticketvote.Summaries{ @@ -1118,7 +934,7 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { return backend.PluginUserError{ PluginID: pi.ID, ErrorCode: int(pi.ErrorStatusPropLinkToInvalid), - ErrorContext: []string{"rfp vote not approved"}, + ErrorContext: "rfp vote not approved", } } } @@ -1127,7 +943,8 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { } func (p *piPlugin) hookNewRecordPost(payload string) error { - nr, err := decodeHookNewRecord([]byte(payload)) + var nr plugins.HookNewRecord + err := json.Unmarshal([]byte(payload), &nr) if err != nil { return err } @@ -1151,7 +968,8 @@ func (p *piPlugin) hookNewRecordPost(payload string) error { } func (p *piPlugin) hookEditRecordPre(payload string) error { - er, err := decodeHookEditRecord([]byte(payload)) + var er plugins.HookEditRecord + err := json.Unmarshal([]byte(payload), &er) if err != nil { return err } @@ -1176,7 +994,7 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { return backend.PluginUserError{ PluginID: pi.ID, ErrorCode: int(pi.ErrorStatusPropLinkToInvalid), - ErrorContext: []string{"linkto cannot change on public proposal"}, + ErrorContext: "linkto cannot change on public proposal", } } } @@ -1213,7 +1031,7 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { return backend.PluginUserError{ PluginID: pi.ID, ErrorCode: int(pi.ErrorStatusVoteStatusInvalid), - ErrorContext: []string{e}, + ErrorContext: e, } } } @@ -1222,7 +1040,8 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { } func (p *piPlugin) hookSetRecordStatusPost(payload string) error { - srs, err := decodeHookSetRecordStatus([]byte(payload)) + var srs plugins.HookSetRecordStatus + err := json.Unmarshal([]byte(payload), &srs) if err != nil { return err } @@ -1234,7 +1053,8 @@ func (p *piPlugin) hookSetRecordStatusPost(payload string) error { continue } - sc, err = pi.DecodeStatusChange([]byte(v.Payload)) + var sc pi.StatusChange + err := json.Unmarshal([]byte(v.Payload), &sc) if err != nil { return err } @@ -1251,7 +1071,7 @@ func (p *piPlugin) hookSetRecordStatusPost(payload string) error { continue } - statuses, err = pi.DecodeStatusChanges([]byte(v.Payload)) + statuses, err = decodeStatusChanges([]byte(v.Payload)) if err != nil { return err } @@ -1265,7 +1085,7 @@ func (p *piPlugin) hookSetRecordStatusPost(payload string) error { return backend.PluginUserError{ PluginID: pi.ID, ErrorCode: int(pi.ErrorStatusPropVersionInvalid), - ErrorContext: []string{e}, + ErrorContext: e, } } @@ -1284,7 +1104,7 @@ func (p *piPlugin) hookSetRecordStatusPost(payload string) error { return backend.PluginUserError{ PluginID: pi.ID, ErrorCode: int(pi.ErrorStatusPropStatusChangeInvalid), - ErrorContext: []string{e}, + ErrorContext: e, } } @@ -1323,22 +1143,22 @@ func (p *piPlugin) hookSetRecordStatusPost(payload string) error { return nil } -// setup performs any plugin setup work that needs to be done. +// Setup performs any plugin setup work that needs to be done. // -// This function satisfies the Plugin interface. -func (p *piPlugin) setup() error { - log.Tracef("pi setup") +// This function satisfies the plugins.Client interface. +func (p *piPlugin) Setup() error { + log.Tracef("Setup") // TODO Verify vote and comment plugin dependency return nil } -// cmd executes a plugin command. +// Cmd executes a plugin command. // -// This function satisfies the pluginClient interface. -func (p *piPlugin) cmd(cmd, payload string) (string, error) { - log.Tracef("pi cmd: %v %v", cmd, payload) +// This function satisfies the plugins.Client interface. +func (p *piPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { + log.Tracef("Cmd: %v %x %v %v", treeID, token, cmd, payload) switch cmd { case pi.CmdProposals: @@ -1347,64 +1167,44 @@ func (p *piPlugin) cmd(cmd, payload string) (string, error) { return p.cmdProposalInv(payload) case pi.CmdVoteInventory: return p.cmdVoteInventory(payload) - case pi.CmdPassThrough: - return p.cmdPassThrough(payload) } return "", backend.ErrPluginCmdInvalid } -// hook executes a plugin hook. +// Hook executes a plugin hook. // -// This function satisfies the pluginClient interface. -func (p *piPlugin) hook(h hookT, payload string) error { - log.Tracef("pi hook: %v", hooks[h]) +// This function satisfies the plugins.Client interface. +func (p *piPlugin) Hook(h plugins.HookT, payload string) error { + log.Tracef("Hook: %v %v", plugins.Hooks[h], payload) switch h { - case hookNewRecordPre: + case plugins.HookNewRecordPre: return p.hookNewRecordPre(payload) - case hookNewRecordPost: + case plugins.HookNewRecordPost: return p.hookNewRecordPost(payload) - case hookEditRecordPre: + case plugins.HookEditRecordPre: return p.hookEditRecordPre(payload) - case hookSetRecordStatusPost: + case plugins.HookSetRecordStatusPost: return p.hookSetRecordStatusPost(payload) } return nil } -// fsck performs a plugin filesystem check. +// Fsck performs a plugin filesystem check. // -// This function satisfies the pluginClient interface. -func (p *piPlugin) fsck() error { - log.Tracef("pi fsck") +// This function satisfies the plugins.Client interface. +func (p *piPlugin) Fsck() error { + log.Tracef("Fsck") // linkedFrom cache return nil } -func newPiPlugin(backend backend.Backend, tlog tlogClient, settings []backend.PluginSetting, activeNetParams *chaincfg.Params) (*piPlugin, error) { - // Unpack plugin settings - var dataDir string - for _, v := range settings { - switch v.Key { - case pluginSettingDataDir: - dataDir = v.Value - default: - return nil, fmt.Errorf("invalid plugin setting '%v'", v.Key) - } - } - - // Verify plugin settings - switch { - case dataDir == "": - return nil, fmt.Errorf("plugin setting not found: %v", - pluginSettingDataDir) - } - - // Create the plugin data directory +func New(backend backend.Backend, tlog tlogclient.Client, settings []backend.PluginSetting, dataDir string, activeNetParams *chaincfg.Params) (*piPlugin, error) { + // Create plugin data directory dataDir = filepath.Join(dataDir, pi.ID) err := os.MkdirAll(dataDir, 0700) if err != nil { diff --git a/politeiad/backend/tlogbe/plugins/pi/pi_test.go b/politeiad/backend/tlogbe/plugins/pi/pi_test.go index d89bb0f8a..bea634995 100644 --- a/politeiad/backend/tlogbe/plugins/pi/pi_test.go +++ b/politeiad/backend/tlogbe/plugins/pi/pi_test.go @@ -4,18 +4,7 @@ package pi -import ( - "encoding/hex" - "errors" - "testing" - - "github.com/decred/politeia/politeiad/api/v1/identity" - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/plugins/comments" - "github.com/decred/politeia/politeiad/plugins/pi" - "github.com/google/uuid" -) - +/* func newTestPiPlugin(t *testing.T) (*piPlugin, *tlogBackend, func()) { t.Helper() @@ -351,3 +340,4 @@ func TestCommentDel(t *testing.T) { } } +*/ diff --git a/politeiad/backend/tlogbe/plugins/pi/user.go b/politeiad/backend/tlogbe/plugins/pi/user.go new file mode 100644 index 000000000..ea9dfdcf5 --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/pi/user.go @@ -0,0 +1,106 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package pi + +import ( + "encoding/json" + "errors" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +const ( + // fnUserData is the filename for the cached userData that is + // saved to the pi plugin data dir. + fnUserData = "{userid}.json" +) + +// userData contains cached pi plugin data for a specific user. The userData +// JSON is saved to disk in the pi plugin data dir. The user ID is included in +// the filename. +type userData struct { + // Tokens contains a list of all the proposals that have been + // submitted by this user. This data is cached so that the + // ProposalInv command can filter proposals by user ID. + Tokens []string `json:"tokens"` +} + +// userDataPath returns the filepath to the cached userData struct for the +// specified user. +func (p *piPlugin) userDataPath(userID string) string { + fn := strings.Replace(fnUserData, "{userid}", userID, 1) + return filepath.Join(p.dataDir, fn) +} + +// userDataWithLock returns the cached userData struct for the specified user. +// +// This function must be called WITH the lock held. +func (p *piPlugin) userDataWithLock(userID string) (*userData, error) { + fp := p.userDataPath(userID) + b, err := ioutil.ReadFile(fp) + if err != nil { + var e *os.PathError + if errors.As(err, &e) && !os.IsExist(err) { + // File does't exist. Return an empty userData. + return &userData{ + Tokens: []string{}, + }, nil + } + } + + var ud userData + err = json.Unmarshal(b, &ud) + if err != nil { + return nil, err + } + + return &ud, nil +} + +// userData returns the cached userData struct for the specified user. +// +// This function must be called WITHOUT the lock held. +func (p *piPlugin) userData(userID string) (*userData, error) { + p.Lock() + defer p.Unlock() + + return p.userDataWithLock(userID) +} + +// userDataSaveWithLock saves the provided userData to the pi plugin data dir. +// +// This function must be called WITH the lock held. +func (p *piPlugin) userDataSaveWithLock(userID string, ud userData) error { + b, err := json.Marshal(ud) + if err != nil { + return err + } + + fp := p.userDataPath(userID) + return ioutil.WriteFile(fp, b, 0664) +} + +// userDataAddToken adds the provided token to the cached userData for the +// provided user. +// +// This function must be called WITHOUT the lock held. +func (p *piPlugin) userDataAddToken(userID string, token string) error { + p.Lock() + defer p.Unlock() + + // Get current user data + ud, err := p.userDataWithLock(userID) + if err != nil { + return err + } + + // Add token + ud.Tokens = append(ud.Tokens, token) + + // Save changes + return p.userDataSaveWithLock(userID, *ud) +} diff --git a/politeiad/backend/tlogbe/tlog/plugin.go b/politeiad/backend/tlogbe/tlog/plugin.go index 0f649fd3d..9d8decee4 100644 --- a/politeiad/backend/tlogbe/tlog/plugin.go +++ b/politeiad/backend/tlogbe/tlog/plugin.go @@ -13,8 +13,10 @@ import ( "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/comments" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/dcrdata" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/pi" cmplugin "github.com/decred/politeia/politeiad/plugins/comments" ddplugin "github.com/decred/politeia/politeiad/plugins/dcrdata" + piplugin "github.com/decred/politeia/politeiad/plugins/pi" ) const ( @@ -71,13 +73,12 @@ func (t *Tlog) PluginRegister(b backend.Backend, p backend.Plugin) error { if err != nil { return err } + case piplugin.ID: + client, err = pi.New(b, t, p.Settings, dataDir, t.activeNetParams) + if err != nil { + return err + } /* - case pi.ID: - client, err = newPiPlugin(t, newBackendClient(t), p.Settings, - t.activeNetParams) - if err != nil { - return err - } case ticketvote.ID: client, err = newTicketVotePlugin(t, newBackendClient(t), p.Settings, p.Identity, t.activeNetParams) diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 594a69ac3..70056f2ad 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -94,64 +94,6 @@ func tokenIsFullLength(token []byte) bool { return len(token) == v1.TokenSizeTlog } -// tokenDecode decodes the provided hex encoded record token. This function -// requires that the token be the full length. Token prefixes will return an -// error. -func tokenDecode(token string) ([]byte, error) { - t, err := hex.DecodeString(token) - if err != nil { - return nil, fmt.Errorf("invalid hex") - } - if !tokenIsFullLength(t) { - return nil, fmt.Errorf("invalid token size") - } - return t, nil -} - -// tokenDecodeAnyLength decodes the provided hex encoded record token. This -// function accepts both full length tokens and token prefixes. -func tokenDecodeAnyLength(token string) ([]byte, error) { - // If provided token has odd length add padding - if len(token)%2 == 1 { - token = token + "0" - } - t, err := hex.DecodeString(token) - if err != nil { - return nil, fmt.Errorf("invalid hex") - } - switch { - case tokenIsFullLength(t): - // Full length tokens are allowed; continue - case len(t) == tokenPrefixSize(): - // Token prefixes are allowed; continue - default: - return nil, fmt.Errorf("invalid token size") - } - return t, nil -} - -// tokenPrefixSize returns the size in bytes of a token prefix. -func tokenPrefixSize() int { - // If the token prefix length is an odd number of characters then - // padding would have needed to be added to it prior to decoding it - // to hex to prevent a hex.ErrLenth (odd length hex string) error. - // Account for this padding in the prefix size. - var size int - if v1.TokenPrefixLength%2 == 1 { - // Add 1 to the length to account for padding - size = (v1.TokenPrefixLength + 1) / 2 - } else { - // No padding was required - size = v1.TokenPrefixLength / 2 - } - return size -} - -// tokenPrefix returns the token prefix as defined by the politeiad API. -func tokenPrefix(token []byte) string { - return hex.EncodeToString(token)[:v1.TokenPrefixLength] -} - func tokenFromTreeID(treeID int64) []byte { b := make([]byte, v1.TokenSizeTlog) // Converting between int64 and uint64 doesn't change @@ -174,7 +116,7 @@ func (t *tlogBackend) fullLengthToken(prefix []byte) ([]byte, bool) { t.Lock() defer t.Unlock() - token, ok := t.prefixes[tokenPrefix(prefix)] + token, ok := t.prefixes[util.TokenPrefix(prefix)] return token, ok } @@ -183,7 +125,7 @@ func (t *tlogBackend) fullLengthToken(prefix []byte) ([]byte, bool) { // // This function must be called WITHOUT the lock held. func (t *tlogBackend) unvettedTreeIDFromToken(token []byte) int64 { - if len(token) == tokenPrefixSize() { + if len(token) == util.TokenPrefixSize() { // This is a token prefix. Get the full token from the cache. var ok bool token, ok = t.fullLengthToken(token) @@ -198,7 +140,7 @@ func (t *tlogBackend) prefixExists(fullToken []byte) bool { t.RLock() defer t.RUnlock() - _, ok := t.prefixes[tokenPrefix(fullToken)] + _, ok := t.prefixes[util.TokenPrefix(fullToken)] return ok } @@ -206,7 +148,7 @@ func (t *tlogBackend) prefixAdd(fullToken []byte) { t.Lock() defer t.Unlock() - prefix := tokenPrefix(fullToken) + prefix := util.TokenPrefix(fullToken) t.prefixes[prefix] = fullToken log.Debugf("Add token prefix: %v", prefix) @@ -752,7 +694,7 @@ func (t *tlogBackend) New(metadata []backend.MetadataStream, files []backend.Fil } log.Infof("Token prefix collision %v, creating new token", - tokenPrefix(token)) + util.TokenPrefix(token)) } // Create record metadata diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index 1c5c31837..134cb01d0 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -6,25 +6,13 @@ // decred's proposal system. package pi -import ( - "encoding/json" - "errors" - "io" - "strings" - - "github.com/decred/politeia/politeiad/plugins/comments" -) - -type PropStateT int type PropStatusT int type ErrorStatusT int -type CommentVoteT int const ( ID = "pi" // Plugin commands - CmdPassThrough = "passthrough" // Plugin command pass through CmdProposalInv = "proposalinv" // Get inventory by proposal status CmdVoteInventory = "voteinv" // Get inventory by vote status @@ -42,11 +30,6 @@ const ( // root that politeiad signs. FileNameProposalMetadata = "proposalmetadata.json" - // Proposal states - PropStateInvalid PropStateT = 0 - PropStateUnvetted PropStateT = 1 - PropStateVetted PropStateT = 2 - // Proposal status codes PropStatusInvalid PropStatusT = 0 // Invalid status PropStatusUnvetted PropStatusT = 1 // Prop has not been vetted @@ -118,21 +101,6 @@ type ProposalMetadata struct { LinkBy int64 `json:"linkby,omitempty"` } -// EncodeProposalMetadata encodes a ProposalMetadata into a JSON byte slice. -func EncodeProposalMetadata(pm ProposalMetadata) ([]byte, error) { - return json.Marshal(pm) -} - -// DecodeProposalMetadata decodes a JSON byte slice into a ProposalMetadata. -func DecodeProposalMetadata(payload []byte) (*ProposalMetadata, error) { - var pm ProposalMetadata - err := json.Unmarshal(payload, &pm) - if err != nil { - return nil, err - } - return &pm, nil -} - // GeneralMetadata contains general metadata about a politeiad record. It is // saved to politeiad as a metadata stream. // @@ -145,21 +113,6 @@ type GeneralMetadata struct { Timestamp int64 `json:"timestamp"` // Submission UNIX timestamp } -// EncodeGeneralMetadata encodes a GeneralMetadata into a JSON byte slice. -func EncodeGeneralMetadata(gm GeneralMetadata) ([]byte, error) { - return json.Marshal(gm) -} - -// DecodeGeneralMetadata decodes a JSON byte slice into a GeneralMetadata. -func DecodeGeneralMetadata(payload []byte) (*GeneralMetadata, error) { - var gm GeneralMetadata - err := json.Unmarshal(payload, &gm) - if err != nil { - return nil, err - } - return &gm, nil -} - // StatusChange represents a proposal status change. // // Signature is the client signature of the Token+Version+Status+Reason. @@ -173,112 +126,11 @@ type StatusChange struct { Timestamp int64 `json:"timestamp"` } -// EncodeStatusChange encodes a StatusChange into a JSON byte slice. -func EncodeStatusChange(sc StatusChange) ([]byte, error) { - return json.Marshal(sc) -} - -// DecodeStatusChange decodes a JSON byte slice into a StatusChange. -func DecodeStatusChange(payload []byte) (*StatusChange, error) { - var sc StatusChange - err := json.Unmarshal(payload, &sc) - if err != nil { - return nil, err - } - return &sc, nil -} - -// DecodeStatusChanges decodes a JSON byte slice into a []StatusChange. -func DecodeStatusChanges(payload []byte) ([]StatusChange, error) { - var statuses []StatusChange - d := json.NewDecoder(strings.NewReader(string(payload))) - for { - var sc StatusChange - err := d.Decode(&sc) - if errors.Is(err, io.EOF) { - break - } else if err != nil { - return nil, err - } - statuses = append(statuses, sc) - } - - return statuses, nil -} - -// PassThrough is used to extended a command from a different plugin package -// with functionality that is specific to the pi plugin, without needing to -// define a new pi command and new pi types. The command passes through the pi -// plugin before being executed. -// -// Example, the pi plugin does not allow comments to be made on a proposal if -// the proposal vote has ended. This validation is specific to the pi plugin -// and does not require the comment payloads to be altered. PassThrough is used -// to pass the comments plugin command through the pi plugin, where the pi -// plugin can first perform pi specific validation before executing the -// comments plugin command. -type PassThrough struct { - PluginID string `json:"pluginid"` - PluginCmd string `json:"plugincmd"` - Payload string `json:"payload"` -} - -// EncodePassThrough encodes a PassThrough into a JSON byte slice. -func EncodePassThrough(p PassThrough) ([]byte, error) { - return json.Marshal(p) -} - -// DecodePassThrough decodes a JSON byte slice into a PassThrough. -func DecodePassThrough(payload []byte) (*PassThrough, error) { - var p PassThrough - err := json.Unmarshal(payload, &p) - if err != nil { - return nil, err - } - return &p, nil -} - -// PassThroughReply is the reply to the PassThrough command. -type PassThroughReply struct { - Payload string `json:"payload"` -} - -// EncodePassThroughReply encodes a PassThroughReply into a JSON byte slice. -func EncodePassThroughReply(p PassThroughReply) ([]byte, error) { - return json.Marshal(p) -} - -// DecodePassThroughReply decodes a JSON byte slice into a PassThroughReply. -func DecodePassThroughReply(payload []byte) (*PassThroughReply, error) { - var p PassThroughReply - err := json.Unmarshal(payload, &p) - if err != nil { - return nil, err - } - return &p, nil -} - // Proposals requests the plugin data for the provided proposals. This includes // pi plugin data as well as other plugin data such as comment plugin data. // This command aggregates all proposal plugin data into a single call. type Proposals struct { - State PropStateT `json:"state"` - Tokens []string `json:"tokens"` -} - -// EncodeProposals encodes a Proposals into a JSON byte slice. -func EncodeProposals(p Proposals) ([]byte, error) { - return json.Marshal(p) -} - -// DecodeProposals decodes a JSON byte slice into a Proposals. -func DecodeProposals(payload []byte) (*Proposals, error) { - var p Proposals - err := json.Unmarshal(payload, &p) - if err != nil { - return nil, err - } - return &p, nil + Tokens []string `json:"tokens"` } // ProposalPluginData contains all the plugin data for a proposal. @@ -293,183 +145,6 @@ type ProposalsReply struct { Proposals map[string]ProposalPluginData `json:"proposals"` } -// EncodeProposalsReply encodes a ProposalsReply into a JSON byte slice. -func EncodeProposalsReply(pr ProposalsReply) ([]byte, error) { - return json.Marshal(pr) -} - -// DecodeProposalsReply decodes a JSON byte slice into a ProposalsReply. -func DecodeProposalsReply(payload []byte) (*ProposalsReply, error) { - var pr ProposalsReply - err := json.Unmarshal(payload, &pr) - if err != nil { - return nil, err - } - return &pr, nil -} - -// CommentNew creates a new comment. This command relies on the comments plugin -// New command, but also performs additional vote status validation that is -// specific to pi. -// -// The parent ID is used to reply to an existing comment. A parent ID of 0 -// indicates that the comment is a base level comment and not a reply commment. -// -// Signature is the client signature of State+Token+ParentID+Comment. -type CommentNew struct { - UserID string `json:"userid"` // Unique user ID - State PropStateT `json:"state"` // Record state - Token string `json:"token"` // Record token - ParentID uint32 `json:"parentid"` // Parent comment ID - Comment string `json:"comment"` // Comment text - PublicKey string `json:"publickey"` // Pubkey used for Signature - Signature string `json:"signature"` // Client signature -} - -// EncodeCommentNew encodes a CommentNew into a JSON byte slice. -func EncodeCommentNew(cn CommentNew) ([]byte, error) { - return json.Marshal(cn) -} - -// DecodeCommentNew decodes a JSON byte slice into a CommentNew. -func DecodeCommentNew(payload []byte) (*CommentNew, error) { - var cn CommentNew - err := json.Unmarshal(payload, &cn) - if err != nil { - return nil, err - } - return &cn, nil -} - -// CommentNewReply is the reply to the CommentNew command. -type CommentNewReply struct { - Comment comments.Comment `json:"comment"` -} - -// EncodeCommentNewReply encodes a CommentNewReply into a JSON byte slice. -func EncodeCommentNewReply(cnr CommentNewReply) ([]byte, error) { - return json.Marshal(cnr) -} - -// DecodeCommentNewReply decodes a JSON byte slice into a CommentNewReply. -func DecodeCommentNewReply(payload []byte) (*CommentNewReply, error) { - var cnr CommentNewReply - err := json.Unmarshal(payload, &cnr) - if err != nil { - return nil, err - } - return &cnr, nil -} - -// CommentCensor permanently deletes the provided comment. This command relies -// on the comments plugin Del command, but also performs additional vote status -// validation that is specific to pi. -// -// Signature is the client signature of the State+Token+CommentID+Reason -type CommentCensor struct { - State PropStateT `json:"state"` // Record state - Token string `json:"token"` // Record token - CommentID uint32 `json:"commentid"` // Comment ID - Reason string `json:"reason"` // Reason for deletion - PublicKey string `json:"publickey"` // Public key used for signature - Signature string `json:"signature"` // Client signature -} - -// EncodeCommentCensor encodes a CommentCensor into a JSON byte slice. -func EncodeCommentCensor(cc CommentCensor) ([]byte, error) { - return json.Marshal(cc) -} - -// DecodeCommentCensor decodes a JSON byte slice into a CommentCensor. -func DecodeCommentCensor(payload []byte) (*CommentCensor, error) { - var cc CommentCensor - err := json.Unmarshal(payload, &cc) - if err != nil { - return nil, err - } - return &cc, nil -} - -// CommentCensorReply is the reply to the CommentCensor command. -type CommentCensorReply struct { - Comment comments.Comment `json:"comment"` -} - -// EncodeCommentCensorReply encodes a CommentCensorReply into a JSON byte -// slice. -func EncodeCommentCensorReply(ccr CommentCensorReply) ([]byte, error) { - return json.Marshal(ccr) -} - -// DecodeCommentCensorReply decodes a JSON byte slice into CommentCensorReply. -func DecodeCommentCensorReply(payload []byte) (*CommentCensorReply, error) { - var d CommentCensorReply - err := json.Unmarshal(payload, &d) - if err != nil { - return nil, err - } - return &d, nil -} - -// CommentVote casts a comment vote (upvote or downvote). This command relies -// on the comments plugin Del command, but also performs additional vote status -// validation that is specific to pi. -// -// The effect of a new vote on a comment score depends on the previous vote -// from that user ID. Example, a user upvotes a comment that they have already -// upvoted, the resulting vote score is 0 due to the second upvote removing the -// original upvote. The public key cannot be relied on to remain the same for -// each user so a user ID must be included. -// -// Signature is the client signature of the State+Token+CommentID+Vote. -type CommentVote struct { - UserID string `json:"userid"` // Unique user ID - State PropStateT `json:"state"` // Record state - Token string `json:"token"` // Record token - CommentID uint32 `json:"commentid"` // Comment ID - Vote CommentVoteT `json:"vote"` // Upvote or downvote - PublicKey string `json:"publickey"` // Public key used for signature - Signature string `json:"signature"` // Client signature -} - -// EncodeCommentVote encodes a CommentVote into a JSON byte slice. -func EncodeCommentVote(cv CommentVote) ([]byte, error) { - return json.Marshal(cv) -} - -// DecodeCommentVote decodes a JSON byte slice into a CommentVote. -func DecodeCommentVote(payload []byte) (*CommentVote, error) { - var cv CommentVote - err := json.Unmarshal(payload, &cv) - if err != nil { - return nil, err - } - return &cv, nil -} - -// CommentVoteReply is the reply to the CommentVote command. -type CommentVoteReply struct { - Downvotes uint64 `json:"downvotes"` // Tolal downvotes on comment - Upvotes uint64 `json:"upvotes"` // Total upvotes on comment - Timestamp int64 `json:"timestamp"` // Received UNIX timestamp - Receipt string `json:"receipt"` // Server signature of client signature -} - -// EncodeCommentVoteReply encodes a CommentVoteReply into a JSON byte slice. -func EncodeCommentVoteReply(cvr CommentVoteReply) ([]byte, error) { - return json.Marshal(cvr) -} - -// DecodeCommentVoteReply decodes a JSON byte slice into a CommentVoteReply. -func DecodeCommentVoteReply(payload []byte) (*CommentVoteReply, error) { - var cvr CommentVoteReply - err := json.Unmarshal(payload, &cvr) - if err != nil { - return nil, err - } - return &cvr, nil -} - // ProposalInv retrieves the tokens of all proposals in the inventory that // match the provided filtering criteria. The returned proposals are // categorized by proposal state and status. If no filtering criteria is @@ -478,21 +153,6 @@ type ProposalInv struct { UserID string `json:"userid,omitempty"` } -// EncodeProposalInv encodes a ProposalInv into a JSON byte slice. -func EncodeProposalInv(pi ProposalInv) ([]byte, error) { - return json.Marshal(pi) -} - -// DecodeProposalInv decodes a JSON byte slice into a ProposalInv. -func DecodeProposalInv(payload []byte) (*ProposalInv, error) { - var pi ProposalInv - err := json.Unmarshal(payload, &pi) - if err != nil { - return nil, err - } - return &pi, nil -} - // ProposalInvReply is the reply to the ProposalInv command. The returned maps // contains map[status][]token where the status is the human readable proposal // status and the token is the proposal token. @@ -501,44 +161,12 @@ type ProposalInvReply struct { Vetted map[string][]string `json:"vetted"` } -// EncodeProposalInvReply encodes a ProposalInvReply into a JSON -// byte slice. -func EncodeProposalInvReply(pir ProposalInvReply) ([]byte, error) { - return json.Marshal(pir) -} - -// DecodeProposalInvReply decodes a JSON byte slice into a -// ProposalInvReply. -func DecodeProposalInvReply(payload []byte) (*ProposalInvReply, error) { - var pir ProposalInvReply - err := json.Unmarshal(payload, &pir) - if err != nil { - return nil, err - } - return &pir, nil -} - // VoteInventory requests the tokens of all proposals in the inventory // categorized by their vote status. This call relies on the ticketvote // Inventory call, but breaks the Finished vote status out into Approved and // Rejected categories. This functionality is specific to pi. type VoteInventory struct{} -// EncodeVoteInventory encodes a VoteInventory into a JSON byte slice. -func EncodeVoteInventory(vi VoteInventory) ([]byte, error) { - return json.Marshal(vi) -} - -// DecodeVoteInventory decodes a JSON byte slice into a VoteInventory. -func DecodeVoteInventory(payload []byte) (*VoteInventory, error) { - var vi VoteInventory - err := json.Unmarshal(payload, &vi) - if err != nil { - return nil, err - } - return &vi, nil -} - // VoteInventoryReply is the reply to the VoteInventory command. type VoteInventoryReply struct { Unauthorized []string `json:"unauthorized"` @@ -551,19 +179,3 @@ type VoteInventoryReply struct { // inventory. BestBlock uint32 `json:"bestblock"` } - -// EncodeVoteInventoryReply encodes a VoteInventoryReply into a JSON byte -// slice. -func EncodeVoteInventoryReply(vir VoteInventoryReply) ([]byte, error) { - return json.Marshal(vir) -} - -// DecodeVoteInventoryReply decodes a JSON byte slice into VoteInventoryReply. -func DecodeVoteInventoryReply(payload []byte) (*VoteInventoryReply, error) { - var vir VoteInventoryReply - err := json.Unmarshal(payload, &vir) - if err != nil { - return nil, err - } - return &vir, nil -} diff --git a/util/token.go b/util/token.go index 4d1fb899b..a87d446d5 100644 --- a/util/token.go +++ b/util/token.go @@ -28,7 +28,7 @@ func tokenIsFullLength(tokenType string, token []byte) bool { } } -func tokenPrefixSize() int { +func TokenPrefixSize() int { // If the token prefix length is an odd number of characters then // padding would have needed to be added to it prior to decoding it // to hex to prevent a hex.ErrLenth (odd length hex string) error. @@ -81,7 +81,7 @@ func TokenDecodeAnyLength(tokenType, token string) ([]byte, error) { // Verify token byte slice is either a token prefix or a valid // full length token. switch { - case len(t) == tokenPrefixSize(): + case len(t) == TokenPrefixSize(): // This is a token prefix. Token prefixes are the same size // regardless of token type. case tokenIsFullLength(TokenTypeGit, t): From a5f33d676091f9fb3b4ab65b665e1589fb29d73e Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 15 Jan 2021 17:37:16 -0600 Subject: [PATCH 225/449] tlogbe: Add hooks to plugin commands. --- politeiad/backend/tlogbe/plugins/pi/pi.go | 8 +- politeiad/backend/tlogbe/plugins/plugins.go | 57 +++++++----- politeiad/backend/tlogbe/tlogbe.go | 98 +++++++++++++++++---- 3 files changed, 122 insertions(+), 41 deletions(-) diff --git a/politeiad/backend/tlogbe/plugins/pi/pi.go b/politeiad/backend/tlogbe/plugins/pi/pi.go index df77bc270..f60cb00fb 100644 --- a/politeiad/backend/tlogbe/plugins/pi/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi/pi.go @@ -1179,13 +1179,13 @@ func (p *piPlugin) Hook(h plugins.HookT, payload string) error { log.Tracef("Hook: %v %v", plugins.Hooks[h], payload) switch h { - case plugins.HookNewRecordPre: + case plugins.HookTypeNewRecordPre: return p.hookNewRecordPre(payload) - case plugins.HookNewRecordPost: + case plugins.HookTypeNewRecordPost: return p.hookNewRecordPost(payload) - case plugins.HookEditRecordPre: + case plugins.HookTypeEditRecordPre: return p.hookEditRecordPre(payload) - case plugins.HookSetRecordStatusPost: + case plugins.HookTypeSetRecordStatusPost: return p.hookSetRecordStatusPost(payload) } diff --git a/politeiad/backend/tlogbe/plugins/plugins.go b/politeiad/backend/tlogbe/plugins/plugins.go index 260e48b46..276b6f73f 100644 --- a/politeiad/backend/tlogbe/plugins/plugins.go +++ b/politeiad/backend/tlogbe/plugins/plugins.go @@ -10,32 +10,32 @@ import "github.com/decred/politeia/politeiad/backend" type HookT int const ( - HookInvalid HookT = 0 - HookNewRecordPre HookT = 1 - HookNewRecordPost HookT = 2 - HookEditRecordPre HookT = 3 - HookEditRecordPost HookT = 4 - HookEditMetadataPre HookT = 5 - HookEditMetadataPost HookT = 6 - HookSetRecordStatusPre HookT = 7 - HookSetRecordStatusPost HookT = 8 - HookPluginPre HookT = 9 - HookPluginPost HookT = 10 + HookTypeInvalid HookT = 0 + HookTypeNewRecordPre HookT = 1 + HookTypeNewRecordPost HookT = 2 + HookTypeEditRecordPre HookT = 3 + HookTypeEditRecordPost HookT = 4 + HookTypeEditMetadataPre HookT = 5 + HookTypeEditMetadataPost HookT = 6 + HookTypeSetRecordStatusPre HookT = 7 + HookTypeSetRecordStatusPost HookT = 8 + HookTypePluginPre HookT = 9 + HookTypePluginPost HookT = 10 ) var ( // Hooks contains human readable descriptions of the plugin hooks. Hooks = map[HookT]string{ - HookNewRecordPre: "new record pre", - HookNewRecordPost: "new record post", - HookEditRecordPre: "edit record pre", - HookEditRecordPost: "edit record post", - HookEditMetadataPre: "edit metadata pre", - HookEditMetadataPost: "edit metadata post", - HookSetRecordStatusPre: "set record status pre", - HookSetRecordStatusPost: "set record status post", - HookPluginPre: "plugin pre", - HookPluginPost: "plugin post", + HookTypeNewRecordPre: "new record pre", + HookTypeNewRecordPost: "new record post", + HookTypeEditRecordPre: "edit record pre", + HookTypeEditRecordPost: "edit record post", + HookTypeEditMetadataPre: "edit metadata pre", + HookTypeEditMetadataPost: "edit metadata post", + HookTypeSetRecordStatusPre: "set record status pre", + HookTypeSetRecordStatusPost: "set record status post", + HookTypePluginPre: "plugin pre", + HookTypePluginPost: "plugin post", } ) @@ -85,6 +85,21 @@ type HookSetRecordStatus struct { MDOverwrite []backend.MetadataStream `json:"mdoverwrite"` } +// HookPluginPre is the payload for the plugin pre hook. +type HookPluginPre struct { + PluginID string `json:"pluginid"` + Cmd string `json:"cmd"` + Payload string `json:"payload"` +} + +// HookPluginPost is the payload for the plugin post hook. +type HookPluginPost struct { + PluginID string `json:"pluginid"` + Cmd string `json:"cmd"` + Payload string `json:"payload"` + Reply string `json:"reply"` +} + // Client provides an API for a tlog instance to use when interacting with a // plugin. All tlog plugins must implement the Client interface. type Client interface { diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 70056f2ad..404491df5 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -666,7 +666,7 @@ func (t *tlogBackend) New(metadata []backend.MetadataStream, files []backend.Fil if err != nil { return nil, err } - err = t.unvetted.PluginHookPre(plugins.HookNewRecordPre, string(b)) + err = t.unvetted.PluginHookPre(plugins.HookTypeNewRecordPre, string(b)) if err != nil { return nil, err } @@ -719,7 +719,7 @@ func (t *tlogBackend) New(metadata []backend.MetadataStream, files []backend.Fil if err != nil { return nil, err } - t.unvetted.PluginHookPost(plugins.HookNewRecordPost, string(b)) + t.unvetted.PluginHookPost(plugins.HookTypeNewRecordPost, string(b)) // Update the inventory cache t.inventoryAdd(stateUnvetted, token, backend.MDStatusUnvetted) @@ -800,7 +800,7 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ if err != nil { return nil, err } - err = t.unvetted.PluginHookPre(plugins.HookEditRecordPre, string(b)) + err = t.unvetted.PluginHookPre(plugins.HookTypeEditRecordPre, string(b)) if err != nil { return nil, err } @@ -818,7 +818,7 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ } // Call post plugin hooks - t.unvetted.PluginHookPost(plugins.HookEditRecordPost, string(b)) + t.unvetted.PluginHookPost(plugins.HookTypeEditRecordPost, string(b)) // Return updated record r, err = t.unvetted.RecordLatest(treeID) @@ -902,7 +902,7 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b if err != nil { return nil, err } - err = t.vetted.PluginHookPre(plugins.HookEditRecordPre, string(b)) + err = t.vetted.PluginHookPre(plugins.HookTypeEditRecordPre, string(b)) if err != nil { return nil, err } @@ -920,7 +920,7 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b } // Call post plugin hooks - t.vetted.PluginHookPost(plugins.HookEditRecordPost, string(b)) + t.vetted.PluginHookPost(plugins.HookTypeEditRecordPost, string(b)) // Return updated record r, err = t.vetted.RecordLatest(treeID) @@ -993,7 +993,7 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite if err != nil { return err } - err = t.unvetted.PluginHookPre(plugins.HookEditMetadataPre, string(b)) + err = t.unvetted.PluginHookPre(plugins.HookTypeEditMetadataPre, string(b)) if err != nil { return err } @@ -1014,7 +1014,7 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite } // Call post plugin hooks - t.unvetted.PluginHookPost(plugins.HookEditMetadataPost, string(b)) + t.unvetted.PluginHookPost(plugins.HookTypeEditMetadataPost, string(b)) return nil } @@ -1083,7 +1083,7 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ if err != nil { return err } - err = t.vetted.PluginHookPre(plugins.HookEditMetadataPre, string(b)) + err = t.vetted.PluginHookPre(plugins.HookTypeEditMetadataPre, string(b)) if err != nil { return err } @@ -1104,7 +1104,7 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ } // Call post plugin hooks - t.vetted.PluginHookPost(plugins.HookEditMetadataPost, string(b)) + t.vetted.PluginHookPost(plugins.HookTypeEditMetadataPost, string(b)) return nil } @@ -1371,7 +1371,7 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, if err != nil { return nil, err } - err = t.unvetted.PluginHookPre(plugins.HookSetRecordStatusPre, string(b)) + err = t.unvetted.PluginHookPre(plugins.HookTypeSetRecordStatusPre, string(b)) if err != nil { return nil, err } @@ -1394,7 +1394,7 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, } // Call post plugin hooks - t.unvetted.PluginHookPost(plugins.HookSetRecordStatusPost, string(b)) + t.unvetted.PluginHookPost(plugins.HookTypeSetRecordStatusPost, string(b)) log.Debugf("Status change %x from %v (%v) to %v (%v)", token, backend.MDStatus[currStatus], currStatus, @@ -1523,7 +1523,7 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md if err != nil { return nil, err } - err = t.vetted.PluginHookPre(plugins.HookSetRecordStatusPre, string(b)) + err = t.vetted.PluginHookPre(plugins.HookTypeSetRecordStatusPre, string(b)) if err != nil { return nil, err } @@ -1546,7 +1546,7 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md } // Call post plugin hooks - t.vetted.PluginHookPost(plugins.HookSetRecordStatusPost, string(b)) + t.vetted.PluginHookPost(plugins.HookTypeSetRecordStatusPost, string(b)) // Update inventory cache t.inventoryUpdate(stateVetted, token, currStatus, status) @@ -1650,7 +1650,40 @@ func (t *tlogBackend) UnvettedPlugin(token []byte, pluginID, cmd, payload string return "", backend.ErrRecordNotFound } - return t.unvetted.PluginCmd(treeID, token, pluginID, cmd, payload) + // Call pre plugin hooks + hp := plugins.HookPluginPre{ + PluginID: pluginID, + Cmd: cmd, + Payload: payload, + } + b, err := json.Marshal(hp) + if err != nil { + return "", err + } + err = t.unvetted.PluginHookPre(plugins.HookTypePluginPre, string(b)) + if err != nil { + return "", err + } + + reply, err := t.unvetted.PluginCmd(treeID, token, pluginID, cmd, payload) + if err != nil { + return "", err + } + + // Call post plugin hooks + hpp := plugins.HookPluginPost{ + PluginID: pluginID, + Cmd: cmd, + Payload: payload, + Reply: reply, + } + b, err = json.Marshal(hpp) + if err != nil { + return "", err + } + t.unvetted.PluginHookPost(plugins.HookTypePluginPost, string(b)) + + return reply, nil } // VettedPlugin executes a plugin command on an unvetted record. @@ -1669,7 +1702,40 @@ func (t *tlogBackend) VettedPlugin(token []byte, pluginID, cmd, payload string) return "", backend.ErrRecordNotFound } - return t.vetted.PluginCmd(treeID, token, pluginID, cmd, payload) + // Call pre plugin hooks + hp := plugins.HookPluginPre{ + PluginID: pluginID, + Cmd: cmd, + Payload: payload, + } + b, err := json.Marshal(hp) + if err != nil { + return "", err + } + err = t.vetted.PluginHookPre(plugins.HookTypePluginPre, string(b)) + if err != nil { + return "", err + } + + reply, err := t.vetted.PluginCmd(treeID, token, pluginID, cmd, payload) + if err != nil { + return "", err + } + + // Call post plugin hooks + hpp := plugins.HookPluginPost{ + PluginID: pluginID, + Cmd: cmd, + Payload: payload, + Reply: reply, + } + b, err = json.Marshal(hpp) + if err != nil { + return "", err + } + t.vetted.PluginHookPost(plugins.HookTypePluginPost, string(b)) + + return reply, nil } // GetUnvettedPlugins returns the unvetted plugins that have been registered. From e70800d4d44cf3f88ce3d22740fb20d66cabe03e Mon Sep 17 00:00:00 2001 From: luke Date: Sat, 16 Jan 2021 14:32:52 -0600 Subject: [PATCH 226/449] tlogbe: Add plugin hooks. --- .../tlogclient.go => clients/clients.go} | 26 +- .../tlogbe/plugins/comments/comments.go | 10 +- .../tlogbe/plugins/comments/comments_test.go | 7 +- .../backend/tlogbe/plugins/dcrdata/dcrdata.go | 4 +- politeiad/backend/tlogbe/plugins/pi/pi.go | 165 ++++----- .../backend/tlogbe/plugins/pi/pi_test.go | 343 ------------------ politeiad/backend/tlogbe/plugins/plugins.go | 81 +++-- politeiad/backend/tlogbe/tlog/plugin.go | 28 +- politeiad/backend/tlogbe/tlogbe.go | 310 +++++++++------- .../tlogclient/testclient/testclient.go | 55 --- politeiad/plugins/pi/pi.go | 2 - 11 files changed, 332 insertions(+), 699 deletions(-) rename politeiad/backend/tlogbe/{tlogclient/tlogclient.go => clients/clients.go} (68%) delete mode 100644 politeiad/backend/tlogbe/plugins/pi/pi_test.go delete mode 100644 politeiad/backend/tlogbe/tlogclient/testclient/testclient.go diff --git a/politeiad/backend/tlogbe/tlogclient/tlogclient.go b/politeiad/backend/tlogbe/clients/clients.go similarity index 68% rename from politeiad/backend/tlogbe/tlogclient/tlogclient.go rename to politeiad/backend/tlogbe/clients/clients.go index 14733f777..d4c999e4b 100644 --- a/politeiad/backend/tlogbe/tlogclient/tlogclient.go +++ b/politeiad/backend/tlogbe/clients/clients.go @@ -2,17 +2,37 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlogclient +package clients import ( "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/store" ) -// Client provides an API for the plugins to interact with a tlog instance. +// BackendClient provides an API for plugins to interact with backend records. +// This an abridged version of the backend.Backend interface. +type BackendClient interface { + // Check if an unvetted record exists + UnvettedExists(token []byte) bool + + // Check if a vetted record exists + VettedExists(token []byte) bool + + // Get unvetted record + GetUnvetted(token []byte, version string) (*backend.Record, error) + + // Get vetted record + GetVetted(token []byte, version string) (*backend.Record, error) + + // InventoryByStatus returns the record tokens of all records in the + // inventory categorized by MDStatusT + InventoryByStatus() (*backend.InventoryByStatus, error) +} + +// TlogClient provides an API for plugins to interact with a tlog instance. // Plugins are allowed to save, delete, and get plugin data to/from the tlog // backend. Editing plugin data is not allowed. -type Client interface { +type TlogClient interface { // BlobSave saves a BlobEntry to the tlog backend. The BlobEntry // will be encrypted prior to being written to disk if the tlog // instance has an encryption key set. The merkle leaf hash for the diff --git a/politeiad/backend/tlogbe/plugins/comments/comments.go b/politeiad/backend/tlogbe/plugins/comments/comments.go index 43635e77f..bcfdc561b 100644 --- a/politeiad/backend/tlogbe/plugins/comments/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments/comments.go @@ -21,9 +21,9 @@ import ( "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/clients" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/politeiad/backend/tlogbe/store" - "github.com/decred/politeia/politeiad/backend/tlogbe/tlogclient" "github.com/decred/politeia/politeiad/plugins/comments" "github.com/decred/politeia/util" ) @@ -59,7 +59,7 @@ var ( // commentsPlugin satisfies the plugins.Client interface. type commentsPlugin struct { sync.Mutex - tlog tlogclient.Client + tlog clients.TlogClient // dataDir is the comments plugin data directory. The only data // that is stored here is cached data that can be re-created at any @@ -1568,8 +1568,8 @@ func (p *commentsPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (s // Hook executes a plugin hook. // // This function satisfies the plugins.Client interface. -func (p *commentsPlugin) Hook(h plugins.HookT, payload string) error { - log.Tracef("Hook: %v", plugins.Hooks[h]) +func (p *commentsPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { + log.Tracef("Hook: %v %x %v", treeID, token, plugins.Hooks[h]) return nil } @@ -1586,7 +1586,7 @@ func (p *commentsPlugin) Fsck() error { } // New returns a new comments plugin. -func New(tlog tlogclient.Client, settings []backend.PluginSetting, id *identity.FullIdentity, dataDir string) (*commentsPlugin, error) { +func New(tlog clients.TlogClient, settings []backend.PluginSetting, id *identity.FullIdentity, dataDir string) (*commentsPlugin, error) { // Setup comments plugin data dir dataDir = filepath.Join(dataDir, comments.ID) err := os.MkdirAll(dataDir, 0700) diff --git a/politeiad/backend/tlogbe/plugins/comments/comments_test.go b/politeiad/backend/tlogbe/plugins/comments/comments_test.go index a7e4efd0d..7ed067b6c 100644 --- a/politeiad/backend/tlogbe/plugins/comments/comments_test.go +++ b/politeiad/backend/tlogbe/plugins/comments/comments_test.go @@ -15,7 +15,7 @@ import ( "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/tlogclient/testclient" + "github.com/decred/politeia/politeiad/backend/tlogbe/clients" "github.com/decred/politeia/politeiad/plugins/comments" "github.com/google/uuid" ) @@ -47,8 +47,9 @@ func newTestCommentsPlugin(t *testing.T) (*commentsPlugin, func()) { t.Fatal(err) } + // TODO Implement a test clients.TlogClient // Setup tlog client - client := testclient.New() + var tlog clients.TlogClient // Setup plugin identity fid, err := identity.New() @@ -57,7 +58,7 @@ func newTestCommentsPlugin(t *testing.T) (*commentsPlugin, func()) { } // Setup comment plugins - c, err := New(client, []backend.PluginSetting{}, fid, dataDir) + c, err := New(tlog, []backend.PluginSetting{}, fid, dataDir) if err != nil { t.Fatal(err) } diff --git a/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go b/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go index c4b5051e8..17568cb4a 100644 --- a/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go +++ b/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go @@ -612,8 +612,8 @@ func (p *dcrdataPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (st // Hook executes a plugin hook. // // This function satisfies the plugins.Client interface. -func (p *dcrdataPlugin) Hook(h plugins.HookT, payload string) error { - log.Tracef("Hook: %v", plugins.Hooks[h]) +func (p *dcrdataPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { + log.Tracef("Hook: %v %x %v", treeID, token, plugins.Hooks[h]) return nil } diff --git a/politeiad/backend/tlogbe/plugins/pi/pi.go b/politeiad/backend/tlogbe/plugins/pi/pi.go index f60cb00fb..090f1b80c 100644 --- a/politeiad/backend/tlogbe/plugins/pi/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi/pi.go @@ -19,8 +19,8 @@ import ( "github.com/decred/dcrd/chaincfg/v3" "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/clients" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" - "github.com/decred/politeia/politeiad/backend/tlogbe/tlogclient" "github.com/decred/politeia/politeiad/plugins/comments" "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/ticketvote" @@ -33,8 +33,8 @@ var ( // piPlugin satisfies the plugins.Client interface. type piPlugin struct { sync.Mutex - backend backend.Backend - tlog tlogclient.Client + backend clients.BackendClient + tlog clients.TlogClient activeNetParams *chaincfg.Params // dataDir is the pi plugin data directory. The only data that is @@ -227,9 +227,10 @@ func (p *piPlugin) cmdProposals(payload string) (string, error) { return "", nil } -func (p *piPlugin) voteSummary(token string) (*ticketvote.Summary, error) { +func (p *piPlugin) voteSummary(token []byte) (*ticketvote.Summary, error) { + t := hex.EncodeToString(token) s := ticketvote.Summaries{ - Tokens: []string{token}, + Tokens: []string{t}, } b, err := ticketvote.EncodeSummaries(s) if err != nil { @@ -244,7 +245,7 @@ func (p *piPlugin) voteSummary(token string) (*ticketvote.Summary, error) { if err != nil { return nil, err } - summary, ok := sr.Summaries[token] + summary, ok := sr.Summaries[t] if !ok { return nil, fmt.Errorf("proposal not found %v", token) } @@ -413,82 +414,34 @@ func (p *piPlugin) cmdVoteInventory(payload string) (string, error) { return string(reply), nil } -func (p *piPlugin) commentNew(payload string) (string, error) { - // TODO - /* - n, err := comments.DecodeNew([]byte(payload)) - if err != nil { - return "", err - } - - // Verifying the state, token, and that the record exists is also - // done in the comments plugin but we duplicate it here so that the - // vote summary request can complete successfully. - - // Verify state - switch n.State { - case comments.StateUnvetted, comments.StateVetted: - // Allowed; continue - default: - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropStateInvalid), - } - } - - // Verify token - token, err := tokenDecodeAnyLength(n.Token) - if err != nil { - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropTokenInvalid), - } - } - - // Verify record exists - var exists bool - switch n.State { - case comments.StateUnvetted: - exists = p.backend.UnvettedExists(token) - case comments.StateVetted: - exists = p.backend.VettedExists(token) - default: - // Should not happen. State has already been validated. - return "", fmt.Errorf("invalid state %v", n.State) - } - if !exists { - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropNotFound), - } - } +func (p *piPlugin) hookCommentNew(treeID int64, token []byte, payload string) error { + var n comments.New + err := json.Unmarshal([]byte(payload), &n) + if err != nil { + return err + } - // Verify vote status - if n.State == comments.StateVetted { - vs, err := p.voteSummary(n.Token) - if err != nil { - return "", fmt.Errorf("voteSummary: %v", err) - } - switch vs.Status { - case ticketvote.VoteStatusUnauthorized, ticketvote.VoteStatusAuthorized, - ticketvote.VoteStatusStarted: - // Comments are allowed on these vote statuses; continue - default: - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusVoteStatusInvalid), - ErrorContext: []string{"vote has ended; proposal is locked"}, - } - } + // Verify vote status + vs, err := p.voteSummary(treeID, token) + if err != nil { + return fmt.Errorf("voteSummary: %v", err) + } + switch vs.Status { + case ticketvote.VoteStatusUnauthorized, ticketvote.VoteStatusAuthorized, + ticketvote.VoteStatusStarted: + // Comments are allowed on these vote statuses; continue + default: + return "", backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusVoteStatusInvalid), + ErrorContext: "vote has ended; proposal is locked", } + } - // Send plugin command - return p.backend.Plugin(comments.ID, comments.CmdNew, "", payload) - */ - return "", nil + return nil } -func (p *piPlugin) commentDel(payload string) (string, error) { +func (p *piPlugin) commentDel(payload string) error { // TODO /* d, err := comments.DecodeDel([]byte(payload)) @@ -560,10 +513,10 @@ func (p *piPlugin) commentDel(payload string) (string, error) { // Send plugin command return p.backend.Plugin(comments.ID, comments.CmdDel, "", payload) */ - return "", nil + return nil } -func (p *piPlugin) commentVote(payload string) (string, error) { +func (p *piPlugin) commentVote(payload string) error { // TODO /* v, err := comments.DecodeVote([]byte(payload)) @@ -649,10 +602,10 @@ func (p *piPlugin) commentVote(payload string) (string, error) { // Send plugin command return p.backend.Plugin(comments.ID, comments.CmdVote, "", payload) */ - return "", nil + return nil } -func (p *piPlugin) ticketVoteStart(payload string) (string, error) { +func (p *piPlugin) ticketVoteStart(payload string) error { // TODO /* // Decode payload @@ -803,34 +756,36 @@ func (p *piPlugin) ticketVoteStart(payload string) (string, error) { // ticketvote plugin. return p.backend.Plugin(ticketvote.ID, ticketvote.CmdStart, "", payload) */ - return "", nil + return nil } -func (p *piPlugin) passThrough(pluginID, cmd, payload string) (string, error) { - switch pluginID { +func (p *piPlugin) hookPluginPre(payload string) error { + // Decode payload + var hpp plugins.HookPluginPre + err := json.Unmarshal([]byte(payload), &hpp) + if err != nil { + return err + } + + // Call plugin hook + switch hpp.PluginID { case comments.ID: - switch cmd { + switch hpp.Cmd { case comments.CmdNew: - return p.commentNew(payload) - case comments.CmdDel: - return p.commentDel(payload) - case comments.CmdVote: - return p.commentVote(payload) - default: - return "", fmt.Errorf("invalid %v plugin cmd '%v'", - pluginID, cmd) + return p.hookCommentNew(hpp.Payload) + // case comments.CmdDel: + // return p.commentDel(hpp.Payload) + // case comments.CmdVote: + // return p.commentVote(hpp.Payload) } case ticketvote.ID: - switch cmd { - case ticketvote.CmdStart: - return p.ticketVoteStart(payload) - default: - return "", fmt.Errorf("invalid %v plugin cmd '%v'", - pluginID, cmd) + switch hpp.Cmd { + // case ticketvote.CmdStart: + // return p.ticketVoteStart(hpp.Payload) } - default: - return "", fmt.Errorf("invalid plugin id '%v'", pluginID) } + + return nil } func (p *piPlugin) hookNewRecordPre(payload string) error { @@ -1175,8 +1130,8 @@ func (p *piPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, // Hook executes a plugin hook. // // This function satisfies the plugins.Client interface. -func (p *piPlugin) Hook(h plugins.HookT, payload string) error { - log.Tracef("Hook: %v %v", plugins.Hooks[h], payload) +func (p *piPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { + log.Tracef("Hook: %v %x %v", treeID, plugins.Hooks[h]) switch h { case plugins.HookTypeNewRecordPre: @@ -1187,6 +1142,8 @@ func (p *piPlugin) Hook(h plugins.HookT, payload string) error { return p.hookEditRecordPre(payload) case plugins.HookTypeSetRecordStatusPost: return p.hookSetRecordStatusPost(payload) + case plugins.HookTypePluginPre: + return p.hookPluginPre(payload) } return nil @@ -1203,7 +1160,7 @@ func (p *piPlugin) Fsck() error { return nil } -func New(backend backend.Backend, tlog tlogclient.Client, settings []backend.PluginSetting, dataDir string, activeNetParams *chaincfg.Params) (*piPlugin, error) { +func New(backend clients.BackendClient, tlog clients.TlogClient, settings []backend.PluginSetting, dataDir string, activeNetParams *chaincfg.Params) (*piPlugin, error) { // Create plugin data directory dataDir = filepath.Join(dataDir, pi.ID) err := os.MkdirAll(dataDir, 0700) diff --git a/politeiad/backend/tlogbe/plugins/pi/pi_test.go b/politeiad/backend/tlogbe/plugins/pi/pi_test.go deleted file mode 100644 index bea634995..000000000 --- a/politeiad/backend/tlogbe/plugins/pi/pi_test.go +++ /dev/null @@ -1,343 +0,0 @@ -// Copyright (c) 2020-2021 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package pi - -/* -func newTestPiPlugin(t *testing.T) (*piPlugin, *tlogBackend, func()) { - t.Helper() - - tlogBackend, cleanup := newTestTlogBackend(t) - - settings := []backend.PluginSetting{{ - Key: pluginSettingDataDir, - Value: tlogBackend.dataDir, - }} - - piPlugin, err := newPiPlugin(tlogBackend, newBackendClient(tlogBackend), - settings, tlogBackend.activeNetParams) - if err != nil { - t.Fatalf("newPiPlugin: %v", err) - } - - return piPlugin, tlogBackend, cleanup -} - -func TestCommentNew(t *testing.T) { - piPlugin, tlogBackend, cleanup := newTestPiPlugin(t) - defer cleanup() - - // Register comments plugin - id, err := identity.New() - if err != nil { - t.Fatal(err) - } - settings := []backend.PluginSetting{{ - Key: pluginSettingDataDir, - Value: tlogBackend.dataDir, - }} - tlogBackend.RegisterPlugin(backend.Plugin{ - ID: comments.ID, - Settings: settings, - Identity: id, - }) - - // New record - md := []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs := []backend.File{ - newBackendFile(t, "index.md"), - } - rec, err := tlogBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - - // Helpers - comment := "random comment" - tokenRandom := hex.EncodeToString(tokenFromTreeID(123)) - parentID := uint32(0) - - uid, err := identity.New() - if err != nil { - t.Fatal(err) - } - - // Setup comment new pi plugin tests - var tests = []struct { - description string - payload comments.New - wantErr error - }{ - { - "invalid comment state", - comments.New{ - UserID: uuid.New().String(), - State: comments.StateInvalid, - Token: rec.Token, - ParentID: parentID, - Comment: comment, - PublicKey: uid.Public.String(), - Signature: commentSignature(t, uid, comments.StateInvalid, - rec.Token, comment, parentID), - }, - backend.PluginUserError{ - ErrorCode: int(pi.ErrorStatusPropStateInvalid), - }, - }, - { - "invalid token", - comments.New{ - UserID: uuid.New().String(), - State: comments.StateUnvetted, - Token: "invalid", - ParentID: parentID, - Comment: comment, - PublicKey: uid.Public.String(), - Signature: commentSignature(t, uid, comments.StateUnvetted, - rec.Token, comment, parentID), - }, - backend.PluginUserError{ - ErrorCode: int(pi.ErrorStatusPropTokenInvalid), - }, - }, - { - "record not found", - comments.New{ - UserID: uuid.New().String(), - State: comments.StateUnvetted, - Token: tokenRandom, - ParentID: parentID, - Comment: comment, - PublicKey: uid.Public.String(), - Signature: commentSignature(t, uid, comments.StateUnvetted, - tokenRandom, comment, parentID), - }, - backend.PluginUserError{ - ErrorCode: int(pi.ErrorStatusPropNotFound), - }, - }, - // TODO: bad vote status test case. waiting on plugin architecture - // refactor - { - "success", - comments.New{ - UserID: uuid.New().String(), - State: comments.StateUnvetted, - Token: rec.Token, - ParentID: parentID, - Comment: comment, - PublicKey: uid.Public.String(), - Signature: commentSignature(t, uid, comments.StateUnvetted, - rec.Token, comment, parentID), - }, - nil, - }, - } - - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - // New Comment - ncEncoded, err := comments.EncodeNew(test.payload) - if err != nil { - t.Error(err) - } - - // Execute plugin command - _, err = piPlugin.commentNew(string(ncEncoded)) - - // Parse plugin user error - var pluginUserError backend.PluginUserError - if errors.As(err, &pluginUserError) { - if test.wantErr == nil { - t.Errorf("got error %v, want nil2", err) - return - } - wantErr := test.wantErr.(backend.PluginUserError) - if pluginUserError.ErrorCode != wantErr.ErrorCode { - t.Errorf("got error %v, want %v", - pluginUserError.ErrorCode, - wantErr.ErrorCode) - } - return - } - - // Expectations not met - if err != test.wantErr { - t.Errorf("got error %v, want %v", err, test.wantErr) - } - }) - } -} - -func TestCommentDel(t *testing.T) { - piPlugin, tlogBackend, cleanup := newTestPiPlugin(t) - defer cleanup() - - // Register comments plugin - id, err := identity.New() - if err != nil { - t.Fatal(err) - } - settings := []backend.PluginSetting{{ - Key: pluginSettingDataDir, - Value: tlogBackend.dataDir, - }} - tlogBackend.RegisterPlugin(backend.Plugin{ - ID: comments.ID, - Settings: settings, - Identity: id, - }) - - // New record - md := []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs := []backend.File{ - newBackendFile(t, "index.md"), - } - rec, err := tlogBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - - // Helpers - comment := "random comment" - reason := "random reason" - tokenRandom := hex.EncodeToString(tokenFromTreeID(123)) - parentID := uint32(0) - - uid, err := identity.New() - if err != nil { - t.Fatal(err) - } - - // New comment - ncEncoded, err := comments.EncodeNew( - comments.New{ - UserID: uuid.New().String(), - State: comments.StateUnvetted, - Token: rec.Token, - ParentID: parentID, - Comment: comment, - PublicKey: uid.Public.String(), - Signature: commentSignature(t, uid, comments.StateUnvetted, - rec.Token, comment, parentID), - }, - ) - if err != nil { - t.Fatal(err) - } - reply, err := piPlugin.commentNew(string(ncEncoded)) - if err != nil { - t.Fatal(err) - } - nr, err := comments.DecodeNewReply([]byte(reply)) - if err != nil { - t.Fatal(err) - } - - // Setup comment del pi plugin tests - var tests = []struct { - description string - payload comments.Del - wantErr error - }{ - { - "invalid comment state", - comments.Del{ - State: comments.StateInvalid, - Token: rec.Token, - CommentID: nr.Comment.CommentID, - Reason: reason, - PublicKey: uid.Public.String(), - Signature: commentSignature(t, uid, comments.StateInvalid, - rec.Token, reason, nr.Comment.CommentID), - }, - backend.PluginUserError{ - ErrorCode: int(pi.ErrorStatusPropStateInvalid), - }, - }, - { - "invalid token", - comments.Del{ - State: comments.StateUnvetted, - Token: "invalid", - CommentID: nr.Comment.CommentID, - Reason: reason, - PublicKey: uid.Public.String(), - Signature: commentSignature(t, uid, comments.StateUnvetted, - "invalid", reason, nr.Comment.CommentID), - }, - backend.PluginUserError{ - ErrorCode: int(pi.ErrorStatusPropTokenInvalid), - }, - }, - { - "proposal not found", - comments.Del{ - State: comments.StateUnvetted, - Token: tokenRandom, - CommentID: nr.Comment.CommentID, - Reason: reason, - PublicKey: uid.Public.String(), - Signature: commentSignature(t, uid, comments.StateUnvetted, - tokenRandom, reason, nr.Comment.CommentID), - }, - backend.PluginUserError{ - ErrorCode: int(pi.ErrorStatusPropNotFound), - }, - }, - { - "success", - comments.Del{ - State: comments.StateUnvetted, - Token: rec.Token, - CommentID: nr.Comment.CommentID, - Reason: reason, - PublicKey: uid.Public.String(), - Signature: commentSignature(t, uid, comments.StateUnvetted, - rec.Token, reason, nr.Comment.CommentID), - }, - nil, - }, - } - - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - // New Comment - dcEncoded, err := comments.EncodeDel(test.payload) - if err != nil { - t.Error(err) - } - - // Execute plugin command - _, err = piPlugin.commentDel(string(dcEncoded)) - - // Parse plugin user error - var pluginUserError backend.PluginUserError - if errors.As(err, &pluginUserError) { - if test.wantErr == nil { - t.Errorf("got error %v, want nil", err) - return - } - wantErr := test.wantErr.(backend.PluginUserError) - if pluginUserError.ErrorCode != wantErr.ErrorCode { - t.Errorf("got error %v, want %v", - pluginUserError.ErrorCode, - wantErr.ErrorCode) - } - return - } - - // Expectations not met - if err != test.wantErr { - t.Errorf("got error %v, want %v", err, test.wantErr) - } - }) - } - -} -*/ diff --git a/politeiad/backend/tlogbe/plugins/plugins.go b/politeiad/backend/tlogbe/plugins/plugins.go index 276b6f73f..3799d9314 100644 --- a/politeiad/backend/tlogbe/plugins/plugins.go +++ b/politeiad/backend/tlogbe/plugins/plugins.go @@ -39,22 +39,46 @@ var ( } ) -// HookNewRecord is the payload for the new record hooks. -type HookNewRecord struct { +// RecordStateT represents a record state. The record state is included in all +// hook payloads so that a plugin has the ability to implement different +// behaviors for different states. +type RecordStateT int + +const ( + // RecordStateInvalid is an invalid record state. + RecordStateInvalid RecordStateT = 0 + + // RecordStateUnvetted represents an unvetted record. + RecordStateUnvetted RecordStateT = 1 + + // RecordStateVetted represents a vetted record. + RecordStateVetted RecordStateT = 2 +) + +// HookNewRecordPre is the payload for the pre new record hook. The record +// state is not inlcuded since all new records will have a record state of +// unvetted. +type HookNewRecordPre struct { Metadata []backend.MetadataStream `json:"metadata"` Files []backend.File `json:"files"` +} - // RecordMetadata will only be present on the post new record hook. - // This is because the record metadata requires the creation of a - // trillian tree and the pre new record hook should execute before - // any politeiad state is changed in case of validation errors. - RecordMetadata *backend.RecordMetadata `json:"recordmetadata"` +// HookNewRecordPost is the payload for the post new record hook. The record +// state is not inlcuded since all new records will have a record state of +// unvetted. RecordMetadata is only be present on the post new record hook +// since the record metadata requires the creation of a trillian tree and the +// pre new record hook should execute before any politeiad state is changed in +// case of validation errors. +type HookNewRecordPost struct { + Metadata []backend.MetadataStream `json:"metadata"` + Files []backend.File `json:"files"` + RecordMetadata *backend.RecordMetadata `json:"recordmetadata"` } -// HookEditRecord is the payload for the edit record hooks. +// HookEditRecord is the payload for the pre and post edit record hooks. type HookEditRecord struct { - // Current record - Current backend.Record `json:"record"` + State RecordStateT `json:"state"` + Current backend.Record `json:"record"` // Current record // Updated fields RecordMetadata backend.RecordMetadata `json:"recordmetadata"` @@ -64,20 +88,21 @@ type HookEditRecord struct { FilesDel []string `json:"filesdel"` } -// HookEditMetadata is the payload for the edit metadata hooks. +// HookEditMetadata is the payload for the pre and post edit metadata hooks. type HookEditMetadata struct { - // Current record - Current backend.Record `json:"record"` + State RecordStateT `json:"state"` + Current backend.Record `json:"record"` // Current record // Updated fields MDAppend []backend.MetadataStream `json:"mdappend"` MDOverwrite []backend.MetadataStream `json:"mdoverwrite"` } -// HookSetRecordStatus is the payload for the set record status hooks. +// HookSetRecordStatus is the payload for the pre and post set record status +// hooks. type HookSetRecordStatus struct { - // Current record - Current backend.Record `json:"record"` + State RecordStateT `json:"state"` + Current backend.Record `json:"record"` // Current record // Updated fields RecordMetadata backend.RecordMetadata `json:"recordmetadata"` @@ -85,19 +110,23 @@ type HookSetRecordStatus struct { MDOverwrite []backend.MetadataStream `json:"mdoverwrite"` } -// HookPluginPre is the payload for the plugin pre hook. +// HookPluginPre is the payload for the pre plugin hook. type HookPluginPre struct { - PluginID string `json:"pluginid"` - Cmd string `json:"cmd"` - Payload string `json:"payload"` + State RecordStateT `json:"state"` + Token string `json:"token"` + PluginID string `json:"pluginid"` + Cmd string `json:"cmd"` + Payload string `json:"payload"` } -// HookPluginPost is the payload for the plugin post hook. +// HookPluginPost is the payload for the post plugin hook. The post plugin hook +// includes the plugin reply. type HookPluginPost struct { - PluginID string `json:"pluginid"` - Cmd string `json:"cmd"` - Payload string `json:"payload"` - Reply string `json:"reply"` + State RecordStateT `json:"state"` + PluginID string `json:"pluginid"` + Cmd string `json:"cmd"` + Payload string `json:"payload"` + Reply string `json:"reply"` } // Client provides an API for a tlog instance to use when interacting with a @@ -110,7 +139,7 @@ type Client interface { Cmd(treeID int64, token []byte, cmd, payload string) (string, error) // Hook executes a plugin hook. - Hook(h HookT, payload string) error + Hook(treeID int64, token []byte, h HookT, payload string) error // Fsck performs a plugin file system check. Fsck() error diff --git a/politeiad/backend/tlogbe/tlog/plugin.go b/politeiad/backend/tlogbe/tlog/plugin.go index 9d8decee4..9f95c333d 100644 --- a/politeiad/backend/tlogbe/tlog/plugin.go +++ b/politeiad/backend/tlogbe/tlog/plugin.go @@ -13,10 +13,8 @@ import ( "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/comments" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/dcrdata" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/pi" cmplugin "github.com/decred/politeia/politeiad/plugins/comments" ddplugin "github.com/decred/politeia/politeiad/plugins/dcrdata" - piplugin "github.com/decred/politeia/politeiad/plugins/pi" ) const ( @@ -73,12 +71,12 @@ func (t *Tlog) PluginRegister(b backend.Backend, p backend.Plugin) error { if err != nil { return err } - case piplugin.ID: - client, err = pi.New(b, t, p.Settings, dataDir, t.activeNetParams) - if err != nil { - return err - } /* + case piplugin.ID: + client, err = pi.New(b, t, p.Settings, dataDir, t.activeNetParams) + if err != nil { + return err + } case ticketvote.ID: client, err = newTicketVotePlugin(t, newBackendClient(t), p.Settings, p.Identity, t.activeNetParams) @@ -113,13 +111,13 @@ func (t *Tlog) PluginSetup(pluginID string) error { return p.client.Setup() } -func (t *Tlog) PluginHookPre(h plugins.HookT, payload string) error { - log.Tracef("%v PluginHookPre: %v %v", t.id, h, payload) +func (t *Tlog) PluginHookPre(treeID int64, token []byte, h plugins.HookT, payload string) error { + log.Tracef("%v PluginHookPre: %v %x %v", t.id, plugins.Hooks[h]) // Pass hook event and payload to each plugin for _, v := range t.pluginIDs() { p, _ := t.plugin(v) - err := p.client.Hook(h, payload) + err := p.client.Hook(treeID, token, h, payload) if err != nil { var e backend.PluginUserError if errors.As(err, &e) { @@ -132,8 +130,8 @@ func (t *Tlog) PluginHookPre(h plugins.HookT, payload string) error { return nil } -func (t *Tlog) PluginHookPost(h plugins.HookT, payload string) { - log.Tracef("%v PluginHookPost: %v %v", t.id, h, payload) +func (t *Tlog) PluginHookPost(treeID int64, token []byte, h plugins.HookT, payload string) { + log.Tracef("%v PluginHookPost: %v %x %v", t.id, plugins.Hooks[h]) // Pass hook event and payload to each plugin for _, v := range t.pluginIDs() { @@ -142,13 +140,13 @@ func (t *Tlog) PluginHookPost(h plugins.HookT, payload string) { log.Errorf("%v PluginHookPost: plugin not found %v", t.id, v) continue } - err := p.client.Hook(h, payload) + err := p.client.Hook(treeID, token, h, payload) if err != nil { // This is the post plugin hook so the data has already been // saved to tlog. We do not have the ability to unwind. Log // the error and continue. - log.Criticalf("%v PluginHookPost %v %v: %v: %v", - t.id, v, h, err, payload) + log.Criticalf("%v PluginHookPost %v %v %v %x %v: %v", + t.id, v, treeID, token, h, err, payload) continue } } diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 404491df5..1cf781e9a 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -658,15 +658,16 @@ func (t *tlogBackend) New(metadata []backend.MetadataStream, files []backend.Fil } // Call pre plugin hooks - hnr := plugins.HookNewRecord{ + pre := plugins.HookNewRecordPre{ Metadata: metadata, Files: files, } - b, err := json.Marshal(hnr) + b, err := json.Marshal(pre) if err != nil { return nil, err } - err = t.unvetted.PluginHookPre(plugins.HookTypeNewRecordPre, string(b)) + err = t.unvetted.PluginHookPre(0, []byte{}, + plugins.HookTypeNewRecordPre, string(b)) if err != nil { return nil, err } @@ -710,16 +711,17 @@ func (t *tlogBackend) New(metadata []backend.MetadataStream, files []backend.Fil } // Call post plugin hooks - hnr = plugins.HookNewRecord{ + post := plugins.HookNewRecordPost{ Metadata: metadata, Files: files, RecordMetadata: rm, } - b, err = json.Marshal(hnr) + b, err = json.Marshal(post) if err != nil { return nil, err } - t.unvetted.PluginHookPost(plugins.HookTypeNewRecordPost, string(b)) + t.unvetted.PluginHookPost(treeID, token, + plugins.HookTypeNewRecordPost, string(b)) // Update the inventory cache t.inventoryAdd(stateUnvetted, token, backend.MDStatusUnvetted) @@ -789,6 +791,7 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ // Call pre plugin hooks her := plugins.HookEditRecord{ + State: plugins.RecordStateUnvetted, Current: *r, RecordMetadata: *recordMD, MDAppend: mdAppend, @@ -800,7 +803,8 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ if err != nil { return nil, err } - err = t.unvetted.PluginHookPre(plugins.HookTypeEditRecordPre, string(b)) + err = t.unvetted.PluginHookPre(treeID, token, + plugins.HookTypeEditRecordPre, string(b)) if err != nil { return nil, err } @@ -818,7 +822,8 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ } // Call post plugin hooks - t.unvetted.PluginHookPost(plugins.HookTypeEditRecordPost, string(b)) + t.unvetted.PluginHookPost(treeID, token, + plugins.HookTypeEditRecordPost, string(b)) // Return updated record r, err = t.unvetted.RecordLatest(treeID) @@ -891,6 +896,7 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b // Call pre plugin hooks her := plugins.HookEditRecord{ + State: plugins.RecordStateVetted, Current: *r, RecordMetadata: *recordMD, MDAppend: mdAppend, @@ -902,7 +908,8 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b if err != nil { return nil, err } - err = t.vetted.PluginHookPre(plugins.HookTypeEditRecordPre, string(b)) + err = t.vetted.PluginHookPre(treeID, token, + plugins.HookTypeEditRecordPre, string(b)) if err != nil { return nil, err } @@ -920,7 +927,8 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b } // Call post plugin hooks - t.vetted.PluginHookPost(plugins.HookTypeEditRecordPost, string(b)) + t.vetted.PluginHookPost(treeID, token, + plugins.HookTypeEditRecordPost, string(b)) // Return updated record r, err = t.vetted.RecordLatest(treeID) @@ -985,6 +993,7 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite // Call pre plugin hooks hem := plugins.HookEditMetadata{ + State: plugins.RecordStateUnvetted, Current: *r, MDAppend: mdAppend, MDOverwrite: mdOverwrite, @@ -993,7 +1002,8 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite if err != nil { return err } - err = t.unvetted.PluginHookPre(plugins.HookTypeEditMetadataPre, string(b)) + err = t.unvetted.PluginHookPre(treeID, token, + plugins.HookTypeEditMetadataPre, string(b)) if err != nil { return err } @@ -1014,7 +1024,8 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite } // Call post plugin hooks - t.unvetted.PluginHookPost(plugins.HookTypeEditMetadataPost, string(b)) + t.unvetted.PluginHookPost(treeID, token, + plugins.HookTypeEditMetadataPost, string(b)) return nil } @@ -1075,6 +1086,7 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ // Call pre plugin hooks hem := plugins.HookEditMetadata{ + State: plugins.RecordStateVetted, Current: *r, MDAppend: mdAppend, MDOverwrite: mdOverwrite, @@ -1083,7 +1095,8 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ if err != nil { return err } - err = t.vetted.PluginHookPre(plugins.HookTypeEditMetadataPre, string(b)) + err = t.vetted.PluginHookPre(treeID, token, + plugins.HookTypeEditMetadataPre, string(b)) if err != nil { return err } @@ -1104,7 +1117,8 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ } // Call post plugin hooks - t.vetted.PluginHookPost(plugins.HookTypeEditMetadataPost, string(b)) + t.vetted.PluginHookPost(treeID, token, + plugins.HookTypeEditMetadataPost, string(b)) return nil } @@ -1137,125 +1151,6 @@ func (t *tlogBackend) VettedExists(token []byte) bool { return ok } -// This function satisfies the backend.Backend interface. -func (t *tlogBackend) GetUnvetted(token []byte, version string) (*backend.Record, error) { - log.Tracef("GetUnvetted: %x %v", token, version) - - if t.isShutdown() { - return nil, backend.ErrShutdown - } - - treeID := t.unvettedTreeIDFromToken(token) - var v uint32 - if version != "" { - u, err := strconv.ParseUint(version, 10, 64) - if err != nil { - return nil, backend.ErrRecordNotFound - } - v = uint32(u) - } - - // Verify record exists and is unvetted - if !t.UnvettedExists(token) { - return nil, backend.ErrRecordNotFound - } - - // Get unvetted record - r, err := t.unvetted.Record(treeID, v) - if err != nil { - return nil, fmt.Errorf("unvetted record: %v", err) - } - - return r, nil -} - -// This function satisfies the backend.Backend interface. -func (t *tlogBackend) GetVetted(token []byte, version string) (*backend.Record, error) { - log.Tracef("GetVetted: %x %v", token, version) - - if t.isShutdown() { - return nil, backend.ErrShutdown - } - - // Get tree ID - treeID, ok := t.vettedTreeIDFromToken(token) - if !ok { - return nil, backend.ErrRecordNotFound - } - - // Parse version - var v uint32 - if version != "" { - u, err := strconv.ParseUint(version, 10, 64) - if err != nil { - return nil, backend.ErrRecordNotFound - } - v = uint32(u) - } - - r, err := t.vetted.Record(treeID, v) - if err != nil { - return nil, err - } - - return r, nil -} - -// This function satisfies the backend.Backend interface. -func (t *tlogBackend) GetUnvettedTimestamps(token []byte, version string) (*backend.RecordTimestamps, error) { - log.Tracef("GetUnvettedTimestamps: %x %v", token, version) - - if t.isShutdown() { - return nil, backend.ErrShutdown - } - - treeID := t.unvettedTreeIDFromToken(token) - var v uint32 - if version != "" { - u, err := strconv.ParseUint(version, 10, 64) - if err != nil { - return nil, backend.ErrRecordNotFound - } - v = uint32(u) - } - - // Verify record exists and is unvetted - if !t.UnvettedExists(token) { - return nil, backend.ErrRecordNotFound - } - - // Get timestamps - return t.unvetted.RecordTimestamps(treeID, v, token) -} - -// This function satisfies the backend.Backend interface. -func (t *tlogBackend) GetVettedTimestamps(token []byte, version string) (*backend.RecordTimestamps, error) { - log.Tracef("GetVettedTimestamps: %x %v", token, version) - - if t.isShutdown() { - return nil, backend.ErrShutdown - } - - // Get tree ID - treeID, ok := t.vettedTreeIDFromToken(token) - if !ok { - return nil, backend.ErrRecordNotFound - } - - // Parse version - var v uint32 - if version != "" { - u, err := strconv.ParseUint(version, 10, 64) - if err != nil { - return nil, backend.ErrRecordNotFound - } - v = uint32(u) - } - - // Get timestamps - return t.vetted.RecordTimestamps(treeID, v, token) -} - // This function must be called WITH the unvetted lock held. func (t *tlogBackend) unvettedPublish(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { // Create a vetted tree @@ -1362,6 +1257,7 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, // Call pre plugin hooks hsrs := plugins.HookSetRecordStatus{ + State: plugins.RecordStateUnvetted, Current: *r, RecordMetadata: rm, MDAppend: mdAppend, @@ -1371,7 +1267,8 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, if err != nil { return nil, err } - err = t.unvetted.PluginHookPre(plugins.HookTypeSetRecordStatusPre, string(b)) + err = t.unvetted.PluginHookPre(treeID, token, + plugins.HookTypeSetRecordStatusPre, string(b)) if err != nil { return nil, err } @@ -1394,7 +1291,8 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, } // Call post plugin hooks - t.unvetted.PluginHookPost(plugins.HookTypeSetRecordStatusPost, string(b)) + t.unvetted.PluginHookPost(treeID, token, + plugins.HookTypeSetRecordStatusPost, string(b)) log.Debugf("Status change %x from %v (%v) to %v (%v)", token, backend.MDStatus[currStatus], currStatus, @@ -1514,6 +1412,7 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md // Call pre plugin hooks srs := plugins.HookSetRecordStatus{ + State: plugins.RecordStateVetted, Current: *r, RecordMetadata: rm, MDAppend: mdAppend, @@ -1523,7 +1422,8 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md if err != nil { return nil, err } - err = t.vetted.PluginHookPre(plugins.HookTypeSetRecordStatusPre, string(b)) + err = t.vetted.PluginHookPre(treeID, token, + plugins.HookTypeSetRecordStatusPre, string(b)) if err != nil { return nil, err } @@ -1546,7 +1446,8 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md } // Call post plugin hooks - t.vetted.PluginHookPost(plugins.HookTypeSetRecordStatusPost, string(b)) + t.vetted.PluginHookPost(treeID, token, + plugins.HookTypeSetRecordStatusPost, string(b)) // Update inventory cache t.inventoryUpdate(stateVetted, token, currStatus, status) @@ -1564,6 +1465,125 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md return r, nil } +// This function satisfies the backend.Backend interface. +func (t *tlogBackend) GetUnvetted(token []byte, version string) (*backend.Record, error) { + log.Tracef("GetUnvetted: %x %v", token, version) + + if t.isShutdown() { + return nil, backend.ErrShutdown + } + + treeID := t.unvettedTreeIDFromToken(token) + var v uint32 + if version != "" { + u, err := strconv.ParseUint(version, 10, 64) + if err != nil { + return nil, backend.ErrRecordNotFound + } + v = uint32(u) + } + + // Verify record exists and is unvetted + if !t.UnvettedExists(token) { + return nil, backend.ErrRecordNotFound + } + + // Get unvetted record + r, err := t.unvetted.Record(treeID, v) + if err != nil { + return nil, fmt.Errorf("unvetted record: %v", err) + } + + return r, nil +} + +// This function satisfies the backend.Backend interface. +func (t *tlogBackend) GetVetted(token []byte, version string) (*backend.Record, error) { + log.Tracef("GetVetted: %x %v", token, version) + + if t.isShutdown() { + return nil, backend.ErrShutdown + } + + // Get tree ID + treeID, ok := t.vettedTreeIDFromToken(token) + if !ok { + return nil, backend.ErrRecordNotFound + } + + // Parse version + var v uint32 + if version != "" { + u, err := strconv.ParseUint(version, 10, 64) + if err != nil { + return nil, backend.ErrRecordNotFound + } + v = uint32(u) + } + + r, err := t.vetted.Record(treeID, v) + if err != nil { + return nil, err + } + + return r, nil +} + +// This function satisfies the backend.Backend interface. +func (t *tlogBackend) GetUnvettedTimestamps(token []byte, version string) (*backend.RecordTimestamps, error) { + log.Tracef("GetUnvettedTimestamps: %x %v", token, version) + + if t.isShutdown() { + return nil, backend.ErrShutdown + } + + treeID := t.unvettedTreeIDFromToken(token) + var v uint32 + if version != "" { + u, err := strconv.ParseUint(version, 10, 64) + if err != nil { + return nil, backend.ErrRecordNotFound + } + v = uint32(u) + } + + // Verify record exists and is unvetted + if !t.UnvettedExists(token) { + return nil, backend.ErrRecordNotFound + } + + // Get timestamps + return t.unvetted.RecordTimestamps(treeID, v, token) +} + +// This function satisfies the backend.Backend interface. +func (t *tlogBackend) GetVettedTimestamps(token []byte, version string) (*backend.RecordTimestamps, error) { + log.Tracef("GetVettedTimestamps: %x %v", token, version) + + if t.isShutdown() { + return nil, backend.ErrShutdown + } + + // Get tree ID + treeID, ok := t.vettedTreeIDFromToken(token) + if !ok { + return nil, backend.ErrRecordNotFound + } + + // Parse version + var v uint32 + if version != "" { + u, err := strconv.ParseUint(version, 10, 64) + if err != nil { + return nil, backend.ErrRecordNotFound + } + v = uint32(u) + } + + // Get timestamps + return t.vetted.RecordTimestamps(treeID, v, token) +} + // InventoryByStatus returns the record tokens of all records in the inventory // categorized by MDStatusT. // @@ -1652,6 +1672,7 @@ func (t *tlogBackend) UnvettedPlugin(token []byte, pluginID, cmd, payload string // Call pre plugin hooks hp := plugins.HookPluginPre{ + State: plugins.RecordStateUnvetted, PluginID: pluginID, Cmd: cmd, Payload: payload, @@ -1660,7 +1681,8 @@ func (t *tlogBackend) UnvettedPlugin(token []byte, pluginID, cmd, payload string if err != nil { return "", err } - err = t.unvetted.PluginHookPre(plugins.HookTypePluginPre, string(b)) + err = t.unvetted.PluginHookPre(treeID, token, + plugins.HookTypePluginPre, string(b)) if err != nil { return "", err } @@ -1672,6 +1694,7 @@ func (t *tlogBackend) UnvettedPlugin(token []byte, pluginID, cmd, payload string // Call post plugin hooks hpp := plugins.HookPluginPost{ + State: plugins.RecordStateUnvetted, PluginID: pluginID, Cmd: cmd, Payload: payload, @@ -1681,7 +1704,8 @@ func (t *tlogBackend) UnvettedPlugin(token []byte, pluginID, cmd, payload string if err != nil { return "", err } - t.unvetted.PluginHookPost(plugins.HookTypePluginPost, string(b)) + t.unvetted.PluginHookPost(treeID, token, + plugins.HookTypePluginPost, string(b)) return reply, nil } @@ -1704,6 +1728,7 @@ func (t *tlogBackend) VettedPlugin(token []byte, pluginID, cmd, payload string) // Call pre plugin hooks hp := plugins.HookPluginPre{ + State: plugins.RecordStateVetted, PluginID: pluginID, Cmd: cmd, Payload: payload, @@ -1712,7 +1737,8 @@ func (t *tlogBackend) VettedPlugin(token []byte, pluginID, cmd, payload string) if err != nil { return "", err } - err = t.vetted.PluginHookPre(plugins.HookTypePluginPre, string(b)) + err = t.vetted.PluginHookPre(treeID, token, + plugins.HookTypePluginPre, string(b)) if err != nil { return "", err } @@ -1724,6 +1750,7 @@ func (t *tlogBackend) VettedPlugin(token []byte, pluginID, cmd, payload string) // Call post plugin hooks hpp := plugins.HookPluginPost{ + State: plugins.RecordStateVetted, PluginID: pluginID, Cmd: cmd, Payload: payload, @@ -1733,7 +1760,8 @@ func (t *tlogBackend) VettedPlugin(token []byte, pluginID, cmd, payload string) if err != nil { return "", err } - t.vetted.PluginHookPost(plugins.HookTypePluginPost, string(b)) + t.vetted.PluginHookPost(treeID, token, + plugins.HookTypePluginPost, string(b)) return reply, nil } diff --git a/politeiad/backend/tlogbe/tlogclient/testclient/testclient.go b/politeiad/backend/tlogbe/tlogclient/testclient/testclient.go deleted file mode 100644 index b60cbb2b4..000000000 --- a/politeiad/backend/tlogbe/tlogclient/testclient/testclient.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) 2020-2021 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package testclient - -import ( - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/store" -) - -// testClient implements the tlogclient.Client interface and is used for -// plugin testing. -type testClient struct{} - -// BlobSave saves a BlobEntry to the tlog backend. The merkle leaf hash for the -// blob will be returned. This merkle leaf hash can be though of as the blob ID -// and can be used to retrieve or delete the blob. -func (t *testClient) BlobSave(treeID int64, keyPrefix string, be store.BlobEntry) ([]byte, error) { - return nil, nil -} - -// BlobsDel deletes the blobs that correspond to the provided merkle leaf -// hashes. -func (t *testClient) BlobsDel(treeID int64, merkles [][]byte) error { - return nil -} - -// BlobsByMerkle returns the blobs with the provided merkle leaf hashes. If a -// blob does not exist it will not be included in the returned map. -func (t *testClient) BlobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, error) { - return nil, nil -} - -// BlobsByKeyPrefix returns all blobs that match the key prefix. -func (t *testClient) BlobsByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) { - return nil, nil -} - -// MerklesByKeyPrefix returns the merkle leaf hashes for all blobs that match -// the key prefix. -func (t *testClient) MerklesByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) { - return nil, nil -} - -// Timestamp returns the timestamp for the blob that correpsonds to the merkle -// leaf hash. -func (t *testClient) Timestamp(treeID int64, merkle []byte) (*backend.Timestamp, error) { - return nil, nil -} - -// New returns a new testClient. -func New() *testClient { - return &testClient{} -} diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index 134cb01d0..dbccd9633 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -41,8 +41,6 @@ const ( // TODO number error codes and add human readable error messages ErrorStatusInvalid ErrorStatusT = 0 ErrorStatusPageSizeExceeded ErrorStatusT = iota - ErrorStatusPropNotFound - ErrorStatusPropStateInvalid ErrorStatusPropTokenInvalid ErrorStatusPropStatusInvalid ErrorStatusPropVersionInvalid From ccac5cd6595cd1b624218db734109525f9cd5fc3 Mon Sep 17 00:00:00 2001 From: luke Date: Sat, 16 Jan 2021 16:38:54 -0600 Subject: [PATCH 227/449] Cleanup tlog errors. --- politeiad/backend/tlogbe/tlog/tlog.go | 68 ++++++--------------- politeiad/backend/tlogbe/tlog/tlogclient.go | 12 ++-- politeiad/backend/tlogbe/tlogbe.go | 36 +++++------ 3 files changed, 42 insertions(+), 74 deletions(-) diff --git a/politeiad/backend/tlogbe/tlog/tlog.go b/politeiad/backend/tlogbe/tlog/tlog.go index 8e33e0256..09a164d7f 100644 --- a/politeiad/backend/tlogbe/tlog/tlog.go +++ b/politeiad/backend/tlogbe/tlog/tlog.go @@ -55,34 +55,6 @@ const ( keyPrefixAnchorRecord = "anchor:" ) -var ( - // ErrRecordNotFound is emitted when a record is not found. This - // can be because a tree does not exists for the provided tree id - // or when a tree does exist but the specified record version does - // not exist. - ErrRecordNotFound = errors.New("record not found") - - // ErrNoFileChanges is emitted when there are no files being - // changed. - ErrNoFileChanges = errors.New("no file changes") - - // ErrNoMetadataChanges is emitted when there are no metadata - // changes being made. - ErrNoMetadataChanges = errors.New("no metadata changes") - - // ErrFreezeRecordNotFound is emitted when a freeze record does not - // exist for a tree. - ErrFreezeRecordNotFound = errors.New("freeze record not found") - - // ErrTreeIsFrozen is emitted when a frozen tree is attempted to be - // altered. - ErrTreeIsFrozen = errors.New("tree is frozen") - - // ErrTreeIsNotFrozen is emitted when a tree is expected to be - // frozen but is actually not frozen. - ErrTreeIsNotFrozen = errors.New("tree is not frozen") -) - // We do not unwind. type Tlog struct { sync.Mutex @@ -523,7 +495,7 @@ func (t *Tlog) TreePointer(treeID int64) (int64, bool) { } idx, err = t.recordIndexLatest(leavesAll) if err != nil { - if err == ErrRecordNotFound { + if err == backend.ErrRecordNotFound { // This is an empty tree. This can happen sometimes if a error // occurred during record creation. Return gracefully. return 0, false @@ -610,8 +582,8 @@ func (t *Tlog) recordIndexSave(treeID int64, ri recordIndex) error { // recordIndexVersion takes a list of record indexes for a record and returns // the most recent iteration of the specified version. A version of 0 indicates -// that the latest version should be returned. A ErrRecordNotFound is returned -// if the provided version does not exist. +// that the latest version should be returned. A backend.ErrRecordNotFound is +// returned if the provided version does not exist. func recordIndexVersion(indexes []recordIndex, version uint32) (*recordIndex, error) { // Return the record index for the specified version var ri *recordIndex @@ -632,7 +604,7 @@ func recordIndexVersion(indexes []recordIndex, version uint32) (*recordIndex, er } if ri == nil { // The specified version does not exist - return nil, ErrRecordNotFound + return nil, backend.ErrRecordNotFound } return ri, nil @@ -668,7 +640,7 @@ func (t *Tlog) recordIndexes(leaves []*trillian.LogLeaf) ([]recordIndex, error) if len(keys) == 0 { // No records have been added to this tree yet - return nil, ErrRecordNotFound + return nil, backend.ErrRecordNotFound } // Get record indexes from store @@ -1044,7 +1016,7 @@ func (t *Tlog) RecordSave(treeID int64, rm backend.RecordMetadata, metadata []ba // Verify tree exists if !t.TreeExists(treeID) { - return ErrRecordNotFound + return backend.ErrRecordNotFound } // Get tree leaves @@ -1055,7 +1027,7 @@ func (t *Tlog) RecordSave(treeID int64, rm backend.RecordMetadata, metadata []ba // Get the existing record index currIdx, err := t.recordIndexLatest(leavesAll) - if errors.Is(err, ErrRecordNotFound) { + if errors.Is(err, backend.ErrRecordNotFound) { // No record versions exist yet. This is ok. currIdx = &recordIndex{ Metadata: make(map[uint64][]byte), @@ -1067,7 +1039,7 @@ func (t *Tlog) RecordSave(treeID int64, rm backend.RecordMetadata, metadata []ba // Verify tree state if currIdx.Frozen { - return ErrTreeIsFrozen + return backend.ErrRecordLocked } // Prepare kv store blobs @@ -1115,7 +1087,7 @@ func (t *Tlog) RecordSave(treeID int64, rm backend.RecordMetadata, metadata []ba } } if !fileChanges { - return ErrNoFileChanges + return backend.ErrNoChanges } // Save blobs @@ -1165,7 +1137,7 @@ func (t *Tlog) RecordSave(treeID int64, rm backend.RecordMetadata, metadata []ba func (t *Tlog) metadataSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) (*recordIndex, error) { // Verify tree exists if !t.TreeExists(treeID) { - return nil, ErrRecordNotFound + return nil, backend.ErrRecordNotFound } // Get tree leaves @@ -1180,7 +1152,7 @@ func (t *Tlog) metadataSave(treeID int64, rm backend.RecordMetadata, metadata [] return nil, err } if currIdx.Frozen { - return nil, ErrTreeIsFrozen + return nil, backend.ErrRecordLocked } // Prepare kv store blobs @@ -1191,7 +1163,7 @@ func (t *Tlog) metadataSave(treeID int64, rm backend.RecordMetadata, metadata [] // Verify at least one new blob is being saved to the kv store if len(bpr.blobs) == 0 { - return nil, ErrNoMetadataChanges + return nil, backend.ErrNoChanges } // Save the blobs @@ -1265,7 +1237,7 @@ func (t *Tlog) RecordDel(treeID int64) error { // Verify tree exists if !t.TreeExists(treeID) { - return ErrRecordNotFound + return backend.ErrRecordNotFound } // Get all tree leaves @@ -1281,7 +1253,7 @@ func (t *Tlog) RecordDel(treeID int64) error { return err } if !currIdx.Frozen { - return ErrTreeIsNotFrozen + return fmt.Errorf("tree is not frozen") } // Retrieve all the record indexes @@ -1349,7 +1321,7 @@ func (t *Tlog) RecordExists(treeID int64) bool { } idx, err = t.recordIndexLatest(leavesAll) if err != nil { - if err == ErrRecordNotFound { + if err == backend.ErrRecordNotFound { // This is an empty tree. This can happen sometimes if a error // occurred during record creation. Return gracefully. return false @@ -1376,7 +1348,7 @@ func (t *Tlog) Record(treeID int64, version uint32) (*backend.Record, error) { // Verify tree exists if !t.TreeExists(treeID) { - return nil, ErrRecordNotFound + return nil, backend.ErrRecordNotFound } // Get tree leaves @@ -1390,8 +1362,8 @@ func (t *Tlog) Record(treeID int64, version uint32) (*backend.Record, error) { // exists on the tree being pointed to, but not on this one. This // happens in situations such as when an unvetted record is made // public and copied to a vetted tree. Querying the unvetted tree - // will result in a ErrRecordNotFound error being returned and the - // vetted tree must be queried instead. + // will result in a backend.ErrRecordNotFound error being returned + // and the vetted tree must be queried instead. indexes, err := t.recordIndexes(leaves) if err != nil { return nil, err @@ -1401,7 +1373,7 @@ func (t *Tlog) Record(treeID int64, version uint32) (*backend.Record, error) { return nil, err } if treePointerExists(*idxLatest) { - return nil, ErrRecordNotFound + return nil, backend.ErrRecordNotFound } // Use the record index to pull the record content from the store. @@ -1669,7 +1641,7 @@ func (t *Tlog) RecordTimestamps(treeID int64, version uint32, token []byte) (*ba // Verify tree exists if !t.TreeExists(treeID) { - return nil, ErrRecordNotFound + return nil, backend.ErrRecordNotFound } // Get record index diff --git a/politeiad/backend/tlogbe/tlog/tlogclient.go b/politeiad/backend/tlogbe/tlog/tlogclient.go index 3e1125dcf..5e88bfe23 100644 --- a/politeiad/backend/tlogbe/tlog/tlogclient.go +++ b/politeiad/backend/tlogbe/tlog/tlogclient.go @@ -45,7 +45,7 @@ func (t *Tlog) BlobSave(treeID int64, keyPrefix string, be store.BlobEntry) ([]b // Verify tree exists if !t.TreeExists(treeID) { - return nil, ErrRecordNotFound + return nil, backend.ErrRecordNotFound } // Verify tree is not frozen @@ -58,7 +58,7 @@ func (t *Tlog) BlobSave(treeID int64, keyPrefix string, be store.BlobEntry) ([]b return nil, err } if idx.Frozen { - return nil, ErrTreeIsFrozen + return nil, backend.ErrRecordLocked } // Save blobs to store @@ -111,7 +111,7 @@ func (t *Tlog) BlobsDel(treeID int64, merkles [][]byte) error { // Verify tree exists. We allow blobs to be deleted from both // frozen and non frozen trees. if !t.TreeExists(treeID) { - return ErrRecordNotFound + return backend.ErrRecordNotFound } // Get all tree leaves @@ -159,7 +159,7 @@ func (t *Tlog) BlobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, // Verify tree exists if !t.TreeExists(treeID) { - return nil, ErrRecordNotFound + return nil, backend.ErrRecordNotFound } // Get leaves @@ -244,7 +244,7 @@ func (t *Tlog) BlobsByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error // Verify tree exists if !t.TreeExists(treeID) { - return nil, ErrRecordNotFound + return nil, backend.ErrRecordNotFound } // Get leaves @@ -308,7 +308,7 @@ func (t *Tlog) MerklesByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, err // Verify tree exists if !t.TreeExists(treeID) { - return nil, ErrRecordNotFound + return nil, backend.ErrRecordNotFound } // Get leaves diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 1cf781e9a..bd7163b06 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -813,12 +813,11 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ err = t.unvetted.RecordSave(treeID, *recordMD, metadata, files) if err != nil { switch err { - case tlog.ErrTreeIsFrozen: - return nil, backend.ErrRecordLocked - case tlog.ErrNoFileChanges: - return nil, backend.ErrNoChanges + case backend.ErrRecordLocked, backend.ErrNoChanges: + return nil, err + default: + return nil, fmt.Errorf("RecordSave: %v", err) } - return nil, fmt.Errorf("RecordSave: %v", err) } // Call post plugin hooks @@ -918,12 +917,11 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b err = t.vetted.RecordSave(treeID, *recordMD, metadata, files) if err != nil { switch err { - case tlog.ErrTreeIsFrozen: - return nil, backend.ErrRecordLocked - case tlog.ErrNoFileChanges: - return nil, backend.ErrNoChanges + case backend.ErrRecordLocked, backend.ErrNoChanges: + return nil, err + default: + return nil, fmt.Errorf("RecordSave: %v", err) } - return nil, fmt.Errorf("RecordSave: %v", err) } // Call post plugin hooks @@ -1015,12 +1013,11 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite err = t.unvetted.RecordMetadataSave(treeID, r.RecordMetadata, metadata) if err != nil { switch err { - case tlog.ErrTreeIsFrozen: - return backend.ErrRecordLocked - case tlog.ErrNoMetadataChanges: - return backend.ErrNoChanges + case backend.ErrRecordLocked, backend.ErrNoChanges: + return err + default: + return fmt.Errorf("RecordMetadataSave: %v", err) } - return err } // Call post plugin hooks @@ -1108,12 +1105,11 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ err = t.vetted.RecordMetadataSave(treeID, r.RecordMetadata, metadata) if err != nil { switch err { - case tlog.ErrTreeIsFrozen: - return backend.ErrRecordLocked - case tlog.ErrNoMetadataChanges: - return backend.ErrNoChanges + case backend.ErrRecordLocked, backend.ErrNoChanges: + return err + default: + return fmt.Errorf("RecordMetadataSave: %v", err) } - return fmt.Errorf("RecordMetadataSave: %v", err) } // Call post plugin hooks From f1b07038f0ca5537547c3b810bf1f1f25f89767f Mon Sep 17 00:00:00 2001 From: luke Date: Sat, 16 Jan 2021 17:56:50 -0600 Subject: [PATCH 228/449] Update tlogclient implementation. --- politeiad/backend/backend.go | 22 +- politeiad/backend/tlogbe/clients/clients.go | 62 ----- .../plugins/comments/{index.go => indexes.go} | 0 politeiad/backend/tlogbe/tlog/plugin.go | 53 ++-- politeiad/backend/tlogbe/tlog/tlog.go | 22 ++ politeiad/backend/tlogbe/tlog/tlogclient.go | 227 ++++++++---------- .../backend/tlogbe/tlog/trillianclient.go | 7 +- politeiad/backend/tlogbe/tlogbe.go | 1 + .../backend/tlogbe/tlogclient/tlogclient.go | 42 ++++ 9 files changed, 223 insertions(+), 213 deletions(-) delete mode 100644 politeiad/backend/tlogbe/clients/clients.go rename politeiad/backend/tlogbe/plugins/comments/{index.go => indexes.go} (100%) create mode 100644 politeiad/backend/tlogbe/tlogclient/tlogclient.go diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index e1ef45270..a3e3c9252 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Decred developers +// Copyright (c) 2017-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -314,3 +314,23 @@ type Backend interface { // Close performs cleanup of the backend Close() } + +// BackendReadOnly provides a read only version of the Backend interface. +// Plugins interact with backend records using this interface. +type BackendReadOnly interface { + // Unvetted exists returns whether an unvetted record exists. + UnvettedExists(token []byte) bool + + // Vetted exists returns whether a vetted record exists. + VettedExists(token []byte) bool + + // GetUnvetted returns an unvetted record. + GetUnvetted(token []byte, version string) (*Record, error) + + // GetVetted returns a vetted record. + GetVetted(token []byte, version string) (*Record, error) + + // InventoryByStatus returns the record tokens of all records in the + // inventory categorized by MDStatusT. + InventoryByStatus() (*InventoryByStatus, error) +} diff --git a/politeiad/backend/tlogbe/clients/clients.go b/politeiad/backend/tlogbe/clients/clients.go deleted file mode 100644 index d4c999e4b..000000000 --- a/politeiad/backend/tlogbe/clients/clients.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) 2020-2021 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package clients - -import ( - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/store" -) - -// BackendClient provides an API for plugins to interact with backend records. -// This an abridged version of the backend.Backend interface. -type BackendClient interface { - // Check if an unvetted record exists - UnvettedExists(token []byte) bool - - // Check if a vetted record exists - VettedExists(token []byte) bool - - // Get unvetted record - GetUnvetted(token []byte, version string) (*backend.Record, error) - - // Get vetted record - GetVetted(token []byte, version string) (*backend.Record, error) - - // InventoryByStatus returns the record tokens of all records in the - // inventory categorized by MDStatusT - InventoryByStatus() (*backend.InventoryByStatus, error) -} - -// TlogClient provides an API for plugins to interact with a tlog instance. -// Plugins are allowed to save, delete, and get plugin data to/from the tlog -// backend. Editing plugin data is not allowed. -type TlogClient interface { - // BlobSave saves a BlobEntry to the tlog backend. The BlobEntry - // will be encrypted prior to being written to disk if the tlog - // instance has an encryption key set. The merkle leaf hash for the - // blob will be returned. This merkle leaf hash can be though of as - // the blob ID and can be used to retrieve or delete the blob. - BlobSave(treeID int64, keyPrefix string, be store.BlobEntry) ([]byte, error) - - // BlobsDel deletes the blobs that correspond to the provided - // merkle leaf hashes. - BlobsDel(treeID int64, merkles [][]byte) error - - // BlobsByMerkle returns the blobs with the provided merkle leaf - // hashes. If a blob does not exist it will not be included in the - // returned map. - BlobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, error) - - // BlobsByKeyPrefix returns all blobs that match the key prefix. - BlobsByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) - - // MerklesByKeyPrefix returns the merkle leaf hashes for all blobs - // that match the key prefix. - MerklesByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) - - // Timestamp returns the timestamp for the blob that correpsonds - // to the merkle leaf hash. - Timestamp(treeID int64, merkle []byte) (*backend.Timestamp, error) -} diff --git a/politeiad/backend/tlogbe/plugins/comments/index.go b/politeiad/backend/tlogbe/plugins/comments/indexes.go similarity index 100% rename from politeiad/backend/tlogbe/plugins/comments/index.go rename to politeiad/backend/tlogbe/plugins/comments/indexes.go diff --git a/politeiad/backend/tlogbe/tlog/plugin.go b/politeiad/backend/tlogbe/tlog/plugin.go index 9f95c333d..1200daa47 100644 --- a/politeiad/backend/tlogbe/tlog/plugin.go +++ b/politeiad/backend/tlogbe/tlog/plugin.go @@ -11,10 +11,6 @@ import ( "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/comments" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/dcrdata" - cmplugin "github.com/decred/politeia/politeiad/plugins/comments" - ddplugin "github.com/decred/politeia/politeiad/plugins/dcrdata" ) const ( @@ -59,31 +55,34 @@ func (t *Tlog) PluginRegister(b backend.Backend, p backend.Plugin) error { err error dataDir = filepath.Join(t.dataDir, pluginDataDirname) + + _ = err + _ = dataDir ) switch p.ID { - case cmplugin.ID: - client, err = comments.New(t, p.Settings, p.Identity, dataDir) - if err != nil { - return err - } - case ddplugin.ID: - client, err = dcrdata.New(p.Settings, t.activeNetParams) - if err != nil { - return err - } - /* - case piplugin.ID: - client, err = pi.New(b, t, p.Settings, dataDir, t.activeNetParams) - if err != nil { - return err - } - case ticketvote.ID: - client, err = newTicketVotePlugin(t, newBackendClient(t), - p.Settings, p.Identity, t.activeNetParams) - if err != nil { - return err - } - */ + /* + case cmplugin.ID: + client, err = comments.New(t, p.Settings, p.Identity, dataDir) + if err != nil { + return err + } + case ddplugin.ID: + client, err = dcrdata.New(p.Settings, t.activeNetParams) + if err != nil { + return err + } + case piplugin.ID: + client, err = pi.New(b, t, p.Settings, dataDir, t.activeNetParams) + if err != nil { + return err + } + case ticketvote.ID: + client, err = newTicketVotePlugin(t, newBackendClient(t), + p.Settings, p.Identity, t.activeNetParams) + if err != nil { + return err + } + */ default: return backend.ErrPluginInvalid } diff --git a/politeiad/backend/tlogbe/tlog/tlog.go b/politeiad/backend/tlogbe/tlog/tlog.go index 09a164d7f..20e430fc0 100644 --- a/politeiad/backend/tlogbe/tlog/tlog.go +++ b/politeiad/backend/tlogbe/tlog/tlog.go @@ -185,6 +185,20 @@ func leafIsAnchor(l *trillian.LogLeaf) bool { return bytes.HasPrefix(l.ExtraData, []byte(keyPrefixAnchorRecord)) } +func leafExtraData(dataType, storeKey string) []byte { + return []byte(dataType + ":" + storeKey) +} + +func leafDataType(l *trillian.LogLeaf) string { + s := bytes.SplitAfter(l.ExtraData, []byte(":")) + if len(s) != 2 { + e := fmt.Sprintf("invalid key '%s' for leaf %x", + l.ExtraData, l.MerkleLeafHash) + panic(e) + } + return string(s[0]) +} + func extractKeyFromLeaf(l *trillian.LogLeaf) string { s := bytes.SplitAfter(l.ExtraData, []byte(":")) if len(s) != 2 { @@ -531,6 +545,14 @@ func (t *Tlog) TreesAll() ([]int64, error) { return treeIDs, nil } +func (t *Tlog) treeIsFrozen(leaves []*trillian.LogLeaf) bool { + r, err := t.recordIndexLatest(leaves) + if err != nil { + panic(err) + } + return r.Frozen +} + func (t *Tlog) recordIndexSave(treeID int64, ri recordIndex) error { // Save record index to the store be, err := convertBlobEntryFromRecordIndex(ri) diff --git a/politeiad/backend/tlogbe/tlog/tlogclient.go b/politeiad/backend/tlogbe/tlog/tlogclient.go index 5e88bfe23..3cc519806 100644 --- a/politeiad/backend/tlogbe/tlog/tlogclient.go +++ b/politeiad/backend/tlogbe/tlog/tlogclient.go @@ -5,108 +5,99 @@ package tlog import ( - "bytes" "encoding/hex" "fmt" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/store" + "github.com/decred/politeia/politeiad/backend/tlogbe/tlogclient" "github.com/google/trillian" "google.golang.org/grpc/codes" ) -// BlobSave saves a BlobEntry to the tlog backend. The BlobEntry will be +var ( + _ tlogclient.Client = (*Tlog)(nil) +) + +// BlobSave saves a BlobEntry to the tlog instance. The BlobEntry will be // encrypted prior to being written to disk if the tlog instance has an -// encryption key set. The merkle leaf hash for the blob will be returned. This -// merkle leaf hash can be though of as the blob ID and can be used to retrieve -// or delete the blob. +// encryption key set. The digest of the data, i.e. BlobEntry.Digest, can be +// thought of as the blob ID and can be used to get/del the blob from tlog. // -// This function satisfies the plugins.TlogClient interface. -func (t *Tlog) BlobSave(treeID int64, keyPrefix string, be store.BlobEntry) ([]byte, error) { - log.Tracef("%v BlobSave: %v %v", t.id, treeID, keyPrefix) +// This function satisfies the tlogclient.Client interface. +func (t *Tlog) BlobSave(treeID int64, dataType string, be store.BlobEntry) error { + log.Tracef("%v BlobSave: %v %v", t.id, treeID, dataType) // Prepare blob and digest digest, err := hex.DecodeString(be.Hash) if err != nil { - return nil, err + return err } blob, err := store.Blobify(be) if err != nil { - return nil, err + return err } // Encrypt blob if an encryption key has been set if t.encryptionKey != nil { blob, err = t.encrypt(blob) if err != nil { - return nil, err + return err } } // Verify tree exists if !t.TreeExists(treeID) { - return nil, backend.ErrRecordNotFound + return backend.ErrRecordNotFound } // Verify tree is not frozen - leavesAll, err := t.trillian.leavesAll(treeID) - if err != nil { - return nil, fmt.Errorf("leavesAll: %v", err) - } - idx, err := t.recordIndexLatest(leavesAll) + leaves, err := t.trillian.leavesAll(treeID) if err != nil { - return nil, err + return fmt.Errorf("leavesAll: %v", err) } - if idx.Frozen { - return nil, backend.ErrRecordLocked + if t.treeIsFrozen(leaves) { + return backend.ErrRecordLocked } // Save blobs to store keys, err := t.store.Put([][]byte{blob}) if err != nil { - return nil, fmt.Errorf("store Put: %v", err) + return fmt.Errorf("store Put: %v", err) } if len(keys) != 1 { - return nil, fmt.Errorf("wrong number of keys: got %v, want 1", + return fmt.Errorf("wrong number of keys: got %v, want 1", len(keys)) } // Prepare log leaf - extraData := []byte(keyPrefix + keys[0]) - leaves := []*trillian.LogLeaf{ + extraData := leafExtraData(dataType, keys[0]) + leaves = []*trillian.LogLeaf{ newLogLeaf(digest, extraData), } // Append log leaf to trillian tree queued, _, err := t.trillian.leavesAppend(treeID, leaves) if err != nil { - return nil, fmt.Errorf("leavesAppend: %v", err) + return fmt.Errorf("leavesAppend: %v", err) } if len(queued) != 1 { - return nil, fmt.Errorf("wrong number of queued leaves: "+ + return fmt.Errorf("wrong number of queued leaves: "+ "got %v, want 1", len(queued)) } - failed := make([]string, 0, len(queued)) - for _, v := range queued { - c := codes.Code(v.QueuedLeaf.GetStatus().GetCode()) - if c != codes.OK { - failed = append(failed, fmt.Sprintf("%v", c)) - } - } - if len(failed) > 0 { - return nil, fmt.Errorf("append leaves failed: %v", failed) + c := codes.Code(queued[0].QueuedLeaf.GetStatus().GetCode()) + if c != codes.OK { + return fmt.Errorf("queued leaf error: %v", c) } - return queued[0].QueuedLeaf.Leaf.MerkleLeafHash, nil + return nil } -// BlobsDel deletes the blobs in the kv store that correspond to the provided -// merkle leaf hashes. The kv store keys in store in the ExtraData field of the -// leaves specified by the provided merkle leaf hashes. +// BlobsDel deletes the blobs that correspond to the provided digests. // -// This function satisfies the plugins.TlogClient interface. -func (t *Tlog) BlobsDel(treeID int64, merkles [][]byte) error { - log.Tracef("%v BlobsDel: %v", t.id, treeID) +// This function satisfies the tlogclient.Client interface. +func (t *Tlog) BlobsDel(treeID int64, digests [][]byte) error { + log.Tracef("%v BlobsDel: %v %x", t.id, treeID, digests) // Verify tree exists. We allow blobs to be deleted from both // frozen and non frozen trees. @@ -117,20 +108,21 @@ func (t *Tlog) BlobsDel(treeID int64, merkles [][]byte) error { // Get all tree leaves leaves, err := t.trillian.leavesAll(treeID) if err != nil { - return err + return fmt.Errorf("leavesAll: %v", err) } // Put merkle leaf hashes into a map so that we can tell if a leaf // corresponds to one of the target merkle leaf hashes in O(n) // time. merkleHashes := make(map[string]struct{}, len(leaves)) - for _, v := range merkles { - merkleHashes[hex.EncodeToString(v)] = struct{}{} + for _, v := range digests { + m := hex.EncodeToString(merkleLeafHash(v)) + merkleHashes[m] = struct{}{} } // Aggregate the key-value store keys for the provided merkle leaf // hashes. - keys := make([]string, 0, len(merkles)) + keys := make([]string, 0, len(digests)) for _, v := range leaves { _, ok := merkleHashes[hex.EncodeToString(v.MerkleLeafHash)] if ok { @@ -147,15 +139,12 @@ func (t *Tlog) BlobsDel(treeID int64, merkles [][]byte) error { return nil } -// BlobsByMerkle returns the blobs with the provided merkle leaf hashes. +// Blobs returns the blobs that correspond to the provided digests. If a blob +// does not exist it will not be included in the returned map. // -// If a blob does not exist it will not be included in the returned map. It is -// the responsibility of the caller to check that a blob is returned for each -// of the provided merkle hashes. -// -// This function satisfies the plugins.TlogClient interface. -func (t *Tlog) BlobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, error) { - log.Tracef("%v BlobsByMerkle: %v", t.id, treeID) +// This function satisfies the tlogclient.Client interface. +func (t *Tlog) Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEntry, error) { + log.Tracef("%v Blobs: %v %x", t.id, treeID, digests) // Verify tree exists if !t.TreeExists(treeID) { @@ -170,10 +159,11 @@ func (t *Tlog) BlobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, // Aggregate the leaves that correspond to the provided merkle // hashes. - // map[merkleHash]*trillian.LogLeaf - leaves := make(map[string]*trillian.LogLeaf, len(merkles)) - for _, v := range merkles { - leaves[hex.EncodeToString(v)] = nil + // map[merkleLeafHash]*trillian.LogLeaf + leaves := make(map[string]*trillian.LogLeaf, len(digests)) + for _, v := range digests { + m := hex.EncodeToString(merkleLeafHash(v)) + leaves[m] = nil } for _, v := range leavesAll { m := hex.EncodeToString(v.MerkleLeafHash) @@ -185,22 +175,23 @@ func (t *Tlog) BlobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, // Ensure a leaf was found for all provided merkle hashes for k, v := range leaves { if v == nil { - return nil, fmt.Errorf("leaf not found for merkle hash: %v", k) + return nil, fmt.Errorf("leaf not found: %v", k) } } // Extract the key-value store keys. These keys MUST be put in the - // same order that the merkle hashes were provided in. + // same order that the digests were provided in. keys := make([]string, 0, len(leaves)) - for _, v := range merkles { - l, ok := leaves[hex.EncodeToString(v)] + for _, v := range digests { + m := hex.EncodeToString(merkleLeafHash(v)) + l, ok := leaves[m] if !ok { - return nil, fmt.Errorf("leaf not found for merkle hash: %x", v) + return nil, fmt.Errorf("leaf not found: %x", v) } keys = append(keys, extractKeyFromLeaf(l)) } - // Pull the blobs from the store. If is ok if one or more blobs is + // Pull the blobs from the store. It's ok if one or more blobs is // not found. It is the responsibility of the caller to decide how // this should be handled. blobs, err := t.store.Get(keys) @@ -208,39 +199,31 @@ func (t *Tlog) BlobsByMerkle(treeID int64, merkles [][]byte) (map[string][]byte, return nil, fmt.Errorf("store Get: %v", err) } - // Decrypt any encrypted blobs - for k, v := range blobs { - if blobIsEncrypted(v) { - b, _, err := t.encryptionKey.decrypt(v) - if err != nil { - return nil, err - } - blobs[k] = b - } - } - - // Put blobs in a map so the caller can determine if any of the - // provided merkle hashes did not correspond to a blob in the - // store. - b := make(map[string][]byte, len(blobs)) // [merkleHash]blob + // Deblob the blobs and put them in a map so the caller can + // determine if any blob entries are missing. + entries := make(map[string]store.BlobEntry, len(blobs)) // [digest]BlobEntry for k, v := range keys { - // The merkle hashes slice and keys slice share the same order - merkleHash := hex.EncodeToString(merkles[k]) - blob, ok := blobs[v] + // The digests slice and the keys slice share the same order + digest := hex.EncodeToString(digests[k]) + b, ok := blobs[v] if !ok { - return nil, fmt.Errorf("blob not found for key %v", v) + return nil, fmt.Errorf("blob not found: %v", v) + } + be, err := t.deblob(b) + if err != nil { + return nil, fmt.Errorf("deblob %v: %v", digest, err) } - b[merkleHash] = blob + entries[digest] = *be } - return b, nil + return entries, nil } -// BlobsByKeyPrefix returns all blobs that match the provided key prefix. +// BlobsByDataType returns all blobs that match the data type. // -// This function satisfies the plugins.TlogClient interface. -func (t *Tlog) BlobsByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) { - log.Tracef("%v BlobsByKeyPrefix: %v %v", t.id, treeID, keyPrefix) +// This function satisfies the tlogclient.Client interface. +func (t *Tlog) BlobsByDataType(treeID int64, dataType string) ([]store.BlobEntry, error) { + log.Tracef("%v BlobsByDataType: %v %v", t.id, treeID, dataType) // Verify tree exists if !t.TreeExists(treeID) { @@ -257,7 +240,7 @@ func (t *Tlog) BlobsByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error // leaves with a matching key prefix. keys := make([]string, 0, len(leaves)) for _, v := range leaves { - if bytes.HasPrefix(v.ExtraData, []byte(keyPrefix)) { + if leafDataType(v) == dataType { keys = append(keys, extractKeyFromLeaf(v)) } } @@ -279,32 +262,29 @@ func (t *Tlog) BlobsByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error return nil, fmt.Errorf("blobs not found: %v", missing) } - // Decrypt any encrypted blobs - for k, v := range blobs { - if blobIsEncrypted(v) { - b, _, err := t.encryptionKey.decrypt(v) - if err != nil { - return nil, err - } - blobs[k] = b + // Prepare reply. The blob entries should be in the same order as + // the keys, i.e. ordered from oldest to newest. + entries := make([]store.BlobEntry, 0, len(keys)) + for _, v := range keys { + b, ok := blobs[v] + if !ok { + return nil, fmt.Errorf("blob not found: %v", v) } + be, err := t.deblob(b) + if err != nil { + return nil, err + } + entries = append(entries, *be) } - // Covert blobs from map to slice - b := make([][]byte, 0, len(blobs)) - for _, v := range blobs { - b = append(b, v) - } - - return b, nil + return entries, nil } -// MerklesByKeyPrefix returns the merkle leaf hashes for all blobs that match -// the key prefix. +// DigestsByDataType returns the digests of all blobs that match the data type. // -// This function satisfies the plugins.TlogClient interface. -func (t *Tlog) MerklesByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, error) { - log.Tracef("%v MerklesByKeyPrefix: %v %v", t.id, treeID, keyPrefix) +// This function satisfies the tlogclient.Client interface. +func (t *Tlog) DigestsByDataType(treeID int64, dataType string) ([][]byte, error) { + log.Tracef("%v DigestsByDataType: %v %v", t.id, treeID, dataType) // Verify tree exists if !t.TreeExists(treeID) { @@ -317,31 +297,34 @@ func (t *Tlog) MerklesByKeyPrefix(treeID int64, keyPrefix string) ([][]byte, err return nil, fmt.Errorf("leavesAll: %v", err) } - // Walk leaves and aggregate the merkle leaf hashes with a matching - // key prefix. - merkles := make([][]byte, 0, len(leaves)) + // Walk leaves and aggregate the digests, i.e. the leaf value, of + // all leaves that match the provided data type. + digests := make([][]byte, 0, len(leaves)) for _, v := range leaves { - if bytes.HasPrefix(v.ExtraData, []byte(keyPrefix)) { - merkles = append(merkles, v.MerkleLeafHash) + if leafDataType(v) == dataType { + digests = append(digests, v.LeafValue) } } - return merkles, nil + return digests, nil } // Timestamp returns the timestamp for the data blob that corresponds to the -// provided merkle leaf hash. +// provided digest. // -// This function satisfies the plugins.TlogClient interface. -func (t *Tlog) Timestamp(treeID int64, merkleLeafHash []byte) (*backend.Timestamp, error) { - log.Tracef("%v Timestamp: %v %x", t.id, treeID, merkleLeafHash) +// This function satisfies the tlogclient.Client interface. +func (t *Tlog) Timestamp(treeID int64, digest []byte) (*backend.Timestamp, error) { + log.Tracef("%v Timestamp: %v %x", t.id, treeID, digest) - // Get all tree leaves + // Get tree leaves leaves, err := t.trillian.leavesAll(treeID) if err != nil { return nil, fmt.Errorf("leavesAll: %v", err) } + // Get merkle leaf hash + m := merkleLeafHash(digest) + // Get timestamp - return t.timestamp(treeID, merkleLeafHash, leaves) + return t.timestamp(treeID, m, leaves) } diff --git a/politeiad/backend/tlogbe/tlog/trillianclient.go b/politeiad/backend/tlogbe/tlog/trillianclient.go index 5462c2521..aacd17985 100644 --- a/politeiad/backend/tlogbe/tlog/trillianclient.go +++ b/politeiad/backend/tlogbe/tlog/trillianclient.go @@ -475,7 +475,12 @@ func (t *tclient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { } // Get all leaves - return t.leavesByRange(treeID, 0, int64(lr.TreeSize)) + leaves, err := t.leavesByRange(treeID, 0, int64(lr.TreeSize)) + if err != nil { + return nil, fmt.Errorf("leavesByRange: %v", err) + } + + return leaves, nil } // close closes the trillian grpc connection. diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index bd7163b06..2bf6a5a52 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -1739,6 +1739,7 @@ func (t *tlogBackend) VettedPlugin(token []byte, pluginID, cmd, payload string) return "", err } + // Execute plugin command reply, err := t.vetted.PluginCmd(treeID, token, pluginID, cmd, payload) if err != nil { return "", err diff --git a/politeiad/backend/tlogbe/tlogclient/tlogclient.go b/politeiad/backend/tlogbe/tlogclient/tlogclient.go new file mode 100644 index 000000000..49e80acc7 --- /dev/null +++ b/politeiad/backend/tlogbe/tlogclient/tlogclient.go @@ -0,0 +1,42 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlogclient + +import ( + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/store" +) + +// Client provides an API for plugins to interact with a tlog instance. +// Plugins are allowed to save, delete, and get plugin data to/from the tlog +// backend. Editing plugin data is not allowed. +type Client interface { + // BlobSave saves a BlobEntry to the tlog instance. The BlobEntry + // will be encrypted prior to being written to disk if the tlog + // instance has an encryption key set. The digest of the data, + // i.e. BlobEntry.Digest, can be thought of as the blob ID and can + // be used to get/del the blob from tlog. + BlobSave(treeID int64, dataType string, be store.BlobEntry) error + + // BlobsDel deletes the blobs that correspond to the provided + // digests. + BlobsDel(treeID int64, digests [][]byte) error + + // Blobs returns the blobs that correspond to the provided digests. + // If a blob does not exist it will not be included in the returned + // map. + Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEntry, error) + + // BlobsByDataType returns all blobs that match the data type. + BlobsByDataType(treeID int64, keyPrefix string) ([]store.BlobEntry, error) + + // DigestsByDataType returns the digests of all blobs that match + // the data type. + DigestsByDataType(treeID int64, dataType string) ([][]byte, error) + + // Timestamp returns the timestamp for the blob that correpsonds + // to the digest. + Timestamp(treeID int64, digest []byte) (*backend.Timestamp, error) +} From ed8e0618ac1ebe18406f517a5ae9561c42be03e3 Mon Sep 17 00:00:00 2001 From: luke Date: Sat, 16 Jan 2021 18:37:33 -0600 Subject: [PATCH 229/449] Cleanup tlog. --- politeiad/backend/tlogbe/tlog/anchor.go | 11 +- politeiad/backend/tlogbe/tlog/recordindex.go | 283 ++++++++++++++ politeiad/backend/tlogbe/tlog/tlog.go | 344 ++---------------- politeiad/backend/tlogbe/tlog/tlogclient.go | 16 +- .../tlogbe/tlog/{timestamp.go => verify.go} | 0 politeiad/backend/tlogbe/tlogbe.go | 18 +- util/token.go | 11 +- 7 files changed, 345 insertions(+), 338 deletions(-) create mode 100644 politeiad/backend/tlogbe/tlog/recordindex.go rename politeiad/backend/tlogbe/tlog/{timestamp.go => verify.go} (100%) diff --git a/politeiad/backend/tlogbe/tlog/anchor.go b/politeiad/backend/tlogbe/tlog/anchor.go index 904d94dc9..3d945f99e 100644 --- a/politeiad/backend/tlogbe/tlog/anchor.go +++ b/politeiad/backend/tlogbe/tlog/anchor.go @@ -80,7 +80,7 @@ func (t *Tlog) anchorForLeaf(treeID int64, merkleLeafHash []byte, leaves []*tril var anchorKey string for i := int(l.LeafIndex); i < len(leaves); i++ { l := leaves[i] - if leafIsAnchor(l) { + if leafDataType(l) == dataTypeAnchorRecord { anchorKey = extractKeyFromLeaf(l) break } @@ -123,8 +123,9 @@ func (t *Tlog) anchorLatest(treeID int64) (*anchor, error) { // Find the most recent anchor leaf var key string for i := len(leavesAll) - 1; i >= 0; i-- { - if leafIsAnchor(leavesAll[i]) { - key = extractKeyFromLeaf(leavesAll[i]) + l := leavesAll[i] + if leafDataType(l) == dataTypeAnchorRecord { + key = extractKeyFromLeaf(l) } } if key == "" { @@ -192,9 +193,9 @@ func (t *Tlog) anchorSave(a anchor) error { if err != nil { return err } - prefixedKey := []byte(keyPrefixAnchorRecord + keys[0]) + extraData := leafExtraData(dataTypeAnchorRecord, keys[0]) leaves := []*trillian.LogLeaf{ - newLogLeaf(h, prefixedKey), + newLogLeaf(h, extraData), } queued, _, err := t.trillian.leavesAppend(a.TreeID, leaves) if err != nil { diff --git a/politeiad/backend/tlogbe/tlog/recordindex.go b/politeiad/backend/tlogbe/tlog/recordindex.go new file mode 100644 index 000000000..a8120f156 --- /dev/null +++ b/politeiad/backend/tlogbe/tlog/recordindex.go @@ -0,0 +1,283 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlog + +import ( + "encoding/hex" + "fmt" + "sort" + + "github.com/decred/politeia/politeiad/backend" + "github.com/google/trillian" + "google.golang.org/grpc/codes" +) + +// recordIndex contains the merkle leaf hashes of all the record content leaves +// for a specific record version and iteration. The record index can be used to +// lookup the trillian log leaves for the record content and the log leaves can +// be used to lookup the kv store blobs. +// +// A record is updated in three steps: +// +// 1. Record content blobs are saved to the kv store. +// +// 2. A trillian leaf is created for each record content blob. The kv store +// key for the blob is stuffed into the LogLeaf.ExtraData field. All leaves +// are appended onto the trillian tree. +// +// 3. If there are failures in steps 1 or 2 for any of the blobs then the +// update will exit without completing. No unwinding is performed. Blobs +// will be left in the kv store as orphaned blobs. The trillian tree is +// append only so once a leaf is appended, it's there permanently. If steps +// 1 and 2 are successful then a recordIndex will be created, saved to the +// kv store, and appended onto the trillian tree. +// +// Appending a recordIndex onto the trillian tree is the last operation that +// occurs during a record update. If a recordIndex exists in the tree then the +// update is considered successful. Any record content leaves that are not part +// of a recordIndex are considered to be orphaned and can be disregarded. +type recordIndex struct { + // Version represents the version of the record. The version is + // only incremented when the record files are updated. Metadata + // only updates do no increment the version. + Version uint32 `json:"version"` + + // Iteration represents the iteration of the record. The iteration + // is incremented anytime any record content changes. This includes + // file changes that bump the version as well metadata stream and + // record metadata changes that don't bump the version. + // + // Note, this field is not the same as the backend RecordMetadata + // iteration field, which does not get incremented on metadata + // updates. + // + // TODO maybe it should be the same. The original iteration field + // was to track unvetted changes in gitbe since unvetted gitbe + // records are not versioned. tlogbe unvetted records are versioned + // so the original use for the iteration field isn't needed anymore. + Iteration uint32 `json:"iteration"` + + // The following fields contain the merkle leaf hashes of the + // trillian log leaves for the record content. The merkle leaf hash + // can be used to lookup the log leaf. The log leaf ExtraData field + // contains the key for the record content in the key-value store. + RecordMetadata []byte `json:"recordmetadata"` + Metadata map[uint64][]byte `json:"metadata"` // [metadataID]merkle + Files map[string][]byte `json:"files"` // [filename]merkle + + // Frozen is used to indicate that the tree for this record has + // been frozen. This happens as a result of certain record status + // changes. The only thing that can be appended onto a frozen tree + // is one additional anchor record. Once a frozen tree has been + // anchored, the tlog fsck function will update the status of the + // tree to frozen in trillian, at which point trillian will not + // allow any additional leaves to be appended onto the tree. + Frozen bool `json:"frozen,omitempty"` + + // TODO make this a generic ExtraData field + // TreePointer is the tree ID of the tree that is the new location + // of this record. A record can be copied to a new tree after + // certain status changes, such as when a record is made public and + // the record is copied from an unvetted tree to a vetted tree. + // TreePointer should only be set if the tree has been frozen. + TreePointer int64 `json:"treepointer,omitempty"` +} + +// treePointerExists returns whether the provided record index has a tree +// pointer set. +func treePointerExists(r recordIndex) bool { + // Sanity checks + switch { + case !r.Frozen && r.TreePointer > 0: + // Tree pointer should only be set if the record is frozen + e := fmt.Sprintf("tree pointer set without record being frozen %v", + r.TreePointer) + panic(e) + case r.TreePointer < 0: + // Tree pointer should never be negative. Trillian uses a int64 + // for the tree ID so we do too. + e := fmt.Sprintf("tree pointer is < 0: %v", r.TreePointer) + panic(e) + } + + return r.TreePointer > 0 +} + +// parseRecordIndex takes a list of record indexes and returns the most recent +// iteration of the specified version. A version of 0 indicates that the latest +// version should be returned. A backend.ErrRecordNotFound is returned if the +// provided version does not exist. +func parseRecordIndex(indexes []recordIndex, version uint32) (*recordIndex, error) { + // Return the record index for the specified version + var ri *recordIndex + if version == 0 { + // A version of 0 indicates that the most recent version should + // be returned. + ri = &indexes[len(indexes)-1] + } else { + // Walk the indexes backwards so the most recent iteration of the + // specified version is selected. + for i := len(indexes) - 1; i >= 0; i-- { + r := indexes[i] + if r.Version == version { + ri = &r + break + } + } + } + if ri == nil { + // The specified version does not exist + return nil, backend.ErrRecordNotFound + } + + return ri, nil +} + +// recordIndexSave saves a record index to tlog. +func (t *Tlog) recordIndexSave(treeID int64, ri recordIndex) error { + // Save record index to the store + be, err := convertBlobEntryFromRecordIndex(ri) + if err != nil { + return err + } + b, err := t.blobify(*be) + if err != nil { + return err + } + keys, err := t.store.Put([][]byte{b}) + if err != nil { + return fmt.Errorf("store Put: %v", err) + } + if len(keys) != 1 { + return fmt.Errorf("wrong number of keys: got %v, want 1", + len(keys)) + } + + // Append record index leaf to trillian tree + h, err := hex.DecodeString(be.Hash) + if err != nil { + return err + } + extraData := leafExtraData(dataTypeRecordIndex, keys[0]) + leaves := []*trillian.LogLeaf{ + newLogLeaf(h, extraData), + } + queued, _, err := t.trillian.leavesAppend(treeID, leaves) + if err != nil { + return fmt.Errorf("leavesAppend: %v", err) + } + if len(queued) != 1 { + return fmt.Errorf("wrong number of queud leaves: got %v, want 1", + len(queued)) + } + failed := make([]string, 0, len(queued)) + for _, v := range queued { + c := codes.Code(v.QueuedLeaf.GetStatus().GetCode()) + if c != codes.OK { + failed = append(failed, fmt.Sprintf("%v", c)) + } + } + if len(failed) > 0 { + return fmt.Errorf("append leaves failed: %v", failed) + } + + return nil +} + +// recordIndexes returns all record indexes found in the provided trillian +// leaves. +func (t *Tlog) recordIndexes(leaves []*trillian.LogLeaf) ([]recordIndex, error) { + // Walk the leaves and compile the keys for all record indexes. It + // is possible for multiple indexes to exist for the same record + // version (they will have different iterations due to metadata + // only updates) so we have to pull the index blobs from the store + // in order to find the most recent iteration for the specified + // version. + keys := make([]string, 0, 64) + for _, v := range leaves { + if leafDataType(v) == dataTypeRecordIndex { + // This is a record index leaf. Save the kv store key. + keys = append(keys, extractKeyFromLeaf(v)) + } + } + + if len(keys) == 0 { + // No records have been added to this tree yet + return nil, backend.ErrRecordNotFound + } + + // Get record indexes from store + blobs, err := t.store.Get(keys) + if err != nil { + return nil, fmt.Errorf("store Get: %v", err) + } + missing := make([]string, 0, len(keys)) + for _, v := range keys { + if _, ok := blobs[v]; !ok { + missing = append(missing, v) + } + } + if len(missing) > 0 { + return nil, fmt.Errorf("record index not found: %v", missing) + } + + indexes := make([]recordIndex, 0, len(blobs)) + for _, v := range blobs { + be, err := t.deblob(v) + if err != nil { + return nil, err + } + ri, err := convertRecordIndexFromBlobEntry(*be) + if err != nil { + return nil, err + } + indexes = append(indexes, *ri) + } + + // Sort indexes by iteration, smallest to largets. The leaves + // ordering was not preserved in the returned blobs map. + sort.SliceStable(indexes, func(i, j int) bool { + return indexes[i].Iteration < indexes[j].Iteration + }) + + // Sanity check. Index iterations should start with 1 and be + // sequential. Index versions should start with 1 and also be + // sequential, but duplicate versions can exist as long as the + // iteration has been incremented. + var versionPrev uint32 + var i uint32 = 1 + for _, v := range indexes { + if v.Iteration != i { + return nil, fmt.Errorf("invalid record index iteration: "+ + "got %v, want %v", v.Iteration, i) + } + diff := v.Version - versionPrev + if diff != 0 && diff != 1 { + return nil, fmt.Errorf("invalid record index version: "+ + "curr version %v, prev version %v", v.Version, versionPrev) + } + + i++ + versionPrev = v.Version + } + + return indexes, nil +} + +// recordIndex returns the specified version of a record index for a slice of +// trillian leaves. +func (t *Tlog) recordIndex(leaves []*trillian.LogLeaf, version uint32) (*recordIndex, error) { + indexes, err := t.recordIndexes(leaves) + if err != nil { + return nil, err + } + return parseRecordIndex(indexes, version) +} + +// recordIndexLatest returns the most recent record index for a slice of +// trillian leaves. +func (t *Tlog) recordIndexLatest(leaves []*trillian.LogLeaf) (*recordIndex, error) { + return t.recordIndex(leaves, 0) +} diff --git a/politeiad/backend/tlogbe/tlog/tlog.go b/politeiad/backend/tlogbe/tlog/tlog.go index 20e430fc0..cabd92ebd 100644 --- a/politeiad/backend/tlogbe/tlog/tlog.go +++ b/politeiad/backend/tlogbe/tlog/tlog.go @@ -13,7 +13,6 @@ import ( "fmt" "os" "path/filepath" - "sort" "strconv" "sync" @@ -32,11 +31,11 @@ const ( defaultStoreDirname = "store" // Blob entry data descriptors - dataDescriptorFile = "file" - dataDescriptorRecordMetadata = "recordmetadata" - dataDescriptorMetadataStream = "metadatastream" - dataDescriptorRecordIndex = "recordindex" - dataDescriptorAnchor = "anchor" + dataDescriptorFile = "file_v1" + dataDescriptorRecordMetadata = "recordmetadata_v1" + dataDescriptorMetadataStream = "metadatastream_v1" + dataDescriptorRecordIndex = "recordindex_v1" + dataDescriptorAnchor = "anchor_v1" // The keys for kv store blobs are saved by stuffing them into the // ExtraData field of their corresponding trillian log leaf. The @@ -49,10 +48,10 @@ const ( // TODO key prefix app-dataID: // TODO the leaf ExtraData field should be hinted. Similar to what // we do for blobs. - keyPrefixRecordIndex = "recordindex:" - keyPrefixRecordContent = "record:" - keyPrefixFreezeRecord = "freeze:" - keyPrefixAnchorRecord = "anchor:" + dataTypeSeperator = ":" + dataTypeRecordIndex = "rindex" + dataTypeRecordContent = "rcontent" + dataTypeAnchorRecord = "anchor" ) // We do not unwind. @@ -78,113 +77,12 @@ type Tlog struct { droppingAnchor bool } -// recordIndex contains the merkle leaf hashes of all the record content leaves -// for a specific record version and iteration. The record index can be used to -// lookup the trillian log leaves for the record content and the log leaves can -// be used to lookup the kv store blobs. -// -// A record is updated in three steps: -// -// 1. Record content blobs are saved to the kv store. -// -// 2. A trillian leaf is created for each record content blob. The kv store -// key for the blob is stuffed into the LogLeaf.ExtraData field. All leaves -// are appended onto the trillian tree. -// -// 3. If there are failures in steps 1 or 2 for any of the blobs then the -// update will exit without completing. No unwinding is performed. Blobs -// will be left in the kv store as orphaned blobs. The trillian tree is -// append only so once a leaf is appended, it's there permanently. If steps -// 1 and 2 are successful then a recordIndex will be created, saved to the -// kv store, and appended onto the trillian tree. -// -// Appending a recordIndex onto the trillian tree is the last operation that -// occurs during a record update. If a recordIndex exists in the tree then the -// update is considered successful. Any record content leaves that are not part -// of a recordIndex are considered to be orphaned and can be disregarded. -type recordIndex struct { - // Version represents the version of the record. The version is - // only incremented when the record files are updated. Metadata - // only updates do no increment the version. - Version uint32 `json:"version"` - - // Iteration represents the iteration of the record. The iteration - // is incremented anytime any record content changes. This includes - // file changes that bump the version as well metadata stream and - // record metadata changes that don't bump the version. - // - // Note, this field is not the same as the backend RecordMetadata - // iteration field, which does not get incremented on metadata - // updates. - // - // TODO maybe it should be the same. The original iteration field - // was to track unvetted changes in gitbe since unvetted gitbe - // records are not versioned. tlogbe unvetted records are versioned - // so the original use for the iteration field isn't needed anymore. - Iteration uint32 `json:"iteration"` - - // The following fields contain the merkle leaf hashes of the - // trillian log leaves for the record content. The merkle leaf hash - // can be used to lookup the log leaf. The log leaf ExtraData field - // contains the key for the record content in the key-value store. - RecordMetadata []byte `json:"recordmetadata"` - Metadata map[uint64][]byte `json:"metadata"` // [metadataID]merkle - Files map[string][]byte `json:"files"` // [filename]merkle - - // Frozen is used to indicate that the tree for this record has - // been frozen. This happens as a result of certain record status - // changes. The only thing that can be appended onto a frozen tree - // is one additional anchor record. Once a frozen tree has been - // anchored, the tlog fsck function will update the status of the - // tree to frozen in trillian, at which point trillian will not - // allow any additional leaves to be appended onto the tree. - Frozen bool `json:"frozen,omitempty"` - - // TODO make this a generic ExtraData field - // TreePointer is the tree ID of the tree that is the new location - // of this record. A record can be copied to a new tree after - // certain status changes, such as when a record is made public and - // the record is copied from an unvetted tree to a vetted tree. - // TreePointer should only be set if the tree has been frozen. - TreePointer int64 `json:"treepointer,omitempty"` -} - -func treePointerExists(r recordIndex) bool { - // Sanity checks - switch { - case !r.Frozen && r.TreePointer > 0: - // Tree pointer should only be set if the record is frozen - e := fmt.Sprintf("tree pointer set without record being frozen %v", - r.TreePointer) - panic(e) - case r.TreePointer < 0: - // Tree pointer should never be negative. Trillian uses a int64 - // for the tree ID so we do too. - e := fmt.Sprintf("tree pointer is < 0: %v", r.TreePointer) - panic(e) - } - - return r.TreePointer > 0 -} - // blobIsEncrypted returns whether the provided blob has been prefixed with an // sbox header, indicating that it is an encrypted blob. func blobIsEncrypted(b []byte) bool { return bytes.HasPrefix(b, []byte("sbox")) } -func leafIsRecordIndex(l *trillian.LogLeaf) bool { - return bytes.HasPrefix(l.ExtraData, []byte(keyPrefixRecordIndex)) -} - -func leafIsRecordContent(l *trillian.LogLeaf) bool { - return bytes.HasPrefix(l.ExtraData, []byte(keyPrefixRecordContent)) -} - -func leafIsAnchor(l *trillian.LogLeaf) bool { - return bytes.HasPrefix(l.ExtraData, []byte(keyPrefixAnchorRecord)) -} - func leafExtraData(dataType, storeKey string) []byte { return []byte(dataType + ":" + storeKey) } @@ -370,17 +268,26 @@ func convertAnchorFromBlobEntry(be store.BlobEntry) (*anchor, error) { return &a, nil } -func (t *Tlog) encrypt(b []byte) ([]byte, error) { - if t.encryptionKey == nil { - return nil, fmt.Errorf("cannot encrypt blob; encryption key "+ - "not set for tlog instance %v", t.id) +func (t *Tlog) blobify(be store.BlobEntry) ([]byte, error) { + b, err := store.Blobify(be) + if err != nil { + return nil, err } - return t.encryptionKey.encrypt(0, b) + if t.encryptionKey != nil { + b, err = t.encryptionKey.encrypt(0, b) + if err != nil { + return nil, err + } + } + return b, nil } func (t *Tlog) deblob(b []byte) (*store.BlobEntry, error) { var err error - if t.encryptionKey != nil && blobIsEncrypted(b) { + if t.encryptionKey != nil { + if !blobIsEncrypted(b) { + return nil, fmt.Errorf("attempted to decrypt an unecrypted blob") + } b, _, err = t.encryptionKey.decrypt(b) if err != nil { return nil, err @@ -388,11 +295,6 @@ func (t *Tlog) deblob(b []byte) (*store.BlobEntry, error) { } be, err := store.Deblob(b) if err != nil { - // Check if this is an encrypted blob that was not decrypted - if t.encryptionKey == nil && blobIsEncrypted(b) { - return nil, fmt.Errorf("blob is encrypted but no encryption " + - "key found to decrypt blob") - } return nil, err } return be, nil @@ -450,7 +352,7 @@ func (t *Tlog) TreeFreeze(treeID int64, rm backend.RecordMetadata, metadata []ba if err != nil { return err } - b, err := store.Blobify(*be) + b, err := t.blobify(*be) if err != nil { return err } @@ -465,8 +367,9 @@ func (t *Tlog) TreeFreeze(treeID int64, rm backend.RecordMetadata, metadata []ba } // Append record index leaf to the trillian tree + extraData := leafExtraData(dataTypeRecordIndex, keys[0]) leaves := []*trillian.LogLeaf{ - newLogLeaf(idxHash, []byte(keyPrefixRecordIndex+keys[0])), + newLogLeaf(idxHash, extraData), } queued, _, err := t.trillian.leavesAppend(treeID, leaves) if err != nil { @@ -553,176 +456,6 @@ func (t *Tlog) treeIsFrozen(leaves []*trillian.LogLeaf) bool { return r.Frozen } -func (t *Tlog) recordIndexSave(treeID int64, ri recordIndex) error { - // Save record index to the store - be, err := convertBlobEntryFromRecordIndex(ri) - if err != nil { - return err - } - b, err := store.Blobify(*be) - if err != nil { - return err - } - keys, err := t.store.Put([][]byte{b}) - if err != nil { - return fmt.Errorf("store Put: %v", err) - } - if len(keys) != 1 { - return fmt.Errorf("wrong number of keys: got %v, want 1", - len(keys)) - } - - // Append record index leaf to trillian tree - h, err := hex.DecodeString(be.Hash) - if err != nil { - return err - } - leaves := []*trillian.LogLeaf{ - newLogLeaf(h, []byte(keyPrefixRecordIndex+keys[0])), - } - queued, _, err := t.trillian.leavesAppend(treeID, leaves) - if err != nil { - return fmt.Errorf("leavesAppend: %v", err) - } - if len(queued) != 1 { - return fmt.Errorf("wrong number of queud leaves: got %v, want 1", - len(queued)) - } - failed := make([]string, 0, len(queued)) - for _, v := range queued { - c := codes.Code(v.QueuedLeaf.GetStatus().GetCode()) - if c != codes.OK { - failed = append(failed, fmt.Sprintf("%v", c)) - } - } - if len(failed) > 0 { - return fmt.Errorf("append leaves failed: %v", failed) - } - - return nil -} - -// recordIndexVersion takes a list of record indexes for a record and returns -// the most recent iteration of the specified version. A version of 0 indicates -// that the latest version should be returned. A backend.ErrRecordNotFound is -// returned if the provided version does not exist. -func recordIndexVersion(indexes []recordIndex, version uint32) (*recordIndex, error) { - // Return the record index for the specified version - var ri *recordIndex - if version == 0 { - // A version of 0 indicates that the most recent version should - // be returned. - ri = &indexes[len(indexes)-1] - } else { - // Walk the indexes backwards so the most recent iteration of the - // specified version is selected. - for i := len(indexes) - 1; i >= 0; i-- { - r := indexes[i] - if r.Version == version { - ri = &r - break - } - } - } - if ri == nil { - // The specified version does not exist - return nil, backend.ErrRecordNotFound - } - - return ri, nil -} - -func (t *Tlog) recordIndexVersion(leaves []*trillian.LogLeaf, version uint32) (*recordIndex, error) { - indexes, err := t.recordIndexes(leaves) - if err != nil { - return nil, err - } - - return recordIndexVersion(indexes, version) -} - -func (t *Tlog) recordIndexLatest(leaves []*trillian.LogLeaf) (*recordIndex, error) { - return t.recordIndexVersion(leaves, 0) -} - -func (t *Tlog) recordIndexes(leaves []*trillian.LogLeaf) ([]recordIndex, error) { - // Walk the leaves and compile the keys for all record indexes. It - // is possible for multiple indexes to exist for the same record - // version (they will have different iterations due to metadata - // only updates) so we have to pull the index blobs from the store - // in order to find the most recent iteration for the specified - // version. - keys := make([]string, 0, 64) - for _, v := range leaves { - if leafIsRecordIndex(v) { - // This is a record index leaf. Save the kv store key. - keys = append(keys, extractKeyFromLeaf(v)) - } - } - - if len(keys) == 0 { - // No records have been added to this tree yet - return nil, backend.ErrRecordNotFound - } - - // Get record indexes from store - blobs, err := t.store.Get(keys) - if err != nil { - return nil, fmt.Errorf("store Get: %v", err) - } - missing := make([]string, 0, len(keys)) - for _, v := range keys { - if _, ok := blobs[v]; !ok { - missing = append(missing, v) - } - } - if len(missing) > 0 { - return nil, fmt.Errorf("record index not found: %v", missing) - } - - indexes := make([]recordIndex, 0, len(blobs)) - for _, v := range blobs { - be, err := t.deblob(v) - if err != nil { - return nil, err - } - ri, err := convertRecordIndexFromBlobEntry(*be) - if err != nil { - return nil, err - } - indexes = append(indexes, *ri) - } - - // Sort indexes by iteration, smallest to largets. The leaves - // ordering was not preserved in the returned blobs map. - sort.SliceStable(indexes, func(i, j int) bool { - return indexes[i].Iteration < indexes[j].Iteration - }) - - // Sanity check. Index iterations should start with 1 and be - // sequential. Index versions should start with 1 and also be - // sequential, but duplicate versions can exist as long as the - // iteration has been incremented. - var versionPrev uint32 - var i uint32 = 1 - for _, v := range indexes { - if v.Iteration != i { - return nil, fmt.Errorf("invalid record index iteration: "+ - "got %v, want %v", v.Iteration, i) - } - diff := v.Version - versionPrev - if diff != 0 && diff != 1 { - return nil, fmt.Errorf("invalid record index version: "+ - "curr version %v, prev version %v", v.Version, versionPrev) - } - - i++ - versionPrev = v.Version - } - - return indexes, nil -} - type recordHashes struct { recordMetadata string // Record metadata hash metadata map[string]uint64 // [hash]metadataID @@ -875,7 +608,7 @@ func (t *Tlog) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backen if err != nil { return nil, err } - b, err = store.Blobify(*be) + b, err = t.blobify(*be) if err != nil { return nil, err } @@ -896,7 +629,7 @@ func (t *Tlog) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backen if err != nil { return nil, err } - b, err := store.Blobify(*be) + b, err := t.blobify(*be) if err != nil { return nil, err } @@ -918,17 +651,10 @@ func (t *Tlog) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backen if err != nil { return nil, err } - b, err := store.Blobify(*be) + b, err := t.blobify(*be) if err != nil { return nil, err } - // Encypt file blobs if encryption key has been set - if t.encryptionKey != nil { - b, err = t.encrypt(b) - if err != nil { - return nil, err - } - } _, ok := dups[be.Hash] if !ok { // Not a duplicate. Save blob to the store. @@ -971,8 +697,8 @@ func (t *Tlog) recordBlobsSave(treeID int64, rbpr recordBlobsPrepareReply) (*rec // Prepare log leaves. hashes and keys share the same ordering. leaves := make([]*trillian.LogLeaf, 0, len(blobs)) for k := range blobs { - pk := []byte(keyPrefixRecordContent + keys[k]) - leaves = append(leaves, newLogLeaf(hashes[k], pk)) + extraData := leafExtraData(dataTypeRecordContent, keys[k]) + leaves = append(leaves, newLogLeaf(hashes[k], extraData)) } // Append leaves to trillian tree @@ -1390,7 +1116,7 @@ func (t *Tlog) Record(treeID int64, version uint32) (*backend.Record, error) { if err != nil { return nil, err } - idxLatest, err := recordIndexVersion(indexes, 0) + idxLatest, err := parseRecordIndex(indexes, 0) if err != nil { return nil, err } @@ -1401,7 +1127,7 @@ func (t *Tlog) Record(treeID int64, version uint32) (*backend.Record, error) { // Use the record index to pull the record content from the store. // The keys for the record content first need to be extracted from // their associated log leaf. - idx, err := recordIndexVersion(indexes, version) + idx, err := parseRecordIndex(indexes, version) if err != nil { return nil, err } @@ -1671,7 +1397,7 @@ func (t *Tlog) RecordTimestamps(treeID int64, version uint32, token []byte) (*ba if err != nil { return nil, fmt.Errorf("leavesAll %v: %v", treeID, err) } - idx, err := t.recordIndexVersion(leaves, version) + idx, err := t.recordIndex(leaves, version) if err != nil { return nil, err } diff --git a/politeiad/backend/tlogbe/tlog/tlogclient.go b/politeiad/backend/tlogbe/tlog/tlogclient.go index 3cc519806..c43be5dc7 100644 --- a/politeiad/backend/tlogbe/tlog/tlogclient.go +++ b/politeiad/backend/tlogbe/tlog/tlogclient.go @@ -7,6 +7,7 @@ package tlog import ( "encoding/hex" "fmt" + "strings" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/store" @@ -28,24 +29,21 @@ var ( func (t *Tlog) BlobSave(treeID int64, dataType string, be store.BlobEntry) error { log.Tracef("%v BlobSave: %v %v", t.id, treeID, dataType) + // Verify data type + if strings.Contains(dataType, dataTypeSeperator) { + return fmt.Errorf("data type cannot contain '%v'", dataTypeSeperator) + } + // Prepare blob and digest digest, err := hex.DecodeString(be.Hash) if err != nil { return err } - blob, err := store.Blobify(be) + blob, err := t.blobify(be) if err != nil { return err } - // Encrypt blob if an encryption key has been set - if t.encryptionKey != nil { - blob, err = t.encrypt(blob) - if err != nil { - return err - } - } - // Verify tree exists if !t.TreeExists(treeID) { return backend.ErrRecordNotFound diff --git a/politeiad/backend/tlogbe/tlog/timestamp.go b/politeiad/backend/tlogbe/tlog/verify.go similarity index 100% rename from politeiad/backend/tlogbe/tlog/timestamp.go rename to politeiad/backend/tlogbe/tlog/verify.go diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 2bf6a5a52..3088fdb68 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -90,10 +90,6 @@ type recordInventory struct { vetted map[backend.MDStatusT][]string } -func tokenIsFullLength(token []byte) bool { - return len(token) == v1.TokenSizeTlog -} - func tokenFromTreeID(treeID int64) []byte { b := make([]byte, v1.TokenSizeTlog) // Converting between int64 and uint64 doesn't change @@ -103,7 +99,7 @@ func tokenFromTreeID(treeID int64) []byte { } func treeIDFromToken(token []byte) int64 { - if !tokenIsFullLength(token) { + if !util.TokenIsFullLength(util.TokenTypeTlog, token) { return 0 } return int64(binary.LittleEndian.Uint64(token)) @@ -752,7 +748,7 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ // Verify token is valid. The full length token must be used when // writing data. - if !tokenIsFullLength(token) { + if !util.TokenIsFullLength(util.TokenTypeTlog, token) { return nil, backend.ContentVerificationError{ ErrorCode: v1.ErrorStatusInvalidToken, } @@ -856,7 +852,7 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b // Verify token is valid. The full length token must be used when // writing data. - if !tokenIsFullLength(token) { + if !util.TokenIsFullLength(util.TokenTypeTlog, token) { return nil, backend.ContentVerificationError{ ErrorCode: v1.ErrorStatusInvalidToken, } @@ -963,7 +959,7 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite // Verify token is valid. The full length token must be used when // writing data. - if !tokenIsFullLength(token) { + if !util.TokenIsFullLength(util.TokenTypeTlog, token) { return backend.ContentVerificationError{ ErrorCode: v1.ErrorStatusInvalidToken, } @@ -1055,7 +1051,7 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ // Verify token is valid. The full length token must be used when // writing data. - if !tokenIsFullLength(token) { + if !util.TokenIsFullLength(util.TokenTypeTlog, token) { return backend.ContentVerificationError{ ErrorCode: v1.ErrorStatusInvalidToken, } @@ -1207,7 +1203,7 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, // Verify token is valid. The full length token must be used when // writing data. - if !tokenIsFullLength(token) { + if !util.TokenIsFullLength(util.TokenTypeTlog, token) { return nil, backend.ContentVerificationError{ ErrorCode: v1.ErrorStatusInvalidToken, } @@ -1362,7 +1358,7 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md // Verify token is valid. The full length token must be used when // writing data. - if !tokenIsFullLength(token) { + if !util.TokenIsFullLength(util.TokenTypeTlog, token) { return nil, backend.ContentVerificationError{ ErrorCode: v1.ErrorStatusInvalidToken, } diff --git a/util/token.go b/util/token.go index a87d446d5..0950f38d1 100644 --- a/util/token.go +++ b/util/token.go @@ -16,7 +16,9 @@ var ( TokenTypeTlog = "tlog" ) -func tokenIsFullLength(tokenType string, token []byte) bool { +// TokenIsFullLength returns whether a token is a valid, full length politeiad +// censorship token. +func TokenIsFullLength(tokenType string, token []byte) bool { switch tokenType { case TokenTypeTlog: return len(token) == pdv1.TokenSizeTlog @@ -28,6 +30,7 @@ func tokenIsFullLength(tokenType string, token []byte) bool { } } +// TokenPrefixSize returns the size (in bytes) of a politeiad token prefix. func TokenPrefixSize() int { // If the token prefix length is an odd number of characters then // padding would have needed to be added to it prior to decoding it @@ -59,7 +62,7 @@ func TokenDecode(tokenType, token string) ([]byte, error) { } // Verify token is full length - if !tokenIsFullLength(tokenType, t) { + if !TokenIsFullLength(tokenType, t) { return nil, fmt.Errorf("invalid token size") } @@ -84,9 +87,9 @@ func TokenDecodeAnyLength(tokenType, token string) ([]byte, error) { case len(t) == TokenPrefixSize(): // This is a token prefix. Token prefixes are the same size // regardless of token type. - case tokenIsFullLength(TokenTypeGit, t): + case TokenIsFullLength(TokenTypeGit, t): // Token is a valid git backend token - case tokenIsFullLength(TokenTypeTlog, t): + case TokenIsFullLength(TokenTypeTlog, t): // Token is a valid tlog backend token default: return nil, fmt.Errorf("invalid token size") From 411170e07f606e57bee3188af2f7834732c6efb5 Mon Sep 17 00:00:00 2001 From: luke Date: Sat, 16 Jan 2021 22:00:39 -0600 Subject: [PATCH 230/449] tlogbe: Plugins update. --- politeiad/backend/backend.go | 20 - .../tlogbe/plugins/comments/comments.go | 195 ++-- .../tlogbe/plugins/comments/indexes.go | 17 +- .../backend/tlogbe/plugins/dcrdata/dcrdata.go | 4 +- .../backend/tlogbe/plugins/pi/linkedfrom.go | 8 +- politeiad/backend/tlogbe/plugins/pi/pi.go | 14 +- .../tlogbe/plugins/ticketvote/inventory.go | 269 ++++++ .../backend/tlogbe/plugins/ticketvote/log.go | 25 + .../tlogbe/plugins/ticketvote/ticketvote.go | 867 +++++------------- politeiad/backend/tlogbe/store/store.go | 6 +- politeiad/backend/tlogbe/tlog/anchor.go | 4 +- politeiad/backend/tlogbe/tlog/plugin.go | 27 +- politeiad/backend/tlogbe/tlog/recordindex.go | 4 +- politeiad/backend/tlogbe/tlog/tlog.go | 44 +- politeiad/backend/tlogbe/tlog/tlogclient.go | 2 +- politeiad/backend/tlogbe/tlogbe.go | 47 +- .../backend/tlogbe/tlogclient/tlogclient.go | 3 +- politeiad/log.go | 20 +- politeiad/plugins/ticketvote/ticketvote.go | 233 +---- util/signature.go | 8 +- util/token.go | 4 +- 21 files changed, 759 insertions(+), 1062 deletions(-) create mode 100644 politeiad/backend/tlogbe/plugins/ticketvote/inventory.go create mode 100644 politeiad/backend/tlogbe/plugins/ticketvote/log.go diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index a3e3c9252..96a8bac59 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -314,23 +314,3 @@ type Backend interface { // Close performs cleanup of the backend Close() } - -// BackendReadOnly provides a read only version of the Backend interface. -// Plugins interact with backend records using this interface. -type BackendReadOnly interface { - // Unvetted exists returns whether an unvetted record exists. - UnvettedExists(token []byte) bool - - // Vetted exists returns whether a vetted record exists. - VettedExists(token []byte) bool - - // GetUnvetted returns an unvetted record. - GetUnvetted(token []byte, version string) (*Record, error) - - // GetVetted returns a vetted record. - GetVetted(token []byte, version string) (*Record, error) - - // InventoryByStatus returns the record tokens of all records in the - // inventory categorized by MDStatusT. - InventoryByStatus() (*InventoryByStatus, error) -} diff --git a/politeiad/backend/tlogbe/plugins/comments/comments.go b/politeiad/backend/tlogbe/plugins/comments/comments.go index bcfdc561b..288bb161b 100644 --- a/politeiad/backend/tlogbe/plugins/comments/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments/comments.go @@ -15,15 +15,14 @@ import ( "path/filepath" "sort" "strconv" - "strings" "sync" "time" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/clients" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/politeiad/backend/tlogbe/store" + "github.com/decred/politeia/politeiad/backend/tlogbe/tlogclient" "github.com/decred/politeia/politeiad/plugins/comments" "github.com/decred/politeia/util" ) @@ -34,20 +33,14 @@ import ( const ( // Blob entry data descriptors - dataDescriptorCommentAdd = "commentadd" - dataDescriptorCommentDel = "commentdel" - dataDescriptorCommentVote = "commentvote" - - // Prefixes that are appended to key-value store keys before - // storing them in the log leaf ExtraData field. - keyPrefixCommentAdd = "commentadd:" - keyPrefixCommentDel = "commentdel:" - keyPrefixCommentVote = "commentvote:" - - // Filenames of cached data saved to the plugin data dir. Brackets - // are used to indicate a variable that should be replaced in the - // filename. - filenameRecordIndex = "{tokenPrefix}-index.json" + dataDescriptorCommentAdd = "cadd_v1" + dataDescriptorCommentDel = "cdel_v1" + dataDescriptorCommentVote = "cvote_v1" + + // Data types + dataTypeCommentAdd = "cadd" + dataTypeCommentDel = "cdel" + dataTypeCommentVote = "cvote" ) var ( @@ -59,7 +52,7 @@ var ( // commentsPlugin satisfies the plugins.Client interface. type commentsPlugin struct { sync.Mutex - tlog clients.TlogClient + tlog tlogclient.Client // dataDir is the comments plugin data directory. The only data // that is stored here is cached data that can be re-created at any @@ -110,7 +103,7 @@ func convertSignatureError(err error) backend.PluginUserError { return backend.PluginUserError{ PluginID: comments.ID, ErrorCode: int(s), - ErrorContext: strings.Join(e.ErrorContext, ", "), + ErrorContext: e.ErrorContext, } } @@ -186,13 +179,13 @@ func convertCommentAddFromBlobEntry(be store.BlobEntry) (*comments.CommentAdd, e if err != nil { return nil, fmt.Errorf("decode Data: %v", err) } - hash, err := hex.DecodeString(be.Hash) + digest, err := hex.DecodeString(be.Digest) if err != nil { - return nil, fmt.Errorf("decode hash: %v", err) + return nil, fmt.Errorf("decode digest: %v", err) } - if !bytes.Equal(util.Digest(b), hash) { + if !bytes.Equal(util.Digest(b), digest) { return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), hash) + util.Digest(b), digest) } var c comments.CommentAdd err = json.Unmarshal(b, &c) @@ -224,13 +217,13 @@ func convertCommentDelFromBlobEntry(be store.BlobEntry) (*comments.CommentDel, e if err != nil { return nil, fmt.Errorf("decode Data: %v", err) } - hash, err := hex.DecodeString(be.Hash) + digest, err := hex.DecodeString(be.Digest) if err != nil { - return nil, fmt.Errorf("decode hash: %v", err) + return nil, fmt.Errorf("decode digest: %v", err) } - if !bytes.Equal(util.Digest(b), hash) { + if !bytes.Equal(util.Digest(b), digest) { return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), hash) + util.Digest(b), digest) } var c comments.CommentDel err = json.Unmarshal(b, &c) @@ -262,13 +255,13 @@ func convertCommentVoteFromBlobEntry(be store.BlobEntry) (*comments.CommentVote, if err != nil { return nil, fmt.Errorf("decode Data: %v", err) } - hash, err := hex.DecodeString(be.Hash) + digest, err := hex.DecodeString(be.Digest) if err != nil { - return nil, fmt.Errorf("decode hash: %v", err) + return nil, fmt.Errorf("decode digest: %v", err) } - if !bytes.Equal(util.Digest(b), hash) { + if !bytes.Equal(util.Digest(b), digest) { return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), hash) + util.Digest(b), digest) } var cv comments.CommentVote err = json.Unmarshal(b, &cv) @@ -350,23 +343,27 @@ func (p *commentsPlugin) commentAddSave(treeID int64, ca comments.CommentAdd) ([ if err != nil { return nil, err } - merkle, err := p.tlog.BlobSave(treeID, keyPrefixCommentAdd, *be) + d, err := hex.DecodeString(be.Digest) if err != nil { return nil, err } - return merkle, nil + err = p.tlog.BlobSave(treeID, dataTypeCommentAdd, *be) + if err != nil { + return nil, err + } + return d, nil } -// commentAdds returns the commentAdd for all specified merkle hashes. -func (p *commentsPlugin) commentAdds(treeID int64, merkles [][]byte) ([]comments.CommentAdd, error) { +// commentAdds returns the commentAdd for all specified digests. +func (p *commentsPlugin) commentAdds(treeID int64, digests [][]byte) ([]comments.CommentAdd, error) { // Retrieve blobs - blobs, err := p.tlog.BlobsByMerkle(treeID, merkles) + blobs, err := p.tlog.Blobs(treeID, digests) if err != nil { return nil, err } - if len(blobs) != len(merkles) { + if len(blobs) != len(digests) { notFound := make([]string, 0, len(blobs)) - for _, v := range merkles { + for _, v := range digests { m := hex.EncodeToString(v) _, ok := blobs[m] if !ok { @@ -379,11 +376,7 @@ func (p *commentsPlugin) commentAdds(treeID int64, merkles [][]byte) ([]comments // Decode blobs adds := make([]comments.CommentAdd, 0, len(blobs)) for _, v := range blobs { - be, err := store.Deblob(v) - if err != nil { - return nil, err - } - c, err := convertCommentAddFromBlobEntry(*be) + c, err := convertCommentAddFromBlobEntry(v) if err != nil { return nil, err } @@ -398,22 +391,26 @@ func (p *commentsPlugin) commentDelSave(treeID int64, cd comments.CommentDel) ([ if err != nil { return nil, err } - merkle, err := p.tlog.BlobSave(treeID, keyPrefixCommentDel, *be) + d, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, err + } + err = p.tlog.BlobSave(treeID, dataTypeCommentDel, *be) if err != nil { return nil, err } - return merkle, nil + return d, nil } -func (p *commentsPlugin) commentDels(treeID int64, merkles [][]byte) ([]comments.CommentDel, error) { +func (p *commentsPlugin) commentDels(treeID int64, digests [][]byte) ([]comments.CommentDel, error) { // Retrieve blobs - blobs, err := p.tlog.BlobsByMerkle(treeID, merkles) + blobs, err := p.tlog.Blobs(treeID, digests) if err != nil { return nil, err } - if len(blobs) != len(merkles) { + if len(blobs) != len(digests) { notFound := make([]string, 0, len(blobs)) - for _, v := range merkles { + for _, v := range digests { m := hex.EncodeToString(v) _, ok := blobs[m] if !ok { @@ -426,15 +423,11 @@ func (p *commentsPlugin) commentDels(treeID int64, merkles [][]byte) ([]comments // Decode blobs dels := make([]comments.CommentDel, 0, len(blobs)) for _, v := range blobs { - be, err := store.Deblob(v) + d, err := convertCommentDelFromBlobEntry(v) if err != nil { return nil, err } - c, err := convertCommentDelFromBlobEntry(*be) - if err != nil { - return nil, err - } - dels = append(dels, *c) + dels = append(dels, *d) } return dels, nil @@ -445,22 +438,26 @@ func (p *commentsPlugin) commentVoteSave(treeID int64, cv comments.CommentVote) if err != nil { return nil, err } - merkle, err := p.tlog.BlobSave(treeID, keyPrefixCommentVote, *be) + d, err := hex.DecodeString(be.Digest) if err != nil { return nil, err } - return merkle, nil + err = p.tlog.BlobSave(treeID, dataTypeCommentVote, *be) + if err != nil { + return nil, err + } + return d, nil } -func (p *commentsPlugin) commentVotes(treeID int64, merkles [][]byte) ([]comments.CommentVote, error) { +func (p *commentsPlugin) commentVotes(treeID int64, digests [][]byte) ([]comments.CommentVote, error) { // Retrieve blobs - blobs, err := p.tlog.BlobsByMerkle(treeID, merkles) + blobs, err := p.tlog.Blobs(treeID, digests) if err != nil { return nil, err } - if len(blobs) != len(merkles) { + if len(blobs) != len(digests) { notFound := make([]string, 0, len(blobs)) - for _, v := range merkles { + for _, v := range digests { m := hex.EncodeToString(v) _, ok := blobs[m] if !ok { @@ -473,11 +470,7 @@ func (p *commentsPlugin) commentVotes(treeID int64, merkles [][]byte) ([]comment // Decode blobs votes := make([]comments.CommentVote, 0, len(blobs)) for _, v := range blobs { - be, err := store.Deblob(v) - if err != nil { - return nil, err - } - c, err := convertCommentVoteFromBlobEntry(*be) + c, err := convertCommentVoteFromBlobEntry(v) if err != nil { return nil, err } @@ -493,14 +486,14 @@ func (p *commentsPlugin) commentVotes(treeID int64, merkles [][]byte) ([]comment // responsibility of the caller to ensure a comment is returned for each of the // provided comment IDs. func (p *commentsPlugin) comments(treeID int64, ridx recordIndex, commentIDs []uint32) (map[uint32]comments.Comment, error) { - // Aggregate the merkle hashes for all records that need to be - // looked up. If a comment has been deleted then the only record - // that will still exist is the comment del record. If the comment - // has not been deleted then the comment add record will need to be + // Aggregate the digests for all records that need to be looked up. + // If a comment has been deleted then the only record that will + // still exist is the comment del record. If the comment has not + // been deleted then the comment add record will need to be // retrieved for the latest version of the comment. var ( - merkleAdds = make([][]byte, 0, len(commentIDs)) - merkleDels = make([][]byte, 0, len(commentIDs)) + digestAdds = make([][]byte, 0, len(commentIDs)) + digestDels = make([][]byte, 0, len(commentIDs)) ) for _, v := range commentIDs { cidx, ok := ridx.Comments[v] @@ -511,33 +504,33 @@ func (p *commentsPlugin) comments(treeID int64, ridx recordIndex, commentIDs []u // Comment del record if cidx.Del != nil { - merkleDels = append(merkleDels, cidx.Del) + digestDels = append(digestDels, cidx.Del) continue } // Comment add record version := commentVersionLatest(cidx) - merkleAdds = append(merkleAdds, cidx.Adds[version]) + digestAdds = append(digestAdds, cidx.Adds[version]) } // Get comment add records - adds, err := p.commentAdds(treeID, merkleAdds) + adds, err := p.commentAdds(treeID, digestAdds) if err != nil { return nil, fmt.Errorf("commentAdds: %v", err) } - if len(adds) != len(merkleAdds) { + if len(adds) != len(digestAdds) { return nil, fmt.Errorf("wrong comment adds count; got %v, want %v", - len(adds), len(merkleAdds)) + len(adds), len(digestAdds)) } // Get comment del records - dels, err := p.commentDels(treeID, merkleDels) + dels, err := p.commentDels(treeID, digestDels) if err != nil { return nil, fmt.Errorf("commentDels: %v", err) } - if len(dels) != len(merkleDels) { + if len(dels) != len(digestDels) { return nil, fmt.Errorf("wrong comment dels count; got %v, want %v", - len(dels), len(merkleDels)) + len(dels), len(digestDels)) } // Prepare comments @@ -572,9 +565,9 @@ func (p *commentsPlugin) comment(treeID int64, ridx recordIndex, commentID uint3 return &c, nil } -func (p *commentsPlugin) timestamp(treeID int64, merkle []byte) (*comments.Timestamp, error) { +func (p *commentsPlugin) timestamp(treeID int64, digest []byte) (*comments.Timestamp, error) { // Get timestamp - t, err := p.tlog.Timestamp(treeID, merkle) + t, err := p.tlog.Timestamp(treeID, digest) if err != nil { return nil, err } @@ -733,7 +726,7 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str } // Save comment - merkleHash, err := p.commentAddSave(treeID, ca) + digest, err := p.commentAddSave(treeID, ca) if err != nil { return "", fmt.Errorf("commentAddSave: %v", err) } @@ -741,7 +734,7 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str // Update index ridx.Comments[ca.CommentID] = commentIndex{ Adds: map[uint32][]byte{ - 1: merkleHash, + 1: digest, }, Del: nil, Votes: make(map[string][]voteIndex), @@ -888,13 +881,13 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st } // Save comment - merkle, err := p.commentAddSave(treeID, ca) + digest, err := p.commentAddSave(treeID, ca) if err != nil { return "", fmt.Errorf("commentAddSave: %v", err) } // Update index - ridx.Comments[ca.CommentID].Adds[ca.Version] = merkle + ridx.Comments[ca.CommentID].Adds[ca.Version] = digest // Save index err = p.recordIndexSaveLocked(token, *ridx) @@ -999,7 +992,7 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str } // Save comment del - merkle, err := p.commentDelSave(treeID, cd) + digest, err := p.commentDelSave(treeID, cd) if err != nil { return "", fmt.Errorf("commentDelSave: %v", err) } @@ -1011,7 +1004,7 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str e := fmt.Sprintf("comment not found in index: %v", d.CommentID) panic(e) } - cidx.Del = merkle + cidx.Del = digest ridx.Comments[d.CommentID] = cidx // Save index @@ -1021,11 +1014,11 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str } // Delete all comment versions - merkles := make([][]byte, 0, len(cidx.Adds)) + digests := make([][]byte, 0, len(cidx.Adds)) for _, v := range cidx.Adds { - merkles = append(merkles, v) + digests = append(digests, v) } - err = p.tlog.BlobsDel(treeID, merkles) + err = p.tlog.BlobsDel(treeID, digests) if err != nil { return "", fmt.Errorf("del: %v", err) } @@ -1160,7 +1153,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st } // Save comment vote - merkle, err := p.commentVoteSave(treeID, cv) + digest, err := p.commentVoteSave(treeID, cv) if err != nil { return "", fmt.Errorf("commentVoteSave: %v", err) } @@ -1172,7 +1165,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st } votes = append(votes, voteIndex{ Vote: cv.Vote, - Merkle: merkle, + Digest: digest, }) cidx.Votes[cv.UserID] = votes @@ -1317,7 +1310,7 @@ func (p *commentsPlugin) cmdGetVersion(treeID int64, token []byte, payload strin ErrorContext: "comment has been deleted", } } - merkle, ok := cidx.Adds[gv.Version] + digest, ok := cidx.Adds[gv.Version] if !ok { e := fmt.Sprintf("comment %v does not have version %v", gv.CommentID, gv.Version) @@ -1329,7 +1322,7 @@ func (p *commentsPlugin) cmdGetVersion(treeID int64, token []byte, payload strin } // Get comment add record - adds, err := p.commentAdds(treeID, [][]byte{merkle}) + adds, err := p.commentAdds(treeID, [][]byte{digest}) if err != nil { return "", fmt.Errorf("commentAdds: %v", err) } @@ -1398,9 +1391,9 @@ func (p *commentsPlugin) cmdVotes(treeID int64, token []byte, payload string) (s return "", err } - // Compile the comment vote merkles for all votes that were cast + // Compile the comment vote digests for all votes that were cast // by the specified user. - merkles := make([][]byte, 0, 256) + digests := make([][]byte, 0, 256) for _, cidx := range ridx.Comments { voteIdxs, ok := cidx.Votes[v.UserID] if !ok { @@ -1410,12 +1403,12 @@ func (p *commentsPlugin) cmdVotes(treeID int64, token []byte, payload string) (s // User has cast votes on this comment for _, vidx := range voteIdxs { - merkles = append(merkles, vidx.Merkle) + digests = append(digests, vidx.Digest) } } // Lookup votes - votes, err := p.commentVotes(treeID, merkles) + votes, err := p.commentVotes(treeID, digests) if err != nil { return "", fmt.Errorf("commentVotes: %v", err) } @@ -1499,7 +1492,7 @@ func (p *commentsPlugin) cmdTimestamps(treeID int64, token []byte, payload strin ts = make([]comments.Timestamp, 0, len(cidx.Votes)) for _, votes := range cidx.Votes { for _, v := range votes { - t, err := p.timestamp(treeID, v.Merkle) + t, err := p.timestamp(treeID, v.Digest) if err != nil { return "", err } @@ -1537,7 +1530,7 @@ func (p *commentsPlugin) Setup() error { // // This function satisfies the plugins.Client interface. func (p *commentsPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { - log.Tracef("Cmd: %v %v %x %v", treeID, token, cmd, payload) + log.Tracef("Cmd: %v %x %v", treeID, token, cmd) switch cmd { case comments.CmdNew: @@ -1586,7 +1579,7 @@ func (p *commentsPlugin) Fsck() error { } // New returns a new comments plugin. -func New(tlog clients.TlogClient, settings []backend.PluginSetting, id *identity.FullIdentity, dataDir string) (*commentsPlugin, error) { +func New(tlog tlogclient.Client, settings []backend.PluginSetting, dataDir string, id *identity.FullIdentity) (*commentsPlugin, error) { // Setup comments plugin data dir dataDir = filepath.Join(dataDir, comments.ID) err := os.MkdirAll(dataDir, 0700) diff --git a/politeiad/backend/tlogbe/plugins/comments/indexes.go b/politeiad/backend/tlogbe/plugins/comments/indexes.go index 404125401..658f13113 100644 --- a/politeiad/backend/tlogbe/plugins/comments/indexes.go +++ b/politeiad/backend/tlogbe/plugins/comments/indexes.go @@ -16,17 +16,22 @@ import ( "github.com/decred/politeia/util" ) -// voteIndex contains the comment vote and the merkle leaf hash of the vote -// record. +const ( + // filenameRecordIndex is the file name of the record index that + // is saved to the comments plugin data dir. + filenameRecordIndex = "{tokenPrefix}-index.json" +) + +// voteIndex contains the comment vote and the digest of the vote record. type voteIndex struct { Vote comments.VoteT `json:"vote"` - Merkle []byte `json:"merkle"` // Merkle leaf hash + Digest []byte `json:"digest"` } -// commentIndex contains the merkle leaf hashes of all comment add, dels, and -// votes for a comment ID. +// commentIndex contains the digests of all comment add, dels, and votes for a +// comment ID. type commentIndex struct { - Adds map[uint32][]byte `json:"adds"` // [version]merkleLeafHash + Adds map[uint32][]byte `json:"adds"` // [version]digest Del []byte `json:"del"` // Votes contains the vote history for each uuid that voted on the diff --git a/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go b/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go index 17568cb4a..0f2de6235 100644 --- a/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go +++ b/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go @@ -227,7 +227,7 @@ func (p *dcrdataPlugin) txsTrimmedHTTP(txIDs []string) ([]v5.TrimmedTx, error) { // along with a status of StatusDisconnected. It is the callers responsibility // to determine if the stale best block should be used. func (p *dcrdataPlugin) cmdBestBlock(payload string) (string, error) { - log.Tracef("cmdBestBlock") + log.Tracef("cmdBestBlock: %v", payload) // Payload is empty. Nothing to decode. @@ -593,7 +593,7 @@ func (p *dcrdataPlugin) Setup() error { // // This function satisfies the plugins.Client interface. func (p *dcrdataPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { - log.Tracef("Cmd: %v %v", cmd, payload) + log.Tracef("Cmd: %v %x %v", treeID, token, cmd) switch cmd { case dcrdata.CmdBestBlock: diff --git a/politeiad/backend/tlogbe/plugins/pi/linkedfrom.go b/politeiad/backend/tlogbe/plugins/pi/linkedfrom.go index c1ad74677..f229de117 100644 --- a/politeiad/backend/tlogbe/plugins/pi/linkedfrom.go +++ b/politeiad/backend/tlogbe/plugins/pi/linkedfrom.go @@ -103,11 +103,11 @@ func (p *piPlugin) linkedFromAdd(parentToken, childToken string) error { defer p.Unlock() // Verify tokens - parent, err := util.TokenDecode(util.TokenTypeTlog, parentToken) + parent, err := tokenDecode(parentToken) if err != nil { return err } - _, err = util.TokenDecode(util.TokenTypeTlog, childToken) + _, err = tokenDecode(childToken) if err != nil { return err } @@ -134,11 +134,11 @@ func (p *piPlugin) linkedFromDel(parentToken, childToken string) error { defer p.Unlock() // Verify tokens - parent, err := util.TokenDecode(util.TokenTypeTlog, parentToken) + parent, err := tokenDecode(parentToken) if err != nil { return err } - _, err = util.TokenDecode(util.TokenTypeTlog, childToken) + _, err = tokenDecode(childToken) if err != nil { return err } diff --git a/politeiad/backend/tlogbe/plugins/pi/pi.go b/politeiad/backend/tlogbe/plugins/pi/pi.go index 090f1b80c..0802ad78e 100644 --- a/politeiad/backend/tlogbe/plugins/pi/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi/pi.go @@ -19,11 +19,12 @@ import ( "github.com/decred/dcrd/chaincfg/v3" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/clients" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" + "github.com/decred/politeia/politeiad/backend/tlogbe/tlogclient" "github.com/decred/politeia/politeiad/plugins/comments" "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/ticketvote" + "github.com/decred/politeia/util" ) var ( @@ -33,8 +34,8 @@ var ( // piPlugin satisfies the plugins.Client interface. type piPlugin struct { sync.Mutex - backend clients.BackendClient - tlog clients.TlogClient + backend backend.Backend + tlog tlogclient.Client activeNetParams *chaincfg.Params // dataDir is the pi plugin data directory. The only data that is @@ -49,6 +50,11 @@ func isRFP(pm pi.ProposalMetadata) bool { return pm.LinkBy != 0 } +// tokenDecode decodes a token string. +func tokenDecode(token string) ([]byte, error) { + return util.TokenDecode(util.TokenTypeTlog, token) +} + // decodeProposalMetadata decodes and returns the ProposalMetadata from the // provided backend files. If a ProposalMetadata is not found, nil is returned. func decodeProposalMetadata(files []backend.File) (*pi.ProposalMetadata, error) { @@ -1160,7 +1166,7 @@ func (p *piPlugin) Fsck() error { return nil } -func New(backend clients.BackendClient, tlog clients.TlogClient, settings []backend.PluginSetting, dataDir string, activeNetParams *chaincfg.Params) (*piPlugin, error) { +func New(backend backend.Backend, tlog tlogclient.Client, settings []backend.PluginSetting, dataDir string, activeNetParams *chaincfg.Params) (*piPlugin, error) { // Create plugin data directory dataDir = filepath.Join(dataDir, pi.ID) err := os.MkdirAll(dataDir, 0700) diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go b/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go new file mode 100644 index 000000000..12af1cef6 --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go @@ -0,0 +1,269 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ticketvote + +import ( + "fmt" + + "github.com/decred/politeia/politeiad/plugins/ticketvote" +) + +// voteInventory contains the record inventory categorized by vote status. The +// authorized and started lists are updated in real-time since ticket vote +// plugin commands initiate those actions. The unauthorized and finished lists +// are lazy loaded since those lists depends on external state. +type voteInventory struct { + unauthorized []string // Unauthorized tokens + authorized []string // Authorized tokens + started map[string]uint32 // [token]endHeight + finished []string // Finished tokens + bestBlock uint32 // Height of last inventory update +} + +func (p *ticketVotePlugin) inventorySetToAuthorized(token string) { + p.Lock() + defer p.Unlock() + + // Remove the token from the unauthorized list. The unauthorize + // list is lazy loaded so it may or may not exist. + var i int + var found bool + for k, v := range p.inv.unauthorized { + if v == token { + i = k + found = true + break + } + } + if found { + // Remove the token from unauthorized + u := p.inv.unauthorized + u = append(u[:i], u[i+1:]...) + p.inv.unauthorized = u + + log.Debugf("ticketvote: removed from unauthorized inv: %v", token) + } + + // Prepend the token to the authorized list + a := p.inv.authorized + a = append([]string{token}, a...) + p.inv.authorized = a + + log.Debugf("ticketvote: added to authorized inv: %v", token) +} + +func (p *ticketVotePlugin) inventorySetToUnauthorized(token string) { + p.Lock() + defer p.Unlock() + + // Remove the token from the authorized list if it exists. Going + // from authorized to unauthorized can happen when a vote + // authorization is revoked. + var i int + var found bool + for k, v := range p.inv.authorized { + if v == token { + i = k + found = true + break + } + } + if found { + // Remove the token from authorized + a := p.inv.authorized + a = append(a[:i], a[i+1:]...) + p.inv.authorized = a + + log.Debugf("ticketvote: removed from authorized inv: %v", token) + } + + // Prepend the token to the unauthorized list + u := p.inv.unauthorized + u = append([]string{token}, u...) + p.inv.unauthorized = u + + log.Debugf("ticketvote: added to unauthorized inv: %v", token) +} + +func (p *ticketVotePlugin) inventorySetToStarted(token string, t ticketvote.VoteT, endHeight uint32) { + p.Lock() + defer p.Unlock() + + switch t { + case ticketvote.VoteTypeStandard: + // Remove the token from the authorized list. The token should + // always be in the authorized list prior to the vote being + // started for standard votes so panicing when this is not the + // case is ok. + var i int + var found bool + for k, v := range p.inv.authorized { + if v == token { + i = k + found = true + break + } + } + if !found { + e := fmt.Sprintf("token not found in authorized list: %v", token) + panic(e) + } + + a := p.inv.authorized + a = append(a[:i], a[i+1:]...) + p.inv.authorized = a + + log.Debugf("ticketvote: removed from authorized inv: %v", token) + + case ticketvote.VoteTypeRunoff: + // A runoff vote does not require the submission votes be + // authorized prior to the vote starting. The token might be in + // the unauthorized list, but its also possible that its not + // since the unauthorized list is lazy loaded and it might not + // have been added yet. Remove it only if it is found. + var i int + var found bool + for k, v := range p.inv.unauthorized { + if v == token { + i = k + found = true + break + } + } + if found { + // Remove the token from unauthorized + u := p.inv.unauthorized + u = append(u[:i], u[i+1:]...) + p.inv.unauthorized = u + + log.Debugf("ticketvote: removed from unauthorized inv: %v", token) + } + + default: + e := fmt.Sprintf("invalid vote type %v", t) + panic(e) + } + + // Add the token to the started list + p.inv.started[token] = endHeight + + log.Debugf("ticketvote: added to started inv: %v", token) +} + +func (p *ticketVotePlugin) inventory(bestBlock uint32) (*voteInventory, error) { + p.Lock() + defer p.Unlock() + + // Check backend inventory for new records + invBackend, err := p.backend.InventoryByStatus() + if err != nil { + return nil, fmt.Errorf("InventoryByStatus: %v", err) + } + + // Find number of records in the vetted inventory + var vettedInvCount int + for _, tokens := range invBackend.Vetted { + vettedInvCount += len(tokens) + } + + // Find number of records in the vote inventory + voteInvCount := len(p.inv.unauthorized) + len(p.inv.authorized) + + len(p.inv.started) + len(p.inv.finished) + + // The vetted inventory count and the vote inventory count should + // be the same. If they're not then it means we there are records + // missing from vote inventory. + if vettedInvCount != voteInvCount { + // Records are missing from the vote inventory. Put all ticket + // vote inventory records into a map so we can easily find what + // backend records are missing. + all := make(map[string]struct{}, voteInvCount) + for _, v := range p.inv.unauthorized { + all[v] = struct{}{} + } + for _, v := range p.inv.authorized { + all[v] = struct{}{} + } + for k := range p.inv.started { + all[k] = struct{}{} + } + for _, v := range p.inv.finished { + all[v] = struct{}{} + } + + // Add missing records to the vote inventory + for _, tokens := range invBackend.Vetted { + for _, v := range tokens { + if _, ok := all[v]; ok { + // Record is already in the vote inventory + continue + } + // We can assume that the record vote status is unauthorized + // since it would have already been added to the vote + // inventory during the authorization request if one had + // occurred. + p.inv.unauthorized = append(p.inv.unauthorized, v) + + log.Debugf("ticketvote: added to unauthorized inv: %v", v) + } + } + } + + // The records are moved to their correct vote status category in + // the inventory on authorization, revoking the authorization, and + // on starting the vote. We can assume these lists are already + // up-to-date. The last thing we must check for is whether any + // votes have finished since the last inventory update. + + // Check if the inventory has been updated for this block height. + if p.inv.bestBlock == bestBlock { + // Inventory already updated. Nothing else to do. + goto reply + } + + // Inventory has not been updated for this block height. Check if + // any proposal votes have finished. + for token, endHeight := range p.inv.started { + if bestBlock >= endHeight { + // Vote has finished. Remove it from the started list. + delete(p.inv.started, token) + + log.Debugf("ticketvote: removed from started inv: %v", token) + + // Add it to the finished list + p.inv.finished = append(p.inv.finished, token) + + log.Debugf("ticketvote: added to finished inv: %v", token) + } + } + + // Update best block + p.inv.bestBlock = bestBlock + + log.Debugf("ticketvote: inv updated for best block %v", bestBlock) + +reply: + // Return a copy of the inventory + var ( + unauthorized = make([]string, len(p.inv.unauthorized)) + authorized = make([]string, len(p.inv.authorized)) + started = make(map[string]uint32, len(p.inv.started)) + finished = make([]string, len(p.inv.finished)) + ) + copy(unauthorized, p.inv.unauthorized) + copy(authorized, p.inv.authorized) + copy(finished, p.inv.finished) + for k, v := range p.inv.started { + started[k] = v + } + + return &voteInventory{ + unauthorized: unauthorized, + authorized: authorized, + started: started, + finished: finished, + bestBlock: p.inv.bestBlock, + }, nil +} diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/log.go b/politeiad/backend/tlogbe/plugins/ticketvote/log.go new file mode 100644 index 000000000..f775a3117 --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/ticketvote/log.go @@ -0,0 +1,25 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ticketvote + +import "github.com/decred/slog" + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log = slog.Disabled + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = slog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using slog. +func UseLogger(logger slog.Logger) { + log = logger +} diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go index fb26b6794..ae3c629ed 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go @@ -27,7 +27,9 @@ import ( "github.com/decred/dcrd/wire" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/politeiad/backend/tlogbe/store" + "github.com/decred/politeia/politeiad/backend/tlogbe/tlogclient" "github.com/decred/politeia/politeiad/plugins/dcrdata" "github.com/decred/politeia/politeiad/plugins/ticketvote" "github.com/decred/politeia/util" @@ -41,22 +43,21 @@ const ( // Filenames of cached data saved to the plugin data dir. Brackets // are used to indicate a variable that should be replaced in the // filename. - filenameSummary = "{token}-summary.json" + filenameSummary = "{tokenprefix}-summary.json" // Blob entry data descriptors - dataDescriptorAuthDetails = "authdetails" - dataDescriptorVoteDetails = "votedetails" - dataDescriptorCastVoteDetails = "castvotedetails" - - // Prefixes that are appended to key-value store keys before - // storing them in the log leaf ExtraData field. - keyPrefixAuthDetails = "authdetails:" - keyPrefixVoteDetails = "votedetails:" - keyPrefixCastVoteDetails = "castvotedetails:" + dataDescriptorAuthDetails = "authdetails_v1" + dataDescriptorVoteDetails = "votedetails_v1" + dataDescriptorCastVoteDetails = "castvotedetails_v1" + + // Data types + dataTypeAuthDetails = "authdetails" + dataTypeVoteDetails = "votedetails" + dataTypeCastVoteDetails = "castvotedetails" ) var ( - _ pluginClient = (*ticketVotePlugin)(nil) + _ plugins.Client = (*ticketVotePlugin)(nil) ) // TODO holding the lock before verifying the token can allow the mutexes to @@ -66,23 +67,23 @@ var ( // TODO verify all writes only accept full length tokens -// ticketVotePlugin satisfies the pluginClient interface. +// ticketVotePlugin satisfies the plugins.Client interface. type ticketVotePlugin struct { sync.Mutex backend backend.Backend - tlog tlogClient + tlog tlogclient.Client activeNetParams *chaincfg.Params - // Plugin settings - voteDurationMin uint32 // In blocks - voteDurationMax uint32 // In blocks - // dataDir is the ticket vote plugin data directory. The only data // that is stored here is cached data that can be re-created at any // time by walking the trillian trees. Ex, the vote summary once a // record vote has ended. dataDir string + // Plugin settings + voteDurationMin uint32 // In blocks + voteDurationMax uint32 // In blocks + // identity contains the full identity that the plugin uses to // create receipts, i.e. signatures of user provided data that // prove the backend received and processed a plugin command. @@ -106,262 +107,12 @@ type ticketVotePlugin struct { mutexes map[string]*sync.Mutex // [string]mutex } -// voteInventory contains the record inventory categorized by vote status. The -// authorized and started lists are updated in real-time since ticket vote -// plugin commands initiate those actions. The unauthorized and finished lists -// are lazy loaded since those lists depends on external state. -type voteInventory struct { - unauthorized []string // Unauthorized tokens - authorized []string // Authorized tokens - started map[string]uint32 // [token]endHeight - finished []string // Finished tokens - bestBlock uint32 // Height of last inventory update +func tokenDecode(token string) ([]byte, error) { + return util.TokenDecode(util.TokenTypeTlog, token) } -func (p *ticketVotePlugin) inventorySetToAuthorized(token string) { - p.Lock() - defer p.Unlock() - - // Remove the token from the unauthorized list. The unauthorize - // list is lazy loaded so it may or may not exist. - var i int - var found bool - for k, v := range p.inv.unauthorized { - if v == token { - i = k - found = true - break - } - } - if found { - // Remove the token from unauthorized - u := p.inv.unauthorized - u = append(u[:i], u[i+1:]...) - p.inv.unauthorized = u - - log.Debugf("ticketvote: removed from unauthorized inv: %v", token) - } - - // Prepend the token to the authorized list - a := p.inv.authorized - a = append([]string{token}, a...) - p.inv.authorized = a - - log.Debugf("ticketvote: added to authorized inv: %v", token) -} - -func (p *ticketVotePlugin) inventorySetToUnauthorized(token string) { - p.Lock() - defer p.Unlock() - - // Remove the token from the authorized list if it exists. Going - // from authorized to unauthorized can happen when a vote - // authorization is revoked. - var i int - var found bool - for k, v := range p.inv.authorized { - if v == token { - i = k - found = true - break - } - } - if found { - // Remove the token from authorized - a := p.inv.authorized - a = append(a[:i], a[i+1:]...) - p.inv.authorized = a - - log.Debugf("ticketvote: removed from authorized inv: %v", token) - } - - // Prepend the token to the unauthorized list - u := p.inv.unauthorized - u = append([]string{token}, u...) - p.inv.unauthorized = u - - log.Debugf("ticketvote: added to unauthorized inv: %v", token) -} - -func (p *ticketVotePlugin) inventorySetToStarted(token string, t ticketvote.VoteT, endHeight uint32) { - p.Lock() - defer p.Unlock() - - switch t { - case ticketvote.VoteTypeStandard: - // Remove the token from the authorized list. The token should - // always be in the authorized list prior to the vote being - // started for standard votes so panicing when this is not the - // case is ok. - var i int - var found bool - for k, v := range p.inv.authorized { - if v == token { - i = k - found = true - break - } - } - if !found { - e := fmt.Sprintf("token not found in authorized list: %v", token) - panic(e) - } - - a := p.inv.authorized - a = append(a[:i], a[i+1:]...) - p.inv.authorized = a - - log.Debugf("ticketvote: removed from authorized inv: %v", token) - - case ticketvote.VoteTypeRunoff: - // A runoff vote does not require the submission votes be - // authorized prior to the vote starting. The token might be in - // the unauthorized list, but its also possible that its not - // since the unauthorized list is lazy loaded and it might not - // have been added yet. Remove it only if it is found. - var i int - var found bool - for k, v := range p.inv.unauthorized { - if v == token { - i = k - found = true - break - } - } - if found { - // Remove the token from unauthorized - u := p.inv.unauthorized - u = append(u[:i], u[i+1:]...) - p.inv.unauthorized = u - - log.Debugf("ticketvote: removed from unauthorized inv: %v", token) - } - - default: - e := fmt.Sprintf("invalid vote type %v", t) - panic(e) - } - - // Add the token to the started list - p.inv.started[token] = endHeight - - log.Debugf("ticketvote: added to started inv: %v", token) -} - -func (p *ticketVotePlugin) inventory(bestBlock uint32) (*voteInventory, error) { - p.Lock() - defer p.Unlock() - - // Check backend inventory for new records - invBackend, err := p.backend.InventoryByStatus() - if err != nil { - return nil, fmt.Errorf("InventoryByStatus: %v", err) - } - - // Find number of records in the vetted inventory - var vettedInvCount int - for _, tokens := range invBackend.Vetted { - vettedInvCount += len(tokens) - } - - // Find number of records in the vote inventory - voteInvCount := len(p.inv.unauthorized) + len(p.inv.authorized) + - len(p.inv.started) + len(p.inv.finished) - - // The vetted inventory count and the vote inventory count should - // be the same. If they're not then it means we there are records - // missing from vote inventory. - if vettedInvCount != voteInvCount { - // Records are missing from the vote inventory. Put all ticket - // vote inventory records into a map so we can easily find what - // backend records are missing. - all := make(map[string]struct{}, voteInvCount) - for _, v := range p.inv.unauthorized { - all[v] = struct{}{} - } - for _, v := range p.inv.authorized { - all[v] = struct{}{} - } - for k := range p.inv.started { - all[k] = struct{}{} - } - for _, v := range p.inv.finished { - all[v] = struct{}{} - } - - // Add missing records to the vote inventory - for _, tokens := range invBackend.Vetted { - for _, v := range tokens { - if _, ok := all[v]; ok { - // Record is already in the vote inventory - continue - } - // We can assume that the record vote status is unauthorized - // since it would have already been added to the vote - // inventory during the authorization request if one had - // occurred. - p.inv.unauthorized = append(p.inv.unauthorized, v) - - log.Debugf("ticketvote: added to unauthorized inv: %v", v) - } - } - } - - // The records are moved to their correct vote status category in - // the inventory on authorization, revoking the authorization, and - // on starting the vote. We can assume these lists are already - // up-to-date. The last thing we must check for is whether any - // votes have finished since the last inventory update. - - // Check if the inventory has been updated for this block height. - if p.inv.bestBlock == bestBlock { - // Inventory already updated. Nothing else to do. - goto reply - } - - // Inventory has not been updated for this block height. Check if - // any proposal votes have finished. - for token, endHeight := range p.inv.started { - if bestBlock >= endHeight { - // Vote has finished. Remove it from the started list. - delete(p.inv.started, token) - - log.Debugf("ticketvote: removed from started inv: %v", token) - - // Add it to the finished list - p.inv.finished = append(p.inv.finished, token) - - log.Debugf("ticketvote: added to finished inv: %v", token) - } - } - - // Update best block - p.inv.bestBlock = bestBlock - - log.Debugf("ticketvote: inv updated for best block %v", bestBlock) - -reply: - // Return a copy of the inventory - var ( - unauthorized = make([]string, len(p.inv.unauthorized)) - authorized = make([]string, len(p.inv.authorized)) - started = make(map[string]uint32, len(p.inv.started)) - finished = make([]string, len(p.inv.finished)) - ) - copy(unauthorized, p.inv.unauthorized) - copy(authorized, p.inv.authorized) - copy(finished, p.inv.finished) - for k, v := range p.inv.started { - started[k] = v - } - - return &voteInventory{ - unauthorized: unauthorized, - authorized: authorized, - started: started, - finished: finished, - bestBlock: p.inv.bestBlock, - }, nil +func tokenDecodeAnyLength(token string) ([]byte, error) { + return util.TokenDecodeAnyLength(util.TokenTypeTlog, token) } func (p *ticketVotePlugin) cachedVotes(token []byte) map[string]string { @@ -413,11 +164,15 @@ func (p *ticketVotePlugin) cachedSummaryPath(token string) (string, error) { if err != nil { return "", err } - token = tokenPrefix(t) - fn := strings.Replace(filenameSummary, "{token}", token, 1) + token = util.TokenPrefix(t) + fn := strings.Replace(filenameSummary, "{tokenprefix}", token, 1) return filepath.Join(p.dataDir, fn), nil } +var ( + errSummaryNotFound = errors.New("summary not found") +) + func (p *ticketVotePlugin) cachedSummary(token string) (*ticketvote.Summary, error) { p.Lock() defer p.Unlock() @@ -431,7 +186,7 @@ func (p *ticketVotePlugin) cachedSummary(token string) (*ticketvote.Summary, err var e *os.PathError if errors.As(err, &e) && !os.IsExist(err) { // File does't exist - return nil, errRecordNotFound + return nil, errSummaryNotFound } return nil, err } @@ -483,7 +238,7 @@ func (p *ticketVotePlugin) mutex(token string) *sync.Mutex { return m } -func convertTicketVoteErrFromSignatureErr(err error) backend.PluginUserError { +func convertSignatureError(err error) backend.PluginUserError { var e util.SignatureError var s ticketvote.ErrorStatusT if errors.As(err, &e) { @@ -522,13 +277,13 @@ func convertAuthDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.AuthDetail if err != nil { return nil, fmt.Errorf("decode Data: %v", err) } - hash, err := hex.DecodeString(be.Hash) + digest, err := hex.DecodeString(be.Digest) if err != nil { - return nil, fmt.Errorf("decode hash: %v", err) + return nil, fmt.Errorf("decode digest: %v", err) } - if !bytes.Equal(util.Digest(b), hash) { + if !bytes.Equal(util.Digest(b), digest) { return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), hash) + util.Digest(b), digest) } var ad ticketvote.AuthDetails err = json.Unmarshal(b, &ad) @@ -560,13 +315,13 @@ func convertVoteDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.VoteDetail if err != nil { return nil, fmt.Errorf("decode Data: %v", err) } - hash, err := hex.DecodeString(be.Hash) + digest, err := hex.DecodeString(be.Digest) if err != nil { - return nil, fmt.Errorf("decode hash: %v", err) + return nil, fmt.Errorf("decode digest: %v", err) } - if !bytes.Equal(util.Digest(b), hash) { + if !bytes.Equal(util.Digest(b), digest) { return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), hash) + util.Digest(b), digest) } var vd ticketvote.VoteDetails err = json.Unmarshal(b, &vd) @@ -598,13 +353,13 @@ func convertCastVoteDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.CastVo if err != nil { return nil, fmt.Errorf("decode Data: %v", err) } - hash, err := hex.DecodeString(be.Hash) + digest, err := hex.DecodeString(be.Digest) if err != nil { - return nil, fmt.Errorf("decode hash: %v", err) + return nil, fmt.Errorf("decode digest: %v", err) } - if !bytes.Equal(util.Digest(b), hash) { + if !bytes.Equal(util.Digest(b), digest) { return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), hash) + util.Digest(b), digest) } var cv ticketvote.CastVoteDetails err = json.Unmarshal(b, &cv) @@ -666,44 +421,20 @@ func convertBlobEntryFromCastVoteDetails(cv ticketvote.CastVoteDetails) (*store. return &be, nil } -func (p *ticketVotePlugin) authSave(ad ticketvote.AuthDetails) error { - token, err := hex.DecodeString(ad.Token) - if err != nil { - return err - } - +func (p *ticketVotePlugin) authSave(treeID int64, ad ticketvote.AuthDetails) error { // Prepare blob be, err := convertBlobEntryFromAuthDetails(ad) if err != nil { return err } - h, err := hex.DecodeString(be.Hash) - if err != nil { - return err - } - b, err := store.Blobify(*be) - if err != nil { - return err - } // Save blob - merkles, err := p.tlog.save(tlogIDVetted, token, keyPrefixAuthDetails, - [][]byte{b}, [][]byte{h}, false) - if err != nil { - return fmt.Errorf("save: %v", err) - } - if len(merkles) != 1 { - return fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", - len(merkles)) - } - - return nil + return p.tlog.BlobSave(treeID, dataTypeAuthDetails, *be) } -func (p *ticketVotePlugin) auths(token []byte) ([]ticketvote.AuthDetails, error) { +func (p *ticketVotePlugin) auths(treeID int64) ([]ticketvote.AuthDetails, error) { // Retrieve blobs - blobs, err := p.tlog.blobsByKeyPrefix(tlogIDVetted, token, - keyPrefixAuthDetails) + blobs, err := p.tlog.BlobsByDataType(treeID, dataTypeAuthDetails) if err != nil { return nil, err } @@ -711,18 +442,15 @@ func (p *ticketVotePlugin) auths(token []byte) ([]ticketvote.AuthDetails, error) // Decode blobs auths := make([]ticketvote.AuthDetails, 0, len(blobs)) for _, v := range blobs { - be, err := store.Deblob(v) - if err != nil { - return nil, err - } - a, err := convertAuthDetailsFromBlobEntry(*be) + a, err := convertAuthDetailsFromBlobEntry(v) if err != nil { return nil, err } auths = append(auths, *a) } - // Sort from oldest to newest + // Sanity check. They should already be sorted from oldest to + // newest. sort.SliceStable(auths, func(i, j int) bool { return auths[i].Timestamp < auths[j].Timestamp }) @@ -730,44 +458,20 @@ func (p *ticketVotePlugin) auths(token []byte) ([]ticketvote.AuthDetails, error) return auths, nil } -func (p *ticketVotePlugin) voteDetailsSave(vd ticketvote.VoteDetails) error { - token, err := hex.DecodeString(vd.Params.Token) - if err != nil { - return err - } - +func (p *ticketVotePlugin) voteDetailsSave(treeID int64, vd ticketvote.VoteDetails) error { // Prepare blob be, err := convertBlobEntryFromVoteDetails(vd) if err != nil { return err } - h, err := hex.DecodeString(be.Hash) - if err != nil { - return err - } - b, err := store.Blobify(*be) - if err != nil { - return err - } // Save blob - merkles, err := p.tlog.save(tlogIDVetted, token, keyPrefixVoteDetails, - [][]byte{b}, [][]byte{h}, false) - if err != nil { - return fmt.Errorf("Save: %v", err) - } - if len(merkles) != 1 { - return fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", - len(merkles)) - } - - return nil + return p.tlog.BlobSave(treeID, dataTypeVoteDetails, *be) } -func (p *ticketVotePlugin) voteDetails(token []byte) (*ticketvote.VoteDetails, error) { +func (p *ticketVotePlugin) voteDetails(treeID int64) (*ticketvote.VoteDetails, error) { // Retrieve blobs - blobs, err := p.tlog.blobsByKeyPrefix(tlogIDVetted, token, - keyPrefixVoteDetails) + blobs, err := p.tlog.BlobsByDataType(treeID, dataTypeVoteDetails) if err != nil { return nil, err } @@ -780,16 +484,12 @@ func (p *ticketVotePlugin) voteDetails(token []byte) (*ticketvote.VoteDetails, e default: // This should not happen. There should only ever be a max of // one vote details. - return nil, fmt.Errorf("multiple vote detailss found (%v) for record %x", - len(blobs), token) + return nil, fmt.Errorf("multiple vote details found (%v) on %x", + len(blobs), treeID) } // Decode blob - be, err := store.Deblob(blobs[0]) - if err != nil { - return nil, err - } - vd, err := convertVoteDetailsFromBlobEntry(*be) + vd, err := convertVoteDetailsFromBlobEntry(blobs[0]) if err != nil { return nil, err } @@ -797,44 +497,20 @@ func (p *ticketVotePlugin) voteDetails(token []byte) (*ticketvote.VoteDetails, e return vd, nil } -func (p *ticketVotePlugin) castVoteSave(cv ticketvote.CastVoteDetails) error { - token, err := hex.DecodeString(cv.Token) - if err != nil { - return err - } - +func (p *ticketVotePlugin) castVoteSave(treeID int64, cv ticketvote.CastVoteDetails) error { // Prepare blob be, err := convertBlobEntryFromCastVoteDetails(cv) if err != nil { return err } - h, err := hex.DecodeString(be.Hash) - if err != nil { - return err - } - b, err := store.Blobify(*be) - if err != nil { - return err - } // Save blob - merkles, err := p.tlog.save(tlogIDVetted, token, keyPrefixCastVoteDetails, - [][]byte{b}, [][]byte{h}, false) - if err != nil { - return fmt.Errorf("save: %v", err) - } - if len(merkles) != 1 { - return fmt.Errorf("invalid merkle leaf hash count; got %v, want 1", - len(merkles)) - } - - return nil + return p.tlog.BlobSave(treeID, dataTypeCastVoteDetails, *be) } -func (p *ticketVotePlugin) castVotes(token []byte) ([]ticketvote.CastVoteDetails, error) { +func (p *ticketVotePlugin) castVotes(treeID int64) ([]ticketvote.CastVoteDetails, error) { // Retrieve blobs - blobs, err := p.tlog.blobsByKeyPrefix(tlogIDVetted, token, - keyPrefixCastVoteDetails) + blobs, err := p.tlog.BlobsByDataType(treeID, dataTypeCastVoteDetails) if err != nil { return nil, err } @@ -842,11 +518,7 @@ func (p *ticketVotePlugin) castVotes(token []byte) ([]ticketvote.CastVoteDetails // Decode blobs votes := make([]ticketvote.CastVoteDetails, 0, len(blobs)) for _, v := range blobs { - be, err := store.Deblob(v) - if err != nil { - return nil, err - } - cv, err := convertCastVoteDetailsFromBlobEntry(*be) + cv, err := convertCastVoteDetailsFromBlobEntry(v) if err != nil { return nil, err } @@ -861,10 +533,10 @@ func (p *ticketVotePlugin) castVotes(token []byte) ([]ticketvote.CastVoteDetails return votes, nil } -func (p *ticketVotePlugin) timestamp(token []byte, merkle []byte) (*ticketvote.Timestamp, error) { - t, err := p.tlog.timestamp(tlogIDVetted, token, merkle) +func (p *ticketVotePlugin) timestamp(treeID int64, digest []byte) (*ticketvote.Timestamp, error) { + t, err := p.tlog.Timestamp(treeID, digest) if err != nil { - return nil, fmt.Errorf("timestamp %x %x: %v", token, merkle, err) + return nil, fmt.Errorf("timestamp %v %x: %v", treeID, digest, err) } // Convert response @@ -891,7 +563,7 @@ func (p *ticketVotePlugin) timestamp(token []byte, merkle []byte) (*ticketvote.T // the dcrdata connection is not active, an error will be returned. func (p *ticketVotePlugin) bestBlock() (uint32, error) { // Get best block - payload, err := dcrdata.EncodeBestBlock(dcrdata.BestBlock{}) + payload, err := json.Marshal(dcrdata.BestBlock{}) if err != nil { return 0, err } @@ -903,7 +575,8 @@ func (p *ticketVotePlugin) bestBlock() (uint32, error) { } // Handle response - bbr, err := dcrdata.DecodeBestBlockReply([]byte(reply)) + var bbr dcrdata.BestBlockReply + err = json.Unmarshal([]byte(reply), &bbr) if err != nil { return 0, err } @@ -926,7 +599,7 @@ func (p *ticketVotePlugin) bestBlock() (uint32, error) { // block is not stale. func (p *ticketVotePlugin) bestBlockUnsafe() (uint32, error) { // Get best block - payload, err := dcrdata.EncodeBestBlock(dcrdata.BestBlock{}) + payload, err := json.Marshal(dcrdata.BestBlock{}) if err != nil { return 0, err } @@ -938,7 +611,8 @@ func (p *ticketVotePlugin) bestBlockUnsafe() (uint32, error) { } // Handle response - bbr, err := dcrdata.DecodeBestBlockReply([]byte(reply)) + var bbr dcrdata.BestBlockReply + err = json.Unmarshal([]byte(reply), &bbr) if err != nil { return 0, err } @@ -960,7 +634,7 @@ func (p *ticketVotePlugin) largestCommitmentAddrs(tickets []string) ([]commitmen tt := dcrdata.TxsTrimmed{ TxIDs: tickets, } - payload, err := dcrdata.EncodeTxsTrimmed(tt) + payload, err := json.Marshal(tt) if err != nil { return nil, err } @@ -970,7 +644,8 @@ func (p *ticketVotePlugin) largestCommitmentAddrs(tickets []string) ([]commitmen return nil, fmt.Errorf("Plugin %v %v: %v", dcrdata.ID, dcrdata.CmdTxsTrimmed, err) } - ttr, err := dcrdata.DecodeTxsTrimmedReply([]byte(reply)) + var ttr dcrdata.TxsTrimmedReply + err = json.Unmarshal([]byte(reply), &ttr) if err != nil { return nil, err } @@ -1029,7 +704,7 @@ func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, bd := dcrdata.BlockDetails{ Height: snapshotHeight, } - payload, err := dcrdata.EncodeBlockDetails(bd) + payload, err := json.Marshal(bd) if err != nil { return nil, err } @@ -1039,7 +714,8 @@ func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, return nil, fmt.Errorf("Plugin %v %v: %v", dcrdata.ID, dcrdata.CmdBlockDetails, err) } - bdr, err := dcrdata.DecodeBlockDetailsReply([]byte(reply)) + var bdr dcrdata.BlockDetailsReply + err = json.Unmarshal([]byte(reply), &bdr) if err != nil { return nil, err } @@ -1052,7 +728,7 @@ func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, tp := dcrdata.TicketPool{ BlockHash: snapshotHash, } - payload, err = dcrdata.EncodeTicketPool(tp) + payload, err = json.Marshal(tp) if err != nil { return nil, err } @@ -1062,7 +738,8 @@ func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, return nil, fmt.Errorf("Plugin %v %v: %v", dcrdata.ID, dcrdata.CmdTicketPool, err) } - tpr, err := dcrdata.DecodeTicketPoolReply([]byte(reply)) + var tpr dcrdata.TicketPoolReply + err = json.Unmarshal([]byte(reply), &tpr) if err != nil { return nil, err } @@ -1163,30 +840,40 @@ func (p *ticketVotePlugin) castVoteSignatureVerify(cv ticketvote.CastVote, addr return nil } -func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { - log.Tracef("ticketvote cmdAuthorize: %v", payload) +func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdAuthorize: %v %x %v", treeID, token, payload) // Decode payload - a, err := ticketvote.DecodeAuthorize([]byte(payload)) + var a ticketvote.Authorize + err := json.Unmarshal([]byte(payload), &a) if err != nil { return "", err } // Verify token - token, err := tokenDecode(a.Token) + t, err := tokenDecode(a.Token) if err != nil { return "", backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusTokenInvalid), } } + if !bytes.Equal(t, token) { + e := fmt.Sprintf("plugin token does not match route token: "+ + "got %x, want %x", t, token) + return "", backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorStatusTokenInvalid), + ErrorContext: e, + } + } // Verify signature version := strconv.FormatUint(uint64(a.Version), 10) msg := a.Token + version + string(a.Action) err = util.VerifySignature(a.Signature, a.PublicKey, msg) if err != nil { - return "", convertTicketVoteErrFromSignatureErr(err) + return "", convertSignatureError(err) } // Verify action @@ -1200,7 +887,7 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { return "", backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusAuthorizationInvalid), - ErrorContext: []string{e}, + ErrorContext: e, } } @@ -1213,7 +900,7 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { // Get any previous authorizations to verify that the new action // is allowed based on the previous action. - auths, err := p.auths(token) + auths, err := p.auths(treeID) if err != nil { return "", err } @@ -1228,7 +915,7 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { return "", backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusAuthorizationInvalid), - ErrorContext: []string{"no prev action; action must be authorize"}, + ErrorContext: "no prev action; action must be authorize", } } case prevAction == ticketvote.AuthActionAuthorize && @@ -1237,7 +924,7 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { return "", backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusAuthorizationInvalid), - ErrorContext: []string{"prev action was authorize"}, + ErrorContext: "prev action was authorize", } case prevAction == ticketvote.AuthActionRevoke && a.Action != ticketvote.AuthActionAuthorize: @@ -1245,7 +932,7 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { return "", backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusAuthorizationInvalid), - ErrorContext: []string{"prev action was revoke"}, + ErrorContext: "prev action was revoke", } } @@ -1262,7 +949,7 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { } // Save authorize vote - err = p.authSave(auth) + err = p.authSave(treeID, auth) if err != nil { return "", err } @@ -1284,7 +971,7 @@ func (p *ticketVotePlugin) cmdAuthorize(payload string) (string, error) { Timestamp: auth.Timestamp, Receipt: auth.Receipt, } - reply, err := ticketvote.EncodeAuthorizeReply(ar) + reply, err := json.Marshal(ar) if err != nil { return "", err } @@ -1329,7 +1016,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM return backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), - ErrorContext: []string{e}, + ErrorContext: e, } } @@ -1341,7 +1028,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM return backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), - ErrorContext: []string{e}, + ErrorContext: e, } case vote.Duration < voteDurationMin: e := fmt.Sprintf("duration %v under min duration %v", @@ -1349,7 +1036,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM return backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), - ErrorContext: []string{e}, + ErrorContext: e, } case vote.QuorumPercentage > 100: e := fmt.Sprintf("quorum percent %v exceeds 100 percent", @@ -1357,7 +1044,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM return backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), - ErrorContext: []string{e}, + ErrorContext: e, } case vote.PassPercentage > 100: e := fmt.Sprintf("pass percent %v exceeds 100 percent", @@ -1365,7 +1052,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM return backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), - ErrorContext: []string{e}, + ErrorContext: e, } } @@ -1375,7 +1062,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM return backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), - ErrorContext: []string{"no vote options found"}, + ErrorContext: "no vote options found", } } switch vote.Type { @@ -1389,7 +1076,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM return backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), - ErrorContext: []string{e}, + ErrorContext: e, } } // map[optionID]found @@ -1418,7 +1105,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM return backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), - ErrorContext: []string{e}, + ErrorContext: e, } } } @@ -1430,7 +1117,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM return backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), - ErrorContext: []string{err.Error()}, + ErrorContext: err.Error(), } } } @@ -1442,7 +1129,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM return backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), - ErrorContext: []string{e}, + ErrorContext: e, } case vote.Type == ticketvote.VoteTypeRunoff: _, err := tokenDecode(vote.Parent) @@ -1451,7 +1138,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM return backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), - ErrorContext: []string{e}, + ErrorContext: e, } } } @@ -1460,13 +1147,13 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM } // startStandard starts a standard vote. -func (p *ticketVotePlugin) startStandard(s ticketvote.Start) (*ticketvote.StartReply, error) { +func (p *ticketVotePlugin) startStandard(treeID int64, s ticketvote.Start) (*ticketvote.StartReply, error) { // Verify there is only one start details if len(s.Starts) != 1 { return nil, backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusStartDetailsInvalid), - ErrorContext: []string{"more than one start details found"}, + ErrorContext: "more than one start details found", } } sd := s.Starts[0] @@ -1488,7 +1175,7 @@ func (p *ticketVotePlugin) startStandard(s ticketvote.Start) (*ticketvote.StartR msg := hex.EncodeToString(util.Digest(vb)) err = util.VerifySignature(sd.Signature, sd.PublicKey, msg) if err != nil { - return nil, convertTicketVoteErrFromSignatureErr(err) + return nil, convertSignatureError(err) } // Verify vote options and params @@ -1512,12 +1199,6 @@ func (p *ticketVotePlugin) startStandard(s ticketvote.Start) (*ticketvote.StartR // Verify record version r, err := p.backend.GetVetted(token, "") if err != nil { - if errors.Is(err, backend.ErrRecordNotFound) { - return nil, backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusRecordNotFound), - } - } return nil, fmt.Errorf("GetVetted: %v", err) } version := strconv.FormatUint(uint64(sd.Params.Version), 10) @@ -1527,12 +1208,12 @@ func (p *ticketVotePlugin) startStandard(s ticketvote.Start) (*ticketvote.StartR return nil, backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusRecordVersionInvalid), - ErrorContext: []string{e}, + ErrorContext: e, } } // Verify vote authorization - auths, err := p.auths(token) + auths, err := p.auths(treeID) if err != nil { return nil, err } @@ -1540,7 +1221,7 @@ func (p *ticketVotePlugin) startStandard(s ticketvote.Start) (*ticketvote.StartR return nil, backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusAuthorizationInvalid), - ErrorContext: []string{"authorization not found"}, + ErrorContext: "authorization not found", } } action := ticketvote.AuthActionT(auths[len(auths)-1].Action) @@ -1548,12 +1229,12 @@ func (p *ticketVotePlugin) startStandard(s ticketvote.Start) (*ticketvote.StartR return nil, backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusAuthorizationInvalid), - ErrorContext: []string{"not authorized"}, + ErrorContext: "not authorized", } } // Verify vote has not already been started - svp, err := p.voteDetails(token) + svp, err := p.voteDetails(treeID) if err != nil { return nil, err } @@ -1562,7 +1243,7 @@ func (p *ticketVotePlugin) startStandard(s ticketvote.Start) (*ticketvote.StartR return nil, backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusVoteStatusInvalid), - ErrorContext: []string{"vote already started"}, + ErrorContext: "vote already started", } } @@ -1578,7 +1259,7 @@ func (p *ticketVotePlugin) startStandard(s ticketvote.Start) (*ticketvote.StartR } // Save vote details - err = p.voteDetailsSave(vd) + err = p.voteDetailsSave(treeID, vd) if err != nil { return nil, fmt.Errorf("voteDetailsSave: %v", err) } @@ -1620,7 +1301,7 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep return nil, backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), - ErrorContext: []string{e}, + ErrorContext: e, } case v.Params.Mask != mask: e := fmt.Sprintf("%v mask invalid: all must be the same", @@ -1628,7 +1309,7 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep return nil, backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), - ErrorContext: []string{e}, + ErrorContext: e, } case v.Params.Duration != duration: e := fmt.Sprintf("%v duration invalid: all must be the same", @@ -1636,7 +1317,7 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep return nil, backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), - ErrorContext: []string{e}, + ErrorContext: e, } case v.Params.QuorumPercentage != quorum: e := fmt.Sprintf("%v quorum invalid: must be the same", @@ -1644,7 +1325,7 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep return nil, backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), - ErrorContext: []string{e}, + ErrorContext: e, } case v.Params.PassPercentage != pass: e := fmt.Sprintf("%v pass rate invalid: all must be the same", @@ -1652,7 +1333,7 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep return nil, backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), - ErrorContext: []string{e}, + ErrorContext: e, } case v.Params.Parent != parent: e := fmt.Sprintf("%v parent invalid: all must be the same", @@ -1660,7 +1341,7 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep return nil, backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), - ErrorContext: []string{e}, + ErrorContext: e, } } @@ -1670,7 +1351,7 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep return nil, backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusTokenInvalid), - ErrorContext: []string{v.Params.Token}, + ErrorContext: v.Params.Token, } } @@ -1682,7 +1363,7 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep msg := hex.EncodeToString(util.Digest(vb)) err = util.VerifySignature(v.Signature, v.PublicKey, msg) if err != nil { - return nil, convertTicketVoteErrFromSignatureErr(err) + return nil, convertSignatureError(err) } // Verify vote options and params. Vote optoins are required to @@ -1709,7 +1390,7 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep return nil, backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), - ErrorContext: []string{e}, + ErrorContext: e, } } @@ -1731,13 +1412,6 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep // Verify record version r, err := p.backend.GetVetted(token, "") if err != nil { - if errors.Is(err, backend.ErrRecordNotFound) { - return nil, backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusRecordNotFound), - ErrorContext: []string{v.Params.Token}, - } - } return nil, fmt.Errorf("GetVetted: %v", err) } version := strconv.FormatUint(uint64(v.Params.Version), 10) @@ -1747,12 +1421,15 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep return nil, backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusRecordVersionInvalid), - ErrorContext: []string{e}, + ErrorContext: e, } } + // TODO figure this out + var treeID int64 + // Verify vote has not already been started - svp, err := p.voteDetails(token) + svp, err := p.voteDetails(treeID) if err != nil { return nil, err } @@ -1761,7 +1438,7 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep return nil, backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusVoteStatusInvalid), - ErrorContext: []string{"vote already started"}, + ErrorContext: "vote already started", } } } @@ -1778,8 +1455,11 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep EligibleTickets: sr.EligibleTickets, } + // TODO figure this out + var treeID int64 + // Save vote details - err = p.voteDetailsSave(vd) + err = p.voteDetailsSave(treeID, vd) if err != nil { return nil, fmt.Errorf("voteDetailsSave: %v", err) } @@ -1797,11 +1477,12 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep }, nil } -func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { - log.Tracef("ticketvote cmdStart: %v", payload) +func (p *ticketVotePlugin) cmdStart(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdStart: %v %x %v", treeID, token, payload) // Decode payload - s, err := ticketvote.DecodeStart([]byte(payload)) + var s ticketvote.Start + err := json.Unmarshal([]byte(payload), &s) if err != nil { return "", err } @@ -1811,7 +1492,7 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { return "", backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusStartDetailsInvalid), - ErrorContext: []string{"no start details found"}, + ErrorContext: "no start details found", } } vtype := s.Starts[0].Params.Type @@ -1822,12 +1503,12 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { var sr *ticketvote.StartReply switch vtype { case ticketvote.VoteTypeStandard: - sr, err = p.startStandard(*s) + sr, err = p.startStandard(treeID, s) if err != nil { return "", err } case ticketvote.VoteTypeRunoff: - sr, err = p.startRunoff(*s) + sr, err = p.startRunoff(s) if err != nil { return "", err } @@ -1836,12 +1517,12 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { return "", backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), - ErrorContext: []string{e}, + ErrorContext: e, } } // Prepare reply - reply, err := ticketvote.EncodeStartReply(*sr) + reply, err := json.Marshal(*sr) if err != nil { return "", err } @@ -1854,7 +1535,7 @@ func (p *ticketVotePlugin) cmdStart(payload string) (string, error) { // waits until all provided votes have been cast before returning. // // This function must be called WITH the record lock held. -func (p *ticketVotePlugin) ballot(votes []ticketvote.CastVote, results chan ticketvote.CastVoteReply) { +func (p *ticketVotePlugin) ballot(treeID int64, votes []ticketvote.CastVote, results chan ticketvote.CastVoteReply) { // Cast the votes concurrently var wg sync.WaitGroup for _, v := range votes { @@ -1877,7 +1558,7 @@ func (p *ticketVotePlugin) ballot(votes []ticketvote.CastVote, results chan tick // Save cast vote var cvr ticketvote.CastVoteReply - err := p.castVoteSave(cv) + err := p.castVoteSave(treeID, cv) if err != nil { t := time.Now().Unix() log.Errorf("cmdCastBallot: castVoteSave %v: %v", t, err) @@ -1909,11 +1590,12 @@ func (p *ticketVotePlugin) ballot(votes []ticketvote.CastVote, results chan tick // cmdCastBallot casts a ballot of votes. This function will not return a user // error if one occurs. It will instead return the ballot reply with the error // included in the invidiual cast vote reply that it applies to. -func (p *ticketVotePlugin) cmdCastBallot(payload string) (string, error) { - log.Tracef("ticketvote cmdCastBallot: %v", payload) +func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdCastBallot: %v %x %v", treeID, token, payload) // Decode payload - cb, err := ticketvote.DecodeCastBallot([]byte(payload)) + var cb ticketvote.CastBallot + err := json.Unmarshal([]byte(payload), &cb) if err != nil { return "", err } @@ -1925,7 +1607,7 @@ func (p *ticketVotePlugin) cmdCastBallot(payload string) (string, error) { cbr := ticketvote.CastBallotReply{ Receipts: []ticketvote.CastVoteReply{}, } - reply, err := ticketvote.EncodeCastBallotReply(cbr) + reply, err := json.Marshal(cbr) if err != nil { return "", err } @@ -1935,7 +1617,6 @@ func (p *ticketVotePlugin) cmdCastBallot(payload string) (string, error) { // Verify that all tokens in the ballot are valid, full length // tokens and that they are all voting for the same record. var ( - token []byte receipts = make([]ticketvote.CastVoteReply, len(votes)) ) for k, v := range votes { @@ -1949,12 +1630,6 @@ func (p *ticketVotePlugin) cmdCastBallot(payload string) (string, error) { ticketvote.VoteErrors[e]) continue } - if token == nil { - // Set token to the first valid one we come across. All votes - // in the ballot with a valid token are required to be the same - // as this token. - token = t - } // Verify token is the same if !bytes.Equal(t, token) { @@ -1970,7 +1645,7 @@ func (p *ticketVotePlugin) cmdCastBallot(payload string) (string, error) { // have not had their error set are voting for the same record. Get // the record and vote data that we need to perform the remaining // inexpensive validation before we have to hold the lock. - voteDetails, err := p.voteDetails(token) + voteDetails, err := p.voteDetails(treeID) if err != nil { return "", err } @@ -2178,7 +1853,7 @@ func (p *ticketVotePlugin) cmdCastBallot(payload string) (string, error) { for i, batch := range queue { log.Debugf("Casting %v votes in batch %v/%v", len(batch), i+1, len(queue)) - p.ballot(batch, results) + p.ballot(treeID, batch, results) } // Empty out the results channel @@ -2218,7 +1893,7 @@ func (p *ticketVotePlugin) cmdCastBallot(payload string) (string, error) { cbr := ticketvote.CastBallotReply{ Receipts: receipts, } - reply, err := ticketvote.EncodeCastBallotReply(cbr) + reply, err := json.Marshal(cbr) if err != nil { return "", err } @@ -2226,14 +1901,17 @@ func (p *ticketVotePlugin) cmdCastBallot(payload string) (string, error) { return string(reply), nil } -func (p *ticketVotePlugin) cmdDetails(payload string) (string, error) { - log.Tracef("ticketvote cmdDetails: %v", payload) +func (p *ticketVotePlugin) cmdDetails(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdDetails: %v %x %v", treeID, token, payload) - d, err := ticketvote.DecodeDetails([]byte(payload)) + var d ticketvote.Details + err := json.Unmarshal([]byte(payload), &d) if err != nil { return "", err } + // TODO this needs to be for a single record + votes := make(map[string]ticketvote.RecordVote, len(d.Tokens)) for _, v := range d.Tokens { // Verify token @@ -2241,21 +1919,16 @@ func (p *ticketVotePlugin) cmdDetails(payload string) (string, error) { if err != nil { continue } + _ = token // Get authorize votes - auths, err := p.auths(token) + auths, err := p.auths(treeID) if err != nil { - if errors.Is(err, errRecordNotFound) { - return "", backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusRecordNotFound), - } - } return "", fmt.Errorf("auths: %v", err) } // Get vote details - vd, err := p.voteDetails(token) + vd, err := p.voteDetails(treeID) if err != nil { return "", fmt.Errorf("startDetails: %v", err) } @@ -2271,7 +1944,7 @@ func (p *ticketVotePlugin) cmdDetails(payload string) (string, error) { dr := ticketvote.DetailsReply{ Votes: votes, } - reply, err := ticketvote.EncodeDetailsReply(dr) + reply, err := json.Marshal(dr) if err != nil { return "", err } @@ -2279,33 +1952,12 @@ func (p *ticketVotePlugin) cmdDetails(payload string) (string, error) { return string(reply), nil } -func (p *ticketVotePlugin) cmdResults(payload string) (string, error) { - log.Tracef("ticketvote cmdResults: %v", payload) - - // Decode payload - r, err := ticketvote.DecodeResults([]byte(payload)) - if err != nil { - return "", err - } - - // Verify token - token, err := tokenDecodeAnyLength(r.Token) - if err != nil { - return "", backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusTokenInvalid), - } - } +func (p *ticketVotePlugin) cmdResults(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdResults: %v %x %v", treeID, token, payload) // Get cast votes - votes, err := p.castVotes(token) + votes, err := p.castVotes(treeID) if err != nil { - if errors.Is(err, errRecordNotFound) { - return "", backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusRecordNotFound), - } - } return "", err } @@ -2313,7 +1965,7 @@ func (p *ticketVotePlugin) cmdResults(payload string) (string, error) { rr := ticketvote.ResultsReply{ Votes: votes, } - reply, err := ticketvote.EncodeResultsReply(rr) + reply, err := json.Marshal(rr) if err != nil { return "", err } @@ -2375,13 +2027,12 @@ func voteIsApproved(vd ticketvote.VoteDetails, results []ticketvote.VoteOptionRe return approved } -// summary returns the vote summary for a record. If a vetted record does not -// exist for the token a errRecordNotFound error is returned. -func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote.Summary, error) { +// summary returns the vote summary for a record. +func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) (*ticketvote.Summary, error) { // Check if the summary has been cached s, err := p.cachedSummary(hex.EncodeToString(token)) switch { - case errors.Is(err, errRecordNotFound): + case errors.Is(err, errSummaryNotFound): // Cached summary not found. Continue. case err != nil: // Some other error @@ -2398,13 +2049,8 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. status := ticketvote.VoteStatusUnauthorized // Check if the vote has been authorized - auths, err := p.auths(token) + auths, err := p.auths(treeID) if err != nil { - if errors.Is(err, errRecordNotFound) { - // Let the calling function decide how to handle when a vetted - // record does not exist for the token. - return nil, err - } return nil, fmt.Errorf("auths: %v", err) } if len(auths) > 0 { @@ -2424,7 +2070,7 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. } // Check if the vote has been started - vd, err := p.voteDetails(token) + vd, err := p.voteDetails(treeID) if err != nil { return nil, fmt.Errorf("startDetails: %v", err) } @@ -2514,11 +2160,12 @@ func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote. return &summary, nil } -func (p *ticketVotePlugin) cmdSummaries(payload string) (string, error) { - log.Tracef("ticketvote cmdSummaries: %v", payload) +func (p *ticketVotePlugin) cmdSummaries(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdSummaries: %v %x %v", treeID, token, payload) // Decode payload - s, err := ticketvote.DecodeSummaries([]byte(payload)) + var s ticketvote.Summaries + err := json.Unmarshal([]byte(payload), &s) if err != nil { return "", err } @@ -2530,6 +2177,7 @@ func (p *ticketVotePlugin) cmdSummaries(payload string) (string, error) { return "", fmt.Errorf("bestBlockUnsafe: %v", err) } + // TODO this route can only be for one summary // Get summaries summaries := make(map[string]ticketvote.Summary, len(s.Tokens)) for _, v := range s.Tokens { @@ -2537,13 +2185,8 @@ func (p *ticketVotePlugin) cmdSummaries(payload string) (string, error) { if err != nil { return "", err } - s, err := p.summary(token, bb) + s, err := p.summary(treeID, token, bb) if err != nil { - if errors.Is(err, errRecordNotFound) { - // Record does not exist for token. Do not include this token - // in the reply. - continue - } return "", fmt.Errorf("summary %v: %v", v, err) } summaries[v] = *s @@ -2554,7 +2197,7 @@ func (p *ticketVotePlugin) cmdSummaries(payload string) (string, error) { Summaries: summaries, BestBlock: bb, } - reply, err := ticketvote.EncodeSummariesReply(sr) + reply, err := json.Marshal(sr) if err != nil { return "", err } @@ -2596,10 +2239,8 @@ func convertInventoryReply(v voteInventory) ticketvote.InventoryReply { } } -func (p *ticketVotePlugin) cmdInventory(payload string) (string, error) { - log.Tracef("ticketvote cmdInventory: %v", payload) - - // Payload is empty. Nothing to decode. +func (p *ticketVotePlugin) cmdInventory() (string, error) { + log.Tracef("cmdInventory") // Get best block. This command does not write any data so we can // use the unsafe best block. @@ -2616,7 +2257,7 @@ func (p *ticketVotePlugin) cmdInventory(payload string) (string, error) { ir := convertInventoryReply(*inv) // Prepare reply - reply, err := ticketvote.EncodeInventoryReply(ir) + reply, err := json.Marshal(ir) if err != nil { return "", err } @@ -2624,8 +2265,8 @@ func (p *ticketVotePlugin) cmdInventory(payload string) (string, error) { return string(reply), nil } -func (p *ticketVotePlugin) cmdTimestamps(payload string) (string, error) { - log.Tracef("ticketvote cmdTicketvote: %v", payload) +func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdTimestamps: %v %x %v", treeID, token, payload) // Decode payload var t ticketvote.Timestamps @@ -2634,54 +2275,37 @@ func (p *ticketVotePlugin) cmdTimestamps(payload string) (string, error) { return "", err } - // Verify token - token, err := tokenDecodeAnyLength(t.Token) - if err != nil { - return "", backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusTokenInvalid), - } - } - // Get authorization timestamps - merkles, err := p.tlog.merklesByKeyPrefix(tlogIDVetted, token, - keyPrefixAuthDetails) + digests, err := p.tlog.DigestsByDataType(treeID, dataTypeAuthDetails) if err != nil { - if errors.Is(err, errRecordNotFound) { - return "", backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusRecordNotFound), - } - } - return "", fmt.Errorf("merklesByKeyPrefix %x %v: %v", - token, keyPrefixAuthDetails, err) + return "", fmt.Errorf("DigestByDataType %v %v: %v", + treeID, dataTypeAuthDetails, err) } - auths := make([]ticketvote.Timestamp, 0, len(merkles)) - for _, v := range merkles { - ts, err := p.timestamp(token, v) + auths := make([]ticketvote.Timestamp, 0, len(digests)) + for _, v := range digests { + ts, err := p.timestamp(treeID, v) if err != nil { return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) } auths = append(auths, *ts) } - // Get vote details merkle leaf hash. There should never be more - // than one vote details. - merkles, err = p.tlog.merklesByKeyPrefix(tlogIDVetted, token, - keyPrefixVoteDetails) + // Get vote details timestamp. There should never be more than one + // vote details. + digests, err = p.tlog.DigestsByDataType(treeID, dataTypeVoteDetails) if err != nil { - return "", fmt.Errorf("merklesByKeyPrefix %x %v: %v", - token, keyPrefixVoteDetails, err) + return "", fmt.Errorf("DigestsByDataType %v %v: %v", + treeID, dataTypeVoteDetails, err) } - if len(merkles) > 1 { + if len(digests) > 1 { return "", fmt.Errorf("invalid vote details count: got %v, want 1", - len(merkles)) + len(digests)) } var details ticketvote.Timestamp - for _, v := range merkles { - ts, err := p.timestamp(token, v) + for _, v := range digests { + ts, err := p.timestamp(treeID, v) if err != nil { return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) } @@ -2689,16 +2313,15 @@ func (p *ticketVotePlugin) cmdTimestamps(payload string) (string, error) { } // Get cast vote timestamps - merkles, err = p.tlog.merklesByKeyPrefix(tlogIDVetted, token, - keyPrefixCastVoteDetails) + digests, err = p.tlog.DigestsByDataType(treeID, dataTypeCastVoteDetails) if err != nil { - return "", fmt.Errorf("merklesByKeyPrefix %x %v: %v", - token, keyPrefixVoteDetails, err) + return "", fmt.Errorf("digestsByKeyPrefix %v %v: %v", + treeID, dataTypeVoteDetails, err) } - votes := make(map[string]ticketvote.Timestamp, len(merkles)) - for _, v := range merkles { - ts, err := p.timestamp(token, v) + votes := make(map[string]ticketvote.Timestamp, len(digests)) + for _, v := range digests { + ts, err := p.timestamp(treeID, v) if err != nil { return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) } @@ -2726,11 +2349,11 @@ func (p *ticketVotePlugin) cmdTimestamps(payload string) (string, error) { return string(reply), nil } -// setup performs any plugin setup work that needs to be done. +// Setup performs any plugin setup work that needs to be done. // -// This function satisfies the pluginClient interface. -func (p *ticketVotePlugin) setup() error { - log.Tracef("ticketvote setup") +// This function satisfies the plugins.Client interface. +func (p *ticketVotePlugin) Setup() error { + log.Tracef("Setup") // Verify plugin dependencies plugins, err := p.backend.GetPlugins() @@ -2772,7 +2395,9 @@ func (p *ticketVotePlugin) setup() error { if err != nil { return err } - s, err := p.summary(token, bestBlock) + // TODO this needs to use the summary plugin command + var treeID int64 + s, err := p.summary(treeID, token, bestBlock) if err != nil { return fmt.Errorf("summary %v: %v", v, err) } @@ -2809,7 +2434,9 @@ func (p *ticketVotePlugin) setup() error { if err != nil { return err } - votes, err := p.castVotes(token) + // TODO this needs to use the results plugin command + var treeID int64 + votes, err := p.castVotes(treeID) if err != nil { return fmt.Errorf("castVotes %v: %v", token, err) } @@ -2821,56 +2448,55 @@ func (p *ticketVotePlugin) setup() error { return nil } -// cmd executes a plugin command. +// Cmd executes a plugin command. // -// This function satisfies the pluginClient interface. -func (p *ticketVotePlugin) cmd(cmd, payload string) (string, error) { - log.Tracef("ticketvote cmd: %v %v", cmd, payload) +// This function satisfies the plugins.Client interface. +func (p *ticketVotePlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { + log.Tracef("Cmd: %v %x %v", treeID, token, cmd) switch cmd { case ticketvote.CmdAuthorize: - return p.cmdAuthorize(payload) + return p.cmdAuthorize(treeID, token, payload) case ticketvote.CmdStart: - return p.cmdStart(payload) + return p.cmdStart(treeID, token, payload) case ticketvote.CmdCastBallot: - return p.cmdCastBallot(payload) + return p.cmdCastBallot(treeID, token, payload) case ticketvote.CmdDetails: - return p.cmdDetails(payload) + return p.cmdDetails(treeID, token, payload) case ticketvote.CmdResults: - return p.cmdResults(payload) + return p.cmdResults(treeID, token, payload) case ticketvote.CmdSummaries: - return p.cmdSummaries(payload) + return p.cmdSummaries(treeID, token, payload) case ticketvote.CmdInventory: - return p.cmdInventory(payload) + return p.cmdInventory() case ticketvote.CmdTimestamps: - return p.cmdTimestamps(payload) + return p.cmdTimestamps(treeID, token, payload) } return "", backend.ErrPluginCmdInvalid } -// hook executes a plugin hook. +// Hook executes a plugin hook. // -// This function satisfies the pluginClient interface. -func (p *ticketVotePlugin) hook(h hookT, payload string) error { - log.Tracef("ticketvote hook: %v", hooks[h]) +// This function satisfies the plugins.Client interface. +func (p *ticketVotePlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { + log.Tracef("Hook: %v %x %v", treeID, token, plugins.Hooks[h]) return nil } // Fsck performs a plugin filesystem check. // -// This function satisfies the pluginClient interface. -func (p *ticketVotePlugin) fsck() error { - log.Tracef("ticketvote fsck") +// This function satisfies the plugins.Client interface. +func (p *ticketVotePlugin) Fsck() error { + log.Tracef("Fsck") return nil } -func newTicketVotePlugin(backend backend.Backend, tlog tlogClient, settings []backend.PluginSetting, id *identity.FullIdentity, activeNetParams *chaincfg.Params) (*ticketVotePlugin, error) { +func newTicketVotePlugin(backend backend.Backend, tlog tlogclient.Client, settings []backend.PluginSetting, dataDir string, id *identity.FullIdentity, activeNetParams *chaincfg.Params) (*ticketVotePlugin, error) { // Plugin settings var ( - dataDir string voteDurationMin uint32 voteDurationMax uint32 ) @@ -2894,8 +2520,6 @@ func newTicketVotePlugin(backend backend.Backend, tlog tlogClient, settings []ba // Parse user provided plugin settings for _, v := range settings { switch v.Key { - case pluginSettingDataDir: - dataDir = v.Value case pluginSettingVoteDurationMin: u, err := strconv.ParseUint(v.Value, 10, 64) if err != nil { @@ -2915,13 +2539,6 @@ func newTicketVotePlugin(backend backend.Backend, tlog tlogClient, settings []ba } } - // Verify required plugin settings - switch { - case dataDir == "": - return nil, fmt.Errorf("plugin setting not found: %v", - pluginSettingDataDir) - } - // Create the plugin data directory dataDir = filepath.Join(dataDir, ticketvote.ID) err := os.MkdirAll(dataDir, 0700) diff --git a/politeiad/backend/tlogbe/store/store.go b/politeiad/backend/tlogbe/store/store.go index 4bc480ebe..4468844ac 100644 --- a/politeiad/backend/tlogbe/store/store.go +++ b/politeiad/backend/tlogbe/store/store.go @@ -36,9 +36,7 @@ type DataDescriptor struct { // BlobEntry is the structure used to store data in the Blob key-value store. // All data in the Blob key-value store will be encoded as a BlobEntry. type BlobEntry struct { - // TODO change this to digest so that we are consistent with the - // terminology used throughout the backend. - Hash string `json:"hash"` // SHA256 hash of data payload, hex encoded + Digest string `json:"digest"` // SHA256 digest of data, hex encoded DataHint string `json:"datahint"` // Hint that describes data, base64 encoded Data string `json:"data"` // Data payload, base64 encoded } @@ -46,7 +44,7 @@ type BlobEntry struct { // NewBlobEntry returns a new BlobEntry. func NewBlobEntry(dataHint, data []byte) BlobEntry { return BlobEntry{ - Hash: hex.EncodeToString(util.Digest(data)), + Digest: hex.EncodeToString(util.Digest(data)), DataHint: base64.StdEncoding.EncodeToString(dataHint), Data: base64.StdEncoding.EncodeToString(data), } diff --git a/politeiad/backend/tlogbe/tlog/anchor.go b/politeiad/backend/tlogbe/tlog/anchor.go index 3d945f99e..76666c1c5 100644 --- a/politeiad/backend/tlogbe/tlog/anchor.go +++ b/politeiad/backend/tlogbe/tlog/anchor.go @@ -189,13 +189,13 @@ func (t *Tlog) anchorSave(a anchor) error { } // Append anchor leaf to trillian tree - h, err := hex.DecodeString(be.Hash) + d, err := hex.DecodeString(be.Digest) if err != nil { return err } extraData := leafExtraData(dataTypeAnchorRecord, keys[0]) leaves := []*trillian.LogLeaf{ - newLogLeaf(h, extraData), + newLogLeaf(d, extraData), } queued, _, err := t.trillian.leavesAppend(a.TreeID, leaves) if err != nil { diff --git a/politeiad/backend/tlogbe/tlog/plugin.go b/politeiad/backend/tlogbe/tlog/plugin.go index 1200daa47..25d08e481 100644 --- a/politeiad/backend/tlogbe/tlog/plugin.go +++ b/politeiad/backend/tlogbe/tlog/plugin.go @@ -11,6 +11,10 @@ import ( "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/comments" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/dcrdata" + cmplugin "github.com/decred/politeia/politeiad/plugins/comments" + ddplugin "github.com/decred/politeia/politeiad/plugins/dcrdata" ) const ( @@ -55,22 +59,19 @@ func (t *Tlog) PluginRegister(b backend.Backend, p backend.Plugin) error { err error dataDir = filepath.Join(t.dataDir, pluginDataDirname) - - _ = err - _ = dataDir ) switch p.ID { + case cmplugin.ID: + client, err = comments.New(t, p.Settings, dataDir, p.Identity) + if err != nil { + return err + } + case ddplugin.ID: + client, err = dcrdata.New(p.Settings, t.activeNetParams) + if err != nil { + return err + } /* - case cmplugin.ID: - client, err = comments.New(t, p.Settings, p.Identity, dataDir) - if err != nil { - return err - } - case ddplugin.ID: - client, err = dcrdata.New(p.Settings, t.activeNetParams) - if err != nil { - return err - } case piplugin.ID: client, err = pi.New(b, t, p.Settings, dataDir, t.activeNetParams) if err != nil { diff --git a/politeiad/backend/tlogbe/tlog/recordindex.go b/politeiad/backend/tlogbe/tlog/recordindex.go index a8120f156..fe629bb3b 100644 --- a/politeiad/backend/tlogbe/tlog/recordindex.go +++ b/politeiad/backend/tlogbe/tlog/recordindex.go @@ -156,13 +156,13 @@ func (t *Tlog) recordIndexSave(treeID int64, ri recordIndex) error { } // Append record index leaf to trillian tree - h, err := hex.DecodeString(be.Hash) + d, err := hex.DecodeString(be.Digest) if err != nil { return err } extraData := leafExtraData(dataTypeRecordIndex, keys[0]) leaves := []*trillian.LogLeaf{ - newLogLeaf(h, extraData), + newLogLeaf(d, extraData), } queued, _, err := t.trillian.leavesAppend(treeID, leaves) if err != nil { diff --git a/politeiad/backend/tlogbe/tlog/tlog.go b/politeiad/backend/tlogbe/tlog/tlog.go index cabd92ebd..d976c3ebd 100644 --- a/politeiad/backend/tlogbe/tlog/tlog.go +++ b/politeiad/backend/tlogbe/tlog/tlog.go @@ -32,9 +32,9 @@ const ( // Blob entry data descriptors dataDescriptorFile = "file_v1" - dataDescriptorRecordMetadata = "recordmetadata_v1" - dataDescriptorMetadataStream = "metadatastream_v1" - dataDescriptorRecordIndex = "recordindex_v1" + dataDescriptorRecordMetadata = "recordmd_v1" + dataDescriptorMetadataStream = "mdstream_v1" + dataDescriptorRecordIndex = "rindex_v1" dataDescriptorAnchor = "anchor_v1" // The keys for kv store blobs are saved by stuffing them into the @@ -213,13 +213,13 @@ func convertRecordIndexFromBlobEntry(be store.BlobEntry) (*recordIndex, error) { if err != nil { return nil, fmt.Errorf("decode Data: %v", err) } - hash, err := hex.DecodeString(be.Hash) + digest, err := hex.DecodeString(be.Digest) if err != nil { - return nil, fmt.Errorf("decode hash: %v", err) + return nil, fmt.Errorf("decode digest: %v", err) } - if !bytes.Equal(util.Digest(b), hash) { + if !bytes.Equal(util.Digest(b), digest) { return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), hash) + util.Digest(b), digest) } var ri recordIndex err = json.Unmarshal(b, &ri) @@ -251,13 +251,13 @@ func convertAnchorFromBlobEntry(be store.BlobEntry) (*anchor, error) { if err != nil { return nil, fmt.Errorf("decode Data: %v", err) } - hash, err := hex.DecodeString(be.Hash) + digest, err := hex.DecodeString(be.Digest) if err != nil { - return nil, fmt.Errorf("decode hash: %v", err) + return nil, fmt.Errorf("decode digest: %v", err) } - if !bytes.Equal(util.Digest(b), hash) { + if !bytes.Equal(util.Digest(b), digest) { return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), hash) + util.Digest(b), digest) } var a anchor err = json.Unmarshal(b, &a) @@ -348,7 +348,7 @@ func (t *Tlog) TreeFreeze(treeID int64, rm backend.RecordMetadata, metadata []ba if err != nil { return err } - idxHash, err := hex.DecodeString(be.Hash) + idxDigest, err := hex.DecodeString(be.Digest) if err != nil { return err } @@ -369,7 +369,7 @@ func (t *Tlog) TreeFreeze(treeID int64, rm backend.RecordMetadata, metadata []ba // Append record index leaf to the trillian tree extraData := leafExtraData(dataTypeRecordIndex, keys[0]) leaves := []*trillian.LogLeaf{ - newLogLeaf(idxHash, extraData), + newLogLeaf(idxDigest, extraData), } queued, _, err := t.trillian.leavesAppend(treeID, leaves) if err != nil { @@ -604,7 +604,7 @@ func (t *Tlog) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backen if err != nil { return nil, err } - h, err := hex.DecodeString(be.Hash) + h, err := hex.DecodeString(be.Digest) if err != nil { return nil, err } @@ -612,7 +612,7 @@ func (t *Tlog) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backen if err != nil { return nil, err } - _, ok := dups[be.Hash] + _, ok := dups[be.Digest] if !ok { // Not a duplicate. Save blob to the store. hashes = append(hashes, h) @@ -625,7 +625,7 @@ func (t *Tlog) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backen if err != nil { return nil, err } - h, err := hex.DecodeString(be.Hash) + h, err := hex.DecodeString(be.Digest) if err != nil { return nil, err } @@ -633,7 +633,7 @@ func (t *Tlog) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backen if err != nil { return nil, err } - _, ok := dups[be.Hash] + _, ok := dups[be.Digest] if !ok { // Not a duplicate. Save blob to the store. hashes = append(hashes, h) @@ -647,7 +647,7 @@ func (t *Tlog) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backen if err != nil { return nil, err } - h, err := hex.DecodeString(be.Hash) + h, err := hex.DecodeString(be.Digest) if err != nil { return nil, err } @@ -655,7 +655,7 @@ func (t *Tlog) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backen if err != nil { return nil, err } - _, ok := dups[be.Hash] + _, ok := dups[be.Digest] if !ok { // Not a duplicate. Save blob to the store. hashes = append(hashes, h) @@ -1205,13 +1205,13 @@ func (t *Tlog) Record(treeID int64, version uint32) (*backend.Record, error) { if err != nil { return nil, fmt.Errorf("decode Data: %v", err) } - hash, err := hex.DecodeString(v.Hash) + digest, err := hex.DecodeString(v.Digest) if err != nil { return nil, fmt.Errorf("decode Hash: %v", err) } - if !bytes.Equal(util.Digest(b), hash) { + if !bytes.Equal(util.Digest(b), digest) { return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), hash) + util.Digest(b), digest) } switch dd.Descriptor { case dataDescriptorRecordMetadata: diff --git a/politeiad/backend/tlogbe/tlog/tlogclient.go b/politeiad/backend/tlogbe/tlog/tlogclient.go index c43be5dc7..b36bbc0f6 100644 --- a/politeiad/backend/tlogbe/tlog/tlogclient.go +++ b/politeiad/backend/tlogbe/tlog/tlogclient.go @@ -35,7 +35,7 @@ func (t *Tlog) BlobSave(treeID int64, dataType string, be store.BlobEntry) error } // Prepare blob and digest - digest, err := hex.DecodeString(be.Hash) + digest, err := hex.DecodeString(be.Digest) if err != nil { return err } diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 3088fdb68..76cba03ca 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -98,8 +98,12 @@ func tokenFromTreeID(treeID int64) []byte { return b } +func tokenIsFullLength(token []byte) bool { + return util.TokenIsFullLength(util.TokenTypeTlog, token) +} + func treeIDFromToken(token []byte) int64 { - if !util.TokenIsFullLength(util.TokenTypeTlog, token) { + if !tokenIsFullLength(token) { return 0 } return int64(binary.LittleEndian.Uint64(token)) @@ -748,7 +752,7 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ // Verify token is valid. The full length token must be used when // writing data. - if !util.TokenIsFullLength(util.TokenTypeTlog, token) { + if !tokenIsFullLength(token) { return nil, backend.ContentVerificationError{ ErrorCode: v1.ErrorStatusInvalidToken, } @@ -852,7 +856,7 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b // Verify token is valid. The full length token must be used when // writing data. - if !util.TokenIsFullLength(util.TokenTypeTlog, token) { + if !tokenIsFullLength(token) { return nil, backend.ContentVerificationError{ ErrorCode: v1.ErrorStatusInvalidToken, } @@ -959,7 +963,7 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite // Verify token is valid. The full length token must be used when // writing data. - if !util.TokenIsFullLength(util.TokenTypeTlog, token) { + if !tokenIsFullLength(token) { return backend.ContentVerificationError{ ErrorCode: v1.ErrorStatusInvalidToken, } @@ -1051,7 +1055,7 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ // Verify token is valid. The full length token must be used when // writing data. - if !util.TokenIsFullLength(util.TokenTypeTlog, token) { + if !tokenIsFullLength(token) { return backend.ContentVerificationError{ ErrorCode: v1.ErrorStatusInvalidToken, } @@ -1203,7 +1207,7 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, // Verify token is valid. The full length token must be used when // writing data. - if !util.TokenIsFullLength(util.TokenTypeTlog, token) { + if !tokenIsFullLength(token) { return nil, backend.ContentVerificationError{ ErrorCode: v1.ErrorStatusInvalidToken, } @@ -1358,7 +1362,7 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md // Verify token is valid. The full length token must be used when // writing data. - if !util.TokenIsFullLength(util.TokenTypeTlog, token) { + if !tokenIsFullLength(token) { return nil, backend.ContentVerificationError{ ErrorCode: v1.ErrorStatusInvalidToken, } @@ -1654,12 +1658,17 @@ func (t *tlogBackend) UnvettedPlugin(token []byte, pluginID, cmd, payload string return "", backend.ErrShutdown } - // Get tree ID - treeID := t.unvettedTreeIDFromToken(token) + // The token is optional. If a token is not provided then a tree ID + // will not be provided to the plugin. + var treeID int64 + if token != nil { + // Get tree ID + treeID = t.unvettedTreeIDFromToken(token) - // Verify record exists and is unvetted - if !t.UnvettedExists(token) { - return "", backend.ErrRecordNotFound + // Verify record exists and is unvetted + if !t.UnvettedExists(token) { + return "", backend.ErrRecordNotFound + } } // Call pre plugin hooks @@ -1712,10 +1721,16 @@ func (t *tlogBackend) VettedPlugin(token []byte, pluginID, cmd, payload string) return "", backend.ErrShutdown } - // Get tree ID - treeID, ok := t.vettedTreeIDFromToken(token) - if !ok { - return "", backend.ErrRecordNotFound + // The token is optional. If a token is not provided then a tree ID + // will not be provided to the plugin. + var treeID int64 + var ok bool + if token != nil { + // Get tree ID + treeID, ok = t.vettedTreeIDFromToken(token) + if !ok { + return "", backend.ErrRecordNotFound + } } // Call pre plugin hooks diff --git a/politeiad/backend/tlogbe/tlogclient/tlogclient.go b/politeiad/backend/tlogbe/tlogclient/tlogclient.go index 49e80acc7..16e4bdc0c 100644 --- a/politeiad/backend/tlogbe/tlogclient/tlogclient.go +++ b/politeiad/backend/tlogbe/tlogclient/tlogclient.go @@ -29,7 +29,8 @@ type Client interface { // map. Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEntry, error) - // BlobsByDataType returns all blobs that match the data type. + // BlobsByDataType returns all blobs that match the data type. The + // blobs will be ordered from oldest to newest. BlobsByDataType(treeID int64, keyPrefix string) ([]store.BlobEntry, error) // DigestsByDataType returns the digests of all blobs that match diff --git a/politeiad/log.go b/politeiad/log.go index 27b2e34c4..c4f141434 100644 --- a/politeiad/log.go +++ b/politeiad/log.go @@ -13,6 +13,7 @@ import ( "github.com/decred/politeia/politeiad/backend/tlogbe" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/comments" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/dcrdata" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/ticketvote" "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" "github.com/decred/politeia/wsdcrdata" @@ -47,14 +48,15 @@ var ( // application shutdown. logRotator *rotator.Rotator - log = backendLog.Logger("POLI") - gitbeLog = backendLog.Logger("GITB") - tlogbeLog = backendLog.Logger("BACK") - tlogLog = backendLog.Logger("TLOG") - storeLog = backendLog.Logger("STOR") - wsdcrdataLog = backendLog.Logger("WSDD") - commentsLog = backendLog.Logger("COMM") - dcrdataLog = backendLog.Logger("DCRL") + log = backendLog.Logger("POLI") + gitbeLog = backendLog.Logger("GITB") + tlogbeLog = backendLog.Logger("BACK") + tlogLog = backendLog.Logger("TLOG") + storeLog = backendLog.Logger("STOR") + wsdcrdataLog = backendLog.Logger("WSDD") + commentsLog = backendLog.Logger("COMM") + dcrdataLog = backendLog.Logger("DCRL") + ticketvoteLog = backendLog.Logger("TIKV") ) // Initialize package-global logger variables. @@ -66,6 +68,7 @@ func init() { wsdcrdata.UseLogger(wsdcrdataLog) comments.UseLogger(commentsLog) dcrdata.UseLogger(dcrdataLog) + ticketvote.UseLogger(ticketvoteLog) } // subsystemLoggers maps each subsystem identifier to its associated logger. @@ -78,6 +81,7 @@ var subsystemLoggers = map[string]slog.Logger{ "WSDD": wsdcrdataLog, "COMM": commentsLog, "DCRL": dcrdataLog, + "TIKV": ticketvoteLog, } // initLogRotator initializes the logging rotater to write logs to logFile and diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index 33b7b9e95..eb13ec967 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -6,10 +6,6 @@ // tickets to participate. package ticketvote -import ( - "encoding/json" -) - // TODO VoteDetails, StartReply, StartRunoffReply should contain a receipt. // The receipt should be the server signature of Signature+StartBlockHash. // TODO Update politeiavoter @@ -54,14 +50,13 @@ const ( ErrorStatusTokenInvalid ErrorStatusT = 1 ErrorStatusPublicKeyInvalid ErrorStatusT = 2 ErrorStatusSignatureInvalid ErrorStatusT = 3 - ErrorStatusRecordNotFound ErrorStatusT = 4 - ErrorStatusRecordVersionInvalid ErrorStatusT = 5 - ErrorStatusRecordStatusInvalid ErrorStatusT = 6 - ErrorStatusAuthorizationInvalid ErrorStatusT = 7 - ErrorStatusStartDetailsInvalid ErrorStatusT = 8 - ErrorStatusVoteParamsInvalid ErrorStatusT = 9 - ErrorStatusVoteStatusInvalid ErrorStatusT = 10 - ErrorStatusPageSizeExceeded ErrorStatusT = 11 + ErrorStatusRecordVersionInvalid ErrorStatusT = 4 + ErrorStatusRecordStatusInvalid ErrorStatusT = 5 + ErrorStatusAuthorizationInvalid ErrorStatusT = 6 + ErrorStatusStartDetailsInvalid ErrorStatusT = 7 + ErrorStatusVoteParamsInvalid ErrorStatusT = 8 + ErrorStatusVoteStatusInvalid ErrorStatusT = 9 + ErrorStatusPageSizeExceeded ErrorStatusT = 10 ) // AuthDetails is the structure that is saved to disk when a vote is authorized @@ -207,42 +202,12 @@ type Authorize struct { Signature string `json:"signature"` // Client signature } -// EncodeAuthorize encodes an Authorize into a JSON byte slice. -func EncodeAuthorize(a Authorize) ([]byte, error) { - return json.Marshal(a) -} - -// DecodeAuthorize decodes a JSON byte slice into a Authorize. -func DecodeAuthorize(payload []byte) (*Authorize, error) { - var a Authorize - err := json.Unmarshal(payload, &a) - if err != nil { - return nil, err - } - return &a, nil -} - // AuthorizeReply is the reply to the Authorize command. type AuthorizeReply struct { Timestamp int64 `json:"timestamp"` // Received UNIX timestamp Receipt string `json:"receipt"` // Server signature of client signature } -// EncodeAuthorizeReply encodes an AuthorizeReply into a JSON byte slice. -func EncodeAuthorizeReply(ar AuthorizeReply) ([]byte, error) { - return json.Marshal(ar) -} - -// DecodeAuthorizeReply decodes a JSON byte slice into a AuthorizeReply. -func DecodeAuthorizeReply(payload []byte) (*AuthorizeReply, error) { - var ar AuthorizeReply - err := json.Unmarshal(payload, &ar) - if err != nil { - return nil, err - } - return &ar, nil -} - // StartDetails is the structure that is provided when starting a ticket vote. // // Signature is the signature of a SHA256 digest of the JSON encoded VoteParams @@ -261,21 +226,6 @@ type Start struct { Starts []StartDetails `json:"starts"` } -// EncodeStart encodes a Start into a JSON byte slice. -func EncodeStart(s Start) ([]byte, error) { - return json.Marshal(s) -} - -// DecodeStart decodes a JSON byte slice into a Start. -func DecodeStart(payload []byte) (*Start, error) { - var s Start - err := json.Unmarshal(payload, &s) - if err != nil { - return nil, err - } - return &s, nil -} - // StartReply is the reply to the Start command. type StartReply struct { StartBlockHeight uint32 `json:"startblockheight"` @@ -284,21 +234,6 @@ type StartReply struct { EligibleTickets []string `json:"eligibletickets"` } -// EncodeStartReply encodes a StartReply into a JSON byte slice. -func EncodeStartReply(sr StartReply) ([]byte, error) { - return json.Marshal(sr) -} - -// DecodeStartReply decodes a JSON byte slice into a StartReply. -func DecodeStartReply(payload []byte) (*StartReply, error) { - var sr StartReply - err := json.Unmarshal(payload, &sr) - if err != nil { - return nil, err - } - return &sr, nil -} - // VoteErrorT represents errors that can occur while attempting to cast ticket // votes. type VoteErrorT int @@ -388,61 +323,16 @@ type CastBallot struct { Ballot []CastVote `json:"ballot"` } -// EncodeCastBallot encodes a CastBallot into a JSON byte slice. -func EncodeCastBallot(b CastBallot) ([]byte, error) { - return json.Marshal(b) -} - -// DecodeCastBallot decodes a JSON byte slice into a CastBallot. -func DecodeCastBallot(payload []byte) (*CastBallot, error) { - var b CastBallot - err := json.Unmarshal(payload, &b) - if err != nil { - return nil, err - } - return &b, nil -} - // CastBallotReply is a reply to a batched list of votes. type CastBallotReply struct { Receipts []CastVoteReply `json:"receipts"` } -// EncodeCastBallotReply encodes a CastBallot into a JSON byte slice. -func EncodeCastBallotReply(b CastBallotReply) ([]byte, error) { - return json.Marshal(b) -} - -// DecodeCastBallotReply decodes a JSON byte slice into a CastBallotReply. -func DecodeCastBallotReply(payload []byte) (*CastBallotReply, error) { - var b CastBallotReply - err := json.Unmarshal(payload, &b) - if err != nil { - return nil, err - } - return &b, nil -} - // Details returns the vote details for each of the provided record tokens. type Details struct { Tokens []string `json:"tokens"` } -// EncodeDetails encodes a Details into a JSON byte slice. -func EncodeDetails(d Details) ([]byte, error) { - return json.Marshal(d) -} - -// DecodeDetails decodes a JSON byte slice into a Details. -func DecodeDetails(payload []byte) (*Details, error) { - var d Details - err := json.Unmarshal(payload, &d) - if err != nil { - return nil, err - } - return &d, nil -} - // RecordVote contains all vote authorizations and the vote details for a // record. The VoteDetails will be nil if the vote has been started. type RecordVote struct { @@ -458,81 +348,21 @@ type DetailsReply struct { Votes map[string]RecordVote `json:"votes"` } -// EncodeDetailsReply encodes a DetailsReply into a JSON byte slice. -func EncodeDetailsReply(dr DetailsReply) ([]byte, error) { - return json.Marshal(dr) -} - -// DecodeDetailsReply decodes a JSON byte slice into a DetailsReply. -func DecodeDetailsReply(payload []byte) (*DetailsReply, error) { - var dr DetailsReply - err := json.Unmarshal(payload, &dr) - if err != nil { - return nil, err - } - return &dr, nil -} - // Results requests the results of a vote. type Results struct { Token string `json:"token"` } -// EncodeResults encodes a Results into a JSON byte slice. -func EncodeResults(r Results) ([]byte, error) { - return json.Marshal(r) -} - -// DecodeResults decodes a JSON byte slice into a Results. -func DecodeResults(payload []byte) (*Results, error) { - var r Results - err := json.Unmarshal(payload, &r) - if err != nil { - return nil, err - } - return &r, nil -} - // ResultsReply is the rely to the Results command. type ResultsReply struct { Votes []CastVoteDetails `json:"votes"` } -// EncodeResultsReply encodes a ResultsReply into a JSON byte slice. -func EncodeResultsReply(rr ResultsReply) ([]byte, error) { - return json.Marshal(rr) -} - -// DecodeResultsReply decodes a JSON byte slice into a ResultsReply. -func DecodeResultsReply(payload []byte) (*ResultsReply, error) { - var rr ResultsReply - err := json.Unmarshal(payload, &rr) - if err != nil { - return nil, err - } - return &rr, nil -} - // Summaries requests the vote summaries for the provided record tokens. type Summaries struct { Tokens []string `json:"tokens"` } -// EncodeSummaries encodes a Summaries into a JSON byte slice. -func EncodeSummaries(s Summaries) ([]byte, error) { - return json.Marshal(s) -} - -// DecodeSummaries decodes a JSON byte slice into a Summaries. -func DecodeSummaries(payload []byte) (*Summaries, error) { - var s Summaries - err := json.Unmarshal(payload, &s) - if err != nil { - return nil, err - } - return &s, nil -} - // VoteStatusT represents the status of a ticket vote. type VoteStatusT int @@ -609,40 +439,10 @@ type SummariesReply struct { BestBlock uint32 `json:"bestblock"` } -// EncodeSummariesReply encodes a SummariesReply into a JSON byte slice. -func EncodeSummariesReply(sr SummariesReply) ([]byte, error) { - return json.Marshal(sr) -} - -// DecodeSummariesReply decodes a JSON byte slice into a SummariesReply. -func DecodeSummariesReply(payload []byte) (*SummariesReply, error) { - var sr SummariesReply - err := json.Unmarshal(payload, &sr) - if err != nil { - return nil, err - } - return &sr, nil -} - // Inventory requests the tokens of all public, non-abandoned records // categorized by vote status. type Inventory struct{} -// EncodeInventory encodes a Inventory into a JSON byte slice. -func EncodeInventory(i Inventory) ([]byte, error) { - return json.Marshal(i) -} - -// DecodeInventory decodes a JSON byte slice into a Inventory. -func DecodeInventory(payload []byte) (*Inventory, error) { - var i Inventory - err := json.Unmarshal(payload, &i) - if err != nil { - return nil, err - } - return &i, nil -} - // InventoryReply is the reply to the Inventory command. It contains the tokens // of all public, non-abandoned records categorized by vote status. // @@ -662,21 +462,6 @@ type InventoryReply struct { BestBlock uint32 `json:"bestblock"` } -// EncodeInventoryReply encodes a InventoryReply into a JSON byte slice. -func EncodeInventoryReply(ir InventoryReply) ([]byte, error) { - return json.Marshal(ir) -} - -// DecodeInventoryReply decodes a JSON byte slice into a InventoryReply. -func DecodeInventoryReply(payload []byte) (*InventoryReply, error) { - var ir InventoryReply - err := json.Unmarshal(payload, &ir) - if err != nil { - return nil, err - } - return &ir, nil -} - // Proof contains an inclusion proof for the digest in the merkle root. The // ExtraData field is used by certain types of proofs to include additional // data that is required to validate the proof. @@ -706,9 +491,7 @@ type Timestamp struct { } // Timestamps requests the timestamps for ticket vote data. -type Timestamps struct { - Token string `json:"token"` -} +type Timestamps struct{} // TimestampsReply is the reply to the Timestamps command. type TimestampsReply struct { diff --git a/util/signature.go b/util/signature.go index 00e252a83..cf76a8bb8 100644 --- a/util/signature.go +++ b/util/signature.go @@ -24,7 +24,7 @@ const ( // signature. type SignatureError struct { ErrorCode ErrorStatusT - ErrorContext []string + ErrorContext string } // Error satisfies the error interface. @@ -38,21 +38,21 @@ func VerifySignature(signature, pubKey, msg string) error { if err != nil { return SignatureError{ ErrorCode: ErrorStatusSignatureInvalid, - ErrorContext: []string{err.Error()}, + ErrorContext: err.Error(), } } b, err := hex.DecodeString(pubKey) if err != nil { return SignatureError{ ErrorCode: ErrorStatusPublicKeyInvalid, - ErrorContext: []string{"key is not hex"}, + ErrorContext: "key is not hex", } } pk, err := identity.PublicIdentityFromBytes(b) if err != nil { return SignatureError{ ErrorCode: ErrorStatusPublicKeyInvalid, - ErrorContext: []string{err.Error()}, + ErrorContext: err.Error(), } } if !pk.VerifyMessage([]byte(msg), sig) { diff --git a/util/token.go b/util/token.go index 0950f38d1..71311c6d3 100644 --- a/util/token.go +++ b/util/token.go @@ -87,9 +87,9 @@ func TokenDecodeAnyLength(tokenType, token string) ([]byte, error) { case len(t) == TokenPrefixSize(): // This is a token prefix. Token prefixes are the same size // regardless of token type. - case TokenIsFullLength(TokenTypeGit, t): + case tokenType == TokenTypeGit && TokenIsFullLength(TokenTypeGit, t): // Token is a valid git backend token - case TokenIsFullLength(TokenTypeTlog, t): + case tokenType == TokenTypeTlog && TokenIsFullLength(TokenTypeTlog, t): // Token is a valid tlog backend token default: return nil, fmt.Errorf("invalid token size") From 29d1231c31e94d485f1c0eb850b0ef9328e7ba27 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 17 Jan 2021 19:25:24 -0600 Subject: [PATCH 231/449] Update ticketvote plugin. --- politeiad/backend/backend.go | 10 +- politeiad/backend/tlogbe/plugins/pi/pi.go | 415 +---- .../backend/tlogbe/plugins/ticketvote/cmds.go | 22 + .../plugins/{pi => ticketvote}/linkedfrom.go | 68 +- .../tlogbe/plugins/ticketvote/ticketvote.go | 1400 ++++++++++++----- politeiad/plugins/pi/pi.go | 14 +- politeiad/plugins/ticketvote/ticketvote.go | 114 +- politeiawww/proposals.go | 24 - 8 files changed, 1200 insertions(+), 867 deletions(-) create mode 100644 politeiad/backend/tlogbe/plugins/ticketvote/cmds.go rename politeiad/backend/tlogbe/plugins/{pi => ticketvote}/linkedfrom.go (60%) diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index 96a8bac59..f1784eb36 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -284,11 +284,13 @@ type Backend interface { // Perform any plugin setup that is required SetupVettedPlugin(pluginID string) error - // Execute a plugin command on an unvetted record - UnvettedPlugin(token []byte, pluginID, cmd, payload string) (string, error) + // Execute a unvetted plugin command + UnvettedPluginCmd(token []byte, pluginID, + cmd, payload string) (string, error) - // Execute a plugin command on a vetted record - VettedPlugin(token []byte, pluginID, cmd, payload string) (string, error) + // Execute a vetted plugin command + VettedPluginCmd(token []byte, pluginID, + cmd, payload string) (string, error) // Get unvetted plugins GetUnvettedPlugins() []Plugin diff --git a/politeiad/backend/tlogbe/plugins/pi/pi.go b/politeiad/backend/tlogbe/plugins/pi/pi.go index 0802ad78e..1ab54c32f 100644 --- a/politeiad/backend/tlogbe/plugins/pi/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi/pi.go @@ -5,7 +5,6 @@ package pi import ( - "encoding/base64" "encoding/hex" "encoding/json" "errors" @@ -15,7 +14,6 @@ import ( "path/filepath" "strings" "sync" - "time" "github.com/decred/dcrd/chaincfg/v3" "github.com/decred/politeia/politeiad/backend" @@ -55,28 +53,6 @@ func tokenDecode(token string) ([]byte, error) { return util.TokenDecode(util.TokenTypeTlog, token) } -// decodeProposalMetadata decodes and returns the ProposalMetadata from the -// provided backend files. If a ProposalMetadata is not found, nil is returned. -func decodeProposalMetadata(files []backend.File) (*pi.ProposalMetadata, error) { - var propMD *pi.ProposalMetadata - for _, v := range files { - if v.Name == pi.FileNameProposalMetadata { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return nil, err - } - var pm pi.ProposalMetadata - err = json.Unmarshal(b, &pm) - if err != nil { - return nil, err - } - propMD = &pm - break - } - } - return propMD, nil -} - // decodeGeneralMetadata decodes and returns the GeneralMetadata from the // provided backend metadata streams. If a GeneralMetadata is not found, nil is // returned. @@ -534,17 +510,6 @@ func (p *piPlugin) commentVote(payload string) error { // done in the comments plugin but we duplicate it here so that the // vote summary request can complete successfully. - // Verify state - switch v.State { - case comments.StateUnvetted, comments.StateVetted: - // Allowed; continue - default: - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropStateInvalid), - } - } - // Verify token token, err := tokenDecodeAnyLength(v.Token) if err != nil { @@ -612,156 +577,16 @@ func (p *piPlugin) commentVote(payload string) error { } func (p *piPlugin) ticketVoteStart(payload string) error { - // TODO - /* - // Decode payload - s, err := ticketvote.DecodeStart([]byte(payload)) - if err != nil { - return "", err - } - - // Verify work needs to be done - if len(s.Starts) == 0 { - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusStartDetailsInvalid), - ErrorContext: []string{"no start details found"}, - } - } - - // Sanity check. This pass through command should only be used for - // RFP runoff votes. - if s.Starts[0].Params.Type != ticketvote.VoteTypeRunoff { - return "", fmt.Errorf("not a runoff vote") - } - - // Get RFP token. Just use the parent token from the first vote - // params. If the different vote params use different parent - // tokens, the ticketvote plugin will catch it. - rfpToken := s.Starts[0].Params.Parent - rfpTokenb, err := tokenDecode(rfpToken) - if err != nil { - e := fmt.Sprintf("invalid rfp token '%v'", rfpToken) - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusVoteParentInvalid), - ErrorContext: []string{e}, - } - } - - // Get RFP record - rfp, err := p.backend.GetVetted(rfpTokenb, "") - if err != nil { - if errors.Is(err, errRecordNotFound) { - e := fmt.Sprintf("rfp not found %x", rfpToken) - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusVoteParentInvalid), - ErrorContext: []string{e}, - } - } - return "", fmt.Errorf("GetVetted %x: %v", rfpToken, err) - } + // TODO If runoff vote, verify that parent record has passed a + // vote itself. This functionality is specific to pi. - // Verify RFP linkby has expired. The runoff vote is not allowed to - // start until after the linkby deadline has passed. - rfpPM, err := decodeProposalMetadata(rfp.Files) - if err != nil { - return "", err - } - if rfpPM == nil { - e := fmt.Sprintf("rfp is not a proposal %v", rfpToken) - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusVoteParentInvalid), - ErrorContext: []string{e}, - } - } - isExpired := rfpPM.LinkBy < time.Now().Unix() - isMainNet := p.activeNetParams.Name == chaincfg.MainNetParams().Name - switch { - case !isExpired && isMainNet: - e := fmt.Sprintf("rfp %v linkby deadline not met %v", - rfpToken, rfpPM.LinkBy) - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusLinkByNotExpired), - ErrorContext: []string{e}, - } - case !isExpired: - // Allow the vote to be started before the link by deadline - // expires on testnet and simnet only. This makes testing the - // RFP process easier. - log.Warnf("RFP linkby deadline has not been met; disregarding " + - "since this is not mainnet") - } - - // Compile a list of the expected RFP submissions that should be in - // the runoff vote. This will be all of the public proposals that - // have linked to the RFP. The RFP's linked from list will include - // abandoned proposals that need to be filtered out. - linkedFrom, err := p.linkedFrom(rfpToken) - if err != nil { - return "", err - } - // map[token]struct{} - expected := make(map[string]struct{}, len(linkedFrom.Tokens)) - for k := range linkedFrom.Tokens { - token, err := tokenDecode(k) - if err != nil { - return "", err - } - r, err := p.backend.GetVetted(token, "") - if err != nil { - return "", err - } - if r.RecordMetadata.Status != backend.MDStatusVetted { - // This proposal is not public and should not be included in - // the runoff vote. - continue - } - - // This is a public proposal that is part of the RFP linked from - // list. It is required to be in the runoff vote. - expected[k] = struct{}{} - } - - // Verify that there are no extra submissions in the runoff vote - for _, v := range s.Starts { - _, ok := expected[v.Params.Token] - if !ok { - // This submission should not be here. - e := fmt.Sprintf("found token that should not be included: %v", - v.Params.Token) - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusStartDetailsInvalid), - ErrorContext: []string{e}, - } - } - } - - // Verify that the runoff vote is not missing any submissions - subs := make(map[string]struct{}, len(s.Starts)) - for _, v := range s.Starts { - subs[v.Params.Token] = struct{}{} - } - for token := range expected { - _, ok := subs[token] - if !ok { - // This proposal is missing from the runoff vote - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusStartDetailsMissing), - ErrorContext: []string{token}, - } - } - } + // Decode payload + s, err := ticketvote.DecodeStart([]byte(payload)) + if err != nil { + return "", err + } + _ = s - // Pi plugin validation complete! Pass the plugin command to the - // ticketvote plugin. - return p.backend.Plugin(ticketvote.ID, ticketvote.CmdStart, "", payload) - */ return nil } @@ -795,13 +620,13 @@ func (p *piPlugin) hookPluginPre(payload string) error { } func (p *piPlugin) hookNewRecordPre(payload string) error { - var nr plugins.HookNewRecord + var nr plugins.HookNewRecordPre err := json.Unmarshal([]byte(payload), &nr) if err != nil { return err } - // Decode ProposalMetadata + // Verify a proposal metadata has been included pm, err := decodeProposalMetadata(nr.Files) if err != nil { return err @@ -810,95 +635,7 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { return fmt.Errorf("proposal metadata not found") } - // TODO is linkby validated anywhere? It should be validated here - // and in the edit proposal. - - // Verify the linkto is an RFP and that the RFP is eligible to be - // linked to. We currently only allow linking to RFP proposals that - // have been approved by a ticket vote. - if pm.LinkTo != "" { - if isRFP(*pm) { - return backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropLinkToInvalid), - ErrorContext: "an rfp cannot have linkto set", - } - } - tokenb, err := hex.DecodeString(pm.LinkTo) - if err != nil { - return backend.PluginUserError{ - - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropLinkToInvalid), - ErrorContext: "invalid hex", - } - } - r, err := p.backend.GetVetted(tokenb, "") - if err != nil { - if errors.Is(err, backend.ErrRecordNotFound) { - return backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropLinkToInvalid), - ErrorContext: "proposal not found", - } - } - return err - } - linkToPM, err := decodeProposalMetadata(r.Files) - if err != nil { - return err - } - if linkToPM == nil { - return backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropLinkToInvalid), - ErrorContext: "proposal not an rfp", - } - } - if !isRFP(*linkToPM) { - return backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropLinkToInvalid), - ErrorContext: "proposal not an rfp", - } - } - if time.Now().Unix() > linkToPM.LinkBy { - // Link by deadline has expired. New links are not allowed. - return backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropLinkToInvalid), - ErrorContext: "rfp link by deadline expired", - } - } - s := ticketvote.Summaries{ - Tokens: []string{pm.LinkTo}, - } - b, err := ticketvote.EncodeSummaries(s) - if err != nil { - return err - } - reply, err := p.backend.Plugin(ticketvote.ID, - ticketvote.CmdSummaries, "", string(b)) - if err != nil { - return fmt.Errorf("Plugin %v %v: %v", - ticketvote.ID, ticketvote.CmdSummaries, err) - } - sr, err := ticketvote.DecodeSummariesReply([]byte(reply)) - if err != nil { - return err - } - summary, ok := sr.Summaries[pm.LinkTo] - if !ok { - return fmt.Errorf("summary not found %v", pm.LinkTo) - } - if !summary.Approved { - return backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropLinkToInvalid), - ErrorContext: "rfp vote not approved", - } - } - } + // TODO Verify proposal name return nil } @@ -929,73 +666,51 @@ func (p *piPlugin) hookNewRecordPost(payload string) error { } func (p *piPlugin) hookEditRecordPre(payload string) error { - var er plugins.HookEditRecord - err := json.Unmarshal([]byte(payload), &er) - if err != nil { - return err - } - - // TODO verify files were changed. Before adding this, verify that - // politeiad will also error if no files were changed. - - // Verify that the linkto has not changed. This only applies to - // public proposal. Unvetted proposals are allowed to change their - // linkto. - status := convertPropStatusFromMDStatus(er.Current.RecordMetadata.Status) - if status == pi.PropStatusPublic { - pmCurr, err := decodeProposalMetadata(er.Current.Files) - if err != nil { - return err - } - pmNew, err := decodeProposalMetadata(er.FilesAdd) + /* + var er plugins.HookEditRecord + err := json.Unmarshal([]byte(payload), &er) if err != nil { return err } - if pmCurr.LinkTo != pmNew.LinkTo { - return backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropLinkToInvalid), - ErrorContext: "linkto cannot change on public proposal", - } - } - } - // TODO verify linkto is allowed + // TODO verify files were changed. Before adding this, verify that + // politeiad will also error if no files were changed. - // Verify vote status. This is only required for public proposals. - if status == pi.PropStatusPublic { - token := er.RecordMetadata.Token - s := ticketvote.Summaries{ - Tokens: []string{token}, - } - b, err := ticketvote.EncodeSummaries(s) - if err != nil { - return err - } - reply, err := p.backend.Plugin(ticketvote.ID, - ticketvote.CmdSummaries, "", string(b)) - if err != nil { - return fmt.Errorf("ticketvote Summaries: %v", err) - } - sr, err := ticketvote.DecodeSummariesReply([]byte(reply)) - if err != nil { - return err - } - summary, ok := sr.Summaries[token] - if !ok { - return fmt.Errorf("ticketvote summmary not found") - } - if summary.Status != ticketvote.VoteStatusUnauthorized { - e := fmt.Sprintf("vote status got %v, want %v", - ticketvote.VoteStatuses[summary.Status], - ticketvote.VoteStatuses[ticketvote.VoteStatusUnauthorized]) - return backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusVoteStatusInvalid), - ErrorContext: e, + // Verify vote status. This is only required for public proposals. + if status == pi.PropStatusPublic { + token := er.RecordMetadata.Token + s := ticketvote.Summaries{ + Tokens: []string{token}, + } + b, err := ticketvote.EncodeSummaries(s) + if err != nil { + return err + } + reply, err := p.backend.Plugin(ticketvote.ID, + ticketvote.CmdSummaries, "", string(b)) + if err != nil { + return fmt.Errorf("ticketvote Summaries: %v", err) + } + sr, err := ticketvote.DecodeSummariesReply([]byte(reply)) + if err != nil { + return err + } + summary, ok := sr.Summaries[token] + if !ok { + return fmt.Errorf("ticketvote summmary not found") + } + if summary.Status != ticketvote.VoteStatusUnauthorized { + e := fmt.Sprintf("vote status got %v, want %v", + ticketvote.VoteStatuses[summary.Status], + ticketvote.VoteStatuses[ticketvote.VoteStatusUnauthorized]) + return backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorStatusVoteStatusInvalid), + ErrorContext: e, + } } } - } + */ return nil } @@ -1069,38 +784,6 @@ func (p *piPlugin) hookSetRecordStatusPost(payload string) error { } } - // If the LinkTo field has been set then the linkedFrom - // list might need to be updated for the proposal that is being - // linked to, depending on the status change that is being made. - pm, err := decodeProposalMetadata(srs.Current.Files) - if err != nil { - return err - } - if pm != nil && pm.LinkTo != "" { - // Link from has been set. Check if the status change requires - // the parent proposal's linked from list to be updated. - var ( - parentToken = pm.LinkTo - childToken = srs.RecordMetadata.Token - ) - switch srs.RecordMetadata.Status { - case backend.MDStatusVetted: - // Proposal has been made public. Add child token to parent - // token's linked from list. - err := p.linkedFromAdd(parentToken, childToken) - if err != nil { - return fmt.Errorf("linkedFromAdd: %v", err) - } - case backend.MDStatusCensored: - // Proposal has been censored. Delete child token from parent - // token's linked from list. - err := p.linkedFromDel(parentToken, childToken) - if err != nil { - return fmt.Errorf("linkedFromDel: %v", err) - } - } - } - return nil } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go new file mode 100644 index 000000000..f959be277 --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go @@ -0,0 +1,22 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ticketvote + +import "github.com/decred/politeia/politeiad/plugins/ticketvote" + +const ( + // Internal plugin commands + cmdStartRunoffSub = "startrunoffsub" +) + +// startRunoffSub is an internal plugin command that is used to start the +// voting period on a runoff vote submission. +type startRunoffSub struct { + ParentTreeID int64 `json:"parenttreeid"` + StartDetails ticketvote.StartDetails `json:"startdetails"` +} + +// startRunoffSubReply is the reply to the startRunoffSub command. +type startRunoffSubReply struct{} diff --git a/politeiad/backend/tlogbe/plugins/pi/linkedfrom.go b/politeiad/backend/tlogbe/plugins/ticketvote/linkedfrom.go similarity index 60% rename from politeiad/backend/tlogbe/plugins/pi/linkedfrom.go rename to politeiad/backend/tlogbe/plugins/ticketvote/linkedfrom.go index f229de117..ccd204701 100644 --- a/politeiad/backend/tlogbe/plugins/pi/linkedfrom.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/linkedfrom.go @@ -2,12 +2,11 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package pi +package ticketvote import ( "encoding/json" "errors" - "fmt" "io/ioutil" "os" "path/filepath" @@ -22,34 +21,32 @@ const ( fnLinkedFrom = "{tokenprefix}-linkedfrom.json" ) -// linkedFrom is the the structure that is updated and cached for proposal A -// when proposal B links to proposal A. Proposals can link to one another using -// the ProposalMetadata LinkTo field. The linkedFrom list contains all -// proposals that have linked to proposal A. The list will only contain public -// proposals. The linkedFrom list is saved to disk in the pi plugin data dir, -// specifying the parent proposal token in the filename. +// linkedFrom is the the structure that is updated and cached for record A when +// record B links to record A. Recordss can link to one another using the +// VoteMetadata LinkTo field. The linkedFrom list contains all records that +// have linked to record A. The list will only contain public records. The +// linkedFrom list is saved to disk in the ticketvote plugin data dir, with the +// parent record token in the filename. // -// Example: the linked from list for an RFP proposal will contain all public -// RFP submissions. The cached list can be found in the pi plugin data dir -// at the path specified by linkedFromPath(). +// Example: the linked from list for a runoff vote parent record will contain +// all public runoff vote submissions. type linkedFrom struct { Tokens map[string]struct{} `json:"tokens"` } // linkedFromPath returns the path to the linkedFrom list for the provided -// proposal token. The token prefix is used in the file path so that the linked +// record token. The token prefix is used in the file path so that the linked // from list can be retrieved using either the full token or the token prefix. -func (p *piPlugin) linkedFromPath(token []byte) string { +func (p *ticketVotePlugin) linkedFromPath(token []byte) string { t := util.TokenPrefix(token) fn := strings.Replace(fnLinkedFrom, "{tokenprefix}", t, 1) return filepath.Join(p.dataDir, fn) } -// linkedFromWithLock return the linkedFrom list for the provided proposal -// token. +// linkedFromWithLock return the linkedFrom list for a record token. // // This function must be called WITH the lock held. -func (p *piPlugin) linkedFromWithLock(token []byte) (*linkedFrom, error) { +func (p *ticketVotePlugin) linkedFromWithLock(token []byte) (*linkedFrom, error) { fp := p.linkedFromPath(token) b, err := ioutil.ReadFile(fp) if err != nil { @@ -71,21 +68,20 @@ func (p *piPlugin) linkedFromWithLock(token []byte) (*linkedFrom, error) { return &lf, nil } -// linkedFrom return the linkedFrom list for the provided proposal token. +// linkedFrom return the linkedFrom list for a record token. // // This function must be called WITHOUT the lock held. -func (p *piPlugin) linkedFrom(token []byte) (*linkedFrom, error) { +func (p *ticketVotePlugin) linkedFrom(token []byte) (*linkedFrom, error) { p.Lock() defer p.Unlock() return p.linkedFromWithLock(token) } -// linkedFromSaveWithLock saves the provided linkedFrom list to the pi plugin -// data dir. +// linkedFromSaveWithLock saves a linkedFrom to the plugin data dir. // // This function must be called WITH the lock held. -func (p *piPlugin) linkedFromSaveWithLock(token []byte, lf linkedFrom) error { +func (p *ticketVotePlugin) linkedFromSaveWithLock(token []byte, lf linkedFrom) error { b, err := json.Marshal(lf) if err != nil { return err @@ -95,10 +91,10 @@ func (p *piPlugin) linkedFromSaveWithLock(token []byte, lf linkedFrom) error { } // linkedFromAdd updates the cached linkedFrom list for the parentToken, adding -// the childToken to the list. +// the childToken to the list. The full length token MUST be used. // // This function must be called WITHOUT the lock held. -func (p *piPlugin) linkedFromAdd(parentToken, childToken string) error { +func (p *ticketVotePlugin) linkedFromAdd(parentToken, childToken string) error { p.Lock() defer p.Unlock() @@ -115,21 +111,29 @@ func (p *piPlugin) linkedFromAdd(parentToken, childToken string) error { // Get existing linked from list lf, err := p.linkedFromWithLock(parent) if err != nil { - return fmt.Errorf("linkedFromWithLock %x: %v", parent, err) + return err } // Update list lf.Tokens[childToken] = struct{}{} // Save list - return p.linkedFromSaveWithLock(parent, *lf) + err = p.linkedFromSaveWithLock(parent, *lf) + if err != nil { + return err + } + + log.Debugf("Linked from list updated. Child %v added to parent %v", + childToken, parentToken) + + return nil } // linkedFromDel updates the cached linkedFrom list for the parentToken, // deleting the childToken from the list. // // This function must be called WITHOUT the lock held. -func (p *piPlugin) linkedFromDel(parentToken, childToken string) error { +func (p *ticketVotePlugin) linkedFromDel(parentToken, childToken string) error { p.Lock() defer p.Unlock() @@ -146,12 +150,20 @@ func (p *piPlugin) linkedFromDel(parentToken, childToken string) error { // Get existing linked from list lf, err := p.linkedFromWithLock(parent) if err != nil { - return fmt.Errorf("linkedFromWithLock %x: %v", parent, err) + return err } // Update list delete(lf.Tokens, childToken) // Save list - return p.linkedFromSaveWithLock(parent, *lf) + err = p.linkedFromSaveWithLock(parent, *lf) + if err != nil { + return err + } + + log.Debugf("Linked from list updated. Child %v deleted from parent %v", + childToken, parentToken) + + return nil } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go index ae3c629ed..cca53ae76 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go @@ -49,22 +49,19 @@ const ( dataDescriptorAuthDetails = "authdetails_v1" dataDescriptorVoteDetails = "votedetails_v1" dataDescriptorCastVoteDetails = "castvotedetails_v1" + dataDescriptorStartRunoff = "startrunoff_v1" // Data types dataTypeAuthDetails = "authdetails" dataTypeVoteDetails = "votedetails" dataTypeCastVoteDetails = "castvotedetails" + dataTypeStartRunoff = "startrunoff" ) var ( _ plugins.Client = (*ticketVotePlugin)(nil) ) -// TODO holding the lock before verifying the token can allow the mutexes to -// be spammed. Create an infinite amount of them with invalid tokens. The fix -// is to check if the record exists in the mutexes function to ensure a token -// is valid before holding the lock on it. - // TODO verify all writes only accept full length tokens // ticketVotePlugin satisfies the plugins.Client interface. @@ -83,6 +80,8 @@ type ticketVotePlugin struct { // Plugin settings voteDurationMin uint32 // In blocks voteDurationMax uint32 // In blocks + linkByPeriodMin int64 // In seconds + linkByPeriodMax int64 // In seconds // identity contains the full identity that the plugin uses to // create receipts, i.e. signatures of user provided data that @@ -96,25 +95,44 @@ type ticketVotePlugin struct { // votes contains the cast votes of ongoing record votes. This // cache is built on startup and record entries are removed once - // the vote has ended and the vote summary has been saved. + // the vote has ended and a vote summary has been cached. votes map[string]map[string]string // [token][ticket]voteBit // Mutexes contains a mutex for each record and are used to lock // the trillian tree for a given record to prevent concurrent - // ticket vote plugin updates to the same tree. They are not used - // to update any of the ticket vote plugin memory caches. These - // mutexes are lazy loaded. + // ticket vote plugin updates on the same tree. These mutexes are + // lazy loaded and should only be used for tree updates, not for + // cache updates. mutexes map[string]*sync.Mutex // [string]mutex } +// tokenDecode decodes a record token and only accepts full length tokens. func tokenDecode(token string) ([]byte, error) { return util.TokenDecode(util.TokenTypeTlog, token) } +// tokenDecode decodes a record token and accepts both full length tokens and +// token prefixes. func tokenDecodeAnyLength(token string) ([]byte, error) { return util.TokenDecodeAnyLength(util.TokenTypeTlog, token) } +// mutex returns the mutex for the specified record. +func (p *ticketVotePlugin) mutex(token []byte) *sync.Mutex { + p.Lock() + defer p.Unlock() + + t := hex.EncodeToString(token) + m, ok := p.mutexes[t] + if !ok { + // Mutexes is lazy loaded + m = &sync.Mutex{} + p.mutexes[t] = m + } + + return m +} + func (p *ticketVotePlugin) cachedVotes(token []byte) map[string]string { p.Lock() defer p.Unlock() @@ -143,7 +161,7 @@ func (p *ticketVotePlugin) cachedVotesSet(token, ticket, voteBit string) { p.votes[token][ticket] = voteBit - log.Debugf("ticketvote: added vote to cache: %v %v %v", + log.Debugf("Added vote to cache: %v %v %v", token, ticket, voteBit) } @@ -153,7 +171,7 @@ func (p *ticketVotePlugin) cachedVotesDel(token string) { delete(p.votes, token) - log.Debugf("ticketvote: deleted votes cache: %v", token) + log.Debugf("Deleted votes cache: %v", token) } // cachedSummaryPath accepts both full tokens and token prefixes, however it @@ -173,7 +191,7 @@ var ( errSummaryNotFound = errors.New("summary not found") ) -func (p *ticketVotePlugin) cachedSummary(token string) (*ticketvote.Summary, error) { +func (p *ticketVotePlugin) cachedSummary(token string) (*ticketvote.VoteSummary, error) { p.Lock() defer p.Unlock() @@ -191,17 +209,17 @@ func (p *ticketVotePlugin) cachedSummary(token string) (*ticketvote.Summary, err return nil, err } - var s ticketvote.Summary - err = json.Unmarshal(b, &s) + var vs ticketvote.VoteSummary + err = json.Unmarshal(b, &vs) if err != nil { return nil, err } - return &s, nil + return &vs, nil } -func (p *ticketVotePlugin) cachedSummarySave(token string, s ticketvote.Summary) error { - b, err := json.Marshal(s) +func (p *ticketVotePlugin) cachedSummarySave(token string, vs ticketvote.VoteSummary) error { + b, err := json.Marshal(vs) if err != nil { return err } @@ -218,35 +236,20 @@ func (p *ticketVotePlugin) cachedSummarySave(token string, s ticketvote.Summary) return err } - log.Debugf("ticketvote: saved votes summary: %v", token) + log.Debugf("Saved votes summary: %v", token) return nil } -// mutex returns the mutex for the specified record. -func (p *ticketVotePlugin) mutex(token string) *sync.Mutex { - p.Lock() - defer p.Unlock() - - m, ok := p.mutexes[token] - if !ok { - // Mutexes is lazy loaded - m = &sync.Mutex{} - p.mutexes[token] = m - } - - return m -} - func convertSignatureError(err error) backend.PluginUserError { var e util.SignatureError - var s ticketvote.ErrorStatusT + var s ticketvote.ErrorCodeT if errors.As(err, &e) { switch e.ErrorCode { case util.ErrorStatusPublicKeyInvalid: - s = ticketvote.ErrorStatusPublicKeyInvalid + s = ticketvote.ErrorCodePublicKeyInvalid case util.ErrorStatusSignatureInvalid: - s = ticketvote.ErrorStatusSignatureInvalid + s = ticketvote.ErrorCodeSignatureInvalid } } return backend.PluginUserError{ @@ -370,6 +373,44 @@ func convertCastVoteDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.CastVo return &cv, nil } +func convertStartRunoffFromBlobEntry(be store.BlobEntry) (*startRunoffRecord, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorStartRunoff { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorStartRunoff) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, fmt.Errorf("decode digest: %v", err) + } + if !bytes.Equal(util.Digest(b), digest) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), digest) + } + var srr startRunoffRecord + err = json.Unmarshal(b, &srr) + if err != nil { + return nil, fmt.Errorf("unmarshal StartRunoffRecord: %v", err) + } + + return &srr, nil +} + func convertBlobEntryFromAuthDetails(ad ticketvote.AuthDetails) (*store.BlobEntry, error) { data, err := json.Marshal(ad) if err != nil { @@ -421,6 +462,23 @@ func convertBlobEntryFromCastVoteDetails(cv ticketvote.CastVoteDetails) (*store. return &be, nil } +func convertBlobEntryFromStartRunoff(srr startRunoffRecord) (*store.BlobEntry, error) { + data, err := json.Marshal(srr) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorStartRunoff, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + func (p *ticketVotePlugin) authSave(treeID int64, ad ticketvote.AuthDetails) error { // Prepare blob be, err := convertBlobEntryFromAuthDetails(ad) @@ -533,6 +591,154 @@ func (p *ticketVotePlugin) castVotes(treeID int64) ([]ticketvote.CastVoteDetails return votes, nil } +// summary returns the vote summary for a record. +func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) (*ticketvote.VoteSummary, error) { + // Check if the summary has been cached + s, err := p.cachedSummary(hex.EncodeToString(token)) + switch { + case errors.Is(err, errSummaryNotFound): + // Cached summary not found. Continue. + case err != nil: + // Some other error + return nil, fmt.Errorf("cachedSummary: %v", err) + default: + // Caches summary was found. Return it. + return s, nil + } + + // Summary has not been cached. Get it manually. + + // Assume vote is unauthorized. Only update the status when the + // appropriate record has been found that proves otherwise. + status := ticketvote.VoteStatusUnauthorized + + // Check if the vote has been authorized + auths, err := p.auths(treeID) + if err != nil { + return nil, fmt.Errorf("auths: %v", err) + } + if len(auths) > 0 { + lastAuth := auths[len(auths)-1] + switch ticketvote.AuthActionT(lastAuth.Action) { + case ticketvote.AuthActionAuthorize: + // Vote has been authorized; continue + status = ticketvote.VoteStatusAuthorized + case ticketvote.AuthActionRevoke: + // Vote authorization has been revoked. Its not possible for + // the vote to have been started. We can stop looking. + return &ticketvote.VoteSummary{ + Status: status, + Results: []ticketvote.VoteOptionResult{}, + }, nil + } + } + + // Check if the vote has been started + vd, err := p.voteDetails(treeID) + if err != nil { + return nil, fmt.Errorf("startDetails: %v", err) + } + if vd == nil { + // Vote has not been started yet + return &ticketvote.VoteSummary{ + Status: status, + Results: []ticketvote.VoteOptionResult{}, + }, nil + } + + // Vote has been started. Check if it is still in progress or has + // already ended. + if bestBlock < vd.EndBlockHeight { + status = ticketvote.VoteStatusStarted + } else { + status = ticketvote.VoteStatusFinished + } + + // Pull the cast votes from the cache and tally the results + votes := p.cachedVotes(token) + tally := make(map[string]int, len(vd.Params.Options)) + for _, voteBit := range votes { + tally[voteBit]++ + } + results := make([]ticketvote.VoteOptionResult, 0, len(vd.Params.Options)) + for _, v := range vd.Params.Options { + bit := strconv.FormatUint(v.Bit, 16) + results = append(results, ticketvote.VoteOptionResult{ + ID: v.ID, + Description: v.Description, + VoteBit: v.Bit, + Votes: uint64(tally[bit]), + }) + } + + // Prepare summary + summary := ticketvote.VoteSummary{ + Type: vd.Params.Type, + Status: status, + Duration: vd.Params.Duration, + StartBlockHeight: vd.StartBlockHeight, + StartBlockHash: vd.StartBlockHash, + EndBlockHeight: vd.EndBlockHeight, + EligibleTickets: uint32(len(vd.EligibleTickets)), + QuorumPercentage: vd.Params.QuorumPercentage, + PassPercentage: vd.Params.PassPercentage, + Results: results, + } + + // If the vote has not finished yet then we are done for now. + if status == ticketvote.VoteStatusStarted { + return &summary, nil + } + + // The vote has finished. We can calculate if the vote was approved + // for certain vote types and cache the results. + switch vd.Params.Type { + case ticketvote.VoteTypeStandard, ticketvote.VoteTypeRunoff: + // These vote types are strictly approve/reject votes so we can + // calculate the vote approval. Continue. + default: + // Nothing else to do for all other vote types + return &summary, nil + } + + // Calculate vote approval + approved := voteIsApproved(*vd, results) + + // If this is a standard vote then we can take the results as is. + // A runoff vote requires that we pull all other runoff vote + // submissions to determine if the vote actually passed. + // TODO make summary for runoff vote submissions + summary.Approved = approved + + // Cache the summary + err = p.cachedSummarySave(vd.Params.Token, summary) + if err != nil { + return nil, fmt.Errorf("cachedSummarySave %v: %v %v", + vd.Params.Token, err, summary) + } + + // Remove record from the votes cache now that a summary has been + // saved for it. + p.cachedVotesDel(vd.Params.Token) + + return &summary, nil +} + +func (p *ticketVotePlugin) summaryByToken(token []byte) (*ticketvote.VoteSummary, error) { + reply, err := p.backend.VettedPluginCmd(token, ticketvote.ID, + ticketvote.CmdSummary, "") + if err != nil { + return nil, fmt.Errorf("VettedPluginCmd %x %v %v: %v", + token, ticketvote.ID, ticketvote.CmdSummary, err) + } + var sr ticketvote.SummaryReply + err = json.Unmarshal([]byte(reply), &sr) + if err != nil { + return nil, err + } + return &sr.Summary, nil +} + func (p *ticketVotePlugin) timestamp(treeID int64, digest []byte) (*ticketvote.Timestamp, error) { t, err := p.tlog.Timestamp(treeID, digest) if err != nil { @@ -855,7 +1061,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri if err != nil { return "", backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusTokenInvalid), + ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), } } if !bytes.Equal(t, token) { @@ -863,7 +1069,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri "got %x, want %x", t, token) return "", backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusTokenInvalid), + ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), ErrorContext: e, } } @@ -886,7 +1092,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri e := fmt.Sprintf("%v not a valid action", a.Action) return "", backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusAuthorizationInvalid), + ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), ErrorContext: e, } } @@ -894,7 +1100,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri // The previous authorize votes must be retrieved to validate the // new autorize vote. The lock must be held for the remainder of // this function. - m := p.mutex(a.Token) + m := p.mutex(token) m.Lock() defer m.Unlock() @@ -914,7 +1120,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri if a.Action != ticketvote.AuthActionAuthorize { return "", backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusAuthorizationInvalid), + ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), ErrorContext: "no prev action; action must be authorize", } } @@ -923,7 +1129,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri // Previous action was a authorize. This action must be revoke. return "", backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusAuthorizationInvalid), + ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), ErrorContext: "prev action was authorize", } case prevAction == ticketvote.AuthActionRevoke && @@ -931,7 +1137,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri // Previous action was a revoke. This action must be authorize. return "", backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusAuthorizationInvalid), + ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), ErrorContext: "prev action was revoke", } } @@ -1015,7 +1221,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM e := fmt.Sprintf("invalid type %v", vote.Type) return backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } } @@ -1027,7 +1233,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM vote.Duration, voteDurationMax) return backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } case vote.Duration < voteDurationMin: @@ -1035,7 +1241,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM vote.Duration, voteDurationMin) return backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } case vote.QuorumPercentage > 100: @@ -1043,7 +1249,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM vote.QuorumPercentage) return backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } case vote.PassPercentage > 100: @@ -1051,7 +1257,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM vote.PassPercentage) return backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } } @@ -1061,7 +1267,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM if len(vote.Options) == 0 { return backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: "no vote options found", } } @@ -1075,7 +1281,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM len(vote.Options)) return backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } } @@ -1104,7 +1310,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM strings.Join(missing, ",")) return backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } } @@ -1116,7 +1322,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM if err != nil { return backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: err.Error(), } } @@ -1128,7 +1334,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM e := "parent token should not be provided for a standard vote" return backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } case vote.Type == ticketvote.VoteTypeRunoff: @@ -1137,7 +1343,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM e := fmt.Sprintf("invalid parent %v", vote.Parent) return backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } } @@ -1147,23 +1353,32 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM } // startStandard starts a standard vote. -func (p *ticketVotePlugin) startStandard(treeID int64, s ticketvote.Start) (*ticketvote.StartReply, error) { +func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvote.Start) (*ticketvote.StartReply, error) { // Verify there is only one start details if len(s.Starts) != 1 { return nil, backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusStartDetailsInvalid), + ErrorCode: int(ticketvote.ErrorCodeStartDetailsInvalid), ErrorContext: "more than one start details found", } } sd := s.Starts[0] // Verify token - token, err := tokenDecode(sd.Params.Token) + t, err := tokenDecode(sd.Params.Token) if err != nil { return nil, backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusTokenInvalid), + ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), + } + } + if !bytes.Equal(t, token) { + e := fmt.Sprintf("plugin token does not match route token: "+ + "got %x, want %x", t, token) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), + ErrorContext: e, } } @@ -1192,7 +1407,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, s ticketvote.Start) (*tic // Validate existing record state. The lock for this record must be // held for the remainder of this function. - m := p.mutex(sd.Params.Token) + m := p.mutex(token) m.Lock() defer m.Unlock() @@ -1207,7 +1422,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, s ticketvote.Start) (*tic sd.Params.Version, r.Version) return nil, backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusRecordVersionInvalid), + ErrorCode: int(ticketvote.ErrorCodeRecordVersionInvalid), ErrorContext: e, } } @@ -1220,7 +1435,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, s ticketvote.Start) (*tic if len(auths) == 0 { return nil, backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusAuthorizationInvalid), + ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), ErrorContext: "authorization not found", } } @@ -1228,7 +1443,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, s ticketvote.Start) (*tic if action != ticketvote.AuthActionAuthorize { return nil, backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusAuthorizationInvalid), + ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), ErrorContext: "not authorized", } } @@ -1242,7 +1457,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, s ticketvote.Start) (*tic // Vote has already been started return nil, backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusVoteStatusInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteStatusInvalid), ErrorContext: "vote already started", } } @@ -1276,31 +1491,340 @@ func (p *ticketVotePlugin) startStandard(treeID int64, s ticketvote.Start) (*tic }, nil } -// startRunoff starts a runoff vote. -func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartReply, error) { +// startRunoffRecord is the record that is saved to the runoff vote's parent +// tree as the first step in starting a runoff vote. Plugins are not able to +// update multiple records atomically, so if this call gets interrupted before +// if can start the voting period on all runoff vote submissions, subsequent +// calls will use this record to pick up where the previous call left off. This +// allows us to recover from unexpected errors, such as network errors, and not +// leave a runoff vote in a weird state. +type startRunoffRecord struct { + Submissions []string `json:"submissions"` + Mask uint64 `json:"mask"` + Duration uint32 `json:"duration"` + QuorumPercentage uint32 `json:"quorumpercentage"` + PassPercentage uint32 `json:"passpercentage"` + StartBlockHeight uint32 `json:"startblockheight"` + StartBlockHash string `json:"startblockhash"` + EndBlockHeight uint32 `json:"endblockheight"` + EligibleTickets []string `json:"eligibletickets"` +} + +func (p *ticketVotePlugin) startRunoffRecordSave(treeID int64, srr startRunoffRecord) error { + be, err := convertBlobEntryFromStartRunoff(srr) + if err != nil { + return err + } + err = p.tlog.BlobSave(treeID, dataTypeStartRunoff, *be) + if err != nil { + return fmt.Errorf("BlobSave %v %v: %v", + treeID, dataTypeStartRunoff, err) + } + return nil +} + +// startRunoffRecord returns the startRunoff record if one exists on a tree. +// nil will be returned if a startRunoff record is not found. +func (p *ticketVotePlugin) startRunoffRecord(treeID int64) (*startRunoffRecord, error) { + blobs, err := p.tlog.BlobsByDataType(treeID, dataTypeStartRunoff) + if err != nil { + return nil, fmt.Errorf("BlobsByDataType %v %v: %v", + treeID, dataTypeStartRunoff, err) + } + + var srr *startRunoffRecord + switch len(blobs) { + case 0: + // Nothing found + return nil, nil + case 1: + // A start runoff record was found + srr, err = convertStartRunoffFromBlobEntry(blobs[0]) + if err != nil { + return nil, err + } + default: + // This should not be possible + e := fmt.Sprintf("%v start runoff blobs found", len(blobs)) + panic(e) + } + + return srr, nil +} + +func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs startRunoffSub) error { // Sanity check - if len(s.Starts) == 0 { - return nil, fmt.Errorf("no start details found") + s := srs.StartDetails + t, err := tokenDecode(s.Params.Token) + if err != nil { + return err + } + if bytes.Equal(token, t) { + return fmt.Errorf("invalid token") } - // Perform validation that can be done without fetching any records - // from the backend. - var ( - mask = s.Starts[0].Params.Mask - duration = s.Starts[0].Params.Duration - quorum = s.Starts[0].Params.QuorumPercentage - pass = s.Starts[0].Params.PassPercentage - parent = s.Starts[0].Params.Parent - ) - for _, v := range s.Starts { - // Verify vote params are the same for all submissions - switch { - case v.Params.Type != ticketvote.VoteTypeRunoff: + // Get the start runoff record from the parent tree + srr, err := p.startRunoffRecord(srs.ParentTreeID) + if err != nil { + return err + } + + // Sanity check. Verify token is part of the start runoff record + // submissions. + var found bool + for _, v := range srr.Submissions { + if hex.EncodeToString(token) == v { + found = true + break + } + } + if !found { + // This submission should not be here + return fmt.Errorf("record not in submission list") + } + + // If the vote has already been started, exit gracefully. This + // allows us to recover from unexpected errors to the start runoff + // vote call as it updates the state of multiple records. If the + // call were to fail before completing, we can simply call the + // command again with the same arguments and it will pick up where + // it left off. + svp, err := p.voteDetails(treeID) + if err != nil { + return err + } + if svp != nil { + // Vote has already been started. Exit gracefully. + return nil + } + + // Verify record version + r, err := p.backend.GetVetted(token, "") + if err != nil { + return fmt.Errorf("GetVetted: %v", err) + } + version := strconv.FormatUint(uint64(s.Params.Version), 10) + if r.Version != version { + e := fmt.Sprintf("version is not latest %v: got %v, want %v", + s.Params.Token, s.Params.Version, r.Version) + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeRecordVersionInvalid), + ErrorContext: e, + } + } + + // Prepare vote details + vd := ticketvote.VoteDetails{ + Params: s.Params, + PublicKey: s.PublicKey, + Signature: s.Signature, + StartBlockHeight: srr.StartBlockHeight, + StartBlockHash: srr.StartBlockHash, + EndBlockHeight: srr.EndBlockHeight, + EligibleTickets: srr.EligibleTickets, + } + + // Save vote details + err = p.voteDetailsSave(treeID, vd) + if err != nil { + return fmt.Errorf("voteDetailsSave: %v", err) + } + + // Update inventory + p.inventorySetToStarted(vd.Params.Token, ticketvote.VoteTypeRunoff, + vd.EndBlockHeight) + + return nil +} + +func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ticketvote.Start) (*startRunoffRecord, error) { + // Check if the runoff vote data already exists on the parent tree. + srr, err := p.startRunoffRecord(treeID) + if err != nil { + return nil, err + } + if srr != nil { + // We already have a start runoff record for this runoff vote. + // This can happen if the previous call failed due to an + // unexpected error such as a network error. Return the start + // runoff record so we can pick up where we left off. + return srr, nil + } + + // Get blockchain data + var ( + mask = s.Starts[0].Params.Mask + duration = s.Starts[0].Params.Duration + quorum = s.Starts[0].Params.QuorumPercentage + pass = s.Starts[0].Params.PassPercentage + ) + sr, err := p.startReply(duration) + if err != nil { + return nil, err + } + + // The parent record must be validated. Hold the lock on the parent + // record for the remainder of this function. + m := p.mutex(token) + m.Lock() + defer m.Unlock() + + // Verify parent has a LinkBy and the LinkBy deadline is expired. + r, err := p.backend.GetVetted(token, "") + if err != nil { + if errors.Is(err, backend.ErrRecordNotFound) { + e := fmt.Sprintf("parent record not found %x", token) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorContext: e, + } + } + } + vm, err := decodeVoteMetadata(r.Files) + if err != nil { + return nil, err + } + if vm == nil || vm.LinkBy == 0 { + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeRunoffVoteParentInvalid), + } + } + isExpired := vm.LinkBy < time.Now().Unix() + isMainNet := p.activeNetParams.Name == chaincfg.MainNetParams().Name + switch { + case !isExpired && isMainNet: + e := fmt.Sprintf("parent record %x linkby deadline not met %v", + token, vm.LinkBy) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeLinkByNotExpired), + ErrorContext: e, + } + case !isExpired: + // Allow the vote to be started before the link by deadline + // expires on testnet and simnet only. This makes testing the + // runoff vote process easier. + log.Warnf("Parent record linkby deadline has not been met; " + + "disregarding deadline since this is not mainnet") + } + + // Compile a list of the expected submissions that should be in the + // runoff vote. This will be all of the public records that have + // linked to the parent record. The parent records' linked from + // list will include abandoned proposals that need to be filtered + // out. + lf, err := p.linkedFrom(token) + if err != nil { + return nil, err + } + expected := make(map[string]struct{}, len(lf.Tokens)) // [token]struct{} + for k := range lf.Tokens { + token, err := tokenDecode(k) + if err != nil { + return nil, err + } + r, err := p.backend.GetVetted(token, "") + if err != nil { + return nil, err + } + if r.RecordMetadata.Status != backend.MDStatusVetted { + // This record is not public and should not be included + // in the runoff vote. + continue + } + + // This is a public record that is part of the parent record's + // linked from list. It is required to be in the runoff vote. + expected[k] = struct{}{} + } + + // Verify that there are no extra submissions in the runoff vote + for _, v := range s.Starts { + _, ok := expected[v.Params.Token] + if !ok { + // This submission should not be here + e := fmt.Sprintf("record %v should not be included", + v.Params.Token) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeStartDetailsInvalid), + ErrorContext: e, + } + } + } + + // Verify that the runoff vote is not missing any submissions + subs := make(map[string]struct{}, len(s.Starts)) + for _, v := range s.Starts { + subs[v.Params.Token] = struct{}{} + } + for k := range expected { + _, ok := subs[k] + if !ok { + // This records is missing from the runoff vote + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeStartDetailsMissing), + ErrorContext: k, + } + } + } + + // Prepare start runoff record + submissions := make([]string, 0, len(subs)) + for k := range subs { + submissions = append(submissions, k) + } + srr = &startRunoffRecord{ + Submissions: submissions, + Mask: mask, + Duration: duration, + QuorumPercentage: quorum, + PassPercentage: pass, + StartBlockHeight: sr.StartBlockHeight, + StartBlockHash: sr.StartBlockHash, + EndBlockHeight: sr.EndBlockHeight, + EligibleTickets: sr.EligibleTickets, + } + + // Save start runoff record + err = p.startRunoffRecordSave(treeID, *srr) + if err != nil { + return nil, fmt.Errorf("startRunoffRecordSave %v: %v", + treeID, err) + } + + return srr, nil +} + +// startRunoff starts a runoff vote. +func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote.Start) (*ticketvote.StartReply, error) { + // Sanity check + if len(s.Starts) == 0 { + return nil, fmt.Errorf("no start details found") + } + + // Perform validation that can be done without fetching any records + // from the backend. + var ( + mask = s.Starts[0].Params.Mask + duration = s.Starts[0].Params.Duration + quorum = s.Starts[0].Params.QuorumPercentage + pass = s.Starts[0].Params.PassPercentage + parent = s.Starts[0].Params.Parent + ) + for _, v := range s.Starts { + // Verify vote params are the same for all submissions + switch { + case v.Params.Type != ticketvote.VoteTypeRunoff: e := fmt.Sprintf("%v vote type invalid: got %v, want %v", v.Params.Token, v.Params.Type, ticketvote.VoteTypeRunoff) return nil, backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } case v.Params.Mask != mask: @@ -1308,7 +1832,7 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep v.Params.Token) return nil, backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } case v.Params.Duration != duration: @@ -1316,7 +1840,7 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep v.Params.Token) return nil, backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } case v.Params.QuorumPercentage != quorum: @@ -1324,7 +1848,7 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep v.Params.Token) return nil, backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } case v.Params.PassPercentage != pass: @@ -1332,7 +1856,7 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep v.Params.Token) return nil, backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } case v.Params.Parent != parent: @@ -1340,7 +1864,7 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep v.Params.Token) return nil, backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } } @@ -1350,11 +1874,22 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep if err != nil { return nil, backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusTokenInvalid), + ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), ErrorContext: v.Params.Token, } } + // Verify parent token + _, err = tokenDecode(v.Params.Parent) + if err != nil { + e := fmt.Sprintf("parent token %v", v.Params.Parent) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), + ErrorContext: e, + } + } + // Verify signature vb, err := json.Marshal(v.Params) if err != nil { @@ -1374,107 +1909,83 @@ func (p *ticketVotePlugin) startRunoff(s ticketvote.Start) (*ticketvote.StartRep } } - // Get vote blockchain data - sr, err := p.startReply(duration) - if err != nil { - return nil, err - } - - // Verify parent exists - parentb, err := tokenDecode(parent) - if err != nil { - return nil, err - } - if !p.backend.VettedExists(parentb) { - e := fmt.Sprintf("parent record not found %v", parent) + // Verify plugin command is being executed on the parent record + if hex.EncodeToString(token) != parent { + e := fmt.Sprintf("runoff vote must be started on parent record %v", + parent) return nil, backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeRunoffVoteParentInvalid), ErrorContext: e, } } - // TODO handle the case where part of the votes are started but - // not all. + // This function is being invoked on the runoff vote parent record. + // Create and save a start runoff record onto the parent record's + // tree. + srr, err := p.startRunoffForParent(treeID, token, s) + if err != nil { + return nil, err + } - // Validate existing record state. The lock for each record must - // be held for the remainder of this function. + // Start the voting period of each runoff vote submissions by + // using the internal plugin command startRunoffSub. for _, v := range s.Starts { - m := p.mutex(v.Params.Token) - m.Lock() - defer m.Unlock() - - token, err := tokenDecode(v.Params.Token) + token, err = tokenDecode(v.Params.Token) if err != nil { return nil, err } - - // Verify record version - r, err := p.backend.GetVetted(token, "") - if err != nil { - return nil, fmt.Errorf("GetVetted: %v", err) - } - version := strconv.FormatUint(uint64(v.Params.Version), 10) - if r.Version != version { - e := fmt.Sprintf("version is not latest %v: got %v, want %v", - v.Params.Token, v.Params.Version, r.Version) - return nil, backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusRecordVersionInvalid), - ErrorContext: e, - } + srs := startRunoffSub{ + ParentTreeID: treeID, + StartDetails: v, } - - // TODO figure this out - var treeID int64 - - // Verify vote has not already been started - svp, err := p.voteDetails(treeID) + b, err := json.Marshal(srs) if err != nil { return nil, err } - if svp != nil { - // Vote has already been started - return nil, backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusVoteStatusInvalid), - ErrorContext: "vote already started", + _, err = p.backend.VettedPluginCmd(token, ticketvote.ID, + cmdStartRunoffSub, string(b)) + if err != nil { + var ue backend.PluginUserError + if errors.As(err, &ue) { + return nil, err } + return nil, fmt.Errorf("VettedPluginCmd %x %v %v %v: %v", + token, ticketvote.ID, cmdStartRunoffSub, b, err) } } - for _, v := range s.Starts { - // Prepare vote details - vd := ticketvote.VoteDetails{ - Params: v.Params, - PublicKey: v.PublicKey, - Signature: v.Signature, - StartBlockHeight: sr.StartBlockHeight, - StartBlockHash: sr.StartBlockHash, - EndBlockHeight: sr.EndBlockHeight, - EligibleTickets: sr.EligibleTickets, - } - - // TODO figure this out - var treeID int64 - - // Save vote details - err = p.voteDetailsSave(treeID, vd) - if err != nil { - return nil, fmt.Errorf("voteDetailsSave: %v", err) - } + return &ticketvote.StartReply{ + StartBlockHeight: srr.StartBlockHeight, + StartBlockHash: srr.StartBlockHash, + EndBlockHeight: srr.EndBlockHeight, + EligibleTickets: srr.EligibleTickets, + }, nil +} + +func (p *ticketVotePlugin) cmdStartRunoffSub(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdStartRunoffSub: %v %x %v", treeID, token, payload) - // Update inventory - p.inventorySetToStarted(vd.Params.Token, ticketvote.VoteTypeRunoff, - vd.EndBlockHeight) + // Decode payload + var srs startRunoffSub + err := json.Unmarshal([]byte(payload), &srs) + if err != nil { + return "", err } - return &ticketvote.StartReply{ - StartBlockHeight: sr.StartBlockHeight, - StartBlockHash: sr.StartBlockHash, - EndBlockHeight: sr.EndBlockHeight, - EligibleTickets: sr.EligibleTickets, - }, nil + // Start voting period on runoff vote submission + err = p.startRunoffForSub(treeID, token, srs) + if err != nil { + return "", err + } + + // Prepare reply + reply, err := json.Marshal(startRunoffSubReply{}) + if err != nil { + return "", err + } + + return string(reply), nil } func (p *ticketVotePlugin) cmdStart(treeID int64, token []byte, payload string) (string, error) { @@ -1491,7 +2002,7 @@ func (p *ticketVotePlugin) cmdStart(treeID int64, token []byte, payload string) if len(s.Starts) == 0 { return "", backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusStartDetailsInvalid), + ErrorCode: int(ticketvote.ErrorCodeStartDetailsMissing), ErrorContext: "no start details found", } } @@ -1503,12 +2014,12 @@ func (p *ticketVotePlugin) cmdStart(treeID int64, token []byte, payload string) var sr *ticketvote.StartReply switch vtype { case ticketvote.VoteTypeStandard: - sr, err = p.startStandard(treeID, s) + sr, err = p.startStandard(treeID, token, s) if err != nil { return "", err } case ticketvote.VoteTypeRunoff: - sr, err = p.startRunoff(s) + sr, err = p.startRunoff(treeID, token, s) if err != nil { return "", err } @@ -1516,7 +2027,7 @@ func (p *ticketVotePlugin) cmdStart(treeID int64, token []byte, payload string) e := fmt.Sprintf("invalid vote type %v", vtype) return "", backend.PluginUserError{ PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorStatusVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } } @@ -1766,7 +2277,7 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str // The record lock must be held for the remainder of the function to // ensure duplicate votes cannot be cast. - m := p.mutex(hex.EncodeToString(token)) + m := p.mutex(token) m.Lock() defer m.Unlock() @@ -1910,39 +2421,22 @@ func (p *ticketVotePlugin) cmdDetails(treeID int64, token []byte, payload string return "", err } - // TODO this needs to be for a single record - - votes := make(map[string]ticketvote.RecordVote, len(d.Tokens)) - for _, v := range d.Tokens { - // Verify token - token, err := tokenDecodeAnyLength(v) - if err != nil { - continue - } - _ = token - - // Get authorize votes - auths, err := p.auths(treeID) - if err != nil { - return "", fmt.Errorf("auths: %v", err) - } - - // Get vote details - vd, err := p.voteDetails(treeID) - if err != nil { - return "", fmt.Errorf("startDetails: %v", err) - } + // Get vote authorizations + auths, err := p.auths(treeID) + if err != nil { + return "", fmt.Errorf("auths: %v", err) + } - // Add record vote - votes[v] = ticketvote.RecordVote{ - Auths: auths, - Vote: vd, - } + // Get vote details + vd, err := p.voteDetails(treeID) + if err != nil { + return "", fmt.Errorf("voteDetails: %v", err) } // Prepare rely dr := ticketvote.DetailsReply{ - Votes: votes, + Auths: auths, + Vote: vd, } reply, err := json.Marshal(dr) if err != nil { @@ -2027,174 +2521,25 @@ func voteIsApproved(vd ticketvote.VoteDetails, results []ticketvote.VoteOptionRe return approved } -// summary returns the vote summary for a record. -func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) (*ticketvote.Summary, error) { - // Check if the summary has been cached - s, err := p.cachedSummary(hex.EncodeToString(token)) - switch { - case errors.Is(err, errSummaryNotFound): - // Cached summary not found. Continue. - case err != nil: - // Some other error - return nil, fmt.Errorf("cachedSummary: %v", err) - default: - // Caches summary was found. Return it. - return s, nil - } - - // Summary has not been cached. Get it manually. +func (p *ticketVotePlugin) cmdSummary(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdSummaries: %v %x %v", treeID, token, payload) - // Assume vote is unauthorized. Only update the status when the - // appropriate record has been found that proves otherwise. - status := ticketvote.VoteStatusUnauthorized + // Get best block. This cmd does not write any data so we do not + // have to use the safe best block. + bb, err := p.bestBlockUnsafe() + if err != nil { + return "", fmt.Errorf("bestBlockUnsafe: %v", err) + } - // Check if the vote has been authorized - auths, err := p.auths(treeID) + // Get summary + s, err := p.summary(treeID, token, bb) if err != nil { - return nil, fmt.Errorf("auths: %v", err) - } - if len(auths) > 0 { - lastAuth := auths[len(auths)-1] - switch ticketvote.AuthActionT(lastAuth.Action) { - case ticketvote.AuthActionAuthorize: - // Vote has been authorized; continue - status = ticketvote.VoteStatusAuthorized - case ticketvote.AuthActionRevoke: - // Vote authorization has been revoked. Its not possible for - // the vote to have been started. We can stop looking. - return &ticketvote.Summary{ - Status: status, - Results: []ticketvote.VoteOptionResult{}, - }, nil - } - } - - // Check if the vote has been started - vd, err := p.voteDetails(treeID) - if err != nil { - return nil, fmt.Errorf("startDetails: %v", err) - } - if vd == nil { - // Vote has not been started yet - return &ticketvote.Summary{ - Status: status, - Results: []ticketvote.VoteOptionResult{}, - }, nil - } - - // Vote has been started. Check if it is still in progress or has - // already ended. - if bestBlock < vd.EndBlockHeight { - status = ticketvote.VoteStatusStarted - } else { - status = ticketvote.VoteStatusFinished - } - - // Pull the cast votes from the cache and tally the results - votes := p.cachedVotes(token) - tally := make(map[string]int, len(vd.Params.Options)) - for _, voteBit := range votes { - tally[voteBit]++ - } - results := make([]ticketvote.VoteOptionResult, 0, len(vd.Params.Options)) - for _, v := range vd.Params.Options { - bit := strconv.FormatUint(v.Bit, 16) - results = append(results, ticketvote.VoteOptionResult{ - ID: v.ID, - Description: v.Description, - VoteBit: v.Bit, - Votes: uint64(tally[bit]), - }) - } - - // Prepare summary - summary := ticketvote.Summary{ - Type: vd.Params.Type, - Status: status, - Duration: vd.Params.Duration, - StartBlockHeight: vd.StartBlockHeight, - StartBlockHash: vd.StartBlockHash, - EndBlockHeight: vd.EndBlockHeight, - EligibleTickets: uint32(len(vd.EligibleTickets)), - QuorumPercentage: vd.Params.QuorumPercentage, - PassPercentage: vd.Params.PassPercentage, - Results: results, - } - - // If the vote has not finished yet then we are done for now. - if status == ticketvote.VoteStatusStarted { - return &summary, nil - } - - // The vote has finished. We can calculate if the vote was approved - // for certain vote types and cache the results. - switch vd.Params.Type { - case ticketvote.VoteTypeStandard, ticketvote.VoteTypeRunoff: - // These vote types are strictly approve/reject votes so we can - // calculate the vote approval. Continue. - default: - // Nothing else to do for all other vote types - return &summary, nil - } - - // Calculate vote approval - approved := voteIsApproved(*vd, results) - - // If this is a standard vote then we can take the results as is. A - // runoff vote requires that we pull all other runoff vote - // submissions to determine if the vote actually passed. - // TODO - summary.Approved = approved - - // Cache the summary - err = p.cachedSummarySave(vd.Params.Token, summary) - if err != nil { - return nil, fmt.Errorf("cachedSummarySave %v: %v %v", - vd.Params.Token, err, summary) - } - - // Remove record from the votes cache now that a summary has been - // saved for it. - p.cachedVotesDel(vd.Params.Token) - - return &summary, nil -} - -func (p *ticketVotePlugin) cmdSummaries(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdSummaries: %v %x %v", treeID, token, payload) - - // Decode payload - var s ticketvote.Summaries - err := json.Unmarshal([]byte(payload), &s) - if err != nil { - return "", err - } - - // Get best block. This cmd does not write any data so we do not - // have to use the safe best block. - bb, err := p.bestBlockUnsafe() - if err != nil { - return "", fmt.Errorf("bestBlockUnsafe: %v", err) - } - - // TODO this route can only be for one summary - // Get summaries - summaries := make(map[string]ticketvote.Summary, len(s.Tokens)) - for _, v := range s.Tokens { - token, err := tokenDecodeAnyLength(v) - if err != nil { - return "", err - } - s, err := p.summary(treeID, token, bb) - if err != nil { - return "", fmt.Errorf("summary %v: %v", v, err) - } - summaries[v] = *s + return "", fmt.Errorf("summary: %v", err) } // Prepare reply - sr := ticketvote.SummariesReply{ - Summaries: summaries, + sr := ticketvote.SummaryReply{ + Summary: *s, BestBlock: bb, } reply, err := json.Marshal(sr) @@ -2349,6 +2694,260 @@ func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload str return string(reply), nil } +// decodeVoteMetadata decodes and returns the VoteMetadata from the +// provided backend files. If a VoteMetadata is not found, nil is returned. +func decodeVoteMetadata(files []backend.File) (*ticketvote.VoteMetadata, error) { + var voteMD *ticketvote.VoteMetadata + for _, v := range files { + if v.Name == ticketvote.FileNameVoteMetadata { + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return nil, err + } + var pm ticketvote.VoteMetadata + err = json.Unmarshal(b, &pm) + if err != nil { + return nil, err + } + voteMD = &pm + break + } + } + return voteMD, nil +} + +func (p *ticketVotePlugin) linkByVerify(linkBy int64) error { + if linkBy == 0 { + // LinkBy as not been set + return nil + } + min := time.Now().Unix() + p.linkByPeriodMin + max := time.Now().Unix() + p.linkByPeriodMax + switch { + case linkBy < min: + e := fmt.Sprintf("linkby %v is less than min required of %v", + linkBy, min) + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeLinkByInvalid), + ErrorContext: e, + } + case linkBy > max: + e := fmt.Sprintf("linkby %v is more than max allowed of %v", + linkBy, max) + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeLinkByInvalid), + ErrorContext: e, + } + } + return nil +} + +func (p *ticketVotePlugin) linkToVerify(linkTo string) error { + // LinkTo must be a public record that is the parent of a runoff + // vote, i.e. has the VoteMetadata.LinkBy field set. + token, err := tokenDecode(linkTo) + if err != nil { + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), + ErrorContext: "invalid hex", + } + } + r, err := p.backend.GetVetted(token, "") + if err != nil { + if errors.Is(err, backend.ErrRecordNotFound) { + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), + ErrorContext: "record not found", + } + } + return err + } + if r.RecordMetadata.Status != backend.MDStatusCensored { + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), + ErrorContext: "record is censored", + } + } + parentVM, err := decodeVoteMetadata(r.Files) + if err != nil { + return err + } + if parentVM == nil || parentVM.LinkBy == 0 { + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), + ErrorContext: "record not a runoff vote parent", + } + } + if time.Now().Unix() > parentVM.LinkBy { + // Linkby deadline has expired. New links are not allowed. + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), + ErrorContext: "parent record linkby deadline has expired", + } + } + return nil +} + +func (p *ticketVotePlugin) voteMetadataVerify(vm ticketvote.VoteMetadata) error { + switch { + case vm.LinkBy == 0 && vm.LinkTo == "": + // Vote metadata is empty + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeVoteMetadataInvalid), + ErrorContext: "md is empty", + } + + case vm.LinkBy != 0 && vm.LinkTo != "": + // LinkBy and LinkTo cannot both be set + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeVoteMetadataInvalid), + ErrorContext: "cannot set both linkby and linkto", + } + + case vm.LinkBy != 0: + err := p.linkByVerify(vm.LinkBy) + if err != nil { + return err + } + + case vm.LinkTo != "": + err := p.linkToVerify(vm.LinkTo) + if err != nil { + return err + } + } + + return nil +} + +// TODO this save validation needs to be done in the edit record hook too +func (p *ticketVotePlugin) hookNewRecordPre(payload string) error { + var nr plugins.HookNewRecordPre + err := json.Unmarshal([]byte(payload), &nr) + if err != nil { + return err + } + + // Verify the vote metadata if the record contains one + vm, err := decodeVoteMetadata(nr.Files) + if err != nil { + return err + } + if vm != nil { + err = p.voteMetadataVerify(*vm) + if err != nil { + return err + } + } + + return nil +} + +func (p *ticketVotePlugin) hookEditRecordPre(payload string) error { + var er plugins.HookEditRecord + err := json.Unmarshal([]byte(payload), &er) + if err != nil { + return err + } + + // The LinkTo field is not allowed to change once the record has + // become public. If this is a vetted record, verify that any + // previously set LinkTo has not changed. + if er.State == plugins.RecordStateVetted { + var ( + oldLinkTo string + newLinkTo string + ) + vm, err := decodeVoteMetadata(er.Current.Files) + if err != nil { + return err + } + if vm != nil { + oldLinkTo = vm.LinkTo + } + vm, err = decodeVoteMetadata(er.FilesAdd) + if err != nil { + return err + } + if vm != nil { + newLinkTo = vm.LinkTo + } + if newLinkTo != oldLinkTo { + e := fmt.Sprintf("linkto cannot change on vetted record: "+ + "got '%v', want '%v'", newLinkTo, oldLinkTo) + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), + ErrorContext: e, + } + } + } + + // Verify LinkBy if one was included. The VoteMetadata is optional + // so the record may not contain one. + vm, err := decodeVoteMetadata(er.FilesAdd) + if err != nil { + return err + } + if vm != nil { + err = p.linkByVerify(vm.LinkBy) + if err != nil { + return err + } + } + + return nil +} + +func (p *ticketVotePlugin) hookSetRecordStatusPost(payload string) error { + var srs plugins.HookSetRecordStatus + err := json.Unmarshal([]byte(payload), &srs) + if err != nil { + return err + } + + // Check if the LinkTo has been set + vm, err := decodeVoteMetadata(srs.Current.Files) + if err != nil { + return err + } + if vm != nil && vm.LinkTo != "" { + // LinkTo has been set. Check if the status change requires the + // linked from list of the linked record to be updated. + var ( + parentToken = vm.LinkTo + childToken = srs.RecordMetadata.Token + ) + switch srs.RecordMetadata.Status { + case backend.MDStatusVetted: + // Record has been made public. Add child token to parent's + // linked from list. + err := p.linkedFromAdd(parentToken, childToken) + if err != nil { + return fmt.Errorf("linkedFromAdd: %v", err) + } + case backend.MDStatusCensored: + // Record has been censored. Delete child token from parent's + // linked from list. + err := p.linkedFromDel(parentToken, childToken) + if err != nil { + return fmt.Errorf("linkedFromDel: %v", err) + } + } + } + + return nil +} + // Setup performs any plugin setup work that needs to be done. // // This function satisfies the plugins.Client interface. @@ -2356,12 +2955,8 @@ func (p *ticketVotePlugin) Setup() error { log.Tracef("Setup") // Verify plugin dependencies - plugins, err := p.backend.GetPlugins() - if err != nil { - return fmt.Errorf("Plugins: %v", err) - } var dcrdataFound bool - for _, v := range plugins { + for _, v := range p.backend.GetVettedPlugins() { if v.ID == dcrdata.ID { dcrdataFound = true } @@ -2371,7 +2966,7 @@ func (p *ticketVotePlugin) Setup() error { } // Build inventory cache - log.Infof("ticketvote: building inventory cache") + log.Infof("Building inventory cache") ibs, err := p.backend.InventoryByStatus() if err != nil { @@ -2395,12 +2990,7 @@ func (p *ticketVotePlugin) Setup() error { if err != nil { return err } - // TODO this needs to use the summary plugin command - var treeID int64 - s, err := p.summary(treeID, token, bestBlock) - if err != nil { - return fmt.Errorf("summary %v: %v", v, err) - } + s, err := p.summaryByToken(token) switch s.Status { case ticketvote.VoteStatusUnauthorized: unauthorized = append(unauthorized, v) @@ -2427,20 +3017,25 @@ func (p *ticketVotePlugin) Setup() error { p.Unlock() // Build votes cache - log.Infof("ticketvote: building votes cache") + log.Infof("Building votes cache") for k := range started { token, err := tokenDecode(k) if err != nil { return err } - // TODO this needs to use the results plugin command - var treeID int64 - votes, err := p.castVotes(treeID) + reply, err := p.backend.VettedPluginCmd(token, ticketvote.ID, + ticketvote.CmdResults, "") if err != nil { - return fmt.Errorf("castVotes %v: %v", token, err) + return fmt.Errorf("VettedPluginCmd %x %v %v: %v", + token, ticketvote.ID, ticketvote.CmdResults, err) + } + var rr ticketvote.ResultsReply + err = json.Unmarshal([]byte(reply), &rr) + if err != nil { + return err } - for _, v := range votes { + for _, v := range rr.Votes { p.cachedVotesSet(v.Token, v.Ticket, v.VoteBit) } } @@ -2465,12 +3060,16 @@ func (p *ticketVotePlugin) Cmd(treeID int64, token []byte, cmd, payload string) return p.cmdDetails(treeID, token, payload) case ticketvote.CmdResults: return p.cmdResults(treeID, token, payload) - case ticketvote.CmdSummaries: - return p.cmdSummaries(treeID, token, payload) + case ticketvote.CmdSummary: + return p.cmdSummary(treeID, token, payload) case ticketvote.CmdInventory: return p.cmdInventory() case ticketvote.CmdTimestamps: return p.cmdTimestamps(treeID, token, payload) + + // Internal plugin commands + case cmdStartRunoffSub: + return p.cmdStartRunoffSub(treeID, token, payload) } return "", backend.ErrPluginCmdInvalid @@ -2482,6 +3081,15 @@ func (p *ticketVotePlugin) Cmd(treeID int64, token []byte, cmd, payload string) func (p *ticketVotePlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { log.Tracef("Hook: %v %x %v", treeID, token, plugins.Hooks[h]) + switch h { + case plugins.HookTypeNewRecordPre: + return p.hookNewRecordPre(payload) + case plugins.HookTypeEditRecordPre: + return p.hookEditRecordPre(payload) + case plugins.HookTypeSetRecordStatusPost: + return p.hookSetRecordStatusPost(payload) + } + return nil } @@ -2494,6 +3102,36 @@ func (p *ticketVotePlugin) Fsck() error { return nil } +/* +// linkByPeriodMin returns the minimum amount of time, in seconds, that the +// LinkBy period must be set to. This is determined by adding 1 week onto the +// minimum voting period so that RFP proposal submissions have at least one +// week to be submitted after the proposal vote ends. +func (p *politeiawww) linkByPeriodMin() int64 { + var ( + submissionPeriod int64 = 604800 // One week in seconds + blockTime int64 // In seconds + ) + switch { + case p.cfg.TestNet: + blockTime = int64(testNet3Params.TargetTimePerBlock.Seconds()) + case p.cfg.SimNet: + blockTime = int64(simNetParams.TargetTimePerBlock.Seconds()) + default: + blockTime = int64(mainNetParams.TargetTimePerBlock.Seconds()) + } + return (int64(p.cfg.VoteDurationMin) * blockTime) + submissionPeriod +} + +// linkByPeriodMax returns the maximum amount of time, in seconds, that the +// LinkBy period can be set to. 3 months is currently hard coded with no real +// reason for deciding on 3 months besides that it sounds like a sufficient +// amount of time. This can be changed if there is a valid reason to. +func (p *politeiawww) linkByPeriodMax() int64 { + return 7776000 // 3 months in seconds +} +*/ + func newTicketVotePlugin(backend backend.Backend, tlog tlogclient.Client, settings []backend.PluginSetting, dataDir string, id *identity.FullIdentity, activeNetParams *chaincfg.Params) (*ticketVotePlugin, error) { // Plugin settings var ( diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index dbccd9633..36a23ffc6 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -19,6 +19,7 @@ const ( // TODO get rid of CmdProposals CmdProposals = "proposals" // Get plugin data for proposals + // TODO MDStream IDs need to be plugin specific // Metadata stream IDs MDStreamIDGeneralMetadata = 1 MDStreamIDStatusChanges = 2 @@ -50,7 +51,6 @@ const ( ErrorStatusStartDetailsInvalid ErrorStatusStartDetailsMissing ErrorStatusVoteParentInvalid - ErrorStatusLinkByNotExpired ) var ( @@ -86,21 +86,11 @@ var ( // object is saved to politeiad as a file, not as a metadata stream, since it // needs to be included in the merkle root that politeiad signs. type ProposalMetadata struct { - // Name is the name of the proposal. Name string `json:"name"` - - // LinkTo specifies a public proposal token to link this proposal - // to. Ex, an RFP sumbssion must link to the RFP proposal. - LinkTo string `json:"linkto,omitempty"` - - // LinkBy is a UNIX timestamp that serves as a deadline for other - // proposals to link to this proposal. Ex, an RFP submission cannot - // link to an RFP proposal once the RFP's LinkBy deadline is past. - LinkBy int64 `json:"linkby,omitempty"` } // GeneralMetadata contains general metadata about a politeiad record. It is -// saved to politeiad as a metadata stream. +// generated by the server and saved to politeiad as a metadata stream. // // Signature is the client signature of the record merkle root. The merkle root // is the ordered merkle root of all politeiad Files. diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index eb13ec967..ceb10a7ef 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -24,7 +24,7 @@ const ( CmdCastBallot = "castballot" // Cast a ballot of votes CmdDetails = "details" // Get vote details CmdResults = "results" // Get vote results - CmdSummaries = "summaries" // Get vote summaries + CmdSummary = "summary" // Get vote summary CmdInventory = "inventory" // Get inventory by vote status CmdTimestamps = "timestamps" // Get vote data timestamps @@ -42,23 +42,56 @@ const ( PolicyVotesPageSize = 20 ) -// ErrorStatusT represents and error that is caused by the user. -type ErrorStatusT int +// ErrorCodeT represents and error that is caused by the user. +type ErrorCodeT int const ( - ErrorStatusInvalid ErrorStatusT = 0 - ErrorStatusTokenInvalid ErrorStatusT = 1 - ErrorStatusPublicKeyInvalid ErrorStatusT = 2 - ErrorStatusSignatureInvalid ErrorStatusT = 3 - ErrorStatusRecordVersionInvalid ErrorStatusT = 4 - ErrorStatusRecordStatusInvalid ErrorStatusT = 5 - ErrorStatusAuthorizationInvalid ErrorStatusT = 6 - ErrorStatusStartDetailsInvalid ErrorStatusT = 7 - ErrorStatusVoteParamsInvalid ErrorStatusT = 8 - ErrorStatusVoteStatusInvalid ErrorStatusT = 9 - ErrorStatusPageSizeExceeded ErrorStatusT = 10 + // TODO number these + ErrorCodeInvalid ErrorCodeT = 0 + ErrorCodeTokenInvalid ErrorCodeT = iota + ErrorCodePublicKeyInvalid + ErrorCodeSignatureInvalid + ErrorCodeRecordVersionInvalid + ErrorCodeRecordStatusInvalid + ErrorCodeAuthorizationInvalid + ErrorCodeStartDetailsMissing + ErrorCodeStartDetailsInvalid + ErrorCodeVoteParamsInvalid + ErrorCodeVoteStatusInvalid + ErrorCodePageSizeExceeded + ErrorCodeVoteMetadataInvalid + ErrorCodeLinkByInvalid + ErrorCodeLinkToInvalid + + ErrorCodeRunoffVoteParentInvalid + ErrorCodeLinkByNotExpired ) +const ( + // FileNameVoteMetadata is the filename of the VoteMetadata file + // that is saved to politeiad. VoteMetadata is saved to politeiad + // as a file, not as a metadata stream, since it contains user + // provided metadata and needs to be included in the merkle root + // that politeiad signs. + FileNameVoteMetadata = "votemetadata.json" +) + +// VoteMetadata is metadata that is specified by the user and attached to +// a record on submission. This metadata is required for certain types of +// votes. +type VoteMetadata struct { + // LinkBy is set when the user intends for the record to be the + // parent record in a runoff vote. It is a UNIX timestamp that + // serves as the deadline for other records to declare their intent + // to participate in the runoff vote. + LinkBy int64 `json:"linkby,omitempty"` + + // LinkTo is the censorship token of a runoff vote parent record. + // It is set when a record is being submitted as a vote options in + // the runoff vote. + LinkTo string `json:"linkto,omitempty"` +} + // AuthDetails is the structure that is saved to disk when a vote is authorized // or a previous authorization is revoked. It contains all the fields from a // Authorize and a AuthorizeReply. @@ -219,9 +252,6 @@ type StartDetails struct { } // Start starts a ticket vote. -// -// Signature is the signature of a SHA256 digest of the JSON encoded VoteParams -// structure. type Start struct { Starts []StartDetails `json:"starts"` } @@ -328,41 +358,23 @@ type CastBallotReply struct { Receipts []CastVoteReply `json:"receipts"` } -// Details returns the vote details for each of the provided record tokens. -type Details struct { - Tokens []string `json:"tokens"` -} +// Details returns the vote details for a record. +type Details struct{} -// RecordVote contains all vote authorizations and the vote details for a -// record. The VoteDetails will be nil if the vote has been started. -type RecordVote struct { +// DetailsReply is the reply to the Details command. +type DetailsReply struct { Auths []AuthDetails `json:"auths"` Vote *VoteDetails `json:"vote"` } -// DetailsReply is the reply to the Details command. The returned map will not -// contain an entry for any tokens that did not correspond to an actual record. -// It is the callers responsibility to ensure that a entry is returned for all -// of the provided tokens. -type DetailsReply struct { - Votes map[string]RecordVote `json:"votes"` -} - // Results requests the results of a vote. -type Results struct { - Token string `json:"token"` -} +type Results struct{} // ResultsReply is the rely to the Results command. type ResultsReply struct { Votes []CastVoteDetails `json:"votes"` } -// Summaries requests the vote summaries for the provided record tokens. -type Summaries struct { - Tokens []string `json:"tokens"` -} - // VoteStatusT represents the status of a ticket vote. type VoteStatusT int @@ -405,8 +417,8 @@ type VoteOptionResult struct { Votes uint64 `json:"votes"` // Votes cast for this option } -// Summary summarizes the vote params and results for a ticket vote. -type Summary struct { +// VoteSummary contains a summary of a record vote. +type VoteSummary struct { Type VoteT `json:"type"` Status VoteStatusT `json:"status"` Duration uint32 `json:"duration"` @@ -425,17 +437,15 @@ type Summary struct { Approved bool `json:"approved,omitempty"` } -// SummariesReply is the reply to the Summaries command. -type SummariesReply struct { - // Summaries contains a vote summary for each of the provided - // tokens. The map will not contain an entry for any tokens that - // did not correspond to an actual record. It is the callers - // responsibility to ensure that a summary is returned for all of - // the provided tokens. - Summaries map[string]Summary `json:"summaries"` // [token]Summary +// Summary requests the vote summaries for a record. +type Summary struct{} - // BestBlock is the best block value that was used to prepare the - // summaries. +// SummaryReply is the reply to the Summary command. +type SummaryReply struct { + Summary VoteSummary `json:"summary"` + + // BestBlock is the best block value that was used to + // prepare this summary. BestBlock uint32 `json:"bestblock"` } diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index e22aaf7d4..cb06f9032 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -417,27 +417,3 @@ func (p *politeiawww) processTokenInventory(ctx context.Context, isAdmin bool) ( Abandoned: archived, }, nil } - -/* -func (p *politeiawww) linkByValidate(linkBy int64) error { - min := time.Now().Unix() + p.linkByPeriodMin() - max := time.Now().Unix() + p.linkByPeriodMax() - switch { - case linkBy < min: - e := fmt.Sprintf("linkby %v is less than min required of %v", - linkBy, min) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkBy, - ErrorContext: []string{e}, - } - case linkBy > max: - e := fmt.Sprintf("linkby %v is more than max allowed of %v", - linkBy, max) - return www.UserError{ - ErrorCode: www.ErrorStatusInvalidLinkBy, - ErrorContext: []string{e}, - } - } - return nil -} -*/ From 6a7c9b0402241381467f30a5befb504257cc3adf Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 18 Jan 2021 09:42:00 -0600 Subject: [PATCH 232/449] All plugins working with new design. --- politeiad/backend/tlogbe/plugins/pi/pi.go | 160 +- .../backend/tlogbe/plugins/ticketvote/cmds.go | 2496 +++++++++++++- .../tlogbe/plugins/ticketvote/hooks.go | 270 ++ .../tlogbe/plugins/ticketvote/inventory.go | 8 +- .../tlogbe/plugins/ticketvote/summary.go | 91 + .../tlogbe/plugins/ticketvote/ticketvote.go | 2874 +---------------- .../tlogbe/plugins/ticketvote/votes.go | 50 + politeiad/backend/tlogbe/tlogbe.go | 12 +- politeiad/plugins/pi/pi.go | 140 +- politeiad/plugins/ticketvote/ticketvote.go | 4 + 10 files changed, 3067 insertions(+), 3038 deletions(-) create mode 100644 politeiad/backend/tlogbe/plugins/ticketvote/hooks.go create mode 100644 politeiad/backend/tlogbe/plugins/ticketvote/summary.go create mode 100644 politeiad/backend/tlogbe/plugins/ticketvote/votes.go diff --git a/politeiad/backend/tlogbe/plugins/pi/pi.go b/politeiad/backend/tlogbe/plugins/pi/pi.go index 1ab54c32f..de4fa8429 100644 --- a/politeiad/backend/tlogbe/plugins/pi/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi/pi.go @@ -5,7 +5,7 @@ package pi import ( - "encoding/hex" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -42,17 +42,33 @@ type piPlugin struct { dataDir string } -// isRFP returns whether the provided proposal metadata belongs to an RFP -// proposal. -func isRFP(pm pi.ProposalMetadata) bool { - return pm.LinkBy != 0 -} - // tokenDecode decodes a token string. func tokenDecode(token string) ([]byte, error) { return util.TokenDecode(util.TokenTypeTlog, token) } +// decodeProposalMetadata decodes and returns the ProposalMetadata from the +// provided backend files. If a ProposalMetadata is not found, nil is returned. +func decodeProposalMetadata(files []backend.File) (*pi.ProposalMetadata, error) { + var propMD *pi.ProposalMetadata + for _, v := range files { + if v.Name == pi.FileNameProposalMetadata { + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return nil, err + } + var m pi.ProposalMetadata + err = json.Unmarshal(b, &m) + if err != nil { + return nil, err + } + propMD = &m + break + } + } + return propMD, nil +} + // decodeGeneralMetadata decodes and returns the GeneralMetadata from the // provided backend metadata streams. If a GeneralMetadata is not found, nil is // returned. @@ -123,7 +139,7 @@ func (p *piPlugin) cmdProposals(payload string) (string, error) { default: return "", backend.PluginUserError{ PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropStateInvalid), + ErrorCode: int(pi.ErrorCodePropStateInvalid), } } @@ -209,29 +225,18 @@ func (p *piPlugin) cmdProposals(payload string) (string, error) { return "", nil } -func (p *piPlugin) voteSummary(token []byte) (*ticketvote.Summary, error) { - t := hex.EncodeToString(token) - s := ticketvote.Summaries{ - Tokens: []string{t}, - } - b, err := ticketvote.EncodeSummaries(s) - if err != nil { - return nil, err - } - r, err := p.backend.Plugin(ticketvote.ID, - ticketvote.CmdSummaries, "", string(b)) +func (p *piPlugin) voteSummary(token []byte) (*ticketvote.VoteSummary, error) { + reply, err := p.backend.VettedPluginCmd(token, + ticketvote.ID, ticketvote.CmdSummary, "") if err != nil { return nil, err } - sr, err := ticketvote.DecodeSummariesReply([]byte(r)) + var sr ticketvote.SummaryReply + err = json.Unmarshal([]byte(reply), &sr) if err != nil { return nil, err } - summary, ok := sr.Summaries[t] - if !ok { - return nil, fmt.Errorf("proposal not found %v", token) - } - return &summary, nil + return &sr.Summary, nil } func (p *piPlugin) cmdProposalInv(payload string) (string, error) { @@ -333,49 +338,37 @@ func (p *piPlugin) cmdProposalInv(payload string) (string, error) { return string(reply), nil } -func (p *piPlugin) cmdVoteInventory(payload string) (string, error) { - // Payload is empty. Nothing to decode. - +func (p *piPlugin) cmdVoteInventory() (string, error) { // Get ticketvote inventory - r, err := p.backend.Plugin(ticketvote.ID, ticketvote.CmdInventory, "", "") - if err != nil { - return "", fmt.Errorf("ticketvote inventory: %v", err) - } - ir, err := ticketvote.DecodeInventoryReply([]byte(r)) + r, err := p.backend.VettedPluginCmd([]byte{}, + ticketvote.ID, ticketvote.CmdInventory, "") if err != nil { - return "", err - } - - // Get vote summaries for all finished proposal votes - s := ticketvote.Summaries{ - Tokens: ir.Finished, + return "", fmt.Errorf("VettedPluginCmd %v %v: %v", + ticketvote.ID, ticketvote.CmdInventory, err) } - b, err := ticketvote.EncodeSummaries(s) + var ir ticketvote.InventoryReply + err = json.Unmarshal([]byte(r), &ir) if err != nil { return "", err } - r, err = p.backend.Plugin(ticketvote.ID, ticketvote.CmdSummaries, - "", string(b)) - if err != nil { - return "", fmt.Errorf("ticketvote summaries: %v", err) - } - sr, err := ticketvote.DecodeSummariesReply([]byte(r)) - if err != nil { - return "", err - } - if len(sr.Summaries) != len(ir.Finished) { - return "", fmt.Errorf("unexpected number of summaries: got %v, want %v", - len(sr.Summaries), len(ir.Finished)) - } - // Categorize votes - approved := make([]string, 0, len(sr.Summaries)) - rejected := make([]string, 0, len(sr.Summaries)) - for token, v := range sr.Summaries { - if v.Approved { - approved = append(approved, token) + // Get vote summaries for all finished proposal votes and + // categorize by approved/rejected. + approved := make([]string, 0, len(ir.Finished)) + rejected := make([]string, 0, len(ir.Finished)) + for _, v := range ir.Finished { + t, err := tokenDecode(v) + if err != nil { + return "", err + } + vs, err := p.voteSummary(t) + if err != nil { + return "", err + } + if vs.Approved { + approved = append(approved, v) } else { - rejected = append(rejected, token) + rejected = append(rejected, v) } } @@ -404,7 +397,7 @@ func (p *piPlugin) hookCommentNew(treeID int64, token []byte, payload string) er } // Verify vote status - vs, err := p.voteSummary(treeID, token) + vs, err := p.voteSummary(token) if err != nil { return fmt.Errorf("voteSummary: %v", err) } @@ -413,9 +406,9 @@ func (p *piPlugin) hookCommentNew(treeID int64, token []byte, payload string) er ticketvote.VoteStatusStarted: // Comments are allowed on these vote statuses; continue default: - return "", backend.PluginUserError{ + return backend.PluginUserError{ PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusVoteStatusInvalid), + ErrorCode: int(pi.ErrorCodeVoteStatusInvalid), ErrorContext: "vote has ended; proposal is locked", } } @@ -442,7 +435,7 @@ func (p *piPlugin) commentDel(payload string) error { default: return "", backend.PluginUserError{ PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropStateInvalid), + ErrorCode: int(pi.ErrorCodePropStateInvalid), } } @@ -451,7 +444,7 @@ func (p *piPlugin) commentDel(payload string) error { if err != nil { return "", backend.PluginUserError{ PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropTokenInvalid), + ErrorCode: int(pi.ErrorCodePropTokenInvalid), } } @@ -469,7 +462,7 @@ func (p *piPlugin) commentDel(payload string) error { if !exists { return "", backend.PluginUserError{ PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropNotFound), + ErrorCode: int(pi.ErrorCodePropNotFound), } } @@ -486,7 +479,7 @@ func (p *piPlugin) commentDel(payload string) error { default: return "", backend.PluginUserError{ PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusVoteStatusInvalid), + ErrorCode: int(pi.ErrorCodeVoteStatusInvalid), ErrorContext: []string{"vote has ended; proposal is locked"}, } } @@ -515,7 +508,7 @@ func (p *piPlugin) commentVote(payload string) error { if err != nil { return "", backend.PluginUserError{ PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropTokenInvalid), + ErrorCode: int(pi.ErrorCodePropTokenInvalid), } } @@ -534,7 +527,7 @@ func (p *piPlugin) commentVote(payload string) error { if errors.Is(err, backend.ErrRecordNotFound) { return "", backend.PluginUserError{ PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropNotFound), + ErrorCode: int(pi.ErrorCodePropNotFound), } } return "", fmt.Errorf("get record: %v", err) @@ -548,7 +541,7 @@ func (p *piPlugin) commentVote(payload string) error { default: return "", backend.PluginUserError{ PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropStatusInvalid), + ErrorCode: int(pi.ErrorCodePropStatusInvalid), ErrorContext: []string{"proposal is not public"}, } } @@ -565,7 +558,7 @@ func (p *piPlugin) commentVote(payload string) error { default: return "", backend.PluginUserError{ PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusVoteStatusInvalid), + ErrorCode: int(pi.ErrorCodeVoteStatusInvalid), ErrorContext: []string{"vote has ended; proposal is locked"}, } } @@ -580,17 +573,10 @@ func (p *piPlugin) ticketVoteStart(payload string) error { // TODO If runoff vote, verify that parent record has passed a // vote itself. This functionality is specific to pi. - // Decode payload - s, err := ticketvote.DecodeStart([]byte(payload)) - if err != nil { - return "", err - } - _ = s - return nil } -func (p *piPlugin) hookPluginPre(payload string) error { +func (p *piPlugin) hookPluginPre(treeID int64, token []byte, payload string) error { // Decode payload var hpp plugins.HookPluginPre err := json.Unmarshal([]byte(payload), &hpp) @@ -603,7 +589,7 @@ func (p *piPlugin) hookPluginPre(payload string) error { case comments.ID: switch hpp.Cmd { case comments.CmdNew: - return p.hookCommentNew(hpp.Payload) + return p.hookCommentNew(treeID, token, hpp.Payload) // case comments.CmdDel: // return p.commentDel(hpp.Payload) // case comments.CmdVote: @@ -641,7 +627,7 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { } func (p *piPlugin) hookNewRecordPost(payload string) error { - var nr plugins.HookNewRecord + var nr plugins.HookNewRecordPost err := json.Unmarshal([]byte(payload), &nr) if err != nil { return err @@ -705,7 +691,7 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { ticketvote.VoteStatuses[ticketvote.VoteStatusUnauthorized]) return backend.PluginUserError{ PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusVoteStatusInvalid), + ErrorCode: int(pi.ErrorCodeVoteStatusInvalid), ErrorContext: e, } } @@ -760,7 +746,7 @@ func (p *piPlugin) hookSetRecordStatusPost(payload string) error { sc.Version, srs.Current.Version) return backend.PluginUserError{ PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropVersionInvalid), + ErrorCode: int(pi.ErrorCodePropVersionInvalid), ErrorContext: e, } } @@ -779,7 +765,7 @@ func (p *piPlugin) hookSetRecordStatusPost(payload string) error { from, sc.Status) return backend.PluginUserError{ PluginID: pi.ID, - ErrorCode: int(pi.ErrorStatusPropStatusChangeInvalid), + ErrorCode: int(pi.ErrorCodePropStatusChangeInvalid), ErrorContext: e, } } @@ -805,12 +791,10 @@ func (p *piPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, log.Tracef("Cmd: %v %x %v %v", treeID, token, cmd, payload) switch cmd { - case pi.CmdProposals: - return p.cmdProposals(payload) case pi.CmdProposalInv: return p.cmdProposalInv(payload) case pi.CmdVoteInventory: - return p.cmdVoteInventory(payload) + return p.cmdVoteInventory() } return "", backend.ErrPluginCmdInvalid @@ -832,7 +816,7 @@ func (p *piPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload str case plugins.HookTypeSetRecordStatusPost: return p.hookSetRecordStatusPost(payload) case plugins.HookTypePluginPre: - return p.hookPluginPre(payload) + return p.hookPluginPre(treeID, token, payload) } return nil diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go index f959be277..b9dfff9ac 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go @@ -4,13 +4,1072 @@ package ticketvote -import "github.com/decred/politeia/politeiad/plugins/ticketvote" +import ( + "bytes" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/decred/dcrd/chaincfg/chainhash" + "github.com/decred/dcrd/chaincfg/v3" + "github.com/decred/dcrd/dcrec/secp256k1/v3/ecdsa" + "github.com/decred/dcrd/dcrutil/v3" + "github.com/decred/dcrd/wire" + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/store" + "github.com/decred/politeia/politeiad/plugins/dcrdata" + "github.com/decred/politeia/politeiad/plugins/ticketvote" + "github.com/decred/politeia/util" +) const ( + // Blob entry data descriptors + dataDescriptorAuthDetails = "authdetails_v1" + dataDescriptorVoteDetails = "votedetails_v1" + dataDescriptorCastVoteDetails = "castvotedetails_v1" + dataDescriptorStartRunoff = "startrunoff_v1" + + // Data types + dataTypeAuthDetails = "authdetails" + dataTypeVoteDetails = "votedetails" + dataTypeCastVoteDetails = "castvotedetails" + dataTypeStartRunoff = "startrunoff" + // Internal plugin commands cmdStartRunoffSub = "startrunoffsub" ) +// tokenDecode decodes a record token and only accepts full length tokens. +func tokenDecode(token string) ([]byte, error) { + return util.TokenDecode(util.TokenTypeTlog, token) +} + +// tokenDecode decodes a record token and accepts both full length tokens and +// token prefixes. +func tokenDecodeAnyLength(token string) ([]byte, error) { + return util.TokenDecodeAnyLength(util.TokenTypeTlog, token) +} + +func convertSignatureError(err error) backend.PluginUserError { + var e util.SignatureError + var s ticketvote.ErrorCodeT + if errors.As(err, &e) { + switch e.ErrorCode { + case util.ErrorStatusPublicKeyInvalid: + s = ticketvote.ErrorCodePublicKeyInvalid + case util.ErrorStatusSignatureInvalid: + s = ticketvote.ErrorCodeSignatureInvalid + } + } + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(s), + ErrorContext: e.ErrorContext, + } +} + +func (p *ticketVotePlugin) authSave(treeID int64, ad ticketvote.AuthDetails) error { + // Prepare blob + be, err := convertBlobEntryFromAuthDetails(ad) + if err != nil { + return err + } + + // Save blob + return p.tlog.BlobSave(treeID, dataTypeAuthDetails, *be) +} + +func (p *ticketVotePlugin) auths(treeID int64) ([]ticketvote.AuthDetails, error) { + // Retrieve blobs + blobs, err := p.tlog.BlobsByDataType(treeID, dataTypeAuthDetails) + if err != nil { + return nil, err + } + + // Decode blobs + auths := make([]ticketvote.AuthDetails, 0, len(blobs)) + for _, v := range blobs { + a, err := convertAuthDetailsFromBlobEntry(v) + if err != nil { + return nil, err + } + auths = append(auths, *a) + } + + // Sanity check. They should already be sorted from oldest to + // newest. + sort.SliceStable(auths, func(i, j int) bool { + return auths[i].Timestamp < auths[j].Timestamp + }) + + return auths, nil +} + +func (p *ticketVotePlugin) voteDetailsSave(treeID int64, vd ticketvote.VoteDetails) error { + // Prepare blob + be, err := convertBlobEntryFromVoteDetails(vd) + if err != nil { + return err + } + + // Save blob + return p.tlog.BlobSave(treeID, dataTypeVoteDetails, *be) +} + +func (p *ticketVotePlugin) voteDetails(treeID int64) (*ticketvote.VoteDetails, error) { + // Retrieve blobs + blobs, err := p.tlog.BlobsByDataType(treeID, dataTypeVoteDetails) + if err != nil { + return nil, err + } + switch len(blobs) { + case 0: + // A vote details does not exist + return nil, nil + case 1: + // A vote details exists; continue + default: + // This should not happen. There should only ever be a max of + // one vote details. + return nil, fmt.Errorf("multiple vote details found (%v) on %x", + len(blobs), treeID) + } + + // Decode blob + vd, err := convertVoteDetailsFromBlobEntry(blobs[0]) + if err != nil { + return nil, err + } + + return vd, nil +} + +func (p *ticketVotePlugin) castVoteSave(treeID int64, cv ticketvote.CastVoteDetails) error { + // Prepare blob + be, err := convertBlobEntryFromCastVoteDetails(cv) + if err != nil { + return err + } + + // Save blob + return p.tlog.BlobSave(treeID, dataTypeCastVoteDetails, *be) +} + +func (p *ticketVotePlugin) castVotes(treeID int64) ([]ticketvote.CastVoteDetails, error) { + // Retrieve blobs + blobs, err := p.tlog.BlobsByDataType(treeID, dataTypeCastVoteDetails) + if err != nil { + return nil, err + } + + // Decode blobs + votes := make([]ticketvote.CastVoteDetails, 0, len(blobs)) + for _, v := range blobs { + cv, err := convertCastVoteDetailsFromBlobEntry(v) + if err != nil { + return nil, err + } + votes = append(votes, *cv) + } + + // Sort by ticket hash + sort.SliceStable(votes, func(i, j int) bool { + return votes[i].Ticket < votes[j].Ticket + }) + + return votes, nil +} + +// summary returns the vote summary for a record. +func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) (*ticketvote.VoteSummary, error) { + // Check if the summary has been cached + s, err := p.cachedSummary(hex.EncodeToString(token)) + switch { + case errors.Is(err, errSummaryNotFound): + // Cached summary not found. Continue. + case err != nil: + // Some other error + return nil, fmt.Errorf("cachedSummary: %v", err) + default: + // Caches summary was found. Return it. + return s, nil + } + + // Summary has not been cached. Get it manually. + + // Assume vote is unauthorized. Only update the status when the + // appropriate record has been found that proves otherwise. + status := ticketvote.VoteStatusUnauthorized + + // Check if the vote has been authorized + auths, err := p.auths(treeID) + if err != nil { + return nil, fmt.Errorf("auths: %v", err) + } + if len(auths) > 0 { + lastAuth := auths[len(auths)-1] + switch ticketvote.AuthActionT(lastAuth.Action) { + case ticketvote.AuthActionAuthorize: + // Vote has been authorized; continue + status = ticketvote.VoteStatusAuthorized + case ticketvote.AuthActionRevoke: + // Vote authorization has been revoked. Its not possible for + // the vote to have been started. We can stop looking. + return &ticketvote.VoteSummary{ + Status: status, + Results: []ticketvote.VoteOptionResult{}, + }, nil + } + } + + // Check if the vote has been started + vd, err := p.voteDetails(treeID) + if err != nil { + return nil, fmt.Errorf("startDetails: %v", err) + } + if vd == nil { + // Vote has not been started yet + return &ticketvote.VoteSummary{ + Status: status, + Results: []ticketvote.VoteOptionResult{}, + }, nil + } + + // Vote has been started. Check if it is still in progress or has + // already ended. + if bestBlock < vd.EndBlockHeight { + status = ticketvote.VoteStatusStarted + } else { + status = ticketvote.VoteStatusFinished + } + + // Pull the cast votes from the cache and tally the results + votes := p.cachedVotes(token) + tally := make(map[string]int, len(vd.Params.Options)) + for _, voteBit := range votes { + tally[voteBit]++ + } + results := make([]ticketvote.VoteOptionResult, 0, len(vd.Params.Options)) + for _, v := range vd.Params.Options { + bit := strconv.FormatUint(v.Bit, 16) + results = append(results, ticketvote.VoteOptionResult{ + ID: v.ID, + Description: v.Description, + VoteBit: v.Bit, + Votes: uint64(tally[bit]), + }) + } + + // Prepare summary + summary := ticketvote.VoteSummary{ + Type: vd.Params.Type, + Status: status, + Duration: vd.Params.Duration, + StartBlockHeight: vd.StartBlockHeight, + StartBlockHash: vd.StartBlockHash, + EndBlockHeight: vd.EndBlockHeight, + EligibleTickets: uint32(len(vd.EligibleTickets)), + QuorumPercentage: vd.Params.QuorumPercentage, + PassPercentage: vd.Params.PassPercentage, + Results: results, + } + + // If the vote has not finished yet then we are done for now. + if status == ticketvote.VoteStatusStarted { + return &summary, nil + } + + // The vote has finished. We can calculate if the vote was approved + // for certain vote types and cache the results. + switch vd.Params.Type { + case ticketvote.VoteTypeStandard, ticketvote.VoteTypeRunoff: + // These vote types are strictly approve/reject votes so we can + // calculate the vote approval. Continue. + default: + // Nothing else to do for all other vote types + return &summary, nil + } + + // Calculate vote approval + approved := voteIsApproved(*vd, results) + + // If this is a standard vote then we can take the results as is. + // A runoff vote requires that we pull all other runoff vote + // submissions to determine if the vote actually passed. + // TODO make summary for runoff vote submissions + summary.Approved = approved + + // Cache the summary + err = p.cachedSummarySave(vd.Params.Token, summary) + if err != nil { + return nil, fmt.Errorf("cachedSummarySave %v: %v %v", + vd.Params.Token, err, summary) + } + + // Remove record from the votes cache now that a summary has been + // saved for it. + p.cachedVotesDel(vd.Params.Token) + + return &summary, nil +} + +func (p *ticketVotePlugin) summaryByToken(token []byte) (*ticketvote.VoteSummary, error) { + reply, err := p.backend.VettedPluginCmd(token, ticketvote.ID, + ticketvote.CmdSummary, "") + if err != nil { + return nil, fmt.Errorf("VettedPluginCmd %x %v %v: %v", + token, ticketvote.ID, ticketvote.CmdSummary, err) + } + var sr ticketvote.SummaryReply + err = json.Unmarshal([]byte(reply), &sr) + if err != nil { + return nil, err + } + return &sr.Summary, nil +} + +func (p *ticketVotePlugin) timestamp(treeID int64, digest []byte) (*ticketvote.Timestamp, error) { + t, err := p.tlog.Timestamp(treeID, digest) + if err != nil { + return nil, fmt.Errorf("timestamp %v %x: %v", treeID, digest, err) + } + + // Convert response + proofs := make([]ticketvote.Proof, 0, len(t.Proofs)) + for _, v := range t.Proofs { + proofs = append(proofs, ticketvote.Proof{ + Type: v.Type, + Digest: v.Digest, + MerkleRoot: v.MerkleRoot, + MerklePath: v.MerklePath, + ExtraData: v.ExtraData, + }) + } + return &ticketvote.Timestamp{ + Data: t.Data, + Digest: t.Digest, + TxID: t.TxID, + MerkleRoot: t.MerkleRoot, + Proofs: proofs, + }, nil +} + +// bestBlock fetches the best block from the dcrdata plugin and returns it. If +// the dcrdata connection is not active, an error will be returned. +func (p *ticketVotePlugin) bestBlock() (uint32, error) { + // Get best block + payload, err := json.Marshal(dcrdata.BestBlock{}) + if err != nil { + return 0, err + } + reply, err := p.backend.Plugin(dcrdata.ID, + dcrdata.CmdBestBlock, "", string(payload)) + if err != nil { + return 0, fmt.Errorf("Plugin %v %v: %v", + dcrdata.ID, dcrdata.CmdBestBlock, err) + } + + // Handle response + var bbr dcrdata.BestBlockReply + err = json.Unmarshal([]byte(reply), &bbr) + if err != nil { + return 0, err + } + if bbr.Status != dcrdata.StatusConnected { + // The dcrdata connection is down. The best block cannot be + // trusted as being accurate. + return 0, fmt.Errorf("dcrdata connection is down") + } + if bbr.Height == 0 { + return 0, fmt.Errorf("invalid best block height 0") + } + + return bbr.Height, nil +} + +// bestBlockUnsafe fetches the best block from the dcrdata plugin and returns +// it. If the dcrdata connection is not active, an error WILL NOT be returned. +// The dcrdata cached best block height will be returned even though it may be +// stale. Use bestBlock() if the caller requires a guarantee that the best +// block is not stale. +func (p *ticketVotePlugin) bestBlockUnsafe() (uint32, error) { + // Get best block + payload, err := json.Marshal(dcrdata.BestBlock{}) + if err != nil { + return 0, err + } + reply, err := p.backend.Plugin(dcrdata.ID, + dcrdata.CmdBestBlock, "", string(payload)) + if err != nil { + return 0, fmt.Errorf("Plugin %v %v: %v", + dcrdata.ID, dcrdata.CmdBestBlock, err) + } + + // Handle response + var bbr dcrdata.BestBlockReply + err = json.Unmarshal([]byte(reply), &bbr) + if err != nil { + return 0, err + } + if bbr.Height == 0 { + return 0, fmt.Errorf("invalid best block height 0") + } + + return bbr.Height, nil +} + +type commitmentAddr struct { + ticket string // Ticket hash + addr string // Commitment address + err error // Error if one occurred +} + +func (p *ticketVotePlugin) largestCommitmentAddrs(tickets []string) ([]commitmentAddr, error) { + // Get tx details + tt := dcrdata.TxsTrimmed{ + TxIDs: tickets, + } + payload, err := json.Marshal(tt) + if err != nil { + return nil, err + } + reply, err := p.backend.Plugin(dcrdata.ID, + dcrdata.CmdTxsTrimmed, "", string(payload)) + if err != nil { + return nil, fmt.Errorf("Plugin %v %v: %v", + dcrdata.ID, dcrdata.CmdTxsTrimmed, err) + } + var ttr dcrdata.TxsTrimmedReply + err = json.Unmarshal([]byte(reply), &ttr) + if err != nil { + return nil, err + } + + // Find the largest commitment address for each tx + addrs := make([]commitmentAddr, 0, len(ttr.Txs)) + for _, tx := range ttr.Txs { + var ( + bestAddr string // Addr with largest commitment amount + bestAmt float64 // Largest commitment amount + addrErr error // Error if one is encountered + ) + for _, vout := range tx.Vout { + scriptPubKey := vout.ScriptPubKeyDecoded + switch { + case scriptPubKey.CommitAmt == nil: + // No commitment amount; continue + case len(scriptPubKey.Addresses) == 0: + // No commitment address; continue + case *scriptPubKey.CommitAmt > bestAmt: + // New largest commitment address found + bestAddr = scriptPubKey.Addresses[0] + bestAmt = *scriptPubKey.CommitAmt + } + } + if bestAddr == "" || bestAmt == 0.0 { + addrErr = fmt.Errorf("no largest commitment address found") + } + + // Store result + addrs = append(addrs, commitmentAddr{ + ticket: tx.TxID, + addr: bestAddr, + err: addrErr, + }) + } + + return addrs, nil +} + +// startReply fetches all required data and returns a StartReply. +func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, error) { + // Get the best block height + bb, err := p.bestBlock() + if err != nil { + return nil, fmt.Errorf("bestBlock: %v", err) + } + + // Find the snapshot height. Subtract the ticket maturity from the + // block height to get into unforkable territory. + ticketMaturity := uint32(p.activeNetParams.TicketMaturity) + snapshotHeight := bb - ticketMaturity + + // Fetch the block details for the snapshot height. We need the + // block hash in order to fetch the ticket pool snapshot. + bd := dcrdata.BlockDetails{ + Height: snapshotHeight, + } + payload, err := json.Marshal(bd) + if err != nil { + return nil, err + } + reply, err := p.backend.Plugin(dcrdata.ID, + dcrdata.CmdBlockDetails, "", string(payload)) + if err != nil { + return nil, fmt.Errorf("Plugin %v %v: %v", + dcrdata.ID, dcrdata.CmdBlockDetails, err) + } + var bdr dcrdata.BlockDetailsReply + err = json.Unmarshal([]byte(reply), &bdr) + if err != nil { + return nil, err + } + if bdr.Block.Hash == "" { + return nil, fmt.Errorf("invalid block hash for height %v", snapshotHeight) + } + snapshotHash := bdr.Block.Hash + + // Fetch the ticket pool snapshot + tp := dcrdata.TicketPool{ + BlockHash: snapshotHash, + } + payload, err = json.Marshal(tp) + if err != nil { + return nil, err + } + reply, err = p.backend.Plugin(dcrdata.ID, + dcrdata.CmdTicketPool, "", string(payload)) + if err != nil { + return nil, fmt.Errorf("Plugin %v %v: %v", + dcrdata.ID, dcrdata.CmdTicketPool, err) + } + var tpr dcrdata.TicketPoolReply + err = json.Unmarshal([]byte(reply), &tpr) + if err != nil { + return nil, err + } + if len(tpr.Tickets) == 0 { + return nil, fmt.Errorf("no tickets found for block %v %v", + snapshotHeight, snapshotHash) + } + + // The start block height has the ticket maturity subtracted from + // it to prevent forking issues. This means we the vote starts in + // the past. The ticket maturity needs to be added to the end block + // height to correct for this. + endBlockHeight := snapshotHeight + duration + ticketMaturity + + return &ticketvote.StartReply{ + StartBlockHeight: snapshotHeight, + StartBlockHash: snapshotHash, + EndBlockHeight: endBlockHeight, + EligibleTickets: tpr.Tickets, + }, nil +} + +func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdAuthorize: %v %x %v", treeID, token, payload) + + // Decode payload + var a ticketvote.Authorize + err := json.Unmarshal([]byte(payload), &a) + if err != nil { + return "", err + } + + // Verify token + t, err := tokenDecode(a.Token) + if err != nil { + return "", backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), + } + } + if !bytes.Equal(t, token) { + e := fmt.Sprintf("plugin token does not match route token: "+ + "got %x, want %x", t, token) + return "", backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), + ErrorContext: e, + } + } + + // Verify signature + version := strconv.FormatUint(uint64(a.Version), 10) + msg := a.Token + version + string(a.Action) + err = util.VerifySignature(a.Signature, a.PublicKey, msg) + if err != nil { + return "", convertSignatureError(err) + } + + // Verify action + switch a.Action { + case ticketvote.AuthActionAuthorize: + // This is allowed + case ticketvote.AuthActionRevoke: + // This is allowed + default: + e := fmt.Sprintf("%v not a valid action", a.Action) + return "", backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), + ErrorContext: e, + } + } + + // The previous authorize votes must be retrieved to validate the + // new autorize vote. The lock must be held for the remainder of + // this function. + m := p.mutex(token) + m.Lock() + defer m.Unlock() + + // Get any previous authorizations to verify that the new action + // is allowed based on the previous action. + auths, err := p.auths(treeID) + if err != nil { + return "", err + } + var prevAction ticketvote.AuthActionT + if len(auths) > 0 { + prevAction = ticketvote.AuthActionT(auths[len(auths)-1].Action) + } + switch { + case len(auths) == 0: + // No previous actions. New action must be an authorize. + if a.Action != ticketvote.AuthActionAuthorize { + return "", backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), + ErrorContext: "no prev action; action must be authorize", + } + } + case prevAction == ticketvote.AuthActionAuthorize && + a.Action != ticketvote.AuthActionRevoke: + // Previous action was a authorize. This action must be revoke. + return "", backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), + ErrorContext: "prev action was authorize", + } + case prevAction == ticketvote.AuthActionRevoke && + a.Action != ticketvote.AuthActionAuthorize: + // Previous action was a revoke. This action must be authorize. + return "", backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), + ErrorContext: "prev action was revoke", + } + } + + // Prepare authorize vote + receipt := p.identity.SignMessage([]byte(a.Signature)) + auth := ticketvote.AuthDetails{ + Token: a.Token, + Version: a.Version, + Action: string(a.Action), + PublicKey: a.PublicKey, + Signature: a.Signature, + Timestamp: time.Now().Unix(), + Receipt: hex.EncodeToString(receipt[:]), + } + + // Save authorize vote + err = p.authSave(treeID, auth) + if err != nil { + return "", err + } + + // Update inventory + switch a.Action { + case ticketvote.AuthActionAuthorize: + p.inventorySetToAuthorized(a.Token) + case ticketvote.AuthActionRevoke: + p.inventorySetToUnauthorized(a.Token) + default: + // Should not happen + e := fmt.Sprintf("invalid authorize action: %v", a.Action) + panic(e) + } + + // Prepare reply + ar := ticketvote.AuthorizeReply{ + Timestamp: auth.Timestamp, + Receipt: auth.Receipt, + } + reply, err := json.Marshal(ar) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func voteBitVerify(options []ticketvote.VoteOption, mask, bit uint64) error { + if len(options) == 0 { + return fmt.Errorf("no vote options found") + } + if bit == 0 { + return fmt.Errorf("invalid bit 0x%x", bit) + } + + // Verify bit is included in mask + if mask&bit != bit { + return fmt.Errorf("invalid mask 0x%x bit 0x%x", mask, bit) + } + + // Verify bit is included in vote options + for _, v := range options { + if v.Bit == bit { + // Bit matches one of the options. We're done. + return nil + } + } + + return fmt.Errorf("bit 0x%x not found in vote options", bit) +} + +// TODO test this function +func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationMax uint32) error { + // Verify vote type + switch vote.Type { + case ticketvote.VoteTypeStandard: + // This is allowed + case ticketvote.VoteTypeRunoff: + // This is allowed + default: + e := fmt.Sprintf("invalid type %v", vote.Type) + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorContext: e, + } + } + + // Verify vote params + switch { + case vote.Duration > voteDurationMax: + e := fmt.Sprintf("duration %v exceeds max duration %v", + vote.Duration, voteDurationMax) + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorContext: e, + } + case vote.Duration < voteDurationMin: + e := fmt.Sprintf("duration %v under min duration %v", + vote.Duration, voteDurationMin) + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorContext: e, + } + case vote.QuorumPercentage > 100: + e := fmt.Sprintf("quorum percent %v exceeds 100 percent", + vote.QuorumPercentage) + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorContext: e, + } + case vote.PassPercentage > 100: + e := fmt.Sprintf("pass percent %v exceeds 100 percent", + vote.PassPercentage) + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorContext: e, + } + } + + // Verify vote options. Different vote types have different + // requirements. + if len(vote.Options) == 0 { + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorContext: "no vote options found", + } + } + switch vote.Type { + case ticketvote.VoteTypeStandard, ticketvote.VoteTypeRunoff: + // These vote types only allow for approve/reject votes. Ensure + // that the only options present are approve/reject and that they + // use the vote option IDs specified by the ticketvote API. + if len(vote.Options) != 2 { + e := fmt.Sprintf("vote options count got %v, want 2", + len(vote.Options)) + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorContext: e, + } + } + // map[optionID]found + options := map[string]bool{ + ticketvote.VoteOptionIDApprove: false, + ticketvote.VoteOptionIDReject: false, + } + for _, v := range vote.Options { + switch v.ID { + case ticketvote.VoteOptionIDApprove: + options[v.ID] = true + case ticketvote.VoteOptionIDReject: + options[v.ID] = true + } + } + missing := make([]string, 0, 2) + for k, v := range options { + if !v { + // Option ID was not found + missing = append(missing, k) + } + } + if len(missing) > 0 { + e := fmt.Sprintf("vote option IDs not found: %v", + strings.Join(missing, ",")) + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorContext: e, + } + } + } + + // Verify vote bits are somewhat sane + for _, v := range vote.Options { + err := voteBitVerify(vote.Options, vote.Mask, v.Bit) + if err != nil { + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorContext: err.Error(), + } + } + } + + // Verify parent token + switch { + case vote.Type == ticketvote.VoteTypeStandard && vote.Parent != "": + e := "parent token should not be provided for a standard vote" + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorContext: e, + } + case vote.Type == ticketvote.VoteTypeRunoff: + _, err := tokenDecode(vote.Parent) + if err != nil { + e := fmt.Sprintf("invalid parent %v", vote.Parent) + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorContext: e, + } + } + } + + return nil +} + +// startStandard starts a standard vote. +func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvote.Start) (*ticketvote.StartReply, error) { + // Verify there is only one start details + if len(s.Starts) != 1 { + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeStartDetailsInvalid), + ErrorContext: "more than one start details found", + } + } + sd := s.Starts[0] + + // Verify token + t, err := tokenDecode(sd.Params.Token) + if err != nil { + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), + } + } + if !bytes.Equal(t, token) { + e := fmt.Sprintf("plugin token does not match route token: "+ + "got %x, want %x", t, token) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), + ErrorContext: e, + } + } + + // Verify signature + vb, err := json.Marshal(sd.Params) + if err != nil { + return nil, err + } + msg := hex.EncodeToString(util.Digest(vb)) + err = util.VerifySignature(sd.Signature, sd.PublicKey, msg) + if err != nil { + return nil, convertSignatureError(err) + } + + // Verify vote options and params + err = voteParamsVerify(sd.Params, p.voteDurationMin, p.voteDurationMax) + if err != nil { + return nil, err + } + + // Get vote blockchain data + sr, err := p.startReply(sd.Params.Duration) + if err != nil { + return nil, err + } + + // Validate existing record state. The lock for this record must be + // held for the remainder of this function. + m := p.mutex(token) + m.Lock() + defer m.Unlock() + + // Verify record version + r, err := p.backend.GetVetted(token, "") + if err != nil { + return nil, fmt.Errorf("GetVetted: %v", err) + } + version := strconv.FormatUint(uint64(sd.Params.Version), 10) + if r.Version != version { + e := fmt.Sprintf("version is not latest: got %v, want %v", + sd.Params.Version, r.Version) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeRecordVersionInvalid), + ErrorContext: e, + } + } + + // Verify vote authorization + auths, err := p.auths(treeID) + if err != nil { + return nil, err + } + if len(auths) == 0 { + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), + ErrorContext: "authorization not found", + } + } + action := ticketvote.AuthActionT(auths[len(auths)-1].Action) + if action != ticketvote.AuthActionAuthorize { + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), + ErrorContext: "not authorized", + } + } + + // Verify vote has not already been started + svp, err := p.voteDetails(treeID) + if err != nil { + return nil, err + } + if svp != nil { + // Vote has already been started + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeVoteStatusInvalid), + ErrorContext: "vote already started", + } + } + + // Prepare vote details + vd := ticketvote.VoteDetails{ + Params: sd.Params, + PublicKey: sd.PublicKey, + Signature: sd.Signature, + StartBlockHeight: sr.StartBlockHeight, + StartBlockHash: sr.StartBlockHash, + EndBlockHeight: sr.EndBlockHeight, + EligibleTickets: sr.EligibleTickets, + } + + // Save vote details + err = p.voteDetailsSave(treeID, vd) + if err != nil { + return nil, fmt.Errorf("voteDetailsSave: %v", err) + } + + // Update inventory + p.inventorySetToStarted(vd.Params.Token, ticketvote.VoteTypeStandard, + vd.EndBlockHeight) + + return &ticketvote.StartReply{ + StartBlockHeight: sr.StartBlockHeight, + StartBlockHash: sr.StartBlockHash, + EndBlockHeight: sr.EndBlockHeight, + EligibleTickets: sr.EligibleTickets, + }, nil +} + +// startRunoffRecord is the record that is saved to the runoff vote's parent +// tree as the first step in starting a runoff vote. Plugins are not able to +// update multiple records atomically, so if this call gets interrupted before +// if can start the voting period on all runoff vote submissions, subsequent +// calls will use this record to pick up where the previous call left off. This +// allows us to recover from unexpected errors, such as network errors, and not +// leave a runoff vote in a weird state. +type startRunoffRecord struct { + Submissions []string `json:"submissions"` + Mask uint64 `json:"mask"` + Duration uint32 `json:"duration"` + QuorumPercentage uint32 `json:"quorumpercentage"` + PassPercentage uint32 `json:"passpercentage"` + StartBlockHeight uint32 `json:"startblockheight"` + StartBlockHash string `json:"startblockhash"` + EndBlockHeight uint32 `json:"endblockheight"` + EligibleTickets []string `json:"eligibletickets"` +} + +func (p *ticketVotePlugin) startRunoffRecordSave(treeID int64, srr startRunoffRecord) error { + be, err := convertBlobEntryFromStartRunoff(srr) + if err != nil { + return err + } + err = p.tlog.BlobSave(treeID, dataTypeStartRunoff, *be) + if err != nil { + return fmt.Errorf("BlobSave %v %v: %v", + treeID, dataTypeStartRunoff, err) + } + return nil +} + +// startRunoffRecord returns the startRunoff record if one exists on a tree. +// nil will be returned if a startRunoff record is not found. +func (p *ticketVotePlugin) startRunoffRecord(treeID int64) (*startRunoffRecord, error) { + blobs, err := p.tlog.BlobsByDataType(treeID, dataTypeStartRunoff) + if err != nil { + return nil, fmt.Errorf("BlobsByDataType %v %v: %v", + treeID, dataTypeStartRunoff, err) + } + + var srr *startRunoffRecord + switch len(blobs) { + case 0: + // Nothing found + return nil, nil + case 1: + // A start runoff record was found + srr, err = convertStartRunoffFromBlobEntry(blobs[0]) + if err != nil { + return nil, err + } + default: + // This should not be possible + e := fmt.Sprintf("%v start runoff blobs found", len(blobs)) + panic(e) + } + + return srr, nil +} + // startRunoffSub is an internal plugin command that is used to start the // voting period on a runoff vote submission. type startRunoffSub struct { @@ -18,5 +1077,1436 @@ type startRunoffSub struct { StartDetails ticketvote.StartDetails `json:"startdetails"` } -// startRunoffSubReply is the reply to the startRunoffSub command. -type startRunoffSubReply struct{} +func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs startRunoffSub) error { + // Sanity check + s := srs.StartDetails + t, err := tokenDecode(s.Params.Token) + if err != nil { + return err + } + if bytes.Equal(token, t) { + return fmt.Errorf("invalid token") + } + + // Get the start runoff record from the parent tree + srr, err := p.startRunoffRecord(srs.ParentTreeID) + if err != nil { + return err + } + + // Sanity check. Verify token is part of the start runoff record + // submissions. + var found bool + for _, v := range srr.Submissions { + if hex.EncodeToString(token) == v { + found = true + break + } + } + if !found { + // This submission should not be here + return fmt.Errorf("record not in submission list") + } + + // If the vote has already been started, exit gracefully. This + // allows us to recover from unexpected errors to the start runoff + // vote call as it updates the state of multiple records. If the + // call were to fail before completing, we can simply call the + // command again with the same arguments and it will pick up where + // it left off. + svp, err := p.voteDetails(treeID) + if err != nil { + return err + } + if svp != nil { + // Vote has already been started. Exit gracefully. + return nil + } + + // Verify record version + r, err := p.backend.GetVetted(token, "") + if err != nil { + return fmt.Errorf("GetVetted: %v", err) + } + version := strconv.FormatUint(uint64(s.Params.Version), 10) + if r.Version != version { + e := fmt.Sprintf("version is not latest %v: got %v, want %v", + s.Params.Token, s.Params.Version, r.Version) + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeRecordVersionInvalid), + ErrorContext: e, + } + } + + // Prepare vote details + vd := ticketvote.VoteDetails{ + Params: s.Params, + PublicKey: s.PublicKey, + Signature: s.Signature, + StartBlockHeight: srr.StartBlockHeight, + StartBlockHash: srr.StartBlockHash, + EndBlockHeight: srr.EndBlockHeight, + EligibleTickets: srr.EligibleTickets, + } + + // Save vote details + err = p.voteDetailsSave(treeID, vd) + if err != nil { + return fmt.Errorf("voteDetailsSave: %v", err) + } + + // Update inventory + p.inventorySetToStarted(vd.Params.Token, ticketvote.VoteTypeRunoff, + vd.EndBlockHeight) + + return nil +} + +func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ticketvote.Start) (*startRunoffRecord, error) { + // Check if the runoff vote data already exists on the parent tree. + srr, err := p.startRunoffRecord(treeID) + if err != nil { + return nil, err + } + if srr != nil { + // We already have a start runoff record for this runoff vote. + // This can happen if the previous call failed due to an + // unexpected error such as a network error. Return the start + // runoff record so we can pick up where we left off. + return srr, nil + } + + // Get blockchain data + var ( + mask = s.Starts[0].Params.Mask + duration = s.Starts[0].Params.Duration + quorum = s.Starts[0].Params.QuorumPercentage + pass = s.Starts[0].Params.PassPercentage + ) + sr, err := p.startReply(duration) + if err != nil { + return nil, err + } + + // The parent record must be validated. Hold the lock on the parent + // record for the remainder of this function. + m := p.mutex(token) + m.Lock() + defer m.Unlock() + + // Verify parent has a LinkBy and the LinkBy deadline is expired. + r, err := p.backend.GetVetted(token, "") + if err != nil { + if errors.Is(err, backend.ErrRecordNotFound) { + e := fmt.Sprintf("parent record not found %x", token) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorContext: e, + } + } + } + vm, err := decodeVoteMetadata(r.Files) + if err != nil { + return nil, err + } + if vm == nil || vm.LinkBy == 0 { + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeRunoffVoteParentInvalid), + } + } + isExpired := vm.LinkBy < time.Now().Unix() + isMainNet := p.activeNetParams.Name == chaincfg.MainNetParams().Name + switch { + case !isExpired && isMainNet: + e := fmt.Sprintf("parent record %x linkby deadline not met %v", + token, vm.LinkBy) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeLinkByNotExpired), + ErrorContext: e, + } + case !isExpired: + // Allow the vote to be started before the link by deadline + // expires on testnet and simnet only. This makes testing the + // runoff vote process easier. + log.Warnf("Parent record linkby deadline has not been met; " + + "disregarding deadline since this is not mainnet") + } + + // Compile a list of the expected submissions that should be in the + // runoff vote. This will be all of the public records that have + // linked to the parent record. The parent record's linked from + // list will include abandoned proposals that need to be filtered + // out. + lf, err := p.linkedFrom(token) + if err != nil { + return nil, err + } + expected := make(map[string]struct{}, len(lf.Tokens)) // [token]struct{} + for k := range lf.Tokens { + token, err := tokenDecode(k) + if err != nil { + return nil, err + } + r, err := p.backend.GetVetted(token, "") + if err != nil { + return nil, err + } + if r.RecordMetadata.Status != backend.MDStatusVetted { + // This record is not public and should not be included + // in the runoff vote. + continue + } + + // This is a public record that is part of the parent record's + // linked from list. It is required to be in the runoff vote. + expected[k] = struct{}{} + } + + // Verify that there are no extra submissions in the runoff vote + for _, v := range s.Starts { + _, ok := expected[v.Params.Token] + if !ok { + // This submission should not be here + e := fmt.Sprintf("record %v should not be included", + v.Params.Token) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeStartDetailsInvalid), + ErrorContext: e, + } + } + } + + // Verify that the runoff vote is not missing any submissions + subs := make(map[string]struct{}, len(s.Starts)) + for _, v := range s.Starts { + subs[v.Params.Token] = struct{}{} + } + for k := range expected { + _, ok := subs[k] + if !ok { + // This records is missing from the runoff vote + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeStartDetailsMissing), + ErrorContext: k, + } + } + } + + // Prepare start runoff record + submissions := make([]string, 0, len(subs)) + for k := range subs { + submissions = append(submissions, k) + } + srr = &startRunoffRecord{ + Submissions: submissions, + Mask: mask, + Duration: duration, + QuorumPercentage: quorum, + PassPercentage: pass, + StartBlockHeight: sr.StartBlockHeight, + StartBlockHash: sr.StartBlockHash, + EndBlockHeight: sr.EndBlockHeight, + EligibleTickets: sr.EligibleTickets, + } + + // Save start runoff record + err = p.startRunoffRecordSave(treeID, *srr) + if err != nil { + return nil, fmt.Errorf("startRunoffRecordSave %v: %v", + treeID, err) + } + + return srr, nil +} + +// startRunoff starts a runoff vote. +func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote.Start) (*ticketvote.StartReply, error) { + // Sanity check + if len(s.Starts) == 0 { + return nil, fmt.Errorf("no start details found") + } + + // Perform validation that can be done without fetching any records + // from the backend. + var ( + mask = s.Starts[0].Params.Mask + duration = s.Starts[0].Params.Duration + quorum = s.Starts[0].Params.QuorumPercentage + pass = s.Starts[0].Params.PassPercentage + parent = s.Starts[0].Params.Parent + ) + for _, v := range s.Starts { + // Verify vote params are the same for all submissions + switch { + case v.Params.Type != ticketvote.VoteTypeRunoff: + e := fmt.Sprintf("%v vote type invalid: got %v, want %v", + v.Params.Token, v.Params.Type, ticketvote.VoteTypeRunoff) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorContext: e, + } + case v.Params.Mask != mask: + e := fmt.Sprintf("%v mask invalid: all must be the same", + v.Params.Token) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorContext: e, + } + case v.Params.Duration != duration: + e := fmt.Sprintf("%v duration invalid: all must be the same", + v.Params.Token) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorContext: e, + } + case v.Params.QuorumPercentage != quorum: + e := fmt.Sprintf("%v quorum invalid: must be the same", + v.Params.Token) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorContext: e, + } + case v.Params.PassPercentage != pass: + e := fmt.Sprintf("%v pass rate invalid: all must be the same", + v.Params.Token) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorContext: e, + } + case v.Params.Parent != parent: + e := fmt.Sprintf("%v parent invalid: all must be the same", + v.Params.Token) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorContext: e, + } + } + + // Verify token + _, err := tokenDecode(v.Params.Token) + if err != nil { + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), + ErrorContext: v.Params.Token, + } + } + + // Verify parent token + _, err = tokenDecode(v.Params.Parent) + if err != nil { + e := fmt.Sprintf("parent token %v", v.Params.Parent) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), + ErrorContext: e, + } + } + + // Verify signature + vb, err := json.Marshal(v.Params) + if err != nil { + return nil, err + } + msg := hex.EncodeToString(util.Digest(vb)) + err = util.VerifySignature(v.Signature, v.PublicKey, msg) + if err != nil { + return nil, convertSignatureError(err) + } + + // Verify vote options and params. Vote optoins are required to + // be approve and reject. + err = voteParamsVerify(v.Params, p.voteDurationMin, p.voteDurationMax) + if err != nil { + return nil, err + } + } + + // Verify plugin command is being executed on the parent record + if hex.EncodeToString(token) != parent { + e := fmt.Sprintf("runoff vote must be started on parent record %v", + parent) + return nil, backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeRunoffVoteParentInvalid), + ErrorContext: e, + } + } + + // This function is being invoked on the runoff vote parent record. + // Create and save a start runoff record onto the parent record's + // tree. + srr, err := p.startRunoffForParent(treeID, token, s) + if err != nil { + return nil, err + } + + // Start the voting period of each runoff vote submissions by + // using the internal plugin command startRunoffSub. + for _, v := range s.Starts { + token, err = tokenDecode(v.Params.Token) + if err != nil { + return nil, err + } + srs := startRunoffSub{ + ParentTreeID: treeID, + StartDetails: v, + } + b, err := json.Marshal(srs) + if err != nil { + return nil, err + } + _, err = p.backend.VettedPluginCmd(token, ticketvote.ID, + cmdStartRunoffSub, string(b)) + if err != nil { + var ue backend.PluginUserError + if errors.As(err, &ue) { + return nil, err + } + return nil, fmt.Errorf("VettedPluginCmd %x %v %v %v: %v", + token, ticketvote.ID, cmdStartRunoffSub, b, err) + } + } + + return &ticketvote.StartReply{ + StartBlockHeight: srr.StartBlockHeight, + StartBlockHash: srr.StartBlockHash, + EndBlockHeight: srr.EndBlockHeight, + EligibleTickets: srr.EligibleTickets, + }, nil +} + +func (p *ticketVotePlugin) cmdStartRunoffSub(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdStartRunoffSub: %v %x %v", treeID, token, payload) + + // Decode payload + var srs startRunoffSub + err := json.Unmarshal([]byte(payload), &srs) + if err != nil { + return "", err + } + + // Start voting period on runoff vote submission + err = p.startRunoffForSub(treeID, token, srs) + if err != nil { + return "", err + } + + return "", nil +} + +func (p *ticketVotePlugin) cmdStart(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdStart: %v %x %v", treeID, token, payload) + + // Decode payload + var s ticketvote.Start + err := json.Unmarshal([]byte(payload), &s) + if err != nil { + return "", err + } + + // Parse vote type + if len(s.Starts) == 0 { + return "", backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeStartDetailsMissing), + ErrorContext: "no start details found", + } + } + vtype := s.Starts[0].Params.Type + + // Start vote + // TODO these vote user errors need to become more granular. Update + // this when writing tests. + var sr *ticketvote.StartReply + switch vtype { + case ticketvote.VoteTypeStandard: + sr, err = p.startStandard(treeID, token, s) + if err != nil { + return "", err + } + case ticketvote.VoteTypeRunoff: + sr, err = p.startRunoff(treeID, token, s) + if err != nil { + return "", err + } + default: + e := fmt.Sprintf("invalid vote type %v", vtype) + return "", backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorContext: e, + } + } + + // Prepare reply + reply, err := json.Marshal(*sr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +// voteMessageVerify verifies a cast vote message is properly signed. Copied +// from: github.com/decred/dcrd/blob/0fc55252f912756c23e641839b1001c21442c38a/rpcserver.go#L5605 +func (p *ticketVotePlugin) voteMessageVerify(address, message, signature string) (bool, error) { + // Decode the provided address. + addr, err := dcrutil.DecodeAddress(address, p.activeNetParams) + if err != nil { + return false, fmt.Errorf("Could not decode address: %v", + err) + } + + // Only P2PKH addresses are valid for signing. + if _, ok := addr.(*dcrutil.AddressPubKeyHash); !ok { + return false, fmt.Errorf("Address is not a pay-to-pubkey-hash "+ + "address: %v", address) + } + + // Decode base64 signature. + sig, err := base64.StdEncoding.DecodeString(signature) + if err != nil { + return false, fmt.Errorf("Malformed base64 encoding: %v", err) + } + + // Validate the signature - this just shows that it was valid at all. + // we will compare it with the key next. + var buf bytes.Buffer + wire.WriteVarString(&buf, 0, "Decred Signed Message:\n") + wire.WriteVarString(&buf, 0, message) + expectedMessageHash := chainhash.HashB(buf.Bytes()) + pk, wasCompressed, err := ecdsa.RecoverCompact(sig, + expectedMessageHash) + if err != nil { + // Mirror Bitcoin Core behavior, which treats error in + // RecoverCompact as invalid signature. + return false, nil + } + + // Reconstruct the pubkey hash. + dcrPK := pk + var serializedPK []byte + if wasCompressed { + serializedPK = dcrPK.SerializeCompressed() + } else { + serializedPK = dcrPK.SerializeUncompressed() + } + a, err := dcrutil.NewAddressSecpPubKey(serializedPK, p.activeNetParams) + if err != nil { + // Again mirror Bitcoin Core behavior, which treats error in + // public key reconstruction as invalid signature. + return false, nil + } + + // Return boolean if addresses match. + return a.Address() == address, nil +} + +func (p *ticketVotePlugin) castVoteSignatureVerify(cv ticketvote.CastVote, addr string) error { + msg := cv.Token + cv.Ticket + cv.VoteBit + + // Convert hex signature to base64. The voteMessageVerify function + // expects base64. + b, err := hex.DecodeString(cv.Signature) + if err != nil { + return fmt.Errorf("invalid hex") + } + sig := base64.StdEncoding.EncodeToString(b) + + // Verify message + validated, err := p.voteMessageVerify(addr, msg, sig) + if err != nil { + return err + } + if !validated { + return fmt.Errorf("could not verify message") + } + + return nil +} + +// ballot casts the provided votes concurrently. The vote results are passed +// back through the results channel to the calling function. This function +// waits until all provided votes have been cast before returning. +// +// This function must be called WITH the record lock held. +func (p *ticketVotePlugin) ballot(treeID int64, votes []ticketvote.CastVote, results chan ticketvote.CastVoteReply) { + // Cast the votes concurrently + var wg sync.WaitGroup + for _, v := range votes { + // Increment the wait group counter + wg.Add(1) + + go func(v ticketvote.CastVote) { + // Decrement wait group counter once vote is cast + defer wg.Done() + + // Setup cast vote details + receipt := p.identity.SignMessage([]byte(v.Signature)) + cv := ticketvote.CastVoteDetails{ + Token: v.Token, + Ticket: v.Ticket, + VoteBit: v.VoteBit, + Signature: v.Signature, + Receipt: hex.EncodeToString(receipt[:]), + } + + // Save cast vote + var cvr ticketvote.CastVoteReply + err := p.castVoteSave(treeID, cv) + if err != nil { + t := time.Now().Unix() + log.Errorf("cmdCastBallot: castVoteSave %v: %v", t, err) + e := ticketvote.VoteErrorInternalError + cvr.Ticket = v.Ticket + cvr.ErrorCode = e + cvr.ErrorContext = fmt.Sprintf("%v: %v", + ticketvote.VoteErrors[e], t) + goto sendResult + } + + // Update receipt + cvr.Ticket = v.Ticket + cvr.Receipt = cv.Receipt + + // Update cast votes cache + p.cachedVotesSet(v.Token, v.Ticket, v.VoteBit) + + sendResult: + // Send result back to calling function + results <- cvr + }(v) + } + + // Wait for the full ballot to be cast before returning. + wg.Wait() +} + +// cmdCastBallot casts a ballot of votes. This function will not return a user +// error if one occurs. It will instead return the ballot reply with the error +// included in the invidiual cast vote reply that it applies to. +func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdCastBallot: %v %x %v", treeID, token, payload) + + // Decode payload + var cb ticketvote.CastBallot + err := json.Unmarshal([]byte(payload), &cb) + if err != nil { + return "", err + } + votes := cb.Ballot + + // Verify there is work to do + if len(votes) == 0 { + // Nothing to do + cbr := ticketvote.CastBallotReply{ + Receipts: []ticketvote.CastVoteReply{}, + } + reply, err := json.Marshal(cbr) + if err != nil { + return "", err + } + return string(reply), nil + } + + // Verify that all tokens in the ballot are valid, full length + // tokens and that they are all voting for the same record. + var ( + receipts = make([]ticketvote.CastVoteReply, len(votes)) + ) + for k, v := range votes { + // Verify token + t, err := tokenDecode(v.Token) + if err != nil { + e := ticketvote.VoteErrorTokenInvalid + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = fmt.Sprintf("%v: not hex", + ticketvote.VoteErrors[e]) + continue + } + + // Verify token is the same + if !bytes.Equal(t, token) { + e := ticketvote.VoteErrorMultipleRecordVotes + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = ticketvote.VoteErrors[e] + continue + } + } + + // From this point forward, it can be assumed that all votes that + // have not had their error set are voting for the same record. Get + // the record and vote data that we need to perform the remaining + // inexpensive validation before we have to hold the lock. + voteDetails, err := p.voteDetails(treeID) + if err != nil { + return "", err + } + bestBlock, err := p.bestBlock() + if err != nil { + return "", err + } + + // eligible contains the ticket hashes of all eligble tickets. They + // are put into a map for O(n) lookups. + eligible := make(map[string]struct{}, len(voteDetails.EligibleTickets)) + for _, v := range voteDetails.EligibleTickets { + eligible[v] = struct{}{} + } + + // addrs contains the largest commitment addresses for each ticket. + // The vote must be signed using the largest commitment address. + tickets := make([]string, 0, len(cb.Ballot)) + for _, v := range cb.Ballot { + tickets = append(tickets, v.Ticket) + } + addrs, err := p.largestCommitmentAddrs(tickets) + if err != nil { + return "", fmt.Errorf("largestCommitmentAddrs: %v", err) + } + + // Perform validation that doesn't require holding the record lock. + for k, v := range votes { + if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { + // Vote has an error. Skip it. + continue + } + + // Verify record vote status + if voteDetails == nil { + // Vote has not been started yet + e := ticketvote.VoteErrorVoteStatusInvalid + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = fmt.Sprintf("%v: vote not started", + ticketvote.VoteErrors[e]) + continue + } + if bestBlock >= voteDetails.EndBlockHeight { + // Vote has ended + e := ticketvote.VoteErrorVoteStatusInvalid + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = fmt.Sprintf("%v: vote has ended", + ticketvote.VoteErrors[e]) + continue + } + + // Verify vote bit + bit, err := strconv.ParseUint(v.VoteBit, 16, 64) + if err != nil { + e := ticketvote.VoteErrorVoteBitInvalid + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = ticketvote.VoteErrors[e] + continue + } + err = voteBitVerify(voteDetails.Params.Options, + voteDetails.Params.Mask, bit) + if err != nil { + e := ticketvote.VoteErrorVoteBitInvalid + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = fmt.Sprintf("%v: %v", + ticketvote.VoteErrors[e], err) + continue + } + + // Verify vote signature + commitmentAddr := addrs[k] + if commitmentAddr.ticket != v.Ticket { + t := time.Now().Unix() + log.Errorf("cmdCastBallot: commitment addr mismatch %v: %v %v", + t, commitmentAddr.ticket, v.Ticket) + e := ticketvote.VoteErrorInternalError + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = fmt.Sprintf("%v: %v", + ticketvote.VoteErrors[e], t) + continue + } + if commitmentAddr.err != nil { + t := time.Now().Unix() + log.Errorf("cmdCastBallot: commitment addr error %v: %v %v", + t, commitmentAddr.ticket, commitmentAddr.err) + e := ticketvote.VoteErrorInternalError + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = fmt.Sprintf("%v: %v", + ticketvote.VoteErrors[e], t) + continue + } + err = p.castVoteSignatureVerify(v, commitmentAddr.addr) + if err != nil { + e := ticketvote.VoteErrorSignatureInvalid + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = fmt.Sprintf("%v: %v", + ticketvote.VoteErrors[e], err) + continue + } + + // Verify ticket is eligible to vote + _, ok := eligible[v.Ticket] + if !ok { + e := ticketvote.VoteErrorTicketNotEligible + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = ticketvote.VoteErrors[e] + continue + } + } + + // The record lock must be held for the remainder of the function to + // ensure duplicate votes cannot be cast. + m := p.mutex(token) + m.Lock() + defer m.Unlock() + + // cachedVotes contains the tickets that have alread voted + cachedVotes := p.cachedVotes(token) + for k, v := range votes { + if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { + // Vote has an error. Skip it. + continue + } + + // Verify ticket has not already vote + _, ok := cachedVotes[v.Ticket] + if ok { + e := ticketvote.VoteErrorTicketAlreadyVoted + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = ticketvote.VoteErrors[e] + continue + } + } + + // The votes that have passed validation will be cast in batches of + // size batchSize. Each batch of votes is cast concurrently in order + // to accommodate the trillian log signer bottleneck. The log signer + // picks up queued leaves and appends them onto the trillian tree + // every xxx ms, where xxx is a configurable value on the log signer, + // but is typically a few hundred milliseconds. Lets use 200ms as an + // example. If we don't cast the votes in batches then every vote in + // the ballot will take 200 milliseconds since we wait for the leaf + // to be fully appended before considering the trillian call + // successful. A person casting hundreds of votes in a single ballot + // would cause UX issues for all the voting clients since the lock is + // held during these calls. + // + // The second variable that we must watch out for is the max trillian + // queued leaf batch size. This is also a configurable trillian value + // that represents the maximum number of leaves that can be waiting + // in the queue for all trees in the trillian instance. This value is + // typically around the order of magnitude of 1000 queued leaves. + // + // This is why a vote batch size of 5 was chosen. It is large enough + // to alleviate performance bottlenecks from the log signer interval, + // but small enough to still allow multiple records votes be held + // concurrently without running into the queued leaf batch size limit. + + // Prepare work + var ( + batchSize = 5 + batch = make([]ticketvote.CastVote, 0, batchSize) + queue = make([][]ticketvote.CastVote, 0, len(votes)/batchSize) + + // ballotCount is the number of votes that have passed validation + // and are being cast in this ballot. + ballotCount int + ) + for k, v := range votes { + if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { + // Vote has an error. Skip it. + continue + } + + // Add vote to the current batch + batch = append(batch, v) + ballotCount++ + + if len(batch) == batchSize { + // This batch is full. Add the batch to the queue and start + // a new batch. + queue = append(queue, batch) + batch = make([]ticketvote.CastVote, 0, batchSize) + } + } + if len(batch) != 0 { + // Add leftover batch to the queue + queue = append(queue, batch) + } + + log.Debugf("Casting %v votes in %v batches of size %v", + ballotCount, len(queue), batchSize) + + // Cast ballot in batches + results := make(chan ticketvote.CastVoteReply, ballotCount) + for i, batch := range queue { + log.Debugf("Casting %v votes in batch %v/%v", len(batch), i+1, len(queue)) + + p.ballot(treeID, batch, results) + } + + // Empty out the results channel + r := make(map[string]ticketvote.CastVoteReply, ballotCount) + close(results) + for v := range results { + r[v.Ticket] = v + } + + if len(r) != ballotCount { + log.Errorf("Missing results: got %v, want %v", len(r), ballotCount) + } + + // Fill in the receipts + for k, v := range votes { + if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { + // Vote has an error. Skip it. + continue + } + cvr, ok := r[v.Ticket] + if !ok { + t := time.Now().Unix() + log.Errorf("cmdCastBallot: vote result not found %v: %v", t, v.Ticket) + e := ticketvote.VoteErrorInternalError + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = fmt.Sprintf("%v: %v", + ticketvote.VoteErrors[e], t) + continue + } + + // Fill in receipt + receipts[k] = cvr + } + + // Prepare reply + cbr := ticketvote.CastBallotReply{ + Receipts: receipts, + } + reply, err := json.Marshal(cbr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *ticketVotePlugin) cmdDetails(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdDetails: %v %x %v", treeID, token, payload) + + var d ticketvote.Details + err := json.Unmarshal([]byte(payload), &d) + if err != nil { + return "", err + } + + // Get vote authorizations + auths, err := p.auths(treeID) + if err != nil { + return "", fmt.Errorf("auths: %v", err) + } + + // Get vote details + vd, err := p.voteDetails(treeID) + if err != nil { + return "", fmt.Errorf("voteDetails: %v", err) + } + + // Prepare rely + dr := ticketvote.DetailsReply{ + Auths: auths, + Vote: vd, + } + reply, err := json.Marshal(dr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *ticketVotePlugin) cmdResults(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdResults: %v %x %v", treeID, token, payload) + + // Get cast votes + votes, err := p.castVotes(treeID) + if err != nil { + return "", err + } + + // Prepare reply + rr := ticketvote.ResultsReply{ + Votes: votes, + } + reply, err := json.Marshal(rr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +// voteIsApproved returns whether the provided vote option results met the +// provided quorum and pass percentage requirements. This function can only be +// called on votes that use VoteOptionIDApprove and VoteOptionIDReject. Any +// other vote option IDs will cause this function to panic. +func voteIsApproved(vd ticketvote.VoteDetails, results []ticketvote.VoteOptionResult) bool { + // Tally the total votes + var total uint64 + for _, v := range results { + total += v.Votes + } + + // Calculate required thresholds + var ( + eligible = float64(len(vd.EligibleTickets)) + quorumPerc = float64(vd.Params.QuorumPercentage) + passPerc = float64(vd.Params.PassPercentage) + quorum = uint64(quorumPerc / 100 * eligible) + pass = uint64(passPerc / 100 * float64(total)) + + approvedVotes uint64 + ) + + // Tally approve votes + for _, v := range results { + switch v.ID { + case ticketvote.VoteOptionIDApprove: + // Valid vote option + approvedVotes++ + case ticketvote.VoteOptionIDReject: + // Valid vote option + default: + // Invalid vote option + e := fmt.Sprintf("invalid vote option id found: %v", v.ID) + panic(e) + } + } + + // Check tally against thresholds + var approved bool + switch { + case total < quorum: + // Quorum not met + approved = false + case approvedVotes < pass: + // Pass percentage not met + approved = false + default: + // Vote was approved + approved = true + } + + return approved +} + +func (p *ticketVotePlugin) cmdSummary(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdSummaries: %v %x %v", treeID, token, payload) + + // Get best block. This cmd does not write any data so we do not + // have to use the safe best block. + bb, err := p.bestBlockUnsafe() + if err != nil { + return "", fmt.Errorf("bestBlockUnsafe: %v", err) + } + + // Get summary + s, err := p.summary(treeID, token, bb) + if err != nil { + return "", fmt.Errorf("summary: %v", err) + } + + // Prepare reply + sr := ticketvote.SummaryReply{ + Summary: *s, + BestBlock: bb, + } + reply, err := json.Marshal(sr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func convertInventoryReply(v inventory) ticketvote.InventoryReply { + // Started needs to be converted from a map to a slice where the + // slice is sorted by end block height from smallest to largest. + tokensByHeight := make(map[uint32][]string, len(v.started)) + for token, height := range v.started { + tokens, ok := tokensByHeight[height] + if !ok { + tokens = make([]string, 0, len(v.started)) + } + tokens = append(tokens, token) + tokensByHeight[height] = tokens + } + sortedHeights := make([]uint32, 0, len(tokensByHeight)) + for k := range tokensByHeight { + sortedHeights = append(sortedHeights, k) + } + // Sort smallest to largest block height + sort.SliceStable(sortedHeights, func(i, j int) bool { + return sortedHeights[i] < sortedHeights[j] + }) + started := make([]string, 0, len(v.started)) + for _, height := range sortedHeights { + tokens := tokensByHeight[height] + started = append(started, tokens...) + } + return ticketvote.InventoryReply{ + Unauthorized: v.unauthorized, + Authorized: v.authorized, + Started: started, + Finished: v.finished, + BestBlock: v.bestBlock, + } +} + +func (p *ticketVotePlugin) cmdInventory() (string, error) { + log.Tracef("cmdInventory") + + // Get best block. This command does not write any data so we can + // use the unsafe best block. + bb, err := p.bestBlockUnsafe() + if err != nil { + return "", fmt.Errorf("bestBlockUnsafe: %v", err) + } + + // Get the inventory + inv, err := p.inventory(bb) + if err != nil { + return "", fmt.Errorf("inventory: %v", err) + } + ir := convertInventoryReply(*inv) + + // Prepare reply + reply, err := json.Marshal(ir) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdTimestamps: %v %x %v", treeID, token, payload) + + // Decode payload + var t ticketvote.Timestamps + err := json.Unmarshal([]byte(payload), &t) + if err != nil { + return "", err + } + + // Get authorization timestamps + digests, err := p.tlog.DigestsByDataType(treeID, dataTypeAuthDetails) + if err != nil { + return "", fmt.Errorf("DigestByDataType %v %v: %v", + treeID, dataTypeAuthDetails, err) + } + + auths := make([]ticketvote.Timestamp, 0, len(digests)) + for _, v := range digests { + ts, err := p.timestamp(treeID, v) + if err != nil { + return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) + } + auths = append(auths, *ts) + } + + // Get vote details timestamp. There should never be more than one + // vote details. + digests, err = p.tlog.DigestsByDataType(treeID, dataTypeVoteDetails) + if err != nil { + return "", fmt.Errorf("DigestsByDataType %v %v: %v", + treeID, dataTypeVoteDetails, err) + } + if len(digests) > 1 { + return "", fmt.Errorf("invalid vote details count: got %v, want 1", + len(digests)) + } + + var details ticketvote.Timestamp + for _, v := range digests { + ts, err := p.timestamp(treeID, v) + if err != nil { + return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) + } + details = *ts + } + + // Get cast vote timestamps + digests, err = p.tlog.DigestsByDataType(treeID, dataTypeCastVoteDetails) + if err != nil { + return "", fmt.Errorf("digestsByKeyPrefix %v %v: %v", + treeID, dataTypeVoteDetails, err) + } + + votes := make(map[string]ticketvote.Timestamp, len(digests)) + for _, v := range digests { + ts, err := p.timestamp(treeID, v) + if err != nil { + return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) + } + + var cv ticketvote.CastVoteDetails + err = json.Unmarshal([]byte(ts.Data), &cv) + if err != nil { + return "", err + } + + votes[cv.Ticket] = *ts + } + + // Prepare reply + tr := ticketvote.TimestampsReply{ + Auths: auths, + Details: details, + Votes: votes, + } + reply, err := json.Marshal(tr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func convertAuthDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.AuthDetails, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorAuthDetails { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorAuthDetails) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, fmt.Errorf("decode digest: %v", err) + } + if !bytes.Equal(util.Digest(b), digest) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), digest) + } + var ad ticketvote.AuthDetails + err = json.Unmarshal(b, &ad) + if err != nil { + return nil, fmt.Errorf("unmarshal AuthDetails: %v", err) + } + + return &ad, nil +} + +func convertVoteDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.VoteDetails, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorVoteDetails { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorVoteDetails) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, fmt.Errorf("decode digest: %v", err) + } + if !bytes.Equal(util.Digest(b), digest) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), digest) + } + var vd ticketvote.VoteDetails + err = json.Unmarshal(b, &vd) + if err != nil { + return nil, fmt.Errorf("unmarshal VoteDetails: %v", err) + } + + return &vd, nil +} + +func convertCastVoteDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.CastVoteDetails, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorCastVoteDetails { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorCastVoteDetails) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, fmt.Errorf("decode digest: %v", err) + } + if !bytes.Equal(util.Digest(b), digest) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), digest) + } + var cv ticketvote.CastVoteDetails + err = json.Unmarshal(b, &cv) + if err != nil { + return nil, fmt.Errorf("unmarshal CastVoteDetails: %v", err) + } + + return &cv, nil +} + +func convertStartRunoffFromBlobEntry(be store.BlobEntry) (*startRunoffRecord, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorStartRunoff { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorStartRunoff) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, fmt.Errorf("decode digest: %v", err) + } + if !bytes.Equal(util.Digest(b), digest) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), digest) + } + var srr startRunoffRecord + err = json.Unmarshal(b, &srr) + if err != nil { + return nil, fmt.Errorf("unmarshal StartRunoffRecord: %v", err) + } + + return &srr, nil +} + +func convertBlobEntryFromAuthDetails(ad ticketvote.AuthDetails) (*store.BlobEntry, error) { + data, err := json.Marshal(ad) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorAuthDetails, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertBlobEntryFromVoteDetails(vd ticketvote.VoteDetails) (*store.BlobEntry, error) { + data, err := json.Marshal(vd) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorVoteDetails, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertBlobEntryFromCastVoteDetails(cv ticketvote.CastVoteDetails) (*store.BlobEntry, error) { + data, err := json.Marshal(cv) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorCastVoteDetails, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertBlobEntryFromStartRunoff(srr startRunoffRecord) (*store.BlobEntry, error) { + data, err := json.Marshal(srr) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorStartRunoff, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go b/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go new file mode 100644 index 000000000..e4c6b199a --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go @@ -0,0 +1,270 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ticketvote + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" + "github.com/decred/politeia/politeiad/plugins/ticketvote" +) + +// decodeVoteMetadata decodes and returns the VoteMetadata from the +// provided backend files. If a VoteMetadata is not found, nil is returned. +func decodeVoteMetadata(files []backend.File) (*ticketvote.VoteMetadata, error) { + var voteMD *ticketvote.VoteMetadata + for _, v := range files { + if v.Name == ticketvote.FileNameVoteMetadata { + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return nil, err + } + var m ticketvote.VoteMetadata + err = json.Unmarshal(b, &m) + if err != nil { + return nil, err + } + voteMD = &m + break + } + } + return voteMD, nil +} + +func (p *ticketVotePlugin) linkByVerify(linkBy int64) error { + if linkBy == 0 { + // LinkBy as not been set + return nil + } + min := time.Now().Unix() + p.linkByPeriodMin + max := time.Now().Unix() + p.linkByPeriodMax + switch { + case linkBy < min: + e := fmt.Sprintf("linkby %v is less than min required of %v", + linkBy, min) + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeLinkByInvalid), + ErrorContext: e, + } + case linkBy > max: + e := fmt.Sprintf("linkby %v is more than max allowed of %v", + linkBy, max) + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeLinkByInvalid), + ErrorContext: e, + } + } + return nil +} + +func (p *ticketVotePlugin) linkToVerify(linkTo string) error { + // LinkTo must be a public record that is the parent of a runoff + // vote, i.e. has the VoteMetadata.LinkBy field set. + token, err := tokenDecode(linkTo) + if err != nil { + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), + ErrorContext: "invalid hex", + } + } + r, err := p.backend.GetVetted(token, "") + if err != nil { + if errors.Is(err, backend.ErrRecordNotFound) { + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), + ErrorContext: "record not found", + } + } + return err + } + if r.RecordMetadata.Status != backend.MDStatusCensored { + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), + ErrorContext: "record is censored", + } + } + parentVM, err := decodeVoteMetadata(r.Files) + if err != nil { + return err + } + if parentVM == nil || parentVM.LinkBy == 0 { + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), + ErrorContext: "record not a runoff vote parent", + } + } + if time.Now().Unix() > parentVM.LinkBy { + // Linkby deadline has expired. New links are not allowed. + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), + ErrorContext: "parent record linkby deadline has expired", + } + } + return nil +} + +func (p *ticketVotePlugin) voteMetadataVerify(vm ticketvote.VoteMetadata) error { + switch { + case vm.LinkBy == 0 && vm.LinkTo == "": + // Vote metadata is empty + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeVoteMetadataInvalid), + ErrorContext: "md is empty", + } + + case vm.LinkBy != 0 && vm.LinkTo != "": + // LinkBy and LinkTo cannot both be set + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeVoteMetadataInvalid), + ErrorContext: "cannot set both linkby and linkto", + } + + case vm.LinkBy != 0: + err := p.linkByVerify(vm.LinkBy) + if err != nil { + return err + } + + case vm.LinkTo != "": + err := p.linkToVerify(vm.LinkTo) + if err != nil { + return err + } + } + + return nil +} + +func (p *ticketVotePlugin) hookNewRecordPre(payload string) error { + var nr plugins.HookNewRecordPre + err := json.Unmarshal([]byte(payload), &nr) + if err != nil { + return err + } + + // Verify the vote metadata if the record contains one + vm, err := decodeVoteMetadata(nr.Files) + if err != nil { + return err + } + if vm != nil { + err = p.voteMetadataVerify(*vm) + if err != nil { + return err + } + } + + return nil +} + +func (p *ticketVotePlugin) hookEditRecordPre(payload string) error { + var er plugins.HookEditRecord + err := json.Unmarshal([]byte(payload), &er) + if err != nil { + return err + } + + // The LinkTo field is not allowed to change once the record has + // become public. If this is a vetted record, verify that any + // previously set LinkTo has not changed. + if er.State == plugins.RecordStateVetted { + var ( + oldLinkTo string + newLinkTo string + ) + vm, err := decodeVoteMetadata(er.Current.Files) + if err != nil { + return err + } + if vm != nil { + oldLinkTo = vm.LinkTo + } + vm, err = decodeVoteMetadata(er.FilesAdd) + if err != nil { + return err + } + if vm != nil { + newLinkTo = vm.LinkTo + } + if newLinkTo != oldLinkTo { + e := fmt.Sprintf("linkto cannot change on vetted record: "+ + "got '%v', want '%v'", newLinkTo, oldLinkTo) + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), + ErrorContext: e, + } + } + } + + // Verify LinkBy if one was included. The VoteMetadata is optional + // so the record may not contain one. + vm, err := decodeVoteMetadata(er.FilesAdd) + if err != nil { + return err + } + if vm != nil { + err = p.linkByVerify(vm.LinkBy) + if err != nil { + return err + } + } + + return nil +} + +func (p *ticketVotePlugin) hookSetRecordStatusPost(payload string) error { + var srs plugins.HookSetRecordStatus + err := json.Unmarshal([]byte(payload), &srs) + if err != nil { + return err + } + + // Check if the LinkTo has been set + vm, err := decodeVoteMetadata(srs.Current.Files) + if err != nil { + return err + } + if vm != nil && vm.LinkTo != "" { + // LinkTo has been set. Check if the status change requires the + // linked from list of the linked record to be updated. + var ( + parentToken = vm.LinkTo + childToken = srs.RecordMetadata.Token + ) + switch srs.RecordMetadata.Status { + case backend.MDStatusVetted: + // Record has been made public. Add child token to parent's + // linked from list. + err := p.linkedFromAdd(parentToken, childToken) + if err != nil { + return fmt.Errorf("linkedFromAdd: %v", err) + } + case backend.MDStatusCensored: + // Record has been censored. Delete child token from parent's + // linked from list. + err := p.linkedFromDel(parentToken, childToken) + if err != nil { + return fmt.Errorf("linkedFromDel: %v", err) + } + } + } + + return nil +} diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go b/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go index 12af1cef6..e8f2ec746 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go @@ -10,11 +10,11 @@ import ( "github.com/decred/politeia/politeiad/plugins/ticketvote" ) -// voteInventory contains the record inventory categorized by vote status. The +// inventory contains the record inventory categorized by vote status. The // authorized and started lists are updated in real-time since ticket vote // plugin commands initiate those actions. The unauthorized and finished lists // are lazy loaded since those lists depends on external state. -type voteInventory struct { +type inventory struct { unauthorized []string // Unauthorized tokens authorized []string // Authorized tokens started map[string]uint32 // [token]endHeight @@ -152,7 +152,7 @@ func (p *ticketVotePlugin) inventorySetToStarted(token string, t ticketvote.Vote log.Debugf("ticketvote: added to started inv: %v", token) } -func (p *ticketVotePlugin) inventory(bestBlock uint32) (*voteInventory, error) { +func (p *ticketVotePlugin) inventory(bestBlock uint32) (*inventory, error) { p.Lock() defer p.Unlock() @@ -259,7 +259,7 @@ reply: started[k] = v } - return &voteInventory{ + return &inventory{ unauthorized: unauthorized, authorized: authorized, started: started, diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/summary.go b/politeiad/backend/tlogbe/plugins/ticketvote/summary.go new file mode 100644 index 000000000..a799bbc20 --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/ticketvote/summary.go @@ -0,0 +1,91 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ticketvote + +import ( + "encoding/json" + "errors" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/decred/politeia/politeiad/plugins/ticketvote" + "github.com/decred/politeia/util" +) + +const ( + // Filenames of cached data saved to the plugin data dir. Brackets + // are used to indicate a variable that should be replaced in the + // filename. + filenameSummary = "{tokenprefix}-summary.json" +) + +// cachedSummaryPath accepts both full tokens and token prefixes, however it +// always uses the token prefix when generatig the path. +func (p *ticketVotePlugin) cachedSummaryPath(token string) (string, error) { + // Use token prefix + t, err := tokenDecodeAnyLength(token) + if err != nil { + return "", err + } + token = util.TokenPrefix(t) + fn := strings.Replace(filenameSummary, "{tokenprefix}", token, 1) + return filepath.Join(p.dataDir, fn), nil +} + +var ( + errSummaryNotFound = errors.New("summary not found") +) + +func (p *ticketVotePlugin) cachedSummary(token string) (*ticketvote.VoteSummary, error) { + p.Lock() + defer p.Unlock() + + fp, err := p.cachedSummaryPath(token) + if err != nil { + return nil, err + } + b, err := ioutil.ReadFile(fp) + if err != nil { + var e *os.PathError + if errors.As(err, &e) && !os.IsExist(err) { + // File does't exist + return nil, errSummaryNotFound + } + return nil, err + } + + var vs ticketvote.VoteSummary + err = json.Unmarshal(b, &vs) + if err != nil { + return nil, err + } + + return &vs, nil +} + +func (p *ticketVotePlugin) cachedSummarySave(token string, vs ticketvote.VoteSummary) error { + b, err := json.Marshal(vs) + if err != nil { + return err + } + + p.Lock() + defer p.Unlock() + + fp, err := p.cachedSummaryPath(token) + if err != nil { + return err + } + err = ioutil.WriteFile(fp, b, 0664) + if err != nil { + return err + } + + log.Debugf("Saved votes summary: %v", token) + + return nil +} diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go index cca53ae76..8e5122700 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go @@ -5,57 +5,21 @@ package ticketvote import ( - "bytes" - "encoding/base64" "encoding/hex" "encoding/json" - "errors" "fmt" - "io/ioutil" "os" "path/filepath" - "sort" "strconv" - "strings" "sync" - "time" - "github.com/decred/dcrd/chaincfg/chainhash" "github.com/decred/dcrd/chaincfg/v3" - "github.com/decred/dcrd/dcrec/secp256k1/v3/ecdsa" - "github.com/decred/dcrd/dcrutil/v3" - "github.com/decred/dcrd/wire" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" - "github.com/decred/politeia/politeiad/backend/tlogbe/store" "github.com/decred/politeia/politeiad/backend/tlogbe/tlogclient" "github.com/decred/politeia/politeiad/plugins/dcrdata" "github.com/decred/politeia/politeiad/plugins/ticketvote" - "github.com/decred/politeia/util" -) - -const ( - // Plugin setting IDs - pluginSettingVoteDurationMin = "votedurationmin" - pluginSettingVoteDurationMax = "votedurationmax" - - // Filenames of cached data saved to the plugin data dir. Brackets - // are used to indicate a variable that should be replaced in the - // filename. - filenameSummary = "{tokenprefix}-summary.json" - - // Blob entry data descriptors - dataDescriptorAuthDetails = "authdetails_v1" - dataDescriptorVoteDetails = "votedetails_v1" - dataDescriptorCastVoteDetails = "castvotedetails_v1" - dataDescriptorStartRunoff = "startrunoff_v1" - - // Data types - dataTypeAuthDetails = "authdetails" - dataTypeVoteDetails = "votedetails" - dataTypeCastVoteDetails = "castvotedetails" - dataTypeStartRunoff = "startrunoff" ) var ( @@ -91,7 +55,7 @@ type ticketVotePlugin struct { // inv contains the record inventory categorized by vote status. // The inventory will only contain public, non-abandoned records. // This cache is built on startup. - inv voteInventory + inv inventory // votes contains the cast votes of ongoing record votes. This // cache is built on startup and record entries are removed once @@ -106,17 +70,6 @@ type ticketVotePlugin struct { mutexes map[string]*sync.Mutex // [string]mutex } -// tokenDecode decodes a record token and only accepts full length tokens. -func tokenDecode(token string) ([]byte, error) { - return util.TokenDecode(util.TokenTypeTlog, token) -} - -// tokenDecode decodes a record token and accepts both full length tokens and -// token prefixes. -func tokenDecodeAnyLength(token string) ([]byte, error) { - return util.TokenDecodeAnyLength(util.TokenTypeTlog, token) -} - // mutex returns the mutex for the specified record. func (p *ticketVotePlugin) mutex(token []byte) *sync.Mutex { p.Lock() @@ -133,2821 +86,6 @@ func (p *ticketVotePlugin) mutex(token []byte) *sync.Mutex { return m } -func (p *ticketVotePlugin) cachedVotes(token []byte) map[string]string { - p.Lock() - defer p.Unlock() - - // Return a copy of the map - cv, ok := p.votes[hex.EncodeToString(token)] - if !ok { - return map[string]string{} - } - c := make(map[string]string, len(cv)) - for k, v := range cv { - c[k] = v - } - - return c -} - -func (p *ticketVotePlugin) cachedVotesSet(token, ticket, voteBit string) { - p.Lock() - defer p.Unlock() - - _, ok := p.votes[token] - if !ok { - p.votes[token] = make(map[string]string, 40960) // Ticket pool size - } - - p.votes[token][ticket] = voteBit - - log.Debugf("Added vote to cache: %v %v %v", - token, ticket, voteBit) -} - -func (p *ticketVotePlugin) cachedVotesDel(token string) { - p.Lock() - defer p.Unlock() - - delete(p.votes, token) - - log.Debugf("Deleted votes cache: %v", token) -} - -// cachedSummaryPath accepts both full tokens and token prefixes, however it -// always uses the token prefix when generatig the path. -func (p *ticketVotePlugin) cachedSummaryPath(token string) (string, error) { - // Use token prefix - t, err := tokenDecodeAnyLength(token) - if err != nil { - return "", err - } - token = util.TokenPrefix(t) - fn := strings.Replace(filenameSummary, "{tokenprefix}", token, 1) - return filepath.Join(p.dataDir, fn), nil -} - -var ( - errSummaryNotFound = errors.New("summary not found") -) - -func (p *ticketVotePlugin) cachedSummary(token string) (*ticketvote.VoteSummary, error) { - p.Lock() - defer p.Unlock() - - fp, err := p.cachedSummaryPath(token) - if err != nil { - return nil, err - } - b, err := ioutil.ReadFile(fp) - if err != nil { - var e *os.PathError - if errors.As(err, &e) && !os.IsExist(err) { - // File does't exist - return nil, errSummaryNotFound - } - return nil, err - } - - var vs ticketvote.VoteSummary - err = json.Unmarshal(b, &vs) - if err != nil { - return nil, err - } - - return &vs, nil -} - -func (p *ticketVotePlugin) cachedSummarySave(token string, vs ticketvote.VoteSummary) error { - b, err := json.Marshal(vs) - if err != nil { - return err - } - - p.Lock() - defer p.Unlock() - - fp, err := p.cachedSummaryPath(token) - if err != nil { - return err - } - err = ioutil.WriteFile(fp, b, 0664) - if err != nil { - return err - } - - log.Debugf("Saved votes summary: %v", token) - - return nil -} - -func convertSignatureError(err error) backend.PluginUserError { - var e util.SignatureError - var s ticketvote.ErrorCodeT - if errors.As(err, &e) { - switch e.ErrorCode { - case util.ErrorStatusPublicKeyInvalid: - s = ticketvote.ErrorCodePublicKeyInvalid - case util.ErrorStatusSignatureInvalid: - s = ticketvote.ErrorCodeSignatureInvalid - } - } - return backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(s), - ErrorContext: e.ErrorContext, - } -} - -func convertAuthDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.AuthDetails, error) { - // Decode and validate data hint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorAuthDetails { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorAuthDetails) - } - - // Decode data - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - digest, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, fmt.Errorf("decode digest: %v", err) - } - if !bytes.Equal(util.Digest(b), digest) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), digest) - } - var ad ticketvote.AuthDetails - err = json.Unmarshal(b, &ad) - if err != nil { - return nil, fmt.Errorf("unmarshal AuthDetails: %v", err) - } - - return &ad, nil -} - -func convertVoteDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.VoteDetails, error) { - // Decode and validate data hint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorVoteDetails { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorVoteDetails) - } - - // Decode data - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - digest, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, fmt.Errorf("decode digest: %v", err) - } - if !bytes.Equal(util.Digest(b), digest) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), digest) - } - var vd ticketvote.VoteDetails - err = json.Unmarshal(b, &vd) - if err != nil { - return nil, fmt.Errorf("unmarshal VoteDetails: %v", err) - } - - return &vd, nil -} - -func convertCastVoteDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.CastVoteDetails, error) { - // Decode and validate data hint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorCastVoteDetails { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorCastVoteDetails) - } - - // Decode data - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - digest, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, fmt.Errorf("decode digest: %v", err) - } - if !bytes.Equal(util.Digest(b), digest) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), digest) - } - var cv ticketvote.CastVoteDetails - err = json.Unmarshal(b, &cv) - if err != nil { - return nil, fmt.Errorf("unmarshal CastVoteDetails: %v", err) - } - - return &cv, nil -} - -func convertStartRunoffFromBlobEntry(be store.BlobEntry) (*startRunoffRecord, error) { - // Decode and validate data hint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorStartRunoff { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorStartRunoff) - } - - // Decode data - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - digest, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, fmt.Errorf("decode digest: %v", err) - } - if !bytes.Equal(util.Digest(b), digest) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), digest) - } - var srr startRunoffRecord - err = json.Unmarshal(b, &srr) - if err != nil { - return nil, fmt.Errorf("unmarshal StartRunoffRecord: %v", err) - } - - return &srr, nil -} - -func convertBlobEntryFromAuthDetails(ad ticketvote.AuthDetails) (*store.BlobEntry, error) { - data, err := json.Marshal(ad) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorAuthDetails, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func convertBlobEntryFromVoteDetails(vd ticketvote.VoteDetails) (*store.BlobEntry, error) { - data, err := json.Marshal(vd) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorVoteDetails, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func convertBlobEntryFromCastVoteDetails(cv ticketvote.CastVoteDetails) (*store.BlobEntry, error) { - data, err := json.Marshal(cv) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorCastVoteDetails, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func convertBlobEntryFromStartRunoff(srr startRunoffRecord) (*store.BlobEntry, error) { - data, err := json.Marshal(srr) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorStartRunoff, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func (p *ticketVotePlugin) authSave(treeID int64, ad ticketvote.AuthDetails) error { - // Prepare blob - be, err := convertBlobEntryFromAuthDetails(ad) - if err != nil { - return err - } - - // Save blob - return p.tlog.BlobSave(treeID, dataTypeAuthDetails, *be) -} - -func (p *ticketVotePlugin) auths(treeID int64) ([]ticketvote.AuthDetails, error) { - // Retrieve blobs - blobs, err := p.tlog.BlobsByDataType(treeID, dataTypeAuthDetails) - if err != nil { - return nil, err - } - - // Decode blobs - auths := make([]ticketvote.AuthDetails, 0, len(blobs)) - for _, v := range blobs { - a, err := convertAuthDetailsFromBlobEntry(v) - if err != nil { - return nil, err - } - auths = append(auths, *a) - } - - // Sanity check. They should already be sorted from oldest to - // newest. - sort.SliceStable(auths, func(i, j int) bool { - return auths[i].Timestamp < auths[j].Timestamp - }) - - return auths, nil -} - -func (p *ticketVotePlugin) voteDetailsSave(treeID int64, vd ticketvote.VoteDetails) error { - // Prepare blob - be, err := convertBlobEntryFromVoteDetails(vd) - if err != nil { - return err - } - - // Save blob - return p.tlog.BlobSave(treeID, dataTypeVoteDetails, *be) -} - -func (p *ticketVotePlugin) voteDetails(treeID int64) (*ticketvote.VoteDetails, error) { - // Retrieve blobs - blobs, err := p.tlog.BlobsByDataType(treeID, dataTypeVoteDetails) - if err != nil { - return nil, err - } - switch len(blobs) { - case 0: - // A vote details does not exist - return nil, nil - case 1: - // A vote details exists; continue - default: - // This should not happen. There should only ever be a max of - // one vote details. - return nil, fmt.Errorf("multiple vote details found (%v) on %x", - len(blobs), treeID) - } - - // Decode blob - vd, err := convertVoteDetailsFromBlobEntry(blobs[0]) - if err != nil { - return nil, err - } - - return vd, nil -} - -func (p *ticketVotePlugin) castVoteSave(treeID int64, cv ticketvote.CastVoteDetails) error { - // Prepare blob - be, err := convertBlobEntryFromCastVoteDetails(cv) - if err != nil { - return err - } - - // Save blob - return p.tlog.BlobSave(treeID, dataTypeCastVoteDetails, *be) -} - -func (p *ticketVotePlugin) castVotes(treeID int64) ([]ticketvote.CastVoteDetails, error) { - // Retrieve blobs - blobs, err := p.tlog.BlobsByDataType(treeID, dataTypeCastVoteDetails) - if err != nil { - return nil, err - } - - // Decode blobs - votes := make([]ticketvote.CastVoteDetails, 0, len(blobs)) - for _, v := range blobs { - cv, err := convertCastVoteDetailsFromBlobEntry(v) - if err != nil { - return nil, err - } - votes = append(votes, *cv) - } - - // Sort by ticket hash - sort.SliceStable(votes, func(i, j int) bool { - return votes[i].Ticket < votes[j].Ticket - }) - - return votes, nil -} - -// summary returns the vote summary for a record. -func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) (*ticketvote.VoteSummary, error) { - // Check if the summary has been cached - s, err := p.cachedSummary(hex.EncodeToString(token)) - switch { - case errors.Is(err, errSummaryNotFound): - // Cached summary not found. Continue. - case err != nil: - // Some other error - return nil, fmt.Errorf("cachedSummary: %v", err) - default: - // Caches summary was found. Return it. - return s, nil - } - - // Summary has not been cached. Get it manually. - - // Assume vote is unauthorized. Only update the status when the - // appropriate record has been found that proves otherwise. - status := ticketvote.VoteStatusUnauthorized - - // Check if the vote has been authorized - auths, err := p.auths(treeID) - if err != nil { - return nil, fmt.Errorf("auths: %v", err) - } - if len(auths) > 0 { - lastAuth := auths[len(auths)-1] - switch ticketvote.AuthActionT(lastAuth.Action) { - case ticketvote.AuthActionAuthorize: - // Vote has been authorized; continue - status = ticketvote.VoteStatusAuthorized - case ticketvote.AuthActionRevoke: - // Vote authorization has been revoked. Its not possible for - // the vote to have been started. We can stop looking. - return &ticketvote.VoteSummary{ - Status: status, - Results: []ticketvote.VoteOptionResult{}, - }, nil - } - } - - // Check if the vote has been started - vd, err := p.voteDetails(treeID) - if err != nil { - return nil, fmt.Errorf("startDetails: %v", err) - } - if vd == nil { - // Vote has not been started yet - return &ticketvote.VoteSummary{ - Status: status, - Results: []ticketvote.VoteOptionResult{}, - }, nil - } - - // Vote has been started. Check if it is still in progress or has - // already ended. - if bestBlock < vd.EndBlockHeight { - status = ticketvote.VoteStatusStarted - } else { - status = ticketvote.VoteStatusFinished - } - - // Pull the cast votes from the cache and tally the results - votes := p.cachedVotes(token) - tally := make(map[string]int, len(vd.Params.Options)) - for _, voteBit := range votes { - tally[voteBit]++ - } - results := make([]ticketvote.VoteOptionResult, 0, len(vd.Params.Options)) - for _, v := range vd.Params.Options { - bit := strconv.FormatUint(v.Bit, 16) - results = append(results, ticketvote.VoteOptionResult{ - ID: v.ID, - Description: v.Description, - VoteBit: v.Bit, - Votes: uint64(tally[bit]), - }) - } - - // Prepare summary - summary := ticketvote.VoteSummary{ - Type: vd.Params.Type, - Status: status, - Duration: vd.Params.Duration, - StartBlockHeight: vd.StartBlockHeight, - StartBlockHash: vd.StartBlockHash, - EndBlockHeight: vd.EndBlockHeight, - EligibleTickets: uint32(len(vd.EligibleTickets)), - QuorumPercentage: vd.Params.QuorumPercentage, - PassPercentage: vd.Params.PassPercentage, - Results: results, - } - - // If the vote has not finished yet then we are done for now. - if status == ticketvote.VoteStatusStarted { - return &summary, nil - } - - // The vote has finished. We can calculate if the vote was approved - // for certain vote types and cache the results. - switch vd.Params.Type { - case ticketvote.VoteTypeStandard, ticketvote.VoteTypeRunoff: - // These vote types are strictly approve/reject votes so we can - // calculate the vote approval. Continue. - default: - // Nothing else to do for all other vote types - return &summary, nil - } - - // Calculate vote approval - approved := voteIsApproved(*vd, results) - - // If this is a standard vote then we can take the results as is. - // A runoff vote requires that we pull all other runoff vote - // submissions to determine if the vote actually passed. - // TODO make summary for runoff vote submissions - summary.Approved = approved - - // Cache the summary - err = p.cachedSummarySave(vd.Params.Token, summary) - if err != nil { - return nil, fmt.Errorf("cachedSummarySave %v: %v %v", - vd.Params.Token, err, summary) - } - - // Remove record from the votes cache now that a summary has been - // saved for it. - p.cachedVotesDel(vd.Params.Token) - - return &summary, nil -} - -func (p *ticketVotePlugin) summaryByToken(token []byte) (*ticketvote.VoteSummary, error) { - reply, err := p.backend.VettedPluginCmd(token, ticketvote.ID, - ticketvote.CmdSummary, "") - if err != nil { - return nil, fmt.Errorf("VettedPluginCmd %x %v %v: %v", - token, ticketvote.ID, ticketvote.CmdSummary, err) - } - var sr ticketvote.SummaryReply - err = json.Unmarshal([]byte(reply), &sr) - if err != nil { - return nil, err - } - return &sr.Summary, nil -} - -func (p *ticketVotePlugin) timestamp(treeID int64, digest []byte) (*ticketvote.Timestamp, error) { - t, err := p.tlog.Timestamp(treeID, digest) - if err != nil { - return nil, fmt.Errorf("timestamp %v %x: %v", treeID, digest, err) - } - - // Convert response - proofs := make([]ticketvote.Proof, 0, len(t.Proofs)) - for _, v := range t.Proofs { - proofs = append(proofs, ticketvote.Proof{ - Type: v.Type, - Digest: v.Digest, - MerkleRoot: v.MerkleRoot, - MerklePath: v.MerklePath, - ExtraData: v.ExtraData, - }) - } - return &ticketvote.Timestamp{ - Data: t.Data, - Digest: t.Digest, - TxID: t.TxID, - MerkleRoot: t.MerkleRoot, - Proofs: proofs, - }, nil -} - -// bestBlock fetches the best block from the dcrdata plugin and returns it. If -// the dcrdata connection is not active, an error will be returned. -func (p *ticketVotePlugin) bestBlock() (uint32, error) { - // Get best block - payload, err := json.Marshal(dcrdata.BestBlock{}) - if err != nil { - return 0, err - } - reply, err := p.backend.Plugin(dcrdata.ID, - dcrdata.CmdBestBlock, "", string(payload)) - if err != nil { - return 0, fmt.Errorf("Plugin %v %v: %v", - dcrdata.ID, dcrdata.CmdBestBlock, err) - } - - // Handle response - var bbr dcrdata.BestBlockReply - err = json.Unmarshal([]byte(reply), &bbr) - if err != nil { - return 0, err - } - if bbr.Status != dcrdata.StatusConnected { - // The dcrdata connection is down. The best block cannot be - // trusted as being accurate. - return 0, fmt.Errorf("dcrdata connection is down") - } - if bbr.Height == 0 { - return 0, fmt.Errorf("invalid best block height 0") - } - - return bbr.Height, nil -} - -// bestBlockUnsafe fetches the best block from the dcrdata plugin and returns -// it. If the dcrdata connection is not active, an error WILL NOT be returned. -// The dcrdata cached best block height will be returned even though it may be -// stale. Use bestBlock() if the caller requires a guarantee that the best -// block is not stale. -func (p *ticketVotePlugin) bestBlockUnsafe() (uint32, error) { - // Get best block - payload, err := json.Marshal(dcrdata.BestBlock{}) - if err != nil { - return 0, err - } - reply, err := p.backend.Plugin(dcrdata.ID, - dcrdata.CmdBestBlock, "", string(payload)) - if err != nil { - return 0, fmt.Errorf("Plugin %v %v: %v", - dcrdata.ID, dcrdata.CmdBestBlock, err) - } - - // Handle response - var bbr dcrdata.BestBlockReply - err = json.Unmarshal([]byte(reply), &bbr) - if err != nil { - return 0, err - } - if bbr.Height == 0 { - return 0, fmt.Errorf("invalid best block height 0") - } - - return bbr.Height, nil -} - -type commitmentAddr struct { - ticket string // Ticket hash - addr string // Commitment address - err error // Error if one occurred -} - -func (p *ticketVotePlugin) largestCommitmentAddrs(tickets []string) ([]commitmentAddr, error) { - // Get tx details - tt := dcrdata.TxsTrimmed{ - TxIDs: tickets, - } - payload, err := json.Marshal(tt) - if err != nil { - return nil, err - } - reply, err := p.backend.Plugin(dcrdata.ID, - dcrdata.CmdTxsTrimmed, "", string(payload)) - if err != nil { - return nil, fmt.Errorf("Plugin %v %v: %v", - dcrdata.ID, dcrdata.CmdTxsTrimmed, err) - } - var ttr dcrdata.TxsTrimmedReply - err = json.Unmarshal([]byte(reply), &ttr) - if err != nil { - return nil, err - } - - // Find the largest commitment address for each tx - addrs := make([]commitmentAddr, 0, len(ttr.Txs)) - for _, tx := range ttr.Txs { - var ( - bestAddr string // Addr with largest commitment amount - bestAmt float64 // Largest commitment amount - addrErr error // Error if one is encountered - ) - for _, vout := range tx.Vout { - scriptPubKey := vout.ScriptPubKeyDecoded - switch { - case scriptPubKey.CommitAmt == nil: - // No commitment amount; continue - case len(scriptPubKey.Addresses) == 0: - // No commitment address; continue - case *scriptPubKey.CommitAmt > bestAmt: - // New largest commitment address found - bestAddr = scriptPubKey.Addresses[0] - bestAmt = *scriptPubKey.CommitAmt - } - } - if bestAddr == "" || bestAmt == 0.0 { - addrErr = fmt.Errorf("no largest commitment address found") - } - - // Store result - addrs = append(addrs, commitmentAddr{ - ticket: tx.TxID, - addr: bestAddr, - err: addrErr, - }) - } - - return addrs, nil -} - -// startReply fetches all required data and returns a StartReply. -func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, error) { - // Get the best block height - bb, err := p.bestBlock() - if err != nil { - return nil, fmt.Errorf("bestBlock: %v", err) - } - - // Find the snapshot height. Subtract the ticket maturity from the - // block height to get into unforkable territory. - ticketMaturity := uint32(p.activeNetParams.TicketMaturity) - snapshotHeight := bb - ticketMaturity - - // Fetch the block details for the snapshot height. We need the - // block hash in order to fetch the ticket pool snapshot. - bd := dcrdata.BlockDetails{ - Height: snapshotHeight, - } - payload, err := json.Marshal(bd) - if err != nil { - return nil, err - } - reply, err := p.backend.Plugin(dcrdata.ID, - dcrdata.CmdBlockDetails, "", string(payload)) - if err != nil { - return nil, fmt.Errorf("Plugin %v %v: %v", - dcrdata.ID, dcrdata.CmdBlockDetails, err) - } - var bdr dcrdata.BlockDetailsReply - err = json.Unmarshal([]byte(reply), &bdr) - if err != nil { - return nil, err - } - if bdr.Block.Hash == "" { - return nil, fmt.Errorf("invalid block hash for height %v", snapshotHeight) - } - snapshotHash := bdr.Block.Hash - - // Fetch the ticket pool snapshot - tp := dcrdata.TicketPool{ - BlockHash: snapshotHash, - } - payload, err = json.Marshal(tp) - if err != nil { - return nil, err - } - reply, err = p.backend.Plugin(dcrdata.ID, - dcrdata.CmdTicketPool, "", string(payload)) - if err != nil { - return nil, fmt.Errorf("Plugin %v %v: %v", - dcrdata.ID, dcrdata.CmdTicketPool, err) - } - var tpr dcrdata.TicketPoolReply - err = json.Unmarshal([]byte(reply), &tpr) - if err != nil { - return nil, err - } - if len(tpr.Tickets) == 0 { - return nil, fmt.Errorf("no tickets found for block %v %v", - snapshotHeight, snapshotHash) - } - - // The start block height has the ticket maturity subtracted from - // it to prevent forking issues. This means we the vote starts in - // the past. The ticket maturity needs to be added to the end block - // height to correct for this. - endBlockHeight := snapshotHeight + duration + ticketMaturity - - return &ticketvote.StartReply{ - StartBlockHeight: snapshotHeight, - StartBlockHash: snapshotHash, - EndBlockHeight: endBlockHeight, - EligibleTickets: tpr.Tickets, - }, nil -} - -// voteMessageVerify verifies a cast vote message is properly signed. Copied -// from: github.com/decred/dcrd/blob/0fc55252f912756c23e641839b1001c21442c38a/rpcserver.go#L5605 -func (p *ticketVotePlugin) voteMessageVerify(address, message, signature string) (bool, error) { - // Decode the provided address. - addr, err := dcrutil.DecodeAddress(address, p.activeNetParams) - if err != nil { - return false, fmt.Errorf("Could not decode address: %v", - err) - } - - // Only P2PKH addresses are valid for signing. - if _, ok := addr.(*dcrutil.AddressPubKeyHash); !ok { - return false, fmt.Errorf("Address is not a pay-to-pubkey-hash "+ - "address: %v", address) - } - - // Decode base64 signature. - sig, err := base64.StdEncoding.DecodeString(signature) - if err != nil { - return false, fmt.Errorf("Malformed base64 encoding: %v", err) - } - - // Validate the signature - this just shows that it was valid at all. - // we will compare it with the key next. - var buf bytes.Buffer - wire.WriteVarString(&buf, 0, "Decred Signed Message:\n") - wire.WriteVarString(&buf, 0, message) - expectedMessageHash := chainhash.HashB(buf.Bytes()) - pk, wasCompressed, err := ecdsa.RecoverCompact(sig, - expectedMessageHash) - if err != nil { - // Mirror Bitcoin Core behavior, which treats error in - // RecoverCompact as invalid signature. - return false, nil - } - - // Reconstruct the pubkey hash. - dcrPK := pk - var serializedPK []byte - if wasCompressed { - serializedPK = dcrPK.SerializeCompressed() - } else { - serializedPK = dcrPK.SerializeUncompressed() - } - a, err := dcrutil.NewAddressSecpPubKey(serializedPK, p.activeNetParams) - if err != nil { - // Again mirror Bitcoin Core behavior, which treats error in - // public key reconstruction as invalid signature. - return false, nil - } - - // Return boolean if addresses match. - return a.Address() == address, nil -} - -func (p *ticketVotePlugin) castVoteSignatureVerify(cv ticketvote.CastVote, addr string) error { - msg := cv.Token + cv.Ticket + cv.VoteBit - - // Convert hex signature to base64. The voteMessageVerify function - // expects base64. - b, err := hex.DecodeString(cv.Signature) - if err != nil { - return fmt.Errorf("invalid hex") - } - sig := base64.StdEncoding.EncodeToString(b) - - // Verify message - validated, err := p.voteMessageVerify(addr, msg, sig) - if err != nil { - return err - } - if !validated { - return fmt.Errorf("could not verify message") - } - - return nil -} - -func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdAuthorize: %v %x %v", treeID, token, payload) - - // Decode payload - var a ticketvote.Authorize - err := json.Unmarshal([]byte(payload), &a) - if err != nil { - return "", err - } - - // Verify token - t, err := tokenDecode(a.Token) - if err != nil { - return "", backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), - } - } - if !bytes.Equal(t, token) { - e := fmt.Sprintf("plugin token does not match route token: "+ - "got %x, want %x", t, token) - return "", backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), - ErrorContext: e, - } - } - - // Verify signature - version := strconv.FormatUint(uint64(a.Version), 10) - msg := a.Token + version + string(a.Action) - err = util.VerifySignature(a.Signature, a.PublicKey, msg) - if err != nil { - return "", convertSignatureError(err) - } - - // Verify action - switch a.Action { - case ticketvote.AuthActionAuthorize: - // This is allowed - case ticketvote.AuthActionRevoke: - // This is allowed - default: - e := fmt.Sprintf("%v not a valid action", a.Action) - return "", backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), - ErrorContext: e, - } - } - - // The previous authorize votes must be retrieved to validate the - // new autorize vote. The lock must be held for the remainder of - // this function. - m := p.mutex(token) - m.Lock() - defer m.Unlock() - - // Get any previous authorizations to verify that the new action - // is allowed based on the previous action. - auths, err := p.auths(treeID) - if err != nil { - return "", err - } - var prevAction ticketvote.AuthActionT - if len(auths) > 0 { - prevAction = ticketvote.AuthActionT(auths[len(auths)-1].Action) - } - switch { - case len(auths) == 0: - // No previous actions. New action must be an authorize. - if a.Action != ticketvote.AuthActionAuthorize { - return "", backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), - ErrorContext: "no prev action; action must be authorize", - } - } - case prevAction == ticketvote.AuthActionAuthorize && - a.Action != ticketvote.AuthActionRevoke: - // Previous action was a authorize. This action must be revoke. - return "", backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), - ErrorContext: "prev action was authorize", - } - case prevAction == ticketvote.AuthActionRevoke && - a.Action != ticketvote.AuthActionAuthorize: - // Previous action was a revoke. This action must be authorize. - return "", backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), - ErrorContext: "prev action was revoke", - } - } - - // Prepare authorize vote - receipt := p.identity.SignMessage([]byte(a.Signature)) - auth := ticketvote.AuthDetails{ - Token: a.Token, - Version: a.Version, - Action: string(a.Action), - PublicKey: a.PublicKey, - Signature: a.Signature, - Timestamp: time.Now().Unix(), - Receipt: hex.EncodeToString(receipt[:]), - } - - // Save authorize vote - err = p.authSave(treeID, auth) - if err != nil { - return "", err - } - - // Update inventory - switch a.Action { - case ticketvote.AuthActionAuthorize: - p.inventorySetToAuthorized(a.Token) - case ticketvote.AuthActionRevoke: - p.inventorySetToUnauthorized(a.Token) - default: - // Should not happen - e := fmt.Sprintf("invalid authorize action: %v", a.Action) - panic(e) - } - - // Prepare reply - ar := ticketvote.AuthorizeReply{ - Timestamp: auth.Timestamp, - Receipt: auth.Receipt, - } - reply, err := json.Marshal(ar) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func voteBitVerify(options []ticketvote.VoteOption, mask, bit uint64) error { - if len(options) == 0 { - return fmt.Errorf("no vote options found") - } - if bit == 0 { - return fmt.Errorf("invalid bit 0x%x", bit) - } - - // Verify bit is included in mask - if mask&bit != bit { - return fmt.Errorf("invalid mask 0x%x bit 0x%x", mask, bit) - } - - // Verify bit is included in vote options - for _, v := range options { - if v.Bit == bit { - // Bit matches one of the options. We're done. - return nil - } - } - - return fmt.Errorf("bit 0x%x not found in vote options", bit) -} - -// TODO test this function -func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationMax uint32) error { - // Verify vote type - switch vote.Type { - case ticketvote.VoteTypeStandard: - // This is allowed - case ticketvote.VoteTypeRunoff: - // This is allowed - default: - e := fmt.Sprintf("invalid type %v", vote.Type) - return backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), - ErrorContext: e, - } - } - - // Verify vote params - switch { - case vote.Duration > voteDurationMax: - e := fmt.Sprintf("duration %v exceeds max duration %v", - vote.Duration, voteDurationMax) - return backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), - ErrorContext: e, - } - case vote.Duration < voteDurationMin: - e := fmt.Sprintf("duration %v under min duration %v", - vote.Duration, voteDurationMin) - return backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), - ErrorContext: e, - } - case vote.QuorumPercentage > 100: - e := fmt.Sprintf("quorum percent %v exceeds 100 percent", - vote.QuorumPercentage) - return backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), - ErrorContext: e, - } - case vote.PassPercentage > 100: - e := fmt.Sprintf("pass percent %v exceeds 100 percent", - vote.PassPercentage) - return backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), - ErrorContext: e, - } - } - - // Verify vote options. Different vote types have different - // requirements. - if len(vote.Options) == 0 { - return backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), - ErrorContext: "no vote options found", - } - } - switch vote.Type { - case ticketvote.VoteTypeStandard, ticketvote.VoteTypeRunoff: - // These vote types only allow for approve/reject votes. Ensure - // that the only options present are approve/reject and that they - // use the vote option IDs specified by the ticketvote API. - if len(vote.Options) != 2 { - e := fmt.Sprintf("vote options count got %v, want 2", - len(vote.Options)) - return backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), - ErrorContext: e, - } - } - // map[optionID]found - options := map[string]bool{ - ticketvote.VoteOptionIDApprove: false, - ticketvote.VoteOptionIDReject: false, - } - for _, v := range vote.Options { - switch v.ID { - case ticketvote.VoteOptionIDApprove: - options[v.ID] = true - case ticketvote.VoteOptionIDReject: - options[v.ID] = true - } - } - missing := make([]string, 0, 2) - for k, v := range options { - if !v { - // Option ID was not found - missing = append(missing, k) - } - } - if len(missing) > 0 { - e := fmt.Sprintf("vote option IDs not found: %v", - strings.Join(missing, ",")) - return backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), - ErrorContext: e, - } - } - } - - // Verify vote bits are somewhat sane - for _, v := range vote.Options { - err := voteBitVerify(vote.Options, vote.Mask, v.Bit) - if err != nil { - return backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), - ErrorContext: err.Error(), - } - } - } - - // Verify parent token - switch { - case vote.Type == ticketvote.VoteTypeStandard && vote.Parent != "": - e := "parent token should not be provided for a standard vote" - return backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), - ErrorContext: e, - } - case vote.Type == ticketvote.VoteTypeRunoff: - _, err := tokenDecode(vote.Parent) - if err != nil { - e := fmt.Sprintf("invalid parent %v", vote.Parent) - return backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), - ErrorContext: e, - } - } - } - - return nil -} - -// startStandard starts a standard vote. -func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvote.Start) (*ticketvote.StartReply, error) { - // Verify there is only one start details - if len(s.Starts) != 1 { - return nil, backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeStartDetailsInvalid), - ErrorContext: "more than one start details found", - } - } - sd := s.Starts[0] - - // Verify token - t, err := tokenDecode(sd.Params.Token) - if err != nil { - return nil, backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), - } - } - if !bytes.Equal(t, token) { - e := fmt.Sprintf("plugin token does not match route token: "+ - "got %x, want %x", t, token) - return nil, backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), - ErrorContext: e, - } - } - - // Verify signature - vb, err := json.Marshal(sd.Params) - if err != nil { - return nil, err - } - msg := hex.EncodeToString(util.Digest(vb)) - err = util.VerifySignature(sd.Signature, sd.PublicKey, msg) - if err != nil { - return nil, convertSignatureError(err) - } - - // Verify vote options and params - err = voteParamsVerify(sd.Params, p.voteDurationMin, p.voteDurationMax) - if err != nil { - return nil, err - } - - // Get vote blockchain data - sr, err := p.startReply(sd.Params.Duration) - if err != nil { - return nil, err - } - - // Validate existing record state. The lock for this record must be - // held for the remainder of this function. - m := p.mutex(token) - m.Lock() - defer m.Unlock() - - // Verify record version - r, err := p.backend.GetVetted(token, "") - if err != nil { - return nil, fmt.Errorf("GetVetted: %v", err) - } - version := strconv.FormatUint(uint64(sd.Params.Version), 10) - if r.Version != version { - e := fmt.Sprintf("version is not latest: got %v, want %v", - sd.Params.Version, r.Version) - return nil, backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeRecordVersionInvalid), - ErrorContext: e, - } - } - - // Verify vote authorization - auths, err := p.auths(treeID) - if err != nil { - return nil, err - } - if len(auths) == 0 { - return nil, backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), - ErrorContext: "authorization not found", - } - } - action := ticketvote.AuthActionT(auths[len(auths)-1].Action) - if action != ticketvote.AuthActionAuthorize { - return nil, backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), - ErrorContext: "not authorized", - } - } - - // Verify vote has not already been started - svp, err := p.voteDetails(treeID) - if err != nil { - return nil, err - } - if svp != nil { - // Vote has already been started - return nil, backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeVoteStatusInvalid), - ErrorContext: "vote already started", - } - } - - // Prepare vote details - vd := ticketvote.VoteDetails{ - Params: sd.Params, - PublicKey: sd.PublicKey, - Signature: sd.Signature, - StartBlockHeight: sr.StartBlockHeight, - StartBlockHash: sr.StartBlockHash, - EndBlockHeight: sr.EndBlockHeight, - EligibleTickets: sr.EligibleTickets, - } - - // Save vote details - err = p.voteDetailsSave(treeID, vd) - if err != nil { - return nil, fmt.Errorf("voteDetailsSave: %v", err) - } - - // Update inventory - p.inventorySetToStarted(vd.Params.Token, ticketvote.VoteTypeStandard, - vd.EndBlockHeight) - - return &ticketvote.StartReply{ - StartBlockHeight: sr.StartBlockHeight, - StartBlockHash: sr.StartBlockHash, - EndBlockHeight: sr.EndBlockHeight, - EligibleTickets: sr.EligibleTickets, - }, nil -} - -// startRunoffRecord is the record that is saved to the runoff vote's parent -// tree as the first step in starting a runoff vote. Plugins are not able to -// update multiple records atomically, so if this call gets interrupted before -// if can start the voting period on all runoff vote submissions, subsequent -// calls will use this record to pick up where the previous call left off. This -// allows us to recover from unexpected errors, such as network errors, and not -// leave a runoff vote in a weird state. -type startRunoffRecord struct { - Submissions []string `json:"submissions"` - Mask uint64 `json:"mask"` - Duration uint32 `json:"duration"` - QuorumPercentage uint32 `json:"quorumpercentage"` - PassPercentage uint32 `json:"passpercentage"` - StartBlockHeight uint32 `json:"startblockheight"` - StartBlockHash string `json:"startblockhash"` - EndBlockHeight uint32 `json:"endblockheight"` - EligibleTickets []string `json:"eligibletickets"` -} - -func (p *ticketVotePlugin) startRunoffRecordSave(treeID int64, srr startRunoffRecord) error { - be, err := convertBlobEntryFromStartRunoff(srr) - if err != nil { - return err - } - err = p.tlog.BlobSave(treeID, dataTypeStartRunoff, *be) - if err != nil { - return fmt.Errorf("BlobSave %v %v: %v", - treeID, dataTypeStartRunoff, err) - } - return nil -} - -// startRunoffRecord returns the startRunoff record if one exists on a tree. -// nil will be returned if a startRunoff record is not found. -func (p *ticketVotePlugin) startRunoffRecord(treeID int64) (*startRunoffRecord, error) { - blobs, err := p.tlog.BlobsByDataType(treeID, dataTypeStartRunoff) - if err != nil { - return nil, fmt.Errorf("BlobsByDataType %v %v: %v", - treeID, dataTypeStartRunoff, err) - } - - var srr *startRunoffRecord - switch len(blobs) { - case 0: - // Nothing found - return nil, nil - case 1: - // A start runoff record was found - srr, err = convertStartRunoffFromBlobEntry(blobs[0]) - if err != nil { - return nil, err - } - default: - // This should not be possible - e := fmt.Sprintf("%v start runoff blobs found", len(blobs)) - panic(e) - } - - return srr, nil -} - -func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs startRunoffSub) error { - // Sanity check - s := srs.StartDetails - t, err := tokenDecode(s.Params.Token) - if err != nil { - return err - } - if bytes.Equal(token, t) { - return fmt.Errorf("invalid token") - } - - // Get the start runoff record from the parent tree - srr, err := p.startRunoffRecord(srs.ParentTreeID) - if err != nil { - return err - } - - // Sanity check. Verify token is part of the start runoff record - // submissions. - var found bool - for _, v := range srr.Submissions { - if hex.EncodeToString(token) == v { - found = true - break - } - } - if !found { - // This submission should not be here - return fmt.Errorf("record not in submission list") - } - - // If the vote has already been started, exit gracefully. This - // allows us to recover from unexpected errors to the start runoff - // vote call as it updates the state of multiple records. If the - // call were to fail before completing, we can simply call the - // command again with the same arguments and it will pick up where - // it left off. - svp, err := p.voteDetails(treeID) - if err != nil { - return err - } - if svp != nil { - // Vote has already been started. Exit gracefully. - return nil - } - - // Verify record version - r, err := p.backend.GetVetted(token, "") - if err != nil { - return fmt.Errorf("GetVetted: %v", err) - } - version := strconv.FormatUint(uint64(s.Params.Version), 10) - if r.Version != version { - e := fmt.Sprintf("version is not latest %v: got %v, want %v", - s.Params.Token, s.Params.Version, r.Version) - return backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeRecordVersionInvalid), - ErrorContext: e, - } - } - - // Prepare vote details - vd := ticketvote.VoteDetails{ - Params: s.Params, - PublicKey: s.PublicKey, - Signature: s.Signature, - StartBlockHeight: srr.StartBlockHeight, - StartBlockHash: srr.StartBlockHash, - EndBlockHeight: srr.EndBlockHeight, - EligibleTickets: srr.EligibleTickets, - } - - // Save vote details - err = p.voteDetailsSave(treeID, vd) - if err != nil { - return fmt.Errorf("voteDetailsSave: %v", err) - } - - // Update inventory - p.inventorySetToStarted(vd.Params.Token, ticketvote.VoteTypeRunoff, - vd.EndBlockHeight) - - return nil -} - -func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ticketvote.Start) (*startRunoffRecord, error) { - // Check if the runoff vote data already exists on the parent tree. - srr, err := p.startRunoffRecord(treeID) - if err != nil { - return nil, err - } - if srr != nil { - // We already have a start runoff record for this runoff vote. - // This can happen if the previous call failed due to an - // unexpected error such as a network error. Return the start - // runoff record so we can pick up where we left off. - return srr, nil - } - - // Get blockchain data - var ( - mask = s.Starts[0].Params.Mask - duration = s.Starts[0].Params.Duration - quorum = s.Starts[0].Params.QuorumPercentage - pass = s.Starts[0].Params.PassPercentage - ) - sr, err := p.startReply(duration) - if err != nil { - return nil, err - } - - // The parent record must be validated. Hold the lock on the parent - // record for the remainder of this function. - m := p.mutex(token) - m.Lock() - defer m.Unlock() - - // Verify parent has a LinkBy and the LinkBy deadline is expired. - r, err := p.backend.GetVetted(token, "") - if err != nil { - if errors.Is(err, backend.ErrRecordNotFound) { - e := fmt.Sprintf("parent record not found %x", token) - return nil, backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), - ErrorContext: e, - } - } - } - vm, err := decodeVoteMetadata(r.Files) - if err != nil { - return nil, err - } - if vm == nil || vm.LinkBy == 0 { - return nil, backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeRunoffVoteParentInvalid), - } - } - isExpired := vm.LinkBy < time.Now().Unix() - isMainNet := p.activeNetParams.Name == chaincfg.MainNetParams().Name - switch { - case !isExpired && isMainNet: - e := fmt.Sprintf("parent record %x linkby deadline not met %v", - token, vm.LinkBy) - return nil, backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeLinkByNotExpired), - ErrorContext: e, - } - case !isExpired: - // Allow the vote to be started before the link by deadline - // expires on testnet and simnet only. This makes testing the - // runoff vote process easier. - log.Warnf("Parent record linkby deadline has not been met; " + - "disregarding deadline since this is not mainnet") - } - - // Compile a list of the expected submissions that should be in the - // runoff vote. This will be all of the public records that have - // linked to the parent record. The parent records' linked from - // list will include abandoned proposals that need to be filtered - // out. - lf, err := p.linkedFrom(token) - if err != nil { - return nil, err - } - expected := make(map[string]struct{}, len(lf.Tokens)) // [token]struct{} - for k := range lf.Tokens { - token, err := tokenDecode(k) - if err != nil { - return nil, err - } - r, err := p.backend.GetVetted(token, "") - if err != nil { - return nil, err - } - if r.RecordMetadata.Status != backend.MDStatusVetted { - // This record is not public and should not be included - // in the runoff vote. - continue - } - - // This is a public record that is part of the parent record's - // linked from list. It is required to be in the runoff vote. - expected[k] = struct{}{} - } - - // Verify that there are no extra submissions in the runoff vote - for _, v := range s.Starts { - _, ok := expected[v.Params.Token] - if !ok { - // This submission should not be here - e := fmt.Sprintf("record %v should not be included", - v.Params.Token) - return nil, backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeStartDetailsInvalid), - ErrorContext: e, - } - } - } - - // Verify that the runoff vote is not missing any submissions - subs := make(map[string]struct{}, len(s.Starts)) - for _, v := range s.Starts { - subs[v.Params.Token] = struct{}{} - } - for k := range expected { - _, ok := subs[k] - if !ok { - // This records is missing from the runoff vote - return nil, backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeStartDetailsMissing), - ErrorContext: k, - } - } - } - - // Prepare start runoff record - submissions := make([]string, 0, len(subs)) - for k := range subs { - submissions = append(submissions, k) - } - srr = &startRunoffRecord{ - Submissions: submissions, - Mask: mask, - Duration: duration, - QuorumPercentage: quorum, - PassPercentage: pass, - StartBlockHeight: sr.StartBlockHeight, - StartBlockHash: sr.StartBlockHash, - EndBlockHeight: sr.EndBlockHeight, - EligibleTickets: sr.EligibleTickets, - } - - // Save start runoff record - err = p.startRunoffRecordSave(treeID, *srr) - if err != nil { - return nil, fmt.Errorf("startRunoffRecordSave %v: %v", - treeID, err) - } - - return srr, nil -} - -// startRunoff starts a runoff vote. -func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote.Start) (*ticketvote.StartReply, error) { - // Sanity check - if len(s.Starts) == 0 { - return nil, fmt.Errorf("no start details found") - } - - // Perform validation that can be done without fetching any records - // from the backend. - var ( - mask = s.Starts[0].Params.Mask - duration = s.Starts[0].Params.Duration - quorum = s.Starts[0].Params.QuorumPercentage - pass = s.Starts[0].Params.PassPercentage - parent = s.Starts[0].Params.Parent - ) - for _, v := range s.Starts { - // Verify vote params are the same for all submissions - switch { - case v.Params.Type != ticketvote.VoteTypeRunoff: - e := fmt.Sprintf("%v vote type invalid: got %v, want %v", - v.Params.Token, v.Params.Type, ticketvote.VoteTypeRunoff) - return nil, backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), - ErrorContext: e, - } - case v.Params.Mask != mask: - e := fmt.Sprintf("%v mask invalid: all must be the same", - v.Params.Token) - return nil, backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), - ErrorContext: e, - } - case v.Params.Duration != duration: - e := fmt.Sprintf("%v duration invalid: all must be the same", - v.Params.Token) - return nil, backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), - ErrorContext: e, - } - case v.Params.QuorumPercentage != quorum: - e := fmt.Sprintf("%v quorum invalid: must be the same", - v.Params.Token) - return nil, backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), - ErrorContext: e, - } - case v.Params.PassPercentage != pass: - e := fmt.Sprintf("%v pass rate invalid: all must be the same", - v.Params.Token) - return nil, backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), - ErrorContext: e, - } - case v.Params.Parent != parent: - e := fmt.Sprintf("%v parent invalid: all must be the same", - v.Params.Token) - return nil, backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), - ErrorContext: e, - } - } - - // Verify token - _, err := tokenDecode(v.Params.Token) - if err != nil { - return nil, backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), - ErrorContext: v.Params.Token, - } - } - - // Verify parent token - _, err = tokenDecode(v.Params.Parent) - if err != nil { - e := fmt.Sprintf("parent token %v", v.Params.Parent) - return nil, backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), - ErrorContext: e, - } - } - - // Verify signature - vb, err := json.Marshal(v.Params) - if err != nil { - return nil, err - } - msg := hex.EncodeToString(util.Digest(vb)) - err = util.VerifySignature(v.Signature, v.PublicKey, msg) - if err != nil { - return nil, convertSignatureError(err) - } - - // Verify vote options and params. Vote optoins are required to - // be approve and reject. - err = voteParamsVerify(v.Params, p.voteDurationMin, p.voteDurationMax) - if err != nil { - return nil, err - } - } - - // Verify plugin command is being executed on the parent record - if hex.EncodeToString(token) != parent { - e := fmt.Sprintf("runoff vote must be started on parent record %v", - parent) - return nil, backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeRunoffVoteParentInvalid), - ErrorContext: e, - } - } - - // This function is being invoked on the runoff vote parent record. - // Create and save a start runoff record onto the parent record's - // tree. - srr, err := p.startRunoffForParent(treeID, token, s) - if err != nil { - return nil, err - } - - // Start the voting period of each runoff vote submissions by - // using the internal plugin command startRunoffSub. - for _, v := range s.Starts { - token, err = tokenDecode(v.Params.Token) - if err != nil { - return nil, err - } - srs := startRunoffSub{ - ParentTreeID: treeID, - StartDetails: v, - } - b, err := json.Marshal(srs) - if err != nil { - return nil, err - } - _, err = p.backend.VettedPluginCmd(token, ticketvote.ID, - cmdStartRunoffSub, string(b)) - if err != nil { - var ue backend.PluginUserError - if errors.As(err, &ue) { - return nil, err - } - return nil, fmt.Errorf("VettedPluginCmd %x %v %v %v: %v", - token, ticketvote.ID, cmdStartRunoffSub, b, err) - } - } - - return &ticketvote.StartReply{ - StartBlockHeight: srr.StartBlockHeight, - StartBlockHash: srr.StartBlockHash, - EndBlockHeight: srr.EndBlockHeight, - EligibleTickets: srr.EligibleTickets, - }, nil -} - -func (p *ticketVotePlugin) cmdStartRunoffSub(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdStartRunoffSub: %v %x %v", treeID, token, payload) - - // Decode payload - var srs startRunoffSub - err := json.Unmarshal([]byte(payload), &srs) - if err != nil { - return "", err - } - - // Start voting period on runoff vote submission - err = p.startRunoffForSub(treeID, token, srs) - if err != nil { - return "", err - } - - // Prepare reply - reply, err := json.Marshal(startRunoffSubReply{}) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *ticketVotePlugin) cmdStart(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdStart: %v %x %v", treeID, token, payload) - - // Decode payload - var s ticketvote.Start - err := json.Unmarshal([]byte(payload), &s) - if err != nil { - return "", err - } - - // Parse vote type - if len(s.Starts) == 0 { - return "", backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeStartDetailsMissing), - ErrorContext: "no start details found", - } - } - vtype := s.Starts[0].Params.Type - - // Start vote - // TODO these vote user errors need to become more granular. Update - // this when writing tests. - var sr *ticketvote.StartReply - switch vtype { - case ticketvote.VoteTypeStandard: - sr, err = p.startStandard(treeID, token, s) - if err != nil { - return "", err - } - case ticketvote.VoteTypeRunoff: - sr, err = p.startRunoff(treeID, token, s) - if err != nil { - return "", err - } - default: - e := fmt.Sprintf("invalid vote type %v", vtype) - return "", backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), - ErrorContext: e, - } - } - - // Prepare reply - reply, err := json.Marshal(*sr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -// ballot casts the provided votes concurrently. The vote results are passed -// back through the results channel to the calling function. This function -// waits until all provided votes have been cast before returning. -// -// This function must be called WITH the record lock held. -func (p *ticketVotePlugin) ballot(treeID int64, votes []ticketvote.CastVote, results chan ticketvote.CastVoteReply) { - // Cast the votes concurrently - var wg sync.WaitGroup - for _, v := range votes { - // Increment the wait group counter - wg.Add(1) - - go func(v ticketvote.CastVote) { - // Decrement wait group counter once vote is cast - defer wg.Done() - - // Setup cast vote details - receipt := p.identity.SignMessage([]byte(v.Signature)) - cv := ticketvote.CastVoteDetails{ - Token: v.Token, - Ticket: v.Ticket, - VoteBit: v.VoteBit, - Signature: v.Signature, - Receipt: hex.EncodeToString(receipt[:]), - } - - // Save cast vote - var cvr ticketvote.CastVoteReply - err := p.castVoteSave(treeID, cv) - if err != nil { - t := time.Now().Unix() - log.Errorf("cmdCastBallot: castVoteSave %v: %v", t, err) - e := ticketvote.VoteErrorInternalError - cvr.Ticket = v.Ticket - cvr.ErrorCode = e - cvr.ErrorContext = fmt.Sprintf("%v: %v", - ticketvote.VoteErrors[e], t) - goto sendResult - } - - // Update receipt - cvr.Ticket = v.Ticket - cvr.Receipt = cv.Receipt - - // Update cast votes cache - p.cachedVotesSet(v.Token, v.Ticket, v.VoteBit) - - sendResult: - // Send result back to calling function - results <- cvr - }(v) - } - - // Wait for the full ballot to be cast before returning. - wg.Wait() -} - -// cmdCastBallot casts a ballot of votes. This function will not return a user -// error if one occurs. It will instead return the ballot reply with the error -// included in the invidiual cast vote reply that it applies to. -func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdCastBallot: %v %x %v", treeID, token, payload) - - // Decode payload - var cb ticketvote.CastBallot - err := json.Unmarshal([]byte(payload), &cb) - if err != nil { - return "", err - } - votes := cb.Ballot - - // Verify there is work to do - if len(votes) == 0 { - // Nothing to do - cbr := ticketvote.CastBallotReply{ - Receipts: []ticketvote.CastVoteReply{}, - } - reply, err := json.Marshal(cbr) - if err != nil { - return "", err - } - return string(reply), nil - } - - // Verify that all tokens in the ballot are valid, full length - // tokens and that they are all voting for the same record. - var ( - receipts = make([]ticketvote.CastVoteReply, len(votes)) - ) - for k, v := range votes { - // Verify token - t, err := tokenDecode(v.Token) - if err != nil { - e := ticketvote.VoteErrorTokenInvalid - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = fmt.Sprintf("%v: not hex", - ticketvote.VoteErrors[e]) - continue - } - - // Verify token is the same - if !bytes.Equal(t, token) { - e := ticketvote.VoteErrorMultipleRecordVotes - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = ticketvote.VoteErrors[e] - continue - } - } - - // From this point forward, it can be assumed that all votes that - // have not had their error set are voting for the same record. Get - // the record and vote data that we need to perform the remaining - // inexpensive validation before we have to hold the lock. - voteDetails, err := p.voteDetails(treeID) - if err != nil { - return "", err - } - bestBlock, err := p.bestBlock() - if err != nil { - return "", err - } - - // eligible contains the ticket hashes of all eligble tickets. They - // are put into a map for O(n) lookups. - eligible := make(map[string]struct{}, len(voteDetails.EligibleTickets)) - for _, v := range voteDetails.EligibleTickets { - eligible[v] = struct{}{} - } - - // addrs contains the largest commitment addresses for each ticket. - // The vote must be signed using the largest commitment address. - tickets := make([]string, 0, len(cb.Ballot)) - for _, v := range cb.Ballot { - tickets = append(tickets, v.Ticket) - } - addrs, err := p.largestCommitmentAddrs(tickets) - if err != nil { - return "", fmt.Errorf("largestCommitmentAddrs: %v", err) - } - - // Perform validation that doesn't require holding the record lock. - for k, v := range votes { - if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { - // Vote has an error. Skip it. - continue - } - - // Verify record vote status - if voteDetails == nil { - // Vote has not been started yet - e := ticketvote.VoteErrorVoteStatusInvalid - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = fmt.Sprintf("%v: vote not started", - ticketvote.VoteErrors[e]) - continue - } - if bestBlock >= voteDetails.EndBlockHeight { - // Vote has ended - e := ticketvote.VoteErrorVoteStatusInvalid - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = fmt.Sprintf("%v: vote has ended", - ticketvote.VoteErrors[e]) - continue - } - - // Verify vote bit - bit, err := strconv.ParseUint(v.VoteBit, 16, 64) - if err != nil { - e := ticketvote.VoteErrorVoteBitInvalid - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = ticketvote.VoteErrors[e] - continue - } - err = voteBitVerify(voteDetails.Params.Options, - voteDetails.Params.Mask, bit) - if err != nil { - e := ticketvote.VoteErrorVoteBitInvalid - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = fmt.Sprintf("%v: %v", - ticketvote.VoteErrors[e], err) - continue - } - - // Verify vote signature - commitmentAddr := addrs[k] - if commitmentAddr.ticket != v.Ticket { - t := time.Now().Unix() - log.Errorf("cmdCastBallot: commitment addr mismatch %v: %v %v", - t, commitmentAddr.ticket, v.Ticket) - e := ticketvote.VoteErrorInternalError - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = fmt.Sprintf("%v: %v", - ticketvote.VoteErrors[e], t) - continue - } - if commitmentAddr.err != nil { - t := time.Now().Unix() - log.Errorf("cmdCastBallot: commitment addr error %v: %v %v", - t, commitmentAddr.ticket, commitmentAddr.err) - e := ticketvote.VoteErrorInternalError - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = fmt.Sprintf("%v: %v", - ticketvote.VoteErrors[e], t) - continue - } - err = p.castVoteSignatureVerify(v, commitmentAddr.addr) - if err != nil { - e := ticketvote.VoteErrorSignatureInvalid - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = fmt.Sprintf("%v: %v", - ticketvote.VoteErrors[e], err) - continue - } - - // Verify ticket is eligible to vote - _, ok := eligible[v.Ticket] - if !ok { - e := ticketvote.VoteErrorTicketNotEligible - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = ticketvote.VoteErrors[e] - continue - } - } - - // The record lock must be held for the remainder of the function to - // ensure duplicate votes cannot be cast. - m := p.mutex(token) - m.Lock() - defer m.Unlock() - - // cachedVotes contains the tickets that have alread voted - cachedVotes := p.cachedVotes(token) - for k, v := range votes { - if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { - // Vote has an error. Skip it. - continue - } - - // Verify ticket has not already vote - _, ok := cachedVotes[v.Ticket] - if ok { - e := ticketvote.VoteErrorTicketAlreadyVoted - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = ticketvote.VoteErrors[e] - continue - } - } - - // The votes that have passed validation will be cast in batches of - // size batchSize. Each batch of votes is cast concurrently in order - // to accommodate the trillian log signer bottleneck. The log signer - // picks up queued leaves and appends them onto the trillian tree - // every xxx ms, where xxx is a configurable value on the log signer, - // but is typically a few hundred milliseconds. Lets use 200ms as an - // example. If we don't cast the votes in batches then every vote in - // the ballot will take 200 milliseconds since we wait for the leaf - // to be fully appended before considering the trillian call - // successful. A person casting hundreds of votes in a single ballot - // would cause UX issues for all the voting clients since the lock is - // held during these calls. - // - // The second variable that we must watch out for is the max trillian - // queued leaf batch size. This is also a configurable trillian value - // that represents the maximum number of leaves that can be waiting - // in the queue for all trees in the trillian instance. This value is - // typically around the order of magnitude of 1000 queued leaves. - // - // This is why a vote batch size of 5 was chosen. It is large enough - // to alleviate performance bottlenecks from the log signer interval, - // but small enough to still allow multiple records votes be held - // concurrently without running into the queued leaf batch size limit. - - // Prepare work - var ( - batchSize = 5 - batch = make([]ticketvote.CastVote, 0, batchSize) - queue = make([][]ticketvote.CastVote, 0, len(votes)/batchSize) - - // ballotCount is the number of votes that have passed validation - // and are being cast in this ballot. - ballotCount int - ) - for k, v := range votes { - if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { - // Vote has an error. Skip it. - continue - } - - // Add vote to the current batch - batch = append(batch, v) - ballotCount++ - - if len(batch) == batchSize { - // This batch is full. Add the batch to the queue and start - // a new batch. - queue = append(queue, batch) - batch = make([]ticketvote.CastVote, 0, batchSize) - } - } - if len(batch) != 0 { - // Add leftover batch to the queue - queue = append(queue, batch) - } - - log.Debugf("Casting %v votes in %v batches of size %v", - ballotCount, len(queue), batchSize) - - // Cast ballot in batches - results := make(chan ticketvote.CastVoteReply, ballotCount) - for i, batch := range queue { - log.Debugf("Casting %v votes in batch %v/%v", len(batch), i+1, len(queue)) - - p.ballot(treeID, batch, results) - } - - // Empty out the results channel - r := make(map[string]ticketvote.CastVoteReply, ballotCount) - close(results) - for v := range results { - r[v.Ticket] = v - } - - if len(r) != ballotCount { - log.Errorf("Missing results: got %v, want %v", len(r), ballotCount) - } - - // Fill in the receipts - for k, v := range votes { - if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { - // Vote has an error. Skip it. - continue - } - cvr, ok := r[v.Ticket] - if !ok { - t := time.Now().Unix() - log.Errorf("cmdCastBallot: vote result not found %v: %v", t, v.Ticket) - e := ticketvote.VoteErrorInternalError - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = fmt.Sprintf("%v: %v", - ticketvote.VoteErrors[e], t) - continue - } - - // Fill in receipt - receipts[k] = cvr - } - - // Prepare reply - cbr := ticketvote.CastBallotReply{ - Receipts: receipts, - } - reply, err := json.Marshal(cbr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *ticketVotePlugin) cmdDetails(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdDetails: %v %x %v", treeID, token, payload) - - var d ticketvote.Details - err := json.Unmarshal([]byte(payload), &d) - if err != nil { - return "", err - } - - // Get vote authorizations - auths, err := p.auths(treeID) - if err != nil { - return "", fmt.Errorf("auths: %v", err) - } - - // Get vote details - vd, err := p.voteDetails(treeID) - if err != nil { - return "", fmt.Errorf("voteDetails: %v", err) - } - - // Prepare rely - dr := ticketvote.DetailsReply{ - Auths: auths, - Vote: vd, - } - reply, err := json.Marshal(dr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *ticketVotePlugin) cmdResults(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdResults: %v %x %v", treeID, token, payload) - - // Get cast votes - votes, err := p.castVotes(treeID) - if err != nil { - return "", err - } - - // Prepare reply - rr := ticketvote.ResultsReply{ - Votes: votes, - } - reply, err := json.Marshal(rr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -// voteIsApproved returns whether the provided vote option results met the -// provided quorum and pass percentage requirements. This function can only be -// called on votes that use VoteOptionIDApprove and VoteOptionIDReject. Any -// other vote option IDs will cause this function to panic. -func voteIsApproved(vd ticketvote.VoteDetails, results []ticketvote.VoteOptionResult) bool { - // Tally the total votes - var total uint64 - for _, v := range results { - total += v.Votes - } - - // Calculate required thresholds - var ( - eligible = float64(len(vd.EligibleTickets)) - quorumPerc = float64(vd.Params.QuorumPercentage) - passPerc = float64(vd.Params.PassPercentage) - quorum = uint64(quorumPerc / 100 * eligible) - pass = uint64(passPerc / 100 * float64(total)) - - approvedVotes uint64 - ) - - // Tally approve votes - for _, v := range results { - switch v.ID { - case ticketvote.VoteOptionIDApprove: - // Valid vote option - approvedVotes++ - case ticketvote.VoteOptionIDReject: - // Valid vote option - default: - // Invalid vote option - e := fmt.Sprintf("invalid vote option id found: %v", v.ID) - panic(e) - } - } - - // Check tally against thresholds - var approved bool - switch { - case total < quorum: - // Quorum not met - approved = false - case approvedVotes < pass: - // Pass percentage not met - approved = false - default: - // Vote was approved - approved = true - } - - return approved -} - -func (p *ticketVotePlugin) cmdSummary(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdSummaries: %v %x %v", treeID, token, payload) - - // Get best block. This cmd does not write any data so we do not - // have to use the safe best block. - bb, err := p.bestBlockUnsafe() - if err != nil { - return "", fmt.Errorf("bestBlockUnsafe: %v", err) - } - - // Get summary - s, err := p.summary(treeID, token, bb) - if err != nil { - return "", fmt.Errorf("summary: %v", err) - } - - // Prepare reply - sr := ticketvote.SummaryReply{ - Summary: *s, - BestBlock: bb, - } - reply, err := json.Marshal(sr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func convertInventoryReply(v voteInventory) ticketvote.InventoryReply { - // Started needs to be converted from a map to a slice where the - // slice is sorted by end block height from smallest to largest. - tokensByHeight := make(map[uint32][]string, len(v.started)) - for token, height := range v.started { - tokens, ok := tokensByHeight[height] - if !ok { - tokens = make([]string, 0, len(v.started)) - } - tokens = append(tokens, token) - tokensByHeight[height] = tokens - } - sortedHeights := make([]uint32, 0, len(tokensByHeight)) - for k := range tokensByHeight { - sortedHeights = append(sortedHeights, k) - } - // Sort smallest to largest block height - sort.SliceStable(sortedHeights, func(i, j int) bool { - return sortedHeights[i] < sortedHeights[j] - }) - started := make([]string, 0, len(v.started)) - for _, height := range sortedHeights { - tokens := tokensByHeight[height] - started = append(started, tokens...) - } - return ticketvote.InventoryReply{ - Unauthorized: v.unauthorized, - Authorized: v.authorized, - Started: started, - Finished: v.finished, - BestBlock: v.bestBlock, - } -} - -func (p *ticketVotePlugin) cmdInventory() (string, error) { - log.Tracef("cmdInventory") - - // Get best block. This command does not write any data so we can - // use the unsafe best block. - bb, err := p.bestBlockUnsafe() - if err != nil { - return "", fmt.Errorf("bestBlockUnsafe: %v", err) - } - - // Get the inventory - inv, err := p.inventory(bb) - if err != nil { - return "", fmt.Errorf("inventory: %v", err) - } - ir := convertInventoryReply(*inv) - - // Prepare reply - reply, err := json.Marshal(ir) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdTimestamps: %v %x %v", treeID, token, payload) - - // Decode payload - var t ticketvote.Timestamps - err := json.Unmarshal([]byte(payload), &t) - if err != nil { - return "", err - } - - // Get authorization timestamps - digests, err := p.tlog.DigestsByDataType(treeID, dataTypeAuthDetails) - if err != nil { - return "", fmt.Errorf("DigestByDataType %v %v: %v", - treeID, dataTypeAuthDetails, err) - } - - auths := make([]ticketvote.Timestamp, 0, len(digests)) - for _, v := range digests { - ts, err := p.timestamp(treeID, v) - if err != nil { - return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) - } - auths = append(auths, *ts) - } - - // Get vote details timestamp. There should never be more than one - // vote details. - digests, err = p.tlog.DigestsByDataType(treeID, dataTypeVoteDetails) - if err != nil { - return "", fmt.Errorf("DigestsByDataType %v %v: %v", - treeID, dataTypeVoteDetails, err) - } - if len(digests) > 1 { - return "", fmt.Errorf("invalid vote details count: got %v, want 1", - len(digests)) - } - - var details ticketvote.Timestamp - for _, v := range digests { - ts, err := p.timestamp(treeID, v) - if err != nil { - return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) - } - details = *ts - } - - // Get cast vote timestamps - digests, err = p.tlog.DigestsByDataType(treeID, dataTypeCastVoteDetails) - if err != nil { - return "", fmt.Errorf("digestsByKeyPrefix %v %v: %v", - treeID, dataTypeVoteDetails, err) - } - - votes := make(map[string]ticketvote.Timestamp, len(digests)) - for _, v := range digests { - ts, err := p.timestamp(treeID, v) - if err != nil { - return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) - } - - var cv ticketvote.CastVoteDetails - err = json.Unmarshal([]byte(ts.Data), &cv) - if err != nil { - return "", err - } - - votes[cv.Ticket] = *ts - } - - // Prepare reply - tr := ticketvote.TimestampsReply{ - Auths: auths, - Details: details, - Votes: votes, - } - reply, err := json.Marshal(tr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -// decodeVoteMetadata decodes and returns the VoteMetadata from the -// provided backend files. If a VoteMetadata is not found, nil is returned. -func decodeVoteMetadata(files []backend.File) (*ticketvote.VoteMetadata, error) { - var voteMD *ticketvote.VoteMetadata - for _, v := range files { - if v.Name == ticketvote.FileNameVoteMetadata { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return nil, err - } - var pm ticketvote.VoteMetadata - err = json.Unmarshal(b, &pm) - if err != nil { - return nil, err - } - voteMD = &pm - break - } - } - return voteMD, nil -} - -func (p *ticketVotePlugin) linkByVerify(linkBy int64) error { - if linkBy == 0 { - // LinkBy as not been set - return nil - } - min := time.Now().Unix() + p.linkByPeriodMin - max := time.Now().Unix() + p.linkByPeriodMax - switch { - case linkBy < min: - e := fmt.Sprintf("linkby %v is less than min required of %v", - linkBy, min) - return backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeLinkByInvalid), - ErrorContext: e, - } - case linkBy > max: - e := fmt.Sprintf("linkby %v is more than max allowed of %v", - linkBy, max) - return backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeLinkByInvalid), - ErrorContext: e, - } - } - return nil -} - -func (p *ticketVotePlugin) linkToVerify(linkTo string) error { - // LinkTo must be a public record that is the parent of a runoff - // vote, i.e. has the VoteMetadata.LinkBy field set. - token, err := tokenDecode(linkTo) - if err != nil { - return backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), - ErrorContext: "invalid hex", - } - } - r, err := p.backend.GetVetted(token, "") - if err != nil { - if errors.Is(err, backend.ErrRecordNotFound) { - return backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), - ErrorContext: "record not found", - } - } - return err - } - if r.RecordMetadata.Status != backend.MDStatusCensored { - return backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), - ErrorContext: "record is censored", - } - } - parentVM, err := decodeVoteMetadata(r.Files) - if err != nil { - return err - } - if parentVM == nil || parentVM.LinkBy == 0 { - return backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), - ErrorContext: "record not a runoff vote parent", - } - } - if time.Now().Unix() > parentVM.LinkBy { - // Linkby deadline has expired. New links are not allowed. - return backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), - ErrorContext: "parent record linkby deadline has expired", - } - } - return nil -} - -func (p *ticketVotePlugin) voteMetadataVerify(vm ticketvote.VoteMetadata) error { - switch { - case vm.LinkBy == 0 && vm.LinkTo == "": - // Vote metadata is empty - return backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeVoteMetadataInvalid), - ErrorContext: "md is empty", - } - - case vm.LinkBy != 0 && vm.LinkTo != "": - // LinkBy and LinkTo cannot both be set - return backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeVoteMetadataInvalid), - ErrorContext: "cannot set both linkby and linkto", - } - - case vm.LinkBy != 0: - err := p.linkByVerify(vm.LinkBy) - if err != nil { - return err - } - - case vm.LinkTo != "": - err := p.linkToVerify(vm.LinkTo) - if err != nil { - return err - } - } - - return nil -} - -// TODO this save validation needs to be done in the edit record hook too -func (p *ticketVotePlugin) hookNewRecordPre(payload string) error { - var nr plugins.HookNewRecordPre - err := json.Unmarshal([]byte(payload), &nr) - if err != nil { - return err - } - - // Verify the vote metadata if the record contains one - vm, err := decodeVoteMetadata(nr.Files) - if err != nil { - return err - } - if vm != nil { - err = p.voteMetadataVerify(*vm) - if err != nil { - return err - } - } - - return nil -} - -func (p *ticketVotePlugin) hookEditRecordPre(payload string) error { - var er plugins.HookEditRecord - err := json.Unmarshal([]byte(payload), &er) - if err != nil { - return err - } - - // The LinkTo field is not allowed to change once the record has - // become public. If this is a vetted record, verify that any - // previously set LinkTo has not changed. - if er.State == plugins.RecordStateVetted { - var ( - oldLinkTo string - newLinkTo string - ) - vm, err := decodeVoteMetadata(er.Current.Files) - if err != nil { - return err - } - if vm != nil { - oldLinkTo = vm.LinkTo - } - vm, err = decodeVoteMetadata(er.FilesAdd) - if err != nil { - return err - } - if vm != nil { - newLinkTo = vm.LinkTo - } - if newLinkTo != oldLinkTo { - e := fmt.Sprintf("linkto cannot change on vetted record: "+ - "got '%v', want '%v'", newLinkTo, oldLinkTo) - return backend.PluginUserError{ - PluginID: ticketvote.ID, - ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), - ErrorContext: e, - } - } - } - - // Verify LinkBy if one was included. The VoteMetadata is optional - // so the record may not contain one. - vm, err := decodeVoteMetadata(er.FilesAdd) - if err != nil { - return err - } - if vm != nil { - err = p.linkByVerify(vm.LinkBy) - if err != nil { - return err - } - } - - return nil -} - -func (p *ticketVotePlugin) hookSetRecordStatusPost(payload string) error { - var srs plugins.HookSetRecordStatus - err := json.Unmarshal([]byte(payload), &srs) - if err != nil { - return err - } - - // Check if the LinkTo has been set - vm, err := decodeVoteMetadata(srs.Current.Files) - if err != nil { - return err - } - if vm != nil && vm.LinkTo != "" { - // LinkTo has been set. Check if the status change requires the - // linked from list of the linked record to be updated. - var ( - parentToken = vm.LinkTo - childToken = srs.RecordMetadata.Token - ) - switch srs.RecordMetadata.Status { - case backend.MDStatusVetted: - // Record has been made public. Add child token to parent's - // linked from list. - err := p.linkedFromAdd(parentToken, childToken) - if err != nil { - return fmt.Errorf("linkedFromAdd: %v", err) - } - case backend.MDStatusCensored: - // Record has been censored. Delete child token from parent's - // linked from list. - err := p.linkedFromDel(parentToken, childToken) - if err != nil { - return fmt.Errorf("linkedFromDel: %v", err) - } - } - } - - return nil -} - // Setup performs any plugin setup work that needs to be done. // // This function satisfies the plugins.Client interface. @@ -3007,7 +145,7 @@ func (p *ticketVotePlugin) Setup() error { } p.Lock() - p.inv = voteInventory{ + p.inv = inventory{ unauthorized: unauthorized, authorized: authorized, started: started, @@ -3132,7 +270,7 @@ func (p *politeiawww) linkByPeriodMax() int64 { } */ -func newTicketVotePlugin(backend backend.Backend, tlog tlogclient.Client, settings []backend.PluginSetting, dataDir string, id *identity.FullIdentity, activeNetParams *chaincfg.Params) (*ticketVotePlugin, error) { +func New(backend backend.Backend, tlog tlogclient.Client, settings []backend.PluginSetting, dataDir string, id *identity.FullIdentity, activeNetParams *chaincfg.Params) (*ticketVotePlugin, error) { // Plugin settings var ( voteDurationMin uint32 @@ -3158,14 +296,14 @@ func newTicketVotePlugin(backend backend.Backend, tlog tlogclient.Client, settin // Parse user provided plugin settings for _, v := range settings { switch v.Key { - case pluginSettingVoteDurationMin: + case ticketvote.SettingKeyVoteDurationMin: u, err := strconv.ParseUint(v.Value, 10, 64) if err != nil { return nil, fmt.Errorf("plugin setting '%v': ParseUint(%v): %v", v.Key, v.Value, err) } voteDurationMin = uint32(u) - case pluginSettingVoteDurationMax: + case ticketvote.SettingKeyVoteDurationMax: u, err := strconv.ParseUint(v.Value, 10, 64) if err != nil { return nil, fmt.Errorf("plugin setting '%v': ParseUint(%v): %v", @@ -3192,7 +330,7 @@ func newTicketVotePlugin(backend backend.Backend, tlog tlogclient.Client, settin voteDurationMax: voteDurationMax, dataDir: dataDir, identity: id, - inv: voteInventory{ + inv: inventory{ unauthorized: make([]string, 0, 1024), authorized: make([]string, 0, 1024), started: make(map[string]uint32, 1024), diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/votes.go b/politeiad/backend/tlogbe/plugins/ticketvote/votes.go new file mode 100644 index 000000000..7d75b7a62 --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/ticketvote/votes.go @@ -0,0 +1,50 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ticketvote + +import ( + "encoding/hex" +) + +func (p *ticketVotePlugin) cachedVotes(token []byte) map[string]string { + p.Lock() + defer p.Unlock() + + // Return a copy of the map + cv, ok := p.votes[hex.EncodeToString(token)] + if !ok { + return map[string]string{} + } + c := make(map[string]string, len(cv)) + for k, v := range cv { + c[k] = v + } + + return c +} + +func (p *ticketVotePlugin) cachedVotesSet(token, ticket, voteBit string) { + p.Lock() + defer p.Unlock() + + _, ok := p.votes[token] + if !ok { + p.votes[token] = make(map[string]string, 40960) // Ticket pool size + } + + p.votes[token][ticket] = voteBit + + log.Debugf("Added vote to cache: %v %v %v", + token, ticket, voteBit) +} + +func (p *ticketVotePlugin) cachedVotesDel(token string) { + p.Lock() + defer p.Unlock() + + delete(p.votes, token) + + log.Debugf("Deleted votes cache: %v", token) +} diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 76cba03ca..c959d7246 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -1648,11 +1648,11 @@ func (t *tlogBackend) SetupVettedPlugin(pluginID string) error { return t.vetted.PluginSetup(pluginID) } -// UnvettedPlugin executes a plugin command on an unvetted record. +// UnvettedPluginCmd executes a plugin command on an unvetted record. // // This function satisfies the backend.Backend interface. -func (t *tlogBackend) UnvettedPlugin(token []byte, pluginID, cmd, payload string) (string, error) { - log.Tracef("UnvettedPlugin: %x %v %v", token, pluginID, cmd) +func (t *tlogBackend) UnvettedPluginCmd(token []byte, pluginID, cmd, payload string) (string, error) { + log.Tracef("UnvettedPluginCmd: %x %v %v", token, pluginID, cmd) if t.isShutdown() { return "", backend.ErrShutdown @@ -1711,11 +1711,11 @@ func (t *tlogBackend) UnvettedPlugin(token []byte, pluginID, cmd, payload string return reply, nil } -// VettedPlugin executes a plugin command on an unvetted record. +// VettedPluginCmd executes a plugin command on an unvetted record. // // This function satisfies the backend.Backend interface. -func (t *tlogBackend) VettedPlugin(token []byte, pluginID, cmd, payload string) (string, error) { - log.Tracef("VettedPlugin: %x %v %v", token, pluginID, cmd) +func (t *tlogBackend) VettedPluginCmd(token []byte, pluginID, cmd, payload string) (string, error) { + log.Tracef("VettedPluginCmd: %x %v %v", token, pluginID, cmd) if t.isShutdown() { return "", backend.ErrShutdown diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index 36a23ffc6..ffdca359f 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -6,24 +6,34 @@ // decred's proposal system. package pi -type PropStatusT int -type ErrorStatusT int - const ( ID = "pi" // Plugin commands CmdProposalInv = "proposalinv" // Get inventory by proposal status CmdVoteInventory = "voteinv" // Get inventory by vote status +) - // TODO get rid of CmdProposals - CmdProposals = "proposals" // Get plugin data for proposals +// ErrorCodeT represents a plugin error that was caused by the user. +type ErrorCodeT int - // TODO MDStream IDs need to be plugin specific - // Metadata stream IDs - MDStreamIDGeneralMetadata = 1 - MDStreamIDStatusChanges = 2 +const ( + // User error status codes + // TODO number error codes and add human readable error messages + ErrorCodeInvalid ErrorCodeT = 0 + ErrorCodePageSizeExceeded ErrorCodeT = iota + ErrorCodePropTokenInvalid + ErrorCodePropStatusInvalid + ErrorCodePropVersionInvalid + ErrorCodePropStatusChangeInvalid + ErrorCodePropLinkToInvalid + ErrorCodeVoteStatusInvalid + ErrorCodeStartDetailsInvalid + ErrorCodeStartDetailsMissing + ErrorCodeVoteParentInvalid +) +const ( // FileNameProposalMetadata is the filename of the ProposalMetadata // file that is saved to politeiad. ProposalMetadata is saved to // politeiad as a file, not as a metadata stream, since it contains @@ -31,26 +41,58 @@ const ( // root that politeiad signs. FileNameProposalMetadata = "proposalmetadata.json" - // Proposal status codes - PropStatusInvalid PropStatusT = 0 // Invalid status - PropStatusUnvetted PropStatusT = 1 // Prop has not been vetted - PropStatusPublic PropStatusT = 2 // Prop has been made public - PropStatusCensored PropStatusT = 3 // Prop has been censored - PropStatusAbandoned PropStatusT = 4 // Prop has been abandoned + // TODO MDStream IDs need to be plugin specific + // MDStreamIDGeneralMetadata is the politeiad metadata stream ID + // for the GeneralMetadata structure. + MDStreamIDGeneralMetadata = 1 - // User error status codes - // TODO number error codes and add human readable error messages - ErrorStatusInvalid ErrorStatusT = 0 - ErrorStatusPageSizeExceeded ErrorStatusT = iota - ErrorStatusPropTokenInvalid - ErrorStatusPropStatusInvalid - ErrorStatusPropVersionInvalid - ErrorStatusPropStatusChangeInvalid - ErrorStatusPropLinkToInvalid - ErrorStatusVoteStatusInvalid - ErrorStatusStartDetailsInvalid - ErrorStatusStartDetailsMissing - ErrorStatusVoteParentInvalid + // MDStreamIDStatusChanges is the politeiad metadata stream ID + // that the StatusesChange structure is appended onto. + MDStreamIDStatusChanges = 2 +) + +// ProposalMetadata contains metadata that is provided by the user as part of +// the proposal submission bundle. The proposal metadata is included in the +// proposal signature since it is user specified data. The ProposalMetadata +// object is saved to politeiad as a file, not as a metadata stream, since it +// needs to be included in the merkle root that politeiad signs. +type ProposalMetadata struct { + Name string `json:"name"` +} + +// GeneralMetadata contains general metadata about a politeiad record. It is +// generated by the server and saved to politeiad as a metadata stream. +// +// Signature is the client signature of the record merkle root. The merkle root +// is the ordered merkle root of all politeiad Files. +type GeneralMetadata struct { + UserID string `json:"userid"` // Author user ID + PublicKey string `json:"publickey"` // Key used for signature + Signature string `json:"signature"` // Signature of merkle root + Timestamp int64 `json:"timestamp"` // Submission UNIX timestamp +} + +// PropStatusT represents a proposal status. +type PropStatusT int + +const ( + // PropStatusInvalid is an invalid proposal status. + PropStatusInvalid PropStatusT = 0 + + // PropStatusUnvetted represents a proposal that has not been made + // public yet. + PropStatusUnvetted PropStatusT = 1 + + // PropStatusPublic represents a proposal that has been made + // public. + PropStatusPublic PropStatusT = 2 + + // PropStatusCensored represents a proposal that has been censored. + PropStatusCensored PropStatusT = 3 + + // PropStatusAbandoned represents a proposal that has been + // abandoned by the author. + PropStatusAbandoned PropStatusT = 4 ) var ( @@ -80,27 +122,6 @@ var ( } ) -// ProposalMetadata contains metadata that is provided by the user as part of -// the proposal submission bundle. The proposal metadata is included in the -// proposal signature since it is user specified data. The ProposalMetadata -// object is saved to politeiad as a file, not as a metadata stream, since it -// needs to be included in the merkle root that politeiad signs. -type ProposalMetadata struct { - Name string `json:"name"` -} - -// GeneralMetadata contains general metadata about a politeiad record. It is -// generated by the server and saved to politeiad as a metadata stream. -// -// Signature is the client signature of the record merkle root. The merkle root -// is the ordered merkle root of all politeiad Files. -type GeneralMetadata struct { - UserID string `json:"userid"` // Author user ID - PublicKey string `json:"publickey"` // Key used for signature - Signature string `json:"signature"` // Signature of merkle root - Timestamp int64 `json:"timestamp"` // Submission UNIX timestamp -} - // StatusChange represents a proposal status change. // // Signature is the client signature of the Token+Version+Status+Reason. @@ -114,25 +135,6 @@ type StatusChange struct { Timestamp int64 `json:"timestamp"` } -// Proposals requests the plugin data for the provided proposals. This includes -// pi plugin data as well as other plugin data such as comment plugin data. -// This command aggregates all proposal plugin data into a single call. -type Proposals struct { - Tokens []string `json:"tokens"` -} - -// ProposalPluginData contains all the plugin data for a proposal. -type ProposalPluginData struct { - Comments uint64 `json:"comments"` // Number of comments - LinkedFrom []string `json:"linkedfrom"` // Linked from list -} - -// ProposalsReply is the reply to the Proposals command. The proposals map will -// not contain an entry for tokens that do not correspond to actual proposals. -type ProposalsReply struct { - Proposals map[string]ProposalPluginData `json:"proposals"` -} - // ProposalInv retrieves the tokens of all proposals in the inventory that // match the provided filtering criteria. The returned proposals are // categorized by proposal state and status. If no filtering criteria is diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index ceb10a7ef..3b41117b1 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -28,6 +28,10 @@ const ( CmdInventory = "inventory" // Get inventory by vote status CmdTimestamps = "timestamps" // Get vote data timestamps + // Plugin setting keys + SettingKeyVoteDurationMin = "votedurationmin" + SettingKeyVoteDurationMax = "votedurationmax" + // Default plugin settings DefaultMainNetVoteDurationMin = 2016 DefaultMainNetVoteDurationMax = 4032 From 6e6af3d1995b74fddba9d2fbacbbb3b4937e0842 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 18 Jan 2021 09:59:06 -0600 Subject: [PATCH 233/449] ticketvote: Add linkedfrom command. --- .../backend/tlogbe/plugins/ticketvote/cmds.go | 443 +++++++++--------- .../tlogbe/plugins/ticketvote/linkedfrom.go | 7 +- .../tlogbe/plugins/ticketvote/ticketvote.go | 2 + politeiad/plugins/ticketvote/ticketvote.go | 12 + 4 files changed, 253 insertions(+), 211 deletions(-) diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go index b9dfff9ac..4520407d3 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go @@ -75,6 +75,226 @@ func convertSignatureError(err error) backend.PluginUserError { } } +func convertAuthDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.AuthDetails, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorAuthDetails { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorAuthDetails) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, fmt.Errorf("decode digest: %v", err) + } + if !bytes.Equal(util.Digest(b), digest) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), digest) + } + var ad ticketvote.AuthDetails + err = json.Unmarshal(b, &ad) + if err != nil { + return nil, fmt.Errorf("unmarshal AuthDetails: %v", err) + } + + return &ad, nil +} + +func convertVoteDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.VoteDetails, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorVoteDetails { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorVoteDetails) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, fmt.Errorf("decode digest: %v", err) + } + if !bytes.Equal(util.Digest(b), digest) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), digest) + } + var vd ticketvote.VoteDetails + err = json.Unmarshal(b, &vd) + if err != nil { + return nil, fmt.Errorf("unmarshal VoteDetails: %v", err) + } + + return &vd, nil +} + +func convertCastVoteDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.CastVoteDetails, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorCastVoteDetails { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorCastVoteDetails) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, fmt.Errorf("decode digest: %v", err) + } + if !bytes.Equal(util.Digest(b), digest) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), digest) + } + var cv ticketvote.CastVoteDetails + err = json.Unmarshal(b, &cv) + if err != nil { + return nil, fmt.Errorf("unmarshal CastVoteDetails: %v", err) + } + + return &cv, nil +} + +func convertStartRunoffFromBlobEntry(be store.BlobEntry) (*startRunoffRecord, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorStartRunoff { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorStartRunoff) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, fmt.Errorf("decode digest: %v", err) + } + if !bytes.Equal(util.Digest(b), digest) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), digest) + } + var srr startRunoffRecord + err = json.Unmarshal(b, &srr) + if err != nil { + return nil, fmt.Errorf("unmarshal StartRunoffRecord: %v", err) + } + + return &srr, nil +} + +func convertBlobEntryFromAuthDetails(ad ticketvote.AuthDetails) (*store.BlobEntry, error) { + data, err := json.Marshal(ad) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorAuthDetails, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertBlobEntryFromVoteDetails(vd ticketvote.VoteDetails) (*store.BlobEntry, error) { + data, err := json.Marshal(vd) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorVoteDetails, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertBlobEntryFromCastVoteDetails(cv ticketvote.CastVoteDetails) (*store.BlobEntry, error) { + data, err := json.Marshal(cv) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorCastVoteDetails, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertBlobEntryFromStartRunoff(srr startRunoffRecord) (*store.BlobEntry, error) { + data, err := json.Marshal(srr) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorStartRunoff, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + func (p *ticketVotePlugin) authSave(treeID int64, ad ticketvote.AuthDetails) error { // Prepare blob be, err := convertBlobEntryFromAuthDetails(ad) @@ -2291,222 +2511,27 @@ func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload str return string(reply), nil } -func convertAuthDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.AuthDetails, error) { - // Decode and validate data hint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorAuthDetails { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorAuthDetails) - } - - // Decode data - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - digest, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, fmt.Errorf("decode digest: %v", err) - } - if !bytes.Equal(util.Digest(b), digest) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), digest) - } - var ad ticketvote.AuthDetails - err = json.Unmarshal(b, &ad) - if err != nil { - return nil, fmt.Errorf("unmarshal AuthDetails: %v", err) - } - - return &ad, nil -} - -func convertVoteDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.VoteDetails, error) { - // Decode and validate data hint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorVoteDetails { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorVoteDetails) - } +func (p *ticketVotePlugin) cmdLinkedFrom(token []byte) (string, error) { + log.Tracef("cmdLinkedFrom: %x", token) - // Decode data - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - digest, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, fmt.Errorf("decode digest: %v", err) - } - if !bytes.Equal(util.Digest(b), digest) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), digest) - } - var vd ticketvote.VoteDetails - err = json.Unmarshal(b, &vd) - if err != nil { - return nil, fmt.Errorf("unmarshal VoteDetails: %v", err) - } - - return &vd, nil -} - -func convertCastVoteDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.CastVoteDetails, error) { - // Decode and validate data hint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorCastVoteDetails { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorCastVoteDetails) - } - - // Decode data - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - digest, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, fmt.Errorf("decode digest: %v", err) - } - if !bytes.Equal(util.Digest(b), digest) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), digest) - } - var cv ticketvote.CastVoteDetails - err = json.Unmarshal(b, &cv) - if err != nil { - return nil, fmt.Errorf("unmarshal CastVoteDetails: %v", err) - } - - return &cv, nil -} - -func convertStartRunoffFromBlobEntry(be store.BlobEntry) (*startRunoffRecord, error) { - // Decode and validate data hint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorStartRunoff { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorStartRunoff) - } - - // Decode data - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - digest, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, fmt.Errorf("decode digest: %v", err) - } - if !bytes.Equal(util.Digest(b), digest) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), digest) - } - var srr startRunoffRecord - err = json.Unmarshal(b, &srr) - if err != nil { - return nil, fmt.Errorf("unmarshal StartRunoffRecord: %v", err) - } - - return &srr, nil -} - -func convertBlobEntryFromAuthDetails(ad ticketvote.AuthDetails) (*store.BlobEntry, error) { - data, err := json.Marshal(ad) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorAuthDetails, - }) + // Get linked from list + lf, err := p.linkedFrom(token) if err != nil { - return nil, err + return "", err } - be := store.NewBlobEntry(hint, data) - return &be, nil -} -func convertBlobEntryFromVoteDetails(vd ticketvote.VoteDetails) (*store.BlobEntry, error) { - data, err := json.Marshal(vd) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorVoteDetails, - }) - if err != nil { - return nil, err + // Prepare reply + tokens := make([]string, 0, len(lf.Tokens)) + for k := range lf.Tokens { + tokens = append(tokens, k) } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func convertBlobEntryFromCastVoteDetails(cv ticketvote.CastVoteDetails) (*store.BlobEntry, error) { - data, err := json.Marshal(cv) - if err != nil { - return nil, err + lfr := ticketvote.LinkedFromReply{ + Tokens: tokens, } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorCastVoteDetails, - }) + reply, err := json.Marshal(lfr) if err != nil { - return nil, err + return "", err } - be := store.NewBlobEntry(hint, data) - return &be, nil -} -func convertBlobEntryFromStartRunoff(srr startRunoffRecord) (*store.BlobEntry, error) { - data, err := json.Marshal(srr) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorStartRunoff, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil + return string(reply), nil } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/linkedfrom.go b/politeiad/backend/tlogbe/plugins/ticketvote/linkedfrom.go index ccd204701..ef6f965c3 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/linkedfrom.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/linkedfrom.go @@ -43,7 +43,9 @@ func (p *ticketVotePlugin) linkedFromPath(token []byte) string { return filepath.Join(p.dataDir, fn) } -// linkedFromWithLock return the linkedFrom list for a record token. +// linkedFromWithLock return the linked from list for a record token. If a +// linked from list does not exist for the token then an empty list will be +// returned. // // This function must be called WITH the lock held. func (p *ticketVotePlugin) linkedFromWithLock(token []byte) (*linkedFrom, error) { @@ -68,7 +70,8 @@ func (p *ticketVotePlugin) linkedFromWithLock(token []byte) (*linkedFrom, error) return &lf, nil } -// linkedFrom return the linkedFrom list for a record token. +// linkedFrom return the linked from list for a record token. If a linked from +// list does not exist for the token then an empty list will be returned. // // This function must be called WITHOUT the lock held. func (p *ticketVotePlugin) linkedFrom(token []byte) (*linkedFrom, error) { diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go index 8e5122700..3deb9b511 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go @@ -204,6 +204,8 @@ func (p *ticketVotePlugin) Cmd(treeID int64, token []byte, cmd, payload string) return p.cmdInventory() case ticketvote.CmdTimestamps: return p.cmdTimestamps(treeID, token, payload) + case ticketvote.CmdLinkedFrom: + return p.cmdLinkedFrom(token) // Internal plugin commands case cmdStartRunoffSub: diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index 3b41117b1..ac31271d5 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -27,6 +27,7 @@ const ( CmdSummary = "summary" // Get vote summary CmdInventory = "inventory" // Get inventory by vote status CmdTimestamps = "timestamps" // Get vote data timestamps + CmdLinkedFrom = "linkedfrom" // Get record linked from list // Plugin setting keys SettingKeyVoteDurationMin = "votedurationmin" @@ -513,3 +514,14 @@ type TimestampsReply struct { Details Timestamp `json:"details,omitempty"` Votes map[string]Timestamp `json:"votes,omitempty"` // [ticket]Timestamp } + +// LinkedFrom requests the linked from list for a record. The only records that +// will have a linked from list are the parent records in a runoff vote. The +// linked from list will contain all runoff vote submissions, i.e. records that +// linked to the runoff parent record using the VoteMetadata.LinkTo field. +type LinkedFrom struct{} + +// LinkedFromReply is the reply to the LinkedFrom command. +type LinkedFromReply struct { + Tokens []string `json:"tokens"` +} From b249ff3ca2b193469c84b6de804bba966f445f30 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 18 Jan 2021 15:15:26 -0600 Subject: [PATCH 234/449] Add runoff vote winner calc. --- politeiad/backend/tlogbe/plugins/pi/hooks.go | 318 ++++++++++ politeiad/backend/tlogbe/plugins/pi/pi.go | 554 ------------------ .../backend/tlogbe/plugins/ticketvote/cmds.go | 365 +++++++++--- .../tlogbe/plugins/ticketvote/hooks.go | 47 +- .../tlogbe/plugins/ticketvote/inventory.go | 8 +- .../tlogbe/plugins/ticketvote/linkedfrom.go | 42 +- .../tlogbe/plugins/ticketvote/summary.go | 12 +- .../tlogbe/plugins/ticketvote/ticketvote.go | 10 +- .../tlogbe/plugins/ticketvote/votes.go | 6 +- politeiad/backend/tlogbe/tlog/plugin.go | 3 + .../backend/tlogbe/tlogclient/tlogclient.go | 10 + 11 files changed, 689 insertions(+), 686 deletions(-) create mode 100644 politeiad/backend/tlogbe/plugins/pi/hooks.go diff --git a/politeiad/backend/tlogbe/plugins/pi/hooks.go b/politeiad/backend/tlogbe/plugins/pi/hooks.go new file mode 100644 index 000000000..8d971c6a6 --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/pi/hooks.go @@ -0,0 +1,318 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package pi + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "strings" + + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" + "github.com/decred/politeia/politeiad/plugins/comments" + "github.com/decred/politeia/politeiad/plugins/pi" + "github.com/decred/politeia/politeiad/plugins/ticketvote" +) + +// proposalMetadataDecode decodes and returns the ProposalMetadata from the +// provided backend files. If a ProposalMetadata is not found, nil is returned. +func proposalMetadataDecode(files []backend.File) (*pi.ProposalMetadata, error) { + var propMD *pi.ProposalMetadata + for _, v := range files { + if v.Name == pi.FileNameProposalMetadata { + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return nil, err + } + var m pi.ProposalMetadata + err = json.Unmarshal(b, &m) + if err != nil { + return nil, err + } + propMD = &m + break + } + } + return propMD, nil +} + +// generalMetadataDecode decodes and returns the GeneralMetadata from the +// provided backend metadata streams. If a GeneralMetadata is not found, nil is +// returned. +func generalMetadataDecode(metadata []backend.MetadataStream) (*pi.GeneralMetadata, error) { + var generalMD *pi.GeneralMetadata + for _, v := range metadata { + if v.ID == pi.MDStreamIDGeneralMetadata { + var gm pi.GeneralMetadata + err := json.Unmarshal([]byte(v.Payload), &gm) + if err != nil { + return nil, err + } + generalMD = &gm + break + } + } + return generalMD, nil +} + +// statusChangesDecode decodes a JSON byte slice into a []StatusChange slice. +func statusChangesDecode(payload []byte) ([]pi.StatusChange, error) { + var statuses []pi.StatusChange + d := json.NewDecoder(strings.NewReader(string(payload))) + for { + var sc pi.StatusChange + err := d.Decode(&sc) + if errors.Is(err, io.EOF) { + break + } else if err != nil { + return nil, err + } + statuses = append(statuses, sc) + } + + return statuses, nil +} + +func (p *piPlugin) hookNewRecordPre(payload string) error { + var nr plugins.HookNewRecordPre + err := json.Unmarshal([]byte(payload), &nr) + if err != nil { + return err + } + + // Verify a proposal metadata has been included + pm, err := proposalMetadataDecode(nr.Files) + if err != nil { + return err + } + if pm == nil { + return fmt.Errorf("proposal metadata not found") + } + + // TODO Verify proposal name + + return nil +} + +func (p *piPlugin) hookNewRecordPost(payload string) error { + var nr plugins.HookNewRecordPost + err := json.Unmarshal([]byte(payload), &nr) + if err != nil { + return err + } + + // Decode GeneralMetadata + gm, err := generalMetadataDecode(nr.Metadata) + if err != nil { + return err + } + if gm == nil { + panic("general metadata not found") + } + + // Add token to the user data cache + err = p.userDataAddToken(gm.UserID, nr.RecordMetadata.Token) + if err != nil { + return err + } + + return nil +} + +func (p *piPlugin) hookEditRecordPre(payload string) error { + /* + var er plugins.HookEditRecord + err := json.Unmarshal([]byte(payload), &er) + if err != nil { + return err + } + + // TODO verify files were changed. Before adding this, verify that + // politeiad will also error if no files were changed. + + // Verify vote status. This is only required for public proposals. + if status == pi.PropStatusPublic { + token := er.RecordMetadata.Token + s := ticketvote.Summaries{ + Tokens: []string{token}, + } + b, err := ticketvote.EncodeSummaries(s) + if err != nil { + return err + } + reply, err := p.backend.Plugin(ticketvote.ID, + ticketvote.CmdSummaries, "", string(b)) + if err != nil { + return fmt.Errorf("ticketvote Summaries: %v", err) + } + sr, err := ticketvote.DecodeSummariesReply([]byte(reply)) + if err != nil { + return err + } + summary, ok := sr.Summaries[token] + if !ok { + return fmt.Errorf("ticketvote summmary not found") + } + if summary.Status != ticketvote.VoteStatusUnauthorized { + e := fmt.Sprintf("vote status got %v, want %v", + ticketvote.VoteStatuses[summary.Status], + ticketvote.VoteStatuses[ticketvote.VoteStatusUnauthorized]) + return backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorCodeVoteStatusInvalid), + ErrorContext: e, + } + } + } + */ + + return nil +} + +func (p *piPlugin) hookSetRecordStatusPost(payload string) error { + var srs plugins.HookSetRecordStatus + err := json.Unmarshal([]byte(payload), &srs) + if err != nil { + return err + } + + // Parse the status change metadata + var sc *pi.StatusChange + for _, v := range srs.MDAppend { + if v.ID != pi.MDStreamIDStatusChanges { + continue + } + + var sc pi.StatusChange + err := json.Unmarshal([]byte(v.Payload), &sc) + if err != nil { + return err + } + break + } + if sc == nil { + return fmt.Errorf("status change append metadata not found") + } + + // Parse the existing status changes metadata stream + var statuses []pi.StatusChange + for _, v := range srs.Current.Metadata { + if v.ID != pi.MDStreamIDStatusChanges { + continue + } + + statuses, err = statusChangesDecode([]byte(v.Payload)) + if err != nil { + return err + } + break + } + + // Verify version is the latest version + if sc.Version != srs.Current.Version { + e := fmt.Sprintf("version not current: got %v, want %v", + sc.Version, srs.Current.Version) + return backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorCodePropVersionInvalid), + ErrorContext: e, + } + } + + // Verify status change is allowed + var from pi.PropStatusT + if len(statuses) == 0 { + // No previous status changes exist. Proposal is unvetted. + from = pi.PropStatusUnvetted + } else { + from = statuses[len(statuses)-1].Status + } + _, isAllowed := pi.StatusChanges[from][sc.Status] + if !isAllowed { + e := fmt.Sprintf("from %v to %v status change not allowed", + from, sc.Status) + return backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorCodePropStatusChangeInvalid), + ErrorContext: e, + } + } + + return nil +} + +// commentWritesVerify verifies that a record's vote status allows writes from +// the comments plugin. +func (p *piPlugin) commentWritesVerify(token []byte) error { + // Verify the ticketvote plugin is registered + var found bool + for _, v := range p.tlog.Plugins() { + if v.ID == ticketvote.ID { + found = true + break + } + } + if !found { + // Ticket vote plugin is not registered. Nothing to verify. + return nil + } + + // Verify that the vote status allows comment writes + vs, err := p.voteSummary(token) + if err != nil { + return err + } + switch vs.Status { + case ticketvote.VoteStatusUnauthorized, ticketvote.VoteStatusAuthorized, + ticketvote.VoteStatusStarted: + // Writes are allowed on these vote statuses + return nil + default: + return backend.PluginUserError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorCodeVoteStatusInvalid), + ErrorContext: "vote has ended; proposal is locked", + } + } +} + +func (p *piPlugin) hookCommentNew(token []byte) error { + return p.commentWritesVerify(token) +} + +func (p *piPlugin) hookCommentDel(token []byte) error { + return p.commentWritesVerify(token) +} + +func (p *piPlugin) hookCommentVote(token []byte) error { + return p.commentWritesVerify(token) +} + +func (p *piPlugin) hookPluginPre(treeID int64, token []byte, payload string) error { + // Decode payload + var hpp plugins.HookPluginPre + err := json.Unmarshal([]byte(payload), &hpp) + if err != nil { + return err + } + + // Call plugin hook + switch hpp.PluginID { + case comments.ID: + switch hpp.Cmd { + case comments.CmdNew: + return p.hookCommentNew(token) + case comments.CmdDel: + return p.hookCommentDel(token) + case comments.CmdVote: + return p.hookCommentVote(token) + } + } + + return nil +} diff --git a/politeiad/backend/tlogbe/plugins/pi/pi.go b/politeiad/backend/tlogbe/plugins/pi/pi.go index de4fa8429..9201d5b64 100644 --- a/politeiad/backend/tlogbe/plugins/pi/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi/pi.go @@ -5,21 +5,16 @@ package pi import ( - "encoding/base64" "encoding/json" - "errors" "fmt" - "io" "os" "path/filepath" - "strings" "sync" "github.com/decred/dcrd/chaincfg/v3" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/politeiad/backend/tlogbe/tlogclient" - "github.com/decred/politeia/politeiad/plugins/comments" "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/ticketvote" "github.com/decred/politeia/util" @@ -47,65 +42,6 @@ func tokenDecode(token string) ([]byte, error) { return util.TokenDecode(util.TokenTypeTlog, token) } -// decodeProposalMetadata decodes and returns the ProposalMetadata from the -// provided backend files. If a ProposalMetadata is not found, nil is returned. -func decodeProposalMetadata(files []backend.File) (*pi.ProposalMetadata, error) { - var propMD *pi.ProposalMetadata - for _, v := range files { - if v.Name == pi.FileNameProposalMetadata { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return nil, err - } - var m pi.ProposalMetadata - err = json.Unmarshal(b, &m) - if err != nil { - return nil, err - } - propMD = &m - break - } - } - return propMD, nil -} - -// decodeGeneralMetadata decodes and returns the GeneralMetadata from the -// provided backend metadata streams. If a GeneralMetadata is not found, nil is -// returned. -func decodeGeneralMetadata(metadata []backend.MetadataStream) (*pi.GeneralMetadata, error) { - var generalMD *pi.GeneralMetadata - for _, v := range metadata { - if v.ID == pi.MDStreamIDGeneralMetadata { - var gm pi.GeneralMetadata - err := json.Unmarshal([]byte(v.Payload), &gm) - if err != nil { - return nil, err - } - generalMD = &gm - break - } - } - return generalMD, nil -} - -// decodeStatusChanges decodes a JSON byte slice into a []StatusChange slice. -func decodeStatusChanges(payload []byte) ([]pi.StatusChange, error) { - var statuses []pi.StatusChange - d := json.NewDecoder(strings.NewReader(string(payload))) - for { - var sc pi.StatusChange - err := d.Decode(&sc) - if errors.Is(err, io.EOF) { - break - } else if err != nil { - return nil, err - } - statuses = append(statuses, sc) - } - - return statuses, nil -} - func convertPropStatusFromMDStatus(s backend.MDStatusT) pi.PropStatusT { var status pi.PropStatusT switch s { @@ -121,110 +57,6 @@ func convertPropStatusFromMDStatus(s backend.MDStatusT) pi.PropStatusT { return status } -func (p *piPlugin) cmdProposals(payload string) (string, error) { - // TODO - /* - ps, err := pi.DecodeProposals([]byte(payload)) - if err != nil { - return "", err - } - - // Verify state - var existsFn func([]byte) bool - switch ps.State { - case pi.PropStateUnvetted: - existsFn = p.backend.UnvettedExists - case pi.PropStateVetted: - existsFn = p.backend.VettedExists - default: - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorCodePropStateInvalid), - } - } - - // Setup the returned map with entries for all tokens that - // correspond to records. - // map[token]ProposalPluginData - proposals := make(map[string]pi.ProposalPluginData, len(ps.Tokens)) - for _, v := range ps.Tokens { - token, err := tokenDecodeAnyLength(v) - if err != nil { - // Not a valid token - continue - } - ok := existsFn(token) - if !ok { - // Record doesn't exists - continue - } - - // Record exists. Include it in the reply. - proposals[v] = pi.ProposalPluginData{} - } - - // Get linked from list for each proposal - for k, v := range proposals { - lf, err := p.linkedFrom(k) - if err != nil { - return "", fmt.Errorf("linkedFrom %v: %v", k, err) - } - - // Convert map to a slice - linkedFrom := make([]string, 0, len(lf.Tokens)) - for token := range lf.Tokens { - linkedFrom = append(linkedFrom, token) - } - - v.LinkedFrom = linkedFrom - proposals[k] = v - } - - // Get comments count for each proposal - for k, v := range proposals { - // Prepare plugin command - c := comments.Count{ - State: comments.StateT(ps.State), - Token: k, - } - b, err := comments.EncodeCount(c) - if err != nil { - return "", err - } - - // Send plugin command - reply, err := p.backend.Plugin(comments.ID, - comments.CmdCount, "", string(b)) - if err != nil { - return "", fmt.Errorf("backend Plugin %v %v: %v", - comments.ID, comments.CmdCount, err) - } - - // Decode reply - cr, err := comments.DecodeCountReply([]byte(reply)) - if err != nil { - return "", err - } - - // Update proposal plugin data - v.Comments = cr.Count - proposals[k] = v - } - - // Prepare reply - pr := pi.ProposalsReply{ - Proposals: proposals, - } - reply, err := pi.EncodeProposalsReply(pr) - if err != nil { - return "", err - } - - return string(reply), nil - */ - return "", nil -} - func (p *piPlugin) voteSummary(token []byte) (*ticketvote.VoteSummary, error) { reply, err := p.backend.VettedPluginCmd(token, ticketvote.ID, ticketvote.CmdSummary, "") @@ -389,390 +221,6 @@ func (p *piPlugin) cmdVoteInventory() (string, error) { return string(reply), nil } -func (p *piPlugin) hookCommentNew(treeID int64, token []byte, payload string) error { - var n comments.New - err := json.Unmarshal([]byte(payload), &n) - if err != nil { - return err - } - - // Verify vote status - vs, err := p.voteSummary(token) - if err != nil { - return fmt.Errorf("voteSummary: %v", err) - } - switch vs.Status { - case ticketvote.VoteStatusUnauthorized, ticketvote.VoteStatusAuthorized, - ticketvote.VoteStatusStarted: - // Comments are allowed on these vote statuses; continue - default: - return backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorCodeVoteStatusInvalid), - ErrorContext: "vote has ended; proposal is locked", - } - } - - return nil -} - -func (p *piPlugin) commentDel(payload string) error { - // TODO - /* - d, err := comments.DecodeDel([]byte(payload)) - if err != nil { - return "", err - } - - // Verifying the state, token, and that the record exists is also - // done in the comments plugin but we duplicate it here so that the - // vote summary request can complete successfully. - - // Verify state - switch d.State { - case comments.StateUnvetted, comments.StateVetted: - // Allowed; continue - default: - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorCodePropStateInvalid), - } - } - - // Verify token - token, err := tokenDecodeAnyLength(d.Token) - if err != nil { - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorCodePropTokenInvalid), - } - } - - // Verify record exists - var exists bool - switch d.State { - case comments.StateUnvetted: - exists = p.backend.UnvettedExists(token) - case comments.StateVetted: - exists = p.backend.VettedExists(token) - default: - // Should not happen. State has already been validated. - return "", fmt.Errorf("invalid state %v", d.State) - } - if !exists { - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorCodePropNotFound), - } - } - - // Verify vote status - if d.State == comments.StateVetted { - vs, err := p.voteSummary(d.Token) - if err != nil { - return "", fmt.Errorf("voteSummary: %v", err) - } - switch vs.Status { - case ticketvote.VoteStatusUnauthorized, ticketvote.VoteStatusAuthorized, - ticketvote.VoteStatusStarted: - // Deleting is allowed on these vote statuses; continue - default: - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorCodeVoteStatusInvalid), - ErrorContext: []string{"vote has ended; proposal is locked"}, - } - } - } - - // Send plugin command - return p.backend.Plugin(comments.ID, comments.CmdDel, "", payload) - */ - return nil -} - -func (p *piPlugin) commentVote(payload string) error { - // TODO - /* - v, err := comments.DecodeVote([]byte(payload)) - if err != nil { - return "", err - } - - // Verifying the state, token, and that the record exists is also - // done in the comments plugin but we duplicate it here so that the - // vote summary request can complete successfully. - - // Verify token - token, err := tokenDecodeAnyLength(v.Token) - if err != nil { - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorCodePropTokenInvalid), - } - } - - // Verify record exists - var record *backend.Record - switch v.State { - case comments.StateUnvetted: - record, err = p.backend.GetUnvetted(token, "") - case comments.StateVetted: - record, err = p.backend.GetVetted(token, "") - default: - // Should not happen. State has already been validated. - return "", fmt.Errorf("invalid state %v", v.State) - } - if err != nil { - if errors.Is(err, backend.ErrRecordNotFound) { - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorCodePropNotFound), - } - } - return "", fmt.Errorf("get record: %v", err) - } - - // Verify record status - status := convertPropStatusFromMDStatus(record.RecordMetadata.Status) - switch status { - case pi.PropStatusPublic: - // Comment votes are only allowed on public proposals; continue - default: - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorCodePropStatusInvalid), - ErrorContext: []string{"proposal is not public"}, - } - } - - // Verify vote status - vs, err := p.voteSummary(v.Token) - if err != nil { - return "", fmt.Errorf("voteSummary: %v", err) - } - switch vs.Status { - case ticketvote.VoteStatusUnauthorized, ticketvote.VoteStatusAuthorized, - ticketvote.VoteStatusStarted: - // Comment votes are allowed on these vote statuses; continue - default: - return "", backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorCodeVoteStatusInvalid), - ErrorContext: []string{"vote has ended; proposal is locked"}, - } - } - - // Send plugin command - return p.backend.Plugin(comments.ID, comments.CmdVote, "", payload) - */ - return nil -} - -func (p *piPlugin) ticketVoteStart(payload string) error { - // TODO If runoff vote, verify that parent record has passed a - // vote itself. This functionality is specific to pi. - - return nil -} - -func (p *piPlugin) hookPluginPre(treeID int64, token []byte, payload string) error { - // Decode payload - var hpp plugins.HookPluginPre - err := json.Unmarshal([]byte(payload), &hpp) - if err != nil { - return err - } - - // Call plugin hook - switch hpp.PluginID { - case comments.ID: - switch hpp.Cmd { - case comments.CmdNew: - return p.hookCommentNew(treeID, token, hpp.Payload) - // case comments.CmdDel: - // return p.commentDel(hpp.Payload) - // case comments.CmdVote: - // return p.commentVote(hpp.Payload) - } - case ticketvote.ID: - switch hpp.Cmd { - // case ticketvote.CmdStart: - // return p.ticketVoteStart(hpp.Payload) - } - } - - return nil -} - -func (p *piPlugin) hookNewRecordPre(payload string) error { - var nr plugins.HookNewRecordPre - err := json.Unmarshal([]byte(payload), &nr) - if err != nil { - return err - } - - // Verify a proposal metadata has been included - pm, err := decodeProposalMetadata(nr.Files) - if err != nil { - return err - } - if pm == nil { - return fmt.Errorf("proposal metadata not found") - } - - // TODO Verify proposal name - - return nil -} - -func (p *piPlugin) hookNewRecordPost(payload string) error { - var nr plugins.HookNewRecordPost - err := json.Unmarshal([]byte(payload), &nr) - if err != nil { - return err - } - - // Decode GeneralMetadata - gm, err := decodeGeneralMetadata(nr.Metadata) - if err != nil { - return err - } - if gm == nil { - panic("general metadata not found") - } - - // Add token to the user data cache - err = p.userDataAddToken(gm.UserID, nr.RecordMetadata.Token) - if err != nil { - return err - } - - return nil -} - -func (p *piPlugin) hookEditRecordPre(payload string) error { - /* - var er plugins.HookEditRecord - err := json.Unmarshal([]byte(payload), &er) - if err != nil { - return err - } - - // TODO verify files were changed. Before adding this, verify that - // politeiad will also error if no files were changed. - - // Verify vote status. This is only required for public proposals. - if status == pi.PropStatusPublic { - token := er.RecordMetadata.Token - s := ticketvote.Summaries{ - Tokens: []string{token}, - } - b, err := ticketvote.EncodeSummaries(s) - if err != nil { - return err - } - reply, err := p.backend.Plugin(ticketvote.ID, - ticketvote.CmdSummaries, "", string(b)) - if err != nil { - return fmt.Errorf("ticketvote Summaries: %v", err) - } - sr, err := ticketvote.DecodeSummariesReply([]byte(reply)) - if err != nil { - return err - } - summary, ok := sr.Summaries[token] - if !ok { - return fmt.Errorf("ticketvote summmary not found") - } - if summary.Status != ticketvote.VoteStatusUnauthorized { - e := fmt.Sprintf("vote status got %v, want %v", - ticketvote.VoteStatuses[summary.Status], - ticketvote.VoteStatuses[ticketvote.VoteStatusUnauthorized]) - return backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorCodeVoteStatusInvalid), - ErrorContext: e, - } - } - } - */ - - return nil -} - -func (p *piPlugin) hookSetRecordStatusPost(payload string) error { - var srs plugins.HookSetRecordStatus - err := json.Unmarshal([]byte(payload), &srs) - if err != nil { - return err - } - - // Parse the status change metadata - var sc *pi.StatusChange - for _, v := range srs.MDAppend { - if v.ID != pi.MDStreamIDStatusChanges { - continue - } - - var sc pi.StatusChange - err := json.Unmarshal([]byte(v.Payload), &sc) - if err != nil { - return err - } - break - } - if sc == nil { - return fmt.Errorf("status change append metadata not found") - } - - // Parse the existing status changes metadata stream - var statuses []pi.StatusChange - for _, v := range srs.Current.Metadata { - if v.ID != pi.MDStreamIDStatusChanges { - continue - } - - statuses, err = decodeStatusChanges([]byte(v.Payload)) - if err != nil { - return err - } - break - } - - // Verify version is the latest version - if sc.Version != srs.Current.Version { - e := fmt.Sprintf("version not current: got %v, want %v", - sc.Version, srs.Current.Version) - return backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorCodePropVersionInvalid), - ErrorContext: e, - } - } - - // Verify status change is allowed - var from pi.PropStatusT - if len(statuses) == 0 { - // No previous status changes exist. Proposal is unvetted. - from = pi.PropStatusUnvetted - } else { - from = statuses[len(statuses)-1].Status - } - _, isAllowed := pi.StatusChanges[from][sc.Status] - if !isAllowed { - e := fmt.Sprintf("from %v to %v status change not allowed", - from, sc.Status) - return backend.PluginUserError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorCodePropStatusChangeInvalid), - ErrorContext: e, - } - } - - return nil -} - // Setup performs any plugin setup work that needs to be done. // // This function satisfies the plugins.Client interface. @@ -828,8 +276,6 @@ func (p *piPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload str func (p *piPlugin) Fsck() error { log.Tracef("Fsck") - // linkedFrom cache - return nil } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go index 4520407d3..7526d4343 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go @@ -43,7 +43,8 @@ const ( dataTypeStartRunoff = "startrunoff" // Internal plugin commands - cmdStartRunoffSub = "startrunoffsub" + cmdStartRunoffSubmission = "startrunoffsub" + cmdRunoffDetails = "runoffdetails" ) // tokenDecode decodes a record token and only accepts full length tokens. @@ -371,6 +372,20 @@ func (p *ticketVotePlugin) voteDetails(treeID int64) (*ticketvote.VoteDetails, e return vd, nil } +func (p *ticketVotePlugin) voteDetailsByToken(token []byte) (*ticketvote.VoteDetails, error) { + reply, err := p.backend.VettedPluginCmd(token, ticketvote.ID, + ticketvote.CmdDetails, "") + if err != nil { + return nil, err + } + var dr ticketvote.DetailsReply + err = json.Unmarshal([]byte(reply), &dr) + if err != nil { + return nil, err + } + return dr.Vote, nil +} + func (p *ticketVotePlugin) castVoteSave(treeID int64, cv ticketvote.CastVoteDetails) error { // Prepare blob be, err := convertBlobEntryFromCastVoteDetails(cv) @@ -407,16 +422,176 @@ func (p *ticketVotePlugin) castVotes(treeID int64) ([]ticketvote.CastVoteDetails return votes, nil } +func (p *ticketVotePlugin) voteOptionResults(token []byte, options []ticketvote.VoteOption) ([]ticketvote.VoteOptionResult, error) { + // Ongoing votes will have the cast votes cached. Calculate the + // results using the cached votes if we can since it will be much + // faster. + var ( + tally = make(map[string]int, len(options)) + votes = p.votesCache(token) + ) + switch { + case len(votes) > 0: + // Vote are in the cache. Tally the results. + for _, voteBit := range votes { + tally[voteBit]++ + } + + default: + // Votes are not in the cache. Pull them from the backend. + reply, err := p.backend.VettedPluginCmd(token, ticketvote.ID, + ticketvote.CmdResults, "") + var rr ticketvote.ResultsReply + err = json.Unmarshal([]byte(reply), &rr) + if err != nil { + return nil, err + } + + // Tally the results + for _, v := range rr.Votes { + tally[v.VoteBit]++ + } + } + + // Prepare reply + results := make([]ticketvote.VoteOptionResult, 0, len(options)) + for _, v := range options { + bit := strconv.FormatUint(v.Bit, 16) + results = append(results, ticketvote.VoteOptionResult{ + ID: v.ID, + Description: v.Description, + VoteBit: v.Bit, + Votes: uint64(tally[bit]), + }) + } + + return results, nil +} + +// voteSummariesForRunoff returns the vote summaries of all submissions in a +// runoff vote. This should only be called once the vote has finished. +func (p *ticketVotePlugin) summariesForRunoff(parentToken string) (map[string]ticketvote.VoteSummary, error) { + // Get runoff vote details + parent, err := tokenDecode(parentToken) + if err != nil { + return nil, err + } + reply, err := p.backend.VettedPluginCmd(parent, ticketvote.ID, + cmdRunoffDetails, "") + if err != nil { + return nil, fmt.Errorf("VettedPluginCmd %x %v %v: %v", + parent, ticketvote.ID, cmdRunoffDetails, err) + } + var rdr runoffDetailsReply + err = json.Unmarshal([]byte(reply), &rdr) + if err != nil { + return nil, err + } + + // Verify submissions exist + subs := rdr.Runoff.Submissions + if len(subs) == 0 { + return map[string]ticketvote.VoteSummary{}, nil + } + + // Compile summaries for all submissions + var ( + summaries = make(map[string]ticketvote.VoteSummary, len(subs)) + winnerNetApprove int // Net number of approve votes of the winner + winnerToken string // Token of the winner + ) + for _, v := range subs { + token, err := tokenDecode(v) + if err != nil { + return nil, err + } + + // Get vote details + vd, err := p.voteDetailsByToken(token) + if err != nil { + return nil, err + } + + // Get vote options results + results, err := p.voteOptionResults(token, vd.Params.Options) + if err != nil { + return nil, err + } + + // Save summary + s := ticketvote.VoteSummary{ + Type: vd.Params.Type, + Status: ticketvote.VoteStatusFinished, + Duration: vd.Params.Duration, + StartBlockHeight: vd.StartBlockHeight, + StartBlockHash: vd.StartBlockHash, + EndBlockHeight: vd.EndBlockHeight, + EligibleTickets: uint32(len(vd.EligibleTickets)), + QuorumPercentage: vd.Params.QuorumPercentage, + PassPercentage: vd.Params.PassPercentage, + Results: results, + Approved: false, + } + summaries[v] = s + + // We now check if this record has the most net yes votes. + + // Verify the vote met quorum and pass requirements + approved := voteIsApproved(*vd, results) + if !approved { + // Vote did not meet quorum and pass requirements. Nothing + // else to do. Record vote is not approved. + continue + } + + // Check if this record has more net approved votes then current + // highest. + var ( + votesApprove uint64 // Number of approve votes + votesReject uint64 // Number of reject votes + ) + for _, vor := range s.Results { + switch vor.ID { + case ticketvote.VoteOptionIDApprove: + votesApprove = vor.Votes + case ticketvote.VoteOptionIDReject: + votesReject = vor.Votes + default: + // Runoff vote options can only be approve/reject + return nil, fmt.Errorf("unknown runoff vote option %v", vor.ID) + } + + netApprove := int(votesApprove) - int(votesReject) + if netApprove > winnerNetApprove { + // New winner! + winnerToken = v + winnerNetApprove = netApprove + } + + // This function doesn't handle the unlikely case that the + // runoff vote results in a tie. + } + } + if winnerToken != "" { + // A winner was found. Mark their summary as approved. + s := summaries[winnerToken] + s.Approved = true + summaries[winnerToken] = s + } + + return summaries, nil +} + // summary returns the vote summary for a record. func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) (*ticketvote.VoteSummary, error) { // Check if the summary has been cached - s, err := p.cachedSummary(hex.EncodeToString(token)) + s, err := p.summaryCache(hex.EncodeToString(token)) switch { case errors.Is(err, errSummaryNotFound): // Cached summary not found. Continue. case err != nil: // Some other error - return nil, fmt.Errorf("cachedSummary: %v", err) + return nil, fmt.Errorf("summaryCache: %v", err) default: // Caches summary was found. Return it. return s, nil @@ -428,7 +603,8 @@ func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) // appropriate record has been found that proves otherwise. status := ticketvote.VoteStatusUnauthorized - // Check if the vote has been authorized + // Check if the vote has been authorized. Not all vote types + // require an authorization. auths, err := p.auths(treeID) if err != nil { return nil, fmt.Errorf("auths: %v", err) @@ -470,21 +646,10 @@ func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) status = ticketvote.VoteStatusFinished } - // Pull the cast votes from the cache and tally the results - votes := p.cachedVotes(token) - tally := make(map[string]int, len(vd.Params.Options)) - for _, voteBit := range votes { - tally[voteBit]++ - } - results := make([]ticketvote.VoteOptionResult, 0, len(vd.Params.Options)) - for _, v := range vd.Params.Options { - bit := strconv.FormatUint(v.Bit, 16) - results = append(results, ticketvote.VoteOptionResult{ - ID: v.ID, - Description: v.Description, - VoteBit: v.Bit, - Votes: uint64(tally[bit]), - }) + // Tally vote results + results, err := p.voteOptionResults(token, vd.Params.Options) + if err != nil { + return nil, err } // Prepare summary @@ -506,36 +671,47 @@ func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) return &summary, nil } - // The vote has finished. We can calculate if the vote was approved - // for certain vote types and cache the results. + // The vote has finished. Find whether the vote was approved and + // cache the vote summary. switch vd.Params.Type { - case ticketvote.VoteTypeStandard, ticketvote.VoteTypeRunoff: - // These vote types are strictly approve/reject votes so we can - // calculate the vote approval. Continue. - default: - // Nothing else to do for all other vote types - return &summary, nil - } + case ticketvote.VoteTypeStandard: + // Standard vote uses a simple approve/reject result + summary.Approved = voteIsApproved(*vd, results) - // Calculate vote approval - approved := voteIsApproved(*vd, results) + // Cache summary + err = p.summaryCacheSave(vd.Params.Token, summary) + if err != nil { + return nil, err + } - // If this is a standard vote then we can take the results as is. - // A runoff vote requires that we pull all other runoff vote - // submissions to determine if the vote actually passed. - // TODO make summary for runoff vote submissions - summary.Approved = approved + // Remove record from votes cache. The votes cache is only for + // records with ongoing votes. + p.votesCacheDel(vd.Params.Token) - // Cache the summary - err = p.cachedSummarySave(vd.Params.Token, summary) - if err != nil { - return nil, fmt.Errorf("cachedSummarySave %v: %v %v", - vd.Params.Token, err, summary) - } + case ticketvote.VoteTypeRunoff: + // A runoff vote requires that we pull all other runoff vote + // submissions to determine if the vote actually passed. + summaries, err := p.summariesForRunoff(vd.Params.Parent) + if err != nil { + return nil, err + } + for k, v := range summaries { + // Cache summary + err = p.summaryCacheSave(k, v) + if err != nil { + return nil, err + } + + // Remove record from votes cache. The votes cache is only for + // records with ongoing votes. + p.votesCacheDel(k) + } - // Remove record from the votes cache now that a summary has been - // saved for it. - p.cachedVotesDel(vd.Params.Token) + summary = summaries[vd.Params.Token] + + default: + return nil, fmt.Errorf("unknown vote type") + } return &summary, nil } @@ -589,8 +765,8 @@ func (p *ticketVotePlugin) bestBlock() (uint32, error) { if err != nil { return 0, err } - reply, err := p.backend.Plugin(dcrdata.ID, - dcrdata.CmdBestBlock, "", string(payload)) + reply, err := p.backend.VettedPluginCmd([]byte{}, dcrdata.ID, + dcrdata.CmdBestBlock, string(payload)) if err != nil { return 0, fmt.Errorf("Plugin %v %v: %v", dcrdata.ID, dcrdata.CmdBestBlock, err) @@ -625,8 +801,8 @@ func (p *ticketVotePlugin) bestBlockUnsafe() (uint32, error) { if err != nil { return 0, err } - reply, err := p.backend.Plugin(dcrdata.ID, - dcrdata.CmdBestBlock, "", string(payload)) + reply, err := p.backend.VettedPluginCmd([]byte{}, dcrdata.ID, + dcrdata.CmdBestBlock, string(payload)) if err != nil { return 0, fmt.Errorf("Plugin %v %v: %v", dcrdata.ID, dcrdata.CmdBestBlock, err) @@ -660,8 +836,8 @@ func (p *ticketVotePlugin) largestCommitmentAddrs(tickets []string) ([]commitmen if err != nil { return nil, err } - reply, err := p.backend.Plugin(dcrdata.ID, - dcrdata.CmdTxsTrimmed, "", string(payload)) + reply, err := p.backend.VettedPluginCmd([]byte{}, dcrdata.ID, + dcrdata.CmdTxsTrimmed, string(payload)) if err != nil { return nil, fmt.Errorf("Plugin %v %v: %v", dcrdata.ID, dcrdata.CmdTxsTrimmed, err) @@ -730,8 +906,8 @@ func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, if err != nil { return nil, err } - reply, err := p.backend.Plugin(dcrdata.ID, - dcrdata.CmdBlockDetails, "", string(payload)) + reply, err := p.backend.VettedPluginCmd([]byte{}, dcrdata.ID, + dcrdata.CmdBlockDetails, string(payload)) if err != nil { return nil, fmt.Errorf("Plugin %v %v: %v", dcrdata.ID, dcrdata.CmdBlockDetails, err) @@ -754,8 +930,8 @@ func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, if err != nil { return nil, err } - reply, err = p.backend.Plugin(dcrdata.ID, - dcrdata.CmdTicketPool, "", string(payload)) + reply, err = p.backend.VettedPluginCmd([]byte{}, dcrdata.ID, + dcrdata.CmdTicketPool, string(payload)) if err != nil { return nil, fmt.Errorf("Plugin %v %v: %v", dcrdata.ID, dcrdata.CmdTicketPool, err) @@ -901,9 +1077,9 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri // Update inventory switch a.Action { case ticketvote.AuthActionAuthorize: - p.inventorySetToAuthorized(a.Token) + p.invCacheSetToAuthorized(a.Token) case ticketvote.AuthActionRevoke: - p.inventorySetToUnauthorized(a.Token) + p.invCacheSetToUnauthorized(a.Token) default: // Should not happen e := fmt.Sprintf("invalid authorize action: %v", a.Action) @@ -947,7 +1123,6 @@ func voteBitVerify(options []ticketvote.VoteOption, mask, bit uint64) error { return fmt.Errorf("bit 0x%x not found in vote options", bit) } -// TODO test this function func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationMax uint32) error { // Verify vote type switch vote.Type { @@ -1218,7 +1393,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot } // Update inventory - p.inventorySetToStarted(vd.Params.Token, ticketvote.VoteTypeStandard, + p.invCacheSetToStarted(vd.Params.Token, ticketvote.VoteTypeStandard, vd.EndBlockHeight) return &ticketvote.StartReply{ @@ -1290,14 +1465,37 @@ func (p *ticketVotePlugin) startRunoffRecord(treeID int64) (*startRunoffRecord, return srr, nil } -// startRunoffSub is an internal plugin command that is used to start the -// voting period on a runoff vote submission. -type startRunoffSub struct { - ParentTreeID int64 `json:"parenttreeid"` - StartDetails ticketvote.StartDetails `json:"startdetails"` +// runoffDetails is an internal plugin command that requests the details of a +// runoff vote. +type runoffDetails struct{} + +// runoffDetailsReply is the reply to the runoffDetails command. +type runoffDetailsReply struct { + Runoff startRunoffRecord `json:"runoff"` +} + +func (p *ticketVotePlugin) cmdRunoffDetails(treeID int64) (string, error) { + log.Tracef("cmdRunoffDetails: %x", treeID) + + // Get start runoff record + srs, err := p.startRunoffRecord(treeID) + if err != nil { + return "", err + } + + // Prepare reply + r := runoffDetailsReply{ + Runoff: *srs, + } + reply, err := json.Marshal(r) + if err != nil { + return "", err + } + + return string(reply), nil } -func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs startRunoffSub) error { +func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs startRunoffSubmission) error { // Sanity check s := srs.StartDetails t, err := tokenDecode(s.Params.Token) @@ -1377,7 +1575,7 @@ func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs sta } // Update inventory - p.inventorySetToStarted(vd.Params.Token, ticketvote.VoteTypeRunoff, + p.invCacheSetToStarted(vd.Params.Token, ticketvote.VoteTypeRunoff, vd.EndBlockHeight) return nil @@ -1427,7 +1625,7 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti } } } - vm, err := decodeVoteMetadata(r.Files) + vm, err := voteMetadataDecode(r.Files) if err != nil { return nil, err } @@ -1461,7 +1659,7 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti // linked to the parent record. The parent record's linked from // list will include abandoned proposals that need to be filtered // out. - lf, err := p.linkedFrom(token) + lf, err := p.linkedFromCache(token) if err != nil { return nil, err } @@ -1674,13 +1872,13 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. } // Start the voting period of each runoff vote submissions by - // using the internal plugin command startRunoffSub. + // using the internal plugin command startRunoffSubmission. for _, v := range s.Starts { token, err = tokenDecode(v.Params.Token) if err != nil { return nil, err } - srs := startRunoffSub{ + srs := startRunoffSubmission{ ParentTreeID: treeID, StartDetails: v, } @@ -1689,14 +1887,14 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. return nil, err } _, err = p.backend.VettedPluginCmd(token, ticketvote.ID, - cmdStartRunoffSub, string(b)) + cmdStartRunoffSubmission, string(b)) if err != nil { var ue backend.PluginUserError if errors.As(err, &ue) { return nil, err } return nil, fmt.Errorf("VettedPluginCmd %x %v %v %v: %v", - token, ticketvote.ID, cmdStartRunoffSub, b, err) + token, ticketvote.ID, cmdStartRunoffSubmission, b, err) } } @@ -1708,11 +1906,18 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. }, nil } -func (p *ticketVotePlugin) cmdStartRunoffSub(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdStartRunoffSub: %v %x %v", treeID, token, payload) +// startRunoffSubmission is an internal plugin command that is used to start +// the voting period on a runoff vote submission. +type startRunoffSubmission struct { + ParentTreeID int64 `json:"parenttreeid"` + StartDetails ticketvote.StartDetails `json:"startdetails"` +} + +func (p *ticketVotePlugin) cmdStartRunoffSubmission(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdStartRunoffSubmission: %v %x %v", treeID, token, payload) // Decode payload - var srs startRunoffSub + var srs startRunoffSubmission err := json.Unmarshal([]byte(payload), &srs) if err != nil { return "", err @@ -1903,7 +2108,7 @@ func (p *ticketVotePlugin) ballot(treeID int64, votes []ticketvote.CastVote, res cvr.Receipt = cv.Receipt // Update cast votes cache - p.cachedVotesSet(v.Token, v.Ticket, v.VoteBit) + p.votesCacheSet(v.Token, v.Ticket, v.VoteBit) sendResult: // Send result back to calling function @@ -2098,8 +2303,8 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str m.Lock() defer m.Unlock() - // cachedVotes contains the tickets that have alread voted - cachedVotes := p.cachedVotes(token) + // votesCache contains the tickets that have alread voted + votesCache := p.votesCache(token) for k, v := range votes { if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { // Vote has an error. Skip it. @@ -2107,7 +2312,7 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str } // Verify ticket has not already vote - _, ok := cachedVotes[v.Ticket] + _, ok := votesCache[v.Ticket] if ok { e := ticketvote.VoteErrorTicketAlreadyVoted receipts[k].Ticket = v.Ticket @@ -2412,7 +2617,7 @@ func (p *ticketVotePlugin) cmdInventory() (string, error) { } // Get the inventory - inv, err := p.inventory(bb) + inv, err := p.invCache(bb) if err != nil { return "", fmt.Errorf("inventory: %v", err) } @@ -2515,7 +2720,7 @@ func (p *ticketVotePlugin) cmdLinkedFrom(token []byte) (string, error) { log.Tracef("cmdLinkedFrom: %x", token) // Get linked from list - lf, err := p.linkedFrom(token) + lf, err := p.linkedFromCache(token) if err != nil { return "", err } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go b/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go index e4c6b199a..e40606af2 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go @@ -16,9 +16,9 @@ import ( "github.com/decred/politeia/politeiad/plugins/ticketvote" ) -// decodeVoteMetadata decodes and returns the VoteMetadata from the +// voteMetadataDecode decodes and returns the VoteMetadata from the // provided backend files. If a VoteMetadata is not found, nil is returned. -func decodeVoteMetadata(files []backend.File) (*ticketvote.VoteMetadata, error) { +func voteMetadataDecode(files []backend.File) (*ticketvote.VoteMetadata, error) { var voteMD *ticketvote.VoteMetadata for _, v := range files { if v.Name == ticketvote.FileNameVoteMetadata { @@ -67,8 +67,7 @@ func (p *ticketVotePlugin) linkByVerify(linkBy int64) error { } func (p *ticketVotePlugin) linkToVerify(linkTo string) error { - // LinkTo must be a public record that is the parent of a runoff - // vote, i.e. has the VoteMetadata.LinkBy field set. + // LinkTo must be a public record token, err := tokenDecode(linkTo) if err != nil { return backend.PluginUserError{ @@ -95,7 +94,10 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { ErrorContext: "record is censored", } } - parentVM, err := decodeVoteMetadata(r.Files) + + // LinkTo must be a runoff vote parent record, i.e. has specified + // a LinkBy deadline. + parentVM, err := voteMetadataDecode(r.Files) if err != nil { return err } @@ -106,14 +108,29 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { ErrorContext: "record not a runoff vote parent", } } + + // The LinkBy deadline must not be expired if time.Now().Unix() > parentVM.LinkBy { - // Linkby deadline has expired. New links are not allowed. return backend.PluginUserError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), ErrorContext: "parent record linkby deadline has expired", } } + + // The runoff vote parent record must have been approved in a vote. + vs, err := p.summaryByToken(token) + if err != nil { + return err + } + if !vs.Approved { + return backend.PluginUserError{ + PluginID: ticketvote.ID, + ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), + ErrorContext: "parent record vote is not approved", + } + } + return nil } @@ -159,7 +176,7 @@ func (p *ticketVotePlugin) hookNewRecordPre(payload string) error { } // Verify the vote metadata if the record contains one - vm, err := decodeVoteMetadata(nr.Files) + vm, err := voteMetadataDecode(nr.Files) if err != nil { return err } @@ -188,14 +205,14 @@ func (p *ticketVotePlugin) hookEditRecordPre(payload string) error { oldLinkTo string newLinkTo string ) - vm, err := decodeVoteMetadata(er.Current.Files) + vm, err := voteMetadataDecode(er.Current.Files) if err != nil { return err } if vm != nil { oldLinkTo = vm.LinkTo } - vm, err = decodeVoteMetadata(er.FilesAdd) + vm, err = voteMetadataDecode(er.FilesAdd) if err != nil { return err } @@ -215,7 +232,7 @@ func (p *ticketVotePlugin) hookEditRecordPre(payload string) error { // Verify LinkBy if one was included. The VoteMetadata is optional // so the record may not contain one. - vm, err := decodeVoteMetadata(er.FilesAdd) + vm, err := voteMetadataDecode(er.FilesAdd) if err != nil { return err } @@ -237,7 +254,7 @@ func (p *ticketVotePlugin) hookSetRecordStatusPost(payload string) error { } // Check if the LinkTo has been set - vm, err := decodeVoteMetadata(srs.Current.Files) + vm, err := voteMetadataDecode(srs.Current.Files) if err != nil { return err } @@ -252,16 +269,16 @@ func (p *ticketVotePlugin) hookSetRecordStatusPost(payload string) error { case backend.MDStatusVetted: // Record has been made public. Add child token to parent's // linked from list. - err := p.linkedFromAdd(parentToken, childToken) + err := p.linkedFromCacheAdd(parentToken, childToken) if err != nil { - return fmt.Errorf("linkedFromAdd: %v", err) + return fmt.Errorf("linkedFromCacheAdd: %v", err) } case backend.MDStatusCensored: // Record has been censored. Delete child token from parent's // linked from list. - err := p.linkedFromDel(parentToken, childToken) + err := p.linkedFromCacheDel(parentToken, childToken) if err != nil { - return fmt.Errorf("linkedFromDel: %v", err) + return fmt.Errorf("linkedFromCacheDel: %v", err) } } } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go b/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go index e8f2ec746..f2bf08fd3 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go @@ -22,7 +22,7 @@ type inventory struct { bestBlock uint32 // Height of last inventory update } -func (p *ticketVotePlugin) inventorySetToAuthorized(token string) { +func (p *ticketVotePlugin) invCacheSetToAuthorized(token string) { p.Lock() defer p.Unlock() @@ -54,7 +54,7 @@ func (p *ticketVotePlugin) inventorySetToAuthorized(token string) { log.Debugf("ticketvote: added to authorized inv: %v", token) } -func (p *ticketVotePlugin) inventorySetToUnauthorized(token string) { +func (p *ticketVotePlugin) invCacheSetToUnauthorized(token string) { p.Lock() defer p.Unlock() @@ -87,7 +87,7 @@ func (p *ticketVotePlugin) inventorySetToUnauthorized(token string) { log.Debugf("ticketvote: added to unauthorized inv: %v", token) } -func (p *ticketVotePlugin) inventorySetToStarted(token string, t ticketvote.VoteT, endHeight uint32) { +func (p *ticketVotePlugin) invCacheSetToStarted(token string, t ticketvote.VoteT, endHeight uint32) { p.Lock() defer p.Unlock() @@ -152,7 +152,7 @@ func (p *ticketVotePlugin) inventorySetToStarted(token string, t ticketvote.Vote log.Debugf("ticketvote: added to started inv: %v", token) } -func (p *ticketVotePlugin) inventory(bestBlock uint32) (*inventory, error) { +func (p *ticketVotePlugin) invCache(bestBlock uint32) (*inventory, error) { p.Lock() defer p.Unlock() diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/linkedfrom.go b/politeiad/backend/tlogbe/plugins/ticketvote/linkedfrom.go index ef6f965c3..bc7b5b3e4 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/linkedfrom.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/linkedfrom.go @@ -34,22 +34,22 @@ type linkedFrom struct { Tokens map[string]struct{} `json:"tokens"` } -// linkedFromPath returns the path to the linkedFrom list for the provided +// linkedFromCachePath returns the path to the linked fromlist for the provided // record token. The token prefix is used in the file path so that the linked // from list can be retrieved using either the full token or the token prefix. -func (p *ticketVotePlugin) linkedFromPath(token []byte) string { +func (p *ticketVotePlugin) linkedFromCachePath(token []byte) string { t := util.TokenPrefix(token) fn := strings.Replace(fnLinkedFrom, "{tokenprefix}", t, 1) return filepath.Join(p.dataDir, fn) } -// linkedFromWithLock return the linked from list for a record token. If a +// linkedFromCacheWithLock return the linked from list for a record token. If a // linked from list does not exist for the token then an empty list will be // returned. // // This function must be called WITH the lock held. -func (p *ticketVotePlugin) linkedFromWithLock(token []byte) (*linkedFrom, error) { - fp := p.linkedFromPath(token) +func (p *ticketVotePlugin) linkedFromCacheWithLock(token []byte) (*linkedFrom, error) { + fp := p.linkedFromCachePath(token) b, err := ioutil.ReadFile(fp) if err != nil { var e *os.PathError @@ -70,34 +70,34 @@ func (p *ticketVotePlugin) linkedFromWithLock(token []byte) (*linkedFrom, error) return &lf, nil } -// linkedFrom return the linked from list for a record token. If a linked from -// list does not exist for the token then an empty list will be returned. +// linkedFromCache return the linked from list for a record token. If a linked +// from list does not exist for the token then an empty list will be returned. // // This function must be called WITHOUT the lock held. -func (p *ticketVotePlugin) linkedFrom(token []byte) (*linkedFrom, error) { +func (p *ticketVotePlugin) linkedFromCache(token []byte) (*linkedFrom, error) { p.Lock() defer p.Unlock() - return p.linkedFromWithLock(token) + return p.linkedFromCacheWithLock(token) } -// linkedFromSaveWithLock saves a linkedFrom to the plugin data dir. +// linkedFromCacheSaveWithLock saves a linkedFrom to the plugin data dir. // // This function must be called WITH the lock held. -func (p *ticketVotePlugin) linkedFromSaveWithLock(token []byte, lf linkedFrom) error { +func (p *ticketVotePlugin) linkedFromCacheSaveWithLock(token []byte, lf linkedFrom) error { b, err := json.Marshal(lf) if err != nil { return err } - fp := p.linkedFromPath(token) + fp := p.linkedFromCachePath(token) return ioutil.WriteFile(fp, b, 0664) } -// linkedFromAdd updates the cached linkedFrom list for the parentToken, adding -// the childToken to the list. The full length token MUST be used. +// linkedFromCacheAdd updates the cached linkedFrom list for the parentToken, +// adding the childToken to the list. The full length token MUST be used. // // This function must be called WITHOUT the lock held. -func (p *ticketVotePlugin) linkedFromAdd(parentToken, childToken string) error { +func (p *ticketVotePlugin) linkedFromCacheAdd(parentToken, childToken string) error { p.Lock() defer p.Unlock() @@ -112,7 +112,7 @@ func (p *ticketVotePlugin) linkedFromAdd(parentToken, childToken string) error { } // Get existing linked from list - lf, err := p.linkedFromWithLock(parent) + lf, err := p.linkedFromCacheWithLock(parent) if err != nil { return err } @@ -121,7 +121,7 @@ func (p *ticketVotePlugin) linkedFromAdd(parentToken, childToken string) error { lf.Tokens[childToken] = struct{}{} // Save list - err = p.linkedFromSaveWithLock(parent, *lf) + err = p.linkedFromCacheSaveWithLock(parent, *lf) if err != nil { return err } @@ -132,11 +132,11 @@ func (p *ticketVotePlugin) linkedFromAdd(parentToken, childToken string) error { return nil } -// linkedFromDel updates the cached linkedFrom list for the parentToken, +// linkedFromCacheDel updates the cached linkedFrom list for the parentToken, // deleting the childToken from the list. // // This function must be called WITHOUT the lock held. -func (p *ticketVotePlugin) linkedFromDel(parentToken, childToken string) error { +func (p *ticketVotePlugin) linkedFromCacheDel(parentToken, childToken string) error { p.Lock() defer p.Unlock() @@ -151,7 +151,7 @@ func (p *ticketVotePlugin) linkedFromDel(parentToken, childToken string) error { } // Get existing linked from list - lf, err := p.linkedFromWithLock(parent) + lf, err := p.linkedFromCacheWithLock(parent) if err != nil { return err } @@ -160,7 +160,7 @@ func (p *ticketVotePlugin) linkedFromDel(parentToken, childToken string) error { delete(lf.Tokens, childToken) // Save list - err = p.linkedFromSaveWithLock(parent, *lf) + err = p.linkedFromCacheSaveWithLock(parent, *lf) if err != nil { return err } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/summary.go b/politeiad/backend/tlogbe/plugins/ticketvote/summary.go index a799bbc20..d5d3ba0ea 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/summary.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/summary.go @@ -23,9 +23,9 @@ const ( filenameSummary = "{tokenprefix}-summary.json" ) -// cachedSummaryPath accepts both full tokens and token prefixes, however it +// summaryCachePath accepts both full tokens and token prefixes, however it // always uses the token prefix when generatig the path. -func (p *ticketVotePlugin) cachedSummaryPath(token string) (string, error) { +func (p *ticketVotePlugin) summaryCachePath(token string) (string, error) { // Use token prefix t, err := tokenDecodeAnyLength(token) if err != nil { @@ -40,11 +40,11 @@ var ( errSummaryNotFound = errors.New("summary not found") ) -func (p *ticketVotePlugin) cachedSummary(token string) (*ticketvote.VoteSummary, error) { +func (p *ticketVotePlugin) summaryCache(token string) (*ticketvote.VoteSummary, error) { p.Lock() defer p.Unlock() - fp, err := p.cachedSummaryPath(token) + fp, err := p.summaryCachePath(token) if err != nil { return nil, err } @@ -67,7 +67,7 @@ func (p *ticketVotePlugin) cachedSummary(token string) (*ticketvote.VoteSummary, return &vs, nil } -func (p *ticketVotePlugin) cachedSummarySave(token string, vs ticketvote.VoteSummary) error { +func (p *ticketVotePlugin) summaryCacheSave(token string, vs ticketvote.VoteSummary) error { b, err := json.Marshal(vs) if err != nil { return err @@ -76,7 +76,7 @@ func (p *ticketVotePlugin) cachedSummarySave(token string, vs ticketvote.VoteSum p.Lock() defer p.Unlock() - fp, err := p.cachedSummaryPath(token) + fp, err := p.summaryCachePath(token) if err != nil { return err } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go index 3deb9b511..27f6fa68b 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go @@ -174,7 +174,7 @@ func (p *ticketVotePlugin) Setup() error { return err } for _, v := range rr.Votes { - p.cachedVotesSet(v.Token, v.Ticket, v.VoteBit) + p.votesCacheSet(v.Token, v.Ticket, v.VoteBit) } } @@ -208,8 +208,10 @@ func (p *ticketVotePlugin) Cmd(treeID int64, token []byte, cmd, payload string) return p.cmdLinkedFrom(token) // Internal plugin commands - case cmdStartRunoffSub: - return p.cmdStartRunoffSub(treeID, token, payload) + case cmdStartRunoffSubmission: + return p.cmdStartRunoffSubmission(treeID, token, payload) + case cmdRunoffDetails: + return p.cmdRunoffDetails(treeID) } return "", backend.ErrPluginCmdInvalid @@ -239,6 +241,8 @@ func (p *ticketVotePlugin) Hook(treeID int64, token []byte, h plugins.HookT, pay func (p *ticketVotePlugin) Fsck() error { log.Tracef("Fsck") + // Verify all caches + return nil } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/votes.go b/politeiad/backend/tlogbe/plugins/ticketvote/votes.go index 7d75b7a62..74d781e28 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/votes.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/votes.go @@ -8,7 +8,7 @@ import ( "encoding/hex" ) -func (p *ticketVotePlugin) cachedVotes(token []byte) map[string]string { +func (p *ticketVotePlugin) votesCache(token []byte) map[string]string { p.Lock() defer p.Unlock() @@ -25,7 +25,7 @@ func (p *ticketVotePlugin) cachedVotes(token []byte) map[string]string { return c } -func (p *ticketVotePlugin) cachedVotesSet(token, ticket, voteBit string) { +func (p *ticketVotePlugin) votesCacheSet(token, ticket, voteBit string) { p.Lock() defer p.Unlock() @@ -40,7 +40,7 @@ func (p *ticketVotePlugin) cachedVotesSet(token, ticket, voteBit string) { token, ticket, voteBit) } -func (p *ticketVotePlugin) cachedVotesDel(token string) { +func (p *ticketVotePlugin) votesCacheDel(token string) { p.Lock() defer p.Unlock() diff --git a/politeiad/backend/tlogbe/tlog/plugin.go b/politeiad/backend/tlogbe/tlog/plugin.go index 25d08e481..289c36750 100644 --- a/politeiad/backend/tlogbe/tlog/plugin.go +++ b/politeiad/backend/tlogbe/tlog/plugin.go @@ -165,6 +165,9 @@ func (t *Tlog) PluginCmd(treeID int64, token []byte, pluginID, cmd, payload stri return p.client.Cmd(treeID, token, cmd, payload) } +// Plugins returns all registered plugins for the tlog instance. +// +// This function satisfies the tlogclient.Client interface. func (t *Tlog) Plugins() []backend.Plugin { log.Tracef("%v Plugins", t.id) diff --git a/politeiad/backend/tlogbe/tlogclient/tlogclient.go b/politeiad/backend/tlogbe/tlogclient/tlogclient.go index 16e4bdc0c..0aae94b1b 100644 --- a/politeiad/backend/tlogbe/tlogclient/tlogclient.go +++ b/politeiad/backend/tlogbe/tlogclient/tlogclient.go @@ -9,6 +9,13 @@ import ( "github.com/decred/politeia/politeiad/backend/tlogbe/store" ) +// TODO plugins should only have access to the backend methods for the tlog +// instance that they're registered on. +// TODO the plugin hook state should not really be required. This issue is that +// some vetted plugins require unvetted hooks, ex. verifying the linkto in +// vote metadata. Possile solution, keep the layer violations in the +// application plugin (pi) instead of the functionality plugin (ticketvote). + // Client provides an API for plugins to interact with a tlog instance. // Plugins are allowed to save, delete, and get plugin data to/from the tlog // backend. Editing plugin data is not allowed. @@ -40,4 +47,7 @@ type Client interface { // Timestamp returns the timestamp for the blob that correpsonds // to the digest. Timestamp(treeID int64, digest []byte) (*backend.Timestamp, error) + + // Plugins returns all registered plugins for the tlog instance. + Plugins() []backend.Plugin } From 571a5d1ddc1fdedc9154118e8e66be627fc58ead Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 18 Jan 2021 18:46:45 -0600 Subject: [PATCH 235/449] politeiad: Add plugin batch route. --- politeiad/api/v1/v1.go | 93 +++++++-- politeiad/backend/backend.go | 6 +- politeiad/backend/gitbe/gitbe.go | 8 +- politeiad/backend/tlogbe/inventory.go | 156 ++++++++++++++ .../tlogbe/plugins/comments/comments.go | 50 ++--- politeiad/backend/tlogbe/plugins/pi/hooks.go | 13 -- .../backend/tlogbe/plugins/ticketvote/cmds.go | 88 ++++---- .../tlogbe/plugins/ticketvote/hooks.go | 22 +- politeiad/backend/tlogbe/testing.go | 2 +- politeiad/backend/tlogbe/tlog/plugin.go | 4 +- politeiad/backend/tlogbe/tlogbe.go | 144 +------------ .../backend/tlogbe/tlogclient/tlogclient.go | 3 - politeiad/politeiad.go | 192 +++++++++++++++--- politeiawww/api/comments/v1/v1.go | 2 +- politeiawww/api/records/v1/v1.go | 2 +- politeiawww/api/ticketvote/v1/v1.go | 2 +- 16 files changed, 494 insertions(+), 293 deletions(-) create mode 100644 politeiad/backend/tlogbe/inventory.go diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index 2784dcbc0..cf0fffe9f 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -9,6 +9,7 @@ import ( "encoding/base64" "encoding/hex" "errors" + "fmt" "regexp" "github.com/decred/dcrtime/merkle" @@ -21,24 +22,25 @@ type RecordStatusT int const ( // Routes - IdentityRoute = "/v1/identity/" // Retrieve identity - NewRecordRoute = "/v1/newrecord/" // New record - UpdateUnvettedRoute = "/v1/updateunvetted/" // Update unvetted record - UpdateUnvettedMetadataRoute = "/v1/updateunvettedmd/" // Update unvetted metadata - UpdateVettedRoute = "/v1/updatevetted/" // Update vetted record - UpdateVettedMetadataRoute = "/v1/updatevettedmd/" // Update vetted metadata - GetUnvettedRoute = "/v1/getunvetted/" // Retrieve unvetted record - GetVettedRoute = "/v1/getvetted/" // Retrieve vetted record - GetUnvettedTimestampsRoute = "/v1/getunvettedts/" - GetVettedTimestampsRoute = "/v1/getvettedts/" - InventoryByStatusRoute = "/v1/inventorybystatus/" + IdentityRoute = "/v1/identity/" // Retrieve identity + NewRecordRoute = "/v1/newrecord/" // New record + UpdateUnvettedRoute = "/v1/updateunvetted/" // Update unvetted record + UpdateUnvettedMetadataRoute = "/v1/updateunvettedmd/" // Update unvetted metadata + UpdateVettedRoute = "/v1/updatevetted/" // Update vetted record + UpdateVettedMetadataRoute = "/v1/updatevettedmd/" // Update vetted metadata + GetUnvettedRoute = "/v1/getunvetted/" // Retrieve unvetted record + GetVettedRoute = "/v1/getvetted/" // Retrieve vetted record + GetUnvettedTimestampsRoute = "/v1/getunvettedts/" // Get unvetted timestamps + GetVettedTimestampsRoute = "/v1/getvettedts/" // Get vetted timestamps + InventoryByStatusRoute = "/v1/inventorybystatus/" // Get token inventory // Auth required - InventoryRoute = "/v1/inventory/" // Inventory records - SetUnvettedStatusRoute = "/v1/setunvettedstatus/" // Set unvetted status - SetVettedStatusRoute = "/v1/setvettedstatus/" // Set vetted status - PluginCommandRoute = "/v1/plugin/" // Send a command to a plugin - PluginInventoryRoute = "/v1/plugin/inventory/" // Inventory all plugins + InventoryRoute = "/v1/inventory/" // Inventory records + SetUnvettedStatusRoute = "/v1/setunvettedstatus/" // Set unvetted status + SetVettedStatusRoute = "/v1/setvettedstatus/" // Set vetted status + PluginCommandRoute = "/v1/plugin/" // Send a command to a plugin + PluginCommandBatchRoute = "/v1/plugin/batch" // Send a batch of plugin cmds + PluginInventoryRoute = "/v1/plugin/inventory/" // Inventory all plugins ChallengeSize = 32 // Size of challenge token in bytes @@ -69,6 +71,12 @@ const ( ErrorStatusInvalidRPCCredentials ErrorStatusT = 16 ErrorStatusRecordNotFound ErrorStatusT = 17 ErrorStatusInvalidToken ErrorStatusT = 18 + ErrorStatusRecordLocked ErrorStatusT = 19 + ErrorStatusInvalidRecordState ErrorStatusT = 20 + + // Record states + RecordStateUnvetted = "unvetted" + RecordStateVetted = "vetted" // Record status codes (set and get) RecordStatusInvalid RecordStatusT = 0 // Invalid status @@ -478,20 +486,35 @@ type UserErrorReply struct { ErrorContext []string `json:"errorcontext,omitempty"` // Additional error information } +// Error satisfies the error interface. +func (e UserErrorReply) Error() string { + return fmt.Sprintf("user error code: %v", e.ErrorCode) +} + // PluginUserErrorReply returns details about a plugin error that occurred // while trying to execute a command due to bad input from the client. -type PluginUserErrorReply struct { - Plugin string `json:"plugin"` +type PluginErrorReply struct { + PluginID string `json:"plugin"` ErrorCode int `json:"errorcode"` ErrorContext []string `json:"errorcontext,omitempty"` } +// Error satisfies the error interface. +func (e PluginErrorReply) Error() string { + return fmt.Sprintf("plugin user error code: %v", e.ErrorCode) +} + // ServerErrorReply returns an error code that can be correlated with // server logs. type ServerErrorReply struct { ErrorCode int64 `json:"code"` // Server error code } +// Error satisfies the error interface. +func (e ServerErrorReply) Error() string { + return fmt.Sprintf("server error: %v", e.ErrorCode) +} + // PluginSetting is a structure that holds key/value pairs of a plugin setting. type PluginSetting struct { Key string `json:"key"` // Name of setting @@ -533,3 +556,37 @@ type PluginCommandReply struct { CommandID string `json:"commandid"` // User setable command identifier Payload string `json:"payload"` // Actual command reply } + +// PluginCommandV2 sends a command to a plugin. +type PluginCommandV2 struct { + State string `json:"state"` // Unvetted or vetted + Token string `json:"token"` // Censorship token + ID string `json:"id"` // Plugin identifier + Command string `json:"command"` // Plugin command + Payload string `json:"payload"` // Command payload +} + +// PluginCommandReplyV2 is the reply to a PluginCommandV2. +type PluginCommandReplyV2 struct { + State string `json:"state"` // Unvetted or vetted + Token string `json:"token"` // Censorship token + ID string `json:"id"` // Plugin identifier + Command string `json:"command"` // Plugin command + Payload string `json:"payload"` // Response payload + + // Error will only be present if an error occured while executing + // the plugin command on a batched request. + Error error `json:"error,omitempty"` +} + +// PluginCommandBatch executes a batch of plugin commands. +type PluginCommandBatch struct { + Challenge string `json:"challenge"` // Random challenge + Commands []PluginCommandV2 `json:"commands"` +} + +// PluginCommandBatchReply is the reply to a PluginCommandBatch. +type PluginCommandBatchReply struct { + Response string `json:"response"` // Challenge response + Replies []PluginCommandReplyV2 `json:"replies"` +} diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index f1784eb36..bdc3dab61 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -111,15 +111,15 @@ func (s StateTransitionError) Error() string { s.From, MDStatus[s.From], s.To, MDStatus[s.To]) } -// PluginUserError represents a plugin error that is caused by the user. -type PluginUserError struct { +// PluginError represents a plugin error that is caused by the user. +type PluginError struct { PluginID string ErrorCode int ErrorContext string } // Error satisfies the error interface. -func (e PluginUserError) Error() string { +func (e PluginError) Error() string { return fmt.Sprintf("plugin id '%v' error code %v", e.PluginID, e.ErrorCode) } diff --git a/politeiad/backend/gitbe/gitbe.go b/politeiad/backend/gitbe/gitbe.go index abf85f198..aeebadde5 100644 --- a/politeiad/backend/gitbe/gitbe.go +++ b/politeiad/backend/gitbe/gitbe.go @@ -2875,17 +2875,17 @@ func (g *gitBackEnd) SetupVettedPlugin(pluginID string) error { return fmt.Errorf("not implemented") } -// UnvettedPlugin has not been implemented. +// UnvettedPluginCmd has not been implemented. // // This function satisfies the backend.Backend interface. -func (g *gitBackEnd) UnvettedPlugin(token []byte, pluginID, cmd, payload string) (string, error) { +func (g *gitBackEnd) UnvettedPluginCmd(token []byte, pluginID, cmd, payload string) (string, error) { return "", fmt.Errorf("not implemented") } -// VettedPlugin has not been implemented. +// VettedPluginCmd has not been implemented. // // This function satisfies the backend.Backend interface. -func (g *gitBackEnd) VettedPlugin(token []byte, pluginID, cmd, payload string) (string, error) { +func (g *gitBackEnd) VettedPluginCmd(token []byte, pluginID, cmd, payload string) (string, error) { return "", fmt.Errorf("not implemented") } diff --git a/politeiad/backend/tlogbe/inventory.go b/politeiad/backend/tlogbe/inventory.go new file mode 100644 index 000000000..c54009530 --- /dev/null +++ b/politeiad/backend/tlogbe/inventory.go @@ -0,0 +1,156 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlogbe + +import ( + "encoding/hex" + "fmt" + + "github.com/decred/politeia/politeiad/backend" +) + +const ( + // Record states + stateUnvetted = "unvetted" + stateVetted = "vetted" +) + +// inventory contains the tokens of all records in the inventory catagorized by +// MDStatusT. +type inventory struct { + unvetted map[backend.MDStatusT][]string + vetted map[backend.MDStatusT][]string +} + +func (t *tlogBackend) inventory() inventory { + t.RLock() + defer t.RUnlock() + + // Return a copy of the inventory + var ( + unvetted = make(map[backend.MDStatusT][]string, len(t.inv.unvetted)) + vetted = make(map[backend.MDStatusT][]string, len(t.inv.vetted)) + ) + for status, tokens := range t.inv.unvetted { + s := make([]string, len(tokens)) + copy(s, tokens) + unvetted[status] = s + } + for status, tokens := range t.inv.vetted { + s := make([]string, len(tokens)) + copy(s, tokens) + vetted[status] = s + } + + return inventory{ + unvetted: unvetted, + vetted: vetted, + } +} + +func (t *tlogBackend) inventoryAdd(state string, tokenb []byte, s backend.MDStatusT) { + t.Lock() + defer t.Unlock() + + token := hex.EncodeToString(tokenb) + switch state { + case stateUnvetted: + t.inv.unvetted[s] = append([]string{token}, t.inv.unvetted[s]...) + case stateVetted: + t.inv.vetted[s] = append([]string{token}, t.inv.vetted[s]...) + default: + e := fmt.Sprintf("unknown state '%v'", state) + panic(e) + } + + log.Debugf("Add to inv %v: %v %v", state, token, backend.MDStatus[s]) +} + +func (t *tlogBackend) inventoryUpdate(state string, tokenb []byte, currStatus, newStatus backend.MDStatusT) { + token := hex.EncodeToString(tokenb) + + t.Lock() + defer t.Unlock() + + var inv map[backend.MDStatusT][]string + switch state { + case stateUnvetted: + inv = t.inv.unvetted + case stateVetted: + inv = t.inv.vetted + default: + e := fmt.Sprintf("unknown state '%v'", state) + panic(e) + } + + // Find the index of the token in its current status list + var idx int + var found bool + for k, v := range inv[currStatus] { + if v == token { + // Token found + idx = k + found = true + break + } + } + if !found { + // Token was never found. This should not happen. + e := fmt.Sprintf("inventoryUpdate: token not found: %v %v %v", + token, currStatus, newStatus) + panic(e) + } + + // Remove the token from its current status list + tokens := inv[currStatus] + inv[currStatus] = append(tokens[:idx], tokens[idx+1:]...) + + // Prepend token to new status + inv[newStatus] = append([]string{token}, inv[newStatus]...) + + log.Debugf("Update inv %v: %v %v to %v", state, token, + backend.MDStatus[currStatus], backend.MDStatus[newStatus]) +} + +// inventoryMoveToVetted moves a token from the unvetted inventory to the +// vetted inventory. The unvettedStatus is the status of the record prior to +// the update and the vettedStatus is the status of the record after the +// update. +func (t *tlogBackend) inventoryMoveToVetted(tokenb []byte, unvettedStatus, vettedStatus backend.MDStatusT) { + t.Lock() + defer t.Unlock() + + token := hex.EncodeToString(tokenb) + unvetted := t.inv.unvetted + vetted := t.inv.vetted + + // Find the index of the token in its current status list + var idx int + var found bool + for k, v := range unvetted[unvettedStatus] { + if v == token { + // Token found + idx = k + found = true + break + } + } + if !found { + // Token was never found. This should not happen. + e := fmt.Sprintf("inventoryMoveToVetted: unvetted token not found: %v %v", + token, unvettedStatus) + panic(e) + } + + // Remove the token from the unvetted status list + tokens := unvetted[unvettedStatus] + unvetted[unvettedStatus] = append(tokens[:idx], tokens[idx+1:]...) + + // Prepend token to vetted status + vetted[vettedStatus] = append([]string{token}, vetted[vettedStatus]...) + + log.Debugf("Inv move to vetted: %v %v to %v", token, + backend.MDStatus[unvettedStatus], backend.MDStatus[vettedStatus]) +} diff --git a/politeiad/backend/tlogbe/plugins/comments/comments.go b/politeiad/backend/tlogbe/plugins/comments/comments.go index 288bb161b..e7b0cc4a1 100644 --- a/politeiad/backend/tlogbe/plugins/comments/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments/comments.go @@ -89,7 +89,7 @@ func tokenDecode(token string) ([]byte, error) { return util.TokenDecode(util.TokenTypeTlog, token) } -func convertSignatureError(err error) backend.PluginUserError { +func convertSignatureError(err error) backend.PluginError { var e util.SignatureError var s comments.ErrorCodeT if errors.As(err, &e) { @@ -100,7 +100,7 @@ func convertSignatureError(err error) backend.PluginUserError { s = comments.ErrorCodeSignatureInvalid } } - return backend.PluginUserError{ + return backend.PluginError{ PluginID: comments.ID, ErrorCode: int(s), ErrorContext: e.ErrorContext, @@ -656,7 +656,7 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str // Verify token t, err := tokenDecode(n.Token) if err != nil { - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorCodeTokenInvalid), ErrorContext: err.Error(), @@ -665,7 +665,7 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str if !bytes.Equal(t, token) { e := fmt.Sprintf("comment token does not match route token: "+ "got %x, want %x", t, token) - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorCodeTokenInvalid), ErrorContext: e, @@ -681,7 +681,7 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str // Verify comment if len(n.Comment) > comments.PolicyCommentLengthMax { - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorCodeCommentTextInvalid), ErrorContext: "exceeds max length", @@ -703,7 +703,7 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str // Verify parent comment exists if set. A parent ID of 0 means that // this is a base level comment, not a reply to another comment. if n.ParentID > 0 && !commentExists(*ridx, n.ParentID) { - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorCodeParentIDInvalid), ErrorContext: "parent ID comment not found", @@ -780,7 +780,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st // Verify token t, err := tokenDecode(e.Token) if err != nil { - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorCodeTokenInvalid), ErrorContext: err.Error(), @@ -789,7 +789,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st if !bytes.Equal(t, token) { e := fmt.Sprintf("comment token does not match route token: "+ "got %x, want %x", t, token) - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorCodeTokenInvalid), ErrorContext: e, @@ -805,7 +805,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st // Verify comment if len(e.Comment) > comments.PolicyCommentLengthMax { - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorCodeCommentTextInvalid), ErrorContext: "exceeds max length", @@ -831,7 +831,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st } existing, ok := cs[e.CommentID] if !ok { - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorCodeCommentNotFound), } @@ -839,7 +839,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st // Verify the user ID if e.UserID != existing.UserID { - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorCodeUserUnauthorized), } @@ -849,7 +849,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st if e.ParentID != existing.ParentID { e := fmt.Sprintf("parent id cannot change; got %v, want %v", e.ParentID, existing.ParentID) - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorCodeParentIDInvalid), ErrorContext: e, @@ -858,7 +858,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st // Verify comment changes if e.Comment == existing.Comment { - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorCodeCommentTextInvalid), ErrorContext: "comment did not change", @@ -929,7 +929,7 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str // Verify token t, err := tokenDecode(d.Token) if err != nil { - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorCodeTokenInvalid), ErrorContext: err.Error(), @@ -938,7 +938,7 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str if !bytes.Equal(t, token) { e := fmt.Sprintf("comment token does not match route token: "+ "got %x, want %x", t, token) - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorCodeTokenInvalid), ErrorContext: e, @@ -971,7 +971,7 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str } existing, ok := cs[d.CommentID] if !ok { - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorCodeCommentNotFound), } @@ -1054,7 +1054,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st // Verify token t, err := tokenDecode(v.Token) if err != nil { - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorCodeTokenInvalid), ErrorContext: err.Error(), @@ -1063,7 +1063,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st if !bytes.Equal(t, token) { e := fmt.Sprintf("comment token does not match route token: "+ "got %x, want %x", t, token) - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorCodeTokenInvalid), ErrorContext: e, @@ -1075,7 +1075,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st case comments.VoteDownvote, comments.VoteUpvote: // These are allowed default: - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorCodeVoteInvalid), } @@ -1104,7 +1104,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st // Verify comment exists cidx, ok := ridx.Comments[v.CommentID] if !ok { - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorCodeCommentNotFound), } @@ -1116,7 +1116,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st uvotes = make([]voteIndex, 0) } if len(uvotes) > comments.PolicyVoteChangesMax { - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorCodeVoteChangesMax), } @@ -1132,7 +1132,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st return "", fmt.Errorf("comment not found %v", v.CommentID) } if v.UserID == c.UserID { - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorCodeVoteInvalid), ErrorContext: "user cannot vote on their own comment", @@ -1298,13 +1298,13 @@ func (p *commentsPlugin) cmdGetVersion(treeID int64, token []byte, payload strin // Verify comment exists cidx, ok := ridx.Comments[gv.CommentID] if !ok { - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorCodeCommentNotFound), } } if cidx.Del != nil { - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorCodeCommentNotFound), ErrorContext: "comment has been deleted", @@ -1314,7 +1314,7 @@ func (p *commentsPlugin) cmdGetVersion(treeID int64, token []byte, payload strin if !ok { e := fmt.Sprintf("comment %v does not have version %v", gv.CommentID, gv.Version) - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: comments.ID, ErrorCode: int(comments.ErrorCodeCommentNotFound), ErrorContext: e, diff --git a/politeiad/backend/tlogbe/plugins/pi/hooks.go b/politeiad/backend/tlogbe/plugins/pi/hooks.go index 8d971c6a6..922bd7f75 100644 --- a/politeiad/backend/tlogbe/plugins/pi/hooks.go +++ b/politeiad/backend/tlogbe/plugins/pi/hooks.go @@ -249,19 +249,6 @@ func (p *piPlugin) hookSetRecordStatusPost(payload string) error { // commentWritesVerify verifies that a record's vote status allows writes from // the comments plugin. func (p *piPlugin) commentWritesVerify(token []byte) error { - // Verify the ticketvote plugin is registered - var found bool - for _, v := range p.tlog.Plugins() { - if v.ID == ticketvote.ID { - found = true - break - } - } - if !found { - // Ticket vote plugin is not registered. Nothing to verify. - return nil - } - // Verify that the vote status allows comment writes vs, err := p.voteSummary(token) if err != nil { diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go index 7526d4343..36a631bde 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go @@ -58,7 +58,7 @@ func tokenDecodeAnyLength(token string) ([]byte, error) { return util.TokenDecodeAnyLength(util.TokenTypeTlog, token) } -func convertSignatureError(err error) backend.PluginUserError { +func convertSignatureError(err error) backend.PluginError { var e util.SignatureError var s ticketvote.ErrorCodeT if errors.As(err, &e) { @@ -69,7 +69,7 @@ func convertSignatureError(err error) backend.PluginUserError { s = ticketvote.ErrorCodeSignatureInvalid } } - return backend.PluginUserError{ + return backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(s), ErrorContext: e.ErrorContext, @@ -973,7 +973,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri // Verify token t, err := tokenDecode(a.Token) if err != nil { - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), } @@ -981,7 +981,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri if !bytes.Equal(t, token) { e := fmt.Sprintf("plugin token does not match route token: "+ "got %x, want %x", t, token) - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), ErrorContext: e, @@ -1004,7 +1004,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri // This is allowed default: e := fmt.Sprintf("%v not a valid action", a.Action) - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), ErrorContext: e, @@ -1032,7 +1032,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri case len(auths) == 0: // No previous actions. New action must be an authorize. if a.Action != ticketvote.AuthActionAuthorize { - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), ErrorContext: "no prev action; action must be authorize", @@ -1041,7 +1041,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri case prevAction == ticketvote.AuthActionAuthorize && a.Action != ticketvote.AuthActionRevoke: // Previous action was a authorize. This action must be revoke. - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), ErrorContext: "prev action was authorize", @@ -1049,7 +1049,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri case prevAction == ticketvote.AuthActionRevoke && a.Action != ticketvote.AuthActionAuthorize: // Previous action was a revoke. This action must be authorize. - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), ErrorContext: "prev action was revoke", @@ -1132,7 +1132,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM // This is allowed default: e := fmt.Sprintf("invalid type %v", vote.Type) - return backend.PluginUserError{ + return backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, @@ -1144,7 +1144,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM case vote.Duration > voteDurationMax: e := fmt.Sprintf("duration %v exceeds max duration %v", vote.Duration, voteDurationMax) - return backend.PluginUserError{ + return backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, @@ -1152,7 +1152,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM case vote.Duration < voteDurationMin: e := fmt.Sprintf("duration %v under min duration %v", vote.Duration, voteDurationMin) - return backend.PluginUserError{ + return backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, @@ -1160,7 +1160,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM case vote.QuorumPercentage > 100: e := fmt.Sprintf("quorum percent %v exceeds 100 percent", vote.QuorumPercentage) - return backend.PluginUserError{ + return backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, @@ -1168,7 +1168,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM case vote.PassPercentage > 100: e := fmt.Sprintf("pass percent %v exceeds 100 percent", vote.PassPercentage) - return backend.PluginUserError{ + return backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, @@ -1178,7 +1178,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM // Verify vote options. Different vote types have different // requirements. if len(vote.Options) == 0 { - return backend.PluginUserError{ + return backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: "no vote options found", @@ -1192,7 +1192,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM if len(vote.Options) != 2 { e := fmt.Sprintf("vote options count got %v, want 2", len(vote.Options)) - return backend.PluginUserError{ + return backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, @@ -1221,7 +1221,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM if len(missing) > 0 { e := fmt.Sprintf("vote option IDs not found: %v", strings.Join(missing, ",")) - return backend.PluginUserError{ + return backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, @@ -1233,7 +1233,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM for _, v := range vote.Options { err := voteBitVerify(vote.Options, vote.Mask, v.Bit) if err != nil { - return backend.PluginUserError{ + return backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: err.Error(), @@ -1245,7 +1245,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM switch { case vote.Type == ticketvote.VoteTypeStandard && vote.Parent != "": e := "parent token should not be provided for a standard vote" - return backend.PluginUserError{ + return backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, @@ -1254,7 +1254,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM _, err := tokenDecode(vote.Parent) if err != nil { e := fmt.Sprintf("invalid parent %v", vote.Parent) - return backend.PluginUserError{ + return backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, @@ -1269,7 +1269,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvote.Start) (*ticketvote.StartReply, error) { // Verify there is only one start details if len(s.Starts) != 1 { - return nil, backend.PluginUserError{ + return nil, backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeStartDetailsInvalid), ErrorContext: "more than one start details found", @@ -1280,7 +1280,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot // Verify token t, err := tokenDecode(sd.Params.Token) if err != nil { - return nil, backend.PluginUserError{ + return nil, backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), } @@ -1288,7 +1288,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot if !bytes.Equal(t, token) { e := fmt.Sprintf("plugin token does not match route token: "+ "got %x, want %x", t, token) - return nil, backend.PluginUserError{ + return nil, backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), ErrorContext: e, @@ -1333,7 +1333,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot if r.Version != version { e := fmt.Sprintf("version is not latest: got %v, want %v", sd.Params.Version, r.Version) - return nil, backend.PluginUserError{ + return nil, backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeRecordVersionInvalid), ErrorContext: e, @@ -1346,7 +1346,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot return nil, err } if len(auths) == 0 { - return nil, backend.PluginUserError{ + return nil, backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), ErrorContext: "authorization not found", @@ -1354,7 +1354,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot } action := ticketvote.AuthActionT(auths[len(auths)-1].Action) if action != ticketvote.AuthActionAuthorize { - return nil, backend.PluginUserError{ + return nil, backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), ErrorContext: "not authorized", @@ -1368,7 +1368,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot } if svp != nil { // Vote has already been started - return nil, backend.PluginUserError{ + return nil, backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeVoteStatusInvalid), ErrorContext: "vote already started", @@ -1550,7 +1550,7 @@ func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs sta if r.Version != version { e := fmt.Sprintf("version is not latest %v: got %v, want %v", s.Params.Token, s.Params.Version, r.Version) - return backend.PluginUserError{ + return backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeRecordVersionInvalid), ErrorContext: e, @@ -1618,7 +1618,7 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti if err != nil { if errors.Is(err, backend.ErrRecordNotFound) { e := fmt.Sprintf("parent record not found %x", token) - return nil, backend.PluginUserError{ + return nil, backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, @@ -1630,7 +1630,7 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti return nil, err } if vm == nil || vm.LinkBy == 0 { - return nil, backend.PluginUserError{ + return nil, backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeRunoffVoteParentInvalid), } @@ -1641,7 +1641,7 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti case !isExpired && isMainNet: e := fmt.Sprintf("parent record %x linkby deadline not met %v", token, vm.LinkBy) - return nil, backend.PluginUserError{ + return nil, backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeLinkByNotExpired), ErrorContext: e, @@ -1691,7 +1691,7 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti // This submission should not be here e := fmt.Sprintf("record %v should not be included", v.Params.Token) - return nil, backend.PluginUserError{ + return nil, backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeStartDetailsInvalid), ErrorContext: e, @@ -1708,7 +1708,7 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti _, ok := subs[k] if !ok { // This records is missing from the runoff vote - return nil, backend.PluginUserError{ + return nil, backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeStartDetailsMissing), ErrorContext: k, @@ -1765,7 +1765,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. case v.Params.Type != ticketvote.VoteTypeRunoff: e := fmt.Sprintf("%v vote type invalid: got %v, want %v", v.Params.Token, v.Params.Type, ticketvote.VoteTypeRunoff) - return nil, backend.PluginUserError{ + return nil, backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, @@ -1773,7 +1773,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. case v.Params.Mask != mask: e := fmt.Sprintf("%v mask invalid: all must be the same", v.Params.Token) - return nil, backend.PluginUserError{ + return nil, backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, @@ -1781,7 +1781,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. case v.Params.Duration != duration: e := fmt.Sprintf("%v duration invalid: all must be the same", v.Params.Token) - return nil, backend.PluginUserError{ + return nil, backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, @@ -1789,7 +1789,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. case v.Params.QuorumPercentage != quorum: e := fmt.Sprintf("%v quorum invalid: must be the same", v.Params.Token) - return nil, backend.PluginUserError{ + return nil, backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, @@ -1797,7 +1797,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. case v.Params.PassPercentage != pass: e := fmt.Sprintf("%v pass rate invalid: all must be the same", v.Params.Token) - return nil, backend.PluginUserError{ + return nil, backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, @@ -1805,7 +1805,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. case v.Params.Parent != parent: e := fmt.Sprintf("%v parent invalid: all must be the same", v.Params.Token) - return nil, backend.PluginUserError{ + return nil, backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, @@ -1815,7 +1815,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. // Verify token _, err := tokenDecode(v.Params.Token) if err != nil { - return nil, backend.PluginUserError{ + return nil, backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), ErrorContext: v.Params.Token, @@ -1826,7 +1826,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. _, err = tokenDecode(v.Params.Parent) if err != nil { e := fmt.Sprintf("parent token %v", v.Params.Parent) - return nil, backend.PluginUserError{ + return nil, backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), ErrorContext: e, @@ -1856,7 +1856,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. if hex.EncodeToString(token) != parent { e := fmt.Sprintf("runoff vote must be started on parent record %v", parent) - return nil, backend.PluginUserError{ + return nil, backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeRunoffVoteParentInvalid), ErrorContext: e, @@ -1889,7 +1889,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. _, err = p.backend.VettedPluginCmd(token, ticketvote.ID, cmdStartRunoffSubmission, string(b)) if err != nil { - var ue backend.PluginUserError + var ue backend.PluginError if errors.As(err, &ue) { return nil, err } @@ -1944,7 +1944,7 @@ func (p *ticketVotePlugin) cmdStart(treeID int64, token []byte, payload string) // Parse vote type if len(s.Starts) == 0 { - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeStartDetailsMissing), ErrorContext: "no start details found", @@ -1969,7 +1969,7 @@ func (p *ticketVotePlugin) cmdStart(treeID int64, token []byte, payload string) } default: e := fmt.Sprintf("invalid vote type %v", vtype) - return "", backend.PluginUserError{ + return "", backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go b/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go index e40606af2..f486b6852 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go @@ -49,7 +49,7 @@ func (p *ticketVotePlugin) linkByVerify(linkBy int64) error { case linkBy < min: e := fmt.Sprintf("linkby %v is less than min required of %v", linkBy, min) - return backend.PluginUserError{ + return backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeLinkByInvalid), ErrorContext: e, @@ -57,7 +57,7 @@ func (p *ticketVotePlugin) linkByVerify(linkBy int64) error { case linkBy > max: e := fmt.Sprintf("linkby %v is more than max allowed of %v", linkBy, max) - return backend.PluginUserError{ + return backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeLinkByInvalid), ErrorContext: e, @@ -70,7 +70,7 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { // LinkTo must be a public record token, err := tokenDecode(linkTo) if err != nil { - return backend.PluginUserError{ + return backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), ErrorContext: "invalid hex", @@ -79,7 +79,7 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { r, err := p.backend.GetVetted(token, "") if err != nil { if errors.Is(err, backend.ErrRecordNotFound) { - return backend.PluginUserError{ + return backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), ErrorContext: "record not found", @@ -88,7 +88,7 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { return err } if r.RecordMetadata.Status != backend.MDStatusCensored { - return backend.PluginUserError{ + return backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), ErrorContext: "record is censored", @@ -102,7 +102,7 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { return err } if parentVM == nil || parentVM.LinkBy == 0 { - return backend.PluginUserError{ + return backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), ErrorContext: "record not a runoff vote parent", @@ -111,7 +111,7 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { // The LinkBy deadline must not be expired if time.Now().Unix() > parentVM.LinkBy { - return backend.PluginUserError{ + return backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), ErrorContext: "parent record linkby deadline has expired", @@ -124,7 +124,7 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { return err } if !vs.Approved { - return backend.PluginUserError{ + return backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), ErrorContext: "parent record vote is not approved", @@ -138,7 +138,7 @@ func (p *ticketVotePlugin) voteMetadataVerify(vm ticketvote.VoteMetadata) error switch { case vm.LinkBy == 0 && vm.LinkTo == "": // Vote metadata is empty - return backend.PluginUserError{ + return backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeVoteMetadataInvalid), ErrorContext: "md is empty", @@ -146,7 +146,7 @@ func (p *ticketVotePlugin) voteMetadataVerify(vm ticketvote.VoteMetadata) error case vm.LinkBy != 0 && vm.LinkTo != "": // LinkBy and LinkTo cannot both be set - return backend.PluginUserError{ + return backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeVoteMetadataInvalid), ErrorContext: "cannot set both linkby and linkto", @@ -222,7 +222,7 @@ func (p *ticketVotePlugin) hookEditRecordPre(payload string) error { if newLinkTo != oldLinkTo { e := fmt.Sprintf("linkto cannot change on vetted record: "+ "got '%v', want '%v'", newLinkTo, oldLinkTo) - return backend.PluginUserError{ + return backend.PluginError{ PluginID: ticketvote.ID, ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), ErrorContext: e, diff --git a/politeiad/backend/tlogbe/testing.go b/politeiad/backend/tlogbe/testing.go index 4d189db8b..ad03d9673 100644 --- a/politeiad/backend/tlogbe/testing.go +++ b/politeiad/backend/tlogbe/testing.go @@ -35,7 +35,7 @@ func NewTestTlogBackend(t *testing.T) (*tlogBackend, func()) { vetted: tlog.NewTestTlogEncrypted(t, dataDir, "vetted"), prefixes: make(map[string][]byte), vettedTreeIDs: make(map[string]int64), - inv: recordInventory{ + inv: inventory{ unvetted: make(map[backend.MDStatusT][]string), vetted: make(map[backend.MDStatusT][]string), }, diff --git a/politeiad/backend/tlogbe/tlog/plugin.go b/politeiad/backend/tlogbe/tlog/plugin.go index 289c36750..bb4d943fd 100644 --- a/politeiad/backend/tlogbe/tlog/plugin.go +++ b/politeiad/backend/tlogbe/tlog/plugin.go @@ -119,7 +119,7 @@ func (t *Tlog) PluginHookPre(treeID int64, token []byte, h plugins.HookT, payloa p, _ := t.plugin(v) err := p.client.Hook(treeID, token, h, payload) if err != nil { - var e backend.PluginUserError + var e backend.PluginError if errors.As(err, &e) { return err } @@ -166,8 +166,6 @@ func (t *Tlog) PluginCmd(treeID int64, token []byte, pluginID, cmd, payload stri } // Plugins returns all registered plugins for the tlog instance. -// -// This function satisfies the tlogclient.Client interface. func (t *Tlog) Plugins() []backend.Plugin { log.Tracef("%v Plugins", t.id) diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index c959d7246..f0b4fb6a6 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -42,10 +42,6 @@ const ( // Tlog instance IDs tlogIDUnvetted = "unvetted" tlogIDVetted = "vetted" - - // Record states - stateUnvetted = "unvetted" - stateVetted = "vetted" ) var ( @@ -82,12 +78,7 @@ type tlogBackend struct { // status. Each list of tokens is sorted by the timestamp of the // status change from newest to oldest. This cache is built on // startup. - inv recordInventory -} - -type recordInventory struct { - unvetted map[backend.MDStatusT][]string - vetted map[backend.MDStatusT][]string + inv inventory } func tokenFromTreeID(treeID int64) []byte { @@ -210,137 +201,6 @@ func (t *tlogBackend) vettedTreeIDFromToken(token []byte) (int64, bool) { return vettedTreeID, true } -func (t *tlogBackend) inventory() recordInventory { - t.RLock() - defer t.RUnlock() - - // Return a copy of the inventory - var ( - unvetted = make(map[backend.MDStatusT][]string, len(t.inv.unvetted)) - vetted = make(map[backend.MDStatusT][]string, len(t.inv.vetted)) - ) - for status, tokens := range t.inv.unvetted { - s := make([]string, len(tokens)) - copy(s, tokens) - unvetted[status] = s - } - for status, tokens := range t.inv.vetted { - s := make([]string, len(tokens)) - copy(s, tokens) - vetted[status] = s - } - - return recordInventory{ - unvetted: unvetted, - vetted: vetted, - } -} - -func (t *tlogBackend) inventoryAdd(state string, tokenb []byte, s backend.MDStatusT) { - t.Lock() - defer t.Unlock() - - token := hex.EncodeToString(tokenb) - switch state { - case stateUnvetted: - t.inv.unvetted[s] = append([]string{token}, t.inv.unvetted[s]...) - case stateVetted: - t.inv.vetted[s] = append([]string{token}, t.inv.vetted[s]...) - default: - e := fmt.Sprintf("unknown state '%v'", state) - panic(e) - } - - log.Debugf("Add to inv %v: %v %v", state, token, backend.MDStatus[s]) -} - -func (t *tlogBackend) inventoryUpdate(state string, tokenb []byte, currStatus, newStatus backend.MDStatusT) { - token := hex.EncodeToString(tokenb) - - t.Lock() - defer t.Unlock() - - var inv map[backend.MDStatusT][]string - switch state { - case stateUnvetted: - inv = t.inv.unvetted - case stateVetted: - inv = t.inv.vetted - default: - e := fmt.Sprintf("unknown state '%v'", state) - panic(e) - } - - // Find the index of the token in its current status list - var idx int - var found bool - for k, v := range inv[currStatus] { - if v == token { - // Token found - idx = k - found = true - break - } - } - if !found { - // Token was never found. This should not happen. - e := fmt.Sprintf("inventoryUpdate: token not found: %v %v %v", - token, currStatus, newStatus) - panic(e) - } - - // Remove the token from its current status list - tokens := inv[currStatus] - inv[currStatus] = append(tokens[:idx], tokens[idx+1:]...) - - // Prepend token to new status - inv[newStatus] = append([]string{token}, inv[newStatus]...) - - log.Debugf("Update inv %v: %v %v to %v", state, token, - backend.MDStatus[currStatus], backend.MDStatus[newStatus]) -} - -// inventoryMoveToVetted moves a token from the unvetted inventory to the -// vetted inventory. The unvettedStatus is the status of the record prior to -// the update and the vettedStatus is the status of the record after the -// update. -func (t *tlogBackend) inventoryMoveToVetted(tokenb []byte, unvettedStatus, vettedStatus backend.MDStatusT) { - t.Lock() - defer t.Unlock() - - token := hex.EncodeToString(tokenb) - unvetted := t.inv.unvetted - vetted := t.inv.vetted - - // Find the index of the token in its current status list - var idx int - var found bool - for k, v := range unvetted[unvettedStatus] { - if v == token { - // Token found - idx = k - found = true - break - } - } - if !found { - // Token was never found. This should not happen. - e := fmt.Sprintf("inventoryMoveToVetted: unvetted token not found: %v %v", - token, unvettedStatus) - panic(e) - } - - // Remove the token from the unvetted status list - tokens := unvetted[unvettedStatus] - unvetted[unvettedStatus] = append(tokens[:idx], tokens[idx+1:]...) - - // Prepend token to vetted status - vetted[vettedStatus] = append([]string{token}, vetted[vettedStatus]...) - - log.Debugf("Inv move to vetted: %v %v to %v", token, - backend.MDStatus[unvettedStatus], backend.MDStatus[vettedStatus]) -} - // verifyContent verifies that all provided MetadataStream and File are sane. func verifyContent(metadata []backend.MetadataStream, files []backend.File, filesDel []string) error { // Make sure all metadata is within maxima. @@ -1947,7 +1807,7 @@ func New(anp *chaincfg.Params, homeDir, dataDir, unvettedTrillianHost, unvettedT vetted: vetted, prefixes: make(map[string][]byte), vettedTreeIDs: make(map[string]int64), - inv: recordInventory{ + inv: inventory{ unvetted: make(map[backend.MDStatusT][]string), vetted: make(map[backend.MDStatusT][]string), }, diff --git a/politeiad/backend/tlogbe/tlogclient/tlogclient.go b/politeiad/backend/tlogbe/tlogclient/tlogclient.go index 0aae94b1b..a44ed1762 100644 --- a/politeiad/backend/tlogbe/tlogclient/tlogclient.go +++ b/politeiad/backend/tlogbe/tlogclient/tlogclient.go @@ -47,7 +47,4 @@ type Client interface { // Timestamp returns the timestamp for the blob that correpsonds // to the digest. Timestamp(treeID int64, digest []byte) (*backend.Timestamp, error) - - // Plugins returns all registered plugins for the tlog instance. - Plugins() []backend.Plugin } diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index df5a6832f..4e67425ff 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2020 The Decred developers +// Copyright (c) 2017-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -258,9 +258,9 @@ func (p *politeia) respondWithUserError(w http.ResponseWriter, errorCode v1.Erro }) } -func (p *politeia) respondWithPluginUserError(w http.ResponseWriter, plugin string, errorCode int, errorContext string) { - util.RespondWithJSON(w, http.StatusBadRequest, v1.PluginUserErrorReply{ - Plugin: plugin, +func (p *politeia) respondWithPluginError(w http.ResponseWriter, plugin string, errorCode int, errorContext string) { + util.RespondWithJSON(w, http.StatusBadRequest, v1.PluginErrorReply{ + PluginID: plugin, ErrorCode: errorCode, ErrorContext: []string{errorContext}, }) @@ -328,11 +328,11 @@ func (p *politeia) newRecord(w http.ResponseWriter, r *http.Request) { } // Check for plugin error - var e backend.PluginUserError + var e backend.PluginError if errors.As(err, &e) { log.Debugf("%v plugin user error: %v %v", remoteAddr(r), e.PluginID, e.ErrorCode) - p.respondWithPluginUserError(w, e.PluginID, e.ErrorCode, + p.respondWithPluginError(w, e.PluginID, e.ErrorCode, e.ErrorContext) return } @@ -438,11 +438,11 @@ func (p *politeia) updateRecord(w http.ResponseWriter, r *http.Request, vetted b return } // Check for plugin error - var e backend.PluginUserError + var e backend.PluginError if errors.As(err, &e) { log.Debugf("%v plugin user error: %v %v", remoteAddr(r), e.PluginID, e.ErrorCode) - p.respondWithPluginUserError(w, e.PluginID, e.ErrorCode, + p.respondWithPluginError(w, e.PluginID, e.ErrorCode, e.ErrorContext) return } @@ -889,11 +889,11 @@ func (p *politeia) setVettedStatus(w http.ResponseWriter, r *http.Request) { return } // Check for plugin error - var e backend.PluginUserError + var e backend.PluginError if errors.As(err, &e) { log.Debugf("%v plugin user error: %v %v", remoteAddr(r), e.PluginID, e.ErrorCode) - p.respondWithPluginUserError(w, e.PluginID, e.ErrorCode, + p.respondWithPluginError(w, e.PluginID, e.ErrorCode, e.ErrorContext) return } @@ -961,11 +961,11 @@ func (p *politeia) setUnvettedStatus(w http.ResponseWriter, r *http.Request) { return } // Check for plugin error - var e backend.PluginUserError + var e backend.PluginError if errors.As(err, &e) { log.Debugf("%v plugin user error: %v %v", remoteAddr(r), e.PluginID, e.ErrorCode) - p.respondWithPluginUserError(w, e.PluginID, e.ErrorCode, + p.respondWithPluginError(w, e.PluginID, e.ErrorCode, e.ErrorContext) return } @@ -1036,11 +1036,11 @@ func (p *politeia) updateVettedMetadata(w http.ResponseWriter, r *http.Request) return } // Check for plugin error - var e backend.PluginUserError + var e backend.PluginError if errors.As(err, &e) { log.Debugf("%v plugin user error: %v %v", remoteAddr(r), e.PluginID, e.ErrorCode) - p.respondWithPluginUserError(w, e.PluginID, e.ErrorCode, + p.respondWithPluginError(w, e.PluginID, e.ErrorCode, e.ErrorContext) return } @@ -1106,11 +1106,11 @@ func (p *politeia) updateUnvettedMetadata(w http.ResponseWriter, r *http.Request return } // Check for plugin error - var e backend.PluginUserError + var e backend.PluginError if errors.As(err, &e) { log.Debugf("%v plugin user error: %v %v", remoteAddr(r), e.PluginID, e.ErrorCode) - p.respondWithPluginUserError(w, e.PluginID, e.ErrorCode, + p.respondWithPluginError(w, e.PluginID, e.ErrorCode, e.ErrorContext) return } @@ -1179,11 +1179,11 @@ func (p *politeia) pluginCommand(w http.ResponseWriter, r *http.Request) { pc.CommandID, pc.Payload) if err != nil { // Check for a user error - var e backend.PluginUserError + var e backend.PluginError if errors.As(err, &e) { log.Infof("%v plugin user error: %v %v", remoteAddr(r), e.PluginID, e.ErrorCode) - p.respondWithPluginUserError(w, e.PluginID, e.ErrorCode, + p.respondWithPluginError(w, e.PluginID, e.ErrorCode, e.ErrorContext) return } @@ -1211,6 +1211,144 @@ func (p *politeia) pluginCommand(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, reply) } +func convertBackendError(e error) error { + // Check for a plugin error + var pe backend.PluginError + if errors.As(e, &pe) { + return v1.PluginErrorReply{ + PluginID: pe.PluginID, + ErrorCode: pe.ErrorCode, + ErrorContext: []string{pe.ErrorContext}, + } + } + + // Check for a backend error + var s v1.ErrorStatusT + switch e { + case backend.ErrRecordNotFound: + s = v1.ErrorStatusRecordNotFound + case backend.ErrRecordFound: + // Intentionally omitted + case backend.ErrFileNotFound: + // Intentionally omitted + case backend.ErrNoChanges: + s = v1.ErrorStatusNoChanges + case backend.ErrChangesRecord: + // Intentionally omitted + case backend.ErrRecordLocked: + s = v1.ErrorStatusRecordLocked + case backend.ErrJournalsNotReplayed: + // Intentionally omitted + case backend.ErrPluginInvalid: + // Intentionally omitted + case backend.ErrPluginCmdInvalid: + // Intentionally omitted + } + if s != v1.ErrorStatusInvalid { + return v1.UserErrorReply{ + ErrorCode: s, + } + } + + return v1.ServerErrorReply{ + ErrorCode: time.Now().Unix(), + } +} + +func (p *politeia) pluginCommandBatch(w http.ResponseWriter, r *http.Request) { + var pcb v1.PluginCommandBatch + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&pcb); err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, + nil) + return + } + + // Verify challenge + challenge, err := hex.DecodeString(pcb.Challenge) + if err != nil || len(challenge) != v1.ChallengeSize { + p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) + return + } + + // Execute plugin commands + replies := make([]v1.PluginCommandReplyV2, len(pcb.Commands)) + for k, pc := range pcb.Commands { + // Verify token + token, err := util.ConvertStringToken(pc.Token) + if err != nil { + replies[k] = v1.PluginCommandReplyV2{ + Error: v1.UserErrorReply{ + ErrorCode: v1.ErrorStatusInvalidToken, + }, + } + continue + } + + // Execute plugin command + var payload string + switch pc.State { + case v1.RecordStateUnvetted: + payload, err = p.backend.UnvettedPluginCmd(token, + pc.ID, pc.Command, pc.Payload) + case v1.RecordStateVetted: + payload, err = p.backend.VettedPluginCmd(token, + pc.ID, pc.Command, pc.Payload) + default: + replies[k] = v1.PluginCommandReplyV2{ + Error: v1.UserErrorReply{ + ErrorCode: v1.ErrorStatusInvalidRecordState, + }, + } + continue + } + if err != nil { + // Add error to reply + e := convertBackendError(err) + replies[k] = v1.PluginCommandReplyV2{ + Error: e, + } + + // Log any internal server errors + var se v1.ServerErrorReply + if errors.As(e, &se) { + log.Errorf("%v %v: backend plugin failed: pluginID:%v cmd:%v "+ + "payload:%v err:%v", remoteAddr(r), se.ErrorCode, pc.ID, + pc.Command, pc.Payload, err) + } + + continue + } + + // Update reply + replies[k] = v1.PluginCommandReplyV2{ + Payload: payload, + } + } + + // Fill in remaining data for the replies + for k, v := range replies { + replies[k] = v1.PluginCommandReplyV2{ + State: pcb.Commands[k].State, + Token: pcb.Commands[k].Token, + ID: pcb.Commands[k].ID, + Command: pcb.Commands[k].Command, + Payload: v.Payload, + Error: v.Error, + } + } + + response := p.identity.SignMessage(challenge) + reply := v1.PluginCommandBatchReply{ + Response: hex.EncodeToString(response[:]), + Replies: replies, + } + + log.Infof("%v Plugin cmd batch executed", remoteAddr(r)) + + util.RespondWithJSON(w, http.StatusOK, reply) +} + func logging(f http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { // Trace incoming request @@ -1474,10 +1612,14 @@ func _main() error { return fmt.Errorf("unknown plugin '%v'", v) } - // Register with backend - err := p.backend.RegisterPlugin(plugin) + // Register plugin + err = p.backend.RegisterUnvettedPlugin(plugin) + if err != nil { + return fmt.Errorf("register unvetted plugin %v: %v", v, err) + } + err = p.backend.RegisterVettedPlugin(plugin) if err != nil { - return fmt.Errorf("RegisterPlugin %v: %v", v, err) + return fmt.Errorf("register vetted plugin %v: %v", v, err) } // Add plugin to politeiad context @@ -1489,9 +1631,13 @@ func _main() error { // Setup plugins for _, v := range cfg.Plugins { log.Infof("Setting up plugin: %v", v) - err := p.backend.SetupPlugin(v) + err = p.backend.SetupUnvettedPlugin(v) + if err != nil { + return fmt.Errorf("setup unvetted plugin %v: %v", v, err) + } + err = p.backend.SetupVettedPlugin(v) if err != nil { - return fmt.Errorf("SetupPlugin %v: %v", v, err) + return fmt.Errorf("setup vetted plugin %v: %v", v, err) } } } diff --git a/politeiawww/api/comments/v1/v1.go b/politeiawww/api/comments/v1/v1.go index 0435b51ab..47af3cc8e 100644 --- a/politeiawww/api/comments/v1/v1.go +++ b/politeiawww/api/comments/v1/v1.go @@ -53,7 +53,7 @@ type PluginErrorReply struct { // Error satisfies the error interface. func (e PluginErrorReply) Error() string { - return fmt.Sprintf("user error code: %v", e.ErrorCode) + return fmt.Sprintf("plugin error code: %v", e.ErrorCode) } // ServerErrorReply is the reply that the server returns when it encounters an diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go index a927df912..0bd552f42 100644 --- a/politeiawww/api/records/v1/v1.go +++ b/politeiawww/api/records/v1/v1.go @@ -58,7 +58,7 @@ type PluginErrorReply struct { // Error satisfies the error interface. func (e PluginErrorReply) Error() string { - return fmt.Sprintf("user error code: %v", e.ErrorCode) + return fmt.Sprintf("plugin user error code: %v", e.ErrorCode) } // ServerErrorReply is the reply that the server returns when it encounters an diff --git a/politeiawww/api/ticketvote/v1/v1.go b/politeiawww/api/ticketvote/v1/v1.go index 12f66624f..6aa23a8c7 100644 --- a/politeiawww/api/ticketvote/v1/v1.go +++ b/politeiawww/api/ticketvote/v1/v1.go @@ -53,7 +53,7 @@ type PluginErrorReply struct { // Error satisfies the error interface. func (e PluginErrorReply) Error() string { - return fmt.Sprintf("user error code: %v", e.ErrorCode) + return fmt.Sprintf("plugin error code: %v", e.ErrorCode) } // ServerErrorReply is the reply that the server returns when it encounters an From 70c3513f859b45c9b6ace375384a1b4f1d570293 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 19 Jan 2021 09:09:16 -0600 Subject: [PATCH 236/449] Move comment routes to comments api. --- .../tlogbe/plugins/comments/comments.go | 79 +- politeiad/backend/tlogbe/plugins/pi/hooks.go | 6 +- politeiad/backend/tlogbe/plugins/pi/pi.go | 5 +- politeiad/backend/tlogbe/plugins/plugins.go | 45 +- .../tlogbe/plugins/ticketvote/ticketvote.go | 5 +- politeiad/backend/tlogbe/tlog/tlogclient.go | 16 +- .../backend/tlogbe/tlogclient/tlogclient.go | 50 -- politeiad/plugins/comments/comments.go | 18 +- politeiad/plugins/pi/pi.go | 1 - politeiawww/api/comments/v1/v1.go | 171 +++- politeiawww/api/pi/v1/v1.go | 189 +---- politeiawww/comments.go | 503 ++++++++++-- politeiawww/eventmanager.go | 7 +- politeiawww/pi.go | 60 +- politeiawww/piwww.go | 731 ++---------------- politeiawww/proposals.go | 51 +- politeiawww/ticketvote.go | 122 +-- politeiawww/www.go | 121 ++- 18 files changed, 923 insertions(+), 1257 deletions(-) delete mode 100644 politeiad/backend/tlogbe/tlogclient/tlogclient.go diff --git a/politeiad/backend/tlogbe/plugins/comments/comments.go b/politeiad/backend/tlogbe/plugins/comments/comments.go index e7b0cc4a1..f517eed42 100644 --- a/politeiad/backend/tlogbe/plugins/comments/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments/comments.go @@ -22,7 +22,6 @@ import ( "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/politeiad/backend/tlogbe/store" - "github.com/decred/politeia/politeiad/backend/tlogbe/tlogclient" "github.com/decred/politeia/politeiad/plugins/comments" "github.com/decred/politeia/util" ) @@ -52,7 +51,7 @@ var ( // commentsPlugin satisfies the plugins.Client interface. type commentsPlugin struct { sync.Mutex - tlog tlogclient.Client + tlog plugins.TlogClient // dataDir is the comments plugin data directory. The only data // that is stored here is cached data that can be re-created at any @@ -274,20 +273,22 @@ func convertCommentVoteFromBlobEntry(be store.BlobEntry) (*comments.CommentVote, func convertCommentFromCommentAdd(ca comments.CommentAdd) comments.Comment { return comments.Comment{ - UserID: ca.UserID, - Token: ca.Token, - ParentID: ca.ParentID, - Comment: ca.Comment, - PublicKey: ca.PublicKey, - Signature: ca.Signature, - CommentID: ca.CommentID, - Version: ca.Version, - Timestamp: ca.Timestamp, - Receipt: ca.Receipt, - Downvotes: 0, // Not part of commentAdd data - Upvotes: 0, // Not part of commentAdd data - Deleted: false, - Reason: "", + UserID: ca.UserID, + Token: ca.Token, + ParentID: ca.ParentID, + Comment: ca.Comment, + PublicKey: ca.PublicKey, + Signature: ca.Signature, + CommentID: ca.CommentID, + Version: ca.Version, + Timestamp: ca.Timestamp, + Receipt: ca.Receipt, + Downvotes: 0, // Not part of commentAdd data + Upvotes: 0, // Not part of commentAdd data + Deleted: false, + Reason: "", + ExtraData: ca.ExtraData, + ExtraDataHint: ca.ExtraDataHint, } } @@ -713,16 +714,18 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str // Setup comment receipt := p.identity.SignMessage([]byte(n.Signature)) ca := comments.CommentAdd{ - UserID: n.UserID, - Token: n.Token, - ParentID: n.ParentID, - Comment: n.Comment, - PublicKey: n.PublicKey, - Signature: n.Signature, - CommentID: commentIDLatest(*ridx) + 1, - Version: 1, - Timestamp: time.Now().Unix(), - Receipt: hex.EncodeToString(receipt[:]), + UserID: n.UserID, + Token: n.Token, + ParentID: n.ParentID, + Comment: n.Comment, + PublicKey: n.PublicKey, + Signature: n.Signature, + CommentID: commentIDLatest(*ridx) + 1, + Version: 1, + Timestamp: time.Now().Unix(), + Receipt: hex.EncodeToString(receipt[:]), + ExtraData: n.ExtraData, + ExtraDataHint: n.ExtraDataHint, } // Save comment @@ -868,16 +871,18 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st // Create a new comment version receipt := p.identity.SignMessage([]byte(e.Signature)) ca := comments.CommentAdd{ - UserID: e.UserID, - Token: e.Token, - ParentID: e.ParentID, - Comment: e.Comment, - PublicKey: e.PublicKey, - Signature: e.Signature, - CommentID: e.CommentID, - Version: existing.Version + 1, - Timestamp: time.Now().Unix(), - Receipt: hex.EncodeToString(receipt[:]), + UserID: e.UserID, + Token: e.Token, + ParentID: e.ParentID, + Comment: e.Comment, + PublicKey: e.PublicKey, + Signature: e.Signature, + CommentID: e.CommentID, + Version: existing.Version + 1, + Timestamp: time.Now().Unix(), + Receipt: hex.EncodeToString(receipt[:]), + ExtraData: e.ExtraData, + ExtraDataHint: e.ExtraDataHint, } // Save comment @@ -1579,7 +1584,7 @@ func (p *commentsPlugin) Fsck() error { } // New returns a new comments plugin. -func New(tlog tlogclient.Client, settings []backend.PluginSetting, dataDir string, id *identity.FullIdentity) (*commentsPlugin, error) { +func New(tlog plugins.TlogClient, settings []backend.PluginSetting, dataDir string, id *identity.FullIdentity) (*commentsPlugin, error) { // Setup comments plugin data dir dataDir = filepath.Join(dataDir, comments.ID) err := os.MkdirAll(dataDir, 0700) diff --git a/politeiad/backend/tlogbe/plugins/pi/hooks.go b/politeiad/backend/tlogbe/plugins/pi/hooks.go index 922bd7f75..c2180e955 100644 --- a/politeiad/backend/tlogbe/plugins/pi/hooks.go +++ b/politeiad/backend/tlogbe/plugins/pi/hooks.go @@ -217,7 +217,7 @@ func (p *piPlugin) hookSetRecordStatusPost(payload string) error { if sc.Version != srs.Current.Version { e := fmt.Sprintf("version not current: got %v, want %v", sc.Version, srs.Current.Version) - return backend.PluginUserError{ + return backend.PluginError{ PluginID: pi.ID, ErrorCode: int(pi.ErrorCodePropVersionInvalid), ErrorContext: e, @@ -236,7 +236,7 @@ func (p *piPlugin) hookSetRecordStatusPost(payload string) error { if !isAllowed { e := fmt.Sprintf("from %v to %v status change not allowed", from, sc.Status) - return backend.PluginUserError{ + return backend.PluginError{ PluginID: pi.ID, ErrorCode: int(pi.ErrorCodePropStatusChangeInvalid), ErrorContext: e, @@ -260,7 +260,7 @@ func (p *piPlugin) commentWritesVerify(token []byte) error { // Writes are allowed on these vote statuses return nil default: - return backend.PluginUserError{ + return backend.PluginError{ PluginID: pi.ID, ErrorCode: int(pi.ErrorCodeVoteStatusInvalid), ErrorContext: "vote has ended; proposal is locked", diff --git a/politeiad/backend/tlogbe/plugins/pi/pi.go b/politeiad/backend/tlogbe/plugins/pi/pi.go index 9201d5b64..3b82250d0 100644 --- a/politeiad/backend/tlogbe/plugins/pi/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi/pi.go @@ -14,7 +14,6 @@ import ( "github.com/decred/dcrd/chaincfg/v3" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" - "github.com/decred/politeia/politeiad/backend/tlogbe/tlogclient" "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/ticketvote" "github.com/decred/politeia/util" @@ -28,7 +27,7 @@ var ( type piPlugin struct { sync.Mutex backend backend.Backend - tlog tlogclient.Client + tlog plugins.TlogClient activeNetParams *chaincfg.Params // dataDir is the pi plugin data directory. The only data that is @@ -279,7 +278,7 @@ func (p *piPlugin) Fsck() error { return nil } -func New(backend backend.Backend, tlog tlogclient.Client, settings []backend.PluginSetting, dataDir string, activeNetParams *chaincfg.Params) (*piPlugin, error) { +func New(backend backend.Backend, tlog plugins.TlogClient, settings []backend.PluginSetting, dataDir string, activeNetParams *chaincfg.Params) (*piPlugin, error) { // Create plugin data directory dataDir = filepath.Join(dataDir, pi.ID) err := os.MkdirAll(dataDir, 0700) diff --git a/politeiad/backend/tlogbe/plugins/plugins.go b/politeiad/backend/tlogbe/plugins/plugins.go index 3799d9314..dd3112345 100644 --- a/politeiad/backend/tlogbe/plugins/plugins.go +++ b/politeiad/backend/tlogbe/plugins/plugins.go @@ -4,7 +4,10 @@ package plugins -import "github.com/decred/politeia/politeiad/backend" +import ( + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/store" +) // HookT represents the types of plugin hooks. type HookT int @@ -144,3 +147,43 @@ type Client interface { // Fsck performs a plugin file system check. Fsck() error } + +// TODO plugins should only have access to the backend methods for the tlog +// instance that they're registered on. +// TODO the plugin hook state should not really be required. This issue is that +// some vetted plugins require unvetted hooks, ex. verifying the linkto in +// vote metadata. Possile solution, keep the layer violations in the +// application plugin (pi) instead of the functionality plugin (ticketvote). + +// TlogClient provides an API for plugins to interact with a tlog instance. +// Plugins are allowed to save, delete, and get plugin data to/from the tlog +// backend. Editing plugin data is not allowed. +type TlogClient interface { + // BlobSave saves a BlobEntry to the tlog instance. The BlobEntry + // will be encrypted prior to being written to disk if the tlog + // instance has an encryption key set. The digest of the data, + // i.e. BlobEntry.Digest, can be thought of as the blob ID and can + // be used to get/del the blob from tlog. + BlobSave(treeID int64, dataType string, be store.BlobEntry) error + + // BlobsDel deletes the blobs that correspond to the provided + // digests. + BlobsDel(treeID int64, digests [][]byte) error + + // Blobs returns the blobs that correspond to the provided digests. + // If a blob does not exist it will not be included in the returned + // map. + Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEntry, error) + + // BlobsByDataType returns all blobs that match the data type. The + // blobs will be ordered from oldest to newest. + BlobsByDataType(treeID int64, keyPrefix string) ([]store.BlobEntry, error) + + // DigestsByDataType returns the digests of all blobs that match + // the data type. + DigestsByDataType(treeID int64, dataType string) ([][]byte, error) + + // Timestamp returns the timestamp for the blob that correpsonds + // to the digest. + Timestamp(treeID int64, digest []byte) (*backend.Timestamp, error) +} diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go index 27f6fa68b..6856c3db0 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go @@ -17,7 +17,6 @@ import ( "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" - "github.com/decred/politeia/politeiad/backend/tlogbe/tlogclient" "github.com/decred/politeia/politeiad/plugins/dcrdata" "github.com/decred/politeia/politeiad/plugins/ticketvote" ) @@ -32,7 +31,7 @@ var ( type ticketVotePlugin struct { sync.Mutex backend backend.Backend - tlog tlogclient.Client + tlog plugins.TlogClient activeNetParams *chaincfg.Params // dataDir is the ticket vote plugin data directory. The only data @@ -276,7 +275,7 @@ func (p *politeiawww) linkByPeriodMax() int64 { } */ -func New(backend backend.Backend, tlog tlogclient.Client, settings []backend.PluginSetting, dataDir string, id *identity.FullIdentity, activeNetParams *chaincfg.Params) (*ticketVotePlugin, error) { +func New(backend backend.Backend, tlog plugins.TlogClient, settings []backend.PluginSetting, dataDir string, id *identity.FullIdentity, activeNetParams *chaincfg.Params) (*ticketVotePlugin, error) { // Plugin settings var ( voteDurationMin uint32 diff --git a/politeiad/backend/tlogbe/tlog/tlogclient.go b/politeiad/backend/tlogbe/tlog/tlogclient.go index b36bbc0f6..1853fa735 100644 --- a/politeiad/backend/tlogbe/tlog/tlogclient.go +++ b/politeiad/backend/tlogbe/tlog/tlogclient.go @@ -10,14 +10,14 @@ import ( "strings" "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/politeiad/backend/tlogbe/store" - "github.com/decred/politeia/politeiad/backend/tlogbe/tlogclient" "github.com/google/trillian" "google.golang.org/grpc/codes" ) var ( - _ tlogclient.Client = (*Tlog)(nil) + _ plugins.TlogClient = (*Tlog)(nil) ) // BlobSave saves a BlobEntry to the tlog instance. The BlobEntry will be @@ -25,7 +25,7 @@ var ( // encryption key set. The digest of the data, i.e. BlobEntry.Digest, can be // thought of as the blob ID and can be used to get/del the blob from tlog. // -// This function satisfies the tlogclient.Client interface. +// This function satisfies the plugins.TlogClient interface. func (t *Tlog) BlobSave(treeID int64, dataType string, be store.BlobEntry) error { log.Tracef("%v BlobSave: %v %v", t.id, treeID, dataType) @@ -93,7 +93,7 @@ func (t *Tlog) BlobSave(treeID int64, dataType string, be store.BlobEntry) error // BlobsDel deletes the blobs that correspond to the provided digests. // -// This function satisfies the tlogclient.Client interface. +// This function satisfies the plugins.TlogClient interface. func (t *Tlog) BlobsDel(treeID int64, digests [][]byte) error { log.Tracef("%v BlobsDel: %v %x", t.id, treeID, digests) @@ -140,7 +140,7 @@ func (t *Tlog) BlobsDel(treeID int64, digests [][]byte) error { // Blobs returns the blobs that correspond to the provided digests. If a blob // does not exist it will not be included in the returned map. // -// This function satisfies the tlogclient.Client interface. +// This function satisfies the plugins.TlogClient interface. func (t *Tlog) Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEntry, error) { log.Tracef("%v Blobs: %v %x", t.id, treeID, digests) @@ -219,7 +219,7 @@ func (t *Tlog) Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEntry // BlobsByDataType returns all blobs that match the data type. // -// This function satisfies the tlogclient.Client interface. +// This function satisfies the plugins.TlogClient interface. func (t *Tlog) BlobsByDataType(treeID int64, dataType string) ([]store.BlobEntry, error) { log.Tracef("%v BlobsByDataType: %v %v", t.id, treeID, dataType) @@ -280,7 +280,7 @@ func (t *Tlog) BlobsByDataType(treeID int64, dataType string) ([]store.BlobEntry // DigestsByDataType returns the digests of all blobs that match the data type. // -// This function satisfies the tlogclient.Client interface. +// This function satisfies the plugins.TlogClient interface. func (t *Tlog) DigestsByDataType(treeID int64, dataType string) ([][]byte, error) { log.Tracef("%v DigestsByDataType: %v %v", t.id, treeID, dataType) @@ -310,7 +310,7 @@ func (t *Tlog) DigestsByDataType(treeID int64, dataType string) ([][]byte, error // Timestamp returns the timestamp for the data blob that corresponds to the // provided digest. // -// This function satisfies the tlogclient.Client interface. +// This function satisfies the plugins.TlogClient interface. func (t *Tlog) Timestamp(treeID int64, digest []byte) (*backend.Timestamp, error) { log.Tracef("%v Timestamp: %v %x", t.id, treeID, digest) diff --git a/politeiad/backend/tlogbe/tlogclient/tlogclient.go b/politeiad/backend/tlogbe/tlogclient/tlogclient.go deleted file mode 100644 index a44ed1762..000000000 --- a/politeiad/backend/tlogbe/tlogclient/tlogclient.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) 2020-2021 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package tlogclient - -import ( - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/store" -) - -// TODO plugins should only have access to the backend methods for the tlog -// instance that they're registered on. -// TODO the plugin hook state should not really be required. This issue is that -// some vetted plugins require unvetted hooks, ex. verifying the linkto in -// vote metadata. Possile solution, keep the layer violations in the -// application plugin (pi) instead of the functionality plugin (ticketvote). - -// Client provides an API for plugins to interact with a tlog instance. -// Plugins are allowed to save, delete, and get plugin data to/from the tlog -// backend. Editing plugin data is not allowed. -type Client interface { - // BlobSave saves a BlobEntry to the tlog instance. The BlobEntry - // will be encrypted prior to being written to disk if the tlog - // instance has an encryption key set. The digest of the data, - // i.e. BlobEntry.Digest, can be thought of as the blob ID and can - // be used to get/del the blob from tlog. - BlobSave(treeID int64, dataType string, be store.BlobEntry) error - - // BlobsDel deletes the blobs that correspond to the provided - // digests. - BlobsDel(treeID int64, digests [][]byte) error - - // Blobs returns the blobs that correspond to the provided digests. - // If a blob does not exist it will not be included in the returned - // map. - Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEntry, error) - - // BlobsByDataType returns all blobs that match the data type. The - // blobs will be ordered from oldest to newest. - BlobsByDataType(treeID int64, keyPrefix string) ([]store.BlobEntry, error) - - // DigestsByDataType returns the digests of all blobs that match - // the data type. - DigestsByDataType(treeID int64, dataType string) ([][]byte, error) - - // Timestamp returns the timestamp for the blob that correpsonds - // to the digest. - Timestamp(treeID int64, digest []byte) (*backend.Timestamp, error) -} diff --git a/politeiad/plugins/comments/comments.go b/politeiad/plugins/comments/comments.go index df52a3458..e7ec14a9f 100644 --- a/politeiad/plugins/comments/comments.go +++ b/politeiad/plugins/comments/comments.go @@ -6,8 +6,6 @@ // records. package comments -// TODO add a hint to comments that can be used freely by the client. This -// is how we'll distinguish proposal comments from update comments. const ( ID = "comments" @@ -69,6 +67,10 @@ type Comment struct { Deleted bool `json:"deleted,omitempty"` // Comment has been deleted Reason string `json:"reason,omitempty"` // Reason for deletion + + // Optional fields to be used freely + ExtraData string `json:"extradata,omitempty"` + ExtraDataHint string `json:"extradatahint,omitempty"` } // CommentAdd is the structure that is saved to disk when a comment is created @@ -89,6 +91,10 @@ type CommentAdd struct { Version uint32 `json:"version"` // Comment version Timestamp int64 `json:"timestamp"` // Received UNIX timestamp Receipt string `json:"receipt"` // Server signature of client signature + + // Optional fields to be used freely + ExtraData string `json:"extradata,omitempty"` + ExtraDataHint string `json:"extradatahint,omitempty"` } // CommentDel is the structure that is saved to disk when a comment is deleted. @@ -158,6 +164,10 @@ type New struct { Comment string `json:"comment"` // Comment text PublicKey string `json:"publickey"` // Pubkey used for Signature Signature string `json:"signature"` // Client signature + + // Optional fields to be used freely + ExtraData string `json:"extradata,omitempty"` + ExtraDataHint string `json:"extradatahint,omitempty"` } // NewReply is the reply to the New command. @@ -176,6 +186,10 @@ type Edit struct { Comment string `json:"comment"` // Comment text PublicKey string `json:"publickey"` // Pubkey used for Signature Signature string `json:"signature"` // Client signature + + // Optional fields to be used freely + ExtraData string `json:"extradata,omitempty"` + ExtraDataHint string `json:"extradatahint,omitempty"` } // EditReply is the reply to the Edit command. diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index ffdca359f..dd4cc5758 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -69,7 +69,6 @@ type GeneralMetadata struct { UserID string `json:"userid"` // Author user ID PublicKey string `json:"publickey"` // Key used for signature Signature string `json:"signature"` // Signature of merkle root - Timestamp int64 `json:"timestamp"` // Submission UNIX timestamp } // PropStatusT represents a proposal status. diff --git a/politeiawww/api/comments/v1/v1.go b/politeiawww/api/comments/v1/v1.go index 47af3cc8e..6e9ebe6b5 100644 --- a/politeiawww/api/comments/v1/v1.go +++ b/politeiawww/api/comments/v1/v1.go @@ -10,6 +10,12 @@ const ( // APIRoute is prefixed onto all routes defined in this package. APIRoute = "/comments/v1" + RouteNew = "/new" + RouteVote = "/vote" + RouteDel = "/del" + RouteCount = "/count" + RouteComments = "/comments" + RouteVotes = "/votes" RouteTimestamps = "/timestamps" ) @@ -19,14 +25,17 @@ type ErrorCodeT int const ( // Error codes ErrorCodeInvalid ErrorCodeT = 0 - ErrorCodeInputInvalid ErrorCodeT = 1 + ErrorCodeInputInvalid ErrorCodeT = iota + ErrorStatusPublicKeyInvalid + ErrorStatusSignatureInvalid ) var ( // ErrorCodes contains the human readable errors. ErrorCodes = map[ErrorCodeT]string{ - ErrorCodeInvalid: "error invalid", - ErrorCodeInputInvalid: "input invalid", + ErrorCodeInvalid: "error invalid", + ErrorCodeInputInvalid: "input invalid", + ErrorStatusPublicKeyInvalid: "public key invalid", } ) @@ -84,6 +93,162 @@ const ( RecordStateVetted RecordStateT = 2 ) +// Comment represent a record comment. +// +// Signature is the client signature of Token+ParentID+Comment. +type Comment struct { + UserID string `json:"userid"` // Unique user ID + Username string `json:"username"` // Username + Token string `json:"token"` // Record token + ParentID uint32 `json:"parentid"` // Parent comment ID if reply + Comment string `json:"comment"` // Comment text + PublicKey string `json:"publickey"` // Public key used for Signature + Signature string `json:"signature"` // Client signature + CommentID uint32 `json:"commentid"` // Comment ID + Timestamp int64 `json:"timestamp"` // UNIX timestamp of last edit + Receipt string `json:"receipt"` // Server signature of client signature + Downvotes uint64 `json:"downvotes"` // Tolal downvotes on comment + Upvotes uint64 `json:"upvotes"` // Total upvotes on comment + + Deleted bool `json:"deleted,omitempty"` // Comment has been deleted + Reason string `json:"reason,omitempty"` // Reason for deletion + + // Optional fields to be used freely + ExtraData string `json:"extradata,omitempty"` + ExtraDataHint string `json:"extradatahint,omitempty"` +} + +// CommentVote represents a comment vote (upvote/downvote). +// +// Signature is the client signature of the Token+CommentID+Vote. +type CommentVote struct { + UserID string `json:"userid"` // Unique user ID + Username string `json:"username"` // Username + Token string `json:"token"` // Record token + CommentID uint32 `json:"commentid"` // Comment ID + Vote VoteT `json:"vote"` // Upvote or downvote + PublicKey string `json:"publickey"` // Public key used for signature + Signature string `json:"signature"` // Client signature + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + Receipt string `json:"receipt"` // Server signature of client signature +} + +// New creates a new comment. +// +// The parent ID is used to reply to an existing comment. A parent ID of 0 +// indicates that the comment is a base level comment and not a reply commment. +// +// Signature is the client signature of Token+ParentID+Comment. +type New struct { + State RecordStateT `json:"state"` + Token string `json:"token"` + ParentID uint32 `json:"parentid"` + Comment string `json:"comment"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` + + // Optional fields to be used freely + ExtraData string `json:"extradata,omitempty"` + ExtraDataHint string `json:"extradatahint,omitempty"` +} + +// NewReply is the reply to the New command. +type NewReply struct { + Comment Comment `json:"comment"` +} + +// VoteT represents a comment upvote/downvote. +type VoteT int + +const ( + // VoteInvalid is an invalid comment vote. + VoteInvalid VoteT = 0 + + // VoteDownvote represents a comment downvote. + VoteDownvote VoteT = -1 + + // VoteUpvote represents a comment upvote. + VoteUpvote VoteT = 1 +) + +// Vote casts a comment vote (upvote or downvote). +// +// The effect of a new vote on a comment score depends on the previous vote +// from that user ID. Example, a user upvotes a comment that they have already +// upvoted, the resulting vote score is 0 due to the second upvote removing the +// original upvote. +// +// Signature is the client signature of the Token+CommentID+Vote. +type Vote struct { + State RecordStateT `json:"state"` + Token string `json:"token"` + CommentID uint32 `json:"commentid"` + Vote VoteT `json:"vote"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` +} + +// VoteReply is the reply to the Vote command. +type VoteReply struct { + Downvotes uint64 `json:"downvotes"` // Tolal downvotes on comment + Upvotes uint64 `json:"upvotes"` // Total upvotes on comment + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + Receipt string `json:"receipt"` // Server signature of client signature +} + +// Del permanently deletes the provided comment. Only admins can delete +// comments. A reason must be given for the deletion. +// +// Signature is the client signature of the Token+CommentID+Reason +type Del struct { + State RecordStateT `json:"state"` + Token string `json:"token"` + CommentID uint32 `json:"commentid"` + Reason string `json:"reason"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` +} + +// DelReply is the reply to the Del command. +type DelReply struct { + Comment Comment `json:"comment"` +} + +// Count requests the number of comments on that have been made on the given +// records. If a record is not found for a token then it will not be included +// in the reply. +type Count struct { + State RecordStateT `json:"state"` + Tokens []string `json:"tokens"` +} + +// CountReply is the reply to the count command. +type CountReply struct { + Count map[string]uint32 `json:"count"` +} + +// Comments requests a record's comments. +type Comments struct { + State RecordStateT `json:"state"` + Token string `json:"token"` +} + +// CommentsReply is the reply to the comments command. +type CommentsReply struct { + Comments []Comment `json:"comments"` +} + +// Votes returns the comment votes that meet the provided filtering criteria. +type Votes struct { + State RecordStateT `json:"state"` + UserID string `json:"userid"` +} + +// VotesReply is the reply to the Votes command. +type VotesReply struct { + Votes []CommentVote `json:"votes"` +} + // Proof contains an inclusion proof for the digest in the merkle root. The // ExtraData field is used by certain types of proofs to include additional // data that is required to validate the proof. diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 80656d0ab..8084ee0bb 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -8,7 +8,6 @@ import ( "fmt" ) -type CommentVoteT int type VoteStatusT int type VoteAuthActionT string type VoteT int @@ -16,11 +15,7 @@ type VoteT int // TODO verify that all batched request have a page size limit // TODO comments count and linked from should be pulled out of the proposal // record struct. These should be separate endpoints: -// /comments/count -// /proposal/linkedfrom -// TODO routes that map directly to plugin commands (comment and vote routes) -// should be added to their own API package so that they can be used by -// multiple politeia applications (pi, cms, forum). +// /ticketvote/linkedfrom // TODO make RouteVoteResults a batched route but that only currently allows // for 1 result to be returned so that we have the option to change this if // we want to. @@ -39,13 +34,6 @@ const ( RouteProposals = "/proposals" RouteProposalInventory = "/proposals/inventory" - // Comment routes - RouteCommentNew = "/comment/new" - RouteCommentVote = "/comment/vote" - RouteCommentCensor = "/comment/censor" - RouteComments = "/comments" - RouteCommentVotes = "/comments/votes" - // Vote routes RouteVoteAuthorize = "/vote/authorize" RouteVoteStart = "/vote/start" @@ -55,11 +43,6 @@ const ( RouteVoteSummaries = "/votes/summaries" RouteVoteInventory = "/votes/inventory" - // Comment vote types - CommentVoteInvalid CommentVoteT = 0 - CommentVoteDownvote CommentVoteT = -1 - CommentVoteUpvote CommentVoteT = 1 - // Vote statuses VoteStatusInvalid VoteStatusT = 0 // Invalid status VoteStatusUnauthorized VoteStatusT = 1 // Vote has not been authorized @@ -151,13 +134,6 @@ const ( ErrorStatusPropStatusChangeReasonInvalid ErrorStatusNoPropChanges - // Comment errors - ErrorStatusCommentTextInvalid - ErrorStatusCommentParentIDInvalid - ErrorStatusCommentVoteInvalid - ErrorStatusCommentNotFound - ErrorStatusCommentVoteChangesMax - // Vote errors ErrorStatusVoteAuthInvalid ErrorStatusVoteStatusInvalid @@ -213,13 +189,6 @@ var ( ErrorStatusPropStatusChangeReasonInvalid: "proposal status reason invalid", ErrorStatusNoPropChanges: "no proposal changes", - // Comment errors - ErrorStatusCommentTextInvalid: "comment text invalid", - ErrorStatusCommentParentIDInvalid: "comment parent ID invalid", - ErrorStatusCommentVoteInvalid: "comment vote invalid", - ErrorStatusCommentNotFound: "comment not found", - ErrorStatusCommentVoteChangesMax: "comment vote changes exceeded max", - // Vote errors ErrorStatusVoteStatusInvalid: "vote status invalid", ErrorStatusVoteParamsInvalid: "vote params invalid", @@ -231,7 +200,7 @@ var ( // timing, etc). The HTTP status code will be 400. type UserErrorReply struct { ErrorCode ErrorStatusT `json:"errorcode"` - ErrorContext []string `json:"errorcontext"` + ErrorContext string `json:"errorcontext"` } // Error satisfies the error interface. @@ -337,7 +306,12 @@ const ( // submission. It is attached to a proposal submission as a Metadata object. type ProposalMetadata struct { Name string `json:"name"` // Proposal name +} +// VoteMetadata that is specified by the user on proposal submission in order +// to host or participate in certain types of votes. It is attached to a +// proposal submission as a Metadata object. +type VoteMetadata struct { // LinkBy is a UNIX timestamp that serves as a deadline for other // proposals to link to this proposal. Ex, an RFP submission cannot // link to an RFP proposal once the RFP's LinkBy deadline is past. @@ -389,16 +363,9 @@ type ProposalRecord struct { Username string `json:"username"` // Author username PublicKey string `json:"publickey"` // Key used in signature Signature string `json:"signature"` // Signature of merkle root - Comments uint64 `json:"comments"` // Number of comments - Statuses []StatusChange `json:"statuses"` // Status change history Files []File `json:"files"` // Proposal files Metadata []Metadata `json:"metadata"` // User defined metadata - - // LinkedFrom contains a list of public proposals that have linked - // to this proposal. A link is established when a child proposal - // specifies this proposal using the LinkTo field of the - // ProposalMetadata. - LinkedFrom []string `json:"linkedfrom"` + Statuses []StatusChange `json:"statuses"` // Status change history // CensorshipRecord contains cryptographic proof that the proposal // was received and processed by the server. @@ -508,146 +475,6 @@ type ProposalInventoryReply struct { Vetted map[string][]string `json:"vetted"` } -// Comment represent a proposal comment. -// -// The parent ID is used to reply to an existing comment. A parent ID of 0 -// indicates that the comment is a base level comment and not a reply commment. -// -// Signature is the client signature of State+Token+ParentID+Comment. -type Comment struct { - UserID string `json:"userid"` // User ID - Username string `json:"username"` // Username - State PropStateT `json:"state"` // Proposal state - Token string `json:"token"` // Proposal token - ParentID uint32 `json:"parentid"` // Parent comment ID - Comment string `json:"comment"` // Comment text - PublicKey string `json:"publickey"` // Public key used for Signature - Signature string `json:"signature"` // Client signature - CommentID uint32 `json:"commentid"` // Comment ID - Timestamp int64 `json:"timestamp"` // UNIX timestamp of last edit - Receipt string `json:"receipt"` // Server sig of client sig - Downvotes uint64 `json:"downvotes"` // Tolal downvotes - Upvotes uint64 `json:"upvotes"` // Total upvotes - - Censored bool `json:"censored,omitempty"` // Comment has been censored - Reason string `json:"reason,omitempty"` // Reason for censoring -} - -// CommentNew creates a new comment. Only the proposal author and admins can -// comment on unvetted proposals. All users can comment on public proposals. -// -// The parent ID is used to reply to an existing comment. A parent ID of 0 -// indicates that the comment is a base level comment and not a reply commment. -// -// Signature is the client signature of State+Token+ParentID+Comment. -type CommentNew struct { - State PropStateT `json:"state"` - Token string `json:"token"` - ParentID uint32 `json:"parentid"` - Comment string `json:"comment"` - PublicKey string `json:"publickey"` - Signature string `json:"signature"` -} - -// CommentNewReply is the reply to the CommentNew command. -// -// Receipt is the server signature of the client signature. This is proof that -// the server received and processed the CommentNew command. -type CommentNewReply struct { - Comment Comment `json:"comment"` -} - -// CommentCensor permanently censors a comment. The comment will be deleted -// and cannot be retrieved once censored. Only admins can censor a comment. -// -// Reason contains the reason why the comment is being censored and must always -// be included. -type CommentCensor struct { - State PropStateT `json:"state"` - Token string `json:"token"` - CommentID uint32 `json:"commentid"` - Reason string `json:"reason"` - PublicKey string `json:"publickey"` - Signature string `json:"signature"` -} - -// CommentCensorReply is the reply to the CommentCensor command. -// -// Receipt is the server signature of the client signature. This is proof that -// the server received and processed the CommentCensor command. -type CommentCensorReply struct { - Comment Comment `json:"comment"` -} - -// CommentVote casts a comment vote (upvote or downvote). Only allowed on -// vetted proposals. -// -// The effect of a new vote on a comment score depends on the previous vote -// from that uuid. Example, a user upvotes a comment that they have already -// upvoted, the resulting vote score is 0 due to the second upvote removing the -// original upvote. -// -// Signature is the client signature of the State+Token+CommentID+Vote. -type CommentVote struct { - State PropStateT `json:"state"` - Token string `json:"token"` - CommentID uint32 `json:"commentid"` - Vote CommentVoteT `json:"vote"` - PublicKey string `json:"publickey"` - Signature string `json:"signature"` -} - -// CommentVoteReply is the reply to the CommentVote command. -// -// Receipt is the server signature of the client signature. This is proof that -// the server received and processed the CommentVote command. -type CommentVoteReply struct { - Downvotes uint64 `json:"downvotes"` // Total downvotes - Upvotes uint64 `json:"upvotes"` // Total upvotes - Timestamp int64 `json:"timestamp"` - Receipt string `json:"receipt"` -} - -// Comments returns all comments for a proposal. Unvetted proposal comments -// are only returned to the proposal author and admins. Retrieving proposal -// comments on vetted proposals does not require a user to be logged in. -type Comments struct { - State PropStateT `json:"state"` - Token string `json:"token"` -} - -// CommentsReply is the reply to the comments command. -type CommentsReply struct { - Comments []Comment `json:"comments"` -} - -// CommentVoteDetails represents all user generated data and server generated -// metadata for a comment vote. -type CommentVoteDetails struct { - UserID string `json:"userid"` - State PropStateT `json:"state"` - Token string `json:"token"` - CommentID uint32 `json:"commentid"` - Vote CommentVoteT `json:"vote"` - PublicKey string `json:"publickey"` - Signature string `json:"signature"` - Timestamp int64 `json:"timestamp"` - Receipt string `json:"receipt"` -} - -// CommentVotes returns all comment votes that meet the provided filtering -// criteria. Comment votes are only allowed on vetted proposals. -type CommentVotes struct { - State PropStateT `json:"state"` - Token string `json:"token"` - UserID string `json:"userid"` -} - -// CommentVotesReply is the reply to the CommentVotes command. -type CommentVotesReply struct { - Votes []CommentVoteDetails `json:"votes"` -} - // AuthDetails contains the details of a vote authorization. type AuthDetails struct { Token string `json:"token"` // Proposal token diff --git a/politeiawww/comments.go b/politeiawww/comments.go index c3759a651..8882ee5d8 100644 --- a/politeiawww/comments.go +++ b/politeiawww/comments.go @@ -16,19 +16,12 @@ import ( "github.com/decred/politeia/politeiad/plugins/comments" cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" + piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" + "github.com/decred/politeia/politeiawww/user" "github.com/decred/politeia/util" + "github.com/google/uuid" ) -func convertCommentState(s cmv1.RecordStateT) comments.StateT { - switch s { - case cmv1.RecordStateUnvetted: - return comments.StateUnvetted - case cmv1.RecordStateVetted: - return comments.StateVetted - } - return comments.StateInvalid -} - func convertProofFromCommentsPlugin(p comments.Proof) cmv1.Proof { return cmv1.Proof{ Type: p.Type, @@ -53,73 +46,294 @@ func convertTimestampFromCommentsPlugin(t comments.Timestamp) cmv1.Timestamp { } } -// commentsAll returns all comments for the provided record. -func (p *politeiawww) commentsAll(ctx context.Context, cp comments.GetAll) (*comments.GetAllReply, error) { - b, err := comments.EncodeGetAll(cp) - if err != nil { - return nil, err +func (p *politeiawww) commentsAll(ctx context.Context, ga comments.GetAll) (*comments.GetAllReply, error) { + return nil, nil +} + +func (p *politeiawww) commentsGet(ctx context.Context, cg comments.Get) (*comments.GetReply, error) { + return nil, nil +} + +func (p *politeiawww) commentVotes(ctx context.Context, vs comments.Votes) (*comments.VotesReply, error) { + return nil, nil +} + +func (p *politeiawww) commentTimestamps(ctx context.Context, t comments.Timestamps) (*comments.TimestampsReply, error) { + return nil, nil +} + +// commentPopulateUser populates the provided comment with user data that is +// not stored in politeiad. +func commentPopulateUser(c piv1.Comment, u user.User) cmv1.Comment { + c.Username = u.Username + return c +} + +func (p *politeiawww) piCommentNew(ctx context.Context, n cmv1.CommentNew, u user.User) error { + // Verify user has paid registration paywall + if !p.userHasPaid(u) { + return nil, cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodeUserRegistrationNotPaid, + } } - r, err := p.pluginCommand(ctx, comments.ID, comments.CmdGetAll, string(b)) - if err != nil { - return nil, err + +} + +func (p *politeiawww) processCommentNew(ctx context.Context, n cmv1.CommentNew, u user.User) (*cmv1.CommentNewReply, error) { + log.Tracef("processCommentNew: %v %v", n.Token, u.Username) + + // This is temporary until user plugins are implemented. + switch p.mode { + case politeiaWWWMode: + err := piCommentNew(ctx, n, u) + if err != nil { + return nil, err + } } - cr, err := comments.DecodeGetAllReply([]byte(r)) - if err != nil { - return nil, err + + // Verify user signed using active identity + if u.PublicKey() != n.PublicKey { + return nil, cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodePublicKeyInvalid, + ErrorContext: "not active identity", + } + } + + // Only admins and the record author are allowed to comment on + // unvetted records. + if n.State == cmv1.PropStateUnvetted && !u.Admin { + // Get the record author + // TODO create a user politeiad plugin + // TODO add command to get author for record + // Fetch the proposal so we can see who the author is + pr, err := p.proposalRecordLatest(ctx, n.State, n.Token) + if err != nil { + if errors.Is(err, errProposalNotFound) { + return nil, cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodeRecordNotFound, + } + } + return nil, fmt.Errorf("proposalRecordLatest: %v", err) + } + if u.ID.String() != pr.UserID { + return nil, cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodeUnauthorized, + ErrorContext: "user is not author or admin", + } + } + } + + // Send plugin command + n := comments.New{ + UserID: usr.ID.String(), + Token: n.Token, + ParentID: n.ParentID, + Comment: n.Comment, + PublicKey: n.PublicKey, + Signature: n.Signature, } - return cr, nil + // TODO + _ = n + var nr comments.NewReply + + // Prepare reply + c := convertCommentFromPlugin(nr.Comment) + c = commentPopulateUser(c, u) + + // Emit event + p.eventManager.emit(eventProposalComment, + dataProposalComment{ + state: c.State, + token: c.Token, + commentID: c.CommentID, + parentID: c.ParentID, + username: c.Username, + }) + + return &cmv1.CommentNewReply{ + Comment: c, + }, nil } -// commentsGet returns the set of comments specified in the comment's id slice. -func (p *politeiawww) commentsGet(ctx context.Context, cg comments.Get) (*comments.GetReply, error) { - b, err := comments.EncodeGet(cg) - if err != nil { - return nil, err +func (p *politeiawww) processCommentVote(ctx context.Context, cv cmv1.CommentVote, usr user.User) (*cmv1.CommentVoteReply, error) { + log.Tracef("processCommentVote: %v %v %v", cv.Token, cv.CommentID, cv.Vote) + + // Verify state + if cv.State != cmv1.PropStateVetted { + return nil, cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodePropStateInvalid, + ErrorContext: "proposal must be vetted", + } } - r, err := p.pluginCommand(ctx, comments.ID, comments.CmdGet, string(b)) - if err != nil { - return nil, err + + // Verify user has paid registration paywall + if !p.userHasPaid(usr) { + return nil, cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodeUserRegistrationNotPaid, + } } - cgr, err := comments.DecodeGetReply([]byte(r)) - if err != nil { - return nil, err + + // Verify user signed using active identity + if usr.PublicKey() != cv.PublicKey { + return nil, cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodePublicKeyInvalid, + ErrorContext: "not active identity", + } + } + + // Send plugin command + v := comments.Vote{ + UserID: usr.ID.String(), + Token: cv.Token, + CommentID: cv.CommentID, + Vote: convertCommentVoteFromPi(cv.Vote), + PublicKey: cv.PublicKey, + Signature: cv.Signature, } - return cgr, nil + // TODO + _ = v + var vr comments.VoteReply + + return &cmv1.CommentVoteReply{ + Downvotes: vr.Downvotes, + Upvotes: vr.Upvotes, + Timestamp: vr.Timestamp, + Receipt: vr.Receipt, + }, nil } -// commentVotes returns the comment votes that meet the provided criteria. -func (p *politeiawww) commentVotes(ctx context.Context, vs comments.Votes) (*comments.VotesReply, error) { - b, err := comments.EncodeVotes(vs) - if err != nil { - return nil, err +func (p *politeiawww) processCommentCensor(ctx context.Context, cc cmv1.CommentCensor, usr user.User) (*cmv1.CommentCensorReply, error) { + log.Tracef("processCommentCensor: %v %v", cc.Token, cc.CommentID) + + // Sanity check + if !usr.Admin { + return nil, fmt.Errorf("not an admin") } - r, err := p.pluginCommand(ctx, comments.ID, comments.CmdVotes, string(b)) - if err != nil { - return nil, err + + // Verify user signed with their active identity + if usr.PublicKey() != cc.PublicKey { + return nil, cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodePublicKeyInvalid, + ErrorContext: "not active identity", + } } - vsr, err := comments.DecodeVotesReply([]byte(r)) - if err != nil { - return nil, err + + // Send plugin command + d := comments.Del{ + Token: cc.Token, + CommentID: cc.CommentID, + Reason: cc.Reason, + PublicKey: cc.PublicKey, + Signature: cc.Signature, } - return vsr, nil + // TODO + _ = d + var dr comments.DelReply + + // Prepare reply + c := convertCommentFromPlugin(dr.Comment) + c = commentPopulateUser(c, usr) + + return &cmv1.CommentCensorReply{ + Comment: c, + }, nil } -func (p *politeiawww) commentTimestamps(ctx context.Context, t comments.Timestamps) (*comments.TimestampsReply, error) { - b, err := json.Marshal(t) - if err != nil { - return nil, err +func (p *politeiawww) processComments(ctx context.Context, c cmv1.Comments, usr *user.User) (*cmv1.CommentsReply, error) { + log.Tracef("processComments: %v", c.Token) + + // Only admins and the proposal author are allowed to retrieve + // unvetted comments. This is a public route so a user might not + // exist. + if c.State == cmv1.PropStateUnvetted { + var isAllowed bool + switch { + case usr == nil: + // No logged in user. Unvetted not allowed. + case usr.Admin: + // User is an admin. Unvetted is allowed. + isAllowed = true + default: + // Logged in user is not an admin. Check if they are the + // proposal author. + pr, err := p.proposalRecordLatest(ctx, c.State, c.Token) + if err != nil { + if errors.Is(err, errProposalNotFound) { + return nil, cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodePropNotFound, + } + } + return nil, fmt.Errorf("proposalRecordLatest: %v", err) + } + if usr.ID.String() == pr.UserID { + // User is the proposal author. Unvetted is allowed. + isAllowed = true + } + } + if !isAllowed { + return nil, cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodeUnauthorized, + ErrorContext: "user is not author or admin", + } + } } - r, err := p.pluginCommand(ctx, comments.ID, - comments.CmdTimestamps, string(b)) + + // Send plugin command + reply, err := p.commentsAll(ctx, comments.GetAll{}) if err != nil { return nil, err } - var tr comments.TimestampsReply - err = json.Unmarshal([]byte(r), &tr) + + // Prepare reply. Comments contain user data that needs to be + // pulled from the user database. + cs := make([]cmv1.Comment, 0, len(reply.Comments)) + for _, cm := range reply.Comments { + // Convert comment + pic := convertCommentFromPlugin(cm) + + // Get comment user data + uuid, err := uuid.Parse(cm.UserID) + if err != nil { + return nil, err + } + u, err := p.db.UserGetById(uuid) + if err != nil { + return nil, err + } + pic.Username = u.Username + + // Add comment + cs = append(cs, pic) + } + + return &cmv1.CommentsReply{ + Comments: cs, + }, nil +} + +func (p *politeiawww) processCommentVotes(ctx context.Context, cv cmv1.CommentVotes) (*cmv1.CommentVotesReply, error) { + log.Tracef("processCommentVotes: %v %v", cv.Token, cv.UserID) + + // Verify state + if cv.State != cmv1.PropStateVetted { + return nil, cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodePropStateInvalid, + ErrorContext: "proposal must be vetted", + } + } + + // Send plugin command + v := comments.Votes{ + UserID: cv.UserID, + } + cvr, err := p.commentVotes(ctx, v) if err != nil { return nil, err } - return &tr, nil + + return &cmv1.CommentVotesReply{ + Votes: convertCommentVoteDetailsFromPlugin(cvr.Votes), + }, nil } func (p *politeiawww) processCommentTimestamps(ctx context.Context, t cmv1.Timestamps, isAdmin bool) (*cmv1.TimestampsReply, error) { @@ -128,8 +342,6 @@ func (p *politeiawww) processCommentTimestamps(ctx context.Context, t cmv1.Times // Get timestamps ct := comments.Timestamps{ - State: convertCommentState(t.State), - Token: t.Token, CommentIDs: t.CommentIDs, IncludeVotes: false, } @@ -157,6 +369,183 @@ func (p *politeiawww) processCommentTimestamps(ctx context.Context, t cmv1.Times }, nil } +func (p *politeiawww) handleCommentNew(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleCommentNew") + + var n cmv1.New + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&n); err != nil { + respondWithPiError(w, r, "handleCommentNew: unmarshal", + cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodeInputInvalid, + }) + return + } + + usr, err := p.getSessionUser(w, r) + if err != nil { + respondWithPiError(w, r, + "handleCommentNew: getSessionUser: %v", err) + return + } + + nr, err := p.processCommentNew(r.Context(), n, *usr) + if err != nil { + respondWithPiError(w, r, + "handleCommentNew: processCommentNew: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, nr) +} + +func (p *politeiawww) handleCommentVote(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleCommentVote") + + var v cmv1.Vote + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&v); err != nil { + respondWithPiError(w, r, "handleCommentVote: unmarshal", + cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodeInputInvalid, + }) + return + } + + usr, err := p.getSessionUser(w, r) + if err != nil { + respondWithPiError(w, r, + "handleCommentVote: getSessionUser: %v", err) + return + } + + vr, err := p.processCommentVote(r.Context(), v, *usr) + if err != nil { + respondWithPiError(w, r, + "handleCommentVote: processCommentVote: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, vr) +} + +func (p *politeiawww) handleCommentDel(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleCommentDel") + + var d cmv1.Del + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&d); err != nil { + respondWithPiError(w, r, "handleCommentDel: unmarshal", + cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodeInputInvalid, + }) + return + } + + usr, err := p.getSessionUser(w, r) + if err != nil { + respondWithPiError(w, r, + "handleCommentDel: getSessionUser: %v", err) + return + } + + dr, err := p.processCommentDel(r.Context(), d, *usr) + if err != nil { + respondWithPiError(w, r, + "handleCommentDel: processCommentDel: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, dr) +} + +func (p *politeiawww) handleCommentsCount(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleCommentsCount") + + var c cmv1.Comments + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&c); err != nil { + respondWithPiError(w, r, "handleCommentsCount: unmarshal", + cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodeInputInvalid, + }) + return + } + + // Lookup session user. This is a public route so a session may not + // exist. Ignore any session not found errors. + usr, err := p.getSessionUser(w, r) + if err != nil && err != errSessionNotFound { + respondWithPiError(w, r, + "handleProposalInventory: getSessionUser: %v", err) + return + } + + cr, err := p.processCommentsCount(r.Context(), c, usr) + if err != nil { + respondWithPiError(w, r, + "handleCommentVote: processCommentsCount: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, cr) +} + +func (p *politeiawww) handleComments(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleComments") + + var c cmv1.Comments + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&c); err != nil { + respondWithPiError(w, r, "handleComments: unmarshal", + cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodeInputInvalid, + }) + return + } + + // Lookup session user. This is a public route so a session may not + // exist. Ignore any session not found errors. + usr, err := p.getSessionUser(w, r) + if err != nil && err != errSessionNotFound { + respondWithPiError(w, r, + "handleProposalInventory: getSessionUser: %v", err) + return + } + + cr, err := p.processComments(r.Context(), c, usr) + if err != nil { + respondWithPiError(w, r, + "handleCommentVote: processComments: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, cr) +} + +func (p *politeiawww) handleCommentVotes(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleCommentVotes") + + var v cmv1.Votes + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&cv); err != nil { + respondWithPiError(w, r, "handleCommentVotes: unmarshal", + cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodeInputInvalid, + }) + return + } + + vr, err := p.processCommentVotes(r.Context(), v) + if err != nil { + respondWithPiError(w, r, + "handleCommentVotes: processCommentVotes: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, vr) +} + func (p *politeiawww) handleCommentTimestamps(w http.ResponseWriter, r *http.Request) { log.Tracef("handleCommentTimestamps") diff --git a/politeiawww/eventmanager.go b/politeiawww/eventmanager.go index 9f1ba858d..7c51f978b 100644 --- a/politeiawww/eventmanager.go +++ b/politeiawww/eventmanager.go @@ -380,11 +380,10 @@ func (p *politeiawww) notifyParentAuthorOnComment(d dataProposalComment, proposa // Lookup the parent comment author to check if they should receive // a reply notification. - parentComment, err := p.commentsGet(context.Background(), comments.Get{ - State: convertCommentsStateFromPi(d.state), - Token: d.token, + g := comments.Get{ CommentIDs: []uint32{d.parentID}, - }) + } + parentComment, err := p.commentsGet(context.Background(), g) if err != nil { return err } diff --git a/politeiawww/pi.go b/politeiawww/pi.go index 9dbffd91d..71e634ee1 100644 --- a/politeiawww/pi.go +++ b/politeiawww/pi.go @@ -6,74 +6,40 @@ package main import ( "context" + "encoding/json" "github.com/decred/politeia/politeiad/plugins/pi" - piplugin "github.com/decred/politeia/politeiad/plugins/pi" ) -// piPassThrough executes the pi plugin PassThrough command. -func (p *politeiawww) piPassThrough(ctx context.Context, pt pi.PassThrough) (*pi.PassThroughReply, error) { - b, err := piplugin.EncodePassThrough(pt) - if err != nil { - return nil, err - } - r, err := p.pluginCommand(ctx, piplugin.ID, - piplugin.CmdPassThrough, string(b)) - if err != nil { - return nil, err - } - ptr, err := piplugin.DecodePassThroughReply(([]byte(r))) - if err != nil { - return nil, err - } - return ptr, nil -} - -// piProposals returns the pi plugin data for the provided proposals. -func (p *politeiawww) piProposals(ctx context.Context, ps piplugin.Proposals) (*piplugin.ProposalsReply, error) { - b, err := piplugin.EncodeProposals(ps) - if err != nil { - return nil, err - } - r, err := p.pluginCommand(ctx, piplugin.ID, - piplugin.CmdProposals, string(b)) - if err != nil { - return nil, err - } - pr, err := piplugin.DecodeProposalsReply([]byte(r)) - if err != nil { - return nil, err - } - return pr, nil -} - // proposalInv returns the pi plugin proposal inventory. -func (p *politeiawww) proposalInv(ctx context.Context, inv piplugin.ProposalInv) (*piplugin.ProposalInvReply, error) { - b, err := piplugin.EncodeProposalInv(inv) +func (p *politeiawww) proposalInv(ctx context.Context, inv pi.ProposalInv) (*pi.ProposalInvReply, error) { + b, err := json.Marshal(inv) if err != nil { return nil, err } - r, err := p.pluginCommand(ctx, piplugin.ID, - piplugin.CmdProposalInv, string(b)) + r, err := p.pluginCommand(ctx, pi.ID, + pi.CmdProposalInv, string(b)) if err != nil { return nil, err } - reply, err := piplugin.DecodeProposalInvReply(([]byte(r))) + var ir pi.ProposalInvReply + err = json.Unmarshal([]byte(r), &ir) if err != nil { return nil, err } - return reply, nil + return &ir, nil } // piVoteInventory returns the pi plugin vote inventory. -func (p *politeiawww) piVoteInventory(ctx context.Context) (*piplugin.VoteInventoryReply, error) { - r, err := p.pluginCommand(ctx, piplugin.ID, piplugin.CmdVoteInventory, "") +func (p *politeiawww) piVoteInventory(ctx context.Context) (*pi.VoteInventoryReply, error) { + r, err := p.pluginCommand(ctx, pi.ID, pi.CmdVoteInventory, "") if err != nil { return nil, err } - vir, err := piplugin.DecodeVoteInventoryReply(([]byte(r))) + var vir pi.VoteInventoryReply + err = json.Unmarshal([]byte(r), &vir) if err != nil { return nil, err } - return vir, nil + return &vir, nil } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 309b5670b..f9333e77b 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -12,10 +12,12 @@ import ( "encoding/json" "errors" "fmt" + "io" "mime" "net/http" "regexp" "strconv" + "strings" "time" pdv1 "github.com/decred/politeia/politeiad/api/v1" @@ -30,7 +32,6 @@ import ( "github.com/decred/politeia/politeiawww/user" wwwutil "github.com/decred/politeia/politeiawww/util" "github.com/decred/politeia/util" - "github.com/google/uuid" ) // TODO www package references should be completely gone from this file @@ -138,7 +139,8 @@ func proposalName(pr piv1.ProposalRecord) string { if err != nil { return "" } - pm, err := pi.DecodeProposalMetadata(b) + var pm pi.ProposalMetadata + err = json.Unmarshal(b, &pm) if err != nil { return "" } @@ -156,14 +158,6 @@ func proposalRecordFillInUser(pr piv1.ProposalRecord, u user.User) piv1.Proposal return pr } -// commentFillInUser populates the provided comment with user data that is not -// store in politeiad and must be populated separately after pulling the -// comment from politeiad. -func commentFillInUser(c piv1.Comment, u user.User) piv1.Comment { - c.Username = u.Username - return c -} - func convertUserErrorFromSignatureError(err error) piv1.UserErrorReply { var e util.SignatureError var s piv1.ErrorStatusT @@ -181,16 +175,6 @@ func convertUserErrorFromSignatureError(err error) piv1.UserErrorReply { } } -func convertPropStateFromPi(s piv1.PropStateT) pi.PropStateT { - switch s { - case piv1.PropStateUnvetted: - return pi.PropStateUnvetted - case piv1.PropStateVetted: - return pi.PropStateVetted - } - return pi.PropStateInvalid -} - func convertRecordStatusFromPropStatus(s piv1.PropStatusT) pdv1.RecordStatusT { switch s { case piv1.PropStatusUnreviewed: @@ -285,6 +269,23 @@ func convertFilesFromPD(f []pdv1.File) ([]piv1.File, []piv1.Metadata) { return files, metadata } +func statusChangesDecode(payload []byte) ([]pi.StatusChange, error) { + var statuses []pi.StatusChange + d := json.NewDecoder(strings.NewReader(string(payload))) + for { + var sc pi.StatusChange + err := d.Decode(&sc) + if errors.Is(err, io.EOF) { + break + } else if err != nil { + return nil, err + } + statuses = append(statuses, sc) + } + + return statuses, nil +} + func convertProposalRecordFromPD(r pdv1.Record, state piv1.PropStateT) (*piv1.ProposalRecord, error) { // Decode metadata streams var ( @@ -295,12 +296,13 @@ func convertProposalRecordFromPD(r pdv1.Record, state piv1.PropStateT) (*piv1.Pr for _, v := range r.Metadata { switch v.ID { case pi.MDStreamIDGeneralMetadata: - gm, err = pi.DecodeGeneralMetadata([]byte(v.Payload)) + var gm pi.GeneralMetadata + err = json.Unmarshal([]byte(v.Payload), &gm) if err != nil { return nil, err } case pi.MDStreamIDStatusChanges: - sc, err = pi.DecodeStatusChanges([]byte(v.Payload)) + sc, err = statusChangesDecode([]byte(v.Payload)) if err != nil { return nil, err } @@ -337,45 +339,13 @@ func convertProposalRecordFromPD(r pdv1.Record, state piv1.PropStateT) (*piv1.Pr Username: "", // Intentionally omitted PublicKey: gm.PublicKey, Signature: gm.Signature, - Comments: 0, // Intentionally omitted Statuses: statuses, Files: files, Metadata: metadata, - LinkedFrom: []string{}, // Intentionally omitted CensorshipRecord: convertCensorshipRecordFromPD(r.CensorshipRecord), }, nil } -func convertCommentsStateFromPi(s piv1.PropStateT) comments.StateT { - switch s { - case piv1.PropStateUnvetted: - return comments.StateUnvetted - case piv1.PropStateVetted: - return comments.StateVetted - } - return comments.StateInvalid -} - -func convertPropStateFromComments(s comments.StateT) piv1.PropStateT { - switch s { - case comments.StateUnvetted: - return piv1.PropStateUnvetted - case comments.StateVetted: - return piv1.PropStateVetted - } - return piv1.PropStateInvalid -} - -func convertCommentStateFromPi(s piv1.PropStateT) comments.StateT { - switch s { - case piv1.PropStateUnvetted: - return comments.StateUnvetted - case piv1.PropStateVetted: - return comments.StateVetted - } - return comments.StateInvalid -} - func convertCommentVoteFromPi(cv piv1.CommentVoteT) comments.VoteT { switch cv { case piv1.CommentVoteDownvote: @@ -390,7 +360,6 @@ func convertCommentFromPlugin(c comments.Comment) piv1.Comment { return piv1.Comment{ UserID: c.UserID, Username: "", // Intentionally omitted, needs to be pulled from userdb - State: convertPropStateFromComments(c.State), Token: c.Token, ParentID: c.ParentID, Comment: c.Comment, @@ -421,7 +390,6 @@ func convertCommentVoteDetailsFromPlugin(cv []comments.CommentVote) []piv1.Comme for _, v := range cv { c = append(c, piv1.CommentVoteDetails{ UserID: v.UserID, - State: convertPropStateFromComments(v.State), Token: v.Token, CommentID: v.CommentID, Vote: convertCommentVoteFromPlugin(v.Vote), @@ -647,22 +615,6 @@ func convertCastVoteDetailsFromPlugin(votes []ticketvote.CastVoteDetails) []piv1 return vs } -func convertProposalVotesFromPlugin(votes map[string]ticketvote.RecordVote) map[string]piv1.ProposalVote { - pv := make(map[string]piv1.ProposalVote, len(votes)) - for k, v := range votes { - var vdp *piv1.VoteDetails - if v.Vote != nil { - vd := convertVoteDetailsFromPlugin(*v.Vote) - vdp = &vd - } - pv[k] = piv1.ProposalVote{ - Auths: convertAuthDetailsFromPlugin(v.Auths), - Vote: vdp, - } - } - return pv -} - func convertVoteStatusFromPlugin(s ticketvote.VoteStatusT) piv1.VoteStatusT { switch s { case ticketvote.VoteStatusInvalid: @@ -680,7 +632,7 @@ func convertVoteStatusFromPlugin(s ticketvote.VoteStatusT) piv1.VoteStatusT { } } -func convertVoteSummaryFromPlugin(s ticketvote.Summary) piv1.VoteSummary { +func convertVoteSummaryFromPlugin(s ticketvote.VoteSummary) piv1.VoteSummary { results := make([]piv1.VoteResult, 0, len(s.Results)) for _, v := range s.Results { results = append(results, piv1.VoteResult{ @@ -705,42 +657,6 @@ func convertVoteSummaryFromPlugin(s ticketvote.Summary) piv1.VoteSummary { } } -func convertVoteSummariesFromPlugin(ts map[string]ticketvote.Summary) map[string]piv1.VoteSummary { - s := make(map[string]piv1.VoteSummary, len(ts)) - for k, v := range ts { - s[k] = convertVoteSummaryFromPlugin(v) - } - return s -} - -// linkByPeriodMin returns the minimum amount of time, in seconds, that the -// LinkBy period must be set to. This is determined by adding 1 week onto the -// minimum voting period so that RFP proposal submissions have at least one -// week to be submitted after the proposal vote ends. -func (p *politeiawww) linkByPeriodMin() int64 { - var ( - submissionPeriod int64 = 604800 // One week in seconds - blockTime int64 // In seconds - ) - switch { - case p.cfg.TestNet: - blockTime = int64(testNet3Params.TargetTimePerBlock.Seconds()) - case p.cfg.SimNet: - blockTime = int64(simNetParams.TargetTimePerBlock.Seconds()) - default: - blockTime = int64(mainNetParams.TargetTimePerBlock.Seconds()) - } - return (int64(p.cfg.VoteDurationMin) * blockTime) + submissionPeriod -} - -// linkByPeriodMax returns the maximum amount of time, in seconds, that the -// LinkBy period can be set to. 3 months is currently hard coded with no real -// reason for deciding on 3 months besides that it sounds like a sufficient -// amount of time. This can be changed if there is a valid reason to. -func (p *politeiawww) linkByPeriodMax() int64 { - return 7776000 // 3 months in seconds -} - // proposalRecords returns the ProposalRecord for each of the provided proposal // requests. If a token does not correspond to an actual proposal then it will // not be included in the returned map. @@ -791,29 +707,6 @@ func (p *politeiawww) proposalRecords(ctx context.Context, state piv1.PropStateT return map[string]piv1.ProposalRecord{}, nil } - // Get proposal plugin data - tokens := make([]string, 0, len(props)) - for _, v := range props { - tokens = append(tokens, v.CensorshipRecord.Token) - } - ps := pi.Proposals{ - State: convertPropStateFromPi(state), - Tokens: tokens, - } - psr, err := p.piProposals(ctx, ps) - if err != nil { - return nil, fmt.Errorf("proposalPluginData: %v", err) - } - for k, v := range props { - token := v.CensorshipRecord.Token - d, ok := psr.Proposals[token] - if !ok { - return nil, fmt.Errorf("proposal plugin data not found %v", token) - } - props[k].Comments = d.Comments - props[k].LinkedFrom = d.LinkedFrom - } - // Get user data pubkeys := make([]string, 0, len(props)) for _, v := range props { @@ -870,12 +763,20 @@ func (p *politeiawww) proposalRecordLatest(ctx context.Context, state piv1.PropS return p.proposalRecord(ctx, state, token, "") } +func (p *politeiawww) linkByPeriodMin() int64 { + return 0 +} + +func (p *politeiawww) linkByPeriodMax() int64 { + return 0 +} + func (p *politeiawww) verifyProposalMetadata(pm piv1.ProposalMetadata) error { // Verify name if !proposalNameIsValid(pm.Name) { return piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusPropNameInvalid, - ErrorContext: []string{proposalNameRegex()}, + ErrorContext: proposalNameRegex(), } } @@ -884,7 +785,7 @@ func (p *politeiawww) verifyProposalMetadata(pm piv1.ProposalMetadata) error { if !tokenIsFullLength(pm.LinkTo) { return piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusPropLinkToInvalid, - ErrorContext: []string{"invalid token"}, + ErrorContext: "invalid token", } } } @@ -899,14 +800,14 @@ func (p *politeiawww) verifyProposalMetadata(pm piv1.ProposalMetadata) error { pm.LinkBy, min) return piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusPropLinkByInvalid, - ErrorContext: []string{e}, + ErrorContext: e, } case pm.LinkBy > max: e := fmt.Sprintf("linkby %v is more than max allowed of %v", pm.LinkBy, max) return piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusPropLinkByInvalid, - ErrorContext: []string{e}, + ErrorContext: e, } } } @@ -918,7 +819,7 @@ func (p *politeiawww) verifyProposal(files []piv1.File, metadata []piv1.Metadata if len(files) == 0 { return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusFileCountInvalid, - ErrorContext: []string{"no files found"}, + ErrorContext: "no files found", } } @@ -936,7 +837,7 @@ func (p *politeiawww) verifyProposal(files []piv1.File, metadata []piv1.Metadata e := fmt.Sprintf("duplicate name %v", v.Name) return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusFileNameInvalid, - ErrorContext: []string{e}, + ErrorContext: e, } } filenames[v.Name] = struct{}{} @@ -946,7 +847,7 @@ func (p *politeiawww) verifyProposal(files []piv1.File, metadata []piv1.Metadata e := fmt.Sprintf("file %v empty payload", v.Name) return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusFilePayloadInvalid, - ErrorContext: []string{e}, + ErrorContext: e, } } payloadb, err := base64.StdEncoding.DecodeString(v.Payload) @@ -954,7 +855,7 @@ func (p *politeiawww) verifyProposal(files []piv1.File, metadata []piv1.Metadata e := fmt.Sprintf("file %v invalid base64", v.Name) return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusFilePayloadInvalid, - ErrorContext: []string{e}, + ErrorContext: e, } } @@ -964,7 +865,7 @@ func (p *politeiawww) verifyProposal(files []piv1.File, metadata []piv1.Metadata if !ok { return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusFileDigestInvalid, - ErrorContext: []string{v.Name}, + ErrorContext: v.Name, } } if !bytes.Equal(digest, d[:]) { @@ -972,7 +873,7 @@ func (p *politeiawww) verifyProposal(files []piv1.File, metadata []piv1.Metadata v.Name, v.Digest, digest) return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusFileDigestInvalid, - ErrorContext: []string{e}, + ErrorContext: e, } } @@ -987,7 +888,7 @@ func (p *politeiawww) verifyProposal(files []piv1.File, metadata []piv1.Metadata e := fmt.Sprintf("file %v mime '%v' not parsable", v.Name, v.MIME) return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusFileMIMEInvalid, - ErrorContext: []string{e}, + ErrorContext: e, } } if mimeFile != mimePayload { @@ -995,7 +896,7 @@ func (p *politeiawww) verifyProposal(files []piv1.File, metadata []piv1.Metadata v.Name, mimeFile, mimePayload) return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusFileMIMEInvalid, - ErrorContext: []string{e}, + ErrorContext: e, } } @@ -1010,7 +911,7 @@ func (p *politeiawww) verifyProposal(files []piv1.File, metadata []piv1.Metadata v.Name, len(payloadb), www.PolicyMaxMDSize) return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusIndexFileSizeInvalid, - ErrorContext: []string{e}, + ErrorContext: e, } } @@ -1020,7 +921,7 @@ func (p *politeiawww) verifyProposal(files []piv1.File, metadata []piv1.Metadata e := fmt.Sprintf("want %v, got %v", www.PolicyIndexFilename, v.Name) return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusIndexFileNameInvalid, - ErrorContext: []string{e}, + ErrorContext: e, } } if foundIndexFile { @@ -1028,7 +929,7 @@ func (p *politeiawww) verifyProposal(files []piv1.File, metadata []piv1.Metadata www.PolicyIndexFilename) return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusIndexFileCountInvalid, - ErrorContext: []string{e}, + ErrorContext: e, } } @@ -1044,14 +945,14 @@ func (p *politeiawww) verifyProposal(files []piv1.File, metadata []piv1.Metadata v.Name, len(payloadb), www.PolicyMaxImageSize) return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusImageFileSizeInvalid, - ErrorContext: []string{e}, + ErrorContext: e, } } default: return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusFileMIMEInvalid, - ErrorContext: []string{v.MIME}, + ErrorContext: v.MIME, } } } @@ -1061,7 +962,7 @@ func (p *politeiawww) verifyProposal(files []piv1.File, metadata []piv1.Metadata e := fmt.Sprintf("%v file not found", www.PolicyIndexFilename) return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusIndexFileCountInvalid, - ErrorContext: []string{e}, + ErrorContext: e, } } @@ -1071,7 +972,7 @@ func (p *politeiawww) verifyProposal(files []piv1.File, metadata []piv1.Metadata countTextFiles, www.PolicyMaxMDs) return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusTextFileCountInvalid, - ErrorContext: []string{e}, + ErrorContext: e, } } if countImageFiles > www.PolicyMaxImages { @@ -1079,7 +980,7 @@ func (p *politeiawww) verifyProposal(files []piv1.File, metadata []piv1.Metadata countImageFiles, www.PolicyMaxImages) return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusImageFileCountInvalid, - ErrorContext: []string{e}, + ErrorContext: e, } } @@ -1095,7 +996,7 @@ func (p *politeiawww) verifyProposal(files []piv1.File, metadata []piv1.Metadata www.HintProposalMetadata) return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusMetadataCountInvalid, - ErrorContext: []string{e}, + ErrorContext: e, } } md := metadata[0] @@ -1111,7 +1012,7 @@ func (p *politeiawww) verifyProposal(files []piv1.File, metadata []piv1.Metadata e := fmt.Sprintf("metadata with hint %v invalid base64 payload", md.Hint) return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusMetadataPayloadInvalid, - ErrorContext: []string{e}, + ErrorContext: e, } } digest := util.Digest(b) @@ -1120,7 +1021,7 @@ func (p *politeiawww) verifyProposal(files []piv1.File, metadata []piv1.Metadata md.Hint, md.Digest, hex.EncodeToString(digest)) return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusMetadataDigestInvalid, - ErrorContext: []string{e}, + ErrorContext: e, } } @@ -1133,7 +1034,7 @@ func (p *politeiawww) verifyProposal(files []piv1.File, metadata []piv1.Metadata e := fmt.Sprintf("unable to decode %v payload", md.Hint) return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusMetadataPayloadInvalid, - ErrorContext: []string{e}, + ErrorContext: e, } } @@ -1177,7 +1078,7 @@ func (p *politeiawww) processProposalNew(ctx context.Context, pn piv1.ProposalNe if usr.PublicKey() != pn.PublicKey { return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusPublicKeyInvalid, - ErrorContext: []string{"not active identity"}, + ErrorContext: "not active identity", } } @@ -1207,7 +1108,7 @@ func (p *politeiawww) processProposalNew(ctx context.Context, pn piv1.ProposalNe Signature: pn.Signature, Timestamp: timestamp, } - b, err := pi.EncodeGeneralMetadata(gm) + b, err := json.Marshal(gm) if err != nil { return nil, err } @@ -1319,7 +1220,7 @@ func (p *politeiawww) processProposalEdit(ctx context.Context, pe piv1.ProposalE if usr.PublicKey() != pe.PublicKey { return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusPublicKeyInvalid, - ErrorContext: []string{"not active identity"}, + ErrorContext: "not active identity", } } @@ -1339,7 +1240,7 @@ func (p *politeiawww) processProposalEdit(ctx context.Context, pe piv1.ProposalE if curr.UserID != usr.ID.String() { return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusUnauthorized, - ErrorContext: []string{"user is not author"}, + ErrorContext: "user is not author", } } @@ -1379,7 +1280,7 @@ func (p *politeiawww) processProposalEdit(ctx context.Context, pe piv1.ProposalE Signature: pe.Signature, Timestamp: timestamp, } - b, err := pi.EncodeGeneralMetadata(gm) + b, err := json.Marshal(gm) if err != nil { return nil, err } @@ -1467,7 +1368,7 @@ func (p *politeiawww) processProposalSetStatus(ctx context.Context, pss piv1.Pro if required && pss.Reason == "" { return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusPropStatusChangeReasonInvalid, - ErrorContext: []string{"reason not given"}, + ErrorContext: "reason not given", } } @@ -1475,7 +1376,7 @@ func (p *politeiawww) processProposalSetStatus(ctx context.Context, pss piv1.Pro if !usr.Admin { return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusUnauthorized, - ErrorContext: []string{"user is not an admin"}, + ErrorContext: "user is not an admin", } } @@ -1483,7 +1384,7 @@ func (p *politeiawww) processProposalSetStatus(ctx context.Context, pss piv1.Pro if usr.PublicKey() != pss.PublicKey { return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusPublicKeyInvalid, - ErrorContext: []string{"not active identity"}, + ErrorContext: "not active identity", } } @@ -1513,7 +1414,7 @@ func (p *politeiawww) processProposalSetStatus(ctx context.Context, pss piv1.Pro Signature: pss.Signature, Timestamp: timestamp, } - b, err := pi.EncodeStatusChange(sc) + b, err := json.Marshal(sc) if err != nil { return nil, err } @@ -1640,308 +1541,6 @@ func (p *politeiawww) processProposalInventory(ctx context.Context, inv piv1.Pro }, nil } -func (p *politeiawww) processCommentNew(ctx context.Context, cn piv1.CommentNew, usr user.User) (*piv1.CommentNewReply, error) { - log.Tracef("processCommentNew: %v", usr.Username) - - // Verify user has paid registration paywall - if !p.userHasPaid(usr) { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusUserRegistrationNotPaid, - } - } - - // Verify user signed using active identity - if usr.PublicKey() != cn.PublicKey { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusPublicKeyInvalid, - ErrorContext: []string{"not active identity"}, - } - } - - // Only admins and the proposal author are allowed to comment on - // unvetted proposals. - if cn.State == piv1.PropStateUnvetted && !usr.Admin { - // Fetch the proposal so we can see who the author is - pr, err := p.proposalRecordLatest(ctx, cn.State, cn.Token) - if err != nil { - if errors.Is(err, errProposalNotFound) { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusPropNotFound, - } - } - return nil, fmt.Errorf("proposalRecordLatest: %v", err) - } - if usr.ID.String() != pr.UserID { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusUnauthorized, - ErrorContext: []string{"user is not author or admin"}, - } - } - } - - // Send plugin command - n := comments.New{ - UserID: usr.ID.String(), - State: convertCommentStateFromPi(cn.State), - Token: cn.Token, - ParentID: cn.ParentID, - Comment: cn.Comment, - PublicKey: cn.PublicKey, - Signature: cn.Signature, - } - b, err := comments.EncodeNew(n) - if err != nil { - return nil, err - } - pt := pi.PassThrough{ - PluginID: comments.ID, - PluginCmd: comments.CmdNew, - Payload: string(b), - } - ptr, err := p.piPassThrough(ctx, pt) - if err != nil { - return nil, err - } - nr, err := comments.DecodeNewReply([]byte(ptr.Payload)) - if err != nil { - return nil, err - } - - // Prepare reply - c := convertCommentFromPlugin(nr.Comment) - c = commentFillInUser(c, usr) - - // Emit event - p.eventManager.emit(eventProposalComment, - dataProposalComment{ - state: c.State, - token: c.Token, - commentID: c.CommentID, - parentID: c.ParentID, - username: c.Username, - }) - - return &piv1.CommentNewReply{ - Comment: c, - }, nil -} - -func (p *politeiawww) processCommentVote(ctx context.Context, cv piv1.CommentVote, usr user.User) (*piv1.CommentVoteReply, error) { - log.Tracef("processCommentVote: %v %v %v", cv.Token, cv.CommentID, cv.Vote) - - // Verify state - if cv.State != piv1.PropStateVetted { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusPropStateInvalid, - ErrorContext: []string{"proposal must be vetted"}, - } - } - - // Verify user has paid registration paywall - if !p.userHasPaid(usr) { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusUserRegistrationNotPaid, - } - } - - // Verify user signed using active identity - if usr.PublicKey() != cv.PublicKey { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusPublicKeyInvalid, - ErrorContext: []string{"not active identity"}, - } - } - - // Send plugin command - v := comments.Vote{ - UserID: usr.ID.String(), - State: convertCommentStateFromPi(cv.State), - Token: cv.Token, - CommentID: cv.CommentID, - Vote: convertCommentVoteFromPi(cv.Vote), - PublicKey: cv.PublicKey, - Signature: cv.Signature, - } - b, err := comments.EncodeVote(v) - if err != nil { - return nil, err - } - pt := pi.PassThrough{ - PluginID: comments.ID, - PluginCmd: comments.CmdVote, - Payload: string(b), - } - ptr, err := p.piPassThrough(ctx, pt) - if err != nil { - return nil, err - } - vr, err := comments.DecodeVoteReply([]byte(ptr.Payload)) - if err != nil { - return nil, err - } - - return &piv1.CommentVoteReply{ - Downvotes: vr.Downvotes, - Upvotes: vr.Upvotes, - Timestamp: vr.Timestamp, - Receipt: vr.Receipt, - }, nil -} - -func (p *politeiawww) processCommentCensor(ctx context.Context, cc piv1.CommentCensor, usr user.User) (*piv1.CommentCensorReply, error) { - log.Tracef("processCommentCensor: %v %v", cc.Token, cc.CommentID) - - // Sanity check - if !usr.Admin { - return nil, fmt.Errorf("not an admin") - } - - // Verify user signed with their active identity - if usr.PublicKey() != cc.PublicKey { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusPublicKeyInvalid, - ErrorContext: []string{"not active identity"}, - } - } - - // Send plugin command - d := comments.Del{ - State: convertCommentStateFromPi(cc.State), - Token: cc.Token, - CommentID: cc.CommentID, - Reason: cc.Reason, - PublicKey: cc.PublicKey, - Signature: cc.Signature, - } - b, err := comments.EncodeDel(d) - if err != nil { - return nil, err - } - pt := pi.PassThrough{ - PluginID: comments.ID, - PluginCmd: comments.CmdDel, - Payload: string(b), - } - ptr, err := p.piPassThrough(ctx, pt) - if err != nil { - return nil, err - } - dr, err := comments.DecodeDelReply([]byte(ptr.Payload)) - if err != nil { - return nil, err - } - - // Prepare reply - c := convertCommentFromPlugin(dr.Comment) - c = commentFillInUser(c, usr) - - return &piv1.CommentCensorReply{ - Comment: c, - }, nil -} - -func (p *politeiawww) processComments(ctx context.Context, c piv1.Comments, usr *user.User) (*piv1.CommentsReply, error) { - log.Tracef("processComments: %v", c.Token) - - // Only admins and the proposal author are allowed to retrieve - // unvetted comments. This is a public route so a user might not - // exist. - if c.State == piv1.PropStateUnvetted { - var isAllowed bool - switch { - case usr == nil: - // No logged in user. Unvetted not allowed. - case usr.Admin: - // User is an admin. Unvetted is allowed. - isAllowed = true - default: - // Logged in user is not an admin. Check if they are the - // proposal author. - pr, err := p.proposalRecordLatest(ctx, c.State, c.Token) - if err != nil { - if errors.Is(err, errProposalNotFound) { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusPropNotFound, - } - } - return nil, fmt.Errorf("proposalRecordLatest: %v", err) - } - if usr.ID.String() == pr.UserID { - // User is the proposal author. Unvetted is allowed. - isAllowed = true - } - } - if !isAllowed { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusUnauthorized, - ErrorContext: []string{"user is not author or admin"}, - } - } - } - - // Send plugin command - reply, err := p.commentsAll(ctx, comments.GetAll{ - State: convertCommentsStateFromPi(c.State), - Token: c.Token, - }) - if err != nil { - return nil, err - } - - // Prepare reply. Comments contain user data that needs to be - // pulled from the user database. - cs := make([]piv1.Comment, 0, len(reply.Comments)) - for _, cm := range reply.Comments { - // Convert comment - pic := convertCommentFromPlugin(cm) - - // Get comment user data - uuid, err := uuid.Parse(cm.UserID) - if err != nil { - return nil, err - } - u, err := p.db.UserGetById(uuid) - if err != nil { - return nil, err - } - pic.Username = u.Username - - // Add comment - cs = append(cs, pic) - } - - return &piv1.CommentsReply{ - Comments: cs, - }, nil -} - -func (p *politeiawww) processCommentVotes(ctx context.Context, cv piv1.CommentVotes) (*piv1.CommentVotesReply, error) { - log.Tracef("processCommentVotes: %v %v", cv.Token, cv.UserID) - - // Verify state - if cv.State != piv1.PropStateVetted { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusPropStateInvalid, - ErrorContext: []string{"proposal must be vetted"}, - } - } - - // Send plugin command - v := comments.Votes{ - State: convertCommentsStateFromPi(cv.State), - Token: cv.Token, - UserID: cv.UserID, - } - cvr, err := p.commentVotes(ctx, v) - if err != nil { - return nil, err - } - - return &piv1.CommentVotesReply{ - Votes: convertCommentVoteDetailsFromPlugin(cvr.Votes), - }, nil -} - func (p *politeiawww) processVoteAuthorize(ctx context.Context, va piv1.VoteAuthorize, usr user.User) (*piv1.VoteAuthorizeReply, error) { log.Tracef("processVoteAuthorize: %v", va.Token) @@ -1949,7 +1548,7 @@ func (p *politeiawww) processVoteAuthorize(ctx context.Context, va piv1.VoteAuth if usr.PublicKey() != va.PublicKey { return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusPublicKeyInvalid, - ErrorContext: []string{"not active identity"}, + ErrorContext: "not active identity", } } @@ -1982,7 +1581,7 @@ func (p *politeiawww) processVoteStart(ctx context.Context, vs piv1.VoteStart, u if len(vs.Starts) == 0 { return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusStartDetailsInvalid, - ErrorContext: []string{"no start details found"}, + ErrorContext: "no start details found", } } @@ -1991,7 +1590,7 @@ func (p *politeiawww) processVoteStart(ctx context.Context, vs piv1.VoteStart, u if usr.PublicKey() != v.PublicKey { return nil, piv1.UserErrorReply{ ErrorCode: piv1.ErrorStatusPublicKeyInvalid, - ErrorContext: []string{"not active identity"}, + ErrorContext: "not active identity", } } } @@ -2016,23 +1615,7 @@ func (p *politeiawww) processVoteStart(ctx context.Context, vs piv1.VoteStart, u // through the pi plugin so that it can perform this additional // validation. s := convertVoteStartFromPi(vs) - payload, err := ticketvote.EncodeStart(s) - if err != nil { - return nil, err - } - pt := pi.PassThrough{ - PluginID: ticketvote.ID, - PluginCmd: ticketvote.CmdStart, - Payload: string(payload), - } - ptr, err := p.piPassThrough(ctx, pt) - if err != nil { - return nil, err - } - sr, err = ticketvote.DecodeStartReply([]byte(ptr.Payload)) - if err != nil { - return nil, err - } + _ = s default: return nil, piv1.UserErrorReply{ @@ -2069,14 +1652,9 @@ func (p *politeiawww) processCastBallot(ctx context.Context, vc piv1.CastBallot) func (p *politeiawww) processVotes(ctx context.Context, v piv1.Votes) (*piv1.VotesReply, error) { log.Tracef("processVotes: %v", v.Tokens) - vd, err := p.voteDetails(ctx, v.Tokens) - if err != nil { - return nil, err - } + // TODO - return &piv1.VotesReply{ - Votes: convertProposalVotesFromPlugin(vd.Votes), - }, nil + return nil, nil } func (p *politeiawww) processVoteResults(ctx context.Context, vr piv1.VoteResults) (*piv1.VoteResultsReply, error) { @@ -2095,15 +1673,19 @@ func (p *politeiawww) processVoteResults(ctx context.Context, vr piv1.VoteResult func (p *politeiawww) processVoteSummaries(ctx context.Context, vs piv1.VoteSummaries) (*piv1.VoteSummariesReply, error) { log.Tracef("processVoteSummaries: %v", vs.Tokens) - r, err := p.voteSummaries(ctx, vs.Tokens) - if err != nil { - return nil, err - } + /* + r, err := p.voteSummaries(ctx, vs.Tokens) + if err != nil { + return nil, err + } - return &piv1.VoteSummariesReply{ - Summaries: convertVoteSummariesFromPlugin(r.Summaries), - BestBlock: r.BestBlock, - }, nil + return &piv1.VoteSummariesReply{ + Summaries: convertVoteSummariesFromPlugin(r.Summaries), + BestBlock: r.BestBlock, + }, nil + */ + + return nil, nil } func (p *politeiawww) processVoteInventory(ctx context.Context) (*piv1.VoteInventoryReply, error) { @@ -2279,151 +1861,6 @@ func (p *politeiawww) handleProposalInventory(w http.ResponseWriter, r *http.Req util.RespondWithJSON(w, http.StatusOK, pir) } -func (p *politeiawww) handleCommentNew(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleCommentNew") - - var cn piv1.CommentNew - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&cn); err != nil { - respondWithPiError(w, r, "handleCommentNew: unmarshal", - piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusInputInvalid, - }) - return - } - - usr, err := p.getSessionUser(w, r) - if err != nil { - respondWithPiError(w, r, - "handleCommentNew: getSessionUser: %v", err) - return - } - - cnr, err := p.processCommentNew(r.Context(), cn, *usr) - if err != nil { - respondWithPiError(w, r, - "handleCommentNew: processCommentNew: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, cnr) -} - -func (p *politeiawww) handleCommentVote(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleCommentVote") - - var cv piv1.CommentVote - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&cv); err != nil { - respondWithPiError(w, r, "handleCommentVote: unmarshal", - piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusInputInvalid, - }) - return - } - - usr, err := p.getSessionUser(w, r) - if err != nil { - respondWithPiError(w, r, - "handleCommentVote: getSessionUser: %v", err) - return - } - - vcr, err := p.processCommentVote(r.Context(), cv, *usr) - if err != nil { - respondWithPiError(w, r, - "handleCommentVote: processCommentVote: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, vcr) -} - -func (p *politeiawww) handleCommentCensor(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleCommentCensor") - - var cc piv1.CommentCensor - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&cc); err != nil { - respondWithPiError(w, r, "handleCommentCensor: unmarshal", - piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusInputInvalid, - }) - return - } - - usr, err := p.getSessionUser(w, r) - if err != nil { - respondWithPiError(w, r, - "handleCommentCensor: getSessionUser: %v", err) - return - } - - ccr, err := p.processCommentCensor(r.Context(), cc, *usr) - if err != nil { - respondWithPiError(w, r, - "handleCommentCensor: processCommentCensor: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, ccr) -} - -func (p *politeiawww) handleComments(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleComments") - - var c piv1.Comments - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&c); err != nil { - respondWithPiError(w, r, "handleComments: unmarshal", - piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusInputInvalid, - }) - return - } - - // Lookup session user. This is a public route so a session may not - // exist. Ignore any session not found errors. - usr, err := p.getSessionUser(w, r) - if err != nil && err != errSessionNotFound { - respondWithPiError(w, r, - "handleProposalInventory: getSessionUser: %v", err) - return - } - - cr, err := p.processComments(r.Context(), c, usr) - if err != nil { - respondWithPiError(w, r, - "handleCommentVote: processComments: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, cr) -} - -func (p *politeiawww) handleCommentVotes(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleCommentVotes") - - var cv piv1.CommentVotes - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&cv); err != nil { - respondWithPiError(w, r, "handleCommentVotes: unmarshal", - piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusInputInvalid, - }) - return - } - - cvr, err := p.processCommentVotes(r.Context(), cv) - if err != nil { - respondWithPiError(w, r, - "handleCommentVotes: processCommentVotes: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, cvr) -} - func (p *politeiawww) handleVoteAuthorize(w http.ResponseWriter, r *http.Request) { log.Tracef("handleVoteAuthorize") diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index cb06f9032..8cf8322f8 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -7,6 +7,7 @@ package main import ( "context" "encoding/base64" + "encoding/json" "strconv" "github.com/decred/politeia/decredplugin" @@ -55,7 +56,8 @@ func convertProposalToWWW(pr *pi.ProposalRecord) (*www.ProposalRecord, error) { if err != nil { return nil, err } - pm, err = piplugin.DecodeProposalMetadata(b) + var pm pi.ProposalMetadata + err = json.Unmarshal(b, &pm) if err != nil { return nil, err } @@ -112,15 +114,11 @@ func convertProposalToWWW(pr *pi.ProposalRecord) (*www.ProposalRecord, error) { Username: pr.Username, PublicKey: pr.PublicKey, Signature: pr.Signature, - NumComments: uint(pr.Comments), Version: pr.Version, StatusChangeMessage: changeMsg, PublishedAt: publishedAt, CensoredAt: censoredAt, AbandonedAt: abandonedAt, - LinkTo: pm.LinkTo, - LinkBy: pm.LinkBy, - LinkedFrom: pr.LinkedFrom, Files: files, Metadata: metadata, CensorshipRecord: www.CensorshipRecord{ @@ -235,22 +233,16 @@ func (p *politeiawww) processVoteResultsWWW(ctx context.Context, token string) ( log.Tracef("processVoteResultsWWW: %v", token) // Get vote details - vd, err := p.voteDetails(ctx, []string{token}) + vd, err := p.voteDetails(ctx, token) if err != nil { return nil, err } - vote, ok := vd.Votes[token] - if !ok { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - } - } // Convert to www - startHeight := strconv.FormatUint(uint64(vote.Vote.StartBlockHeight), 10) - endHeight := strconv.FormatUint(uint64(vote.Vote.EndBlockHeight), 10) - options := make([]www.VoteOption, 0, len(vote.Vote.Params.Options)) - for _, o := range vote.Vote.Params.Options { + startHeight := strconv.FormatUint(uint64(vd.Vote.StartBlockHeight), 10) + endHeight := strconv.FormatUint(uint64(vd.Vote.EndBlockHeight), 10) + options := make([]www.VoteOption, 0, len(vd.Vote.Params.Options)) + for _, o := range vd.Vote.Params.Options { options = append(options, www.VoteOption{ Id: o.ID, Description: o.Description, @@ -277,22 +269,22 @@ func (p *politeiawww) processVoteResultsWWW(ctx context.Context, token string) ( return &www.VoteResultsReply{ StartVote: www.StartVote{ - PublicKey: vote.Vote.PublicKey, - Signature: vote.Vote.Signature, + PublicKey: vd.Vote.PublicKey, + Signature: vd.Vote.Signature, Vote: www.Vote{ - Token: vote.Vote.Params.Token, - Mask: vote.Vote.Params.Mask, - Duration: vote.Vote.Params.Duration, - QuorumPercentage: vote.Vote.Params.QuorumPercentage, - PassPercentage: vote.Vote.Params.PassPercentage, + Token: vd.Vote.Params.Token, + Mask: vd.Vote.Params.Mask, + Duration: vd.Vote.Params.Duration, + QuorumPercentage: vd.Vote.Params.QuorumPercentage, + PassPercentage: vd.Vote.Params.PassPercentage, Options: options, }, }, StartVoteReply: www.StartVoteReply{ StartBlockHeight: startHeight, - StartBlockHash: vote.Vote.StartBlockHash, + StartBlockHash: vd.Vote.StartBlockHash, EndHeight: endHeight, - EligibleTickets: vote.Vote.EligibleTickets, + EligibleTickets: vd.Vote.EligibleTickets, }, CastVotes: votes, }, nil @@ -302,14 +294,14 @@ func (p *politeiawww) processBatchVoteSummary(ctx context.Context, bvs www.Batch log.Tracef("processBatchVoteSummary: %v", bvs.Tokens) // Get vote summaries - sm, err := p.voteSummaries(ctx, bvs.Tokens) + vs, err := p.voteSummaries(ctx, bvs.Tokens) if err != nil { return nil, err } // Prepare reply - summaries := make(map[string]www.VoteSummary, len(sm.Summaries)) - for k, v := range sm.Summaries { + summaries := make(map[string]www.VoteSummary, len(vs)) + for k, v := range vs { results := make([]www.VoteOptionResult, 0, len(v.Results)) for _, r := range v.Results { results = append(results, www.VoteOptionResult{ @@ -336,7 +328,8 @@ func (p *politeiawww) processBatchVoteSummary(ctx context.Context, bvs www.Batch return &www.BatchVoteSummaryReply{ Summaries: summaries, - BestBlock: uint64(sm.BestBlock), + // TODO + // BestBlock: uint64(sm.BestBlock), }, nil } diff --git a/politeiawww/ticketvote.go b/politeiawww/ticketvote.go index b2d418c1f..266a00c62 100644 --- a/politeiawww/ticketvote.go +++ b/politeiawww/ticketvote.go @@ -44,128 +44,34 @@ func convertTimestampFromTicketVotePlugin(t ticketvote.Timestamp) tkv1.Timestamp } } -// voteAuthorize uses the ticketvote plugin to authorize a vote. func (p *politeiawww) voteAuthorize(ctx context.Context, a ticketvote.Authorize) (*ticketvote.AuthorizeReply, error) { - b, err := ticketvote.EncodeAuthorize(a) - if err != nil { - return nil, err - } - r, err := p.pluginCommand(ctx, ticketvote.ID, - ticketvote.CmdAuthorize, string(b)) - if err != nil { - return nil, err - } - va, err := ticketvote.DecodeAuthorizeReply([]byte(r)) - if err != nil { - return nil, err - } - return va, nil + return nil, nil } -// voteStart uses the ticketvote plugin to start a vote. func (p *politeiawww) voteStart(ctx context.Context, s ticketvote.Start) (*ticketvote.StartReply, error) { - b, err := ticketvote.EncodeStart(s) - if err != nil { - return nil, err - } - r, err := p.pluginCommand(ctx, ticketvote.ID, - ticketvote.CmdStart, string(b)) - if err != nil { - return nil, err - } - sr, err := ticketvote.DecodeStartReply([]byte(r)) - if err != nil { - return nil, err - } - return sr, nil + return nil, nil } -// castBallot uses the ticketvote plugin to cast a ballot of votes. func (p *politeiawww) castBallot(ctx context.Context, tb ticketvote.CastBallot) (*ticketvote.CastBallotReply, error) { - b, err := ticketvote.EncodeCastBallot(tb) - if err != nil { - return nil, err - } - r, err := p.pluginCommand(ctx, ticketvote.ID, - ticketvote.CmdCastBallot, string(b)) - if err != nil { - return nil, err - } - br, err := ticketvote.DecodeCastBallotReply([]byte(r)) - if err != nil { - return nil, err - } - return br, nil + return nil, nil } -// voteDetails uses the ticketvote plugin to fetch the details of a vote. -func (p *politeiawww) voteDetails(ctx context.Context, tokens []string) (*ticketvote.DetailsReply, error) { - d := ticketvote.Details{ - Tokens: tokens, - } - b, err := ticketvote.EncodeDetails(d) - if err != nil { - return nil, err - } - r, err := p.pluginCommand(ctx, ticketvote.ID, - ticketvote.CmdDetails, string(b)) - if err != nil { - return nil, err - } - dr, err := ticketvote.DecodeDetailsReply([]byte(r)) - if err != nil { - return nil, err - } - return dr, nil +func (p *politeiawww) voteDetails(ctx context.Context, token string) (*ticketvote.DetailsReply, error) { + return nil, nil } -// voteResults uses the ticketvote plugin to fetch cast votes for a record. func (p *politeiawww) voteResults(ctx context.Context, token string) (*ticketvote.ResultsReply, error) { - cv := ticketvote.Results{ - Token: token, - } - b, err := ticketvote.EncodeResults(cv) - if err != nil { - return nil, err - } - r, err := p.pluginCommand(ctx, ticketvote.ID, - ticketvote.CmdResults, string(b)) - if err != nil { - return nil, err - } - cvr, err := ticketvote.DecodeResultsReply([]byte(r)) - if err != nil { - return nil, err - } - return cvr, nil + return nil, nil } -// voteSummaries uses the ticketvote plugin to fetch vote summaries. -func (p *politeiawww) voteSummaries(ctx context.Context, tokens []string) (*ticketvote.SummariesReply, error) { - s := ticketvote.Summaries{ - Tokens: tokens, - } - b, err := ticketvote.EncodeSummaries(s) - if err != nil { - return nil, err - } - r, err := p.pluginCommand(ctx, ticketvote.ID, - ticketvote.CmdSummaries, string(b)) - if err != nil { - return nil, err - } - sr, err := ticketvote.DecodeSummariesReply([]byte(r)) - if err != nil { - return nil, err - } - return sr, nil +func (p *politeiawww) voteSummaries(ctx context.Context, tokens []string) (map[string]ticketvote.VoteSummary, error) { + + return nil, nil } -func (p *politeiawww) voteTimestamps(ctx context.Context, t ticketvote.Timestamps) (*ticketvote.TimestampsReply, error) { - b, err := json.Marshal(t) - if err != nil { - return nil, err - } +func (p *politeiawww) voteTimestamps(ctx context.Context, token string) (*ticketvote.TimestampsReply, error) { + _ = token + var b []byte r, err := p.pluginCommand(ctx, ticketvote.ID, ticketvote.CmdTimestamps, string(b)) if err != nil { @@ -183,9 +89,7 @@ func (p *politeiawww) processTicketVoteTimestamps(ctx context.Context, t tkv1.Ti log.Tracef("processTicketVoteTimestamps: %v", t.Token) // Send plugin command - r, err := p.voteTimestamps(ctx, ticketvote.Timestamps{ - Token: t.Token, - }) + r, err := p.voteTimestamps(ctx, t.Token) if err != nil { return nil, err } diff --git a/politeiawww/www.go b/politeiawww/www.go index f1736db3d..6f040f3bd 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -85,45 +85,35 @@ func convertWWWErrorStatusFromPD(e pd.ErrorStatusT) www.ErrorStatusT { // TODO verify all plugin errors have been added to these www conversion // functions -func convertWWWErrorStatusFromPiPlugin(e piplugin.ErrorStatusT) www.ErrorStatusT { - switch e { - case piplugin.ErrorStatusPropLinkToInvalid: - return www.ErrorStatusInvalidLinkTo - case piplugin.ErrorStatusVoteStatusInvalid: - return www.ErrorStatusWrongVoteStatus - } +func convertWWWErrorStatusFromPiPlugin(e piplugin.ErrorCodeT) www.ErrorStatusT { return www.ErrorStatusInvalid } -func convertWWWErrorStatusFromComments(e comments.ErrorStatusT) www.ErrorStatusT { +func convertWWWErrorStatusFromComments(e comments.ErrorCodeT) www.ErrorStatusT { switch e { - case comments.ErrorStatusTokenInvalid: + case comments.ErrorCodeTokenInvalid: return www.ErrorStatusInvalidCensorshipToken - case comments.ErrorStatusRecordNotFound: - return www.ErrorStatusProposalNotFound - case comments.ErrorStatusCommentNotFound: + case comments.ErrorCodeCommentNotFound: return www.ErrorStatusCommentNotFound - case comments.ErrorStatusParentIDInvalid: + case comments.ErrorCodeParentIDInvalid: return www.ErrorStatusCommentNotFound } return www.ErrorStatusInvalid } -func convertWWWErrorStatusFromTicketVote(e ticketvote.ErrorStatusT) www.ErrorStatusT { +func convertWWWErrorStatusFromTicketVote(e ticketvote.ErrorCodeT) www.ErrorStatusT { switch e { - case ticketvote.ErrorStatusTokenInvalid: + case ticketvote.ErrorCodeTokenInvalid: return www.ErrorStatusInvalidCensorshipToken - case ticketvote.ErrorStatusPublicKeyInvalid: + case ticketvote.ErrorCodePublicKeyInvalid: return www.ErrorStatusInvalidPublicKey - case ticketvote.ErrorStatusSignatureInvalid: + case ticketvote.ErrorCodeSignatureInvalid: return www.ErrorStatusInvalidSignature - case ticketvote.ErrorStatusRecordNotFound: - return www.ErrorStatusProposalNotFound - case ticketvote.ErrorStatusRecordStatusInvalid: + case ticketvote.ErrorCodeRecordStatusInvalid: return www.ErrorStatusWrongStatus - case ticketvote.ErrorStatusVoteParamsInvalid: + case ticketvote.ErrorCodeVoteParamsInvalid: return www.ErrorStatusInvalidPropVoteParams - case ticketvote.ErrorStatusVoteStatusInvalid: + case ticketvote.ErrorCodeVoteStatusInvalid: return www.ErrorStatusInvalidPropVoteStatus } return www.ErrorStatusInvalid @@ -142,15 +132,15 @@ func convertWWWErrorStatus(pluginID string, errCode int) www.ErrorStatusT { return convertWWWErrorStatusFromPD(e) case piplugin.ID: // Pi plugin - e := piplugin.ErrorStatusT(errCode) + e := piplugin.ErrorCodeT(errCode) return convertWWWErrorStatusFromPiPlugin(e) case comments.ID: // Comments plugin - e := comments.ErrorStatusT(errCode) + e := comments.ErrorCodeT(errCode) return convertWWWErrorStatusFromComments(e) case ticketvote.ID: // Ticket vote plugin - e := ticketvote.ErrorStatusT(errCode) + e := ticketvote.ErrorCodeT(errCode) return convertWWWErrorStatusFromTicketVote(e) } @@ -188,83 +178,71 @@ func convertPiErrorStatusFromPD(e pd.ErrorStatusT) pi.ErrorStatusT { return pi.ErrorStatusInvalid } -func convertPiErrorStatusFromPiPlugin(e piplugin.ErrorStatusT) pi.ErrorStatusT { +func convertPiErrorStatusFromPiPlugin(e piplugin.ErrorCodeT) pi.ErrorStatusT { switch e { - case piplugin.ErrorStatusPageSizeExceeded: + case piplugin.ErrorCodePageSizeExceeded: return pi.ErrorStatusPageSizeExceeded - case piplugin.ErrorStatusPropNotFound: - return pi.ErrorStatusPropNotFound - case piplugin.ErrorStatusPropStateInvalid: - return pi.ErrorStatusPropStateInvalid - case piplugin.ErrorStatusPropTokenInvalid: + case piplugin.ErrorCodePropTokenInvalid: return pi.ErrorStatusPropTokenInvalid - case piplugin.ErrorStatusPropStatusInvalid: + case piplugin.ErrorCodePropStatusInvalid: return pi.ErrorStatusPropStatusInvalid - case piplugin.ErrorStatusPropVersionInvalid: + case piplugin.ErrorCodePropVersionInvalid: return pi.ErrorStatusPropVersionInvalid - case piplugin.ErrorStatusPropStatusChangeInvalid: + case piplugin.ErrorCodePropStatusChangeInvalid: return pi.ErrorStatusPropStatusChangeInvalid - case piplugin.ErrorStatusPropLinkToInvalid: + case piplugin.ErrorCodePropLinkToInvalid: return pi.ErrorStatusPropLinkToInvalid - case piplugin.ErrorStatusVoteStatusInvalid: + case piplugin.ErrorCodeVoteStatusInvalid: return pi.ErrorStatusVoteStatusInvalid - case piplugin.ErrorStatusStartDetailsInvalid: + case piplugin.ErrorCodeStartDetailsInvalid: return pi.ErrorStatusStartDetailsInvalid - case piplugin.ErrorStatusStartDetailsMissing: + case piplugin.ErrorCodeStartDetailsMissing: return pi.ErrorStatusStartDetailsMissing - case piplugin.ErrorStatusVoteParentInvalid: + case piplugin.ErrorCodeVoteParentInvalid: return pi.ErroStatusVoteParentInvalid - case piplugin.ErrorStatusLinkByNotExpired: - return pi.ErrorStatusLinkByNotExpired } return pi.ErrorStatusInvalid } -func convertPiErrorStatusFromComments(e comments.ErrorStatusT) pi.ErrorStatusT { +func convertPiErrorStatusFromComments(e comments.ErrorCodeT) pi.ErrorStatusT { switch e { - case comments.ErrorStatusStateInvalid: - return pi.ErrorStatusPropStateInvalid - case comments.ErrorStatusTokenInvalid: + case comments.ErrorCodeTokenInvalid: return pi.ErrorStatusPropTokenInvalid - case comments.ErrorStatusPublicKeyInvalid: + case comments.ErrorCodePublicKeyInvalid: return pi.ErrorStatusPublicKeyInvalid - case comments.ErrorStatusSignatureInvalid: + case comments.ErrorCodeSignatureInvalid: return pi.ErrorStatusSignatureInvalid - case comments.ErrorStatusCommentTextInvalid: + case comments.ErrorCodeCommentTextInvalid: return pi.ErrorStatusCommentTextInvalid - case comments.ErrorStatusRecordNotFound: - return pi.ErrorStatusPropNotFound - case comments.ErrorStatusCommentNotFound: + case comments.ErrorCodeCommentNotFound: return pi.ErrorStatusCommentNotFound - case comments.ErrorStatusUserUnauthorized: + case comments.ErrorCodeUserUnauthorized: return pi.ErrorStatusUnauthorized - case comments.ErrorStatusParentIDInvalid: + case comments.ErrorCodeParentIDInvalid: return pi.ErrorStatusCommentParentIDInvalid - case comments.ErrorStatusVoteInvalid: + case comments.ErrorCodeVoteInvalid: return pi.ErrorStatusCommentVoteInvalid - case comments.ErrorStatusVoteChangesMax: + case comments.ErrorCodeVoteChangesMax: return pi.ErrorStatusCommentVoteChangesMax } return pi.ErrorStatusInvalid } -func convertPiErrorStatusFromTicketVote(e ticketvote.ErrorStatusT) pi.ErrorStatusT { +func convertPiErrorStatusFromTicketVote(e ticketvote.ErrorCodeT) pi.ErrorStatusT { switch e { - case ticketvote.ErrorStatusTokenInvalid: + case ticketvote.ErrorCodeTokenInvalid: return pi.ErrorStatusPropTokenInvalid - case ticketvote.ErrorStatusPublicKeyInvalid: + case ticketvote.ErrorCodePublicKeyInvalid: return pi.ErrorStatusPublicKeyInvalid - case ticketvote.ErrorStatusSignatureInvalid: + case ticketvote.ErrorCodeSignatureInvalid: return pi.ErrorStatusSignatureInvalid - case ticketvote.ErrorStatusRecordNotFound: - return pi.ErrorStatusPropNotFound - case ticketvote.ErrorStatusRecordStatusInvalid: + case ticketvote.ErrorCodeRecordStatusInvalid: return pi.ErrorStatusPropStatusInvalid - case ticketvote.ErrorStatusAuthorizationInvalid: + case ticketvote.ErrorCodeAuthorizationInvalid: return pi.ErrorStatusVoteAuthInvalid - case ticketvote.ErrorStatusVoteParamsInvalid: + case ticketvote.ErrorCodeVoteParamsInvalid: return pi.ErrorStatusVoteParamsInvalid - case ticketvote.ErrorStatusVoteStatusInvalid: + case ticketvote.ErrorCodeVoteStatusInvalid: return pi.ErrorStatusVoteStatusInvalid } return pi.ErrorStatusInvalid @@ -283,15 +261,15 @@ func convertPiErrorStatus(pluginID string, errCode int) pi.ErrorStatusT { return convertPiErrorStatusFromPD(e) case piplugin.ID: // Pi plugin - e := piplugin.ErrorStatusT(errCode) + e := piplugin.ErrorCodeT(errCode) return convertPiErrorStatusFromPiPlugin(e) case comments.ID: // Comments plugin - e := comments.ErrorStatusT(errCode) + e := comments.ErrorCodeT(errCode) return convertPiErrorStatusFromComments(e) case ticketvote.ID: // Ticket vote plugin - e := ticketvote.ErrorStatusT(errCode) + e := ticketvote.ErrorCodeT(errCode) return convertPiErrorStatusFromTicketVote(e) } @@ -355,8 +333,7 @@ func respondWithPiError(w http.ResponseWriter, r *http.Request, format string, e } else { log.Errorf("Pi user error: %v %v %v: %v", remoteAddr(r), int64(ue.ErrorCode), - pi.ErrorStatus[ue.ErrorCode], - strings.Join(ue.ErrorContext, ", ")) + pi.ErrorStatus[ue.ErrorCode], ue.ErrorContext) } util.RespondWithJSON(w, http.StatusBadRequest, @@ -415,7 +392,7 @@ func respondWithPiError(w http.ResponseWriter, r *http.Request, format string, e util.RespondWithJSON(w, http.StatusBadRequest, pi.UserErrorReply{ ErrorCode: piErrCode, - ErrorContext: errContext, + ErrorContext: strings.Join(errContext, ", "), }) return From f1fd945b0b0223146107967099f397e430c8242b Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 20 Jan 2021 08:53:09 -0600 Subject: [PATCH 237/449] Add user politeiad plugin. --- .../backend/tlogbe/plugins/comments/cmds.go | 1472 +++++++++++++++++ .../tlogbe/plugins/comments/comments.go | 1454 +--------------- .../comments/{indexes.go => recordindex.go} | 0 .../backend/tlogbe/plugins/dcrdata/dcrdata.go | 4 +- politeiad/backend/tlogbe/plugins/pi/cmds.go | 133 ++ politeiad/backend/tlogbe/plugins/pi/hooks.go | 150 +- politeiad/backend/tlogbe/plugins/pi/pi.go | 215 +-- politeiad/backend/tlogbe/plugins/pi/user.go | 106 -- politeiad/backend/tlogbe/plugins/plugins.go | 78 +- .../tlogbe/plugins/ticketvote/hooks.go | 4 +- .../tlogbe/plugins/ticketvote/inventory.go | 22 +- .../tlogbe/plugins/ticketvote/ticketvote.go | 4 +- .../backend/tlogbe/plugins/user/cache.go | 105 ++ politeiad/backend/tlogbe/plugins/user/cmds.go | 64 + .../backend/tlogbe/plugins/user/hooks.go | 227 +++ politeiad/backend/tlogbe/plugins/user/log.go | 25 + politeiad/backend/tlogbe/plugins/user/user.go | 100 ++ politeiad/backend/tlogbe/tlog/plugin.go | 32 +- politeiad/backend/tlogbe/tlog/tlog.go | 6 + politeiad/backend/tlogbe/tlog/tlogclient.go | 5 - politeiad/backend/tlogbe/tlogbe.go | 37 +- politeiad/log.go | 16 +- politeiad/plugins/pi/pi.go | 42 +- politeiad/plugins/user/user.go | 77 + politeiad/util/util.go | 50 + politeiawww/api/comments/v1/v1.go | 1 + politeiawww/comments.go | 6 +- util/signature.go | 2 +- 28 files changed, 2452 insertions(+), 1985 deletions(-) create mode 100644 politeiad/backend/tlogbe/plugins/comments/cmds.go rename politeiad/backend/tlogbe/plugins/comments/{indexes.go => recordindex.go} (100%) create mode 100644 politeiad/backend/tlogbe/plugins/pi/cmds.go delete mode 100644 politeiad/backend/tlogbe/plugins/pi/user.go create mode 100644 politeiad/backend/tlogbe/plugins/user/cache.go create mode 100644 politeiad/backend/tlogbe/plugins/user/cmds.go create mode 100644 politeiad/backend/tlogbe/plugins/user/hooks.go create mode 100644 politeiad/backend/tlogbe/plugins/user/log.go create mode 100644 politeiad/backend/tlogbe/plugins/user/user.go create mode 100644 politeiad/plugins/user/user.go create mode 100644 politeiad/util/util.go diff --git a/politeiad/backend/tlogbe/plugins/comments/cmds.go b/politeiad/backend/tlogbe/plugins/comments/cmds.go new file mode 100644 index 000000000..6f150cef9 --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/comments/cmds.go @@ -0,0 +1,1472 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package comments + +import ( + "bytes" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "sort" + "strconv" + "time" + + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/store" + "github.com/decred/politeia/politeiad/plugins/comments" + "github.com/decred/politeia/util" +) + +const ( + // Blob entry data descriptors + dataDescriptorCommentAdd = "cadd_v1" + dataDescriptorCommentDel = "cdel_v1" + dataDescriptorCommentVote = "cvote_v1" + + // Data types + dataTypeCommentAdd = "cadd" + dataTypeCommentDel = "cdel" + dataTypeCommentVote = "cvote" +) + +func tokenDecode(token string) ([]byte, error) { + return util.TokenDecode(util.TokenTypeTlog, token) +} + +func convertSignatureError(err error) backend.PluginError { + var e util.SignatureError + var s comments.ErrorCodeT + if errors.As(err, &e) { + switch e.ErrorCode { + case util.ErrorStatusPublicKeyInvalid: + s = comments.ErrorCodePublicKeyInvalid + case util.ErrorStatusSignatureInvalid: + s = comments.ErrorCodeSignatureInvalid + } + } + return backend.PluginError{ + PluginID: comments.ID, + ErrorCode: int(s), + ErrorContext: e.ErrorContext, + } +} + +func convertBlobEntryFromCommentAdd(c comments.CommentAdd) (*store.BlobEntry, error) { + data, err := json.Marshal(c) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorCommentAdd, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertBlobEntryFromCommentDel(c comments.CommentDel) (*store.BlobEntry, error) { + data, err := json.Marshal(c) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorCommentDel, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertBlobEntryFromCommentVote(c comments.CommentVote) (*store.BlobEntry, error) { + data, err := json.Marshal(c) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorCommentVote, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertCommentAddFromBlobEntry(be store.BlobEntry) (*comments.CommentAdd, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorCommentAdd { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorCommentAdd) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, fmt.Errorf("decode digest: %v", err) + } + if !bytes.Equal(util.Digest(b), digest) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), digest) + } + var c comments.CommentAdd + err = json.Unmarshal(b, &c) + if err != nil { + return nil, fmt.Errorf("unmarshal CommentAdd: %v", err) + } + + return &c, nil +} + +func convertCommentDelFromBlobEntry(be store.BlobEntry) (*comments.CommentDel, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorCommentDel { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorCommentDel) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, fmt.Errorf("decode digest: %v", err) + } + if !bytes.Equal(util.Digest(b), digest) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), digest) + } + var c comments.CommentDel + err = json.Unmarshal(b, &c) + if err != nil { + return nil, fmt.Errorf("unmarshal CommentDel: %v", err) + } + + return &c, nil +} + +func convertCommentVoteFromBlobEntry(be store.BlobEntry) (*comments.CommentVote, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorCommentVote { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorCommentVote) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, fmt.Errorf("decode digest: %v", err) + } + if !bytes.Equal(util.Digest(b), digest) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), digest) + } + var cv comments.CommentVote + err = json.Unmarshal(b, &cv) + if err != nil { + return nil, fmt.Errorf("unmarshal CommentVote: %v", err) + } + + return &cv, nil +} + +func convertCommentFromCommentAdd(ca comments.CommentAdd) comments.Comment { + return comments.Comment{ + UserID: ca.UserID, + Token: ca.Token, + ParentID: ca.ParentID, + Comment: ca.Comment, + PublicKey: ca.PublicKey, + Signature: ca.Signature, + CommentID: ca.CommentID, + Version: ca.Version, + Timestamp: ca.Timestamp, + Receipt: ca.Receipt, + Downvotes: 0, // Not part of commentAdd data + Upvotes: 0, // Not part of commentAdd data + Deleted: false, + Reason: "", + ExtraData: ca.ExtraData, + ExtraDataHint: ca.ExtraDataHint, + } +} + +func convertCommentFromCommentDel(cd comments.CommentDel) comments.Comment { + // Score needs to be filled in separately + return comments.Comment{ + UserID: cd.UserID, + Token: cd.Token, + ParentID: cd.ParentID, + Comment: "", + Signature: "", + CommentID: cd.CommentID, + Version: 0, + Timestamp: cd.Timestamp, + Receipt: cd.Receipt, + Downvotes: 0, + Upvotes: 0, + Deleted: true, + Reason: cd.Reason, + } +} + +// commentVersionLatest returns the latest comment version. +func commentVersionLatest(cidx commentIndex) uint32 { + var maxVersion uint32 + for version := range cidx.Adds { + if version > maxVersion { + maxVersion = version + } + } + return maxVersion +} + +// commentExists returns whether the provided comment ID exists. +func commentExists(ridx recordIndex, commentID uint32) bool { + _, ok := ridx.Comments[commentID] + return ok +} + +// commentIDLatest returns the latest comment ID. +func commentIDLatest(idx recordIndex) uint32 { + var maxID uint32 + for id := range idx.Comments { + if id > maxID { + maxID = id + } + } + return maxID +} + +func (p *commentsPlugin) commentAddSave(treeID int64, ca comments.CommentAdd) ([]byte, error) { + be, err := convertBlobEntryFromCommentAdd(ca) + if err != nil { + return nil, err + } + d, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, err + } + err = p.tlog.BlobSave(treeID, dataTypeCommentAdd, *be) + if err != nil { + return nil, err + } + return d, nil +} + +// commentAdds returns the commentAdd for all specified digests. +func (p *commentsPlugin) commentAdds(treeID int64, digests [][]byte) ([]comments.CommentAdd, error) { + // Retrieve blobs + blobs, err := p.tlog.Blobs(treeID, digests) + if err != nil { + return nil, err + } + if len(blobs) != len(digests) { + notFound := make([]string, 0, len(blobs)) + for _, v := range digests { + m := hex.EncodeToString(v) + _, ok := blobs[m] + if !ok { + notFound = append(notFound, m) + } + } + return nil, fmt.Errorf("blobs not found: %v", notFound) + } + + // Decode blobs + adds := make([]comments.CommentAdd, 0, len(blobs)) + for _, v := range blobs { + c, err := convertCommentAddFromBlobEntry(v) + if err != nil { + return nil, err + } + adds = append(adds, *c) + } + + return adds, nil +} + +func (p *commentsPlugin) commentDelSave(treeID int64, cd comments.CommentDel) ([]byte, error) { + be, err := convertBlobEntryFromCommentDel(cd) + if err != nil { + return nil, err + } + d, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, err + } + err = p.tlog.BlobSave(treeID, dataTypeCommentDel, *be) + if err != nil { + return nil, err + } + return d, nil +} + +func (p *commentsPlugin) commentDels(treeID int64, digests [][]byte) ([]comments.CommentDel, error) { + // Retrieve blobs + blobs, err := p.tlog.Blobs(treeID, digests) + if err != nil { + return nil, err + } + if len(blobs) != len(digests) { + notFound := make([]string, 0, len(blobs)) + for _, v := range digests { + m := hex.EncodeToString(v) + _, ok := blobs[m] + if !ok { + notFound = append(notFound, m) + } + } + return nil, fmt.Errorf("blobs not found: %v", notFound) + } + + // Decode blobs + dels := make([]comments.CommentDel, 0, len(blobs)) + for _, v := range blobs { + d, err := convertCommentDelFromBlobEntry(v) + if err != nil { + return nil, err + } + dels = append(dels, *d) + } + + return dels, nil +} + +func (p *commentsPlugin) commentVoteSave(treeID int64, cv comments.CommentVote) ([]byte, error) { + be, err := convertBlobEntryFromCommentVote(cv) + if err != nil { + return nil, err + } + d, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, err + } + err = p.tlog.BlobSave(treeID, dataTypeCommentVote, *be) + if err != nil { + return nil, err + } + return d, nil +} + +func (p *commentsPlugin) commentVotes(treeID int64, digests [][]byte) ([]comments.CommentVote, error) { + // Retrieve blobs + blobs, err := p.tlog.Blobs(treeID, digests) + if err != nil { + return nil, err + } + if len(blobs) != len(digests) { + notFound := make([]string, 0, len(blobs)) + for _, v := range digests { + m := hex.EncodeToString(v) + _, ok := blobs[m] + if !ok { + notFound = append(notFound, m) + } + } + return nil, fmt.Errorf("blobs not found: %v", notFound) + } + + // Decode blobs + votes := make([]comments.CommentVote, 0, len(blobs)) + for _, v := range blobs { + c, err := convertCommentVoteFromBlobEntry(v) + if err != nil { + return nil, err + } + votes = append(votes, *c) + } + + return votes, nil +} + +// comments returns the most recent version of the specified comments. Deleted +// comments are returned with limited data. Comment IDs that do not correspond +// to an actual comment are not included in the returned map. It is the +// responsibility of the caller to ensure a comment is returned for each of the +// provided comment IDs. +func (p *commentsPlugin) comments(treeID int64, ridx recordIndex, commentIDs []uint32) (map[uint32]comments.Comment, error) { + // Aggregate the digests for all records that need to be looked up. + // If a comment has been deleted then the only record that will + // still exist is the comment del record. If the comment has not + // been deleted then the comment add record will need to be + // retrieved for the latest version of the comment. + var ( + digestAdds = make([][]byte, 0, len(commentIDs)) + digestDels = make([][]byte, 0, len(commentIDs)) + ) + for _, v := range commentIDs { + cidx, ok := ridx.Comments[v] + if !ok { + // Comment does not exist + continue + } + + // Comment del record + if cidx.Del != nil { + digestDels = append(digestDels, cidx.Del) + continue + } + + // Comment add record + version := commentVersionLatest(cidx) + digestAdds = append(digestAdds, cidx.Adds[version]) + } + + // Get comment add records + adds, err := p.commentAdds(treeID, digestAdds) + if err != nil { + return nil, fmt.Errorf("commentAdds: %v", err) + } + if len(adds) != len(digestAdds) { + return nil, fmt.Errorf("wrong comment adds count; got %v, want %v", + len(adds), len(digestAdds)) + } + + // Get comment del records + dels, err := p.commentDels(treeID, digestDels) + if err != nil { + return nil, fmt.Errorf("commentDels: %v", err) + } + if len(dels) != len(digestDels) { + return nil, fmt.Errorf("wrong comment dels count; got %v, want %v", + len(dels), len(digestDels)) + } + + // Prepare comments + cs := make(map[uint32]comments.Comment, len(commentIDs)) + for _, v := range adds { + c := convertCommentFromCommentAdd(v) + cidx, ok := ridx.Comments[c.CommentID] + if !ok { + return nil, fmt.Errorf("comment index not found %v", c.CommentID) + } + c.Downvotes, c.Upvotes = calcVoteScore(cidx) + cs[v.CommentID] = c + } + for _, v := range dels { + c := convertCommentFromCommentDel(v) + cs[v.CommentID] = c + } + + return cs, nil +} + +// comment returns the latest version of the provided comment. +func (p *commentsPlugin) comment(treeID int64, ridx recordIndex, commentID uint32) (*comments.Comment, error) { + cs, err := p.comments(treeID, ridx, []uint32{commentID}) + if err != nil { + return nil, fmt.Errorf("comments: %v", err) + } + c, ok := cs[commentID] + if !ok { + return nil, fmt.Errorf("comment not found") + } + return &c, nil +} + +func (p *commentsPlugin) timestamp(treeID int64, digest []byte) (*comments.Timestamp, error) { + // Get timestamp + t, err := p.tlog.Timestamp(treeID, digest) + if err != nil { + return nil, err + } + + // Convert response + proofs := make([]comments.Proof, 0, len(t.Proofs)) + for _, v := range t.Proofs { + proofs = append(proofs, comments.Proof{ + Type: v.Type, + Digest: v.Digest, + MerkleRoot: v.MerkleRoot, + MerklePath: v.MerklePath, + ExtraData: v.ExtraData, + }) + } + return &comments.Timestamp{ + Data: t.Data, + Digest: t.Digest, + TxID: t.TxID, + MerkleRoot: t.MerkleRoot, + Proofs: proofs, + }, nil +} + +// calcVoteScore returns the vote score for the provided comment index. The +// returned values are the downvotes and upvotes, respectively. +func calcVoteScore(cidx commentIndex) (uint64, uint64) { + // Find the vote score by replaying all existing votes from all + // users. The net effect of a new vote on a comment score depends + // on the previous vote from that uuid. Example, a user upvotes a + // comment that they have already upvoted, the resulting vote score + // is 0 due to the second upvote removing the original upvote. + var upvotes uint64 + var downvotes uint64 + for _, votes := range cidx.Votes { + // Calculate the vote score that this user is contributing. This + // can only ever be -1, 0, or 1. + var score int64 + for _, v := range votes { + vote := int64(v.Vote) + switch { + case score == 0: + // No previous vote. New vote becomes the score. + score = vote + + case score == vote: + // New vote is the same as the previous vote. The vote gets + // removed from the score, making the score 0. + score = 0 + + case score != vote: + // New vote is different than the previous vote. New vote + // becomes the score. + score = vote + } + } + + // Add the net result of all votes from this user to the totals. + switch score { + case 0: + // Nothing to do + case -1: + downvotes++ + case 1: + upvotes++ + default: + // Something went wrong + e := fmt.Errorf("unexpected vote score %v", score) + panic(e) + } + } + + return downvotes, upvotes +} + +func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdNew: %v %x %v", treeID, token, payload) + + // Decode payload + var n comments.New + err := json.Unmarshal([]byte(payload), &n) + if err != nil { + return "", err + } + + // Verify token + t, err := tokenDecode(n.Token) + if err != nil { + return "", backend.PluginError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeTokenInvalid), + ErrorContext: err.Error(), + } + } + if !bytes.Equal(t, token) { + e := fmt.Sprintf("comment token does not match route token: "+ + "got %x, want %x", t, token) + return "", backend.PluginError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeTokenInvalid), + ErrorContext: e, + } + } + + // Verify signature + msg := n.Token + strconv.FormatUint(uint64(n.ParentID), 10) + n.Comment + err = util.VerifySignature(n.Signature, n.PublicKey, msg) + if err != nil { + return "", convertSignatureError(err) + } + + // Verify comment + if len(n.Comment) > comments.PolicyCommentLengthMax { + return "", backend.PluginError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeCommentTextInvalid), + ErrorContext: "exceeds max length", + } + } + + // The record index must be pulled and updated. The record lock + // must be held for the remainder of this function. + m := p.mutex(token) + m.Lock() + defer m.Unlock() + + // Get record index + ridx, err := p.recordIndexLocked(token) + if err != nil { + return "", err + } + + // Verify parent comment exists if set. A parent ID of 0 means that + // this is a base level comment, not a reply to another comment. + if n.ParentID > 0 && !commentExists(*ridx, n.ParentID) { + return "", backend.PluginError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeParentIDInvalid), + ErrorContext: "parent ID comment not found", + } + } + + // Setup comment + receipt := p.identity.SignMessage([]byte(n.Signature)) + ca := comments.CommentAdd{ + UserID: n.UserID, + Token: n.Token, + ParentID: n.ParentID, + Comment: n.Comment, + PublicKey: n.PublicKey, + Signature: n.Signature, + CommentID: commentIDLatest(*ridx) + 1, + Version: 1, + Timestamp: time.Now().Unix(), + Receipt: hex.EncodeToString(receipt[:]), + ExtraData: n.ExtraData, + ExtraDataHint: n.ExtraDataHint, + } + + // Save comment + digest, err := p.commentAddSave(treeID, ca) + if err != nil { + return "", fmt.Errorf("commentAddSave: %v", err) + } + + // Update index + ridx.Comments[ca.CommentID] = commentIndex{ + Adds: map[uint32][]byte{ + 1: digest, + }, + Del: nil, + Votes: make(map[string][]voteIndex), + } + + // Save index + err = p.recordIndexSaveLocked(token, *ridx) + if err != nil { + return "", err + } + + log.Debugf("Comment saved to record %v comment ID %v", + ca.Token, ca.CommentID) + + // Return new comment + c, err := p.comment(treeID, *ridx, ca.CommentID) + if err != nil { + return "", fmt.Errorf("comment %x %v: %v", token, ca.CommentID, err) + } + + // Prepare reply + nr := comments.NewReply{ + Comment: *c, + } + reply, err := json.Marshal(nr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdEdit: %v %x %v", treeID, token, payload) + + // Decode payload + var e comments.Edit + err := json.Unmarshal([]byte(payload), &e) + if err != nil { + return "", err + } + + // Verify token + t, err := tokenDecode(e.Token) + if err != nil { + return "", backend.PluginError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeTokenInvalid), + ErrorContext: err.Error(), + } + } + if !bytes.Equal(t, token) { + e := fmt.Sprintf("comment token does not match route token: "+ + "got %x, want %x", t, token) + return "", backend.PluginError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeTokenInvalid), + ErrorContext: e, + } + } + + // Verify signature + msg := e.Token + strconv.FormatUint(uint64(e.ParentID), 10) + e.Comment + err = util.VerifySignature(e.Signature, e.PublicKey, msg) + if err != nil { + return "", convertSignatureError(err) + } + + // Verify comment + if len(e.Comment) > comments.PolicyCommentLengthMax { + return "", backend.PluginError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeCommentTextInvalid), + ErrorContext: "exceeds max length", + } + } + + // The record index must be pulled and updated. The record lock + // must be held for the remainder of this function. + m := p.mutex(token) + m.Lock() + defer m.Unlock() + + // Get record index + ridx, err := p.recordIndexLocked(token) + if err != nil { + return "", err + } + + // Get the existing comment + cs, err := p.comments(treeID, *ridx, []uint32{e.CommentID}) + if err != nil { + return "", fmt.Errorf("comments %v: %v", e.CommentID, err) + } + existing, ok := cs[e.CommentID] + if !ok { + return "", backend.PluginError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeCommentNotFound), + } + } + + // Verify the user ID + if e.UserID != existing.UserID { + return "", backend.PluginError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeUserUnauthorized), + } + } + + // Verify the parent ID + if e.ParentID != existing.ParentID { + e := fmt.Sprintf("parent id cannot change; got %v, want %v", + e.ParentID, existing.ParentID) + return "", backend.PluginError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeParentIDInvalid), + ErrorContext: e, + } + } + + // Verify comment changes + if e.Comment == existing.Comment { + return "", backend.PluginError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeCommentTextInvalid), + ErrorContext: "comment did not change", + } + } + + // Create a new comment version + receipt := p.identity.SignMessage([]byte(e.Signature)) + ca := comments.CommentAdd{ + UserID: e.UserID, + Token: e.Token, + ParentID: e.ParentID, + Comment: e.Comment, + PublicKey: e.PublicKey, + Signature: e.Signature, + CommentID: e.CommentID, + Version: existing.Version + 1, + Timestamp: time.Now().Unix(), + Receipt: hex.EncodeToString(receipt[:]), + ExtraData: e.ExtraData, + ExtraDataHint: e.ExtraDataHint, + } + + // Save comment + digest, err := p.commentAddSave(treeID, ca) + if err != nil { + return "", fmt.Errorf("commentAddSave: %v", err) + } + + // Update index + ridx.Comments[ca.CommentID].Adds[ca.Version] = digest + + // Save index + err = p.recordIndexSaveLocked(token, *ridx) + if err != nil { + return "", err + } + + log.Debugf("Comment edited on record %v comment ID %v", + ca.Token, ca.CommentID) + + // Return updated comment + c, err := p.comment(treeID, *ridx, e.CommentID) + if err != nil { + return "", fmt.Errorf("comment %x %v: %v", token, e.CommentID, err) + } + + // Prepare reply + er := comments.EditReply{ + Comment: *c, + } + reply, err := json.Marshal(er) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdDel: %v %x %v", treeID, token, payload) + + // Decode payload + var d comments.Del + err := json.Unmarshal([]byte(payload), &d) + if err != nil { + return "", err + } + + // Verify token + t, err := tokenDecode(d.Token) + if err != nil { + return "", backend.PluginError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeTokenInvalid), + ErrorContext: err.Error(), + } + } + if !bytes.Equal(t, token) { + e := fmt.Sprintf("comment token does not match route token: "+ + "got %x, want %x", t, token) + return "", backend.PluginError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeTokenInvalid), + ErrorContext: e, + } + } + + // Verify signature + msg := d.Token + strconv.FormatUint(uint64(d.CommentID), 10) + d.Reason + err = util.VerifySignature(d.Signature, d.PublicKey, msg) + if err != nil { + return "", convertSignatureError(err) + } + + // The record index must be pulled and updated. The record lock + // must be held for the remainder of this function. + m := p.mutex(token) + m.Lock() + defer m.Unlock() + + // Get record index + ridx, err := p.recordIndexLocked(token) + if err != nil { + return "", err + } + + // Get the existing comment + cs, err := p.comments(treeID, *ridx, []uint32{d.CommentID}) + if err != nil { + return "", fmt.Errorf("comments %v: %v", d.CommentID, err) + } + existing, ok := cs[d.CommentID] + if !ok { + return "", backend.PluginError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeCommentNotFound), + } + } + + // Prepare comment delete + receipt := p.identity.SignMessage([]byte(d.Signature)) + cd := comments.CommentDel{ + Token: d.Token, + CommentID: d.CommentID, + Reason: d.Reason, + PublicKey: d.PublicKey, + Signature: d.Signature, + ParentID: existing.ParentID, + UserID: existing.UserID, + Timestamp: time.Now().Unix(), + Receipt: hex.EncodeToString(receipt[:]), + } + + // Save comment del + digest, err := p.commentDelSave(treeID, cd) + if err != nil { + return "", fmt.Errorf("commentDelSave: %v", err) + } + + // Update index + cidx, ok := ridx.Comments[d.CommentID] + if !ok { + // This should not be possible + e := fmt.Sprintf("comment not found in index: %v", d.CommentID) + panic(e) + } + cidx.Del = digest + ridx.Comments[d.CommentID] = cidx + + // Save index + err = p.recordIndexSaveLocked(token, *ridx) + if err != nil { + return "", err + } + + // Delete all comment versions + digests := make([][]byte, 0, len(cidx.Adds)) + for _, v := range cidx.Adds { + digests = append(digests, v) + } + err = p.tlog.BlobsDel(treeID, digests) + if err != nil { + return "", fmt.Errorf("del: %v", err) + } + + // Return updated comment + c, err := p.comment(treeID, *ridx, d.CommentID) + if err != nil { + return "", fmt.Errorf("comment %v: %v", d.CommentID, err) + } + + // Prepare reply + dr := comments.DelReply{ + Comment: *c, + } + reply, err := json.Marshal(dr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdVote: %v %x %v", treeID, token, payload) + + // Decode payload + var v comments.Vote + err := json.Unmarshal([]byte(payload), &v) + if err != nil { + return "", err + } + + // Verify token + t, err := tokenDecode(v.Token) + if err != nil { + return "", backend.PluginError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeTokenInvalid), + ErrorContext: err.Error(), + } + } + if !bytes.Equal(t, token) { + e := fmt.Sprintf("comment token does not match route token: "+ + "got %x, want %x", t, token) + return "", backend.PluginError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeTokenInvalid), + ErrorContext: e, + } + } + + // Verify vote + switch v.Vote { + case comments.VoteDownvote, comments.VoteUpvote: + // These are allowed + default: + return "", backend.PluginError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeVoteInvalid), + } + } + + // Verify signature + msg := v.Token + strconv.FormatUint(uint64(v.CommentID), 10) + + strconv.FormatInt(int64(v.Vote), 10) + err = util.VerifySignature(v.Signature, v.PublicKey, msg) + if err != nil { + return "", convertSignatureError(err) + } + + // The record index must be pulled and updated. The record lock + // must be held for the remainder of this function. + m := p.mutex(token) + m.Lock() + defer m.Unlock() + + // Get record index + ridx, err := p.recordIndexLocked(token) + if err != nil { + return "", err + } + + // Verify comment exists + cidx, ok := ridx.Comments[v.CommentID] + if !ok { + return "", backend.PluginError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeCommentNotFound), + } + } + + // Verify user has not exceeded max allowed vote changes + uvotes, ok := cidx.Votes[v.UserID] + if !ok { + uvotes = make([]voteIndex, 0) + } + if len(uvotes) > comments.PolicyVoteChangesMax { + return "", backend.PluginError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeVoteChangesMax), + } + } + + // Verify user is not voting on their own comment + cs, err := p.comments(treeID, *ridx, []uint32{v.CommentID}) + if err != nil { + return "", fmt.Errorf("comments %v: %v", v.CommentID, err) + } + c, ok := cs[v.CommentID] + if !ok { + return "", fmt.Errorf("comment not found %v", v.CommentID) + } + if v.UserID == c.UserID { + return "", backend.PluginError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeVoteInvalid), + ErrorContext: "user cannot vote on their own comment", + } + } + + // Prepare comment vote + receipt := p.identity.SignMessage([]byte(v.Signature)) + cv := comments.CommentVote{ + UserID: v.UserID, + Token: v.Token, + CommentID: v.CommentID, + Vote: v.Vote, + PublicKey: v.PublicKey, + Signature: v.Signature, + Timestamp: time.Now().Unix(), + Receipt: hex.EncodeToString(receipt[:]), + } + + // Save comment vote + digest, err := p.commentVoteSave(treeID, cv) + if err != nil { + return "", fmt.Errorf("commentVoteSave: %v", err) + } + + // Add vote to the comment index + votes, ok := cidx.Votes[cv.UserID] + if !ok { + votes = make([]voteIndex, 0, 1) + } + votes = append(votes, voteIndex{ + Vote: cv.Vote, + Digest: digest, + }) + cidx.Votes[cv.UserID] = votes + + // Update the record index + ridx.Comments[cv.CommentID] = cidx + + // Save index + err = p.recordIndexSaveLocked(token, *ridx) + if err != nil { + return "", err + } + + // Calculate the new vote scores + downvotes, upvotes := calcVoteScore(cidx) + + // Prepare reply + vr := comments.VoteReply{ + Downvotes: downvotes, + Upvotes: upvotes, + Timestamp: cv.Timestamp, + Receipt: cv.Receipt, + } + reply, err := json.Marshal(vr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *commentsPlugin) cmdGet(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdGet: %v %x %v", treeID, token, payload) + + // Decode payload + var g comments.Get + err := json.Unmarshal([]byte(payload), &g) + if err != nil { + return "", err + } + + // Get record index + ridx, err := p.recordIndex(token) + if err != nil { + return "", err + } + + // Get comments + cs, err := p.comments(treeID, *ridx, g.CommentIDs) + if err != nil { + return "", fmt.Errorf("comments: %v", err) + } + + // Prepare reply + gr := comments.GetReply{ + Comments: cs, + } + reply, err := json.Marshal(gr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *commentsPlugin) cmdGetAll(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdGetAll: %v %x %v", treeID, token, payload) + + // Decode payload + var ga comments.GetAll + err := json.Unmarshal([]byte(payload), &ga) + if err != nil { + return "", err + } + + // Compile comment IDs + ridx, err := p.recordIndex(token) + if err != nil { + return "", err + } + commentIDs := make([]uint32, 0, len(ridx.Comments)) + for k := range ridx.Comments { + commentIDs = append(commentIDs, k) + } + + // Get comments + c, err := p.comments(treeID, *ridx, commentIDs) + if err != nil { + return "", fmt.Errorf("comments: %v", err) + } + + // Convert comments from a map to a slice + cs := make([]comments.Comment, 0, len(c)) + for _, v := range c { + cs = append(cs, v) + } + + // Order comments by comment ID + sort.SliceStable(cs, func(i, j int) bool { + return cs[i].CommentID < cs[j].CommentID + }) + + // Prepare reply + gar := comments.GetAllReply{ + Comments: cs, + } + reply, err := json.Marshal(gar) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *commentsPlugin) cmdGetVersion(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdGetVersion: %v %x %v", treeID, token, payload) + + // Decode payload + var gv comments.GetVersion + err := json.Unmarshal([]byte(payload), &gv) + if err != nil { + return "", err + } + + // Get record index + ridx, err := p.recordIndex(token) + if err != nil { + return "", err + } + + // Verify comment exists + cidx, ok := ridx.Comments[gv.CommentID] + if !ok { + return "", backend.PluginError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeCommentNotFound), + } + } + if cidx.Del != nil { + return "", backend.PluginError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeCommentNotFound), + ErrorContext: "comment has been deleted", + } + } + digest, ok := cidx.Adds[gv.Version] + if !ok { + e := fmt.Sprintf("comment %v does not have version %v", + gv.CommentID, gv.Version) + return "", backend.PluginError{ + PluginID: comments.ID, + ErrorCode: int(comments.ErrorCodeCommentNotFound), + ErrorContext: e, + } + } + + // Get comment add record + adds, err := p.commentAdds(treeID, [][]byte{digest}) + if err != nil { + return "", fmt.Errorf("commentAdds: %v", err) + } + if len(adds) != 1 { + return "", fmt.Errorf("wrong comment adds count; got %v, want 1", + len(adds)) + } + + // Convert to a comment + c := convertCommentFromCommentAdd(adds[0]) + c.Downvotes, c.Upvotes = calcVoteScore(cidx) + + // Prepare reply + gvr := comments.GetVersionReply{ + Comment: c, + } + reply, err := json.Marshal(gvr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *commentsPlugin) cmdCount(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdCount: %v %x %v", treeID, token, payload) + + // Decode payload + var c comments.Count + err := json.Unmarshal([]byte(payload), &c) + if err != nil { + return "", err + } + + // Get record index + ridx, err := p.recordIndex(token) + if err != nil { + return "", err + } + + // Prepare reply + cr := comments.CountReply{ + Count: uint32(len(ridx.Comments)), + } + reply, err := json.Marshal(cr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *commentsPlugin) cmdVotes(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdVotes: %v %x %v", treeID, token, payload) + + // Decode payload + var v comments.Votes + err := json.Unmarshal([]byte(payload), &v) + if err != nil { + return "", err + } + + // Get record index + ridx, err := p.recordIndex(token) + if err != nil { + return "", err + } + + // Compile the comment vote digests for all votes that were cast + // by the specified user. + digests := make([][]byte, 0, 256) + for _, cidx := range ridx.Comments { + voteIdxs, ok := cidx.Votes[v.UserID] + if !ok { + // User has not cast any votes for this comment + continue + } + + // User has cast votes on this comment + for _, vidx := range voteIdxs { + digests = append(digests, vidx.Digest) + } + } + + // Lookup votes + votes, err := p.commentVotes(treeID, digests) + if err != nil { + return "", fmt.Errorf("commentVotes: %v", err) + } + + // Prepare reply + vr := comments.VotesReply{ + Votes: votes, + } + reply, err := json.Marshal(vr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *commentsPlugin) cmdTimestamps(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdTimestamps: %v %x %v", treeID, token, payload) + + // Decode payload + var t comments.Timestamps + err := json.Unmarshal([]byte(payload), &t) + if err != nil { + return "", err + } + + // Get record index + ridx, err := p.recordIndex(token) + if err != nil { + return "", err + } + + // If no comment IDs were given then we need to return the + // timestamps for all comments. + if len(t.CommentIDs) == 0 { + commentIDs := make([]uint32, 0, len(ridx.Comments)) + for k := range ridx.Comments { + commentIDs = append(commentIDs, k) + } + t.CommentIDs = commentIDs + } + + // Get timestamps + cmts := make(map[uint32][]comments.Timestamp, len(t.CommentIDs)) + votes := make(map[uint32][]comments.Timestamp, len(t.CommentIDs)) + for _, commentID := range t.CommentIDs { + cidx, ok := ridx.Comments[commentID] + if !ok { + // Comment ID does not exist. Skip it. + continue + } + + // Get timestamps for adds + ts := make([]comments.Timestamp, 0, len(cidx.Adds)+1) + for _, v := range cidx.Adds { + t, err := p.timestamp(treeID, v) + if err != nil { + return "", err + } + ts = append(ts, *t) + } + + // Get timestamp for del + if cidx.Del != nil { + t, err := p.timestamp(treeID, cidx.Del) + if err != nil { + return "", err + } + ts = append(ts, *t) + } + + // Save timestamps + cmts[commentID] = ts + + // Only get the comment vote timestamps if specified + if !t.IncludeVotes { + continue + } + + // Get timestamps for votes + ts = make([]comments.Timestamp, 0, len(cidx.Votes)) + for _, votes := range cidx.Votes { + for _, v := range votes { + t, err := p.timestamp(treeID, v.Digest) + if err != nil { + return "", err + } + ts = append(ts, *t) + } + } + + // Save timestamps + votes[commentID] = ts + } + + // Prepare reply + ts := comments.TimestampsReply{ + Comments: cmts, + Votes: votes, + } + reply, err := json.Marshal(ts) + if err != nil { + return "", err + } + + return string(reply), nil +} diff --git a/politeiad/backend/tlogbe/plugins/comments/comments.go b/politeiad/backend/tlogbe/plugins/comments/comments.go index f517eed42..4db7d7093 100644 --- a/politeiad/backend/tlogbe/plugins/comments/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments/comments.go @@ -5,25 +5,15 @@ package comments import ( - "bytes" - "encoding/base64" "encoding/hex" - "encoding/json" - "errors" - "fmt" "os" "path/filepath" - "sort" - "strconv" "sync" - "time" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" - "github.com/decred/politeia/politeiad/backend/tlogbe/store" "github.com/decred/politeia/politeiad/plugins/comments" - "github.com/decred/politeia/util" ) // TODO prevent duplicate comments @@ -84,1445 +74,7 @@ func (p *commentsPlugin) mutex(token []byte) *sync.Mutex { return m } -func tokenDecode(token string) ([]byte, error) { - return util.TokenDecode(util.TokenTypeTlog, token) -} - -func convertSignatureError(err error) backend.PluginError { - var e util.SignatureError - var s comments.ErrorCodeT - if errors.As(err, &e) { - switch e.ErrorCode { - case util.ErrorStatusPublicKeyInvalid: - s = comments.ErrorCodePublicKeyInvalid - case util.ErrorStatusSignatureInvalid: - s = comments.ErrorCodeSignatureInvalid - } - } - return backend.PluginError{ - PluginID: comments.ID, - ErrorCode: int(s), - ErrorContext: e.ErrorContext, - } -} - -func convertBlobEntryFromCommentAdd(c comments.CommentAdd) (*store.BlobEntry, error) { - data, err := json.Marshal(c) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorCommentAdd, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func convertBlobEntryFromCommentDel(c comments.CommentDel) (*store.BlobEntry, error) { - data, err := json.Marshal(c) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorCommentDel, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func convertBlobEntryFromCommentVote(c comments.CommentVote) (*store.BlobEntry, error) { - data, err := json.Marshal(c) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorCommentVote, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func convertCommentAddFromBlobEntry(be store.BlobEntry) (*comments.CommentAdd, error) { - // Decode and validate data hint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorCommentAdd { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorCommentAdd) - } - - // Decode data - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - digest, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, fmt.Errorf("decode digest: %v", err) - } - if !bytes.Equal(util.Digest(b), digest) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), digest) - } - var c comments.CommentAdd - err = json.Unmarshal(b, &c) - if err != nil { - return nil, fmt.Errorf("unmarshal CommentAdd: %v", err) - } - - return &c, nil -} - -func convertCommentDelFromBlobEntry(be store.BlobEntry) (*comments.CommentDel, error) { - // Decode and validate data hint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorCommentDel { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorCommentDel) - } - - // Decode data - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - digest, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, fmt.Errorf("decode digest: %v", err) - } - if !bytes.Equal(util.Digest(b), digest) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), digest) - } - var c comments.CommentDel - err = json.Unmarshal(b, &c) - if err != nil { - return nil, fmt.Errorf("unmarshal CommentDel: %v", err) - } - - return &c, nil -} - -func convertCommentVoteFromBlobEntry(be store.BlobEntry) (*comments.CommentVote, error) { - // Decode and validate data hint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorCommentVote { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorCommentVote) - } - - // Decode data - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - digest, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, fmt.Errorf("decode digest: %v", err) - } - if !bytes.Equal(util.Digest(b), digest) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), digest) - } - var cv comments.CommentVote - err = json.Unmarshal(b, &cv) - if err != nil { - return nil, fmt.Errorf("unmarshal CommentVote: %v", err) - } - - return &cv, nil -} - -func convertCommentFromCommentAdd(ca comments.CommentAdd) comments.Comment { - return comments.Comment{ - UserID: ca.UserID, - Token: ca.Token, - ParentID: ca.ParentID, - Comment: ca.Comment, - PublicKey: ca.PublicKey, - Signature: ca.Signature, - CommentID: ca.CommentID, - Version: ca.Version, - Timestamp: ca.Timestamp, - Receipt: ca.Receipt, - Downvotes: 0, // Not part of commentAdd data - Upvotes: 0, // Not part of commentAdd data - Deleted: false, - Reason: "", - ExtraData: ca.ExtraData, - ExtraDataHint: ca.ExtraDataHint, - } -} - -func convertCommentFromCommentDel(cd comments.CommentDel) comments.Comment { - // Score needs to be filled in separately - return comments.Comment{ - UserID: cd.UserID, - Token: cd.Token, - ParentID: cd.ParentID, - Comment: "", - Signature: "", - CommentID: cd.CommentID, - Version: 0, - Timestamp: cd.Timestamp, - Receipt: cd.Receipt, - Downvotes: 0, - Upvotes: 0, - Deleted: true, - Reason: cd.Reason, - } -} - -// commentVersionLatest returns the latest comment version. -func commentVersionLatest(cidx commentIndex) uint32 { - var maxVersion uint32 - for version := range cidx.Adds { - if version > maxVersion { - maxVersion = version - } - } - return maxVersion -} - -// commentExists returns whether the provided comment ID exists. -func commentExists(ridx recordIndex, commentID uint32) bool { - _, ok := ridx.Comments[commentID] - return ok -} - -// commentIDLatest returns the latest comment ID. -func commentIDLatest(idx recordIndex) uint32 { - var maxID uint32 - for id := range idx.Comments { - if id > maxID { - maxID = id - } - } - return maxID -} - -func (p *commentsPlugin) commentAddSave(treeID int64, ca comments.CommentAdd) ([]byte, error) { - be, err := convertBlobEntryFromCommentAdd(ca) - if err != nil { - return nil, err - } - d, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, err - } - err = p.tlog.BlobSave(treeID, dataTypeCommentAdd, *be) - if err != nil { - return nil, err - } - return d, nil -} - -// commentAdds returns the commentAdd for all specified digests. -func (p *commentsPlugin) commentAdds(treeID int64, digests [][]byte) ([]comments.CommentAdd, error) { - // Retrieve blobs - blobs, err := p.tlog.Blobs(treeID, digests) - if err != nil { - return nil, err - } - if len(blobs) != len(digests) { - notFound := make([]string, 0, len(blobs)) - for _, v := range digests { - m := hex.EncodeToString(v) - _, ok := blobs[m] - if !ok { - notFound = append(notFound, m) - } - } - return nil, fmt.Errorf("blobs not found: %v", notFound) - } - - // Decode blobs - adds := make([]comments.CommentAdd, 0, len(blobs)) - for _, v := range blobs { - c, err := convertCommentAddFromBlobEntry(v) - if err != nil { - return nil, err - } - adds = append(adds, *c) - } - - return adds, nil -} - -func (p *commentsPlugin) commentDelSave(treeID int64, cd comments.CommentDel) ([]byte, error) { - be, err := convertBlobEntryFromCommentDel(cd) - if err != nil { - return nil, err - } - d, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, err - } - err = p.tlog.BlobSave(treeID, dataTypeCommentDel, *be) - if err != nil { - return nil, err - } - return d, nil -} - -func (p *commentsPlugin) commentDels(treeID int64, digests [][]byte) ([]comments.CommentDel, error) { - // Retrieve blobs - blobs, err := p.tlog.Blobs(treeID, digests) - if err != nil { - return nil, err - } - if len(blobs) != len(digests) { - notFound := make([]string, 0, len(blobs)) - for _, v := range digests { - m := hex.EncodeToString(v) - _, ok := blobs[m] - if !ok { - notFound = append(notFound, m) - } - } - return nil, fmt.Errorf("blobs not found: %v", notFound) - } - - // Decode blobs - dels := make([]comments.CommentDel, 0, len(blobs)) - for _, v := range blobs { - d, err := convertCommentDelFromBlobEntry(v) - if err != nil { - return nil, err - } - dels = append(dels, *d) - } - - return dels, nil -} - -func (p *commentsPlugin) commentVoteSave(treeID int64, cv comments.CommentVote) ([]byte, error) { - be, err := convertBlobEntryFromCommentVote(cv) - if err != nil { - return nil, err - } - d, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, err - } - err = p.tlog.BlobSave(treeID, dataTypeCommentVote, *be) - if err != nil { - return nil, err - } - return d, nil -} - -func (p *commentsPlugin) commentVotes(treeID int64, digests [][]byte) ([]comments.CommentVote, error) { - // Retrieve blobs - blobs, err := p.tlog.Blobs(treeID, digests) - if err != nil { - return nil, err - } - if len(blobs) != len(digests) { - notFound := make([]string, 0, len(blobs)) - for _, v := range digests { - m := hex.EncodeToString(v) - _, ok := blobs[m] - if !ok { - notFound = append(notFound, m) - } - } - return nil, fmt.Errorf("blobs not found: %v", notFound) - } - - // Decode blobs - votes := make([]comments.CommentVote, 0, len(blobs)) - for _, v := range blobs { - c, err := convertCommentVoteFromBlobEntry(v) - if err != nil { - return nil, err - } - votes = append(votes, *c) - } - - return votes, nil -} - -// comments returns the most recent version of the specified comments. Deleted -// comments are returned with limited data. Comment IDs that do not correspond -// to an actual comment are not included in the returned map. It is the -// responsibility of the caller to ensure a comment is returned for each of the -// provided comment IDs. -func (p *commentsPlugin) comments(treeID int64, ridx recordIndex, commentIDs []uint32) (map[uint32]comments.Comment, error) { - // Aggregate the digests for all records that need to be looked up. - // If a comment has been deleted then the only record that will - // still exist is the comment del record. If the comment has not - // been deleted then the comment add record will need to be - // retrieved for the latest version of the comment. - var ( - digestAdds = make([][]byte, 0, len(commentIDs)) - digestDels = make([][]byte, 0, len(commentIDs)) - ) - for _, v := range commentIDs { - cidx, ok := ridx.Comments[v] - if !ok { - // Comment does not exist - continue - } - - // Comment del record - if cidx.Del != nil { - digestDels = append(digestDels, cidx.Del) - continue - } - - // Comment add record - version := commentVersionLatest(cidx) - digestAdds = append(digestAdds, cidx.Adds[version]) - } - - // Get comment add records - adds, err := p.commentAdds(treeID, digestAdds) - if err != nil { - return nil, fmt.Errorf("commentAdds: %v", err) - } - if len(adds) != len(digestAdds) { - return nil, fmt.Errorf("wrong comment adds count; got %v, want %v", - len(adds), len(digestAdds)) - } - - // Get comment del records - dels, err := p.commentDels(treeID, digestDels) - if err != nil { - return nil, fmt.Errorf("commentDels: %v", err) - } - if len(dels) != len(digestDels) { - return nil, fmt.Errorf("wrong comment dels count; got %v, want %v", - len(dels), len(digestDels)) - } - - // Prepare comments - cs := make(map[uint32]comments.Comment, len(commentIDs)) - for _, v := range adds { - c := convertCommentFromCommentAdd(v) - cidx, ok := ridx.Comments[c.CommentID] - if !ok { - return nil, fmt.Errorf("comment index not found %v", c.CommentID) - } - c.Downvotes, c.Upvotes = calcVoteScore(cidx) - cs[v.CommentID] = c - } - for _, v := range dels { - c := convertCommentFromCommentDel(v) - cs[v.CommentID] = c - } - - return cs, nil -} - -// comment returns the latest version of the provided comment. -func (p *commentsPlugin) comment(treeID int64, ridx recordIndex, commentID uint32) (*comments.Comment, error) { - cs, err := p.comments(treeID, ridx, []uint32{commentID}) - if err != nil { - return nil, fmt.Errorf("comments: %v", err) - } - c, ok := cs[commentID] - if !ok { - return nil, fmt.Errorf("comment not found") - } - return &c, nil -} - -func (p *commentsPlugin) timestamp(treeID int64, digest []byte) (*comments.Timestamp, error) { - // Get timestamp - t, err := p.tlog.Timestamp(treeID, digest) - if err != nil { - return nil, err - } - - // Convert response - proofs := make([]comments.Proof, 0, len(t.Proofs)) - for _, v := range t.Proofs { - proofs = append(proofs, comments.Proof{ - Type: v.Type, - Digest: v.Digest, - MerkleRoot: v.MerkleRoot, - MerklePath: v.MerklePath, - ExtraData: v.ExtraData, - }) - } - return &comments.Timestamp{ - Data: t.Data, - Digest: t.Digest, - TxID: t.TxID, - MerkleRoot: t.MerkleRoot, - Proofs: proofs, - }, nil -} - -// calcVoteScore returns the vote score for the provided comment index. The -// returned values are the downvotes and upvotes, respectively. -func calcVoteScore(cidx commentIndex) (uint64, uint64) { - // Find the vote score by replaying all existing votes from all - // users. The net effect of a new vote on a comment score depends - // on the previous vote from that uuid. Example, a user upvotes a - // comment that they have already upvoted, the resulting vote score - // is 0 due to the second upvote removing the original upvote. - var upvotes uint64 - var downvotes uint64 - for _, votes := range cidx.Votes { - // Calculate the vote score that this user is contributing. This - // can only ever be -1, 0, or 1. - var score int64 - for _, v := range votes { - vote := int64(v.Vote) - switch { - case score == 0: - // No previous vote. New vote becomes the score. - score = vote - - case score == vote: - // New vote is the same as the previous vote. The vote gets - // removed from the score, making the score 0. - score = 0 - - case score != vote: - // New vote is different than the previous vote. New vote - // becomes the score. - score = vote - } - } - - // Add the net result of all votes from this user to the totals. - switch score { - case 0: - // Nothing to do - case -1: - downvotes++ - case 1: - upvotes++ - default: - // Something went wrong - e := fmt.Errorf("unexpected vote score %v", score) - panic(e) - } - } - - return downvotes, upvotes -} - -func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdNew: %v %x %v", treeID, token, payload) - - // Decode payload - var n comments.New - err := json.Unmarshal([]byte(payload), &n) - if err != nil { - return "", err - } - - // Verify token - t, err := tokenDecode(n.Token) - if err != nil { - return "", backend.PluginError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorCodeTokenInvalid), - ErrorContext: err.Error(), - } - } - if !bytes.Equal(t, token) { - e := fmt.Sprintf("comment token does not match route token: "+ - "got %x, want %x", t, token) - return "", backend.PluginError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorCodeTokenInvalid), - ErrorContext: e, - } - } - - // Verify signature - msg := n.Token + strconv.FormatUint(uint64(n.ParentID), 10) + n.Comment - err = util.VerifySignature(n.Signature, n.PublicKey, msg) - if err != nil { - return "", convertSignatureError(err) - } - - // Verify comment - if len(n.Comment) > comments.PolicyCommentLengthMax { - return "", backend.PluginError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorCodeCommentTextInvalid), - ErrorContext: "exceeds max length", - } - } - - // The record index must be pulled and updated. The record lock - // must be held for the remainder of this function. - m := p.mutex(token) - m.Lock() - defer m.Unlock() - - // Get record index - ridx, err := p.recordIndexLocked(token) - if err != nil { - return "", err - } - - // Verify parent comment exists if set. A parent ID of 0 means that - // this is a base level comment, not a reply to another comment. - if n.ParentID > 0 && !commentExists(*ridx, n.ParentID) { - return "", backend.PluginError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorCodeParentIDInvalid), - ErrorContext: "parent ID comment not found", - } - } - - // Setup comment - receipt := p.identity.SignMessage([]byte(n.Signature)) - ca := comments.CommentAdd{ - UserID: n.UserID, - Token: n.Token, - ParentID: n.ParentID, - Comment: n.Comment, - PublicKey: n.PublicKey, - Signature: n.Signature, - CommentID: commentIDLatest(*ridx) + 1, - Version: 1, - Timestamp: time.Now().Unix(), - Receipt: hex.EncodeToString(receipt[:]), - ExtraData: n.ExtraData, - ExtraDataHint: n.ExtraDataHint, - } - - // Save comment - digest, err := p.commentAddSave(treeID, ca) - if err != nil { - return "", fmt.Errorf("commentAddSave: %v", err) - } - - // Update index - ridx.Comments[ca.CommentID] = commentIndex{ - Adds: map[uint32][]byte{ - 1: digest, - }, - Del: nil, - Votes: make(map[string][]voteIndex), - } - - // Save index - err = p.recordIndexSaveLocked(token, *ridx) - if err != nil { - return "", err - } - - log.Debugf("Comment saved to record %v comment ID %v", - ca.Token, ca.CommentID) - - // Return new comment - c, err := p.comment(treeID, *ridx, ca.CommentID) - if err != nil { - return "", fmt.Errorf("comment %x %v: %v", token, ca.CommentID, err) - } - - // Prepare reply - nr := comments.NewReply{ - Comment: *c, - } - reply, err := json.Marshal(nr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdEdit: %v %x %v", treeID, token, payload) - - // Decode payload - var e comments.Edit - err := json.Unmarshal([]byte(payload), &e) - if err != nil { - return "", err - } - - // Verify token - t, err := tokenDecode(e.Token) - if err != nil { - return "", backend.PluginError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorCodeTokenInvalid), - ErrorContext: err.Error(), - } - } - if !bytes.Equal(t, token) { - e := fmt.Sprintf("comment token does not match route token: "+ - "got %x, want %x", t, token) - return "", backend.PluginError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorCodeTokenInvalid), - ErrorContext: e, - } - } - - // Verify signature - msg := e.Token + strconv.FormatUint(uint64(e.ParentID), 10) + e.Comment - err = util.VerifySignature(e.Signature, e.PublicKey, msg) - if err != nil { - return "", convertSignatureError(err) - } - - // Verify comment - if len(e.Comment) > comments.PolicyCommentLengthMax { - return "", backend.PluginError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorCodeCommentTextInvalid), - ErrorContext: "exceeds max length", - } - } - - // The record index must be pulled and updated. The record lock - // must be held for the remainder of this function. - m := p.mutex(token) - m.Lock() - defer m.Unlock() - - // Get record index - ridx, err := p.recordIndexLocked(token) - if err != nil { - return "", err - } - - // Get the existing comment - cs, err := p.comments(treeID, *ridx, []uint32{e.CommentID}) - if err != nil { - return "", fmt.Errorf("comments %v: %v", e.CommentID, err) - } - existing, ok := cs[e.CommentID] - if !ok { - return "", backend.PluginError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorCodeCommentNotFound), - } - } - - // Verify the user ID - if e.UserID != existing.UserID { - return "", backend.PluginError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorCodeUserUnauthorized), - } - } - - // Verify the parent ID - if e.ParentID != existing.ParentID { - e := fmt.Sprintf("parent id cannot change; got %v, want %v", - e.ParentID, existing.ParentID) - return "", backend.PluginError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorCodeParentIDInvalid), - ErrorContext: e, - } - } - - // Verify comment changes - if e.Comment == existing.Comment { - return "", backend.PluginError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorCodeCommentTextInvalid), - ErrorContext: "comment did not change", - } - } - - // Create a new comment version - receipt := p.identity.SignMessage([]byte(e.Signature)) - ca := comments.CommentAdd{ - UserID: e.UserID, - Token: e.Token, - ParentID: e.ParentID, - Comment: e.Comment, - PublicKey: e.PublicKey, - Signature: e.Signature, - CommentID: e.CommentID, - Version: existing.Version + 1, - Timestamp: time.Now().Unix(), - Receipt: hex.EncodeToString(receipt[:]), - ExtraData: e.ExtraData, - ExtraDataHint: e.ExtraDataHint, - } - - // Save comment - digest, err := p.commentAddSave(treeID, ca) - if err != nil { - return "", fmt.Errorf("commentAddSave: %v", err) - } - - // Update index - ridx.Comments[ca.CommentID].Adds[ca.Version] = digest - - // Save index - err = p.recordIndexSaveLocked(token, *ridx) - if err != nil { - return "", err - } - - log.Debugf("Comment edited on record %v comment ID %v", - ca.Token, ca.CommentID) - - // Return updated comment - c, err := p.comment(treeID, *ridx, e.CommentID) - if err != nil { - return "", fmt.Errorf("comment %x %v: %v", token, e.CommentID, err) - } - - // Prepare reply - er := comments.EditReply{ - Comment: *c, - } - reply, err := json.Marshal(er) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdDel: %v %x %v", treeID, token, payload) - - // Decode payload - var d comments.Del - err := json.Unmarshal([]byte(payload), &d) - if err != nil { - return "", err - } - - // Verify token - t, err := tokenDecode(d.Token) - if err != nil { - return "", backend.PluginError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorCodeTokenInvalid), - ErrorContext: err.Error(), - } - } - if !bytes.Equal(t, token) { - e := fmt.Sprintf("comment token does not match route token: "+ - "got %x, want %x", t, token) - return "", backend.PluginError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorCodeTokenInvalid), - ErrorContext: e, - } - } - - // Verify signature - msg := d.Token + strconv.FormatUint(uint64(d.CommentID), 10) + d.Reason - err = util.VerifySignature(d.Signature, d.PublicKey, msg) - if err != nil { - return "", convertSignatureError(err) - } - - // The record index must be pulled and updated. The record lock - // must be held for the remainder of this function. - m := p.mutex(token) - m.Lock() - defer m.Unlock() - - // Get record index - ridx, err := p.recordIndexLocked(token) - if err != nil { - return "", err - } - - // Get the existing comment - cs, err := p.comments(treeID, *ridx, []uint32{d.CommentID}) - if err != nil { - return "", fmt.Errorf("comments %v: %v", d.CommentID, err) - } - existing, ok := cs[d.CommentID] - if !ok { - return "", backend.PluginError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorCodeCommentNotFound), - } - } - - // Prepare comment delete - receipt := p.identity.SignMessage([]byte(d.Signature)) - cd := comments.CommentDel{ - Token: d.Token, - CommentID: d.CommentID, - Reason: d.Reason, - PublicKey: d.PublicKey, - Signature: d.Signature, - ParentID: existing.ParentID, - UserID: existing.UserID, - Timestamp: time.Now().Unix(), - Receipt: hex.EncodeToString(receipt[:]), - } - - // Save comment del - digest, err := p.commentDelSave(treeID, cd) - if err != nil { - return "", fmt.Errorf("commentDelSave: %v", err) - } - - // Update index - cidx, ok := ridx.Comments[d.CommentID] - if !ok { - // This should not be possible - e := fmt.Sprintf("comment not found in index: %v", d.CommentID) - panic(e) - } - cidx.Del = digest - ridx.Comments[d.CommentID] = cidx - - // Save index - err = p.recordIndexSaveLocked(token, *ridx) - if err != nil { - return "", err - } - - // Delete all comment versions - digests := make([][]byte, 0, len(cidx.Adds)) - for _, v := range cidx.Adds { - digests = append(digests, v) - } - err = p.tlog.BlobsDel(treeID, digests) - if err != nil { - return "", fmt.Errorf("del: %v", err) - } - - // Return updated comment - c, err := p.comment(treeID, *ridx, d.CommentID) - if err != nil { - return "", fmt.Errorf("comment %v: %v", d.CommentID, err) - } - - // Prepare reply - dr := comments.DelReply{ - Comment: *c, - } - reply, err := json.Marshal(dr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdVote: %v %x %v", treeID, token, payload) - - // Decode payload - var v comments.Vote - err := json.Unmarshal([]byte(payload), &v) - if err != nil { - return "", err - } - - // Verify token - t, err := tokenDecode(v.Token) - if err != nil { - return "", backend.PluginError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorCodeTokenInvalid), - ErrorContext: err.Error(), - } - } - if !bytes.Equal(t, token) { - e := fmt.Sprintf("comment token does not match route token: "+ - "got %x, want %x", t, token) - return "", backend.PluginError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorCodeTokenInvalid), - ErrorContext: e, - } - } - - // Verify vote - switch v.Vote { - case comments.VoteDownvote, comments.VoteUpvote: - // These are allowed - default: - return "", backend.PluginError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorCodeVoteInvalid), - } - } - - // Verify signature - msg := v.Token + strconv.FormatUint(uint64(v.CommentID), 10) + - strconv.FormatInt(int64(v.Vote), 10) - err = util.VerifySignature(v.Signature, v.PublicKey, msg) - if err != nil { - return "", convertSignatureError(err) - } - - // The record index must be pulled and updated. The record lock - // must be held for the remainder of this function. - m := p.mutex(token) - m.Lock() - defer m.Unlock() - - // Get record index - ridx, err := p.recordIndexLocked(token) - if err != nil { - return "", err - } - - // Verify comment exists - cidx, ok := ridx.Comments[v.CommentID] - if !ok { - return "", backend.PluginError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorCodeCommentNotFound), - } - } - - // Verify user has not exceeded max allowed vote changes - uvotes, ok := cidx.Votes[v.UserID] - if !ok { - uvotes = make([]voteIndex, 0) - } - if len(uvotes) > comments.PolicyVoteChangesMax { - return "", backend.PluginError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorCodeVoteChangesMax), - } - } - - // Verify user is not voting on their own comment - cs, err := p.comments(treeID, *ridx, []uint32{v.CommentID}) - if err != nil { - return "", fmt.Errorf("comments %v: %v", v.CommentID, err) - } - c, ok := cs[v.CommentID] - if !ok { - return "", fmt.Errorf("comment not found %v", v.CommentID) - } - if v.UserID == c.UserID { - return "", backend.PluginError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorCodeVoteInvalid), - ErrorContext: "user cannot vote on their own comment", - } - } - - // Prepare comment vote - receipt := p.identity.SignMessage([]byte(v.Signature)) - cv := comments.CommentVote{ - UserID: v.UserID, - Token: v.Token, - CommentID: v.CommentID, - Vote: v.Vote, - PublicKey: v.PublicKey, - Signature: v.Signature, - Timestamp: time.Now().Unix(), - Receipt: hex.EncodeToString(receipt[:]), - } - - // Save comment vote - digest, err := p.commentVoteSave(treeID, cv) - if err != nil { - return "", fmt.Errorf("commentVoteSave: %v", err) - } - - // Add vote to the comment index - votes, ok := cidx.Votes[cv.UserID] - if !ok { - votes = make([]voteIndex, 0, 1) - } - votes = append(votes, voteIndex{ - Vote: cv.Vote, - Digest: digest, - }) - cidx.Votes[cv.UserID] = votes - - // Update the record index - ridx.Comments[cv.CommentID] = cidx - - // Save index - err = p.recordIndexSaveLocked(token, *ridx) - if err != nil { - return "", err - } - - // Calculate the new vote scores - downvotes, upvotes := calcVoteScore(cidx) - - // Prepare reply - vr := comments.VoteReply{ - Downvotes: downvotes, - Upvotes: upvotes, - Timestamp: cv.Timestamp, - Receipt: cv.Receipt, - } - reply, err := json.Marshal(vr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *commentsPlugin) cmdGet(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdGet: %v %x %v", treeID, token, payload) - - // Decode payload - var g comments.Get - err := json.Unmarshal([]byte(payload), &g) - if err != nil { - return "", err - } - - // Get record index - ridx, err := p.recordIndex(token) - if err != nil { - return "", err - } - - // Get comments - cs, err := p.comments(treeID, *ridx, g.CommentIDs) - if err != nil { - return "", fmt.Errorf("comments: %v", err) - } - - // Prepare reply - gr := comments.GetReply{ - Comments: cs, - } - reply, err := json.Marshal(gr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *commentsPlugin) cmdGetAll(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdGetAll: %v %x %v", treeID, token, payload) - - // Decode payload - var ga comments.GetAll - err := json.Unmarshal([]byte(payload), &ga) - if err != nil { - return "", err - } - - // Compile comment IDs - ridx, err := p.recordIndex(token) - if err != nil { - return "", err - } - commentIDs := make([]uint32, 0, len(ridx.Comments)) - for k := range ridx.Comments { - commentIDs = append(commentIDs, k) - } - - // Get comments - c, err := p.comments(treeID, *ridx, commentIDs) - if err != nil { - return "", fmt.Errorf("comments: %v", err) - } - - // Convert comments from a map to a slice - cs := make([]comments.Comment, 0, len(c)) - for _, v := range c { - cs = append(cs, v) - } - - // Order comments by comment ID - sort.SliceStable(cs, func(i, j int) bool { - return cs[i].CommentID < cs[j].CommentID - }) - - // Prepare reply - gar := comments.GetAllReply{ - Comments: cs, - } - reply, err := json.Marshal(gar) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *commentsPlugin) cmdGetVersion(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdGetVersion: %v %x %v", treeID, token, payload) - - // Decode payload - var gv comments.GetVersion - err := json.Unmarshal([]byte(payload), &gv) - if err != nil { - return "", err - } - - // Get record index - ridx, err := p.recordIndex(token) - if err != nil { - return "", err - } - - // Verify comment exists - cidx, ok := ridx.Comments[gv.CommentID] - if !ok { - return "", backend.PluginError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorCodeCommentNotFound), - } - } - if cidx.Del != nil { - return "", backend.PluginError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorCodeCommentNotFound), - ErrorContext: "comment has been deleted", - } - } - digest, ok := cidx.Adds[gv.Version] - if !ok { - e := fmt.Sprintf("comment %v does not have version %v", - gv.CommentID, gv.Version) - return "", backend.PluginError{ - PluginID: comments.ID, - ErrorCode: int(comments.ErrorCodeCommentNotFound), - ErrorContext: e, - } - } - - // Get comment add record - adds, err := p.commentAdds(treeID, [][]byte{digest}) - if err != nil { - return "", fmt.Errorf("commentAdds: %v", err) - } - if len(adds) != 1 { - return "", fmt.Errorf("wrong comment adds count; got %v, want 1", - len(adds)) - } - - // Convert to a comment - c := convertCommentFromCommentAdd(adds[0]) - c.Downvotes, c.Upvotes = calcVoteScore(cidx) - - // Prepare reply - gvr := comments.GetVersionReply{ - Comment: c, - } - reply, err := json.Marshal(gvr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *commentsPlugin) cmdCount(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdCount: %v %x %v", treeID, token, payload) - - // Decode payload - var c comments.Count - err := json.Unmarshal([]byte(payload), &c) - if err != nil { - return "", err - } - - // Get record index - ridx, err := p.recordIndex(token) - if err != nil { - return "", err - } - - // Prepare reply - cr := comments.CountReply{ - Count: uint32(len(ridx.Comments)), - } - reply, err := json.Marshal(cr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *commentsPlugin) cmdVotes(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdVotes: %v %x %v", treeID, token, payload) - - // Decode payload - var v comments.Votes - err := json.Unmarshal([]byte(payload), &v) - if err != nil { - return "", err - } - - // Get record index - ridx, err := p.recordIndex(token) - if err != nil { - return "", err - } - - // Compile the comment vote digests for all votes that were cast - // by the specified user. - digests := make([][]byte, 0, 256) - for _, cidx := range ridx.Comments { - voteIdxs, ok := cidx.Votes[v.UserID] - if !ok { - // User has not cast any votes for this comment - continue - } - - // User has cast votes on this comment - for _, vidx := range voteIdxs { - digests = append(digests, vidx.Digest) - } - } - - // Lookup votes - votes, err := p.commentVotes(treeID, digests) - if err != nil { - return "", fmt.Errorf("commentVotes: %v", err) - } - - // Prepare reply - vr := comments.VotesReply{ - Votes: votes, - } - reply, err := json.Marshal(vr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *commentsPlugin) cmdTimestamps(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdTimestamps: %v %x %v", treeID, token, payload) - - // Decode payload - var t comments.Timestamps - err := json.Unmarshal([]byte(payload), &t) - if err != nil { - return "", err - } - - // Get record index - ridx, err := p.recordIndex(token) - if err != nil { - return "", err - } - - // If no comment IDs were given then we need to return the - // timestamps for all comments. - if len(t.CommentIDs) == 0 { - commentIDs := make([]uint32, 0, len(ridx.Comments)) - for k := range ridx.Comments { - commentIDs = append(commentIDs, k) - } - t.CommentIDs = commentIDs - } - - // Get timestamps - cmts := make(map[uint32][]comments.Timestamp, len(t.CommentIDs)) - votes := make(map[uint32][]comments.Timestamp, len(t.CommentIDs)) - for _, commentID := range t.CommentIDs { - cidx, ok := ridx.Comments[commentID] - if !ok { - // Comment ID does not exist. Skip it. - continue - } - - // Get timestamps for adds - ts := make([]comments.Timestamp, 0, len(cidx.Adds)+1) - for _, v := range cidx.Adds { - t, err := p.timestamp(treeID, v) - if err != nil { - return "", err - } - ts = append(ts, *t) - } - - // Get timestamp for del - if cidx.Del != nil { - t, err := p.timestamp(treeID, cidx.Del) - if err != nil { - return "", err - } - ts = append(ts, *t) - } - - // Save timestamps - cmts[commentID] = ts - - // Only get the comment vote timestamps if specified - if !t.IncludeVotes { - continue - } - - // Get timestamps for votes - ts = make([]comments.Timestamp, 0, len(cidx.Votes)) - for _, votes := range cidx.Votes { - for _, v := range votes { - t, err := p.timestamp(treeID, v.Digest) - if err != nil { - return "", err - } - ts = append(ts, *t) - } - } - - // Save timestamps - votes[commentID] = ts - } - - // Prepare reply - ts := comments.TimestampsReply{ - Comments: cmts, - Votes: votes, - } - reply, err := json.Marshal(ts) - if err != nil { - return "", err - } - - return string(reply), nil -} - -// Setup performs any plugin setup work that needs to be done. +// Setup performs any plugin setup that is required. // // This function satisfies the plugins.Client interface. func (p *commentsPlugin) Setup() error { @@ -1575,10 +127,10 @@ func (p *commentsPlugin) Hook(treeID int64, token []byte, h plugins.HookT, paylo // Fsck performs a plugin filesystem check. // // This function satisfies the plugins.Client interface. -func (p *commentsPlugin) Fsck() error { +func (p *commentsPlugin) Fsck(treeIDs []int64) error { log.Tracef("Fsck") - // TODO Make sure CommentDel blobs were actually deleted + // Verify CommentDel blobs were actually deleted return nil } diff --git a/politeiad/backend/tlogbe/plugins/comments/indexes.go b/politeiad/backend/tlogbe/plugins/comments/recordindex.go similarity index 100% rename from politeiad/backend/tlogbe/plugins/comments/indexes.go rename to politeiad/backend/tlogbe/plugins/comments/recordindex.go diff --git a/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go b/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go index 0f2de6235..e2aed877a 100644 --- a/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go +++ b/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go @@ -574,7 +574,7 @@ func (p *dcrdataPlugin) websocketSetup() { go p.websocketMonitor() } -// Setup performs any plugin setup work that needs to be done. +// Setup performs any plugin setup that is required. // // This function satisfies the plugins.Client interface. func (p *dcrdataPlugin) Setup() error { @@ -621,7 +621,7 @@ func (p *dcrdataPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payloa // Fsck performs a plugin filesystem check. // // This function satisfies the plugins.Client interface. -func (p *dcrdataPlugin) Fsck() error { +func (p *dcrdataPlugin) Fsck(treeIDs []int64) error { log.Tracef("Fsck") return nil diff --git a/politeiad/backend/tlogbe/plugins/pi/cmds.go b/politeiad/backend/tlogbe/plugins/pi/cmds.go new file mode 100644 index 000000000..29b8fefa1 --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/pi/cmds.go @@ -0,0 +1,133 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package pi + +import ( + "encoding/json" + "fmt" + + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/plugins/pi" + "github.com/decred/politeia/politeiad/plugins/ticketvote" + "github.com/decred/politeia/util" +) + +func tokenDecode(token string) ([]byte, error) { + return util.TokenDecode(util.TokenTypeTlog, token) +} + +func convertPropStatusFromMDStatus(s backend.MDStatusT) pi.PropStatusT { + var status pi.PropStatusT + switch s { + case backend.MDStatusUnvetted, backend.MDStatusIterationUnvetted: + status = pi.PropStatusUnvetted + case backend.MDStatusVetted: + status = pi.PropStatusPublic + case backend.MDStatusCensored: + status = pi.PropStatusCensored + case backend.MDStatusArchived: + status = pi.PropStatusAbandoned + } + return status +} + +func (p *piPlugin) cmdProposalInv() (string, error) { + log.Tracef("cmdProposalInv") + + // Get full record inventory + ibs, err := p.backend.InventoryByStatus() + if err != nil { + return "", err + } + + // Convert MDStatus keys to human readable proposal statuses + unvetted := make(map[string][]string, len(ibs.Unvetted)) + vetted := make(map[string][]string, len(ibs.Vetted)) + for k, v := range ibs.Unvetted { + s := pi.PropStatuses[convertPropStatusFromMDStatus(k)] + unvetted[s] = v + } + for k, v := range ibs.Vetted { + s := pi.PropStatuses[convertPropStatusFromMDStatus(k)] + vetted[s] = v + } + + // Prepare reply + pir := pi.ProposalInvReply{ + Unvetted: unvetted, + Vetted: vetted, + } + reply, err := json.Marshal(pir) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *piPlugin) voteSummary(token []byte) (*ticketvote.VoteSummary, error) { + reply, err := p.backend.VettedPluginCmd(token, + ticketvote.ID, ticketvote.CmdSummary, "") + if err != nil { + return nil, err + } + var sr ticketvote.SummaryReply + err = json.Unmarshal([]byte(reply), &sr) + if err != nil { + return nil, err + } + return &sr.Summary, nil +} + +func (p *piPlugin) cmdVoteInv() (string, error) { + // Get ticketvote inventory + r, err := p.backend.VettedPluginCmd([]byte{}, + ticketvote.ID, ticketvote.CmdInventory, "") + if err != nil { + return "", fmt.Errorf("VettedPluginCmd %v %v: %v", + ticketvote.ID, ticketvote.CmdInventory, err) + } + var ir ticketvote.InventoryReply + err = json.Unmarshal([]byte(r), &ir) + if err != nil { + return "", err + } + + // Get vote summaries for all finished proposal votes and + // categorize by approved/rejected. + approved := make([]string, 0, len(ir.Finished)) + rejected := make([]string, 0, len(ir.Finished)) + for _, v := range ir.Finished { + t, err := tokenDecode(v) + if err != nil { + return "", err + } + vs, err := p.voteSummary(t) + if err != nil { + return "", err + } + if vs.Approved { + approved = append(approved, v) + } else { + rejected = append(rejected, v) + } + } + + // Prepare reply + vir := pi.VoteInventoryReply{ + Unauthorized: ir.Unauthorized, + Authorized: ir.Authorized, + Started: ir.Started, + Approved: approved, + Rejected: rejected, + BestBlock: ir.BestBlock, + } + reply, err := json.Marshal(vir) + if err != nil { + return "", err + } + + return string(reply), nil +} diff --git a/politeiad/backend/tlogbe/plugins/pi/hooks.go b/politeiad/backend/tlogbe/plugins/pi/hooks.go index c2180e955..2f8c39344 100644 --- a/politeiad/backend/tlogbe/plugins/pi/hooks.go +++ b/politeiad/backend/tlogbe/plugins/pi/hooks.go @@ -41,25 +41,6 @@ func proposalMetadataDecode(files []backend.File) (*pi.ProposalMetadata, error) return propMD, nil } -// generalMetadataDecode decodes and returns the GeneralMetadata from the -// provided backend metadata streams. If a GeneralMetadata is not found, nil is -// returned. -func generalMetadataDecode(metadata []backend.MetadataStream) (*pi.GeneralMetadata, error) { - var generalMD *pi.GeneralMetadata - for _, v := range metadata { - if v.ID == pi.MDStreamIDGeneralMetadata { - var gm pi.GeneralMetadata - err := json.Unmarshal([]byte(v.Payload), &gm) - if err != nil { - return nil, err - } - generalMD = &gm - break - } - } - return generalMD, nil -} - // statusChangesDecode decodes a JSON byte slice into a []StatusChange slice. func statusChangesDecode(payload []byte) ([]pi.StatusChange, error) { var statuses []pi.StatusChange @@ -99,31 +80,6 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { return nil } -func (p *piPlugin) hookNewRecordPost(payload string) error { - var nr plugins.HookNewRecordPost - err := json.Unmarshal([]byte(payload), &nr) - if err != nil { - return err - } - - // Decode GeneralMetadata - gm, err := generalMetadataDecode(nr.Metadata) - if err != nil { - return err - } - if gm == nil { - panic("general metadata not found") - } - - // Add token to the user data cache - err = p.userDataAddToken(gm.UserID, nr.RecordMetadata.Token) - if err != nil { - return err - } - - return nil -} - func (p *piPlugin) hookEditRecordPre(payload string) error { /* var er plugins.HookEditRecord @@ -181,67 +137,69 @@ func (p *piPlugin) hookSetRecordStatusPost(payload string) error { return err } - // Parse the status change metadata - var sc *pi.StatusChange - for _, v := range srs.MDAppend { - if v.ID != pi.MDStreamIDStatusChanges { - continue - } + /* + // Parse the status change metadata + var sc *pi.StatusChange + for _, v := range srs.MDAppend { + if v.ID != pi.MDStreamIDStatusChanges { + continue + } - var sc pi.StatusChange - err := json.Unmarshal([]byte(v.Payload), &sc) - if err != nil { - return err + var sc pi.StatusChange + err := json.Unmarshal([]byte(v.Payload), &sc) + if err != nil { + return err + } + break } - break - } - if sc == nil { - return fmt.Errorf("status change append metadata not found") - } - - // Parse the existing status changes metadata stream - var statuses []pi.StatusChange - for _, v := range srs.Current.Metadata { - if v.ID != pi.MDStreamIDStatusChanges { - continue + if sc == nil { + return fmt.Errorf("status change append metadata not found") } - statuses, err = statusChangesDecode([]byte(v.Payload)) - if err != nil { - return err + // Parse the existing status changes metadata stream + var statuses []pi.StatusChange + for _, v := range srs.Current.Metadata { + if v.ID != pi.MDStreamIDStatusChanges { + continue + } + + statuses, err = statusChangesDecode([]byte(v.Payload)) + if err != nil { + return err + } + break } - break - } - // Verify version is the latest version - if sc.Version != srs.Current.Version { - e := fmt.Sprintf("version not current: got %v, want %v", - sc.Version, srs.Current.Version) - return backend.PluginError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorCodePropVersionInvalid), - ErrorContext: e, + // Verify version is the latest version + if sc.Version != srs.Current.Version { + e := fmt.Sprintf("version not current: got %v, want %v", + sc.Version, srs.Current.Version) + return backend.PluginError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorCodePropVersionInvalid), + ErrorContext: e, + } } - } - // Verify status change is allowed - var from pi.PropStatusT - if len(statuses) == 0 { - // No previous status changes exist. Proposal is unvetted. - from = pi.PropStatusUnvetted - } else { - from = statuses[len(statuses)-1].Status - } - _, isAllowed := pi.StatusChanges[from][sc.Status] - if !isAllowed { - e := fmt.Sprintf("from %v to %v status change not allowed", - from, sc.Status) - return backend.PluginError{ - PluginID: pi.ID, - ErrorCode: int(pi.ErrorCodePropStatusChangeInvalid), - ErrorContext: e, + // Verify status change is allowed + var from pi.PropStatusT + if len(statuses) == 0 { + // No previous status changes exist. Proposal is unvetted. + from = pi.PropStatusUnvetted + } else { + from = statuses[len(statuses)-1].Status } - } + _, isAllowed := pi.StatusChanges[from][sc.Status] + if !isAllowed { + e := fmt.Sprintf("from %v to %v status change not allowed", + from, sc.Status) + return backend.PluginError{ + PluginID: pi.ID, + ErrorCode: int(pi.ErrorCodePropStatusChangeInvalid), + ErrorContext: e, + } + } + */ return nil } diff --git a/politeiad/backend/tlogbe/plugins/pi/pi.go b/politeiad/backend/tlogbe/plugins/pi/pi.go index 3b82250d0..d32700957 100644 --- a/politeiad/backend/tlogbe/plugins/pi/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi/pi.go @@ -5,18 +5,13 @@ package pi import ( - "encoding/json" - "fmt" "os" "path/filepath" "sync" - "github.com/decred/dcrd/chaincfg/v3" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/politeiad/plugins/pi" - "github.com/decred/politeia/politeiad/plugins/ticketvote" - "github.com/decred/politeia/util" ) var ( @@ -26,9 +21,7 @@ var ( // piPlugin satisfies the plugins.Client interface. type piPlugin struct { sync.Mutex - backend backend.Backend - tlog plugins.TlogClient - activeNetParams *chaincfg.Params + backend backend.Backend // dataDir is the pi plugin data directory. The only data that is // stored here is cached data that can be re-created at any time @@ -36,191 +29,7 @@ type piPlugin struct { dataDir string } -// tokenDecode decodes a token string. -func tokenDecode(token string) ([]byte, error) { - return util.TokenDecode(util.TokenTypeTlog, token) -} - -func convertPropStatusFromMDStatus(s backend.MDStatusT) pi.PropStatusT { - var status pi.PropStatusT - switch s { - case backend.MDStatusUnvetted, backend.MDStatusIterationUnvetted: - status = pi.PropStatusUnvetted - case backend.MDStatusVetted: - status = pi.PropStatusPublic - case backend.MDStatusCensored: - status = pi.PropStatusCensored - case backend.MDStatusArchived: - status = pi.PropStatusAbandoned - } - return status -} - -func (p *piPlugin) voteSummary(token []byte) (*ticketvote.VoteSummary, error) { - reply, err := p.backend.VettedPluginCmd(token, - ticketvote.ID, ticketvote.CmdSummary, "") - if err != nil { - return nil, err - } - var sr ticketvote.SummaryReply - err = json.Unmarshal([]byte(reply), &sr) - if err != nil { - return nil, err - } - return &sr.Summary, nil -} - -func (p *piPlugin) cmdProposalInv(payload string) (string, error) { - // Decode payload - var inv pi.ProposalInv - err := json.Unmarshal([]byte(payload), &inv) - if err != nil { - return "", err - } - - // Get full record inventory - ibs, err := p.backend.InventoryByStatus() - if err != nil { - return "", err - } - - // Apply user ID filtering criteria - if inv.UserID != "" { - // Lookup the proposal tokens that have been submitted by the - // specified user. - ud, err := p.userData(inv.UserID) - if err != nil { - return "", fmt.Errorf("userData %v: %v", inv.UserID, err) - } - userTokens := make(map[string]struct{}, len(ud.Tokens)) - for _, v := range ud.Tokens { - userTokens[v] = struct{}{} - } - - // Compile a list of unvetted tokens categorized by MDStatusT - // that were submitted by the user. - filtered := make(map[backend.MDStatusT][]string, len(ibs.Unvetted)) - for status, tokens := range ibs.Unvetted { - for _, v := range tokens { - _, ok := userTokens[v] - if !ok { - // Proposal was not submitted by the user - continue - } - - // Proposal was submitted by the user - ftokens, ok := filtered[status] - if !ok { - ftokens = make([]string, 0, len(tokens)) - } - filtered[status] = append(ftokens, v) - } - } - - // Update unvetted inventory with filtered tokens - ibs.Unvetted = filtered - - // Compile a list of vetted tokens categorized by MDStatusT that - // were submitted by the user. - filtered = make(map[backend.MDStatusT][]string, len(ibs.Vetted)) - for status, tokens := range ibs.Vetted { - for _, v := range tokens { - _, ok := userTokens[v] - if !ok { - // Proposal was not submitted by the user - continue - } - - // Proposal was submitted by the user - ftokens, ok := filtered[status] - if !ok { - ftokens = make([]string, 0, len(tokens)) - } - filtered[status] = append(ftokens, v) - } - } - - // Update vetted inventory with filtered tokens - ibs.Vetted = filtered - } - - // Convert MDStatus keys to human readable proposal statuses - unvetted := make(map[string][]string, len(ibs.Unvetted)) - vetted := make(map[string][]string, len(ibs.Vetted)) - for k, v := range ibs.Unvetted { - s := pi.PropStatuses[convertPropStatusFromMDStatus(k)] - unvetted[s] = v - } - for k, v := range ibs.Vetted { - s := pi.PropStatuses[convertPropStatusFromMDStatus(k)] - vetted[s] = v - } - - // Prepare reply - pir := pi.ProposalInvReply{ - Unvetted: unvetted, - Vetted: vetted, - } - reply, err := json.Marshal(pir) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *piPlugin) cmdVoteInventory() (string, error) { - // Get ticketvote inventory - r, err := p.backend.VettedPluginCmd([]byte{}, - ticketvote.ID, ticketvote.CmdInventory, "") - if err != nil { - return "", fmt.Errorf("VettedPluginCmd %v %v: %v", - ticketvote.ID, ticketvote.CmdInventory, err) - } - var ir ticketvote.InventoryReply - err = json.Unmarshal([]byte(r), &ir) - if err != nil { - return "", err - } - - // Get vote summaries for all finished proposal votes and - // categorize by approved/rejected. - approved := make([]string, 0, len(ir.Finished)) - rejected := make([]string, 0, len(ir.Finished)) - for _, v := range ir.Finished { - t, err := tokenDecode(v) - if err != nil { - return "", err - } - vs, err := p.voteSummary(t) - if err != nil { - return "", err - } - if vs.Approved { - approved = append(approved, v) - } else { - rejected = append(rejected, v) - } - } - - // Prepare reply - vir := pi.VoteInventoryReply{ - Unauthorized: ir.Unauthorized, - Authorized: ir.Authorized, - Started: ir.Started, - Approved: approved, - Rejected: rejected, - BestBlock: ir.BestBlock, - } - reply, err := json.Marshal(vir) - if err != nil { - return "", err - } - - return string(reply), nil -} - -// Setup performs any plugin setup work that needs to be done. +// Setup performs any plugin setup that is required. // // This function satisfies the plugins.Client interface. func (p *piPlugin) Setup() error { @@ -239,9 +48,9 @@ func (p *piPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, switch cmd { case pi.CmdProposalInv: - return p.cmdProposalInv(payload) - case pi.CmdVoteInventory: - return p.cmdVoteInventory() + return p.cmdProposalInv() + case pi.CmdVoteInv: + return p.cmdVoteInv() } return "", backend.ErrPluginCmdInvalid @@ -251,13 +60,11 @@ func (p *piPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, // // This function satisfies the plugins.Client interface. func (p *piPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { - log.Tracef("Hook: %v %x %v", treeID, plugins.Hooks[h]) + log.Tracef("Hook: %v %x %v", treeID, token, plugins.Hooks[h]) switch h { case plugins.HookTypeNewRecordPre: return p.hookNewRecordPre(payload) - case plugins.HookTypeNewRecordPost: - return p.hookNewRecordPost(payload) case plugins.HookTypeEditRecordPre: return p.hookEditRecordPre(payload) case plugins.HookTypeSetRecordStatusPost: @@ -272,13 +79,13 @@ func (p *piPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload str // Fsck performs a plugin filesystem check. // // This function satisfies the plugins.Client interface. -func (p *piPlugin) Fsck() error { +func (p *piPlugin) Fsck(treeIDs []int64) error { log.Tracef("Fsck") return nil } -func New(backend backend.Backend, tlog plugins.TlogClient, settings []backend.PluginSetting, dataDir string, activeNetParams *chaincfg.Params) (*piPlugin, error) { +func New(backend backend.Backend, settings []backend.PluginSetting, dataDir string) (*piPlugin, error) { // Create plugin data directory dataDir = filepath.Join(dataDir, pi.ID) err := os.MkdirAll(dataDir, 0700) @@ -287,9 +94,7 @@ func New(backend backend.Backend, tlog plugins.TlogClient, settings []backend.Pl } return &piPlugin{ - dataDir: dataDir, - backend: backend, - activeNetParams: activeNetParams, - tlog: tlog, + dataDir: dataDir, + backend: backend, }, nil } diff --git a/politeiad/backend/tlogbe/plugins/pi/user.go b/politeiad/backend/tlogbe/plugins/pi/user.go deleted file mode 100644 index ea9dfdcf5..000000000 --- a/politeiad/backend/tlogbe/plugins/pi/user.go +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright (c) 2020-2021 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package pi - -import ( - "encoding/json" - "errors" - "io/ioutil" - "os" - "path/filepath" - "strings" -) - -const ( - // fnUserData is the filename for the cached userData that is - // saved to the pi plugin data dir. - fnUserData = "{userid}.json" -) - -// userData contains cached pi plugin data for a specific user. The userData -// JSON is saved to disk in the pi plugin data dir. The user ID is included in -// the filename. -type userData struct { - // Tokens contains a list of all the proposals that have been - // submitted by this user. This data is cached so that the - // ProposalInv command can filter proposals by user ID. - Tokens []string `json:"tokens"` -} - -// userDataPath returns the filepath to the cached userData struct for the -// specified user. -func (p *piPlugin) userDataPath(userID string) string { - fn := strings.Replace(fnUserData, "{userid}", userID, 1) - return filepath.Join(p.dataDir, fn) -} - -// userDataWithLock returns the cached userData struct for the specified user. -// -// This function must be called WITH the lock held. -func (p *piPlugin) userDataWithLock(userID string) (*userData, error) { - fp := p.userDataPath(userID) - b, err := ioutil.ReadFile(fp) - if err != nil { - var e *os.PathError - if errors.As(err, &e) && !os.IsExist(err) { - // File does't exist. Return an empty userData. - return &userData{ - Tokens: []string{}, - }, nil - } - } - - var ud userData - err = json.Unmarshal(b, &ud) - if err != nil { - return nil, err - } - - return &ud, nil -} - -// userData returns the cached userData struct for the specified user. -// -// This function must be called WITHOUT the lock held. -func (p *piPlugin) userData(userID string) (*userData, error) { - p.Lock() - defer p.Unlock() - - return p.userDataWithLock(userID) -} - -// userDataSaveWithLock saves the provided userData to the pi plugin data dir. -// -// This function must be called WITH the lock held. -func (p *piPlugin) userDataSaveWithLock(userID string, ud userData) error { - b, err := json.Marshal(ud) - if err != nil { - return err - } - - fp := p.userDataPath(userID) - return ioutil.WriteFile(fp, b, 0664) -} - -// userDataAddToken adds the provided token to the cached userData for the -// provided user. -// -// This function must be called WITHOUT the lock held. -func (p *piPlugin) userDataAddToken(userID string, token string) error { - p.Lock() - defer p.Unlock() - - // Get current user data - ud, err := p.userDataWithLock(userID) - if err != nil { - return err - } - - // Add token - ud.Tokens = append(ud.Tokens, token) - - // Save changes - return p.userDataSaveWithLock(userID, *ud) -} diff --git a/politeiad/backend/tlogbe/plugins/plugins.go b/politeiad/backend/tlogbe/plugins/plugins.go index dd3112345..8d31015b6 100644 --- a/politeiad/backend/tlogbe/plugins/plugins.go +++ b/politeiad/backend/tlogbe/plugins/plugins.go @@ -9,21 +9,50 @@ import ( "github.com/decred/politeia/politeiad/backend/tlogbe/store" ) -// HookT represents the types of plugin hooks. +// HookT represents a plugin hook. type HookT int const ( - HookTypeInvalid HookT = 0 - HookTypeNewRecordPre HookT = 1 - HookTypeNewRecordPost HookT = 2 - HookTypeEditRecordPre HookT = 3 - HookTypeEditRecordPost HookT = 4 - HookTypeEditMetadataPre HookT = 5 - HookTypeEditMetadataPost HookT = 6 - HookTypeSetRecordStatusPre HookT = 7 + // HookTypeInvalid is an invalid plugin hook. + HookTypeInvalid HookT = 0 + + // HootTypeNewRecordPre is called before new record is saved to + // disk. + HookTypeNewRecordPre HookT = 1 + + // HootTypeNewRecordPost is called after a new record is saved to + // disk. + HookTypeNewRecordPost HookT = 2 + + // HookTypeEditRecordPre is called before a record update is saved + // to disk. + HookTypeEditRecordPre HookT = 3 + + // HookTypeEditRecordPost is called after a record update is saved + // to disk. + HookTypeEditRecordPost HookT = 4 + + // HookTypeEditMetadataPre is called before a metadata update is + // saved to disk. + HookTypeEditMetadataPre HookT = 5 + + // HookTypeEditMetadataPost is called after a metadata update is + // saved to disk. + HookTypeEditMetadataPost HookT = 6 + + // HookTypeSetRecordStatusPre is called before a record status + // change is saved to disk. + HookTypeSetRecordStatusPre HookT = 7 + + // HookTypeSetRecordStatusPost is called after a record status + // change is saved to disk. HookTypeSetRecordStatusPost HookT = 8 - HookTypePluginPre HookT = 9 - HookTypePluginPost HookT = 10 + + // HookTypePluginPre is called before a plugin command is executed. + HookTypePluginPre HookT = 9 + + // HookTypePluginPost is called after a plugin command is executed. + HookTypePluginPost HookT = 10 ) var ( @@ -85,10 +114,8 @@ type HookEditRecord struct { // Updated fields RecordMetadata backend.RecordMetadata `json:"recordmetadata"` - MDAppend []backend.MetadataStream `json:"mdappend"` - MDOverwrite []backend.MetadataStream `json:"mdoverwrite"` - FilesAdd []backend.File `json:"filesadd"` - FilesDel []string `json:"filesdel"` + Metadata []backend.MetadataStream `json:"metadata"` + Files []backend.File `json:"files"` } // HookEditMetadata is the payload for the pre and post edit metadata hooks. @@ -97,8 +124,7 @@ type HookEditMetadata struct { Current backend.Record `json:"record"` // Current record // Updated fields - MDAppend []backend.MetadataStream `json:"mdappend"` - MDOverwrite []backend.MetadataStream `json:"mdoverwrite"` + Metadata []backend.MetadataStream `json:"metadata"` } // HookSetRecordStatus is the payload for the pre and post set record status @@ -109,8 +135,7 @@ type HookSetRecordStatus struct { // Updated fields RecordMetadata backend.RecordMetadata `json:"recordmetadata"` - MDAppend []backend.MetadataStream `json:"mdappend"` - MDOverwrite []backend.MetadataStream `json:"mdoverwrite"` + Metadata []backend.MetadataStream `json:"metadata"` } // HookPluginPre is the payload for the pre plugin hook. @@ -145,16 +170,9 @@ type Client interface { Hook(treeID int64, token []byte, h HookT, payload string) error // Fsck performs a plugin file system check. - Fsck() error + Fsck(treeIDs []int64) error } -// TODO plugins should only have access to the backend methods for the tlog -// instance that they're registered on. -// TODO the plugin hook state should not really be required. This issue is that -// some vetted plugins require unvetted hooks, ex. verifying the linkto in -// vote metadata. Possile solution, keep the layer violations in the -// application plugin (pi) instead of the functionality plugin (ticketvote). - // TlogClient provides an API for plugins to interact with a tlog instance. // Plugins are allowed to save, delete, and get plugin data to/from the tlog // backend. Editing plugin data is not allowed. @@ -186,4 +204,10 @@ type TlogClient interface { // Timestamp returns the timestamp for the blob that correpsonds // to the digest. Timestamp(treeID int64, digest []byte) (*backend.Timestamp, error) + + // Record returns a version of a record. + Record(treeID int64, version uint32) (*backend.Record, error) + + // RecordLatest returns the most recent version of a record. + RecordLatest(treeID int64) (*backend.Record, error) } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go b/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go index f486b6852..658a4f2ce 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go @@ -212,7 +212,7 @@ func (p *ticketVotePlugin) hookEditRecordPre(payload string) error { if vm != nil { oldLinkTo = vm.LinkTo } - vm, err = voteMetadataDecode(er.FilesAdd) + vm, err = voteMetadataDecode(er.Files) if err != nil { return err } @@ -232,7 +232,7 @@ func (p *ticketVotePlugin) hookEditRecordPre(payload string) error { // Verify LinkBy if one was included. The VoteMetadata is optional // so the record may not contain one. - vm, err := voteMetadataDecode(er.FilesAdd) + vm, err := voteMetadataDecode(er.Files) if err != nil { return err } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go b/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go index f2bf08fd3..0d9fd5ec6 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go @@ -43,7 +43,7 @@ func (p *ticketVotePlugin) invCacheSetToAuthorized(token string) { u = append(u[:i], u[i+1:]...) p.inv.unauthorized = u - log.Debugf("ticketvote: removed from unauthorized inv: %v", token) + log.Debugf("Removed from unauthorized inv: %v", token) } // Prepend the token to the authorized list @@ -51,7 +51,7 @@ func (p *ticketVotePlugin) invCacheSetToAuthorized(token string) { a = append([]string{token}, a...) p.inv.authorized = a - log.Debugf("ticketvote: added to authorized inv: %v", token) + log.Debugf("Added to authorized inv: %v", token) } func (p *ticketVotePlugin) invCacheSetToUnauthorized(token string) { @@ -76,7 +76,7 @@ func (p *ticketVotePlugin) invCacheSetToUnauthorized(token string) { a = append(a[:i], a[i+1:]...) p.inv.authorized = a - log.Debugf("ticketvote: removed from authorized inv: %v", token) + log.Debugf("Removed from authorized inv: %v", token) } // Prepend the token to the unauthorized list @@ -84,7 +84,7 @@ func (p *ticketVotePlugin) invCacheSetToUnauthorized(token string) { u = append([]string{token}, u...) p.inv.unauthorized = u - log.Debugf("ticketvote: added to unauthorized inv: %v", token) + log.Debugf("Added to unauthorized inv: %v", token) } func (p *ticketVotePlugin) invCacheSetToStarted(token string, t ticketvote.VoteT, endHeight uint32) { @@ -115,7 +115,7 @@ func (p *ticketVotePlugin) invCacheSetToStarted(token string, t ticketvote.VoteT a = append(a[:i], a[i+1:]...) p.inv.authorized = a - log.Debugf("ticketvote: removed from authorized inv: %v", token) + log.Debugf("Removed from authorized inv: %v", token) case ticketvote.VoteTypeRunoff: // A runoff vote does not require the submission votes be @@ -138,7 +138,7 @@ func (p *ticketVotePlugin) invCacheSetToStarted(token string, t ticketvote.VoteT u = append(u[:i], u[i+1:]...) p.inv.unauthorized = u - log.Debugf("ticketvote: removed from unauthorized inv: %v", token) + log.Debugf("Removed from unauthorized inv: %v", token) } default: @@ -149,7 +149,7 @@ func (p *ticketVotePlugin) invCacheSetToStarted(token string, t ticketvote.VoteT // Add the token to the started list p.inv.started[token] = endHeight - log.Debugf("ticketvote: added to started inv: %v", token) + log.Debugf("Added to started inv: %v", token) } func (p *ticketVotePlugin) invCache(bestBlock uint32) (*inventory, error) { @@ -206,7 +206,7 @@ func (p *ticketVotePlugin) invCache(bestBlock uint32) (*inventory, error) { // occurred. p.inv.unauthorized = append(p.inv.unauthorized, v) - log.Debugf("ticketvote: added to unauthorized inv: %v", v) + log.Debugf("Added to unauthorized inv: %v", v) } } } @@ -230,19 +230,19 @@ func (p *ticketVotePlugin) invCache(bestBlock uint32) (*inventory, error) { // Vote has finished. Remove it from the started list. delete(p.inv.started, token) - log.Debugf("ticketvote: removed from started inv: %v", token) + log.Debugf("Removed from started inv: %v", token) // Add it to the finished list p.inv.finished = append(p.inv.finished, token) - log.Debugf("ticketvote: added to finished inv: %v", token) + log.Debugf("Added to finished inv: %v", token) } } // Update best block p.inv.bestBlock = bestBlock - log.Debugf("ticketvote: inv updated for best block %v", bestBlock) + log.Debugf("Inv updated for best block %v", bestBlock) reply: // Return a copy of the inventory diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go index 6856c3db0..094223a87 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go @@ -85,7 +85,7 @@ func (p *ticketVotePlugin) mutex(token []byte) *sync.Mutex { return m } -// Setup performs any plugin setup work that needs to be done. +// Setup performs any plugin setup that is required. // // This function satisfies the plugins.Client interface. func (p *ticketVotePlugin) Setup() error { @@ -237,7 +237,7 @@ func (p *ticketVotePlugin) Hook(treeID int64, token []byte, h plugins.HookT, pay // Fsck performs a plugin filesystem check. // // This function satisfies the plugins.Client interface. -func (p *ticketVotePlugin) Fsck() error { +func (p *ticketVotePlugin) Fsck(treeIDs []int64) error { log.Tracef("Fsck") // Verify all caches diff --git a/politeiad/backend/tlogbe/plugins/user/cache.go b/politeiad/backend/tlogbe/plugins/user/cache.go new file mode 100644 index 000000000..f40ecacf6 --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/user/cache.go @@ -0,0 +1,105 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package user + +import ( + "encoding/json" + "errors" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +const ( + // fnUserCache is the filename for the cached userCache data that + // is saved to the plugin data dir. + fnUserCache = "{userid}.json" +) + +// userCache contains cached user metadata. The userCache JSON is saved to disk +// in the user plugin data dir. The user ID is included in the filename. +type userCache struct { + // Tokens contains a list of all record tokens that have been + // submitted by this user, ordered newest to oldest. + Tokens []string `json:"tokens"` +} + +// userCachePath returns the filepath to the cached userCache struct for the +// specified user. +func (p *userPlugin) userCachePath(userID string) string { + fn := strings.Replace(fnUserCache, "{userid}", userID, 1) + return filepath.Join(p.dataDir, fn) +} + +// userCacheWithLock returns the cached userCache struct for the specified +// user. +// +// This function must be called WITH the lock held. +func (p *userPlugin) userCacheWithLock(userID string) (*userCache, error) { + fp := p.userCachePath(userID) + b, err := ioutil.ReadFile(fp) + if err != nil { + var e *os.PathError + if errors.As(err, &e) && !os.IsExist(err) { + // File does't exist. Return an empty userCache. + return &userCache{ + Tokens: []string{}, + }, nil + } + } + + var uc userCache + err = json.Unmarshal(b, &uc) + if err != nil { + return nil, err + } + + return &uc, nil +} + +// userCache returns the cached userCache struct for the specified user. +// +// This function must be called WITHOUT the lock held. +func (p *userPlugin) userCache(userID string) (*userCache, error) { + p.Lock() + defer p.Unlock() + + return p.userCacheWithLock(userID) +} + +// userCacheSaveWithLock saves the provided userCache to the pi plugin data dir. +// +// This function must be called WITH the lock held. +func (p *userPlugin) userCacheSaveWithLock(userID string, uc userCache) error { + b, err := json.Marshal(uc) + if err != nil { + return err + } + + fp := p.userCachePath(userID) + return ioutil.WriteFile(fp, b, 0664) +} + +// userCacheAddToken adds the provided token to the cached userCache for the +// provided user. +// +// This function must be called WITHOUT the lock held. +func (p *userPlugin) userCacheAddToken(userID string, token string) error { + p.Lock() + defer p.Unlock() + + // Get current user data + uc, err := p.userCacheWithLock(userID) + if err != nil { + return err + } + + // Add token + uc.Tokens = append(uc.Tokens, token) + + // Save changes + return p.userCacheSaveWithLock(userID, *uc) +} diff --git a/politeiad/backend/tlogbe/plugins/user/cmds.go b/politeiad/backend/tlogbe/plugins/user/cmds.go new file mode 100644 index 000000000..f6b877e28 --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/user/cmds.go @@ -0,0 +1,64 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package user + +import ( + "encoding/json" + + "github.com/decred/politeia/politeiad/plugins/user" +) + +func (p *userPlugin) cmdAuthor(treeID int64) (string, error) { + log.Tracef("cmdAuthor: %v", treeID) + + // Get user metadata + r, err := p.tlog.RecordLatest(treeID) + if err != nil { + return "", err + } + um, err := userMetadataDecode(r.Metadata) + if err != nil { + return "", err + } + + // Prepare reply + ar := user.AuthorReply{ + UserID: um.UserID, + } + reply, err := json.Marshal(ar) + if err != nil { + return "", err + } + + return string(reply), nil +} + +func (p *userPlugin) cmdUserRecords(payload string) (string, error) { + log.Tracef("cmdUserRecords: %v", payload) + + // Decode payload + var ur user.UserRecords + err := json.Unmarshal([]byte(payload), &ur) + if err != nil { + return "", err + } + + // Get user records + uc, err := p.userCache(ur.UserID) + if err != nil { + return "", err + } + + // Prepare reply + urr := user.UserRecordsReply{ + Records: uc.Tokens, + } + reply, err := json.Marshal(urr) + if err != nil { + return "", err + } + + return string(reply), nil +} diff --git a/politeiad/backend/tlogbe/plugins/user/hooks.go b/politeiad/backend/tlogbe/plugins/user/hooks.go new file mode 100644 index 000000000..4db5edd07 --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/user/hooks.go @@ -0,0 +1,227 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package user + +import ( + "encoding/hex" + "encoding/json" + "errors" + "fmt" + + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" + "github.com/decred/politeia/politeiad/plugins/user" + pdutil "github.com/decred/politeia/politeiad/util" + "github.com/decred/politeia/util" + "github.com/google/uuid" +) + +func convertSignatureError(err error) backend.PluginError { + var e util.SignatureError + var s user.ErrorCodeT + if errors.As(err, &e) { + switch e.ErrorCode { + case util.ErrorStatusPublicKeyInvalid: + s = user.ErrorCodePublicKeyInvalid + case util.ErrorStatusSignatureInvalid: + s = user.ErrorCodeSignatureInvalid + } + } + return backend.PluginError{ + PluginID: user.PluginID, + ErrorCode: int(s), + ErrorContext: e.ErrorContext, + } +} + +// userMetadataDecode decodes and returns the UserMetadata from the provided +// backend metadata streams. If a UserMetadata is not found, nil is returned. +func userMetadataDecode(metadata []backend.MetadataStream) (*user.UserMetadata, error) { + var userMD *user.UserMetadata + for _, v := range metadata { + if v.ID == user.MDStreamIDUserMetadata { + var um user.UserMetadata + err := json.Unmarshal([]byte(v.Payload), &um) + if err != nil { + return nil, err + } + userMD = &um + break + } + } + return userMD, nil +} + +// userMetadataVerify parses a UserMetadata from the metadata streams and +// verifies its contents are valid. +func userMetadataVerify(metadata []backend.MetadataStream, files []backend.File) error { + // Decode user metadata + um, err := userMetadataDecode(metadata) + if err != nil { + return err + } + if um == nil { + return backend.PluginError{ + PluginID: user.PluginID, + ErrorCode: int(user.ErrorCodeUserMetadataNotFound), + } + } + + // Verify user ID + _, err = uuid.Parse(um.UserID) + if err != nil { + return backend.PluginError{ + PluginID: user.PluginID, + ErrorCode: int(user.ErrorCodeUserIDInvalid), + } + } + + // Verify signature + m, err := pdutil.MerkleRoot(files) + if err != nil { + return err + } + msg := hex.EncodeToString(m[:]) + err = util.VerifySignature(um.Signature, um.PublicKey, msg) + if err != nil { + return convertSignatureError(err) + } + + return nil +} + +// userMetadataPreventUpdates errors if the UserMetadata is being updated. +func userMetadataPreventUpdates(current, update []backend.MetadataStream) error { + // Decode user metadata + c, err := userMetadataDecode(current) + if err != nil { + return err + } + u, err := userMetadataDecode(update) + if err != nil { + return err + } + + // Verify user metadata has not changed + switch { + case u.UserID != c.UserID: + e := fmt.Sprintf("user id cannot change: got %v, want %v", + u.UserID, c.UserID) + return backend.PluginError{ + PluginID: user.PluginID, + ErrorCode: int(user.ErrorCodeUserIDInvalid), + ErrorContext: e, + } + + case u.PublicKey != c.PublicKey: + e := fmt.Sprintf("public key cannot change: got %v, want %v", + u.PublicKey, c.PublicKey) + return backend.PluginError{ + PluginID: user.PluginID, + ErrorCode: int(user.ErrorCodePublicKeyInvalid), + ErrorContext: e, + } + + case c.Signature != c.Signature: + e := fmt.Sprintf("signature cannot change: got %v, want %v", + u.Signature, c.Signature) + return backend.PluginError{ + PluginID: user.PluginID, + ErrorCode: int(user.ErrorCodeSignatureInvalid), + ErrorContext: e, + } + } + + return nil +} + +func (p *userPlugin) hookNewRecordPre(payload string) error { + var nr plugins.HookNewRecordPre + err := json.Unmarshal([]byte(payload), &nr) + if err != nil { + return err + } + + return userMetadataVerify(nr.Metadata, nr.Files) +} + +func (p *userPlugin) hookNewRecordPost(payload string) error { + var nr plugins.HookNewRecordPost + err := json.Unmarshal([]byte(payload), &nr) + if err != nil { + return err + } + + // Decode user metadata + um, err := userMetadataDecode(nr.Metadata) + if err != nil { + return err + } + + // Add token to the user cache + err = p.userCacheAddToken(um.UserID, nr.RecordMetadata.Token) + if err != nil { + return err + } + + return nil +} + +func (p *userPlugin) hookEditRecordPre(payload string) error { + var er plugins.HookEditRecord + err := json.Unmarshal([]byte(payload), &er) + if err != nil { + return err + } + + // Verify user metadata + err = userMetadataVerify(er.Metadata, er.Files) + if err != nil { + return err + } + + // Verify user ID has not changed + um, err := userMetadataDecode(er.Metadata) + if err != nil { + return err + } + umCurr, err := userMetadataDecode(er.Current.Metadata) + if err != nil { + return err + } + if um.UserID != umCurr.UserID { + e := fmt.Sprintf("user id cannot change: got %v, want %v", + um.UserID, umCurr.UserID) + return backend.PluginError{ + PluginID: user.PluginID, + ErrorCode: int(user.ErrorCodeUserIDInvalid), + ErrorContext: e, + } + } + + return nil +} + +func (p *userPlugin) hookEditMetadataPre(payload string) error { + var em plugins.HookEditMetadata + err := json.Unmarshal([]byte(payload), &em) + if err != nil { + return err + } + + // User metadata should not change on metadata updates + return userMetadataPreventUpdates(em.Current.Metadata, em.Metadata) +} + +func (p *userPlugin) hookSetRecordStatusPre(payload string) error { + var srs plugins.HookSetRecordStatus + err := json.Unmarshal([]byte(payload), &srs) + if err != nil { + return err + } + + // User metadata should not change on status changes + return userMetadataPreventUpdates(srs.Current.Metadata, srs.Metadata) +} diff --git a/politeiad/backend/tlogbe/plugins/user/log.go b/politeiad/backend/tlogbe/plugins/user/log.go new file mode 100644 index 000000000..b8ab45de7 --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/user/log.go @@ -0,0 +1,25 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package user + +import "github.com/decred/slog" + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log = slog.Disabled + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = slog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using slog. +func UseLogger(logger slog.Logger) { + log = logger +} diff --git a/politeiad/backend/tlogbe/plugins/user/user.go b/politeiad/backend/tlogbe/plugins/user/user.go new file mode 100644 index 000000000..4e03f9162 --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/user/user.go @@ -0,0 +1,100 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package user + +import ( + "os" + "path/filepath" + "sync" + + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" + "github.com/decred/politeia/politeiad/plugins/user" +) + +var ( + _ plugins.Client = (*userPlugin)(nil) +) + +type userPlugin struct { + sync.Mutex + tlog plugins.TlogClient + + // dataDir is the pi plugin data directory. The only data that is + // stored here is cached data that can be re-created at any time + // by walking the trillian trees. + dataDir string +} + +// Setup performs any plugin setup that is required. +// +// This function satisfies the plugins.Client interface. +func (p *userPlugin) Setup() error { + log.Tracef("Setup") + + return nil +} + +// Cmd executes a plugin command. +// +// This function satisfies the plugins.Client interface. +func (p *userPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { + log.Tracef("Cmd: %v %x %v %v", treeID, token, cmd, payload) + + switch cmd { + case user.CmdAuthor: + return p.cmdAuthor(treeID) + case user.CmdUserRecords: + return p.cmdUserRecords(payload) + } + + return "", backend.ErrPluginCmdInvalid +} + +// Hook executes a plugin hook. +// +// This function satisfies the plugins.Client interface. +func (p *userPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { + log.Tracef("Hook: %v %x %v", treeID, token, plugins.Hooks[h]) + + switch h { + case plugins.HookTypeNewRecordPre: + return p.hookNewRecordPre(payload) + case plugins.HookTypeNewRecordPost: + return p.hookNewRecordPost(payload) + case plugins.HookTypeEditRecordPre: + return p.hookEditRecordPre(payload) + case plugins.HookTypeEditMetadataPre: + return p.hookEditMetadataPre(payload) + case plugins.HookTypeSetRecordStatusPre: + return p.hookSetRecordStatusPre(payload) + } + + return nil +} + +// Fsck performs a plugin filesystem check. +// +// This function satisfies the plugins.Client interface. +func (p *userPlugin) Fsck(treeIDs []int64) error { + log.Tracef("Fsck") + + return nil +} + +// New returns a new userPlugin. +func New(tlog plugins.TlogClient, settings []backend.PluginSetting, dataDir string) (*userPlugin, error) { + // Create plugin data directory + dataDir = filepath.Join(dataDir, user.PluginID) + err := os.MkdirAll(dataDir, 0700) + if err != nil { + return nil, err + } + + return &userPlugin{ + tlog: tlog, + dataDir: dataDir, + }, nil +} diff --git a/politeiad/backend/tlogbe/tlog/plugin.go b/politeiad/backend/tlogbe/tlog/plugin.go index bb4d943fd..2540a7382 100644 --- a/politeiad/backend/tlogbe/tlog/plugin.go +++ b/politeiad/backend/tlogbe/tlog/plugin.go @@ -13,8 +13,14 @@ import ( "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/comments" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/dcrdata" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/pi" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/ticketvote" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/user" cmplugin "github.com/decred/politeia/politeiad/plugins/comments" ddplugin "github.com/decred/politeia/politeiad/plugins/dcrdata" + piplugin "github.com/decred/politeia/politeiad/plugins/pi" + tkplugin "github.com/decred/politeia/politeiad/plugins/ticketvote" + userplugin "github.com/decred/politeia/politeiad/plugins/user" ) const ( @@ -71,19 +77,19 @@ func (t *Tlog) PluginRegister(b backend.Backend, p backend.Plugin) error { if err != nil { return err } - /* - case piplugin.ID: - client, err = pi.New(b, t, p.Settings, dataDir, t.activeNetParams) - if err != nil { - return err - } - case ticketvote.ID: - client, err = newTicketVotePlugin(t, newBackendClient(t), - p.Settings, p.Identity, t.activeNetParams) - if err != nil { - return err - } - */ + case piplugin.ID: + client, err = pi.New(b, p.Settings, dataDir) + if err != nil { + return err + } + case tkplugin.ID: + client, err = ticketvote.New(b, t, p.Settings, dataDir, + p.Identity, t.activeNetParams) + if err != nil { + return err + } + case userplugin.PluginID: + client, err = user.New(t, p.Settings, dataDir) default: return backend.ErrPluginInvalid } diff --git a/politeiad/backend/tlogbe/tlog/tlog.go b/politeiad/backend/tlogbe/tlog/tlog.go index d976c3ebd..e69e08ce6 100644 --- a/politeiad/backend/tlogbe/tlog/tlog.go +++ b/politeiad/backend/tlogbe/tlog/tlog.go @@ -18,6 +18,7 @@ import ( "github.com/decred/dcrd/chaincfg/v3" "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/politeiad/backend/tlogbe/store" "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" "github.com/decred/politeia/util" @@ -54,6 +55,11 @@ const ( dataTypeAnchorRecord = "anchor" ) +var ( + _ plugins.TlogClient = (*Tlog)(nil) +) + +// TODO change tlog name to tstore. // We do not unwind. type Tlog struct { sync.Mutex diff --git a/politeiad/backend/tlogbe/tlog/tlogclient.go b/politeiad/backend/tlogbe/tlog/tlogclient.go index 1853fa735..80c9138c8 100644 --- a/politeiad/backend/tlogbe/tlog/tlogclient.go +++ b/politeiad/backend/tlogbe/tlog/tlogclient.go @@ -10,16 +10,11 @@ import ( "strings" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/politeiad/backend/tlogbe/store" "github.com/google/trillian" "google.golang.org/grpc/codes" ) -var ( - _ plugins.TlogClient = (*Tlog)(nil) -) - // BlobSave saves a BlobEntry to the tlog instance. The BlobEntry will be // encrypted prior to being written to disk if the tlog instance has an // encryption key set. The digest of the data, i.e. BlobEntry.Digest, can be diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index f0b4fb6a6..25145df76 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -6,7 +6,6 @@ package tlogbe import ( "bytes" - "crypto/sha256" "encoding/base64" "encoding/binary" "encoding/hex" @@ -21,12 +20,12 @@ import ( "time" "github.com/decred/dcrd/chaincfg/v3" - "github.com/decred/dcrtime/merkle" v1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/mime" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" + pdutil "github.com/decred/politeia/politeiad/util" "github.com/decred/politeia/util" "github.com/marcopeereboom/sbox" "github.com/subosito/gozaru" @@ -397,22 +396,8 @@ func statusChangeIsAllowed(from, to backend.MDStatusT) bool { return ok } -func merkleRoot(files []backend.File) (*[sha256.Size]byte, error) { - hashes := make([]*[sha256.Size]byte, 0, len(files)) - for _, v := range files { - b, err := hex.DecodeString(v.Digest) - if err != nil { - return nil, err - } - var d [sha256.Size]byte - copy(d[:], b) - hashes = append(hashes, &d) - } - return merkle.Root(hashes), nil -} - func recordMetadataNew(token []byte, files []backend.File, status backend.MDStatusT, iteration uint64) (*backend.RecordMetadata, error) { - m, err := merkleRoot(files) + m, err := pdutil.MerkleRoot(files) if err != nil { return nil, err } @@ -654,10 +639,8 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ State: plugins.RecordStateUnvetted, Current: *r, RecordMetadata: *recordMD, - MDAppend: mdAppend, - MDOverwrite: mdOverwrite, - FilesAdd: filesAdd, - FilesDel: filesDel, + Metadata: metadata, + Files: files, } b, err := json.Marshal(her) if err != nil { @@ -758,10 +741,8 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b State: plugins.RecordStateVetted, Current: *r, RecordMetadata: *recordMD, - MDAppend: mdAppend, - MDOverwrite: mdOverwrite, - FilesAdd: filesAdd, - FilesDel: filesDel, + Metadata: metadata, + Files: files, } b, err := json.Marshal(her) if err != nil { @@ -1116,8 +1097,7 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, State: plugins.RecordStateUnvetted, Current: *r, RecordMetadata: rm, - MDAppend: mdAppend, - MDOverwrite: mdOverwrite, + Metadata: metadata, } b, err := json.Marshal(hsrs) if err != nil { @@ -1271,8 +1251,7 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md State: plugins.RecordStateVetted, Current: *r, RecordMetadata: rm, - MDAppend: mdAppend, - MDOverwrite: mdOverwrite, + Metadata: metadata, } b, err := json.Marshal(srs) if err != nil { diff --git a/politeiad/log.go b/politeiad/log.go index c4f141434..1ec19ce22 100644 --- a/politeiad/log.go +++ b/politeiad/log.go @@ -14,6 +14,7 @@ import ( "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/comments" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/dcrdata" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/ticketvote" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/user" "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" "github.com/decred/politeia/wsdcrdata" @@ -54,9 +55,10 @@ var ( tlogLog = backendLog.Logger("TLOG") storeLog = backendLog.Logger("STOR") wsdcrdataLog = backendLog.Logger("WSDD") - commentsLog = backendLog.Logger("COMM") - dcrdataLog = backendLog.Logger("DCRL") - ticketvoteLog = backendLog.Logger("TIKV") + commentsLog = backendLog.Logger("COMT") + dcrdataLog = backendLog.Logger("DCDA") + ticketvoteLog = backendLog.Logger("TICK") + userLog = backendLog.Logger("USER") ) // Initialize package-global logger variables. @@ -69,6 +71,7 @@ func init() { comments.UseLogger(commentsLog) dcrdata.UseLogger(dcrdataLog) ticketvote.UseLogger(ticketvoteLog) + user.UseLogger(userLog) } // subsystemLoggers maps each subsystem identifier to its associated logger. @@ -79,9 +82,10 @@ var subsystemLoggers = map[string]slog.Logger{ "TLOG": tlogLog, "STOR": storeLog, "WSDD": wsdcrdataLog, - "COMM": commentsLog, - "DCRL": dcrdataLog, - "TIKV": ticketvoteLog, + "COMT": commentsLog, + "DCDA": dcrdataLog, + "TICK": ticketvoteLog, + "USER": userLog, } // initLogRotator initializes the logging rotater to write logs to logFile and diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index dd4cc5758..b7e1d1a3d 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -10,16 +10,17 @@ const ( ID = "pi" // Plugin commands - CmdProposalInv = "proposalinv" // Get inventory by proposal status - CmdVoteInventory = "voteinv" // Get inventory by vote status + // TODO I might not need the ProposalInv command + CmdProposalInv = "proposalinv" // Get inventory by proposal status + CmdVoteInv = "voteinv" // Get inventory by vote status ) // ErrorCodeT represents a plugin error that was caused by the user. type ErrorCodeT int const ( - // User error status codes - // TODO number error codes and add human readable error messages + // TODO number + // User error codes ErrorCodeInvalid ErrorCodeT = 0 ErrorCodePageSizeExceeded ErrorCodeT = iota ErrorCodePropTokenInvalid @@ -33,6 +34,14 @@ const ( ErrorCodeVoteParentInvalid ) +var ( + // TODO fill in + // ErrorCodes contains the human readable errors. + ErrorCodes = map[ErrorCodeT]string{ + ErrorCodeInvalid: "error code invalid", + } +) + const ( // FileNameProposalMetadata is the filename of the ProposalMetadata // file that is saved to politeiad. ProposalMetadata is saved to @@ -41,13 +50,8 @@ const ( // root that politeiad signs. FileNameProposalMetadata = "proposalmetadata.json" - // TODO MDStream IDs need to be plugin specific - // MDStreamIDGeneralMetadata is the politeiad metadata stream ID - // for the GeneralMetadata structure. - MDStreamIDGeneralMetadata = 1 - - // MDStreamIDStatusChanges is the politeiad metadata stream ID - // that the StatusesChange structure is appended onto. + // MDStreamIDStatusChanges is the politeiad metadata stream ID that + // the StatusesChange structure is appended onto. MDStreamIDStatusChanges = 2 ) @@ -60,17 +64,6 @@ type ProposalMetadata struct { Name string `json:"name"` } -// GeneralMetadata contains general metadata about a politeiad record. It is -// generated by the server and saved to politeiad as a metadata stream. -// -// Signature is the client signature of the record merkle root. The merkle root -// is the ordered merkle root of all politeiad Files. -type GeneralMetadata struct { - UserID string `json:"userid"` // Author user ID - PublicKey string `json:"publickey"` // Key used for signature - Signature string `json:"signature"` // Signature of merkle root -} - // PropStatusT represents a proposal status. type PropStatusT int @@ -134,10 +127,7 @@ type StatusChange struct { Timestamp int64 `json:"timestamp"` } -// ProposalInv retrieves the tokens of all proposals in the inventory that -// match the provided filtering criteria. The returned proposals are -// categorized by proposal state and status. If no filtering criteria is -// provided then the full proposal inventory is returned. +// ProposalInv retrieves the tokens of all proposals in the inventory. type ProposalInv struct { UserID string `json:"userid,omitempty"` } diff --git a/politeiad/plugins/user/user.go b/politeiad/plugins/user/user.go new file mode 100644 index 000000000..d2282a380 --- /dev/null +++ b/politeiad/plugins/user/user.go @@ -0,0 +1,77 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +// Package user provides a politeiad plugin that extends records with user +// metadata and provides an API for retrieving records by user metadata. +package user + +const ( + PluginID = "user" + + // Plugin commands + CmdAuthor = "author" // Get record author + CmdUserRecords = "userrecords" // Get user submitted records + + // TODO MDStream IDs need to be plugin specific. If we can't then + // we need to make a mdstream package to aggregate all the mdstream + // ID. + + // MDStreamIDUserMetadata is the politeiad metadata stream ID for + // the UserMetadata structure. + MDStreamIDUserMetadata = 1 +) + +// ErrorCodeT represents a plugin error that was caused by the user. +type ErrorCodeT int + +const ( + // TODO number + // User error codes + ErrorCodeInvalid ErrorCodeT = iota + ErrorCodeUserMetadataNotFound + ErrorCodeUserIDInvalid + ErrorCodePublicKeyInvalid + ErrorCodeSignatureInvalid + ErrorCodeUpdateNotAllowed +) + +var ( + // TODO fill in + // ErrorCodes contains the human readable errors. + ErrorCodes = map[ErrorCodeT]string{ + ErrorCodeInvalid: "error code invalid", + } +) + +// UserMetadata contains user metadata about a politeiad record. It is +// generated by the server and saved to politeiad as a metadata stream. +// +// Signature is the client signature of the record merkle root. The merkle root +// is the ordered merkle root of all user submitted politeiad files. +type UserMetadata struct { + UserID string `json:"userid"` // Author user ID + PublicKey string `json:"publickey"` // Key used for signature + Signature string `json:"signature"` // Signature of merkle root +} + +// Author returns the user ID of a record's author. If no UserMetadata is +// present for the record then an empty string will be returned. +type Author struct{} + +// AuthorReply is the reply to the Author command. +type AuthorReply struct { + UserID string `json:"userid"` +} + +// UserRecords retrieves the tokens of all records that were +// submitted by the provided user ID. The returned tokens are sorted from +// newest to oldest. +type UserRecords struct { + UserID string `json:"userid"` +} + +// UserRecordsReply is the reply to the UserInv command. +type UserRecordsReply struct { + Records []string `json:"records"` +} diff --git a/politeiad/util/util.go b/politeiad/util/util.go new file mode 100644 index 000000000..68ece2635 --- /dev/null +++ b/politeiad/util/util.go @@ -0,0 +1,50 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package util + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + + "github.com/decred/dcrtime/merkle" + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/util" +) + +// MerkleRoot computes and returns the merkle root of the backend files. +func MerkleRoot(files []backend.File) (*[sha256.Size]byte, error) { + digests := make([]*[sha256.Size]byte, 0, len(files)) + for _, v := range files { + // Decode digest + digest, err := hex.DecodeString(v.Digest) + if err != nil { + return nil, err + } + + // Decode payload + payload, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return nil, err + } + + // Verify digest + d := util.Digest(payload) + if bytes.Equal(digest, d) { + return nil, fmt.Errorf("invalid digest for payload: got %x, want %x", + digest, d) + } + + // Save digest + var s [sha256.Size]byte + copy(s[:], d) + digests = append(digests, &s) + } + + // Calc merkle root + return merkle.Root(digests), nil +} diff --git a/politeiawww/api/comments/v1/v1.go b/politeiawww/api/comments/v1/v1.go index 6e9ebe6b5..dcde06357 100644 --- a/politeiawww/api/comments/v1/v1.go +++ b/politeiawww/api/comments/v1/v1.go @@ -36,6 +36,7 @@ var ( ErrorCodeInvalid: "error invalid", ErrorCodeInputInvalid: "input invalid", ErrorStatusPublicKeyInvalid: "public key invalid", + ErrorStatusSignatureInvalid: "signature invalid", } ) diff --git a/politeiawww/comments.go b/politeiawww/comments.go index 8882ee5d8..12918b925 100644 --- a/politeiawww/comments.go +++ b/politeiawww/comments.go @@ -69,14 +69,14 @@ func commentPopulateUser(c piv1.Comment, u user.User) cmv1.Comment { return c } -func (p *politeiawww) piCommentNew(ctx context.Context, n cmv1.CommentNew, u user.User) error { +func (p *politeiawww) commentNewPi(ctx context.Context, n cmv1.CommentNew, u user.User) error { // Verify user has paid registration paywall if !p.userHasPaid(u) { return nil, cmv1.UserErrorReply{ ErrorCode: cmv1.ErrorCodeUserRegistrationNotPaid, } } - + return nil } func (p *politeiawww) processCommentNew(ctx context.Context, n cmv1.CommentNew, u user.User) (*cmv1.CommentNewReply, error) { @@ -85,7 +85,7 @@ func (p *politeiawww) processCommentNew(ctx context.Context, n cmv1.CommentNew, // This is temporary until user plugins are implemented. switch p.mode { case politeiaWWWMode: - err := piCommentNew(ctx, n, u) + err := p.commentNewPi(ctx, n, u) if err != nil { return nil, err } diff --git a/util/signature.go b/util/signature.go index cf76a8bb8..6684f901c 100644 --- a/util/signature.go +++ b/util/signature.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. From 942abb4fca1312de1ff845cea6b7463149d8f3b8 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 20 Jan 2021 10:26:39 -0600 Subject: [PATCH 238/449] Add politeiad client. --- politeiad/client/client.go | 112 ++++++++ politeiad/client/pdv1.go | 544 +++++++++++++++++++++++++++++++++++++ politeiawww/politeiad.go | 494 +-------------------------------- politeiawww/politeiawww.go | 2 + politeiawww/www.go | 8 + 5 files changed, 670 insertions(+), 490 deletions(-) create mode 100644 politeiad/client/client.go create mode 100644 politeiad/client/pdv1.go diff --git a/politeiad/client/client.go b/politeiad/client/client.go new file mode 100644 index 000000000..7598d09ef --- /dev/null +++ b/politeiad/client/client.go @@ -0,0 +1,112 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package client + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + + "github.com/decred/politeia/politeiad/api/v1/identity" + "github.com/decred/politeia/util" +) + +// Client provides a client for interacting with the politeiad API. +type Client struct { + rpcHost string + rpcUser string + rpcPass string + http *http.Client + pid *identity.PublicIdentity +} + +// ErrorReply represents the request body that is returned from politeaid when +// an error occurs. PluginID will only be populated if this is a plugin error. +type ErrorReply struct { + PluginID string + ErrorCode int + ErrorContext []string +} + +// Error represents a politeiad error. Error is returned anytime the politeiad +// is not a 200. +type Error struct { + HTTPCode int + ErrorReply ErrorReply +} + +// Error satisfies the error interface. +func (e Error) Error() string { + if e.ErrorReply.PluginID != "" { + return fmt.Sprintf("politeiad plugin error: %v %v %v", + e.HTTPCode, e.ErrorReply.PluginID, e.ErrorReply.ErrorCode) + } + return fmt.Sprintf("politeiad error: %v %v", + e.HTTPCode, e.ErrorReply.ErrorCode) +} + +// makeReq makes a politeiad http request to the method and route provided, +// serializing the provided object as the request body, and returning a byte +// slice of the repsonse body. An Error is returned if politeiad responds with +// anything other than a 200 http status code. +func (c *Client) makeReq(ctx context.Context, method string, route string, v interface{}) ([]byte, error) { + // Serialize body + var ( + reqBody []byte + err error + ) + if v != nil { + reqBody, err = json.Marshal(v) + if err != nil { + return nil, err + } + } + + // Send request + fullRoute := c.rpcHost + route + req, err := http.NewRequestWithContext(ctx, method, + fullRoute, bytes.NewReader(reqBody)) + if err != nil { + return nil, err + } + req.SetBasicAuth(c.rpcUser, c.rpcPass) + r, err := c.http.Do(req) + if err != nil { + return nil, err + } + defer r.Body.Close() + + // Handle reply + if r.StatusCode != http.StatusOK { + var e ErrorReply + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&e); err != nil { + return nil, fmt.Errorf("status code %v: %v", r.StatusCode, err) + } + return nil, Error{ + HTTPCode: r.StatusCode, + ErrorReply: e, + } + } + + respBody := util.ConvertBodyToByteArray(r.Body, false) + return respBody, nil +} + +// New returns a new politeiad client. +func New(rpcHost, rpcUser, rpcPass string, pid *identity.PublicIdentity) (*Client, error) { + h, err := util.NewHTTPClient(false, "") + if err != nil { + return nil, err + } + return &Client{ + rpcHost: rpcHost, + rpcUser: rpcUser, + rpcPass: rpcPass, + http: h, + }, nil +} diff --git a/politeiad/client/pdv1.go b/politeiad/client/pdv1.go new file mode 100644 index 000000000..f32c09262 --- /dev/null +++ b/politeiad/client/pdv1.go @@ -0,0 +1,544 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package client + +import ( + "context" + "encoding/hex" + "encoding/json" + "net/http" + + pdv1 "github.com/decred/politeia/politeiad/api/v1" + "github.com/decred/politeia/util" +) + +// NewRecord sends a NewRecord request to the politeiad v1 API. +func (c *Client) NewRecord(ctx context.Context, metadata []pdv1.MetadataStream, files []pdv1.File) (*pdv1.CensorshipRecord, error) { + // Setup request + challenge, err := util.Random(pdv1.ChallengeSize) + if err != nil { + return nil, err + } + nr := pdv1.NewRecord{ + Challenge: hex.EncodeToString(challenge), + Metadata: metadata, + Files: files, + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, pdv1.NewRecordRoute, nr) + if err != nil { + return nil, err + } + + // Decode reply + var nrr pdv1.NewRecordReply + err = json.Unmarshal(resBody, &nrr) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(c.pid, challenge, nrr.Response) + if err != nil { + return nil, err + } + + return &nrr.CensorshipRecord, nil +} + +func (c *Client) updateRecord(ctx context.Context, route, token string, mdAppend, mdOverwrite []pdv1.MetadataStream, filesAdd []pdv1.File, filesDel []string) (*pdv1.Record, error) { + // Setup request + challenge, err := util.Random(pdv1.ChallengeSize) + if err != nil { + return nil, err + } + ur := pdv1.UpdateRecord{ + Token: token, + Challenge: hex.EncodeToString(challenge), + MDOverwrite: mdOverwrite, + FilesAdd: filesAdd, + FilesDel: filesDel, + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, route, ur) + if err != nil { + return nil, err + } + + // Decode reply + var urr pdv1.UpdateRecordReply + err = json.Unmarshal(resBody, &urr) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(c.pid, challenge, urr.Response) + if err != nil { + return nil, err + } + + return &urr.Record, nil +} + +// UpdateUnvetted sends a UpdateRecord request to the unvetted politeiad v1 +// API. +func (c *Client) UpdateUnvetted(ctx context.Context, token string, mdAppend, mdOverwrite []pdv1.MetadataStream, filesAdd []pdv1.File, filesDel []string) (*pdv1.Record, error) { + return c.updateRecord(ctx, pdv1.UpdateUnvettedRoute, token, + mdAppend, mdOverwrite, filesAdd, filesDel) +} + +// UpdateVetted sends a UpdateRecord request to the vetted politeiad v1 API. +func (c *Client) UpdateVetted(ctx context.Context, token string, mdAppend, mdOverwrite []pdv1.MetadataStream, filesAdd []pdv1.File, filesDel []string) (*pdv1.Record, error) { + return c.updateRecord(ctx, pdv1.UpdateVettedRoute, token, + mdAppend, mdOverwrite, filesAdd, filesDel) +} + +// UpdateUnvettedMetadata sends a UpdateVettedMetadata request to the politeiad +// v1 API. +func (c *Client) UpdateUnvettedMetadata(ctx context.Context, token string, mdAppend, mdOverwrite []pdv1.MetadataStream) error { + // Setup request + challenge, err := util.Random(pdv1.ChallengeSize) + if err != nil { + return err + } + uum := pdv1.UpdateUnvettedMetadata{ + Challenge: hex.EncodeToString(challenge), + Token: token, + MDAppend: mdAppend, + MDOverwrite: mdOverwrite, + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, + pdv1.UpdateUnvettedMetadataRoute, uum) + if err != nil { + return nil + } + + // Decode reply + var uumr pdv1.UpdateUnvettedMetadataReply + err = json.Unmarshal(resBody, &uumr) + if err != nil { + return err + } + err = util.VerifyChallenge(c.pid, challenge, uumr.Response) + if err != nil { + return err + } + + return nil +} + +// UpdateVettedMetadata sends a UpdateVettedMetadata request to the politeiad +// v1 API. +func (c *Client) UpdateVettedMetadata(ctx context.Context, token string, mdAppend, mdOverwrite []pdv1.MetadataStream) error { + // Setup request + challenge, err := util.Random(pdv1.ChallengeSize) + if err != nil { + return err + } + uvm := pdv1.UpdateVettedMetadata{ + Challenge: hex.EncodeToString(challenge), + Token: token, + MDAppend: mdAppend, + MDOverwrite: mdOverwrite, + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, + pdv1.UpdateVettedMetadataRoute, uvm) + if err != nil { + return nil + } + + // Decode reply + var uvmr pdv1.UpdateVettedMetadataReply + err = json.Unmarshal(resBody, &uvmr) + if err != nil { + return err + } + err = util.VerifyChallenge(c.pid, challenge, uvmr.Response) + if err != nil { + return err + } + + return nil +} + +// SetUnvettedStatus sends a SetUnvettedStatus request to the politeiad v1 +// API. +func (c *Client) SetUnvettedStatus(ctx context.Context, token string, status pdv1.RecordStatusT, mdAppend, mdOverwrite []pdv1.MetadataStream) (*pdv1.Record, error) { + // Setup request + challenge, err := util.Random(pdv1.ChallengeSize) + if err != nil { + return nil, err + } + sus := pdv1.SetUnvettedStatus{ + Challenge: hex.EncodeToString(challenge), + Token: token, + Status: status, + MDAppend: mdAppend, + MDOverwrite: mdOverwrite, + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, + pdv1.SetUnvettedStatusRoute, sus) + if err != nil { + return nil, err + } + + // Decode reply + var susr pdv1.SetUnvettedStatusReply + err = json.Unmarshal(resBody, &susr) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(c.pid, challenge, susr.Response) + if err != nil { + return nil, err + } + + return &susr.Record, nil +} + +// SetVettedStatus sends a SetVettedStatus request to the politeiad v1 API. +func (c *Client) SetVettedStatus(ctx context.Context, token string, status pdv1.RecordStatusT, mdAppend, mdOverwrite []pdv1.MetadataStream) (*pdv1.Record, error) { + // Setup request + challenge, err := util.Random(pdv1.ChallengeSize) + if err != nil { + return nil, err + } + svs := pdv1.SetVettedStatus{ + Challenge: hex.EncodeToString(challenge), + Token: token, + Status: status, + MDAppend: mdAppend, + MDOverwrite: mdOverwrite, + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, + pdv1.SetVettedStatusRoute, svs) + if err != nil { + return nil, err + } + + // Decode reply + var svsr pdv1.SetVettedStatusReply + err = json.Unmarshal(resBody, &svsr) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(c.pid, challenge, svsr.Response) + if err != nil { + return nil, err + } + + return &svsr.Record, nil +} + +// GetUnvetted sends a GetUnvetted request to the politeiad v1 API. +func (c *Client) GetUnvetted(ctx context.Context, token, version string) (*pdv1.Record, error) { + // Setup request + challenge, err := util.Random(pdv1.ChallengeSize) + if err != nil { + return nil, err + } + gu := pdv1.GetUnvetted{ + Challenge: hex.EncodeToString(challenge), + Token: token, + Version: version, + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, pdv1.GetUnvettedRoute, gu) + if err != nil { + return nil, err + } + + // Decode reply + var gur pdv1.GetUnvettedReply + err = json.Unmarshal(resBody, &gur) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(c.pid, challenge, gur.Response) + if err != nil { + return nil, err + } + + return &gur.Record, nil +} + +// GetVetted sends a GetVetted request to the politeiad v1 API. +func (c *Client) GetVetted(ctx context.Context, token, version string) (*pdv1.Record, error) { + // Setup request + challenge, err := util.Random(pdv1.ChallengeSize) + if err != nil { + return nil, err + } + gv := pdv1.GetVetted{ + Challenge: hex.EncodeToString(challenge), + Token: token, + Version: version, + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, + pdv1.GetVettedRoute, gv) + if err != nil { + return nil, err + } + + // Decode reply + var gvr pdv1.GetVettedReply + err = json.Unmarshal(resBody, &gvr) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(c.pid, challenge, gvr.Response) + if err != nil { + return nil, err + } + + return &gvr.Record, nil +} + +// GetUnvettedTimestamps sends a GetUnvettedTimestamps request to the politeiad +// v1 API. +func (c *Client) GetUnvettedTimestamps(ctx context.Context, token, version string) (*pdv1.RecordTimestamps, error) { + // Setup request + challenge, err := util.Random(pdv1.ChallengeSize) + if err != nil { + return nil, err + } + gut := pdv1.GetUnvettedTimestamps{ + Challenge: hex.EncodeToString(challenge), + Token: token, + Version: version, + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, + pdv1.GetUnvettedTimestampsRoute, gut) + if err != nil { + return nil, err + } + + // Decode reply + var reply pdv1.GetUnvettedTimestampsReply + err = json.Unmarshal(resBody, &reply) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(c.pid, challenge, reply.Response) + if err != nil { + return nil, err + } + + return &reply.RecordTimestamps, nil +} + +// GetVettedTimestamps sends a GetVettedTimestamps request to the politeiad +// v1 API. +func (c *Client) GetVettedTimestamps(ctx context.Context, token, version string) (*pdv1.RecordTimestamps, error) { + // Setup request + challenge, err := util.Random(pdv1.ChallengeSize) + if err != nil { + return nil, err + } + gvt := pdv1.GetVettedTimestamps{ + Challenge: hex.EncodeToString(challenge), + Token: token, + Version: version, + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, + pdv1.GetVettedTimestampsRoute, gvt) + if err != nil { + return nil, err + } + + // Decode reply + var reply pdv1.GetVettedTimestampsReply + err = json.Unmarshal(resBody, &reply) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(c.pid, challenge, reply.Response) + if err != nil { + return nil, err + } + + return &reply.RecordTimestamps, nil +} + +// Inventory sends a Inventory request to the politeiad v1 API. +func (c *Client) Inventory(ctx context.Context) (*pdv1.InventoryReply, error) { + // Setup request + challenge, err := util.Random(pdv1.ChallengeSize) + if err != nil { + return nil, err + } + ibs := pdv1.Inventory{ + Challenge: hex.EncodeToString(challenge), + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, + pdv1.InventoryRoute, ibs) + if err != nil { + return nil, err + } + + // Decode reply + var ir pdv1.InventoryReply + err = json.Unmarshal(resBody, &ir) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(c.pid, challenge, ir.Response) + if err != nil { + return nil, err + } + + return &ir, nil +} + +// InventoryByStatus sends a InventoryByStatus request to the politeiad v1 API. +func (c *Client) InventoryByStatus(ctx context.Context) (*pdv1.InventoryByStatusReply, error) { + // Setup request + challenge, err := util.Random(pdv1.ChallengeSize) + if err != nil { + return nil, err + } + ibs := pdv1.InventoryByStatus{ + Challenge: hex.EncodeToString(challenge), + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, + pdv1.InventoryByStatusRoute, ibs) + if err != nil { + return nil, err + } + + // Decode reply + var ibsr pdv1.InventoryByStatusReply + err = json.Unmarshal(resBody, &ibsr) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(c.pid, challenge, ibsr.Response) + if err != nil { + return nil, err + } + + return &ibsr, nil +} + +// PluginCommand sends a PluginCommand request to the politeiad v1 API. +func (c *Client) PluginCommand(ctx context.Context, pluginID, cmd, payload string) (string, error) { + // Setup request + challenge, err := util.Random(pdv1.ChallengeSize) + if err != nil { + return "", err + } + pc := pdv1.PluginCommand{ + Challenge: hex.EncodeToString(challenge), + ID: pluginID, + Command: cmd, + CommandID: "", + Payload: payload, + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, + pdv1.PluginCommandRoute, pc) + if err != nil { + return "", err + } + + // Decode reply + var pcr pdv1.PluginCommandReply + err = json.Unmarshal(resBody, &pcr) + if err != nil { + return "", err + } + err = util.VerifyChallenge(c.pid, challenge, pcr.Response) + if err != nil { + return "", err + } + + return pcr.Payload, nil +} + +// PluginCommandBatch sends a PluginCommandBatch request to the politeiad v1 API. +func (c *Client) PluginCommandBatch(ctx context.Context, cmds []pdv1.PluginCommandV2) ([]pdv1.PluginCommandReplyV2, error) { + // Setup request + challenge, err := util.Random(pdv1.ChallengeSize) + if err != nil { + return nil, err + } + pcb := pdv1.PluginCommandBatch{ + Challenge: hex.EncodeToString(challenge), + Commands: cmds, + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, + pdv1.PluginCommandBatchRoute, pcb) + if err != nil { + return nil, err + } + + // Decode reply + var pcbr pdv1.PluginCommandBatchReply + err = json.Unmarshal(resBody, &pcbr) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(c.pid, challenge, pcbr.Response) + if err != nil { + return nil, err + } + + return pcbr.Replies, nil +} + +// PluginInventory sends a PluginInventory request to the politeiad v1 API. +func (c *Client) PluginInventory(ctx context.Context) ([]pdv1.Plugin, error) { + // Setup request + challenge, err := util.Random(pdv1.ChallengeSize) + if err != nil { + return nil, err + } + pi := pdv1.PluginInventory{ + Challenge: hex.EncodeToString(challenge), + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, + pdv1.PluginInventoryRoute, pi) + if err != nil { + return nil, err + } + + // Receive reply + var pir pdv1.PluginInventoryReply + err = json.Unmarshal(resBody, &pir) + if err != nil { + return nil, err + } + + // Verify challenge + err = util.VerifyChallenge(c.pid, challenge, pir.Response) + if err != nil { + return nil, err + } + + return pir.Plugins, nil +} diff --git a/politeiawww/politeiad.go b/politeiawww/politeiad.go index 37a89c76e..2010c2140 100644 --- a/politeiawww/politeiad.go +++ b/politeiawww/politeiad.go @@ -7,12 +7,10 @@ package main import ( "bytes" "context" - "encoding/hex" "encoding/json" "fmt" "net/http" - pd "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/util" ) @@ -39,6 +37,10 @@ func (e pdError) Error() string { // makeRequest makes a politeiad http request to the method and route provided, // serializing the provided object as the request body. A pdError is returned // if politeiad does not respond with a 200. +// +// This method has been DEPRECATED. The politeiad client on the politeiawww +// context should be used instead. This method can be removed once all of the +// cms invocations have been switched over to use the politeaid client. func (p *politeiawww) makeRequest(ctx context.Context, method string, route string, v interface{}) ([]byte, error) { var ( reqBody []byte @@ -83,491 +85,3 @@ func (p *politeiawww) makeRequest(ctx context.Context, method string, route stri responseBody := util.ConvertBodyToByteArray(r.Body, false) return responseBody, nil } - -// newRecord creates a record in politeiad. This route returns the censorship -// record from the new created record. -func (p *politeiawww) newRecord(ctx context.Context, metadata []pd.MetadataStream, files []pd.File) (*pd.CensorshipRecord, error) { - // Setup request - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - nr := pd.NewRecord{ - Challenge: hex.EncodeToString(challenge), - Metadata: metadata, - Files: files, - } - - // Send request - resBody, err := p.makeRequest(ctx, http.MethodPost, pd.NewRecordRoute, nr) - if err != nil { - return nil, err - } - - // Receive reply - var nrr pd.NewRecordReply - err = json.Unmarshal(resBody, &nrr) - if err != nil { - return nil, err - } - - // Verify challenge - err = util.VerifyChallenge(p.cfg.Identity, challenge, nrr.Response) - if err != nil { - return nil, err - } - - return &nrr.CensorshipRecord, nil -} - -// updateRecord updates a record in politeiad. This can be used to update -// unvetted or vetted records depending on the route that is provided. -func (p *politeiawww) updateRecord(ctx context.Context, route, token string, mdAppend, mdOverwrite []pd.MetadataStream, filesAdd []pd.File, filesDel []string) (*pd.Record, error) { - // Setup request - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - ur := pd.UpdateRecord{ - Token: token, - Challenge: hex.EncodeToString(challenge), - MDOverwrite: mdOverwrite, - FilesAdd: filesAdd, - FilesDel: filesDel, - } - - // Send request - resBody, err := p.makeRequest(ctx, http.MethodPost, route, ur) - if err != nil { - return nil, err - } - - // Receive reply - var urr pd.UpdateRecordReply - err = json.Unmarshal(resBody, &urr) - if err != nil { - return nil, err - } - - // Verify challenge - err = util.VerifyChallenge(p.cfg.Identity, challenge, urr.Response) - if err != nil { - return nil, err - } - - return &urr.Record, nil -} - -// updateUnvetted updates an unvetted record in politeiad. -func (p *politeiawww) updateUnvetted(ctx context.Context, token string, mdAppend, mdOverwrite []pd.MetadataStream, filesAdd []pd.File, filesDel []string) (*pd.Record, error) { - return p.updateRecord(ctx, pd.UpdateUnvettedRoute, token, - mdAppend, mdOverwrite, filesAdd, filesDel) -} - -// updateVetted updates a vetted record in politeiad. -func (p *politeiawww) updateVetted(ctx context.Context, token string, mdAppend, mdOverwrite []pd.MetadataStream, filesAdd []pd.File, filesDel []string) (*pd.Record, error) { - return p.updateRecord(ctx, pd.UpdateVettedRoute, token, - mdAppend, mdOverwrite, filesAdd, filesDel) -} - -// updateUnvettedMetadata updates the metadata of a unvetted record in politeiad. -func (p *politeiawww) updateUnvettedMetadata(ctx context.Context, token string, mdAppend, mdOverwrite []pd.MetadataStream) error { - // Setup request - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return err - } - uum := pd.UpdateUnvettedMetadata{ - Challenge: hex.EncodeToString(challenge), - Token: token, - MDAppend: mdAppend, - MDOverwrite: mdOverwrite, - } - - // Send request - resBody, err := p.makeRequest(ctx, http.MethodPost, - pd.UpdateUnvettedMetadataRoute, uum) - if err != nil { - return nil - } - - // Receive reply - var uumr pd.UpdateUnvettedMetadataReply - err = json.Unmarshal(resBody, &uumr) - if err != nil { - return err - } - - // Verify challenge - err = util.VerifyChallenge(p.cfg.Identity, challenge, uumr.Response) - if err != nil { - return err - } - - return nil -} - -// updateVettedMetadata updates the metadata of a vetted record in politeiad. -func (p *politeiawww) updateVettedMetadata(ctx context.Context, token string, mdAppend, mdOverwrite []pd.MetadataStream) error { - // Setup request - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return err - } - uvm := pd.UpdateVettedMetadata{ - Challenge: hex.EncodeToString(challenge), - Token: token, - MDAppend: mdAppend, - MDOverwrite: mdOverwrite, - } - - // Send request - resBody, err := p.makeRequest(ctx, http.MethodPost, - pd.UpdateVettedMetadataRoute, uvm) - if err != nil { - return nil - } - - // Receive reply - var uvmr pd.UpdateVettedMetadataReply - err = json.Unmarshal(resBody, &uvmr) - if err != nil { - return err - } - - // Verify challenge - err = util.VerifyChallenge(p.cfg.Identity, challenge, uvmr.Response) - if err != nil { - return err - } - - return nil -} - -// setUnvettedStatus sets the status of a unvetted record in politeiad. -func (p *politeiawww) setUnvettedStatus(ctx context.Context, token string, status pd.RecordStatusT, mdAppend, mdOverwrite []pd.MetadataStream) (*pd.Record, error) { - // Setup request - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - sus := pd.SetUnvettedStatus{ - Challenge: hex.EncodeToString(challenge), - Token: token, - Status: status, - MDAppend: mdAppend, - MDOverwrite: mdOverwrite, - } - - // Send request - resBody, err := p.makeRequest(ctx, http.MethodPost, - pd.SetUnvettedStatusRoute, sus) - if err != nil { - return nil, err - } - - // Receive reply - var susr pd.SetUnvettedStatusReply - err = json.Unmarshal(resBody, &susr) - if err != nil { - return nil, err - } - - // Verify challenge - err = util.VerifyChallenge(p.cfg.Identity, challenge, susr.Response) - if err != nil { - return nil, err - } - - return &susr.Record, nil -} - -// setVettedStatus sets the status of a vetted record in politeiad. -func (p *politeiawww) setVettedStatus(ctx context.Context, token string, status pd.RecordStatusT, mdAppend, mdOverwrite []pd.MetadataStream) (*pd.Record, error) { - // Setup request - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - svs := pd.SetVettedStatus{ - Challenge: hex.EncodeToString(challenge), - Token: token, - Status: status, - MDAppend: mdAppend, - MDOverwrite: mdOverwrite, - } - - // Send request - resBody, err := p.makeRequest(ctx, http.MethodPost, - pd.SetVettedStatusRoute, svs) - if err != nil { - return nil, err - } - - // Receive reply - var svsr pd.SetVettedStatusReply - err = json.Unmarshal(resBody, &svsr) - if err != nil { - return nil, err - } - - // Verify challenge - err = util.VerifyChallenge(p.cfg.Identity, challenge, svsr.Response) - if err != nil { - return nil, err - } - - return &svsr.Record, nil -} - -// getUnvetted retrieves an unvetted record from politeiad. -func (p *politeiawww) getUnvetted(ctx context.Context, token, version string) (*pd.Record, error) { - // Setup request - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - gu := pd.GetUnvetted{ - Challenge: hex.EncodeToString(challenge), - Token: token, - Version: version, - } - - // Send request - resBody, err := p.makeRequest(ctx, http.MethodPost, pd.GetUnvettedRoute, gu) - if err != nil { - return nil, err - } - - // Receive reply - var gur pd.GetUnvettedReply - err = json.Unmarshal(resBody, &gur) - if err != nil { - return nil, err - } - - // Verify challenge - err = util.VerifyChallenge(p.cfg.Identity, challenge, gur.Response) - if err != nil { - return nil, err - } - - return &gur.Record, nil -} - -// getUnvettedTimestamps retrieves the timestamps for an unvetted record. -func (p *politeiawww) getUnvettedTimestamps(ctx context.Context, token, version string) (*pd.RecordTimestamps, error) { - // Setup request - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - gut := pd.GetUnvettedTimestamps{ - Challenge: hex.EncodeToString(challenge), - Token: token, - Version: version, - } - - // Send request - resBody, err := p.makeRequest(ctx, http.MethodPost, - pd.GetUnvettedTimestampsRoute, gut) - if err != nil { - return nil, err - } - - // Receive reply - var reply pd.GetUnvettedTimestampsReply - err = json.Unmarshal(resBody, &reply) - if err != nil { - return nil, err - } - - // Verify challenge - err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) - if err != nil { - return nil, err - } - - return &reply.RecordTimestamps, nil -} - -// getVetted retrieves a vetted record from politeiad. -func (p *politeiawww) getVetted(ctx context.Context, token, version string) (*pd.Record, error) { - // Setup request - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - gv := pd.GetVetted{ - Challenge: hex.EncodeToString(challenge), - Token: token, - Version: version, - } - - // Send request - resBody, err := p.makeRequest(ctx, http.MethodPost, - pd.GetVettedRoute, gv) - if err != nil { - return nil, err - } - - // Receive reply - var gvr pd.GetVettedReply - err = json.Unmarshal(resBody, &gvr) - if err != nil { - return nil, err - } - - // Verify challenge - err = util.VerifyChallenge(p.cfg.Identity, challenge, gvr.Response) - if err != nil { - return nil, err - } - - return &gvr.Record, nil -} - -// getVettedTimestamps retrieves the timestamps for an unvetted record. -func (p *politeiawww) getVettedTimestamps(ctx context.Context, token, version string) (*pd.RecordTimestamps, error) { - // Setup request - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - gvt := pd.GetVettedTimestamps{ - Challenge: hex.EncodeToString(challenge), - Token: token, - Version: version, - } - - // Send request - resBody, err := p.makeRequest(ctx, http.MethodPost, - pd.GetVettedTimestampsRoute, gvt) - if err != nil { - return nil, err - } - - // Receive reply - var reply pd.GetVettedTimestampsReply - err = json.Unmarshal(resBody, &reply) - if err != nil { - return nil, err - } - - // Verify challenge - err = util.VerifyChallenge(p.cfg.Identity, challenge, reply.Response) - if err != nil { - return nil, err - } - - return &reply.RecordTimestamps, nil -} - -// pluginInventory requests the plugin inventory from politeiad and returns -// inventoryByStatus retrieves the censorship record tokens filtered by status. -func (p *politeiawww) inventoryByStatus(ctx context.Context) (*pd.InventoryByStatusReply, error) { - // Setup request - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - ibs := pd.InventoryByStatus{ - Challenge: hex.EncodeToString(challenge), - } - - // Send request - resBody, err := p.makeRequest(ctx, http.MethodPost, - pd.InventoryByStatusRoute, ibs) - if err != nil { - return nil, err - } - - // Receive reply - var ibsr pd.InventoryByStatusReply - err = json.Unmarshal(resBody, &ibsr) - if err != nil { - return nil, err - } - - // Verify challenge - err = util.VerifyChallenge(p.cfg.Identity, challenge, ibsr.Response) - if err != nil { - return nil, err - } - - return &ibsr, nil -} - -// pluginInventory requests the plugin inventory from politeiad and returns -// the available plugins slice. -func (p *politeiawww) pluginInventory(ctx context.Context) ([]pd.Plugin, error) { - // Setup request - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return nil, err - } - pi := pd.PluginInventory{ - Challenge: hex.EncodeToString(challenge), - } - - // Send request - resBody, err := p.makeRequest(ctx, http.MethodPost, - pd.PluginInventoryRoute, pi) - if err != nil { - return nil, err - } - - // Receive reply - var pir pd.PluginInventoryReply - err = json.Unmarshal(resBody, &pir) - if err != nil { - return nil, err - } - - // Verify challenge - err = util.VerifyChallenge(p.cfg.Identity, challenge, pir.Response) - if err != nil { - return nil, err - } - - return pir.Plugins, nil -} - -// pluginCommand fires a plugin command on politeiad and returns the reply -// payload. -func (p *politeiawww) pluginCommand(ctx context.Context, pluginID, cmd, payload string) (string, error) { - // Setup request - challenge, err := util.Random(pd.ChallengeSize) - if err != nil { - return "", err - } - pc := pd.PluginCommand{ - Challenge: hex.EncodeToString(challenge), - ID: pluginID, - Command: cmd, - CommandID: "", - Payload: payload, - } - - // Send request - resBody, err := p.makeRequest(ctx, http.MethodPost, - pd.PluginCommandRoute, pc) - if err != nil { - return "", err - } - - // Receive reply - var pcr pd.PluginCommandReply - err = json.Unmarshal(resBody, &pcr) - if err != nil { - return "", err - } - - // Verify challenge - err = util.VerifyChallenge(p.cfg.Identity, challenge, pcr.Response) - if err != nil { - return "", err - } - - return pcr.Payload, nil -} diff --git a/politeiawww/politeiawww.go b/politeiawww/politeiawww.go index f977d89f5..f27b218d6 100644 --- a/politeiawww/politeiawww.go +++ b/politeiawww/politeiawww.go @@ -17,6 +17,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/decred/dcrd/chaincfg/v3" "github.com/decred/politeia/politeiad/api/v1/mime" + pdclient "github.com/decred/politeia/politeiad/client" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/cmsdatabase" "github.com/decred/politeia/politeiawww/codetracker" @@ -66,6 +67,7 @@ type politeiawww struct { params *chaincfg.Params router *mux.Router auth *mux.Router // CSRF protected subrouter + politeiad *pdclient.Client client *http.Client smtp *smtp db user.Database diff --git a/politeiawww/www.go b/politeiawww/www.go index 6f040f3bd..da4555fbc 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -872,6 +872,13 @@ func _main() error { auth := router.NewRoute().Subrouter() auth.Use(csrfMiddleware) + // Setup the politeiad client + pdc, err := pdclient.New(loadedCfg.RPCHost, loadedCfg.RPCUser, + loadedCfg.RPCPass, loadedCfg.Identity) + if err != nil { + return err + } + // Setup user database log.Infof("User db: %v", loadedCfg.UserDB) @@ -953,6 +960,7 @@ func _main() error { params: activeNetParams.Params, router: router, auth: auth, + politeiad: pdc, client: client, smtp: smtp, db: userDB, From 3909c966c64b47ee401e24e1f9647469025959e1 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 20 Jan 2021 14:06:13 -0600 Subject: [PATCH 239/449] Fix all politeiawww build errors. --- politeiawww/comments.go | 75 +++++++++++---- politeiawww/decred.go | 12 +-- politeiawww/eventmanager.go | 13 ++- politeiawww/pi.go | 28 +----- politeiawww/piwww.go | 184 +++++++++--------------------------- politeiawww/plugin.go | 2 +- politeiawww/proposals.go | 2 +- politeiawww/records.go | 4 +- politeiawww/ticketvote.go | 2 +- politeiawww/www.go | 29 +----- 10 files changed, 128 insertions(+), 223 deletions(-) diff --git a/politeiawww/comments.go b/politeiawww/comments.go index 12918b925..76707ac51 100644 --- a/politeiawww/comments.go +++ b/politeiawww/comments.go @@ -4,23 +4,63 @@ package main -import ( - "context" - "encoding/json" - "errors" - "fmt" - "net/http" - "runtime/debug" - "strings" - "time" - - "github.com/decred/politeia/politeiad/plugins/comments" - cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" - piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" - "github.com/decred/politeia/politeiawww/user" - "github.com/decred/politeia/util" - "github.com/google/uuid" -) +/* +func convertCommentVoteFromPi(cv piv1.CommentVoteT) comments.VoteT { + switch cv { + case piv1.CommentVoteDownvote: + return comments.VoteUpvote + case piv1.CommentVoteUpvote: + return comments.VoteDownvote + } + return comments.VoteInvalid +} + +func convertCommentFromPlugin(c comments.Comment) piv1.Comment { + return piv1.Comment{ + UserID: c.UserID, + Username: "", // Intentionally omitted, needs to be pulled from userdb + Token: c.Token, + ParentID: c.ParentID, + Comment: c.Comment, + PublicKey: c.PublicKey, + Signature: c.Signature, + CommentID: c.CommentID, + Timestamp: c.Timestamp, + Receipt: c.Receipt, + Downvotes: c.Downvotes, + Upvotes: c.Upvotes, + Censored: c.Deleted, + Reason: c.Reason, + } +} + +func convertCommentVoteFromPlugin(v comments.VoteT) piv1.CommentVoteT { + switch v { + case comments.VoteDownvote: + return piv1.CommentVoteDownvote + case comments.VoteUpvote: + return piv1.CommentVoteUpvote + } + return piv1.CommentVoteInvalid +} + +func convertCommentVoteDetailsFromPlugin(cv []comments.CommentVote) []piv1.CommentVoteDetails { + c := make([]piv1.CommentVoteDetails, 0, len(cv)) + for _, v := range cv { + c = append(c, piv1.CommentVoteDetails{ + UserID: v.UserID, + Token: v.Token, + CommentID: v.CommentID, + Vote: convertCommentVoteFromPlugin(v.Vote), + PublicKey: v.PublicKey, + Signature: v.Signature, + Timestamp: v.Timestamp, + Receipt: v.Receipt, + }) + } + return c +} + func convertProofFromCommentsPlugin(p comments.Proof) cmv1.Proof { return cmv1.Proof{ @@ -653,3 +693,4 @@ func respondWithCommentsError(w http.ResponseWriter, r *http.Request, format str return } } +*/ diff --git a/politeiawww/decred.go b/politeiawww/decred.go index 93500f78d..a3fb36f5f 100644 --- a/politeiawww/decred.go +++ b/politeiawww/decred.go @@ -24,10 +24,10 @@ func (p *politeiawww) decredGetComments(ctx context.Context, token string) ([]de } // Execute plugin command - reply, err := p.pluginCommand(ctx, decredplugin.ID, decredplugin.CmdGetComments, - string(payload)) + reply, err := p.politeiad.PluginCommand(ctx, decredplugin.ID, + decredplugin.CmdGetComments, string(payload)) if err != nil { - return nil, fmt.Errorf("pluginCommand %v %v: %v", + return nil, fmt.Errorf("PluginCommand %v %v: %v", decredplugin.ID, decredplugin.CmdGetComments, err) } @@ -48,10 +48,10 @@ func (p *politeiawww) decredBestBlock(ctx context.Context) (uint32, error) { } // Execute plugin command - reply, err := p.pluginCommand(ctx, decredplugin.ID, decredplugin.CmdBestBlock, - string(payload)) + reply, err := p.politeiad.PluginCommand(ctx, decredplugin.ID, + decredplugin.CmdBestBlock, string(payload)) if err != nil { - return 0, fmt.Errorf("pluginCommand %v %v: %v", + return 0, fmt.Errorf("PluginCommand %v %v: %v", decredplugin.ID, decredplugin.CmdBestBlock, err) } diff --git a/politeiawww/eventmanager.go b/politeiawww/eventmanager.go index 7c51f978b..fad3cd05d 100644 --- a/politeiawww/eventmanager.go +++ b/politeiawww/eventmanager.go @@ -383,10 +383,15 @@ func (p *politeiawww) notifyParentAuthorOnComment(d dataProposalComment, proposa g := comments.Get{ CommentIDs: []uint32{d.parentID}, } - parentComment, err := p.commentsGet(context.Background(), g) - if err != nil { - return err - } + // TODO + _ = g + var parentComment comments.GetReply + /* + parentComment, err := p.commentsGet(context.Background(), g) + if err != nil { + return err + } + */ userID, err := uuid.Parse(parentComment.Comments[0].UserID) if err != nil { return err diff --git a/politeiawww/pi.go b/politeiawww/pi.go index 71e634ee1..37424062f 100644 --- a/politeiawww/pi.go +++ b/politeiawww/pi.go @@ -6,40 +6,16 @@ package main import ( "context" - "encoding/json" "github.com/decred/politeia/politeiad/plugins/pi" ) // proposalInv returns the pi plugin proposal inventory. func (p *politeiawww) proposalInv(ctx context.Context, inv pi.ProposalInv) (*pi.ProposalInvReply, error) { - b, err := json.Marshal(inv) - if err != nil { - return nil, err - } - r, err := p.pluginCommand(ctx, pi.ID, - pi.CmdProposalInv, string(b)) - if err != nil { - return nil, err - } - var ir pi.ProposalInvReply - err = json.Unmarshal([]byte(r), &ir) - if err != nil { - return nil, err - } - return &ir, nil + return nil, nil } // piVoteInventory returns the pi plugin vote inventory. func (p *politeiawww) piVoteInventory(ctx context.Context) (*pi.VoteInventoryReply, error) { - r, err := p.pluginCommand(ctx, pi.ID, pi.CmdVoteInventory, "") - if err != nil { - return nil, err - } - var vir pi.VoteInventoryReply - err = json.Unmarshal([]byte(r), &vir) - if err != nil { - return nil, err - } - return &vir, nil + return nil, nil } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index f9333e77b..0e2569d63 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -21,10 +21,9 @@ import ( "time" pdv1 "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/politeiad/plugins/comments" "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/ticketvote" - cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" + usermd "github.com/decred/politeia/politeiad/plugins/user" piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" @@ -289,15 +288,15 @@ func statusChangesDecode(payload []byte) ([]pi.StatusChange, error) { func convertProposalRecordFromPD(r pdv1.Record, state piv1.PropStateT) (*piv1.ProposalRecord, error) { // Decode metadata streams var ( - gm *pi.GeneralMetadata + um *usermd.UserMetadata sc = make([]pi.StatusChange, 0, 16) err error ) for _, v := range r.Metadata { switch v.ID { - case pi.MDStreamIDGeneralMetadata: - var gm pi.GeneralMetadata - err = json.Unmarshal([]byte(v.Payload), &gm) + case usermd.MDStreamIDUserMetadata: + var um usermd.UserMetadata + err = json.Unmarshal([]byte(v.Payload), &um) if err != nil { return nil, err } @@ -332,13 +331,13 @@ func convertProposalRecordFromPD(r pdv1.Record, state piv1.PropStateT) (*piv1.Pr // plugin command. return &piv1.ProposalRecord{ Version: r.Version, - Timestamp: gm.Timestamp, + Timestamp: r.Timestamp, State: state, Status: status, UserID: "", // Intentionally omitted Username: "", // Intentionally omitted - PublicKey: gm.PublicKey, - Signature: gm.Signature, + PublicKey: um.PublicKey, + Signature: um.Signature, Statuses: statuses, Files: files, Metadata: metadata, @@ -346,62 +345,6 @@ func convertProposalRecordFromPD(r pdv1.Record, state piv1.PropStateT) (*piv1.Pr }, nil } -func convertCommentVoteFromPi(cv piv1.CommentVoteT) comments.VoteT { - switch cv { - case piv1.CommentVoteDownvote: - return comments.VoteUpvote - case piv1.CommentVoteUpvote: - return comments.VoteDownvote - } - return comments.VoteInvalid -} - -func convertCommentFromPlugin(c comments.Comment) piv1.Comment { - return piv1.Comment{ - UserID: c.UserID, - Username: "", // Intentionally omitted, needs to be pulled from userdb - Token: c.Token, - ParentID: c.ParentID, - Comment: c.Comment, - PublicKey: c.PublicKey, - Signature: c.Signature, - CommentID: c.CommentID, - Timestamp: c.Timestamp, - Receipt: c.Receipt, - Downvotes: c.Downvotes, - Upvotes: c.Upvotes, - Censored: c.Deleted, - Reason: c.Reason, - } -} - -func convertCommentVoteFromPlugin(v comments.VoteT) piv1.CommentVoteT { - switch v { - case comments.VoteDownvote: - return piv1.CommentVoteDownvote - case comments.VoteUpvote: - return piv1.CommentVoteUpvote - } - return piv1.CommentVoteInvalid -} - -func convertCommentVoteDetailsFromPlugin(cv []comments.CommentVote) []piv1.CommentVoteDetails { - c := make([]piv1.CommentVoteDetails, 0, len(cv)) - for _, v := range cv { - c = append(c, piv1.CommentVoteDetails{ - UserID: v.UserID, - Token: v.Token, - CommentID: v.CommentID, - Vote: convertCommentVoteFromPlugin(v.Vote), - PublicKey: v.PublicKey, - Signature: v.Signature, - Timestamp: v.Timestamp, - Receipt: v.Receipt, - }) - } - return c -} - func convertVoteAuthActionFromPi(a piv1.VoteAuthActionT) ticketvote.AuthActionT { switch a { case piv1.VoteAuthActionAuthorize: @@ -669,13 +612,13 @@ func (p *politeiawww) proposalRecords(ctx context.Context, state piv1.PropStateT switch state { case piv1.PropStateUnvetted: // Unvetted politeiad record - r, err = p.getUnvetted(ctx, v.Token, v.Version) + r, err = p.politeiad.GetUnvetted(ctx, v.Token, v.Version) if err != nil { return nil, err } case piv1.PropStateVetted: // Vetted politeiad record - r, err = p.getVetted(ctx, v.Token, v.Version) + r, err = p.politeiad.GetVetted(ctx, v.Token, v.Version) if err != nil { return nil, err } @@ -779,39 +722,6 @@ func (p *politeiawww) verifyProposalMetadata(pm piv1.ProposalMetadata) error { ErrorContext: proposalNameRegex(), } } - - // Verify linkto - if pm.LinkTo != "" { - if !tokenIsFullLength(pm.LinkTo) { - return piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusPropLinkToInvalid, - ErrorContext: "invalid token", - } - } - } - - // Verify linkby - if pm.LinkBy != 0 { - min := time.Now().Unix() + p.linkByPeriodMin() - max := time.Now().Unix() + p.linkByPeriodMax() - switch { - case pm.LinkBy < min: - e := fmt.Sprintf("linkby %v is less than min required of %v", - pm.LinkBy, min) - return piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusPropLinkByInvalid, - ErrorContext: e, - } - case pm.LinkBy > max: - e := fmt.Sprintf("linkby %v is more than max allowed of %v", - pm.LinkBy, max) - return piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusPropLinkByInvalid, - ErrorContext: e, - } - } - } - return nil } @@ -1101,26 +1011,24 @@ func (p *politeiawww) processProposalNew(ctx context.Context, pn piv1.ProposalNe } // Setup metadata stream - timestamp := time.Now().Unix() - gm := pi.GeneralMetadata{ + um := usermd.UserMetadata{ UserID: usr.ID.String(), PublicKey: pn.PublicKey, Signature: pn.Signature, - Timestamp: timestamp, } - b, err := json.Marshal(gm) + b, err := json.Marshal(um) if err != nil { return nil, err } metadata := []pdv1.MetadataStream{ { - ID: pi.MDStreamIDGeneralMetadata, + ID: usermd.MDStreamIDUserMetadata, Payload: string(b), }, } // Send politeiad request - dcr, err := p.newRecord(ctx, metadata, files) + dcr, err := p.politeiad.NewRecord(ctx, metadata, files) if err != nil { return nil, err } @@ -1273,20 +1181,18 @@ func (p *politeiawww) processProposalEdit(ctx context.Context, pe piv1.ProposalE filesDel := filesToDel(curr.Files, pe.Files) // Setup politeiad metadata - timestamp := time.Now().Unix() - gm := pi.GeneralMetadata{ + um := usermd.UserMetadata{ UserID: usr.ID.String(), PublicKey: pe.PublicKey, Signature: pe.Signature, - Timestamp: timestamp, } - b, err := json.Marshal(gm) + b, err := json.Marshal(um) if err != nil { return nil, err } mdOverwrite := []pdv1.MetadataStream{ { - ID: pi.MDStreamIDGeneralMetadata, + ID: usermd.MDStreamIDUserMetadata, Payload: string(b), }, } @@ -1298,14 +1204,14 @@ func (p *politeiawww) processProposalEdit(ctx context.Context, pe piv1.ProposalE var r *pdv1.Record switch pe.State { case piv1.PropStateUnvetted: - r, err = p.updateUnvetted(ctx, pe.Token, mdAppend, mdOverwrite, - filesAdd, filesDel) + r, err = p.politeiad.UpdateUnvetted(ctx, pe.Token, + mdAppend, mdOverwrite, filesAdd, filesDel) if err != nil { return nil, err } case piv1.PropStateVetted: - r, err = p.updateVetted(ctx, pe.Token, mdAppend, mdOverwrite, - filesAdd, filesDel) + r, err = p.politeiad.UpdateVetted(ctx, pe.Token, + mdAppend, mdOverwrite, filesAdd, filesDel) if err != nil { return nil, err } @@ -1431,12 +1337,14 @@ func (p *politeiawww) processProposalSetStatus(ctx context.Context, pss piv1.Pro status := convertRecordStatusFromPropStatus(pss.Status) switch pss.State { case piv1.PropStateUnvetted: - r, err = p.setUnvettedStatus(ctx, pss.Token, status, mdAppend, mdOverwrite) + r, err = p.politeiad.SetUnvettedStatus(ctx, pss.Token, + status, mdAppend, mdOverwrite) if err != nil { return nil, err } case piv1.PropStateVetted: - r, err = p.setVettedStatus(ctx, pss.Token, status, mdAppend, mdOverwrite) + r, err = p.politeiad.SetVettedStatus(ctx, pss.Token, + status, mdAppend, mdOverwrite) if err != nil { return nil, err } @@ -2056,21 +1964,28 @@ func (p *politeiawww) setPiRoutes() { permissionPublic) // Pi routes - comments - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteCommentNew, p.handleCommentNew, - permissionLogin) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteCommentVote, p.handleCommentVote, - permissionLogin) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteCommentCensor, p.handleCommentCensor, - permissionAdmin) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteComments, p.handleComments, - permissionPublic) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteCommentVotes, p.handleCommentVotes, - permissionPublic) + /* + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteCommentNew, p.handleCommentNew, + permissionLogin) + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteCommentVote, p.handleCommentVote, + permissionLogin) + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteCommentCensor, p.handleCommentCensor, + permissionAdmin) + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteComments, p.handleComments, + permissionPublic) + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteCommentVotes, p.handleCommentVotes, + permissionPublic) + + // Comment routes + p.addRoute(http.MethodPost, cmv1.APIRoute, + cmv1.RouteTimestamps, p.handleCommentTimestamps, + permissionPublic) + */ // Pi routes - vote p.addRoute(http.MethodPost, piv1.APIRoute, @@ -2100,11 +2015,6 @@ func (p *politeiawww) setPiRoutes() { rcv1.RouteTimestamps, p.handleTimestamps, permissionPublic) - // Comment routes - p.addRoute(http.MethodPost, cmv1.APIRoute, - cmv1.RouteTimestamps, p.handleCommentTimestamps, - permissionPublic) - // Ticket vote routes p.addRoute(http.MethodPost, tkv1.APIRoute, tkv1.RouteTimestamps, p.handleTicketVoteTimestamps, diff --git a/politeiawww/plugin.go b/politeiawww/plugin.go index bb4ce2208..457f8410f 100644 --- a/politeiawww/plugin.go +++ b/politeiawww/plugin.go @@ -62,7 +62,7 @@ func (p *politeiawww) getPluginInventory() ([]plugin, error) { return nil, fmt.Errorf("max retries exceeded") } - pi, err := p.pluginInventory(ctx) + pi, err := p.politeiad.PluginInventory(ctx) if err != nil { log.Infof("cannot get politeiad plugin inventory: %v: retry in %v", err, sleepInterval) diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 8cf8322f8..9b9969bec 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -374,7 +374,7 @@ func (p *politeiawww) processTokenInventory(ctx context.Context, isAdmin bool) ( log.Tracef("processTokenInventory") // Get record inventory - ir, err := p.inventoryByStatus(ctx) + ir, err := p.politeiad.InventoryByStatus(ctx) if err != nil { return nil, err } diff --git a/politeiawww/records.go b/politeiawww/records.go index 34f22dc8a..12012e528 100644 --- a/politeiawww/records.go +++ b/politeiawww/records.go @@ -29,12 +29,12 @@ func (p *politeiawww) processTimestamps(ctx context.Context, t rcv1.Timestamps, ) switch t.State { case rcv1.StateUnvetted: - rt, err = p.getUnvettedTimestamps(ctx, t.Token, t.Version) + rt, err = p.politeiad.GetUnvettedTimestamps(ctx, t.Token, t.Version) if err != nil { return nil, err } case rcv1.StateVetted: - rt, err = p.getVettedTimestamps(ctx, t.Token, t.Version) + rt, err = p.politeiad.GetVettedTimestamps(ctx, t.Token, t.Version) if err != nil { return nil, err } diff --git a/politeiawww/ticketvote.go b/politeiawww/ticketvote.go index 266a00c62..cba1655b1 100644 --- a/politeiawww/ticketvote.go +++ b/politeiawww/ticketvote.go @@ -72,7 +72,7 @@ func (p *politeiawww) voteSummaries(ctx context.Context, tokens []string) (map[s func (p *politeiawww) voteTimestamps(ctx context.Context, token string) (*ticketvote.TimestampsReply, error) { _ = token var b []byte - r, err := p.pluginCommand(ctx, ticketvote.ID, + r, err := p.politeiad.PluginCommand(ctx, ticketvote.ID, ticketvote.CmdTimestamps, string(b)) if err != nil { return nil, err diff --git a/politeiawww/www.go b/politeiawww/www.go index da4555fbc..724890748 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -26,6 +26,7 @@ import ( "github.com/decred/politeia/mdstream" pd "github.com/decred/politeia/politeiad/api/v1" + pdclient "github.com/decred/politeia/politeiad/client" "github.com/decred/politeia/politeiad/plugins/comments" piplugin "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/ticketvote" @@ -204,30 +205,6 @@ func convertPiErrorStatusFromPiPlugin(e piplugin.ErrorCodeT) pi.ErrorStatusT { return pi.ErrorStatusInvalid } -func convertPiErrorStatusFromComments(e comments.ErrorCodeT) pi.ErrorStatusT { - switch e { - case comments.ErrorCodeTokenInvalid: - return pi.ErrorStatusPropTokenInvalid - case comments.ErrorCodePublicKeyInvalid: - return pi.ErrorStatusPublicKeyInvalid - case comments.ErrorCodeSignatureInvalid: - return pi.ErrorStatusSignatureInvalid - case comments.ErrorCodeCommentTextInvalid: - return pi.ErrorStatusCommentTextInvalid - case comments.ErrorCodeCommentNotFound: - return pi.ErrorStatusCommentNotFound - case comments.ErrorCodeUserUnauthorized: - return pi.ErrorStatusUnauthorized - case comments.ErrorCodeParentIDInvalid: - return pi.ErrorStatusCommentParentIDInvalid - case comments.ErrorCodeVoteInvalid: - return pi.ErrorStatusCommentVoteInvalid - case comments.ErrorCodeVoteChangesMax: - return pi.ErrorStatusCommentVoteChangesMax - } - return pi.ErrorStatusInvalid -} - func convertPiErrorStatusFromTicketVote(e ticketvote.ErrorCodeT) pi.ErrorStatusT { switch e { case ticketvote.ErrorCodeTokenInvalid: @@ -263,10 +240,6 @@ func convertPiErrorStatus(pluginID string, errCode int) pi.ErrorStatusT { // Pi plugin e := piplugin.ErrorCodeT(errCode) return convertPiErrorStatusFromPiPlugin(e) - case comments.ID: - // Comments plugin - e := comments.ErrorCodeT(errCode) - return convertPiErrorStatusFromComments(e) case ticketvote.ID: // Ticket vote plugin e := ticketvote.ErrorCodeT(errCode) From 1958fec93270821122413ce75775e658cd4e1028 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 20 Jan 2021 14:06:58 -0600 Subject: [PATCH 240/449] Package config. --- politeiawww/config.go | 104 +++++------------------ politeiawww/config/config.go | 100 ++++++++++++++++++++++ politeiawww/politeiawww.go | 3 +- politeiawww/sharedconfig/sharedconfig.go | 33 ------- politeiawww/testing.go | 5 +- 5 files changed, 124 insertions(+), 121 deletions(-) create mode 100644 politeiawww/config/config.go delete mode 100644 politeiawww/sharedconfig/sharedconfig.go diff --git a/politeiawww/config.go b/politeiawww/config.go index 25cda2bea..3f635efe9 100644 --- a/politeiawww/config.go +++ b/politeiawww/config.go @@ -25,10 +25,10 @@ import ( "github.com/decred/dcrd/hdkeychain/v3" "github.com/decred/politeia/politeiad/api/v1/identity" + "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/util/version" v1 "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/politeiawww/sharedconfig" "github.com/decred/politeia/util" flags "github.com/jessevdk/go-flags" ) @@ -79,11 +79,11 @@ const ( ) var ( - defaultHTTPSKeyFile = filepath.Join(sharedconfig.DefaultHomeDir, "https.key") - defaultHTTPSCertFile = filepath.Join(sharedconfig.DefaultHomeDir, "https.cert") - defaultRPCCertFile = filepath.Join(sharedconfig.DefaultHomeDir, "rpc.cert") - defaultCookieKeyFile = filepath.Join(sharedconfig.DefaultHomeDir, "cookie.key") - defaultLogDir = filepath.Join(sharedconfig.DefaultHomeDir, defaultLogDirname) + defaultHTTPSKeyFile = filepath.Join(config.DefaultHomeDir, "https.key") + defaultHTTPSCertFile = filepath.Join(config.DefaultHomeDir, "https.cert") + defaultRPCCertFile = filepath.Join(config.DefaultHomeDir, "rpc.cert") + defaultCookieKeyFile = filepath.Join(config.DefaultHomeDir, "cookie.key") + defaultLogDir = filepath.Join(config.DefaultHomeDir, defaultLogDirname) // Default start date to start pulling code statistics if none specified. defaultCodeStatStart = time.Now().Add(-1 * time.Minute * 60 * 24 * 7 * 26) // 6 months in minutes 60min * 24h * 7days * 26 weeks @@ -99,72 +99,6 @@ var ( // to parse and execute service commands specified via the -s flag. var runServiceCommand func(string) error -// config defines the configuration options for politeiawww. -// -// See loadConfig for details on the configuration load process. -type config struct { - HomeDir string `short:"A" long:"appdata" description:"Path to application home directory"` - ShowVersion bool `short:"V" long:"version" description:"Display version information and exit"` - ConfigFile string `short:"C" long:"configfile" description:"Path to configuration file"` - DataDir string `short:"b" long:"datadir" description:"Directory to store data"` - LogDir string `long:"logdir" description:"Directory to log output."` - TestNet bool `long:"testnet" description:"Use the test network"` - SimNet bool `long:"simnet" description:"Use the simulation test network"` - Profile string `long:"profile" description:"Enable HTTP profiling on given port -- NOTE port must be between 1024 and 65536"` - CookieKeyFile string `long:"cookiekey" description:"File containing the secret cookies key"` - DebugLevel string `short:"d" long:"debuglevel" description:"Logging level for all subsystems {trace, debug, info, warn, error, critical} -- You may also specify =,=,... to set the log level for individual subsystems -- Use show to list available subsystems"` - Listeners []string `long:"listen" description:"Add an interface/port to listen for connections (default all interfaces port: 4443)"` - HTTPSCert string `long:"httpscert" description:"File containing the https certificate file"` - HTTPSKey string `long:"httpskey" description:"File containing the https certificate key"` - RPCHost string `long:"rpchost" description:"Host for politeiad in this format"` - RPCCert string `long:"rpccert" description:"File containing the https certificate file"` - RPCIdentityFile string `long:"rpcidentityfile" description:"Path to file containing the politeiad identity"` - RPCUser string `long:"rpcuser" description:"RPC user name for privileged politeaid commands"` - RPCPass string `long:"rpcpass" description:"RPC password for privileged politeiad commands"` - FetchIdentity bool `long:"fetchidentity" description:"Whether or not politeiawww fetches the identity from politeiad."` - Interactive string `long:"interactive" description:"Set to i-know-this-is-a-bad-idea to turn off interactive mode during --fetchidentity."` - AdminLogFile string `long:"adminlogfile" description:"admin log filename (Default: admin.log)"` - Mode string `long:"mode" description:"Mode www runs as. Supported values: piwww, cmswww"` - - // User database settings - UserDB string `long:"userdb" description:"Database choice for the user database"` - DBHost string `long:"dbhost" description:"Database ip:port"` - DBRootCert string `long:"dbrootcert" description:"File containing the CA certificate for the database"` - DBCert string `long:"dbcert" description:"File containing the politeiawww client certificate for the database"` - DBKey string `long:"dbkey" description:"File containing the politeiawww client certificate key for the database"` - EncryptionKey string `long:"encryptionkey" description:"File containing encryption key used for encrypting user data at rest"` - OldEncryptionKey string `long:"oldencryptionkey" description:"File containing old encryption key (only set when rotating keys)"` - - // SMTP settings - MailHost string `long:"mailhost" description:"Email server address in this format: :"` - MailUser string `long:"mailuser" description:"Email server username"` - MailPass string `long:"mailpass" description:"Email server password"` - MailAddress string `long:"mailaddress" description:"Email address for outgoing email in the format: name
"` - SMTPCert string `long:"smtpcert" description:"File containing the smtp certificate file"` - SMTPSkipVerify bool `long:"smtpskipverify" description:"Skip SMTP TLS cert verification. Will only skip if SMTPCert is empty"` - WebServerAddress string `long:"webserveraddress" description:"Address for the Politeia web server; it should have this format: ://[:]"` - - // XXX These should be plugin settings - DcrdataHost string `long:"dcrdatahost" description:"Dcrdata ip:port"` - PaywallAmount uint64 `long:"paywallamount" description:"Amount of DCR (in atoms) required for a user to register or submit a proposal."` - PaywallXpub string `long:"paywallxpub" description:"Extended public key for deriving paywall addresses."` - MinConfirmationsRequired uint64 `long:"minconfirmations" description:"Minimum blocks confirmation for accepting paywall as paid. Only works in TestNet."` - BuildCMSDB bool `long:"buildcmsdb" description:"Build the cmsdb from scratch"` - GithubAPIToken string `long:"githubapitoken" description:"API Token used to communicate with github API. When populated in cmswww mode, github-tracker is enabled."` - CodeStatRepos []string `long:"codestatrepos" description:"Repositories under the organization to crawl for code statistics"` - CodeStatOrganization string `long:"codestatorg" description:"Organization to crawl for code statistics"` - CodeStatStart int64 `long:"codestatstart" description:"Date in which to look back to for code stat crawl (default 6 months back)"` - CodeStatEnd int64 `long:"codestatend" description:"Date in which to end look back to for code stat crawl (default today)"` - - // TODO these need to be removed - VoteDurationMin uint32 `long:"votedurationmin" description:"Minimum duration of a proposal vote in blocks"` - VoteDurationMax uint32 `long:"votedurationmax" description:"Maximum duration of a proposal vote in blocks"` - - Version string - Identity *identity.PublicIdentity - SystemCerts *x509.CertPool -} - // serviceOptions defines the configuration options for the rpc as a service // on Windows. type serviceOptions struct { @@ -290,7 +224,7 @@ func fileExists(name string) bool { } // newConfigParser returns a new command line flags parser. -func newConfigParser(cfg *config, so *serviceOptions, options flags.Options) *flags.Parser { +func newConfigParser(cfg *config.Config, so *serviceOptions, options flags.Options) *flags.Parser { parser := flags.NewParser(cfg, options) if runtime.GOOS == "windows" { parser.AddGroup("Service Options", "Service Options", so) @@ -299,7 +233,7 @@ func newConfigParser(cfg *config, so *serviceOptions, options flags.Options) *fl } // loadIdentity fetches an identity from politeiad if necessary. -func loadIdentity(cfg *config) error { +func loadIdentity(cfg *config.Config) error { // Set up the path to the politeiad identity file. if cfg.RPCIdentityFile == "" { cfg.RPCIdentityFile = filepath.Join(cfg.HomeDir, @@ -342,13 +276,13 @@ func loadIdentity(cfg *config) error { // The above results in rpc functioning properly without any config settings // while still allowing the user to override settings with config files and // command line options. Command line options always take precedence. -func loadConfig() (*config, []string, error) { +func loadConfig() (*config.Config, []string, error) { // Default config. - cfg := config{ - HomeDir: sharedconfig.DefaultHomeDir, - ConfigFile: sharedconfig.DefaultConfigFile, + cfg := config.Config{ + HomeDir: config.DefaultHomeDir, + ConfigFile: config.DefaultConfigFile, DebugLevel: defaultLogLevel, - DataDir: sharedconfig.DefaultDataDir, + DataDir: config.DefaultDataDir, LogDir: defaultLogDir, HTTPSKey: defaultHTTPSKeyFile, HTTPSCert: defaultHTTPSCertFile, @@ -409,13 +343,13 @@ func loadConfig() (*config, []string, error) { if preCfg.HomeDir != "" { cfg.HomeDir, _ = filepath.Abs(preCfg.HomeDir) - if preCfg.ConfigFile == sharedconfig.DefaultConfigFile { - cfg.ConfigFile = filepath.Join(cfg.HomeDir, sharedconfig.DefaultConfigFilename) + if preCfg.ConfigFile == config.DefaultConfigFile { + cfg.ConfigFile = filepath.Join(cfg.HomeDir, config.DefaultConfigFilename) } else { cfg.ConfigFile = preCfg.ConfigFile } - if preCfg.DataDir == sharedconfig.DefaultDataDir { - cfg.DataDir = filepath.Join(cfg.HomeDir, sharedconfig.DefaultDataDirname) + if preCfg.DataDir == config.DefaultDataDir { + cfg.DataDir = filepath.Join(cfg.HomeDir, config.DefaultDataDirname) } else { cfg.DataDir = preCfg.DataDir } @@ -449,7 +383,7 @@ func loadConfig() (*config, []string, error) { // Load additional config from file. var configFileError error parser := newConfigParser(&cfg, &serviceOpts, flags.Default) - if !(preCfg.SimNet) || cfg.ConfigFile != sharedconfig.DefaultConfigFile { + if !(preCfg.SimNet) || cfg.ConfigFile != config.DefaultConfigFile { err := flags.NewIniParser(parser).ParseFile(cfg.ConfigFile) if err != nil { var e *os.PathError @@ -498,7 +432,7 @@ func loadConfig() (*config, []string, error) { // Create the home directory if it doesn't already exist. funcName := "loadConfig" - err = os.MkdirAll(sharedconfig.DefaultHomeDir, 0700) + err = os.MkdirAll(config.DefaultHomeDir, 0700) if err != nil { // Show a nicer error message if it's because a symlink is // linked to a directory that does not exist (probably because diff --git a/politeiawww/config/config.go b/politeiawww/config/config.go new file mode 100644 index 000000000..516dd7001 --- /dev/null +++ b/politeiawww/config/config.go @@ -0,0 +1,100 @@ +// Copyright (c) 2013-2014 The btcsuite developers +// Copyright (c) 2015-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package config + +import ( + "crypto/x509" + "path/filepath" + + "github.com/decred/dcrd/dcrutil/v3" + "github.com/decred/politeia/politeiad/api/v1/identity" +) + +const ( + // DefaultConfigFilename is the default configuration file name. + DefaultConfigFilename = "politeiawww.conf" + + // DefaultDataDirname is the default data directory name. The data + // directory is located in the application home directory. + DefaultDataDirname = "data" +) + +var ( + // DefaultHomeDir points to politeiawww's default home directory. + DefaultHomeDir = dcrutil.AppDataDir("politeiawww", false) + + // DefaultConfigFile points to politeiawww's default config file + // path. + DefaultConfigFile = filepath.Join(DefaultHomeDir, DefaultConfigFilename) + + // DefaultDataDir points to politeiawww's default data directory + // path. + DefaultDataDir = filepath.Join(DefaultHomeDir, DefaultDataDirname) +) + +// Config defines the configuration options for politeiawww. +type Config struct { + HomeDir string `short:"A" long:"appdata" description:"Path to application home directory"` + ShowVersion bool `short:"V" long:"version" description:"Display version information and exit"` + ConfigFile string `short:"C" long:"configfile" description:"Path to configuration file"` + DataDir string `short:"b" long:"datadir" description:"Directory to store data"` + LogDir string `long:"logdir" description:"Directory to log output."` + TestNet bool `long:"testnet" description:"Use the test network"` + SimNet bool `long:"simnet" description:"Use the simulation test network"` + Profile string `long:"profile" description:"Enable HTTP profiling on given port -- NOTE port must be between 1024 and 65536"` + CookieKeyFile string `long:"cookiekey" description:"File containing the secret cookies key"` + DebugLevel string `short:"d" long:"debuglevel" description:"Logging level for all subsystems {trace, debug, info, warn, error, critical} -- You may also specify =,=,... to set the log level for individual subsystems -- Use show to list available subsystems"` + Listeners []string `long:"listen" description:"Add an interface/port to listen for connections (default all interfaces port: 4443)"` + HTTPSCert string `long:"httpscert" description:"File containing the https certificate file"` + HTTPSKey string `long:"httpskey" description:"File containing the https certificate key"` + RPCHost string `long:"rpchost" description:"Host for politeiad in this format"` + RPCCert string `long:"rpccert" description:"File containing the https certificate file"` + RPCIdentityFile string `long:"rpcidentityfile" description:"Path to file containing the politeiad identity"` + RPCUser string `long:"rpcuser" description:"RPC user name for privileged politeaid commands"` + RPCPass string `long:"rpcpass" description:"RPC password for privileged politeiad commands"` + FetchIdentity bool `long:"fetchidentity" description:"Whether or not politeiawww fetches the identity from politeiad."` + Interactive string `long:"interactive" description:"Set to i-know-this-is-a-bad-idea to turn off interactive mode during --fetchidentity."` + AdminLogFile string `long:"adminlogfile" description:"admin log filename (Default: admin.log)"` + Mode string `long:"mode" description:"Mode www runs as. Supported values: piwww, cmswww"` + + // User database settings + UserDB string `long:"userdb" description:"Database choice for the user database"` + DBHost string `long:"dbhost" description:"Database ip:port"` + DBRootCert string `long:"dbrootcert" description:"File containing the CA certificate for the database"` + DBCert string `long:"dbcert" description:"File containing the politeiawww client certificate for the database"` + DBKey string `long:"dbkey" description:"File containing the politeiawww client certificate key for the database"` + EncryptionKey string `long:"encryptionkey" description:"File containing encryption key used for encrypting user data at rest"` + OldEncryptionKey string `long:"oldencryptionkey" description:"File containing old encryption key (only set when rotating keys)"` + + // SMTP settings + MailHost string `long:"mailhost" description:"Email server address in this format: :"` + MailUser string `long:"mailuser" description:"Email server username"` + MailPass string `long:"mailpass" description:"Email server password"` + MailAddress string `long:"mailaddress" description:"Email address for outgoing email in the format: name
"` + SMTPCert string `long:"smtpcert" description:"File containing the smtp certificate file"` + SMTPSkipVerify bool `long:"smtpskipverify" description:"Skip SMTP TLS cert verification. Will only skip if SMTPCert is empty"` + WebServerAddress string `long:"webserveraddress" description:"Address for the Politeia web server; it should have this format: ://[:]"` + + // XXX These should be plugin settings + DcrdataHost string `long:"dcrdatahost" description:"Dcrdata ip:port"` + PaywallAmount uint64 `long:"paywallamount" description:"Amount of DCR (in atoms) required for a user to register or submit a proposal."` + PaywallXpub string `long:"paywallxpub" description:"Extended public key for deriving paywall addresses."` + MinConfirmationsRequired uint64 `long:"minconfirmations" description:"Minimum blocks confirmation for accepting paywall as paid. Only works in TestNet."` + BuildCMSDB bool `long:"buildcmsdb" description:"Build the cmsdb from scratch"` + GithubAPIToken string `long:"githubapitoken" description:"API Token used to communicate with github API. When populated in cmswww mode, github-tracker is enabled."` + CodeStatRepos []string `long:"codestatrepos" description:"Repositories under the organization to crawl for code statistics"` + CodeStatOrganization string `long:"codestatorg" description:"Organization to crawl for code statistics"` + CodeStatStart int64 `long:"codestatstart" description:"Date in which to look back to for code stat crawl (default 6 months back)"` + CodeStatEnd int64 `long:"codestatend" description:"Date in which to end look back to for code stat crawl (default today)"` + + // TODO these need to be removed + VoteDurationMin uint32 `long:"votedurationmin" description:"Minimum duration of a proposal vote in blocks"` + VoteDurationMax uint32 `long:"votedurationmax" description:"Maximum duration of a proposal vote in blocks"` + + Version string + Identity *identity.PublicIdentity + SystemCerts *x509.CertPool +} diff --git a/politeiawww/politeiawww.go b/politeiawww/politeiawww.go index f27b218d6..30e2b6745 100644 --- a/politeiawww/politeiawww.go +++ b/politeiawww/politeiawww.go @@ -21,6 +21,7 @@ import ( www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/cmsdatabase" "github.com/decred/politeia/politeiawww/codetracker" + "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/user" utilwww "github.com/decred/politeia/politeiawww/util" "github.com/decred/politeia/util" @@ -63,7 +64,7 @@ func (w *wsContext) isAuthenticated() bool { // politeiawww represents the politeiawww server. type politeiawww struct { sync.RWMutex - cfg *config + cfg *config.Config params *chaincfg.Params router *mux.Router auth *mux.Router // CSRF protected subrouter diff --git a/politeiawww/sharedconfig/sharedconfig.go b/politeiawww/sharedconfig/sharedconfig.go deleted file mode 100644 index c316619cd..000000000 --- a/politeiawww/sharedconfig/sharedconfig.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) 2017-2019 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package sharedconfig - -import ( - "path/filepath" - - "github.com/decred/dcrd/dcrutil/v3" -) - -const ( - // DefaultConfigFilename is the default configuration file name. - DefaultConfigFilename = "politeiawww.conf" - - // DefaultDataDirname is the default data directory name. The data - // directory is located in the application home directory. - DefaultDataDirname = "data" -) - -var ( - // DefaultHomeDir points to politeiawww's default home directory. - DefaultHomeDir = dcrutil.AppDataDir("politeiawww", false) - - // DefaultConfigFile points to politeiawww's default config file - // path. - DefaultConfigFile = filepath.Join(DefaultHomeDir, DefaultConfigFilename) - - // DefaultDataDir points to politeiawww's default data directory - // path. - DefaultDataDir = filepath.Join(DefaultHomeDir, DefaultDataDirname) -) diff --git a/politeiawww/testing.go b/politeiawww/testing.go index 3e128d7f2..7127f1865 100644 --- a/politeiawww/testing.go +++ b/politeiawww/testing.go @@ -26,6 +26,7 @@ import ( cms "github.com/decred/politeia/politeiawww/api/cms/v1" pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" + "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/user" "github.com/decred/politeia/politeiawww/user/localdb" "github.com/decred/politeia/util" @@ -316,7 +317,7 @@ func newTestPoliteiawww(t *testing.T) (*politeiawww, func()) { // Setup config xpub := "tpubVobLtToNtTq6TZNw4raWQok35PRPZou53vegZqNubtBTJMMFm" + "uMpWybFCfweJ52N8uZJPZZdHE5SRnBBuuRPfC5jdNstfKjiAs8JtbYG9jx" - cfg := &config{ + cfg := &config.Config{ DataDir: dataDir, PaywallAmount: 1e7, PaywallXpub: xpub, @@ -405,7 +406,7 @@ func newTestCMSwww(t *testing.T) (*politeiawww, func()) { // Setup config xpub := "tpubVobLtToNtTq6TZNw4raWQok35PRPZou53vegZqNubtBTJMMFm" + "uMpWybFCfweJ52N8uZJPZZdHE5SRnBBuuRPfC5jdNstfKjiAs8JtbYG9jx" - cfg := &config{ + cfg := &config.Config{ DataDir: dataDir, PaywallAmount: 1e7, PaywallXpub: xpub, From 8713ebecd900922ae7ef4fe7513278afb6145a49 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 20 Jan 2021 16:44:15 -0600 Subject: [PATCH 241/449] Add sessions package. --- politeiawww/cmswww.go | 53 ++++++------ politeiawww/log.go | 4 + politeiawww/middleware.go | 4 +- politeiawww/piwww.go | 19 +++-- politeiawww/politeiawww.go | 24 +++--- politeiawww/records.go | 5 +- politeiawww/sessions/log.go | 25 ++++++ politeiawww/{ => sessions}/sessions.go | 85 ++++++++++++------- .../{sessionstore.go => sessions/store.go} | 57 ++++++------- politeiawww/testing.go | 5 +- politeiawww/userwww.go | 47 +++++----- politeiawww/www.go | 6 +- 12 files changed, 193 insertions(+), 141 deletions(-) create mode 100644 politeiawww/sessions/log.go rename politeiawww/{ => sessions}/sessions.go (55%) rename politeiawww/{sessionstore.go => sessions/store.go} (84%) diff --git a/politeiawww/cmswww.go b/politeiawww/cmswww.go index ac8fc03c9..c627038a4 100644 --- a/politeiawww/cmswww.go +++ b/politeiawww/cmswww.go @@ -12,6 +12,7 @@ import ( cms "github.com/decred/politeia/politeiawww/api/cms/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" + "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/util" "github.com/google/uuid" "github.com/gorilla/mux" @@ -56,7 +57,7 @@ func (p *politeiawww) handleNewInvoice(w http.ResponseWriter, r *http.Request) { return } - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleNewInvoice: getSessionUser %v", err) @@ -95,9 +96,9 @@ func (p *politeiawww) handleInvoiceDetails(w http.ResponseWriter, r *http.Reques pathParams := mux.Vars(r) pd.Token = pathParams["token"] - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { - if !errors.Is(err, errSessionNotFound) { + if !errors.Is(err, sessions.ErrSessionNotFound) { RespondWithError(w, r, 0, "handleInvoiceDetails: getSessionUser %v", err) return @@ -119,7 +120,7 @@ func (p *politeiawww) handleInvoiceDetails(w http.ResponseWriter, r *http.Reques func (p *politeiawww) handleUserInvoices(w http.ResponseWriter, r *http.Request) { log.Tracef("handleUserInvoices") - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleUserInvoices: getSessionUser %v", err) @@ -184,7 +185,7 @@ func (p *politeiawww) handleSetInvoiceStatus(w http.ResponseWriter, r *http.Requ return } - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleSetInvoiceStatus: getSessionUser %v", err) @@ -215,7 +216,7 @@ func (p *politeiawww) handleInvoices(w http.ResponseWriter, r *http.Request) { return } - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleInvoices: getSessionUser %v", err) @@ -247,7 +248,7 @@ func (p *politeiawww) handleEditInvoice(w http.ResponseWriter, r *http.Request) return } - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleEditInvoice: getSessionUser %v", err) @@ -282,7 +283,7 @@ func (p *politeiawww) handleGeneratePayouts(w http.ResponseWriter, r *http.Reque return } - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleGeneratePayouts: getSessionUser %v", err) @@ -313,7 +314,7 @@ func (p *politeiawww) handleNewCommentInvoice(w http.ResponseWriter, r *http.Req return } - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleNewCommentInvoice: getSessionUser %v", err) @@ -337,9 +338,9 @@ func (p *politeiawww) handleInvoiceComments(w http.ResponseWriter, r *http.Reque pathParams := mux.Vars(r) token := pathParams["token"] - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { - if !errors.Is(err, errSessionNotFound) { + if !errors.Is(err, sessions.ErrSessionNotFound) { RespondWithError(w, r, 0, "handleInvoiceComments: getSessionUser %v", err) return @@ -415,7 +416,7 @@ func (p *politeiawww) handleCMSPolicy(w http.ResponseWriter, r *http.Request) { func (p *politeiawww) handlePayInvoices(w http.ResponseWriter, r *http.Request) { log.Tracef("handlePayInvoices") - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handlePayInvoices: getSessionUser %v", err) @@ -447,7 +448,7 @@ func (p *politeiawww) handleEditCMSUser(w http.ResponseWriter, r *http.Request) return } - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleEditCMSUser: getSessionUser %v", err) @@ -505,7 +506,7 @@ func (p *politeiawww) handleCMSUserDetails(w http.ResponseWriter, r *http.Reques return } - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleCMSUserDetails: getSessionUser %v", err) @@ -562,7 +563,7 @@ func (p *politeiawww) handleNewDCC(w http.ResponseWriter, r *http.Request) { }) return } - u, err := p.getSessionUser(w, r) + u, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleNewDCC: getSessionUser %v", err) @@ -618,7 +619,7 @@ func (p *politeiawww) handleGetDCCs(w http.ResponseWriter, r *http.Request) { }) return } - _, err := p.getSessionUser(w, r) + _, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleGetDCCs: getSessionUser %v", err) @@ -647,7 +648,7 @@ func (p *politeiawww) handleSupportOpposeDCC(w http.ResponseWriter, r *http.Requ }) return } - u, err := p.getSessionUser(w, r) + u, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleSupportOpposeDCC: getSessionUser %v", err) @@ -678,7 +679,7 @@ func (p *politeiawww) handleNewCommentDCC(w http.ResponseWriter, r *http.Request return } - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleNewCommentDCC: getSessionUser %v", err) @@ -702,9 +703,9 @@ func (p *politeiawww) handleDCCComments(w http.ResponseWriter, r *http.Request) pathParams := mux.Vars(r) token := pathParams["token"] - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { - if !errors.Is(err, errSessionNotFound) { + if !errors.Is(err, sessions.ErrSessionNotFound) { RespondWithError(w, r, 0, "handleDCCComments: getSessionUser %v", err) return @@ -731,7 +732,7 @@ func (p *politeiawww) handleSetDCCStatus(w http.ResponseWriter, r *http.Request) }) return } - u, err := p.getSessionUser(w, r) + u, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleSetDCCStatus: getSessionUser %v", err) @@ -751,7 +752,7 @@ func (p *politeiawww) handleSetDCCStatus(w http.ResponseWriter, r *http.Request) func (p *politeiawww) handleUserSubContractors(w http.ResponseWriter, r *http.Request) { log.Tracef("handleUserSubContractors") - u, err := p.getSessionUser(w, r) + u, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleUserSubContractors: getSessionUser %v", err) @@ -803,7 +804,7 @@ func (p *politeiawww) handleProposalBilling(w http.ResponseWriter, r *http.Reque return } - u, err := p.getSessionUser(w, r) + u, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleProposalBilling: getSessionUser %v", err) @@ -832,7 +833,7 @@ func (p *politeiawww) handleCastVoteDCC(w http.ResponseWriter, r *http.Request) return } - u, err := p.getSessionUser(w, r) + u, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleCastVoteDCC: getSessionUser %v", err) @@ -898,7 +899,7 @@ func (p *politeiawww) handleStartVoteDCC(w http.ResponseWriter, r *http.Request) }) return } - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleStartVoteDCC: getSessionUser %v", err) @@ -1094,7 +1095,7 @@ func (p *politeiawww) handleUserCodeStats(w http.ResponseWriter, r *http.Request return } - u, err := p.getSessionUser(w, r) + u, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleUserCodeStats: getSessionUser %v", err) diff --git a/politeiawww/log.go b/politeiawww/log.go index b81ea0837..09b1390fe 100644 --- a/politeiawww/log.go +++ b/politeiawww/log.go @@ -12,6 +12,7 @@ import ( cmsdb "github.com/decred/politeia/politeiawww/cmsdatabase/cockroachdb" "github.com/decred/politeia/politeiawww/codetracker/github" ghdb "github.com/decred/politeia/politeiawww/codetracker/github/database/cockroachdb" + "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/politeiawww/user/cockroachdb" "github.com/decred/politeia/politeiawww/user/localdb" "github.com/decred/politeia/wsdcrdata" @@ -52,6 +53,7 @@ var ( wsdcrdataLog = backendLog.Logger("WSDD") githubTrackerLog = backendLog.Logger("GHTR") githubdbLog = backendLog.Logger("GHDB") + sessionsLog = backendLog.Logger("SESS") ) // Initialize package-global logger variables. @@ -62,6 +64,7 @@ func init() { wsdcrdata.UseLogger(wsdcrdataLog) github.UseLogger(githubTrackerLog) ghdb.UseLogger(githubdbLog) + sessions.UseLogger(sessionsLog) } // subsystemLoggers maps each subsystem identifier to its associated logger. @@ -72,6 +75,7 @@ var subsystemLoggers = map[string]slog.Logger{ "WSDD": wsdcrdataLog, "GHTR": githubTrackerLog, "GHDB": githubdbLog, + "SESS": sessionsLog, } // initLogRotator initializes the logging rotater to write logs to logFile and diff --git a/politeiawww/middleware.go b/politeiawww/middleware.go index b1be9922e..5c7b206b0 100644 --- a/politeiawww/middleware.go +++ b/politeiawww/middleware.go @@ -31,7 +31,7 @@ func (p *politeiawww) isLoggedIn(f http.HandlerFunc) http.HandlerFunc { log.Debugf("isLoggedIn: %v %v %v %v", remoteAddr(r), r.Method, r.URL, r.Proto) - id, err := p.getSessionUserID(w, r) + id, err := p.sessions.GetSessionUserID(w, r) if err != nil { util.RespondWithJSON(w, http.StatusUnauthorized, www.UserError{ ErrorCode: www.ErrorStatusNotLoggedIn, @@ -53,7 +53,7 @@ func (p *politeiawww) isLoggedIn(f http.HandlerFunc) http.HandlerFunc { // isAdmin returns true if the current session has admin privileges. func (p *politeiawww) isAdmin(w http.ResponseWriter, r *http.Request) (bool, error) { - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { return false, err } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 0e2569d63..2c2b6b8e9 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -28,6 +28,7 @@ import ( rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" + "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/politeiawww/user" wwwutil "github.com/decred/politeia/politeiawww/util" "github.com/decred/politeia/util" @@ -1627,7 +1628,7 @@ func (p *politeiawww) handleProposalNew(w http.ResponseWriter, r *http.Request) return } - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { respondWithPiError(w, r, "handleProposalNew: getSessionUser: %v", err) @@ -1657,7 +1658,7 @@ func (p *politeiawww) handleProposalEdit(w http.ResponseWriter, r *http.Request) return } - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { respondWithPiError(w, r, "handleProposalEdit: getSessionUser: %v", err) @@ -1687,7 +1688,7 @@ func (p *politeiawww) handleProposalSetStatus(w http.ResponseWriter, r *http.Req return } - usr, err := p.getSessionUser(w, r) + usr, err := p.sessions.GetSessionUser(w, r) if err != nil { respondWithPiError(w, r, "handleProposalSetStatus: getSessionUser: %v", err) @@ -1719,8 +1720,8 @@ func (p *politeiawww) handleProposals(w http.ResponseWriter, r *http.Request) { // Lookup session user. This is a public route so a session may not // exist. Ignore any session not found errors. - usr, err := p.getSessionUser(w, r) - if err != nil && err != errSessionNotFound { + usr, err := p.sessions.GetSessionUser(w, r) + if err != nil && err != sessions.ErrSessionNotFound { respondWithPiError(w, r, "handleProposals: getSessionUser: %v", err) return @@ -1752,8 +1753,8 @@ func (p *politeiawww) handleProposalInventory(w http.ResponseWriter, r *http.Req // Lookup session user. This is a public route so a session may not // exist. Ignore any session not found errors. - usr, err := p.getSessionUser(w, r) - if err != nil && err != errSessionNotFound { + usr, err := p.sessions.GetSessionUser(w, r) + if err != nil && err != sessions.ErrSessionNotFound { respondWithPiError(w, r, "handleProposalInventory: getSessionUser: %v", err) return @@ -1782,7 +1783,7 @@ func (p *politeiawww) handleVoteAuthorize(w http.ResponseWriter, r *http.Request return } - usr, err := p.getSessionUser(w, r) + usr, err := p.sessions.GetSessionUser(w, r) if err != nil { respondWithPiError(w, r, "handleVoteAuthorize: getSessionUser: %v", err) @@ -1812,7 +1813,7 @@ func (p *politeiawww) handleVoteStart(w http.ResponseWriter, r *http.Request) { return } - usr, err := p.getSessionUser(w, r) + usr, err := p.sessions.GetSessionUser(w, r) if err != nil { respondWithPiError(w, r, "handleVoteStart: getSessionUser: %v", err) diff --git a/politeiawww/politeiawww.go b/politeiawww/politeiawww.go index 30e2b6745..6cd10e5fa 100644 --- a/politeiawww/politeiawww.go +++ b/politeiawww/politeiawww.go @@ -22,6 +22,7 @@ import ( "github.com/decred/politeia/politeiawww/cmsdatabase" "github.com/decred/politeia/politeiawww/codetracker" "github.com/decred/politeia/politeiawww/config" + "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/politeiawww/user" utilwww "github.com/decred/politeia/politeiawww/util" "github.com/decred/politeia/util" @@ -30,7 +31,6 @@ import ( "github.com/google/uuid" "github.com/gorilla/csrf" "github.com/gorilla/mux" - "github.com/gorilla/sessions" "github.com/gorilla/websocket" "github.com/robfig/cron" ) @@ -72,7 +72,7 @@ type politeiawww struct { client *http.Client smtp *smtp db user.Database - sessions sessions.Store + sessions *sessions.Sessions eventManager *eventManager plugins []plugin @@ -134,7 +134,7 @@ func (p *politeiawww) handleVersion(w http.ResponseWriter, r *http.Request) { Mode: p.cfg.Mode, } - _, err := p.getSessionUser(w, r) + _, err := p.sessions.GetSessionUser(w, r) if err == nil { versionReply.ActiveUserSession = true } @@ -196,8 +196,8 @@ func (p *politeiawww) handleTokenInventory(w http.ResponseWriter, r *http.Reques log.Tracef("handleTokenInventory") // Get session user. This is a public route so one might not exist. - user, err := p.getSessionUser(w, r) - if err != nil && !errors.Is(err, errSessionNotFound) { + user, err := p.sessions.GetSessionUser(w, r) + if err != nil && !errors.Is(err, sessions.ErrSessionNotFound) { RespondWithError(w, r, 0, "handleTokenInventory: getSessionUser %v", err) return @@ -234,8 +234,8 @@ func (p *politeiawww) handleProposalDetails(w http.ResponseWriter, r *http.Reque pd.Token = pathParams["token"] // Get session user. This is a public route so one might not exist. - user, err := p.getSessionUser(w, r) - if err != nil && err != errSessionNotFound { + user, err := p.sessions.GetSessionUser(w, r) + if err != nil && err != sessions.ErrSessionNotFound { RespondWithError(w, r, 0, "handleProposalDetails: getSessionUser %v", err) return @@ -268,8 +268,8 @@ func (p *politeiawww) handleBatchProposals(w http.ResponseWriter, r *http.Reques } // Get session user. This is a public route so one might not exist. - user, err := p.getSessionUser(w, r) - if err != nil && err != errSessionNotFound { + user, err := p.sessions.GetSessionUser(w, r) + if err != nil && err != sessions.ErrSessionNotFound { RespondWithError(w, r, 0, "handleBatchProposals: getSessionUser %v", err) return @@ -561,8 +561,8 @@ func (p *politeiawww) handleWebsocket(w http.ResponseWriter, r *http.Request, id func (p *politeiawww) handleUnauthenticatedWebsocket(w http.ResponseWriter, r *http.Request) { // We are retrieving the uuid here to make sure it is NOT set. This // check looks backwards but is correct. - id, err := p.getSessionUserID(w, r) - if err != nil && !errors.Is(err, errSessionNotFound) { + id, err := p.sessions.GetSessionUserID(w, r) + if err != nil && !errors.Is(err, sessions.ErrSessionNotFound) { http.Error(w, "Could not get session uuid", http.StatusBadRequest) return @@ -580,7 +580,7 @@ func (p *politeiawww) handleUnauthenticatedWebsocket(w http.ResponseWriter, r *h // handleAuthenticatedWebsocket attempts to upgrade the current authenticated // connection to a websocket connection. func (p *politeiawww) handleAuthenticatedWebsocket(w http.ResponseWriter, r *http.Request) { - id, err := p.getSessionUserID(w, r) + id, err := p.sessions.GetSessionUserID(w, r) if err != nil { http.Error(w, "Could not get session uuid", http.StatusBadRequest) diff --git a/politeiawww/records.go b/politeiawww/records.go index 12012e528..ffc20901a 100644 --- a/politeiawww/records.go +++ b/politeiawww/records.go @@ -16,6 +16,7 @@ import ( pdv1 "github.com/decred/politeia/politeiad/api/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/util" ) @@ -92,8 +93,8 @@ func (p *politeiawww) handleTimestamps(w http.ResponseWriter, r *http.Request) { // Lookup session user. This is a public route so a session may not // exist. Ignore any session not found errors. - usr, err := p.getSessionUser(w, r) - if err != nil && err != errSessionNotFound { + usr, err := p.sessions.GetSessionUser(w, r) + if err != nil && err != sessions.ErrSessionNotFound { respondWithRecordError(w, r, "handleTimestamps: getSessionUser: %v", err) return diff --git a/politeiawww/sessions/log.go b/politeiawww/sessions/log.go new file mode 100644 index 000000000..71ca0dc60 --- /dev/null +++ b/politeiawww/sessions/log.go @@ -0,0 +1,25 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package sessions + +import "github.com/decred/slog" + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log = slog.Disabled + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = slog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using slog. +func UseLogger(logger slog.Logger) { + log = logger +} diff --git a/politeiawww/sessions.go b/politeiawww/sessions/sessions.go similarity index 55% rename from politeiawww/sessions.go rename to politeiawww/sessions/sessions.go index 78e7d9430..9e64589fa 100644 --- a/politeiawww/sessions.go +++ b/politeiawww/sessions/sessions.go @@ -1,8 +1,8 @@ -// Copyright (c) 2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package main +package sessions import ( "errors" @@ -16,7 +16,8 @@ import ( ) const ( - sessionMaxAge = 86400 // One day + // SessionMaxAge is the max age for a session in seconds. + SessionMaxAge = 86400 // One day // Session value keys. A user session contains a map that is used // for application specific values. The following is a list of the @@ -26,55 +27,65 @@ const ( ) var ( - // errSessionNotFound is emitted when a session is not found in the + // ErrSessionNotFound is emitted when a session is not found in the // session store. - errSessionNotFound = errors.New("session not found") + ErrSessionNotFound = errors.New("session not found") ) +// Sessions manages politeiawww sessions. +type Sessions struct { + store sessions.Store + userdb user.Database +} + func sessionIsExpired(session *sessions.Session) bool { createdAt := session.Values[sessionValueCreatedAt].(int64) expiresAt := createdAt + int64(session.Options.MaxAge) return time.Now().Unix() > expiresAt } -// getSession returns the Session for the session ID from the given http +// GetSession returns the Session for the session ID from the given http // request cookie. If no session exists then a new session object is returned. // Access IsNew on the session to check if it is an existing session or a new // one. The new session will not have any sessions values set, such as user_id, // and will not have been saved to the session store yet. -func (p *politeiawww) getSession(r *http.Request) (*sessions.Session, error) { - return p.sessions.Get(r, www.CookieSession) +func (s *Sessions) GetSession(r *http.Request) (*sessions.Session, error) { + log.Tracef("GetSession") + + return s.store.Get(r, www.CookieSession) } -// getSessionUserID returns the user ID of the user for the given session. A -// errSessionNotFound error is returned if a user session does not exist or +// GetSessionUserID returns the user ID of the user for the given session. A +// ErrSessionNotFound error is returned if a user session does not exist or // has expired. -func (p *politeiawww) getSessionUserID(w http.ResponseWriter, r *http.Request) (string, error) { - session, err := p.getSession(r) +func (s *Sessions) GetSessionUserID(w http.ResponseWriter, r *http.Request) (string, error) { + log.Tracef("GetSessionUserID") + + session, err := s.GetSession(r) if err != nil { return "", err } if session.IsNew { - return "", errSessionNotFound + return "", ErrSessionNotFound } // Delete the session if its expired. Setting the MaxAge // to <= 0 and then saving it will trigger a deletion. if sessionIsExpired(session) { session.Options.MaxAge = -1 - p.sessions.Save(r, w, session) - return "", errSessionNotFound + s.store.Save(r, w, session) + return "", ErrSessionNotFound } return session.Values[sessionValueUserID].(string), nil } -// getSessionUser returns the User for the given session. A errSessionFound +// GetSessionUser returns the User for the given session. A errSessionFound // error is returned if a user session does not exist or has expired. -func (p *politeiawww) getSessionUser(w http.ResponseWriter, r *http.Request) (*user.User, error) { - log.Tracef("getSessionUser") +func (s *Sessions) GetSessionUser(w http.ResponseWriter, r *http.Request) (*user.User, error) { + log.Tracef("GetSessionUser") - uid, err := p.getSessionUserID(w, r) + uid, err := s.GetSessionUserID(w, r) if err != nil { return nil, err } @@ -84,32 +95,32 @@ func (p *politeiawww) getSessionUser(w http.ResponseWriter, r *http.Request) (*u return nil, err } - user, err := p.db.UserGetById(pid) + user, err := s.userdb.UserGetById(pid) if err != nil { return nil, err } if user.Deactivated { - err := p.removeSession(w, r) + err := s.DelSession(w, r) if err != nil { return nil, err } - return nil, errSessionNotFound + return nil, ErrSessionNotFound } return user, nil } -// removeSession removes the given session from the session store. -func (p *politeiawww) removeSession(w http.ResponseWriter, r *http.Request) error { - log.Tracef("removeSession") +// DelSession removes the given session from the session store. +func (s *Sessions) DelSession(w http.ResponseWriter, r *http.Request) error { + log.Tracef("DelSession") - session, err := p.getSession(r) + session, err := s.GetSession(r) if err != nil { return err } if session.IsNew { - return errSessionNotFound + return ErrSessionNotFound } log.Debugf("Deleting user session: %v %v", @@ -118,18 +129,18 @@ func (p *politeiawww) removeSession(w http.ResponseWriter, r *http.Request) erro // Saving the session with a negative MaxAge will cause it to be // deleted. session.Options.MaxAge = -1 - return p.sessions.Save(r, w, session) + return s.store.Save(r, w, session) } -// initSession creates a new session, adds it to the given http response +// NewSession creates a new session, adds it to the given http response // session cookie, and saves it to the session store. If the http request // already contains a session cookie then the session values will be updated // and the session will be updated in the session store. -func (p *politeiawww) initSession(w http.ResponseWriter, r *http.Request, userID string) error { - log.Tracef("initSession: %v", userID) +func (s *Sessions) NewSession(w http.ResponseWriter, r *http.Request, userID string) error { + log.Tracef("NewSession: %v", userID) // Init session - session, err := p.getSession(r) + session, err := s.GetSession(r) if err != nil { return err } @@ -139,5 +150,13 @@ func (p *politeiawww) initSession(w http.ResponseWriter, r *http.Request, userID session.Values[sessionValueUserID] = userID // Update session in the store and update the response cookie - return p.sessions.Save(r, w, session) + return s.store.Save(r, w, session) +} + +// New returns a new Sessions context. +func New(userdb user.Database, keyPairs ...[]byte) *Sessions { + return &Sessions{ + store: newSessionStore(userdb, keyPairs...), + userdb: userdb, + } } diff --git a/politeiawww/sessionstore.go b/politeiawww/sessions/store.go similarity index 84% rename from politeiawww/sessionstore.go rename to politeiawww/sessions/store.go index 9f04ed9a1..a80525560 100644 --- a/politeiawww/sessionstore.go +++ b/politeiawww/sessions/store.go @@ -1,8 +1,8 @@ -// Copyright (c) 2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package main +package sessions import ( "encoding/base32" @@ -16,10 +16,14 @@ import ( "github.com/gorilla/sessions" ) -// SessionStore is a session store backed by the user database. +var ( + _ sessions.Store = (*sessionStore)(nil) +) + +// sessionStore is a session store backed by the user database. // -// SessionStore impelements the sessions.Store interface. -type SessionStore struct { +// sessionStore impelements the sessions.Store interface. +type sessionStore struct { Codecs []securecookie.Codec Options *sessions.Options db user.Database @@ -44,8 +48,8 @@ func newSessionID() string { // be decoded. // // This function satisfies the sessions.Store interface. -func (s *SessionStore) Get(r *http.Request, name string) (*sessions.Session, error) { - log.Tracef("SessionStore.Get: %v", name) +func (s *sessionStore) Get(r *http.Request, name string) (*sessions.Session, error) { + log.Tracef("Get: %v", name) return sessions.GetRegistry(r).Get(s, name) } @@ -61,8 +65,8 @@ func (s *SessionStore) Get(r *http.Request, name string) (*sessions.Session, err // decoded session after the first call. // // This function satisfies the sessions.Store interface. -func (s *SessionStore) New(r *http.Request, name string) (*sessions.Session, error) { - log.Tracef("SessStore.New: %v", name) +func (s *sessionStore) New(r *http.Request, name string) (*sessions.Session, error) { + log.Tracef("New: %v", name) // Setup new session session := sessions.NewSession(s, name) @@ -128,8 +132,8 @@ func (s *SessionStore) New(r *http.Request, name string) (*sessions.Session, err // browser. // // This function satisfies the sessions.Store interface. -func (s *SessionStore) Save(r *http.Request, w http.ResponseWriter, session *sessions.Session) error { - log.Tracef("SessionStore.Save: %v", session.ID) +func (s *sessionStore) Save(r *http.Request, w http.ResponseWriter, session *sessions.Session) error { + log.Tracef("Save: %v", session.ID) // Delete session if max-age is <= 0 if session.Options.MaxAge <= 0 { @@ -183,18 +187,7 @@ func (s *SessionStore) Save(r *http.Request, w http.ResponseWriter, session *ses return nil } -// newSessionOptions returns the default session configuration for politeiawww. -func newSessionOptions() *sessions.Options { - return &sessions.Options{ - Path: "/", - MaxAge: sessionMaxAge, // Max age for the store - Secure: true, - HttpOnly: true, - SameSite: http.SameSiteStrictMode, - } -} - -// newSessionStore returns a new SessionStore. +// new returns a new sessionStore. // // Keys are defined in pairs to allow key rotation, but the common case is // to set a single authentication key and optionally an encryption key. @@ -206,18 +199,24 @@ func newSessionOptions() *sessions.Options { // It is recommended to use an authentication key with 32 or 64 bytes. // The encryption key, if set, must be either 16, 24, or 32 bytes to select // AES-128, AES-192, or AES-256 modes. -func newSessionStore(db user.Database, sessionMaxAge int, keyPairs ...[]byte) *SessionStore { +func newSessionStore(db user.Database, keyPairs ...[]byte) *sessionStore { // Set the maxAge for each securecookie instance codecs := securecookie.CodecsFromPairs(keyPairs...) for _, codec := range codecs { if sc, ok := codec.(*securecookie.SecureCookie); ok { - sc.MaxAge(sessionMaxAge) + sc.MaxAge(SessionMaxAge) } } - return &SessionStore{ - Codecs: codecs, - Options: newSessionOptions(), - db: db, + return &sessionStore{ + Codecs: codecs, + Options: &sessions.Options{ + Path: "/", + MaxAge: SessionMaxAge, // Max age for the store + Secure: true, + HttpOnly: true, + SameSite: http.SameSiteStrictMode, + }, + db: db, } } diff --git a/politeiawww/testing.go b/politeiawww/testing.go index 7127f1865..838e27524 100644 --- a/politeiawww/testing.go +++ b/politeiawww/testing.go @@ -27,6 +27,7 @@ import ( pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/config" + "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/politeiawww/user" "github.com/decred/politeia/politeiawww/user/localdb" "github.com/decred/politeia/util" @@ -350,7 +351,7 @@ func newTestPoliteiawww(t *testing.T) (*politeiawww, func()) { params: chaincfg.TestNet3Params(), router: mux.NewRouter(), auth: mux.NewRouter(), - sessions: newSessionStore(db, sessionMaxAge, cookieKey), + sessions: sessions.New(db, cookieKey), smtp: smtp, db: db, test: true, @@ -454,7 +455,7 @@ func newTestCMSwww(t *testing.T) (*politeiawww, func()) { params: chaincfg.TestNet3Params(), router: mux.NewRouter(), auth: mux.NewRouter(), - sessions: newSessionStore(db, sessionMaxAge, cookieKey), + sessions: sessions.New(db, cookieKey), smtp: smtp, test: true, userEmails: make(map[string]uuid.UUID), diff --git a/politeiawww/userwww.go b/politeiawww/userwww.go index f9713ab26..5c22023c0 100644 --- a/politeiawww/userwww.go +++ b/politeiawww/userwww.go @@ -12,6 +12,7 @@ import ( cms "github.com/decred/politeia/politeiawww/api/cms/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" + "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/util" "github.com/google/uuid" "github.com/gorilla/mux" @@ -137,7 +138,7 @@ func (p *politeiawww) handleLogin(w http.ResponseWriter, r *http.Request) { } // Initialize a session for the logged in user - err = p.initSession(w, r, reply.UserID) + err = p.sessions.NewSession(w, r, reply.UserID) if err != nil { RespondWithError(w, r, 0, "handleLogin: initSession: %v", err) @@ -145,7 +146,7 @@ func (p *politeiawww) handleLogin(w http.ResponseWriter, r *http.Request) { } // Set session max age - reply.SessionMaxAge = sessionMaxAge + reply.SessionMaxAge = sessions.SessionMaxAge // Reply with the user information. util.RespondWithJSON(w, http.StatusOK, reply) @@ -155,7 +156,7 @@ func (p *politeiawww) handleLogin(w http.ResponseWriter, r *http.Request) { func (p *politeiawww) handleLogout(w http.ResponseWriter, r *http.Request) { log.Tracef("handleLogout") - _, err := p.getSessionUser(w, r) + _, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleLogout: getSessionUser", www.UserError{ ErrorCode: www.ErrorStatusNotLoggedIn, @@ -163,7 +164,7 @@ func (p *politeiawww) handleLogout(w http.ResponseWriter, r *http.Request) { return } - err = p.removeSession(w, r) + err = p.sessions.DelSession(w, r) if err != nil { RespondWithError(w, r, 0, "handleLogout: removeSession %v", err) @@ -259,8 +260,8 @@ func (p *politeiawww) handleUserDetails(w http.ResponseWriter, r *http.Request) } // Get session user. This is a public route so one might not exist. - user, err := p.getSessionUser(w, r) - if err != nil && !errors.Is(err, errSessionNotFound) { + user, err := p.sessions.GetSessionUser(w, r) + if err != nil && !errors.Is(err, sessions.ErrSessionNotFound) { RespondWithError(w, r, 0, "handleUserDetails: getSessionUser %v", err) return @@ -292,7 +293,7 @@ func (p *politeiawww) handleSecret(w http.ResponseWriter, r *http.Request) { func (p *politeiawww) handleMe(w http.ResponseWriter, r *http.Request) { log.Tracef("handleMe") - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleMe: getSessionUser %v", err) @@ -307,7 +308,7 @@ func (p *politeiawww) handleMe(w http.ResponseWriter, r *http.Request) { } // Set session max age - reply.SessionMaxAge = sessionMaxAge + reply.SessionMaxAge = sessions.SessionMaxAge util.RespondWithJSON(w, http.StatusOK, *reply) } @@ -328,7 +329,7 @@ func (p *politeiawww) handleUpdateUserKey(w http.ResponseWriter, r *http.Request return } - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleUpdateUserKey: getSessionUser %v", err) @@ -362,7 +363,7 @@ func (p *politeiawww) handleVerifyUpdateUserKey(w http.ResponseWriter, r *http.R return } - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleVerifyUpdateUserKey: getSessionUser %v", err) @@ -394,7 +395,7 @@ func (p *politeiawww) handleChangeUsername(w http.ResponseWriter, r *http.Reques return } - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleChangeUsername: getSessionUser %v", err) @@ -427,13 +428,13 @@ func (p *politeiawww) handleChangePassword(w http.ResponseWriter, r *http.Reques return } - session, err := p.getSession(r) + session, err := p.sessions.GetSession(r) if err != nil { RespondWithError(w, r, 0, "handleChangePassword: getSession %v", err) return } - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleChangePassword: getSessionUser %v", err) @@ -474,7 +475,7 @@ func (p *politeiawww) handleEditUser(w http.ResponseWriter, r *http.Request) { return } - adminUser, err := p.getSessionUser(w, r) + adminUser, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleEditUser: getSessionUser %v", err) @@ -506,8 +507,8 @@ func (p *politeiawww) handleUsers(w http.ResponseWriter, r *http.Request) { } // Get session user. This is a public route so one might not exist. - user, err := p.getSessionUser(w, r) - if err != nil && !errors.Is(err, errSessionNotFound) { + user, err := p.sessions.GetSessionUser(w, r) + if err != nil && !errors.Is(err, sessions.ErrSessionNotFound) { RespondWithError(w, r, 0, "handleUsers: getSessionUser %v", err) return @@ -563,7 +564,7 @@ func (p *politeiawww) handleManageUser(w http.ResponseWriter, r *http.Request) { return } - adminUser, err := p.getSessionUser(w, r) + adminUser, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleManageUser: getSessionUser %v", err) @@ -586,7 +587,7 @@ func (p *politeiawww) handleManageUser(w http.ResponseWriter, r *http.Request) { func (p *politeiawww) handleUserRegistrationPayment(w http.ResponseWriter, r *http.Request) { log.Tracef("handleUserRegistrationPayment") - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleUserRegistrationPayment: getSessionUser %v", err) @@ -609,7 +610,7 @@ func (p *politeiawww) handleUserRegistrationPayment(w http.ResponseWriter, r *ht func (p *politeiawww) handleUserProposalPaywall(w http.ResponseWriter, r *http.Request) { log.Tracef("handleUserProposalPaywall") - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleUserProposalPaywall: getSessionUser %v", err) @@ -631,7 +632,7 @@ func (p *politeiawww) handleUserProposalPaywall(w http.ResponseWriter, r *http.R func (p *politeiawww) handleUserProposalPaywallTx(w http.ResponseWriter, r *http.Request) { log.Tracef("handleUserProposalPaywallTx") - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleUserProposalPaywallTx: getSessionUser %v", err) @@ -654,7 +655,7 @@ func (p *politeiawww) handleUserProposalPaywallTx(w http.ResponseWriter, r *http func (p *politeiawww) handleUserProposalCredits(w http.ResponseWriter, r *http.Request) { log.Tracef("handleUserProposalCredits") - user, err := p.getSessionUser(w, r) + user, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleUserProposalCredits: getSessionUser %v", err) @@ -736,7 +737,7 @@ func (p *politeiawww) handleSetTOTP(w http.ResponseWriter, r *http.Request) { return } - u, err := p.getSessionUser(w, r) + u, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleSetTOTP: getSessionUser %v", err) @@ -767,7 +768,7 @@ func (p *politeiawww) handleVerifyTOTP(w http.ResponseWriter, r *http.Request) { return } - u, err := p.getSessionUser(w, r) + u, err := p.sessions.GetSessionUser(w, r) if err != nil { RespondWithError(w, r, 0, "handleVerifyTOTP: getSessionUser %v", err) diff --git a/politeiawww/www.go b/politeiawww/www.go index 724890748..07e3db675 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -36,6 +36,7 @@ import ( database "github.com/decred/politeia/politeiawww/cmsdatabase" cmsdb "github.com/decred/politeia/politeiawww/cmsdatabase/cockroachdb" ghtracker "github.com/decred/politeia/politeiawww/codetracker/github" + "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/politeiawww/user" "github.com/decred/politeia/politeiawww/user/cockroachdb" "github.com/decred/politeia/politeiawww/user/localdb" @@ -829,7 +830,7 @@ func _main() error { csrfMiddleware := csrf.Protect( csrfKey, csrf.Path("/"), - csrf.MaxAge(sessionMaxAge), + csrf.MaxAge(sessions.SessionMaxAge), ) // Setup router @@ -911,7 +912,6 @@ func _main() error { } log.Infof("Cookie key generated") } - sessions := newSessionStore(userDB, sessionMaxAge, cookieKey) // Setup smtp client smtp, err := newSMTP(loadedCfg.MailHost, loadedCfg.MailUser, @@ -937,7 +937,7 @@ func _main() error { client: client, smtp: smtp, db: userDB, - sessions: sessions, + sessions: sessions.New(userDB, cookieKey), eventManager: newEventManager(), ws: make(map[string]map[string]*wsContext), userEmails: make(map[string]uuid.UUID), From 5a8f90468f69a96765c869bff343858e3846cecf Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 20 Jan 2021 20:12:29 -0600 Subject: [PATCH 242/449] politeiawww: Add comments package. --- .../backend/tlogbe/plugins/comments/cmds.go | 48 +-- .../tlogbe/plugins/comments/comments.go | 14 +- politeiad/backend/tlogbe/plugins/pi/cmds.go | 6 +- politeiad/backend/tlogbe/plugins/pi/hooks.go | 12 +- politeiad/backend/tlogbe/plugins/pi/pi.go | 2 +- .../backend/tlogbe/plugins/ticketvote/cmds.go | 120 +++---- .../tlogbe/plugins/ticketvote/hooks.go | 22 +- .../tlogbe/plugins/ticketvote/ticketvote.go | 11 +- politeiad/backend/tlogbe/tlog/plugin.go | 8 +- politeiad/backend/tlogbe/tlogbe.go | 26 +- politeiad/client/comments.go | 92 +++++ politeiad/client/pdv1.go | 4 +- politeiad/client/user.go | 94 +++++ politeiad/plugins/comments/comments.go | 2 +- politeiad/plugins/dcrdata/dcrdata.go | 2 +- politeiad/plugins/pi/pi.go | 2 +- politeiad/plugins/ticketvote/ticketvote.go | 2 +- politeiawww/api/comments/v1/v1.go | 87 ++--- politeiawww/comments.go | 338 +----------------- politeiawww/comments/comments.go | 30 ++ politeiawww/comments/handle.go | 154 ++++++++ politeiawww/comments/log.go | 25 ++ politeiawww/comments/process.go | 197 ++++++++++ politeiawww/config.go | 10 +- politeiawww/config/config.go | 5 + politeiawww/log.go | 4 + politeiawww/middleware.go | 24 +- politeiawww/politeiawww.go | 4 +- politeiawww/records.go | 10 +- politeiawww/testing.go | 2 +- politeiawww/ticketvote.go | 11 +- politeiawww/totp.go | 3 +- politeiawww/www.go | 47 +-- util/json.go | 15 +- 34 files changed, 850 insertions(+), 583 deletions(-) create mode 100644 politeiad/client/comments.go create mode 100644 politeiad/client/user.go create mode 100644 politeiawww/comments/comments.go create mode 100644 politeiawww/comments/handle.go create mode 100644 politeiawww/comments/log.go create mode 100644 politeiawww/comments/process.go diff --git a/politeiad/backend/tlogbe/plugins/comments/cmds.go b/politeiad/backend/tlogbe/plugins/comments/cmds.go index 6f150cef9..ba8a3bd0b 100644 --- a/politeiad/backend/tlogbe/plugins/comments/cmds.go +++ b/politeiad/backend/tlogbe/plugins/comments/cmds.go @@ -49,7 +49,7 @@ func convertSignatureError(err error) backend.PluginError { } } return backend.PluginError{ - PluginID: comments.ID, + PluginID: comments.PluginID, ErrorCode: int(s), ErrorContext: e.ErrorContext, } @@ -607,7 +607,7 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str t, err := tokenDecode(n.Token) if err != nil { return "", backend.PluginError{ - PluginID: comments.ID, + PluginID: comments.PluginID, ErrorCode: int(comments.ErrorCodeTokenInvalid), ErrorContext: err.Error(), } @@ -616,7 +616,7 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str e := fmt.Sprintf("comment token does not match route token: "+ "got %x, want %x", t, token) return "", backend.PluginError{ - PluginID: comments.ID, + PluginID: comments.PluginID, ErrorCode: int(comments.ErrorCodeTokenInvalid), ErrorContext: e, } @@ -632,7 +632,7 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str // Verify comment if len(n.Comment) > comments.PolicyCommentLengthMax { return "", backend.PluginError{ - PluginID: comments.ID, + PluginID: comments.PluginID, ErrorCode: int(comments.ErrorCodeCommentTextInvalid), ErrorContext: "exceeds max length", } @@ -654,7 +654,7 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str // this is a base level comment, not a reply to another comment. if n.ParentID > 0 && !commentExists(*ridx, n.ParentID) { return "", backend.PluginError{ - PluginID: comments.ID, + PluginID: comments.PluginID, ErrorCode: int(comments.ErrorCodeParentIDInvalid), ErrorContext: "parent ID comment not found", } @@ -733,7 +733,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st t, err := tokenDecode(e.Token) if err != nil { return "", backend.PluginError{ - PluginID: comments.ID, + PluginID: comments.PluginID, ErrorCode: int(comments.ErrorCodeTokenInvalid), ErrorContext: err.Error(), } @@ -742,7 +742,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st e := fmt.Sprintf("comment token does not match route token: "+ "got %x, want %x", t, token) return "", backend.PluginError{ - PluginID: comments.ID, + PluginID: comments.PluginID, ErrorCode: int(comments.ErrorCodeTokenInvalid), ErrorContext: e, } @@ -758,7 +758,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st // Verify comment if len(e.Comment) > comments.PolicyCommentLengthMax { return "", backend.PluginError{ - PluginID: comments.ID, + PluginID: comments.PluginID, ErrorCode: int(comments.ErrorCodeCommentTextInvalid), ErrorContext: "exceeds max length", } @@ -784,7 +784,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st existing, ok := cs[e.CommentID] if !ok { return "", backend.PluginError{ - PluginID: comments.ID, + PluginID: comments.PluginID, ErrorCode: int(comments.ErrorCodeCommentNotFound), } } @@ -792,7 +792,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st // Verify the user ID if e.UserID != existing.UserID { return "", backend.PluginError{ - PluginID: comments.ID, + PluginID: comments.PluginID, ErrorCode: int(comments.ErrorCodeUserUnauthorized), } } @@ -802,7 +802,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st e := fmt.Sprintf("parent id cannot change; got %v, want %v", e.ParentID, existing.ParentID) return "", backend.PluginError{ - PluginID: comments.ID, + PluginID: comments.PluginID, ErrorCode: int(comments.ErrorCodeParentIDInvalid), ErrorContext: e, } @@ -811,7 +811,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st // Verify comment changes if e.Comment == existing.Comment { return "", backend.PluginError{ - PluginID: comments.ID, + PluginID: comments.PluginID, ErrorCode: int(comments.ErrorCodeCommentTextInvalid), ErrorContext: "comment did not change", } @@ -884,7 +884,7 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str t, err := tokenDecode(d.Token) if err != nil { return "", backend.PluginError{ - PluginID: comments.ID, + PluginID: comments.PluginID, ErrorCode: int(comments.ErrorCodeTokenInvalid), ErrorContext: err.Error(), } @@ -893,7 +893,7 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str e := fmt.Sprintf("comment token does not match route token: "+ "got %x, want %x", t, token) return "", backend.PluginError{ - PluginID: comments.ID, + PluginID: comments.PluginID, ErrorCode: int(comments.ErrorCodeTokenInvalid), ErrorContext: e, } @@ -926,7 +926,7 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str existing, ok := cs[d.CommentID] if !ok { return "", backend.PluginError{ - PluginID: comments.ID, + PluginID: comments.PluginID, ErrorCode: int(comments.ErrorCodeCommentNotFound), } } @@ -1009,7 +1009,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st t, err := tokenDecode(v.Token) if err != nil { return "", backend.PluginError{ - PluginID: comments.ID, + PluginID: comments.PluginID, ErrorCode: int(comments.ErrorCodeTokenInvalid), ErrorContext: err.Error(), } @@ -1018,7 +1018,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st e := fmt.Sprintf("comment token does not match route token: "+ "got %x, want %x", t, token) return "", backend.PluginError{ - PluginID: comments.ID, + PluginID: comments.PluginID, ErrorCode: int(comments.ErrorCodeTokenInvalid), ErrorContext: e, } @@ -1030,7 +1030,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st // These are allowed default: return "", backend.PluginError{ - PluginID: comments.ID, + PluginID: comments.PluginID, ErrorCode: int(comments.ErrorCodeVoteInvalid), } } @@ -1059,7 +1059,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st cidx, ok := ridx.Comments[v.CommentID] if !ok { return "", backend.PluginError{ - PluginID: comments.ID, + PluginID: comments.PluginID, ErrorCode: int(comments.ErrorCodeCommentNotFound), } } @@ -1071,7 +1071,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st } if len(uvotes) > comments.PolicyVoteChangesMax { return "", backend.PluginError{ - PluginID: comments.ID, + PluginID: comments.PluginID, ErrorCode: int(comments.ErrorCodeVoteChangesMax), } } @@ -1087,7 +1087,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st } if v.UserID == c.UserID { return "", backend.PluginError{ - PluginID: comments.ID, + PluginID: comments.PluginID, ErrorCode: int(comments.ErrorCodeVoteInvalid), ErrorContext: "user cannot vote on their own comment", } @@ -1253,13 +1253,13 @@ func (p *commentsPlugin) cmdGetVersion(treeID int64, token []byte, payload strin cidx, ok := ridx.Comments[gv.CommentID] if !ok { return "", backend.PluginError{ - PluginID: comments.ID, + PluginID: comments.PluginID, ErrorCode: int(comments.ErrorCodeCommentNotFound), } } if cidx.Del != nil { return "", backend.PluginError{ - PluginID: comments.ID, + PluginID: comments.PluginID, ErrorCode: int(comments.ErrorCodeCommentNotFound), ErrorContext: "comment has been deleted", } @@ -1269,7 +1269,7 @@ func (p *commentsPlugin) cmdGetVersion(treeID int64, token []byte, payload strin e := fmt.Sprintf("comment %v does not have version %v", gv.CommentID, gv.Version) return "", backend.PluginError{ - PluginID: comments.ID, + PluginID: comments.PluginID, ErrorCode: int(comments.ErrorCodeCommentNotFound), ErrorContext: e, } diff --git a/politeiad/backend/tlogbe/plugins/comments/comments.go b/politeiad/backend/tlogbe/plugins/comments/comments.go index 4db7d7093..92a8dc45a 100644 --- a/politeiad/backend/tlogbe/plugins/comments/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments/comments.go @@ -20,18 +20,6 @@ import ( // TODO upvoting a comment twice in the same second causes a duplicate leaf // error which causes a 500. Solution: add the timestamp to the vote index. -const ( - // Blob entry data descriptors - dataDescriptorCommentAdd = "cadd_v1" - dataDescriptorCommentDel = "cdel_v1" - dataDescriptorCommentVote = "cvote_v1" - - // Data types - dataTypeCommentAdd = "cadd" - dataTypeCommentDel = "cdel" - dataTypeCommentVote = "cvote" -) - var ( _ plugins.Client = (*commentsPlugin)(nil) ) @@ -138,7 +126,7 @@ func (p *commentsPlugin) Fsck(treeIDs []int64) error { // New returns a new comments plugin. func New(tlog plugins.TlogClient, settings []backend.PluginSetting, dataDir string, id *identity.FullIdentity) (*commentsPlugin, error) { // Setup comments plugin data dir - dataDir = filepath.Join(dataDir, comments.ID) + dataDir = filepath.Join(dataDir, comments.PluginID) err := os.MkdirAll(dataDir, 0700) if err != nil { return nil, err diff --git a/politeiad/backend/tlogbe/plugins/pi/cmds.go b/politeiad/backend/tlogbe/plugins/pi/cmds.go index 29b8fefa1..39e654f31 100644 --- a/politeiad/backend/tlogbe/plugins/pi/cmds.go +++ b/politeiad/backend/tlogbe/plugins/pi/cmds.go @@ -69,7 +69,7 @@ func (p *piPlugin) cmdProposalInv() (string, error) { func (p *piPlugin) voteSummary(token []byte) (*ticketvote.VoteSummary, error) { reply, err := p.backend.VettedPluginCmd(token, - ticketvote.ID, ticketvote.CmdSummary, "") + ticketvote.PluginID, ticketvote.CmdSummary, "") if err != nil { return nil, err } @@ -84,10 +84,10 @@ func (p *piPlugin) voteSummary(token []byte) (*ticketvote.VoteSummary, error) { func (p *piPlugin) cmdVoteInv() (string, error) { // Get ticketvote inventory r, err := p.backend.VettedPluginCmd([]byte{}, - ticketvote.ID, ticketvote.CmdInventory, "") + ticketvote.PluginID, ticketvote.CmdInventory, "") if err != nil { return "", fmt.Errorf("VettedPluginCmd %v %v: %v", - ticketvote.ID, ticketvote.CmdInventory, err) + ticketvote.PluginID, ticketvote.CmdInventory, err) } var ir ticketvote.InventoryReply err = json.Unmarshal([]byte(r), &ir) diff --git a/politeiad/backend/tlogbe/plugins/pi/hooks.go b/politeiad/backend/tlogbe/plugins/pi/hooks.go index 2f8c39344..5f6eb6231 100644 --- a/politeiad/backend/tlogbe/plugins/pi/hooks.go +++ b/politeiad/backend/tlogbe/plugins/pi/hooks.go @@ -101,7 +101,7 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { if err != nil { return err } - reply, err := p.backend.Plugin(ticketvote.ID, + reply, err := p.backend.Plugin(ticketvote.PluginID, ticketvote.CmdSummaries, "", string(b)) if err != nil { return fmt.Errorf("ticketvote Summaries: %v", err) @@ -119,7 +119,7 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { ticketvote.VoteStatuses[summary.Status], ticketvote.VoteStatuses[ticketvote.VoteStatusUnauthorized]) return backend.PluginUserError{ - PluginID: pi.ID, + PluginID: pi.PluginID, ErrorCode: int(pi.ErrorCodeVoteStatusInvalid), ErrorContext: e, } @@ -175,7 +175,7 @@ func (p *piPlugin) hookSetRecordStatusPost(payload string) error { e := fmt.Sprintf("version not current: got %v, want %v", sc.Version, srs.Current.Version) return backend.PluginError{ - PluginID: pi.ID, + PluginID: pi.PluginID, ErrorCode: int(pi.ErrorCodePropVersionInvalid), ErrorContext: e, } @@ -194,7 +194,7 @@ func (p *piPlugin) hookSetRecordStatusPost(payload string) error { e := fmt.Sprintf("from %v to %v status change not allowed", from, sc.Status) return backend.PluginError{ - PluginID: pi.ID, + PluginID: pi.PluginID, ErrorCode: int(pi.ErrorCodePropStatusChangeInvalid), ErrorContext: e, } @@ -219,7 +219,7 @@ func (p *piPlugin) commentWritesVerify(token []byte) error { return nil default: return backend.PluginError{ - PluginID: pi.ID, + PluginID: pi.PluginID, ErrorCode: int(pi.ErrorCodeVoteStatusInvalid), ErrorContext: "vote has ended; proposal is locked", } @@ -248,7 +248,7 @@ func (p *piPlugin) hookPluginPre(treeID int64, token []byte, payload string) err // Call plugin hook switch hpp.PluginID { - case comments.ID: + case comments.PluginID: switch hpp.Cmd { case comments.CmdNew: return p.hookCommentNew(token) diff --git a/politeiad/backend/tlogbe/plugins/pi/pi.go b/politeiad/backend/tlogbe/plugins/pi/pi.go index d32700957..425bca797 100644 --- a/politeiad/backend/tlogbe/plugins/pi/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi/pi.go @@ -87,7 +87,7 @@ func (p *piPlugin) Fsck(treeIDs []int64) error { func New(backend backend.Backend, settings []backend.PluginSetting, dataDir string) (*piPlugin, error) { // Create plugin data directory - dataDir = filepath.Join(dataDir, pi.ID) + dataDir = filepath.Join(dataDir, pi.PluginID) err := os.MkdirAll(dataDir, 0700) if err != nil { return nil, err diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go index 36a631bde..b61510901 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go @@ -70,7 +70,7 @@ func convertSignatureError(err error) backend.PluginError { } } return backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(s), ErrorContext: e.ErrorContext, } @@ -373,7 +373,7 @@ func (p *ticketVotePlugin) voteDetails(treeID int64) (*ticketvote.VoteDetails, e } func (p *ticketVotePlugin) voteDetailsByToken(token []byte) (*ticketvote.VoteDetails, error) { - reply, err := p.backend.VettedPluginCmd(token, ticketvote.ID, + reply, err := p.backend.VettedPluginCmd(token, ticketvote.PluginID, ticketvote.CmdDetails, "") if err != nil { return nil, err @@ -439,7 +439,7 @@ func (p *ticketVotePlugin) voteOptionResults(token []byte, options []ticketvote. default: // Votes are not in the cache. Pull them from the backend. - reply, err := p.backend.VettedPluginCmd(token, ticketvote.ID, + reply, err := p.backend.VettedPluginCmd(token, ticketvote.PluginID, ticketvote.CmdResults, "") var rr ticketvote.ResultsReply err = json.Unmarshal([]byte(reply), &rr) @@ -476,11 +476,11 @@ func (p *ticketVotePlugin) summariesForRunoff(parentToken string) (map[string]ti if err != nil { return nil, err } - reply, err := p.backend.VettedPluginCmd(parent, ticketvote.ID, + reply, err := p.backend.VettedPluginCmd(parent, ticketvote.PluginID, cmdRunoffDetails, "") if err != nil { return nil, fmt.Errorf("VettedPluginCmd %x %v %v: %v", - parent, ticketvote.ID, cmdRunoffDetails, err) + parent, ticketvote.PluginID, cmdRunoffDetails, err) } var rdr runoffDetailsReply err = json.Unmarshal([]byte(reply), &rdr) @@ -717,11 +717,11 @@ func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) } func (p *ticketVotePlugin) summaryByToken(token []byte) (*ticketvote.VoteSummary, error) { - reply, err := p.backend.VettedPluginCmd(token, ticketvote.ID, + reply, err := p.backend.VettedPluginCmd(token, ticketvote.PluginID, ticketvote.CmdSummary, "") if err != nil { return nil, fmt.Errorf("VettedPluginCmd %x %v %v: %v", - token, ticketvote.ID, ticketvote.CmdSummary, err) + token, ticketvote.PluginID, ticketvote.CmdSummary, err) } var sr ticketvote.SummaryReply err = json.Unmarshal([]byte(reply), &sr) @@ -765,11 +765,11 @@ func (p *ticketVotePlugin) bestBlock() (uint32, error) { if err != nil { return 0, err } - reply, err := p.backend.VettedPluginCmd([]byte{}, dcrdata.ID, + reply, err := p.backend.VettedPluginCmd([]byte{}, dcrdata.PluginID, dcrdata.CmdBestBlock, string(payload)) if err != nil { return 0, fmt.Errorf("Plugin %v %v: %v", - dcrdata.ID, dcrdata.CmdBestBlock, err) + dcrdata.PluginID, dcrdata.CmdBestBlock, err) } // Handle response @@ -801,11 +801,11 @@ func (p *ticketVotePlugin) bestBlockUnsafe() (uint32, error) { if err != nil { return 0, err } - reply, err := p.backend.VettedPluginCmd([]byte{}, dcrdata.ID, + reply, err := p.backend.VettedPluginCmd([]byte{}, dcrdata.PluginID, dcrdata.CmdBestBlock, string(payload)) if err != nil { return 0, fmt.Errorf("Plugin %v %v: %v", - dcrdata.ID, dcrdata.CmdBestBlock, err) + dcrdata.PluginID, dcrdata.CmdBestBlock, err) } // Handle response @@ -836,11 +836,11 @@ func (p *ticketVotePlugin) largestCommitmentAddrs(tickets []string) ([]commitmen if err != nil { return nil, err } - reply, err := p.backend.VettedPluginCmd([]byte{}, dcrdata.ID, + reply, err := p.backend.VettedPluginCmd([]byte{}, dcrdata.PluginID, dcrdata.CmdTxsTrimmed, string(payload)) if err != nil { return nil, fmt.Errorf("Plugin %v %v: %v", - dcrdata.ID, dcrdata.CmdTxsTrimmed, err) + dcrdata.PluginID, dcrdata.CmdTxsTrimmed, err) } var ttr dcrdata.TxsTrimmedReply err = json.Unmarshal([]byte(reply), &ttr) @@ -906,11 +906,11 @@ func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, if err != nil { return nil, err } - reply, err := p.backend.VettedPluginCmd([]byte{}, dcrdata.ID, + reply, err := p.backend.VettedPluginCmd([]byte{}, dcrdata.PluginID, dcrdata.CmdBlockDetails, string(payload)) if err != nil { return nil, fmt.Errorf("Plugin %v %v: %v", - dcrdata.ID, dcrdata.CmdBlockDetails, err) + dcrdata.PluginID, dcrdata.CmdBlockDetails, err) } var bdr dcrdata.BlockDetailsReply err = json.Unmarshal([]byte(reply), &bdr) @@ -930,11 +930,11 @@ func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, if err != nil { return nil, err } - reply, err = p.backend.VettedPluginCmd([]byte{}, dcrdata.ID, + reply, err = p.backend.VettedPluginCmd([]byte{}, dcrdata.PluginID, dcrdata.CmdTicketPool, string(payload)) if err != nil { return nil, fmt.Errorf("Plugin %v %v: %v", - dcrdata.ID, dcrdata.CmdTicketPool, err) + dcrdata.PluginID, dcrdata.CmdTicketPool, err) } var tpr dcrdata.TicketPoolReply err = json.Unmarshal([]byte(reply), &tpr) @@ -974,7 +974,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri t, err := tokenDecode(a.Token) if err != nil { return "", backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), } } @@ -982,7 +982,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri e := fmt.Sprintf("plugin token does not match route token: "+ "got %x, want %x", t, token) return "", backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), ErrorContext: e, } @@ -1005,7 +1005,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri default: e := fmt.Sprintf("%v not a valid action", a.Action) return "", backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), ErrorContext: e, } @@ -1033,7 +1033,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri // No previous actions. New action must be an authorize. if a.Action != ticketvote.AuthActionAuthorize { return "", backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), ErrorContext: "no prev action; action must be authorize", } @@ -1042,7 +1042,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri a.Action != ticketvote.AuthActionRevoke: // Previous action was a authorize. This action must be revoke. return "", backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), ErrorContext: "prev action was authorize", } @@ -1050,7 +1050,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri a.Action != ticketvote.AuthActionAuthorize: // Previous action was a revoke. This action must be authorize. return "", backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), ErrorContext: "prev action was revoke", } @@ -1133,7 +1133,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM default: e := fmt.Sprintf("invalid type %v", vote.Type) return backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } @@ -1145,7 +1145,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM e := fmt.Sprintf("duration %v exceeds max duration %v", vote.Duration, voteDurationMax) return backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } @@ -1153,7 +1153,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM e := fmt.Sprintf("duration %v under min duration %v", vote.Duration, voteDurationMin) return backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } @@ -1161,7 +1161,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM e := fmt.Sprintf("quorum percent %v exceeds 100 percent", vote.QuorumPercentage) return backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } @@ -1169,7 +1169,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM e := fmt.Sprintf("pass percent %v exceeds 100 percent", vote.PassPercentage) return backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } @@ -1179,7 +1179,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM // requirements. if len(vote.Options) == 0 { return backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: "no vote options found", } @@ -1193,7 +1193,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM e := fmt.Sprintf("vote options count got %v, want 2", len(vote.Options)) return backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } @@ -1222,7 +1222,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM e := fmt.Sprintf("vote option IDs not found: %v", strings.Join(missing, ",")) return backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } @@ -1234,7 +1234,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM err := voteBitVerify(vote.Options, vote.Mask, v.Bit) if err != nil { return backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: err.Error(), } @@ -1246,7 +1246,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM case vote.Type == ticketvote.VoteTypeStandard && vote.Parent != "": e := "parent token should not be provided for a standard vote" return backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } @@ -1255,7 +1255,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM if err != nil { e := fmt.Sprintf("invalid parent %v", vote.Parent) return backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } @@ -1270,7 +1270,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot // Verify there is only one start details if len(s.Starts) != 1 { return nil, backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeStartDetailsInvalid), ErrorContext: "more than one start details found", } @@ -1281,7 +1281,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot t, err := tokenDecode(sd.Params.Token) if err != nil { return nil, backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), } } @@ -1289,7 +1289,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot e := fmt.Sprintf("plugin token does not match route token: "+ "got %x, want %x", t, token) return nil, backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), ErrorContext: e, } @@ -1334,7 +1334,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot e := fmt.Sprintf("version is not latest: got %v, want %v", sd.Params.Version, r.Version) return nil, backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeRecordVersionInvalid), ErrorContext: e, } @@ -1347,7 +1347,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot } if len(auths) == 0 { return nil, backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), ErrorContext: "authorization not found", } @@ -1355,7 +1355,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot action := ticketvote.AuthActionT(auths[len(auths)-1].Action) if action != ticketvote.AuthActionAuthorize { return nil, backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), ErrorContext: "not authorized", } @@ -1369,7 +1369,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot if svp != nil { // Vote has already been started return nil, backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeVoteStatusInvalid), ErrorContext: "vote already started", } @@ -1551,7 +1551,7 @@ func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs sta e := fmt.Sprintf("version is not latest %v: got %v, want %v", s.Params.Token, s.Params.Version, r.Version) return backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeRecordVersionInvalid), ErrorContext: e, } @@ -1619,7 +1619,7 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti if errors.Is(err, backend.ErrRecordNotFound) { e := fmt.Sprintf("parent record not found %x", token) return nil, backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } @@ -1631,7 +1631,7 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti } if vm == nil || vm.LinkBy == 0 { return nil, backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeRunoffVoteParentInvalid), } } @@ -1642,7 +1642,7 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti e := fmt.Sprintf("parent record %x linkby deadline not met %v", token, vm.LinkBy) return nil, backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeLinkByNotExpired), ErrorContext: e, } @@ -1692,7 +1692,7 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti e := fmt.Sprintf("record %v should not be included", v.Params.Token) return nil, backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeStartDetailsInvalid), ErrorContext: e, } @@ -1709,7 +1709,7 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti if !ok { // This records is missing from the runoff vote return nil, backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeStartDetailsMissing), ErrorContext: k, } @@ -1766,7 +1766,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. e := fmt.Sprintf("%v vote type invalid: got %v, want %v", v.Params.Token, v.Params.Type, ticketvote.VoteTypeRunoff) return nil, backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } @@ -1774,7 +1774,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. e := fmt.Sprintf("%v mask invalid: all must be the same", v.Params.Token) return nil, backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } @@ -1782,7 +1782,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. e := fmt.Sprintf("%v duration invalid: all must be the same", v.Params.Token) return nil, backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } @@ -1790,7 +1790,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. e := fmt.Sprintf("%v quorum invalid: must be the same", v.Params.Token) return nil, backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } @@ -1798,7 +1798,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. e := fmt.Sprintf("%v pass rate invalid: all must be the same", v.Params.Token) return nil, backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } @@ -1806,7 +1806,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. e := fmt.Sprintf("%v parent invalid: all must be the same", v.Params.Token) return nil, backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } @@ -1816,7 +1816,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. _, err := tokenDecode(v.Params.Token) if err != nil { return nil, backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), ErrorContext: v.Params.Token, } @@ -1827,7 +1827,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. if err != nil { e := fmt.Sprintf("parent token %v", v.Params.Parent) return nil, backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), ErrorContext: e, } @@ -1857,7 +1857,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. e := fmt.Sprintf("runoff vote must be started on parent record %v", parent) return nil, backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeRunoffVoteParentInvalid), ErrorContext: e, } @@ -1886,7 +1886,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. if err != nil { return nil, err } - _, err = p.backend.VettedPluginCmd(token, ticketvote.ID, + _, err = p.backend.VettedPluginCmd(token, ticketvote.PluginID, cmdStartRunoffSubmission, string(b)) if err != nil { var ue backend.PluginError @@ -1894,7 +1894,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. return nil, err } return nil, fmt.Errorf("VettedPluginCmd %x %v %v %v: %v", - token, ticketvote.ID, cmdStartRunoffSubmission, b, err) + token, ticketvote.PluginID, cmdStartRunoffSubmission, b, err) } } @@ -1945,7 +1945,7 @@ func (p *ticketVotePlugin) cmdStart(treeID int64, token []byte, payload string) // Parse vote type if len(s.Starts) == 0 { return "", backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeStartDetailsMissing), ErrorContext: "no start details found", } @@ -1970,7 +1970,7 @@ func (p *ticketVotePlugin) cmdStart(treeID int64, token []byte, payload string) default: e := fmt.Sprintf("invalid vote type %v", vtype) return "", backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), ErrorContext: e, } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go b/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go index 658a4f2ce..9b499caea 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go @@ -50,7 +50,7 @@ func (p *ticketVotePlugin) linkByVerify(linkBy int64) error { e := fmt.Sprintf("linkby %v is less than min required of %v", linkBy, min) return backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeLinkByInvalid), ErrorContext: e, } @@ -58,7 +58,7 @@ func (p *ticketVotePlugin) linkByVerify(linkBy int64) error { e := fmt.Sprintf("linkby %v is more than max allowed of %v", linkBy, max) return backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeLinkByInvalid), ErrorContext: e, } @@ -71,7 +71,7 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { token, err := tokenDecode(linkTo) if err != nil { return backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), ErrorContext: "invalid hex", } @@ -80,7 +80,7 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { if err != nil { if errors.Is(err, backend.ErrRecordNotFound) { return backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), ErrorContext: "record not found", } @@ -89,7 +89,7 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { } if r.RecordMetadata.Status != backend.MDStatusCensored { return backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), ErrorContext: "record is censored", } @@ -103,7 +103,7 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { } if parentVM == nil || parentVM.LinkBy == 0 { return backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), ErrorContext: "record not a runoff vote parent", } @@ -112,7 +112,7 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { // The LinkBy deadline must not be expired if time.Now().Unix() > parentVM.LinkBy { return backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), ErrorContext: "parent record linkby deadline has expired", } @@ -125,7 +125,7 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { } if !vs.Approved { return backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), ErrorContext: "parent record vote is not approved", } @@ -139,7 +139,7 @@ func (p *ticketVotePlugin) voteMetadataVerify(vm ticketvote.VoteMetadata) error case vm.LinkBy == 0 && vm.LinkTo == "": // Vote metadata is empty return backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeVoteMetadataInvalid), ErrorContext: "md is empty", } @@ -147,7 +147,7 @@ func (p *ticketVotePlugin) voteMetadataVerify(vm ticketvote.VoteMetadata) error case vm.LinkBy != 0 && vm.LinkTo != "": // LinkBy and LinkTo cannot both be set return backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeVoteMetadataInvalid), ErrorContext: "cannot set both linkby and linkto", } @@ -223,7 +223,7 @@ func (p *ticketVotePlugin) hookEditRecordPre(payload string) error { e := fmt.Sprintf("linkto cannot change on vetted record: "+ "got '%v', want '%v'", newLinkTo, oldLinkTo) return backend.PluginError{ - PluginID: ticketvote.ID, + PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), ErrorContext: e, } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go index 094223a87..de90445a0 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go @@ -94,12 +94,13 @@ func (p *ticketVotePlugin) Setup() error { // Verify plugin dependencies var dcrdataFound bool for _, v := range p.backend.GetVettedPlugins() { - if v.ID == dcrdata.ID { + if v.ID == dcrdata.PluginID { dcrdataFound = true } } if !dcrdataFound { - return fmt.Errorf("plugin dependency not registered: %v", dcrdata.ID) + return fmt.Errorf("plugin dependency not registered: %v", + dcrdata.PluginID) } // Build inventory cache @@ -161,11 +162,11 @@ func (p *ticketVotePlugin) Setup() error { if err != nil { return err } - reply, err := p.backend.VettedPluginCmd(token, ticketvote.ID, + reply, err := p.backend.VettedPluginCmd(token, ticketvote.PluginID, ticketvote.CmdResults, "") if err != nil { return fmt.Errorf("VettedPluginCmd %x %v %v: %v", - token, ticketvote.ID, ticketvote.CmdResults, err) + token, ticketvote.PluginID, ticketvote.CmdResults, err) } var rr ticketvote.ResultsReply err = json.Unmarshal([]byte(reply), &rr) @@ -321,7 +322,7 @@ func New(backend backend.Backend, tlog plugins.TlogClient, settings []backend.Pl } // Create the plugin data directory - dataDir = filepath.Join(dataDir, ticketvote.ID) + dataDir = filepath.Join(dataDir, ticketvote.PluginID) err := os.MkdirAll(dataDir, 0700) if err != nil { return nil, err diff --git a/politeiad/backend/tlogbe/tlog/plugin.go b/politeiad/backend/tlogbe/tlog/plugin.go index 2540a7382..aead92479 100644 --- a/politeiad/backend/tlogbe/tlog/plugin.go +++ b/politeiad/backend/tlogbe/tlog/plugin.go @@ -67,22 +67,22 @@ func (t *Tlog) PluginRegister(b backend.Backend, p backend.Plugin) error { dataDir = filepath.Join(t.dataDir, pluginDataDirname) ) switch p.ID { - case cmplugin.ID: + case cmplugin.PluginID: client, err = comments.New(t, p.Settings, dataDir, p.Identity) if err != nil { return err } - case ddplugin.ID: + case ddplugin.PluginID: client, err = dcrdata.New(p.Settings, t.activeNetParams) if err != nil { return err } - case piplugin.ID: + case piplugin.PluginID: client, err = pi.New(b, p.Settings, dataDir) if err != nil { return err } - case tkplugin.ID: + case tkplugin.PluginID: client, err = ticketvote.New(b, t, p.Settings, dataDir, p.Identity, t.activeNetParams) if err != nil { diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 25145df76..4f0be5f86 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -830,12 +830,14 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite return fmt.Errorf("RecordLatest: %v", err) } + // Apply changes + metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) + // Call pre plugin hooks hem := plugins.HookEditMetadata{ - State: plugins.RecordStateUnvetted, - Current: *r, - MDAppend: mdAppend, - MDOverwrite: mdOverwrite, + State: plugins.RecordStateUnvetted, + Current: *r, + Metadata: metadata, } b, err := json.Marshal(hem) if err != nil { @@ -847,9 +849,6 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite return err } - // Apply changes - metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) - // Update metadata err = t.unvetted.RecordMetadataSave(treeID, r.RecordMetadata, metadata) if err != nil { @@ -922,12 +921,14 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ return fmt.Errorf("RecordLatest: %v", err) } + // Apply changes + metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) + // Call pre plugin hooks hem := plugins.HookEditMetadata{ - State: plugins.RecordStateVetted, - Current: *r, - MDAppend: mdAppend, - MDOverwrite: mdOverwrite, + State: plugins.RecordStateVetted, + Current: *r, + Metadata: metadata, } b, err := json.Marshal(hem) if err != nil { @@ -939,9 +940,6 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ return err } - // Apply changes - metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) - // Update metadata err = t.vetted.RecordMetadataSave(treeID, r.RecordMetadata, metadata) if err != nil { diff --git a/politeiad/client/comments.go b/politeiad/client/comments.go new file mode 100644 index 000000000..78ed611d6 --- /dev/null +++ b/politeiad/client/comments.go @@ -0,0 +1,92 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package client + +import ( + "context" + "encoding/json" + "fmt" + + pdv1 "github.com/decred/politeia/politeiad/api/v1" + "github.com/decred/politeia/politeiad/plugins/comments" +) + +func (c *Client) CommentNew(ctx context.Context, state, token string, n comments.New) (*comments.NewReply, error) { + // Setup request + b, err := json.Marshal(n) + if err != nil { + return nil, err + } + cmds := []pdv1.PluginCommandV2{ + { + State: state, + Token: token, + ID: comments.PluginID, + Command: comments.CmdNew, + Payload: string(b), + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + + // Decode reply + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + if pcr.Error != nil { + return nil, pcr.Error + } + var nr comments.NewReply + err = json.Unmarshal([]byte(pcr.Payload), &nr) + if err != nil { + return nil, err + } + + return &nr, nil +} + +func (c *Client) CommentVote(ctx context.Context, state, token string, v comments.Vote) (*comments.VoteReply, error) { + // Setup request + b, err := json.Marshal(v) + if err != nil { + return nil, err + } + cmds := []pdv1.PluginCommandV2{ + { + State: state, + Token: token, + ID: comments.PluginID, + Command: comments.CmdVote, + Payload: string(b), + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + + // Decode reply + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + if pcr.Error != nil { + return nil, pcr.Error + } + var nr comments.VoteReply + err = json.Unmarshal([]byte(pcr.Payload), &nr) + if err != nil { + return nil, err + } + + return &nr, nil +} diff --git a/politeiad/client/pdv1.go b/politeiad/client/pdv1.go index f32c09262..c8c558743 100644 --- a/politeiad/client/pdv1.go +++ b/politeiad/client/pdv1.go @@ -527,14 +527,12 @@ func (c *Client) PluginInventory(ctx context.Context) ([]pdv1.Plugin, error) { return nil, err } - // Receive reply + // Decode reply var pir pdv1.PluginInventoryReply err = json.Unmarshal(resBody, &pir) if err != nil { return nil, err } - - // Verify challenge err = util.VerifyChallenge(c.pid, challenge, pir.Response) if err != nil { return nil, err diff --git a/politeiad/client/user.go b/politeiad/client/user.go new file mode 100644 index 000000000..553486056 --- /dev/null +++ b/politeiad/client/user.go @@ -0,0 +1,94 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package client + +import ( + "context" + "encoding/json" + "fmt" + + pdv1 "github.com/decred/politeia/politeiad/api/v1" + "github.com/decred/politeia/politeiad/plugins/user" +) + +// Author sends the user plugin Author command to the politeiad v1 API. +func (c *Client) Author(ctx context.Context, state, token string) (string, error) { + // Setup request + cmds := []pdv1.PluginCommandV2{ + { + State: state, + Token: token, + ID: user.PluginID, + Command: user.CmdAuthor, + Payload: "", + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return "", err + } + + // Decode reply + if len(replies) == 0 { + return "", fmt.Errorf("no replies found") + } + pcr := replies[0] + if pcr.Error != nil { + return "", pcr.Error + } + var ar user.AuthorReply + err = json.Unmarshal([]byte(pcr.Payload), &ar) + if err != nil { + return "", err + } + + return ar.UserID, nil +} + +// UserRecords sends the user plugin UserRecords command to the politeiad v1 +// API. +func (c *Client) UserRecords(ctx context.Context, state, token, userID string) ([]string, error) { + // Setup request + ur := user.UserRecords{ + UserID: userID, + } + b, err := json.Marshal(ur) + if err != nil { + return nil, err + } + cmds := []pdv1.PluginCommandV2{ + { + State: state, + Token: token, + ID: user.PluginID, + Command: user.CmdUserRecords, + Payload: string(b), + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + + // Decode reply + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + if pcr.Error != nil { + return nil, pcr.Error + } + var urr user.UserRecordsReply + err = json.Unmarshal([]byte(pcr.Payload), &urr) + if err != nil { + return nil, err + } + + return urr.Records, nil +} diff --git a/politeiad/plugins/comments/comments.go b/politeiad/plugins/comments/comments.go index e7ec14a9f..1203736cf 100644 --- a/politeiad/plugins/comments/comments.go +++ b/politeiad/plugins/comments/comments.go @@ -7,7 +7,7 @@ package comments const ( - ID = "comments" + PluginID = "comments" // Plugin commands CmdNew = "new" // Create a new comment diff --git a/politeiad/plugins/dcrdata/dcrdata.go b/politeiad/plugins/dcrdata/dcrdata.go index daceff121..1a9235625 100644 --- a/politeiad/plugins/dcrdata/dcrdata.go +++ b/politeiad/plugins/dcrdata/dcrdata.go @@ -9,7 +9,7 @@ package dcrdata type StatusT int const ( - ID = "dcrdata" + PluginID = "dcrdata" // Plugin commands CmdBestBlock = "bestblock" // Get best block diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index b7e1d1a3d..28d90907b 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -7,7 +7,7 @@ package pi const ( - ID = "pi" + PluginID = "pi" // Plugin commands // TODO I might not need the ProposalInv command diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index ac31271d5..3606651a9 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -16,7 +16,7 @@ package ticketvote // Max (41k votes): 82MB const ( - ID = "ticketvote" + PluginID = "ticketvote" // Plugin commands CmdAuthorize = "authorize" // Authorize a vote diff --git a/politeiawww/api/comments/v1/v1.go b/politeiawww/api/comments/v1/v1.go index dcde06357..8c44df207 100644 --- a/politeiawww/api/comments/v1/v1.go +++ b/politeiawww/api/comments/v1/v1.go @@ -10,6 +10,7 @@ const ( // APIRoute is prefixed onto all routes defined in this package. APIRoute = "/comments/v1" + // Routes RouteNew = "/new" RouteVote = "/vote" RouteDel = "/del" @@ -17,6 +18,10 @@ const ( RouteComments = "/comments" RouteVotes = "/votes" RouteTimestamps = "/timestamps" + + // Record states + RecordStateUnvetted = "unvetted" + RecordStateVetted = "vetted" ) // ErrorCodeT represents a user error code. @@ -26,17 +31,18 @@ const ( // Error codes ErrorCodeInvalid ErrorCodeT = 0 ErrorCodeInputInvalid ErrorCodeT = iota - ErrorStatusPublicKeyInvalid - ErrorStatusSignatureInvalid + ErrorCodePublicKeyInvalid + ErrorCodeSignatureInvalid + ErrorCodeRecordStateInvalid ) var ( // ErrorCodes contains the human readable errors. ErrorCodes = map[ErrorCodeT]string{ - ErrorCodeInvalid: "error invalid", - ErrorCodeInputInvalid: "input invalid", - ErrorStatusPublicKeyInvalid: "public key invalid", - ErrorStatusSignatureInvalid: "signature invalid", + ErrorCodeInvalid: "error invalid", + ErrorCodeInputInvalid: "input invalid", + ErrorCodePublicKeyInvalid: "public key invalid", + ErrorCodeSignatureInvalid: "signature invalid", } ) @@ -79,21 +85,6 @@ func (e ServerErrorReply) Error() string { return fmt.Sprintf("server error: %v", e.ErrorCode) } -// RecordStateT represents a record state. -type RecordStateT int - -const ( - // RecordStateInvalid indicates an invalid record state. - RecordStateInvalid RecordStateT = 0 - - // RecordStateUnvetted indicates a record has not been made public - // yet. - RecordStateUnvetted RecordStateT = 1 - - // RecordStateVetted indicates a record has been made public. - RecordStateVetted RecordStateT = 2 -) - // Comment represent a record comment. // // Signature is the client signature of Token+ParentID+Comment. @@ -141,12 +132,12 @@ type CommentVote struct { // // Signature is the client signature of Token+ParentID+Comment. type New struct { - State RecordStateT `json:"state"` - Token string `json:"token"` - ParentID uint32 `json:"parentid"` - Comment string `json:"comment"` - PublicKey string `json:"publickey"` - Signature string `json:"signature"` + State string `json:"state"` + Token string `json:"token"` + ParentID uint32 `json:"parentid"` + Comment string `json:"comment"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` // Optional fields to be used freely ExtraData string `json:"extradata,omitempty"` @@ -181,12 +172,12 @@ const ( // // Signature is the client signature of the Token+CommentID+Vote. type Vote struct { - State RecordStateT `json:"state"` - Token string `json:"token"` - CommentID uint32 `json:"commentid"` - Vote VoteT `json:"vote"` - PublicKey string `json:"publickey"` - Signature string `json:"signature"` + State string `json:"state"` + Token string `json:"token"` + CommentID uint32 `json:"commentid"` + Vote VoteT `json:"vote"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` } // VoteReply is the reply to the Vote command. @@ -202,12 +193,12 @@ type VoteReply struct { // // Signature is the client signature of the Token+CommentID+Reason type Del struct { - State RecordStateT `json:"state"` - Token string `json:"token"` - CommentID uint32 `json:"commentid"` - Reason string `json:"reason"` - PublicKey string `json:"publickey"` - Signature string `json:"signature"` + State string `json:"state"` + Token string `json:"token"` + CommentID uint32 `json:"commentid"` + Reason string `json:"reason"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` } // DelReply is the reply to the Del command. @@ -219,8 +210,8 @@ type DelReply struct { // records. If a record is not found for a token then it will not be included // in the reply. type Count struct { - State RecordStateT `json:"state"` - Tokens []string `json:"tokens"` + State string `json:"state"` + Tokens []string `json:"tokens"` } // CountReply is the reply to the count command. @@ -230,8 +221,8 @@ type CountReply struct { // Comments requests a record's comments. type Comments struct { - State RecordStateT `json:"state"` - Token string `json:"token"` + State string `json:"state"` + Token string `json:"token"` } // CommentsReply is the reply to the comments command. @@ -241,8 +232,8 @@ type CommentsReply struct { // Votes returns the comment votes that meet the provided filtering criteria. type Votes struct { - State RecordStateT `json:"state"` - UserID string `json:"userid"` + State string `json:"state"` + UserID string `json:"userid"` } // VotesReply is the reply to the Votes command. @@ -282,9 +273,9 @@ type Timestamp struct { // comment IDs are provided then the timestamps for all comments will be // returned. type Timestamps struct { - State RecordStateT `json:"state"` - Token string `json:"token"` - CommentIDs []uint32 `json:"commentids,omitempty"` + State string `json:"state"` + Token string `json:"token"` + CommentIDs []uint32 `json:"commentids,omitempty"` } // TimestampsReply is the reply to the Timestamps command. diff --git a/politeiawww/comments.go b/politeiawww/comments.go index 76707ac51..edc7fc9b7 100644 --- a/politeiawww/comments.go +++ b/politeiawww/comments.go @@ -5,35 +5,6 @@ package main /* -func convertCommentVoteFromPi(cv piv1.CommentVoteT) comments.VoteT { - switch cv { - case piv1.CommentVoteDownvote: - return comments.VoteUpvote - case piv1.CommentVoteUpvote: - return comments.VoteDownvote - } - return comments.VoteInvalid -} - -func convertCommentFromPlugin(c comments.Comment) piv1.Comment { - return piv1.Comment{ - UserID: c.UserID, - Username: "", // Intentionally omitted, needs to be pulled from userdb - Token: c.Token, - ParentID: c.ParentID, - Comment: c.Comment, - PublicKey: c.PublicKey, - Signature: c.Signature, - CommentID: c.CommentID, - Timestamp: c.Timestamp, - Receipt: c.Receipt, - Downvotes: c.Downvotes, - Upvotes: c.Upvotes, - Censored: c.Deleted, - Reason: c.Reason, - } -} - func convertCommentVoteFromPlugin(v comments.VoteT) piv1.CommentVoteT { switch v { case comments.VoteDownvote: @@ -61,7 +32,6 @@ func convertCommentVoteDetailsFromPlugin(cv []comments.CommentVote) []piv1.Comme return c } - func convertProofFromCommentsPlugin(p comments.Proof) cmv1.Proof { return cmv1.Proof{ Type: p.Type, @@ -86,22 +56,6 @@ func convertTimestampFromCommentsPlugin(t comments.Timestamp) cmv1.Timestamp { } } -func (p *politeiawww) commentsAll(ctx context.Context, ga comments.GetAll) (*comments.GetAllReply, error) { - return nil, nil -} - -func (p *politeiawww) commentsGet(ctx context.Context, cg comments.Get) (*comments.GetReply, error) { - return nil, nil -} - -func (p *politeiawww) commentVotes(ctx context.Context, vs comments.Votes) (*comments.VotesReply, error) { - return nil, nil -} - -func (p *politeiawww) commentTimestamps(ctx context.Context, t comments.Timestamps) (*comments.TimestampsReply, error) { - return nil, nil -} - // commentPopulateUser populates the provided comment with user data that is // not stored in politeiad. func commentPopulateUser(c piv1.Comment, u user.User) cmv1.Comment { @@ -109,140 +63,7 @@ func commentPopulateUser(c piv1.Comment, u user.User) cmv1.Comment { return c } -func (p *politeiawww) commentNewPi(ctx context.Context, n cmv1.CommentNew, u user.User) error { - // Verify user has paid registration paywall - if !p.userHasPaid(u) { - return nil, cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodeUserRegistrationNotPaid, - } - } - return nil -} - -func (p *politeiawww) processCommentNew(ctx context.Context, n cmv1.CommentNew, u user.User) (*cmv1.CommentNewReply, error) { - log.Tracef("processCommentNew: %v %v", n.Token, u.Username) - - // This is temporary until user plugins are implemented. - switch p.mode { - case politeiaWWWMode: - err := p.commentNewPi(ctx, n, u) - if err != nil { - return nil, err - } - } - - // Verify user signed using active identity - if u.PublicKey() != n.PublicKey { - return nil, cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodePublicKeyInvalid, - ErrorContext: "not active identity", - } - } - - // Only admins and the record author are allowed to comment on - // unvetted records. - if n.State == cmv1.PropStateUnvetted && !u.Admin { - // Get the record author - // TODO create a user politeiad plugin - // TODO add command to get author for record - // Fetch the proposal so we can see who the author is - pr, err := p.proposalRecordLatest(ctx, n.State, n.Token) - if err != nil { - if errors.Is(err, errProposalNotFound) { - return nil, cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodeRecordNotFound, - } - } - return nil, fmt.Errorf("proposalRecordLatest: %v", err) - } - if u.ID.String() != pr.UserID { - return nil, cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodeUnauthorized, - ErrorContext: "user is not author or admin", - } - } - } - - // Send plugin command - n := comments.New{ - UserID: usr.ID.String(), - Token: n.Token, - ParentID: n.ParentID, - Comment: n.Comment, - PublicKey: n.PublicKey, - Signature: n.Signature, - } - // TODO - _ = n - var nr comments.NewReply - - // Prepare reply - c := convertCommentFromPlugin(nr.Comment) - c = commentPopulateUser(c, u) - - // Emit event - p.eventManager.emit(eventProposalComment, - dataProposalComment{ - state: c.State, - token: c.Token, - commentID: c.CommentID, - parentID: c.ParentID, - username: c.Username, - }) - - return &cmv1.CommentNewReply{ - Comment: c, - }, nil -} - -func (p *politeiawww) processCommentVote(ctx context.Context, cv cmv1.CommentVote, usr user.User) (*cmv1.CommentVoteReply, error) { - log.Tracef("processCommentVote: %v %v %v", cv.Token, cv.CommentID, cv.Vote) - - // Verify state - if cv.State != cmv1.PropStateVetted { - return nil, cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodePropStateInvalid, - ErrorContext: "proposal must be vetted", - } - } - - // Verify user has paid registration paywall - if !p.userHasPaid(usr) { - return nil, cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodeUserRegistrationNotPaid, - } - } - - // Verify user signed using active identity - if usr.PublicKey() != cv.PublicKey { - return nil, cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodePublicKeyInvalid, - ErrorContext: "not active identity", - } - } - - // Send plugin command - v := comments.Vote{ - UserID: usr.ID.String(), - Token: cv.Token, - CommentID: cv.CommentID, - Vote: convertCommentVoteFromPi(cv.Vote), - PublicKey: cv.PublicKey, - Signature: cv.Signature, - } - // TODO - _ = v - var vr comments.VoteReply - - return &cmv1.CommentVoteReply{ - Downvotes: vr.Downvotes, - Upvotes: vr.Upvotes, - Timestamp: vr.Timestamp, - Receipt: vr.Receipt, - }, nil -} - -func (p *politeiawww) processCommentCensor(ctx context.Context, cc cmv1.CommentCensor, usr user.User) (*cmv1.CommentCensorReply, error) { +func (c *Comments) processCommentCensor(ctx context.Context, cc cmv1.CommentCensor, usr user.User) (*cmv1.CommentCensorReply, error) { log.Tracef("processCommentCensor: %v %v", cc.Token, cc.CommentID) // Sanity check @@ -279,13 +100,13 @@ func (p *politeiawww) processCommentCensor(ctx context.Context, cc cmv1.CommentC }, nil } -func (p *politeiawww) processComments(ctx context.Context, c cmv1.Comments, usr *user.User) (*cmv1.CommentsReply, error) { +func (c *Comments) processComments(ctx context.Context, c cmv1.Comments, usr *user.User) (*cmv1.CommentsReply, error) { log.Tracef("processComments: %v", c.Token) // Only admins and the proposal author are allowed to retrieve // unvetted comments. This is a public route so a user might not // exist. - if c.State == cmv1.PropStateUnvetted { + if c.State == cmv1.RecordStateUnvetted { var isAllowed bool switch { case usr == nil: @@ -351,13 +172,13 @@ func (p *politeiawww) processComments(ctx context.Context, c cmv1.Comments, usr }, nil } -func (p *politeiawww) processCommentVotes(ctx context.Context, cv cmv1.CommentVotes) (*cmv1.CommentVotesReply, error) { +func (c *Comments) processCommentVotes(ctx context.Context, cv cmv1.CommentVotes) (*cmv1.CommentVotesReply, error) { log.Tracef("processCommentVotes: %v %v", cv.Token, cv.UserID) // Verify state - if cv.State != cmv1.PropStateVetted { + if cv.State != cmv1.RecordStateVetted { return nil, cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodePropStateInvalid, + ErrorCode: cmv1.ErrorCodeRecordStateInvalid, ErrorContext: "proposal must be vetted", } } @@ -376,7 +197,7 @@ func (p *politeiawww) processCommentVotes(ctx context.Context, cv cmv1.CommentVo }, nil } -func (p *politeiawww) processCommentTimestamps(ctx context.Context, t cmv1.Timestamps, isAdmin bool) (*cmv1.TimestampsReply, error) { +func (c *Comments) processCommentTimestamps(ctx context.Context, t cmv1.Timestamps, isAdmin bool) (*cmv1.TimestampsReply, error) { log.Tracef("processCommentTimestamps: %v %v %v", t.State, t.Token, t.CommentIDs) @@ -409,67 +230,7 @@ func (p *politeiawww) processCommentTimestamps(ctx context.Context, t cmv1.Times }, nil } -func (p *politeiawww) handleCommentNew(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleCommentNew") - - var n cmv1.New - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&n); err != nil { - respondWithPiError(w, r, "handleCommentNew: unmarshal", - cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodeInputInvalid, - }) - return - } - - usr, err := p.getSessionUser(w, r) - if err != nil { - respondWithPiError(w, r, - "handleCommentNew: getSessionUser: %v", err) - return - } - - nr, err := p.processCommentNew(r.Context(), n, *usr) - if err != nil { - respondWithPiError(w, r, - "handleCommentNew: processCommentNew: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, nr) -} - -func (p *politeiawww) handleCommentVote(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleCommentVote") - - var v cmv1.Vote - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&v); err != nil { - respondWithPiError(w, r, "handleCommentVote: unmarshal", - cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodeInputInvalid, - }) - return - } - - usr, err := p.getSessionUser(w, r) - if err != nil { - respondWithPiError(w, r, - "handleCommentVote: getSessionUser: %v", err) - return - } - - vr, err := p.processCommentVote(r.Context(), v, *usr) - if err != nil { - respondWithPiError(w, r, - "handleCommentVote: processCommentVote: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, vr) -} - -func (p *politeiawww) handleCommentDel(w http.ResponseWriter, r *http.Request) { +func (c *Comments) handleCommentDel(w http.ResponseWriter, r *http.Request) { log.Tracef("handleCommentDel") var d cmv1.Del @@ -499,7 +260,7 @@ func (p *politeiawww) handleCommentDel(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, dr) } -func (p *politeiawww) handleCommentsCount(w http.ResponseWriter, r *http.Request) { +func (c *Comments) handleCommentsCount(w http.ResponseWriter, r *http.Request) { log.Tracef("handleCommentsCount") var c cmv1.Comments @@ -531,7 +292,7 @@ func (p *politeiawww) handleCommentsCount(w http.ResponseWriter, r *http.Request util.RespondWithJSON(w, http.StatusOK, cr) } -func (p *politeiawww) handleComments(w http.ResponseWriter, r *http.Request) { +func (c *Comments) handleComments(w http.ResponseWriter, r *http.Request) { log.Tracef("handleComments") var c cmv1.Comments @@ -563,7 +324,7 @@ func (p *politeiawww) handleComments(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, cr) } -func (p *politeiawww) handleCommentVotes(w http.ResponseWriter, r *http.Request) { +func (c *Comments) handleCommentVotes(w http.ResponseWriter, r *http.Request) { log.Tracef("handleCommentVotes") var v cmv1.Votes @@ -586,7 +347,7 @@ func (p *politeiawww) handleCommentVotes(w http.ResponseWriter, r *http.Request) util.RespondWithJSON(w, http.StatusOK, vr) } -func (p *politeiawww) handleCommentTimestamps(w http.ResponseWriter, r *http.Request) { +func (c *Comments) handleCommentTimestamps(w http.ResponseWriter, r *http.Request) { log.Tracef("handleCommentTimestamps") var t cmv1.Timestamps @@ -618,79 +379,4 @@ func (p *politeiawww) handleCommentTimestamps(w http.ResponseWriter, r *http.Req util.RespondWithJSON(w, http.StatusOK, tr) } - -func respondWithCommentsError(w http.ResponseWriter, r *http.Request, format string, err error) { - var ( - ue cmv1.UserErrorReply - pe pdError - ) - switch { - case errors.As(err, &ue): - // Comments user error - m := fmt.Sprintf("Comments user error: %v %v %v", - remoteAddr(r), ue.ErrorCode, cmv1.ErrorCodes[ue.ErrorCode]) - if ue.ErrorContext != "" { - m += fmt.Sprintf(": %v", ue.ErrorContext) - } - log.Infof(m) - util.RespondWithJSON(w, http.StatusBadRequest, - cmv1.UserErrorReply{ - ErrorCode: ue.ErrorCode, - ErrorContext: ue.ErrorContext, - }) - return - - case errors.As(err, &pe): - // Politeiad error - var ( - pluginID = pe.ErrorReply.Plugin - errCode = pe.ErrorReply.ErrorCode - errContext = pe.ErrorReply.ErrorContext - ) - switch { - case pluginID != "": - // Politeiad plugin error. Log it and return a 400. - m := fmt.Sprintf("Plugin error: %v %v %v", - remoteAddr(r), pluginID, errCode) - if len(errContext) > 0 { - m += fmt.Sprintf(": %v", strings.Join(errContext, ", ")) - } - log.Infof(m) - util.RespondWithJSON(w, http.StatusBadRequest, - cmv1.PluginErrorReply{ - PluginID: pluginID, - ErrorCode: errCode, - ErrorContext: strings.Join(errContext, ", "), - }) - return - - default: - // Unknown politeiad error. Log it and return a 500. - ts := time.Now().Unix() - log.Errorf("%v %v %v %v Internal error %v: error code "+ - "from politeiad: %v", remoteAddr(r), r.Method, r.URL, - r.Proto, ts, errCode) - - util.RespondWithJSON(w, http.StatusInternalServerError, - cmv1.ServerErrorReply{ - ErrorCode: ts, - }) - return - } - - default: - // Internal server error. Log it and return a 500. - t := time.Now().Unix() - e := fmt.Sprintf(format, err) - log.Errorf("%v %v %v %v Internal error %v: %v", - remoteAddr(r), r.Method, r.URL, r.Proto, t, e) - log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) - - util.RespondWithJSON(w, http.StatusInternalServerError, - cmv1.ServerErrorReply{ - ErrorCode: t, - }) - return - } -} */ diff --git a/politeiawww/comments/comments.go b/politeiawww/comments/comments.go new file mode 100644 index 000000000..70f1f8ed5 --- /dev/null +++ b/politeiawww/comments/comments.go @@ -0,0 +1,30 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package comments + +import ( + pdclient "github.com/decred/politeia/politeiad/client" + "github.com/decred/politeia/politeiawww/config" + "github.com/decred/politeia/politeiawww/sessions" + "github.com/decred/politeia/politeiawww/user" +) + +// Comments is the context for the comments API. +type Comments struct { + cfg *config.Config + politeiad *pdclient.Client + userdb user.Database + sessions sessions.Sessions + // events *events.Manager +} + +// New returns a new Comments context. +func New(cfg *config.Config, politeiad *pdclient.Client, userdb user.Database) *Comments { + return &Comments{ + cfg: cfg, + politeiad: politeiad, + userdb: userdb, + } +} diff --git a/politeiawww/comments/handle.go b/politeiawww/comments/handle.go new file mode 100644 index 000000000..9bf070861 --- /dev/null +++ b/politeiawww/comments/handle.go @@ -0,0 +1,154 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package comments + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "runtime/debug" + "strings" + "time" + + pdclient "github.com/decred/politeia/politeiad/client" + cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" + "github.com/decred/politeia/util" +) + +func (c *Comments) HandleNew(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleNew") + + var n cmv1.New + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&n); err != nil { + respondWithError(w, r, "handleNew: unmarshal", + cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodeInputInvalid, + }) + return + } + + usr, err := c.sessions.GetSessionUser(w, r) + if err != nil { + respondWithError(w, r, + "handleNew: getSessionUser: %v", err) + return + } + + nr, err := c.processNew(r.Context(), n, *usr) + if err != nil { + respondWithError(w, r, + "handleNew: processNew: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, nr) +} + +func (c *Comments) HandleVote(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleVote") + + var v cmv1.Vote + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&v); err != nil { + respondWithError(w, r, "handleVote: unmarshal", + cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodeInputInvalid, + }) + return + } + + usr, err := c.sessions.GetSessionUser(w, r) + if err != nil { + respondWithError(w, r, + "handleVote: getSessionUser: %v", err) + return + } + + vr, err := c.processVote(r.Context(), v, *usr) + if err != nil { + respondWithError(w, r, + "handleVote: processVote: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, vr) +} + +func respondWithError(w http.ResponseWriter, r *http.Request, format string, err error) { + var ( + ue cmv1.UserErrorReply + pe pdclient.Error + ) + switch { + case errors.As(err, &ue): + // Comments user error + m := fmt.Sprintf("Comments user error: %v %v %v", + util.RemoteAddr(r), ue.ErrorCode, cmv1.ErrorCodes[ue.ErrorCode]) + if ue.ErrorContext != "" { + m += fmt.Sprintf(": %v", ue.ErrorContext) + } + log.Infof(m) + util.RespondWithJSON(w, http.StatusBadRequest, + cmv1.UserErrorReply{ + ErrorCode: ue.ErrorCode, + ErrorContext: ue.ErrorContext, + }) + return + + case errors.As(err, &pe): + // Politeiad error + var ( + pluginID = pe.ErrorReply.PluginID + errCode = pe.ErrorReply.ErrorCode + errContext = pe.ErrorReply.ErrorContext + ) + switch { + case pluginID != "": + // Politeiad plugin error. Log it and return a 400. + m := fmt.Sprintf("Plugin error: %v %v %v", + util.RemoteAddr(r), pluginID, errCode) + if len(errContext) > 0 { + m += fmt.Sprintf(": %v", strings.Join(errContext, ", ")) + } + log.Infof(m) + util.RespondWithJSON(w, http.StatusBadRequest, + cmv1.PluginErrorReply{ + PluginID: pluginID, + ErrorCode: errCode, + ErrorContext: strings.Join(errContext, ", "), + }) + return + + default: + // Unknown politeiad error. Log it and return a 500. + ts := time.Now().Unix() + log.Errorf("%v %v %v %v Internal error %v: error code "+ + "from politeiad: %v", util.RemoteAddr(r), r.Method, r.URL, + r.Proto, ts, errCode) + + util.RespondWithJSON(w, http.StatusInternalServerError, + cmv1.ServerErrorReply{ + ErrorCode: ts, + }) + return + } + + default: + // Internal server error. Log it and return a 500. + t := time.Now().Unix() + e := fmt.Sprintf(format, err) + log.Errorf("%v %v %v %v Internal error %v: %v", + util.RemoteAddr(r), r.Method, r.URL, r.Proto, t, e) + log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) + + util.RespondWithJSON(w, http.StatusInternalServerError, + cmv1.ServerErrorReply{ + ErrorCode: t, + }) + return + } +} diff --git a/politeiawww/comments/log.go b/politeiawww/comments/log.go new file mode 100644 index 000000000..f1d18be7c --- /dev/null +++ b/politeiawww/comments/log.go @@ -0,0 +1,25 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package comments + +import "github.com/decred/slog" + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log = slog.Disabled + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = slog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using slog. +func UseLogger(logger slog.Logger) { + log = logger +} diff --git a/politeiawww/comments/process.go b/politeiawww/comments/process.go new file mode 100644 index 000000000..3b029f367 --- /dev/null +++ b/politeiawww/comments/process.go @@ -0,0 +1,197 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package comments + +import ( + "context" + + "github.com/decred/politeia/politeiad/plugins/comments" + cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" + "github.com/decred/politeia/politeiawww/config" + "github.com/decred/politeia/politeiawww/user" +) + +func convertComment(c comments.Comment) cmv1.Comment { + // Fields that are intentionally omitted are not stored in + // politeiad. They need to be pulled from the userdb. + return cmv1.Comment{ + UserID: c.UserID, + Username: "", // Intentionally omitted + Token: c.Token, + ParentID: c.ParentID, + Comment: c.Comment, + PublicKey: c.PublicKey, + Signature: c.Signature, + CommentID: c.CommentID, + Timestamp: c.Timestamp, + Receipt: c.Receipt, + Downvotes: c.Downvotes, + Upvotes: c.Upvotes, + Deleted: c.Deleted, + Reason: c.Reason, + ExtraData: c.ExtraData, + ExtraDataHint: c.ExtraDataHint, + } +} + +func convertVote(v cmv1.VoteT) comments.VoteT { + switch v { + case cmv1.VoteDownvote: + return comments.VoteUpvote + case cmv1.VoteUpvote: + return comments.VoteDownvote + } + return comments.VoteInvalid +} + +// commentPopulateUserData populates the comment with user data that is not +// stored in politeiad. +func commentPopulateUser(c cmv1.Comment, u user.User) cmv1.Comment { + c.Username = u.Username + return c +} + +// This function is a temporary function that will be removed once user plugins +// have been implemented. +func (c *Comments) paywallIsEnabled() bool { + return c.cfg.PaywallAmount != 0 && c.cfg.PaywallXpub != "" +} + +// This function is a temporary function that will be removed once user plugins +// have been implemented. +func (c *Comments) userHasPaid(u user.User) bool { + if !c.paywallIsEnabled() { + return true + } + return u.NewUserPaywallTx != "" +} + +func (c *Comments) processNew(ctx context.Context, n cmv1.New, u user.User) (*cmv1.NewReply, error) { + log.Tracef("processNew: %v %v %v", n.Token, n.ParentID, u.Username) + + // Checking the mode is a temporary measure until user plugins + // have been implemented. + switch c.cfg.Mode { + case config.PoliteiaWWWMode: + // Verify user has paid registration paywall + if !c.userHasPaid(u) { + return nil, cmv1.UserErrorReply{ + // ErrorCode: cmv1.ErrorCodeUserRegistrationNotPaid, + } + } + } + + // Verify user signed using active identity + if u.PublicKey() != n.PublicKey { + return nil, cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodePublicKeyInvalid, + ErrorContext: "not active identity", + } + } + + // Only admins and the record author are allowed to comment on + // unvetted records. + if n.State == cmv1.RecordStateUnvetted && !u.Admin { + // Get the record author + authorID, err := c.politeiad.Author(ctx, n.State, n.Token) + if err != nil { + return nil, err + } + if u.ID.String() != authorID { + return nil, cmv1.UserErrorReply{ + // ErrorCode: cmv1.ErrorCodeUnauthorized, + ErrorContext: "user is not author or admin", + } + } + } + + // Send plugin command + cn := comments.New{ + UserID: u.ID.String(), + Token: n.Token, + ParentID: n.ParentID, + Comment: n.Comment, + PublicKey: n.PublicKey, + Signature: n.Signature, + } + cnr, err := c.politeiad.CommentNew(ctx, n.State, n.Token, cn) + if err != nil { + return nil, err + } + + // Prepare reply + cm := convertComment(cnr.Comment) + cm = commentPopulateUser(cm, u) + + /* + // TODO + // Emit event + c.eventManager.emit(eventProposalComment, + dataProposalComment{ + state: c.State, + token: c.Token, + commentID: c.CommentID, + parentID: c.ParentID, + username: c.Username, + }) + */ + + return &cmv1.NewReply{ + Comment: cm, + }, nil +} + +func (c *Comments) processVote(ctx context.Context, v cmv1.Vote, u user.User) (*cmv1.VoteReply, error) { + log.Tracef("processVote: %v %v %v", v.Token, v.CommentID, v.Vote) + + // Checking the mode is a temporary measure until user plugins + // have been implemented. + switch c.cfg.Mode { + case config.PoliteiaWWWMode: + // Verify user has paid registration paywall + if !c.userHasPaid(u) { + return nil, cmv1.UserErrorReply{ + // ErrorCode: cmv1.ErrorCodeUserRegistrationNotPaid, + } + } + } + + // Verify state + if v.State != cmv1.RecordStateVetted { + return nil, cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodeRecordStateInvalid, + ErrorContext: "record must be vetted", + } + } + + // Verify user signed using active identity + if u.PublicKey() != v.PublicKey { + return nil, cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodePublicKeyInvalid, + ErrorContext: "not active identity", + } + } + + // Send plugin command + cv := comments.Vote{ + UserID: u.ID.String(), + Token: v.Token, + CommentID: v.CommentID, + Vote: convertVote(v.Vote), + PublicKey: v.PublicKey, + Signature: v.Signature, + } + vr, err := c.politeiad.CommentVote(ctx, v.State, v.Token, cv) + if err != nil { + return nil, err + } + + return &cmv1.VoteReply{ + Downvotes: vr.Downvotes, + Upvotes: vr.Upvotes, + Timestamp: vr.Timestamp, + Receipt: vr.Receipt, + }, nil +} diff --git a/politeiawww/config.go b/politeiawww/config.go index 3f635efe9..38ed5572f 100644 --- a/politeiawww/config.go +++ b/politeiawww/config.go @@ -65,11 +65,7 @@ const ( // } dust = 60300 - // Currently available modes to run politeia, by default piwww, is used. - politeiaWWWMode = "piwww" - cmsWWWMode = "cmswww" - - defaultWWWMode = politeiaWWWMode + defaultWWWMode = config.PoliteiaWWWMode // User database options userDBLevel = "leveldb" @@ -409,11 +405,11 @@ func loadConfig() (*config.Config, []string, error) { // Verify mode and set mode specific defaults switch cfg.Mode { - case cmsWWWMode: + case config.CMSWWWMode: if cfg.MailAddress == "" { cfg.MailAddress = defaultMailAddressCMS } - case politeiaWWWMode: + case config.PoliteiaWWWMode: if cfg.MailAddress == "" { cfg.MailAddress = defaultMailAddressPi } diff --git a/politeiawww/config/config.go b/politeiawww/config/config.go index 516dd7001..b2cf2c8df 100644 --- a/politeiawww/config/config.go +++ b/politeiawww/config/config.go @@ -20,6 +20,11 @@ const ( // DefaultDataDirname is the default data directory name. The data // directory is located in the application home directory. DefaultDataDirname = "data" + + // Currently available modes to run politeia, by default piwww, is + // used. + PoliteiaWWWMode = "piwww" + CMSWWWMode = "cmswww" ) var ( diff --git a/politeiawww/log.go b/politeiawww/log.go index 09b1390fe..e44743397 100644 --- a/politeiawww/log.go +++ b/politeiawww/log.go @@ -12,6 +12,7 @@ import ( cmsdb "github.com/decred/politeia/politeiawww/cmsdatabase/cockroachdb" "github.com/decred/politeia/politeiawww/codetracker/github" ghdb "github.com/decred/politeia/politeiawww/codetracker/github/database/cockroachdb" + "github.com/decred/politeia/politeiawww/comments" "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/politeiawww/user/cockroachdb" "github.com/decred/politeia/politeiawww/user/localdb" @@ -54,6 +55,7 @@ var ( githubTrackerLog = backendLog.Logger("GHTR") githubdbLog = backendLog.Logger("GHDB") sessionsLog = backendLog.Logger("SESS") + commentsLog = backendLog.Logger("COMT") ) // Initialize package-global logger variables. @@ -65,6 +67,7 @@ func init() { github.UseLogger(githubTrackerLog) ghdb.UseLogger(githubdbLog) sessions.UseLogger(sessionsLog) + comments.UseLogger(commentsLog) } // subsystemLoggers maps each subsystem identifier to its associated logger. @@ -76,6 +79,7 @@ var subsystemLoggers = map[string]slog.Logger{ "GHTR": githubTrackerLog, "GHDB": githubdbLog, "SESS": sessionsLog, + "COMT": commentsLog, } // initLogRotator initializes the logging rotater to write logs to logFile and diff --git a/politeiawww/middleware.go b/politeiawww/middleware.go index 5c7b206b0..c4388a174 100644 --- a/politeiawww/middleware.go +++ b/politeiawww/middleware.go @@ -15,21 +15,12 @@ import ( "github.com/decred/politeia/util" ) -func remoteAddr(r *http.Request) string { - via := r.RemoteAddr - xff := r.Header.Get(www.Forward) - if xff != "" { - return fmt.Sprintf("%v via %v", xff, r.RemoteAddr) - } - return via -} - // isLoggedIn ensures that a user is logged in before calling the next // function. func (p *politeiawww) isLoggedIn(f http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - log.Debugf("isLoggedIn: %v %v %v %v", remoteAddr(r), r.Method, - r.URL, r.Proto) + log.Debugf("isLoggedIn: %v %v %v %v", + util.RemoteAddr(r), r.Method, r.URL, r.Proto) id, err := p.sessions.GetSessionUserID(w, r) if err != nil { @@ -65,8 +56,8 @@ func (p *politeiawww) isAdmin(w http.ResponseWriter, r *http.Request) (bool, err // before calling the next function. func (p *politeiawww) isLoggedInAsAdmin(f http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - log.Debugf("isLoggedInAsAdmin: %v %v %v %v", remoteAddr(r), - r.Method, r.URL, r.Proto) + log.Debugf("isLoggedInAsAdmin: %v %v %v %v", + util.RemoteAddr(r), r.Method, r.URL, r.Proto) // Check if user is admin isAdmin, err := p.isAdmin(w, r) @@ -112,7 +103,7 @@ func loggingMiddleware(next http.Handler) http.Handler { })) // Log incoming connection - log.Infof("%v %v %v %v", remoteAddr(r), r.Method, r.URL, r.Proto) + log.Infof("%v %v %v %v", util.RemoteAddr(r), r.Method, r.URL, r.Proto) // Call next handler next.ServeHTTP(w, r) @@ -126,8 +117,9 @@ func recoverMiddleware(next http.Handler) http.Handler { defer func() { if err := recover(); err != nil { errorCode := time.Now().Unix() - log.Criticalf("%v %v %v %v Internal error %v: %v", remoteAddr(r), - r.Method, r.URL, r.Proto, errorCode, err) + log.Criticalf("%v %v %v %v Internal error %v: %v", + util.RemoteAddr(r), r.Method, r.URL, r.Proto, errorCode, err) + log.Criticalf("Stacktrace (THIS IS AN ACTUAL PANIC): %s", debug.Stack()) diff --git a/politeiawww/politeiawww.go b/politeiawww/politeiawww.go index 6cd10e5fa..d6d927197 100644 --- a/politeiawww/politeiawww.go +++ b/politeiawww/politeiawww.go @@ -104,8 +104,8 @@ type politeiawww struct { // handleNotFound is a generic handler for an invalid route. func (p *politeiawww) handleNotFound(w http.ResponseWriter, r *http.Request) { // Log incoming connection - log.Debugf("Invalid route: %v %v %v %v", remoteAddr(r), r.Method, r.URL, - r.Proto) + log.Debugf("Invalid route: %v %v %v %v", + util.RemoteAddr(r), r.Method, r.URL, r.Proto) // Trace incoming request log.Tracef("%v", newLogClosure(func() string { diff --git a/politeiawww/records.go b/politeiawww/records.go index ffc20901a..6b8c60d4c 100644 --- a/politeiawww/records.go +++ b/politeiawww/records.go @@ -133,7 +133,7 @@ func respondWithRecordError(w http.ResponseWriter, r *http.Request, format strin case errors.As(err, &ue): // Record user error m := fmt.Sprintf("Records user error: %v %v %v", - remoteAddr(r), ue.ErrorCode, rcv1.ErrorCodes[ue.ErrorCode]) + util.RemoteAddr(r), ue.ErrorCode, rcv1.ErrorCodes[ue.ErrorCode]) if ue.ErrorContext != "" { m += fmt.Sprintf(": %v", ue.ErrorContext) } @@ -157,7 +157,7 @@ func respondWithRecordError(w http.ResponseWriter, r *http.Request, format strin case pluginID != "": // politeiad plugin error. Log it and return a 400. m := fmt.Sprintf("Plugin error: %v %v %v", - remoteAddr(r), pluginID, errCode) + util.RemoteAddr(r), pluginID, errCode) if len(errContext) > 0 { m += fmt.Sprintf(": %v", strings.Join(errContext, ", ")) } @@ -175,7 +175,7 @@ func respondWithRecordError(w http.ResponseWriter, r *http.Request, format strin // and return a 500. ts := time.Now().Unix() log.Errorf("%v %v %v %v Internal error %v: error code "+ - "from politeiad: %v", remoteAddr(r), r.Method, r.URL, + "from politeiad: %v", util.RemoteAddr(r), r.Method, r.URL, r.Proto, ts, errCode) util.RespondWithJSON(w, http.StatusInternalServerError, @@ -188,7 +188,7 @@ func respondWithRecordError(w http.ResponseWriter, r *http.Request, format strin // politeiad error does correspond to a user error. Log it and // return a 400. m := fmt.Sprintf("Records user error: %v %v %v", - remoteAddr(r), e, rcv1.ErrorCodes[e]) + util.RemoteAddr(r), e, rcv1.ErrorCodes[e]) if len(errContext) > 0 { m += fmt.Sprintf(": %v", strings.Join(errContext, ", ")) } @@ -206,7 +206,7 @@ func respondWithRecordError(w http.ResponseWriter, r *http.Request, format strin t := time.Now().Unix() e := fmt.Sprintf(format, err) log.Errorf("%v %v %v %v Internal error %v: %v", - remoteAddr(r), r.Method, r.URL, r.Proto, t, e) + util.RemoteAddr(r), r.Method, r.URL, r.Proto, t, e) log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) util.RespondWithJSON(w, http.StatusInternalServerError, diff --git a/politeiawww/testing.go b/politeiawww/testing.go index 838e27524..33d305efe 100644 --- a/politeiawww/testing.go +++ b/politeiawww/testing.go @@ -414,7 +414,7 @@ func newTestCMSwww(t *testing.T) (*politeiawww, func()) { TestNet: true, VoteDurationMin: 2016, VoteDurationMax: 4032, - Mode: cmsWWWMode, + Mode: config.CMSWWWMode, } // Setup database diff --git a/politeiawww/ticketvote.go b/politeiawww/ticketvote.go index cba1655b1..a3ca438b3 100644 --- a/politeiawww/ticketvote.go +++ b/politeiawww/ticketvote.go @@ -72,7 +72,7 @@ func (p *politeiawww) voteSummaries(ctx context.Context, tokens []string) (map[s func (p *politeiawww) voteTimestamps(ctx context.Context, token string) (*ticketvote.TimestampsReply, error) { _ = token var b []byte - r, err := p.politeiad.PluginCommand(ctx, ticketvote.ID, + r, err := p.politeiad.PluginCommand(ctx, ticketvote.PluginID, ticketvote.CmdTimestamps, string(b)) if err != nil { return nil, err @@ -147,7 +147,7 @@ func respondWithTicketVoteError(w http.ResponseWriter, r *http.Request, format s case errors.As(err, &ue): // Ticket vote user error m := fmt.Sprintf("Ticket vote user error: %v %v %v", - remoteAddr(r), ue.ErrorCode, tkv1.ErrorCodes[ue.ErrorCode]) + util.RemoteAddr(r), ue.ErrorCode, tkv1.ErrorCodes[ue.ErrorCode]) if ue.ErrorContext != "" { m += fmt.Sprintf(": %v", ue.ErrorContext) } @@ -170,7 +170,7 @@ func respondWithTicketVoteError(w http.ResponseWriter, r *http.Request, format s case pluginID != "": // Politeiad plugin error. Log it and return a 400. m := fmt.Sprintf("Plugin error: %v %v %v", - remoteAddr(r), pluginID, errCode) + util.RemoteAddr(r), pluginID, errCode) if len(errContext) > 0 { m += fmt.Sprintf(": %v", strings.Join(errContext, ", ")) } @@ -187,7 +187,7 @@ func respondWithTicketVoteError(w http.ResponseWriter, r *http.Request, format s // Unknown politeiad error. Log it and return a 500. ts := time.Now().Unix() log.Errorf("%v %v %v %v Internal error %v: error code "+ - "from politeiad: %v", remoteAddr(r), r.Method, r.URL, + "from politeiad: %v", util.RemoteAddr(r), r.Method, r.URL, r.Proto, ts, errCode) util.RespondWithJSON(w, http.StatusInternalServerError, @@ -202,7 +202,8 @@ func respondWithTicketVoteError(w http.ResponseWriter, r *http.Request, format s t := time.Now().Unix() e := fmt.Sprintf(format, err) log.Errorf("%v %v %v %v Internal error %v: %v", - remoteAddr(r), r.Method, r.URL, r.Proto, t, e) + util.RemoteAddr(r), r.Method, r.URL, r.Proto, t, e) + log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) util.RespondWithJSON(w, http.StatusInternalServerError, diff --git a/politeiawww/totp.go b/politeiawww/totp.go index 544000005..0c2062e63 100644 --- a/politeiawww/totp.go +++ b/politeiawww/totp.go @@ -10,6 +10,7 @@ import ( "time" www "github.com/decred/politeia/politeiawww/api/www/v1" + "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/user" "github.com/pquerna/otp" "github.com/pquerna/otp/totp" @@ -59,7 +60,7 @@ func (p *politeiawww) processSetTOTP(st www.SetTOTP, u *user.User) (*www.SetTOTP } issuer := defaultPoliteiaIssuer - if p.cfg.Mode == cmsWWWMode { + if p.cfg.Mode == config.CMSWWWMode { issuer = defaultCMSIssuer } opts := p.totpGenerateOpts(issuer, u.Username) diff --git a/politeiawww/www.go b/politeiawww/www.go index 07e3db675..86bbebbf1 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -36,6 +36,7 @@ import ( database "github.com/decred/politeia/politeiawww/cmsdatabase" cmsdb "github.com/decred/politeia/politeiawww/cmsdatabase/cockroachdb" ghtracker "github.com/decred/politeia/politeiawww/codetracker/github" + "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/politeiawww/user" "github.com/decred/politeia/politeiawww/user/cockroachdb" @@ -132,15 +133,15 @@ func convertWWWErrorStatus(pluginID string, errCode int) www.ErrorStatusT { // politeiad API e := pd.ErrorStatusT(errCode) return convertWWWErrorStatusFromPD(e) - case piplugin.ID: + case piplugin.PluginID: // Pi plugin e := piplugin.ErrorCodeT(errCode) return convertWWWErrorStatusFromPiPlugin(e) - case comments.ID: + case comments.PluginID: // Comments plugin e := comments.ErrorCodeT(errCode) return convertWWWErrorStatusFromComments(e) - case ticketvote.ID: + case ticketvote.PluginID: // Ticket vote plugin e := ticketvote.ErrorCodeT(errCode) return convertWWWErrorStatusFromTicketVote(e) @@ -237,11 +238,11 @@ func convertPiErrorStatus(pluginID string, errCode int) pi.ErrorStatusT { // politeiad API e := pd.ErrorStatusT(errCode) return convertPiErrorStatusFromPD(e) - case piplugin.ID: + case piplugin.PluginID: // Pi plugin e := piplugin.ErrorCodeT(errCode) return convertPiErrorStatusFromPiPlugin(e) - case ticketvote.ID: + case ticketvote.PluginID: // Ticket vote plugin e := ticketvote.ErrorCodeT(errCode) return convertPiErrorStatusFromTicketVote(e) @@ -302,11 +303,11 @@ func respondWithPiError(w http.ResponseWriter, r *http.Request, format string, e // Error is a pi user error. Log it and return a 400. if len(ue.ErrorContext) == 0 { log.Infof("Pi user error: %v %v %v", - remoteAddr(r), int64(ue.ErrorCode), + util.RemoteAddr(r), int64(ue.ErrorCode), pi.ErrorStatus[ue.ErrorCode]) } else { log.Errorf("Pi user error: %v %v %v: %v", - remoteAddr(r), int64(ue.ErrorCode), + util.RemoteAddr(r), int64(ue.ErrorCode), pi.ErrorStatus[ue.ErrorCode], ue.ErrorContext) } @@ -335,11 +336,11 @@ func respondWithPiError(w http.ResponseWriter, r *http.Request, format string, e t := time.Now().Unix() if pluginID == "" { log.Errorf("%v %v %v %v Internal error %v: error "+ - "code from politeiad: %v", remoteAddr(r), r.Method, + "code from politeiad: %v", util.RemoteAddr(r), r.Method, r.URL, r.Proto, t, errCode) } else { log.Errorf("%v %v %v %v Internal error %v: error "+ - "code from politeiad plugin %v: %v %v", remoteAddr(r), + "code from politeiad plugin %v: %v %v", util.RemoteAddr(r), r.Method, r.URL, r.Proto, t, pluginID, errCode, errContext) } @@ -354,11 +355,11 @@ func respondWithPiError(w http.ResponseWriter, r *http.Request, format string, e // return a 400. if len(errContext) == 0 { log.Infof("Pi user error: %v %v %v", - remoteAddr(r), int64(piErrCode), + util.RemoteAddr(r), int64(piErrCode), pi.ErrorStatus[piErrCode]) } else { log.Infof("Pi user error: %v %v %v: %v", - remoteAddr(r), int64(piErrCode), + util.RemoteAddr(r), int64(piErrCode), pi.ErrorStatus[piErrCode], strings.Join(errContext, ", ")) } @@ -376,7 +377,7 @@ func respondWithPiError(w http.ResponseWriter, r *http.Request, format string, e t := time.Now().Unix() e := fmt.Sprintf(format, err) log.Errorf("%v %v %v %v Internal error %v: %v", - remoteAddr(r), r.Method, r.URL, r.Proto, t, e) + util.RemoteAddr(r), r.Method, r.URL, r.Proto, t, e) log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) util.RespondWithJSON(w, http.StatusInternalServerError, @@ -421,11 +422,11 @@ func RespondWithError(w http.ResponseWriter, r *http.Request, userHttpCode int, if len(userErr.ErrorContext) == 0 { log.Infof("WWW user error: %v %v %v", - remoteAddr(r), int64(userErr.ErrorCode), + util.RemoteAddr(r), int64(userErr.ErrorCode), userErrorStatus(userErr.ErrorCode)) } else { log.Infof("WWW user error: %v %v %v: %v", - remoteAddr(r), int64(userErr.ErrorCode), + util.RemoteAddr(r), int64(userErr.ErrorCode), userErrorStatus(userErr.ErrorCode), strings.Join(userErr.ErrorContext, ", ")) } @@ -454,11 +455,11 @@ func RespondWithError(w http.ResponseWriter, r *http.Request, userHttpCode int, t := time.Now().Unix() if pluginID == "" { log.Errorf("%v %v %v %v Internal error %v: error "+ - "code from politeiad: %v", remoteAddr(r), r.Method, + "code from politeiad: %v", util.RemoteAddr(r), r.Method, r.URL, r.Proto, t, errCode) } else { log.Errorf("%v %v %v %v Internal error %v: error "+ - "code from politeiad plugin %v: %v", remoteAddr(r), + "code from politeiad plugin %v: %v", util.RemoteAddr(r), r.Method, r.URL, r.Proto, t, pluginID, errCode) } @@ -473,11 +474,11 @@ func RespondWithError(w http.ResponseWriter, r *http.Request, userHttpCode int, // and return a 400. if len(errContext) == 0 { log.Infof("WWW user error: %v %v %v", - remoteAddr(r), int64(wwwErrCode), + util.RemoteAddr(r), int64(wwwErrCode), userErrorStatus(wwwErrCode)) } else { log.Infof("WWW user error: %v %v %v: %v", - remoteAddr(r), int64(wwwErrCode), + util.RemoteAddr(r), int64(wwwErrCode), userErrorStatus(wwwErrCode), strings.Join(errContext, ", ")) } @@ -492,7 +493,7 @@ func RespondWithError(w http.ResponseWriter, r *http.Request, userHttpCode int, // Error is a politeiawww server error. Log it and return a 500. t := time.Now().Unix() - ec := fmt.Sprintf("%v %v %v %v Internal error %v: ", remoteAddr(r), + ec := fmt.Sprintf("%v %v %v %v Internal error %v: ", util.RemoteAddr(r), r.Method, r.URL, r.Proto, t) log.Errorf(ec+format, args...) log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) @@ -957,12 +958,12 @@ func _main() error { // Perform application specific setup switch p.cfg.Mode { - case politeiaWWWMode: + case config.PoliteiaWWWMode: err = p.setupPi() if err != nil { return fmt.Errorf("setupPi: %v", err) } - case cmsWWWMode: + case config.CMSWWWMode: err = p.setupCMS() if err != nil { return fmt.Errorf("setupCMS: %v", err) @@ -1030,9 +1031,9 @@ done: // Perform application specific shutdown switch p.cfg.Mode { - case politeiaWWWMode: + case config.PoliteiaWWWMode: // Nothing to do - case cmsWWWMode: + case config.CMSWWWMode: p.wsDcrdata.Close() } diff --git a/util/json.go b/util/json.go index b36f5bdd4..637248c32 100644 --- a/util/json.go +++ b/util/json.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Decred developers +// Copyright (c) 2017-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -6,9 +6,11 @@ package util import ( "encoding/json" + "fmt" "io" "net/http" + pdv1 "github.com/decred/politeia/politeiad/api/v1" "github.com/gorilla/schema" ) @@ -65,3 +67,14 @@ func ParseGetParams(r *http.Request, dst interface{}) error { return schema.NewDecoder().Decode(dst, r.Form) } + +// RemoteAddr returns a string of the remote address, i.e. the address that +// sent the request. +func RemoteAddr(r *http.Request) string { + via := r.RemoteAddr + xff := r.Header.Get(pdv1.Forward) + if xff != "" { + return fmt.Sprintf("%v via %v", xff, r.RemoteAddr) + } + return via +} From 715abe69f851f5aad90c069c4604bb491c736a85 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 20 Jan 2021 20:45:09 -0600 Subject: [PATCH 243/449] Fix build issues. --- go.mod | 2 +- .../tlogbe/plugins/comments/comments_test.go | 14 +++++++------- politeiad/backend/tlogbe/tlogbe_test.go | 2 ++ politeiad/politeiad.go | 16 ++++++++-------- .../cmd/politeiawww_dbutil/politeiawww_dbutil.go | 6 +++--- 5 files changed, 21 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index 774b4ec84..1ee89f815 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/otiai10/curr v0.0.0-20190513014714-f5a3d24e5776 // indirect github.com/pmezard/go-difflib v1.0.0 github.com/pquerna/otp v1.2.0 - github.com/prometheus/common v0.10.0 + github.com/prometheus/common v0.10.0 // indirect github.com/robfig/cron v1.2.0 github.com/subosito/gozaru v0.0.0-20190625071150-416082cce636 github.com/syndtr/goleveldb v1.0.0 diff --git a/politeiad/backend/tlogbe/plugins/comments/comments_test.go b/politeiad/backend/tlogbe/plugins/comments/comments_test.go index 7ed067b6c..8661399b7 100644 --- a/politeiad/backend/tlogbe/plugins/comments/comments_test.go +++ b/politeiad/backend/tlogbe/plugins/comments/comments_test.go @@ -15,7 +15,7 @@ import ( "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/clients" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/politeiad/plugins/comments" "github.com/google/uuid" ) @@ -49,7 +49,7 @@ func newTestCommentsPlugin(t *testing.T) (*commentsPlugin, func()) { // TODO Implement a test clients.TlogClient // Setup tlog client - var tlog clients.TlogClient + var tlog plugins.TlogClient // Setup plugin identity fid, err := identity.New() @@ -58,7 +58,7 @@ func newTestCommentsPlugin(t *testing.T) (*commentsPlugin, func()) { } // Setup comment plugins - c, err := New(tlog, []backend.PluginSetting{}, fid, dataDir) + c, err := New(tlog, []backend.PluginSetting{}, dataDir, fid) if err != nil { t.Fatal(err) } @@ -113,8 +113,8 @@ func TestCmdNew(t *testing.T) { PublicKey: publicKey, Signature: commentSignature(t, fid, token, parentIDZero, comment), }, - backend.PluginUserError{ - PluginID: comments.ID, + backend.PluginError{ + PluginID: comments.PluginID, ErrorCode: int(comments.ErrorCodeTokenInvalid), }, "", @@ -136,8 +136,8 @@ func TestCmdNew(t *testing.T) { if tc.wantErr != nil { // We expect an error. Verify that the returned error is // correct. - want := tc.wantErr.(backend.PluginUserError) - var ue backend.PluginUserError + want := tc.wantErr.(backend.PluginError) + var ue backend.PluginError switch { case errors.As(err, &ue) && want.PluginID == ue.PluginID && diff --git a/politeiad/backend/tlogbe/tlogbe_test.go b/politeiad/backend/tlogbe/tlogbe_test.go index 8b2669fd2..3ca1beed5 100644 --- a/politeiad/backend/tlogbe/tlogbe_test.go +++ b/politeiad/backend/tlogbe/tlogbe_test.go @@ -390,6 +390,7 @@ func TestNewRecord(t *testing.T) { } } +/* func TestUpdateUnvettedRecord(t *testing.T) { tlogBackend, cleanup := NewTestTlogBackend(t) defer cleanup() @@ -1658,3 +1659,4 @@ func TestSetVettedStatus(t *testing.T) { }) } } +*/ diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 4e67425ff..e45bd6d3b 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -1582,25 +1582,25 @@ func _main() error { // Setup plugin var plugin backend.Plugin switch v { - case comments.ID: + case comments.PluginID: plugin = backend.Plugin{ - ID: comments.ID, + ID: comments.PluginID, Settings: ps, Identity: p.identity, } - case dcrdata.ID: + case dcrdata.PluginID: plugin = backend.Plugin{ - ID: dcrdata.ID, + ID: dcrdata.PluginID, Settings: ps, } - case pi.ID: + case pi.PluginID: plugin = backend.Plugin{ - ID: pi.ID, + ID: pi.PluginID, Settings: ps, } - case ticketvote.ID: + case ticketvote.PluginID: plugin = backend.Plugin{ - ID: ticketvote.ID, + ID: ticketvote.PluginID, Settings: ps, Identity: p.identity, } diff --git a/politeiawww/cmd/politeiawww_dbutil/politeiawww_dbutil.go b/politeiawww/cmd/politeiawww_dbutil/politeiawww_dbutil.go index 94d879b9c..a0ba9c141 100644 --- a/politeiawww/cmd/politeiawww_dbutil/politeiawww_dbutil.go +++ b/politeiawww/cmd/politeiawww_dbutil/politeiawww_dbutil.go @@ -28,7 +28,7 @@ import ( "github.com/decred/dcrd/chaincfg/v3" "github.com/decred/politeia/decredplugin" "github.com/decred/politeia/politeiad/backend/gitbe" - "github.com/decred/politeia/politeiawww/sharedconfig" + "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/user" "github.com/decred/politeia/politeiawww/user/cockroachdb" "github.com/decred/politeia/politeiawww/user/localdb" @@ -54,8 +54,8 @@ const ( ) var ( - defaultHomeDir = sharedconfig.DefaultHomeDir - defaultDataDir = sharedconfig.DefaultDataDir + defaultHomeDir = config.DefaultHomeDir + defaultDataDir = config.DefaultDataDir defaultEncryptionKey = filepath.Join(defaultHomeDir, "sbox.key") // Database options From 3bc93cf8c06f3a70236a1cd31bf486d1447007f2 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 20 Jan 2021 21:45:55 -0600 Subject: [PATCH 244/449] Package event manager. --- politeiawww/comments/comments.go | 3 +- politeiawww/comments/events.go | 20 +++ .../comments/{handle.go => handlers.go} | 0 politeiawww/comments/process.go | 131 +++++++++--------- politeiawww/dcc.go | 4 +- politeiawww/{eventmanager.go => events.go} | 91 +++--------- politeiawww/events/events.go | 53 +++++++ politeiawww/invoices.go | 4 +- politeiawww/piwww.go | 6 +- politeiawww/politeiawww.go | 23 +-- politeiawww/www.go | 25 ++-- 11 files changed, 191 insertions(+), 169 deletions(-) create mode 100644 politeiawww/comments/events.go rename politeiawww/comments/{handle.go => handlers.go} (100%) rename politeiawww/{eventmanager.go => events.go} (89%) create mode 100644 politeiawww/events/events.go diff --git a/politeiawww/comments/comments.go b/politeiawww/comments/comments.go index 70f1f8ed5..042da62fe 100644 --- a/politeiawww/comments/comments.go +++ b/politeiawww/comments/comments.go @@ -7,6 +7,7 @@ package comments import ( pdclient "github.com/decred/politeia/politeiad/client" "github.com/decred/politeia/politeiawww/config" + "github.com/decred/politeia/politeiawww/events" "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/politeiawww/user" ) @@ -17,7 +18,7 @@ type Comments struct { politeiad *pdclient.Client userdb user.Database sessions sessions.Sessions - // events *events.Manager + events *events.Manager } // New returns a new Comments context. diff --git a/politeiawww/comments/events.go b/politeiawww/comments/events.go new file mode 100644 index 000000000..c01065372 --- /dev/null +++ b/politeiawww/comments/events.go @@ -0,0 +1,20 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package comments + +const ( + // EventNew is the event that is emitted when a new comment is + // made. + EventNew = "commentnew" +) + +// EventDataNew is the event data that is emitted when a new comment is made. +type EventDataNew struct { + State string + Token string + CommentID uint32 + ParentID uint32 + Username string +} diff --git a/politeiawww/comments/handle.go b/politeiawww/comments/handlers.go similarity index 100% rename from politeiawww/comments/handle.go rename to politeiawww/comments/handlers.go diff --git a/politeiawww/comments/process.go b/politeiawww/comments/process.go index 3b029f367..d5274f5c9 100644 --- a/politeiawww/comments/process.go +++ b/politeiawww/comments/process.go @@ -13,61 +13,6 @@ import ( "github.com/decred/politeia/politeiawww/user" ) -func convertComment(c comments.Comment) cmv1.Comment { - // Fields that are intentionally omitted are not stored in - // politeiad. They need to be pulled from the userdb. - return cmv1.Comment{ - UserID: c.UserID, - Username: "", // Intentionally omitted - Token: c.Token, - ParentID: c.ParentID, - Comment: c.Comment, - PublicKey: c.PublicKey, - Signature: c.Signature, - CommentID: c.CommentID, - Timestamp: c.Timestamp, - Receipt: c.Receipt, - Downvotes: c.Downvotes, - Upvotes: c.Upvotes, - Deleted: c.Deleted, - Reason: c.Reason, - ExtraData: c.ExtraData, - ExtraDataHint: c.ExtraDataHint, - } -} - -func convertVote(v cmv1.VoteT) comments.VoteT { - switch v { - case cmv1.VoteDownvote: - return comments.VoteUpvote - case cmv1.VoteUpvote: - return comments.VoteDownvote - } - return comments.VoteInvalid -} - -// commentPopulateUserData populates the comment with user data that is not -// stored in politeiad. -func commentPopulateUser(c cmv1.Comment, u user.User) cmv1.Comment { - c.Username = u.Username - return c -} - -// This function is a temporary function that will be removed once user plugins -// have been implemented. -func (c *Comments) paywallIsEnabled() bool { - return c.cfg.PaywallAmount != 0 && c.cfg.PaywallXpub != "" -} - -// This function is a temporary function that will be removed once user plugins -// have been implemented. -func (c *Comments) userHasPaid(u user.User) bool { - if !c.paywallIsEnabled() { - return true - } - return u.NewUserPaywallTx != "" -} - func (c *Comments) processNew(ctx context.Context, n cmv1.New, u user.User) (*cmv1.NewReply, error) { log.Tracef("processNew: %v %v %v", n.Token, n.ParentID, u.Username) @@ -125,18 +70,15 @@ func (c *Comments) processNew(ctx context.Context, n cmv1.New, u user.User) (*cm cm := convertComment(cnr.Comment) cm = commentPopulateUser(cm, u) - /* - // TODO - // Emit event - c.eventManager.emit(eventProposalComment, - dataProposalComment{ - state: c.State, - token: c.Token, - commentID: c.CommentID, - parentID: c.ParentID, - username: c.Username, - }) - */ + // Emit event + c.events.Emit(EventNew, + EventDataNew{ + State: n.State, + Token: cm.Token, + CommentID: cm.CommentID, + ParentID: cm.ParentID, + Username: cm.Username, + }) return &cmv1.NewReply{ Comment: cm, @@ -195,3 +137,58 @@ func (c *Comments) processVote(ctx context.Context, v cmv1.Vote, u user.User) (* Receipt: vr.Receipt, }, nil } + +func convertComment(c comments.Comment) cmv1.Comment { + // Fields that are intentionally omitted are not stored in + // politeiad. They need to be pulled from the userdb. + return cmv1.Comment{ + UserID: c.UserID, + Username: "", // Intentionally omitted + Token: c.Token, + ParentID: c.ParentID, + Comment: c.Comment, + PublicKey: c.PublicKey, + Signature: c.Signature, + CommentID: c.CommentID, + Timestamp: c.Timestamp, + Receipt: c.Receipt, + Downvotes: c.Downvotes, + Upvotes: c.Upvotes, + Deleted: c.Deleted, + Reason: c.Reason, + ExtraData: c.ExtraData, + ExtraDataHint: c.ExtraDataHint, + } +} + +func convertVote(v cmv1.VoteT) comments.VoteT { + switch v { + case cmv1.VoteDownvote: + return comments.VoteUpvote + case cmv1.VoteUpvote: + return comments.VoteDownvote + } + return comments.VoteInvalid +} + +// commentPopulateUserData populates the comment with user data that is not +// stored in politeiad. +func commentPopulateUser(c cmv1.Comment, u user.User) cmv1.Comment { + c.Username = u.Username + return c +} + +// This function is a temporary function that will be removed once user plugins +// have been implemented. +func (c *Comments) paywallIsEnabled() bool { + return c.cfg.PaywallAmount != 0 && c.cfg.PaywallXpub != "" +} + +// This function is a temporary function that will be removed once user plugins +// have been implemented. +func (c *Comments) userHasPaid(u user.User) bool { + if !c.paywallIsEnabled() { + return true + } + return u.NewUserPaywallTx != "" +} diff --git a/politeiawww/dcc.go b/politeiawww/dcc.go index 0e82b8c23..5f343edcf 100644 --- a/politeiawww/dcc.go +++ b/politeiawww/dcc.go @@ -577,7 +577,7 @@ func (p *politeiawww) processNewDCC(ctx context.Context, nd cms.NewDCC, u *user. } // Emit event notification for new DCC being submitted - p.eventManager.emit(eventDCCNew, + p.events.Emit(eventDCCNew, dataDCCNew{ token: pdReply.CensorshipRecord.Token, }) @@ -1016,7 +1016,7 @@ func (p *politeiawww) processSupportOpposeDCC(ctx context.Context, sd cms.Suppor } // Emit event notification for a DCC being supported/opposed - p.eventManager.emit(eventDCCSupportOppose, + p.events.Emit(eventDCCSupportOppose, dataDCCSupportOppose{ token: sd.Token, }) diff --git a/politeiawww/eventmanager.go b/politeiawww/events.go similarity index 89% rename from politeiawww/eventmanager.go rename to politeiawww/events.go index fad3cd05d..3122eb40e 100644 --- a/politeiawww/eventmanager.go +++ b/politeiawww/events.go @@ -8,7 +8,6 @@ import ( "context" "fmt" "strconv" - "sync" "github.com/decred/politeia/politeiad/plugins/comments" pi "github.com/decred/politeia/politeiawww/api/pi/v1" @@ -17,72 +16,22 @@ import ( "github.com/google/uuid" ) -type eventT int - const ( - // Event types - eventTypeInvalid eventT = iota - // Pi events - eventProposalSubmitted - eventProposalEdited - eventProposalStatusChange - eventProposalComment - eventProposalVoteAuthorized - eventProposalVoteStarted + eventProposalSubmitted = "eventProposalSubmitted" + eventProposalEdited = "eventProposalEdited" + eventProposalStatusChange = "eventProposalStatusChange" + eventProposalComment = "eventProposalComment" + eventProposalVoteAuthorized = "eventProposalVoteAuthorized" + eventProposalVoteStarted = "eventProposalVoteStarted" // CMS events - eventInvoiceComment - eventInvoiceStatusUpdate - eventDCCNew - eventDCCSupportOppose + eventInvoiceComment = "eventInvoiceComment" + eventInvoiceStatusUpdate = "eventInvoiceStatusUpdate" + eventDCCNew = "eventDCCNew" + eventDCCSupportOppose = "eventDCCSupportOppose" ) -// eventManager manages event listeners for different event types. -type eventManager struct { - sync.Mutex - listeners map[eventT][]chan interface{} -} - -// register registers an event listener (channel) to listen for the provided -// event type. -func (e *eventManager) register(event eventT, listener chan interface{}) { - e.Lock() - defer e.Unlock() - - l, ok := e.listeners[event] - if !ok { - l = make([]chan interface{}, 0) - } - - l = append(l, listener) - e.listeners[event] = l -} - -// emit emits an event by passing it to all channels that have been registered -// to listen for the event. -func (e *eventManager) emit(event eventT, data interface{}) { - e.Lock() - defer e.Unlock() - - listeners, ok := e.listeners[event] - if !ok { - log.Errorf("fire: unregistered event %v", event) - return - } - - for _, ch := range listeners { - ch <- data - } -} - -// newEventManager returns a new eventManager context. -func newEventManager() *eventManager { - return &eventManager{ - listeners: make(map[eventT][]chan interface{}), - } -} - func (p *politeiawww) setupEventListenersPi() { // Setup process for each event: // 1. Create a channel for the event @@ -91,54 +40,54 @@ func (p *politeiawww) setupEventListenersPi() { // Setup proposal submitted event ch := make(chan interface{}) - p.eventManager.register(eventProposalSubmitted, ch) + p.events.Register(eventProposalSubmitted, ch) go p.handleEventProposalSubmitted(ch) // Setup proposal edit event ch = make(chan interface{}) - p.eventManager.register(eventProposalEdited, ch) + p.events.Register(eventProposalEdited, ch) go p.handleEventProposalEdited(ch) // Setup proposal status change event ch = make(chan interface{}) - p.eventManager.register(eventProposalStatusChange, ch) + p.events.Register(eventProposalStatusChange, ch) go p.handleEventProposalStatusChange(ch) // Setup proposal comment event ch = make(chan interface{}) - p.eventManager.register(eventProposalComment, ch) + p.events.Register(eventProposalComment, ch) go p.handleEventProposalComment(ch) // Setup proposal vote authorized event ch = make(chan interface{}) - p.eventManager.register(eventProposalVoteAuthorized, ch) + p.events.Register(eventProposalVoteAuthorized, ch) go p.handleEventProposalVoteAuthorized(ch) // Setup proposal vote started event ch = make(chan interface{}) - p.eventManager.register(eventProposalVoteStarted, ch) + p.events.Register(eventProposalVoteStarted, ch) go p.handleEventProposalVoteStarted(ch) } func (p *politeiawww) setupEventListenersCMS() { // Setup invoice comment event ch := make(chan interface{}) - p.eventManager.register(eventInvoiceComment, ch) + p.events.Register(eventInvoiceComment, ch) go p.handleEventInvoiceComment(ch) // Setup invoice status update event ch = make(chan interface{}) - p.eventManager.register(eventInvoiceStatusUpdate, ch) + p.events.Register(eventInvoiceStatusUpdate, ch) go p.handleEventInvoiceStatusUpdate(ch) // Setup DCC new update event ch = make(chan interface{}) - p.eventManager.register(eventDCCNew, ch) + p.events.Register(eventDCCNew, ch) go p.handleEventDCCNew(ch) // Setup DCC support/oppose event ch = make(chan interface{}) - p.eventManager.register(eventDCCSupportOppose, ch) + p.events.Register(eventDCCSupportOppose, ch) go p.handleEventDCCSupportOppose(ch) } diff --git a/politeiawww/events/events.go b/politeiawww/events/events.go new file mode 100644 index 000000000..e15b7b9e1 --- /dev/null +++ b/politeiawww/events/events.go @@ -0,0 +1,53 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package events + +import ( + "sync" +) + +// Manager manages event listeners for different event types. +type Manager struct { + sync.Mutex + listeners map[string][]chan interface{} +} + +// Register registers an event listener (channel) to listen for the provided +// event type. +func (e *Manager) Register(event string, listener chan interface{}) { + e.Lock() + defer e.Unlock() + + l, ok := e.listeners[event] + if !ok { + l = make([]chan interface{}, 0) + } + + l = append(l, listener) + e.listeners[event] = l +} + +// Emit emits an event by passing it to all channels that have been registered +// to listen for the event. +func (e *Manager) Emit(event string, data interface{}) { + e.Lock() + defer e.Unlock() + + listeners, ok := e.listeners[event] + if !ok { + return + } + + for _, ch := range listeners { + ch <- data + } +} + +// NewManager returns a new Manager context. +func NewManager() *Manager { + return &Manager{ + listeners: make(map[string][]chan interface{}), + } +} diff --git a/politeiawww/invoices.go b/politeiawww/invoices.go index 87b66453d..4381c803e 100644 --- a/politeiawww/invoices.go +++ b/politeiawww/invoices.go @@ -1337,7 +1337,7 @@ func (p *politeiawww) processSetInvoiceStatus(ctx context.Context, sis cms.SetIn } // Emit event notification for invoice status update - p.eventManager.emit(eventInvoiceStatusUpdate, + p.events.Emit(eventInvoiceStatusUpdate, dataInvoiceStatusUpdate{ token: dbInvoice.Token, email: invoiceUser.Email, @@ -2025,7 +2025,7 @@ func (p *politeiawww) processNewCommentInvoice(ctx context.Context, nc www.NewCo ir.Username, err) } // Emit event notification for a invoice comment - p.eventManager.emit(eventInvoiceComment, + p.events.Emit(eventInvoiceComment, dataInvoiceComment{ token: nc.Token, email: invoiceUser.Email, diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 2c2b6b8e9..71c838beb 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -1042,7 +1042,7 @@ func (p *politeiawww) processProposalNew(ctx context.Context, pn piv1.ProposalNe } // Emit a new proposal event - p.eventManager.emit(eventProposalSubmitted, + p.events.Emit(eventProposalSubmitted, dataProposalSubmitted{ token: cr.Token, name: pm.Name, @@ -1221,7 +1221,7 @@ func (p *politeiawww) processProposalEdit(ctx context.Context, pe piv1.ProposalE } // Emit an edit proposal event - p.eventManager.emit(eventProposalEdited, dataProposalEdited{ + p.events.Emit(eventProposalEdited, dataProposalEdited{ userID: usr.ID.String(), username: usr.Username, token: pe.Token, @@ -1359,7 +1359,7 @@ func (p *politeiawww) processProposalSetStatus(ctx context.Context, pss piv1.Pro } // Emit status change event - p.eventManager.emit(eventProposalStatusChange, + p.events.Emit(eventProposalStatusChange, dataProposalStatusChange{ token: pss.Token, state: state, diff --git a/politeiawww/politeiawww.go b/politeiawww/politeiawww.go index d6d927197..146479139 100644 --- a/politeiawww/politeiawww.go +++ b/politeiawww/politeiawww.go @@ -22,6 +22,7 @@ import ( "github.com/decred/politeia/politeiawww/cmsdatabase" "github.com/decred/politeia/politeiawww/codetracker" "github.com/decred/politeia/politeiawww/config" + "github.com/decred/politeia/politeiawww/events" "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/politeiawww/user" utilwww "github.com/decred/politeia/politeiawww/util" @@ -64,17 +65,17 @@ func (w *wsContext) isAuthenticated() bool { // politeiawww represents the politeiawww server. type politeiawww struct { sync.RWMutex - cfg *config.Config - params *chaincfg.Params - router *mux.Router - auth *mux.Router // CSRF protected subrouter - politeiad *pdclient.Client - client *http.Client - smtp *smtp - db user.Database - sessions *sessions.Sessions - eventManager *eventManager - plugins []plugin + cfg *config.Config + params *chaincfg.Params + router *mux.Router + auth *mux.Router // CSRF protected subrouter + politeiad *pdclient.Client + client *http.Client + smtp *smtp + db user.Database + sessions *sessions.Sessions + events *events.Manager + plugins []plugin // Client websocket connections ws map[string]map[string]*wsContext // [uuid][]*context diff --git a/politeiawww/www.go b/politeiawww/www.go index 86bbebbf1..1653ad2e7 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -37,6 +37,7 @@ import ( cmsdb "github.com/decred/politeia/politeiawww/cmsdatabase/cockroachdb" ghtracker "github.com/decred/politeia/politeiawww/codetracker/github" "github.com/decred/politeia/politeiawww/config" + "github.com/decred/politeia/politeiawww/events" "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/politeiawww/user" "github.com/decred/politeia/politeiawww/user/cockroachdb" @@ -930,18 +931,18 @@ func _main() error { // Setup application context p := &politeiawww{ - cfg: loadedCfg, - params: activeNetParams.Params, - router: router, - auth: auth, - politeiad: pdc, - client: client, - smtp: smtp, - db: userDB, - sessions: sessions.New(userDB, cookieKey), - eventManager: newEventManager(), - ws: make(map[string]map[string]*wsContext), - userEmails: make(map[string]uuid.UUID), + cfg: loadedCfg, + params: activeNetParams.Params, + router: router, + auth: auth, + politeiad: pdc, + client: client, + smtp: smtp, + db: userDB, + sessions: sessions.New(userDB, cookieKey), + events: events.NewManager(), + ws: make(map[string]map[string]*wsContext), + userEmails: make(map[string]uuid.UUID), } // Setup politeiad plugins From 707b9b0494def19e44d4b4fedd470d2ab5f0f39d Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 21 Jan 2021 13:36:40 -0600 Subject: [PATCH 245/449] Add support for legacy routes dcrdata uses. --- politeiad/plugins/ticketvote/ticketvote.go | 4 - politeiawww/api/www/v1/v1.go | 11 +- politeiawww/comments/handlers.go | 4 +- politeiawww/piwww.go | 46 ++++- politeiawww/politeiawww.go | 180 ------------------- politeiawww/proposals.go | 200 +++++++++++++++++++-- politeiawww/testing.go | 1 - politeiawww/www.go | 1 - 8 files changed, 239 insertions(+), 208 deletions(-) diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index 3606651a9..66f15fa8f 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -202,10 +202,6 @@ type VoteDetails struct { } // CastVoteDetails is the structure that is saved to disk when a vote is cast. -// -// TODO VoteOption.Bit is a uint64, but the CastVote.VoteBit is a string in -// decredplugin. Do we want to make them consistent or was that done on -// purpose? It was probably done that way so that way for the signature. type CastVoteDetails struct { // Data generated by client Token string `json:"token"` // Record token diff --git a/politeiawww/api/www/v1/v1.go b/politeiawww/api/www/v1/v1.go index 6c8665834..6302108f4 100644 --- a/politeiawww/api/www/v1/v1.go +++ b/politeiawww/api/www/v1/v1.go @@ -54,18 +54,17 @@ const ( RouteUnauthenticatedWebSocket = "/ws" RouteAuthenticatedWebSocket = "/aws" - // The following routes have been DEPRECATED and support will be - // removed in the near future. + // The following routes have been DEPRECATED. RouteTokenInventory = "/proposals/tokeninventory" RouteProposalDetails = "/proposals/{token:[A-Fa-f0-9]{7,64}}" + RouteAllVetted = "/proposals/vetted" RouteBatchProposals = "/proposals/batch" + RouteActiveVote = "/proposals/activevote" RouteVoteResults = "/proposals/{token:[A-Fa-f0-9]{7,64}}/votes" RouteCastVotes = "/proposals/castvotes" RouteBatchVoteSummary = "/proposals/batchvotesummary" // The following routes are NO LONGER SUPPORTED. - RouteActiveVote = "/proposals/activevote" - RouteAllVetted = "/proposals/vetted" RouteNewProposal = "/proposals/new" RouteEditProposal = "/proposals/edit" RouteAuthorizeVote = "/proposals/authorizevote" @@ -956,7 +955,7 @@ type SetProposalStatusReply struct { // If Before is specified, the "page" returned starts before the provided // proposal censorship token, when sorted in reverse chronological order. // -// This request is NO LONGER SUPPORTED. +// This request is DEPRECATED. type GetAllVetted struct { Before string `schema:"before"` After string `schema:"after"` @@ -964,7 +963,7 @@ type GetAllVetted struct { // GetAllVettedReply is used to reply with a list of vetted proposals. // -// This request is NO LONGER SUPPORTED. +// This request is DEPRECATED. type GetAllVettedReply struct { Proposals []ProposalRecord `json:"proposals"` } diff --git a/politeiawww/comments/handlers.go b/politeiawww/comments/handlers.go index 9bf070861..567c00353 100644 --- a/politeiawww/comments/handlers.go +++ b/politeiawww/comments/handlers.go @@ -34,7 +34,7 @@ func (c *Comments) HandleNew(w http.ResponseWriter, r *http.Request) { usr, err := c.sessions.GetSessionUser(w, r) if err != nil { respondWithError(w, r, - "handleNew: getSessionUser: %v", err) + "handleNew: GetSessionUser: %v", err) return } @@ -64,7 +64,7 @@ func (c *Comments) HandleVote(w http.ResponseWriter, r *http.Request) { usr, err := c.sessions.GetSessionUser(w, r) if err != nil { respondWithError(w, r, - "handleVote: getSessionUser: %v", err) + "handleVote: GetSessionUser: %v", err) return } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 71c838beb..4bc64a47b 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -1566,7 +1566,7 @@ func (p *politeiawww) processVotes(ctx context.Context, v piv1.Votes) (*piv1.Vot return nil, nil } -func (p *politeiawww) processVoteResults(ctx context.Context, vr piv1.VoteResults) (*piv1.VoteResultsReply, error) { +func (p *politeiawww) processVoteResultsPi(ctx context.Context, vr piv1.VoteResults) (*piv1.VoteResultsReply, error) { log.Tracef("processVoteResults: %v", vr.Token) cvr, err := p.voteResults(ctx, vr.Token) @@ -1876,7 +1876,7 @@ func (p *politeiawww) handleVotes(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, vr) } -func (p *politeiawww) handleVoteResults(w http.ResponseWriter, r *http.Request) { +func (p *politeiawww) handleVoteResultsPi(w http.ResponseWriter, r *http.Request) { log.Tracef("handleVoteResults") var vr piv1.VoteResults @@ -1889,7 +1889,7 @@ func (p *politeiawww) handleVoteResults(w http.ResponseWriter, r *http.Request) return } - vrr, err := p.processVoteResults(r.Context(), vr) + vrr, err := p.processVoteResultsPi(r.Context(), vr) if err != nil { respondWithPiError(w, r, "handleVoteResults: prcoessVoteResults: %v", err) @@ -1947,6 +1947,46 @@ func (p *politeiawww) handleVoteInventory(w http.ResponseWriter, r *http.Request // setPiRoutes sets the pi API routes. func (p *politeiawww) setPiRoutes() { + // Return a 404 when a route is not found + p.router.NotFoundHandler = http.HandlerFunc(p.handleNotFound) + + // The version routes set the CSRF token and thus need to be part + // of the CSRF protected auth router. + p.auth.HandleFunc("/", p.handleVersion).Methods(http.MethodGet) + p.auth.StrictSlash(true). + HandleFunc(www.PoliteiaWWWAPIRoute+www.RouteVersion, p.handleVersion). + Methods(http.MethodGet) + + // www routes. These routes have been deprecated and support will + // be removed in the future. + p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, + www.RoutePolicy, p.handlePolicy, + permissionPublic) + p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, + www.RouteTokenInventory, p.handleTokenInventory, + permissionPublic) + p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, + www.RouteAllVetted, p.handleAllVetted, + permissionPublic) + p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, + www.RouteProposalDetails, p.handleProposalDetails, + permissionPublic) + p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, + www.RouteBatchProposals, p.handleBatchProposals, + permissionPublic) + p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, + www.RouteActiveVote, p.handleActiveVote, + permissionPublic) + p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, + www.RouteCastVotes, p.handleCastVotes, + permissionPublic) + p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, + www.RouteVoteResults, p.handleVoteResults, + permissionPublic) + p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, + www.RouteBatchVoteSummary, p.handleBatchVoteSummary, + permissionPublic) + // Pi routes - proposals p.addRoute(http.MethodPost, piv1.APIRoute, piv1.RouteProposalNew, p.handleProposalNew, diff --git a/politeiawww/politeiawww.go b/politeiawww/politeiawww.go index 146479139..2ac3849c4 100644 --- a/politeiawww/politeiawww.go +++ b/politeiawww/politeiawww.go @@ -192,141 +192,6 @@ func (p *politeiawww) handlePolicy(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, reply) } -// handleTokenInventory returns the tokens of all proposals in the inventory. -func (p *politeiawww) handleTokenInventory(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleTokenInventory") - - // Get session user. This is a public route so one might not exist. - user, err := p.sessions.GetSessionUser(w, r) - if err != nil && !errors.Is(err, sessions.ErrSessionNotFound) { - RespondWithError(w, r, 0, - "handleTokenInventory: getSessionUser %v", err) - return - } - - isAdmin := user != nil && user.Admin - reply, err := p.processTokenInventory(r.Context(), isAdmin) - if err != nil { - RespondWithError(w, r, 0, - "handleTokenInventory: processTokenInventory: %v", err) - return - } - util.RespondWithJSON(w, http.StatusOK, reply) -} - -// handleProposalDetails handles the incoming proposal details command. It -// fetches the complete details for an existing proposal. -func (p *politeiawww) handleProposalDetails(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleProposalDetails") - - // Get version from query string parameters - var pd www.ProposalsDetails - err := util.ParseGetParams(r, &pd) - if err != nil { - RespondWithError(w, r, 0, "handleProposalDetails: ParseGetParams", - www.UserError{ - ErrorCode: www.ErrorStatusInvalidInput, - }) - return - } - - // Get proposal token from path parameters - pathParams := mux.Vars(r) - pd.Token = pathParams["token"] - - // Get session user. This is a public route so one might not exist. - user, err := p.sessions.GetSessionUser(w, r) - if err != nil && err != sessions.ErrSessionNotFound { - RespondWithError(w, r, 0, - "handleProposalDetails: getSessionUser %v", err) - return - } - - reply, err := p.processProposalDetails(r.Context(), pd, user) - if err != nil { - RespondWithError(w, r, 0, - "handleProposalDetails: processProposalDetails %v", err) - return - } - - // Reply with the proposal details. - util.RespondWithJSON(w, http.StatusOK, reply) -} - -// handleBatchProposals handles the incoming batch proposals command. It -// returns a ProposalRecord for each of the provided censorship tokens. -func (p *politeiawww) handleBatchProposals(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleBatchProposals") - - var bp www.BatchProposals - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&bp); err != nil { - RespondWithError(w, r, 0, "handleBatchProposals: unmarshal", - www.UserError{ - ErrorCode: www.ErrorStatusInvalidInput, - }) - return - } - - // Get session user. This is a public route so one might not exist. - user, err := p.sessions.GetSessionUser(w, r) - if err != nil && err != sessions.ErrSessionNotFound { - RespondWithError(w, r, 0, - "handleBatchProposals: getSessionUser %v", err) - return - } - - reply, err := p.processBatchProposals(r.Context(), bp, user) - if err != nil { - RespondWithError(w, r, 0, - "handleBatchProposals: processBatchProposals %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, reply) -} - -// handleCastVotes casts dcr ticket votes for a proposal vote. -func (p *politeiawww) handleCastVotes(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleCastVotes") - - var cv www.Ballot - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&cv); err != nil { - RespondWithError(w, r, 0, "handleCastVotes: unmarshal", www.UserError{ - ErrorCode: www.ErrorStatusInvalidInput, - }) - return - } - - avr, err := p.processCastVotes(r.Context(), &cv) - if err != nil { - RespondWithError(w, r, 0, - "handleCastVotes: processCastVotes %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, avr) -} - -// handleVoteResultsWWW returns a proposal + all voting action. -func (p *politeiawww) handleVoteResultsWWW(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleVoteResultsWWW") - - pathParams := mux.Vars(r) - token := pathParams["token"] - - vrr, err := p.processVoteResultsWWW(r.Context(), token) - if err != nil { - RespondWithError(w, r, 0, - "handleVoteResultsWWW: processVoteResultsWWW %v", - err) - return - } - - util.RespondWithJSON(w, http.StatusOK, vrr) -} - // handleBatchVoteSummary handles the incoming batch vote summary command. It // returns a VoteSummary for each of the provided censorship tokens. func (p *politeiawww) handleBatchVoteSummary(w http.ResponseWriter, r *http.Request) { @@ -593,48 +458,3 @@ func (p *politeiawww) handleAuthenticatedWebsocket(w http.ResponseWriter, r *htt p.handleWebsocket(w, r, id) } - -// setPoliteiaWWWRoutes sets up the politeia routes. -func (p *politeiawww) setPoliteiaWWWRoutes() { - // Return a 404 when a route is not found - p.router.NotFoundHandler = http.HandlerFunc(p.handleNotFound) - - // The version routes set the CSRF token and thus need to be part - // of the CSRF protected auth router. - p.auth.HandleFunc("/", p.handleVersion).Methods(http.MethodGet) - p.auth.StrictSlash(true). - HandleFunc(www.PoliteiaWWWAPIRoute+www.RouteVersion, p.handleVersion). - Methods(http.MethodGet) - - // Public routes. - p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RoutePolicy, p.handlePolicy, - permissionPublic) - p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteTokenInventory, p.handleTokenInventory, - permissionPublic) - p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteProposalDetails, p.handleProposalDetails, - permissionPublic) - p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, - www.RouteBatchProposals, p.handleBatchProposals, - permissionPublic) - p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteCastVotes, p.handleCastVotes, - permissionPublic) - p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, - www.RouteVoteResults, p.handleVoteResultsWWW, - permissionPublic) - p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, - www.RouteBatchVoteSummary, p.handleBatchVoteSummary, - permissionPublic) - - // Unauthenticated websocket - p.addRoute("", www.PoliteiaWWWAPIRoute, - www.RouteUnauthenticatedWebSocket, p.handleUnauthenticatedWebsocket, - permissionPublic) - // Authenticated websocket - p.addRoute("", www.PoliteiaWWWAPIRoute, - www.RouteAuthenticatedWebSocket, p.handleAuthenticatedWebsocket, - permissionLogin) -} diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 9b9969bec..983fc06ee 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -8,6 +8,8 @@ import ( "context" "encoding/base64" "encoding/json" + "errors" + "net/http" "strconv" "github.com/decred/politeia/decredplugin" @@ -16,7 +18,10 @@ import ( ticketvote "github.com/decred/politeia/politeiad/plugins/ticketvote" pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" + "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/politeiawww/user" + "github.com/decred/politeia/util" + "github.com/gorilla/mux" ) func convertStateToWWW(state pi.PropStateT) www.PropStateT { @@ -197,6 +202,11 @@ func (p *politeiawww) processProposalDetails(ctx context.Context, pd www.Proposa }, nil } +func (p *politeiawww) processAllVetted(ctx context.Context, gav www.GetAllVetted) (*www.GetAllVettedReply, error) { + // TODO + return nil, nil +} + func (p *politeiawww) processBatchProposals(ctx context.Context, bp www.BatchProposals, u *user.User) (*www.BatchProposalsReply, error) { log.Tracef("processBatchProposals: %v", bp.Tokens) @@ -229,8 +239,8 @@ func (p *politeiawww) processBatchProposals(ctx context.Context, bp www.BatchPro }, nil } -func (p *politeiawww) processVoteResultsWWW(ctx context.Context, token string) (*www.VoteResultsReply, error) { - log.Tracef("processVoteResultsWWW: %v", token) +func (p *politeiawww) processVoteResults(ctx context.Context, token string) (*www.VoteResultsReply, error) { + log.Tracef("processVoteResults: %v", token) // Get vote details vd, err := p.voteDetails(ctx, token) @@ -293,15 +303,13 @@ func (p *politeiawww) processVoteResultsWWW(ctx context.Context, token string) ( func (p *politeiawww) processBatchVoteSummary(ctx context.Context, bvs www.BatchVoteSummary) (*www.BatchVoteSummaryReply, error) { log.Tracef("processBatchVoteSummary: %v", bvs.Tokens) - // Get vote summaries - vs, err := p.voteSummaries(ctx, bvs.Tokens) - if err != nil { - return nil, err - } + // TODO + var bestBlock uint32 + var vs []ticketvote.VoteSummary // Prepare reply summaries := make(map[string]www.VoteSummary, len(vs)) - for k, v := range vs { + for _, v := range vs { results := make([]www.VoteOptionResult, 0, len(v.Results)) for _, r := range v.Results { results = append(results, www.VoteOptionResult{ @@ -313,7 +321,9 @@ func (p *politeiawww) processBatchVoteSummary(ctx context.Context, bvs www.Batch }, }) } - summaries[k] = www.VoteSummary{ + // TODO + var token string + summaries[token] = www.VoteSummary{ Status: convertVoteStatusToWWW(v.Status), Type: convertVoteTypeToWWW(v.Type), Approved: v.Approved, @@ -328,11 +338,15 @@ func (p *politeiawww) processBatchVoteSummary(ctx context.Context, bvs www.Batch return &www.BatchVoteSummaryReply{ Summaries: summaries, - // TODO - // BestBlock: uint64(sm.BestBlock), + BestBlock: uint64(bestBlock), }, nil } +func (p *politeiawww) processActiveVote(ctx context.Context) (*www.ActiveVoteReply, error) { + // TODO + return nil, nil +} + func (p *politeiawww) processCastVotes(ctx context.Context, ballot *www.Ballot) (*www.BallotReply, error) { log.Tracef("processCastVotes") @@ -410,3 +424,167 @@ func (p *politeiawww) processTokenInventory(ctx context.Context, isAdmin bool) ( Abandoned: archived, }, nil } + +func (p *politeiawww) handleTokenInventory(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleTokenInventory") + + // Get session user. This is a public route so one might not exist. + user, err := p.sessions.GetSessionUser(w, r) + if err != nil && !errors.Is(err, sessions.ErrSessionNotFound) { + RespondWithError(w, r, 0, + "handleTokenInventory: getSessionUser %v", err) + return + } + + isAdmin := user != nil && user.Admin + reply, err := p.processTokenInventory(r.Context(), isAdmin) + if err != nil { + RespondWithError(w, r, 0, + "handleTokenInventory: processTokenInventory: %v", err) + return + } + util.RespondWithJSON(w, http.StatusOK, reply) +} + +func (p *politeiawww) handleAllVetted(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleAllVetted") + + var v www.GetAllVetted + err := util.ParseGetParams(r, &v) + if err != nil { + RespondWithError(w, r, 0, "handleAllVetted: ParseGetParams", + www.UserError{ + ErrorCode: www.ErrorStatusInvalidInput, + }) + return + } + + vr, err := p.processAllVetted(r.Context(), v) + if err != nil { + RespondWithError(w, r, 0, + "handleAllVetted: processAllVetted %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, vr) +} + +func (p *politeiawww) handleProposalDetails(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleProposalDetails") + + // Get version from query string parameters + var pd www.ProposalsDetails + err := util.ParseGetParams(r, &pd) + if err != nil { + RespondWithError(w, r, 0, "handleProposalDetails: ParseGetParams", + www.UserError{ + ErrorCode: www.ErrorStatusInvalidInput, + }) + return + } + + // Get proposal token from path parameters + pathParams := mux.Vars(r) + pd.Token = pathParams["token"] + + // Get session user. This is a public route so one might not exist. + user, err := p.sessions.GetSessionUser(w, r) + if err != nil && err != sessions.ErrSessionNotFound { + RespondWithError(w, r, 0, + "handleProposalDetails: getSessionUser %v", err) + return + } + + reply, err := p.processProposalDetails(r.Context(), pd, user) + if err != nil { + RespondWithError(w, r, 0, + "handleProposalDetails: processProposalDetails %v", err) + return + } + + // Reply with the proposal details. + util.RespondWithJSON(w, http.StatusOK, reply) +} + +func (p *politeiawww) handleBatchProposals(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleBatchProposals") + + var bp www.BatchProposals + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&bp); err != nil { + RespondWithError(w, r, 0, "handleBatchProposals: unmarshal", + www.UserError{ + ErrorCode: www.ErrorStatusInvalidInput, + }) + return + } + + // Get session user. This is a public route so one might not exist. + user, err := p.sessions.GetSessionUser(w, r) + if err != nil && err != sessions.ErrSessionNotFound { + RespondWithError(w, r, 0, + "handleBatchProposals: getSessionUser %v", err) + return + } + + reply, err := p.processBatchProposals(r.Context(), bp, user) + if err != nil { + RespondWithError(w, r, 0, + "handleBatchProposals: processBatchProposals %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, reply) +} + +func (p *politeiawww) handleActiveVote(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleActiveVote") + + avr, err := p.processActiveVote(r.Context()) + if err != nil { + RespondWithError(w, r, 0, + "handleActiveVote: processActiveVote %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, avr) +} + +func (p *politeiawww) handleCastVotes(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleCastVotes") + + var cv www.Ballot + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&cv); err != nil { + RespondWithError(w, r, 0, "handleCastVotes: unmarshal", www.UserError{ + ErrorCode: www.ErrorStatusInvalidInput, + }) + return + } + + avr, err := p.processCastVotes(r.Context(), &cv) + if err != nil { + RespondWithError(w, r, 0, + "handleCastVotes: processCastVotes %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, avr) +} + +func (p *politeiawww) handleVoteResults(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleVoteResults") + + pathParams := mux.Vars(r) + token := pathParams["token"] + + vrr, err := p.processVoteResults(r.Context(), token) + if err != nil { + RespondWithError(w, r, 0, + "handleVoteResults: processVoteResults %v", + err) + return + } + + util.RespondWithJSON(w, http.StatusOK, vrr) +} diff --git a/politeiawww/testing.go b/politeiawww/testing.go index 33d305efe..bbd9f031b 100644 --- a/politeiawww/testing.go +++ b/politeiawww/testing.go @@ -360,7 +360,6 @@ func newTestPoliteiawww(t *testing.T) (*politeiawww, func()) { } // Setup routes - p.setPoliteiaWWWRoutes() p.setUserWWWRoutes() p.setPiRoutes() diff --git a/politeiawww/www.go b/politeiawww/www.go index 1653ad2e7..5f7567546 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -537,7 +537,6 @@ func (p *politeiawww) addRoute(method string, routeVersion string, route string, func (p *politeiawww) setupPi() error { // Setup routes - p.setPoliteiaWWWRoutes() p.setUserWWWRoutes() p.setPiRoutes() From 162609f357b0d0bcb65aae4e8cc870ee171dec9a Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 21 Jan 2021 21:28:00 -0600 Subject: [PATCH 246/449] Add routes to comments api. --- politeiad/client/comments.go | 174 ++++++++++- politeiawww/api/comments/v1/v1.go | 5 +- politeiawww/comments.go | 271 +----------------- politeiawww/comments/comments.go | 181 +++++++++++- .../comments/{handlers.go => error.go} | 61 ---- politeiawww/comments/events.go | 8 +- politeiawww/comments/process.go | 157 +++++++++- 7 files changed, 504 insertions(+), 353 deletions(-) rename politeiawww/comments/{handlers.go => error.go} (64%) diff --git a/politeiad/client/comments.go b/politeiad/client/comments.go index 78ed611d6..f8877b52f 100644 --- a/politeiad/client/comments.go +++ b/politeiad/client/comments.go @@ -13,6 +13,7 @@ import ( "github.com/decred/politeia/politeiad/plugins/comments" ) +// CommentNew sends the comments plugin New command to the politeiad v1 API. func (c *Client) CommentNew(ctx context.Context, state, token string, n comments.New) (*comments.NewReply, error) { // Setup request b, err := json.Marshal(n) @@ -34,8 +35,6 @@ func (c *Client) CommentNew(ctx context.Context, state, token string, n comments if err != nil { return nil, err } - - // Decode reply if len(replies) == 0 { return nil, fmt.Errorf("no replies found") } @@ -43,6 +42,8 @@ func (c *Client) CommentNew(ctx context.Context, state, token string, n comments if pcr.Error != nil { return nil, pcr.Error } + + // Decode reply var nr comments.NewReply err = json.Unmarshal([]byte(pcr.Payload), &nr) if err != nil { @@ -52,6 +53,7 @@ func (c *Client) CommentNew(ctx context.Context, state, token string, n comments return &nr, nil } +// CommentVote sends the comments plugin Vote command to the politeiad v1 API. func (c *Client) CommentVote(ctx context.Context, state, token string, v comments.Vote) (*comments.VoteReply, error) { // Setup request b, err := json.Marshal(v) @@ -73,8 +75,6 @@ func (c *Client) CommentVote(ctx context.Context, state, token string, v comment if err != nil { return nil, err } - - // Decode reply if len(replies) == 0 { return nil, fmt.Errorf("no replies found") } @@ -82,6 +82,8 @@ func (c *Client) CommentVote(ctx context.Context, state, token string, v comment if pcr.Error != nil { return nil, pcr.Error } + + // Decode reply var nr comments.VoteReply err = json.Unmarshal([]byte(pcr.Payload), &nr) if err != nil { @@ -90,3 +92,167 @@ func (c *Client) CommentVote(ctx context.Context, state, token string, v comment return &nr, nil } + +// CommentDel sends the comments plugin Del command to the politeiad v1 API. +func (c *Client) CommentDel(ctx context.Context, state, token string, d comments.Del) (*comments.DelReply, error) { + // Setup request + b, err := json.Marshal(d) + if err != nil { + return nil, err + } + cmds := []pdv1.PluginCommandV2{ + { + State: state, + Token: token, + ID: comments.PluginID, + Command: comments.CmdDel, + Payload: string(b), + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + + // Decode reply + pcr := replies[0] + if pcr.Error != nil { + return nil, pcr.Error + } + var dr comments.DelReply + err = json.Unmarshal([]byte(pcr.Payload), &dr) + if err != nil { + return nil, err + } + + return &dr, nil +} + +// CommentCounts sends a batch of comment plugin Count commands to the +// politeiad v1 API and returns a map[token]count with the results. If a record +// is not found for a token or any other error occurs, that token will not be +// included in the reply. +func (c *Client) CommentCounts(ctx context.Context, state string, tokens []string) (map[string]uint32, error) { + // Setup request + cmds := make([]pdv1.PluginCommandV2, 0, len(tokens)) + for _, v := range tokens { + cmds = append(cmds, pdv1.PluginCommandV2{ + State: state, + Token: v, + ID: comments.PluginID, + Command: comments.CmdCount, + }) + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == len(cmds) { + return nil, fmt.Errorf("replies missing") + } + + // Decode replies + counts := make(map[string]uint32, len(replies)) + for _, v := range replies { + // This command swallows individual errors. The token of the + // command that errored will not be included in the reply. + if v.Error != nil { + continue + } + var cr comments.CountReply + err = json.Unmarshal([]byte(v.Payload), cr) + if err != nil { + continue + } + counts[v.Token] = cr.Count + } + + return counts, nil +} + +// CommentGetAll sends the comments plugin GetAll command to the politeiad v1 +// API. +func (c *Client) CommentGetAll(ctx context.Context, state, token string) ([]comments.Comment, error) { + // Setup request + cmds := []pdv1.PluginCommandV2{ + { + State: state, + Token: token, + ID: comments.PluginID, + Command: comments.CmdGetAll, + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + if pcr.Error != nil { + return nil, pcr.Error + } + + // Decode reply + var gar comments.GetAllReply + err = json.Unmarshal([]byte(pcr.Payload), &gar) + if err != nil { + return nil, err + } + + return gar.Comments, nil +} + +// CommentVotes sends the comments plugin Votes command to the politeiad v1 +// API. +func (c *Client) CommentVotes(ctx context.Context, state, token, userID string) ([]comments.CommentVote, error) { + // Setup request + cm := comments.Votes{ + UserID: userID, + } + b, err := json.Marshal(cm) + if err != nil { + return nil, err + } + cmds := []pdv1.PluginCommandV2{ + { + State: state, + Token: token, + ID: comments.PluginID, + Command: comments.CmdVotes, + Payload: string(b), + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + if pcr.Error != nil { + return nil, pcr.Error + } + + // Decode reply + var vr comments.VotesReply + err = json.Unmarshal([]byte(pcr.Payload), &vr) + if err != nil { + return nil, err + } + + return vr.Votes, nil +} diff --git a/politeiawww/api/comments/v1/v1.go b/politeiawww/api/comments/v1/v1.go index 8c44df207..816b95b55 100644 --- a/politeiawww/api/comments/v1/v1.go +++ b/politeiawww/api/comments/v1/v1.go @@ -208,7 +208,7 @@ type DelReply struct { // Count requests the number of comments on that have been made on the given // records. If a record is not found for a token then it will not be included -// in the reply. +// in the returned map. type Count struct { State string `json:"state"` Tokens []string `json:"tokens"` @@ -216,7 +216,7 @@ type Count struct { // CountReply is the reply to the count command. type CountReply struct { - Count map[string]uint32 `json:"count"` + Counts map[string]uint32 `json:"counts"` } // Comments requests a record's comments. @@ -233,6 +233,7 @@ type CommentsReply struct { // Votes returns the comment votes that meet the provided filtering criteria. type Votes struct { State string `json:"state"` + Token string `json:"token"` UserID string `json:"userid"` } diff --git a/politeiawww/comments.go b/politeiawww/comments.go index edc7fc9b7..9af361c8d 100644 --- a/politeiawww/comments.go +++ b/politeiawww/comments.go @@ -15,23 +15,6 @@ func convertCommentVoteFromPlugin(v comments.VoteT) piv1.CommentVoteT { return piv1.CommentVoteInvalid } -func convertCommentVoteDetailsFromPlugin(cv []comments.CommentVote) []piv1.CommentVoteDetails { - c := make([]piv1.CommentVoteDetails, 0, len(cv)) - for _, v := range cv { - c = append(c, piv1.CommentVoteDetails{ - UserID: v.UserID, - Token: v.Token, - CommentID: v.CommentID, - Vote: convertCommentVoteFromPlugin(v.Vote), - PublicKey: v.PublicKey, - Signature: v.Signature, - Timestamp: v.Timestamp, - Receipt: v.Receipt, - }) - } - return c -} - func convertProofFromCommentsPlugin(p comments.Proof) cmv1.Proof { return cmv1.Proof{ Type: p.Type, @@ -63,140 +46,6 @@ func commentPopulateUser(c piv1.Comment, u user.User) cmv1.Comment { return c } -func (c *Comments) processCommentCensor(ctx context.Context, cc cmv1.CommentCensor, usr user.User) (*cmv1.CommentCensorReply, error) { - log.Tracef("processCommentCensor: %v %v", cc.Token, cc.CommentID) - - // Sanity check - if !usr.Admin { - return nil, fmt.Errorf("not an admin") - } - - // Verify user signed with their active identity - if usr.PublicKey() != cc.PublicKey { - return nil, cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodePublicKeyInvalid, - ErrorContext: "not active identity", - } - } - - // Send plugin command - d := comments.Del{ - Token: cc.Token, - CommentID: cc.CommentID, - Reason: cc.Reason, - PublicKey: cc.PublicKey, - Signature: cc.Signature, - } - // TODO - _ = d - var dr comments.DelReply - - // Prepare reply - c := convertCommentFromPlugin(dr.Comment) - c = commentPopulateUser(c, usr) - - return &cmv1.CommentCensorReply{ - Comment: c, - }, nil -} - -func (c *Comments) processComments(ctx context.Context, c cmv1.Comments, usr *user.User) (*cmv1.CommentsReply, error) { - log.Tracef("processComments: %v", c.Token) - - // Only admins and the proposal author are allowed to retrieve - // unvetted comments. This is a public route so a user might not - // exist. - if c.State == cmv1.RecordStateUnvetted { - var isAllowed bool - switch { - case usr == nil: - // No logged in user. Unvetted not allowed. - case usr.Admin: - // User is an admin. Unvetted is allowed. - isAllowed = true - default: - // Logged in user is not an admin. Check if they are the - // proposal author. - pr, err := p.proposalRecordLatest(ctx, c.State, c.Token) - if err != nil { - if errors.Is(err, errProposalNotFound) { - return nil, cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodePropNotFound, - } - } - return nil, fmt.Errorf("proposalRecordLatest: %v", err) - } - if usr.ID.String() == pr.UserID { - // User is the proposal author. Unvetted is allowed. - isAllowed = true - } - } - if !isAllowed { - return nil, cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodeUnauthorized, - ErrorContext: "user is not author or admin", - } - } - } - - // Send plugin command - reply, err := p.commentsAll(ctx, comments.GetAll{}) - if err != nil { - return nil, err - } - - // Prepare reply. Comments contain user data that needs to be - // pulled from the user database. - cs := make([]cmv1.Comment, 0, len(reply.Comments)) - for _, cm := range reply.Comments { - // Convert comment - pic := convertCommentFromPlugin(cm) - - // Get comment user data - uuid, err := uuid.Parse(cm.UserID) - if err != nil { - return nil, err - } - u, err := p.db.UserGetById(uuid) - if err != nil { - return nil, err - } - pic.Username = u.Username - - // Add comment - cs = append(cs, pic) - } - - return &cmv1.CommentsReply{ - Comments: cs, - }, nil -} - -func (c *Comments) processCommentVotes(ctx context.Context, cv cmv1.CommentVotes) (*cmv1.CommentVotesReply, error) { - log.Tracef("processCommentVotes: %v %v", cv.Token, cv.UserID) - - // Verify state - if cv.State != cmv1.RecordStateVetted { - return nil, cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodeRecordStateInvalid, - ErrorContext: "proposal must be vetted", - } - } - - // Send plugin command - v := comments.Votes{ - UserID: cv.UserID, - } - cvr, err := p.commentVotes(ctx, v) - if err != nil { - return nil, err - } - - return &cmv1.CommentVotesReply{ - Votes: convertCommentVoteDetailsFromPlugin(cvr.Votes), - }, nil -} - func (c *Comments) processCommentTimestamps(ctx context.Context, t cmv1.Timestamps, isAdmin bool) (*cmv1.TimestampsReply, error) { log.Tracef("processCommentTimestamps: %v %v %v", t.State, t.Token, t.CommentIDs) @@ -230,122 +79,6 @@ func (c *Comments) processCommentTimestamps(ctx context.Context, t cmv1.Timestam }, nil } -func (c *Comments) handleCommentDel(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleCommentDel") - - var d cmv1.Del - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&d); err != nil { - respondWithPiError(w, r, "handleCommentDel: unmarshal", - cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodeInputInvalid, - }) - return - } - - usr, err := p.getSessionUser(w, r) - if err != nil { - respondWithPiError(w, r, - "handleCommentDel: getSessionUser: %v", err) - return - } - - dr, err := p.processCommentDel(r.Context(), d, *usr) - if err != nil { - respondWithPiError(w, r, - "handleCommentDel: processCommentDel: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, dr) -} - -func (c *Comments) handleCommentsCount(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleCommentsCount") - - var c cmv1.Comments - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&c); err != nil { - respondWithPiError(w, r, "handleCommentsCount: unmarshal", - cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodeInputInvalid, - }) - return - } - - // Lookup session user. This is a public route so a session may not - // exist. Ignore any session not found errors. - usr, err := p.getSessionUser(w, r) - if err != nil && err != errSessionNotFound { - respondWithPiError(w, r, - "handleProposalInventory: getSessionUser: %v", err) - return - } - - cr, err := p.processCommentsCount(r.Context(), c, usr) - if err != nil { - respondWithPiError(w, r, - "handleCommentVote: processCommentsCount: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, cr) -} - -func (c *Comments) handleComments(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleComments") - - var c cmv1.Comments - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&c); err != nil { - respondWithPiError(w, r, "handleComments: unmarshal", - cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodeInputInvalid, - }) - return - } - - // Lookup session user. This is a public route so a session may not - // exist. Ignore any session not found errors. - usr, err := p.getSessionUser(w, r) - if err != nil && err != errSessionNotFound { - respondWithPiError(w, r, - "handleProposalInventory: getSessionUser: %v", err) - return - } - - cr, err := p.processComments(r.Context(), c, usr) - if err != nil { - respondWithPiError(w, r, - "handleCommentVote: processComments: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, cr) -} - -func (c *Comments) handleCommentVotes(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleCommentVotes") - - var v cmv1.Votes - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&cv); err != nil { - respondWithPiError(w, r, "handleCommentVotes: unmarshal", - cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodeInputInvalid, - }) - return - } - - vr, err := p.processCommentVotes(r.Context(), v) - if err != nil { - respondWithPiError(w, r, - "handleCommentVotes: processCommentVotes: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, vr) -} func (c *Comments) handleCommentTimestamps(w http.ResponseWriter, r *http.Request) { log.Tracef("handleCommentTimestamps") @@ -362,10 +95,10 @@ func (c *Comments) handleCommentTimestamps(w http.ResponseWriter, r *http.Reques // Lookup session user. This is a public route so a session may not // exist. Ignore any session not found errors. - usr, err := p.getSessionUser(w, r) + usr, err := p.GetSessionUser(w, r) if err != nil && err != errSessionNotFound { respondWithCommentsError(w, r, - "handleCommentTimestamps: getSessionUser: %v", err) + "handleCommentTimestamps: GetSessionUser: %v", err) return } diff --git a/politeiawww/comments/comments.go b/politeiawww/comments/comments.go index 042da62fe..54c9e789d 100644 --- a/politeiawww/comments/comments.go +++ b/politeiawww/comments/comments.go @@ -5,14 +5,19 @@ package comments import ( + "encoding/json" + "net/http" + pdclient "github.com/decred/politeia/politeiad/client" + cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/events" "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/politeiawww/user" + "github.com/decred/politeia/util" ) -// Comments is the context for the comments API. +// Comments is the context that handles the comments API. type Comments struct { cfg *config.Config politeiad *pdclient.Client @@ -21,6 +26,180 @@ type Comments struct { events *events.Manager } +// HandleNew is the request handler for the comments v1 New route. +func (c *Comments) HandleNew(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleNew") + + var n cmv1.New + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&n); err != nil { + respondWithError(w, r, "HandleNew: unmarshal", + cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodeInputInvalid, + }) + return + } + + u, err := c.sessions.GetSessionUser(w, r) + if err != nil { + respondWithError(w, r, + "HandleNew: GetSessionUser: %v", err) + return + } + + nr, err := c.processNew(r.Context(), n, *u) + if err != nil { + respondWithError(w, r, + "HandleNew: processNew: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, nr) +} + +// HandleVote is the request handler for the comments v1 Vote route. +func (c *Comments) HandleVote(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleVote") + + var v cmv1.Vote + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&v); err != nil { + respondWithError(w, r, "HandleVote: unmarshal", + cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodeInputInvalid, + }) + return + } + + u, err := c.sessions.GetSessionUser(w, r) + if err != nil { + respondWithError(w, r, + "HandleVote: GetSessionUser: %v", err) + return + } + + vr, err := c.processVote(r.Context(), v, *u) + if err != nil { + respondWithError(w, r, + "HandleVote: processVote: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, vr) +} + +// HandleDel is the request handler for the comments v1 Del route. +func (c *Comments) HandleDel(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleDel") + + var d cmv1.Del + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&d); err != nil { + respondWithError(w, r, "HandleDel: unmarshal", + cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodeInputInvalid, + }) + return + } + + u, err := c.sessions.GetSessionUser(w, r) + if err != nil { + respondWithError(w, r, + "HandleDel: GetSessionUser: %v", err) + return + } + + dr, err := c.processDel(r.Context(), d, *u) + if err != nil { + respondWithError(w, r, + "HandleDel: processDel: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, dr) +} + +// HandleCount is the request handler for the comments v1 Count route. +func (c *Comments) HandleCount(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleCount") + + var ct cmv1.Count + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&c); err != nil { + respondWithError(w, r, "HandleCount: unmarshal", + cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodeInputInvalid, + }) + return + } + + cr, err := c.processCount(r.Context(), ct) + if err != nil { + respondWithError(w, r, + "HandleCount: processCount: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, cr) +} + +// HandleComments is the request handler for the comments v1 Comments route. +func (c *Comments) HandleComments(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleComments") + + var cs cmv1.Comments + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&cs); err != nil { + respondWithError(w, r, "HandleComments: unmarshal", + cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodeInputInvalid, + }) + return + } + + // Lookup session user. This is a public route so a session may not + // exist. Ignore any session not found errors. + u, err := c.sessions.GetSessionUser(w, r) + if err != nil && err != sessions.ErrSessionNotFound { + respondWithError(w, r, + "HandleComments: GetSessionUser: %v", err) + return + } + + cr, err := c.processComments(r.Context(), cs, u) + if err != nil { + respondWithError(w, r, + "HandleComments: processComments: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, cr) +} + +// HandleVotes is the request handler for the comments v1 Votes route. +func (c *Comments) HandleVotes(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleVotes") + + var v cmv1.Votes + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&v); err != nil { + respondWithError(w, r, "HandleVotes: unmarshal", + cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodeInputInvalid, + }) + return + } + + vr, err := c.processVotes(r.Context(), v) + if err != nil { + respondWithError(w, r, + "HandleVotes: processVotes: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, vr) +} + // New returns a new Comments context. func New(cfg *config.Config, politeiad *pdclient.Client, userdb user.Database) *Comments { return &Comments{ diff --git a/politeiawww/comments/handlers.go b/politeiawww/comments/error.go similarity index 64% rename from politeiawww/comments/handlers.go rename to politeiawww/comments/error.go index 567c00353..2ba771837 100644 --- a/politeiawww/comments/handlers.go +++ b/politeiawww/comments/error.go @@ -5,7 +5,6 @@ package comments import ( - "encoding/json" "errors" "fmt" "net/http" @@ -18,66 +17,6 @@ import ( "github.com/decred/politeia/util" ) -func (c *Comments) HandleNew(w http.ResponseWriter, r *http.Request) { - log.Tracef("HandleNew") - - var n cmv1.New - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&n); err != nil { - respondWithError(w, r, "handleNew: unmarshal", - cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodeInputInvalid, - }) - return - } - - usr, err := c.sessions.GetSessionUser(w, r) - if err != nil { - respondWithError(w, r, - "handleNew: GetSessionUser: %v", err) - return - } - - nr, err := c.processNew(r.Context(), n, *usr) - if err != nil { - respondWithError(w, r, - "handleNew: processNew: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, nr) -} - -func (c *Comments) HandleVote(w http.ResponseWriter, r *http.Request) { - log.Tracef("HandleVote") - - var v cmv1.Vote - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&v); err != nil { - respondWithError(w, r, "handleVote: unmarshal", - cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodeInputInvalid, - }) - return - } - - usr, err := c.sessions.GetSessionUser(w, r) - if err != nil { - respondWithError(w, r, - "handleVote: GetSessionUser: %v", err) - return - } - - vr, err := c.processVote(r.Context(), v, *usr) - if err != nil { - respondWithError(w, r, - "handleVote: processVote: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, vr) -} - func respondWithError(w http.ResponseWriter, r *http.Request, format string, err error) { var ( ue cmv1.UserErrorReply diff --git a/politeiawww/comments/events.go b/politeiawww/comments/events.go index c01065372..53625e652 100644 --- a/politeiawww/comments/events.go +++ b/politeiawww/comments/events.go @@ -5,13 +5,13 @@ package comments const ( - // EventNew is the event that is emitted when a new comment is + // EventTypeNew is the event that is emitted when a new comment is // made. - EventNew = "commentnew" + EventTypeNew = "commentnew" ) -// EventDataNew is the event data that is emitted when a new comment is made. -type EventDataNew struct { +// EventNew is the event data that is emitted when a new comment is made. +type EventNew struct { State string Token string CommentID uint32 diff --git a/politeiawww/comments/process.go b/politeiawww/comments/process.go index d5274f5c9..b1f200495 100644 --- a/politeiawww/comments/process.go +++ b/politeiawww/comments/process.go @@ -11,10 +11,11 @@ import ( cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/user" + "github.com/google/uuid" ) func (c *Comments) processNew(ctx context.Context, n cmv1.New, u user.User) (*cmv1.NewReply, error) { - log.Tracef("processNew: %v %v %v", n.Token, n.ParentID, u.Username) + log.Tracef("processNew: %v %v %v", n.Token, u.Username) // Checking the mode is a temporary measure until user plugins // have been implemented. @@ -39,7 +40,7 @@ func (c *Comments) processNew(ctx context.Context, n cmv1.New, u user.User) (*cm // Only admins and the record author are allowed to comment on // unvetted records. if n.State == cmv1.RecordStateUnvetted && !u.Admin { - // Get the record author + // User is not an admin. Get the record author. authorID, err := c.politeiad.Author(ctx, n.State, n.Token) if err != nil { return nil, err @@ -71,8 +72,8 @@ func (c *Comments) processNew(ctx context.Context, n cmv1.New, u user.User) (*cm cm = commentPopulateUser(cm, u) // Emit event - c.events.Emit(EventNew, - EventDataNew{ + c.events.Emit(EventTypeNew, + EventNew{ State: n.State, Token: cm.Token, CommentID: cm.CommentID, @@ -121,7 +122,7 @@ func (c *Comments) processVote(ctx context.Context, v cmv1.Vote, u user.User) (* UserID: u.ID.String(), Token: v.Token, CommentID: v.CommentID, - Vote: convertVote(v.Vote), + Vote: comments.VoteT(v.Vote), PublicKey: v.PublicKey, Signature: v.Signature, } @@ -138,6 +139,131 @@ func (c *Comments) processVote(ctx context.Context, v cmv1.Vote, u user.User) (* }, nil } +func (c *Comments) processDel(ctx context.Context, d cmv1.Del, u user.User) (*cmv1.DelReply, error) { + log.Tracef("processDel: %v %v %v", d.Token, d.CommentID, d.Reason) + + // Verify user signed with their active identity + if u.PublicKey() != d.PublicKey { + return nil, cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodePublicKeyInvalid, + ErrorContext: "not active identity", + } + } + + // Send plugin command + cd := comments.Del{ + Token: d.Token, + CommentID: d.CommentID, + Reason: d.Reason, + PublicKey: d.PublicKey, + Signature: d.Signature, + } + cdr, err := c.politeiad.CommentDel(ctx, d.State, d.Token, cd) + if err != nil { + return nil, err + } + + // Prepare reply + cm := convertComment(cdr.Comment) + cm = commentPopulateUser(cm, u) + + return &cmv1.DelReply{ + Comment: cm, + }, nil +} + +func (c *Comments) processCount(ctx context.Context, ct cmv1.Count) (*cmv1.CountReply, error) { + log.Tracef("processCount: %v", ct.Tokens) + + counts, err := c.politeiad.CommentCounts(ctx, ct.State, ct.Tokens) + if err != nil { + return nil, err + } + + return &cmv1.CountReply{ + Counts: counts, + }, nil +} + +func (c *Comments) processComments(ctx context.Context, cs cmv1.Comments, u *user.User) (*cmv1.CommentsReply, error) { + log.Tracef("processComments: %v", cs.Token) + + // Only admins and the record author are allowed to retrieve + // unvetted comments. This is a public route so a user might + // not exist. + if cs.State == cmv1.RecordStateUnvetted { + var isAllowed bool + switch { + case u == nil: + // No logged in user. Not allowed. + isAllowed = false + case u.Admin: + // User is an admin. Allowed. + isAllowed = true + default: + // User is not an admin. Get the record author. + authorID, err := c.politeiad.Author(ctx, cs.State, cs.Token) + if err != nil { + return nil, err + } + if u.ID.String() == authorID { + // User is the author. Allowed. + isAllowed = true + } + } + if !isAllowed { + return nil, cmv1.UserErrorReply{ + // ErrorCode: cmv1.ErrorCodeUnauthorized, + ErrorContext: "user is not author or admin", + } + } + } + + // Send plugin command + pcomments, err := c.politeiad.CommentGetAll(ctx, cs.State, cs.Token) + if err != nil { + return nil, err + } + + // Prepare reply. Comment user data must be pulled from the + // userdb. + comments := make([]cmv1.Comment, 0, len(pcomments)) + for _, v := range pcomments { + cm := convertComment(v) + + // Get comment user data + uuid, err := uuid.Parse(cm.UserID) + if err != nil { + return nil, err + } + u, err := c.userdb.UserGetById(uuid) + if err != nil { + return nil, err + } + cm = commentPopulateUser(cm, *u) + + // Add comment + comments = append(comments, cm) + } + + return &cmv1.CommentsReply{ + Comments: comments, + }, nil +} + +func (c *Comments) processVotes(ctx context.Context, v cmv1.Votes) (*cmv1.VotesReply, error) { + log.Tracef("processVotes: %v %v", v.Token, v.UserID) + + votes, err := c.politeiad.CommentVotes(ctx, v.State, v.Token, v.UserID) + if err != nil { + return nil, err + } + + return &cmv1.VotesReply{ + Votes: convertCommentVotes(votes), + }, nil +} + func convertComment(c comments.Comment) cmv1.Comment { // Fields that are intentionally omitted are not stored in // politeiad. They need to be pulled from the userdb. @@ -161,14 +287,21 @@ func convertComment(c comments.Comment) cmv1.Comment { } } -func convertVote(v cmv1.VoteT) comments.VoteT { - switch v { - case cmv1.VoteDownvote: - return comments.VoteUpvote - case cmv1.VoteUpvote: - return comments.VoteDownvote +func convertCommentVotes(cv []comments.CommentVote) []cmv1.CommentVote { + c := make([]cmv1.CommentVote, 0, len(cv)) + for _, v := range cv { + c = append(c, cmv1.CommentVote{ + UserID: v.UserID, + Token: v.Token, + CommentID: v.CommentID, + Vote: cmv1.VoteT(v.Vote), + PublicKey: v.PublicKey, + Signature: v.Signature, + Timestamp: v.Timestamp, + Receipt: v.Receipt, + }) } - return comments.VoteInvalid + return c } // commentPopulateUserData populates the comment with user data that is not From b4987589bb361393009bc1c9f0373c768fac5b7c Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 22 Jan 2021 10:09:20 -0600 Subject: [PATCH 247/449] Finish transfering comment routes. --- politeiad/client/client.go | 5 +- politeiad/client/comments.go | 48 +++++++++++-- politeiawww/comments.go | 115 ------------------------------- politeiawww/comments/comments.go | 35 ++++++++++ politeiawww/comments/process.go | 71 +++++++++++++++++-- 5 files changed, 146 insertions(+), 128 deletions(-) delete mode 100644 politeiawww/comments.go diff --git a/politeiad/client/client.go b/politeiad/client/client.go index 7598d09ef..b6107279b 100644 --- a/politeiad/client/client.go +++ b/politeiad/client/client.go @@ -25,7 +25,8 @@ type Client struct { } // ErrorReply represents the request body that is returned from politeaid when -// an error occurs. PluginID will only be populated if this is a plugin error. +// an error occurs. PluginID will only be populated if the error occured during +// execution of a plugin command. type ErrorReply struct { PluginID string ErrorCode int @@ -33,7 +34,7 @@ type ErrorReply struct { } // Error represents a politeiad error. Error is returned anytime the politeiad -// is not a 200. +// response is not a 200. type Error struct { HTTPCode int ErrorReply ErrorReply diff --git a/politeiad/client/comments.go b/politeiad/client/comments.go index f8877b52f..5aaa69589 100644 --- a/politeiad/client/comments.go +++ b/politeiad/client/comments.go @@ -215,12 +215,9 @@ func (c *Client) CommentGetAll(ctx context.Context, state, token string) ([]comm // CommentVotes sends the comments plugin Votes command to the politeiad v1 // API. -func (c *Client) CommentVotes(ctx context.Context, state, token, userID string) ([]comments.CommentVote, error) { +func (c *Client) CommentVotes(ctx context.Context, state, token string, v comments.Votes) ([]comments.CommentVote, error) { // Setup request - cm := comments.Votes{ - UserID: userID, - } - b, err := json.Marshal(cm) + b, err := json.Marshal(v) if err != nil { return nil, err } @@ -256,3 +253,44 @@ func (c *Client) CommentVotes(ctx context.Context, state, token, userID string) return vr.Votes, nil } + +// CommentTimestamps sends the comments plugin Timestamps command to the +// politeiad v1 API. +func (c *Client) CommentTimestamps(ctx context.Context, state, token string, t comments.Timestamps) (*comments.TimestampsReply, error) { + // Setup request + b, err := json.Marshal(t) + if err != nil { + return nil, err + } + cmds := []pdv1.PluginCommandV2{ + { + State: state, + Token: token, + ID: comments.PluginID, + Command: comments.CmdTimestamps, + Payload: string(b), + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + if pcr.Error != nil { + return nil, pcr.Error + } + + // Decode reply + var tr comments.TimestampsReply + err = json.Unmarshal([]byte(pcr.Payload), &tr) + if err != nil { + return nil, err + } + + return &tr, nil +} diff --git a/politeiawww/comments.go b/politeiawww/comments.go deleted file mode 100644 index 9af361c8d..000000000 --- a/politeiawww/comments.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright (c) 2017-2021 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -/* -func convertCommentVoteFromPlugin(v comments.VoteT) piv1.CommentVoteT { - switch v { - case comments.VoteDownvote: - return piv1.CommentVoteDownvote - case comments.VoteUpvote: - return piv1.CommentVoteUpvote - } - return piv1.CommentVoteInvalid -} - -func convertProofFromCommentsPlugin(p comments.Proof) cmv1.Proof { - return cmv1.Proof{ - Type: p.Type, - Digest: p.Digest, - MerkleRoot: p.MerkleRoot, - MerklePath: p.MerklePath, - ExtraData: p.ExtraData, - } -} - -func convertTimestampFromCommentsPlugin(t comments.Timestamp) cmv1.Timestamp { - proofs := make([]cmv1.Proof, 0, len(t.Proofs)) - for _, v := range t.Proofs { - proofs = append(proofs, convertProofFromCommentsPlugin(v)) - } - return cmv1.Timestamp{ - Data: t.Data, - Digest: t.Digest, - TxID: t.TxID, - MerkleRoot: t.MerkleRoot, - Proofs: proofs, - } -} - -// commentPopulateUser populates the provided comment with user data that is -// not stored in politeiad. -func commentPopulateUser(c piv1.Comment, u user.User) cmv1.Comment { - c.Username = u.Username - return c -} - -func (c *Comments) processCommentTimestamps(ctx context.Context, t cmv1.Timestamps, isAdmin bool) (*cmv1.TimestampsReply, error) { - log.Tracef("processCommentTimestamps: %v %v %v", - t.State, t.Token, t.CommentIDs) - - // Get timestamps - ct := comments.Timestamps{ - CommentIDs: t.CommentIDs, - IncludeVotes: false, - } - ctr, err := p.commentTimestamps(ctx, ct) - if err != nil { - return nil, err - } - - // Prepare reply - comments := make(map[uint32][]cmv1.Timestamp, len(ctr.Comments)) - for commentID, timestamps := range ctr.Comments { - ts := make([]cmv1.Timestamp, 0, len(timestamps)) - for _, v := range timestamps { - // Strip unvetted data blobs if the user is not an admin - if t.State == cmv1.RecordStateUnvetted && !isAdmin { - v.Data = "" - } - ts = append(ts, convertTimestampFromCommentsPlugin(v)) - } - comments[commentID] = ts - } - - return &cmv1.TimestampsReply{ - Comments: comments, - }, nil -} - - -func (c *Comments) handleCommentTimestamps(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleCommentTimestamps") - - var t cmv1.Timestamps - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&t); err != nil { - respondWithCommentsError(w, r, "handleCommentTimestamps: unmarshal", - cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodeInputInvalid, - }) - return - } - - // Lookup session user. This is a public route so a session may not - // exist. Ignore any session not found errors. - usr, err := p.GetSessionUser(w, r) - if err != nil && err != errSessionNotFound { - respondWithCommentsError(w, r, - "handleCommentTimestamps: GetSessionUser: %v", err) - return - } - - isAdmin := usr != nil && usr.Admin - tr, err := p.processCommentTimestamps(r.Context(), t, isAdmin) - if err != nil { - respondWithCommentsError(w, r, - "handleCommentTimestamps: processCommentTimestamps: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, tr) -} -*/ diff --git a/politeiawww/comments/comments.go b/politeiawww/comments/comments.go index 54c9e789d..ef276f13c 100644 --- a/politeiawww/comments/comments.go +++ b/politeiawww/comments/comments.go @@ -200,6 +200,41 @@ func (c *Comments) HandleVotes(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, vr) } +// HandleTimestamps is the request handler for the comments v1 Timestamps +// route. +func (c *Comments) HandleTimestamps(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleTimestamps") + + var t cmv1.Timestamps + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&t); err != nil { + respondWithError(w, r, "HandleTimestamps: unmarshal", + cmv1.UserErrorReply{ + ErrorCode: cmv1.ErrorCodeInputInvalid, + }) + return + } + + // Lookup session user. This is a public route so a session may not + // exist. Ignore any session not found errors. + u, err := c.sessions.GetSessionUser(w, r) + if err != nil && err != sessions.ErrSessionNotFound { + respondWithError(w, r, + "HandleTimestamps: GetSessionUser: %v", err) + return + } + + isAdmin := u != nil && u.Admin + tr, err := c.processTimestamps(r.Context(), t, isAdmin) + if err != nil { + respondWithError(w, r, + "HandleTimestamps: processTimestamps: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, tr) +} + // New returns a new Comments context. func New(cfg *config.Config, politeiad *pdclient.Client, userdb user.Database) *Comments { return &Comments{ diff --git a/politeiawww/comments/process.go b/politeiawww/comments/process.go index b1f200495..79f5d8492 100644 --- a/politeiawww/comments/process.go +++ b/politeiawww/comments/process.go @@ -254,7 +254,11 @@ func (c *Comments) processComments(ctx context.Context, cs cmv1.Comments, u *use func (c *Comments) processVotes(ctx context.Context, v cmv1.Votes) (*cmv1.VotesReply, error) { log.Tracef("processVotes: %v %v", v.Token, v.UserID) - votes, err := c.politeiad.CommentVotes(ctx, v.State, v.Token, v.UserID) + // Get comment votes + cm := comments.Votes{ + UserID: v.UserID, + } + votes, err := c.politeiad.CommentVotes(ctx, v.State, v.Token, cm) if err != nil { return nil, err } @@ -264,6 +268,44 @@ func (c *Comments) processVotes(ctx context.Context, v cmv1.Votes) (*cmv1.VotesR }, nil } +func (c *Comments) processTimestamps(ctx context.Context, t cmv1.Timestamps, isAdmin bool) (*cmv1.TimestampsReply, error) { + log.Tracef("processTimestamps: %v %v", t.Token, t.CommentIDs) + + // Get timestamps + ct := comments.Timestamps{ + CommentIDs: t.CommentIDs, + } + ctr, err := c.politeiad.CommentTimestamps(ctx, t.State, t.Token, ct) + if err != nil { + return nil, err + } + + // Prepare reply + comments := make(map[uint32][]cmv1.Timestamp, len(ctr.Comments)) + for commentID, timestamps := range ctr.Comments { + ts := make([]cmv1.Timestamp, 0, len(timestamps)) + for _, v := range timestamps { + // Strip unvetted data blobs if the user is not an admin + if t.State == cmv1.RecordStateUnvetted && !isAdmin { + v.Data = "" + } + ts = append(ts, convertTimestamp(v)) + } + comments[commentID] = ts + } + + return &cmv1.TimestampsReply{ + Comments: comments, + }, nil +} + +// commentPopulateUserData populates the comment with user data that is not +// stored in politeiad. +func commentPopulateUser(c cmv1.Comment, u user.User) cmv1.Comment { + c.Username = u.Username + return c +} + func convertComment(c comments.Comment) cmv1.Comment { // Fields that are intentionally omitted are not stored in // politeiad. They need to be pulled from the userdb. @@ -304,11 +346,28 @@ func convertCommentVotes(cv []comments.CommentVote) []cmv1.CommentVote { return c } -// commentPopulateUserData populates the comment with user data that is not -// stored in politeiad. -func commentPopulateUser(c cmv1.Comment, u user.User) cmv1.Comment { - c.Username = u.Username - return c +func convertProof(p comments.Proof) cmv1.Proof { + return cmv1.Proof{ + Type: p.Type, + Digest: p.Digest, + MerkleRoot: p.MerkleRoot, + MerklePath: p.MerklePath, + ExtraData: p.ExtraData, + } +} + +func convertTimestamp(t comments.Timestamp) cmv1.Timestamp { + proofs := make([]cmv1.Proof, 0, len(t.Proofs)) + for _, v := range t.Proofs { + proofs = append(proofs, convertProof(v)) + } + return cmv1.Timestamp{ + Data: t.Data, + Digest: t.Digest, + TxID: t.TxID, + MerkleRoot: t.MerkleRoot, + Proofs: proofs, + } } // This function is a temporary function that will be removed once user plugins From 7912ace9fa6c19eebf35b42a4f9e04d50d25ca25 Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 22 Jan 2021 10:20:37 -0600 Subject: [PATCH 248/449] Setup comment routes. --- politeiawww/piwww.go | 49 ++++++++++++++++++++---------------------- politeiawww/testing.go | 6 +++++- politeiawww/www.go | 40 ++++++---------------------------- 3 files changed, 35 insertions(+), 60 deletions(-) diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 4bc64a47b..3c5cc3226 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -24,10 +24,12 @@ import ( "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/ticketvote" usermd "github.com/decred/politeia/politeiad/plugins/user" + cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" + "github.com/decred/politeia/politeiawww/comments" "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/politeiawww/user" wwwutil "github.com/decred/politeia/politeiawww/util" @@ -1946,7 +1948,7 @@ func (p *politeiawww) handleVoteInventory(w http.ResponseWriter, r *http.Request } // setPiRoutes sets the pi API routes. -func (p *politeiawww) setPiRoutes() { +func (p *politeiawww) setPiRoutes(c *comments.Comments) { // Return a 404 when a route is not found p.router.NotFoundHandler = http.HandlerFunc(p.handleNotFound) @@ -1957,8 +1959,7 @@ func (p *politeiawww) setPiRoutes() { HandleFunc(www.PoliteiaWWWAPIRoute+www.RouteVersion, p.handleVersion). Methods(http.MethodGet) - // www routes. These routes have been deprecated and support will - // be removed in the future. + // www routes. These routes have been DEPRECATED. p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, www.RoutePolicy, p.handlePolicy, permissionPublic) @@ -2004,29 +2005,25 @@ func (p *politeiawww) setPiRoutes() { piv1.RouteProposalInventory, p.handleProposalInventory, permissionPublic) - // Pi routes - comments - /* - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteCommentNew, p.handleCommentNew, - permissionLogin) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteCommentVote, p.handleCommentVote, - permissionLogin) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteCommentCensor, p.handleCommentCensor, - permissionAdmin) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteComments, p.handleComments, - permissionPublic) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteCommentVotes, p.handleCommentVotes, - permissionPublic) - - // Comment routes - p.addRoute(http.MethodPost, cmv1.APIRoute, - cmv1.RouteTimestamps, p.handleCommentTimestamps, - permissionPublic) - */ + // Comment routes + p.addRoute(http.MethodPost, cmv1.APIRoute, + cmv1.RouteNew, c.HandleNew, + permissionLogin) + p.addRoute(http.MethodPost, cmv1.APIRoute, + cmv1.RouteVote, c.HandleVote, + permissionLogin) + p.addRoute(http.MethodPost, cmv1.APIRoute, + cmv1.RouteDel, c.HandleDel, + permissionAdmin) + p.addRoute(http.MethodPost, cmv1.APIRoute, + cmv1.RouteComments, c.HandleComments, + permissionPublic) + p.addRoute(http.MethodPost, cmv1.APIRoute, + cmv1.RouteVotes, c.HandleVotes, + permissionPublic) + p.addRoute(http.MethodPost, cmv1.APIRoute, + cmv1.RouteTimestamps, c.HandleTimestamps, + permissionPublic) // Pi routes - vote p.addRoute(http.MethodPost, piv1.APIRoute, diff --git a/politeiawww/testing.go b/politeiawww/testing.go index bbd9f031b..f1b7b064f 100644 --- a/politeiawww/testing.go +++ b/politeiawww/testing.go @@ -26,6 +26,7 @@ import ( cms "github.com/decred/politeia/politeiawww/api/cms/v1" pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" + "github.com/decred/politeia/politeiawww/comments" "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/politeiawww/user" @@ -359,9 +360,12 @@ func newTestPoliteiawww(t *testing.T) (*politeiawww, func()) { userPaywallPool: make(map[uuid.UUID]paywallPoolMember), } + // TODO setup testing + var c *comments.Comments + // Setup routes p.setUserWWWRoutes() - p.setPiRoutes() + p.setPiRoutes(c) // The cleanup is handled using a closure so that the temp dir // can be deleted using the local variable and not cfg.DataDir. diff --git a/politeiawww/www.go b/politeiawww/www.go index 5f7567546..53c1fdafc 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -27,7 +27,6 @@ import ( "github.com/decred/politeia/mdstream" pd "github.com/decred/politeia/politeiad/api/v1" pdclient "github.com/decred/politeia/politeiad/client" - "github.com/decred/politeia/politeiad/plugins/comments" piplugin "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/ticketvote" cms "github.com/decred/politeia/politeiawww/api/cms/v1" @@ -36,6 +35,7 @@ import ( database "github.com/decred/politeia/politeiawww/cmsdatabase" cmsdb "github.com/decred/politeia/politeiawww/cmsdatabase/cockroachdb" ghtracker "github.com/decred/politeia/politeiawww/codetracker/github" + "github.com/decred/politeia/politeiawww/comments" "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/events" "github.com/decred/politeia/politeiawww/sessions" @@ -87,24 +87,6 @@ func convertWWWErrorStatusFromPD(e pd.ErrorStatusT) www.ErrorStatusT { return www.ErrorStatusInvalid } -// TODO verify all plugin errors have been added to these www conversion -// functions -func convertWWWErrorStatusFromPiPlugin(e piplugin.ErrorCodeT) www.ErrorStatusT { - return www.ErrorStatusInvalid -} - -func convertWWWErrorStatusFromComments(e comments.ErrorCodeT) www.ErrorStatusT { - switch e { - case comments.ErrorCodeTokenInvalid: - return www.ErrorStatusInvalidCensorshipToken - case comments.ErrorCodeCommentNotFound: - return www.ErrorStatusCommentNotFound - case comments.ErrorCodeParentIDInvalid: - return www.ErrorStatusCommentNotFound - } - return www.ErrorStatusInvalid -} - func convertWWWErrorStatusFromTicketVote(e ticketvote.ErrorCodeT) www.ErrorStatusT { switch e { case ticketvote.ErrorCodeTokenInvalid: @@ -123,25 +105,14 @@ func convertWWWErrorStatusFromTicketVote(e ticketvote.ErrorCodeT) www.ErrorStatu return www.ErrorStatusInvalid } -// convertWWWErrorStatus attempts to convert the provided politeiad plugin ID -// and error code into a www ErrorStatusT. If a plugin ID is provided the error -// code is assumed to be a user error code from the specified plugin API. If -// no plugin ID is provided the error code is assumed to be a user error code -// from the politeiad API. +// TODO make sure the legacy www routes have the plugin error conversions that +// they need. func convertWWWErrorStatus(pluginID string, errCode int) www.ErrorStatusT { switch pluginID { case "": // politeiad API e := pd.ErrorStatusT(errCode) return convertWWWErrorStatusFromPD(e) - case piplugin.PluginID: - // Pi plugin - e := piplugin.ErrorCodeT(errCode) - return convertWWWErrorStatusFromPiPlugin(e) - case comments.PluginID: - // Comments plugin - e := comments.ErrorCodeT(errCode) - return convertWWWErrorStatusFromComments(e) case ticketvote.PluginID: // Ticket vote plugin e := ticketvote.ErrorCodeT(errCode) @@ -536,9 +507,12 @@ func (p *politeiawww) addRoute(method string, routeVersion string, route string, } func (p *politeiawww) setupPi() error { + // Setup api contexts + c := comments.New(p.cfg, p.politeiad, p.db) + // Setup routes p.setUserWWWRoutes() - p.setPiRoutes() + p.setPiRoutes(c) // Verify paywall settings switch { From b201806d7b11180127b67986c943d230262f64b4 Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 22 Jan 2021 16:58:36 -0600 Subject: [PATCH 249/449] Add ticketvote api. --- politeiawww/api/pi/v1/v1.go | 364 ++---------------------- politeiawww/api/ticketvote/v1/v1.go | 410 +++++++++++++++++++++++++++ politeiawww/piwww.go | 231 ++------------- politeiawww/ticketvote/ticketvote.go | 202 +++++++++++++ 4 files changed, 662 insertions(+), 545 deletions(-) create mode 100644 politeiawww/ticketvote/ticketvote.go diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 8084ee0bb..59787cf78 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -8,19 +8,11 @@ import ( "fmt" ) -type VoteStatusT int -type VoteAuthActionT string -type VoteT int - // TODO verify that all batched request have a page size limit -// TODO comments count and linked from should be pulled out of the proposal -// record struct. These should be separate endpoints: -// /ticketvote/linkedfrom -// TODO make RouteVoteResults a batched route but that only currently allows -// for 1 result to be returned so that we have the option to change this if -// we want to. -// TODO pi needs a Version route and a Policy route. The policies should be -// defined in the plugin packages and returned in the policy route. +// TODO pi needs a Version route that returns the APIs and versions that pi +// uses. +// TODO new APIs need a Policy route. The policies should be defined in the +// plugin packages as plugin settings and returned in a policy command. // TODO module these API packages const ( @@ -35,56 +27,7 @@ const ( RouteProposalInventory = "/proposals/inventory" // Vote routes - RouteVoteAuthorize = "/vote/authorize" - RouteVoteStart = "/vote/start" - RouteCastBallot = "/vote/castballot" - RouteVotes = "/votes" - RouteVoteResults = "/votes/results" - RouteVoteSummaries = "/votes/summaries" RouteVoteInventory = "/votes/inventory" - - // Vote statuses - VoteStatusInvalid VoteStatusT = 0 // Invalid status - VoteStatusUnauthorized VoteStatusT = 1 // Vote has not been authorized - VoteStatusAuthorized VoteStatusT = 2 // Vote has been authorized - VoteStatusStarted VoteStatusT = 3 // Vote has been started - VoteStatusFinished VoteStatusT = 4 // Vote has finished - - // Vote authorization actions - VoteAuthActionAuthorize VoteAuthActionT = "authorize" - VoteAuthActionRevoke VoteAuthActionT = "revoke" - - // VoteTypeInvalid represents and invalid vote type. - VoteTypeInvalid VoteT = 0 - - // VoteTypeStandard is used to indicate a simple approve or reject - // vote where the winner is the voting option that has met the - // specified quorum and pass requirements. - VoteTypeStandard VoteT = 1 - - // VoteTypeRunoff specifies a runoff vote that multiple proposals - // compete in. All proposals are voted on like normal and all votes - // are simple approve/reject votes, but there can only be one - // winner in a runoff vote. The winner is the proposal that meets - // the quorum requirement, meets the pass requirement, and that has - // the most net yes votes. The winning proposal is considered - // approved and all other proposals are considered to be rejected. - // If no proposals meet the quorum and pass requirements then all - // proposals are considered rejected. Note, in a runoff vote it is - // possible for a proposal to meet both the quorum and pass - // requirements but still be rejected if it does not have the most - // net yes votes. - VoteTypeRunoff VoteT = 2 - - // VoteOptionIDApprove is the vote option ID that indicates the - // proposal should be approved. Proposal votes are required to use - // this vote option ID. - VoteOptionIDApprove = "yes" - - // VoteOptionIDReject is the vote option ID that indicates the - // proposal should be rejected. Proposal votes are required to use - // this vote option ID. - VoteOptionIDReject = "no" ) // ErrorStatusT represents a user error status code. @@ -133,16 +76,6 @@ const ( ErrorStatusPropStatusChangeInvalid ErrorStatusPropStatusChangeReasonInvalid ErrorStatusNoPropChanges - - // Vote errors - ErrorStatusVoteAuthInvalid - ErrorStatusVoteStatusInvalid - ErrorStatusStartDetailsInvalid - ErrorStatusStartDetailsMissing - ErrorStatusVoteParamsInvalid - ErrorStatusVoteTypeInvalid - ErroStatusVoteParentInvalid - ErrorStatusLinkByNotExpired ) var ( @@ -188,10 +121,6 @@ var ( ErrorStatusPropStatusChangeInvalid: "proposal status change invalid", ErrorStatusPropStatusChangeReasonInvalid: "proposal status reason invalid", ErrorStatusNoPropChanges: "no proposal changes", - - // Vote errors - ErrorStatusVoteStatusInvalid: "vote status invalid", - ErrorStatusVoteParamsInvalid: "vote params invalid", } ) @@ -296,10 +225,14 @@ type Metadata struct { Payload string `json:"payload"` // JSON metadata content, base64 encoded } -// Metadata hints const ( - // HintProposalMetadata is the proposal metadata hint - HintProposalMetadata = "proposalmetadata" + // HintProposalMetadata is the Metadata object hint that is used + // when the payload contains a ProposalMetadata. + HintProposalMetadata = "proposalmd" + + // HintVoteMetadata is the Metadata object hint that is used when + // the payload contains a VoteMetadata. + HintVoteMetadata = "votemd" ) // ProposalMetadata contains metadata that is specified by the user on proposal @@ -308,17 +241,19 @@ type ProposalMetadata struct { Name string `json:"name"` // Proposal name } -// VoteMetadata that is specified by the user on proposal submission in order -// to host or participate in certain types of votes. It is attached to a -// proposal submission as a Metadata object. +// VoteMetadata that is specified by the user on proposal submission in order to +// host or participate in certain types of votes. It is attached to a proposal +// submission as a Metadata object. type VoteMetadata struct { - // LinkBy is a UNIX timestamp that serves as a deadline for other - // proposals to link to this proposal. Ex, an RFP submission cannot - // link to an RFP proposal once the RFP's LinkBy deadline is past. + // LinkBy is set when the user intends for the proposal to be the + // parent proposal in a runoff vote. It is a UNIX timestamp that + // serves as the deadline for other proposals to declare their + // intent to participate in the runoff vote. LinkBy int64 `json:"linkby,omitempty"` - // LinkTo specifies a public proposal token to link this proposal - // to. Ex, an RFP submission must link to the RFP proposal. + // LinkTo is the censorship token of a runoff vote parent proposal. + // It is set when a proposal is being submitted as a vote options + // in the runoff vote. LinkTo string `json:"linkto,omitempty"` } @@ -475,263 +410,6 @@ type ProposalInventoryReply struct { Vetted map[string][]string `json:"vetted"` } -// AuthDetails contains the details of a vote authorization. -type AuthDetails struct { - Token string `json:"token"` // Proposal token - Version uint32 `json:"version"` // Proposal version - Action string `json:"action"` // Authorize or revoke - PublicKey string `json:"publickey"` // Public key used for signature - Signature string `json:"signature"` // Signature of token+version+action - Timestamp int64 `json:"timestamp"` // Received UNIX timestamp - Receipt string `json:"receipt"` // Server signature of client signature -} - -// VoteOption describes a single vote option. -type VoteOption struct { - ID string `json:"id"` // Single, unique word (e.g. yes) - Description string `json:"description"` // Longer description of the vote - Bit uint64 `json:"bit"` // Bit used for this option -} - -// VoteParams contains all client defined vote params required by server to -// start a proposal vote. -type VoteParams struct { - Token string `json:"token"` // Proposal token - Version uint32 `json:"version"` // Proposal version - Type VoteT `json:"type"` // Vote type - Mask uint64 `json:"mask"` // Valid vote bits - Duration uint32 `json:"duration"` // Duration in blocks - - // QuorumPercentage is the percent of elligible votes required for - // the vote to meet a quorum. - QuorumPercentage uint32 `json:"quorumpercentage"` - - // PassPercentage is the percent of total votes that are required - // to consider a vote option as passed. - PassPercentage uint32 `json:"passpercentage"` - - Options []VoteOption `json:"options"` - - // Parent is the token of the parent proposal. This field will only - // be populated for runoff votes. - Parent string `json:"parent,omitempty"` -} - -// VoteDetails contains the details of a proposal vote. -// -// Signature is the client signature of the SHA256 digest of the JSON encoded -// Vote struct. -type VoteDetails struct { - Params VoteParams `json:"params"` - PublicKey string `json:"publickey"` - Signature string `json:"signature"` - StartBlockHeight uint32 `json:"startblockheight"` - StartBlockHash string `json:"startblockhash"` - EndBlockHeight uint32 `json:"endblockheight"` - EligibleTickets []string `json:"eligibletickets"` // Ticket hashes -} - -// CastVoteDetails contains the details of a cast vote. -type CastVoteDetails struct { - Token string `json:"token"` // Proposal token - Ticket string `json:"ticket"` // Ticket hash - VoteBit string `json:"votebits"` // Selected vote bit, hex encoded - Signature string `json:"signature"` // Signature of Token+Ticket+VoteBit - Receipt string `json:"receipt"` // Server signature of client signature -} - -// VoteResult describes a vote option and the total number of votes that have -// been cast for this option. -type VoteResult struct { - ID string `json:"id"` // Single unique word (e.g. yes) - Description string `json:"description"` // Longer description of the vote - VoteBit uint64 `json:"votebit"` // Bits used for this option - Votes uint64 `json:"votes"` // Votes cast for this option -} - -// VoteSummary summarizes the vote params and results of a proposal vote. -type VoteSummary struct { - Type VoteT `json:"type"` - Status VoteStatusT `json:"status"` - Duration uint32 `json:"duration"` // In blocks - StartBlockHeight uint32 `json:"startblockheight"` - StartBlockHash string `json:"startblockhash"` - EndBlockHeight uint32 `json:"endblockheight"` - - // EligibleTickets is the number of tickets that are eligible to - // cast a vote. - EligibleTickets uint32 `json:"eligibletickets"` - - // QuorumPercentage is the percent of eligible tickets required to - // vote in order to have a quorum. - QuorumPercentage uint32 `json:"quorumpercentage"` - - // PassPercentage is the percent of total votes required to approve - // the vote in order for the vote to pass. - PassPercentage uint32 `json:"passpercentage"` - - Results []VoteResult `json:"results"` - Approved bool `json:"approved"` // Was the vote approved -} - -// VoteAuthorize authorizes a proposal vote or revokes a previous vote -// authorization. All proposal votes must be authorized by the proposal author -// before an admin is able to start the voting process. -// -// Signature contains the client signature of the Token+Version+Action. -type VoteAuthorize struct { - Token string `json:"token"` - Version uint32 `json:"version"` - Action VoteAuthActionT `json:"action"` - PublicKey string `json:"publickey"` - Signature string `json:"signature"` -} - -// VoteAuthorizeReply is the reply to the VoteAuthorize command. -// -// Receipt is the server signature of the client signature. This is proof that -// the server received and processed the VoteAuthorize command. -type VoteAuthorizeReply struct { - Timestamp int64 `json:"timestamp"` - Receipt string `json:"receipt"` -} - -// StartDetails is the structure that is provided when starting a proposal vote. -// -// Signature is the signature of a SHA256 digest of the JSON encoded VoteParams -// structure. -type StartDetails struct { - Params VoteParams `json:"params"` - PublicKey string `json:"publickey"` - Signature string `json:"signature"` -} - -// VoteStart starts a proposal vote or multiple proposal votes if the vote is -// a runoff vote. -// -// Standard votes require that the vote have been authorized by the proposal -// author before an admin will able to start the voting process. The -// StartDetails list should only contain a single StartDetails. -// -// Runoff votes can be started by an admin at any point once the RFP link by -// deadline has expired. Runoff votes DO NOT require the votes to have been -// authorized by the submission authors prior to an admin starting the runoff -// vote. All public, non-abandoned RFP submissions should be included in the -// list of StartDetails. -type VoteStart struct { - Starts []StartDetails `json:"starts"` -} - -// VoteStartReply is the reply to the VoteStart command. -type VoteStartReply struct { - StartBlockHeight uint32 `json:"startblockheight"` - StartBlockHash string `json:"startblockhash"` - EndBlockHeight uint32 `json:"endblockheight"` - EligibleTickets []string `json:"eligibletickets"` -} - -// VoteErrorT represents an error that occurred while attempting to cast a -// ticket vote. -type VoteErrorT int - -const ( - // Cast vote errors - // TODO these need human readable equivalents - VoteErrorInvalid VoteErrorT = 0 - VoteErrorInternalError VoteErrorT = 1 - VoteErrorTokenInvalid VoteErrorT = 2 - VoteErrorRecordNotFound VoteErrorT = 3 - VoteErrorMultipleRecordVotes VoteErrorT = 4 - VoteErrorVoteStatusInvalid VoteErrorT = 5 - VoteErrorVoteBitInvalid VoteErrorT = 6 - VoteErrorSignatureInvalid VoteErrorT = 7 - VoteErrorTicketNotEligible VoteErrorT = 8 - VoteErrorTicketAlreadyVoted VoteErrorT = 9 -) - -// CastVote is a signed ticket vote. -type CastVote struct { - Token string `json:"token"` // Proposal token - Ticket string `json:"ticket"` // Ticket ID - VoteBit string `json:"votebits"` // Selected vote bit, hex encoded - Signature string `json:"signature"` // Signature of Token+Ticket+VoteBit -} - -// CastVoteReply contains the receipt for the cast vote. -type CastVoteReply struct { - Ticket string `json:"ticket"` // Ticket ID - Receipt string `json:"receipt"` // Server signature of client signature - - // The follwing fields will only be present if an error occurred - // while attempting to cast the vote. - ErrorCode VoteErrorT `json:"errorcode,omitempty"` - ErrorContext string `json:"errorcontext,omitempty"` -} - -// CastBallot casts a ballot of votes. A ballot can only contain the votes for -// a single record. -type CastBallot struct { - Votes []CastVote `json:"votes"` -} - -// CastBallotReply is a reply to a batched list of votes. -type CastBallotReply struct { - Receipts []CastVoteReply `json:"receipts"` -} - -// ProposalVote contains all vote authorizations and the vote details for a -// proposal vote. The vote details will be null if the proposal vote has not -// been started yet. -type ProposalVote struct { - Auths []AuthDetails `json:"auths"` - Vote *VoteDetails `json:"vote"` -} - -// Votes returns the vote authorizations and vote details for each of the -// provided proposal tokens. -type Votes struct { - Tokens []string `json:"tokens"` -} - -// VotesReply is the reply to the Votes command. The returned map will not -// contain an entry for any tokens that did not correspond to an actual -// proposal. It is the callers responsibility to ensure that a entry is -// returned for all of the provided tokens. -type VotesReply struct { - Votes map[string]ProposalVote `json:"votes"` -} - -// VoteResults returns the votes that have been cast for the specified -// proposal. -type VoteResults struct { - Token string `json:"token"` -} - -// VoteResultsReply is the reply to the VoteResults command. -type VoteResultsReply struct { - Votes []CastVoteDetails `json:"votes"` -} - -// VoteSummaries summarizes the vote params and results for a ticket vote. -type VoteSummaries struct { - Tokens []string `json:"tokens"` -} - -// VoteSummariesReply is the reply to the VoteSummaries command. -// -// Summaries field contains a vote summary for each of the provided -// tokens. The map will not contain an entry for any tokens that -// did not correspond to an actual record. It is the callers -// responsibility to ensure that a summary is returned for all of -// the provided tokens. -type VoteSummariesReply struct { - Summaries map[string]VoteSummary `json:"summaries"` // [token]Summary - - // BestBlock is the best block value that was used to prepare the - // summaries. - BestBlock uint32 `json:"bestblock"` -} - // VoteInventory retrieves the tokens of all public, non-abandoned proposals // categorized by their vote status. type VoteInventory struct{} diff --git a/politeiawww/api/ticketvote/v1/v1.go b/politeiawww/api/ticketvote/v1/v1.go index 6aa23a8c7..3944b8703 100644 --- a/politeiawww/api/ticketvote/v1/v1.go +++ b/politeiawww/api/ticketvote/v1/v1.go @@ -10,6 +10,14 @@ const ( // APIRoute is prefixed onto all routes defined in this package. APIRoute = "/ticketvote/v1" + RouteAuthorize = "/authorize" + RouteStart = "/start" + RouteCastBallot = "/castballot" + RouteDetails = "/details" + RouteResults = "/results" + RouteSummaries = "/summaries" + RouteInventory = "/inventory" + RouteLinkedFrom = "/linkedfrom" RouteTimestamps = "/timestamps" ) @@ -69,6 +77,408 @@ func (e ServerErrorReply) Error() string { return fmt.Sprintf("server error: %v", e.ErrorCode) } +// AuthActionT represents an Authorize action. +type AuthActionT string + +const ( + // AuthActionAuthorize is used to authorize a record vote. + AuthActionAuthorize AuthActionT = "authorize" + + // AuthActionRevoke is used to revoke a previous authorization. + AuthActionRevoke AuthActionT = "revoke" +) + +// Authorize authorizes a record vote or revokes a previous vote authorization. +// Not all vote types require an authorization. +// +// Signature contains the client signature of the Token+Version+Action. +type Authorize struct { + Token string `json:"token"` + Version uint32 `json:"version"` + Action AuthActionT `json:"action"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` +} + +// AuthorizeReply is the reply to the Authorize command. +// +// Receipt is the server signature of the client signature. This is proof that +// the server received and processed the Authorize command. +type AuthorizeReply struct { + Timestamp int64 `json:"timestamp"` + Receipt string `json:"receipt"` +} + +// VoteT represents a vote type. +type VoteT int + +const ( + // VoteTypeInvalid represents and invalid vote type. + VoteTypeInvalid VoteT = 0 + + // VoteTypeStandard is used to indicate a simple approve or reject + // vote where the winner is the voting option that has met the + // specified quorum and pass requirements. Standard votes require + // an authorization from the record author before the voting period + // can be started by an admin. + VoteTypeStandard VoteT = 1 + + // VoteTypeRunoff specifies a runoff vote that multiple records + // compete in. All records are voted on like normal and all votes + // are simple approve/reject votes, but there can only be one + // winner in a runoff vote. The winner is the record that meets + // the quorum requirement, meets the pass requirement, and that has + // the most net yes votes. The winning record is considered + // approved and all other records are considered to be rejected. + // If no records meet the quorum and pass requirements then all + // records are considered rejected. Note, in a runoff vote it is + // possible for a record to meet both the quorum and pass + // requirements but still be rejected if it does not have the most + // net yes votes. + VoteTypeRunoff VoteT = 2 + + // VoteOptionIDApprove is the vote option ID that indicates the + // record should be approved. Standard votes and runoff vote + // submissions are required to use this vote option ID. + VoteOptionIDApprove = "yes" + + // VoteOptionIDReject is the vote option ID that indicates the + // record should be rejected. Standard votes and runoff vote + // submissions are required to use this vote option ID. + VoteOptionIDReject = "no" +) + +// VoteStatusT represents a vote status. +type VoteStatusT int + +const ( + // VoteStatusInvalid represents an invalid vote status. + VoteStatusInvalid VoteStatusT = 0 + + // VoteStatusUnauthorized represents a vote that has not been + // authorized yet. Some vote types require prior authorization from + // the record author before an admin can start the voting period. + VoteStatusUnauthorized VoteStatusT = 1 + + // VoteStatusAuthorized represents a vote that has been authorized. + // Some vote types require prior authorization from the record + // author before an admin can start the voting period. + VoteStatusAuthorized VoteStatusT = 2 + + // VoteStatusStarted represents a vote that has been started and + // is still ongoing. + VoteStatusStarted VoteStatusT = 3 + + // VoteStatusFinished represents a vote that has ended. + VoteStatusFinished VoteStatusT = 4 +) + +var ( + // VoteStatuses contains the human readable vote statuses. + VoteStatuses = map[VoteStatusT]string{ + VoteStatusInvalid: "invalid", + VoteStatusUnauthorized: "unauthorized", + VoteStatusAuthorized: "authorized", + VoteStatusStarted: "started", + VoteStatusFinished: "finished", + } +) + +// VoteMetadata that is specified by the user on record submission in order to +// host or participate in certain types of votes. It is attached to a record +// submission as a metadata stream. +type VoteMetadata struct { + // LinkBy is set when the user intends for the record to be the + // parent record in a runoff vote. It is a UNIX timestamp that + // serves as the deadline for other records to declare their intent + // to participate in the runoff vote. + LinkBy int64 `json:"linkby,omitempty"` + + // LinkTo is the censorship token of a runoff vote parent record. + // It is set when a record is being submitted as a vote options in + // the runoff vote. + LinkTo string `json:"linkto,omitempty"` +} + +// VoteOption describes a single vote option. +type VoteOption struct { + ID string `json:"id"` // Single, unique word (e.g. yes) + Description string `json:"description"` // Longer description of the vote + Bit uint64 `json:"bit"` // Bit used for this option +} + +// VoteParams contains all client defined vote params required by server to +// start a record vote. +type VoteParams struct { + Token string `json:"token"` // Record token + Version uint32 `json:"version"` // Record version + Type VoteT `json:"type"` // Vote type + Mask uint64 `json:"mask"` // Valid vote bits + Duration uint32 `json:"duration"` // Duration in blocks + + // QuorumPercentage is the percent of elligible votes required for + // the vote to meet a quorum. + QuorumPercentage uint32 `json:"quorumpercentage"` + + // PassPercentage is the percent of total votes that are required + // to consider a vote option as passed. + PassPercentage uint32 `json:"passpercentage"` + + Options []VoteOption `json:"options"` + + // Parent is the token of the parent record. This field will only + // be populated for runoff votes. + Parent string `json:"parent,omitempty"` +} + +// StartDetails is the structure that is provided when starting a record +// vote. +// +// Signature is the signature of a SHA256 digest of the JSON encoded +// VoteParams. +type StartDetails struct { + Params VoteParams `json:"params"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` +} + +// Start starts a record vote or multiple record votes if the vote is a runoff +// vote. +// +// Standard votes require that the vote have been authorized by the record +// author before an admin will able to start the voting process. The +// StartDetails list should only contain a single StartDetails. +// +// Runoff votes can be started by an admin at any point once the RFP link by +// deadline has expired. Runoff votes DO NOT require the votes to have been +// authorized by the submission authors prior to an admin starting the runoff +// vote. All public, non-abandoned RFP submissions should be included in the +// list of StartDetails. +type Start struct { + Starts []StartDetails `json:"starts"` +} + +// StartReply is the reply to the Start command. +type StartReply struct { + StartBlockHeight uint32 `json:"startblockheight"` + StartBlockHash string `json:"startblockhash"` + EndBlockHeight uint32 `json:"endblockheight"` + EligibleTickets []string `json:"eligibletickets"` +} + +// VoteErrorT represents an error that occurred while attempting to cast a +// ticket vote. +type VoteErrorT int + +const ( + // VoteErrorInvalid is an invalid vote error. + VoteErrorInvalid VoteErrorT = 0 + + // VoteErrorInternalError is returned when an internal server error + // occurred while attempting to cast a vote. + VoteErrorInternalError VoteErrorT = 1 + + // VoteErrorTokenInvalid is returned when a cast vote token is an + // invalid record token. + VoteErrorTokenInvalid VoteErrorT = 2 + + // VoteErrorRecordNotFound is returned when a cast vote token does + // not/correspond to a record. + VoteErrorRecordNotFound VoteErrorT = 3 + + // VoteErrorMultipleRecordVotes is returned when a ballot contains + // cast votes for multiple records. A ballot can only contain votes + // for a single record at a time. + VoteErrorMultipleRecordVotes VoteErrorT = 4 + + // VoteStatusInvalid is returned when a vote is cast on a record + // that is not being actively voted on. + VoteErrorVoteStatusInvalid VoteErrorT = 5 + + // VoteErrorVoteBitInvalid is returned when a cast vote's vote bit + // is not a valid vote option. + VoteErrorVoteBitInvalid VoteErrorT = 6 + + // VoteErrorSignatureInvalid is returned when a cast vote signature + // is invalid. + VoteErrorSignatureInvalid VoteErrorT = 7 + + // VoteErrorTicketNotEligible is returned when attempting to cast + // a vote using a dcr ticket that is not eligible. + VoteErrorTicketNotEligible VoteErrorT = 8 + + // VoteErrorTicketAlreadyVoted is returned when attempting to cast + // a vote using a dcr ticket that has already voted. + VoteErrorTicketAlreadyVoted VoteErrorT = 9 +) + +// CastVote is a signed ticket vote. +type CastVote struct { + Token string `json:"token"` // Record token + Ticket string `json:"ticket"` // Ticket ID + VoteBit string `json:"votebits"` // Selected vote bit, hex encoded + Signature string `json:"signature"` // Signature of Token+Ticket+VoteBit +} + +// CastVoteReply contains the receipt for the cast vote. +type CastVoteReply struct { + Ticket string `json:"ticket"` // Ticket ID + Receipt string `json:"receipt"` // Server signature of client signature + + // The follwing fields will only be present if an error occurred + // while attempting to cast the vote. + ErrorCode VoteErrorT `json:"errorcode,omitempty"` + ErrorContext string `json:"errorcontext,omitempty"` +} + +// CastBallot casts a ballot of votes. A ballot can only contain the votes for +// a single record. +type CastBallot struct { + Votes []CastVote `json:"votes"` +} + +// CastBallotReply is a reply to a batched list of votes. +type CastBallotReply struct { + Receipts []CastVoteReply `json:"receipts"` +} + +// AuthDetails contains the details of a vote authorization. +// +// Signature is the client signature of the Token+Version+Action. +type AuthDetails struct { + Token string `json:"token"` // Record token + Version uint32 `json:"version"` // Record version + Action string `json:"action"` // Authorization or revoke + PublicKey string `json:"publickey"` // Public key used for signature + Signature string `json:"signature"` // Client signature + Timestamp int64 `json:"timestamp"` // Server timestamp + Receipt string `json:"receipt"` // Server sig of client sig +} + +// VoteDetails contains the details of a record vote. +// +// Signature is the client signature of the SHA256 digest of the JSON encoded +// VoteParams struct. +type VoteDetails struct { + Params VoteParams `json:"params"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` + StartBlockHeight uint32 `json:"startblockheight"` + StartBlockHash string `json:"startblockhash"` + EndBlockHeight uint32 `json:"endblockheight"` + EligibleTickets []string `json:"eligibletickets"` // Ticket hashes +} + +// Details requests the vote details for a record vote. +type Details struct { + Token string `json:"token"` +} + +// DetailsReply is the reply to the Details command. +type DetailsReply struct { + Auths []AuthDetails `json:"auths"` + Vote VoteDetails `json:"vote"` +} + +// CastVoteDetails contains the details of a cast vote. +// +// Signature is the client signature of the Token+Ticket+VoteBit. +type CastVoteDetails struct { + Token string `json:"token"` // Record token + Ticket string `json:"ticket"` // Ticket hash + VoteBit string `json:"votebits"` // Selected vote bit, hex encoded + Signature string `json:"signature"` // Client signature + Receipt string `json:"receipt"` // Server sig of client sig +} + +// Results returns the cast votes for a record. +type Results struct { + Token string `json:"token"` +} + +// ResultsReply is the reply to the Results command. +type ResultsReply struct { + Votes []CastVoteDetails `json:"votes"` +} + +// VoteResult describes a vote option and the total number of votes that have +// been cast for this option. +type VoteResult struct { + ID string `json:"id"` // Single unique word (e.g. yes) + Description string `json:"description"` // Longer description of the vote + VoteBit uint64 `json:"votebit"` // Bits used for this option + Votes uint64 `json:"votes"` // Votes cast for this option +} + +// Summary summarizes the vote params and results of a record vote. +type Summary struct { + Type VoteT `json:"type"` + Status VoteStatusT `json:"status"` + Duration uint32 `json:"duration"` // In blocks + StartBlockHeight uint32 `json:"startblockheight"` + StartBlockHash string `json:"startblockhash"` + EndBlockHeight uint32 `json:"endblockheight"` + + // EligibleTickets is the number of tickets that are eligible to + // cast a vote. + EligibleTickets uint32 `json:"eligibletickets"` + + // QuorumPercentage is the percent of eligible tickets required to + // vote in order to have a quorum. + QuorumPercentage uint32 `json:"quorumpercentage"` + + // PassPercentage is the percent of total votes required to approve + // the vote in order for the vote to pass. + PassPercentage uint32 `json:"passpercentage"` + + Results []VoteResult `json:"results"` + Approved bool `json:"approved"` // Was the vote approved + + // BestBlock is the best block value that was used to prepare the + // summary. + BestBlock uint32 `json:"bestblock"` +} + +// Summaries requests the vote summaries for the provided record tokens. +type Summaries struct { + Tokens []string `json:"tokens"` +} + +// SummariesReply is the reply to the Summaries command. +// +// Summaries field contains a vote summary for each of the provided tokens. The +// map will not contain an entry for any tokens that did not correspond to an +// actual record. It is the callers responsibility to ensure that a summary is +// returned for all provided tokens. +type SummariesReply struct { + Summaries map[string]Summary `json:"summaries"` // [token]Summary +} + +// Inventory requests the record inventory categorized by vote status. +type Inventory struct{} + +// InventoryReply is the reply to the Inventory command. The returned map is +// a map[votestatus][]token where the votestatus key is the human readable vote +// status defined by the VoteStatuses array in this package. +type InventoryReply struct { + Records map[string][]string `json:"records"` +} + +// LinkedFrom requests the linked from list for a record. The only records that +// will have a linked from list are the parent records in a runoff vote. The +// linked from list will contain all runoff vote submissions, i.e. records that +// linked to the runoff parent record using the VoteMetadata.LinkTo field. +type LinkedFrom struct { + Tokens []string `json:"tokens"` +} + +// LinkedFromReply is the reply to the LinkedFrom command. If a provided token +// does not correspond to a record then it will not be included in the returned +// map. +type LinkedFromReply struct { + LinkedFrom map[string][]string `json:"linkedfrom"` +} + // Proof contains an inclusion proof for the digest in the merkle root. The // ExtraData field is used by certain types of proofs to include additional // data that is required to validate the proof. diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 3c5cc3226..1ad0be820 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -1633,7 +1633,7 @@ func (p *politeiawww) handleProposalNew(w http.ResponseWriter, r *http.Request) user, err := p.sessions.GetSessionUser(w, r) if err != nil { respondWithPiError(w, r, - "handleProposalNew: getSessionUser: %v", err) + "handleProposalNew: GetSessionUser: %v", err) return } @@ -1663,7 +1663,7 @@ func (p *politeiawww) handleProposalEdit(w http.ResponseWriter, r *http.Request) user, err := p.sessions.GetSessionUser(w, r) if err != nil { respondWithPiError(w, r, - "handleProposalEdit: getSessionUser: %v", err) + "handleProposalEdit: GetSessionUser: %v", err) return } @@ -1693,7 +1693,7 @@ func (p *politeiawww) handleProposalSetStatus(w http.ResponseWriter, r *http.Req usr, err := p.sessions.GetSessionUser(w, r) if err != nil { respondWithPiError(w, r, - "handleProposalSetStatus: getSessionUser: %v", err) + "handleProposalSetStatus: GetSessionUser: %v", err) return } @@ -1725,7 +1725,7 @@ func (p *politeiawww) handleProposals(w http.ResponseWriter, r *http.Request) { usr, err := p.sessions.GetSessionUser(w, r) if err != nil && err != sessions.ErrSessionNotFound { respondWithPiError(w, r, - "handleProposals: getSessionUser: %v", err) + "handleProposals: GetSessionUser: %v", err) return } @@ -1758,7 +1758,7 @@ func (p *politeiawww) handleProposalInventory(w http.ResponseWriter, r *http.Req usr, err := p.sessions.GetSessionUser(w, r) if err != nil && err != sessions.ErrSessionNotFound { respondWithPiError(w, r, - "handleProposalInventory: getSessionUser: %v", err) + "handleProposalInventory: GetSessionUser: %v", err) return } @@ -1772,181 +1772,6 @@ func (p *politeiawww) handleProposalInventory(w http.ResponseWriter, r *http.Req util.RespondWithJSON(w, http.StatusOK, pir) } -func (p *politeiawww) handleVoteAuthorize(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleVoteAuthorize") - - var va piv1.VoteAuthorize - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&va); err != nil { - respondWithPiError(w, r, "handleVoteAuthorize: unmarshal", - piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusInputInvalid, - }) - return - } - - usr, err := p.sessions.GetSessionUser(w, r) - if err != nil { - respondWithPiError(w, r, - "handleVoteAuthorize: getSessionUser: %v", err) - return - } - - vr, err := p.processVoteAuthorize(r.Context(), va, *usr) - if err != nil { - respondWithPiError(w, r, - "handleVoteAuthorize: processVoteAuthorize: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, vr) -} - -func (p *politeiawww) handleVoteStart(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleVoteStart") - - var vs piv1.VoteStart - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&vs); err != nil { - respondWithPiError(w, r, "handleVoteStart: unmarshal", - piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusInputInvalid, - }) - return - } - - usr, err := p.sessions.GetSessionUser(w, r) - if err != nil { - respondWithPiError(w, r, - "handleVoteStart: getSessionUser: %v", err) - return - } - - vsr, err := p.processVoteStart(r.Context(), vs, *usr) - if err != nil { - respondWithPiError(w, r, - "handleVoteStart: processVoteStart: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, vsr) -} - -func (p *politeiawww) handleCastBallot(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleCastBallot") - - var vb piv1.CastBallot - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&vb); err != nil { - respondWithPiError(w, r, "handleCastBallot: unmarshal", - piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusInputInvalid, - }) - return - } - - vbr, err := p.processCastBallot(r.Context(), vb) - if err != nil { - respondWithPiError(w, r, - "handleCastBallot: processCastBallot: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, vbr) -} - -func (p *politeiawww) handleVotes(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleVotes") - - var v piv1.Votes - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&v); err != nil { - respondWithPiError(w, r, "handleVotes: unmarshal", - piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusInputInvalid, - }) - return - } - - vr, err := p.processVotes(r.Context(), v) - if err != nil { - respondWithPiError(w, r, - "handleVotes: processVotes: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, vr) -} - -func (p *politeiawww) handleVoteResultsPi(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleVoteResults") - - var vr piv1.VoteResults - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&vr); err != nil { - respondWithPiError(w, r, "handleVoteResults: unmarshal", - piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusInputInvalid, - }) - return - } - - vrr, err := p.processVoteResultsPi(r.Context(), vr) - if err != nil { - respondWithPiError(w, r, - "handleVoteResults: prcoessVoteResults: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, vrr) -} - -func (p *politeiawww) handleVoteSummaries(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleVoteSummaries") - - var vs piv1.VoteSummaries - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&vs); err != nil { - respondWithPiError(w, r, "handleVoteSummaries: unmarshal", - piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusInputInvalid, - }) - return - } - - vsr, err := p.processVoteSummaries(r.Context(), vs) - if err != nil { - respondWithPiError(w, r, "handleVoteSummaries: processVoteSummaries: %v", - err) - return - } - - util.RespondWithJSON(w, http.StatusOK, vsr) -} - -func (p *politeiawww) handleVoteInventory(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleVoteInventory") - - var vi piv1.VoteInventory - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&vi); err != nil { - respondWithPiError(w, r, "handleVoteInventory: unmarshal", - piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusInputInvalid, - }) - return - } - - vir, err := p.processVoteInventory(r.Context()) - if err != nil { - respondWithPiError(w, r, "handleVoteInventory: processVoteInventory: %v", - err) - return - } - - util.RespondWithJSON(w, http.StatusOK, vir) -} - // setPiRoutes sets the pi API routes. func (p *politeiawww) setPiRoutes(c *comments.Comments) { // Return a 404 when a route is not found @@ -2025,28 +1850,30 @@ func (p *politeiawww) setPiRoutes(c *comments.Comments) { cmv1.RouteTimestamps, c.HandleTimestamps, permissionPublic) - // Pi routes - vote - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteVoteAuthorize, p.handleVoteAuthorize, - permissionLogin) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteVoteStart, p.handleVoteStart, - permissionAdmin) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteCastBallot, p.handleCastBallot, - permissionPublic) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteVotes, p.handleVotes, - permissionPublic) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteVoteResults, p.handleVoteResults, - permissionPublic) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteVoteSummaries, p.handleVoteSummaries, - permissionPublic) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteVoteInventory, p.handleVoteInventory, - permissionPublic) + /* + // Voute routes + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteVoteAuthorize, p.handleVoteAuthorize, + permissionLogin) + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteVoteStart, p.handleVoteStart, + permissionAdmin) + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteCastBallot, p.handleCastBallot, + permissionPublic) + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteVotes, p.handleVotes, + permissionPublic) + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteVoteResults, p.handleVoteResults, + permissionPublic) + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteVoteSummaries, p.handleVoteSummaries, + permissionPublic) + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteVoteInventory, p.handleVoteInventory, + permissionPublic) + */ // Record routes p.addRoute(http.MethodPost, rcv1.APIRoute, diff --git a/politeiawww/ticketvote/ticketvote.go b/politeiawww/ticketvote/ticketvote.go new file mode 100644 index 000000000..428d61448 --- /dev/null +++ b/politeiawww/ticketvote/ticketvote.go @@ -0,0 +1,202 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package comments + +import ( + "encoding/json" + "net/http" + + pdclient "github.com/decred/politeia/politeiad/client" + tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" + "github.com/decred/politeia/politeiawww/config" + "github.com/decred/politeia/politeiawww/events" + "github.com/decred/politeia/politeiawww/sessions" + "github.com/decred/politeia/politeiawww/user" + "github.com/decred/politeia/util" +) + +// TicketVote is the context that handles the ticketvote API. +type TicketVote struct { + cfg *config.Config + politeiad *pdclient.Client + userdb user.Database + sessions sessions.Sessions + events *events.Manager +} + +func (p *politeiawww) handleAuthorize(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleAuthorize") + + var a tkv1.Authorize + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&va); err != nil { + respondWithError(w, r, "handleAuthorize: unmarshal", + tkv1.UserErrorReply{ + ErrorCode: tkv1.ErrorStatusInputInvalid, + }) + return + } + + u, err := p.sessions.GetSessionUser(w, r) + if err != nil { + respondWithError(w, r, + "handleAuthorize: GetSessionUser: %v", err) + return + } + + ar, err := p.processAuthorize(r.Context(), a, *u) + if err != nil { + respondWithError(w, r, + "handleAuthorize: processAuthorize: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, vr) +} + +func (p *politeiawww) HandleStart(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleStart") + + var vs piv1.VoteStart + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&vs); err != nil { + respondWithPiError(w, r, "HandleStart: unmarshal", + piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusInputInvalid, + }) + return + } + + usr, err := p.sessions.GetSessionUser(w, r) + if err != nil { + respondWithPiError(w, r, + "HandleStart: GetSessionUser: %v", err) + return + } + + vsr, err := p.processVoteStart(r.Context(), vs, *usr) + if err != nil { + respondWithPiError(w, r, + "HandleStart: processVoteStart: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, vsr) +} + +func (p *politeiawww) handleCastBallot(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleCastBallot") + + var vb piv1.CastBallot + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&vb); err != nil { + respondWithPiError(w, r, "handleCastBallot: unmarshal", + piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusInputInvalid, + }) + return + } + + vbr, err := p.processCastBallot(r.Context(), vb) + if err != nil { + respondWithPiError(w, r, + "handleCastBallot: processCastBallot: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, vbr) +} + +func (p *politeiawww) handleVotes(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleVotes") + + var v piv1.Votes + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&v); err != nil { + respondWithPiError(w, r, "handleVotes: unmarshal", + piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusInputInvalid, + }) + return + } + + vr, err := p.processVotes(r.Context(), v) + if err != nil { + respondWithPiError(w, r, + "handleVotes: processVotes: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, vr) +} + +func (p *politeiawww) handleVoteResultsPi(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleVoteResults") + + var vr piv1.VoteResults + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&vr); err != nil { + respondWithPiError(w, r, "handleVoteResults: unmarshal", + piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusInputInvalid, + }) + return + } + + vrr, err := p.processVoteResultsPi(r.Context(), vr) + if err != nil { + respondWithPiError(w, r, + "handleVoteResults: prcoessVoteResults: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, vrr) +} + +func (p *politeiawww) handleVoteSummaries(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleVoteSummaries") + + var vs piv1.VoteSummaries + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&vs); err != nil { + respondWithPiError(w, r, "handleVoteSummaries: unmarshal", + piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusInputInvalid, + }) + return + } + + vsr, err := p.processVoteSummaries(r.Context(), vs) + if err != nil { + respondWithPiError(w, r, "handleVoteSummaries: processVoteSummaries: %v", + err) + return + } + + util.RespondWithJSON(w, http.StatusOK, vsr) +} + +func (p *politeiawww) handleVoteInventory(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleVoteInventory") + + var vi piv1.VoteInventory + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&vi); err != nil { + respondWithPiError(w, r, "handleVoteInventory: unmarshal", + piv1.UserErrorReply{ + ErrorCode: piv1.ErrorStatusInputInvalid, + }) + return + } + + vir, err := p.processVoteInventory(r.Context()) + if err != nil { + respondWithPiError(w, r, "handleVoteInventory: processVoteInventory: %v", + err) + return + } + + util.RespondWithJSON(w, http.StatusOK, vir) +} From 943550136d729c91270361b38347018c7768af65 Mon Sep 17 00:00:00 2001 From: luke Date: Sat, 23 Jan 2021 20:12:04 -0600 Subject: [PATCH 250/449] Add ticketvote routes. --- politeiad/backend/tlogbe/plugins/pi/cmds.go | 4 +- .../backend/tlogbe/plugins/ticketvote/cmds.go | 37 +- .../tlogbe/plugins/ticketvote/summary.go | 12 +- politeiad/client/comments.go | 12 +- politeiad/client/ticketvote.go | 403 ++++++++++++++++ politeiad/plugins/ticketvote/ticketvote.go | 25 +- politeiawww/api/ticketvote/v1/v1.go | 27 +- politeiawww/api/www/v2/v2.go | 3 +- politeiawww/comments/events.go | 16 +- politeiawww/comments/process.go | 19 +- politeiawww/log.go | 4 + politeiawww/piwww.go | 414 +--------------- politeiawww/proposals.go | 47 +- politeiawww/ticketvote.go | 215 --------- politeiawww/ticketvote/error.go | 93 ++++ politeiawww/ticketvote/events.go | 30 ++ politeiawww/ticketvote/log.go | 25 + politeiawww/ticketvote/process.go | 451 ++++++++++++++++++ politeiawww/ticketvote/ticketvote.go | 213 ++++++--- politeiawww/www.go | 45 +- 20 files changed, 1239 insertions(+), 856 deletions(-) create mode 100644 politeiad/client/ticketvote.go delete mode 100644 politeiawww/ticketvote.go create mode 100644 politeiawww/ticketvote/error.go create mode 100644 politeiawww/ticketvote/events.go create mode 100644 politeiawww/ticketvote/log.go create mode 100644 politeiawww/ticketvote/process.go diff --git a/politeiad/backend/tlogbe/plugins/pi/cmds.go b/politeiad/backend/tlogbe/plugins/pi/cmds.go index 39e654f31..eda04823f 100644 --- a/politeiad/backend/tlogbe/plugins/pi/cmds.go +++ b/politeiad/backend/tlogbe/plugins/pi/cmds.go @@ -67,7 +67,7 @@ func (p *piPlugin) cmdProposalInv() (string, error) { return string(reply), nil } -func (p *piPlugin) voteSummary(token []byte) (*ticketvote.VoteSummary, error) { +func (p *piPlugin) voteSummary(token []byte) (*ticketvote.SummaryReply, error) { reply, err := p.backend.VettedPluginCmd(token, ticketvote.PluginID, ticketvote.CmdSummary, "") if err != nil { @@ -78,7 +78,7 @@ func (p *piPlugin) voteSummary(token []byte) (*ticketvote.VoteSummary, error) { if err != nil { return nil, err } - return &sr.Summary, nil + return &sr, nil } func (p *piPlugin) cmdVoteInv() (string, error) { diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go index b61510901..e90725513 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go @@ -470,7 +470,7 @@ func (p *ticketVotePlugin) voteOptionResults(token []byte, options []ticketvote. // voteSummariesForRunoff returns the vote summaries of all submissions in a // runoff vote. This should only be called once the vote has finished. -func (p *ticketVotePlugin) summariesForRunoff(parentToken string) (map[string]ticketvote.VoteSummary, error) { +func (p *ticketVotePlugin) summariesForRunoff(parentToken string) (map[string]ticketvote.SummaryReply, error) { // Get runoff vote details parent, err := tokenDecode(parentToken) if err != nil { @@ -491,12 +491,12 @@ func (p *ticketVotePlugin) summariesForRunoff(parentToken string) (map[string]ti // Verify submissions exist subs := rdr.Runoff.Submissions if len(subs) == 0 { - return map[string]ticketvote.VoteSummary{}, nil + return map[string]ticketvote.SummaryReply{}, nil } // Compile summaries for all submissions var ( - summaries = make(map[string]ticketvote.VoteSummary, len(subs)) + summaries = make(map[string]ticketvote.SummaryReply, len(subs)) winnerNetApprove int // Net number of approve votes of the winner winnerToken string // Token of the winner ) @@ -519,7 +519,7 @@ func (p *ticketVotePlugin) summariesForRunoff(parentToken string) (map[string]ti } // Save summary - s := ticketvote.VoteSummary{ + s := ticketvote.SummaryReply{ Type: vd.Params.Type, Status: ticketvote.VoteStatusFinished, Duration: vd.Params.Duration, @@ -583,7 +583,7 @@ func (p *ticketVotePlugin) summariesForRunoff(parentToken string) (map[string]ti } // summary returns the vote summary for a record. -func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) (*ticketvote.VoteSummary, error) { +func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) (*ticketvote.SummaryReply, error) { // Check if the summary has been cached s, err := p.summaryCache(hex.EncodeToString(token)) switch { @@ -618,9 +618,10 @@ func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) case ticketvote.AuthActionRevoke: // Vote authorization has been revoked. Its not possible for // the vote to have been started. We can stop looking. - return &ticketvote.VoteSummary{ - Status: status, - Results: []ticketvote.VoteOptionResult{}, + return &ticketvote.SummaryReply{ + Status: status, + Results: []ticketvote.VoteOptionResult{}, + BestBlock: bestBlock, }, nil } } @@ -632,9 +633,10 @@ func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) } if vd == nil { // Vote has not been started yet - return &ticketvote.VoteSummary{ - Status: status, - Results: []ticketvote.VoteOptionResult{}, + return &ticketvote.SummaryReply{ + Status: status, + Results: []ticketvote.VoteOptionResult{}, + BestBlock: bestBlock, }, nil } @@ -653,7 +655,7 @@ func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) } // Prepare summary - summary := ticketvote.VoteSummary{ + summary := ticketvote.SummaryReply{ Type: vd.Params.Type, Status: status, Duration: vd.Params.Duration, @@ -664,6 +666,7 @@ func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) QuorumPercentage: vd.Params.QuorumPercentage, PassPercentage: vd.Params.PassPercentage, Results: results, + BestBlock: bestBlock, } // If the vote has not finished yet then we are done for now. @@ -716,7 +719,7 @@ func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) return &summary, nil } -func (p *ticketVotePlugin) summaryByToken(token []byte) (*ticketvote.VoteSummary, error) { +func (p *ticketVotePlugin) summaryByToken(token []byte) (*ticketvote.SummaryReply, error) { reply, err := p.backend.VettedPluginCmd(token, ticketvote.PluginID, ticketvote.CmdSummary, "") if err != nil { @@ -728,7 +731,7 @@ func (p *ticketVotePlugin) summaryByToken(token []byte) (*ticketvote.VoteSummary if err != nil { return nil, err } - return &sr.Summary, nil + return &sr, nil } func (p *ticketVotePlugin) timestamp(treeID int64, digest []byte) (*ticketvote.Timestamp, error) { @@ -2554,16 +2557,12 @@ func (p *ticketVotePlugin) cmdSummary(treeID int64, token []byte, payload string } // Get summary - s, err := p.summary(treeID, token, bb) + sr, err := p.summary(treeID, token, bb) if err != nil { return "", fmt.Errorf("summary: %v", err) } // Prepare reply - sr := ticketvote.SummaryReply{ - Summary: *s, - BestBlock: bb, - } reply, err := json.Marshal(sr) if err != nil { return "", err diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/summary.go b/politeiad/backend/tlogbe/plugins/ticketvote/summary.go index d5d3ba0ea..78a86f9d0 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/summary.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/summary.go @@ -40,7 +40,7 @@ var ( errSummaryNotFound = errors.New("summary not found") ) -func (p *ticketVotePlugin) summaryCache(token string) (*ticketvote.VoteSummary, error) { +func (p *ticketVotePlugin) summaryCache(token string) (*ticketvote.SummaryReply, error) { p.Lock() defer p.Unlock() @@ -58,17 +58,17 @@ func (p *ticketVotePlugin) summaryCache(token string) (*ticketvote.VoteSummary, return nil, err } - var vs ticketvote.VoteSummary - err = json.Unmarshal(b, &vs) + var sr ticketvote.SummaryReply + err = json.Unmarshal(b, &sr) if err != nil { return nil, err } - return &vs, nil + return &sr, nil } -func (p *ticketVotePlugin) summaryCacheSave(token string, vs ticketvote.VoteSummary) error { - b, err := json.Marshal(vs) +func (p *ticketVotePlugin) summaryCacheSave(token string, sr ticketvote.SummaryReply) error { + b, err := json.Marshal(sr) if err != nil { return err } diff --git a/politeiad/client/comments.go b/politeiad/client/comments.go index 5aaa69589..449f7d1e8 100644 --- a/politeiad/client/comments.go +++ b/politeiad/client/comments.go @@ -14,7 +14,7 @@ import ( ) // CommentNew sends the comments plugin New command to the politeiad v1 API. -func (c *Client) CommentNew(ctx context.Context, state, token string, n comments.New) (*comments.NewReply, error) { +func (c *Client) CommentNew(ctx context.Context, state string, n comments.New) (*comments.NewReply, error) { // Setup request b, err := json.Marshal(n) if err != nil { @@ -23,7 +23,7 @@ func (c *Client) CommentNew(ctx context.Context, state, token string, n comments cmds := []pdv1.PluginCommandV2{ { State: state, - Token: token, + Token: n.Token, ID: comments.PluginID, Command: comments.CmdNew, Payload: string(b), @@ -54,7 +54,7 @@ func (c *Client) CommentNew(ctx context.Context, state, token string, n comments } // CommentVote sends the comments plugin Vote command to the politeiad v1 API. -func (c *Client) CommentVote(ctx context.Context, state, token string, v comments.Vote) (*comments.VoteReply, error) { +func (c *Client) CommentVote(ctx context.Context, state string, v comments.Vote) (*comments.VoteReply, error) { // Setup request b, err := json.Marshal(v) if err != nil { @@ -63,7 +63,7 @@ func (c *Client) CommentVote(ctx context.Context, state, token string, v comment cmds := []pdv1.PluginCommandV2{ { State: state, - Token: token, + Token: v.Token, ID: comments.PluginID, Command: comments.CmdVote, Payload: string(b), @@ -94,7 +94,7 @@ func (c *Client) CommentVote(ctx context.Context, state, token string, v comment } // CommentDel sends the comments plugin Del command to the politeiad v1 API. -func (c *Client) CommentDel(ctx context.Context, state, token string, d comments.Del) (*comments.DelReply, error) { +func (c *Client) CommentDel(ctx context.Context, state string, d comments.Del) (*comments.DelReply, error) { // Setup request b, err := json.Marshal(d) if err != nil { @@ -103,7 +103,7 @@ func (c *Client) CommentDel(ctx context.Context, state, token string, d comments cmds := []pdv1.PluginCommandV2{ { State: state, - Token: token, + Token: d.Token, ID: comments.PluginID, Command: comments.CmdDel, Payload: string(b), diff --git a/politeiad/client/ticketvote.go b/politeiad/client/ticketvote.go new file mode 100644 index 000000000..263d60288 --- /dev/null +++ b/politeiad/client/ticketvote.go @@ -0,0 +1,403 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package client + +import ( + "context" + "encoding/json" + "fmt" + + pdv1 "github.com/decred/politeia/politeiad/api/v1" + "github.com/decred/politeia/politeiad/plugins/ticketvote" +) + +// TicketVoteAuthorize sends the ticketvote plugin Authorize command to the +// politeiad v1 API. +func (c *Client) TicketVoteAuthorize(ctx context.Context, a ticketvote.Authorize) (*ticketvote.AuthorizeReply, error) { + // Setup request + b, err := json.Marshal(a) + if err != nil { + return nil, err + } + cmds := []pdv1.PluginCommandV2{ + { + State: pdv1.RecordStateVetted, + Token: a.Token, + ID: ticketvote.PluginID, + Command: ticketvote.CmdAuthorize, + Payload: string(b), + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + if pcr.Error != nil { + return nil, pcr.Error + } + + // Decode reply + var ar ticketvote.AuthorizeReply + err = json.Unmarshal([]byte(pcr.Payload), &ar) + if err != nil { + return nil, err + } + + return &ar, nil +} + +// TicketVoteStart sends the ticketvote plugin Start command to the politeiad +// v1 API. +func (c *Client) TicketVoteStart(ctx context.Context, token string, s ticketvote.Start) (*ticketvote.StartReply, error) { + // Setup request + b, err := json.Marshal(s) + if err != nil { + return nil, err + } + cmds := []pdv1.PluginCommandV2{ + { + State: pdv1.RecordStateVetted, + Token: token, + ID: ticketvote.PluginID, + Command: ticketvote.CmdStart, + Payload: string(b), + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + if pcr.Error != nil { + return nil, pcr.Error + } + + // Decode reply + var sr ticketvote.StartReply + err = json.Unmarshal([]byte(pcr.Payload), &sr) + if err != nil { + return nil, err + } + + return &sr, nil +} + +// TicketVoteCastBallot sends the ticketvote plugin CastBallot command to the +// politeiad v1 API. +func (c *Client) TicketVoteCastBallot(ctx context.Context, token string, cb ticketvote.CastBallot) (*ticketvote.CastBallotReply, error) { + // Setup request + b, err := json.Marshal(cb) + if err != nil { + return nil, err + } + cmds := []pdv1.PluginCommandV2{ + { + State: pdv1.RecordStateVetted, + Token: token, + ID: ticketvote.PluginID, + Command: ticketvote.CmdCastBallot, + Payload: string(b), + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + if pcr.Error != nil { + return nil, pcr.Error + } + + // Decode reply + var cbr ticketvote.CastBallotReply + err = json.Unmarshal([]byte(pcr.Payload), &cbr) + if err != nil { + return nil, err + } + + return &cbr, nil +} + +// TicketVoteDetails sends the ticketvote plugin Details command to the +// politeiad v1 API. +func (c *Client) TicketVoteDetails(ctx context.Context, token string) (*ticketvote.DetailsReply, error) { + // Setup request + cmds := []pdv1.PluginCommandV2{ + { + State: pdv1.RecordStateVetted, + Token: token, + ID: ticketvote.PluginID, + Command: ticketvote.CmdDetails, + Payload: "", + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + if pcr.Error != nil { + return nil, pcr.Error + } + + // Decode reply + var dr ticketvote.DetailsReply + err = json.Unmarshal([]byte(pcr.Payload), &dr) + if err != nil { + return nil, err + } + + return &dr, nil +} + +// TicketVoteResults sends the ticketvote plugin Results command to the +// politeiad v1 API. +func (c *Client) TicketVoteResults(ctx context.Context, token string) (*ticketvote.ResultsReply, error) { + // Setup request + cmds := []pdv1.PluginCommandV2{ + { + State: pdv1.RecordStateVetted, + Token: token, + ID: ticketvote.PluginID, + Command: ticketvote.CmdResults, + Payload: "", + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + if pcr.Error != nil { + return nil, pcr.Error + } + + // Decode reply + var rr ticketvote.ResultsReply + err = json.Unmarshal([]byte(pcr.Payload), &rr) + if err != nil { + return nil, err + } + + return &rr, nil +} + +// TicketVoteSummary sends the ticketvote plugin Summary command to the +// politeiad v1 API. +func (c *Client) TicketVoteSummary(ctx context.Context, token string) (*ticketvote.SummaryReply, error) { + // Setup request + cmds := []pdv1.PluginCommandV2{ + { + State: pdv1.RecordStateVetted, + ID: ticketvote.PluginID, + Command: ticketvote.CmdSummary, + Token: token, + Payload: "", + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + if pcr.Error != nil { + return nil, pcr.Error + } + + // Decode reply + var sr ticketvote.SummaryReply + err = json.Unmarshal([]byte(pcr.Payload), &sr) + if err != nil { + return nil, err + } + + return &sr, nil +} + +// TicketVoteSummaries sends a batch of ticketvote plugin Summary commands to +// the politeiad v1 API. Individual summary errors are not returned, the token +// will simply be left out of the returned map. +func (c *Client) TicketVoteSummaries(ctx context.Context, tokens []string) (map[string]ticketvote.SummaryReply, error) { + // Setup request + cmds := make([]pdv1.PluginCommandV2, 0, len(tokens)) + for _, v := range tokens { + cmds = append(cmds, pdv1.PluginCommandV2{ + State: pdv1.RecordStateVetted, + Token: v, + ID: ticketvote.PluginID, + Command: ticketvote.CmdSummary, + Payload: "", + }) + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + + // Prepare reply + summaries := make(map[string]ticketvote.SummaryReply, len(replies)) + for _, v := range replies { + if v.Error != nil { + // Individual summary errors are ignored. The token will not + // be included in the returned summaries map. + continue + } + var sr ticketvote.SummaryReply + err = json.Unmarshal([]byte(v.Payload), &sr) + if err != nil { + return nil, err + } + summaries[v.Token] = sr + } + + return summaries, nil +} + +// TicketVoteLinkedFrom sends a batch of ticketvote plugin LinkedFrom commands +// to the politeiad v1 API. Individual record errors are not returned, the +// token will simply be left out of the returned map. +func (c *Client) TicketVoteLinkedFrom(ctx context.Context, tokens []string) (map[string][]string, error) { + // Setup request + cmds := make([]pdv1.PluginCommandV2, 0, len(tokens)) + for _, v := range tokens { + cmds = append(cmds, pdv1.PluginCommandV2{ + State: pdv1.RecordStateVetted, + Token: v, + ID: ticketvote.PluginID, + Command: ticketvote.CmdLinkedFrom, + Payload: "", + }) + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + + // Prepare reply + linkedFrom := make(map[string][]string, len(replies)) + for _, v := range replies { + if v.Error != nil { + // Individual record errors are ignored. The token will not be + // included in the returned linkedFrom map. + continue + } + var lfr ticketvote.LinkedFromReply + err = json.Unmarshal([]byte(v.Payload), &lfr) + if err != nil { + return nil, err + } + linkedFrom[v.Token] = lfr.Tokens + } + + return linkedFrom, nil +} + +// TicketVoteInventory sends the ticketvote plugin Inventory command to the +// politeiad v1 API. +func (c *Client) TicketVoteInventory(ctx context.Context) (*ticketvote.InventoryReply, error) { + // Setup request + cmds := []pdv1.PluginCommandV2{ + { + State: pdv1.RecordStateVetted, + ID: ticketvote.PluginID, + Command: ticketvote.CmdInventory, + Payload: "", + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + if pcr.Error != nil { + return nil, pcr.Error + } + + // Decode reply + var ir ticketvote.InventoryReply + err = json.Unmarshal([]byte(pcr.Payload), &ir) + if err != nil { + return nil, err + } + + return &ir, nil +} + +// TicketVoteTimestamps sends the ticketvote plugin Timestamps command to the +// politeiad v1 API. +func (c *Client) TicketVoteTimestamps(ctx context.Context, token string) (*ticketvote.TimestampsReply, error) { + // Setup request + cmds := []pdv1.PluginCommandV2{ + { + State: pdv1.RecordStateVetted, + ID: ticketvote.PluginID, + Command: ticketvote.CmdTimestamps, + Token: token, + Payload: "", + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + if pcr.Error != nil { + return nil, pcr.Error + } + + // Decode reply + var sr ticketvote.TimestampsReply + err = json.Unmarshal([]byte(pcr.Payload), &sr) + if err != nil { + return nil, err + } + + return &sr, nil +} diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index 66f15fa8f..60863e5c6 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -6,9 +6,7 @@ // tickets to participate. package ticketvote -// TODO VoteDetails, StartReply, StartRunoffReply should contain a receipt. -// The receipt should be the server signature of Signature+StartBlockHash. -// TODO Update politeiavoter +// TODO should VoteDetails, StartReply, StartRunoffReply contain a receipt? // TODO the timestamps reply is going to be too large. Each ticket vote // timestamp is ~2000 bytes. // Avg (15k votes): 30MB @@ -25,9 +23,9 @@ const ( CmdDetails = "details" // Get vote details CmdResults = "results" // Get vote results CmdSummary = "summary" // Get vote summary + CmdLinkedFrom = "linkedfrom" // Get linked from list CmdInventory = "inventory" // Get inventory by vote status CmdTimestamps = "timestamps" // Get vote data timestamps - CmdLinkedFrom = "linkedfrom" // Get record linked from list // Plugin setting keys SettingKeyVoteDurationMin = "votedurationmin" @@ -418,8 +416,11 @@ type VoteOptionResult struct { Votes uint64 `json:"votes"` // Votes cast for this option } -// VoteSummary contains a summary of a record vote. -type VoteSummary struct { +// Summary requests the vote summaries for a record. +type Summary struct{} + +// SummaryReply is the reply to the Summary command. +type SummaryReply struct { Type VoteT `json:"type"` Status VoteStatusT `json:"status"` Duration uint32 `json:"duration"` @@ -436,17 +437,9 @@ type VoteSummary struct { // VoteTypeRunoff, both of which only allow for approve/reject // voting options. Approved bool `json:"approved,omitempty"` -} - -// Summary requests the vote summaries for a record. -type Summary struct{} - -// SummaryReply is the reply to the Summary command. -type SummaryReply struct { - Summary VoteSummary `json:"summary"` - // BestBlock is the best block value that was used to - // prepare this summary. + // BestBlock is the best block value that was used to prepare this + // summary. BestBlock uint32 `json:"bestblock"` } diff --git a/politeiawww/api/ticketvote/v1/v1.go b/politeiawww/api/ticketvote/v1/v1.go index 3944b8703..95da2f415 100644 --- a/politeiawww/api/ticketvote/v1/v1.go +++ b/politeiawww/api/ticketvote/v1/v1.go @@ -16,8 +16,8 @@ const ( RouteDetails = "/details" RouteResults = "/results" RouteSummaries = "/summaries" - RouteInventory = "/inventory" RouteLinkedFrom = "/linkedfrom" + RouteInventory = "/inventory" RouteTimestamps = "/timestamps" ) @@ -28,6 +28,9 @@ const ( // Error codes ErrorCodeInvalid ErrorCodeT = 0 ErrorCodeInputInvalid ErrorCodeT = 1 + + ErrorCodePublicKeyInvalid ErrorCodeT = iota + ErrorCodeUnauthorized ) var ( @@ -377,7 +380,7 @@ type Details struct { // DetailsReply is the reply to the Details command. type DetailsReply struct { Auths []AuthDetails `json:"auths"` - Vote VoteDetails `json:"vote"` + Vote *VoteDetails `json:"vote"` } // CastVoteDetails contains the details of a cast vote. @@ -454,16 +457,6 @@ type SummariesReply struct { Summaries map[string]Summary `json:"summaries"` // [token]Summary } -// Inventory requests the record inventory categorized by vote status. -type Inventory struct{} - -// InventoryReply is the reply to the Inventory command. The returned map is -// a map[votestatus][]token where the votestatus key is the human readable vote -// status defined by the VoteStatuses array in this package. -type InventoryReply struct { - Records map[string][]string `json:"records"` -} - // LinkedFrom requests the linked from list for a record. The only records that // will have a linked from list are the parent records in a runoff vote. The // linked from list will contain all runoff vote submissions, i.e. records that @@ -479,6 +472,16 @@ type LinkedFromReply struct { LinkedFrom map[string][]string `json:"linkedfrom"` } +// Inventory requests the record inventory categorized by vote status. +type Inventory struct{} + +// InventoryReply is the reply to the Inventory command. The returned map is +// a map[votestatus][]token where the votestatus key is the human readable vote +// status defined by the VoteStatuses array in this package. +type InventoryReply struct { + Records map[string][]string `json:"records"` +} + // Proof contains an inclusion proof for the digest in the merkle root. The // ExtraData field is used by certain types of proofs to include additional // data that is required to validate the proof. diff --git a/politeiawww/api/www/v2/v2.go b/politeiawww/api/www/v2/v2.go index ca7b2c4a3..ff89b959b 100644 --- a/politeiawww/api/www/v2/v2.go +++ b/politeiawww/api/www/v2/v2.go @@ -14,8 +14,7 @@ type VoteT int const ( APIVersion = 2 - // All routes in the package are NO LONGER SUPPORTED. The pi v1 API - // should be used instead. + // All routes in the package are NO LONGER SUPPORTED. RouteStartVote = "/vote/start" RouteStartVoteRunoff = "/vote/startrunoff" RouteVoteDetails = "/vote/{token:[A-z0-9]{64}}" diff --git a/politeiawww/comments/events.go b/politeiawww/comments/events.go index 53625e652..99a0a9e0e 100644 --- a/politeiawww/comments/events.go +++ b/politeiawww/comments/events.go @@ -4,17 +4,15 @@ package comments +import cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" + const ( - // EventTypeNew is the event that is emitted when a new comment is - // made. - EventTypeNew = "commentnew" + // EventTypeNew is emitted when a new comment is made. + EventTypeNew = "comments-new" ) -// EventNew is the event data that is emitted when a new comment is made. +// EventNew is the event data for EventTypeNew. type EventNew struct { - State string - Token string - CommentID uint32 - ParentID uint32 - Username string + State string + Comment cmv1.Comment } diff --git a/politeiawww/comments/process.go b/politeiawww/comments/process.go index 79f5d8492..46e1af27b 100644 --- a/politeiawww/comments/process.go +++ b/politeiawww/comments/process.go @@ -24,7 +24,7 @@ func (c *Comments) processNew(ctx context.Context, n cmv1.New, u user.User) (*cm // Verify user has paid registration paywall if !c.userHasPaid(u) { return nil, cmv1.UserErrorReply{ - // ErrorCode: cmv1.ErrorCodeUserRegistrationNotPaid, + // TODO ErrorCode: cmv1.ErrorCodeUserRegistrationNotPaid, } } } @@ -47,7 +47,7 @@ func (c *Comments) processNew(ctx context.Context, n cmv1.New, u user.User) (*cm } if u.ID.String() != authorID { return nil, cmv1.UserErrorReply{ - // ErrorCode: cmv1.ErrorCodeUnauthorized, + // TODO ErrorCode: cmv1.ErrorCodeUnauthorized, ErrorContext: "user is not author or admin", } } @@ -62,7 +62,7 @@ func (c *Comments) processNew(ctx context.Context, n cmv1.New, u user.User) (*cm PublicKey: n.PublicKey, Signature: n.Signature, } - cnr, err := c.politeiad.CommentNew(ctx, n.State, n.Token, cn) + cnr, err := c.politeiad.CommentNew(ctx, n.State, cn) if err != nil { return nil, err } @@ -74,11 +74,8 @@ func (c *Comments) processNew(ctx context.Context, n cmv1.New, u user.User) (*cm // Emit event c.events.Emit(EventTypeNew, EventNew{ - State: n.State, - Token: cm.Token, - CommentID: cm.CommentID, - ParentID: cm.ParentID, - Username: cm.Username, + State: n.State, + Comment: cm, }) return &cmv1.NewReply{ @@ -126,7 +123,7 @@ func (c *Comments) processVote(ctx context.Context, v cmv1.Vote, u user.User) (* PublicKey: v.PublicKey, Signature: v.Signature, } - vr, err := c.politeiad.CommentVote(ctx, v.State, v.Token, cv) + vr, err := c.politeiad.CommentVote(ctx, v.State, cv) if err != nil { return nil, err } @@ -158,7 +155,7 @@ func (c *Comments) processDel(ctx context.Context, d cmv1.Del, u user.User) (*cm PublicKey: d.PublicKey, Signature: d.Signature, } - cdr, err := c.politeiad.CommentDel(ctx, d.State, d.Token, cd) + cdr, err := c.politeiad.CommentDel(ctx, d.State, cd) if err != nil { return nil, err } @@ -213,7 +210,7 @@ func (c *Comments) processComments(ctx context.Context, cs cmv1.Comments, u *use } if !isAllowed { return nil, cmv1.UserErrorReply{ - // ErrorCode: cmv1.ErrorCodeUnauthorized, + // TODO ErrorCode: cmv1.ErrorCodeUnauthorized, ErrorContext: "user is not author or admin", } } diff --git a/politeiawww/log.go b/politeiawww/log.go index e44743397..cdbf2e612 100644 --- a/politeiawww/log.go +++ b/politeiawww/log.go @@ -14,6 +14,7 @@ import ( ghdb "github.com/decred/politeia/politeiawww/codetracker/github/database/cockroachdb" "github.com/decred/politeia/politeiawww/comments" "github.com/decred/politeia/politeiawww/sessions" + "github.com/decred/politeia/politeiawww/ticketvote" "github.com/decred/politeia/politeiawww/user/cockroachdb" "github.com/decred/politeia/politeiawww/user/localdb" "github.com/decred/politeia/wsdcrdata" @@ -56,6 +57,7 @@ var ( githubdbLog = backendLog.Logger("GHDB") sessionsLog = backendLog.Logger("SESS") commentsLog = backendLog.Logger("COMT") + ticketvoteLog = backendLog.Logger("TICK") ) // Initialize package-global logger variables. @@ -68,6 +70,7 @@ func init() { ghdb.UseLogger(githubdbLog) sessions.UseLogger(sessionsLog) comments.UseLogger(commentsLog) + ticketvote.UseLogger(ticketvoteLog) } // subsystemLoggers maps each subsystem identifier to its associated logger. @@ -80,6 +83,7 @@ var subsystemLoggers = map[string]slog.Logger{ "GHDB": githubdbLog, "SESS": sessionsLog, "COMT": commentsLog, + "TICK": ticketvoteLog, } // initLogRotator initializes the logging rotater to write logs to logFile and diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 1ad0be820..5dc493776 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -22,12 +22,10 @@ import ( pdv1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/plugins/pi" - "github.com/decred/politeia/politeiad/plugins/ticketvote" usermd "github.com/decred/politeia/politeiad/plugins/user" cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" - tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/comments" "github.com/decred/politeia/politeiawww/sessions" @@ -348,261 +346,6 @@ func convertProposalRecordFromPD(r pdv1.Record, state piv1.PropStateT) (*piv1.Pr }, nil } -func convertVoteAuthActionFromPi(a piv1.VoteAuthActionT) ticketvote.AuthActionT { - switch a { - case piv1.VoteAuthActionAuthorize: - return ticketvote.AuthActionAuthorize - case piv1.VoteAuthActionRevoke: - return ticketvote.AuthActionRevoke - default: - return ticketvote.AuthActionAuthorize - } -} - -func convertVoteAuthorizeFromPi(va piv1.VoteAuthorize) ticketvote.Authorize { - return ticketvote.Authorize{ - Token: va.Token, - Version: va.Version, - Action: convertVoteAuthActionFromPi(va.Action), - PublicKey: va.PublicKey, - Signature: va.Signature, - } -} - -func convertVoteAuthsFromPi(auths []piv1.VoteAuthorize) []ticketvote.Authorize { - a := make([]ticketvote.Authorize, 0, len(auths)) - for _, v := range auths { - a = append(a, convertVoteAuthorizeFromPi(v)) - } - return a -} - -func convertVoteTypeFromPi(t piv1.VoteT) ticketvote.VoteT { - switch t { - case piv1.VoteTypeStandard: - return ticketvote.VoteTypeStandard - case piv1.VoteTypeRunoff: - return ticketvote.VoteTypeRunoff - } - return ticketvote.VoteTypeInvalid -} - -func convertVoteParamsFromPi(v piv1.VoteParams) ticketvote.VoteParams { - tv := ticketvote.VoteParams{ - Token: v.Token, - Version: v.Version, - Type: convertVoteTypeFromPi(v.Type), - Mask: v.Mask, - Duration: v.Duration, - QuorumPercentage: v.QuorumPercentage, - PassPercentage: v.PassPercentage, - Parent: v.Parent, - } - // Convert vote options - vo := make([]ticketvote.VoteOption, 0, len(v.Options)) - for _, vi := range v.Options { - vo = append(vo, ticketvote.VoteOption{ - ID: vi.ID, - Description: vi.Description, - Bit: vi.Bit, - }) - } - tv.Options = vo - - return tv -} - -func convertStartDetailsFromPi(sd piv1.StartDetails) ticketvote.StartDetails { - return ticketvote.StartDetails{ - Params: convertVoteParamsFromPi(sd.Params), - PublicKey: sd.PublicKey, - Signature: sd.Signature, - } -} - -func convertVoteStartFromPi(vs piv1.VoteStart) ticketvote.Start { - starts := make([]ticketvote.StartDetails, 0, len(vs.Starts)) - for _, v := range vs.Starts { - starts = append(starts, convertStartDetailsFromPi(v)) - } - return ticketvote.Start{ - Starts: starts, - } -} - -func convertVoteStartsFromPi(starts []piv1.VoteStart) []ticketvote.Start { - s := make([]ticketvote.Start, 0, len(starts)) - for _, v := range starts { - s = append(s, convertVoteStartFromPi(v)) - } - return s -} - -func convertCastVotesFromPi(votes []piv1.CastVote) []ticketvote.CastVote { - cv := make([]ticketvote.CastVote, 0, len(votes)) - for _, v := range votes { - cv = append(cv, ticketvote.CastVote{ - Token: v.Token, - Ticket: v.Ticket, - VoteBit: v.VoteBit, - Signature: v.Signature, - }) - } - return cv -} - -func convertVoteErrorFromPlugin(e ticketvote.VoteErrorT) piv1.VoteErrorT { - switch e { - case ticketvote.VoteErrorInvalid: - return piv1.VoteErrorInvalid - case ticketvote.VoteErrorInternalError: - return piv1.VoteErrorInternalError - case ticketvote.VoteErrorRecordNotFound: - return piv1.VoteErrorRecordNotFound - case ticketvote.VoteErrorVoteBitInvalid: - return piv1.VoteErrorVoteBitInvalid - case ticketvote.VoteErrorVoteStatusInvalid: - return piv1.VoteErrorVoteStatusInvalid - case ticketvote.VoteErrorTicketAlreadyVoted: - return piv1.VoteErrorTicketAlreadyVoted - case ticketvote.VoteErrorTicketNotEligible: - return piv1.VoteErrorTicketNotEligible - default: - return piv1.VoteErrorInternalError - } -} - -func convertVoteTypeFromPlugin(t ticketvote.VoteT) piv1.VoteT { - switch t { - case ticketvote.VoteTypeStandard: - return piv1.VoteTypeStandard - case ticketvote.VoteTypeRunoff: - return piv1.VoteTypeRunoff - } - return piv1.VoteTypeInvalid - -} - -func convertVoteParamsFromPlugin(v ticketvote.VoteParams) piv1.VoteParams { - vp := piv1.VoteParams{ - Token: v.Token, - Version: v.Version, - Type: convertVoteTypeFromPlugin(v.Type), - Mask: v.Mask, - Duration: v.Duration, - QuorumPercentage: v.QuorumPercentage, - PassPercentage: v.PassPercentage, - } - vo := make([]piv1.VoteOption, 0, len(v.Options)) - for _, o := range v.Options { - vo = append(vo, piv1.VoteOption{ - ID: o.ID, - Description: o.Description, - Bit: o.Bit, - }) - } - vp.Options = vo - - return vp -} - -func convertCastVoteRepliesFromPlugin(replies []ticketvote.CastVoteReply) []piv1.CastVoteReply { - r := make([]piv1.CastVoteReply, 0, len(replies)) - for _, v := range replies { - r = append(r, piv1.CastVoteReply{ - Ticket: v.Ticket, - Receipt: v.Receipt, - ErrorCode: convertVoteErrorFromPlugin(v.ErrorCode), - ErrorContext: v.ErrorContext, - }) - } - return r -} - -func convertVoteDetailsFromPlugin(vd ticketvote.VoteDetails) piv1.VoteDetails { - return piv1.VoteDetails{ - Params: convertVoteParamsFromPlugin(vd.Params), - PublicKey: vd.PublicKey, - Signature: vd.Signature, - StartBlockHeight: vd.StartBlockHeight, - StartBlockHash: vd.StartBlockHash, - EndBlockHeight: vd.EndBlockHeight, - EligibleTickets: vd.EligibleTickets, - } -} - -func convertAuthDetailsFromPlugin(auths []ticketvote.AuthDetails) []piv1.AuthDetails { - a := make([]piv1.AuthDetails, 0, len(auths)) - for _, v := range auths { - a = append(a, piv1.AuthDetails{ - Token: v.Token, - Version: v.Version, - Action: v.Action, - PublicKey: v.PublicKey, - Signature: v.Signature, - Timestamp: v.Timestamp, - Receipt: v.Receipt, - }) - } - return a -} - -func convertCastVoteDetailsFromPlugin(votes []ticketvote.CastVoteDetails) []piv1.CastVoteDetails { - vs := make([]piv1.CastVoteDetails, 0, len(votes)) - for _, v := range votes { - vs = append(vs, piv1.CastVoteDetails{ - Token: v.Token, - Ticket: v.Ticket, - VoteBit: v.VoteBit, - Signature: v.Signature, - Receipt: v.Receipt, - }) - } - return vs -} - -func convertVoteStatusFromPlugin(s ticketvote.VoteStatusT) piv1.VoteStatusT { - switch s { - case ticketvote.VoteStatusInvalid: - return piv1.VoteStatusInvalid - case ticketvote.VoteStatusUnauthorized: - return piv1.VoteStatusUnauthorized - case ticketvote.VoteStatusAuthorized: - return piv1.VoteStatusAuthorized - case ticketvote.VoteStatusStarted: - return piv1.VoteStatusStarted - case ticketvote.VoteStatusFinished: - return piv1.VoteStatusFinished - default: - return piv1.VoteStatusInvalid - } -} - -func convertVoteSummaryFromPlugin(s ticketvote.VoteSummary) piv1.VoteSummary { - results := make([]piv1.VoteResult, 0, len(s.Results)) - for _, v := range s.Results { - results = append(results, piv1.VoteResult{ - ID: v.ID, - Description: v.Description, - VoteBit: v.VoteBit, - Votes: v.Votes, - }) - } - return piv1.VoteSummary{ - Type: convertVoteTypeFromPlugin(s.Type), - Status: convertVoteStatusFromPlugin(s.Status), - Duration: s.Duration, - StartBlockHeight: s.StartBlockHeight, - StartBlockHash: s.StartBlockHash, - EndBlockHeight: s.EndBlockHeight, - EligibleTickets: s.EligibleTickets, - QuorumPercentage: s.QuorumPercentage, - PassPercentage: s.PassPercentage, - Results: results, - Approved: s.Approved, - } -} - // proposalRecords returns the ProposalRecord for each of the provided proposal // requests. If a token does not correspond to an actual proposal then it will // not be included in the returned map. @@ -1452,153 +1195,6 @@ func (p *politeiawww) processProposalInventory(ctx context.Context, inv piv1.Pro }, nil } -func (p *politeiawww) processVoteAuthorize(ctx context.Context, va piv1.VoteAuthorize, usr user.User) (*piv1.VoteAuthorizeReply, error) { - log.Tracef("processVoteAuthorize: %v", va.Token) - - // Verify user signed with their active identity - if usr.PublicKey() != va.PublicKey { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusPublicKeyInvalid, - ErrorContext: "not active identity", - } - } - - // TODO Verify user is the proposal author. Hmmm I think the userID - // probably needs to be attached to the vote authorization. - - // Send plugin command - ar, err := p.voteAuthorize(ctx, convertVoteAuthorizeFromPi(va)) - if err != nil { - return nil, err - } - - // TODO Emit notification - - return &piv1.VoteAuthorizeReply{ - Timestamp: ar.Timestamp, - Receipt: ar.Receipt, - }, nil -} - -func (p *politeiawww) processVoteStart(ctx context.Context, vs piv1.VoteStart, usr user.User) (*piv1.VoteStartReply, error) { - log.Tracef("processVoteStart: %v", len(vs.Starts)) - - // Sanity check - if !usr.Admin { - return nil, fmt.Errorf("not an admin") - } - - // Verify there is work to be done - if len(vs.Starts) == 0 { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusStartDetailsInvalid, - ErrorContext: "no start details found", - } - } - - // Verify admin signed with their active identity - for _, v := range vs.Starts { - if usr.PublicKey() != v.PublicKey { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusPublicKeyInvalid, - ErrorContext: "not active identity", - } - } - } - - // Start vote - var ( - sr *ticketvote.StartReply - err error - ) - switch vs.Starts[0].Params.Type { - case piv1.VoteTypeStandard: - // A standard vote can be started using the ticketvote plugin - // directly. - sr, err = p.voteStart(ctx, convertVoteStartFromPi(vs)) - if err != nil { - return nil, err - } - - case piv1.VoteTypeRunoff: - // A runoff vote requires additional validation that is not part - // of the ticketvote plugin. We pass the ticketvote command - // through the pi plugin so that it can perform this additional - // validation. - s := convertVoteStartFromPi(vs) - _ = s - - default: - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusVoteTypeInvalid, - } - } - - // TODO Emit notification for each start - - return &piv1.VoteStartReply{ - StartBlockHeight: sr.StartBlockHeight, - StartBlockHash: sr.StartBlockHash, - EndBlockHeight: sr.EndBlockHeight, - EligibleTickets: sr.EligibleTickets, - }, nil -} - -func (p *politeiawww) processCastBallot(ctx context.Context, vc piv1.CastBallot) (*piv1.CastBallotReply, error) { - log.Tracef("processCastBallot") - - cb := ticketvote.CastBallot{ - Ballot: convertCastVotesFromPi(vc.Votes), - } - reply, err := p.castBallot(ctx, cb) - if err != nil { - return nil, err - } - - return &piv1.CastBallotReply{ - Receipts: convertCastVoteRepliesFromPlugin(reply.Receipts), - }, nil -} - -func (p *politeiawww) processVotes(ctx context.Context, v piv1.Votes) (*piv1.VotesReply, error) { - log.Tracef("processVotes: %v", v.Tokens) - - // TODO - - return nil, nil -} - -func (p *politeiawww) processVoteResultsPi(ctx context.Context, vr piv1.VoteResults) (*piv1.VoteResultsReply, error) { - log.Tracef("processVoteResults: %v", vr.Token) - - cvr, err := p.voteResults(ctx, vr.Token) - if err != nil { - return nil, err - } - - return &piv1.VoteResultsReply{ - Votes: convertCastVoteDetailsFromPlugin(cvr.Votes), - }, nil -} - -func (p *politeiawww) processVoteSummaries(ctx context.Context, vs piv1.VoteSummaries) (*piv1.VoteSummariesReply, error) { - log.Tracef("processVoteSummaries: %v", vs.Tokens) - - /* - r, err := p.voteSummaries(ctx, vs.Tokens) - if err != nil { - return nil, err - } - - return &piv1.VoteSummariesReply{ - Summaries: convertVoteSummariesFromPlugin(r.Summaries), - BestBlock: r.BestBlock, - }, nil - */ - - return nil, nil -} - func (p *politeiawww) processVoteInventory(ctx context.Context) (*piv1.VoteInventoryReply, error) { log.Tracef("processVoteInventory") @@ -1873,15 +1469,15 @@ func (p *politeiawww) setPiRoutes(c *comments.Comments) { p.addRoute(http.MethodPost, piv1.APIRoute, piv1.RouteVoteInventory, p.handleVoteInventory, permissionPublic) + + // Ticket vote routes + p.addRoute(http.MethodPost, tkv1.APIRoute, + tkv1.RouteTimestamps, p.handleTicketVoteTimestamps, + permissionPublic) */ // Record routes p.addRoute(http.MethodPost, rcv1.APIRoute, rcv1.RouteTimestamps, p.handleTimestamps, permissionPublic) - - // Ticket vote routes - p.addRoute(http.MethodPost, tkv1.APIRoute, - tkv1.RouteTimestamps, p.handleTicketVoteTimestamps, - permissionPublic) } diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 983fc06ee..5e8fafefe 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -242,17 +242,14 @@ func (p *politeiawww) processBatchProposals(ctx context.Context, bp www.BatchPro func (p *politeiawww) processVoteResults(ctx context.Context, token string) (*www.VoteResultsReply, error) { log.Tracef("processVoteResults: %v", token) - // Get vote details - vd, err := p.voteDetails(ctx, token) - if err != nil { - return nil, err - } + // TODO Get vote details + var vd ticketvote.VoteDetails // Convert to www - startHeight := strconv.FormatUint(uint64(vd.Vote.StartBlockHeight), 10) - endHeight := strconv.FormatUint(uint64(vd.Vote.EndBlockHeight), 10) - options := make([]www.VoteOption, 0, len(vd.Vote.Params.Options)) - for _, o := range vd.Vote.Params.Options { + startHeight := strconv.FormatUint(uint64(vd.StartBlockHeight), 10) + endHeight := strconv.FormatUint(uint64(vd.EndBlockHeight), 10) + options := make([]www.VoteOption, 0, len(vd.Params.Options)) + for _, o := range vd.Params.Options { options = append(options, www.VoteOption{ Id: o.ID, Description: o.Description, @@ -260,11 +257,8 @@ func (p *politeiawww) processVoteResults(ctx context.Context, token string) (*ww }) } - // Get cast votes - rr, err := p.voteResults(ctx, token) - if err != nil { - return nil, err - } + // TODO Get cast votes + var rr ticketvote.ResultsReply // Convert to www votes := make([]www.CastVote, 0, len(rr.Votes)) @@ -279,22 +273,22 @@ func (p *politeiawww) processVoteResults(ctx context.Context, token string) (*ww return &www.VoteResultsReply{ StartVote: www.StartVote{ - PublicKey: vd.Vote.PublicKey, - Signature: vd.Vote.Signature, + PublicKey: vd.PublicKey, + Signature: vd.Signature, Vote: www.Vote{ - Token: vd.Vote.Params.Token, - Mask: vd.Vote.Params.Mask, - Duration: vd.Vote.Params.Duration, - QuorumPercentage: vd.Vote.Params.QuorumPercentage, - PassPercentage: vd.Vote.Params.PassPercentage, + Token: vd.Params.Token, + Mask: vd.Params.Mask, + Duration: vd.Params.Duration, + QuorumPercentage: vd.Params.QuorumPercentage, + PassPercentage: vd.Params.PassPercentage, Options: options, }, }, StartVoteReply: www.StartVoteReply{ StartBlockHeight: startHeight, - StartBlockHash: vd.Vote.StartBlockHash, + StartBlockHash: vd.StartBlockHash, EndHeight: endHeight, - EligibleTickets: vd.Vote.EligibleTickets, + EligibleTickets: vd.EligibleTickets, }, CastVotes: votes, }, nil @@ -363,10 +357,9 @@ func (p *politeiawww) processCastVotes(ctx context.Context, ballot *www.Ballot) cb := ticketvote.CastBallot{ Ballot: votes, } - cbr, err := p.castBallot(ctx, cb) - if err != nil { - return nil, err - } + // TODO + _ = cb + var cbr ticketvote.CastBallotReply // Prepare reply receipts := make([]www.CastVoteReply, 0, len(cbr.Receipts)) diff --git a/politeiawww/ticketvote.go b/politeiawww/ticketvote.go deleted file mode 100644 index a3ca438b3..000000000 --- a/politeiawww/ticketvote.go +++ /dev/null @@ -1,215 +0,0 @@ -// Copyright (c) 2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "net/http" - "runtime/debug" - "strings" - "time" - - "github.com/decred/politeia/politeiad/plugins/ticketvote" - cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" - tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" - "github.com/decred/politeia/util" -) - -func convertProofFromTicketVotePlugin(p ticketvote.Proof) tkv1.Proof { - return tkv1.Proof{ - Type: p.Type, - Digest: p.Digest, - MerkleRoot: p.MerkleRoot, - MerklePath: p.MerklePath, - ExtraData: p.ExtraData, - } -} - -func convertTimestampFromTicketVotePlugin(t ticketvote.Timestamp) tkv1.Timestamp { - proofs := make([]tkv1.Proof, 0, len(t.Proofs)) - for _, v := range t.Proofs { - proofs = append(proofs, convertProofFromTicketVotePlugin(v)) - } - return tkv1.Timestamp{ - Data: t.Data, - Digest: t.Digest, - TxID: t.TxID, - MerkleRoot: t.MerkleRoot, - Proofs: proofs, - } -} - -func (p *politeiawww) voteAuthorize(ctx context.Context, a ticketvote.Authorize) (*ticketvote.AuthorizeReply, error) { - return nil, nil -} - -func (p *politeiawww) voteStart(ctx context.Context, s ticketvote.Start) (*ticketvote.StartReply, error) { - return nil, nil -} - -func (p *politeiawww) castBallot(ctx context.Context, tb ticketvote.CastBallot) (*ticketvote.CastBallotReply, error) { - return nil, nil -} - -func (p *politeiawww) voteDetails(ctx context.Context, token string) (*ticketvote.DetailsReply, error) { - return nil, nil -} - -func (p *politeiawww) voteResults(ctx context.Context, token string) (*ticketvote.ResultsReply, error) { - return nil, nil -} - -func (p *politeiawww) voteSummaries(ctx context.Context, tokens []string) (map[string]ticketvote.VoteSummary, error) { - - return nil, nil -} - -func (p *politeiawww) voteTimestamps(ctx context.Context, token string) (*ticketvote.TimestampsReply, error) { - _ = token - var b []byte - r, err := p.politeiad.PluginCommand(ctx, ticketvote.PluginID, - ticketvote.CmdTimestamps, string(b)) - if err != nil { - return nil, err - } - var tr ticketvote.TimestampsReply - err = json.Unmarshal([]byte(r), &tr) - if err != nil { - return nil, err - } - return &tr, nil -} - -func (p *politeiawww) processTicketVoteTimestamps(ctx context.Context, t tkv1.Timestamps) (*tkv1.TimestampsReply, error) { - log.Tracef("processTicketVoteTimestamps: %v", t.Token) - - // Send plugin command - r, err := p.voteTimestamps(ctx, t.Token) - if err != nil { - return nil, err - } - - // Prepare reply - var ( - auths = make([]tkv1.Timestamp, 0, len(r.Auths)) - votes = make(map[string]tkv1.Timestamp, len(r.Votes)) - - details = convertTimestampFromTicketVotePlugin(r.Details) - ) - for _, v := range r.Auths { - auths = append(auths, convertTimestampFromTicketVotePlugin(v)) - } - for k, v := range r.Votes { - votes[k] = convertTimestampFromTicketVotePlugin(v) - } - - return &tkv1.TimestampsReply{ - Auths: auths, - Details: details, - Votes: votes, - }, nil -} - -func (p *politeiawww) handleTicketVoteTimestamps(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleTicketVoteTimestamps") - - var t tkv1.Timestamps - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&t); err != nil { - respondWithTicketVoteError(w, r, "handleTicketVoteTimestamps: unmarshal", - cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodeInputInvalid, - }) - return - } - - tr, err := p.processTicketVoteTimestamps(r.Context(), t) - if err != nil { - respondWithTicketVoteError(w, r, - "handleTicketVoteTimestamps: processTicketVoteTimestamps: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, tr) -} - -func respondWithTicketVoteError(w http.ResponseWriter, r *http.Request, format string, err error) { - var ( - ue tkv1.UserErrorReply - pe pdError - ) - switch { - case errors.As(err, &ue): - // Ticket vote user error - m := fmt.Sprintf("Ticket vote user error: %v %v %v", - util.RemoteAddr(r), ue.ErrorCode, tkv1.ErrorCodes[ue.ErrorCode]) - if ue.ErrorContext != "" { - m += fmt.Sprintf(": %v", ue.ErrorContext) - } - log.Infof(m) - util.RespondWithJSON(w, http.StatusBadRequest, - tkv1.UserErrorReply{ - ErrorCode: ue.ErrorCode, - ErrorContext: ue.ErrorContext, - }) - return - - case errors.As(err, &pe): - // Politeiad error - var ( - pluginID = pe.ErrorReply.Plugin - errCode = pe.ErrorReply.ErrorCode - errContext = pe.ErrorReply.ErrorContext - ) - switch { - case pluginID != "": - // Politeiad plugin error. Log it and return a 400. - m := fmt.Sprintf("Plugin error: %v %v %v", - util.RemoteAddr(r), pluginID, errCode) - if len(errContext) > 0 { - m += fmt.Sprintf(": %v", strings.Join(errContext, ", ")) - } - log.Infof(m) - util.RespondWithJSON(w, http.StatusBadRequest, - tkv1.PluginErrorReply{ - PluginID: pluginID, - ErrorCode: errCode, - ErrorContext: strings.Join(errContext, ", "), - }) - return - - default: - // Unknown politeiad error. Log it and return a 500. - ts := time.Now().Unix() - log.Errorf("%v %v %v %v Internal error %v: error code "+ - "from politeiad: %v", util.RemoteAddr(r), r.Method, r.URL, - r.Proto, ts, errCode) - - util.RespondWithJSON(w, http.StatusInternalServerError, - tkv1.ServerErrorReply{ - ErrorCode: ts, - }) - return - } - - default: - // Internal server error. Log it and return a 500. - t := time.Now().Unix() - e := fmt.Sprintf(format, err) - log.Errorf("%v %v %v %v Internal error %v: %v", - util.RemoteAddr(r), r.Method, r.URL, r.Proto, t, e) - - log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) - - util.RespondWithJSON(w, http.StatusInternalServerError, - tkv1.ServerErrorReply{ - ErrorCode: t, - }) - return - } -} diff --git a/politeiawww/ticketvote/error.go b/politeiawww/ticketvote/error.go new file mode 100644 index 000000000..455fc7d8b --- /dev/null +++ b/politeiawww/ticketvote/error.go @@ -0,0 +1,93 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ticketvote + +import ( + "errors" + "fmt" + "net/http" + "runtime/debug" + "strings" + "time" + + pdclient "github.com/decred/politeia/politeiad/client" + tkv1 "github.com/decred/politeia/politeiawww/api/comments/v1" + "github.com/decred/politeia/util" +) + +func respondWithError(w http.ResponseWriter, r *http.Request, format string, err error) { + var ( + ue tkv1.UserErrorReply + pe pdclient.Error + ) + switch { + case errors.As(err, &ue): + // Ticketvote user error + m := fmt.Sprintf("Ticketvote user error: %v %v %v", + util.RemoteAddr(r), ue.ErrorCode, tkv1.ErrorCodes[ue.ErrorCode]) + if ue.ErrorContext != "" { + m += fmt.Sprintf(": %v", ue.ErrorContext) + } + log.Infof(m) + util.RespondWithJSON(w, http.StatusBadRequest, + tkv1.UserErrorReply{ + ErrorCode: ue.ErrorCode, + ErrorContext: ue.ErrorContext, + }) + return + + case errors.As(err, &pe): + // Politeiad error + var ( + pluginID = pe.ErrorReply.PluginID + errCode = pe.ErrorReply.ErrorCode + errContext = pe.ErrorReply.ErrorContext + ) + switch { + case pluginID != "": + // Politeiad plugin error. Log it and return a 400. + m := fmt.Sprintf("Plugin error: %v %v %v", + util.RemoteAddr(r), pluginID, errCode) + if len(errContext) > 0 { + m += fmt.Sprintf(": %v", strings.Join(errContext, ", ")) + } + log.Infof(m) + util.RespondWithJSON(w, http.StatusBadRequest, + tkv1.PluginErrorReply{ + PluginID: pluginID, + ErrorCode: errCode, + ErrorContext: strings.Join(errContext, ", "), + }) + return + + default: + // Unknown politeiad error. Log it and return a 500. + ts := time.Now().Unix() + log.Errorf("%v %v %v %v Internal error %v: error code "+ + "from politeiad: %v", util.RemoteAddr(r), r.Method, r.URL, + r.Proto, ts, errCode) + + util.RespondWithJSON(w, http.StatusInternalServerError, + tkv1.ServerErrorReply{ + ErrorCode: ts, + }) + return + } + + default: + // Internal server error. Log it and return a 500. + t := time.Now().Unix() + e := fmt.Sprintf(format, err) + log.Errorf("%v %v %v %v Internal error %v: %v", + util.RemoteAddr(r), r.Method, r.URL, r.Proto, t, e) + log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) + + util.RespondWithJSON(w, http.StatusInternalServerError, + tkv1.ServerErrorReply{ + ErrorCode: t, + }) + return + } +} diff --git a/politeiawww/ticketvote/events.go b/politeiawww/ticketvote/events.go new file mode 100644 index 000000000..4d6dd47e6 --- /dev/null +++ b/politeiawww/ticketvote/events.go @@ -0,0 +1,30 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ticketvote + +import ( + tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" + "github.com/decred/politeia/politeiawww/user" +) + +const ( + // EventTypeAuthorize is emitted when a vote is authorized. + EventTypeAuthorize = "ticketvote-authorize" + + // EventTypeStart is emitted when a vote is started. + EventTypeStart = "ticketvote-start" +) + +// EventAuthorize is the event data for EventTypeAuthorize. +type EventAuthorize struct { + Auth tkv1.Authorize + User user.User +} + +// EventStart is the event data for EventTypeStart. +type EventStart struct { + Start tkv1.Start + User user.User +} diff --git a/politeiawww/ticketvote/log.go b/politeiawww/ticketvote/log.go new file mode 100644 index 000000000..f775a3117 --- /dev/null +++ b/politeiawww/ticketvote/log.go @@ -0,0 +1,25 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ticketvote + +import "github.com/decred/slog" + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log = slog.Disabled + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = slog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using slog. +func UseLogger(logger slog.Logger) { + log = logger +} diff --git a/politeiawww/ticketvote/process.go b/politeiawww/ticketvote/process.go new file mode 100644 index 000000000..e8d317531 --- /dev/null +++ b/politeiawww/ticketvote/process.go @@ -0,0 +1,451 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ticketvote + +import ( + "context" + + pdv1 "github.com/decred/politeia/politeiad/api/v1" + "github.com/decred/politeia/politeiad/plugins/ticketvote" + tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" + "github.com/decred/politeia/politeiawww/user" +) + +func (t *TicketVote) processAuthorize(ctx context.Context, a tkv1.Authorize, u user.User) (*tkv1.AuthorizeReply, error) { + log.Tracef("processAuthorize: %v", a.Token) + + // Verify user signed with their active identity + if u.PublicKey() != a.PublicKey { + return nil, tkv1.UserErrorReply{ + ErrorCode: tkv1.ErrorCodePublicKeyInvalid, + ErrorContext: "not active identity", + } + } + + // Verify user is the record author + authorID, err := t.politeiad.Author(ctx, pdv1.RecordStateVetted, a.Token) + if err != nil { + return nil, err + } + if u.ID.String() != authorID { + return nil, tkv1.UserErrorReply{ + ErrorCode: tkv1.ErrorCodeUnauthorized, + ErrorContext: "user is not record author", + } + } + + // Send plugin command + ta := ticketvote.Authorize{ + Token: a.Token, + Version: a.Version, + Action: ticketvote.AuthActionT(a.Action), + PublicKey: a.PublicKey, + Signature: a.Signature, + } + tar, err := t.politeiad.TicketVoteAuthorize(ctx, ta) + if err != nil { + return nil, err + } + + // Emit event + t.events.Emit(EventTypeAuthorize, + EventAuthorize{ + Auth: a, + User: u, + }) + + return &tkv1.AuthorizeReply{ + Timestamp: tar.Timestamp, + Receipt: tar.Receipt, + }, nil +} + +func (t *TicketVote) processStart(ctx context.Context, s tkv1.Start, u user.User) (*tkv1.StartReply, error) { + log.Tracef("processStart: %v", len(s.Starts)) + + // Verify user signed with their active identity + for _, v := range s.Starts { + if u.PublicKey() != v.PublicKey { + return nil, tkv1.UserErrorReply{ + ErrorCode: tkv1.ErrorCodePublicKeyInvalid, + ErrorContext: "not active identity", + } + } + } + + // Get token from start details + var token string + for _, v := range s.Starts { + switch v.Params.Type { + case tkv1.VoteTypeRunoff: + // This is a runoff vote. Execute the plugin command on the + // parent record. + token = v.Params.Parent + case tkv1.VoteTypeStandard: + // This is a standard vote. Execute the plugin command on the + // record specified in the vote params. + token = v.Params.Token + } + } + + // Send plugin command + ts := convertStartToPlugin(s) + tsr, err := t.politeiad.TicketVoteStart(ctx, token, ts) + if err != nil { + return nil, err + } + + // Emit notification for each start + t.events.Emit(EventTypeStart, + EventStart{ + Start: s, + User: u, + }) + + return &tkv1.StartReply{ + StartBlockHeight: tsr.StartBlockHeight, + StartBlockHash: tsr.StartBlockHash, + EndBlockHeight: tsr.EndBlockHeight, + EligibleTickets: tsr.EligibleTickets, + }, nil +} + +func (t *TicketVote) processCastBallot(ctx context.Context, cb tkv1.CastBallot) (*tkv1.CastBallotReply, error) { + log.Tracef("processCastBallot") + + // Get token from one of the votes + var token string + for _, v := range cb.Votes { + token = v.Token + break + } + + // Send plugin command + tcb := ticketvote.CastBallot{ + Ballot: convertCastVotesToPlugin(cb.Votes), + } + tcbr, err := t.politeiad.TicketVoteCastBallot(ctx, token, tcb) + if err != nil { + return nil, err + } + + return &tkv1.CastBallotReply{ + Receipts: convertCastVoteRepliesToV1(tcbr.Receipts), + }, nil +} + +func (t *TicketVote) processDetails(ctx context.Context, d tkv1.Details) (*tkv1.DetailsReply, error) { + log.Tracef("processsDetails: %v", d.Token) + + tdr, err := t.politeiad.TicketVoteDetails(ctx, d.Token) + if err != nil { + return nil, err + } + var vote *tkv1.VoteDetails + if tdr.Vote != nil { + vd := convertVoteDetailsToV1(*tdr.Vote) + vote = &vd + } + + return &tkv1.DetailsReply{ + Auths: convertAuthDetailsToV1(tdr.Auths), + Vote: vote, + }, nil +} + +func (t *TicketVote) processResults(ctx context.Context, r tkv1.Results) (*tkv1.ResultsReply, error) { + log.Tracef("processResults: %v", r.Token) + + return nil, nil +} + +func (t *TicketVote) processSummaries(ctx context.Context, s tkv1.Summaries) (*tkv1.SummariesReply, error) { + log.Tracef("processSummaries: %v", s.Tokens) + + return nil, nil +} + +func (t *TicketVote) processLinkedFrom(ctx context.Context, lf tkv1.LinkedFrom) (*tkv1.LinkedFromReply, error) { + log.Tracef("processLinkedFrom: %v", lf.Tokens) + + return nil, nil +} + +func (t *TicketVote) processInventory(ctx context.Context) (*tkv1.InventoryReply, error) { + log.Tracef("processInventory") + + return nil, nil +} + +func (t *TicketVote) processTimestamps(ctx context.Context, ts tkv1.Timestamps) (*tkv1.TimestampsReply, error) { + log.Tracef("processTimestamps: %v", ts.Token) + + // TODO Send plugin command + var r ticketvote.TimestampsReply + + // Prepare reply + var ( + auths = make([]tkv1.Timestamp, 0, len(r.Auths)) + votes = make(map[string]tkv1.Timestamp, len(r.Votes)) + + details = convertTimestampToV1(r.Details) + ) + for _, v := range r.Auths { + auths = append(auths, convertTimestampToV1(v)) + } + for k, v := range r.Votes { + votes[k] = convertTimestampToV1(v) + } + + return &tkv1.TimestampsReply{ + Auths: auths, + Details: details, + Votes: votes, + }, nil +} + +func convertVoteTypeToPlugin(t tkv1.VoteT) ticketvote.VoteT { + switch t { + case tkv1.VoteTypeStandard: + return ticketvote.VoteTypeStandard + case tkv1.VoteTypeRunoff: + return ticketvote.VoteTypeRunoff + } + return ticketvote.VoteTypeInvalid +} + +func convertVoteTypeToV1(t ticketvote.VoteT) tkv1.VoteT { + switch t { + case ticketvote.VoteTypeStandard: + return tkv1.VoteTypeStandard + case ticketvote.VoteTypeRunoff: + return tkv1.VoteTypeRunoff + } + return tkv1.VoteTypeInvalid + +} + +func convertVoteParamsToPlugin(v tkv1.VoteParams) ticketvote.VoteParams { + tv := ticketvote.VoteParams{ + Token: v.Token, + Version: v.Version, + Type: convertVoteTypeToPlugin(v.Type), + Mask: v.Mask, + Duration: v.Duration, + QuorumPercentage: v.QuorumPercentage, + PassPercentage: v.PassPercentage, + Parent: v.Parent, + } + // Convert vote options + vo := make([]ticketvote.VoteOption, 0, len(v.Options)) + for _, vi := range v.Options { + vo = append(vo, ticketvote.VoteOption{ + ID: vi.ID, + Description: vi.Description, + Bit: vi.Bit, + }) + } + tv.Options = vo + + return tv +} + +func convertVoteParamsToV1(v ticketvote.VoteParams) tkv1.VoteParams { + vp := tkv1.VoteParams{ + Token: v.Token, + Version: v.Version, + Type: convertVoteTypeToV1(v.Type), + Mask: v.Mask, + Duration: v.Duration, + QuorumPercentage: v.QuorumPercentage, + PassPercentage: v.PassPercentage, + } + vo := make([]tkv1.VoteOption, 0, len(v.Options)) + for _, o := range v.Options { + vo = append(vo, tkv1.VoteOption{ + ID: o.ID, + Description: o.Description, + Bit: o.Bit, + }) + } + vp.Options = vo + + return vp +} + +func convertStartDetailsToPlugin(sd tkv1.StartDetails) ticketvote.StartDetails { + return ticketvote.StartDetails{ + Params: convertVoteParamsToPlugin(sd.Params), + PublicKey: sd.PublicKey, + Signature: sd.Signature, + } +} + +func convertStartToPlugin(vs tkv1.Start) ticketvote.Start { + starts := make([]ticketvote.StartDetails, 0, len(vs.Starts)) + for _, v := range vs.Starts { + starts = append(starts, convertStartDetailsToPlugin(v)) + } + return ticketvote.Start{ + Starts: starts, + } +} + +func convertCastVotesToPlugin(votes []tkv1.CastVote) []ticketvote.CastVote { + cv := make([]ticketvote.CastVote, 0, len(votes)) + for _, v := range votes { + cv = append(cv, ticketvote.CastVote{ + Token: v.Token, + Ticket: v.Ticket, + VoteBit: v.VoteBit, + Signature: v.Signature, + }) + } + return cv +} + +func convertVoteErrorToV1(e ticketvote.VoteErrorT) tkv1.VoteErrorT { + switch e { + case ticketvote.VoteErrorInvalid: + return tkv1.VoteErrorInvalid + case ticketvote.VoteErrorInternalError: + return tkv1.VoteErrorInternalError + case ticketvote.VoteErrorRecordNotFound: + return tkv1.VoteErrorRecordNotFound + case ticketvote.VoteErrorVoteBitInvalid: + return tkv1.VoteErrorVoteBitInvalid + case ticketvote.VoteErrorVoteStatusInvalid: + return tkv1.VoteErrorVoteStatusInvalid + case ticketvote.VoteErrorTicketAlreadyVoted: + return tkv1.VoteErrorTicketAlreadyVoted + case ticketvote.VoteErrorTicketNotEligible: + return tkv1.VoteErrorTicketNotEligible + default: + return tkv1.VoteErrorInternalError + } +} + +func convertCastVoteRepliesToV1(replies []ticketvote.CastVoteReply) []tkv1.CastVoteReply { + r := make([]tkv1.CastVoteReply, 0, len(replies)) + for _, v := range replies { + r = append(r, tkv1.CastVoteReply{ + Ticket: v.Ticket, + Receipt: v.Receipt, + ErrorCode: convertVoteErrorToV1(v.ErrorCode), + ErrorContext: v.ErrorContext, + }) + } + return r +} + +func convertVoteDetailsToV1(vd ticketvote.VoteDetails) tkv1.VoteDetails { + return tkv1.VoteDetails{ + Params: convertVoteParamsToV1(vd.Params), + PublicKey: vd.PublicKey, + Signature: vd.Signature, + StartBlockHeight: vd.StartBlockHeight, + StartBlockHash: vd.StartBlockHash, + EndBlockHeight: vd.EndBlockHeight, + EligibleTickets: vd.EligibleTickets, + } +} + +func convertAuthDetailsToV1(auths []ticketvote.AuthDetails) []tkv1.AuthDetails { + a := make([]tkv1.AuthDetails, 0, len(auths)) + for _, v := range auths { + a = append(a, tkv1.AuthDetails{ + Token: v.Token, + Version: v.Version, + Action: v.Action, + PublicKey: v.PublicKey, + Signature: v.Signature, + Timestamp: v.Timestamp, + Receipt: v.Receipt, + }) + } + return a +} + +/* +func convertCastVoteDetails(votes []ticketvote.CastVoteDetails) []tkv1.CastVoteDetails { + vs := make([]tkv1.CastVoteDetails, 0, len(votes)) + for _, v := range votes { + vs = append(vs, tkv1.CastVoteDetails{ + Token: v.Token, + Ticket: v.Ticket, + VoteBit: v.VoteBit, + Signature: v.Signature, + Receipt: v.Receipt, + }) + } + return vs +} + +func convertVoteStatus(s ticketvote.VoteStatusT) tkv1.VoteStatusT { + switch s { + case ticketvote.VoteStatusInvalid: + return tkv1.VoteStatusInvalid + case ticketvote.VoteStatusUnauthorized: + return tkv1.VoteStatusUnauthorized + case ticketvote.VoteStatusAuthorized: + return tkv1.VoteStatusAuthorized + case ticketvote.VoteStatusStarted: + return tkv1.VoteStatusStarted + case ticketvote.VoteStatusFinished: + return tkv1.VoteStatusFinished + default: + return tkv1.VoteStatusInvalid + } +} + +func convertSummary(s ticketvote.VoteSummary) tkv1.Summary { + results := make([]tkv1.VoteResult, 0, len(s.Results)) + for _, v := range s.Results { + results = append(results, tkv1.VoteResult{ + ID: v.ID, + Description: v.Description, + VoteBit: v.VoteBit, + Votes: v.Votes, + }) + } + return tkv1.Summary{ + Type: convertVoteType(s.Type), + Status: convertVoteStatus(s.Status), + Duration: s.Duration, + StartBlockHeight: s.StartBlockHeight, + StartBlockHash: s.StartBlockHash, + EndBlockHeight: s.EndBlockHeight, + EligibleTickets: s.EligibleTickets, + QuorumPercentage: s.QuorumPercentage, + PassPercentage: s.PassPercentage, + Results: results, + Approved: s.Approved, + } +} +*/ + +func convertProofToV1(p ticketvote.Proof) tkv1.Proof { + return tkv1.Proof{ + Type: p.Type, + Digest: p.Digest, + MerkleRoot: p.MerkleRoot, + MerklePath: p.MerklePath, + ExtraData: p.ExtraData, + } +} + +func convertTimestampToV1(t ticketvote.Timestamp) tkv1.Timestamp { + proofs := make([]tkv1.Proof, 0, len(t.Proofs)) + for _, v := range t.Proofs { + proofs = append(proofs, convertProofToV1(v)) + } + return tkv1.Timestamp{ + Data: t.Data, + Digest: t.Digest, + TxID: t.TxID, + MerkleRoot: t.MerkleRoot, + Proofs: proofs, + } +} diff --git a/politeiawww/ticketvote/ticketvote.go b/politeiawww/ticketvote/ticketvote.go index 428d61448..be91c6459 100644 --- a/politeiawww/ticketvote/ticketvote.go +++ b/politeiawww/ticketvote/ticketvote.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package comments +package ticketvote import ( "encoding/json" @@ -13,7 +13,6 @@ import ( "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/events" "github.com/decred/politeia/politeiawww/sessions" - "github.com/decred/politeia/politeiawww/user" "github.com/decred/politeia/util" ) @@ -21,182 +20,240 @@ import ( type TicketVote struct { cfg *config.Config politeiad *pdclient.Client - userdb user.Database sessions sessions.Sessions events *events.Manager } -func (p *politeiawww) handleAuthorize(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleAuthorize") +// HandleAuthorize is the request handler for the ticketvote v1 Authorize +// route. +func (t *TicketVote) HandleAuthorize(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleAuthorize") var a tkv1.Authorize decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&va); err != nil { - respondWithError(w, r, "handleAuthorize: unmarshal", + if err := decoder.Decode(&a); err != nil { + respondWithError(w, r, "HandleAuthorize: unmarshal", tkv1.UserErrorReply{ - ErrorCode: tkv1.ErrorStatusInputInvalid, + ErrorCode: tkv1.ErrorCodeInputInvalid, }) return } - u, err := p.sessions.GetSessionUser(w, r) + u, err := t.sessions.GetSessionUser(w, r) if err != nil { respondWithError(w, r, - "handleAuthorize: GetSessionUser: %v", err) + "HandleAuthorize: GetSessionUser: %v", err) return } - ar, err := p.processAuthorize(r.Context(), a, *u) + ar, err := t.processAuthorize(r.Context(), a, *u) if err != nil { respondWithError(w, r, - "handleAuthorize: processAuthorize: %v", err) + "HandleAuthorize: processAuthorize: %v", err) return } - util.RespondWithJSON(w, http.StatusOK, vr) + util.RespondWithJSON(w, http.StatusOK, ar) } -func (p *politeiawww) HandleStart(w http.ResponseWriter, r *http.Request) { +// HandleStart is the requeset handler for the ticketvote v1 Start route. +func (t *TicketVote) HandleStart(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleStart") - var vs piv1.VoteStart + var s tkv1.Start decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&vs); err != nil { - respondWithPiError(w, r, "HandleStart: unmarshal", - piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusInputInvalid, + if err := decoder.Decode(&s); err != nil { + respondWithError(w, r, "HandleStart: unmarshal", + tkv1.UserErrorReply{ + ErrorCode: tkv1.ErrorCodeInputInvalid, }) return } - usr, err := p.sessions.GetSessionUser(w, r) + u, err := t.sessions.GetSessionUser(w, r) if err != nil { - respondWithPiError(w, r, + respondWithError(w, r, "HandleStart: GetSessionUser: %v", err) return } - vsr, err := p.processVoteStart(r.Context(), vs, *usr) + sr, err := t.processStart(r.Context(), s, *u) if err != nil { - respondWithPiError(w, r, - "HandleStart: processVoteStart: %v", err) + respondWithError(w, r, + "HandleStart: processStart: %v", err) return } - util.RespondWithJSON(w, http.StatusOK, vsr) + util.RespondWithJSON(w, http.StatusOK, sr) } -func (p *politeiawww) handleCastBallot(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleCastBallot") +// HandleCastBallot is the request handler for the ticketvote v1 CastBallot +// route. +func (t *TicketVote) HandleCastBallot(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleCastBallot") - var vb piv1.CastBallot + var cb tkv1.CastBallot decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&vb); err != nil { - respondWithPiError(w, r, "handleCastBallot: unmarshal", - piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusInputInvalid, + if err := decoder.Decode(&cb); err != nil { + respondWithError(w, r, "HandleCastBallot: unmarshal", + tkv1.UserErrorReply{ + ErrorCode: tkv1.ErrorCodeInputInvalid, }) return } - vbr, err := p.processCastBallot(r.Context(), vb) + cbr, err := t.processCastBallot(r.Context(), cb) if err != nil { - respondWithPiError(w, r, - "handleCastBallot: processCastBallot: %v", err) + respondWithError(w, r, + "HandleCastBallot: processCastBallot: %v", err) return } - util.RespondWithJSON(w, http.StatusOK, vbr) + util.RespondWithJSON(w, http.StatusOK, cbr) } -func (p *politeiawww) handleVotes(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleVotes") +// HandleDetails is the request handler for the ticketvote v1 Details route. +func (t *TicketVote) HandleDetails(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleDetails") - var v piv1.Votes + var d tkv1.Details decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&v); err != nil { - respondWithPiError(w, r, "handleVotes: unmarshal", - piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusInputInvalid, + if err := decoder.Decode(&d); err != nil { + respondWithError(w, r, "HandleDetails: unmarshal", + tkv1.UserErrorReply{ + ErrorCode: tkv1.ErrorCodeInputInvalid, }) return } - vr, err := p.processVotes(r.Context(), v) + dr, err := t.processDetails(r.Context(), d) if err != nil { - respondWithPiError(w, r, - "handleVotes: processVotes: %v", err) + respondWithError(w, r, + "HandleDetails: processDetails: %v", err) return } - util.RespondWithJSON(w, http.StatusOK, vr) + util.RespondWithJSON(w, http.StatusOK, dr) } -func (p *politeiawww) handleVoteResultsPi(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleVoteResults") +// HandleResults is the request handler for the ticketvote v1 Results route. +func (t *TicketVote) HandleResults(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleResults") - var vr piv1.VoteResults + var rs tkv1.Results decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&vr); err != nil { - respondWithPiError(w, r, "handleVoteResults: unmarshal", - piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusInputInvalid, + if err := decoder.Decode(&rs); err != nil { + respondWithError(w, r, "HandleResults: unmarshal", + tkv1.UserErrorReply{ + ErrorCode: tkv1.ErrorCodeInputInvalid, }) return } - vrr, err := p.processVoteResultsPi(r.Context(), vr) + rsr, err := t.processResults(r.Context(), rs) if err != nil { - respondWithPiError(w, r, - "handleVoteResults: prcoessVoteResults: %v", err) + respondWithError(w, r, + "HandleResults: processResults: %v", err) return } - util.RespondWithJSON(w, http.StatusOK, vrr) + util.RespondWithJSON(w, http.StatusOK, rsr) } -func (p *politeiawww) handleVoteSummaries(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleVoteSummaries") +// HandleSummaries is the request handler for the ticketvote v1 Summaries +// route. +func (t *TicketVote) HandleSummaries(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleSummaries") - var vs piv1.VoteSummaries + var s tkv1.Summaries decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&vs); err != nil { - respondWithPiError(w, r, "handleVoteSummaries: unmarshal", - piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusInputInvalid, + if err := decoder.Decode(&s); err != nil { + respondWithError(w, r, "HandleSummaries: unmarshal", + tkv1.UserErrorReply{ + ErrorCode: tkv1.ErrorCodeInputInvalid, }) return } - vsr, err := p.processVoteSummaries(r.Context(), vs) + sr, err := t.processSummaries(r.Context(), s) if err != nil { - respondWithPiError(w, r, "handleVoteSummaries: processVoteSummaries: %v", + respondWithError(w, r, "HandleSummaries: processSummaries: %v", err) return } - util.RespondWithJSON(w, http.StatusOK, vsr) + util.RespondWithJSON(w, http.StatusOK, sr) } -func (p *politeiawww) handleVoteInventory(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleVoteInventory") +// HandleLinkedFrom is the request handler for the ticketvote v1 LinkedFrom +// route. +func (t *TicketVote) HandleLinkedFrom(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleLinkedFrom") - var vi piv1.VoteInventory + var lf tkv1.LinkedFrom decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&vi); err != nil { - respondWithPiError(w, r, "handleVoteInventory: unmarshal", - piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusInputInvalid, + if err := decoder.Decode(&lf); err != nil { + respondWithError(w, r, "HandleLinkedFrom: unmarshal", + tkv1.UserErrorReply{ + ErrorCode: tkv1.ErrorCodeInputInvalid, }) return } - vir, err := p.processVoteInventory(r.Context()) + lfr, err := t.processLinkedFrom(r.Context(), lf) + if err != nil { + respondWithError(w, r, "HandleLinkedFrom: processLinkedFrom: %v", + err) + return + } + + util.RespondWithJSON(w, http.StatusOK, lfr) +} + +// HandleInventory is the request handler for the ticketvote v1 Inventory +// route. +func (t *TicketVote) HandleInventory(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleInventory") + + ir, err := t.processInventory(r.Context()) if err != nil { - respondWithPiError(w, r, "handleVoteInventory: processVoteInventory: %v", + respondWithError(w, r, "HandleInventory: processInventory: %v", err) return } - util.RespondWithJSON(w, http.StatusOK, vir) + util.RespondWithJSON(w, http.StatusOK, ir) +} + +// HandleTimestamps is the request handler for the ticketvote v1 Timestamps +// route. +func (t *TicketVote) HandleTimestamps(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleTimestamps") + + var ts tkv1.Timestamps + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&t); err != nil { + respondWithError(w, r, "HandleTimestamps: unmarshal", + tkv1.UserErrorReply{ + ErrorCode: tkv1.ErrorCodeInputInvalid, + }) + return + } + + tsr, err := t.processTimestamps(r.Context(), ts) + if err != nil { + respondWithError(w, r, + "HandleTimestamps: processTimestamps: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, tsr) +} + +// New returns a new TicketVote context. +func New(cfg *config.Config, politeiad *pdclient.Client) *TicketVote { + return &TicketVote{ + cfg: cfg, + politeiad: politeiad, + } } diff --git a/politeiawww/www.go b/politeiawww/www.go index 53c1fdafc..9584ef52b 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -155,46 +155,7 @@ func convertPiErrorStatusFromPD(e pd.ErrorStatusT) pi.ErrorStatusT { func convertPiErrorStatusFromPiPlugin(e piplugin.ErrorCodeT) pi.ErrorStatusT { switch e { - case piplugin.ErrorCodePageSizeExceeded: - return pi.ErrorStatusPageSizeExceeded - case piplugin.ErrorCodePropTokenInvalid: - return pi.ErrorStatusPropTokenInvalid - case piplugin.ErrorCodePropStatusInvalid: - return pi.ErrorStatusPropStatusInvalid - case piplugin.ErrorCodePropVersionInvalid: - return pi.ErrorStatusPropVersionInvalid - case piplugin.ErrorCodePropStatusChangeInvalid: - return pi.ErrorStatusPropStatusChangeInvalid - case piplugin.ErrorCodePropLinkToInvalid: - return pi.ErrorStatusPropLinkToInvalid - case piplugin.ErrorCodeVoteStatusInvalid: - return pi.ErrorStatusVoteStatusInvalid - case piplugin.ErrorCodeStartDetailsInvalid: - return pi.ErrorStatusStartDetailsInvalid - case piplugin.ErrorCodeStartDetailsMissing: - return pi.ErrorStatusStartDetailsMissing - case piplugin.ErrorCodeVoteParentInvalid: - return pi.ErroStatusVoteParentInvalid - } - return pi.ErrorStatusInvalid -} - -func convertPiErrorStatusFromTicketVote(e ticketvote.ErrorCodeT) pi.ErrorStatusT { - switch e { - case ticketvote.ErrorCodeTokenInvalid: - return pi.ErrorStatusPropTokenInvalid - case ticketvote.ErrorCodePublicKeyInvalid: - return pi.ErrorStatusPublicKeyInvalid - case ticketvote.ErrorCodeSignatureInvalid: - return pi.ErrorStatusSignatureInvalid - case ticketvote.ErrorCodeRecordStatusInvalid: - return pi.ErrorStatusPropStatusInvalid - case ticketvote.ErrorCodeAuthorizationInvalid: - return pi.ErrorStatusVoteAuthInvalid - case ticketvote.ErrorCodeVoteParamsInvalid: - return pi.ErrorStatusVoteParamsInvalid - case ticketvote.ErrorCodeVoteStatusInvalid: - return pi.ErrorStatusVoteStatusInvalid + // TODO } return pi.ErrorStatusInvalid } @@ -214,10 +175,6 @@ func convertPiErrorStatus(pluginID string, errCode int) pi.ErrorStatusT { // Pi plugin e := piplugin.ErrorCodeT(errCode) return convertPiErrorStatusFromPiPlugin(e) - case ticketvote.PluginID: - // Ticket vote plugin - e := ticketvote.ErrorCodeT(errCode) - return convertPiErrorStatusFromTicketVote(e) } // No corresponding pi error status found From d000bfc84bad4e34b68ae4fc8e55e64218e79a10 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 24 Jan 2021 11:36:01 -0600 Subject: [PATCH 251/449] Finish ticketvote routes. --- .../backend/tlogbe/plugins/ticketvote/cmds.go | 13 +- politeiad/plugins/ticketvote/ticketvote.go | 11 +- politeiawww/api/ticketvote/v1/v1.go | 18 ++- politeiawww/comments/comments.go | 10 +- politeiawww/piwww.go | 100 ++++++++---- politeiawww/proposals.go | 2 +- politeiawww/testing.go | 4 +- politeiawww/ticketvote/process.go | 152 ++++++++++++------ politeiawww/ticketvote/ticketvote.go | 8 +- politeiawww/www.go | 41 ----- 10 files changed, 212 insertions(+), 147 deletions(-) diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go index e90725513..8d2659b32 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go @@ -2596,12 +2596,15 @@ func convertInventoryReply(v inventory) ticketvote.InventoryReply { tokens := tokensByHeight[height] started = append(started, tokens...) } + return ticketvote.InventoryReply{ - Unauthorized: v.unauthorized, - Authorized: v.authorized, - Started: started, - Finished: v.finished, - BestBlock: v.bestBlock, + Records: map[ticketvote.VoteStatusT][]string{ + ticketvote.VoteStatusUnauthorized: v.unauthorized, + ticketvote.VoteStatusAuthorized: v.authorized, + ticketvote.VoteStatusStarted: started, + ticketvote.VoteStatusFinished: v.finished, + }, + BestBlock: v.bestBlock, } } diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index 60863e5c6..980061638 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -399,7 +399,7 @@ const ( var ( // VoteStatuses contains the human readable vote statuses. VoteStatuses = map[VoteStatusT]string{ - VoteStatusInvalid: "vote status invalid", + VoteStatusInvalid: "invalid", VoteStatusUnauthorized: "unauthorized", VoteStatusAuthorized: "authorized", VoteStatusStarted: "started", @@ -450,16 +450,13 @@ type Inventory struct{} // InventoryReply is the reply to the Inventory command. It contains the tokens // of all public, non-abandoned records categorized by vote status. // -// Sorted by timestamp in descending order: +// Statuses sorted by timestamp in descending order: // Unauthorized, Authorized // -// Sorted by voting period end block height in descending order: +// Statuses sorted by voting period end block height in descending order: // Started, Finished type InventoryReply struct { - Unauthorized []string `json:"unauthorized"` - Authorized []string `json:"authorized"` - Started []string `json:"started"` - Finished []string `json:"finished"` + Records map[VoteStatusT][]string `json:"records"` // BestBlock is the best block value that was used to prepare the // inventory. diff --git a/politeiawww/api/ticketvote/v1/v1.go b/politeiawww/api/ticketvote/v1/v1.go index 95da2f415..a8d14bf62 100644 --- a/politeiawww/api/ticketvote/v1/v1.go +++ b/politeiawww/api/ticketvote/v1/v1.go @@ -475,11 +475,23 @@ type LinkedFromReply struct { // Inventory requests the record inventory categorized by vote status. type Inventory struct{} -// InventoryReply is the reply to the Inventory command. The returned map is -// a map[votestatus][]token where the votestatus key is the human readable vote -// status defined by the VoteStatuses array in this package. +// InventoryReply is the reply to the Inventory command. It contains the tokens +// of all public, non-abandoned records categorized by vote status. The +// returned map is a map[votestatus][]token where the votestatus key is the +// human readable vote status defined by the VoteStatuses array in this +// package. +// +// Statuses sorted by timestamp in descending order: +// Unauthorized, Authorized +// +// Statuses sorted by voting period end block height in descending order: +// Started, Finished type InventoryReply struct { Records map[string][]string `json:"records"` + + // BestBlock is the best block value that was used to prepare the + // inventory. + BestBlock uint32 `json:"bestblock"` } // Proof contains an inclusion proof for the digest in the merkle root. The diff --git a/politeiawww/comments/comments.go b/politeiawww/comments/comments.go index ef276f13c..e3d0cca37 100644 --- a/politeiawww/comments/comments.go +++ b/politeiawww/comments/comments.go @@ -22,7 +22,7 @@ type Comments struct { cfg *config.Config politeiad *pdclient.Client userdb user.Database - sessions sessions.Sessions + sessions *sessions.Sessions events *events.Manager } @@ -236,10 +236,12 @@ func (c *Comments) HandleTimestamps(w http.ResponseWriter, r *http.Request) { } // New returns a new Comments context. -func New(cfg *config.Config, politeiad *pdclient.Client, userdb user.Database) *Comments { +func New(cfg *config.Config, pdc *pdclient.Client, udb user.Database, s *sessions.Sessions, e *events.Manager) *Comments { return &Comments{ cfg: cfg, - politeiad: politeiad, - userdb: userdb, + politeiad: pdc, + userdb: udb, + sessions: s, + events: e, } } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 5dc493776..ceefac829 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -26,12 +26,15 @@ import ( cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/comments" "github.com/decred/politeia/politeiawww/sessions" + "github.com/decred/politeia/politeiawww/ticketvote" "github.com/decred/politeia/politeiawww/user" wwwutil "github.com/decred/politeia/politeiawww/util" "github.com/decred/politeia/util" + "github.com/google/uuid" ) // TODO www package references should be completely gone from this file @@ -1369,7 +1372,7 @@ func (p *politeiawww) handleProposalInventory(w http.ResponseWriter, r *http.Req } // setPiRoutes sets the pi API routes. -func (p *politeiawww) setPiRoutes(c *comments.Comments) { +func (p *politeiawww) setPiRoutes(c *comments.Comments, t *ticketvote.TicketVote) { // Return a 404 when a route is not found p.router.NotFoundHandler = http.HandlerFunc(p.handleNotFound) @@ -1446,38 +1449,75 @@ func (p *politeiawww) setPiRoutes(c *comments.Comments) { cmv1.RouteTimestamps, c.HandleTimestamps, permissionPublic) - /* - // Voute routes - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteVoteAuthorize, p.handleVoteAuthorize, - permissionLogin) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteVoteStart, p.handleVoteStart, - permissionAdmin) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteCastBallot, p.handleCastBallot, - permissionPublic) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteVotes, p.handleVotes, - permissionPublic) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteVoteResults, p.handleVoteResults, - permissionPublic) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteVoteSummaries, p.handleVoteSummaries, - permissionPublic) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteVoteInventory, p.handleVoteInventory, - permissionPublic) - - // Ticket vote routes - p.addRoute(http.MethodPost, tkv1.APIRoute, - tkv1.RouteTimestamps, p.handleTicketVoteTimestamps, - permissionPublic) - */ + // Ticket vote routes + p.addRoute(http.MethodPost, tkv1.APIRoute, + tkv1.RouteAuthorize, t.HandleAuthorize, + permissionLogin) + p.addRoute(http.MethodPost, tkv1.APIRoute, + tkv1.RouteStart, t.HandleStart, + permissionAdmin) + p.addRoute(http.MethodPost, tkv1.APIRoute, + tkv1.RouteCastBallot, t.HandleCastBallot, + permissionPublic) + p.addRoute(http.MethodPost, tkv1.APIRoute, + tkv1.RouteDetails, t.HandleDetails, + permissionPublic) + p.addRoute(http.MethodPost, tkv1.APIRoute, + tkv1.RouteResults, t.HandleResults, + permissionPublic) + p.addRoute(http.MethodPost, tkv1.APIRoute, + tkv1.RouteSummaries, t.HandleSummaries, + permissionPublic) + p.addRoute(http.MethodPost, tkv1.APIRoute, + tkv1.RouteInventory, t.HandleInventory, + permissionPublic) + p.addRoute(http.MethodPost, tkv1.APIRoute, + tkv1.RouteTimestamps, t.HandleTimestamps, + permissionPublic) // Record routes p.addRoute(http.MethodPost, rcv1.APIRoute, rcv1.RouteTimestamps, p.handleTimestamps, permissionPublic) } + +func (p *politeiawww) setupPi() error { + // Setup api contexts + c := comments.New(p.cfg, p.politeiad, p.db, p.sessions, p.events) + tv := ticketvote.New(p.cfg, p.politeiad, p.sessions, p.events) + + // Setup routes + p.setUserWWWRoutes() + p.setPiRoutes(c, tv) + + // Verify paywall settings + switch { + case p.cfg.PaywallAmount != 0 && p.cfg.PaywallXpub != "": + // Paywall is enabled + paywallAmountInDcr := float64(p.cfg.PaywallAmount) / 1e8 + log.Infof("Paywall : %v DCR", paywallAmountInDcr) + + case p.cfg.PaywallAmount == 0 && p.cfg.PaywallXpub == "": + // Paywall is disabled + log.Infof("Paywall: DISABLED") + + default: + // Invalid paywall setting + return fmt.Errorf("paywall settings invalid, both an amount " + + "and public key MUST be set") + } + + // Setup paywall pool + p.userPaywallPool = make(map[uuid.UUID]paywallPoolMember) + err := p.initPaywallChecker() + if err != nil { + return err + } + + // Setup event manager + p.setupEventListenersPi() + + // TODO Verify politeiad plugins + + return nil +} diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 5e8fafefe..1f33d25c2 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -299,7 +299,7 @@ func (p *politeiawww) processBatchVoteSummary(ctx context.Context, bvs www.Batch // TODO var bestBlock uint32 - var vs []ticketvote.VoteSummary + var vs []ticketvote.SummaryReply // Prepare reply summaries := make(map[string]www.VoteSummary, len(vs)) diff --git a/politeiawww/testing.go b/politeiawww/testing.go index f1b7b064f..afe91e474 100644 --- a/politeiawww/testing.go +++ b/politeiawww/testing.go @@ -29,6 +29,7 @@ import ( "github.com/decred/politeia/politeiawww/comments" "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/sessions" + "github.com/decred/politeia/politeiawww/ticketvote" "github.com/decred/politeia/politeiawww/user" "github.com/decred/politeia/politeiawww/user/localdb" "github.com/decred/politeia/util" @@ -362,10 +363,11 @@ func newTestPoliteiawww(t *testing.T) (*politeiawww, func()) { // TODO setup testing var c *comments.Comments + var tv *ticketvote.TicketVote // Setup routes p.setUserWWWRoutes() - p.setPiRoutes(c) + p.setPiRoutes(c, tv) // The cleanup is handled using a closure so that the temp dir // can be deleted using the local variable and not cfg.DataDir. diff --git a/politeiawww/ticketvote/process.go b/politeiawww/ticketvote/process.go index e8d317531..58396cd4f 100644 --- a/politeiawww/ticketvote/process.go +++ b/politeiawww/ticketvote/process.go @@ -143,6 +143,7 @@ func (t *TicketVote) processDetails(ctx context.Context, d tkv1.Details) (*tkv1. if err != nil { return nil, err } + var vote *tkv1.VoteDetails if tdr.Vote != nil { vd := convertVoteDetailsToV1(*tdr.Vote) @@ -158,44 +159,84 @@ func (t *TicketVote) processDetails(ctx context.Context, d tkv1.Details) (*tkv1. func (t *TicketVote) processResults(ctx context.Context, r tkv1.Results) (*tkv1.ResultsReply, error) { log.Tracef("processResults: %v", r.Token) - return nil, nil + rr, err := t.politeiad.TicketVoteResults(ctx, r.Token) + if err != nil { + return nil, err + } + + return &tkv1.ResultsReply{ + Votes: convertCastVoteDetailsToV1(rr.Votes), + }, nil } func (t *TicketVote) processSummaries(ctx context.Context, s tkv1.Summaries) (*tkv1.SummariesReply, error) { log.Tracef("processSummaries: %v", s.Tokens) - return nil, nil + ts, err := t.politeiad.TicketVoteSummaries(ctx, s.Tokens) + if err != nil { + return nil, err + } + + return &tkv1.SummariesReply{ + Summaries: convertSummariesToV1(ts), + }, nil } func (t *TicketVote) processLinkedFrom(ctx context.Context, lf tkv1.LinkedFrom) (*tkv1.LinkedFromReply, error) { log.Tracef("processLinkedFrom: %v", lf.Tokens) - return nil, nil + tlf, err := t.politeiad.TicketVoteLinkedFrom(ctx, lf.Tokens) + if err != nil { + return nil, err + } + + return &tkv1.LinkedFromReply{ + LinkedFrom: tlf, + }, nil } func (t *TicketVote) processInventory(ctx context.Context) (*tkv1.InventoryReply, error) { log.Tracef("processInventory") - return nil, nil + // Send plugin command + ir, err := t.politeiad.TicketVoteInventory(ctx) + if err != nil { + return nil, err + } + + // Convert vote statuses to human readable equivalents + records := make(map[string][]string, len(ir.Records)) + for k, v := range ir.Records { + s := convertVoteStatusToV1(k) + records[tkv1.VoteStatuses[s]] = v + } + + return &tkv1.InventoryReply{ + Records: records, + BestBlock: ir.BestBlock, + }, nil } func (t *TicketVote) processTimestamps(ctx context.Context, ts tkv1.Timestamps) (*tkv1.TimestampsReply, error) { log.Tracef("processTimestamps: %v", ts.Token) - // TODO Send plugin command - var r ticketvote.TimestampsReply + // Send plugin command + tsr, err := t.politeiad.TicketVoteTimestamps(ctx, ts.Token) + if err != nil { + return nil, err + } // Prepare reply var ( - auths = make([]tkv1.Timestamp, 0, len(r.Auths)) - votes = make(map[string]tkv1.Timestamp, len(r.Votes)) + auths = make([]tkv1.Timestamp, 0, len(tsr.Auths)) + votes = make(map[string]tkv1.Timestamp, len(tsr.Votes)) - details = convertTimestampToV1(r.Details) + details = convertTimestampToV1(tsr.Details) ) - for _, v := range r.Auths { + for _, v := range tsr.Auths { auths = append(auths, convertTimestampToV1(v)) } - for k, v := range r.Votes { + for k, v := range tsr.Votes { votes[k] = convertTimestampToV1(v) } @@ -216,17 +257,6 @@ func convertVoteTypeToPlugin(t tkv1.VoteT) ticketvote.VoteT { return ticketvote.VoteTypeInvalid } -func convertVoteTypeToV1(t ticketvote.VoteT) tkv1.VoteT { - switch t { - case ticketvote.VoteTypeStandard: - return tkv1.VoteTypeStandard - case ticketvote.VoteTypeRunoff: - return tkv1.VoteTypeRunoff - } - return tkv1.VoteTypeInvalid - -} - func convertVoteParamsToPlugin(v tkv1.VoteParams) ticketvote.VoteParams { tv := ticketvote.VoteParams{ Token: v.Token, @@ -252,29 +282,6 @@ func convertVoteParamsToPlugin(v tkv1.VoteParams) ticketvote.VoteParams { return tv } -func convertVoteParamsToV1(v ticketvote.VoteParams) tkv1.VoteParams { - vp := tkv1.VoteParams{ - Token: v.Token, - Version: v.Version, - Type: convertVoteTypeToV1(v.Type), - Mask: v.Mask, - Duration: v.Duration, - QuorumPercentage: v.QuorumPercentage, - PassPercentage: v.PassPercentage, - } - vo := make([]tkv1.VoteOption, 0, len(v.Options)) - for _, o := range v.Options { - vo = append(vo, tkv1.VoteOption{ - ID: o.ID, - Description: o.Description, - Bit: o.Bit, - }) - } - vp.Options = vo - - return vp -} - func convertStartDetailsToPlugin(sd tkv1.StartDetails) ticketvote.StartDetails { return ticketvote.StartDetails{ Params: convertVoteParamsToPlugin(sd.Params), @@ -306,6 +313,40 @@ func convertCastVotesToPlugin(votes []tkv1.CastVote) []ticketvote.CastVote { return cv } +func convertVoteTypeToV1(t ticketvote.VoteT) tkv1.VoteT { + switch t { + case ticketvote.VoteTypeStandard: + return tkv1.VoteTypeStandard + case ticketvote.VoteTypeRunoff: + return tkv1.VoteTypeRunoff + } + return tkv1.VoteTypeInvalid + +} + +func convertVoteParamsToV1(v ticketvote.VoteParams) tkv1.VoteParams { + vp := tkv1.VoteParams{ + Token: v.Token, + Version: v.Version, + Type: convertVoteTypeToV1(v.Type), + Mask: v.Mask, + Duration: v.Duration, + QuorumPercentage: v.QuorumPercentage, + PassPercentage: v.PassPercentage, + } + vo := make([]tkv1.VoteOption, 0, len(v.Options)) + for _, o := range v.Options { + vo = append(vo, tkv1.VoteOption{ + ID: o.ID, + Description: o.Description, + Bit: o.Bit, + }) + } + vp.Options = vo + + return vp +} + func convertVoteErrorToV1(e ticketvote.VoteErrorT) tkv1.VoteErrorT { switch e { case ticketvote.VoteErrorInvalid: @@ -368,8 +409,7 @@ func convertAuthDetailsToV1(auths []ticketvote.AuthDetails) []tkv1.AuthDetails { return a } -/* -func convertCastVoteDetails(votes []ticketvote.CastVoteDetails) []tkv1.CastVoteDetails { +func convertCastVoteDetailsToV1(votes []ticketvote.CastVoteDetails) []tkv1.CastVoteDetails { vs := make([]tkv1.CastVoteDetails, 0, len(votes)) for _, v := range votes { vs = append(vs, tkv1.CastVoteDetails{ @@ -383,7 +423,7 @@ func convertCastVoteDetails(votes []ticketvote.CastVoteDetails) []tkv1.CastVoteD return vs } -func convertVoteStatus(s ticketvote.VoteStatusT) tkv1.VoteStatusT { +func convertVoteStatusToV1(s ticketvote.VoteStatusT) tkv1.VoteStatusT { switch s { case ticketvote.VoteStatusInvalid: return tkv1.VoteStatusInvalid @@ -400,7 +440,7 @@ func convertVoteStatus(s ticketvote.VoteStatusT) tkv1.VoteStatusT { } } -func convertSummary(s ticketvote.VoteSummary) tkv1.Summary { +func convertSummaryToV1(s ticketvote.SummaryReply) tkv1.Summary { results := make([]tkv1.VoteResult, 0, len(s.Results)) for _, v := range s.Results { results = append(results, tkv1.VoteResult{ @@ -411,8 +451,8 @@ func convertSummary(s ticketvote.VoteSummary) tkv1.Summary { }) } return tkv1.Summary{ - Type: convertVoteType(s.Type), - Status: convertVoteStatus(s.Status), + Type: convertVoteTypeToV1(s.Type), + Status: convertVoteStatusToV1(s.Status), Duration: s.Duration, StartBlockHeight: s.StartBlockHeight, StartBlockHash: s.StartBlockHash, @@ -422,9 +462,17 @@ func convertSummary(s ticketvote.VoteSummary) tkv1.Summary { PassPercentage: s.PassPercentage, Results: results, Approved: s.Approved, + BestBlock: s.BestBlock, + } +} + +func convertSummariesToV1(s map[string]ticketvote.SummaryReply) map[string]tkv1.Summary { + ts := make(map[string]tkv1.Summary, len(s)) + for k, v := range s { + ts[k] = convertSummaryToV1(v) } + return ts } -*/ func convertProofToV1(p ticketvote.Proof) tkv1.Proof { return tkv1.Proof{ diff --git a/politeiawww/ticketvote/ticketvote.go b/politeiawww/ticketvote/ticketvote.go index be91c6459..217e23ba7 100644 --- a/politeiawww/ticketvote/ticketvote.go +++ b/politeiawww/ticketvote/ticketvote.go @@ -20,7 +20,7 @@ import ( type TicketVote struct { cfg *config.Config politeiad *pdclient.Client - sessions sessions.Sessions + sessions *sessions.Sessions events *events.Manager } @@ -251,9 +251,11 @@ func (t *TicketVote) HandleTimestamps(w http.ResponseWriter, r *http.Request) { } // New returns a new TicketVote context. -func New(cfg *config.Config, politeiad *pdclient.Client) *TicketVote { +func New(cfg *config.Config, pdc *pdclient.Client, s *sessions.Sessions, e *events.Manager) *TicketVote { return &TicketVote{ cfg: cfg, - politeiad: politeiad, + politeiad: pdc, + sessions: s, + events: e, } } diff --git a/politeiawww/www.go b/politeiawww/www.go index 9584ef52b..373bb65e4 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -35,7 +35,6 @@ import ( database "github.com/decred/politeia/politeiawww/cmsdatabase" cmsdb "github.com/decred/politeia/politeiawww/cmsdatabase/cockroachdb" ghtracker "github.com/decred/politeia/politeiawww/codetracker/github" - "github.com/decred/politeia/politeiawww/comments" "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/events" "github.com/decred/politeia/politeiawww/sessions" @@ -463,46 +462,6 @@ func (p *politeiawww) addRoute(method string, routeVersion string, route string, } } -func (p *politeiawww) setupPi() error { - // Setup api contexts - c := comments.New(p.cfg, p.politeiad, p.db) - - // Setup routes - p.setUserWWWRoutes() - p.setPiRoutes(c) - - // Verify paywall settings - switch { - case p.cfg.PaywallAmount != 0 && p.cfg.PaywallXpub != "": - // Paywall is enabled - paywallAmountInDcr := float64(p.cfg.PaywallAmount) / 1e8 - log.Infof("Paywall : %v DCR", paywallAmountInDcr) - - case p.cfg.PaywallAmount == 0 && p.cfg.PaywallXpub == "": - // Paywall is disabled - log.Infof("Paywall: DISABLED") - - default: - // Invalid paywall setting - return fmt.Errorf("paywall settings invalid, both an amount " + - "and public key MUST be set") - } - - // Setup paywall pool - p.userPaywallPool = make(map[uuid.UUID]paywallPoolMember) - err := p.initPaywallChecker() - if err != nil { - return err - } - - // Setup event manager - p.setupEventListenersPi() - - // TODO Verify politeiad plugins - - return nil -} - func (p *politeiawww) setupCMS() error { // Setup routes p.setCMSWWWRoutes() From a1db093a4abdcbee4c6579d0a1f96836e1f27106 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 24 Jan 2021 16:47:24 -0600 Subject: [PATCH 252/449] Add records API. --- politeiad/api/v1/v1.go | 3 +- politeiad/backend/tlogbe/plugins/pi/cmds.go | 65 +---- politeiad/backend/tlogbe/plugins/pi/pi.go | 2 - politeiad/backend/tlogbe/tlog/testing.go | 10 +- politeiad/client/{pdv1.go => politeiad.go} | 0 politeiad/plugins/pi/pi.go | 34 +-- politeiawww/api/pi/v1/v1.go | 172 ++++++------- politeiawww/api/records/v1/v1.go | 195 ++++++++++++++- politeiawww/api/ticketvote/v1/v1.go | 3 +- politeiawww/comments/comments.go | 46 ++-- politeiawww/comments/error.go | 14 +- politeiawww/comments/events.go | 4 +- politeiawww/comments/process.go | 100 ++++---- politeiawww/piwww.go | 190 ++------------- politeiawww/records.go | 242 ------------------- politeiawww/records/error.go | 125 ++++++++++ politeiawww/records/events.go | 5 + politeiawww/records/log.go | 25 ++ politeiawww/records/process.go | 94 ++++++++ politeiawww/records/records.go | 253 ++++++++++++++++++++ politeiawww/ticketvote/error.go | 14 +- politeiawww/ticketvote/events.go | 6 +- politeiawww/ticketvote/process.go | 168 ++++++------- politeiawww/ticketvote/ticketvote.go | 52 ++-- 24 files changed, 1012 insertions(+), 810 deletions(-) rename politeiad/client/{pdv1.go => politeiad.go} (100%) delete mode 100644 politeiawww/records.go create mode 100644 politeiawww/records/error.go create mode 100644 politeiawww/records/events.go create mode 100644 politeiawww/records/log.go create mode 100644 politeiawww/records/process.go create mode 100644 politeiawww/records/records.go diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index cf0fffe9f..72c018de3 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -245,8 +245,7 @@ type Record struct { } // NewRecord creates a new record. It must include all files that are part of -// the record and it may contain an optional metatda record. Thet optional -// metadatarecord must be string encoded. +// the record and it may contain an optional metatda record. type NewRecord struct { Challenge string `json:"challenge"` // Random challenge Metadata []MetadataStream `json:"metadata"` // Metadata streams diff --git a/politeiad/backend/tlogbe/plugins/pi/cmds.go b/politeiad/backend/tlogbe/plugins/pi/cmds.go index eda04823f..cc172d312 100644 --- a/politeiad/backend/tlogbe/plugins/pi/cmds.go +++ b/politeiad/backend/tlogbe/plugins/pi/cmds.go @@ -8,7 +8,6 @@ import ( "encoding/json" "fmt" - "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/ticketvote" "github.com/decred/politeia/util" @@ -18,55 +17,6 @@ func tokenDecode(token string) ([]byte, error) { return util.TokenDecode(util.TokenTypeTlog, token) } -func convertPropStatusFromMDStatus(s backend.MDStatusT) pi.PropStatusT { - var status pi.PropStatusT - switch s { - case backend.MDStatusUnvetted, backend.MDStatusIterationUnvetted: - status = pi.PropStatusUnvetted - case backend.MDStatusVetted: - status = pi.PropStatusPublic - case backend.MDStatusCensored: - status = pi.PropStatusCensored - case backend.MDStatusArchived: - status = pi.PropStatusAbandoned - } - return status -} - -func (p *piPlugin) cmdProposalInv() (string, error) { - log.Tracef("cmdProposalInv") - - // Get full record inventory - ibs, err := p.backend.InventoryByStatus() - if err != nil { - return "", err - } - - // Convert MDStatus keys to human readable proposal statuses - unvetted := make(map[string][]string, len(ibs.Unvetted)) - vetted := make(map[string][]string, len(ibs.Vetted)) - for k, v := range ibs.Unvetted { - s := pi.PropStatuses[convertPropStatusFromMDStatus(k)] - unvetted[s] = v - } - for k, v := range ibs.Vetted { - s := pi.PropStatuses[convertPropStatusFromMDStatus(k)] - vetted[s] = v - } - - // Prepare reply - pir := pi.ProposalInvReply{ - Unvetted: unvetted, - Vetted: vetted, - } - reply, err := json.Marshal(pir) - if err != nil { - return "", err - } - - return string(reply), nil -} - func (p *piPlugin) voteSummary(token []byte) (*ticketvote.SummaryReply, error) { reply, err := p.backend.VettedPluginCmd(token, ticketvote.PluginID, ticketvote.CmdSummary, "") @@ -97,9 +47,12 @@ func (p *piPlugin) cmdVoteInv() (string, error) { // Get vote summaries for all finished proposal votes and // categorize by approved/rejected. - approved := make([]string, 0, len(ir.Finished)) - rejected := make([]string, 0, len(ir.Finished)) - for _, v := range ir.Finished { + var ( + finished = ir.Records[ticketvote.VoteStatusFinished] + approved = make([]string, 0, len(finished)) + rejected = make([]string, 0, len(finished)) + ) + for _, v := range finished { t, err := tokenDecode(v) if err != nil { return "", err @@ -117,9 +70,9 @@ func (p *piPlugin) cmdVoteInv() (string, error) { // Prepare reply vir := pi.VoteInventoryReply{ - Unauthorized: ir.Unauthorized, - Authorized: ir.Authorized, - Started: ir.Started, + Unauthorized: ir.Records[ticketvote.VoteStatusUnauthorized], + Authorized: ir.Records[ticketvote.VoteStatusAuthorized], + Started: ir.Records[ticketvote.VoteStatusStarted], Approved: approved, Rejected: rejected, BestBlock: ir.BestBlock, diff --git a/politeiad/backend/tlogbe/plugins/pi/pi.go b/politeiad/backend/tlogbe/plugins/pi/pi.go index 425bca797..d2b02b7a8 100644 --- a/politeiad/backend/tlogbe/plugins/pi/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi/pi.go @@ -47,8 +47,6 @@ func (p *piPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, log.Tracef("Cmd: %v %x %v %v", treeID, token, cmd, payload) switch cmd { - case pi.CmdProposalInv: - return p.cmdProposalInv() case pi.CmdVoteInv: return p.cmdVoteInv() } diff --git a/politeiad/backend/tlogbe/tlog/testing.go b/politeiad/backend/tlogbe/tlog/testing.go index b1a274c2c..a46bd8e64 100644 --- a/politeiad/backend/tlogbe/tlog/testing.go +++ b/politeiad/backend/tlogbe/tlog/testing.go @@ -5,8 +5,7 @@ package tlog import ( - "os" - "path/filepath" + "io/ioutil" "testing" "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" @@ -17,15 +16,14 @@ func newTestTlog(t *testing.T, tlogID, dataDir string, encrypt bool) *Tlog { t.Helper() // Setup datadir for this tlog instance - dataDir = filepath.Join(dataDir, tlogID) - err := os.MkdirAll(dataDir, 0700) + var err error + dataDir, err = ioutil.TempDir(dataDir, tlogID) if err != nil { t.Fatal(err) } // Setup key-value store - fp := filepath.Join(dataDir, defaultStoreDirname) - err = os.MkdirAll(fp, 0700) + fp, err := ioutil.TempDir(dataDir, defaultStoreDirname) if err != nil { t.Fatal(err) } diff --git a/politeiad/client/pdv1.go b/politeiad/client/politeiad.go similarity index 100% rename from politeiad/client/pdv1.go rename to politeiad/client/politeiad.go diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index 28d90907b..788a8ff75 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -2,25 +2,22 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -// Package pi provides a politeiad plugin for functionality that is specific to -// decred's proposal system. +// Package pi provides a plugin for functionality that is specific to decred's +// proposal system. package pi const ( PluginID = "pi" // Plugin commands - // TODO I might not need the ProposalInv command - CmdProposalInv = "proposalinv" // Get inventory by proposal status - CmdVoteInv = "voteinv" // Get inventory by vote status + CmdVoteInv = "voteinv" // Get inventory by vote status ) // ErrorCodeT represents a plugin error that was caused by the user. type ErrorCodeT int const ( - // TODO number - // User error codes + // TODO User error codes ErrorCodeInvalid ErrorCodeT = 0 ErrorCodePageSizeExceeded ErrorCodeT = iota ErrorCodePropTokenInvalid @@ -35,8 +32,7 @@ const ( ) var ( - // TODO fill in - // ErrorCodes contains the human readable errors. + // TODO ErrorCodes contains the human readable errors. ErrorCodes = map[ErrorCodeT]string{ ErrorCodeInvalid: "error code invalid", } @@ -82,9 +78,14 @@ const ( // PropStatusCensored represents a proposal that has been censored. PropStatusCensored PropStatusT = 3 + // PropStatusUnreviewedChanges is a deprecated proposal status that + // has only been included so that the proposal statuses map + // directly to the politeiad record statuses. + PropStatusUnreviewedChanges PropStatusT = 4 + // PropStatusAbandoned represents a proposal that has been // abandoned by the author. - PropStatusAbandoned PropStatusT = 4 + PropStatusAbandoned PropStatusT = 5 ) var ( @@ -127,19 +128,6 @@ type StatusChange struct { Timestamp int64 `json:"timestamp"` } -// ProposalInv retrieves the tokens of all proposals in the inventory. -type ProposalInv struct { - UserID string `json:"userid,omitempty"` -} - -// ProposalInvReply is the reply to the ProposalInv command. The returned maps -// contains map[status][]token where the status is the human readable proposal -// status and the token is the proposal token. -type ProposalInvReply struct { - Unvetted map[string][]string `json:"unvetted"` - Vetted map[string][]string `json:"vetted"` -} - // VoteInventory requests the tokens of all proposals in the inventory // categorized by their vote status. This call relies on the ticketvote // Inventory call, but breaks the Finished vote status out into Approved and diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 59787cf78..1f2242658 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -24,103 +24,102 @@ const ( RouteProposalEdit = "/proposal/edit" RouteProposalSetStatus = "/proposal/setstatus" RouteProposals = "/proposals" - RouteProposalInventory = "/proposals/inventory" // Vote routes RouteVoteInventory = "/votes/inventory" ) -// ErrorStatusT represents a user error status code. -type ErrorStatusT int +// ErrorCodeT represents a user error code. +type ErrorCodeT int const ( // Error status codes - ErrorStatusInvalid ErrorStatusT = 0 - ErrorStatusInputInvalid ErrorStatusT = 1 - ErrorStatusPageSizeExceeded ErrorStatusT = 2 + ErrorCodeInvalid ErrorCodeT = 0 + ErrorCodeInputInvalid ErrorCodeT = 1 + ErrorCodePageSizeExceeded ErrorCodeT = 2 // User errors - ErrorStatusUserRegistrationNotPaid ErrorStatusT = 100 - ErrorStatusUserBalanceInsufficient ErrorStatusT = 101 - ErrorStatusUnauthorized ErrorStatusT = 102 + ErrorCodeUserRegistrationNotPaid ErrorCodeT = 100 + ErrorCodeUserBalanceInsufficient ErrorCodeT = 101 + ErrorCodeUnauthorized ErrorCodeT = 102 // Signature errors - ErrorStatusPublicKeyInvalid ErrorStatusT = 200 - ErrorStatusSignatureInvalid ErrorStatusT = 201 + ErrorCodePublicKeyInvalid ErrorCodeT = 200 + ErrorCodeSignatureInvalid ErrorCodeT = 201 // Proposal errors // TODO number error codes - ErrorStatusFileCountInvalid ErrorStatusT = 300 - ErrorStatusFileNameInvalid ErrorStatusT = iota - ErrorStatusFileMIMEInvalid - ErrorStatusFileDigestInvalid - ErrorStatusFilePayloadInvalid - ErrorStatusIndexFileNameInvalid - ErrorStatusIndexFileCountInvalid - ErrorStatusIndexFileSizeInvalid - ErrorStatusTextFileCountInvalid - ErrorStatusImageFileCountInvalid - ErrorStatusImageFileSizeInvalid - ErrorStatusMetadataCountInvalid - ErrorStatusMetadataDigestInvalid - ErrorStatusMetadataPayloadInvalid - ErrorStatusPropNotFound - ErrorStatusPropMetadataNotFound - ErrorStatusPropTokenInvalid - ErrorStatusPropVersionInvalid - ErrorStatusPropNameInvalid - ErrorStatusPropLinkToInvalid - ErrorStatusPropLinkByInvalid - ErrorStatusPropStateInvalid - ErrorStatusPropStatusInvalid - ErrorStatusPropStatusChangeInvalid - ErrorStatusPropStatusChangeReasonInvalid - ErrorStatusNoPropChanges + ErrorCodeFileCountInvalid ErrorCodeT = 300 + ErrorCodeFileNameInvalid ErrorCodeT = iota + ErrorCodeFileMIMEInvalid + ErrorCodeFileDigestInvalid + ErrorCodeFilePayloadInvalid + ErrorCodeIndexFileNameInvalid + ErrorCodeIndexFileCountInvalid + ErrorCodeIndexFileSizeInvalid + ErrorCodeTextFileCountInvalid + ErrorCodeImageFileCountInvalid + ErrorCodeImageFileSizeInvalid + ErrorCodeMetadataCountInvalid + ErrorCodeMetadataDigestInvalid + ErrorCodeMetadataPayloadInvalid + ErrorCodePropNotFound + ErrorCodePropMetadataNotFound + ErrorCodePropTokenInvalid + ErrorCodePropVersionInvalid + ErrorCodePropNameInvalid + ErrorCodePropLinkToInvalid + ErrorCodePropLinkByInvalid + ErrorCodePropStateInvalid + ErrorCodePropStatusInvalid + ErrorCodePropStatusChangeInvalid + ErrorCodePropStatusChangeReasonInvalid + ErrorCodeNoPropChanges ) var ( - // ErrorStatus contains human readable error messages. + // ErrorCode contains human readable error messages. // TODO fill in error status messages - ErrorStatus = map[ErrorStatusT]string{ - ErrorStatusInvalid: "error status invalid", - ErrorStatusInputInvalid: "input invalid", - ErrorStatusPageSizeExceeded: "page size exceeded", + ErrorCode = map[ErrorCodeT]string{ + ErrorCodeInvalid: "error status invalid", + ErrorCodeInputInvalid: "input invalid", + ErrorCodePageSizeExceeded: "page size exceeded", // User errors - ErrorStatusUserRegistrationNotPaid: "user registration not paid", - ErrorStatusUserBalanceInsufficient: "user balance insufficient", - ErrorStatusUnauthorized: "user is unauthorized", + ErrorCodeUserRegistrationNotPaid: "user registration not paid", + ErrorCodeUserBalanceInsufficient: "user balance insufficient", + ErrorCodeUnauthorized: "user is unauthorized", // Signature errors - ErrorStatusPublicKeyInvalid: "public key invalid", - ErrorStatusSignatureInvalid: "signature invalid", + ErrorCodePublicKeyInvalid: "public key invalid", + ErrorCodeSignatureInvalid: "signature invalid", // Proposal errors - ErrorStatusFileCountInvalid: "file count invalid", - ErrorStatusFileNameInvalid: "file name invalid", - ErrorStatusFileMIMEInvalid: "file mime invalid", - ErrorStatusFileDigestInvalid: "file digest invalid", - ErrorStatusFilePayloadInvalid: "file payload invalid", - ErrorStatusIndexFileNameInvalid: "index filename invalid", - ErrorStatusIndexFileCountInvalid: "index file count invalid", - ErrorStatusIndexFileSizeInvalid: "index file size invalid", - ErrorStatusTextFileCountInvalid: "text file count invalid", - ErrorStatusImageFileCountInvalid: "file count invalid", - ErrorStatusImageFileSizeInvalid: "file size invalid", - ErrorStatusMetadataCountInvalid: "metadata count invalid", - ErrorStatusMetadataDigestInvalid: "metadata digest invalid", - ErrorStatusMetadataPayloadInvalid: "metadata pyaload invalid", - ErrorStatusPropMetadataNotFound: "proposal metadata not found", - ErrorStatusPropNameInvalid: "proposal name invalid", - ErrorStatusPropLinkToInvalid: "proposal link to invalid", - ErrorStatusPropLinkByInvalid: "proposal link by invalid", - ErrorStatusPropTokenInvalid: "proposal token invalid", - ErrorStatusPropNotFound: "proposal not found", - ErrorStatusPropStateInvalid: "proposal state invalid", - ErrorStatusPropStatusInvalid: "proposal status invalid", - ErrorStatusPropStatusChangeInvalid: "proposal status change invalid", - ErrorStatusPropStatusChangeReasonInvalid: "proposal status reason invalid", - ErrorStatusNoPropChanges: "no proposal changes", + ErrorCodeFileCountInvalid: "file count invalid", + ErrorCodeFileNameInvalid: "file name invalid", + ErrorCodeFileMIMEInvalid: "file mime invalid", + ErrorCodeFileDigestInvalid: "file digest invalid", + ErrorCodeFilePayloadInvalid: "file payload invalid", + ErrorCodeIndexFileNameInvalid: "index filename invalid", + ErrorCodeIndexFileCountInvalid: "index file count invalid", + ErrorCodeIndexFileSizeInvalid: "index file size invalid", + ErrorCodeTextFileCountInvalid: "text file count invalid", + ErrorCodeImageFileCountInvalid: "file count invalid", + ErrorCodeImageFileSizeInvalid: "file size invalid", + ErrorCodeMetadataCountInvalid: "metadata count invalid", + ErrorCodeMetadataDigestInvalid: "metadata digest invalid", + ErrorCodeMetadataPayloadInvalid: "metadata pyaload invalid", + ErrorCodePropMetadataNotFound: "proposal metadata not found", + ErrorCodePropNameInvalid: "proposal name invalid", + ErrorCodePropLinkToInvalid: "proposal link to invalid", + ErrorCodePropLinkByInvalid: "proposal link by invalid", + ErrorCodePropTokenInvalid: "proposal token invalid", + ErrorCodePropNotFound: "proposal not found", + ErrorCodePropStateInvalid: "proposal state invalid", + ErrorCodePropStatusInvalid: "proposal status invalid", + ErrorCodePropStatusChangeInvalid: "proposal status change invalid", + ErrorCodePropStatusChangeReasonInvalid: "proposal status reason invalid", + ErrorCodeNoPropChanges: "no proposal changes", } ) @@ -128,8 +127,8 @@ var ( // error that is caused by something that the user did (malformed input, bad // timing, etc). The HTTP status code will be 400. type UserErrorReply struct { - ErrorCode ErrorStatusT `json:"errorcode"` - ErrorContext string `json:"errorcontext"` + ErrorCode ErrorCodeT `json:"errorcode"` + ErrorContext string `json:"errorcontext"` } // Error satisfies the error interface. @@ -187,7 +186,7 @@ const ( PropStatusPublic PropStatusT = 2 // PropStatusCensored indicates that a proposal has been censored - // by an admin for violating the proposal guidlines.. Both unvetted + // by an admin for violating the proposal guidlines. Both unvetted // and vetted proposals can be censored so a proposal with this // status can have a state of either PropStateUnvetted or // PropStateVetted depending on whether the proposal was censored @@ -391,27 +390,10 @@ type ProposalsReply struct { Proposals map[string]ProposalRecord `json:"proposals"` // [token]Proposal } -// ProposalInventory retrieves the tokens of all proposals in the inventory, -// categorized by proposal state and proposal status, that match the provided -// filtering criteria. If no filtering criteria is provided then the full -// proposal inventory is returned. Unvetted proposal tokens are only returned -// to admins and the proposal author. -type ProposalInventory struct { - UserID string `json:"userid,omitempty"` -} - -// ProposalInventoryReply is the reply to the ProposalInventory command. The -// inventory maps contain map[status][]tokens where the status is the human -// readable proposal status, as defined by the PropStatuses map, and the tokens -// are a list of proposal tokens for that status. Each list is ordered by -// timestamp of the status change from newest to oldest. -type ProposalInventoryReply struct { - Unvetted map[string][]string `json:"unvetted"` - Vetted map[string][]string `json:"vetted"` -} - // VoteInventory retrieves the tokens of all public, non-abandoned proposals -// categorized by their vote status. +// categorized by their vote status. This is the same inventory as the +// ticketvote API returns except the Finished vote status is broken out into +// Approved and Rejected. type VoteInventory struct{} // VoteInventoryReply in the reply to the VoteInventory command. diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go index 0bd552f42..4a412f6e4 100644 --- a/politeiawww/api/records/v1/v1.go +++ b/politeiawww/api/records/v1/v1.go @@ -11,7 +11,20 @@ const ( APIRoute = "/records/v1" // Record routes + RouteNew = "/new" + RouteEdit = "/edit" + RouteSetStatus = "/setstatus" + RouteDetails = "/details" + RouteRecords = "/records" + RouteInventory = "/inventory" RouteTimestamps = "/timestamps" + + // User metadata routes + RouteUserRecords = "/userrecords" + + // Record states + RecordStateUnvetted = "unvetted" + RecordStateVetted = "vetted" ) // ErrorCodeT represents a user error code. @@ -74,20 +87,184 @@ func (e ServerErrorReply) Error() string { return fmt.Sprintf("server error: %v", e.ErrorCode) } -// StateT represents a record state. -type StateT int +// StatusT represents a record status. +type StatusT int const ( - // StateInvalid indicates an invalid record state. - StateInvalid StateT = 0 + // StatusInvalid is an invalid record status. + StatusInvalid StatusT = 0 + + // StatusUnreviewed indicates that a record has been submitted but + // has not been made public yet. A record with this status will + // have a state of unvetted. + StatusUnreviewed StatusT = 1 + + // StatusPublic indicates that a record has been made public. A + // record with this status will have a state of vetted. + StatusPublic StatusT = 2 - // StateUnvetted indicates a record has not been made public yet. - StateUnvetted StateT = 1 + // StatusCensored indicates that a record has been censored. The + // record state can be either unvetted or vetted depending on + // whether the record was censored before or after it was made + // public. All user submitted content of a censored record will + // have been permanently deleted. + StatusCensored StatusT = 3 - // StateVetted indicates a record has been made public. - StateVetted StateT = 2 + // StatusUnreviewedChanges has been deprecated. + StatusUnreviewedChanges StatusT = 4 + + // StatusArchived represents a record that has been archived. Both + // unvetted and vetted records can be marked as archived. Unlike + // with censored records, the user submitted content of an archived + // record is not deleted. + StatusArchived StatusT = 5 ) +var ( + // Statuses contains the human readable record statuses. + Statuses = map[StatusT]string{ + StatusInvalid: "invalid", + StatusUnreviewed: "unreviewed", + StatusPublic: "public", + StatusCensored: "censored", + StatusArchived: "archived", + } +) + +// File describes an individual file that is part of the record. +type File struct { + Name string `json:"name"` // Filename + MIME string `json:"mime"` // Mime type + Digest string `json:"digest"` // SHA256 digest of unencoded payload + Payload string `json:"payload"` // File content, base64 encoded +} + +// MetadataStream describes a record metadata stream. +type MetadataStream struct { + ID uint64 `json:"id"` + Payload string `json:"payload"` // JSON encoded +} + +// CensorshipRecord contains cryptographic proof that a record was accepted for +// review by the server. The proof is verifiable by the client. +type CensorshipRecord struct { + // Token is a random censorship token that is generated by the + // server. It serves as a unique identifier for the record. + Token string `json:"token"` + + // Merkle is the ordered merkle root of all files and metadata in + // in the record. + Merkle string `json:"merkle"` + + // Signature is the server signature of the Merkle+Token. + Signature string `json:"signature"` +} + +// Record represents a record and all of its content. +type Record struct { + State string `json:"state"` // Record state + Status StatusT `json:"status"` // Record status + Version string `json:"version"` // Version of this record + Timestamp int64 `json:"timestamp"` // Last update + Metadata []MetadataStream `json:"metadata"` // Metadata streams + Files []File `json:"files"` // User submitted files + + CensorshipRecord CensorshipRecord `json:"censorshiprecord"` +} + +// New submits a new record. +// +// Signature is the client signature of the record merkle root. The merkle root +// is the ordered merkle root of all record Files. +type New struct { + Files []File `json:"files"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` +} + +// NewReply is the reply to the New command. +type NewReply struct { + Record Record `json:"record"` +} + +// Edit edits an existing record. +// +// Signature is the client signature of the record merkle root. The merkle root +// is the ordered merkle root of all record Files. +type Edit struct { + State string `json:"state"` + Token string `json:"token"` + Files []File `json:"files"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` +} + +// EditReply is the reply to the Edit command. +type EditReply struct { + Record Record `json:"record"` +} + +// SetStatus sets the status of a record. Some status changes require a reason +// to be included. +// +// Signature is the client signature of the Token+Version+Status+Reason. +type SetStatus struct { + State string `json:"state"` + Token string `json:"token"` + Version string `json:"version"` + Status StatusT `json:"status"` + Reason string `json:"reason,omitempty"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` +} + +// SetStatusReply is the reply to the SetStatus command. +type SetStatusReply struct { + Record Record `json:"record"` +} + +// Details requests the details of a record. The full record will be returned. +type Details struct { + Token string `json:"token"` // Censorship token + State string `json:"state"` // Record state +} + +// DetailsReply is the reply to the Details command. +type DetailsReply struct { + Record Record `json:"record"` +} + +// RecordRequest is used to request a record version. If the version is +// omitted, the most recent version will be returned. +type RecordRequest struct { + Token string `json:"token"` + Version string `json:"version,omitempty"` +} + +// Records requests a batch of records. Only the record metadata is returned. +// The Details command must be used to retrieve the record files. +type Records struct { + State string `json:"state"` // Record state +} + +// RecordsReply is the reply to the Records command. Any tokens that did not +// correspond to a record will not be included in the reply. +type RecordsReply struct { + Records map[string]Record `json:"records"` // [token]Record +} + +// Inventory requests the tokens of all records in the inventory, categorized +// by record state and record status. +type Inventory struct{} + +// InventoryReply is the reply to the Inventory command. The returned maps are +// map[status][]token where the status is the human readable record status +// defined by the Statuses array in this package. +type InventoryReply struct { + Unvetted map[string][]string `json:"unvetted"` + Vetted map[string][]string `json:"vetted"` +} + // Proof contains an inclusion proof for the digest in the merkle root. The // ExtraData field is used by certain types of proofs to include additional // data that is required to validate the proof. @@ -120,7 +297,7 @@ type Timestamp struct { // version is omitted, the timestamps for the most recent version will be // returned. type Timestamps struct { - State StateT `json:"state"` + State string `json:"state"` Token string `json:"token"` Version string `json:"version,omitempty"` } diff --git a/politeiawww/api/ticketvote/v1/v1.go b/politeiawww/api/ticketvote/v1/v1.go index a8d14bf62..72f843c94 100644 --- a/politeiawww/api/ticketvote/v1/v1.go +++ b/politeiawww/api/ticketvote/v1/v1.go @@ -472,7 +472,8 @@ type LinkedFromReply struct { LinkedFrom map[string][]string `json:"linkedfrom"` } -// Inventory requests the record inventory categorized by vote status. +// Inventory requests the tokens of all records in the inventory, categorized +// by vote status. type Inventory struct{} // InventoryReply is the reply to the Inventory command. It contains the tokens diff --git a/politeiawww/comments/comments.go b/politeiawww/comments/comments.go index e3d0cca37..4d3f0aa77 100644 --- a/politeiawww/comments/comments.go +++ b/politeiawww/comments/comments.go @@ -9,7 +9,7 @@ import ( "net/http" pdclient "github.com/decred/politeia/politeiad/client" - cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" + v1 "github.com/decred/politeia/politeiawww/api/comments/v1" "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/events" "github.com/decred/politeia/politeiawww/sessions" @@ -17,7 +17,7 @@ import ( "github.com/decred/politeia/util" ) -// Comments is the context that handles the comments API. +// Comments is the context for the comments API. type Comments struct { cfg *config.Config politeiad *pdclient.Client @@ -30,12 +30,12 @@ type Comments struct { func (c *Comments) HandleNew(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleNew") - var n cmv1.New + var n v1.New decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&n); err != nil { respondWithError(w, r, "HandleNew: unmarshal", - cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodeInputInvalid, + v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeInputInvalid, }) return } @@ -61,12 +61,12 @@ func (c *Comments) HandleNew(w http.ResponseWriter, r *http.Request) { func (c *Comments) HandleVote(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleVote") - var v cmv1.Vote + var v v1.Vote decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&v); err != nil { respondWithError(w, r, "HandleVote: unmarshal", - cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodeInputInvalid, + v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeInputInvalid, }) return } @@ -92,12 +92,12 @@ func (c *Comments) HandleVote(w http.ResponseWriter, r *http.Request) { func (c *Comments) HandleDel(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleDel") - var d cmv1.Del + var d v1.Del decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&d); err != nil { respondWithError(w, r, "HandleDel: unmarshal", - cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodeInputInvalid, + v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeInputInvalid, }) return } @@ -123,12 +123,12 @@ func (c *Comments) HandleDel(w http.ResponseWriter, r *http.Request) { func (c *Comments) HandleCount(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleCount") - var ct cmv1.Count + var ct v1.Count decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&c); err != nil { respondWithError(w, r, "HandleCount: unmarshal", - cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodeInputInvalid, + v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeInputInvalid, }) return } @@ -147,12 +147,12 @@ func (c *Comments) HandleCount(w http.ResponseWriter, r *http.Request) { func (c *Comments) HandleComments(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleComments") - var cs cmv1.Comments + var cs v1.Comments decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&cs); err != nil { respondWithError(w, r, "HandleComments: unmarshal", - cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodeInputInvalid, + v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeInputInvalid, }) return } @@ -180,12 +180,12 @@ func (c *Comments) HandleComments(w http.ResponseWriter, r *http.Request) { func (c *Comments) HandleVotes(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleVotes") - var v cmv1.Votes + var v v1.Votes decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&v); err != nil { respondWithError(w, r, "HandleVotes: unmarshal", - cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodeInputInvalid, + v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeInputInvalid, }) return } @@ -205,12 +205,12 @@ func (c *Comments) HandleVotes(w http.ResponseWriter, r *http.Request) { func (c *Comments) HandleTimestamps(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleTimestamps") - var t cmv1.Timestamps + var t v1.Timestamps decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&t); err != nil { respondWithError(w, r, "HandleTimestamps: unmarshal", - cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodeInputInvalid, + v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeInputInvalid, }) return } diff --git a/politeiawww/comments/error.go b/politeiawww/comments/error.go index 2ba771837..e6608f6bf 100644 --- a/politeiawww/comments/error.go +++ b/politeiawww/comments/error.go @@ -13,26 +13,26 @@ import ( "time" pdclient "github.com/decred/politeia/politeiad/client" - cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" + v1 "github.com/decred/politeia/politeiawww/api/comments/v1" "github.com/decred/politeia/util" ) func respondWithError(w http.ResponseWriter, r *http.Request, format string, err error) { var ( - ue cmv1.UserErrorReply + ue v1.UserErrorReply pe pdclient.Error ) switch { case errors.As(err, &ue): // Comments user error m := fmt.Sprintf("Comments user error: %v %v %v", - util.RemoteAddr(r), ue.ErrorCode, cmv1.ErrorCodes[ue.ErrorCode]) + util.RemoteAddr(r), ue.ErrorCode, v1.ErrorCodes[ue.ErrorCode]) if ue.ErrorContext != "" { m += fmt.Sprintf(": %v", ue.ErrorContext) } log.Infof(m) util.RespondWithJSON(w, http.StatusBadRequest, - cmv1.UserErrorReply{ + v1.UserErrorReply{ ErrorCode: ue.ErrorCode, ErrorContext: ue.ErrorContext, }) @@ -55,7 +55,7 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err } log.Infof(m) util.RespondWithJSON(w, http.StatusBadRequest, - cmv1.PluginErrorReply{ + v1.PluginErrorReply{ PluginID: pluginID, ErrorCode: errCode, ErrorContext: strings.Join(errContext, ", "), @@ -70,7 +70,7 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err r.Proto, ts, errCode) util.RespondWithJSON(w, http.StatusInternalServerError, - cmv1.ServerErrorReply{ + v1.ServerErrorReply{ ErrorCode: ts, }) return @@ -85,7 +85,7 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) util.RespondWithJSON(w, http.StatusInternalServerError, - cmv1.ServerErrorReply{ + v1.ServerErrorReply{ ErrorCode: t, }) return diff --git a/politeiawww/comments/events.go b/politeiawww/comments/events.go index 99a0a9e0e..8ab3aa439 100644 --- a/politeiawww/comments/events.go +++ b/politeiawww/comments/events.go @@ -4,7 +4,7 @@ package comments -import cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" +import v1 "github.com/decred/politeia/politeiawww/api/comments/v1" const ( // EventTypeNew is emitted when a new comment is made. @@ -14,5 +14,5 @@ const ( // EventNew is the event data for EventTypeNew. type EventNew struct { State string - Comment cmv1.Comment + Comment v1.Comment } diff --git a/politeiawww/comments/process.go b/politeiawww/comments/process.go index 46e1af27b..2ca5e81fb 100644 --- a/politeiawww/comments/process.go +++ b/politeiawww/comments/process.go @@ -8,13 +8,13 @@ import ( "context" "github.com/decred/politeia/politeiad/plugins/comments" - cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" + v1 "github.com/decred/politeia/politeiawww/api/comments/v1" "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/user" "github.com/google/uuid" ) -func (c *Comments) processNew(ctx context.Context, n cmv1.New, u user.User) (*cmv1.NewReply, error) { +func (c *Comments) processNew(ctx context.Context, n v1.New, u user.User) (*v1.NewReply, error) { log.Tracef("processNew: %v %v %v", n.Token, u.Username) // Checking the mode is a temporary measure until user plugins @@ -23,31 +23,31 @@ func (c *Comments) processNew(ctx context.Context, n cmv1.New, u user.User) (*cm case config.PoliteiaWWWMode: // Verify user has paid registration paywall if !c.userHasPaid(u) { - return nil, cmv1.UserErrorReply{ - // TODO ErrorCode: cmv1.ErrorCodeUserRegistrationNotPaid, + return nil, v1.UserErrorReply{ + // TODO ErrorCode: v1.ErrorCodeUserRegistrationNotPaid, } } } // Verify user signed using active identity if u.PublicKey() != n.PublicKey { - return nil, cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodePublicKeyInvalid, + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodePublicKeyInvalid, ErrorContext: "not active identity", } } // Only admins and the record author are allowed to comment on // unvetted records. - if n.State == cmv1.RecordStateUnvetted && !u.Admin { + if n.State == v1.RecordStateUnvetted && !u.Admin { // User is not an admin. Get the record author. authorID, err := c.politeiad.Author(ctx, n.State, n.Token) if err != nil { return nil, err } if u.ID.String() != authorID { - return nil, cmv1.UserErrorReply{ - // TODO ErrorCode: cmv1.ErrorCodeUnauthorized, + return nil, v1.UserErrorReply{ + // TODO ErrorCode: v1.ErrorCodeUnauthorized, ErrorContext: "user is not author or admin", } } @@ -78,12 +78,12 @@ func (c *Comments) processNew(ctx context.Context, n cmv1.New, u user.User) (*cm Comment: cm, }) - return &cmv1.NewReply{ + return &v1.NewReply{ Comment: cm, }, nil } -func (c *Comments) processVote(ctx context.Context, v cmv1.Vote, u user.User) (*cmv1.VoteReply, error) { +func (c *Comments) processVote(ctx context.Context, v v1.Vote, u user.User) (*v1.VoteReply, error) { log.Tracef("processVote: %v %v %v", v.Token, v.CommentID, v.Vote) // Checking the mode is a temporary measure until user plugins @@ -92,24 +92,24 @@ func (c *Comments) processVote(ctx context.Context, v cmv1.Vote, u user.User) (* case config.PoliteiaWWWMode: // Verify user has paid registration paywall if !c.userHasPaid(u) { - return nil, cmv1.UserErrorReply{ - // ErrorCode: cmv1.ErrorCodeUserRegistrationNotPaid, + return nil, v1.UserErrorReply{ + // ErrorCode: v1.ErrorCodeUserRegistrationNotPaid, } } } // Verify state - if v.State != cmv1.RecordStateVetted { - return nil, cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodeRecordStateInvalid, + if v.State != v1.RecordStateVetted { + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeRecordStateInvalid, ErrorContext: "record must be vetted", } } // Verify user signed using active identity if u.PublicKey() != v.PublicKey { - return nil, cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodePublicKeyInvalid, + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodePublicKeyInvalid, ErrorContext: "not active identity", } } @@ -128,7 +128,7 @@ func (c *Comments) processVote(ctx context.Context, v cmv1.Vote, u user.User) (* return nil, err } - return &cmv1.VoteReply{ + return &v1.VoteReply{ Downvotes: vr.Downvotes, Upvotes: vr.Upvotes, Timestamp: vr.Timestamp, @@ -136,13 +136,13 @@ func (c *Comments) processVote(ctx context.Context, v cmv1.Vote, u user.User) (* }, nil } -func (c *Comments) processDel(ctx context.Context, d cmv1.Del, u user.User) (*cmv1.DelReply, error) { +func (c *Comments) processDel(ctx context.Context, d v1.Del, u user.User) (*v1.DelReply, error) { log.Tracef("processDel: %v %v %v", d.Token, d.CommentID, d.Reason) // Verify user signed with their active identity if u.PublicKey() != d.PublicKey { - return nil, cmv1.UserErrorReply{ - ErrorCode: cmv1.ErrorCodePublicKeyInvalid, + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodePublicKeyInvalid, ErrorContext: "not active identity", } } @@ -164,12 +164,12 @@ func (c *Comments) processDel(ctx context.Context, d cmv1.Del, u user.User) (*cm cm := convertComment(cdr.Comment) cm = commentPopulateUser(cm, u) - return &cmv1.DelReply{ + return &v1.DelReply{ Comment: cm, }, nil } -func (c *Comments) processCount(ctx context.Context, ct cmv1.Count) (*cmv1.CountReply, error) { +func (c *Comments) processCount(ctx context.Context, ct v1.Count) (*v1.CountReply, error) { log.Tracef("processCount: %v", ct.Tokens) counts, err := c.politeiad.CommentCounts(ctx, ct.State, ct.Tokens) @@ -177,18 +177,18 @@ func (c *Comments) processCount(ctx context.Context, ct cmv1.Count) (*cmv1.Count return nil, err } - return &cmv1.CountReply{ + return &v1.CountReply{ Counts: counts, }, nil } -func (c *Comments) processComments(ctx context.Context, cs cmv1.Comments, u *user.User) (*cmv1.CommentsReply, error) { +func (c *Comments) processComments(ctx context.Context, cs v1.Comments, u *user.User) (*v1.CommentsReply, error) { log.Tracef("processComments: %v", cs.Token) // Only admins and the record author are allowed to retrieve // unvetted comments. This is a public route so a user might // not exist. - if cs.State == cmv1.RecordStateUnvetted { + if cs.State == v1.RecordStateUnvetted { var isAllowed bool switch { case u == nil: @@ -209,8 +209,8 @@ func (c *Comments) processComments(ctx context.Context, cs cmv1.Comments, u *use } } if !isAllowed { - return nil, cmv1.UserErrorReply{ - // TODO ErrorCode: cmv1.ErrorCodeUnauthorized, + return nil, v1.UserErrorReply{ + // TODO ErrorCode: v1.ErrorCodeUnauthorized, ErrorContext: "user is not author or admin", } } @@ -224,7 +224,7 @@ func (c *Comments) processComments(ctx context.Context, cs cmv1.Comments, u *use // Prepare reply. Comment user data must be pulled from the // userdb. - comments := make([]cmv1.Comment, 0, len(pcomments)) + comments := make([]v1.Comment, 0, len(pcomments)) for _, v := range pcomments { cm := convertComment(v) @@ -243,12 +243,12 @@ func (c *Comments) processComments(ctx context.Context, cs cmv1.Comments, u *use comments = append(comments, cm) } - return &cmv1.CommentsReply{ + return &v1.CommentsReply{ Comments: comments, }, nil } -func (c *Comments) processVotes(ctx context.Context, v cmv1.Votes) (*cmv1.VotesReply, error) { +func (c *Comments) processVotes(ctx context.Context, v v1.Votes) (*v1.VotesReply, error) { log.Tracef("processVotes: %v %v", v.Token, v.UserID) // Get comment votes @@ -260,12 +260,12 @@ func (c *Comments) processVotes(ctx context.Context, v cmv1.Votes) (*cmv1.VotesR return nil, err } - return &cmv1.VotesReply{ + return &v1.VotesReply{ Votes: convertCommentVotes(votes), }, nil } -func (c *Comments) processTimestamps(ctx context.Context, t cmv1.Timestamps, isAdmin bool) (*cmv1.TimestampsReply, error) { +func (c *Comments) processTimestamps(ctx context.Context, t v1.Timestamps, isAdmin bool) (*v1.TimestampsReply, error) { log.Tracef("processTimestamps: %v %v", t.Token, t.CommentIDs) // Get timestamps @@ -278,12 +278,12 @@ func (c *Comments) processTimestamps(ctx context.Context, t cmv1.Timestamps, isA } // Prepare reply - comments := make(map[uint32][]cmv1.Timestamp, len(ctr.Comments)) + comments := make(map[uint32][]v1.Timestamp, len(ctr.Comments)) for commentID, timestamps := range ctr.Comments { - ts := make([]cmv1.Timestamp, 0, len(timestamps)) + ts := make([]v1.Timestamp, 0, len(timestamps)) for _, v := range timestamps { // Strip unvetted data blobs if the user is not an admin - if t.State == cmv1.RecordStateUnvetted && !isAdmin { + if t.State == v1.RecordStateUnvetted && !isAdmin { v.Data = "" } ts = append(ts, convertTimestamp(v)) @@ -291,22 +291,22 @@ func (c *Comments) processTimestamps(ctx context.Context, t cmv1.Timestamps, isA comments[commentID] = ts } - return &cmv1.TimestampsReply{ + return &v1.TimestampsReply{ Comments: comments, }, nil } // commentPopulateUserData populates the comment with user data that is not // stored in politeiad. -func commentPopulateUser(c cmv1.Comment, u user.User) cmv1.Comment { +func commentPopulateUser(c v1.Comment, u user.User) v1.Comment { c.Username = u.Username return c } -func convertComment(c comments.Comment) cmv1.Comment { +func convertComment(c comments.Comment) v1.Comment { // Fields that are intentionally omitted are not stored in // politeiad. They need to be pulled from the userdb. - return cmv1.Comment{ + return v1.Comment{ UserID: c.UserID, Username: "", // Intentionally omitted Token: c.Token, @@ -326,14 +326,14 @@ func convertComment(c comments.Comment) cmv1.Comment { } } -func convertCommentVotes(cv []comments.CommentVote) []cmv1.CommentVote { - c := make([]cmv1.CommentVote, 0, len(cv)) +func convertCommentVotes(cv []comments.CommentVote) []v1.CommentVote { + c := make([]v1.CommentVote, 0, len(cv)) for _, v := range cv { - c = append(c, cmv1.CommentVote{ + c = append(c, v1.CommentVote{ UserID: v.UserID, Token: v.Token, CommentID: v.CommentID, - Vote: cmv1.VoteT(v.Vote), + Vote: v1.VoteT(v.Vote), PublicKey: v.PublicKey, Signature: v.Signature, Timestamp: v.Timestamp, @@ -343,8 +343,8 @@ func convertCommentVotes(cv []comments.CommentVote) []cmv1.CommentVote { return c } -func convertProof(p comments.Proof) cmv1.Proof { - return cmv1.Proof{ +func convertProof(p comments.Proof) v1.Proof { + return v1.Proof{ Type: p.Type, Digest: p.Digest, MerkleRoot: p.MerkleRoot, @@ -353,12 +353,12 @@ func convertProof(p comments.Proof) cmv1.Proof { } } -func convertTimestamp(t comments.Timestamp) cmv1.Timestamp { - proofs := make([]cmv1.Proof, 0, len(t.Proofs)) +func convertTimestamp(t comments.Timestamp) v1.Timestamp { + proofs := make([]v1.Proof, 0, len(t.Proofs)) for _, v := range t.Proofs { proofs = append(proofs, convertProof(v)) } - return cmv1.Timestamp{ + return v1.Timestamp{ Data: t.Data, Digest: t.Digest, TxID: t.TxID, diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index ceefac829..c5ed55d77 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -29,7 +29,6 @@ import ( tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/comments" - "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/politeiawww/ticketvote" "github.com/decred/politeia/politeiawww/user" wwwutil "github.com/decred/politeia/politeiawww/util" @@ -1216,161 +1215,6 @@ func (p *politeiawww) processVoteInventory(ctx context.Context) (*piv1.VoteInven }, nil } -func (p *politeiawww) handleProposalNew(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleProposalNew") - - var pn piv1.ProposalNew - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&pn); err != nil { - respondWithPiError(w, r, "handleProposalNew: unmarshal", - piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusInputInvalid, - }) - return - } - - user, err := p.sessions.GetSessionUser(w, r) - if err != nil { - respondWithPiError(w, r, - "handleProposalNew: GetSessionUser: %v", err) - return - } - - pnr, err := p.processProposalNew(r.Context(), pn, *user) - if err != nil { - respondWithPiError(w, r, - "handleProposalNew: processProposalNew: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, pnr) -} - -func (p *politeiawww) handleProposalEdit(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleProposalEdit") - - var pe piv1.ProposalEdit - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&pe); err != nil { - respondWithPiError(w, r, "handleProposalEdit: unmarshal", - piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusInputInvalid, - }) - return - } - - user, err := p.sessions.GetSessionUser(w, r) - if err != nil { - respondWithPiError(w, r, - "handleProposalEdit: GetSessionUser: %v", err) - return - } - - per, err := p.processProposalEdit(r.Context(), pe, *user) - if err != nil { - respondWithPiError(w, r, - "handleProposalEdit: processProposalEdit: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, per) -} - -func (p *politeiawww) handleProposalSetStatus(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleProposalSetStatus") - - var pss piv1.ProposalSetStatus - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&pss); err != nil { - respondWithPiError(w, r, "handleProposalSetStatus: unmarshal", - piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusInputInvalid, - }) - return - } - - usr, err := p.sessions.GetSessionUser(w, r) - if err != nil { - respondWithPiError(w, r, - "handleProposalSetStatus: GetSessionUser: %v", err) - return - } - - pssr, err := p.processProposalSetStatus(r.Context(), pss, *usr) - if err != nil { - respondWithPiError(w, r, - "handleProposalSetStatus: processProposalSetStatus: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, pssr) -} - -func (p *politeiawww) handleProposals(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleProposals") - - var ps piv1.Proposals - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&ps); err != nil { - respondWithPiError(w, r, "handleProposals: unmarshal", - piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusInputInvalid, - }) - return - } - - // Lookup session user. This is a public route so a session may not - // exist. Ignore any session not found errors. - usr, err := p.sessions.GetSessionUser(w, r) - if err != nil && err != sessions.ErrSessionNotFound { - respondWithPiError(w, r, - "handleProposals: GetSessionUser: %v", err) - return - } - - isAdmin := usr != nil && usr.Admin - ppi, err := p.processProposals(r.Context(), ps, isAdmin) - if err != nil { - respondWithPiError(w, r, - "handleProposals: processProposals: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, ppi) -} - -func (p *politeiawww) handleProposalInventory(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleProposalInventory") - - var inv piv1.ProposalInventory - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&inv); err != nil { - respondWithPiError(w, r, "handleProposalInventory: unmarshal", - piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusInputInvalid, - }) - return - } - - // Lookup session user. This is a public route so a session may not - // exist. Ignore any session not found errors. - usr, err := p.sessions.GetSessionUser(w, r) - if err != nil && err != sessions.ErrSessionNotFound { - respondWithPiError(w, r, - "handleProposalInventory: GetSessionUser: %v", err) - return - } - - pir, err := p.processProposalInventory(r.Context(), inv, usr) - if err != nil { - respondWithPiError(w, r, - "handleProposalInventory: processProposalInventory: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, pir) -} - // setPiRoutes sets the pi API routes. func (p *politeiawww) setPiRoutes(c *comments.Comments, t *ticketvote.TicketVote) { // Return a 404 when a route is not found @@ -1412,22 +1256,24 @@ func (p *politeiawww) setPiRoutes(c *comments.Comments, t *ticketvote.TicketVote www.RouteBatchVoteSummary, p.handleBatchVoteSummary, permissionPublic) - // Pi routes - proposals - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteProposalNew, p.handleProposalNew, - permissionLogin) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteProposalEdit, p.handleProposalEdit, - permissionLogin) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteProposalSetStatus, p.handleProposalSetStatus, - permissionAdmin) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteProposals, p.handleProposals, - permissionPublic) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteProposalInventory, p.handleProposalInventory, - permissionPublic) + /* + // Pi routes - proposals + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteProposalNew, p.handleProposalNew, + permissionLogin) + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteProposalEdit, p.handleProposalEdit, + permissionLogin) + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteProposalSetStatus, p.handleProposalSetStatus, + permissionAdmin) + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteProposals, p.handleProposals, + permissionPublic) + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteProposalInventory, p.handleProposalInventory, + permissionPublic) + */ // Comment routes p.addRoute(http.MethodPost, cmv1.APIRoute, diff --git a/politeiawww/records.go b/politeiawww/records.go deleted file mode 100644 index 6b8c60d4c..000000000 --- a/politeiawww/records.go +++ /dev/null @@ -1,242 +0,0 @@ -// Copyright (c) 2020-2021 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "net/http" - "runtime/debug" - "strings" - "time" - - pdv1 "github.com/decred/politeia/politeiad/api/v1" - rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" - "github.com/decred/politeia/politeiawww/sessions" - "github.com/decred/politeia/util" -) - -func (p *politeiawww) processTimestamps(ctx context.Context, t rcv1.Timestamps, isAdmin bool) (*rcv1.TimestampsReply, error) { - log.Tracef("processTimestamps: %v %v %v", t.State, t.Token, t.Version) - - // Get record timestamps - var ( - rt *pdv1.RecordTimestamps - err error - ) - switch t.State { - case rcv1.StateUnvetted: - rt, err = p.politeiad.GetUnvettedTimestamps(ctx, t.Token, t.Version) - if err != nil { - return nil, err - } - case rcv1.StateVetted: - rt, err = p.politeiad.GetVettedTimestamps(ctx, t.Token, t.Version) - if err != nil { - return nil, err - } - default: - return nil, rcv1.UserErrorReply{ - ErrorCode: rcv1.ErrorCodeRecordStateInvalid, - } - } - - var ( - recordMD = convertTimestampFromPD(rt.RecordMetadata) - metadata = make(map[uint64]rcv1.Timestamp, len(rt.Files)) - files = make(map[string]rcv1.Timestamp, len(rt.Files)) - ) - for k, v := range rt.Metadata { - metadata[k] = convertTimestampFromPD(v) - } - for k, v := range rt.Files { - files[k] = convertTimestampFromPD(v) - } - - // Unvetted data blobs are stripped if the user is not an admin. - // The rest of the timestamp is still returned. - if t.State != rcv1.StateVetted && !isAdmin { - recordMD.Data = "" - for k, v := range files { - v.Data = "" - files[k] = v - } - for k, v := range metadata { - v.Data = "" - metadata[k] = v - } - } - - return &rcv1.TimestampsReply{ - RecordMetadata: recordMD, - Files: files, - Metadata: metadata, - }, nil -} - -func (p *politeiawww) handleTimestamps(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleTimestamps") - - var t rcv1.Timestamps - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&t); err != nil { - respondWithRecordError(w, r, "handleTimestamps: unmarshal", - rcv1.UserErrorReply{ - ErrorCode: rcv1.ErrorCodeInputInvalid, - }) - return - } - - // Lookup session user. This is a public route so a session may not - // exist. Ignore any session not found errors. - usr, err := p.sessions.GetSessionUser(w, r) - if err != nil && err != sessions.ErrSessionNotFound { - respondWithRecordError(w, r, - "handleTimestamps: getSessionUser: %v", err) - return - } - - isAdmin := usr != nil && usr.Admin - tr, err := p.processTimestamps(r.Context(), t, isAdmin) - if err != nil { - respondWithRecordError(w, r, - "handleTimestamps: processTimestamps: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, tr) -} - -func convertRecordsErrorCode(errCode int) rcv1.ErrorCodeT { - switch pdv1.ErrorStatusT(errCode) { - case pdv1.ErrorStatusInvalidRequestPayload: - // Intentionally omitted. This indicates an internal server error. - case pdv1.ErrorStatusInvalidChallenge: - // Intentionally omitted. This indicates an internal server error. - case pdv1.ErrorStatusRecordNotFound: - return rcv1.ErrorCodeRecordNotFound - } - // No record API error code found - return rcv1.ErrorCodeInvalid -} - -func respondWithRecordError(w http.ResponseWriter, r *http.Request, format string, err error) { - var ( - ue rcv1.UserErrorReply - pe pdError - ) - switch { - case errors.As(err, &ue): - // Record user error - m := fmt.Sprintf("Records user error: %v %v %v", - util.RemoteAddr(r), ue.ErrorCode, rcv1.ErrorCodes[ue.ErrorCode]) - if ue.ErrorContext != "" { - m += fmt.Sprintf(": %v", ue.ErrorContext) - } - log.Infof(m) - util.RespondWithJSON(w, http.StatusBadRequest, - rcv1.UserErrorReply{ - ErrorCode: ue.ErrorCode, - ErrorContext: ue.ErrorContext, - }) - return - - case errors.As(err, &pe): - // Politeiad error - var ( - pluginID = pe.ErrorReply.Plugin - errCode = pe.ErrorReply.ErrorCode - errContext = pe.ErrorReply.ErrorContext - ) - e := convertRecordsErrorCode(errCode) - switch { - case pluginID != "": - // politeiad plugin error. Log it and return a 400. - m := fmt.Sprintf("Plugin error: %v %v %v", - util.RemoteAddr(r), pluginID, errCode) - if len(errContext) > 0 { - m += fmt.Sprintf(": %v", strings.Join(errContext, ", ")) - } - log.Infof(m) - util.RespondWithJSON(w, http.StatusBadRequest, - rcv1.PluginErrorReply{ - PluginID: pluginID, - ErrorCode: errCode, - ErrorContext: strings.Join(errContext, ", "), - }) - return - - case e == rcv1.ErrorCodeInvalid: - // politeiad error does not correspond to a user error. Log it - // and return a 500. - ts := time.Now().Unix() - log.Errorf("%v %v %v %v Internal error %v: error code "+ - "from politeiad: %v", util.RemoteAddr(r), r.Method, r.URL, - r.Proto, ts, errCode) - - util.RespondWithJSON(w, http.StatusInternalServerError, - rcv1.ServerErrorReply{ - ErrorCode: ts, - }) - return - - default: - // politeiad error does correspond to a user error. Log it and - // return a 400. - m := fmt.Sprintf("Records user error: %v %v %v", - util.RemoteAddr(r), e, rcv1.ErrorCodes[e]) - if len(errContext) > 0 { - m += fmt.Sprintf(": %v", strings.Join(errContext, ", ")) - } - log.Infof(m) - util.RespondWithJSON(w, http.StatusBadRequest, - rcv1.UserErrorReply{ - ErrorCode: e, - ErrorContext: strings.Join(errContext, ", "), - }) - return - } - - default: - // Internal server error. Log it and return a 500. - t := time.Now().Unix() - e := fmt.Sprintf(format, err) - log.Errorf("%v %v %v %v Internal error %v: %v", - util.RemoteAddr(r), r.Method, r.URL, r.Proto, t, e) - log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) - - util.RespondWithJSON(w, http.StatusInternalServerError, - rcv1.ServerErrorReply{ - ErrorCode: t, - }) - return - } -} - -func convertProofFromPD(p pdv1.Proof) rcv1.Proof { - return rcv1.Proof{ - Type: p.Type, - Digest: p.Digest, - MerkleRoot: p.MerkleRoot, - MerklePath: p.MerklePath, - ExtraData: p.ExtraData, - } -} - -func convertTimestampFromPD(t pdv1.Timestamp) rcv1.Timestamp { - proofs := make([]rcv1.Proof, 0, len(t.Proofs)) - for _, v := range t.Proofs { - proofs = append(proofs, convertProofFromPD(v)) - } - return rcv1.Timestamp{ - Data: t.Data, - Digest: t.Digest, - TxID: t.TxID, - MerkleRoot: t.MerkleRoot, - Proofs: proofs, - } -} diff --git a/politeiawww/records/error.go b/politeiawww/records/error.go new file mode 100644 index 000000000..8da9dd1ed --- /dev/null +++ b/politeiawww/records/error.go @@ -0,0 +1,125 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package records + +import ( + "errors" + "fmt" + "net/http" + "runtime/debug" + "strings" + "time" + + pdv1 "github.com/decred/politeia/politeiad/api/v1" + pdclient "github.com/decred/politeia/politeiad/client" + v1 "github.com/decred/politeia/politeiawww/api/records/v1" + "github.com/decred/politeia/util" +) + +func convertRecordsErrorCode(errCode int) v1.ErrorCodeT { + switch pdv1.ErrorStatusT(errCode) { + case pdv1.ErrorStatusInvalidRequestPayload: + // Intentionally omitted. This indicates an internal server error. + case pdv1.ErrorStatusInvalidChallenge: + // Intentionally omitted. This indicates an internal server error. + case pdv1.ErrorStatusRecordNotFound: + return v1.ErrorCodeRecordNotFound + } + // No record API error code found + return v1.ErrorCodeInvalid +} + +func respondWithRecordError(w http.ResponseWriter, r *http.Request, format string, err error) { + var ( + ue v1.UserErrorReply + pe pdclient.Error + ) + switch { + case errors.As(err, &ue): + // Record user error + m := fmt.Sprintf("Records user error: %v %v %v", + util.RemoteAddr(r), ue.ErrorCode, v1.ErrorCodes[ue.ErrorCode]) + if ue.ErrorContext != "" { + m += fmt.Sprintf(": %v", ue.ErrorContext) + } + log.Infof(m) + util.RespondWithJSON(w, http.StatusBadRequest, + v1.UserErrorReply{ + ErrorCode: ue.ErrorCode, + ErrorContext: ue.ErrorContext, + }) + return + + case errors.As(err, &pe): + // Politeiad error + var ( + pluginID = pe.ErrorReply.PluginID + errCode = pe.ErrorReply.ErrorCode + errContext = pe.ErrorReply.ErrorContext + ) + e := convertRecordsErrorCode(errCode) + switch { + case pluginID != "": + // politeiad plugin error. Log it and return a 400. + m := fmt.Sprintf("Plugin error: %v %v %v", + util.RemoteAddr(r), pluginID, errCode) + if len(errContext) > 0 { + m += fmt.Sprintf(": %v", strings.Join(errContext, ", ")) + } + log.Infof(m) + util.RespondWithJSON(w, http.StatusBadRequest, + v1.PluginErrorReply{ + PluginID: pluginID, + ErrorCode: errCode, + ErrorContext: strings.Join(errContext, ", "), + }) + return + + case e == v1.ErrorCodeInvalid: + // politeiad error does not correspond to a user error. Log it + // and return a 500. + ts := time.Now().Unix() + log.Errorf("%v %v %v %v Internal error %v: error code "+ + "from politeiad: %v", util.RemoteAddr(r), r.Method, r.URL, + r.Proto, ts, errCode) + + util.RespondWithJSON(w, http.StatusInternalServerError, + v1.ServerErrorReply{ + ErrorCode: ts, + }) + return + + default: + // politeiad error does correspond to a user error. Log it and + // return a 400. + m := fmt.Sprintf("Records user error: %v %v %v", + util.RemoteAddr(r), e, v1.ErrorCodes[e]) + if len(errContext) > 0 { + m += fmt.Sprintf(": %v", strings.Join(errContext, ", ")) + } + log.Infof(m) + util.RespondWithJSON(w, http.StatusBadRequest, + v1.UserErrorReply{ + ErrorCode: e, + ErrorContext: strings.Join(errContext, ", "), + }) + return + } + + default: + // Internal server error. Log it and return a 500. + t := time.Now().Unix() + e := fmt.Sprintf(format, err) + log.Errorf("%v %v %v %v Internal error %v: %v", + util.RemoteAddr(r), r.Method, r.URL, r.Proto, t, e) + log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) + + util.RespondWithJSON(w, http.StatusInternalServerError, + v1.ServerErrorReply{ + ErrorCode: t, + }) + return + } +} diff --git a/politeiawww/records/events.go b/politeiawww/records/events.go new file mode 100644 index 000000000..b5ae42240 --- /dev/null +++ b/politeiawww/records/events.go @@ -0,0 +1,5 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package records diff --git a/politeiawww/records/log.go b/politeiawww/records/log.go new file mode 100644 index 000000000..12d150b9f --- /dev/null +++ b/politeiawww/records/log.go @@ -0,0 +1,25 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package records + +import "github.com/decred/slog" + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log = slog.Disabled + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = slog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using slog. +func UseLogger(logger slog.Logger) { + log = logger +} diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go new file mode 100644 index 000000000..881363cb6 --- /dev/null +++ b/politeiawww/records/process.go @@ -0,0 +1,94 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package records + +import ( + "context" + + pdv1 "github.com/decred/politeia/politeiad/api/v1" + v1 "github.com/decred/politeia/politeiawww/api/records/v1" +) + +func (r *Records) processTimestamps(ctx context.Context, t v1.Timestamps, isAdmin bool) (*v1.TimestampsReply, error) { + log.Tracef("processTimestamps: %v %v %v", t.State, t.Token, t.Version) + + // Get record timestamps + var ( + rt *pdv1.RecordTimestamps + err error + ) + switch t.State { + case v1.RecordStateUnvetted: + rt, err = r.politeiad.GetUnvettedTimestamps(ctx, t.Token, t.Version) + if err != nil { + return nil, err + } + case v1.RecordStateVetted: + rt, err = r.politeiad.GetVettedTimestamps(ctx, t.Token, t.Version) + if err != nil { + return nil, err + } + default: + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeRecordStateInvalid, + } + } + + var ( + recordMD = convertTimestampToV1(rt.RecordMetadata) + metadata = make(map[uint64]v1.Timestamp, len(rt.Files)) + files = make(map[string]v1.Timestamp, len(rt.Files)) + ) + for k, v := range rt.Metadata { + metadata[k] = convertTimestampToV1(v) + } + for k, v := range rt.Files { + files[k] = convertTimestampToV1(v) + } + + // Unvetted data blobs are stripped if the user is not an admin. + // The rest of the timestamp is still returned. + if t.State == v1.RecordStateUnvetted && !isAdmin { + recordMD.Data = "" + for k, v := range files { + v.Data = "" + files[k] = v + } + for k, v := range metadata { + v.Data = "" + metadata[k] = v + } + } + + return &v1.TimestampsReply{ + RecordMetadata: recordMD, + Files: files, + Metadata: metadata, + }, nil +} + +func convertProofToV1(p pdv1.Proof) v1.Proof { + return v1.Proof{ + Type: p.Type, + Digest: p.Digest, + MerkleRoot: p.MerkleRoot, + MerklePath: p.MerklePath, + ExtraData: p.ExtraData, + } +} + +func convertTimestampToV1(t pdv1.Timestamp) v1.Timestamp { + proofs := make([]v1.Proof, 0, len(t.Proofs)) + for _, v := range t.Proofs { + proofs = append(proofs, convertProofToV1(v)) + } + return v1.Timestamp{ + Data: t.Data, + Digest: t.Digest, + TxID: t.TxID, + MerkleRoot: t.MerkleRoot, + Proofs: proofs, + } +} diff --git a/politeiawww/records/records.go b/politeiawww/records/records.go new file mode 100644 index 000000000..08101d936 --- /dev/null +++ b/politeiawww/records/records.go @@ -0,0 +1,253 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package records + +import ( + "encoding/json" + "net/http" + + pdclient "github.com/decred/politeia/politeiad/client" + v1 "github.com/decred/politeia/politeiawww/api/records/v1" + "github.com/decred/politeia/politeiawww/config" + "github.com/decred/politeia/politeiawww/events" + "github.com/decred/politeia/politeiawww/sessions" + "github.com/decred/politeia/util" +) + +// Records is the context for the records API. +type Records struct { + cfg *config.Config + politeiad *pdclient.Client + sessions *sessions.Sessions + events *events.Manager +} + +func (c *Records) HandleNew(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleNew") + + var n v1.New + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&n); err != nil { + respondWithError(w, r, "HandleNew: unmarshal", + v1.UserErrorReply{ + ErrorCode: v1.ErrorStatusInputInvalid, + }) + return + } + + u, err := c.sessions.GetSessionUser(w, r) + if err != nil { + respondWithError(w, r, + "HandleNew: GetSessionUser: %v", err) + return + } + + nr, err := c.processNew(r.Context(), n, *u) + if err != nil { + respondWithError(w, r, + "HandleNew: processNew: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, nr) +} + +func (c *Records) HandleEdit(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleEdit") + + var e v1.Edit + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&e); err != nil { + respondWithError(w, r, "HandleEdit: unmarshal", + v1.UserErrorReply{ + ErrorCode: v1.ErrorStatusInputInvalid, + }) + return + } + + u, err := c.sessions.GetSessionUser(w, r) + if err != nil { + respondWithError(w, r, + "HandleEdit: GetSessionUser: %v", err) + return + } + + er, err := c.processEdit(r.Context(), e, *u) + if err != nil { + respondWithError(w, r, + "HandleEdit: processEdit: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, er) +} + +func (c *Records) HandleSetStatus(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleSetStatus") + + var ss v1.SetStatus + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&ss); err != nil { + respondWithError(w, r, "HandleSetStatus: unmarshal", + v1.UserErrorReply{ + ErrorCode: v1.ErrorStatusInputInvalid, + }) + return + } + + u, err := c.sessions.GetSessionUser(w, r) + if err != nil { + respondWithError(w, r, + "HandleSetStatus: GetSessionUser: %v", err) + return + } + + ssr, err := c.processSetStatus(r.Context(), ss, *u) + if err != nil { + respondWithError(w, r, + "HandleSetStatus: processSetStatus: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, ssr) +} + +func (c *Records) HandleDetails(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleDetails") + + var d v1.Details + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&d); err != nil { + respondWithError(w, r, "HandleDetails: unmarshal", + v1.UserErrorReply{ + ErrorCode: v1.ErrorStatusInputInvalid, + }) + return + } + + u, err := c.sessions.GetSessionUser(w, r) + if err != nil { + respondWithError(w, r, + "HandleDetails: GetSessionUser: %v", err) + return + } + + dr, err := c.processDetails(r.Context(), d, *u) + if err != nil { + respondWithError(w, r, + "HandleDetails: processDetails: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, dr) +} + +func (c *Records) HandleRecords(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleRecords") + + var rs v1.Records + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&rs); err != nil { + respondWithError(w, r, "HandleRecords: unmarshal", + v1.UserErrorReply{ + ErrorCode: v1.ErrorStatusInputInvalid, + }) + return + } + + // Lookup session user. This is a public route so a session may not + // exist. Ignore any session not found errors. + u, err := c.sessions.GetSessionUser(w, r) + if err != nil && err != sessions.ErrSessionNotFound { + respondWithError(w, r, + "HandleRecords: GetSessionUser: %v", err) + return + } + + isAdmin := u != nil && u.Admin + rsr, err := c.processs(r.Context(), rs, isAdmin) + if err != nil { + respondWithError(w, r, + "HandleRecords: processs: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, rsr) +} + +func (c *Records) HandleInventory(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleInventory") + + var i v1.Inventory + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&i); err != nil { + respondWithError(w, r, "HandleInventory: unmarshal", + v1.UserErrorReply{ + ErrorCode: v1.ErrorStatusInputInvalid, + }) + return + } + + // Lookup session user. This is a public route so a session may not + // exist. Ignore any session not found errors. + u, err := c.sessions.GetSessionUser(w, r) + if err != nil && err != sessions.ErrSessionNotFound { + respondWithError(w, r, + "HandleInventory: GetSessionUser: %v", err) + return + } + + ir, err := c.processInventory(r.Context(), i, u) + if err != nil { + respondWithError(w, r, + "HandleInventory: processInventory: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, ir) +} + +func (p *politeiawww) HandleTimestamps(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleTimestamps") + + var t rcv1.Timestamps + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&t); err != nil { + respondWithError(w, r, "HandleTimestamps: unmarshal", + rcv1.UserErrorReply{ + ErrorCode: rcv1.ErrorCodeInputInvalid, + }) + return + } + + // Lookup session user. This is a public route so a session may not + // exist. Ignore any session not found errors. + u, err := c.sessions.GetSessionUser(w, r) + if err != nil && err != sessions.ErrSessionNotFound { + respondWithError(w, r, + "HandleTimestamps: getSessionUser: %v", err) + return + } + + isAdmin := u != nil && u.Admin + tr, err := c.processTimestamps(r.Context(), t, isAdmin) + if err != nil { + respondWithError(w, r, + "HandleTimestamps: processTimestamps: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, tr) +} + +// New returns a new Records context. +func New(cfg *config.Config, pdc *pdclient.Client, s *sessions.Sessions, e *events.Manager) *Comments { + return &Comments{ + cfg: cfg, + politeiad: pdc, + sessions: s, + events: e, + } +} diff --git a/politeiawww/ticketvote/error.go b/politeiawww/ticketvote/error.go index 455fc7d8b..7636246b3 100644 --- a/politeiawww/ticketvote/error.go +++ b/politeiawww/ticketvote/error.go @@ -13,26 +13,26 @@ import ( "time" pdclient "github.com/decred/politeia/politeiad/client" - tkv1 "github.com/decred/politeia/politeiawww/api/comments/v1" + v1 "github.com/decred/politeia/politeiawww/api/comments/v1" "github.com/decred/politeia/util" ) func respondWithError(w http.ResponseWriter, r *http.Request, format string, err error) { var ( - ue tkv1.UserErrorReply + ue v1.UserErrorReply pe pdclient.Error ) switch { case errors.As(err, &ue): // Ticketvote user error m := fmt.Sprintf("Ticketvote user error: %v %v %v", - util.RemoteAddr(r), ue.ErrorCode, tkv1.ErrorCodes[ue.ErrorCode]) + util.RemoteAddr(r), ue.ErrorCode, v1.ErrorCodes[ue.ErrorCode]) if ue.ErrorContext != "" { m += fmt.Sprintf(": %v", ue.ErrorContext) } log.Infof(m) util.RespondWithJSON(w, http.StatusBadRequest, - tkv1.UserErrorReply{ + v1.UserErrorReply{ ErrorCode: ue.ErrorCode, ErrorContext: ue.ErrorContext, }) @@ -55,7 +55,7 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err } log.Infof(m) util.RespondWithJSON(w, http.StatusBadRequest, - tkv1.PluginErrorReply{ + v1.PluginErrorReply{ PluginID: pluginID, ErrorCode: errCode, ErrorContext: strings.Join(errContext, ", "), @@ -70,7 +70,7 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err r.Proto, ts, errCode) util.RespondWithJSON(w, http.StatusInternalServerError, - tkv1.ServerErrorReply{ + v1.ServerErrorReply{ ErrorCode: ts, }) return @@ -85,7 +85,7 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) util.RespondWithJSON(w, http.StatusInternalServerError, - tkv1.ServerErrorReply{ + v1.ServerErrorReply{ ErrorCode: t, }) return diff --git a/politeiawww/ticketvote/events.go b/politeiawww/ticketvote/events.go index 4d6dd47e6..f1880edca 100644 --- a/politeiawww/ticketvote/events.go +++ b/politeiawww/ticketvote/events.go @@ -5,7 +5,7 @@ package ticketvote import ( - tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" + v1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" "github.com/decred/politeia/politeiawww/user" ) @@ -19,12 +19,12 @@ const ( // EventAuthorize is the event data for EventTypeAuthorize. type EventAuthorize struct { - Auth tkv1.Authorize + Auth v1.Authorize User user.User } // EventStart is the event data for EventTypeStart. type EventStart struct { - Start tkv1.Start + Start v1.Start User user.User } diff --git a/politeiawww/ticketvote/process.go b/politeiawww/ticketvote/process.go index 58396cd4f..b2be496b4 100644 --- a/politeiawww/ticketvote/process.go +++ b/politeiawww/ticketvote/process.go @@ -9,17 +9,17 @@ import ( pdv1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/plugins/ticketvote" - tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" + v1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" "github.com/decred/politeia/politeiawww/user" ) -func (t *TicketVote) processAuthorize(ctx context.Context, a tkv1.Authorize, u user.User) (*tkv1.AuthorizeReply, error) { +func (t *TicketVote) processAuthorize(ctx context.Context, a v1.Authorize, u user.User) (*v1.AuthorizeReply, error) { log.Tracef("processAuthorize: %v", a.Token) // Verify user signed with their active identity if u.PublicKey() != a.PublicKey { - return nil, tkv1.UserErrorReply{ - ErrorCode: tkv1.ErrorCodePublicKeyInvalid, + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodePublicKeyInvalid, ErrorContext: "not active identity", } } @@ -30,8 +30,8 @@ func (t *TicketVote) processAuthorize(ctx context.Context, a tkv1.Authorize, u u return nil, err } if u.ID.String() != authorID { - return nil, tkv1.UserErrorReply{ - ErrorCode: tkv1.ErrorCodeUnauthorized, + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeUnauthorized, ErrorContext: "user is not record author", } } @@ -56,20 +56,20 @@ func (t *TicketVote) processAuthorize(ctx context.Context, a tkv1.Authorize, u u User: u, }) - return &tkv1.AuthorizeReply{ + return &v1.AuthorizeReply{ Timestamp: tar.Timestamp, Receipt: tar.Receipt, }, nil } -func (t *TicketVote) processStart(ctx context.Context, s tkv1.Start, u user.User) (*tkv1.StartReply, error) { +func (t *TicketVote) processStart(ctx context.Context, s v1.Start, u user.User) (*v1.StartReply, error) { log.Tracef("processStart: %v", len(s.Starts)) // Verify user signed with their active identity for _, v := range s.Starts { if u.PublicKey() != v.PublicKey { - return nil, tkv1.UserErrorReply{ - ErrorCode: tkv1.ErrorCodePublicKeyInvalid, + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodePublicKeyInvalid, ErrorContext: "not active identity", } } @@ -79,11 +79,11 @@ func (t *TicketVote) processStart(ctx context.Context, s tkv1.Start, u user.User var token string for _, v := range s.Starts { switch v.Params.Type { - case tkv1.VoteTypeRunoff: + case v1.VoteTypeRunoff: // This is a runoff vote. Execute the plugin command on the // parent record. token = v.Params.Parent - case tkv1.VoteTypeStandard: + case v1.VoteTypeStandard: // This is a standard vote. Execute the plugin command on the // record specified in the vote params. token = v.Params.Token @@ -104,7 +104,7 @@ func (t *TicketVote) processStart(ctx context.Context, s tkv1.Start, u user.User User: u, }) - return &tkv1.StartReply{ + return &v1.StartReply{ StartBlockHeight: tsr.StartBlockHeight, StartBlockHash: tsr.StartBlockHash, EndBlockHeight: tsr.EndBlockHeight, @@ -112,7 +112,7 @@ func (t *TicketVote) processStart(ctx context.Context, s tkv1.Start, u user.User }, nil } -func (t *TicketVote) processCastBallot(ctx context.Context, cb tkv1.CastBallot) (*tkv1.CastBallotReply, error) { +func (t *TicketVote) processCastBallot(ctx context.Context, cb v1.CastBallot) (*v1.CastBallotReply, error) { log.Tracef("processCastBallot") // Get token from one of the votes @@ -131,12 +131,12 @@ func (t *TicketVote) processCastBallot(ctx context.Context, cb tkv1.CastBallot) return nil, err } - return &tkv1.CastBallotReply{ + return &v1.CastBallotReply{ Receipts: convertCastVoteRepliesToV1(tcbr.Receipts), }, nil } -func (t *TicketVote) processDetails(ctx context.Context, d tkv1.Details) (*tkv1.DetailsReply, error) { +func (t *TicketVote) processDetails(ctx context.Context, d v1.Details) (*v1.DetailsReply, error) { log.Tracef("processsDetails: %v", d.Token) tdr, err := t.politeiad.TicketVoteDetails(ctx, d.Token) @@ -144,19 +144,19 @@ func (t *TicketVote) processDetails(ctx context.Context, d tkv1.Details) (*tkv1. return nil, err } - var vote *tkv1.VoteDetails + var vote *v1.VoteDetails if tdr.Vote != nil { vd := convertVoteDetailsToV1(*tdr.Vote) vote = &vd } - return &tkv1.DetailsReply{ + return &v1.DetailsReply{ Auths: convertAuthDetailsToV1(tdr.Auths), Vote: vote, }, nil } -func (t *TicketVote) processResults(ctx context.Context, r tkv1.Results) (*tkv1.ResultsReply, error) { +func (t *TicketVote) processResults(ctx context.Context, r v1.Results) (*v1.ResultsReply, error) { log.Tracef("processResults: %v", r.Token) rr, err := t.politeiad.TicketVoteResults(ctx, r.Token) @@ -164,12 +164,12 @@ func (t *TicketVote) processResults(ctx context.Context, r tkv1.Results) (*tkv1. return nil, err } - return &tkv1.ResultsReply{ + return &v1.ResultsReply{ Votes: convertCastVoteDetailsToV1(rr.Votes), }, nil } -func (t *TicketVote) processSummaries(ctx context.Context, s tkv1.Summaries) (*tkv1.SummariesReply, error) { +func (t *TicketVote) processSummaries(ctx context.Context, s v1.Summaries) (*v1.SummariesReply, error) { log.Tracef("processSummaries: %v", s.Tokens) ts, err := t.politeiad.TicketVoteSummaries(ctx, s.Tokens) @@ -177,12 +177,12 @@ func (t *TicketVote) processSummaries(ctx context.Context, s tkv1.Summaries) (*t return nil, err } - return &tkv1.SummariesReply{ + return &v1.SummariesReply{ Summaries: convertSummariesToV1(ts), }, nil } -func (t *TicketVote) processLinkedFrom(ctx context.Context, lf tkv1.LinkedFrom) (*tkv1.LinkedFromReply, error) { +func (t *TicketVote) processLinkedFrom(ctx context.Context, lf v1.LinkedFrom) (*v1.LinkedFromReply, error) { log.Tracef("processLinkedFrom: %v", lf.Tokens) tlf, err := t.politeiad.TicketVoteLinkedFrom(ctx, lf.Tokens) @@ -190,12 +190,12 @@ func (t *TicketVote) processLinkedFrom(ctx context.Context, lf tkv1.LinkedFrom) return nil, err } - return &tkv1.LinkedFromReply{ + return &v1.LinkedFromReply{ LinkedFrom: tlf, }, nil } -func (t *TicketVote) processInventory(ctx context.Context) (*tkv1.InventoryReply, error) { +func (t *TicketVote) processInventory(ctx context.Context) (*v1.InventoryReply, error) { log.Tracef("processInventory") // Send plugin command @@ -208,16 +208,16 @@ func (t *TicketVote) processInventory(ctx context.Context) (*tkv1.InventoryReply records := make(map[string][]string, len(ir.Records)) for k, v := range ir.Records { s := convertVoteStatusToV1(k) - records[tkv1.VoteStatuses[s]] = v + records[v1.VoteStatuses[s]] = v } - return &tkv1.InventoryReply{ + return &v1.InventoryReply{ Records: records, BestBlock: ir.BestBlock, }, nil } -func (t *TicketVote) processTimestamps(ctx context.Context, ts tkv1.Timestamps) (*tkv1.TimestampsReply, error) { +func (t *TicketVote) processTimestamps(ctx context.Context, ts v1.Timestamps) (*v1.TimestampsReply, error) { log.Tracef("processTimestamps: %v", ts.Token) // Send plugin command @@ -228,8 +228,8 @@ func (t *TicketVote) processTimestamps(ctx context.Context, ts tkv1.Timestamps) // Prepare reply var ( - auths = make([]tkv1.Timestamp, 0, len(tsr.Auths)) - votes = make(map[string]tkv1.Timestamp, len(tsr.Votes)) + auths = make([]v1.Timestamp, 0, len(tsr.Auths)) + votes = make(map[string]v1.Timestamp, len(tsr.Votes)) details = convertTimestampToV1(tsr.Details) ) @@ -240,24 +240,24 @@ func (t *TicketVote) processTimestamps(ctx context.Context, ts tkv1.Timestamps) votes[k] = convertTimestampToV1(v) } - return &tkv1.TimestampsReply{ + return &v1.TimestampsReply{ Auths: auths, Details: details, Votes: votes, }, nil } -func convertVoteTypeToPlugin(t tkv1.VoteT) ticketvote.VoteT { +func convertVoteTypeToPlugin(t v1.VoteT) ticketvote.VoteT { switch t { - case tkv1.VoteTypeStandard: + case v1.VoteTypeStandard: return ticketvote.VoteTypeStandard - case tkv1.VoteTypeRunoff: + case v1.VoteTypeRunoff: return ticketvote.VoteTypeRunoff } return ticketvote.VoteTypeInvalid } -func convertVoteParamsToPlugin(v tkv1.VoteParams) ticketvote.VoteParams { +func convertVoteParamsToPlugin(v v1.VoteParams) ticketvote.VoteParams { tv := ticketvote.VoteParams{ Token: v.Token, Version: v.Version, @@ -282,7 +282,7 @@ func convertVoteParamsToPlugin(v tkv1.VoteParams) ticketvote.VoteParams { return tv } -func convertStartDetailsToPlugin(sd tkv1.StartDetails) ticketvote.StartDetails { +func convertStartDetailsToPlugin(sd v1.StartDetails) ticketvote.StartDetails { return ticketvote.StartDetails{ Params: convertVoteParamsToPlugin(sd.Params), PublicKey: sd.PublicKey, @@ -290,7 +290,7 @@ func convertStartDetailsToPlugin(sd tkv1.StartDetails) ticketvote.StartDetails { } } -func convertStartToPlugin(vs tkv1.Start) ticketvote.Start { +func convertStartToPlugin(vs v1.Start) ticketvote.Start { starts := make([]ticketvote.StartDetails, 0, len(vs.Starts)) for _, v := range vs.Starts { starts = append(starts, convertStartDetailsToPlugin(v)) @@ -300,7 +300,7 @@ func convertStartToPlugin(vs tkv1.Start) ticketvote.Start { } } -func convertCastVotesToPlugin(votes []tkv1.CastVote) []ticketvote.CastVote { +func convertCastVotesToPlugin(votes []v1.CastVote) []ticketvote.CastVote { cv := make([]ticketvote.CastVote, 0, len(votes)) for _, v := range votes { cv = append(cv, ticketvote.CastVote{ @@ -313,19 +313,19 @@ func convertCastVotesToPlugin(votes []tkv1.CastVote) []ticketvote.CastVote { return cv } -func convertVoteTypeToV1(t ticketvote.VoteT) tkv1.VoteT { +func convertVoteTypeToV1(t ticketvote.VoteT) v1.VoteT { switch t { case ticketvote.VoteTypeStandard: - return tkv1.VoteTypeStandard + return v1.VoteTypeStandard case ticketvote.VoteTypeRunoff: - return tkv1.VoteTypeRunoff + return v1.VoteTypeRunoff } - return tkv1.VoteTypeInvalid + return v1.VoteTypeInvalid } -func convertVoteParamsToV1(v ticketvote.VoteParams) tkv1.VoteParams { - vp := tkv1.VoteParams{ +func convertVoteParamsToV1(v ticketvote.VoteParams) v1.VoteParams { + vp := v1.VoteParams{ Token: v.Token, Version: v.Version, Type: convertVoteTypeToV1(v.Type), @@ -334,9 +334,9 @@ func convertVoteParamsToV1(v ticketvote.VoteParams) tkv1.VoteParams { QuorumPercentage: v.QuorumPercentage, PassPercentage: v.PassPercentage, } - vo := make([]tkv1.VoteOption, 0, len(v.Options)) + vo := make([]v1.VoteOption, 0, len(v.Options)) for _, o := range v.Options { - vo = append(vo, tkv1.VoteOption{ + vo = append(vo, v1.VoteOption{ ID: o.ID, Description: o.Description, Bit: o.Bit, @@ -347,31 +347,31 @@ func convertVoteParamsToV1(v ticketvote.VoteParams) tkv1.VoteParams { return vp } -func convertVoteErrorToV1(e ticketvote.VoteErrorT) tkv1.VoteErrorT { +func convertVoteErrorToV1(e ticketvote.VoteErrorT) v1.VoteErrorT { switch e { case ticketvote.VoteErrorInvalid: - return tkv1.VoteErrorInvalid + return v1.VoteErrorInvalid case ticketvote.VoteErrorInternalError: - return tkv1.VoteErrorInternalError + return v1.VoteErrorInternalError case ticketvote.VoteErrorRecordNotFound: - return tkv1.VoteErrorRecordNotFound + return v1.VoteErrorRecordNotFound case ticketvote.VoteErrorVoteBitInvalid: - return tkv1.VoteErrorVoteBitInvalid + return v1.VoteErrorVoteBitInvalid case ticketvote.VoteErrorVoteStatusInvalid: - return tkv1.VoteErrorVoteStatusInvalid + return v1.VoteErrorVoteStatusInvalid case ticketvote.VoteErrorTicketAlreadyVoted: - return tkv1.VoteErrorTicketAlreadyVoted + return v1.VoteErrorTicketAlreadyVoted case ticketvote.VoteErrorTicketNotEligible: - return tkv1.VoteErrorTicketNotEligible + return v1.VoteErrorTicketNotEligible default: - return tkv1.VoteErrorInternalError + return v1.VoteErrorInternalError } } -func convertCastVoteRepliesToV1(replies []ticketvote.CastVoteReply) []tkv1.CastVoteReply { - r := make([]tkv1.CastVoteReply, 0, len(replies)) +func convertCastVoteRepliesToV1(replies []ticketvote.CastVoteReply) []v1.CastVoteReply { + r := make([]v1.CastVoteReply, 0, len(replies)) for _, v := range replies { - r = append(r, tkv1.CastVoteReply{ + r = append(r, v1.CastVoteReply{ Ticket: v.Ticket, Receipt: v.Receipt, ErrorCode: convertVoteErrorToV1(v.ErrorCode), @@ -381,8 +381,8 @@ func convertCastVoteRepliesToV1(replies []ticketvote.CastVoteReply) []tkv1.CastV return r } -func convertVoteDetailsToV1(vd ticketvote.VoteDetails) tkv1.VoteDetails { - return tkv1.VoteDetails{ +func convertVoteDetailsToV1(vd ticketvote.VoteDetails) v1.VoteDetails { + return v1.VoteDetails{ Params: convertVoteParamsToV1(vd.Params), PublicKey: vd.PublicKey, Signature: vd.Signature, @@ -393,10 +393,10 @@ func convertVoteDetailsToV1(vd ticketvote.VoteDetails) tkv1.VoteDetails { } } -func convertAuthDetailsToV1(auths []ticketvote.AuthDetails) []tkv1.AuthDetails { - a := make([]tkv1.AuthDetails, 0, len(auths)) +func convertAuthDetailsToV1(auths []ticketvote.AuthDetails) []v1.AuthDetails { + a := make([]v1.AuthDetails, 0, len(auths)) for _, v := range auths { - a = append(a, tkv1.AuthDetails{ + a = append(a, v1.AuthDetails{ Token: v.Token, Version: v.Version, Action: v.Action, @@ -409,10 +409,10 @@ func convertAuthDetailsToV1(auths []ticketvote.AuthDetails) []tkv1.AuthDetails { return a } -func convertCastVoteDetailsToV1(votes []ticketvote.CastVoteDetails) []tkv1.CastVoteDetails { - vs := make([]tkv1.CastVoteDetails, 0, len(votes)) +func convertCastVoteDetailsToV1(votes []ticketvote.CastVoteDetails) []v1.CastVoteDetails { + vs := make([]v1.CastVoteDetails, 0, len(votes)) for _, v := range votes { - vs = append(vs, tkv1.CastVoteDetails{ + vs = append(vs, v1.CastVoteDetails{ Token: v.Token, Ticket: v.Ticket, VoteBit: v.VoteBit, @@ -423,34 +423,34 @@ func convertCastVoteDetailsToV1(votes []ticketvote.CastVoteDetails) []tkv1.CastV return vs } -func convertVoteStatusToV1(s ticketvote.VoteStatusT) tkv1.VoteStatusT { +func convertVoteStatusToV1(s ticketvote.VoteStatusT) v1.VoteStatusT { switch s { case ticketvote.VoteStatusInvalid: - return tkv1.VoteStatusInvalid + return v1.VoteStatusInvalid case ticketvote.VoteStatusUnauthorized: - return tkv1.VoteStatusUnauthorized + return v1.VoteStatusUnauthorized case ticketvote.VoteStatusAuthorized: - return tkv1.VoteStatusAuthorized + return v1.VoteStatusAuthorized case ticketvote.VoteStatusStarted: - return tkv1.VoteStatusStarted + return v1.VoteStatusStarted case ticketvote.VoteStatusFinished: - return tkv1.VoteStatusFinished + return v1.VoteStatusFinished default: - return tkv1.VoteStatusInvalid + return v1.VoteStatusInvalid } } -func convertSummaryToV1(s ticketvote.SummaryReply) tkv1.Summary { - results := make([]tkv1.VoteResult, 0, len(s.Results)) +func convertSummaryToV1(s ticketvote.SummaryReply) v1.Summary { + results := make([]v1.VoteResult, 0, len(s.Results)) for _, v := range s.Results { - results = append(results, tkv1.VoteResult{ + results = append(results, v1.VoteResult{ ID: v.ID, Description: v.Description, VoteBit: v.VoteBit, Votes: v.Votes, }) } - return tkv1.Summary{ + return v1.Summary{ Type: convertVoteTypeToV1(s.Type), Status: convertVoteStatusToV1(s.Status), Duration: s.Duration, @@ -466,16 +466,16 @@ func convertSummaryToV1(s ticketvote.SummaryReply) tkv1.Summary { } } -func convertSummariesToV1(s map[string]ticketvote.SummaryReply) map[string]tkv1.Summary { - ts := make(map[string]tkv1.Summary, len(s)) +func convertSummariesToV1(s map[string]ticketvote.SummaryReply) map[string]v1.Summary { + ts := make(map[string]v1.Summary, len(s)) for k, v := range s { ts[k] = convertSummaryToV1(v) } return ts } -func convertProofToV1(p ticketvote.Proof) tkv1.Proof { - return tkv1.Proof{ +func convertProofToV1(p ticketvote.Proof) v1.Proof { + return v1.Proof{ Type: p.Type, Digest: p.Digest, MerkleRoot: p.MerkleRoot, @@ -484,12 +484,12 @@ func convertProofToV1(p ticketvote.Proof) tkv1.Proof { } } -func convertTimestampToV1(t ticketvote.Timestamp) tkv1.Timestamp { - proofs := make([]tkv1.Proof, 0, len(t.Proofs)) +func convertTimestampToV1(t ticketvote.Timestamp) v1.Timestamp { + proofs := make([]v1.Proof, 0, len(t.Proofs)) for _, v := range t.Proofs { proofs = append(proofs, convertProofToV1(v)) } - return tkv1.Timestamp{ + return v1.Timestamp{ Data: t.Data, Digest: t.Digest, TxID: t.TxID, diff --git a/politeiawww/ticketvote/ticketvote.go b/politeiawww/ticketvote/ticketvote.go index 217e23ba7..d04ad5364 100644 --- a/politeiawww/ticketvote/ticketvote.go +++ b/politeiawww/ticketvote/ticketvote.go @@ -9,14 +9,14 @@ import ( "net/http" pdclient "github.com/decred/politeia/politeiad/client" - tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" + v1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/events" "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/util" ) -// TicketVote is the context that handles the ticketvote API. +// TicketVote is the context for the ticketvote API. type TicketVote struct { cfg *config.Config politeiad *pdclient.Client @@ -29,12 +29,12 @@ type TicketVote struct { func (t *TicketVote) HandleAuthorize(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleAuthorize") - var a tkv1.Authorize + var a v1.Authorize decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&a); err != nil { respondWithError(w, r, "HandleAuthorize: unmarshal", - tkv1.UserErrorReply{ - ErrorCode: tkv1.ErrorCodeInputInvalid, + v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeInputInvalid, }) return } @@ -60,12 +60,12 @@ func (t *TicketVote) HandleAuthorize(w http.ResponseWriter, r *http.Request) { func (t *TicketVote) HandleStart(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleStart") - var s tkv1.Start + var s v1.Start decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&s); err != nil { respondWithError(w, r, "HandleStart: unmarshal", - tkv1.UserErrorReply{ - ErrorCode: tkv1.ErrorCodeInputInvalid, + v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeInputInvalid, }) return } @@ -92,12 +92,12 @@ func (t *TicketVote) HandleStart(w http.ResponseWriter, r *http.Request) { func (t *TicketVote) HandleCastBallot(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleCastBallot") - var cb tkv1.CastBallot + var cb v1.CastBallot decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&cb); err != nil { respondWithError(w, r, "HandleCastBallot: unmarshal", - tkv1.UserErrorReply{ - ErrorCode: tkv1.ErrorCodeInputInvalid, + v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeInputInvalid, }) return } @@ -116,12 +116,12 @@ func (t *TicketVote) HandleCastBallot(w http.ResponseWriter, r *http.Request) { func (t *TicketVote) HandleDetails(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleDetails") - var d tkv1.Details + var d v1.Details decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&d); err != nil { respondWithError(w, r, "HandleDetails: unmarshal", - tkv1.UserErrorReply{ - ErrorCode: tkv1.ErrorCodeInputInvalid, + v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeInputInvalid, }) return } @@ -140,12 +140,12 @@ func (t *TicketVote) HandleDetails(w http.ResponseWriter, r *http.Request) { func (t *TicketVote) HandleResults(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleResults") - var rs tkv1.Results + var rs v1.Results decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&rs); err != nil { respondWithError(w, r, "HandleResults: unmarshal", - tkv1.UserErrorReply{ - ErrorCode: tkv1.ErrorCodeInputInvalid, + v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeInputInvalid, }) return } @@ -165,12 +165,12 @@ func (t *TicketVote) HandleResults(w http.ResponseWriter, r *http.Request) { func (t *TicketVote) HandleSummaries(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleSummaries") - var s tkv1.Summaries + var s v1.Summaries decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&s); err != nil { respondWithError(w, r, "HandleSummaries: unmarshal", - tkv1.UserErrorReply{ - ErrorCode: tkv1.ErrorCodeInputInvalid, + v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeInputInvalid, }) return } @@ -190,12 +190,12 @@ func (t *TicketVote) HandleSummaries(w http.ResponseWriter, r *http.Request) { func (t *TicketVote) HandleLinkedFrom(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleLinkedFrom") - var lf tkv1.LinkedFrom + var lf v1.LinkedFrom decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&lf); err != nil { respondWithError(w, r, "HandleLinkedFrom: unmarshal", - tkv1.UserErrorReply{ - ErrorCode: tkv1.ErrorCodeInputInvalid, + v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeInputInvalid, }) return } @@ -230,12 +230,12 @@ func (t *TicketVote) HandleInventory(w http.ResponseWriter, r *http.Request) { func (t *TicketVote) HandleTimestamps(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleTimestamps") - var ts tkv1.Timestamps + var ts v1.Timestamps decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&t); err != nil { respondWithError(w, r, "HandleTimestamps: unmarshal", - tkv1.UserErrorReply{ - ErrorCode: tkv1.ErrorCodeInputInvalid, + v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeInputInvalid, }) return } From 8bfd66be0fbe0f90826f90309ebd65d48fe30de8 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 26 Jan 2021 21:30:07 -0600 Subject: [PATCH 253/449] Add record routes. --- politeiad/backend/tlogbe/plugins/pi/cmds.go | 5 - politeiad/backend/tlogbe/plugins/pi/hooks.go | 302 ++-- .../backend/tlogbe/plugins/pi/hooks_test.go | 18 +- politeiad/backend/tlogbe/plugins/pi/pi.go | 29 +- .../backend/tlogbe/plugins/user/hooks.go | 22 + politeiad/backend/tlogbe/tlogbe.go | 10 + politeiad/plugins/pi/pi.go | 78 +- politeiad/plugins/user/user.go | 12 + politeiawww/api/pi/v1/v1.go | 180 +-- politeiawww/api/records/v1/v1.go | 135 +- politeiawww/api/www/v1/v1.go | 8 +- politeiawww/comments/events.go | 2 +- politeiawww/comments/process.go | 16 +- politeiawww/email.go | 8 +- politeiawww/events.go | 179 +-- politeiawww/log.go | 12 +- politeiawww/paywall.go | 48 - politeiawww/pi.go | 21 - politeiawww/pi/pi.go | 319 +++++ politeiawww/piwww.go | 1226 +---------------- politeiawww/politeiawww.go | 8 +- politeiawww/proposals.go | 56 +- politeiawww/records/error.go | 2 +- politeiawww/records/events.go | 26 + politeiawww/records/process.go | 599 ++++++++ politeiawww/records/records.go | 71 +- politeiawww/testing.go | 2 +- politeiawww/user.go | 12 +- politeiawww/www.go | 155 --- 29 files changed, 1551 insertions(+), 2010 deletions(-) rename politeiawww/piwww_test.go => politeiad/backend/tlogbe/plugins/pi/hooks_test.go (79%) delete mode 100644 politeiawww/pi.go create mode 100644 politeiawww/pi/pi.go diff --git a/politeiad/backend/tlogbe/plugins/pi/cmds.go b/politeiad/backend/tlogbe/plugins/pi/cmds.go index cc172d312..b89ad2ea0 100644 --- a/politeiad/backend/tlogbe/plugins/pi/cmds.go +++ b/politeiad/backend/tlogbe/plugins/pi/cmds.go @@ -10,13 +10,8 @@ import ( "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/ticketvote" - "github.com/decred/politeia/util" ) -func tokenDecode(token string) ([]byte, error) { - return util.TokenDecode(util.TokenTypeTlog, token) -} - func (p *piPlugin) voteSummary(token []byte) (*ticketvote.SummaryReply, error) { reply, err := p.backend.VettedPluginCmd(token, ticketvote.PluginID, ticketvote.CmdSummary, "") diff --git a/politeiad/backend/tlogbe/plugins/pi/hooks.go b/politeiad/backend/tlogbe/plugins/pi/hooks.go index 5f6eb6231..df42c1e7a 100644 --- a/politeiad/backend/tlogbe/plugins/pi/hooks.go +++ b/politeiad/backend/tlogbe/plugins/pi/hooks.go @@ -5,20 +5,31 @@ package pi import ( + "bytes" "encoding/base64" "encoding/json" - "errors" "fmt" - "io" - "strings" + "strconv" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/politeiad/plugins/comments" "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/ticketvote" + "github.com/decred/politeia/util" ) +const ( + // Accepted MIME types + mimeTypeText = "text/plain" + mimeTypeTextUTF8 = "text/plain; charset=utf-8" + mimeTypePNG = "image/png" +) + +func tokenDecode(token string) ([]byte, error) { + return util.TokenDecode(util.TokenTypeTlog, token) +} + // proposalMetadataDecode decodes and returns the ProposalMetadata from the // provided backend files. If a ProposalMetadata is not found, nil is returned. func proposalMetadataDecode(files []backend.File) (*pi.ProposalMetadata, error) { @@ -41,165 +52,206 @@ func proposalMetadataDecode(files []backend.File) (*pi.ProposalMetadata, error) return propMD, nil } -// statusChangesDecode decodes a JSON byte slice into a []StatusChange slice. -func statusChangesDecode(payload []byte) ([]pi.StatusChange, error) { - var statuses []pi.StatusChange - d := json.NewDecoder(strings.NewReader(string(payload))) - for { - var sc pi.StatusChange - err := d.Decode(&sc) - if errors.Is(err, io.EOF) { - break - } else if err != nil { - return nil, err +// proposalNameRegexp returns the regexp string for validating a proposal name. +func (p *piPlugin) proposalNameRegexpString() string { + var b bytes.Buffer + + b.WriteString("^[") + for _, v := range p.proposalNameSupportedChars { + if len(v) > 1 { + b.WriteString(v) + } else { + b.WriteString(`\` + v) } - statuses = append(statuses, sc) } + minNameLength := strconv.Itoa(p.proposalNameLengthMin) + maxNameLength := strconv.Itoa(p.proposalNameLengthMax) + b.WriteString("]{") + b.WriteString(minNameLength + ",") + b.WriteString(maxNameLength + "}$") - return statuses, nil + return b.String() } -func (p *piPlugin) hookNewRecordPre(payload string) error { - var nr plugins.HookNewRecordPre - err := json.Unmarshal([]byte(payload), &nr) - if err != nil { - return err - } - - // Verify a proposal metadata has been included - pm, err := proposalMetadataDecode(nr.Files) - if err != nil { - return err - } - if pm == nil { - return fmt.Errorf("proposal metadata not found") - } - - // TODO Verify proposal name - - return nil +// proposalNameIsValid returns whether the provided name is a valid proposal +// name. +func (p *piPlugin) proposalNameIsValid(name string) bool { + return p.proposalNameRegexp.MatchString(name) } -func (p *piPlugin) hookEditRecordPre(payload string) error { - /* - var er plugins.HookEditRecord - err := json.Unmarshal([]byte(payload), &er) +// proposalFilesVerify verifies the files adhere to all plugin setting +// requirements. If this hook is being executed then the files have already +// passed politeia validation so we can assume that the file has a unique name, +// a valid base64 payload, and that the file digest and MIME type are correct. +func (p *piPlugin) proposalFilesVerify(files []backend.File) error { + var ( + textFilesCount int + imageFilesCount int + indexFileFound bool + ) + for _, v := range files { + payload, err := base64.StdEncoding.DecodeString(v.Payload) if err != nil { - return err + return fmt.Errorf("invalid base64 %v", v.Name) } - // TODO verify files were changed. Before adding this, verify that - // politeiad will also error if no files were changed. + // MIME type specific validation + switch v.MIME { + case mimeTypeText: + textFilesCount++ - // Verify vote status. This is only required for public proposals. - if status == pi.PropStatusPublic { - token := er.RecordMetadata.Token - s := ticketvote.Summaries{ - Tokens: []string{token}, - } - b, err := ticketvote.EncodeSummaries(s) - if err != nil { - return err - } - reply, err := p.backend.Plugin(ticketvote.PluginID, - ticketvote.CmdSummaries, "", string(b)) - if err != nil { - return fmt.Errorf("ticketvote Summaries: %v", err) + // The text file must be the proposal index file + if v.Name != p.indexFileName { + e := fmt.Sprintf("want %v, got %v", p.indexFileName, v.Name) + return backend.PluginError{ + PluginID: pi.PluginID, + ErrorCode: int(pi.ErrorCodeIndexFileNameInvalid), + ErrorContext: e, + } } - sr, err := ticketvote.DecodeSummariesReply([]byte(reply)) - if err != nil { - return err + + // Verify text file size + if len(payload) > p.textFileSizeMax { + e := fmt.Sprintf("file %v size %v exceeds max size %v", + v.Name, len(payload), p.textFileSizeMax) + return backend.PluginError{ + PluginID: pi.PluginID, + ErrorCode: int(pi.ErrorCodeIndexFileSizeInvalid), + ErrorContext: e, + } } - summary, ok := sr.Summaries[token] - if !ok { - return fmt.Errorf("ticketvote summmary not found") + + // Verify there isn't more than one index file + if indexFileFound { + e := fmt.Sprintf("more than one %v file found", + p.indexFileName) + return backend.PluginError{ + PluginID: pi.PluginID, + ErrorCode: int(pi.ErrorCodeIndexFileCountInvalid), + ErrorContext: e, + } } - if summary.Status != ticketvote.VoteStatusUnauthorized { - e := fmt.Sprintf("vote status got %v, want %v", - ticketvote.VoteStatuses[summary.Status], - ticketvote.VoteStatuses[ticketvote.VoteStatusUnauthorized]) - return backend.PluginUserError{ + + // Set index file as being found + indexFileFound = true + + case mimeTypePNG: + imageFilesCount++ + + // Verify image file size + if len(payload) > p.imageFileSizeMax { + e := fmt.Sprintf("image %v size %v exceeds max size %v", + v.Name, len(payload), p.imageFileSizeMax) + return backend.PluginError{ PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeVoteStatusInvalid), + ErrorCode: int(pi.ErrorCodeImageFileSizeInvalid), ErrorContext: e, } } + + default: + return fmt.Errorf("invalid mime") + } + } + + // Verify that an index file is present + if !indexFileFound { + e := fmt.Sprintf("%v file not found", p.indexFileName) + return backend.PluginError{ + PluginID: pi.PluginID, + ErrorCode: int(pi.ErrorCodeIndexFileCountInvalid), + ErrorContext: e, + } + } + + // Verify file counts are acceptable + if textFilesCount > p.textFileCountMax { + e := fmt.Sprintf("got %v text files, max is %v", + textFilesCount, p.textFileCountMax) + return backend.PluginError{ + PluginID: pi.PluginID, + ErrorCode: int(pi.ErrorCodeTextFileCountInvalid), + ErrorContext: e, + } + } + if imageFilesCount > p.imageFileCountMax { + e := fmt.Sprintf("got %v image files, max is %v", + imageFilesCount, p.imageFileCountMax) + return backend.PluginError{ + PluginID: pi.PluginID, + ErrorCode: int(pi.ErrorCodeImageFileCountInvalid), + ErrorContext: e, + } + } + + // Verify a proposal metadata has been included + pm, err := proposalMetadataDecode(files) + if err != nil { + return err + } + if pm == nil { + return backend.PluginError{ + PluginID: pi.PluginID, + ErrorCode: int(pi.ErrorCodeProposalMetadataInvalid), + ErrorContext: "metadata not found", } - */ + } + + // Verify proposal name + if !p.proposalNameIsValid(pm.Name) { + return backend.PluginError{ + PluginID: pi.PluginID, + ErrorCode: int(pi.ErrorCodeProposalNameInvalid), + ErrorContext: p.proposalNameRegexpString(), + } + } return nil } -func (p *piPlugin) hookSetRecordStatusPost(payload string) error { - var srs plugins.HookSetRecordStatus - err := json.Unmarshal([]byte(payload), &srs) +func (p *piPlugin) hookNewRecordPre(payload string) error { + var nr plugins.HookNewRecordPre + err := json.Unmarshal([]byte(payload), &nr) if err != nil { return err } - /* - // Parse the status change metadata - var sc *pi.StatusChange - for _, v := range srs.MDAppend { - if v.ID != pi.MDStreamIDStatusChanges { - continue - } - - var sc pi.StatusChange - err := json.Unmarshal([]byte(v.Payload), &sc) - if err != nil { - return err - } - break - } - if sc == nil { - return fmt.Errorf("status change append metadata not found") - } + return p.proposalFilesVerify(nr.Files) +} - // Parse the existing status changes metadata stream - var statuses []pi.StatusChange - for _, v := range srs.Current.Metadata { - if v.ID != pi.MDStreamIDStatusChanges { - continue - } +func (p *piPlugin) hookEditRecordPre(payload string) error { + var er plugins.HookEditRecord + err := json.Unmarshal([]byte(payload), &er) + if err != nil { + return err + } - statuses, err = statusChangesDecode([]byte(v.Payload)) - if err != nil { - return err - } - break - } + // Verify proposal files + err = p.proposalFilesVerify(er.Files) + if err != nil { + return err + } - // Verify version is the latest version - if sc.Version != srs.Current.Version { - e := fmt.Sprintf("version not current: got %v, want %v", - sc.Version, srs.Current.Version) - return backend.PluginError{ - PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodePropVersionInvalid), - ErrorContext: e, - } + // Verify vote status allows proposal edits + if er.RecordMetadata.Status == backend.MDStatusVetted { + t, err := tokenDecode(er.RecordMetadata.Token) + if err != nil { + return err } - - // Verify status change is allowed - var from pi.PropStatusT - if len(statuses) == 0 { - // No previous status changes exist. Proposal is unvetted. - from = pi.PropStatusUnvetted - } else { - from = statuses[len(statuses)-1].Status + s, err := p.voteSummary(t) + if err != nil { + return err } - _, isAllowed := pi.StatusChanges[from][sc.Status] - if !isAllowed { - e := fmt.Sprintf("from %v to %v status change not allowed", - from, sc.Status) + if s.Status != ticketvote.VoteStatusUnauthorized { + e := fmt.Sprintf("vote status '%v' does not allow for proposal edits", + ticketvote.VoteStatuses[s.Status]) return backend.PluginError{ PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodePropStatusChangeInvalid), + ErrorCode: int(pi.ErrorCodeVoteStatusInvalid), ErrorContext: e, } } - */ + } return nil } diff --git a/politeiawww/piwww_test.go b/politeiad/backend/tlogbe/plugins/pi/hooks_test.go similarity index 79% rename from politeiawww/piwww_test.go rename to politeiad/backend/tlogbe/plugins/pi/hooks_test.go index 492cc78a5..73bf7339e 100644 --- a/politeiawww/piwww_test.go +++ b/politeiad/backend/tlogbe/plugins/pi/hooks_test.go @@ -1,8 +1,8 @@ -// Copyright (c) 2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package main +package pi import "testing" @@ -69,11 +69,13 @@ func TestProposalNameIsValid(t *testing.T) { true, }, } - - for _, test := range tests { - isValid := proposalNameIsValid(test.name) - if isValid != test.want { - t.Errorf("got %v, want %v", isValid, test.want) + // TODO + /* + for _, test := range tests { + isValid := proposalNameIsValid(test.name) + if isValid != test.want { + t.Errorf("got %v, want %v", isValid, test.want) + } } - } + */ } diff --git a/politeiad/backend/tlogbe/plugins/pi/pi.go b/politeiad/backend/tlogbe/plugins/pi/pi.go index d2b02b7a8..5569ee8b1 100644 --- a/politeiad/backend/tlogbe/plugins/pi/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi/pi.go @@ -7,6 +7,7 @@ package pi import ( "os" "path/filepath" + "regexp" "sync" "github.com/decred/politeia/politeiad/backend" @@ -27,6 +28,18 @@ type piPlugin struct { // stored here is cached data that can be re-created at any time // by walking the trillian trees. dataDir string + + // Plugin settings + indexFileName string + textFileCountMax int + textFileSizeMax int // In bytes + imageFileCountMax int + imageFileSizeMax int // In bytes + + proposalNameSupportedChars []string + proposalNameLengthMin int // In characters + proposalNameLengthMax int // In characters + proposalNameRegexp *regexp.Regexp } // Setup performs any plugin setup that is required. @@ -65,8 +78,6 @@ func (p *piPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload str return p.hookNewRecordPre(payload) case plugins.HookTypeEditRecordPre: return p.hookEditRecordPre(payload) - case plugins.HookTypeSetRecordStatusPost: - return p.hookSetRecordStatusPost(payload) case plugins.HookTypePluginPre: return p.hookPluginPre(treeID, token, payload) } @@ -91,8 +102,22 @@ func New(backend backend.Backend, settings []backend.PluginSetting, dataDir stri return nil, err } + // Setup proposal name regex + // pregexp, err = regexp.Compile() + var pregexp *regexp.Regexp + return &piPlugin{ dataDir: dataDir, backend: backend, + // TODO pi plugin settings + indexFileName: "", + textFileCountMax: 0, + textFileSizeMax: 0, + imageFileCountMax: 0, + imageFileSizeMax: 0, + proposalNameSupportedChars: []string{}, + proposalNameLengthMin: 0, + proposalNameLengthMax: 0, + proposalNameRegexp: pregexp, }, nil } diff --git a/politeiad/backend/tlogbe/plugins/user/hooks.go b/politeiad/backend/tlogbe/plugins/user/hooks.go index 4db5edd07..1cff472e6 100644 --- a/politeiad/backend/tlogbe/plugins/user/hooks.go +++ b/politeiad/backend/tlogbe/plugins/user/hooks.go @@ -137,6 +137,28 @@ func userMetadataPreventUpdates(current, update []backend.MetadataStream) error return nil } +/* +// statusChangesDecode decodes a JSON byte slice into a []StatusChange slice. +func statusChangesDecode(payload []byte) ([]pi.StatusChange, error) { + var statuses []pi.StatusChange + d := json.NewDecoder(strings.NewReader(string(payload))) + for { + var sc pi.StatusChange + err := d.Decode(&sc) + if errors.Is(err, io.EOF) { + break + } else if err != nil { + return nil, err + } + statuses = append(statuses, sc) + } + + return statuses, nil +} + + +*/ + func (p *userPlugin) hookNewRecordPre(payload string) error { var nr plugins.HookNewRecordPre err := json.Unmarshal([]byte(payload), &nr) diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 4f0be5f86..8296d3255 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -306,6 +306,16 @@ func verifyContent(metadata []backend.MetadataStream, files []backend.File, file } } + // Verify payload is not empty + if files[i].Payload == "" { + return backend.ContentVerificationError{ + ErrorCode: v1.ErrorStatusInvalidBase64, + ErrorContext: []string{ + files[i].Name, + }, + } + } + // Decode base64 payload var err error payload, err := base64.StdEncoding.DecodeString(files[i].Payload) diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index 788a8ff75..60d576e20 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -17,18 +17,19 @@ const ( type ErrorCodeT int const ( - // TODO User error codes + // TODO number error codes ErrorCodeInvalid ErrorCodeT = 0 ErrorCodePageSizeExceeded ErrorCodeT = iota - ErrorCodePropTokenInvalid - ErrorCodePropStatusInvalid - ErrorCodePropVersionInvalid - ErrorCodePropStatusChangeInvalid - ErrorCodePropLinkToInvalid + ErrorCodeFileNameInvalid + ErrorCodeIndexFileNameInvalid + ErrorCodeIndexFileCountInvalid + ErrorCodeIndexFileSizeInvalid + ErrorCodeTextFileCountInvalid + ErrorCodeImageFileCountInvalid + ErrorCodeImageFileSizeInvalid + ErrorCodeProposalMetadataInvalid + ErrorCodeProposalNameInvalid ErrorCodeVoteStatusInvalid - ErrorCodeStartDetailsInvalid - ErrorCodeStartDetailsMissing - ErrorCodeVoteParentInvalid ) var ( @@ -45,10 +46,6 @@ const ( // user provided metadata and needs to be included in the merkle // root that politeiad signs. FileNameProposalMetadata = "proposalmetadata.json" - - // MDStreamIDStatusChanges is the politeiad metadata stream ID that - // the StatusesChange structure is appended onto. - MDStreamIDStatusChanges = 2 ) // ProposalMetadata contains metadata that is provided by the user as part of @@ -60,22 +57,32 @@ type ProposalMetadata struct { Name string `json:"name"` } -// PropStatusT represents a proposal status. +// PropStatusT represents a proposal status. These map directly to the +// politeiad record statuses, but some have had their names changed to better +// reflect their intended use case by proposals. type PropStatusT int const ( // PropStatusInvalid is an invalid proposal status. PropStatusInvalid PropStatusT = 0 - // PropStatusUnvetted represents a proposal that has not been made - // public yet. + // PropStatusUnreviewed indicates the proposal has been submitted, + // but has not yet been reviewed and made public by an admin. A + // proposal with this status will have a proposal state of + // PropStateUnvetted. PropStatusUnvetted PropStatusT = 1 - // PropStatusPublic represents a proposal that has been made - // public. + // PropStatusPublic indicates that a proposal has been reviewed and + // made public by an admin. A proposal with this status will have + // a proposal state of PropStateVetted. PropStatusPublic PropStatusT = 2 - // PropStatusCensored represents a proposal that has been censored. + // PropStatusCensored indicates that a proposal has been censored + // by an admin for violating the proposal guidlines. Both unvetted + // and vetted proposals can be censored so a proposal with this + // status can have a state of either PropStateUnvetted or + // PropStateVetted depending on whether the proposal was censored + // before or after it was made public. PropStatusCensored PropStatusT = 3 // PropStatusUnreviewedChanges is a deprecated proposal status that @@ -83,8 +90,8 @@ const ( // directly to the politeiad record statuses. PropStatusUnreviewedChanges PropStatusT = 4 - // PropStatusAbandoned represents a proposal that has been - // abandoned by the author. + // PropStatusAbandoned indicates that a proposal has been marked + // as abandoned by an admin due to the author being inactive. PropStatusAbandoned PropStatusT = 5 ) @@ -97,37 +104,8 @@ var ( PropStatusCensored: "censored", PropStatusAbandoned: "abandoned", } - - // StatusChanges contains the allowed proposal status change - // transitions. If StatusChanges[currentStatus][newStatus] exists - // then the status change is allowed. - StatusChanges = map[PropStatusT]map[PropStatusT]struct{}{ - PropStatusUnvetted: { - PropStatusPublic: {}, - PropStatusCensored: {}, - }, - PropStatusPublic: { - PropStatusAbandoned: {}, - PropStatusCensored: {}, - }, - PropStatusCensored: {}, - PropStatusAbandoned: {}, - } ) -// StatusChange represents a proposal status change. -// -// Signature is the client signature of the Token+Version+Status+Reason. -type StatusChange struct { - Token string `json:"token"` - Version string `json:"version"` - Status PropStatusT `json:"status"` - Reason string `json:"message,omitempty"` - PublicKey string `json:"publickey"` - Signature string `json:"signature"` - Timestamp int64 `json:"timestamp"` -} - // VoteInventory requests the tokens of all proposals in the inventory // categorized by their vote status. This call relies on the ticketvote // Inventory call, but breaks the Finished vote status out into Approved and diff --git a/politeiad/plugins/user/user.go b/politeiad/plugins/user/user.go index d2282a380..8ad0d59ad 100644 --- a/politeiad/plugins/user/user.go +++ b/politeiad/plugins/user/user.go @@ -13,6 +13,9 @@ const ( CmdAuthor = "author" // Get record author CmdUserRecords = "userrecords" // Get user submitted records + // TODO add record status change mdstream + // TODO make whether user md is required a plugin setting + // TODO MDStream IDs need to be plugin specific. If we can't then // we need to make a mdstream package to aggregate all the mdstream // ID. @@ -42,6 +45,15 @@ var ( ErrorCodes = map[ErrorCodeT]string{ ErrorCodeInvalid: "error code invalid", } + + /* + // statusReasonRequired contains the list of proposal statuses that + // require an accompanying reason to be given for the status change. + statusReasonRequired = map[piv1.PropStatusT]struct{}{ + piv1.PropStatusCensored: {}, + piv1.PropStatusAbandoned: {}, + } + */ ) // UserMetadata contains user metadata about a politeiad record. It is diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 1f2242658..13dad1786 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -19,14 +19,13 @@ const ( // APIRoute is prefixed onto all routes defined in this package. APIRoute = "/pi/v1" - // Proposal routes - RouteProposalNew = "/proposal/new" - RouteProposalEdit = "/proposal/edit" - RouteProposalSetStatus = "/proposal/setstatus" - RouteProposals = "/proposals" - - // Vote routes - RouteVoteInventory = "/votes/inventory" + // Routes + RouteProposals = "/proposals" + RouteVoteInventory = "/voteinventory" + + // Proposal states + ProposalStateUnvetted = "unvetted" + ProposalStateVetted = "vetted" ) // ErrorCodeT represents a user error code. @@ -46,35 +45,6 @@ const ( // Signature errors ErrorCodePublicKeyInvalid ErrorCodeT = 200 ErrorCodeSignatureInvalid ErrorCodeT = 201 - - // Proposal errors - // TODO number error codes - ErrorCodeFileCountInvalid ErrorCodeT = 300 - ErrorCodeFileNameInvalid ErrorCodeT = iota - ErrorCodeFileMIMEInvalid - ErrorCodeFileDigestInvalid - ErrorCodeFilePayloadInvalid - ErrorCodeIndexFileNameInvalid - ErrorCodeIndexFileCountInvalid - ErrorCodeIndexFileSizeInvalid - ErrorCodeTextFileCountInvalid - ErrorCodeImageFileCountInvalid - ErrorCodeImageFileSizeInvalid - ErrorCodeMetadataCountInvalid - ErrorCodeMetadataDigestInvalid - ErrorCodeMetadataPayloadInvalid - ErrorCodePropNotFound - ErrorCodePropMetadataNotFound - ErrorCodePropTokenInvalid - ErrorCodePropVersionInvalid - ErrorCodePropNameInvalid - ErrorCodePropLinkToInvalid - ErrorCodePropLinkByInvalid - ErrorCodePropStateInvalid - ErrorCodePropStatusInvalid - ErrorCodePropStatusChangeInvalid - ErrorCodePropStatusChangeReasonInvalid - ErrorCodeNoPropChanges ) var ( @@ -95,31 +65,6 @@ var ( ErrorCodeSignatureInvalid: "signature invalid", // Proposal errors - ErrorCodeFileCountInvalid: "file count invalid", - ErrorCodeFileNameInvalid: "file name invalid", - ErrorCodeFileMIMEInvalid: "file mime invalid", - ErrorCodeFileDigestInvalid: "file digest invalid", - ErrorCodeFilePayloadInvalid: "file payload invalid", - ErrorCodeIndexFileNameInvalid: "index filename invalid", - ErrorCodeIndexFileCountInvalid: "index file count invalid", - ErrorCodeIndexFileSizeInvalid: "index file size invalid", - ErrorCodeTextFileCountInvalid: "text file count invalid", - ErrorCodeImageFileCountInvalid: "file count invalid", - ErrorCodeImageFileSizeInvalid: "file size invalid", - ErrorCodeMetadataCountInvalid: "metadata count invalid", - ErrorCodeMetadataDigestInvalid: "metadata digest invalid", - ErrorCodeMetadataPayloadInvalid: "metadata pyaload invalid", - ErrorCodePropMetadataNotFound: "proposal metadata not found", - ErrorCodePropNameInvalid: "proposal name invalid", - ErrorCodePropLinkToInvalid: "proposal link to invalid", - ErrorCodePropLinkByInvalid: "proposal link by invalid", - ErrorCodePropTokenInvalid: "proposal token invalid", - ErrorCodePropNotFound: "proposal not found", - ErrorCodePropStateInvalid: "proposal state invalid", - ErrorCodePropStatusInvalid: "proposal status invalid", - ErrorCodePropStatusChangeInvalid: "proposal status change invalid", - ErrorCodePropStatusChangeReasonInvalid: "proposal status reason invalid", - ErrorCodeNoPropChanges: "no proposal changes", } ) @@ -149,24 +94,6 @@ func (e ServerErrorReply) Error() string { return fmt.Sprintf("server error: %v", e.ErrorCode) } -// PropStateT represents a proposal state. A proposal state can be either -// unvetted or vetted. The PropStatusT type further breaks down these two -// states into more granular statuses. -type PropStateT int - -const ( - // PropStateInvalid indicates an invalid proposal state. - PropStateInvalid PropStateT = 0 - - // PropStateUnvetted indicates a proposal has not been made public - // yet. Only admins and the proposal author are able to view - // unvetted proposals. - PropStateUnvetted PropStateT = 1 - - // PropStateVetted indicates a proposal has been made public. - PropStateVetted PropStateT = 2 -) - // PropStatusT represents a proposal status. type PropStatusT int @@ -193,10 +120,14 @@ const ( // before or after it was made public. PropStatusCensored PropStatusT = 3 + // PropStatusUnreviewedChanges is deprecated. It is only here so + // the proposal status numbering maps directly to the record status + // numbering. + PropStatusUnreviewedChanges PropStatusT = 4 + // PropStatusAbandoned indicates that a proposal has been marked // as abandoned by an admin due to the author being inactive. - // TODO can a unvetted proposal be abandoned? - PropStatusAbandoned PropStatusT = 4 + PropStatusAbandoned PropStatusT = 5 ) // PropStatuses contains the human readable proposal statuses. @@ -240,9 +171,9 @@ type ProposalMetadata struct { Name string `json:"name"` // Proposal name } -// VoteMetadata that is specified by the user on proposal submission in order to -// host or participate in certain types of votes. It is attached to a proposal -// submission as a Metadata object. +// VoteMetadata is metadata that is specified by the user on proposal +// submission in order to host or participate in certain types of votes. It is +// attached to a proposal submission as a Metadata object. type VoteMetadata struct { // LinkBy is set when the user intends for the proposal to be the // parent proposal in a runoff vote. It is a UNIX timestamp that @@ -291,7 +222,7 @@ type StatusChange struct { type ProposalRecord struct { Version string `json:"version"` // Proposal version Timestamp int64 `json:"timestamp"` // Submission UNIX timestamp - State PropStateT `json:"state"` // Proposal state + State string `json:"state"` // Proposal state Status PropStatusT `json:"status"` // Proposal status UserID string `json:"userid"` // Author ID Username string `json:"username"` // Author username @@ -306,82 +237,23 @@ type ProposalRecord struct { CensorshipRecord CensorshipRecord `json:"censorshiprecord"` } -// ProposalNew submits a new proposal. -// -// Metadata must contain a ProposalMetadata object. -// -// Signature is the client signature of the proposal merkle root. The merkle -// root is the ordered merkle root of all proposal Files and Metadata. -type ProposalNew struct { - Files []File `json:"files"` // Proposal files - Metadata []Metadata `json:"metadata"` // User defined metadata - PublicKey string `json:"publickey"` // Key used for signature - Signature string `json:"signature"` // Signature of merkle root -} - -// ProposalNewReply is the reply to the ProposalNew command. -type ProposalNewReply struct { - Proposal ProposalRecord `json:"proposal"` -} - -// ProposalEdit edits an existing proposal. -// -// Metadata must contain a ProposalMetadata object. -// -// Signature is the client signature of the proposal merkle root. The merkle -// root is the ordered merkle root of all proposal Files and Metadata. -type ProposalEdit struct { - Token string `json:"token"` // Censorship token - State PropStateT `json:"state"` // Proposal state - Files []File `json:"files"` // Proposal files - Metadata []Metadata `json:"metadata"` // User defined metadata - PublicKey string `json:"publickey"` // Key used for signature - Signature string `json:"signature"` // Signature of merkle root -} - -// ProposalEditReply is the reply to the ProposalEdit command. -type ProposalEditReply struct { - Proposal ProposalRecord `json:"proposal"` -} - -// ProposalSetStatus sets the status of a proposal. Some status changes require -// a reason to be included. -// -// Signature is the client signature of the Token+Version+Status+Reason. -type ProposalSetStatus struct { - Token string `json:"token"` // Censorship token - State PropStateT `json:"state"` // Proposal state - Version string `json:"version"` // Proposal version - Status PropStatusT `json:"status"` // New status - Reason string `json:"reason,omitempty"` // Reason for status change - PublicKey string `json:"publickey"` // Key used for signature - Signature string `json:"signature"` // Client signature -} - -// ProposalSetStatusReply is the reply to the ProposalSetStatus command. -type ProposalSetStatusReply struct { - Proposal ProposalRecord `json:"proposal"` -} - -// ProposalRequest is used to request the ProposalRecord of the provided -// proposal token and version. If the version is omitted, the most recent -// version will be returned. +// ProposalRequest is used to request a ProposalRecord. If the version is +// omitted, the most recent version will be returned. type ProposalRequest struct { Token string `json:"token"` Version string `json:"version,omitempty"` } // Proposals retrieves the ProposalRecord for each of the provided proposal -// requests. Unvetted proposals are stripped of their user defined files and -// metadata when being returned to non-admins. +// requests. // -// IncludeFiles specifies whether the proposal files should be returned. The -// user defined metadata will still be returned even when IncludeFiles is set -// to false. +// This command does not return user submitted proposal files or metadata, +// except for the ProposalMetadata, which contains the proposal name. All other +// user submitted data isi removed. Unvetted proposals are also stripped of the +// ProposalMetadata when being returned to non-admins. type Proposals struct { - State PropStateT `json:"state"` - Requests []ProposalRequest `json:"requests,omitempty"` - IncludeFiles bool `json:"includefiles,omitempty"` + State string `json:"state"` + Requests []ProposalRequest `json:"requests"` } // ProposalsReply is the reply to the Proposals command. Any tokens that did diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go index 4a412f6e4..2811b8297 100644 --- a/politeiawww/api/records/v1/v1.go +++ b/politeiawww/api/records/v1/v1.go @@ -19,7 +19,7 @@ const ( RouteInventory = "/inventory" RouteTimestamps = "/timestamps" - // User metadata routes + // Metadata routes RouteUserRecords = "/userrecords" // Record states @@ -32,19 +32,19 @@ type ErrorCodeT int const ( // Error codes - ErrorCodeInvalid ErrorCodeT = 0 - ErrorCodeInputInvalid ErrorCodeT = 1 - ErrorCodeRecordNotFound ErrorCodeT = 2 - ErrorCodeRecordStateInvalid ErrorCodeT = 3 + ErrorCodeInvalid ErrorCodeT = 0 + ErrorCodeInputInvalid ErrorCodeT = iota + ErrorCodePublicKeyInvalid + ErrorCodeSignatureInvalid + ErrorCodeRecordStateInvalid + ErrorCodeRecordNotFound ) var ( // ErrorCodes contains the human readable errors. ErrorCodes = map[ErrorCodeT]string{ - ErrorCodeInvalid: "error invalid", - ErrorCodeInputInvalid: "input invalid", - ErrorCodeRecordNotFound: "record not found", - ErrorCodeRecordStateInvalid: "record state invalid", + ErrorCodeInvalid: "error invalid", + ErrorCodeInputInvalid: "input invalid", } ) @@ -87,47 +87,47 @@ func (e ServerErrorReply) Error() string { return fmt.Sprintf("server error: %v", e.ErrorCode) } -// StatusT represents a record status. -type StatusT int +// RecordStatusT represents a record status. +type RecordStatusT int const ( - // StatusInvalid is an invalid record status. - StatusInvalid StatusT = 0 + // RecordStatusInvalid is an invalid record status. + RecordStatusInvalid RecordStatusT = 0 - // StatusUnreviewed indicates that a record has been submitted but - // has not been made public yet. A record with this status will - // have a state of unvetted. - StatusUnreviewed StatusT = 1 + // RecordStatusUnreviewed indicates that a record has been + // submitted but has not been made public yet. A record with + // this status will have a state of unvetted. + RecordStatusUnreviewed RecordStatusT = 1 - // StatusPublic indicates that a record has been made public. A - // record with this status will have a state of vetted. - StatusPublic StatusT = 2 + // RecordStatusPublic indicates that a record has been made public. + // A record with this status will have a state of vetted. + RecordStatusPublic RecordStatusT = 2 - // StatusCensored indicates that a record has been censored. The - // record state can be either unvetted or vetted depending on + // RecordStatusCensored indicates that a record has been censored. + // The record state can be either unvetted or vetted depending on // whether the record was censored before or after it was made // public. All user submitted content of a censored record will // have been permanently deleted. - StatusCensored StatusT = 3 + RecordStatusCensored RecordStatusT = 3 - // StatusUnreviewedChanges has been deprecated. - StatusUnreviewedChanges StatusT = 4 + // RecordStatusUnreviewedChanges has been deprecated. + RecordStatusUnreviewedChanges RecordStatusT = 4 - // StatusArchived represents a record that has been archived. Both - // unvetted and vetted records can be marked as archived. Unlike - // with censored records, the user submitted content of an archived - // record is not deleted. - StatusArchived StatusT = 5 + // RecordStatusArchived represents a record that has been archived. + // Both unvetted and vetted records can be marked as archived. + // Unlike with censored records, the user submitted content of an + // archived record is not deleted. + RecordStatusArchived RecordStatusT = 5 ) var ( - // Statuses contains the human readable record statuses. - Statuses = map[StatusT]string{ - StatusInvalid: "invalid", - StatusUnreviewed: "unreviewed", - StatusPublic: "public", - StatusCensored: "censored", - StatusArchived: "archived", + // RecordStatuses contains the human readable record statuses. + RecordStatuses = map[RecordStatusT]string{ + RecordStatusInvalid: "invalid", + RecordStatusUnreviewed: "unreviewed", + RecordStatusPublic: "public", + RecordStatusCensored: "censored", + RecordStatusArchived: "archived", } ) @@ -163,15 +163,41 @@ type CensorshipRecord struct { // Record represents a record and all of its content. type Record struct { State string `json:"state"` // Record state - Status StatusT `json:"status"` // Record status + Status RecordStatusT `json:"status"` // Record status Version string `json:"version"` // Version of this record Timestamp int64 `json:"timestamp"` // Last update + Username string `json:"username"` // Author username Metadata []MetadataStream `json:"metadata"` // Metadata streams Files []File `json:"files"` // User submitted files CensorshipRecord CensorshipRecord `json:"censorshiprecord"` } +// UserMetadata contains user metadata about a politeiad record. It is +// generated by the server and saved to politeiad as a metadata stream. +// +// Signature is the client signature of the record merkle root. The merkle root +// is the ordered merkle root of all user submitted politeiad files. +type UserMetadata struct { + UserID string `json:"userid"` // Author user ID + PublicKey string `json:"publickey"` // Key used for signature + Signature string `json:"signature"` // Signature of merkle root +} + +// StatusChange represents a record status change. It is generated by the +// server and saved to politeiad as a metadata stream. +// +// Signature is the client signature of the Token+Version+Status+Reason. +type StatusChange struct { + Token string `json:"token"` + Version string `json:"version"` + Status RecordStatusT `json:"status"` + Reason string `json:"message,omitempty"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` + Timestamp int64 `json:"timestamp"` +} + // New submits a new record. // // Signature is the client signature of the record merkle root. The merkle root @@ -209,13 +235,13 @@ type EditReply struct { // // Signature is the client signature of the Token+Version+Status+Reason. type SetStatus struct { - State string `json:"state"` - Token string `json:"token"` - Version string `json:"version"` - Status StatusT `json:"status"` - Reason string `json:"reason,omitempty"` - PublicKey string `json:"publickey"` - Signature string `json:"signature"` + State string `json:"state"` + Token string `json:"token"` + Version string `json:"version"` + Status RecordStatusT `json:"status"` + Reason string `json:"reason,omitempty"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` } // SetStatusReply is the reply to the SetStatus command. @@ -225,8 +251,9 @@ type SetStatusReply struct { // Details requests the details of a record. The full record will be returned. type Details struct { - Token string `json:"token"` // Censorship token - State string `json:"state"` // Record state + Token string `json:"token"` + State string `json:"state"` + Version string `json:"version"` } // DetailsReply is the reply to the Details command. @@ -244,7 +271,8 @@ type RecordRequest struct { // Records requests a batch of records. Only the record metadata is returned. // The Details command must be used to retrieve the record files. type Records struct { - State string `json:"state"` // Record state + State string `json:"state"` + Requests []RecordRequest `json:"requests"` } // RecordsReply is the reply to the Records command. Any tokens that did not @@ -312,3 +340,14 @@ type TimestampsReply struct { // map[filename]Timestamp Files map[string]Timestamp `json:"files"` } + +// UserRecords requests the tokens of all records submitted by a user. +type UserRecords struct { + State string `json:"state"` + UserID string `json:"userid"` +} + +// UserRecordsReply is the reply to the UserRecords command. +type UserRecordsReply struct { + Tokens []string `json:"tokens"` +} diff --git a/politeiawww/api/www/v1/v1.go b/politeiawww/api/www/v1/v1.go index 6302108f4..e2386ce82 100644 --- a/politeiawww/api/www/v1/v1.go +++ b/politeiawww/api/www/v1/v1.go @@ -995,10 +995,10 @@ type PolicyReply struct { TokenPrefixLength int `json:"tokenprefixlength"` BuildInformation []string `json:"buildinformation"` IndexFilename string `json:"indexfilename"` - MinLinkByPeriod int64 `json:"minlinkbyperiod"` - MaxLinkByPeriod int64 `json:"maxlinkbyperiod"` - MinVoteDuration uint32 `json:"minvoteduration"` - MaxVoteDuration uint32 `json:"maxvoteduration"` + MinLinkByPeriod int64 `json:"minlinkbyperiod"` // DEPRECATED + MaxLinkByPeriod int64 `json:"maxlinkbyperiod"` // DEPRECATED + MinVoteDuration uint32 `json:"minvoteduration"` // DEPRECATED + MaxVoteDuration uint32 `json:"maxvoteduration"` // DEPRECATED } // VoteOption describes a single vote option. diff --git a/politeiawww/comments/events.go b/politeiawww/comments/events.go index 8ab3aa439..1ba453fd0 100644 --- a/politeiawww/comments/events.go +++ b/politeiawww/comments/events.go @@ -11,7 +11,7 @@ const ( EventTypeNew = "comments-new" ) -// EventNew is the event data for EventTypeNew. +// EventNew is the event data for the EventTypeNew. type EventNew struct { State string Comment v1.Comment diff --git a/politeiawww/comments/process.go b/politeiawww/comments/process.go index 2ca5e81fb..bbb30ca4d 100644 --- a/politeiawww/comments/process.go +++ b/politeiawww/comments/process.go @@ -17,8 +17,8 @@ import ( func (c *Comments) processNew(ctx context.Context, n v1.New, u user.User) (*v1.NewReply, error) { log.Tracef("processNew: %v %v %v", n.Token, u.Username) - // Checking the mode is a temporary measure until user plugins - // have been implemented. + // Execute pre plugin hooks. Checking the mode is a temporary + // measure until user plugins have been properly implemented. switch c.cfg.Mode { case config.PoliteiaWWWMode: // Verify user has paid registration paywall @@ -69,7 +69,7 @@ func (c *Comments) processNew(ctx context.Context, n v1.New, u user.User) (*v1.N // Prepare reply cm := convertComment(cnr.Comment) - cm = commentPopulateUser(cm, u) + cm = commentPopulateUserData(cm, u) // Emit event c.events.Emit(EventTypeNew, @@ -162,7 +162,7 @@ func (c *Comments) processDel(ctx context.Context, d v1.Del, u user.User) (*v1.D // Prepare reply cm := convertComment(cdr.Comment) - cm = commentPopulateUser(cm, u) + cm = commentPopulateUserData(cm, u) return &v1.DelReply{ Comment: cm, @@ -237,7 +237,7 @@ func (c *Comments) processComments(ctx context.Context, cs v1.Comments, u *user. if err != nil { return nil, err } - cm = commentPopulateUser(cm, *u) + cm = commentPopulateUserData(cm, *u) // Add comment comments = append(comments, cm) @@ -298,7 +298,7 @@ func (c *Comments) processTimestamps(ctx context.Context, t v1.Timestamps, isAdm // commentPopulateUserData populates the comment with user data that is not // stored in politeiad. -func commentPopulateUser(c v1.Comment, u user.User) v1.Comment { +func commentPopulateUserData(c v1.Comment, u user.User) v1.Comment { c.Username = u.Username return c } @@ -367,12 +367,16 @@ func convertTimestamp(t comments.Timestamp) v1.Timestamp { } } +// paywallIsEnabled returns whether the user paywall is enabled. +// // This function is a temporary function that will be removed once user plugins // have been implemented. func (c *Comments) paywallIsEnabled() bool { return c.cfg.PaywallAmount != 0 && c.cfg.PaywallXpub != "" } +// userHasPaid returns whether the user has paid their user registration fee. +// // This function is a temporary function that will be removed once user plugins // have been implemented. func (c *Comments) userHasPaid(u user.User) bool { diff --git a/politeiawww/email.go b/politeiawww/email.go index 9481ebae3..51a4835bc 100644 --- a/politeiawww/email.go +++ b/politeiawww/email.go @@ -12,7 +12,7 @@ import ( "text/template" "time" - pi "github.com/decred/politeia/politeiawww/api/pi/v1" + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" ) @@ -119,7 +119,7 @@ func (p *politeiawww) emailProposalStatusChange(d dataProposalStatusChange, prop body string ) switch d.status { - case pi.PropStatusPublic: + case rcv1.StatusPublic: subject = "New Proposal Published" tmplData := proposalVetted{ Name: proposalName, @@ -152,7 +152,7 @@ func (p *politeiawww) emailProposalStatusChangeToAuthor(d dataProposalStatusChan body string ) switch d.status { - case pi.PropStatusPublic: + case rcv1.StatusPublic: subject = "Your Proposal Has Been Published" tmplData := proposalVettedToAuthor{ Name: proposalName, @@ -163,7 +163,7 @@ func (p *politeiawww) emailProposalStatusChangeToAuthor(d dataProposalStatusChan return err } - case pi.PropStatusCensored: + case rcv1.StatusCensored: subject = "Your Proposal Has Been Censored" tmplData := proposalCensoredToAuthor{ Name: proposalName, diff --git a/politeiawww/events.go b/politeiawww/events.go index 3122eb40e..8a41884c1 100644 --- a/politeiawww/events.go +++ b/politeiawww/events.go @@ -5,12 +5,11 @@ package main import ( - "context" "fmt" "strconv" "github.com/decred/politeia/politeiad/plugins/comments" - pi "github.com/decred/politeia/politeiawww/api/pi/v1" + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/user" "github.com/google/uuid" @@ -206,12 +205,12 @@ func (p *politeiawww) handleEventProposalEdited(ch chan interface{}) { } type dataProposalStatusChange struct { - token string // Proposal censorship token - state pi.PropStateT // Updated proposal state - status pi.PropStatusT // Updated proposal status - version string // Proposal version - reason string // Status change reason - adminID string // Admin uuid + token string // Proposal censorship token + state string // Updated proposal state + status rcv1.StatusT // Updated proposal status + version string // Proposal version + reason string // Status change reason + adminID string // Admin uuid } func (p *politeiawww) handleEventProposalStatusChange(ch chan interface{}) { @@ -224,7 +223,7 @@ func (p *politeiawww) handleEventProposalStatusChange(ch chan interface{}) { // Check if proposal is in correct status for notification switch d.status { - case pi.PropStatusPublic, pi.PropStatusCensored: + case rcv1.StatusPublic, rcv1.StatusCensored: // The status requires a notification be sent default: // The status does not require a notification be sent. Listen @@ -232,61 +231,64 @@ func (p *politeiawww) handleEventProposalStatusChange(ch chan interface{}) { continue } - // Get the proposal author - pr, err := p.proposalRecordLatest(context.Background(), d.state, d.token) - if err != nil { - log.Errorf("handleEventProposalStatusChange: proposalRecordLatest "+ - "%v %v: %v", d.state, d.token, err) - continue - } - author, err := p.db.UserGetByPubKey(pr.PublicKey) - if err != nil { - log.Errorf("handleEventProposalStatusChange: UserGetByPubKey %v: %v", - pr.PublicKey, err) - continue - } - - // Email author - proposalName := proposalName(*pr) - notification := www.NotificationEmailRegularProposalVetted - if userNotificationEnabled(*author, notification) { - err = p.emailProposalStatusChangeToAuthor(d, proposalName, author.Email) + // TODO + /* + // Get the proposal author + pr, err := p.proposalRecordLatest(context.Background(), d.state, d.token) if err != nil { - log.Errorf("emailProposalStatusChangeToAuthor: %v", err) + log.Errorf("handleEventProposalStatusChange: proposalRecordLatest "+ + "%v %v: %v", d.state, d.token, err) + continue + } + author, err := p.db.UserGetByPubKey(pr.PublicKey) + if err != nil { + log.Errorf("handleEventProposalStatusChange: UserGetByPubKey %v: %v", + pr.PublicKey, err) continue } - } - // Compile list of users to send the notification to - emails := make([]string, 0, 256) - err = p.db.AllUsers(func(u *user.User) { - switch { - case u.ID.String() == d.adminID: - // User is the admin that made the status change - return - case u.ID.String() == author.ID.String(): - // User is the author. The author is sent a different - // notification. Don't include them in the users list. - return - case !userNotificationEnabled(*u, notification): - // User does not have notification bit set - return + // Email author + proposalName := proposalName(*pr) + notification := www.NotificationEmailRegularProposalVetted + if userNotificationEnabled(*author, notification) { + err = p.emailProposalStatusChangeToAuthor(d, proposalName, author.Email) + if err != nil { + log.Errorf("emailProposalStatusChangeToAuthor: %v", err) + continue + } } - // Add user to notification list - emails = append(emails, u.Email) - }) - if err != nil { - log.Errorf("handleEventProposalStatusChange: AllUsers: %v", err) - continue - } + // Compile list of users to send the notification to + emails := make([]string, 0, 256) + err = p.db.AllUsers(func(u *user.User) { + switch { + case u.ID.String() == d.adminID: + // User is the admin that made the status change + return + case u.ID.String() == author.ID.String(): + // User is the author. The author is sent a different + // notification. Don't include them in the users list. + return + case !userNotificationEnabled(*u, notification): + // User does not have notification bit set + return + } + + // Add user to notification list + emails = append(emails, u.Email) + }) + if err != nil { + log.Errorf("handleEventProposalStatusChange: AllUsers: %v", err) + continue + } - // Email users - err = p.emailProposalStatusChange(d, proposalName, emails) - if err != nil { - log.Errorf("emailProposalStatusChange: %v", err) - continue - } + // Email users + err = p.emailProposalStatusChange(d, proposalName, emails) + if err != nil { + log.Errorf("emailProposalStatusChange: %v", err) + continue + } + */ log.Debugf("Sent proposal status change notifications %v", d.token) } @@ -368,7 +370,7 @@ func (p *politeiawww) notifyParentAuthorOnComment(d dataProposalComment, proposa } type dataProposalComment struct { - state pi.PropStateT + state string token string commentID uint32 parentID uint32 @@ -383,39 +385,42 @@ func (p *politeiawww) handleEventProposalComment(ch chan interface{}) { continue } - // Fetch the proposal record here to avoid calling this two times - // on the notify functions below - pr, err := p.proposalRecordLatest(context.Background(), d.state, - d.token) - if err != nil { - err = fmt.Errorf("proposalRecordLatest %v %v: %v", - d.state, d.token, err) - goto next - } + _ = d + /* TODO + // Fetch the proposal record here to avoid calling this two times + // on the notify functions below + pr, err := p.proposalRecordLatest(context.Background(), d.state, + d.token) + if err != nil { + err = fmt.Errorf("proposalRecordLatest %v %v: %v", + d.state, d.token, err) + goto next + } - // Notify the proposal author - err = p.notifyProposalAuthorOnComment(d, pr.UserID, proposalName(*pr)) - if err != nil { - err = fmt.Errorf("notifyProposalAuthorOnComment: %v", err) - goto next - } + // Notify the proposal author + err = p.notifyProposalAuthorOnComment(d, pr.UserID, proposalName(*pr)) + if err != nil { + err = fmt.Errorf("notifyProposalAuthorOnComment: %v", err) + goto next + } - // Notify the parent comment author - err = p.notifyParentAuthorOnComment(d, proposalName(*pr)) - if err != nil { - err = fmt.Errorf("notifyParentAuthorOnComment: %v", err) - goto next - } + // Notify the parent comment author + err = p.notifyParentAuthorOnComment(d, proposalName(*pr)) + if err != nil { + err = fmt.Errorf("notifyParentAuthorOnComment: %v", err) + goto next + } - // Notifications successfully sent - log.Debugf("Sent proposal commment notification %v", d.token) - continue + // Notifications successfully sent + log.Debugf("Sent proposal commment notification %v", d.token) + continue - next: - // If we made it here then there was an error. Log the error - // before listening for the next event. - log.Errorf("handleEventProposalComment: %v", err) - continue + next: + // If we made it here then there was an error. Log the error + // before listening for the next event. + log.Errorf("handleEventProposalComment: %v", err) + continue + */ } } diff --git a/politeiawww/log.go b/politeiawww/log.go index cdbf2e612..ea7f9981c 100644 --- a/politeiawww/log.go +++ b/politeiawww/log.go @@ -55,9 +55,6 @@ var ( wsdcrdataLog = backendLog.Logger("WSDD") githubTrackerLog = backendLog.Logger("GHTR") githubdbLog = backendLog.Logger("GHDB") - sessionsLog = backendLog.Logger("SESS") - commentsLog = backendLog.Logger("COMT") - ticketvoteLog = backendLog.Logger("TICK") ) // Initialize package-global logger variables. @@ -68,9 +65,9 @@ func init() { wsdcrdata.UseLogger(wsdcrdataLog) github.UseLogger(githubTrackerLog) ghdb.UseLogger(githubdbLog) - sessions.UseLogger(sessionsLog) - comments.UseLogger(commentsLog) - ticketvote.UseLogger(ticketvoteLog) + sessions.UseLogger(log) + comments.UseLogger(log) + ticketvote.UseLogger(log) } // subsystemLoggers maps each subsystem identifier to its associated logger. @@ -81,9 +78,6 @@ var subsystemLoggers = map[string]slog.Logger{ "WSDD": wsdcrdataLog, "GHTR": githubTrackerLog, "GHDB": githubdbLog, - "SESS": sessionsLog, - "COMT": commentsLog, - "TICK": ticketvoteLog, } // initLogRotator initializes the logging rotater to write logs to logFile and diff --git a/politeiawww/paywall.go b/politeiawww/paywall.go index 40a8b0f0b..10b3fff69 100644 --- a/politeiawww/paywall.go +++ b/politeiawww/paywall.go @@ -10,7 +10,6 @@ import ( "fmt" "time" - www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/user" "github.com/decred/politeia/util" "github.com/google/uuid" @@ -266,50 +265,3 @@ func (p *politeiawww) verifyProposalPayment(ctx context.Context, u *user.User) ( return nil, nil } - -// userHasPaid returns whether the user has paid the user registration paywall. -func (p *politeiawww) userHasPaid(u user.User) bool { - // Return true if paywall is disabled - if !p.paywallIsEnabled() { - return true - } - - return u.NewUserPaywallTx != "" -} - -func proposalCreditBalance(u user.User) uint64 { - return uint64(len(u.UnspentProposalCredits)) -} - -// userHasProposalCredits returns whether the user has at least 1 unspent -// proposal credit. -func (p *politeiawww) userHasProposalCredits(u user.User) bool { - if !p.paywallIsEnabled() { - return true - } - return proposalCreditBalance(u) > 0 -} - -// spendProposalCredit updates an unspent proposal credit with the passed in -// censorship token, moves the credit into the user's spent proposal credits -// list, and then updates the user database. -func (p *politeiawww) spendProposalCredit(u *user.User, token string) error { - // Skip when running unit tests or if paywall is disabled. - if !p.paywallIsEnabled() { - return nil - } - - if !p.userHasProposalCredits(*u) { - return www.UserError{ - ErrorCode: www.ErrorStatusNoProposalCredits, - } - } - - creditToSpend := u.UnspentProposalCredits[0] - creditToSpend.CensorshipToken = token - u.SpentProposalCredits = append(u.SpentProposalCredits, creditToSpend) - u.UnspentProposalCredits = u.UnspentProposalCredits[1:] - - err := p.db.UserUpdate(*u) - return err -} diff --git a/politeiawww/pi.go b/politeiawww/pi.go deleted file mode 100644 index 37424062f..000000000 --- a/politeiawww/pi.go +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) 2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import ( - "context" - - "github.com/decred/politeia/politeiad/plugins/pi" -) - -// proposalInv returns the pi plugin proposal inventory. -func (p *politeiawww) proposalInv(ctx context.Context, inv pi.ProposalInv) (*pi.ProposalInvReply, error) { - return nil, nil -} - -// piVoteInventory returns the pi plugin vote inventory. -func (p *politeiawww) piVoteInventory(ctx context.Context) (*pi.VoteInventoryReply, error) { - return nil, nil -} diff --git a/politeiawww/pi/pi.go b/politeiawww/pi/pi.go new file mode 100644 index 000000000..bedcea9dd --- /dev/null +++ b/politeiawww/pi/pi.go @@ -0,0 +1,319 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package pi + +/* +var ( + // errProposalNotFound is emitted when a proposal is not found in + // politeiad for a specified token and version. + errProposalNotFound = errors.New("proposal not found") +) + +// proposalName parses the proposal name from the ProposalMetadata and returns +// it. An empty string will be returned if any errors occur or if a name is not +// found. +func proposalName(r pdv1.Record) string { + var name string + for _, v := range r.Files { + if v.Name == pi.FileNameProposalMetadata { + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return "" + } + var pm pi.ProposalMetadata + err = json.Unmarshal(b, &pm) + if err != nil { + return "" + } + name = pm.Name + } + } + return name +} + +// proposalRecordFillInUser fills in all user fields that are store in the +// user database and not in politeiad. +func proposalRecordFillInUser(pr piv1.ProposalRecord, u user.User) piv1.ProposalRecord { + pr.UserID = u.ID.String() + pr.Username = u.Username + return pr +} + +func convertRecordStatusFromPropStatus(s piv1.PropStatusT) pdv1.RecordStatusT { + switch s { + case piv1.PropStatusUnreviewed: + return pdv1.RecordStatusNotReviewed + case piv1.PropStatusPublic: + return pdv1.RecordStatusPublic + case piv1.PropStatusCensored: + return pdv1.RecordStatusCensored + case piv1.PropStatusAbandoned: + return pdv1.RecordStatusArchived + } + return pdv1.RecordStatusInvalid +} + +func convertFileFromMetadata(m piv1.Metadata) pdv1.File { + var name string + switch m.Hint { + case piv1.HintProposalMetadata: + name = pi.FileNameProposalMetadata + } + return pdv1.File{ + Name: name, + MIME: mimeTypeTextUTF8, + Digest: m.Digest, + Payload: m.Payload, + } +} + +func convertFileFromPi(f piv1.File) pdv1.File { + return pdv1.File{ + Name: f.Name, + MIME: f.MIME, + Digest: f.Digest, + Payload: f.Payload, + } +} + +func convertFilesFromPi(files []piv1.File) []pdv1.File { + f := make([]pdv1.File, 0, len(files)) + for _, v := range files { + f = append(f, convertFileFromPi(v)) + } + return f +} + +func convertPropStatusFromPD(s pdv1.RecordStatusT) piv1.PropStatusT { + switch s { + case pdv1.RecordStatusNotFound: + // Intentionally omitted. No corresponding PropStatusT. + case pdv1.RecordStatusNotReviewed: + return piv1.PropStatusUnreviewed + case pdv1.RecordStatusCensored: + return piv1.PropStatusCensored + case pdv1.RecordStatusPublic: + return piv1.PropStatusPublic + case pdv1.RecordStatusUnreviewedChanges: + return piv1.PropStatusUnreviewed + case pdv1.RecordStatusArchived: + return piv1.PropStatusAbandoned + } + return piv1.PropStatusInvalid +} + +func convertCensorshipRecordFromPD(cr pdv1.CensorshipRecord) piv1.CensorshipRecord { + return piv1.CensorshipRecord{ + Token: cr.Token, + Merkle: cr.Merkle, + Signature: cr.Signature, + } +} + +func convertFilesFromPD(f []pdv1.File) ([]piv1.File, []piv1.Metadata) { + files := make([]piv1.File, 0, len(f)) + metadata := make([]piv1.Metadata, 0, len(f)) + for _, v := range f { + switch v.Name { + case pi.FileNameProposalMetadata: + metadata = append(metadata, piv1.Metadata{ + Hint: piv1.HintProposalMetadata, + Digest: v.Digest, + Payload: v.Payload, + }) + default: + files = append(files, piv1.File{ + Name: v.Name, + MIME: v.MIME, + Digest: v.Digest, + Payload: v.Payload, + }) + } + } + return files, metadata +} + +func statusChangesDecode(payload []byte) ([]rcv1.StatusChange, error) { + var statuses []rcv1.StatusChange + d := json.NewDecoder(strings.NewReader(string(payload))) + for { + var sc rcv1.StatusChange + err := d.Decode(&sc) + if errors.Is(err, io.EOF) { + break + } else if err != nil { + return nil, err + } + statuses = append(statuses, sc) + } + return statuses, nil +} + +func convertProposalRecordFromPD(r pdv1.Record, state piv1.PropStateT) (*piv1.ProposalRecord, error) { + // Decode metadata streams + var ( + um *usermd.UserMetadata + sc = make([]pi.StatusChange, 0, 16) + err error + ) + for _, v := range r.Metadata { + switch v.ID { + case usermd.MDStreamIDUserMetadata: + var um usermd.UserMetadata + err = json.Unmarshal([]byte(v.Payload), &um) + if err != nil { + return nil, err + } + case pi.MDStreamIDStatusChanges: + sc, err = statusChangesDecode([]byte(v.Payload)) + if err != nil { + return nil, err + } + } + } + + // Convert to pi types + files, metadata := convertFilesFromPD(r.Files) + status := convertPropStatusFromPD(r.Status) + + statuses := make([]piv1.StatusChange, 0, len(sc)) + for _, v := range sc { + statuses = append(statuses, piv1.StatusChange{ + Token: v.Token, + Version: v.Version, + Status: piv1.PropStatusT(v.Status), + Reason: v.Reason, + PublicKey: v.PublicKey, + Signature: v.Signature, + Timestamp: v.Timestamp, + }) + } + + // Some fields are intentionally omitted because they are either + // user data that needs to be pulled from the user database or they + // are politeiad plugin data that needs to be retrieved using a + // plugin command. + return &piv1.ProposalRecord{ + Version: r.Version, + Timestamp: r.Timestamp, + State: state, + Status: status, + UserID: um.UserID, + Username: "", // Intentionally omitted + PublicKey: um.PublicKey, + Signature: um.Signature, + Statuses: statuses, + Files: files, + Metadata: metadata, + CensorshipRecord: convertCensorshipRecordFromPD(r.CensorshipRecord), + }, nil +} + +// proposalRecords returns the ProposalRecord for each of the provided proposal +// requests. If a token does not correspond to an actual proposal then it will +// not be included in the returned map. +func (p *politeiawww) proposalRecords(ctx context.Context, state piv1.PropStateT, reqs []piv1.ProposalRequest, includeFiles bool) (map[string]piv1.ProposalRecord, error) { + // Get politeiad records + props := make([]piv1.ProposalRecord, 0, len(reqs)) + for _, v := range reqs { + var r *pdv1.Record + var err error + switch state { + case piv1.PropStateUnvetted: + // Unvetted politeiad record + r, err = p.politeiad.GetUnvetted(ctx, v.Token, v.Version) + if err != nil { + return nil, err + } + case piv1.PropStateVetted: + // Vetted politeiad record + r, err = p.politeiad.GetVetted(ctx, v.Token, v.Version) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unknown state %v", state) + } + + if r.Status == pdv1.RecordStatusNotFound { + // Record wasn't found. Don't include token in the results. + continue + } + + pr, err := convertProposalRecordFromPD(*r, state) + if err != nil { + return nil, err + } + + // Remove files if specified. The Metadata objects will still be + // returned. + if !includeFiles { + pr.Files = []piv1.File{} + } + + props = append(props, *pr) + } + + // Verify we've got some results + if len(props) == 0 { + return map[string]piv1.ProposalRecord{}, nil + } + + // Get user data + pubkeys := make([]string, 0, len(props)) + for _, v := range props { + pubkeys = append(pubkeys, v.PublicKey) + } + ur, err := p.db.UsersGetByPubKey(pubkeys) + if err != nil { + return nil, err + } + for k, v := range props { + token := v.CensorshipRecord.Token + u, ok := ur[v.PublicKey] + if !ok { + return nil, fmt.Errorf("user not found for pubkey %v from proposal %v", + v.PublicKey, token) + } + props[k] = proposalRecordFillInUser(v, u) + } + + // Convert proposals to a map + proposals := make(map[string]piv1.ProposalRecord, len(props)) + for _, v := range props { + proposals[v.CensorshipRecord.Token] = v + } + + return proposals, nil +} + +// proposalRecord returns the proposal record for the provided token and +// version. A blank version will return the most recent version. A +// errProposalNotFound error will be returned if a proposal is not found for +// the provided token/version combination. +func (p *politeiawww) proposalRecord(ctx context.Context, state piv1.PropStateT, token, version string) (*piv1.ProposalRecord, error) { + prs, err := p.proposalRecords(ctx, state, []piv1.ProposalRequest{ + { + Token: token, + Version: version, + }, + }, true) + if err != nil { + return nil, err + } + pr, ok := prs[token] + if !ok { + return nil, errProposalNotFound + } + return &pr, nil +} + +// proposalRecordLatest returns the latest version of the proposal record for +// the provided token. A errProposalNotFound error will be returned if a +// proposal is not found for the provided token. +func (p *politeiawww) proposalRecordLatest(ctx context.Context, state piv1.PropStateT, token string) (*piv1.ProposalRecord, error) { + return p.proposalRecord(ctx, state, token, "") +} +*/ diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index c5ed55d77..ad79f5888 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -5,1218 +5,19 @@ package main import ( - "bytes" - "context" - "encoding/base64" - "encoding/hex" - "encoding/json" - "errors" "fmt" - "io" - "mime" "net/http" - "regexp" - "strconv" - "strings" - "time" - pdv1 "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/politeiad/plugins/pi" - usermd "github.com/decred/politeia/politeiad/plugins/user" cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" - piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" - rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/comments" "github.com/decred/politeia/politeiawww/ticketvote" - "github.com/decred/politeia/politeiawww/user" - wwwutil "github.com/decred/politeia/politeiawww/util" - "github.com/decred/politeia/util" "github.com/google/uuid" ) -// TODO www package references should be completely gone from this file -// TODO use pi policies. Should the policies be defined in the pi plugin -// or the pi api spec? -// TODO ensure plugins can't write data using short proposal token. -// TODO politeiad needs batched calls for retrieving unvetted and vetted -// records. - -const ( - // MIME types - mimeTypeText = "text/plain" - mimeTypeTextUTF8 = "text/plain; charset=utf-8" - mimeTypePNG = "image/png" -) - -var ( - // validProposalName contains the regex that matches a valid - // proposal name. - validProposalName = regexp.MustCompile(proposalNameRegex()) - - // statusReasonRequired contains the list of proposal statuses that - // require an accompanying reason to be given for the status change. - statusReasonRequired = map[piv1.PropStatusT]struct{}{ - piv1.PropStatusCensored: {}, - piv1.PropStatusAbandoned: {}, - } - - // errProposalNotFound is emitted when a proposal is not found in - // politeiad for a specified token and version. - errProposalNotFound = errors.New("proposal not found") -) - -// tokenIsValid returns whether the provided string is a valid politeiad -// censorship record token. This CAN BE EITHER the full length token or the -// token prefix. -// -// Short tokens should only be used when retrieving data. Data that is written -// to disk should always reference the full length token. -func tokenIsValid(token string) bool { - // Verify token size - switch { - case len(token) == pdv1.TokenPrefixLength: - // Token is a short proposal token - case len(token) == pdv1.TokenSizeTlog*2: - // Token is a full length token - default: - // Unknown token size - return false - } - - // Verify token is valid hex - _, err := hex.DecodeString(token) - return err == nil -} - -// tokenIsFullLength returns whether the provided string a is valid, full -// length politeiad censorship record token. Short tokens are considered -// invalid by this function. -func tokenIsFullLength(token string) bool { - b, err := hex.DecodeString(token) - if err != nil { - return false - } - if len(b) != pdv1.TokenSizeTlog { - return false - } - return true -} - -// proposalNameIsValid returns whether the provided proposal name is a valid. -func proposalNameIsValid(name string) bool { - return validProposalName.MatchString(name) -} - -// proposalNameRegex returns a regex string for validating the proposal name. -func proposalNameRegex() string { - var validProposalNameBuffer bytes.Buffer - validProposalNameBuffer.WriteString("^[") - - for _, supportedChar := range www.PolicyProposalNameSupportedChars { - if len(supportedChar) > 1 { - validProposalNameBuffer.WriteString(supportedChar) - } else { - validProposalNameBuffer.WriteString(`\` + supportedChar) - } - } - minNameLength := strconv.Itoa(www.PolicyMinProposalNameLength) - maxNameLength := strconv.Itoa(www.PolicyMaxProposalNameLength) - validProposalNameBuffer.WriteString("]{") - validProposalNameBuffer.WriteString(minNameLength + ",") - validProposalNameBuffer.WriteString(maxNameLength + "}$") - - return validProposalNameBuffer.String() -} - -// proposalName parses the proposal name from the ProposalMetadata and returns -// it. An empty string will be returned if any errors occur or if a name is not -// found. -func proposalName(pr piv1.ProposalRecord) string { - var name string - for _, v := range pr.Metadata { - if v.Hint == piv1.HintProposalMetadata { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return "" - } - var pm pi.ProposalMetadata - err = json.Unmarshal(b, &pm) - if err != nil { - return "" - } - name = pm.Name - } - } - return name -} - -// proposalRecordFillInUser fills in all user fields that are store in the -// user database and not in politeiad. -func proposalRecordFillInUser(pr piv1.ProposalRecord, u user.User) piv1.ProposalRecord { - pr.UserID = u.ID.String() - pr.Username = u.Username - return pr -} - -func convertUserErrorFromSignatureError(err error) piv1.UserErrorReply { - var e util.SignatureError - var s piv1.ErrorStatusT - if errors.As(err, &e) { - switch e.ErrorCode { - case util.ErrorStatusPublicKeyInvalid: - s = piv1.ErrorStatusPublicKeyInvalid - case util.ErrorStatusSignatureInvalid: - s = piv1.ErrorStatusSignatureInvalid - } - } - return piv1.UserErrorReply{ - ErrorCode: s, - ErrorContext: e.ErrorContext, - } -} - -func convertRecordStatusFromPropStatus(s piv1.PropStatusT) pdv1.RecordStatusT { - switch s { - case piv1.PropStatusUnreviewed: - return pdv1.RecordStatusNotReviewed - case piv1.PropStatusPublic: - return pdv1.RecordStatusPublic - case piv1.PropStatusCensored: - return pdv1.RecordStatusCensored - case piv1.PropStatusAbandoned: - return pdv1.RecordStatusArchived - } - return pdv1.RecordStatusInvalid -} - -func convertFileFromMetadata(m piv1.Metadata) pdv1.File { - var name string - switch m.Hint { - case piv1.HintProposalMetadata: - name = pi.FileNameProposalMetadata - } - return pdv1.File{ - Name: name, - MIME: mimeTypeTextUTF8, - Digest: m.Digest, - Payload: m.Payload, - } -} - -func convertFileFromPi(f piv1.File) pdv1.File { - return pdv1.File{ - Name: f.Name, - MIME: f.MIME, - Digest: f.Digest, - Payload: f.Payload, - } -} - -func convertFilesFromPi(files []piv1.File) []pdv1.File { - f := make([]pdv1.File, 0, len(files)) - for _, v := range files { - f = append(f, convertFileFromPi(v)) - } - return f -} - -func convertPropStatusFromPD(s pdv1.RecordStatusT) piv1.PropStatusT { - switch s { - case pdv1.RecordStatusNotFound: - // Intentionally omitted. No corresponding PropStatusT. - case pdv1.RecordStatusNotReviewed: - return piv1.PropStatusUnreviewed - case pdv1.RecordStatusCensored: - return piv1.PropStatusCensored - case pdv1.RecordStatusPublic: - return piv1.PropStatusPublic - case pdv1.RecordStatusUnreviewedChanges: - return piv1.PropStatusUnreviewed - case pdv1.RecordStatusArchived: - return piv1.PropStatusAbandoned - } - return piv1.PropStatusInvalid -} - -func convertCensorshipRecordFromPD(cr pdv1.CensorshipRecord) piv1.CensorshipRecord { - return piv1.CensorshipRecord{ - Token: cr.Token, - Merkle: cr.Merkle, - Signature: cr.Signature, - } -} - -func convertFilesFromPD(f []pdv1.File) ([]piv1.File, []piv1.Metadata) { - files := make([]piv1.File, 0, len(f)) - metadata := make([]piv1.Metadata, 0, len(f)) - for _, v := range f { - switch v.Name { - case pi.FileNameProposalMetadata: - metadata = append(metadata, piv1.Metadata{ - Hint: piv1.HintProposalMetadata, - Digest: v.Digest, - Payload: v.Payload, - }) - default: - files = append(files, piv1.File{ - Name: v.Name, - MIME: v.MIME, - Digest: v.Digest, - Payload: v.Payload, - }) - } - } - return files, metadata -} - -func statusChangesDecode(payload []byte) ([]pi.StatusChange, error) { - var statuses []pi.StatusChange - d := json.NewDecoder(strings.NewReader(string(payload))) - for { - var sc pi.StatusChange - err := d.Decode(&sc) - if errors.Is(err, io.EOF) { - break - } else if err != nil { - return nil, err - } - statuses = append(statuses, sc) - } - - return statuses, nil -} - -func convertProposalRecordFromPD(r pdv1.Record, state piv1.PropStateT) (*piv1.ProposalRecord, error) { - // Decode metadata streams - var ( - um *usermd.UserMetadata - sc = make([]pi.StatusChange, 0, 16) - err error - ) - for _, v := range r.Metadata { - switch v.ID { - case usermd.MDStreamIDUserMetadata: - var um usermd.UserMetadata - err = json.Unmarshal([]byte(v.Payload), &um) - if err != nil { - return nil, err - } - case pi.MDStreamIDStatusChanges: - sc, err = statusChangesDecode([]byte(v.Payload)) - if err != nil { - return nil, err - } - } - } - - // Convert to pi types - files, metadata := convertFilesFromPD(r.Files) - status := convertPropStatusFromPD(r.Status) - - statuses := make([]piv1.StatusChange, 0, len(sc)) - for _, v := range sc { - statuses = append(statuses, piv1.StatusChange{ - Token: v.Token, - Version: v.Version, - Status: piv1.PropStatusT(v.Status), - Reason: v.Reason, - PublicKey: v.PublicKey, - Signature: v.Signature, - Timestamp: v.Timestamp, - }) - } - - // Some fields are intentionally omitted because they are either - // user data that needs to be pulled from the user database or they - // are politeiad plugin data that needs to be retrieved using a - // plugin command. - return &piv1.ProposalRecord{ - Version: r.Version, - Timestamp: r.Timestamp, - State: state, - Status: status, - UserID: "", // Intentionally omitted - Username: "", // Intentionally omitted - PublicKey: um.PublicKey, - Signature: um.Signature, - Statuses: statuses, - Files: files, - Metadata: metadata, - CensorshipRecord: convertCensorshipRecordFromPD(r.CensorshipRecord), - }, nil -} - -// proposalRecords returns the ProposalRecord for each of the provided proposal -// requests. If a token does not correspond to an actual proposal then it will -// not be included in the returned map. -func (p *politeiawww) proposalRecords(ctx context.Context, state piv1.PropStateT, reqs []piv1.ProposalRequest, includeFiles bool) (map[string]piv1.ProposalRecord, error) { - // Get politeiad records - props := make([]piv1.ProposalRecord, 0, len(reqs)) - for _, v := range reqs { - var r *pdv1.Record - var err error - switch state { - case piv1.PropStateUnvetted: - // Unvetted politeiad record - r, err = p.politeiad.GetUnvetted(ctx, v.Token, v.Version) - if err != nil { - return nil, err - } - case piv1.PropStateVetted: - // Vetted politeiad record - r, err = p.politeiad.GetVetted(ctx, v.Token, v.Version) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("unknown state %v", state) - } - - if r.Status == pdv1.RecordStatusNotFound { - // Record wasn't found. Don't include token in the results. - continue - } - - pr, err := convertProposalRecordFromPD(*r, state) - if err != nil { - return nil, err - } - - // Remove files if specified. The Metadata objects will still be - // returned. - if !includeFiles { - pr.Files = []piv1.File{} - } - - props = append(props, *pr) - } - - // Verify we've got some results - if len(props) == 0 { - return map[string]piv1.ProposalRecord{}, nil - } - - // Get user data - pubkeys := make([]string, 0, len(props)) - for _, v := range props { - pubkeys = append(pubkeys, v.PublicKey) - } - ur, err := p.db.UsersGetByPubKey(pubkeys) - if err != nil { - return nil, err - } - for k, v := range props { - token := v.CensorshipRecord.Token - u, ok := ur[v.PublicKey] - if !ok { - return nil, fmt.Errorf("user not found for pubkey %v from proposal %v", - v.PublicKey, token) - } - props[k] = proposalRecordFillInUser(v, u) - } - - // Convert proposals to a map - proposals := make(map[string]piv1.ProposalRecord, len(props)) - for _, v := range props { - proposals[v.CensorshipRecord.Token] = v - } - - return proposals, nil -} - -// proposalRecord returns the proposal record for the provided token and -// version. A blank version will return the most recent version. A -// errProposalNotFound error will be returned if a proposal is not found for -// the provided token/version combination. -func (p *politeiawww) proposalRecord(ctx context.Context, state piv1.PropStateT, token, version string) (*piv1.ProposalRecord, error) { - prs, err := p.proposalRecords(ctx, state, []piv1.ProposalRequest{ - { - Token: token, - Version: version, - }, - }, true) - if err != nil { - return nil, err - } - pr, ok := prs[token] - if !ok { - return nil, errProposalNotFound - } - return &pr, nil -} - -// proposalRecordLatest returns the latest version of the proposal record for -// the provided token. A errProposalNotFound error will be returned if a -// proposal is not found for the provided token. -func (p *politeiawww) proposalRecordLatest(ctx context.Context, state piv1.PropStateT, token string) (*piv1.ProposalRecord, error) { - return p.proposalRecord(ctx, state, token, "") -} - -func (p *politeiawww) linkByPeriodMin() int64 { - return 0 -} - -func (p *politeiawww) linkByPeriodMax() int64 { - return 0 -} - -func (p *politeiawww) verifyProposalMetadata(pm piv1.ProposalMetadata) error { - // Verify name - if !proposalNameIsValid(pm.Name) { - return piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusPropNameInvalid, - ErrorContext: proposalNameRegex(), - } - } - return nil -} - -func (p *politeiawww) verifyProposal(files []piv1.File, metadata []piv1.Metadata, publicKey, signature string) (*piv1.ProposalMetadata, error) { - if len(files) == 0 { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusFileCountInvalid, - ErrorContext: "no files found", - } - } - - // Verify the files adhere to all policy requirements - var ( - countTextFiles int - countImageFiles int - foundIndexFile bool - ) - filenames := make(map[string]struct{}, len(files)) - for _, v := range files { - // Validate file name - _, ok := filenames[v.Name] - if ok { - e := fmt.Sprintf("duplicate name %v", v.Name) - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusFileNameInvalid, - ErrorContext: e, - } - } - filenames[v.Name] = struct{}{} - - // Validate file payload - if v.Payload == "" { - e := fmt.Sprintf("file %v empty payload", v.Name) - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusFilePayloadInvalid, - ErrorContext: e, - } - } - payloadb, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - e := fmt.Sprintf("file %v invalid base64", v.Name) - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusFilePayloadInvalid, - ErrorContext: e, - } - } - - // Verify computed file digest matches given file digest - digest := util.Digest(payloadb) - d, ok := util.ConvertDigest(v.Digest) - if !ok { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusFileDigestInvalid, - ErrorContext: v.Name, - } - } - if !bytes.Equal(digest, d[:]) { - e := fmt.Sprintf("file %v digest got %v, want %x", - v.Name, v.Digest, digest) - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusFileDigestInvalid, - ErrorContext: e, - } - } - - // Verify detected MIME type matches given mime type - ct := http.DetectContentType(payloadb) - mimePayload, _, err := mime.ParseMediaType(ct) - if err != nil { - return nil, err - } - mimeFile, _, err := mime.ParseMediaType(v.MIME) - if err != nil { - e := fmt.Sprintf("file %v mime '%v' not parsable", v.Name, v.MIME) - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusFileMIMEInvalid, - ErrorContext: e, - } - } - if mimeFile != mimePayload { - e := fmt.Sprintf("file %v mime got %v, want %v", - v.Name, mimeFile, mimePayload) - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusFileMIMEInvalid, - ErrorContext: e, - } - } - - // Run MIME type specific validation - switch mimeFile { - case mimeTypeText: - countTextFiles++ - - // Verify text file size - if len(payloadb) > www.PolicyMaxMDSize { - e := fmt.Sprintf("file %v size %v exceeds max size %v", - v.Name, len(payloadb), www.PolicyMaxMDSize) - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusIndexFileSizeInvalid, - ErrorContext: e, - } - } - - // The only text file that is allowed is the index markdown - // file. - if v.Name != www.PolicyIndexFilename { - e := fmt.Sprintf("want %v, got %v", www.PolicyIndexFilename, v.Name) - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusIndexFileNameInvalid, - ErrorContext: e, - } - } - if foundIndexFile { - e := fmt.Sprintf("more than one %v file found", - www.PolicyIndexFilename) - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusIndexFileCountInvalid, - ErrorContext: e, - } - } - - // Set index file as being found - foundIndexFile = true - - case mimeTypePNG: - countImageFiles++ - - // Verify image file size - if len(payloadb) > www.PolicyMaxImageSize { - e := fmt.Sprintf("image %v size %v exceeds max size %v", - v.Name, len(payloadb), www.PolicyMaxImageSize) - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusImageFileSizeInvalid, - ErrorContext: e, - } - } - - default: - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusFileMIMEInvalid, - ErrorContext: v.MIME, - } - } - } - - // Verify that an index file is present - if !foundIndexFile { - e := fmt.Sprintf("%v file not found", www.PolicyIndexFilename) - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusIndexFileCountInvalid, - ErrorContext: e, - } - } - - // Verify file counts are acceptable - if countTextFiles > www.PolicyMaxMDs { - e := fmt.Sprintf("got %v text files, max is %v", - countTextFiles, www.PolicyMaxMDs) - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusTextFileCountInvalid, - ErrorContext: e, - } - } - if countImageFiles > www.PolicyMaxImages { - e := fmt.Sprintf("got %v image files, max is %v", - countImageFiles, www.PolicyMaxImages) - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusImageFileCountInvalid, - ErrorContext: e, - } - } - - // Verify that the metadata contains a ProposalMetadata and only - // a ProposalMetadata. - switch { - case len(metadata) == 0: - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusPropMetadataNotFound, - } - case len(metadata) > 1: - e := fmt.Sprintf("metadata should only contain %v", - www.HintProposalMetadata) - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusMetadataCountInvalid, - ErrorContext: e, - } - } - md := metadata[0] - if md.Hint != www.HintProposalMetadata { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusPropMetadataNotFound, - } - } - - // Verify metadata fields - b, err := base64.StdEncoding.DecodeString(md.Payload) - if err != nil { - e := fmt.Sprintf("metadata with hint %v invalid base64 payload", md.Hint) - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusMetadataPayloadInvalid, - ErrorContext: e, - } - } - digest := util.Digest(b) - if md.Digest != hex.EncodeToString(digest) { - e := fmt.Sprintf("metadata with hint %v got digest %v, want %v", - md.Hint, md.Digest, hex.EncodeToString(digest)) - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusMetadataDigestInvalid, - ErrorContext: e, - } - } - - // Decode ProposalMetadata - d := json.NewDecoder(bytes.NewReader(b)) - d.DisallowUnknownFields() - var pm piv1.ProposalMetadata - err = d.Decode(&pm) - if err != nil { - e := fmt.Sprintf("unable to decode %v payload", md.Hint) - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusMetadataPayloadInvalid, - ErrorContext: e, - } - } - - // Verify ProposalMetadata - err = p.verifyProposalMetadata(pm) - if err != nil { - return nil, err - } - - // Verify signature - mr, err := wwwutil.MerkleRoot(files, metadata) - if err != nil { - return nil, err - } - err = util.VerifySignature(signature, publicKey, mr) - if err != nil { - return nil, convertUserErrorFromSignatureError(err) - } - - return &pm, nil -} - -func (p *politeiawww) processProposalNew(ctx context.Context, pn piv1.ProposalNew, usr user.User) (*piv1.ProposalNewReply, error) { - log.Tracef("processProposalNew: %v", usr.Username) - - // Verify user has paid registration paywall - if !p.userHasPaid(usr) { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusUserRegistrationNotPaid, - } - } - - // Verify user has a proposal credit - if !p.userHasProposalCredits(usr) { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusUserBalanceInsufficient, - } - } - - // Verify user signed using active identity - if usr.PublicKey() != pn.PublicKey { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusPublicKeyInvalid, - ErrorContext: "not active identity", - } - } - - // Verify proposal - pm, err := p.verifyProposal(pn.Files, pn.Metadata, - pn.PublicKey, pn.Signature) - if err != nil { - return nil, err - } - - // Setup politeiad files. The Metadata objects are converted to - // politeiad files since they contain user defined data that needs - // to be included in the merkle root that politeiad signs. - files := convertFilesFromPi(pn.Files) - for _, v := range pn.Metadata { - switch v.Hint { - case piv1.HintProposalMetadata: - files = append(files, convertFileFromMetadata(v)) - } - } - - // Setup metadata stream - um := usermd.UserMetadata{ - UserID: usr.ID.String(), - PublicKey: pn.PublicKey, - Signature: pn.Signature, - } - b, err := json.Marshal(um) - if err != nil { - return nil, err - } - metadata := []pdv1.MetadataStream{ - { - ID: usermd.MDStreamIDUserMetadata, - Payload: string(b), - }, - } - - // Send politeiad request - dcr, err := p.politeiad.NewRecord(ctx, metadata, files) - if err != nil { - return nil, err - } - cr := convertCensorshipRecordFromPD(*dcr) - - // Deduct proposal credit from author's account - err = p.spendProposalCredit(&usr, cr.Token) - if err != nil { - return nil, err - } - - // Emit a new proposal event - p.events.Emit(eventProposalSubmitted, - dataProposalSubmitted{ - token: cr.Token, - name: pm.Name, - username: usr.Username, - }) - - log.Infof("Proposal submitted: %v %v", cr.Token, pm.Name) - for k, f := range pn.Files { - log.Infof("%02v: %v %v", k, f.Name, f.Digest) - } - - // Get full proposal record - pr, err := p.proposalRecordLatest(ctx, piv1.PropStateUnvetted, cr.Token) - if err != nil { - return nil, err - } - - return &piv1.ProposalNewReply{ - Proposal: *pr, - }, nil -} - -// filenameIsMetadata returns whether the politeiad filename represents a pi -// Metadata object. These are provided as user defined metadata in proposal -// submissions but are stored as files in politeiad since they are part of the -// merkle root that the user signs and must also be part of the merkle root -// that politeiad signs. -func filenameIsMetadata(filename string) bool { - switch filename { - case pi.FileNameProposalMetadata: - return true - } - return false -} - -// filesToDel returns the names of the files that are included in current but -// are not included in updated. These are the files that need to be deleted -// from a proposal on update. -func filesToDel(current []piv1.File, updated []piv1.File) []string { - curr := make(map[string]struct{}, len(current)) // [name]struct - for _, v := range updated { - curr[v.Name] = struct{}{} - } - - del := make([]string, 0, len(current)) - for _, v := range current { - _, ok := curr[v.Name] - if !ok { - del = append(del, v.Name) - } - } - - return del -} - -func (p *politeiawww) processProposalEdit(ctx context.Context, pe piv1.ProposalEdit, usr user.User) (*piv1.ProposalEditReply, error) { - log.Tracef("processProposalEdit: %v", pe.Token) - - // Verify token - if !tokenIsFullLength(pe.Token) { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusPropTokenInvalid, - } - } - - // Verify state - switch pe.State { - case piv1.PropStateUnvetted, piv1.PropStateVetted: - // Allowed; continue - default: - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusPropStateInvalid, - } - } - - // Verify proposal - pm, err := p.verifyProposal(pe.Files, pe.Metadata, - pe.PublicKey, pe.Signature) - if err != nil { - return nil, err - } - - // Verify user signed using active identity - if usr.PublicKey() != pe.PublicKey { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusPublicKeyInvalid, - ErrorContext: "not active identity", - } - } - - // Get the current proposal - curr, err := p.proposalRecordLatest(ctx, pe.State, pe.Token) - if err != nil { - if errors.Is(err, errProposalNotFound) { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusPropNotFound, - } - } - return nil, err - } - - // Verify the user is the author. The public keys are not static - // values so the user IDs must be compared directly. - if curr.UserID != usr.ID.String() { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusUnauthorized, - ErrorContext: "user is not author", - } - } - - // Verify proposal status - switch curr.Status { - case piv1.PropStatusUnreviewed, piv1.PropStatusPublic: - // Allowed; continue - default: - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusPropStatusInvalid, - } - } - - // Verification that requires plugin data or querying additional - // proposal data is done in the politeiad pi plugin hook. This - // includes: - // -Verify linkto - // -Verify vote status - - // Setup politeiad files. The Metadata objects are converted to - // politeiad files since they contain user defined data that needs - // to be included in the merkle root that politeiad signs. - filesAdd := convertFilesFromPi(pe.Files) - for _, v := range pe.Metadata { - switch v.Hint { - case piv1.HintProposalMetadata: - filesAdd = append(filesAdd, convertFileFromMetadata(v)) - } - } - filesDel := filesToDel(curr.Files, pe.Files) - - // Setup politeiad metadata - um := usermd.UserMetadata{ - UserID: usr.ID.String(), - PublicKey: pe.PublicKey, - Signature: pe.Signature, - } - b, err := json.Marshal(um) - if err != nil { - return nil, err - } - mdOverwrite := []pdv1.MetadataStream{ - { - ID: usermd.MDStreamIDUserMetadata, - Payload: string(b), - }, - } - mdAppend := []pdv1.MetadataStream{} - - // Send politeiad request - // TODO verify that this will throw an error if no proposal files - // were changed. - var r *pdv1.Record - switch pe.State { - case piv1.PropStateUnvetted: - r, err = p.politeiad.UpdateUnvetted(ctx, pe.Token, - mdAppend, mdOverwrite, filesAdd, filesDel) - if err != nil { - return nil, err - } - case piv1.PropStateVetted: - r, err = p.politeiad.UpdateVetted(ctx, pe.Token, - mdAppend, mdOverwrite, filesAdd, filesDel) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("unknown state %v", pe.State) - } - - // Emit an edit proposal event - p.events.Emit(eventProposalEdited, dataProposalEdited{ - userID: usr.ID.String(), - username: usr.Username, - token: pe.Token, - name: pm.Name, - version: r.Version, - }) - - log.Infof("Proposal edited: %v %v", pe.Token, pm.Name) - for k, f := range pe.Files { - log.Infof("%02v: %v %v", k, f.Name, f.Digest) - } - - // Get updated proposal - pr, err := p.proposalRecordLatest(ctx, pe.State, pe.Token) - if err != nil { - return nil, err - } - - return &piv1.ProposalEditReply{ - Proposal: *pr, - }, nil -} - -func (p *politeiawww) processProposalSetStatus(ctx context.Context, pss piv1.ProposalSetStatus, usr user.User) (*piv1.ProposalSetStatusReply, error) { - log.Tracef("processProposalSetStatus: %v %v", pss.Token, pss.Status) - - // Sanity check - if !usr.Admin { - return nil, fmt.Errorf("not an admin") - } - - // Verify token - if !tokenIsFullLength(pss.Token) { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusPropTokenInvalid, - } - } - - // Verify state - switch pss.State { - case piv1.PropStateUnvetted, piv1.PropStateVetted: - // Allowed; continue - default: - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusPropStateInvalid, - } - } - - // Verify reason - _, required := statusReasonRequired[pss.Status] - if required && pss.Reason == "" { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusPropStatusChangeReasonInvalid, - ErrorContext: "reason not given", - } - } - - // Verify user is an admin - if !usr.Admin { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusUnauthorized, - ErrorContext: "user is not an admin", - } - } - - // Verify user signed with their active identity - if usr.PublicKey() != pss.PublicKey { - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusPublicKeyInvalid, - ErrorContext: "not active identity", - } - } - - // Verify signature - msg := pss.Token + pss.Version + strconv.Itoa(int(pss.Status)) + pss.Reason - err := util.VerifySignature(pss.Signature, pss.PublicKey, msg) - if err != nil { - return nil, convertUserErrorFromSignatureError(err) - } - - // TODO don't allow censoring a proposal once the vote has started - // Verification that requires retrieving the existing proposal is - // done in politeiad. This includes: - // -Verify proposal exists (politeiad) - // -Verify proposal state is correct (politeiad) - // -Verify version is the latest version (pi plugin) - // -Verify status change is allowed (pi plugin) - - // Setup metadata - timestamp := time.Now().Unix() - sc := pi.StatusChange{ - Token: pss.Token, - Version: pss.Version, - Status: pi.PropStatusT(pss.Status), - Reason: pss.Reason, - PublicKey: pss.PublicKey, - Signature: pss.Signature, - Timestamp: timestamp, - } - b, err := json.Marshal(sc) - if err != nil { - return nil, err - } - mdAppend := []pdv1.MetadataStream{ - { - ID: pi.MDStreamIDStatusChanges, - Payload: string(b), - }, - } - mdOverwrite := []pdv1.MetadataStream{} - - // Send politeiad request - var r *pdv1.Record - status := convertRecordStatusFromPropStatus(pss.Status) - switch pss.State { - case piv1.PropStateUnvetted: - r, err = p.politeiad.SetUnvettedStatus(ctx, pss.Token, - status, mdAppend, mdOverwrite) - if err != nil { - return nil, err - } - case piv1.PropStateVetted: - r, err = p.politeiad.SetVettedStatus(ctx, pss.Token, - status, mdAppend, mdOverwrite) - if err != nil { - return nil, err - } - } - - // The proposal state will have changed if the proposal was made - // public. - state := pss.State - if pss.Status == piv1.PropStatusPublic { - state = piv1.PropStateVetted - } - - // Emit status change event - p.events.Emit(eventProposalStatusChange, - dataProposalStatusChange{ - token: pss.Token, - state: state, - status: convertPropStatusFromPD(r.Status), - version: r.Version, - reason: pss.Reason, - adminID: usr.ID.String(), - }) - - // Get updated proposal - pr, err := p.proposalRecordLatest(ctx, state, pss.Token) - if err != nil { - return nil, err - } - - return &piv1.ProposalSetStatusReply{ - Proposal: *pr, - }, nil -} - -func (p *politeiawww) processProposals(ctx context.Context, ps piv1.Proposals, isAdmin bool) (*piv1.ProposalsReply, error) { - log.Tracef("processProposals: %v", ps.Requests) - - // Verify state - switch ps.State { - case piv1.PropStateUnvetted, piv1.PropStateVetted: - // Allowed; continue - default: - return nil, piv1.UserErrorReply{ - ErrorCode: piv1.ErrorStatusPropStateInvalid, - } - } - - // Get proposal records - props, err := p.proposalRecords(ctx, ps.State, ps.Requests, ps.IncludeFiles) - if err != nil { - return nil, err - } - - // Only admins are allowed to retrieve unvetted proposal files. - // Remove all unvetted proposal files and user defined metadata if - // the user is not an admin. - if !isAdmin { - for k, v := range props { - if v.State == piv1.PropStateVetted { - continue - } - v.Files = []piv1.File{} - v.Metadata = []piv1.Metadata{} - props[k] = v - } - } - - return &piv1.ProposalsReply{ - Proposals: props, - }, nil -} - -func (p *politeiawww) processProposalInventory(ctx context.Context, inv piv1.ProposalInventory, u *user.User) (*piv1.ProposalInventoryReply, error) { - log.Tracef("processProposalInventory: %v", inv.UserID) - - // Send plugin command - i := pi.ProposalInv{ - UserID: inv.UserID, - } - pir, err := p.proposalInv(ctx, i) - if err != nil { - return nil, err - } - - // Determine if unvetted tokens should be returned - switch { - case u == nil: - // No user session. Remove unvetted. - pir.Unvetted = nil - case u.Admin: - // User is an admin. Return unvetted. - case inv.UserID == u.ID.String(): - // User is requesting their own proposals. Return unvetted. - default: - // Remove unvetted for all other cases - pir.Unvetted = nil - } - - return &piv1.ProposalInventoryReply{ - Unvetted: pir.Unvetted, - Vetted: pir.Vetted, - }, nil -} - -func (p *politeiawww) processVoteInventory(ctx context.Context) (*piv1.VoteInventoryReply, error) { - log.Tracef("processVoteInventory") - - r, err := p.piVoteInventory(ctx) - if err != nil { - return nil, err - } - - return &piv1.VoteInventoryReply{ - Unauthorized: r.Unauthorized, - Authorized: r.Authorized, - Started: r.Started, - Approved: r.Approved, - Rejected: r.Rejected, - BestBlock: r.BestBlock, - }, nil -} - -// setPiRoutes sets the pi API routes. -func (p *politeiawww) setPiRoutes(c *comments.Comments, t *ticketvote.TicketVote) { +// setupPiRoutes sets up the API routes for piwww mode. +func (p *politeiawww) setupPiRoutes(c *comments.Comments, t *ticketvote.TicketVote) { // Return a 404 when a route is not found p.router.NotFoundHandler = http.HandlerFunc(p.handleNotFound) @@ -1257,7 +58,14 @@ func (p *politeiawww) setPiRoutes(c *comments.Comments, t *ticketvote.TicketVote permissionPublic) /* - // Pi routes - proposals + // Pi routes + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteProposals, p.handleProposals, + permissionPublic) + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteVoteInventory, p.handleVoteInventory, + permissionPublic) + p.addRoute(http.MethodPost, piv1.APIRoute, piv1.RouteProposalNew, p.handleProposalNew, permissionLogin) @@ -1267,12 +75,14 @@ func (p *politeiawww) setPiRoutes(c *comments.Comments, t *ticketvote.TicketVote p.addRoute(http.MethodPost, piv1.APIRoute, piv1.RouteProposalSetStatus, p.handleProposalSetStatus, permissionAdmin) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteProposals, p.handleProposals, - permissionPublic) p.addRoute(http.MethodPost, piv1.APIRoute, piv1.RouteProposalInventory, p.handleProposalInventory, permissionPublic) + + // Record routes + p.addRoute(http.MethodPost, rcv1.APIRoute, + rcv1.RouteTimestamps, p.handleTimestamps, + permissionPublic) */ // Comment routes @@ -1321,10 +131,6 @@ func (p *politeiawww) setPiRoutes(c *comments.Comments, t *ticketvote.TicketVote tkv1.RouteTimestamps, t.HandleTimestamps, permissionPublic) - // Record routes - p.addRoute(http.MethodPost, rcv1.APIRoute, - rcv1.RouteTimestamps, p.handleTimestamps, - permissionPublic) } func (p *politeiawww) setupPi() error { @@ -1334,7 +140,7 @@ func (p *politeiawww) setupPi() error { // Setup routes p.setUserWWWRoutes() - p.setPiRoutes(c, tv) + p.setupPiRoutes(c, tv) // Verify paywall settings switch { diff --git a/politeiawww/politeiawww.go b/politeiawww/politeiawww.go index 2ac3849c4..b4905f67e 100644 --- a/politeiawww/politeiawww.go +++ b/politeiawww/politeiawww.go @@ -183,10 +183,10 @@ func (p *politeiawww) handlePolicy(w http.ResponseWriter, r *http.Request) { TokenPrefixLength: www.TokenPrefixLength, BuildInformation: version.BuildInformation(), IndexFilename: www.PolicyIndexFilename, - MinLinkByPeriod: p.linkByPeriodMin(), - MaxLinkByPeriod: p.linkByPeriodMax(), - MinVoteDuration: p.cfg.VoteDurationMin, - MaxVoteDuration: p.cfg.VoteDurationMax, + MinLinkByPeriod: 0, + MaxLinkByPeriod: 0, + MinVoteDuration: 0, + MaxVoteDuration: 0, } util.RespondWithJSON(w, http.StatusOK, reply) diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 1f33d25c2..7751b71cb 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -6,7 +6,6 @@ package main import ( "context" - "encoding/base64" "encoding/json" "errors" "net/http" @@ -14,7 +13,6 @@ import ( "github.com/decred/politeia/decredplugin" pd "github.com/decred/politeia/politeiad/api/v1" - piplugin "github.com/decred/politeia/politeiad/plugins/pi" ticketvote "github.com/decred/politeia/politeiad/plugins/ticketvote" pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" @@ -24,6 +22,7 @@ import ( "github.com/gorilla/mux" ) +/* func convertStateToWWW(state pi.PropStateT) www.PropStateT { switch state { case pi.PropStateInvalid: @@ -36,6 +35,7 @@ func convertStateToWWW(state pi.PropStateT) www.PropStateT { return www.PropStateInvalid } } +*/ func convertStatusToWWW(status pi.PropStatusT) www.PropStatusT { switch status { @@ -52,6 +52,7 @@ func convertStatusToWWW(status pi.PropStatusT) www.PropStatusT { } } +/* func convertProposalToWWW(pr *pi.ProposalRecord) (*www.ProposalRecord, error) { // Decode metadata var pm *piplugin.ProposalMetadata @@ -133,6 +134,7 @@ func convertProposalToWWW(pr *pi.ProposalRecord) (*www.ProposalRecord, error) { }, }, nil } +*/ func convertVoteStatusToWWW(status ticketvote.VoteStatusT) www.PropVoteStatusT { switch status { @@ -188,55 +190,18 @@ func convertVoteErrorCodeToWWW(errcode ticketvote.VoteErrorT) decredplugin.Error func (p *politeiawww) processProposalDetails(ctx context.Context, pd www.ProposalsDetails, u *user.User) (*www.ProposalDetailsReply, error) { log.Tracef("processProposalDetails: %v", pd.Token) - pr, err := p.proposalRecord(ctx, pi.PropStateVetted, pd.Token, pd.Version) - if err != nil { - return nil, err - } - pw, err := convertProposalToWWW(pr) - if err != nil { - return nil, err - } - - return &www.ProposalDetailsReply{ - Proposal: *pw, - }, nil + return nil, nil } func (p *politeiawww) processAllVetted(ctx context.Context, gav www.GetAllVetted) (*www.GetAllVettedReply, error) { - // TODO + return nil, nil } func (p *politeiawww) processBatchProposals(ctx context.Context, bp www.BatchProposals, u *user.User) (*www.BatchProposalsReply, error) { log.Tracef("processBatchProposals: %v", bp.Tokens) - // Setup requests - prs := make([]pi.ProposalRequest, 0, len(bp.Tokens)) - for _, t := range bp.Tokens { - prs = append(prs, pi.ProposalRequest{ - Token: t, - }) - } - - // Get proposals - props, err := p.proposalRecords(ctx, pi.PropStateVetted, prs, false) - if err != nil { - return nil, err - } - - // Prepare reply - propsw := make([]www.ProposalRecord, 0, len(bp.Tokens)) - for _, pr := range props { - propw, err := convertProposalToWWW(&pr) - if err != nil { - return nil, err - } - propsw = append(propsw, *propw) - } - - return &www.BatchProposalsReply{ - Proposals: propsw, - }, nil + return nil, nil } func (p *politeiawww) processVoteResults(ctx context.Context, token string) (*www.VoteResultsReply, error) { @@ -386,11 +351,8 @@ func (p *politeiawww) processTokenInventory(ctx context.Context, isAdmin bool) ( return nil, err } - // Get vote inventory - vir, err := p.piVoteInventory(ctx) - if err != nil { - return nil, err - } + // TODO Get vote inventory + var vir pi.VoteInventoryReply // Unpack record inventory var ( diff --git a/politeiawww/records/error.go b/politeiawww/records/error.go index 8da9dd1ed..00812b944 100644 --- a/politeiawww/records/error.go +++ b/politeiawww/records/error.go @@ -31,7 +31,7 @@ func convertRecordsErrorCode(errCode int) v1.ErrorCodeT { return v1.ErrorCodeInvalid } -func respondWithRecordError(w http.ResponseWriter, r *http.Request, format string, err error) { +func respondWithError(w http.ResponseWriter, r *http.Request, format string, err error) { var ( ue v1.UserErrorReply pe pdclient.Error diff --git a/politeiawww/records/events.go b/politeiawww/records/events.go index b5ae42240..f9c2a454b 100644 --- a/politeiawww/records/events.go +++ b/politeiawww/records/events.go @@ -3,3 +3,29 @@ // license that can be found in the LICENSE file. package records + +import ( + v1 "github.com/decred/politeia/politeiawww/api/records/v1" + "github.com/decred/politeia/politeiawww/user" +) + +const ( + // EventTypeNew is emitted when a new record is submitted. + EventTypeNew = "records-new" + + // EventTypeEdit is emitted when a a record is edited. + EventTypeEdit = "records-edit" +) + +// EventNew is the event data for the EventTypeNew. +type EventNew struct { + User user.User + Record v1.Record +} + +// EventEdit is the event data for the EventTypeEdit. +type EventEdit struct { + User user.User + State string + Record v1.Record +} diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index 881363cb6..ebb443b55 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -6,11 +6,393 @@ package records import ( "context" + "encoding/json" + "fmt" pdv1 "github.com/decred/politeia/politeiad/api/v1" + pduser "github.com/decred/politeia/politeiad/plugins/user" v1 "github.com/decred/politeia/politeiawww/api/records/v1" + "github.com/decred/politeia/politeiawww/config" + "github.com/decred/politeia/politeiawww/user" + "github.com/google/uuid" ) +func (r *Records) processNew(ctx context.Context, n v1.New, u user.User) (*v1.NewReply, error) { + log.Tracef("processNew: %v", u.Username) + + // Verify user signed using active identity + if u.PublicKey() != n.PublicKey { + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodePublicKeyInvalid, + ErrorContext: "not active identity", + } + } + + // Execute pre plugin hooks. Checking the mode is a temporary + // measure until user plugins have been properly implemented. + switch r.cfg.Mode { + case config.PoliteiaWWWMode: + err := r.piHookNewRecordPre(u) + if err != nil { + return nil, err + } + } + + // Setup metadata stream + um := pduser.UserMetadata{ + UserID: u.ID.String(), + PublicKey: n.PublicKey, + Signature: n.Signature, + } + b, err := json.Marshal(um) + if err != nil { + return nil, err + } + metadata := []pdv1.MetadataStream{ + { + ID: pduser.MDStreamIDUserMetadata, + Payload: string(b), + }, + } + + // Save record to politeiad + f := convertFilesToPD(n.Files) + cr, err := r.politeiad.NewRecord(ctx, metadata, f) + if err != nil { + return nil, err + } + + // Get full record + pdr, err := r.politeiad.GetUnvetted(ctx, cr.Token, "") + if err != nil { + return nil, err + } + rc := convertRecordToV1(*pdr, v1.RecordStateUnvetted) + + // Execute post plugin hooks. Checking the mode is a temporary + // measure until user plugins have been properly implemented. + switch r.cfg.Mode { + case config.PoliteiaWWWMode: + err := r.piHookNewRecordPost(u, rc.CensorshipRecord.Token) + if err != nil { + return nil, err + } + } + + // Emit event + r.events.Emit(EventTypeNew, + EventNew{ + User: u, + Record: rc, + }) + + log.Infof("Record submitted: %v", rc.CensorshipRecord.Token) + for k, f := range rc.Files { + log.Infof("%02v: %v %v", k, f.Name, f.Digest) + } + + return &v1.NewReply{ + Record: rc, + }, nil +} + +// filesToDel returns the names of the files that are included in the current +// files but are not included in updated files. These are the files that need +// to be deleted from a record on update. +func filesToDel(current []pdv1.File, updated []pdv1.File) []string { + curr := make(map[string]struct{}, len(current)) // [name]struct + for _, v := range updated { + curr[v.Name] = struct{}{} + } + + del := make([]string, 0, len(current)) + for _, v := range current { + _, ok := curr[v.Name] + if !ok { + del = append(del, v.Name) + } + } + + return del +} + +func (r *Records) processEdit(ctx context.Context, e v1.Edit, u user.User) (*v1.EditReply, error) { + log.Tracef("processEdit: %v %v", e.Token, u.Username) + + // Verify user signed using active identity + if u.PublicKey() != e.PublicKey { + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodePublicKeyInvalid, + ErrorContext: "not active identity", + } + } + + // Get current record + var ( + curr *pdv1.Record + err error + ) + switch e.State { + case v1.RecordStateUnvetted: + curr, err = r.politeiad.GetUnvetted(ctx, e.Token, "") + if err != nil { + return nil, err + } + case v1.RecordStateVetted: + curr, err = r.politeiad.GetVetted(ctx, e.Token, "") + if err != nil { + return nil, err + } + default: + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeRecordStateInvalid, + } + } + + // Setup files + filesAdd := convertFilesToPD(e.Files) + filesDel := filesToDel(curr.Files, filesAdd) + + // Setup metadata + um := pduser.UserMetadata{ + UserID: u.ID.String(), + PublicKey: e.PublicKey, + Signature: e.Signature, + } + b, err := json.Marshal(um) + if err != nil { + return nil, err + } + mdOverwrite := []pdv1.MetadataStream{ + { + ID: pduser.MDStreamIDUserMetadata, + Payload: string(b), + }, + } + mdAppend := []pdv1.MetadataStream{} + + // Save update to politeiad + var pdr *pdv1.Record + switch e.State { + case v1.RecordStateUnvetted: + pdr, err = r.politeiad.UpdateUnvetted(ctx, e.Token, mdAppend, + mdOverwrite, filesAdd, filesDel) + if err != nil { + return nil, err + } + case v1.RecordStateVetted: + pdr, err = r.politeiad.UpdateUnvetted(ctx, e.Token, mdAppend, + mdOverwrite, filesAdd, filesDel) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("invalid state %v", e.State) + } + + rc := convertRecordToV1(*pdr, e.State) + + // Emit event + r.events.Emit(EventTypeEdit, + EventEdit{ + User: u, + State: e.State, + Record: rc, + }) + + log.Infof("Record edited: %v %v", e.State, rc.CensorshipRecord.Token) + for k, f := range rc.Files { + log.Infof("%02v: %v %v", k, f.Name, f.Digest) + } + + return &v1.EditReply{ + Record: rc, + }, nil +} + +func (r *Records) processSetStatus(ctx context.Context, ss v1.SetStatus, u user.User) (*v1.SetStatusReply, error) { + + /* + // Verify user signed with their active identity + if u.PublicKey() != pss.PublicKey { + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodePublicKeyInvalid, + ErrorContext: "not active identity", + } + } + */ + + // Status change user metadata + /* + timestamp := time.Now().Unix() + sc := pi.StatusChange{ + Token: pss.Token, + Version: pss.Version, + Status: pi.PropStatusT(pss.Status), + Reason: pss.Reason, + PublicKey: pss.PublicKey, + Signature: pss.Signature, + Timestamp: timestamp, + } + b, err := json.Marshal(sc) + if err != nil { + return nil, err + } + mdAppend := []pdv1.MetadataStream{ + { + ID: pi.MDStreamIDStatusChanges, + Payload: string(b), + }, + } + mdOverwrite := []pdv1.MetadataStream{} + */ + + /* + // This goes in the user plugin + // Verify reason + _, required := statusReasonRequired[pss.Status] + if required && pss.Reason == "" { + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodePropStatusChangeReasonInvalid, + ErrorContext: "reason not given", + } + } + + msg := pss.Token + pss.Version + strconv.Itoa(int(pss.Status)) + pss.Reason + err := util.VerifySignature(pss.Signature, pss.PublicKey, msg) + if err != nil { + return nil, convertUserErrorFromSignatureError(err) + } + */ + + // Emit event + + return nil, nil +} + +// record returns a version of a record from politeiad. If version is an empty +// string then the most recent version will be returned. +func (r *Records) record(ctx context.Context, state, token, version string) (*v1.Record, error) { + var ( + pdr *pdv1.Record + err error + ) + switch state { + case v1.RecordStateUnvetted: + pdr, err = r.politeiad.GetUnvetted(ctx, token, version) + if err != nil { + return nil, err + } + case v1.RecordStateVetted: + pdr, err = r.politeiad.GetVetted(ctx, token, version) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("invalid state %v", state) + } + + rc := convertRecordToV1(*pdr, state) + + // Fill in user data + userID := userIDFromMetadataStreams(rc.Metadata) + uid, err := uuid.Parse(userID) + u, err := r.userdb.UserGetById(uid) + if err != nil { + return nil, err + } + rc = recordPopulateUserData(rc, *u) + + return &rc, nil +} + +func (r *Records) processDetails(ctx context.Context, d v1.Details, u *user.User) (*v1.DetailsReply, error) { + log.Tracef("processDetails: %v %v %v", d.State, d.Token, d.Version) + + // Verify state + switch d.State { + case v1.RecordStateUnvetted, v1.RecordStateVetted: + // Allowed; continue + default: + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeRecordStateInvalid, + } + } + + // Get record + rc, err := r.record(ctx, d.State, d.Token, d.Version) + if err != nil { + return nil, err + } + + // Only admins and the record author are allowed to retrieve + // unvetted record files. Remove files if the user is not an admin + // or the author. This is a public route so a user may not be + // present. + var ( + authorID = userIDFromMetadataStreams(rc.Metadata) + isAuthor = u != nil && u.ID.String() == authorID + isAdmin = u != nil && u.Admin + ) + if !isAuthor && !isAdmin { + rc.Files = []v1.File{} + } + + return &v1.DetailsReply{ + Record: *rc, + }, nil +} + +func (r *Records) processRecords(ctx context.Context, rs v1.Records, u *user.User) (*v1.RecordsReply, error) { + log.Tracef("processRecords: %v %v", rs.State, len(rs.Requests)) + + // Verify state + switch rs.State { + case v1.RecordStateUnvetted, v1.RecordStateVetted: + // Allowed; continue + default: + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeRecordStateInvalid, + } + } + + // TODO Verify page size + + // Get all records in the batch + records := make(map[string]v1.Record, len(rs.Requests)) + for _, v := range rs.Requests { + rc, err := r.record(ctx, rs.State, v.Token, v.Version) + if err != nil { + // If any error occured simply skip this record. It will not + // be included in the reply. + continue + } + + // Only admins and the record author are allowed to retrieve + // unvetted record files. Remove files if the user is not an admin + // or the author. This is a public route so a user may not be + // present. + var ( + authorID = userIDFromMetadataStreams(rc.Metadata) + isAuthor = u != nil && u.ID.String() == authorID + isAdmin = u != nil && u.Admin + ) + if !isAuthor && !isAdmin { + rc.Files = []v1.File{} + } + + records[rc.CensorshipRecord.Token] = *rc + } + + return &v1.RecordsReply{ + Records: records, + }, nil +} + +func (r *Records) processInventory(ctx context.Context, u *user.User) (*v1.InventoryReply, error) { + return nil, nil +} + func (r *Records) processTimestamps(ctx context.Context, t v1.Timestamps, isAdmin bool) (*v1.TimestampsReply, error) { log.Tracef("processTimestamps: %v %v %v", t.State, t.Token, t.Version) @@ -69,6 +451,223 @@ func (r *Records) processTimestamps(ctx context.Context, t v1.Timestamps, isAdmi }, nil } +func (r *Records) processUserRecords(ctx context.Context, ur v1.UserRecords, u *user.User) (*v1.UserRecordsReply, error) { + /* + // Determine if unvetted tokens should be returned + switch { + case u == nil: + // No user session. Remove unvetted. + pir.Unvetted = nil + case u.Admin: + // User is an admin. Return unvetted. + case inv.UserID == u.ID.String(): + // User is requesting their own proposals. Return unvetted. + default: + // Remove unvetted for all other cases + pir.Unvetted = nil + } + */ + + return nil, nil +} + +// paywallIsEnabled returns whether the user paywall is enabled. +// +// This function is a temporary function that will be removed once user plugins +// have been implemented. +func (r *Records) paywallIsEnabled() bool { + return r.cfg.PaywallAmount != 0 && r.cfg.PaywallXpub != "" +} + +// userHasPaid returns whether the user has paid their user registration fee. +// +// This function is a temporary function that will be removed once user plugins +// have been implemented. +func (r *Records) userHasPaid(u user.User) bool { + if !r.paywallIsEnabled() { + return true + } + return u.NewUserPaywallTx != "" +} + +// userHashProposalCredits returns whether the user has any unspent proposal +// credits. +// +// This function is a temporary function that will be removed once user plugins +// have been implemented. +func userHasProposalCredits(u user.User) bool { + return len(u.UnspentProposalCredits) > 0 +} + +// spendProposalCredit moves a unspent credit to the spent credit list and +// updates the user in the database. +// +// This function is a temporary function that will be removed once user plugins +// have been implemented. +func (r *Records) spendProposalCredit(u user.User, token string) error { + // Skip if the paywall is enabled + if !r.paywallIsEnabled() { + return nil + } + + // Verify there are credits to be spent + if !userHasProposalCredits(u) { + return fmt.Errorf("no proposal credits found") + } + + // Credits are spent FIFO + c := u.UnspentProposalCredits[0] + c.CensorshipToken = token + u.SpentProposalCredits = append(u.SpentProposalCredits, c) + u.UnspentProposalCredits = u.UnspentProposalCredits[1:] + + return r.userdb.UserUpdate(u) +} + +// piHookNewRecordpre executes the new record pre hook for pi. +// +// This function is a temporary function that will be removed once user plugins +// have been implemented. +func (r *Records) piHookNewRecordPre(u user.User) error { + // Verify user has paid registration paywall + if !r.userHasPaid(u) { + return v1.UserErrorReply{ + // TODO + // ErrorCode: v1.ErrorCodeUserRegistrationNotPaid, + } + } + + // Verify user has a proposal credit + if !userHasProposalCredits(u) { + return v1.UserErrorReply{ + // TODO + // ErrorCode: v1.ErrorCodeUserBalanceInsufficient, + } + } + return nil +} + +// piHoonNewRecordPost executes the new record post hook for pi. +// +// This function is a temporary function that will be removed once user plugins +// have been implemented. +func (r *Records) piHookNewRecordPost(u user.User, token string) error { + return r.spendProposalCredit(u, token) +} + +// recordPopulateUserData populates the record with user data that is not +// stored in politeiad. +func recordPopulateUserData(r v1.Record, u user.User) v1.Record { + r.Username = u.Username + return r +} + +// userMetadataDecode decodes and returns the UserMetadata from the provided +// metadata streams. If a UserMetadata is not found, nil is returned. +func userMetadataDecode(ms []v1.MetadataStream) (*pduser.UserMetadata, error) { + var userMD *pduser.UserMetadata + for _, v := range ms { + if v.ID == pduser.MDStreamIDUserMetadata { + var um pduser.UserMetadata + err := json.Unmarshal([]byte(v.Payload), &um) + if err != nil { + return nil, err + } + userMD = &um + break + } + } + return userMD, nil +} + +// userIDFromMetadataStreams searches for a UserMetadata and parses the user ID +// from it if found. An empty string is returned if no UserMetadata is found. +func userIDFromMetadataStreams(ms []v1.MetadataStream) string { + um, err := userMetadataDecode(ms) + if err != nil { + return "" + } + if um == nil { + return "" + } + return um.UserID +} + +func convertFilesToPD(f []v1.File) []pdv1.File { + files := make([]pdv1.File, 0, len(f)) + for _, v := range f { + files = append(files, pdv1.File{ + Name: v.Name, + MIME: v.MIME, + Digest: v.Digest, + Payload: v.Payload, + }) + } + return files +} + +func convertStatusToV1(s pdv1.RecordStatusT) v1.RecordStatusT { + switch s { + case pdv1.RecordStatusNotReviewed: + return v1.RecordStatusUnreviewed + case pdv1.RecordStatusPublic: + return v1.RecordStatusPublic + case pdv1.RecordStatusCensored: + return v1.RecordStatusCensored + case pdv1.RecordStatusArchived: + return v1.RecordStatusArchived + } + return v1.RecordStatusInvalid +} + +func convertFilesToV1(f []pdv1.File) []v1.File { + files := make([]v1.File, 0, len(f)) + for _, v := range f { + files = append(files, v1.File{ + Name: v.Name, + MIME: v.MIME, + Digest: v.Digest, + Payload: v.Payload, + }) + } + return files +} + +func convertMetadataStreamsToV1(ms []pdv1.MetadataStream) []v1.MetadataStream { + metadata := make([]v1.MetadataStream, 0, len(ms)) + for _, v := range ms { + metadata = append(metadata, v1.MetadataStream{ + ID: v.ID, + Payload: v.Payload, + }) + } + return metadata +} + +func convertCensorshipRecordToV1(cr pdv1.CensorshipRecord) v1.CensorshipRecord { + return v1.CensorshipRecord{ + Token: cr.Token, + Merkle: cr.Merkle, + Signature: cr.Signature, + } +} + +func convertRecordToV1(r pdv1.Record, state string) v1.Record { + // User fields that are not part of the politeiad record have + // been intentionally left blank. These fields must be pulled + // from the user database. + return v1.Record{ + State: state, + Status: convertStatusToV1(r.Status), + Version: r.Version, + Timestamp: r.Timestamp, + Username: "", // Intentionally left blank + Metadata: convertMetadataStreamsToV1(r.Metadata), + Files: convertFilesToV1(r.Files), + CensorshipRecord: convertCensorshipRecordToV1(r.CensorshipRecord), + } +} + func convertProofToV1(p pdv1.Proof) v1.Proof { return v1.Proof{ Type: p.Type, diff --git a/politeiawww/records/records.go b/politeiawww/records/records.go index 08101d936..77168d248 100644 --- a/politeiawww/records/records.go +++ b/politeiawww/records/records.go @@ -13,6 +13,7 @@ import ( "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/events" "github.com/decred/politeia/politeiawww/sessions" + "github.com/decred/politeia/politeiawww/user" "github.com/decred/politeia/util" ) @@ -20,6 +21,7 @@ import ( type Records struct { cfg *config.Config politeiad *pdclient.Client + userdb user.Database sessions *sessions.Sessions events *events.Manager } @@ -32,7 +34,7 @@ func (c *Records) HandleNew(w http.ResponseWriter, r *http.Request) { if err := decoder.Decode(&n); err != nil { respondWithError(w, r, "HandleNew: unmarshal", v1.UserErrorReply{ - ErrorCode: v1.ErrorStatusInputInvalid, + ErrorCode: v1.ErrorCodeInputInvalid, }) return } @@ -62,7 +64,7 @@ func (c *Records) HandleEdit(w http.ResponseWriter, r *http.Request) { if err := decoder.Decode(&e); err != nil { respondWithError(w, r, "HandleEdit: unmarshal", v1.UserErrorReply{ - ErrorCode: v1.ErrorStatusInputInvalid, + ErrorCode: v1.ErrorCodeInputInvalid, }) return } @@ -92,7 +94,7 @@ func (c *Records) HandleSetStatus(w http.ResponseWriter, r *http.Request) { if err := decoder.Decode(&ss); err != nil { respondWithError(w, r, "HandleSetStatus: unmarshal", v1.UserErrorReply{ - ErrorCode: v1.ErrorStatusInputInvalid, + ErrorCode: v1.ErrorCodeInputInvalid, }) return } @@ -122,19 +124,21 @@ func (c *Records) HandleDetails(w http.ResponseWriter, r *http.Request) { if err := decoder.Decode(&d); err != nil { respondWithError(w, r, "HandleDetails: unmarshal", v1.UserErrorReply{ - ErrorCode: v1.ErrorStatusInputInvalid, + ErrorCode: v1.ErrorCodeInputInvalid, }) return } + // Lookup session user. This is a public route so a session may not + // exist. Ignore any session not found errors. u, err := c.sessions.GetSessionUser(w, r) - if err != nil { + if err != nil && err != sessions.ErrSessionNotFound { respondWithError(w, r, "HandleDetails: GetSessionUser: %v", err) return } - dr, err := c.processDetails(r.Context(), d, *u) + dr, err := c.processDetails(r.Context(), d, u) if err != nil { respondWithError(w, r, "HandleDetails: processDetails: %v", err) @@ -152,7 +156,7 @@ func (c *Records) HandleRecords(w http.ResponseWriter, r *http.Request) { if err := decoder.Decode(&rs); err != nil { respondWithError(w, r, "HandleRecords: unmarshal", v1.UserErrorReply{ - ErrorCode: v1.ErrorStatusInputInvalid, + ErrorCode: v1.ErrorCodeInputInvalid, }) return } @@ -166,11 +170,10 @@ func (c *Records) HandleRecords(w http.ResponseWriter, r *http.Request) { return } - isAdmin := u != nil && u.Admin - rsr, err := c.processs(r.Context(), rs, isAdmin) + rsr, err := c.processRecords(r.Context(), rs, u) if err != nil { respondWithError(w, r, - "HandleRecords: processs: %v", err) + "HandleRecords: processRecords: %v", err) return } @@ -185,7 +188,7 @@ func (c *Records) HandleInventory(w http.ResponseWriter, r *http.Request) { if err := decoder.Decode(&i); err != nil { respondWithError(w, r, "HandleInventory: unmarshal", v1.UserErrorReply{ - ErrorCode: v1.ErrorStatusInputInvalid, + ErrorCode: v1.ErrorCodeInputInvalid, }) return } @@ -199,7 +202,7 @@ func (c *Records) HandleInventory(w http.ResponseWriter, r *http.Request) { return } - ir, err := c.processInventory(r.Context(), i, u) + ir, err := c.processInventory(r.Context(), u) if err != nil { respondWithError(w, r, "HandleInventory: processInventory: %v", err) @@ -209,15 +212,15 @@ func (c *Records) HandleInventory(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, ir) } -func (p *politeiawww) HandleTimestamps(w http.ResponseWriter, r *http.Request) { +func (c *Records) HandleTimestamps(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleTimestamps") - var t rcv1.Timestamps + var t v1.Timestamps decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&t); err != nil { respondWithError(w, r, "HandleTimestamps: unmarshal", - rcv1.UserErrorReply{ - ErrorCode: rcv1.ErrorCodeInputInvalid, + v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeInputInvalid, }) return } @@ -242,9 +245,41 @@ func (p *politeiawww) HandleTimestamps(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, tr) } +func (c *Records) HandleUserRecords(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleUserRecords") + + var ur v1.UserRecords + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&ur); err != nil { + respondWithError(w, r, "HandleUserRecords: unmaurhal", + v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeInputInvalid, + }) + return + } + + // Lookup session user. This is a public route so a session may not + // exist. Ignore any session not found errour. + u, err := c.sessions.GetSessionUser(w, r) + if err != nil && err != sessions.ErrSessionNotFound { + respondWithError(w, r, + "HandleUserRecords: GetSessionUser: %v", err) + return + } + + urr, err := c.processUserRecords(r.Context(), ur, u) + if err != nil { + respondWithError(w, r, + "HandleUserRecords: processsUserRecords: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, urr) +} + // New returns a new Records context. -func New(cfg *config.Config, pdc *pdclient.Client, s *sessions.Sessions, e *events.Manager) *Comments { - return &Comments{ +func New(cfg *config.Config, pdc *pdclient.Client, s *sessions.Sessions, e *events.Manager) *Records { + return &Records{ cfg: cfg, politeiad: pdc, sessions: s, diff --git a/politeiawww/testing.go b/politeiawww/testing.go index afe91e474..1ec8978c9 100644 --- a/politeiawww/testing.go +++ b/politeiawww/testing.go @@ -367,7 +367,7 @@ func newTestPoliteiawww(t *testing.T) (*politeiawww, func()) { // Setup routes p.setUserWWWRoutes() - p.setPiRoutes(c, tv) + p.setupPiRoutes(c, tv) // The cleanup is handled using a closure so that the temp dir // can be deleted using the local variable and not cfg.DataDir. diff --git a/politeiawww/user.go b/politeiawww/user.go index 29b2b8215..9136b2ab0 100644 --- a/politeiawww/user.go +++ b/politeiawww/user.go @@ -116,7 +116,7 @@ func convertWWWUserFromDatabaseUser(user *user.User) www.User { Deactivated: user.Deactivated, Locked: userIsLocked(user.FailedLoginAttempts), Identities: convertWWWIdentitiesFromDatabaseIdentities(user.Identities), - ProposalCredits: proposalCreditBalance(*user), + ProposalCredits: uint64(len(user.UnspentProposalCredits)), EmailNotifications: user.EmailNotifications, } } @@ -635,6 +635,14 @@ func (p *politeiawww) processEditUser(eu *www.EditUser, user *user.User) (*www.E return &www.EditUserReply{}, nil } +// userHasPaid returns whether the user has paid the user registration paywall. +func (p *politeiawww) userHasPaid(u user.User) bool { + if !p.paywallIsEnabled() { + return true + } + return u.NewUserPaywallTx != "" +} + // createLoginReply creates a login reply. func (p *politeiawww) createLoginReply(u *user.User, lastLoginTime int64) (*www.LoginReply, error) { reply := www.LoginReply{ @@ -644,7 +652,7 @@ func (p *politeiawww) createLoginReply(u *user.User, lastLoginTime int64) (*www. Username: u.Username, PublicKey: u.PublicKey(), PaywallTxID: u.NewUserPaywallTx, - ProposalCredits: proposalCreditBalance(*u), + ProposalCredits: uint64(len(u.UnspentProposalCredits)), LastLoginTime: lastLoginTime, } diff --git a/politeiawww/www.go b/politeiawww/www.go index 373bb65e4..f3894bd29 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -27,10 +27,8 @@ import ( "github.com/decred/politeia/mdstream" pd "github.com/decred/politeia/politeiad/api/v1" pdclient "github.com/decred/politeia/politeiad/client" - piplugin "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/ticketvote" cms "github.com/decred/politeia/politeiawww/api/cms/v1" - pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" database "github.com/decred/politeia/politeiawww/cmsdatabase" cmsdb "github.com/decred/politeia/politeiawww/cmsdatabase/cockroachdb" @@ -122,64 +120,6 @@ func convertWWWErrorStatus(pluginID string, errCode int) www.ErrorStatusT { return www.ErrorStatusInvalid } -func convertPiErrorStatusFromPD(e pd.ErrorStatusT) pi.ErrorStatusT { - switch e { - case pd.ErrorStatusInvalidRequestPayload: - // Intentionally omitted because this indicates a politeiawww - // server error so a ErrorStatusInvalid should be returned. - case pd.ErrorStatusInvalidChallenge: - // Intentionally omitted because this indicates a politeiawww - // server error so a ErrorStatusInvalid should be returned. - case pd.ErrorStatusInvalidFilename: - return pi.ErrorStatusFileNameInvalid - case pd.ErrorStatusInvalidFileDigest: - return pi.ErrorStatusFileDigestInvalid - case pd.ErrorStatusInvalidBase64: - return pi.ErrorStatusFilePayloadInvalid - case pd.ErrorStatusInvalidMIMEType: - return pi.ErrorStatusFileMIMEInvalid - case pd.ErrorStatusUnsupportedMIMEType: - return pi.ErrorStatusFileMIMEInvalid - case pd.ErrorStatusInvalidRecordStatusTransition: - return pi.ErrorStatusPropStatusChangeInvalid - case pd.ErrorStatusNoChanges: - return pi.ErrorStatusNoPropChanges - case pd.ErrorStatusRecordNotFound: - return pi.ErrorStatusPropNotFound - case pd.ErrorStatusInvalidToken: - return pi.ErrorStatusPropTokenInvalid - } - return pi.ErrorStatusInvalid -} - -func convertPiErrorStatusFromPiPlugin(e piplugin.ErrorCodeT) pi.ErrorStatusT { - switch e { - // TODO - } - return pi.ErrorStatusInvalid -} - -// convertPiErrorStatus attempts to convert the provided politeiad error code -// into a pi ErrorStatusT. If a plugin ID is provided the error code is assumed -// to be a user error code from the specified plugin API. If no plugin ID is -// provided the error code is assumed to be a user error code from the -// politeiad API. -func convertPiErrorStatus(pluginID string, errCode int) pi.ErrorStatusT { - switch pluginID { - case "": - // politeiad API - e := pd.ErrorStatusT(errCode) - return convertPiErrorStatusFromPD(e) - case piplugin.PluginID: - // Pi plugin - e := piplugin.ErrorCodeT(errCode) - return convertPiErrorStatusFromPiPlugin(e) - } - - // No corresponding pi error status found - return pi.ErrorStatusInvalid -} - // Fetch remote identity func getIdentity(rpcHost, rpcCert, rpcIdentityFile, interactive string) error { id, err := util.RemoteIdentity(false, rpcHost, rpcCert) @@ -219,101 +159,6 @@ func getIdentity(rpcHost, rpcCert, rpcIdentityFile, interactive string) error { return nil } -// respondWithPiError returns an HTTP error status to the client. If it's a pi -// user error, it returns a 4xx HTTP status and the specific user error code. -// If it's an internal server error, it returns 500 and a UNIX timestamp which -// is also outputted to the logs so that it can be correlated later if the user -// files a complaint. -func respondWithPiError(w http.ResponseWriter, r *http.Request, format string, err error) { - // Check for pi user error - var ue pi.UserErrorReply - if errors.As(err, &ue) { - // Error is a pi user error. Log it and return a 400. - if len(ue.ErrorContext) == 0 { - log.Infof("Pi user error: %v %v %v", - util.RemoteAddr(r), int64(ue.ErrorCode), - pi.ErrorStatus[ue.ErrorCode]) - } else { - log.Errorf("Pi user error: %v %v %v: %v", - util.RemoteAddr(r), int64(ue.ErrorCode), - pi.ErrorStatus[ue.ErrorCode], ue.ErrorContext) - } - - util.RespondWithJSON(w, http.StatusBadRequest, - pi.UserErrorReply{ - ErrorCode: ue.ErrorCode, - ErrorContext: ue.ErrorContext, - }) - return - } - - // Check for politeiad error - var pdErr pdError - if errors.As(err, &pdErr) { - var ( - pluginID = pdErr.ErrorReply.Plugin - errCode = pdErr.ErrorReply.ErrorCode - errContext = pdErr.ErrorReply.ErrorContext - ) - - // Check if the politeiad error corresponds to a pi user error - piErrCode := convertPiErrorStatus(pluginID, errCode) - if piErrCode == pi.ErrorStatusInvalid { - // politeiad error does not correspond to a pi user error. Log - // it and return a 500. - t := time.Now().Unix() - if pluginID == "" { - log.Errorf("%v %v %v %v Internal error %v: error "+ - "code from politeiad: %v", util.RemoteAddr(r), r.Method, - r.URL, r.Proto, t, errCode) - } else { - log.Errorf("%v %v %v %v Internal error %v: error "+ - "code from politeiad plugin %v: %v %v", util.RemoteAddr(r), - r.Method, r.URL, r.Proto, t, pluginID, errCode, errContext) - } - - util.RespondWithJSON(w, http.StatusInternalServerError, - pi.ServerErrorReply{ - ErrorCode: t, - }) - return - } - - // politeiad error does correspond to a pi user error. Log it and - // return a 400. - if len(errContext) == 0 { - log.Infof("Pi user error: %v %v %v", - util.RemoteAddr(r), int64(piErrCode), - pi.ErrorStatus[piErrCode]) - } else { - log.Infof("Pi user error: %v %v %v: %v", - util.RemoteAddr(r), int64(piErrCode), - pi.ErrorStatus[piErrCode], - strings.Join(errContext, ", ")) - } - - util.RespondWithJSON(w, http.StatusBadRequest, - pi.UserErrorReply{ - ErrorCode: piErrCode, - ErrorContext: strings.Join(errContext, ", "), - }) - return - - } - - // Error is a politeiawww server error. Log it and return a 500. - t := time.Now().Unix() - e := fmt.Sprintf(format, err) - log.Errorf("%v %v %v %v Internal error %v: %v", - util.RemoteAddr(r), r.Method, r.URL, r.Proto, t, e) - log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) - - util.RespondWithJSON(w, http.StatusInternalServerError, - pi.ServerErrorReply{ - ErrorCode: t, - }) -} - // userErrorStatus retrieves the human readable error message for an error // status code. The status code can be from either the pi or cms api. func userErrorStatus(e www.ErrorStatusT) string { From 758b3f23a2756ebca955761e41736281f041a5bd Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 27 Jan 2021 12:32:01 -0600 Subject: [PATCH 254/449] Cleanup. --- .../backend/tlogbe/plugins/comments/cmds.go | 6 ++--- .../tlogbe/plugins/comments/comments.go | 12 ++++----- .../backend/tlogbe/plugins/dcrdata/dcrdata.go | 12 ++++----- politeiad/backend/tlogbe/plugins/pi/pi.go | 12 ++++----- politeiad/backend/tlogbe/plugins/plugins.go | 12 ++++----- .../backend/tlogbe/plugins/ticketvote/cmds.go | 8 +++--- .../tlogbe/plugins/ticketvote/ticketvote.go | 14 +++++------ politeiad/backend/tlogbe/plugins/user/user.go | 10 ++++---- politeiad/backend/tlogbe/testing.go | 14 +++++------ politeiad/backend/tlogbe/tlog/plugin.go | 4 +-- politeiad/backend/tlogbe/tlog/tlog.go | 14 ++++------- politeiad/backend/tlogbe/tlogbe.go | 25 ++++++++----------- politeiad/plugins/comments/comments.go | 1 - 13 files changed, 65 insertions(+), 79 deletions(-) diff --git a/politeiad/backend/tlogbe/plugins/comments/cmds.go b/politeiad/backend/tlogbe/plugins/comments/cmds.go index ba8a3bd0b..98513922b 100644 --- a/politeiad/backend/tlogbe/plugins/comments/cmds.go +++ b/politeiad/backend/tlogbe/plugins/comments/cmds.go @@ -23,9 +23,9 @@ import ( const ( // Blob entry data descriptors - dataDescriptorCommentAdd = "cadd_v1" - dataDescriptorCommentDel = "cdel_v1" - dataDescriptorCommentVote = "cvote_v1" + dataDescriptorCommentAdd = "cadd-v1" + dataDescriptorCommentDel = "cdel-v1" + dataDescriptorCommentVote = "cvote-v1" // Data types dataTypeCommentAdd = "cadd" diff --git a/politeiad/backend/tlogbe/plugins/comments/comments.go b/politeiad/backend/tlogbe/plugins/comments/comments.go index 92a8dc45a..7cf3d1c5a 100644 --- a/politeiad/backend/tlogbe/plugins/comments/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments/comments.go @@ -21,12 +21,12 @@ import ( // error which causes a 500. Solution: add the timestamp to the vote index. var ( - _ plugins.Client = (*commentsPlugin)(nil) + _ plugins.PluginClient = (*commentsPlugin)(nil) ) // commentsPlugin is the tlog backend implementation of the comments plugin. // -// commentsPlugin satisfies the plugins.Client interface. +// commentsPlugin satisfies the plugins.PluginClient interface. type commentsPlugin struct { sync.Mutex tlog plugins.TlogClient @@ -64,7 +64,7 @@ func (p *commentsPlugin) mutex(token []byte) *sync.Mutex { // Setup performs any plugin setup that is required. // -// This function satisfies the plugins.Client interface. +// This function satisfies the plugins.PluginClient interface. func (p *commentsPlugin) Setup() error { log.Tracef("Setup") @@ -73,7 +73,7 @@ func (p *commentsPlugin) Setup() error { // Cmd executes a plugin command. // -// This function satisfies the plugins.Client interface. +// This function satisfies the plugins.PluginClient interface. func (p *commentsPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { log.Tracef("Cmd: %v %x %v", treeID, token, cmd) @@ -105,7 +105,7 @@ func (p *commentsPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (s // Hook executes a plugin hook. // -// This function satisfies the plugins.Client interface. +// This function satisfies the plugins.PluginClient interface. func (p *commentsPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { log.Tracef("Hook: %v %x %v", treeID, token, plugins.Hooks[h]) @@ -114,7 +114,7 @@ func (p *commentsPlugin) Hook(treeID int64, token []byte, h plugins.HookT, paylo // Fsck performs a plugin filesystem check. // -// This function satisfies the plugins.Client interface. +// This function satisfies the plugins.PluginClient interface. func (p *commentsPlugin) Fsck(treeIDs []int64) error { log.Tracef("Fsck") diff --git a/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go b/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go index e2aed877a..7f3103ca8 100644 --- a/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go +++ b/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go @@ -45,10 +45,10 @@ const ( ) var ( - _ plugins.Client = (*dcrdataPlugin)(nil) + _ plugins.PluginClient = (*dcrdataPlugin)(nil) ) -// dcrdataplugin satisfies the plugins.Client interface. +// dcrdataplugin satisfies the plugins.PluginClient interface. type dcrdataPlugin struct { sync.Mutex activeNetParams *chaincfg.Params @@ -576,7 +576,7 @@ func (p *dcrdataPlugin) websocketSetup() { // Setup performs any plugin setup that is required. // -// This function satisfies the plugins.Client interface. +// This function satisfies the plugins.PluginClient interface. func (p *dcrdataPlugin) Setup() error { log.Tracef("setup") @@ -591,7 +591,7 @@ func (p *dcrdataPlugin) Setup() error { // Cmd executes a plugin command. // -// This function satisfies the plugins.Client interface. +// This function satisfies the plugins.PluginClient interface. func (p *dcrdataPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { log.Tracef("Cmd: %v %x %v", treeID, token, cmd) @@ -611,7 +611,7 @@ func (p *dcrdataPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (st // Hook executes a plugin hook. // -// This function satisfies the plugins.Client interface. +// This function satisfies the plugins.PluginClient interface. func (p *dcrdataPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { log.Tracef("Hook: %v %x %v", treeID, token, plugins.Hooks[h]) @@ -620,7 +620,7 @@ func (p *dcrdataPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payloa // Fsck performs a plugin filesystem check. // -// This function satisfies the plugins.Client interface. +// This function satisfies the plugins.PluginClient interface. func (p *dcrdataPlugin) Fsck(treeIDs []int64) error { log.Tracef("Fsck") diff --git a/politeiad/backend/tlogbe/plugins/pi/pi.go b/politeiad/backend/tlogbe/plugins/pi/pi.go index 5569ee8b1..80313b7da 100644 --- a/politeiad/backend/tlogbe/plugins/pi/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi/pi.go @@ -16,10 +16,10 @@ import ( ) var ( - _ plugins.Client = (*piPlugin)(nil) + _ plugins.PluginClient = (*piPlugin)(nil) ) -// piPlugin satisfies the plugins.Client interface. +// piPlugin satisfies the plugins.PluginClient interface. type piPlugin struct { sync.Mutex backend backend.Backend @@ -44,7 +44,7 @@ type piPlugin struct { // Setup performs any plugin setup that is required. // -// This function satisfies the plugins.Client interface. +// This function satisfies the plugins.PluginClient interface. func (p *piPlugin) Setup() error { log.Tracef("Setup") @@ -55,7 +55,7 @@ func (p *piPlugin) Setup() error { // Cmd executes a plugin command. // -// This function satisfies the plugins.Client interface. +// This function satisfies the plugins.PluginClient interface. func (p *piPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { log.Tracef("Cmd: %v %x %v %v", treeID, token, cmd, payload) @@ -69,7 +69,7 @@ func (p *piPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, // Hook executes a plugin hook. // -// This function satisfies the plugins.Client interface. +// This function satisfies the plugins.PluginClient interface. func (p *piPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { log.Tracef("Hook: %v %x %v", treeID, token, plugins.Hooks[h]) @@ -87,7 +87,7 @@ func (p *piPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload str // Fsck performs a plugin filesystem check. // -// This function satisfies the plugins.Client interface. +// This function satisfies the plugins.PluginClient interface. func (p *piPlugin) Fsck(treeIDs []int64) error { log.Tracef("Fsck") diff --git a/politeiad/backend/tlogbe/plugins/plugins.go b/politeiad/backend/tlogbe/plugins/plugins.go index 8d31015b6..f2fb73035 100644 --- a/politeiad/backend/tlogbe/plugins/plugins.go +++ b/politeiad/backend/tlogbe/plugins/plugins.go @@ -16,7 +16,7 @@ const ( // HookTypeInvalid is an invalid plugin hook. HookTypeInvalid HookT = 0 - // HootTypeNewRecordPre is called before new record is saved to + // HootTypeNewRecordPre is called before a new record is saved to // disk. HookTypeNewRecordPre HookT = 1 @@ -71,9 +71,7 @@ var ( } ) -// RecordStateT represents a record state. The record state is included in all -// hook payloads so that a plugin has the ability to implement different -// behaviors for different states. +// RecordStateT represents a record state. type RecordStateT int const ( @@ -157,9 +155,9 @@ type HookPluginPost struct { Reply string `json:"reply"` } -// Client provides an API for a tlog instance to use when interacting with a -// plugin. All tlog plugins must implement the Client interface. -type Client interface { +// PluginClient provides an API for a tlog instance to use when interacting +// with a plugin. All tlog plugins must implement the PluginClient interface. +type PluginClient interface { // Setup performs any required plugin setup. Setup() error diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go index 8d2659b32..15fba52eb 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go @@ -31,10 +31,10 @@ import ( const ( // Blob entry data descriptors - dataDescriptorAuthDetails = "authdetails_v1" - dataDescriptorVoteDetails = "votedetails_v1" - dataDescriptorCastVoteDetails = "castvotedetails_v1" - dataDescriptorStartRunoff = "startrunoff_v1" + dataDescriptorAuthDetails = "authdetails-v1" + dataDescriptorVoteDetails = "votedetails-v1" + dataDescriptorCastVoteDetails = "castvotedetails-v1" + dataDescriptorStartRunoff = "startrunoff-v1" // Data types dataTypeAuthDetails = "authdetails" diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go index de90445a0..e16c86faf 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go @@ -22,12 +22,10 @@ import ( ) var ( - _ plugins.Client = (*ticketVotePlugin)(nil) + _ plugins.PluginClient = (*ticketVotePlugin)(nil) ) -// TODO verify all writes only accept full length tokens - -// ticketVotePlugin satisfies the plugins.Client interface. +// ticketVotePlugin satisfies the plugins.PluginClient interface. type ticketVotePlugin struct { sync.Mutex backend backend.Backend @@ -87,7 +85,7 @@ func (p *ticketVotePlugin) mutex(token []byte) *sync.Mutex { // Setup performs any plugin setup that is required. // -// This function satisfies the plugins.Client interface. +// This function satisfies the plugins.PluginClient interface. func (p *ticketVotePlugin) Setup() error { log.Tracef("Setup") @@ -183,7 +181,7 @@ func (p *ticketVotePlugin) Setup() error { // Cmd executes a plugin command. // -// This function satisfies the plugins.Client interface. +// This function satisfies the plugins.PluginClient interface. func (p *ticketVotePlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { log.Tracef("Cmd: %v %x %v", treeID, token, cmd) @@ -219,7 +217,7 @@ func (p *ticketVotePlugin) Cmd(treeID int64, token []byte, cmd, payload string) // Hook executes a plugin hook. // -// This function satisfies the plugins.Client interface. +// This function satisfies the plugins.PluginClient interface. func (p *ticketVotePlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { log.Tracef("Hook: %v %x %v", treeID, token, plugins.Hooks[h]) @@ -237,7 +235,7 @@ func (p *ticketVotePlugin) Hook(treeID int64, token []byte, h plugins.HookT, pay // Fsck performs a plugin filesystem check. // -// This function satisfies the plugins.Client interface. +// This function satisfies the plugins.PluginClient interface. func (p *ticketVotePlugin) Fsck(treeIDs []int64) error { log.Tracef("Fsck") diff --git a/politeiad/backend/tlogbe/plugins/user/user.go b/politeiad/backend/tlogbe/plugins/user/user.go index 4e03f9162..5a58dd72f 100644 --- a/politeiad/backend/tlogbe/plugins/user/user.go +++ b/politeiad/backend/tlogbe/plugins/user/user.go @@ -15,7 +15,7 @@ import ( ) var ( - _ plugins.Client = (*userPlugin)(nil) + _ plugins.PluginClient = (*userPlugin)(nil) ) type userPlugin struct { @@ -30,7 +30,7 @@ type userPlugin struct { // Setup performs any plugin setup that is required. // -// This function satisfies the plugins.Client interface. +// This function satisfies the plugins.PluginClient interface. func (p *userPlugin) Setup() error { log.Tracef("Setup") @@ -39,7 +39,7 @@ func (p *userPlugin) Setup() error { // Cmd executes a plugin command. // -// This function satisfies the plugins.Client interface. +// This function satisfies the plugins.PluginClient interface. func (p *userPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { log.Tracef("Cmd: %v %x %v %v", treeID, token, cmd, payload) @@ -55,7 +55,7 @@ func (p *userPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (strin // Hook executes a plugin hook. // -// This function satisfies the plugins.Client interface. +// This function satisfies the plugins.PluginClient interface. func (p *userPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { log.Tracef("Hook: %v %x %v", treeID, token, plugins.Hooks[h]) @@ -77,7 +77,7 @@ func (p *userPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload s // Fsck performs a plugin filesystem check. // -// This function satisfies the plugins.Client interface. +// This function satisfies the plugins.PluginClient interface. func (p *userPlugin) Fsck(treeIDs []int64) error { log.Tracef("Fsck") diff --git a/politeiad/backend/tlogbe/testing.go b/politeiad/backend/tlogbe/testing.go index ad03d9673..b08e13c83 100644 --- a/politeiad/backend/tlogbe/testing.go +++ b/politeiad/backend/tlogbe/testing.go @@ -10,7 +10,6 @@ import ( "path/filepath" "testing" - "github.com/decred/dcrd/chaincfg/v3" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" ) @@ -28,13 +27,12 @@ func NewTestTlogBackend(t *testing.T) (*tlogBackend, func()) { dataDir := filepath.Join(homeDir, "data") tlogBackend := tlogBackend{ - activeNetParams: chaincfg.TestNet3Params(), - homeDir: homeDir, - dataDir: dataDir, - unvetted: tlog.NewTestTlogUnencrypted(t, dataDir, "unvetted"), - vetted: tlog.NewTestTlogEncrypted(t, dataDir, "vetted"), - prefixes: make(map[string][]byte), - vettedTreeIDs: make(map[string]int64), + homeDir: homeDir, + dataDir: dataDir, + unvetted: tlog.NewTestTlogUnencrypted(t, dataDir, "unvetted"), + vetted: tlog.NewTestTlogEncrypted(t, dataDir, "vetted"), + prefixes: make(map[string][]byte), + vettedTreeIDs: make(map[string]int64), inv: inventory{ unvetted: make(map[backend.MDStatusT][]string), vetted: make(map[backend.MDStatusT][]string), diff --git a/politeiad/backend/tlogbe/tlog/plugin.go b/politeiad/backend/tlogbe/tlog/plugin.go index aead92479..631da3730 100644 --- a/politeiad/backend/tlogbe/tlog/plugin.go +++ b/politeiad/backend/tlogbe/tlog/plugin.go @@ -34,7 +34,7 @@ const ( type plugin struct { id string settings []backend.PluginSetting - client plugins.Client + client plugins.PluginClient } func (t *Tlog) plugin(pluginID string) (plugin, bool) { @@ -61,7 +61,7 @@ func (t *Tlog) PluginRegister(b backend.Backend, p backend.Plugin) error { log.Tracef("%v PluginRegister: %v", t.id, p.ID) var ( - client plugins.Client + client plugins.PluginClient err error dataDir = filepath.Join(t.dataDir, pluginDataDirname) diff --git a/politeiad/backend/tlogbe/tlog/tlog.go b/politeiad/backend/tlogbe/tlog/tlog.go index e69e08ce6..9d79b80f6 100644 --- a/politeiad/backend/tlogbe/tlog/tlog.go +++ b/politeiad/backend/tlogbe/tlog/tlog.go @@ -32,11 +32,11 @@ const ( defaultStoreDirname = "store" // Blob entry data descriptors - dataDescriptorFile = "file_v1" - dataDescriptorRecordMetadata = "recordmd_v1" - dataDescriptorMetadataStream = "mdstream_v1" - dataDescriptorRecordIndex = "rindex_v1" - dataDescriptorAnchor = "anchor_v1" + dataDescriptorFile = "file-v1" + dataDescriptorRecordMetadata = "recordmd-v1" + dataDescriptorMetadataStream = "mdstream-v1" + dataDescriptorRecordIndex = "rindex-v1" + dataDescriptorAnchor = "anchor-v1" // The keys for kv store blobs are saved by stuffing them into the // ExtraData field of their corresponding trillian log leaf. The @@ -46,9 +46,6 @@ const ( // data out of the store, which can become an issue in situations // such as searching for a record index that has been buried by // thousands of leaves from plugin data. - // TODO key prefix app-dataID: - // TODO the leaf ExtraData field should be hinted. Similar to what - // we do for blobs. dataTypeSeperator = ":" dataTypeRecordIndex = "rindex" dataTypeRecordContent = "rcontent" @@ -59,7 +56,6 @@ var ( _ plugins.TlogClient = (*Tlog)(nil) ) -// TODO change tlog name to tstore. // We do not unwind. type Tlog struct { sync.Mutex diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 8296d3255..141f233fb 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -33,7 +33,6 @@ import ( // TODO testnet vs mainnet trillian databases // TODO fsck -// TODO move memory caches to filesystem const ( defaultEncryptionKeyFilename = "tlogbe.key" @@ -50,12 +49,11 @@ var ( // tlogBackend implements the backend.Backend interface. type tlogBackend struct { sync.RWMutex - activeNetParams *chaincfg.Params - homeDir string - dataDir string - shutdown bool - unvetted *tlog.Tlog - vetted *tlog.Tlog + homeDir string + dataDir string + shutdown bool + unvetted *tlog.Tlog + vetted *tlog.Tlog // prefixes contains the prefix to full token mapping for unvetted // records. The prefix is the first n characters of the hex encoded @@ -1787,13 +1785,12 @@ func New(anp *chaincfg.Params, homeDir, dataDir, unvettedTrillianHost, unvettedT // Setup tlogbe t := tlogBackend{ - activeNetParams: anp, - homeDir: homeDir, - dataDir: dataDir, - unvetted: unvetted, - vetted: vetted, - prefixes: make(map[string][]byte), - vettedTreeIDs: make(map[string]int64), + homeDir: homeDir, + dataDir: dataDir, + unvetted: unvetted, + vetted: vetted, + prefixes: make(map[string][]byte), + vettedTreeIDs: make(map[string]int64), inv: inventory{ unvetted: make(map[backend.MDStatusT][]string), vetted: make(map[backend.MDStatusT][]string), diff --git a/politeiad/plugins/comments/comments.go b/politeiad/plugins/comments/comments.go index 1203736cf..39a7e3c9f 100644 --- a/politeiad/plugins/comments/comments.go +++ b/politeiad/plugins/comments/comments.go @@ -21,7 +21,6 @@ const ( CmdVotes = "votes" // Get comment votes CmdTimestamps = "timestamps" // Get timestamps - // TODO make these default settings, not policies // PolicyCommentLengthMax is the maximum number of characters // accepted for comments. PolicyCommentLengthMax = 8000 From 3039409a92bd600c3b419d2d673a68ad8d64d74b Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 28 Jan 2021 17:50:07 -0600 Subject: [PATCH 255/449] Finish record routes. --- .../backend/tlogbe/plugins/user/hooks.go | 130 ++++++++++-- politeiad/client/user.go | 40 ++-- politeiad/plugins/user/user.go | 36 ++-- politeiawww/api/records/v1/v1.go | 7 +- politeiawww/email.go | 6 +- politeiawww/events.go | 14 +- politeiawww/piwww.go | 59 +++--- politeiawww/records/events.go | 9 + politeiawww/records/process.go | 198 ++++++++++++------ politeiawww/records/records.go | 9 + politeiawww/testing.go | 4 +- 11 files changed, 354 insertions(+), 158 deletions(-) diff --git a/politeiad/backend/tlogbe/plugins/user/hooks.go b/politeiad/backend/tlogbe/plugins/user/hooks.go index 1cff472e6..de424b693 100644 --- a/politeiad/backend/tlogbe/plugins/user/hooks.go +++ b/politeiad/backend/tlogbe/plugins/user/hooks.go @@ -9,6 +9,9 @@ import ( "encoding/json" "errors" "fmt" + "io" + "strconv" + "strings" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" @@ -137,28 +140,6 @@ func userMetadataPreventUpdates(current, update []backend.MetadataStream) error return nil } -/* -// statusChangesDecode decodes a JSON byte slice into a []StatusChange slice. -func statusChangesDecode(payload []byte) ([]pi.StatusChange, error) { - var statuses []pi.StatusChange - d := json.NewDecoder(strings.NewReader(string(payload))) - for { - var sc pi.StatusChange - err := d.Decode(&sc) - if errors.Is(err, io.EOF) { - break - } else if err != nil { - return nil, err - } - statuses = append(statuses, sc) - } - - return statuses, nil -} - - -*/ - func (p *userPlugin) hookNewRecordPre(payload string) error { var nr plugins.HookNewRecordPre err := json.Unmarshal([]byte(payload), &nr) @@ -237,6 +218,98 @@ func (p *userPlugin) hookEditMetadataPre(payload string) error { return userMetadataPreventUpdates(em.Current.Metadata, em.Metadata) } +// statusChangesDecode decodes a []StatusChange from the provided metadata +// streams. If a status change metadata stream is not found, nil is returned. +func statusChangesDecode(metadata []backend.MetadataStream) ([]user.StatusChangeMetadata, error) { + var statuses []user.StatusChangeMetadata + for _, v := range metadata { + if v.ID == user.MDStreamIDUserMetadata { + d := json.NewDecoder(strings.NewReader(v.Payload)) + for { + var sc user.StatusChangeMetadata + err := d.Decode(&sc) + if errors.Is(err, io.EOF) { + break + } else if err != nil { + return nil, err + } + statuses = append(statuses, sc) + } + } + break + } + return statuses, nil +} + +var ( + // statusReasonRequired contains the list of record statuses that + // require an accompanying reason to be given in the status change. + statusReasonRequired = map[backend.MDStatusT]struct{}{ + backend.MDStatusCensored: {}, + backend.MDStatusArchived: {}, + } +) + +// statusChangeMetadataVerify parses the status change metadata from the +// metadata streams and verifies that its contents are valid. +func statusChangeMetadataVerify(rm backend.RecordMetadata, metadata []backend.MetadataStream) error { + // Decode status change metadata + statusChanges, err := statusChangesDecode(metadata) + if err != nil { + return err + } + + // Verify that status change metadata is present + if len(statusChanges) == 0 { + return backend.PluginError{ + PluginID: user.PluginID, + ErrorCode: int(user.ErrorCodeStatusChangeMetadataNotFound), + } + } + scm := statusChanges[len(statusChanges)-1] + + // Verify token matches + if scm.Token != rm.Token { + e := fmt.Sprintf("status change token does not match record "+ + "metadata token: got %v, want %v", scm.Token, rm.Token) + return backend.PluginError{ + PluginID: user.PluginID, + ErrorCode: int(user.ErrorCodeTokenInvalid), + ErrorContext: e, + } + } + + // Verify status matches + if scm.Status != int(rm.Status) { + e := fmt.Sprintf("status from metadata does not match status from "+ + "record metadata: got %v, want %v", scm.Status, rm.Status) + return backend.PluginError{ + PluginID: user.PluginID, + ErrorCode: int(user.ErrorCodeStatusInvalid), + ErrorContext: e, + } + } + + // Verify reason was included on required status changes + _, ok := statusReasonRequired[rm.Status] + if ok && scm.Reason == "" { + return backend.PluginError{ + PluginID: user.PluginID, + ErrorCode: int(user.ErrorCodeReasonInvalid), + ErrorContext: "a reason must be given for this status change", + } + } + + // Verify signature + msg := scm.Token + scm.Version + strconv.Itoa(scm.Status) + scm.Reason + err = util.VerifySignature(scm.Signature, scm.PublicKey, msg) + if err != nil { + return convertSignatureError(err) + } + + return nil +} + func (p *userPlugin) hookSetRecordStatusPre(payload string) error { var srs plugins.HookSetRecordStatus err := json.Unmarshal([]byte(payload), &srs) @@ -245,5 +318,16 @@ func (p *userPlugin) hookSetRecordStatusPre(payload string) error { } // User metadata should not change on status changes - return userMetadataPreventUpdates(srs.Current.Metadata, srs.Metadata) + err = userMetadataPreventUpdates(srs.Current.Metadata, srs.Metadata) + if err != nil { + return err + } + + // Verify status change metadata + err = statusChangeMetadataVerify(srs.RecordMetadata, srs.Metadata) + if err != nil { + return err + } + + return nil } diff --git a/politeiad/client/user.go b/politeiad/client/user.go index 553486056..cc93b5dd5 100644 --- a/politeiad/client/user.go +++ b/politeiad/client/user.go @@ -50,8 +50,9 @@ func (c *Client) Author(ctx context.Context, state, token string) (string, error } // UserRecords sends the user plugin UserRecords command to the politeiad v1 -// API. -func (c *Client) UserRecords(ctx context.Context, state, token, userID string) ([]string, error) { +// API. A seperate command is sent for the unvetted and vetted records. The +// returned map is a map[recordState][]token. +func (c *Client) UserRecords(ctx context.Context, userID string) (map[string][]string, error) { // Setup request ur := user.UserRecords{ UserID: userID, @@ -62,8 +63,13 @@ func (c *Client) UserRecords(ctx context.Context, state, token, userID string) ( } cmds := []pdv1.PluginCommandV2{ { - State: state, - Token: token, + State: pdv1.RecordStateUnvetted, + ID: user.PluginID, + Command: user.CmdUserRecords, + Payload: string(b), + }, + { + State: pdv1.RecordStateVetted, ID: user.PluginID, Command: user.CmdUserRecords, Payload: string(b), @@ -75,20 +81,24 @@ func (c *Client) UserRecords(ctx context.Context, state, token, userID string) ( if err != nil { return nil, err } - - // Decode reply if len(replies) == 0 { return nil, fmt.Errorf("no replies found") } - pcr := replies[0] - if pcr.Error != nil { - return nil, pcr.Error - } - var urr user.UserRecordsReply - err = json.Unmarshal([]byte(pcr.Payload), &urr) - if err != nil { - return nil, err + + // Decode replies + reply := make(map[string][]string, 2) // [recordState][]token + for _, v := range replies { + if v.Error != nil { + // Swallow individual errors + continue + } + var urr user.UserRecordsReply + err = json.Unmarshal([]byte(v.Payload), &urr) + if err != nil { + return nil, err + } + reply[v.State] = urr.Records } - return urr.Records, nil + return reply, nil } diff --git a/politeiad/plugins/user/user.go b/politeiad/plugins/user/user.go index 8ad0d59ad..5ee3a447e 100644 --- a/politeiad/plugins/user/user.go +++ b/politeiad/plugins/user/user.go @@ -13,16 +13,18 @@ const ( CmdAuthor = "author" // Get record author CmdUserRecords = "userrecords" // Get user submitted records - // TODO add record status change mdstream - // TODO make whether user md is required a plugin setting - // TODO MDStream IDs need to be plugin specific. If we can't then // we need to make a mdstream package to aggregate all the mdstream - // ID. + // IDs. // MDStreamIDUserMetadata is the politeiad metadata stream ID for // the UserMetadata structure. MDStreamIDUserMetadata = 1 + + // MDStreamIDStatusChanges is the politeiad metadata stream ID for + // the status changes metadata. Status changes should be appended + // onto this metadata stream. + MDStreamIDStatusChanges = 2 ) // ErrorCodeT represents a plugin error that was caused by the user. @@ -36,7 +38,10 @@ const ( ErrorCodeUserIDInvalid ErrorCodePublicKeyInvalid ErrorCodeSignatureInvalid - ErrorCodeUpdateNotAllowed + ErrorCodeStatusChangeMetadataNotFound + ErrorCodeTokenInvalid + ErrorCodeStatusInvalid + ErrorCodeReasonInvalid ) var ( @@ -45,15 +50,6 @@ var ( ErrorCodes = map[ErrorCodeT]string{ ErrorCodeInvalid: "error code invalid", } - - /* - // statusReasonRequired contains the list of proposal statuses that - // require an accompanying reason to be given for the status change. - statusReasonRequired = map[piv1.PropStatusT]struct{}{ - piv1.PropStatusCensored: {}, - piv1.PropStatusAbandoned: {}, - } - */ ) // UserMetadata contains user metadata about a politeiad record. It is @@ -67,6 +63,18 @@ type UserMetadata struct { Signature string `json:"signature"` // Signature of merkle root } +// StatusChangeMetadata contains the user signature for a record status change. +// +// Signature is the client signature of the Token+Version+Status+Reason. +type StatusChangeMetadata struct { + Token string `json:"token"` + Version string `json:"version"` + Status int `json:"status"` + Reason string `json:"message,omitempty"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` +} + // Author returns the user ID of a record's author. If no UserMetadata is // present for the record then an empty string will be returned. type Author struct{} diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go index 2811b8297..7223a485a 100644 --- a/politeiawww/api/records/v1/v1.go +++ b/politeiawww/api/records/v1/v1.go @@ -341,13 +341,14 @@ type TimestampsReply struct { Files map[string]Timestamp `json:"files"` } -// UserRecords requests the tokens of all records submitted by a user. +// UserRecords requests the tokens of all records submitted by a user. Unvetted +// record tokens are only returned to admins and the record author. type UserRecords struct { - State string `json:"state"` UserID string `json:"userid"` } // UserRecordsReply is the reply to the UserRecords command. type UserRecordsReply struct { - Tokens []string `json:"tokens"` + Unvetted []string `json:"unvetted"` + Vetted []string `json:"Vetted"` } diff --git a/politeiawww/email.go b/politeiawww/email.go index 51a4835bc..97f19338c 100644 --- a/politeiawww/email.go +++ b/politeiawww/email.go @@ -119,7 +119,7 @@ func (p *politeiawww) emailProposalStatusChange(d dataProposalStatusChange, prop body string ) switch d.status { - case rcv1.StatusPublic: + case rcv1.RecordStatusPublic: subject = "New Proposal Published" tmplData := proposalVetted{ Name: proposalName, @@ -152,7 +152,7 @@ func (p *politeiawww) emailProposalStatusChangeToAuthor(d dataProposalStatusChan body string ) switch d.status { - case rcv1.StatusPublic: + case rcv1.RecordStatusPublic: subject = "Your Proposal Has Been Published" tmplData := proposalVettedToAuthor{ Name: proposalName, @@ -163,7 +163,7 @@ func (p *politeiawww) emailProposalStatusChangeToAuthor(d dataProposalStatusChan return err } - case rcv1.StatusCensored: + case rcv1.RecordStatusCensored: subject = "Your Proposal Has Been Censored" tmplData := proposalCensoredToAuthor{ Name: proposalName, diff --git a/politeiawww/events.go b/politeiawww/events.go index 8a41884c1..cace30620 100644 --- a/politeiawww/events.go +++ b/politeiawww/events.go @@ -205,12 +205,12 @@ func (p *politeiawww) handleEventProposalEdited(ch chan interface{}) { } type dataProposalStatusChange struct { - token string // Proposal censorship token - state string // Updated proposal state - status rcv1.StatusT // Updated proposal status - version string // Proposal version - reason string // Status change reason - adminID string // Admin uuid + token string // Proposal censorship token + state string // Updated proposal state + status rcv1.RecordStatusT // Updated proposal status + version string // Proposal version + reason string // Status change reason + adminID string // Admin uuid } func (p *politeiawww) handleEventProposalStatusChange(ch chan interface{}) { @@ -223,7 +223,7 @@ func (p *politeiawww) handleEventProposalStatusChange(ch chan interface{}) { // Check if proposal is in correct status for notification switch d.status { - case rcv1.StatusPublic, rcv1.StatusCensored: + case rcv1.RecordStatusPublic, rcv1.RecordStatusCensored: // The status requires a notification be sent default: // The status does not require a notification be sent. Listen diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index ad79f5888..6b856c48b 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -9,15 +9,17 @@ import ( "net/http" cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/comments" + "github.com/decred/politeia/politeiawww/records" "github.com/decred/politeia/politeiawww/ticketvote" "github.com/google/uuid" ) // setupPiRoutes sets up the API routes for piwww mode. -func (p *politeiawww) setupPiRoutes(c *comments.Comments, t *ticketvote.TicketVote) { +func (p *politeiawww) setupPiRoutes(r *records.Records, c *comments.Comments, t *ticketvote.TicketVote) { // Return a 404 when a route is not found p.router.NotFoundHandler = http.HandlerFunc(p.handleNotFound) @@ -57,33 +59,22 @@ func (p *politeiawww) setupPiRoutes(c *comments.Comments, t *ticketvote.TicketVo www.RouteBatchVoteSummary, p.handleBatchVoteSummary, permissionPublic) - /* - // Pi routes - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteProposals, p.handleProposals, - permissionPublic) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteVoteInventory, p.handleVoteInventory, - permissionPublic) - - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteProposalNew, p.handleProposalNew, - permissionLogin) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteProposalEdit, p.handleProposalEdit, - permissionLogin) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteProposalSetStatus, p.handleProposalSetStatus, - permissionAdmin) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteProposalInventory, p.handleProposalInventory, - permissionPublic) - - // Record routes - p.addRoute(http.MethodPost, rcv1.APIRoute, - rcv1.RouteTimestamps, p.handleTimestamps, - permissionPublic) - */ + // Record routes + p.addRoute(http.MethodPost, rcv1.APIRoute, + rcv1.RouteNew, r.HandleNew, + permissionLogin) + p.addRoute(http.MethodPost, rcv1.APIRoute, + rcv1.RouteEdit, r.HandleEdit, + permissionLogin) + p.addRoute(http.MethodPost, rcv1.APIRoute, + rcv1.RouteSetStatus, r.HandleSetStatus, + permissionAdmin) + p.addRoute(http.MethodPost, rcv1.APIRoute, + rcv1.RouteInventory, r.HandleInventory, + permissionPublic) + p.addRoute(http.MethodPost, rcv1.APIRoute, + rcv1.RouteTimestamps, r.HandleTimestamps, + permissionPublic) // Comment routes p.addRoute(http.MethodPost, cmv1.APIRoute, @@ -131,16 +122,26 @@ func (p *politeiawww) setupPiRoutes(c *comments.Comments, t *ticketvote.TicketVo tkv1.RouteTimestamps, t.HandleTimestamps, permissionPublic) + /* + // Pi routes + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteProposals, p.handleProposals, + permissionPublic) + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteVoteInventory, p.handleVoteInventory, + permissionPublic) + */ } func (p *politeiawww) setupPi() error { // Setup api contexts c := comments.New(p.cfg, p.politeiad, p.db, p.sessions, p.events) tv := ticketvote.New(p.cfg, p.politeiad, p.sessions, p.events) + r := records.New(p.cfg, p.politeiad, p.sessions, p.events) // Setup routes p.setUserWWWRoutes() - p.setupPiRoutes(c, tv) + p.setupPiRoutes(r, c, tv) // Verify paywall settings switch { diff --git a/politeiawww/records/events.go b/politeiawww/records/events.go index f9c2a454b..0400b83cd 100644 --- a/politeiawww/records/events.go +++ b/politeiawww/records/events.go @@ -15,6 +15,9 @@ const ( // EventTypeEdit is emitted when a a record is edited. EventTypeEdit = "records-edit" + + // EventTypeSetStatus is emitted when a a record is edited. + EventTypeSetStatus = "records-setstatus" ) // EventNew is the event data for the EventTypeNew. @@ -29,3 +32,9 @@ type EventEdit struct { State string Record v1.Record } + +// EventSetStatus is the event data for the EventTypeSetStatus. +type EventSetStatus struct { + State string + Record v1.Record +} diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index ebb443b55..380580d3e 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -211,63 +211,73 @@ func (r *Records) processEdit(ctx context.Context, e v1.Edit, u user.User) (*v1. } func (r *Records) processSetStatus(ctx context.Context, ss v1.SetStatus, u user.User) (*v1.SetStatusReply, error) { + log.Tracef("processSetStatus: %v %v %v", ss.Token, ss.Status, ss.Reason) - /* - // Verify user signed with their active identity - if u.PublicKey() != pss.PublicKey { - return nil, v1.UserErrorReply{ - ErrorCode: v1.ErrorCodePublicKeyInvalid, - ErrorContext: "not active identity", - } - } - */ - - // Status change user metadata - /* - timestamp := time.Now().Unix() - sc := pi.StatusChange{ - Token: pss.Token, - Version: pss.Version, - Status: pi.PropStatusT(pss.Status), - Reason: pss.Reason, - PublicKey: pss.PublicKey, - Signature: pss.Signature, - Timestamp: timestamp, + // Verify user signed using active identity + if u.PublicKey() != ss.PublicKey { + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodePublicKeyInvalid, + ErrorContext: "not active identity", } - b, err := json.Marshal(sc) + } + + // Setup status change metadata + scm := pduser.StatusChangeMetadata{ + Token: ss.Token, + Version: ss.Version, + Status: int(ss.Status), + Reason: ss.Reason, + PublicKey: ss.PublicKey, + Signature: ss.Signature, + } + b, err := json.Marshal(scm) + if err != nil { + return nil, err + } + mdAppend := []pdv1.MetadataStream{ + { + ID: pduser.MDStreamIDStatusChanges, + Payload: string(b), + }, + } + mdOverwrite := []pdv1.MetadataStream{} + + // Send politeiad request + var ( + s = convertStatusToPD(ss.Status) + pdr *pdv1.Record + ) + switch ss.State { + case v1.RecordStateUnvetted: + pdr, err = r.politeiad.SetUnvettedStatus(ctx, ss.Token, + s, mdAppend, mdOverwrite) if err != nil { return nil, err } - mdAppend := []pdv1.MetadataStream{ - { - ID: pi.MDStreamIDStatusChanges, - Payload: string(b), - }, + case v1.RecordStateVetted: + pdr, err = r.politeiad.SetVettedStatus(ctx, ss.Token, + s, mdAppend, mdOverwrite) + if err != nil { + return nil, err } - mdOverwrite := []pdv1.MetadataStream{} - */ - - /* - // This goes in the user plugin - // Verify reason - _, required := statusReasonRequired[pss.Status] - if required && pss.Reason == "" { - return nil, v1.UserErrorReply{ - ErrorCode: v1.ErrorCodePropStatusChangeReasonInvalid, - ErrorContext: "reason not given", - } + default: + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeRecordStateInvalid, } + } - msg := pss.Token + pss.Version + strconv.Itoa(int(pss.Status)) + pss.Reason - err := util.VerifySignature(pss.Signature, pss.PublicKey, msg) - if err != nil { - return nil, convertUserErrorFromSignatureError(err) - } - */ + rc := convertRecordToV1(*pdr, ss.State) // Emit event + r.events.Emit(EventTypeSetStatus, + EventSetStatus{ + State: ss.State, + Record: rc, + }) - return nil, nil + return &v1.SetStatusReply{ + Record: rc, + }, nil } // record returns a version of a record from politeiad. If version is an empty @@ -390,7 +400,33 @@ func (r *Records) processRecords(ctx context.Context, rs v1.Records, u *user.Use } func (r *Records) processInventory(ctx context.Context, u *user.User) (*v1.InventoryReply, error) { - return nil, nil + log.Tracef("processInventory") + + ir, err := r.politeiad.InventoryByStatus(ctx) + if err != nil { + return nil, err + } + + unvetted := make(map[string][]string, len(ir.Unvetted)) + vetted := make(map[string][]string, len(ir.Vetted)) + for k, v := range ir.Vetted { + ks := v1.RecordStatuses[convertStatusToV1(k)] + vetted[ks] = v + } + + // Only admins are allowed to retrieve unvetted tokens. A user may + // or may not exist. + if u != nil && u.Admin { + for k, v := range ir.Unvetted { + ks := v1.RecordStatuses[convertStatusToV1(k)] + unvetted[ks] = v + } + } + + return &v1.InventoryReply{ + Unvetted: unvetted, + Vetted: vetted, + }, nil } func (r *Records) processTimestamps(ctx context.Context, t v1.Timestamps, isAdmin bool) (*v1.TimestampsReply, error) { @@ -452,23 +488,45 @@ func (r *Records) processTimestamps(ctx context.Context, t v1.Timestamps, isAdmi } func (r *Records) processUserRecords(ctx context.Context, ur v1.UserRecords, u *user.User) (*v1.UserRecordsReply, error) { - /* - // Determine if unvetted tokens should be returned - switch { - case u == nil: - // No user session. Remove unvetted. - pir.Unvetted = nil - case u.Admin: - // User is an admin. Return unvetted. - case inv.UserID == u.ID.String(): - // User is requesting their own proposals. Return unvetted. - default: - // Remove unvetted for all other cases - pir.Unvetted = nil - } - */ + log.Tracef("processUserRecords: %v", ur.UserID) + + reply, err := r.politeiad.UserRecords(ctx, ur.UserID) + if err != nil { + return nil, err + } - return nil, nil + // Unpack reply + var ( + unvetted = make([]string, 0) + vetted = make([]string, 0) + ) + tokens, ok := reply[v1.RecordStateUnvetted] + if ok { + unvetted = tokens + } + tokens, ok = reply[v1.RecordStateUnvetted] + if ok { + vetted = tokens + } + + // Determine if unvetted tokens should be returned + switch { + case u == nil: + // No user session. Remove unvetted. + unvetted = []string{} + case u.Admin: + // User is an admin. Return unvetted. + case ur.UserID == u.ID.String(): + // User is requesting their own records. Return unvetted. + default: + // Remove unvetted for all other cases + unvetted = []string{} + } + + return &v1.UserRecordsReply{ + Unvetted: unvetted, + Vetted: vetted, + }, nil } // paywallIsEnabled returns whether the user paywall is enabled. @@ -606,6 +664,20 @@ func convertFilesToPD(f []v1.File) []pdv1.File { return files } +func convertStatusToPD(s v1.RecordStatusT) pdv1.RecordStatusT { + switch s { + case v1.RecordStatusUnreviewed: + return pdv1.RecordStatusNotReviewed + case v1.RecordStatusPublic: + return pdv1.RecordStatusPublic + case v1.RecordStatusCensored: + return pdv1.RecordStatusCensored + case v1.RecordStatusArchived: + return pdv1.RecordStatusArchived + } + return pdv1.RecordStatusInvalid +} + func convertStatusToV1(s pdv1.RecordStatusT) v1.RecordStatusT { switch s { case pdv1.RecordStatusNotReviewed: diff --git a/politeiawww/records/records.go b/politeiawww/records/records.go index 77168d248..d16a8eb82 100644 --- a/politeiawww/records/records.go +++ b/politeiawww/records/records.go @@ -26,6 +26,7 @@ type Records struct { events *events.Manager } +// HandleNew is the request handler for the records v1 New route. func (c *Records) HandleNew(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleNew") @@ -56,6 +57,7 @@ func (c *Records) HandleNew(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, nr) } +// HandleEdit is the request handler for the records v1 Edit route. func (c *Records) HandleEdit(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleEdit") @@ -86,6 +88,7 @@ func (c *Records) HandleEdit(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, er) } +// HandleSetStatus is the request handler for the records v1 SetStatus route. func (c *Records) HandleSetStatus(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleSetStatus") @@ -116,6 +119,7 @@ func (c *Records) HandleSetStatus(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, ssr) } +// HandleDetails is the request handler for the records v1 Details route. func (c *Records) HandleDetails(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleDetails") @@ -148,6 +152,7 @@ func (c *Records) HandleDetails(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, dr) } +// HandleRecords is the request handler for the records v1 Records route. func (c *Records) HandleRecords(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleRecords") @@ -180,6 +185,7 @@ func (c *Records) HandleRecords(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, rsr) } +// HandleInventory is the request handler for the records v1 Inventory route. func (c *Records) HandleInventory(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleInventory") @@ -212,6 +218,7 @@ func (c *Records) HandleInventory(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, ir) } +// HandleTimestamps is the request handler for the records v1 Timestamps route. func (c *Records) HandleTimestamps(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleTimestamps") @@ -245,6 +252,8 @@ func (c *Records) HandleTimestamps(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, tr) } +// HandleUserRecords is the request handler for the records v1 UserRecords +// route. func (c *Records) HandleUserRecords(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleUserRecords") diff --git a/politeiawww/testing.go b/politeiawww/testing.go index 1ec8978c9..07c67ccfc 100644 --- a/politeiawww/testing.go +++ b/politeiawww/testing.go @@ -28,6 +28,7 @@ import ( www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/comments" "github.com/decred/politeia/politeiawww/config" + "github.com/decred/politeia/politeiawww/records" "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/politeiawww/ticketvote" "github.com/decred/politeia/politeiawww/user" @@ -364,10 +365,11 @@ func newTestPoliteiawww(t *testing.T) (*politeiawww, func()) { // TODO setup testing var c *comments.Comments var tv *ticketvote.TicketVote + var r *records.Records // Setup routes p.setUserWWWRoutes() - p.setupPiRoutes(c, tv) + p.setupPiRoutes(r, c, tv) // The cleanup is handled using a closure so that the temp dir // can be deleted using the local variable and not cfg.DataDir. From 54b854566d97445af0a63ad37953bf9ff46eabb9 Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 29 Jan 2021 17:33:25 -0600 Subject: [PATCH 256/449] Add pi routes. --- politeiad/client/pi.go | 50 +++++ politeiad/plugins/pi/pi.go | 50 +---- politeiawww/api/pi/v1/v1.go | 114 ++++------ politeiawww/api/records/v1/v1.go | 27 ++- politeiawww/dcc.go | 4 +- politeiawww/events/events.go | 2 +- politeiawww/invoices.go | 24 ++- politeiawww/log.go | 4 + politeiawww/pi/error.go | 93 +++++++++ politeiawww/pi/log.go | 25 +++ politeiawww/pi/pi.go | 346 +++++-------------------------- politeiawww/pi/process.go | 299 ++++++++++++++++++++++++++ politeiawww/records/events.go | 2 +- politeiawww/records/process.go | 40 ++-- politeiawww/util/merkle.go | 48 ----- wsdcrdata/wsdcrdata.go | 2 +- 16 files changed, 631 insertions(+), 499 deletions(-) create mode 100644 politeiad/client/pi.go create mode 100644 politeiawww/pi/error.go create mode 100644 politeiawww/pi/log.go create mode 100644 politeiawww/pi/process.go delete mode 100644 politeiawww/util/merkle.go diff --git a/politeiad/client/pi.go b/politeiad/client/pi.go new file mode 100644 index 000000000..b9671ce3d --- /dev/null +++ b/politeiad/client/pi.go @@ -0,0 +1,50 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package client + +import ( + "context" + "encoding/json" + "fmt" + + pdv1 "github.com/decred/politeia/politeiad/api/v1" + "github.com/decred/politeia/politeiad/plugins/pi" +) + +// PiVoteInv sends the pi plugin VoteInv command to the politeiad v1 API. +func (c *Client) PiVoteInv(ctx context.Context) (*pi.VoteInventoryReply, error) { + // Setup request + cmds := []pdv1.PluginCommandV2{ + { + State: pdv1.RecordStateVetted, + Token: "", + ID: pi.PluginID, + Command: pi.CmdVoteInv, + Payload: "", + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + if pcr.Error != nil { + return nil, pcr.Error + } + + // Decode reply + var vir pi.VoteInventoryReply + err = json.Unmarshal([]byte(pcr.Payload), &vir) + if err != nil { + return nil, err + } + + return &vir, nil +} diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index 60d576e20..ab4769484 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -7,6 +7,7 @@ package pi const ( + // PluginID is the pi plugin ID. PluginID = "pi" // Plugin commands @@ -57,55 +58,6 @@ type ProposalMetadata struct { Name string `json:"name"` } -// PropStatusT represents a proposal status. These map directly to the -// politeiad record statuses, but some have had their names changed to better -// reflect their intended use case by proposals. -type PropStatusT int - -const ( - // PropStatusInvalid is an invalid proposal status. - PropStatusInvalid PropStatusT = 0 - - // PropStatusUnreviewed indicates the proposal has been submitted, - // but has not yet been reviewed and made public by an admin. A - // proposal with this status will have a proposal state of - // PropStateUnvetted. - PropStatusUnvetted PropStatusT = 1 - - // PropStatusPublic indicates that a proposal has been reviewed and - // made public by an admin. A proposal with this status will have - // a proposal state of PropStateVetted. - PropStatusPublic PropStatusT = 2 - - // PropStatusCensored indicates that a proposal has been censored - // by an admin for violating the proposal guidlines. Both unvetted - // and vetted proposals can be censored so a proposal with this - // status can have a state of either PropStateUnvetted or - // PropStateVetted depending on whether the proposal was censored - // before or after it was made public. - PropStatusCensored PropStatusT = 3 - - // PropStatusUnreviewedChanges is a deprecated proposal status that - // has only been included so that the proposal statuses map - // directly to the politeiad record statuses. - PropStatusUnreviewedChanges PropStatusT = 4 - - // PropStatusAbandoned indicates that a proposal has been marked - // as abandoned by an admin due to the author being inactive. - PropStatusAbandoned PropStatusT = 5 -) - -var ( - // PropStatuses contains the human readable proposal statuses. - PropStatuses = map[PropStatusT]string{ - PropStatusInvalid: "invalid", - PropStatusUnvetted: "unvetted", - PropStatusPublic: "public", - PropStatusCensored: "censored", - PropStatusAbandoned: "abandoned", - } -) - // VoteInventory requests the tokens of all proposals in the inventory // categorized by their vote status. This call relies on the ticketvote // Inventory call, but breaks the Finished vote status out into Approved and diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 13dad1786..2086494fb 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -33,38 +33,19 @@ type ErrorCodeT int const ( // Error status codes - ErrorCodeInvalid ErrorCodeT = 0 - ErrorCodeInputInvalid ErrorCodeT = 1 - ErrorCodePageSizeExceeded ErrorCodeT = 2 - - // User errors - ErrorCodeUserRegistrationNotPaid ErrorCodeT = 100 - ErrorCodeUserBalanceInsufficient ErrorCodeT = 101 - ErrorCodeUnauthorized ErrorCodeT = 102 - - // Signature errors - ErrorCodePublicKeyInvalid ErrorCodeT = 200 - ErrorCodeSignatureInvalid ErrorCodeT = 201 + ErrorCodeInvalid ErrorCodeT = 0 + ErrorCodeInputInvalid ErrorCodeT = 1 + ErrorCodePageSizeExceeded ErrorCodeT = 2 + ErrorCodeProposalStateInvalid ErrorCodeT = 3 ) var ( - // ErrorCode contains human readable error messages. + // ErrorCodes contains human readable error messages. // TODO fill in error status messages - ErrorCode = map[ErrorCodeT]string{ + ErrorCodes = map[ErrorCodeT]string{ ErrorCodeInvalid: "error status invalid", ErrorCodeInputInvalid: "input invalid", ErrorCodePageSizeExceeded: "page size exceeded", - - // User errors - ErrorCodeUserRegistrationNotPaid: "user registration not paid", - ErrorCodeUserBalanceInsufficient: "user balance insufficient", - ErrorCodeUnauthorized: "user is unauthorized", - - // Signature errors - ErrorCodePublicKeyInvalid: "public key invalid", - ErrorCodeSignatureInvalid: "signature invalid", - - // Proposal errors } ) @@ -81,6 +62,19 @@ func (e UserErrorReply) Error() string { return fmt.Sprintf("user error code: %v", e.ErrorCode) } +// PluginErrorReply is the reply that the server returns when it encounters +// a plugin error. +type PluginErrorReply struct { + PluginID string `json:"pluginid"` + ErrorCode int `json:"errorcode"` + ErrorContext string `json:"errorcontext"` +} + +// Error satisfies the error interface. +func (e PluginErrorReply) Error() string { + return fmt.Sprintf("plugin error code: %v", e.ErrorCode) +} + // ServerErrorReply is the reply that the server returns when it encounters an // unrecoverable error while executing a command. The HTTP status code will be // 500 and the ErrorCode field will contain a UNIX timestamp that the user can @@ -130,15 +124,6 @@ const ( PropStatusAbandoned PropStatusT = 5 ) -// PropStatuses contains the human readable proposal statuses. -var PropStatuses = map[PropStatusT]string{ - PropStatusInvalid: "invalid", - PropStatusUnreviewed: "unreviewed", - PropStatusPublic: "public", - PropStatusCensored: "censored", - PropStatusAbandoned: "abandoned", -} - // File describes an individual file that is part of the proposal. The // directory structure must be flattened. type File struct { @@ -148,32 +133,24 @@ type File struct { Payload string `json:"payload"` // File content, base64 encoded } -// Metadata describes user specified proposal metadata. -type Metadata struct { - Hint string `json:"hint"` // Hint that describes the payload - Digest string `json:"digest"` // SHA256 digest of unencoded payload - Payload string `json:"payload"` // JSON metadata content, base64 encoded -} - const ( - // HintProposalMetadata is the Metadata object hint that is used - // when the payload contains a ProposalMetadata. - HintProposalMetadata = "proposalmd" + // FileNameProposalMetadata is the file name of the user submitted + // ProposalMetadata. + FileNameProposalMetadata = "proposalmetadata.json" - // HintVoteMetadata is the Metadata object hint that is used when - // the payload contains a VoteMetadata. - HintVoteMetadata = "votemd" + // FileNameVoteMetadata is the file name of the user submitted + // VoteMetadata. + FileNameVoteMetadata = "votemetadata.json" ) // ProposalMetadata contains metadata that is specified by the user on proposal -// submission. It is attached to a proposal submission as a Metadata object. +// submission. type ProposalMetadata struct { Name string `json:"name"` // Proposal name } // VoteMetadata is metadata that is specified by the user on proposal -// submission in order to host or participate in certain types of votes. It is -// attached to a proposal submission as a Metadata object. +// submission in order to host or participate in a runoff vote. type VoteMetadata struct { // LinkBy is set when the user intends for the proposal to be the // parent proposal in a runoff vote. It is a UNIX timestamp that @@ -215,11 +192,11 @@ type StatusChange struct { Timestamp int64 `json:"timestamp"` } -// ProposalRecord represents a proposal submission and its metadata. +// Proposal represents a proposal submission and its metadata. // // Signature is the client signature of the proposal merkle root. The merkle -// root is the ordered merkle root of all proposal Files and Metadata. -type ProposalRecord struct { +// root is the ordered merkle root of all proposal files. +type Proposal struct { Version string `json:"version"` // Proposal version Timestamp int64 `json:"timestamp"` // Submission UNIX timestamp State string `json:"state"` // Proposal state @@ -229,7 +206,6 @@ type ProposalRecord struct { PublicKey string `json:"publickey"` // Key used in signature Signature string `json:"signature"` // Signature of merkle root Files []File `json:"files"` // Proposal files - Metadata []Metadata `json:"metadata"` // User defined metadata Statuses []StatusChange `json:"statuses"` // Status change history // CensorshipRecord contains cryptographic proof that the proposal @@ -237,29 +213,27 @@ type ProposalRecord struct { CensorshipRecord CensorshipRecord `json:"censorshiprecord"` } -// ProposalRequest is used to request a ProposalRecord. If the version is -// omitted, the most recent version will be returned. -type ProposalRequest struct { - Token string `json:"token"` - Version string `json:"version,omitempty"` -} +const ( + // ProposalsPageSize is the maximum number of proposals that can be + // requested in a Proposals request. + ProposalsPageSize = 10 +) -// Proposals retrieves the ProposalRecord for each of the provided proposal -// requests. +// Proposals retrieves the Proposal for each of the provided tokens. // -// This command does not return user submitted proposal files or metadata, -// except for the ProposalMetadata, which contains the proposal name. All other -// user submitted data isi removed. Unvetted proposals are also stripped of the -// ProposalMetadata when being returned to non-admins. +// This command does not return the proposal index file or any attachment +// files. It will return the ProposalMetadata file and the VoteMetadata file if +// one is present. Unvetted proposals are stripped of all user submitted data +// when being returned to non-admins. type Proposals struct { - State string `json:"state"` - Requests []ProposalRequest `json:"requests"` + State string `json:"state"` + Tokens []string `json:"tokens"` } // ProposalsReply is the reply to the Proposals command. Any tokens that did -// not correspond to a ProposalRecord will not be included in the reply. +// not correspond to a Proposal will not be included in the reply. type ProposalsReply struct { - Proposals map[string]ProposalRecord `json:"proposals"` // [token]Proposal + Proposals map[string]Proposal `json:"proposals"` // [token]Proposal } // VoteInventory retrieves the tokens of all public, non-abandoned proposals diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go index 7223a485a..6d1a4c46f 100644 --- a/politeiawww/api/records/v1/v1.go +++ b/politeiawww/api/records/v1/v1.go @@ -38,6 +38,7 @@ const ( ErrorCodeSignatureInvalid ErrorCodeRecordStateInvalid ErrorCodeRecordNotFound + ErrorCodePageSizeExceeded ) var ( @@ -261,18 +262,21 @@ type DetailsReply struct { Record Record `json:"record"` } -// RecordRequest is used to request a record version. If the version is -// omitted, the most recent version will be returned. -type RecordRequest struct { - Token string `json:"token"` - Version string `json:"version,omitempty"` -} +const ( + // RecordsPageSize is the maximum number of records that can be + // requested in a Records request. + RecordsPageSize = 10 +) -// Records requests a batch of records. Only the record metadata is returned. -// The Details command must be used to retrieve the record files. +// Records requests a batch of records. +// +// Only the record metadata is returned. The Details command must be used to +// retrieve the record files or a specific version of the record. Since record +// files are not included in the reply, unvetted records are returned to all +// users. type Records struct { - State string `json:"state"` - Requests []RecordRequest `json:"requests"` + State string `json:"state"` + Tokens []string `json:"tokens"` } // RecordsReply is the reply to the Records command. Any tokens that did not @@ -282,7 +286,8 @@ type RecordsReply struct { } // Inventory requests the tokens of all records in the inventory, categorized -// by record state and record status. +// by record state and record status. Unvetted record tokens will only be +// returned to admins. type Inventory struct{} // InventoryReply is the reply to the Inventory command. The returned maps are diff --git a/politeiawww/dcc.go b/politeiawww/dcc.go index 5f343edcf..83bd11ce2 100644 --- a/politeiawww/dcc.go +++ b/politeiawww/dcc.go @@ -28,7 +28,6 @@ import ( www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/cmsdatabase" "github.com/decred/politeia/politeiawww/user" - wwwutil "github.com/decred/politeia/politeiawww/util" "github.com/decred/politeia/util" ) @@ -703,8 +702,7 @@ func (p *politeiawww) validateDCC(nd cms.NewDCC, u *user.User) error { } // Note that we need validate the string representation of the merkle - files := convertPiFilesFromWWW([]www.File{nd.File}) - mr, err := wwwutil.MerkleRoot(files, nil) + mr, err := merkleRoot([]www.File{nd.File}) if err != nil { return err } diff --git a/politeiawww/events/events.go b/politeiawww/events/events.go index e15b7b9e1..cf42a58e8 100644 --- a/politeiawww/events/events.go +++ b/politeiawww/events/events.go @@ -8,7 +8,7 @@ import ( "sync" ) -// Manager manages event listeners for different event types. +// Manager manages event listeners. type Manager struct { sync.Mutex listeners map[string][]chan interface{} diff --git a/politeiawww/invoices.go b/politeiawww/invoices.go index 4381c803e..441418ae3 100644 --- a/politeiawww/invoices.go +++ b/politeiawww/invoices.go @@ -7,6 +7,7 @@ package main import ( "bytes" "context" + "crypto/sha256" "encoding/base64" "encoding/hex" "encoding/json" @@ -20,6 +21,7 @@ import ( "time" "github.com/decred/dcrd/dcrutil/v3" + "github.com/decred/dcrtime/merkle" "github.com/decred/politeia/decredplugin" "github.com/decred/politeia/mdstream" pd "github.com/decred/politeia/politeiad/api/v1" @@ -29,7 +31,6 @@ import ( "github.com/decred/politeia/politeiawww/cmsdatabase" database "github.com/decred/politeia/politeiawww/cmsdatabase" "github.com/decred/politeia/politeiawww/user" - wwwutil "github.com/decred/politeia/politeiawww/util" "github.com/decred/politeia/util" ) @@ -798,6 +799,24 @@ func (p *politeiawww) processNewInvoice(ctx context.Context, ni cms.NewInvoice, }, nil } +func merkleRoot(files []www.File) (string, error) { + // Calculate file digests + digests := make([]*[sha256.Size]byte, 0, len(files)) + for _, f := range files { + b, err := base64.StdEncoding.DecodeString(f.Payload) + if err != nil { + return "", err + } + digest := util.Digest(b) + var hf [sha256.Size]byte + copy(hf[:], digest) + digests = append(digests, &hf) + } + + // Return merkle root + return hex.EncodeToString(merkle.Root(digests)[:]), nil +} + func (p *politeiawww) validateInvoice(ni cms.NewInvoice, u *user.CMSUser) error { log.Tracef("validateInvoice") @@ -1118,8 +1137,7 @@ func (p *politeiawww) validateInvoice(ni cms.NewInvoice, u *user.CMSUser) error } // Note that we need validate the string representation of the merkle - files := convertPiFilesFromWWW(ni.Files) - mr, err := wwwutil.MerkleRoot(files, nil) + mr, err := merkleRoot(ni.Files) if err != nil { return err } diff --git a/politeiawww/log.go b/politeiawww/log.go index ea7f9981c..4a7ac673b 100644 --- a/politeiawww/log.go +++ b/politeiawww/log.go @@ -13,6 +13,8 @@ import ( "github.com/decred/politeia/politeiawww/codetracker/github" ghdb "github.com/decred/politeia/politeiawww/codetracker/github/database/cockroachdb" "github.com/decred/politeia/politeiawww/comments" + "github.com/decred/politeia/politeiawww/pi" + "github.com/decred/politeia/politeiawww/records" "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/politeiawww/ticketvote" "github.com/decred/politeia/politeiawww/user/cockroachdb" @@ -68,6 +70,8 @@ func init() { sessions.UseLogger(log) comments.UseLogger(log) ticketvote.UseLogger(log) + records.UseLogger(log) + pi.UseLogger(log) } // subsystemLoggers maps each subsystem identifier to its associated logger. diff --git a/politeiawww/pi/error.go b/politeiawww/pi/error.go new file mode 100644 index 000000000..c8bdda039 --- /dev/null +++ b/politeiawww/pi/error.go @@ -0,0 +1,93 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package pi + +import ( + "errors" + "fmt" + "net/http" + "runtime/debug" + "strings" + "time" + + pdclient "github.com/decred/politeia/politeiad/client" + v1 "github.com/decred/politeia/politeiawww/api/pi/v1" + "github.com/decred/politeia/util" +) + +func respondWithError(w http.ResponseWriter, r *http.Request, format string, err error) { + var ( + ue v1.UserErrorReply + pe pdclient.Error + ) + switch { + case errors.As(err, &ue): + // Comments user error + m := fmt.Sprintf("Comments user error: %v %v %v", + util.RemoteAddr(r), ue.ErrorCode, v1.ErrorCodes[ue.ErrorCode]) + if ue.ErrorContext != "" { + m += fmt.Sprintf(": %v", ue.ErrorContext) + } + log.Infof(m) + util.RespondWithJSON(w, http.StatusBadRequest, + v1.UserErrorReply{ + ErrorCode: ue.ErrorCode, + ErrorContext: ue.ErrorContext, + }) + return + + case errors.As(err, &pe): + // Politeiad error + var ( + pluginID = pe.ErrorReply.PluginID + errCode = pe.ErrorReply.ErrorCode + errContext = pe.ErrorReply.ErrorContext + ) + switch { + case pluginID != "": + // Politeiad plugin error. Log it and return a 400. + m := fmt.Sprintf("Plugin error: %v %v %v", + util.RemoteAddr(r), pluginID, errCode) + if len(errContext) > 0 { + m += fmt.Sprintf(": %v", strings.Join(errContext, ", ")) + } + log.Infof(m) + util.RespondWithJSON(w, http.StatusBadRequest, + v1.PluginErrorReply{ + PluginID: pluginID, + ErrorCode: errCode, + ErrorContext: strings.Join(errContext, ", "), + }) + return + + default: + // Unknown politeiad error. Log it and return a 500. + ts := time.Now().Unix() + log.Errorf("%v %v %v %v Internal error %v: error code "+ + "from politeiad: %v", util.RemoteAddr(r), r.Method, r.URL, + r.Proto, ts, errCode) + + util.RespondWithJSON(w, http.StatusInternalServerError, + v1.ServerErrorReply{ + ErrorCode: ts, + }) + return + } + + default: + // Internal server error. Log it and return a 500. + t := time.Now().Unix() + e := fmt.Sprintf(format, err) + log.Errorf("%v %v %v %v Internal error %v: %v", + util.RemoteAddr(r), r.Method, r.URL, r.Proto, t, e) + log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) + + util.RespondWithJSON(w, http.StatusInternalServerError, + v1.ServerErrorReply{ + ErrorCode: t, + }) + return + } +} diff --git a/politeiawww/pi/log.go b/politeiawww/pi/log.go new file mode 100644 index 000000000..48e6bee30 --- /dev/null +++ b/politeiawww/pi/log.go @@ -0,0 +1,25 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package pi + +import "github.com/decred/slog" + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log = slog.Disabled + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = slog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using slog. +func UseLogger(logger slog.Logger) { + log = logger +} diff --git a/politeiawww/pi/pi.go b/politeiawww/pi/pi.go index bedcea9dd..d5cdc5188 100644 --- a/politeiawww/pi/pi.go +++ b/politeiawww/pi/pi.go @@ -4,316 +4,80 @@ package pi -/* -var ( - // errProposalNotFound is emitted when a proposal is not found in - // politeiad for a specified token and version. - errProposalNotFound = errors.New("proposal not found") +import ( + "encoding/json" + "net/http" + + pdclient "github.com/decred/politeia/politeiad/client" + v1 "github.com/decred/politeia/politeiawww/api/pi/v1" + "github.com/decred/politeia/politeiawww/config" + "github.com/decred/politeia/politeiawww/sessions" + "github.com/decred/politeia/politeiawww/user" + "github.com/decred/politeia/util" ) -// proposalName parses the proposal name from the ProposalMetadata and returns -// it. An empty string will be returned if any errors occur or if a name is not -// found. -func proposalName(r pdv1.Record) string { - var name string - for _, v := range r.Files { - if v.Name == pi.FileNameProposalMetadata { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return "" - } - var pm pi.ProposalMetadata - err = json.Unmarshal(b, &pm) - if err != nil { - return "" - } - name = pm.Name - } - } - return name -} - -// proposalRecordFillInUser fills in all user fields that are store in the -// user database and not in politeiad. -func proposalRecordFillInUser(pr piv1.ProposalRecord, u user.User) piv1.ProposalRecord { - pr.UserID = u.ID.String() - pr.Username = u.Username - return pr -} - -func convertRecordStatusFromPropStatus(s piv1.PropStatusT) pdv1.RecordStatusT { - switch s { - case piv1.PropStatusUnreviewed: - return pdv1.RecordStatusNotReviewed - case piv1.PropStatusPublic: - return pdv1.RecordStatusPublic - case piv1.PropStatusCensored: - return pdv1.RecordStatusCensored - case piv1.PropStatusAbandoned: - return pdv1.RecordStatusArchived - } - return pdv1.RecordStatusInvalid -} - -func convertFileFromMetadata(m piv1.Metadata) pdv1.File { - var name string - switch m.Hint { - case piv1.HintProposalMetadata: - name = pi.FileNameProposalMetadata - } - return pdv1.File{ - Name: name, - MIME: mimeTypeTextUTF8, - Digest: m.Digest, - Payload: m.Payload, - } -} - -func convertFileFromPi(f piv1.File) pdv1.File { - return pdv1.File{ - Name: f.Name, - MIME: f.MIME, - Digest: f.Digest, - Payload: f.Payload, - } +// Pi is the context for the pi API. +type Pi struct { + cfg *config.Config + politeiad *pdclient.Client + userdb user.Database + sessions *sessions.Sessions } -func convertFilesFromPi(files []piv1.File) []pdv1.File { - f := make([]pdv1.File, 0, len(files)) - for _, v := range files { - f = append(f, convertFileFromPi(v)) - } - return f -} +// HandleProposals is the request handler for the pi v1 Proposals route. +func (p *Pi) HandleProposals(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleProposals") -func convertPropStatusFromPD(s pdv1.RecordStatusT) piv1.PropStatusT { - switch s { - case pdv1.RecordStatusNotFound: - // Intentionally omitted. No corresponding PropStatusT. - case pdv1.RecordStatusNotReviewed: - return piv1.PropStatusUnreviewed - case pdv1.RecordStatusCensored: - return piv1.PropStatusCensored - case pdv1.RecordStatusPublic: - return piv1.PropStatusPublic - case pdv1.RecordStatusUnreviewedChanges: - return piv1.PropStatusUnreviewed - case pdv1.RecordStatusArchived: - return piv1.PropStatusAbandoned - } - return piv1.PropStatusInvalid -} - -func convertCensorshipRecordFromPD(cr pdv1.CensorshipRecord) piv1.CensorshipRecord { - return piv1.CensorshipRecord{ - Token: cr.Token, - Merkle: cr.Merkle, - Signature: cr.Signature, - } -} - -func convertFilesFromPD(f []pdv1.File) ([]piv1.File, []piv1.Metadata) { - files := make([]piv1.File, 0, len(f)) - metadata := make([]piv1.Metadata, 0, len(f)) - for _, v := range f { - switch v.Name { - case pi.FileNameProposalMetadata: - metadata = append(metadata, piv1.Metadata{ - Hint: piv1.HintProposalMetadata, - Digest: v.Digest, - Payload: v.Payload, - }) - default: - files = append(files, piv1.File{ - Name: v.Name, - MIME: v.MIME, - Digest: v.Digest, - Payload: v.Payload, + var ps v1.Proposals + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&ps); err != nil { + respondWithError(w, r, "HandleProposals: unmarshal", + v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeInputInvalid, }) - } + return } - return files, metadata -} - -func statusChangesDecode(payload []byte) ([]rcv1.StatusChange, error) { - var statuses []rcv1.StatusChange - d := json.NewDecoder(strings.NewReader(string(payload))) - for { - var sc rcv1.StatusChange - err := d.Decode(&sc) - if errors.Is(err, io.EOF) { - break - } else if err != nil { - return nil, err - } - statuses = append(statuses, sc) - } - return statuses, nil -} -func convertProposalRecordFromPD(r pdv1.Record, state piv1.PropStateT) (*piv1.ProposalRecord, error) { - // Decode metadata streams - var ( - um *usermd.UserMetadata - sc = make([]pi.StatusChange, 0, 16) - err error - ) - for _, v := range r.Metadata { - switch v.ID { - case usermd.MDStreamIDUserMetadata: - var um usermd.UserMetadata - err = json.Unmarshal([]byte(v.Payload), &um) - if err != nil { - return nil, err - } - case pi.MDStreamIDStatusChanges: - sc, err = statusChangesDecode([]byte(v.Payload)) - if err != nil { - return nil, err - } - } + // Lookup session user. This is a public route so a session may not + // exist. Ignore any session not found errors. + u, err := p.sessions.GetSessionUser(w, r) + if err != nil && err != sessions.ErrSessionNotFound { + respondWithError(w, r, + "HandleDetails: GetSessionUser: %v", err) + return } - // Convert to pi types - files, metadata := convertFilesFromPD(r.Files) - status := convertPropStatusFromPD(r.Status) - - statuses := make([]piv1.StatusChange, 0, len(sc)) - for _, v := range sc { - statuses = append(statuses, piv1.StatusChange{ - Token: v.Token, - Version: v.Version, - Status: piv1.PropStatusT(v.Status), - Reason: v.Reason, - PublicKey: v.PublicKey, - Signature: v.Signature, - Timestamp: v.Timestamp, - }) + psr, err := p.processProposals(r.Context(), ps, u) + if err != nil { + respondWithError(w, r, + "HandleProposals: processProposals: %v", err) + return } - // Some fields are intentionally omitted because they are either - // user data that needs to be pulled from the user database or they - // are politeiad plugin data that needs to be retrieved using a - // plugin command. - return &piv1.ProposalRecord{ - Version: r.Version, - Timestamp: r.Timestamp, - State: state, - Status: status, - UserID: um.UserID, - Username: "", // Intentionally omitted - PublicKey: um.PublicKey, - Signature: um.Signature, - Statuses: statuses, - Files: files, - Metadata: metadata, - CensorshipRecord: convertCensorshipRecordFromPD(r.CensorshipRecord), - }, nil + util.RespondWithJSON(w, http.StatusOK, psr) } -// proposalRecords returns the ProposalRecord for each of the provided proposal -// requests. If a token does not correspond to an actual proposal then it will -// not be included in the returned map. -func (p *politeiawww) proposalRecords(ctx context.Context, state piv1.PropStateT, reqs []piv1.ProposalRequest, includeFiles bool) (map[string]piv1.ProposalRecord, error) { - // Get politeiad records - props := make([]piv1.ProposalRecord, 0, len(reqs)) - for _, v := range reqs { - var r *pdv1.Record - var err error - switch state { - case piv1.PropStateUnvetted: - // Unvetted politeiad record - r, err = p.politeiad.GetUnvetted(ctx, v.Token, v.Version) - if err != nil { - return nil, err - } - case piv1.PropStateVetted: - // Vetted politeiad record - r, err = p.politeiad.GetVetted(ctx, v.Token, v.Version) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("unknown state %v", state) - } - - if r.Status == pdv1.RecordStatusNotFound { - // Record wasn't found. Don't include token in the results. - continue - } - - pr, err := convertProposalRecordFromPD(*r, state) - if err != nil { - return nil, err - } - - // Remove files if specified. The Metadata objects will still be - // returned. - if !includeFiles { - pr.Files = []piv1.File{} - } - - props = append(props, *pr) - } +// HandleVoteInventory is the request handler for the pi v1 VoteInventory +// route. +func (p *Pi) HandleVoteInventory(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleVoteInventory") - // Verify we've got some results - if len(props) == 0 { - return map[string]piv1.ProposalRecord{}, nil - } - - // Get user data - pubkeys := make([]string, 0, len(props)) - for _, v := range props { - pubkeys = append(pubkeys, v.PublicKey) - } - ur, err := p.db.UsersGetByPubKey(pubkeys) + vir, err := p.processVoteInventory(r.Context()) if err != nil { - return nil, err - } - for k, v := range props { - token := v.CensorshipRecord.Token - u, ok := ur[v.PublicKey] - if !ok { - return nil, fmt.Errorf("user not found for pubkey %v from proposal %v", - v.PublicKey, token) - } - props[k] = proposalRecordFillInUser(v, u) - } - - // Convert proposals to a map - proposals := make(map[string]piv1.ProposalRecord, len(props)) - for _, v := range props { - proposals[v.CensorshipRecord.Token] = v + respondWithError(w, r, + "HandleVoteInventory: processVoteInventory: %v", err) + return } - return proposals, nil + util.RespondWithJSON(w, http.StatusOK, vir) } -// proposalRecord returns the proposal record for the provided token and -// version. A blank version will return the most recent version. A -// errProposalNotFound error will be returned if a proposal is not found for -// the provided token/version combination. -func (p *politeiawww) proposalRecord(ctx context.Context, state piv1.PropStateT, token, version string) (*piv1.ProposalRecord, error) { - prs, err := p.proposalRecords(ctx, state, []piv1.ProposalRequest{ - { - Token: token, - Version: version, - }, - }, true) - if err != nil { - return nil, err - } - pr, ok := prs[token] - if !ok { - return nil, errProposalNotFound +// New returns a new Pi context. +func New(cfg *config.Config, pdc *pdclient.Client, s *sessions.Sessions, udb user.Database) *Pi { + return &Pi{ + cfg: cfg, + politeiad: pdc, + userdb: udb, + sessions: s, } - return &pr, nil -} - -// proposalRecordLatest returns the latest version of the proposal record for -// the provided token. A errProposalNotFound error will be returned if a -// proposal is not found for the provided token. -func (p *politeiawww) proposalRecordLatest(ctx context.Context, state piv1.PropStateT, token string) (*piv1.ProposalRecord, error) { - return p.proposalRecord(ctx, state, token, "") } -*/ diff --git a/politeiawww/pi/process.go b/politeiawww/pi/process.go new file mode 100644 index 000000000..419f27fd5 --- /dev/null +++ b/politeiawww/pi/process.go @@ -0,0 +1,299 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package pi + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "strings" + + pdv1 "github.com/decred/politeia/politeiad/api/v1" + "github.com/decred/politeia/politeiad/plugins/pi" + pduser "github.com/decred/politeia/politeiad/plugins/user" + v1 "github.com/decred/politeia/politeiawww/api/pi/v1" + "github.com/decred/politeia/politeiawww/user" + "github.com/google/uuid" +) + +// proposal returns a version of a proposal record from politeiad. If version +// is an empty string then the most recent version will be returned. +func (p *Pi) proposal(ctx context.Context, state, token, version string) (*v1.Proposal, error) { + var ( + r *pdv1.Record + err error + ) + switch state { + case v1.ProposalStateUnvetted: + r, err = p.politeiad.GetUnvetted(ctx, token, version) + if err != nil { + return nil, err + } + case v1.ProposalStateVetted: + r, err = p.politeiad.GetVetted(ctx, token, version) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("invalid state %v", state) + } + + // Convert to a proposal + pr, err := convertRecord(*r, state) + if err != nil { + return nil, err + } + + // Fill in user data + userID := userIDFromMetadataStreams(r.Metadata) + uid, err := uuid.Parse(userID) + u, err := p.userdb.UserGetById(uid) + if err != nil { + return nil, err + } + proposalPopulateUserData(pr, *u) + + return pr, nil +} + +func (p *Pi) processProposals(ctx context.Context, ps v1.Proposals, u *user.User) (*v1.ProposalsReply, error) { + log.Tracef("processProposals: %v %v", ps.State, ps.Tokens) + + // Verify state + switch ps.State { + case v1.ProposalStateUnvetted, v1.ProposalStateVetted: + // Allowed; continue + default: + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeProposalStateInvalid, + } + } + + // Verify page size + if len(ps.Tokens) > v1.ProposalsPageSize { + e := fmt.Sprintf("max page size is %v", v1.ProposalsPageSize) + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodePageSizeExceeded, + ErrorContext: e, + } + } + + // Get all proposals in the batch. This should be a batched call to + // politeiad, but the politeiad API does not provided a batched + // records endpoint. + proposals := make(map[string]v1.Proposal, len(ps.Tokens)) + for _, v := range ps.Tokens { + pr, err := p.proposal(ctx, ps.State, v, "") + if err != nil { + // If any error occured simply skip this proposal. It will not + // be included in the reply. + continue + } + + // The only files that are returned in this call are the + // ProposalMetadata and the VoteMetadata files. + files := make([]v1.File, 0, len(pr.Files)) + for k := range pr.Files { + switch pr.Files[k].Name { + case v1.FileNameProposalMetadata, v1.FileNameVoteMetadata: + // Include file + files = append(files, pr.Files[k]) + default: + // All other files are disregarded. Do nothing. + } + } + + pr.Files = files + + proposals[pr.CensorshipRecord.Token] = *pr + } + + return &v1.ProposalsReply{ + Proposals: proposals, + }, nil +} + +func (p *Pi) processVoteInventory(ctx context.Context) (*v1.VoteInventoryReply, error) { + log.Tracef("processVoteInventory") + + vir, err := p.politeiad.PiVoteInv(ctx) + if err != nil { + return nil, err + } + + return &v1.VoteInventoryReply{ + Unauthorized: vir.Unauthorized, + Authorized: vir.Authorized, + Started: vir.Started, + Approved: vir.Approved, + Rejected: vir.Rejected, + BestBlock: vir.BestBlock, + }, nil +} + +// proposalName parses the proposal name from the ProposalMetadata and returns +// it. An empty string will be returned if any errors occur or if a name is not +// found. +func proposalName(r pdv1.Record) string { + var name string + for _, v := range r.Files { + if v.Name == pi.FileNameProposalMetadata { + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return "" + } + var pm pi.ProposalMetadata + err = json.Unmarshal(b, &pm) + if err != nil { + return "" + } + name = pm.Name + } + } + return name +} + +// proposalPopulateUserData populates a proposal with user data that is stored +// in the user database and not in politeiad. +func proposalPopulateUserData(pr *v1.Proposal, u user.User) { + pr.Username = u.Username +} + +func convertStatus(s pdv1.RecordStatusT) v1.PropStatusT { + switch s { + case pdv1.RecordStatusNotFound: + // Intentionally omitted. No corresponding PropStatusT. + case pdv1.RecordStatusNotReviewed: + return v1.PropStatusUnreviewed + case pdv1.RecordStatusCensored: + return v1.PropStatusCensored + case pdv1.RecordStatusPublic: + return v1.PropStatusPublic + case pdv1.RecordStatusUnreviewedChanges: + return v1.PropStatusUnreviewed + case pdv1.RecordStatusArchived: + return v1.PropStatusAbandoned + } + return v1.PropStatusInvalid +} + +func statusChangesDecode(payload []byte) ([]pduser.StatusChangeMetadata, error) { + var statuses []pduser.StatusChangeMetadata + d := json.NewDecoder(strings.NewReader(string(payload))) + for { + var sc pduser.StatusChangeMetadata + err := d.Decode(&sc) + if errors.Is(err, io.EOF) { + break + } else if err != nil { + return nil, err + } + statuses = append(statuses, sc) + } + return statuses, nil +} + +func convertRecord(r pdv1.Record, state string) (*v1.Proposal, error) { + // Decode metadata streams + var ( + um *pduser.UserMetadata + sc = make([]pduser.StatusChangeMetadata, 0, 16) + err error + ) + for _, v := range r.Metadata { + switch v.ID { + case pduser.MDStreamIDUserMetadata: + var um pduser.UserMetadata + err = json.Unmarshal([]byte(v.Payload), &um) + if err != nil { + return nil, err + } + case pduser.MDStreamIDStatusChanges: + sc, err = statusChangesDecode([]byte(v.Payload)) + if err != nil { + return nil, err + } + } + } + + // Convert files + files := make([]v1.File, 0, len(r.Files)) + for _, v := range r.Files { + files = append(files, v1.File{ + Name: v.Name, + MIME: v.MIME, + Digest: v.Digest, + Payload: v.Payload, + }) + } + + // Convert statuses + statuses := make([]v1.StatusChange, 0, len(sc)) + for _, v := range sc { + statuses = append(statuses, v1.StatusChange{ + Token: v.Token, + Version: v.Version, + Status: v1.PropStatusT(v.Status), + Reason: v.Reason, + PublicKey: v.PublicKey, + Signature: v.Signature, + }) + } + + // Some fields are intentionally omitted because they are user data + // that is not saved to politeiad and needs to be pulled from the + // user database. + return &v1.Proposal{ + Version: r.Version, + Timestamp: r.Timestamp, + State: state, + Status: convertStatus(r.Status), + UserID: um.UserID, + Username: "", // Intentionally omitted + PublicKey: um.PublicKey, + Signature: um.Signature, + Statuses: statuses, + Files: files, + CensorshipRecord: v1.CensorshipRecord{ + Token: r.CensorshipRecord.Token, + Merkle: r.CensorshipRecord.Merkle, + Signature: r.CensorshipRecord.Signature, + }, + }, nil +} + +// userMetadataDecode decodes and returns the UserMetadata from the provided +// metadata streams. If a UserMetadata is not found, nil is returned. +func userMetadataDecode(ms []pdv1.MetadataStream) (*pduser.UserMetadata, error) { + var userMD *pduser.UserMetadata + for _, v := range ms { + if v.ID == pduser.MDStreamIDUserMetadata { + var um pduser.UserMetadata + err := json.Unmarshal([]byte(v.Payload), &um) + if err != nil { + return nil, err + } + userMD = &um + break + } + } + return userMD, nil +} + +// userIDFromMetadataStreams searches for a UserMetadata and parses the user ID +// from it if found. An empty string is returned if no UserMetadata is found. +func userIDFromMetadataStreams(ms []pdv1.MetadataStream) string { + um, err := userMetadataDecode(ms) + if err != nil { + return "" + } + if um == nil { + return "" + } + return um.UserID +} diff --git a/politeiawww/records/events.go b/politeiawww/records/events.go index 0400b83cd..469bc0977 100644 --- a/politeiawww/records/events.go +++ b/politeiawww/records/events.go @@ -16,7 +16,7 @@ const ( // EventTypeEdit is emitted when a a record is edited. EventTypeEdit = "records-edit" - // EventTypeSetStatus is emitted when a a record is edited. + // EventTypeSetStatus is emitted when a a record status is updated. EventTypeSetStatus = "records-setstatus" ) diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index 380580d3e..147b5bc79 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -311,7 +311,7 @@ func (r *Records) record(ctx context.Context, state, token, version string) (*v1 if err != nil { return nil, err } - rc = recordPopulateUserData(rc, *u) + recordPopulateUserData(&rc, *u) return &rc, nil } @@ -354,7 +354,7 @@ func (r *Records) processDetails(ctx context.Context, d v1.Details, u *user.User } func (r *Records) processRecords(ctx context.Context, rs v1.Records, u *user.User) (*v1.RecordsReply, error) { - log.Tracef("processRecords: %v %v", rs.State, len(rs.Requests)) + log.Tracef("processRecords: %v %v", rs.State, len(rs.Tokens)) // Verify state switch rs.State { @@ -366,30 +366,29 @@ func (r *Records) processRecords(ctx context.Context, rs v1.Records, u *user.Use } } - // TODO Verify page size + // Verify page size + if len(rs.Tokens) > v1.RecordsPageSize { + e := fmt.Sprintf("max page size is %v", v1.RecordsPageSize) + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodePageSizeExceeded, + ErrorContext: e, + } + } - // Get all records in the batch - records := make(map[string]v1.Record, len(rs.Requests)) - for _, v := range rs.Requests { - rc, err := r.record(ctx, rs.State, v.Token, v.Version) + // Get all records in the batch. This should be a batched call to + // politeiad, but the politeiad API does not provided a batched + // records endpoint. + records := make(map[string]v1.Record, len(rs.Tokens)) + for _, v := range rs.Tokens { + rc, err := r.record(ctx, rs.State, v, "") if err != nil { // If any error occured simply skip this record. It will not // be included in the reply. continue } - // Only admins and the record author are allowed to retrieve - // unvetted record files. Remove files if the user is not an admin - // or the author. This is a public route so a user may not be - // present. - var ( - authorID = userIDFromMetadataStreams(rc.Metadata) - isAuthor = u != nil && u.ID.String() == authorID - isAdmin = u != nil && u.Admin - ) - if !isAuthor && !isAdmin { - rc.Files = []v1.File{} - } + // Record files are not returned in this call + rc.Files = []v1.File{} records[rc.CensorshipRecord.Token] = *rc } @@ -615,9 +614,8 @@ func (r *Records) piHookNewRecordPost(u user.User, token string) error { // recordPopulateUserData populates the record with user data that is not // stored in politeiad. -func recordPopulateUserData(r v1.Record, u user.User) v1.Record { +func recordPopulateUserData(r *v1.Record, u user.User) { r.Username = u.Username - return r } // userMetadataDecode decodes and returns the UserMetadata from the provided diff --git a/politeiawww/util/merkle.go b/politeiawww/util/merkle.go deleted file mode 100644 index b55dc36c7..000000000 --- a/politeiawww/util/merkle.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) 2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package util - -import ( - "crypto/sha256" - "encoding/base64" - "encoding/hex" - - "github.com/decred/dcrtime/merkle" - pi "github.com/decred/politeia/politeiawww/api/pi/v1" - "github.com/decred/politeia/util" -) - -// MerkleRoot converts the passed in list of files and metadata into SHA256 -// digests then calculates and returns the merkle root of the digests. -func MerkleRoot(files []pi.File, md []pi.Metadata) (string, error) { - digests := make([]*[sha256.Size]byte, 0, len(files)) - - // Calculate file digests - for _, f := range files { - b, err := base64.StdEncoding.DecodeString(f.Payload) - if err != nil { - return "", err - } - digest := util.Digest(b) - var hf [sha256.Size]byte - copy(hf[:], digest) - digests = append(digests, &hf) - } - - // Calculate metadata digests - for _, v := range md { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return "", err - } - digest := util.Digest(b) - var hv [sha256.Size]byte - copy(hv[:], digest) - digests = append(digests, &hv) - } - - // Return merkle root - return hex.EncodeToString(merkle.Root(digests)[:]), nil -} diff --git a/wsdcrdata/wsdcrdata.go b/wsdcrdata/wsdcrdata.go index d7f41fe9a..0741eb57c 100644 --- a/wsdcrdata/wsdcrdata.go +++ b/wsdcrdata/wsdcrdata.go @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2020 The Decred developers +// Copyright (c) 2019-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. From 5559f77c5287f823d26f2f429e24b89d63653011 Mon Sep 17 00:00:00 2001 From: luke Date: Sat, 30 Jan 2021 13:20:31 -0600 Subject: [PATCH 257/449] Get pictl to build. --- politeiawww/cmd/{piwww => pictl}/README.md | 52 +- .../cmd/{piwww => pictl}/castballot.go | 17 +- .../cmd/{piwww => pictl}/commentcensor.go | 12 +- .../cmd/{piwww => pictl}/commentnew.go | 12 +- politeiawww/cmd/{piwww => pictl}/comments.go | 7 +- .../cmd/{piwww => pictl}/commenttimestamps.go | 11 +- .../cmd/{piwww => pictl}/commentvote.go | 12 +- .../cmd/{piwww => pictl}/commentvotes.go | 9 +- politeiawww/cmd/{piwww => pictl}/help.go | 0 .../cmd/{piwww/piwww.go => pictl/pictl.go} | 46 +- politeiawww/cmd/{piwww => pictl}/policy.go | 0 .../cmd/{piwww => pictl}/proposaledit.go | 18 +- .../cmd/{piwww => pictl}/proposalinv.go | 7 +- .../cmd/{piwww => pictl}/proposalnew.go | 17 +- politeiawww/cmd/{piwww => pictl}/proposals.go | 10 +- .../cmd/{piwww => pictl}/proposalstatusset.go | 11 +- .../proposaltimestamps.go} | 37 +- .../cmd/{piwww => pictl}/sample-piwww.conf | 0 .../cmd/{piwww => pictl}/sendfaucettx.go | 0 politeiawww/cmd/{piwww => pictl}/subscribe.go | 0 politeiawww/cmd/{piwww => pictl}/testrun.go | 18 +- .../cmd/{piwww => pictl}/userdetails.go | 0 politeiawww/cmd/{piwww => pictl}/useredit.go | 0 .../cmd/{piwww => pictl}/useremailverify.go | 0 politeiawww/cmd/{piwww => pictl}/usernew.go | 0 .../{piwww => pictl}/userpaymentsrescan.go | 0 .../{piwww => pictl}/userproposalcredits.go | 0 .../{piwww => pictl}/userproposalpaywall.go | 0 .../{piwww => pictl}/userproposalpaywalltx.go | 0 .../userregistrationpayment.go | 0 .../userverificationresend.go | 0 politeiawww/cmd/pictl/util.go | 148 +++++ .../cmd/{piwww => pictl}/voteauthorize.go | 12 +- .../cmd/{piwww => pictl}/voteinventory.go | 7 +- .../cmd/{piwww => pictl}/voteresults.go | 7 +- politeiawww/cmd/{piwww => pictl}/votes.go | 7 +- politeiawww/cmd/{piwww => pictl}/votestart.go | 13 +- .../cmd/{piwww => pictl}/votestartrunoff.go | 14 +- .../cmd/{piwww => pictl}/votesummaries.go | 7 +- .../cmd/{piwww => pictl}/votetimestamps.go | 4 +- politeiawww/cmd/piwww/util.go | 221 ------- politeiawww/cmd/shared/client.go | 562 ------------------ 42 files changed, 262 insertions(+), 1036 deletions(-) rename politeiawww/cmd/{piwww => pictl}/README.md (81%) rename politeiawww/cmd/{piwww => pictl}/castballot.go (94%) rename politeiawww/cmd/{piwww => pictl}/commentcensor.go (93%) rename politeiawww/cmd/{piwww => pictl}/commentnew.go (94%) rename politeiawww/cmd/{piwww => pictl}/comments.go (92%) rename politeiawww/cmd/{piwww => pictl}/commenttimestamps.go (92%) rename politeiawww/cmd/{piwww => pictl}/commentvote.go (93%) rename politeiawww/cmd/{piwww => pictl}/commentvotes.go (92%) rename politeiawww/cmd/{piwww => pictl}/help.go (100%) rename politeiawww/cmd/{piwww/piwww.go => pictl/pictl.go} (88%) rename politeiawww/cmd/{piwww => pictl}/policy.go (100%) rename politeiawww/cmd/{piwww => pictl}/proposaledit.go (95%) rename politeiawww/cmd/{piwww => pictl}/proposalinv.go (91%) rename politeiawww/cmd/{piwww => pictl}/proposalnew.go (95%) rename politeiawww/cmd/{piwww => pictl}/proposals.go (95%) rename politeiawww/cmd/{piwww => pictl}/proposalstatusset.go (96%) rename politeiawww/cmd/{piwww/recordtimestamps.go => pictl/proposaltimestamps.go} (69%) rename politeiawww/cmd/{piwww => pictl}/sample-piwww.conf (100%) rename politeiawww/cmd/{piwww => pictl}/sendfaucettx.go (100%) rename politeiawww/cmd/{piwww => pictl}/subscribe.go (100%) rename politeiawww/cmd/{piwww => pictl}/testrun.go (98%) rename politeiawww/cmd/{piwww => pictl}/userdetails.go (100%) rename politeiawww/cmd/{piwww => pictl}/useredit.go (100%) rename politeiawww/cmd/{piwww => pictl}/useremailverify.go (100%) rename politeiawww/cmd/{piwww => pictl}/usernew.go (100%) rename politeiawww/cmd/{piwww => pictl}/userpaymentsrescan.go (100%) rename politeiawww/cmd/{piwww => pictl}/userproposalcredits.go (100%) rename politeiawww/cmd/{piwww => pictl}/userproposalpaywall.go (100%) rename politeiawww/cmd/{piwww => pictl}/userproposalpaywalltx.go (100%) rename politeiawww/cmd/{piwww => pictl}/userregistrationpayment.go (100%) rename politeiawww/cmd/{piwww => pictl}/userverificationresend.go (100%) create mode 100644 politeiawww/cmd/pictl/util.go rename politeiawww/cmd/{piwww => pictl}/voteauthorize.go (93%) rename politeiawww/cmd/{piwww => pictl}/voteinventory.go (90%) rename politeiawww/cmd/{piwww => pictl}/voteresults.go (90%) rename politeiawww/cmd/{piwww => pictl}/votes.go (92%) rename politeiawww/cmd/{piwww => pictl}/votestart.go (94%) rename politeiawww/cmd/{piwww => pictl}/votestartrunoff.go (93%) rename politeiawww/cmd/{piwww => pictl}/votesummaries.go (90%) rename politeiawww/cmd/{piwww => pictl}/votetimestamps.go (96%) delete mode 100644 politeiawww/cmd/piwww/util.go diff --git a/politeiawww/cmd/piwww/README.md b/politeiawww/cmd/pictl/README.md similarity index 81% rename from politeiawww/cmd/piwww/README.md rename to politeiawww/cmd/pictl/README.md index 1964eb9cd..ff20a62d5 100644 --- a/politeiawww/cmd/piwww/README.md +++ b/politeiawww/cmd/pictl/README.md @@ -1,59 +1,59 @@ -piwww +pictl ==== -piwww is a command line tool that allows you to interact with the politeiawww +pictl is a command line tool that allows you to interact with the politeiawww pi API. # Available Commands You can view the available commands and application options by using the help flag. - $ piwww -h + $ pictl -h You can view details about a specific command by using the help command. - $ piwww help + $ pictl help # Persisting Data Between Commands -piwww stores user identity data (the user's public/private key pair), session -cookies, and CSRF tokens in the piwww directory. This allows you to login with +pictl stores user identity data (the user's public/private key pair), session +cookies, and CSRF tokens in the pictl directory. This allows you to login with a user and use the same session data for subsequent commands. The data is segmented by host, allowing you to login and interact with multiple hosts simultaneously. -The location of the piwww directory varies based on your operating system. +The location of the pictl directory varies based on your operating system. **macOS** -`/Users//Library/Application Support/Piwww` +`/Users//Library/Application Support/Pictl` **Windows** -`C:\Users\\AppData\Local\Piwww` +`C:\Users\\AppData\Local\Pictl` **Ubuntu** -`~/.piwww` +`~/.pictl` # Setup Configuration File -piwww has a configuration file that you can setup to make execution easier. +pictl has a configuration file that you can setup to make execution easier. You should create the configuration file under the following paths. **macOS** -`/Users//Library/Application Support/Piwww/piwww.conf` +`/Users//Library/Application Support/Pictl/pictl.conf` **Windows** -`C:\Users\\AppData\Local\Piwww/piwww.conf` +`C:\Users\\AppData\Local\Pictl/pictl.conf` **Ubuntu** -`~/.piwww/piwww.conf` +`~/.pictl/pictl.conf` If you're developing locally, you'll want to set the politeiawww host in the configuration file to your local politeiawww instance. The host defaults to -`https://proposals.decred.org`. Copy these lines into your `piwww.conf` file. +`https://proposals.decred.org`. Copy these lines into your `pictl.conf` file. `skipverify` is used to skip TLS certificate verification and should only be used when running politeia locally. @@ -66,7 +66,7 @@ skipverify=true ## Create a new user - $ piwww usernew email@example.com username password --verify --paywall + $ pictl usernew email@example.com username password --verify --paywall `--verify` and `--paywall` are options that can be used when running politeiawww on testnet to make the user registration process quicker. @@ -81,7 +81,7 @@ confirmations before you'll be allowed to submit proposals.** ## Login with the user - $ piwww login email@example.com password + $ pictl login email@example.com password ## Assign admin privileges and create proposal credits @@ -103,9 +103,9 @@ get a `resource temporarily unavailable` error if you don't.** ## Submit a new proposal When submitting a proposal, you can either specify a markdown file or you can -use the `--random` flag to have piwww generate a random proposal for you. +use the `--random` flag to have pictl generate a random proposal for you. - $ piwww proposalnew --random + $ pictl proposalnew --random { "files": [ { @@ -143,7 +143,7 @@ commands. The censorship record token of the proposal example shown above is The proposal must first be vetted by an admin and have the proposal status set to public before it will be publicly viewable. - $ piwww proposalstatusset --unvetted [token] public + $ pictl proposalstatusset --unvetted [token] public Now that the proposal status has been made public, any user can comment on the proposal. Once the proposal author feels the discussion period was sufficient, @@ -154,14 +154,14 @@ they can authorize the voting period to start. Before an admin can start the voting period on a proposal the author must authorize the vote. - $ piwww voteauthorize [token] + $ pictl voteauthorize [token] ## Start a proposal vote (admin privileges required) Once a proposal vote has been authorized by the author, an admin can start the voting period at any point. - $ piwww votestart [token] + $ pictl votestart [token] ## Voting on a proposal @@ -171,16 +171,16 @@ Voting on a proposal can be done using the `politeiavoter` tool. [politeiavoter](https://github.com/decred/politeia/tree/master/politeiawww/cmd/politeiavoter/) -### piwww +### pictl -You can also vote on proposals using the `piwww voteballot` command. This casts +You can also vote on proposals using the `pictl voteballot` command. This casts a ballot of votes. This will only work on testnet and if you are running your dcrwallet locally using the default port. - $ piwww voteballot [token] [voteID] + $ pictl voteballot [token] [voteID] # Reference implementation -The piwww `testrun` command runs a series of tests on all of the politeiawww pi +The pictl `testrun` command runs a series of tests on all of the politeiawww pi API routes. This command can be used as a reference implementation for the pi API. diff --git a/politeiawww/cmd/piwww/castballot.go b/politeiawww/cmd/pictl/castballot.go similarity index 94% rename from politeiawww/cmd/piwww/castballot.go rename to politeiawww/cmd/pictl/castballot.go index c49bed47b..91011b923 100644 --- a/politeiawww/cmd/piwww/castballot.go +++ b/politeiawww/cmd/pictl/castballot.go @@ -4,21 +4,6 @@ package main -import ( - "bytes" - "encoding/hex" - "fmt" - "os" - "strconv" - - "decred.org/dcrwallet/rpc/walletrpc" - "github.com/decred/dcrd/chaincfg/chainhash" - "github.com/decred/politeia/politeiad/api/v1/identity" - pi "github.com/decred/politeia/politeiawww/api/pi/v1" - "github.com/decred/politeia/util" - "golang.org/x/crypto/ssh/terminal" -) - // castBallotCmd casts a ballot of votes for the specified proposal. type castBallotCmd struct { Args struct { @@ -28,6 +13,7 @@ type castBallotCmd struct { Password string `long:"password" optional:"true"` } +/* // Execute executes the castBallotCmd command. // // This function satisfies the go-flags Commander interface. @@ -212,6 +198,7 @@ func (c *castBallotCmd) Execute(args []string) error { return nil } +*/ // castBallotHelpMsg is the help command message. const castBallotHelpMsg = `castballot "token" "voteid" diff --git a/politeiawww/cmd/piwww/commentcensor.go b/politeiawww/cmd/pictl/commentcensor.go similarity index 93% rename from politeiawww/cmd/piwww/commentcensor.go rename to politeiawww/cmd/pictl/commentcensor.go index 3db0cd45e..39668c6fb 100644 --- a/politeiawww/cmd/piwww/commentcensor.go +++ b/politeiawww/cmd/pictl/commentcensor.go @@ -4,16 +4,6 @@ package main -import ( - "encoding/hex" - "fmt" - "strconv" - - pi "github.com/decred/politeia/politeiawww/api/pi/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" - "github.com/decred/politeia/util" -) - // commentCensorCmd censors a proposal comment. type commentCensorCmd struct { Args struct { @@ -26,6 +16,7 @@ type commentCensorCmd struct { Unvetted bool `long:"unvetted" optional:"true"` } +/* // Execute executes the commentCensorCmd command. // // This function satisfies the go-flags Commander interface. @@ -102,6 +93,7 @@ func (cmd *commentCensorCmd) Execute(args []string) error { return nil } +*/ // commentCensorHelpMsg is the help command message. const commentCensorHelpMsg = `commentcensor "token" "commentID" "reason" diff --git a/politeiawww/cmd/piwww/commentnew.go b/politeiawww/cmd/pictl/commentnew.go similarity index 94% rename from politeiawww/cmd/piwww/commentnew.go rename to politeiawww/cmd/pictl/commentnew.go index a04b024b3..abf16334c 100644 --- a/politeiawww/cmd/piwww/commentnew.go +++ b/politeiawww/cmd/pictl/commentnew.go @@ -4,16 +4,6 @@ package main -import ( - "encoding/hex" - "fmt" - "strconv" - - pi "github.com/decred/politeia/politeiawww/api/pi/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" - "github.com/decred/politeia/util" -) - // commentNewCmd submits a new proposal comment. type commentNewCmd struct { Args struct { @@ -26,6 +16,7 @@ type commentNewCmd struct { Unvetted bool `long:"unvetted" optional:"true"` } +/* // Execute executes the commentNewCmd command. // // This function satisfies the go-flags Commander interface. @@ -110,6 +101,7 @@ func (c *commentNewCmd) Execute(args []string) error { return nil } +*/ // commentNewHelpMsg is the help command message. const commentNewHelpMsg = `commentnew "token" "comment" "parentid" diff --git a/politeiawww/cmd/piwww/comments.go b/politeiawww/cmd/pictl/comments.go similarity index 92% rename from politeiawww/cmd/piwww/comments.go rename to politeiawww/cmd/pictl/comments.go index 57e61589f..fdea24942 100644 --- a/politeiawww/cmd/piwww/comments.go +++ b/politeiawww/cmd/pictl/comments.go @@ -4,11 +4,6 @@ package main -import ( - pi "github.com/decred/politeia/politeiawww/api/pi/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" -) - // commentsCmd retreives the comments for the specified proposal. type commentsCmd struct { Args struct { @@ -19,6 +14,7 @@ type commentsCmd struct { Unvetted bool `long:"unvetted" optional:"true"` } +/* // Execute executes the commentsCmd command. // // This function satisfies the go-flags Commander interface. @@ -45,6 +41,7 @@ func (cmd *commentsCmd) Execute(args []string) error { return shared.PrintJSON(gcr) } +*/ // commentsHelpMsg is the output for the help command when 'comments' // is specified. diff --git a/politeiawww/cmd/piwww/commenttimestamps.go b/politeiawww/cmd/pictl/commenttimestamps.go similarity index 92% rename from politeiawww/cmd/piwww/commenttimestamps.go rename to politeiawww/cmd/pictl/commenttimestamps.go index 32e7eba4a..823f0238a 100644 --- a/politeiawww/cmd/piwww/commenttimestamps.go +++ b/politeiawww/cmd/pictl/commenttimestamps.go @@ -4,15 +4,6 @@ package main -import ( - "fmt" - - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe" - cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" -) - // commentTimestampsCmd retrieves the timestamps for politeiawww comments. type commentTimestampsCmd struct { Args struct { @@ -25,6 +16,7 @@ type commentTimestampsCmd struct { Unvetted bool `long:"unvetted" optional:"true"` } +/* // Execute executes the commentTimestampsCmd command. // // This function satisfies the go-flags Commander interface. @@ -103,6 +95,7 @@ func convertCommentTimestamp(t cmv1.Timestamp) backend.Timestamp { Proofs: proofs, } } +*/ const commentTimestampsHelpMsg = `commenttimestamps [flags] "token" commentIDs diff --git a/politeiawww/cmd/piwww/commentvote.go b/politeiawww/cmd/pictl/commentvote.go similarity index 93% rename from politeiawww/cmd/piwww/commentvote.go rename to politeiawww/cmd/pictl/commentvote.go index 9b57417f1..117896ed1 100644 --- a/politeiawww/cmd/piwww/commentvote.go +++ b/politeiawww/cmd/pictl/commentvote.go @@ -4,16 +4,6 @@ package main -import ( - "encoding/hex" - "fmt" - "strconv" - - pi "github.com/decred/politeia/politeiawww/api/pi/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" - "github.com/decred/politeia/util" -) - // commentVoteCmd is used to upvote/downvote a proposal comment using the // logged in the user. type commentVoteCmd struct { @@ -24,6 +14,7 @@ type commentVoteCmd struct { } `positional-args:"true" required:"true"` } +/* // Execute executes the commentVoteCmd command. // // This function satisfies the go-flags Commander interface. @@ -102,6 +93,7 @@ func (c *commentVoteCmd) Execute(args []string) error { return nil } +*/ // commentVoteHelpMsg is the help command message. const commentVoteHelpMsg = `commentvote "token" "commentID" "vote" diff --git a/politeiawww/cmd/piwww/commentvotes.go b/politeiawww/cmd/pictl/commentvotes.go similarity index 92% rename from politeiawww/cmd/piwww/commentvotes.go rename to politeiawww/cmd/pictl/commentvotes.go index 09a6f1a9e..c86d5f29f 100644 --- a/politeiawww/cmd/piwww/commentvotes.go +++ b/politeiawww/cmd/pictl/commentvotes.go @@ -4,13 +4,6 @@ package main -import ( - "fmt" - - pi "github.com/decred/politeia/politeiawww/api/pi/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" -) - // commentVotesCmd retreives like comment objects for // the specified proposal from the provided user. type commentVotesCmd struct { @@ -21,6 +14,7 @@ type commentVotesCmd struct { Me bool `long:"me" optional:"true"` } +/* // Execute executes the commentVotesCmd command. // // This function satisfies the go-flags Commander interface. @@ -52,6 +46,7 @@ func (c *commentVotesCmd) Execute(args []string) error { } return shared.PrintJSON(cvr) } +*/ // commentVotesHelpMsg is the output for the help command when // 'commentvotes' is specified. diff --git a/politeiawww/cmd/piwww/help.go b/politeiawww/cmd/pictl/help.go similarity index 100% rename from politeiawww/cmd/piwww/help.go rename to politeiawww/cmd/pictl/help.go diff --git a/politeiawww/cmd/piwww/piwww.go b/politeiawww/cmd/pictl/pictl.go similarity index 88% rename from politeiawww/cmd/piwww/piwww.go rename to politeiawww/cmd/pictl/pictl.go index 3c91d3514..078a66290 100644 --- a/politeiawww/cmd/piwww/piwww.go +++ b/politeiawww/cmd/pictl/pictl.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2020 The Decred developers +// Copyright (c) 2017-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -17,13 +17,13 @@ import ( const ( // Config settings - defaultHomeDirname = "piwww" + defaultHomeDirname = "pictl" defaultDataDirname = "data" - defaultConfigFilename = "piwww.conf" + defaultConfigFilename = "pictl.conf" ) var ( - // Global variables for piwww commands + // Global variables for pictl commands cfg *shared.Config client *shared.Client @@ -31,7 +31,7 @@ var ( defaultHomeDir = dcrutil.AppDataDir(defaultHomeDirname, false) ) -type piwww struct { +type pictl struct { // This is here to prevent parsing errors caused by config flags. Config shared.Config @@ -41,7 +41,6 @@ type piwww struct { // Server commands Version shared.VersionCmd `command:"version"` Policy policyCmd `command:"policy"` - Secret shared.SecretCmd `command:"secret"` Login shared.LoginCmd `command:"login"` Logout shared.LogoutCmd `command:"logout"` Me shared.MeCmd `command:"me"` @@ -66,14 +65,12 @@ type piwww struct { // TODO replace www policies with pi policies // Proposal commands - ProposalNew proposalNewCmd `command:"proposalnew"` - ProposalEdit proposalEditCmd `command:"proposaledit"` - ProposalSetStatus proposalSetStatusCmd `command:"proposalsetstatus"` - Proposals proposalsCmd `command:"proposals"` - ProposalInv proposalInvCmd `command:"proposalinv"` - - // Record commands - RecordTimestamps recordTimestampsCmd `command:"recordtimestamps"` + ProposalNew proposalNewCmd `command:"proposalnew"` + ProposalEdit proposalEditCmd `command:"proposaledit"` + ProposalSetStatus proposalSetStatusCmd `command:"proposalsetstatus"` + Proposals proposalsCmd `command:"proposals"` + ProposalInv proposalInvCmd `command:"proposalinv"` + proposalTimestamps proposalTimestampsCmd `command:"proposaltimestamps"` // Comments commands CommentNew commentNewCmd `command:"commentnew"` @@ -192,7 +189,7 @@ func _main() error { shared.SetClient(client) // Check for a help flag. This is done separately so that we can - // print our own custom help message + // print our own custom help message. var opts flags.Options = flags.HelpFlag | flags.IgnoreUnknown | flags.PassDoubleDash parser := flags.NewParser(&struct{}{}, opts) @@ -200,24 +197,21 @@ func _main() error { if err != nil { var flagsErr *flags.Error if errors.As(err, &flagsErr) && flagsErr.Type == flags.ErrHelp { + // The -h, --help flag was used. Print the custom help message + // and exit gracefully. fmt.Printf("%v\n", helpMsg) - return nil + os.Exit(0) } return fmt.Errorf("parse help flag: %v", err) } - // Get politeiawww CSRF token - if cfg.CSRF == "" { - _, err := client.Version() - if err != nil { - return fmt.Errorf("Version: %v", err) - } - } - - // Parse subcommand and execute - parser = flags.NewParser(&piwww{Config: *cfg}, flags.Default) + // Parse CLI args and execute command + parser = flags.NewParser(&pictl{Config: *cfg}, flags.Default) _, err = parser.Parse() if err != nil { + // An error has occurred during command execution. go-flags will + // have already printed the error to os.Stdout. Exit with an + // error code. os.Exit(1) } diff --git a/politeiawww/cmd/piwww/policy.go b/politeiawww/cmd/pictl/policy.go similarity index 100% rename from politeiawww/cmd/piwww/policy.go rename to politeiawww/cmd/pictl/policy.go diff --git a/politeiawww/cmd/piwww/proposaledit.go b/politeiawww/cmd/pictl/proposaledit.go similarity index 95% rename from politeiawww/cmd/piwww/proposaledit.go rename to politeiawww/cmd/pictl/proposaledit.go index e47c025b2..d4cef14f7 100644 --- a/politeiawww/cmd/piwww/proposaledit.go +++ b/politeiawww/cmd/pictl/proposaledit.go @@ -4,22 +4,6 @@ package main -import ( - "encoding/base64" - "encoding/hex" - "encoding/json" - "fmt" - "io/ioutil" - "path/filepath" - "time" - - "github.com/decred/politeia/politeiad/api/v1/mime" - pi "github.com/decred/politeia/politeiawww/api/pi/v1" - v1 "github.com/decred/politeia/politeiawww/api/www/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" - "github.com/decred/politeia/util" -) - // proposalEditCmd edits an existing proposal. type proposalEditCmd struct { Args struct { @@ -50,6 +34,7 @@ type proposalEditCmd struct { UseMD bool `long:"usemd" optional:"true"` } +/* // Execute executes the proposalEditCmd command. // // This function satisfies the go-flags Commander interface. @@ -225,6 +210,7 @@ func (cmd *proposalEditCmd) Execute(args []string) error { return nil } +*/ // proposalEditHelpMsg is the output of the help command. const proposalEditHelpMsg = `editproposal [flags] "token" "indexfile" "attachments" diff --git a/politeiawww/cmd/piwww/proposalinv.go b/politeiawww/cmd/pictl/proposalinv.go similarity index 91% rename from politeiawww/cmd/piwww/proposalinv.go rename to politeiawww/cmd/pictl/proposalinv.go index 52610f229..01d96eee0 100644 --- a/politeiawww/cmd/piwww/proposalinv.go +++ b/politeiawww/cmd/pictl/proposalinv.go @@ -4,11 +4,6 @@ package main -import ( - pi "github.com/decred/politeia/politeiawww/api/pi/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" -) - // proposalInvCmd retrieves the censorship record tokens of all proposals in // the inventory that match the provided filtering criteria. If no filtering // criteria is given then the full inventory is returned. @@ -16,6 +11,7 @@ type proposalInvCmd struct { UserID string `long:"userid" optional:"true"` } +/* // Execute executes the proposalInvCmd command. // // This function satisfies the go-flags Commander interface. @@ -37,6 +33,7 @@ func (c *proposalInvCmd) Execute(args []string) error { } return nil } +*/ // proposalInvHelpMsg is the command help message. const proposalInvHelpMsg = `proposalinv diff --git a/politeiawww/cmd/piwww/proposalnew.go b/politeiawww/cmd/pictl/proposalnew.go similarity index 95% rename from politeiawww/cmd/piwww/proposalnew.go rename to politeiawww/cmd/pictl/proposalnew.go index 1da21b819..8b4208939 100644 --- a/politeiawww/cmd/piwww/proposalnew.go +++ b/politeiawww/cmd/pictl/proposalnew.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2020 The Decred developers +// Copyright (c) 2017-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -14,7 +14,7 @@ import ( "time" "github.com/decred/politeia/politeiad/api/v1/mime" - pi "github.com/decred/politeia/politeiawww/api/pi/v1" + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" v1 "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/cmd/shared" "github.com/decred/politeia/util" @@ -27,7 +27,7 @@ type proposalNewCmd struct { Attachments []string `positional-arg-name:"attachments"` } `positional-args:"true" optional:"true"` - // CLI flags + // Metadata fields that can be set by the user Name string `long:"name" optional:"true"` LinkTo string `long:"linkto" optional:"true"` LinkBy int64 `long:"linkby" optional:"true"` @@ -38,8 +38,8 @@ type proposalNewCmd struct { // RFP is a flag that is intended to make submitting an RFP easier // by calculating and inserting a linkby timestamp automatically - // instead of having to pass in a specific timestamp using the - // --linkby flag. + // instead of having to pass in a timestamp using the --linkby + // flag. RFP bool `long:"rfp" optional:"true"` } @@ -47,10 +47,11 @@ type proposalNewCmd struct { // // This function satisfies the go-flags Commander interface. func (cmd *proposalNewCmd) Execute(args []string) error { + // Unpack args indexFile := cmd.Args.IndexFile attachments := cmd.Args.Attachments - // Validate arguments + // Verify args switch { case !cmd.Random && indexFile == "": return fmt.Errorf("index file not found; you must either provide an " + @@ -75,9 +76,11 @@ func (cmd *proposalNewCmd) Execute(args []string) error { return shared.ErrUserIdentityNotFound } + // Get pi policy + // Prepare index file var ( - file *pi.File + file *rcv1.File err error ) if cmd.Random { diff --git a/politeiawww/cmd/piwww/proposals.go b/politeiawww/cmd/pictl/proposals.go similarity index 95% rename from politeiawww/cmd/piwww/proposals.go rename to politeiawww/cmd/pictl/proposals.go index 9ddbae987..257e9f803 100644 --- a/politeiawww/cmd/piwww/proposals.go +++ b/politeiawww/cmd/pictl/proposals.go @@ -4,14 +4,6 @@ package main -import ( - "fmt" - "strings" - - pi "github.com/decred/politeia/politeiawww/api/pi/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" -) - // proposalsCmd retrieves the proposal records of the requested tokens and // versions. type proposalsCmd struct { @@ -26,6 +18,7 @@ type proposalsCmd struct { IncludeFiles bool `long:"includefiles" optional:"true"` } +/* // Execute executes the proposalsCmd command. // // This function satisfies the go-flags Commander interface. @@ -100,6 +93,7 @@ func (c *proposalsCmd) Execute(args []string) error { return nil } +*/ // proposalsHelpMsg is the output of the help command. const proposalsHelpMsg = `proposals [flags] "proposals" diff --git a/politeiawww/cmd/piwww/proposalstatusset.go b/politeiawww/cmd/pictl/proposalstatusset.go similarity index 96% rename from politeiawww/cmd/piwww/proposalstatusset.go rename to politeiawww/cmd/pictl/proposalstatusset.go index 4dc4470bc..f4d913b20 100644 --- a/politeiawww/cmd/piwww/proposalstatusset.go +++ b/politeiawww/cmd/pictl/proposalstatusset.go @@ -4,15 +4,6 @@ package main -import ( - "encoding/hex" - "fmt" - "strconv" - - pi "github.com/decred/politeia/politeiawww/api/pi/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" -) - // proposalSetStatusCmd sets the status of a proposal. type proposalSetStatusCmd struct { Args struct { @@ -25,6 +16,7 @@ type proposalSetStatusCmd struct { Unvetted bool `long:"unvetted" optional:"true"` } +/* // Execute executes the proposalSetStatusCmd command. // // This function satisfies the go-flags Commander interface. @@ -121,6 +113,7 @@ func (cmd *proposalSetStatusCmd) Execute(args []string) error { return nil } +*/ // proposalSetStatusHelpMsg is the output of the help command. const proposalSetStatusHelpMsg = `proposalstatusset "token" "status" "reason" diff --git a/politeiawww/cmd/piwww/recordtimestamps.go b/politeiawww/cmd/pictl/proposaltimestamps.go similarity index 69% rename from politeiawww/cmd/piwww/recordtimestamps.go rename to politeiawww/cmd/pictl/proposaltimestamps.go index a3a6de8a5..5c82a127d 100644 --- a/politeiawww/cmd/piwww/recordtimestamps.go +++ b/politeiawww/cmd/pictl/proposaltimestamps.go @@ -5,32 +5,30 @@ package main import ( - "fmt" - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe" + "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" ) -// recordTimestampsCmd retrieves the timestamps for a politeiawww record. -type recordTimestampsCmd struct { +// proposalTimestampsCmd retrieves the timestamps for a politeiawww proposal. +type proposalTimestampsCmd struct { Args struct { Token string `positional-arg-name:"token" required:"true"` Version string `positional-arg-name:"version" optional:"true"` } `positional-args:"true"` // Unvetted is used to request the timestamps of an unvetted - // record. + // proposal. Unvetted bool `long:"unvetted" optional:"true"` } -// Execute executes the recordTimestampsCmd command. +/* +// Execute executes the proposalTimestampsCmd command. // // This function satisfies the go-flags Commander interface. -func (c *recordTimestampsCmd) Execute(args []string) error { +func (c *proposalTimestampsCmd) Execute(args []string) error { - // Set record state. Defaults to vetted unless the unvetted flag + // Set proposal state. Defaults to vetted unless the unvetted flag // is used. var state rcv1.StateT switch { @@ -64,7 +62,7 @@ func (c *recordTimestampsCmd) Execute(args []string) error { // Verify timestamps err = verifyTimestamp(tr.RecordMetadata) if err != nil { - return fmt.Errorf("verify record metadata timestamp: %v", err) + return fmt.Errorf("verify proposal metadata timestamp: %v", err) } for k, v := range tr.Metadata { err = verifyTimestamp(v) @@ -81,10 +79,11 @@ func (c *recordTimestampsCmd) Execute(args []string) error { return nil } +*/ func verifyTimestamp(t rcv1.Timestamp) error { ts := convertTimestamp(t) - return tlogbe.VerifyTimestamp(ts) + return tlog.VerifyTimestamp(ts) } func convertProof(p rcv1.Proof) backend.Proof { @@ -111,17 +110,17 @@ func convertTimestamp(t rcv1.Timestamp) backend.Timestamp { } } -const recordTimestampsHelpMsg = `recordtimestamps [flags] "token" "version" +const proposalTimestampsHelpMsg = `proposaltimestamps [flags] "token" "version" -Fetch the timestamps a record version. The timestamp contains all necessary -data to verify that user submitted record data has been timestamped onto the +Fetch the timestamps a proposal version. The timestamp contains all necessary +data to verify that user submitted proposal data has been timestamped onto the decred blockchain. Arguments: -1. token (string, required) Record token -2. version (string, optional) Record version +1. token (string, required) Record token +2. version (string, optional) Record version Flags: - --unvetted (bool, optional) Request is for unvetted records instead of vetted - ones (default: false). + --unvetted (bool, optional) Request is for unvetted proposals instead of + vetted ones (default: false). ` diff --git a/politeiawww/cmd/piwww/sample-piwww.conf b/politeiawww/cmd/pictl/sample-piwww.conf similarity index 100% rename from politeiawww/cmd/piwww/sample-piwww.conf rename to politeiawww/cmd/pictl/sample-piwww.conf diff --git a/politeiawww/cmd/piwww/sendfaucettx.go b/politeiawww/cmd/pictl/sendfaucettx.go similarity index 100% rename from politeiawww/cmd/piwww/sendfaucettx.go rename to politeiawww/cmd/pictl/sendfaucettx.go diff --git a/politeiawww/cmd/piwww/subscribe.go b/politeiawww/cmd/pictl/subscribe.go similarity index 100% rename from politeiawww/cmd/piwww/subscribe.go rename to politeiawww/cmd/pictl/subscribe.go diff --git a/politeiawww/cmd/piwww/testrun.go b/politeiawww/cmd/pictl/testrun.go similarity index 98% rename from politeiawww/cmd/piwww/testrun.go rename to politeiawww/cmd/pictl/testrun.go index afcbc0807..4051be688 100644 --- a/politeiawww/cmd/piwww/testrun.go +++ b/politeiawww/cmd/pictl/testrun.go @@ -4,22 +4,6 @@ package main -import ( - "context" - "encoding/base64" - "encoding/hex" - "encoding/json" - "fmt" - "strconv" - "time" - - "github.com/decred/politeia/politeiad/api/v1/identity" - pi "github.com/decred/politeia/politeiawww/api/pi/v1" - www "github.com/decred/politeia/politeiawww/api/www/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" - "github.com/decred/politeia/util" -) - // testRunCmd performs a test run of all the politeiawww routes. type testRunCmd struct { Args struct { @@ -28,6 +12,7 @@ type testRunCmd struct { } `positional-args:"true" required:"true"` } +/* var ( minPasswordLength int publicKey string @@ -1497,6 +1482,7 @@ func (cmd *testRunCmd) Execute(args []string) error { return nil } +*/ const testRunHelpMsg = `testrun "adminusername" "adminpassword" diff --git a/politeiawww/cmd/piwww/userdetails.go b/politeiawww/cmd/pictl/userdetails.go similarity index 100% rename from politeiawww/cmd/piwww/userdetails.go rename to politeiawww/cmd/pictl/userdetails.go diff --git a/politeiawww/cmd/piwww/useredit.go b/politeiawww/cmd/pictl/useredit.go similarity index 100% rename from politeiawww/cmd/piwww/useredit.go rename to politeiawww/cmd/pictl/useredit.go diff --git a/politeiawww/cmd/piwww/useremailverify.go b/politeiawww/cmd/pictl/useremailverify.go similarity index 100% rename from politeiawww/cmd/piwww/useremailverify.go rename to politeiawww/cmd/pictl/useremailverify.go diff --git a/politeiawww/cmd/piwww/usernew.go b/politeiawww/cmd/pictl/usernew.go similarity index 100% rename from politeiawww/cmd/piwww/usernew.go rename to politeiawww/cmd/pictl/usernew.go diff --git a/politeiawww/cmd/piwww/userpaymentsrescan.go b/politeiawww/cmd/pictl/userpaymentsrescan.go similarity index 100% rename from politeiawww/cmd/piwww/userpaymentsrescan.go rename to politeiawww/cmd/pictl/userpaymentsrescan.go diff --git a/politeiawww/cmd/piwww/userproposalcredits.go b/politeiawww/cmd/pictl/userproposalcredits.go similarity index 100% rename from politeiawww/cmd/piwww/userproposalcredits.go rename to politeiawww/cmd/pictl/userproposalcredits.go diff --git a/politeiawww/cmd/piwww/userproposalpaywall.go b/politeiawww/cmd/pictl/userproposalpaywall.go similarity index 100% rename from politeiawww/cmd/piwww/userproposalpaywall.go rename to politeiawww/cmd/pictl/userproposalpaywall.go diff --git a/politeiawww/cmd/piwww/userproposalpaywalltx.go b/politeiawww/cmd/pictl/userproposalpaywalltx.go similarity index 100% rename from politeiawww/cmd/piwww/userproposalpaywalltx.go rename to politeiawww/cmd/pictl/userproposalpaywalltx.go diff --git a/politeiawww/cmd/piwww/userregistrationpayment.go b/politeiawww/cmd/pictl/userregistrationpayment.go similarity index 100% rename from politeiawww/cmd/piwww/userregistrationpayment.go rename to politeiawww/cmd/pictl/userregistrationpayment.go diff --git a/politeiawww/cmd/piwww/userverificationresend.go b/politeiawww/cmd/pictl/userverificationresend.go similarity index 100% rename from politeiawww/cmd/piwww/userverificationresend.go rename to politeiawww/cmd/pictl/userverificationresend.go diff --git a/politeiawww/cmd/pictl/util.go b/politeiawww/cmd/pictl/util.go new file mode 100644 index 000000000..7f766dbf5 --- /dev/null +++ b/politeiawww/cmd/pictl/util.go @@ -0,0 +1,148 @@ +// Copyright (c) 2017-2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "encoding/base64" + "encoding/hex" + "fmt" + + "github.com/decred/dcrd/chaincfg/chainhash" + "github.com/decred/politeia/politeiad/api/v1/identity" + "github.com/decred/politeia/politeiad/api/v1/mime" + pi "github.com/decred/politeia/politeiawww/api/pi/v1" + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + v1 "github.com/decred/politeia/politeiawww/api/www/v1" + "github.com/decred/politeia/util" +) + +// signedMerkleRoot calculates the merkle root of the passed in list of files +// and metadata, signs the merkle root with the passed in identity and returns +// the signature. +func signedMerkleRoot(files []rcv1.File, id *identity.FullIdentity) (string, error) { + /* + if len(files) == 0 { + return "", fmt.Errorf("no proposal files found") + } + mr, err := utilwww.MerkleRoot(files, md) + if err != nil { + return "", err + } + sig := id.SignMessage([]byte(mr)) + return hex.EncodeToString(sig[:]), nil + */ + return "", fmt.Errorf("not implemented") +} + +// convertTicketHashes converts a slice of hexadecimal ticket hashes into +// a slice of byte slices. +func convertTicketHashes(h []string) ([][]byte, error) { + hashes := make([][]byte, 0, len(h)) + for _, v := range h { + h, err := chainhash.NewHashFromStr(v) + if err != nil { + return nil, err + } + hashes = append(hashes, h[:]) + } + return hashes, nil +} + +// createMDFile returns a File object that was created using a markdown file +// filled with random text. +func createMDFile() (*pi.File, error) { + var b bytes.Buffer + b.WriteString("This is the proposal title\n") + + for i := 0; i < 10; i++ { + r, err := util.Random(32) + if err != nil { + return nil, err + } + b.WriteString(base64.StdEncoding.EncodeToString(r) + "\n") + } + + return &pi.File{ + Name: v1.PolicyIndexFilename, + MIME: mime.DetectMimeType(b.Bytes()), + Digest: hex.EncodeToString(util.Digest(b.Bytes())), + Payload: base64.StdEncoding.EncodeToString(b.Bytes()), + }, nil +} + +func verifyDigests(files []rcv1.File) error { + // Validate file digests + for _, f := range files { + b, err := base64.StdEncoding.DecodeString(f.Payload) + if err != nil { + return fmt.Errorf("file: %v decode payload err %v", + f.Name, err) + } + digest := util.Digest(b) + d, ok := util.ConvertDigest(f.Digest) + if !ok { + return fmt.Errorf("file: %v invalid digest %v", + f.Name, f.Digest) + } + if !bytes.Equal(digest, d[:]) { + return fmt.Errorf("file: %v digests do not match", + f.Name) + } + } + + return nil +} + +func verifyRecord(r rcv1.Record, serverPubKey string) error { + /* + if len(p.Files) > 0 { + // Verify digests + err := verifyDigests(p.Files, p.Metadata) + if err != nil { + return err + } + // Verify merkle root + mr, err := utilwww.MerkleRoot(p.Files, p.Metadata) + if err != nil { + return err + } + // Check if merkle roots match + if mr != p.CensorshipRecord.Merkle { + return fmt.Errorf("merkle roots do not match") + } + } + + // Verify proposal signature + pid, err := util.IdentityFromString(p.PublicKey) + if err != nil { + return err + } + sig, err := util.ConvertSignature(p.Signature) + if err != nil { + return err + } + if !pid.VerifyMessage([]byte(p.CensorshipRecord.Merkle), sig) { + return fmt.Errorf("invalid proposal signature") + } + + // Verify censorship record signature + id, err := util.IdentityFromString(serverPubKey) + if err != nil { + return err + } + s, err := util.ConvertSignature(p.CensorshipRecord.Signature) + if err != nil { + return err + } + msg := []byte(p.CensorshipRecord.Merkle + p.CensorshipRecord.Token) + if !id.VerifyMessage(msg, s) { + return fmt.Errorf("invalid censorship record signature") + } + + return nil + */ + return fmt.Errorf("not implemented") +} diff --git a/politeiawww/cmd/piwww/voteauthorize.go b/politeiawww/cmd/pictl/voteauthorize.go similarity index 93% rename from politeiawww/cmd/piwww/voteauthorize.go rename to politeiawww/cmd/pictl/voteauthorize.go index 6a4bed345..c2a7b6db0 100644 --- a/politeiawww/cmd/piwww/voteauthorize.go +++ b/politeiawww/cmd/pictl/voteauthorize.go @@ -4,16 +4,6 @@ package main -import ( - "encoding/hex" - "fmt" - "strconv" - - pi "github.com/decred/politeia/politeiawww/api/pi/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" - "github.com/decred/politeia/util" -) - // voteAuthorizeCmd authorizes a proposal vote or revokes a previous vote // authorization. type voteAuthorizeCmd struct { @@ -23,6 +13,7 @@ type voteAuthorizeCmd struct { } `positional-args:"true"` } +/* // Execute executes the voteAuthorizeCmd command. // // This function satisfies the go-flags Commander interface. @@ -105,6 +96,7 @@ func (cmd *voteAuthorizeCmd) Execute(args []string) error { return nil } +*/ // voteAuthorizeHelpMsg is the help command message. const voteAuthorizeHelpMsg = `voteauthorize "token" "action" diff --git a/politeiawww/cmd/piwww/voteinventory.go b/politeiawww/cmd/pictl/voteinventory.go similarity index 90% rename from politeiawww/cmd/piwww/voteinventory.go rename to politeiawww/cmd/pictl/voteinventory.go index 9da53720b..9338a31fe 100644 --- a/politeiawww/cmd/piwww/voteinventory.go +++ b/politeiawww/cmd/pictl/voteinventory.go @@ -4,15 +4,11 @@ package main -import ( - pi "github.com/decred/politeia/politeiawww/api/pi/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" -) - // voteInventoryCmd retrieves the censorship record tokens of all public, // non-abandoned proposals in inventory categorized by their vote status. type voteInventoryCmd struct{} +/* // Execute executes the voteInventoryCmd command. // // This function satisfies the go-flags Commander interface. @@ -37,6 +33,7 @@ func (cmd *voteInventoryCmd) Execute(args []string) error { return nil } +*/ // voteInventoryHelpMsg is the command help message. const voteInventoryHelpMsg = `voteinv diff --git a/politeiawww/cmd/piwww/voteresults.go b/politeiawww/cmd/pictl/voteresults.go similarity index 90% rename from politeiawww/cmd/piwww/voteresults.go rename to politeiawww/cmd/pictl/voteresults.go index d3ae0fa68..5dcbe052a 100644 --- a/politeiawww/cmd/piwww/voteresults.go +++ b/politeiawww/cmd/pictl/voteresults.go @@ -4,11 +4,6 @@ package main -import ( - pi "github.com/decred/politeia/politeiawww/api/pi/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" -) - // voteResultsCmd retreives the cast votes for the provided proposal. type voteResultsCmd struct { Args struct { @@ -16,6 +11,7 @@ type voteResultsCmd struct { } `positional-args:"true" required:"true"` } +/* // Execute executes the voteResultsCmd command. // // This function satisfies the go-flags Commander interface. @@ -42,6 +38,7 @@ func (cmd *voteResultsCmd) Execute(args []string) error { return nil } +*/ // voteResultsHelpMsg is the help command message. const voteResultsHelpMsg = `voteresults "token" diff --git a/politeiawww/cmd/piwww/votes.go b/politeiawww/cmd/pictl/votes.go similarity index 92% rename from politeiawww/cmd/piwww/votes.go rename to politeiawww/cmd/pictl/votes.go index 221c8eb4e..7ea7908c2 100644 --- a/politeiawww/cmd/piwww/votes.go +++ b/politeiawww/cmd/pictl/votes.go @@ -4,11 +4,6 @@ package main -import ( - pi "github.com/decred/politeia/politeiawww/api/pi/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" -) - // votesCmd retrieves vote details for the specified proposals. type votesCmd struct { Args struct { @@ -16,6 +11,7 @@ type votesCmd struct { } `positional-args:"true" required:"true"` } +/* // Execute executes the votesCmd command. // // This function satisfies the go-flags Commander interface. @@ -55,6 +51,7 @@ func (c *votesCmd) Execute(args []string) error { return nil } +*/ // votesHelpMsg is the help command message. const votesHelpMsg = `votes "tokens" diff --git a/politeiawww/cmd/piwww/votestart.go b/politeiawww/cmd/pictl/votestart.go similarity index 94% rename from politeiawww/cmd/piwww/votestart.go rename to politeiawww/cmd/pictl/votestart.go index d824ae30f..631a4f473 100644 --- a/politeiawww/cmd/piwww/votestart.go +++ b/politeiawww/cmd/pictl/votestart.go @@ -4,17 +4,6 @@ package main -import ( - "encoding/hex" - "encoding/json" - "fmt" - "strconv" - - pi "github.com/decred/politeia/politeiawww/api/pi/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" - "github.com/decred/politeia/util" -) - // voteStartCmd starts the voting period on the specified proposal. type voteStartCmd struct { Args struct { @@ -25,6 +14,7 @@ type voteStartCmd struct { } `positional-args:"true"` } +/* // Execute executes the voteStartCmd command. // // This function satisfies the go-flags Commander interface. @@ -120,6 +110,7 @@ func (cmd *voteStartCmd) Execute(args []string) error { return nil } +*/ // voteStartHelpMsg is the help command message. var voteStartHelpMsg = `votestart diff --git a/politeiawww/cmd/piwww/votestartrunoff.go b/politeiawww/cmd/pictl/votestartrunoff.go similarity index 93% rename from politeiawww/cmd/piwww/votestartrunoff.go rename to politeiawww/cmd/pictl/votestartrunoff.go index fb506e832..e1e55b3e9 100644 --- a/politeiawww/cmd/piwww/votestartrunoff.go +++ b/politeiawww/cmd/pictl/votestartrunoff.go @@ -4,18 +4,6 @@ package main -import ( - "encoding/hex" - "encoding/json" - "strconv" - - "github.com/decred/politeia/politeiad/plugins/ticketvote" - pi "github.com/decred/politeia/politeiawww/api/pi/v1" - v1 "github.com/decred/politeia/politeiawww/api/www/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" - "github.com/decred/politeia/util" -) - // voteStartRunoffCmd starts the voting period on all public submissions to a // request for proposals (RFP). // @@ -31,6 +19,7 @@ type voteStartRunoffCmd struct { } `positional-args:"true"` } +/* // Execute executes the voteStartRunoffCmd command. // // This function satisfies the go-flags Commander interface. @@ -155,6 +144,7 @@ func (cmd *voteStartRunoffCmd) Execute(args []string) error { return nil } +*/ // voteStartRunoffHelpMsg is the help command output for 'votestartrunoff'. var voteStartRunoffHelpMsg = `votestartrunoff diff --git a/politeiawww/cmd/piwww/votesummaries.go b/politeiawww/cmd/pictl/votesummaries.go similarity index 90% rename from politeiawww/cmd/piwww/votesummaries.go rename to politeiawww/cmd/pictl/votesummaries.go index 88b654d0a..16d6d868e 100644 --- a/politeiawww/cmd/piwww/votesummaries.go +++ b/politeiawww/cmd/pictl/votesummaries.go @@ -4,11 +4,6 @@ package main -import ( - pi "github.com/decred/politeia/politeiawww/api/pi/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" -) - // voteSummariesCmd retrieves the vote summaries for the provided proposals. type voteSummariesCmd struct { Args struct { @@ -16,6 +11,7 @@ type voteSummariesCmd struct { } `positional-args:"true" required:"true"` } +/* // Execute executes the voteSummariesCmd command. // // This function satisfies the go-flags Commander interface. @@ -42,6 +38,7 @@ func (cmd *voteSummariesCmd) Execute(args []string) error { return nil } +*/ // voteSummariesHelpMsg is the help command message. const voteSummariesHelpMsg = `votesummaries "tokens" diff --git a/politeiawww/cmd/piwww/votetimestamps.go b/politeiawww/cmd/pictl/votetimestamps.go similarity index 96% rename from politeiawww/cmd/piwww/votetimestamps.go rename to politeiawww/cmd/pictl/votetimestamps.go index 2f3a517ef..9d6656bba 100644 --- a/politeiawww/cmd/piwww/votetimestamps.go +++ b/politeiawww/cmd/pictl/votetimestamps.go @@ -8,7 +8,7 @@ import ( "fmt" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe" + "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" "github.com/decred/politeia/politeiawww/cmd/shared" ) @@ -66,7 +66,7 @@ func (c *voteTimestampsCmd) Execute(args []string) error { func verifyVoteTimestamp(t tkv1.Timestamp) error { ts := convertVoteTimestamp(t) - return tlogbe.VerifyTimestamp(ts) + return tlog.VerifyTimestamp(ts) } func convertVoteProof(p tkv1.Proof) backend.Proof { diff --git a/politeiawww/cmd/piwww/util.go b/politeiawww/cmd/piwww/util.go deleted file mode 100644 index 8bdd355f1..000000000 --- a/politeiawww/cmd/piwww/util.go +++ /dev/null @@ -1,221 +0,0 @@ -// Copyright (c) 2017-2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import ( - "bytes" - "encoding/base64" - "encoding/hex" - "encoding/json" - "fmt" - - "github.com/decred/dcrd/chaincfg/chainhash" - "github.com/decred/politeia/politeiad/api/v1/identity" - "github.com/decred/politeia/politeiad/api/v1/mime" - pi "github.com/decred/politeia/politeiawww/api/pi/v1" - v1 "github.com/decred/politeia/politeiawww/api/www/v1" - utilwww "github.com/decred/politeia/politeiawww/util" - "github.com/decred/politeia/util" -) - -// signedMerkleRoot calculates the merkle root of the passed in list of files -// and metadata, signs the merkle root with the passed in identity and returns -// the signature. -func signedMerkleRoot(files []pi.File, md []pi.Metadata, id *identity.FullIdentity) (string, error) { - if len(files) == 0 { - return "", fmt.Errorf("no proposal files found") - } - mr, err := utilwww.MerkleRoot(files, md) - if err != nil { - return "", err - } - sig := id.SignMessage([]byte(mr)) - return hex.EncodeToString(sig[:]), nil -} - -// convertTicketHashes converts a slice of hexadecimal ticket hashes into -// a slice of byte slices. -func convertTicketHashes(h []string) ([][]byte, error) { - hashes := make([][]byte, 0, len(h)) - for _, v := range h { - h, err := chainhash.NewHashFromStr(v) - if err != nil { - return nil, err - } - hashes = append(hashes, h[:]) - } - return hashes, nil -} - -// proposalRecord returns the ProposalRecord for the provided token and -// version. -func proposalRecord(state pi.PropStateT, token, version string) (*pi.ProposalRecord, error) { - ps := pi.Proposals{ - State: state, - Requests: []pi.ProposalRequest{ - { - Token: token, - Version: version, - }, - }, - IncludeFiles: true, - } - psr, err := client.Proposals(ps) - if err != nil { - return nil, err - } - pr, ok := psr.Proposals[token] - if !ok { - return nil, fmt.Errorf("proposal not found") - } - - return &pr, nil -} - -// createMDFile returns a File object that was created using a markdown file -// filled with random text. -func createMDFile() (*pi.File, error) { - var b bytes.Buffer - b.WriteString("This is the proposal title\n") - - for i := 0; i < 10; i++ { - r, err := util.Random(32) - if err != nil { - return nil, err - } - b.WriteString(base64.StdEncoding.EncodeToString(r) + "\n") - } - - return &pi.File{ - Name: v1.PolicyIndexFilename, - MIME: mime.DetectMimeType(b.Bytes()), - Digest: hex.EncodeToString(util.Digest(b.Bytes())), - Payload: base64.StdEncoding.EncodeToString(b.Bytes()), - }, nil -} - -// proposalRecord returns the latest ProposalRecrord version for the provided -// token. -func proposalRecordLatest(state pi.PropStateT, token string) (*pi.ProposalRecord, error) { - return proposalRecord(state, token, "") -} - -// decodeProposalMetadata decodes and returns a ProposalMetadata given the -// metadata array from a ProposalRecord. -func decodeProposalMetadata(metadata []pi.Metadata) (*pi.ProposalMetadata, error) { - var pm *pi.ProposalMetadata - for _, v := range metadata { - if v.Hint == pi.HintProposalMetadata { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return nil, err - } - pm = &pi.ProposalMetadata{} - err = json.Unmarshal(b, pm) - if err != nil { - return nil, err - } - } - } - if pm == nil { - return nil, fmt.Errorf("proposal metadata not found") - } - return pm, nil -} - -// verifyDigests verifies if the list of files and metadatas have valid -// digests. It compares digests that came with the file/metadata with -// digests calculated from their payload. -func verifyDigests(files []pi.File, md []pi.Metadata) error { - // Validate file digests - for _, f := range files { - b, err := base64.StdEncoding.DecodeString(f.Payload) - if err != nil { - return fmt.Errorf("file: %v decode payload err %v", - f.Name, err) - } - digest := util.Digest(b) - d, ok := util.ConvertDigest(f.Digest) - if !ok { - return fmt.Errorf("file: %v invalid digest %v", - f.Name, f.Digest) - } - if !bytes.Equal(digest, d[:]) { - return fmt.Errorf("file: %v digests do not match", - f.Name) - } - } - - // Validate metadata digests - for _, v := range md { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return fmt.Errorf("metadata: %v decode payload err %v", - v.Hint, err) - } - digest := util.Digest(b) - d, ok := util.ConvertDigest(v.Digest) - if !ok { - return fmt.Errorf("metadata: %v invalid digest %v", - v.Hint, v.Digest) - } - if !bytes.Equal(digest, d[:]) { - return fmt.Errorf("metadata: %v digests do not match metadata", - v.Hint) - } - } - - return nil -} - -// verifyProposal verifies the merkle root, author signature and censorship -// record of a given proposal. -func verifyProposal(p pi.ProposalRecord, serverPubKey string) error { - if len(p.Files) > 0 { - // Verify digests - err := verifyDigests(p.Files, p.Metadata) - if err != nil { - return err - } - // Verify merkle root - mr, err := utilwww.MerkleRoot(p.Files, p.Metadata) - if err != nil { - return err - } - // Check if merkle roots match - if mr != p.CensorshipRecord.Merkle { - return fmt.Errorf("merkle roots do not match") - } - } - - // Verify proposal signature - pid, err := util.IdentityFromString(p.PublicKey) - if err != nil { - return err - } - sig, err := util.ConvertSignature(p.Signature) - if err != nil { - return err - } - if !pid.VerifyMessage([]byte(p.CensorshipRecord.Merkle), sig) { - return fmt.Errorf("invalid proposal signature") - } - - // Verify censorship record signature - id, err := util.IdentityFromString(serverPubKey) - if err != nil { - return err - } - s, err := util.ConvertSignature(p.CensorshipRecord.Signature) - if err != nil { - return err - } - msg := []byte(p.CensorshipRecord.Merkle + p.CensorshipRecord.Token) - if !id.VerifyMessage(msg, s) { - return fmt.Errorf("invalid censorship record signature") - } - - return nil -} diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index 781bda63a..3ddde9038 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -21,8 +21,6 @@ import ( "decred.org/dcrwallet/rpc/walletrpc" cms "github.com/decred/politeia/politeiawww/api/cms/v1" - cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" - pi "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" @@ -69,16 +67,6 @@ func userWWWErrorStatus(e www.ErrorStatusT) string { return "" } -// userPiErrorStatus retrieves the human readable error message for an error -// status code. The status code error message comes from the pi api. -func userPiErrorStatus(e pi.ErrorStatusT) string { - s, ok := pi.ErrorStatus[e] - if ok { - return s - } - return "" -} - // wwwError unmarshals the response body from makeRequest, and handles any // status code errors from the server. Parses the error code and error context // from the www api, in case of user error. @@ -144,47 +132,6 @@ func ticketVoteError(body []byte, statusCode int) error { return fmt.Errorf("%v %s", statusCode, body) } -// piError unmarshals the response body from makeRequest, and handles any -// status code errors from the server. Parses the error code and error context -// from the pi api, in case of user error. -func piError(body []byte, statusCode int) error { - switch statusCode { - case http.StatusBadRequest: - // User Error - var ue pi.UserErrorReply - err := json.Unmarshal(body, &ue) - if err != nil { - return fmt.Errorf("unmarshal UserError: %v", err) - } - if ue.ErrorCode != 0 { - var e error - errMsg := userPiErrorStatus(ue.ErrorCode) - if len(ue.ErrorContext) == 0 { - // Error format when an ErrorContext is not included - e = fmt.Errorf("%v, %v", statusCode, errMsg) - } else { - // Error format when an ErrorContext is included - e = fmt.Errorf("%v, %v: %v", statusCode, errMsg, - strings.Join(ue.ErrorContext, ", ")) - } - return e - } - case http.StatusInternalServerError: - // Server Error - var ser pi.ServerErrorReply - err := json.Unmarshal(body, &ser) - if err != nil { - return fmt.Errorf("unmarshal ServerError: %v", err) - } - return fmt.Errorf("ServerError timestamp: %v", ser.ErrorCode) - default: - // Return Status Code Error - return fmt.Errorf("%v", statusCode) - } - - return nil -} - // makeRequest sends the provided request to the politeiawww backend specified // by the Client config. This function handles verbose printing when specified // by the Client config since verbose printing includes details such as the @@ -857,90 +804,6 @@ func (c *Client) UserProposalPaywall() (*www.UserProposalPaywallReply, error) { return &ppdr, nil } -// ProposalNew submits a new proposal. -func (c *Client) ProposalNew(pn pi.ProposalNew) (*pi.ProposalNewReply, error) { - statusCode, respBody, err := c.makeRequest(http.MethodPost, - pi.APIRoute, pi.RouteProposalNew, pn) - if err != nil { - return nil, err - } - - if statusCode != http.StatusOK { - return nil, piError(respBody, statusCode) - } - - var pnr pi.ProposalNewReply - err = json.Unmarshal(respBody, &pnr) - if err != nil { - return nil, fmt.Errorf("unmarshal ProposalNewReply: %v", err) - } - - if c.cfg.Verbose { - err := prettyPrintJSON(pnr) - if err != nil { - return nil, err - } - } - - return &pnr, nil -} - -// ProposalEdit edits a proposal. -func (c *Client) ProposalEdit(pe pi.ProposalEdit) (*pi.ProposalEditReply, error) { - statusCode, respBody, err := c.makeRequest(http.MethodPost, - pi.APIRoute, pi.RouteProposalEdit, pe) - if err != nil { - return nil, err - } - - if statusCode != http.StatusOK { - return nil, piError(respBody, statusCode) - } - - var per pi.ProposalEditReply - err = json.Unmarshal(respBody, &per) - if err != nil { - return nil, fmt.Errorf("unmarshal ProposalEditReply: %v", err) - } - - if c.cfg.Verbose { - err := prettyPrintJSON(per) - if err != nil { - return nil, err - } - } - - return &per, nil -} - -// ProposalSetStatus sends the ProposalSetStatus command to politeiawww. -func (c *Client) ProposalSetStatus(pss pi.ProposalSetStatus) (*pi.ProposalSetStatusReply, error) { - statusCode, respBody, err := c.makeRequest(http.MethodPost, - pi.APIRoute, pi.RouteProposalSetStatus, pss) - if err != nil { - return nil, err - } - - if statusCode != http.StatusOK { - return nil, piError(respBody, statusCode) - } - - var pssr pi.ProposalSetStatusReply - err = json.Unmarshal(respBody, &pssr) - if err != nil { - return nil, fmt.Errorf("unmarshal ProposalSetStatusReply: %v", err) - } - - if c.cfg.Verbose { - err := prettyPrintJSON(pssr) - if err != nil { - return nil, err - } - } - - return &pssr, nil -} - // RecordTimestamps sends the Timestamps command to politeiawww records API. func (c *Client) RecordTimestamps(t rcv1.Timestamps) (*rcv1.TimestampsReply, error) { statusCode, respBody, err := c.makeRequest(http.MethodPost, @@ -997,62 +860,6 @@ func (c *Client) TicketVoteTimestamps(t tkv1.Timestamps) (*tkv1.TimestampsReply, return &tr, nil } -// Proposals retrieves a proposal for each of the provided proposal requests. -func (c *Client) Proposals(p pi.Proposals) (*pi.ProposalsReply, error) { - statusCode, respBody, err := c.makeRequest(http.MethodPost, - pi.APIRoute, pi.RouteProposals, p) - if err != nil { - return nil, err - } - - if statusCode != http.StatusOK { - return nil, piError(respBody, statusCode) - } - - var pr pi.ProposalsReply - err = json.Unmarshal(respBody, &pr) - if err != nil { - return nil, fmt.Errorf("unmarshal ProposalsReply: %v", err) - } - - if c.cfg.Verbose { - err := prettyPrintJSON(pr) - if err != nil { - return nil, err - } - } - - return &pr, nil -} - -// ProposalInventory sends a ProposalInventory request to politeiawww. -func (c *Client) ProposalInventory(p pi.ProposalInventory) (*pi.ProposalInventoryReply, error) { - statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, - pi.RouteProposalInventory, p) - if err != nil { - return nil, err - } - - if statusCode != http.StatusOK { - return nil, piError(respBody, statusCode) - } - - var pir pi.ProposalInventoryReply - err = json.Unmarshal(respBody, &pir) - if err != nil { - return nil, fmt.Errorf("unmarshal ProposalInventory: %v", err) - } - - if c.cfg.Verbose { - err := prettyPrintJSON(pir) - if err != nil { - return nil, err - } - } - - return &pir, nil -} - // NewInvoice submits the specified invoice to politeiawww for the logged in // user. func (c *Client) NewInvoice(ni *cms.NewInvoice) (*cms.NewInvoiceReply, error) { @@ -1333,35 +1140,6 @@ func (c *Client) PayInvoices(pi *cms.PayInvoices) (*cms.PayInvoicesReply, error) return &pir, nil } -// VoteInventory retrieves the tokens of all proposals in the inventory -// categorized by their vote status. -func (c *Client) VoteInventory(vi pi.VoteInventory) (*pi.VoteInventoryReply, error) { - statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, - pi.RouteVoteInventory, vi) - if err != nil { - return nil, err - } - - if statusCode != http.StatusOK { - return nil, piError(respBody, statusCode) - } - - var vir pi.VoteInventoryReply - err = json.Unmarshal(respBody, &vir) - if err != nil { - return nil, fmt.Errorf("unmarshal VoteInventory: %v", err) - } - - if c.cfg.Verbose { - err := prettyPrintJSON(vir) - if err != nil { - return nil, err - } - } - - return &vir, nil -} - // BatchProposals retrieves a list of proposals func (c *Client) BatchProposals(bp *www.BatchProposals) (*www.BatchProposalsReply, error) { statusCode, respBody, err := c.makeRequest(http.MethodPost, @@ -1390,35 +1168,6 @@ func (c *Client) BatchProposals(bp *www.BatchProposals) (*www.BatchProposalsRepl return &bpr, nil } -// VoteSummaries retrieves a summary of the voting process for a set of -// proposals. -func (c *Client) VoteSummaries(vs pi.VoteSummaries) (*pi.VoteSummariesReply, error) { - statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, - pi.RouteVoteSummaries, vs) - if err != nil { - return nil, err - } - - if statusCode != http.StatusOK { - return nil, piError(respBody, statusCode) - } - - var vsr pi.VoteSummariesReply - err = json.Unmarshal(respBody, &vsr) - if err != nil { - return nil, err - } - - if c.cfg.Verbose { - err := prettyPrintJSON(vsr) - if err != nil { - return nil, err - } - } - - return &vsr, nil -} - // GetAllVetted retrieves a page of vetted proposals. func (c *Client) GetAllVetted(gav *www.GetAllVetted) (*www.GetAllVettedReply, error) { statusCode, respBody, err := c.makeRequest(http.MethodGet, @@ -1475,175 +1224,6 @@ func (c *Client) WWWNewComment(nc *www.NewComment) (*www.NewCommentReply, error) return &ncr, nil } -// CommentNew submits a new proposal comment for the logged in user. -func (c *Client) CommentNew(cn pi.CommentNew) (*pi.CommentNewReply, error) { - statusCode, respBody, err := c.makeRequest(http.MethodPost, - pi.APIRoute, pi.RouteCommentNew, cn) - if err != nil { - return nil, err - } - - if statusCode != http.StatusOK { - return nil, piError(respBody, statusCode) - } - - var cnr pi.CommentNewReply - err = json.Unmarshal(respBody, &cnr) - if err != nil { - return nil, fmt.Errorf("unmarshal CommentNewReply: %v", err) - } - - if c.cfg.Verbose { - err := prettyPrintJSON(cnr) - if err != nil { - return nil, err - } - } - - return &cnr, nil -} - -// CommentVote casts a like comment action (upvote/downvote) for the logged in -// user. -func (c *Client) CommentVote(cv pi.CommentVote) (*pi.CommentVoteReply, error) { - statusCode, respBody, err := c.makeRequest(http.MethodPost, - pi.APIRoute, pi.RouteCommentVote, cv) - if err != nil { - return nil, err - } - - if statusCode != http.StatusOK { - return nil, piError(respBody, statusCode) - } - - var cvr pi.CommentVoteReply - err = json.Unmarshal(respBody, &cvr) - if err != nil { - return nil, fmt.Errorf("unmarshal CommentVoteReply: %v", err) - } - - if c.cfg.Verbose { - err := prettyPrintJSON(cvr) - if err != nil { - return nil, err - } - } - - return &cvr, nil -} - -// CommentCensor censors the specified proposal comment. -func (c *Client) CommentCensor(cc pi.CommentCensor) (*pi.CommentCensorReply, error) { - statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, - pi.RouteCommentCensor, cc) - if err != nil { - return nil, err - } - - if statusCode != http.StatusOK { - return nil, piError(respBody, statusCode) - } - - var ccr pi.CommentCensorReply - err = json.Unmarshal(respBody, &ccr) - if err != nil { - return nil, fmt.Errorf("unmarshal CensorCommentReply: %v", err) - } - - if c.cfg.Verbose { - err := prettyPrintJSON(ccr) - if err != nil { - return nil, err - } - } - - return &ccr, nil -} - -// Comments retrieves the comments for the specified proposal. -func (c *Client) Comments(cs pi.Comments) (*pi.CommentsReply, error) { - statusCode, respBody, err := c.makeRequest(http.MethodPost, - pi.APIRoute, pi.RouteComments, &cs) - if err != nil { - return nil, err - } - - if statusCode != http.StatusOK { - return nil, piError(respBody, statusCode) - } - - var cr pi.CommentsReply - err = json.Unmarshal(respBody, &cr) - if err != nil { - return nil, fmt.Errorf("unmarshal CommentsReply: %v", err) - } - - if c.cfg.Verbose { - err := prettyPrintJSON(cr) - if err != nil { - return nil, err - } - } - - return &cr, nil -} - -// CommentVotes retrieves the comment likes (upvotes/downvotes) for the -// specified proposal that are from the privoded user. -func (c *Client) CommentVotes(cv pi.CommentVotes) (*pi.CommentVotesReply, error) { - statusCode, respBody, err := c.makeRequest(http.MethodPost, - pi.APIRoute, pi.RouteCommentVotes, cv) - if err != nil { - return nil, err - } - - if statusCode != http.StatusOK { - return nil, piError(respBody, statusCode) - } - - var cvr pi.CommentVotesReply - err = json.Unmarshal(respBody, &cvr) - if err != nil { - return nil, fmt.Errorf("unmarshal CommentVotes: %v", err) - } - - if c.cfg.Verbose { - err := prettyPrintJSON(cvr) - if err != nil { - return nil, err - } - } - - return &cvr, nil -} - -// CommentTimestamps sends the Timestamps command to politeiawww comments API. -func (c *Client) CommentTimestamps(t cmv1.Timestamps) (*cmv1.TimestampsReply, error) { - statusCode, respBody, err := c.makeRequest(http.MethodPost, - cmv1.APIRoute, cmv1.RouteTimestamps, t) - if err != nil { - return nil, err - } - if statusCode != http.StatusOK { - return nil, commentsError(respBody, statusCode) - } - - var tr cmv1.TimestampsReply - err = json.Unmarshal(respBody, &tr) - if err != nil { - return nil, err - } - - if c.cfg.Verbose { - err := prettyPrintJSON(tr) - if err != nil { - return nil, err - } - } - - return &tr, nil -} - // InvoiceComments retrieves the comments for the specified proposal. func (c *Client) InvoiceComments(token string) (*www.GetCommentsReply, error) { route := "/invoices/" + token + "/comments" @@ -1673,34 +1253,6 @@ func (c *Client) InvoiceComments(token string) (*www.GetCommentsReply, error) { return &gcr, nil } -// Votes rerieves the vote details for a given proposal. -func (c *Client) Votes(vs pi.Votes) (*pi.VotesReply, error) { - statusCode, respBody, err := c.makeRequest(http.MethodPost, - pi.APIRoute, pi.RouteVotes, vs) - if err != nil { - return nil, err - } - - if statusCode != http.StatusOK { - return nil, piError(respBody, statusCode) - } - - var vsr pi.VotesReply - err = json.Unmarshal(respBody, &vsr) - if err != nil { - return nil, fmt.Errorf("unmarshal Votes: %v", err) - } - - if c.cfg.Verbose { - err := prettyPrintJSON(vsr) - if err != nil { - return nil, err - } - } - - return &vsr, nil -} - // WWWCensorComment censors the specified proposal comment. func (c *Client) WWWCensorComment(cc *www.CensorComment) (*www.CensorCommentReply, error) { statusCode, respBody, err := c.makeRequest(http.MethodPost, @@ -1729,35 +1281,6 @@ func (c *Client) WWWCensorComment(cc *www.CensorComment) (*www.CensorCommentRepl return &ccr, nil } -// VoteStart sends the provided VoteStart to pi. -func (c *Client) VoteStart(vs pi.VoteStart) (*pi.VoteStartReply, error) { - statusCode, respBody, err := c.makeRequest(http.MethodPost, - pi.APIRoute, pi.RouteVoteStart, vs) - if err != nil { - return nil, err - } - - if statusCode != http.StatusOK { - return nil, piError(respBody, statusCode) - } - - var vsr pi.VoteStartReply - err = json.Unmarshal(respBody, &vsr) - if err != nil { - return nil, fmt.Errorf("unmarshal VoteStartReply: %v", err) - } - - if c.cfg.Verbose { - vsr.EligibleTickets = []string{"removed by piwww for readability"} - err := prettyPrintJSON(vsr) - if err != nil { - return nil, err - } - } - - return &vsr, nil -} - // UserRegistrationPayment checks whether the logged in user has paid their user // registration fee. func (c *Client) UserRegistrationPayment() (*www.UserRegistrationPaymentReply, error) { @@ -1788,34 +1311,6 @@ func (c *Client) UserRegistrationPayment() (*www.UserRegistrationPaymentReply, e return &urpr, nil } -// VoteResults retrieves the vote results for the specified proposal. -func (c *Client) VoteResults(vr pi.VoteResults) (*pi.VoteResultsReply, error) { - statusCode, respBody, err := c.makeRequest(http.MethodPost, - pi.APIRoute, pi.RouteVoteResults, vr) - if err != nil { - return nil, err - } - - if statusCode != http.StatusOK { - return nil, piError(respBody, statusCode) - } - - var vrr pi.VoteResultsReply - err = json.Unmarshal(respBody, &vrr) - if err != nil { - return nil, err - } - - if c.cfg.Verbose { - err := prettyPrintJSON(vrr) - if err != nil { - return nil, err - } - } - - return &vrr, nil -} - // VoteDetailsV2 returns the proposal vote details for the given token using // the www v2 VoteDetails route. func (c *Client) VoteDetailsV2(token string) (*www2.VoteDetailsReply, error) { @@ -1992,35 +1487,6 @@ func (c *Client) EditUser(eu *www.EditUser) (*www.EditUserReply, error) { return &eur, nil } -// VoteAuthorize authorizes the voting period for the specified proposal using -// the logged in user. -func (c *Client) VoteAuthorize(va pi.VoteAuthorize) (*pi.VoteAuthorizeReply, error) { - statusCode, respBody, err := c.makeRequest(http.MethodPost, pi.APIRoute, - pi.RouteVoteAuthorize, va) - if err != nil { - return nil, err - } - - if statusCode != http.StatusOK { - return nil, piError(respBody, statusCode) - } - - var vr pi.VoteAuthorizeReply - err = json.Unmarshal(respBody, &vr) - if err != nil { - return nil, fmt.Errorf("unmarshal VoteAuthorizeReply: %v", err) - } - - if c.cfg.Verbose { - err := prettyPrintJSON(vr) - if err != nil { - return nil, err - } - } - - return &vr, nil -} - // VoteStatus retrieves the vote status for the specified proposal. func (c *Client) VoteStatus(token string) (*www.VoteStatusReply, error) { route := "/proposals/" + token + "/votestatus" @@ -2106,34 +1572,6 @@ func (c *Client) ActiveVotesDCC() (*cms.ActiveVoteReply, error) { return &avr, nil } -// CastBallot casts a ballot of votes for a proposal. -func (c *Client) CastBallot(cb pi.CastBallot) (*pi.CastBallotReply, error) { - statusCode, respBody, err := c.makeRequest(http.MethodPost, - pi.APIRoute, pi.RouteCastBallot, cb) - if err != nil { - return nil, err - } - - if statusCode != http.StatusOK { - return nil, piError(respBody, statusCode) - } - - var cbr pi.CastBallotReply - err = json.Unmarshal(respBody, &cbr) - if err != nil { - return nil, fmt.Errorf("unmarshal CastBallotReply: %v", err) - } - - if c.cfg.Verbose { - err := prettyPrintJSON(cbr) - if err != nil { - return nil, err - } - } - - return &cbr, nil -} - // UpdateUserKey updates the identity of the logged in user. func (c *Client) UpdateUserKey(uuk *www.UpdateUserKey) (*www.UpdateUserKeyReply, error) { statusCode, respBody, err := c.makeRequest(http.MethodPost, From 9fd2aa3e4cfe0ad9847fd3cb943f3af92e83c3ed Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 31 Jan 2021 11:32:10 -0600 Subject: [PATCH 258/449] Add pi api policy route. --- politeiad/backend/tlogbe/plugins/pi/hooks.go | 41 +---- politeiad/backend/tlogbe/plugins/pi/pi.go | 169 ++++++++++++++++--- politeiad/backend/tlogbe/plugins/plugins.go | 3 + politeiad/backend/tlogbe/tlog/plugin.go | 12 +- politeiad/plugins/comments/comments.go | 1 + politeiad/plugins/pi/pi.go | 53 +++++- politeiad/plugins/ticketvote/ticketvote.go | 1 + politeiad/politeiad.go | 60 ++++--- politeiawww/api/pi/v1/v1.go | 41 +++-- politeiawww/api/www/v1/v1.go | 8 +- politeiawww/pi/pi.go | 135 ++++++++++++++- politeiawww/pi/process.go | 14 ++ politeiawww/piwww.go | 66 ++++++-- politeiawww/plugin.go | 80 --------- politeiawww/politeiawww.go | 3 +- politeiawww/testing.go | 10 +- politeiawww/www.go | 44 ++++- util/regexp.go | 48 ++++++ 18 files changed, 585 insertions(+), 204 deletions(-) delete mode 100644 politeiawww/plugin.go create mode 100644 util/regexp.go diff --git a/politeiad/backend/tlogbe/plugins/pi/hooks.go b/politeiad/backend/tlogbe/plugins/pi/hooks.go index df42c1e7a..3077fb295 100644 --- a/politeiad/backend/tlogbe/plugins/pi/hooks.go +++ b/politeiad/backend/tlogbe/plugins/pi/hooks.go @@ -5,11 +5,9 @@ package pi import ( - "bytes" "encoding/base64" "encoding/json" "fmt" - "strconv" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" @@ -52,27 +50,6 @@ func proposalMetadataDecode(files []backend.File) (*pi.ProposalMetadata, error) return propMD, nil } -// proposalNameRegexp returns the regexp string for validating a proposal name. -func (p *piPlugin) proposalNameRegexpString() string { - var b bytes.Buffer - - b.WriteString("^[") - for _, v := range p.proposalNameSupportedChars { - if len(v) > 1 { - b.WriteString(v) - } else { - b.WriteString(`\` + v) - } - } - minNameLength := strconv.Itoa(p.proposalNameLengthMin) - maxNameLength := strconv.Itoa(p.proposalNameLengthMax) - b.WriteString("]{") - b.WriteString(minNameLength + ",") - b.WriteString(maxNameLength + "}$") - - return b.String() -} - // proposalNameIsValid returns whether the provided name is a valid proposal // name. func (p *piPlugin) proposalNameIsValid(name string) bool { @@ -85,8 +62,8 @@ func (p *piPlugin) proposalNameIsValid(name string) bool { // a valid base64 payload, and that the file digest and MIME type are correct. func (p *piPlugin) proposalFilesVerify(files []backend.File) error { var ( - textFilesCount int - imageFilesCount int + textFilesCount uint32 + imageFilesCount uint32 indexFileFound bool ) for _, v := range files { @@ -101,8 +78,8 @@ func (p *piPlugin) proposalFilesVerify(files []backend.File) error { textFilesCount++ // The text file must be the proposal index file - if v.Name != p.indexFileName { - e := fmt.Sprintf("want %v, got %v", p.indexFileName, v.Name) + if v.Name != pi.FileNameIndexFile { + e := fmt.Sprintf("want %v, got %v", pi.FileNameIndexFile, v.Name) return backend.PluginError{ PluginID: pi.PluginID, ErrorCode: int(pi.ErrorCodeIndexFileNameInvalid), @@ -111,7 +88,7 @@ func (p *piPlugin) proposalFilesVerify(files []backend.File) error { } // Verify text file size - if len(payload) > p.textFileSizeMax { + if len(payload) > int(p.textFileSizeMax) { e := fmt.Sprintf("file %v size %v exceeds max size %v", v.Name, len(payload), p.textFileSizeMax) return backend.PluginError{ @@ -124,7 +101,7 @@ func (p *piPlugin) proposalFilesVerify(files []backend.File) error { // Verify there isn't more than one index file if indexFileFound { e := fmt.Sprintf("more than one %v file found", - p.indexFileName) + pi.FileNameIndexFile) return backend.PluginError{ PluginID: pi.PluginID, ErrorCode: int(pi.ErrorCodeIndexFileCountInvalid), @@ -139,7 +116,7 @@ func (p *piPlugin) proposalFilesVerify(files []backend.File) error { imageFilesCount++ // Verify image file size - if len(payload) > p.imageFileSizeMax { + if len(payload) > int(p.imageFileSizeMax) { e := fmt.Sprintf("image %v size %v exceeds max size %v", v.Name, len(payload), p.imageFileSizeMax) return backend.PluginError{ @@ -156,7 +133,7 @@ func (p *piPlugin) proposalFilesVerify(files []backend.File) error { // Verify that an index file is present if !indexFileFound { - e := fmt.Sprintf("%v file not found", p.indexFileName) + e := fmt.Sprintf("%v file not found", pi.FileNameIndexFile) return backend.PluginError{ PluginID: pi.PluginID, ErrorCode: int(pi.ErrorCodeIndexFileCountInvalid), @@ -202,7 +179,7 @@ func (p *piPlugin) proposalFilesVerify(files []backend.File) error { return backend.PluginError{ PluginID: pi.PluginID, ErrorCode: int(pi.ErrorCodeProposalNameInvalid), - ErrorContext: p.proposalNameRegexpString(), + ErrorContext: p.proposalNameRegexp.String(), } } diff --git a/politeiad/backend/tlogbe/plugins/pi/pi.go b/politeiad/backend/tlogbe/plugins/pi/pi.go index 80313b7da..aba1322be 100644 --- a/politeiad/backend/tlogbe/plugins/pi/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi/pi.go @@ -5,14 +5,18 @@ package pi import ( + "encoding/json" + "fmt" "os" "path/filepath" "regexp" + "strconv" "sync" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/politeiad/plugins/pi" + "github.com/decred/politeia/util" ) var ( @@ -30,15 +34,13 @@ type piPlugin struct { dataDir string // Plugin settings - indexFileName string - textFileCountMax int - textFileSizeMax int // In bytes - imageFileCountMax int - imageFileSizeMax int // In bytes - - proposalNameSupportedChars []string - proposalNameLengthMin int // In characters - proposalNameLengthMax int // In characters + textFileCountMax uint32 + textFileSizeMax uint32 // In bytes + imageFileCountMax uint32 + imageFileSizeMax uint32 // In bytes + proposalNameSupportedChars string // JSON encoded []string + proposalNameLengthMin uint32 // In characters + proposalNameLengthMax uint32 // In characters proposalNameRegexp *regexp.Regexp } @@ -94,6 +96,49 @@ func (p *piPlugin) Fsck(treeIDs []int64) error { return nil } +// Settings returns the plugin's settings. +// +// This function satisfies the plugins.PluginClient interface. +func (p *piPlugin) Settings() []backend.PluginSetting { + log.Tracef("Settings") + + return []backend.PluginSetting{ + { + Key: pi.SettingKeyTextFileCountMax, + Value: strconv.FormatUint(uint64(p.textFileCountMax), 10), + }, + { + Key: pi.SettingKeyTextFileSizeMax, + Value: strconv.FormatUint(uint64(p.textFileSizeMax), 10), + }, + { + Key: pi.SettingKeyImageFileCountMax, + Value: strconv.FormatUint(uint64(p.imageFileCountMax), 10), + }, + { + Key: pi.SettingKeyImageFileCountMax, + Value: strconv.FormatUint(uint64(p.imageFileCountMax), 10), + }, + { + Key: pi.SettingKeyImageFileSizeMax, + Value: strconv.FormatUint(uint64(p.imageFileSizeMax), 10), + }, + { + Key: pi.SettingKeyProposalNameLengthMin, + Value: strconv.FormatUint(uint64(p.proposalNameLengthMin), 10), + }, + { + Key: pi.SettingKeyProposalNameLengthMax, + Value: strconv.FormatUint(uint64(p.proposalNameLengthMax), 10), + }, + { + Key: pi.SettingKeyProposalNameSupportedChars, + Value: p.proposalNameSupportedChars, + }, + } +} + +// New returns a new piPlugin. func New(backend backend.Backend, settings []backend.PluginSetting, dataDir string) (*piPlugin, error) { // Create plugin data directory dataDir = filepath.Join(dataDir, pi.PluginID) @@ -102,22 +147,100 @@ func New(backend backend.Backend, settings []backend.PluginSetting, dataDir stri return nil, err } + // Setup plugin setting default values + var ( + textFileCountMax = pi.SettingTextFileCountMax + textFileSizeMax = pi.SettingTextFileSizeMax + imageFileCountMax = pi.SettingImageFileCountMax + imageFileSizeMax = pi.SettingImageFileSizeMax + nameLengthMin = pi.SettingProposalNameLengthMin + nameLengthMax = pi.SettingProposalNameLengthMax + nameSupportedChars = pi.SettingProposalNameSupportedChars + ) + + // Override default plugin settings with any passed in settings + for _, v := range settings { + switch v.Key { + case pi.SettingKeyTextFileCountMax: + u, err := strconv.ParseUint(v.Value, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid plugin setting %v '%v': %v", + v.Key, v.Value, err) + } + textFileCountMax = uint32(u) + case pi.SettingKeyTextFileSizeMax: + u, err := strconv.ParseUint(v.Value, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid plugin setting %v '%v': %v", + v.Key, v.Value, err) + } + textFileSizeMax = uint32(u) + case pi.SettingKeyImageFileCountMax: + u, err := strconv.ParseUint(v.Value, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid plugin setting %v '%v': %v", + v.Key, v.Value, err) + } + imageFileCountMax = uint32(u) + case pi.SettingKeyImageFileSizeMax: + u, err := strconv.ParseUint(v.Value, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid plugin setting %v '%v': %v", + v.Key, v.Value, err) + } + imageFileSizeMax = uint32(u) + case pi.SettingKeyProposalNameLengthMin: + u, err := strconv.ParseUint(v.Value, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid plugin setting %v '%v': %v", + v.Key, v.Value, err) + } + nameLengthMin = uint32(u) + case pi.SettingKeyProposalNameLengthMax: + u, err := strconv.ParseUint(v.Value, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid plugin setting %v '%v': %v", + v.Key, v.Value, err) + } + nameLengthMax = uint32(u) + case pi.SettingKeyProposalNameSupportedChars: + var sc []string + err := json.Unmarshal([]byte(v.Value), &sc) + if err != nil { + return nil, fmt.Errorf("invalid plugin setting %v '%v': %v", + v.Key, v.Value, err) + } + nameSupportedChars = sc + default: + return nil, fmt.Errorf("invalid plugin setting: %v", v.Key) + } + } + // Setup proposal name regex - // pregexp, err = regexp.Compile() - var pregexp *regexp.Regexp + rexp, err := util.Regexp(nameSupportedChars, uint64(nameLengthMin), + uint64(nameLengthMax)) + if err != nil { + return nil, fmt.Errorf("proposal name regexp: %v", err) + } + + // Encode the supported chars so that they can be returned as a + // string plugin setting. + b, err := json.Marshal(nameSupportedChars) + if err != nil { + return nil, err + } + nameSupportedCharsString := string(b) return &piPlugin{ - dataDir: dataDir, - backend: backend, - // TODO pi plugin settings - indexFileName: "", - textFileCountMax: 0, - textFileSizeMax: 0, - imageFileCountMax: 0, - imageFileSizeMax: 0, - proposalNameSupportedChars: []string{}, - proposalNameLengthMin: 0, - proposalNameLengthMax: 0, - proposalNameRegexp: pregexp, + dataDir: dataDir, + backend: backend, + textFileCountMax: textFileCountMax, + textFileSizeMax: textFileSizeMax, + imageFileCountMax: imageFileCountMax, + imageFileSizeMax: imageFileSizeMax, + proposalNameLengthMin: nameLengthMin, + proposalNameLengthMax: nameLengthMax, + proposalNameSupportedChars: nameSupportedCharsString, + proposalNameRegexp: rexp, }, nil } diff --git a/politeiad/backend/tlogbe/plugins/plugins.go b/politeiad/backend/tlogbe/plugins/plugins.go index f2fb73035..5022b860b 100644 --- a/politeiad/backend/tlogbe/plugins/plugins.go +++ b/politeiad/backend/tlogbe/plugins/plugins.go @@ -169,6 +169,9 @@ type PluginClient interface { // Fsck performs a plugin file system check. Fsck(treeIDs []int64) error + + // Settings returns the plugin settings. + Settings() []backend.PluginSetting } // TlogClient provides an API for plugins to interact with a tlog instance. diff --git a/politeiad/backend/tlogbe/tlog/plugin.go b/politeiad/backend/tlogbe/tlog/plugin.go index 631da3730..a5ca53c79 100644 --- a/politeiad/backend/tlogbe/tlog/plugin.go +++ b/politeiad/backend/tlogbe/tlog/plugin.go @@ -32,9 +32,8 @@ const ( // plugin represents a tlog plugin. type plugin struct { - id string - settings []backend.PluginSetting - client plugins.PluginClient + id string + client plugins.PluginClient } func (t *Tlog) plugin(pluginID string) (plugin, bool) { @@ -98,9 +97,8 @@ func (t *Tlog) PluginRegister(b backend.Backend, p backend.Plugin) error { defer t.Unlock() t.plugins[p.ID] = plugin{ - id: p.ID, - settings: p.Settings, - client: client, + id: p.ID, + client: client, } return nil @@ -182,7 +180,7 @@ func (t *Tlog) Plugins() []backend.Plugin { for _, v := range t.plugins { plugins = append(plugins, backend.Plugin{ ID: v.id, - Settings: v.settings, + Settings: v.client.Settings(), }) } diff --git a/politeiad/plugins/comments/comments.go b/politeiad/plugins/comments/comments.go index 39a7e3c9f..4d3f2d4d3 100644 --- a/politeiad/plugins/comments/comments.go +++ b/politeiad/plugins/comments/comments.go @@ -7,6 +7,7 @@ package comments const ( + // PluginID is the comments plugin ID. PluginID = "comments" // Plugin commands diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index ab4769484..c8196ac65 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -11,7 +11,53 @@ const ( PluginID = "pi" // Plugin commands - CmdVoteInv = "voteinv" // Get inventory by vote status + CmdVoteInv = "voteinv" +) + +const ( + // Setting keys are the plugin setting keys that can be used to + // override a default plugin setting. Defaults will be overridden + // if a plugin setting is provided to the plugin on startup. + SettingKeyTextFileCountMax = "textfilecountmax" + SettingKeyTextFileSizeMax = "textfilesizemax" + SettingKeyImageFileCountMax = "imagefilecountmax" + SettingKeyImageFileSizeMax = "imagefilesizemax" + SettingKeyProposalNameLengthMin = "proposalnamelengthmin" + SettingKeyProposalNameLengthMax = "proposalnamelengthmax" + SettingKeyProposalNameSupportedChars = "proposalnamesupportedchars" + + // SettingTextFileCountMax is the maximum number of text files that + // can be included a proposal. + SettingTextFileCountMax uint32 = 1 + + // SettingTextFileSizeMax is the maximum allowed size of a text + // file in bytes. + SettingTextFileSizeMax uint32 = 512 * 1024 + + // SettingImageFileCountMax is the maximum number of image files + // that can be included in a proposal. + SettingImageFileCountMax uint32 = 5 + + // SettingImageFileSizeMax is the maximum allowed size of a image + // file in bytes. + SettingImageFileSizeMax uint32 = 512 * 1024 + + // SettingProposalNameLengthMin is the minimum number of characters + // that a proposal name can be. + SettingProposalNameLengthMin uint32 = 8 + + // SettingProposalNameLengthMax is the maximum number of characters + // that a proposal name can be. + SettingProposalNameLengthMax uint32 = 80 +) + +var ( + // SettingProposalNameSupportedChars contains the supported + // characters in a proposal name. + SettingProposalNameSupportedChars = []string{ + "A-z", "0-9", "&", ".", ",", ":", ";", "-", " ", "@", "+", "#", "/", + "(", ")", "!", "?", "\"", "'", + } ) // ErrorCodeT represents a plugin error that was caused by the user. @@ -41,6 +87,11 @@ var ( ) const ( + // FileNameIndexFile is the file name of the proposal markdown + // file. Every proposal is required to have an index file. The + // index file should contain the proposal content. + FileNameIndexFile = "index.md" + // FileNameProposalMetadata is the filename of the ProposalMetadata // file that is saved to politeiad. ProposalMetadata is saved to // politeiad as a file, not as a metadata stream, since it contains diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index 980061638..20d212d6d 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -14,6 +14,7 @@ package ticketvote // Max (41k votes): 82MB const ( + // PluginID is the ticketvote plugin ID. PluginID = "ticketvote" // Plugin commands diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index e45bd6d3b..efc379e2d 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -50,7 +50,6 @@ type politeia struct { cfg *config router *mux.Router identity *identity.FullIdentity - plugins map[string]v1.Plugin } func remoteAddr(r *http.Request) string { @@ -69,16 +68,20 @@ func convertBackendPluginSetting(bpi backend.PluginSetting) v1.PluginSetting { } } -func convertBackendPlugin(bpi backend.Plugin) v1.Plugin { - p := v1.Plugin{ - ID: bpi.ID, - Version: bpi.Version, - } - for _, v := range bpi.Settings { - p.Settings = append(p.Settings, convertBackendPluginSetting(v)) +func convertBackendPlugins(bplugins []backend.Plugin) []v1.Plugin { + plugins := make([]v1.Plugin, 0, len(bplugins)) + for _, v := range bplugins { + p := v1.Plugin{ + ID: v.ID, + Version: v.Version, + Settings: make([]v1.PluginSetting, 0, len(v.Settings)), + } + for _, v := range v.Settings { + p.Settings = append(p.Settings, convertBackendPluginSetting(v)) + } + plugins = append(plugins, p) } - - return p + return plugins } // convertBackendMetadataStream converts a backend metadata stream to an API @@ -1149,12 +1152,35 @@ func (p *politeia) pluginInventory(w http.ResponseWriter, r *http.Request) { } response := p.identity.SignMessage(challenge) - reply := v1.PluginInventoryReply{ - Response: hex.EncodeToString(response[:]), + // Get plugins + unvetted := p.backend.GetUnvettedPlugins() + vetted := p.backend.GetVettedPlugins() + + // Aggregate unique plugins + pid := make(map[string]struct{}, len(unvetted)+len(vetted)) + plugins := make([]backend.Plugin, len(unvetted)+len(vetted)) + for _, v := range unvetted { + _, ok := pid[v.ID] + if ok { + // Already added + continue + } + plugins = append(plugins, v) + pid[v.ID] = struct{}{} + } + for _, v := range vetted { + _, ok := pid[v.ID] + if ok { + // Already added + continue + } + plugins = append(plugins, v) + pid[v.ID] = struct{}{} } - for _, v := range p.plugins { - reply.Plugins = append(reply.Plugins, v) + reply := v1.PluginInventoryReply{ + Plugins: convertBackendPlugins(plugins), + Response: hex.EncodeToString(response[:]), } util.RespondWithJSON(w, http.StatusOK, reply) @@ -1440,8 +1466,7 @@ func _main() error { // Setup application context. p := &politeia{ - cfg: cfg, - plugins: make(map[string]v1.Plugin), + cfg: cfg, } // Load identity. @@ -1622,9 +1647,6 @@ func _main() error { return fmt.Errorf("register vetted plugin %v: %v", v, err) } - // Add plugin to politeiad context - p.plugins[plugin.ID] = convertBackendPlugin(plugin) - log.Infof("Registered plugin: %v", v) } diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 2086494fb..63f8d3b28 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -20,6 +20,7 @@ const ( APIRoute = "/pi/v1" // Routes + RoutePolicy = "/policy" RouteProposals = "/proposals" RouteVoteInventory = "/voteinventory" @@ -88,7 +89,23 @@ func (e ServerErrorReply) Error() string { return fmt.Sprintf("server error: %v", e.ErrorCode) } -// PropStatusT represents a proposal status. +// Policy requests the policy settings for the pi API. +type Policy struct{} + +// PolicyReply is the reply to the Policy command. +type PolicyReply struct { + TextFileCountMax uint32 `json:"textfilecountmax"` + TextFileSizeMax uint32 `json:"textfilesizemax"` // In bytes + ImageFileCountMax uint32 `json:"imagefilecountmax"` + ImageFileSizeMax uint32 `json:"imagefilesizemax"` // In bytes + NameLengthMin uint32 `json:"namelengthmin"` // In characters + NameLengthMax uint32 `json:"namelengthmax"` // In characters + NameSupportedChars []string `json:"namesupportedchars"` +} + +// PropStatusT represents a proposal status. The proposal status codes map +// directly to the record status codes. Some have been renamed to give a more +// accurate representation of their use in pi. type PropStatusT int const ( @@ -124,16 +141,11 @@ const ( PropStatusAbandoned PropStatusT = 5 ) -// File describes an individual file that is part of the proposal. The -// directory structure must be flattened. -type File struct { - Name string `json:"name"` // Filename - MIME string `json:"mime"` // Mime type - Digest string `json:"digest"` // SHA256 digest of unencoded payload - Payload string `json:"payload"` // File content, base64 encoded -} - const ( + // FileNameIndexFile is the file name of the proposal markdown + // file that contains the proposal contents. + FileNameIndexFile = "index.md" + // FileNameProposalMetadata is the file name of the user submitted // ProposalMetadata. FileNameProposalMetadata = "proposalmetadata.json" @@ -143,6 +155,15 @@ const ( FileNameVoteMetadata = "votemetadata.json" ) +// File describes an individual file that is part of the proposal. The +// directory structure must be flattened. +type File struct { + Name string `json:"name"` // Filename + MIME string `json:"mime"` // Mime type + Digest string `json:"digest"` // SHA256 digest of unencoded payload + Payload string `json:"payload"` // File content, base64 encoded +} + // ProposalMetadata contains metadata that is specified by the user on proposal // submission. type ProposalMetadata struct { diff --git a/politeiawww/api/www/v1/v1.go b/politeiawww/api/www/v1/v1.go index e2386ce82..6302108f4 100644 --- a/politeiawww/api/www/v1/v1.go +++ b/politeiawww/api/www/v1/v1.go @@ -995,10 +995,10 @@ type PolicyReply struct { TokenPrefixLength int `json:"tokenprefixlength"` BuildInformation []string `json:"buildinformation"` IndexFilename string `json:"indexfilename"` - MinLinkByPeriod int64 `json:"minlinkbyperiod"` // DEPRECATED - MaxLinkByPeriod int64 `json:"maxlinkbyperiod"` // DEPRECATED - MinVoteDuration uint32 `json:"minvoteduration"` // DEPRECATED - MaxVoteDuration uint32 `json:"maxvoteduration"` // DEPRECATED + MinLinkByPeriod int64 `json:"minlinkbyperiod"` + MaxLinkByPeriod int64 `json:"maxlinkbyperiod"` + MinVoteDuration uint32 `json:"minvoteduration"` + MaxVoteDuration uint32 `json:"maxvoteduration"` } // VoteOption describes a single vote option. diff --git a/politeiawww/pi/pi.go b/politeiawww/pi/pi.go index d5cdc5188..32430a80a 100644 --- a/politeiawww/pi/pi.go +++ b/politeiawww/pi/pi.go @@ -6,9 +6,13 @@ package pi import ( "encoding/json" + "fmt" "net/http" + "strconv" + pdv1 "github.com/decred/politeia/politeiad/api/v1" pdclient "github.com/decred/politeia/politeiad/client" + "github.com/decred/politeia/politeiad/plugins/pi" v1 "github.com/decred/politeia/politeiawww/api/pi/v1" "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/sessions" @@ -22,6 +26,29 @@ type Pi struct { politeiad *pdclient.Client userdb user.Database sessions *sessions.Sessions + + // Plugin settings + textFileCountMax uint32 + textFileSizeMax uint32 // In bytes + imageFileCountMax uint32 + imageFileSizeMax uint32 // In bytes + nameLengthMin uint32 // In characters + nameLengthMax uint32 // In characters + nameSupportedChars []string +} + +// HandlePolicy is the request handler for the pi v1 Policy route. +func (p *Pi) HandlePolicy(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandlePolicy") + + pr, err := p.processPolicy(r.Context()) + if err != nil { + respondWithError(w, r, + "HandlePolicy: processPolicy: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, pr) } // HandleProposals is the request handler for the pi v1 Proposals route. @@ -73,11 +100,107 @@ func (p *Pi) HandleVoteInventory(w http.ResponseWriter, r *http.Request) { } // New returns a new Pi context. -func New(cfg *config.Config, pdc *pdclient.Client, s *sessions.Sessions, udb user.Database) *Pi { - return &Pi{ - cfg: cfg, - politeiad: pdc, - userdb: udb, - sessions: s, +func New(cfg *config.Config, pdc *pdclient.Client, udb user.Database, s *sessions.Sessions, plugins []pdv1.Plugin) (*Pi, error) { + // Parse pi plugin settings + var ( + textFileCountMax uint32 + textFileSizeMax uint32 + imageFileCountMax uint32 + imageFileSizeMax uint32 + nameLengthMin uint32 + nameLengthMax uint32 + nameSupportedChars []string + ) + for _, v := range plugins { + if v.ID != pi.PluginID { + // Not the pi plugin; skip + continue + } + for _, s := range v.Settings { + switch s.Key { + case pi.SettingKeyTextFileCountMax: + u, err := strconv.ParseUint(s.Value, 10, 64) + if err != nil { + return nil, err + } + textFileCountMax = uint32(u) + case pi.SettingKeyTextFileSizeMax: + u, err := strconv.ParseUint(s.Value, 10, 64) + if err != nil { + return nil, err + } + textFileSizeMax = uint32(u) + case pi.SettingKeyImageFileCountMax: + u, err := strconv.ParseUint(s.Value, 10, 64) + if err != nil { + return nil, err + } + imageFileCountMax = uint32(u) + case pi.SettingKeyImageFileSizeMax: + u, err := strconv.ParseUint(s.Value, 10, 64) + if err != nil { + return nil, err + } + imageFileSizeMax = uint32(u) + case pi.SettingKeyProposalNameLengthMin: + u, err := strconv.ParseUint(s.Value, 10, 64) + if err != nil { + return nil, err + } + nameLengthMin = uint32(u) + case pi.SettingKeyProposalNameLengthMax: + u, err := strconv.ParseUint(s.Value, 10, 64) + if err != nil { + return nil, err + } + nameLengthMax = uint32(u) + case pi.SettingKeyProposalNameSupportedChars: + var sc []string + err := json.Unmarshal([]byte(s.Value), &sc) + if err != nil { + return nil, err + } + nameSupportedChars = sc + default: + // Skip unknown settings + log.Warnf("Unknown plugin setting %v; Skipping...", s.Key) + } + } } + + // Verify all plugin settings have been provided + switch { + case textFileCountMax == 0: + return nil, fmt.Errorf("plugin setting not found: %v", + pi.SettingKeyTextFileCountMax) + case textFileSizeMax == 0: + return nil, fmt.Errorf("plugin setting not found: %v", + pi.SettingKeyTextFileSizeMax) + case imageFileCountMax == 0: + return nil, fmt.Errorf("plugin setting not found: %v", + pi.SettingKeyImageFileCountMax) + case imageFileSizeMax == 0: + return nil, fmt.Errorf("plugin setting not found: %v", + pi.SettingKeyImageFileSizeMax) + case nameLengthMin == 0: + return nil, fmt.Errorf("plugin setting not found: %v", + pi.SettingKeyProposalNameLengthMin) + case nameLengthMax == 0: + return nil, fmt.Errorf("plugin setting not found: %v", + pi.SettingKeyProposalNameLengthMax) + } + + return &Pi{ + cfg: cfg, + politeiad: pdc, + userdb: udb, + sessions: s, + textFileCountMax: textFileCountMax, + textFileSizeMax: textFileSizeMax, + imageFileCountMax: imageFileCountMax, + imageFileSizeMax: imageFileSizeMax, + nameLengthMin: nameLengthMin, + nameLengthMax: nameLengthMax, + nameSupportedChars: nameSupportedChars, + }, nil } diff --git a/politeiawww/pi/process.go b/politeiawww/pi/process.go index 419f27fd5..9a82577ef 100644 --- a/politeiawww/pi/process.go +++ b/politeiawww/pi/process.go @@ -21,6 +21,20 @@ import ( "github.com/google/uuid" ) +func (p *Pi) processPolicy(ctx context.Context) (*v1.PolicyReply, error) { + log.Tracef("Policy") + + return &v1.PolicyReply{ + TextFileCountMax: p.textFileCountMax, + TextFileSizeMax: p.textFileSizeMax, + ImageFileCountMax: p.imageFileCountMax, + ImageFileSizeMax: p.imageFileSizeMax, + NameLengthMin: p.nameLengthMin, + NameLengthMax: p.nameLengthMax, + NameSupportedChars: p.nameSupportedChars, + }, nil +} + // proposal returns a version of a proposal record from politeiad. If version // is an empty string then the most recent version will be returned. func (p *Pi) proposal(ctx context.Context, state, token, version string) (*v1.Proposal, error) { diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 6b856c48b..da75ab8f1 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -8,18 +8,24 @@ import ( "fmt" "net/http" + pdv1 "github.com/decred/politeia/politeiad/api/v1" + cmplugin "github.com/decred/politeia/politeiad/plugins/comments" + piplugin "github.com/decred/politeia/politeiad/plugins/pi" + tkplugin "github.com/decred/politeia/politeiad/plugins/ticketvote" cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" + piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/comments" + "github.com/decred/politeia/politeiawww/pi" "github.com/decred/politeia/politeiawww/records" "github.com/decred/politeia/politeiawww/ticketvote" "github.com/google/uuid" ) // setupPiRoutes sets up the API routes for piwww mode. -func (p *politeiawww) setupPiRoutes(r *records.Records, c *comments.Comments, t *ticketvote.TicketVote) { +func (p *politeiawww) setupPiRoutes(r *records.Records, c *comments.Comments, t *ticketvote.TicketVote, pic *pi.Pi) { // Return a 404 when a route is not found p.router.NotFoundHandler = http.HandlerFunc(p.handleNotFound) @@ -75,6 +81,9 @@ func (p *politeiawww) setupPiRoutes(r *records.Records, c *comments.Comments, t p.addRoute(http.MethodPost, rcv1.APIRoute, rcv1.RouteTimestamps, r.HandleTimestamps, permissionPublic) + p.addRoute(http.MethodPost, rcv1.APIRoute, + rcv1.RouteUserRecords, r.HandleUserRecords, + permissionPublic) // Comment routes p.addRoute(http.MethodPost, cmv1.APIRoute, @@ -122,26 +131,55 @@ func (p *politeiawww) setupPiRoutes(r *records.Records, c *comments.Comments, t tkv1.RouteTimestamps, t.HandleTimestamps, permissionPublic) - /* - // Pi routes - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteProposals, p.handleProposals, - permissionPublic) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteVoteInventory, p.handleVoteInventory, - permissionPublic) - */ + // Pi routes + p.addRoute(http.MethodGet, piv1.APIRoute, + piv1.RoutePolicy, pic.HandlePolicy, + permissionPublic) + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteProposals, pic.HandleProposals, + permissionPublic) + p.addRoute(http.MethodPost, piv1.APIRoute, + piv1.RouteVoteInventory, pic.HandleVoteInventory, + permissionPublic) } -func (p *politeiawww) setupPi() error { +func (p *politeiawww) setupPi(plugins []pdv1.Plugin) error { + // Verify all required politeiad plugins have been registered + required := map[string]bool{ + piplugin.PluginID: false, + cmplugin.PluginID: false, + tkplugin.PluginID: false, + } + for _, v := range plugins { + _, ok := required[v.ID] + if !ok { + // Not a required plugin. Skip. + continue + } + required[v.ID] = true + } + notFound := make([]string, 0, len(required)) + for pluginID, wasFound := range required { + if !wasFound { + notFound = append(notFound, pluginID) + } + } + if len(notFound) > 0 { + return fmt.Errorf("required politeiad plugins not found: %v", notFound) + } + // Setup api contexts c := comments.New(p.cfg, p.politeiad, p.db, p.sessions, p.events) tv := ticketvote.New(p.cfg, p.politeiad, p.sessions, p.events) r := records.New(p.cfg, p.politeiad, p.sessions, p.events) + pic, err := pi.New(p.cfg, p.politeiad, p.db, p.sessions, plugins) + if err != nil { + return fmt.Errorf("new pi: %v", err) + } // Setup routes p.setUserWWWRoutes() - p.setupPiRoutes(r, c, tv) + p.setupPiRoutes(r, c, tv, pic) // Verify paywall settings switch { @@ -162,7 +200,7 @@ func (p *politeiawww) setupPi() error { // Setup paywall pool p.userPaywallPool = make(map[uuid.UUID]paywallPoolMember) - err := p.initPaywallChecker() + err = p.initPaywallChecker() if err != nil { return err } @@ -170,7 +208,5 @@ func (p *politeiawww) setupPi() error { // Setup event manager p.setupEventListenersPi() - // TODO Verify politeiad plugins - return nil } diff --git a/politeiawww/plugin.go b/politeiawww/plugin.go deleted file mode 100644 index 457f8410f..000000000 --- a/politeiawww/plugin.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright (c) 2017-2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import ( - "context" - "fmt" - "time" - - pd "github.com/decred/politeia/politeiad/api/v1" -) - -// pluginSetting is a structure that holds the key-value pairs of a plugin -// setting. -type pluginSetting struct { - Key string - Value string -} - -// plugin describes a plugin and its settings. -type plugin struct { - ID string // Identifier - Version string // Version - Settings []pluginSetting // Settings -} - -func convertPluginFromPD(p pd.Plugin) plugin { - ps := make([]pluginSetting, 0, len(p.Settings)) - for _, v := range p.Settings { - ps = append(ps, pluginSetting{ - Key: v.Key, - Value: v.Value, - }) - } - return plugin{ - ID: p.ID, - Version: p.Version, - Settings: ps, - } -} - -// getPluginInventory returns the politeiad plugin inventory. If a politeiad -// connection cannot be made, the call will be retried every 5 seconds for up -// to 1000 tries. -func (p *politeiawww) getPluginInventory() ([]plugin, error) { - // Attempt to fetch the plugin inventory from politeiad until - // either it is successful or the maxRetries has been exceeded. - var ( - done bool - maxRetries = 1000 - sleepInterval = 5 * time.Second - plugins = make([]plugin, 0, 16) - ctx = context.Background() - ) - for retries := 0; !done; retries++ { - if ctx.Err() != nil { - return nil, ctx.Err() - } - if retries == maxRetries { - return nil, fmt.Errorf("max retries exceeded") - } - - pi, err := p.politeiad.PluginInventory(ctx) - if err != nil { - log.Infof("cannot get politeiad plugin inventory: %v: retry in %v", - err, sleepInterval) - time.Sleep(sleepInterval) - continue - } - for _, v := range pi { - plugins = append(plugins, convertPluginFromPD(v)) - } - - done = true - } - - return plugins, nil -} diff --git a/politeiawww/politeiawww.go b/politeiawww/politeiawww.go index b4905f67e..a5a45b7d9 100644 --- a/politeiawww/politeiawww.go +++ b/politeiawww/politeiawww.go @@ -16,6 +16,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/decred/dcrd/chaincfg/v3" + pdv1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/mime" pdclient "github.com/decred/politeia/politeiad/client" www "github.com/decred/politeia/politeiawww/api/www/v1" @@ -75,7 +76,7 @@ type politeiawww struct { db user.Database sessions *sessions.Sessions events *events.Manager - plugins []plugin + plugins []pdv1.Plugin // Client websocket connections ws map[string]map[string]*wsContext // [uuid][]*context diff --git a/politeiawww/testing.go b/politeiawww/testing.go index 07c67ccfc..8c17fe179 100644 --- a/politeiawww/testing.go +++ b/politeiawww/testing.go @@ -24,10 +24,11 @@ import ( "github.com/decred/politeia/politeiad/api/v1/mime" "github.com/decred/politeia/politeiad/testpoliteiad" cms "github.com/decred/politeia/politeiawww/api/cms/v1" - pi "github.com/decred/politeia/politeiawww/api/pi/v1" + piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/comments" "github.com/decred/politeia/politeiawww/config" + "github.com/decred/politeia/politeiawww/pi" "github.com/decred/politeia/politeiawww/records" "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/politeiawww/ticketvote" @@ -58,7 +59,7 @@ func errToStr(e error) string { // by default but can be filled in with random rgb colors by setting the // addColor parameter to true. The png without color will be ~3kB. The png with // color will be ~2MB. -func newFilePNG(t *testing.T, addColor bool) *pi.File { +func newFilePNG(t *testing.T, addColor bool) *piv1.File { t.Helper() b := new(bytes.Buffer) @@ -89,7 +90,7 @@ func newFilePNG(t *testing.T, addColor bool) *pi.File { t.Fatalf("%v", err) } - return &pi.File{ + return &piv1.File{ Name: hex.EncodeToString(r) + ".png", MIME: mime.DetectMimeType(b.Bytes()), Digest: hex.EncodeToString(util.Digest(b.Bytes())), @@ -366,10 +367,11 @@ func newTestPoliteiawww(t *testing.T) (*politeiawww, func()) { var c *comments.Comments var tv *ticketvote.TicketVote var r *records.Records + var pic *pi.Pi // Setup routes p.setUserWWWRoutes() - p.setupPiRoutes(r, c, tv) + p.setupPiRoutes(r, c, tv, pic) // The cleanup is handled using a closure so that the temp dir // can be deleted using the local variable and not cfg.DataDir. diff --git a/politeiawww/www.go b/politeiawww/www.go index f3894bd29..e536e5bf2 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -26,6 +26,7 @@ import ( "github.com/decred/politeia/mdstream" pd "github.com/decred/politeia/politeiad/api/v1" + pdv1 "github.com/decred/politeia/politeiad/api/v1" pdclient "github.com/decred/politeia/politeiad/client" "github.com/decred/politeia/politeiad/plugins/ticketvote" cms "github.com/decred/politeia/politeiawww/api/cms/v1" @@ -307,6 +308,44 @@ func (p *politeiawww) addRoute(method string, routeVersion string, route string, } } +// getPluginInventory returns the politeiad plugin inventory. If a politeiad +// connection cannot be made, the call will be retried every 5 seconds for up +// to 1000 tries. +func (p *politeiawww) getPluginInventory() ([]pdv1.Plugin, error) { + // Attempt to fetch the plugin inventory from politeiad until + // either it is successful or the maxRetries has been exceeded. + var ( + done bool + maxRetries = 1000 + sleepInterval = 5 * time.Second + plugins = make([]pdv1.Plugin, 0, 32) + ctx = context.Background() + ) + for retries := 0; !done; retries++ { + if ctx.Err() != nil { + return nil, ctx.Err() + } + if retries == maxRetries { + return nil, fmt.Errorf("max retries exceeded") + } + + pi, err := p.politeiad.PluginInventory(ctx) + if err != nil { + log.Infof("cannot get politeiad plugin inventory: %v: retry in %v", + err, sleepInterval) + time.Sleep(sleepInterval) + continue + } + for _, v := range pi { + plugins = append(plugins, v) + } + + done = true + } + + return plugins, nil +} + func (p *politeiawww) setupCMS() error { // Setup routes p.setCMSWWWRoutes() @@ -680,10 +719,11 @@ func _main() error { } // Setup politeiad plugins - p.plugins, err = p.getPluginInventory() + plugins, err := p.getPluginInventory() if err != nil { return fmt.Errorf("getPluginInventory: %v", err) } + p.plugins = plugins // Setup email-userID cache err = p.initUserEmailsCache() @@ -694,7 +734,7 @@ func _main() error { // Perform application specific setup switch p.cfg.Mode { case config.PoliteiaWWWMode: - err = p.setupPi() + err = p.setupPi(plugins) if err != nil { return fmt.Errorf("setupPi: %v", err) } diff --git a/util/regexp.go b/util/regexp.go new file mode 100644 index 000000000..fe2eb3711 --- /dev/null +++ b/util/regexp.go @@ -0,0 +1,48 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package util + +import ( + "bytes" + "fmt" + "regexp" + "strconv" +) + +// Regexp returns a compiled Regexp for the provided parameters. +func Regexp(supportedChars []string, lengthMin, lengthMax uint64) (*regexp.Regexp, error) { + // Match begining of string + var b bytes.Buffer + b.WriteString("^") + + // Set allowed character set + b.WriteString("[") + for _, v := range supportedChars { + switch v { + case `\`, `"`, "[", "]", "^", "-", " ": + // These characters must be escaped + b.WriteString(`\` + v) + default: + b.WriteString(v) + } + } + b.WriteString("]") + + // Set min and max length + min := strconv.FormatUint(lengthMin, 10) + max := strconv.FormatUint(lengthMax, 10) + b.WriteString(fmt.Sprintf("{%v,%v}", min, max)) + + // Match end of string + b.WriteString("$") + + // Compile regexp + r, err := regexp.Compile(b.String()) + if err != nil { + return nil, err + } + + return r, nil +} From 5109a5f7a5d7c4d29df3ccb5d542457ed9bcb2e7 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 31 Jan 2021 20:06:48 -0600 Subject: [PATCH 259/449] Work through all new record issues and details. --- .gitignore | 1 + politeiad/api/v1/v1.go | 4 +- .../tlogbe/plugins/comments/comments.go | 9 + .../backend/tlogbe/plugins/dcrdata/dcrdata.go | 9 + politeiad/backend/tlogbe/plugins/pi/hooks.go | 182 ++++++++++-------- .../tlogbe/plugins/ticketvote/ticketvote.go | 9 + .../backend/tlogbe/plugins/user/hooks.go | 11 +- politeiad/backend/tlogbe/plugins/user/user.go | 9 + politeiad/backend/tlogbe/tlog/plugin.go | 6 + politeiad/backend/tlogbe/tlog/tlog.go | 1 + politeiad/backend/tlogbe/tlogbe.go | 11 +- politeiad/client/client.go | 13 +- politeiad/plugins/pi/pi.go | 23 +-- politeiad/plugins/user/user.go | 19 +- politeiad/politeiad.go | 103 +++++----- politeiad/util/util.go | 50 ----- politeiawww/api/pi/v1/v1.go | 3 +- politeiawww/api/records/v1/v1.go | 45 ++++- politeiawww/client/client.go | 165 ++++++++++++++++ politeiawww/client/pi.go | 29 +++ politeiawww/client/records.go | 29 +++ politeiawww/cmd/pictl/proposalnew.go | 152 ++++++++------- politeiawww/cmd/pictl/sample-pictl.conf | 6 + politeiawww/cmd/pictl/sample-piwww.conf | 6 - politeiawww/cmd/pictl/util.go | 110 +++++------ politeiawww/cmd/shared/client.go | 17 +- politeiawww/cmd/shared/config.go | 10 +- politeiawww/config.go | 5 +- politeiawww/config/config.go | 4 + politeiawww/records/error.go | 47 ++++- politeiawww/www.go | 7 +- util/merkle.go | 33 ++++ 32 files changed, 727 insertions(+), 401 deletions(-) delete mode 100644 politeiad/util/util.go create mode 100644 politeiawww/client/client.go create mode 100644 politeiawww/client/pi.go create mode 100644 politeiawww/client/records.go create mode 100644 politeiawww/cmd/pictl/sample-pictl.conf delete mode 100644 politeiawww/cmd/pictl/sample-piwww.conf create mode 100644 util/merkle.go diff --git a/.gitignore b/.gitignore index 189351b00..7fce4193e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ politeiad/politeiad politeiawww/cmd/politeiawww_dbutil/politeiawww_dbutil politeiawww/cmd/politeiawww_refclient/politeiawww_refclient politeiawww/cmd/politeiawwwcli/politeiawwwcli +politeiawww/cmd/pictl/pictl politeiawww/cmd/politeiawwwtest/politeiawwwtest politeiawww/cmd/politeiavoter/politeiavoter politeiawww/politeiawww diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index 72c018de3..035286ccc 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -493,14 +493,14 @@ func (e UserErrorReply) Error() string { // PluginUserErrorReply returns details about a plugin error that occurred // while trying to execute a command due to bad input from the client. type PluginErrorReply struct { - PluginID string `json:"plugin"` + PluginID string `json:"pluginid"` ErrorCode int `json:"errorcode"` ErrorContext []string `json:"errorcontext,omitempty"` } // Error satisfies the error interface. func (e PluginErrorReply) Error() string { - return fmt.Sprintf("plugin user error code: %v", e.ErrorCode) + return fmt.Sprintf("plugin '%v' error code: %v", e.PluginID, e.ErrorCode) } // ServerErrorReply returns an error code that can be correlated with diff --git a/politeiad/backend/tlogbe/plugins/comments/comments.go b/politeiad/backend/tlogbe/plugins/comments/comments.go index 7cf3d1c5a..3ae749991 100644 --- a/politeiad/backend/tlogbe/plugins/comments/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments/comments.go @@ -123,6 +123,15 @@ func (p *commentsPlugin) Fsck(treeIDs []int64) error { return nil } +// TODO Settings returns the plugin's settings. +// +// This function satisfies the plugins.PluginClient interface. +func (p *commentsPlugin) Settings() []backend.PluginSetting { + log.Tracef("Settings") + + return nil +} + // New returns a new comments plugin. func New(tlog plugins.TlogClient, settings []backend.PluginSetting, dataDir string, id *identity.FullIdentity) (*commentsPlugin, error) { // Setup comments plugin data dir diff --git a/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go b/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go index 7f3103ca8..780dbc90c 100644 --- a/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go +++ b/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go @@ -627,6 +627,15 @@ func (p *dcrdataPlugin) Fsck(treeIDs []int64) error { return nil } +// TODO Settings returns the plugin's settings. +// +// This function satisfies the plugins.PluginClient interface. +func (p *dcrdataPlugin) Settings() []backend.PluginSetting { + log.Tracef("Settings") + + return nil +} + func New(settings []backend.PluginSetting, activeNetParams *chaincfg.Params) (*dcrdataPlugin, error) { // Unpack plugin settings var ( diff --git a/politeiad/backend/tlogbe/plugins/pi/hooks.go b/politeiad/backend/tlogbe/plugins/pi/hooks.go index 3077fb295..f0341c249 100644 --- a/politeiad/backend/tlogbe/plugins/pi/hooks.go +++ b/politeiad/backend/tlogbe/plugins/pi/hooks.go @@ -24,6 +24,16 @@ const ( mimeTypePNG = "image/png" ) +var ( + // allowedTextFileNames contains the only file names that are + // allowed for text files. + allowedTextFileNames = map[string]struct{}{ + pi.FileNameIndexFile: {}, + pi.FileNameProposalMetadata: {}, + ticketvote.FileNameVoteMetadata: {}, + } +) + func tokenDecode(token string) ([]byte, error) { return util.TokenDecode(util.TokenTypeTlog, token) } @@ -61,105 +71,109 @@ func (p *piPlugin) proposalNameIsValid(name string) bool { // passed politeia validation so we can assume that the file has a unique name, // a valid base64 payload, and that the file digest and MIME type are correct. func (p *piPlugin) proposalFilesVerify(files []backend.File) error { - var ( - textFilesCount uint32 - imageFilesCount uint32 - indexFileFound bool - ) - for _, v := range files { - payload, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return fmt.Errorf("invalid base64 %v", v.Name) - } - - // MIME type specific validation - switch v.MIME { - case mimeTypeText: - textFilesCount++ - - // The text file must be the proposal index file - if v.Name != pi.FileNameIndexFile { - e := fmt.Sprintf("want %v, got %v", pi.FileNameIndexFile, v.Name) - return backend.PluginError{ - PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeIndexFileNameInvalid), - ErrorContext: e, - } + // TODO this verification assumes the user provided is passed + // in as metadata and not files. + /* + var ( + textFilesCount uint32 + imageFilesCount uint32 + indexFileFound bool + ) + for _, v := range files { + payload, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return fmt.Errorf("invalid base64 %v", v.Name) } - // Verify text file size - if len(payload) > int(p.textFileSizeMax) { - e := fmt.Sprintf("file %v size %v exceeds max size %v", - v.Name, len(payload), p.textFileSizeMax) - return backend.PluginError{ - PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeIndexFileSizeInvalid), - ErrorContext: e, + // MIME type specific validation + switch v.MIME { + case mimeTypeText, mimeTypeTextUTF8: + textFilesCount++ + + // The text file must be the proposal index file + if v.Name != pi.FileNameIndexFile { + e := fmt.Sprintf("want %v, got %v", pi.FileNameIndexFile, v.Name) + return backend.PluginError{ + PluginID: pi.PluginID, + ErrorCode: int(pi.ErrorCodeIndexFileNameInvalid), + ErrorContext: e, + } } - } - // Verify there isn't more than one index file - if indexFileFound { - e := fmt.Sprintf("more than one %v file found", - pi.FileNameIndexFile) - return backend.PluginError{ - PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeIndexFileCountInvalid), - ErrorContext: e, + // Verify text file size + if len(payload) > int(p.textFileSizeMax) { + e := fmt.Sprintf("file %v size %v exceeds max size %v", + v.Name, len(payload), p.textFileSizeMax) + return backend.PluginError{ + PluginID: pi.PluginID, + ErrorCode: int(pi.ErrorCodeIndexFileSizeInvalid), + ErrorContext: e, + } } - } - - // Set index file as being found - indexFileFound = true - case mimeTypePNG: - imageFilesCount++ + // Verify there isn't more than one index file + if indexFileFound { + e := fmt.Sprintf("more than one %v file found", + pi.FileNameIndexFile) + return backend.PluginError{ + PluginID: pi.PluginID, + ErrorCode: int(pi.ErrorCodeIndexFileCountInvalid), + ErrorContext: e, + } + } - // Verify image file size - if len(payload) > int(p.imageFileSizeMax) { - e := fmt.Sprintf("image %v size %v exceeds max size %v", - v.Name, len(payload), p.imageFileSizeMax) - return backend.PluginError{ - PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeImageFileSizeInvalid), - ErrorContext: e, + // Set index file as being found + indexFileFound = true + + case mimeTypePNG: + imageFilesCount++ + + // Verify image file size + if len(payload) > int(p.imageFileSizeMax) { + e := fmt.Sprintf("image %v size %v exceeds max size %v", + v.Name, len(payload), p.imageFileSizeMax) + return backend.PluginError{ + PluginID: pi.PluginID, + ErrorCode: int(pi.ErrorCodeImageFileSizeInvalid), + ErrorContext: e, + } } - } - default: - return fmt.Errorf("invalid mime") + default: + return fmt.Errorf("invalid mime") + } } - } - // Verify that an index file is present - if !indexFileFound { - e := fmt.Sprintf("%v file not found", pi.FileNameIndexFile) - return backend.PluginError{ - PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeIndexFileCountInvalid), - ErrorContext: e, + // Verify that an index file is present + if !indexFileFound { + e := fmt.Sprintf("%v file not found", pi.FileNameIndexFile) + return backend.PluginError{ + PluginID: pi.PluginID, + ErrorCode: int(pi.ErrorCodeIndexFileCountInvalid), + ErrorContext: e, + } } - } - // Verify file counts are acceptable - if textFilesCount > p.textFileCountMax { - e := fmt.Sprintf("got %v text files, max is %v", - textFilesCount, p.textFileCountMax) - return backend.PluginError{ - PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeTextFileCountInvalid), - ErrorContext: e, + // Verify file counts are acceptable + if textFilesCount > p.textFileCountMax { + e := fmt.Sprintf("got %v text files, max is %v", + textFilesCount, p.textFileCountMax) + return backend.PluginError{ + PluginID: pi.PluginID, + ErrorCode: int(pi.ErrorCodeTextFileCountInvalid), + ErrorContext: e, + } } - } - if imageFilesCount > p.imageFileCountMax { - e := fmt.Sprintf("got %v image files, max is %v", - imageFilesCount, p.imageFileCountMax) - return backend.PluginError{ - PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeImageFileCountInvalid), - ErrorContext: e, + if imageFilesCount > p.imageFileCountMax { + e := fmt.Sprintf("got %v image files, max is %v", + imageFilesCount, p.imageFileCountMax) + return backend.PluginError{ + PluginID: pi.PluginID, + ErrorCode: int(pi.ErrorCodeImageFileCountInvalid), + ErrorContext: e, + } } - } + */ // Verify a proposal metadata has been included pm, err := proposalMetadataDecode(files) diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go index e16c86faf..399723035 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go @@ -244,6 +244,15 @@ func (p *ticketVotePlugin) Fsck(treeIDs []int64) error { return nil } +// TODO Settings returns the plugin's settings. +// +// This function satisfies the plugins.PluginClient interface. +func (p *ticketVotePlugin) Settings() []backend.PluginSetting { + log.Tracef("Settings") + + return nil +} + /* // linkByPeriodMin returns the minimum amount of time, in seconds, that the // LinkBy period must be set to. This is determined by adding 1 week onto the diff --git a/politeiad/backend/tlogbe/plugins/user/hooks.go b/politeiad/backend/tlogbe/plugins/user/hooks.go index de424b693..4b93364f6 100644 --- a/politeiad/backend/tlogbe/plugins/user/hooks.go +++ b/politeiad/backend/tlogbe/plugins/user/hooks.go @@ -5,7 +5,6 @@ package user import ( - "encoding/hex" "encoding/json" "errors" "fmt" @@ -16,7 +15,6 @@ import ( "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/politeiad/plugins/user" - pdutil "github.com/decred/politeia/politeiad/util" "github.com/decred/politeia/util" "github.com/google/uuid" ) @@ -82,12 +80,15 @@ func userMetadataVerify(metadata []backend.MetadataStream, files []backend.File) } // Verify signature - m, err := pdutil.MerkleRoot(files) + digests := make([]string, 0, len(files)) + for _, v := range files { + digests = append(digests, v.Digest) + } + m, err := util.MerkleRoot(digests) if err != nil { return err } - msg := hex.EncodeToString(m[:]) - err = util.VerifySignature(um.Signature, um.PublicKey, msg) + err = util.VerifySignature(um.Signature, um.PublicKey, string(m[:])) if err != nil { return convertSignatureError(err) } diff --git a/politeiad/backend/tlogbe/plugins/user/user.go b/politeiad/backend/tlogbe/plugins/user/user.go index 5a58dd72f..cad8369a5 100644 --- a/politeiad/backend/tlogbe/plugins/user/user.go +++ b/politeiad/backend/tlogbe/plugins/user/user.go @@ -84,6 +84,15 @@ func (p *userPlugin) Fsck(treeIDs []int64) error { return nil } +// TODO Settings returns the plugin's settings. +// +// This function satisfies the plugins.PluginClient interface. +func (p *userPlugin) Settings() []backend.PluginSetting { + log.Tracef("Settings") + + return nil +} + // New returns a new userPlugin. func New(tlog plugins.TlogClient, settings []backend.PluginSetting, dataDir string) (*userPlugin, error) { // Create plugin data directory diff --git a/politeiad/backend/tlogbe/tlog/plugin.go b/politeiad/backend/tlogbe/tlog/plugin.go index a5ca53c79..19ee60f84 100644 --- a/politeiad/backend/tlogbe/tlog/plugin.go +++ b/politeiad/backend/tlogbe/tlog/plugin.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "path/filepath" + "sort" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" @@ -53,6 +54,11 @@ func (t *Tlog) pluginIDs() []string { ids = append(ids, k) } + // Sort IDs so the returned order is deterministic + sort.SliceStable(ids, func(i, j int) bool { + return ids[i] < ids[j] + }) + return ids } diff --git a/politeiad/backend/tlogbe/tlog/tlog.go b/politeiad/backend/tlogbe/tlog/tlog.go index 9d79b80f6..df7819632 100644 --- a/politeiad/backend/tlogbe/tlog/tlog.go +++ b/politeiad/backend/tlogbe/tlog/tlog.go @@ -1527,6 +1527,7 @@ func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trilli store: store, dcrtime: dcrtimeClient, cron: cron.New(), + plugins: make(map[string]plugin), encryptionKey: ek, } diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 141f233fb..8efca3d72 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -25,7 +25,6 @@ import ( "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" - pdutil "github.com/decred/politeia/politeiad/util" "github.com/decred/politeia/util" "github.com/marcopeereboom/sbox" "github.com/subosito/gozaru" @@ -405,7 +404,11 @@ func statusChangeIsAllowed(from, to backend.MDStatusT) bool { } func recordMetadataNew(token []byte, files []backend.File, status backend.MDStatusT, iteration uint64) (*backend.RecordMetadata, error) { - m, err := pdutil.MerkleRoot(files) + digests := make([]string, 0, len(files)) + for _, v := range files { + digests = append(digests, v.Digest) + } + m, err := util.MerkleRoot(digests) if err != nil { return nil, err } @@ -1506,7 +1509,7 @@ func (t *tlogBackend) UnvettedPluginCmd(token []byte, pluginID, cmd, payload str // The token is optional. If a token is not provided then a tree ID // will not be provided to the plugin. var treeID int64 - if token != nil { + if len(token) > 0 { // Get tree ID treeID = t.unvettedTreeIDFromToken(token) @@ -1570,7 +1573,7 @@ func (t *tlogBackend) VettedPluginCmd(token []byte, pluginID, cmd, payload strin // will not be provided to the plugin. var treeID int64 var ok bool - if token != nil { + if len(token) > 0 { // Get tree ID treeID, ok = t.vettedTreeIDFromToken(token) if !ok { diff --git a/politeiad/client/client.go b/politeiad/client/client.go index b6107279b..d4f3d64a9 100644 --- a/politeiad/client/client.go +++ b/politeiad/client/client.go @@ -18,6 +18,7 @@ import ( // Client provides a client for interacting with the politeiad API. type Client struct { rpcHost string + rpcCert string rpcUser string rpcPass string http *http.Client @@ -28,9 +29,9 @@ type Client struct { // an error occurs. PluginID will only be populated if the error occured during // execution of a plugin command. type ErrorReply struct { - PluginID string - ErrorCode int - ErrorContext []string + PluginID string `json:"pluginid"` + ErrorCode int `json:"errorcode"` + ErrorContext []string `json:"errorcontext"` } // Error represents a politeiad error. Error is returned anytime the politeiad @@ -99,15 +100,17 @@ func (c *Client) makeReq(ctx context.Context, method string, route string, v int } // New returns a new politeiad client. -func New(rpcHost, rpcUser, rpcPass string, pid *identity.PublicIdentity) (*Client, error) { - h, err := util.NewHTTPClient(false, "") +func New(rpcHost, rpcCert, rpcUser, rpcPass string, pid *identity.PublicIdentity) (*Client, error) { + h, err := util.NewHTTPClient(false, rpcCert) if err != nil { return nil, err } return &Client{ rpcHost: rpcHost, + rpcCert: rpcCert, rpcUser: rpcUser, rpcPass: rpcPass, http: h, + pid: pid, }, nil } diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index c8196ac65..f9f66067a 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -64,19 +64,16 @@ var ( type ErrorCodeT int const ( - // TODO number error codes - ErrorCodeInvalid ErrorCodeT = 0 - ErrorCodePageSizeExceeded ErrorCodeT = iota - ErrorCodeFileNameInvalid - ErrorCodeIndexFileNameInvalid - ErrorCodeIndexFileCountInvalid - ErrorCodeIndexFileSizeInvalid - ErrorCodeTextFileCountInvalid - ErrorCodeImageFileCountInvalid - ErrorCodeImageFileSizeInvalid - ErrorCodeProposalMetadataInvalid - ErrorCodeProposalNameInvalid - ErrorCodeVoteStatusInvalid + ErrorCodeInvalid ErrorCodeT = 0 + ErrorCodeIndexFileNameInvalid ErrorCodeT = 1 + ErrorCodeIndexFileCountInvalid ErrorCodeT = 2 + ErrorCodeIndexFileSizeInvalid ErrorCodeT = 3 + ErrorCodeTextFileCountInvalid ErrorCodeT = 4 + ErrorCodeImageFileCountInvalid ErrorCodeT = 5 + ErrorCodeImageFileSizeInvalid ErrorCodeT = 6 + ErrorCodeProposalMetadataInvalid ErrorCodeT = 7 + ErrorCodeProposalNameInvalid ErrorCodeT = 8 + ErrorCodeVoteStatusInvalid ErrorCodeT = 9 ) var ( diff --git a/politeiad/plugins/user/user.go b/politeiad/plugins/user/user.go index 5ee3a447e..f0edeb1c1 100644 --- a/politeiad/plugins/user/user.go +++ b/politeiad/plugins/user/user.go @@ -31,17 +31,16 @@ const ( type ErrorCodeT int const ( - // TODO number // User error codes - ErrorCodeInvalid ErrorCodeT = iota - ErrorCodeUserMetadataNotFound - ErrorCodeUserIDInvalid - ErrorCodePublicKeyInvalid - ErrorCodeSignatureInvalid - ErrorCodeStatusChangeMetadataNotFound - ErrorCodeTokenInvalid - ErrorCodeStatusInvalid - ErrorCodeReasonInvalid + ErrorCodeInvalid ErrorCodeT = 0 + ErrorCodeUserMetadataNotFound ErrorCodeT = 1 + ErrorCodeUserIDInvalid ErrorCodeT = 2 + ErrorCodePublicKeyInvalid ErrorCodeT = 3 + ErrorCodeSignatureInvalid ErrorCodeT = 4 + ErrorCodeStatusChangeMetadataNotFound ErrorCodeT = 5 + ErrorCodeTokenInvalid ErrorCodeT = 6 + ErrorCodeStatusInvalid ErrorCodeT = 7 + ErrorCodeReasonInvalid ErrorCodeT = 8 ) var ( diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index efc379e2d..02b02d978 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -32,6 +32,7 @@ import ( "github.com/decred/politeia/politeiad/plugins/dcrdata" "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/ticketvote" + "github.com/decred/politeia/politeiad/plugins/user" "github.com/decred/politeia/util" "github.com/decred/politeia/util/version" "github.com/gorilla/mux" @@ -261,9 +262,9 @@ func (p *politeia) respondWithUserError(w http.ResponseWriter, errorCode v1.Erro }) } -func (p *politeia) respondWithPluginError(w http.ResponseWriter, plugin string, errorCode int, errorContext string) { +func (p *politeia) respondWithPluginError(w http.ResponseWriter, pluginID string, errorCode int, errorContext string) { util.RespondWithJSON(w, http.StatusBadRequest, v1.PluginErrorReply{ - PluginID: plugin, + PluginID: pluginID, ErrorCode: errorCode, ErrorContext: []string{errorContext}, }) @@ -314,8 +315,6 @@ func (p *politeia) newRecord(w http.ResponseWriter, r *http.Request) { return } - log.Infof("New record submitted %v", remoteAddr(r)) - md := convertFrontendMetadataStream(t.Metadata) files := convertFrontendFiles(t.Files) rm, err := p.backend.New(md, files) @@ -1237,50 +1236,6 @@ func (p *politeia) pluginCommand(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, reply) } -func convertBackendError(e error) error { - // Check for a plugin error - var pe backend.PluginError - if errors.As(e, &pe) { - return v1.PluginErrorReply{ - PluginID: pe.PluginID, - ErrorCode: pe.ErrorCode, - ErrorContext: []string{pe.ErrorContext}, - } - } - - // Check for a backend error - var s v1.ErrorStatusT - switch e { - case backend.ErrRecordNotFound: - s = v1.ErrorStatusRecordNotFound - case backend.ErrRecordFound: - // Intentionally omitted - case backend.ErrFileNotFound: - // Intentionally omitted - case backend.ErrNoChanges: - s = v1.ErrorStatusNoChanges - case backend.ErrChangesRecord: - // Intentionally omitted - case backend.ErrRecordLocked: - s = v1.ErrorStatusRecordLocked - case backend.ErrJournalsNotReplayed: - // Intentionally omitted - case backend.ErrPluginInvalid: - // Intentionally omitted - case backend.ErrPluginCmdInvalid: - // Intentionally omitted - } - if s != v1.ErrorStatusInvalid { - return v1.UserErrorReply{ - ErrorCode: s, - } - } - - return v1.ServerErrorReply{ - ErrorCode: time.Now().Unix(), - } -} - func (p *politeia) pluginCommandBatch(w http.ResponseWriter, r *http.Request) { var pcb v1.PluginCommandBatch decoder := json.NewDecoder(r.Body) @@ -1329,18 +1284,43 @@ func (p *politeia) pluginCommandBatch(w http.ResponseWriter, r *http.Request) { continue } if err != nil { - // Add error to reply - e := convertBackendError(err) - replies[k] = v1.PluginCommandReplyV2{ - Error: e, - } - - // Log any internal server errors - var se v1.ServerErrorReply - if errors.As(e, &se) { - log.Errorf("%v %v: backend plugin failed: pluginID:%v cmd:%v "+ - "payload:%v err:%v", remoteAddr(r), se.ErrorCode, pc.ID, + var e backend.PluginError + switch { + case errors.As(err, &e): + log.Infof("%v batched plugin cmd user error: %v %v", + remoteAddr(r), e.PluginID, e.ErrorCode) + + replies[k] = v1.PluginCommandReplyV2{ + Error: v1.PluginErrorReply{ + PluginID: e.PluginID, + ErrorCode: e.ErrorCode, + ErrorContext: []string{e.ErrorContext}, + }, + } + case err == backend.ErrRecordNotFound: + replies[k] = v1.PluginCommandReplyV2{ + Error: v1.UserErrorReply{ + ErrorCode: v1.ErrorStatusRecordNotFound, + }, + } + case err == backend.ErrRecordLocked: + replies[k] = v1.PluginCommandReplyV2{ + Error: v1.UserErrorReply{ + ErrorCode: v1.ErrorStatusRecordLocked, + }, + } + default: + // Unkown error. Log is as an internal server error. + t := time.Now().Unix() + log.Errorf("%v %v: batched plugin cmd failed: pluginID:%v "+ + "cmd:%v payload:%v err:%v", remoteAddr(r), t, pc.ID, pc.Command, pc.Payload, err) + + replies[k] = v1.PluginCommandReplyV2{ + Error: v1.ServerErrorReply{ + ErrorCode: t, + }, + } } continue @@ -1629,6 +1609,11 @@ func _main() error { Settings: ps, Identity: p.identity, } + case user.PluginID: + plugin = backend.Plugin{ + ID: user.PluginID, + Settings: ps, + } case decredplugin.ID: // TODO plugin setup for cms case cmsplugin.ID: diff --git a/politeiad/util/util.go b/politeiad/util/util.go deleted file mode 100644 index 68ece2635..000000000 --- a/politeiad/util/util.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) 2020-2021 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package util - -import ( - "bytes" - "crypto/sha256" - "encoding/base64" - "encoding/hex" - "fmt" - - "github.com/decred/dcrtime/merkle" - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/util" -) - -// MerkleRoot computes and returns the merkle root of the backend files. -func MerkleRoot(files []backend.File) (*[sha256.Size]byte, error) { - digests := make([]*[sha256.Size]byte, 0, len(files)) - for _, v := range files { - // Decode digest - digest, err := hex.DecodeString(v.Digest) - if err != nil { - return nil, err - } - - // Decode payload - payload, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return nil, err - } - - // Verify digest - d := util.Digest(payload) - if bytes.Equal(digest, d) { - return nil, fmt.Errorf("invalid digest for payload: got %x, want %x", - digest, d) - } - - // Save digest - var s [sha256.Size]byte - copy(s[:], d) - digests = append(digests, &s) - } - - // Calc merkle root - return merkle.Root(digests), nil -} diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 63f8d3b28..46a59aa29 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -89,7 +89,8 @@ func (e ServerErrorReply) Error() string { return fmt.Sprintf("server error: %v", e.ErrorCode) } -// Policy requests the policy settings for the pi API. +// Policy requests the policy settings for the pi API. It includes the policy +// guidlines for the contents of a proposal record. type Policy struct{} // PolicyReply is the reply to the Policy command. diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go index 6d1a4c46f..570c37c8f 100644 --- a/politeiawww/api/records/v1/v1.go +++ b/politeiawww/api/records/v1/v1.go @@ -32,20 +32,47 @@ type ErrorCodeT int const ( // Error codes - ErrorCodeInvalid ErrorCodeT = 0 - ErrorCodeInputInvalid ErrorCodeT = iota - ErrorCodePublicKeyInvalid - ErrorCodeSignatureInvalid - ErrorCodeRecordStateInvalid - ErrorCodeRecordNotFound - ErrorCodePageSizeExceeded + ErrorCodeInvalid ErrorCodeT = 0 + ErrorCodeInputInvalid ErrorCodeT = 1 + ErrorCodeFileNameInvalid ErrorCodeT = 2 + ErrorCodeFileMIMEInvalid ErrorCodeT = 3 + ErrorCodeFileDigestInvalid ErrorCodeT = 4 + ErrorCodeFilePayloadInvalid ErrorCodeT = 5 + ErrorCodeMetadataStreamIDInvalid ErrorCodeT = 6 + ErrorCodeMetadataStreamPayloadInvalid ErrorCodeT = 8 + ErrorCodePublicKeyInvalid ErrorCodeT = 9 + ErrorCodeSignatureInvalid ErrorCodeT = 10 + ErrorCodeRecordTokenInvalid ErrorCodeT = 11 + ErrorCodeRecordStateInvalid ErrorCodeT = 12 + ErrorCodeRecordNotFound ErrorCodeT = 13 + ErrorCodeRecordLocked ErrorCodeT = 14 + ErrorCodeNoRecordChanges ErrorCodeT = 15 + ErrorCodeRecordStatusInvalid ErrorCodeT = 16 + ErrorCodeStatusReasonNotFound ErrorCodeT = 17 + ErrorCodePageSizeExceeded ErrorCodeT = 18 ) var ( // ErrorCodes contains the human readable errors. ErrorCodes = map[ErrorCodeT]string{ - ErrorCodeInvalid: "error invalid", - ErrorCodeInputInvalid: "input invalid", + ErrorCodeInvalid: "error invalid", + ErrorCodeInputInvalid: "input invalid", + ErrorCodeFileNameInvalid: "file name invalid", + ErrorCodeFileMIMEInvalid: "file mime invalid", + ErrorCodeFileDigestInvalid: "file digest invalid", + ErrorCodeFilePayloadInvalid: "file payload invalid", + ErrorCodeMetadataStreamIDInvalid: "mdstream id invalid", + ErrorCodeMetadataStreamPayloadInvalid: "mdstream payload invalid", + ErrorCodePublicKeyInvalid: "public key invalid", + ErrorCodeSignatureInvalid: "signature invalid", + ErrorCodeRecordTokenInvalid: "record token invalid", + ErrorCodeRecordStateInvalid: "record state invalid", + ErrorCodeRecordNotFound: "record not found", + ErrorCodeRecordLocked: "record locked", + ErrorCodeNoRecordChanges: "no record changes", + ErrorCodeRecordStatusInvalid: "record status invalid", + ErrorCodeStatusReasonNotFound: "status reason not found", + ErrorCodePageSizeExceeded: "page size exceeded", } ) diff --git a/politeiawww/client/client.go b/politeiawww/client/client.go new file mode 100644 index 000000000..fc7767b51 --- /dev/null +++ b/politeiawww/client/client.go @@ -0,0 +1,165 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package client + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/http/cookiejar" + "net/url" + "reflect" + + "github.com/decred/politeia/util" + "github.com/gorilla/schema" + "golang.org/x/net/publicsuffix" +) + +var ( + headerCSRF = "X-CSRF-Token" +) + +// Client provides a client for interacting with the politeiawww API. +type Client struct { + host string + cert string + headerCSRF string // Header csrf token + http *http.Client +} + +// ErrorReply represents the request body that is returned from politeiawww +// when an error occurs. PluginID will only be populated if the error occured +// during execution of a plugin command. +type ErrorReply struct { + PluginID string + ErrorCode int + ErrorContext string +} + +// httpsError represents a politeiawww error. Error is returned anytime the +// politeiawww response is not a 200. +type Error struct { + HTTPCode int + ErrorReply ErrorReply +} + +// Error satisfies the error interface. +func (e Error) Error() string { + switch { + case e.HTTPCode == http.StatusNotFound: + return fmt.Sprintf("404 not found") + case e.ErrorReply.PluginID != "": + return fmt.Sprintf("politeiawww plugin error: %v %v %v", + e.HTTPCode, e.ErrorReply.PluginID, e.ErrorReply.ErrorCode) + default: + return fmt.Sprintf("politeiawww error: %v %v", + e.HTTPCode, e.ErrorReply.ErrorCode) + } +} + +// makeReq makes a politeiawww http request to the method and route provided, +// serializing the provided object as the request body, and returning a byte +// slice of the repsonse body. An Error is returned if politeiawww responds +// with anything other than a 200 http status code. +func (c *Client) makeReq(method string, route string, v interface{}) ([]byte, error) { + // Serialize body + var ( + reqBody []byte + queryParams string + err error + ) + if v != nil { + switch method { + case http.MethodGet: + // Use reflection in case the interface value is nil but the + // interface type is not. This can happen when query params + // exist but are not used. + if reflect.ValueOf(v).IsNil() { + break + } + + // Populate GET request query params + form := url.Values{} + if err := schema.NewEncoder().Encode(v, form); err != nil { + return nil, err + } + queryParams = "?" + form.Encode() + + case http.MethodPost, http.MethodPut: + reqBody, err = json.Marshal(v) + if err != nil { + return nil, err + } + + default: + return nil, fmt.Errorf("unknown http method '%v'", method) + } + } + + // Send request + fullRoute := c.host + route + queryParams + req, err := http.NewRequest(method, fullRoute, bytes.NewReader(reqBody)) + if err != nil { + return nil, err + } + if c.headerCSRF != "" { + req.Header.Add(headerCSRF, c.headerCSRF) + } + r, err := c.http.Do(req) + if err != nil { + return nil, err + } + defer r.Body.Close() + + // Handle reply + if r.StatusCode != http.StatusOK { + var e ErrorReply + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&e); err != nil { + return nil, fmt.Errorf("status code %v: %v", r.StatusCode, err) + } + return nil, Error{ + HTTPCode: r.StatusCode, + ErrorReply: e, + } + } + + respBody := util.ConvertBodyToByteArray(r.Body, false) + return respBody, nil +} + +// New returns a new politeiawww client. +func New(host, cert string, cookies []*http.Cookie, headerCSRF string) (*Client, error) { + // Setup http client + h, err := util.NewHTTPClient(false, cert) + if err != nil { + return nil, err + } + + // Setup cookies + if cookies != nil { + opt := cookiejar.Options{ + PublicSuffixList: publicsuffix.List, + } + jar, err := cookiejar.New(&opt) + if err != nil { + return nil, err + } + u, err := url.Parse(host) + if err != nil { + return nil, err + } + jar.SetCookies(u, cookies) + h.Jar = jar + } + + return &Client{ + host: host, + cert: cert, + headerCSRF: headerCSRF, + http: h, + }, nil +} diff --git a/politeiawww/client/pi.go b/politeiawww/client/pi.go new file mode 100644 index 000000000..61680b8ef --- /dev/null +++ b/politeiawww/client/pi.go @@ -0,0 +1,29 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package client + +import ( + "encoding/json" + "net/http" + + piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" +) + +// PiPolicy sends a pi v1 Policy request to politeiawww. +func (c *Client) PiPolicy() (*piv1.PolicyReply, error) { + route := piv1.APIRoute + piv1.RoutePolicy + resBody, err := c.makeReq(http.MethodGet, route, nil) + if err != nil { + return nil, err + } + + var pr piv1.PolicyReply + err = json.Unmarshal(resBody, &pr) + if err != nil { + return nil, err + } + + return &pr, nil +} diff --git a/politeiawww/client/records.go b/politeiawww/client/records.go new file mode 100644 index 000000000..d3f4268b6 --- /dev/null +++ b/politeiawww/client/records.go @@ -0,0 +1,29 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package client + +import ( + "encoding/json" + "net/http" + + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" +) + +// RecordNew sends a records v1 New request to politeiawww. +func (c *Client) RecordNew(n rcv1.New) (*rcv1.NewReply, error) { + route := rcv1.APIRoute + rcv1.RouteNew + resBody, err := c.makeReq(http.MethodPost, route, n) + if err != nil { + return nil, err + } + + var nr rcv1.NewReply + err = json.Unmarshal(resBody, &nr) + if err != nil { + return nil, err + } + + return &nr, nil +} diff --git a/politeiawww/cmd/pictl/proposalnew.go b/politeiawww/cmd/pictl/proposalnew.go index 8b4208939..719d983a0 100644 --- a/politeiawww/cmd/pictl/proposalnew.go +++ b/politeiawww/cmd/pictl/proposalnew.go @@ -14,8 +14,10 @@ import ( "time" "github.com/decred/politeia/politeiad/api/v1/mime" + piplugin "github.com/decred/politeia/politeiad/plugins/pi" + piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" - v1 "github.com/decred/politeia/politeiawww/api/www/v1" + pclient "github.com/decred/politeia/politeiawww/client" "github.com/decred/politeia/politeiawww/cmd/shared" "github.com/decred/politeia/util" ) @@ -46,69 +48,77 @@ type proposalNewCmd struct { // Execute executes the proposalNewCmd command. // // This function satisfies the go-flags Commander interface. -func (cmd *proposalNewCmd) Execute(args []string) error { +func (c *proposalNewCmd) Execute(args []string) error { // Unpack args - indexFile := cmd.Args.IndexFile - attachments := cmd.Args.Attachments + indexFile := c.Args.IndexFile + attachments := c.Args.Attachments - // Verify args + // Verify args and flags switch { - case !cmd.Random && indexFile == "": - return fmt.Errorf("index file not found; you must either provide an " + - "index.md file or use --random") + case !c.Random && indexFile == "": + return fmt.Errorf("index file not found; you must either " + + "provide an index.md file or use --random") - case cmd.Random && indexFile != "": - return fmt.Errorf("you cannot provide file arguments and use the " + - "--random flag at the same time") + case c.Random && indexFile != "": + return fmt.Errorf("you cannot provide file arguments and use " + + "the --random flag at the same time") - case !cmd.Random && cmd.Name == "": - return fmt.Errorf("you must either provide a proposal name using the " + - "--name flag or use the --random flag to generate a random name") + case !c.Random && c.Name == "": + return fmt.Errorf("you must either provide a proposal name " + + "using the --name flag or use the --random flag to generate " + + "a random name") - case cmd.RFP && cmd.LinkBy != 0: - return fmt.Errorf("you cannot use both the --rfp and --linkby flags " + - "at the same time") + case c.RFP && c.LinkBy != 0: + return fmt.Errorf("you cannot use both the --rfp and --linkby " + + "flags at the same time") } // Check for user identity. A user identity is required to sign - // the proposal files and metadata. + // the proposal files. if cfg.Identity == nil { return shared.ErrUserIdentityNotFound } - // Get pi policy + // Setup client + pc, err := pclient.New(cfg.Host, cfg.HTTPSCert, cfg.Cookies, cfg.CSRF) + if err != nil { + return err + } - // Prepare index file - var ( - file *rcv1.File - err error - ) - if cmd.Random { + // Get the pi policy. It contains the proposal requirements. + pr, err := pc.PiPolicy() + if err != nil { + return err + } + + // Setup index file + var file *rcv1.File + if c.Random { // Generate random text for the index file file, err = createMDFile() if err != nil { return err } } else { - // Read the index file from disk + // Read index file from disk fp := util.CleanAndExpandPath(indexFile) var err error payload, err := ioutil.ReadFile(fp) if err != nil { return fmt.Errorf("ReadFile %v: %v", fp, err) } - file = &pi.File{ - Name: v1.PolicyIndexFilename, + file = &rcv1.File{ + Name: piplugin.FileNameIndexFile, MIME: mime.DetectMimeType(payload), Digest: hex.EncodeToString(util.Digest(payload)), Payload: base64.StdEncoding.EncodeToString(payload), } } - files := []pi.File{ + files := []rcv1.File{ *file, } - // Prepare attachment files + // Setup attachment files for _, fn := range attachments { fp := util.CleanAndExpandPath(fn) payload, err := ioutil.ReadFile(fp) @@ -116,7 +126,7 @@ func (cmd *proposalNewCmd) Execute(args []string) error { return fmt.Errorf("ReadFile %v: %v", fp, err) } - files = append(files, pi.File{ + files = append(files, rcv1.File{ Name: filepath.Base(fn), MIME: mime.DetectMimeType(payload), Digest: hex.EncodeToString(util.Digest(payload)), @@ -124,70 +134,84 @@ func (cmd *proposalNewCmd) Execute(args []string) error { }) } - // Setup metadata - if cmd.Random { - r, err := util.Random(v1.PolicyMinProposalNameLength) + // Setup proposal metadata + if c.Random { + r, err := util.Random(int(pr.NameLengthMin)) if err != nil { return err } - cmd.Name = hex.EncodeToString(r) - } - if cmd.RFP { - // Set linkby to a month from now - cmd.LinkBy = time.Now().Add(time.Hour * 24 * 30).Unix() + c.Name = hex.EncodeToString(r) } - pm := pi.ProposalMetadata{ - Name: cmd.Name, - LinkTo: cmd.LinkTo, - LinkBy: cmd.LinkBy, + pm := piv1.ProposalMetadata{ + Name: c.Name, } pmb, err := json.Marshal(pm) if err != nil { return err } - metadata := []pi.Metadata{ - { - Digest: hex.EncodeToString(util.Digest(pmb)), - Hint: pi.HintProposalMetadata, - Payload: base64.StdEncoding.EncodeToString(pmb), - }, + files = append(files, rcv1.File{ + Name: piv1.FileNameProposalMetadata, + MIME: mime.DetectMimeType(pmb), + Digest: hex.EncodeToString(util.Digest(pmb)), + Payload: base64.StdEncoding.EncodeToString(pmb), + }) + + // Setup vote metadata + if c.RFP { + // Set linkby to a month from now + c.LinkBy = time.Now().Add(time.Hour * 24 * 30).Unix() + } + if c.LinkBy != 0 || c.LinkTo != "" { + vm := piv1.VoteMetadata{ + LinkTo: c.LinkTo, + LinkBy: c.LinkBy, + } + vmb, err := json.Marshal(vm) + if err != nil { + return err + } + files = append(files, rcv1.File{ + Name: piv1.FileNameProposalMetadata, + MIME: mime.DetectMimeType(vmb), + Digest: hex.EncodeToString(util.Digest(vmb)), + Payload: base64.StdEncoding.EncodeToString(vmb), + }) } - // Setup new proposal request - sig, err := signedMerkleRoot(files, metadata, cfg.Identity) + // Setup request + sig, err := signedMerkleRoot(files, cfg.Identity) if err != nil { return err } - pn := pi.ProposalNew{ + n := rcv1.New{ Files: files, - Metadata: metadata, - PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), + PublicKey: cfg.Identity.Public.String(), Signature: sig, } // Send request. The request and response details are printed to // the console based on the logging flags that were used. - err = shared.PrintJSON(pn) + err = shared.PrintJSON(n) if err != nil { return err } - pnr, err := client.ProposalNew(pn) + nr, err := pc.RecordNew(n) if err != nil { return err } - err = shared.PrintJSON(pnr) + err = shared.PrintJSON(nr) if err != nil { return err } - // Verify proposal + // Verify record vr, err := client.Version() if err != nil { return err } - err = verifyProposal(pnr.Proposal, vr.PubKey) + err = verifyRecord(nr.Record, vr.PubKey) if err != nil { - return fmt.Errorf("unable to verify proposal: %v", err) + return fmt.Errorf("unable to verify record: %v", err) } return nil @@ -196,23 +220,21 @@ func (cmd *proposalNewCmd) Execute(args []string) error { // proposalNewHelpMsg is the output of the help command. const proposalNewHelpMsg = `proposalnew [flags] "indexfile" "attachments" -Submit a new proposal to Politeia. A proposal is defined as a single markdown -file with the filename "index.md" and optional attachment png files. No other -file types are allowed. +Submit a new proposal to Politeia. A proposal can be submitted as an RFP (Request for Proposals) by using either the --rfp flag or by manually specifying a link by deadline using the --linkby flag. Only one of these flags can be used at a time. A proposal can be submitted as an RFP submission by using the --linkto flag -to link to and existing RFP proposal. +to link to and an existing RFP proposal. Arguments: 1. indexfile (string, required) Index file 2. attachments (string, optional) Attachment files Flags: - --name (string, optional) The name of the proposal. + --name (string, optional) Name of the proposal. --linkto (string, optional) Token of an existing public proposal to link to. --linkby (int64, optional) UNIX timestamp of the RFP deadline. Setting this field will make the proposal an RFP with a diff --git a/politeiawww/cmd/pictl/sample-pictl.conf b/politeiawww/cmd/pictl/sample-pictl.conf new file mode 100644 index 000000000..c5542ffc6 --- /dev/null +++ b/politeiawww/cmd/pictl/sample-pictl.conf @@ -0,0 +1,6 @@ +[Application Options] + +; appdata=~/.pictl +; httpscert=~/.politeiawww/https.cert +; host=https://127.0.0.1:4443 +; skipverify=false diff --git a/politeiawww/cmd/pictl/sample-piwww.conf b/politeiawww/cmd/pictl/sample-piwww.conf deleted file mode 100644 index 42c1c0fac..000000000 --- a/politeiawww/cmd/pictl/sample-piwww.conf +++ /dev/null @@ -1,6 +0,0 @@ -[Application Options] - -; appdata=~/.piwww -; host=https://proposals.decred.org/api -; testnet=false -; skipverify=false diff --git a/politeiawww/cmd/pictl/util.go b/politeiawww/cmd/pictl/util.go index 7f766dbf5..6ad7b6786 100644 --- a/politeiawww/cmd/pictl/util.go +++ b/politeiawww/cmd/pictl/util.go @@ -13,28 +13,27 @@ import ( "github.com/decred/dcrd/chaincfg/chainhash" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/api/v1/mime" - pi "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" v1 "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/util" ) -// signedMerkleRoot calculates the merkle root of the passed in list of files -// and metadata, signs the merkle root with the passed in identity and returns -// the signature. -func signedMerkleRoot(files []rcv1.File, id *identity.FullIdentity) (string, error) { - /* - if len(files) == 0 { - return "", fmt.Errorf("no proposal files found") - } - mr, err := utilwww.MerkleRoot(files, md) - if err != nil { - return "", err - } - sig := id.SignMessage([]byte(mr)) - return hex.EncodeToString(sig[:]), nil - */ - return "", fmt.Errorf("not implemented") +// signedMerkleRoot returns the signed merkle root of the provided files. The +// signature is created using the provided identity. +func signedMerkleRoot(files []rcv1.File, fid *identity.FullIdentity) (string, error) { + if len(files) == 0 { + return "", fmt.Errorf("no proposal files found") + } + digests := make([]string, 0, len(files)) + for _, v := range files { + digests = append(digests, v.Digest) + } + mr, err := util.MerkleRoot(digests) + if err != nil { + return "", err + } + sig := fid.SignMessage(mr[:]) + return hex.EncodeToString(sig[:]), nil } // convertTicketHashes converts a slice of hexadecimal ticket hashes into @@ -53,10 +52,9 @@ func convertTicketHashes(h []string) ([][]byte, error) { // createMDFile returns a File object that was created using a markdown file // filled with random text. -func createMDFile() (*pi.File, error) { +// TODO fill to max size +func createMDFile() (*rcv1.File, error) { var b bytes.Buffer - b.WriteString("This is the proposal title\n") - for i := 0; i < 10; i++ { r, err := util.Random(32) if err != nil { @@ -65,7 +63,7 @@ func createMDFile() (*pi.File, error) { b.WriteString(base64.StdEncoding.EncodeToString(r) + "\n") } - return &pi.File{ + return &rcv1.File{ Name: v1.PolicyIndexFilename, MIME: mime.DetectMimeType(b.Bytes()), Digest: hex.EncodeToString(util.Digest(b.Bytes())), @@ -73,8 +71,9 @@ func createMDFile() (*pi.File, error) { }, nil } +// verifyDigests verifies that all file digests match the calculated SHA256 +// digests of the file payloads. func verifyDigests(files []rcv1.File) error { - // Validate file digests for _, f := range files { b, err := base64.StdEncoding.DecodeString(f.Payload) if err != nil { @@ -92,57 +91,44 @@ func verifyDigests(files []rcv1.File) error { f.Name) } } - return nil } func verifyRecord(r rcv1.Record, serverPubKey string) error { - /* - if len(p.Files) > 0 { - // Verify digests - err := verifyDigests(p.Files, p.Metadata) - if err != nil { - return err - } - // Verify merkle root - mr, err := utilwww.MerkleRoot(p.Files, p.Metadata) - if err != nil { - return err - } - // Check if merkle roots match - if mr != p.CensorshipRecord.Merkle { - return fmt.Errorf("merkle roots do not match") - } - } - - // Verify proposal signature - pid, err := util.IdentityFromString(p.PublicKey) + if len(r.Files) > 0 { + // Verify digests + err := verifyDigests(r.Files) if err != nil { return err } - sig, err := util.ConvertSignature(p.Signature) - if err != nil { - return err + // Verify merkle root + digests := make([]string, 0, len(r.Files)) + for _, v := range r.Files { + digests = append(digests, v.Digest) } - if !pid.VerifyMessage([]byte(p.CensorshipRecord.Merkle), sig) { - return fmt.Errorf("invalid proposal signature") - } - - // Verify censorship record signature - id, err := util.IdentityFromString(serverPubKey) - if err != nil { - return err - } - s, err := util.ConvertSignature(p.CensorshipRecord.Signature) + mr, err := util.MerkleRoot(digests) if err != nil { return err } - msg := []byte(p.CensorshipRecord.Merkle + p.CensorshipRecord.Token) - if !id.VerifyMessage(msg, s) { - return fmt.Errorf("invalid censorship record signature") + // Check if merkle roots match + if hex.EncodeToString(mr[:]) != r.CensorshipRecord.Merkle { + return fmt.Errorf("merkle roots do not match") } + } - return nil - */ - return fmt.Errorf("not implemented") + // Verify censorship record signature + id, err := util.IdentityFromString(serverPubKey) + if err != nil { + return err + } + s, err := util.ConvertSignature(r.CensorshipRecord.Signature) + if err != nil { + return err + } + msg := []byte(r.CensorshipRecord.Merkle + r.CensorshipRecord.Token) + if !id.VerifyMessage(msg, s) { + return fmt.Errorf("invalid censorship record signature") + } + + return nil } diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index 3ddde9038..f610e326c 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -2517,15 +2517,13 @@ func (c *Client) Close() { // NewClient returns a new politeiawww client. func NewClient(cfg *Config) (*Client, error) { - // Create http client - tlsConfig := &tls.Config{ - InsecureSkipVerify: cfg.SkipVerify, - } - tr := &http.Transport{ - TLSClientConfig: tlsConfig, + // Setup http client + httpClient, err := util.NewHTTPClient(cfg.SkipVerify, cfg.HTTPSCert) + if err != nil { + return nil, err } - // Set cookies + // Setup cookies jar, err := cookiejar.New(&cookiejar.Options{ PublicSuffixList: publicsuffix.List, }) @@ -2537,10 +2535,7 @@ func NewClient(cfg *Config) (*Client, error) { return nil, err } jar.SetCookies(u, cfg.Cookies) - httpClient := &http.Client{ - Transport: tr, - Jar: jar, - } + httpClient.Jar = jar return &Client{ http: httpClient, diff --git a/politeiawww/cmd/shared/config.go b/politeiawww/cmd/shared/config.go index d587075ca..57910e68c 100644 --- a/politeiawww/cmd/shared/config.go +++ b/politeiawww/cmd/shared/config.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2020 The Decred developers +// Copyright (c) 2017-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -18,13 +18,14 @@ import ( "github.com/decred/dcrd/dcrutil/v3" "github.com/decred/politeia/politeiad/api/v1/identity" + "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/util" "github.com/decred/politeia/util/version" flags "github.com/jessevdk/go-flags" ) const ( - defaultHost = "https://proposals.decred.org/api" + defaultHost = "https://127.0.0.1:4443" defaultFaucetHost = "https://faucet.decred.org/requestfaucet" defaultWalletHost = "127.0.0.1" defaultWalletTestnetPort = "19111" @@ -38,6 +39,7 @@ const ( ) var ( + defaultHTTPSCert = config.DefaultHTTPSCertFile dcrwalletHomeDir = dcrutil.AppDataDir("dcrwallet", false) defaultWalletCertFile = filepath.Join(dcrwalletHomeDir, "rpc.cert") ) @@ -46,6 +48,7 @@ var ( type Config struct { HomeDir string `long:"appdata" description:"Path to application home directory"` Host string `long:"host" description:"politeiawww host"` + HTTPSCert string `long:"httpscert" description:"politeiawww https cert"` RawJSON bool `short:"j" long:"json" description:"Print raw JSON output"` ShowVersion bool `short:"V" long:"version" description:"Display version information and exit"` SkipVerify bool `long:"skipverify" description:"Skip verifying the server's certifcate chain and host name"` @@ -85,6 +88,7 @@ func LoadConfig(homeDir, dataDirname, configFilename string) (*Config, error) { HomeDir: homeDir, DataDir: filepath.Join(homeDir, dataDirname), Host: defaultHost, + HTTPSCert: defaultHTTPSCert, WalletHost: defaultWalletHost + ":" + defaultWalletTestnetPort, WalletCert: defaultWalletCertFile, FaucetHost: defaultFaucetHost, @@ -127,7 +131,7 @@ func LoadConfig(homeDir, dataDirname, configFilename string) (*Config, error) { if err != nil { var e *os.PathError if errors.As(err, &e) { - fmt.Printf("Warning: no config file found at %v\n", cfgFile) + // No config file found. Do nothing. } else { return nil, fmt.Errorf("parsing config file: %v", err) } diff --git a/politeiawww/config.go b/politeiawww/config.go index 38ed5572f..60abc2b0e 100644 --- a/politeiawww/config.go +++ b/politeiawww/config.go @@ -76,7 +76,6 @@ const ( var ( defaultHTTPSKeyFile = filepath.Join(config.DefaultHomeDir, "https.key") - defaultHTTPSCertFile = filepath.Join(config.DefaultHomeDir, "https.cert") defaultRPCCertFile = filepath.Join(config.DefaultHomeDir, "rpc.cert") defaultCookieKeyFile = filepath.Join(config.DefaultHomeDir, "cookie.key") defaultLogDir = filepath.Join(config.DefaultHomeDir, defaultLogDirname) @@ -281,7 +280,7 @@ func loadConfig() (*config.Config, []string, error) { DataDir: config.DefaultDataDir, LogDir: defaultLogDir, HTTPSKey: defaultHTTPSKeyFile, - HTTPSCert: defaultHTTPSCertFile, + HTTPSCert: config.DefaultHTTPSCertFile, RPCCert: defaultRPCCertFile, CookieKeyFile: defaultCookieKeyFile, Version: version.String(), @@ -354,7 +353,7 @@ func loadConfig() (*config.Config, []string, error) { } else { cfg.HTTPSKey = preCfg.HTTPSKey } - if preCfg.HTTPSCert == defaultHTTPSCertFile { + if preCfg.HTTPSCert == config.DefaultHTTPSCertFile { cfg.HTTPSCert = filepath.Join(cfg.HomeDir, "https.cert") } else { cfg.HTTPSCert = preCfg.HTTPSCert diff --git a/politeiawww/config/config.go b/politeiawww/config/config.go index b2cf2c8df..08dc3227a 100644 --- a/politeiawww/config/config.go +++ b/politeiawww/config/config.go @@ -38,6 +38,10 @@ var ( // DefaultDataDir points to politeiawww's default data directory // path. DefaultDataDir = filepath.Join(DefaultHomeDir, DefaultDataDirname) + + // DefaultHTTPSCertFile contains the file path to the politeiawww + // https certificate. + DefaultHTTPSCertFile = filepath.Join(DefaultHomeDir, "https.cert") ) // Config defines the configuration options for politeiawww. diff --git a/politeiawww/records/error.go b/politeiawww/records/error.go index 00812b944..4a92a37be 100644 --- a/politeiawww/records/error.go +++ b/politeiawww/records/error.go @@ -18,20 +18,57 @@ import ( "github.com/decred/politeia/util" ) -func convertRecordsErrorCode(errCode int) v1.ErrorCodeT { +func convertPDErrorCode(errCode int) v1.ErrorCodeT { + // Any error statuses that are intentionally omitted means that + // politeiawww should 500. switch pdv1.ErrorStatusT(errCode) { case pdv1.ErrorStatusInvalidRequestPayload: - // Intentionally omitted. This indicates an internal server error. + // Intentionally omitted case pdv1.ErrorStatusInvalidChallenge: - // Intentionally omitted. This indicates an internal server error. + // Intentionally omitted + case pdv1.ErrorStatusInvalidFilename: + return v1.ErrorCodeFileNameInvalid + case pdv1.ErrorStatusInvalidFileDigest: + return v1.ErrorCodeFileDigestInvalid + case pdv1.ErrorStatusInvalidBase64: + return v1.ErrorCodeFilePayloadInvalid + case pdv1.ErrorStatusInvalidMIMEType: + return v1.ErrorCodeFileMIMEInvalid + case pdv1.ErrorStatusUnsupportedMIMEType: + return v1.ErrorCodeFileMIMEInvalid + case pdv1.ErrorStatusInvalidRecordStatusTransition: + return v1.ErrorCodeRecordStatusInvalid + case pdv1.ErrorStatusEmpty: + return v1.ErrorCodeRecordStatusInvalid + case pdv1.ErrorStatusInvalidMDID: + return v1.ErrorCodeMetadataStreamIDInvalid + case pdv1.ErrorStatusDuplicateMDID: + return v1.ErrorCodeMetadataStreamIDInvalid + case pdv1.ErrorStatusDuplicateFilename: + return v1.ErrorCodeFileNameInvalid + case pdv1.ErrorStatusFileNotFound: + // Intentionally omitted + case pdv1.ErrorStatusNoChanges: + return v1.ErrorCodeNoRecordChanges + case pdv1.ErrorStatusRecordFound: + // Intentionally omitted + case pdv1.ErrorStatusInvalidRPCCredentials: + // Intentionally omitted case pdv1.ErrorStatusRecordNotFound: return v1.ErrorCodeRecordNotFound + case pdv1.ErrorStatusInvalidToken: + return v1.ErrorCodeRecordTokenInvalid + case pdv1.ErrorStatusRecordLocked: + return v1.ErrorCodeRecordLocked + case pdv1.ErrorStatusInvalidRecordState: + return v1.ErrorCodeRecordStateInvalid } - // No record API error code found return v1.ErrorCodeInvalid } func respondWithError(w http.ResponseWriter, r *http.Request, format string, err error) { + log.Tracef("respondWithError: %v %v", format, err) + var ( ue v1.UserErrorReply pe pdclient.Error @@ -59,7 +96,7 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err errCode = pe.ErrorReply.ErrorCode errContext = pe.ErrorReply.ErrorContext ) - e := convertRecordsErrorCode(errCode) + e := convertPDErrorCode(errCode) switch { case pluginID != "": // politeiad plugin error. Log it and return a 400. diff --git a/politeiawww/www.go b/politeiawww/www.go index e536e5bf2..26a70b25c 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -279,8 +279,7 @@ func RespondWithError(w http.ResponseWriter, r *http.Request, userHttpCode int, } // addRoute sets up a handler for a specific method+route. If method is not -// specified it adds a websocket. The routeVersion should be in the format -// "/v1". +// specified it adds a websocket. func (p *politeiawww) addRoute(method string, routeVersion string, route string, handler http.HandlerFunc, perm permission) { fullRoute := routeVersion + route @@ -622,8 +621,8 @@ func _main() error { auth.Use(csrfMiddleware) // Setup the politeiad client - pdc, err := pdclient.New(loadedCfg.RPCHost, loadedCfg.RPCUser, - loadedCfg.RPCPass, loadedCfg.Identity) + pdc, err := pdclient.New(loadedCfg.RPCHost, loadedCfg.RPCCert, + loadedCfg.RPCUser, loadedCfg.RPCPass, loadedCfg.Identity) if err != nil { return err } diff --git a/util/merkle.go b/util/merkle.go new file mode 100644 index 000000000..d18ef7d91 --- /dev/null +++ b/util/merkle.go @@ -0,0 +1,33 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package util + +import ( + "crypto/sha256" + "encoding/hex" + + "github.com/decred/dcrtime/merkle" +) + +// MerkleRoot computes and returns the merkle root of the provided digests. +// The digests should be hex encoded SHA256 digests. +func MerkleRoot(digests []string) (*[sha256.Size]byte, error) { + sha := make([]*[sha256.Size]byte, 0, len(digests)) + for _, v := range digests { + // Decode digest + d, err := hex.DecodeString(v) + if err != nil { + return nil, err + } + + // Save digest + var s [sha256.Size]byte + copy(s[:], d) + sha = append(sha, &s) + } + + // Calc merkle root + return merkle.Root(sha), nil +} From efcada8f31d11635127822ff982e31ca2811b485 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 1 Feb 2021 12:22:04 -0600 Subject: [PATCH 260/449] Fix record details bug. --- politeiad/backend/tlogbe/plugins/pi/pi.go | 13 - politeiad/backend/tlogbe/tlog/tlog.go | 2 +- politeiad/plugins/pi/pi.go | 5 - politeiawww/api/pi/v1/v1.go | 6 - politeiawww/client/client.go | 6 + politeiawww/client/records.go | 85 +++++++ politeiawww/cmd/pictl/castballot.go | 10 +- politeiawww/cmd/pictl/pictl.go | 7 +- politeiawww/cmd/pictl/proposaldetails.go | 68 ++++++ politeiawww/cmd/pictl/proposaledit.go | 285 +++++++++------------- politeiawww/cmd/pictl/proposalnew.go | 34 ++- politeiawww/cmd/pictl/util.go | 102 ++------ politeiawww/pi/pi.go | 12 - politeiawww/pi/process.go | 1 - politeiawww/piwww.go | 7 +- politeiawww/records/records.go | 3 +- 16 files changed, 336 insertions(+), 310 deletions(-) create mode 100644 politeiawww/cmd/pictl/proposaldetails.go diff --git a/politeiad/backend/tlogbe/plugins/pi/pi.go b/politeiad/backend/tlogbe/plugins/pi/pi.go index aba1322be..ba3f03256 100644 --- a/politeiad/backend/tlogbe/plugins/pi/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi/pi.go @@ -103,10 +103,6 @@ func (p *piPlugin) Settings() []backend.PluginSetting { log.Tracef("Settings") return []backend.PluginSetting{ - { - Key: pi.SettingKeyTextFileCountMax, - Value: strconv.FormatUint(uint64(p.textFileCountMax), 10), - }, { Key: pi.SettingKeyTextFileSizeMax, Value: strconv.FormatUint(uint64(p.textFileSizeMax), 10), @@ -149,7 +145,6 @@ func New(backend backend.Backend, settings []backend.PluginSetting, dataDir stri // Setup plugin setting default values var ( - textFileCountMax = pi.SettingTextFileCountMax textFileSizeMax = pi.SettingTextFileSizeMax imageFileCountMax = pi.SettingImageFileCountMax imageFileSizeMax = pi.SettingImageFileSizeMax @@ -161,13 +156,6 @@ func New(backend backend.Backend, settings []backend.PluginSetting, dataDir stri // Override default plugin settings with any passed in settings for _, v := range settings { switch v.Key { - case pi.SettingKeyTextFileCountMax: - u, err := strconv.ParseUint(v.Value, 10, 64) - if err != nil { - return nil, fmt.Errorf("invalid plugin setting %v '%v': %v", - v.Key, v.Value, err) - } - textFileCountMax = uint32(u) case pi.SettingKeyTextFileSizeMax: u, err := strconv.ParseUint(v.Value, 10, 64) if err != nil { @@ -234,7 +222,6 @@ func New(backend backend.Backend, settings []backend.PluginSetting, dataDir stri return &piPlugin{ dataDir: dataDir, backend: backend, - textFileCountMax: textFileCountMax, textFileSizeMax: textFileSizeMax, imageFileCountMax: imageFileCountMax, imageFileSizeMax: imageFileSizeMax, diff --git a/politeiad/backend/tlogbe/tlog/tlog.go b/politeiad/backend/tlogbe/tlog/tlog.go index df7819632..a58a66ef1 100644 --- a/politeiad/backend/tlogbe/tlog/tlog.go +++ b/politeiad/backend/tlogbe/tlog/tlog.go @@ -90,7 +90,7 @@ func leafExtraData(dataType, storeKey string) []byte { } func leafDataType(l *trillian.LogLeaf) string { - s := bytes.SplitAfter(l.ExtraData, []byte(":")) + s := bytes.Split(l.ExtraData, []byte(":")) if len(s) != 2 { e := fmt.Sprintf("invalid key '%s' for leaf %x", l.ExtraData, l.MerkleLeafHash) diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index f9f66067a..add8cbbaa 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -18,7 +18,6 @@ const ( // Setting keys are the plugin setting keys that can be used to // override a default plugin setting. Defaults will be overridden // if a plugin setting is provided to the plugin on startup. - SettingKeyTextFileCountMax = "textfilecountmax" SettingKeyTextFileSizeMax = "textfilesizemax" SettingKeyImageFileCountMax = "imagefilecountmax" SettingKeyImageFileSizeMax = "imagefilesizemax" @@ -26,10 +25,6 @@ const ( SettingKeyProposalNameLengthMax = "proposalnamelengthmax" SettingKeyProposalNameSupportedChars = "proposalnamesupportedchars" - // SettingTextFileCountMax is the maximum number of text files that - // can be included a proposal. - SettingTextFileCountMax uint32 = 1 - // SettingTextFileSizeMax is the maximum allowed size of a text // file in bytes. SettingTextFileSizeMax uint32 = 512 * 1024 diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 46a59aa29..90b2bd6b5 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -9,10 +9,6 @@ import ( ) // TODO verify that all batched request have a page size limit -// TODO pi needs a Version route that returns the APIs and versions that pi -// uses. -// TODO new APIs need a Policy route. The policies should be defined in the -// plugin packages as plugin settings and returned in a policy command. // TODO module these API packages const ( @@ -95,7 +91,6 @@ type Policy struct{} // PolicyReply is the reply to the Policy command. type PolicyReply struct { - TextFileCountMax uint32 `json:"textfilecountmax"` TextFileSizeMax uint32 `json:"textfilesizemax"` // In bytes ImageFileCountMax uint32 `json:"imagefilecountmax"` ImageFileSizeMax uint32 `json:"imagefilesizemax"` // In bytes @@ -211,7 +206,6 @@ type StatusChange struct { Reason string `json:"message,omitempty"` PublicKey string `json:"publickey"` Signature string `json:"signature"` - Timestamp int64 `json:"timestamp"` } // Proposal represents a proposal submission and its metadata. diff --git a/politeiawww/client/client.go b/politeiawww/client/client.go index fc7767b51..1e577f8b2 100644 --- a/politeiawww/client/client.go +++ b/politeiawww/client/client.go @@ -132,6 +132,12 @@ func (c *Client) makeReq(method string, route string, v interface{}) ([]byte, er } // New returns a new politeiawww client. +// +// The cert argument is optional. Any provided cert will be added to the http +// client's trust cert pool. This allows you to interact with a politeiawww +// instance that uses a self signed cert. +// +// The cookies and headerCSRF arguments are optional. func New(host, cert string, cookies []*http.Cookie, headerCSRF string) (*Client, error) { // Setup http client h, err := util.NewHTTPClient(false, cert) diff --git a/politeiawww/client/records.go b/politeiawww/client/records.go index d3f4268b6..62db08419 100644 --- a/politeiawww/client/records.go +++ b/politeiawww/client/records.go @@ -5,10 +5,15 @@ package client import ( + "bytes" + "encoding/base64" + "encoding/hex" "encoding/json" + "fmt" "net/http" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + "github.com/decred/politeia/util" ) // RecordNew sends a records v1 New request to politeiawww. @@ -27,3 +32,83 @@ func (c *Client) RecordNew(n rcv1.New) (*rcv1.NewReply, error) { return &nr, nil } + +// RecordDetails sends a records v1 Details request to politeiawww. +func (c *Client) RecordDetails(d rcv1.Details) (*rcv1.DetailsReply, error) { + route := rcv1.APIRoute + rcv1.RouteDetails + resBody, err := c.makeReq(http.MethodPost, route, d) + if err != nil { + return nil, err + } + + var dr rcv1.DetailsReply + err = json.Unmarshal(resBody, &dr) + if err != nil { + return nil, err + } + + return &dr, nil +} + +// digestsVerify verifies that all file digests match the calculated SHA256 +// digests of the file payloads. +func digestsVerify(files []rcv1.File) error { + for _, f := range files { + b, err := base64.StdEncoding.DecodeString(f.Payload) + if err != nil { + return fmt.Errorf("file: %v decode payload err %v", + f.Name, err) + } + digest := util.Digest(b) + d, ok := util.ConvertDigest(f.Digest) + if !ok { + return fmt.Errorf("file: %v invalid digest %v", + f.Name, f.Digest) + } + if !bytes.Equal(digest, d[:]) { + return fmt.Errorf("file: %v digests do not match", + f.Name) + } + } + return nil +} + +// RecordVerify verifies the censorship record of a records v1 Record. +func RecordVerify(r rcv1.Record, serverPubKey string) error { + // Verify censorship record merkle root + if len(r.Files) > 0 { + // Verify digests + err := digestsVerify(r.Files) + if err != nil { + return err + } + // Verify merkle root + digests := make([]string, 0, len(r.Files)) + for _, v := range r.Files { + digests = append(digests, v.Digest) + } + mr, err := util.MerkleRoot(digests) + if err != nil { + return err + } + if hex.EncodeToString(mr[:]) != r.CensorshipRecord.Merkle { + return fmt.Errorf("merkle roots do not match") + } + } + + // Verify censorship record signature + id, err := util.IdentityFromString(serverPubKey) + if err != nil { + return err + } + s, err := util.ConvertSignature(r.CensorshipRecord.Signature) + if err != nil { + return err + } + msg := []byte(r.CensorshipRecord.Merkle + r.CensorshipRecord.Token) + if !id.VerifyMessage(msg, s) { + return fmt.Errorf("invalid censorship record signature") + } + + return nil +} diff --git a/politeiawww/cmd/pictl/castballot.go b/politeiawww/cmd/pictl/castballot.go index 91011b923..fd91c01ce 100644 --- a/politeiawww/cmd/pictl/castballot.go +++ b/politeiawww/cmd/pictl/castballot.go @@ -56,9 +56,13 @@ func (c *castBallotCmd) Execute(args []string) error { defer client.Close() // Get the user's tickets that are eligible to vote - ticketPool, err := convertTicketHashes(pv.Vote.EligibleTickets) - if err != nil { - return err + ticketpool := make([][]byte, 0, len(pv.Vote.EligibleTickets)) + for _, v := range pv.Vote.EligibleTickets { + h, err := chainhash.NewHashFromStr(v) + if err != nil { + return nil, err + } + ticketpool = append(ticketpool, h[:]) } ctr, err := client.CommittedTickets( &walletrpc.CommittedTicketsRequest{ diff --git a/politeiawww/cmd/pictl/pictl.go b/politeiawww/cmd/pictl/pictl.go index 078a66290..9f4af3ab1 100644 --- a/politeiawww/cmd/pictl/pictl.go +++ b/politeiawww/cmd/pictl/pictl.go @@ -63,11 +63,11 @@ type pictl struct { UserDetails userDetailsCmd `command:"userdetails"` Users shared.UsersCmd `command:"users"` - // TODO replace www policies with pi policies // Proposal commands ProposalNew proposalNewCmd `command:"proposalnew"` ProposalEdit proposalEditCmd `command:"proposaledit"` ProposalSetStatus proposalSetStatusCmd `command:"proposalsetstatus"` + ProposalDetails proposalDetailsCmd `command:"proposaldetails"` Proposals proposalsCmd `command:"proposals"` ProposalInv proposalInvCmd `command:"proposalinv"` proposalTimestamps proposalTimestampsCmd `command:"proposaltimestamps"` @@ -141,8 +141,9 @@ Proposal commands proposalnew (user) Submit a new proposal proposaledit (user) Edit an existing proposal proposalstatusset (admin) Set the status of a proposal - proposals (public) Get proposals - proposalinv (public) Get proposal inventory by proposal status + proposaldetials (public) Get a full proposal record + proposals (public) Get proposals without their files + proposalinv (public) Get inventory by proposal status Comment commands commentnew (user) Submit a new comment diff --git a/politeiawww/cmd/pictl/proposaldetails.go b/politeiawww/cmd/pictl/proposaldetails.go new file mode 100644 index 000000000..2f724b6b9 --- /dev/null +++ b/politeiawww/cmd/pictl/proposaldetails.go @@ -0,0 +1,68 @@ +// Copyright (c) 2017-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + pclient "github.com/decred/politeia/politeiawww/client" + "github.com/decred/politeia/politeiawww/cmd/shared" +) + +// proposalDetails retrieves a full proposal record. +type proposalDetailsCmd struct { + Args struct { + Token string `positional-arg-name:"token"` + Version string `postional-arg-name:"version" optional:"true"` + } `positional-args:"true"` + + // Unvetted is used to indicate that the state of the requested + // proposal is unvetted. If this flag is not used it will be + // assumed that a vetted proposal is being requested. + Unvetted bool `long:"unvetted" optional:"true"` +} + +// Execute executes the proposalDetailsCmd command. +// +// This function satisfies the go-flags Commander interface. +func (c *proposalDetailsCmd) Execute(args []string) error { + // Setup client + pc, err := pclient.New(cfg.Host, cfg.HTTPSCert, cfg.Cookies, cfg.CSRF) + if err != nil { + return err + } + + // Setup state + var state string + switch { + case c.Unvetted: + state = rcv1.RecordStateUnvetted + default: + state = rcv1.RecordStateVetted + } + + // Setup request + d := rcv1.Details{ + State: state, + Token: c.Args.Token, + Version: c.Args.Version, + } + + // Send request. The request and response details are printed to + // the console based on the logging flags that were used. + err = shared.PrintJSON(d) + if err != nil { + return err + } + dr, err := pc.RecordDetails(d) + if err != nil { + return err + } + err = shared.PrintJSON(dr) + if err != nil { + return err + } + + return nil +} diff --git a/politeiawww/cmd/pictl/proposaledit.go b/politeiawww/cmd/pictl/proposaledit.go index d4cef14f7..d7c6a5613 100644 --- a/politeiawww/cmd/pictl/proposaledit.go +++ b/politeiawww/cmd/pictl/proposaledit.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2020 The Decred developers +// Copyright (c) 2017-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -12,16 +12,21 @@ type proposalEditCmd struct { Attachments []string `positional-arg-name:"attachmets"` } `positional-args:"true" optional:"true"` - // CLI flags - Unvetted bool `long:"unvetted" optional:"true"` - Name string `long:"name" optional:"true"` - LinkTo string `long:"linkto" optional:"true"` - LinkBy int64 `long:"linkby" optional:"true"` + // Unvetted is used to indicate that the state of the requested + // proposal is unvetted. If this flag is not used it will be + // assumed that a vetted proposal is being requested. + Unvetted bool `long:"unvetted" optional:"true"` // Random generates random proposal data. An IndexFile and // Attachments are not required when using this flag. Random bool `long:"random" optional:"true"` + // The following flags can be used to specify user defined proposal + // metadata values. + Name string `long:"name" optional:"true"` + LinkTo string `long:"linkto" optional:"true"` + LinkBy int64 `long:"linkby" optional:"true"` + // RFP is a flag that is intended to make editing an RFP easier // by calculating and inserting a linkby timestamp automatically // instead of having to pass in a specific timestamp using the @@ -30,187 +35,141 @@ type proposalEditCmd struct { // UseMD is a flag that is intended to make editing proposal // metadata easier by using exisiting proposal metadata values - // instead of having to pass in specific values + // instead of having to pass in specific values. UseMD bool `long:"usemd" optional:"true"` } -/* // Execute executes the proposalEditCmd command. // // This function satisfies the go-flags Commander interface. -func (cmd *proposalEditCmd) Execute(args []string) error { - token := cmd.Args.Token - indexFile := cmd.Args.IndexFile - attachments := cmd.Args.Attachments - - // Verify arguments - switch { - case !cmd.Random && indexFile == "": - return fmt.Errorf("index file not found; you must either provide an " + - "index.md file or use --random") - case cmd.RFP && cmd.LinkBy != 0: - return fmt.Errorf("--rfp and --linkby can not be used together, as " + - "--rfp sets the linkby one month from now") - case cmd.Random && cmd.Name != "": - return fmt.Errorf("--random and --name can not be used together, as " + - "--random generates a random name and random proposal data") - } - - // Verify state. Defaults to vetted if the --unvetted flag - // is not used. - var state pi.PropStateT - switch { - case cmd.Unvetted: - state = pi.PropStateUnvetted - default: - state = pi.PropStateVetted - } - - // Check for user identity. A user identity is required to sign - // the proposal files and metadata. - if cfg.Identity == nil { - return shared.ErrUserIdentityNotFound - } - - // Prepare index file - var ( - file *pi.File - err error - ) - if cmd.Random { - // Generate random text for the index file - file, err = createMDFile() - if err != nil { - return err - } - } else { - // Read the index file from disk - fp := util.CleanAndExpandPath(indexFile) - var err error - payload, err := ioutil.ReadFile(fp) - if err != nil { - return fmt.Errorf("ReadFile %v: %v", fp, err) - } - file = &pi.File{ - Name: v1.PolicyIndexFilename, - MIME: mime.DetectMimeType(payload), - Digest: hex.EncodeToString(util.Digest(payload)), - Payload: base64.StdEncoding.EncodeToString(payload), - } - } - files := []pi.File{ - *file, - } - - // Prepare attachment files - for _, fn := range attachments { - fp := util.CleanAndExpandPath(fn) - payload, err := ioutil.ReadFile(fp) - if err != nil { - return fmt.Errorf("ReadFile %v: %v", fp, err) +func (c *proposalEditCmd) Execute(args []string) error { + /* + // Unpack args + token := c.Args.Token + indexFile := c.Args.IndexFile + attachments := c.Args.Attachments + + // Verify args + switch { + case !c.Random && indexFile == "": + return fmt.Errorf("index file not found; you must either " + + "provide an index.md file or use --random") + + case c.Random && indexFile != "": + return fmt.Errorf("you cannot provide file arguments and use " + + "the --random flag at the same time") + + case !c.Random && c.Name == "": + return fmt.Errorf("you must either provide a proposal name " + + "using the --name flag or use the --random flag to generate " + + "a random proposal") + + case c.RFP && c.LinkBy != 0: + return fmt.Errorf("you cannot use both the --rfp and --linkby " + + "flags at the same time") } - files = append(files, pi.File{ - Name: filepath.Base(fn), - MIME: mime.DetectMimeType(payload), - Digest: hex.EncodeToString(util.Digest(payload)), - Payload: base64.StdEncoding.EncodeToString(payload), - }) - } + // Check for user identity. A user identity is required to sign + // the proposal files. + if cfg.Identity == nil { + return shared.ErrUserIdentityNotFound + } - // Setup metadata - var pm pi.ProposalMetadata - if cmd.UseMD { - // Get the existing proposal metadata - pr, err := proposalRecordLatest(state, cmd.Args.Token) + // Setup client + pc, err := pclient.New(cfg.Host, cfg.HTTPSCert, cfg.Cookies, cfg.CSRF) if err != nil { return err } - pmCurr, err := decodeProposalMetadata(pr.Metadata) + + // Get the pi policy. It contains the proposal requirements. + pr, err := pc.PiPolicy() if err != nil { return err } - // Prefill proposal metadata with existing values - pm.Name = pmCurr.Name - pm.LinkTo = pmCurr.LinkTo - pm.LinkBy = pmCurr.LinkBy - } - if cmd.Random { - // Generate random name - r, err := util.Random(v1.PolicyMinProposalNameLength) + // Setup state + var state string + switch { + case c.Unvetted: + state = rcv1.RecordStateUnvetted + default: + state = rcv1.RecordStateVetted + } + + // Setup index file + var ( + file *rcv1.File + files = make([]rcv1.File, 0, 16) + ) + if c.Random { + // Generate random text for the index file + file, err = createMDFile() + if err != nil { + return err + } + } else { + // Read index file from disk + fp := util.CleanAndExpandPath(indexFile) + var err error + payload, err := ioutil.ReadFile(fp) + if err != nil { + return fmt.Errorf("ReadFile %v: %v", fp, err) + } + file = &rcv1.File{ + Name: piplugin.FileNameIndexFile, + MIME: mime.DetectMimeType(payload), + Digest: hex.EncodeToString(util.Digest(payload)), + Payload: base64.StdEncoding.EncodeToString(payload), + } + } + files = append(files, *file) + + // Setup attachment files + for _, fn := range attachments { + fp := util.CleanAndExpandPath(fn) + payload, err := ioutil.ReadFile(fp) + if err != nil { + return fmt.Errorf("ReadFile %v: %v", fp, err) + } + + files = append(files, rcv1.File{ + Name: filepath.Base(fn), + MIME: mime.DetectMimeType(payload), + Digest: hex.EncodeToString(util.Digest(payload)), + Payload: base64.StdEncoding.EncodeToString(payload), + }) + } + + // Setup proposal metadata + switch { + case c.UseMD: + // Use the prexisting proposal name + + case c.Random: + // Create a random proposal name + r, err := util.Random(int(pr.NameLengthMin)) + if err != nil { + return err + } + c.Name = hex.EncodeToString(r) + } + pm := piv1.ProposalMetadata{ + Name: c.Name, + } + pmb, err := json.Marshal(pm) if err != nil { return err } - pm.Name = hex.EncodeToString(r) - } - if cmd.RFP { - // Set linkby to a month from now - pm.LinkBy = time.Now().Add(time.Hour * 24 * 30).Unix() - } - if cmd.Name != "" { - pm.Name = cmd.Name - } - if cmd.LinkBy != 0 { - pm.LinkBy = cmd.LinkBy - } - if cmd.LinkTo != "" { - pm.LinkTo = cmd.LinkTo - } - pmb, err := json.Marshal(pm) - if err != nil { - return err - } - metadata := []pi.Metadata{ - { + files = append(files, rcv1.File{ + Name: piv1.FileNameProposalMetadata, + MIME: mime.DetectMimeType(pmb), Digest: hex.EncodeToString(util.Digest(pmb)), - Hint: pi.HintProposalMetadata, Payload: base64.StdEncoding.EncodeToString(pmb), - }, - } - - // Setup edit proposal request - sig, err := signedMerkleRoot(files, metadata, cfg.Identity) - if err != nil { - return err - } - pe := pi.ProposalEdit{ - Token: token, - State: state, - Files: files, - Metadata: metadata, - PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), - Signature: sig, - } - - // Send request. The request and response details are printed to - // the console based on the logging flags that were used. - err = shared.PrintJSON(pe) - if err != nil { - return err - } - per, err := client.ProposalEdit(pe) - if err != nil { - return err - } - err = shared.PrintJSON(per) - if err != nil { - return err - } - - // Verify proposal - vr, err := client.Version() - if err != nil { - return err - } - err = verifyProposal(per.Proposal, vr.PubKey) - if err != nil { - return fmt.Errorf("unable to verify proposal: %v", err) - } + }) + */ return nil } -*/ // proposalEditHelpMsg is the output of the help command. const proposalEditHelpMsg = `editproposal [flags] "token" "indexfile" "attachments" diff --git a/politeiawww/cmd/pictl/proposalnew.go b/politeiawww/cmd/pictl/proposalnew.go index 719d983a0..20501e2f2 100644 --- a/politeiawww/cmd/pictl/proposalnew.go +++ b/politeiawww/cmd/pictl/proposalnew.go @@ -92,10 +92,13 @@ func (c *proposalNewCmd) Execute(args []string) error { } // Setup index file - var file *rcv1.File + var ( + file *rcv1.File + files = make([]rcv1.File, 0, 16) + ) if c.Random { // Generate random text for the index file - file, err = createMDFile() + file, err = indexFileRandom() if err != nil { return err } @@ -114,9 +117,7 @@ func (c *proposalNewCmd) Execute(args []string) error { Payload: base64.StdEncoding.EncodeToString(payload), } } - files := []rcv1.File{ - *file, - } + files = append(files, *file) // Setup attachment files for _, fn := range attachments { @@ -140,7 +141,7 @@ func (c *proposalNewCmd) Execute(args []string) error { if err != nil { return err } - c.Name = hex.EncodeToString(r) + c.Name = fmt.Sprintf("Name %x", r) } pm := piv1.ProposalMetadata{ Name: c.Name, @@ -171,7 +172,7 @@ func (c *proposalNewCmd) Execute(args []string) error { return err } files = append(files, rcv1.File{ - Name: piv1.FileNameProposalMetadata, + Name: piv1.FileNameVoteMetadata, MIME: mime.DetectMimeType(vmb), Digest: hex.EncodeToString(util.Digest(vmb)), Payload: base64.StdEncoding.EncodeToString(vmb), @@ -189,31 +190,28 @@ func (c *proposalNewCmd) Execute(args []string) error { Signature: sig, } - // Send request. The request and response details are printed to - // the console based on the logging flags that were used. - err = shared.PrintJSON(n) - if err != nil { - return err - } + // Send request nr, err := pc.RecordNew(n) if err != nil { return err } - err = shared.PrintJSON(nr) - if err != nil { - return err - } // Verify record vr, err := client.Version() if err != nil { return err } - err = verifyRecord(nr.Record, vr.PubKey) + err = pclient.RecordVerify(nr.Record, vr.PubKey) if err != nil { return fmt.Errorf("unable to verify record: %v", err) } + // Print details to stdout + printf("Proposal '%v' submitted\n", pm.Name) + for _, v := range nr.Record.Files { + printf(" %-22v %v\n", v.Name, v.MIME) + } + return nil } diff --git a/politeiawww/cmd/pictl/util.go b/politeiawww/cmd/pictl/util.go index 6ad7b6786..be9b4b9ca 100644 --- a/politeiawww/cmd/pictl/util.go +++ b/politeiawww/cmd/pictl/util.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2020 The Decred developers +// Copyright (c) 2017-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -10,14 +10,27 @@ import ( "encoding/hex" "fmt" - "github.com/decred/dcrd/chaincfg/chainhash" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/api/v1/mime" + piplugin "github.com/decred/politeia/politeiad/plugins/pi" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" - v1 "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/util" ) +// printf prints the provided string to stdout if the global config settings +// allows for it. +func printf(s string, args ...interface{}) { + switch { + case cfg.Verbose, cfg.RawJSON: + // These are handled by the politeiawwww client + case cfg.Silent: + // Do nothing + default: + // Print to stdout + fmt.Printf(s, args...) + } +} + // signedMerkleRoot returns the signed merkle root of the provided files. The // signature is created using the provided identity. func signedMerkleRoot(files []rcv1.File, fid *identity.FullIdentity) (string, error) { @@ -36,24 +49,9 @@ func signedMerkleRoot(files []rcv1.File, fid *identity.FullIdentity) (string, er return hex.EncodeToString(sig[:]), nil } -// convertTicketHashes converts a slice of hexadecimal ticket hashes into -// a slice of byte slices. -func convertTicketHashes(h []string) ([][]byte, error) { - hashes := make([][]byte, 0, len(h)) - for _, v := range h { - h, err := chainhash.NewHashFromStr(v) - if err != nil { - return nil, err - } - hashes = append(hashes, h[:]) - } - return hashes, nil -} - -// createMDFile returns a File object that was created using a markdown file -// filled with random text. +// indexFileRandom returns a proposal index file filled with random data. // TODO fill to max size -func createMDFile() (*rcv1.File, error) { +func indexFileRandom() (*rcv1.File, error) { var b bytes.Buffer for i := 0; i < 10; i++ { r, err := util.Random(32) @@ -64,71 +62,9 @@ func createMDFile() (*rcv1.File, error) { } return &rcv1.File{ - Name: v1.PolicyIndexFilename, + Name: piplugin.FileNameIndexFile, MIME: mime.DetectMimeType(b.Bytes()), Digest: hex.EncodeToString(util.Digest(b.Bytes())), Payload: base64.StdEncoding.EncodeToString(b.Bytes()), }, nil } - -// verifyDigests verifies that all file digests match the calculated SHA256 -// digests of the file payloads. -func verifyDigests(files []rcv1.File) error { - for _, f := range files { - b, err := base64.StdEncoding.DecodeString(f.Payload) - if err != nil { - return fmt.Errorf("file: %v decode payload err %v", - f.Name, err) - } - digest := util.Digest(b) - d, ok := util.ConvertDigest(f.Digest) - if !ok { - return fmt.Errorf("file: %v invalid digest %v", - f.Name, f.Digest) - } - if !bytes.Equal(digest, d[:]) { - return fmt.Errorf("file: %v digests do not match", - f.Name) - } - } - return nil -} - -func verifyRecord(r rcv1.Record, serverPubKey string) error { - if len(r.Files) > 0 { - // Verify digests - err := verifyDigests(r.Files) - if err != nil { - return err - } - // Verify merkle root - digests := make([]string, 0, len(r.Files)) - for _, v := range r.Files { - digests = append(digests, v.Digest) - } - mr, err := util.MerkleRoot(digests) - if err != nil { - return err - } - // Check if merkle roots match - if hex.EncodeToString(mr[:]) != r.CensorshipRecord.Merkle { - return fmt.Errorf("merkle roots do not match") - } - } - - // Verify censorship record signature - id, err := util.IdentityFromString(serverPubKey) - if err != nil { - return err - } - s, err := util.ConvertSignature(r.CensorshipRecord.Signature) - if err != nil { - return err - } - msg := []byte(r.CensorshipRecord.Merkle + r.CensorshipRecord.Token) - if !id.VerifyMessage(msg, s) { - return fmt.Errorf("invalid censorship record signature") - } - - return nil -} diff --git a/politeiawww/pi/pi.go b/politeiawww/pi/pi.go index 32430a80a..c7753ad0f 100644 --- a/politeiawww/pi/pi.go +++ b/politeiawww/pi/pi.go @@ -28,7 +28,6 @@ type Pi struct { sessions *sessions.Sessions // Plugin settings - textFileCountMax uint32 textFileSizeMax uint32 // In bytes imageFileCountMax uint32 imageFileSizeMax uint32 // In bytes @@ -103,7 +102,6 @@ func (p *Pi) HandleVoteInventory(w http.ResponseWriter, r *http.Request) { func New(cfg *config.Config, pdc *pdclient.Client, udb user.Database, s *sessions.Sessions, plugins []pdv1.Plugin) (*Pi, error) { // Parse pi plugin settings var ( - textFileCountMax uint32 textFileSizeMax uint32 imageFileCountMax uint32 imageFileSizeMax uint32 @@ -118,12 +116,6 @@ func New(cfg *config.Config, pdc *pdclient.Client, udb user.Database, s *session } for _, s := range v.Settings { switch s.Key { - case pi.SettingKeyTextFileCountMax: - u, err := strconv.ParseUint(s.Value, 10, 64) - if err != nil { - return nil, err - } - textFileCountMax = uint32(u) case pi.SettingKeyTextFileSizeMax: u, err := strconv.ParseUint(s.Value, 10, 64) if err != nil { @@ -170,9 +162,6 @@ func New(cfg *config.Config, pdc *pdclient.Client, udb user.Database, s *session // Verify all plugin settings have been provided switch { - case textFileCountMax == 0: - return nil, fmt.Errorf("plugin setting not found: %v", - pi.SettingKeyTextFileCountMax) case textFileSizeMax == 0: return nil, fmt.Errorf("plugin setting not found: %v", pi.SettingKeyTextFileSizeMax) @@ -195,7 +184,6 @@ func New(cfg *config.Config, pdc *pdclient.Client, udb user.Database, s *session politeiad: pdc, userdb: udb, sessions: s, - textFileCountMax: textFileCountMax, textFileSizeMax: textFileSizeMax, imageFileCountMax: imageFileCountMax, imageFileSizeMax: imageFileSizeMax, diff --git a/politeiawww/pi/process.go b/politeiawww/pi/process.go index 9a82577ef..baeb97322 100644 --- a/politeiawww/pi/process.go +++ b/politeiawww/pi/process.go @@ -25,7 +25,6 @@ func (p *Pi) processPolicy(ctx context.Context) (*v1.PolicyReply, error) { log.Tracef("Policy") return &v1.PolicyReply{ - TextFileCountMax: p.textFileCountMax, TextFileSizeMax: p.textFileSizeMax, ImageFileCountMax: p.imageFileCountMax, ImageFileSizeMax: p.imageFileSizeMax, diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index da75ab8f1..a45ac6ea4 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -12,6 +12,7 @@ import ( cmplugin "github.com/decred/politeia/politeiad/plugins/comments" piplugin "github.com/decred/politeia/politeiad/plugins/pi" tkplugin "github.com/decred/politeia/politeiad/plugins/ticketvote" + usplugin "github.com/decred/politeia/politeiad/plugins/user" cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" @@ -75,6 +76,9 @@ func (p *politeiawww) setupPiRoutes(r *records.Records, c *comments.Comments, t p.addRoute(http.MethodPost, rcv1.APIRoute, rcv1.RouteSetStatus, r.HandleSetStatus, permissionAdmin) + p.addRoute(http.MethodPost, rcv1.APIRoute, + rcv1.RouteDetails, r.HandleDetails, + permissionPublic) p.addRoute(http.MethodPost, rcv1.APIRoute, rcv1.RouteInventory, r.HandleInventory, permissionPublic) @@ -149,6 +153,7 @@ func (p *politeiawww) setupPi(plugins []pdv1.Plugin) error { piplugin.PluginID: false, cmplugin.PluginID: false, tkplugin.PluginID: false, + usplugin.PluginID: false, } for _, v := range plugins { _, ok := required[v.ID] @@ -171,7 +176,7 @@ func (p *politeiawww) setupPi(plugins []pdv1.Plugin) error { // Setup api contexts c := comments.New(p.cfg, p.politeiad, p.db, p.sessions, p.events) tv := ticketvote.New(p.cfg, p.politeiad, p.sessions, p.events) - r := records.New(p.cfg, p.politeiad, p.sessions, p.events) + r := records.New(p.cfg, p.politeiad, p.db, p.sessions, p.events) pic, err := pi.New(p.cfg, p.politeiad, p.db, p.sessions, plugins) if err != nil { return fmt.Errorf("new pi: %v", err) diff --git a/politeiawww/records/records.go b/politeiawww/records/records.go index d16a8eb82..24731f47e 100644 --- a/politeiawww/records/records.go +++ b/politeiawww/records/records.go @@ -287,10 +287,11 @@ func (c *Records) HandleUserRecords(w http.ResponseWriter, r *http.Request) { } // New returns a new Records context. -func New(cfg *config.Config, pdc *pdclient.Client, s *sessions.Sessions, e *events.Manager) *Records { +func New(cfg *config.Config, pdc *pdclient.Client, udb user.Database, s *sessions.Sessions, e *events.Manager) *Records { return &Records{ cfg: cfg, politeiad: pdc, + userdb: udb, sessions: s, events: e, } From 4f31167fb3c895cbf6414d6e6d9f1e5758815b7d Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 1 Feb 2021 14:37:56 -0600 Subject: [PATCH 261/449] ticketvote plugin settings and policy route. --- .../tlogbe/plugins/ticketvote/ticketvote.go | 102 +++++++++--------- politeiad/plugins/pi/pi.go | 22 ++-- politeiad/plugins/ticketvote/ticketvote.go | 51 ++++++--- politeiawww/api/ticketvote/v1/v1.go | 12 +++ politeiawww/client/ticketvote.go | 29 +++++ politeiawww/cmd/pictl/pictl.go | 2 + politeiawww/cmd/pictl/util.go | 16 +++ politeiawww/cmd/pictl/votepolicy.go | 39 +++++++ politeiawww/pi/pi.go | 64 +++++------ politeiawww/pi/process.go | 13 --- politeiawww/piwww.go | 11 +- politeiawww/ticketvote/ticketvote.go | 82 +++++++++++++- 12 files changed, 315 insertions(+), 128 deletions(-) create mode 100644 politeiawww/client/ticketvote.go create mode 100644 politeiawww/cmd/pictl/votepolicy.go diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go index 399723035..3e237a432 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go @@ -38,12 +38,6 @@ type ticketVotePlugin struct { // record vote has ended. dataDir string - // Plugin settings - voteDurationMin uint32 // In blocks - voteDurationMax uint32 // In blocks - linkByPeriodMin int64 // In seconds - linkByPeriodMax int64 // In seconds - // identity contains the full identity that the plugin uses to // create receipts, i.e. signatures of user provided data that // prove the backend received and processed a plugin command. @@ -65,6 +59,12 @@ type ticketVotePlugin struct { // lazy loaded and should only be used for tree updates, not for // cache updates. mutexes map[string]*sync.Mutex // [string]mutex + + // Plugin settings + linkByPeriodMin int64 // In seconds + linkByPeriodMax int64 // In seconds + voteDurationMin uint32 // In blocks + voteDurationMax uint32 // In blocks } // mutex returns the mutex for the specified record. @@ -244,48 +244,37 @@ func (p *ticketVotePlugin) Fsck(treeIDs []int64) error { return nil } -// TODO Settings returns the plugin's settings. +// Settings returns the plugin's settings. // // This function satisfies the plugins.PluginClient interface. func (p *ticketVotePlugin) Settings() []backend.PluginSetting { log.Tracef("Settings") - return nil -} - -/* -// linkByPeriodMin returns the minimum amount of time, in seconds, that the -// LinkBy period must be set to. This is determined by adding 1 week onto the -// minimum voting period so that RFP proposal submissions have at least one -// week to be submitted after the proposal vote ends. -func (p *politeiawww) linkByPeriodMin() int64 { - var ( - submissionPeriod int64 = 604800 // One week in seconds - blockTime int64 // In seconds - ) - switch { - case p.cfg.TestNet: - blockTime = int64(testNet3Params.TargetTimePerBlock.Seconds()) - case p.cfg.SimNet: - blockTime = int64(simNetParams.TargetTimePerBlock.Seconds()) - default: - blockTime = int64(mainNetParams.TargetTimePerBlock.Seconds()) + return []backend.PluginSetting{ + { + Key: ticketvote.SettingKeyLinkByPeriodMin, + Value: strconv.FormatInt(p.linkByPeriodMin, 10), + }, + { + Key: ticketvote.SettingKeyLinkByPeriodMax, + Value: strconv.FormatInt(p.linkByPeriodMax, 10), + }, + { + Key: ticketvote.SettingKeyVoteDurationMin, + Value: strconv.FormatUint(uint64(p.voteDurationMin), 10), + }, + { + Key: ticketvote.SettingKeyVoteDurationMax, + Value: strconv.FormatUint(uint64(p.voteDurationMax), 10), + }, } - return (int64(p.cfg.VoteDurationMin) * blockTime) + submissionPeriod } -// linkByPeriodMax returns the maximum amount of time, in seconds, that the -// LinkBy period can be set to. 3 months is currently hard coded with no real -// reason for deciding on 3 months besides that it sounds like a sufficient -// amount of time. This can be changed if there is a valid reason to. -func (p *politeiawww) linkByPeriodMax() int64 { - return 7776000 // 3 months in seconds -} -*/ - func New(backend backend.Backend, tlog plugins.TlogClient, settings []backend.PluginSetting, dataDir string, id *identity.FullIdentity, activeNetParams *chaincfg.Params) (*ticketVotePlugin, error) { // Plugin settings var ( + linkByPeriodMin = ticketvote.SettingLinkByPeriodMin + linkByPeriodMax = ticketvote.SettingLinkByPeriodMax voteDurationMin uint32 voteDurationMax uint32 ) @@ -294,21 +283,36 @@ func New(backend backend.Backend, tlog plugins.TlogClient, settings []backend.Pl // the setting was specified by the user. switch activeNetParams.Name { case chaincfg.MainNetParams().Name: - voteDurationMin = ticketvote.DefaultMainNetVoteDurationMin - voteDurationMax = ticketvote.DefaultMainNetVoteDurationMax + voteDurationMin = ticketvote.SettingMainNetVoteDurationMin + voteDurationMax = ticketvote.SettingMainNetVoteDurationMax case chaincfg.TestNet3Params().Name: - voteDurationMin = ticketvote.DefaultTestNetVoteDurationMin - voteDurationMax = ticketvote.DefaultTestNetVoteDurationMax + voteDurationMin = ticketvote.SettingTestNetVoteDurationMin + voteDurationMax = ticketvote.SettingTestNetVoteDurationMax case chaincfg.SimNetParams().Name: - voteDurationMin = ticketvote.DefaultSimNetVoteDurationMin - voteDurationMax = ticketvote.DefaultSimNetVoteDurationMax + voteDurationMin = ticketvote.SettingSimNetVoteDurationMin + voteDurationMax = ticketvote.SettingSimNetVoteDurationMax default: return nil, fmt.Errorf("unknown active net: %v", activeNetParams.Name) } - // Parse user provided plugin settings + // Override default plugin settings with user provided plugin + // settings. for _, v := range settings { switch v.Key { + case ticketvote.SettingKeyLinkByPeriodMin: + i, err := strconv.ParseInt(v.Value, 10, 64) + if err != nil { + return nil, fmt.Errorf("plugin setting '%v': ParseInt(%v): %v", + v.Key, v.Value, err) + } + linkByPeriodMin = i + case ticketvote.SettingKeyLinkByPeriodMax: + i, err := strconv.ParseInt(v.Value, 10, 64) + if err != nil { + return nil, fmt.Errorf("plugin setting '%v': ParseInt(%v): %v", + v.Key, v.Value, err) + } + linkByPeriodMax = i case ticketvote.SettingKeyVoteDurationMin: u, err := strconv.ParseUint(v.Value, 10, 64) if err != nil { @@ -339,8 +343,6 @@ func New(backend backend.Backend, tlog plugins.TlogClient, settings []backend.Pl activeNetParams: activeNetParams, backend: backend, tlog: tlog, - voteDurationMin: voteDurationMin, - voteDurationMax: voteDurationMax, dataDir: dataDir, identity: id, inv: inventory{ @@ -350,7 +352,11 @@ func New(backend backend.Backend, tlog plugins.TlogClient, settings []backend.Pl finished: make([]string, 0, 1024), bestBlock: 0, }, - votes: make(map[string]map[string]string), - mutexes: make(map[string]*sync.Mutex), + votes: make(map[string]map[string]string), + mutexes: make(map[string]*sync.Mutex), + linkByPeriodMin: linkByPeriodMin, + linkByPeriodMax: linkByPeriodMax, + voteDurationMin: voteDurationMin, + voteDurationMax: voteDurationMax, }, nil } diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index add8cbbaa..22cf4b50d 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -12,9 +12,7 @@ const ( // Plugin commands CmdVoteInv = "voteinv" -) -const ( // Setting keys are the plugin setting keys that can be used to // override a default plugin setting. Defaults will be overridden // if a plugin setting is provided to the plugin on startup. @@ -25,24 +23,24 @@ const ( SettingKeyProposalNameLengthMax = "proposalnamelengthmax" SettingKeyProposalNameSupportedChars = "proposalnamesupportedchars" - // SettingTextFileSizeMax is the maximum allowed size of a text - // file in bytes. + // SettingTextFileSizeMax is the default maximum allowed size of a + // text file in bytes. SettingTextFileSizeMax uint32 = 512 * 1024 - // SettingImageFileCountMax is the maximum number of image files - // that can be included in a proposal. + // SettingImageFileCountMax is the default maximum number of image + // files that can be included in a proposal. SettingImageFileCountMax uint32 = 5 - // SettingImageFileSizeMax is the maximum allowed size of a image - // file in bytes. + // SettingImageFileSizeMax is the default maximum allowed size of + // an image file in bytes. SettingImageFileSizeMax uint32 = 512 * 1024 - // SettingProposalNameLengthMin is the minimum number of characters - // that a proposal name can be. + // SettingProposalNameLengthMin is the default minimum number of + // characters that a proposal name can be. SettingProposalNameLengthMin uint32 = 8 - // SettingProposalNameLengthMax is the maximum number of characters - // that a proposal name can be. + // SettingProposalNameLengthMax is the default maximum number of + // characters that a proposal name can be. SettingProposalNameLengthMax uint32 = 80 ) diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index 20d212d6d..ba56a4ba4 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -28,22 +28,47 @@ const ( CmdInventory = "inventory" // Get inventory by vote status CmdTimestamps = "timestamps" // Get vote data timestamps - // Plugin setting keys + // Setting keys are the plugin setting keys that can be used to + // override a default plugin setting. Defaults will be overridden + // if a plugin setting is provided to the plugin on startup. + SettingKeyLinkByPeriodMin = "linkbyperiodmin" + SettingKeyLinkByPeriodMax = "linkbyperiodmax" SettingKeyVoteDurationMin = "votedurationmin" SettingKeyVoteDurationMax = "votedurationmax" - // Default plugin settings - DefaultMainNetVoteDurationMin = 2016 - DefaultMainNetVoteDurationMax = 4032 - DefaultTestNetVoteDurationMin = 0 - DefaultTestNetVoteDurationMax = 4032 - DefaultSimNetVoteDurationMin = 0 - DefaultSimNetVoteDurationMax = 4032 - - // TODO implement PolicyVotesPageSize - // PolicyVotesPageSize is the maximum number of results that can be - // returned from any of the batched vote commands. - PolicyVotesPageSize = 20 + // SettingLinkByPeriodMin is the default minimum amount of time, + // in seconds, that the link by period can be set to. This value + // of 2 weeks was chosen arbitrarily. + SettingLinkByPeriodMin int64 = 1209600 + + // SettingLinkByPeriodMax is the default maximum amount of time, + // in seconds, that the link by period can be set to. This value + // of 3 months was chosen arbitrarily. + SettingLinkByPeriodMax int64 = 7776000 + + // SettingMainNetVoteDurationMin is the default minimum vote + // duration on mainnet in blocks. + SettingMainNetVoteDurationMin uint32 = 2016 + + // SettingMainNetVoteDurationMax is the default maximum vote + // duration on mainnet in blocks. + SettingMainNetVoteDurationMax uint32 = 4032 + + // SettingTestNetVoteDurationMin is the default minimum vote + // duration on testnet in blocks. + SettingTestNetVoteDurationMin uint32 = 1 + + // SettingTestNetVoteDurationMax is the default maximum vote + // duration on testnet in blocks. + SettingTestNetVoteDurationMax uint32 = 4032 + + // SettingSimNetVoteDurationMin is the default minimum vote + // duration on simnet in blocks. + SettingSimNetVoteDurationMin uint32 = 1 + + // SettingSimNetVoteDurationMax is the default maximum vote + // duration on simnet in blocks. + SettingSimNetVoteDurationMax uint32 = 4032 ) // ErrorCodeT represents and error that is caused by the user. diff --git a/politeiawww/api/ticketvote/v1/v1.go b/politeiawww/api/ticketvote/v1/v1.go index 72f843c94..edc38b5c2 100644 --- a/politeiawww/api/ticketvote/v1/v1.go +++ b/politeiawww/api/ticketvote/v1/v1.go @@ -10,6 +10,7 @@ const ( // APIRoute is prefixed onto all routes defined in this package. APIRoute = "/ticketvote/v1" + RoutePolicy = "/policy" RouteAuthorize = "/authorize" RouteStart = "/start" RouteCastBallot = "/castballot" @@ -80,6 +81,17 @@ func (e ServerErrorReply) Error() string { return fmt.Sprintf("server error: %v", e.ErrorCode) } +// Policy requests the ticketvote policy. +type Policy struct{} + +// PolicyReply is the reply to the Policy command. +type PolicyReply struct { + LinkByPeriodMin int64 `json:"linkbyperiodmin"` // In seconds + LinkByPeriodMax int64 `json:"linkbyperiodmax"` // In seconds + VoteDurationMin uint32 `json:"votedurationmin"` // In blocks + VoteDurationMax uint32 `json:"votedurationmax"` // In blocks +} + // AuthActionT represents an Authorize action. type AuthActionT string diff --git a/politeiawww/client/ticketvote.go b/politeiawww/client/ticketvote.go new file mode 100644 index 000000000..96546fda8 --- /dev/null +++ b/politeiawww/client/ticketvote.go @@ -0,0 +1,29 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package client + +import ( + "encoding/json" + "net/http" + + tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" +) + +// TicketVotePolicy sends a pi v1 Policy request to politeiawww. +func (c *Client) TicketVotePolicy() (*tkv1.PolicyReply, error) { + route := tkv1.APIRoute + tkv1.RoutePolicy + resBody, err := c.makeReq(http.MethodGet, route, nil) + if err != nil { + return nil, err + } + + var pr tkv1.PolicyReply + err = json.Unmarshal(resBody, &pr) + if err != nil { + return nil, err + } + + return &pr, nil +} diff --git a/politeiawww/cmd/pictl/pictl.go b/politeiawww/cmd/pictl/pictl.go index 9f4af3ab1..b43f3ae87 100644 --- a/politeiawww/cmd/pictl/pictl.go +++ b/politeiawww/cmd/pictl/pictl.go @@ -64,6 +64,7 @@ type pictl struct { Users shared.UsersCmd `command:"users"` // Proposal commands + // TODO ProposalPolicy ProposalNew proposalNewCmd `command:"proposalnew"` ProposalEdit proposalEditCmd `command:"proposaledit"` ProposalSetStatus proposalSetStatusCmd `command:"proposalsetstatus"` @@ -81,6 +82,7 @@ type pictl struct { CommentTimestamps commentTimestampsCmd `command:"commenttimestamps"` // Vote commands + VotePolicy votePolicyCmd `command:"votepolicy"` VoteAuthorize voteAuthorizeCmd `command:"voteauthorize"` VoteStart voteStartCmd `command:"votestart"` VoteStartRunoff voteStartRunoffCmd `command:"votestartrunoff"` diff --git a/politeiawww/cmd/pictl/util.go b/politeiawww/cmd/pictl/util.go index be9b4b9ca..8a0feeb93 100644 --- a/politeiawww/cmd/pictl/util.go +++ b/politeiawww/cmd/pictl/util.go @@ -8,6 +8,7 @@ import ( "bytes" "encoding/base64" "encoding/hex" + "encoding/json" "fmt" "github.com/decred/politeia/politeiad/api/v1/identity" @@ -31,6 +32,21 @@ func printf(s string, args ...interface{}) { } } +// println prints the provided string to stdout if the global config settings +// allows for it. +func println(s string, args ...interface{}) { + printf(s+"\n", args...) +} + +// formatJSON returns a pretty printed JSON string for the provided structure. +func formatJSON(v interface{}) string { + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + return "" + } + return string(b) +} + // signedMerkleRoot returns the signed merkle root of the provided files. The // signature is created using the provided identity. func signedMerkleRoot(files []rcv1.File, fid *identity.FullIdentity) (string, error) { diff --git a/politeiawww/cmd/pictl/votepolicy.go b/politeiawww/cmd/pictl/votepolicy.go new file mode 100644 index 000000000..75a7a2ebd --- /dev/null +++ b/politeiawww/cmd/pictl/votepolicy.go @@ -0,0 +1,39 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + pclient "github.com/decred/politeia/politeiawww/client" +) + +// votePolicy retrieves the ticketvote API policy. +type votePolicyCmd struct{} + +// Execute executes the votePolicyCmd command. +// +// This function satisfies the go-flags Commander interface. +func (cmd *votePolicyCmd) Execute(args []string) error { + // Setup client + pc, err := pclient.New(cfg.Host, cfg.HTTPSCert, nil, "") + if err != nil { + return err + } + + // Get policy + pr, err := pc.TicketVotePolicy() + if err != nil { + return err + } + + // Print policy + println(formatJSON(pr)) + + return nil +} + +// votePolicyHelpMsg is the command help message. +const votePolicyHelpMsg = `votepolicy + +Fetch the ticketvote API policy.` diff --git a/politeiawww/pi/pi.go b/politeiawww/pi/pi.go index c7753ad0f..11d717cf5 100644 --- a/politeiawww/pi/pi.go +++ b/politeiawww/pi/pi.go @@ -26,28 +26,14 @@ type Pi struct { politeiad *pdclient.Client userdb user.Database sessions *sessions.Sessions - - // Plugin settings - textFileSizeMax uint32 // In bytes - imageFileCountMax uint32 - imageFileSizeMax uint32 // In bytes - nameLengthMin uint32 // In characters - nameLengthMax uint32 // In characters - nameSupportedChars []string + policy *v1.PolicyReply } // HandlePolicy is the request handler for the pi v1 Policy route. func (p *Pi) HandlePolicy(w http.ResponseWriter, r *http.Request) { log.Tracef("HandlePolicy") - pr, err := p.processPolicy(r.Context()) - if err != nil { - respondWithError(w, r, - "HandlePolicy: processPolicy: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, pr) + util.RespondWithJSON(w, http.StatusOK, p.policy) } // HandleProposals is the request handler for the pi v1 Proposals route. @@ -100,7 +86,7 @@ func (p *Pi) HandleVoteInventory(w http.ResponseWriter, r *http.Request) { // New returns a new Pi context. func New(cfg *config.Config, pdc *pdclient.Client, udb user.Database, s *sessions.Sessions, plugins []pdv1.Plugin) (*Pi, error) { - // Parse pi plugin settings + // Parse plugin settings var ( textFileSizeMax uint32 imageFileCountMax uint32 @@ -109,53 +95,53 @@ func New(cfg *config.Config, pdc *pdclient.Client, udb user.Database, s *session nameLengthMax uint32 nameSupportedChars []string ) - for _, v := range plugins { - if v.ID != pi.PluginID { + for _, p := range plugins { + if p.ID != pi.PluginID { // Not the pi plugin; skip continue } - for _, s := range v.Settings { - switch s.Key { + for _, v := range p.Settings { + switch v.Key { case pi.SettingKeyTextFileSizeMax: - u, err := strconv.ParseUint(s.Value, 10, 64) + u, err := strconv.ParseUint(v.Value, 10, 64) if err != nil { return nil, err } textFileSizeMax = uint32(u) case pi.SettingKeyImageFileCountMax: - u, err := strconv.ParseUint(s.Value, 10, 64) + u, err := strconv.ParseUint(v.Value, 10, 64) if err != nil { return nil, err } imageFileCountMax = uint32(u) case pi.SettingKeyImageFileSizeMax: - u, err := strconv.ParseUint(s.Value, 10, 64) + u, err := strconv.ParseUint(v.Value, 10, 64) if err != nil { return nil, err } imageFileSizeMax = uint32(u) case pi.SettingKeyProposalNameLengthMin: - u, err := strconv.ParseUint(s.Value, 10, 64) + u, err := strconv.ParseUint(v.Value, 10, 64) if err != nil { return nil, err } nameLengthMin = uint32(u) case pi.SettingKeyProposalNameLengthMax: - u, err := strconv.ParseUint(s.Value, 10, 64) + u, err := strconv.ParseUint(v.Value, 10, 64) if err != nil { return nil, err } nameLengthMax = uint32(u) case pi.SettingKeyProposalNameSupportedChars: var sc []string - err := json.Unmarshal([]byte(s.Value), &sc) + err := json.Unmarshal([]byte(v.Value), &sc) if err != nil { return nil, err } nameSupportedChars = sc default: // Skip unknown settings - log.Warnf("Unknown plugin setting %v; Skipping...", s.Key) + log.Warnf("Unknown plugin setting %v; Skipping...", v.Key) } } } @@ -180,15 +166,17 @@ func New(cfg *config.Config, pdc *pdclient.Client, udb user.Database, s *session } return &Pi{ - cfg: cfg, - politeiad: pdc, - userdb: udb, - sessions: s, - textFileSizeMax: textFileSizeMax, - imageFileCountMax: imageFileCountMax, - imageFileSizeMax: imageFileSizeMax, - nameLengthMin: nameLengthMin, - nameLengthMax: nameLengthMax, - nameSupportedChars: nameSupportedChars, + cfg: cfg, + politeiad: pdc, + userdb: udb, + sessions: s, + policy: &v1.PolicyReply{ + TextFileSizeMax: textFileSizeMax, + ImageFileCountMax: imageFileCountMax, + ImageFileSizeMax: imageFileSizeMax, + NameLengthMin: nameLengthMin, + NameLengthMax: nameLengthMax, + NameSupportedChars: nameSupportedChars, + }, }, nil } diff --git a/politeiawww/pi/process.go b/politeiawww/pi/process.go index baeb97322..419f27fd5 100644 --- a/politeiawww/pi/process.go +++ b/politeiawww/pi/process.go @@ -21,19 +21,6 @@ import ( "github.com/google/uuid" ) -func (p *Pi) processPolicy(ctx context.Context) (*v1.PolicyReply, error) { - log.Tracef("Policy") - - return &v1.PolicyReply{ - TextFileSizeMax: p.textFileSizeMax, - ImageFileCountMax: p.imageFileCountMax, - ImageFileSizeMax: p.imageFileSizeMax, - NameLengthMin: p.nameLengthMin, - NameLengthMax: p.nameLengthMax, - NameSupportedChars: p.nameSupportedChars, - }, nil -} - // proposal returns a version of a proposal record from politeiad. If version // is an empty string then the most recent version will be returned. func (p *Pi) proposal(ctx context.Context, state, token, version string) (*v1.Proposal, error) { diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index a45ac6ea4..67ce85137 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -110,6 +110,9 @@ func (p *politeiawww) setupPiRoutes(r *records.Records, c *comments.Comments, t permissionPublic) // Ticket vote routes + p.addRoute(http.MethodGet, tkv1.APIRoute, + tkv1.RoutePolicy, t.HandlePolicy, + permissionPublic) p.addRoute(http.MethodPost, tkv1.APIRoute, tkv1.RouteAuthorize, t.HandleAuthorize, permissionLogin) @@ -174,9 +177,13 @@ func (p *politeiawww) setupPi(plugins []pdv1.Plugin) error { } // Setup api contexts - c := comments.New(p.cfg, p.politeiad, p.db, p.sessions, p.events) - tv := ticketvote.New(p.cfg, p.politeiad, p.sessions, p.events) r := records.New(p.cfg, p.politeiad, p.db, p.sessions, p.events) + c := comments.New(p.cfg, p.politeiad, p.db, p.sessions, p.events) + tv, err := ticketvote.New(p.cfg, p.politeiad, + p.sessions, p.events, plugins) + if err != nil { + return fmt.Errorf("new ticketvote: %v", err) + } pic, err := pi.New(p.cfg, p.politeiad, p.db, p.sessions, plugins) if err != nil { return fmt.Errorf("new pi: %v", err) diff --git a/politeiawww/ticketvote/ticketvote.go b/politeiawww/ticketvote/ticketvote.go index d04ad5364..5cbbcd0bd 100644 --- a/politeiawww/ticketvote/ticketvote.go +++ b/politeiawww/ticketvote/ticketvote.go @@ -6,9 +6,13 @@ package ticketvote import ( "encoding/json" + "fmt" "net/http" + "strconv" + pdv1 "github.com/decred/politeia/politeiad/api/v1" pdclient "github.com/decred/politeia/politeiad/client" + "github.com/decred/politeia/politeiad/plugins/ticketvote" v1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/events" @@ -22,6 +26,14 @@ type TicketVote struct { politeiad *pdclient.Client sessions *sessions.Sessions events *events.Manager + policy *v1.PolicyReply +} + +// HandlePolicy is the request handler for the ticketvote v1 Policy route. +func (t *TicketVote) HandlePolicy(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandlePolicy") + + util.RespondWithJSON(w, http.StatusOK, t.policy) } // HandleAuthorize is the request handler for the ticketvote v1 Authorize @@ -251,11 +263,77 @@ func (t *TicketVote) HandleTimestamps(w http.ResponseWriter, r *http.Request) { } // New returns a new TicketVote context. -func New(cfg *config.Config, pdc *pdclient.Client, s *sessions.Sessions, e *events.Manager) *TicketVote { +func New(cfg *config.Config, pdc *pdclient.Client, s *sessions.Sessions, e *events.Manager, plugins []pdv1.Plugin) (*TicketVote, error) { + // Parse plugin settings + var ( + linkByPeriodMin int64 + linkByPeriodMax int64 + voteDurationMin uint32 + voteDurationMax uint32 + ) + for _, p := range plugins { + if p.ID != ticketvote.PluginID { + // Wrong plugin; skip + continue + } + for _, v := range p.Settings { + switch v.Key { + case ticketvote.SettingKeyLinkByPeriodMin: + i, err := strconv.ParseInt(v.Value, 10, 64) + if err != nil { + return nil, err + } + linkByPeriodMin = i + case ticketvote.SettingKeyLinkByPeriodMax: + i, err := strconv.ParseInt(v.Value, 10, 64) + if err != nil { + return nil, err + } + linkByPeriodMax = i + case ticketvote.SettingKeyVoteDurationMin: + u, err := strconv.ParseUint(v.Value, 10, 64) + if err != nil { + return nil, err + } + voteDurationMin = uint32(u) + case ticketvote.SettingKeyVoteDurationMax: + u, err := strconv.ParseUint(v.Value, 10, 64) + if err != nil { + return nil, err + } + voteDurationMax = uint32(u) + default: + log.Warnf("Unknown plugin setting %v; Skipping...", v.Key) + } + } + } + + // Verify all plugin settings have been provided + switch { + case linkByPeriodMin == 0: + return nil, fmt.Errorf("plugin setting not found: %v", + ticketvote.SettingKeyLinkByPeriodMin) + case linkByPeriodMax == 0: + return nil, fmt.Errorf("plugin setting not found: %v", + ticketvote.SettingKeyLinkByPeriodMax) + case voteDurationMin == 0: + return nil, fmt.Errorf("plugin setting not found: %v", + ticketvote.SettingKeyVoteDurationMin) + case voteDurationMax == 0: + return nil, fmt.Errorf("plugin setting not found: %v", + ticketvote.SettingKeyVoteDurationMax) + } + return &TicketVote{ cfg: cfg, politeiad: pdc, sessions: s, events: e, - } + policy: &v1.PolicyReply{ + LinkByPeriodMin: linkByPeriodMin, + LinkByPeriodMax: linkByPeriodMax, + VoteDurationMin: voteDurationMin, + VoteDurationMax: voteDurationMax, + }, + }, nil } From a03db1d80d11a9c7eb42ef74c5336264ae98b24c Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 1 Feb 2021 15:16:09 -0600 Subject: [PATCH 262/449] Fix proposal validation bug. --- politeiad/backend/tlogbe/plugins/pi/hooks.go | 161 ++++++++----------- politeiad/plugins/pi/pi.go | 15 +- politeiawww/cmd/pictl/pictl.go | 4 +- politeiawww/cmd/pictl/proposaledit.go | 5 - politeiawww/cmd/pictl/proposalnew.go | 11 +- politeiawww/cmd/pictl/proposalpolicy.go | 39 +++++ 6 files changed, 119 insertions(+), 116 deletions(-) create mode 100644 politeiawww/cmd/pictl/proposalpolicy.go diff --git a/politeiad/backend/tlogbe/plugins/pi/hooks.go b/politeiad/backend/tlogbe/plugins/pi/hooks.go index f0341c249..2ac0d20de 100644 --- a/politeiad/backend/tlogbe/plugins/pi/hooks.go +++ b/politeiad/backend/tlogbe/plugins/pi/hooks.go @@ -25,9 +25,9 @@ const ( ) var ( - // allowedTextFileNames contains the only file names that are - // allowed for text files. - allowedTextFileNames = map[string]struct{}{ + // allowedTextFiles contains the only text files that are allowed + // to be submitted as part of a proposal. + allowedTextFiles = map[string]struct{}{ pi.FileNameIndexFile: {}, pi.FileNameProposalMetadata: {}, ticketvote.FileNameVoteMetadata: {}, @@ -71,109 +71,82 @@ func (p *piPlugin) proposalNameIsValid(name string) bool { // passed politeia validation so we can assume that the file has a unique name, // a valid base64 payload, and that the file digest and MIME type are correct. func (p *piPlugin) proposalFilesVerify(files []backend.File) error { - // TODO this verification assumes the user provided is passed - // in as metadata and not files. - /* - var ( - textFilesCount uint32 - imageFilesCount uint32 - indexFileFound bool - ) - for _, v := range files { - payload, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return fmt.Errorf("invalid base64 %v", v.Name) - } - - // MIME type specific validation - switch v.MIME { - case mimeTypeText, mimeTypeTextUTF8: - textFilesCount++ - - // The text file must be the proposal index file - if v.Name != pi.FileNameIndexFile { - e := fmt.Sprintf("want %v, got %v", pi.FileNameIndexFile, v.Name) - return backend.PluginError{ - PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeIndexFileNameInvalid), - ErrorContext: e, - } - } + var imagesCount uint32 + for _, v := range files { + payload, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return fmt.Errorf("invalid base64 %v", v.Name) + } - // Verify text file size - if len(payload) > int(p.textFileSizeMax) { - e := fmt.Sprintf("file %v size %v exceeds max size %v", - v.Name, len(payload), p.textFileSizeMax) - return backend.PluginError{ - PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeIndexFileSizeInvalid), - ErrorContext: e, - } + // MIME type specific validation + switch v.MIME { + case mimeTypeText, mimeTypeTextUTF8: + // Verify text file is allowed + _, ok := allowedTextFiles[v.Name] + if !ok { + return backend.PluginError{ + PluginID: pi.PluginID, + ErrorCode: int(pi.ErrorCodeIndexFileNameInvalid), + ErrorContext: v.Name, } + } - // Verify there isn't more than one index file - if indexFileFound { - e := fmt.Sprintf("more than one %v file found", - pi.FileNameIndexFile) - return backend.PluginError{ - PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeIndexFileCountInvalid), - ErrorContext: e, - } + // Verify text file size + if len(payload) > int(p.textFileSizeMax) { + e := fmt.Sprintf("file %v size %v exceeds max size %v", + v.Name, len(payload), p.textFileSizeMax) + return backend.PluginError{ + PluginID: pi.PluginID, + ErrorCode: int(pi.ErrorCodeIndexFileSizeInvalid), + ErrorContext: e, } + } - // Set index file as being found - indexFileFound = true - - case mimeTypePNG: - imageFilesCount++ - - // Verify image file size - if len(payload) > int(p.imageFileSizeMax) { - e := fmt.Sprintf("image %v size %v exceeds max size %v", - v.Name, len(payload), p.imageFileSizeMax) - return backend.PluginError{ - PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeImageFileSizeInvalid), - ErrorContext: e, - } + case mimeTypePNG: + imagesCount++ + + // Verify image file size + if len(payload) > int(p.imageFileSizeMax) { + e := fmt.Sprintf("image %v size %v exceeds max size %v", + v.Name, len(payload), p.imageFileSizeMax) + return backend.PluginError{ + PluginID: pi.PluginID, + ErrorCode: int(pi.ErrorCodeImageFileSizeInvalid), + ErrorContext: e, } - - default: - return fmt.Errorf("invalid mime") } - } - // Verify that an index file is present - if !indexFileFound { - e := fmt.Sprintf("%v file not found", pi.FileNameIndexFile) - return backend.PluginError{ - PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeIndexFileCountInvalid), - ErrorContext: e, - } + default: + return fmt.Errorf("invalid mime") } + } - // Verify file counts are acceptable - if textFilesCount > p.textFileCountMax { - e := fmt.Sprintf("got %v text files, max is %v", - textFilesCount, p.textFileCountMax) - return backend.PluginError{ - PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeTextFileCountInvalid), - ErrorContext: e, - } + // Verify that an index file is present + var found bool + for _, v := range files { + if v.Name == pi.FileNameIndexFile { + found = true + break } - if imageFilesCount > p.imageFileCountMax { - e := fmt.Sprintf("got %v image files, max is %v", - imageFilesCount, p.imageFileCountMax) - return backend.PluginError{ - PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeImageFileCountInvalid), - ErrorContext: e, - } + } + if !found { + return backend.PluginError{ + PluginID: pi.PluginID, + ErrorCode: int(pi.ErrorCodeFileMissing), + ErrorContext: pi.FileNameIndexFile, + } + } + + // Verify image file count is acceptable + if imagesCount > p.imageFileCountMax { + e := fmt.Sprintf("got %v image files, max is %v", + imagesCount, p.imageFileCountMax) + return backend.PluginError{ + PluginID: pi.PluginID, + ErrorCode: int(pi.ErrorCodeImageFileCountInvalid), + ErrorContext: e, } - */ + } // Verify a proposal metadata has been included pm, err := proposalMetadataDecode(files) diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index 22cf4b50d..f26538365 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -59,14 +59,13 @@ type ErrorCodeT int const ( ErrorCodeInvalid ErrorCodeT = 0 ErrorCodeIndexFileNameInvalid ErrorCodeT = 1 - ErrorCodeIndexFileCountInvalid ErrorCodeT = 2 - ErrorCodeIndexFileSizeInvalid ErrorCodeT = 3 - ErrorCodeTextFileCountInvalid ErrorCodeT = 4 - ErrorCodeImageFileCountInvalid ErrorCodeT = 5 - ErrorCodeImageFileSizeInvalid ErrorCodeT = 6 - ErrorCodeProposalMetadataInvalid ErrorCodeT = 7 - ErrorCodeProposalNameInvalid ErrorCodeT = 8 - ErrorCodeVoteStatusInvalid ErrorCodeT = 9 + ErrorCodeIndexFileSizeInvalid ErrorCodeT = 2 + ErrorCodeFileMissing ErrorCodeT = 3 + ErrorCodeImageFileCountInvalid ErrorCodeT = 4 + ErrorCodeImageFileSizeInvalid ErrorCodeT = 5 + ErrorCodeProposalMetadataInvalid ErrorCodeT = 6 + ErrorCodeProposalNameInvalid ErrorCodeT = 7 + ErrorCodeVoteStatusInvalid ErrorCodeT = 8 ) var ( diff --git a/politeiawww/cmd/pictl/pictl.go b/politeiawww/cmd/pictl/pictl.go index b43f3ae87..4a22c8cdc 100644 --- a/politeiawww/cmd/pictl/pictl.go +++ b/politeiawww/cmd/pictl/pictl.go @@ -64,7 +64,7 @@ type pictl struct { Users shared.UsersCmd `command:"users"` // Proposal commands - // TODO ProposalPolicy + ProposalPolicy proposalPolicyCmd `command:"proposalpolicy"` ProposalNew proposalNewCmd `command:"proposalnew"` ProposalEdit proposalEditCmd `command:"proposaledit"` ProposalSetStatus proposalSetStatusCmd `command:"proposalsetstatus"` @@ -140,6 +140,7 @@ User commands users (public) Get users Proposal commands + proposalpolicy (public) Get the pi api policy proposalnew (user) Submit a new proposal proposaledit (user) Edit an existing proposal proposalstatusset (admin) Set the status of a proposal @@ -155,6 +156,7 @@ Comment commands commentvotes (public) Get comment votes Vote commands + votepolicy (public) Get the ticketvote api policy voteauthorize (user) Authorize a proposal vote votestart (admin) Start a proposal vote votestartrunoff (admin) Start a runoff vote diff --git a/politeiawww/cmd/pictl/proposaledit.go b/politeiawww/cmd/pictl/proposaledit.go index d7c6a5613..dc77ec3b7 100644 --- a/politeiawww/cmd/pictl/proposaledit.go +++ b/politeiawww/cmd/pictl/proposaledit.go @@ -59,11 +59,6 @@ func (c *proposalEditCmd) Execute(args []string) error { return fmt.Errorf("you cannot provide file arguments and use " + "the --random flag at the same time") - case !c.Random && c.Name == "": - return fmt.Errorf("you must either provide a proposal name " + - "using the --name flag or use the --random flag to generate " + - "a random proposal") - case c.RFP && c.LinkBy != 0: return fmt.Errorf("you cannot use both the --rfp and --linkby " + "flags at the same time") diff --git a/politeiawww/cmd/pictl/proposalnew.go b/politeiawww/cmd/pictl/proposalnew.go index 20501e2f2..e9a05c541 100644 --- a/politeiawww/cmd/pictl/proposalnew.go +++ b/politeiawww/cmd/pictl/proposalnew.go @@ -63,11 +63,6 @@ func (c *proposalNewCmd) Execute(args []string) error { return fmt.Errorf("you cannot provide file arguments and use " + "the --random flag at the same time") - case !c.Random && c.Name == "": - return fmt.Errorf("you must either provide a proposal name " + - "using the --name flag or use the --random flag to generate " + - "a random name") - case c.RFP && c.LinkBy != 0: return fmt.Errorf("you cannot use both the --rfp and --linkby " + "flags at the same time") @@ -136,7 +131,7 @@ func (c *proposalNewCmd) Execute(args []string) error { } // Setup proposal metadata - if c.Random { + if c.Random && c.Name == "" { r, err := util.Random(int(pr.NameLengthMin)) if err != nil { return err @@ -228,8 +223,8 @@ A proposal can be submitted as an RFP submission by using the --linkto flag to link to and an existing RFP proposal. Arguments: -1. indexfile (string, required) Index file -2. attachments (string, optional) Attachment files +1. indexfile (string, required) Index file +2. attachments (string, optional) Attachment files Flags: --name (string, optional) Name of the proposal. diff --git a/politeiawww/cmd/pictl/proposalpolicy.go b/politeiawww/cmd/pictl/proposalpolicy.go new file mode 100644 index 000000000..686cacca8 --- /dev/null +++ b/politeiawww/cmd/pictl/proposalpolicy.go @@ -0,0 +1,39 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + pclient "github.com/decred/politeia/politeiawww/client" +) + +// proposalPolicy retrieves the pi API policy. +type proposalPolicyCmd struct{} + +// Execute executes the proposalPolicyCmd command. +// +// This function satisfies the go-flags Commander interface. +func (cmd *proposalPolicyCmd) Execute(args []string) error { + // Setup client + pc, err := pclient.New(cfg.Host, cfg.HTTPSCert, nil, "") + if err != nil { + return err + } + + // Get policy + pr, err := pc.PiPolicy() + if err != nil { + return err + } + + // Print policy + println(formatJSON(pr)) + + return nil +} + +// proposalPolicyHelpMsg is the command help message. +const proposalPolicyHelpMsg = `proposalpolicy + +Fetch the pi API policy.` From 3ede3a8b75d9420412495f7661db537ee4d9a2c1 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 1 Feb 2021 16:12:42 -0600 Subject: [PATCH 263/449] Add printing to politeiawww client. --- politeiad/backend/tlogbe/plugins/pi/hooks.go | 10 +-- politeiad/plugins/pi/pi.go | 62 +++++++++++---- politeiawww/client/client.go | 79 ++++++++++++++++---- politeiawww/client/pi.go | 4 + politeiawww/client/records.go | 6 ++ politeiawww/client/ticketvote.go | 4 + politeiawww/cmd/pictl/proposaldetails.go | 9 ++- politeiawww/cmd/pictl/proposalnew.go | 9 ++- politeiawww/cmd/pictl/proposalpolicy.go | 7 +- politeiawww/cmd/pictl/votepolicy.go | 7 +- politeiawww/cmd/pictl/votetimestamps.go | 67 ++++++++--------- politeiawww/cmd/shared/client.go | 73 ------------------ 12 files changed, 189 insertions(+), 148 deletions(-) diff --git a/politeiad/backend/tlogbe/plugins/pi/hooks.go b/politeiad/backend/tlogbe/plugins/pi/hooks.go index 2ac0d20de..24ecd7996 100644 --- a/politeiad/backend/tlogbe/plugins/pi/hooks.go +++ b/politeiad/backend/tlogbe/plugins/pi/hooks.go @@ -86,7 +86,7 @@ func (p *piPlugin) proposalFilesVerify(files []backend.File) error { if !ok { return backend.PluginError{ PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeIndexFileNameInvalid), + ErrorCode: int(pi.ErrorCodeTextFileNameInvalid), ErrorContext: v.Name, } } @@ -97,7 +97,7 @@ func (p *piPlugin) proposalFilesVerify(files []backend.File) error { v.Name, len(payload), p.textFileSizeMax) return backend.PluginError{ PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeIndexFileSizeInvalid), + ErrorCode: int(pi.ErrorCodeTextFileSizeInvalid), ErrorContext: e, } } @@ -132,7 +132,7 @@ func (p *piPlugin) proposalFilesVerify(files []backend.File) error { if !found { return backend.PluginError{ PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeFileMissing), + ErrorCode: int(pi.ErrorCodeTextFileMissing), ErrorContext: pi.FileNameIndexFile, } } @@ -156,8 +156,8 @@ func (p *piPlugin) proposalFilesVerify(files []backend.File) error { if pm == nil { return backend.PluginError{ PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeProposalMetadataInvalid), - ErrorContext: "metadata not found", + ErrorCode: int(pi.ErrorCodeTextFileMissing), + ErrorContext: pi.FileNameProposalMetadata, } } diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index f26538365..0a6ebc943 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -12,10 +12,12 @@ const ( // Plugin commands CmdVoteInv = "voteinv" +) - // Setting keys are the plugin setting keys that can be used to - // override a default plugin setting. Defaults will be overridden - // if a plugin setting is provided to the plugin on startup. +// Plugin setting keys and default values. Default plugin setting values can be +// overridden by passing in a custom plugin setting key and value on startup. +const ( + // Plugin setting keys SettingKeyTextFileSizeMax = "textfilesizemax" SettingKeyImageFileCountMax = "imagefilecountmax" SettingKeyImageFileSizeMax = "imagefilesizemax" @@ -48,8 +50,8 @@ var ( // SettingProposalNameSupportedChars contains the supported // characters in a proposal name. SettingProposalNameSupportedChars = []string{ - "A-z", "0-9", "&", ".", ",", ":", ";", "-", " ", "@", "+", "#", "/", - "(", ")", "!", "?", "\"", "'", + "A-z", "0-9", "&", ".", ",", ":", ";", "-", " ", "@", "+", "#", + "/", "(", ")", "!", "?", "\"", "'", } ) @@ -57,21 +59,49 @@ var ( type ErrorCodeT int const ( - ErrorCodeInvalid ErrorCodeT = 0 - ErrorCodeIndexFileNameInvalid ErrorCodeT = 1 - ErrorCodeIndexFileSizeInvalid ErrorCodeT = 2 - ErrorCodeFileMissing ErrorCodeT = 3 - ErrorCodeImageFileCountInvalid ErrorCodeT = 4 - ErrorCodeImageFileSizeInvalid ErrorCodeT = 5 - ErrorCodeProposalMetadataInvalid ErrorCodeT = 6 - ErrorCodeProposalNameInvalid ErrorCodeT = 7 - ErrorCodeVoteStatusInvalid ErrorCodeT = 8 + // ErrorCodeInvalid represents and invalid error code. + ErrorCodeInvalid ErrorCodeT = 0 + + // ErrorCodeTextFileNameInvalid is returned when a text file has + // a file name that is not allowed. + ErrorCodeTextFileNameInvalid ErrorCodeT = 1 + + // ErrorCodeTextFileSizeInvalid is returned when a text file size + // exceedes the TextFileSizeMax setting. + ErrorCodeTextFileSizeInvalid ErrorCodeT = 2 + + // ErrorCodeTextFileMissing is returned when the proposal does not + // contain one or more of the required text files. + ErrorCodeTextFileMissing ErrorCodeT = 3 + + // ErrorCodeImageFileCountInvalid is returned when the number of + // image attachments exceedes the ImageFileCountMax setting. + ErrorCodeImageFileCountInvalid ErrorCodeT = 4 + + // ErrorCodeImageFileSizeInvalid is returned when an image file + // size exceedes the ImageFileSizeMax setting. + ErrorCodeImageFileSizeInvalid ErrorCodeT = 5 + + // ErrorCodeProposalNameInvalid is returned when a proposal name + // does not adhere to the proposal name settings. + ErrorCodeProposalNameInvalid ErrorCodeT = 6 + + // ErrorCodeVoteStatusInvalid is returned when a proposal vote + // status does not allow changes to be made to the proposal. + ErrorCodeVoteStatusInvalid ErrorCodeT = 7 ) var ( - // TODO ErrorCodes contains the human readable errors. + // ErrorCodes contains the human readable errors. ErrorCodes = map[ErrorCodeT]string{ - ErrorCodeInvalid: "error code invalid", + ErrorCodeInvalid: "error code invalid", + ErrorCodeTextFileNameInvalid: "text file name invalid", + ErrorCodeTextFileSizeInvalid: "text file size invalid", + ErrorCodeTextFileMissing: "text file is misisng", + ErrorCodeImageFileCountInvalid: "image file count invalid", + ErrorCodeImageFileSizeInvalid: "image file size invalid", + ErrorCodeProposalNameInvalid: "proposal name invalid", + ErrorCodeVoteStatusInvalid: "vote status invalid", } ) diff --git a/politeiawww/client/client.go b/politeiawww/client/client.go index 1e577f8b2..0dac848c3 100644 --- a/politeiawww/client/client.go +++ b/politeiawww/client/client.go @@ -25,8 +25,9 @@ var ( // Client provides a client for interacting with the politeiawww API. type Client struct { host string - cert string headerCSRF string // Header csrf token + verbose bool + rawJSON bool http *http.Client } @@ -60,6 +61,15 @@ func (e Error) Error() string { } } +// formatJSON returns a pretty printed JSON string for the provided structure. +func formatJSON(v interface{}) string { + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + return fmt.Sprintf("MarshalIndent: %v", err) + } + return string(b) +} + // makeReq makes a politeiawww http request to the method and route provided, // serializing the provided object as the request body, and returning a byte // slice of the repsonse body. An Error is returned if politeiawww responds @@ -99,8 +109,23 @@ func (c *Client) makeReq(method string, route string, v interface{}) ([]byte, er } } - // Send request + // Setup route fullRoute := c.host + route + queryParams + + // Print request details + switch { + case method == http.MethodGet && c.verbose: + fmt.Printf("Request: %v %v\n", method, fullRoute) + case method == http.MethodGet && c.rawJSON: + // No JSON to print + case c.verbose: + fmt.Printf("Request: %v %v\n", method, fullRoute) + fmt.Printf("%v\n", formatJSON(v)) + case c.rawJSON: + fmt.Printf("%s\n", reqBody) + } + + // Send request req, err := http.NewRequest(method, fullRoute, bytes.NewReader(reqBody)) if err != nil { return nil, err @@ -114,6 +139,11 @@ func (c *Client) makeReq(method string, route string, v interface{}) ([]byte, er } defer r.Body.Close() + // Print response code + if c.verbose { + fmt.Printf("Response: %v\n", r.StatusCode) + } + // Handle reply if r.StatusCode != http.StatusOK { var e ErrorReply @@ -127,30 +157,46 @@ func (c *Client) makeReq(method string, route string, v interface{}) ([]byte, er } } + // Decode response body respBody := util.ConvertBodyToByteArray(r.Body, false) + + // Print response body. Pretty printing the response body for the + // verbose output must be handled by the calling function once it + // has unmarshalled the body. + if c.rawJSON { + fmt.Printf("%s\n", respBody) + } + return respBody, nil } -// New returns a new politeiawww client. -// -// The cert argument is optional. Any provided cert will be added to the http -// client's trust cert pool. This allows you to interact with a politeiawww -// instance that uses a self signed cert. +// Opts contains the politeiawww client options. All values are optional. // -// The cookies and headerCSRF arguments are optional. -func New(host, cert string, cookies []*http.Cookie, headerCSRF string) (*Client, error) { +// Any provided HTTPSCert will be added to the http client's trust cert pool. +// This allows you to interact with a politeiawww instance that uses a self +// signed cert. +type Opts struct { + HTTPSCert string + Cookies []*http.Cookie + HeaderCSRF string + Verbose bool // Pretty print details + RawJSON bool // Print raw json +} + +// New returns a new politeiawww client. +func New(host string, opts Opts) (*Client, error) { // Setup http client - h, err := util.NewHTTPClient(false, cert) + h, err := util.NewHTTPClient(false, opts.HTTPSCert) if err != nil { return nil, err } // Setup cookies - if cookies != nil { - opt := cookiejar.Options{ + if opts.Cookies != nil { + copt := cookiejar.Options{ PublicSuffixList: publicsuffix.List, } - jar, err := cookiejar.New(&opt) + jar, err := cookiejar.New(&copt) if err != nil { return nil, err } @@ -158,14 +204,15 @@ func New(host, cert string, cookies []*http.Cookie, headerCSRF string) (*Client, if err != nil { return nil, err } - jar.SetCookies(u, cookies) + jar.SetCookies(u, opts.Cookies) h.Jar = jar } return &Client{ host: host, - cert: cert, - headerCSRF: headerCSRF, + headerCSRF: opts.HeaderCSRF, + verbose: opts.Verbose, + rawJSON: opts.RawJSON, http: h, }, nil } diff --git a/politeiawww/client/pi.go b/politeiawww/client/pi.go index 61680b8ef..02ae61813 100644 --- a/politeiawww/client/pi.go +++ b/politeiawww/client/pi.go @@ -6,6 +6,7 @@ package client import ( "encoding/json" + "fmt" "net/http" piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" @@ -24,6 +25,9 @@ func (c *Client) PiPolicy() (*piv1.PolicyReply, error) { if err != nil { return nil, err } + if c.verbose { + fmt.Printf("%v\n", formatJSON(pr)) + } return &pr, nil } diff --git a/politeiawww/client/records.go b/politeiawww/client/records.go index 62db08419..4fb72c6b9 100644 --- a/politeiawww/client/records.go +++ b/politeiawww/client/records.go @@ -29,6 +29,9 @@ func (c *Client) RecordNew(n rcv1.New) (*rcv1.NewReply, error) { if err != nil { return nil, err } + if c.verbose { + fmt.Printf("%v\n", formatJSON(nr)) + } return &nr, nil } @@ -46,6 +49,9 @@ func (c *Client) RecordDetails(d rcv1.Details) (*rcv1.DetailsReply, error) { if err != nil { return nil, err } + if c.verbose { + fmt.Printf("%v\n", formatJSON(dr)) + } return &dr, nil } diff --git a/politeiawww/client/ticketvote.go b/politeiawww/client/ticketvote.go index 96546fda8..ff7025cb5 100644 --- a/politeiawww/client/ticketvote.go +++ b/politeiawww/client/ticketvote.go @@ -6,6 +6,7 @@ package client import ( "encoding/json" + "fmt" "net/http" tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" @@ -24,6 +25,9 @@ func (c *Client) TicketVotePolicy() (*tkv1.PolicyReply, error) { if err != nil { return nil, err } + if c.verbose { + fmt.Printf("%v\n", formatJSON(pr)) + } return &pr, nil } diff --git a/politeiawww/cmd/pictl/proposaldetails.go b/politeiawww/cmd/pictl/proposaldetails.go index 2f724b6b9..f51c809fc 100644 --- a/politeiawww/cmd/pictl/proposaldetails.go +++ b/politeiawww/cmd/pictl/proposaldetails.go @@ -28,7 +28,14 @@ type proposalDetailsCmd struct { // This function satisfies the go-flags Commander interface. func (c *proposalDetailsCmd) Execute(args []string) error { // Setup client - pc, err := pclient.New(cfg.Host, cfg.HTTPSCert, cfg.Cookies, cfg.CSRF) + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Cookies: cfg.Cookies, + HeaderCSRF: cfg.CSRF, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, + } + pc, err := pclient.New(cfg.Host, opts) if err != nil { return err } diff --git a/politeiawww/cmd/pictl/proposalnew.go b/politeiawww/cmd/pictl/proposalnew.go index e9a05c541..0cefc6a36 100644 --- a/politeiawww/cmd/pictl/proposalnew.go +++ b/politeiawww/cmd/pictl/proposalnew.go @@ -75,7 +75,14 @@ func (c *proposalNewCmd) Execute(args []string) error { } // Setup client - pc, err := pclient.New(cfg.Host, cfg.HTTPSCert, cfg.Cookies, cfg.CSRF) + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Cookies: cfg.Cookies, + HeaderCSRF: cfg.CSRF, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, + } + pc, err := pclient.New(cfg.Host, opts) if err != nil { return err } diff --git a/politeiawww/cmd/pictl/proposalpolicy.go b/politeiawww/cmd/pictl/proposalpolicy.go index 686cacca8..476a37c03 100644 --- a/politeiawww/cmd/pictl/proposalpolicy.go +++ b/politeiawww/cmd/pictl/proposalpolicy.go @@ -16,7 +16,12 @@ type proposalPolicyCmd struct{} // This function satisfies the go-flags Commander interface. func (cmd *proposalPolicyCmd) Execute(args []string) error { // Setup client - pc, err := pclient.New(cfg.Host, cfg.HTTPSCert, nil, "") + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, + } + pc, err := pclient.New(cfg.Host, opts) if err != nil { return err } diff --git a/politeiawww/cmd/pictl/votepolicy.go b/politeiawww/cmd/pictl/votepolicy.go index 75a7a2ebd..99814a742 100644 --- a/politeiawww/cmd/pictl/votepolicy.go +++ b/politeiawww/cmd/pictl/votepolicy.go @@ -16,7 +16,12 @@ type votePolicyCmd struct{} // This function satisfies the go-flags Commander interface. func (cmd *votePolicyCmd) Execute(args []string) error { // Setup client - pc, err := pclient.New(cfg.Host, cfg.HTTPSCert, nil, "") + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, + } + pc, err := pclient.New(cfg.Host, opts) if err != nil { return err } diff --git a/politeiawww/cmd/pictl/votetimestamps.go b/politeiawww/cmd/pictl/votetimestamps.go index 9d6656bba..6e1883e74 100644 --- a/politeiawww/cmd/pictl/votetimestamps.go +++ b/politeiawww/cmd/pictl/votetimestamps.go @@ -5,12 +5,9 @@ package main import ( - "fmt" - "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" ) // voteTimestampsCmd retrieves the timestamps for a politeiawww ticket vote. @@ -24,42 +21,44 @@ type voteTimestampsCmd struct { // // This function satisfies the go-flags Commander interface. func (c *voteTimestampsCmd) Execute(args []string) error { - // Setup request - t := tkv1.Timestamps{ - Token: c.Args.Token, - } - - // Send request - err := shared.PrintJSON(t) - if err != nil { - return err - } - tr, err := client.TicketVoteTimestamps(t) - if err != nil { - return err - } - err = shared.PrintJSON(tr) - if err != nil { - return err - } + /* + // Setup request + t := tkv1.Timestamps{ + Token: c.Args.Token, + } - // Verify timestamps - for k, v := range tr.Auths { - err = verifyVoteTimestamp(v) + // Send request + err := shared.PrintJSON(t) if err != nil { - return fmt.Errorf("verify authorization %v timestamp: %v", k, err) + return err } - } - err = verifyVoteTimestamp(tr.Details) - if err != nil { - return fmt.Errorf("verify vote details timestamp: %v", err) - } - for k, v := range tr.Votes { - err = verifyVoteTimestamp(v) + tr, err := client.TicketVoteTimestamps(t) if err != nil { - return fmt.Errorf("verify vote %v timestamp: %v", k, err) + return err } - } + err = shared.PrintJSON(tr) + if err != nil { + return err + } + + // Verify timestamps + for k, v := range tr.Auths { + err = verifyVoteTimestamp(v) + if err != nil { + return fmt.Errorf("verify authorization %v timestamp: %v", k, err) + } + } + err = verifyVoteTimestamp(tr.Details) + if err != nil { + return fmt.Errorf("verify vote details timestamp: %v", err) + } + for k, v := range tr.Votes { + err = verifyVoteTimestamp(v) + if err != nil { + return fmt.Errorf("verify vote %v timestamp: %v", k, err) + } + } + */ return nil } diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index f610e326c..80aad4f62 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -21,8 +21,6 @@ import ( "decred.org/dcrwallet/rpc/walletrpc" cms "github.com/decred/politeia/politeiawww/api/cms/v1" - rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" - tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" www2 "github.com/decred/politeia/politeiawww/api/www/v2" "github.com/decred/politeia/util" @@ -117,21 +115,6 @@ func wwwError(body []byte, statusCode int) error { return nil } -// TODO implement recordsError -func recordsError(body []byte, statusCode int) error { - return fmt.Errorf("%v %s", statusCode, body) -} - -// TODO implement commentsError -func commentsError(body []byte, statusCode int) error { - return fmt.Errorf("%v %s", statusCode, body) -} - -// TODO implement ticketVoteError -func ticketVoteError(body []byte, statusCode int) error { - return fmt.Errorf("%v %s", statusCode, body) -} - // makeRequest sends the provided request to the politeiawww backend specified // by the Client config. This function handles verbose printing when specified // by the Client config since verbose printing includes details such as the @@ -804,62 +787,6 @@ func (c *Client) UserProposalPaywall() (*www.UserProposalPaywallReply, error) { return &ppdr, nil } -// RecordTimestamps sends the Timestamps command to politeiawww records API. -func (c *Client) RecordTimestamps(t rcv1.Timestamps) (*rcv1.TimestampsReply, error) { - statusCode, respBody, err := c.makeRequest(http.MethodPost, - rcv1.APIRoute, rcv1.RouteTimestamps, t) - if err != nil { - return nil, err - } - if statusCode != http.StatusOK { - return nil, recordsError(respBody, statusCode) - } - - var tr rcv1.TimestampsReply - err = json.Unmarshal(respBody, &tr) - if err != nil { - return nil, err - } - - if c.cfg.Verbose { - err := prettyPrintJSON(tr) - if err != nil { - return nil, err - } - } - - return &tr, nil -} - -// TicketVoteTimestamps sends the Timestamps command to politeiawww ticketvote -// API. -func (c *Client) TicketVoteTimestamps(t tkv1.Timestamps) (*tkv1.TimestampsReply, - error) { - statusCode, respBody, err := c.makeRequest(http.MethodPost, - tkv1.APIRoute, tkv1.RouteTimestamps, t) - if err != nil { - return nil, err - } - if statusCode != http.StatusOK { - return nil, ticketVoteError(respBody, statusCode) - } - - var tr tkv1.TimestampsReply - err = json.Unmarshal(respBody, &tr) - if err != nil { - return nil, err - } - - if c.cfg.Verbose { - err := prettyPrintJSON(tr) - if err != nil { - return nil, err - } - } - - return &tr, nil -} - // NewInvoice submits the specified invoice to politeiawww for the logged in // user. func (c *Client) NewInvoice(ni *cms.NewInvoice) (*cms.NewInvoiceReply, error) { From c5ebbceb9c6662e0028622fbe47938a13e2b941b Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 1 Feb 2021 16:51:40 -0600 Subject: [PATCH 264/449] add temp pi user errors --- politeiad/plugins/comments/comments.go | 2 +- politeiad/plugins/dcrdata/dcrdata.go | 2 +- politeiad/plugins/pi/pi.go | 10 +-- politeiad/plugins/ticketvote/ticketvote.go | 2 +- politeiad/plugins/user/user.go | 2 +- politeiawww/api/comments/v1/v1.go | 2 + politeiawww/comments/process.go | 9 +- politeiawww/pi/user.go | 36 ++++++++ politeiawww/records/error.go | 98 +++++++++++----------- politeiawww/records/pi.go | 96 +++++++++++++++++++++ politeiawww/records/process.go | 84 ------------------- 11 files changed, 195 insertions(+), 148 deletions(-) create mode 100644 politeiawww/pi/user.go create mode 100644 politeiawww/records/pi.go diff --git a/politeiad/plugins/comments/comments.go b/politeiad/plugins/comments/comments.go index 4d3f2d4d3..7ed7b2b5d 100644 --- a/politeiad/plugins/comments/comments.go +++ b/politeiad/plugins/comments/comments.go @@ -8,7 +8,7 @@ package comments const ( // PluginID is the comments plugin ID. - PluginID = "comments" + PluginID = "politeiad-comments" // Plugin commands CmdNew = "new" // Create a new comment diff --git a/politeiad/plugins/dcrdata/dcrdata.go b/politeiad/plugins/dcrdata/dcrdata.go index 1a9235625..b3252c948 100644 --- a/politeiad/plugins/dcrdata/dcrdata.go +++ b/politeiad/plugins/dcrdata/dcrdata.go @@ -9,7 +9,7 @@ package dcrdata type StatusT int const ( - PluginID = "dcrdata" + PluginID = "politeiad-dcrdata" // Plugin commands CmdBestBlock = "bestblock" // Get best block diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index 0a6ebc943..cb7f62acb 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -8,16 +8,14 @@ package pi const ( // PluginID is the pi plugin ID. - PluginID = "pi" + PluginID = "politeiad-pi" // Plugin commands CmdVoteInv = "voteinv" -) -// Plugin setting keys and default values. Default plugin setting values can be -// overridden by passing in a custom plugin setting key and value on startup. -const ( - // Plugin setting keys + // Plugin setting keys and default values. Default plugin setting + // values can be overridden by passing in a custom plugin setting + // key and value on startup. SettingKeyTextFileSizeMax = "textfilesizemax" SettingKeyImageFileCountMax = "imagefilecountmax" SettingKeyImageFileSizeMax = "imagefilesizemax" diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index ba56a4ba4..0b9786c50 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -15,7 +15,7 @@ package ticketvote const ( // PluginID is the ticketvote plugin ID. - PluginID = "ticketvote" + PluginID = "politeiad-ticketvote" // Plugin commands CmdAuthorize = "authorize" // Authorize a vote diff --git a/politeiad/plugins/user/user.go b/politeiad/plugins/user/user.go index f0edeb1c1..8e7d4c40d 100644 --- a/politeiad/plugins/user/user.go +++ b/politeiad/plugins/user/user.go @@ -7,7 +7,7 @@ package user const ( - PluginID = "user" + PluginID = "politeiad-user" // Plugin commands CmdAuthor = "author" // Get record author diff --git a/politeiawww/api/comments/v1/v1.go b/politeiawww/api/comments/v1/v1.go index 816b95b55..8fd191612 100644 --- a/politeiawww/api/comments/v1/v1.go +++ b/politeiawww/api/comments/v1/v1.go @@ -31,6 +31,7 @@ const ( // Error codes ErrorCodeInvalid ErrorCodeT = 0 ErrorCodeInputInvalid ErrorCodeT = iota + ErrorCodeUnauthorized ErrorCodePublicKeyInvalid ErrorCodeSignatureInvalid ErrorCodeRecordStateInvalid @@ -41,6 +42,7 @@ var ( ErrorCodes = map[ErrorCodeT]string{ ErrorCodeInvalid: "error invalid", ErrorCodeInputInvalid: "input invalid", + ErrorCodeUnauthorized: "unauthorized", ErrorCodePublicKeyInvalid: "public key invalid", ErrorCodeSignatureInvalid: "signature invalid", } diff --git a/politeiawww/comments/process.go b/politeiawww/comments/process.go index bbb30ca4d..c7140a6f1 100644 --- a/politeiawww/comments/process.go +++ b/politeiawww/comments/process.go @@ -10,6 +10,7 @@ import ( "github.com/decred/politeia/politeiad/plugins/comments" v1 "github.com/decred/politeia/politeiawww/api/comments/v1" "github.com/decred/politeia/politeiawww/config" + "github.com/decred/politeia/politeiawww/pi" "github.com/decred/politeia/politeiawww/user" "github.com/google/uuid" ) @@ -23,8 +24,8 @@ func (c *Comments) processNew(ctx context.Context, n v1.New, u user.User) (*v1.N case config.PoliteiaWWWMode: // Verify user has paid registration paywall if !c.userHasPaid(u) { - return nil, v1.UserErrorReply{ - // TODO ErrorCode: v1.ErrorCodeUserRegistrationNotPaid, + return nil, v1.PluginErrorReply{ + ErrorCode: pi.ErrorCodeUserRegistrationNotPaid, } } } @@ -47,7 +48,7 @@ func (c *Comments) processNew(ctx context.Context, n v1.New, u user.User) (*v1.N } if u.ID.String() != authorID { return nil, v1.UserErrorReply{ - // TODO ErrorCode: v1.ErrorCodeUnauthorized, + ErrorCode: v1.ErrorCodeUnauthorized, ErrorContext: "user is not author or admin", } } @@ -210,7 +211,7 @@ func (c *Comments) processComments(ctx context.Context, cs v1.Comments, u *user. } if !isAllowed { return nil, v1.UserErrorReply{ - // TODO ErrorCode: v1.ErrorCodeUnauthorized, + ErrorCode: v1.ErrorCodeUnauthorized, ErrorContext: "user is not author or admin", } } diff --git a/politeiawww/pi/user.go b/politeiawww/pi/user.go new file mode 100644 index 000000000..8761c39b6 --- /dev/null +++ b/politeiawww/pi/user.go @@ -0,0 +1,36 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package pi + +// Everything defined in this file is a temporary measure until proper user +// plugins have been added to politeiawww, at which point these errors will be +// deprecated. + +const ( + // UserPluginID is a temporary plugin ID for user functionality + // that is specific to pi. + UserPluginID = "politeiawww-piuser" + + // ErrorCodeInvalid is an invalid error code. + ErrorCodeInvalid = 0 + + // ErrorCodeUserRegistrationNotPaid is returned when a user + // attempts to write data to politeia prior to paying their user + // registration fee. + ErrorCodeUserRegistrationNotPaid = 1 + + // ErrorCodeBalanceInsufficient is returned when a user attempts + // to submit a proposal but does not have a proposal credit. + ErrorCodeUserBalanceInsufficient = 2 +) + +var ( + // ErrorCodes contains the human readable error codes. + ErrorCodes = map[int]string{ + ErrorCodeInvalid: "error code invalid", + ErrorCodeUserRegistrationNotPaid: "user registration not paid", + ErrorCodeUserBalanceInsufficient: "user balance insufficient", + } +) diff --git a/politeiawww/records/error.go b/politeiawww/records/error.go index 4a92a37be..9baacfb6c 100644 --- a/politeiawww/records/error.go +++ b/politeiawww/records/error.go @@ -18,57 +18,7 @@ import ( "github.com/decred/politeia/util" ) -func convertPDErrorCode(errCode int) v1.ErrorCodeT { - // Any error statuses that are intentionally omitted means that - // politeiawww should 500. - switch pdv1.ErrorStatusT(errCode) { - case pdv1.ErrorStatusInvalidRequestPayload: - // Intentionally omitted - case pdv1.ErrorStatusInvalidChallenge: - // Intentionally omitted - case pdv1.ErrorStatusInvalidFilename: - return v1.ErrorCodeFileNameInvalid - case pdv1.ErrorStatusInvalidFileDigest: - return v1.ErrorCodeFileDigestInvalid - case pdv1.ErrorStatusInvalidBase64: - return v1.ErrorCodeFilePayloadInvalid - case pdv1.ErrorStatusInvalidMIMEType: - return v1.ErrorCodeFileMIMEInvalid - case pdv1.ErrorStatusUnsupportedMIMEType: - return v1.ErrorCodeFileMIMEInvalid - case pdv1.ErrorStatusInvalidRecordStatusTransition: - return v1.ErrorCodeRecordStatusInvalid - case pdv1.ErrorStatusEmpty: - return v1.ErrorCodeRecordStatusInvalid - case pdv1.ErrorStatusInvalidMDID: - return v1.ErrorCodeMetadataStreamIDInvalid - case pdv1.ErrorStatusDuplicateMDID: - return v1.ErrorCodeMetadataStreamIDInvalid - case pdv1.ErrorStatusDuplicateFilename: - return v1.ErrorCodeFileNameInvalid - case pdv1.ErrorStatusFileNotFound: - // Intentionally omitted - case pdv1.ErrorStatusNoChanges: - return v1.ErrorCodeNoRecordChanges - case pdv1.ErrorStatusRecordFound: - // Intentionally omitted - case pdv1.ErrorStatusInvalidRPCCredentials: - // Intentionally omitted - case pdv1.ErrorStatusRecordNotFound: - return v1.ErrorCodeRecordNotFound - case pdv1.ErrorStatusInvalidToken: - return v1.ErrorCodeRecordTokenInvalid - case pdv1.ErrorStatusRecordLocked: - return v1.ErrorCodeRecordLocked - case pdv1.ErrorStatusInvalidRecordState: - return v1.ErrorCodeRecordStateInvalid - } - return v1.ErrorCodeInvalid -} - func respondWithError(w http.ResponseWriter, r *http.Request, format string, err error) { - log.Tracef("respondWithError: %v %v", format, err) - var ( ue v1.UserErrorReply pe pdclient.Error @@ -160,3 +110,51 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err return } } + +func convertPDErrorCode(errCode int) v1.ErrorCodeT { + // Any error statuses that are intentionally omitted means that + // politeiawww should 500. + switch pdv1.ErrorStatusT(errCode) { + case pdv1.ErrorStatusInvalidRequestPayload: + // Intentionally omitted + case pdv1.ErrorStatusInvalidChallenge: + // Intentionally omitted + case pdv1.ErrorStatusInvalidFilename: + return v1.ErrorCodeFileNameInvalid + case pdv1.ErrorStatusInvalidFileDigest: + return v1.ErrorCodeFileDigestInvalid + case pdv1.ErrorStatusInvalidBase64: + return v1.ErrorCodeFilePayloadInvalid + case pdv1.ErrorStatusInvalidMIMEType: + return v1.ErrorCodeFileMIMEInvalid + case pdv1.ErrorStatusUnsupportedMIMEType: + return v1.ErrorCodeFileMIMEInvalid + case pdv1.ErrorStatusInvalidRecordStatusTransition: + return v1.ErrorCodeRecordStatusInvalid + case pdv1.ErrorStatusEmpty: + return v1.ErrorCodeRecordStatusInvalid + case pdv1.ErrorStatusInvalidMDID: + return v1.ErrorCodeMetadataStreamIDInvalid + case pdv1.ErrorStatusDuplicateMDID: + return v1.ErrorCodeMetadataStreamIDInvalid + case pdv1.ErrorStatusDuplicateFilename: + return v1.ErrorCodeFileNameInvalid + case pdv1.ErrorStatusFileNotFound: + // Intentionally omitted + case pdv1.ErrorStatusNoChanges: + return v1.ErrorCodeNoRecordChanges + case pdv1.ErrorStatusRecordFound: + // Intentionally omitted + case pdv1.ErrorStatusInvalidRPCCredentials: + // Intentionally omitted + case pdv1.ErrorStatusRecordNotFound: + return v1.ErrorCodeRecordNotFound + case pdv1.ErrorStatusInvalidToken: + return v1.ErrorCodeRecordTokenInvalid + case pdv1.ErrorStatusRecordLocked: + return v1.ErrorCodeRecordLocked + case pdv1.ErrorStatusInvalidRecordState: + return v1.ErrorCodeRecordStateInvalid + } + return v1.ErrorCodeInvalid +} diff --git a/politeiawww/records/pi.go b/politeiawww/records/pi.go new file mode 100644 index 000000000..d113b5179 --- /dev/null +++ b/politeiawww/records/pi.go @@ -0,0 +1,96 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package records + +import ( + "fmt" + + v1 "github.com/decred/politeia/politeiawww/api/records/v1" + "github.com/decred/politeia/politeiawww/pi" + "github.com/decred/politeia/politeiawww/user" +) + +// paywallIsEnabled returns whether the user paywall is enabled. +// +// This function is a temporary function that will be removed once user plugins +// have been implemented. +func (r *Records) paywallIsEnabled() bool { + return r.cfg.PaywallAmount != 0 && r.cfg.PaywallXpub != "" +} + +// userHasPaid returns whether the user has paid their user registration fee. +// +// This function is a temporary function that will be removed once user plugins +// have been implemented. +func (r *Records) userHasPaid(u user.User) bool { + if !r.paywallIsEnabled() { + return true + } + return u.NewUserPaywallTx != "" +} + +// userHashProposalCredits returns whether the user has any unspent proposal +// credits. +// +// This function is a temporary function that will be removed once user plugins +// have been implemented. +func userHasProposalCredits(u user.User) bool { + return len(u.UnspentProposalCredits) > 0 +} + +// spendProposalCredit moves a unspent credit to the spent credit list and +// updates the user in the database. +// +// This function is a temporary function that will be removed once user plugins +// have been implemented. +func (r *Records) spendProposalCredit(u user.User, token string) error { + // Skip if the paywall is enabled + if !r.paywallIsEnabled() { + return nil + } + + // Verify there are credits to be spent + if !userHasProposalCredits(u) { + return fmt.Errorf("no proposal credits found") + } + + // Credits are spent FIFO + c := u.UnspentProposalCredits[0] + c.CensorshipToken = token + u.SpentProposalCredits = append(u.SpentProposalCredits, c) + u.UnspentProposalCredits = u.UnspentProposalCredits[1:] + + return r.userdb.UserUpdate(u) +} + +// piHookNewRecordpre executes the new record pre hook for pi. +// +// This function is a temporary function that will be removed once user plugins +// have been implemented. +func (r *Records) piHookNewRecordPre(u user.User) error { + // Verify user has paid registration paywall + if !r.userHasPaid(u) { + return v1.PluginErrorReply{ + PluginID: pi.UserPluginID, + ErrorCode: pi.ErrorCodeUserRegistrationNotPaid, + } + } + + // Verify user has a proposal credit + if !userHasProposalCredits(u) { + return v1.UserErrorReply{ + ErrorCode: pi.ErrorCodeUserBalanceInsufficient, + } + } + return nil +} + +// piHoonNewRecordPost executes the new record post hook for pi. +// +// This function is a temporary function that will be removed once user plugins +// have been implemented. +func (r *Records) piHookNewRecordPost(u user.User, token string) error { + return r.spendProposalCredit(u, token) +} diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index 147b5bc79..8dfced995 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -528,90 +528,6 @@ func (r *Records) processUserRecords(ctx context.Context, ur v1.UserRecords, u * }, nil } -// paywallIsEnabled returns whether the user paywall is enabled. -// -// This function is a temporary function that will be removed once user plugins -// have been implemented. -func (r *Records) paywallIsEnabled() bool { - return r.cfg.PaywallAmount != 0 && r.cfg.PaywallXpub != "" -} - -// userHasPaid returns whether the user has paid their user registration fee. -// -// This function is a temporary function that will be removed once user plugins -// have been implemented. -func (r *Records) userHasPaid(u user.User) bool { - if !r.paywallIsEnabled() { - return true - } - return u.NewUserPaywallTx != "" -} - -// userHashProposalCredits returns whether the user has any unspent proposal -// credits. -// -// This function is a temporary function that will be removed once user plugins -// have been implemented. -func userHasProposalCredits(u user.User) bool { - return len(u.UnspentProposalCredits) > 0 -} - -// spendProposalCredit moves a unspent credit to the spent credit list and -// updates the user in the database. -// -// This function is a temporary function that will be removed once user plugins -// have been implemented. -func (r *Records) spendProposalCredit(u user.User, token string) error { - // Skip if the paywall is enabled - if !r.paywallIsEnabled() { - return nil - } - - // Verify there are credits to be spent - if !userHasProposalCredits(u) { - return fmt.Errorf("no proposal credits found") - } - - // Credits are spent FIFO - c := u.UnspentProposalCredits[0] - c.CensorshipToken = token - u.SpentProposalCredits = append(u.SpentProposalCredits, c) - u.UnspentProposalCredits = u.UnspentProposalCredits[1:] - - return r.userdb.UserUpdate(u) -} - -// piHookNewRecordpre executes the new record pre hook for pi. -// -// This function is a temporary function that will be removed once user plugins -// have been implemented. -func (r *Records) piHookNewRecordPre(u user.User) error { - // Verify user has paid registration paywall - if !r.userHasPaid(u) { - return v1.UserErrorReply{ - // TODO - // ErrorCode: v1.ErrorCodeUserRegistrationNotPaid, - } - } - - // Verify user has a proposal credit - if !userHasProposalCredits(u) { - return v1.UserErrorReply{ - // TODO - // ErrorCode: v1.ErrorCodeUserBalanceInsufficient, - } - } - return nil -} - -// piHoonNewRecordPost executes the new record post hook for pi. -// -// This function is a temporary function that will be removed once user plugins -// have been implemented. -func (r *Records) piHookNewRecordPost(u user.User, token string) error { - return r.spendProposalCredit(u, token) -} - // recordPopulateUserData populates the record with user data that is not // stored in politeiad. func recordPopulateUserData(r *v1.Record, u user.User) { From d00b095579463329c63effacd44cc1fb11ca3088 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 2 Feb 2021 10:32:30 -0600 Subject: [PATCH 265/449] Fix tlogbe locking issue. --- politeiad/backend/tlogbe/plugins/pi/hooks.go | 2 +- politeiad/backend/tlogbe/tlog/anchor.go | 94 +++-- politeiad/backend/tlogbe/tlog/plugin.go | 4 +- politeiad/backend/tlogbe/tlog/tlog.go | 5 +- .../backend/tlogbe/tlog/trillianclient.go | 14 +- politeiad/backend/tlogbe/tlogbe.go | 88 +++-- politeiad/plugins/comments/comments.go | 2 +- politeiad/plugins/dcrdata/dcrdata.go | 2 +- politeiad/plugins/pi/pi.go | 2 +- politeiad/plugins/ticketvote/ticketvote.go | 2 +- politeiad/plugins/user/user.go | 2 +- politeiad/politeiad.go | 3 +- politeiawww/client/records.go | 24 +- politeiawww/cmd/pictl/help.go | 23 +- politeiawww/cmd/pictl/proposaldetails.go | 16 +- politeiawww/cmd/pictl/proposaledit.go | 373 +++++++++++------- politeiawww/cmd/pictl/proposalnew.go | 31 +- politeiawww/cmd/pictl/proposalpolicy.go | 2 +- politeiawww/cmd/pictl/util.go | 119 +++++- politeiawww/cmd/pictl/votepolicy.go | 2 +- politeiawww/politeiawww_test.go | 3 - 21 files changed, 522 insertions(+), 291 deletions(-) diff --git a/politeiad/backend/tlogbe/plugins/pi/hooks.go b/politeiad/backend/tlogbe/plugins/pi/hooks.go index 24ecd7996..4f7f7c85d 100644 --- a/politeiad/backend/tlogbe/plugins/pi/hooks.go +++ b/politeiad/backend/tlogbe/plugins/pi/hooks.go @@ -117,7 +117,7 @@ func (p *piPlugin) proposalFilesVerify(files []backend.File) error { } default: - return fmt.Errorf("invalid mime") + return fmt.Errorf("invalid mime: %v", v.MIME) } } diff --git a/politeiad/backend/tlogbe/tlog/anchor.go b/politeiad/backend/tlogbe/tlog/anchor.go index 76666c1c5..78228acb6 100644 --- a/politeiad/backend/tlogbe/tlog/anchor.go +++ b/politeiad/backend/tlogbe/tlog/anchor.go @@ -53,6 +53,20 @@ type anchor struct { VerifyDigest *dcrtime.VerifyDigest `json:"verifydigest"` } +func (t *Tlog) droppingAnchorGet() bool { + t.Lock() + defer t.Unlock() + + return t.droppingAnchor +} + +func (t *Tlog) droppingAnchorSet(b bool) { + t.Lock() + defer t.Unlock() + + t.droppingAnchor = b +} + var ( errAnchorNotFound = errors.New("anchor not found") ) @@ -229,21 +243,19 @@ func (t *Tlog) anchorSave(a anchor) error { // saved to the key-value store and the record histories of the corresponding // timestamped trees are updated. func (t *Tlog) anchorWait(anchors []anchor, digests []string) { - // Ensure we are not reentrant - t.Lock() - if t.droppingAnchor { + // Verify we are not reentrant + if t.droppingAnchorGet() { log.Errorf("waitForAchor: called reentrantly") return } - t.droppingAnchor = true - t.Unlock() + + // We are now condsidered to be dropping an anchor + t.droppingAnchorSet(true) // Whatever happens in this function we must clear droppingAnchor var exitErr error defer func() { - t.Lock() - t.droppingAnchor = false - t.Unlock() + t.droppingAnchorSet(false) if exitErr != nil { log.Errorf("anchorWait: %v", exitErr) @@ -396,20 +408,27 @@ func (t *Tlog) anchorWait(anchors []anchor, digests []string) { // time of function invocation. A SHA256 digest of the tree's log root at its // current height is timestamped onto the decred blockchain using the dcrtime // service. The anchor data is saved to the key-value store. -func (t *Tlog) anchorTrees() { +func (t *Tlog) anchorTrees() error { log.Debugf("Start %v anchor process", t.id) - var exitErr error // Set on exit if there is an error - defer func() { - if exitErr != nil { - log.Errorf("anchor %v: %v", t.id, exitErr) - } - }() + // Ensure we are not reentrant + if t.droppingAnchorGet() { + // An anchor is not considered dropped until dcrtime returns the + // ChainTimestamp in the VerifyReply. dcrtime does not do this + // until the anchor tx has 6 confirmations, therefor, this code + // path can be hit if 6 blocks are not mined within the period + // specified by the anchor schedule. Though rare, the probability + // of this happening is not zero and should not be considered an + // error. We simply exit and will drop a new anchor at the next + // anchor period. + log.Infof("Attempting to drop an anchor while previous anchor " + + "has not finished dropping; skipping current anchor period") + return nil + } trees, err := t.trillian.treesAll() if err != nil { - exitErr = fmt.Errorf("treesAll: %v", err) - return + return fmt.Errorf("treesAll: %v", err) } // digests contains the SHA256 digests of the LogRootV1.RootHash @@ -437,8 +456,7 @@ func (t *Tlog) anchorTrees() { // leaves. A tree with no leaves does not need to be anchored. leavesAll, err := t.trillian.leavesAll(v.TreeId) if err != nil { - exitErr = fmt.Errorf("leavesAll: %v", err) - return + return fmt.Errorf("leavesAll: %v", err) } if len(leavesAll) == 0 { // Tree does not have any leaves. Nothing to do. @@ -447,16 +465,14 @@ func (t *Tlog) anchorTrees() { case err != nil: // All other errors - exitErr = fmt.Errorf("anchorLatest %v: %v", v.TreeId, err) - return + return fmt.Errorf("anchorLatest %v: %v", v.TreeId, err) default: // Anchor record found. If the anchor height differs from the // current height then the tree needs to be anchored. _, lr, err := t.trillian.signedLogRoot(v) if err != nil { - exitErr = fmt.Errorf("signedLogRoot %v: %v", v.TreeId, err) - return + return fmt.Errorf("signedLogRoot %v: %v", v.TreeId, err) } // Subtract one from the current height to account for the // anchor leaf. @@ -471,8 +487,7 @@ func (t *Tlog) anchorTrees() { // list of anchors. _, lr, err := t.trillian.signedLogRoot(v) if err != nil { - exitErr = fmt.Errorf("signedLogRoot %v: %v", v.TreeId, err) - return + return fmt.Errorf("signedLogRoot %v: %v", v.TreeId, err) } anchors = append(anchors, anchor{ TreeID: v.TreeId, @@ -488,34 +503,15 @@ func (t *Tlog) anchorTrees() { } if len(anchors) == 0 { log.Infof("No %v trees to to anchor", t.id) - return + return nil } - // Ensure we are not reentrant - t.Lock() - if t.droppingAnchor { - // An anchor is not considered dropped until dcrtime returns the - // ChainTimestamp in the VerifyReply. dcrtime does not do this - // until the anchor tx has 6 confirmations, therefor, this code - // path can be hit if 6 blocks are not mined within the period - // specified by the anchor schedule. Though rare, the probability - // of this happening is not zero and should not be considered an - // error. We simply exit and will drop a new anchor at the next - // anchor period. - t.Unlock() - log.Infof("Attempting to drop an anchor while previous anchor " + - "has not finished dropping; skipping current anchor period") - return - } - t.Unlock() - // Submit dcrtime anchor request log.Infof("Anchoring %v %v trees", len(anchors), t.id) tbr, err := t.dcrtime.timestampBatch(anchorID, digests) if err != nil { - exitErr = fmt.Errorf("timestampBatch: %v", err) - return + return fmt.Errorf("timestampBatch: %v", err) } var failed bool for i, v := range tbr.Results { @@ -536,9 +532,11 @@ func (t *Tlog) anchorTrees() { } } if failed { - exitErr = fmt.Errorf("dcrtime failed to timestamp digests") - return + return fmt.Errorf("dcrtime failed to timestamp digests") } + // Launch go routine that polls dcrtime for the anchor tx go t.anchorWait(anchors, digests) + + return nil } diff --git a/politeiad/backend/tlogbe/tlog/plugin.go b/politeiad/backend/tlogbe/tlog/plugin.go index 19ee60f84..d7abf44d2 100644 --- a/politeiad/backend/tlogbe/tlog/plugin.go +++ b/politeiad/backend/tlogbe/tlog/plugin.go @@ -122,7 +122,7 @@ func (t *Tlog) PluginSetup(pluginID string) error { } func (t *Tlog) PluginHookPre(treeID int64, token []byte, h plugins.HookT, payload string) error { - log.Tracef("%v PluginHookPre: %v %x %v", t.id, plugins.Hooks[h]) + log.Tracef("%v PluginHookPre: %v %v", t.id, treeID, plugins.Hooks[h]) // Pass hook event and payload to each plugin for _, v := range t.pluginIDs() { @@ -141,7 +141,7 @@ func (t *Tlog) PluginHookPre(treeID int64, token []byte, h plugins.HookT, payloa } func (t *Tlog) PluginHookPost(treeID int64, token []byte, h plugins.HookT, payload string) { - log.Tracef("%v PluginHookPost: %v %x %v", t.id, plugins.Hooks[h]) + log.Tracef("%v PluginHookPost: %v %v", t.id, treeID, plugins.Hooks[h]) // Pass hook event and payload to each plugin for _, v := range t.pluginIDs() { diff --git a/politeiad/backend/tlogbe/tlog/tlog.go b/politeiad/backend/tlogbe/tlog/tlog.go index a58a66ef1..c01f80349 100644 --- a/politeiad/backend/tlogbe/tlog/tlog.go +++ b/politeiad/backend/tlogbe/tlog/tlog.go @@ -1534,7 +1534,10 @@ func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trilli // Launch cron log.Infof("Launch %v cron anchor job", id) err = t.cron.AddFunc(anchorSchedule, func() { - t.anchorTrees() + err := t.anchorTrees() + if err != nil { + log.Errorf("%v anchorTrees: %v", id, err) + } }) if err != nil { return nil, err diff --git a/politeiad/backend/tlogbe/tlog/trillianclient.go b/politeiad/backend/tlogbe/tlog/trillianclient.go index aacd17985..599cc0780 100644 --- a/politeiad/backend/tlogbe/tlog/trillianclient.go +++ b/politeiad/backend/tlogbe/tlog/trillianclient.go @@ -571,7 +571,7 @@ var ( // testTClient implements the trillianClient interface and is used for testing. type testTClient struct { - sync.RWMutex + sync.Mutex trees map[int64]*trillian.Tree // [treeID]Tree leaves map[int64][]*trillian.LogLeaf // [treeID][]LogLeaf @@ -630,8 +630,8 @@ func (t *testTClient) treeFreeze(treeID int64) (*trillian.Tree, error) { // // This function satisfies the trillianClient interface. func (t *testTClient) tree(treeID int64) (*trillian.Tree, error) { - t.RLock() - defer t.RUnlock() + t.Lock() + defer t.Unlock() if tree, ok := t.trees[treeID]; ok { return tree, nil @@ -645,8 +645,8 @@ func (t *testTClient) tree(treeID int64) (*trillian.Tree, error) { // // This function satisfies the trillianClient interface. func (t *testTClient) treesAll() ([]*trillian.Tree, error) { - t.RLock() - defer t.RUnlock() + t.Lock() + defer t.Unlock() trees := make([]*trillian.Tree, len(t.trees)) for _, t := range t.trees { @@ -697,8 +697,8 @@ func (t *testTClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([] // // This function satisfies the trillianClient interface. func (t *testTClient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { - t.RLock() - defer t.RUnlock() + t.Lock() + defer t.Unlock() // Check if treeID entry exists if _, ok := t.leaves[treeID]; !ok { diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 8efca3d72..170805861 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -75,6 +75,11 @@ type tlogBackend struct { // status change from newest to oldest. This cache is built on // startup. inv inventory + + // recordMtxs allows a the backend to hold a lock on a record so + // that it can perform multiple read/write operations in a + // concurrency safe manner. These mutexes are lazy loaded. + recordMtxs map[string]*sync.Mutex } func tokenFromTreeID(treeID int64) []byte { @@ -96,6 +101,22 @@ func treeIDFromToken(token []byte) int64 { return int64(binary.LittleEndian.Uint64(token)) } +// recordMutex returns the mutex for a record. +func (t *tlogBackend) recordMutex(token []byte) *sync.Mutex { + t.Lock() + defer t.Unlock() + + ts := hex.EncodeToString(token) + m, ok := t.recordMtxs[ts] + if !ok { + // recordMtxs is lazy loaded + m = &sync.Mutex{} + t.recordMtxs[ts] = m + } + + return m +} + // fullLengthToken returns the full length token given the token prefix. // // This function must be called WITHOUT the lock held. @@ -619,13 +640,14 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ return nil, backend.ErrRecordNotFound } - // Apply the record changes and save the new version. The lock - // needs to be held for the remainder of the function. - t.unvetted.Lock() - defer t.unvetted.Unlock() - if t.shutdown { + // Apply the record changes and save the new version. The record + // lock needs to be held for the remainder of the function. + if t.isShutdown() { return nil, backend.ErrShutdown } + m := t.recordMutex(token) + m.Lock() + defer m.Unlock() // Get existing record treeID := treeIDFromToken(token) @@ -722,13 +744,14 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b return nil, backend.ErrRecordNotFound } - // Apply the record changes and save the new version. The lock - // needs to be held for the remainder of the function. - t.vetted.Lock() - defer t.vetted.Unlock() - if t.shutdown { + // Apply the record changes and save the new version. The record + // lock needs to be held for the remainder of the function. + if t.isShutdown() { return nil, backend.ErrShutdown } + m := t.recordMutex(token) + m.Lock() + defer m.Unlock() // Get existing record r, err := t.vetted.RecordLatest(treeID) @@ -827,12 +850,13 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite } // Pull the existing record and apply the metadata updates. The - // unvetted lock must be held for the remainder of this function. - t.unvetted.Lock() - defer t.unvetted.Unlock() - if t.shutdown { + // record lock must be held for the remainder of this function. + if t.isShutdown() { return backend.ErrShutdown } + m := t.recordMutex(token) + m.Lock() + defer m.Unlock() // Get existing record treeID := treeIDFromToken(token) @@ -919,12 +943,13 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ } // Pull the existing record and apply the metadata updates. The - // vetted lock must be held for the remainder of this function. - t.vetted.Lock() - defer t.vetted.Unlock() - if t.shutdown { + // record lock must be held for the remainder of this function. + if t.isShutdown() { return backend.ErrShutdown } + m := t.recordMutex(token) + m.Lock() + defer m.Unlock() // Get existing record r, err := t.vetted.RecordLatest(treeID) @@ -997,7 +1022,7 @@ func (t *tlogBackend) VettedExists(token []byte) bool { return ok } -// This function must be called WITH the unvetted lock held. +// This function must be called WITH the record lock held. func (t *tlogBackend) unvettedPublish(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { // Create a vetted tree vettedTreeID, err := t.vetted.TreeNew() @@ -1029,7 +1054,7 @@ func (t *tlogBackend) unvettedPublish(token []byte, rm backend.RecordMetadata, m return nil } -// This function must be called WITH the unvetted lock held. +// This function must be called WITH the record lock held. func (t *tlogBackend) unvettedCensor(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { // Freeze the tree treeID := treeIDFromToken(token) @@ -1068,13 +1093,14 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, return nil, backend.ErrRecordNotFound } - // The existing record must be pulled and updated. The unvetted + // The existing record must be pulled and updated. The record // lock must be held for the rest of this function. - t.unvetted.Lock() - defer t.unvetted.Unlock() - if t.shutdown { + if t.isShutdown() { return nil, backend.ErrShutdown } + m := t.recordMutex(token) + m.Lock() + defer m.Unlock() // Get existing record treeID := treeIDFromToken(token) @@ -1161,7 +1187,7 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, return t.GetUnvetted(token, "") } -// This function must be called WITH the vetted lock held. +// This function must be called WITH the record lock held. func (t *tlogBackend) vettedCensor(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { // Freeze the tree treeID, ok := t.vettedTreeID(token) @@ -1186,7 +1212,7 @@ func (t *tlogBackend) vettedCensor(token []byte, rm backend.RecordMetadata, meta return nil } -// This function must be called WITH the vetted lock held. +// This function must be called WITH the record lock held. func (t *tlogBackend) vettedArchive(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { // Freeze the tree. Nothing else needs to be done for an archived // record. @@ -1223,13 +1249,14 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md return nil, backend.ErrRecordNotFound } - // The existing record must be pulled and updated. The vetted lock + // The existing record must be pulled and updated. The record lock // must be held for the rest of this function. - t.vetted.Lock() - defer t.vetted.Unlock() - if t.shutdown { + if t.isShutdown() { return nil, backend.ErrShutdown } + m := t.recordMutex(token) + m.Lock() + defer m.Unlock() // Get existing record r, err := t.vetted.RecordLatest(treeID) @@ -1798,6 +1825,7 @@ func New(anp *chaincfg.Params, homeDir, dataDir, unvettedTrillianHost, unvettedT unvetted: make(map[backend.MDStatusT][]string), vetted: make(map[backend.MDStatusT][]string), }, + recordMtxs: make(map[string]*sync.Mutex), } err = t.setup() diff --git a/politeiad/plugins/comments/comments.go b/politeiad/plugins/comments/comments.go index 7ed7b2b5d..4d3f2d4d3 100644 --- a/politeiad/plugins/comments/comments.go +++ b/politeiad/plugins/comments/comments.go @@ -8,7 +8,7 @@ package comments const ( // PluginID is the comments plugin ID. - PluginID = "politeiad-comments" + PluginID = "comments" // Plugin commands CmdNew = "new" // Create a new comment diff --git a/politeiad/plugins/dcrdata/dcrdata.go b/politeiad/plugins/dcrdata/dcrdata.go index b3252c948..1a9235625 100644 --- a/politeiad/plugins/dcrdata/dcrdata.go +++ b/politeiad/plugins/dcrdata/dcrdata.go @@ -9,7 +9,7 @@ package dcrdata type StatusT int const ( - PluginID = "politeiad-dcrdata" + PluginID = "dcrdata" // Plugin commands CmdBestBlock = "bestblock" // Get best block diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index cb7f62acb..fc5609c64 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -8,7 +8,7 @@ package pi const ( // PluginID is the pi plugin ID. - PluginID = "politeiad-pi" + PluginID = "pi" // Plugin commands CmdVoteInv = "voteinv" diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index 0b9786c50..ba56a4ba4 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -15,7 +15,7 @@ package ticketvote const ( // PluginID is the ticketvote plugin ID. - PluginID = "politeiad-ticketvote" + PluginID = "ticketvote" // Plugin commands CmdAuthorize = "authorize" // Authorize a vote diff --git a/politeiad/plugins/user/user.go b/politeiad/plugins/user/user.go index 8e7d4c40d..f0edeb1c1 100644 --- a/politeiad/plugins/user/user.go +++ b/politeiad/plugins/user/user.go @@ -7,7 +7,7 @@ package user const ( - PluginID = "politeiad-user" + PluginID = "user" // Plugin commands CmdAuthor = "author" // Get record author diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 02b02d978..cd0ea519d 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -1535,7 +1535,6 @@ func _main() error { p.addRoute(http.MethodPost, v1.UpdateUnvettedMetadataRoute, p.updateUnvettedMetadata, permissionAuth) - // TODO document plugins and plugin settings in README // Setup plugins if len(cfg.Plugins) > 0 { // Set plugin routes. Requires auth. @@ -1619,7 +1618,7 @@ func _main() error { case cmsplugin.ID: // TODO plugin setup for cms default: - return fmt.Errorf("unknown plugin '%v'", v) + return fmt.Errorf("unknown plugin provided by config '%v'", v) } // Register plugin diff --git a/politeiawww/client/records.go b/politeiawww/client/records.go index 4fb72c6b9..17c3aaded 100644 --- a/politeiawww/client/records.go +++ b/politeiawww/client/records.go @@ -36,8 +36,28 @@ func (c *Client) RecordNew(n rcv1.New) (*rcv1.NewReply, error) { return &nr, nil } +// RecordEdit sends a records v1 Edit request to politeiawww. +func (c *Client) RecordEdit(e rcv1.Edit) (*rcv1.EditReply, error) { + route := rcv1.APIRoute + rcv1.RouteEdit + resBody, err := c.makeReq(http.MethodPost, route, e) + if err != nil { + return nil, err + } + + var er rcv1.EditReply + err = json.Unmarshal(resBody, &er) + if err != nil { + return nil, err + } + if c.verbose { + fmt.Printf("%v\n", formatJSON(er)) + } + + return &er, nil +} + // RecordDetails sends a records v1 Details request to politeiawww. -func (c *Client) RecordDetails(d rcv1.Details) (*rcv1.DetailsReply, error) { +func (c *Client) RecordDetails(d rcv1.Details) (*rcv1.Record, error) { route := rcv1.APIRoute + rcv1.RouteDetails resBody, err := c.makeReq(http.MethodPost, route, d) if err != nil { @@ -53,7 +73,7 @@ func (c *Client) RecordDetails(d rcv1.Details) (*rcv1.DetailsReply, error) { fmt.Printf("%v\n", formatJSON(dr)) } - return &dr, nil + return &dr.Record, nil } // digestsVerify verifies that all file digests match the calculated SHA256 diff --git a/politeiawww/cmd/pictl/help.go b/politeiawww/cmd/pictl/help.go index a2fa62f16..f1fa9d0d9 100644 --- a/politeiawww/cmd/pictl/help.go +++ b/politeiawww/cmd/pictl/help.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Decred developers +// Copyright (c) 2017-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -13,20 +13,15 @@ import ( // helpCmd prints a detailed help message for the specified command. type helpCmd struct { Args struct { - Topic string `positional-arg-name:"topic"` // Topic to print help message for - } `positional-args:"true"` + Command string `positional-arg-name:"command"` + } `positional-args:"true" required:"true"` } // Execute executes the helpCmd command. // // This function satisfies the go-flags Commander interface. func (cmd *helpCmd) Execute(args []string) error { - if cmd.Args.Topic == "" { - return fmt.Errorf("Specify a command to print a detailed help " + - "message for. Example: piwww help login") - } - - switch cmd.Args.Topic { + switch cmd.Args.Command { // Server commands case "version": fmt.Printf("%s\n", shared.VersionHelpMsg) @@ -39,7 +34,9 @@ func (cmd *helpCmd) Execute(args []string) error { case "me": fmt.Printf("%s\n", shared.MeHelpMsg) - // Proposal commands + // Proposal commands + case "proposalpolicy": + fmt.Printf("%s\n", proposalPolicyHelpMsg) case "proposalnew": fmt.Printf("%s\n", proposalNewHelpMsg) case "proposaledit": @@ -64,6 +61,8 @@ func (cmd *helpCmd) Execute(args []string) error { fmt.Printf("%s\n", commentVotesHelpMsg) // Vote commands + case "votepolicy": + fmt.Printf("%s\n", votePolicyHelpMsg) case "voteauthorize": fmt.Printf("%s\n", voteAuthorizeHelpMsg) case "votestart": @@ -126,8 +125,8 @@ func (cmd *helpCmd) Execute(args []string) error { fmt.Printf("%s\n", sendFaucetTxHelpMsg) default: - fmt.Printf("invalid command: use 'piwww -h' " + - "to view a list of valid commands\n") + fmt.Printf("invalid command: use the -h,--help flag to view the " + + "full list of valid commands\n") } return nil diff --git a/politeiawww/cmd/pictl/proposaldetails.go b/politeiawww/cmd/pictl/proposaldetails.go index f51c809fc..9472fe766 100644 --- a/politeiawww/cmd/pictl/proposaldetails.go +++ b/politeiawww/cmd/pictl/proposaldetails.go @@ -7,7 +7,6 @@ package main import ( rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" pclient "github.com/decred/politeia/politeiawww/client" - "github.com/decred/politeia/politeiawww/cmd/shared" ) // proposalDetails retrieves a full proposal record. @@ -49,24 +48,19 @@ func (c *proposalDetailsCmd) Execute(args []string) error { state = rcv1.RecordStateVetted } - // Setup request + // Get proposal details d := rcv1.Details{ State: state, Token: c.Args.Token, Version: c.Args.Version, } - - // Send request. The request and response details are printed to - // the console based on the logging flags that were used. - err = shared.PrintJSON(d) + r, err := pc.RecordDetails(d) if err != nil { return err } - dr, err := pc.RecordDetails(d) - if err != nil { - return err - } - err = shared.PrintJSON(dr) + + // Print proposal to stdout + err = printProposal(*r) if err != nil { return err } diff --git a/politeiawww/cmd/pictl/proposaledit.go b/politeiawww/cmd/pictl/proposaledit.go index dc77ec3b7..872d063f5 100644 --- a/politeiawww/cmd/pictl/proposaledit.go +++ b/politeiawww/cmd/pictl/proposaledit.go @@ -4,6 +4,24 @@ package main +import ( + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "io/ioutil" + "path/filepath" + "time" + + "github.com/decred/politeia/politeiad/api/v1/mime" + piplugin "github.com/decred/politeia/politeiad/plugins/pi" + piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + pclient "github.com/decred/politeia/politeiawww/client" + "github.com/decred/politeia/politeiawww/cmd/shared" + "github.com/decred/politeia/util" +) + // proposalEditCmd edits an existing proposal. type proposalEditCmd struct { Args struct { @@ -12,166 +30,256 @@ type proposalEditCmd struct { Attachments []string `positional-arg-name:"attachmets"` } `positional-args:"true" optional:"true"` - // Unvetted is used to indicate that the state of the requested - // proposal is unvetted. If this flag is not used it will be - // assumed that a vetted proposal is being requested. + // Unvetted is used to indicate the state of the proposal is + // unvetted. If this flag is not used it will be assumed that + // the proposal is vetted. Unvetted bool `long:"unvetted" optional:"true"` - // Random generates random proposal data. An IndexFile and - // Attachments are not required when using this flag. - Random bool `long:"random" optional:"true"` + // UseMD is a flag that is intended to make editing proposal + // metadata easier by using exisiting proposal metadata values + // instead of having to pass in specific values. + UseMD bool `long:"usemd" optional:"true"` - // The following flags can be used to specify user defined proposal - // metadata values. + // Metadata fields that can be set by the user Name string `long:"name" optional:"true"` LinkTo string `long:"linkto" optional:"true"` LinkBy int64 `long:"linkby" optional:"true"` - // RFP is a flag that is intended to make editing an RFP easier + // Random generates random proposal data. An IndexFile and + // Attachments are not required when using this flag. + Random bool `long:"random" optional:"true"` + + // RFP is a flag that is intended to make submitting an RFP easier // by calculating and inserting a linkby timestamp automatically - // instead of having to pass in a specific timestamp using the - // --linkby flag. + // instead of having to pass in a timestamp using the --linkby + // flag. RFP bool `long:"rfp" optional:"true"` - - // UseMD is a flag that is intended to make editing proposal - // metadata easier by using exisiting proposal metadata values - // instead of having to pass in specific values. - UseMD bool `long:"usemd" optional:"true"` } // Execute executes the proposalEditCmd command. // // This function satisfies the go-flags Commander interface. func (c *proposalEditCmd) Execute(args []string) error { - /* - // Unpack args - token := c.Args.Token - indexFile := c.Args.IndexFile - attachments := c.Args.Attachments - - // Verify args - switch { - case !c.Random && indexFile == "": - return fmt.Errorf("index file not found; you must either " + - "provide an index.md file or use --random") - - case c.Random && indexFile != "": - return fmt.Errorf("you cannot provide file arguments and use " + - "the --random flag at the same time") - - case c.RFP && c.LinkBy != 0: - return fmt.Errorf("you cannot use both the --rfp and --linkby " + - "flags at the same time") - } + // Unpack args + token := c.Args.Token + indexFile := c.Args.IndexFile + attachments := c.Args.Attachments - // Check for user identity. A user identity is required to sign - // the proposal files. - if cfg.Identity == nil { - return shared.ErrUserIdentityNotFound - } + // Verify args + switch { + case !c.Random && indexFile == "": + return fmt.Errorf("index file not found; you must either " + + "provide an index.md file or use --random") + + case c.Random && indexFile != "": + return fmt.Errorf("you cannot provide file arguments and use " + + "the --random flag at the same time") + + case c.RFP && c.LinkBy != 0: + return fmt.Errorf("you cannot use both the --rfp and --linkby " + + "flags at the same time") + } + + // Check for user identity. A user identity is required to sign + // the proposal files. + if cfg.Identity == nil { + return shared.ErrUserIdentityNotFound + } + + // Setup client + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Cookies: cfg.Cookies, + HeaderCSRF: cfg.CSRF, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, + } + pc, err := pclient.New(cfg.Host, opts) + if err != nil { + return err + } + + // Get the pi policy. It contains the proposal requirements. + pr, err := pc.PiPolicy() + if err != nil { + return err + } - // Setup client - pc, err := pclient.New(cfg.Host, cfg.HTTPSCert, cfg.Cookies, cfg.CSRF) + // Setup state + var state string + switch { + case c.Unvetted: + state = rcv1.RecordStateUnvetted + default: + state = rcv1.RecordStateVetted + } + + // Setup index file + files := make([]rcv1.File, 0, 16) + if c.Random { + // Generate random text for the index file + f, err := indexFileRandom(1024) if err != nil { return err } + files = append(files, *f) + } else { + // Read index file from disk + fp := util.CleanAndExpandPath(indexFile) + var err error + payload, err := ioutil.ReadFile(fp) + if err != nil { + return fmt.Errorf("ReadFile %v: %v", fp, err) + } + files = append(files, rcv1.File{ + Name: piplugin.FileNameIndexFile, + MIME: mime.DetectMimeType(payload), + Digest: hex.EncodeToString(util.Digest(payload)), + Payload: base64.StdEncoding.EncodeToString(payload), + }) + } - // Get the pi policy. It contains the proposal requirements. - pr, err := pc.PiPolicy() + // Setup attachment files + for _, fn := range attachments { + fp := util.CleanAndExpandPath(fn) + payload, err := ioutil.ReadFile(fp) if err != nil { - return err + return fmt.Errorf("ReadFile %v: %v", fp, err) } - // Setup state - var state string - switch { - case c.Unvetted: - state = rcv1.RecordStateUnvetted - default: - state = rcv1.RecordStateVetted + files = append(files, rcv1.File{ + Name: filepath.Base(fn), + MIME: mime.DetectMimeType(payload), + Digest: hex.EncodeToString(util.Digest(payload)), + Payload: base64.StdEncoding.EncodeToString(payload), + }) + } + + // Get current proposal if we are using the existing metadata + var curr *rcv1.Record + if c.UseMD { + d := rcv1.Details{ + State: state, + Token: token, + } + curr, err = pc.RecordDetails(d) + if err != nil { + return err } + } - // Setup index file - var ( - file *rcv1.File - files = make([]rcv1.File, 0, 16) - ) - if c.Random { - // Generate random text for the index file - file, err = createMDFile() - if err != nil { - return err - } - } else { - // Read index file from disk - fp := util.CleanAndExpandPath(indexFile) - var err error - payload, err := ioutil.ReadFile(fp) - if err != nil { - return fmt.Errorf("ReadFile %v: %v", fp, err) - } - file = &rcv1.File{ - Name: piplugin.FileNameIndexFile, - MIME: mime.DetectMimeType(payload), - Digest: hex.EncodeToString(util.Digest(payload)), - Payload: base64.StdEncoding.EncodeToString(payload), - } + // Setup proposal metadata + switch { + case c.UseMD: + // Use the existing proposal name + pm, err := proposalMetadataDecode(curr.Files) + if err != nil { + return err } - files = append(files, *file) - - // Setup attachment files - for _, fn := range attachments { - fp := util.CleanAndExpandPath(fn) - payload, err := ioutil.ReadFile(fp) - if err != nil { - return fmt.Errorf("ReadFile %v: %v", fp, err) - } - - files = append(files, rcv1.File{ - Name: filepath.Base(fn), - MIME: mime.DetectMimeType(payload), - Digest: hex.EncodeToString(util.Digest(payload)), - Payload: base64.StdEncoding.EncodeToString(payload), - }) + c.Name = pm.Name + case c.Random && c.Name == "": + // Create a random proposal name + r, err := util.Random(int(pr.NameLengthMin)) + if err != nil { + return err } + c.Name = hex.EncodeToString(r) + } + pm := piv1.ProposalMetadata{ + Name: c.Name, + } + pmb, err := json.Marshal(pm) + if err != nil { + return err + } + files = append(files, rcv1.File{ + Name: piv1.FileNameProposalMetadata, + MIME: mime.DetectMimeType(pmb), + Digest: hex.EncodeToString(util.Digest(pmb)), + Payload: base64.StdEncoding.EncodeToString(pmb), + }) - // Setup proposal metadata - switch { - case c.UseMD: - // Use the prexisting proposal name - - case c.Random: - // Create a random proposal name - r, err := util.Random(int(pr.NameLengthMin)) - if err != nil { - return err - } - c.Name = hex.EncodeToString(r) + // Setup vote metadata + if c.UseMD { + vm, err := voteMetadataDecode(curr.Files) + if err != nil { + return err } - pm := piv1.ProposalMetadata{ - Name: c.Name, + c.LinkBy = vm.LinkBy + c.LinkTo = vm.LinkTo + } + if c.RFP { + // Set linkby to a month from now + c.LinkBy = time.Now().Add(time.Hour * 24 * 30).Unix() + } + if c.LinkBy != 0 || c.LinkTo != "" { + vm := piv1.VoteMetadata{ + LinkTo: c.LinkTo, + LinkBy: c.LinkBy, } - pmb, err := json.Marshal(pm) + vmb, err := json.Marshal(vm) if err != nil { return err } files = append(files, rcv1.File{ - Name: piv1.FileNameProposalMetadata, - MIME: mime.DetectMimeType(pmb), - Digest: hex.EncodeToString(util.Digest(pmb)), - Payload: base64.StdEncoding.EncodeToString(pmb), + Name: piv1.FileNameVoteMetadata, + MIME: mime.DetectMimeType(vmb), + Digest: hex.EncodeToString(util.Digest(vmb)), + Payload: base64.StdEncoding.EncodeToString(vmb), }) - */ + } + + // Print proposal to stdout + printf("Proposal editted\n") + err = printProposalFiles(files) + if err != nil { + return err + } + + // Setup request + sig, err := signedMerkleRoot(files, cfg.Identity) + if err != nil { + return err + } + e := rcv1.Edit{ + State: state, + Token: token, + Files: files, + PublicKey: cfg.Identity.Public.String(), + Signature: sig, + } + + // Send request + nr, err := pc.RecordEdit(e) + if err != nil { + return err + } + + // Verify record + vr, err := client.Version() + if err != nil { + return err + } + err = pclient.RecordVerify(nr.Record, vr.PubKey) + if err != nil { + return fmt.Errorf("unable to verify record: %v", err) + } return nil } -// proposalEditHelpMsg is the output of the help command. +// proposalEditHelpMsg is the printed to stdout by the help command. const proposalEditHelpMsg = `editproposal [flags] "token" "indexfile" "attachments" -Edit a proposal. This command assumes the proposal is a vetted record. If -the proposal is unvetted, the --unvetted flag must be used. Requires admin -priviledges. +Edit a proposal. This command assumes the proposal is a vetted record. If the +proposal is unvetted, the --unvetted flag must be used. + +A proposal can be submitted as an RFP (Request for Proposals) by using either +the --rfp flag or by manually specifying a link by deadline using the --linkby +flag. Only one of these flags can be used at a time. + +A proposal can be submitted as an RFP submission by using the --linkto flag +to link to and an existing RFP proposal. Arguments: 1. token (string, required) Proposal censorship token @@ -179,18 +287,17 @@ Arguments: 3. attachments (string, optional) Attachment files Flags: - --unvetted (bool, optional) Comment on unvetted record. - --random (bool, optional) Generate a random proposal name & files to - submit. If this flag is used then the markdown - file argument is no longer required and any - provided files will be ignored. - --usemd (bool, optional) Use the existing proposal metadata. - --name (string, optional) The name of the proposal. - --linkto (string, optional) Censorship token of an existing public proposal - to link to. - --linkby (int64, optional) UNIX timestamp of RFP deadline. - --rfp (bool, optional) Make the proposal an RFP by inserting a LinkBy - timestamp into the proposal metadata. The LinkBy - timestamp is set to be one month from the - current time. This is intended to be used in - place of --linkby.` + --unvetted (bool, optional) Edit an unvetted record. + --usemd (bool, optional) Use the existing proposal metadata. + --name (string, optional) Name of the proposal. + --linkto (string, optional) Token of an existing public proposal to link to. + --linkby (int64, optional) UNIX timestamp of the RFP deadline. Setting this + field will make the proposal an RFP with a + submission deadline specified by the linkby. + --random (bool, optional) Generate a random proposal. If this flag is used + then the markdownfile argument is no longer + required and any provided files will be ignored. + --rfp (bool, optional) Make the proposal an RFP by setting the linkby + to one month from the current time. This is + intended to be used in place of --linkby. +` diff --git a/politeiawww/cmd/pictl/proposalnew.go b/politeiawww/cmd/pictl/proposalnew.go index 0cefc6a36..87cd0d2b5 100644 --- a/politeiawww/cmd/pictl/proposalnew.go +++ b/politeiawww/cmd/pictl/proposalnew.go @@ -94,16 +94,14 @@ func (c *proposalNewCmd) Execute(args []string) error { } // Setup index file - var ( - file *rcv1.File - files = make([]rcv1.File, 0, 16) - ) + files := make([]rcv1.File, 0, 16) if c.Random { // Generate random text for the index file - file, err = indexFileRandom() + f, err := indexFileRandom(1024) if err != nil { return err } + files = append(files, *f) } else { // Read index file from disk fp := util.CleanAndExpandPath(indexFile) @@ -112,14 +110,13 @@ func (c *proposalNewCmd) Execute(args []string) error { if err != nil { return fmt.Errorf("ReadFile %v: %v", fp, err) } - file = &rcv1.File{ + files = append(files, rcv1.File{ Name: piplugin.FileNameIndexFile, MIME: mime.DetectMimeType(payload), Digest: hex.EncodeToString(util.Digest(payload)), Payload: base64.StdEncoding.EncodeToString(payload), - } + }) } - files = append(files, *file) // Setup attachment files for _, fn := range attachments { @@ -143,7 +140,7 @@ func (c *proposalNewCmd) Execute(args []string) error { if err != nil { return err } - c.Name = fmt.Sprintf("Name %x", r) + c.Name = fmt.Sprintf("A Proposal Name %x", r) } pm := piv1.ProposalMetadata{ Name: c.Name, @@ -181,6 +178,13 @@ func (c *proposalNewCmd) Execute(args []string) error { }) } + // Print proposal to stdout + printf("Proposal submitted\n") + err = printProposalFiles(files) + if err != nil { + return err + } + // Setup request sig, err := signedMerkleRoot(files, cfg.Identity) if err != nil { @@ -208,16 +212,13 @@ func (c *proposalNewCmd) Execute(args []string) error { return fmt.Errorf("unable to verify record: %v", err) } - // Print details to stdout - printf("Proposal '%v' submitted\n", pm.Name) - for _, v := range nr.Record.Files { - printf(" %-22v %v\n", v.Name, v.MIME) - } + // Print token to stdout + printf("Token: %v\n", nr.Record.CensorshipRecord.Token) return nil } -// proposalNewHelpMsg is the output of the help command. +// proposalNewHelpMsg is the printed to stdout by the help command. const proposalNewHelpMsg = `proposalnew [flags] "indexfile" "attachments" Submit a new proposal to Politeia. diff --git a/politeiawww/cmd/pictl/proposalpolicy.go b/politeiawww/cmd/pictl/proposalpolicy.go index 476a37c03..1b60783d5 100644 --- a/politeiawww/cmd/pictl/proposalpolicy.go +++ b/politeiawww/cmd/pictl/proposalpolicy.go @@ -38,7 +38,7 @@ func (cmd *proposalPolicyCmd) Execute(args []string) error { return nil } -// proposalPolicyHelpMsg is the command help message. +// proposalEditHelpMsg is the printed to stdout by the help command. const proposalPolicyHelpMsg = `proposalpolicy Fetch the pi API policy.` diff --git a/politeiawww/cmd/pictl/util.go b/politeiawww/cmd/pictl/util.go index 8a0feeb93..f58332797 100644 --- a/politeiawww/cmd/pictl/util.go +++ b/politeiawww/cmd/pictl/util.go @@ -5,15 +5,16 @@ package main import ( - "bytes" "encoding/base64" "encoding/hex" "encoding/json" "fmt" + "math/rand" + "strings" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/api/v1/mime" - piplugin "github.com/decred/politeia/politeiad/plugins/pi" + piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" "github.com/decred/politeia/util" ) @@ -47,6 +48,63 @@ func formatJSON(v interface{}) string { return string(b) } +// byteCountSI converts the provided bytes to a string representation of the +// closest SI unit (kB, MB, GB, etc). +func byteCountSI(b int64) string { + const unit = 1000 + if b < unit { + return fmt.Sprintf("%d B", b) + } + div, exp := int64(unit), 0 + for n := b / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %cB", + float64(b)/float64(div), "kMGTPE"[exp]) +} + +func printProposalFiles(files []rcv1.File) error { + for _, v := range files { + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return err + } + s := byteCountSI(int64(len(b))) + printf(" %-22v %-26v %v\n", v.Name, v.MIME, s) + } + return nil +} + +func printProposal(r rcv1.Record) error { + printf("Token: %v\n", r.CensorshipRecord.Token) + return printProposalFiles(r.Files) +} + +// indexFileRandom returns a proposal index file filled with random data. +func indexFileRandom(sizeInBytes int) (*rcv1.File, error) { + // Create lines of text that are 80 characters long + charSet := "abcdefghijklmnopqrstuvwxyz" + var b strings.Builder + for i := 0; i < sizeInBytes; i++ { + if i%80 == 0 && i != 0 { + b.WriteString("\n") + continue + } + r := rand.Intn(len(charSet)) + char := charSet[r] + b.WriteString(string(char)) + } + payload := []byte(b.String()) + + return &rcv1.File{ + Name: piv1.FileNameIndexFile, + MIME: mime.DetectMimeType(payload), + Digest: hex.EncodeToString(util.Digest(payload)), + Payload: base64.StdEncoding.EncodeToString(payload), + }, nil +} + // signedMerkleRoot returns the signed merkle root of the provided files. The // signature is created using the provided identity. func signedMerkleRoot(files []rcv1.File, fid *identity.FullIdentity) (string, error) { @@ -65,22 +123,49 @@ func signedMerkleRoot(files []rcv1.File, fid *identity.FullIdentity) (string, er return hex.EncodeToString(sig[:]), nil } -// indexFileRandom returns a proposal index file filled with random data. -// TODO fill to max size -func indexFileRandom() (*rcv1.File, error) { - var b bytes.Buffer - for i := 0; i < 10; i++ { - r, err := util.Random(32) - if err != nil { - return nil, err +// proposalMetadataDecode decodes and returns the ProposalMetadata from the +// provided record files. An error is returned if a ProposalMetadata is not +// found. +func proposalMetadataDecode(files []rcv1.File) (*piv1.ProposalMetadata, error) { + var propMD *piv1.ProposalMetadata + for _, v := range files { + if v.Name == piv1.FileNameProposalMetadata { + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return nil, err + } + var m piv1.ProposalMetadata + err = json.Unmarshal(b, &m) + if err != nil { + return nil, err + } + propMD = &m + break } - b.WriteString(base64.StdEncoding.EncodeToString(r) + "\n") } + if propMD == nil { + return nil, fmt.Errorf("proposal metadata not found") + } + return propMD, nil +} - return &rcv1.File{ - Name: piplugin.FileNameIndexFile, - MIME: mime.DetectMimeType(b.Bytes()), - Digest: hex.EncodeToString(util.Digest(b.Bytes())), - Payload: base64.StdEncoding.EncodeToString(b.Bytes()), - }, nil +// voteMetadataDecode decodes and returns the VoteMetadata from the provided +// backend files. If a VoteMetadata is not found, an empty one will be +// returned. +func voteMetadataDecode(files []rcv1.File) (*piv1.VoteMetadata, error) { + var vm piv1.VoteMetadata + for _, v := range files { + if v.Name == piv1.FileNameVoteMetadata { + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return nil, err + } + err = json.Unmarshal(b, &vm) + if err != nil { + return nil, err + } + break + } + } + return &vm, nil } diff --git a/politeiawww/cmd/pictl/votepolicy.go b/politeiawww/cmd/pictl/votepolicy.go index 99814a742..f9b65fece 100644 --- a/politeiawww/cmd/pictl/votepolicy.go +++ b/politeiawww/cmd/pictl/votepolicy.go @@ -38,7 +38,7 @@ func (cmd *votePolicyCmd) Execute(args []string) error { return nil } -// votePolicyHelpMsg is the command help message. +// votePolicyHelpMsg is printed to stdout by the help command. const votePolicyHelpMsg = `votepolicy Fetch the ticketvote API policy.` diff --git a/politeiawww/politeiawww_test.go b/politeiawww/politeiawww_test.go index c4917bab9..a97098ffe 100644 --- a/politeiawww/politeiawww_test.go +++ b/politeiawww/politeiawww_test.go @@ -18,9 +18,6 @@ func TestHandleVersion(t *testing.T) { p, cleanup := newTestPoliteiawww(t) defer cleanup() - d := newTestPoliteiad(t, p) - defer d.Close() - expectedReply := www.VersionReply{ Version: www.PoliteiaWWWAPIVersion, BuildVersion: version.BuildMainVersion(), From fcdcabc4e61777a841c17e492f7e716bb420a5ec Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 2 Feb 2021 14:59:43 -0600 Subject: [PATCH 266/449] Improve error handling of politeiawww client. --- .../backend/tlogbe/plugins/user/hooks.go | 6 +- politeiad/plugins/comments/comments.go | 7 + politeiad/plugins/ticketvote/ticketvote.go | 7 + politeiad/plugins/user/user.go | 11 +- politeiad/politeiad.go | 46 +++-- politeiawww/client/client.go | 46 +---- politeiawww/client/error.go | 100 ++++++++++ politeiawww/client/pi.go | 7 +- politeiawww/client/records.go | 38 +++- politeiawww/client/ticketvote.go | 7 +- ...oposaldetails.go => cmdproposaldetails.go} | 6 +- .../{proposaledit.go => cmdproposaledit.go} | 26 +-- .../{proposalnew.go => cmdproposalnew.go} | 8 +- ...proposalpolicy.go => cmdproposalpolicy.go} | 10 +- politeiawww/cmd/pictl/cmdproposalstatusset.go | 171 ++++++++++++++++++ politeiawww/cmd/pictl/pictl.go | 10 +- politeiawww/cmd/pictl/print.go | 47 +++++ .../cmd/pictl/{util.go => proposal.go} | 63 ++----- politeiawww/cmd/pictl/proposalstatusset.go | 139 -------------- politeiawww/cmd/pictl/votepolicy.go | 2 +- politeiawww/pi/process.go | 2 +- politeiawww/records/error.go | 26 +-- util/json.go | 27 +-- util/req.go | 36 ++++ 24 files changed, 517 insertions(+), 331 deletions(-) create mode 100644 politeiawww/client/error.go rename politeiawww/cmd/pictl/{proposaldetails.go => cmdproposaldetails.go} (91%) rename politeiawww/cmd/pictl/{proposaledit.go => cmdproposaledit.go} (96%) rename politeiawww/cmd/pictl/{proposalnew.go => cmdproposalnew.go} (97%) rename politeiawww/cmd/pictl/{proposalpolicy.go => cmdproposalpolicy.go} (77%) create mode 100644 politeiawww/cmd/pictl/cmdproposalstatusset.go create mode 100644 politeiawww/cmd/pictl/print.go rename politeiawww/cmd/pictl/{util.go => proposal.go} (71%) delete mode 100644 politeiawww/cmd/pictl/proposalstatusset.go create mode 100644 util/req.go diff --git a/politeiad/backend/tlogbe/plugins/user/hooks.go b/politeiad/backend/tlogbe/plugins/user/hooks.go index 4b93364f6..534a594a0 100644 --- a/politeiad/backend/tlogbe/plugins/user/hooks.go +++ b/politeiad/backend/tlogbe/plugins/user/hooks.go @@ -219,12 +219,10 @@ func (p *userPlugin) hookEditMetadataPre(payload string) error { return userMetadataPreventUpdates(em.Current.Metadata, em.Metadata) } -// statusChangesDecode decodes a []StatusChange from the provided metadata -// streams. If a status change metadata stream is not found, nil is returned. func statusChangesDecode(metadata []backend.MetadataStream) ([]user.StatusChangeMetadata, error) { - var statuses []user.StatusChangeMetadata + statuses := make([]user.StatusChangeMetadata, 0, 16) for _, v := range metadata { - if v.ID == user.MDStreamIDUserMetadata { + if v.ID == user.MDStreamIDStatusChanges { d := json.NewDecoder(strings.NewReader(v.Payload)) for { var sc user.StatusChangeMetadata diff --git a/politeiad/plugins/comments/comments.go b/politeiad/plugins/comments/comments.go index 4d3f2d4d3..0d24f77f7 100644 --- a/politeiad/plugins/comments/comments.go +++ b/politeiad/plugins/comments/comments.go @@ -48,6 +48,13 @@ const ( ErrorCodeVoteChangesMax ErrorCodeT = 9 ) +var ( + // TODO ErrorCodes contains the human readable error messages. + ErrorCodes = map[ErrorCodeT]string{ + ErrorCodeInvalid: "error code invalid", + } +) + // Comment represent a record comment. // // Signature is the client signature of Token+ParentID+Comment. diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index ba56a4ba4..effdc8fa5 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -96,6 +96,13 @@ const ( ErrorCodeLinkByNotExpired ) +var ( + // TODO ErrorCodes contains the human readable error messages. + ErrorCodes = map[ErrorCodeT]string{ + ErrorCodeInvalid: "error code invalid", + } +) + const ( // FileNameVoteMetadata is the filename of the VoteMetadata file // that is saved to politeiad. VoteMetadata is saved to politeiad diff --git a/politeiad/plugins/user/user.go b/politeiad/plugins/user/user.go index f0edeb1c1..c63e12267 100644 --- a/politeiad/plugins/user/user.go +++ b/politeiad/plugins/user/user.go @@ -44,10 +44,17 @@ const ( ) var ( - // TODO fill in // ErrorCodes contains the human readable errors. ErrorCodes = map[ErrorCodeT]string{ - ErrorCodeInvalid: "error code invalid", + ErrorCodeInvalid: "error code invalid", + ErrorCodeUserMetadataNotFound: "user metadata not found", + ErrorCodeUserIDInvalid: "user id invalid", + ErrorCodePublicKeyInvalid: "public key invalid", + ErrorCodeSignatureInvalid: "signature invalid", + ErrorCodeStatusChangeMetadataNotFound: "status change metadata not found", + ErrorCodeTokenInvalid: "token invalid", + ErrorCodeStatusInvalid: "status invalid", + ErrorCodeReasonInvalid: "status reason invalid", } ) diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index cd0ea519d..591c4dfb3 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -1584,6 +1584,10 @@ func _main() error { } // Setup plugin + var ( + unvetted bool // Register as unvetted plugin + vetted bool // Register as vetted plugin + ) var plugin backend.Plugin switch v { case comments.PluginID: @@ -1592,27 +1596,36 @@ func _main() error { Settings: ps, Identity: p.identity, } + unvetted = true + vetted = true case dcrdata.PluginID: plugin = backend.Plugin{ ID: dcrdata.PluginID, Settings: ps, } + vetted = true case pi.PluginID: plugin = backend.Plugin{ ID: pi.PluginID, Settings: ps, } + unvetted = true + vetted = true case ticketvote.PluginID: plugin = backend.Plugin{ ID: ticketvote.PluginID, Settings: ps, Identity: p.identity, } + unvetted = true + vetted = true case user.PluginID: plugin = backend.Plugin{ ID: user.PluginID, Settings: ps, } + unvetted = true + vetted = true case decredplugin.ID: // TODO plugin setup for cms case cmsplugin.ID: @@ -1622,28 +1635,35 @@ func _main() error { } // Register plugin - err = p.backend.RegisterUnvettedPlugin(plugin) - if err != nil { - return fmt.Errorf("register unvetted plugin %v: %v", v, err) + if unvetted { + log.Infof("Register unvetted plugin: %v", v) + err = p.backend.RegisterUnvettedPlugin(plugin) + if err != nil { + return fmt.Errorf("register unvetted plugin %v: %v", v, err) + } } - err = p.backend.RegisterVettedPlugin(plugin) - if err != nil { - return fmt.Errorf("register vetted plugin %v: %v", v, err) + if vetted { + log.Infof("Register vetted plugin: %v", v) + err = p.backend.RegisterVettedPlugin(plugin) + if err != nil { + return fmt.Errorf("register vetted plugin %v: %v", v, err) + } } - - log.Infof("Registered plugin: %v", v) } // Setup plugins - for _, v := range cfg.Plugins { - log.Infof("Setting up plugin: %v", v) - err = p.backend.SetupUnvettedPlugin(v) + for _, v := range p.backend.GetUnvettedPlugins() { + log.Infof("Setup unvetted plugin: %v", v.ID) + err = p.backend.SetupUnvettedPlugin(v.ID) if err != nil { return fmt.Errorf("setup unvetted plugin %v: %v", v, err) } - err = p.backend.SetupVettedPlugin(v) + } + for _, v := range p.backend.GetVettedPlugins() { + log.Infof("Setup vetted plugin: %v", v.ID) + err = p.backend.SetupVettedPlugin(v.ID) if err != nil { - return fmt.Errorf("setup vetted plugin %v: %v", v, err) + return fmt.Errorf("setup vetted plugin %v: %v", v.ID, err) } } } diff --git a/politeiawww/client/client.go b/politeiawww/client/client.go index 0dac848c3..ef1f66292 100644 --- a/politeiawww/client/client.go +++ b/politeiawww/client/client.go @@ -31,50 +31,11 @@ type Client struct { http *http.Client } -// ErrorReply represents the request body that is returned from politeiawww -// when an error occurs. PluginID will only be populated if the error occured -// during execution of a plugin command. -type ErrorReply struct { - PluginID string - ErrorCode int - ErrorContext string -} - -// httpsError represents a politeiawww error. Error is returned anytime the -// politeiawww response is not a 200. -type Error struct { - HTTPCode int - ErrorReply ErrorReply -} - -// Error satisfies the error interface. -func (e Error) Error() string { - switch { - case e.HTTPCode == http.StatusNotFound: - return fmt.Sprintf("404 not found") - case e.ErrorReply.PluginID != "": - return fmt.Sprintf("politeiawww plugin error: %v %v %v", - e.HTTPCode, e.ErrorReply.PluginID, e.ErrorReply.ErrorCode) - default: - return fmt.Sprintf("politeiawww error: %v %v", - e.HTTPCode, e.ErrorReply.ErrorCode) - } -} - -// formatJSON returns a pretty printed JSON string for the provided structure. -func formatJSON(v interface{}) string { - b, err := json.MarshalIndent(v, "", " ") - if err != nil { - return fmt.Sprintf("MarshalIndent: %v", err) - } - return string(b) -} - // makeReq makes a politeiawww http request to the method and route provided, // serializing the provided object as the request body, and returning a byte // slice of the repsonse body. An Error is returned if politeiawww responds // with anything other than a 200 http status code. -func (c *Client) makeReq(method string, route string, v interface{}) ([]byte, error) { +func (c *Client) makeReq(method string, api, route string, v interface{}) ([]byte, error) { // Serialize body var ( reqBody []byte @@ -110,7 +71,7 @@ func (c *Client) makeReq(method string, route string, v interface{}) ([]byte, er } // Setup route - fullRoute := c.host + route + queryParams + fullRoute := c.host + api + route + queryParams // Print request details switch { @@ -120,7 +81,7 @@ func (c *Client) makeReq(method string, route string, v interface{}) ([]byte, er // No JSON to print case c.verbose: fmt.Printf("Request: %v %v\n", method, fullRoute) - fmt.Printf("%v\n", formatJSON(v)) + fmt.Printf("%v\n", util.FormatJSON(v)) case c.rawJSON: fmt.Printf("%s\n", reqBody) } @@ -153,6 +114,7 @@ func (c *Client) makeReq(method string, route string, v interface{}) ([]byte, er } return nil, Error{ HTTPCode: r.StatusCode, + API: api, ErrorReply: e, } } diff --git a/politeiawww/client/error.go b/politeiawww/client/error.go new file mode 100644 index 000000000..8d0e44b8f --- /dev/null +++ b/politeiawww/client/error.go @@ -0,0 +1,100 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package client + +import ( + "fmt" + "net/http" + + cmplugin "github.com/decred/politeia/politeiad/plugins/comments" + piplugin "github.com/decred/politeia/politeiad/plugins/pi" + tkplugin "github.com/decred/politeia/politeiad/plugins/ticketvote" + usplugin "github.com/decred/politeia/politeiad/plugins/user" + piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" +) + +// ErrorReply represents the request body that is returned from politeiawww +// when an error occurs. PluginID will only be populated if the error occured +// during execution of a plugin command. +type ErrorReply struct { + PluginID string + ErrorCode int + ErrorContext string +} + +// Error represents a politeiawww error. Error is returned anytime the +// politeiawww response is not a 200. +type Error struct { + HTTPCode int + API string + ErrorReply ErrorReply +} + +// Error satisfies the error interface. +func (e Error) Error() string { + switch e.HTTPCode { + case http.StatusNotFound: + return fmt.Sprintf("404 not found") + case http.StatusInternalServerError: + return fmt.Sprintf("500 internal server error: %v", + e.ErrorReply.ErrorCode) + case http.StatusBadRequest: + var msg string + if e.ErrorReply.PluginID == "" { + // API user error + msg = apiUserErr(e.API, e.ErrorReply) + } else { + // Plugin user error + msg = pluginUserErr(e.ErrorReply) + } + return fmt.Sprintf("%v %v", e.HTTPCode, msg) + default: + return fmt.Sprintf("%v %+v", e.HTTPCode, e.ErrorReply) + } +} + +func apiUserErr(api string, e ErrorReply) string { + var errMsg string + switch api { + case piv1.APIRoute: + errMsg = piv1.ErrorCodes[piv1.ErrorCodeT(e.ErrorCode)] + case rcv1.APIRoute: + errMsg = rcv1.ErrorCodes[rcv1.ErrorCodeT(e.ErrorCode)] + case tkv1.APIRoute: + errMsg = tkv1.ErrorCodes[tkv1.ErrorCodeT(e.ErrorCode)] + } + m := fmt.Sprintf("user error code %v", e.ErrorCode) + if errMsg != "" { + m += fmt.Sprintf(", %v", errMsg) + } + if e.ErrorContext != "" { + m += fmt.Sprintf(": %v", e.ErrorContext) + } + return m +} + +func pluginUserErr(e ErrorReply) string { + var errMsg string + switch e.PluginID { + case cmplugin.PluginID: + errMsg = cmplugin.ErrorCodes[cmplugin.ErrorCodeT(e.ErrorCode)] + case piplugin.PluginID: + errMsg = piplugin.ErrorCodes[piplugin.ErrorCodeT(e.ErrorCode)] + case tkplugin.PluginID: + errMsg = tkplugin.ErrorCodes[tkplugin.ErrorCodeT(e.ErrorCode)] + case usplugin.PluginID: + errMsg = usplugin.ErrorCodes[usplugin.ErrorCodeT(e.ErrorCode)] + } + m := fmt.Sprintf("%v plugin error code %v", e.PluginID, e.ErrorCode) + if errMsg != "" { + m += fmt.Sprintf(", %v", errMsg) + } + if e.ErrorContext != "" { + m += fmt.Sprintf(": %v", e.ErrorContext) + } + return m +} diff --git a/politeiawww/client/pi.go b/politeiawww/client/pi.go index 02ae61813..f668df268 100644 --- a/politeiawww/client/pi.go +++ b/politeiawww/client/pi.go @@ -10,12 +10,13 @@ import ( "net/http" piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" + "github.com/decred/politeia/util" ) // PiPolicy sends a pi v1 Policy request to politeiawww. func (c *Client) PiPolicy() (*piv1.PolicyReply, error) { - route := piv1.APIRoute + piv1.RoutePolicy - resBody, err := c.makeReq(http.MethodGet, route, nil) + resBody, err := c.makeReq(http.MethodGet, + piv1.APIRoute, piv1.RoutePolicy, nil) if err != nil { return nil, err } @@ -26,7 +27,7 @@ func (c *Client) PiPolicy() (*piv1.PolicyReply, error) { return nil, err } if c.verbose { - fmt.Printf("%v\n", formatJSON(pr)) + fmt.Printf("%v\n", util.FormatJSON(pr)) } return &pr, nil diff --git a/politeiawww/client/records.go b/politeiawww/client/records.go index 17c3aaded..da0f5ea86 100644 --- a/politeiawww/client/records.go +++ b/politeiawww/client/records.go @@ -18,8 +18,8 @@ import ( // RecordNew sends a records v1 New request to politeiawww. func (c *Client) RecordNew(n rcv1.New) (*rcv1.NewReply, error) { - route := rcv1.APIRoute + rcv1.RouteNew - resBody, err := c.makeReq(http.MethodPost, route, n) + resBody, err := c.makeReq(http.MethodPost, + rcv1.APIRoute, rcv1.RouteNew, n) if err != nil { return nil, err } @@ -30,7 +30,7 @@ func (c *Client) RecordNew(n rcv1.New) (*rcv1.NewReply, error) { return nil, err } if c.verbose { - fmt.Printf("%v\n", formatJSON(nr)) + fmt.Printf("%v\n", util.FormatJSON(nr)) } return &nr, nil @@ -38,8 +38,8 @@ func (c *Client) RecordNew(n rcv1.New) (*rcv1.NewReply, error) { // RecordEdit sends a records v1 Edit request to politeiawww. func (c *Client) RecordEdit(e rcv1.Edit) (*rcv1.EditReply, error) { - route := rcv1.APIRoute + rcv1.RouteEdit - resBody, err := c.makeReq(http.MethodPost, route, e) + resBody, err := c.makeReq(http.MethodPost, + rcv1.APIRoute, rcv1.RouteEdit, e) if err != nil { return nil, err } @@ -50,16 +50,36 @@ func (c *Client) RecordEdit(e rcv1.Edit) (*rcv1.EditReply, error) { return nil, err } if c.verbose { - fmt.Printf("%v\n", formatJSON(er)) + fmt.Printf("%v\n", util.FormatJSON(er)) } return &er, nil } +// RecordSetStatus sends a records v1 SetStatus request to politeiawww. +func (c *Client) RecordSetStatus(ss rcv1.SetStatus) (*rcv1.SetStatusReply, error) { + resBody, err := c.makeReq(http.MethodPost, + rcv1.APIRoute, rcv1.RouteSetStatus, ss) + if err != nil { + return nil, err + } + + var ssr rcv1.SetStatusReply + err = json.Unmarshal(resBody, &ssr) + if err != nil { + return nil, err + } + if c.verbose { + fmt.Printf("%v\n", util.FormatJSON(ssr)) + } + + return &ssr, nil +} + // RecordDetails sends a records v1 Details request to politeiawww. func (c *Client) RecordDetails(d rcv1.Details) (*rcv1.Record, error) { - route := rcv1.APIRoute + rcv1.RouteDetails - resBody, err := c.makeReq(http.MethodPost, route, d) + resBody, err := c.makeReq(http.MethodPost, + rcv1.APIRoute, rcv1.RouteDetails, d) if err != nil { return nil, err } @@ -70,7 +90,7 @@ func (c *Client) RecordDetails(d rcv1.Details) (*rcv1.Record, error) { return nil, err } if c.verbose { - fmt.Printf("%v\n", formatJSON(dr)) + fmt.Printf("%v\n", util.FormatJSON(dr)) } return &dr.Record, nil diff --git a/politeiawww/client/ticketvote.go b/politeiawww/client/ticketvote.go index ff7025cb5..f16c88845 100644 --- a/politeiawww/client/ticketvote.go +++ b/politeiawww/client/ticketvote.go @@ -10,12 +10,13 @@ import ( "net/http" tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" + "github.com/decred/politeia/util" ) // TicketVotePolicy sends a pi v1 Policy request to politeiawww. func (c *Client) TicketVotePolicy() (*tkv1.PolicyReply, error) { - route := tkv1.APIRoute + tkv1.RoutePolicy - resBody, err := c.makeReq(http.MethodGet, route, nil) + resBody, err := c.makeReq(http.MethodGet, + tkv1.APIRoute, tkv1.RoutePolicy, nil) if err != nil { return nil, err } @@ -26,7 +27,7 @@ func (c *Client) TicketVotePolicy() (*tkv1.PolicyReply, error) { return nil, err } if c.verbose { - fmt.Printf("%v\n", formatJSON(pr)) + fmt.Printf("%v\n", util.FormatJSON(pr)) } return &pr, nil diff --git a/politeiawww/cmd/pictl/proposaldetails.go b/politeiawww/cmd/pictl/cmdproposaldetails.go similarity index 91% rename from politeiawww/cmd/pictl/proposaldetails.go rename to politeiawww/cmd/pictl/cmdproposaldetails.go index 9472fe766..9025a626f 100644 --- a/politeiawww/cmd/pictl/proposaldetails.go +++ b/politeiawww/cmd/pictl/cmdproposaldetails.go @@ -10,7 +10,7 @@ import ( ) // proposalDetails retrieves a full proposal record. -type proposalDetailsCmd struct { +type cmdProposalDetails struct { Args struct { Token string `positional-arg-name:"token"` Version string `postional-arg-name:"version" optional:"true"` @@ -22,10 +22,10 @@ type proposalDetailsCmd struct { Unvetted bool `long:"unvetted" optional:"true"` } -// Execute executes the proposalDetailsCmd command. +// Execute executes the cmdProposalDetails command. // // This function satisfies the go-flags Commander interface. -func (c *proposalDetailsCmd) Execute(args []string) error { +func (c *cmdProposalDetails) Execute(args []string) error { // Setup client opts := pclient.Opts{ HTTPSCert: cfg.HTTPSCert, diff --git a/politeiawww/cmd/pictl/proposaledit.go b/politeiawww/cmd/pictl/cmdproposaledit.go similarity index 96% rename from politeiawww/cmd/pictl/proposaledit.go rename to politeiawww/cmd/pictl/cmdproposaledit.go index 872d063f5..c7921e024 100644 --- a/politeiawww/cmd/pictl/proposaledit.go +++ b/politeiawww/cmd/pictl/cmdproposaledit.go @@ -22,8 +22,8 @@ import ( "github.com/decred/politeia/util" ) -// proposalEditCmd edits an existing proposal. -type proposalEditCmd struct { +// cmdProposalEdit edits an existing proposal. +type cmdProposalEdit struct { Args struct { Token string `positional-arg-name:"token" required:"true"` IndexFile string `positional-arg-name:"indexfile"` @@ -56,10 +56,10 @@ type proposalEditCmd struct { RFP bool `long:"rfp" optional:"true"` } -// Execute executes the proposalEditCmd command. +// Execute executes the cmdProposalEdit command. // // This function satisfies the go-flags Commander interface. -func (c *proposalEditCmd) Execute(args []string) error { +func (c *cmdProposalEdit) Execute(args []string) error { // Unpack args token := c.Args.Token indexFile := c.Args.IndexFile @@ -229,13 +229,6 @@ func (c *proposalEditCmd) Execute(args []string) error { }) } - // Print proposal to stdout - printf("Proposal editted\n") - err = printProposalFiles(files) - if err != nil { - return err - } - // Setup request sig, err := signedMerkleRoot(files, cfg.Identity) if err != nil { @@ -250,7 +243,7 @@ func (c *proposalEditCmd) Execute(args []string) error { } // Send request - nr, err := pc.RecordEdit(e) + er, err := pc.RecordEdit(e) if err != nil { return err } @@ -260,11 +253,18 @@ func (c *proposalEditCmd) Execute(args []string) error { if err != nil { return err } - err = pclient.RecordVerify(nr.Record, vr.PubKey) + err = pclient.RecordVerify(er.Record, vr.PubKey) if err != nil { return fmt.Errorf("unable to verify record: %v", err) } + // Print proposal to stdout + printf("Proposal editted\n") + err = printProposal(er.Record) + if err != nil { + return err + } + return nil } diff --git a/politeiawww/cmd/pictl/proposalnew.go b/politeiawww/cmd/pictl/cmdproposalnew.go similarity index 97% rename from politeiawww/cmd/pictl/proposalnew.go rename to politeiawww/cmd/pictl/cmdproposalnew.go index 87cd0d2b5..6103cd720 100644 --- a/politeiawww/cmd/pictl/proposalnew.go +++ b/politeiawww/cmd/pictl/cmdproposalnew.go @@ -22,8 +22,8 @@ import ( "github.com/decred/politeia/util" ) -// proposalNewCmd submits a new proposal. -type proposalNewCmd struct { +// cmdProposalNew submits a new proposal. +type cmdProposalNew struct { Args struct { IndexFile string `positional-arg-name:"indexfile"` Attachments []string `positional-arg-name:"attachments"` @@ -45,10 +45,10 @@ type proposalNewCmd struct { RFP bool `long:"rfp" optional:"true"` } -// Execute executes the proposalNewCmd command. +// Execute executes the cmdProposalNew command. // // This function satisfies the go-flags Commander interface. -func (c *proposalNewCmd) Execute(args []string) error { +func (c *cmdProposalNew) Execute(args []string) error { // Unpack args indexFile := c.Args.IndexFile attachments := c.Args.Attachments diff --git a/politeiawww/cmd/pictl/proposalpolicy.go b/politeiawww/cmd/pictl/cmdproposalpolicy.go similarity index 77% rename from politeiawww/cmd/pictl/proposalpolicy.go rename to politeiawww/cmd/pictl/cmdproposalpolicy.go index 1b60783d5..3776987ff 100644 --- a/politeiawww/cmd/pictl/proposalpolicy.go +++ b/politeiawww/cmd/pictl/cmdproposalpolicy.go @@ -8,13 +8,13 @@ import ( pclient "github.com/decred/politeia/politeiawww/client" ) -// proposalPolicy retrieves the pi API policy. -type proposalPolicyCmd struct{} +// cmdProposalPolicy retrieves the pi API policy. +type cmdProposalPolicy struct{} -// Execute executes the proposalPolicyCmd command. +// Execute executes the cmdProposalPolicy command. // // This function satisfies the go-flags Commander interface. -func (cmd *proposalPolicyCmd) Execute(args []string) error { +func (c *cmdProposalPolicy) Execute(args []string) error { // Setup client opts := pclient.Opts{ HTTPSCert: cfg.HTTPSCert, @@ -33,7 +33,7 @@ func (cmd *proposalPolicyCmd) Execute(args []string) error { } // Print policy - println(formatJSON(pr)) + printJSON(pr) return nil } diff --git a/politeiawww/cmd/pictl/cmdproposalstatusset.go b/politeiawww/cmd/pictl/cmdproposalstatusset.go new file mode 100644 index 000000000..69724b088 --- /dev/null +++ b/politeiawww/cmd/pictl/cmdproposalstatusset.go @@ -0,0 +1,171 @@ +// Copyright (c) 2017-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "encoding/hex" + "fmt" + "strconv" + + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + pclient "github.com/decred/politeia/politeiawww/client" + "github.com/decred/politeia/politeiawww/cmd/shared" +) + +// cmdProposalSetStatus sets the status of a proposal. +type cmdProposalSetStatus struct { + Args struct { + Token string `positional-arg-name:"token" required:"true"` + Status string `positional-arg-name:"status" required:"true"` + Reason string `positional-arg-name:"reason"` + Version string `positional-arg-name:"version"` + } `positional-args:"true"` + + // Unvetted is used to indicate the state of the proposal is + // unvetted. If this flag is not used it will be assumed that + // the proposal is vetted. + Unvetted bool `long:"unvetted" optional:"true"` +} + +// Execute executes the cmdProposalSetStatus command. +// +// This function satisfies the go-flags Commander interface. +func (c *cmdProposalSetStatus) Execute(args []string) error { + // Verify user identity. This will be needed to sign the status + // change. + if cfg.Identity == nil { + return shared.ErrUserIdentityNotFound + } + + // Setup client + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Cookies: cfg.Cookies, + HeaderCSRF: cfg.CSRF, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, + } + pc, err := pclient.New(cfg.Host, opts) + if err != nil { + return err + } + + // Setup state + var state string + switch { + case c.Unvetted: + state = rcv1.RecordStateUnvetted + default: + state = rcv1.RecordStateVetted + } + + // Parse status. This can be either the numeric status code or the + // human readable equivalent. + var ( + status rcv1.RecordStatusT + + statuses = map[string]rcv1.RecordStatusT{ + "public": rcv1.RecordStatusPublic, + "censored": rcv1.RecordStatusCensored, + "abandoned": rcv1.RecordStatusArchived, + "2": rcv1.RecordStatusPublic, + "3": rcv1.RecordStatusCensored, + "4": rcv1.RecordStatusArchived, + } + ) + s, err := strconv.ParseUint(c.Args.Status, 10, 32) + if err == nil { + // Numeric status code found + status = rcv1.RecordStatusT(s) + } else if s, ok := statuses[c.Args.Status]; ok { + // Human readable status code found + status = s + } else { + return fmt.Errorf("invalid proposal status '%v'\n %v", + c.Args.Status, proposalSetStatusHelpMsg) + } + + // Setup version + var version string + if c.Args.Version != "" { + version = c.Args.Version + } else { + // Get the version manually + d := rcv1.Details{ + State: state, + Token: c.Args.Token, + } + r, err := pc.RecordDetails(d) + if err != nil { + return err + } + version = r.Version + } + + // Setup request + msg := c.Args.Token + version + strconv.Itoa(int(status)) + c.Args.Reason + sig := cfg.Identity.SignMessage([]byte(msg)) + ss := rcv1.SetStatus{ + Token: c.Args.Token, + State: state, + Version: version, + Status: status, + Reason: c.Args.Reason, + PublicKey: cfg.Identity.Public.String(), + Signature: hex.EncodeToString(sig[:]), + } + + // Send request + ssr, err := pc.RecordSetStatus(ss) + if err != nil { + return err + } + + // Verify record + vr, err := client.Version() + if err != nil { + return err + } + err = pclient.RecordVerify(ssr.Record, vr.PubKey) + if err != nil { + return fmt.Errorf("unable to verify record: %v", err) + } + + // Print proposal to stdout + printf("Proposal status updated\n") + err = printProposal(ssr.Record) + if err != nil { + return err + } + + return nil +} + +// proposalSetStatusHelpMsg is printed to stdout by the help command. +const proposalSetStatusHelpMsg = `proposalstatusset "token" "status" "reason" + +Set the status of a proposal. This command assumes the proposal is a vetted +record. If the proposal is unvetted, the --unvetted flag must be used. Requires +admin priviledges. + +Valid statuses: + public + censored + abandoned + +The following statuses require a status change reason to be included: + censored + abandoned + +Arguments: +1. token (string, required) Proposal censorship token +2. status (string, required) New status +3. message (string, optional) Status change message +4. version (string, optional) Proposal version. This will be retrieved from + the backend if one is not provided. + +Flags: + --unvetted (bool, optional) Set status of an unvetted record. +` diff --git a/politeiawww/cmd/pictl/pictl.go b/politeiawww/cmd/pictl/pictl.go index 4a22c8cdc..7dd942eae 100644 --- a/politeiawww/cmd/pictl/pictl.go +++ b/politeiawww/cmd/pictl/pictl.go @@ -64,11 +64,11 @@ type pictl struct { Users shared.UsersCmd `command:"users"` // Proposal commands - ProposalPolicy proposalPolicyCmd `command:"proposalpolicy"` - ProposalNew proposalNewCmd `command:"proposalnew"` - ProposalEdit proposalEditCmd `command:"proposaledit"` - ProposalSetStatus proposalSetStatusCmd `command:"proposalsetstatus"` - ProposalDetails proposalDetailsCmd `command:"proposaldetails"` + ProposalPolicy cmdProposalPolicy `command:"proposalpolicy"` + ProposalNew cmdProposalNew `command:"proposalnew"` + ProposalEdit cmdProposalEdit `command:"proposaledit"` + ProposalSetStatus cmdProposalSetStatus `command:"proposalsetstatus"` + ProposalDetails cmdProposalDetails `command:"proposaldetails"` Proposals proposalsCmd `command:"proposals"` ProposalInv proposalInvCmd `command:"proposalinv"` proposalTimestamps proposalTimestampsCmd `command:"proposaltimestamps"` diff --git a/politeiawww/cmd/pictl/print.go b/politeiawww/cmd/pictl/print.go new file mode 100644 index 000000000..eafe52d69 --- /dev/null +++ b/politeiawww/cmd/pictl/print.go @@ -0,0 +1,47 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + + "github.com/decred/politeia/util" +) + +// printf prints the provided string to stdout if the global config settings +// allows for it. +func printf(s string, args ...interface{}) { + switch { + case cfg.Verbose, cfg.RawJSON: + // These are handled by the politeiawwww client + case cfg.Silent: + // Do nothing + default: + // Print to stdout + fmt.Printf(s, args...) + } +} + +// printJSON pretty prints the provided structure if the global config settings +// allow for it. +func printJSON(v interface{}) { + printf("%v\n", util.FormatJSON(v)) +} + +// byteCountSI converts the provided bytes to a string representation of the +// closest SI unit (kB, MB, GB, etc). +func byteCountSI(b int64) string { + const unit = 1000 + if b < unit { + return fmt.Sprintf("%d B", b) + } + div, exp := int64(unit), 0 + for n := b / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %cB", + float64(b)/float64(div), "kMGTPE"[exp]) +} diff --git a/politeiawww/cmd/pictl/util.go b/politeiawww/cmd/pictl/proposal.go similarity index 71% rename from politeiawww/cmd/pictl/util.go rename to politeiawww/cmd/pictl/proposal.go index f58332797..f12cc4207 100644 --- a/politeiawww/cmd/pictl/util.go +++ b/politeiawww/cmd/pictl/proposal.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2021 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -19,65 +19,30 @@ import ( "github.com/decred/politeia/util" ) -// printf prints the provided string to stdout if the global config settings -// allows for it. -func printf(s string, args ...interface{}) { - switch { - case cfg.Verbose, cfg.RawJSON: - // These are handled by the politeiawwww client - case cfg.Silent: - // Do nothing - default: - // Print to stdout - fmt.Printf(s, args...) - } -} - -// println prints the provided string to stdout if the global config settings -// allows for it. -func println(s string, args ...interface{}) { - printf(s+"\n", args...) -} - -// formatJSON returns a pretty printed JSON string for the provided structure. -func formatJSON(v interface{}) string { - b, err := json.MarshalIndent(v, "", " ") - if err != nil { - return "" - } - return string(b) -} - -// byteCountSI converts the provided bytes to a string representation of the -// closest SI unit (kB, MB, GB, etc). -func byteCountSI(b int64) string { - const unit = 1000 - if b < unit { - return fmt.Sprintf("%d B", b) - } - div, exp := int64(unit), 0 - for n := b / unit; n >= unit; n /= unit { - div *= unit - exp++ - } - return fmt.Sprintf("%.1f %cB", - float64(b)/float64(div), "kMGTPE"[exp]) -} - func printProposalFiles(files []rcv1.File) error { for _, v := range files { b, err := base64.StdEncoding.DecodeString(v.Payload) if err != nil { return err } - s := byteCountSI(int64(len(b))) - printf(" %-22v %-26v %v\n", v.Name, v.MIME, s) + size := byteCountSI(int64(len(b))) + printf(" %-22v %-26v %v\n", v.Name, v.MIME, size) } return nil } func printProposal(r rcv1.Record) error { - printf("Token: %v\n", r.CensorshipRecord.Token) + printf("Token : %v\n", r.CensorshipRecord.Token) + printf("Version : %v\n", r.Version) + printf("Status : %v\n", rcv1.RecordStatuses[r.Status]) + printf("Timestamp: %v\n", r.Timestamp) + printf("Username : %v\n", r.Username) + printf("Metadata\n") + for _, v := range r.Metadata { + size := byteCountSI(int64(len([]byte(v.Payload)))) + printf(" %-2v %v\n", v.ID, size) + } + printf("Files\n") return printProposalFiles(r.Files) } diff --git a/politeiawww/cmd/pictl/proposalstatusset.go b/politeiawww/cmd/pictl/proposalstatusset.go deleted file mode 100644 index f4d913b20..000000000 --- a/politeiawww/cmd/pictl/proposalstatusset.go +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright (c) 2017-2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -// proposalSetStatusCmd sets the status of a proposal. -type proposalSetStatusCmd struct { - Args struct { - Token string `positional-arg-name:"token" required:"true"` - Status string `positional-arg-name:"status" required:"true"` - Reason string `positional-arg-name:"reason"` - Version string `positional-arg-name:"version"` - } `positional-args:"true"` - - Unvetted bool `long:"unvetted" optional:"true"` -} - -/* -// Execute executes the proposalSetStatusCmd command. -// -// This function satisfies the go-flags Commander interface. -func (cmd *proposalSetStatusCmd) Execute(args []string) error { - propStatus := map[string]pi.PropStatusT{ - "public": pi.PropStatusPublic, - "censored": pi.PropStatusCensored, - "abandoned": pi.PropStatusAbandoned, - "2": pi.PropStatusPublic, - "3": pi.PropStatusCensored, - "4": pi.PropStatusAbandoned, - } - - // Verify state. Defaults to vetted if the --unvetted flag - // is not used. - var state pi.PropStateT - switch { - case cmd.Unvetted: - state = pi.PropStateUnvetted - default: - state = pi.PropStateVetted - } - - // Validate user identity - if cfg.Identity == nil { - return shared.ErrUserIdentityNotFound - } - - // Parse proposal status. This can be either the numeric status - // code or the human readable equivalent. - var status pi.PropStatusT - s, err := strconv.ParseUint(cmd.Args.Status, 10, 32) - if err == nil { - // Numeric status code found - status = pi.PropStatusT(s) - } else if s, ok := propStatus[cmd.Args.Status]; ok { - // Human readable status code found - status = s - } else { - return fmt.Errorf("Invalid proposal status '%v'\n %v", - cmd.Args.Status, proposalSetStatusHelpMsg) - } - - // Verify version - var version string - if cmd.Args.Version != "" { - version = cmd.Args.Version - } else { - // Get the version manually - pr, err := proposalRecordLatest(state, cmd.Args.Token) - if err != nil { - return err - } - version = pr.Version - } - - // Setup request - msg := cmd.Args.Token + version + strconv.Itoa(int(status)) + cmd.Args.Reason - sig := cfg.Identity.SignMessage([]byte(msg)) - pss := pi.ProposalSetStatus{ - Token: cmd.Args.Token, - State: state, - Version: version, - Status: status, - Reason: cmd.Args.Reason, - Signature: hex.EncodeToString(sig[:]), - PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), - } - - // Send request. The request and response details are printed to - // the console based on the logging flags that were used. - err = shared.PrintJSON(pss) - if err != nil { - return err - } - pssr, err := client.ProposalSetStatus(pss) - if err != nil { - return err - } - err = shared.PrintJSON(pssr) - if err != nil { - return err - } - - // Verify proposal - vr, err := client.Version() - if err != nil { - return err - } - err = verifyProposal(pssr.Proposal, vr.PubKey) - if err != nil { - return fmt.Errorf("unable to verify proposal: %v", err) - } - - return nil -} -*/ - -// proposalSetStatusHelpMsg is the output of the help command. -const proposalSetStatusHelpMsg = `proposalstatusset "token" "status" "reason" - -Set the status of a proposal. This command assumes the proposal is a vetted -record. If the proposal is unvetted, the --unvetted flag must be used. Requires -admin priviledges. - -Valid statuses: - public - censored - abandoned - -Arguments: -1. token (string, required) Proposal censorship token -2. status (string, required) New status -3. message (string, optional) Status change message -4. version (string, optional) Proposal version. This will be fetched manually - if one is not provided. - -Flags: - --unvetted (bool, optional) Set status of an unvetted record. -` diff --git a/politeiawww/cmd/pictl/votepolicy.go b/politeiawww/cmd/pictl/votepolicy.go index f9b65fece..5e433370f 100644 --- a/politeiawww/cmd/pictl/votepolicy.go +++ b/politeiawww/cmd/pictl/votepolicy.go @@ -33,7 +33,7 @@ func (cmd *votePolicyCmd) Execute(args []string) error { } // Print policy - println(formatJSON(pr)) + printJSON(pr) return nil } diff --git a/politeiawww/pi/process.go b/politeiawww/pi/process.go index 419f27fd5..fc75393d6 100644 --- a/politeiawww/pi/process.go +++ b/politeiawww/pi/process.go @@ -183,7 +183,7 @@ func convertStatus(s pdv1.RecordStatusT) v1.PropStatusT { } func statusChangesDecode(payload []byte) ([]pduser.StatusChangeMetadata, error) { - var statuses []pduser.StatusChangeMetadata + statuses := make([]pduser.StatusChangeMetadata, 0, 16) d := json.NewDecoder(strings.NewReader(string(payload))) for { var sc pduser.StatusChangeMetadata diff --git a/politeiawww/records/error.go b/politeiawww/records/error.go index 9baacfb6c..2d37fd95c 100644 --- a/politeiawww/records/error.go +++ b/politeiawww/records/error.go @@ -25,8 +25,8 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err ) switch { case errors.As(err, &ue): - // Record user error - m := fmt.Sprintf("Records user error: %v %v %v", + // Record user error from politeiawww + m := fmt.Sprintf("%v Records user error: %v %v", util.RemoteAddr(r), ue.ErrorCode, v1.ErrorCodes[ue.ErrorCode]) if ue.ErrorContext != "" { m += fmt.Sprintf(": %v", ue.ErrorContext) @@ -44,23 +44,23 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err var ( pluginID = pe.ErrorReply.PluginID errCode = pe.ErrorReply.ErrorCode - errContext = pe.ErrorReply.ErrorContext + errContext = strings.Join(pe.ErrorReply.ErrorContext, ",") ) e := convertPDErrorCode(errCode) switch { case pluginID != "": // politeiad plugin error. Log it and return a 400. - m := fmt.Sprintf("Plugin error: %v %v %v", + m := fmt.Sprintf("%v Plugin error: %v %v", util.RemoteAddr(r), pluginID, errCode) - if len(errContext) > 0 { - m += fmt.Sprintf(": %v", strings.Join(errContext, ", ")) + if errContext != "" { + m += fmt.Sprintf(": %v", errContext) } log.Infof(m) util.RespondWithJSON(w, http.StatusBadRequest, v1.PluginErrorReply{ PluginID: pluginID, ErrorCode: errCode, - ErrorContext: strings.Join(errContext, ", "), + ErrorContext: errContext, }) return @@ -79,18 +79,18 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err return default: - // politeiad error does correspond to a user error. Log it and - // return a 400. - m := fmt.Sprintf("Records user error: %v %v %v", + // User error from politeiad that corresponds to a records user + // error. Log it and return a 400. + m := fmt.Sprintf("%v Records user error: %v %v", util.RemoteAddr(r), e, v1.ErrorCodes[e]) - if len(errContext) > 0 { - m += fmt.Sprintf(": %v", strings.Join(errContext, ", ")) + if errContext != "" { + m += fmt.Sprintf(": %v", errContext) } log.Infof(m) util.RespondWithJSON(w, http.StatusBadRequest, v1.UserErrorReply{ ErrorCode: e, - ErrorContext: strings.Join(errContext, ", "), + ErrorContext: errContext, }) return } diff --git a/util/json.go b/util/json.go index 637248c32..54d746698 100644 --- a/util/json.go +++ b/util/json.go @@ -9,9 +9,6 @@ import ( "fmt" "io" "net/http" - - pdv1 "github.com/decred/politeia/politeiad/api/v1" - "github.com/gorilla/schema" ) func RespondWithError(w http.ResponseWriter, code int, message string) { @@ -56,25 +53,11 @@ func GetErrorFromJSON(r io.Reader) (interface{}, error) { return e, nil } -// ParseGetParams parses the query params from the GET request into -// a struct. This method requires the struct type to be defined -// with `schema` tags. -func ParseGetParams(r *http.Request, dst interface{}) error { - err := r.ParseForm() +// FormatJSON returns a pretty printed JSON string for the provided structure. +func FormatJSON(v interface{}) string { + b, err := json.MarshalIndent(v, "", " ") if err != nil { - return err - } - - return schema.NewDecoder().Decode(dst, r.Form) -} - -// RemoteAddr returns a string of the remote address, i.e. the address that -// sent the request. -func RemoteAddr(r *http.Request) string { - via := r.RemoteAddr - xff := r.Header.Get(pdv1.Forward) - if xff != "" { - return fmt.Sprintf("%v via %v", xff, r.RemoteAddr) + return fmt.Sprintf("MarshalIndent: %v", err) } - return via + return string(b) } diff --git a/util/req.go b/util/req.go new file mode 100644 index 000000000..ad16484d0 --- /dev/null +++ b/util/req.go @@ -0,0 +1,36 @@ +// Copyright (c) 2017-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package util + +import ( + "fmt" + "net/http" + + pdv1 "github.com/decred/politeia/politeiad/api/v1" + "github.com/gorilla/schema" +) + +// ParseGetParams parses the query params from the GET request into +// a struct. This method requires the struct type to be defined +// with `schema` tags. +func ParseGetParams(r *http.Request, dst interface{}) error { + err := r.ParseForm() + if err != nil { + return err + } + + return schema.NewDecoder().Decode(dst, r.Form) +} + +// RemoteAddr returns a string of the remote address, i.e. the address that +// sent the request. +func RemoteAddr(r *http.Request) string { + via := r.RemoteAddr + xff := r.Header.Get(pdv1.Forward) + if xff != "" { + return fmt.Sprintf("%v via %v", xff, r.RemoteAddr) + } + return via +} From 3c1f56c4fbc056d987f713d89c85b451862ec4ad Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 2 Feb 2021 16:21:00 -0600 Subject: [PATCH 267/449] pictl proposals command. --- politeiad/plugins/user/user.go | 1 + politeiawww/api/pi/v1/v1.go | 1 + politeiawww/client/pi.go | 20 +++ politeiawww/cmd/pictl/cmdproposaldetails.go | 15 +++ politeiawww/cmd/pictl/cmdproposals.go | 90 +++++++++++++ politeiawww/cmd/pictl/cmdproposalstatusset.go | 2 +- politeiawww/cmd/pictl/pictl.go | 2 +- politeiawww/cmd/pictl/proposal.go | 68 ++++++++++ politeiawww/cmd/pictl/proposals.go | 120 ------------------ politeiawww/pi/process.go | 4 +- politeiawww/records/process.go | 2 + 11 files changed, 201 insertions(+), 124 deletions(-) create mode 100644 politeiawww/cmd/pictl/cmdproposals.go delete mode 100644 politeiawww/cmd/pictl/proposals.go diff --git a/politeiad/plugins/user/user.go b/politeiad/plugins/user/user.go index c63e12267..e0befc47c 100644 --- a/politeiad/plugins/user/user.go +++ b/politeiad/plugins/user/user.go @@ -79,6 +79,7 @@ type StatusChangeMetadata struct { Reason string `json:"message,omitempty"` PublicKey string `json:"publickey"` Signature string `json:"signature"` + Timestamp int64 `json:"timestamp"` } // Author returns the user ID of a record's author. If no UserMetadata is diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 90b2bd6b5..49ab13421 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -206,6 +206,7 @@ type StatusChange struct { Reason string `json:"message,omitempty"` PublicKey string `json:"publickey"` Signature string `json:"signature"` + Timestamp int64 `json:"timestamp"` } // Proposal represents a proposal submission and its metadata. diff --git a/politeiawww/client/pi.go b/politeiawww/client/pi.go index f668df268..b5fcc5bab 100644 --- a/politeiawww/client/pi.go +++ b/politeiawww/client/pi.go @@ -32,3 +32,23 @@ func (c *Client) PiPolicy() (*piv1.PolicyReply, error) { return &pr, nil } + +// PiProposals sends a pi v1 Proposals request to politeiawww. +func (c *Client) PiProposals(p piv1.Proposals) (*piv1.ProposalsReply, error) { + resBody, err := c.makeReq(http.MethodPost, + piv1.APIRoute, piv1.RouteProposals, p) + if err != nil { + return nil, err + } + + var pr piv1.ProposalsReply + err = json.Unmarshal(resBody, &pr) + if err != nil { + return nil, err + } + if c.verbose { + fmt.Printf("%v\n", util.FormatJSON(pr)) + } + + return &pr, nil +} diff --git a/politeiawww/cmd/pictl/cmdproposaldetails.go b/politeiawww/cmd/pictl/cmdproposaldetails.go index 9025a626f..1e0de9eb6 100644 --- a/politeiawww/cmd/pictl/cmdproposaldetails.go +++ b/politeiawww/cmd/pictl/cmdproposaldetails.go @@ -67,3 +67,18 @@ func (c *cmdProposalDetails) Execute(args []string) error { return nil } + +// proposalDetailsHelpMsg is printed to stdout by the help command. +const proposalDetailsHelpMsg = `proposaldetails [flags] "token" "version" + +Retrive a full proposal record. + +This command defaults to retrieving vetted proposals unless the --unvetted flag +is used. + +Arguments: +1. token (string, required) Proposal token. + +Flags: + --unvetted (bool, optional) Retrieve an unvetted proposal. +` diff --git a/politeiawww/cmd/pictl/cmdproposals.go b/politeiawww/cmd/pictl/cmdproposals.go new file mode 100644 index 000000000..1fe5bf01f --- /dev/null +++ b/politeiawww/cmd/pictl/cmdproposals.go @@ -0,0 +1,90 @@ +// Copyright (c) 2020 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" + pclient "github.com/decred/politeia/politeiawww/client" +) + +// cmdProposals retrieves the proposal records for the provided tokens. +type cmdProposals struct { + Args struct { + Tokens []string `positional-arg-name:"proposals" required:"true"` + } `positional-args:"true" optional:"true"` + + // Unvetted is used to indicate the state of the proposals are + // unvetted. If this flag is not used it will be assumed that the + // proposals are vetted. + Unvetted bool `long:"unvetted" optional:"true"` +} + +// Execute executes the cmdProposals command. +// +// This function satisfies the go-flags Commander interface. +func (c *cmdProposals) Execute(args []string) error { + // Setup client + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Cookies: cfg.Cookies, + HeaderCSRF: cfg.CSRF, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, + } + pc, err := pclient.New(cfg.Host, opts) + if err != nil { + return err + } + + // Setup state + var state string + switch { + case c.Unvetted: + state = piv1.ProposalStateUnvetted + default: + state = piv1.ProposalStateVetted + } + + // Get proposal details + p := piv1.Proposals{ + State: state, + Tokens: c.Args.Tokens, + } + pr, err := pc.PiProposals(p) + if err != nil { + return err + } + + // Print proposals to stdout + for _, v := range pr.Proposals { + r, err := convertProposal(v) + if err != nil { + return err + } + err = printProposal(*r) + if err != nil { + return err + } + } + + return nil +} + +// proposalsHelpMsg is printed to stdout by the help command. +const proposalsHelpMsg = `proposals [flags] "tokens..." + +Retrive the proposals for the provided tokens. The proposal index file and the +proposal attachments are not returned from this command. Use the proposal +details command if you are trying to retieve the full proposal. + +This command defaults to retrieving vetted proposals unless the --unvetted flag +is used. + +Arguments: +1. tokens ([]string, required) Proposal tokens. + +Flags: + --unvetted (bool, optional) Retrieve unvetted proposals. +` diff --git a/politeiawww/cmd/pictl/cmdproposalstatusset.go b/politeiawww/cmd/pictl/cmdproposalstatusset.go index 69724b088..13a1d9cd4 100644 --- a/politeiawww/cmd/pictl/cmdproposalstatusset.go +++ b/politeiawww/cmd/pictl/cmdproposalstatusset.go @@ -167,5 +167,5 @@ Arguments: the backend if one is not provided. Flags: - --unvetted (bool, optional) Set status of an unvetted record. + --unvetted (bool, optional) Set status of an unvetted record. ` diff --git a/politeiawww/cmd/pictl/pictl.go b/politeiawww/cmd/pictl/pictl.go index 7dd942eae..feebf486d 100644 --- a/politeiawww/cmd/pictl/pictl.go +++ b/politeiawww/cmd/pictl/pictl.go @@ -69,7 +69,7 @@ type pictl struct { ProposalEdit cmdProposalEdit `command:"proposaledit"` ProposalSetStatus cmdProposalSetStatus `command:"proposalsetstatus"` ProposalDetails cmdProposalDetails `command:"proposaldetails"` - Proposals proposalsCmd `command:"proposals"` + Proposals cmdProposals `command:"proposals"` ProposalInv proposalInvCmd `command:"proposalinv"` proposalTimestamps proposalTimestampsCmd `command:"proposaltimestamps"` diff --git a/politeiawww/cmd/pictl/proposal.go b/politeiawww/cmd/pictl/proposal.go index f12cc4207..d0ef03613 100644 --- a/politeiawww/cmd/pictl/proposal.go +++ b/politeiawww/cmd/pictl/proposal.go @@ -5,6 +5,7 @@ package main import ( + "bytes" "encoding/base64" "encoding/hex" "encoding/json" @@ -14,11 +15,78 @@ import ( "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/api/v1/mime" + usplugin "github.com/decred/politeia/politeiad/plugins/user" piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" "github.com/decred/politeia/util" ) +func convertProposal(p piv1.Proposal) (*rcv1.Record, error) { + // Setup files + files := make([]rcv1.File, 0, len(p.Files)) + for _, v := range p.Files { + files = append(files, rcv1.File{ + Name: v.Name, + MIME: v.MIME, + Digest: v.Digest, + Payload: v.Payload, + }) + } + + // Setup metadata + um := usplugin.UserMetadata{ + UserID: p.UserID, + PublicKey: p.PublicKey, + Signature: p.Signature, + } + umb, err := json.Marshal(um) + if err != nil { + return nil, err + } + var buf bytes.Buffer + for _, v := range p.Statuses { + sc := rcv1.StatusChange{ + Token: v.Token, + Version: v.Version, + Status: rcv1.RecordStatusT(v.Status), + Reason: v.Reason, + PublicKey: v.PublicKey, + Signature: v.Signature, + Timestamp: v.Timestamp, + } + b, err := json.Marshal(sc) + if err != nil { + return nil, err + } + buf.Write(b) + } + metadata := []rcv1.MetadataStream{ + { + ID: usplugin.MDStreamIDUserMetadata, + Payload: string(umb), + }, + { + ID: usplugin.MDStreamIDStatusChanges, + Payload: buf.String(), + }, + } + + return &rcv1.Record{ + State: p.State, + Status: rcv1.RecordStatusT(p.Status), + Version: p.Version, + Timestamp: p.Timestamp, + Username: p.Username, + Metadata: metadata, + Files: files, + CensorshipRecord: rcv1.CensorshipRecord{ + Token: p.CensorshipRecord.Token, + Merkle: p.CensorshipRecord.Merkle, + Signature: p.CensorshipRecord.Signature, + }, + }, nil +} + func printProposalFiles(files []rcv1.File) error { for _, v := range files { b, err := base64.StdEncoding.DecodeString(v.Payload) diff --git a/politeiawww/cmd/pictl/proposals.go b/politeiawww/cmd/pictl/proposals.go deleted file mode 100644 index 257e9f803..000000000 --- a/politeiawww/cmd/pictl/proposals.go +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright (c) 2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -// proposalsCmd retrieves the proposal records of the requested tokens and -// versions. -type proposalsCmd struct { - Args struct { - Proposals []string `positional-arg-name:"proposals" required:"true"` - } `positional-args:"true" optional:"true"` - - // Unvetted requests for unvetted proposals instead of vetted ones. - Unvetted bool `long:"unvetted" optional:"true"` - - // IncludeFiles adds the proposals files to the response payload. - IncludeFiles bool `long:"includefiles" optional:"true"` -} - -/* -// Execute executes the proposalsCmd command. -// -// This function satisfies the go-flags Commander interface. -func (c *proposalsCmd) Execute(args []string) error { - proposals := c.Args.Proposals - - // Set state to get unvetted or vetted proposals. Defaults - // to vetted unless the unvetted flag is used. - var state pi.PropStateT - switch { - case c.Unvetted: - state = pi.PropStateUnvetted - default: - state = pi.PropStateVetted - } - - // Build proposals request - var requests []pi.ProposalRequest - for _, p := range proposals { - // Parse token and version - var r pi.ProposalRequest - tokenAndVersion := strings.Split(p, ",") - switch len(tokenAndVersion) { - case 1: - // No version provided - r.Token = tokenAndVersion[0] - case 2: - // Version provided - r.Token = tokenAndVersion[0] - r.Version = tokenAndVersion[1] - default: - return fmt.Errorf("invalid format for proposal request. check " + - "the help command for usage example") - } - - requests = append(requests, r) - } - - // Setup request - p := pi.Proposals{ - State: state, - Requests: requests, - IncludeFiles: c.IncludeFiles, - } - - // Send request - err := shared.PrintJSON(p) - if err != nil { - return err - } - reply, err := client.Proposals(p) - if err != nil { - return err - } - err = shared.PrintJSON(reply) - if err != nil { - return err - } - - // Verify proposals - vr, err := client.Version() - if err != nil { - return err - } - for _, p := range reply.Proposals { - err = verifyProposal(p, vr.PubKey) - if err != nil { - return fmt.Errorf("unable to verify proposal %v: %v", - p.CensorshipRecord.Token, err) - } - } - - return nil -} -*/ - -// proposalsHelpMsg is the output of the help command. -const proposalsHelpMsg = `proposals [flags] "proposals" - -Fetch the proposal record for the requested tokens in "proposals". A request -is set by providing the censorship record token and the desired version, -comma-separated. Providing only the token will default to the latest proposal -version. - -This command defaults to fetching vetted proposals unless the --unvetted flag -is used. - -Arguments: -1. proposals ([]string, required) Proposals request - -Flags: - --unvetted (bool, optional) Request is for unvetted proposals instead of - vetted ones (default: false). - --includefiles (bool, optional) Include proposal files in the returned - proposal records (default: false). - -Example: - piwww proposals ... -` diff --git a/politeiawww/pi/process.go b/politeiawww/pi/process.go index fc75393d6..f8ab2f082 100644 --- a/politeiawww/pi/process.go +++ b/politeiawww/pi/process.go @@ -201,14 +201,13 @@ func statusChangesDecode(payload []byte) ([]pduser.StatusChangeMetadata, error) func convertRecord(r pdv1.Record, state string) (*v1.Proposal, error) { // Decode metadata streams var ( - um *pduser.UserMetadata + um pduser.UserMetadata sc = make([]pduser.StatusChangeMetadata, 0, 16) err error ) for _, v := range r.Metadata { switch v.ID { case pduser.MDStreamIDUserMetadata: - var um pduser.UserMetadata err = json.Unmarshal([]byte(v.Payload), &um) if err != nil { return nil, err @@ -242,6 +241,7 @@ func convertRecord(r pdv1.Record, state string) (*v1.Proposal, error) { Reason: v.Reason, PublicKey: v.PublicKey, Signature: v.Signature, + Timestamp: v.Timestamp, }) } diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index 8dfced995..798120505 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -8,6 +8,7 @@ import ( "context" "encoding/json" "fmt" + "time" pdv1 "github.com/decred/politeia/politeiad/api/v1" pduser "github.com/decred/politeia/politeiad/plugins/user" @@ -229,6 +230,7 @@ func (r *Records) processSetStatus(ctx context.Context, ss v1.SetStatus, u user. Reason: ss.Reason, PublicKey: ss.PublicKey, Signature: ss.Signature, + Timestamp: time.Now().Unix(), } b, err := json.Marshal(scm) if err != nil { From 857c8d0aa89ac8c591d52035ce4db365f04e1135 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 2 Feb 2021 18:30:51 -0600 Subject: [PATCH 268/449] All proposal commands and routes. --- .../backend/tlogbe/plugins/dcrdata/dcrdata.go | 3 +- .../backend/tlogbe/plugins/user/hooks.go | 1 - politeiad/backend/tlogbe/plugins/user/user.go | 2 +- politeiad/backend/tlogbe/tlog/dcrtime.go | 3 +- politeiad/client/client.go | 3 +- politeiawww/api/records/v1/v1.go | 11 +++- politeiawww/client/client.go | 29 +++++++---- politeiawww/client/error.go | 2 - politeiawww/client/records.go | 40 ++++++++++++++ politeiawww/cmd/pictl/cmdproposaldetails.go | 3 +- politeiawww/cmd/pictl/cmdproposalinv.go | 49 +++++++++++++++++ politeiawww/cmd/pictl/cmdproposals.go | 7 ++- ...timestamps.go => cmdproposaltimestamps.go} | 52 ++++++++++--------- politeiawww/cmd/pictl/pictl.go | 5 +- politeiawww/cmd/pictl/proposalinv.go | 50 ------------------ util/net.go | 36 ++++++++++++- util/req.go | 36 ------------- 17 files changed, 196 insertions(+), 136 deletions(-) create mode 100644 politeiawww/cmd/pictl/cmdproposalinv.go rename politeiawww/cmd/pictl/{proposaltimestamps.go => cmdproposaltimestamps.go} (75%) delete mode 100644 politeiawww/cmd/pictl/proposalinv.go delete mode 100644 util/req.go diff --git a/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go b/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go index 780dbc90c..e97ec7e7c 100644 --- a/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go +++ b/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go @@ -139,8 +139,7 @@ func (p *dcrdataPlugin) makeReq(method string, route string, headers map[string] r.StatusCode, method, url, body) } - resBody := util.ConvertBodyToByteArray(r.Body, false) - return resBody, nil + return util.RespBody(r), nil } // bestBlockHTTP fetches and returns the best block from the dcrdata http API. diff --git a/politeiad/backend/tlogbe/plugins/user/hooks.go b/politeiad/backend/tlogbe/plugins/user/hooks.go index 534a594a0..9f298dbde 100644 --- a/politeiad/backend/tlogbe/plugins/user/hooks.go +++ b/politeiad/backend/tlogbe/plugins/user/hooks.go @@ -235,7 +235,6 @@ func statusChangesDecode(metadata []backend.MetadataStream) ([]user.StatusChange statuses = append(statuses, sc) } } - break } return statuses, nil } diff --git a/politeiad/backend/tlogbe/plugins/user/user.go b/politeiad/backend/tlogbe/plugins/user/user.go index cad8369a5..76a46f1ef 100644 --- a/politeiad/backend/tlogbe/plugins/user/user.go +++ b/politeiad/backend/tlogbe/plugins/user/user.go @@ -84,7 +84,7 @@ func (p *userPlugin) Fsck(treeIDs []int64) error { return nil } -// TODO Settings returns the plugin's settings. +// Settings returns the plugin's settings. // // This function satisfies the plugins.PluginClient interface. func (p *userPlugin) Settings() []backend.PluginSetting { diff --git a/politeiad/backend/tlogbe/tlog/dcrtime.go b/politeiad/backend/tlogbe/tlog/dcrtime.go index 97f3d5c6d..10e0f60af 100644 --- a/politeiad/backend/tlogbe/tlog/dcrtime.go +++ b/politeiad/backend/tlogbe/tlog/dcrtime.go @@ -67,8 +67,7 @@ func (c *dcrtimeClient) makeReq(method string, route string, v interface{}) ([]b return nil, fmt.Errorf("%v: %v", r.Status, e) } - respBody := util.ConvertBodyToByteArray(r.Body, false) - return respBody, nil + return util.RespBody(r), nil } // timestampBatch posts digests to the dcrtime v2 batch timestamp route. diff --git a/politeiad/client/client.go b/politeiad/client/client.go index d4f3d64a9..41ed7ded4 100644 --- a/politeiad/client/client.go +++ b/politeiad/client/client.go @@ -95,8 +95,7 @@ func (c *Client) makeReq(ctx context.Context, method string, route string, v int } } - respBody := util.ConvertBodyToByteArray(r.Body, false) - return respBody, nil + return util.RespBody(r), nil } // New returns a new politeiad client. diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go index 570c37c8f..f5ff3f9c1 100644 --- a/politeiawww/api/records/v1/v1.go +++ b/politeiawww/api/records/v1/v1.go @@ -312,10 +312,19 @@ type RecordsReply struct { Records map[string]Record `json:"records"` // [token]Record } +const ( + // TODO implement + InventoryPageSize = 60 +) + // Inventory requests the tokens of all records in the inventory, categorized // by record state and record status. Unvetted record tokens will only be // returned to admins. -type Inventory struct{} +type Inventory struct { + State string `json:"state,omitempty"` + Status string `json:"status,omitempty"` + Page int32 `json:"page,omitempty"` +} // InventoryReply is the reply to the Inventory command. The returned maps are // map[status][]token where the status is the human readable record status diff --git a/politeiawww/client/client.go b/politeiawww/client/client.go index ef1f66292..593f98d42 100644 --- a/politeiawww/client/client.go +++ b/politeiawww/client/client.go @@ -107,20 +107,29 @@ func (c *Client) makeReq(method string, api, route string, v interface{}) ([]byt // Handle reply if r.StatusCode != http.StatusOK { - var e ErrorReply - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&e); err != nil { - return nil, fmt.Errorf("status code %v: %v", r.StatusCode, err) - } - return nil, Error{ - HTTPCode: r.StatusCode, - API: api, - ErrorReply: e, + switch r.StatusCode { + case http.StatusNotFound: + return nil, fmt.Errorf("404 not found") + case http.StatusForbidden: + return nil, fmt.Errorf("403 %s", util.RespBody(r)) + default: + // All other http status codes should have a request body that + // decodes into a ErrorReply. + var e ErrorReply + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&e); err != nil { + return nil, fmt.Errorf("status code %v: %v", r.StatusCode, err) + } + return nil, Error{ + HTTPCode: r.StatusCode, + API: api, + ErrorReply: e, + } } } // Decode response body - respBody := util.ConvertBodyToByteArray(r.Body, false) + respBody := util.RespBody(r) // Print response body. Pretty printing the response body for the // verbose output must be handled by the calling function once it diff --git a/politeiawww/client/error.go b/politeiawww/client/error.go index 8d0e44b8f..ef1e07ec5 100644 --- a/politeiawww/client/error.go +++ b/politeiawww/client/error.go @@ -37,8 +37,6 @@ type Error struct { // Error satisfies the error interface. func (e Error) Error() string { switch e.HTTPCode { - case http.StatusNotFound: - return fmt.Sprintf("404 not found") case http.StatusInternalServerError: return fmt.Sprintf("500 internal server error: %v", e.ErrorReply.ErrorCode) diff --git a/politeiawww/client/records.go b/politeiawww/client/records.go index da0f5ea86..5d4670940 100644 --- a/politeiawww/client/records.go +++ b/politeiawww/client/records.go @@ -96,6 +96,46 @@ func (c *Client) RecordDetails(d rcv1.Details) (*rcv1.Record, error) { return &dr.Record, nil } +// RecordInventory sends a records v1 Inventory request to politeiawww. +func (c *Client) RecordInventory(i rcv1.Inventory) (*rcv1.InventoryReply, error) { + resBody, err := c.makeReq(http.MethodPost, + rcv1.APIRoute, rcv1.RouteInventory, i) + if err != nil { + return nil, err + } + + var ir rcv1.InventoryReply + err = json.Unmarshal(resBody, &ir) + if err != nil { + return nil, err + } + if c.verbose { + fmt.Printf("%v\n", util.FormatJSON(ir)) + } + + return &ir, nil +} + +// RecordTimestamps sends a records v1 Timestamps request to politeiawww. +func (c *Client) RecordTimestamps(t rcv1.Timestamps) (*rcv1.TimestampsReply, error) { + resBody, err := c.makeReq(http.MethodPost, + rcv1.APIRoute, rcv1.RouteTimestamps, t) + if err != nil { + return nil, err + } + + var tr rcv1.TimestampsReply + err = json.Unmarshal(resBody, &tr) + if err != nil { + return nil, err + } + if c.verbose { + fmt.Printf("%v\n", util.FormatJSON(tr)) + } + + return &tr, nil +} + // digestsVerify verifies that all file digests match the calculated SHA256 // digests of the file payloads. func digestsVerify(files []rcv1.File) error { diff --git a/politeiawww/cmd/pictl/cmdproposaldetails.go b/politeiawww/cmd/pictl/cmdproposaldetails.go index 1e0de9eb6..f804e26df 100644 --- a/politeiawww/cmd/pictl/cmdproposaldetails.go +++ b/politeiawww/cmd/pictl/cmdproposaldetails.go @@ -74,7 +74,8 @@ const proposalDetailsHelpMsg = `proposaldetails [flags] "token" "version" Retrive a full proposal record. This command defaults to retrieving vetted proposals unless the --unvetted flag -is used. +is used. This command accepts both the full tokens or the shortened token +prefixes. Arguments: 1. token (string, required) Proposal token. diff --git a/politeiawww/cmd/pictl/cmdproposalinv.go b/politeiawww/cmd/pictl/cmdproposalinv.go new file mode 100644 index 000000000..5f616abb8 --- /dev/null +++ b/politeiawww/cmd/pictl/cmdproposalinv.go @@ -0,0 +1,49 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + pclient "github.com/decred/politeia/politeiawww/client" +) + +// cmdProposalInv retrieves the censorship record tokens of all proposals in +// the inventory, categorized by status. +type cmdProposalInv struct{} + +// Execute executes the cmdProposalInv command. +// +// This function satisfies the go-flags Commander interface. +func (c *cmdProposalInv) Execute(args []string) error { + // Setup client + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Cookies: cfg.Cookies, + HeaderCSRF: cfg.CSRF, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, + } + pc, err := pclient.New(cfg.Host, opts) + if err != nil { + return err + } + + // Get inventory + ir, err := pc.RecordInventory(rcv1.Inventory{}) + if err != nil { + return err + } + + // Print inventory + printJSON(ir) + + return nil +} + +// proposalInvHelpMsg is the command help message. +const proposalInvHelpMsg = `proposalinv + +Retrieve the censorship record tokens of all proposals in the inventory, +categorized by status. Unvetted proposals are only returned to admins.` diff --git a/politeiawww/cmd/pictl/cmdproposals.go b/politeiawww/cmd/pictl/cmdproposals.go index 1fe5bf01f..a7c8a5b17 100644 --- a/politeiawww/cmd/pictl/cmdproposals.go +++ b/politeiawww/cmd/pictl/cmdproposals.go @@ -80,11 +80,16 @@ proposal attachments are not returned from this command. Use the proposal details command if you are trying to retieve the full proposal. This command defaults to retrieving vetted proposals unless the --unvetted flag -is used. +is used. This command accepts both the full tokens or the shortened token +prefixes. Arguments: 1. tokens ([]string, required) Proposal tokens. Flags: --unvetted (bool, optional) Retrieve unvetted proposals. + +Example: +$ pictl proposals f6458c2d8d9ef41c 9f9af91cf609d839 917c6fde9bcc2118 +$ pictl proposals f6458c2 9f9af91 917c6fd ` diff --git a/politeiawww/cmd/pictl/proposaltimestamps.go b/politeiawww/cmd/pictl/cmdproposaltimestamps.go similarity index 75% rename from politeiawww/cmd/pictl/proposaltimestamps.go rename to politeiawww/cmd/pictl/cmdproposaltimestamps.go index 5c82a127d..c38ff50cd 100644 --- a/politeiawww/cmd/pictl/proposaltimestamps.go +++ b/politeiawww/cmd/pictl/cmdproposaltimestamps.go @@ -5,56 +5,61 @@ package main import ( + "fmt" + "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" + piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + pclient "github.com/decred/politeia/politeiawww/client" ) -// proposalTimestampsCmd retrieves the timestamps for a politeiawww proposal. -type proposalTimestampsCmd struct { +// cmdProposalTimestamps retrieves the timestamps for a politeiawww proposal. +type cmdProposalTimestamps struct { Args struct { Token string `positional-arg-name:"token" required:"true"` Version string `positional-arg-name:"version" optional:"true"` } `positional-args:"true"` // Unvetted is used to request the timestamps of an unvetted - // proposal. + // proposal. If this flag is not used it will be assume that the + // proposal is vetted. Unvetted bool `long:"unvetted" optional:"true"` } -/* -// Execute executes the proposalTimestampsCmd command. +// Execute executes the cmdProposalTimestamps command. // // This function satisfies the go-flags Commander interface. -func (c *proposalTimestampsCmd) Execute(args []string) error { +func (c *cmdProposalTimestamps) Execute(args []string) error { + // Setup client + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Cookies: cfg.Cookies, + HeaderCSRF: cfg.CSRF, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, + } + pc, err := pclient.New(cfg.Host, opts) + if err != nil { + return err + } - // Set proposal state. Defaults to vetted unless the unvetted flag - // is used. - var state rcv1.StateT + // Setup state + var state string switch { case c.Unvetted: - state = rcv1.StateUnvetted + state = piv1.ProposalStateUnvetted default: - state = rcv1.StateVetted + state = piv1.ProposalStateVetted } - // Setup request + // Get timestamps t := rcv1.Timestamps{ State: state, Token: c.Args.Token, Version: c.Args.Version, } - - // Send request - err := shared.PrintJSON(t) - if err != nil { - return err - } - tr, err := client.RecordTimestamps(t) - if err != nil { - return err - } - err = shared.PrintJSON(tr) + tr, err := pc.RecordTimestamps(t) if err != nil { return err } @@ -79,7 +84,6 @@ func (c *proposalTimestampsCmd) Execute(args []string) error { return nil } -*/ func verifyTimestamp(t rcv1.Timestamp) error { ts := convertTimestamp(t) diff --git a/politeiawww/cmd/pictl/pictl.go b/politeiawww/cmd/pictl/pictl.go index feebf486d..a72587618 100644 --- a/politeiawww/cmd/pictl/pictl.go +++ b/politeiawww/cmd/pictl/pictl.go @@ -70,8 +70,8 @@ type pictl struct { ProposalSetStatus cmdProposalSetStatus `command:"proposalsetstatus"` ProposalDetails cmdProposalDetails `command:"proposaldetails"` Proposals cmdProposals `command:"proposals"` - ProposalInv proposalInvCmd `command:"proposalinv"` - proposalTimestamps proposalTimestampsCmd `command:"proposaltimestamps"` + ProposalInv cmdProposalInv `command:"proposalinv"` + ProposalTimestamps cmdProposalTimestamps `command:"proposaltimestamps"` // Comments commands CommentNew commentNewCmd `command:"commentnew"` @@ -147,6 +147,7 @@ Proposal commands proposaldetials (public) Get a full proposal record proposals (public) Get proposals without their files proposalinv (public) Get inventory by proposal status + proposaltimestamps (public) Get timestamps for a proposal Comment commands commentnew (user) Submit a new comment diff --git a/politeiawww/cmd/pictl/proposalinv.go b/politeiawww/cmd/pictl/proposalinv.go deleted file mode 100644 index 01d96eee0..000000000 --- a/politeiawww/cmd/pictl/proposalinv.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) 2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -// proposalInvCmd retrieves the censorship record tokens of all proposals in -// the inventory that match the provided filtering criteria. If no filtering -// criteria is given then the full inventory is returned. -type proposalInvCmd struct { - UserID string `long:"userid" optional:"true"` -} - -/* -// Execute executes the proposalInvCmd command. -// -// This function satisfies the go-flags Commander interface. -func (c *proposalInvCmd) Execute(args []string) error { - p := pi.ProposalInventory{ - UserID: c.UserID, - } - err := shared.PrintJSON(p) - if err != nil { - return err - } - pir, err := client.ProposalInventory(p) - if err != nil { - return err - } - err = shared.PrintJSON(pir) - if err != nil { - return err - } - return nil -} -*/ - -// proposalInvHelpMsg is the command help message. -const proposalInvHelpMsg = `proposalinv - -Fetch the censorship record tokens for all proposals that match the provided -filtering criteria. If no filtering criteria is provided, the full proposal -inventory will be returned. The returned proposals are categorized by their -proposal state and proposal status. Unvetted tokens are only returned if the -logged in user is an admin. - - -Flags: - --userid (string, optional) Filter by user ID -` diff --git a/util/net.go b/util/net.go index d68db6604..addfaa541 100644 --- a/util/net.go +++ b/util/net.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Decred developers +// Copyright (c) 2017-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -15,6 +15,9 @@ import ( "net/http" "os" "time" + + pdv1 "github.com/decred/politeia/politeiad/api/v1" + "github.com/gorilla/schema" ) // NormalizeAddress returns addr with the passed default port appended if @@ -70,3 +73,34 @@ func ConvertBodyToByteArray(r io.Reader, print bool) []byte { return body.Bytes() } + +// ParseGetParams parses the query params from the GET request into a struct. +// This method requires the struct type to be defined with `schema` tags. +func ParseGetParams(r *http.Request, dst interface{}) error { + err := r.ParseForm() + if err != nil { + return err + } + + return schema.NewDecoder().Decode(dst, r.Form) +} + +// RespBody returns the reponse body as a byte slice. +func RespBody(r *http.Response) []byte { + var mw io.Writer + var body bytes.Buffer + mw = io.MultiWriter(&body) + io.Copy(mw, r.Body) + return body.Bytes() +} + +// RemoteAddr returns a string of the remote address, i.e. the address that +// sent the request. +func RemoteAddr(r *http.Request) string { + via := r.RemoteAddr + xff := r.Header.Get(pdv1.Forward) + if xff != "" { + return fmt.Sprintf("%v via %v", xff, r.RemoteAddr) + } + return via +} diff --git a/util/req.go b/util/req.go deleted file mode 100644 index ad16484d0..000000000 --- a/util/req.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) 2017-2021 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package util - -import ( - "fmt" - "net/http" - - pdv1 "github.com/decred/politeia/politeiad/api/v1" - "github.com/gorilla/schema" -) - -// ParseGetParams parses the query params from the GET request into -// a struct. This method requires the struct type to be defined -// with `schema` tags. -func ParseGetParams(r *http.Request, dst interface{}) error { - err := r.ParseForm() - if err != nil { - return err - } - - return schema.NewDecoder().Decode(dst, r.Form) -} - -// RemoteAddr returns a string of the remote address, i.e. the address that -// sent the request. -func RemoteAddr(r *http.Request) string { - via := r.RemoteAddr - xff := r.Header.Get(pdv1.Forward) - if xff != "" { - return fmt.Sprintf("%v via %v", xff, r.RemoteAddr) - } - return via -} From ce683e1e8d06083367f534ec0f2d70656f95d35f Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 2 Feb 2021 20:56:16 -0600 Subject: [PATCH 269/449] cleanup --- politeiad/api/v1/v1.go | 6 ++++-- politeiad/backend/backend.go | 6 ++++-- politeiad/backend/tlogbe/tlog/verify.go | 6 ++---- politeiawww/api/comments/v1/v1.go | 6 ++++-- politeiawww/api/pi/v1/v1.go | 10 ++++++---- politeiawww/api/records/v1/v1.go | 6 ++++-- politeiawww/api/ticketvote/v1/v1.go | 6 ++++-- politeiawww/client/client.go | 2 +- politeiawww/client/error.go | 4 ++-- politeiawww/cmd/pictl/proposal.go | 2 +- 10 files changed, 32 insertions(+), 22 deletions(-) diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index 035286ccc..823d8ad25 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -368,8 +368,10 @@ type UpdateUnvettedMetadataReply struct { Response string `json:"response"` // Challenge response } -// Proof contains an inclusion proof for the digest in the merkle root. The -// ExtraData field is used by certain types of proofs to include additional +// Proof contains an inclusion proof for the digest in the merkle root. All +// digests are hex encoded SHA256 digests. +// +// The ExtraData field is used by certain types of proofs to include additional // data that is required to validate the proof. type Proof struct { Type string `json:"type"` diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index bdc3dab61..f6399da28 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -153,8 +153,10 @@ type Record struct { Files []File // User provided files } -// Proof contains an inclusion proof for the digest in the merkle root. The -// ExtraData field is used by certain types of proofs to include additional +// Proof contains an inclusion proof for the digest in the merkle root. All +// digests are hex encoded SHA256 digests. +// +// The ExtraData field is used by certain types of proofs to include additional // data that is required to validate the proof. type Proof struct { Type string diff --git a/politeiad/backend/tlogbe/tlog/verify.go b/politeiad/backend/tlogbe/tlog/verify.go index 621fcd64b..ef34f09ba 100644 --- a/politeiad/backend/tlogbe/tlog/verify.go +++ b/politeiad/backend/tlogbe/tlog/verify.go @@ -18,8 +18,6 @@ import ( "github.com/google/trillian/merkle/hashers" ) -// TODO test all of this validation - const ( // ProofTypeTrillianRFC6962 indicates a trillian proof that uses // the trillian hashing strategy HashStrategy_RFC6962_SHA256. @@ -174,10 +172,10 @@ func VerifyTimestamp(t backend.Timestamp) error { } // Verify proofs - for i, v := range t.Proofs { + for _, v := range t.Proofs { err := verifyProof(v) if err != nil { - return fmt.Errorf("invalid proof %v: %v", i, err) + return fmt.Errorf("invalid %v proof: %v", v.Type, err) } } diff --git a/politeiawww/api/comments/v1/v1.go b/politeiawww/api/comments/v1/v1.go index 8fd191612..3661ae8c9 100644 --- a/politeiawww/api/comments/v1/v1.go +++ b/politeiawww/api/comments/v1/v1.go @@ -244,8 +244,10 @@ type VotesReply struct { Votes []CommentVote `json:"votes"` } -// Proof contains an inclusion proof for the digest in the merkle root. The -// ExtraData field is used by certain types of proofs to include additional +// Proof contains an inclusion proof for the digest in the merkle root. All +// digests are hex encoded SHA256 digests. +// +// The ExtraData field is used by certain types of proofs to include additional // data that is required to validate the proof. type Proof struct { Type string `json:"type"` diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 49ab13421..3013ee2a9 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -38,7 +38,6 @@ const ( var ( // ErrorCodes contains human readable error messages. - // TODO fill in error status messages ErrorCodes = map[ErrorCodeT]string{ ErrorCodeInvalid: "error status invalid", ErrorCodeInputInvalid: "input invalid", @@ -139,15 +138,18 @@ const ( const ( // FileNameIndexFile is the file name of the proposal markdown - // file that contains the proposal contents. + // file that contains the main proposal contents. All proposal + // submissions must contain an index file. FileNameIndexFile = "index.md" // FileNameProposalMetadata is the file name of the user submitted - // ProposalMetadata. + // ProposalMetadata. All proposal submissions must contain a + // proposal metadata file. FileNameProposalMetadata = "proposalmetadata.json" // FileNameVoteMetadata is the file name of the user submitted - // VoteMetadata. + // VoteMetadata. This file will only be present when proposals + // are hosting or participating in certain types of votes. FileNameVoteMetadata = "votemetadata.json" ) diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go index f5ff3f9c1..79a0e0395 100644 --- a/politeiawww/api/records/v1/v1.go +++ b/politeiawww/api/records/v1/v1.go @@ -334,8 +334,10 @@ type InventoryReply struct { Vetted map[string][]string `json:"vetted"` } -// Proof contains an inclusion proof for the digest in the merkle root. The -// ExtraData field is used by certain types of proofs to include additional +// Proof contains an inclusion proof for the digest in the merkle root. All +// digests are hex encoded SHA256 digests. +// +// The ExtraData field is used by certain types of proofs to include additional // data that is required to validate the proof. type Proof struct { Type string `json:"type"` diff --git a/politeiawww/api/ticketvote/v1/v1.go b/politeiawww/api/ticketvote/v1/v1.go index edc38b5c2..0388236d1 100644 --- a/politeiawww/api/ticketvote/v1/v1.go +++ b/politeiawww/api/ticketvote/v1/v1.go @@ -507,8 +507,10 @@ type InventoryReply struct { BestBlock uint32 `json:"bestblock"` } -// Proof contains an inclusion proof for the digest in the merkle root. The -// ExtraData field is used by certain types of proofs to include additional +// Proof contains an inclusion proof for the digest in the merkle root. All +// digests are hex encoded SHA256 digests. +// +// The ExtraData field is used by certain types of proofs to include additional // data that is required to validate the proof. type Proof struct { Type string `json:"type"` diff --git a/politeiawww/client/client.go b/politeiawww/client/client.go index 593f98d42..e5e03f5bd 100644 --- a/politeiawww/client/client.go +++ b/politeiawww/client/client.go @@ -33,7 +33,7 @@ type Client struct { // makeReq makes a politeiawww http request to the method and route provided, // serializing the provided object as the request body, and returning a byte -// slice of the repsonse body. An Error is returned if politeiawww responds +// slice of the response body. An Error is returned if politeiawww responds // with anything other than a 200 http status code. func (c *Client) makeReq(method string, api, route string, v interface{}) ([]byte, error) { // Serialize body diff --git a/politeiawww/client/error.go b/politeiawww/client/error.go index ef1e07ec5..5c5d36387 100644 --- a/politeiawww/client/error.go +++ b/politeiawww/client/error.go @@ -26,8 +26,8 @@ type ErrorReply struct { ErrorContext string } -// Error represents a politeiawww error. Error is returned anytime the -// politeiawww response is not a 200. +// Error represents a politeiawww response error. An Error is returned anytime +// the politeiawww response is not a 200. type Error struct { HTTPCode int API string diff --git a/politeiawww/cmd/pictl/proposal.go b/politeiawww/cmd/pictl/proposal.go index d0ef03613..0bbc88b50 100644 --- a/politeiawww/cmd/pictl/proposal.go +++ b/politeiawww/cmd/pictl/proposal.go @@ -34,7 +34,7 @@ func convertProposal(p piv1.Proposal) (*rcv1.Record, error) { } // Setup metadata - um := usplugin.UserMetadata{ + um := rcv1.UserMetadata{ UserID: p.UserID, PublicKey: p.PublicKey, Signature: p.Signature, From b184541500c568746f47b0cc07b2b13f73dd1c28 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 3 Feb 2021 08:51:56 -0600 Subject: [PATCH 270/449] Fix timestamp bug. --- politeiad/backend/tlogbe/tlog/tlog.go | 19 ++++++++++-- politeiad/backend/tlogbe/tlog/verify.go | 40 +++++++++++++++++++++---- 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/politeiad/backend/tlogbe/tlog/tlog.go b/politeiad/backend/tlogbe/tlog/tlog.go index c01f80349..c033d4f81 100644 --- a/politeiad/backend/tlogbe/tlog/tlog.go +++ b/politeiad/backend/tlogbe/tlog/tlog.go @@ -1332,11 +1332,11 @@ func (t *Tlog) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trillian } // Setup proof for data digest inclusion in the log merkle root - ed := ExtraDataTrillianRFC6962{ + edt := ExtraDataTrillianRFC6962{ LeafIndex: p.LeafIndex, TreeSize: int64(a.LogRoot.TreeSize), } - extraData, err := json.Marshal(ed) + extraData, err := json.Marshal(edt) if err != nil { return nil, err } @@ -1357,7 +1357,19 @@ func (t *Tlog) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trillian if a.VerifyDigest.Digest != trillianProof.MerkleRoot { return nil, fmt.Errorf("trillian merkle root not anchored") } - hashes := a.VerifyDigest.ChainInformation.MerklePath.Hashes + var ( + numLeaves = a.VerifyDigest.ChainInformation.MerklePath.NumLeaves + hashes = a.VerifyDigest.ChainInformation.MerklePath.Hashes + flags = a.VerifyDigest.ChainInformation.MerklePath.Flags + ) + edd := ExtraDataDcrtime{ + NumLeaves: numLeaves, + Flags: base64.StdEncoding.EncodeToString(flags), + } + extraData, err = json.Marshal(edd) + if err != nil { + return nil, err + } merklePath = make([]string, 0, len(hashes)) for _, v := range hashes { merklePath = append(merklePath, hex.EncodeToString(v[:])) @@ -1367,6 +1379,7 @@ func (t *Tlog) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trillian Digest: a.VerifyDigest.Digest, MerkleRoot: a.VerifyDigest.ChainInformation.MerkleRoot, MerklePath: merklePath, + ExtraData: string(extraData), } // Update timestamp diff --git a/politeiad/backend/tlogbe/tlog/verify.go b/politeiad/backend/tlogbe/tlog/verify.go index ef34f09ba..762ba25cf 100644 --- a/politeiad/backend/tlogbe/tlog/verify.go +++ b/politeiad/backend/tlogbe/tlog/verify.go @@ -6,10 +6,12 @@ package tlog import ( "crypto/sha256" + "encoding/base64" "encoding/hex" "encoding/json" "fmt" + "github.com/decred/dcrtime/merkle" dmerkle "github.com/decred/dcrtime/merkle" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/util" @@ -34,6 +36,13 @@ type ExtraDataTrillianRFC6962 struct { TreeSize int64 `json:"treesize"` } +// ExtraDataDcrtime requires the extra data required to verify a dcrtime +// inclusion proof. +type ExtraDataDcrtime struct { + NumLeaves uint32 // Nuber of leaves + Flags string // Bitmap of merkle tree, base64 encoded +} + func verifyProofTrillian(p backend.Proof) error { // Verify type if p.Type != ProofTypeTrillianRFC6962 { @@ -96,19 +105,38 @@ func verifyProofDcrtime(p backend.Proof) error { p.Digest, p.MerklePath) } + // Decode extra data + var ed ExtraDataDcrtime + err := json.Unmarshal([]byte(p.ExtraData), &ed) + if err != nil { + return err + } + flags, err := base64.StdEncoding.DecodeString(ed.Flags) + if err != nil { + return err + } + // Calculate merkle root - digests := make([]*[sha256.Size]byte, 0, len(p.MerklePath)) + digests := make([][sha256.Size]byte, 0, len(p.MerklePath)) for _, v := range p.MerklePath { b, err := hex.DecodeString(v) if err != nil { return err } - var h [sha256.Size]byte - copy(h[:], b) - digests = append(digests, &h) + var d [sha256.Size]byte + copy(d[:], b) + digests = append(digests, d) + } + mb := merkle.Branch{ + NumLeaves: ed.NumLeaves, + Hashes: digests, + Flags: flags, + } + mr, err := dmerkle.VerifyAuthPath(&mb) + if err != nil { + return err } - r := dmerkle.Root(digests) - merkleRoot := hex.EncodeToString(r[:]) + merkleRoot := hex.EncodeToString(mr[:]) // Verify merkle root matches if merkleRoot != p.MerkleRoot { From 2a27e071b279e9e4a24ec544a73842b510af1464 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 4 Feb 2021 09:05:06 -0600 Subject: [PATCH 271/449] Hook up comments plugin settings and api policy. --- .../backend/tlogbe/plugins/comments/cmds.go | 23 ++-- .../tlogbe/plugins/comments/comments.go | 55 +++++++- politeiad/backend/tlogbe/plugins/pi/pi.go | 2 +- .../tlogbe/plugins/ticketvote/hooks.go | 9 +- .../tlogbe/plugins/ticketvote/ticketvote.go | 3 +- politeiad/plugins/comments/comments.go | 96 +++++++++++--- politeiad/plugins/pi/pi.go | 6 +- politeiad/plugins/ticketvote/ticketvote.go | 19 ++- politeiawww/api/comments/v1/v1.go | 34 +++-- politeiawww/client/comments.go | 34 +++++ .../{commentcensor.go => cmdcommentcensor.go} | 10 +- politeiawww/cmd/pictl/cmdcommentnew.go | 121 ++++++++++++++++++ .../cmd/pictl/{comments.go => cmdcomments.go} | 11 +- politeiawww/cmd/pictl/cmdcommentspolicy.go | 44 +++++++ ...ttimestamps.go => cmdcommenttimestamps.go} | 9 +- .../{commentvote.go => cmdcommentvote.go} | 10 +- .../{commentvotes.go => cmdcommentvotes.go} | 13 +- politeiawww/cmd/pictl/cmdproposalinv.go | 2 +- .../cmd/pictl/cmdproposaltimestamps.go | 10 +- politeiawww/cmd/pictl/commentnew.go | 121 ------------------ politeiawww/cmd/pictl/pictl.go | 13 +- politeiawww/comments/comments.go | 61 ++++++++- politeiawww/piwww.go | 13 +- 23 files changed, 498 insertions(+), 221 deletions(-) create mode 100644 politeiawww/client/comments.go rename politeiawww/cmd/pictl/{commentcensor.go => cmdcommentcensor.go} (91%) create mode 100644 politeiawww/cmd/pictl/cmdcommentnew.go rename politeiawww/cmd/pictl/{comments.go => cmdcomments.go} (80%) create mode 100644 politeiawww/cmd/pictl/cmdcommentspolicy.go rename politeiawww/cmd/pictl/{commenttimestamps.go => cmdcommenttimestamps.go} (90%) rename politeiawww/cmd/pictl/{commentvote.go => cmdcommentvote.go} (91%) rename politeiawww/cmd/pictl/{commentvotes.go => cmdcommentvotes.go} (79%) delete mode 100644 politeiawww/cmd/pictl/commentnew.go diff --git a/politeiad/backend/tlogbe/plugins/comments/cmds.go b/politeiad/backend/tlogbe/plugins/comments/cmds.go index 98513922b..25f58880d 100644 --- a/politeiad/backend/tlogbe/plugins/comments/cmds.go +++ b/politeiad/backend/tlogbe/plugins/comments/cmds.go @@ -630,11 +630,12 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str } // Verify comment - if len(n.Comment) > comments.PolicyCommentLengthMax { + if len(n.Comment) > int(p.commentLengthMax) { + e := fmt.Sprintf("max length is %v characters", p.commentLengthMax) return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeCommentTextInvalid), - ErrorContext: "exceeds max length", + ErrorCode: int(comments.ErrorCodeMaxLengthExceeded), + ErrorContext: e, } } @@ -756,11 +757,12 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st } // Verify comment - if len(e.Comment) > comments.PolicyCommentLengthMax { + if len(e.Comment) > int(p.commentLengthMax) { + e := fmt.Sprintf("max length is %v characters", p.commentLengthMax) return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeCommentTextInvalid), - ErrorContext: "exceeds max length", + ErrorCode: int(comments.ErrorCodeMaxLengthExceeded), + ErrorContext: e, } } @@ -811,9 +813,8 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st // Verify comment changes if e.Comment == existing.Comment { return "", backend.PluginError{ - PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeCommentTextInvalid), - ErrorContext: "comment did not change", + PluginID: comments.PluginID, + ErrorCode: int(comments.ErrorCodeNoChanges), } } @@ -1069,10 +1070,10 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st if !ok { uvotes = make([]voteIndex, 0) } - if len(uvotes) > comments.PolicyVoteChangesMax { + if len(uvotes) > int(p.voteChangesMax) { return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeVoteChangesMax), + ErrorCode: int(comments.ErrorCodeVoteChangesMaxExceeded), } } diff --git a/politeiad/backend/tlogbe/plugins/comments/comments.go b/politeiad/backend/tlogbe/plugins/comments/comments.go index 3ae749991..9cc710173 100644 --- a/politeiad/backend/tlogbe/plugins/comments/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments/comments.go @@ -6,8 +6,10 @@ package comments import ( "encoding/hex" + "fmt" "os" "path/filepath" + "strconv" "sync" "github.com/decred/politeia/politeiad/api/v1/identity" @@ -44,6 +46,10 @@ type commentsPlugin struct { // Mutexes contains a mutex for each record. The mutexes are lazy // loaded. mutexes map[string]*sync.Mutex // [string]mutex + + // Plugin settings + commentLengthMax uint32 + voteChangesMax uint32 } // mutex returns the mutex for a record. @@ -123,13 +129,22 @@ func (p *commentsPlugin) Fsck(treeIDs []int64) error { return nil } -// TODO Settings returns the plugin's settings. +// Settings returns the plugin's settings. // // This function satisfies the plugins.PluginClient interface. func (p *commentsPlugin) Settings() []backend.PluginSetting { log.Tracef("Settings") - return nil + return []backend.PluginSetting{ + { + Key: comments.SettingKeyCommentLengthMax, + Value: strconv.FormatUint(uint64(p.commentLengthMax), 10), + }, + { + Key: comments.SettingKeyVoteChangesMax, + Value: strconv.FormatUint(uint64(p.voteChangesMax), 10), + }, + } } // New returns a new comments plugin. @@ -141,10 +156,38 @@ func New(tlog plugins.TlogClient, settings []backend.PluginSetting, dataDir stri return nil, err } + // Default plugin settings + var ( + commentLengthMax = comments.SettingCommentLengthMax + voteChangesMax = comments.SettingVoteChangesMax + ) + + // Override defaults with any passed in settings + for _, v := range settings { + switch v.Key { + case comments.SettingKeyCommentLengthMax: + u, err := strconv.ParseUint(v.Value, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid plugin setting %v '%v': %v", + v.Key, v.Value, err) + } + commentLengthMax = uint32(u) + case comments.SettingKeyVoteChangesMax: + u, err := strconv.ParseUint(v.Value, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid plugin setting %v '%v': %v", + v.Key, v.Value, err) + } + voteChangesMax = uint32(u) + } + } + return &commentsPlugin{ - tlog: tlog, - identity: id, - dataDir: dataDir, - mutexes: make(map[string]*sync.Mutex), + tlog: tlog, + identity: id, + dataDir: dataDir, + mutexes: make(map[string]*sync.Mutex), + commentLengthMax: commentLengthMax, + voteChangesMax: voteChangesMax, }, nil } diff --git a/politeiad/backend/tlogbe/plugins/pi/pi.go b/politeiad/backend/tlogbe/plugins/pi/pi.go index ba3f03256..ac8a541e7 100644 --- a/politeiad/backend/tlogbe/plugins/pi/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi/pi.go @@ -153,7 +153,7 @@ func New(backend backend.Backend, settings []backend.PluginSetting, dataDir stri nameSupportedChars = pi.SettingProposalNameSupportedChars ) - // Override default plugin settings with any passed in settings + // Override defaults with any passed in settings for _, v := range settings { switch v.Key { case pi.SettingKeyTextFileSizeMax: diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go b/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go index 9b499caea..e26c8708a 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go @@ -87,11 +87,14 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { } return err } - if r.RecordMetadata.Status != backend.MDStatusCensored { + if r.RecordMetadata.Status != backend.MDStatusVetted { + e := fmt.Sprintf("record status is invalid: got %v, want %v", + backend.MDStatus[r.RecordMetadata.Status], + backend.MDStatus[backend.MDStatusVetted]) return backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), - ErrorContext: "record is censored", + ErrorContext: e, } } @@ -141,7 +144,7 @@ func (p *ticketVotePlugin) voteMetadataVerify(vm ticketvote.VoteMetadata) error return backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeVoteMetadataInvalid), - ErrorContext: "md is empty", + ErrorContext: "metadata is empty", } case vm.LinkBy != 0 && vm.LinkTo != "": diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go index 3e237a432..622b01102 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go @@ -295,8 +295,7 @@ func New(backend backend.Backend, tlog plugins.TlogClient, settings []backend.Pl return nil, fmt.Errorf("unknown active net: %v", activeNetParams.Name) } - // Override default plugin settings with user provided plugin - // settings. + // Override defaults with any passed in settings for _, v := range settings { switch v.Key { case ticketvote.SettingKeyLinkByPeriodMin: diff --git a/politeiad/plugins/comments/comments.go b/politeiad/plugins/comments/comments.go index 0d24f77f7..b89bb19c9 100644 --- a/politeiad/plugins/comments/comments.go +++ b/politeiad/plugins/comments/comments.go @@ -21,37 +21,95 @@ const ( CmdCount = "count" // Get comments count for a record CmdVotes = "votes" // Get comment votes CmdTimestamps = "timestamps" // Get timestamps +) + +// Plugin setting keys can be used to specify custom plugin settings. Default +// plugin setting values can be overridden by providing a plugin setting key +// and value to the plugin on startup. +const ( + // SettingKeyCommentLengthMax is the plugin setting key for the + // comment length max plugin setting. + SettingKeyCommentLengthMax = "commentlengthmax" - // PolicyCommentLengthMax is the maximum number of characters - // accepted for comments. - PolicyCommentLengthMax = 8000 + // SettingKeyVoteChangesMax is the plugin setting key for the vote + // changes max plugin setting. + SettingKeyVoteChangesMax = "votechangesmax" +) - // PolicayVoteChangesMax is the maximum number times a user can - // change their vote on a comment. This prevents a malicious user - // from being able to spam comment votes. - PolicyVoteChangesMax = 5 +// Plugin setting default values. These can be overridden by providing a plugin +// setting key and value to the plugin on startup. +const ( + // SettingCommentLengthMax is the default maximum number of + // characters that are allowed in a comment. + SettingCommentLengthMax uint32 = 8000 + + // SettingVoteChangesMax is the defualt maximum number of times a + // user can change their vote on a comment. This prevents a + // malicious user from being able to spam comment votes. + SettingVoteChangesMax uint32 = 5 ) // ErrorCodeT represents a error that was caused by the user. type ErrorCodeT int const ( - ErrorCodeInvalid ErrorCodeT = 0 - ErrorCodeTokenInvalid ErrorCodeT = 1 - ErrorCodePublicKeyInvalid ErrorCodeT = 2 - ErrorCodeSignatureInvalid ErrorCodeT = 3 - ErrorCodeCommentTextInvalid ErrorCodeT = 4 - ErrorCodeCommentNotFound ErrorCodeT = 5 - ErrorCodeUserUnauthorized ErrorCodeT = 6 - ErrorCodeParentIDInvalid ErrorCodeT = 7 - ErrorCodeVoteInvalid ErrorCodeT = 8 - ErrorCodeVoteChangesMax ErrorCodeT = 9 + // ErrorCodeInvalid is an invalid error code. + ErrorCodeInvalid ErrorCodeT = 0 + + // ErrorCodeTokenInvalid is returned when a token is invalid. + ErrorCodeTokenInvalid ErrorCodeT = 1 + + // ErrorCodePublicKeyInvalid is returned when a public key is + // invalid. + ErrorCodePublicKeyInvalid ErrorCodeT = 2 + + // ErrorCodeSignatureInvalid is returned when a signature is + // invalid. + ErrorCodeSignatureInvalid ErrorCodeT = 3 + + // ErrorCodeMaxLengthExceeded is returned when a comment exceeds the + // max length plugin setting. + ErrorCodeMaxLengthExceeded ErrorCodeT = 4 + + // ErrorCodeNoChanges is returned when a comment edit does not + // contain any changes. + ErrorCodeNoChanges ErrorCodeT = 5 + + // ErrorCodeCommentNotFound is returned when a comment could not be + // found. + ErrorCodeCommentNotFound ErrorCodeT = 6 + + // ErrorCodeUserUnauthorized is returned when a user is attempting + // to edit a comment that they did not submit. + ErrorCodeUserUnauthorized ErrorCodeT = 7 + + // ErrorCodeParentIDInvalid is returned when a comment parent ID + // does not correspond to an actual comment. + ErrorCodeParentIDInvalid ErrorCodeT = 8 + + // ErrorCodeVoteInvalid is returned when a comment vote is invalid. + ErrorCodeVoteInvalid ErrorCodeT = 9 + + // ErrorCodeVoteChangesMaxExceeded is returned when the number of + // times the user has changed their vote has exceeded the vote + // changes max plugin setting. + ErrorCodeVoteChangesMaxExceeded ErrorCodeT = 10 ) var ( - // TODO ErrorCodes contains the human readable error messages. + // ErrorCodes contains the human readable error messages. ErrorCodes = map[ErrorCodeT]string{ - ErrorCodeInvalid: "error code invalid", + ErrorCodeInvalid: "error code invalid", + ErrorCodeTokenInvalid: "token invalid", + ErrorCodePublicKeyInvalid: "public key invalid", + ErrorCodeSignatureInvalid: "signature invalid", + ErrorCodeMaxLengthExceeded: "max length exceeded", + ErrorCodeNoChanges: "no changes", + ErrorCodeCommentNotFound: "comment not found", + ErrorCodeUserUnauthorized: "user unauthorized", + ErrorCodeParentIDInvalid: "parent id invalid", + ErrorCodeVoteInvalid: "vote invalid", + ErrorCodeVoteChangesMaxExceeded: "vote changes max exceeded", } ) diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index fc5609c64..516ca9d0d 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -13,9 +13,9 @@ const ( // Plugin commands CmdVoteInv = "voteinv" - // Plugin setting keys and default values. Default plugin setting - // values can be overridden by passing in a custom plugin setting - // key and value on startup. + // Setting keys are the plugin setting keys that can be used to + // override a default plugin setting. Defaults will be overridden + // if a plugin setting is provided to the plugin on startup. SettingKeyTextFileSizeMax = "textfilesizemax" SettingKeyImageFileCountMax = "imagefilecountmax" SettingKeyImageFileSizeMax = "imagefilesizemax" diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index effdc8fa5..3f0129150 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -75,7 +75,6 @@ const ( type ErrorCodeT int const ( - // TODO number these ErrorCodeInvalid ErrorCodeT = 0 ErrorCodeTokenInvalid ErrorCodeT = iota ErrorCodePublicKeyInvalid @@ -99,7 +98,23 @@ const ( var ( // TODO ErrorCodes contains the human readable error messages. ErrorCodes = map[ErrorCodeT]string{ - ErrorCodeInvalid: "error code invalid", + ErrorCodeInvalid: "error code invalid", + ErrorCodeTokenInvalid: "token invalid", + ErrorCodePublicKeyInvalid: "public key invalid", + ErrorCodeSignatureInvalid: "signature invalid", + ErrorCodeRecordVersionInvalid: "record version invalid", + ErrorCodeRecordStatusInvalid: "record status invalid", + ErrorCodeAuthorizationInvalid: "authorization invalid", + ErrorCodeStartDetailsMissing: "start details missing", + ErrorCodeStartDetailsInvalid: "start details invalid", + ErrorCodeVoteParamsInvalid: "vote params invalid", + ErrorCodeVoteStatusInvalid: "vote status invalid", + ErrorCodePageSizeExceeded: "page size exceeded", + ErrorCodeVoteMetadataInvalid: "vote metadata invalid", + ErrorCodeLinkByInvalid: "linkby invalid", + ErrorCodeLinkToInvalid: "linkto invalid", + ErrorCodeRunoffVoteParentInvalid: "runoff vote parent invalid", + ErrorCodeLinkByNotExpired: "linkby not exipred", } ) diff --git a/politeiawww/api/comments/v1/v1.go b/politeiawww/api/comments/v1/v1.go index 3661ae8c9..da332ff12 100644 --- a/politeiawww/api/comments/v1/v1.go +++ b/politeiawww/api/comments/v1/v1.go @@ -11,6 +11,7 @@ const ( APIRoute = "/comments/v1" // Routes + RoutePolicy = "/policy" RouteNew = "/new" RouteVote = "/vote" RouteDel = "/del" @@ -28,23 +29,23 @@ const ( type ErrorCodeT int const ( - // Error codes - ErrorCodeInvalid ErrorCodeT = 0 - ErrorCodeInputInvalid ErrorCodeT = iota - ErrorCodeUnauthorized - ErrorCodePublicKeyInvalid - ErrorCodeSignatureInvalid - ErrorCodeRecordStateInvalid + ErrorCodeInvalid ErrorCodeT = 0 + ErrorCodeInputInvalid ErrorCodeT = 1 + ErrorCodeUnauthorized ErrorCodeT = 2 + ErrorCodePublicKeyInvalid ErrorCodeT = 3 + ErrorCodeSignatureInvalid ErrorCodeT = 4 + ErrorCodeRecordStateInvalid ErrorCodeT = 5 ) var ( // ErrorCodes contains the human readable errors. ErrorCodes = map[ErrorCodeT]string{ - ErrorCodeInvalid: "error invalid", - ErrorCodeInputInvalid: "input invalid", - ErrorCodeUnauthorized: "unauthorized", - ErrorCodePublicKeyInvalid: "public key invalid", - ErrorCodeSignatureInvalid: "signature invalid", + ErrorCodeInvalid: "error invalid", + ErrorCodeInputInvalid: "input invalid", + ErrorCodeUnauthorized: "unauthorized", + ErrorCodePublicKeyInvalid: "public key invalid", + ErrorCodeSignatureInvalid: "signature invalid", + ErrorCodeRecordStateInvalid: "record state invalid", } ) @@ -87,6 +88,15 @@ func (e ServerErrorReply) Error() string { return fmt.Sprintf("server error: %v", e.ErrorCode) } +// Policy requests the comments API policy. +type Policy struct{} + +// PolicyReply is the reply to the policy command. +type PolicyReply struct { + LengthMax uint32 `json:"lengthmax"` // In characters + VoteChangesMax uint32 `json:"votechangesmax"` +} + // Comment represent a record comment. // // Signature is the client signature of Token+ParentID+Comment. diff --git a/politeiawww/client/comments.go b/politeiawww/client/comments.go new file mode 100644 index 000000000..6b9ad655f --- /dev/null +++ b/politeiawww/client/comments.go @@ -0,0 +1,34 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package client + +import ( + "encoding/json" + "fmt" + "net/http" + + cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" + "github.com/decred/politeia/util" +) + +// CommentsPolicy sends a pi v1 Policy request to politeiawww. +func (c *Client) CommentsPolicy() (*cmv1.PolicyReply, error) { + resBody, err := c.makeReq(http.MethodGet, + cmv1.APIRoute, cmv1.RoutePolicy, nil) + if err != nil { + return nil, err + } + + var pr cmv1.PolicyReply + err = json.Unmarshal(resBody, &pr) + if err != nil { + return nil, err + } + if c.verbose { + fmt.Printf("%v\n", util.FormatJSON(pr)) + } + + return &pr, nil +} diff --git a/politeiawww/cmd/pictl/commentcensor.go b/politeiawww/cmd/pictl/cmdcommentcensor.go similarity index 91% rename from politeiawww/cmd/pictl/commentcensor.go rename to politeiawww/cmd/pictl/cmdcommentcensor.go index 39668c6fb..f0b6f5d8c 100644 --- a/politeiawww/cmd/pictl/commentcensor.go +++ b/politeiawww/cmd/pictl/cmdcommentcensor.go @@ -4,8 +4,8 @@ package main -// commentCensorCmd censors a proposal comment. -type commentCensorCmd struct { +// cmdCommentCensor censors a proposal comment. +type cmdCommentCensor struct { Args struct { Token string `positional-arg-name:"token"` CommentID string `positional-arg-name:"commentid"` @@ -17,10 +17,10 @@ type commentCensorCmd struct { } /* -// Execute executes the commentCensorCmd command. +// Execute executes the cmdCommentCensor command. // // This function satisfies the go-flags Commander interface. -func (cmd *commentCensorCmd) Execute(args []string) error { +func (cmd *cmdCommentCensor) Execute(args []string) error { // Unpack args token := cmd.Args.Token reason := cmd.Args.Reason @@ -95,7 +95,7 @@ func (cmd *commentCensorCmd) Execute(args []string) error { } */ -// commentCensorHelpMsg is the help command message. +// commentCensorHelpMsg is printed to stdout by the help command. const commentCensorHelpMsg = `commentcensor "token" "commentID" "reason" Censor a user comment. This command assumes the record is a vetted record. If diff --git a/politeiawww/cmd/pictl/cmdcommentnew.go b/politeiawww/cmd/pictl/cmdcommentnew.go new file mode 100644 index 000000000..09eb36a16 --- /dev/null +++ b/politeiawww/cmd/pictl/cmdcommentnew.go @@ -0,0 +1,121 @@ +// Copyright (c) 2017-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +// cmdCommentNew submits a new proposal comment. +type cmdCommentNew struct { + Args struct { + Token string `positional-arg-name:"token" required:"true"` + Comment string `positional-arg-name:"comment" required:"true"` + ParentID string `positional-arg-name:"parentid"` + } `positional-args:"true"` + + // CLI flags + Unvetted bool `long:"unvetted" optional:"true"` +} + +// Execute executes the cmdCommentNew command. +// +// This function satisfies the go-flags Commander interface. +func (c *cmdCommentNew) Execute(args []string) error { + /* + // Unpack args + token := c.Args.Token + comment := c.Args.Comment + + var parentID uint64 + var err error + if c.Args.ParentID == "" { + parentID = 0 + } else { + parentID, err = strconv.ParseUint(c.Args.ParentID, 10, 32) + if err != nil { + return fmt.Errorf("ParseUint(%v): %v", c.Args.ParentID, err) + } + } + + // Verify identity + if cfg.Identity == nil { + return shared.ErrUserIdentityNotFound + } + + // Verify state. Defaults to vetted if the --unvetted flag + // is not used. + var state pi.PropStateT + switch { + case c.Unvetted: + state = pi.PropStateUnvetted + default: + state = pi.PropStateVetted + } + + // Sign comment data + msg := strconv.Itoa(int(state)) + token + + strconv.FormatUint(parentID, 10) + comment + b := cfg.Identity.SignMessage([]byte(msg)) + signature := hex.EncodeToString(b[:]) + + // Setup request + cn := pi.CommentNew{ + Token: token, + State: state, + ParentID: uint32(parentID), + Comment: comment, + Signature: signature, + PublicKey: cfg.Identity.Public.String(), + } + + // Send request. The request and response details are printed to + // the console. + err = shared.PrintJSON(cn) + if err != nil { + return err + } + cnr, err := client.CommentNew(cn) + if err != nil { + return err + } + err = shared.PrintJSON(cnr) + if err != nil { + return err + } + + // Verify receipt + vr, err := client.Version() + if err != nil { + return err + } + serverID, err := util.IdentityFromString(vr.PubKey) + if err != nil { + return err + } + receiptb, err := util.ConvertSignature(cnr.Comment.Receipt) + if err != nil { + return err + } + if !serverID.VerifyMessage([]byte(signature), receiptb) { + return fmt.Errorf("could not verify receipt") + } + */ + + return nil +} + +// commentNewHelpMsg is printed to stdout by the help command. +const commentNewHelpMsg = `commentnew "token" "comment" "parentid" + +Comment on a record as logged in user. This command assumes the record is a +vetted record. If the record is unvetted, the --unvetted flag must be used. +Requires admin priviledges. + +Arguments: +1. token (string, required) Proposal censorship token +2. comment (string, required) Comment +3. parentid (string, optional) ID of parent commment. Including a parent ID + indicates that the comment is a reply. + +Flags: + --unvetted (bool, optional) Comment on unvetted record. +` diff --git a/politeiawww/cmd/pictl/comments.go b/politeiawww/cmd/pictl/cmdcomments.go similarity index 80% rename from politeiawww/cmd/pictl/comments.go rename to politeiawww/cmd/pictl/cmdcomments.go index fdea24942..ef703b56c 100644 --- a/politeiawww/cmd/pictl/comments.go +++ b/politeiawww/cmd/pictl/cmdcomments.go @@ -4,8 +4,8 @@ package main -// commentsCmd retreives the comments for the specified proposal. -type commentsCmd struct { +// cmdComments retreives the comments for the specified proposal. +type cmdComments struct { Args struct { Token string `positional-arg-name:"token"` // Censorship token } `positional-args:"true" required:"true"` @@ -15,10 +15,10 @@ type commentsCmd struct { } /* -// Execute executes the commentsCmd command. +// Execute executes the cmdComments command. // // This function satisfies the go-flags Commander interface. -func (cmd *commentsCmd) Execute(args []string) error { +func (cmd *cmdComments) Execute(args []string) error { token := cmd.Args.Token // Verify state. Defaults to vetted if the --unvetted flag @@ -43,8 +43,7 @@ func (cmd *commentsCmd) Execute(args []string) error { } */ -// commentsHelpMsg is the output for the help command when 'comments' -// is specified. +// commentsHelpMsg is printed to stdout by the help command. const commentsHelpMsg = `comments "token" Get the comments for a record. This command assumes the record is a vetted diff --git a/politeiawww/cmd/pictl/cmdcommentspolicy.go b/politeiawww/cmd/pictl/cmdcommentspolicy.go new file mode 100644 index 000000000..219ec4fd8 --- /dev/null +++ b/politeiawww/cmd/pictl/cmdcommentspolicy.go @@ -0,0 +1,44 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + pclient "github.com/decred/politeia/politeiawww/client" +) + +// cmdCommentsPolicy retrieves the comments API policy. +type cmdCommentsPolicy struct{} + +// Execute executes the cmdCommentsPolicy command. +// +// This function satisfies the go-flags Commander interface. +func (c *cmdCommentsPolicy) Execute(args []string) error { + // Setup client + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, + } + pc, err := pclient.New(cfg.Host, opts) + if err != nil { + return err + } + + // Get policy + pr, err := pc.CommentsPolicy() + if err != nil { + return err + } + + // Print policy + printJSON(pr) + + return nil +} + +// commentsEditHelpMsg is the printed to stdout by the help command. +const commentsPolicyHelpMsg = `commentspolicy + +Fetch the comments API policy.` diff --git a/politeiawww/cmd/pictl/commenttimestamps.go b/politeiawww/cmd/pictl/cmdcommenttimestamps.go similarity index 90% rename from politeiawww/cmd/pictl/commenttimestamps.go rename to politeiawww/cmd/pictl/cmdcommenttimestamps.go index 823f0238a..da4c63c3c 100644 --- a/politeiawww/cmd/pictl/commenttimestamps.go +++ b/politeiawww/cmd/pictl/cmdcommenttimestamps.go @@ -4,8 +4,8 @@ package main -// commentTimestampsCmd retrieves the timestamps for politeiawww comments. -type commentTimestampsCmd struct { +// cmdCommentTimestamps retrieves the timestamps for politeiawww comments. +type cmdCommentTimestamps struct { Args struct { Token string `positional-arg-name:"token" required:"true"` CommentIDs []uint32 `positional-arg-name:"commentids" optional:"true"` @@ -17,10 +17,10 @@ type commentTimestampsCmd struct { } /* -// Execute executes the commentTimestampsCmd command. +// Execute executes the cmdCommentTimestamps command. // // This function satisfies the go-flags Commander interface. -func (c *commentTimestampsCmd) Execute(args []string) error { +func (c *cmdCommentTimestamps) Execute(args []string) error { // Set comment state. Defaults to vetted unless the unvetted flag // is used. @@ -97,6 +97,7 @@ func convertCommentTimestamp(t cmv1.Timestamp) backend.Timestamp { } */ +// commentTimestampsHelpMsg is printed to stdout by the help command const commentTimestampsHelpMsg = `commenttimestamps [flags] "token" commentIDs Fetch the timestamps for a record's comments. The timestamp contains all diff --git a/politeiawww/cmd/pictl/commentvote.go b/politeiawww/cmd/pictl/cmdcommentvote.go similarity index 91% rename from politeiawww/cmd/pictl/commentvote.go rename to politeiawww/cmd/pictl/cmdcommentvote.go index 117896ed1..ef15651ee 100644 --- a/politeiawww/cmd/pictl/commentvote.go +++ b/politeiawww/cmd/pictl/cmdcommentvote.go @@ -4,9 +4,9 @@ package main -// commentVoteCmd is used to upvote/downvote a proposal comment using the +// cmdCommentVote is used to upvote/downvote a proposal comment using the // logged in the user. -type commentVoteCmd struct { +type cmdCommentVote struct { Args struct { Token string `positional-arg-name:"token"` CommentID string `positional-arg-name:"commentID"` @@ -15,10 +15,10 @@ type commentVoteCmd struct { } /* -// Execute executes the commentVoteCmd command. +// Execute executes the cmdCommentVote command. // // This function satisfies the go-flags Commander interface. -func (c *commentVoteCmd) Execute(args []string) error { +func (c *cmdCommentVote) Execute(args []string) error { votes := map[string]pi.CommentVoteT{ "upvote": pi.CommentVoteUpvote, "downvote": pi.CommentVoteDownvote, @@ -95,7 +95,7 @@ func (c *commentVoteCmd) Execute(args []string) error { } */ -// commentVoteHelpMsg is the help command message. +// commentVoteHelpMsg is printed to stdout by the help command. const commentVoteHelpMsg = `commentvote "token" "commentID" "vote" Upvote or downvote a comment as the logged in user. diff --git a/politeiawww/cmd/pictl/commentvotes.go b/politeiawww/cmd/pictl/cmdcommentvotes.go similarity index 79% rename from politeiawww/cmd/pictl/commentvotes.go rename to politeiawww/cmd/pictl/cmdcommentvotes.go index c86d5f29f..016821210 100644 --- a/politeiawww/cmd/pictl/commentvotes.go +++ b/politeiawww/cmd/pictl/cmdcommentvotes.go @@ -4,9 +4,9 @@ package main -// commentVotesCmd retreives like comment objects for -// the specified proposal from the provided user. -type commentVotesCmd struct { +// cmdCommentVotes retrieves the comment upvotes/downvotes for a user on a +// record. +type cmdCommentVotes struct { Args struct { Token string `positional-arg-name:"token" required:"true"` UserID string `positional-arg-name:"userid"` @@ -15,10 +15,10 @@ type commentVotesCmd struct { } /* -// Execute executes the commentVotesCmd command. +// Execute executes the cmdCommentVotes command. // // This function satisfies the go-flags Commander interface. -func (c *commentVotesCmd) Execute(args []string) error { +func (c *cmdCommentVotes) Execute(args []string) error { token := c.Args.Token userID := c.Args.UserID @@ -48,8 +48,7 @@ func (c *commentVotesCmd) Execute(args []string) error { } */ -// commentVotesHelpMsg is the output for the help command when -// 'commentvotes' is specified. +// commentVotesHelpMsg is printed to stdout by the help command. const commentVotesHelpMsg = `commentvotes "token" "userid" Get the provided user comment upvote/downvotes for a proposal. diff --git a/politeiawww/cmd/pictl/cmdproposalinv.go b/politeiawww/cmd/pictl/cmdproposalinv.go index 5f616abb8..d4c1711af 100644 --- a/politeiawww/cmd/pictl/cmdproposalinv.go +++ b/politeiawww/cmd/pictl/cmdproposalinv.go @@ -42,7 +42,7 @@ func (c *cmdProposalInv) Execute(args []string) error { return nil } -// proposalInvHelpMsg is the command help message. +// proposalInvHelpMsg is printed to stdout by the help command. const proposalInvHelpMsg = `proposalinv Retrieve the censorship record tokens of all proposals in the inventory, diff --git a/politeiawww/cmd/pictl/cmdproposaltimestamps.go b/politeiawww/cmd/pictl/cmdproposaltimestamps.go index c38ff50cd..b0ae4bf4b 100644 --- a/politeiawww/cmd/pictl/cmdproposaltimestamps.go +++ b/politeiawww/cmd/pictl/cmdproposaltimestamps.go @@ -82,6 +82,9 @@ func (c *cmdProposalTimestamps) Execute(args []string) error { } } + // Print timestamps + printJSON(tr) + return nil } @@ -114,17 +117,20 @@ func convertTimestamp(t rcv1.Timestamp) backend.Timestamp { } } +// proposalTimestampsHelpMsg is printed to stdout by the help command. const proposalTimestampsHelpMsg = `proposaltimestamps [flags] "token" "version" Fetch the timestamps a proposal version. The timestamp contains all necessary data to verify that user submitted proposal data has been timestamped onto the decred blockchain. +This command defaults to requesting vetted proposals unless the --unvetted flag +is used. + Arguments: 1. token (string, required) Record token 2. version (string, optional) Record version Flags: - --unvetted (bool, optional) Request is for unvetted proposals instead of - vetted ones (default: false). + --unvetted (bool, optional) Request is for unvetted proposals. ` diff --git a/politeiawww/cmd/pictl/commentnew.go b/politeiawww/cmd/pictl/commentnew.go deleted file mode 100644 index abf16334c..000000000 --- a/politeiawww/cmd/pictl/commentnew.go +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright (c) 2017-2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -// commentNewCmd submits a new proposal comment. -type commentNewCmd struct { - Args struct { - Token string `positional-arg-name:"token" required:"true"` - Comment string `positional-arg-name:"comment" required:"true"` - ParentID string `positional-arg-name:"parentid"` - } `positional-args:"true"` - - // CLI flags - Unvetted bool `long:"unvetted" optional:"true"` -} - -/* -// Execute executes the commentNewCmd command. -// -// This function satisfies the go-flags Commander interface. -func (c *commentNewCmd) Execute(args []string) error { - // Unpack args - token := c.Args.Token - comment := c.Args.Comment - - var parentID uint64 - var err error - if c.Args.ParentID == "" { - parentID = 0 - } else { - parentID, err = strconv.ParseUint(c.Args.ParentID, 10, 32) - if err != nil { - return fmt.Errorf("ParseUint(%v): %v", c.Args.ParentID, err) - } - } - - // Verify identity - if cfg.Identity == nil { - return shared.ErrUserIdentityNotFound - } - - // Verify state. Defaults to vetted if the --unvetted flag - // is not used. - var state pi.PropStateT - switch { - case c.Unvetted: - state = pi.PropStateUnvetted - default: - state = pi.PropStateVetted - } - - // Sign comment data - msg := strconv.Itoa(int(state)) + token + - strconv.FormatUint(parentID, 10) + comment - b := cfg.Identity.SignMessage([]byte(msg)) - signature := hex.EncodeToString(b[:]) - - // Setup request - cn := pi.CommentNew{ - Token: token, - State: state, - ParentID: uint32(parentID), - Comment: comment, - Signature: signature, - PublicKey: cfg.Identity.Public.String(), - } - - // Send request. The request and response details are printed to - // the console. - err = shared.PrintJSON(cn) - if err != nil { - return err - } - cnr, err := client.CommentNew(cn) - if err != nil { - return err - } - err = shared.PrintJSON(cnr) - if err != nil { - return err - } - - // Verify receipt - vr, err := client.Version() - if err != nil { - return err - } - serverID, err := util.IdentityFromString(vr.PubKey) - if err != nil { - return err - } - receiptb, err := util.ConvertSignature(cnr.Comment.Receipt) - if err != nil { - return err - } - if !serverID.VerifyMessage([]byte(signature), receiptb) { - return fmt.Errorf("could not verify receipt") - } - - return nil -} -*/ - -// commentNewHelpMsg is the help command message. -const commentNewHelpMsg = `commentnew "token" "comment" "parentid" - -Comment on a record as logged in user. This command assumes the record is a -vetted record. If the record is unvetted, the --unvetted flag must be used. -Requires admin priviledges. - -Arguments: -1. token (string, required) Proposal censorship token -2. comment (string, required) Comment -3. parentid (string, optional) ID of parent commment. Including a parent ID - indicates that the comment is a reply. - -Flags: - --unvetted (bool, optional) Comment on unvetted record. -` diff --git a/politeiawww/cmd/pictl/pictl.go b/politeiawww/cmd/pictl/pictl.go index a72587618..112ba475a 100644 --- a/politeiawww/cmd/pictl/pictl.go +++ b/politeiawww/cmd/pictl/pictl.go @@ -74,12 +74,13 @@ type pictl struct { ProposalTimestamps cmdProposalTimestamps `command:"proposaltimestamps"` // Comments commands - CommentNew commentNewCmd `command:"commentnew"` - CommentVote commentVoteCmd `command:"commentvote"` - CommentCensor commentCensorCmd `command:"commentcensor"` - Comments commentsCmd `command:"comments"` - CommentVotes commentVotesCmd `command:"commentvotes"` - CommentTimestamps commentTimestampsCmd `command:"commenttimestamps"` + CommentsPolicy cmdCommentsPolicy `command:"commentspolicy"` + CommentNew cmdCommentNew `command:"commentnew"` + CommentVote cmdCommentVote `command:"commentvote"` + CommentCensor cmdCommentCensor `command:"commentcensor"` + Comments cmdComments `command:"comments"` + CommentVotes cmdCommentVotes `command:"commentvotes"` + CommentTimestamps cmdCommentTimestamps `command:"commenttimestamps"` // Vote commands VotePolicy votePolicyCmd `command:"votepolicy"` diff --git a/politeiawww/comments/comments.go b/politeiawww/comments/comments.go index 4d3f0aa77..b3876f0c5 100644 --- a/politeiawww/comments/comments.go +++ b/politeiawww/comments/comments.go @@ -6,9 +6,13 @@ package comments import ( "encoding/json" + "fmt" "net/http" + "strconv" + pdv1 "github.com/decred/politeia/politeiad/api/v1" pdclient "github.com/decred/politeia/politeiad/client" + "github.com/decred/politeia/politeiad/plugins/comments" v1 "github.com/decred/politeia/politeiawww/api/comments/v1" "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/events" @@ -24,6 +28,14 @@ type Comments struct { userdb user.Database sessions *sessions.Sessions events *events.Manager + policy *v1.PolicyReply +} + +// HandlePolicy is the request handler for the comments v1 Policy route. +func (c *Comments) HandlePolicy(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandlePolicy") + + util.RespondWithJSON(w, http.StatusOK, c.policy) } // HandleNew is the request handler for the comments v1 New route. @@ -236,12 +248,57 @@ func (c *Comments) HandleTimestamps(w http.ResponseWriter, r *http.Request) { } // New returns a new Comments context. -func New(cfg *config.Config, pdc *pdclient.Client, udb user.Database, s *sessions.Sessions, e *events.Manager) *Comments { +func New(cfg *config.Config, pdc *pdclient.Client, udb user.Database, s *sessions.Sessions, e *events.Manager, plugins []pdv1.Plugin) (*Comments, error) { + // Parse plugin settings + var ( + lengthMax uint32 + voteChangesMax uint32 + ) + for _, p := range plugins { + if p.ID != comments.PluginID { + // Not the comments plugin; skip + continue + } + for _, v := range p.Settings { + switch v.Key { + case comments.SettingKeyCommentLengthMax: + u, err := strconv.ParseUint(v.Value, 10, 64) + if err != nil { + return nil, err + } + lengthMax = uint32(u) + case comments.SettingKeyVoteChangesMax: + u, err := strconv.ParseUint(v.Value, 10, 64) + if err != nil { + return nil, err + } + voteChangesMax = uint32(u) + default: + // Skip unknown settings + log.Warnf("Unknown plugin setting %v; Skipping...", v.Key) + } + } + } + + // Verify all plugin settings have been provided + switch { + case lengthMax == 0: + return nil, fmt.Errorf("plugin setting not found: %v", + comments.SettingKeyCommentLengthMax) + case voteChangesMax == 0: + return nil, fmt.Errorf("plugin setting not found: %v", + comments.SettingKeyVoteChangesMax) + } + return &Comments{ cfg: cfg, politeiad: pdc, userdb: udb, sessions: s, events: e, - } + policy: &v1.PolicyReply{ + LengthMax: lengthMax, + VoteChangesMax: voteChangesMax, + }, + }, nil } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 67ce85137..be5e88704 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -90,6 +90,9 @@ func (p *politeiawww) setupPiRoutes(r *records.Records, c *comments.Comments, t permissionPublic) // Comment routes + p.addRoute(http.MethodGet, cmv1.APIRoute, + cmv1.RoutePolicy, c.HandlePolicy, + permissionPublic) p.addRoute(http.MethodPost, cmv1.APIRoute, cmv1.RouteNew, c.HandleNew, permissionLogin) @@ -178,15 +181,19 @@ func (p *politeiawww) setupPi(plugins []pdv1.Plugin) error { // Setup api contexts r := records.New(p.cfg, p.politeiad, p.db, p.sessions, p.events) - c := comments.New(p.cfg, p.politeiad, p.db, p.sessions, p.events) + c, err := comments.New(p.cfg, p.politeiad, p.db, + p.sessions, p.events, plugins) + if err != nil { + return fmt.Errorf("new comments api: %v", err) + } tv, err := ticketvote.New(p.cfg, p.politeiad, p.sessions, p.events, plugins) if err != nil { - return fmt.Errorf("new ticketvote: %v", err) + return fmt.Errorf("new ticketvote api: %v", err) } pic, err := pi.New(p.cfg, p.politeiad, p.db, p.sessions, plugins) if err != nil { - return fmt.Errorf("new pi: %v", err) + return fmt.Errorf("new pi api: %v", err) } // Setup routes From 939c21d1bed64cb1214092bb2f8265c726b3e90d Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 4 Feb 2021 22:14:07 -0600 Subject: [PATCH 272/449] Fix politeiad client and www API error handling. --- politeiad/api/v1/v1.go | 14 +- .../tlogbe/store/filesystem/filesystem.go | 4 + .../backend/tlogbe/tlog/trillianclient.go | 11 +- politeiad/backend/tlogbe/tlogbe.go | 16 ++ politeiad/client/comments.go | 44 +++-- politeiad/client/pi.go | 5 +- politeiad/client/politeiad.go | 23 +++ politeiad/client/ticketvote.go | 47 +++-- politeiad/client/user.go | 12 +- politeiad/log.go | 4 +- politeiad/politeiad.go | 96 +++------ politeiawww/api/comments/v1/v1.go | 6 + politeiawww/client/comments.go | 144 +++++++++++++- politeiawww/cmd/pictl/cmdcommentnew.go | 184 +++++++++--------- politeiawww/cmd/pictl/cmdcommentspolicy.go | 10 +- politeiawww/cmd/pictl/cmdcommentvote.go | 91 +++++---- politeiawww/cmd/pictl/cmdproposaledit.go | 5 +- politeiawww/cmd/pictl/pictl.go | 2 +- politeiawww/comments/error.go | 53 ++++- politeiawww/records/error.go | 2 +- 20 files changed, 500 insertions(+), 273 deletions(-) diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index 823d8ad25..cab451b27 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -117,6 +117,8 @@ var ( ErrorStatusRecordFound: "record found", ErrorStatusInvalidRPCCredentials: "invalid RPC client credentials", ErrorStatusInvalidToken: "invalid token", + ErrorStatusRecordLocked: "record locked", + ErrorStatusInvalidRecordState: "invalid record state", } // RecordStatus converts record status codes to human readable text. @@ -575,9 +577,15 @@ type PluginCommandReplyV2 struct { Command string `json:"command"` // Plugin command Payload string `json:"payload"` // Response payload - // Error will only be present if an error occured while executing - // the plugin command on a batched request. - Error error `json:"error,omitempty"` + // UserError will be populated if a ErrorStatusT is encountered + // before the plugin command could be executed. Ex, the provided + // token does not correspond to a record. + UserError *UserErrorReply `json:"usererror,omitempty"` + + // PluginError will be populated if a plugin error occured during + // plugin command execution. These errors will be specific to the + // plugin. + PluginError *PluginErrorReply `json:"pluginerror,omitempty"` } // PluginCommandBatch executes a batch of plugin commands. diff --git a/politeiad/backend/tlogbe/store/filesystem/filesystem.go b/politeiad/backend/tlogbe/store/filesystem/filesystem.go index 759476fd7..ed7401b78 100644 --- a/politeiad/backend/tlogbe/store/filesystem/filesystem.go +++ b/politeiad/backend/tlogbe/store/filesystem/filesystem.go @@ -91,6 +91,8 @@ func (f *fileSystem) Put(blobs [][]byte) ([]string, error) { keys = append(keys, key) } + log.Debugf("Saved blobs (%v) to store", len(blobs)) + return keys, nil } @@ -140,6 +142,8 @@ func (f *fileSystem) Del(keys []string) error { deleted = append(deleted, v) } + log.Debugf("Deleted blobs (%v) from store", len(keys)) + return nil } diff --git a/politeiad/backend/tlogbe/tlog/trillianclient.go b/politeiad/backend/tlogbe/tlog/trillianclient.go index 599cc0780..aaec3ac4e 100644 --- a/politeiad/backend/tlogbe/tlog/trillianclient.go +++ b/politeiad/backend/tlogbe/tlog/trillianclient.go @@ -325,7 +325,7 @@ func (t *tclient) signedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoot, * // // This function satisfies the trillianClient interface. func (t *tclient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, *types.LogRootV1, error) { - log.Tracef("trillian leavesAppend: %v", treeID) + log.Tracef("trillian leavesAppend: %v %v", treeID, len(leaves)) // Get the latest signed log root tree, err := t.tree(treeID) @@ -342,8 +342,6 @@ func (t *tclient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queu return nil, nil, fmt.Errorf("tree is frozen") } - log.Debugf("Appending %v leaves to tree id %v", len(leaves), treeID) - // Append leaves to log qlr, err := t.log.QueueLeaves(t.ctx, &trillian.QueueLeavesRequest{ LogId: treeID, @@ -369,7 +367,8 @@ func (t *tclient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queu } } - log.Debugf("Queued/Ignored leaves: %v/%v", len(leaves)-n, n) + log.Trace("Queued/Ignored leaves: %v/%v", len(leaves)-n, n) + log.Tracef("Waiting for inclusion of queued leaves...") var logRoot types.LogRootV1 err = logRoot.UnmarshalBinary(slr.LogRoot) @@ -389,6 +388,8 @@ func (t *tclient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queu } } + log.Debugf("Appended leaves (%v) to tree %v", len(leaves), treeID) + // Get the latest signed log root _, lr, err := t.signedLogRoot(tree) if err != nil { @@ -418,7 +419,7 @@ func (t *tclient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queu panic(e) } - // The LeafIndex of a QueuedLogLeaf will not be set yet. Get the + // The LeafIndex of a QueuedLogLeaf will not be set. Get the // inclusion proof by MerkleLeafHash. qlp.Proof, err = t.inclusionProof(treeID, v.Leaf.MerkleLeafHash, lr) if err != nil { diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 170805861..6a84e4715 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -1546,6 +1546,14 @@ func (t *tlogBackend) UnvettedPluginCmd(token []byte, pluginID, cmd, payload str } } + if len(token) > 0 { + log.Debugf("Unvetted '%v' plugin cmd '%v' on record %x", + pluginID, cmd, token) + } else { + log.Debugf("Unvetted '%v' plugin command '%v'", + pluginID, cmd) + } + // Call pre plugin hooks hp := plugins.HookPluginPre{ State: plugins.RecordStateUnvetted, @@ -1608,6 +1616,14 @@ func (t *tlogBackend) VettedPluginCmd(token []byte, pluginID, cmd, payload strin } } + if len(token) > 0 { + log.Debugf("Vetted '%v' plugin cmd '%v' on record %x", + pluginID, cmd, token) + } else { + log.Debugf("Vetted '%v' plugin command '%v'", + pluginID, cmd) + } + // Call pre plugin hooks hp := plugins.HookPluginPre{ State: plugins.RecordStateVetted, diff --git a/politeiad/client/comments.go b/politeiad/client/comments.go index 449f7d1e8..3fa860b9d 100644 --- a/politeiad/client/comments.go +++ b/politeiad/client/comments.go @@ -39,8 +39,9 @@ func (c *Client) CommentNew(ctx context.Context, state string, n comments.New) ( return nil, fmt.Errorf("no replies found") } pcr := replies[0] - if pcr.Error != nil { - return nil, pcr.Error + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err } // Decode reply @@ -79,18 +80,19 @@ func (c *Client) CommentVote(ctx context.Context, state string, v comments.Vote) return nil, fmt.Errorf("no replies found") } pcr := replies[0] - if pcr.Error != nil { - return nil, pcr.Error + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err } // Decode reply - var nr comments.VoteReply - err = json.Unmarshal([]byte(pcr.Payload), &nr) + var vr comments.VoteReply + err = json.Unmarshal([]byte(pcr.Payload), &vr) if err != nil { return nil, err } - return &nr, nil + return &vr, nil } // CommentDel sends the comments plugin Del command to the politeiad v1 API. @@ -118,12 +120,13 @@ func (c *Client) CommentDel(ctx context.Context, state string, d comments.Del) ( if len(replies) == 0 { return nil, fmt.Errorf("no replies found") } - - // Decode reply pcr := replies[0] - if pcr.Error != nil { - return nil, pcr.Error + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err } + + // Decode reply var dr comments.DelReply err = json.Unmarshal([]byte(pcr.Payload), &dr) if err != nil { @@ -163,9 +166,11 @@ func (c *Client) CommentCounts(ctx context.Context, state string, tokens []strin for _, v := range replies { // This command swallows individual errors. The token of the // command that errored will not be included in the reply. - if v.Error != nil { + err = extractPluginCommandError(v) + if err != nil { continue } + var cr comments.CountReply err = json.Unmarshal([]byte(v.Payload), cr) if err != nil { @@ -199,8 +204,9 @@ func (c *Client) CommentGetAll(ctx context.Context, state, token string) ([]comm return nil, fmt.Errorf("no replies found") } pcr := replies[0] - if pcr.Error != nil { - return nil, pcr.Error + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err } // Decode reply @@ -240,8 +246,9 @@ func (c *Client) CommentVotes(ctx context.Context, state, token string, v commen return nil, fmt.Errorf("no replies found") } pcr := replies[0] - if pcr.Error != nil { - return nil, pcr.Error + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err } // Decode reply @@ -281,8 +288,9 @@ func (c *Client) CommentTimestamps(ctx context.Context, state, token string, t c return nil, fmt.Errorf("no replies found") } pcr := replies[0] - if pcr.Error != nil { - return nil, pcr.Error + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err } // Decode reply diff --git a/politeiad/client/pi.go b/politeiad/client/pi.go index b9671ce3d..c82cfa4ee 100644 --- a/politeiad/client/pi.go +++ b/politeiad/client/pi.go @@ -35,8 +35,9 @@ func (c *Client) PiVoteInv(ctx context.Context) (*pi.VoteInventoryReply, error) return nil, fmt.Errorf("no replies found") } pcr := replies[0] - if pcr.Error != nil { - return nil, pcr.Error + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err } // Decode reply diff --git a/politeiad/client/politeiad.go b/politeiad/client/politeiad.go index c8c558743..7870c9ca3 100644 --- a/politeiad/client/politeiad.go +++ b/politeiad/client/politeiad.go @@ -540,3 +540,26 @@ func (c *Client) PluginInventory(ctx context.Context) ([]pdv1.Plugin, error) { return pir.Plugins, nil } + +func extractPluginCommandError(pcr pdv1.PluginCommandReplyV2) error { + switch { + case pcr.UserError != nil: + return Error{ + HTTPCode: http.StatusBadRequest, + ErrorReply: ErrorReply{ + ErrorCode: int(pcr.UserError.ErrorCode), + ErrorContext: pcr.UserError.ErrorContext, + }, + } + case pcr.PluginError != nil: + return Error{ + HTTPCode: http.StatusBadRequest, + ErrorReply: ErrorReply{ + PluginID: pcr.PluginError.PluginID, + ErrorCode: pcr.PluginError.ErrorCode, + ErrorContext: pcr.PluginError.ErrorContext, + }, + } + } + return nil +} diff --git a/politeiad/client/ticketvote.go b/politeiad/client/ticketvote.go index 263d60288..ac486d22a 100644 --- a/politeiad/client/ticketvote.go +++ b/politeiad/client/ticketvote.go @@ -40,8 +40,9 @@ func (c *Client) TicketVoteAuthorize(ctx context.Context, a ticketvote.Authorize return nil, fmt.Errorf("no replies found") } pcr := replies[0] - if pcr.Error != nil { - return nil, pcr.Error + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err } // Decode reply @@ -81,8 +82,9 @@ func (c *Client) TicketVoteStart(ctx context.Context, token string, s ticketvote return nil, fmt.Errorf("no replies found") } pcr := replies[0] - if pcr.Error != nil { - return nil, pcr.Error + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err } // Decode reply @@ -122,8 +124,9 @@ func (c *Client) TicketVoteCastBallot(ctx context.Context, token string, cb tick return nil, fmt.Errorf("no replies found") } pcr := replies[0] - if pcr.Error != nil { - return nil, pcr.Error + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err } // Decode reply @@ -159,8 +162,9 @@ func (c *Client) TicketVoteDetails(ctx context.Context, token string) (*ticketvo return nil, fmt.Errorf("no replies found") } pcr := replies[0] - if pcr.Error != nil { - return nil, pcr.Error + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err } // Decode reply @@ -196,8 +200,9 @@ func (c *Client) TicketVoteResults(ctx context.Context, token string) (*ticketvo return nil, fmt.Errorf("no replies found") } pcr := replies[0] - if pcr.Error != nil { - return nil, pcr.Error + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err } // Decode reply @@ -233,8 +238,9 @@ func (c *Client) TicketVoteSummary(ctx context.Context, token string) (*ticketvo return nil, fmt.Errorf("no replies found") } pcr := replies[0] - if pcr.Error != nil { - return nil, pcr.Error + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err } // Decode reply @@ -272,7 +278,8 @@ func (c *Client) TicketVoteSummaries(ctx context.Context, tokens []string) (map[ // Prepare reply summaries := make(map[string]ticketvote.SummaryReply, len(replies)) for _, v := range replies { - if v.Error != nil { + err = extractPluginCommandError(v) + if err != nil { // Individual summary errors are ignored. The token will not // be included in the returned summaries map. continue @@ -313,11 +320,13 @@ func (c *Client) TicketVoteLinkedFrom(ctx context.Context, tokens []string) (map // Prepare reply linkedFrom := make(map[string][]string, len(replies)) for _, v := range replies { - if v.Error != nil { + err = extractPluginCommandError(v) + if err != nil { // Individual record errors are ignored. The token will not be // included in the returned linkedFrom map. continue } + var lfr ticketvote.LinkedFromReply err = json.Unmarshal([]byte(v.Payload), &lfr) if err != nil { @@ -351,8 +360,9 @@ func (c *Client) TicketVoteInventory(ctx context.Context) (*ticketvote.Inventory return nil, fmt.Errorf("no replies found") } pcr := replies[0] - if pcr.Error != nil { - return nil, pcr.Error + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err } // Decode reply @@ -388,8 +398,9 @@ func (c *Client) TicketVoteTimestamps(ctx context.Context, token string) (*ticke return nil, fmt.Errorf("no replies found") } pcr := replies[0] - if pcr.Error != nil { - return nil, pcr.Error + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err } // Decode reply diff --git a/politeiad/client/user.go b/politeiad/client/user.go index cc93b5dd5..f94cb7f8d 100644 --- a/politeiad/client/user.go +++ b/politeiad/client/user.go @@ -31,15 +31,16 @@ func (c *Client) Author(ctx context.Context, state, token string) (string, error if err != nil { return "", err } - - // Decode reply if len(replies) == 0 { return "", fmt.Errorf("no replies found") } pcr := replies[0] - if pcr.Error != nil { - return "", pcr.Error + err = extractPluginCommandError(pcr) + if err != nil { + return "", err } + + // Decode reply var ar user.AuthorReply err = json.Unmarshal([]byte(pcr.Payload), &ar) if err != nil { @@ -88,7 +89,8 @@ func (c *Client) UserRecords(ctx context.Context, userID string) (map[string][]s // Decode replies reply := make(map[string][]string, 2) // [recordState][]token for _, v := range replies { - if v.Error != nil { + err = extractPluginCommandError(v) + if err != nil { // Swallow individual errors continue } diff --git a/politeiad/log.go b/politeiad/log.go index 1ec19ce22..9f4699f2c 100644 --- a/politeiad/log.go +++ b/politeiad/log.go @@ -53,7 +53,6 @@ var ( gitbeLog = backendLog.Logger("GITB") tlogbeLog = backendLog.Logger("BACK") tlogLog = backendLog.Logger("TLOG") - storeLog = backendLog.Logger("STOR") wsdcrdataLog = backendLog.Logger("WSDD") commentsLog = backendLog.Logger("COMT") dcrdataLog = backendLog.Logger("DCDA") @@ -66,7 +65,7 @@ func init() { gitbe.UseLogger(gitbeLog) tlogbe.UseLogger(tlogbeLog) tlog.UseLogger(tlogLog) - filesystem.UseLogger(storeLog) + filesystem.UseLogger(tlogLog) wsdcrdata.UseLogger(wsdcrdataLog) comments.UseLogger(commentsLog) dcrdata.UseLogger(dcrdataLog) @@ -80,7 +79,6 @@ var subsystemLoggers = map[string]slog.Logger{ "GITB": gitbeLog, "BACK": tlogbeLog, "TLOG": tlogLog, - "STOR": storeLog, "WSDD": wsdcrdataLog, "COMT": commentsLog, "DCDA": dcrdataLog, diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 591c4dfb3..361f86f7b 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -28,11 +28,7 @@ import ( "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/gitbe" "github.com/decred/politeia/politeiad/backend/tlogbe" - "github.com/decred/politeia/politeiad/plugins/comments" "github.com/decred/politeia/politeiad/plugins/dcrdata" - "github.com/decred/politeia/politeiad/plugins/pi" - "github.com/decred/politeia/politeiad/plugins/ticketvote" - "github.com/decred/politeia/politeiad/plugins/user" "github.com/decred/politeia/util" "github.com/decred/politeia/util/version" "github.com/gorilla/mux" @@ -1259,7 +1255,7 @@ func (p *politeia) pluginCommandBatch(w http.ResponseWriter, r *http.Request) { token, err := util.ConvertStringToken(pc.Token) if err != nil { replies[k] = v1.PluginCommandReplyV2{ - Error: v1.UserErrorReply{ + UserError: &v1.UserErrorReply{ ErrorCode: v1.ErrorStatusInvalidToken, }, } @@ -1277,7 +1273,7 @@ func (p *politeia) pluginCommandBatch(w http.ResponseWriter, r *http.Request) { pc.ID, pc.Command, pc.Payload) default: replies[k] = v1.PluginCommandReplyV2{ - Error: v1.UserErrorReply{ + UserError: &v1.UserErrorReply{ ErrorCode: v1.ErrorStatusInvalidRecordState, }, } @@ -1291,7 +1287,7 @@ func (p *politeia) pluginCommandBatch(w http.ResponseWriter, r *http.Request) { remoteAddr(r), e.PluginID, e.ErrorCode) replies[k] = v1.PluginCommandReplyV2{ - Error: v1.PluginErrorReply{ + PluginError: &v1.PluginErrorReply{ PluginID: e.PluginID, ErrorCode: e.ErrorCode, ErrorContext: []string{e.ErrorContext}, @@ -1299,28 +1295,26 @@ func (p *politeia) pluginCommandBatch(w http.ResponseWriter, r *http.Request) { } case err == backend.ErrRecordNotFound: replies[k] = v1.PluginCommandReplyV2{ - Error: v1.UserErrorReply{ + UserError: &v1.UserErrorReply{ ErrorCode: v1.ErrorStatusRecordNotFound, }, } case err == backend.ErrRecordLocked: replies[k] = v1.PluginCommandReplyV2{ - Error: v1.UserErrorReply{ + UserError: &v1.UserErrorReply{ ErrorCode: v1.ErrorStatusRecordLocked, }, } default: - // Unkown error. Log is as an internal server error. + // Unkown error. Log is as an internal server error and + // respond with a server error. t := time.Now().Unix() log.Errorf("%v %v: batched plugin cmd failed: pluginID:%v "+ "cmd:%v payload:%v err:%v", remoteAddr(r), t, pc.ID, pc.Command, pc.Payload, err) - replies[k] = v1.PluginCommandReplyV2{ - Error: v1.ServerErrorReply{ - ErrorCode: t, - }, - } + p.respondWithServerError(w, t) + return } continue @@ -1335,12 +1329,13 @@ func (p *politeia) pluginCommandBatch(w http.ResponseWriter, r *http.Request) { // Fill in remaining data for the replies for k, v := range replies { replies[k] = v1.PluginCommandReplyV2{ - State: pcb.Commands[k].State, - Token: pcb.Commands[k].Token, - ID: pcb.Commands[k].ID, - Command: pcb.Commands[k].Command, - Payload: v.Payload, - Error: v.Error, + State: pcb.Commands[k].State, + Token: pcb.Commands[k].Token, + ID: pcb.Commands[k].ID, + Command: pcb.Commands[k].Command, + Payload: v.Payload, + UserError: v.UserError, + PluginError: v.PluginError, } } @@ -1540,6 +1535,8 @@ func _main() error { // Set plugin routes. Requires auth. p.addRoute(http.MethodPost, v1.PluginCommandRoute, p.pluginCommand, permissionAuth) + p.addRoute(http.MethodPost, v1.PluginCommandBatchRoute, + p.pluginCommandBatch, permissionAuth) p.addRoute(http.MethodPost, v1.PluginInventoryRoute, p.pluginInventory, permissionAuth) @@ -1572,9 +1569,10 @@ func _main() error { // Register plugins for _, v := range cfg.Plugins { - // Verify plugin ID is lowercase + // Verify plugin ID format if backend.PluginRE.FindString(v) != v { - return fmt.Errorf("invalid plugin id: %v", v) + return fmt.Errorf("invalid plugin id format: %v %v", + v, backend.PluginRE.String()) } // Get plugin settings @@ -1583,55 +1581,23 @@ func _main() error { ps = make([]backend.PluginSetting, 0) } - // Setup plugin + // Prepare plugin var ( - unvetted bool // Register as unvetted plugin - vetted bool // Register as vetted plugin - ) - var plugin backend.Plugin - switch v { - case comments.PluginID: - plugin = backend.Plugin{ - ID: comments.PluginID, + unvetted = true // Register as unvetted plugin + vetted = true // Register as vetted plugin + plugin = backend.Plugin{ + ID: v, Settings: ps, Identity: p.identity, } - unvetted = true - vetted = true + ) + switch v { case dcrdata.PluginID: - plugin = backend.Plugin{ - ID: dcrdata.PluginID, - Settings: ps, - } - vetted = true - case pi.PluginID: - plugin = backend.Plugin{ - ID: pi.PluginID, - Settings: ps, - } - unvetted = true - vetted = true - case ticketvote.PluginID: - plugin = backend.Plugin{ - ID: ticketvote.PluginID, - Settings: ps, - Identity: p.identity, - } - unvetted = true - vetted = true - case user.PluginID: - plugin = backend.Plugin{ - ID: user.PluginID, - Settings: ps, - } - unvetted = true - vetted = true + unvetted = false case decredplugin.ID: - // TODO plugin setup for cms + // TODO decredplugin setup for cms case cmsplugin.ID: - // TODO plugin setup for cms - default: - return fmt.Errorf("unknown plugin provided by config '%v'", v) + // TODO cmsplugin setup for cms } // Register plugin diff --git a/politeiawww/api/comments/v1/v1.go b/politeiawww/api/comments/v1/v1.go index da332ff12..2db18b0a0 100644 --- a/politeiawww/api/comments/v1/v1.go +++ b/politeiawww/api/comments/v1/v1.go @@ -35,6 +35,9 @@ const ( ErrorCodePublicKeyInvalid ErrorCodeT = 3 ErrorCodeSignatureInvalid ErrorCodeT = 4 ErrorCodeRecordStateInvalid ErrorCodeT = 5 + ErrorCodeTokenInvalid ErrorCodeT = 6 + ErrorCodeRecordNotFound ErrorCodeT = 7 + ErrorCodeRecordLocked ErrorCodeT = 8 ) var ( @@ -46,6 +49,9 @@ var ( ErrorCodePublicKeyInvalid: "public key invalid", ErrorCodeSignatureInvalid: "signature invalid", ErrorCodeRecordStateInvalid: "record state invalid", + ErrorCodeTokenInvalid: "token invalid", + ErrorCodeRecordNotFound: "record not found", + ErrorCodeRecordLocked: "record is locked", } ) diff --git a/politeiawww/client/comments.go b/politeiawww/client/comments.go index 6b9ad655f..233015499 100644 --- a/politeiawww/client/comments.go +++ b/politeiawww/client/comments.go @@ -13,8 +13,8 @@ import ( "github.com/decred/politeia/util" ) -// CommentsPolicy sends a pi v1 Policy request to politeiawww. -func (c *Client) CommentsPolicy() (*cmv1.PolicyReply, error) { +// CommentPolicy sends a pi v1 Policy request to politeiawww. +func (c *Client) CommentPolicy() (*cmv1.PolicyReply, error) { resBody, err := c.makeReq(http.MethodGet, cmv1.APIRoute, cmv1.RoutePolicy, nil) if err != nil { @@ -32,3 +32,143 @@ func (c *Client) CommentsPolicy() (*cmv1.PolicyReply, error) { return &pr, nil } + +// CommentNew sends a comments v1 New request to politeiawww. +func (c *Client) CommentNew(n cmv1.New) (*cmv1.NewReply, error) { + resBody, err := c.makeReq(http.MethodPost, + cmv1.APIRoute, cmv1.RouteNew, n) + if err != nil { + return nil, err + } + + var nr cmv1.NewReply + err = json.Unmarshal(resBody, &nr) + if err != nil { + return nil, err + } + if c.verbose { + fmt.Printf("%v\n", util.FormatJSON(nr)) + } + + return &nr, nil +} + +// CommentVote sends a comments v1 Vote request to politeiawww. +func (c *Client) CommentVote(v cmv1.Vote) (*cmv1.VoteReply, error) { + resBody, err := c.makeReq(http.MethodPost, + cmv1.APIRoute, cmv1.RouteVote, v) + if err != nil { + return nil, err + } + + var vr cmv1.VoteReply + err = json.Unmarshal(resBody, &vr) + if err != nil { + return nil, err + } + if c.verbose { + fmt.Printf("%v\n", util.FormatJSON(vr)) + } + + return &vr, nil +} + +// CommentDel sends a comments v1 Del request to politeiawww. +func (c *Client) CommentDel(d cmv1.Del) (*cmv1.DelReply, error) { + resBody, err := c.makeReq(http.MethodPost, + cmv1.APIRoute, cmv1.RouteDel, d) + if err != nil { + return nil, err + } + + var dr cmv1.DelReply + err = json.Unmarshal(resBody, &dr) + if err != nil { + return nil, err + } + if c.verbose { + fmt.Printf("%v\n", util.FormatJSON(dr)) + } + + return &dr, nil +} + +// CommentCount sends a comments v1 Count request to politeiawww. +func (c *Client) CommentCount(cn cmv1.Count) (*cmv1.CountReply, error) { + resBody, err := c.makeReq(http.MethodPost, + cmv1.APIRoute, cmv1.RouteCount, cn) + if err != nil { + return nil, err + } + + var cr cmv1.CountReply + err = json.Unmarshal(resBody, &cr) + if err != nil { + return nil, err + } + if c.verbose { + fmt.Printf("%v\n", util.FormatJSON(cr)) + } + + return &cr, nil +} + +// Comments sends a comments v1 Comments request to politeiawww. +func (c *Client) Comments(cm cmv1.Comments) (*cmv1.CommentsReply, error) { + resBody, err := c.makeReq(http.MethodPost, + cmv1.APIRoute, cmv1.RouteComments, cm) + if err != nil { + return nil, err + } + + var cr cmv1.CommentsReply + err = json.Unmarshal(resBody, &cr) + if err != nil { + return nil, err + } + if c.verbose { + fmt.Printf("%v\n", util.FormatJSON(cr)) + } + + return &cr, nil +} + +// CommentVotes sends a comments v1 Votes request to politeiawww. +func (c *Client) CommentVotes(v cmv1.Votes) (*cmv1.VotesReply, error) { + resBody, err := c.makeReq(http.MethodPost, + cmv1.APIRoute, cmv1.RouteVotes, v) + if err != nil { + return nil, err + } + + var vr cmv1.VotesReply + err = json.Unmarshal(resBody, &vr) + if err != nil { + return nil, err + } + if c.verbose { + fmt.Printf("%v\n", util.FormatJSON(vr)) + } + + return &vr, nil +} + +// CommentTimestamps sends a comments v1 Timestamps request to politeiawww. +func (c *Client) CommentTimestamps(t cmv1.Timestamps) (*cmv1.TimestampsReply, error) { + resBody, err := c.makeReq(http.MethodPost, + cmv1.APIRoute, cmv1.RouteTimestamps, t) + if err != nil { + return nil, err + } + + var tr cmv1.TimestampsReply + err = json.Unmarshal(resBody, &tr) + if err != nil { + return nil, err + } + if c.verbose { + fmt.Printf("%v\n", util.FormatJSON(tr)) + } + + return &tr, nil +} diff --git a/politeiawww/cmd/pictl/cmdcommentnew.go b/politeiawww/cmd/pictl/cmdcommentnew.go index 09eb36a16..4c162c3b7 100644 --- a/politeiawww/cmd/pictl/cmdcommentnew.go +++ b/politeiawww/cmd/pictl/cmdcommentnew.go @@ -4,15 +4,27 @@ package main -// cmdCommentNew submits a new proposal comment. +import ( + "encoding/hex" + "fmt" + "strconv" + + cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" + pclient "github.com/decred/politeia/politeiawww/client" + "github.com/decred/politeia/politeiawww/cmd/shared" + "github.com/decred/politeia/util" +) + +// cmdCommentNew submits a new comment. type cmdCommentNew struct { Args struct { Token string `positional-arg-name:"token" required:"true"` Comment string `positional-arg-name:"comment" required:"true"` - ParentID string `positional-arg-name:"parentid"` + ParentID uint32 `positional-arg-name:"parentid"` } `positional-args:"true"` - // CLI flags + // Unvetted is used to comment on an unvetted record. If this flag + // is not used the command assumes the record is vetted. Unvetted bool `long:"unvetted" optional:"true"` } @@ -20,101 +32,93 @@ type cmdCommentNew struct { // // This function satisfies the go-flags Commander interface. func (c *cmdCommentNew) Execute(args []string) error { - /* - // Unpack args - token := c.Args.Token - comment := c.Args.Comment - - var parentID uint64 - var err error - if c.Args.ParentID == "" { - parentID = 0 - } else { - parentID, err = strconv.ParseUint(c.Args.ParentID, 10, 32) - if err != nil { - return fmt.Errorf("ParseUint(%v): %v", c.Args.ParentID, err) - } - } - - // Verify identity - if cfg.Identity == nil { - return shared.ErrUserIdentityNotFound - } - - // Verify state. Defaults to vetted if the --unvetted flag - // is not used. - var state pi.PropStateT - switch { - case c.Unvetted: - state = pi.PropStateUnvetted - default: - state = pi.PropStateVetted - } - - // Sign comment data - msg := strconv.Itoa(int(state)) + token + - strconv.FormatUint(parentID, 10) + comment - b := cfg.Identity.SignMessage([]byte(msg)) - signature := hex.EncodeToString(b[:]) - - // Setup request - cn := pi.CommentNew{ - Token: token, - State: state, - ParentID: uint32(parentID), - Comment: comment, - Signature: signature, - PublicKey: cfg.Identity.Public.String(), - } - - // Send request. The request and response details are printed to - // the console. - err = shared.PrintJSON(cn) - if err != nil { - return err - } - cnr, err := client.CommentNew(cn) - if err != nil { - return err - } - err = shared.PrintJSON(cnr) - if err != nil { - return err - } - - // Verify receipt - vr, err := client.Version() - if err != nil { - return err - } - serverID, err := util.IdentityFromString(vr.PubKey) - if err != nil { - return err - } - receiptb, err := util.ConvertSignature(cnr.Comment.Receipt) - if err != nil { - return err - } - if !serverID.VerifyMessage([]byte(signature), receiptb) { - return fmt.Errorf("could not verify receipt") - } - */ + // Unpack args + var ( + token = c.Args.Token + comment = c.Args.Comment + parentID = c.Args.ParentID + ) + + // Check for user identity. A user identity is required to sign + // the comment. + if cfg.Identity == nil { + return shared.ErrUserIdentityNotFound + } + + // Setup client + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Cookies: cfg.Cookies, + HeaderCSRF: cfg.CSRF, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, + } + pc, err := pclient.New(cfg.Host, opts) + if err != nil { + return err + } + + // Setup state + var state string + switch { + case c.Unvetted: + state = cmv1.RecordStateUnvetted + default: + state = cmv1.RecordStateVetted + } + + // Setup request + msg := token + strconv.FormatUint(uint64(parentID), 10) + comment + sig := cfg.Identity.SignMessage([]byte(msg)) + n := cmv1.New{ + Token: token, + State: state, + ParentID: parentID, + Comment: comment, + Signature: hex.EncodeToString(sig[:]), + PublicKey: cfg.Identity.Public.String(), + } + + // Send request + nr, err := pc.CommentNew(n) + if err != nil { + return err + } + + // Verify receipt + vr, err := client.Version() + if err != nil { + return err + } + serverID, err := util.IdentityFromString(vr.PubKey) + if err != nil { + return err + } + receiptb, err := util.ConvertSignature(nr.Comment.Receipt) + if err != nil { + return err + } + if !serverID.VerifyMessage([]byte(n.Signature), receiptb) { + return fmt.Errorf("could not verify receipt") + } return nil } // commentNewHelpMsg is printed to stdout by the help command. -const commentNewHelpMsg = `commentnew "token" "comment" "parentid" +const commentNewHelpMsg = `commentnew "token" "comment" parentid -Comment on a record as logged in user. This command assumes the record is a -vetted record. If the record is unvetted, the --unvetted flag must be used. -Requires admin priviledges. +Comment on a record. Requires the user to be logged in. + +This command assumes the record is a vetted record. If the record is unvetted, +the --unvetted flag must be used. Commenting on unvetted records requires admin +priviledges. Arguments: -1. token (string, required) Proposal censorship token -2. comment (string, required) Comment -3. parentid (string, optional) ID of parent commment. Including a parent ID - indicates that the comment is a reply. +1. token (string, required) Proposal censorship token. +2. comment (string, required) Comment text. +3. parentid (uint32, optional) ID of parent commment. Including a parent ID + indicates that the comment is a reply. Flags: --unvetted (bool, optional) Comment on unvetted record. diff --git a/politeiawww/cmd/pictl/cmdcommentspolicy.go b/politeiawww/cmd/pictl/cmdcommentspolicy.go index 219ec4fd8..375b7936c 100644 --- a/politeiawww/cmd/pictl/cmdcommentspolicy.go +++ b/politeiawww/cmd/pictl/cmdcommentspolicy.go @@ -8,13 +8,13 @@ import ( pclient "github.com/decred/politeia/politeiawww/client" ) -// cmdCommentsPolicy retrieves the comments API policy. -type cmdCommentsPolicy struct{} +// cmdCommentPolicy retrieves the comments API policy. +type cmdCommentPolicy struct{} -// Execute executes the cmdCommentsPolicy command. +// Execute executes the cmdCommentPolicy command. // // This function satisfies the go-flags Commander interface. -func (c *cmdCommentsPolicy) Execute(args []string) error { +func (c *cmdCommentPolicy) Execute(args []string) error { // Setup client opts := pclient.Opts{ HTTPSCert: cfg.HTTPSCert, @@ -27,7 +27,7 @@ func (c *cmdCommentsPolicy) Execute(args []string) error { } // Get policy - pr, err := pc.CommentsPolicy() + pr, err := pc.CommentPolicy() if err != nil { return err } diff --git a/politeiawww/cmd/pictl/cmdcommentvote.go b/politeiawww/cmd/pictl/cmdcommentvote.go index ef15651ee..f742fcf91 100644 --- a/politeiawww/cmd/pictl/cmdcommentvote.go +++ b/politeiawww/cmd/pictl/cmdcommentvote.go @@ -4,33 +4,56 @@ package main +import ( + "encoding/hex" + "fmt" + "strconv" + + cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" + pclient "github.com/decred/politeia/politeiawww/client" + "github.com/decred/politeia/politeiawww/cmd/shared" + "github.com/decred/politeia/util" +) + // cmdCommentVote is used to upvote/downvote a proposal comment using the // logged in the user. type cmdCommentVote struct { Args struct { Token string `positional-arg-name:"token"` - CommentID string `positional-arg-name:"commentID"` + CommentID uint32 `positional-arg-name:"commentID"` Vote string `positional-arg-name:"vote"` } `positional-args:"true" required:"true"` } -/* // Execute executes the cmdCommentVote command. // // This function satisfies the go-flags Commander interface. func (c *cmdCommentVote) Execute(args []string) error { - votes := map[string]pi.CommentVoteT{ - "upvote": pi.CommentVoteUpvote, - "downvote": pi.CommentVoteDownvote, - "1": pi.CommentVoteUpvote, - "-1": pi.CommentVoteDownvote, + // Check for user identity. A user identity is required to sign + // the comment vote. + if cfg.Identity == nil { + return shared.ErrUserIdentityNotFound } - // Unpack args - token := c.Args.Token - commentID, err := strconv.ParseUint(c.Args.CommentID, 10, 32) + // Setup client + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Cookies: cfg.Cookies, + HeaderCSRF: cfg.CSRF, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, + } + pc, err := pclient.New(cfg.Host, opts) if err != nil { - return fmt.Errorf("ParseUint(%v): %v", c.Args.CommentID, err) + return err + } + + // Parse vote preference + votes := map[string]cmv1.VoteT{ + "upvote": cmv1.VoteUpvote, + "downvote": cmv1.VoteDownvote, + "1": cmv1.VoteUpvote, + "-1": cmv1.VoteDownvote, } vote, ok := votes[c.Args.Vote] if !ok { @@ -38,38 +61,21 @@ func (c *cmdCommentVote) Execute(args []string) error { c.Args.Vote, commentVoteHelpMsg) } - // Verify identity - if cfg.Identity == nil { - return shared.ErrUserIdentityNotFound - } - - // Sign vote choice - msg := strconv.Itoa(int(pi.PropStateVetted)) + token + - c.Args.CommentID + strconv.FormatInt(int64(vote), 10) - b := cfg.Identity.SignMessage([]byte(msg)) - signature := hex.EncodeToString(b[:]) - // Setup request - cv := pi.CommentVote{ - Token: token, - State: pi.PropStateVetted, - CommentID: uint32(commentID), + msg := c.Args.Token + strconv.FormatUint(uint64(c.Args.CommentID), 10) + + strconv.FormatInt(int64(vote), 10) + sig := cfg.Identity.SignMessage([]byte(msg)) + v := cmv1.Vote{ + Token: c.Args.Token, + State: cmv1.RecordStateVetted, + CommentID: c.Args.CommentID, Vote: vote, - Signature: signature, + Signature: hex.EncodeToString(sig[:]), PublicKey: cfg.Identity.Public.String(), } - // Send request. The request and response details are printed to - // the console. - err = shared.PrintJSON(cv) - if err != nil { - return err - } - cvr, err := client.CommentVote(cv) - if err != nil { - return err - } - err = shared.PrintJSON(cvr) + // Send request + cvr, err := pc.CommentVote(v) if err != nil { return err } @@ -87,18 +93,17 @@ func (c *cmdCommentVote) Execute(args []string) error { if err != nil { return err } - if !serverID.VerifyMessage([]byte(signature), receiptb) { + if !serverID.VerifyMessage([]byte(v.Signature), receiptb) { return fmt.Errorf("could not verify receipt") } return nil } -*/ // commentVoteHelpMsg is printed to stdout by the help command. const commentVoteHelpMsg = `commentvote "token" "commentID" "vote" -Upvote or downvote a comment as the logged in user. +Upvote or downvote a comment. Requires the user to be logged in. Arguments: 1. token (string, required) Proposal censorship token @@ -111,6 +116,6 @@ upvote (1) downvote (-1) Example usage -$ commentvote d594fbadef0f93780000 3 downvote -$ commentvote d594fbadef0f93780000 3 -1 +$ commentvote d594fbadef0f9378 3 downvote +$ commentvote d594fbadef0f9378 3 -1 ` diff --git a/politeiawww/cmd/pictl/cmdproposaledit.go b/politeiawww/cmd/pictl/cmdproposaledit.go index c7921e024..4abae36ce 100644 --- a/politeiawww/cmd/pictl/cmdproposaledit.go +++ b/politeiawww/cmd/pictl/cmdproposaledit.go @@ -30,9 +30,8 @@ type cmdProposalEdit struct { Attachments []string `positional-arg-name:"attachmets"` } `positional-args:"true" optional:"true"` - // Unvetted is used to indicate the state of the proposal is - // unvetted. If this flag is not used it will be assumed that - // the proposal is vetted. + // Unvetted is used to edit an unvetted proposal. If this flag is + // not used the command assumes the proposal is vetted. Unvetted bool `long:"unvetted" optional:"true"` // UseMD is a flag that is intended to make editing proposal diff --git a/politeiawww/cmd/pictl/pictl.go b/politeiawww/cmd/pictl/pictl.go index 112ba475a..f074b1c03 100644 --- a/politeiawww/cmd/pictl/pictl.go +++ b/politeiawww/cmd/pictl/pictl.go @@ -74,7 +74,7 @@ type pictl struct { ProposalTimestamps cmdProposalTimestamps `command:"proposaltimestamps"` // Comments commands - CommentsPolicy cmdCommentsPolicy `command:"commentspolicy"` + CommentsPolicy cmdCommentPolicy `command:"commentpolicy"` CommentNew cmdCommentNew `command:"commentnew"` CommentVote cmdCommentVote `command:"commentvote"` CommentCensor cmdCommentCensor `command:"commentcensor"` diff --git a/politeiawww/comments/error.go b/politeiawww/comments/error.go index e6608f6bf..6530a868f 100644 --- a/politeiawww/comments/error.go +++ b/politeiawww/comments/error.go @@ -12,6 +12,7 @@ import ( "strings" "time" + pdv1 "github.com/decred/politeia/politeiad/api/v1" pdclient "github.com/decred/politeia/politeiad/client" v1 "github.com/decred/politeia/politeiawww/api/comments/v1" "github.com/decred/politeia/util" @@ -25,7 +26,7 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err switch { case errors.As(err, &ue): // Comments user error - m := fmt.Sprintf("Comments user error: %v %v %v", + m := fmt.Sprintf("%v Records user error: %v %v", util.RemoteAddr(r), ue.ErrorCode, v1.ErrorCodes[ue.ErrorCode]) if ue.ErrorContext != "" { m += fmt.Sprintf(": %v", ue.ErrorContext) @@ -43,27 +44,29 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err var ( pluginID = pe.ErrorReply.PluginID errCode = pe.ErrorReply.ErrorCode - errContext = pe.ErrorReply.ErrorContext + errContext = strings.Join(pe.ErrorReply.ErrorContext, ",") ) + e := convertPDErrorCode(errCode) switch { case pluginID != "": - // Politeiad plugin error. Log it and return a 400. - m := fmt.Sprintf("Plugin error: %v %v %v", + // politeiad plugin error. Log it and return a 400. + m := fmt.Sprintf("%v Plugin error: %v %v", util.RemoteAddr(r), pluginID, errCode) - if len(errContext) > 0 { - m += fmt.Sprintf(": %v", strings.Join(errContext, ", ")) + if errContext != "" { + m += fmt.Sprintf(": %v", errContext) } log.Infof(m) util.RespondWithJSON(w, http.StatusBadRequest, v1.PluginErrorReply{ PluginID: pluginID, ErrorCode: errCode, - ErrorContext: strings.Join(errContext, ", "), + ErrorContext: errContext, }) return - default: - // Unknown politeiad error. Log it and return a 500. + case e == v1.ErrorCodeInvalid: + // politeiad error does not correspond to a user error. Log it + // and return a 500. ts := time.Now().Unix() log.Errorf("%v %v %v %v Internal error %v: error code "+ "from politeiad: %v", util.RemoteAddr(r), r.Method, r.URL, @@ -74,6 +77,22 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err ErrorCode: ts, }) return + + default: + // User error from politeiad that corresponds to a comments + // user error. Log it and return a 400. + m := fmt.Sprintf("%v Records user error: %v %v", + util.RemoteAddr(r), e, v1.ErrorCodes[e]) + if errContext != "" { + m += fmt.Sprintf(": %v", errContext) + } + log.Infof(m) + util.RespondWithJSON(w, http.StatusBadRequest, + v1.UserErrorReply{ + ErrorCode: e, + ErrorContext: errContext, + }) + return } default: @@ -91,3 +110,19 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err return } } + +func convertPDErrorCode(errCode int) v1.ErrorCodeT { + // These are the only politeiad user errors that the comments + // API expects to encounter. + switch pdv1.ErrorStatusT(errCode) { + case pdv1.ErrorStatusInvalidToken: + return v1.ErrorCodeTokenInvalid + case pdv1.ErrorStatusInvalidRecordState: + return v1.ErrorCodeRecordStateInvalid + case pdv1.ErrorStatusRecordNotFound: + return v1.ErrorCodeRecordNotFound + case pdv1.ErrorStatusRecordLocked: + return v1.ErrorCodeRecordLocked + } + return v1.ErrorCodeInvalid +} diff --git a/politeiawww/records/error.go b/politeiawww/records/error.go index 2d37fd95c..483e2df23 100644 --- a/politeiawww/records/error.go +++ b/politeiawww/records/error.go @@ -25,7 +25,7 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err ) switch { case errors.As(err, &ue): - // Record user error from politeiawww + // Records user error m := fmt.Sprintf("%v Records user error: %v %v", util.RemoteAddr(r), ue.ErrorCode, v1.ErrorCodes[ue.ErrorCode]) if ue.ErrorContext != "" { From 02b55b2850ec51704da4f27e51f0d53f4393a9af Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 5 Feb 2021 08:39:49 -0600 Subject: [PATCH 273/449] Record route bug fixes. --- politeiad/client/politeiad.go | 2 + politeiawww/cmd/pictl/cmdcommentnew.go | 14 ++++-- politeiawww/cmd/pictl/cmdcomments.go | 66 +++++++++++++++++-------- politeiawww/cmd/pictl/cmdcommentvote.go | 2 + politeiawww/cmd/pictl/cmdproposalnew.go | 6 ++- politeiawww/cmd/pictl/proposal.go | 2 + politeiawww/records/process.go | 9 ++-- 7 files changed, 70 insertions(+), 31 deletions(-) diff --git a/politeiad/client/politeiad.go b/politeiad/client/politeiad.go index 7870c9ca3..e597ac501 100644 --- a/politeiad/client/politeiad.go +++ b/politeiad/client/politeiad.go @@ -541,6 +541,8 @@ func (c *Client) PluginInventory(ctx context.Context) ([]pdv1.Plugin, error) { return pir.Plugins, nil } +// extractPluginCommandError extracts the error from a plugin command reply if +// one exists and converts it to a politeiad client Error. func extractPluginCommandError(pcr pdv1.PluginCommandReplyV2) error { switch { case pcr.UserError != nil: diff --git a/politeiawww/cmd/pictl/cmdcommentnew.go b/politeiawww/cmd/pictl/cmdcommentnew.go index 4c162c3b7..8cb07c562 100644 --- a/politeiawww/cmd/pictl/cmdcommentnew.go +++ b/politeiawww/cmd/pictl/cmdcommentnew.go @@ -102,6 +102,11 @@ func (c *cmdCommentNew) Execute(args []string) error { return fmt.Errorf("could not verify receipt") } + // Print receipt + fmt.Printf("Comment Submitted\n") + fmt.Printf("ID : %v\n", nr.Comment.CommentID) + fmt.Printf("Receipt: %v\n", nr.Comment.Receipt) + return nil } @@ -110,9 +115,10 @@ const commentNewHelpMsg = `commentnew "token" "comment" parentid Comment on a record. Requires the user to be logged in. -This command assumes the record is a vetted record. If the record is unvetted, -the --unvetted flag must be used. Commenting on unvetted records requires admin -priviledges. +This command assumes the record is a vetted record. + +If the record is unvetted, the --unvetted flag must be used. Commenting on +unvetted records requires admin priviledges. Arguments: 1. token (string, required) Proposal censorship token. @@ -121,5 +127,5 @@ Arguments: indicates that the comment is a reply. Flags: - --unvetted (bool, optional) Comment on unvetted record. + --unvetted (bool, optional) Record is unvetted. ` diff --git a/politeiawww/cmd/pictl/cmdcomments.go b/politeiawww/cmd/pictl/cmdcomments.go index ef703b56c..4f5bb8c04 100644 --- a/politeiawww/cmd/pictl/cmdcomments.go +++ b/politeiawww/cmd/pictl/cmdcomments.go @@ -1,58 +1,82 @@ -// Copyright (c) 2017-2019 The Decred developers +// Copyright (c) 2017-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. package main +import ( + cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" + pclient "github.com/decred/politeia/politeiawww/client" +) + // cmdComments retreives the comments for the specified proposal. type cmdComments struct { Args struct { Token string `positional-arg-name:"token"` // Censorship token } `positional-args:"true" required:"true"` - // CLI flags + // Unvetted is used to request the comments of an unvetted record. + // If this flag is not used the command assumes the record is + // vetted. Unvetted bool `long:"unvetted" optional:"true"` } -/* // Execute executes the cmdComments command. // // This function satisfies the go-flags Commander interface. -func (cmd *cmdComments) Execute(args []string) error { - token := cmd.Args.Token +func (c *cmdComments) Execute(args []string) error { + // Setup client + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Cookies: cfg.Cookies, + HeaderCSRF: cfg.CSRF, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, + } + pc, err := pclient.New(cfg.Host, opts) + if err != nil { + return err + } - // Verify state. Defaults to vetted if the --unvetted flag - // is not used. - var state pi.PropStateT + // Setup state + var state string switch { - case cmd.Unvetted: - state = pi.PropStateUnvetted + case c.Unvetted: + state = cmv1.RecordStateUnvetted default: - state = pi.PropStateVetted + state = cmv1.RecordStateVetted } - gcr, err := client.Comments(pi.Comments{ - Token: token, + // Get comments + cm := cmv1.Comments{ State: state, - }) + Token: c.Args.Token, + } + cr, err := pc.Comments(cm) if err != nil { return err } - return shared.PrintJSON(gcr) + // Print comments + for _, v := range cr.Comments { + _ = v + } + + return nil } -*/ // commentsHelpMsg is printed to stdout by the help command. const commentsHelpMsg = `comments "token" -Get the comments for a record. This command assumes the record is a vetted -record. If the record is unvetted, the --unvetted flag must be used. Requires -admin priviledges. +Get the comments for a record. + +If the record is unvetted, the --unvetted flag must be used. Retrieving the +comments on an unvetted record requires the user be either an admin or the +record author. Arguments: -1. token (string, required) Proposal censorship token +1. token (string, required) Proposal censorship token Flags: - --unvetted (bool, optional) Comment on unvetted record. + --unvetted (bool, optional) Record is unvetted. ` diff --git a/politeiawww/cmd/pictl/cmdcommentvote.go b/politeiawww/cmd/pictl/cmdcommentvote.go index f742fcf91..2ebc03906 100644 --- a/politeiawww/cmd/pictl/cmdcommentvote.go +++ b/politeiawww/cmd/pictl/cmdcommentvote.go @@ -97,6 +97,8 @@ func (c *cmdCommentVote) Execute(args []string) error { return fmt.Errorf("could not verify receipt") } + // Print receipt + return nil } diff --git a/politeiawww/cmd/pictl/cmdproposalnew.go b/politeiawww/cmd/pictl/cmdproposalnew.go index 6103cd720..c8d512b78 100644 --- a/politeiawww/cmd/pictl/cmdproposalnew.go +++ b/politeiawww/cmd/pictl/cmdproposalnew.go @@ -212,8 +212,10 @@ func (c *cmdProposalNew) Execute(args []string) error { return fmt.Errorf("unable to verify record: %v", err) } - // Print token to stdout - printf("Token: %v\n", nr.Record.CensorshipRecord.Token) + // Print censorship record + printf("Token : %v\n", nr.Record.CensorshipRecord.Token) + printf("Merkle : %v\n", nr.Record.CensorshipRecord.Merkle) + printf("Receipt: %v\n", nr.Record.CensorshipRecord.Signature) return nil } diff --git a/politeiawww/cmd/pictl/proposal.go b/politeiawww/cmd/pictl/proposal.go index 0bbc88b50..3d45c524a 100644 --- a/politeiawww/cmd/pictl/proposal.go +++ b/politeiawww/cmd/pictl/proposal.go @@ -105,6 +105,8 @@ func printProposal(r rcv1.Record) error { printf("Status : %v\n", rcv1.RecordStatuses[r.Status]) printf("Timestamp: %v\n", r.Timestamp) printf("Username : %v\n", r.Username) + printf("Merkle : %v\n", r.CensorshipRecord.Merkle) + printf("Receipt : %v\n", r.CensorshipRecord.Signature) printf("Metadata\n") for _, v := range r.Metadata { size := byteCountSI(int64(len([]byte(v.Payload)))) diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index 798120505..32057929e 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -64,11 +64,10 @@ func (r *Records) processNew(ctx context.Context, n v1.New, u user.User) (*v1.Ne } // Get full record - pdr, err := r.politeiad.GetUnvetted(ctx, cr.Token, "") + rc, err := r.record(ctx, v1.RecordStateUnvetted, cr.Token, "") if err != nil { return nil, err } - rc := convertRecordToV1(*pdr, v1.RecordStateUnvetted) // Execute post plugin hooks. Checking the mode is a temporary // measure until user plugins have been properly implemented. @@ -84,7 +83,7 @@ func (r *Records) processNew(ctx context.Context, n v1.New, u user.User) (*v1.Ne r.events.Emit(EventTypeNew, EventNew{ User: u, - Record: rc, + Record: *rc, }) log.Infof("Record submitted: %v", rc.CensorshipRecord.Token) @@ -93,7 +92,7 @@ func (r *Records) processNew(ctx context.Context, n v1.New, u user.User) (*v1.Ne } return &v1.NewReply{ - Record: rc, + Record: *rc, }, nil } @@ -192,6 +191,7 @@ func (r *Records) processEdit(ctx context.Context, e v1.Edit, u user.User) (*v1. } rc := convertRecordToV1(*pdr, e.State) + recordPopulateUserData(&rc, u) // Emit event r.events.Emit(EventTypeEdit, @@ -269,6 +269,7 @@ func (r *Records) processSetStatus(ctx context.Context, ss v1.SetStatus, u user. } rc := convertRecordToV1(*pdr, ss.State) + recordPopulateUserData(&rc, u) // Emit event r.events.Emit(EventTypeSetStatus, From 591a0eae7e0d297d68d178c0f1a33deefa98f442 Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 5 Feb 2021 14:20:57 -0600 Subject: [PATCH 274/449] All comment commands and routes working. --- .../backend/tlogbe/plugins/comments/cmds.go | 22 +--- .../tlogbe/plugins/comments/comments.go | 14 +-- .../backend/tlogbe/plugins/dcrdata/dcrdata.go | 10 +- politeiad/backend/tlogbe/plugins/pi/pi.go | 10 +- .../tlogbe/plugins/ticketvote/ticketvote.go | 10 +- politeiad/backend/tlogbe/plugins/user/user.go | 10 +- politeiad/log.go | 28 ++--- politeiawww/api/comments/v1/v1.go | 5 +- politeiawww/api/pi/v1/v1.go | 1 + politeiawww/client/error.go | 13 ++- politeiawww/cmd/pictl/cmdcommentcensor.go | 106 ++++++++++-------- politeiawww/cmd/pictl/cmdcommentnew.go | 4 +- politeiawww/cmd/pictl/cmdcomments.go | 8 +- politeiawww/cmd/pictl/cmdcommenttimestamps.go | 60 +++++----- politeiawww/cmd/pictl/cmdcommentvote.go | 9 +- politeiawww/cmd/pictl/cmdcommentvotes.go | 73 ++++++++---- politeiawww/cmd/pictl/comment.go | 70 ++++++++++++ politeiawww/cmd/pictl/pictl.go | 3 +- politeiawww/cmd/pictl/proposal.go | 60 +++++----- politeiawww/cmd/pictl/time.go | 23 ++++ politeiawww/comments/process.go | 41 ++++--- 21 files changed, 365 insertions(+), 215 deletions(-) create mode 100644 politeiawww/cmd/pictl/comment.go create mode 100644 politeiawww/cmd/pictl/time.go diff --git a/politeiad/backend/tlogbe/plugins/comments/cmds.go b/politeiad/backend/tlogbe/plugins/comments/cmds.go index 25f58880d..2e050376e 100644 --- a/politeiad/backend/tlogbe/plugins/comments/cmds.go +++ b/politeiad/backend/tlogbe/plugins/comments/cmds.go @@ -1185,15 +1185,8 @@ func (p *commentsPlugin) cmdGet(treeID int64, token []byte, payload string) (str return string(reply), nil } -func (p *commentsPlugin) cmdGetAll(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdGetAll: %v %x %v", treeID, token, payload) - - // Decode payload - var ga comments.GetAll - err := json.Unmarshal([]byte(payload), &ga) - if err != nil { - return "", err - } +func (p *commentsPlugin) cmdGetAll(treeID int64, token []byte) (string, error) { + log.Tracef("cmdGetAll: %v %x", treeID, token) // Compile comment IDs ridx, err := p.recordIndex(token) @@ -1302,15 +1295,8 @@ func (p *commentsPlugin) cmdGetVersion(treeID int64, token []byte, payload strin return string(reply), nil } -func (p *commentsPlugin) cmdCount(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdCount: %v %x %v", treeID, token, payload) - - // Decode payload - var c comments.Count - err := json.Unmarshal([]byte(payload), &c) - if err != nil { - return "", err - } +func (p *commentsPlugin) cmdCount(treeID int64, token []byte) (string, error) { + log.Tracef("cmdCount: %v %x", treeID, token) // Get record index ridx, err := p.recordIndex(token) diff --git a/politeiad/backend/tlogbe/plugins/comments/comments.go b/politeiad/backend/tlogbe/plugins/comments/comments.go index 9cc710173..8e86d5b86 100644 --- a/politeiad/backend/tlogbe/plugins/comments/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments/comments.go @@ -72,7 +72,7 @@ func (p *commentsPlugin) mutex(token []byte) *sync.Mutex { // // This function satisfies the plugins.PluginClient interface. func (p *commentsPlugin) Setup() error { - log.Tracef("Setup") + log.Tracef("comments Setup") return nil } @@ -81,7 +81,7 @@ func (p *commentsPlugin) Setup() error { // // This function satisfies the plugins.PluginClient interface. func (p *commentsPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { - log.Tracef("Cmd: %v %x %v", treeID, token, cmd) + log.Tracef("comments Cmd: %v %x %v", treeID, token, cmd) switch cmd { case comments.CmdNew: @@ -95,11 +95,11 @@ func (p *commentsPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (s case comments.CmdGet: return p.cmdGet(treeID, token, payload) case comments.CmdGetAll: - return p.cmdGetAll(treeID, token, payload) + return p.cmdGetAll(treeID, token) case comments.CmdGetVersion: return p.cmdGetVersion(treeID, token, payload) case comments.CmdCount: - return p.cmdCount(treeID, token, payload) + return p.cmdCount(treeID, token) case comments.CmdVotes: return p.cmdVotes(treeID, token, payload) case comments.CmdTimestamps: @@ -113,7 +113,7 @@ func (p *commentsPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (s // // This function satisfies the plugins.PluginClient interface. func (p *commentsPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { - log.Tracef("Hook: %v %x %v", treeID, token, plugins.Hooks[h]) + log.Tracef("comments Hook: %v %x %v", treeID, token, plugins.Hooks[h]) return nil } @@ -122,7 +122,7 @@ func (p *commentsPlugin) Hook(treeID int64, token []byte, h plugins.HookT, paylo // // This function satisfies the plugins.PluginClient interface. func (p *commentsPlugin) Fsck(treeIDs []int64) error { - log.Tracef("Fsck") + log.Tracef("comments Fsck") // Verify CommentDel blobs were actually deleted @@ -133,7 +133,7 @@ func (p *commentsPlugin) Fsck(treeIDs []int64) error { // // This function satisfies the plugins.PluginClient interface. func (p *commentsPlugin) Settings() []backend.PluginSetting { - log.Tracef("Settings") + log.Tracef("comments Settings") return []backend.PluginSetting{ { diff --git a/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go b/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go index e97ec7e7c..0cb98d903 100644 --- a/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go +++ b/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go @@ -577,7 +577,7 @@ func (p *dcrdataPlugin) websocketSetup() { // // This function satisfies the plugins.PluginClient interface. func (p *dcrdataPlugin) Setup() error { - log.Tracef("setup") + log.Tracef("dcrdata Setup") // Setup dcrdata websocket subscriptions and monitoring. This is // done in a go routine so setup will continue in the event that @@ -592,7 +592,7 @@ func (p *dcrdataPlugin) Setup() error { // // This function satisfies the plugins.PluginClient interface. func (p *dcrdataPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { - log.Tracef("Cmd: %v %x %v", treeID, token, cmd) + log.Tracef("dcrdata Cmd: %v %x %v", treeID, token, cmd) switch cmd { case dcrdata.CmdBestBlock: @@ -612,7 +612,7 @@ func (p *dcrdataPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (st // // This function satisfies the plugins.PluginClient interface. func (p *dcrdataPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { - log.Tracef("Hook: %v %x %v", treeID, token, plugins.Hooks[h]) + log.Tracef("dcrdata Hook: %v %x %v", treeID, token, plugins.Hooks[h]) return nil } @@ -621,7 +621,7 @@ func (p *dcrdataPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payloa // // This function satisfies the plugins.PluginClient interface. func (p *dcrdataPlugin) Fsck(treeIDs []int64) error { - log.Tracef("Fsck") + log.Tracef("dcrdata Fsck") return nil } @@ -630,7 +630,7 @@ func (p *dcrdataPlugin) Fsck(treeIDs []int64) error { // // This function satisfies the plugins.PluginClient interface. func (p *dcrdataPlugin) Settings() []backend.PluginSetting { - log.Tracef("Settings") + log.Tracef("dcrdata Settings") return nil } diff --git a/politeiad/backend/tlogbe/plugins/pi/pi.go b/politeiad/backend/tlogbe/plugins/pi/pi.go index ac8a541e7..51ddb4579 100644 --- a/politeiad/backend/tlogbe/plugins/pi/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi/pi.go @@ -48,7 +48,7 @@ type piPlugin struct { // // This function satisfies the plugins.PluginClient interface. func (p *piPlugin) Setup() error { - log.Tracef("Setup") + log.Tracef("pi Setup") // TODO Verify vote and comment plugin dependency @@ -59,7 +59,7 @@ func (p *piPlugin) Setup() error { // // This function satisfies the plugins.PluginClient interface. func (p *piPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { - log.Tracef("Cmd: %v %x %v %v", treeID, token, cmd, payload) + log.Tracef("pi Cmd: %v %x %v %v", treeID, token, cmd, payload) switch cmd { case pi.CmdVoteInv: @@ -73,7 +73,7 @@ func (p *piPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, // // This function satisfies the plugins.PluginClient interface. func (p *piPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { - log.Tracef("Hook: %v %x %v", treeID, token, plugins.Hooks[h]) + log.Tracef("pi Hook: %v %x %v", treeID, token, plugins.Hooks[h]) switch h { case plugins.HookTypeNewRecordPre: @@ -91,7 +91,7 @@ func (p *piPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload str // // This function satisfies the plugins.PluginClient interface. func (p *piPlugin) Fsck(treeIDs []int64) error { - log.Tracef("Fsck") + log.Tracef("pi Fsck") return nil } @@ -100,7 +100,7 @@ func (p *piPlugin) Fsck(treeIDs []int64) error { // // This function satisfies the plugins.PluginClient interface. func (p *piPlugin) Settings() []backend.PluginSetting { - log.Tracef("Settings") + log.Tracef("pi Settings") return []backend.PluginSetting{ { diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go index 622b01102..dc4a5ef11 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go @@ -87,7 +87,7 @@ func (p *ticketVotePlugin) mutex(token []byte) *sync.Mutex { // // This function satisfies the plugins.PluginClient interface. func (p *ticketVotePlugin) Setup() error { - log.Tracef("Setup") + log.Tracef("ticketvote Setup") // Verify plugin dependencies var dcrdataFound bool @@ -183,7 +183,7 @@ func (p *ticketVotePlugin) Setup() error { // // This function satisfies the plugins.PluginClient interface. func (p *ticketVotePlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { - log.Tracef("Cmd: %v %x %v", treeID, token, cmd) + log.Tracef("ticketvote Cmd: %v %x %v", treeID, token, cmd) switch cmd { case ticketvote.CmdAuthorize: @@ -219,7 +219,7 @@ func (p *ticketVotePlugin) Cmd(treeID int64, token []byte, cmd, payload string) // // This function satisfies the plugins.PluginClient interface. func (p *ticketVotePlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { - log.Tracef("Hook: %v %x %v", treeID, token, plugins.Hooks[h]) + log.Tracef("ticketvote Hook: %v %x %v", treeID, token, plugins.Hooks[h]) switch h { case plugins.HookTypeNewRecordPre: @@ -237,7 +237,7 @@ func (p *ticketVotePlugin) Hook(treeID int64, token []byte, h plugins.HookT, pay // // This function satisfies the plugins.PluginClient interface. func (p *ticketVotePlugin) Fsck(treeIDs []int64) error { - log.Tracef("Fsck") + log.Tracef("ticketvote Fsck") // Verify all caches @@ -248,7 +248,7 @@ func (p *ticketVotePlugin) Fsck(treeIDs []int64) error { // // This function satisfies the plugins.PluginClient interface. func (p *ticketVotePlugin) Settings() []backend.PluginSetting { - log.Tracef("Settings") + log.Tracef("ticketvote Settings") return []backend.PluginSetting{ { diff --git a/politeiad/backend/tlogbe/plugins/user/user.go b/politeiad/backend/tlogbe/plugins/user/user.go index 76a46f1ef..56f9166a3 100644 --- a/politeiad/backend/tlogbe/plugins/user/user.go +++ b/politeiad/backend/tlogbe/plugins/user/user.go @@ -32,7 +32,7 @@ type userPlugin struct { // // This function satisfies the plugins.PluginClient interface. func (p *userPlugin) Setup() error { - log.Tracef("Setup") + log.Tracef("user Setup") return nil } @@ -41,7 +41,7 @@ func (p *userPlugin) Setup() error { // // This function satisfies the plugins.PluginClient interface. func (p *userPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { - log.Tracef("Cmd: %v %x %v %v", treeID, token, cmd, payload) + log.Tracef("user Cmd: %v %x %v %v", treeID, token, cmd, payload) switch cmd { case user.CmdAuthor: @@ -57,7 +57,7 @@ func (p *userPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (strin // // This function satisfies the plugins.PluginClient interface. func (p *userPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { - log.Tracef("Hook: %v %x %v", treeID, token, plugins.Hooks[h]) + log.Tracef("user Hook: %v %x %v", treeID, token, plugins.Hooks[h]) switch h { case plugins.HookTypeNewRecordPre: @@ -79,7 +79,7 @@ func (p *userPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload s // // This function satisfies the plugins.PluginClient interface. func (p *userPlugin) Fsck(treeIDs []int64) error { - log.Tracef("Fsck") + log.Tracef("user Fsck") return nil } @@ -88,7 +88,7 @@ func (p *userPlugin) Fsck(treeIDs []int64) error { // // This function satisfies the plugins.PluginClient interface. func (p *userPlugin) Settings() []backend.PluginSetting { - log.Tracef("Settings") + log.Tracef("user Settings") return nil } diff --git a/politeiad/log.go b/politeiad/log.go index 9f4699f2c..4ea5f870f 100644 --- a/politeiad/log.go +++ b/politeiad/log.go @@ -49,15 +49,12 @@ var ( // application shutdown. logRotator *rotator.Rotator - log = backendLog.Logger("POLI") - gitbeLog = backendLog.Logger("GITB") - tlogbeLog = backendLog.Logger("BACK") - tlogLog = backendLog.Logger("TLOG") - wsdcrdataLog = backendLog.Logger("WSDD") - commentsLog = backendLog.Logger("COMT") - dcrdataLog = backendLog.Logger("DCDA") - ticketvoteLog = backendLog.Logger("TICK") - userLog = backendLog.Logger("USER") + log = backendLog.Logger("POLI") + gitbeLog = backendLog.Logger("GITB") + tlogbeLog = backendLog.Logger("BACK") + tlogLog = backendLog.Logger("TLOG") + wsdcrdataLog = backendLog.Logger("WSDD") + pluginLog = backendLog.Logger("PLUG") ) // Initialize package-global logger variables. @@ -67,10 +64,10 @@ func init() { tlog.UseLogger(tlogLog) filesystem.UseLogger(tlogLog) wsdcrdata.UseLogger(wsdcrdataLog) - comments.UseLogger(commentsLog) - dcrdata.UseLogger(dcrdataLog) - ticketvote.UseLogger(ticketvoteLog) - user.UseLogger(userLog) + comments.UseLogger(pluginLog) + dcrdata.UseLogger(pluginLog) + ticketvote.UseLogger(pluginLog) + user.UseLogger(pluginLog) } // subsystemLoggers maps each subsystem identifier to its associated logger. @@ -80,10 +77,7 @@ var subsystemLoggers = map[string]slog.Logger{ "BACK": tlogbeLog, "TLOG": tlogLog, "WSDD": wsdcrdataLog, - "COMT": commentsLog, - "DCDA": dcrdataLog, - "TICK": ticketvoteLog, - "USER": userLog, + "PLUG": pluginLog, } // initLogRotator initializes the logging rotater to write logs to logFile and diff --git a/politeiawww/api/comments/v1/v1.go b/politeiawww/api/comments/v1/v1.go index 2db18b0a0..998e6ef92 100644 --- a/politeiawww/api/comments/v1/v1.go +++ b/politeiawww/api/comments/v1/v1.go @@ -181,7 +181,8 @@ const ( VoteUpvote VoteT = 1 ) -// Vote casts a comment vote (upvote or downvote). +// Vote casts a comment vote (upvote or downvote). Votes can only be cast +// on vetted records. // // The effect of a new vote on a comment score depends on the previous vote // from that user ID. Example, a user upvotes a comment that they have already @@ -190,7 +191,6 @@ const ( // // Signature is the client signature of the Token+CommentID+Vote. type Vote struct { - State string `json:"state"` Token string `json:"token"` CommentID uint32 `json:"commentid"` Vote VoteT `json:"vote"` @@ -250,7 +250,6 @@ type CommentsReply struct { // Votes returns the comment votes that meet the provided filtering criteria. type Votes struct { - State string `json:"state"` Token string `json:"token"` UserID string `json:"userid"` } diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 3013ee2a9..d85df26b7 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -9,6 +9,7 @@ import ( ) // TODO verify that all batched request have a page size limit +// comment timestamps, vote timestamps // TODO module these API packages const ( diff --git a/politeiawww/client/error.go b/politeiawww/client/error.go index 5c5d36387..3b5fdac5d 100644 --- a/politeiawww/client/error.go +++ b/politeiawww/client/error.go @@ -7,11 +7,13 @@ package client import ( "fmt" "net/http" + "strings" cmplugin "github.com/decred/politeia/politeiad/plugins/comments" piplugin "github.com/decred/politeia/politeiad/plugins/pi" tkplugin "github.com/decred/politeia/politeiad/plugins/ticketvote" usplugin "github.com/decred/politeia/politeiad/plugins/user" + cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" @@ -58,6 +60,8 @@ func (e Error) Error() string { func apiUserErr(api string, e ErrorReply) string { var errMsg string switch api { + case cmv1.APIRoute: + errMsg = cmv1.ErrorCodes[cmv1.ErrorCodeT(e.ErrorCode)] case piv1.APIRoute: errMsg = piv1.ErrorCodes[piv1.ErrorCodeT(e.ErrorCode)] case rcv1.APIRoute: @@ -65,7 +69,14 @@ func apiUserErr(api string, e ErrorReply) string { case tkv1.APIRoute: errMsg = tkv1.ErrorCodes[tkv1.ErrorCodeT(e.ErrorCode)] } - m := fmt.Sprintf("user error code %v", e.ErrorCode) + + // Remove "/" from api string. "/records/v1" to "records v1". + s := strings.Split(api, "/") + api = strings.Join(s, " ") + api = strings.Trim(api, " ") + + // Create error string + m := fmt.Sprintf("%v user error code %v", api, e.ErrorCode) if errMsg != "" { m += fmt.Sprintf(", %v", errMsg) } diff --git a/politeiawww/cmd/pictl/cmdcommentcensor.go b/politeiawww/cmd/pictl/cmdcommentcensor.go index f0b6f5d8c..8d489274c 100644 --- a/politeiawww/cmd/pictl/cmdcommentcensor.go +++ b/politeiawww/cmd/pictl/cmdcommentcensor.go @@ -1,75 +1,86 @@ -// Copyright (c) 2017-2020 The Decred developers +// Copyright (c) 2017-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. package main +import ( + "encoding/hex" + "fmt" + "strconv" + + cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" + pclient "github.com/decred/politeia/politeiawww/client" + "github.com/decred/politeia/politeiawww/cmd/shared" + "github.com/decred/politeia/util" +) + // cmdCommentCensor censors a proposal comment. type cmdCommentCensor struct { Args struct { Token string `positional-arg-name:"token"` - CommentID string `positional-arg-name:"commentid"` + CommentID uint32 `positional-arg-name:"commentid"` Reason string `positional-arg-name:"reason"` } `positional-args:"true" required:"true"` - // CLI flags + // Unvetted is used to censor the comment on an unvetted record. If + // this flag is not used the command assumes the record is vetted. Unvetted bool `long:"unvetted" optional:"true"` } -/* // Execute executes the cmdCommentCensor command. // // This function satisfies the go-flags Commander interface. -func (cmd *cmdCommentCensor) Execute(args []string) error { +func (c *cmdCommentCensor) Execute(args []string) error { // Unpack args - token := cmd.Args.Token - reason := cmd.Args.Reason - commentID, err := strconv.ParseUint(cmd.Args.CommentID, 10, 32) - if err != nil { - return fmt.Errorf("ParseUint(%v): %v", cmd.Args.CommentID, err) - } + var ( + token = c.Args.Token + commentID = c.Args.CommentID + reason = c.Args.Reason + ) - // Verify user identity + // Check for user identity. A user identity is required to sign + // the censor request. if cfg.Identity == nil { return shared.ErrUserIdentityNotFound } - // Verify state. Defaults to vetted if the --unvetted flag - // is not used. - var state pi.PropStateT + // Setup state + var state string switch { - case cmd.Unvetted: - state = pi.PropStateUnvetted + case c.Unvetted: + state = cmv1.RecordStateUnvetted default: - state = pi.PropStateVetted + state = cmv1.RecordStateVetted } - // Sign comment data - msg := strconv.Itoa(int(state)) + token + cmd.Args.CommentID + reason - b := cfg.Identity.SignMessage([]byte(msg)) - signature := hex.EncodeToString(b[:]) + // Setup client + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Cookies: cfg.Cookies, + HeaderCSRF: cfg.CSRF, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, + } + pc, err := pclient.New(cfg.Host, opts) + if err != nil { + return err + } // Setup request - cc := pi.CommentCensor{ - Token: token, + msg := token + strconv.FormatUint(uint64(commentID), 10) + reason + sig := cfg.Identity.SignMessage([]byte(msg)) + d := cmv1.Del{ State: state, + Token: token, CommentID: uint32(commentID), Reason: reason, - Signature: signature, + Signature: hex.EncodeToString(sig[:]), PublicKey: cfg.Identity.Public.String(), } - // Send request. The request and response details are printed to - // the console. - err = shared.PrintJSON(cc) - if err != nil { - return err - } - ccr, err := client.CommentCensor(cc) - if err != nil { - return err - } - err = shared.PrintJSON(ccr) + // Send request + dr, err := pc.CommentDel(d) if err != nil { return err } @@ -83,30 +94,33 @@ func (cmd *cmdCommentCensor) Execute(args []string) error { if err != nil { return err } - receiptb, err := util.ConvertSignature(ccr.Comment.Receipt) + receiptb, err := util.ConvertSignature(dr.Comment.Receipt) if err != nil { return err } - if !serverID.VerifyMessage([]byte(signature), receiptb) { + if !serverID.VerifyMessage([]byte(d.Signature), receiptb) { return fmt.Errorf("could not verify receipt") } + // Print comment + printComment(dr.Comment) + return nil } -*/ // commentCensorHelpMsg is printed to stdout by the help command. const commentCensorHelpMsg = `commentcensor "token" "commentID" "reason" -Censor a user comment. This command assumes the record is a vetted record. If -the record is unvetted, the --unvetted flag must be used. Requires admin -privileges. +Censor a comment. + +If the record is unvetted, the --unvetted flag must be used. This command +requires admin priviledges. Arguments: -1. token (string, required) Proposal censorship token -2. commentid (string, required) ID of the comment -3. reason (string, required) Reason for censoring the comment +1. token (string, required) Proposal censorship token +2. commentid (string, required) ID of the comment +3. reason (string, required) Reason for censoring the comment Flags: - --unvetted (bool, optional) Comment on unvetted record. + --unvetted (bool, optional) Record is unvetted. ` diff --git a/politeiawww/cmd/pictl/cmdcommentnew.go b/politeiawww/cmd/pictl/cmdcommentnew.go index 8cb07c562..187da0273 100644 --- a/politeiawww/cmd/pictl/cmdcommentnew.go +++ b/politeiawww/cmd/pictl/cmdcommentnew.go @@ -103,9 +103,7 @@ func (c *cmdCommentNew) Execute(args []string) error { } // Print receipt - fmt.Printf("Comment Submitted\n") - fmt.Printf("ID : %v\n", nr.Comment.CommentID) - fmt.Printf("Receipt: %v\n", nr.Comment.Receipt) + printComment(nr.Comment) return nil } diff --git a/politeiawww/cmd/pictl/cmdcomments.go b/politeiawww/cmd/pictl/cmdcomments.go index 4f5bb8c04..45d99bf08 100644 --- a/politeiawww/cmd/pictl/cmdcomments.go +++ b/politeiawww/cmd/pictl/cmdcomments.go @@ -5,6 +5,8 @@ package main import ( + "fmt" + cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" pclient "github.com/decred/politeia/politeiawww/client" ) @@ -59,7 +61,8 @@ func (c *cmdComments) Execute(args []string) error { // Print comments for _, v := range cr.Comments { - _ = v + printComment(v) + fmt.Printf("\n") } return nil @@ -78,5 +81,4 @@ Arguments: 1. token (string, required) Proposal censorship token Flags: - --unvetted (bool, optional) Record is unvetted. -` + --unvetted (bool, optional) Record is unvetted.` diff --git a/politeiawww/cmd/pictl/cmdcommenttimestamps.go b/politeiawww/cmd/pictl/cmdcommenttimestamps.go index da4c63c3c..3102d3a5e 100644 --- a/politeiawww/cmd/pictl/cmdcommenttimestamps.go +++ b/politeiawww/cmd/pictl/cmdcommenttimestamps.go @@ -4,27 +4,47 @@ package main -// cmdCommentTimestamps retrieves the timestamps for politeiawww comments. +import ( + "fmt" + + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" + cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" + pclient "github.com/decred/politeia/politeiawww/client" +) + +// cmdCommentTimestamps retrieves the timestamps for a record's comments. type cmdCommentTimestamps struct { Args struct { Token string `positional-arg-name:"token" required:"true"` CommentIDs []uint32 `positional-arg-name:"commentids" optional:"true"` } `positional-args:"true"` - // Unvetted is used to request the comment timestamps of an - // unvetted record. + // Unvetted is used to request the timestamps of an unvetted + // record. If this flag is not used the command assumes the record + // is vetted. Unvetted bool `long:"unvetted" optional:"true"` } -/* // Execute executes the cmdCommentTimestamps command. // // This function satisfies the go-flags Commander interface. func (c *cmdCommentTimestamps) Execute(args []string) error { + // Setup client + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Cookies: cfg.Cookies, + HeaderCSRF: cfg.CSRF, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, + } + pc, err := pclient.New(cfg.Host, opts) + if err != nil { + return err + } - // Set comment state. Defaults to vetted unless the unvetted flag - // is used. - var state cmv1.RecordStateT + // Setup state + var state string switch { case c.Unvetted: state = cmv1.RecordStateUnvetted @@ -32,23 +52,13 @@ func (c *cmdCommentTimestamps) Execute(args []string) error { state = cmv1.RecordStateVetted } - // Setup request + // Get timestamps t := cmv1.Timestamps{ State: state, Token: c.Args.Token, CommentIDs: c.Args.CommentIDs, } - - // Send request - err := shared.PrintJSON(t) - if err != nil { - return err - } - tr, err := client.CommentTimestamps(t) - if err != nil { - return err - } - err = shared.PrintJSON(tr) + tr, err := pc.CommentTimestamps(t) if err != nil { return err } @@ -69,7 +79,7 @@ func (c *cmdCommentTimestamps) Execute(args []string) error { func verifyCommentTimestamp(t cmv1.Timestamp) error { ts := convertCommentTimestamp(t) - return tlogbe.VerifyTimestamp(ts) + return tlog.VerifyTimestamp(ts) } func convertCommentProof(p cmv1.Proof) backend.Proof { @@ -95,7 +105,6 @@ func convertCommentTimestamp(t cmv1.Timestamp) backend.Timestamp { Proofs: proofs, } } -*/ // commentTimestampsHelpMsg is printed to stdout by the help command const commentTimestampsHelpMsg = `commenttimestamps [flags] "token" commentIDs @@ -104,17 +113,18 @@ Fetch the timestamps for a record's comments. The timestamp contains all necessary data to verify that user submitted comment data has been timestamped onto the decred blockchain. +If comment IDs are not provided then the timestamps for all comments will be +returned. If the record is unvetted, the --unvetted flag must be used. + Arguments: 1. token (string, required) Proposal token 2. commentIDs ([]uint32, optional) Proposal version Flags: - --unvetted (bool, optional) Request is for an unvetted record instead of - vetted ones. + --unvetted (bool, optional) Record is unvetted. Example: Fetch all record comment timestamps $ piwww commenttimestamps 0a265dd93e9bae6d Example: Fetch comment timestamps for comment IDs 1, 6, and 7 -$ piwww commenttimestamps 0a265dd93e9bae6d 1 6 7 -` +$ pictl commenttimestamps 0a265dd93e9bae6d 1 6 7` diff --git a/politeiawww/cmd/pictl/cmdcommentvote.go b/politeiawww/cmd/pictl/cmdcommentvote.go index 2ebc03906..ea0b7615f 100644 --- a/politeiawww/cmd/pictl/cmdcommentvote.go +++ b/politeiawww/cmd/pictl/cmdcommentvote.go @@ -67,7 +67,6 @@ func (c *cmdCommentVote) Execute(args []string) error { sig := cfg.Identity.SignMessage([]byte(msg)) v := cmv1.Vote{ Token: c.Args.Token, - State: cmv1.RecordStateVetted, CommentID: c.Args.CommentID, Vote: vote, Signature: hex.EncodeToString(sig[:]), @@ -98,6 +97,10 @@ func (c *cmdCommentVote) Execute(args []string) error { } // Print receipt + printf("Downvotes: %v\n", int64(cvr.Downvotes)*-1) + printf("Upvotes : %v\n", cvr.Upvotes) + printf("Timestamp: %v\n", timestampFromUnix(cvr.Timestamp)) + printf("Receipt : %v\n", cvr.Receipt) return nil } @@ -105,7 +108,9 @@ func (c *cmdCommentVote) Execute(args []string) error { // commentVoteHelpMsg is printed to stdout by the help command. const commentVoteHelpMsg = `commentvote "token" "commentID" "vote" -Upvote or downvote a comment. Requires the user to be logged in. +Upvote or downvote a comment. + +Requires the user to be logged in. Votes can only be cast on vetted records. Arguments: 1. token (string, required) Proposal censorship token diff --git a/politeiawww/cmd/pictl/cmdcommentvotes.go b/politeiawww/cmd/pictl/cmdcommentvotes.go index 016821210..2bd828ed9 100644 --- a/politeiawww/cmd/pictl/cmdcommentvotes.go +++ b/politeiawww/cmd/pictl/cmdcommentvotes.go @@ -4,6 +4,13 @@ package main +import ( + "fmt" + + cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" + pclient "github.com/decred/politeia/politeiawww/client" +) + // cmdCommentVotes retrieves the comment upvotes/downvotes for a user on a // record. type cmdCommentVotes struct { @@ -11,52 +18,72 @@ type cmdCommentVotes struct { Token string `positional-arg-name:"token" required:"true"` UserID string `positional-arg-name:"userid"` } `positional-args:"true"` - Me bool `long:"me" optional:"true"` } -/* // Execute executes the cmdCommentVotes command. // // This function satisfies the go-flags Commander interface. func (c *cmdCommentVotes) Execute(args []string) error { - token := c.Args.Token - userID := c.Args.UserID - - if userID == "" && !c.Me { - return fmt.Errorf("you must either provide a user id or use " + - "the --me flag to use the user ID of the logged in user") - } + // Unpack args + var ( + token = c.Args.Token + userID = c.Args.UserID + ) - // Get user ID of logged in user if specified - if c.Me { + // If a user ID was not provided this command assumes the user + // is requesting their own comment votes. + if userID == "" { + // Get user ID of logged in user lr, err := client.Me() if err != nil { + if err.Error() == "401" { + return fmt.Errorf("no user ID provided and no logged in "+ + "user found. Command usage: \n\n%v", commentVotesHelpMsg) + } return err } userID = lr.UserID } - cvr, err := client.CommentVotes(pi.CommentVotes{ + // Setup client + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Cookies: cfg.Cookies, + HeaderCSRF: cfg.CSRF, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, + } + pc, err := pclient.New(cfg.Host, opts) + if err != nil { + return err + } + + // Get comment votes + v := cmv1.Votes{ Token: token, - State: pi.PropStateVetted, UserID: userID, - }) + } + vr, err := pc.CommentVotes(v) if err != nil { return err } - return shared.PrintJSON(cvr) + + // Print votes + if len(vr.Votes) == 0 { + printf("No comment votes found for user %v\n", userID) + } + printCommentVotes(vr.Votes) + + return nil } -*/ // commentVotesHelpMsg is printed to stdout by the help command. const commentVotesHelpMsg = `commentvotes "token" "userid" -Get the provided user comment upvote/downvotes for a proposal. +Get the provided user comment upvote/downvotes for a proposal. If no user ID +is provded then the command will assume the logged in user is requesting their +own comment votes. Arguments: -1. token (string, required) Proposal censorship token -2. userid (string, required) User ID - -Flags: - --me (bool, optional) Use the user ID of the logged in user -` +1. token (string, required) Proposal censorship token +2. userid (string, optional) User ID` diff --git a/politeiawww/cmd/pictl/comment.go b/politeiawww/cmd/pictl/comment.go new file mode 100644 index 000000000..da24bccf5 --- /dev/null +++ b/politeiawww/cmd/pictl/comment.go @@ -0,0 +1,70 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "sort" + "strings" + + cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" +) + +func printComment(c cmv1.Comment) { + downvotes := int64(c.Downvotes) * -1 + + printf("Comment %v\n", c.CommentID) + printf(" Score : %v %v\n", downvotes, c.Upvotes) + printf(" Username : %v\n", c.Username) + printf(" Parent ID: %v\n", c.ParentID) + printf(" Timestamp: %v\n", timestampFromUnix(c.Timestamp)) + + // If the comment has been deleted the comment text will not be + // present. Print the reason for deletion instead and exit. + if c.Deleted { + printf(" Deleted : %v\n", c.Deleted) + printf(" Reason : %v\n", c.Reason) + return + } + + // Print the fist line as is if its less than the 80 character + // limit (including the leading comment label). + if len(c.Comment) < 66 { + printf(" Comment : %v\n", c.Comment) + return + } + + // Format lines as 80 characters that start with two spaces. + var b strings.Builder + b.WriteString(" ") + for i, v := range c.Comment { + if i != 0 && i%78 == 0 { + b.WriteString("\n") + b.WriteString(" ") + } + b.WriteString(string(v)) + } + printf(" Comment :\n") + printf("%v\n", b.String()) +} + +func printCommentVotes(votes []cmv1.CommentVote) { + if len(votes) == 0 { + return + } + printf("Token : %v\n", votes[0].Token) + printf("UserID : %v\n", votes[0].UserID) + printf("Username: %v\n", votes[0].Username) + printf("Votes\n") + + // Order votes by timestamp. Oldest to newest. + sort.SliceStable(votes, func(i, j int) bool { + return votes[i].Timestamp < votes[j].Timestamp + }) + + for _, v := range votes { + printf(" %-22v comment %v vote %v\n", + timestampFromUnix(v.Timestamp), v.CommentID, v.Vote) + } +} diff --git a/politeiawww/cmd/pictl/pictl.go b/politeiawww/cmd/pictl/pictl.go index f074b1c03..004125895 100644 --- a/politeiawww/cmd/pictl/pictl.go +++ b/politeiawww/cmd/pictl/pictl.go @@ -115,7 +115,7 @@ Help commands help Print detailed help message for a command Basic commands - version (public) Get politeiawww server version + version (public) Get politeiawww server version and CSRF policy (public) Get politeiawww server policy secret (public) Ping the server login (public) Login to politeiawww @@ -151,6 +151,7 @@ Proposal commands proposaltimestamps (public) Get timestamps for a proposal Comment commands + commentpolicy (public) Get the comments api policy commentnew (user) Submit a new comment commentvote (user) Upvote/downvote a comment commentcensor (admin) Censor a comment diff --git a/politeiawww/cmd/pictl/proposal.go b/politeiawww/cmd/pictl/proposal.go index 3d45c524a..137fbe3e3 100644 --- a/politeiawww/cmd/pictl/proposal.go +++ b/politeiawww/cmd/pictl/proposal.go @@ -21,6 +21,35 @@ import ( "github.com/decred/politeia/util" ) +func printProposalFiles(files []rcv1.File) error { + for _, v := range files { + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return err + } + size := byteCountSI(int64(len(b))) + printf(" %-22v %-26v %v\n", v.Name, v.MIME, size) + } + return nil +} + +func printProposal(r rcv1.Record) error { + printf("Token : %v\n", r.CensorshipRecord.Token) + printf("Version : %v\n", r.Version) + printf("Status : %v\n", rcv1.RecordStatuses[r.Status]) + printf("Timestamp: %v\n", r.Timestamp) + printf("Username : %v\n", r.Username) + printf("Merkle : %v\n", r.CensorshipRecord.Merkle) + printf("Receipt : %v\n", r.CensorshipRecord.Signature) + printf("Metadata\n") + for _, v := range r.Metadata { + size := byteCountSI(int64(len([]byte(v.Payload)))) + printf(" %-2v %v\n", v.ID, size) + } + printf("Files\n") + return printProposalFiles(r.Files) +} + func convertProposal(p piv1.Proposal) (*rcv1.Record, error) { // Setup files files := make([]rcv1.File, 0, len(p.Files)) @@ -87,42 +116,13 @@ func convertProposal(p piv1.Proposal) (*rcv1.Record, error) { }, nil } -func printProposalFiles(files []rcv1.File) error { - for _, v := range files { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return err - } - size := byteCountSI(int64(len(b))) - printf(" %-22v %-26v %v\n", v.Name, v.MIME, size) - } - return nil -} - -func printProposal(r rcv1.Record) error { - printf("Token : %v\n", r.CensorshipRecord.Token) - printf("Version : %v\n", r.Version) - printf("Status : %v\n", rcv1.RecordStatuses[r.Status]) - printf("Timestamp: %v\n", r.Timestamp) - printf("Username : %v\n", r.Username) - printf("Merkle : %v\n", r.CensorshipRecord.Merkle) - printf("Receipt : %v\n", r.CensorshipRecord.Signature) - printf("Metadata\n") - for _, v := range r.Metadata { - size := byteCountSI(int64(len([]byte(v.Payload)))) - printf(" %-2v %v\n", v.ID, size) - } - printf("Files\n") - return printProposalFiles(r.Files) -} - // indexFileRandom returns a proposal index file filled with random data. func indexFileRandom(sizeInBytes int) (*rcv1.File, error) { // Create lines of text that are 80 characters long charSet := "abcdefghijklmnopqrstuvwxyz" var b strings.Builder for i := 0; i < sizeInBytes; i++ { - if i%80 == 0 && i != 0 { + if i != 0 && i%80 == 0 { b.WriteString("\n") continue } diff --git a/politeiawww/cmd/pictl/time.go b/politeiawww/cmd/pictl/time.go new file mode 100644 index 000000000..acbfcb0c1 --- /dev/null +++ b/politeiawww/cmd/pictl/time.go @@ -0,0 +1,23 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import "time" + +const ( + // timeFormat contains the reference time format that is used + // throughout this CLI tool. This format is how timestamps are + // printed when we want to print the human readable version. + // + // Mon Jan 2 15:04:05 -0700 MST 2006 + timeFormat = "01/02/2006 3:04pm MST" +) + +// timestampFromUnix converts a unix timestamp into a human readable timestamp +// string formatted according to the timeFormat global variable. +func timestampFromUnix(unixTime int64) string { + t := time.Unix(unixTime, 0) + return t.Format(timeFormat) +} diff --git a/politeiawww/comments/process.go b/politeiawww/comments/process.go index c7140a6f1..f606926b6 100644 --- a/politeiawww/comments/process.go +++ b/politeiawww/comments/process.go @@ -70,7 +70,7 @@ func (c *Comments) processNew(ctx context.Context, n v1.New, u user.User) (*v1.N // Prepare reply cm := convertComment(cnr.Comment) - cm = commentPopulateUserData(cm, u) + commentPopulateUserData(&cm, u) // Emit event c.events.Emit(EventTypeNew, @@ -99,14 +99,6 @@ func (c *Comments) processVote(ctx context.Context, v v1.Vote, u user.User) (*v1 } } - // Verify state - if v.State != v1.RecordStateVetted { - return nil, v1.UserErrorReply{ - ErrorCode: v1.ErrorCodeRecordStateInvalid, - ErrorContext: "record must be vetted", - } - } - // Verify user signed using active identity if u.PublicKey() != v.PublicKey { return nil, v1.UserErrorReply{ @@ -124,7 +116,7 @@ func (c *Comments) processVote(ctx context.Context, v v1.Vote, u user.User) (*v1 PublicKey: v.PublicKey, Signature: v.Signature, } - vr, err := c.politeiad.CommentVote(ctx, v.State, cv) + vr, err := c.politeiad.CommentVote(ctx, v1.RecordStateVetted, cv) if err != nil { return nil, err } @@ -163,7 +155,7 @@ func (c *Comments) processDel(ctx context.Context, d v1.Del, u user.User) (*v1.D // Prepare reply cm := convertComment(cdr.Comment) - cm = commentPopulateUserData(cm, u) + commentPopulateUserData(&cm, u) return &v1.DelReply{ Comment: cm, @@ -238,7 +230,7 @@ func (c *Comments) processComments(ctx context.Context, cs v1.Comments, u *user. if err != nil { return nil, err } - cm = commentPopulateUserData(cm, *u) + commentPopulateUserData(&cm, *u) // Add comment comments = append(comments, cm) @@ -256,13 +248,23 @@ func (c *Comments) processVotes(ctx context.Context, v v1.Votes) (*v1.VotesReply cm := comments.Votes{ UserID: v.UserID, } - votes, err := c.politeiad.CommentVotes(ctx, v.State, v.Token, cm) + votes, err := c.politeiad.CommentVotes(ctx, + v1.RecordStateVetted, v.Token, cm) + if err != nil { + return nil, err + } + cv := convertCommentVotes(votes) + + // Populate comment votes with user data + uid, err := uuid.Parse(v.UserID) + u, err := c.userdb.UserGetById(uid) if err != nil { return nil, err } + commentVotePopulateUserData(cv, *u) return &v1.VotesReply{ - Votes: convertCommentVotes(votes), + Votes: cv, }, nil } @@ -299,9 +301,16 @@ func (c *Comments) processTimestamps(ctx context.Context, t v1.Timestamps, isAdm // commentPopulateUserData populates the comment with user data that is not // stored in politeiad. -func commentPopulateUserData(c v1.Comment, u user.User) v1.Comment { +func commentPopulateUserData(c *v1.Comment, u user.User) { c.Username = u.Username - return c +} + +// commentVotePopulateUserData populates the comment vote with user data that +// is not stored in politeiad. +func commentVotePopulateUserData(votes []v1.CommentVote, u user.User) { + for k := range votes { + votes[k].Username = u.Username + } } func convertComment(c comments.Comment) v1.Comment { From 27d45cfd2abc6720993e5992fedab73dfeb3de20 Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 5 Feb 2021 15:41:57 -0600 Subject: [PATCH 275/449] Comment count route and command. --- politeiad/client/comments.go | 10 ++- politeiawww/api/comments/v1/v1.go | 2 + politeiawww/client/comments.go | 4 +- politeiawww/cmd/pictl/cmdcommentcount.go | 87 ++++++++++++++++++++++ politeiawww/cmd/pictl/cmdcommentspolicy.go | 4 +- politeiawww/cmd/pictl/cmdproposals.go | 6 +- politeiawww/cmd/pictl/help.go | 14 +++- politeiawww/cmd/pictl/pictl.go | 5 +- politeiawww/comments/comments.go | 2 +- politeiawww/comments/process.go | 8 +- politeiawww/piwww.go | 3 + 11 files changed, 127 insertions(+), 18 deletions(-) create mode 100644 politeiawww/cmd/pictl/cmdcommentcount.go diff --git a/politeiad/client/comments.go b/politeiad/client/comments.go index 3fa860b9d..d8ff21adf 100644 --- a/politeiad/client/comments.go +++ b/politeiad/client/comments.go @@ -9,6 +9,7 @@ import ( "encoding/json" "fmt" + "github.com/davecgh/go-spew/spew" pdv1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/plugins/comments" ) @@ -136,11 +137,11 @@ func (c *Client) CommentDel(ctx context.Context, state string, d comments.Del) ( return &dr, nil } -// CommentCounts sends a batch of comment plugin Count commands to the +// CommentCount sends a batch of comment plugin Count commands to the // politeiad v1 API and returns a map[token]count with the results. If a record // is not found for a token or any other error occurs, that token will not be // included in the reply. -func (c *Client) CommentCounts(ctx context.Context, state string, tokens []string) (map[string]uint32, error) { +func (c *Client) CommentCount(ctx context.Context, state string, tokens []string) (map[string]uint32, error) { // Setup request cmds := make([]pdv1.PluginCommandV2, 0, len(tokens)) for _, v := range tokens { @@ -157,7 +158,7 @@ func (c *Client) CommentCounts(ctx context.Context, state string, tokens []strin if err != nil { return nil, err } - if len(replies) == len(cmds) { + if len(replies) != len(cmds) { return nil, fmt.Errorf("replies missing") } @@ -168,11 +169,12 @@ func (c *Client) CommentCounts(ctx context.Context, state string, tokens []strin // command that errored will not be included in the reply. err = extractPluginCommandError(v) if err != nil { + spew.Dump(err) continue } var cr comments.CountReply - err = json.Unmarshal([]byte(v.Payload), cr) + err = json.Unmarshal([]byte(v.Payload), &cr) if err != nil { continue } diff --git a/politeiawww/api/comments/v1/v1.go b/politeiawww/api/comments/v1/v1.go index 998e6ef92..1d151c42e 100644 --- a/politeiawww/api/comments/v1/v1.go +++ b/politeiawww/api/comments/v1/v1.go @@ -38,6 +38,7 @@ const ( ErrorCodeTokenInvalid ErrorCodeT = 6 ErrorCodeRecordNotFound ErrorCodeT = 7 ErrorCodeRecordLocked ErrorCodeT = 8 + ErrorCodeNoTokensFound ErrorCodeT = 9 ) var ( @@ -52,6 +53,7 @@ var ( ErrorCodeTokenInvalid: "token invalid", ErrorCodeRecordNotFound: "record not found", ErrorCodeRecordLocked: "record is locked", + ErrorCodeNoTokensFound: "no tokens found", } ) diff --git a/politeiawww/client/comments.go b/politeiawww/client/comments.go index 233015499..d4ff0dab9 100644 --- a/politeiawww/client/comments.go +++ b/politeiawww/client/comments.go @@ -94,9 +94,9 @@ func (c *Client) CommentDel(d cmv1.Del) (*cmv1.DelReply, error) { } // CommentCount sends a comments v1 Count request to politeiawww. -func (c *Client) CommentCount(cn cmv1.Count) (*cmv1.CountReply, error) { +func (c *Client) CommentCount(cc cmv1.Count) (*cmv1.CountReply, error) { resBody, err := c.makeReq(http.MethodPost, - cmv1.APIRoute, cmv1.RouteCount, cn) + cmv1.APIRoute, cmv1.RouteCount, cc) if err != nil { return nil, err } diff --git a/politeiawww/cmd/pictl/cmdcommentcount.go b/politeiawww/cmd/pictl/cmdcommentcount.go new file mode 100644 index 000000000..a9bb3d449 --- /dev/null +++ b/politeiawww/cmd/pictl/cmdcommentcount.go @@ -0,0 +1,87 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + + cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" + pclient "github.com/decred/politeia/politeiawww/client" +) + +// cmdCommentCount retreives the comments for the specified proposal. +type cmdCommentCount struct { + Args struct { + Tokens []string `positional-arg-name:"tokens"` + } `positional-args:"true" required:"true"` + + // Unvetted is used to request the comment counts of unvetted + // records. If this flag is not used the command assumes the + // records are vetted. + Unvetted bool `long:"unvetted" optional:"true"` +} + +// Execute executes the cmdCommentCount command. +// +// This function satisfies the go-flags Commander interface. +func (c *cmdCommentCount) Execute(args []string) error { + // Setup client + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Cookies: cfg.Cookies, + HeaderCSRF: cfg.CSRF, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, + } + pc, err := pclient.New(cfg.Host, opts) + if err != nil { + return err + } + + // Setup state + var state string + switch { + case c.Unvetted: + state = cmv1.RecordStateUnvetted + default: + state = cmv1.RecordStateVetted + } + + // Get comments + cc := cmv1.Count{ + State: state, + Tokens: c.Args.Tokens, + } + cr, err := pc.CommentCount(cc) + if err != nil { + return err + } + + // Print counts + for k, v := range cr.Counts { + fmt.Printf("%v %v\n", k, v) + } + + return nil +} + +// commentCountHelpMsg is printed to stdout by the help command. +const commentCountHelpMsg = `commentcount "tokens..." + +Get the number of comments that have been made on each of the provided +records. + +If the record is unvetted, the --unvetted flag must be used. This command +accepts both full length tokens or the token prefixes. + +Arguments: +1. token (string, required) Proposal censorship token + +Flags: + --unvetted (bool, optional) Record is unvetted. + +Examples: +$ pictl commentcount f6458c2d8d9ef41c 9f9af91cf609d839 917c6fde9bcc2118 +$ pictl commentcount f6458c2 9f9af91 917c6fd` diff --git a/politeiawww/cmd/pictl/cmdcommentspolicy.go b/politeiawww/cmd/pictl/cmdcommentspolicy.go index 375b7936c..abc2e4edf 100644 --- a/politeiawww/cmd/pictl/cmdcommentspolicy.go +++ b/politeiawww/cmd/pictl/cmdcommentspolicy.go @@ -38,7 +38,7 @@ func (c *cmdCommentPolicy) Execute(args []string) error { return nil } -// commentsEditHelpMsg is the printed to stdout by the help command. -const commentsPolicyHelpMsg = `commentspolicy +// commentEditHelpMsg is the printed to stdout by the help command. +const commentPolicyHelpMsg = `commentpolicy Fetch the comments API policy.` diff --git a/politeiawww/cmd/pictl/cmdproposals.go b/politeiawww/cmd/pictl/cmdproposals.go index a7c8a5b17..7966b43b7 100644 --- a/politeiawww/cmd/pictl/cmdproposals.go +++ b/politeiawww/cmd/pictl/cmdproposals.go @@ -80,8 +80,7 @@ proposal attachments are not returned from this command. Use the proposal details command if you are trying to retieve the full proposal. This command defaults to retrieving vetted proposals unless the --unvetted flag -is used. This command accepts both the full tokens or the shortened token -prefixes. +is used. This command accepts both the full tokens or the token prefixes. Arguments: 1. tokens ([]string, required) Proposal tokens. @@ -91,5 +90,4 @@ Flags: Example: $ pictl proposals f6458c2d8d9ef41c 9f9af91cf609d839 917c6fde9bcc2118 -$ pictl proposals f6458c2 9f9af91 917c6fd -` +$ pictl proposals f6458c2 9f9af91 917c6fd` diff --git a/politeiawww/cmd/pictl/help.go b/politeiawww/cmd/pictl/help.go index f1fa9d0d9..f089619d5 100644 --- a/politeiawww/cmd/pictl/help.go +++ b/politeiawww/cmd/pictl/help.go @@ -41,24 +41,32 @@ func (cmd *helpCmd) Execute(args []string) error { fmt.Printf("%s\n", proposalNewHelpMsg) case "proposaledit": fmt.Printf("%s\n", proposalEditHelpMsg) - case "proposalstatusset": + case "proposalsetstatus": fmt.Printf("%s\n", proposalSetStatusHelpMsg) case "proposals": fmt.Printf("%s\n", proposalsHelpMsg) - case "proposalinventory": + case "proposalinv": fmt.Printf("%s\n", proposalInvHelpMsg) + case "proposaltimestamps": + fmt.Printf("%s\n", proposalTimestampsHelpMsg) - // Comment commands + // Comment commands + case "commentpolicy": + fmt.Printf("%s\n", commentPolicyHelpMsg) case "commentnew": fmt.Printf("%s\n", commentNewHelpMsg) case "commentvote": fmt.Printf("%s\n", commentVoteHelpMsg) case "commentcensor": fmt.Printf("%s\n", commentCensorHelpMsg) + case "commentcount": + fmt.Printf("%s\n", commentCountHelpMsg) case "comments": fmt.Printf("%s\n", commentsHelpMsg) case "commentvotes": fmt.Printf("%s\n", commentVotesHelpMsg) + case "commenttimestamps": + fmt.Printf("%s\n", commentTimestampsHelpMsg) // Vote commands case "votepolicy": diff --git a/politeiawww/cmd/pictl/pictl.go b/politeiawww/cmd/pictl/pictl.go index 004125895..d040672e1 100644 --- a/politeiawww/cmd/pictl/pictl.go +++ b/politeiawww/cmd/pictl/pictl.go @@ -78,6 +78,7 @@ type pictl struct { CommentNew cmdCommentNew `command:"commentnew"` CommentVote cmdCommentVote `command:"commentvote"` CommentCensor cmdCommentCensor `command:"commentcensor"` + CommentCount cmdCommentCount `command:"commentcount"` Comments cmdComments `command:"comments"` CommentVotes cmdCommentVotes `command:"commentvotes"` CommentTimestamps cmdCommentTimestamps `command:"commenttimestamps"` @@ -145,7 +146,7 @@ Proposal commands proposalnew (user) Submit a new proposal proposaledit (user) Edit an existing proposal proposalstatusset (admin) Set the status of a proposal - proposaldetials (public) Get a full proposal record + proposaldetails (public) Get a full proposal record proposals (public) Get proposals without their files proposalinv (public) Get inventory by proposal status proposaltimestamps (public) Get timestamps for a proposal @@ -155,8 +156,10 @@ Comment commands commentnew (user) Submit a new comment commentvote (user) Upvote/downvote a comment commentcensor (admin) Censor a comment + commentcount (public) Get the number of comments comments (public) Get comments commentvotes (public) Get comment votes + commenttimestamps (public) Get comment timestamps Vote commands votepolicy (public) Get the ticketvote api policy diff --git a/politeiawww/comments/comments.go b/politeiawww/comments/comments.go index b3876f0c5..1d48d082f 100644 --- a/politeiawww/comments/comments.go +++ b/politeiawww/comments/comments.go @@ -137,7 +137,7 @@ func (c *Comments) HandleCount(w http.ResponseWriter, r *http.Request) { var ct v1.Count decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&c); err != nil { + if err := decoder.Decode(&ct); err != nil { respondWithError(w, r, "HandleCount: unmarshal", v1.UserErrorReply{ ErrorCode: v1.ErrorCodeInputInvalid, diff --git a/politeiawww/comments/process.go b/politeiawww/comments/process.go index f606926b6..60d6af114 100644 --- a/politeiawww/comments/process.go +++ b/politeiawww/comments/process.go @@ -165,7 +165,13 @@ func (c *Comments) processDel(ctx context.Context, d v1.Del, u user.User) (*v1.D func (c *Comments) processCount(ctx context.Context, ct v1.Count) (*v1.CountReply, error) { log.Tracef("processCount: %v", ct.Tokens) - counts, err := c.politeiad.CommentCounts(ctx, ct.State, ct.Tokens) + if len(ct.Tokens) == 0 { + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeNoTokensFound, + } + } + + counts, err := c.politeiad.CommentCount(ctx, ct.State, ct.Tokens) if err != nil { return nil, err } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index be5e88704..17080e1a4 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -102,6 +102,9 @@ func (p *politeiawww) setupPiRoutes(r *records.Records, c *comments.Comments, t p.addRoute(http.MethodPost, cmv1.APIRoute, cmv1.RouteDel, c.HandleDel, permissionAdmin) + p.addRoute(http.MethodPost, cmv1.APIRoute, + cmv1.RouteCount, c.HandleCount, + permissionPublic) p.addRoute(http.MethodPost, cmv1.APIRoute, cmv1.RouteComments, c.HandleComments, permissionPublic) From 39f2d05957f78c09dbefdf0a1752dc9da184c68d Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 7 Feb 2021 09:35:29 -0600 Subject: [PATCH 276/449] Pi user plugin error fixes. --- politeiawww/comments/process.go | 10 ++++++---- politeiawww/pi/user.go | 2 +- politeiawww/records/pi.go | 3 ++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/politeiawww/comments/process.go b/politeiawww/comments/process.go index 60d6af114..b6670c63c 100644 --- a/politeiawww/comments/process.go +++ b/politeiawww/comments/process.go @@ -25,6 +25,7 @@ func (c *Comments) processNew(ctx context.Context, n v1.New, u user.User) (*v1.N // Verify user has paid registration paywall if !c.userHasPaid(u) { return nil, v1.PluginErrorReply{ + PluginID: pi.UserPluginID, ErrorCode: pi.ErrorCodeUserRegistrationNotPaid, } } @@ -87,14 +88,15 @@ func (c *Comments) processNew(ctx context.Context, n v1.New, u user.User) (*v1.N func (c *Comments) processVote(ctx context.Context, v v1.Vote, u user.User) (*v1.VoteReply, error) { log.Tracef("processVote: %v %v %v", v.Token, v.CommentID, v.Vote) - // Checking the mode is a temporary measure until user plugins - // have been implemented. + // Execute pre plugin hooks. Checking the mode is a temporary + // measure until user plugins have been properly implemented. switch c.cfg.Mode { case config.PoliteiaWWWMode: // Verify user has paid registration paywall if !c.userHasPaid(u) { - return nil, v1.UserErrorReply{ - // ErrorCode: v1.ErrorCodeUserRegistrationNotPaid, + return nil, v1.PluginErrorReply{ + PluginID: pi.UserPluginID, + ErrorCode: pi.ErrorCodeUserRegistrationNotPaid, } } } diff --git a/politeiawww/pi/user.go b/politeiawww/pi/user.go index 8761c39b6..a58615030 100644 --- a/politeiawww/pi/user.go +++ b/politeiawww/pi/user.go @@ -11,7 +11,7 @@ package pi const ( // UserPluginID is a temporary plugin ID for user functionality // that is specific to pi. - UserPluginID = "politeiawww-piuser" + UserPluginID = "piuser" // ErrorCodeInvalid is an invalid error code. ErrorCodeInvalid = 0 diff --git a/politeiawww/records/pi.go b/politeiawww/records/pi.go index d113b5179..c7343d447 100644 --- a/politeiawww/records/pi.go +++ b/politeiawww/records/pi.go @@ -80,7 +80,8 @@ func (r *Records) piHookNewRecordPre(u user.User) error { // Verify user has a proposal credit if !userHasProposalCredits(u) { - return v1.UserErrorReply{ + return v1.PluginErrorReply{ + PluginID: pi.UserPluginID, ErrorCode: pi.ErrorCodeUserBalanceInsufficient, } } From 6b9b2998a28c3a81082367157f5ceddd411b609b Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 7 Feb 2021 11:11:46 -0600 Subject: [PATCH 277/449] Sign hex encoded merkle. --- politeiad/backend/tlogbe/plugins/user/hooks.go | 4 +++- politeiad/plugins/user/user.go | 6 ++++-- politeiawww/cmd/pictl/proposal.go | 5 +++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/politeiad/backend/tlogbe/plugins/user/hooks.go b/politeiad/backend/tlogbe/plugins/user/hooks.go index 9f298dbde..f3a4125a3 100644 --- a/politeiad/backend/tlogbe/plugins/user/hooks.go +++ b/politeiad/backend/tlogbe/plugins/user/hooks.go @@ -5,6 +5,7 @@ package user import ( + "encoding/hex" "encoding/json" "errors" "fmt" @@ -88,7 +89,8 @@ func userMetadataVerify(metadata []backend.MetadataStream, files []backend.File) if err != nil { return err } - err = util.VerifySignature(um.Signature, um.PublicKey, string(m[:])) + mr := hex.EncodeToString(m[:]) + err = util.VerifySignature(um.Signature, um.PublicKey, mr) if err != nil { return convertSignatureError(err) } diff --git a/politeiad/plugins/user/user.go b/politeiad/plugins/user/user.go index e0befc47c..49a7334d6 100644 --- a/politeiad/plugins/user/user.go +++ b/politeiad/plugins/user/user.go @@ -61,8 +61,10 @@ var ( // UserMetadata contains user metadata about a politeiad record. It is // generated by the server and saved to politeiad as a metadata stream. // -// Signature is the client signature of the record merkle root. The merkle root -// is the ordered merkle root of all user submitted politeiad files. +// Signature is the client signature of the hex encoded record merkle root.The +// merkle root is the ordered merkle root of all user submitted politeiad +// files. The merkle root is hex encoded before being signed so that the +// signature is consistent with how politeiad signs the merkle root. type UserMetadata struct { UserID string `json:"userid"` // Author user ID PublicKey string `json:"publickey"` // Key used for signature diff --git a/politeiawww/cmd/pictl/proposal.go b/politeiawww/cmd/pictl/proposal.go index 137fbe3e3..6eec8f9e6 100644 --- a/politeiawww/cmd/pictl/proposal.go +++ b/politeiawww/cmd/pictl/proposal.go @@ -150,11 +150,12 @@ func signedMerkleRoot(files []rcv1.File, fid *identity.FullIdentity) (string, er for _, v := range files { digests = append(digests, v.Digest) } - mr, err := util.MerkleRoot(digests) + m, err := util.MerkleRoot(digests) if err != nil { return "", err } - sig := fid.SignMessage(mr[:]) + mr := hex.EncodeToString(m[:]) + sig := fid.SignMessage([]byte(mr)) return hex.EncodeToString(sig[:]), nil } From de04c0f4d82468bd0f2e11a5e0cb90506f166e63 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 7 Feb 2021 15:16:37 -0600 Subject: [PATCH 278/449] Vote policy, auth, start, inv. --- politeiawww/api/records/v1/v1.go | 2 +- politeiawww/api/ticketvote/v1/v1.go | 9 +- politeiawww/client/comments.go | 2 +- politeiawww/client/ticketvote.go | 185 ++++++++++++- .../pictl/{castballot.go => cmdcastballot.go} | 20 +- politeiawww/cmd/pictl/cmdcommentcensor.go | 2 +- politeiawww/cmd/pictl/cmdcommentnew.go | 2 +- politeiawww/cmd/pictl/cmdcomments.go | 2 +- politeiawww/cmd/pictl/cmdcommenttimestamps.go | 2 +- politeiawww/cmd/pictl/cmdcommentvote.go | 2 +- politeiawww/cmd/pictl/cmdcommentvotes.go | 2 +- politeiawww/cmd/pictl/cmdproposaldetails.go | 2 +- politeiawww/cmd/pictl/cmdproposaledit.go | 2 +- politeiawww/cmd/pictl/cmdproposalnew.go | 2 +- politeiawww/cmd/pictl/cmdproposals.go | 2 +- politeiawww/cmd/pictl/cmdproposalstatusset.go | 2 +- politeiawww/cmd/pictl/cmdvoteauthorize.go | 136 +++++++++ .../cmd/pictl/{votes.go => cmdvotedetails.go} | 24 +- politeiawww/cmd/pictl/cmdvoteinv.go | 44 +++ .../pictl/{votepolicy.go => cmdvotepolicy.go} | 6 +- .../{voteresults.go => cmdvoteresults.go} | 12 +- politeiawww/cmd/pictl/cmdvotestart.go | 257 ++++++++++++++++++ .../{votesummaries.go => cmdvotesummaries.go} | 12 +- ...votetimestamps.go => cmdvotetimestamps.go} | 11 +- politeiawww/cmd/pictl/help.go | 16 +- politeiawww/cmd/pictl/pictl.go | 25 +- politeiawww/cmd/pictl/voteauthorize.go | 113 -------- politeiawww/cmd/pictl/voteinventory.go | 43 --- politeiawww/cmd/pictl/votestart.go | 128 --------- politeiawww/cmd/pictl/votestartrunoff.go | 164 ----------- politeiawww/ticketvote/process.go | 2 +- 31 files changed, 703 insertions(+), 530 deletions(-) rename politeiawww/cmd/pictl/{castballot.go => cmdcastballot.go} (90%) create mode 100644 politeiawww/cmd/pictl/cmdvoteauthorize.go rename politeiawww/cmd/pictl/{votes.go => cmdvotedetails.go} (61%) create mode 100644 politeiawww/cmd/pictl/cmdvoteinv.go rename politeiawww/cmd/pictl/{votepolicy.go => cmdvotepolicy.go} (86%) rename politeiawww/cmd/pictl/{voteresults.go => cmdvoteresults.go} (72%) create mode 100644 politeiawww/cmd/pictl/cmdvotestart.go rename politeiawww/cmd/pictl/{votesummaries.go => cmdvotesummaries.go} (73%) rename politeiawww/cmd/pictl/{votetimestamps.go => cmdvotetimestamps.go} (86%) delete mode 100644 politeiawww/cmd/pictl/voteauthorize.go delete mode 100644 politeiawww/cmd/pictl/voteinventory.go delete mode 100644 politeiawww/cmd/pictl/votestart.go delete mode 100644 politeiawww/cmd/pictl/votestartrunoff.go diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go index 79a0e0395..779b48f99 100644 --- a/politeiawww/api/records/v1/v1.go +++ b/politeiawww/api/records/v1/v1.go @@ -313,7 +313,7 @@ type RecordsReply struct { } const ( - // TODO implement + // TODO implement Inventory pagnation InventoryPageSize = 60 ) diff --git a/politeiawww/api/ticketvote/v1/v1.go b/politeiawww/api/ticketvote/v1/v1.go index 0388236d1..9ae72430c 100644 --- a/politeiawww/api/ticketvote/v1/v1.go +++ b/politeiawww/api/ticketvote/v1/v1.go @@ -275,8 +275,8 @@ type Start struct { // StartReply is the reply to the Start command. type StartReply struct { - StartBlockHeight uint32 `json:"startblockheight"` StartBlockHash string `json:"startblockhash"` + StartBlockHeight uint32 `json:"startblockheight"` EndBlockHeight uint32 `json:"endblockheight"` EligibleTickets []string `json:"eligibletickets"` } @@ -484,6 +484,11 @@ type LinkedFromReply struct { LinkedFrom map[string][]string `json:"linkedfrom"` } +const ( + // TODO implement Inventory pagnation + InventoryPageSize = 60 +) + // Inventory requests the tokens of all records in the inventory, categorized // by vote status. type Inventory struct{} @@ -500,7 +505,7 @@ type Inventory struct{} // Statuses sorted by voting period end block height in descending order: // Started, Finished type InventoryReply struct { - Records map[string][]string `json:"records"` + Vetted map[string][]string `json:"vetted"` // BestBlock is the best block value that was used to prepare the // inventory. diff --git a/politeiawww/client/comments.go b/politeiawww/client/comments.go index d4ff0dab9..9d166773b 100644 --- a/politeiawww/client/comments.go +++ b/politeiawww/client/comments.go @@ -13,7 +13,7 @@ import ( "github.com/decred/politeia/util" ) -// CommentPolicy sends a pi v1 Policy request to politeiawww. +// CommentPolicy sends a comments v1 Policy request to politeiawww. func (c *Client) CommentPolicy() (*cmv1.PolicyReply, error) { resBody, err := c.makeReq(http.MethodGet, cmv1.APIRoute, cmv1.RoutePolicy, nil) diff --git a/politeiawww/client/ticketvote.go b/politeiawww/client/ticketvote.go index f16c88845..8e7c233b5 100644 --- a/politeiawww/client/ticketvote.go +++ b/politeiawww/client/ticketvote.go @@ -13,7 +13,7 @@ import ( "github.com/decred/politeia/util" ) -// TicketVotePolicy sends a pi v1 Policy request to politeiawww. +// TicketVotePolicy sends a ticketvote v1 Policy request to politeiawww. func (c *Client) TicketVotePolicy() (*tkv1.PolicyReply, error) { resBody, err := c.makeReq(http.MethodGet, tkv1.APIRoute, tkv1.RoutePolicy, nil) @@ -32,3 +32,186 @@ func (c *Client) TicketVotePolicy() (*tkv1.PolicyReply, error) { return &pr, nil } + +// TicketVoteAuthorize sends a ticketvote v1 Authorize request to politeiawww. +func (c *Client) TicketVoteAuthorize(a tkv1.Authorize) (*tkv1.AuthorizeReply, error) { + resBody, err := c.makeReq(http.MethodPost, + tkv1.APIRoute, tkv1.RouteAuthorize, a) + if err != nil { + return nil, err + } + + var ar tkv1.AuthorizeReply + err = json.Unmarshal(resBody, &ar) + if err != nil { + return nil, err + } + if c.verbose { + fmt.Printf("%v\n", util.FormatJSON(ar)) + } + + return &ar, nil +} + +// TicketVoteStart sends a ticketvote v1 Start request to politeiawww. +func (c *Client) TicketVoteStart(s tkv1.Start) (*tkv1.StartReply, error) { + resBody, err := c.makeReq(http.MethodPost, + tkv1.APIRoute, tkv1.RouteStart, s) + if err != nil { + return nil, err + } + + var sr tkv1.StartReply + err = json.Unmarshal(resBody, &sr) + if err != nil { + return nil, err + } + if c.verbose { + fmt.Printf("%v\n", util.FormatJSON(sr)) + } + + return &sr, nil +} + +// TicketVoteCastBallot sends a ticketvote v1 CastBallot request to +// politeiawww. +func (c *Client) TicketVoteCastBallot(cb tkv1.CastBallot) (*tkv1.CastBallotReply, error) { + resBody, err := c.makeReq(http.MethodPost, + tkv1.APIRoute, tkv1.RouteCastBallot, cb) + if err != nil { + return nil, err + } + + var cbr tkv1.CastBallotReply + err = json.Unmarshal(resBody, &cbr) + if err != nil { + return nil, err + } + if c.verbose { + fmt.Printf("%v\n", util.FormatJSON(cbr)) + } + + return &cbr, nil +} + +// TicketVoteDetails sends a ticketvote v1 Details request to politeiawww. +func (c *Client) TicketVoteDetails(d tkv1.Details) (*tkv1.DetailsReply, error) { + resBody, err := c.makeReq(http.MethodPost, + tkv1.APIRoute, tkv1.RouteDetails, d) + if err != nil { + return nil, err + } + + var dr tkv1.DetailsReply + err = json.Unmarshal(resBody, &dr) + if err != nil { + return nil, err + } + if c.verbose { + fmt.Printf("%v\n", util.FormatJSON(dr)) + } + + return &dr, nil +} + +// TicketVoteResults sends a ticketvote v1 Results request to politeiawww. +func (c *Client) TicketVoteResults(r tkv1.Results) (*tkv1.ResultsReply, error) { + resBody, err := c.makeReq(http.MethodPost, + tkv1.APIRoute, tkv1.RouteResults, r) + if err != nil { + return nil, err + } + + var rr tkv1.ResultsReply + err = json.Unmarshal(resBody, &rr) + if err != nil { + return nil, err + } + if c.verbose { + fmt.Printf("%v\n", util.FormatJSON(rr)) + } + + return &rr, nil +} + +// TicketVoteSummaries sends a ticketvote v1 Summaries request to politeiawww. +func (c *Client) TicketVoteSummaries(s tkv1.Summaries) (*tkv1.SummariesReply, error) { + resBody, err := c.makeReq(http.MethodPost, + tkv1.APIRoute, tkv1.RouteSummaries, s) + if err != nil { + return nil, err + } + + var sr tkv1.SummariesReply + err = json.Unmarshal(resBody, &sr) + if err != nil { + return nil, err + } + if c.verbose { + fmt.Printf("%v\n", util.FormatJSON(sr)) + } + + return &sr, nil +} + +// TicketVoteLinkedFrom sends a ticketvote v1 LinkedFrom request to +// politeiawww. +func (c *Client) TicketVoteLinkedFrom(lf tkv1.LinkedFrom) (*tkv1.LinkedFromReply, error) { + resBody, err := c.makeReq(http.MethodPost, + tkv1.APIRoute, tkv1.RouteLinkedFrom, lf) + if err != nil { + return nil, err + } + + var lfr tkv1.LinkedFromReply + err = json.Unmarshal(resBody, &lfr) + if err != nil { + return nil, err + } + if c.verbose { + fmt.Printf("%v\n", util.FormatJSON(lfr)) + } + + return &lfr, nil +} + +// TicketVoteInventory sends a ticketvote v1 Inventory request to politeiawww. +func (c *Client) TicketVoteInventory() (*tkv1.InventoryReply, error) { + resBody, err := c.makeReq(http.MethodPost, + tkv1.APIRoute, tkv1.RouteInventory, nil) + if err != nil { + return nil, err + } + + var ir tkv1.InventoryReply + err = json.Unmarshal(resBody, &ir) + if err != nil { + return nil, err + } + if c.verbose { + fmt.Printf("%v\n", util.FormatJSON(ir)) + } + + return &ir, nil +} + +// TicketVoteTimestamps sends a ticketvote v1 Timestamps request to +// politeiawww. +func (c *Client) TicketVoteTimestamps(t tkv1.Timestamps) (*tkv1.TimestampsReply, error) { + resBody, err := c.makeReq(http.MethodPost, + tkv1.APIRoute, tkv1.RouteTimestamps, t) + if err != nil { + return nil, err + } + + var tr tkv1.TimestampsReply + err = json.Unmarshal(resBody, &tr) + if err != nil { + return nil, err + } + if c.verbose { + fmt.Printf("%v\n", util.FormatJSON(tr)) + } + + return &tr, nil +} diff --git a/politeiawww/cmd/pictl/castballot.go b/politeiawww/cmd/pictl/cmdcastballot.go similarity index 90% rename from politeiawww/cmd/pictl/castballot.go rename to politeiawww/cmd/pictl/cmdcastballot.go index fd91c01ce..62da9cfdf 100644 --- a/politeiawww/cmd/pictl/castballot.go +++ b/politeiawww/cmd/pictl/cmdcastballot.go @@ -1,11 +1,11 @@ -// Copyright (c) 2017-2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. package main -// castBallotCmd casts a ballot of votes for the specified proposal. -type castBallotCmd struct { +// cmdCastBallot casts a ballot of votes. +type cmdCastBallot struct { Args struct { Token string `positional-arg-name:"token"` VoteID string `positional-arg-name:"voteid"` @@ -14,10 +14,10 @@ type castBallotCmd struct { } /* -// Execute executes the castBallotCmd command. +// Execute executes the cmdCastBallot command. // // This function satisfies the go-flags Commander interface. -func (c *castBallotCmd) Execute(args []string) error { +func (c *cmdCastBallot) Execute(args []string) error { token := c.Args.Token voteID := c.Args.VoteID @@ -204,17 +204,17 @@ func (c *castBallotCmd) Execute(args []string) error { } */ -// castBallotHelpMsg is the help command message. +// castBallotHelpMsg is printed to stdout by the help command. const castBallotHelpMsg = `castballot "token" "voteid" -Cast a ballot of ticket votes for a proposal. This command will only work when -on testnet and when running dcrwallet locally on the default port. +Cast a ballot of dcr ticket votes. This command will only work when on testnet +and when running dcrwallet locally on the default port. Arguments: 1. token (string, optional) Proposal censorship token 2. voteid (string, optional) Vote option ID (e.g. yes) Flags: - --password (string, optional) Wallet password. You will be prompted for the - password if one is not provided. + --password (string, optional) Wallet password. You will be prompted for the + password if one is not provided. ` diff --git a/politeiawww/cmd/pictl/cmdcommentcensor.go b/politeiawww/cmd/pictl/cmdcommentcensor.go index 8d489274c..45c56742e 100644 --- a/politeiawww/cmd/pictl/cmdcommentcensor.go +++ b/politeiawww/cmd/pictl/cmdcommentcensor.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2021 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/politeiawww/cmd/pictl/cmdcommentnew.go b/politeiawww/cmd/pictl/cmdcommentnew.go index 187da0273..14bdef4a4 100644 --- a/politeiawww/cmd/pictl/cmdcommentnew.go +++ b/politeiawww/cmd/pictl/cmdcommentnew.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2021 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/politeiawww/cmd/pictl/cmdcomments.go b/politeiawww/cmd/pictl/cmdcomments.go index 45d99bf08..026128a53 100644 --- a/politeiawww/cmd/pictl/cmdcomments.go +++ b/politeiawww/cmd/pictl/cmdcomments.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2021 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/politeiawww/cmd/pictl/cmdcommenttimestamps.go b/politeiawww/cmd/pictl/cmdcommenttimestamps.go index 3102d3a5e..30c424586 100644 --- a/politeiawww/cmd/pictl/cmdcommenttimestamps.go +++ b/politeiawww/cmd/pictl/cmdcommenttimestamps.go @@ -106,7 +106,7 @@ func convertCommentTimestamp(t cmv1.Timestamp) backend.Timestamp { } } -// commentTimestampsHelpMsg is printed to stdout by the help command +// commentTimestampsHelpMsg is printed to stdout by the help command. const commentTimestampsHelpMsg = `commenttimestamps [flags] "token" commentIDs Fetch the timestamps for a record's comments. The timestamp contains all diff --git a/politeiawww/cmd/pictl/cmdcommentvote.go b/politeiawww/cmd/pictl/cmdcommentvote.go index ea0b7615f..b2de528e1 100644 --- a/politeiawww/cmd/pictl/cmdcommentvote.go +++ b/politeiawww/cmd/pictl/cmdcommentvote.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2020 The Decred developers +// Copyright (c) 2020-2020 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/politeiawww/cmd/pictl/cmdcommentvotes.go b/politeiawww/cmd/pictl/cmdcommentvotes.go index 2bd828ed9..bfd2efee4 100644 --- a/politeiawww/cmd/pictl/cmdcommentvotes.go +++ b/politeiawww/cmd/pictl/cmdcommentvotes.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/politeiawww/cmd/pictl/cmdproposaldetails.go b/politeiawww/cmd/pictl/cmdproposaldetails.go index f804e26df..b52f60642 100644 --- a/politeiawww/cmd/pictl/cmdproposaldetails.go +++ b/politeiawww/cmd/pictl/cmdproposaldetails.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2021 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/politeiawww/cmd/pictl/cmdproposaledit.go b/politeiawww/cmd/pictl/cmdproposaledit.go index 4abae36ce..54f0dfa4b 100644 --- a/politeiawww/cmd/pictl/cmdproposaledit.go +++ b/politeiawww/cmd/pictl/cmdproposaledit.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2021 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/politeiawww/cmd/pictl/cmdproposalnew.go b/politeiawww/cmd/pictl/cmdproposalnew.go index c8d512b78..47284bacf 100644 --- a/politeiawww/cmd/pictl/cmdproposalnew.go +++ b/politeiawww/cmd/pictl/cmdproposalnew.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2021 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/politeiawww/cmd/pictl/cmdproposals.go b/politeiawww/cmd/pictl/cmdproposals.go index 7966b43b7..7b415baf9 100644 --- a/politeiawww/cmd/pictl/cmdproposals.go +++ b/politeiawww/cmd/pictl/cmdproposals.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/politeiawww/cmd/pictl/cmdproposalstatusset.go b/politeiawww/cmd/pictl/cmdproposalstatusset.go index 13a1d9cd4..1cee81b8a 100644 --- a/politeiawww/cmd/pictl/cmdproposalstatusset.go +++ b/politeiawww/cmd/pictl/cmdproposalstatusset.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2021 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/politeiawww/cmd/pictl/cmdvoteauthorize.go b/politeiawww/cmd/pictl/cmdvoteauthorize.go new file mode 100644 index 000000000..a8b51d9f3 --- /dev/null +++ b/politeiawww/cmd/pictl/cmdvoteauthorize.go @@ -0,0 +1,136 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "encoding/hex" + "fmt" + "strconv" + + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" + pclient "github.com/decred/politeia/politeiawww/client" + "github.com/decred/politeia/politeiawww/cmd/shared" + "github.com/decred/politeia/util" +) + +// cmdVoteAuthorize authorizes a ticket vote or revokes a previous +// authorization. +type cmdVoteAuthorize struct { + Args struct { + Token string `positional-arg-name:"token" required:"true"` + Action string `positional-arg-name:"action"` + } `positional-args:"true"` +} + +// Execute executes the cmdVoteAuthorize command. +// +// This function satisfies the go-flags Commander interface. +func (c *cmdVoteAuthorize) Execute(args []string) error { + // Verify action + var action tkv1.AuthActionT + switch c.Args.Action { + case "authorize": + action = tkv1.AuthActionAuthorize + case "revoke": + action = tkv1.AuthActionRevoke + case "": + // Default to authorize + action = tkv1.AuthActionAuthorize + default: + return fmt.Errorf("Invalid action; \n%v", voteAuthorizeHelpMsg) + } + + // Verify user identity. An identity is required to sign the vote + // authorization. + if cfg.Identity == nil { + return shared.ErrUserIdentityNotFound + } + + // Setup client + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Cookies: cfg.Cookies, + HeaderCSRF: cfg.CSRF, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, + } + pc, err := pclient.New(cfg.Host, opts) + if err != nil { + return err + } + + // Get record version + d := rcv1.Details{ + State: rcv1.RecordStateVetted, + Token: c.Args.Token, + } + r, err := pc.RecordDetails(d) + if err != nil { + return err + } + version, err := strconv.ParseUint(r.Version, 10, 64) + if err != nil { + return err + } + + // Setup request + msg := c.Args.Token + r.Version + string(action) + sig := cfg.Identity.SignMessage([]byte(msg)) + a := tkv1.Authorize{ + Token: c.Args.Token, + Version: uint32(version), + Action: action, + PublicKey: cfg.Identity.Public.String(), + Signature: hex.EncodeToString(sig[:]), + } + + // Send request + ar, err := pc.TicketVoteAuthorize(a) + if err != nil { + return err + } + + // Verify receipt + vr, err := client.Version() + if err != nil { + return err + } + serverID, err := util.IdentityFromString(vr.PubKey) + if err != nil { + return err + } + s, err := util.ConvertSignature(ar.Receipt) + if err != nil { + return err + } + if !serverID.VerifyMessage([]byte(a.Signature), s) { + return fmt.Errorf("could not verify receipt") + } + + // Print receipt + printf("Token : %v\n", a.Token) + printf("Action : %v\n", a.Action) + printf("Timestamp: %v\n", timestampFromUnix(ar.Timestamp)) + printf("Receipt : %v\n", ar.Receipt) + + return nil +} + +// voteAuthorizeHelpMsg is printed to stdout by the help command. +const voteAuthorizeHelpMsg = `voteauthorize "token" "action" + +Authorize or revoke a ticket vote. + +If an action is not provided this command defaults to authorizing a ticket +vote. The user must be the record author. + +Valid actions: + authorize authorize a vote + revoke revoke a previous authorization + +Arguments: +1. token (string, required) Record token. +2. action (string, optional) Authorize vote action.` diff --git a/politeiawww/cmd/pictl/votes.go b/politeiawww/cmd/pictl/cmdvotedetails.go similarity index 61% rename from politeiawww/cmd/pictl/votes.go rename to politeiawww/cmd/pictl/cmdvotedetails.go index 7ea7908c2..6239438c4 100644 --- a/politeiawww/cmd/pictl/votes.go +++ b/politeiawww/cmd/pictl/cmdvotedetails.go @@ -1,21 +1,21 @@ -// Copyright (c) 2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. package main -// votesCmd retrieves vote details for the specified proposals. -type votesCmd struct { +// cmdVoteDetails retrieves vote details for the specified proposals. +type cmdVoteDetails struct { Args struct { - Tokens []string `positional-arg-name:"token"` + Token string `positional-arg-name:"token"` } `positional-args:"true" required:"true"` } /* -// Execute executes the votesCmd command. +// Execute executes the cmdVoteDetails command. // // This function satisfies the go-flags Commander interface. -func (c *votesCmd) Execute(args []string) error { +func (c *cmdVoteDetails) Execute(args []string) error { // Setup request v := pi.Votes{ Tokens: c.Args.Tokens, @@ -53,14 +53,10 @@ func (c *votesCmd) Execute(args []string) error { } */ -// votesHelpMsg is the help command message. -const votesHelpMsg = `votes "tokens" +// voteDetailsHelpMsg is printed to stdout by the help command. +const voteDetailsHelpMsg = `votedetails "token" -Fetch the vote details for the provided proposal tokens. +Fetch the vote details for a proposal. Arguments: -1. tokens (string, required) Proposal censorship tokens - -Example usage: -$ piwww votes cda97ace0a4765140000 71dd3a110500fb6a0000 -` +1. token (string, required) Proposal censorship token.` diff --git a/politeiawww/cmd/pictl/cmdvoteinv.go b/politeiawww/cmd/pictl/cmdvoteinv.go new file mode 100644 index 000000000..9f2492126 --- /dev/null +++ b/politeiawww/cmd/pictl/cmdvoteinv.go @@ -0,0 +1,44 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import pclient "github.com/decred/politeia/politeiawww/client" + +// cmdVoteInv retrieves the censorship record tokens of all public, +// non-abandoned records in the inventory, categorized by their vote status. +type cmdVoteInv struct{} + +// Execute executes the cmdVoteInv command. +// +// This function satisfies the go-flags Commander interface. +func (c *cmdVoteInv) Execute(args []string) error { + // Setup client + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, + } + pc, err := pclient.New(cfg.Host, opts) + if err != nil { + return err + } + + // Get vote inventory + ir, err := pc.TicketVoteInventory() + if err != nil { + return err + } + + // Print inventory + printJSON(ir) + + return nil +} + +// voteInvHelpMsg is printed to stdout by the help command. +const voteInvHelpMsg = `voteinv + +Fetch the censorship record tokens of all public, non-abandoned records, +categorized by their vote status.` diff --git a/politeiawww/cmd/pictl/votepolicy.go b/politeiawww/cmd/pictl/cmdvotepolicy.go similarity index 86% rename from politeiawww/cmd/pictl/votepolicy.go rename to politeiawww/cmd/pictl/cmdvotepolicy.go index 5e433370f..de97ffa32 100644 --- a/politeiawww/cmd/pictl/votepolicy.go +++ b/politeiawww/cmd/pictl/cmdvotepolicy.go @@ -9,12 +9,12 @@ import ( ) // votePolicy retrieves the ticketvote API policy. -type votePolicyCmd struct{} +type cmdVotePolicy struct{} -// Execute executes the votePolicyCmd command. +// Execute executes the cmdVotePolicy command. // // This function satisfies the go-flags Commander interface. -func (cmd *votePolicyCmd) Execute(args []string) error { +func (c *cmdVotePolicy) Execute(args []string) error { // Setup client opts := pclient.Opts{ HTTPSCert: cfg.HTTPSCert, diff --git a/politeiawww/cmd/pictl/voteresults.go b/politeiawww/cmd/pictl/cmdvoteresults.go similarity index 72% rename from politeiawww/cmd/pictl/voteresults.go rename to politeiawww/cmd/pictl/cmdvoteresults.go index 5dcbe052a..e27c38621 100644 --- a/politeiawww/cmd/pictl/voteresults.go +++ b/politeiawww/cmd/pictl/cmdvoteresults.go @@ -1,21 +1,21 @@ -// Copyright (c) 2017-2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. package main -// voteResultsCmd retreives the cast votes for the provided proposal. -type voteResultsCmd struct { +// cmdVoteResults retreives the cast votes for the provided proposal. +type cmdVoteResults struct { Args struct { Token string `positional-arg-name:"token"` } `positional-args:"true" required:"true"` } /* -// Execute executes the voteResultsCmd command. +// Execute executes the cmdVoteResults command. // // This function satisfies the go-flags Commander interface. -func (cmd *voteResultsCmd) Execute(args []string) error { +func (c *cmdVoteResults) Execute(args []string) error { // Setup request vr := pi.VoteResults{ Token: cmd.Args.Token, @@ -40,7 +40,7 @@ func (cmd *voteResultsCmd) Execute(args []string) error { } */ -// voteResultsHelpMsg is the help command message. +// voteResultsHelpMsg is printed to stdout by the help command. const voteResultsHelpMsg = `voteresults "token" Fetch vote results for the provided proposal. diff --git a/politeiawww/cmd/pictl/cmdvotestart.go b/politeiawww/cmd/pictl/cmdvotestart.go new file mode 100644 index 000000000..eba12f1c2 --- /dev/null +++ b/politeiawww/cmd/pictl/cmdvotestart.go @@ -0,0 +1,257 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "strconv" + + "github.com/decred/politeia/politeiad/plugins/ticketvote" + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" + pclient "github.com/decred/politeia/politeiawww/client" + "github.com/decred/politeia/politeiawww/cmd/shared" + "github.com/decred/politeia/util" +) + +// cmdVoteStart starts the voting period on a record. +// +// QuorumPercentage and PassPercentage are strings and not uint32 so that a +// value of 0 can be passed in and not be overwritten by the defaults. This is +// sometimes desirable when testing. +type cmdVoteStart struct { + Args struct { + Token string `positional-arg-name:"token" required:"true"` + Duration uint32 `positional-arg-name:"duration"` + QuorumPercentage uint32 `positional-arg-name:"quorumpercentage"` + PassPercentage uint32 `positional-arg-name:"passpercentage"` + } `positional-args:"true"` + + // Runoff is used to indicate the vote is a runoff vote and the + // provided token is the parent token of the runoff vote. + Runoff bool `long:"random" optional:"true"` +} + +func voteStartStandard(token string, duration, quorum, pass uint32, pc *pclient.Client) (*tkv1.StartReply, error) { + // Get record version + d := rcv1.Details{ + State: rcv1.RecordStateVetted, + Token: token, + } + r, err := pc.RecordDetails(d) + if err != nil { + return nil, err + } + version, err := strconv.ParseUint(r.Version, 10, 64) + if err != nil { + return nil, err + } + + // Setup request + vp := tkv1.VoteParams{ + Token: token, + Version: uint32(version), + Type: tkv1.VoteTypeStandard, + Mask: 0x03, + Duration: duration, + QuorumPercentage: quorum, + PassPercentage: pass, + Options: []tkv1.VoteOption{ + { + ID: tkv1.VoteOptionIDApprove, + Description: "Approve the proposal", + Bit: 0x01, + }, + { + ID: tkv1.VoteOptionIDReject, + Description: "Reject the proposal", + Bit: 0x02, + }, + }, + } + vpb, err := json.Marshal(vp) + if err != nil { + return nil, err + } + msg := hex.EncodeToString(util.Digest(vpb)) + b := cfg.Identity.SignMessage([]byte(msg)) + signature := hex.EncodeToString(b[:]) + s := tkv1.Start{ + Starts: []tkv1.StartDetails{ + { + Params: vp, + PublicKey: cfg.Identity.Public.String(), + Signature: signature, + }, + }, + } + + // Send request + return pc.TicketVoteStart(s) +} + +func voteStartRunoff(parentToken string, duration, quorum, pass uint32, pc *pclient.Client) (*tkv1.StartReply, error) { + // Get runoff vote submissions + lf := tkv1.LinkedFrom{ + Tokens: []string{parentToken}, + } + lfr, err := pc.TicketVoteLinkedFrom(lf) + if err != nil { + return nil, fmt.Errorf("TicketVoteLinkedFrom: %v", err) + } + linkedFrom, ok := lfr.LinkedFrom[parentToken] + if !ok { + return nil, fmt.Errorf("linked from not found %v", parentToken) + } + + // Prepare start details for each submission + starts := make([]tkv1.StartDetails, 0, len(linkedFrom)) + for _, v := range linkedFrom { + // Get record + d := rcv1.Details{ + State: rcv1.RecordStateVetted, + Token: v, + } + r, err := pc.RecordDetails(d) + if err != nil { + return nil, fmt.Errorf("RecordDetails %v: %v", v, err) + } + version, err := strconv.ParseUint(r.Version, 10, 64) + if err != nil { + return nil, err + } + + // Don't include the record if it has been abandoned. + if r.Status == rcv1.RecordStatusArchived { + continue + } + + // Setup vote params + vp := tkv1.VoteParams{ + Token: r.CensorshipRecord.Token, + Version: uint32(version), + Type: tkv1.VoteTypeRunoff, + Mask: 0x03, // bit 0 no, bit 1 yes + Duration: duration, + QuorumPercentage: quorum, + PassPercentage: pass, + Options: []tkv1.VoteOption{ + { + ID: ticketvote.VoteOptionIDApprove, + Description: "Approve the proposal", + Bit: 0x01, + }, + { + ID: ticketvote.VoteOptionIDReject, + Description: "Reject the proposal", + Bit: 0x02, + }, + }, + Parent: parentToken, + } + vpb, err := json.Marshal(vp) + if err != nil { + return nil, err + } + msg := hex.EncodeToString(util.Digest(vpb)) + sig := cfg.Identity.SignMessage([]byte(msg)) + starts = append(starts, tkv1.StartDetails{ + Params: vp, + PublicKey: cfg.Identity.Public.String(), + Signature: hex.EncodeToString(sig[:]), + }) + } + + // Send request + s := tkv1.Start{ + Starts: starts, + } + return pc.TicketVoteStart(s) +} + +// Execute executes the cmdVoteStart command. +// +// This function satisfies the go-flags Commander interface. +func (c *cmdVoteStart) Execute(args []string) error { + token := c.Args.Token + + // Verify user identity. An identity is required to sign the vote + // start. + if cfg.Identity == nil { + return shared.ErrUserIdentityNotFound + } + + // Setup client + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Cookies: cfg.Cookies, + HeaderCSRF: cfg.CSRF, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, + } + pc, err := pclient.New(cfg.Host, opts) + if err != nil { + return err + } + + // Setup vote params + var ( + // Default values + duration uint32 = 2016 + quorum uint32 = 20 + pass uint32 = 60 + ) + if c.Args.Duration != 0 { + duration = c.Args.Duration + } + if c.Args.QuorumPercentage != 0 { + quorum = c.Args.QuorumPercentage + } + if c.Args.PassPercentage != 0 { + pass = c.Args.PassPercentage + } + + var sr *tkv1.StartReply + if c.Runoff { + sr, err = voteStartRunoff(token, duration, quorum, pass, pc) + if err != nil { + return err + } + } else { + sr, err = voteStartStandard(token, duration, quorum, pass, pc) + if err != nil { + return err + } + } + + // Print reply + printf("StartBlockHash : %v\n", sr.StartBlockHash) + printf("StartBlockHeight: %v\n", sr.StartBlockHeight) + printf("EndBlockHeight : %v\n", sr.EndBlockHeight) + + return nil +} + +// voteStartHelpMsg is printed to stdout by the help command. +var voteStartHelpMsg = `votestart + +Start the voting period for a proposal. Requires admin privileges. + +If the vote is a runoff vote then the --runoff flag must be used. The provided +token should be the parent token of the runoff vote. + +Arguments: +1. token (string, required) Proposal censorship token +2. duration (uint32, optional) Duration of vote in blocks + (default: 2016) +3. quorumpercentage (uint32, optional) Percent of total votes required to + reach a quorum (default: 10) +4. passpercentage (uint32, optional) Percent of cast votes required for + vote to be approved (default: 60) +Flags: + --runoff (bool, optional) Start a runoff vote. +` diff --git a/politeiawww/cmd/pictl/votesummaries.go b/politeiawww/cmd/pictl/cmdvotesummaries.go similarity index 73% rename from politeiawww/cmd/pictl/votesummaries.go rename to politeiawww/cmd/pictl/cmdvotesummaries.go index 16d6d868e..bbbe5ed28 100644 --- a/politeiawww/cmd/pictl/votesummaries.go +++ b/politeiawww/cmd/pictl/cmdvotesummaries.go @@ -1,21 +1,21 @@ -// Copyright (c) 2017-2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. package main -// voteSummariesCmd retrieves the vote summaries for the provided proposals. -type voteSummariesCmd struct { +// cmdVoteSummaries retrieves the vote summaries for the provided proposals. +type cmdVoteSummaries struct { Args struct { Tokens []string `positional-arg-name:"token"` } `positional-args:"true" required:"true"` } /* -// Execute executes the voteSummariesCmd command. +// Execute executes the cmdVoteSummaries command. // // This function satisfies the go-flags Commander interface. -func (cmd *voteSummariesCmd) Execute(args []string) error { +func (c *cmdVoteSummaries) Execute(args []string) error { // Setup request vs := pi.VoteSummaries{ Tokens: cmd.Args.Tokens, @@ -40,7 +40,7 @@ func (cmd *voteSummariesCmd) Execute(args []string) error { } */ -// voteSummariesHelpMsg is the help command message. +// voteSummariesHelpMsg is printed to stdout by the help command. const voteSummariesHelpMsg = `votesummaries "tokens" Fetch the vote summaries for the provided proposal tokens. diff --git a/politeiawww/cmd/pictl/votetimestamps.go b/politeiawww/cmd/pictl/cmdvotetimestamps.go similarity index 86% rename from politeiawww/cmd/pictl/votetimestamps.go rename to politeiawww/cmd/pictl/cmdvotetimestamps.go index 6e1883e74..58cb7daa3 100644 --- a/politeiawww/cmd/pictl/votetimestamps.go +++ b/politeiawww/cmd/pictl/cmdvotetimestamps.go @@ -10,17 +10,17 @@ import ( tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" ) -// voteTimestampsCmd retrieves the timestamps for a politeiawww ticket vote. -type voteTimestampsCmd struct { +// cmdVoteTimestamps retrieves the timestamps for a politeiawww ticket vote. +type cmdVoteTimestamps struct { Args struct { Token string `positional-arg-name:"token" required:"true"` } `positional-args:"true"` } -// Execute executes the voteTimestampsCmd command. +// Execute executes the cmdVoteTimestamps command. // // This function satisfies the go-flags Commander interface. -func (c *voteTimestampsCmd) Execute(args []string) error { +func (c *cmdVoteTimestamps) Execute(args []string) error { /* // Setup request t := tkv1.Timestamps{ @@ -92,7 +92,8 @@ func convertVoteTimestamp(t tkv1.Timestamp) backend.Timestamp { } } -const voteTimestampsHelpMsg = `votetimestamps [flags] "token" +// voteTimestampsHelpMsg is printed to stdout by the help command. +const voteTimestampsHelpMsg = `votetimestamps "token" Fetch the timestamps for a ticket vote. This includes timestamps for all authorizations, the vote details, and all cast votes. diff --git a/politeiawww/cmd/pictl/help.go b/politeiawww/cmd/pictl/help.go index f089619d5..0b6d047af 100644 --- a/politeiawww/cmd/pictl/help.go +++ b/politeiawww/cmd/pictl/help.go @@ -75,18 +75,18 @@ func (cmd *helpCmd) Execute(args []string) error { fmt.Printf("%s\n", voteAuthorizeHelpMsg) case "votestart": fmt.Printf("%s\n", voteStartHelpMsg) - case "votestartrunoff": - fmt.Printf("%s\n", voteStartRunoffHelpMsg) - case "voteballot": - fmt.Printf("%s\n", voteInventoryHelpMsg) - case "votes": - fmt.Printf("%s\n", votesHelpMsg) + case "castballot": + fmt.Printf("%s\n", castBallotHelpMsg) + case "votedetails": + fmt.Printf("%s\n", voteDetailsHelpMsg) case "voteresults": fmt.Printf("%s\n", voteResultsHelpMsg) case "votesummaries": fmt.Printf("%s\n", voteSummariesHelpMsg) - case "voteinventory": - fmt.Printf("%s\n", voteInventoryHelpMsg) + case "voteinv": + fmt.Printf("%s\n", voteInvHelpMsg) + case "votetimestamps": + fmt.Printf("%s\n", voteTimestampsHelpMsg) // User commands case "usernew": diff --git a/politeiawww/cmd/pictl/pictl.go b/politeiawww/cmd/pictl/pictl.go index d040672e1..b7a0d1b08 100644 --- a/politeiawww/cmd/pictl/pictl.go +++ b/politeiawww/cmd/pictl/pictl.go @@ -84,16 +84,15 @@ type pictl struct { CommentTimestamps cmdCommentTimestamps `command:"commenttimestamps"` // Vote commands - VotePolicy votePolicyCmd `command:"votepolicy"` - VoteAuthorize voteAuthorizeCmd `command:"voteauthorize"` - VoteStart voteStartCmd `command:"votestart"` - VoteStartRunoff voteStartRunoffCmd `command:"votestartrunoff"` - CastBallot castBallotCmd `command:"castballot"` - Votes votesCmd `command:"votes"` - VoteResults voteResultsCmd `command:"voteresults"` - VoteSummaries voteSummariesCmd `command:"votesummaries"` - VoteInventory voteInventoryCmd `command:"voteinv"` - VoteTimestamps voteTimestampsCmd `command:"votetimestamps"` + VotePolicy cmdVotePolicy `command:"votepolicy"` + VoteAuthorize cmdVoteAuthorize `command:"voteauthorize"` + VoteStart cmdVoteStart `command:"votestart"` + CastBallot cmdCastBallot `command:"castballot"` + Votes cmdVoteDetails `command:"votedetails"` + VoteResults cmdVoteResults `command:"voteresults"` + VoteSummaries cmdVoteSummaries `command:"votesummaries"` + VoteInv cmdVoteInv `command:"voteinv"` + VoteTimestamps cmdVoteTimestamps `command:"votetimestamps"` // Websocket commands Subscribe subscribeCmd `command:"subscribe"` @@ -108,7 +107,7 @@ const helpMsg = `Application Options: --host= politeiawww host -j, --json Print raw JSON output --version Display version information and exit - --skipverify Skip verifying the server's certifcate chain and host name + --skipverify Skip verifying the server's certificate chain and host name -v, --verbose Print verbose output --silent Suppress all output @@ -165,12 +164,12 @@ Vote commands votepolicy (public) Get the ticketvote api policy voteauthorize (user) Authorize a proposal vote votestart (admin) Start a proposal vote - votestartrunoff (admin) Start a runoff vote castballot (public) Cast a ballot of votes - votes (public) Get vote details + votedetails (public) Get details for a vote voteresults (public) Get full vote results votesummaries (public) Get vote summaries voteinv (public) Get proposal inventory by vote status + votetimestamps (public) Get vote timestamps Websocket commands subscribe (public) Subscribe/unsubscribe to websocket event diff --git a/politeiawww/cmd/pictl/voteauthorize.go b/politeiawww/cmd/pictl/voteauthorize.go deleted file mode 100644 index c2a7b6db0..000000000 --- a/politeiawww/cmd/pictl/voteauthorize.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) 2017-2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -// voteAuthorizeCmd authorizes a proposal vote or revokes a previous vote -// authorization. -type voteAuthorizeCmd struct { - Args struct { - Token string `positional-arg-name:"token" required:"true"` - Action string `positional-arg-name:"action"` - } `positional-args:"true"` -} - -/* -// Execute executes the voteAuthorizeCmd command. -// -// This function satisfies the go-flags Commander interface. -func (cmd *voteAuthorizeCmd) Execute(args []string) error { - token := cmd.Args.Token - - // Verify user identity - if cfg.Identity == nil { - return shared.ErrUserIdentityNotFound - } - - // Verify action - var action pi.VoteAuthActionT - switch cmd.Args.Action { - case "authorize": - action = pi.VoteAuthActionAuthorize - case "revoke": - action = pi.VoteAuthActionRevoke - case "": - // Default to authorize - action = pi.VoteAuthActionAuthorize - default: - return fmt.Errorf("Invalid action; \n%v", voteAuthorizeHelpMsg) - } - - // Get proposal version - pr, err := proposalRecordLatest(pi.PropStateVetted, token) - if err != nil { - return fmt.Errorf("proposalRecordLatest: %v", err) - } - // Parse version - version, err := strconv.ParseUint(pr.Version, 10, 32) - if err != nil { - return err - } - - // Setup request - msg := token + pr.Version + string(action) - b := cfg.Identity.SignMessage([]byte(msg)) - signature := hex.EncodeToString(b[:]) - va := pi.VoteAuthorize{ - Token: token, - Version: uint32(version), - Action: action, - PublicKey: cfg.Identity.Public.String(), - Signature: signature, - } - - // Send request. The request and response details are printed to - // the console. - err = shared.PrintJSON(va) - if err != nil { - return err - } - ar, err := client.VoteAuthorize(va) - if err != nil { - return err - } - err = shared.PrintJSON(ar) - if err != nil { - return err - } - - // Verify receipt - vr, err := client.Version() - if err != nil { - return err - } - serverID, err := util.IdentityFromString(vr.PubKey) - if err != nil { - return err - } - s, err := util.ConvertSignature(ar.Receipt) - if err != nil { - return err - } - if !serverID.VerifyMessage([]byte(signature), s) { - return fmt.Errorf("could not verify receipt") - } - - return nil -} -*/ - -// voteAuthorizeHelpMsg is the help command message. -const voteAuthorizeHelpMsg = `voteauthorize "token" "action" - -Authorize or revoke a proposal vote. Must be proposal author. - -Valid actions: - authorize authorize a vote - revoke revoke a previous authorization - -Arguments: -1. token (string, required) Proposal censorship token -2. action (string, optional) Authorize vote actions (default: authorize) -` diff --git a/politeiawww/cmd/pictl/voteinventory.go b/politeiawww/cmd/pictl/voteinventory.go deleted file mode 100644 index 9338a31fe..000000000 --- a/politeiawww/cmd/pictl/voteinventory.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) 2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -// voteInventoryCmd retrieves the censorship record tokens of all public, -// non-abandoned proposals in inventory categorized by their vote status. -type voteInventoryCmd struct{} - -/* -// Execute executes the voteInventoryCmd command. -// -// This function satisfies the go-flags Commander interface. -func (cmd *voteInventoryCmd) Execute(args []string) error { - // Setup request - vi := pi.VoteInventory{} - - // Send request. The request and response details are printed to - // the console. - err := shared.PrintJSON(vi) - if err != nil { - return err - } - vir, err := client.VoteInventory(vi) - if err != nil { - return err - } - err = shared.PrintJSON(vir) - if err != nil { - return err - } - - return nil -} -*/ - -// voteInventoryHelpMsg is the command help message. -const voteInventoryHelpMsg = `voteinv - -Fetch the censorship record tokens for all proposals, categorized by their -vote status. The unvetted tokens are only returned if the logged in user is an -admin.` diff --git a/politeiawww/cmd/pictl/votestart.go b/politeiawww/cmd/pictl/votestart.go deleted file mode 100644 index 631a4f473..000000000 --- a/politeiawww/cmd/pictl/votestart.go +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright (c) 2017-2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -// voteStartCmd starts the voting period on the specified proposal. -type voteStartCmd struct { - Args struct { - Token string `positional-arg-name:"token" required:"true"` - Duration uint32 `positional-arg-name:"duration"` - QuorumPercentage uint32 `positional-arg-name:"quorumpercentage"` - PassPercentage uint32 `positional-arg-name:"passpercentage"` - } `positional-args:"true"` -} - -/* -// Execute executes the voteStartCmd command. -// -// This function satisfies the go-flags Commander interface. -func (cmd *voteStartCmd) Execute(args []string) error { - token := cmd.Args.Token - - // Verify user identity - if cfg.Identity == nil { - return shared.ErrUserIdentityNotFound - } - - // Get proposal version - pr, err := proposalRecordLatest(pi.PropStateVetted, token) - if err != nil { - return fmt.Errorf("proposalRecordLatest: %v", err) - } - version, err := strconv.ParseUint(pr.Version, 10, 32) - if err != nil { - return err - } - - // Setup vote params - var ( - // Default values - duration uint32 = 2016 - quorum uint32 = 20 - pass uint32 = 60 - ) - if cmd.Args.Duration != 0 { - duration = cmd.Args.Duration - } - if cmd.Args.QuorumPercentage != 0 { - quorum = cmd.Args.QuorumPercentage - } - if cmd.Args.PassPercentage != 0 { - pass = cmd.Args.PassPercentage - } - - // Setup request - vote := pi.VoteParams{ - Token: token, - Version: uint32(version), - Type: pi.VoteTypeStandard, - Mask: 0x03, - Duration: duration, - QuorumPercentage: quorum, - PassPercentage: pass, - Options: []pi.VoteOption{ - { - ID: pi.VoteOptionIDApprove, - Description: "Approve proposal", - Bit: 0x01, - }, - { - ID: pi.VoteOptionIDReject, - Description: "Don't approve proposal", - Bit: 0x02, - }, - }, - } - vb, err := json.Marshal(vote) - if err != nil { - return err - } - msg := hex.EncodeToString(util.Digest(vb)) - b := cfg.Identity.SignMessage([]byte(msg)) - signature := hex.EncodeToString(b[:]) - vs := pi.VoteStart{ - Starts: []pi.StartDetails{ - { - Params: vote, - PublicKey: cfg.Identity.Public.String(), - Signature: signature, - }, - }, - } - - // Send request. The request and response details are printed to - // the console. - err = shared.PrintJSON(vs) - if err != nil { - return err - } - vsr, err := client.VoteStart(vs) - if err != nil { - return err - } - vsr.EligibleTickets = []string{"removed by piwww for readability"} - err = shared.PrintJSON(vsr) - if err != nil { - return err - } - - return nil -} -*/ - -// voteStartHelpMsg is the help command message. -var voteStartHelpMsg = `votestart - -Start the voting period for a proposal. Requires admin privileges. - -Arguments: -1. token (string, required) Proposal censorship token -2. duration (uint32, optional) Duration of vote in blocks - (default: 2016) -3. quorumpercentage (uint32, optional) Percent of total votes required to - reach a quorum (default: 10) -4. passpercentage (uint32, optional) Percent of cast votes required for - vote to be approved (default: 60) -` diff --git a/politeiawww/cmd/pictl/votestartrunoff.go b/politeiawww/cmd/pictl/votestartrunoff.go deleted file mode 100644 index e1e55b3e9..000000000 --- a/politeiawww/cmd/pictl/votestartrunoff.go +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright (c) 2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -// voteStartRunoffCmd starts the voting period on all public submissions to a -// request for proposals (RFP). -// -// The QuorumPercentage and PassPercentage are strings and not uint32 so that a -// value of 0 can be passed in and not be overwritten by the defaults. This is -// sometimes desirable when testing. -type voteStartRunoffCmd struct { - Args struct { - TokenRFP string `positional-arg-name:"token" required:"true"` - Duration uint32 `positional-arg-name:"duration"` - QuorumPercentage string `positional-arg-name:"quorumpercentage"` - PassPercentage string `positional-arg-name:"passpercentage"` - } `positional-args:"true"` -} - -/* -// Execute executes the voteStartRunoffCmd command. -// -// This function satisfies the go-flags Commander interface. -func (cmd *voteStartRunoffCmd) Execute(args []string) error { - // Check for user identity - if cfg.Identity == nil { - return shared.ErrUserIdentityNotFound - } - - // Setup vote params - var ( - // Default values - duration uint32 = 2016 - quorum uint32 = 10 - pass uint32 = 60 - ) - if cmd.Args.Duration != 0 { - duration = cmd.Args.Duration - } - if cmd.Args.QuorumPercentage != "" { - i, err := strconv.ParseUint(cmd.Args.QuorumPercentage, 10, 32) - if err != nil { - return err - } - quorum = uint32(i) - } - if cmd.Args.PassPercentage != "" { - i, err := strconv.ParseUint(cmd.Args.PassPercentage, 10, 32) - if err != nil { - return err - } - pass = uint32(i) - } - - // Fetch RFP proposal and RFP submissions - pdr, err := client.ProposalDetails(cmd.Args.TokenRFP, - &v1.ProposalsDetails{}) - if err != nil { - return err - } - bpr, err := client.BatchProposals(&v1.BatchProposals{ - Tokens: pdr.Proposal.LinkedFrom, - }) - if err != nil { - return err - } - - // Only include submissions that are public. This will - // filter out any submissions that have been abandoned. - submissions := make([]v1.ProposalRecord, 0, len(bpr.Proposals)) - for _, v := range bpr.Proposals { - if v.Status == v1.PropStatusPublic { - submissions = append(submissions, v) - } - } - - // Prepare VoteStart for each submission - starts := make([]pi.StartDetails, 0, len(submissions)) - for _, v := range submissions { - version, err := strconv.ParseUint(v.Version, 10, 32) - if err != nil { - return err - } - - vote := pi.VoteParams{ - Token: v.CensorshipRecord.Token, - Version: uint32(version), - Type: pi.VoteTypeRunoff, - Mask: 0x03, // bit 0 no, bit 1 yes - Duration: duration, - QuorumPercentage: quorum, - PassPercentage: pass, - Options: []pi.VoteOption{ - { - ID: ticketvote.VoteOptionIDApprove, - Description: "Approve proposal", - Bit: 0x01, - }, - { - ID: ticketvote.VoteOptionIDReject, - Description: "Don't approve proposal", - Bit: 0x02, - }, - }, - Parent: cmd.Args.TokenRFP, - } - vb, err := json.Marshal(vote) - if err != nil { - return err - } - msg := hex.EncodeToString(util.Digest(vb)) - sig := cfg.Identity.SignMessage([]byte(msg)) - - starts = append(starts, pi.StartDetails{ - Params: vote, - PublicKey: hex.EncodeToString(cfg.Identity.Public.Key[:]), - Signature: hex.EncodeToString(sig[:]), - }) - } - - // Prepare and send request - vs := pi.VoteStart{ - Starts: starts, - } - err = shared.PrintJSON(vs) - if err != nil { - return err - } - vsr, err := client.VoteStart(vs) - if err != nil { - return err - } - - // Print response details. Remove ticket snapshot from - // the response before printing so that the output is - // legible. - vsr.EligibleTickets = []string{"removed by piwww for readability"} - err = shared.PrintJSON(vsr) - if err != nil { - return err - } - - return nil -} -*/ - -// voteStartRunoffHelpMsg is the help command output for 'votestartrunoff'. -var voteStartRunoffHelpMsg = `votestartrunoff - -Start the voting period on all public submissions to an RFP proposal. The -optional arguments must either all be used or none be used. - -The quorumpercentage and passpercentage are strings and not uint32 so that a -value of 0 can be passed in and not be overwritten by the defaults. This is -sometimes desirable when testing. - -Arguments: -1. token (string, required) Proposal censorship token -2. duration (uint32, optional) Duration of vote in blocks (default: 2016) -3. quorumpercentage (uint32, optional) Percent of votes required for quorum (default: 10) -4. passpercentage (uint32, optional) Percent of votes required to pass (default: 60) -` diff --git a/politeiawww/ticketvote/process.go b/politeiawww/ticketvote/process.go index b2be496b4..63b119c31 100644 --- a/politeiawww/ticketvote/process.go +++ b/politeiawww/ticketvote/process.go @@ -212,7 +212,7 @@ func (t *TicketVote) processInventory(ctx context.Context) (*v1.InventoryReply, } return &v1.InventoryReply{ - Records: records, + Vetted: records, BestBlock: ir.BestBlock, }, nil } From 14115f57cb3d3a2e0bc96147db2fa7f82c47335d Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 7 Feb 2021 15:45:14 -0600 Subject: [PATCH 279/449] Edit vetted record bug fix. --- politeiawww/records/process.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index 32057929e..44164be9b 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -181,7 +181,7 @@ func (r *Records) processEdit(ctx context.Context, e v1.Edit, u user.User) (*v1. return nil, err } case v1.RecordStateVetted: - pdr, err = r.politeiad.UpdateUnvetted(ctx, e.Token, mdAppend, + pdr, err = r.politeiad.UpdateVetted(ctx, e.Token, mdAppend, mdOverwrite, filesAdd, filesDel) if err != nil { return nil, err From f5dbbb7ef8c1546bff3e918f1015758d4e08014e Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 8 Feb 2021 08:55:20 -0600 Subject: [PATCH 280/449] Fix unvetted comment bug. --- politeiad/backend/tlogbe/plugins/pi/hooks.go | 26 +++++++++++--------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/politeiad/backend/tlogbe/plugins/pi/hooks.go b/politeiad/backend/tlogbe/plugins/pi/hooks.go index 4f7f7c85d..f92a14717 100644 --- a/politeiad/backend/tlogbe/plugins/pi/hooks.go +++ b/politeiad/backend/tlogbe/plugins/pi/hooks.go @@ -222,8 +222,12 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { // commentWritesVerify verifies that a record's vote status allows writes from // the comments plugin. -func (p *piPlugin) commentWritesVerify(token []byte) error { - // Verify that the vote status allows comment writes +func (p *piPlugin) commentWritesVerify(s plugins.RecordStateT, token []byte) error { + // Verify that the vote status allows comment writes. This only + // applies to vetted records. + if s == plugins.RecordStateUnvetted { + return nil + } vs, err := p.voteSummary(token) if err != nil { return err @@ -242,16 +246,16 @@ func (p *piPlugin) commentWritesVerify(token []byte) error { } } -func (p *piPlugin) hookCommentNew(token []byte) error { - return p.commentWritesVerify(token) +func (p *piPlugin) hookCommentNew(s plugins.RecordStateT, token []byte) error { + return p.commentWritesVerify(s, token) } -func (p *piPlugin) hookCommentDel(token []byte) error { - return p.commentWritesVerify(token) +func (p *piPlugin) hookCommentDel(s plugins.RecordStateT, token []byte) error { + return p.commentWritesVerify(s, token) } -func (p *piPlugin) hookCommentVote(token []byte) error { - return p.commentWritesVerify(token) +func (p *piPlugin) hookCommentVote(s plugins.RecordStateT, token []byte) error { + return p.commentWritesVerify(s, token) } func (p *piPlugin) hookPluginPre(treeID int64, token []byte, payload string) error { @@ -267,11 +271,11 @@ func (p *piPlugin) hookPluginPre(treeID int64, token []byte, payload string) err case comments.PluginID: switch hpp.Cmd { case comments.CmdNew: - return p.hookCommentNew(token) + return p.hookCommentNew(hpp.State, token) case comments.CmdDel: - return p.hookCommentDel(token) + return p.hookCommentDel(hpp.State, token) case comments.CmdVote: - return p.hookCommentVote(token) + return p.hookCommentVote(hpp.State, token) } } From ab831a5eb045e781a0f2670488e6d5c675c643eb Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 8 Feb 2021 17:14:56 -0600 Subject: [PATCH 281/449] Vote timestamps command. --- .../backend/tlogbe/plugins/ticketvote/cmds.go | 29 ++---- .../tlogbe/plugins/ticketvote/ticketvote.go | 8 +- politeiawww/api/ticketvote/v1/v1.go | 9 ++ politeiawww/cmd/pictl/cmdcommenttimestamps.go | 2 +- politeiawww/cmd/pictl/cmdvotedetails.go | 55 ++++++------ politeiawww/cmd/pictl/cmdvoteresults.go | 38 ++++---- politeiawww/cmd/pictl/cmdvotesummaries.go | 48 ++++++---- politeiawww/cmd/pictl/cmdvotetimestamps.go | 70 ++++++++------- politeiawww/cmd/pictl/pictl.go | 2 +- politeiawww/cmd/pictl/testrun.go | 2 +- politeiawww/cmd/pictl/ticketvote.go | 88 +++++++++++++++++++ politeiawww/ticketvote/ticketvote.go | 2 +- 12 files changed, 228 insertions(+), 125 deletions(-) create mode 100644 politeiawww/cmd/pictl/ticketvote.go diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go index 15fba52eb..97657aff4 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go @@ -2437,14 +2437,8 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str return string(reply), nil } -func (p *ticketVotePlugin) cmdDetails(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdDetails: %v %x %v", treeID, token, payload) - - var d ticketvote.Details - err := json.Unmarshal([]byte(payload), &d) - if err != nil { - return "", err - } +func (p *ticketVotePlugin) cmdDetails(treeID int64, token []byte) (string, error) { + log.Tracef("cmdDetails: %v %x", treeID, token) // Get vote authorizations auths, err := p.auths(treeID) @@ -2471,8 +2465,8 @@ func (p *ticketVotePlugin) cmdDetails(treeID int64, token []byte, payload string return string(reply), nil } -func (p *ticketVotePlugin) cmdResults(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdResults: %v %x %v", treeID, token, payload) +func (p *ticketVotePlugin) cmdResults(treeID int64, token []byte) (string, error) { + log.Tracef("cmdResults: %v %x", treeID, token) // Get cast votes votes, err := p.castVotes(treeID) @@ -2546,8 +2540,8 @@ func voteIsApproved(vd ticketvote.VoteDetails, results []ticketvote.VoteOptionRe return approved } -func (p *ticketVotePlugin) cmdSummary(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdSummaries: %v %x %v", treeID, token, payload) +func (p *ticketVotePlugin) cmdSummary(treeID int64, token []byte) (string, error) { + log.Tracef("cmdSummaries: %v %x %v", treeID, token) // Get best block. This cmd does not write any data so we do not // have to use the safe best block. @@ -2634,15 +2628,8 @@ func (p *ticketVotePlugin) cmdInventory() (string, error) { return string(reply), nil } -func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdTimestamps: %v %x %v", treeID, token, payload) - - // Decode payload - var t ticketvote.Timestamps - err := json.Unmarshal([]byte(payload), &t) - if err != nil { - return "", err - } +func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte) (string, error) { + log.Tracef("cmdTimestamps: %v %x", treeID, token) // Get authorization timestamps digests, err := p.tlog.DigestsByDataType(treeID, dataTypeAuthDetails) diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go index dc4a5ef11..7c7f2a9af 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go @@ -193,15 +193,15 @@ func (p *ticketVotePlugin) Cmd(treeID int64, token []byte, cmd, payload string) case ticketvote.CmdCastBallot: return p.cmdCastBallot(treeID, token, payload) case ticketvote.CmdDetails: - return p.cmdDetails(treeID, token, payload) + return p.cmdDetails(treeID, token) case ticketvote.CmdResults: - return p.cmdResults(treeID, token, payload) + return p.cmdResults(treeID, token) case ticketvote.CmdSummary: - return p.cmdSummary(treeID, token, payload) + return p.cmdSummary(treeID, token) case ticketvote.CmdInventory: return p.cmdInventory() case ticketvote.CmdTimestamps: - return p.cmdTimestamps(treeID, token, payload) + return p.cmdTimestamps(treeID, token) case ticketvote.CmdLinkedFrom: return p.cmdLinkedFrom(token) diff --git a/politeiawww/api/ticketvote/v1/v1.go b/politeiawww/api/ticketvote/v1/v1.go index 9ae72430c..95e80c01b 100644 --- a/politeiawww/api/ticketvote/v1/v1.go +++ b/politeiawww/api/ticketvote/v1/v1.go @@ -163,6 +163,15 @@ const ( VoteOptionIDReject = "no" ) +var ( + // VoteType contains the human readable vote types. + VoteTypes = map[VoteT]string{ + VoteTypeInvalid: "invalid vote type", + VoteTypeStandard: "standard", + VoteTypeRunoff: "runoff", + } +) + // VoteStatusT represents a vote status. type VoteStatusT int diff --git a/politeiawww/cmd/pictl/cmdcommenttimestamps.go b/politeiawww/cmd/pictl/cmdcommenttimestamps.go index 30c424586..47ecfa8d9 100644 --- a/politeiawww/cmd/pictl/cmdcommenttimestamps.go +++ b/politeiawww/cmd/pictl/cmdcommenttimestamps.go @@ -124,7 +124,7 @@ Flags: --unvetted (bool, optional) Record is unvetted. Example: Fetch all record comment timestamps -$ piwww commenttimestamps 0a265dd93e9bae6d +$ pictl commenttimestamps 0a265dd93e9bae6d Example: Fetch comment timestamps for comment IDs 1, 6, and 7 $ pictl commenttimestamps 0a265dd93e9bae6d 1 6 7` diff --git a/politeiawww/cmd/pictl/cmdvotedetails.go b/politeiawww/cmd/pictl/cmdvotedetails.go index 6239438c4..50235550e 100644 --- a/politeiawww/cmd/pictl/cmdvotedetails.go +++ b/politeiawww/cmd/pictl/cmdvotedetails.go @@ -4,59 +4,58 @@ package main -// cmdVoteDetails retrieves vote details for the specified proposals. +import ( + tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" + pclient "github.com/decred/politeia/politeiawww/client" +) + +// cmdVoteDetails retrieves vote details for the provided record. type cmdVoteDetails struct { Args struct { Token string `positional-arg-name:"token"` } `positional-args:"true" required:"true"` } -/* // Execute executes the cmdVoteDetails command. // // This function satisfies the go-flags Commander interface. func (c *cmdVoteDetails) Execute(args []string) error { - // Setup request - v := pi.Votes{ - Tokens: c.Args.Tokens, + // Setup client + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, } - - // Send request. The request and response details are printed to - // the console. - err := shared.PrintJSON(v) + pc, err := pclient.New(cfg.Host, opts) if err != nil { return err } - vr, err := client.Votes(v) + + // Get vote details + d := tkv1.Details{ + Token: c.Args.Token, + } + dr, err := pc.TicketVoteDetails(d) if err != nil { return err } - if !cfg.RawJSON { - // Remove the eligible ticket pool from the response for - // readability. - for k, v := range vr.Votes { - if v.Vote == nil { - continue - } - v.Vote.EligibleTickets = []string{ - "removed by piwww for readability", - } - vr.Votes[k] = v - } + + // Print results + for _, v := range dr.Auths { + printAuthDetails(v) + printf("\n") } - err = shared.PrintJSON(vr) - if err != nil { - return err + if dr.Vote != nil { + printVoteDetails(*dr.Vote) } return nil } -*/ // voteDetailsHelpMsg is printed to stdout by the help command. const voteDetailsHelpMsg = `votedetails "token" -Fetch the vote details for a proposal. +Fetch the vote details for a record. Arguments: -1. token (string, required) Proposal censorship token.` +1. token (string, required) Record token.` diff --git a/politeiawww/cmd/pictl/cmdvoteresults.go b/politeiawww/cmd/pictl/cmdvoteresults.go index e27c38621..4f760a1bb 100644 --- a/politeiawww/cmd/pictl/cmdvoteresults.go +++ b/politeiawww/cmd/pictl/cmdvoteresults.go @@ -4,47 +4,53 @@ package main -// cmdVoteResults retreives the cast votes for the provided proposal. +import ( + tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" + pclient "github.com/decred/politeia/politeiawww/client" +) + +// cmdVoteResults retreives the cast ticket votes for a record. type cmdVoteResults struct { Args struct { Token string `positional-arg-name:"token"` } `positional-args:"true" required:"true"` } -/* // Execute executes the cmdVoteResults command. // // This function satisfies the go-flags Commander interface. func (c *cmdVoteResults) Execute(args []string) error { - // Setup request - vr := pi.VoteResults{ - Token: cmd.Args.Token, + // Setup client + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, } - - // Send request. The request and response details are printed to - // the console. - err := shared.PrintJSON(vr) + pc, err := pclient.New(cfg.Host, opts) if err != nil { return err } - vrr, err := client.VoteResults(vr) - if err != nil { - return err + + // Get vote results + r := tkv1.Results{ + Token: c.Args.Token, } - err = shared.PrintJSON(vrr) + rr, err := pc.TicketVoteResults(r) if err != nil { return err } + // Print results summary + printVoteResults(rr.Votes) + return nil } -*/ // voteResultsHelpMsg is printed to stdout by the help command. const voteResultsHelpMsg = `voteresults "token" -Fetch vote results for the provided proposal. +Fetch vote results for a record. Arguments: -1. token (string, required) Proposal censorship token +1. token (string, required) Record token. ` diff --git a/politeiawww/cmd/pictl/cmdvotesummaries.go b/politeiawww/cmd/pictl/cmdvotesummaries.go index bbbe5ed28..8687bd87f 100644 --- a/politeiawww/cmd/pictl/cmdvotesummaries.go +++ b/politeiawww/cmd/pictl/cmdvotesummaries.go @@ -4,47 +4,57 @@ package main -// cmdVoteSummaries retrieves the vote summaries for the provided proposals. +import ( + tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" + pclient "github.com/decred/politeia/politeiawww/client" +) + +// cmdVoteSummaries retrieves the vote summaries for the provided records. type cmdVoteSummaries struct { Args struct { - Tokens []string `positional-arg-name:"token"` + Tokens []string `positional-arg-name:"tokens"` } `positional-args:"true" required:"true"` } -/* // Execute executes the cmdVoteSummaries command. // // This function satisfies the go-flags Commander interface. func (c *cmdVoteSummaries) Execute(args []string) error { - // Setup request - vs := pi.VoteSummaries{ - Tokens: cmd.Args.Tokens, + // Setup client + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, } - - // Send request. The request and response details are printed to - // the console. - err := shared.PrintJSON(vs) + pc, err := pclient.New(cfg.Host, opts) if err != nil { return err } - vsr, err := client.VoteSummaries(vs) - if err != nil { - return err + + // Get vote summaries + s := tkv1.Summaries{ + Tokens: c.Args.Tokens, } - err = shared.PrintJSON(vsr) + sr, err := pc.TicketVoteSummaries(s) if err != nil { return err } + // Print summaries + for k, v := range sr.Summaries { + printVoteSummary(k, v) + printf("-----\n") + } + return nil } -*/ // voteSummariesHelpMsg is printed to stdout by the help command. -const voteSummariesHelpMsg = `votesummaries "tokens" +const voteSummariesHelpMsg = `votesummaries "tokens..." -Fetch the vote summaries for the provided proposal tokens. +Fetch the vote summaries for the provided records. This command accepts both +full length tokens and token prefixes. Example usage: -$ piww votesummaries cda97ace0a4765140000 71dd3a110500fb6a0000 -` +$ pictl votesummaries cda97ace0a476514 71dd3a110500fb6a +$ pictl votesummaries cda97ac 71dd3a1` diff --git a/politeiawww/cmd/pictl/cmdvotetimestamps.go b/politeiawww/cmd/pictl/cmdvotetimestamps.go index 58cb7daa3..294291cc1 100644 --- a/politeiawww/cmd/pictl/cmdvotetimestamps.go +++ b/politeiawww/cmd/pictl/cmdvotetimestamps.go @@ -5,9 +5,12 @@ package main import ( + "fmt" + "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" + pclient "github.com/decred/politeia/politeiawww/client" ) // cmdVoteTimestamps retrieves the timestamps for a politeiawww ticket vote. @@ -21,44 +24,45 @@ type cmdVoteTimestamps struct { // // This function satisfies the go-flags Commander interface. func (c *cmdVoteTimestamps) Execute(args []string) error { - /* - // Setup request - t := tkv1.Timestamps{ - Token: c.Args.Token, - } + // Setup client + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Cookies: cfg.Cookies, + HeaderCSRF: cfg.CSRF, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, + } + pc, err := pclient.New(cfg.Host, opts) + if err != nil { + return err + } - // Send request - err := shared.PrintJSON(t) - if err != nil { - return err - } - tr, err := client.TicketVoteTimestamps(t) - if err != nil { - return err - } - err = shared.PrintJSON(tr) - if err != nil { - return err - } + // Get timestamps + t := tkv1.Timestamps{ + Token: c.Args.Token, + } + tr, err := pc.TicketVoteTimestamps(t) + if err != nil { + return err + } - // Verify timestamps - for k, v := range tr.Auths { - err = verifyVoteTimestamp(v) - if err != nil { - return fmt.Errorf("verify authorization %v timestamp: %v", k, err) - } - } - err = verifyVoteTimestamp(tr.Details) + // Verify timestamps + for k, v := range tr.Auths { + err = verifyVoteTimestamp(v) if err != nil { - return fmt.Errorf("verify vote details timestamp: %v", err) + return fmt.Errorf("verify authorization %v timestamp: %v", k, err) } - for k, v := range tr.Votes { - err = verifyVoteTimestamp(v) - if err != nil { - return fmt.Errorf("verify vote %v timestamp: %v", k, err) - } + } + err = verifyVoteTimestamp(tr.Details) + if err != nil { + return fmt.Errorf("verify vote details timestamp: %v", err) + } + for k, v := range tr.Votes { + err = verifyVoteTimestamp(v) + if err != nil { + return fmt.Errorf("verify vote %v timestamp: %v", k, err) } - */ + } return nil } diff --git a/politeiawww/cmd/pictl/pictl.go b/politeiawww/cmd/pictl/pictl.go index b7a0d1b08..6b2f7eebc 100644 --- a/politeiawww/cmd/pictl/pictl.go +++ b/politeiawww/cmd/pictl/pictl.go @@ -88,7 +88,7 @@ type pictl struct { VoteAuthorize cmdVoteAuthorize `command:"voteauthorize"` VoteStart cmdVoteStart `command:"votestart"` CastBallot cmdCastBallot `command:"castballot"` - Votes cmdVoteDetails `command:"votedetails"` + VoteDetails cmdVoteDetails `command:"votedetails"` VoteResults cmdVoteResults `command:"voteresults"` VoteSummaries cmdVoteSummaries `command:"votesummaries"` VoteInv cmdVoteInv `command:"voteinv"` diff --git a/politeiawww/cmd/pictl/testrun.go b/politeiawww/cmd/pictl/testrun.go index 4051be688..67e37ad51 100644 --- a/politeiawww/cmd/pictl/testrun.go +++ b/politeiawww/cmd/pictl/testrun.go @@ -164,7 +164,7 @@ func userDetails(u *testUser) error { return nil } -// testUser tests piwww user specific routes. +// testUser tests pictl user specific routes. func testUserRoutes(admin testUser) error { // sleepInterval is the time to wait in between requests // when polling politeiawww for paywall tx confirmations. diff --git a/politeiawww/cmd/pictl/ticketvote.go b/politeiawww/cmd/pictl/ticketvote.go new file mode 100644 index 000000000..4aabb61fe --- /dev/null +++ b/politeiawww/cmd/pictl/ticketvote.go @@ -0,0 +1,88 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "sort" + + tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" +) + +func printAuthDetails(a tkv1.AuthDetails) { + printf("Token : %v\n", a.Token) + printf("Action : %v\n", a.Action) + printf("Timestamp: %v\n", timestampFromUnix(a.Timestamp)) + printf("Receipt : %v\n", a.Receipt) +} + +func printVoteDetails(v tkv1.VoteDetails) { + printf("Token : %v\n", v.Params.Token) + printf("Type : %v\n", v.Params.Type) + if v.Params.Type == tkv1.VoteTypeRunoff { + printf("Parent : %v\n", v.Params.Parent) + } + printf("Pass Percentage : %v%%\n", v.Params.PassPercentage) + printf("Quorum Percentage : %v%%\n", v.Params.QuorumPercentage) + printf("Duration : %v blocks\n", v.Params.Duration) + printf("Start Block Hash : %v\n", v.StartBlockHash) + printf("Start Block Height: %v\n", v.StartBlockHeight) + printf("End Block Height : %v\n", v.EndBlockHeight) + printf("Vote options\n") + for _, v := range v.Params.Options { + printf(" %v %v %v\n", v.Bit, v.ID, v.Description) + } +} + +func printVoteResults(votes []tkv1.CastVoteDetails) { + if len(votes) == 0 { + return + } + + // Tally results + results := make(map[string]int) + for _, v := range votes { + results[v.VoteBit]++ + } + + // Order results + r := make([]string, 0, len(results)) + for k := range results { + r = append(r, k) + } + sort.SliceStable(r, func(i, j int) bool { + return r[i] < r[j] + }) + + // Print results + printf("Token: %v\n", votes[0].Token) + printf("Results\n") + for _, v := range r { + printf(" %v: %v\n", v, results[v]) + } +} + +func printVoteSummary(token string, s tkv1.Summary) { + printf("Token : %v\n", token) + printf("Status : %v\n", tkv1.VoteStatuses[s.Status]) + switch s.Status { + case tkv1.VoteStatusUnauthorized, tkv1.VoteStatusAuthorized: + // Nothing else to print + return + } + printf("Type : %v\n", tkv1.VoteTypes[s.Type]) + printf("Pass Percentage : %v%%\n", s.PassPercentage) + printf("Quorum Percentage : %v%%\n", s.QuorumPercentage) + printf("Duration : %v blocks\n", s.Duration) + printf("Start Block Hash : %v\n", s.StartBlockHash) + printf("Start Block Height: %v\n", s.StartBlockHeight) + printf("End Block Height : %v\n", s.EndBlockHeight) + printf("Eligible Tickets : %v tickets\n", s.EligibleTickets) + printf("Approved : %v\n", s.Approved) + printf("Best Block : %v\n", s.BestBlock) + printf("Results\n") + for _, v := range s.Results { + printf(" %v %-3v %v votes\n", v.VoteBit, v.ID, v.Votes) + } +} diff --git a/politeiawww/ticketvote/ticketvote.go b/politeiawww/ticketvote/ticketvote.go index 5cbbcd0bd..32a1c3e44 100644 --- a/politeiawww/ticketvote/ticketvote.go +++ b/politeiawww/ticketvote/ticketvote.go @@ -244,7 +244,7 @@ func (t *TicketVote) HandleTimestamps(w http.ResponseWriter, r *http.Request) { var ts v1.Timestamps decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&t); err != nil { + if err := decoder.Decode(&ts); err != nil { respondWithError(w, r, "HandleTimestamps: unmarshal", v1.UserErrorReply{ ErrorCode: v1.ErrorCodeInputInvalid, From 7b9dbd410c0661984284e5a604de958ea50b5b50 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 8 Feb 2021 18:38:00 -0600 Subject: [PATCH 282/449] Cast ballot cmd. --- politeiawww/cmd/pictl/cmdcastballot.go | 126 +++++++++++++++---------- politeiawww/cmd/pictl/cmdvotestart.go | 2 +- politeiawww/cmd/pictl/dcrwallet.go | 49 ++++++++++ politeiawww/cmd/pictl/ticketvote.go | 6 +- politeiawww/cmd/shared/client.go | 122 ------------------------ politeiawww/piwww.go | 3 + 6 files changed, 134 insertions(+), 174 deletions(-) create mode 100644 politeiawww/cmd/pictl/dcrwallet.go diff --git a/politeiawww/cmd/pictl/cmdcastballot.go b/politeiawww/cmd/pictl/cmdcastballot.go index 62da9cfdf..5b3b9a806 100644 --- a/politeiawww/cmd/pictl/cmdcastballot.go +++ b/politeiawww/cmd/pictl/cmdcastballot.go @@ -4,6 +4,23 @@ package main +import ( + "bytes" + "context" + "encoding/hex" + "fmt" + "os" + "strconv" + + "decred.org/dcrwallet/rpc/walletrpc" + "github.com/decred/dcrd/chaincfg/chainhash" + "github.com/decred/politeia/politeiad/api/v1/identity" + tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" + pclient "github.com/decred/politeia/politeiawww/client" + "github.com/decred/politeia/util" + "golang.org/x/crypto/ssh/terminal" +) + // cmdCastBallot casts a ballot of votes. type cmdCastBallot struct { Args struct { @@ -13,32 +30,52 @@ type cmdCastBallot struct { Password string `long:"password" optional:"true"` } -/* // Execute executes the cmdCastBallot command. // // This function satisfies the go-flags Commander interface. func (c *cmdCastBallot) Execute(args []string) error { - token := c.Args.Token - voteID := c.Args.VoteID + // Unpack args + var ( + token = c.Args.Token + voteID = c.Args.VoteID + ) + + // Setup politeiawww client + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, + } + pc, err := pclient.New(cfg.Host, opts) + if err != nil { + return err + } - // Get vote details - vr, err := client.Votes(pi.Votes{ - Tokens: []string{token}, - }) + // Setup dcrwallet client + ctx := context.Background() + wc, err := newDcrwalletClient(cfg.WalletHost, cfg.WalletCert, + cfg.ClientCert, cfg.ClientKey) if err != nil { - return fmt.Errorf("Votes: %v", err) + return err } - pv, ok := vr.Votes[token] - if !ok { - return fmt.Errorf("proposal not found: %v", token) + defer wc.conn.Close() + + // Get vote details + d := tkv1.Details{ + Token: token, } - if pv.Vote == nil { - return fmt.Errorf("vote hasn't started yet") + dr, err := pc.TicketVoteDetails(d) + if err != nil { + return err } + if dr.Vote == nil { + return fmt.Errorf("vote not started") + } + voteDetails := dr.Vote // Verify provided vote ID var voteBit string - for _, option := range pv.Vote.Params.Options { + for _, option := range voteDetails.Params.Options { if voteID == option.ID { voteBit = strconv.FormatUint(option.Bit, 16) break @@ -48,26 +85,19 @@ func (c *cmdCastBallot) Execute(args []string) error { return fmt.Errorf("vote id not found: %v", voteID) } - // Connect to user's wallet - err = client.LoadWalletClient() - if err != nil { - return fmt.Errorf("LoadWalletClient: %v", err) - } - defer client.Close() - // Get the user's tickets that are eligible to vote - ticketpool := make([][]byte, 0, len(pv.Vote.EligibleTickets)) - for _, v := range pv.Vote.EligibleTickets { + ticketPool := make([][]byte, 0, len(voteDetails.EligibleTickets)) + for _, v := range voteDetails.EligibleTickets { h, err := chainhash.NewHashFromStr(v) if err != nil { - return nil, err + return err } - ticketpool = append(ticketpool, h[:]) + ticketPool = append(ticketPool, h[:]) } - ctr, err := client.CommittedTickets( - &walletrpc.CommittedTicketsRequest{ - Tickets: ticketPool, - }) + ct := walletrpc.CommittedTicketsRequest{ + Tickets: ticketPool, + } + ctr, err := wc.wallet.CommittedTickets(ctx, &ct) if err != nil { return fmt.Errorf("CommittedTickets: %v", err) } @@ -86,7 +116,7 @@ func (c *cmdCastBallot) Execute(args []string) error { } // The next step is to have the user's wallet sign the proposal - // votes for each ticket. The password wallet is needed for this. + // votes for each ticket. The wallet password is needed for this. var passphrase []byte if c.Password != "" { // Password was provided @@ -94,12 +124,12 @@ func (c *cmdCastBallot) Execute(args []string) error { } else { // Prompt user for password for len(passphrase) == 0 { - fmt.Printf("Enter the private passphrase of your wallet: ") + printf("Enter the private passphrase of your wallet: ") pass, err := terminal.ReadPassword(int(os.Stdin.Fd())) if err != nil { return err } - fmt.Printf("\n") + printf("\n") passphrase = bytes.TrimSpace(pass) } } @@ -115,10 +145,11 @@ func (c *cmdCastBallot) Execute(args []string) error { Message: msg, }) } - sigs, err := client.SignMessages(&walletrpc.SignMessagesRequest{ + sm := walletrpc.SignMessagesRequest{ Passphrase: passphrase, Messages: messages, - }) + } + sigs, err := wc.wallet.SignMessages(ctx, &sm) if err != nil { return fmt.Errorf("SignMessages: %v", err) } @@ -130,24 +161,24 @@ func (c *cmdCastBallot) Execute(args []string) error { } // Setup ballot request - votes := make([]pi.CastVote, 0, len(eligibleTickets)) + votes := make([]tkv1.CastVote, 0, len(eligibleTickets)) for i, ticket := range eligibleTickets { // eligibleTickets and sigs use the same index - votes = append(votes, pi.CastVote{ + votes = append(votes, tkv1.CastVote{ Token: token, Ticket: ticket, VoteBit: voteBit, Signature: hex.EncodeToString(sigs.Replies[i].Signature), }) } - cb := pi.CastBallot{ + cb := tkv1.CastBallot{ Votes: votes, } // Send ballot request - cbr, err := client.CastBallot(cb) + cbr, err := pc.TicketVoteCastBallot(cb) if err != nil { - return fmt.Errorf("CastBallot: %v", err) + return err } // Get the server pubkey so that we can validate the receipts. @@ -164,7 +195,7 @@ func (c *cmdCastBallot) Execute(args []string) error { // ticket hash so in order to associate a failed receipt with a // specific ticket, we need to lookup the ticket hash and store // it separately. - failedReceipts := make([]pi.CastVoteReply, 0, len(cbr.Receipts)) + failedReceipts := make([]tkv1.CastVoteReply, 0, len(cbr.Receipts)) failedTickets := make([]string, 0, len(eligibleTickets)) for i, v := range cbr.Receipts { // Lookup ticket hash. br.Receipts and eligibleTickets use the @@ -181,28 +212,25 @@ func (c *cmdCastBallot) Execute(args []string) error { // Verify receipts sig, err := identity.SignatureFromString(v.Receipt) if err != nil { - fmt.Printf("Failed to decode receipt: %v\n", v.Ticket) + printf("Failed to decode receipt: %v\n", v.Ticket) continue } clientSig := votes[i].Signature if !serverID.VerifyMessage([]byte(clientSig), *sig) { - fmt.Printf("Failed to verify receipt: %v", v.Ticket) + printf("Failed to verify receipt: %v", v.Ticket) continue } } // Print results - if !cfg.Silent { - fmt.Printf("Votes succeeded: %v\n", len(cbr.Receipts)-len(failedReceipts)) - fmt.Printf("Votes failed : %v\n", len(failedReceipts)) - for i, v := range failedReceipts { - fmt.Printf("Failed vote : %v %v\n", failedTickets[i], v.ErrorContext) - } + printf("Votes succeeded: %v\n", len(cbr.Receipts)-len(failedReceipts)) + printf("Votes failed : %v\n", len(failedReceipts)) + for i, v := range failedReceipts { + printf("Failed vote : %v %v\n", failedTickets[i], v.ErrorContext) } return nil } -*/ // castBallotHelpMsg is printed to stdout by the help command. const castBallotHelpMsg = `castballot "token" "voteid" diff --git a/politeiawww/cmd/pictl/cmdvotestart.go b/politeiawww/cmd/pictl/cmdvotestart.go index eba12f1c2..f4680a6bf 100644 --- a/politeiawww/cmd/pictl/cmdvotestart.go +++ b/politeiawww/cmd/pictl/cmdvotestart.go @@ -33,7 +33,7 @@ type cmdVoteStart struct { // Runoff is used to indicate the vote is a runoff vote and the // provided token is the parent token of the runoff vote. - Runoff bool `long:"random" optional:"true"` + Runoff bool `long:"runoff" optional:"true"` } func voteStartStandard(token string, duration, quorum, pass uint32, pc *pclient.Client) (*tkv1.StartReply, error) { diff --git a/politeiawww/cmd/pictl/dcrwallet.go b/politeiawww/cmd/pictl/dcrwallet.go new file mode 100644 index 000000000..846716418 --- /dev/null +++ b/politeiawww/cmd/pictl/dcrwallet.go @@ -0,0 +1,49 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" + + "decred.org/dcrwallet/rpc/walletrpc" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" +) + +type dcrwalletClient struct { + conn *grpc.ClientConn + wallet walletrpc.WalletServiceClient +} + +func newDcrwalletClient(walletHost, walletCert, clientCert, clientKey string) (*dcrwalletClient, error) { + serverCAs := x509.NewCertPool() + serverCert, err := ioutil.ReadFile(walletCert) + if err != nil { + return nil, err + } + if !serverCAs.AppendCertsFromPEM(serverCert) { + return nil, fmt.Errorf("no certificates found in %v", + walletCert) + } + keypair, err := tls.LoadX509KeyPair(clientCert, clientKey) + if err != nil { + return nil, err + } + creds := credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{keypair}, + RootCAs: serverCAs, + }) + conn, err := grpc.Dial(walletHost, grpc.WithTransportCredentials(creds)) + if err != nil { + return nil, err + } + return &dcrwalletClient{ + conn: conn, + wallet: walletrpc.NewWalletServiceClient(conn), + }, nil +} diff --git a/politeiawww/cmd/pictl/ticketvote.go b/politeiawww/cmd/pictl/ticketvote.go index 4aabb61fe..8e71f973e 100644 --- a/politeiawww/cmd/pictl/ticketvote.go +++ b/politeiawww/cmd/pictl/ticketvote.go @@ -71,9 +71,11 @@ func printVoteSummary(token string, s tkv1.Summary) { // Nothing else to print return } + pass := int(float64(s.PassPercentage) / 100 * float64(s.EligibleTickets)) + quorum := int(float64(s.QuorumPercentage) / 100 * float64(s.EligibleTickets)) printf("Type : %v\n", tkv1.VoteTypes[s.Type]) - printf("Pass Percentage : %v%%\n", s.PassPercentage) - printf("Quorum Percentage : %v%%\n", s.QuorumPercentage) + printf("Pass Percentage : %v%% (%v votes)\n", s.PassPercentage, pass) + printf("Quorum Percentage : %v%% (%v votes)\n", s.QuorumPercentage, quorum) printf("Duration : %v blocks\n", s.Duration) printf("Start Block Hash : %v\n", s.StartBlockHash) printf("Start Block Height: %v\n", s.StartBlockHeight) diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index 80aad4f62..2d1f53256 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -7,11 +7,8 @@ package shared import ( "bytes" "context" - "crypto/tls" - "crypto/x509" "encoding/json" "fmt" - "io/ioutil" "net/http" "net/http/cookiejar" "net/url" @@ -2323,125 +2320,6 @@ func (c *Client) VerifyTOTP(vt *www.VerifyTOTP) (*www.VerifyTOTPReply, error) { return &vtr, nil } -// WalletAccounts retrieves the walletprc accounts. -func (c *Client) WalletAccounts() (*walletrpc.AccountsResponse, error) { - if c.wallet == nil { - return nil, fmt.Errorf("walletrpc client not loaded") - } - - if c.cfg.Verbose { - fmt.Printf("walletrpc %v Accounts\n", c.cfg.WalletHost) - } - - ar, err := c.wallet.Accounts(c.ctx, &walletrpc.AccountsRequest{}) - if err != nil { - return nil, err - } - - if c.cfg.Verbose { - err := prettyPrintJSON(ar) - if err != nil { - return nil, err - } - } - - return ar, nil -} - -// CommittedTickets returns the committed tickets that belong to the dcrwallet -// instance out of the the specified list of tickets. -func (c *Client) CommittedTickets(ct *walletrpc.CommittedTicketsRequest) (*walletrpc.CommittedTicketsResponse, error) { - if c.wallet == nil { - return nil, fmt.Errorf("walletrpc client not loaded") - } - - if c.cfg.Verbose { - fmt.Printf("walletrpc %v CommittedTickets\n", c.cfg.WalletHost) - } - - ctr, err := c.wallet.CommittedTickets(c.ctx, ct) - if err != nil { - return nil, err - } - - if c.cfg.Verbose { - err := prettyPrintJSON(ctr) - if err != nil { - return nil, err - } - } - - return ctr, nil -} - -// SignMessages signs the passed in messages using the private keys from the -// specified addresses. -func (c *Client) SignMessages(sm *walletrpc.SignMessagesRequest) (*walletrpc.SignMessagesResponse, error) { - if c.wallet == nil { - return nil, fmt.Errorf("walletrpc client not loaded") - } - - if c.cfg.Verbose { - fmt.Printf("walletrpc %v SignMessages\n", c.cfg.WalletHost) - } - - smr, err := c.wallet.SignMessages(c.ctx, sm) - if err != nil { - return nil, err - } - - if c.cfg.Verbose { - err := prettyPrintJSON(smr) - if err != nil { - return nil, err - } - } - - return smr, nil -} - -// TODO the wallet client should be its own client and it should verify -// that the dcrwallet client certs are set. -// LoadWalletClient connects to a dcrwallet instance. -func (c *Client) LoadWalletClient() error { - serverCAs := x509.NewCertPool() - serverCert, err := ioutil.ReadFile(c.cfg.WalletCert) - if err != nil { - return err - } - if !serverCAs.AppendCertsFromPEM(serverCert) { - return fmt.Errorf("no certificates found in %s", - c.cfg.WalletCert) - } - keypair, err := tls.LoadX509KeyPair(c.cfg.ClientCert, c.cfg.ClientKey) - if err != nil { - return fmt.Errorf("read client keypair: %v", err) - } - creds := credentials.NewTLS(&tls.Config{ - Certificates: []tls.Certificate{keypair}, - RootCAs: serverCAs, - }) - - conn, err := grpc.Dial(c.cfg.WalletHost, - grpc.WithTransportCredentials(creds)) - if err != nil { - return err - } - - c.ctx = context.Background() - c.creds = creds - c.conn = conn - c.wallet = walletrpc.NewWalletServiceClient(conn) - return nil -} - -// Close all client connections. -func (c *Client) Close() { - if c.conn != nil { - c.conn.Close() - } -} - // NewClient returns a new politeiawww client. func NewClient(cfg *Config) (*Client, error) { // Setup http client diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 17080e1a4..61b933312 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -137,6 +137,9 @@ func (p *politeiawww) setupPiRoutes(r *records.Records, c *comments.Comments, t p.addRoute(http.MethodPost, tkv1.APIRoute, tkv1.RouteSummaries, t.HandleSummaries, permissionPublic) + p.addRoute(http.MethodPost, tkv1.APIRoute, + tkv1.RouteLinkedFrom, t.HandleLinkedFrom, + permissionPublic) p.addRoute(http.MethodPost, tkv1.APIRoute, tkv1.RouteInventory, t.HandleInventory, permissionPublic) From 793f83c3a803819558cc1f114850ea9e2fd21d86 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 9 Feb 2021 09:08:46 -0600 Subject: [PATCH 283/449] Change linkedfrom route to submissions. --- .../backend/tlogbe/plugins/ticketvote/cmds.go | 18 +- .../tlogbe/plugins/ticketvote/hooks.go | 14 +- .../tlogbe/plugins/ticketvote/linkedfrom.go | 172 ------------------ .../tlogbe/plugins/ticketvote/submissions.go | 172 ++++++++++++++++++ .../tlogbe/plugins/ticketvote/ticketvote.go | 4 +- politeiad/client/ticketvote.go | 49 +++-- politeiad/plugins/ticketvote/ticketvote.go | 42 +++-- politeiawww/api/ticketvote/v1/v1.go | 40 ++-- politeiawww/client/ticketvote.go | 14 +- politeiawww/cmd/pictl/{help.go => cmdhelp.go} | 12 +- politeiawww/cmd/pictl/cmdvotestart.go | 20 +- politeiawww/cmd/pictl/cmdvotesubmissions.go | 56 ++++++ politeiawww/cmd/pictl/pictl.go | 21 ++- politeiawww/piwww.go | 2 +- politeiawww/ticketvote/process.go | 10 +- politeiawww/ticketvote/ticketvote.go | 18 +- 16 files changed, 357 insertions(+), 307 deletions(-) delete mode 100644 politeiad/backend/tlogbe/plugins/ticketvote/linkedfrom.go create mode 100644 politeiad/backend/tlogbe/plugins/ticketvote/submissions.go rename politeiawww/cmd/pictl/{help.go => cmdhelp.go} (93%) create mode 100644 politeiawww/cmd/pictl/cmdvotesubmissions.go diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go index 97657aff4..1a7ed0577 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go @@ -1659,10 +1659,10 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti // Compile a list of the expected submissions that should be in the // runoff vote. This will be all of the public records that have - // linked to the parent record. The parent record's linked from + // linked to the parent record. The parent record's submissions // list will include abandoned proposals that need to be filtered // out. - lf, err := p.linkedFromCache(token) + lf, err := p.submissionsCache(token) if err != nil { return nil, err } @@ -1683,7 +1683,7 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti } // This is a public record that is part of the parent record's - // linked from list. It is required to be in the runoff vote. + // submissions list. It is required to be in the runoff vote. expected[k] = struct{}{} } @@ -2705,11 +2705,11 @@ func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte) (string, er return string(reply), nil } -func (p *ticketVotePlugin) cmdLinkedFrom(token []byte) (string, error) { - log.Tracef("cmdLinkedFrom: %x", token) +func (p *ticketVotePlugin) cmdSubmissions(token []byte) (string, error) { + log.Tracef("cmdSubmissions: %x", token) - // Get linked from list - lf, err := p.linkedFromCache(token) + // Get submissions list + lf, err := p.submissionsCache(token) if err != nil { return "", err } @@ -2719,8 +2719,8 @@ func (p *ticketVotePlugin) cmdLinkedFrom(token []byte) (string, error) { for k := range lf.Tokens { tokens = append(tokens, k) } - lfr := ticketvote.LinkedFromReply{ - Tokens: tokens, + lfr := ticketvote.SubmissionsReply{ + Submissions: tokens, } reply, err := json.Marshal(lfr) if err != nil { diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go b/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go index e26c8708a..14a0d81bb 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go @@ -263,7 +263,7 @@ func (p *ticketVotePlugin) hookSetRecordStatusPost(payload string) error { } if vm != nil && vm.LinkTo != "" { // LinkTo has been set. Check if the status change requires the - // linked from list of the linked record to be updated. + // submissions list of the linked record to be updated. var ( parentToken = vm.LinkTo childToken = srs.RecordMetadata.Token @@ -271,17 +271,17 @@ func (p *ticketVotePlugin) hookSetRecordStatusPost(payload string) error { switch srs.RecordMetadata.Status { case backend.MDStatusVetted: // Record has been made public. Add child token to parent's - // linked from list. - err := p.linkedFromCacheAdd(parentToken, childToken) + // submissions list. + err := p.submissionsCacheAdd(parentToken, childToken) if err != nil { - return fmt.Errorf("linkedFromCacheAdd: %v", err) + return fmt.Errorf("submissionsFromCacheAdd: %v", err) } case backend.MDStatusCensored: // Record has been censored. Delete child token from parent's - // linked from list. - err := p.linkedFromCacheDel(parentToken, childToken) + // submissions list. + err := p.submissionsCacheDel(parentToken, childToken) if err != nil { - return fmt.Errorf("linkedFromCacheDel: %v", err) + return fmt.Errorf("submissionsCacheDel: %v", err) } } } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/linkedfrom.go b/politeiad/backend/tlogbe/plugins/ticketvote/linkedfrom.go deleted file mode 100644 index bc7b5b3e4..000000000 --- a/politeiad/backend/tlogbe/plugins/ticketvote/linkedfrom.go +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright (c) 2020-2021 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package ticketvote - -import ( - "encoding/json" - "errors" - "io/ioutil" - "os" - "path/filepath" - "strings" - - "github.com/decred/politeia/util" -) - -const ( - // fnLinkedFrom is the filename for the cached linkedFrom data that - // is saved to the pi plugin data dir. - fnLinkedFrom = "{tokenprefix}-linkedfrom.json" -) - -// linkedFrom is the the structure that is updated and cached for record A when -// record B links to record A. Recordss can link to one another using the -// VoteMetadata LinkTo field. The linkedFrom list contains all records that -// have linked to record A. The list will only contain public records. The -// linkedFrom list is saved to disk in the ticketvote plugin data dir, with the -// parent record token in the filename. -// -// Example: the linked from list for a runoff vote parent record will contain -// all public runoff vote submissions. -type linkedFrom struct { - Tokens map[string]struct{} `json:"tokens"` -} - -// linkedFromCachePath returns the path to the linked fromlist for the provided -// record token. The token prefix is used in the file path so that the linked -// from list can be retrieved using either the full token or the token prefix. -func (p *ticketVotePlugin) linkedFromCachePath(token []byte) string { - t := util.TokenPrefix(token) - fn := strings.Replace(fnLinkedFrom, "{tokenprefix}", t, 1) - return filepath.Join(p.dataDir, fn) -} - -// linkedFromCacheWithLock return the linked from list for a record token. If a -// linked from list does not exist for the token then an empty list will be -// returned. -// -// This function must be called WITH the lock held. -func (p *ticketVotePlugin) linkedFromCacheWithLock(token []byte) (*linkedFrom, error) { - fp := p.linkedFromCachePath(token) - b, err := ioutil.ReadFile(fp) - if err != nil { - var e *os.PathError - if errors.As(err, &e) && !os.IsExist(err) { - // File does't exist. Return an empty linked from list. - return &linkedFrom{ - Tokens: make(map[string]struct{}), - }, nil - } - } - - var lf linkedFrom - err = json.Unmarshal(b, &lf) - if err != nil { - return nil, err - } - - return &lf, nil -} - -// linkedFromCache return the linked from list for a record token. If a linked -// from list does not exist for the token then an empty list will be returned. -// -// This function must be called WITHOUT the lock held. -func (p *ticketVotePlugin) linkedFromCache(token []byte) (*linkedFrom, error) { - p.Lock() - defer p.Unlock() - - return p.linkedFromCacheWithLock(token) -} - -// linkedFromCacheSaveWithLock saves a linkedFrom to the plugin data dir. -// -// This function must be called WITH the lock held. -func (p *ticketVotePlugin) linkedFromCacheSaveWithLock(token []byte, lf linkedFrom) error { - b, err := json.Marshal(lf) - if err != nil { - return err - } - fp := p.linkedFromCachePath(token) - return ioutil.WriteFile(fp, b, 0664) -} - -// linkedFromCacheAdd updates the cached linkedFrom list for the parentToken, -// adding the childToken to the list. The full length token MUST be used. -// -// This function must be called WITHOUT the lock held. -func (p *ticketVotePlugin) linkedFromCacheAdd(parentToken, childToken string) error { - p.Lock() - defer p.Unlock() - - // Verify tokens - parent, err := tokenDecode(parentToken) - if err != nil { - return err - } - _, err = tokenDecode(childToken) - if err != nil { - return err - } - - // Get existing linked from list - lf, err := p.linkedFromCacheWithLock(parent) - if err != nil { - return err - } - - // Update list - lf.Tokens[childToken] = struct{}{} - - // Save list - err = p.linkedFromCacheSaveWithLock(parent, *lf) - if err != nil { - return err - } - - log.Debugf("Linked from list updated. Child %v added to parent %v", - childToken, parentToken) - - return nil -} - -// linkedFromCacheDel updates the cached linkedFrom list for the parentToken, -// deleting the childToken from the list. -// -// This function must be called WITHOUT the lock held. -func (p *ticketVotePlugin) linkedFromCacheDel(parentToken, childToken string) error { - p.Lock() - defer p.Unlock() - - // Verify tokens - parent, err := tokenDecode(parentToken) - if err != nil { - return err - } - _, err = tokenDecode(childToken) - if err != nil { - return err - } - - // Get existing linked from list - lf, err := p.linkedFromCacheWithLock(parent) - if err != nil { - return err - } - - // Update list - delete(lf.Tokens, childToken) - - // Save list - err = p.linkedFromCacheSaveWithLock(parent, *lf) - if err != nil { - return err - } - - log.Debugf("Linked from list updated. Child %v deleted from parent %v", - childToken, parentToken) - - return nil -} diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/submissions.go b/politeiad/backend/tlogbe/plugins/ticketvote/submissions.go new file mode 100644 index 000000000..da2db1baf --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/ticketvote/submissions.go @@ -0,0 +1,172 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ticketvote + +import ( + "encoding/json" + "errors" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/decred/politeia/util" +) + +const ( + // fnSubmissions is the filename for the cached submissions data + // that is saved to the plugin data dir. + fnSubmissions = "{tokenprefix}-submissions.json" +) + +// submissions is the the structure that is updated and cached for record A +// when record B links to record A in order to participate in a runoff vote. +// Record A must be a runoff vote parent record. Record B declares its intent +// on being a runoff vote submission using the VoteMetadata LinkTo field. The +// submissions list contains all records that have linked to record A. The list +// will only contain public records. The submissions list is saved to disk in +// the ticketvote plugin data dir, with the parent record token in the +// filename. +type submissions struct { + Tokens map[string]struct{} `json:"tokens"` +} + +// submissionsCachePath returns the path to the submissions list for the +// provided record token. The token prefix is used in the file path so that the +// submissions list can be retrieved using either the full token or the token +// prefix. +func (p *ticketVotePlugin) submissionsCachePath(token []byte) string { + t := util.TokenPrefix(token) + fn := strings.Replace(fnSubmissions, "{tokenprefix}", t, 1) + return filepath.Join(p.dataDir, fn) +} + +// submissionsCacheWithLock return the submissions list for a record token. If +// a submissions list does not exist for the token then an empty list will be +// returned. +// +// This function must be called WITH the lock held. +func (p *ticketVotePlugin) submissionsCacheWithLock(token []byte) (*submissions, error) { + fp := p.submissionsCachePath(token) + b, err := ioutil.ReadFile(fp) + if err != nil { + var e *os.PathError + if errors.As(err, &e) && !os.IsExist(err) { + // File does't exist. Return an empty submissions list. + return &submissions{ + Tokens: make(map[string]struct{}), + }, nil + } + } + + var s submissions + err = json.Unmarshal(b, &s) + if err != nil { + return nil, err + } + + return &s, nil +} + +// submissionsCache return the submissions list for a record token. If a linked +// from list does not exist for the token then an empty list will be returned. +// +// This function must be called WITHOUT the lock held. +func (p *ticketVotePlugin) submissionsCache(token []byte) (*submissions, error) { + p.Lock() + defer p.Unlock() + + return p.submissionsCacheWithLock(token) +} + +// submissionsCacheSaveWithLock saves a submissions to the plugin data dir. +// +// This function must be called WITH the lock held. +func (p *ticketVotePlugin) submissionsCacheSaveWithLock(token []byte, s submissions) error { + b, err := json.Marshal(s) + if err != nil { + return err + } + fp := p.submissionsCachePath(token) + return ioutil.WriteFile(fp, b, 0664) +} + +// submissionsCacheAdd updates the cached submissions list for the parentToken, +// adding the childToken to the list. The full length token MUST be used. +// +// This function must be called WITHOUT the lock held. +func (p *ticketVotePlugin) submissionsCacheAdd(parentToken, childToken string) error { + p.Lock() + defer p.Unlock() + + // Verify tokens + parent, err := tokenDecode(parentToken) + if err != nil { + return err + } + _, err = tokenDecode(childToken) + if err != nil { + return err + } + + // Get existing submissions list + s, err := p.submissionsCacheWithLock(parent) + if err != nil { + return err + } + + // Update list + s.Tokens[childToken] = struct{}{} + + // Save list + err = p.submissionsCacheSaveWithLock(parent, *s) + if err != nil { + return err + } + + log.Debugf("Submissions list updated. Child %v added to parent %v", + childToken, parentToken) + + return nil +} + +// submissionsCacheDel updates the cached submissions list for the parentToken, +// deleting the childToken from the list. +// +// This function must be called WITHOUT the lock held. +func (p *ticketVotePlugin) submissionsCacheDel(parentToken, childToken string) error { + p.Lock() + defer p.Unlock() + + // Verify tokens + parent, err := tokenDecode(parentToken) + if err != nil { + return err + } + _, err = tokenDecode(childToken) + if err != nil { + return err + } + + // Get existing submissions list + s, err := p.submissionsCacheWithLock(parent) + if err != nil { + return err + } + + // Update list + delete(s.Tokens, childToken) + + // Save list + err = p.submissionsCacheSaveWithLock(parent, *s) + if err != nil { + return err + } + + log.Debugf("Submissions list updated. Child %v deleted from parent %v", + childToken, parentToken) + + return nil +} diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go index 7c7f2a9af..3200069ad 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go @@ -198,12 +198,12 @@ func (p *ticketVotePlugin) Cmd(treeID int64, token []byte, cmd, payload string) return p.cmdResults(treeID, token) case ticketvote.CmdSummary: return p.cmdSummary(treeID, token) + case ticketvote.CmdSubmissions: + return p.cmdSubmissions(token) case ticketvote.CmdInventory: return p.cmdInventory() case ticketvote.CmdTimestamps: return p.cmdTimestamps(treeID, token) - case ticketvote.CmdLinkedFrom: - return p.cmdLinkedFrom(token) // Internal plugin commands case cmdStartRunoffSubmission: diff --git a/politeiad/client/ticketvote.go b/politeiad/client/ticketvote.go index ac486d22a..865d85232 100644 --- a/politeiad/client/ticketvote.go +++ b/politeiad/client/ticketvote.go @@ -295,20 +295,18 @@ func (c *Client) TicketVoteSummaries(ctx context.Context, tokens []string) (map[ return summaries, nil } -// TicketVoteLinkedFrom sends a batch of ticketvote plugin LinkedFrom commands -// to the politeiad v1 API. Individual record errors are not returned, the -// token will simply be left out of the returned map. -func (c *Client) TicketVoteLinkedFrom(ctx context.Context, tokens []string) (map[string][]string, error) { +// TicketVoteSubmissions sends the ticketvote plugin Submissions command to the +// politeiad v1 API. +func (c *Client) TicketVoteSubmissions(ctx context.Context, token string) (*ticketvote.SubmissionsReply, error) { // Setup request - cmds := make([]pdv1.PluginCommandV2, 0, len(tokens)) - for _, v := range tokens { - cmds = append(cmds, pdv1.PluginCommandV2{ + cmds := []pdv1.PluginCommandV2{ + { State: pdv1.RecordStateVetted, - Token: v, + Token: token, ID: ticketvote.PluginID, - Command: ticketvote.CmdLinkedFrom, + Command: ticketvote.CmdSubmissions, Payload: "", - }) + }, } // Send request @@ -316,26 +314,23 @@ func (c *Client) TicketVoteLinkedFrom(ctx context.Context, tokens []string) (map if err != nil { return nil, err } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err + } - // Prepare reply - linkedFrom := make(map[string][]string, len(replies)) - for _, v := range replies { - err = extractPluginCommandError(v) - if err != nil { - // Individual record errors are ignored. The token will not be - // included in the returned linkedFrom map. - continue - } - - var lfr ticketvote.LinkedFromReply - err = json.Unmarshal([]byte(v.Payload), &lfr) - if err != nil { - return nil, err - } - linkedFrom[v.Token] = lfr.Tokens + // Decode reply + var sr ticketvote.SubmissionsReply + err = json.Unmarshal([]byte(pcr.Payload), &sr) + if err != nil { + return nil, err } - return linkedFrom, nil + return &sr, nil } // TicketVoteInventory sends the ticketvote plugin Inventory command to the diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index 3f0129150..e3df56874 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -18,15 +18,15 @@ const ( PluginID = "ticketvote" // Plugin commands - CmdAuthorize = "authorize" // Authorize a vote - CmdStart = "start" // Start a vote - CmdCastBallot = "castballot" // Cast a ballot of votes - CmdDetails = "details" // Get vote details - CmdResults = "results" // Get vote results - CmdSummary = "summary" // Get vote summary - CmdLinkedFrom = "linkedfrom" // Get linked from list - CmdInventory = "inventory" // Get inventory by vote status - CmdTimestamps = "timestamps" // Get vote data timestamps + CmdAuthorize = "authorize" // Authorize a vote + CmdStart = "start" // Start a vote + CmdCastBallot = "castballot" // Cast a ballot of votes + CmdDetails = "details" // Get vote details + CmdResults = "results" // Get vote results + CmdSummary = "summary" // Get vote summary + CmdSubmissions = "submissions" // Get runoff vote submissions + CmdInventory = "inventory" // Get inventory by vote status + CmdTimestamps = "timestamps" // Get vote data timestamps // Setting keys are the plugin setting keys that can be used to // override a default plugin setting. Defaults will be overridden @@ -491,6 +491,19 @@ type SummaryReply struct { BestBlock uint32 `json:"bestblock"` } +// Submissions requests the submissions of a runoff vote. The only records that +// will have a submissions list are the parent records in a runoff vote. The +// list will contain all public runoff vote submissions, i.e. records that +// have linked to the parent record using the VoteMetadata.LinkTo field. +type Submissions struct { + Token string `json:"token"` +} + +// SubmissionsReply is the reply to the Submissions command. +type SubmissionsReply struct { + Submissions []string `json:"submissions"` +} + // Inventory requests the tokens of all public, non-abandoned records // categorized by vote status. type Inventory struct{} @@ -548,14 +561,3 @@ type TimestampsReply struct { Details Timestamp `json:"details,omitempty"` Votes map[string]Timestamp `json:"votes,omitempty"` // [ticket]Timestamp } - -// LinkedFrom requests the linked from list for a record. The only records that -// will have a linked from list are the parent records in a runoff vote. The -// linked from list will contain all runoff vote submissions, i.e. records that -// linked to the runoff parent record using the VoteMetadata.LinkTo field. -type LinkedFrom struct{} - -// LinkedFromReply is the reply to the LinkedFrom command. -type LinkedFromReply struct { - Tokens []string `json:"tokens"` -} diff --git a/politeiawww/api/ticketvote/v1/v1.go b/politeiawww/api/ticketvote/v1/v1.go index 95e80c01b..f2909bfc8 100644 --- a/politeiawww/api/ticketvote/v1/v1.go +++ b/politeiawww/api/ticketvote/v1/v1.go @@ -10,16 +10,16 @@ const ( // APIRoute is prefixed onto all routes defined in this package. APIRoute = "/ticketvote/v1" - RoutePolicy = "/policy" - RouteAuthorize = "/authorize" - RouteStart = "/start" - RouteCastBallot = "/castballot" - RouteDetails = "/details" - RouteResults = "/results" - RouteSummaries = "/summaries" - RouteLinkedFrom = "/linkedfrom" - RouteInventory = "/inventory" - RouteTimestamps = "/timestamps" + RoutePolicy = "/policy" + RouteAuthorize = "/authorize" + RouteStart = "/start" + RouteCastBallot = "/castballot" + RouteDetails = "/details" + RouteResults = "/results" + RouteSummaries = "/summaries" + RouteSubmissions = "/submissions" + RouteInventory = "/inventory" + RouteTimestamps = "/timestamps" ) // ErrorCodeT represents a user error code. @@ -478,19 +478,17 @@ type SummariesReply struct { Summaries map[string]Summary `json:"summaries"` // [token]Summary } -// LinkedFrom requests the linked from list for a record. The only records that -// will have a linked from list are the parent records in a runoff vote. The -// linked from list will contain all runoff vote submissions, i.e. records that -// linked to the runoff parent record using the VoteMetadata.LinkTo field. -type LinkedFrom struct { - Tokens []string `json:"tokens"` +// Submissions requests the submissions of a runoff vote. The only records that +// will have a submissions list are the parent records in a runoff vote. The +// list will contain all public runoff vote submissions, i.e. records that +// have linked to the parent record using the VoteMetadata.LinkTo field. +type Submissions struct { + Token string `json:"token"` } -// LinkedFromReply is the reply to the LinkedFrom command. If a provided token -// does not correspond to a record then it will not be included in the returned -// map. -type LinkedFromReply struct { - LinkedFrom map[string][]string `json:"linkedfrom"` +// SubmissionsReply is the reply to the Submissions command. +type SubmissionsReply struct { + Submissions []string `json:"submissions"` } const ( diff --git a/politeiawww/client/ticketvote.go b/politeiawww/client/ticketvote.go index 8e7c233b5..9c646d6e2 100644 --- a/politeiawww/client/ticketvote.go +++ b/politeiawww/client/ticketvote.go @@ -154,25 +154,25 @@ func (c *Client) TicketVoteSummaries(s tkv1.Summaries) (*tkv1.SummariesReply, er return &sr, nil } -// TicketVoteLinkedFrom sends a ticketvote v1 LinkedFrom request to +// TicketVoteSubmissions sends a ticketvote v1 Submissions request to // politeiawww. -func (c *Client) TicketVoteLinkedFrom(lf tkv1.LinkedFrom) (*tkv1.LinkedFromReply, error) { +func (c *Client) TicketVoteSubmissions(s tkv1.Submissions) (*tkv1.SubmissionsReply, error) { resBody, err := c.makeReq(http.MethodPost, - tkv1.APIRoute, tkv1.RouteLinkedFrom, lf) + tkv1.APIRoute, tkv1.RouteSubmissions, s) if err != nil { return nil, err } - var lfr tkv1.LinkedFromReply - err = json.Unmarshal(resBody, &lfr) + var sr tkv1.SubmissionsReply + err = json.Unmarshal(resBody, &sr) if err != nil { return nil, err } if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(lfr)) + fmt.Printf("%v\n", util.FormatJSON(sr)) } - return &lfr, nil + return &sr, nil } // TicketVoteInventory sends a ticketvote v1 Inventory request to politeiawww. diff --git a/politeiawww/cmd/pictl/help.go b/politeiawww/cmd/pictl/cmdhelp.go similarity index 93% rename from politeiawww/cmd/pictl/help.go rename to politeiawww/cmd/pictl/cmdhelp.go index 0b6d047af..ac4c87a42 100644 --- a/politeiawww/cmd/pictl/help.go +++ b/politeiawww/cmd/pictl/cmdhelp.go @@ -10,18 +10,18 @@ import ( "github.com/decred/politeia/politeiawww/cmd/shared" ) -// helpCmd prints a detailed help message for the specified command. -type helpCmd struct { +// cmdHelp prints a detailed help message for the specified command. +type cmdHelp struct { Args struct { Command string `positional-arg-name:"command"` } `positional-args:"true" required:"true"` } -// Execute executes the helpCmd command. +// Execute executes the cmdHelp command. // // This function satisfies the go-flags Commander interface. -func (cmd *helpCmd) Execute(args []string) error { - switch cmd.Args.Command { +func (c *cmdHelp) Execute(args []string) error { + switch c.Args.Command { // Server commands case "version": fmt.Printf("%s\n", shared.VersionHelpMsg) @@ -83,6 +83,8 @@ func (cmd *helpCmd) Execute(args []string) error { fmt.Printf("%s\n", voteResultsHelpMsg) case "votesummaries": fmt.Printf("%s\n", voteSummariesHelpMsg) + case "votesubmissions": + fmt.Printf("%s\n", voteSubmissionsHelpMsg) case "voteinv": fmt.Printf("%s\n", voteInvHelpMsg) case "votetimestamps": diff --git a/politeiawww/cmd/pictl/cmdvotestart.go b/politeiawww/cmd/pictl/cmdvotestart.go index f4680a6bf..04bfd6c16 100644 --- a/politeiawww/cmd/pictl/cmdvotestart.go +++ b/politeiawww/cmd/pictl/cmdvotestart.go @@ -96,21 +96,17 @@ func voteStartStandard(token string, duration, quorum, pass uint32, pc *pclient. func voteStartRunoff(parentToken string, duration, quorum, pass uint32, pc *pclient.Client) (*tkv1.StartReply, error) { // Get runoff vote submissions - lf := tkv1.LinkedFrom{ - Tokens: []string{parentToken}, + s := tkv1.Submissions{ + Token: parentToken, } - lfr, err := pc.TicketVoteLinkedFrom(lf) + sr, err := pc.TicketVoteSubmissions(s) if err != nil { - return nil, fmt.Errorf("TicketVoteLinkedFrom: %v", err) - } - linkedFrom, ok := lfr.LinkedFrom[parentToken] - if !ok { - return nil, fmt.Errorf("linked from not found %v", parentToken) + return nil, fmt.Errorf("TicketVoteSubmissions: %v", err) } // Prepare start details for each submission - starts := make([]tkv1.StartDetails, 0, len(linkedFrom)) - for _, v := range linkedFrom { + starts := make([]tkv1.StartDetails, 0, len(sr.Submissions)) + for _, v := range sr.Submissions { // Get record d := rcv1.Details{ State: rcv1.RecordStateVetted, @@ -167,10 +163,10 @@ func voteStartRunoff(parentToken string, duration, quorum, pass uint32, pc *pcli } // Send request - s := tkv1.Start{ + ts := tkv1.Start{ Starts: starts, } - return pc.TicketVoteStart(s) + return pc.TicketVoteStart(ts) } // Execute executes the cmdVoteStart command. diff --git a/politeiawww/cmd/pictl/cmdvotesubmissions.go b/politeiawww/cmd/pictl/cmdvotesubmissions.go new file mode 100644 index 000000000..9eefe2dad --- /dev/null +++ b/politeiawww/cmd/pictl/cmdvotesubmissions.go @@ -0,0 +1,56 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" + pclient "github.com/decred/politeia/politeiawww/client" +) + +// cmdVoteSubmissions retrieves vote details for the provided record. +type cmdVoteSubmissions struct { + Args struct { + Token string `positional-arg-name:"token"` + } `positional-args:"true" required:"true"` +} + +// Execute executes the cmdVoteSubmissions command. +// +// This function satisfies the go-flags Commander interface. +func (c *cmdVoteSubmissions) Execute(args []string) error { + // Setup client + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, + } + pc, err := pclient.New(cfg.Host, opts) + if err != nil { + return err + } + + // Get vote details + s := tkv1.Submissions{ + Token: c.Args.Token, + } + sr, err := pc.TicketVoteSubmissions(s) + if err != nil { + return err + } + + // Print submissions + printJSON(sr) + + return nil +} + +// voteSubmissionsHelpMsg is printed to stdout by the help command. +const voteSubmissionsHelpMsg = `votesubmissions "token" + +Get the list of submissions for a runoff vote. The token should be the token of +the runoff vote parent record. + +Arguments: +1. token (string, required) Record token.` diff --git a/politeiawww/cmd/pictl/pictl.go b/politeiawww/cmd/pictl/pictl.go index 6b2f7eebc..fafbbbe35 100644 --- a/politeiawww/cmd/pictl/pictl.go +++ b/politeiawww/cmd/pictl/pictl.go @@ -36,7 +36,7 @@ type pictl struct { Config shared.Config // Basic commands - Help helpCmd `command:"help"` + Help cmdHelp `command:"help"` // Server commands Version shared.VersionCmd `command:"version"` @@ -84,15 +84,16 @@ type pictl struct { CommentTimestamps cmdCommentTimestamps `command:"commenttimestamps"` // Vote commands - VotePolicy cmdVotePolicy `command:"votepolicy"` - VoteAuthorize cmdVoteAuthorize `command:"voteauthorize"` - VoteStart cmdVoteStart `command:"votestart"` - CastBallot cmdCastBallot `command:"castballot"` - VoteDetails cmdVoteDetails `command:"votedetails"` - VoteResults cmdVoteResults `command:"voteresults"` - VoteSummaries cmdVoteSummaries `command:"votesummaries"` - VoteInv cmdVoteInv `command:"voteinv"` - VoteTimestamps cmdVoteTimestamps `command:"votetimestamps"` + VotePolicy cmdVotePolicy `command:"votepolicy"` + VoteAuthorize cmdVoteAuthorize `command:"voteauthorize"` + VoteStart cmdVoteStart `command:"votestart"` + CastBallot cmdCastBallot `command:"castballot"` + VoteDetails cmdVoteDetails `command:"votedetails"` + VoteResults cmdVoteResults `command:"voteresults"` + VoteSummaries cmdVoteSummaries `command:"votesummaries"` + VoteSubmissions cmdVoteSubmissions `command:"votesubmissions"` + VoteInv cmdVoteInv `command:"voteinv"` + VoteTimestamps cmdVoteTimestamps `command:"votetimestamps"` // Websocket commands Subscribe subscribeCmd `command:"subscribe"` diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 61b933312..3de05c4c3 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -138,7 +138,7 @@ func (p *politeiawww) setupPiRoutes(r *records.Records, c *comments.Comments, t tkv1.RouteSummaries, t.HandleSummaries, permissionPublic) p.addRoute(http.MethodPost, tkv1.APIRoute, - tkv1.RouteLinkedFrom, t.HandleLinkedFrom, + tkv1.RouteSubmissions, t.HandleSubmissions, permissionPublic) p.addRoute(http.MethodPost, tkv1.APIRoute, tkv1.RouteInventory, t.HandleInventory, diff --git a/politeiawww/ticketvote/process.go b/politeiawww/ticketvote/process.go index 63b119c31..8c6fc8dc2 100644 --- a/politeiawww/ticketvote/process.go +++ b/politeiawww/ticketvote/process.go @@ -182,16 +182,16 @@ func (t *TicketVote) processSummaries(ctx context.Context, s v1.Summaries) (*v1. }, nil } -func (t *TicketVote) processLinkedFrom(ctx context.Context, lf v1.LinkedFrom) (*v1.LinkedFromReply, error) { - log.Tracef("processLinkedFrom: %v", lf.Tokens) +func (t *TicketVote) processSubmissions(ctx context.Context, s v1.Submissions) (*v1.SubmissionsReply, error) { + log.Tracef("processSubmissions: %v", s.Token) - tlf, err := t.politeiad.TicketVoteLinkedFrom(ctx, lf.Tokens) + sr, err := t.politeiad.TicketVoteSubmissions(ctx, s.Token) if err != nil { return nil, err } - return &v1.LinkedFromReply{ - LinkedFrom: tlf, + return &v1.SubmissionsReply{ + Submissions: sr.Submissions, }, nil } diff --git a/politeiawww/ticketvote/ticketvote.go b/politeiawww/ticketvote/ticketvote.go index 32a1c3e44..644c5f116 100644 --- a/politeiawww/ticketvote/ticketvote.go +++ b/politeiawww/ticketvote/ticketvote.go @@ -197,29 +197,29 @@ func (t *TicketVote) HandleSummaries(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, sr) } -// HandleLinkedFrom is the request handler for the ticketvote v1 LinkedFrom +// HandleSubmissions is the request handler for the ticketvote v1 Submissions // route. -func (t *TicketVote) HandleLinkedFrom(w http.ResponseWriter, r *http.Request) { - log.Tracef("HandleLinkedFrom") +func (t *TicketVote) HandleSubmissions(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleSubmissions") - var lf v1.LinkedFrom + var s v1.Submissions decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&lf); err != nil { - respondWithError(w, r, "HandleLinkedFrom: unmarshal", + if err := decoder.Decode(&s); err != nil { + respondWithError(w, r, "HandleSubmissions: unmarshal", v1.UserErrorReply{ ErrorCode: v1.ErrorCodeInputInvalid, }) return } - lfr, err := t.processLinkedFrom(r.Context(), lf) + sr, err := t.processSubmissions(r.Context(), s) if err != nil { - respondWithError(w, r, "HandleLinkedFrom: processLinkedFrom: %v", + respondWithError(w, r, "HandleSubmissions: processSubmissions: %v", err) return } - util.RespondWithJSON(w, http.StatusOK, lfr) + util.RespondWithJSON(w, http.StatusOK, sr) } // HandleInventory is the request handler for the ticketvote v1 Inventory From f836f33004b5d03745685e82b64b8beb7483b452 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 9 Feb 2021 10:21:06 -0600 Subject: [PATCH 284/449] Fix runoff vote bug. --- .../backend/tlogbe/plugins/ticketvote/cmds.go | 20 ++++---- .../tlogbe/plugins/ticketvote/submissions.go | 4 +- politeiad/backend/tlogbe/tlog/anchor.go | 5 +- politeiad/backend/tlogbe/tlogbe.go | 46 +++++++++++++------ politeiad/politeiad.go | 4 +- politeiawww/cmd/pictl/cmdproposals.go | 1 + politeiawww/cmd/pictl/pictl.go | 1 + 7 files changed, 48 insertions(+), 33 deletions(-) diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go index 1a7ed0577..bbd61eb87 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go @@ -1500,12 +1500,12 @@ func (p *ticketVotePlugin) cmdRunoffDetails(treeID int64) (string, error) { func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs startRunoffSubmission) error { // Sanity check - s := srs.StartDetails - t, err := tokenDecode(s.Params.Token) + sd := srs.StartDetails + t, err := tokenDecode(sd.Params.Token) if err != nil { return err } - if bytes.Equal(token, t) { + if !bytes.Equal(token, t) { return fmt.Errorf("invalid token") } @@ -1549,10 +1549,10 @@ func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs sta if err != nil { return fmt.Errorf("GetVetted: %v", err) } - version := strconv.FormatUint(uint64(s.Params.Version), 10) + version := strconv.FormatUint(uint64(sd.Params.Version), 10) if r.Version != version { e := fmt.Sprintf("version is not latest %v: got %v, want %v", - s.Params.Token, s.Params.Version, r.Version) + sd.Params.Token, sd.Params.Version, r.Version) return backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeRecordVersionInvalid), @@ -1562,9 +1562,9 @@ func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs sta // Prepare vote details vd := ticketvote.VoteDetails{ - Params: s.Params, - PublicKey: s.PublicKey, - Signature: s.Signature, + Params: sd.Params, + PublicKey: sd.PublicKey, + Signature: sd.Signature, StartBlockHeight: srr.StartBlockHeight, StartBlockHash: srr.StartBlockHash, EndBlockHeight: srr.EndBlockHeight, @@ -1896,8 +1896,8 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. if errors.As(err, &ue) { return nil, err } - return nil, fmt.Errorf("VettedPluginCmd %x %v %v %v: %v", - token, ticketvote.PluginID, cmdStartRunoffSubmission, b, err) + return nil, fmt.Errorf("VettedPluginCmd %x %v %v: %v", + token, ticketvote.PluginID, cmdStartRunoffSubmission, err) } } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/submissions.go b/politeiad/backend/tlogbe/plugins/ticketvote/submissions.go index da2db1baf..4c18390ca 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/submissions.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/submissions.go @@ -126,7 +126,7 @@ func (p *ticketVotePlugin) submissionsCacheAdd(parentToken, childToken string) e return err } - log.Debugf("Submissions list updated. Child %v added to parent %v", + log.Debugf("Submissions list add: child %v added to parent %v", childToken, parentToken) return nil @@ -165,7 +165,7 @@ func (p *ticketVotePlugin) submissionsCacheDel(parentToken, childToken string) e return err } - log.Debugf("Submissions list updated. Child %v deleted from parent %v", + log.Debugf("Submissions list del: child %v deleted from parent %v", childToken, parentToken) return nil diff --git a/politeiad/backend/tlogbe/tlog/anchor.go b/politeiad/backend/tlogbe/tlog/anchor.go index 78228acb6..16e4e4ee5 100644 --- a/politeiad/backend/tlogbe/tlog/anchor.go +++ b/politeiad/backend/tlogbe/tlog/anchor.go @@ -30,10 +30,7 @@ const ( // currently drops an anchor on the hour mark so we submit new // anchors a few minutes prior to that. // Seconds Minutes Hours Days Months DayOfWeek - - // TODO put back when done testing - // anchorSchedule = "0 56 * * * *" // At minute 56 of every hour - anchorSchedule = "0 */5 * * * *" // Every 5 minutes + anchorSchedule = "0 56 * * * *" // At minute 56 of every hour // anchorID is included in the timestamp and verify requests as a // unique identifier. diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 6a84e4715..a24f82dfb 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -1161,30 +1161,46 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, backend.MDStatus[status], status) } - // Call post plugin hooks - t.unvetted.PluginHookPost(treeID, token, - plugins.HookTypeSetRecordStatusPost, string(b)) - log.Debugf("Status change %x from %v (%v) to %v (%v)", token, backend.MDStatus[currStatus], currStatus, backend.MDStatus[status], status) - // Update inventory cache - if status == backend.MDStatusVetted { - // Record was made public + switch status { + case backend.MDStatusVetted: + // Record was made public. Actions must now be executed on the + // vetted tlog instance. + + // Call post plugin hooks + t.vetted.PluginHookPost(treeID, token, + plugins.HookTypeSetRecordStatusPost, string(b)) + + // Update inventory cache t.inventoryMoveToVetted(token, currStatus, status) - } else { - // All other status changes + + // Retrieve record + r, err = t.GetVetted(token, "") + if err != nil { + return nil, err + } + + default: + // All other status changes. The record is still unvetted. + + // Call post plugin hooks + t.unvetted.PluginHookPost(treeID, token, + plugins.HookTypeSetRecordStatusPost, string(b)) + + // Update inventory cache t.inventoryUpdate(stateUnvetted, token, currStatus, status) - } - // Return the updated record. If the record was made public it is - // now a vetted record and must be fetched accordingly. - if status == backend.MDStatusVetted { - return t.GetVetted(token, "") + // Retrieve record + r, err = t.GetUnvetted(token, "") + if err != nil { + return nil, err + } } - return t.GetUnvetted(token, "") + return r, nil } // This function must be called WITH the record lock held. diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 361f86f7b..5601f36ff 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -1310,8 +1310,8 @@ func (p *politeia) pluginCommandBatch(w http.ResponseWriter, r *http.Request) { // respond with a server error. t := time.Now().Unix() log.Errorf("%v %v: batched plugin cmd failed: pluginID:%v "+ - "cmd:%v payload:%v err:%v", remoteAddr(r), t, pc.ID, - pc.Command, pc.Payload, err) + "cmd:%v err:'%v' payload:%v", remoteAddr(r), t, pc.ID, + pc.Command, err, pc.Payload) p.respondWithServerError(w, t) return diff --git a/politeiawww/cmd/pictl/cmdproposals.go b/politeiawww/cmd/pictl/cmdproposals.go index 7b415baf9..d74e75473 100644 --- a/politeiawww/cmd/pictl/cmdproposals.go +++ b/politeiawww/cmd/pictl/cmdproposals.go @@ -67,6 +67,7 @@ func (c *cmdProposals) Execute(args []string) error { if err != nil { return err } + printf("-----\n") } return nil diff --git a/politeiawww/cmd/pictl/pictl.go b/politeiawww/cmd/pictl/pictl.go index fafbbbe35..910d017ec 100644 --- a/politeiawww/cmd/pictl/pictl.go +++ b/politeiawww/cmd/pictl/pictl.go @@ -169,6 +169,7 @@ Vote commands votedetails (public) Get details for a vote voteresults (public) Get full vote results votesummaries (public) Get vote summaries + votesubmissions (public) Get runoff vote submissions voteinv (public) Get proposal inventory by vote status votetimestamps (public) Get vote timestamps From 535c82ddf2f30260d2f683a890d921c163cd83c6 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 9 Feb 2021 13:17:31 -0600 Subject: [PATCH 285/449] Fix record state bug on status change. --- politeiawww/records/process.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index 44164be9b..37fa0a0e8 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -268,13 +268,23 @@ func (r *Records) processSetStatus(ctx context.Context, ss v1.SetStatus, u user. } } - rc := convertRecordToV1(*pdr, ss.State) + // Convert the record. The state may need to be updated if the + // record was made public. + var state string + switch ss.Status { + case v1.RecordStatusPublic: + // Flip state from unvetted to vetted + state = pdv1.RecordStateVetted + default: + state = ss.State + } + rc := convertRecordToV1(*pdr, state) recordPopulateUserData(&rc, u) // Emit event r.events.Emit(EventTypeSetStatus, EventSetStatus{ - State: ss.State, + State: state, Record: rc, }) From 9f8811e9862f2fb0207fcb108b446594eabc2fa5 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 9 Feb 2021 13:45:48 -0600 Subject: [PATCH 286/449] Prevent expired linkby on set status. --- .../tlogbe/plugins/ticketvote/hooks.go | 56 +++++++++++++++++++ .../tlogbe/plugins/ticketvote/ticketvote.go | 14 +++++ politeiad/plugins/ticketvote/ticketvote.go | 3 +- politeiad/plugins/user/user.go | 4 -- politeiawww/cmd/pictl/cmdproposalnew.go | 56 ++++++++++++------- politeiawww/cmd/pictl/cmdproposalstatusset.go | 1 - politeiawww/cmd/pictl/proposal.go | 33 +++++++++-- 7 files changed, 137 insertions(+), 30 deletions(-) diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go b/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go index 14a0d81bb..730641d97 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go @@ -249,6 +249,62 @@ func (p *ticketVotePlugin) hookEditRecordPre(payload string) error { return nil } +func (p *ticketVotePlugin) hookSetRecordStatusPre(payload string) error { + var srs plugins.HookSetRecordStatus + err := json.Unmarshal([]byte(payload), &srs) + if err != nil { + return err + } + + // Check if the LinkTo has been set + vm, err := voteMetadataDecode(srs.Current.Files) + if err != nil { + return err + } + if vm != nil && vm.LinkTo != "" { + // LinkTo has been set. Verify that the deadline to link to this + // record has not expired. We only need to do this when a record + // is being made public since the submissions list of the parent + // record is only updated for public records. This update occurs + // in the set status post hook. + switch srs.RecordMetadata.Status { + case backend.MDStatusVetted: + // Get the parent record + token, err := tokenDecode(vm.LinkTo) + if err != nil { + return err + } + r, err := p.backend.GetVetted(token, "") + if err != nil { + return err + } + + // Verify linkby has not expired + vmParent, err := voteMetadataDecode(r.Files) + if err != nil { + return err + } + if vmParent == nil { + e := fmt.Sprintf("vote metadata does not exist on parent record %v", + srs.RecordMetadata.Token) + panic(e) + } + if time.Now().Unix() > vmParent.LinkBy { + return backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), + ErrorContext: "parent record linkby has expired", + } + } + + default: + // Nothing to do + } + } + + return nil +} + func (p *ticketVotePlugin) hookSetRecordStatusPost(payload string) error { var srs plugins.HookSetRecordStatus err := json.Unmarshal([]byte(payload), &srs) diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go index 3200069ad..b717ef1c6 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go @@ -226,6 +226,8 @@ func (p *ticketVotePlugin) Hook(treeID int64, token []byte, h plugins.HookT, pay return p.hookNewRecordPre(payload) case plugins.HookTypeEditRecordPre: return p.hookEditRecordPre(payload) + case plugins.HookTypeSetRecordStatusPre: + return p.hookSetRecordStatusPre(payload) case plugins.HookTypeSetRecordStatusPost: return p.hookSetRecordStatusPost(payload) } @@ -305,6 +307,9 @@ func New(backend backend.Backend, tlog plugins.TlogClient, settings []backend.Pl v.Key, v.Value, err) } linkByPeriodMin = i + log.Infof("Plugin setting updated: ticketvote %v %v", + ticketvote.SettingKeyLinkByPeriodMin, linkByPeriodMin) + case ticketvote.SettingKeyLinkByPeriodMax: i, err := strconv.ParseInt(v.Value, 10, 64) if err != nil { @@ -312,6 +317,9 @@ func New(backend backend.Backend, tlog plugins.TlogClient, settings []backend.Pl v.Key, v.Value, err) } linkByPeriodMax = i + log.Infof("Plugin setting updated: ticketvote %v %v", + ticketvote.SettingKeyLinkByPeriodMax, linkByPeriodMax) + case ticketvote.SettingKeyVoteDurationMin: u, err := strconv.ParseUint(v.Value, 10, 64) if err != nil { @@ -319,6 +327,9 @@ func New(backend backend.Backend, tlog plugins.TlogClient, settings []backend.Pl v.Key, v.Value, err) } voteDurationMin = uint32(u) + log.Infof("Plugin setting updated: ticketvote %v %v", + ticketvote.SettingKeyVoteDurationMin, voteDurationMin) + case ticketvote.SettingKeyVoteDurationMax: u, err := strconv.ParseUint(v.Value, 10, 64) if err != nil { @@ -326,6 +337,9 @@ func New(backend backend.Backend, tlog plugins.TlogClient, settings []backend.Pl v.Key, v.Value, err) } voteDurationMax = uint32(u) + log.Infof("Plugin setting updated: ticketvote %v %v", + ticketvote.SettingKeyVoteDurationMax, voteDurationMax) + default: return nil, fmt.Errorf("invalid plugin setting '%v'", v.Key) } diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index e3df56874..a633b22fc 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -38,7 +38,8 @@ const ( // SettingLinkByPeriodMin is the default minimum amount of time, // in seconds, that the link by period can be set to. This value - // of 2 weeks was chosen arbitrarily. + // of 2 weeks was chosen assuming a 1 week voting period on + // mainnet. SettingLinkByPeriodMin int64 = 1209600 // SettingLinkByPeriodMax is the default maximum amount of time, diff --git a/politeiad/plugins/user/user.go b/politeiad/plugins/user/user.go index 49a7334d6..fdbdb92fc 100644 --- a/politeiad/plugins/user/user.go +++ b/politeiad/plugins/user/user.go @@ -13,10 +13,6 @@ const ( CmdAuthor = "author" // Get record author CmdUserRecords = "userrecords" // Get user submitted records - // TODO MDStream IDs need to be plugin specific. If we can't then - // we need to make a mdstream package to aggregate all the mdstream - // IDs. - // MDStreamIDUserMetadata is the politeiad metadata stream ID for // the UserMetadata structure. MDStreamIDUserMetadata = 1 diff --git a/politeiawww/cmd/pictl/cmdproposalnew.go b/politeiawww/cmd/pictl/cmdproposalnew.go index 47284bacf..342de6c70 100644 --- a/politeiawww/cmd/pictl/cmdproposalnew.go +++ b/politeiawww/cmd/pictl/cmdproposalnew.go @@ -32,7 +32,7 @@ type cmdProposalNew struct { // Metadata fields that can be set by the user Name string `long:"name" optional:"true"` LinkTo string `long:"linkto" optional:"true"` - LinkBy int64 `long:"linkby" optional:"true"` + LinkBy string `long:"linkby" optional:"true"` // Random generates random proposal data. An IndexFile and // Attachments are not required when using this flag. @@ -63,7 +63,7 @@ func (c *cmdProposalNew) Execute(args []string) error { return fmt.Errorf("you cannot provide file arguments and use " + "the --random flag at the same time") - case c.RFP && c.LinkBy != 0: + case c.RFP && c.LinkBy != "": return fmt.Errorf("you cannot use both the --rfp and --linkby " + "flags at the same time") } @@ -157,14 +157,23 @@ func (c *cmdProposalNew) Execute(args []string) error { }) // Setup vote metadata - if c.RFP { + var linkBy int64 + switch { + case c.RFP: // Set linkby to a month from now - c.LinkBy = time.Now().Add(time.Hour * 24 * 30).Unix() + linkBy = time.Now().Add(time.Hour * 24 * 30).Unix() + case c.LinkBy != "": + // Parse the provided linkby + d, err := time.ParseDuration(c.LinkBy) + if err != nil { + return fmt.Errorf("unable to parse linkby: %v", err) + } + linkBy = time.Now().Add(d).Unix() } - if c.LinkBy != 0 || c.LinkTo != "" { + if linkBy != 0 || c.LinkTo != "" { vm := piv1.VoteMetadata{ LinkTo: c.LinkTo, - LinkBy: c.LinkBy, + LinkBy: linkBy, } vmb, err := json.Marshal(vm) if err != nil { @@ -179,7 +188,7 @@ func (c *cmdProposalNew) Execute(args []string) error { } // Print proposal to stdout - printf("Proposal submitted\n") + printf("Files\n") err = printProposalFiles(files) if err != nil { return err @@ -234,18 +243,27 @@ to link to and an existing RFP proposal. Arguments: 1. indexfile (string, required) Index file -2. attachments (string, optional) Attachment files +2. attachments (string) Attachment files Flags: - --name (string, optional) Name of the proposal. - --linkto (string, optional) Token of an existing public proposal to link to. - --linkby (int64, optional) UNIX timestamp of the RFP deadline. Setting this - field will make the proposal an RFP with a - submission deadline specified by the linkby. - --random (bool, optional) Generate a random proposal. If this flag is used - then the markdownfile argument is no longer - required and any provided files will be ignored. - --rfp (bool, optional) Make the proposal an RFP by setting the linkby to - one month from the current time. This is intended - to be used in place of --linkby. + --name (string) Name of the proposal. + --linkto (string) Token of an existing public proposal to link to. + --linkby (string) UNIX timestamp of the RFP deadline. Setting this field will + make the proposal an RFP with a submission deadline set by + the linkby. Valid linkby units are: + s (seconds), m (minutes), h (hours) + --random (bool) Generate a random proposal. If this flag is used then the + markdownfile argument is no longer required and any + provided files will be ignored. + --rfp (bool) Make the proposal an RFP by setting the linkby to one month + from the current time. This is intended to be used in place + of --linkby. + +Examples: + +# Set linkby 24 hours from current time +$ pictl proposalnew --random --linkby=24h + +# Use --rfp to set the linky 1 month from current time +$ pictl proposalnew --rfp index.md proposalmetadata.json ` diff --git a/politeiawww/cmd/pictl/cmdproposalstatusset.go b/politeiawww/cmd/pictl/cmdproposalstatusset.go index 1cee81b8a..8dc134d6a 100644 --- a/politeiawww/cmd/pictl/cmdproposalstatusset.go +++ b/politeiawww/cmd/pictl/cmdproposalstatusset.go @@ -134,7 +134,6 @@ func (c *cmdProposalSetStatus) Execute(args []string) error { } // Print proposal to stdout - printf("Proposal status updated\n") err = printProposal(ssr.Record) if err != nil { return err diff --git a/politeiawww/cmd/pictl/proposal.go b/politeiawww/cmd/pictl/proposal.go index 6eec8f9e6..00618e704 100644 --- a/politeiawww/cmd/pictl/proposal.go +++ b/politeiawww/cmd/pictl/proposal.go @@ -30,6 +30,33 @@ func printProposalFiles(files []rcv1.File) error { size := byteCountSI(int64(len(b))) printf(" %-22v %-26v %v\n", v.Name, v.MIME, size) } + + // Its possible for a proposal metadata to not exist if the + // proposal has been censored. + pm, err := proposalMetadataDecode(files) + if err != nil { + return err + } + if pm != nil { + printf("%v\n", piv1.FileNameProposalMetadata) + printf(" Name: %v\n", pm.Name) + } + + // A vote metadata file is optional + vm, err := voteMetadataDecode(files) + if err != nil { + return err + } + if vm != nil { + printf("%v\n", piv1.FileNameVoteMetadata) + if vm.LinkTo != "" { + printf(" LinkTo: %v\n", vm.LinkTo) + } + if vm.LinkBy != 0 { + printf(" LinkBy: %v\n", timestampFromUnix(vm.LinkBy)) + } + } + return nil } @@ -160,8 +187,7 @@ func signedMerkleRoot(files []rcv1.File, fid *identity.FullIdentity) (string, er } // proposalMetadataDecode decodes and returns the ProposalMetadata from the -// provided record files. An error is returned if a ProposalMetadata is not -// found. +// provided record files. nil is returned if a ProposalMetadata is not found. func proposalMetadataDecode(files []rcv1.File) (*piv1.ProposalMetadata, error) { var propMD *piv1.ProposalMetadata for _, v := range files { @@ -179,9 +205,6 @@ func proposalMetadataDecode(files []rcv1.File) (*piv1.ProposalMetadata, error) { break } } - if propMD == nil { - return nil, fmt.Errorf("proposal metadata not found") - } return propMD, nil } From ba14ba60716df1a6919aa86c46d1ebc5d7100f4b Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 9 Feb 2021 14:19:44 -0600 Subject: [PATCH 287/449] Cleanup. --- .../backend/tlogbe/plugins/dcrdata/dcrdata.go | 56 +++++++++---------- politeiad/backend/tlogbe/tlog/anchor.go | 4 +- politeiad/backend/tlogbe/tlog/tlog.go | 2 - politeiad/backend/tlogbe/tlogbe.go | 2 - politeiad/plugins/dcrdata/dcrdata.go | 51 +++++++++++------ politeiad/plugins/ticketvote/ticketvote.go | 37 ++++++------ politeiad/politeiad.go | 10 +++- 7 files changed, 87 insertions(+), 75 deletions(-) diff --git a/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go b/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go index 0cb98d903..06f01297e 100644 --- a/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go +++ b/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go @@ -27,10 +27,6 @@ import ( ) const ( - // Plugin settings - pluginSettingHostHTTP = "hosthttp" - pluginSettingHostWS = "hostws" - // Dcrdata routes routeBestBlock = "/api/block/best" routeBlockDetails = "/api/block/{height}" @@ -626,7 +622,7 @@ func (p *dcrdataPlugin) Fsck(treeIDs []int64) error { return nil } -// TODO Settings returns the plugin's settings. +// Settings returns the plugin's settings. // // This function satisfies the plugins.PluginClient interface. func (p *dcrdataPlugin) Settings() []backend.PluginSetting { @@ -636,42 +632,40 @@ func (p *dcrdataPlugin) Settings() []backend.PluginSetting { } func New(settings []backend.PluginSetting, activeNetParams *chaincfg.Params) (*dcrdataPlugin, error) { - // Unpack plugin settings + // Plugin setting var ( hostHTTP string hostWS string ) + + // Set plugin settings to defaults. These will be overwritten if + // the setting was specified by the user. + switch activeNetParams.Name { + case chaincfg.MainNetParams().Name: + hostHTTP = dcrdata.SettingHostHTTPMainNet + hostWS = dcrdata.SettingHostWSMainNet + case chaincfg.TestNet3Params().Name: + hostHTTP = dcrdata.SettingHostHTTPTestNet + hostWS = dcrdata.SettingHostWSTestNet + default: + return nil, fmt.Errorf("unknown active net: %v", activeNetParams.Name) + } + + // Override defaults with any passed in settings for _, v := range settings { switch v.Key { - case pluginSettingHostHTTP: + case dcrdata.SettingKeyHostHTTP: hostHTTP = v.Value - case pluginSettingHostWS: + log.Infof("Plugin setting updated: dcrdata %v %v", + dcrdata.SettingKeyHostHTTP, hostHTTP) + + case dcrdata.SettingKeyHostWS: hostWS = v.Value - default: - return nil, fmt.Errorf("invalid plugin setting '%v'", v.Key) - } - } + log.Infof("Plugin setting updated: dcrdata %v %v", + dcrdata.SettingKeyHostWS, hostWS) - // Set optional plugin settings to default values if a value was - // not specified. - if hostHTTP == "" { - switch activeNetParams.Name { - case chaincfg.MainNetParams().Name: - hostHTTP = dcrdata.DefaultHostHTTPMainNet - case chaincfg.TestNet3Params().Name: - hostHTTP = dcrdata.DefaultHostHTTPTestNet default: - return nil, fmt.Errorf("unknown active net: %v", activeNetParams.Name) - } - } - if hostWS == "" { - switch activeNetParams.Name { - case chaincfg.MainNetParams().Name: - hostWS = dcrdata.DefaultHostWSMainNet - case chaincfg.TestNet3Params().Name: - hostWS = dcrdata.DefaultHostWSTestNet - default: - return nil, fmt.Errorf("unknown active net: %v", activeNetParams.Name) + return nil, fmt.Errorf("invalid plugin setting '%v'", v.Key) } } diff --git a/politeiad/backend/tlogbe/tlog/anchor.go b/politeiad/backend/tlogbe/tlog/anchor.go index 16e4e4ee5..b534b835c 100644 --- a/politeiad/backend/tlogbe/tlog/anchor.go +++ b/politeiad/backend/tlogbe/tlog/anchor.go @@ -272,9 +272,7 @@ func (t *Tlog) anchorWait(anchors []anchor, digests []string) { // enough time is given for the anchor transaction to receive 6 // confirmations. This is based on the fact that each block has // a 99.75% chance of being mined within 30 minutes. - // - // TODO change period to 5 minutes when done testing - period = 1 * time.Minute // check every 5 minute + period = 5 * time.Minute // check every 5 minute retries = 180 / int(period.Minutes()) // for up to 180 minutes ticker = time.NewTicker(period) ) diff --git a/politeiad/backend/tlogbe/tlog/tlog.go b/politeiad/backend/tlogbe/tlog/tlog.go index c033d4f81..dffeb8614 100644 --- a/politeiad/backend/tlogbe/tlog/tlog.go +++ b/politeiad/backend/tlogbe/tlog/tlog.go @@ -491,8 +491,6 @@ type recordBlobsPrepareReply struct { // recordBlobsPrepare prepares the provided record content to be saved to // the blob kv store and appended onto a trillian tree. -// -// TODO test this function func (t *Tlog) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) (*recordBlobsPrepareReply, error) { // Verify there are no duplicate or empty mdstream IDs mdstreamIDs := make(map[uint64]struct{}, len(metadata)) diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index a24f82dfb..cdd072f23 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -443,7 +443,6 @@ func recordMetadataNew(token []byte, files []backend.File, status backend.MDStat }, nil } -// TODO test this function func filesUpdate(filesCurr, filesAdd []backend.File, filesDel []string) []backend.File { // Put current files into a map curr := make(map[string]backend.File, len(filesCurr)) // [filename]File @@ -473,7 +472,6 @@ func filesUpdate(filesCurr, filesAdd []backend.File, filesDel []string) []backen return f } -// TODO test this function func metadataStreamsUpdate(curr, mdAppend, mdOverwrite []backend.MetadataStream) []backend.MetadataStream { // Put current metadata into a map md := make(map[uint64]backend.MetadataStream, len(curr)) diff --git a/politeiad/plugins/dcrdata/dcrdata.go b/politeiad/plugins/dcrdata/dcrdata.go index 1a9235625..20dd231ec 100644 --- a/politeiad/plugins/dcrdata/dcrdata.go +++ b/politeiad/plugins/dcrdata/dcrdata.go @@ -6,8 +6,6 @@ // explorer. package dcrdata -type StatusT int - const ( PluginID = "dcrdata" @@ -17,20 +15,41 @@ const ( CmdTicketPool = "ticketpool" // Get ticket pool CmdTxsTrimmed = "txstrimmed" // Get trimmed transactions - // Default plugin settings - DefaultHostHTTPMainNet = "https://dcrdata.decred.org" - DefaultHostHTTPTestNet = "https://testnet.decred.org" - DefaultHostWSMainNet = "wss://dcrdata.decred.org/ps" - DefaultHostWSTestNet = "wss://testnet.decred.org/ps" - - // Dcrdata connection statuses. - // - // Some commands will return cached results with the connection - // status when dcrdata cannot be reached. It is the callers - // responsibility to determine the correct course of action when - // dcrdata cannot be reached. - StatusInvalid StatusT = 0 - StatusConnected StatusT = 1 + // Setting keys are the plugin setting keys that can be used to + // override a default plugin setting. Defaults will be overridden + // if a plugin setting is provided to the plugin on startup. + SettingKeyHostHTTP = "hosthttp" + SettingKeyHostWS = "hostws" + + // SettingHostHTTPMainNet is the default dcrdata mainnet http host. + SettingHostHTTPMainNet = "https://dcrdata.decred.org" + + // SettingHostHTTPTestNet is the default dcrdata testnet http host. + SettingHostHTTPTestNet = "https://testnet.decred.org" + + // SettingHostWSMainNet is the default dcrdata mainnet websocket + // host. + SettingHostWSMainNet = "wss://dcrdata.decred.org/ps" + + // SettingHostWSTestNet is the default dcrdata testnet websocket + // host. + SettingHostWSTestNet = "wss://testnet.decred.org/ps" +) + +// StatusT represents a dcrdata connection status. Some commands will returned +// cached results and the connection status to let the caller know that the +// cached data may be stale. It is the callers responsibility to determine the +// correct course of action when dcrdata cannot be reached. +type StatusT int + +const ( + // StatusInvalid is an invalid connection status. + StatusInvalid StatusT = 0 + + // StatusConnected is returned when the dcrdata connection is ok. + StatusConnected StatusT = 1 + + // StatusDisconnected is returned when dcrdata cannot be reached. StatusDisconnected StatusT = 2 ) diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index a633b22fc..c5803e14a 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -76,28 +76,27 @@ const ( type ErrorCodeT int const ( - ErrorCodeInvalid ErrorCodeT = 0 - ErrorCodeTokenInvalid ErrorCodeT = iota - ErrorCodePublicKeyInvalid - ErrorCodeSignatureInvalid - ErrorCodeRecordVersionInvalid - ErrorCodeRecordStatusInvalid - ErrorCodeAuthorizationInvalid - ErrorCodeStartDetailsMissing - ErrorCodeStartDetailsInvalid - ErrorCodeVoteParamsInvalid - ErrorCodeVoteStatusInvalid - ErrorCodePageSizeExceeded - ErrorCodeVoteMetadataInvalid - ErrorCodeLinkByInvalid - ErrorCodeLinkToInvalid - - ErrorCodeRunoffVoteParentInvalid - ErrorCodeLinkByNotExpired + ErrorCodeInvalid ErrorCodeT = 0 + ErrorCodeTokenInvalid ErrorCodeT = 1 + ErrorCodePublicKeyInvalid ErrorCodeT = 2 + ErrorCodeSignatureInvalid ErrorCodeT = 3 + ErrorCodeRecordVersionInvalid ErrorCodeT = 4 + ErrorCodeRecordStatusInvalid ErrorCodeT = 5 + ErrorCodeAuthorizationInvalid ErrorCodeT = 6 + ErrorCodeStartDetailsMissing ErrorCodeT = 7 + ErrorCodeStartDetailsInvalid ErrorCodeT = 8 + ErrorCodeVoteParamsInvalid ErrorCodeT = 9 + ErrorCodeVoteStatusInvalid ErrorCodeT = 10 + ErrorCodePageSizeExceeded ErrorCodeT = 11 + ErrorCodeVoteMetadataInvalid ErrorCodeT = 12 + ErrorCodeLinkByInvalid ErrorCodeT = 13 + ErrorCodeLinkToInvalid ErrorCodeT = 14 + ErrorCodeRunoffVoteParentInvalid ErrorCodeT = 15 + ErrorCodeLinkByNotExpired ErrorCodeT = 16 ) var ( - // TODO ErrorCodes contains the human readable error messages. + // ErrorCodes contains the human readable error messages. ErrorCodes = map[ErrorCodeT]string{ ErrorCodeInvalid: "error code invalid", ErrorCodeTokenInvalid: "token invalid", diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 5601f36ff..de3565a14 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -1595,9 +1595,15 @@ func _main() error { case dcrdata.PluginID: unvetted = false case decredplugin.ID: - // TODO decredplugin setup for cms + // decredplugin uses the deprecated plugin methods. This + // plugin is also deprecated will eventually be removed. + unvetted = false + vetted = false case cmsplugin.ID: - // TODO cmsplugin setup for cms + // cmsplugin uses the deprecated plugin methods. This + // plugin is also deprecated will eventually be removed. + unvetted = false + vetted = false } // Register plugin From e32e23838bbd6a6767d8b8f9f964be1a36609e00 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 10 Feb 2021 08:43:26 -0600 Subject: [PATCH 288/449] Fix JSON typo. --- politeiawww/api/records/v1/v1.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go index 779b48f99..898594933 100644 --- a/politeiawww/api/records/v1/v1.go +++ b/politeiawww/api/records/v1/v1.go @@ -393,5 +393,5 @@ type UserRecords struct { // UserRecordsReply is the reply to the UserRecords command. type UserRecordsReply struct { Unvetted []string `json:"unvetted"` - Vetted []string `json:"Vetted"` + Vetted []string `json:"vetted"` } From f27a5435c0c6d226c4fa873f6d52c812475f6250 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 10 Feb 2021 09:03:29 -0600 Subject: [PATCH 289/449] Fix user records vetted bug. --- politeiawww/records/process.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index 37fa0a0e8..029cc65f5 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -516,7 +516,7 @@ func (r *Records) processUserRecords(ctx context.Context, ur v1.UserRecords, u * if ok { unvetted = tokens } - tokens, ok = reply[v1.RecordStateUnvetted] + tokens, ok = reply[v1.RecordStateVetted] if ok { vetted = tokens } From 4d463dad43f879965eabf7a99daf4344c2742ace Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 10 Feb 2021 10:15:45 -0600 Subject: [PATCH 290/449] Redo vote inv and remove pi vote route. --- .../tlogbe/plugins/comments/recordindex.go | 14 +- politeiad/backend/tlogbe/plugins/pi/cmds.go | 81 --- politeiad/backend/tlogbe/plugins/pi/hooks.go | 14 + politeiad/backend/tlogbe/plugins/pi/pi.go | 5 - .../backend/tlogbe/plugins/ticketvote/cmds.go | 77 +-- .../tlogbe/plugins/ticketvote/hooks.go | 9 +- .../tlogbe/plugins/ticketvote/inventory.go | 460 +++++++++++------- .../tlogbe/plugins/ticketvote/summary.go | 5 +- .../tlogbe/plugins/ticketvote/ticketvote.go | 69 +-- politeiad/client/pi.go | 51 -- politeiad/plugins/pi/pi.go | 22 - politeiad/plugins/ticketvote/ticketvote.go | 40 +- politeiawww/api/pi/v1/v1.go | 24 +- politeiawww/api/ticketvote/v1/v1.go | 34 +- politeiawww/cmd/pictl/proposal.go | 2 +- politeiawww/cmd/pictl/ticketvote.go | 1 - politeiawww/pi/pi.go | 15 - politeiawww/pi/process.go | 18 - politeiawww/piwww.go | 5 +- politeiawww/proposals.go | 19 +- politeiawww/ticketvote/process.go | 14 +- 21 files changed, 410 insertions(+), 569 deletions(-) delete mode 100644 politeiad/backend/tlogbe/plugins/pi/cmds.go delete mode 100644 politeiad/client/pi.go diff --git a/politeiad/backend/tlogbe/plugins/comments/recordindex.go b/politeiad/backend/tlogbe/plugins/comments/recordindex.go index 658f13113..d36356f97 100644 --- a/politeiad/backend/tlogbe/plugins/comments/recordindex.go +++ b/politeiad/backend/tlogbe/plugins/comments/recordindex.go @@ -50,10 +50,10 @@ type recordIndex struct { // recordIndexPath accepts full length token or token prefixes, but always uses // prefix when generating the comments index path string. -func (p *commentsPlugin) recordIndexPath(token []byte) (string, error) { +func (p *commentsPlugin) recordIndexPath(token []byte) string { tp := util.TokenPrefix(token) fn := strings.Replace(filenameRecordIndex, "{tokenPrefix}", tp, 1) - return filepath.Join(p.dataDir, fn), nil + return filepath.Join(p.dataDir, fn) } // recordIndexLocked returns the cached recordIndex for the provided record. @@ -61,10 +61,7 @@ func (p *commentsPlugin) recordIndexPath(token []byte) (string, error) { // // This function must be called WITH the lock held. func (p *commentsPlugin) recordIndexLocked(token []byte) (*recordIndex, error) { - fp, err := p.recordIndexPath(token) - if err != nil { - return nil, err - } + fp := p.recordIndexPath(token) b, err := ioutil.ReadFile(fp) if err != nil { var e *os.PathError @@ -107,10 +104,7 @@ func (p *commentsPlugin) recordIndexSaveLocked(token []byte, ridx recordIndex) e if err != nil { return err } - fp, err := p.recordIndexPath(token) - if err != nil { - return err - } + fp := p.recordIndexPath(token) err = ioutil.WriteFile(fp, b, 0664) if err != nil { return err diff --git a/politeiad/backend/tlogbe/plugins/pi/cmds.go b/politeiad/backend/tlogbe/plugins/pi/cmds.go deleted file mode 100644 index b89ad2ea0..000000000 --- a/politeiad/backend/tlogbe/plugins/pi/cmds.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) 2020-2021 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package pi - -import ( - "encoding/json" - "fmt" - - "github.com/decred/politeia/politeiad/plugins/pi" - "github.com/decred/politeia/politeiad/plugins/ticketvote" -) - -func (p *piPlugin) voteSummary(token []byte) (*ticketvote.SummaryReply, error) { - reply, err := p.backend.VettedPluginCmd(token, - ticketvote.PluginID, ticketvote.CmdSummary, "") - if err != nil { - return nil, err - } - var sr ticketvote.SummaryReply - err = json.Unmarshal([]byte(reply), &sr) - if err != nil { - return nil, err - } - return &sr, nil -} - -func (p *piPlugin) cmdVoteInv() (string, error) { - // Get ticketvote inventory - r, err := p.backend.VettedPluginCmd([]byte{}, - ticketvote.PluginID, ticketvote.CmdInventory, "") - if err != nil { - return "", fmt.Errorf("VettedPluginCmd %v %v: %v", - ticketvote.PluginID, ticketvote.CmdInventory, err) - } - var ir ticketvote.InventoryReply - err = json.Unmarshal([]byte(r), &ir) - if err != nil { - return "", err - } - - // Get vote summaries for all finished proposal votes and - // categorize by approved/rejected. - var ( - finished = ir.Records[ticketvote.VoteStatusFinished] - approved = make([]string, 0, len(finished)) - rejected = make([]string, 0, len(finished)) - ) - for _, v := range finished { - t, err := tokenDecode(v) - if err != nil { - return "", err - } - vs, err := p.voteSummary(t) - if err != nil { - return "", err - } - if vs.Approved { - approved = append(approved, v) - } else { - rejected = append(rejected, v) - } - } - - // Prepare reply - vir := pi.VoteInventoryReply{ - Unauthorized: ir.Records[ticketvote.VoteStatusUnauthorized], - Authorized: ir.Records[ticketvote.VoteStatusAuthorized], - Started: ir.Records[ticketvote.VoteStatusStarted], - Approved: approved, - Rejected: rejected, - BestBlock: ir.BestBlock, - } - reply, err := json.Marshal(vir) - if err != nil { - return "", err - } - - return string(reply), nil -} diff --git a/politeiad/backend/tlogbe/plugins/pi/hooks.go b/politeiad/backend/tlogbe/plugins/pi/hooks.go index f92a14717..40ca4377e 100644 --- a/politeiad/backend/tlogbe/plugins/pi/hooks.go +++ b/politeiad/backend/tlogbe/plugins/pi/hooks.go @@ -220,6 +220,20 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { return nil } +func (p *piPlugin) voteSummary(token []byte) (*ticketvote.SummaryReply, error) { + reply, err := p.backend.VettedPluginCmd(token, + ticketvote.PluginID, ticketvote.CmdSummary, "") + if err != nil { + return nil, err + } + var sr ticketvote.SummaryReply + err = json.Unmarshal([]byte(reply), &sr) + if err != nil { + return nil, err + } + return &sr, nil +} + // commentWritesVerify verifies that a record's vote status allows writes from // the comments plugin. func (p *piPlugin) commentWritesVerify(s plugins.RecordStateT, token []byte) error { diff --git a/politeiad/backend/tlogbe/plugins/pi/pi.go b/politeiad/backend/tlogbe/plugins/pi/pi.go index 51ddb4579..8ae32d2a6 100644 --- a/politeiad/backend/tlogbe/plugins/pi/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi/pi.go @@ -61,11 +61,6 @@ func (p *piPlugin) Setup() error { func (p *piPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { log.Tracef("pi Cmd: %v %x %v %v", treeID, token, cmd, payload) - switch cmd { - case pi.CmdVoteInv: - return p.cmdVoteInv() - } - return "", backend.ErrPluginCmdInvalid } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go index bbd61eb87..4cb68ce73 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go @@ -521,7 +521,7 @@ func (p *ticketVotePlugin) summariesForRunoff(parentToken string) (map[string]ti // Save summary s := ticketvote.SummaryReply{ Type: vd.Params.Type, - Status: ticketvote.VoteStatusFinished, + Status: ticketvote.VoteStatusRejected, Duration: vd.Params.Duration, StartBlockHeight: vd.StartBlockHeight, StartBlockHash: vd.StartBlockHash, @@ -530,7 +530,6 @@ func (p *ticketVotePlugin) summariesForRunoff(parentToken string) (map[string]ti QuorumPercentage: vd.Params.QuorumPercentage, PassPercentage: vd.Params.PassPercentage, Results: results, - Approved: false, } summaries[v] = s @@ -575,7 +574,7 @@ func (p *ticketVotePlugin) summariesForRunoff(parentToken string) (map[string]ti if winnerToken != "" { // A winner was found. Mark their summary as approved. s := summaries[winnerToken] - s.Approved = true + s.Status = ticketvote.VoteStatusApproved summaries[winnerToken] = s } @@ -640,13 +639,9 @@ func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) }, nil } - // Vote has been started. Check if it is still in progress or has - // already ended. - if bestBlock < vd.EndBlockHeight { - status = ticketvote.VoteStatusStarted - } else { - status = ticketvote.VoteStatusFinished - } + // Vote has been started. We need to check if the vote has ended + // yet and if it can be considered approved or rejected. + status = ticketvote.VoteStatusStarted // Tally vote results results, err := p.voteOptionResults(token, vd.Params.Options) @@ -670,7 +665,7 @@ func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) } // If the vote has not finished yet then we are done for now. - if status == ticketvote.VoteStatusStarted { + if bestBlock < vd.EndBlockHeight { return &summary, nil } @@ -679,7 +674,11 @@ func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) switch vd.Params.Type { case ticketvote.VoteTypeStandard: // Standard vote uses a simple approve/reject result - summary.Approved = voteIsApproved(*vd, results) + if voteIsApproved(*vd, results) { + summary.Status = ticketvote.VoteStatusApproved + } else { + summary.Status = ticketvote.VoteStatusRejected + } // Cache summary err = p.summaryCacheSave(vd.Params.Token, summary) @@ -1080,9 +1079,9 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri // Update inventory switch a.Action { case ticketvote.AuthActionAuthorize: - p.invCacheSetToAuthorized(a.Token) + p.invAddToAuthorized(a.Token) case ticketvote.AuthActionRevoke: - p.invCacheSetToUnauthorized(a.Token) + p.invAddToUnauthorized(a.Token) default: // Should not happen e := fmt.Sprintf("invalid authorize action: %v", a.Action) @@ -1396,7 +1395,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot } // Update inventory - p.invCacheSetToStarted(vd.Params.Token, ticketvote.VoteTypeStandard, + p.invAddToStarted(vd.Params.Token, ticketvote.VoteTypeStandard, vd.EndBlockHeight) return &ticketvote.StartReply{ @@ -1578,7 +1577,7 @@ func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs sta } // Update inventory - p.invCacheSetToStarted(vd.Params.Token, ticketvote.VoteTypeRunoff, + p.invAddToStarted(vd.Params.Token, ticketvote.VoteTypeRunoff, vd.EndBlockHeight) return nil @@ -2565,43 +2564,6 @@ func (p *ticketVotePlugin) cmdSummary(treeID int64, token []byte) (string, error return string(reply), nil } -func convertInventoryReply(v inventory) ticketvote.InventoryReply { - // Started needs to be converted from a map to a slice where the - // slice is sorted by end block height from smallest to largest. - tokensByHeight := make(map[uint32][]string, len(v.started)) - for token, height := range v.started { - tokens, ok := tokensByHeight[height] - if !ok { - tokens = make([]string, 0, len(v.started)) - } - tokens = append(tokens, token) - tokensByHeight[height] = tokens - } - sortedHeights := make([]uint32, 0, len(tokensByHeight)) - for k := range tokensByHeight { - sortedHeights = append(sortedHeights, k) - } - // Sort smallest to largest block height - sort.SliceStable(sortedHeights, func(i, j int) bool { - return sortedHeights[i] < sortedHeights[j] - }) - started := make([]string, 0, len(v.started)) - for _, height := range sortedHeights { - tokens := tokensByHeight[height] - started = append(started, tokens...) - } - - return ticketvote.InventoryReply{ - Records: map[ticketvote.VoteStatusT][]string{ - ticketvote.VoteStatusUnauthorized: v.unauthorized, - ticketvote.VoteStatusAuthorized: v.authorized, - ticketvote.VoteStatusStarted: started, - ticketvote.VoteStatusFinished: v.finished, - }, - BestBlock: v.bestBlock, - } -} - func (p *ticketVotePlugin) cmdInventory() (string, error) { log.Tracef("cmdInventory") @@ -2613,13 +2575,16 @@ func (p *ticketVotePlugin) cmdInventory() (string, error) { } // Get the inventory - inv, err := p.invCache(bb) + inv, err := p.invGet(bb) if err != nil { - return "", fmt.Errorf("inventory: %v", err) + return "", fmt.Errorf("invGet: %v", err) } - ir := convertInventoryReply(*inv) // Prepare reply + ir := ticketvote.InventoryReply{ + Tokens: inv.Tokens, + BestBlock: inv.BestBlock, + } reply, err := json.Marshal(ir) if err != nil { return "", err diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go b/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go index 730641d97..8b846b572 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go @@ -126,7 +126,7 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { if err != nil { return err } - if !vs.Approved { + if vs.Status != ticketvote.VoteStatusApproved { return backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), @@ -312,7 +312,12 @@ func (p *ticketVotePlugin) hookSetRecordStatusPost(payload string) error { return err } - // Check if the LinkTo has been set + // Update the inventory cache if the record is being made public. + if srs.RecordMetadata.Status == backend.MDStatusVetted { + p.invAddToUnauthorized(srs.RecordMetadata.Token) + } + + // Update the submissions cache if the linkto has been set. vm, err := voteMetadataDecode(srs.Current.Files) if err != nil { return err diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go b/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go index 0d9fd5ec6..e6adbf13a 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go @@ -5,265 +5,365 @@ package ticketvote import ( + "encoding/json" + "errors" "fmt" + "io/ioutil" + "os" + "path/filepath" + "sort" "github.com/decred/politeia/politeiad/plugins/ticketvote" ) +const ( + // filenameInventory is the file name of the ticketvote inventory + // that is cached to the plugin data dir. + filenameInventory = "inventory.json" +) + // inventory contains the record inventory categorized by vote status. The -// authorized and started lists are updated in real-time since ticket vote -// plugin commands initiate those actions. The unauthorized and finished lists -// are lazy loaded since those lists depends on external state. +// unauthorized, authorized, and started lists are updated in real-time since +// ticket vote plugin commands or hooks initiate those actions. The finished, +// approved, and rejected statuses are lazy loaded since those lists depends on +// external state (DCR block height). type inventory struct { - unauthorized []string // Unauthorized tokens - authorized []string // Authorized tokens - started map[string]uint32 // [token]endHeight - finished []string // Finished tokens - bestBlock uint32 // Height of last inventory update + Tokens map[string][]string `json:"tokens"` // [status][]token + Active []activeVote `json:"active"` // Active votes + BestBlock uint32 `json:"bestblock"` // Last updated +} + +// activeVote is used to track active votes. The end height is stored so that +// we can check what votes have finished when a new best blocks come in. +type activeVote struct { + Token string `json:"token"` + EndHeight uint32 `json:"endheight"` +} + +// invPath returns the full path for the cached ticket vote inventory. +func (p *ticketVotePlugin) invPath() string { + return filepath.Join(p.dataDir, filenameInventory) +} + +// invGetLocked retrieves the inventory from disk. A new inventory is returned +// if one does not exist yet. +// +// This function must be called WITH the lock held. +func (p *ticketVotePlugin) invGetLocked() (*inventory, error) { + b, err := ioutil.ReadFile(p.invPath()) + if err != nil { + var e *os.PathError + if errors.As(err, &e) && !os.IsExist(err) { + // File does't exist. Return a new inventory. + return &inventory{ + Tokens: make(map[string][]string, 256), + Active: make([]activeVote, 0, 256), + BestBlock: 0, + }, nil + } + return nil, err + } + + var inv inventory + err = json.Unmarshal(b, &inv) + if err != nil { + return nil, err + } + + return &inv, nil +} + +// invSetLocked writes the inventory to disk. +// +// This function must be called WITH the lock held. +func (p *ticketVotePlugin) invSetLocked(inv inventory) error { + b, err := json.Marshal(inv) + if err != nil { + return err + } + err = ioutil.WriteFile(p.invPath(), b, 0664) + if err != nil { + return err + } + return nil } -func (p *ticketVotePlugin) invCacheSetToAuthorized(token string) { +// invAddToUnauthorized adds the token to the unauthorized vote status list. +// This is done when a unvetted record is made vetted or when a previous vote +// authorization is revoked. +func (p *ticketVotePlugin) invAddToUnauthorized(token string) { p.Lock() defer p.Unlock() - // Remove the token from the unauthorized list. The unauthorize - // list is lazy loaded so it may or may not exist. - var i int - var found bool - for k, v := range p.inv.unauthorized { - if v == token { - i = k - found = true - break - } + inv, err := p.invGetLocked() + if err != nil { + panic(err) } - if found { - // Remove the token from unauthorized - u := p.inv.unauthorized - u = append(u[:i], u[i+1:]...) - p.inv.unauthorized = u - log.Debugf("Removed from unauthorized inv: %v", token) + var ( + // Human readable vote statuses + unauth = ticketvote.VoteStatuses[ticketvote.VoteStatusUnauthorized] + auth = ticketvote.VoteStatuses[ticketvote.VoteStatusAuthorized] + ) + + // Remove token from the authorized list. It will only exit in the + // authorized list if the user is revoking a previous authorization. + ok := invDel(inv.Tokens, auth, token) + if ok { + log.Debugf("Vote inv del %v from authorized", token) } - // Prepend the token to the authorized list - a := p.inv.authorized - a = append([]string{token}, a...) - p.inv.authorized = a + // Add the token to unauthorized + invAdd(inv.Tokens, unauth, token) - log.Debugf("Added to authorized inv: %v", token) + log.Debugf("Vote inv add %v to unauthorized", token) + + // Save inventory + err = p.invSetLocked(*inv) + if err != nil { + panic(err) + } } -func (p *ticketVotePlugin) invCacheSetToUnauthorized(token string) { +// invAddToAuthorized moves a record from the unauthorized to the authorized +// list. This is done by the ticketvote authorize command. +func (p *ticketVotePlugin) invAddToAuthorized(token string) { p.Lock() defer p.Unlock() - // Remove the token from the authorized list if it exists. Going - // from authorized to unauthorized can happen when a vote - // authorization is revoked. - var i int - var found bool - for k, v := range p.inv.authorized { - if v == token { - i = k - found = true - break - } + inv, err := p.invGetLocked() + if err != nil { + panic(err) } - if found { - // Remove the token from authorized - a := p.inv.authorized - a = append(a[:i], a[i+1:]...) - p.inv.authorized = a - log.Debugf("Removed from authorized inv: %v", token) + var ( + // Human readable vote statuses + unauth = ticketvote.VoteStatuses[ticketvote.VoteStatusUnauthorized] + auth = ticketvote.VoteStatuses[ticketvote.VoteStatusAuthorized] + ) + + // Remove the token from unauthorized list. The token should always + // be in the unauthorized list. + ok := invDel(inv.Tokens, unauth, token) + if !ok { + e := fmt.Sprintf("token not found in unauthorized list %v", token) + panic(e) } - // Prepend the token to the unauthorized list - u := p.inv.unauthorized - u = append([]string{token}, u...) - p.inv.unauthorized = u + log.Debugf("Vote inv del %v from unauthorized", token) - log.Debugf("Added to unauthorized inv: %v", token) + // Prepend the token to the authorized list + invAdd(inv.Tokens, auth, token) + + log.Debugf("Vote inv add %v to authorized", token) + + // Save inventory + err = p.invSetLocked(*inv) + if err != nil { + panic(err) + } } -func (p *ticketVotePlugin) invCacheSetToStarted(token string, t ticketvote.VoteT, endHeight uint32) { +// invAddToStarted moves a record into the started vote status list. This is +// done by the ticketvote start command. +func (p *ticketVotePlugin) invAddToStarted(token string, t ticketvote.VoteT, endHeight uint32) { p.Lock() defer p.Unlock() + inv, err := p.invGetLocked() + if err != nil { + panic(err) + } + + var ( + // Human readable vote statuses + unauth = ticketvote.VoteStatuses[ticketvote.VoteStatusUnauthorized] + auth = ticketvote.VoteStatuses[ticketvote.VoteStatusAuthorized] + started = ticketvote.VoteStatuses[ticketvote.VoteStatusStarted] + ) + switch t { case ticketvote.VoteTypeStandard: // Remove the token from the authorized list. The token should // always be in the authorized list prior to the vote being - // started for standard votes so panicing when this is not the - // case is ok. - var i int - var found bool - for k, v := range p.inv.authorized { - if v == token { - i = k - found = true - break - } - } - if !found { - e := fmt.Sprintf("token not found in authorized list: %v", token) + // started for standard votes. + ok := invDel(inv.Tokens, auth, token) + if !ok { + e := fmt.Sprintf("token not found in authorized list %v", token) panic(e) } - a := p.inv.authorized - a = append(a[:i], a[i+1:]...) - p.inv.authorized = a - - log.Debugf("Removed from authorized inv: %v", token) + log.Debugf("Vote inv del %v from authorized", token) case ticketvote.VoteTypeRunoff: - // A runoff vote does not require the submission votes be - // authorized prior to the vote starting. The token might be in - // the unauthorized list, but its also possible that its not - // since the unauthorized list is lazy loaded and it might not - // have been added yet. Remove it only if it is found. - var i int - var found bool - for k, v := range p.inv.unauthorized { - if v == token { - i = k - found = true - break - } + // A runoff vote does not require the submissions be authorized + // prior to the vote starting. The token should always be in the + // unauthorized list. + ok := invDel(inv.Tokens, unauth, token) + if !ok { + e := fmt.Sprintf("token not found in unauthorized list %v", token) + panic(e) } - if found { - // Remove the token from unauthorized - u := p.inv.unauthorized - u = append(u[:i], u[i+1:]...) - p.inv.unauthorized = u - log.Debugf("Removed from unauthorized inv: %v", token) - } + log.Debugf("Vote inv del %v from unauthorized", token) default: e := fmt.Sprintf("invalid vote type %v", t) panic(e) } - // Add the token to the started list - p.inv.started[token] = endHeight + // Add token to started list + invAdd(inv.Tokens, started, token) + + // Add token to active votes list + vt := activeVote{ + Token: token, + EndHeight: endHeight, + } + inv.Active = append([]activeVote{vt}, inv.Active...) + + // Sort active votes + sortActiveVotes(inv.Active) - log.Debugf("Added to started inv: %v", token) + log.Debugf("Vote inv add %v to started with end height %v", + token, endHeight) + + // Save inventory + err = p.invSetLocked(*inv) + if err != nil { + panic(err) + } } -func (p *ticketVotePlugin) invCache(bestBlock uint32) (*inventory, error) { +func (p *ticketVotePlugin) invGet(bestBlock uint32) (*inventory, error) { p.Lock() defer p.Unlock() - // Check backend inventory for new records - invBackend, err := p.backend.InventoryByStatus() + inv, err := p.invGetLocked() if err != nil { - return nil, fmt.Errorf("InventoryByStatus: %v", err) + panic(err) } - // Find number of records in the vetted inventory - var vettedInvCount int - for _, tokens := range invBackend.Vetted { - vettedInvCount += len(tokens) + // Check if the inventory has been updated for this block height. + if inv.BestBlock == bestBlock { + // Inventory already updated. Nothing else to do. + return inv, nil } - // Find number of records in the vote inventory - voteInvCount := len(p.inv.unauthorized) + len(p.inv.authorized) + - len(p.inv.started) + len(p.inv.finished) - - // The vetted inventory count and the vote inventory count should - // be the same. If they're not then it means we there are records - // missing from vote inventory. - if vettedInvCount != voteInvCount { - // Records are missing from the vote inventory. Put all ticket - // vote inventory records into a map so we can easily find what - // backend records are missing. - all := make(map[string]struct{}, voteInvCount) - for _, v := range p.inv.unauthorized { - all[v] = struct{}{} + // The active votes should already be sorted, but sort them again + // just to be sure. + sortActiveVotes(inv.Active) + + // The inventory has not been updated for this block height. Check + // if any votes have finished. + active := make([]activeVote, 0, len(inv.Active)) + for _, v := range inv.Active { + if v.EndHeight >= bestBlock { + // Vote has not finished yet. Keep it in the active votes list. + active = append(active, v) } - for _, v := range p.inv.authorized { - all[v] = struct{}{} + + // Vote has finished. Get vote summary. + t, err := tokenDecode(v.Token) + if err != nil { + return nil, err } - for k := range p.inv.started { - all[k] = struct{}{} + sr, err := p.summaryByToken(t) + if err != nil { + return nil, err } - for _, v := range p.inv.finished { - all[v] = struct{}{} + + // Remove token from started list + started := ticketvote.VoteStatuses[ticketvote.VoteStatusStarted] + ok := invDel(inv.Tokens, started, v.Token) + if !ok { + return nil, fmt.Errorf("token not found in started %v", v.Token) } - // Add missing records to the vote inventory - for _, tokens := range invBackend.Vetted { - for _, v := range tokens { - if _, ok := all[v]; ok { - // Record is already in the vote inventory - continue - } - // We can assume that the record vote status is unauthorized - // since it would have already been added to the vote - // inventory during the authorization request if one had - // occurred. - p.inv.unauthorized = append(p.inv.unauthorized, v) - - log.Debugf("Added to unauthorized inv: %v", v) - } + log.Debugf("Vote inv del %v from started", v.Token) + + // Add token to the appropriate list + switch sr.Status { + case ticketvote.VoteStatusFinished, ticketvote.VoteStatusApproved, + ticketvote.VoteStatusRejected: + // These statuses are allowed + status := ticketvote.VoteStatuses[sr.Status] + invAdd(inv.Tokens, status, v.Token) + default: + // Something went wrong + return nil, fmt.Errorf("unexpected vote status %v %v", + v.Token, sr.Status) } + + log.Debugf("Vote inv add %v to %v", + v.Token, ticketvote.VoteStatuses[sr.Status]) } - // The records are moved to their correct vote status category in - // the inventory on authorization, revoking the authorization, and - // on starting the vote. We can assume these lists are already - // up-to-date. The last thing we must check for is whether any - // votes have finished since the last inventory update. + // Update active votes list + inv.Active = active - // Check if the inventory has been updated for this block height. - if p.inv.bestBlock == bestBlock { - // Inventory already updated. Nothing else to do. - goto reply + // Update best block + inv.BestBlock = bestBlock + + log.Debugf("Vote inv updated for block %v", bestBlock) + + // Save inventory + err = p.invSetLocked(*inv) + if err != nil { + panic(err) } - // Inventory has not been updated for this block height. Check if - // any proposal votes have finished. - for token, endHeight := range p.inv.started { - if bestBlock >= endHeight { - // Vote has finished. Remove it from the started list. - delete(p.inv.started, token) + return inv, nil +} - log.Debugf("Removed from started inv: %v", token) +func sortActiveVotes(v []activeVote) { + // Sort by end height from smallest to largest + sort.SliceStable(v, func(i, j int) bool { + return v[i].EndHeight < v[j].EndHeight + }) +} - // Add it to the finished list - p.inv.finished = append(p.inv.finished, token) +func invDel(inv map[string][]string, status, token string) bool { + list, ok := inv[status] + if !ok { + inv[status] = make([]string, 0, 1056) + return false + } - log.Debugf("Added to finished inv: %v", token) + // Find token (linear time) + var i int + var found bool + for k, v := range list { + if v == token { + i = k + found = true + break } } + if !found { + return found + } - // Update best block - p.inv.bestBlock = bestBlock + // Remove token (linear time) + copy(list[i:], list[i+1:]) // Shift list[i+1:] left one index + list[len(list)-1] = "" // Erase last element (write zero token) + list = list[:len(list)-1] // Truncate slice - log.Debugf("Inv updated for best block %v", bestBlock) + // Update inv + inv[status] = list -reply: - // Return a copy of the inventory - var ( - unauthorized = make([]string, len(p.inv.unauthorized)) - authorized = make([]string, len(p.inv.authorized)) - started = make(map[string]uint32, len(p.inv.started)) - finished = make([]string, len(p.inv.finished)) - ) - copy(unauthorized, p.inv.unauthorized) - copy(authorized, p.inv.authorized) - copy(finished, p.inv.finished) - for k, v := range p.inv.started { - started[k] = v + return found +} + +func invAdd(inv map[string][]string, status, token string) { + list, ok := inv[status] + if !ok { + list = make([]string, 0, 1056) } - return &inventory{ - unauthorized: unauthorized, - authorized: authorized, - started: started, - finished: finished, - bestBlock: p.inv.bestBlock, - }, nil + // Prepend token + list = append([]string{token}, list...) + + // Update inv + inv[status] = list } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/summary.go b/politeiad/backend/tlogbe/plugins/ticketvote/summary.go index 78a86f9d0..78f91dccd 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/summary.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/summary.go @@ -17,9 +17,8 @@ import ( ) const ( - // Filenames of cached data saved to the plugin data dir. Brackets - // are used to indicate a variable that should be replaced in the - // filename. + // filenameSummary is the file name of the vote summary for a + // record. These summaries are cached in the plugin data dir. filenameSummary = "{tokenprefix}-summary.json" ) diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go index b717ef1c6..ac373799d 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go @@ -43,11 +43,6 @@ type ticketVotePlugin struct { // prove the backend received and processed a plugin command. identity *identity.FullIdentity - // inv contains the record inventory categorized by vote status. - // The inventory will only contain public, non-abandoned records. - // This cache is built on startup. - inv inventory - // votes contains the cast votes of ongoing record votes. This // cache is built on startup and record entries are removed once // the vote has ended and a vote summary has been cached. @@ -101,62 +96,25 @@ func (p *ticketVotePlugin) Setup() error { dcrdata.PluginID) } - // Build inventory cache - log.Infof("Building inventory cache") - - ibs, err := p.backend.InventoryByStatus() - if err != nil { - return fmt.Errorf("InventoryByStatus: %v", err) - } + // Update the inventory with the current best block. Retrieving + // the inventory will cause it to update. + log.Infof("Updating vote inv") - bestBlock, err := p.bestBlock() + bb, err := p.bestBlock() if err != nil { return fmt.Errorf("bestBlock: %v", err) } - - var ( - unauthorized = make([]string, 0, 256) - authorized = make([]string, 0, 256) - started = make(map[string]uint32, 256) // [token]endHeight - finished = make([]string, 0, 256) - ) - for _, tokens := range ibs.Vetted { - for _, v := range tokens { - token, err := tokenDecode(v) - if err != nil { - return err - } - s, err := p.summaryByToken(token) - switch s.Status { - case ticketvote.VoteStatusUnauthorized: - unauthorized = append(unauthorized, v) - case ticketvote.VoteStatusAuthorized: - authorized = append(authorized, v) - case ticketvote.VoteStatusStarted: - started[v] = s.EndBlockHeight - case ticketvote.VoteStatusFinished: - finished = append(finished, v) - default: - return fmt.Errorf("invalid vote status %v %v", v, s.Status) - } - } - } - - p.Lock() - p.inv = inventory{ - unauthorized: unauthorized, - authorized: authorized, - started: started, - finished: finished, - bestBlock: bestBlock, + inv, err := p.invGet(bb) + if err != nil { + return fmt.Errorf("invGet: %v", err) } - p.Unlock() // Build votes cache log.Infof("Building votes cache") - for k := range started { - token, err := tokenDecode(k) + started := ticketvote.VoteStatuses[ticketvote.VoteStatusStarted] + for _, v := range inv.Tokens[started] { + token, err := tokenDecode(v) if err != nil { return err } @@ -358,13 +316,6 @@ func New(backend backend.Backend, tlog plugins.TlogClient, settings []backend.Pl tlog: tlog, dataDir: dataDir, identity: id, - inv: inventory{ - unauthorized: make([]string, 0, 1024), - authorized: make([]string, 0, 1024), - started: make(map[string]uint32, 1024), - finished: make([]string, 0, 1024), - bestBlock: 0, - }, votes: make(map[string]map[string]string), mutexes: make(map[string]*sync.Mutex), linkByPeriodMin: linkByPeriodMin, diff --git a/politeiad/client/pi.go b/politeiad/client/pi.go deleted file mode 100644 index c82cfa4ee..000000000 --- a/politeiad/client/pi.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (c) 2020-2021 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package client - -import ( - "context" - "encoding/json" - "fmt" - - pdv1 "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/politeiad/plugins/pi" -) - -// PiVoteInv sends the pi plugin VoteInv command to the politeiad v1 API. -func (c *Client) PiVoteInv(ctx context.Context) (*pi.VoteInventoryReply, error) { - // Setup request - cmds := []pdv1.PluginCommandV2{ - { - State: pdv1.RecordStateVetted, - Token: "", - ID: pi.PluginID, - Command: pi.CmdVoteInv, - Payload: "", - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var vir pi.VoteInventoryReply - err = json.Unmarshal([]byte(pcr.Payload), &vir) - if err != nil { - return nil, err - } - - return &vir, nil -} diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index 516ca9d0d..edf5b4295 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -10,9 +10,6 @@ const ( // PluginID is the pi plugin ID. PluginID = "pi" - // Plugin commands - CmdVoteInv = "voteinv" - // Setting keys are the plugin setting keys that can be used to // override a default plugin setting. Defaults will be overridden // if a plugin setting is provided to the plugin on startup. @@ -125,22 +122,3 @@ const ( type ProposalMetadata struct { Name string `json:"name"` } - -// VoteInventory requests the tokens of all proposals in the inventory -// categorized by their vote status. This call relies on the ticketvote -// Inventory call, but breaks the Finished vote status out into Approved and -// Rejected categories. This functionality is specific to pi. -type VoteInventory struct{} - -// VoteInventoryReply is the reply to the VoteInventory command. -type VoteInventoryReply struct { - Unauthorized []string `json:"unauthorized"` - Authorized []string `json:"authorized"` - Started []string `json:"started"` - Approved []string `json:"approved"` - Rejected []string `json:"rejected"` - - // BestBlock is the best block value that was used to prepare the - // inventory. - BestBlock uint32 `json:"bestblock"` -} diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index c5803e14a..b077d8017 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -440,8 +440,22 @@ const ( // VoteStatusStarted indicates the ticket vote has been started. VoteStatusStarted VoteStatusT = 3 - // VoteStatusFinished indicates the ticket vote has finished. + // VoteStatusFinished indicates the ticket vote has finished. This + // vote status is used for vote types that do not have a clear + // approved or rejected outcome, such as multiple choice votes. VoteStatusFinished VoteStatusT = 4 + + // VoteStatusApproved indicates that a vote has finished and the + // vote has met the criteria for being approved. This vote status + // is only used when the vote type allows for a clear approved or + // rejected outcome. + VoteStatusApproved VoteStatusT = 5 + + // VoteStatusRejected indicates that a vote has finished and the + // vote did NOT the criteria for being approved. This vote status + // is only used when the vote type allows for a clear approved or + // rejected outcome. + VoteStatusRejected VoteStatusT = 6 ) var ( @@ -452,6 +466,8 @@ var ( VoteStatusAuthorized: "authorized", VoteStatusStarted: "started", VoteStatusFinished: "finished", + VoteStatusApproved: "approved", + VoteStatusRejected: "rejected", } ) @@ -480,12 +496,6 @@ type SummaryReply struct { PassPercentage uint32 `json:"passpercentage"` Results []VoteOptionResult `json:"results"` - // Approved describes whether the vote has been approved. This will - // only be present when the vote type is VoteTypeStandard or - // VoteTypeRunoff, both of which only allow for approve/reject - // voting options. - Approved bool `json:"approved,omitempty"` - // BestBlock is the best block value that was used to prepare this // summary. BestBlock uint32 `json:"bestblock"` @@ -509,15 +519,21 @@ type SubmissionsReply struct { type Inventory struct{} // InventoryReply is the reply to the Inventory command. It contains the tokens -// of all public, non-abandoned records categorized by vote status. +// of all public, non-abandoned records categorized by vote status. The +// returned map is a map[votestatus][]token where the votestatus key is the +// human readable vote status defined by the VoteStatuses array in this +// package. // -// Statuses sorted by timestamp in descending order: +// Sorted by timestamp in descending order: // Unauthorized, Authorized // -// Statuses sorted by voting period end block height in descending order: -// Started, Finished +// Sorted by vote start block height in descending order: +// Started +// +// Sorted by vote end block height in descending order: +// Finished, Approved, Rejected type InventoryReply struct { - Records map[VoteStatusT][]string `json:"records"` + Tokens map[string][]string `json:"tokens"` // BestBlock is the best block value that was used to prepare the // inventory. diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index d85df26b7..5f79ab68d 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -17,9 +17,8 @@ const ( APIRoute = "/pi/v1" // Routes - RoutePolicy = "/policy" - RouteProposals = "/proposals" - RouteVoteInventory = "/voteinventory" + RoutePolicy = "/policy" + RouteProposals = "/proposals" // Proposal states ProposalStateUnvetted = "unvetted" @@ -255,22 +254,3 @@ type Proposals struct { type ProposalsReply struct { Proposals map[string]Proposal `json:"proposals"` // [token]Proposal } - -// VoteInventory retrieves the tokens of all public, non-abandoned proposals -// categorized by their vote status. This is the same inventory as the -// ticketvote API returns except the Finished vote status is broken out into -// Approved and Rejected. -type VoteInventory struct{} - -// VoteInventoryReply in the reply to the VoteInventory command. -type VoteInventoryReply struct { - Unauthorized []string `json:"unauthorized"` - Authorized []string `json:"authorized"` - Started []string `json:"started"` - Approved []string `json:"approved"` - Rejected []string `json:"rejected"` - - // BestBlock is the best block value that was used to prepare the - // inventory. - BestBlock uint32 `json:"bestblock"` -} diff --git a/politeiawww/api/ticketvote/v1/v1.go b/politeiawww/api/ticketvote/v1/v1.go index f2909bfc8..621ca0f71 100644 --- a/politeiawww/api/ticketvote/v1/v1.go +++ b/politeiawww/api/ticketvote/v1/v1.go @@ -193,8 +193,22 @@ const ( // is still ongoing. VoteStatusStarted VoteStatusT = 3 - // VoteStatusFinished represents a vote that has ended. + // VoteStatusFinished indicates the ticket vote has finished. This + // vote status is used for vote types that do not have a clear + // approved or rejected outcome, such as multiple choice votes. VoteStatusFinished VoteStatusT = 4 + + // VoteStatusApproved indicates that a vote has finished and the + // vote has met the criteria for being approved. This vote status + // is only used when the vote type allows for a clear approved or + // rejected outcome. + VoteStatusApproved VoteStatusT = 5 + + // VoteStatusRejected indicates that a vote has finished and the + // vote did NOT the criteria for being approved. This vote status + // is only used when the vote type allows for a clear approved or + // rejected outcome. + VoteStatusRejected VoteStatusT = 6 ) var ( @@ -205,6 +219,8 @@ var ( VoteStatusAuthorized: "authorized", VoteStatusStarted: "started", VoteStatusFinished: "finished", + VoteStatusApproved: "approved", + VoteStatusRejected: "rejected", } ) @@ -455,8 +471,7 @@ type Summary struct { // the vote in order for the vote to pass. PassPercentage uint32 `json:"passpercentage"` - Results []VoteResult `json:"results"` - Approved bool `json:"approved"` // Was the vote approved + Results []VoteResult `json:"results"` // BestBlock is the best block value that was used to prepare the // summary. @@ -496,8 +511,8 @@ const ( InventoryPageSize = 60 ) -// Inventory requests the tokens of all records in the inventory, categorized -// by vote status. +// Inventory requests the tokens of all public, non-abandoned records +// categorized by vote status. type Inventory struct{} // InventoryReply is the reply to the Inventory command. It contains the tokens @@ -506,11 +521,14 @@ type Inventory struct{} // human readable vote status defined by the VoteStatuses array in this // package. // -// Statuses sorted by timestamp in descending order: +// Sorted by timestamp in descending order: // Unauthorized, Authorized // -// Statuses sorted by voting period end block height in descending order: -// Started, Finished +// Sorted by vote start block height in descending order: +// Started +// +// Sorted by vote end block height in descending order: +// Finished, Approved, Rejected type InventoryReply struct { Vetted map[string][]string `json:"vetted"` diff --git a/politeiawww/cmd/pictl/proposal.go b/politeiawww/cmd/pictl/proposal.go index 00618e704..6ece0156a 100644 --- a/politeiawww/cmd/pictl/proposal.go +++ b/politeiawww/cmd/pictl/proposal.go @@ -64,7 +64,7 @@ func printProposal(r rcv1.Record) error { printf("Token : %v\n", r.CensorshipRecord.Token) printf("Version : %v\n", r.Version) printf("Status : %v\n", rcv1.RecordStatuses[r.Status]) - printf("Timestamp: %v\n", r.Timestamp) + printf("Timestamp: %v\n", timestampFromUnix(r.Timestamp)) printf("Username : %v\n", r.Username) printf("Merkle : %v\n", r.CensorshipRecord.Merkle) printf("Receipt : %v\n", r.CensorshipRecord.Signature) diff --git a/politeiawww/cmd/pictl/ticketvote.go b/politeiawww/cmd/pictl/ticketvote.go index 8e71f973e..c808cf0b1 100644 --- a/politeiawww/cmd/pictl/ticketvote.go +++ b/politeiawww/cmd/pictl/ticketvote.go @@ -81,7 +81,6 @@ func printVoteSummary(token string, s tkv1.Summary) { printf("Start Block Height: %v\n", s.StartBlockHeight) printf("End Block Height : %v\n", s.EndBlockHeight) printf("Eligible Tickets : %v tickets\n", s.EligibleTickets) - printf("Approved : %v\n", s.Approved) printf("Best Block : %v\n", s.BestBlock) printf("Results\n") for _, v := range s.Results { diff --git a/politeiawww/pi/pi.go b/politeiawww/pi/pi.go index 11d717cf5..5525e1e65 100644 --- a/politeiawww/pi/pi.go +++ b/politeiawww/pi/pi.go @@ -69,21 +69,6 @@ func (p *Pi) HandleProposals(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, psr) } -// HandleVoteInventory is the request handler for the pi v1 VoteInventory -// route. -func (p *Pi) HandleVoteInventory(w http.ResponseWriter, r *http.Request) { - log.Tracef("HandleVoteInventory") - - vir, err := p.processVoteInventory(r.Context()) - if err != nil { - respondWithError(w, r, - "HandleVoteInventory: processVoteInventory: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, vir) -} - // New returns a new Pi context. func New(cfg *config.Config, pdc *pdclient.Client, udb user.Database, s *sessions.Sessions, plugins []pdv1.Plugin) (*Pi, error) { // Parse plugin settings diff --git a/politeiawww/pi/process.go b/politeiawww/pi/process.go index f8ab2f082..9404a3847 100644 --- a/politeiawww/pi/process.go +++ b/politeiawww/pi/process.go @@ -118,24 +118,6 @@ func (p *Pi) processProposals(ctx context.Context, ps v1.Proposals, u *user.User }, nil } -func (p *Pi) processVoteInventory(ctx context.Context) (*v1.VoteInventoryReply, error) { - log.Tracef("processVoteInventory") - - vir, err := p.politeiad.PiVoteInv(ctx) - if err != nil { - return nil, err - } - - return &v1.VoteInventoryReply{ - Unauthorized: vir.Unauthorized, - Authorized: vir.Authorized, - Started: vir.Started, - Approved: vir.Approved, - Rejected: vir.Rejected, - BestBlock: vir.BestBlock, - }, nil -} - // proposalName parses the proposal name from the ProposalMetadata and returns // it. An empty string will be returned if any errors occur or if a name is not // found. diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 3de05c4c3..f259f4160 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -37,7 +37,7 @@ func (p *politeiawww) setupPiRoutes(r *records.Records, c *comments.Comments, t HandleFunc(www.PoliteiaWWWAPIRoute+www.RouteVersion, p.handleVersion). Methods(http.MethodGet) - // www routes. These routes have been DEPRECATED. + // Legacy www routes. These routes have been DEPRECATED. p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, www.RoutePolicy, p.handlePolicy, permissionPublic) @@ -154,9 +154,6 @@ func (p *politeiawww) setupPiRoutes(r *records.Records, c *comments.Comments, t p.addRoute(http.MethodPost, piv1.APIRoute, piv1.RouteProposals, pic.HandleProposals, permissionPublic) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteVoteInventory, pic.HandleVoteInventory, - permissionPublic) } func (p *politeiawww) setupPi(plugins []pdv1.Plugin) error { diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 7751b71cb..7fe419e11 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2020 The Decred developers +// Copyright (c) 2017-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -283,9 +283,9 @@ func (p *politeiawww) processBatchVoteSummary(ctx context.Context, bvs www.Batch // TODO var token string summaries[token] = www.VoteSummary{ - Status: convertVoteStatusToWWW(v.Status), - Type: convertVoteTypeToWWW(v.Type), - Approved: v.Approved, + Status: convertVoteStatusToWWW(v.Status), + Type: convertVoteTypeToWWW(v.Type), + // Approved: v.Approved, EligibleTickets: v.EligibleTickets, Duration: v.Duration, EndHeight: uint64(v.EndBlockHeight), @@ -352,7 +352,6 @@ func (p *politeiawww) processTokenInventory(ctx context.Context, isAdmin bool) ( } // TODO Get vote inventory - var vir pi.VoteInventoryReply // Unpack record inventory var ( @@ -372,11 +371,11 @@ func (p *politeiawww) processTokenInventory(ctx context.Context, isAdmin bool) ( return &www.TokenInventoryReply{ Unreviewed: unreviewed, Censored: censored, - Pre: append(vir.Unauthorized, vir.Authorized...), - Active: vir.Started, - Approved: vir.Approved, - Rejected: vir.Rejected, - Abandoned: archived, + // Pre: append(vir.Unauthorized, vir.Authorized...), + // Active: vir.Started, + // Approved: vir.Approved, + // Rejected: vir.Rejected, + Abandoned: archived, }, nil } diff --git a/politeiawww/ticketvote/process.go b/politeiawww/ticketvote/process.go index 8c6fc8dc2..dbcfbc9bf 100644 --- a/politeiawww/ticketvote/process.go +++ b/politeiawww/ticketvote/process.go @@ -204,15 +204,8 @@ func (t *TicketVote) processInventory(ctx context.Context) (*v1.InventoryReply, return nil, err } - // Convert vote statuses to human readable equivalents - records := make(map[string][]string, len(ir.Records)) - for k, v := range ir.Records { - s := convertVoteStatusToV1(k) - records[v1.VoteStatuses[s]] = v - } - return &v1.InventoryReply{ - Vetted: records, + Vetted: ir.Tokens, BestBlock: ir.BestBlock, }, nil } @@ -435,6 +428,10 @@ func convertVoteStatusToV1(s ticketvote.VoteStatusT) v1.VoteStatusT { return v1.VoteStatusStarted case ticketvote.VoteStatusFinished: return v1.VoteStatusFinished + case ticketvote.VoteStatusApproved: + return v1.VoteStatusApproved + case ticketvote.VoteStatusRejected: + return v1.VoteStatusRejected default: return v1.VoteStatusInvalid } @@ -461,7 +458,6 @@ func convertSummaryToV1(s ticketvote.SummaryReply) v1.Summary { QuorumPercentage: s.QuorumPercentage, PassPercentage: s.PassPercentage, Results: results, - Approved: s.Approved, BestBlock: s.BestBlock, } } From deaf8db339f19b3d6723123760b0c476198d4909 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 10 Feb 2021 13:13:19 -0600 Subject: [PATCH 291/449] Fix ticketvote deadlock. --- .../tlogbe/plugins/ticketvote/inventory.go | 16 ++++++++-------- .../tlogbe/plugins/ticketvote/submissions.go | 12 ++++++------ .../backend/tlogbe/plugins/ticketvote/summary.go | 8 ++++---- .../tlogbe/plugins/ticketvote/ticketvote.go | 16 +++++++++++----- .../backend/tlogbe/plugins/ticketvote/votes.go | 12 ++++++------ politeiawww/cmd/pictl/proposal.go | 9 +++++---- 6 files changed, 40 insertions(+), 33 deletions(-) diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go b/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go index e6adbf13a..666cb830d 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go @@ -92,8 +92,8 @@ func (p *ticketVotePlugin) invSetLocked(inv inventory) error { // This is done when a unvetted record is made vetted or when a previous vote // authorization is revoked. func (p *ticketVotePlugin) invAddToUnauthorized(token string) { - p.Lock() - defer p.Unlock() + p.mtxInv.Lock() + defer p.mtxInv.Unlock() inv, err := p.invGetLocked() if err != nil { @@ -128,8 +128,8 @@ func (p *ticketVotePlugin) invAddToUnauthorized(token string) { // invAddToAuthorized moves a record from the unauthorized to the authorized // list. This is done by the ticketvote authorize command. func (p *ticketVotePlugin) invAddToAuthorized(token string) { - p.Lock() - defer p.Unlock() + p.mtxInv.Lock() + defer p.mtxInv.Unlock() inv, err := p.invGetLocked() if err != nil { @@ -167,8 +167,8 @@ func (p *ticketVotePlugin) invAddToAuthorized(token string) { // invAddToStarted moves a record into the started vote status list. This is // done by the ticketvote start command. func (p *ticketVotePlugin) invAddToStarted(token string, t ticketvote.VoteT, endHeight uint32) { - p.Lock() - defer p.Unlock() + p.mtxInv.Lock() + defer p.mtxInv.Unlock() inv, err := p.invGetLocked() if err != nil { @@ -236,8 +236,8 @@ func (p *ticketVotePlugin) invAddToStarted(token string, t ticketvote.VoteT, end } func (p *ticketVotePlugin) invGet(bestBlock uint32) (*inventory, error) { - p.Lock() - defer p.Unlock() + p.mtxInv.Lock() + defer p.mtxInv.Unlock() inv, err := p.invGetLocked() if err != nil { diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/submissions.go b/politeiad/backend/tlogbe/plugins/ticketvote/submissions.go index 4c18390ca..b20152d4a 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/submissions.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/submissions.go @@ -75,8 +75,8 @@ func (p *ticketVotePlugin) submissionsCacheWithLock(token []byte) (*submissions, // // This function must be called WITHOUT the lock held. func (p *ticketVotePlugin) submissionsCache(token []byte) (*submissions, error) { - p.Lock() - defer p.Unlock() + p.mtxSubs.Lock() + defer p.mtxSubs.Unlock() return p.submissionsCacheWithLock(token) } @@ -98,8 +98,8 @@ func (p *ticketVotePlugin) submissionsCacheSaveWithLock(token []byte, s submissi // // This function must be called WITHOUT the lock held. func (p *ticketVotePlugin) submissionsCacheAdd(parentToken, childToken string) error { - p.Lock() - defer p.Unlock() + p.mtxSubs.Lock() + defer p.mtxSubs.Unlock() // Verify tokens parent, err := tokenDecode(parentToken) @@ -137,8 +137,8 @@ func (p *ticketVotePlugin) submissionsCacheAdd(parentToken, childToken string) e // // This function must be called WITHOUT the lock held. func (p *ticketVotePlugin) submissionsCacheDel(parentToken, childToken string) error { - p.Lock() - defer p.Unlock() + p.mtxSubs.Lock() + defer p.mtxSubs.Unlock() // Verify tokens parent, err := tokenDecode(parentToken) diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/summary.go b/politeiad/backend/tlogbe/plugins/ticketvote/summary.go index 78f91dccd..7aa675cf6 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/summary.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/summary.go @@ -40,8 +40,8 @@ var ( ) func (p *ticketVotePlugin) summaryCache(token string) (*ticketvote.SummaryReply, error) { - p.Lock() - defer p.Unlock() + p.mtxSummary.Lock() + defer p.mtxSummary.Unlock() fp, err := p.summaryCachePath(token) if err != nil { @@ -72,8 +72,8 @@ func (p *ticketVotePlugin) summaryCacheSave(token string, sr ticketvote.SummaryR return err } - p.Lock() - defer p.Unlock() + p.mtxSummary.Lock() + defer p.mtxSummary.Unlock() fp, err := p.summaryCachePath(token) if err != nil { diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go index ac373799d..751b25f24 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go @@ -27,7 +27,6 @@ var ( // ticketVotePlugin satisfies the plugins.PluginClient interface. type ticketVotePlugin struct { - sync.Mutex backend backend.Backend tlog plugins.TlogClient activeNetParams *chaincfg.Params @@ -46,14 +45,21 @@ type ticketVotePlugin struct { // votes contains the cast votes of ongoing record votes. This // cache is built on startup and record entries are removed once // the vote has ended and a vote summary has been cached. - votes map[string]map[string]string // [token][ticket]voteBit + votes map[string]map[string]string // [token][ticket]voteBit + mtxVotes sync.Mutex // Mutexes contains a mutex for each record and are used to lock // the trillian tree for a given record to prevent concurrent // ticket vote plugin updates on the same tree. These mutexes are // lazy loaded and should only be used for tree updates, not for // cache updates. - mutexes map[string]*sync.Mutex // [string]mutex + mutexes map[string]*sync.Mutex // [string]mutex + mtxRecords sync.Mutex + + // Mutexes for on-disk caches + mtxInv sync.Mutex // Vote inventory + mtxSummary sync.Mutex // Vote summaries + mtxSubs sync.Mutex // Runoff vote submissions // Plugin settings linkByPeriodMin int64 // In seconds @@ -64,8 +70,8 @@ type ticketVotePlugin struct { // mutex returns the mutex for the specified record. func (p *ticketVotePlugin) mutex(token []byte) *sync.Mutex { - p.Lock() - defer p.Unlock() + p.mtxRecords.Lock() + defer p.mtxRecords.Unlock() t := hex.EncodeToString(token) m, ok := p.mutexes[t] diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/votes.go b/politeiad/backend/tlogbe/plugins/ticketvote/votes.go index 74d781e28..54be48dcf 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/votes.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/votes.go @@ -9,8 +9,8 @@ import ( ) func (p *ticketVotePlugin) votesCache(token []byte) map[string]string { - p.Lock() - defer p.Unlock() + p.mtxVotes.Lock() + defer p.mtxVotes.Unlock() // Return a copy of the map cv, ok := p.votes[hex.EncodeToString(token)] @@ -26,8 +26,8 @@ func (p *ticketVotePlugin) votesCache(token []byte) map[string]string { } func (p *ticketVotePlugin) votesCacheSet(token, ticket, voteBit string) { - p.Lock() - defer p.Unlock() + p.mtxVotes.Lock() + defer p.mtxVotes.Unlock() _, ok := p.votes[token] if !ok { @@ -41,8 +41,8 @@ func (p *ticketVotePlugin) votesCacheSet(token, ticket, voteBit string) { } func (p *ticketVotePlugin) votesCacheDel(token string) { - p.Lock() - defer p.Unlock() + p.mtxVotes.Lock() + defer p.mtxVotes.Unlock() delete(p.votes, token) diff --git a/politeiawww/cmd/pictl/proposal.go b/politeiawww/cmd/pictl/proposal.go index 6ece0156a..3217cb2d0 100644 --- a/politeiawww/cmd/pictl/proposal.go +++ b/politeiawww/cmd/pictl/proposal.go @@ -209,22 +209,23 @@ func proposalMetadataDecode(files []rcv1.File) (*piv1.ProposalMetadata, error) { } // voteMetadataDecode decodes and returns the VoteMetadata from the provided -// backend files. If a VoteMetadata is not found, an empty one will be -// returned. +// backend files. If a VoteMetadata is not found, nil will be returned. func voteMetadataDecode(files []rcv1.File) (*piv1.VoteMetadata, error) { - var vm piv1.VoteMetadata + var voteMD *piv1.VoteMetadata for _, v := range files { if v.Name == piv1.FileNameVoteMetadata { b, err := base64.StdEncoding.DecodeString(v.Payload) if err != nil { return nil, err } + var vm piv1.VoteMetadata err = json.Unmarshal(b, &vm) if err != nil { return nil, err } + voteMD = &vm break } } - return &vm, nil + return voteMD, nil } From b65dacda9b45cf0d142253881ac0b2c3d702d876 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 10 Feb 2021 14:34:29 -0600 Subject: [PATCH 292/449] Work on legacy www route support. --- politeiawww/politeiawww.go | 25 -- politeiawww/proposals.go | 489 ++++++++++++++++++++++--------------- 2 files changed, 289 insertions(+), 225 deletions(-) diff --git a/politeiawww/politeiawww.go b/politeiawww/politeiawww.go index a5a45b7d9..8e94d505d 100644 --- a/politeiawww/politeiawww.go +++ b/politeiawww/politeiawww.go @@ -193,31 +193,6 @@ func (p *politeiawww) handlePolicy(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, reply) } -// handleBatchVoteSummary handles the incoming batch vote summary command. It -// returns a VoteSummary for each of the provided censorship tokens. -func (p *politeiawww) handleBatchVoteSummary(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleBatchVoteSummary") - - var bvs www.BatchVoteSummary - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&bvs); err != nil { - RespondWithError(w, r, 0, "handleBatchVoteSummary: unmarshal", - www.UserError{ - ErrorCode: www.ErrorStatusInvalidInput, - }) - return - } - - reply, err := p.processBatchVoteSummary(r.Context(), bvs) - if err != nil { - RespondWithError(w, r, 0, - "handleBatchVoteSummary: processBatchVoteSummary %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, reply) -} - // websocketPing is used to verify that websockets are operational. func (p *politeiawww) websocketPing(id string) { log.Tracef("websocketPing %v", id) diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 7fe419e11..6f69500a0 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -11,189 +11,113 @@ import ( "net/http" "strconv" - "github.com/decred/politeia/decredplugin" - pd "github.com/decred/politeia/politeiad/api/v1" - ticketvote "github.com/decred/politeia/politeiad/plugins/ticketvote" - pi "github.com/decred/politeia/politeiawww/api/pi/v1" + pdv1 "github.com/decred/politeia/politeiad/api/v1" + v1 "github.com/decred/politeia/politeiad/api/v1" + piplugin "github.com/decred/politeia/politeiad/plugins/pi" + tvplugin "github.com/decred/politeia/politeiad/plugins/ticketvote" + usplugin "github.com/decred/politeia/politeiad/plugins/user" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/politeiawww/user" "github.com/decred/politeia/util" + "github.com/google/uuid" "github.com/gorilla/mux" ) -/* -func convertStateToWWW(state pi.PropStateT) www.PropStateT { - switch state { - case pi.PropStateInvalid: - return www.PropStateInvalid - case pi.PropStateUnvetted: - return www.PropStateUnvetted - case pi.PropStateVetted: - return www.PropStateVetted - default: - return www.PropStateInvalid - } -} -*/ - -func convertStatusToWWW(status pi.PropStatusT) www.PropStatusT { - switch status { - case pi.PropStatusInvalid: - return www.PropStatusInvalid - case pi.PropStatusPublic: - return www.PropStatusPublic - case pi.PropStatusCensored: - return www.PropStatusCensored - case pi.PropStatusAbandoned: - return www.PropStatusAbandoned - default: - return www.PropStatusInvalid - } -} - -/* -func convertProposalToWWW(pr *pi.ProposalRecord) (*www.ProposalRecord, error) { - // Decode metadata - var pm *piplugin.ProposalMetadata - for _, v := range pr.Metadata { - if v.Hint == pi.HintProposalMetadata { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return nil, err - } - var pm pi.ProposalMetadata - err = json.Unmarshal(b, &pm) - if err != nil { - return nil, err - } - } - } +func (p *politeiawww) processTokenInventory(ctx context.Context, isAdmin bool) (*www.TokenInventoryReply, error) { + log.Tracef("processTokenInventory") - // Convert files - files := make([]www.File, 0, len(pr.Files)) - for _, f := range pr.Files { - files = append(files, www.File{ - Name: f.Name, - MIME: f.MIME, - Digest: f.Digest, - Payload: f.Payload, - }) + // Get record inventory + ir, err := p.politeiad.InventoryByStatus(ctx) + if err != nil { + return nil, err } - // Convert metadata - metadata := make([]www.Metadata, 0, len(pr.Metadata)) - for _, md := range pr.Metadata { - metadata = append(metadata, www.Metadata{ - Digest: md.Digest, - Hint: md.Hint, - Payload: md.Payload, - }) + // Get vote inventory + vir, err := p.politeiad.TicketVoteInventory(ctx) + if err != nil { + return nil, err } var ( - publishedAt, censoredAt, abandonedAt int64 - changeMsg string - changeMsgTimestamp int64 + // Unvetted + unvetted = ir.Unvetted[pdv1.RecordStatusNotReviewed] + unvettedChanges = ir.Unvetted[pdv1.RecordStatusUnreviewedChanges] + unreviewed = append(unvetted, unvettedChanges...) + censored = ir.Unvetted[pdv1.RecordStatusCensored] + + // Human readable vote statuses + statusUnauth = tvplugin.VoteStatuses[tvplugin.VoteStatusUnauthorized] + statusAuth = tvplugin.VoteStatuses[tvplugin.VoteStatusAuthorized] + statusStarted = tvplugin.VoteStatuses[tvplugin.VoteStatusStarted] + statusApproved = tvplugin.VoteStatuses[tvplugin.VoteStatusApproved] + statusRejected = tvplugin.VoteStatuses[tvplugin.VoteStatusRejected] + + // Vetted + unauth = vir.Tokens[statusUnauth] + auth = vir.Tokens[statusAuth] + pre = append(unauth, auth...) + active = vir.Tokens[statusStarted] + approved = vir.Tokens[statusApproved] + rejected = vir.Tokens[statusRejected] + abandoned = ir.Vetted[pdv1.RecordStatusArchived] ) - for _, v := range pr.Statuses { - if v.Timestamp > changeMsgTimestamp { - changeMsg = v.Reason - changeMsgTimestamp = v.Timestamp - } - switch v.Status { - case pi.PropStatusPublic: - publishedAt = v.Timestamp - case pi.PropStatusCensored: - censoredAt = v.Timestamp - case pi.PropStatusAbandoned: - abandonedAt = v.Timestamp - } + + // Only return unvetted tokens to admins + if isAdmin { + unreviewed = nil + censored = nil } - return &www.ProposalRecord{ - Name: pm.Name, - State: convertStateToWWW(pr.State), - Status: convertStatusToWWW(pr.Status), - Timestamp: pr.Timestamp, - UserId: pr.UserID, - Username: pr.Username, - PublicKey: pr.PublicKey, - Signature: pr.Signature, - Version: pr.Version, - StatusChangeMessage: changeMsg, - PublishedAt: publishedAt, - CensoredAt: censoredAt, - AbandonedAt: abandonedAt, - Files: files, - Metadata: metadata, - CensorshipRecord: www.CensorshipRecord{ - Token: pr.CensorshipRecord.Token, - Merkle: pr.CensorshipRecord.Merkle, - Signature: pr.CensorshipRecord.Signature, - }, + return &www.TokenInventoryReply{ + Unreviewed: unreviewed, + Censored: censored, + Pre: pre, + Active: active, + Approved: approved, + Rejected: rejected, + Abandoned: abandoned, }, nil } -*/ -func convertVoteStatusToWWW(status ticketvote.VoteStatusT) www.PropVoteStatusT { - switch status { - case ticketvote.VoteStatusInvalid: - return www.PropVoteStatusInvalid - case ticketvote.VoteStatusUnauthorized: - return www.PropVoteStatusNotAuthorized - case ticketvote.VoteStatusAuthorized: - return www.PropVoteStatusAuthorized - case ticketvote.VoteStatusStarted: - return www.PropVoteStatusStarted - case ticketvote.VoteStatusFinished: - return www.PropVoteStatusFinished - default: - return www.PropVoteStatusInvalid - } +func (p *politeiawww) processAllVetted(ctx context.Context, gav www.GetAllVetted) (*www.GetAllVettedReply, error) { + log.Tracef("processAllVetted: %v %v", gav.Before, gav.After) + // TODO + + return nil, nil } -func convertVoteTypeToWWW(t ticketvote.VoteT) www.VoteT { - switch t { - case ticketvote.VoteTypeInvalid: - return www.VoteTypeInvalid - case ticketvote.VoteTypeStandard: - return www.VoteTypeStandard - case ticketvote.VoteTypeRunoff: - return www.VoteTypeRunoff - default: - return www.VoteTypeInvalid +func (p *politeiawww) proposal(ctx context.Context, token, version string) (*www.ProposalRecord, error) { + // Get record + r, err := p.politeiad.GetVetted(ctx, token, version) + if err != nil { + return nil, err + } + pr, err := convertRecordToProposal(*r) + if err != nil { + return nil, err } -} -func convertVoteErrorCodeToWWW(errcode ticketvote.VoteErrorT) decredplugin.ErrorStatusT { - switch errcode { - case ticketvote.VoteErrorInvalid: - return decredplugin.ErrorStatusInvalid - case ticketvote.VoteErrorInternalError: - return decredplugin.ErrorStatusInternalError - case ticketvote.VoteErrorRecordNotFound: - return decredplugin.ErrorStatusProposalNotFound - case ticketvote.VoteErrorVoteBitInvalid: - return decredplugin.ErrorStatusInvalidVoteBit - case ticketvote.VoteErrorVoteStatusInvalid: - return decredplugin.ErrorStatusVoteHasEnded - case ticketvote.VoteErrorTicketAlreadyVoted: - return decredplugin.ErrorStatusDuplicateVote - case ticketvote.VoteErrorTicketNotEligible: - return decredplugin.ErrorStatusIneligibleTicket - default: - return decredplugin.ErrorStatusInternalError + // Fill in user data + userID := userIDFromMetadataStreams(r.Metadata) + uid, err := uuid.Parse(userID) + u, err := p.db.UserGetById(uid) + if err != nil { + return nil, err } + pr.Username = u.Username + + return pr, nil } func (p *politeiawww) processProposalDetails(ctx context.Context, pd www.ProposalsDetails, u *user.User) (*www.ProposalDetailsReply, error) { log.Tracef("processProposalDetails: %v", pd.Token) - return nil, nil -} + // This route will now only return vetted proposal. This is fine + // since API consumers of this legacy route will only need public + // proposals. -func (p *politeiawww) processAllVetted(ctx context.Context, gav www.GetAllVetted) (*www.GetAllVettedReply, error) { + // Remove files if the user is not an admin or the author return nil, nil } @@ -208,7 +132,7 @@ func (p *politeiawww) processVoteResults(ctx context.Context, token string) (*ww log.Tracef("processVoteResults: %v", token) // TODO Get vote details - var vd ticketvote.VoteDetails + var vd tvplugin.VoteDetails // Convert to www startHeight := strconv.FormatUint(uint64(vd.StartBlockHeight), 10) @@ -223,7 +147,7 @@ func (p *politeiawww) processVoteResults(ctx context.Context, token string) (*ww } // TODO Get cast votes - var rr ticketvote.ResultsReply + var rr tvplugin.ResultsReply // Convert to www votes := make([]www.CastVote, 0, len(rr.Votes)) @@ -264,7 +188,7 @@ func (p *politeiawww) processBatchVoteSummary(ctx context.Context, bvs www.Batch // TODO var bestBlock uint32 - var vs []ticketvote.SummaryReply + var vs []tvplugin.SummaryReply // Prepare reply summaries := make(map[string]www.VoteSummary, len(vs)) @@ -283,8 +207,8 @@ func (p *politeiawww) processBatchVoteSummary(ctx context.Context, bvs www.Batch // TODO var token string summaries[token] = www.VoteSummary{ - Status: convertVoteStatusToWWW(v.Status), - Type: convertVoteTypeToWWW(v.Type), + // Status: convertVoteStatusToWWW(v.Status), + // Type: convertVoteTypeToWWW(v.Type), // Approved: v.Approved, EligibleTickets: v.EligibleTickets, Duration: v.Duration, @@ -310,21 +234,21 @@ func (p *politeiawww) processCastVotes(ctx context.Context, ballot *www.Ballot) log.Tracef("processCastVotes") // Prepare plugin command - votes := make([]ticketvote.CastVote, 0, len(ballot.Votes)) + votes := make([]tvplugin.CastVote, 0, len(ballot.Votes)) for _, vote := range ballot.Votes { - votes = append(votes, ticketvote.CastVote{ + votes = append(votes, tvplugin.CastVote{ Token: vote.Ticket, Ticket: vote.Ticket, VoteBit: vote.VoteBit, Signature: vote.Signature, }) } - cb := ticketvote.CastBallot{ + cb := tvplugin.CastBallot{ Ballot: votes, } // TODO _ = cb - var cbr ticketvote.CastBallotReply + var cbr tvplugin.CastBallotReply // Prepare reply receipts := make([]www.CastVoteReply, 0, len(cbr.Receipts)) @@ -333,7 +257,7 @@ func (p *politeiawww) processCastVotes(ctx context.Context, ballot *www.Ballot) ClientSignature: ballot.Votes[k].Signature, Signature: v.Receipt, Error: v.ErrorContext, - ErrorStatus: convertVoteErrorCodeToWWW(v.ErrorCode), + // ErrorStatus: convertVoteErrorCodeToWWW(v.ErrorCode), }) } @@ -342,43 +266,6 @@ func (p *politeiawww) processCastVotes(ctx context.Context, ballot *www.Ballot) }, nil } -func (p *politeiawww) processTokenInventory(ctx context.Context, isAdmin bool) (*www.TokenInventoryReply, error) { - log.Tracef("processTokenInventory") - - // Get record inventory - ir, err := p.politeiad.InventoryByStatus(ctx) - if err != nil { - return nil, err - } - - // TODO Get vote inventory - - // Unpack record inventory - var ( - archived = ir.Vetted[pd.RecordStatusArchived] - unvetted = ir.Unvetted[pd.RecordStatusNotReviewed] - unvettedChanges = ir.Unvetted[pd.RecordStatusUnreviewedChanges] - unreviewed = append(unvetted, unvettedChanges...) - censored = ir.Unvetted[pd.RecordStatusCensored] - ) - - // Only return unvetted tokens to admins - if isAdmin { - unreviewed = nil - censored = nil - } - - return &www.TokenInventoryReply{ - Unreviewed: unreviewed, - Censored: censored, - // Pre: append(vir.Unauthorized, vir.Authorized...), - // Active: vir.Started, - // Approved: vir.Approved, - // Rejected: vir.Rejected, - Abandoned: archived, - }, nil -} - func (p *politeiawww) handleTokenInventory(w http.ResponseWriter, r *http.Request) { log.Tracef("handleTokenInventory") @@ -542,3 +429,205 @@ func (p *politeiawww) handleVoteResults(w http.ResponseWriter, r *http.Request) util.RespondWithJSON(w, http.StatusOK, vrr) } + +func (p *politeiawww) handleBatchVoteSummary(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleBatchVoteSummary") + + var bvs www.BatchVoteSummary + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&bvs); err != nil { + RespondWithError(w, r, 0, "handleBatchVoteSummary: unmarshal", + www.UserError{ + ErrorCode: www.ErrorStatusInvalidInput, + }) + return + } + + reply, err := p.processBatchVoteSummary(r.Context(), bvs) + if err != nil { + RespondWithError(w, r, 0, + "handleBatchVoteSummary: processBatchVoteSummary %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, reply) +} + +// userMetadataDecode decodes and returns the UserMetadata from the provided +// metadata streams. If a UserMetadata is not found, nil is returned. +func userMetadataDecode(ms []v1.MetadataStream) (*usplugin.UserMetadata, error) { + var userMD *usplugin.UserMetadata + for _, v := range ms { + if v.ID == usplugin.MDStreamIDUserMetadata { + var um usplugin.UserMetadata + err := json.Unmarshal([]byte(v.Payload), &um) + if err != nil { + return nil, err + } + userMD = &um + break + } + } + return userMD, nil +} + +// userIDFromMetadataStreams searches for a UserMetadata and parses the user ID +// from it if found. An empty string is returned if no UserMetadata is found. +func userIDFromMetadataStreams(ms []pdv1.MetadataStream) string { + um, err := userMetadataDecode(ms) + if err != nil { + return "" + } + if um == nil { + return "" + } + return um.UserID +} + +func convertStatusToWWW(status pdv1.RecordStatusT) www.PropStatusT { + switch status { + case pdv1.RecordStatusInvalid: + return www.PropStatusInvalid + case pdv1.RecordStatusPublic: + return www.PropStatusPublic + case pdv1.RecordStatusCensored: + return www.PropStatusCensored + case pdv1.RecordStatusArchived: + return www.PropStatusAbandoned + default: + return www.PropStatusInvalid + } +} + +// TODO convertRecordToProposal +func convertRecordToProposal(r pdv1.Record) (*www.ProposalRecord, error) { + // Decode metadata + var um *usplugin.UserMetadata + for _, v := range r.Metadata { + switch v.ID { + case usplugin.MDStreamIDUserMetadata: + } + } + + // Convert files + var ( + pm *piplugin.ProposalMetadata + vm *tvplugin.VoteMetadata + files = make([]www.File, 0, len(r.Files)) + metadata = make([]www.Metadata, 0, len(r.Files)) + ) + for _, v := range r.Files { + switch v.Name { + case piplugin.FileNameProposalMetadata: + case tvplugin.FileNameVoteMetadata: + default: + files = append(files, www.File{ + Name: v.Name, + MIME: v.MIME, + Digest: v.Digest, + Payload: v.Payload, + }) + } + } + + /* + var ( + publishedAt, censoredAt, abandonedAt int64 + changeMsg string + changeMsgTimestamp int64 + ) + for _, v := range pr.Statuses { + if v.Timestamp > changeMsgTimestamp { + changeMsg = v.Reason + changeMsgTimestamp = v.Timestamp + } + switch v.Status { + case piv1.PropStatusPublic: + publishedAt = v.Timestamp + case piv1.PropStatusCensored: + censoredAt = v.Timestamp + case piv1.PropStatusAbandoned: + abandonedAt = v.Timestamp + } + } + */ + + return &www.ProposalRecord{ + Name: pm.Name, + State: www.PropStateVetted, + Status: convertStatusToWWW(r.Status), + Timestamp: r.Timestamp, + UserId: um.UserID, + Username: "", // Intentionally omitted + PublicKey: um.PublicKey, + Signature: um.Signature, + Version: r.Version, + // StatusChangeMessage: changeMsg, + // PublishedAt: publishedAt, + // CensoredAt: censoredAt, + // AbandonedAt: abandonedAt, + LinkTo: vm.LinkTo, + LinkBy: vm.LinkBy, + // LinkedFrom: submissions, + Files: files, + Metadata: metadata, + CensorshipRecord: www.CensorshipRecord{ + Token: r.CensorshipRecord.Token, + Merkle: r.CensorshipRecord.Merkle, + Signature: r.CensorshipRecord.Signature, + }, + }, nil +} + +/* +func convertVoteStatusToWWW(status tvplugin.VoteStatusT) www.PropVoteStatusT { + switch status { + case tvplugin.VoteStatusInvalid: + return www.PropVoteStatusInvalid + case tvplugin.VoteStatusUnauthorized: + return www.PropVoteStatusNotAuthorized + case tvplugin.VoteStatusAuthorized: + return www.PropVoteStatusAuthorized + case tvplugin.VoteStatusStarted: + return www.PropVoteStatusStarted + case tvplugin.VoteStatusFinished: + return www.PropVoteStatusFinished + default: + return www.PropVoteStatusInvalid + } +} + +func convertVoteTypeToWWW(t tvplugin.VoteT) www.VoteT { + switch t { + case tvplugin.VoteTypeInvalid: + return www.VoteTypeInvalid + case tvplugin.VoteTypeStandard: + return www.VoteTypeStandard + case tvplugin.VoteTypeRunoff: + return www.VoteTypeRunoff + default: + return www.VoteTypeInvalid + } +} + +func convertVoteErrorCodeToWWW(errcode tvplugin.VoteErrorT) decredplugin.ErrorStatusT { + switch errcode { + case tvplugin.VoteErrorInvalid: + return decredplugin.ErrorStatusInvalid + case tvplugin.VoteErrorInternalError: + return decredplugin.ErrorStatusInternalError + case tvplugin.VoteErrorRecordNotFound: + return decredplugin.ErrorStatusProposalNotFound + case tvplugin.VoteErrorVoteBitInvalid: + return decredplugin.ErrorStatusInvalidVoteBit + case tvplugin.VoteErrorVoteStatusInvalid: + return decredplugin.ErrorStatusVoteHasEnded + case tvplugin.VoteErrorTicketAlreadyVoted: + return decredplugin.ErrorStatusDuplicateVote + case tvplugin.VoteErrorTicketNotEligible: + return decredplugin.ErrorStatusIneligibleTicket + default: + return decredplugin.ErrorStatusInternalError + } +} +*/ From 5d507d98889d1ae3c1e3194aa12b8bc38d9f8370 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 10 Feb 2021 15:07:15 -0600 Subject: [PATCH 293/449] Fix politeiawww plugin error handling bug. --- politeiawww/api/comments/v1/v1.go | 2 +- politeiawww/api/pi/v1/v1.go | 2 +- politeiawww/api/records/v1/v1.go | 2 +- politeiawww/api/ticketvote/v1/v1.go | 2 +- politeiawww/comments/error.go | 27 ++++++++++++++++++++++----- politeiawww/pi/error.go | 4 ++-- politeiawww/records/error.go | 29 ++++++++++++++++++++++++----- 7 files changed, 52 insertions(+), 16 deletions(-) diff --git a/politeiawww/api/comments/v1/v1.go b/politeiawww/api/comments/v1/v1.go index 1d151c42e..596da6de4 100644 --- a/politeiawww/api/comments/v1/v1.go +++ b/politeiawww/api/comments/v1/v1.go @@ -80,7 +80,7 @@ type PluginErrorReply struct { // Error satisfies the error interface. func (e PluginErrorReply) Error() string { - return fmt.Sprintf("plugin error code: %v", e.ErrorCode) + return fmt.Sprintf("plugin %v error code: %v", e.PluginID, e.ErrorCode) } // ServerErrorReply is the reply that the server returns when it encounters an diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 5f79ab68d..d0140fbf4 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -68,7 +68,7 @@ type PluginErrorReply struct { // Error satisfies the error interface. func (e PluginErrorReply) Error() string { - return fmt.Sprintf("plugin error code: %v", e.ErrorCode) + return fmt.Sprintf("plugin %v error code: %v", e.PluginID, e.ErrorCode) } // ServerErrorReply is the reply that the server returns when it encounters an diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go index 898594933..33eeea90c 100644 --- a/politeiawww/api/records/v1/v1.go +++ b/politeiawww/api/records/v1/v1.go @@ -99,7 +99,7 @@ type PluginErrorReply struct { // Error satisfies the error interface. func (e PluginErrorReply) Error() string { - return fmt.Sprintf("plugin user error code: %v", e.ErrorCode) + return fmt.Sprintf("plugin %v error code: %v", e.PluginID, e.ErrorCode) } // ServerErrorReply is the reply that the server returns when it encounters an diff --git a/politeiawww/api/ticketvote/v1/v1.go b/politeiawww/api/ticketvote/v1/v1.go index 621ca0f71..6f7437d4b 100644 --- a/politeiawww/api/ticketvote/v1/v1.go +++ b/politeiawww/api/ticketvote/v1/v1.go @@ -65,7 +65,7 @@ type PluginErrorReply struct { // Error satisfies the error interface. func (e PluginErrorReply) Error() string { - return fmt.Sprintf("plugin error code: %v", e.ErrorCode) + return fmt.Sprintf("plugin %v error code: %v", e.PluginID, e.ErrorCode) } // ServerErrorReply is the reply that the server returns when it encounters an diff --git a/politeiawww/comments/error.go b/politeiawww/comments/error.go index 6530a868f..f9a7e7713 100644 --- a/politeiawww/comments/error.go +++ b/politeiawww/comments/error.go @@ -20,8 +20,9 @@ import ( func respondWithError(w http.ResponseWriter, r *http.Request, format string, err error) { var ( - ue v1.UserErrorReply - pe pdclient.Error + ue v1.UserErrorReply + pe v1.PluginErrorReply + pde pdclient.Error ) switch { case errors.As(err, &ue): @@ -40,11 +41,27 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err return case errors.As(err, &pe): + // politeiawww plugin error + m := fmt.Sprintf("%v Plugin error: %v %v", + util.RemoteAddr(r), pe.PluginID, pe.ErrorCode) + if pe.ErrorContext != "" { + m += fmt.Sprintf(": %v", pe.ErrorContext) + } + log.Infof(m) + util.RespondWithJSON(w, http.StatusBadRequest, + v1.PluginErrorReply{ + PluginID: pe.PluginID, + ErrorCode: pe.ErrorCode, + ErrorContext: pe.ErrorContext, + }) + return + + case errors.As(err, &pde): // Politeiad error var ( - pluginID = pe.ErrorReply.PluginID - errCode = pe.ErrorReply.ErrorCode - errContext = strings.Join(pe.ErrorReply.ErrorContext, ",") + pluginID = pde.ErrorReply.PluginID + errCode = pde.ErrorReply.ErrorCode + errContext = strings.Join(pde.ErrorReply.ErrorContext, ",") ) e := convertPDErrorCode(errCode) switch { diff --git a/politeiawww/pi/error.go b/politeiawww/pi/error.go index c8bdda039..482a28eba 100644 --- a/politeiawww/pi/error.go +++ b/politeiawww/pi/error.go @@ -24,8 +24,8 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err ) switch { case errors.As(err, &ue): - // Comments user error - m := fmt.Sprintf("Comments user error: %v %v %v", + // Pi user error + m := fmt.Sprintf("Pi user error: %v %v %v", util.RemoteAddr(r), ue.ErrorCode, v1.ErrorCodes[ue.ErrorCode]) if ue.ErrorContext != "" { m += fmt.Sprintf(": %v", ue.ErrorContext) diff --git a/politeiawww/records/error.go b/politeiawww/records/error.go index 483e2df23..9f469f23a 100644 --- a/politeiawww/records/error.go +++ b/politeiawww/records/error.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "github.com/davecgh/go-spew/spew" pdv1 "github.com/decred/politeia/politeiad/api/v1" pdclient "github.com/decred/politeia/politeiad/client" v1 "github.com/decred/politeia/politeiawww/api/records/v1" @@ -19,9 +20,11 @@ import ( ) func respondWithError(w http.ResponseWriter, r *http.Request, format string, err error) { + spew.Dump(err) var ( - ue v1.UserErrorReply - pe pdclient.Error + ue v1.UserErrorReply + pe v1.PluginErrorReply + pde pdclient.Error ) switch { case errors.As(err, &ue): @@ -40,11 +43,27 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err return case errors.As(err, &pe): + // politeiawww plugin error + m := fmt.Sprintf("%v Plugin error: %v %v", + util.RemoteAddr(r), pe.PluginID, pe.ErrorCode) + if pe.ErrorContext != "" { + m += fmt.Sprintf(": %v", pe.ErrorContext) + } + log.Infof(m) + util.RespondWithJSON(w, http.StatusBadRequest, + v1.PluginErrorReply{ + PluginID: pe.PluginID, + ErrorCode: pe.ErrorCode, + ErrorContext: pe.ErrorContext, + }) + return + + case errors.As(err, &pde): // Politeiad error var ( - pluginID = pe.ErrorReply.PluginID - errCode = pe.ErrorReply.ErrorCode - errContext = strings.Join(pe.ErrorReply.ErrorContext, ",") + pluginID = pde.ErrorReply.PluginID + errCode = pde.ErrorReply.ErrorCode + errContext = strings.Join(pde.ErrorReply.ErrorContext, ",") ) e := convertPDErrorCode(errCode) switch { From 686c8bebe687896e127d4a0107962c0c0e9226b6 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 10 Feb 2021 15:51:15 -0600 Subject: [PATCH 294/449] Fix ticketvote bugs. --- .../backend/tlogbe/plugins/ticketvote/cmds.go | 17 +++++++++++++---- .../tlogbe/plugins/ticketvote/inventory.go | 3 ++- politeiawww/cmd/pictl/ticketvote.go | 12 +++++++++--- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go index 4cb68ce73..65e59a21d 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go @@ -2151,9 +2151,7 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str // Verify that all tokens in the ballot are valid, full length // tokens and that they are all voting for the same record. - var ( - receipts = make([]ticketvote.CastVoteReply, len(votes)) - ) + receipts := make([]ticketvote.CastVoteReply, len(votes)) for k, v := range votes { // Verify token t, err := tokenDecode(v.Token) @@ -2512,7 +2510,7 @@ func voteIsApproved(vd ticketvote.VoteDetails, results []ticketvote.VoteOptionRe switch v.ID { case ticketvote.VoteOptionIDApprove: // Valid vote option - approvedVotes++ + approvedVotes = v.Votes case ticketvote.VoteOptionIDReject: // Valid vote option default: @@ -2528,12 +2526,23 @@ func voteIsApproved(vd ticketvote.VoteDetails, results []ticketvote.VoteOptionRe case total < quorum: // Quorum not met approved = false + + log.Debugf("Quorum not met on %v: votes cast %v, quorum %v", + vd.Params.Token, total, quorum) + case approvedVotes < pass: // Pass percentage not met approved = false + + log.Debugf("Pass threshold not met on %v: approved %v, required %v", + vd.Params.Token, total, quorum) + default: // Vote was approved approved = true + + log.Debugf("Vote %v approved: quorum %v, pass %v, total %v, approved %v", + vd.Params.Token, quorum, pass, total, approvedVotes) } return approved diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go b/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go index 666cb830d..946210438 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go @@ -258,9 +258,10 @@ func (p *ticketVotePlugin) invGet(bestBlock uint32) (*inventory, error) { // if any votes have finished. active := make([]activeVote, 0, len(inv.Active)) for _, v := range inv.Active { - if v.EndHeight >= bestBlock { + if bestBlock < v.EndHeight { // Vote has not finished yet. Keep it in the active votes list. active = append(active, v) + continue } // Vote has finished. Get vote summary. diff --git a/politeiawww/cmd/pictl/ticketvote.go b/politeiawww/cmd/pictl/ticketvote.go index c808cf0b1..265d05852 100644 --- a/politeiawww/cmd/pictl/ticketvote.go +++ b/politeiawww/cmd/pictl/ticketvote.go @@ -71,11 +71,17 @@ func printVoteSummary(token string, s tkv1.Summary) { // Nothing else to print return } - pass := int(float64(s.PassPercentage) / 100 * float64(s.EligibleTickets)) + var total uint64 + for _, v := range s.Results { + total += v.Votes + } quorum := int(float64(s.QuorumPercentage) / 100 * float64(s.EligibleTickets)) + pass := int(float64(s.PassPercentage) / 100 * float64(total)) printf("Type : %v\n", tkv1.VoteTypes[s.Type]) - printf("Pass Percentage : %v%% (%v votes)\n", s.PassPercentage, pass) - printf("Quorum Percentage : %v%% (%v votes)\n", s.QuorumPercentage, quorum) + printf("Quorum Percentage : %v%% of eligible votes (%v votes)\n", + s.QuorumPercentage, quorum) + printf("Pass Percentage : %v%% of cast votes (%v votes)\n", + s.PassPercentage, pass) printf("Duration : %v blocks\n", s.Duration) printf("Start Block Hash : %v\n", s.StartBlockHash) printf("Start Block Height: %v\n", s.StartBlockHeight) From f016a07412e59ddc8e2e1a479ddd3f144c88e81a Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 10 Feb 2021 17:15:58 -0600 Subject: [PATCH 295/449] Add userproposals cmd and fix bugs. --- .../backend/tlogbe/plugins/user/hooks.go | 27 +++++++ politeiad/backend/tlogbe/plugins/user/user.go | 2 + politeiawww/client/records.go | 20 ++++++ politeiawww/cmd/pictl/cmdproposals.go | 2 +- politeiawww/cmd/pictl/cmduserproposals.go | 71 +++++++++++++++++++ politeiawww/cmd/pictl/pictl.go | 1 + 6 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 politeiawww/cmd/pictl/cmduserproposals.go diff --git a/politeiad/backend/tlogbe/plugins/user/hooks.go b/politeiad/backend/tlogbe/plugins/user/hooks.go index f3a4125a3..156e11ab3 100644 --- a/politeiad/backend/tlogbe/plugins/user/hooks.go +++ b/politeiad/backend/tlogbe/plugins/user/hooks.go @@ -331,3 +331,30 @@ func (p *userPlugin) hookSetRecordStatusPre(payload string) error { return nil } + +func (p *userPlugin) hookSetRecordStatusPost(payload string) error { + var srs plugins.HookSetRecordStatus + err := json.Unmarshal([]byte(payload), &srs) + if err != nil { + return err + } + + // When a record becomes vetted the post hook will be executed on + // the vetted tstore instance. The record must be added to the + // vetted user cache. + if srs.RecordMetadata.Status == backend.MDStatusVetted { + // Decode user metadata + um, err := userMetadataDecode(srs.Metadata) + if err != nil { + return err + } + + // Add token to the user cache + err = p.userCacheAddToken(um.UserID, srs.RecordMetadata.Token) + if err != nil { + return err + } + } + + return nil +} diff --git a/politeiad/backend/tlogbe/plugins/user/user.go b/politeiad/backend/tlogbe/plugins/user/user.go index 56f9166a3..d0830612a 100644 --- a/politeiad/backend/tlogbe/plugins/user/user.go +++ b/politeiad/backend/tlogbe/plugins/user/user.go @@ -70,6 +70,8 @@ func (p *userPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload s return p.hookEditMetadataPre(payload) case plugins.HookTypeSetRecordStatusPre: return p.hookSetRecordStatusPre(payload) + case plugins.HookTypeSetRecordStatusPost: + return p.hookSetRecordStatusPost(payload) } return nil diff --git a/politeiawww/client/records.go b/politeiawww/client/records.go index 5d4670940..098177f00 100644 --- a/politeiawww/client/records.go +++ b/politeiawww/client/records.go @@ -136,6 +136,26 @@ func (c *Client) RecordTimestamps(t rcv1.Timestamps) (*rcv1.TimestampsReply, err return &tr, nil } +// UserRecords sends a records v1 UserRecords request to politeiawww. +func (c *Client) UserRecords(ur rcv1.UserRecords) (*rcv1.UserRecordsReply, error) { + resBody, err := c.makeReq(http.MethodPost, + rcv1.APIRoute, rcv1.RouteUserRecords, ur) + if err != nil { + return nil, err + } + + var urr rcv1.UserRecordsReply + err = json.Unmarshal(resBody, &urr) + if err != nil { + return nil, err + } + if c.verbose { + fmt.Printf("%v\n", util.FormatJSON(urr)) + } + + return &urr, nil +} + // digestsVerify verifies that all file digests match the calculated SHA256 // digests of the file payloads. func digestsVerify(files []rcv1.File) error { diff --git a/politeiawww/cmd/pictl/cmdproposals.go b/politeiawww/cmd/pictl/cmdproposals.go index d74e75473..e33775832 100644 --- a/politeiawww/cmd/pictl/cmdproposals.go +++ b/politeiawww/cmd/pictl/cmdproposals.go @@ -13,7 +13,7 @@ import ( type cmdProposals struct { Args struct { Tokens []string `positional-arg-name:"proposals" required:"true"` - } `positional-args:"true" optional:"true"` + } `positional-args:"true"` // Unvetted is used to indicate the state of the proposals are // unvetted. If this flag is not used it will be assumed that the diff --git a/politeiawww/cmd/pictl/cmduserproposals.go b/politeiawww/cmd/pictl/cmduserproposals.go new file mode 100644 index 000000000..4c10a09e0 --- /dev/null +++ b/politeiawww/cmd/pictl/cmduserproposals.go @@ -0,0 +1,71 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + pclient "github.com/decred/politeia/politeiawww/client" +) + +// cmdUserProposals retrieves the proposal records for a user. +type cmdUserProposals struct { + Args struct { + UserID string `positional-arg-name:"userID" optional:"true"` + } `positional-args:"true"` +} + +// Execute executes the cmdUserProposals command. +// +// This function satisfies the go-flags Commander interface. +func (c *cmdUserProposals) Execute(args []string) error { + // Setup client + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Cookies: cfg.Cookies, + HeaderCSRF: cfg.CSRF, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, + } + pc, err := pclient.New(cfg.Host, opts) + if err != nil { + return err + } + + // Setup user ID + userID := c.Args.UserID + if userID == "" { + // No user ID provided. Use the user ID of the logged in user. + lr, err := client.Me() + if err != nil { + if err.Error() == "401" { + return fmt.Errorf("no user ID provided and no logged in user found") + } + return err + } + userID = lr.UserID + } + + // Get user proposals + ur := rcv1.UserRecords{ + UserID: userID, + } + urr, err := pc.UserRecords(ur) + if err != nil { + return err + } + + // Print record tokens to stdout + printJSON(urr) + + return nil +} + +// userProposalsHelpMsg is printed to stdout by the help command. +const userProposalsHelpMsg = `userproposals "userID" + +Retrieve the proprosals that were submitted by a user. If no user ID is given, +the ID of the logged in user will be used.` diff --git a/politeiawww/cmd/pictl/pictl.go b/politeiawww/cmd/pictl/pictl.go index 910d017ec..61173a0cf 100644 --- a/politeiawww/cmd/pictl/pictl.go +++ b/politeiawww/cmd/pictl/pictl.go @@ -72,6 +72,7 @@ type pictl struct { Proposals cmdProposals `command:"proposals"` ProposalInv cmdProposalInv `command:"proposalinv"` ProposalTimestamps cmdProposalTimestamps `command:"proposaltimestamps"` + UserProposals cmdUserProposals `command:"userproposals"` // Comments commands CommentsPolicy cmdCommentPolicy `command:"commentpolicy"` From 291a2ed06b38d39b95124b5c8dfa701dfcd30d28 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 10 Feb 2021 17:19:06 -0600 Subject: [PATCH 296/449] Update policy routes. --- politeiawww/client/comments.go | 2 +- politeiawww/client/pi.go | 2 +- politeiawww/client/ticketvote.go | 2 +- politeiawww/cmd/pictl/pictl.go | 1 + politeiawww/piwww.go | 6 +++--- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/politeiawww/client/comments.go b/politeiawww/client/comments.go index 9d166773b..f19099bf7 100644 --- a/politeiawww/client/comments.go +++ b/politeiawww/client/comments.go @@ -15,7 +15,7 @@ import ( // CommentPolicy sends a comments v1 Policy request to politeiawww. func (c *Client) CommentPolicy() (*cmv1.PolicyReply, error) { - resBody, err := c.makeReq(http.MethodGet, + resBody, err := c.makeReq(http.MethodPost, cmv1.APIRoute, cmv1.RoutePolicy, nil) if err != nil { return nil, err diff --git a/politeiawww/client/pi.go b/politeiawww/client/pi.go index b5fcc5bab..687b16fd6 100644 --- a/politeiawww/client/pi.go +++ b/politeiawww/client/pi.go @@ -15,7 +15,7 @@ import ( // PiPolicy sends a pi v1 Policy request to politeiawww. func (c *Client) PiPolicy() (*piv1.PolicyReply, error) { - resBody, err := c.makeReq(http.MethodGet, + resBody, err := c.makeReq(http.MethodPost, piv1.APIRoute, piv1.RoutePolicy, nil) if err != nil { return nil, err diff --git a/politeiawww/client/ticketvote.go b/politeiawww/client/ticketvote.go index 9c646d6e2..92e19f731 100644 --- a/politeiawww/client/ticketvote.go +++ b/politeiawww/client/ticketvote.go @@ -15,7 +15,7 @@ import ( // TicketVotePolicy sends a ticketvote v1 Policy request to politeiawww. func (c *Client) TicketVotePolicy() (*tkv1.PolicyReply, error) { - resBody, err := c.makeReq(http.MethodGet, + resBody, err := c.makeReq(http.MethodPost, tkv1.APIRoute, tkv1.RoutePolicy, nil) if err != nil { return nil, err diff --git a/politeiawww/cmd/pictl/pictl.go b/politeiawww/cmd/pictl/pictl.go index 61173a0cf..a84f9fe02 100644 --- a/politeiawww/cmd/pictl/pictl.go +++ b/politeiawww/cmd/pictl/pictl.go @@ -151,6 +151,7 @@ Proposal commands proposals (public) Get proposals without their files proposalinv (public) Get inventory by proposal status proposaltimestamps (public) Get timestamps for a proposal + userproposals (public) Get proposals submitted by a user Comment commands commentpolicy (public) Get the comments api policy diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index f259f4160..855ecb13f 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -90,7 +90,7 @@ func (p *politeiawww) setupPiRoutes(r *records.Records, c *comments.Comments, t permissionPublic) // Comment routes - p.addRoute(http.MethodGet, cmv1.APIRoute, + p.addRoute(http.MethodPost, cmv1.APIRoute, cmv1.RoutePolicy, c.HandlePolicy, permissionPublic) p.addRoute(http.MethodPost, cmv1.APIRoute, @@ -116,7 +116,7 @@ func (p *politeiawww) setupPiRoutes(r *records.Records, c *comments.Comments, t permissionPublic) // Ticket vote routes - p.addRoute(http.MethodGet, tkv1.APIRoute, + p.addRoute(http.MethodPost, tkv1.APIRoute, tkv1.RoutePolicy, t.HandlePolicy, permissionPublic) p.addRoute(http.MethodPost, tkv1.APIRoute, @@ -148,7 +148,7 @@ func (p *politeiawww) setupPiRoutes(r *records.Records, c *comments.Comments, t permissionPublic) // Pi routes - p.addRoute(http.MethodGet, piv1.APIRoute, + p.addRoute(http.MethodPost, piv1.APIRoute, piv1.RoutePolicy, pic.HandlePolicy, permissionPublic) p.addRoute(http.MethodPost, piv1.APIRoute, From 8139907a394c66332649642f4bc3b5fcfb9620d5 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 11 Feb 2021 09:36:41 -0600 Subject: [PATCH 297/449] Remove unvetted from user cache when made public. --- politeiad/backend/tlogbe/plugins/plugins.go | 3 + .../backend/tlogbe/plugins/user/cache.go | 57 ++++++++++++++++++- .../backend/tlogbe/plugins/user/hooks.go | 31 +++++++--- politeiad/backend/tlogbe/plugins/user/user.go | 2 +- politeiad/backend/tlogbe/tlogbe.go | 22 +++---- 5 files changed, 93 insertions(+), 22 deletions(-) diff --git a/politeiad/backend/tlogbe/plugins/plugins.go b/politeiad/backend/tlogbe/plugins/plugins.go index 5022b860b..a019de962 100644 --- a/politeiad/backend/tlogbe/plugins/plugins.go +++ b/politeiad/backend/tlogbe/plugins/plugins.go @@ -206,6 +206,9 @@ type TlogClient interface { // to the digest. Timestamp(treeID int64, digest []byte) (*backend.Timestamp, error) + // RecordExists returns whether a record exists. + RecordExists(treeID int64) bool + // Record returns a version of a record. Record(treeID int64, version uint32) (*backend.Record, error) diff --git a/politeiad/backend/tlogbe/plugins/user/cache.go b/politeiad/backend/tlogbe/plugins/user/cache.go index f40ecacf6..cc3052d61 100644 --- a/politeiad/backend/tlogbe/plugins/user/cache.go +++ b/politeiad/backend/tlogbe/plugins/user/cache.go @@ -7,6 +7,7 @@ package user import ( "encoding/json" "errors" + "fmt" "io/ioutil" "os" "path/filepath" @@ -83,8 +84,7 @@ func (p *userPlugin) userCacheSaveWithLock(userID string, uc userCache) error { return ioutil.WriteFile(fp, b, 0664) } -// userCacheAddToken adds the provided token to the cached userCache for the -// provided user. +// userCacheAddToken adds a token to a user cache. // // This function must be called WITHOUT the lock held. func (p *userPlugin) userCacheAddToken(userID string, token string) error { @@ -101,5 +101,56 @@ func (p *userPlugin) userCacheAddToken(userID string, token string) error { uc.Tokens = append(uc.Tokens, token) // Save changes - return p.userCacheSaveWithLock(userID, *uc) + err = p.userCacheSaveWithLock(userID, *uc) + if err != nil { + return err + } + + log.Debugf("User cache add %v %v", userID, token) + + return nil +} + +// userCacheDelToken deletes a token from a user cache. +// +// This function must be called WITHOUT the lock held. +func (p *userPlugin) userCacheDelToken(userID string, token string) error { + p.Lock() + defer p.Unlock() + + // Get current user data + uc, err := p.userCacheWithLock(userID) + if err != nil { + return err + } + + // Find token index + var i int + var found bool + for k, v := range uc.Tokens { + if v == token { + i = k + found = true + break + } + } + if !found { + return fmt.Errorf("user token not found %v %v", userID, token) + } + + // Del token (linear time) + t := uc.Tokens + copy(t[i:], t[i+1:]) // Shift t[i+1:] left one index + t[len(t)-1] = "" // Erase last element + uc.Tokens = t[:len(t)-1] // Truncate slice + + // Save changes + err = p.userCacheSaveWithLock(userID, *uc) + if err != nil { + return err + } + + log.Debugf("User cache del %v %v", userID, token) + + return nil } diff --git a/politeiad/backend/tlogbe/plugins/user/hooks.go b/politeiad/backend/tlogbe/plugins/user/hooks.go index 156e11ab3..3850901b6 100644 --- a/politeiad/backend/tlogbe/plugins/user/hooks.go +++ b/politeiad/backend/tlogbe/plugins/user/hooks.go @@ -332,16 +332,16 @@ func (p *userPlugin) hookSetRecordStatusPre(payload string) error { return nil } -func (p *userPlugin) hookSetRecordStatusPost(payload string) error { +func (p *userPlugin) hookSetRecordStatusPost(treeID int64, payload string) error { var srs plugins.HookSetRecordStatus err := json.Unmarshal([]byte(payload), &srs) if err != nil { return err } - // When a record becomes vetted the post hook will be executed on - // the vetted tstore instance. The record must be added to the - // vetted user cache. + // When a record is made public it is moved from the unvetted to + // the vetted tstore instance. The token must be removed from the + // unvetted user cache and added to the vetted user cache. if srs.RecordMetadata.Status == backend.MDStatusVetted { // Decode user metadata um, err := userMetadataDecode(srs.Metadata) @@ -349,10 +349,25 @@ func (p *userPlugin) hookSetRecordStatusPost(payload string) error { return err } - // Add token to the user cache - err = p.userCacheAddToken(um.UserID, srs.RecordMetadata.Token) - if err != nil { - return err + // Determine whether to add or remove this token from the user + // cache. When an unvetted record is made vetted, the record is + // considered to not exist in the unvetted tstore instance + // anymore. We can determine whether we need to add the token to + // the user cache or remove it by checking whether the record + // exists. The record will not exists in unvetted but will exist + // in vetted. + if p.tlog.RecordExists(treeID) { + // Add token to the user cache + err = p.userCacheAddToken(um.UserID, srs.RecordMetadata.Token) + if err != nil { + return err + } + } else { + // Del token from user cache + err = p.userCacheDelToken(um.UserID, srs.RecordMetadata.Token) + if err != nil { + return err + } } } diff --git a/politeiad/backend/tlogbe/plugins/user/user.go b/politeiad/backend/tlogbe/plugins/user/user.go index d0830612a..55586d3ff 100644 --- a/politeiad/backend/tlogbe/plugins/user/user.go +++ b/politeiad/backend/tlogbe/plugins/user/user.go @@ -71,7 +71,7 @@ func (p *userPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload s case plugins.HookTypeSetRecordStatusPre: return p.hookSetRecordStatusPre(payload) case plugins.HookTypeSetRecordStatusPost: - return p.hookSetRecordStatusPost(payload) + return p.hookSetRecordStatusPost(treeID, payload) } return nil diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index cdd072f23..19f29364d 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -1021,17 +1021,17 @@ func (t *tlogBackend) VettedExists(token []byte) bool { } // This function must be called WITH the record lock held. -func (t *tlogBackend) unvettedPublish(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { +func (t *tlogBackend) unvettedPublish(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) (int64, error) { // Create a vetted tree vettedTreeID, err := t.vetted.TreeNew() if err != nil { - return err + return 0, err } // Save the record to the vetted tlog err = t.vetted.RecordSave(vettedTreeID, rm, metadata, files) if err != nil { - return fmt.Errorf("vetted RecordSave: %v", err) + return 0, fmt.Errorf("vetted RecordSave: %v", err) } log.Debugf("Unvetted record %x copied to vetted tree %v", @@ -1041,7 +1041,7 @@ func (t *tlogBackend) unvettedPublish(token []byte, rm backend.RecordMetadata, m treeID := treeIDFromToken(token) err = t.unvetted.TreeFreeze(treeID, rm, metadata, vettedTreeID) if err != nil { - return fmt.Errorf("TreeFreeze %v: %v", treeID, err) + return 0, fmt.Errorf("TreeFreeze %v: %v", treeID, err) } log.Debugf("Unvetted record frozen %x", token) @@ -1049,7 +1049,7 @@ func (t *tlogBackend) unvettedPublish(token []byte, rm backend.RecordMetadata, m // Update the vetted cache t.vettedTreeIDAdd(hex.EncodeToString(token), vettedTreeID) - return nil + return vettedTreeID, nil } // This function must be called WITH the record lock held. @@ -1143,9 +1143,10 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, } // Update record + var vettedTreeID int64 switch status { case backend.MDStatusVetted: - err := t.unvettedPublish(token, rm, metadata, r.Files) + vettedTreeID, err = t.unvettedPublish(token, rm, metadata, r.Files) if err != nil { return nil, fmt.Errorf("unvettedPublish: %v", err) } @@ -1165,11 +1166,12 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, switch status { case backend.MDStatusVetted: - // Record was made public. Actions must now be executed on the - // vetted tlog instance. + // Record was made public. Call the plugin hook on both unvetted + // and vetted since both instances had state changes. + t.unvetted.PluginHookPost(treeID, token, + plugins.HookTypeSetRecordStatusPost, string(b)) - // Call post plugin hooks - t.vetted.PluginHookPost(treeID, token, + t.vetted.PluginHookPost(vettedTreeID, token, plugins.HookTypeSetRecordStatusPost, string(b)) // Update inventory cache From 610b017ac9ce63d995dfe38444f95d558f2f19f9 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 11 Feb 2021 13:14:12 -0600 Subject: [PATCH 298/449] Simplify user cache update. --- politeiad/backend/tlogbe/plugins/plugins.go | 3 --- .../backend/tlogbe/plugins/user/hooks.go | 25 +++++++++++-------- politeiad/backend/tlogbe/tlog/anchor.go | 4 ++- politeiad/backend/tlogbe/tlogbe.go | 3 ++- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/politeiad/backend/tlogbe/plugins/plugins.go b/politeiad/backend/tlogbe/plugins/plugins.go index a019de962..5022b860b 100644 --- a/politeiad/backend/tlogbe/plugins/plugins.go +++ b/politeiad/backend/tlogbe/plugins/plugins.go @@ -206,9 +206,6 @@ type TlogClient interface { // to the digest. Timestamp(treeID int64, digest []byte) (*backend.Timestamp, error) - // RecordExists returns whether a record exists. - RecordExists(treeID int64) bool - // Record returns a version of a record. Record(treeID int64, version uint32) (*backend.Record, error) diff --git a/politeiad/backend/tlogbe/plugins/user/hooks.go b/politeiad/backend/tlogbe/plugins/user/hooks.go index 3850901b6..f9570c714 100644 --- a/politeiad/backend/tlogbe/plugins/user/hooks.go +++ b/politeiad/backend/tlogbe/plugins/user/hooks.go @@ -350,24 +350,29 @@ func (p *userPlugin) hookSetRecordStatusPost(treeID int64, payload string) error } // Determine whether to add or remove this token from the user - // cache. When an unvetted record is made vetted, the record is - // considered to not exist in the unvetted tstore instance - // anymore. We can determine whether we need to add the token to - // the user cache or remove it by checking whether the record - // exists. The record will not exists in unvetted but will exist - // in vetted. - if p.tlog.RecordExists(treeID) { - // Add token to the user cache + // cache. The token will need to be removed from the unvetted + // tstore user cache and added to the vetted tstore user cache. + // We can determine what tstore instance this is by checking if + // the record exists. Once a record has been made vetted, the + // unvetted tstore instance will return a record not found error + // when queried. + _, err = p.tlog.RecordLatest(treeID) + switch err { + case nil: + // Record exists. Add token to the user cache. err = p.userCacheAddToken(um.UserID, srs.RecordMetadata.Token) if err != nil { return err } - } else { - // Del token from user cache + case backend.ErrRecordNotFound: + // Record does not exist. Del token from user cache. err = p.userCacheDelToken(um.UserID, srs.RecordMetadata.Token) if err != nil { return err } + default: + // Unexpected error + return fmt.Errorf("RecordLatest %v: %v", treeID, err) } } diff --git a/politeiad/backend/tlogbe/tlog/anchor.go b/politeiad/backend/tlogbe/tlog/anchor.go index b534b835c..3f967d889 100644 --- a/politeiad/backend/tlogbe/tlog/anchor.go +++ b/politeiad/backend/tlogbe/tlog/anchor.go @@ -30,7 +30,9 @@ const ( // currently drops an anchor on the hour mark so we submit new // anchors a few minutes prior to that. // Seconds Minutes Hours Days Months DayOfWeek - anchorSchedule = "0 56 * * * *" // At minute 56 of every hour + // TODO change back when done testing + // anchorSchedule = "0 56 * * * *" // At minute 56 of every hour + anchorSchedule = "0 */5 * * * *" // Every 5 minutes // anchorID is included in the timestamp and verify requests as a // unique identifier. diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 19f29364d..35665a368 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -1142,7 +1142,8 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, return nil, err } - // Update record + // Update record. When a record is made public it is moved from the + // unvetted instance to the vetted instance. var vettedTreeID int64 switch status { case backend.MDStatusVetted: From e59a434b12eaad535a970ac95eee1fa3dcd07ea1 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 11 Feb 2021 13:40:09 -0600 Subject: [PATCH 299/449] Cleanup. --- politeiad/backend/tlogbe/tlog/convert.go | 178 +++++++++++++++++++++++ politeiad/backend/tlogbe/tlog/tlog.go | 161 -------------------- 2 files changed, 178 insertions(+), 161 deletions(-) create mode 100644 politeiad/backend/tlogbe/tlog/convert.go diff --git a/politeiad/backend/tlogbe/tlog/convert.go b/politeiad/backend/tlogbe/tlog/convert.go new file mode 100644 index 000000000..1145b889c --- /dev/null +++ b/politeiad/backend/tlogbe/tlog/convert.go @@ -0,0 +1,178 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tlog + +import ( + "bytes" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/politeiad/backend/tlogbe/store" + "github.com/decred/politeia/util" +) + +func convertBlobEntryFromFile(f backend.File) (*store.BlobEntry, error) { + data, err := json.Marshal(f) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorFile, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertBlobEntryFromMetadataStream(ms backend.MetadataStream) (*store.BlobEntry, error) { + data, err := json.Marshal(ms) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorMetadataStream, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertBlobEntryFromRecordMetadata(rm backend.RecordMetadata) (*store.BlobEntry, error) { + data, err := json.Marshal(rm) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorRecordMetadata, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertBlobEntryFromRecordIndex(ri recordIndex) (*store.BlobEntry, error) { + data, err := json.Marshal(ri) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorRecordIndex, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertBlobEntryFromAnchor(a anchor) (*store.BlobEntry, error) { + data, err := json.Marshal(a) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorAnchor, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertRecordIndexFromBlobEntry(be store.BlobEntry) (*recordIndex, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorRecordIndex { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorRecordIndex) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, fmt.Errorf("decode digest: %v", err) + } + if !bytes.Equal(util.Digest(b), digest) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), digest) + } + var ri recordIndex + err = json.Unmarshal(b, &ri) + if err != nil { + return nil, fmt.Errorf("unmarshal recordIndex: %v", err) + } + + return &ri, nil +} + +func convertAnchorFromBlobEntry(be store.BlobEntry) (*anchor, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorAnchor { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorAnchor) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, fmt.Errorf("decode digest: %v", err) + } + if !bytes.Equal(util.Digest(b), digest) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), digest) + } + var a anchor + err = json.Unmarshal(b, &a) + if err != nil { + return nil, fmt.Errorf("unmarshal anchor: %v", err) + } + + return &a, nil +} diff --git a/politeiad/backend/tlogbe/tlog/tlog.go b/politeiad/backend/tlogbe/tlog/tlog.go index dffeb8614..adef65133 100644 --- a/politeiad/backend/tlogbe/tlog/tlog.go +++ b/politeiad/backend/tlogbe/tlog/tlog.go @@ -109,167 +109,6 @@ func extractKeyFromLeaf(l *trillian.LogLeaf) string { return string(s[1]) } -func convertBlobEntryFromFile(f backend.File) (*store.BlobEntry, error) { - data, err := json.Marshal(f) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorFile, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func convertBlobEntryFromMetadataStream(ms backend.MetadataStream) (*store.BlobEntry, error) { - data, err := json.Marshal(ms) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorMetadataStream, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func convertBlobEntryFromRecordMetadata(rm backend.RecordMetadata) (*store.BlobEntry, error) { - data, err := json.Marshal(rm) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorRecordMetadata, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func convertBlobEntryFromRecordIndex(ri recordIndex) (*store.BlobEntry, error) { - data, err := json.Marshal(ri) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorRecordIndex, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func convertBlobEntryFromAnchor(a anchor) (*store.BlobEntry, error) { - data, err := json.Marshal(a) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorAnchor, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func convertRecordIndexFromBlobEntry(be store.BlobEntry) (*recordIndex, error) { - // Decode and validate data hint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorRecordIndex { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorRecordIndex) - } - - // Decode data - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - digest, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, fmt.Errorf("decode digest: %v", err) - } - if !bytes.Equal(util.Digest(b), digest) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), digest) - } - var ri recordIndex - err = json.Unmarshal(b, &ri) - if err != nil { - return nil, fmt.Errorf("unmarshal recordIndex: %v", err) - } - - return &ri, nil -} - -func convertAnchorFromBlobEntry(be store.BlobEntry) (*anchor, error) { - // Decode and validate data hint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorAnchor { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorAnchor) - } - - // Decode data - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - digest, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, fmt.Errorf("decode digest: %v", err) - } - if !bytes.Equal(util.Digest(b), digest) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), digest) - } - var a anchor - err = json.Unmarshal(b, &a) - if err != nil { - return nil, fmt.Errorf("unmarshal anchor: %v", err) - } - - return &a, nil -} - func (t *Tlog) blobify(be store.BlobEntry) ([]byte, error) { b, err := store.Blobify(be) if err != nil { From e9f84b5e7879c751c1eec49f59109f9bf68a16a1 Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 12 Feb 2021 10:10:31 -0600 Subject: [PATCH 300/449] Fix unvetted files bug. --- politeiawww/pi/process.go | 16 ++++++++++++++++ politeiawww/records/process.go | 16 +++++++++------- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/politeiawww/pi/process.go b/politeiawww/pi/process.go index 9404a3847..99764e549 100644 --- a/politeiawww/pi/process.go +++ b/politeiawww/pi/process.go @@ -113,6 +113,22 @@ func (p *Pi) processProposals(ctx context.Context, ps v1.Proposals, u *user.User proposals[pr.CensorshipRecord.Token] = *pr } + // Only admins and the proposal author are allowed to retrieve + // unvetted files. Remove files if the user is not an admin or the + // author. This is a public route so a user may not be present. + if ps.State == v1.ProposalStateUnvetted { + for k, v := range proposals { + var ( + isAuthor = u != nil && u.ID.String() == v.UserID + isAdmin = u != nil && u.Admin + ) + if !isAuthor && !isAdmin { + v.Files = []v1.File{} + proposals[k] = v + } + } + } + return &v1.ProposalsReply{ Proposals: proposals, }, nil diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index 029cc65f5..e6a80f304 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -352,13 +352,15 @@ func (r *Records) processDetails(ctx context.Context, d v1.Details, u *user.User // unvetted record files. Remove files if the user is not an admin // or the author. This is a public route so a user may not be // present. - var ( - authorID = userIDFromMetadataStreams(rc.Metadata) - isAuthor = u != nil && u.ID.String() == authorID - isAdmin = u != nil && u.Admin - ) - if !isAuthor && !isAdmin { - rc.Files = []v1.File{} + if d.State == v1.RecordStateUnvetted { + var ( + authorID = userIDFromMetadataStreams(rc.Metadata) + isAuthor = u != nil && u.ID.String() == authorID + isAdmin = u != nil && u.Admin + ) + if !isAuthor && !isAdmin { + rc.Files = []v1.File{} + } } return &v1.DetailsReply{ From 0ce8b034c1f3aa68296be73541407228375efe17 Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 12 Feb 2021 12:17:07 -0600 Subject: [PATCH 301/449] Make record inv command scalable. --- politeiad/api/v1/v1.go | 41 +- politeiad/backend/backend.go | 22 +- politeiad/backend/gitbe/gitbe.go | 2 +- politeiad/backend/tlogbe/inventory.go | 450 ++++++++++++++---- .../tlogbe/plugins/ticketvote/inventory.go | 8 +- politeiad/backend/tlogbe/testing.go | 5 - politeiad/backend/tlogbe/tlog/testing.go | 4 + politeiad/backend/tlogbe/tlogbe.go | 93 +--- politeiad/client/politeiad.go | 5 +- politeiad/plugins/ticketvote/ticketvote.go | 5 - politeiad/politeiad.go | 4 +- politeiawww/api/records/v1/v1.go | 27 +- politeiawww/api/ticketvote/v1/v1.go | 2 +- politeiawww/cmd/pictl/cmdproposalinv.go | 71 ++- politeiawww/cmd/pictl/cmdproposalstatusset.go | 55 ++- politeiawww/proposals.go | 3 +- politeiawww/records/process.go | 26 +- politeiawww/records/records.go | 2 +- 18 files changed, 586 insertions(+), 239 deletions(-) diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index cab451b27..7b618520a 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -22,17 +22,17 @@ type RecordStatusT int const ( // Routes - IdentityRoute = "/v1/identity/" // Retrieve identity - NewRecordRoute = "/v1/newrecord/" // New record - UpdateUnvettedRoute = "/v1/updateunvetted/" // Update unvetted record - UpdateUnvettedMetadataRoute = "/v1/updateunvettedmd/" // Update unvetted metadata - UpdateVettedRoute = "/v1/updatevetted/" // Update vetted record - UpdateVettedMetadataRoute = "/v1/updatevettedmd/" // Update vetted metadata - GetUnvettedRoute = "/v1/getunvetted/" // Retrieve unvetted record - GetVettedRoute = "/v1/getvetted/" // Retrieve vetted record - GetUnvettedTimestampsRoute = "/v1/getunvettedts/" // Get unvetted timestamps - GetVettedTimestampsRoute = "/v1/getvettedts/" // Get vetted timestamps - InventoryByStatusRoute = "/v1/inventorybystatus/" // Get token inventory + IdentityRoute = "/v1/identity/" // Retrieve identity + NewRecordRoute = "/v1/newrecord/" // New record + UpdateUnvettedRoute = "/v1/updateunvetted/" // Update unvetted record + UpdateUnvettedMetadataRoute = "/v1/updateunvettedmd/" // Update unvetted metadata + UpdateVettedRoute = "/v1/updatevetted/" // Update vetted record + UpdateVettedMetadataRoute = "/v1/updatevettedmd/" // Update vetted metadata + GetUnvettedRoute = "/v1/getunvetted/" // Retrieve unvetted record + GetVettedRoute = "/v1/getvetted/" // Retrieve vetted record + GetUnvettedTimestampsRoute = "/v1/getunvettedts/" // Get unvetted timestamps + GetVettedTimestampsRoute = "/v1/getvettedts/" // Get vetted timestamps + InventoryByStatusRoute = "/v1/inventorybystatus/" // Auth required InventoryRoute = "/v1/inventory/" // Inventory records @@ -468,10 +468,23 @@ type InventoryReply struct { Branches []Record `json:"branches"` // Last N branches (censored, new etc) } -// InventoryByStatus requests for the censhorship tokens from all records -// filtered by their status. +const ( + // InventoryPageSize is the maximum number of tokens that will be + // returned for any single status in an inventory reply. + InventoryPageSize uint32 = 20 +) + +// InventoryByStatus requests the tokens of the records in the inventory, +// categorized by record state and record status. Each status will contain a +// list of tokens of size InventoryPageSize. +// +// The client can alternatively request the tokens for a specific status and +// page number. type InventoryByStatus struct { - Challenge string `json:"challenge"` // Random challenge + Challenge string `json:"challenge"` // Random challenge + State string `json:"state,omitempty"` + Status RecordStatusT `json:"status,omitempty"` + Page uint32 `json:"page,omitempty"` } // InventoryByStatusReply returns all censorship record tokens categorized by diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index f6399da28..61cfaf36b 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -210,9 +210,19 @@ type Plugin struct { Identity *identity.FullIdentity } -// InventoryByStatus contains the record tokens of all records in the inventory -// categorized by state and MDStatusT. Each list is sorted by the timestamp of -// the status change from newest to oldest. +const ( + // StateUnvetted is used to request the inventory of an unvetted + // status. + StateUnvetted = "unvetted" + + // StateVetted is used to request the inventory of a vetted status. + StateVetted = "vetted" +) + +// InventoryByStatus contains the tokens of the records in the inventory +// categorized by record state and record status. Each list contains a page of +// tokens that are sorted by the timestamp of the status change from newest to +// oldest. type InventoryByStatus struct { Unvetted map[MDStatusT][]string Vetted map[MDStatusT][]string @@ -270,9 +280,9 @@ type Backend interface { GetVettedTimestamps(token []byte, version string) (*RecordTimestamps, error) - // InventoryByStatus returns the record tokens of all records in the - // inventory categorized by MDStatusT - InventoryByStatus() (*InventoryByStatus, error) + // Get record tokens categorized by MDStatusT + InventoryByStatus(state string, s MDStatusT, + pageSize, page uint32) (*InventoryByStatus, error) // Register an unvetted plugin with the backend RegisterUnvettedPlugin(Plugin) error diff --git a/politeiad/backend/gitbe/gitbe.go b/politeiad/backend/gitbe/gitbe.go index aeebadde5..ad8395291 100644 --- a/politeiad/backend/gitbe/gitbe.go +++ b/politeiad/backend/gitbe/gitbe.go @@ -2843,7 +2843,7 @@ func (g *gitBackEnd) Plugin(pluginID, command, commandID, payload string) (strin // InventoryByStatus has not been not implemented. // // This function satisfies the backend.Backend interface. -func (g *gitBackEnd) InventoryByStatus() (*backend.InventoryByStatus, error) { +func (g *gitBackEnd) InventoryByStatus(state string, s backend.MDStatusT, pageSize, page uint32) (*backend.InventoryByStatus, error) { return nil, fmt.Errorf("not implemented") } diff --git a/politeiad/backend/tlogbe/inventory.go b/politeiad/backend/tlogbe/inventory.go index c54009530..a2e706598 100644 --- a/politeiad/backend/tlogbe/inventory.go +++ b/politeiad/backend/tlogbe/inventory.go @@ -6,151 +6,427 @@ package tlogbe import ( "encoding/hex" + "encoding/json" + "errors" "fmt" + "io/ioutil" + "os" + "path/filepath" "github.com/decred/politeia/politeiad/backend" ) const ( - // Record states - stateUnvetted = "unvetted" - stateVetted = "vetted" + filenameInvUnvetted = "inv-unvetted.json" + filenameInvVetted = "inv-vetted.json" ) -// inventory contains the tokens of all records in the inventory catagorized by -// MDStatusT. +type entry struct { + Token string `json:"token"` + Status backend.MDStatusT `json:"status"` +} + type inventory struct { - unvetted map[backend.MDStatusT][]string - vetted map[backend.MDStatusT][]string + Entries []entry `json:"entries"` } -func (t *tlogBackend) inventory() inventory { - t.RLock() - defer t.RUnlock() +func (t *tlogBackend) invPathUnvetted() string { + return filepath.Join(t.dataDir, filenameInvUnvetted) +} - // Return a copy of the inventory - var ( - unvetted = make(map[backend.MDStatusT][]string, len(t.inv.unvetted)) - vetted = make(map[backend.MDStatusT][]string, len(t.inv.vetted)) - ) - for status, tokens := range t.inv.unvetted { - s := make([]string, len(tokens)) - copy(s, tokens) - unvetted[status] = s +func (t *tlogBackend) invPathVetted() string { + return filepath.Join(t.dataDir, filenameInvVetted) +} + +// invGetLocked retrieves the inventory from disk. A new inventory is returned +// if one does not exist yet. +// +// This function must be called WITH the read lock held. +func (t *tlogBackend) invGetLocked(filePath string) (*inventory, error) { + b, err := ioutil.ReadFile(filePath) + if err != nil { + var e *os.PathError + if errors.As(err, &e) && !os.IsExist(err) { + // File does't exist. Return a new inventory. + return &inventory{ + Entries: make([]entry, 0, 1024), + }, nil + } + return nil, err } - for status, tokens := range t.inv.vetted { - s := make([]string, len(tokens)) - copy(s, tokens) - vetted[status] = s + + var inv inventory + err = json.Unmarshal(b, &inv) + if err != nil { + return nil, err } - return inventory{ - unvetted: unvetted, - vetted: vetted, + return &inv, nil +} + +// invGet retrieves the inventory from disk. A new inventory is returned if one +// does not exist yet. +// +// This function must be called WITHOUT the read lock held. +func (t *tlogBackend) invGet(filePath string) (*inventory, error) { + t.RLock() + defer t.RUnlock() + + return t.invGetLocked(filePath) +} + +// invSaveLocked writes the inventory to disk. +// +// This function must be called WITH the read/write lock held. +func (t *tlogBackend) invSaveLocked(filePath string, inv inventory) error { + b, err := json.Marshal(inv) + if err != nil { + return err } + return ioutil.WriteFile(filePath, b, 0664) } -func (t *tlogBackend) inventoryAdd(state string, tokenb []byte, s backend.MDStatusT) { +func (t *tlogBackend) invAdd(state string, token []byte, s backend.MDStatusT) error { t.Lock() defer t.Unlock() - token := hex.EncodeToString(tokenb) + // Get inventory file path + var fp string switch state { - case stateUnvetted: - t.inv.unvetted[s] = append([]string{token}, t.inv.unvetted[s]...) - case stateVetted: - t.inv.vetted[s] = append([]string{token}, t.inv.vetted[s]...) + case backend.StateUnvetted: + fp = t.invPathUnvetted() + case backend.StateVetted: + fp = t.invPathVetted() default: e := fmt.Sprintf("unknown state '%v'", state) panic(e) } - log.Debugf("Add to inv %v: %v %v", state, token, backend.MDStatus[s]) -} + // Get inventory + inv, err := t.invGetLocked(fp) + if err != nil { + return err + } + + // Prepend token + e := entry{ + Token: hex.EncodeToString(token), + Status: s, + } + inv.Entries = append([]entry{e}, inv.Entries...) + + // Save inventory + err = t.invSaveLocked(fp, *inv) + if err != nil { + return err + } -func (t *tlogBackend) inventoryUpdate(state string, tokenb []byte, currStatus, newStatus backend.MDStatusT) { - token := hex.EncodeToString(tokenb) + log.Debugf("Inv %v add %x %v", state, token, backend.MDStatus[s]) + return nil +} + +func (t *tlogBackend) invUpdate(state string, token []byte, s backend.MDStatusT) error { t.Lock() defer t.Unlock() - var inv map[backend.MDStatusT][]string + // Get inventory file path + var fp string switch state { - case stateUnvetted: - inv = t.inv.unvetted - case stateVetted: - inv = t.inv.vetted + case backend.StateUnvetted: + fp = t.invPathUnvetted() + case backend.StateVetted: + fp = t.invPathVetted() default: e := fmt.Sprintf("unknown state '%v'", state) panic(e) } - // Find the index of the token in its current status list - var idx int - var found bool - for k, v := range inv[currStatus] { - if v == token { - // Token found - idx = k - found = true - break - } + // Get inventory + inv, err := t.invGetLocked(fp) + if err != nil { + return err } - if !found { - // Token was never found. This should not happen. - e := fmt.Sprintf("inventoryUpdate: token not found: %v %v %v", - token, currStatus, newStatus) + + // Del entry + entries, err := entryDel(inv.Entries, token) + if err != nil { + // This should not happen. Panic if it does. + e := fmt.Sprintf("%v entry del: %v", state, err) panic(e) } - // Remove the token from its current status list - tokens := inv[currStatus] - inv[currStatus] = append(tokens[:idx], tokens[idx+1:]...) + // Prepend new entry to inventory + e := entry{ + Token: hex.EncodeToString(token), + Status: s, + } + inv.Entries = append([]entry{e}, entries...) + + // Save inventory + err = t.invSaveLocked(fp, *inv) + if err != nil { + return err + } - // Prepend token to new status - inv[newStatus] = append([]string{token}, inv[newStatus]...) + log.Debugf("Inv %v update %x to %v", state, token, backend.MDStatus[s]) - log.Debugf("Update inv %v: %v %v to %v", state, token, - backend.MDStatus[currStatus], backend.MDStatus[newStatus]) + return nil } -// inventoryMoveToVetted moves a token from the unvetted inventory to the -// vetted inventory. The unvettedStatus is the status of the record prior to -// the update and the vettedStatus is the status of the record after the -// update. -func (t *tlogBackend) inventoryMoveToVetted(tokenb []byte, unvettedStatus, vettedStatus backend.MDStatusT) { +// invMoveToVetted moves a token from the unvetted inventory to the vetted +// inventory. +func (t *tlogBackend) invMoveToVetted(token []byte, s backend.MDStatusT) error { t.Lock() defer t.Unlock() - token := hex.EncodeToString(tokenb) - unvetted := t.inv.unvetted - vetted := t.inv.vetted + // Get unvetted inventory + upath := t.invPathUnvetted() + u, err := t.invGetLocked(upath) + if err != nil { + return err + } + + // Del entry + u.Entries, err = entryDel(u.Entries, token) + if err != nil { + // This should not happen. Panic if it does. + e := fmt.Sprintf("unvetted entry del: %v", err) + panic(e) + } + + // Save unvetted inventory + err = t.invSaveLocked(upath, *u) + if err != nil { + return err + } + + // Get vetted inventory + vpath := t.invPathVetted() + v, err := t.invGetLocked(vpath) + if err != nil { + return err + } + + // Prepend new entry to inventory + e := entry{ + Token: hex.EncodeToString(token), + Status: s, + } + v.Entries = append([]entry{e}, v.Entries...) + + // Save vetted inventory + err = t.invSaveLocked(vpath, *v) + if err != nil { + return err + } + + log.Debugf("Inv move %x from unvetted to vetted status %v", + token, backend.MDStatus[s]) + + return nil +} + +// invByStatus contains the inventory categorized by record state and record +// status. Each list contains a page of tokens that are sorted by the timestamp +// of the status change from newest to oldest. +type invByStatus struct { + Unvetted map[backend.MDStatusT][]string + Vetted map[backend.MDStatusT][]string +} + +func (t *tlogBackend) invByStatusAll(pageSize uint32) (*invByStatus, error) { + // Get unvetted inventory + u, err := t.invGet(t.invPathUnvetted()) + if err != nil { + return nil, err + } + + // Prepare unvetted inventory reply + var ( + unvetted = tokensParse(u.Entries, backend.MDStatusUnvetted, pageSize, 1) + censored = tokensParse(u.Entries, backend.MDStatusCensored, pageSize, 1) + archived = tokensParse(u.Entries, backend.MDStatusArchived, pageSize, 1) + + unvettedInv = make(map[backend.MDStatusT][]string, 16) + ) + if len(unvetted) != 0 { + unvettedInv[backend.MDStatusUnvetted] = unvetted + } + if len(censored) != 0 { + unvettedInv[backend.MDStatusCensored] = censored + } + if len(archived) != 0 { + unvettedInv[backend.MDStatusArchived] = archived + } + + // Get vetted inventory + v, err := t.invGet(t.invPathVetted()) + if err != nil { + return nil, err + } + + // Prepare vetted inventory reply + var ( + vetted = tokensParse(v.Entries, backend.MDStatusVetted, pageSize, 1) + vcensored = tokensParse(v.Entries, backend.MDStatusCensored, pageSize, 1) + varchived = tokensParse(v.Entries, backend.MDStatusArchived, pageSize, 1) + + vettedInv = make(map[backend.MDStatusT][]string, 16) + ) + if len(vetted) != 0 { + vettedInv[backend.MDStatusVetted] = vetted + } + if len(vcensored) != 0 { + vettedInv[backend.MDStatusCensored] = vcensored + } + if len(varchived) != 0 { + vettedInv[backend.MDStatusArchived] = varchived + } + + return &invByStatus{ + Unvetted: unvettedInv, + Vetted: vettedInv, + }, nil +} + +// inventoryAdd is a wrapper around the invAdd method that allows us to decide +// how disk read/write errors should be handled. For now we simply panic. The +// best thing to do would be to kick off a non-block fsck job that checks the +// inventory cache and corrects any mistakes that it finds. +func (t *tlogBackend) inventoryAdd(state string, token []byte, s backend.MDStatusT) { + err := t.invAdd(state, token, s) + if err != nil { + e := fmt.Sprintf("invAdd %v %x %v: %v", state, token, s, err) + panic(e) + } +} + +// inventoryUpdate is a wrapper around the invUpdate method that allows us to +// decide how disk read/write errors should be handled. For now we simply +// panic. The best thing to do would be to kick off a non-block fsck job that +// checks the inventory cache and corrects any mistakes that it finds. +func (t *tlogBackend) inventoryUpdate(state string, token []byte, s backend.MDStatusT) { + err := t.invUpdate(state, token, s) + if err != nil { + e := fmt.Sprintf("invUpdate %v %x %v: %v", state, token, s, err) + panic(e) + } +} + +// inventoryMoveToVetted is a wrapper around the invMoveToVetted method that +// allows us to decide how disk read/write errors should be handled. For now we +// simply panic. The best thing to do would be to kick off a non-block fsck job +// that checks the inventory cache and corrects any mistakes that it finds. +func (t *tlogBackend) inventoryMoveToVetted(token []byte, s backend.MDStatusT) { + err := t.invMoveToVetted(token, s) + if err != nil { + e := fmt.Sprintf("invMoveToVetted %x %v: %v", token, s, err) + panic(e) + } +} + +func (t *tlogBackend) inventory(state string, s backend.MDStatusT, pageSize, page uint32) (*invByStatus, error) { + // If no status is provided a page of tokens for each status should + // be returned. + if s == backend.MDStatusInvalid { + return t.invByStatusAll(pageSize) + } + + // Get inventory file path + var fp string + switch state { + case backend.StateUnvetted: + fp = t.invPathUnvetted() + case backend.StateVetted: + fp = t.invPathVetted() + default: + e := fmt.Sprintf("unknown state '%v'", state) + panic(e) + } + + // Get inventory + inv, err := t.invGet(fp) + if err != nil { + return nil, err + } + + // Get the page of tokens + tokens := tokensParse(inv.Entries, s, pageSize, page) + + // Prepare reply + var ibs invByStatus + switch state { + case backend.StateUnvetted: + ibs = invByStatus{ + Unvetted: map[backend.MDStatusT][]string{ + s: tokens, + }, + Vetted: map[backend.MDStatusT][]string{}, + } + case backend.StateVetted: + ibs = invByStatus{ + Unvetted: map[backend.MDStatusT][]string{}, + Vetted: map[backend.MDStatusT][]string{ + s: tokens, + }, + } + } + + return &ibs, nil +} - // Find the index of the token in its current status list - var idx int +// entryDel removes the entry for the token and returns the updated slice. +func entryDel(entries []entry, token []byte) ([]entry, error) { + // Find token in entries + var i int var found bool - for k, v := range unvetted[unvettedStatus] { - if v == token { - // Token found - idx = k + htoken := hex.EncodeToString(token) + for k, v := range entries { + if v.Token == htoken { + i = k found = true break } } if !found { - // Token was never found. This should not happen. - e := fmt.Sprintf("inventoryMoveToVetted: unvetted token not found: %v %v", - token, unvettedStatus) - panic(e) + return nil, fmt.Errorf("token not found %x", token) + } + + // Del token from entries (linear time) + copy(entries[i:], entries[i+1:]) // Shift entries[i+1:] left one index + entries[len(entries)-1] = entry{} // Del last element (write zero value) + entries = entries[:len(entries)-1] // Truncate slice + + return entries, nil +} + +// tokensParse parses a page of tokens from the provided entries. +func tokensParse(entries []entry, s backend.MDStatusT, countPerPage, page uint32) []string { + tokens := make([]string, 0, countPerPage) + if countPerPage == 0 || page == 0 { + return tokens } - // Remove the token from the unvetted status list - tokens := unvetted[unvettedStatus] - unvetted[unvettedStatus] = append(tokens[:idx], tokens[idx+1:]...) + startAt := (page - 1) * countPerPage + var foundCount uint32 + for _, v := range entries { + if v.Status != s { + // Status does not match + continue + } - // Prepend token to vetted status - vetted[vettedStatus] = append([]string{token}, vetted[vettedStatus]...) + // Matching status found + if foundCount >= startAt { + tokens = append(tokens, v.Token) + if len(tokens) == int(countPerPage) { + // We got a full page. We're done. + return tokens + } + } + + foundCount++ + } - log.Debugf("Inv move to vetted: %v %v to %v", token, - backend.MDStatus[unvettedStatus], backend.MDStatus[vettedStatus]) + return tokens } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go b/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go index 946210438..317fc79a2 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go @@ -81,11 +81,7 @@ func (p *ticketVotePlugin) invSetLocked(inv inventory) error { if err != nil { return err } - err = ioutil.WriteFile(p.invPath(), b, 0664) - if err != nil { - return err - } - return nil + return ioutil.WriteFile(p.invPath(), b, 0664) } // invAddToUnauthorized adds the token to the unauthorized vote status list. @@ -347,7 +343,7 @@ func invDel(inv map[string][]string, status, token string) bool { // Remove token (linear time) copy(list[i:], list[i+1:]) // Shift list[i+1:] left one index - list[len(list)-1] = "" // Erase last element (write zero token) + list[len(list)-1] = "" // Del last element (write zero value) list = list[:len(list)-1] // Truncate slice // Update inv diff --git a/politeiad/backend/tlogbe/testing.go b/politeiad/backend/tlogbe/testing.go index b08e13c83..3d177f3fb 100644 --- a/politeiad/backend/tlogbe/testing.go +++ b/politeiad/backend/tlogbe/testing.go @@ -10,7 +10,6 @@ import ( "path/filepath" "testing" - "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" ) @@ -33,10 +32,6 @@ func NewTestTlogBackend(t *testing.T) (*tlogBackend, func()) { vetted: tlog.NewTestTlogEncrypted(t, dataDir, "vetted"), prefixes: make(map[string][]byte), vettedTreeIDs: make(map[string]int64), - inv: inventory{ - unvetted: make(map[backend.MDStatusT][]string), - vetted: make(map[backend.MDStatusT][]string), - }, } return &tlogBackend, func() { diff --git a/politeiad/backend/tlogbe/tlog/testing.go b/politeiad/backend/tlogbe/tlog/testing.go index a46bd8e64..414ac9807 100644 --- a/politeiad/backend/tlogbe/tlog/testing.go +++ b/politeiad/backend/tlogbe/tlog/testing.go @@ -47,10 +47,14 @@ func newTestTlog(t *testing.T, tlogID, dataDir string, encrypt bool) *Tlog { } } +// NewTestTlogEncrypted returns a tlog instance that encrypts all data blobs +// and that has been setup for testing. func NewTestTlogEncrypted(t *testing.T, tlogID, dataDir string) *Tlog { return newTestTlog(t, tlogID, dataDir, true) } +// NewTestTlogUnencrypted returns a tlog instance that save all data blobs +// as plaintext and that has been setup for testing. func NewTestTlogUnencrypted(t *testing.T, tlogID, dataDir string) *Tlog { return newTestTlog(t, tlogID, dataDir, false) } diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 35665a368..2e5d27efb 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -30,9 +30,6 @@ import ( "github.com/subosito/gozaru" ) -// TODO testnet vs mainnet trillian databases -// TODO fsck - const ( defaultEncryptionKeyFilename = "tlogbe.key" @@ -70,20 +67,19 @@ type tlogBackend struct { // This cache memoizes those results and is lazy loaded. vettedTreeIDs map[string]int64 // [token]treeID - // inventory contains the full record inventory grouped by record - // status. Each list of tokens is sorted by the timestamp of the - // status change from newest to oldest. This cache is built on - // startup. + // inv contains the full record inventory grouped by record status. + // Each list of tokens is sorted by the timestamp of the status + // change from newest to oldest. This cache is built on startup. inv inventory // recordMtxs allows a the backend to hold a lock on a record so // that it can perform multiple read/write operations in a - // concurrency safe manner. These mutexes are lazy loaded. + // concurrent safe manner. These mutexes are lazy loaded. recordMtxs map[string]*sync.Mutex } func tokenFromTreeID(treeID int64) []byte { - b := make([]byte, v1.TokenSizeTlog) + b := make([]byte, 8) // Converting between int64 and uint64 doesn't change // the sign bit, only the way it's interpreted. binary.LittleEndian.PutUint64(b, uint64(treeID)) @@ -599,7 +595,7 @@ func (t *tlogBackend) New(metadata []backend.MetadataStream, files []backend.Fil plugins.HookTypeNewRecordPost, string(b)) // Update the inventory cache - t.inventoryAdd(stateUnvetted, token, backend.MDStatusUnvetted) + t.inventoryAdd(backend.StateUnvetted, token, backend.MDStatusUnvetted) return rm, nil } @@ -1176,7 +1172,7 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, plugins.HookTypeSetRecordStatusPost, string(b)) // Update inventory cache - t.inventoryMoveToVetted(token, currStatus, status) + t.inventoryMoveToVetted(token, status) // Retrieve record r, err = t.GetVetted(token, "") @@ -1192,7 +1188,7 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, plugins.HookTypeSetRecordStatusPost, string(b)) // Update inventory cache - t.inventoryUpdate(stateUnvetted, token, currStatus, status) + t.inventoryUpdate(backend.StateUnvetted, token, status) // Retrieve record r, err = t.GetUnvetted(token, "") @@ -1338,7 +1334,7 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md plugins.HookTypeSetRecordStatusPost, string(b)) // Update inventory cache - t.inventoryUpdate(stateVetted, token, currStatus, status) + t.inventoryUpdate(backend.StateVetted, token, status) log.Debugf("Status change %x from %v (%v) to %v (%v)", token, backend.MDStatus[currStatus], currStatus, @@ -1476,13 +1472,17 @@ func (t *tlogBackend) GetVettedTimestamps(token []byte, version string) (*backen // categorized by MDStatusT. // // This function satisfies the backend.Backend interface. -func (t *tlogBackend) InventoryByStatus() (*backend.InventoryByStatus, error) { - log.Tracef("InventoryByStatus") +func (t *tlogBackend) InventoryByStatus(state string, status backend.MDStatusT, pageSize, page uint32) (*backend.InventoryByStatus, error) { + log.Tracef("InventoryByStatus: %v %v %v %v", state, status, pageSize, page) + + inv, err := t.inventory(state, status, pageSize, page) + if err != nil { + return nil, err + } - inv := t.inventory() return &backend.InventoryByStatus{ - Unvetted: inv.unvetted, - Vetted: inv.vetted, + Unvetted: inv.Unvetted, + Vetted: inv.Vetted, }, nil } @@ -1742,62 +1742,15 @@ func (t *tlogBackend) Close() { func (t *tlogBackend) setup() error { log.Tracef("setup") - // Get all trees + log.Infof("Building backend token prefix cache") + treeIDs, err := t.unvetted.TreesAll() if err != nil { return fmt.Errorf("unvetted TreesAll: %v", err) } - - log.Infof("Building backend caches") - - // Build all memory caches for _, v := range treeIDs { token := tokenFromTreeID(v) - - log.Debugf("Building memory caches for %x", token) - - // Add tree to prefixes cache t.prefixAdd(token) - - // Identify whether the record is unvetted or vetted. - isUnvetted := t.UnvettedExists(token) - isVetted := t.VettedExists(token) - - // Get the record - var r *backend.Record - switch { - case isUnvetted && isVetted: - // Sanity check - e := fmt.Sprintf("records is both unvetted and vetted: %x", token) - panic(e) - - case isUnvetted: - // Get unvetted record - r, err = t.GetUnvetted(token, "") - if err != nil { - return fmt.Errorf("GetUnvetted %x: %v", token, err) - } - - // Add record to the inventory cache - t.inventoryAdd(stateUnvetted, token, r.RecordMetadata.Status) - - case isVetted: - // Get vetted record - r, err = t.GetVetted(token, "") - if err != nil { - return fmt.Errorf("GetUnvetted %x: %v", token, err) - } - - // Add record to the inventory cache - t.inventoryAdd(stateVetted, token, r.RecordMetadata.Status) - - default: - // This is an empty tree. This can happen if there was an error - // during record creation and the record failed to be appended - // to the tree. - log.Debugf("Empty tree found for token %x", token) - } - } return nil @@ -1854,11 +1807,7 @@ func New(anp *chaincfg.Params, homeDir, dataDir, unvettedTrillianHost, unvettedT vetted: vetted, prefixes: make(map[string][]byte), vettedTreeIDs: make(map[string]int64), - inv: inventory{ - unvetted: make(map[backend.MDStatusT][]string), - vetted: make(map[backend.MDStatusT][]string), - }, - recordMtxs: make(map[string]*sync.Mutex), + recordMtxs: make(map[string]*sync.Mutex), } err = t.setup() diff --git a/politeiad/client/politeiad.go b/politeiad/client/politeiad.go index e597ac501..e0faf2a2e 100644 --- a/politeiad/client/politeiad.go +++ b/politeiad/client/politeiad.go @@ -409,7 +409,7 @@ func (c *Client) Inventory(ctx context.Context) (*pdv1.InventoryReply, error) { } // InventoryByStatus sends a InventoryByStatus request to the politeiad v1 API. -func (c *Client) InventoryByStatus(ctx context.Context) (*pdv1.InventoryByStatusReply, error) { +func (c *Client) InventoryByStatus(ctx context.Context, state string, status pdv1.RecordStatusT, page uint32) (*pdv1.InventoryByStatusReply, error) { // Setup request challenge, err := util.Random(pdv1.ChallengeSize) if err != nil { @@ -417,6 +417,9 @@ func (c *Client) InventoryByStatus(ctx context.Context) (*pdv1.InventoryByStatus } ibs := pdv1.InventoryByStatus{ Challenge: hex.EncodeToString(challenge), + State: state, + Status: status, + Page: page, } // Send request diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index b077d8017..34a5110fb 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -7,11 +7,6 @@ package ticketvote // TODO should VoteDetails, StartReply, StartRunoffReply contain a receipt? -// TODO the timestamps reply is going to be too large. Each ticket vote -// timestamp is ~2000 bytes. -// Avg (15k votes): 30MB -// Large (25k votes): 50MB -// Max (41k votes): 82MB const ( // PluginID is the ticketvote plugin ID. diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index de3565a14..1d257d68a 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -788,7 +788,9 @@ func (p *politeia) inventoryByStatus(w http.ResponseWriter, r *http.Request) { return } - inv, err := p.backend.InventoryByStatus() + s := convertFrontendStatus(ibs.Status) + inv, err := p.backend.InventoryByStatus(ibs.State, s, + v1.InventoryPageSize, ibs.Page) if err != nil { // Generic internal error. errorCode := time.Now().Unix() diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go index 33eeea90c..7a357baa5 100644 --- a/politeiawww/api/records/v1/v1.go +++ b/politeiawww/api/records/v1/v1.go @@ -313,22 +313,31 @@ type RecordsReply struct { } const ( - // TODO implement Inventory pagnation - InventoryPageSize = 60 + // InventoryPageSize is the maximum number of tokens that will be + // returned for any single status in an inventory reply. + InventoryPageSize uint32 = 20 ) -// Inventory requests the tokens of all records in the inventory, categorized -// by record state and record status. Unvetted record tokens will only be -// returned to admins. +// Inventory requests the tokens of the records in the inventory, categorized +// by record state and record status. The tokens are ordered by the timestamp +// of their most recent status change, sorted from newest to oldest. +// +// The state, status, and page arguments can be provided to request a specific +// page of record tokens. +// +// If no status is provided then a page of tokens for all statuses are +// returned. The state and page arguments will be ignored. +// +// Unvetted record tokens will only be returned to admins. type Inventory struct { - State string `json:"state,omitempty"` - Status string `json:"status,omitempty"` - Page int32 `json:"page,omitempty"` + State string `json:"state,omitempty"` + Status RecordStatusT `json:"status,omitempty"` + Page uint32 `json:"page,omitempty"` } // InventoryReply is the reply to the Inventory command. The returned maps are // map[status][]token where the status is the human readable record status -// defined by the Statuses array in this package. +// defined by the RecordStatuses array in this package. type InventoryReply struct { Unvetted map[string][]string `json:"unvetted"` Vetted map[string][]string `json:"vetted"` diff --git a/politeiawww/api/ticketvote/v1/v1.go b/politeiawww/api/ticketvote/v1/v1.go index 6f7437d4b..691fb50c8 100644 --- a/politeiawww/api/ticketvote/v1/v1.go +++ b/politeiawww/api/ticketvote/v1/v1.go @@ -496,7 +496,7 @@ type SummariesReply struct { // Submissions requests the submissions of a runoff vote. The only records that // will have a submissions list are the parent records in a runoff vote. The // list will contain all public runoff vote submissions, i.e. records that -// have linked to the parent record using the VoteMetadata.LinkTo field. +// have linked to the parent record using the VoteMetadata LinkTo field. type Submissions struct { Token string `json:"token"` } diff --git a/politeiawww/cmd/pictl/cmdproposalinv.go b/politeiawww/cmd/pictl/cmdproposalinv.go index d4c1711af..cf5320f5c 100644 --- a/politeiawww/cmd/pictl/cmdproposalinv.go +++ b/politeiawww/cmd/pictl/cmdproposalinv.go @@ -11,7 +11,19 @@ import ( // cmdProposalInv retrieves the censorship record tokens of all proposals in // the inventory, categorized by status. -type cmdProposalInv struct{} +type cmdProposalInv struct { + Args struct { + Status string `positional-arg-name:"status"` + Page uint32 `positional-arg-name:"page"` + } `positional-args:"true" optional:"true"` + + // Unvetted is used to indicate the state that should be sent in + // the inventory request. This flag is only required when + // requesting the inventory for a specific status. If a status + // argument is provided and this flag is not, it will be assumed + // that the state being requested is vetted. + Unvetted bool `long:"unvetted" optional:"true"` +} // Execute executes the cmdProposalInv command. // @@ -30,8 +42,39 @@ func (c *cmdProposalInv) Execute(args []string) error { return err } + // Setup state + var state string + switch { + case c.Unvetted: + state = rcv1.RecordStateUnvetted + default: + state = rcv1.RecordStateVetted + } + + // Setup status and page number + var status rcv1.RecordStatusT + if c.Args.Status != "" { + // Parse status. This can be either the numeric status code or the + // human readable equivalent. + status, err = parseStatus(c.Args.Status) + if err != nil { + return err + } + + // If a status was given but no page number was give, default + // to page number 1. + if c.Args.Page == 0 { + c.Args.Page = 1 + } + } + // Get inventory - ir, err := pc.RecordInventory(rcv1.Inventory{}) + i := rcv1.Inventory{ + State: state, + Status: status, + Page: c.Args.Page, + } + ir, err := pc.RecordInventory(i) if err != nil { return err } @@ -45,5 +88,25 @@ func (c *cmdProposalInv) Execute(args []string) error { // proposalInvHelpMsg is printed to stdout by the help command. const proposalInvHelpMsg = `proposalinv -Retrieve the censorship record tokens of all proposals in the inventory, -categorized by status. Unvetted proposals are only returned to admins.` +Inventory returns the tokens of the records in the inventory, categorized by +record state and record status. The tokens are ordered by the timestamp of +their most recent status change, sorted from newest to oldest. + +The status and page arguments can be provided to request a specific page of +record tokens. + +If no status is specified then a page of tokens for each status are returned. +The state and page arguments will be ignored. + +Valid statuses: + public + censored + abandoned + +Arguments: +1. status (string, optional) Status of tokens being requested. +2. page (uint32, optional) Page number. + +Flags: + --unvetted (bool, optional) Set status of an unvetted record. +` diff --git a/politeiawww/cmd/pictl/cmdproposalstatusset.go b/politeiawww/cmd/pictl/cmdproposalstatusset.go index 8dc134d6a..c7c1775a3 100644 --- a/politeiawww/cmd/pictl/cmdproposalstatusset.go +++ b/politeiawww/cmd/pictl/cmdproposalstatusset.go @@ -63,28 +63,9 @@ func (c *cmdProposalSetStatus) Execute(args []string) error { // Parse status. This can be either the numeric status code or the // human readable equivalent. - var ( - status rcv1.RecordStatusT - - statuses = map[string]rcv1.RecordStatusT{ - "public": rcv1.RecordStatusPublic, - "censored": rcv1.RecordStatusCensored, - "abandoned": rcv1.RecordStatusArchived, - "2": rcv1.RecordStatusPublic, - "3": rcv1.RecordStatusCensored, - "4": rcv1.RecordStatusArchived, - } - ) - s, err := strconv.ParseUint(c.Args.Status, 10, 32) - if err == nil { - // Numeric status code found - status = rcv1.RecordStatusT(s) - } else if s, ok := statuses[c.Args.Status]; ok { - // Human readable status code found - status = s - } else { - return fmt.Errorf("invalid proposal status '%v'\n %v", - c.Args.Status, proposalSetStatusHelpMsg) + status, err := parseStatus(c.Args.Status) + if err != nil { + return err } // Setup version @@ -142,6 +123,36 @@ func (c *cmdProposalSetStatus) Execute(args []string) error { return nil } +func parseStatus(status string) (rcv1.RecordStatusT, error) { + // Parse status. This can be either the numeric status code or the + // human readable equivalent. + var ( + rc rcv1.RecordStatusT + + statuses = map[string]rcv1.RecordStatusT{ + "public": rcv1.RecordStatusPublic, + "censored": rcv1.RecordStatusCensored, + "abandoned": rcv1.RecordStatusArchived, + "archived": rcv1.RecordStatusArchived, + "2": rcv1.RecordStatusPublic, + "3": rcv1.RecordStatusCensored, + "4": rcv1.RecordStatusArchived, + } + ) + u, err := strconv.ParseUint(status, 10, 32) + if err == nil { + // Numeric status code found + rc = rcv1.RecordStatusT(u) + } else if s, ok := statuses[status]; ok { + // Human readable status code found + rc = s + } else { + return rc, fmt.Errorf("invalid status '%v'", status) + } + + return rc, nil +} + // proposalSetStatusHelpMsg is printed to stdout by the help command. const proposalSetStatusHelpMsg = `proposalstatusset "token" "status" "reason" diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 6f69500a0..86c128884 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -28,7 +28,8 @@ func (p *politeiawww) processTokenInventory(ctx context.Context, isAdmin bool) ( log.Tracef("processTokenInventory") // Get record inventory - ir, err := p.politeiad.InventoryByStatus(ctx) + ir, err := p.politeiad.InventoryByStatus(ctx, "", + pdv1.RecordStatusInvalid, 0) if err != nil { return nil, err } diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index e6a80f304..db9c13076 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -413,10 +413,30 @@ func (r *Records) processRecords(ctx context.Context, rs v1.Records, u *user.Use }, nil } -func (r *Records) processInventory(ctx context.Context, u *user.User) (*v1.InventoryReply, error) { - log.Tracef("processInventory") +func (r *Records) processInventory(ctx context.Context, i v1.Inventory, u *user.User) (*v1.InventoryReply, error) { + log.Tracef("processInventory: %v %v %v", i.State, i.Status, i.Page) - ir, err := r.politeiad.InventoryByStatus(ctx) + // Verify state + switch i.State { + case v1.RecordStateUnvetted, v1.RecordStateVetted: + // Allowed; continue + default: + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeRecordStateInvalid, + } + } + + // Verify status. The status is optional so only validate it if one + // was provided. + s := convertStatusToPD(i.Status) + if i.Status != v1.RecordStatusInvalid && s == pdv1.RecordStatusInvalid { + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeRecordStatusInvalid, + } + } + + // Get inventory + ir, err := r.politeiad.InventoryByStatus(ctx, i.State, s, i.Page) if err != nil { return nil, err } diff --git a/politeiawww/records/records.go b/politeiawww/records/records.go index 24731f47e..2223408c8 100644 --- a/politeiawww/records/records.go +++ b/politeiawww/records/records.go @@ -208,7 +208,7 @@ func (c *Records) HandleInventory(w http.ResponseWriter, r *http.Request) { return } - ir, err := c.processInventory(r.Context(), u) + ir, err := c.processInventory(r.Context(), i, u) if err != nil { respondWithError(w, r, "HandleInventory: processInventory: %v", err) From 766c932c76ecf4e2732962590692071b097aa2e8 Mon Sep 17 00:00:00 2001 From: luke Date: Sat, 13 Feb 2021 18:22:25 -0600 Subject: [PATCH 302/449] Make vote inv scalable. --- politeiad/api/v1/v1.go | 15 +- politeiad/backend/tlogbe/inventory.go | 69 ++- .../backend/tlogbe/plugins/ticketvote/cmds.go | 43 +- .../tlogbe/plugins/ticketvote/hooks.go | 9 +- .../tlogbe/plugins/ticketvote/inventory.go | 450 ++++++++++-------- .../tlogbe/plugins/ticketvote/ticketvote.go | 23 +- politeiad/backend/tlogbe/tlogbe.go | 7 +- politeiad/client/ticketvote.go | 8 +- politeiad/plugins/ticketvote/ticketvote.go | 31 +- politeiawww/api/ticketvote/v1/v1.go | 30 +- politeiawww/client/ticketvote.go | 4 +- politeiawww/cmd/pictl/cmdproposalinv.go | 2 +- politeiawww/cmd/pictl/cmdproposalstatusset.go | 4 +- politeiawww/cmd/pictl/cmdvoteinv.go | 97 +++- politeiawww/proposals.go | 4 +- politeiawww/ticketvote/error.go | 2 +- politeiawww/ticketvote/process.go | 31 +- politeiawww/ticketvote/ticketvote.go | 12 +- 18 files changed, 523 insertions(+), 318 deletions(-) diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index 7b618520a..b2547eef0 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -470,16 +470,19 @@ type InventoryReply struct { const ( // InventoryPageSize is the maximum number of tokens that will be - // returned for any single status in an inventory reply. + // returned for any single status in an InventoryReply. InventoryPageSize uint32 = 20 ) -// InventoryByStatus requests the tokens of the records in the inventory, -// categorized by record state and record status. Each status will contain a -// list of tokens of size InventoryPageSize. +// Inventory requests the tokens of the records in the inventory, categorized +// by record state and record status. The tokens are ordered by the timestamp +// of their most recent status change, sorted from newest to oldest. // -// The client can alternatively request the tokens for a specific status and -// page number. +// The state, status, and page arguments can be provided to request a specific +// page of record tokens. +// +// If no status is provided then a page of tokens for all statuses will be +// returned. The page argument will be ignored. type InventoryByStatus struct { Challenge string `json:"challenge"` // Random challenge State string `json:"state,omitempty"` diff --git a/politeiad/backend/tlogbe/inventory.go b/politeiad/backend/tlogbe/inventory.go index a2e706598..48151f48e 100644 --- a/politeiad/backend/tlogbe/inventory.go +++ b/politeiad/backend/tlogbe/inventory.go @@ -227,6 +227,37 @@ func (t *tlogBackend) invMoveToVetted(token []byte, s backend.MDStatusT) error { return nil } +// inventoryAdd is a wrapper around the invAdd method that allows us to decide +// how disk read/write errors should be handled. For now we just panic. +func (t *tlogBackend) inventoryAdd(state string, token []byte, s backend.MDStatusT) { + err := t.invAdd(state, token, s) + if err != nil { + e := fmt.Sprintf("invAdd %v %x %v: %v", state, token, s, err) + panic(e) + } +} + +// inventoryUpdate is a wrapper around the invUpdate method that allows us to +// decide how disk read/write errors should be handled. For now we just panic. +func (t *tlogBackend) inventoryUpdate(state string, token []byte, s backend.MDStatusT) { + err := t.invUpdate(state, token, s) + if err != nil { + e := fmt.Sprintf("invUpdate %v %x %v: %v", state, token, s, err) + panic(e) + } +} + +// inventoryMoveToVetted is a wrapper around the invMoveToVetted method that +// allows us to decide how disk read/write errors should be handled. For now we +// just panic. +func (t *tlogBackend) inventoryMoveToVetted(token []byte, s backend.MDStatusT) { + err := t.invMoveToVetted(token, s) + if err != nil { + e := fmt.Sprintf("invMoveToVetted %x %v: %v", token, s, err) + panic(e) + } +} + // invByStatus contains the inventory categorized by record state and record // status. Each list contains a page of tokens that are sorted by the timestamp // of the status change from newest to oldest. @@ -290,43 +321,7 @@ func (t *tlogBackend) invByStatusAll(pageSize uint32) (*invByStatus, error) { }, nil } -// inventoryAdd is a wrapper around the invAdd method that allows us to decide -// how disk read/write errors should be handled. For now we simply panic. The -// best thing to do would be to kick off a non-block fsck job that checks the -// inventory cache and corrects any mistakes that it finds. -func (t *tlogBackend) inventoryAdd(state string, token []byte, s backend.MDStatusT) { - err := t.invAdd(state, token, s) - if err != nil { - e := fmt.Sprintf("invAdd %v %x %v: %v", state, token, s, err) - panic(e) - } -} - -// inventoryUpdate is a wrapper around the invUpdate method that allows us to -// decide how disk read/write errors should be handled. For now we simply -// panic. The best thing to do would be to kick off a non-block fsck job that -// checks the inventory cache and corrects any mistakes that it finds. -func (t *tlogBackend) inventoryUpdate(state string, token []byte, s backend.MDStatusT) { - err := t.invUpdate(state, token, s) - if err != nil { - e := fmt.Sprintf("invUpdate %v %x %v: %v", state, token, s, err) - panic(e) - } -} - -// inventoryMoveToVetted is a wrapper around the invMoveToVetted method that -// allows us to decide how disk read/write errors should be handled. For now we -// simply panic. The best thing to do would be to kick off a non-block fsck job -// that checks the inventory cache and corrects any mistakes that it finds. -func (t *tlogBackend) inventoryMoveToVetted(token []byte, s backend.MDStatusT) { - err := t.invMoveToVetted(token, s) - if err != nil { - e := fmt.Sprintf("invMoveToVetted %x %v: %v", token, s, err) - panic(e) - } -} - -func (t *tlogBackend) inventory(state string, s backend.MDStatusT, pageSize, page uint32) (*invByStatus, error) { +func (t *tlogBackend) invByStatus(state string, s backend.MDStatusT, pageSize, page uint32) (*invByStatus, error) { // If no status is provided a page of tokens for each status should // be returned. if s == backend.MDStatusInvalid { diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go index 65e59a21d..075e21505 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go @@ -47,6 +47,11 @@ const ( cmdRunoffDetails = "runoffdetails" ) +// voteHasEnded returns whether the vote has ended. +func voteHasEnded(bestBlock, endHeight uint32) bool { + return bestBlock >= endHeight +} + // tokenDecode decodes a record token and only accepts full length tokens. func tokenDecode(token string) ([]byte, error) { return util.TokenDecode(util.TokenTypeTlog, token) @@ -665,7 +670,7 @@ func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) } // If the vote has not finished yet then we are done for now. - if bestBlock < vd.EndBlockHeight { + if !voteHasEnded(bestBlock, vd.EndBlockHeight) { return &summary, nil } @@ -1077,16 +1082,17 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri } // Update inventory + var status ticketvote.VoteStatusT switch a.Action { case ticketvote.AuthActionAuthorize: - p.invAddToAuthorized(a.Token) + status = ticketvote.VoteStatusAuthorized case ticketvote.AuthActionRevoke: - p.invAddToUnauthorized(a.Token) + status = ticketvote.VoteStatusUnauthorized default: // Should not happen - e := fmt.Sprintf("invalid authorize action: %v", a.Action) - panic(e) + return "", fmt.Errorf("invalid action %v", a.Action) } + p.inventoryUpdate(a.Token, status) // Prepare reply ar := ticketvote.AuthorizeReply{ @@ -1395,7 +1401,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot } // Update inventory - p.invAddToStarted(vd.Params.Token, ticketvote.VoteTypeStandard, + p.inventoryUpdateToStarted(vd.Params.Token, ticketvote.VoteStatusStarted, vd.EndBlockHeight) return &ticketvote.StartReply{ @@ -1577,7 +1583,7 @@ func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs sta } // Update inventory - p.invAddToStarted(vd.Params.Token, ticketvote.VoteTypeRunoff, + p.inventoryUpdateToStarted(vd.Params.Token, ticketvote.VoteStatusStarted, vd.EndBlockHeight) return nil @@ -2222,7 +2228,7 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str ticketvote.VoteErrors[e]) continue } - if bestBlock >= voteDetails.EndBlockHeight { + if voteHasEnded(bestBlock, voteDetails.EndBlockHeight) { // Vote has ended e := ticketvote.VoteErrorVoteStatusInvalid receipts[k].Ticket = v.Ticket @@ -2573,9 +2579,15 @@ func (p *ticketVotePlugin) cmdSummary(treeID int64, token []byte) (string, error return string(reply), nil } -func (p *ticketVotePlugin) cmdInventory() (string, error) { +func (p *ticketVotePlugin) cmdInventory(payload string) (string, error) { log.Tracef("cmdInventory") + var i ticketvote.Inventory + err := json.Unmarshal([]byte(payload), &i) + if err != nil { + return "", err + } + // Get best block. This command does not write any data so we can // use the unsafe best block. bb, err := p.bestBlockUnsafe() @@ -2584,15 +2596,20 @@ func (p *ticketVotePlugin) cmdInventory() (string, error) { } // Get the inventory - inv, err := p.invGet(bb) + ibs, err := p.invByStatus(bb, i.Status, i.Page) if err != nil { - return "", fmt.Errorf("invGet: %v", err) + return "", fmt.Errorf("invByStatus: %v", err) } // Prepare reply + tokens := make(map[string][]string, len(ibs.Tokens)) + for k, v := range ibs.Tokens { + vs := ticketvote.VoteStatuses[k] + tokens[vs] = v + } ir := ticketvote.InventoryReply{ - Tokens: inv.Tokens, - BestBlock: inv.BestBlock, + Tokens: tokens, + BestBlock: ibs.BestBlock, } reply, err := json.Marshal(ir) if err != nil { diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go b/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go index 8b846b572..5fa0f0d9e 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go @@ -312,9 +312,12 @@ func (p *ticketVotePlugin) hookSetRecordStatusPost(payload string) error { return err } - // Update the inventory cache if the record is being made public. - if srs.RecordMetadata.Status == backend.MDStatusVetted { - p.invAddToUnauthorized(srs.RecordMetadata.Token) + // Update the inventory cache + switch srs.RecordMetadata.Status { + case backend.MDStatusVetted: + // Add to inventory + p.inventoryAdd(srs.RecordMetadata.Token, + ticketvote.VoteStatusUnauthorized) } // Update the submissions cache if the linkto has been set. diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go b/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go index 317fc79a2..a78291ec6 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go @@ -22,22 +22,21 @@ const ( filenameInventory = "inventory.json" ) -// inventory contains the record inventory categorized by vote status. The -// unauthorized, authorized, and started lists are updated in real-time since -// ticket vote plugin commands or hooks initiate those actions. The finished, -// approved, and rejected statuses are lazy loaded since those lists depends on -// external state (DCR block height). -type inventory struct { - Tokens map[string][]string `json:"tokens"` // [status][]token - Active []activeVote `json:"active"` // Active votes - BestBlock uint32 `json:"bestblock"` // Last updated +// entry is an inventory entry. +type entry struct { + Token string `json:"token"` + Status ticketvote.VoteStatusT `json:"status"` + EndHeight uint32 `json:"endheight,omitempty"` } -// activeVote is used to track active votes. The end height is stored so that -// we can check what votes have finished when a new best blocks come in. -type activeVote struct { - Token string `json:"token"` - EndHeight uint32 `json:"endheight"` +// inventory contains the ticketvote inventory. The unauthorized, authorized, +// and started lists are updated in real-time since ticket vote plugin commands +// or hooks initiate those actions. The finished, approved, and rejected +// statuses are lazy loaded since those lists depends on external state (DCR +// block height). +type inventory struct { + Entries []entry `json:"entries"` + BestBlock uint32 `json:"bestblock"` } // invPath returns the full path for the cached ticket vote inventory. @@ -48,7 +47,7 @@ func (p *ticketVotePlugin) invPath() string { // invGetLocked retrieves the inventory from disk. A new inventory is returned // if one does not exist yet. // -// This function must be called WITH the lock held. +// This function must be called WITH the read lock held. func (p *ticketVotePlugin) invGetLocked() (*inventory, error) { b, err := ioutil.ReadFile(p.invPath()) if err != nil { @@ -56,8 +55,7 @@ func (p *ticketVotePlugin) invGetLocked() (*inventory, error) { if errors.As(err, &e) && !os.IsExist(err) { // File does't exist. Return a new inventory. return &inventory{ - Tokens: make(map[string][]string, 256), - Active: make([]activeVote, 0, 256), + Entries: make([]entry, 0, 256), BestBlock: 0, }, nil } @@ -73,10 +71,21 @@ func (p *ticketVotePlugin) invGetLocked() (*inventory, error) { return &inv, nil } -// invSetLocked writes the inventory to disk. +// invGetLocked retrieves the inventory from disk. A new inventory is returned +// if one does not exist yet. +// +// This function must be called WITHOUT the read lock held. +func (p *ticketVotePlugin) invGet() (*inventory, error) { + p.mtxInv.RLock() + defer p.mtxInv.RUnlock() + + return p.invGetLocked() +} + +// invSaveLocked writes the inventory to disk. // -// This function must be called WITH the lock held. -func (p *ticketVotePlugin) invSetLocked(inv inventory) error { +// This function must be called WITH the read/write lock held. +func (p *ticketVotePlugin) invSaveLocked(inv inventory) error { b, err := json.Marshal(inv) if err != nil { return err @@ -84,283 +93,318 @@ func (p *ticketVotePlugin) invSetLocked(inv inventory) error { return ioutil.WriteFile(p.invPath(), b, 0664) } -// invAddToUnauthorized adds the token to the unauthorized vote status list. -// This is done when a unvetted record is made vetted or when a previous vote -// authorization is revoked. -func (p *ticketVotePlugin) invAddToUnauthorized(token string) { +func (p *ticketVotePlugin) invAdd(token string, s ticketvote.VoteStatusT) error { p.mtxInv.Lock() defer p.mtxInv.Unlock() + // Get inventory inv, err := p.invGetLocked() if err != nil { - panic(err) + return err } - var ( - // Human readable vote statuses - unauth = ticketvote.VoteStatuses[ticketvote.VoteStatusUnauthorized] - auth = ticketvote.VoteStatuses[ticketvote.VoteStatusAuthorized] - ) - - // Remove token from the authorized list. It will only exit in the - // authorized list if the user is revoking a previous authorization. - ok := invDel(inv.Tokens, auth, token) - if ok { - log.Debugf("Vote inv del %v from authorized", token) + // Prepend token + e := entry{ + Token: token, + Status: s, } - - // Add the token to unauthorized - invAdd(inv.Tokens, unauth, token) - - log.Debugf("Vote inv add %v to unauthorized", token) + inv.Entries = append([]entry{e}, inv.Entries...) // Save inventory - err = p.invSetLocked(*inv) - if err != nil { - panic(err) - } -} - -// invAddToAuthorized moves a record from the unauthorized to the authorized -// list. This is done by the ticketvote authorize command. -func (p *ticketVotePlugin) invAddToAuthorized(token string) { - p.mtxInv.Lock() - defer p.mtxInv.Unlock() - - inv, err := p.invGetLocked() + err = p.invSaveLocked(*inv) if err != nil { - panic(err) - } - - var ( - // Human readable vote statuses - unauth = ticketvote.VoteStatuses[ticketvote.VoteStatusUnauthorized] - auth = ticketvote.VoteStatuses[ticketvote.VoteStatusAuthorized] - ) - - // Remove the token from unauthorized list. The token should always - // be in the unauthorized list. - ok := invDel(inv.Tokens, unauth, token) - if !ok { - e := fmt.Sprintf("token not found in unauthorized list %v", token) - panic(e) + return err } - log.Debugf("Vote inv del %v from unauthorized", token) - - // Prepend the token to the authorized list - invAdd(inv.Tokens, auth, token) - - log.Debugf("Vote inv add %v to authorized", token) + log.Debugf("Vote inv add %v %v", token, ticketvote.VoteStatuses[s]) - // Save inventory - err = p.invSetLocked(*inv) - if err != nil { - panic(err) - } + return nil } -// invAddToStarted moves a record into the started vote status list. This is -// done by the ticketvote start command. -func (p *ticketVotePlugin) invAddToStarted(token string, t ticketvote.VoteT, endHeight uint32) { +func (p *ticketVotePlugin) invUpdate(token string, s ticketvote.VoteStatusT, endHeight uint32) error { p.mtxInv.Lock() defer p.mtxInv.Unlock() + // Get inventory inv, err := p.invGetLocked() if err != nil { - panic(err) + return err } - var ( - // Human readable vote statuses - unauth = ticketvote.VoteStatuses[ticketvote.VoteStatusUnauthorized] - auth = ticketvote.VoteStatuses[ticketvote.VoteStatusAuthorized] - started = ticketvote.VoteStatuses[ticketvote.VoteStatusStarted] - ) - - switch t { - case ticketvote.VoteTypeStandard: - // Remove the token from the authorized list. The token should - // always be in the authorized list prior to the vote being - // started for standard votes. - ok := invDel(inv.Tokens, auth, token) - if !ok { - e := fmt.Sprintf("token not found in authorized list %v", token) - panic(e) - } - - log.Debugf("Vote inv del %v from authorized", token) - - case ticketvote.VoteTypeRunoff: - // A runoff vote does not require the submissions be authorized - // prior to the vote starting. The token should always be in the - // unauthorized list. - ok := invDel(inv.Tokens, unauth, token) - if !ok { - e := fmt.Sprintf("token not found in unauthorized list %v", token) - panic(e) - } - - log.Debugf("Vote inv del %v from unauthorized", token) - - default: - e := fmt.Sprintf("invalid vote type %v", t) + // Del entry + entries, err := entryDel(inv.Entries, token) + if err != nil { + // This should not happen. Panic if it does. + e := fmt.Sprintf("entry del: %v", err) panic(e) } - // Add token to started list - invAdd(inv.Tokens, started, token) - - // Add token to active votes list - vt := activeVote{ + // Prepend new entry to inventory + e := entry{ Token: token, + Status: s, EndHeight: endHeight, } - inv.Active = append([]activeVote{vt}, inv.Active...) - - // Sort active votes - sortActiveVotes(inv.Active) - - log.Debugf("Vote inv add %v to started with end height %v", - token, endHeight) + inv.Entries = append([]entry{e}, entries...) // Save inventory - err = p.invSetLocked(*inv) + err = p.invSaveLocked(*inv) if err != nil { - panic(err) + return err } + + log.Debugf("Vote inv update %v to %v", token, ticketvote.VoteStatuses[s]) + + return nil } -func (p *ticketVotePlugin) invGet(bestBlock uint32) (*inventory, error) { +func (p *ticketVotePlugin) invUpdateForBlock(bestBlock uint32) (*inventory, error) { p.mtxInv.Lock() defer p.mtxInv.Unlock() inv, err := p.invGetLocked() if err != nil { - panic(err) + return nil, err } - - // Check if the inventory has been updated for this block height. if inv.BestBlock == bestBlock { - // Inventory already updated. Nothing else to do. return inv, nil } - // The active votes should already be sorted, but sort them again - // just to be sure. - sortActiveVotes(inv.Active) - - // The inventory has not been updated for this block height. Check - // if any votes have finished. - active := make([]activeVote, 0, len(inv.Active)) - for _, v := range inv.Active { - if bestBlock < v.EndHeight { - // Vote has not finished yet. Keep it in the active votes list. - active = append(active, v) + // Compile the votes that have ended + ended := make([]entry, 0, 256) + for _, v := range inv.Entries { + if v.EndHeight == 0 { continue } + if voteHasEnded(bestBlock, v.EndHeight) { + ended = append(ended, v) + } + } + + // Sort by end height from smallest to largest so that they're + // added to the inventory in the correct order. + sort.SliceStable(ended, func(i, j int) bool { + return ended[i].EndHeight < ended[j].EndHeight + }) - // Vote has finished. Get vote summary. - t, err := tokenDecode(v.Token) + // Update the inventory for the ended entries + for _, v := range ended { + // Get the vote summary + token, err := tokenDecode(v.Token) if err != nil { return nil, err } - sr, err := p.summaryByToken(t) + sr, err := p.summaryByToken(token) if err != nil { return nil, err } - // Remove token from started list - started := ticketvote.VoteStatuses[ticketvote.VoteStatusStarted] - ok := invDel(inv.Tokens, started, v.Token) - if !ok { - return nil, fmt.Errorf("token not found in started %v", v.Token) - } - - log.Debugf("Vote inv del %v from started", v.Token) - - // Add token to the appropriate list + // Update inventory switch sr.Status { case ticketvote.VoteStatusFinished, ticketvote.VoteStatusApproved, ticketvote.VoteStatusRejected: // These statuses are allowed - status := ticketvote.VoteStatuses[sr.Status] - invAdd(inv.Tokens, status, v.Token) + err := p.invUpdate(v.Token, sr.Status, 0) + if err != nil { + return nil, err + } default: - // Something went wrong return nil, fmt.Errorf("unexpected vote status %v %v", v.Token, sr.Status) } - - log.Debugf("Vote inv add %v to %v", - v.Token, ticketvote.VoteStatuses[sr.Status]) } - // Update active votes list - inv.Active = active - // Update best block + inv, err = p.invGetLocked() + if err != nil { + return nil, err + } inv.BestBlock = bestBlock + // Save inventory + err = p.invSaveLocked(*inv) + if err != nil { + return nil, err + } + log.Debugf("Vote inv updated for block %v", bestBlock) - // Save inventory - err = p.invSetLocked(*inv) + return inv, nil +} + +// inventoryAdd is a wrapper around the invAdd method that allows us to decide +// how disk read/write errors should be handled. For now we just panic. +func (p *ticketVotePlugin) inventoryAdd(token string, s ticketvote.VoteStatusT) { + err := p.invAdd(token, s) + if err != nil { + e := fmt.Sprintf("invAdd %v %v: %v", token, s, err) + panic(e) + } +} + +func (p *ticketVotePlugin) inventoryUpdate(token string, s ticketvote.VoteStatusT) { + err := p.invUpdate(token, s, 0) + if err != nil { + e := fmt.Sprintf("invUpdate %v %v: %v", token, s, err) + panic(e) + } +} + +func (p *ticketVotePlugin) inventoryUpdateToStarted(token string, s ticketvote.VoteStatusT, endHeight uint32) { + err := p.invUpdate(token, s, endHeight) if err != nil { - panic(err) + e := fmt.Sprintf("invUpdate %v %v: %v", token, s, err) + panic(e) + } +} + +func (p *ticketVotePlugin) inventory(bestBlock uint32) (*inventory, error) { + // Get inventory + inv, err := p.invGet() + if err != nil { + return nil, err + } + + // Check if the inventory has been updated for this block height. + if bestBlock > inv.BestBlock { + // Inventory has not been update for this block. Update it. + return p.invUpdateForBlock(bestBlock) } return inv, nil } -func sortActiveVotes(v []activeVote) { - // Sort by end height from smallest to largest - sort.SliceStable(v, func(i, j int) bool { - return v[i].EndHeight < v[j].EndHeight - }) +// invByStatus contains the inventory categorized by vote status. Each list +// contains a page of tokens that are sorted by the timestamp of the status +// change from newest to oldest. +type invByStatus struct { + Tokens map[ticketvote.VoteStatusT][]string + BestBlock uint32 } -func invDel(inv map[string][]string, status, token string) bool { - list, ok := inv[status] - if !ok { - inv[status] = make([]string, 0, 1056) - return false +func (p *ticketVotePlugin) invByStatusAll(bestBlock, pageSize uint32) (*invByStatus, error) { + // Get inventory + i, err := p.inventory(bestBlock) + if err != nil { + return nil, err + } + + // Prepare reply + var ( + unauth = tokensParse(i.Entries, ticketvote.VoteStatusUnauthorized, + pageSize, 1) + auth = tokensParse(i.Entries, ticketvote.VoteStatusAuthorized, + pageSize, 1) + started = tokensParse(i.Entries, ticketvote.VoteStatusStarted, + pageSize, 1) + finished = tokensParse(i.Entries, ticketvote.VoteStatusFinished, + pageSize, 1) + approved = tokensParse(i.Entries, ticketvote.VoteStatusApproved, + pageSize, 1) + rejected = tokensParse(i.Entries, ticketvote.VoteStatusRejected, + pageSize, 1) + + tokens = make(map[ticketvote.VoteStatusT][]string, 16) + ) + if len(unauth) != 0 { + tokens[ticketvote.VoteStatusUnauthorized] = unauth } + if len(auth) != 0 { + tokens[ticketvote.VoteStatusAuthorized] = auth + } + if len(started) != 0 { + tokens[ticketvote.VoteStatusStarted] = started + } + if len(finished) != 0 { + tokens[ticketvote.VoteStatusFinished] = finished + } + if len(approved) != 0 { + tokens[ticketvote.VoteStatusApproved] = approved + } + if len(rejected) != 0 { + tokens[ticketvote.VoteStatusRejected] = rejected + } + + return &invByStatus{ + Tokens: tokens, + BestBlock: i.BestBlock, + }, nil +} + +func (p *ticketVotePlugin) invByStatus(bestBlock uint32, s ticketvote.VoteStatusT, page uint32) (*invByStatus, error) { + pageSize := ticketvote.InventoryPageSize - // Find token (linear time) + // If no status is provided a page of tokens for each status should + // be returned. + if s == ticketvote.VoteStatusInvalid { + return p.invByStatusAll(bestBlock, pageSize) + } + + // A status was provided. Return a page of tokens for the status. + inv, err := p.inventory(bestBlock) + if err != nil { + return nil, err + } + tokens := tokensParse(inv.Entries, s, pageSize, page) + + return &invByStatus{ + Tokens: map[ticketvote.VoteStatusT][]string{ + s: tokens, + }, + BestBlock: inv.BestBlock, + }, nil +} + +// entryDel removes the entry for the token and returns the updated slice. +func entryDel(entries []entry, token string) ([]entry, error) { + // Find token in entries var i int var found bool - for k, v := range list { - if v == token { + for k, v := range entries { + if v.Token == token { i = k found = true break } } if !found { - return found + return nil, fmt.Errorf("token not found %v", token) } - // Remove token (linear time) - copy(list[i:], list[i+1:]) // Shift list[i+1:] left one index - list[len(list)-1] = "" // Del last element (write zero value) - list = list[:len(list)-1] // Truncate slice - - // Update inv - inv[status] = list + // Del token from entries (linear time) + copy(entries[i:], entries[i+1:]) // Shift entries[i+1:] left one index + entries[len(entries)-1] = entry{} // Del last element (write zero value) + entries = entries[:len(entries)-1] // Truncate slice - return found + return entries, nil } -func invAdd(inv map[string][]string, status, token string) { - list, ok := inv[status] - if !ok { - list = make([]string, 0, 1056) +// tokensParse parses a page of tokens from the provided entries. +func tokensParse(entries []entry, s ticketvote.VoteStatusT, countPerPage, page uint32) []string { + tokens := make([]string, 0, countPerPage) + if countPerPage == 0 || page == 0 { + return tokens } - // Prepend token - list = append([]string{token}, list...) + startAt := (page - 1) * countPerPage + var foundCount uint32 + for _, v := range entries { + if v.Status != s { + // Status does not match + continue + } + + // Matching status found + if foundCount >= startAt { + tokens = append(tokens, v.Token) + if len(tokens) == int(countPerPage) { + // We got a full page. We're done. + return tokens + } + } + + foundCount++ + } - // Update inv - inv[status] = list + return tokens } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go index 751b25f24..9a2eeb178 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go @@ -57,9 +57,9 @@ type ticketVotePlugin struct { mtxRecords sync.Mutex // Mutexes for on-disk caches - mtxInv sync.Mutex // Vote inventory - mtxSummary sync.Mutex // Vote summaries - mtxSubs sync.Mutex // Runoff vote submissions + mtxInv sync.RWMutex // Vote inventory cache + mtxSummary sync.Mutex // Vote summaries cache + mtxSubs sync.Mutex // Runoff vote submission cache // Plugin settings linkByPeriodMin int64 // In seconds @@ -104,22 +104,27 @@ func (p *ticketVotePlugin) Setup() error { // Update the inventory with the current best block. Retrieving // the inventory will cause it to update. - log.Infof("Updating vote inv") + log.Infof("Updating vote inventory") bb, err := p.bestBlock() if err != nil { return fmt.Errorf("bestBlock: %v", err) } - inv, err := p.invGet(bb) + inv, err := p.inventory(bb) if err != nil { - return fmt.Errorf("invGet: %v", err) + return fmt.Errorf("inventory: %v", err) } // Build votes cache log.Infof("Building votes cache") - started := ticketvote.VoteStatuses[ticketvote.VoteStatusStarted] - for _, v := range inv.Tokens[started] { + started := make([]string, 0, len(inv.Entries)) + for _, v := range inv.Entries { + if v.Status == ticketvote.VoteStatusStarted { + started = append(started, v.Token) + } + } + for _, v := range started { token, err := tokenDecode(v) if err != nil { return err @@ -165,7 +170,7 @@ func (p *ticketVotePlugin) Cmd(treeID int64, token []byte, cmd, payload string) case ticketvote.CmdSubmissions: return p.cmdSubmissions(token) case ticketvote.CmdInventory: - return p.cmdInventory() + return p.cmdInventory(payload) case ticketvote.CmdTimestamps: return p.cmdTimestamps(treeID, token) diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 2e5d27efb..69d7a322e 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -67,11 +67,6 @@ type tlogBackend struct { // This cache memoizes those results and is lazy loaded. vettedTreeIDs map[string]int64 // [token]treeID - // inv contains the full record inventory grouped by record status. - // Each list of tokens is sorted by the timestamp of the status - // change from newest to oldest. This cache is built on startup. - inv inventory - // recordMtxs allows a the backend to hold a lock on a record so // that it can perform multiple read/write operations in a // concurrent safe manner. These mutexes are lazy loaded. @@ -1475,7 +1470,7 @@ func (t *tlogBackend) GetVettedTimestamps(token []byte, version string) (*backen func (t *tlogBackend) InventoryByStatus(state string, status backend.MDStatusT, pageSize, page uint32) (*backend.InventoryByStatus, error) { log.Tracef("InventoryByStatus: %v %v %v %v", state, status, pageSize, page) - inv, err := t.inventory(state, status, pageSize, page) + inv, err := t.invByStatus(state, status, pageSize, page) if err != nil { return nil, err } diff --git a/politeiad/client/ticketvote.go b/politeiad/client/ticketvote.go index 865d85232..021bc9966 100644 --- a/politeiad/client/ticketvote.go +++ b/politeiad/client/ticketvote.go @@ -335,14 +335,18 @@ func (c *Client) TicketVoteSubmissions(ctx context.Context, token string) (*tick // TicketVoteInventory sends the ticketvote plugin Inventory command to the // politeiad v1 API. -func (c *Client) TicketVoteInventory(ctx context.Context) (*ticketvote.InventoryReply, error) { +func (c *Client) TicketVoteInventory(ctx context.Context, i ticketvote.Inventory) (*ticketvote.InventoryReply, error) { // Setup request + b, err := json.Marshal(i) + if err != nil { + return nil, err + } cmds := []pdv1.PluginCommandV2{ { State: pdv1.RecordStateVetted, ID: ticketvote.PluginID, Command: ticketvote.CmdInventory, - Payload: "", + Payload: string(b), }, } diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index 34a5110fb..caf067507 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -509,15 +509,28 @@ type SubmissionsReply struct { Submissions []string `json:"submissions"` } -// Inventory requests the tokens of all public, non-abandoned records -// categorized by vote status. -type Inventory struct{} - -// InventoryReply is the reply to the Inventory command. It contains the tokens -// of all public, non-abandoned records categorized by vote status. The -// returned map is a map[votestatus][]token where the votestatus key is the -// human readable vote status defined by the VoteStatuses array in this -// package. +const ( + // InventoryPageSize is the maximum number of tokens that will be + // returned for any single status in an InventoryReply. + InventoryPageSize uint32 = 20 +) + +// Inventory requests the tokens of public records in the inventory categorized +// by vote status. +// +// The status and page arguments can be provided to request a specific page of +// record tokens. +// +// If no status is provided then a page of tokens for all statuses will be +// returned. The page argument will be ignored. +type Inventory struct { + Status VoteStatusT `json:"status,omitempty"` + Page uint32 `json:"page,omitempty"` +} + +// InventoryReply is the reply to the Inventory command. The returned map is a +// map[votestatus][]token where the votestatus key is the human readable vote +// status defined by the VoteStatuses array in this package. // // Sorted by timestamp in descending order: // Unauthorized, Authorized diff --git a/politeiawww/api/ticketvote/v1/v1.go b/politeiawww/api/ticketvote/v1/v1.go index 691fb50c8..6df3b3291 100644 --- a/politeiawww/api/ticketvote/v1/v1.go +++ b/politeiawww/api/ticketvote/v1/v1.go @@ -507,21 +507,29 @@ type SubmissionsReply struct { } const ( - // TODO implement Inventory pagnation - InventoryPageSize = 60 + // InventoryPageSize is the maximum number of tokens that will be + // returned for any single status in an InventoryReply. + InventoryPageSize uint32 = 20 ) -// Inventory requests the tokens of all public, non-abandoned records -// categorized by vote status. -type Inventory struct{} +// Inventory requests the tokens of public records in the inventory categorized +// by vote status. +// +// The status and page arguments can be provided to request a specific page of +// record tokens. +// +// If no status is provided then a page of tokens for all statuses will be +// returned. The page argument will be ignored. +type Inventory struct { + Status VoteStatusT `json:"status,omitempty"` + Page uint32 `json:"page,omitempty"` +} -// InventoryReply is the reply to the Inventory command. It contains the tokens -// of all public, non-abandoned records categorized by vote status. The -// returned map is a map[votestatus][]token where the votestatus key is the -// human readable vote status defined by the VoteStatuses array in this -// package. +// InventoryReply is the reply to the Inventory command. The returned map is a +// map[votestatus][]token where the votestatus key is the human readable vote +// status defined by the VoteStatuses array in this package. // -// Sorted by timestamp in descending order: +// Sorted by timestamp newest to oldest: // Unauthorized, Authorized // // Sorted by vote start block height in descending order: diff --git a/politeiawww/client/ticketvote.go b/politeiawww/client/ticketvote.go index 92e19f731..fed3d1c57 100644 --- a/politeiawww/client/ticketvote.go +++ b/politeiawww/client/ticketvote.go @@ -176,9 +176,9 @@ func (c *Client) TicketVoteSubmissions(s tkv1.Submissions) (*tkv1.SubmissionsRep } // TicketVoteInventory sends a ticketvote v1 Inventory request to politeiawww. -func (c *Client) TicketVoteInventory() (*tkv1.InventoryReply, error) { +func (c *Client) TicketVoteInventory(i tkv1.Inventory) (*tkv1.InventoryReply, error) { resBody, err := c.makeReq(http.MethodPost, - tkv1.APIRoute, tkv1.RouteInventory, nil) + tkv1.APIRoute, tkv1.RouteInventory, i) if err != nil { return nil, err } diff --git a/politeiawww/cmd/pictl/cmdproposalinv.go b/politeiawww/cmd/pictl/cmdproposalinv.go index cf5320f5c..080a2e1bc 100644 --- a/politeiawww/cmd/pictl/cmdproposalinv.go +++ b/politeiawww/cmd/pictl/cmdproposalinv.go @@ -56,7 +56,7 @@ func (c *cmdProposalInv) Execute(args []string) error { if c.Args.Status != "" { // Parse status. This can be either the numeric status code or the // human readable equivalent. - status, err = parseStatus(c.Args.Status) + status, err = parseRecordStatus(c.Args.Status) if err != nil { return err } diff --git a/politeiawww/cmd/pictl/cmdproposalstatusset.go b/politeiawww/cmd/pictl/cmdproposalstatusset.go index c7c1775a3..bd66f4844 100644 --- a/politeiawww/cmd/pictl/cmdproposalstatusset.go +++ b/politeiawww/cmd/pictl/cmdproposalstatusset.go @@ -63,7 +63,7 @@ func (c *cmdProposalSetStatus) Execute(args []string) error { // Parse status. This can be either the numeric status code or the // human readable equivalent. - status, err := parseStatus(c.Args.Status) + status, err := parseRecordStatus(c.Args.Status) if err != nil { return err } @@ -123,7 +123,7 @@ func (c *cmdProposalSetStatus) Execute(args []string) error { return nil } -func parseStatus(status string) (rcv1.RecordStatusT, error) { +func parseRecordStatus(status string) (rcv1.RecordStatusT, error) { // Parse status. This can be either the numeric status code or the // human readable equivalent. var ( diff --git a/politeiawww/cmd/pictl/cmdvoteinv.go b/politeiawww/cmd/pictl/cmdvoteinv.go index 9f2492126..e904a20f9 100644 --- a/politeiawww/cmd/pictl/cmdvoteinv.go +++ b/politeiawww/cmd/pictl/cmdvoteinv.go @@ -4,11 +4,22 @@ package main -import pclient "github.com/decred/politeia/politeiawww/client" +import ( + "fmt" + "strconv" -// cmdVoteInv retrieves the censorship record tokens of all public, -// non-abandoned records in the inventory, categorized by their vote status. -type cmdVoteInv struct{} + tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" + pclient "github.com/decred/politeia/politeiawww/client" +) + +// cmdVoteInv retrieves the censorship record tokens of the public records in +// the inventory, categorized by their vote status. +type cmdVoteInv struct { + Args struct { + Status string `positional-arg-name:"status"` + Page uint32 `positional-arg-name:"page"` + } `positional-args:"true" optional:"true"` +} // Execute executes the cmdVoteInv command. // @@ -25,8 +36,29 @@ func (c *cmdVoteInv) Execute(args []string) error { return err } + // Setup status and page number + var status tkv1.VoteStatusT + if c.Args.Status != "" { + // Parse status. This can be either the numeric status code or the + // human readable equivalent. + status, err = parseVoteStatus(c.Args.Status) + if err != nil { + return err + } + + // If a status was given but no page number was give, default + // to page number 1. + if c.Args.Page == 0 { + c.Args.Page = 1 + } + } + // Get vote inventory - ir, err := pc.TicketVoteInventory() + i := tkv1.Inventory{ + Status: status, + Page: c.Args.Page, + } + ir, err := pc.TicketVoteInventory(i) if err != nil { return err } @@ -37,8 +69,59 @@ func (c *cmdVoteInv) Execute(args []string) error { return nil } +func parseVoteStatus(status string) (tkv1.VoteStatusT, error) { + // Parse status. This can be either the numeric status code or the + // human readable equivalent. + var ( + vs tkv1.VoteStatusT + + statuses = map[string]tkv1.VoteStatusT{ + "unauthorized": tkv1.VoteStatusUnauthorized, + "authorized": tkv1.VoteStatusAuthorized, + "started": tkv1.VoteStatusStarted, + "approved": tkv1.VoteStatusApproved, + "rejected": tkv1.VoteStatusRejected, + "1": tkv1.VoteStatusUnauthorized, + "2": tkv1.VoteStatusAuthorized, + "3": tkv1.VoteStatusStarted, + "5": tkv1.VoteStatusApproved, + "6": tkv1.VoteStatusRejected, + } + ) + u, err := strconv.ParseUint(status, 10, 32) + if err == nil { + // Numeric status code found + vs = tkv1.VoteStatusT(u) + } else if s, ok := statuses[status]; ok { + // Human readable status code found + vs = s + } else { + return vs, fmt.Errorf("invalid status '%v'", status) + } + + return vs, nil +} + // voteInvHelpMsg is printed to stdout by the help command. const voteInvHelpMsg = `voteinv -Fetch the censorship record tokens of all public, non-abandoned records, -categorized by their vote status.` +Inventory requests the tokens of public records in the inventory categorized by +vote status. + +The status and page arguments can be provided to request a specific page of +record tokens. + +If no status is provided then a page of tokens for all statuses will be +returned. The page argument will be ignored. + +Valid statuses: + ("1") "unauthorized" + ("2") "authorized" + ("3") "started" + ("5") "approved" + ("6") "rejected" + +Arguments: +1. status (string, optional) Status of tokens being requested. +2. page (uint32, optional) Page number. +` diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 86c128884..6cd22a834 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -14,6 +14,7 @@ import ( pdv1 "github.com/decred/politeia/politeiad/api/v1" v1 "github.com/decred/politeia/politeiad/api/v1" piplugin "github.com/decred/politeia/politeiad/plugins/pi" + "github.com/decred/politeia/politeiad/plugins/ticketvote" tvplugin "github.com/decred/politeia/politeiad/plugins/ticketvote" usplugin "github.com/decred/politeia/politeiad/plugins/user" www "github.com/decred/politeia/politeiawww/api/www/v1" @@ -35,7 +36,8 @@ func (p *politeiawww) processTokenInventory(ctx context.Context, isAdmin bool) ( } // Get vote inventory - vir, err := p.politeiad.TicketVoteInventory(ctx) + ti := ticketvote.Inventory{} + vir, err := p.politeiad.TicketVoteInventory(ctx, ti) if err != nil { return nil, err } diff --git a/politeiawww/ticketvote/error.go b/politeiawww/ticketvote/error.go index 7636246b3..ab46cfc24 100644 --- a/politeiawww/ticketvote/error.go +++ b/politeiawww/ticketvote/error.go @@ -13,7 +13,7 @@ import ( "time" pdclient "github.com/decred/politeia/politeiad/client" - v1 "github.com/decred/politeia/politeiawww/api/comments/v1" + v1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" "github.com/decred/politeia/util" ) diff --git a/politeiawww/ticketvote/process.go b/politeiawww/ticketvote/process.go index dbcfbc9bf..c8eccf955 100644 --- a/politeiawww/ticketvote/process.go +++ b/politeiawww/ticketvote/process.go @@ -195,11 +195,15 @@ func (t *TicketVote) processSubmissions(ctx context.Context, s v1.Submissions) ( }, nil } -func (t *TicketVote) processInventory(ctx context.Context) (*v1.InventoryReply, error) { - log.Tracef("processInventory") +func (t *TicketVote) processInventory(ctx context.Context, i v1.Inventory) (*v1.InventoryReply, error) { + log.Tracef("processInventory: %v %v", i.Status, i.Page) - // Send plugin command - ir, err := t.politeiad.TicketVoteInventory(ctx) + // Get inventory + ti := ticketvote.Inventory{ + Status: convertVoteStatusToPlugin(i.Status), + Page: i.Page, + } + ir, err := t.politeiad.TicketVoteInventory(ctx, ti) if err != nil { return nil, err } @@ -240,6 +244,25 @@ func (t *TicketVote) processTimestamps(ctx context.Context, ts v1.Timestamps) (* }, nil } +func convertVoteStatusToPlugin(s v1.VoteStatusT) ticketvote.VoteStatusT { + switch s { + case v1.VoteStatusUnauthorized: + return ticketvote.VoteStatusUnauthorized + case v1.VoteStatusAuthorized: + return ticketvote.VoteStatusAuthorized + case v1.VoteStatusStarted: + return ticketvote.VoteStatusStarted + case v1.VoteStatusFinished: + return ticketvote.VoteStatusFinished + case v1.VoteStatusApproved: + return ticketvote.VoteStatusApproved + case v1.VoteStatusRejected: + return ticketvote.VoteStatusRejected + default: + return ticketvote.VoteStatusInvalid + } +} + func convertVoteTypeToPlugin(t v1.VoteT) ticketvote.VoteT { switch t { case v1.VoteTypeStandard: diff --git a/politeiawww/ticketvote/ticketvote.go b/politeiawww/ticketvote/ticketvote.go index 644c5f116..2118d57a2 100644 --- a/politeiawww/ticketvote/ticketvote.go +++ b/politeiawww/ticketvote/ticketvote.go @@ -227,7 +227,17 @@ func (t *TicketVote) HandleSubmissions(w http.ResponseWriter, r *http.Request) { func (t *TicketVote) HandleInventory(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleInventory") - ir, err := t.processInventory(r.Context()) + var i v1.Inventory + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&i); err != nil { + respondWithError(w, r, "HandleInventory: unmarshal", + v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeInputInvalid, + }) + return + } + + ir, err := t.processInventory(r.Context(), i) if err != nil { respondWithError(w, r, "HandleInventory: processInventory: %v", err) From 1b7fda4e0b91aefd1cc2e93267dac3665e98004d Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 14 Feb 2021 12:01:49 -0600 Subject: [PATCH 303/449] Fix vote cache bug. --- politeiad/backend/tlogbe/plugins/plugins.go | 3 +++ .../tlogbe/plugins/ticketvote/hooks.go | 17 +++++++++--- .../tlogbe/plugins/ticketvote/ticketvote.go | 2 +- .../backend/tlogbe/plugins/user/hooks.go | 26 +++++++------------ 4 files changed, 28 insertions(+), 20 deletions(-) diff --git a/politeiad/backend/tlogbe/plugins/plugins.go b/politeiad/backend/tlogbe/plugins/plugins.go index 5022b860b..a019de962 100644 --- a/politeiad/backend/tlogbe/plugins/plugins.go +++ b/politeiad/backend/tlogbe/plugins/plugins.go @@ -206,6 +206,9 @@ type TlogClient interface { // to the digest. Timestamp(treeID int64, digest []byte) (*backend.Timestamp, error) + // RecordExists returns whether a record exists. + RecordExists(treeID int64) bool + // Record returns a version of a record. Record(treeID int64, version uint32) (*backend.Record, error) diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go b/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go index 5fa0f0d9e..cc24bd038 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go @@ -305,15 +305,26 @@ func (p *ticketVotePlugin) hookSetRecordStatusPre(payload string) error { return nil } -func (p *ticketVotePlugin) hookSetRecordStatusPost(payload string) error { +func (p *ticketVotePlugin) hookSetRecordStatusPost(treeID int64, payload string) error { var srs plugins.HookSetRecordStatus err := json.Unmarshal([]byte(payload), &srs) if err != nil { return err } + status := srs.RecordMetadata.Status + + // When a record is moved to vetted the plugin hooks are executed + // on both the unvetted and vetted tstore instances. We only need + // to update cached data if this is the vetted instance. We can + // determine this by checking if the record exists. The unvetted + // instance will return false. + if status == backend.MDStatusVetted && !p.tlog.RecordExists(treeID) { + // This is the unvetted instance. Nothing to do. + return nil + } // Update the inventory cache - switch srs.RecordMetadata.Status { + switch status { case backend.MDStatusVetted: // Add to inventory p.inventoryAdd(srs.RecordMetadata.Token, @@ -332,7 +343,7 @@ func (p *ticketVotePlugin) hookSetRecordStatusPost(payload string) error { parentToken = vm.LinkTo childToken = srs.RecordMetadata.Token ) - switch srs.RecordMetadata.Status { + switch status { case backend.MDStatusVetted: // Record has been made public. Add child token to parent's // submissions list. diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go index 9a2eeb178..1ee40a336 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go @@ -198,7 +198,7 @@ func (p *ticketVotePlugin) Hook(treeID int64, token []byte, h plugins.HookT, pay case plugins.HookTypeSetRecordStatusPre: return p.hookSetRecordStatusPre(payload) case plugins.HookTypeSetRecordStatusPost: - return p.hookSetRecordStatusPost(payload) + return p.hookSetRecordStatusPost(treeID, payload) } return nil diff --git a/politeiad/backend/tlogbe/plugins/user/hooks.go b/politeiad/backend/tlogbe/plugins/user/hooks.go index f9570c714..e94118625 100644 --- a/politeiad/backend/tlogbe/plugins/user/hooks.go +++ b/politeiad/backend/tlogbe/plugins/user/hooks.go @@ -349,30 +349,24 @@ func (p *userPlugin) hookSetRecordStatusPost(treeID int64, payload string) error return err } - // Determine whether to add or remove this token from the user - // cache. The token will need to be removed from the unvetted - // tstore user cache and added to the vetted tstore user cache. - // We can determine what tstore instance this is by checking if - // the record exists. Once a record has been made vetted, the - // unvetted tstore instance will return a record not found error - // when queried. - _, err = p.tlog.RecordLatest(treeID) - switch err { - case nil: - // Record exists. Add token to the user cache. + // When a record is moved to vetted the plugin hooks are executed + // on both the unvetted and vetted tstore instances. The token + // needs to be removed from the unvetted tstore user cache and + // added to the vetted tstore user cache. We can determine this + // by checking if the record exists. The unvetted instance will + // return false. + if p.tlog.RecordExists(treeID) { + // This is the vetted tstore. Add token to the user cache. err = p.userCacheAddToken(um.UserID, srs.RecordMetadata.Token) if err != nil { return err } - case backend.ErrRecordNotFound: - // Record does not exist. Del token from user cache. + } else { + // This is the unvetted tstore. Del token from user cache. err = p.userCacheDelToken(um.UserID, srs.RecordMetadata.Token) if err != nil { return err } - default: - // Unexpected error - return fmt.Errorf("RecordLatest %v: %v", treeID, err) } } From bca61f0e5624bb8e94b7898450ef49519fb65edd Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 15 Feb 2021 08:41:40 -0600 Subject: [PATCH 304/449] Drop in mysql kv. --- go.mod | 1 + .../{filesystem/filesystem.go => fs/fs.go} | 52 ++-- .../tlogbe/store/{filesystem => fs}/log.go | 2 +- politeiad/backend/tlogbe/store/mysql/log.go | 25 ++ politeiad/backend/tlogbe/store/mysql/mysql.go | 232 ++++++++++++++++++ politeiad/backend/tlogbe/store/store.go | 14 +- politeiad/backend/tlogbe/tlog/testing.go | 4 +- politeiad/backend/tlogbe/tlog/tlog.go | 46 +++- politeiad/backend/tlogbe/tlogbe.go | 6 +- politeiad/config.go | 29 ++- politeiad/log.go | 17 +- politeiad/politeiad.go | 5 +- 12 files changed, 362 insertions(+), 71 deletions(-) rename politeiad/backend/tlogbe/store/{filesystem/filesystem.go => fs/fs.go} (77%) rename politeiad/backend/tlogbe/store/{filesystem => fs}/log.go (97%) create mode 100644 politeiad/backend/tlogbe/store/mysql/log.go create mode 100644 politeiad/backend/tlogbe/store/mysql/mysql.go diff --git a/go.mod b/go.mod index 1ee89f815..a11e2cd64 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/decred/dcrtime/api/v2 v2.0.0-20200912200806-b1e4dbc46be9 github.com/decred/go-socks v1.1.0 github.com/decred/slog v1.1.0 + github.com/go-sql-driver/mysql v1.5.0 github.com/go-test/deep v1.0.1 github.com/golang/protobuf v1.4.2 github.com/golang/snappy v0.0.1 // indirect diff --git a/politeiad/backend/tlogbe/store/filesystem/filesystem.go b/politeiad/backend/tlogbe/store/fs/fs.go similarity index 77% rename from politeiad/backend/tlogbe/store/filesystem/filesystem.go rename to politeiad/backend/tlogbe/store/fs/fs.go index ed7401b78..42624ed21 100644 --- a/politeiad/backend/tlogbe/store/filesystem/filesystem.go +++ b/politeiad/backend/tlogbe/store/fs/fs.go @@ -1,8 +1,8 @@ -// Copyright (c) 2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package filesystem +package fs import ( "errors" @@ -17,13 +17,15 @@ import ( ) var ( - _ store.Blob = (*fileSystem)(nil) + _ store.BlobKV = (*fs)(nil) + + errNotFound = errors.New("not found") ) -// fileSystem implements the Blob interface using the file system. +// fs implements the store.BlobKV interface using the file system. // // This implementation should be used for TESTING ONLY. -type fileSystem struct { +type fs struct { sync.RWMutex root string // Location of files } @@ -31,16 +33,16 @@ type fileSystem struct { // put saves a files to the file system. // // This function must be called WITH the lock held. -func (f *fileSystem) put(key string, value []byte) error { +func (f *fs) put(key string, value []byte) error { return ioutil.WriteFile(filepath.Join(f.root, key), value, 0600) } // This function must be called WITH the lock held. -func (f *fileSystem) del(key string) error { +func (f *fs) del(key string) error { err := os.Remove(filepath.Join(f.root, key)) if err != nil { if os.IsNotExist(err) { - return store.ErrNotFound + return errNotFound } return err } @@ -50,11 +52,11 @@ func (f *fileSystem) del(key string) error { // get retrieves a file from the file system. // // This function must be called WITH the lock held. -func (f *fileSystem) get(key string) ([]byte, error) { +func (f *fs) get(key string) ([]byte, error) { b, err := ioutil.ReadFile(filepath.Join(f.root, key)) if err != nil { if os.IsNotExist(err) { - return nil, store.ErrNotFound + return nil, errNotFound } return nil, err } @@ -64,8 +66,8 @@ func (f *fileSystem) get(key string) ([]byte, error) { // Put saves the provided blobs to the file system. The keys for the blobs // are generated in this function and returned. // -// This function satisfies the Blob interface. -func (f *fileSystem) Put(blobs [][]byte) ([]string, error) { +// This function satisfies the BlobKV interface. +func (f *fs) Put(blobs [][]byte) ([]string, error) { log.Tracef("Put: %v", len(blobs)) f.Lock() @@ -99,8 +101,8 @@ func (f *fileSystem) Put(blobs [][]byte) ([]string, error) { // Del deletes the files from the file system that correspond to the provided // keys. // -// This function satisfies the Blob interface. -func (f *fileSystem) Del(keys []string) error { +// This function satisfies the BlobKV interface. +func (f *fs) Del(keys []string) error { log.Tracef("Del: %v", keys) f.Lock() @@ -121,7 +123,7 @@ func (f *fileSystem) Del(keys []string) error { for _, v := range keys { err := f.del(v) if err != nil { - if errors.Is(err, store.ErrNotFound) { + if errors.Is(err, errNotFound) { // File does not exist. This is ok. continue } @@ -152,8 +154,8 @@ func (f *fileSystem) Del(keys []string) error { // responsibility of the caller to ensure a blob was returned for all provided // keys. // -// This function satisfies the Blob interface. -func (f *fileSystem) Get(keys []string) (map[string][]byte, error) { +// This function satisfies the BlobKV interface. +func (f *fs) Get(keys []string) (map[string][]byte, error) { log.Tracef("Get: %v", keys) f.RLock() @@ -163,7 +165,7 @@ func (f *fileSystem) Get(keys []string) (map[string][]byte, error) { for _, v := range keys { b, err := f.get(v) if err != nil { - if errors.Is(err, store.ErrNotFound) { + if errors.Is(err, errNotFound) { // File does not exist. This is ok. continue } @@ -177,9 +179,7 @@ func (f *fileSystem) Get(keys []string) (map[string][]byte, error) { // Enum enumerates over all blobs in the store, invoking the provided function // for each blob. -// -// This function satisfies the Blob interface. -func (f *fileSystem) Enum(cb func(key string, blob []byte) error) error { +func (f *fs) Enum(cb func(key string, blob []byte) error) error { log.Tracef("Enum") f.RLock() @@ -209,12 +209,12 @@ func (f *fileSystem) Enum(cb func(key string, blob []byte) error) error { // Closes closes the blob store connection. // -// This function satisfies the Blob interface. -func (f *fileSystem) Close() {} +// This function satisfies the BlobKV interface. +func (f *fs) Close() {} -// New returns a new fileSystem. -func New(root string) *fileSystem { - return &fileSystem{ +// New returns a new fs. +func New(root string) *fs { + return &fs{ root: root, } } diff --git a/politeiad/backend/tlogbe/store/filesystem/log.go b/politeiad/backend/tlogbe/store/fs/log.go similarity index 97% rename from politeiad/backend/tlogbe/store/filesystem/log.go rename to politeiad/backend/tlogbe/store/fs/log.go index 7234a6148..07b628af4 100644 --- a/politeiad/backend/tlogbe/store/filesystem/log.go +++ b/politeiad/backend/tlogbe/store/fs/log.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package filesystem +package fs import "github.com/decred/slog" diff --git a/politeiad/backend/tlogbe/store/mysql/log.go b/politeiad/backend/tlogbe/store/mysql/log.go new file mode 100644 index 000000000..62f76fd60 --- /dev/null +++ b/politeiad/backend/tlogbe/store/mysql/log.go @@ -0,0 +1,25 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package mysql + +import "github.com/decred/slog" + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log = slog.Disabled + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = slog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using slog. +func UseLogger(logger slog.Logger) { + log = logger +} diff --git a/politeiad/backend/tlogbe/store/mysql/mysql.go b/politeiad/backend/tlogbe/store/mysql/mysql.go new file mode 100644 index 000000000..8215f51a2 --- /dev/null +++ b/politeiad/backend/tlogbe/store/mysql/mysql.go @@ -0,0 +1,232 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package mysql + +import ( + "context" + "database/sql" + "fmt" + "net/url" + "time" + + "github.com/decred/politeia/politeiad/backend/tlogbe/store" + "github.com/google/uuid" +) + +const ( + // Database options + connTimeout = 1 * time.Minute + connMaxLifetime = 1 * time.Minute + maxOpenConns = 0 // 0 is unlimited + maxIdleConns = 10 +) + +// tableKeyValue defines the key-value table. The key is a uuid. +const tableKeyValue = ` + k CHAR(36) NOT NULL PRIMARY KEY, + v LONGBLOB NOT NULL +` + +var ( + _ store.BlobKV = (*mysql)(nil) +) + +// mysql implements the store BlobKV interface using a mysql driver. +type mysql struct { + db *sql.DB +} + +func ctxWithTimeout() (context.Context, func()) { + return context.WithTimeout(context.Background(), connTimeout) +} + +// Put saves the provided blobs to the store The keys for the blobs are +// returned using the same odering that the blobs were provided in. This +// operation is performed atomically. +// +// This function satisfies the store BlobKV interface. +func (s *mysql) Put(blobs [][]byte) ([]string, error) { + log.Tracef("Put: %v blobs", len(blobs)) + + ctx, cancel := ctxWithTimeout() + defer cancel() + + // Start transaction + opts := &sql.TxOptions{ + Isolation: sql.LevelDefault, + } + tx, err := s.db.BeginTx(ctx, opts) + if err != nil { + return nil, err + } + + // Save blobs + keys := make([]string, 0, len(blobs)) + for _, v := range blobs { + k := uuid.New().String() + _, err = tx.ExecContext(ctx, "INSERT INTO kv (k, v) VALUES (?, ?);", k, v) + if err != nil { + // Attempt to roll back the transaction + if err2 := tx.Rollback(); err2 != nil { + // We're in trouble! + e := fmt.Sprintf("put: %v, unable to rollback: %v", err, err2) + panic(e) + } + } + + keys = append(keys, k) + } + + // Commit transaction + err = tx.Commit() + if err != nil { + return nil, fmt.Errorf("commit: %v", err) + } + + return keys, nil +} + +// Del deletes the provided blobs from the store This operation is performed +// atomically. +// +// This function satisfies the store BlobKV interface. +func (s *mysql) Del(keys []string) error { + log.Tracef("Del: %v", keys) + + ctx, cancel := ctxWithTimeout() + defer cancel() + + // Start transaction + opts := &sql.TxOptions{ + Isolation: sql.LevelDefault, + } + tx, err := s.db.BeginTx(ctx, opts) + if err != nil { + return err + } + + // Delete blobs + for _, v := range keys { + _, err = tx.ExecContext(ctx, "DELETE FROM kv WHERE k IN (?);", v) + if err != nil { + // Attempt to roll back the transaction + if err2 := tx.Rollback(); err2 != nil { + // We're in trouble! + e := fmt.Sprintf("del: %v, unable to rollback: %v", err, err2) + panic(e) + } + } + } + + // Commit transaction + err = tx.Commit() + if err != nil { + return fmt.Errorf("commit: %v", err) + } + + return nil +} + +// Get returns blobs from the store for the provided keys. An entry will not +// exist in the returned map if for any blobs that are not found. It is the +// responsibility of the caller to ensure a blob was returned for all provided +// keys. +// +// This function satisfies the store BlobKV interface. +func (s *mysql) Get(keys []string) (map[string][]byte, error) { + log.Tracef("Get: %v", keys) + + ctx, cancel := ctxWithTimeout() + defer cancel() + + // Build query. A placeholder parameter (?) is required for each + // key being requested. + // + // Ex 3 keys: "SELECT k, v FROM kv WHERE k IN (?, ?, ?)" + sql := "SELECT k, v FROM kv WHERE k IN (" + for i := 0; i < len(keys); i++ { + sql += "?" + // Don't add a comma on the last one + if i < len(keys)-1 { + sql += "," + } + } + sql += ");" + + // The keys must be converted to []interface{} for the query method + // to accept them. + args := make([]interface{}, len(keys)) + for i, v := range keys { + args[i] = v + } + + // Get blobs + rows, err := s.db.QueryContext(ctx, sql, args...) + if err != nil { + return nil, fmt.Errorf("query: %v", err) + } + defer rows.Close() + + reply := make(map[string][]byte, len(keys)) + for rows.Next() { + var k string + var v []byte + err = rows.Scan(&k, &v) + if err != nil { + return nil, fmt.Errorf("scan: %v", err) + } + reply[k] = v + } + err = rows.Err() + if err != nil { + return nil, fmt.Errorf("next: %v", err) + } + + return reply, nil +} + +// Closes closes the blob store connection. +func (s *mysql) Close() { + s.db.Close() +} + +func New(host, user, dbname, sslRootCert, sslCert, sslKey string) (*mysql, error) { + // Setup database connection + v := url.Values{} + v.Add("sslmode", "require") + v.Add("sslca", sslRootCert) + v.Add("sslcert", sslCert) + v.Add("sslkey", sslKey) + + h := fmt.Sprintf("%v@tcp(%v)/%v?%v", user, host, dbname, v.Encode()) + log.Infof("Store host: %v", h) + + db, err := sql.Open("mysql", h) + if err != nil { + return nil, err + } + + // Setup database options + db.SetConnMaxLifetime(connMaxLifetime) + db.SetMaxOpenConns(maxOpenConns) + db.SetMaxIdleConns(maxIdleConns) + + // Verify database connection + err = db.Ping() + if err != nil { + return nil, err + } + + // Setup key-value table + sql := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS kv (%s)`, tableKeyValue) + _, err = db.Exec(sql) + if err != nil { + return nil, fmt.Errorf("create table: %v", err) + } + + return &mysql{ + db: db, + }, nil +} diff --git a/politeiad/backend/tlogbe/store/store.go b/politeiad/backend/tlogbe/store/store.go index 4468844ac..c746e40b9 100644 --- a/politeiad/backend/tlogbe/store/store.go +++ b/politeiad/backend/tlogbe/store/store.go @@ -10,7 +10,6 @@ import ( "encoding/base64" "encoding/gob" "encoding/hex" - "errors" "github.com/decred/politeia/util" ) @@ -20,11 +19,6 @@ const ( DataTypeStructure = "struct" ) -var ( - // ErrNotFound is emitted when a blob is not found. - ErrNotFound = errors.New("not found") -) - // DataDescriptor provides hints about a data blob. In practise we JSON encode // this struture and stuff it into BlobEntry.DataHint. type DataDescriptor struct { @@ -81,8 +75,8 @@ func Deblob(blob []byte) (*BlobEntry, error) { return &be, nil } -// Blob represents a blob key-value store. -type Blob interface { +// BlobKV represents a blob key-value store. +type BlobKV interface { // Put saves the provided blobs to the store. The keys for the // blobs are returned using the same odering that the blobs were // provided in. This operation is performed atomically. @@ -98,10 +92,6 @@ type Blob interface { // was returned for all provided keys. Get(keys []string) (map[string][]byte, error) - // Enum enumerates over all blobs in the store, invoking the - // provided function for each blob. - Enum(func(key string, blob []byte) error) error - // Closes closes the blob store connection. Close() } diff --git a/politeiad/backend/tlogbe/tlog/testing.go b/politeiad/backend/tlogbe/tlog/testing.go index 414ac9807..132044a06 100644 --- a/politeiad/backend/tlogbe/tlog/testing.go +++ b/politeiad/backend/tlogbe/tlog/testing.go @@ -8,7 +8,7 @@ import ( "io/ioutil" "testing" - "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" + "github.com/decred/politeia/politeiad/backend/tlogbe/store/fs" "github.com/marcopeereboom/sbox" ) @@ -27,7 +27,7 @@ func newTestTlog(t *testing.T, tlogID, dataDir string, encrypt bool) *Tlog { if err != nil { t.Fatal(err) } - store := filesystem.New(fp) + store := fs.New(fp) // Setup encryptin key if specified var ek *encryptionKey diff --git a/politeiad/backend/tlogbe/tlog/tlog.go b/politeiad/backend/tlogbe/tlog/tlog.go index adef65133..2e8720c1d 100644 --- a/politeiad/backend/tlogbe/tlog/tlog.go +++ b/politeiad/backend/tlogbe/tlog/tlog.go @@ -20,7 +20,8 @@ import ( "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" "github.com/decred/politeia/politeiad/backend/tlogbe/store" - "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" + "github.com/decred/politeia/politeiad/backend/tlogbe/store/fs" + "github.com/decred/politeia/politeiad/backend/tlogbe/store/mysql" "github.com/decred/politeia/util" "github.com/google/trillian" "github.com/robfig/cron" @@ -28,6 +29,10 @@ import ( ) const ( + DBTypeFileSystem = "filesystem" + DBTypeMySQL = "mysql" + dbUser = "politeiad" + defaultTrillianKeyFilename = "trillian.key" defaultStoreDirname = "store" @@ -63,7 +68,7 @@ type Tlog struct { dataDir string activeNetParams *chaincfg.Params trillian trillianClient - store store.Blob + store store.BlobKV dcrtime *dcrtimeClient cron *cron.Cron plugins map[string]plugin // [pluginID]plugin @@ -1310,7 +1315,7 @@ func (t *Tlog) Close() { } } -func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianKeyFile, encryptionKeyFile, dcrtimeHost, dcrtimeCert string) (*Tlog, error) { +func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianKeyFile, encryptionKeyFile, dbType, dbHost, dbRootCert, dbCert, dbKey, dcrtimeHost, dcrtimeCert string) (*Tlog, error) { // Load encryption key if provided. An encryption key is optional. var ek *encryptionKey if encryptionKeyFile != "" { @@ -1339,14 +1344,6 @@ func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trilli return nil, err } - // Setup key-value store - fp := filepath.Join(dataDir, defaultStoreDirname) - err = os.MkdirAll(fp, 0700) - if err != nil { - return nil, err - } - store := filesystem.New(fp) - // Setup trillian client if trillianKeyFile == "" { // No file path was given. Use the default path. @@ -1362,6 +1359,31 @@ func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trilli return nil, err } + // Setup key-value store + log.Infof("Database type %v: %v", id, dbType) + var kvstore store.BlobKV + switch dbType { + case DBTypeFileSystem: + fp := filepath.Join(dataDir, defaultStoreDirname) + err = os.MkdirAll(fp, 0700) + if err != nil { + return nil, err + } + kvstore = fs.New(fp) + + case DBTypeMySQL: + // Example db name: testnet3_unvetted_kv + dbName := fmt.Sprintf("%v_%v_kv", anp.Name, id) + kvstore, err = mysql.New(dbHost, dbUser, dbName, + dbRootCert, dbCert, dbKey) + if err != nil { + return nil, err + } + + default: + return nil, fmt.Errorf("invalid db type: %v", dbType) + } + // Setup dcrtime client dcrtimeClient, err := newDcrtimeClient(dcrtimeHost, dcrtimeCert) if err != nil { @@ -1374,7 +1396,7 @@ func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trilli dataDir: dataDir, activeNetParams: anp, trillian: trillianClient, - store: store, + store: kvstore, dcrtime: dcrtimeClient, cron: cron.New(), plugins: make(map[string]plugin), diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 69d7a322e..5d929523d 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -1752,7 +1752,7 @@ func (t *tlogBackend) setup() error { } // New returns a new tlogBackend. -func New(anp *chaincfg.Params, homeDir, dataDir, unvettedTrillianHost, unvettedTrillianKeyFile, vettedTrillianHost, vettedTrillianKeyFile, encryptionKeyFile, dcrtimeHost, dcrtimeCert string) (*tlogBackend, error) { +func New(anp *chaincfg.Params, homeDir, dataDir, unvettedTrillianHost, unvettedTrillianKeyFile, vettedTrillianHost, vettedTrillianKeyFile, encryptionKeyFile, dbType, dbHost, dbRootCert, dbCert, dbKey, dcrtimeHost, dcrtimeCert string) (*tlogBackend, error) { // Setup encryption key file if encryptionKeyFile == "" { // No file path was given. Use the default path. @@ -1783,13 +1783,13 @@ func New(anp *chaincfg.Params, homeDir, dataDir, unvettedTrillianHost, unvettedT // Setup tlog instances unvetted, err := tlog.New(tlogIDUnvetted, homeDir, dataDir, anp, unvettedTrillianHost, unvettedTrillianKeyFile, encryptionKeyFile, - dcrtimeHost, dcrtimeCert) + dbType, dbHost, dbRootCert, dbCert, dbKey, dcrtimeHost, dcrtimeCert) if err != nil { return nil, fmt.Errorf("new tlog unvetted: %v", err) } vetted, err := tlog.New(tlogIDVetted, homeDir, dataDir, anp, vettedTrillianHost, vettedTrillianKeyFile, "", - dcrtimeHost, dcrtimeCert) + dbType, dbHost, dbRootCert, dbCert, dbKey, dcrtimeHost, dcrtimeCert) if err != nil { return nil, fmt.Errorf("new tlog vetted: %v", err) } diff --git a/politeiad/config.go b/politeiad/config.go index 8f8f4a702..7ea1e7ad1 100644 --- a/politeiad/config.go +++ b/politeiad/config.go @@ -18,6 +18,7 @@ import ( "strings" v1 "github.com/decred/dcrtime/api/v1" + "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" "github.com/decred/politeia/politeiad/sharedconfig" "github.com/decred/politeia/util" "github.com/decred/politeia/util/version" @@ -45,6 +46,7 @@ const ( defaultTrillianHostUnvetted = "localhost:8090" defaultTrillianHostVetted = "localhost:8094" + defaultDBType = tlog.DBTypeFileSystem ) var ( @@ -88,15 +90,23 @@ type config struct { GitTrace bool `long:"gittrace" description:"Enable git tracing in logs"` DcrdataHost string `long:"dcrdatahost" description:"Dcrdata ip:port"` - // TODO validate these config params - Backend string `long:"backend"` - TrillianHostUnvetted string `long:"trillianhostunvetted"` - TrillianHostVetted string `long:"trillianhostvetted"` - TrillianKeyUnvetted string `long:"trilliankeyunvetted"` - TrillianKeyVetted string `long:"trilliankeyvetted"` - EncryptionKey string `long:"encryptionkey"` - Plugins []string `long:"plugin"` - PluginSettings []string `long:"pluginsetting"` + // TODO validate these config params and set defaults. Also consider + // making them specific to tstore. Ex: tstore.TrillianHostUnvetted. + Backend string `long:"backend"` + TrillianHostUnvetted string `long:"trillianhostunvetted"` + TrillianHostVetted string `long:"trillianhostvetted"` + TrillianKeyUnvetted string `long:"trilliankeyunvetted"` + TrillianKeyVetted string `long:"trilliankeyvetted"` + EncryptionKey string `long:"encryptionkey"` + DBType string `long:"dbtype"` + DBHost string `long:"dbhost" description:"Database ip:port"` + DBRootCert string `long:"dbrootcert" description:"File containing the CA certificate for the database"` + DBCert string `long:"dbcert" description:"File containing the client certificate for the database"` + DBKey string `long:"dbkey" description:"File containing the client certificate key for the database"` + + // Plugin settings + Plugins []string `long:"plugin"` + PluginSettings []string `long:"pluginsetting"` } // serviceOptions defines the configuration options for the daemon as a service @@ -248,6 +258,7 @@ func loadConfig() (*config, []string, error) { Backend: defaultBackend, TrillianHostUnvetted: defaultTrillianHostUnvetted, TrillianHostVetted: defaultTrillianHostVetted, + DBType: defaultDBType, } // Service options which are only added on Windows. diff --git a/politeiad/log.go b/politeiad/log.go index 4ea5f870f..7088aaaad 100644 --- a/politeiad/log.go +++ b/politeiad/log.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Decred developers +// Copyright (c) 2017-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -15,7 +15,8 @@ import ( "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/dcrdata" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/ticketvote" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/user" - "github.com/decred/politeia/politeiad/backend/tlogbe/store/filesystem" + "github.com/decred/politeia/politeiad/backend/tlogbe/store/fs" + "github.com/decred/politeia/politeiad/backend/tlogbe/store/mysql" "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" "github.com/decred/politeia/wsdcrdata" "github.com/decred/slog" @@ -59,15 +60,23 @@ var ( // Initialize package-global logger variables. func init() { + // Backend loggers gitbe.UseLogger(gitbeLog) tlogbe.UseLogger(tlogbeLog) + + // Tlog loggers tlog.UseLogger(tlogLog) - filesystem.UseLogger(tlogLog) - wsdcrdata.UseLogger(wsdcrdataLog) + fs.UseLogger(tlogLog) + mysql.UseLogger(tlogLog) + + // Plugin loggers comments.UseLogger(pluginLog) dcrdata.UseLogger(pluginLog) ticketvote.UseLogger(pluginLog) user.UseLogger(pluginLog) + + // Other loggers + wsdcrdata.UseLogger(wsdcrdataLog) } // subsystemLoggers maps each subsystem identifier to its associated logger. diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 1d257d68a..f2a59ddd9 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -1484,8 +1484,9 @@ func _main() error { case backendTlog: b, err := tlogbe.New(activeNetParams.Params, cfg.HomeDir, cfg.DataDir, cfg.TrillianHostUnvetted, cfg.TrillianKeyUnvetted, - cfg.TrillianHostVetted, cfg.TrillianKeyVetted, - cfg.EncryptionKey, cfg.DcrtimeHost, cfg.DcrtimeCert) + cfg.TrillianHostVetted, cfg.TrillianKeyVetted, cfg.EncryptionKey, + cfg.DBType, cfg.DBHost, cfg.DBRootCert, cfg.DBCert, cfg.DBKey, + cfg.DcrtimeHost, cfg.DcrtimeCert) if err != nil { return fmt.Errorf("new tlogbe: %v", err) } From d64723fa2e9bc3a7f07a90590650aabde27f9450 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 15 Feb 2021 09:32:56 -0600 Subject: [PATCH 305/449] Fix vote inv locking bug. --- .../tlogbe/plugins/ticketvote/inventory.go | 15 ++++++++++----- politeiad/plugins/ticketvote/ticketvote.go | 2 -- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go b/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go index a78291ec6..77574bf2e 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go @@ -121,10 +121,8 @@ func (p *ticketVotePlugin) invAdd(token string, s ticketvote.VoteStatusT) error return nil } -func (p *ticketVotePlugin) invUpdate(token string, s ticketvote.VoteStatusT, endHeight uint32) error { - p.mtxInv.Lock() - defer p.mtxInv.Unlock() - +// This function must be called WITH the read/write lock held. +func (p *ticketVotePlugin) invUpdateLocked(token string, s ticketvote.VoteStatusT, endHeight uint32) error { // Get inventory inv, err := p.invGetLocked() if err != nil { @@ -158,6 +156,13 @@ func (p *ticketVotePlugin) invUpdate(token string, s ticketvote.VoteStatusT, end return nil } +func (p *ticketVotePlugin) invUpdate(token string, s ticketvote.VoteStatusT, endHeight uint32) error { + p.mtxInv.Lock() + defer p.mtxInv.Unlock() + + return p.invUpdateLocked(token, s, endHeight) +} + func (p *ticketVotePlugin) invUpdateForBlock(bestBlock uint32) (*inventory, error) { p.mtxInv.Lock() defer p.mtxInv.Unlock() @@ -204,7 +209,7 @@ func (p *ticketVotePlugin) invUpdateForBlock(bestBlock uint32) (*inventory, erro case ticketvote.VoteStatusFinished, ticketvote.VoteStatusApproved, ticketvote.VoteStatusRejected: // These statuses are allowed - err := p.invUpdate(v.Token, sr.Status, 0) + err := p.invUpdateLocked(v.Token, sr.Status, 0) if err != nil { return nil, err } diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index caf067507..b0c33b892 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -6,8 +6,6 @@ // tickets to participate. package ticketvote -// TODO should VoteDetails, StartReply, StartRunoffReply contain a receipt? - const ( // PluginID is the ticketvote plugin ID. PluginID = "ticketvote" From 9000ed53b4f8da91a78a089faa287f20faa5719d Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 15 Feb 2021 13:22:56 -0600 Subject: [PATCH 306/449] Paginate vote timestamps. --- politeiad/backend/tlogbe/inventory.go | 2 +- .../backend/tlogbe/plugins/ticketvote/cmds.go | 109 ++++++++++-------- .../tlogbe/plugins/ticketvote/ticketvote.go | 2 +- politeiad/client/ticketvote.go | 10 +- politeiad/plugins/ticketvote/ticketvote.go | 40 +++++-- politeiawww/api/ticketvote/v1/v1.go | 25 +++- politeiawww/cmd/pictl/cmdvotetimestamps.go | 26 +++-- politeiawww/cmd/pictl/ticketvote.go | 2 +- politeiawww/ticketvote/process.go | 20 +++- 9 files changed, 154 insertions(+), 82 deletions(-) diff --git a/politeiad/backend/tlogbe/inventory.go b/politeiad/backend/tlogbe/inventory.go index 48151f48e..d97f626b6 100644 --- a/politeiad/backend/tlogbe/inventory.go +++ b/politeiad/backend/tlogbe/inventory.go @@ -415,7 +415,7 @@ func tokensParse(entries []entry, s backend.MDStatusT, countPerPage, page uint32 if foundCount >= startAt { tokens = append(tokens, v.Token) if len(tokens) == int(countPerPage) { - // We got a full page. We're done. + // We have a full page. We're done. return tokens } } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go index 075e21505..eaa8d2002 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go @@ -2619,67 +2619,84 @@ func (p *ticketVotePlugin) cmdInventory(payload string) (string, error) { return string(reply), nil } -func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte) (string, error) { - log.Tracef("cmdTimestamps: %v %x", treeID, token) +func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload string) (string, error) { + log.Tracef("cmdTimestamps: %v %x %v", treeID, token, payload) - // Get authorization timestamps - digests, err := p.tlog.DigestsByDataType(treeID, dataTypeAuthDetails) + // Decode payload + var t ticketvote.Timestamps + err := json.Unmarshal([]byte(payload), &t) if err != nil { - return "", fmt.Errorf("DigestByDataType %v %v: %v", - treeID, dataTypeAuthDetails, err) + return "", err } - auths := make([]ticketvote.Timestamp, 0, len(digests)) - for _, v := range digests { - ts, err := p.timestamp(treeID, v) + var ( + auths []ticketvote.Timestamp + details *ticketvote.Timestamp + + pageSize = ticketvote.VoteTimestampsPageSize + votes = make([]ticketvote.Timestamp, 0, pageSize) + ) + switch { + case t.VotesPage > 0: + // Return a page of vote timestamps + digests, err := p.tlog.DigestsByDataType(treeID, dataTypeCastVoteDetails) if err != nil { - return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) + return "", fmt.Errorf("digestsByKeyPrefix %v %v: %v", + treeID, dataTypeVoteDetails, err) } - auths = append(auths, *ts) - } - // Get vote details timestamp. There should never be more than one - // vote details. - digests, err = p.tlog.DigestsByDataType(treeID, dataTypeVoteDetails) - if err != nil { - return "", fmt.Errorf("DigestsByDataType %v %v: %v", - treeID, dataTypeVoteDetails, err) - } - if len(digests) > 1 { - return "", fmt.Errorf("invalid vote details count: got %v, want 1", - len(digests)) - } - - var details ticketvote.Timestamp - for _, v := range digests { - ts, err := p.timestamp(treeID, v) - if err != nil { - return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) + startAt := (t.VotesPage - 1) * pageSize + for i, v := range digests { + if i < int(startAt) { + continue + } + ts, err := p.timestamp(treeID, v) + if err != nil { + return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) + } + votes = append(votes, *ts) + if len(votes) == int(pageSize) { + // We have a full page. We're done. + break + } } - details = *ts - } - // Get cast vote timestamps - digests, err = p.tlog.DigestsByDataType(treeID, dataTypeCastVoteDetails) - if err != nil { - return "", fmt.Errorf("digestsByKeyPrefix %v %v: %v", - treeID, dataTypeVoteDetails, err) - } + default: + // Return authorization timestamps and the vote details timestamp. - votes := make(map[string]ticketvote.Timestamp, len(digests)) - for _, v := range digests { - ts, err := p.timestamp(treeID, v) + // Auth timestamps + digests, err := p.tlog.DigestsByDataType(treeID, dataTypeAuthDetails) if err != nil { - return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) + return "", fmt.Errorf("DigestByDataType %v %v: %v", + treeID, dataTypeAuthDetails, err) + } + auths = make([]ticketvote.Timestamp, 0, len(digests)) + for _, v := range digests { + ts, err := p.timestamp(treeID, v) + if err != nil { + return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) + } + auths = append(auths, *ts) } - var cv ticketvote.CastVoteDetails - err = json.Unmarshal([]byte(ts.Data), &cv) + // Vote details timestamp + digests, err = p.tlog.DigestsByDataType(treeID, dataTypeVoteDetails) if err != nil { - return "", err + return "", fmt.Errorf("DigestsByDataType %v %v: %v", + treeID, dataTypeVoteDetails, err) + } + // There should never be more than a one vote details + if len(digests) > 1 { + return "", fmt.Errorf("invalid vote details count: got %v, want 1", + len(digests)) + } + for _, v := range digests { + ts, err := p.timestamp(treeID, v) + if err != nil { + return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) + } + details = ts } - - votes[cv.Ticket] = *ts } // Prepare reply diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go index 1ee40a336..480be8310 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go @@ -172,7 +172,7 @@ func (p *ticketVotePlugin) Cmd(treeID int64, token []byte, cmd, payload string) case ticketvote.CmdInventory: return p.cmdInventory(payload) case ticketvote.CmdTimestamps: - return p.cmdTimestamps(treeID, token) + return p.cmdTimestamps(treeID, token, payload) // Internal plugin commands case cmdStartRunoffSubmission: diff --git a/politeiad/client/ticketvote.go b/politeiad/client/ticketvote.go index 021bc9966..c9bf07aec 100644 --- a/politeiad/client/ticketvote.go +++ b/politeiad/client/ticketvote.go @@ -376,15 +376,19 @@ func (c *Client) TicketVoteInventory(ctx context.Context, i ticketvote.Inventory // TicketVoteTimestamps sends the ticketvote plugin Timestamps command to the // politeiad v1 API. -func (c *Client) TicketVoteTimestamps(ctx context.Context, token string) (*ticketvote.TimestampsReply, error) { +func (c *Client) TicketVoteTimestamps(ctx context.Context, t ticketvote.Timestamps) (*ticketvote.TimestampsReply, error) { // Setup request + b, err := json.Marshal(t) + if err != nil { + return nil, err + } cmds := []pdv1.PluginCommandV2{ { State: pdv1.RecordStateVetted, ID: ticketvote.PluginID, Command: ticketvote.CmdTimestamps, - Token: token, - Payload: "", + Token: t.Token, + Payload: string(b), }, } diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index b0c33b892..31c7db841 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -223,7 +223,8 @@ type VoteParams struct { } // VoteDetails is the structure that is saved to disk when a vote is started. -// It contains all of the fields from a Start and a StartReply. +// It contains all of the fields from a Start and a StartReply. A vote details +// with the eligible tickets snapshot will be ~0.35MB. // // Signature is the client signature of the SHA256 digest of the JSON encoded // Vote struct. @@ -240,16 +241,20 @@ type VoteDetails struct { EligibleTickets []string `json:"eligibletickets"` // Ticket hashes } -// CastVoteDetails is the structure that is saved to disk when a vote is cast. +// CastVoteDetails contains the details of a cast vote. A JSON encoded cast +// vote details is 405 bytes (could vary slightly depending on the votebit). +// +// Signature is the client signature of the Token+Ticket+VoteBit. The receipt +// is the server signature of the client signature. type CastVoteDetails struct { // Data generated by client Token string `json:"token"` // Record token Ticket string `json:"ticket"` // Ticket hash - VoteBit string `json:"votebits"` // Selected vote bit, hex encoded - Signature string `json:"signature"` // Signature of Token+Ticket+VoteBit + VoteBit string `json:"votebits"` // Vote bit, hex encoded + Signature string `json:"signature"` // Client signature // Metdata generated by server - Receipt string `json:"receipt"` // Server signature of client signature + Receipt string `json:"receipt"` // Server signature } // AuthActionT represents the ticket vote authorization actions. @@ -574,12 +579,29 @@ type Timestamp struct { Proofs []Proof `json:"proofs"` } +const ( + // VoteTimetsampsPageSize is the maximum number of vote timestamps + // that will be returned for any single request. A vote timestamp + // is ~2000 bytes so a page of 100 votes will only be 0.2MB, but + // the bottleneck on this call is performance, not size. Its + // expensive to retreive a large number of inclusion proofs from + // trillian. A 100 timestamps will take ~1 second to compile. + VoteTimestampsPageSize uint32 = 100 +) + // Timestamps requests the timestamps for ticket vote data. -type Timestamps struct{} +// +// If no votes page number is provided then the vote authorization and vote +// details timestamps will be returned. If a votes page number is provided then +// the specified page of votes will be returned. +type Timestamps struct { + Token string `json:"token"` + VotesPage uint32 `json:"votespage,omitempty"` +} // TimestampsReply is the reply to the Timestamps command. type TimestampsReply struct { - Auths []Timestamp `json:"auths,omitempty"` - Details Timestamp `json:"details,omitempty"` - Votes map[string]Timestamp `json:"votes,omitempty"` // [ticket]Timestamp + Auths []Timestamp `json:"auths,omitempty"` + Details *Timestamp `json:"details,omitempty"` + Votes []Timestamp `json:"votes,omitempty"` } diff --git a/politeiawww/api/ticketvote/v1/v1.go b/politeiawww/api/ticketvote/v1/v1.go index 6df3b3291..19e243b1f 100644 --- a/politeiawww/api/ticketvote/v1/v1.go +++ b/politeiawww/api/ticketvote/v1/v1.go @@ -395,7 +395,8 @@ type AuthDetails struct { Receipt string `json:"receipt"` // Server sig of client sig } -// VoteDetails contains the details of a record vote. +// VoteDetails contains the details of a record vote. A vote details with the +// eligible tickets snapshot will be ~0.35MB. // // Signature is the client signature of the SHA256 digest of the JSON encoded // VoteParams struct. @@ -420,7 +421,8 @@ type DetailsReply struct { Vote *VoteDetails `json:"vote"` } -// CastVoteDetails contains the details of a cast vote. +// CastVoteDetails contains the details of a cast vote. A JSON encoded cast +// vote details is 405 bytes (could vary slightly depending on the votebit). // // Signature is the client signature of the Token+Ticket+VoteBit. type CastVoteDetails struct { @@ -575,14 +577,25 @@ type Timestamp struct { Proofs []Proof `json:"proofs"` } +const ( + // VoteTimetsampsPageSize is the maximum number of vote timestamps + // that will be returned for any single request. + VoteTimestampsPageSize uint32 = 100 +) + // Timestamps requests the timestamps for ticket vote data. +// +// If no votes page number is provided then the vote authorization and vote +// details timestamps will be returned. If a votes page number is provided then +// the specified page of votes will be returned. type Timestamps struct { - Token string `json:"token"` + Token string `json:"token"` + VotesPage uint32 `json:"votespage,omitempty"` } // TimestampsReply is the reply to the Timestamps command. type TimestampsReply struct { - Auths []Timestamp `json:"auths,omitempty"` - Details Timestamp `json:"details,omitempty"` - Votes map[string]Timestamp `json:"votes,omitempty"` // [ticket]Timestamp + Auths []Timestamp `json:"auths,omitempty"` + Details *Timestamp `json:"details,omitempty"` + Votes []Timestamp `json:"votes,omitempty"` } diff --git a/politeiawww/cmd/pictl/cmdvotetimestamps.go b/politeiawww/cmd/pictl/cmdvotetimestamps.go index 294291cc1..7fef7b0f5 100644 --- a/politeiawww/cmd/pictl/cmdvotetimestamps.go +++ b/politeiawww/cmd/pictl/cmdvotetimestamps.go @@ -16,7 +16,8 @@ import ( // cmdVoteTimestamps retrieves the timestamps for a politeiawww ticket vote. type cmdVoteTimestamps struct { Args struct { - Token string `positional-arg-name:"token" required:"true"` + Token string `positional-arg-name:"token" required:"true"` + VotesPage uint32 `positional-arg-name:"votespage" optional:"true"` } `positional-args:"true"` } @@ -39,7 +40,8 @@ func (c *cmdVoteTimestamps) Execute(args []string) error { // Get timestamps t := tkv1.Timestamps{ - Token: c.Args.Token, + Token: c.Args.Token, + VotesPage: c.Args.VotesPage, } tr, err := pc.TicketVoteTimestamps(t) if err != nil { @@ -53,9 +55,11 @@ func (c *cmdVoteTimestamps) Execute(args []string) error { return fmt.Errorf("verify authorization %v timestamp: %v", k, err) } } - err = verifyVoteTimestamp(tr.Details) - if err != nil { - return fmt.Errorf("verify vote details timestamp: %v", err) + if tr.Details != nil { + err = verifyVoteTimestamp(*tr.Details) + if err != nil { + return fmt.Errorf("verify vote details timestamp: %v", err) + } } for k, v := range tr.Votes { err = verifyVoteTimestamp(v) @@ -97,11 +101,15 @@ func convertVoteTimestamp(t tkv1.Timestamp) backend.Timestamp { } // voteTimestampsHelpMsg is printed to stdout by the help command. -const voteTimestampsHelpMsg = `votetimestamps "token" +const voteTimestampsHelpMsg = `votetimestamps "token" votepage + +Request the timestamps for ticket vote data. -Fetch the timestamps for a ticket vote. This includes timestamps for all -authorizations, the vote details, and all cast votes. +If no votes page number is provided then the vote authorization and vote +details timestamps will be returned. If a votes page number is provided then +the specified page of votes will be returned. Arguments: -1. token (string, required) Record token +1. token (string, required) Record token. +2. votepage (uint32, optional) Page number for cast vote timestamps. ` diff --git a/politeiawww/cmd/pictl/ticketvote.go b/politeiawww/cmd/pictl/ticketvote.go index 265d05852..b83179534 100644 --- a/politeiawww/cmd/pictl/ticketvote.go +++ b/politeiawww/cmd/pictl/ticketvote.go @@ -19,7 +19,7 @@ func printAuthDetails(a tkv1.AuthDetails) { func printVoteDetails(v tkv1.VoteDetails) { printf("Token : %v\n", v.Params.Token) - printf("Type : %v\n", v.Params.Type) + printf("Type : %v\n", tkv1.VoteTypes[v.Params.Type]) if v.Params.Type == tkv1.VoteTypeRunoff { printf("Parent : %v\n", v.Params.Parent) } diff --git a/politeiawww/ticketvote/process.go b/politeiawww/ticketvote/process.go index c8eccf955..b4a46db4d 100644 --- a/politeiawww/ticketvote/process.go +++ b/politeiawww/ticketvote/process.go @@ -215,10 +215,14 @@ func (t *TicketVote) processInventory(ctx context.Context, i v1.Inventory) (*v1. } func (t *TicketVote) processTimestamps(ctx context.Context, ts v1.Timestamps) (*v1.TimestampsReply, error) { - log.Tracef("processTimestamps: %v", ts.Token) + log.Tracef("processTimestamps: %v %v", ts.Token, ts.VotesPage) // Send plugin command - tsr, err := t.politeiad.TicketVoteTimestamps(ctx, ts.Token) + tt := ticketvote.Timestamps{ + Token: ts.Token, + VotesPage: ts.VotesPage, + } + tsr, err := t.politeiad.TicketVoteTimestamps(ctx, tt) if err != nil { return nil, err } @@ -226,15 +230,19 @@ func (t *TicketVote) processTimestamps(ctx context.Context, ts v1.Timestamps) (* // Prepare reply var ( auths = make([]v1.Timestamp, 0, len(tsr.Auths)) - votes = make(map[string]v1.Timestamp, len(tsr.Votes)) + votes = make([]v1.Timestamp, 0, len(tsr.Votes)) - details = convertTimestampToV1(tsr.Details) + details *v1.Timestamp ) + if tsr.Details != nil { + dt := convertTimestampToV1(*tsr.Details) + details = &dt + } for _, v := range tsr.Auths { auths = append(auths, convertTimestampToV1(v)) } - for k, v := range tsr.Votes { - votes[k] = convertTimestampToV1(v) + for _, v := range tsr.Votes { + votes = append(votes, convertTimestampToV1(v)) } return &v1.TimestampsReply{ From 33ad1acd3600e5097f2889843b55dfe2c2ee3a37 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 15 Feb 2021 14:30:26 -0600 Subject: [PATCH 307/449] Cleanup. --- politeiawww/comments/pi.go | 59 +++++++++++++++++++++++++++++ politeiawww/comments/process.go | 66 ++++++++++----------------------- politeiawww/records/pi.go | 19 +++++----- 3 files changed, 88 insertions(+), 56 deletions(-) create mode 100644 politeiawww/comments/pi.go diff --git a/politeiawww/comments/pi.go b/politeiawww/comments/pi.go new file mode 100644 index 000000000..767c135bb --- /dev/null +++ b/politeiawww/comments/pi.go @@ -0,0 +1,59 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package comments + +import ( + v1 "github.com/decred/politeia/politeiawww/api/comments/v1" + "github.com/decred/politeia/politeiawww/pi" + "github.com/decred/politeia/politeiawww/user" +) + +// paywallIsEnabled returns whether the user paywall is enabled. +// +// This function is a temporary function that will be removed once user plugins +// have been implemented. +func (c *Comments) paywallIsEnabled() bool { + return c.cfg.PaywallAmount != 0 && c.cfg.PaywallXpub != "" +} + +// userHasPaid returns whether the user has paid their user registration fee. +// +// This function is a temporary function that will be removed once user plugins +// have been implemented. +func userHasPaid(u user.User) bool { + return u.NewUserPaywallTx != "" +} + +func (c *Comments) piHookNewPre(u user.User) error { + if !c.paywallIsEnabled() { + return nil + } + + // Verify user has paid registration paywall + if !userHasPaid(u) { + return v1.PluginErrorReply{ + PluginID: pi.UserPluginID, + ErrorCode: pi.ErrorCodeUserRegistrationNotPaid, + } + } + + return nil +} + +func (c *Comments) piHookVotePre(u user.User) error { + if !c.paywallIsEnabled() { + return nil + } + + // Verify user has paid registration paywall + if !userHasPaid(u) { + return v1.PluginErrorReply{ + PluginID: pi.UserPluginID, + ErrorCode: pi.ErrorCodeUserRegistrationNotPaid, + } + } + + return nil +} diff --git a/politeiawww/comments/process.go b/politeiawww/comments/process.go index b6670c63c..c57e9581c 100644 --- a/politeiawww/comments/process.go +++ b/politeiawww/comments/process.go @@ -10,7 +10,6 @@ import ( "github.com/decred/politeia/politeiad/plugins/comments" v1 "github.com/decred/politeia/politeiawww/api/comments/v1" "github.com/decred/politeia/politeiawww/config" - "github.com/decred/politeia/politeiawww/pi" "github.com/decred/politeia/politeiawww/user" "github.com/google/uuid" ) @@ -18,19 +17,6 @@ import ( func (c *Comments) processNew(ctx context.Context, n v1.New, u user.User) (*v1.NewReply, error) { log.Tracef("processNew: %v %v %v", n.Token, u.Username) - // Execute pre plugin hooks. Checking the mode is a temporary - // measure until user plugins have been properly implemented. - switch c.cfg.Mode { - case config.PoliteiaWWWMode: - // Verify user has paid registration paywall - if !c.userHasPaid(u) { - return nil, v1.PluginErrorReply{ - PluginID: pi.UserPluginID, - ErrorCode: pi.ErrorCodeUserRegistrationNotPaid, - } - } - } - // Verify user signed using active identity if u.PublicKey() != n.PublicKey { return nil, v1.UserErrorReply{ @@ -55,6 +41,16 @@ func (c *Comments) processNew(ctx context.Context, n v1.New, u user.User) (*v1.N } } + // Execute pre plugin hooks. Checking the mode is a temporary + // measure until user plugins have been properly implemented. + switch c.cfg.Mode { + case config.PoliteiaWWWMode: + err := c.piHookNewPre(u) + if err != nil { + return nil, err + } + } + // Send plugin command cn := comments.New{ UserID: u.ID.String(), @@ -88,19 +84,6 @@ func (c *Comments) processNew(ctx context.Context, n v1.New, u user.User) (*v1.N func (c *Comments) processVote(ctx context.Context, v v1.Vote, u user.User) (*v1.VoteReply, error) { log.Tracef("processVote: %v %v %v", v.Token, v.CommentID, v.Vote) - // Execute pre plugin hooks. Checking the mode is a temporary - // measure until user plugins have been properly implemented. - switch c.cfg.Mode { - case config.PoliteiaWWWMode: - // Verify user has paid registration paywall - if !c.userHasPaid(u) { - return nil, v1.PluginErrorReply{ - PluginID: pi.UserPluginID, - ErrorCode: pi.ErrorCodeUserRegistrationNotPaid, - } - } - } - // Verify user signed using active identity if u.PublicKey() != v.PublicKey { return nil, v1.UserErrorReply{ @@ -109,6 +92,16 @@ func (c *Comments) processVote(ctx context.Context, v v1.Vote, u user.User) (*v1 } } + // Execute pre plugin hooks. Checking the mode is a temporary + // measure until user plugins have been properly implemented. + switch c.cfg.Mode { + case config.PoliteiaWWWMode: + err := c.piHookVotePre(u) + if err != nil { + return nil, err + } + } + // Send plugin command cv := comments.Vote{ UserID: u.ID.String(), @@ -384,22 +377,3 @@ func convertTimestamp(t comments.Timestamp) v1.Timestamp { Proofs: proofs, } } - -// paywallIsEnabled returns whether the user paywall is enabled. -// -// This function is a temporary function that will be removed once user plugins -// have been implemented. -func (c *Comments) paywallIsEnabled() bool { - return c.cfg.PaywallAmount != 0 && c.cfg.PaywallXpub != "" -} - -// userHasPaid returns whether the user has paid their user registration fee. -// -// This function is a temporary function that will be removed once user plugins -// have been implemented. -func (c *Comments) userHasPaid(u user.User) bool { - if !c.paywallIsEnabled() { - return true - } - return u.NewUserPaywallTx != "" -} diff --git a/politeiawww/records/pi.go b/politeiawww/records/pi.go index c7343d447..421b9417c 100644 --- a/politeiawww/records/pi.go +++ b/politeiawww/records/pi.go @@ -24,10 +24,7 @@ func (r *Records) paywallIsEnabled() bool { // // This function is a temporary function that will be removed once user plugins // have been implemented. -func (r *Records) userHasPaid(u user.User) bool { - if !r.paywallIsEnabled() { - return true - } +func userHasPaid(u user.User) bool { return u.NewUserPaywallTx != "" } @@ -46,11 +43,6 @@ func userHasProposalCredits(u user.User) bool { // This function is a temporary function that will be removed once user plugins // have been implemented. func (r *Records) spendProposalCredit(u user.User, token string) error { - // Skip if the paywall is enabled - if !r.paywallIsEnabled() { - return nil - } - // Verify there are credits to be spent if !userHasProposalCredits(u) { return fmt.Errorf("no proposal credits found") @@ -70,8 +62,12 @@ func (r *Records) spendProposalCredit(u user.User, token string) error { // This function is a temporary function that will be removed once user plugins // have been implemented. func (r *Records) piHookNewRecordPre(u user.User) error { + if !r.paywallIsEnabled() { + return nil + } + // Verify user has paid registration paywall - if !r.userHasPaid(u) { + if !userHasPaid(u) { return v1.PluginErrorReply{ PluginID: pi.UserPluginID, ErrorCode: pi.ErrorCodeUserRegistrationNotPaid, @@ -93,5 +89,8 @@ func (r *Records) piHookNewRecordPre(u user.User) error { // This function is a temporary function that will be removed once user plugins // have been implemented. func (r *Records) piHookNewRecordPost(u user.User, token string) error { + if !r.paywallIsEnabled() { + return nil + } return r.spendProposalCredit(u, token) } From 5b13b88097ce255ff42df50f723f39c95623d19a Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 15 Feb 2021 15:25:25 -0600 Subject: [PATCH 308/449] Record inventory bug fix. --- politeiawww/records/error.go | 2 -- politeiawww/records/process.go | 32 ++++++++++++++++++-------------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/politeiawww/records/error.go b/politeiawww/records/error.go index 9f469f23a..b2bb678b9 100644 --- a/politeiawww/records/error.go +++ b/politeiawww/records/error.go @@ -12,7 +12,6 @@ import ( "strings" "time" - "github.com/davecgh/go-spew/spew" pdv1 "github.com/decred/politeia/politeiad/api/v1" pdclient "github.com/decred/politeia/politeiad/client" v1 "github.com/decred/politeia/politeiawww/api/records/v1" @@ -20,7 +19,6 @@ import ( ) func respondWithError(w http.ResponseWriter, r *http.Request, format string, err error) { - spew.Dump(err) var ( ue v1.UserErrorReply pe v1.PluginErrorReply diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index db9c13076..a961fdaf1 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -416,22 +416,26 @@ func (r *Records) processRecords(ctx context.Context, rs v1.Records, u *user.Use func (r *Records) processInventory(ctx context.Context, i v1.Inventory, u *user.User) (*v1.InventoryReply, error) { log.Tracef("processInventory: %v %v %v", i.State, i.Status, i.Page) - // Verify state - switch i.State { - case v1.RecordStateUnvetted, v1.RecordStateVetted: - // Allowed; continue - default: - return nil, v1.UserErrorReply{ - ErrorCode: v1.ErrorCodeRecordStateInvalid, + // The inventory arguments are optional. If a status is provided + // then they all arguments must be provided. + var s pdv1.RecordStatusT + if i.Status != v1.RecordStatusInvalid { + // Verify state + switch i.State { + case v1.RecordStateUnvetted, v1.RecordStateVetted: + // Allowed; continue + default: + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeRecordStateInvalid, + } } - } - // Verify status. The status is optional so only validate it if one - // was provided. - s := convertStatusToPD(i.Status) - if i.Status != v1.RecordStatusInvalid && s == pdv1.RecordStatusInvalid { - return nil, v1.UserErrorReply{ - ErrorCode: v1.ErrorCodeRecordStatusInvalid, + // Verify status + s = convertStatusToPD(i.Status) + if s == pdv1.RecordStatusInvalid { + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeRecordStatusInvalid, + } } } From c5ca793fe793db983e6c773f8cd5b3bfa7d33731 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 15 Feb 2021 19:44:57 -0600 Subject: [PATCH 309/449] Store data desc in leaf extra data. --- .../backend/tlogbe/plugins/comments/cmds.go | 19 ++-- politeiad/backend/tlogbe/plugins/plugins.go | 14 +-- .../backend/tlogbe/plugins/ticketvote/cmds.go | 55 +++++---- politeiad/backend/tlogbe/store/store.go | 2 +- politeiad/backend/tlogbe/tlog/anchor.go | 23 +++- politeiad/backend/tlogbe/tlog/recordindex.go | 13 ++- politeiad/backend/tlogbe/tlog/tlog.go | 105 +++++++++++------- politeiad/backend/tlogbe/tlog/tlogclient.go | 76 +++++++++---- politeiad/plugins/user/user.go | 2 +- 9 files changed, 184 insertions(+), 125 deletions(-) diff --git a/politeiad/backend/tlogbe/plugins/comments/cmds.go b/politeiad/backend/tlogbe/plugins/comments/cmds.go index 2e050376e..c57d9f897 100644 --- a/politeiad/backend/tlogbe/plugins/comments/cmds.go +++ b/politeiad/backend/tlogbe/plugins/comments/cmds.go @@ -22,15 +22,12 @@ import ( ) const ( + pluginID = comments.PluginID + // Blob entry data descriptors - dataDescriptorCommentAdd = "cadd-v1" - dataDescriptorCommentDel = "cdel-v1" - dataDescriptorCommentVote = "cvote-v1" - - // Data types - dataTypeCommentAdd = "cadd" - dataTypeCommentDel = "cdel" - dataTypeCommentVote = "cvote" + dataDescriptorCommentAdd = pluginID + "-add-v1" + dataDescriptorCommentDel = pluginID + "-del-v1" + dataDescriptorCommentVote = pluginID + "-vote-v1" ) func tokenDecode(token string) ([]byte, error) { @@ -297,7 +294,7 @@ func (p *commentsPlugin) commentAddSave(treeID int64, ca comments.CommentAdd) ([ if err != nil { return nil, err } - err = p.tlog.BlobSave(treeID, dataTypeCommentAdd, *be) + err = p.tlog.BlobSave(treeID, *be) if err != nil { return nil, err } @@ -345,7 +342,7 @@ func (p *commentsPlugin) commentDelSave(treeID int64, cd comments.CommentDel) ([ if err != nil { return nil, err } - err = p.tlog.BlobSave(treeID, dataTypeCommentDel, *be) + err = p.tlog.BlobSave(treeID, *be) if err != nil { return nil, err } @@ -392,7 +389,7 @@ func (p *commentsPlugin) commentVoteSave(treeID int64, cv comments.CommentVote) if err != nil { return nil, err } - err = p.tlog.BlobSave(treeID, dataTypeCommentVote, *be) + err = p.tlog.BlobSave(treeID, *be) if err != nil { return nil, err } diff --git a/politeiad/backend/tlogbe/plugins/plugins.go b/politeiad/backend/tlogbe/plugins/plugins.go index a019de962..de45e635b 100644 --- a/politeiad/backend/tlogbe/plugins/plugins.go +++ b/politeiad/backend/tlogbe/plugins/plugins.go @@ -183,7 +183,7 @@ type TlogClient interface { // instance has an encryption key set. The digest of the data, // i.e. BlobEntry.Digest, can be thought of as the blob ID and can // be used to get/del the blob from tlog. - BlobSave(treeID int64, dataType string, be store.BlobEntry) error + BlobSave(treeID int64, be store.BlobEntry) error // BlobsDel deletes the blobs that correspond to the provided // digests. @@ -194,13 +194,13 @@ type TlogClient interface { // map. Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEntry, error) - // BlobsByDataType returns all blobs that match the data type. The - // blobs will be ordered from oldest to newest. - BlobsByDataType(treeID int64, keyPrefix string) ([]store.BlobEntry, error) + // BlobsByDataDesc returns all blobs that match the provided data + // descriptor. The blobs will be ordered from oldest to newest. + BlobsByDataDesc(treeID int64, dataDesc string) ([]store.BlobEntry, error) - // DigestsByDataType returns the digests of all blobs that match - // the data type. - DigestsByDataType(treeID int64, dataType string) ([][]byte, error) + // DigestsByDataDesc returns the digests of all blobs that match + // the provided data descriptor. + DigestsByDataDesc(treeID int64, dataDesc string) ([][]byte, error) // Timestamp returns the timestamp for the blob that correpsonds // to the digest. diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go index eaa8d2002..d8180dac2 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go @@ -30,17 +30,13 @@ import ( ) const ( - // Blob entry data descriptors - dataDescriptorAuthDetails = "authdetails-v1" - dataDescriptorVoteDetails = "votedetails-v1" - dataDescriptorCastVoteDetails = "castvotedetails-v1" - dataDescriptorStartRunoff = "startrunoff-v1" + pluginID = ticketvote.PluginID - // Data types - dataTypeAuthDetails = "authdetails" - dataTypeVoteDetails = "votedetails" - dataTypeCastVoteDetails = "castvotedetails" - dataTypeStartRunoff = "startrunoff" + // Blob entry data descriptors + dataDescriptorAuthDetails = pluginID + "-auth-v1" + dataDescriptorVoteDetails = pluginID + "-vote-v1" + dataDescriptorCastVoteDetails = pluginID + "-cvote-v1" + dataDescriptorStartRunoff = pluginID + "-startrunoff-v1" // Internal plugin commands cmdStartRunoffSubmission = "startrunoffsub" @@ -309,12 +305,12 @@ func (p *ticketVotePlugin) authSave(treeID int64, ad ticketvote.AuthDetails) err } // Save blob - return p.tlog.BlobSave(treeID, dataTypeAuthDetails, *be) + return p.tlog.BlobSave(treeID, *be) } func (p *ticketVotePlugin) auths(treeID int64) ([]ticketvote.AuthDetails, error) { // Retrieve blobs - blobs, err := p.tlog.BlobsByDataType(treeID, dataTypeAuthDetails) + blobs, err := p.tlog.BlobsByDataDesc(treeID, dataDescriptorAuthDetails) if err != nil { return nil, err } @@ -346,12 +342,12 @@ func (p *ticketVotePlugin) voteDetailsSave(treeID int64, vd ticketvote.VoteDetai } // Save blob - return p.tlog.BlobSave(treeID, dataTypeVoteDetails, *be) + return p.tlog.BlobSave(treeID, *be) } func (p *ticketVotePlugin) voteDetails(treeID int64) (*ticketvote.VoteDetails, error) { // Retrieve blobs - blobs, err := p.tlog.BlobsByDataType(treeID, dataTypeVoteDetails) + blobs, err := p.tlog.BlobsByDataDesc(treeID, dataDescriptorVoteDetails) if err != nil { return nil, err } @@ -399,12 +395,12 @@ func (p *ticketVotePlugin) castVoteSave(treeID int64, cv ticketvote.CastVoteDeta } // Save blob - return p.tlog.BlobSave(treeID, dataTypeCastVoteDetails, *be) + return p.tlog.BlobSave(treeID, *be) } func (p *ticketVotePlugin) castVotes(treeID int64) ([]ticketvote.CastVoteDetails, error) { // Retrieve blobs - blobs, err := p.tlog.BlobsByDataType(treeID, dataTypeCastVoteDetails) + blobs, err := p.tlog.BlobsByDataDesc(treeID, dataDescriptorCastVoteDetails) if err != nil { return nil, err } @@ -1436,10 +1432,10 @@ func (p *ticketVotePlugin) startRunoffRecordSave(treeID int64, srr startRunoffRe if err != nil { return err } - err = p.tlog.BlobSave(treeID, dataTypeStartRunoff, *be) + err = p.tlog.BlobSave(treeID, *be) if err != nil { return fmt.Errorf("BlobSave %v %v: %v", - treeID, dataTypeStartRunoff, err) + treeID, dataDescriptorStartRunoff, err) } return nil } @@ -1447,10 +1443,10 @@ func (p *ticketVotePlugin) startRunoffRecordSave(treeID int64, srr startRunoffRe // startRunoffRecord returns the startRunoff record if one exists on a tree. // nil will be returned if a startRunoff record is not found. func (p *ticketVotePlugin) startRunoffRecord(treeID int64) (*startRunoffRecord, error) { - blobs, err := p.tlog.BlobsByDataType(treeID, dataTypeStartRunoff) + blobs, err := p.tlog.BlobsByDataDesc(treeID, dataDescriptorStartRunoff) if err != nil { - return nil, fmt.Errorf("BlobsByDataType %v %v: %v", - treeID, dataTypeStartRunoff, err) + return nil, fmt.Errorf("BlobsByDataDesc %v %v: %v", + treeID, dataDescriptorStartRunoff, err) } var srr *startRunoffRecord @@ -2639,10 +2635,11 @@ func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload str switch { case t.VotesPage > 0: // Return a page of vote timestamps - digests, err := p.tlog.DigestsByDataType(treeID, dataTypeCastVoteDetails) + digests, err := p.tlog.DigestsByDataDesc(treeID, + dataDescriptorCastVoteDetails) if err != nil { return "", fmt.Errorf("digestsByKeyPrefix %v %v: %v", - treeID, dataTypeVoteDetails, err) + treeID, dataDescriptorVoteDetails, err) } startAt := (t.VotesPage - 1) * pageSize @@ -2665,10 +2662,10 @@ func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload str // Return authorization timestamps and the vote details timestamp. // Auth timestamps - digests, err := p.tlog.DigestsByDataType(treeID, dataTypeAuthDetails) + digests, err := p.tlog.DigestsByDataDesc(treeID, dataDescriptorAuthDetails) if err != nil { - return "", fmt.Errorf("DigestByDataType %v %v: %v", - treeID, dataTypeAuthDetails, err) + return "", fmt.Errorf("DigestByDataDesc %v %v: %v", + treeID, dataDescriptorAuthDetails, err) } auths = make([]ticketvote.Timestamp, 0, len(digests)) for _, v := range digests { @@ -2680,10 +2677,10 @@ func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload str } // Vote details timestamp - digests, err = p.tlog.DigestsByDataType(treeID, dataTypeVoteDetails) + digests, err = p.tlog.DigestsByDataDesc(treeID, dataDescriptorVoteDetails) if err != nil { - return "", fmt.Errorf("DigestsByDataType %v %v: %v", - treeID, dataTypeVoteDetails, err) + return "", fmt.Errorf("DigestsByDataDesc %v %v: %v", + treeID, dataDescriptorVoteDetails, err) } // There should never be more than a one vote details if len(digests) > 1 { diff --git a/politeiad/backend/tlogbe/store/store.go b/politeiad/backend/tlogbe/store/store.go index c746e40b9..1bbef420c 100644 --- a/politeiad/backend/tlogbe/store/store.go +++ b/politeiad/backend/tlogbe/store/store.go @@ -19,7 +19,7 @@ const ( DataTypeStructure = "struct" ) -// DataDescriptor provides hints about a data blob. In practise we JSON encode +// DataDescriptor provides hints about a data blob. In practice we JSON encode // this struture and stuff it into BlobEntry.DataHint. type DataDescriptor struct { Type string `json:"type"` // Type of data diff --git a/politeiad/backend/tlogbe/tlog/anchor.go b/politeiad/backend/tlogbe/tlog/anchor.go index 3f967d889..0493da41c 100644 --- a/politeiad/backend/tlogbe/tlog/anchor.go +++ b/politeiad/backend/tlogbe/tlog/anchor.go @@ -93,8 +93,12 @@ func (t *Tlog) anchorForLeaf(treeID int64, merkleLeafHash []byte, leaves []*tril var anchorKey string for i := int(l.LeafIndex); i < len(leaves); i++ { l := leaves[i] - if leafDataType(l) == dataTypeAnchorRecord { - anchorKey = extractKeyFromLeaf(l) + ed, err := extraDataDecode(l.ExtraData) + if err != nil { + return nil, err + } + if ed.Desc == dataDescriptorAnchor { + anchorKey = ed.Key break } } @@ -136,9 +140,13 @@ func (t *Tlog) anchorLatest(treeID int64) (*anchor, error) { // Find the most recent anchor leaf var key string for i := len(leavesAll) - 1; i >= 0; i-- { - l := leavesAll[i] - if leafDataType(l) == dataTypeAnchorRecord { - key = extractKeyFromLeaf(l) + ed, err := extraDataDecode(leavesAll[i].ExtraData) + if err != nil { + return nil, err + } + if ed.Desc == dataDescriptorAnchor { + key = ed.Key + break } } if key == "" { @@ -206,7 +214,10 @@ func (t *Tlog) anchorSave(a anchor) error { if err != nil { return err } - extraData := leafExtraData(dataTypeAnchorRecord, keys[0]) + extraData, err := extraDataEncode(keys[0], dataDescriptorAnchor) + if err != nil { + return err + } leaves := []*trillian.LogLeaf{ newLogLeaf(d, extraData), } diff --git a/politeiad/backend/tlogbe/tlog/recordindex.go b/politeiad/backend/tlogbe/tlog/recordindex.go index fe629bb3b..6e07c34ab 100644 --- a/politeiad/backend/tlogbe/tlog/recordindex.go +++ b/politeiad/backend/tlogbe/tlog/recordindex.go @@ -160,7 +160,10 @@ func (t *Tlog) recordIndexSave(treeID int64, ri recordIndex) error { if err != nil { return err } - extraData := leafExtraData(dataTypeRecordIndex, keys[0]) + extraData, err := extraDataEncode(keys[0], dataDescriptorRecordIndex) + if err != nil { + return err + } leaves := []*trillian.LogLeaf{ newLogLeaf(d, extraData), } @@ -197,9 +200,13 @@ func (t *Tlog) recordIndexes(leaves []*trillian.LogLeaf) ([]recordIndex, error) // version. keys := make([]string, 0, 64) for _, v := range leaves { - if leafDataType(v) == dataTypeRecordIndex { + ed, err := extraDataDecode(v.ExtraData) + if err != nil { + return nil, err + } + if ed.Desc == dataDescriptorRecordIndex { // This is a record index leaf. Save the kv store key. - keys = append(keys, extractKeyFromLeaf(v)) + keys = append(keys, ed.Key) } } diff --git a/politeiad/backend/tlogbe/tlog/tlog.go b/politeiad/backend/tlogbe/tlog/tlog.go index 2e8720c1d..e9546ff9a 100644 --- a/politeiad/backend/tlogbe/tlog/tlog.go +++ b/politeiad/backend/tlogbe/tlog/tlog.go @@ -42,19 +42,6 @@ const ( dataDescriptorMetadataStream = "mdstream-v1" dataDescriptorRecordIndex = "rindex-v1" dataDescriptorAnchor = "anchor-v1" - - // The keys for kv store blobs are saved by stuffing them into the - // ExtraData field of their corresponding trillian log leaf. The - // keys are prefixed with one of the follwing identifiers before - // being added to the log leaf so that we can correlate the leaf - // to the type of data it represents without having to pull the - // data out of the store, which can become an issue in situations - // such as searching for a record index that has been buried by - // thousands of leaves from plugin data. - dataTypeSeperator = ":" - dataTypeRecordIndex = "rindex" - dataTypeRecordContent = "rcontent" - dataTypeAnchorRecord = "anchor" ) var ( @@ -90,28 +77,33 @@ func blobIsEncrypted(b []byte) bool { return bytes.HasPrefix(b, []byte("sbox")) } -func leafExtraData(dataType, storeKey string) []byte { - return []byte(dataType + ":" + storeKey) +// extraData is the data that is stored in the log leaf ExtraData field. It is +// saved as a JSON encoded byte slice. The JSON keys have been abbreviated to +// minimize the size of a trillian log leaf. +type extraData struct { + Key string `json:"k"` // Key-value store key + Desc string `json:"d"` // Blob entry data descriptor } -func leafDataType(l *trillian.LogLeaf) string { - s := bytes.Split(l.ExtraData, []byte(":")) - if len(s) != 2 { - e := fmt.Sprintf("invalid key '%s' for leaf %x", - l.ExtraData, l.MerkleLeafHash) - panic(e) +func extraDataEncode(key, desc string) ([]byte, error) { + ed := extraData{ + Key: key, + Desc: desc, + } + b, err := json.Marshal(ed) + if err != nil { + return nil, err } - return string(s[0]) + return b, nil } -func extractKeyFromLeaf(l *trillian.LogLeaf) string { - s := bytes.SplitAfter(l.ExtraData, []byte(":")) - if len(s) != 2 { - e := fmt.Sprintf("invalid key '%s' for leaf %x", - l.ExtraData, l.MerkleLeafHash) - panic(e) +func extraDataDecode(b []byte) (*extraData, error) { + var ed extraData + err := json.Unmarshal(b, &ed) + if err != nil { + return nil, err } - return string(s[1]) + return &ed, nil } func (t *Tlog) blobify(be store.BlobEntry) ([]byte, error) { @@ -213,7 +205,10 @@ func (t *Tlog) TreeFreeze(treeID int64, rm backend.RecordMetadata, metadata []ba } // Append record index leaf to the trillian tree - extraData := leafExtraData(dataTypeRecordIndex, keys[0]) + extraData, err := extraDataEncode(keys[0], dataDescriptorRecordIndex) + if err != nil { + return err + } leaves := []*trillian.LogLeaf{ newLogLeaf(idxDigest, extraData), } @@ -323,14 +318,18 @@ type recordBlobsPrepareReply struct { recordHashes recordHashes // blobs contains the blobified record content that needs to be - // saved to the kv store. Hashes contains the hashes of the record - // content prior to being blobified. These hashes are saved to - // trilian log leaves. The hashes are SHA256 hashes of the JSON - // encoded data. + // saved to the kv store. + // + // Hashes contains the hashes of the record content prior to being + // blobified. These hashes are saved to trilian log leaves. The + // hashes are SHA256 hashes of the JSON encoded data. + // + // hints contains the data hints of the blob entries. // - // blobs and hashes share the same ordering. + // blobs, hashes, and descriptors share the same ordering. blobs [][]byte hashes [][]byte + hints []string } // recordBlobsPrepare prepares the provided record content to be saved to @@ -442,6 +441,7 @@ func (t *Tlog) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backen l := len(metadata) + len(files) + 1 hashes := make([][]byte, 0, l) blobs := make([][]byte, 0, l) + hints := make([]string, 0, l) // Prepare record metadata blob be, err := convertBlobEntryFromRecordMetadata(recordMD) @@ -461,6 +461,7 @@ func (t *Tlog) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backen // Not a duplicate. Save blob to the store. hashes = append(hashes, h) blobs = append(blobs, b) + hints = append(hints, be.DataHint) } // Prepare metadata blobs @@ -482,6 +483,7 @@ func (t *Tlog) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backen // Not a duplicate. Save blob to the store. hashes = append(hashes, h) blobs = append(blobs, b) + hints = append(hints, be.DataHint) } } @@ -504,6 +506,7 @@ func (t *Tlog) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backen // Not a duplicate. Save blob to the store. hashes = append(hashes, h) blobs = append(blobs, b) + hints = append(hints, be.DataHint) } } @@ -512,6 +515,7 @@ func (t *Tlog) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backen recordHashes: rhashes, blobs: blobs, hashes: hashes, + hints: hints, }, nil } @@ -526,6 +530,7 @@ func (t *Tlog) recordBlobsSave(treeID int64, rbpr recordBlobsPrepareReply) (*rec rhashes = rbpr.recordHashes blobs = rbpr.blobs hashes = rbpr.hashes + hints = rbpr.hints ) // Save blobs to store @@ -541,7 +546,10 @@ func (t *Tlog) recordBlobsSave(treeID int64, rbpr recordBlobsPrepareReply) (*rec // Prepare log leaves. hashes and keys share the same ordering. leaves := make([]*trillian.LogLeaf, 0, len(blobs)) for k := range blobs { - extraData := leafExtraData(dataTypeRecordContent, keys[k]) + extraData, err := extraDataEncode(keys[k], hints[k]) + if err != nil { + return nil, err + } leaves = append(leaves, newLogLeaf(hashes[k], extraData)) } @@ -848,7 +856,7 @@ func (t *Tlog) RecordDel(treeID int64) error { return fmt.Errorf("tree is not frozen") } - // Retrieve all the record indexes + // Retrieve all record indexes indexes, err := t.recordIndexes(leavesAll) if err != nil { return err @@ -867,7 +875,11 @@ func (t *Tlog) RecordDel(treeID int64) error { for _, v := range leavesAll { _, ok := merkles[hex.EncodeToString(v.MerkleLeafHash)] if ok { - keys = append(keys, extractKeyFromLeaf(v)) + ed, err := extraDataDecode(v.ExtraData) + if err != nil { + return err + } + keys = append(keys, ed.Key) } } @@ -996,7 +1008,11 @@ func (t *Tlog) Record(treeID int64, version uint32) (*backend.Record, error) { } // Leaf is part of record content. Save the kv store key. - keys = append(keys, extractKeyFromLeaf(v)) + ed, err := extraDataDecode(v.ExtraData) + if err != nil { + return nil, err + } + keys = append(keys, ed.Key) } // Get record content from store @@ -1120,8 +1136,11 @@ func (t *Tlog) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trillian } // Get blob entry from the kv store - key := extractKeyFromLeaf(l) - blobs, err := t.store.Get([]string{key}) + ed, err := extraDataDecode(l.ExtraData) + if err != nil { + return nil, err + } + blobs, err := t.store.Get([]string{ed.Key}) if err != nil { return nil, fmt.Errorf("store get: %v", err) } @@ -1131,9 +1150,9 @@ func (t *Tlog) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trillian // rest of the timestamp. var data []byte if len(blobs) == 1 { - b, ok := blobs[key] + b, ok := blobs[ed.Key] if !ok { - return nil, fmt.Errorf("blob not found %v", key) + return nil, fmt.Errorf("blob not found %v", ed.Key) } be, err := t.deblob(b) if err != nil { diff --git a/politeiad/backend/tlogbe/tlog/tlogclient.go b/politeiad/backend/tlogbe/tlog/tlogclient.go index 80c9138c8..bea3b8a75 100644 --- a/politeiad/backend/tlogbe/tlog/tlogclient.go +++ b/politeiad/backend/tlogbe/tlog/tlogclient.go @@ -5,9 +5,10 @@ package tlog import ( + "encoding/base64" "encoding/hex" + "encoding/json" "fmt" - "strings" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/store" @@ -20,13 +21,19 @@ import ( // encryption key set. The digest of the data, i.e. BlobEntry.Digest, can be // thought of as the blob ID and can be used to get/del the blob from tlog. // -// This function satisfies the plugins.TlogClient interface. -func (t *Tlog) BlobSave(treeID int64, dataType string, be store.BlobEntry) error { - log.Tracef("%v BlobSave: %v %v", t.id, treeID, dataType) +// This function satisfies the plugins TlogClient interface. +func (t *Tlog) BlobSave(treeID int64, be store.BlobEntry) error { + log.Tracef("%v BlobSave: %v %v", t.id, treeID) - // Verify data type - if strings.Contains(dataType, dataTypeSeperator) { - return fmt.Errorf("data type cannot contain '%v'", dataTypeSeperator) + // Parse the data descriptor + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return err + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return err } // Prepare blob and digest @@ -64,7 +71,10 @@ func (t *Tlog) BlobSave(treeID int64, dataType string, be store.BlobEntry) error } // Prepare log leaf - extraData := leafExtraData(dataType, keys[0]) + extraData, err := extraDataEncode(keys[0], dd.Descriptor) + if err != nil { + return err + } leaves = []*trillian.LogLeaf{ newLogLeaf(digest, extraData), } @@ -88,7 +98,7 @@ func (t *Tlog) BlobSave(treeID int64, dataType string, be store.BlobEntry) error // BlobsDel deletes the blobs that correspond to the provided digests. // -// This function satisfies the plugins.TlogClient interface. +// This function satisfies the plugins TlogClient interface. func (t *Tlog) BlobsDel(treeID int64, digests [][]byte) error { log.Tracef("%v BlobsDel: %v %x", t.id, treeID, digests) @@ -119,7 +129,11 @@ func (t *Tlog) BlobsDel(treeID int64, digests [][]byte) error { for _, v := range leaves { _, ok := merkleHashes[hex.EncodeToString(v.MerkleLeafHash)] if ok { - keys = append(keys, extractKeyFromLeaf(v)) + ed, err := extraDataDecode(v.ExtraData) + if err != nil { + return err + } + keys = append(keys, ed.Key) } } @@ -135,7 +149,7 @@ func (t *Tlog) BlobsDel(treeID int64, digests [][]byte) error { // Blobs returns the blobs that correspond to the provided digests. If a blob // does not exist it will not be included in the returned map. // -// This function satisfies the plugins.TlogClient interface. +// This function satisfies the plugins TlogClient interface. func (t *Tlog) Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEntry, error) { log.Tracef("%v Blobs: %v %x", t.id, treeID, digests) @@ -181,7 +195,11 @@ func (t *Tlog) Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEntry if !ok { return nil, fmt.Errorf("leaf not found: %x", v) } - keys = append(keys, extractKeyFromLeaf(l)) + ed, err := extraDataDecode(l.ExtraData) + if err != nil { + return nil, err + } + keys = append(keys, ed.Key) } // Pull the blobs from the store. It's ok if one or more blobs is @@ -212,11 +230,12 @@ func (t *Tlog) Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEntry return entries, nil } -// BlobsByDataType returns all blobs that match the data type. +// BlobsByDataDesc returns all blobs that match the provided data descriptor. +// The blobs will be ordered from oldest to newest. // -// This function satisfies the plugins.TlogClient interface. -func (t *Tlog) BlobsByDataType(treeID int64, dataType string) ([]store.BlobEntry, error) { - log.Tracef("%v BlobsByDataType: %v %v", t.id, treeID, dataType) +// This function satisfies the plugins TlogClient interface. +func (t *Tlog) BlobsByDataDesc(treeID int64, dataDesc string) ([]store.BlobEntry, error) { + log.Tracef("%v BlobsByDataDesc: %v %v", t.id, treeID, dataDesc) // Verify tree exists if !t.TreeExists(treeID) { @@ -233,8 +252,12 @@ func (t *Tlog) BlobsByDataType(treeID int64, dataType string) ([]store.BlobEntry // leaves with a matching key prefix. keys := make([]string, 0, len(leaves)) for _, v := range leaves { - if leafDataType(v) == dataType { - keys = append(keys, extractKeyFromLeaf(v)) + ed, err := extraDataDecode(v.ExtraData) + if err != nil { + return nil, err + } + if ed.Desc == dataDesc { + keys = append(keys, ed.Key) } } @@ -273,11 +296,12 @@ func (t *Tlog) BlobsByDataType(treeID int64, dataType string) ([]store.BlobEntry return entries, nil } -// DigestsByDataType returns the digests of all blobs that match the data type. +// DigestsByDataDesc returns the digests of all blobs that match the provided +// data descriptor. // -// This function satisfies the plugins.TlogClient interface. -func (t *Tlog) DigestsByDataType(treeID int64, dataType string) ([][]byte, error) { - log.Tracef("%v DigestsByDataType: %v %v", t.id, treeID, dataType) +// This function satisfies the plugins TlogClient interface. +func (t *Tlog) DigestsByDataDesc(treeID int64, dataDesc string) ([][]byte, error) { + log.Tracef("%v DigestsByDataDesc: %v %v", t.id, treeID, dataDesc) // Verify tree exists if !t.TreeExists(treeID) { @@ -294,7 +318,11 @@ func (t *Tlog) DigestsByDataType(treeID int64, dataType string) ([][]byte, error // all leaves that match the provided data type. digests := make([][]byte, 0, len(leaves)) for _, v := range leaves { - if leafDataType(v) == dataType { + ed, err := extraDataDecode(v.ExtraData) + if err != nil { + return nil, err + } + if ed.Desc == dataDesc { digests = append(digests, v.LeafValue) } } @@ -305,7 +333,7 @@ func (t *Tlog) DigestsByDataType(treeID int64, dataType string) ([][]byte, error // Timestamp returns the timestamp for the data blob that corresponds to the // provided digest. // -// This function satisfies the plugins.TlogClient interface. +// This function satisfies the plugins TlogClient interface. func (t *Tlog) Timestamp(treeID int64, digest []byte) (*backend.Timestamp, error) { log.Tracef("%v Timestamp: %v %x", t.id, treeID, digest) diff --git a/politeiad/plugins/user/user.go b/politeiad/plugins/user/user.go index fdbdb92fc..5d3ca100a 100644 --- a/politeiad/plugins/user/user.go +++ b/politeiad/plugins/user/user.go @@ -57,7 +57,7 @@ var ( // UserMetadata contains user metadata about a politeiad record. It is // generated by the server and saved to politeiad as a metadata stream. // -// Signature is the client signature of the hex encoded record merkle root.The +// Signature is the client signature of the hex encoded record merkle root. The // merkle root is the ordered merkle root of all user submitted politeiad // files. The merkle root is hex encoded before being signed so that the // signature is consistent with how politeiad signs the merkle root. From 9527dd046703ad618b771b9252f4b46da2f417a8 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 16 Feb 2021 09:40:00 -0600 Subject: [PATCH 310/449] Add plugin ID to mdstream. --- politeiad/api/v1/v1.go | 5 +- politeiad/backend/backend.go | 5 +- .../backend/tlogbe/plugins/user/hooks.go | 45 +++++++++------- politeiad/plugins/user/user.go | 5 +- politeiad/politeiad.go | 10 ++-- politeiawww/api/records/v1/v1.go | 5 +- politeiawww/cmd/pictl/proposal.go | 2 +- politeiawww/records/process.go | 51 +++++++++++-------- 8 files changed, 73 insertions(+), 55 deletions(-) diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index b2547eef0..382766750 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -229,8 +229,9 @@ type File struct { // MetadataStream identifies a metadata stream by its identity. type MetadataStream struct { - ID uint64 `json:"id"` // Stream identity - Payload string `json:"payload"` // String encoded metadata + PluginID string `json:"pluginid,omitempty"` // Plugin identity + ID uint64 `json:"id"` // Stream identity + Payload string `json:"payload"` // String encoded metadata } // Record is an entire record and it's content. diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index 61cfaf36b..eeeb85f15 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -140,8 +140,9 @@ type RecordMetadata struct { // MetadataStream describes a single metada stream. type MetadataStream struct { - ID uint64 `json:"id"` // Stream identity - Payload string `json:"payload"` // String encoded metadata + PluginID string `json:"pluginid,omitempty"` // Plugin identity + ID uint64 `json:"id"` // Stream identity + Payload string `json:"payload"` // String encoded metadata } // Record is a permanent Record that includes the submitted files, metadata and diff --git a/politeiad/backend/tlogbe/plugins/user/hooks.go b/politeiad/backend/tlogbe/plugins/user/hooks.go index e94118625..edba3d99a 100644 --- a/politeiad/backend/tlogbe/plugins/user/hooks.go +++ b/politeiad/backend/tlogbe/plugins/user/hooks.go @@ -43,15 +43,18 @@ func convertSignatureError(err error) backend.PluginError { func userMetadataDecode(metadata []backend.MetadataStream) (*user.UserMetadata, error) { var userMD *user.UserMetadata for _, v := range metadata { - if v.ID == user.MDStreamIDUserMetadata { - var um user.UserMetadata - err := json.Unmarshal([]byte(v.Payload), &um) - if err != nil { - return nil, err - } - userMD = &um - break + if v.PluginID != user.PluginID || + v.ID != user.MDStreamIDUserMetadata { + // Not the mdstream we're looking for + continue } + var um user.UserMetadata + err := json.Unmarshal([]byte(v.Payload), &um) + if err != nil { + return nil, err + } + userMD = &um + break } return userMD, nil } @@ -224,19 +227,23 @@ func (p *userPlugin) hookEditMetadataPre(payload string) error { func statusChangesDecode(metadata []backend.MetadataStream) ([]user.StatusChangeMetadata, error) { statuses := make([]user.StatusChangeMetadata, 0, 16) for _, v := range metadata { - if v.ID == user.MDStreamIDStatusChanges { - d := json.NewDecoder(strings.NewReader(v.Payload)) - for { - var sc user.StatusChangeMetadata - err := d.Decode(&sc) - if errors.Is(err, io.EOF) { - break - } else if err != nil { - return nil, err - } - statuses = append(statuses, sc) + if v.PluginID != user.PluginID || + v.ID != user.MDStreamIDStatusChanges { + // Not the mdstream we're looking for + continue + } + d := json.NewDecoder(strings.NewReader(v.Payload)) + for { + var sc user.StatusChangeMetadata + err := d.Decode(&sc) + if errors.Is(err, io.EOF) { + break + } else if err != nil { + return nil, err } + statuses = append(statuses, sc) } + break } return statuses, nil } diff --git a/politeiad/plugins/user/user.go b/politeiad/plugins/user/user.go index 5d3ca100a..036ec3ed7 100644 --- a/politeiad/plugins/user/user.go +++ b/politeiad/plugins/user/user.go @@ -89,9 +89,8 @@ type AuthorReply struct { UserID string `json:"userid"` } -// UserRecords retrieves the tokens of all records that were -// submitted by the provided user ID. The returned tokens are sorted from -// newest to oldest. +// UserRecords retrieves the tokens of all records that were submitted by the +// provided user ID. The returned tokens are sorted from newest to oldest. type UserRecords struct { UserID string `json:"userid"` } diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index f2a59ddd9..06496da78 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -85,8 +85,9 @@ func convertBackendPlugins(bplugins []backend.Plugin) []v1.Plugin { // metadata stream. func convertBackendMetadataStream(mds backend.MetadataStream) v1.MetadataStream { return v1.MetadataStream{ - ID: mds.ID, - Payload: mds.Payload, + PluginID: mds.PluginID, + ID: mds.ID, + Payload: mds.Payload, } } @@ -187,8 +188,9 @@ func convertFrontendMetadataStream(mds []v1.MetadataStream) []backend.MetadataSt m := make([]backend.MetadataStream, 0, len(mds)) for _, v := range mds { m = append(m, backend.MetadataStream{ - ID: v.ID, - Payload: v.Payload, + PluginID: v.PluginID, + ID: v.ID, + Payload: v.Payload, }) } return m diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go index 7a357baa5..02b677c8c 100644 --- a/politeiawww/api/records/v1/v1.go +++ b/politeiawww/api/records/v1/v1.go @@ -169,8 +169,9 @@ type File struct { // MetadataStream describes a record metadata stream. type MetadataStream struct { - ID uint64 `json:"id"` - Payload string `json:"payload"` // JSON encoded + PluginID string `json:"pluginid,omitempty"` // Plugin ID + ID uint64 `json:"id"` // Metadata stream ID + Payload string `json:"payload"` // JSON encoded } // CensorshipRecord contains cryptographic proof that a record was accepted for diff --git a/politeiawww/cmd/pictl/proposal.go b/politeiawww/cmd/pictl/proposal.go index 3217cb2d0..f59528f4e 100644 --- a/politeiawww/cmd/pictl/proposal.go +++ b/politeiawww/cmd/pictl/proposal.go @@ -71,7 +71,7 @@ func printProposal(r rcv1.Record) error { printf("Metadata\n") for _, v := range r.Metadata { size := byteCountSI(int64(len([]byte(v.Payload)))) - printf(" %-2v %v\n", v.ID, size) + printf(" %-8v %-2v %v\n", v.PluginID, v.ID, size) } printf("Files\n") return printProposalFiles(r.Files) diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index a961fdaf1..6df9b6d68 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -11,7 +11,7 @@ import ( "time" pdv1 "github.com/decred/politeia/politeiad/api/v1" - pduser "github.com/decred/politeia/politeiad/plugins/user" + usplugin "github.com/decred/politeia/politeiad/plugins/user" v1 "github.com/decred/politeia/politeiawww/api/records/v1" "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/user" @@ -40,7 +40,7 @@ func (r *Records) processNew(ctx context.Context, n v1.New, u user.User) (*v1.Ne } // Setup metadata stream - um := pduser.UserMetadata{ + um := usplugin.UserMetadata{ UserID: u.ID.String(), PublicKey: n.PublicKey, Signature: n.Signature, @@ -51,8 +51,9 @@ func (r *Records) processNew(ctx context.Context, n v1.New, u user.User) (*v1.Ne } metadata := []pdv1.MetadataStream{ { - ID: pduser.MDStreamIDUserMetadata, - Payload: string(b), + PluginID: usplugin.PluginID, + ID: usplugin.MDStreamIDUserMetadata, + Payload: string(b), }, } @@ -154,7 +155,7 @@ func (r *Records) processEdit(ctx context.Context, e v1.Edit, u user.User) (*v1. filesDel := filesToDel(curr.Files, filesAdd) // Setup metadata - um := pduser.UserMetadata{ + um := usplugin.UserMetadata{ UserID: u.ID.String(), PublicKey: e.PublicKey, Signature: e.Signature, @@ -165,8 +166,9 @@ func (r *Records) processEdit(ctx context.Context, e v1.Edit, u user.User) (*v1. } mdOverwrite := []pdv1.MetadataStream{ { - ID: pduser.MDStreamIDUserMetadata, - Payload: string(b), + PluginID: usplugin.PluginID, + ID: usplugin.MDStreamIDUserMetadata, + Payload: string(b), }, } mdAppend := []pdv1.MetadataStream{} @@ -223,7 +225,7 @@ func (r *Records) processSetStatus(ctx context.Context, ss v1.SetStatus, u user. } // Setup status change metadata - scm := pduser.StatusChangeMetadata{ + scm := usplugin.StatusChangeMetadata{ Token: ss.Token, Version: ss.Version, Status: int(ss.Status), @@ -238,8 +240,9 @@ func (r *Records) processSetStatus(ctx context.Context, ss v1.SetStatus, u user. } mdAppend := []pdv1.MetadataStream{ { - ID: pduser.MDStreamIDStatusChanges, - Payload: string(b), + PluginID: usplugin.PluginID, + ID: usplugin.MDStreamIDStatusChanges, + Payload: string(b), }, } mdOverwrite := []pdv1.MetadataStream{} @@ -575,18 +578,21 @@ func recordPopulateUserData(r *v1.Record, u user.User) { // userMetadataDecode decodes and returns the UserMetadata from the provided // metadata streams. If a UserMetadata is not found, nil is returned. -func userMetadataDecode(ms []v1.MetadataStream) (*pduser.UserMetadata, error) { - var userMD *pduser.UserMetadata +func userMetadataDecode(ms []v1.MetadataStream) (*usplugin.UserMetadata, error) { + var userMD *usplugin.UserMetadata for _, v := range ms { - if v.ID == pduser.MDStreamIDUserMetadata { - var um pduser.UserMetadata - err := json.Unmarshal([]byte(v.Payload), &um) - if err != nil { - return nil, err - } - userMD = &um - break + if v.PluginID != usplugin.PluginID || + v.ID != usplugin.MDStreamIDUserMetadata { + // Not the mdstream we're looking for + continue + } + var um usplugin.UserMetadata + err := json.Unmarshal([]byte(v.Payload), &um) + if err != nil { + return nil, err } + userMD = &um + break } return userMD, nil } @@ -662,8 +668,9 @@ func convertMetadataStreamsToV1(ms []pdv1.MetadataStream) []v1.MetadataStream { metadata := make([]v1.MetadataStream, 0, len(ms)) for _, v := range ms { metadata = append(metadata, v1.MetadataStream{ - ID: v.ID, - Payload: v.Payload, + PluginID: v.PluginID, + ID: v.ID, + Payload: v.Payload, }) } return metadata From d5292ba7b2d8aadd8e2659e5468c8b769c4ee2eb Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 16 Feb 2021 10:13:21 -0600 Subject: [PATCH 311/449] Rename user plugin to usermd. --- go.mod | 1 + .../tlogbe/plugins/{user => usermd}/cache.go | 2 +- .../tlogbe/plugins/{user => usermd}/cmds.go | 10 +-- .../tlogbe/plugins/{user => usermd}/hooks.go | 72 +++++++++---------- .../tlogbe/plugins/{user => usermd}/log.go | 2 +- .../{user/user.go => usermd/usermd.go} | 20 +++--- politeiad/backend/tlogbe/tlog/plugin.go | 8 +-- politeiad/client/user.go | 20 +++--- politeiad/log.go | 4 +- politeiad/plugins/comments/comments.go | 2 +- politeiad/plugins/dcrdata/dcrdata.go | 1 + politeiad/plugins/pi/pi.go | 2 +- politeiad/plugins/ticketvote/ticketvote.go | 2 +- .../{user/user.go => usermd/usermd.go} | 7 +- politeiawww/client/error.go | 6 +- politeiawww/cmd/pictl/cmdproposalstatusset.go | 11 +-- politeiawww/cmd/pictl/proposal.go | 6 +- politeiawww/pi/process.go | 24 +++---- politeiawww/piwww.go | 4 +- politeiawww/proposals.go | 14 ++-- politeiawww/records/process.go | 30 ++++---- 21 files changed, 127 insertions(+), 121 deletions(-) rename politeiad/backend/tlogbe/plugins/{user => usermd}/cache.go (99%) rename politeiad/backend/tlogbe/plugins/{user => usermd}/cmds.go (87%) rename politeiad/backend/tlogbe/plugins/{user => usermd}/hooks.go (85%) rename politeiad/backend/tlogbe/plugins/{user => usermd}/log.go (98%) rename politeiad/backend/tlogbe/plugins/{user/user.go => usermd/usermd.go} (86%) rename politeiad/plugins/{user/user.go => usermd/usermd.go} (95%) diff --git a/go.mod b/go.mod index a11e2cd64..db2d10494 100644 --- a/go.mod +++ b/go.mod @@ -56,6 +56,7 @@ require ( golang.org/x/net v0.0.0-20200625001655-4c5254603344 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 + google.golang.org/appengine v1.6.6 google.golang.org/genproto v0.0.0-20200707001353-8e8330bf89df google.golang.org/grpc v1.29.1 ) diff --git a/politeiad/backend/tlogbe/plugins/user/cache.go b/politeiad/backend/tlogbe/plugins/usermd/cache.go similarity index 99% rename from politeiad/backend/tlogbe/plugins/user/cache.go rename to politeiad/backend/tlogbe/plugins/usermd/cache.go index cc3052d61..48c6bdec7 100644 --- a/politeiad/backend/tlogbe/plugins/user/cache.go +++ b/politeiad/backend/tlogbe/plugins/usermd/cache.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package user +package usermd import ( "encoding/json" diff --git a/politeiad/backend/tlogbe/plugins/user/cmds.go b/politeiad/backend/tlogbe/plugins/usermd/cmds.go similarity index 87% rename from politeiad/backend/tlogbe/plugins/user/cmds.go rename to politeiad/backend/tlogbe/plugins/usermd/cmds.go index f6b877e28..24e72ad5e 100644 --- a/politeiad/backend/tlogbe/plugins/user/cmds.go +++ b/politeiad/backend/tlogbe/plugins/usermd/cmds.go @@ -2,12 +2,12 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package user +package usermd import ( "encoding/json" - "github.com/decred/politeia/politeiad/plugins/user" + "github.com/decred/politeia/politeiad/plugins/usermd" ) func (p *userPlugin) cmdAuthor(treeID int64) (string, error) { @@ -24,7 +24,7 @@ func (p *userPlugin) cmdAuthor(treeID int64) (string, error) { } // Prepare reply - ar := user.AuthorReply{ + ar := usermd.AuthorReply{ UserID: um.UserID, } reply, err := json.Marshal(ar) @@ -39,7 +39,7 @@ func (p *userPlugin) cmdUserRecords(payload string) (string, error) { log.Tracef("cmdUserRecords: %v", payload) // Decode payload - var ur user.UserRecords + var ur usermd.UserRecords err := json.Unmarshal([]byte(payload), &ur) if err != nil { return "", err @@ -52,7 +52,7 @@ func (p *userPlugin) cmdUserRecords(payload string) (string, error) { } // Prepare reply - urr := user.UserRecordsReply{ + urr := usermd.UserRecordsReply{ Records: uc.Tokens, } reply, err := json.Marshal(urr) diff --git a/politeiad/backend/tlogbe/plugins/user/hooks.go b/politeiad/backend/tlogbe/plugins/usermd/hooks.go similarity index 85% rename from politeiad/backend/tlogbe/plugins/user/hooks.go rename to politeiad/backend/tlogbe/plugins/usermd/hooks.go index edba3d99a..6509bfe75 100644 --- a/politeiad/backend/tlogbe/plugins/user/hooks.go +++ b/politeiad/backend/tlogbe/plugins/usermd/hooks.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package user +package usermd import ( "encoding/hex" @@ -15,24 +15,24 @@ import ( "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" - "github.com/decred/politeia/politeiad/plugins/user" + "github.com/decred/politeia/politeiad/plugins/usermd" "github.com/decred/politeia/util" "github.com/google/uuid" ) func convertSignatureError(err error) backend.PluginError { var e util.SignatureError - var s user.ErrorCodeT + var s usermd.ErrorCodeT if errors.As(err, &e) { switch e.ErrorCode { case util.ErrorStatusPublicKeyInvalid: - s = user.ErrorCodePublicKeyInvalid + s = usermd.ErrorCodePublicKeyInvalid case util.ErrorStatusSignatureInvalid: - s = user.ErrorCodeSignatureInvalid + s = usermd.ErrorCodeSignatureInvalid } } return backend.PluginError{ - PluginID: user.PluginID, + PluginID: usermd.PluginID, ErrorCode: int(s), ErrorContext: e.ErrorContext, } @@ -40,15 +40,15 @@ func convertSignatureError(err error) backend.PluginError { // userMetadataDecode decodes and returns the UserMetadata from the provided // backend metadata streams. If a UserMetadata is not found, nil is returned. -func userMetadataDecode(metadata []backend.MetadataStream) (*user.UserMetadata, error) { - var userMD *user.UserMetadata +func userMetadataDecode(metadata []backend.MetadataStream) (*usermd.UserMetadata, error) { + var userMD *usermd.UserMetadata for _, v := range metadata { - if v.PluginID != user.PluginID || - v.ID != user.MDStreamIDUserMetadata { + if v.PluginID != usermd.PluginID || + v.ID != usermd.MDStreamIDUserMetadata { // Not the mdstream we're looking for continue } - var um user.UserMetadata + var um usermd.UserMetadata err := json.Unmarshal([]byte(v.Payload), &um) if err != nil { return nil, err @@ -69,8 +69,8 @@ func userMetadataVerify(metadata []backend.MetadataStream, files []backend.File) } if um == nil { return backend.PluginError{ - PluginID: user.PluginID, - ErrorCode: int(user.ErrorCodeUserMetadataNotFound), + PluginID: usermd.PluginID, + ErrorCode: int(usermd.ErrorCodeUserMetadataNotFound), } } @@ -78,8 +78,8 @@ func userMetadataVerify(metadata []backend.MetadataStream, files []backend.File) _, err = uuid.Parse(um.UserID) if err != nil { return backend.PluginError{ - PluginID: user.PluginID, - ErrorCode: int(user.ErrorCodeUserIDInvalid), + PluginID: usermd.PluginID, + ErrorCode: int(usermd.ErrorCodeUserIDInvalid), } } @@ -119,8 +119,8 @@ func userMetadataPreventUpdates(current, update []backend.MetadataStream) error e := fmt.Sprintf("user id cannot change: got %v, want %v", u.UserID, c.UserID) return backend.PluginError{ - PluginID: user.PluginID, - ErrorCode: int(user.ErrorCodeUserIDInvalid), + PluginID: usermd.PluginID, + ErrorCode: int(usermd.ErrorCodeUserIDInvalid), ErrorContext: e, } @@ -128,8 +128,8 @@ func userMetadataPreventUpdates(current, update []backend.MetadataStream) error e := fmt.Sprintf("public key cannot change: got %v, want %v", u.PublicKey, c.PublicKey) return backend.PluginError{ - PluginID: user.PluginID, - ErrorCode: int(user.ErrorCodePublicKeyInvalid), + PluginID: usermd.PluginID, + ErrorCode: int(usermd.ErrorCodePublicKeyInvalid), ErrorContext: e, } @@ -137,8 +137,8 @@ func userMetadataPreventUpdates(current, update []backend.MetadataStream) error e := fmt.Sprintf("signature cannot change: got %v, want %v", u.Signature, c.Signature) return backend.PluginError{ - PluginID: user.PluginID, - ErrorCode: int(user.ErrorCodeSignatureInvalid), + PluginID: usermd.PluginID, + ErrorCode: int(usermd.ErrorCodeSignatureInvalid), ErrorContext: e, } } @@ -204,8 +204,8 @@ func (p *userPlugin) hookEditRecordPre(payload string) error { e := fmt.Sprintf("user id cannot change: got %v, want %v", um.UserID, umCurr.UserID) return backend.PluginError{ - PluginID: user.PluginID, - ErrorCode: int(user.ErrorCodeUserIDInvalid), + PluginID: usermd.PluginID, + ErrorCode: int(usermd.ErrorCodeUserIDInvalid), ErrorContext: e, } } @@ -224,17 +224,17 @@ func (p *userPlugin) hookEditMetadataPre(payload string) error { return userMetadataPreventUpdates(em.Current.Metadata, em.Metadata) } -func statusChangesDecode(metadata []backend.MetadataStream) ([]user.StatusChangeMetadata, error) { - statuses := make([]user.StatusChangeMetadata, 0, 16) +func statusChangesDecode(metadata []backend.MetadataStream) ([]usermd.StatusChangeMetadata, error) { + statuses := make([]usermd.StatusChangeMetadata, 0, 16) for _, v := range metadata { - if v.PluginID != user.PluginID || - v.ID != user.MDStreamIDStatusChanges { + if v.PluginID != usermd.PluginID || + v.ID != usermd.MDStreamIDStatusChanges { // Not the mdstream we're looking for continue } d := json.NewDecoder(strings.NewReader(v.Payload)) for { - var sc user.StatusChangeMetadata + var sc usermd.StatusChangeMetadata err := d.Decode(&sc) if errors.Is(err, io.EOF) { break @@ -269,8 +269,8 @@ func statusChangeMetadataVerify(rm backend.RecordMetadata, metadata []backend.Me // Verify that status change metadata is present if len(statusChanges) == 0 { return backend.PluginError{ - PluginID: user.PluginID, - ErrorCode: int(user.ErrorCodeStatusChangeMetadataNotFound), + PluginID: usermd.PluginID, + ErrorCode: int(usermd.ErrorCodeStatusChangeMetadataNotFound), } } scm := statusChanges[len(statusChanges)-1] @@ -280,8 +280,8 @@ func statusChangeMetadataVerify(rm backend.RecordMetadata, metadata []backend.Me e := fmt.Sprintf("status change token does not match record "+ "metadata token: got %v, want %v", scm.Token, rm.Token) return backend.PluginError{ - PluginID: user.PluginID, - ErrorCode: int(user.ErrorCodeTokenInvalid), + PluginID: usermd.PluginID, + ErrorCode: int(usermd.ErrorCodeTokenInvalid), ErrorContext: e, } } @@ -291,8 +291,8 @@ func statusChangeMetadataVerify(rm backend.RecordMetadata, metadata []backend.Me e := fmt.Sprintf("status from metadata does not match status from "+ "record metadata: got %v, want %v", scm.Status, rm.Status) return backend.PluginError{ - PluginID: user.PluginID, - ErrorCode: int(user.ErrorCodeStatusInvalid), + PluginID: usermd.PluginID, + ErrorCode: int(usermd.ErrorCodeStatusInvalid), ErrorContext: e, } } @@ -301,8 +301,8 @@ func statusChangeMetadataVerify(rm backend.RecordMetadata, metadata []backend.Me _, ok := statusReasonRequired[rm.Status] if ok && scm.Reason == "" { return backend.PluginError{ - PluginID: user.PluginID, - ErrorCode: int(user.ErrorCodeReasonInvalid), + PluginID: usermd.PluginID, + ErrorCode: int(usermd.ErrorCodeReasonInvalid), ErrorContext: "a reason must be given for this status change", } } diff --git a/politeiad/backend/tlogbe/plugins/user/log.go b/politeiad/backend/tlogbe/plugins/usermd/log.go similarity index 98% rename from politeiad/backend/tlogbe/plugins/user/log.go rename to politeiad/backend/tlogbe/plugins/usermd/log.go index b8ab45de7..ea391cd03 100644 --- a/politeiad/backend/tlogbe/plugins/user/log.go +++ b/politeiad/backend/tlogbe/plugins/usermd/log.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package user +package usermd import "github.com/decred/slog" diff --git a/politeiad/backend/tlogbe/plugins/user/user.go b/politeiad/backend/tlogbe/plugins/usermd/usermd.go similarity index 86% rename from politeiad/backend/tlogbe/plugins/user/user.go rename to politeiad/backend/tlogbe/plugins/usermd/usermd.go index 55586d3ff..a5f0e0814 100644 --- a/politeiad/backend/tlogbe/plugins/user/user.go +++ b/politeiad/backend/tlogbe/plugins/usermd/usermd.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package user +package usermd import ( "os" @@ -11,7 +11,7 @@ import ( "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" - "github.com/decred/politeia/politeiad/plugins/user" + "github.com/decred/politeia/politeiad/plugins/usermd" ) var ( @@ -32,7 +32,7 @@ type userPlugin struct { // // This function satisfies the plugins.PluginClient interface. func (p *userPlugin) Setup() error { - log.Tracef("user Setup") + log.Tracef("usermd Setup") return nil } @@ -41,12 +41,12 @@ func (p *userPlugin) Setup() error { // // This function satisfies the plugins.PluginClient interface. func (p *userPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { - log.Tracef("user Cmd: %v %x %v %v", treeID, token, cmd, payload) + log.Tracef("usermd Cmd: %v %x %v %v", treeID, token, cmd, payload) switch cmd { - case user.CmdAuthor: + case usermd.CmdAuthor: return p.cmdAuthor(treeID) - case user.CmdUserRecords: + case usermd.CmdUserRecords: return p.cmdUserRecords(payload) } @@ -57,7 +57,7 @@ func (p *userPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (strin // // This function satisfies the plugins.PluginClient interface. func (p *userPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { - log.Tracef("user Hook: %v %x %v", treeID, token, plugins.Hooks[h]) + log.Tracef("usermd Hook: %v %x %v", treeID, token, plugins.Hooks[h]) switch h { case plugins.HookTypeNewRecordPre: @@ -81,7 +81,7 @@ func (p *userPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload s // // This function satisfies the plugins.PluginClient interface. func (p *userPlugin) Fsck(treeIDs []int64) error { - log.Tracef("user Fsck") + log.Tracef("usermd Fsck") return nil } @@ -90,7 +90,7 @@ func (p *userPlugin) Fsck(treeIDs []int64) error { // // This function satisfies the plugins.PluginClient interface. func (p *userPlugin) Settings() []backend.PluginSetting { - log.Tracef("user Settings") + log.Tracef("usermd Settings") return nil } @@ -98,7 +98,7 @@ func (p *userPlugin) Settings() []backend.PluginSetting { // New returns a new userPlugin. func New(tlog plugins.TlogClient, settings []backend.PluginSetting, dataDir string) (*userPlugin, error) { // Create plugin data directory - dataDir = filepath.Join(dataDir, user.PluginID) + dataDir = filepath.Join(dataDir, usermd.PluginID) err := os.MkdirAll(dataDir, 0700) if err != nil { return nil, err diff --git a/politeiad/backend/tlogbe/tlog/plugin.go b/politeiad/backend/tlogbe/tlog/plugin.go index d7abf44d2..33d22c351 100644 --- a/politeiad/backend/tlogbe/tlog/plugin.go +++ b/politeiad/backend/tlogbe/tlog/plugin.go @@ -16,12 +16,12 @@ import ( "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/dcrdata" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/pi" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/ticketvote" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/user" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/usermd" cmplugin "github.com/decred/politeia/politeiad/plugins/comments" ddplugin "github.com/decred/politeia/politeiad/plugins/dcrdata" piplugin "github.com/decred/politeia/politeiad/plugins/pi" tkplugin "github.com/decred/politeia/politeiad/plugins/ticketvote" - userplugin "github.com/decred/politeia/politeiad/plugins/user" + umplugin "github.com/decred/politeia/politeiad/plugins/usermd" ) const ( @@ -93,8 +93,8 @@ func (t *Tlog) PluginRegister(b backend.Backend, p backend.Plugin) error { if err != nil { return err } - case userplugin.PluginID: - client, err = user.New(t, p.Settings, dataDir) + case umplugin.PluginID: + client, err = usermd.New(t, p.Settings, dataDir) default: return backend.ErrPluginInvalid } diff --git a/politeiad/client/user.go b/politeiad/client/user.go index f94cb7f8d..40a247a36 100644 --- a/politeiad/client/user.go +++ b/politeiad/client/user.go @@ -10,7 +10,7 @@ import ( "fmt" pdv1 "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/politeiad/plugins/user" + "github.com/decred/politeia/politeiad/plugins/usermd" ) // Author sends the user plugin Author command to the politeiad v1 API. @@ -20,8 +20,8 @@ func (c *Client) Author(ctx context.Context, state, token string) (string, error { State: state, Token: token, - ID: user.PluginID, - Command: user.CmdAuthor, + ID: usermd.PluginID, + Command: usermd.CmdAuthor, Payload: "", }, } @@ -41,7 +41,7 @@ func (c *Client) Author(ctx context.Context, state, token string) (string, error } // Decode reply - var ar user.AuthorReply + var ar usermd.AuthorReply err = json.Unmarshal([]byte(pcr.Payload), &ar) if err != nil { return "", err @@ -55,7 +55,7 @@ func (c *Client) Author(ctx context.Context, state, token string) (string, error // returned map is a map[recordState][]token. func (c *Client) UserRecords(ctx context.Context, userID string) (map[string][]string, error) { // Setup request - ur := user.UserRecords{ + ur := usermd.UserRecords{ UserID: userID, } b, err := json.Marshal(ur) @@ -65,14 +65,14 @@ func (c *Client) UserRecords(ctx context.Context, userID string) (map[string][]s cmds := []pdv1.PluginCommandV2{ { State: pdv1.RecordStateUnvetted, - ID: user.PluginID, - Command: user.CmdUserRecords, + ID: usermd.PluginID, + Command: usermd.CmdUserRecords, Payload: string(b), }, { State: pdv1.RecordStateVetted, - ID: user.PluginID, - Command: user.CmdUserRecords, + ID: usermd.PluginID, + Command: usermd.CmdUserRecords, Payload: string(b), }, } @@ -94,7 +94,7 @@ func (c *Client) UserRecords(ctx context.Context, userID string) (map[string][]s // Swallow individual errors continue } - var urr user.UserRecordsReply + var urr usermd.UserRecordsReply err = json.Unmarshal([]byte(v.Payload), &urr) if err != nil { return nil, err diff --git a/politeiad/log.go b/politeiad/log.go index 7088aaaad..41b4695a6 100644 --- a/politeiad/log.go +++ b/politeiad/log.go @@ -14,7 +14,7 @@ import ( "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/comments" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/dcrdata" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/ticketvote" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/user" + "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/usermd" "github.com/decred/politeia/politeiad/backend/tlogbe/store/fs" "github.com/decred/politeia/politeiad/backend/tlogbe/store/mysql" "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" @@ -73,7 +73,7 @@ func init() { comments.UseLogger(pluginLog) dcrdata.UseLogger(pluginLog) ticketvote.UseLogger(pluginLog) - user.UseLogger(pluginLog) + usermd.UseLogger(pluginLog) // Other loggers wsdcrdata.UseLogger(wsdcrdataLog) diff --git a/politeiad/plugins/comments/comments.go b/politeiad/plugins/comments/comments.go index b89bb19c9..f6c72f2fb 100644 --- a/politeiad/plugins/comments/comments.go +++ b/politeiad/plugins/comments/comments.go @@ -7,7 +7,7 @@ package comments const ( - // PluginID is the comments plugin ID. + // PluginID is the unique identifier for this plugin. PluginID = "comments" // Plugin commands diff --git a/politeiad/plugins/dcrdata/dcrdata.go b/politeiad/plugins/dcrdata/dcrdata.go index 20dd231ec..09fafafa6 100644 --- a/politeiad/plugins/dcrdata/dcrdata.go +++ b/politeiad/plugins/dcrdata/dcrdata.go @@ -7,6 +7,7 @@ package dcrdata const ( + // PluginID is the unique identifier for this plugin. PluginID = "dcrdata" // Plugin commands diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index edf5b4295..fca74d7b3 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -7,7 +7,7 @@ package pi const ( - // PluginID is the pi plugin ID. + // PluginID is the unique identifier for this plugin. PluginID = "pi" // Setting keys are the plugin setting keys that can be used to diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index 31c7db841..05662a68f 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -7,7 +7,7 @@ package ticketvote const ( - // PluginID is the ticketvote plugin ID. + // PluginID is the unique identifier for this plugin. PluginID = "ticketvote" // Plugin commands diff --git a/politeiad/plugins/user/user.go b/politeiad/plugins/usermd/usermd.go similarity index 95% rename from politeiad/plugins/user/user.go rename to politeiad/plugins/usermd/usermd.go index 036ec3ed7..f5f2de07b 100644 --- a/politeiad/plugins/user/user.go +++ b/politeiad/plugins/usermd/usermd.go @@ -2,12 +2,13 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -// Package user provides a politeiad plugin that extends records with user +// Package usermd provides a politeiad plugin that extends records with user // metadata and provides an API for retrieving records by user metadata. -package user +package usermd const ( - PluginID = "user" + // PluginID is the unique identifier for this plugin. + PluginID = "usermd" // Plugin commands CmdAuthor = "author" // Get record author diff --git a/politeiawww/client/error.go b/politeiawww/client/error.go index 3b5fdac5d..a0b3062c1 100644 --- a/politeiawww/client/error.go +++ b/politeiawww/client/error.go @@ -12,7 +12,7 @@ import ( cmplugin "github.com/decred/politeia/politeiad/plugins/comments" piplugin "github.com/decred/politeia/politeiad/plugins/pi" tkplugin "github.com/decred/politeia/politeiad/plugins/ticketvote" - usplugin "github.com/decred/politeia/politeiad/plugins/user" + umplugin "github.com/decred/politeia/politeiad/plugins/usermd" cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" @@ -95,8 +95,8 @@ func pluginUserErr(e ErrorReply) string { errMsg = piplugin.ErrorCodes[piplugin.ErrorCodeT(e.ErrorCode)] case tkplugin.PluginID: errMsg = tkplugin.ErrorCodes[tkplugin.ErrorCodeT(e.ErrorCode)] - case usplugin.PluginID: - errMsg = usplugin.ErrorCodes[usplugin.ErrorCodeT(e.ErrorCode)] + case umplugin.PluginID: + errMsg = umplugin.ErrorCodes[umplugin.ErrorCodeT(e.ErrorCode)] } m := fmt.Sprintf("%v plugin error code %v", e.PluginID, e.ErrorCode) if errMsg != "" { diff --git a/politeiawww/cmd/pictl/cmdproposalstatusset.go b/politeiawww/cmd/pictl/cmdproposalstatusset.go index bd66f4844..e05c5ff9b 100644 --- a/politeiawww/cmd/pictl/cmdproposalstatusset.go +++ b/politeiawww/cmd/pictl/cmdproposalstatusset.go @@ -131,8 +131,11 @@ func parseRecordStatus(status string) (rcv1.RecordStatusT, error) { statuses = map[string]rcv1.RecordStatusT{ "public": rcv1.RecordStatusPublic, + "censor": rcv1.RecordStatusCensored, "censored": rcv1.RecordStatusCensored, + "abandon": rcv1.RecordStatusArchived, "abandoned": rcv1.RecordStatusArchived, + "archive": rcv1.RecordStatusArchived, "archived": rcv1.RecordStatusArchived, "2": rcv1.RecordStatusPublic, "3": rcv1.RecordStatusCensored, @@ -162,12 +165,12 @@ admin priviledges. Valid statuses: public - censored - abandoned + censor + abandon The following statuses require a status change reason to be included: - censored - abandoned + censor + abandon Arguments: 1. token (string, required) Proposal censorship token diff --git a/politeiawww/cmd/pictl/proposal.go b/politeiawww/cmd/pictl/proposal.go index f59528f4e..0e559f152 100644 --- a/politeiawww/cmd/pictl/proposal.go +++ b/politeiawww/cmd/pictl/proposal.go @@ -15,7 +15,7 @@ import ( "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/api/v1/mime" - usplugin "github.com/decred/politeia/politeiad/plugins/user" + "github.com/decred/politeia/politeiad/plugins/usermd" piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" "github.com/decred/politeia/util" @@ -118,11 +118,11 @@ func convertProposal(p piv1.Proposal) (*rcv1.Record, error) { } metadata := []rcv1.MetadataStream{ { - ID: usplugin.MDStreamIDUserMetadata, + ID: usermd.MDStreamIDUserMetadata, Payload: string(umb), }, { - ID: usplugin.MDStreamIDStatusChanges, + ID: usermd.MDStreamIDStatusChanges, Payload: buf.String(), }, } diff --git a/politeiawww/pi/process.go b/politeiawww/pi/process.go index 99764e549..5d5307829 100644 --- a/politeiawww/pi/process.go +++ b/politeiawww/pi/process.go @@ -15,7 +15,7 @@ import ( pdv1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/plugins/pi" - pduser "github.com/decred/politeia/politeiad/plugins/user" + "github.com/decred/politeia/politeiad/plugins/usermd" v1 "github.com/decred/politeia/politeiawww/api/pi/v1" "github.com/decred/politeia/politeiawww/user" "github.com/google/uuid" @@ -180,11 +180,11 @@ func convertStatus(s pdv1.RecordStatusT) v1.PropStatusT { return v1.PropStatusInvalid } -func statusChangesDecode(payload []byte) ([]pduser.StatusChangeMetadata, error) { - statuses := make([]pduser.StatusChangeMetadata, 0, 16) +func statusChangesDecode(payload []byte) ([]usermd.StatusChangeMetadata, error) { + statuses := make([]usermd.StatusChangeMetadata, 0, 16) d := json.NewDecoder(strings.NewReader(string(payload))) for { - var sc pduser.StatusChangeMetadata + var sc usermd.StatusChangeMetadata err := d.Decode(&sc) if errors.Is(err, io.EOF) { break @@ -199,18 +199,18 @@ func statusChangesDecode(payload []byte) ([]pduser.StatusChangeMetadata, error) func convertRecord(r pdv1.Record, state string) (*v1.Proposal, error) { // Decode metadata streams var ( - um pduser.UserMetadata - sc = make([]pduser.StatusChangeMetadata, 0, 16) + um usermd.UserMetadata + sc = make([]usermd.StatusChangeMetadata, 0, 16) err error ) for _, v := range r.Metadata { switch v.ID { - case pduser.MDStreamIDUserMetadata: + case usermd.MDStreamIDUserMetadata: err = json.Unmarshal([]byte(v.Payload), &um) if err != nil { return nil, err } - case pduser.MDStreamIDStatusChanges: + case usermd.MDStreamIDStatusChanges: sc, err = statusChangesDecode([]byte(v.Payload)) if err != nil { return nil, err @@ -267,11 +267,11 @@ func convertRecord(r pdv1.Record, state string) (*v1.Proposal, error) { // userMetadataDecode decodes and returns the UserMetadata from the provided // metadata streams. If a UserMetadata is not found, nil is returned. -func userMetadataDecode(ms []pdv1.MetadataStream) (*pduser.UserMetadata, error) { - var userMD *pduser.UserMetadata +func userMetadataDecode(ms []pdv1.MetadataStream) (*usermd.UserMetadata, error) { + var userMD *usermd.UserMetadata for _, v := range ms { - if v.ID == pduser.MDStreamIDUserMetadata { - var um pduser.UserMetadata + if v.ID == usermd.MDStreamIDUserMetadata { + var um usermd.UserMetadata err := json.Unmarshal([]byte(v.Payload), &um) if err != nil { return nil, err diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 855ecb13f..c9495b9e0 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -12,7 +12,7 @@ import ( cmplugin "github.com/decred/politeia/politeiad/plugins/comments" piplugin "github.com/decred/politeia/politeiad/plugins/pi" tkplugin "github.com/decred/politeia/politeiad/plugins/ticketvote" - usplugin "github.com/decred/politeia/politeiad/plugins/user" + umplugin "github.com/decred/politeia/politeiad/plugins/usermd" cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" @@ -162,7 +162,7 @@ func (p *politeiawww) setupPi(plugins []pdv1.Plugin) error { piplugin.PluginID: false, cmplugin.PluginID: false, tkplugin.PluginID: false, - usplugin.PluginID: false, + umplugin.PluginID: false, } for _, v := range plugins { _, ok := required[v.ID] diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 6cd22a834..0ba3b3995 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -16,7 +16,7 @@ import ( piplugin "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/ticketvote" tvplugin "github.com/decred/politeia/politeiad/plugins/ticketvote" - usplugin "github.com/decred/politeia/politeiad/plugins/user" + umplugin "github.com/decred/politeia/politeiad/plugins/usermd" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/politeiawww/user" @@ -458,11 +458,11 @@ func (p *politeiawww) handleBatchVoteSummary(w http.ResponseWriter, r *http.Requ // userMetadataDecode decodes and returns the UserMetadata from the provided // metadata streams. If a UserMetadata is not found, nil is returned. -func userMetadataDecode(ms []v1.MetadataStream) (*usplugin.UserMetadata, error) { - var userMD *usplugin.UserMetadata +func userMetadataDecode(ms []v1.MetadataStream) (*umplugin.UserMetadata, error) { + var userMD *umplugin.UserMetadata for _, v := range ms { - if v.ID == usplugin.MDStreamIDUserMetadata { - var um usplugin.UserMetadata + if v.ID == umplugin.MDStreamIDUserMetadata { + var um umplugin.UserMetadata err := json.Unmarshal([]byte(v.Payload), &um) if err != nil { return nil, err @@ -505,10 +505,10 @@ func convertStatusToWWW(status pdv1.RecordStatusT) www.PropStatusT { // TODO convertRecordToProposal func convertRecordToProposal(r pdv1.Record) (*www.ProposalRecord, error) { // Decode metadata - var um *usplugin.UserMetadata + var um *umplugin.UserMetadata for _, v := range r.Metadata { switch v.ID { - case usplugin.MDStreamIDUserMetadata: + case umplugin.MDStreamIDUserMetadata: } } diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index 6df9b6d68..e00d1f57f 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -11,7 +11,7 @@ import ( "time" pdv1 "github.com/decred/politeia/politeiad/api/v1" - usplugin "github.com/decred/politeia/politeiad/plugins/user" + "github.com/decred/politeia/politeiad/plugins/usermd" v1 "github.com/decred/politeia/politeiawww/api/records/v1" "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/user" @@ -40,7 +40,7 @@ func (r *Records) processNew(ctx context.Context, n v1.New, u user.User) (*v1.Ne } // Setup metadata stream - um := usplugin.UserMetadata{ + um := usermd.UserMetadata{ UserID: u.ID.String(), PublicKey: n.PublicKey, Signature: n.Signature, @@ -51,8 +51,8 @@ func (r *Records) processNew(ctx context.Context, n v1.New, u user.User) (*v1.Ne } metadata := []pdv1.MetadataStream{ { - PluginID: usplugin.PluginID, - ID: usplugin.MDStreamIDUserMetadata, + PluginID: usermd.PluginID, + ID: usermd.MDStreamIDUserMetadata, Payload: string(b), }, } @@ -155,7 +155,7 @@ func (r *Records) processEdit(ctx context.Context, e v1.Edit, u user.User) (*v1. filesDel := filesToDel(curr.Files, filesAdd) // Setup metadata - um := usplugin.UserMetadata{ + um := usermd.UserMetadata{ UserID: u.ID.String(), PublicKey: e.PublicKey, Signature: e.Signature, @@ -166,8 +166,8 @@ func (r *Records) processEdit(ctx context.Context, e v1.Edit, u user.User) (*v1. } mdOverwrite := []pdv1.MetadataStream{ { - PluginID: usplugin.PluginID, - ID: usplugin.MDStreamIDUserMetadata, + PluginID: usermd.PluginID, + ID: usermd.MDStreamIDUserMetadata, Payload: string(b), }, } @@ -225,7 +225,7 @@ func (r *Records) processSetStatus(ctx context.Context, ss v1.SetStatus, u user. } // Setup status change metadata - scm := usplugin.StatusChangeMetadata{ + scm := usermd.StatusChangeMetadata{ Token: ss.Token, Version: ss.Version, Status: int(ss.Status), @@ -240,8 +240,8 @@ func (r *Records) processSetStatus(ctx context.Context, ss v1.SetStatus, u user. } mdAppend := []pdv1.MetadataStream{ { - PluginID: usplugin.PluginID, - ID: usplugin.MDStreamIDStatusChanges, + PluginID: usermd.PluginID, + ID: usermd.MDStreamIDStatusChanges, Payload: string(b), }, } @@ -578,15 +578,15 @@ func recordPopulateUserData(r *v1.Record, u user.User) { // userMetadataDecode decodes and returns the UserMetadata from the provided // metadata streams. If a UserMetadata is not found, nil is returned. -func userMetadataDecode(ms []v1.MetadataStream) (*usplugin.UserMetadata, error) { - var userMD *usplugin.UserMetadata +func userMetadataDecode(ms []v1.MetadataStream) (*usermd.UserMetadata, error) { + var userMD *usermd.UserMetadata for _, v := range ms { - if v.PluginID != usplugin.PluginID || - v.ID != usplugin.MDStreamIDUserMetadata { + if v.PluginID != usermd.PluginID || + v.ID != usermd.MDStreamIDUserMetadata { // Not the mdstream we're looking for continue } - var um usplugin.UserMetadata + var um usermd.UserMetadata err := json.Unmarshal([]byte(v.Payload), &um) if err != nil { return nil, err From 74c804f2cc6880279ab1032a9d0892da002efd11 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 16 Feb 2021 14:43:57 -0600 Subject: [PATCH 312/449] Move plugin record locking to backend layer. --- politeiad/api/v1/v1.go | 27 +++- politeiad/backend/backend.go | 50 +++++--- politeiad/backend/gitbe/gitbe.go | 4 +- .../backend/tlogbe/plugins/comments/cmds.go | 40 ++---- .../tlogbe/plugins/comments/comments.go | 24 +--- .../tlogbe/plugins/comments/recordindex.go | 32 ++--- politeiad/backend/tlogbe/plugins/pi/hooks.go | 4 +- politeiad/backend/tlogbe/plugins/pi/pi.go | 2 - .../backend/tlogbe/plugins/ticketvote/cmds.go | 65 +++------- .../tlogbe/plugins/ticketvote/ticketvote.go | 30 +---- politeiad/backend/tlogbe/tlogbe.go | 121 +++++++++++++++--- politeiad/client/comments.go | 7 + politeiad/client/ticketvote.go | 10 ++ politeiad/client/user.go | 2 + politeiad/politeiad.go | 10 +- politeiawww/api/records/v1/v1.go | 3 +- 16 files changed, 240 insertions(+), 191 deletions(-) diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index 382766750..645813093 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -73,10 +73,7 @@ const ( ErrorStatusInvalidToken ErrorStatusT = 18 ErrorStatusRecordLocked ErrorStatusT = 19 ErrorStatusInvalidRecordState ErrorStatusT = 20 - - // Record states - RecordStateUnvetted = "unvetted" - RecordStateVetted = "vetted" + ErrorStatusInvalidPluginAction ErrorStatusT = 21 // Record status codes (set and get) RecordStatusInvalid RecordStatusT = 0 // Invalid status @@ -119,6 +116,7 @@ var ( ErrorStatusInvalidToken: "invalid token", ErrorStatusRecordLocked: "record locked", ErrorStatusInvalidRecordState: "invalid record state", + ErrorStatusInvalidPluginAction: "invalid plugin action", } // RecordStatus converts record status codes to human readable text. @@ -577,8 +575,29 @@ type PluginCommandReply struct { Payload string `json:"payload"` // Actual command reply } +const ( + // PluginActionRead is passed in as the Action of a PluginCommandV2 + // to indicate that the plugin command is a read only command. + PluginActionRead = "read" + + // PluginActionWrite is passed in as the Action of a PluginCommandV2 + // to indicate that the plugin command requires writing data. + PluginActionWrite = "write" + + // RecordStateUnvetted is passed in as the State field of a + // PluginCommandV2 to indicate that the plugin command is being + // executed on an unvetted record. + RecordStateUnvetted = "unvetted" + + // RecordStateVetted is passed in as the State field of a + // PluginCommandV2 to indicate that the plugin command is being + // executed on an vetted record. + RecordStateVetted = "vetted" +) + // PluginCommandV2 sends a command to a plugin. type PluginCommandV2 struct { + Action string `json:"action"` // Read or write State string `json:"state"` // Unvetted or vetted Token string `json:"token"` // Censorship token ID string `json:"id"` // Plugin identifier diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index eeeb85f15..f698c5d1c 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -50,6 +50,11 @@ var ( // used. ErrPluginCmdInvalid = errors.New("plugin command invalid") + // ErrPluginActionInvalid is emitted when an invalid plugin action + // is used. See PluginActionRead and PluginActionWrite for valid + // plugin actions. + ErrPluginActionInvalid = errors.New("plugin action invalid") + // Plugin names must be all lowercase letters and have a length of <20 PluginRE = regexp.MustCompile(`^[a-z]{1,20}$`) ) @@ -111,19 +116,6 @@ func (s StateTransitionError) Error() string { s.From, MDStatus[s.From], s.To, MDStatus[s.To]) } -// PluginError represents a plugin error that is caused by the user. -type PluginError struct { - PluginID string - ErrorCode int - ErrorContext string -} - -// Error satisfies the error interface. -func (e PluginError) Error() string { - return fmt.Sprintf("plugin id '%v' error code %v", - e.PluginID, e.ErrorCode) -} - // RecordMetadata is the metadata of a record. const VersionRecordMD = 1 @@ -193,6 +185,21 @@ type RecordTimestamps struct { Files map[string]Timestamp // [filename]Timestamp } +const ( + // PluginActionRead is provided to the backend methods that execute + // plugin commands to indicate that the plugin command is a read + // only command. + PluginActionRead = "read" + + // PluginActionWrite is provided to the backend methods that execute + // plugin commands to indicate that the plugin command writes data + // to the backend. This allows the backend to prevent concurrent + // writes to the record so that individual plugin implementations + // do not need to worry about implementing logic to prevent race + // conditions. + PluginActionWrite = "write" +) + // PluginSettings are used to specify settings for a plugin at runtime. type PluginSetting struct { Key string // Name of setting @@ -211,6 +218,19 @@ type Plugin struct { Identity *identity.FullIdentity } +// PluginError represents a plugin error that is caused by the user. +type PluginError struct { + PluginID string + ErrorCode int + ErrorContext string +} + +// Error satisfies the error interface. +func (e PluginError) Error() string { + return fmt.Sprintf("plugin id '%v' error code %v", + e.PluginID, e.ErrorCode) +} + const ( // StateUnvetted is used to request the inventory of an unvetted // status. @@ -298,11 +318,11 @@ type Backend interface { SetupVettedPlugin(pluginID string) error // Execute a unvetted plugin command - UnvettedPluginCmd(token []byte, pluginID, + UnvettedPluginCmd(action string, token []byte, pluginID, cmd, payload string) (string, error) // Execute a vetted plugin command - VettedPluginCmd(token []byte, pluginID, + VettedPluginCmd(action string, token []byte, pluginID, cmd, payload string) (string, error) // Get unvetted plugins diff --git a/politeiad/backend/gitbe/gitbe.go b/politeiad/backend/gitbe/gitbe.go index ad8395291..93180963b 100644 --- a/politeiad/backend/gitbe/gitbe.go +++ b/politeiad/backend/gitbe/gitbe.go @@ -2878,14 +2878,14 @@ func (g *gitBackEnd) SetupVettedPlugin(pluginID string) error { // UnvettedPluginCmd has not been implemented. // // This function satisfies the backend.Backend interface. -func (g *gitBackEnd) UnvettedPluginCmd(token []byte, pluginID, cmd, payload string) (string, error) { +func (g *gitBackEnd) UnvettedPluginCmd(action string, token []byte, pluginID, cmd, payload string) (string, error) { return "", fmt.Errorf("not implemented") } // VettedPluginCmd has not been implemented. // // This function satisfies the backend.Backend interface. -func (g *gitBackEnd) VettedPluginCmd(token []byte, pluginID, cmd, payload string) (string, error) { +func (g *gitBackEnd) VettedPluginCmd(action string, token []byte, pluginID, cmd, payload string) (string, error) { return "", fmt.Errorf("not implemented") } diff --git a/politeiad/backend/tlogbe/plugins/comments/cmds.go b/politeiad/backend/tlogbe/plugins/comments/cmds.go index c57d9f897..db9160d2e 100644 --- a/politeiad/backend/tlogbe/plugins/comments/cmds.go +++ b/politeiad/backend/tlogbe/plugins/comments/cmds.go @@ -636,14 +636,8 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str } } - // The record index must be pulled and updated. The record lock - // must be held for the remainder of this function. - m := p.mutex(token) - m.Lock() - defer m.Unlock() - // Get record index - ridx, err := p.recordIndexLocked(token) + ridx, err := p.recordIndex(token) if err != nil { return "", err } @@ -691,7 +685,7 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str } // Save index - err = p.recordIndexSaveLocked(token, *ridx) + err = p.recordIndexSave(token, *ridx) if err != nil { return "", err } @@ -763,14 +757,8 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st } } - // The record index must be pulled and updated. The record lock - // must be held for the remainder of this function. - m := p.mutex(token) - m.Lock() - defer m.Unlock() - // Get record index - ridx, err := p.recordIndexLocked(token) + ridx, err := p.recordIndex(token) if err != nil { return "", err } @@ -842,7 +830,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st ridx.Comments[ca.CommentID].Adds[ca.Version] = digest // Save index - err = p.recordIndexSaveLocked(token, *ridx) + err = p.recordIndexSave(token, *ridx) if err != nil { return "", err } @@ -904,14 +892,8 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str return "", convertSignatureError(err) } - // The record index must be pulled and updated. The record lock - // must be held for the remainder of this function. - m := p.mutex(token) - m.Lock() - defer m.Unlock() - // Get record index - ridx, err := p.recordIndexLocked(token) + ridx, err := p.recordIndex(token) if err != nil { return "", err } @@ -960,7 +942,7 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str ridx.Comments[d.CommentID] = cidx // Save index - err = p.recordIndexSaveLocked(token, *ridx) + err = p.recordIndexSave(token, *ridx) if err != nil { return "", err } @@ -1041,14 +1023,8 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st return "", convertSignatureError(err) } - // The record index must be pulled and updated. The record lock - // must be held for the remainder of this function. - m := p.mutex(token) - m.Lock() - defer m.Unlock() - // Get record index - ridx, err := p.recordIndexLocked(token) + ridx, err := p.recordIndex(token) if err != nil { return "", err } @@ -1125,7 +1101,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st ridx.Comments[cv.CommentID] = cidx // Save index - err = p.recordIndexSaveLocked(token, *ridx) + err = p.recordIndexSave(token, *ridx) if err != nil { return "", err } diff --git a/politeiad/backend/tlogbe/plugins/comments/comments.go b/politeiad/backend/tlogbe/plugins/comments/comments.go index 8e86d5b86..2add44275 100644 --- a/politeiad/backend/tlogbe/plugins/comments/comments.go +++ b/politeiad/backend/tlogbe/plugins/comments/comments.go @@ -5,7 +5,6 @@ package comments import ( - "encoding/hex" "fmt" "os" "path/filepath" @@ -30,7 +29,7 @@ var ( // // commentsPlugin satisfies the plugins.PluginClient interface. type commentsPlugin struct { - sync.Mutex + sync.RWMutex tlog plugins.TlogClient // dataDir is the comments plugin data directory. The only data @@ -43,31 +42,11 @@ type commentsPlugin struct { // prove the backend received and processed a plugin command. identity *identity.FullIdentity - // Mutexes contains a mutex for each record. The mutexes are lazy - // loaded. - mutexes map[string]*sync.Mutex // [string]mutex - // Plugin settings commentLengthMax uint32 voteChangesMax uint32 } -// mutex returns the mutex for a record. -func (p *commentsPlugin) mutex(token []byte) *sync.Mutex { - p.Lock() - defer p.Unlock() - - t := hex.EncodeToString(token) - m, ok := p.mutexes[t] - if !ok { - // Mutexes is lazy loaded - m = &sync.Mutex{} - p.mutexes[t] = m - } - - return m -} - // Setup performs any plugin setup that is required. // // This function satisfies the plugins.PluginClient interface. @@ -186,7 +165,6 @@ func New(tlog plugins.TlogClient, settings []backend.PluginSetting, dataDir stri tlog: tlog, identity: id, dataDir: dataDir, - mutexes: make(map[string]*sync.Mutex), commentLengthMax: commentLengthMax, voteChangesMax: voteChangesMax, }, nil diff --git a/politeiad/backend/tlogbe/plugins/comments/recordindex.go b/politeiad/backend/tlogbe/plugins/comments/recordindex.go index d36356f97..7dc101591 100644 --- a/politeiad/backend/tlogbe/plugins/comments/recordindex.go +++ b/politeiad/backend/tlogbe/plugins/comments/recordindex.go @@ -56,11 +56,14 @@ func (p *commentsPlugin) recordIndexPath(token []byte) string { return filepath.Join(p.dataDir, fn) } -// recordIndexLocked returns the cached recordIndex for the provided record. -// If a cached recordIndex does not exist, a new one will be returned. +// recordIndex returns the cached recordIndex for the provided record. If a +// cached recordIndex does not exist, a new one will be returned. // -// This function must be called WITH the lock held. -func (p *commentsPlugin) recordIndexLocked(token []byte) (*recordIndex, error) { +// This function must be called WITHOUT the read lock held. +func (p *commentsPlugin) recordIndex(token []byte) (*recordIndex, error) { + p.RLock() + defer p.RUnlock() + fp := p.recordIndexPath(token) b, err := ioutil.ReadFile(fp) if err != nil { @@ -83,23 +86,14 @@ func (p *commentsPlugin) recordIndexLocked(token []byte) (*recordIndex, error) { return &ridx, nil } -// recordIndex returns the cached recordIndex for the provided record. If a -// cached recordIndex does not exist, a new one will be returned. +// recordIndexSave saves the provided recordIndex to the comments plugin data +// dir. // -// This function must be called WITHOUT the lock held. -func (p *commentsPlugin) recordIndex(token []byte) (*recordIndex, error) { - m := p.mutex(token) - m.Lock() - defer m.Unlock() +// This function must be called WITHOUT the read/write lock held. +func (p *commentsPlugin) recordIndexSave(token []byte, ridx recordIndex) error { + p.Lock() + defer p.Unlock() - return p.recordIndexLocked(token) -} - -// recordIndexSaveLocked saves the provided recordIndex to the comments -// plugin data dir. -// -// This function must be called WITH the lock held. -func (p *commentsPlugin) recordIndexSaveLocked(token []byte, ridx recordIndex) error { b, err := json.Marshal(ridx) if err != nil { return err diff --git a/politeiad/backend/tlogbe/plugins/pi/hooks.go b/politeiad/backend/tlogbe/plugins/pi/hooks.go index 40ca4377e..38fc7ee27 100644 --- a/politeiad/backend/tlogbe/plugins/pi/hooks.go +++ b/politeiad/backend/tlogbe/plugins/pi/hooks.go @@ -221,8 +221,8 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { } func (p *piPlugin) voteSummary(token []byte) (*ticketvote.SummaryReply, error) { - reply, err := p.backend.VettedPluginCmd(token, - ticketvote.PluginID, ticketvote.CmdSummary, "") + reply, err := p.backend.VettedPluginCmd(backend.PluginActionRead, + token, ticketvote.PluginID, ticketvote.CmdSummary, "") if err != nil { return nil, err } diff --git a/politeiad/backend/tlogbe/plugins/pi/pi.go b/politeiad/backend/tlogbe/plugins/pi/pi.go index 8ae32d2a6..6c8350dbb 100644 --- a/politeiad/backend/tlogbe/plugins/pi/pi.go +++ b/politeiad/backend/tlogbe/plugins/pi/pi.go @@ -11,7 +11,6 @@ import ( "path/filepath" "regexp" "strconv" - "sync" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" @@ -25,7 +24,6 @@ var ( // piPlugin satisfies the plugins.PluginClient interface. type piPlugin struct { - sync.Mutex backend backend.Backend // dataDir is the pi plugin data directory. The only data that is diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go index d8180dac2..f6ed875fa 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go @@ -374,8 +374,8 @@ func (p *ticketVotePlugin) voteDetails(treeID int64) (*ticketvote.VoteDetails, e } func (p *ticketVotePlugin) voteDetailsByToken(token []byte) (*ticketvote.VoteDetails, error) { - reply, err := p.backend.VettedPluginCmd(token, ticketvote.PluginID, - ticketvote.CmdDetails, "") + reply, err := p.backend.VettedPluginCmd(backend.PluginActionRead, + token, ticketvote.PluginID, ticketvote.CmdDetails, "") if err != nil { return nil, err } @@ -440,8 +440,8 @@ func (p *ticketVotePlugin) voteOptionResults(token []byte, options []ticketvote. default: // Votes are not in the cache. Pull them from the backend. - reply, err := p.backend.VettedPluginCmd(token, ticketvote.PluginID, - ticketvote.CmdResults, "") + reply, err := p.backend.VettedPluginCmd(backend.PluginActionRead, + token, ticketvote.PluginID, ticketvote.CmdResults, "") var rr ticketvote.ResultsReply err = json.Unmarshal([]byte(reply), &rr) if err != nil { @@ -477,8 +477,8 @@ func (p *ticketVotePlugin) summariesForRunoff(parentToken string) (map[string]ti if err != nil { return nil, err } - reply, err := p.backend.VettedPluginCmd(parent, ticketvote.PluginID, - cmdRunoffDetails, "") + reply, err := p.backend.VettedPluginCmd(backend.PluginActionRead, + parent, ticketvote.PluginID, cmdRunoffDetails, "") if err != nil { return nil, fmt.Errorf("VettedPluginCmd %x %v %v: %v", parent, ticketvote.PluginID, cmdRunoffDetails, err) @@ -720,8 +720,8 @@ func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) } func (p *ticketVotePlugin) summaryByToken(token []byte) (*ticketvote.SummaryReply, error) { - reply, err := p.backend.VettedPluginCmd(token, ticketvote.PluginID, - ticketvote.CmdSummary, "") + reply, err := p.backend.VettedPluginCmd(backend.PluginActionRead, + token, ticketvote.PluginID, ticketvote.CmdSummary, "") if err != nil { return nil, fmt.Errorf("VettedPluginCmd %x %v %v: %v", token, ticketvote.PluginID, ticketvote.CmdSummary, err) @@ -768,8 +768,8 @@ func (p *ticketVotePlugin) bestBlock() (uint32, error) { if err != nil { return 0, err } - reply, err := p.backend.VettedPluginCmd([]byte{}, dcrdata.PluginID, - dcrdata.CmdBestBlock, string(payload)) + reply, err := p.backend.VettedPluginCmd(backend.PluginActionRead, + []byte{}, dcrdata.PluginID, dcrdata.CmdBestBlock, string(payload)) if err != nil { return 0, fmt.Errorf("Plugin %v %v: %v", dcrdata.PluginID, dcrdata.CmdBestBlock, err) @@ -804,8 +804,8 @@ func (p *ticketVotePlugin) bestBlockUnsafe() (uint32, error) { if err != nil { return 0, err } - reply, err := p.backend.VettedPluginCmd([]byte{}, dcrdata.PluginID, - dcrdata.CmdBestBlock, string(payload)) + reply, err := p.backend.VettedPluginCmd(backend.PluginActionRead, + []byte{}, dcrdata.PluginID, dcrdata.CmdBestBlock, string(payload)) if err != nil { return 0, fmt.Errorf("Plugin %v %v: %v", dcrdata.PluginID, dcrdata.CmdBestBlock, err) @@ -839,8 +839,8 @@ func (p *ticketVotePlugin) largestCommitmentAddrs(tickets []string) ([]commitmen if err != nil { return nil, err } - reply, err := p.backend.VettedPluginCmd([]byte{}, dcrdata.PluginID, - dcrdata.CmdTxsTrimmed, string(payload)) + reply, err := p.backend.VettedPluginCmd(backend.PluginActionRead, + []byte{}, dcrdata.PluginID, dcrdata.CmdTxsTrimmed, string(payload)) if err != nil { return nil, fmt.Errorf("Plugin %v %v: %v", dcrdata.PluginID, dcrdata.CmdTxsTrimmed, err) @@ -909,8 +909,8 @@ func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, if err != nil { return nil, err } - reply, err := p.backend.VettedPluginCmd([]byte{}, dcrdata.PluginID, - dcrdata.CmdBlockDetails, string(payload)) + reply, err := p.backend.VettedPluginCmd(backend.PluginActionRead, + []byte{}, dcrdata.PluginID, dcrdata.CmdBlockDetails, string(payload)) if err != nil { return nil, fmt.Errorf("Plugin %v %v: %v", dcrdata.PluginID, dcrdata.CmdBlockDetails, err) @@ -933,8 +933,8 @@ func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, if err != nil { return nil, err } - reply, err = p.backend.VettedPluginCmd([]byte{}, dcrdata.PluginID, - dcrdata.CmdTicketPool, string(payload)) + reply, err = p.backend.VettedPluginCmd(backend.PluginActionRead, + []byte{}, dcrdata.PluginID, dcrdata.CmdTicketPool, string(payload)) if err != nil { return nil, fmt.Errorf("Plugin %v %v: %v", dcrdata.PluginID, dcrdata.CmdTicketPool, err) @@ -1014,13 +1014,6 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri } } - // The previous authorize votes must be retrieved to validate the - // new autorize vote. The lock must be held for the remainder of - // this function. - m := p.mutex(token) - m.Lock() - defer m.Unlock() - // Get any previous authorizations to verify that the new action // is allowed based on the previous action. auths, err := p.auths(treeID) @@ -1322,12 +1315,6 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot return nil, err } - // Validate existing record state. The lock for this record must be - // held for the remainder of this function. - m := p.mutex(token) - m.Lock() - defer m.Unlock() - // Verify record version r, err := p.backend.GetVetted(token, "") if err != nil { @@ -1611,12 +1598,6 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti return nil, err } - // The parent record must be validated. Hold the lock on the parent - // record for the remainder of this function. - m := p.mutex(token) - m.Lock() - defer m.Unlock() - // Verify parent has a LinkBy and the LinkBy deadline is expired. r, err := p.backend.GetVetted(token, "") if err != nil { @@ -1890,8 +1871,8 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. if err != nil { return nil, err } - _, err = p.backend.VettedPluginCmd(token, ticketvote.PluginID, - cmdStartRunoffSubmission, string(b)) + _, err = p.backend.VettedPluginCmd(backend.PluginActionWrite, + token, ticketvote.PluginID, cmdStartRunoffSubmission, string(b)) if err != nil { var ue backend.PluginError if errors.As(err, &ue) { @@ -2299,12 +2280,6 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str } } - // The record lock must be held for the remainder of the function to - // ensure duplicate votes cannot be cast. - m := p.mutex(token) - m.Lock() - defer m.Unlock() - // votesCache contains the tickets that have alread voted votesCache := p.votesCache(token) for k, v := range votes { diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go index 480be8310..f848170f7 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go @@ -5,7 +5,6 @@ package ticketvote import ( - "encoding/hex" "encoding/json" "fmt" "os" @@ -48,14 +47,6 @@ type ticketVotePlugin struct { votes map[string]map[string]string // [token][ticket]voteBit mtxVotes sync.Mutex - // Mutexes contains a mutex for each record and are used to lock - // the trillian tree for a given record to prevent concurrent - // ticket vote plugin updates on the same tree. These mutexes are - // lazy loaded and should only be used for tree updates, not for - // cache updates. - mutexes map[string]*sync.Mutex // [string]mutex - mtxRecords sync.Mutex - // Mutexes for on-disk caches mtxInv sync.RWMutex // Vote inventory cache mtxSummary sync.Mutex // Vote summaries cache @@ -68,22 +59,6 @@ type ticketVotePlugin struct { voteDurationMax uint32 // In blocks } -// mutex returns the mutex for the specified record. -func (p *ticketVotePlugin) mutex(token []byte) *sync.Mutex { - p.mtxRecords.Lock() - defer p.mtxRecords.Unlock() - - t := hex.EncodeToString(token) - m, ok := p.mutexes[t] - if !ok { - // Mutexes is lazy loaded - m = &sync.Mutex{} - p.mutexes[t] = m - } - - return m -} - // Setup performs any plugin setup that is required. // // This function satisfies the plugins.PluginClient interface. @@ -129,8 +104,8 @@ func (p *ticketVotePlugin) Setup() error { if err != nil { return err } - reply, err := p.backend.VettedPluginCmd(token, ticketvote.PluginID, - ticketvote.CmdResults, "") + reply, err := p.backend.VettedPluginCmd(backend.PluginActionRead, + token, ticketvote.PluginID, ticketvote.CmdResults, "") if err != nil { return fmt.Errorf("VettedPluginCmd %x %v %v: %v", token, ticketvote.PluginID, ticketvote.CmdResults, err) @@ -328,7 +303,6 @@ func New(backend backend.Backend, tlog plugins.TlogClient, settings []backend.Pl dataDir: dataDir, identity: id, votes: make(map[string]map[string]string), - mutexes: make(map[string]*sync.Mutex), linkByPeriodMin: linkByPeriodMin, linkByPeriodMax: linkByPeriodMax, voteDurationMin: voteDurationMin, diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 5d929523d..b77dad087 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -67,9 +67,9 @@ type tlogBackend struct { // This cache memoizes those results and is lazy loaded. vettedTreeIDs map[string]int64 // [token]treeID - // recordMtxs allows a the backend to hold a lock on a record so - // that it can perform multiple read/write operations in a - // concurrent safe manner. These mutexes are lazy loaded. + // recordMtxs allows the backend to hold a lock on an individual + // record so that it can perform multiple read/write operations + // in a concurrent safe manner. These mutexes are lazy loaded. recordMtxs map[string]*sync.Mutex } @@ -1535,16 +1535,7 @@ func (t *tlogBackend) SetupVettedPlugin(pluginID string) error { return t.vetted.PluginSetup(pluginID) } -// UnvettedPluginCmd executes a plugin command on an unvetted record. -// -// This function satisfies the backend.Backend interface. -func (t *tlogBackend) UnvettedPluginCmd(token []byte, pluginID, cmd, payload string) (string, error) { - log.Tracef("UnvettedPluginCmd: %x %v %v", token, pluginID, cmd) - - if t.isShutdown() { - return "", backend.ErrShutdown - } - +func (t *tlogBackend) unvettedPluginRead(token []byte, pluginID, cmd, payload string) (string, error) { // The token is optional. If a token is not provided then a tree ID // will not be provided to the plugin. var treeID int64 @@ -1566,6 +1557,28 @@ func (t *tlogBackend) UnvettedPluginCmd(token []byte, pluginID, cmd, payload str pluginID, cmd) } + return t.unvetted.PluginCmd(treeID, token, pluginID, cmd, payload) +} + +func (t *tlogBackend) unvettedPluginWrite(token []byte, pluginID, cmd, payload string) (string, error) { + // Get tree ID + treeID := t.unvettedTreeIDFromToken(token) + + // Verify unvetted record exists + if !t.UnvettedExists(token) { + return "", backend.ErrRecordNotFound + } + + log.Debugf("Unvetted '%v' plugin cmd '%v' on record %x", + pluginID, cmd, token) + + // Hold the record lock for the remainder of this function. We + // do this here in the backend so that the individual plugins + // implementations don't need to worry about race conditions. + m := t.recordMutex(token) + m.Lock() + defer m.Unlock() + // Call pre plugin hooks hp := plugins.HookPluginPre{ State: plugins.RecordStateUnvetted, @@ -1583,6 +1596,7 @@ func (t *tlogBackend) UnvettedPluginCmd(token []byte, pluginID, cmd, payload str return "", err } + // Execute plugin command reply, err := t.unvetted.PluginCmd(treeID, token, pluginID, cmd, payload) if err != nil { return "", err @@ -1606,16 +1620,39 @@ func (t *tlogBackend) UnvettedPluginCmd(token []byte, pluginID, cmd, payload str return reply, nil } -// VettedPluginCmd executes a plugin command on an unvetted record. +// UnvettedPluginCmd executes a plugin command on an unvetted record. // // This function satisfies the backend.Backend interface. -func (t *tlogBackend) VettedPluginCmd(token []byte, pluginID, cmd, payload string) (string, error) { - log.Tracef("VettedPluginCmd: %x %v %v", token, pluginID, cmd) +func (t *tlogBackend) UnvettedPluginCmd(action string, token []byte, pluginID, cmd, payload string) (string, error) { + log.Tracef("UnvettedPluginCmd: %v %x %v %v", action, token, pluginID, cmd) if t.isShutdown() { return "", backend.ErrShutdown } + var ( + reply string + err error + ) + switch action { + case backend.PluginActionRead: + reply, err = t.unvettedPluginRead(token, pluginID, cmd, payload) + if err != nil { + return "", err + } + case backend.PluginActionWrite: + reply, err = t.unvettedPluginWrite(token, pluginID, cmd, payload) + if err != nil { + return "", err + } + default: + return "", backend.ErrPluginActionInvalid + } + + return reply, nil +} + +func (t *tlogBackend) vettedPluginRead(token []byte, pluginID, cmd, payload string) (string, error) { // The token is optional. If a token is not provided then a tree ID // will not be provided to the plugin. var treeID int64 @@ -1636,6 +1673,26 @@ func (t *tlogBackend) VettedPluginCmd(token []byte, pluginID, cmd, payload strin pluginID, cmd) } + return t.vetted.PluginCmd(treeID, token, pluginID, cmd, payload) +} + +func (t *tlogBackend) vettedPluginWrite(token []byte, pluginID, cmd, payload string) (string, error) { + // Get tree ID + treeID, ok := t.vettedTreeIDFromToken(token) + if !ok { + return "", backend.ErrRecordNotFound + } + + log.Debugf("Vetted '%v' plugin cmd '%v' on record %x", + pluginID, cmd, token) + + // Hold the record lock for the remainder of this function. We + // do this here in the backend so that the individual plugins + // implementations don't need to worry about race conditions. + m := t.recordMutex(token) + m.Lock() + defer m.Unlock() + // Call pre plugin hooks hp := plugins.HookPluginPre{ State: plugins.RecordStateVetted, @@ -1677,6 +1734,38 @@ func (t *tlogBackend) VettedPluginCmd(token []byte, pluginID, cmd, payload strin return reply, nil } +// VettedPluginCmd executes a plugin command on an unvetted record. +// +// This function satisfies the backend.Backend interface. +func (t *tlogBackend) VettedPluginCmd(action string, token []byte, pluginID, cmd, payload string) (string, error) { + log.Tracef("VettedPluginCmd: %v %x %v %v", action, token, pluginID, cmd) + + if t.isShutdown() { + return "", backend.ErrShutdown + } + + var ( + reply string + err error + ) + switch action { + case backend.PluginActionRead: + reply, err = t.vettedPluginRead(token, pluginID, cmd, payload) + if err != nil { + return "", err + } + case backend.PluginActionWrite: + reply, err = t.vettedPluginWrite(token, pluginID, cmd, payload) + if err != nil { + return "", err + } + default: + return "", backend.ErrPluginActionInvalid + } + + return reply, nil +} + // GetUnvettedPlugins returns the unvetted plugins that have been registered. // // This function satisfies the backend.Backend interface. diff --git a/politeiad/client/comments.go b/politeiad/client/comments.go index d8ff21adf..98dedaf63 100644 --- a/politeiad/client/comments.go +++ b/politeiad/client/comments.go @@ -23,6 +23,7 @@ func (c *Client) CommentNew(ctx context.Context, state string, n comments.New) ( } cmds := []pdv1.PluginCommandV2{ { + Action: pdv1.PluginActionWrite, State: state, Token: n.Token, ID: comments.PluginID, @@ -64,6 +65,7 @@ func (c *Client) CommentVote(ctx context.Context, state string, v comments.Vote) } cmds := []pdv1.PluginCommandV2{ { + Action: pdv1.PluginActionWrite, State: state, Token: v.Token, ID: comments.PluginID, @@ -105,6 +107,7 @@ func (c *Client) CommentDel(ctx context.Context, state string, d comments.Del) ( } cmds := []pdv1.PluginCommandV2{ { + Action: pdv1.PluginActionWrite, State: state, Token: d.Token, ID: comments.PluginID, @@ -146,6 +149,7 @@ func (c *Client) CommentCount(ctx context.Context, state string, tokens []string cmds := make([]pdv1.PluginCommandV2, 0, len(tokens)) for _, v := range tokens { cmds = append(cmds, pdv1.PluginCommandV2{ + Action: pdv1.PluginActionRead, State: state, Token: v, ID: comments.PluginID, @@ -190,6 +194,7 @@ func (c *Client) CommentGetAll(ctx context.Context, state, token string) ([]comm // Setup request cmds := []pdv1.PluginCommandV2{ { + Action: pdv1.PluginActionRead, State: state, Token: token, ID: comments.PluginID, @@ -231,6 +236,7 @@ func (c *Client) CommentVotes(ctx context.Context, state, token string, v commen } cmds := []pdv1.PluginCommandV2{ { + Action: pdv1.PluginActionRead, State: state, Token: token, ID: comments.PluginID, @@ -273,6 +279,7 @@ func (c *Client) CommentTimestamps(ctx context.Context, state, token string, t c } cmds := []pdv1.PluginCommandV2{ { + Action: pdv1.PluginActionRead, State: state, Token: token, ID: comments.PluginID, diff --git a/politeiad/client/ticketvote.go b/politeiad/client/ticketvote.go index c9bf07aec..701e54f3f 100644 --- a/politeiad/client/ticketvote.go +++ b/politeiad/client/ticketvote.go @@ -23,6 +23,7 @@ func (c *Client) TicketVoteAuthorize(ctx context.Context, a ticketvote.Authorize } cmds := []pdv1.PluginCommandV2{ { + Action: pdv1.PluginActionWrite, State: pdv1.RecordStateVetted, Token: a.Token, ID: ticketvote.PluginID, @@ -65,6 +66,7 @@ func (c *Client) TicketVoteStart(ctx context.Context, token string, s ticketvote } cmds := []pdv1.PluginCommandV2{ { + Action: pdv1.PluginActionWrite, State: pdv1.RecordStateVetted, Token: token, ID: ticketvote.PluginID, @@ -107,6 +109,7 @@ func (c *Client) TicketVoteCastBallot(ctx context.Context, token string, cb tick } cmds := []pdv1.PluginCommandV2{ { + Action: pdv1.PluginActionWrite, State: pdv1.RecordStateVetted, Token: token, ID: ticketvote.PluginID, @@ -145,6 +148,7 @@ func (c *Client) TicketVoteDetails(ctx context.Context, token string) (*ticketvo // Setup request cmds := []pdv1.PluginCommandV2{ { + Action: pdv1.PluginActionRead, State: pdv1.RecordStateVetted, Token: token, ID: ticketvote.PluginID, @@ -183,6 +187,7 @@ func (c *Client) TicketVoteResults(ctx context.Context, token string) (*ticketvo // Setup request cmds := []pdv1.PluginCommandV2{ { + Action: pdv1.PluginActionRead, State: pdv1.RecordStateVetted, Token: token, ID: ticketvote.PluginID, @@ -221,6 +226,7 @@ func (c *Client) TicketVoteSummary(ctx context.Context, token string) (*ticketvo // Setup request cmds := []pdv1.PluginCommandV2{ { + Action: pdv1.PluginActionRead, State: pdv1.RecordStateVetted, ID: ticketvote.PluginID, Command: ticketvote.CmdSummary, @@ -261,6 +267,7 @@ func (c *Client) TicketVoteSummaries(ctx context.Context, tokens []string) (map[ cmds := make([]pdv1.PluginCommandV2, 0, len(tokens)) for _, v := range tokens { cmds = append(cmds, pdv1.PluginCommandV2{ + Action: pdv1.PluginActionRead, State: pdv1.RecordStateVetted, Token: v, ID: ticketvote.PluginID, @@ -301,6 +308,7 @@ func (c *Client) TicketVoteSubmissions(ctx context.Context, token string) (*tick // Setup request cmds := []pdv1.PluginCommandV2{ { + Action: pdv1.PluginActionRead, State: pdv1.RecordStateVetted, Token: token, ID: ticketvote.PluginID, @@ -343,6 +351,7 @@ func (c *Client) TicketVoteInventory(ctx context.Context, i ticketvote.Inventory } cmds := []pdv1.PluginCommandV2{ { + Action: pdv1.PluginActionRead, State: pdv1.RecordStateVetted, ID: ticketvote.PluginID, Command: ticketvote.CmdInventory, @@ -384,6 +393,7 @@ func (c *Client) TicketVoteTimestamps(ctx context.Context, t ticketvote.Timestam } cmds := []pdv1.PluginCommandV2{ { + Action: pdv1.PluginActionRead, State: pdv1.RecordStateVetted, ID: ticketvote.PluginID, Command: ticketvote.CmdTimestamps, diff --git a/politeiad/client/user.go b/politeiad/client/user.go index 40a247a36..a95a67bb0 100644 --- a/politeiad/client/user.go +++ b/politeiad/client/user.go @@ -18,6 +18,7 @@ func (c *Client) Author(ctx context.Context, state, token string) (string, error // Setup request cmds := []pdv1.PluginCommandV2{ { + Action: pdv1.PluginActionRead, State: state, Token: token, ID: usermd.PluginID, @@ -64,6 +65,7 @@ func (c *Client) UserRecords(ctx context.Context, userID string) (map[string][]s } cmds := []pdv1.PluginCommandV2{ { + Action: pdv1.PluginActionRead, State: pdv1.RecordStateUnvetted, ID: usermd.PluginID, Command: usermd.CmdUserRecords, diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 06496da78..c4ee84789 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -1270,10 +1270,10 @@ func (p *politeia) pluginCommandBatch(w http.ResponseWriter, r *http.Request) { var payload string switch pc.State { case v1.RecordStateUnvetted: - payload, err = p.backend.UnvettedPluginCmd(token, + payload, err = p.backend.UnvettedPluginCmd(pc.Action, token, pc.ID, pc.Command, pc.Payload) case v1.RecordStateVetted: - payload, err = p.backend.VettedPluginCmd(token, + payload, err = p.backend.VettedPluginCmd(pc.Action, token, pc.ID, pc.Command, pc.Payload) default: replies[k] = v1.PluginCommandReplyV2{ @@ -1297,6 +1297,12 @@ func (p *politeia) pluginCommandBatch(w http.ResponseWriter, r *http.Request) { ErrorContext: []string{e.ErrorContext}, }, } + case err == backend.ErrPluginActionInvalid: + replies[k] = v1.PluginCommandReplyV2{ + UserError: &v1.UserErrorReply{ + ErrorCode: v1.ErrorStatusInvalidPluginAction, + }, + } case err == backend.ErrRecordNotFound: replies[k] = v1.PluginCommandReplyV2{ UserError: &v1.UserErrorReply{ diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go index 02b677c8c..c5703ef9d 100644 --- a/politeiawww/api/records/v1/v1.go +++ b/politeiawww/api/records/v1/v1.go @@ -279,10 +279,11 @@ type SetStatusReply struct { } // Details requests the details of a record. The full record will be returned. +// If no version is specified then the most recent version will be returned. type Details struct { Token string `json:"token"` State string `json:"state"` - Version string `json:"version"` + Version string `json:"version,omitempty"` } // DetailsReply is the reply to the Details command. From 58325a6ca540b5d9e5c83bf1f6141cb98574c241 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 16 Feb 2021 18:47:37 -0600 Subject: [PATCH 313/449] Cleanup. --- politeiawww/api/pi/v1/v1.go | 4 ---- politeiawww/api/ticketvote/v1/v1.go | 15 ++++++++------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index d0140fbf4..05319e599 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -8,10 +8,6 @@ import ( "fmt" ) -// TODO verify that all batched request have a page size limit -// comment timestamps, vote timestamps -// TODO module these API packages - const ( // APIRoute is prefixed onto all routes defined in this package. APIRoute = "/pi/v1" diff --git a/politeiawww/api/ticketvote/v1/v1.go b/politeiawww/api/ticketvote/v1/v1.go index 19e243b1f..a3adec6bf 100644 --- a/politeiawww/api/ticketvote/v1/v1.go +++ b/politeiawww/api/ticketvote/v1/v1.go @@ -27,18 +27,19 @@ type ErrorCodeT int const ( // Error codes - ErrorCodeInvalid ErrorCodeT = 0 - ErrorCodeInputInvalid ErrorCodeT = 1 - - ErrorCodePublicKeyInvalid ErrorCodeT = iota - ErrorCodeUnauthorized + ErrorCodeInvalid ErrorCodeT = 0 + ErrorCodeInputInvalid ErrorCodeT = 1 + ErrorCodePublicKeyInvalid ErrorCodeT = 2 + ErrorCodeUnauthorized ErrorCodeT = 3 ) var ( // ErrorCodes contains the human readable errors. ErrorCodes = map[ErrorCodeT]string{ - ErrorCodeInvalid: "error invalid", - ErrorCodeInputInvalid: "input invalid", + ErrorCodeInvalid: "error invalid", + ErrorCodeInputInvalid: "input invalid", + ErrorCodePublicKeyInvalid: "public key invalid", + ErrorCodeUnauthorized: "unauthorized", } ) From d25988216eae6b483b75d2f5788af459bf9b54b1 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 17 Feb 2021 14:41:42 -0600 Subject: [PATCH 314/449] Add pictl command testproposalcapacity. --- politeiawww/cmd/pictl/cmdproposaledit.go | 191 +++---- politeiawww/cmd/pictl/cmdproposalnew.go | 151 +++--- politeiawww/cmd/pictl/cmdproposalstatusset.go | 29 +- .../{sendfaucettx.go => cmdsendfaucettx.go} | 16 +- .../cmd/pictl/cmdtestproposalcapacity.go | 465 ++++++++++++++++++ .../cmd/pictl/{testrun.go => cmdtestrun.go} | 10 +- politeiawww/cmd/pictl/pictl.go | 5 +- politeiawww/cmd/pictl/print.go | 7 + politeiawww/cmd/pictl/proposal.go | 103 ++++ politeiawww/cmd/pictl/usernew.go | 4 +- politeiawww/config/config.go | 8 +- 11 files changed, 794 insertions(+), 195 deletions(-) rename politeiawww/cmd/pictl/{sendfaucettx.go => cmdsendfaucettx.go} (79%) create mode 100644 politeiawww/cmd/pictl/cmdtestproposalcapacity.go rename politeiawww/cmd/pictl/{testrun.go => cmdtestrun.go} (99%) diff --git a/politeiawww/cmd/pictl/cmdproposaledit.go b/politeiawww/cmd/pictl/cmdproposaledit.go index 54f0dfa4b..4a837923c 100644 --- a/politeiawww/cmd/pictl/cmdproposaledit.go +++ b/politeiawww/cmd/pictl/cmdproposaledit.go @@ -9,12 +9,9 @@ import ( "encoding/hex" "encoding/json" "fmt" - "io/ioutil" - "path/filepath" "time" "github.com/decred/politeia/politeiad/api/v1/mime" - piplugin "github.com/decred/politeia/politeiad/plugins/pi" piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" pclient "github.com/decred/politeia/politeiawww/client" @@ -42,47 +39,61 @@ type cmdProposalEdit struct { // Metadata fields that can be set by the user Name string `long:"name" optional:"true"` LinkTo string `long:"linkto" optional:"true"` - LinkBy int64 `long:"linkby" optional:"true"` - - // Random generates random proposal data. An IndexFile and - // Attachments are not required when using this flag. - Random bool `long:"random" optional:"true"` + LinkBy string `long:"linkby" optional:"true"` // RFP is a flag that is intended to make submitting an RFP easier // by calculating and inserting a linkby timestamp automatically // instead of having to pass in a timestamp using the --linkby // flag. RFP bool `long:"rfp" optional:"true"` + + // Random generates a random index file. The IndexFile argument is + // not allowed when using this flag. + Random bool `long:"random" optional:"true"` + + // RandomImages generates random image attachments. The Attachments + // argument is not allowed when using this flag. + RandomImages bool `long:"randomimages" optional:"true"` } // Execute executes the cmdProposalEdit command. // // This function satisfies the go-flags Commander interface. func (c *cmdProposalEdit) Execute(args []string) error { + _, err := proposalEdit(c) + if err != nil { + return err + } + return nil +} + +// proposalEdit edits a proposal. This function has been pulled out of the +// Execute method so that is can be used in the test commands. +func proposalEdit(c *cmdProposalEdit) (*rcv1.Record, error) { // Unpack args token := c.Args.Token indexFile := c.Args.IndexFile attachments := c.Args.Attachments - // Verify args + // Verify args and flags switch { case !c.Random && indexFile == "": - return fmt.Errorf("index file not found; you must either " + + return nil, fmt.Errorf("index file not found; you must either " + "provide an index.md file or use --random") - case c.Random && indexFile != "": - return fmt.Errorf("you cannot provide file arguments and use " + - "the --random flag at the same time") + case c.RandomImages && len(attachments) == 0: + return nil, fmt.Errorf("you cannot provide file arguments and use " + + "the --randomimages flag at the same time") - case c.RFP && c.LinkBy != 0: - return fmt.Errorf("you cannot use both the --rfp and --linkby " + + case c.RFP && c.LinkBy != "": + return nil, fmt.Errorf("you cannot use both the --rfp and --linkby " + "flags at the same time") } // Check for user identity. A user identity is required to sign // the proposal files. if cfg.Identity == nil { - return shared.ErrUserIdentityNotFound + return nil, shared.ErrUserIdentityNotFound } // Setup client @@ -95,13 +106,13 @@ func (c *cmdProposalEdit) Execute(args []string) error { } pc, err := pclient.New(cfg.Host, opts) if err != nil { - return err + return nil, err } // Get the pi policy. It contains the proposal requirements. pr, err := pc.PiPolicy() if err != nil { - return err + return nil, err } // Setup state @@ -113,45 +124,28 @@ func (c *cmdProposalEdit) Execute(args []string) error { state = rcv1.RecordStateVetted } - // Setup index file - files := make([]rcv1.File, 0, 16) - if c.Random { - // Generate random text for the index file - f, err := indexFileRandom(1024) + // Setup proposal files + var files []rcv1.File + switch { + case c.Random && c.RandomImages: + // Create a random index file and random attachments + files, err = proposalFilesRandom(int(pr.TextFileSizeMax), + int(pr.ImageFileCountMax)) if err != nil { - return err + return nil, err } - files = append(files, *f) - } else { - // Read index file from disk - fp := util.CleanAndExpandPath(indexFile) - var err error - payload, err := ioutil.ReadFile(fp) + case c.Random: + // Create a random index file + files, err = proposalFilesRandom(int(pr.TextFileSizeMax), 0) if err != nil { - return fmt.Errorf("ReadFile %v: %v", fp, err) + return nil, err } - files = append(files, rcv1.File{ - Name: piplugin.FileNameIndexFile, - MIME: mime.DetectMimeType(payload), - Digest: hex.EncodeToString(util.Digest(payload)), - Payload: base64.StdEncoding.EncodeToString(payload), - }) - } - - // Setup attachment files - for _, fn := range attachments { - fp := util.CleanAndExpandPath(fn) - payload, err := ioutil.ReadFile(fp) + default: + // Read files from disk + files, err = proposalFilesFromDisk(indexFile, attachments) if err != nil { - return fmt.Errorf("ReadFile %v: %v", fp, err) + return nil, err } - - files = append(files, rcv1.File{ - Name: filepath.Base(fn), - MIME: mime.DetectMimeType(payload), - Digest: hex.EncodeToString(util.Digest(payload)), - Payload: base64.StdEncoding.EncodeToString(payload), - }) } // Get current proposal if we are using the existing metadata @@ -163,7 +157,7 @@ func (c *cmdProposalEdit) Execute(args []string) error { } curr, err = pc.RecordDetails(d) if err != nil { - return err + return nil, err } } @@ -173,14 +167,14 @@ func (c *cmdProposalEdit) Execute(args []string) error { // Use the existing proposal name pm, err := proposalMetadataDecode(curr.Files) if err != nil { - return err + return nil, err } c.Name = pm.Name case c.Random && c.Name == "": // Create a random proposal name r, err := util.Random(int(pr.NameLengthMin)) if err != nil { - return err + return nil, err } c.Name = hex.EncodeToString(r) } @@ -189,7 +183,7 @@ func (c *cmdProposalEdit) Execute(args []string) error { } pmb, err := json.Marshal(pm) if err != nil { - return err + return nil, err } files = append(files, rcv1.File{ Name: piv1.FileNameProposalMetadata, @@ -199,26 +193,35 @@ func (c *cmdProposalEdit) Execute(args []string) error { }) // Setup vote metadata - if c.UseMD { + var linkBy int64 + switch { + case c.UseMD: + // Use existing vote metadata values vm, err := voteMetadataDecode(curr.Files) if err != nil { - return err + return nil, err } - c.LinkBy = vm.LinkBy + linkBy = vm.LinkBy c.LinkTo = vm.LinkTo - } - if c.RFP { + case c.RFP: // Set linkby to a month from now - c.LinkBy = time.Now().Add(time.Hour * 24 * 30).Unix() + linkBy = time.Now().Add(time.Hour * 24 * 30).Unix() + case c.LinkBy != "": + // Parse the provided linkby + d, err := time.ParseDuration(c.LinkBy) + if err != nil { + return nil, fmt.Errorf("unable to parse linkby: %v", err) + } + linkBy = time.Now().Add(d).Unix() } - if c.LinkBy != 0 || c.LinkTo != "" { + if linkBy != 0 || c.LinkTo != "" { vm := piv1.VoteMetadata{ LinkTo: c.LinkTo, - LinkBy: c.LinkBy, + LinkBy: linkBy, } vmb, err := json.Marshal(vm) if err != nil { - return err + return nil, err } files = append(files, rcv1.File{ Name: piv1.FileNameVoteMetadata, @@ -228,10 +231,10 @@ func (c *cmdProposalEdit) Execute(args []string) error { }) } - // Setup request + // Edit record sig, err := signedMerkleRoot(files, cfg.Identity) if err != nil { - return err + return nil, err } e := rcv1.Edit{ State: state, @@ -240,31 +243,29 @@ func (c *cmdProposalEdit) Execute(args []string) error { PublicKey: cfg.Identity.Public.String(), Signature: sig, } - - // Send request er, err := pc.RecordEdit(e) if err != nil { - return err + return nil, err } // Verify record vr, err := client.Version() if err != nil { - return err + return nil, err } err = pclient.RecordVerify(er.Record, vr.PubKey) if err != nil { - return fmt.Errorf("unable to verify record: %v", err) + return nil, fmt.Errorf("unable to verify record: %v", err) } // Print proposal to stdout printf("Proposal editted\n") err = printProposal(er.Record) if err != nil { - return err + return nil, err } - return nil + return &er.Record, nil } // proposalEditHelpMsg is the printed to stdout by the help command. @@ -281,22 +282,34 @@ A proposal can be submitted as an RFP submission by using the --linkto flag to link to and an existing RFP proposal. Arguments: -1. token (string, required) Proposal censorship token -2. indexfile (string, required) Index file -3. attachments (string, optional) Attachment files +1. token (string, required) Proposal censorship token. +2. indexfile (string, optional) Index file. +3. attachments (string, optional) Attachment files. Flags: - --unvetted (bool, optional) Edit an unvetted record. - --usemd (bool, optional) Use the existing proposal metadata. - --name (string, optional) Name of the proposal. - --linkto (string, optional) Token of an existing public proposal to link to. - --linkby (int64, optional) UNIX timestamp of the RFP deadline. Setting this - field will make the proposal an RFP with a - submission deadline specified by the linkby. - --random (bool, optional) Generate a random proposal. If this flag is used - then the markdownfile argument is no longer - required and any provided files will be ignored. - --rfp (bool, optional) Make the proposal an RFP by setting the linkby - to one month from the current time. This is - intended to be used in place of --linkby. + --unvetted (bool) Edit an unvetted record. + + --usemd (bool) Use the existing proposal metadata. + + --name (string) Name of the proposal. + + --linkto (string) Token of an existing public proposal to link to. + + --linkby (string) Make the proposal and RFP by setting the linkby + deadline. Other proposals must be entered as RFP + submissions by this linkby deadline. The provided + string should be a duration that will be added onto + the current time. Valid duration units are: + s (seconds), m (minutes), h (hours). + + --rfp (bool) Make the proposal an RFP by setting the linkby to one + month from the current time. This is intended to be + used in place of --linkby. + + --random (bool) Generate random proposal data, not including + attachments. The indexFile argument is not allowed + when using this flag. + + --randomimages (bool) Generate random attachments. The attachments argument + is not allowed when using this flag. ` diff --git a/politeiawww/cmd/pictl/cmdproposalnew.go b/politeiawww/cmd/pictl/cmdproposalnew.go index 342de6c70..695bbb6f1 100644 --- a/politeiawww/cmd/pictl/cmdproposalnew.go +++ b/politeiawww/cmd/pictl/cmdproposalnew.go @@ -9,12 +9,9 @@ import ( "encoding/hex" "encoding/json" "fmt" - "io/ioutil" - "path/filepath" "time" "github.com/decred/politeia/politeiad/api/v1/mime" - piplugin "github.com/decred/politeia/politeiad/plugins/pi" piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" pclient "github.com/decred/politeia/politeiawww/client" @@ -34,21 +31,35 @@ type cmdProposalNew struct { LinkTo string `long:"linkto" optional:"true"` LinkBy string `long:"linkby" optional:"true"` - // Random generates random proposal data. An IndexFile and - // Attachments are not required when using this flag. - Random bool `long:"random" optional:"true"` - // RFP is a flag that is intended to make submitting an RFP easier // by calculating and inserting a linkby timestamp automatically // instead of having to pass in a timestamp using the --linkby // flag. RFP bool `long:"rfp" optional:"true"` + + // Random generate random proposal data. The IndexFile argument is + // not allowed when using this flag. + Random bool `long:"random" optional:"true"` + + // RandomImages generates random image attachments. The Attachments + // argument is not allowed when using this flag. + RandomImages bool `long:"randomimages" optional:"true"` } // Execute executes the cmdProposalNew command. // // This function satisfies the go-flags Commander interface. func (c *cmdProposalNew) Execute(args []string) error { + _, err := proposalNew(c) + if err != nil { + return err + } + return nil +} + +// proposalNew creates a new proposal. This function has been pulled out of the +// Execute method so that it can be used in the test commands. +func proposalNew(c *cmdProposalNew) (*rcv1.Record, error) { // Unpack args indexFile := c.Args.IndexFile attachments := c.Args.Attachments @@ -56,22 +67,22 @@ func (c *cmdProposalNew) Execute(args []string) error { // Verify args and flags switch { case !c.Random && indexFile == "": - return fmt.Errorf("index file not found; you must either " + + return nil, fmt.Errorf("index file not found; you must either " + "provide an index.md file or use --random") - case c.Random && indexFile != "": - return fmt.Errorf("you cannot provide file arguments and use " + - "the --random flag at the same time") + case c.RandomImages && len(attachments) == 0: + return nil, fmt.Errorf("you cannot provide file arguments and use " + + "the --randomimages flag at the same time") case c.RFP && c.LinkBy != "": - return fmt.Errorf("you cannot use both the --rfp and --linkby " + + return nil, fmt.Errorf("you cannot use both the --rfp and --linkby " + "flags at the same time") } // Check for user identity. A user identity is required to sign // the proposal files. if cfg.Identity == nil { - return shared.ErrUserIdentityNotFound + return nil, shared.ErrUserIdentityNotFound } // Setup client @@ -84,61 +95,44 @@ func (c *cmdProposalNew) Execute(args []string) error { } pc, err := pclient.New(cfg.Host, opts) if err != nil { - return err + return nil, err } // Get the pi policy. It contains the proposal requirements. pr, err := pc.PiPolicy() if err != nil { - return err + return nil, err } - // Setup index file - files := make([]rcv1.File, 0, 16) - if c.Random { - // Generate random text for the index file - f, err := indexFileRandom(1024) + // Setup proposal files + var files []rcv1.File + switch { + case c.Random && c.RandomImages: + // Create a random index file and random attachments + files, err = proposalFilesRandom(int(pr.TextFileSizeMax), + int(pr.ImageFileCountMax)) if err != nil { - return err + return nil, err } - files = append(files, *f) - } else { - // Read index file from disk - fp := util.CleanAndExpandPath(indexFile) - var err error - payload, err := ioutil.ReadFile(fp) + case c.Random: + // Create a random index file + files, err = proposalFilesRandom(int(pr.TextFileSizeMax), 0) if err != nil { - return fmt.Errorf("ReadFile %v: %v", fp, err) + return nil, err } - files = append(files, rcv1.File{ - Name: piplugin.FileNameIndexFile, - MIME: mime.DetectMimeType(payload), - Digest: hex.EncodeToString(util.Digest(payload)), - Payload: base64.StdEncoding.EncodeToString(payload), - }) - } - - // Setup attachment files - for _, fn := range attachments { - fp := util.CleanAndExpandPath(fn) - payload, err := ioutil.ReadFile(fp) + default: + // Read files from disk + files, err = proposalFilesFromDisk(indexFile, attachments) if err != nil { - return fmt.Errorf("ReadFile %v: %v", fp, err) + return nil, err } - - files = append(files, rcv1.File{ - Name: filepath.Base(fn), - MIME: mime.DetectMimeType(payload), - Digest: hex.EncodeToString(util.Digest(payload)), - Payload: base64.StdEncoding.EncodeToString(payload), - }) } // Setup proposal metadata if c.Random && c.Name == "" { r, err := util.Random(int(pr.NameLengthMin)) if err != nil { - return err + return nil, err } c.Name = fmt.Sprintf("A Proposal Name %x", r) } @@ -147,7 +141,7 @@ func (c *cmdProposalNew) Execute(args []string) error { } pmb, err := json.Marshal(pm) if err != nil { - return err + return nil, err } files = append(files, rcv1.File{ Name: piv1.FileNameProposalMetadata, @@ -166,7 +160,7 @@ func (c *cmdProposalNew) Execute(args []string) error { // Parse the provided linkby d, err := time.ParseDuration(c.LinkBy) if err != nil { - return fmt.Errorf("unable to parse linkby: %v", err) + return nil, fmt.Errorf("unable to parse linkby: %v", err) } linkBy = time.Now().Add(d).Unix() } @@ -177,7 +171,7 @@ func (c *cmdProposalNew) Execute(args []string) error { } vmb, err := json.Marshal(vm) if err != nil { - return err + return nil, err } files = append(files, rcv1.File{ Name: piv1.FileNameVoteMetadata, @@ -191,34 +185,32 @@ func (c *cmdProposalNew) Execute(args []string) error { printf("Files\n") err = printProposalFiles(files) if err != nil { - return err + return nil, err } - // Setup request + // Submit proposal sig, err := signedMerkleRoot(files, cfg.Identity) if err != nil { - return err + return nil, err } n := rcv1.New{ Files: files, PublicKey: cfg.Identity.Public.String(), Signature: sig, } - - // Send request nr, err := pc.RecordNew(n) if err != nil { - return err + return nil, err } // Verify record vr, err := client.Version() if err != nil { - return err + return nil, err } err = pclient.RecordVerify(nr.Record, vr.PubKey) if err != nil { - return fmt.Errorf("unable to verify record: %v", err) + return nil, fmt.Errorf("unable to verify record: %v", err) } // Print censorship record @@ -226,7 +218,7 @@ func (c *cmdProposalNew) Execute(args []string) error { printf("Merkle : %v\n", nr.Record.CensorshipRecord.Merkle) printf("Receipt: %v\n", nr.Record.CensorshipRecord.Signature) - return nil + return &nr.Record, nil } // proposalNewHelpMsg is the printed to stdout by the help command. @@ -242,22 +234,31 @@ A proposal can be submitted as an RFP submission by using the --linkto flag to link to and an existing RFP proposal. Arguments: -1. indexfile (string, required) Index file -2. attachments (string) Attachment files +1. indexfile (string, optional) Index file. +2. attachments (string, optional) Attachment files. Flags: - --name (string) Name of the proposal. - --linkto (string) Token of an existing public proposal to link to. - --linkby (string) UNIX timestamp of the RFP deadline. Setting this field will - make the proposal an RFP with a submission deadline set by - the linkby. Valid linkby units are: - s (seconds), m (minutes), h (hours) - --random (bool) Generate a random proposal. If this flag is used then the - markdownfile argument is no longer required and any - provided files will be ignored. - --rfp (bool) Make the proposal an RFP by setting the linkby to one month - from the current time. This is intended to be used in place - of --linkby. + --name (string) Name of the proposal. + + --linkto (string) Token of an existing public proposal to link to. + + --linkby (string) Make the proposal and RFP by setting the linkby + deadline. Other proposals must be entered as RFP + submissions by this linkby deadline. The provided + string should be a duration that will be added onto + the current time. Valid duration units are: + s (seconds), m (minutes), h (hours). + + --rfp (bool) Make the proposal an RFP by setting the linkby to one + month from the current time. This is intended to be + used in place of --linkby. + + --random (bool) Generate random proposal data, not including + attachments. The indexFile argument is not allowed + when using this flag. + + --randomimages (bool) Generate random attachments. The attachments argument + is not allowed when using this flag. Examples: diff --git a/politeiawww/cmd/pictl/cmdproposalstatusset.go b/politeiawww/cmd/pictl/cmdproposalstatusset.go index e05c5ff9b..41f6d2d25 100644 --- a/politeiawww/cmd/pictl/cmdproposalstatusset.go +++ b/politeiawww/cmd/pictl/cmdproposalstatusset.go @@ -33,10 +33,21 @@ type cmdProposalSetStatus struct { // // This function satisfies the go-flags Commander interface. func (c *cmdProposalSetStatus) Execute(args []string) error { + _, err := proposalSetStatus(c) + if err != nil { + return err + } + return nil +} + +// proposalSetStatus sets the status of a proposal. This function has been +// pulled out of the Execute method so that is can be used in the test +// commands. +func proposalSetStatus(c *cmdProposalSetStatus) (*rcv1.Record, error) { // Verify user identity. This will be needed to sign the status // change. if cfg.Identity == nil { - return shared.ErrUserIdentityNotFound + return nil, shared.ErrUserIdentityNotFound } // Setup client @@ -49,7 +60,7 @@ func (c *cmdProposalSetStatus) Execute(args []string) error { } pc, err := pclient.New(cfg.Host, opts) if err != nil { - return err + return nil, err } // Setup state @@ -65,7 +76,7 @@ func (c *cmdProposalSetStatus) Execute(args []string) error { // human readable equivalent. status, err := parseRecordStatus(c.Args.Status) if err != nil { - return err + return nil, err } // Setup version @@ -80,7 +91,7 @@ func (c *cmdProposalSetStatus) Execute(args []string) error { } r, err := pc.RecordDetails(d) if err != nil { - return err + return nil, err } version = r.Version } @@ -101,26 +112,26 @@ func (c *cmdProposalSetStatus) Execute(args []string) error { // Send request ssr, err := pc.RecordSetStatus(ss) if err != nil { - return err + return nil, err } // Verify record vr, err := client.Version() if err != nil { - return err + return nil, err } err = pclient.RecordVerify(ssr.Record, vr.PubKey) if err != nil { - return fmt.Errorf("unable to verify record: %v", err) + return nil, fmt.Errorf("unable to verify record: %v", err) } // Print proposal to stdout err = printProposal(ssr.Record) if err != nil { - return err + return nil, err } - return nil + return &ssr.Record, nil } func parseRecordStatus(status string) (rcv1.RecordStatusT, error) { diff --git a/politeiawww/cmd/pictl/sendfaucettx.go b/politeiawww/cmd/pictl/cmdsendfaucettx.go similarity index 79% rename from politeiawww/cmd/pictl/sendfaucettx.go rename to politeiawww/cmd/pictl/cmdsendfaucettx.go index 27e3b8f0d..c22df0c30 100644 --- a/politeiawww/cmd/pictl/sendfaucettx.go +++ b/politeiawww/cmd/pictl/cmdsendfaucettx.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2020 The Decred developers +// Copyright (c) 2017-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -11,9 +11,9 @@ import ( "github.com/decred/politeia/util" ) -// sendFaucetTxCmd uses the Decred testnet faucet to send the specified amount +// cmdSendFaucetTx uses the Decred testnet faucet to send the specified amount // of DCR (in atoms) to the specified address. -type sendFaucetTxCmd struct { +type cmdSendFaucetTx struct { Args struct { Address string `positional-arg-name:"address" required:"true"` Amount uint64 `positional-arg-name:"amount" required:"true"` @@ -21,16 +21,16 @@ type sendFaucetTxCmd struct { } `positional-args:"true"` } -// Execute executes the sendFaucetTxCmd command. +// Execute executes the cmdSendFaucetTx command. // // This function satisfies the go-flags Commander interface. -func (cmd *sendFaucetTxCmd) Execute(args []string) error { - address := cmd.Args.Address - atoms := cmd.Args.Amount +func (c *cmdSendFaucetTx) Execute(args []string) error { + address := c.Args.Address + atoms := c.Args.Amount dcr := float64(atoms) / 1e8 txID, err := util.PayWithTestnetFaucet(context.Background(), - cfg.FaucetHost, address, atoms, cmd.Args.OverrideToken) + cfg.FaucetHost, address, atoms, c.Args.OverrideToken) if err != nil { return err } diff --git a/politeiawww/cmd/pictl/cmdtestproposalcapacity.go b/politeiawww/cmd/pictl/cmdtestproposalcapacity.go new file mode 100644 index 000000000..b9f36f857 --- /dev/null +++ b/politeiawww/cmd/pictl/cmdtestproposalcapacity.go @@ -0,0 +1,465 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "encoding/hex" + "fmt" + "math/rand" + "strconv" + + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + "github.com/decred/politeia/politeiawww/cmd/shared" + "github.com/decred/politeia/util" +) + +type cmdTestProposalCapacity struct { + Args struct { + AdminEmail string `positional-arg-name:"adminemail" required:"true"` + AdminPassword string `positional-arg-name:"adminpassword" required:"true"` + } `positional-args:"true"` +} + +// TODO this is temp +const randomImages = false + +// Execute executes the cmdTestProposalCapacity command. +// +// This function satisfies the go-flags Commander interface. +func (c *cmdTestProposalCapacity) Execute(args []string) error { + const ( + // Test run params + userCount = 1 + proposalCount = 5 + commentsPerProposal = 100 + commentVotesPerProposal = 1000 + ) + + // We don't want the output of individual commands printed. + cfg.Verbose = false + cfg.RawJSON = false + cfg.Silent = true + + // Verify the the provided login credentials are for an admin. + admin := user{ + Email: c.Args.AdminEmail, + Password: c.Args.AdminPassword, + } + err := userLogin(admin) + if err != nil { + return fmt.Errorf("failed to login admin: %v", err) + } + lr, err := client.Me() + if err != nil { + return err + } + if !lr.IsAdmin { + return fmt.Errorf("provided user is not an admin") + } + admin.Username = lr.Username + + // Verify that the paywall is disabled. + policyWWW, err := client.Policy() + if err != nil { + return err + } + if policyWWW.PaywallEnabled { + return fmt.Errorf("paywall is not disabled") + } + + // Setup users + users := make([]user, 0, userCount) + for i := 0; i < userCount; i++ { + log := fmt.Sprintf("Creating user %v/%v", i+1, userCount) + printInPlace(log) + + u, err := userNewRandom(policyWWW.MaxUsernameLength) + if err != nil { + return err + } + + users = append(users, *u) + } + fmt.Printf("\n") + + // Setup proposals + var ( + statusUnreviewed = "unreviewed" + statusUnvettedCensored = "unvetted-censored" + statusPublic = "public" + statusVettedCensored = "vetted-cesored" + statusAbandoned = "abandoned" + + statuses = []string{ + statusUnreviewed, + statusUnvettedCensored, + statusPublic, + statusVettedCensored, + statusAbandoned, + } + + // These are used to track the number of proposals that are + // created for each status. + countUnreviewed int + countUnvettedCensored int + countPublic int + countVettedCensored int + countAbandoned int + ) + for i := 0; i < proposalCount; i++ { + // Select a random user + r := rand.Intn(len(users)) + u := users[r] + + // Select a random status. This will be the ending status of + // the proposal. + r = rand.Intn(len(statuses)) + s := statuses[r] + + log := fmt.Sprintf("Submitting proposal %v/%v: %-17v", + i+1, proposalCount, s) + printInPlace(log) + + // Create proposal + switch s { + case statusUnreviewed: + _, err = proposalUnreviewed(u) + if err != nil { + return err + } + countUnreviewed++ + case statusUnvettedCensored: + _, err = proposalUnvettedCensored(u, admin) + if err != nil { + return err + } + countUnvettedCensored++ + case statusPublic: + _, err = proposalPublic(u, admin) + if err != nil { + return err + } + countPublic++ + case statusVettedCensored: + _, err = proposalVettedCensored(u, admin) + if err != nil { + return err + } + countVettedCensored++ + case statusAbandoned: + _, err = proposalAbandoned(u, admin) + if err != nil { + return err + } + countAbandoned++ + default: + return fmt.Errorf("invalid status %v", s) + } + } + fmt.Printf("\n") + + // Setup comments + // Setup comment votes + + _ = commentsPerProposal + _ = commentVotesPerProposal + + return nil +} + +type user struct { + Email string + Password string + Username string +} + +// userNew creates a new user. +// +// This function returns with the user logged out. +func userNew(email, username, password string) (*user, error) { + // Create user + c := userNewCmd{ + Verify: true, + } + c.Args.Email = email + c.Args.Username = username + c.Args.Password = password + err := c.Execute(nil) + if err != nil { + return nil, fmt.Errorf("userNewCmd: %v", err) + } + + // Log out user + err = userLogout() + if err != nil { + return nil, err + } + + return &user{ + Email: email, + Username: username, + Password: password, + }, nil +} + +// userNewRandom creates a new user with random credentials. +// +// This function returns with the user logged out. +func userNewRandom(usernameLength uint) (*user, error) { + // Hex encoding creates 2x the number of characters as bytes. + // Ex: 4 bytes will results in a 8 character hex string. + b, err := util.Random(int(usernameLength / uint(2))) + if err != nil { + return nil, err + } + var ( + r = hex.EncodeToString(b) + email = r + "@example.com" + username = r + password = r + ) + return userNew(email, username, password) +} + +// userLogin logs in the provided user. +func userLogin(u user) error { + c := shared.LoginCmd{} + c.Args.Email = u.Email + c.Args.Password = u.Password + err := c.Execute(nil) + if err != nil { + return fmt.Errorf("LoginCmd: %v", err) + } + return nil +} + +// userLogout logs out any logged in user. +func userLogout() error { + c := shared.LogoutCmd{} + err := c.Execute(nil) + if err != nil { + return fmt.Errorf("LogoutCmd: %v", err) + } + return nil +} + +// proposalUnreviewed creates a new proposal and leaves its status as +// unreviewed. +// +// This function returns with the user logged out. +func proposalUnreviewed(u user) (*rcv1.Record, error) { + // Login user + err := userLogin(u) + if err != nil { + return nil, err + } + + // Submit new proposal + cn := cmdProposalNew{ + Random: true, + RandomImages: randomImages, + } + r, err := proposalNew(&cn) + if err != nil { + return nil, fmt.Errorf("cmdProposalNew: %v", err) + } + + // Edit the proposal + ce := cmdProposalEdit{ + Unvetted: true, + Random: true, + RandomImages: randomImages, + } + ce.Args.Token = r.CensorshipRecord.Token + r, err = proposalEdit(&ce) + if err != nil { + return nil, fmt.Errorf("cmdProposalEdit: %v", err) + } + + // Logout user + err = userLogout() + if err != nil { + return nil, err + } + + return r, nil +} + +// proposalUnvettedCensored creates a new proposal then censors the proposal. +// +// This function returns with all users logged out. +func proposalUnvettedCensored(u user, admin user) (*rcv1.Record, error) { + // Setup an unvetted proposal + r, err := proposalUnreviewed(u) + if err != nil { + return nil, err + } + + // Login admin + err = userLogin(admin) + if err != nil { + return nil, err + } + + // Censor the proposal + cs := cmdProposalSetStatus{ + Unvetted: true, + } + cs.Args.Token = r.CensorshipRecord.Token + cs.Args.Status = strconv.Itoa(int(rcv1.RecordStatusCensored)) + cs.Args.Reason = "Violates proposal rules." + cs.Args.Version = r.Version + r, err = proposalSetStatus(&cs) + if err != nil { + return nil, fmt.Errorf("cmdProposalSetStatus: %v", err) + } + + // Logout admin + err = userLogout() + if err != nil { + return nil, err + } + + return r, nil +} + +// proposalPublic creates and new proposal then makes the proposal public. +// +// This function returns with all users logged out. +func proposalPublic(u user, admin user) (*rcv1.Record, error) { + // Setup an unvetted proposal + r, err := proposalUnreviewed(u) + if err != nil { + return nil, err + } + + // Login admin + err = userLogin(admin) + if err != nil { + return nil, err + } + + // Make the proposal public + cs := cmdProposalSetStatus{ + Unvetted: true, + } + cs.Args.Token = r.CensorshipRecord.Token + cs.Args.Status = strconv.Itoa(int(rcv1.RecordStatusPublic)) + cs.Args.Version = r.Version + r, err = proposalSetStatus(&cs) + if err != nil { + return nil, fmt.Errorf("cmdProposalSetStatus: %v", err) + } + + // Logout admin + err = userLogout() + if err != nil { + return nil, err + } + + // Login user + err = userLogin(u) + if err != nil { + return nil, err + } + + // Edit the proposal + ce := cmdProposalEdit{ + Unvetted: false, + Random: true, + RandomImages: randomImages, + } + ce.Args.Token = r.CensorshipRecord.Token + r, err = proposalEdit(&ce) + if err != nil { + return nil, fmt.Errorf("cmdProposalEdit: %v", err) + } + + // Logout user + err = userLogout() + if err != nil { + return nil, err + } + + return r, nil +} + +// proposalVettedCensored creates a new proposal, makes the proposal public, +// then censors the proposal. +// +// This function returns with all users logged out. +func proposalVettedCensored(u user, admin user) (*rcv1.Record, error) { + // Create a public proposal + r, err := proposalPublic(u, admin) + if err != nil { + return nil, err + } + + // Login admin + err = userLogin(admin) + if err != nil { + return nil, err + } + + // Censor the proposal + cs := cmdProposalSetStatus{ + Unvetted: false, + } + cs.Args.Token = r.CensorshipRecord.Token + cs.Args.Status = strconv.Itoa(int(rcv1.RecordStatusCensored)) + cs.Args.Reason = "Violates proposal rules." + cs.Args.Version = r.Version + r, err = proposalSetStatus(&cs) + if err != nil { + return nil, fmt.Errorf("cmdProposalSetStatus: %v", err) + } + + // Logout admin + err = userLogout() + if err != nil { + return nil, err + } + + return r, nil +} + +// proposalAbandoned creates a new proposal, makes the proposal public, +// then abandones the proposal. +// +// This function returns with all users logged out. +func proposalAbandoned(u user, admin user) (*rcv1.Record, error) { + // Create a public proposal + r, err := proposalPublic(u, admin) + if err != nil { + return nil, err + } + + // Login admin + err = userLogin(admin) + if err != nil { + return nil, err + } + + // Abandone the proposal + cs := cmdProposalSetStatus{ + Unvetted: false, + } + cs.Args.Token = r.CensorshipRecord.Token + cs.Args.Status = strconv.Itoa(int(rcv1.RecordStatusArchived)) + cs.Args.Reason = "No activity from author in 3 weeks." + cs.Args.Version = r.Version + r, err = proposalSetStatus(&cs) + if err != nil { + return nil, fmt.Errorf("cmdProposalSetStatus: %v", err) + } + + // Logout admin + err = userLogout() + if err != nil { + return nil, err + } + + return r, nil +} diff --git a/politeiawww/cmd/pictl/testrun.go b/politeiawww/cmd/pictl/cmdtestrun.go similarity index 99% rename from politeiawww/cmd/pictl/testrun.go rename to politeiawww/cmd/pictl/cmdtestrun.go index 67e37ad51..baceb792f 100644 --- a/politeiawww/cmd/pictl/testrun.go +++ b/politeiawww/cmd/pictl/cmdtestrun.go @@ -1,11 +1,11 @@ -// Copyright (c) 2017-2020 The Decred developers +// Copyright (c) 2017-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. package main -// testRunCmd performs a test run of all the politeiawww routes. -type testRunCmd struct { +// cmdTestRun performs a test run of all the politeiawww routes. +type cmdTestRun struct { Args struct { AdminEmail string `positional-arg-name:"adminemail"` AdminPassword string `positional-arg-name:"adminpassword"` @@ -1391,10 +1391,10 @@ func testCommentRoutes(admin testUser) error { return nil } -// Execute executes the testRunCmd command. +// Execute executes the cmdTestRun command. // // This function satisfies the go-flags Commander interface. -func (cmd *testRunCmd) Execute(args []string) error { +func (cmd *cmdTestRun) Execute(args []string) error { // Suppress output from cli commands cfg.Silent = true diff --git a/politeiawww/cmd/pictl/pictl.go b/politeiawww/cmd/pictl/pictl.go index a84f9fe02..d93a4cead 100644 --- a/politeiawww/cmd/pictl/pictl.go +++ b/politeiawww/cmd/pictl/pictl.go @@ -100,8 +100,9 @@ type pictl struct { Subscribe subscribeCmd `command:"subscribe"` // Dev commands - TestRun testRunCmd `command:"testrun"` - SendFaucetTx sendFaucetTxCmd `command:"sendfaucettx"` + SendFaucetTx cmdSendFaucetTx `command:"sendfaucettx"` + TestRun cmdTestRun `command:"testrun"` + TestProposalCapcity cmdTestProposalCapacity `command:"testproposalcapacity"` } const helpMsg = `Application Options: diff --git a/politeiawww/cmd/pictl/print.go b/politeiawww/cmd/pictl/print.go index eafe52d69..bc5f7cc67 100644 --- a/politeiawww/cmd/pictl/print.go +++ b/politeiawww/cmd/pictl/print.go @@ -45,3 +45,10 @@ func byteCountSI(b int64) string { return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "kMGTPE"[exp]) } + +// printInPlace prints the provided text to stdout in a way that overwrites the +// existing stdout text. This function can be called multiple times. Each +// subsequent call will overwrite the existing text that was printed to stdout. +func printInPlace(s string) { + fmt.Printf("\033[2K\r%s", s) +} diff --git a/politeiawww/cmd/pictl/proposal.go b/politeiawww/cmd/pictl/proposal.go index 0e559f152..7588cd4ff 100644 --- a/politeiawww/cmd/pictl/proposal.go +++ b/politeiawww/cmd/pictl/proposal.go @@ -10,11 +10,17 @@ import ( "encoding/hex" "encoding/json" "fmt" + "image" + "image/color" + "image/png" + "io/ioutil" "math/rand" + "path/filepath" "strings" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/api/v1/mime" + piplugin "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/usermd" piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" @@ -167,6 +173,103 @@ func indexFileRandom(sizeInBytes int) (*rcv1.File, error) { }, nil } +// pngFileRandom returns a record file for a randomly generated PNG image. The +// size of the image will be 0.49MB. +func pngFileRandom() (*rcv1.File, error) { + b := new(bytes.Buffer) + img := image.NewRGBA(image.Rect(0, 0, 500, 250)) + + // Fill in the pixels with random rgb colors + r := rand.New(rand.NewSource(255)) + for y := 0; y < img.Bounds().Max.Y-1; y++ { + for x := 0; x < img.Bounds().Max.X-1; x++ { + a := uint8(r.Float32() * 255) + rgb := uint8(r.Float32() * 255) + img.SetRGBA(x, y, color.RGBA{rgb, rgb, rgb, a}) + } + } + err := png.Encode(b, img) + if err != nil { + return nil, err + } + + // Create a random name + rn, err := util.Random(8) + if err != nil { + return nil, err + } + + return &rcv1.File{ + Name: hex.EncodeToString(rn) + ".png", + MIME: mime.DetectMimeType(b.Bytes()), + Digest: hex.EncodeToString(util.Digest(b.Bytes())), + Payload: base64.StdEncoding.EncodeToString(b.Bytes()), + }, nil +} + +func proposalFilesRandom(textFileSizeMax, imageFileCountMax int) ([]rcv1.File, error) { + files := make([]rcv1.File, 0, 16) + + // Generate random text for the index file. The size of the index + // file is also randomly chosen + size := rand.Intn(textFileSizeMax) + f, err := indexFileRandom(size) + if err != nil { + return nil, err + } + files = append(files, *f) + + // Generate a random number of attachment files + if imageFileCountMax > 0 { + attachmentCount := rand.Intn(imageFileCountMax) + for i := 0; i <= attachmentCount; i++ { + f, err := pngFileRandom() + if err != nil { + return nil, err + } + files = append(files, *f) + } + } + + return files, nil +} + +func proposalFilesFromDisk(indexFile string, attachments []string) ([]rcv1.File, error) { + files := make([]rcv1.File, 0, len(attachments)+1) + + // Setup index file + fp := util.CleanAndExpandPath(indexFile) + var err error + payload, err := ioutil.ReadFile(fp) + if err != nil { + return nil, fmt.Errorf("ReadFile %v: %v", fp, err) + } + files = append(files, rcv1.File{ + Name: piplugin.FileNameIndexFile, + MIME: mime.DetectMimeType(payload), + Digest: hex.EncodeToString(util.Digest(payload)), + Payload: base64.StdEncoding.EncodeToString(payload), + }) + + // Setup attachment files + for _, fn := range attachments { + fp := util.CleanAndExpandPath(fn) + payload, err := ioutil.ReadFile(fp) + if err != nil { + return nil, fmt.Errorf("ReadFile %v: %v", fp, err) + } + + files = append(files, rcv1.File{ + Name: filepath.Base(fn), + MIME: mime.DetectMimeType(payload), + Digest: hex.EncodeToString(util.Digest(payload)), + Payload: base64.StdEncoding.EncodeToString(payload), + }) + } + + return files, nil +} + // signedMerkleRoot returns the signed merkle root of the provided files. The // signature is created using the provided identity. func signedMerkleRoot(files []rcv1.File, fid *identity.FullIdentity) (string, error) { diff --git a/politeiawww/cmd/pictl/usernew.go b/politeiawww/cmd/pictl/usernew.go index 4b95ab7b4..14e5f5516 100644 --- a/politeiawww/cmd/pictl/usernew.go +++ b/politeiawww/cmd/pictl/usernew.go @@ -139,8 +139,8 @@ func (cmd *userNewCmd) Execute(args []string) error { } // Pays paywall fee using faucet - if cmd.Paywall { - faucet := sendFaucetTxCmd{} + if pr.PaywallEnabled && cmd.Paywall { + faucet := cmdSendFaucetTx{} faucet.Args.Address = lr.PaywallAddress faucet.Args.Amount = lr.PaywallAmount err = faucet.Execute(nil) diff --git a/politeiawww/config/config.go b/politeiawww/config/config.go index 08dc3227a..e431ff86d 100644 --- a/politeiawww/config/config.go +++ b/politeiawww/config/config.go @@ -87,7 +87,7 @@ type Config struct { SMTPSkipVerify bool `long:"smtpskipverify" description:"Skip SMTP TLS cert verification. Will only skip if SMTPCert is empty"` WebServerAddress string `long:"webserveraddress" description:"Address for the Politeia web server; it should have this format: ://[:]"` - // XXX These should be plugin settings + // XXX These should all be plugin settings DcrdataHost string `long:"dcrdatahost" description:"Dcrdata ip:port"` PaywallAmount uint64 `long:"paywallamount" description:"Amount of DCR (in atoms) required for a user to register or submit a proposal."` PaywallXpub string `long:"paywallxpub" description:"Extended public key for deriving paywall addresses."` @@ -98,10 +98,8 @@ type Config struct { CodeStatOrganization string `long:"codestatorg" description:"Organization to crawl for code statistics"` CodeStatStart int64 `long:"codestatstart" description:"Date in which to look back to for code stat crawl (default 6 months back)"` CodeStatEnd int64 `long:"codestatend" description:"Date in which to end look back to for code stat crawl (default today)"` - - // TODO these need to be removed - VoteDurationMin uint32 `long:"votedurationmin" description:"Minimum duration of a proposal vote in blocks"` - VoteDurationMax uint32 `long:"votedurationmax" description:"Maximum duration of a proposal vote in blocks"` + VoteDurationMin uint32 `long:"votedurationmin" description:"Minimum duration of a dcc vote in blocks"` + VoteDurationMax uint32 `long:"votedurationmax" description:"Maximum duration of a dcc vote in blocks"` Version string Identity *identity.PublicIdentity From 77f1ee6e50b8f916f71dcc54ab7ec7043b2ec279 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 18 Feb 2021 09:08:31 -0600 Subject: [PATCH 315/449] Fix anchor for leaf bug. --- politeiad/backend/tlogbe/tlog/anchor.go | 61 +++++++++++++++++-------- politeiad/backend/tlogbe/tlog/tlog.go | 4 +- politeiad/backend/tlogbe/tlogbe.go | 6 +++ 3 files changed, 50 insertions(+), 21 deletions(-) diff --git a/politeiad/backend/tlogbe/tlog/anchor.go b/politeiad/backend/tlogbe/tlog/anchor.go index 0493da41c..3f2a8c368 100644 --- a/politeiad/backend/tlogbe/tlog/anchor.go +++ b/politeiad/backend/tlogbe/tlog/anchor.go @@ -89,8 +89,11 @@ func (t *Tlog) anchorForLeaf(treeID int64, merkleLeafHash []byte, leaves []*tril return nil, fmt.Errorf("leaf not found") } - // Find the first anchor that occurs after the leaf - var anchorKey string + // Find the first two anchor that occurs after the leaf. If the + // leaf was added in the middle of an anchor drop then it will not + // be part of the first anchor. It will be part of the second + // anchor. + keys := make([]string, 0, 2) for i := int(l.LeafIndex); i < len(leaves); i++ { l := leaves[i] ed, err := extraDataDecode(l.ExtraData) @@ -98,34 +101,54 @@ func (t *Tlog) anchorForLeaf(treeID int64, merkleLeafHash []byte, leaves []*tril return nil, err } if ed.Desc == dataDescriptorAnchor { - anchorKey = ed.Key - break + keys = append(keys, ed.Key) + if len(keys) == 2 { + break + } } } - if anchorKey == "" { - // This record version has not been anchored yet + if len(keys) == 0 { + // This leaf has not been anchored yet return nil, errAnchorNotFound } - // Get the anchor record - blobs, err := t.store.Get([]string{anchorKey}) + // Get the anchor records + blobs, err := t.store.Get(keys) if err != nil { return nil, fmt.Errorf("store Get: %v", err) } - b, ok := blobs[anchorKey] - if !ok { - return nil, fmt.Errorf("blob not found %v", anchorKey) + if len(blobs) != len(keys) { + return nil, fmt.Errorf("unexpected blobs count: got %v, want %v", + len(blobs), len(keys)) } - be, err := store.Deblob(b) - if err != nil { - return nil, err + + // Find the correct anchor for the leaf + var leafAnchor *anchor + for _, v := range keys { + b, ok := blobs[v] + if !ok { + return nil, fmt.Errorf("blob not found %v", v) + } + be, err := store.Deblob(b) + if err != nil { + return nil, err + } + a, err := convertAnchorFromBlobEntry(*be) + if err != nil { + return nil, err + } + if uint64(l.LeafIndex) < a.LogRoot.TreeSize { + // The leaf is included in this anchor. We're done. + leafAnchor = a + break + } } - a, err := convertAnchorFromBlobEntry(*be) - if err != nil { - return nil, err + if leafAnchor == nil { + // This leaf has not been anchored yet + return nil, errAnchorNotFound } - return a, nil + return leafAnchor, nil } // anchorLatest returns the most recent anchor for the provided tree. A @@ -309,7 +332,7 @@ func (t *Tlog) anchorWait(anchors []anchor, digests []string) { // // Ex: politeiad submits a digest for treeA to dcrtime. politeiad // gets shutdown before an anchor record is added to treeA. - // dcrtime timestamps the treeA digest onto block 1000. politeiad + // dcrtime timestamps the treeA digest into block 1000. politeiad // gets turned back on and a new record, treeB, is submitted // prior to an anchor drop attempt. On the next anchor drop, // politeiad will try to drop an anchor for both treeA and treeB diff --git a/politeiad/backend/tlogbe/tlog/tlog.go b/politeiad/backend/tlogbe/tlog/tlog.go index e9546ff9a..50b018e60 100644 --- a/politeiad/backend/tlogbe/tlog/tlog.go +++ b/politeiad/backend/tlogbe/tlog/tlog.go @@ -1146,8 +1146,8 @@ func (t *Tlog) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trillian } // Extract the data blob. Its possible for the data blob to not - // exist if has been censored. This is ok. We'll still return the - // rest of the timestamp. + // exist if it has been censored. This is ok. We'll still return + // the rest of the timestamp. var data []byte if len(blobs) == 1 { b, ok := blobs[ed.Key] diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index b77dad087..c3670ebd3 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -1828,10 +1828,16 @@ func (t *tlogBackend) setup() error { log.Infof("Building backend token prefix cache") + // A record token is created using the unvetted tree ID so we + // only need to retrieve the unvetted trees in order to build the + // token prefix cache. treeIDs, err := t.unvetted.TreesAll() if err != nil { return fmt.Errorf("unvetted TreesAll: %v", err) } + + log.Infof("%v records in the backend", len(treeIDs)) + for _, v := range treeIDs { token := tokenFromTreeID(v) t.prefixAdd(token) From e6bfbb86ef511ad4d6d2ae0298230e9caea5b375 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 18 Feb 2021 15:42:15 -0600 Subject: [PATCH 316/449] pictl: Finish proposalloadtest cmd. --- politeiawww/cmd/pictl/cmdcommentcount.go | 21 +- politeiawww/cmd/pictl/cmdproposaledit.go | 11 +- politeiawww/cmd/pictl/cmdproposalinv.go | 18 +- politeiawww/cmd/pictl/cmdproposalloadtest.go | 815 ++++++++++++++++++ politeiawww/cmd/pictl/cmdproposalnew.go | 15 +- .../cmd/pictl/cmdtestproposalcapacity.go | 465 ---------- politeiawww/cmd/pictl/pictl.go | 7 +- politeiawww/cmd/pictl/proposal.go | 9 +- 8 files changed, 866 insertions(+), 495 deletions(-) create mode 100644 politeiawww/cmd/pictl/cmdproposalloadtest.go delete mode 100644 politeiawww/cmd/pictl/cmdtestproposalcapacity.go diff --git a/politeiawww/cmd/pictl/cmdcommentcount.go b/politeiawww/cmd/pictl/cmdcommentcount.go index a9bb3d449..b1c4ae5fe 100644 --- a/politeiawww/cmd/pictl/cmdcommentcount.go +++ b/politeiawww/cmd/pictl/cmdcommentcount.go @@ -5,8 +5,6 @@ package main import ( - "fmt" - cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" pclient "github.com/decred/politeia/politeiawww/client" ) @@ -27,6 +25,17 @@ type cmdCommentCount struct { // // This function satisfies the go-flags Commander interface. func (c *cmdCommentCount) Execute(args []string) error { + _, err := commentCount(c) + if err != nil { + return err + } + return nil +} + +// commentCount returns the number of comments that have been made on the +// provided records. This function has been pulled out of the Execute method so +// that it can be used in test commands. +func commentCount(c *cmdCommentCount) (map[string]uint32, error) { // Setup client opts := pclient.Opts{ HTTPSCert: cfg.HTTPSCert, @@ -37,7 +46,7 @@ func (c *cmdCommentCount) Execute(args []string) error { } pc, err := pclient.New(cfg.Host, opts) if err != nil { - return err + return nil, err } // Setup state @@ -56,15 +65,15 @@ func (c *cmdCommentCount) Execute(args []string) error { } cr, err := pc.CommentCount(cc) if err != nil { - return err + return nil, err } // Print counts for k, v := range cr.Counts { - fmt.Printf("%v %v\n", k, v) + printf("%v %v\n", k, v) } - return nil + return cr.Counts, nil } // commentCountHelpMsg is printed to stdout by the help command. diff --git a/politeiawww/cmd/pictl/cmdproposaledit.go b/politeiawww/cmd/pictl/cmdproposaledit.go index 4a837923c..a9a621e00 100644 --- a/politeiawww/cmd/pictl/cmdproposaledit.go +++ b/politeiawww/cmd/pictl/cmdproposaledit.go @@ -81,9 +81,9 @@ func proposalEdit(c *cmdProposalEdit) (*rcv1.Record, error) { return nil, fmt.Errorf("index file not found; you must either " + "provide an index.md file or use --random") - case c.RandomImages && len(attachments) == 0: - return nil, fmt.Errorf("you cannot provide file arguments and use " + - "the --randomimages flag at the same time") + case c.RandomImages && len(attachments) > 0: + return nil, fmt.Errorf("you cannot provide attachment files and " + + "use the --randomimages flag at the same time") case c.RFP && c.LinkBy != "": return nil, fmt.Errorf("you cannot use both the --rfp and --linkby " + @@ -125,18 +125,19 @@ func proposalEdit(c *cmdProposalEdit) (*rcv1.Record, error) { } // Setup proposal files + indexFileSize := 10000 // In bytes var files []rcv1.File switch { case c.Random && c.RandomImages: // Create a random index file and random attachments - files, err = proposalFilesRandom(int(pr.TextFileSizeMax), + files, err = proposalFilesRandom(indexFileSize, int(pr.ImageFileCountMax)) if err != nil { return nil, err } case c.Random: // Create a random index file - files, err = proposalFilesRandom(int(pr.TextFileSizeMax), 0) + files, err = proposalFilesRandom(indexFileSize, 0) if err != nil { return nil, err } diff --git a/politeiawww/cmd/pictl/cmdproposalinv.go b/politeiawww/cmd/pictl/cmdproposalinv.go index 080a2e1bc..9304ca28c 100644 --- a/politeiawww/cmd/pictl/cmdproposalinv.go +++ b/politeiawww/cmd/pictl/cmdproposalinv.go @@ -29,6 +29,16 @@ type cmdProposalInv struct { // // This function satisfies the go-flags Commander interface. func (c *cmdProposalInv) Execute(args []string) error { + _, err := proposalInv(c) + if err != nil { + return err + } + return nil +} + +// proposalInv retrieves the proposal inventory. This function has been pulled +// out of the Execute method so that it can be used in test commands. +func proposalInv(c *cmdProposalInv) (*rcv1.InventoryReply, error) { // Setup client opts := pclient.Opts{ HTTPSCert: cfg.HTTPSCert, @@ -39,7 +49,7 @@ func (c *cmdProposalInv) Execute(args []string) error { } pc, err := pclient.New(cfg.Host, opts) if err != nil { - return err + return nil, err } // Setup state @@ -58,7 +68,7 @@ func (c *cmdProposalInv) Execute(args []string) error { // human readable equivalent. status, err = parseRecordStatus(c.Args.Status) if err != nil { - return err + return nil, err } // If a status was given but no page number was give, default @@ -76,13 +86,13 @@ func (c *cmdProposalInv) Execute(args []string) error { } ir, err := pc.RecordInventory(i) if err != nil { - return err + return nil, err } // Print inventory printJSON(ir) - return nil + return ir, nil } // proposalInvHelpMsg is printed to stdout by the help command. diff --git a/politeiawww/cmd/pictl/cmdproposalloadtest.go b/politeiawww/cmd/pictl/cmdproposalloadtest.go new file mode 100644 index 000000000..9249a748b --- /dev/null +++ b/politeiawww/cmd/pictl/cmdproposalloadtest.go @@ -0,0 +1,815 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "encoding/hex" + "fmt" + "math/rand" + "strconv" + "time" + + cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + "github.com/decred/politeia/politeiawww/cmd/shared" + "github.com/decred/politeia/util" +) + +type cmdProposalLoadTest struct { + Args struct { + AdminEmail string `positional-arg-name:"adminemail" required:"true"` + AdminPassword string `positional-arg-name:"adminpassword" required:"true"` + Users int `positional-arg-name:"users"` + Proposals int `positional-arg-name:"proposals"` + CommentVotes int `positional-arg-name:"commentvotes"` + } `positional-args:"true"` + + // IncludeImages is used to include a random number of images when + // submitting proposals. + IncludeImages bool `long:"includeimages"` +} + +// Execute executes the cmdProposalLoadTest command. +// +// This function satisfies the go-flags Commander interface. +func (c *cmdProposalLoadTest) Execute(args []string) error { + fmt.Printf("Warn: this cmd should be run on a clean politeia instance\n") + + // Setup default test parameters + var ( + userCount = 10 + proposalCount = 25 + commentsPerProposal = 100 + commentSize = 32 // In characters + commentVotesPerProposal = 500 + + includeImages = c.IncludeImages + ) + if c.Args.Users != 0 { + userCount = c.Args.Users + } + if c.Args.Proposals != 0 { + proposalCount = c.Args.Proposals + } + if c.Args.CommentVotes != 0 { + commentVotesPerProposal = c.Args.CommentVotes + } + + // We don't want the output of individual commands printed. + cfg.Verbose = false + cfg.RawJSON = false + cfg.Silent = true + + // User count must be at least 2. A user cannot upvote their own + // comments so we need at least 1 user to make comments and a + // second user to upvote the comments. + if userCount < 2 { + return fmt.Errorf("user count must be >= 2") + } + + // Verify the the provided login credentials are for an admin. + admin := user{ + Email: c.Args.AdminEmail, + Password: c.Args.AdminPassword, + } + err := userLogin(admin) + if err != nil { + return fmt.Errorf("failed to login admin: %v", err) + } + lr, err := client.Me() + if err != nil { + return err + } + if !lr.IsAdmin { + return fmt.Errorf("provided user is not an admin") + } + admin.Username = lr.Username + + // Verify that the paywall is disabled. + policyWWW, err := client.Policy() + if err != nil { + return err + } + if policyWWW.PaywallEnabled { + return fmt.Errorf("paywall is not disabled") + } + + // Setup users + users := make([]user, 0, userCount) + for i := 0; i < userCount; i++ { + log := fmt.Sprintf("Creating user %v/%v", i+1, userCount) + printInPlace(log) + + u, err := userNewRandom() + if err != nil { + return err + } + + users = append(users, *u) + } + fmt.Printf("\n") + + // Log start time + fmt.Printf("Start time: %v\n", timestampFromUnix(time.Now().Unix())) + + // Setup proposals + var ( + statusUnreviewed = "unreviewed" + statusUnvettedCensored = "unvetted-censored" + statusPublic = "public" + statusVettedCensored = "vetted-cesored" + statusAbandoned = "abandoned" + + // statuses specifies the statuses that are rotated through when + // proposals are being submitted. We can increase the proption of + // proposals that are a specific status by increasing the number + // of times the status occurs in this array. + statuses = []string{ + statusPublic, + statusPublic, + statusPublic, + statusPublic, + statusUnreviewed, + statusUnvettedCensored, + statusVettedCensored, + statusAbandoned, + } + + // These are used to track the number of proposals that are + // created for each status. + countUnreviewed int + countUnvettedCensored int + countPublic int + countVettedCensored int + countAbandoned int + + // public is used to aggregate the tokens of public proposals. + // These will be used when we add comments to the proposals. + public = make([]string, 0, proposalCount) + ) + for i := 0; i < proposalCount; i++ { + // Select a random user + r := rand.Intn(len(users)) + u := users[r] + + // Rotate through the statuses + s := statuses[i%len(statuses)] + + log := fmt.Sprintf("Submitting proposal %v/%v: %v", + i+1, proposalCount, s) + printInPlace(log) + + // Create proposal + switch s { + case statusUnreviewed: + _, err = proposalUnreviewed(u, includeImages) + if err != nil { + return err + } + countUnreviewed++ + case statusUnvettedCensored: + _, err = proposalUnvettedCensored(u, admin, includeImages) + if err != nil { + return err + } + countUnvettedCensored++ + case statusPublic: + r, err := proposalPublic(u, admin, includeImages) + if err != nil { + return err + } + countPublic++ + public = append(public, r.CensorshipRecord.Token) + case statusVettedCensored: + _, err = proposalVettedCensored(u, admin, includeImages) + if err != nil { + return err + } + countVettedCensored++ + case statusAbandoned: + _, err = proposalAbandoned(u, admin, includeImages) + if err != nil { + return err + } + countAbandoned++ + default: + return fmt.Errorf("invalid status %v", s) + } + } + fmt.Printf("\n") + + // Verify proposal inventory + var ( + statusesUnvetted = map[rcv1.RecordStatusT]int{ + rcv1.RecordStatusUnreviewed: countUnreviewed, + rcv1.RecordStatusCensored: countUnvettedCensored, + } + + statusesVetted = map[rcv1.RecordStatusT]int{ + rcv1.RecordStatusPublic: countPublic, + rcv1.RecordStatusCensored: countVettedCensored, + rcv1.RecordStatusArchived: countAbandoned, + } + ) + for status, count := range statusesUnvetted { + // Tally up how many records are in the inventory for each + // status. + var tally int + var page uint32 = 1 + for { + log := fmt.Sprintf("Verifying unvetted inv for status %v, page %v", + rcv1.RecordStatuses[status], page) + printInPlace(log) + + tokens, err := invUnvetted(admin, status, page) + if err != nil { + return err + } + if len(tokens) == 0 { + // We've reached the end of the inventory + break + } + tally += len(tokens) + page++ + } + fmt.Printf("\n") + + // The count might be more than the tally if there were already + // proposals in the inventory prior to running this command. The + // tally should never be less than the count. + if tally < count { + return fmt.Errorf("unexpected number of proposals in inventory "+ + "for status %v: got %v, want >=%v", rcv1.RecordStatuses[status], + tally, count) + } + } + for status, count := range statusesVetted { + // Tally up how many records are in the inventory for each + // status. + var tally int + var page uint32 = 1 + for { + log := fmt.Sprintf("Verifying vetted inv for status %v, page %v", + rcv1.RecordStatuses[status], page) + printInPlace(log) + + tokens, err := inv(rcv1.RecordStateVetted, status, page) + if err != nil { + return err + } + if len(tokens) == 0 { + // We've reached the end of the inventory + break + } + tally += len(tokens) + page++ + } + fmt.Printf("\n") + + // The count might be more than the tally if there were already + // proposals in the inventory prior to running this command. The + // tally should never be less than the count. + if tally < count { + return fmt.Errorf("unexpected number of proposals in inventory "+ + "for status %v: got %v, want >=%v", rcv1.RecordStatuses[status], + tally, count) + } + } + + // Users cannot vote on their own comment. Divide the user into two + // groups. Group 1 will create the comments. Group 2 will vote on + // the comments. + users1 := users[:len(users)/2] + users2 := users[len(users)/2:] + + // Setup comments + for i, token := range public { + for j := 0; j < commentsPerProposal; j++ { + log := fmt.Sprintf("Submitting comments for proposal %v/%v, "+ + "comment %v/%v", i+1, len(public), j+1, commentsPerProposal) + printInPlace(log) + + // Select a random user + r := rand.Intn(len(users1)) + u := users1[r] + + // Every 5th comment should be the start of a comment thread, not + // a reply. All other comments should be replies to a random + // existing comment. + var parentID uint32 + switch { + case j%5 == 0: + // This should be a parent comment. Keep the parent ID as 0. + default: + // Reply to a random comment + parentID = uint32(rand.Intn(j + 1)) + } + + // Create random comment + b, err := util.Random(commentSize / 2) + if err != nil { + return err + } + comment := hex.EncodeToString(b) + + // Submit comment + err = commentNew(u, token, comment, parentID) + if err != nil { + return err + } + } + } + fmt.Printf("\n") + + // Setup comment votes + for i, token := range public { + // Get the number of comments this proposal has + count, err := commentCountForRecord(token) + if err != nil { + return err + } + + // We iterate through the users and comments sequentially. Trying + // to vote on comments randomly can cause max vote changes + // exceeded errors. + var ( + userIdx int + commentID uint32 = 1 + ) + for j := 0; j < commentVotesPerProposal; j++ { + log := fmt.Sprintf("Submitting comment votes for proposal %v/%v, "+ + "comment %v/%v", i+1, len(public), j+1, commentVotesPerProposal) + printInPlace(log) + + // Setup the comment ID and the user + if commentID > count { + // We've reached the end of the comments. Start back over + // with a different user. + userIdx++ + commentID = 1 + } + if userIdx == len(users2) { + // We've reached the end of the users. Start back over. + userIdx = 0 + } + u := users2[userIdx] + + // Select a random vote preference + var vote string + if rand.Intn(100)%2 == 0 { + vote = strconv.Itoa(int(cmv1.VoteUpvote)) + } else { + vote = strconv.Itoa(int(cmv1.VoteDownvote)) + } + + // Cast comment vote + err := commentVote(u, token, commentID, vote) + if err != nil { + return err + } + + // Increment comment ID + commentID++ + } + } + fmt.Printf("\n") + + ts := timestampFromUnix(time.Now().Unix()) + fmt.Printf("Done!\n") + fmt.Printf("Stop time : %v\n", ts) + fmt.Printf("Users : %v\n", userCount) + fmt.Printf("Proposals : %v\n", proposalCount) + fmt.Printf("Comments per proposal : %v\n", commentsPerProposal) + fmt.Printf("Comment votes per proposal: %v\n", commentVotesPerProposal) + + return nil +} + +type user struct { + Email string + Password string + Username string +} + +// userNew creates a new user. +// +// This function returns with the user logged out. +func userNew(email, username, password string) (*user, error) { + // Create user + c := userNewCmd{ + Verify: true, + } + c.Args.Email = email + c.Args.Username = username + c.Args.Password = password + err := c.Execute(nil) + if err != nil { + return nil, fmt.Errorf("userNewCmd: %v", err) + } + + // Log out user + err = userLogout() + if err != nil { + return nil, err + } + + return &user{ + Email: email, + Username: username, + Password: password, + }, nil +} + +// userNewRandom creates a new user with random credentials. +// +// This function returns with the user logged out. +func userNewRandom() (*user, error) { + // Hex encoding creates 2x the number of characters as bytes. + // Ex: 4 bytes will results in a 8 character hex string. + b, err := util.Random(5) + if err != nil { + return nil, err + } + var ( + r = hex.EncodeToString(b) + email = r + "@example.com" + username = "user_" + r + password = r + ) + return userNew(email, username, password) +} + +// userLogin logs in the provided user. +func userLogin(u user) error { + c := shared.LoginCmd{} + c.Args.Email = u.Email + c.Args.Password = u.Password + err := c.Execute(nil) + if err != nil { + return fmt.Errorf("LoginCmd: %v", err) + } + return nil +} + +// userLogout logs out any logged in user. +func userLogout() error { + c := shared.LogoutCmd{} + err := c.Execute(nil) + if err != nil { + return fmt.Errorf("LogoutCmd: %v", err) + } + return nil +} + +// proposalUnreviewed creates a new proposal and leaves its status as +// unreviewed. +// +// This function returns with the user logged out. +func proposalUnreviewed(u user, includeImages bool) (*rcv1.Record, error) { + // Login user + err := userLogin(u) + if err != nil { + return nil, err + } + + // Submit new proposal + cn := cmdProposalNew{ + Random: true, + RandomImages: includeImages, + } + r, err := proposalNew(&cn) + if err != nil { + return nil, fmt.Errorf("cmdProposalNew: %v", err) + } + + // Edit the proposal + ce := cmdProposalEdit{ + Unvetted: true, + Random: true, + RandomImages: includeImages, + } + ce.Args.Token = r.CensorshipRecord.Token + r, err = proposalEdit(&ce) + if err != nil { + return nil, fmt.Errorf("cmdProposalEdit: %v", err) + } + + // Logout user + err = userLogout() + if err != nil { + return nil, err + } + + return r, nil +} + +// proposalUnvettedCensored creates a new proposal then censors the proposal. +// +// This function returns with all users logged out. +func proposalUnvettedCensored(u user, admin user, includeImages bool) (*rcv1.Record, error) { + // Setup an unvetted proposal + r, err := proposalUnreviewed(u, includeImages) + if err != nil { + return nil, err + } + + // Login admin + err = userLogin(admin) + if err != nil { + return nil, err + } + + // Censor the proposal + cs := cmdProposalSetStatus{ + Unvetted: true, + } + cs.Args.Token = r.CensorshipRecord.Token + cs.Args.Status = strconv.Itoa(int(rcv1.RecordStatusCensored)) + cs.Args.Reason = "Violates proposal rules." + cs.Args.Version = r.Version + r, err = proposalSetStatus(&cs) + if err != nil { + return nil, fmt.Errorf("cmdProposalSetStatus: %v", err) + } + + // Logout admin + err = userLogout() + if err != nil { + return nil, err + } + + return r, nil +} + +// proposalPublic creates and new proposal then makes the proposal public. +// +// This function returns with all users logged out. +func proposalPublic(u user, admin user, includeImages bool) (*rcv1.Record, error) { + // Setup an unvetted proposal + r, err := proposalUnreviewed(u, includeImages) + if err != nil { + return nil, err + } + + // Login admin + err = userLogin(admin) + if err != nil { + return nil, err + } + + // Make the proposal public + cs := cmdProposalSetStatus{ + Unvetted: true, + } + cs.Args.Token = r.CensorshipRecord.Token + cs.Args.Status = strconv.Itoa(int(rcv1.RecordStatusPublic)) + cs.Args.Version = r.Version + r, err = proposalSetStatus(&cs) + if err != nil { + return nil, fmt.Errorf("cmdProposalSetStatus: %v", err) + } + + // Logout admin + err = userLogout() + if err != nil { + return nil, err + } + + // Login user + err = userLogin(u) + if err != nil { + return nil, err + } + + // Edit the proposal + ce := cmdProposalEdit{ + Unvetted: false, + Random: true, + RandomImages: includeImages, + } + ce.Args.Token = r.CensorshipRecord.Token + r, err = proposalEdit(&ce) + if err != nil { + return nil, fmt.Errorf("cmdProposalEdit: %v", err) + } + + // Logout user + err = userLogout() + if err != nil { + return nil, err + } + + return r, nil +} + +// proposalVettedCensored creates a new proposal, makes the proposal public, +// then censors the proposal. +// +// This function returns with all users logged out. +func proposalVettedCensored(u user, admin user, includeImages bool) (*rcv1.Record, error) { + // Create a public proposal + r, err := proposalPublic(u, admin, includeImages) + if err != nil { + return nil, err + } + + // Login admin + err = userLogin(admin) + if err != nil { + return nil, err + } + + // Censor the proposal + cs := cmdProposalSetStatus{ + Unvetted: false, + } + cs.Args.Token = r.CensorshipRecord.Token + cs.Args.Status = strconv.Itoa(int(rcv1.RecordStatusCensored)) + cs.Args.Reason = "Violates proposal rules." + cs.Args.Version = r.Version + r, err = proposalSetStatus(&cs) + if err != nil { + return nil, fmt.Errorf("cmdProposalSetStatus: %v", err) + } + + // Logout admin + err = userLogout() + if err != nil { + return nil, err + } + + return r, nil +} + +// proposalAbandoned creates a new proposal, makes the proposal public, +// then abandones the proposal. +// +// This function returns with all users logged out. +func proposalAbandoned(u user, admin user, includeImages bool) (*rcv1.Record, error) { + // Create a public proposal + r, err := proposalPublic(u, admin, includeImages) + if err != nil { + return nil, err + } + + // Login admin + err = userLogin(admin) + if err != nil { + return nil, err + } + + // Abandone the proposal + cs := cmdProposalSetStatus{ + Unvetted: false, + } + cs.Args.Token = r.CensorshipRecord.Token + cs.Args.Status = strconv.Itoa(int(rcv1.RecordStatusArchived)) + cs.Args.Reason = "No activity from author in 3 weeks." + cs.Args.Version = r.Version + r, err = proposalSetStatus(&cs) + if err != nil { + return nil, fmt.Errorf("cmdProposalSetStatus: %v", err) + } + + // Logout admin + err = userLogout() + if err != nil { + return nil, err + } + + return r, nil +} + +// inv returns a page of tokens for a record status. +func inv(state string, status rcv1.RecordStatusT, page uint32) ([]string, error) { + // Setup command + c := cmdProposalInv{} + switch state { + case rcv1.RecordStateUnvetted: + c.Unvetted = true + case rcv1.RecordStateVetted: + // Do nothing + default: + return nil, fmt.Errorf("invalid state") + } + c.Args.Status = strconv.Itoa(int(status)) + c.Args.Page = page + + // Get inventory + ir, err := proposalInv(&c) + if err != nil { + return nil, fmt.Errorf("cmdProposalInv: %v", err) + } + + // Unpack reply + s := rcv1.RecordStatuses[status] + var tokens []string + switch state { + case rcv1.RecordStateUnvetted: + tokens = ir.Unvetted[s] + case rcv1.RecordStateVetted: + tokens = ir.Vetted[s] + } + + return tokens, nil +} + +// invUnvetted returns a page of tokens for an unvetted record status. +// +// This function returns with the admin logged out. +func invUnvetted(admin user, status rcv1.RecordStatusT, page uint32) ([]string, error) { + // Login admin + err := userLogin(admin) + if err != nil { + return nil, err + } + + // Get a page of tokens + tokens, err := inv(rcv1.RecordStateUnvetted, status, page) + if err != nil { + return nil, err + } + + // Logout admin + err = userLogout() + if err != nil { + return nil, err + } + + return tokens, nil +} + +// commentNew submits a new comment to a public record. +// +// This function returns with the user logged out. +func commentNew(u user, token, comment string, parentID uint32) error { + // Login user + err := userLogin(u) + if err != nil { + return err + } + + // Submit comment + c := cmdCommentNew{} + c.Args.Token = token + c.Args.Comment = comment + c.Args.ParentID = parentID + err = c.Execute(nil) + if err != nil { + return fmt.Errorf("cmdCommentNew: %v", err) + } + + // Logout user + err = userLogout() + if err != nil { + return err + } + + return nil +} + +// commentCountForRecord returns the number of comments that have been made on +// a record. +func commentCountForRecord(token string) (uint32, error) { + c := cmdCommentCount{} + c.Args.Tokens = []string{token} + counts, err := commentCount(&c) + if err != nil { + return 0, fmt.Errorf("cmdCommentCount: %v", err) + } + count, ok := counts[token] + if !ok { + return 0, fmt.Errorf("cmdCommentCount: record not found %v", token) + } + return count, nil +} + +// commentVoteCasts a comment upvote/downvote. +// +// This function returns with the user logged out. +func commentVote(u user, token string, commentID uint32, vote string) error { + // Login user + err := userLogin(u) + if err != nil { + return err + } + + c := cmdCommentVote{} + c.Args.Token = token + c.Args.CommentID = commentID + c.Args.Vote = vote + err = c.Execute(nil) + if err != nil { + return err + } + + // Logout user + err = userLogout() + if err != nil { + return err + } + + return nil +} diff --git a/politeiawww/cmd/pictl/cmdproposalnew.go b/politeiawww/cmd/pictl/cmdproposalnew.go index 695bbb6f1..6821cf0b4 100644 --- a/politeiawww/cmd/pictl/cmdproposalnew.go +++ b/politeiawww/cmd/pictl/cmdproposalnew.go @@ -70,9 +70,9 @@ func proposalNew(c *cmdProposalNew) (*rcv1.Record, error) { return nil, fmt.Errorf("index file not found; you must either " + "provide an index.md file or use --random") - case c.RandomImages && len(attachments) == 0: - return nil, fmt.Errorf("you cannot provide file arguments and use " + - "the --randomimages flag at the same time") + case c.RandomImages && len(attachments) > 0: + return nil, fmt.Errorf("you cannot provide attachment files and " + + "use the --randomimages flag at the same time") case c.RFP && c.LinkBy != "": return nil, fmt.Errorf("you cannot use both the --rfp and --linkby " + @@ -105,18 +105,19 @@ func proposalNew(c *cmdProposalNew) (*rcv1.Record, error) { } // Setup proposal files + indexFileSize := 10000 // In bytes var files []rcv1.File switch { case c.Random && c.RandomImages: - // Create a random index file and random attachments - files, err = proposalFilesRandom(int(pr.TextFileSizeMax), + // Create a index file and random attachments + files, err = proposalFilesRandom(indexFileSize, int(pr.ImageFileCountMax)) if err != nil { return nil, err } case c.Random: - // Create a random index file - files, err = proposalFilesRandom(int(pr.TextFileSizeMax), 0) + // Create a index file + files, err = proposalFilesRandom(indexFileSize, 0) if err != nil { return nil, err } diff --git a/politeiawww/cmd/pictl/cmdtestproposalcapacity.go b/politeiawww/cmd/pictl/cmdtestproposalcapacity.go deleted file mode 100644 index b9f36f857..000000000 --- a/politeiawww/cmd/pictl/cmdtestproposalcapacity.go +++ /dev/null @@ -1,465 +0,0 @@ -// Copyright (c) 2020-2021 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package main - -import ( - "encoding/hex" - "fmt" - "math/rand" - "strconv" - - rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" - "github.com/decred/politeia/politeiawww/cmd/shared" - "github.com/decred/politeia/util" -) - -type cmdTestProposalCapacity struct { - Args struct { - AdminEmail string `positional-arg-name:"adminemail" required:"true"` - AdminPassword string `positional-arg-name:"adminpassword" required:"true"` - } `positional-args:"true"` -} - -// TODO this is temp -const randomImages = false - -// Execute executes the cmdTestProposalCapacity command. -// -// This function satisfies the go-flags Commander interface. -func (c *cmdTestProposalCapacity) Execute(args []string) error { - const ( - // Test run params - userCount = 1 - proposalCount = 5 - commentsPerProposal = 100 - commentVotesPerProposal = 1000 - ) - - // We don't want the output of individual commands printed. - cfg.Verbose = false - cfg.RawJSON = false - cfg.Silent = true - - // Verify the the provided login credentials are for an admin. - admin := user{ - Email: c.Args.AdminEmail, - Password: c.Args.AdminPassword, - } - err := userLogin(admin) - if err != nil { - return fmt.Errorf("failed to login admin: %v", err) - } - lr, err := client.Me() - if err != nil { - return err - } - if !lr.IsAdmin { - return fmt.Errorf("provided user is not an admin") - } - admin.Username = lr.Username - - // Verify that the paywall is disabled. - policyWWW, err := client.Policy() - if err != nil { - return err - } - if policyWWW.PaywallEnabled { - return fmt.Errorf("paywall is not disabled") - } - - // Setup users - users := make([]user, 0, userCount) - for i := 0; i < userCount; i++ { - log := fmt.Sprintf("Creating user %v/%v", i+1, userCount) - printInPlace(log) - - u, err := userNewRandom(policyWWW.MaxUsernameLength) - if err != nil { - return err - } - - users = append(users, *u) - } - fmt.Printf("\n") - - // Setup proposals - var ( - statusUnreviewed = "unreviewed" - statusUnvettedCensored = "unvetted-censored" - statusPublic = "public" - statusVettedCensored = "vetted-cesored" - statusAbandoned = "abandoned" - - statuses = []string{ - statusUnreviewed, - statusUnvettedCensored, - statusPublic, - statusVettedCensored, - statusAbandoned, - } - - // These are used to track the number of proposals that are - // created for each status. - countUnreviewed int - countUnvettedCensored int - countPublic int - countVettedCensored int - countAbandoned int - ) - for i := 0; i < proposalCount; i++ { - // Select a random user - r := rand.Intn(len(users)) - u := users[r] - - // Select a random status. This will be the ending status of - // the proposal. - r = rand.Intn(len(statuses)) - s := statuses[r] - - log := fmt.Sprintf("Submitting proposal %v/%v: %-17v", - i+1, proposalCount, s) - printInPlace(log) - - // Create proposal - switch s { - case statusUnreviewed: - _, err = proposalUnreviewed(u) - if err != nil { - return err - } - countUnreviewed++ - case statusUnvettedCensored: - _, err = proposalUnvettedCensored(u, admin) - if err != nil { - return err - } - countUnvettedCensored++ - case statusPublic: - _, err = proposalPublic(u, admin) - if err != nil { - return err - } - countPublic++ - case statusVettedCensored: - _, err = proposalVettedCensored(u, admin) - if err != nil { - return err - } - countVettedCensored++ - case statusAbandoned: - _, err = proposalAbandoned(u, admin) - if err != nil { - return err - } - countAbandoned++ - default: - return fmt.Errorf("invalid status %v", s) - } - } - fmt.Printf("\n") - - // Setup comments - // Setup comment votes - - _ = commentsPerProposal - _ = commentVotesPerProposal - - return nil -} - -type user struct { - Email string - Password string - Username string -} - -// userNew creates a new user. -// -// This function returns with the user logged out. -func userNew(email, username, password string) (*user, error) { - // Create user - c := userNewCmd{ - Verify: true, - } - c.Args.Email = email - c.Args.Username = username - c.Args.Password = password - err := c.Execute(nil) - if err != nil { - return nil, fmt.Errorf("userNewCmd: %v", err) - } - - // Log out user - err = userLogout() - if err != nil { - return nil, err - } - - return &user{ - Email: email, - Username: username, - Password: password, - }, nil -} - -// userNewRandom creates a new user with random credentials. -// -// This function returns with the user logged out. -func userNewRandom(usernameLength uint) (*user, error) { - // Hex encoding creates 2x the number of characters as bytes. - // Ex: 4 bytes will results in a 8 character hex string. - b, err := util.Random(int(usernameLength / uint(2))) - if err != nil { - return nil, err - } - var ( - r = hex.EncodeToString(b) - email = r + "@example.com" - username = r - password = r - ) - return userNew(email, username, password) -} - -// userLogin logs in the provided user. -func userLogin(u user) error { - c := shared.LoginCmd{} - c.Args.Email = u.Email - c.Args.Password = u.Password - err := c.Execute(nil) - if err != nil { - return fmt.Errorf("LoginCmd: %v", err) - } - return nil -} - -// userLogout logs out any logged in user. -func userLogout() error { - c := shared.LogoutCmd{} - err := c.Execute(nil) - if err != nil { - return fmt.Errorf("LogoutCmd: %v", err) - } - return nil -} - -// proposalUnreviewed creates a new proposal and leaves its status as -// unreviewed. -// -// This function returns with the user logged out. -func proposalUnreviewed(u user) (*rcv1.Record, error) { - // Login user - err := userLogin(u) - if err != nil { - return nil, err - } - - // Submit new proposal - cn := cmdProposalNew{ - Random: true, - RandomImages: randomImages, - } - r, err := proposalNew(&cn) - if err != nil { - return nil, fmt.Errorf("cmdProposalNew: %v", err) - } - - // Edit the proposal - ce := cmdProposalEdit{ - Unvetted: true, - Random: true, - RandomImages: randomImages, - } - ce.Args.Token = r.CensorshipRecord.Token - r, err = proposalEdit(&ce) - if err != nil { - return nil, fmt.Errorf("cmdProposalEdit: %v", err) - } - - // Logout user - err = userLogout() - if err != nil { - return nil, err - } - - return r, nil -} - -// proposalUnvettedCensored creates a new proposal then censors the proposal. -// -// This function returns with all users logged out. -func proposalUnvettedCensored(u user, admin user) (*rcv1.Record, error) { - // Setup an unvetted proposal - r, err := proposalUnreviewed(u) - if err != nil { - return nil, err - } - - // Login admin - err = userLogin(admin) - if err != nil { - return nil, err - } - - // Censor the proposal - cs := cmdProposalSetStatus{ - Unvetted: true, - } - cs.Args.Token = r.CensorshipRecord.Token - cs.Args.Status = strconv.Itoa(int(rcv1.RecordStatusCensored)) - cs.Args.Reason = "Violates proposal rules." - cs.Args.Version = r.Version - r, err = proposalSetStatus(&cs) - if err != nil { - return nil, fmt.Errorf("cmdProposalSetStatus: %v", err) - } - - // Logout admin - err = userLogout() - if err != nil { - return nil, err - } - - return r, nil -} - -// proposalPublic creates and new proposal then makes the proposal public. -// -// This function returns with all users logged out. -func proposalPublic(u user, admin user) (*rcv1.Record, error) { - // Setup an unvetted proposal - r, err := proposalUnreviewed(u) - if err != nil { - return nil, err - } - - // Login admin - err = userLogin(admin) - if err != nil { - return nil, err - } - - // Make the proposal public - cs := cmdProposalSetStatus{ - Unvetted: true, - } - cs.Args.Token = r.CensorshipRecord.Token - cs.Args.Status = strconv.Itoa(int(rcv1.RecordStatusPublic)) - cs.Args.Version = r.Version - r, err = proposalSetStatus(&cs) - if err != nil { - return nil, fmt.Errorf("cmdProposalSetStatus: %v", err) - } - - // Logout admin - err = userLogout() - if err != nil { - return nil, err - } - - // Login user - err = userLogin(u) - if err != nil { - return nil, err - } - - // Edit the proposal - ce := cmdProposalEdit{ - Unvetted: false, - Random: true, - RandomImages: randomImages, - } - ce.Args.Token = r.CensorshipRecord.Token - r, err = proposalEdit(&ce) - if err != nil { - return nil, fmt.Errorf("cmdProposalEdit: %v", err) - } - - // Logout user - err = userLogout() - if err != nil { - return nil, err - } - - return r, nil -} - -// proposalVettedCensored creates a new proposal, makes the proposal public, -// then censors the proposal. -// -// This function returns with all users logged out. -func proposalVettedCensored(u user, admin user) (*rcv1.Record, error) { - // Create a public proposal - r, err := proposalPublic(u, admin) - if err != nil { - return nil, err - } - - // Login admin - err = userLogin(admin) - if err != nil { - return nil, err - } - - // Censor the proposal - cs := cmdProposalSetStatus{ - Unvetted: false, - } - cs.Args.Token = r.CensorshipRecord.Token - cs.Args.Status = strconv.Itoa(int(rcv1.RecordStatusCensored)) - cs.Args.Reason = "Violates proposal rules." - cs.Args.Version = r.Version - r, err = proposalSetStatus(&cs) - if err != nil { - return nil, fmt.Errorf("cmdProposalSetStatus: %v", err) - } - - // Logout admin - err = userLogout() - if err != nil { - return nil, err - } - - return r, nil -} - -// proposalAbandoned creates a new proposal, makes the proposal public, -// then abandones the proposal. -// -// This function returns with all users logged out. -func proposalAbandoned(u user, admin user) (*rcv1.Record, error) { - // Create a public proposal - r, err := proposalPublic(u, admin) - if err != nil { - return nil, err - } - - // Login admin - err = userLogin(admin) - if err != nil { - return nil, err - } - - // Abandone the proposal - cs := cmdProposalSetStatus{ - Unvetted: false, - } - cs.Args.Token = r.CensorshipRecord.Token - cs.Args.Status = strconv.Itoa(int(rcv1.RecordStatusArchived)) - cs.Args.Reason = "No activity from author in 3 weeks." - cs.Args.Version = r.Version - r, err = proposalSetStatus(&cs) - if err != nil { - return nil, fmt.Errorf("cmdProposalSetStatus: %v", err) - } - - // Logout admin - err = userLogout() - if err != nil { - return nil, err - } - - return r, nil -} diff --git a/politeiawww/cmd/pictl/pictl.go b/politeiawww/cmd/pictl/pictl.go index d93a4cead..81712ca4b 100644 --- a/politeiawww/cmd/pictl/pictl.go +++ b/politeiawww/cmd/pictl/pictl.go @@ -100,9 +100,9 @@ type pictl struct { Subscribe subscribeCmd `command:"subscribe"` // Dev commands - SendFaucetTx cmdSendFaucetTx `command:"sendfaucettx"` - TestRun cmdTestRun `command:"testrun"` - TestProposalCapcity cmdTestProposalCapacity `command:"testproposalcapacity"` + SendFaucetTx cmdSendFaucetTx `command:"sendfaucettx"` + TestRun cmdTestRun `command:"testrun"` + ProposalLoadTest cmdProposalLoadTest `command:"proposalloadtest"` } const helpMsg = `Application Options: @@ -182,6 +182,7 @@ Websocket commands Dev commands sendfaucettx Send a dcr faucet tx testrun Execute a test run of pi routes + proposalloadtest Execute a load test using proposals ` func _main() error { diff --git a/politeiawww/cmd/pictl/proposal.go b/politeiawww/cmd/pictl/proposal.go index 7588cd4ff..f6f41309d 100644 --- a/politeiawww/cmd/pictl/proposal.go +++ b/politeiawww/cmd/pictl/proposal.go @@ -163,6 +163,7 @@ func indexFileRandom(sizeInBytes int) (*rcv1.File, error) { char := charSet[r] b.WriteString(string(char)) } + b.WriteString("\n") payload := []byte(b.String()) return &rcv1.File{ @@ -207,13 +208,11 @@ func pngFileRandom() (*rcv1.File, error) { }, nil } -func proposalFilesRandom(textFileSizeMax, imageFileCountMax int) ([]rcv1.File, error) { +func proposalFilesRandom(textFileSize, imageFileCountMax int) ([]rcv1.File, error) { files := make([]rcv1.File, 0, 16) - // Generate random text for the index file. The size of the index - // file is also randomly chosen - size := rand.Intn(textFileSizeMax) - f, err := indexFileRandom(size) + // Generate random text for the index file + f, err := indexFileRandom(textFileSize) if err != nil { return nil, err } From 2bf612883a4630f9ff7fd61b18cb8ba0c358913a Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 18 Feb 2021 17:50:40 -0600 Subject: [PATCH 317/449] pictl: Add setupvotetest command. --- ...roposalloadtest.go => cmdseedproposals.go} | 30 ++-- politeiawww/cmd/pictl/cmdsetupvotetest.go | 154 ++++++++++++++++++ politeiawww/cmd/pictl/pictl.go | 13 +- 3 files changed, 177 insertions(+), 20 deletions(-) rename politeiawww/cmd/pictl/{cmdproposalloadtest.go => cmdseedproposals.go} (96%) create mode 100644 politeiawww/cmd/pictl/cmdsetupvotetest.go diff --git a/politeiawww/cmd/pictl/cmdproposalloadtest.go b/politeiawww/cmd/pictl/cmdseedproposals.go similarity index 96% rename from politeiawww/cmd/pictl/cmdproposalloadtest.go rename to politeiawww/cmd/pictl/cmdseedproposals.go index 9249a748b..46d6c478b 100644 --- a/politeiawww/cmd/pictl/cmdproposalloadtest.go +++ b/politeiawww/cmd/pictl/cmdseedproposals.go @@ -17,7 +17,7 @@ import ( "github.com/decred/politeia/util" ) -type cmdProposalLoadTest struct { +type cmdSeedProposals struct { Args struct { AdminEmail string `positional-arg-name:"adminemail" required:"true"` AdminPassword string `positional-arg-name:"adminpassword" required:"true"` @@ -31,13 +31,13 @@ type cmdProposalLoadTest struct { IncludeImages bool `long:"includeimages"` } -// Execute executes the cmdProposalLoadTest command. +// Execute executes the cmdSeedProposals command. // // This function satisfies the go-flags Commander interface. -func (c *cmdProposalLoadTest) Execute(args []string) error { +func (c *cmdSeedProposals) Execute(args []string) error { fmt.Printf("Warn: this cmd should be run on a clean politeia instance\n") - // Setup default test parameters + // Setup default parameters var ( userCount = 10 proposalCount = 25 @@ -508,9 +508,9 @@ func proposalUnreviewed(u user, includeImages bool) (*rcv1.Record, error) { // proposalUnvettedCensored creates a new proposal then censors the proposal. // // This function returns with all users logged out. -func proposalUnvettedCensored(u user, admin user, includeImages bool) (*rcv1.Record, error) { +func proposalUnvettedCensored(author, admin user, includeImages bool) (*rcv1.Record, error) { // Setup an unvetted proposal - r, err := proposalUnreviewed(u, includeImages) + r, err := proposalUnreviewed(author, includeImages) if err != nil { return nil, err } @@ -546,9 +546,9 @@ func proposalUnvettedCensored(u user, admin user, includeImages bool) (*rcv1.Rec // proposalPublic creates and new proposal then makes the proposal public. // // This function returns with all users logged out. -func proposalPublic(u user, admin user, includeImages bool) (*rcv1.Record, error) { +func proposalPublic(author, admin user, includeImages bool) (*rcv1.Record, error) { // Setup an unvetted proposal - r, err := proposalUnreviewed(u, includeImages) + r, err := proposalUnreviewed(author, includeImages) if err != nil { return nil, err } @@ -577,8 +577,8 @@ func proposalPublic(u user, admin user, includeImages bool) (*rcv1.Record, error return nil, err } - // Login user - err = userLogin(u) + // Login author + err = userLogin(author) if err != nil { return nil, err } @@ -595,7 +595,7 @@ func proposalPublic(u user, admin user, includeImages bool) (*rcv1.Record, error return nil, fmt.Errorf("cmdProposalEdit: %v", err) } - // Logout user + // Logout author err = userLogout() if err != nil { return nil, err @@ -608,9 +608,9 @@ func proposalPublic(u user, admin user, includeImages bool) (*rcv1.Record, error // then censors the proposal. // // This function returns with all users logged out. -func proposalVettedCensored(u user, admin user, includeImages bool) (*rcv1.Record, error) { +func proposalVettedCensored(author, admin user, includeImages bool) (*rcv1.Record, error) { // Create a public proposal - r, err := proposalPublic(u, admin, includeImages) + r, err := proposalPublic(author, admin, includeImages) if err != nil { return nil, err } @@ -647,9 +647,9 @@ func proposalVettedCensored(u user, admin user, includeImages bool) (*rcv1.Recor // then abandones the proposal. // // This function returns with all users logged out. -func proposalAbandoned(u user, admin user, includeImages bool) (*rcv1.Record, error) { +func proposalAbandoned(author, admin user, includeImages bool) (*rcv1.Record, error) { // Create a public proposal - r, err := proposalPublic(u, admin, includeImages) + r, err := proposalPublic(author, admin, includeImages) if err != nil { return nil, err } diff --git a/politeiawww/cmd/pictl/cmdsetupvotetest.go b/politeiawww/cmd/pictl/cmdsetupvotetest.go new file mode 100644 index 000000000..290e652be --- /dev/null +++ b/politeiawww/cmd/pictl/cmdsetupvotetest.go @@ -0,0 +1,154 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" +) + +type cmdSetupVoteTest struct { + Args struct { + AdminEmail string `positional-arg-name:"adminemail" required:"true"` + AdminPassword string `positional-arg-name:"adminpassword" required:"true"` + Votes uint32 `positional-arg-name:"votes"` + } `positional-args:"true"` + + // IncludeImages is used to include a random number of images when + // submitting proposals. + IncludeImages bool `long:"includeimages"` +} + +// Execute executes the cmdSetupVoteTest command. +// +// This function satisfies the go-flags Commander interface. +func (c *cmdSetupVoteTest) Execute(args []string) error { + // Setup test parameters + var ( + votes uint32 = 10 + duration uint32 = 6 // In blocks + quorum uint32 = 1 // Percentage of total tickets + pass uint32 = 50 // Percentage of votes cast + ) + if c.Args.Votes > 0 { + votes = c.Args.Votes + } + + // We don't want the output of individual commands printed. + cfg.Verbose = false + cfg.RawJSON = false + cfg.Silent = true + + // Verify the the provided login credentials are for an admin. + admin := user{ + Email: c.Args.AdminEmail, + Password: c.Args.AdminPassword, + } + err := userLogin(admin) + if err != nil { + return fmt.Errorf("failed to login admin: %v", err) + } + lr, err := client.Me() + if err != nil { + return err + } + if !lr.IsAdmin { + return fmt.Errorf("provided user is not an admin") + } + admin.Username = lr.Username + + // Verify that the paywall is disabled + policyWWW, err := client.Policy() + if err != nil { + return err + } + if policyWWW.PaywallEnabled { + return fmt.Errorf("paywall is not disabled") + } + + // Setup votes + for i := 0; i < int(votes); i++ { + s := fmt.Sprintf("Starting voting period on proposal %v/%v", i+1, votes) + printInPlace(s) + + // Create a public proposal + r, err := proposalPublic(admin, admin, false) + if err != nil { + return err + } + token := r.CensorshipRecord.Token + + // Authorize vote + err = voteAuthorize(admin, token) + if err != nil { + return err + } + + // Start vote + err = voteStart(admin, token, duration, quorum, pass) + if err != nil { + return err + } + } + fmt.Printf("\n") + + return nil +} + +// voteAuthorize authorizes the ticket vote. +// +// This function returns with the user logged out. +func voteAuthorize(author user, token string) error { + // Login author + err := userLogin(author) + if err != nil { + return err + } + + // Authorize the voting period + c := cmdVoteAuthorize{} + c.Args.Token = token + err = c.Execute(nil) + if err != nil { + return fmt.Errorf("cmdVoteAuthorize: %v", err) + } + + // Logout author + err = userLogout() + if err != nil { + return err + } + + return nil +} + +// voteStart starts the voting period on a record. +// +// This function returns with the admin logged out. +func voteStart(admin user, token string, duration, quorum, pass uint32) error { + // Login admin + err := userLogin(admin) + if err != nil { + return err + } + + // Start the voting period + c := cmdVoteStart{} + c.Args.Token = token + c.Args.Duration = duration + c.Args.QuorumPercentage = quorum + c.Args.PassPercentage = pass + err = c.Execute(nil) + if err != nil { + return fmt.Errorf("cmdVoteStart: %v", err) + } + + // Logout admin + err = userLogout() + if err != nil { + return err + } + + return nil +} diff --git a/politeiawww/cmd/pictl/pictl.go b/politeiawww/cmd/pictl/pictl.go index 81712ca4b..a05eb8b4c 100644 --- a/politeiawww/cmd/pictl/pictl.go +++ b/politeiawww/cmd/pictl/pictl.go @@ -100,9 +100,10 @@ type pictl struct { Subscribe subscribeCmd `command:"subscribe"` // Dev commands - SendFaucetTx cmdSendFaucetTx `command:"sendfaucettx"` - TestRun cmdTestRun `command:"testrun"` - ProposalLoadTest cmdProposalLoadTest `command:"proposalloadtest"` + SendFaucetTx cmdSendFaucetTx `command:"sendfaucettx"` + TestRun cmdTestRun `command:"testrun"` + SeedProposals cmdSeedProposals `command:"seedproposals"` + SetupVoteTest cmdSetupVoteTest `command:"setupvotetest"` } const helpMsg = `Application Options: @@ -181,8 +182,10 @@ Websocket commands Dev commands sendfaucettx Send a dcr faucet tx - testrun Execute a test run of pi routes - proposalloadtest Execute a load test using proposals + testrun Execute a test run of the pi routes + seedproposals Seed the backend with proposals + setupvotetest Setup a vote test + votetest Execute a vote test ` func _main() error { From 4120a37b51a6f89b86e0982d7e8be393cf74481f Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 18 Feb 2021 18:45:05 -0600 Subject: [PATCH 318/449] pictl: Add votetest command. --- politeiawww/cmd/pictl/cmdvoteinv.go | 19 +++- politeiawww/cmd/pictl/cmdvotetest.go | 106 ++++++++++++++++++ ...mdsetupvotetest.go => cmdvotetestsetup.go} | 24 +++- politeiawww/cmd/pictl/pictl.go | 5 +- 4 files changed, 142 insertions(+), 12 deletions(-) create mode 100644 politeiawww/cmd/pictl/cmdvotetest.go rename politeiawww/cmd/pictl/{cmdsetupvotetest.go => cmdvotetestsetup.go} (80%) diff --git a/politeiawww/cmd/pictl/cmdvoteinv.go b/politeiawww/cmd/pictl/cmdvoteinv.go index e904a20f9..faacabaf0 100644 --- a/politeiawww/cmd/pictl/cmdvoteinv.go +++ b/politeiawww/cmd/pictl/cmdvoteinv.go @@ -25,6 +25,16 @@ type cmdVoteInv struct { // // This function satisfies the go-flags Commander interface. func (c *cmdVoteInv) Execute(args []string) error { + _, err := voteInv(c) + if err != nil { + return err + } + return nil +} + +// voteInv returns the vote inventory. It has been pulled out of Execute so +// that it can be used in test commands. +func voteInv(c *cmdVoteInv) (map[string][]string, error) { // Setup client opts := pclient.Opts{ HTTPSCert: cfg.HTTPSCert, @@ -33,7 +43,7 @@ func (c *cmdVoteInv) Execute(args []string) error { } pc, err := pclient.New(cfg.Host, opts) if err != nil { - return err + return nil, err } // Setup status and page number @@ -43,7 +53,7 @@ func (c *cmdVoteInv) Execute(args []string) error { // human readable equivalent. status, err = parseVoteStatus(c.Args.Status) if err != nil { - return err + return nil, err } // If a status was given but no page number was give, default @@ -60,13 +70,14 @@ func (c *cmdVoteInv) Execute(args []string) error { } ir, err := pc.TicketVoteInventory(i) if err != nil { - return err + return nil, err } // Print inventory printJSON(ir) - return nil + return ir.Vetted, nil + } func parseVoteStatus(status string) (tkv1.VoteStatusT, error) { diff --git a/politeiawww/cmd/pictl/cmdvotetest.go b/politeiawww/cmd/pictl/cmdvotetest.go new file mode 100644 index 000000000..3f8f6be66 --- /dev/null +++ b/politeiawww/cmd/pictl/cmdvotetest.go @@ -0,0 +1,106 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "math/rand" + "strconv" + "sync" + + tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" +) + +type cmdVoteTest struct { + Args struct { + Password string `positional-arg-name:"password"` + } `positional-args:"true" required:"true"` +} + +// Execute executes the cmdVoteTest command. +// +// This function satisfies the go-flags Commander interface. +func (c *cmdVoteTest) Execute(args []string) error { + // We don't want the output of individual commands printed. + cfg.Verbose = false + cfg.RawJSON = false + cfg.Silent = true + + // Get all ongoing votes + votes := make([]string, 0, 256) + var page uint32 = 1 + for { + tokens, err := voteInvForStatus(tkv1.VoteStatusStarted, page) + if err != nil { + return err + } + if len(tokens) == 0 { + // We've reached the end of the inventory + break + } + votes = append(votes, tokens...) + page++ + } + + // Setup vote options + voteOptions := []string{ + tkv1.VoteOptionIDApprove, + tkv1.VoteOptionIDReject, + } + + // Cast ballots concurrently + var wg sync.WaitGroup + for _, v := range votes { + // Select vote option randomly + r := rand.Intn(len(voteOptions)) + voteOption := voteOptions[r] + + wg.Add(1) + go func(wg *sync.WaitGroup, token, voteOption, password string) { + defer wg.Done() + + // Turn printing back on for this part + cfg.Silent = false + + // Cast ballot + fmt.Printf("Casting ballot for %v %v\n", token, voteOption) + err := castBallot(token, voteOption, password) + if err != nil { + fmt.Printf("castBallot %v: %v", token, err) + } + }(&wg, v, voteOption, c.Args.Password) + } + + wg.Wait() + + return nil +} + +// voteInvForStatus returns a page of tokens for a vote status. +func voteInvForStatus(s tkv1.VoteStatusT, page uint32) ([]string, error) { + // Setup command + c := cmdVoteInv{} + c.Args.Status = strconv.Itoa(int(s)) + c.Args.Page = page + + // Get inventory + inv, err := voteInv(&c) + if err != nil { + return nil, fmt.Errorf("cmdVoteInv: %v", err) + } + + // Unpack reply + sm := tkv1.VoteStatuses[s] + return inv[sm], nil +} + +func castBallot(token, voteID, password string) error { + c := cmdCastBallot{ + Password: password, + } + c.Args.Token = token + c.Args.VoteID = voteID + return c.Execute(nil) +} diff --git a/politeiawww/cmd/pictl/cmdsetupvotetest.go b/politeiawww/cmd/pictl/cmdvotetestsetup.go similarity index 80% rename from politeiawww/cmd/pictl/cmdsetupvotetest.go rename to politeiawww/cmd/pictl/cmdvotetestsetup.go index 290e652be..2e295f324 100644 --- a/politeiawww/cmd/pictl/cmdsetupvotetest.go +++ b/politeiawww/cmd/pictl/cmdvotetestsetup.go @@ -8,11 +8,14 @@ import ( "fmt" ) -type cmdSetupVoteTest struct { +type cmdVoteTestSetup struct { Args struct { - AdminEmail string `positional-arg-name:"adminemail" required:"true"` - AdminPassword string `positional-arg-name:"adminpassword" required:"true"` - Votes uint32 `positional-arg-name:"votes"` + AdminEmail string `positional-arg-name:"adminemail" required:"true"` + AdminPassword string `positional-arg-name:"adminpassword" required:"true"` + Votes uint32 `positional-arg-name:"votes"` + Duration uint32 `positional-arg-name:"duration"` + QuorumPercentage uint32 `positional-arg-name:"quorumpercentage"` + PassPercentage uint32 `positional-arg-name:"passpercentage"` } `positional-args:"true"` // IncludeImages is used to include a random number of images when @@ -20,10 +23,10 @@ type cmdSetupVoteTest struct { IncludeImages bool `long:"includeimages"` } -// Execute executes the cmdSetupVoteTest command. +// Execute executes the cmdVoteTestSetup command. // // This function satisfies the go-flags Commander interface. -func (c *cmdSetupVoteTest) Execute(args []string) error { +func (c *cmdVoteTestSetup) Execute(args []string) error { // Setup test parameters var ( votes uint32 = 10 @@ -34,6 +37,15 @@ func (c *cmdSetupVoteTest) Execute(args []string) error { if c.Args.Votes > 0 { votes = c.Args.Votes } + if c.Args.Duration > 0 { + duration = c.Args.Duration + } + if c.Args.QuorumPercentage > 0 { + quorum = c.Args.QuorumPercentage + } + if c.Args.PassPercentage > 0 { + pass = c.Args.PassPercentage + } // We don't want the output of individual commands printed. cfg.Verbose = false diff --git a/politeiawww/cmd/pictl/pictl.go b/politeiawww/cmd/pictl/pictl.go index a05eb8b4c..abcc3ff02 100644 --- a/politeiawww/cmd/pictl/pictl.go +++ b/politeiawww/cmd/pictl/pictl.go @@ -103,7 +103,8 @@ type pictl struct { SendFaucetTx cmdSendFaucetTx `command:"sendfaucettx"` TestRun cmdTestRun `command:"testrun"` SeedProposals cmdSeedProposals `command:"seedproposals"` - SetupVoteTest cmdSetupVoteTest `command:"setupvotetest"` + VoteTestSetup cmdVoteTestSetup `command:"votetestsetup"` + VoteTest cmdVoteTest `command:"votetest"` } const helpMsg = `Application Options: @@ -184,7 +185,7 @@ Dev commands sendfaucettx Send a dcr faucet tx testrun Execute a test run of the pi routes seedproposals Seed the backend with proposals - setupvotetest Setup a vote test + votetestsetup Setup a vote test votetest Execute a vote test ` From 745329873fb3960cf74e9ae448872e4754b7c3ed Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 19 Feb 2021 08:45:48 -0600 Subject: [PATCH 319/449] Small tweaks. --- .../backend/tlogbe/plugins/ticketvote/cmds.go | 4 ++-- politeiad/backend/tlogbe/tlog/anchor.go | 4 +--- politeiad/backend/tlogbe/tlogbe.go | 20 +++++++++---------- politeiawww/cmd/pictl/cmdvotetest.go | 10 ++++++++++ util/net.go | 6 +++--- 5 files changed, 26 insertions(+), 18 deletions(-) diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go index f6ed875fa..6df297a50 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go @@ -2309,8 +2309,8 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str // the ballot will take 200 milliseconds since we wait for the leaf // to be fully appended before considering the trillian call // successful. A person casting hundreds of votes in a single ballot - // would cause UX issues for all the voting clients since the lock is - // held during these calls. + // would cause UX issues for all the voting clients since the backend + // lock is held during these calls. // // The second variable that we must watch out for is the max trillian // queued leaf batch size. This is also a configurable trillian value diff --git a/politeiad/backend/tlogbe/tlog/anchor.go b/politeiad/backend/tlogbe/tlog/anchor.go index 3f2a8c368..2643f1206 100644 --- a/politeiad/backend/tlogbe/tlog/anchor.go +++ b/politeiad/backend/tlogbe/tlog/anchor.go @@ -30,9 +30,7 @@ const ( // currently drops an anchor on the hour mark so we submit new // anchors a few minutes prior to that. // Seconds Minutes Hours Days Months DayOfWeek - // TODO change back when done testing - // anchorSchedule = "0 56 * * * *" // At minute 56 of every hour - anchorSchedule = "0 */5 * * * *" // Every 5 minutes + anchorSchedule = "0 56 * * * *" // At minute 56 of every hour // anchorID is included in the timestamp and verify requests as a // unique identifier. diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index c3670ebd3..4b2e5c7ad 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -1550,10 +1550,10 @@ func (t *tlogBackend) unvettedPluginRead(token []byte, pluginID, cmd, payload st } if len(token) > 0 { - log.Debugf("Unvetted '%v' plugin cmd '%v' on record %x", + log.Infof("Unvetted '%v' plugin read cmd '%v' on %x", pluginID, cmd, token) } else { - log.Debugf("Unvetted '%v' plugin command '%v'", + log.Infof("Unvetted '%v' plugin read cmd '%v'", pluginID, cmd) } @@ -1569,9 +1569,6 @@ func (t *tlogBackend) unvettedPluginWrite(token []byte, pluginID, cmd, payload s return "", backend.ErrRecordNotFound } - log.Debugf("Unvetted '%v' plugin cmd '%v' on record %x", - pluginID, cmd, token) - // Hold the record lock for the remainder of this function. We // do this here in the backend so that the individual plugins // implementations don't need to worry about race conditions. @@ -1579,6 +1576,9 @@ func (t *tlogBackend) unvettedPluginWrite(token []byte, pluginID, cmd, payload s m.Lock() defer m.Unlock() + log.Infof("Unvetted '%v' plugin write cmd '%v' on %x", + pluginID, cmd, token) + // Call pre plugin hooks hp := plugins.HookPluginPre{ State: plugins.RecordStateUnvetted, @@ -1666,10 +1666,10 @@ func (t *tlogBackend) vettedPluginRead(token []byte, pluginID, cmd, payload stri } if len(token) > 0 { - log.Debugf("Vetted '%v' plugin cmd '%v' on record %x", + log.Infof("Vetted '%v' plugin read cmd '%v' on %x", pluginID, cmd, token) } else { - log.Debugf("Vetted '%v' plugin command '%v'", + log.Infof("Vetted '%v' plugin read cmd '%v'", pluginID, cmd) } @@ -1683,9 +1683,6 @@ func (t *tlogBackend) vettedPluginWrite(token []byte, pluginID, cmd, payload str return "", backend.ErrRecordNotFound } - log.Debugf("Vetted '%v' plugin cmd '%v' on record %x", - pluginID, cmd, token) - // Hold the record lock for the remainder of this function. We // do this here in the backend so that the individual plugins // implementations don't need to worry about race conditions. @@ -1693,6 +1690,9 @@ func (t *tlogBackend) vettedPluginWrite(token []byte, pluginID, cmd, payload str m.Lock() defer m.Unlock() + log.Infof("Vetted '%v' plugin write cmd '%v' on %x", + pluginID, cmd, token) + // Call pre plugin hooks hp := plugins.HookPluginPre{ State: plugins.RecordStateVetted, diff --git a/politeiawww/cmd/pictl/cmdvotetest.go b/politeiawww/cmd/pictl/cmdvotetest.go index 3f8f6be66..fe7fff5df 100644 --- a/politeiawww/cmd/pictl/cmdvotetest.go +++ b/politeiawww/cmd/pictl/cmdvotetest.go @@ -9,6 +9,7 @@ import ( "math/rand" "strconv" "sync" + "time" tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" ) @@ -43,6 +44,9 @@ func (c *cmdVoteTest) Execute(args []string) error { votes = append(votes, tokens...) page++ } + if len(votes) == 0 { + return fmt.Errorf("no ongoing votes") + } // Setup vote options voteOptions := []string{ @@ -66,10 +70,16 @@ func (c *cmdVoteTest) Execute(args []string) error { // Cast ballot fmt.Printf("Casting ballot for %v %v\n", token, voteOption) + start := time.Now() err := castBallot(token, voteOption, password) if err != nil { fmt.Printf("castBallot %v: %v", token, err) } + end := time.Now() + elapsed := end.Sub(start) + + fmt.Printf("%v elapsed time %v\n", token, elapsed) + }(&wg, v, voteOption, c.Args.Password) } diff --git a/util/net.go b/util/net.go index addfaa541..7728db6f5 100644 --- a/util/net.go +++ b/util/net.go @@ -48,10 +48,10 @@ func NewHTTPClient(skipVerify bool, certPath string) (*http.Client, error) { } return &http.Client{ - Timeout: 1 * time.Minute, + Timeout: 2 * time.Minute, Transport: &http.Transport{ - IdleConnTimeout: 1 * time.Minute, - ResponseHeaderTimeout: 1 * time.Minute, + IdleConnTimeout: 2 * time.Minute, + ResponseHeaderTimeout: 2 * time.Minute, TLSClientConfig: tlsConfig, }}, nil } From 04e9c0129bc9343dfd8085c0f9d8a7818369fe88 Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 19 Feb 2021 10:29:37 -0600 Subject: [PATCH 320/449] Add comment about potential cast ballot issues. --- .../backend/tlogbe/plugins/ticketvote/cmds.go | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go index 6df297a50..01117c76e 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go @@ -2107,7 +2107,18 @@ func (p *ticketVotePlugin) ballot(treeID int64, votes []ticketvote.CastVote, res // cmdCastBallot casts a ballot of votes. This function will not return a user // error if one occurs. It will instead return the ballot reply with the error -// included in the invidiual cast vote reply that it applies to. +// included in the individual cast vote reply that it applies to. +// +// NOTE: Record locking is currently handled by the backend, not by individual +// plugins. This makes the plugin implementations simpler and easier to reason +// about, but it can also lead to performance bottlenecks for expensive plugin +// write commands. This cast ballot command is one such command because of the +// external dcrdata calls that it makes to verify the largest commitment +// addresses. If this becomes an issue then we will need to cache all of the +// valid commitment addresses when we start the voting period. It takes ~1.5 +// minutes to compile this data for 41k eligible tickets. We would need to make +// the start vote call kick off an asynchronous job that fetches and caches +// this data. func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload string) (string, error) { log.Tracef("cmdCastBallot: %v %x %v", treeID, token, payload) @@ -2160,7 +2171,7 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str // From this point forward, it can be assumed that all votes that // have not had their error set are voting for the same record. Get // the record and vote data that we need to perform the remaining - // inexpensive validation before we have to hold the lock. + // validation. voteDetails, err := p.voteDetails(treeID) if err != nil { return "", err @@ -2188,7 +2199,10 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str return "", fmt.Errorf("largestCommitmentAddrs: %v", err) } - // Perform validation that doesn't require holding the record lock. + // votesCache contains the tickets that have alread voted + votesCache := p.votesCache(token) + + // Perform remaining validation for k, v := range votes { if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { // Vote has an error. Skip it. @@ -2278,18 +2292,9 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str receipts[k].ErrorContext = ticketvote.VoteErrors[e] continue } - } - - // votesCache contains the tickets that have alread voted - votesCache := p.votesCache(token) - for k, v := range votes { - if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { - // Vote has an error. Skip it. - continue - } // Verify ticket has not already vote - _, ok := votesCache[v.Ticket] + _, ok = votesCache[v.Ticket] if ok { e := ticketvote.VoteErrorTicketAlreadyVoted receipts[k].Ticket = v.Ticket @@ -2310,7 +2315,7 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str // to be fully appended before considering the trillian call // successful. A person casting hundreds of votes in a single ballot // would cause UX issues for all the voting clients since the backend - // lock is held during these calls. + // lock for this record is held during these calls. // // The second variable that we must watch out for is the max trillian // queued leaf batch size. This is also a configurable trillian value @@ -2318,14 +2323,14 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str // in the queue for all trees in the trillian instance. This value is // typically around the order of magnitude of 1000 queued leaves. // - // This is why a vote batch size of 5 was chosen. It is large enough + // This is why a vote batch size of 10 was chosen. It is large enough // to alleviate performance bottlenecks from the log signer interval, // but small enough to still allow multiple records votes be held // concurrently without running into the queued leaf batch size limit. // Prepare work var ( - batchSize = 5 + batchSize = 10 batch = make([]ticketvote.CastVote, 0, batchSize) queue = make([][]ticketvote.CastVote, 0, len(votes)/batchSize) From c257ba38e75cc6bcdfaa911b544111006943b344 Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 19 Feb 2021 16:44:45 -0600 Subject: [PATCH 321/449] records/v1: Fix user error numbering. --- politeiawww/api/records/v1/v1.go | 70 ++++++++++++++++---------------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go index c5703ef9d..d6b11e45b 100644 --- a/politeiawww/api/records/v1/v1.go +++ b/politeiawww/api/records/v1/v1.go @@ -32,47 +32,45 @@ type ErrorCodeT int const ( // Error codes - ErrorCodeInvalid ErrorCodeT = 0 - ErrorCodeInputInvalid ErrorCodeT = 1 - ErrorCodeFileNameInvalid ErrorCodeT = 2 - ErrorCodeFileMIMEInvalid ErrorCodeT = 3 - ErrorCodeFileDigestInvalid ErrorCodeT = 4 - ErrorCodeFilePayloadInvalid ErrorCodeT = 5 - ErrorCodeMetadataStreamIDInvalid ErrorCodeT = 6 - ErrorCodeMetadataStreamPayloadInvalid ErrorCodeT = 8 - ErrorCodePublicKeyInvalid ErrorCodeT = 9 - ErrorCodeSignatureInvalid ErrorCodeT = 10 - ErrorCodeRecordTokenInvalid ErrorCodeT = 11 - ErrorCodeRecordStateInvalid ErrorCodeT = 12 - ErrorCodeRecordNotFound ErrorCodeT = 13 - ErrorCodeRecordLocked ErrorCodeT = 14 - ErrorCodeNoRecordChanges ErrorCodeT = 15 - ErrorCodeRecordStatusInvalid ErrorCodeT = 16 - ErrorCodeStatusReasonNotFound ErrorCodeT = 17 - ErrorCodePageSizeExceeded ErrorCodeT = 18 + ErrorCodeInvalid ErrorCodeT = 0 + ErrorCodeInputInvalid ErrorCodeT = 1 + ErrorCodeFileNameInvalid ErrorCodeT = 2 + ErrorCodeFileMIMEInvalid ErrorCodeT = 3 + ErrorCodeFileDigestInvalid ErrorCodeT = 4 + ErrorCodeFilePayloadInvalid ErrorCodeT = 5 + ErrorCodeMetadataStreamIDInvalid ErrorCodeT = 6 + ErrorCodePublicKeyInvalid ErrorCodeT = 7 + ErrorCodeSignatureInvalid ErrorCodeT = 8 + ErrorCodeRecordTokenInvalid ErrorCodeT = 9 + ErrorCodeRecordStateInvalid ErrorCodeT = 10 + ErrorCodeRecordNotFound ErrorCodeT = 11 + ErrorCodeRecordLocked ErrorCodeT = 12 + ErrorCodeNoRecordChanges ErrorCodeT = 13 + ErrorCodeRecordStatusInvalid ErrorCodeT = 14 + ErrorCodeStatusReasonNotFound ErrorCodeT = 15 + ErrorCodePageSizeExceeded ErrorCodeT = 16 ) var ( // ErrorCodes contains the human readable errors. ErrorCodes = map[ErrorCodeT]string{ - ErrorCodeInvalid: "error invalid", - ErrorCodeInputInvalid: "input invalid", - ErrorCodeFileNameInvalid: "file name invalid", - ErrorCodeFileMIMEInvalid: "file mime invalid", - ErrorCodeFileDigestInvalid: "file digest invalid", - ErrorCodeFilePayloadInvalid: "file payload invalid", - ErrorCodeMetadataStreamIDInvalid: "mdstream id invalid", - ErrorCodeMetadataStreamPayloadInvalid: "mdstream payload invalid", - ErrorCodePublicKeyInvalid: "public key invalid", - ErrorCodeSignatureInvalid: "signature invalid", - ErrorCodeRecordTokenInvalid: "record token invalid", - ErrorCodeRecordStateInvalid: "record state invalid", - ErrorCodeRecordNotFound: "record not found", - ErrorCodeRecordLocked: "record locked", - ErrorCodeNoRecordChanges: "no record changes", - ErrorCodeRecordStatusInvalid: "record status invalid", - ErrorCodeStatusReasonNotFound: "status reason not found", - ErrorCodePageSizeExceeded: "page size exceeded", + ErrorCodeInvalid: "error invalid", + ErrorCodeInputInvalid: "input invalid", + ErrorCodeFileNameInvalid: "file name invalid", + ErrorCodeFileMIMEInvalid: "file mime invalid", + ErrorCodeFileDigestInvalid: "file digest invalid", + ErrorCodeFilePayloadInvalid: "file payload invalid", + ErrorCodeMetadataStreamIDInvalid: "metadata stream id invalid", + ErrorCodePublicKeyInvalid: "public key invalid", + ErrorCodeSignatureInvalid: "signature invalid", + ErrorCodeRecordTokenInvalid: "record token invalid", + ErrorCodeRecordStateInvalid: "record state invalid", + ErrorCodeRecordNotFound: "record not found", + ErrorCodeRecordLocked: "record locked", + ErrorCodeNoRecordChanges: "no record changes", + ErrorCodeRecordStatusInvalid: "record status invalid", + ErrorCodeStatusReasonNotFound: "status reason not found", + ErrorCodePageSizeExceeded: "page size exceeded", } ) From 63095da791ac1863a1192f7f6e4a7ed674341e06 Mon Sep 17 00:00:00 2001 From: luke Date: Sat, 20 Feb 2021 17:52:27 -0600 Subject: [PATCH 322/449] ticketvote: Cache commitment addresses. --- .../tlogbe/plugins/ticketvote/activevotes.go | 301 ++++++++++++++++++ .../backend/tlogbe/plugins/ticketvote/cmds.go | 217 +++++++------ .../tlogbe/plugins/ticketvote/ticketvote.go | 38 ++- .../tlogbe/plugins/ticketvote/votes.go | 50 --- 4 files changed, 438 insertions(+), 168 deletions(-) create mode 100644 politeiad/backend/tlogbe/plugins/ticketvote/activevotes.go delete mode 100644 politeiad/backend/tlogbe/plugins/ticketvote/votes.go diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/activevotes.go b/politeiad/backend/tlogbe/plugins/ticketvote/activevotes.go new file mode 100644 index 000000000..34a1da39e --- /dev/null +++ b/politeiad/backend/tlogbe/plugins/ticketvote/activevotes.go @@ -0,0 +1,301 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ticketvote + +import ( + "encoding/hex" + "fmt" + "sync" + + "github.com/decred/politeia/politeiad/plugins/ticketvote" +) + +// activeVotes provides a memory cache for data that is required to validate +// vote ballots in a time efficient manner. An active vote is added to the +// cache when a vote is started and is removed from the cache lazily when a +// vote summary is created for the finished vote. +// +// Record locking is handled by the backend, not by individual plugins. This +// makes the plugin implementations simpler and easier to reason about, but it +// can also lead to performance bottlenecks for expensive plugin write +// commands. The cast ballot command is one such command due to a combination +// requiring external dcrdata calls to verify the largest commitment addresses +// for each ticket and the fact that its possible for hundreds of ballots to be +// cast concurrently. We cache the active vote data in order to alleviate this +// bottleneck. +type activeVotes struct { + sync.RWMutex + activeVotes map[string]activeVote // [token]activeVote +} + +// activeVote caches the data required to validate vote ballots for a record +// with an active voting period. +// +// A active vote with 41k tickets will cache a maximum of 10.5 MB of data. +// This includes a 3 MB vote details, 4.5 MB commitment addresses map, and a +// potential 3 MB cast votes map if all 41k votes are cast. +type activeVote struct { + Details *ticketvote.VoteDetails + CastVotes map[string]string // [ticket]voteBit + + // Addrs contains the largest commitment address for each eligble + // ticket. The vote must be signed with the key from this address. + // + // This map is populated by an async job that is kicked off when a + // a vote is started. It takes ~1.5 minutes to fully populate this + // cache when the ticket pool is 41k tickets and when using an off + // premise dcrdata instance with minimal latency. Any functions + // that rely of this cache should fallback to fetching the + // commitment addresses manually in the event the cache has not + // been fully populated yet or has experienced unforseen errors + // during creation (ex. network errors). If the initial job fails + // to complete it will not be retried. + Addrs map[string]string // [ticket]address +} + +// VoteDetails returns the vote details from the active votes cache for the +// provided token. If the token does not correspond to an active vote then nil +// is returned. +func (a *activeVotes) VoteDetails(token []byte) *ticketvote.VoteDetails { + a.RLock() + defer a.RUnlock() + + t := hex.EncodeToString(token) + av, ok := a.activeVotes[t] + if !ok { + return nil + } + + // Return a copy of the vote details + eligible := make([]string, len(av.Details.EligibleTickets)) + for i, v := range av.Details.EligibleTickets { + eligible[i] = v + } + options := make([]ticketvote.VoteOption, len(av.Details.Params.Options)) + for i, v := range av.Details.Params.Options { + options[i] = v + } + return &ticketvote.VoteDetails{ + Params: ticketvote.VoteParams{ + Token: av.Details.Params.Token, + Version: av.Details.Params.Version, + Type: av.Details.Params.Type, + Mask: av.Details.Params.Mask, + Duration: av.Details.Params.Duration, + QuorumPercentage: av.Details.Params.QuorumPercentage, + PassPercentage: av.Details.Params.PassPercentage, + Options: options, + Parent: av.Details.Params.Parent, + }, + PublicKey: av.Details.PublicKey, + Signature: av.Details.Signature, + StartBlockHeight: av.Details.StartBlockHeight, + StartBlockHash: av.Details.StartBlockHash, + EndBlockHeight: av.Details.EndBlockHeight, + EligibleTickets: eligible, + } +} + +// EligibleTickets returns the eligible tickets from the active votes cache for +// the provided token. If the token does not correspond to an active vote then +// nil is returned. +func (a *activeVotes) EligibleTickets(token []byte) map[string]struct{} { + a.RLock() + defer a.RUnlock() + + t := hex.EncodeToString(token) + av, ok := a.activeVotes[t] + if !ok { + return nil + } + + // Return a map of the eligible tickets + eligible := make(map[string]struct{}, len(av.Details.EligibleTickets)) + for _, v := range av.Details.EligibleTickets { + eligible[v] = struct{}{} + } + + return eligible +} + +// VoteIsDuplicate returns whether the vote has already been cast. This +// function will panic if the provided token does not correspond to a record in +// the active votes cache. +func (a *activeVotes) VoteIsDuplicate(token, ticket string) bool { + a.RLock() + defer a.RUnlock() + + av, ok := a.activeVotes[token] + if !ok { + // This should not happen + e := fmt.Errorf("active vote not found %v", token) + panic(e) + } + + _, isDup := av.CastVotes[ticket] + return isDup +} + +// CommitmentAddrs returns the largest comittment address for each of the +// provided tickets. The returned map is a map[ticket]commitmentAddr. Nil is +// returned if the provided token does not correspond to a record in the active +// votes cache. +func (a *activeVotes) CommitmentAddrs(token []byte, tickets []string) map[string]commitmentAddr { + a.RLock() + defer a.RUnlock() + + t := hex.EncodeToString(token) + av, ok := a.activeVotes[t] + if !ok { + return nil + } + + ca := make(map[string]commitmentAddr, len(tickets)) + for _, v := range tickets { + addr, ok := av.Addrs[v] + if ok { + ca[v] = commitmentAddr{ + addr: addr, + } + } + } + + return ca +} + +// Tally returns the tally of the cast votes for each vote option in an active +// vote. The returned map is a map[votebit]tally. An empty map is returned if +// the requested token is not in the active votes cache. +func (a *activeVotes) Tally(token string) map[string]uint32 { + a.RLock() + defer a.RUnlock() + + tally := make(map[string]uint32, 16) + av, ok := a.activeVotes[token] + if !ok { + return tally + } + for _, votebit := range av.CastVotes { + tally[votebit]++ + } + return tally +} + +// AddCastVote adds a cast ticket vote to the active votes cache. This function +// will panic if the provided token does not correspond to a record in the +// active votes cache. +func (a *activeVotes) AddCastVote(token, ticket, votebit string) { + a.Lock() + defer a.Unlock() + + av, ok := a.activeVotes[token] + if !ok { + // This should not happen + e := fmt.Sprintf("active vote not found %v", token) + panic(e) + } + + av.CastVotes[ticket] = votebit +} + +// AddCommitmentAddrs adds commitment addresses to the cache for a record. This +// function will panic if the provided token does not correspond to a record in +// the active votes cache. +func (a *activeVotes) AddCommitmentAddrs(token string, addrs map[string]commitmentAddr) { + a.Lock() + defer a.Unlock() + + av, ok := a.activeVotes[token] + if !ok { + // This should not happen + e := fmt.Sprintf("active vote not found %v", token) + panic(e) + } + + for ticket, v := range addrs { + if v.err != nil { + log.Errorf("Commitment address error %v %v %v", + token, ticket, v.err) + continue + } + av.Addrs[ticket] = v.addr + } +} + +// Del deletes an active vote from the active votes cache. +func (a *activeVotes) Del(token string) { + a.Lock() + defer a.Unlock() + + delete(a.activeVotes, token) + + log.Debugf("Active votes del %v", token) +} + +// Add adds a active vote to the active votes cache. +func (a *activeVotes) Add(vd ticketvote.VoteDetails) { + a.Lock() + defer a.Unlock() + + token := vd.Params.Token + a.activeVotes[token] = activeVote{ + Details: &vd, + CastVotes: make(map[string]string, 40960), // Ticket pool size + Addrs: make(map[string]string, 40960), // Ticket pool size + } + + log.Debugf("Active votes add %v", token) +} + +func newActiveVotes() *activeVotes { + return &activeVotes{ + activeVotes: make(map[string]activeVote, 256), + } +} + +func (p *ticketVotePlugin) activeVotePopulateAddrs(vd ticketvote.VoteDetails) { + // Get largest commitment address for each eligible ticket. A + // TrimmedTxs response for 500 tickets is ~1MB. It takes ~1.5 + // minutes to get the largest commitment address for 41k eligible + // tickets from an off premise dcrdata instance with minimal + // latency. + var ( + token = vd.Params.Token + pageSize = 500 + startIdx int + done bool + ) + for !done { + endIdx := startIdx + pageSize + if endIdx > len(vd.EligibleTickets) { + endIdx = len(vd.EligibleTickets) + done = true + } + + log.Debugf("Get %v commitment addrs %v/%v", + token, endIdx, len(vd.EligibleTickets)) + + tickets := vd.EligibleTickets[startIdx:endIdx] + addrs, err := p.largestCommitmentAddrs(tickets) + if err != nil { + log.Errorf("Populate commitment addresses for %v at %v: %v", + token, startIdx, err) + continue + } + + // Update cached active vote + p.activeVotes.AddCommitmentAddrs(token, addrs) + + startIdx += pageSize + } +} + +func (p *ticketVotePlugin) activeVotesAdd(vd ticketvote.VoteDetails) { + // Add the vote to the active votes cache + p.activeVotes.Add(vd) + + // Fetch the commitment addresses asynchronously + go p.activeVotePopulateAddrs(vd) +} diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go index 01117c76e..66ea61aa6 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go @@ -428,15 +428,14 @@ func (p *ticketVotePlugin) voteOptionResults(token []byte, options []ticketvote. // results using the cached votes if we can since it will be much // faster. var ( - tally = make(map[string]int, len(options)) - votes = p.votesCache(token) + tally = make(map[string]uint32, len(options)) + t = hex.EncodeToString(token) + ctally = p.activeVotes.Tally(t) ) switch { - case len(votes) > 0: - // Vote are in the cache. Tally the results. - for _, voteBit := range votes { - tally[voteBit]++ - } + case len(ctally) > 0: + // Vote are in the cache. Use the cached results. + tally = ctally default: // Votes are not in the cache. Pull them from the backend. @@ -687,9 +686,8 @@ func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) return nil, err } - // Remove record from votes cache. The votes cache is only for - // records with ongoing votes. - p.votesCacheDel(vd.Params.Token) + // Remove record from the active votes cache + p.activeVotes.Del(vd.Params.Token) case ticketvote.VoteTypeRunoff: // A runoff vote requires that we pull all other runoff vote @@ -705,9 +703,8 @@ func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) return nil, err } - // Remove record from votes cache. The votes cache is only for - // records with ongoing votes. - p.votesCacheDel(k) + // Remove record from active votes cache + p.activeVotes.Del(k) } summary = summaries[vd.Params.Token] @@ -825,12 +822,14 @@ func (p *ticketVotePlugin) bestBlockUnsafe() (uint32, error) { } type commitmentAddr struct { - ticket string // Ticket hash - addr string // Commitment address - err error // Error if one occurred + addr string // Commitment address + err error // Error if one occurred } -func (p *ticketVotePlugin) largestCommitmentAddrs(tickets []string) ([]commitmentAddr, error) { +// largestCommitmentAddrs retrieves the largest commitment addresses for each +// of the provided tickets from dcrdata. A map[ticket]commitmentAddr is +// returned. +func (p *ticketVotePlugin) largestCommitmentAddrs(tickets []string) (map[string]commitmentAddr, error) { // Get tx details tt := dcrdata.TxsTrimmed{ TxIDs: tickets, @@ -852,7 +851,7 @@ func (p *ticketVotePlugin) largestCommitmentAddrs(tickets []string) ([]commitmen } // Find the largest commitment address for each tx - addrs := make([]commitmentAddr, 0, len(ttr.Txs)) + addrs := make(map[string]commitmentAddr, len(ttr.Txs)) for _, tx := range ttr.Txs { var ( bestAddr string // Addr with largest commitment amount @@ -877,11 +876,10 @@ func (p *ticketVotePlugin) largestCommitmentAddrs(tickets []string) ([]commitmen } // Store result - addrs = append(addrs, commitmentAddr{ - ticket: tx.TxID, - addr: bestAddr, - err: addrErr, - }) + addrs[tx.TxID] = commitmentAddr{ + addr: bestAddr, + err: addrErr, + } } return addrs, nil @@ -1387,6 +1385,9 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot p.inventoryUpdateToStarted(vd.Params.Token, ticketvote.VoteStatusStarted, vd.EndBlockHeight) + // Update active votes cache + p.activeVotesAdd(vd) + return &ticketvote.StartReply{ StartBlockHeight: sr.StartBlockHeight, StartBlockHash: sr.StartBlockHash, @@ -1569,6 +1570,9 @@ func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs sta p.inventoryUpdateToStarted(vd.Params.Token, ticketvote.VoteStatusStarted, vd.EndBlockHeight) + // Update active votes cache + p.activeVotesAdd(vd) + return nil } @@ -2093,7 +2097,7 @@ func (p *ticketVotePlugin) ballot(treeID int64, votes []ticketvote.CastVote, res cvr.Receipt = cv.Receipt // Update cast votes cache - p.votesCacheSet(v.Token, v.Ticket, v.VoteBit) + p.activeVotes.AddCastVote(v.Token, v.Ticket, v.VoteBit) sendResult: // Send result back to calling function @@ -2108,17 +2112,6 @@ func (p *ticketVotePlugin) ballot(treeID int64, votes []ticketvote.CastVote, res // cmdCastBallot casts a ballot of votes. This function will not return a user // error if one occurs. It will instead return the ballot reply with the error // included in the individual cast vote reply that it applies to. -// -// NOTE: Record locking is currently handled by the backend, not by individual -// plugins. This makes the plugin implementations simpler and easier to reason -// about, but it can also lead to performance bottlenecks for expensive plugin -// write commands. This cast ballot command is one such command because of the -// external dcrdata calls that it makes to verify the largest commitment -// addresses. If this becomes an issue then we will need to cache all of the -// valid commitment addresses when we start the voting period. It takes ~1.5 -// minutes to compile this data for 41k eligible tickets. We would need to make -// the start vote call kick off an asynchronous job that fetches and caches -// this data. func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload string) (string, error) { log.Tracef("cmdCastBallot: %v %x %v", treeID, token, payload) @@ -2143,11 +2136,19 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str return string(reply), nil } - // Verify that all tokens in the ballot are valid, full length - // tokens and that they are all voting for the same record. + // Get the data that we need to validate the votes + voteDetails := p.activeVotes.VoteDetails(token) + eligible := p.activeVotes.EligibleTickets(token) + bestBlock, err := p.bestBlock() + if err != nil { + return "", err + } + + // Perform all validation that does not require fetching the + // commitment addresses. receipts := make([]ticketvote.CastVoteReply, len(votes)) for k, v := range votes { - // Verify token + // Verify token is a valid token t, err := tokenDecode(v.Token) if err != nil { e := ticketvote.VoteErrorTokenInvalid @@ -2158,7 +2159,7 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str continue } - // Verify token is the same + // Verify vote token and command token are the same if !bytes.Equal(t, token) { e := ticketvote.VoteErrorMultipleRecordVotes receipts[k].Ticket = v.Ticket @@ -2166,61 +2167,17 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str receipts[k].ErrorContext = ticketvote.VoteErrors[e] continue } - } - - // From this point forward, it can be assumed that all votes that - // have not had their error set are voting for the same record. Get - // the record and vote data that we need to perform the remaining - // validation. - voteDetails, err := p.voteDetails(treeID) - if err != nil { - return "", err - } - bestBlock, err := p.bestBlock() - if err != nil { - return "", err - } - - // eligible contains the ticket hashes of all eligble tickets. They - // are put into a map for O(n) lookups. - eligible := make(map[string]struct{}, len(voteDetails.EligibleTickets)) - for _, v := range voteDetails.EligibleTickets { - eligible[v] = struct{}{} - } - - // addrs contains the largest commitment addresses for each ticket. - // The vote must be signed using the largest commitment address. - tickets := make([]string, 0, len(cb.Ballot)) - for _, v := range cb.Ballot { - tickets = append(tickets, v.Ticket) - } - addrs, err := p.largestCommitmentAddrs(tickets) - if err != nil { - return "", fmt.Errorf("largestCommitmentAddrs: %v", err) - } - - // votesCache contains the tickets that have alread voted - votesCache := p.votesCache(token) - - // Perform remaining validation - for k, v := range votes { - if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { - // Vote has an error. Skip it. - continue - } - // Verify record vote status + // Verify vote is still active if voteDetails == nil { - // Vote has not been started yet e := ticketvote.VoteErrorVoteStatusInvalid receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e - receipts[k].ErrorContext = fmt.Sprintf("%v: vote not started", + receipts[k].ErrorContext = fmt.Sprintf("%v: vote is not active", ticketvote.VoteErrors[e]) continue } if voteHasEnded(bestBlock, voteDetails.EndBlockHeight) { - // Vote has ended e := ticketvote.VoteErrorVoteStatusInvalid receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e @@ -2249,12 +2206,72 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str continue } + // Verify ticket is eligible to vote + _, ok := eligible[v.Ticket] + if !ok { + e := ticketvote.VoteErrorTicketNotEligible + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = ticketvote.VoteErrors[e] + continue + } + + // Verify ticket has not already vote + if p.activeVotes.VoteIsDuplicate(v.Token, v.Ticket) { + e := ticketvote.VoteErrorTicketAlreadyVoted + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = ticketvote.VoteErrors[e] + continue + } + } + + // Get the largest commitment address for each ticket and verify + // that the vote was signed using the private key from this + // address. We first check the active votes cache to see if the + // commitment addresses have already been fetched. Any tickets + // that are not found in the cache are fetched manually. + tickets := make([]string, 0, len(cb.Ballot)) + for _, v := range cb.Ballot { + tickets = append(tickets, v.Ticket) + } + addrs := p.activeVotes.CommitmentAddrs(token, tickets) + notInCache := make([]string, 0, len(cb.Ballot)) + for _, v := range tickets { + _, ok := addrs[v] + if !ok { + notInCache = append(notInCache, v) + } + } + + log.Debugf("%v commitment addresses not found in cache", len(notInCache)) + + if len(notInCache) > 0 { + // Get commitment addresses from dcrdata + caddrs, err := p.largestCommitmentAddrs(tickets) + if err != nil { + return "", fmt.Errorf("largestCommitmentAddrs: %v", err) + } + + // Add addresses to the existing map + for k, v := range caddrs { + addrs[k] = v + } + } + + // Verify the signatures + for k, v := range votes { + if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { + // Vote has an error. Skip it. + continue + } + // Verify vote signature - commitmentAddr := addrs[k] - if commitmentAddr.ticket != v.Ticket { + commitmentAddr, ok := addrs[v.Ticket] + if !ok { t := time.Now().Unix() - log.Errorf("cmdCastBallot: commitment addr mismatch %v: %v %v", - t, commitmentAddr.ticket, v.Ticket) + log.Errorf("cmdCastBallot: commitment addr not found %v: %v", + t, v.Ticket) e := ticketvote.VoteErrorInternalError receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e @@ -2265,7 +2282,7 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str if commitmentAddr.err != nil { t := time.Now().Unix() log.Errorf("cmdCastBallot: commitment addr error %v: %v %v", - t, commitmentAddr.ticket, commitmentAddr.err) + t, v.Ticket, commitmentAddr.err) e := ticketvote.VoteErrorInternalError receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e @@ -2282,26 +2299,6 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str ticketvote.VoteErrors[e], err) continue } - - // Verify ticket is eligible to vote - _, ok := eligible[v.Ticket] - if !ok { - e := ticketvote.VoteErrorTicketNotEligible - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = ticketvote.VoteErrors[e] - continue - } - - // Verify ticket has not already vote - _, ok = votesCache[v.Ticket] - if ok { - e := ticketvote.VoteErrorTicketAlreadyVoted - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = ticketvote.VoteErrors[e] - continue - } } // The votes that have passed validation will be cast in batches of diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go index f848170f7..95056b0c8 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go @@ -41,11 +41,9 @@ type ticketVotePlugin struct { // prove the backend received and processed a plugin command. identity *identity.FullIdentity - // votes contains the cast votes of ongoing record votes. This - // cache is built on startup and record entries are removed once - // the vote has ended and a vote summary has been cached. - votes map[string]map[string]string // [token][ticket]voteBit - mtxVotes sync.Mutex + // activeVotes is a memeory cache that contains data required to + // validate vote ballots in a time efficient manner. + activeVotes *activeVotes // Mutexes for on-disk caches mtxInv sync.RWMutex // Vote inventory cache @@ -90,7 +88,7 @@ func (p *ticketVotePlugin) Setup() error { return fmt.Errorf("inventory: %v", err) } - // Build votes cache + // Build active votes cache log.Infof("Building votes cache") started := make([]string, 0, len(inv.Entries)) @@ -100,11 +98,34 @@ func (p *ticketVotePlugin) Setup() error { } } for _, v := range started { + // Get the vote details token, err := tokenDecode(v) if err != nil { return err } + reply, err := p.backend.VettedPluginCmd(backend.PluginActionRead, + token, ticketvote.PluginID, ticketvote.CmdDetails, "") + if err != nil { + return fmt.Errorf("VettedPluginCmd %x %v %v: %v", + token, ticketvote.PluginID, ticketvote.CmdDetails, err) + } + var dr ticketvote.DetailsReply + err = json.Unmarshal([]byte(reply), &dr) + if err != nil { + return err + } + if dr.Vote == nil { + // Something is wrong. This should not happen. + return fmt.Errorf("vote details not found for record in "+ + "started inventory %x", token) + } + + // Add active votes entry + p.activeVotesAdd(*dr.Vote) + + // Get cast votes + reply, err = p.backend.VettedPluginCmd(backend.PluginActionRead, token, ticketvote.PluginID, ticketvote.CmdResults, "") if err != nil { return fmt.Errorf("VettedPluginCmd %x %v %v: %v", @@ -116,7 +137,8 @@ func (p *ticketVotePlugin) Setup() error { return err } for _, v := range rr.Votes { - p.votesCacheSet(v.Token, v.Ticket, v.VoteBit) + // Add cast vote to the active votes cache + p.activeVotes.AddCastVote(v.Token, v.Ticket, v.VoteBit) } } @@ -302,7 +324,7 @@ func New(backend backend.Backend, tlog plugins.TlogClient, settings []backend.Pl tlog: tlog, dataDir: dataDir, identity: id, - votes: make(map[string]map[string]string), + activeVotes: newActiveVotes(), linkByPeriodMin: linkByPeriodMin, linkByPeriodMax: linkByPeriodMax, voteDurationMin: voteDurationMin, diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/votes.go b/politeiad/backend/tlogbe/plugins/ticketvote/votes.go deleted file mode 100644 index 54be48dcf..000000000 --- a/politeiad/backend/tlogbe/plugins/ticketvote/votes.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) 2020-2021 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package ticketvote - -import ( - "encoding/hex" -) - -func (p *ticketVotePlugin) votesCache(token []byte) map[string]string { - p.mtxVotes.Lock() - defer p.mtxVotes.Unlock() - - // Return a copy of the map - cv, ok := p.votes[hex.EncodeToString(token)] - if !ok { - return map[string]string{} - } - c := make(map[string]string, len(cv)) - for k, v := range cv { - c[k] = v - } - - return c -} - -func (p *ticketVotePlugin) votesCacheSet(token, ticket, voteBit string) { - p.mtxVotes.Lock() - defer p.mtxVotes.Unlock() - - _, ok := p.votes[token] - if !ok { - p.votes[token] = make(map[string]string, 40960) // Ticket pool size - } - - p.votes[token][ticket] = voteBit - - log.Debugf("Added vote to cache: %v %v %v", - token, ticket, voteBit) -} - -func (p *ticketVotePlugin) votesCacheDel(token string) { - p.mtxVotes.Lock() - defer p.mtxVotes.Unlock() - - delete(p.votes, token) - - log.Debugf("Deleted votes cache: %v", token) -} From 5ecbd2fa71f0d0370f5b52cdc1b76fc12fe25366 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 21 Feb 2021 13:08:33 -0600 Subject: [PATCH 323/449] multi: Add batched politeiad requests. --- politeiad/api/v1/v1.go | 58 ++++++- politeiad/backend/backend.go | 25 +++ politeiad/backend/gitbe/gitbe.go | 14 ++ .../tlogbe/plugins/ticketvote/activevotes.go | 4 + politeiad/backend/tlogbe/tlog/tlog.go | 56 +++++- politeiad/backend/tlogbe/tlogbe.go | 160 ++++++++++++++---- politeiad/client/politeiad.go | 66 ++++++++ politeiad/politeiad.go | 122 +++++++++++++ politeiawww/pi/process.go | 84 +++++---- 9 files changed, 501 insertions(+), 88 deletions(-) diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index 645813093..5b01ad89c 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -28,8 +28,10 @@ const ( UpdateUnvettedMetadataRoute = "/v1/updateunvettedmd/" // Update unvetted metadata UpdateVettedRoute = "/v1/updatevetted/" // Update vetted record UpdateVettedMetadataRoute = "/v1/updatevettedmd/" // Update vetted metadata - GetUnvettedRoute = "/v1/getunvetted/" // Retrieve unvetted record - GetVettedRoute = "/v1/getvetted/" // Retrieve vetted record + GetUnvettedRoute = "/v1/getunvetted/" // Get unvetted record + GetVettedRoute = "/v1/getvetted/" // Get vetted record + GetUnvettedBatchRoute = "/v1/getunvettedbatch/" // Get unvetted records + GetVettedBatchRoute = "/v1/getvettedbatch/" // Get vetted records GetUnvettedTimestampsRoute = "/v1/getunvettedts/" // Get unvetted timestamps GetVettedTimestampsRoute = "/v1/getvettedts/" // Get vetted timestamps InventoryByStatusRoute = "/v1/inventorybystatus/" @@ -39,7 +41,7 @@ const ( SetUnvettedStatusRoute = "/v1/setunvettedstatus/" // Set unvetted status SetVettedStatusRoute = "/v1/setvettedstatus/" // Set vetted status PluginCommandRoute = "/v1/plugin/" // Send a command to a plugin - PluginCommandBatchRoute = "/v1/plugin/batch" // Send a batch of plugin cmds + PluginCommandBatchRoute = "/v1/plugin/batch" // Send commands to plugins PluginInventoryRoute = "/v1/plugin/inventory/" // Inventory all plugins ChallengeSize = 32 // Size of challenge token in bytes @@ -614,8 +616,7 @@ type PluginCommandReplyV2 struct { Payload string `json:"payload"` // Response payload // UserError will be populated if a ErrorStatusT is encountered - // before the plugin command could be executed. Ex, the provided - // token does not correspond to a record. + // before the plugin command could be executed. UserError *UserErrorReply `json:"usererror,omitempty"` // PluginError will be populated if a plugin error occured during @@ -635,3 +636,50 @@ type PluginCommandBatchReply struct { Response string `json:"response"` // Challenge response Replies []PluginCommandReplyV2 `json:"replies"` } + +// RecordRequest is used to requests a record. It gives the client granular +// control over what is returned. The only required field is the token. All +// other fields are optional. +// +// Version is used to request a specific version of a record. If no version is +// provided then the most recent version of the record will be returned. +// +// OmitFiles can be used to retrieve a record without any of the record files +// being returned. +// +// Filenames can be used to request specific files. When filenames is not +// empty, the only files that are returned will be those that are specified. +type RecordRequest struct { + Token string `json:"token"` + Version string `json:"version,omitempty"` + OmitFiles bool `json:"omitfiles,omitempty"` + Filenames []string `json:"filenames,omitempty"` +} + +// GetUnvettedBatch requests a batch of unvetted records. +type GetUnvettedBatch struct { + Challenge string `json:"challenge"` // Random challenge + Requests []RecordRequest `json:"requests"` +} + +// GetUnvettedBatchReply is the reply to the GetUnvettedBatch command. If a +// record was not found or an error occured while retrieving it the token will +// not be included in the returned map. +type GetUnvettedBatchReply struct { + Response string `json:"response"` // Challenge response + Records map[string]Record `json:"record"` +} + +// GetVettedBatch requests a batch of unvetted records. +type GetVettedBatch struct { + Challenge string `json:"challenge"` // Random challenge + Requests []RecordRequest `json:"requests"` +} + +// GetVettedBatchReply is the reply to the GetVettedBatch command. If a record +// was not found or an error occured while retrieving it the token will not be +// included in the returned map. +type GetVettedBatchReply struct { + Response string `json:"response"` // Challenge response + Records map[string]Record `json:"record"` +} diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index f698c5d1c..7446f21b9 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -146,6 +146,25 @@ type Record struct { Files []File // User provided files } +// RecordRequest is used to requests a record. It gives the client granular +// control over what is returned. The only required field is the token. All +// other fields are optional. +// +// Version is used to request a specific version of a record. If no version is +// provided then the most recent version of the record will be returned. +// +// OmitFiles can be used to retrieve a record without any of the record files +// being returned. +// +// Filenames can be used to request specific files. When filenames is not +// empty, the only files that are returned will be those that are specified. +type RecordRequest struct { + Token []byte `json:"token"` + Version string `json:"version,omitempty"` + OmitFiles bool `json:"omitfiles,omitempty"` + Filenames []string `json:"filenames,omitempty"` +} + // Proof contains an inclusion proof for the digest in the merkle root. All // digests are hex encoded SHA256 digests. // @@ -293,6 +312,12 @@ type Backend interface { // Get vetted record GetVetted(token []byte, version string) (*Record, error) + // Get a batch of unvetted records + GetUnvettedBatch(reqs []RecordRequest) (map[string]Record, error) + + // Get a batch of vetted records + GetVettedBatch(reqs []RecordRequest) (map[string]Record, error) + // Get unvetted record timestamps GetUnvettedTimestamps(token []byte, version string) (*RecordTimestamps, error) diff --git a/politeiad/backend/gitbe/gitbe.go b/politeiad/backend/gitbe/gitbe.go index 93180963b..8f5286358 100644 --- a/politeiad/backend/gitbe/gitbe.go +++ b/politeiad/backend/gitbe/gitbe.go @@ -2428,6 +2428,20 @@ func (g *gitBackEnd) GetVetted(token []byte, version string) (*backend.Record, e return g.getRecordLock(token, version, g.vetted, true) } +// GetUnvettedBatch is not implemented. +// +// This function satisfies the Backend interface. +func (g *gitBackEnd) GetUnvettedBatch(reqs []backend.RecordRequest) (map[string]backend.Record, error) { + return nil, fmt.Errorf("not implemented") +} + +// GetVettedBatch is not implemented. +// +// This function satisfies the Backend interface. +func (g *gitBackEnd) GetVettedBatch(reqs []backend.RecordRequest) (map[string]backend.Record, error) { + return nil, fmt.Errorf("not implemented") +} + // GetUnvettedTimestamps is not implemented. // // This function satisfies the Backend interface. diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/activevotes.go b/politeiad/backend/tlogbe/plugins/ticketvote/activevotes.go index 34a1da39e..74b30cc69 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/activevotes.go +++ b/politeiad/backend/tlogbe/plugins/ticketvote/activevotes.go @@ -235,6 +235,10 @@ func (a *activeVotes) Del(token string) { } // Add adds a active vote to the active votes cache. +// +// This function should NOT be used directly. The activeVotesAdd function, +// which also kicks of an async job to fetch the commitment addresses for this +// active votes entry, should be used instead. func (a *activeVotes) Add(vd ticketvote.VoteDetails) { a.Lock() defer a.Unlock() diff --git a/politeiad/backend/tlogbe/tlog/tlog.go b/politeiad/backend/tlogbe/tlog/tlog.go index 50b018e60..cd10c9be3 100644 --- a/politeiad/backend/tlogbe/tlog/tlog.go +++ b/politeiad/backend/tlogbe/tlog/tlog.go @@ -947,9 +947,17 @@ printErr: return false } -func (t *Tlog) Record(treeID int64, version uint32) (*backend.Record, error) { - log.Tracef("%v record: %v %v", t.id, treeID, version) - +// record returns the specified record. +// +// Version is used to request a specific version of a record. If no version is +// provided then the most recent version of the record will be returned. +// +// OmitFiles can be used to retrieve a record without any of the record files +// being returned. +// +// Filenames can be used to request specific files. When filenames is not +// empty, the only files that are returned will be those that are specified. +func (t *Tlog) record(treeID int64, version uint32, omitFiles bool, filenames []string) (*backend.Record, error) { // Verify tree exists if !t.TreeExists(treeID) { return nil, backend.ErrRecordNotFound @@ -994,8 +1002,25 @@ func (t *Tlog) Record(treeID int64, version uint32) (*backend.Record, error) { for _, v := range idx.Metadata { merkles[hex.EncodeToString(v)] = struct{}{} } - for _, v := range idx.Files { - merkles[hex.EncodeToString(v)] = struct{}{} + switch { + case omitFiles: + // Don't include any files + case len(filenames) > 0: + // Only included the specified files + filesToInclude := make(map[string]struct{}, len(filenames)) + for _, v := range filenames { + filesToInclude[v] = struct{}{} + } + for fn, v := range idx.Files { + if _, ok := filesToInclude[fn]; ok { + merkles[hex.EncodeToString(v)] = struct{}{} + } + } + default: + // Include all files + for _, v := range idx.Files { + merkles[hex.EncodeToString(v)] = struct{}{} + } } // Walk the tree and extract the record content keys @@ -1117,9 +1142,28 @@ func (t *Tlog) Record(treeID int64, version uint32) (*backend.Record, error) { }, nil } +// Record returns the specified version of the record. +func (t *Tlog) Record(treeID int64, version uint32) (*backend.Record, error) { + log.Tracef("%v record: %v %v", t.id, treeID, version) + + return t.record(treeID, version, false, []string{}) +} + +// RecordLatest returns the latest version of a record. func (t *Tlog) RecordLatest(treeID int64) (*backend.Record, error) { log.Tracef("%v RecordLatest: %v", t.id, treeID) - return t.Record(treeID, 0) + + return t.record(treeID, 0, false, []string{}) +} + +// RecordPartial returns a partial record. This method gives the caller fine +// grained control over what version and what files are returned. The only +// required field is the token. All other fields are optional. +func (t *Tlog) RecordPartial(treeID int64, version uint32, omitFiles bool, filenames []string) (*backend.Record, error) { + log.Tracef("%v RecordPartial: %v %v %v %v", + t.id, treeID, version, omitFiles, filenames) + + return t.record(treeID, version, omitFiles, filenames) } func (t *Tlog) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trillian.LogLeaf) (*backend.Timestamp, error) { diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tlogbe/tlogbe.go index 4b2e5c7ad..f29fe797d 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tlogbe/tlogbe.go @@ -42,7 +42,7 @@ var ( _ backend.Backend = (*tlogBackend)(nil) ) -// tlogBackend implements the backend.Backend interface. +// tlogBackend implements the Backend interface. type tlogBackend struct { sync.RWMutex homeDir string @@ -513,7 +513,7 @@ func (t *tlogBackend) isShutdown() bool { // New submites a new record. Records are considered unvetted until their // status is changed to a public status. // -// This function satisfies the backend.Backend interface. +// This function satisfies the Backend interface. func (t *tlogBackend) New(metadata []backend.MetadataStream, files []backend.File) (*backend.RecordMetadata, error) { log.Tracef("New") @@ -595,7 +595,7 @@ func (t *tlogBackend) New(metadata []backend.MetadataStream, files []backend.Fil return rm, nil } -// This function satisfies the backend.Backend interface. +// This function satisfies the Backend interface. func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { log.Tracef("UpdateUnvettedRecord: %x", token) @@ -698,7 +698,7 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ return r, nil } -// This function satisfies the backend.Backend interface. +// This function satisfies the Backend interface. func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { log.Tracef("UpdateVettedRecord: %x", token) @@ -801,7 +801,7 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b return r, nil } -// This function satisfies the backend.Backend interface. +// This function satisfies the Backend interface. func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite []backend.MetadataStream) error { // Validate record contents. Send in a single metadata array to // verify there are no dups. @@ -891,7 +891,7 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite return nil } -// This function satisfies the backend.Backend interface. +// This function satisfies the Backend interface. func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite []backend.MetadataStream) error { log.Tracef("UpdateVettedMetadata: %x", token) @@ -986,7 +986,7 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ // UnvettedExists returns whether the provided token corresponds to an unvetted // record. // -// This function satisfies the backend.Backend interface. +// This function satisfies the Backend interface. func (t *tlogBackend) UnvettedExists(token []byte) bool { log.Tracef("UnvettedExists %x", token) @@ -1003,7 +1003,7 @@ func (t *tlogBackend) UnvettedExists(token []byte) bool { return t.unvetted.RecordExists(treeID) } -// This function satisfies the backend.Backend interface. +// This function satisfies the Backend interface. func (t *tlogBackend) VettedExists(token []byte) bool { log.Tracef("VettedExists %x", token) @@ -1238,7 +1238,7 @@ func (t *tlogBackend) vettedArchive(token []byte, rm backend.RecordMetadata, met return nil } -// This function satisfies the backend.Backend interface. +// This function satisfies the Backend interface. func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { log.Tracef("SetVettedStatus: %x %v (%v)", token, status, backend.MDStatus[status]) @@ -1344,7 +1344,7 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md return r, nil } -// This function satisfies the backend.Backend interface. +// This function satisfies the Backend interface. func (t *tlogBackend) GetUnvetted(token []byte, version string) (*backend.Record, error) { log.Tracef("GetUnvetted: %x %v", token, version) @@ -1362,21 +1362,19 @@ func (t *tlogBackend) GetUnvetted(token []byte, version string) (*backend.Record v = uint32(u) } - // Verify record exists and is unvetted - if !t.UnvettedExists(token) { - return nil, backend.ErrRecordNotFound - } - // Get unvetted record r, err := t.unvetted.Record(treeID, v) if err != nil { + if err == backend.ErrRecordNotFound { + return nil, err + } return nil, fmt.Errorf("unvetted record: %v", err) } return r, nil } -// This function satisfies the backend.Backend interface. +// This function satisfies the Backend interface. func (t *tlogBackend) GetVetted(token []byte, version string) (*backend.Record, error) { log.Tracef("GetVetted: %x %v", token, version) @@ -1408,7 +1406,107 @@ func (t *tlogBackend) GetVetted(token []byte, version string) (*backend.Record, return r, nil } -// This function satisfies the backend.Backend interface. +// GetUnvettedBatch returns a batch of unvetted records. +// +// This function satisfies the Backend interface. +func (t *tlogBackend) GetUnvettedBatch(reqs []backend.RecordRequest) (map[string]backend.Record, error) { + log.Tracef("GetUnvettedBatch: %v records ", len(reqs)) + + if t.isShutdown() { + return nil, backend.ErrShutdown + } + + records := make(map[string]backend.Record, len(reqs)) + for _, v := range reqs { + // Get tree ID + treeID := t.unvettedTreeIDFromToken(v.Token) + + // Parse version + var version uint32 + if v.Version != "" { + u, err := strconv.ParseUint(v.Version, 10, 64) + if err != nil { + // Not a valid version. Don't include this token in the + // reply. + continue + } + version = uint32(u) + } + + // Get the record + r, err := t.unvetted.RecordPartial(treeID, version, + v.OmitFiles, v.Filenames) + if err != nil { + if err == backend.ErrRecordNotFound { + // Record doesn't exist. This is ok. It will not be included + // in the reply. + continue + } + // An unexpected error occured. Log it and continue. + log.Debug("RecordPartial %v: %v", treeID, err) + continue + } + + // Update reply + records[r.RecordMetadata.Token] = *r + } + + return records, nil +} + +// GetVettedBatch returns a batch of unvetted records. +// +// This function satisfies the Backend interface. +func (t *tlogBackend) GetVettedBatch(reqs []backend.RecordRequest) (map[string]backend.Record, error) { + log.Tracef("GetVettedBatch: %v records ", len(reqs)) + + if t.isShutdown() { + return nil, backend.ErrShutdown + } + + records := make(map[string]backend.Record, len(reqs)) + for _, v := range reqs { + // Get tree ID + treeID, ok := t.vettedTreeIDFromToken(v.Token) + if !ok { + // Record doesn't exist + continue + } + + // Parse the version + var version uint32 + if v.Version != "" { + u, err := strconv.ParseUint(v.Version, 10, 64) + if err != nil { + // Not a valid version. Don't include this token in the + // reply. + continue + } + version = uint32(u) + } + + // Get the record + r, err := t.vetted.RecordPartial(treeID, version, + v.OmitFiles, v.Filenames) + if err != nil { + if err == backend.ErrRecordNotFound { + // Record doesn't exist. This is ok. It will not be included + // in the reply. + continue + } + // An unexpected error occured. Log it and continue. + log.Debug("RecordPartial %v: %v", treeID, err) + continue + } + + // Update reply + records[r.RecordMetadata.Token] = *r + } + + return records, nil +} + +// This function satisfies the Backend interface. func (t *tlogBackend) GetUnvettedTimestamps(token []byte, version string) (*backend.RecordTimestamps, error) { log.Tracef("GetUnvettedTimestamps: %x %v", token, version) @@ -1435,7 +1533,7 @@ func (t *tlogBackend) GetUnvettedTimestamps(token []byte, version string) (*back return t.unvetted.RecordTimestamps(treeID, v, token) } -// This function satisfies the backend.Backend interface. +// This function satisfies the Backend interface. func (t *tlogBackend) GetVettedTimestamps(token []byte, version string) (*backend.RecordTimestamps, error) { log.Tracef("GetVettedTimestamps: %x %v", token, version) @@ -1466,7 +1564,7 @@ func (t *tlogBackend) GetVettedTimestamps(token []byte, version string) (*backen // InventoryByStatus returns the record tokens of all records in the inventory // categorized by MDStatusT. // -// This function satisfies the backend.Backend interface. +// This function satisfies the Backend interface. func (t *tlogBackend) InventoryByStatus(state string, status backend.MDStatusT, pageSize, page uint32) (*backend.InventoryByStatus, error) { log.Tracef("InventoryByStatus: %v %v %v %v", state, status, pageSize, page) @@ -1483,7 +1581,7 @@ func (t *tlogBackend) InventoryByStatus(state string, status backend.MDStatusT, // RegisterUnvettedPlugin registers a plugin with the unvetted tlog instance. // -// This function satisfies the backend.Backend interface. +// This function satisfies the Backend interface. func (t *tlogBackend) RegisterUnvettedPlugin(p backend.Plugin) error { log.Tracef("RegisterUnvettedPlugin: %v", p.ID) @@ -1496,7 +1594,7 @@ func (t *tlogBackend) RegisterUnvettedPlugin(p backend.Plugin) error { // RegisterVettedPlugin has not been implemented. // -// This function satisfies the backend.Backend interface. +// This function satisfies the Backend interface. func (t *tlogBackend) RegisterVettedPlugin(p backend.Plugin) error { log.Tracef("RegisterVettedPlugin: %v", p.ID) @@ -1510,7 +1608,7 @@ func (t *tlogBackend) RegisterVettedPlugin(p backend.Plugin) error { // SetupUnvettedPlugin performs plugin setup for a previously registered // unvetted plugin. // -// This function satisfies the backend.Backend interface. +// This function satisfies the Backend interface. func (t *tlogBackend) SetupUnvettedPlugin(pluginID string) error { log.Tracef("SetupUnvettedPlugin: %v", pluginID) @@ -1524,7 +1622,7 @@ func (t *tlogBackend) SetupUnvettedPlugin(pluginID string) error { // SetupVettedPlugin performs plugin setup for a previously registered vetted // plugin. // -// This function satisfies the backend.Backend interface. +// This function satisfies the Backend interface. func (t *tlogBackend) SetupVettedPlugin(pluginID string) error { log.Tracef("SetupVettedPlugin: %v", pluginID) @@ -1622,7 +1720,7 @@ func (t *tlogBackend) unvettedPluginWrite(token []byte, pluginID, cmd, payload s // UnvettedPluginCmd executes a plugin command on an unvetted record. // -// This function satisfies the backend.Backend interface. +// This function satisfies the Backend interface. func (t *tlogBackend) UnvettedPluginCmd(action string, token []byte, pluginID, cmd, payload string) (string, error) { log.Tracef("UnvettedPluginCmd: %v %x %v %v", action, token, pluginID, cmd) @@ -1736,7 +1834,7 @@ func (t *tlogBackend) vettedPluginWrite(token []byte, pluginID, cmd, payload str // VettedPluginCmd executes a plugin command on an unvetted record. // -// This function satisfies the backend.Backend interface. +// This function satisfies the Backend interface. func (t *tlogBackend) VettedPluginCmd(action string, token []byte, pluginID, cmd, payload string) (string, error) { log.Tracef("VettedPluginCmd: %v %x %v %v", action, token, pluginID, cmd) @@ -1768,7 +1866,7 @@ func (t *tlogBackend) VettedPluginCmd(action string, token []byte, pluginID, cmd // GetUnvettedPlugins returns the unvetted plugins that have been registered. // -// This function satisfies the backend.Backend interface. +// This function satisfies the Backend interface. func (t *tlogBackend) GetUnvettedPlugins() []backend.Plugin { log.Tracef("GetUnvettedPlugins") @@ -1777,7 +1875,7 @@ func (t *tlogBackend) GetUnvettedPlugins() []backend.Plugin { // GetVettedPlugins returns the vetted plugins that have been registered. // -// This function satisfies the backend.Backend interface. +// This function satisfies the Backend interface. func (t *tlogBackend) GetVettedPlugins() []backend.Plugin { log.Tracef("GetVettedPlugins") @@ -1786,28 +1884,28 @@ func (t *tlogBackend) GetVettedPlugins() []backend.Plugin { // Inventory has been DEPRECATED. // -// This function satisfies the backend.Backend interface. +// This function satisfies the Backend interface. func (t *tlogBackend) Inventory(vettedCount, vettedStart, unvettedCount uint, includeFiles, allVersions bool) ([]backend.Record, []backend.Record, error) { return nil, nil, fmt.Errorf("not implemented") } // GetPlugins has been DEPRECATED. // -// This function satisfies the backend.Backend interface. +// This function satisfies the Backend interface. func (t *tlogBackend) GetPlugins() ([]backend.Plugin, error) { return nil, fmt.Errorf("not implemented") } // Plugin has been DEPRECATED. // -// This function satisfies the backend.Backend interface. +// This function satisfies the Backend interface. func (t *tlogBackend) Plugin(pluginID, cmd, cmdID, payload string) (string, error) { return "", fmt.Errorf("not implemented") } // Close shuts the backend down and performs cleanup. // -// This function satisfies the backend.Backend interface. +// This function satisfies the Backend interface. func (t *tlogBackend) Close() { log.Tracef("Close") diff --git a/politeiad/client/politeiad.go b/politeiad/client/politeiad.go index e0faf2a2e..cf47ac7c6 100644 --- a/politeiad/client/politeiad.go +++ b/politeiad/client/politeiad.go @@ -306,6 +306,72 @@ func (c *Client) GetVetted(ctx context.Context, token, version string) (*pdv1.Re return &gvr.Record, nil } +// GetUnvettedBatch sends a GetUnvettedBatch request to the politeiad v1 API. +func (c *Client) GetUnvettedBatch(ctx context.Context, reqs []pdv1.RecordRequest) (map[string]pdv1.Record, error) { + // Setup request + challenge, err := util.Random(pdv1.ChallengeSize) + if err != nil { + return nil, err + } + gub := pdv1.GetUnvettedBatch{ + Challenge: hex.EncodeToString(challenge), + Requests: reqs, + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, + pdv1.GetUnvettedBatchRoute, gub) + if err != nil { + return nil, err + } + + // Decode reply + var r pdv1.GetUnvettedBatchReply + err = json.Unmarshal(resBody, &r) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(c.pid, challenge, r.Response) + if err != nil { + return nil, err + } + + return r.Records, nil +} + +// GetVettedBatch sends a GetVettedBatch request to the politeiad v1 API. +func (c *Client) GetVettedBatch(ctx context.Context, reqs []pdv1.RecordRequest) (map[string]pdv1.Record, error) { + // Setup request + challenge, err := util.Random(pdv1.ChallengeSize) + if err != nil { + return nil, err + } + gvb := pdv1.GetVettedBatch{ + Challenge: hex.EncodeToString(challenge), + Requests: reqs, + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, + pdv1.GetVettedBatchRoute, gvb) + if err != nil { + return nil, err + } + + // Decode reply + var r pdv1.GetVettedBatchReply + err = json.Unmarshal(resBody, &r) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(c.pid, challenge, r.Response) + if err != nil { + return nil, err + } + + return r.Records, nil +} + // GetUnvettedTimestamps sends a GetUnvettedTimestamps request to the politeiad // v1 API. func (c *Client) GetUnvettedTimestamps(ctx context.Context, token, version string) (*pdv1.RecordTimestamps, error) { diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index c4ee84789..69da3c3c3 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -623,6 +623,124 @@ func (p *politeia) getVetted(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, reply) } +func convertFrontendRecordRequest(r v1.RecordRequest) backend.RecordRequest { + token, _ := util.ConvertStringToken(r.Token) + return backend.RecordRequest{ + Token: token, + Version: r.Version, + OmitFiles: r.OmitFiles, + Filenames: r.Filenames, + } +} + +func (p *politeia) convertBackendRecords(br map[string]backend.Record) map[string]v1.Record { + r := make(map[string]v1.Record, len(br)) + for k, v := range br { + r[k] = p.convertBackendRecord(v) + } + return r +} + +func (p *politeia) getUnvettedBatch(w http.ResponseWriter, r *http.Request) { + var gub v1.GetUnvettedBatch + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&gub); err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) + return + } + + challenge, err := hex.DecodeString(gub.Challenge) + if err != nil || len(challenge) != v1.ChallengeSize { + p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) + return + } + + // Setup backend requests + reqs := make([]backend.RecordRequest, 0, len(gub.Requests)) + for _, v := range gub.Requests { + // Verify token + _, err := util.ConvertStringToken(v.Token) + if err != nil { + // Not a valid token. Do not include it in the reply. + continue + } + reqs = append(reqs, convertFrontendRecordRequest(v)) + } + + // Get records batch + records, err := p.backend.GetUnvettedBatch(reqs) + if err != nil { + // Generic internal error + errorCode := time.Now().Unix() + log.Errorf("%v Get unvetted batch error code %v: %v", + remoteAddr(r), errorCode, err) + p.respondWithServerError(w, errorCode) + return + } + + log.Infof("%v Get unvetted batch %v/%v found", + remoteAddr(r), len(records), len(gub.Requests)) + + // Prepare reply + response := p.identity.SignMessage(challenge) + reply := v1.GetUnvettedBatchReply{ + Response: hex.EncodeToString(response[:]), + Records: p.convertBackendRecords(records), + } + + util.RespondWithJSON(w, http.StatusOK, reply) +} + +func (p *politeia) getVettedBatch(w http.ResponseWriter, r *http.Request) { + var gvb v1.GetVettedBatch + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&gvb); err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) + return + } + + challenge, err := hex.DecodeString(gvb.Challenge) + if err != nil || len(challenge) != v1.ChallengeSize { + p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) + return + } + + // Setup backend requests + reqs := make([]backend.RecordRequest, 0, len(gvb.Requests)) + for _, v := range gvb.Requests { + // Verify token + _, err := util.ConvertStringToken(v.Token) + if err != nil { + // Not a valid token. Do not include it in the reply. + continue + } + reqs = append(reqs, convertFrontendRecordRequest(v)) + } + + // Get records batch + records, err := p.backend.GetVettedBatch(reqs) + if err != nil { + // Generic internal error + errorCode := time.Now().Unix() + log.Errorf("%v Get vetted batch error code %v: %v", + remoteAddr(r), errorCode, err) + p.respondWithServerError(w, errorCode) + return + } + + log.Infof("%v Get vetted batch %v/%v found", + remoteAddr(r), len(records), len(gvb.Requests)) + + // Prepare reply + response := p.identity.SignMessage(challenge) + reply := v1.GetVettedBatchReply{ + Response: hex.EncodeToString(response[:]), + Records: p.convertBackendRecords(records), + } + + util.RespondWithJSON(w, http.StatusOK, reply) +} + func (p *politeia) getUnvettedTimestamps(w http.ResponseWriter, r *http.Request) { var t v1.GetUnvettedTimestamps decoder := json.NewDecoder(r.Body) @@ -1522,6 +1640,10 @@ func _main() error { permissionPublic) p.addRoute(http.MethodPost, v1.GetVettedRoute, p.getVetted, permissionPublic) + p.addRoute(http.MethodPost, v1.GetUnvettedBatchRoute, + p.getUnvettedBatch, permissionPublic) + p.addRoute(http.MethodPost, v1.GetVettedBatchRoute, + p.getVettedBatch, permissionPublic) p.addRoute(http.MethodPost, v1.GetUnvettedTimestampsRoute, p.getUnvettedTimestamps, permissionPublic) p.addRoute(http.MethodPost, v1.GetVettedTimestampsRoute, diff --git a/politeiawww/pi/process.go b/politeiawww/pi/process.go index 5d5307829..6dc772f60 100644 --- a/politeiawww/pi/process.go +++ b/politeiawww/pi/process.go @@ -21,21 +21,19 @@ import ( "github.com/google/uuid" ) -// proposal returns a version of a proposal record from politeiad. If version -// is an empty string then the most recent version will be returned. -func (p *Pi) proposal(ctx context.Context, state, token, version string) (*v1.Proposal, error) { +func (p *Pi) proposals(ctx context.Context, state string, reqs []pdv1.RecordRequest) (map[string]v1.Proposal, error) { var ( - r *pdv1.Record - err error + records map[string]pdv1.Record + err error ) switch state { case v1.ProposalStateUnvetted: - r, err = p.politeiad.GetUnvetted(ctx, token, version) + records, err = p.politeiad.GetUnvettedBatch(ctx, reqs) if err != nil { return nil, err } case v1.ProposalStateVetted: - r, err = p.politeiad.GetVetted(ctx, token, version) + records, err = p.politeiad.GetVettedBatch(ctx, reqs) if err != nil { return nil, err } @@ -43,22 +41,27 @@ func (p *Pi) proposal(ctx context.Context, state, token, version string) (*v1.Pr return nil, fmt.Errorf("invalid state %v", state) } - // Convert to a proposal - pr, err := convertRecord(*r, state) - if err != nil { - return nil, err - } + proposals := make(map[string]v1.Proposal, len(records)) + for k, v := range records { + // Convert to a proposal + pr, err := convertRecord(v, state) + if err != nil { + return nil, err + } - // Fill in user data - userID := userIDFromMetadataStreams(r.Metadata) - uid, err := uuid.Parse(userID) - u, err := p.userdb.UserGetById(uid) - if err != nil { - return nil, err + // Fill in user data + userID := userIDFromMetadataStreams(v.Metadata) + uid, err := uuid.Parse(userID) + u, err := p.userdb.UserGetById(uid) + if err != nil { + return nil, err + } + proposalPopulateUserData(pr, *u) + + proposals[k] = *pr } - proposalPopulateUserData(pr, *u) - return pr, nil + return proposals, nil } func (p *Pi) processProposals(ctx context.Context, ps v1.Proposals, u *user.User) (*v1.ProposalsReply, error) { @@ -83,34 +86,23 @@ func (p *Pi) processProposals(ctx context.Context, ps v1.Proposals, u *user.User } } - // Get all proposals in the batch. This should be a batched call to - // politeiad, but the politeiad API does not provided a batched - // records endpoint. - proposals := make(map[string]v1.Proposal, len(ps.Tokens)) + // Setup record requests. We don't retreive any index files or + // attachment files in order to keep the payload size minimal. + reqs := make([]pdv1.RecordRequest, 0, len(ps.Tokens)) for _, v := range ps.Tokens { - pr, err := p.proposal(ctx, ps.State, v, "") - if err != nil { - // If any error occured simply skip this proposal. It will not - // be included in the reply. - continue - } - - // The only files that are returned in this call are the - // ProposalMetadata and the VoteMetadata files. - files := make([]v1.File, 0, len(pr.Files)) - for k := range pr.Files { - switch pr.Files[k].Name { - case v1.FileNameProposalMetadata, v1.FileNameVoteMetadata: - // Include file - files = append(files, pr.Files[k]) - default: - // All other files are disregarded. Do nothing. - } - } - - pr.Files = files + reqs = append(reqs, pdv1.RecordRequest{ + Token: v, + Filenames: []string{ + v1.FileNameProposalMetadata, + v1.FileNameVoteMetadata, + }, + }) + } - proposals[pr.CensorshipRecord.Token] = *pr + // Get proposals + proposals, err := p.proposals(ctx, ps.State, reqs) + if err != nil { + return nil, err } // Only admins and the proposal author are allowed to retrieve From a2879e0372483ad84a60faadd46eca535e6c5b79 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 21 Feb 2021 14:56:28 -0600 Subject: [PATCH 324/449] pictl: Seed proposals perf improvements. --- politeiawww/cmd/pictl/cmdseedproposals.go | 64 +++++++++++++++++------ 1 file changed, 48 insertions(+), 16 deletions(-) diff --git a/politeiawww/cmd/pictl/cmdseedproposals.go b/politeiawww/cmd/pictl/cmdseedproposals.go index 46d6c478b..23cd4f5a5 100644 --- a/politeiawww/cmd/pictl/cmdseedproposals.go +++ b/politeiawww/cmd/pictl/cmdseedproposals.go @@ -35,13 +35,11 @@ type cmdSeedProposals struct { // // This function satisfies the go-flags Commander interface. func (c *cmdSeedProposals) Execute(args []string) error { - fmt.Printf("Warn: this cmd should be run on a clean politeia instance\n") - // Setup default parameters var ( userCount = 10 proposalCount = 25 - commentsPerProposal = 100 + commentsPerProposal = 150 commentSize = 32 // In characters commentVotesPerProposal = 500 @@ -69,7 +67,7 @@ func (c *cmdSeedProposals) Execute(args []string) error { return fmt.Errorf("user count must be >= 2") } - // Verify the the provided login credentials are for an admin. + // Verify admin login credentials admin := user{ Email: c.Args.AdminEmail, Password: c.Args.AdminPassword, @@ -87,7 +85,7 @@ func (c *cmdSeedProposals) Execute(args []string) error { } admin.Username = lr.Username - // Verify that the paywall is disabled. + // Verify paywall is disabled policyWWW, err := client.Policy() if err != nil { return err @@ -96,6 +94,9 @@ func (c *cmdSeedProposals) Execute(args []string) error { return fmt.Errorf("paywall is not disabled") } + // Log start time + fmt.Printf("Start time: %v\n", timestampFromUnix(time.Now().Unix())) + // Setup users users := make([]user, 0, userCount) for i := 0; i < userCount; i++ { @@ -111,9 +112,6 @@ func (c *cmdSeedProposals) Execute(args []string) error { } fmt.Printf("\n") - // Log start time - fmt.Printf("Start time: %v\n", timestampFromUnix(time.Now().Unix())) - // Setup proposals var ( statusUnreviewed = "unreviewed" @@ -284,6 +282,14 @@ func (c *cmdSeedProposals) Execute(args []string) error { users1 := users[:len(users)/2] users2 := users[len(users)/2:] + // Reverse the ordering of the public records so that comments are + // added to the most recent record first. + reverse := make([]string, 0, len(public)) + for i := len(public) - 1; i >= 0; i-- { + reverse = append(reverse, public[i]) + } + public = reverse + // Setup comments for i, token := range public { for j := 0; j < commentsPerProposal; j++ { @@ -291,9 +297,16 @@ func (c *cmdSeedProposals) Execute(args []string) error { "comment %v/%v", i+1, len(public), j+1, commentsPerProposal) printInPlace(log) - // Select a random user - r := rand.Intn(len(users1)) - u := users1[r] + // Login a new, random user every 10 comments. Selecting a + // new user every comment is too slow. + if j%10 == 0 { + // Select a random user + r := rand.Intn(len(users1)) + u := users1[r] + + // Login user + userLogin(u) + } // Every 5th comment should be the start of a comment thread, not // a reply. All other comments should be replies to a random @@ -315,9 +328,13 @@ func (c *cmdSeedProposals) Execute(args []string) error { comment := hex.EncodeToString(b) // Submit comment - err = commentNew(u, token, comment, parentID) + c := cmdCommentNew{} + c.Args.Token = token + c.Args.Comment = comment + c.Args.ParentID = parentID + err = c.Execute(nil) if err != nil { - return err + return fmt.Errorf("cmdCommentNew: %v", err) } } } @@ -335,8 +352,9 @@ func (c *cmdSeedProposals) Execute(args []string) error { // to vote on comments randomly can cause max vote changes // exceeded errors. var ( - userIdx int - commentID uint32 = 1 + userIdx int + needToLogin bool = true + commentID uint32 = 1 ) for j := 0; j < commentVotesPerProposal; j++ { log := fmt.Sprintf("Submitting comment votes for proposal %v/%v, "+ @@ -349,12 +367,22 @@ func (c *cmdSeedProposals) Execute(args []string) error { // with a different user. userIdx++ commentID = 1 + + userLogout() + needToLogin = true } if userIdx == len(users2) { // We've reached the end of the users. Start back over. userIdx = 0 + userLogout() + needToLogin = true } + u := users2[userIdx] + if needToLogin { + userLogin(u) + needToLogin = false + } // Select a random vote preference var vote string @@ -365,7 +393,11 @@ func (c *cmdSeedProposals) Execute(args []string) error { } // Cast comment vote - err := commentVote(u, token, commentID, vote) + c := cmdCommentVote{} + c.Args.Token = token + c.Args.CommentID = commentID + c.Args.Vote = vote + err = c.Execute(nil) if err != nil { return err } From b6655191fe393e8934ca6ef27bfc7fb6ca25e90b Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 21 Feb 2021 18:50:42 -0600 Subject: [PATCH 325/449] Add legacy support for www active votes. --- politeiad/client/ticketvote.go | 4 +- politeiawww/api/www/v1/v1.go | 2 +- politeiawww/cmd/cmswww/cmswww.go | 2 +- politeiawww/cmd/pictl/cmdactivevotes.go | 32 ++ politeiawww/cmd/pictl/cmdseedproposals.go | 10 +- politeiawww/cmd/pictl/pictl.go | 4 + .../cmd/politeiavoter/politeiavoter.go | 94 ---- politeiawww/cmd/shared/client.go | 27 ++ .../cmd/{cmswww => shared}/tokeninventory.go | 6 +- politeiawww/proposals.go | 442 ++++++++++++++---- politeiawww/ticketvote/process.go | 4 +- 11 files changed, 417 insertions(+), 210 deletions(-) create mode 100644 politeiawww/cmd/pictl/cmdactivevotes.go rename politeiawww/cmd/{cmswww => shared}/tokeninventory.go (81%) diff --git a/politeiad/client/ticketvote.go b/politeiad/client/ticketvote.go index 701e54f3f..aafc02c73 100644 --- a/politeiad/client/ticketvote.go +++ b/politeiad/client/ticketvote.go @@ -304,7 +304,7 @@ func (c *Client) TicketVoteSummaries(ctx context.Context, tokens []string) (map[ // TicketVoteSubmissions sends the ticketvote plugin Submissions command to the // politeiad v1 API. -func (c *Client) TicketVoteSubmissions(ctx context.Context, token string) (*ticketvote.SubmissionsReply, error) { +func (c *Client) TicketVoteSubmissions(ctx context.Context, token string) ([]string, error) { // Setup request cmds := []pdv1.PluginCommandV2{ { @@ -338,7 +338,7 @@ func (c *Client) TicketVoteSubmissions(ctx context.Context, token string) (*tick return nil, err } - return &sr, nil + return sr.Submissions, nil } // TicketVoteInventory sends the ticketvote plugin Inventory command to the diff --git a/politeiawww/api/www/v1/v1.go b/politeiawww/api/www/v1/v1.go index 6302108f4..16c3aa94a 100644 --- a/politeiawww/api/www/v1/v1.go +++ b/politeiawww/api/www/v1/v1.go @@ -1032,7 +1032,7 @@ type ProposalVoteTuple struct { // ActiveVoteReply returns all proposals that have active votes. // -// This request is NO LONGER SUPPORTED. +// This request is DEPRECATED. type ActiveVoteReply struct { Votes []ProposalVoteTuple `json:"votes"` // Active votes } diff --git a/politeiawww/cmd/cmswww/cmswww.go b/politeiawww/cmd/cmswww/cmswww.go index 56dd6016f..aaea35d7d 100644 --- a/politeiawww/cmd/cmswww/cmswww.go +++ b/politeiawww/cmd/cmswww/cmswww.go @@ -99,7 +99,7 @@ type cmswww struct { StartVote StartVoteCmd `command:"startvote" description:"(admin) start the voting period on a dcc"` SupportOpposeDCC SupportOpposeDCCCmd `command:"supportopposedcc" description:"(user) support or oppose a given DCC"` TestRun TestRunCmd `command:"testrun" description:" test cmswww routes"` - TokenInventory TokenInventoryCmd `command:"tokeninventory" description:"(user) get the censorship record tokens of all proposals (passthrough)"` + TokenInventory shared.TokenInventoryCmd `command:"tokeninventory" description:"(user) get the censorship record tokens of all proposals (passthrough)"` UpdateUserKey shared.UserKeyUpdateCmd `command:"updateuserkey" description:"(user) generate a new identity for the logged in user"` UserDetails UserDetailsCmd `command:"userdetails" description:"(user) get current cms user details"` UserInvoices UserInvoicesCmd `command:"userinvoices" description:"(user) get all invoices submitted by a specific user"` diff --git a/politeiawww/cmd/pictl/cmdactivevotes.go b/politeiawww/cmd/pictl/cmdactivevotes.go new file mode 100644 index 000000000..7752e5f90 --- /dev/null +++ b/politeiawww/cmd/pictl/cmdactivevotes.go @@ -0,0 +1,32 @@ +// Copyright (c) 2017-2019 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +// cmdActiveVotes retreives all proposals that are currently being voted on. +type cmdActiveVotes struct{} + +// Execute executes the active votes command. +func (cmd *cmdActiveVotes) Execute(args []string) error { + // Send request + avr, err := client.ActiveVotes() + if err != nil { + return err + } + + // Remove the ticket snapshots from the response so that the + // output is legible + if !cfg.RawJSON { + for k := range avr.Votes { + avr.Votes[k].StartVoteReply.EligibleTickets = []string{ + "removed by politeiawwwcli for readability", + } + } + } + + // Print response details + printJSON(avr) + + return nil +} diff --git a/politeiawww/cmd/pictl/cmdseedproposals.go b/politeiawww/cmd/pictl/cmdseedproposals.go index 23cd4f5a5..6d97670db 100644 --- a/politeiawww/cmd/pictl/cmdseedproposals.go +++ b/politeiawww/cmd/pictl/cmdseedproposals.go @@ -23,7 +23,8 @@ type cmdSeedProposals struct { AdminPassword string `positional-arg-name:"adminpassword" required:"true"` Users int `positional-arg-name:"users"` Proposals int `positional-arg-name:"proposals"` - CommentVotes int `positional-arg-name:"commentvotes"` + Comments *int `positional-arg-name:"comments"` + CommentVotes *int `positional-arg-name:"commentvotes"` } `positional-args:"true"` // IncludeImages is used to include a random number of images when @@ -51,8 +52,11 @@ func (c *cmdSeedProposals) Execute(args []string) error { if c.Args.Proposals != 0 { proposalCount = c.Args.Proposals } - if c.Args.CommentVotes != 0 { - commentVotesPerProposal = c.Args.CommentVotes + if c.Args.Comments != nil { + commentsPerProposal = *c.Args.Comments + } + if c.Args.CommentVotes != nil { + commentVotesPerProposal = *c.Args.CommentVotes } // We don't want the output of individual commands printed. diff --git a/politeiawww/cmd/pictl/pictl.go b/politeiawww/cmd/pictl/pictl.go index abcc3ff02..b53f67bb6 100644 --- a/politeiawww/cmd/pictl/pictl.go +++ b/politeiawww/cmd/pictl/pictl.go @@ -105,6 +105,10 @@ type pictl struct { SeedProposals cmdSeedProposals `command:"seedproposals"` VoteTestSetup cmdVoteTestSetup `command:"votetestsetup"` VoteTest cmdVoteTest `command:"votetest"` + + // Legacy www routes (deprecated) + TokenInventory shared.TokenInventoryCmd `command:"tokeninventory"` + ActiveVotes cmdActiveVotes `command:"activevotes"` } const helpMsg = `Application Options: diff --git a/politeiawww/cmd/politeiavoter/politeiavoter.go b/politeiawww/cmd/politeiavoter/politeiavoter.go index 5ffe9d0b8..b6c1ae9df 100644 --- a/politeiawww/cmd/politeiavoter/politeiavoter.go +++ b/politeiawww/cmd/politeiavoter/politeiavoter.go @@ -1332,96 +1332,6 @@ func (c *ctx) tally(args []string) error { return nil } -func (c *ctx) login(email, password string) (*v1.LoginReply, error) { - l := v1.Login{ - Email: email, - Password: password, - } - - responseBody, err := c.makeRequest(http.MethodPost, v1.RouteLogin, l) - if err != nil { - return nil, err - } - - var lr v1.LoginReply - err = json.Unmarshal(responseBody, &lr) - if err != nil { - return nil, fmt.Errorf("Could not unmarshal LoginReply: %v", - err) - } - - return &lr, nil -} - -func (c *ctx) _startVote(sv *v1.StartVote) (*v1.StartVoteReply, error) { - responseBody, err := c.makeRequest(http.MethodPost, v1.RouteStartVote, sv) - if err != nil { - return nil, err - } - - var svr v1.StartVoteReply - err = json.Unmarshal(responseBody, &svr) - if err != nil { - return nil, fmt.Errorf("Could not unmarshal StartVoteReply: %v", - err) - } - - return &svr, nil -} - -func (c *ctx) startVote(args []string) error { - if len(args) != 4 { - return fmt.Errorf("startvote: not enough arguments, expected:" + - "identityfile email password token") - } - - // startvote identityfile email password token - fi, err := identity.LoadFullIdentity(args[0]) - if err != nil { - return err - } - - // Login as admin - lr, err := c.login(args[1], args[2]) - if err != nil { - return err - } - if !lr.IsAdmin { - return fmt.Errorf("user is not an admin") - } - - sv := v1.StartVote{ - PublicKey: hex.EncodeToString(c.id.Key[:]), - Vote: v1.Vote{ - Token: args[3], - Mask: 0x03, // bit 0 no, bit 1 yes - Duration: 2016, // 1 week - Options: []v1.VoteOption{ - { - Id: "no", - Description: "Don't approve proposal", - Bits: 0x01, - }, - { - Id: "yes", - Description: "Approve proposal", - Bits: 0x02, - }, - }, - }, - } - sig := fi.SignMessage([]byte(args[1])) - sv.Signature = hex.EncodeToString(sig[:]) - - svr, err := c._startVote(&sv) - if err != nil { - return err - } - _ = svr - - return nil -} - type failedTuple struct { Time JSONTime Votes v1.Ballot `json:"votes"` @@ -1942,10 +1852,6 @@ func _main() error { switch action { case "inventory": err = c.inventory() - case "startvote": - // This remains undocumented because it is for - // testing only. - err = c.startVote(args[1:]) case "tally": err = c.tally(args[1:]) case "vote": diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index 2d1f53256..bb9340a9f 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -1468,6 +1468,33 @@ func (c *Client) GetAllVoteStatus() (*www.GetAllVoteStatusReply, error) { return &avsr, nil } +// ActiveVotes retreives the vote status of all public proposals. +func (c *Client) ActiveVotes() (*www.ActiveVoteReply, error) { + statusCode, respBody, err := c.makeRequest(http.MethodGet, + www.PoliteiaWWWAPIRoute, www.RouteActiveVote, nil) + if err != nil { + return nil, err + } + if statusCode != http.StatusOK { + return nil, wwwError(respBody, statusCode) + } + + var avr www.ActiveVoteReply + err = json.Unmarshal(respBody, &avr) + if err != nil { + return nil, err + } + + if c.cfg.Verbose { + err := prettyPrintJSON(avr) + if err != nil { + return nil, err + } + } + + return &avr, nil +} + // ActiveVotesDCC retreives all dccs that are currently being voted on. func (c *Client) ActiveVotesDCC() (*cms.ActiveVoteReply, error) { statusCode, respBody, err := c.makeRequest(http.MethodGet, diff --git a/politeiawww/cmd/cmswww/tokeninventory.go b/politeiawww/cmd/shared/tokeninventory.go similarity index 81% rename from politeiawww/cmd/cmswww/tokeninventory.go rename to politeiawww/cmd/shared/tokeninventory.go index 588567b86..ba3856ca1 100644 --- a/politeiawww/cmd/cmswww/tokeninventory.go +++ b/politeiawww/cmd/shared/tokeninventory.go @@ -2,9 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package main - -import "github.com/decred/politeia/politeiawww/cmd/shared" +package shared // TokenInventoryCmd retrieves the censorship record tokens of all proposals in // the inventory. @@ -17,5 +15,5 @@ func (cmd *TokenInventoryCmd) Execute(args []string) error { return err } - return shared.PrintJSON(reply) + return PrintJSON(reply) } diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 0ba3b3995..91325e093 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -6,17 +6,22 @@ package main import ( "context" + "encoding/base64" + "encoding/hex" "encoding/json" "errors" + "io" "net/http" "strconv" + "strings" pdv1 "github.com/decred/politeia/politeiad/api/v1" v1 "github.com/decred/politeia/politeiad/api/v1" piplugin "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/ticketvote" - tvplugin "github.com/decred/politeia/politeiad/plugins/ticketvote" + tkplugin "github.com/decred/politeia/politeiad/plugins/ticketvote" umplugin "github.com/decred/politeia/politeiad/plugins/usermd" + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/politeiawww/user" @@ -25,6 +30,77 @@ import ( "github.com/gorilla/mux" ) +func (p *politeiawww) proposal(ctx context.Context, token, version string) (*www.ProposalRecord, error) { + // Get record + r, err := p.politeiad.GetVetted(ctx, token, version) + if err != nil { + return nil, err + } + pr, err := convertRecordToProposal(*r) + if err != nil { + return nil, err + } + + // Get submissions list if this is an RFP + if pr.LinkBy != 0 { + subs, err := p.politeiad.TicketVoteSubmissions(ctx, token) + if err != nil { + return nil, err + } + pr.LinkedFrom = subs + } + + // Fill in user data + userID := userIDFromMetadataStreams(r.Metadata) + uid, err := uuid.Parse(userID) + u, err := p.db.UserGetById(uid) + if err != nil { + return nil, err + } + pr.Username = u.Username + + return pr, nil +} + +func (p *politeiawww) proposals(ctx context.Context, reqs []pdv1.RecordRequest) (map[string]www.ProposalRecord, error) { + records, err := p.politeiad.GetVettedBatch(ctx, reqs) + if err != nil { + return nil, err + } + + proposals := make(map[string]www.ProposalRecord, len(records)) + for k, v := range records { + // Convert to a proposal + pr, err := convertRecordToProposal(v) + if err != nil { + return nil, err + } + + // Get submissions list if this is an RFP + if pr.LinkBy != 0 { + subs, err := p.politeiad.TicketVoteSubmissions(ctx, + pr.CensorshipRecord.Token) + if err != nil { + return nil, err + } + pr.LinkedFrom = subs + } + + // Fill in user data + userID := userIDFromMetadataStreams(v.Metadata) + uid, err := uuid.Parse(userID) + u, err := p.db.UserGetById(uid) + if err != nil { + return nil, err + } + pr.Username = u.Username + + proposals[k] = *pr + } + + return proposals, nil +} + func (p *politeiawww) processTokenInventory(ctx context.Context, isAdmin bool) (*www.TokenInventoryReply, error) { log.Tracef("processTokenInventory") @@ -50,11 +126,11 @@ func (p *politeiawww) processTokenInventory(ctx context.Context, isAdmin bool) ( censored = ir.Unvetted[pdv1.RecordStatusCensored] // Human readable vote statuses - statusUnauth = tvplugin.VoteStatuses[tvplugin.VoteStatusUnauthorized] - statusAuth = tvplugin.VoteStatuses[tvplugin.VoteStatusAuthorized] - statusStarted = tvplugin.VoteStatuses[tvplugin.VoteStatusStarted] - statusApproved = tvplugin.VoteStatuses[tvplugin.VoteStatusApproved] - statusRejected = tvplugin.VoteStatuses[tvplugin.VoteStatusRejected] + statusUnauth = tkplugin.VoteStatuses[tkplugin.VoteStatusUnauthorized] + statusAuth = tkplugin.VoteStatuses[tkplugin.VoteStatusAuthorized] + statusStarted = tkplugin.VoteStatuses[tkplugin.VoteStatusStarted] + statusApproved = tkplugin.VoteStatuses[tkplugin.VoteStatusApproved] + statusRejected = tkplugin.VoteStatuses[tkplugin.VoteStatusRejected] // Vetted unauth = vir.Tokens[statusUnauth] @@ -68,8 +144,31 @@ func (p *politeiawww) processTokenInventory(ctx context.Context, isAdmin bool) ( // Only return unvetted tokens to admins if isAdmin { - unreviewed = nil - censored = nil + unreviewed = []string{} + censored = []string{} + } + + // Return empty arrays and not nils + if unreviewed == nil { + unreviewed = []string{} + } + if censored == nil { + censored = []string{} + } + if pre == nil { + pre = []string{} + } + if active == nil { + active = []string{} + } + if approved == nil { + approved = []string{} + } + if rejected == nil { + rejected = []string{} + } + if abandoned == nil { + abandoned = []string{} } return &www.TokenInventoryReply{ @@ -90,29 +189,6 @@ func (p *politeiawww) processAllVetted(ctx context.Context, gav www.GetAllVetted return nil, nil } -func (p *politeiawww) proposal(ctx context.Context, token, version string) (*www.ProposalRecord, error) { - // Get record - r, err := p.politeiad.GetVetted(ctx, token, version) - if err != nil { - return nil, err - } - pr, err := convertRecordToProposal(*r) - if err != nil { - return nil, err - } - - // Fill in user data - userID := userIDFromMetadataStreams(r.Metadata) - uid, err := uuid.Parse(userID) - u, err := p.db.UserGetById(uid) - if err != nil { - return nil, err - } - pr.Username = u.Username - - return pr, nil -} - func (p *politeiawww) processProposalDetails(ctx context.Context, pd www.ProposalsDetails, u *user.User) (*www.ProposalDetailsReply, error) { log.Tracef("processProposalDetails: %v", pd.Token) @@ -135,7 +211,7 @@ func (p *politeiawww) processVoteResults(ctx context.Context, token string) (*ww log.Tracef("processVoteResults: %v", token) // TODO Get vote details - var vd tvplugin.VoteDetails + var vd tkplugin.VoteDetails // Convert to www startHeight := strconv.FormatUint(uint64(vd.StartBlockHeight), 10) @@ -150,7 +226,7 @@ func (p *politeiawww) processVoteResults(ctx context.Context, token string) (*ww } // TODO Get cast votes - var rr tvplugin.ResultsReply + var rr tkplugin.ResultsReply // Convert to www votes := make([]www.CastVote, 0, len(rr.Votes)) @@ -191,7 +267,7 @@ func (p *politeiawww) processBatchVoteSummary(ctx context.Context, bvs www.Batch // TODO var bestBlock uint32 - var vs []tvplugin.SummaryReply + var vs []tkplugin.SummaryReply // Prepare reply summaries := make(map[string]www.VoteSummary, len(vs)) @@ -229,29 +305,126 @@ func (p *politeiawww) processBatchVoteSummary(ctx context.Context, bvs www.Batch } func (p *politeiawww) processActiveVote(ctx context.Context) (*www.ActiveVoteReply, error) { - // TODO - return nil, nil + // Get a page of ongoing votes. This route is deprecated and should + // be deleted before the time comes when more than a page of ongoing + // votes is required. + i := ticketvote.Inventory{} + ir, err := p.politeiad.TicketVoteInventory(ctx, i) + if err != nil { + return nil, err + } + s := ticketvote.VoteStatuses[ticketvote.VoteStatusStarted] + started := ir.Tokens[s] + + if len(started) == 0 { + // No active votes + return &www.ActiveVoteReply{ + Votes: []www.ProposalVoteTuple{}, + }, nil + } + + // Get proposals + reqs := make([]pdv1.RecordRequest, 0, len(started)) + for _, v := range started { + reqs = append(reqs, pdv1.RecordRequest{ + Token: v, + Filenames: []string{ + piplugin.FileNameProposalMetadata, + tkplugin.FileNameVoteMetadata, + }, + }) + } + props, err := p.proposals(ctx, reqs) + if err != nil { + return nil, err + } + + // Get vote details + voteDetails := make(map[string]tkplugin.VoteDetails, len(started)) + for _, v := range started { + dr, err := p.politeiad.TicketVoteDetails(ctx, v) + if err != nil { + return nil, err + } + if dr.Vote == nil { + continue + } + voteDetails[v] = *dr.Vote + } + + // Prepare reply + votes := make([]www.ProposalVoteTuple, 0, len(started)) + for _, v := range started { + var ( + proposal www.ProposalRecord + sv www.StartVote + svr www.StartVoteReply + ok bool + ) + proposal, ok = props[v] + if !ok { + continue + } + vd, ok := voteDetails[v] + if ok { + options := make([]www.VoteOption, 0, len(vd.Params.Options)) + for _, v := range vd.Params.Options { + options = append(options, www.VoteOption{ + Id: v.ID, + Description: v.Description, + Bits: v.Bit, + }) + } + sv = www.StartVote{ + Vote: www.Vote{ + Token: vd.Params.Token, + Mask: vd.Params.Mask, + Duration: vd.Params.Duration, + QuorumPercentage: vd.Params.QuorumPercentage, + PassPercentage: vd.Params.PassPercentage, + Options: options, + }, + PublicKey: vd.PublicKey, + Signature: vd.Signature, + } + svr = www.StartVoteReply{ + StartBlockHeight: strconv.FormatUint(uint64(vd.StartBlockHeight), 10), + StartBlockHash: vd.StartBlockHash, + EndHeight: strconv.FormatUint(uint64(vd.EndBlockHeight), 10), + EligibleTickets: vd.EligibleTickets, + } + } + votes = append(votes, www.ProposalVoteTuple{ + Proposal: proposal, + StartVote: sv, + StartVoteReply: svr, + }) + } + + return &www.ActiveVoteReply{ + Votes: votes, + }, nil } func (p *politeiawww) processCastVotes(ctx context.Context, ballot *www.Ballot) (*www.BallotReply, error) { log.Tracef("processCastVotes") // Prepare plugin command - votes := make([]tvplugin.CastVote, 0, len(ballot.Votes)) + votes := make([]tkplugin.CastVote, 0, len(ballot.Votes)) for _, vote := range ballot.Votes { - votes = append(votes, tvplugin.CastVote{ + votes = append(votes, tkplugin.CastVote{ Token: vote.Ticket, Ticket: vote.Ticket, VoteBit: vote.VoteBit, Signature: vote.Signature, }) } - cb := tvplugin.CastBallot{ + cb := tkplugin.CastBallot{ Ballot: votes, } // TODO _ = cb - var cbr tvplugin.CastBallotReply + var cbr tkplugin.CastBallotReply // Prepare reply receipts := make([]www.CastVoteReply, 0, len(cbr.Receipts)) @@ -502,27 +675,74 @@ func convertStatusToWWW(status pdv1.RecordStatusT) www.PropStatusT { } } -// TODO convertRecordToProposal func convertRecordToProposal(r pdv1.Record) (*www.ProposalRecord, error) { // Decode metadata - var um *umplugin.UserMetadata + var ( + um *umplugin.UserMetadata + statuses = make([]umplugin.StatusChangeMetadata, 0, 16) + ) for _, v := range r.Metadata { + if v.PluginID != umplugin.PluginID { + continue + } + + // This is a usermd plugin metadata stream switch v.ID { case umplugin.MDStreamIDUserMetadata: + var m umplugin.UserMetadata + err := json.Unmarshal([]byte(v.Payload), &m) + if err != nil { + return nil, err + } + um = &m + case umplugin.MDStreamIDStatusChanges: + d := json.NewDecoder(strings.NewReader(v.Payload)) + for { + var sc umplugin.StatusChangeMetadata + err := d.Decode(&sc) + if errors.Is(err, io.EOF) { + break + } else if err != nil { + return nil, err + } + statuses = append(statuses, sc) + } } } // Convert files var ( - pm *piplugin.ProposalMetadata - vm *tvplugin.VoteMetadata - files = make([]www.File, 0, len(r.Files)) - metadata = make([]www.Metadata, 0, len(r.Files)) + name, linkTo string + linkBy int64 + files = make([]www.File, 0, len(r.Files)) ) for _, v := range r.Files { switch v.Name { case piplugin.FileNameProposalMetadata: - case tvplugin.FileNameVoteMetadata: + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return nil, err + } + var pm piplugin.ProposalMetadata + err = json.Unmarshal(b, &pm) + if err != nil { + return nil, err + } + name = pm.Name + + case tkplugin.FileNameVoteMetadata: + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return nil, err + } + var vm tkplugin.VoteMetadata + err = json.Unmarshal(b, &vm) + if err != nil { + return nil, err + } + linkTo = vm.LinkTo + linkBy = vm.LinkBy + default: files = append(files, www.File{ Name: v.Name, @@ -533,47 +753,63 @@ func convertRecordToProposal(r pdv1.Record) (*www.ProposalRecord, error) { } } - /* - var ( - publishedAt, censoredAt, abandonedAt int64 - changeMsg string - changeMsgTimestamp int64 - ) - for _, v := range pr.Statuses { - if v.Timestamp > changeMsgTimestamp { - changeMsg = v.Reason - changeMsgTimestamp = v.Timestamp - } - switch v.Status { - case piv1.PropStatusPublic: - publishedAt = v.Timestamp - case piv1.PropStatusCensored: - censoredAt = v.Timestamp - case piv1.PropStatusAbandoned: - abandonedAt = v.Timestamp - } + // Setup user defined metadata + pm := www.ProposalMetadata{ + Name: name, + LinkTo: linkTo, + LinkBy: linkBy, + } + b, err := json.Marshal(pm) + if err != nil { + return nil, err + } + metadata := []www.Metadata{ + { + Digest: hex.EncodeToString(util.Digest(b)), + Hint: www.HintProposalMetadata, + Payload: base64.StdEncoding.EncodeToString(b), + }, + } + + var ( + publishedAt, censoredAt, abandonedAt int64 + changeMsg string + changeMsgTimestamp int64 + ) + for _, v := range statuses { + if v.Timestamp > changeMsgTimestamp { + changeMsg = v.Reason + changeMsgTimestamp = v.Timestamp } - */ + switch rcv1.RecordStatusT(v.Status) { + case rcv1.RecordStatusPublic: + publishedAt = v.Timestamp + case rcv1.RecordStatusCensored: + censoredAt = v.Timestamp + case rcv1.RecordStatusArchived: + abandonedAt = v.Timestamp + } + } return &www.ProposalRecord{ - Name: pm.Name, - State: www.PropStateVetted, - Status: convertStatusToWWW(r.Status), - Timestamp: r.Timestamp, - UserId: um.UserID, - Username: "", // Intentionally omitted - PublicKey: um.PublicKey, - Signature: um.Signature, - Version: r.Version, - // StatusChangeMessage: changeMsg, - // PublishedAt: publishedAt, - // CensoredAt: censoredAt, - // AbandonedAt: abandonedAt, - LinkTo: vm.LinkTo, - LinkBy: vm.LinkBy, - // LinkedFrom: submissions, - Files: files, - Metadata: metadata, + Name: pm.Name, + State: www.PropStateVetted, + Status: convertStatusToWWW(r.Status), + Timestamp: r.Timestamp, + UserId: um.UserID, + Username: "", // Intentionally omitted + PublicKey: um.PublicKey, + Signature: um.Signature, + Version: r.Version, + StatusChangeMessage: changeMsg, + PublishedAt: publishedAt, + CensoredAt: censoredAt, + AbandonedAt: abandonedAt, + LinkTo: pm.LinkTo, + LinkBy: pm.LinkBy, + LinkedFrom: []string{}, + Files: files, + Metadata: metadata, CensorshipRecord: www.CensorshipRecord{ Token: r.CensorshipRecord.Token, Merkle: r.CensorshipRecord.Merkle, @@ -583,51 +819,51 @@ func convertRecordToProposal(r pdv1.Record) (*www.ProposalRecord, error) { } /* -func convertVoteStatusToWWW(status tvplugin.VoteStatusT) www.PropVoteStatusT { +func convertVoteStatusToWWW(status tkplugin.VoteStatusT) www.PropVoteStatusT { switch status { - case tvplugin.VoteStatusInvalid: + case tkplugin.VoteStatusInvalid: return www.PropVoteStatusInvalid - case tvplugin.VoteStatusUnauthorized: + case tkplugin.VoteStatusUnauthorized: return www.PropVoteStatusNotAuthorized - case tvplugin.VoteStatusAuthorized: + case tkplugin.VoteStatusAuthorized: return www.PropVoteStatusAuthorized - case tvplugin.VoteStatusStarted: + case tkplugin.VoteStatusStarted: return www.PropVoteStatusStarted - case tvplugin.VoteStatusFinished: + case tkplugin.VoteStatusFinished: return www.PropVoteStatusFinished default: return www.PropVoteStatusInvalid } } -func convertVoteTypeToWWW(t tvplugin.VoteT) www.VoteT { +func convertVoteTypeToWWW(t tkplugin.VoteT) www.VoteT { switch t { - case tvplugin.VoteTypeInvalid: + case tkplugin.VoteTypeInvalid: return www.VoteTypeInvalid - case tvplugin.VoteTypeStandard: + case tkplugin.VoteTypeStandard: return www.VoteTypeStandard - case tvplugin.VoteTypeRunoff: + case tkplugin.VoteTypeRunoff: return www.VoteTypeRunoff default: return www.VoteTypeInvalid } } -func convertVoteErrorCodeToWWW(errcode tvplugin.VoteErrorT) decredplugin.ErrorStatusT { +func convertVoteErrorCodeToWWW(errcode tkplugin.VoteErrorT) decredplugin.ErrorStatusT { switch errcode { - case tvplugin.VoteErrorInvalid: + case tkplugin.VoteErrorInvalid: return decredplugin.ErrorStatusInvalid - case tvplugin.VoteErrorInternalError: + case tkplugin.VoteErrorInternalError: return decredplugin.ErrorStatusInternalError - case tvplugin.VoteErrorRecordNotFound: + case tkplugin.VoteErrorRecordNotFound: return decredplugin.ErrorStatusProposalNotFound - case tvplugin.VoteErrorVoteBitInvalid: + case tkplugin.VoteErrorVoteBitInvalid: return decredplugin.ErrorStatusInvalidVoteBit - case tvplugin.VoteErrorVoteStatusInvalid: + case tkplugin.VoteErrorVoteStatusInvalid: return decredplugin.ErrorStatusVoteHasEnded - case tvplugin.VoteErrorTicketAlreadyVoted: + case tkplugin.VoteErrorTicketAlreadyVoted: return decredplugin.ErrorStatusDuplicateVote - case tvplugin.VoteErrorTicketNotEligible: + case tkplugin.VoteErrorTicketNotEligible: return decredplugin.ErrorStatusIneligibleTicket default: return decredplugin.ErrorStatusInternalError diff --git a/politeiawww/ticketvote/process.go b/politeiawww/ticketvote/process.go index b4a46db4d..8e8b5db2d 100644 --- a/politeiawww/ticketvote/process.go +++ b/politeiawww/ticketvote/process.go @@ -185,13 +185,13 @@ func (t *TicketVote) processSummaries(ctx context.Context, s v1.Summaries) (*v1. func (t *TicketVote) processSubmissions(ctx context.Context, s v1.Submissions) (*v1.SubmissionsReply, error) { log.Tracef("processSubmissions: %v", s.Token) - sr, err := t.politeiad.TicketVoteSubmissions(ctx, s.Token) + subs, err := t.politeiad.TicketVoteSubmissions(ctx, s.Token) if err != nil { return nil, err } return &v1.SubmissionsReply{ - Submissions: sr.Submissions, + Submissions: subs, }, nil } From bd44358fa0ff13dd4d044e0e4cfbd6db82a4fc4b Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 21 Feb 2021 19:06:23 -0600 Subject: [PATCH 326/449] Add legacy support for www vote summaries. --- politeiawww/proposals.go | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 91325e093..46a2efd51 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -265,30 +265,32 @@ func (p *politeiawww) processVoteResults(ctx context.Context, token string) (*ww func (p *politeiawww) processBatchVoteSummary(ctx context.Context, bvs www.BatchVoteSummary) (*www.BatchVoteSummaryReply, error) { log.Tracef("processBatchVoteSummary: %v", bvs.Tokens) - // TODO - var bestBlock uint32 - var vs []tkplugin.SummaryReply + // Get vote summaries + vs, err := p.politeiad.TicketVoteSummaries(ctx, bvs.Tokens) + if err != nil { + return nil, err + } // Prepare reply + var bestBlock uint32 summaries := make(map[string]www.VoteSummary, len(vs)) - for _, v := range vs { - results := make([]www.VoteOptionResult, 0, len(v.Results)) - for _, r := range v.Results { - results = append(results, www.VoteOptionResult{ + for token, v := range vs { + bestBlock = v.BestBlock + results := make([]www.VoteOptionResult, len(v.Results)) + for k, r := range v.Results { + results[k] = www.VoteOptionResult{ VotesReceived: r.Votes, Option: www.VoteOption{ Id: r.ID, Description: r.Description, Bits: r.VoteBit, }, - }) + } } - // TODO - var token string summaries[token] = www.VoteSummary{ - // Status: convertVoteStatusToWWW(v.Status), - // Type: convertVoteTypeToWWW(v.Type), - // Approved: v.Approved, + Status: convertVoteStatusToWWW(v.Status), + Type: convertVoteTypeToWWW(v.Type), + Approved: v.Status == tkplugin.VoteStatusApproved, EligibleTickets: v.EligibleTickets, Duration: v.Duration, EndHeight: uint64(v.EndBlockHeight), @@ -818,7 +820,6 @@ func convertRecordToProposal(r pdv1.Record) (*www.ProposalRecord, error) { }, nil } -/* func convertVoteStatusToWWW(status tkplugin.VoteStatusT) www.PropVoteStatusT { switch status { case tkplugin.VoteStatusInvalid: @@ -831,6 +832,10 @@ func convertVoteStatusToWWW(status tkplugin.VoteStatusT) www.PropVoteStatusT { return www.PropVoteStatusStarted case tkplugin.VoteStatusFinished: return www.PropVoteStatusFinished + case tkplugin.VoteStatusApproved: + return www.PropVoteStatusFinished + case tkplugin.VoteStatusRejected: + return www.PropVoteStatusFinished default: return www.PropVoteStatusInvalid } @@ -849,6 +854,7 @@ func convertVoteTypeToWWW(t tkplugin.VoteT) www.VoteT { } } +/* func convertVoteErrorCodeToWWW(errcode tkplugin.VoteErrorT) decredplugin.ErrorStatusT { switch errcode { case tkplugin.VoteErrorInvalid: From f304e9dc68efa7c04b949b5a3d177b8c77a746ae Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 21 Feb 2021 19:17:27 -0600 Subject: [PATCH 327/449] Add legacy support for www vote results. --- politeiawww/proposals.go | 210 +++++++++++++++++++-------------------- 1 file changed, 101 insertions(+), 109 deletions(-) diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 46a2efd51..42b6224ef 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -207,61 +207,6 @@ func (p *politeiawww) processBatchProposals(ctx context.Context, bp www.BatchPro return nil, nil } -func (p *politeiawww) processVoteResults(ctx context.Context, token string) (*www.VoteResultsReply, error) { - log.Tracef("processVoteResults: %v", token) - - // TODO Get vote details - var vd tkplugin.VoteDetails - - // Convert to www - startHeight := strconv.FormatUint(uint64(vd.StartBlockHeight), 10) - endHeight := strconv.FormatUint(uint64(vd.EndBlockHeight), 10) - options := make([]www.VoteOption, 0, len(vd.Params.Options)) - for _, o := range vd.Params.Options { - options = append(options, www.VoteOption{ - Id: o.ID, - Description: o.Description, - Bits: o.Bit, - }) - } - - // TODO Get cast votes - var rr tkplugin.ResultsReply - - // Convert to www - votes := make([]www.CastVote, 0, len(rr.Votes)) - for _, v := range rr.Votes { - votes = append(votes, www.CastVote{ - Token: v.Token, - Ticket: v.Ticket, - VoteBit: v.VoteBit, - Signature: v.Signature, - }) - } - - return &www.VoteResultsReply{ - StartVote: www.StartVote{ - PublicKey: vd.PublicKey, - Signature: vd.Signature, - Vote: www.Vote{ - Token: vd.Params.Token, - Mask: vd.Params.Mask, - Duration: vd.Params.Duration, - QuorumPercentage: vd.Params.QuorumPercentage, - PassPercentage: vd.Params.PassPercentage, - Options: options, - }, - }, - StartVoteReply: www.StartVoteReply{ - StartBlockHeight: startHeight, - StartBlockHash: vd.StartBlockHash, - EndHeight: endHeight, - EligibleTickets: vd.EligibleTickets, - }, - CastVotes: votes, - }, nil -} - func (p *politeiawww) processBatchVoteSummary(ctx context.Context, bvs www.BatchVoteSummary) (*www.BatchVoteSummaryReply, error) { log.Tracef("processBatchVoteSummary: %v", bvs.Tokens) @@ -306,7 +251,40 @@ func (p *politeiawww) processBatchVoteSummary(ctx context.Context, bvs www.Batch }, nil } +func convertVoteDetails(vd tkplugin.VoteDetails) (www.StartVote, www.StartVoteReply) { + options := make([]www.VoteOption, 0, len(vd.Params.Options)) + for _, v := range vd.Params.Options { + options = append(options, www.VoteOption{ + Id: v.ID, + Description: v.Description, + Bits: v.Bit, + }) + } + sv := www.StartVote{ + Vote: www.Vote{ + Token: vd.Params.Token, + Mask: vd.Params.Mask, + Duration: vd.Params.Duration, + QuorumPercentage: vd.Params.QuorumPercentage, + PassPercentage: vd.Params.PassPercentage, + Options: options, + }, + PublicKey: vd.PublicKey, + Signature: vd.Signature, + } + svr := www.StartVoteReply{ + StartBlockHeight: strconv.FormatUint(uint64(vd.StartBlockHeight), 10), + StartBlockHash: vd.StartBlockHash, + EndHeight: strconv.FormatUint(uint64(vd.EndBlockHeight), 10), + EligibleTickets: vd.EligibleTickets, + } + + return sv, svr +} + func (p *politeiawww) processActiveVote(ctx context.Context) (*www.ActiveVoteReply, error) { + log.Tracef("processActiveVotes") + // Get a page of ongoing votes. This route is deprecated and should // be deleted before the time comes when more than a page of ongoing // votes is required. @@ -369,38 +347,13 @@ func (p *politeiawww) processActiveVote(ctx context.Context) (*www.ActiveVoteRep } vd, ok := voteDetails[v] if ok { - options := make([]www.VoteOption, 0, len(vd.Params.Options)) - for _, v := range vd.Params.Options { - options = append(options, www.VoteOption{ - Id: v.ID, - Description: v.Description, - Bits: v.Bit, - }) - } - sv = www.StartVote{ - Vote: www.Vote{ - Token: vd.Params.Token, - Mask: vd.Params.Mask, - Duration: vd.Params.Duration, - QuorumPercentage: vd.Params.QuorumPercentage, - PassPercentage: vd.Params.PassPercentage, - Options: options, - }, - PublicKey: vd.PublicKey, - Signature: vd.Signature, - } - svr = www.StartVoteReply{ - StartBlockHeight: strconv.FormatUint(uint64(vd.StartBlockHeight), 10), - StartBlockHash: vd.StartBlockHash, - EndHeight: strconv.FormatUint(uint64(vd.EndBlockHeight), 10), - EligibleTickets: vd.EligibleTickets, - } + sv, svr = convertVoteDetails(vd) + votes = append(votes, www.ProposalVoteTuple{ + Proposal: proposal, + StartVote: sv, + StartVoteReply: svr, + }) } - votes = append(votes, www.ProposalVoteTuple{ - Proposal: proposal, - StartVote: sv, - StartVoteReply: svr, - }) } return &www.ActiveVoteReply{ @@ -444,6 +397,45 @@ func (p *politeiawww) processCastVotes(ctx context.Context, ballot *www.Ballot) }, nil } +func (p *politeiawww) processVoteResults(ctx context.Context, token string) (*www.VoteResultsReply, error) { + log.Tracef("processVoteResults: %v", token) + + // Get vote details + dr, err := p.politeiad.TicketVoteDetails(ctx, token) + if err != nil { + return nil, err + } + if dr.Vote == nil { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusWrongVoteStatus, + } + } + sv, svr := convertVoteDetails(*dr.Vote) + + // Get cast votes + rr, err := p.politeiad.TicketVoteResults(ctx, token) + if err != nil { + return nil, err + } + + // Convert to www + votes := make([]www.CastVote, 0, len(rr.Votes)) + for _, v := range rr.Votes { + votes = append(votes, www.CastVote{ + Token: v.Token, + Ticket: v.Ticket, + VoteBit: v.VoteBit, + Signature: v.Signature, + }) + } + + return &www.VoteResultsReply{ + StartVote: sv, + StartVoteReply: svr, + CastVotes: votes, + }, nil +} + func (p *politeiawww) handleTokenInventory(w http.ResponseWriter, r *http.Request) { log.Tracef("handleTokenInventory") @@ -556,6 +548,29 @@ func (p *politeiawww) handleBatchProposals(w http.ResponseWriter, r *http.Reques util.RespondWithJSON(w, http.StatusOK, reply) } +func (p *politeiawww) handleBatchVoteSummary(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleBatchVoteSummary") + + var bvs www.BatchVoteSummary + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&bvs); err != nil { + RespondWithError(w, r, 0, "handleBatchVoteSummary: unmarshal", + www.UserError{ + ErrorCode: www.ErrorStatusInvalidInput, + }) + return + } + + reply, err := p.processBatchVoteSummary(r.Context(), bvs) + if err != nil { + RespondWithError(w, r, 0, + "handleBatchVoteSummary: processBatchVoteSummary %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, reply) +} + func (p *politeiawww) handleActiveVote(w http.ResponseWriter, r *http.Request) { log.Tracef("handleActiveVote") @@ -608,29 +623,6 @@ func (p *politeiawww) handleVoteResults(w http.ResponseWriter, r *http.Request) util.RespondWithJSON(w, http.StatusOK, vrr) } -func (p *politeiawww) handleBatchVoteSummary(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleBatchVoteSummary") - - var bvs www.BatchVoteSummary - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&bvs); err != nil { - RespondWithError(w, r, 0, "handleBatchVoteSummary: unmarshal", - www.UserError{ - ErrorCode: www.ErrorStatusInvalidInput, - }) - return - } - - reply, err := p.processBatchVoteSummary(r.Context(), bvs) - if err != nil { - RespondWithError(w, r, 0, - "handleBatchVoteSummary: processBatchVoteSummary %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, reply) -} - // userMetadataDecode decodes and returns the UserMetadata from the provided // metadata streams. If a UserMetadata is not found, nil is returned. func userMetadataDecode(ms []v1.MetadataStream) (*umplugin.UserMetadata, error) { From c89f48d1c2a4fe7aa88351d7361a5b8172be3270 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 21 Feb 2021 19:27:07 -0600 Subject: [PATCH 328/449] Add legacy support for www cast ballot. --- politeiawww/proposals.go | 51 ++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 42b6224ef..b04afbd1e 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -15,6 +15,7 @@ import ( "strconv" "strings" + "github.com/decred/politeia/decredplugin" pdv1 "github.com/decred/politeia/politeiad/api/v1" v1 "github.com/decred/politeia/politeiad/api/v1" piplugin "github.com/decred/politeia/politeiad/plugins/pi" @@ -364,22 +365,34 @@ func (p *politeiawww) processActiveVote(ctx context.Context) (*www.ActiveVoteRep func (p *politeiawww) processCastVotes(ctx context.Context, ballot *www.Ballot) (*www.BallotReply, error) { log.Tracef("processCastVotes") + // Verify there is work to do + if len(ballot.Votes) == 0 { + return &www.BallotReply{ + Receipts: []www.CastVoteReply{}, + }, nil + } + // Prepare plugin command votes := make([]tkplugin.CastVote, 0, len(ballot.Votes)) - for _, vote := range ballot.Votes { + var token string + for _, v := range ballot.Votes { + token = v.Token votes = append(votes, tkplugin.CastVote{ - Token: vote.Ticket, - Ticket: vote.Ticket, - VoteBit: vote.VoteBit, - Signature: vote.Signature, + Token: v.Token, + Ticket: v.Ticket, + VoteBit: v.VoteBit, + Signature: v.Signature, }) } cb := tkplugin.CastBallot{ Ballot: votes, } - // TODO - _ = cb - var cbr tkplugin.CastBallotReply + + // Send plugin command + cbr, err := p.politeiad.TicketVoteCastBallot(ctx, token, cb) + if err != nil { + return nil, err + } // Prepare reply receipts := make([]www.CastVoteReply, 0, len(cbr.Receipts)) @@ -388,7 +401,7 @@ func (p *politeiawww) processCastVotes(ctx context.Context, ballot *www.Ballot) ClientSignature: ballot.Votes[k].Signature, Signature: v.Receipt, Error: v.ErrorContext, - // ErrorStatus: convertVoteErrorCodeToWWW(v.ErrorCode), + ErrorStatus: convertVoteErrorCodeToWWW(v.ErrorCode), }) } @@ -846,25 +859,27 @@ func convertVoteTypeToWWW(t tkplugin.VoteT) www.VoteT { } } -/* -func convertVoteErrorCodeToWWW(errcode tkplugin.VoteErrorT) decredplugin.ErrorStatusT { - switch errcode { +func convertVoteErrorCodeToWWW(e tkplugin.VoteErrorT) decredplugin.ErrorStatusT { + switch e { case tkplugin.VoteErrorInvalid: return decredplugin.ErrorStatusInvalid case tkplugin.VoteErrorInternalError: return decredplugin.ErrorStatusInternalError case tkplugin.VoteErrorRecordNotFound: return decredplugin.ErrorStatusProposalNotFound - case tkplugin.VoteErrorVoteBitInvalid: - return decredplugin.ErrorStatusInvalidVoteBit + case tkplugin.VoteErrorMultipleRecordVotes: + // There is not decredplugin error code for this case tkplugin.VoteErrorVoteStatusInvalid: return decredplugin.ErrorStatusVoteHasEnded - case tkplugin.VoteErrorTicketAlreadyVoted: - return decredplugin.ErrorStatusDuplicateVote + case tkplugin.VoteErrorVoteBitInvalid: + return decredplugin.ErrorStatusInvalidVoteBit + case tkplugin.VoteErrorSignatureInvalid: + // There is not decredplugin error code for this case tkplugin.VoteErrorTicketNotEligible: return decredplugin.ErrorStatusIneligibleTicket + case tkplugin.VoteErrorTicketAlreadyVoted: + return decredplugin.ErrorStatusDuplicateVote default: - return decredplugin.ErrorStatusInternalError } + return decredplugin.ErrorStatusInternalError } -*/ From 64e92c4aa06b8b252b25510c44b1667ff843b81d Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 21 Feb 2021 19:39:08 -0600 Subject: [PATCH 329/449] Add legacy support for www proposal routes. --- politeiawww/proposals.go | 46 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index b04afbd1e..c67262aa1 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -185,6 +185,7 @@ func (p *politeiawww) processTokenInventory(ctx context.Context, isAdmin bool) ( func (p *politeiawww) processAllVetted(ctx context.Context, gav www.GetAllVetted) (*www.GetAllVettedReply, error) { log.Tracef("processAllVetted: %v %v", gav.Before, gav.After) + // TODO return nil, nil @@ -196,21 +197,58 @@ func (p *politeiawww) processProposalDetails(ctx context.Context, pd www.Proposa // This route will now only return vetted proposal. This is fine // since API consumers of this legacy route will only need public // proposals. + pr, err := p.proposal(ctx, pd.Token, pd.Version) + if err != nil { + return nil, err + } - // Remove files if the user is not an admin or the author - - return nil, nil + return &www.ProposalDetailsReply{ + Proposal: *pr, + }, nil } func (p *politeiawww) processBatchProposals(ctx context.Context, bp www.BatchProposals, u *user.User) (*www.BatchProposalsReply, error) { log.Tracef("processBatchProposals: %v", bp.Tokens) - return nil, nil + if len(bp.Tokens) > www.ProposalListPageSize { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusMaxProposalsExceededPolicy, + } + } + + reqs := make([]pdv1.RecordRequest, 0, len(bp.Tokens)) + for _, v := range bp.Tokens { + reqs = append(reqs, pdv1.RecordRequest{ + Token: v, + Filenames: []string{ + piplugin.FileNameProposalMetadata, + tkplugin.FileNameVoteMetadata, + }, + }) + } + props, err := p.proposals(ctx, reqs) + if err != nil { + return nil, err + } + prs := make([]www.ProposalRecord, 0, len(props)) + for _, v := range props { + prs = append(prs, v) + } + + return &www.BatchProposalsReply{ + Proposals: prs, + }, nil } func (p *politeiawww) processBatchVoteSummary(ctx context.Context, bvs www.BatchVoteSummary) (*www.BatchVoteSummaryReply, error) { log.Tracef("processBatchVoteSummary: %v", bvs.Tokens) + if len(bvs.Tokens) > www.ProposalListPageSize { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusMaxProposalsExceededPolicy, + } + } + // Get vote summaries vs, err := p.politeiad.TicketVoteSummaries(ctx, bvs.Tokens) if err != nil { From 27af449e604fb5f3a685b8c83f2a3ca0db722ab6 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 22 Feb 2021 09:53:28 -0600 Subject: [PATCH 330/449] Change tlogbe name to tstorebe. --- go.mod | 2 +- politeiad/api/v1/v1.go | 4 +- .../backend/{tlogbe => tstorebe}/inventory.go | 28 ++-- .../backend/{tlogbe/tlog => tstorebe}/log.go | 2 +- .../plugins/comments/cmds.go | 20 +-- .../plugins/comments/comments.go | 10 +- .../plugins/comments/comments_test.go | 12 +- .../plugins/comments/log.go | 0 .../plugins/comments/recordindex.go | 0 .../plugins/dcrdata/dcrdata.go | 2 +- .../plugins/dcrdata/log.go | 0 .../{tlogbe => tstorebe}/plugins/pi/hooks.go | 4 +- .../plugins/pi/hooks_test.go | 0 .../{tlogbe => tstorebe}/plugins/pi/log.go | 0 .../{tlogbe => tstorebe}/plugins/pi/pi.go | 2 +- .../{tlogbe => tstorebe}/plugins/plugins.go | 18 +-- .../plugins/ticketvote/activevotes.go | 0 .../plugins/ticketvote/cmds.go | 33 ++-- .../plugins/ticketvote/hooks.go | 4 +- .../plugins/ticketvote/inventory.go | 0 .../plugins/ticketvote/log.go | 0 .../plugins/ticketvote/submissions.go | 0 .../plugins/ticketvote/summary.go | 0 .../plugins/ticketvote/ticketvote.go | 8 +- .../plugins/usermd/cache.go | 0 .../plugins/usermd/cmds.go | 2 +- .../plugins/usermd/hooks.go | 4 +- .../plugins/usermd/log.go | 0 .../plugins/usermd/usermd.go | 8 +- .../{tlogbe => tstorebe}/store/fs/fs.go | 2 +- .../{tlogbe => tstorebe}/store/fs/log.go | 0 .../{tlogbe => tstorebe}/store/mysql/log.go | 0 .../{tlogbe => tstorebe}/store/mysql/mysql.go | 2 +- .../{tlogbe => tstorebe}/store/store.go | 0 .../backend/{tlogbe => tstorebe}/testing.go | 18 +-- .../tlog => tstorebe/tstore}/anchor.go | 20 +-- .../tstore/client.go} | 34 ++-- .../tlog => tstorebe/tstore}/convert.go | 4 +- .../tlog => tstorebe/tstore}/dcrtime.go | 2 +- .../tlog => tstorebe/tstore}/encryptionkey.go | 2 +- .../{tlogbe => tstorebe/tstore}/log.go | 2 +- .../tlog => tstorebe/tstore}/plugin.go | 38 ++--- .../tlog => tstorebe/tstore}/recordindex.go | 16 +- .../tlog => tstorebe/tstore}/testing.go | 26 +-- .../tstore}/trillianclient.go | 3 +- .../tlog.go => tstorebe/tstore/tstore.go} | 89 +++++----- .../tlog => tstorebe/tstore}/verify.go | 2 +- .../tstore_test.go} | 142 ++++++++-------- .../tlogbe.go => tstorebe/tstorebe.go} | 152 +++++++++--------- politeiad/config.go | 8 +- politeiad/log.go | 34 ++-- politeiad/politeiad.go | 8 +- politeiawww/cmd/pictl/cmdcommenttimestamps.go | 4 +- .../cmd/pictl/cmdproposaltimestamps.go | 4 +- politeiawww/cmd/pictl/cmdvotetimestamps.go | 4 +- util/convert.go | 4 +- util/token.go | 12 +- 57 files changed, 399 insertions(+), 396 deletions(-) rename politeiad/backend/{tlogbe => tstorebe}/inventory.go (89%) rename politeiad/backend/{tlogbe/tlog => tstorebe}/log.go (97%) rename politeiad/backend/{tlogbe => tstorebe}/plugins/comments/cmds.go (98%) rename politeiad/backend/{tlogbe => tstorebe}/plugins/comments/comments.go (92%) rename politeiad/backend/{tlogbe => tstorebe}/plugins/comments/comments_test.go (92%) rename politeiad/backend/{tlogbe => tstorebe}/plugins/comments/log.go (100%) rename politeiad/backend/{tlogbe => tstorebe}/plugins/comments/recordindex.go (100%) rename politeiad/backend/{tlogbe => tstorebe}/plugins/dcrdata/dcrdata.go (99%) rename politeiad/backend/{tlogbe => tstorebe}/plugins/dcrdata/log.go (100%) rename politeiad/backend/{tlogbe => tstorebe}/plugins/pi/hooks.go (98%) rename politeiad/backend/{tlogbe => tstorebe}/plugins/pi/hooks_test.go (100%) rename politeiad/backend/{tlogbe => tstorebe}/plugins/pi/log.go (100%) rename politeiad/backend/{tlogbe => tstorebe}/plugins/pi/pi.go (99%) rename politeiad/backend/{tlogbe => tstorebe}/plugins/plugins.go (93%) rename politeiad/backend/{tlogbe => tstorebe}/plugins/ticketvote/activevotes.go (100%) rename politeiad/backend/{tlogbe => tstorebe}/plugins/ticketvote/cmds.go (98%) rename politeiad/backend/{tlogbe => tstorebe}/plugins/ticketvote/hooks.go (98%) rename politeiad/backend/{tlogbe => tstorebe}/plugins/ticketvote/inventory.go (100%) rename politeiad/backend/{tlogbe => tstorebe}/plugins/ticketvote/log.go (100%) rename politeiad/backend/{tlogbe => tstorebe}/plugins/ticketvote/submissions.go (100%) rename politeiad/backend/{tlogbe => tstorebe}/plugins/ticketvote/summary.go (100%) rename politeiad/backend/{tlogbe => tstorebe}/plugins/ticketvote/ticketvote.go (96%) rename politeiad/backend/{tlogbe => tstorebe}/plugins/usermd/cache.go (100%) rename politeiad/backend/{tlogbe => tstorebe}/plugins/usermd/cmds.go (96%) rename politeiad/backend/{tlogbe => tstorebe}/plugins/usermd/hooks.go (98%) rename politeiad/backend/{tlogbe => tstorebe}/plugins/usermd/log.go (100%) rename politeiad/backend/{tlogbe => tstorebe}/plugins/usermd/usermd.go (92%) rename politeiad/backend/{tlogbe => tstorebe}/store/fs/fs.go (98%) rename politeiad/backend/{tlogbe => tstorebe}/store/fs/log.go (100%) rename politeiad/backend/{tlogbe => tstorebe}/store/mysql/log.go (100%) rename politeiad/backend/{tlogbe => tstorebe}/store/mysql/mysql.go (98%) rename politeiad/backend/{tlogbe => tstorebe}/store/store.go (100%) rename politeiad/backend/{tlogbe => tstorebe}/testing.go (55%) rename politeiad/backend/{tlogbe/tlog => tstorebe/tstore}/anchor.go (96%) rename politeiad/backend/{tlogbe/tlog/tlogclient.go => tstorebe/tstore/client.go} (88%) rename politeiad/backend/{tlogbe/tlog => tstorebe/tstore}/convert.go (98%) rename politeiad/backend/{tlogbe/tlog => tstorebe/tstore}/dcrtime.go (99%) rename politeiad/backend/{tlogbe/tlog => tstorebe/tstore}/encryptionkey.go (99%) rename politeiad/backend/{tlogbe => tstorebe/tstore}/log.go (98%) rename politeiad/backend/{tlogbe/tlog => tstorebe/tstore}/plugin.go (74%) rename politeiad/backend/{tlogbe/tlog => tstorebe/tstore}/recordindex.go (94%) rename politeiad/backend/{tlogbe/tlog => tstorebe/tstore}/testing.go (52%) rename politeiad/backend/{tlogbe/tlog => tstorebe/tstore}/trillianclient.go (99%) rename politeiad/backend/{tlogbe/tlog/tlog.go => tstorebe/tstore/tstore.go} (92%) rename politeiad/backend/{tlogbe/tlog => tstorebe/tstore}/verify.go (99%) rename politeiad/backend/{tlogbe/tlogbe_test.go => tstorebe/tstore_test.go} (90%) rename politeiad/backend/{tlogbe/tlogbe.go => tstorebe/tstorebe.go} (90%) diff --git a/go.mod b/go.mod index db2d10494..4fcb84181 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,7 @@ require ( github.com/otiai10/curr v0.0.0-20190513014714-f5a3d24e5776 // indirect github.com/pmezard/go-difflib v1.0.0 github.com/pquerna/otp v1.2.0 - github.com/prometheus/common v0.10.0 // indirect + github.com/prometheus/common v0.10.0 github.com/robfig/cron v1.2.0 github.com/subosito/gozaru v0.0.0-20190625071150-416082cce636 github.com/syndtr/goleveldb v1.0.0 diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index 5b01ad89c..dbd4e7a7a 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -48,8 +48,8 @@ const ( // Token sizes. The size of the token depends on the politeiad // backend configuration. - TokenSizeTlog = 8 - TokenSizeGit = 32 + TokenSizeTstore = 8 + TokenSizeGit = 32 MetadataStreamsMax = uint64(16) // Maximum number of metadata streams diff --git a/politeiad/backend/tlogbe/inventory.go b/politeiad/backend/tstorebe/inventory.go similarity index 89% rename from politeiad/backend/tlogbe/inventory.go rename to politeiad/backend/tstorebe/inventory.go index d97f626b6..0f00c9215 100644 --- a/politeiad/backend/tlogbe/inventory.go +++ b/politeiad/backend/tstorebe/inventory.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlogbe +package tstorebe import ( "encoding/hex" @@ -30,11 +30,11 @@ type inventory struct { Entries []entry `json:"entries"` } -func (t *tlogBackend) invPathUnvetted() string { +func (t *tstoreBackend) invPathUnvetted() string { return filepath.Join(t.dataDir, filenameInvUnvetted) } -func (t *tlogBackend) invPathVetted() string { +func (t *tstoreBackend) invPathVetted() string { return filepath.Join(t.dataDir, filenameInvVetted) } @@ -42,7 +42,7 @@ func (t *tlogBackend) invPathVetted() string { // if one does not exist yet. // // This function must be called WITH the read lock held. -func (t *tlogBackend) invGetLocked(filePath string) (*inventory, error) { +func (t *tstoreBackend) invGetLocked(filePath string) (*inventory, error) { b, err := ioutil.ReadFile(filePath) if err != nil { var e *os.PathError @@ -68,7 +68,7 @@ func (t *tlogBackend) invGetLocked(filePath string) (*inventory, error) { // does not exist yet. // // This function must be called WITHOUT the read lock held. -func (t *tlogBackend) invGet(filePath string) (*inventory, error) { +func (t *tstoreBackend) invGet(filePath string) (*inventory, error) { t.RLock() defer t.RUnlock() @@ -78,7 +78,7 @@ func (t *tlogBackend) invGet(filePath string) (*inventory, error) { // invSaveLocked writes the inventory to disk. // // This function must be called WITH the read/write lock held. -func (t *tlogBackend) invSaveLocked(filePath string, inv inventory) error { +func (t *tstoreBackend) invSaveLocked(filePath string, inv inventory) error { b, err := json.Marshal(inv) if err != nil { return err @@ -86,7 +86,7 @@ func (t *tlogBackend) invSaveLocked(filePath string, inv inventory) error { return ioutil.WriteFile(filePath, b, 0664) } -func (t *tlogBackend) invAdd(state string, token []byte, s backend.MDStatusT) error { +func (t *tstoreBackend) invAdd(state string, token []byte, s backend.MDStatusT) error { t.Lock() defer t.Unlock() @@ -126,7 +126,7 @@ func (t *tlogBackend) invAdd(state string, token []byte, s backend.MDStatusT) er return nil } -func (t *tlogBackend) invUpdate(state string, token []byte, s backend.MDStatusT) error { +func (t *tstoreBackend) invUpdate(state string, token []byte, s backend.MDStatusT) error { t.Lock() defer t.Unlock() @@ -176,7 +176,7 @@ func (t *tlogBackend) invUpdate(state string, token []byte, s backend.MDStatusT) // invMoveToVetted moves a token from the unvetted inventory to the vetted // inventory. -func (t *tlogBackend) invMoveToVetted(token []byte, s backend.MDStatusT) error { +func (t *tstoreBackend) invMoveToVetted(token []byte, s backend.MDStatusT) error { t.Lock() defer t.Unlock() @@ -229,7 +229,7 @@ func (t *tlogBackend) invMoveToVetted(token []byte, s backend.MDStatusT) error { // inventoryAdd is a wrapper around the invAdd method that allows us to decide // how disk read/write errors should be handled. For now we just panic. -func (t *tlogBackend) inventoryAdd(state string, token []byte, s backend.MDStatusT) { +func (t *tstoreBackend) inventoryAdd(state string, token []byte, s backend.MDStatusT) { err := t.invAdd(state, token, s) if err != nil { e := fmt.Sprintf("invAdd %v %x %v: %v", state, token, s, err) @@ -239,7 +239,7 @@ func (t *tlogBackend) inventoryAdd(state string, token []byte, s backend.MDStatu // inventoryUpdate is a wrapper around the invUpdate method that allows us to // decide how disk read/write errors should be handled. For now we just panic. -func (t *tlogBackend) inventoryUpdate(state string, token []byte, s backend.MDStatusT) { +func (t *tstoreBackend) inventoryUpdate(state string, token []byte, s backend.MDStatusT) { err := t.invUpdate(state, token, s) if err != nil { e := fmt.Sprintf("invUpdate %v %x %v: %v", state, token, s, err) @@ -250,7 +250,7 @@ func (t *tlogBackend) inventoryUpdate(state string, token []byte, s backend.MDSt // inventoryMoveToVetted is a wrapper around the invMoveToVetted method that // allows us to decide how disk read/write errors should be handled. For now we // just panic. -func (t *tlogBackend) inventoryMoveToVetted(token []byte, s backend.MDStatusT) { +func (t *tstoreBackend) inventoryMoveToVetted(token []byte, s backend.MDStatusT) { err := t.invMoveToVetted(token, s) if err != nil { e := fmt.Sprintf("invMoveToVetted %x %v: %v", token, s, err) @@ -266,7 +266,7 @@ type invByStatus struct { Vetted map[backend.MDStatusT][]string } -func (t *tlogBackend) invByStatusAll(pageSize uint32) (*invByStatus, error) { +func (t *tstoreBackend) invByStatusAll(pageSize uint32) (*invByStatus, error) { // Get unvetted inventory u, err := t.invGet(t.invPathUnvetted()) if err != nil { @@ -321,7 +321,7 @@ func (t *tlogBackend) invByStatusAll(pageSize uint32) (*invByStatus, error) { }, nil } -func (t *tlogBackend) invByStatus(state string, s backend.MDStatusT, pageSize, page uint32) (*invByStatus, error) { +func (t *tstoreBackend) invByStatus(state string, s backend.MDStatusT, pageSize, page uint32) (*invByStatus, error) { // If no status is provided a page of tokens for each status should // be returned. if s == backend.MDStatusInvalid { diff --git a/politeiad/backend/tlogbe/tlog/log.go b/politeiad/backend/tstorebe/log.go similarity index 97% rename from politeiad/backend/tlogbe/tlog/log.go rename to politeiad/backend/tstorebe/log.go index eef45b55f..d58862017 100644 --- a/politeiad/backend/tlogbe/tlog/log.go +++ b/politeiad/backend/tstorebe/log.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlog +package tstorebe import "github.com/decred/slog" diff --git a/politeiad/backend/tlogbe/plugins/comments/cmds.go b/politeiad/backend/tstorebe/plugins/comments/cmds.go similarity index 98% rename from politeiad/backend/tlogbe/plugins/comments/cmds.go rename to politeiad/backend/tstorebe/plugins/comments/cmds.go index db9160d2e..1466713ce 100644 --- a/politeiad/backend/tlogbe/plugins/comments/cmds.go +++ b/politeiad/backend/tstorebe/plugins/comments/cmds.go @@ -16,7 +16,7 @@ import ( "time" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/store" + "github.com/decred/politeia/politeiad/backend/tstorebe/store" "github.com/decred/politeia/politeiad/plugins/comments" "github.com/decred/politeia/util" ) @@ -31,7 +31,7 @@ const ( ) func tokenDecode(token string) ([]byte, error) { - return util.TokenDecode(util.TokenTypeTlog, token) + return util.TokenDecode(util.TokenTypeTstore, token) } func convertSignatureError(err error) backend.PluginError { @@ -294,7 +294,7 @@ func (p *commentsPlugin) commentAddSave(treeID int64, ca comments.CommentAdd) ([ if err != nil { return nil, err } - err = p.tlog.BlobSave(treeID, *be) + err = p.tstore.BlobSave(treeID, *be) if err != nil { return nil, err } @@ -304,7 +304,7 @@ func (p *commentsPlugin) commentAddSave(treeID int64, ca comments.CommentAdd) ([ // commentAdds returns the commentAdd for all specified digests. func (p *commentsPlugin) commentAdds(treeID int64, digests [][]byte) ([]comments.CommentAdd, error) { // Retrieve blobs - blobs, err := p.tlog.Blobs(treeID, digests) + blobs, err := p.tstore.Blobs(treeID, digests) if err != nil { return nil, err } @@ -342,7 +342,7 @@ func (p *commentsPlugin) commentDelSave(treeID int64, cd comments.CommentDel) ([ if err != nil { return nil, err } - err = p.tlog.BlobSave(treeID, *be) + err = p.tstore.BlobSave(treeID, *be) if err != nil { return nil, err } @@ -351,7 +351,7 @@ func (p *commentsPlugin) commentDelSave(treeID int64, cd comments.CommentDel) ([ func (p *commentsPlugin) commentDels(treeID int64, digests [][]byte) ([]comments.CommentDel, error) { // Retrieve blobs - blobs, err := p.tlog.Blobs(treeID, digests) + blobs, err := p.tstore.Blobs(treeID, digests) if err != nil { return nil, err } @@ -389,7 +389,7 @@ func (p *commentsPlugin) commentVoteSave(treeID int64, cv comments.CommentVote) if err != nil { return nil, err } - err = p.tlog.BlobSave(treeID, *be) + err = p.tstore.BlobSave(treeID, *be) if err != nil { return nil, err } @@ -398,7 +398,7 @@ func (p *commentsPlugin) commentVoteSave(treeID int64, cv comments.CommentVote) func (p *commentsPlugin) commentVotes(treeID int64, digests [][]byte) ([]comments.CommentVote, error) { // Retrieve blobs - blobs, err := p.tlog.Blobs(treeID, digests) + blobs, err := p.tstore.Blobs(treeID, digests) if err != nil { return nil, err } @@ -514,7 +514,7 @@ func (p *commentsPlugin) comment(treeID int64, ridx recordIndex, commentID uint3 func (p *commentsPlugin) timestamp(treeID int64, digest []byte) (*comments.Timestamp, error) { // Get timestamp - t, err := p.tlog.Timestamp(treeID, digest) + t, err := p.tstore.Timestamp(treeID, digest) if err != nil { return nil, err } @@ -952,7 +952,7 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str for _, v := range cidx.Adds { digests = append(digests, v) } - err = p.tlog.BlobsDel(treeID, digests) + err = p.tstore.BlobsDel(treeID, digests) if err != nil { return "", fmt.Errorf("del: %v", err) } diff --git a/politeiad/backend/tlogbe/plugins/comments/comments.go b/politeiad/backend/tstorebe/plugins/comments/comments.go similarity index 92% rename from politeiad/backend/tlogbe/plugins/comments/comments.go rename to politeiad/backend/tstorebe/plugins/comments/comments.go index 2add44275..9ce6dc2ac 100644 --- a/politeiad/backend/tlogbe/plugins/comments/comments.go +++ b/politeiad/backend/tstorebe/plugins/comments/comments.go @@ -13,7 +13,7 @@ import ( "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" + "github.com/decred/politeia/politeiad/backend/tstorebe/plugins" "github.com/decred/politeia/politeiad/plugins/comments" ) @@ -25,12 +25,12 @@ var ( _ plugins.PluginClient = (*commentsPlugin)(nil) ) -// commentsPlugin is the tlog backend implementation of the comments plugin. +// commentsPlugin is the tstore backend implementation of the comments plugin. // // commentsPlugin satisfies the plugins.PluginClient interface. type commentsPlugin struct { sync.RWMutex - tlog plugins.TlogClient + tstore plugins.TstoreClient // dataDir is the comments plugin data directory. The only data // that is stored here is cached data that can be re-created at any @@ -127,7 +127,7 @@ func (p *commentsPlugin) Settings() []backend.PluginSetting { } // New returns a new comments plugin. -func New(tlog plugins.TlogClient, settings []backend.PluginSetting, dataDir string, id *identity.FullIdentity) (*commentsPlugin, error) { +func New(tstore plugins.TstoreClient, settings []backend.PluginSetting, dataDir string, id *identity.FullIdentity) (*commentsPlugin, error) { // Setup comments plugin data dir dataDir = filepath.Join(dataDir, comments.PluginID) err := os.MkdirAll(dataDir, 0700) @@ -162,7 +162,7 @@ func New(tlog plugins.TlogClient, settings []backend.PluginSetting, dataDir stri } return &commentsPlugin{ - tlog: tlog, + tstore: tstore, identity: id, dataDir: dataDir, commentLengthMax: commentLengthMax, diff --git a/politeiad/backend/tlogbe/plugins/comments/comments_test.go b/politeiad/backend/tstorebe/plugins/comments/comments_test.go similarity index 92% rename from politeiad/backend/tlogbe/plugins/comments/comments_test.go rename to politeiad/backend/tstorebe/plugins/comments/comments_test.go index 8661399b7..67146c6d7 100644 --- a/politeiad/backend/tlogbe/plugins/comments/comments_test.go +++ b/politeiad/backend/tstorebe/plugins/comments/comments_test.go @@ -15,7 +15,7 @@ import ( "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" + "github.com/decred/politeia/politeiad/backend/tstorebe/plugins" "github.com/decred/politeia/politeiad/plugins/comments" "github.com/google/uuid" ) @@ -42,14 +42,14 @@ func newTestCommentsPlugin(t *testing.T) (*commentsPlugin, func()) { t.Helper() // Setup data dir - dataDir, err := ioutil.TempDir("", "tlogbe.comments.test") + dataDir, err := ioutil.TempDir("", "tstorebe.comments.test") if err != nil { t.Fatal(err) } - // TODO Implement a test clients.TlogClient - // Setup tlog client - var tlog plugins.TlogClient + // TODO Implement a test clients.TstoreClient + // Setup tstore client + var tstore plugins.TstoreClient // Setup plugin identity fid, err := identity.New() @@ -58,7 +58,7 @@ func newTestCommentsPlugin(t *testing.T) (*commentsPlugin, func()) { } // Setup comment plugins - c, err := New(tlog, []backend.PluginSetting{}, dataDir, fid) + c, err := New(tstore, []backend.PluginSetting{}, dataDir, fid) if err != nil { t.Fatal(err) } diff --git a/politeiad/backend/tlogbe/plugins/comments/log.go b/politeiad/backend/tstorebe/plugins/comments/log.go similarity index 100% rename from politeiad/backend/tlogbe/plugins/comments/log.go rename to politeiad/backend/tstorebe/plugins/comments/log.go diff --git a/politeiad/backend/tlogbe/plugins/comments/recordindex.go b/politeiad/backend/tstorebe/plugins/comments/recordindex.go similarity index 100% rename from politeiad/backend/tlogbe/plugins/comments/recordindex.go rename to politeiad/backend/tstorebe/plugins/comments/recordindex.go diff --git a/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go b/politeiad/backend/tstorebe/plugins/dcrdata/dcrdata.go similarity index 99% rename from politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go rename to politeiad/backend/tstorebe/plugins/dcrdata/dcrdata.go index 06f01297e..992fcdd76 100644 --- a/politeiad/backend/tlogbe/plugins/dcrdata/dcrdata.go +++ b/politeiad/backend/tstorebe/plugins/dcrdata/dcrdata.go @@ -20,7 +20,7 @@ import ( exptypes "github.com/decred/dcrdata/explorer/types/v2" pstypes "github.com/decred/dcrdata/pubsub/types/v3" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" + "github.com/decred/politeia/politeiad/backend/tstorebe/plugins" "github.com/decred/politeia/politeiad/plugins/dcrdata" "github.com/decred/politeia/util" "github.com/decred/politeia/wsdcrdata" diff --git a/politeiad/backend/tlogbe/plugins/dcrdata/log.go b/politeiad/backend/tstorebe/plugins/dcrdata/log.go similarity index 100% rename from politeiad/backend/tlogbe/plugins/dcrdata/log.go rename to politeiad/backend/tstorebe/plugins/dcrdata/log.go diff --git a/politeiad/backend/tlogbe/plugins/pi/hooks.go b/politeiad/backend/tstorebe/plugins/pi/hooks.go similarity index 98% rename from politeiad/backend/tlogbe/plugins/pi/hooks.go rename to politeiad/backend/tstorebe/plugins/pi/hooks.go index 38fc7ee27..771bed5f6 100644 --- a/politeiad/backend/tlogbe/plugins/pi/hooks.go +++ b/politeiad/backend/tstorebe/plugins/pi/hooks.go @@ -10,7 +10,7 @@ import ( "fmt" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" + "github.com/decred/politeia/politeiad/backend/tstorebe/plugins" "github.com/decred/politeia/politeiad/plugins/comments" "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/ticketvote" @@ -35,7 +35,7 @@ var ( ) func tokenDecode(token string) ([]byte, error) { - return util.TokenDecode(util.TokenTypeTlog, token) + return util.TokenDecode(util.TokenTypeTstore, token) } // proposalMetadataDecode decodes and returns the ProposalMetadata from the diff --git a/politeiad/backend/tlogbe/plugins/pi/hooks_test.go b/politeiad/backend/tstorebe/plugins/pi/hooks_test.go similarity index 100% rename from politeiad/backend/tlogbe/plugins/pi/hooks_test.go rename to politeiad/backend/tstorebe/plugins/pi/hooks_test.go diff --git a/politeiad/backend/tlogbe/plugins/pi/log.go b/politeiad/backend/tstorebe/plugins/pi/log.go similarity index 100% rename from politeiad/backend/tlogbe/plugins/pi/log.go rename to politeiad/backend/tstorebe/plugins/pi/log.go diff --git a/politeiad/backend/tlogbe/plugins/pi/pi.go b/politeiad/backend/tstorebe/plugins/pi/pi.go similarity index 99% rename from politeiad/backend/tlogbe/plugins/pi/pi.go rename to politeiad/backend/tstorebe/plugins/pi/pi.go index 6c8350dbb..355f3075e 100644 --- a/politeiad/backend/tlogbe/plugins/pi/pi.go +++ b/politeiad/backend/tstorebe/plugins/pi/pi.go @@ -13,7 +13,7 @@ import ( "strconv" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" + "github.com/decred/politeia/politeiad/backend/tstorebe/plugins" "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/util" ) diff --git a/politeiad/backend/tlogbe/plugins/plugins.go b/politeiad/backend/tstorebe/plugins/plugins.go similarity index 93% rename from politeiad/backend/tlogbe/plugins/plugins.go rename to politeiad/backend/tstorebe/plugins/plugins.go index de45e635b..abfeef3cc 100644 --- a/politeiad/backend/tlogbe/plugins/plugins.go +++ b/politeiad/backend/tstorebe/plugins/plugins.go @@ -6,7 +6,7 @@ package plugins import ( "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/store" + "github.com/decred/politeia/politeiad/backend/tstorebe/store" ) // HookT represents a plugin hook. @@ -155,8 +155,8 @@ type HookPluginPost struct { Reply string `json:"reply"` } -// PluginClient provides an API for a tlog instance to use when interacting -// with a plugin. All tlog plugins must implement the PluginClient interface. +// PluginClient provides an API for a tstore instance to use when interacting +// with a plugin. All tstore plugins must implement the PluginClient interface. type PluginClient interface { // Setup performs any required plugin setup. Setup() error @@ -174,15 +174,15 @@ type PluginClient interface { Settings() []backend.PluginSetting } -// TlogClient provides an API for plugins to interact with a tlog instance. -// Plugins are allowed to save, delete, and get plugin data to/from the tlog +// TstoreClient provides an API for plugins to interact with a tstore instance. +// Plugins are allowed to save, delete, and get plugin data to/from the tstore // backend. Editing plugin data is not allowed. -type TlogClient interface { - // BlobSave saves a BlobEntry to the tlog instance. The BlobEntry - // will be encrypted prior to being written to disk if the tlog +type TstoreClient interface { + // BlobSave saves a BlobEntry to the tstore instance. The BlobEntry + // will be encrypted prior to being written to disk if the tstore // instance has an encryption key set. The digest of the data, // i.e. BlobEntry.Digest, can be thought of as the blob ID and can - // be used to get/del the blob from tlog. + // be used to get/del the blob from tstore. BlobSave(treeID int64, be store.BlobEntry) error // BlobsDel deletes the blobs that correspond to the provided diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/activevotes.go b/politeiad/backend/tstorebe/plugins/ticketvote/activevotes.go similarity index 100% rename from politeiad/backend/tlogbe/plugins/ticketvote/activevotes.go rename to politeiad/backend/tstorebe/plugins/ticketvote/activevotes.go diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go b/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go similarity index 98% rename from politeiad/backend/tlogbe/plugins/ticketvote/cmds.go rename to politeiad/backend/tstorebe/plugins/ticketvote/cmds.go index 66ea61aa6..67d883378 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go @@ -23,7 +23,7 @@ import ( "github.com/decred/dcrd/dcrutil/v3" "github.com/decred/dcrd/wire" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/store" + "github.com/decred/politeia/politeiad/backend/tstorebe/store" "github.com/decred/politeia/politeiad/plugins/dcrdata" "github.com/decred/politeia/politeiad/plugins/ticketvote" "github.com/decred/politeia/util" @@ -50,13 +50,13 @@ func voteHasEnded(bestBlock, endHeight uint32) bool { // tokenDecode decodes a record token and only accepts full length tokens. func tokenDecode(token string) ([]byte, error) { - return util.TokenDecode(util.TokenTypeTlog, token) + return util.TokenDecode(util.TokenTypeTstore, token) } // tokenDecode decodes a record token and accepts both full length tokens and // token prefixes. func tokenDecodeAnyLength(token string) ([]byte, error) { - return util.TokenDecodeAnyLength(util.TokenTypeTlog, token) + return util.TokenDecodeAnyLength(util.TokenTypeTstore, token) } func convertSignatureError(err error) backend.PluginError { @@ -305,12 +305,12 @@ func (p *ticketVotePlugin) authSave(treeID int64, ad ticketvote.AuthDetails) err } // Save blob - return p.tlog.BlobSave(treeID, *be) + return p.tstore.BlobSave(treeID, *be) } func (p *ticketVotePlugin) auths(treeID int64) ([]ticketvote.AuthDetails, error) { // Retrieve blobs - blobs, err := p.tlog.BlobsByDataDesc(treeID, dataDescriptorAuthDetails) + blobs, err := p.tstore.BlobsByDataDesc(treeID, dataDescriptorAuthDetails) if err != nil { return nil, err } @@ -342,12 +342,12 @@ func (p *ticketVotePlugin) voteDetailsSave(treeID int64, vd ticketvote.VoteDetai } // Save blob - return p.tlog.BlobSave(treeID, *be) + return p.tstore.BlobSave(treeID, *be) } func (p *ticketVotePlugin) voteDetails(treeID int64) (*ticketvote.VoteDetails, error) { // Retrieve blobs - blobs, err := p.tlog.BlobsByDataDesc(treeID, dataDescriptorVoteDetails) + blobs, err := p.tstore.BlobsByDataDesc(treeID, dataDescriptorVoteDetails) if err != nil { return nil, err } @@ -395,12 +395,13 @@ func (p *ticketVotePlugin) castVoteSave(treeID int64, cv ticketvote.CastVoteDeta } // Save blob - return p.tlog.BlobSave(treeID, *be) + return p.tstore.BlobSave(treeID, *be) } func (p *ticketVotePlugin) castVotes(treeID int64) ([]ticketvote.CastVoteDetails, error) { // Retrieve blobs - blobs, err := p.tlog.BlobsByDataDesc(treeID, dataDescriptorCastVoteDetails) + blobs, err := p.tstore.BlobsByDataDesc(treeID, + dataDescriptorCastVoteDetails) if err != nil { return nil, err } @@ -732,7 +733,7 @@ func (p *ticketVotePlugin) summaryByToken(token []byte) (*ticketvote.SummaryRepl } func (p *ticketVotePlugin) timestamp(treeID int64, digest []byte) (*ticketvote.Timestamp, error) { - t, err := p.tlog.Timestamp(treeID, digest) + t, err := p.tstore.Timestamp(treeID, digest) if err != nil { return nil, fmt.Errorf("timestamp %v %x: %v", treeID, digest, err) } @@ -1420,7 +1421,7 @@ func (p *ticketVotePlugin) startRunoffRecordSave(treeID int64, srr startRunoffRe if err != nil { return err } - err = p.tlog.BlobSave(treeID, *be) + err = p.tstore.BlobSave(treeID, *be) if err != nil { return fmt.Errorf("BlobSave %v %v: %v", treeID, dataDescriptorStartRunoff, err) @@ -1431,7 +1432,7 @@ func (p *ticketVotePlugin) startRunoffRecordSave(treeID int64, srr startRunoffRe // startRunoffRecord returns the startRunoff record if one exists on a tree. // nil will be returned if a startRunoff record is not found. func (p *ticketVotePlugin) startRunoffRecord(treeID int64) (*startRunoffRecord, error) { - blobs, err := p.tlog.BlobsByDataDesc(treeID, dataDescriptorStartRunoff) + blobs, err := p.tstore.BlobsByDataDesc(treeID, dataDescriptorStartRunoff) if err != nil { return nil, fmt.Errorf("BlobsByDataDesc %v %v: %v", treeID, dataDescriptorStartRunoff, err) @@ -2612,7 +2613,7 @@ func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload str switch { case t.VotesPage > 0: // Return a page of vote timestamps - digests, err := p.tlog.DigestsByDataDesc(treeID, + digests, err := p.tstore.DigestsByDataDesc(treeID, dataDescriptorCastVoteDetails) if err != nil { return "", fmt.Errorf("digestsByKeyPrefix %v %v: %v", @@ -2639,7 +2640,8 @@ func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload str // Return authorization timestamps and the vote details timestamp. // Auth timestamps - digests, err := p.tlog.DigestsByDataDesc(treeID, dataDescriptorAuthDetails) + digests, err := p.tstore.DigestsByDataDesc(treeID, + dataDescriptorAuthDetails) if err != nil { return "", fmt.Errorf("DigestByDataDesc %v %v: %v", treeID, dataDescriptorAuthDetails, err) @@ -2654,7 +2656,8 @@ func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload str } // Vote details timestamp - digests, err = p.tlog.DigestsByDataDesc(treeID, dataDescriptorVoteDetails) + digests, err = p.tstore.DigestsByDataDesc(treeID, + dataDescriptorVoteDetails) if err != nil { return "", fmt.Errorf("DigestsByDataDesc %v %v: %v", treeID, dataDescriptorVoteDetails, err) diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go b/politeiad/backend/tstorebe/plugins/ticketvote/hooks.go similarity index 98% rename from politeiad/backend/tlogbe/plugins/ticketvote/hooks.go rename to politeiad/backend/tstorebe/plugins/ticketvote/hooks.go index cc24bd038..8eeeedc99 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/hooks.go +++ b/politeiad/backend/tstorebe/plugins/ticketvote/hooks.go @@ -12,7 +12,7 @@ import ( "time" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" + "github.com/decred/politeia/politeiad/backend/tstorebe/plugins" "github.com/decred/politeia/politeiad/plugins/ticketvote" ) @@ -318,7 +318,7 @@ func (p *ticketVotePlugin) hookSetRecordStatusPost(treeID int64, payload string) // to update cached data if this is the vetted instance. We can // determine this by checking if the record exists. The unvetted // instance will return false. - if status == backend.MDStatusVetted && !p.tlog.RecordExists(treeID) { + if status == backend.MDStatusVetted && !p.tstore.RecordExists(treeID) { // This is the unvetted instance. Nothing to do. return nil } diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/inventory.go b/politeiad/backend/tstorebe/plugins/ticketvote/inventory.go similarity index 100% rename from politeiad/backend/tlogbe/plugins/ticketvote/inventory.go rename to politeiad/backend/tstorebe/plugins/ticketvote/inventory.go diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/log.go b/politeiad/backend/tstorebe/plugins/ticketvote/log.go similarity index 100% rename from politeiad/backend/tlogbe/plugins/ticketvote/log.go rename to politeiad/backend/tstorebe/plugins/ticketvote/log.go diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/submissions.go b/politeiad/backend/tstorebe/plugins/ticketvote/submissions.go similarity index 100% rename from politeiad/backend/tlogbe/plugins/ticketvote/submissions.go rename to politeiad/backend/tstorebe/plugins/ticketvote/submissions.go diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/summary.go b/politeiad/backend/tstorebe/plugins/ticketvote/summary.go similarity index 100% rename from politeiad/backend/tlogbe/plugins/ticketvote/summary.go rename to politeiad/backend/tstorebe/plugins/ticketvote/summary.go diff --git a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tstorebe/plugins/ticketvote/ticketvote.go similarity index 96% rename from politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go rename to politeiad/backend/tstorebe/plugins/ticketvote/ticketvote.go index 95056b0c8..a6ac8d07f 100644 --- a/politeiad/backend/tlogbe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tstorebe/plugins/ticketvote/ticketvote.go @@ -15,7 +15,7 @@ import ( "github.com/decred/dcrd/chaincfg/v3" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" + "github.com/decred/politeia/politeiad/backend/tstorebe/plugins" "github.com/decred/politeia/politeiad/plugins/dcrdata" "github.com/decred/politeia/politeiad/plugins/ticketvote" ) @@ -27,7 +27,7 @@ var ( // ticketVotePlugin satisfies the plugins.PluginClient interface. type ticketVotePlugin struct { backend backend.Backend - tlog plugins.TlogClient + tstore plugins.TstoreClient activeNetParams *chaincfg.Params // dataDir is the ticket vote plugin data directory. The only data @@ -238,7 +238,7 @@ func (p *ticketVotePlugin) Settings() []backend.PluginSetting { } } -func New(backend backend.Backend, tlog plugins.TlogClient, settings []backend.PluginSetting, dataDir string, id *identity.FullIdentity, activeNetParams *chaincfg.Params) (*ticketVotePlugin, error) { +func New(backend backend.Backend, tstore plugins.TstoreClient, settings []backend.PluginSetting, dataDir string, id *identity.FullIdentity, activeNetParams *chaincfg.Params) (*ticketVotePlugin, error) { // Plugin settings var ( linkByPeriodMin = ticketvote.SettingLinkByPeriodMin @@ -321,7 +321,7 @@ func New(backend backend.Backend, tlog plugins.TlogClient, settings []backend.Pl return &ticketVotePlugin{ activeNetParams: activeNetParams, backend: backend, - tlog: tlog, + tstore: tstore, dataDir: dataDir, identity: id, activeVotes: newActiveVotes(), diff --git a/politeiad/backend/tlogbe/plugins/usermd/cache.go b/politeiad/backend/tstorebe/plugins/usermd/cache.go similarity index 100% rename from politeiad/backend/tlogbe/plugins/usermd/cache.go rename to politeiad/backend/tstorebe/plugins/usermd/cache.go diff --git a/politeiad/backend/tlogbe/plugins/usermd/cmds.go b/politeiad/backend/tstorebe/plugins/usermd/cmds.go similarity index 96% rename from politeiad/backend/tlogbe/plugins/usermd/cmds.go rename to politeiad/backend/tstorebe/plugins/usermd/cmds.go index 24e72ad5e..c847f2514 100644 --- a/politeiad/backend/tlogbe/plugins/usermd/cmds.go +++ b/politeiad/backend/tstorebe/plugins/usermd/cmds.go @@ -14,7 +14,7 @@ func (p *userPlugin) cmdAuthor(treeID int64) (string, error) { log.Tracef("cmdAuthor: %v", treeID) // Get user metadata - r, err := p.tlog.RecordLatest(treeID) + r, err := p.tstore.RecordLatest(treeID) if err != nil { return "", err } diff --git a/politeiad/backend/tlogbe/plugins/usermd/hooks.go b/politeiad/backend/tstorebe/plugins/usermd/hooks.go similarity index 98% rename from politeiad/backend/tlogbe/plugins/usermd/hooks.go rename to politeiad/backend/tstorebe/plugins/usermd/hooks.go index 6509bfe75..bf89fed0c 100644 --- a/politeiad/backend/tlogbe/plugins/usermd/hooks.go +++ b/politeiad/backend/tstorebe/plugins/usermd/hooks.go @@ -14,7 +14,7 @@ import ( "strings" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" + "github.com/decred/politeia/politeiad/backend/tstorebe/plugins" "github.com/decred/politeia/politeiad/plugins/usermd" "github.com/decred/politeia/util" "github.com/google/uuid" @@ -362,7 +362,7 @@ func (p *userPlugin) hookSetRecordStatusPost(treeID int64, payload string) error // added to the vetted tstore user cache. We can determine this // by checking if the record exists. The unvetted instance will // return false. - if p.tlog.RecordExists(treeID) { + if p.tstore.RecordExists(treeID) { // This is the vetted tstore. Add token to the user cache. err = p.userCacheAddToken(um.UserID, srs.RecordMetadata.Token) if err != nil { diff --git a/politeiad/backend/tlogbe/plugins/usermd/log.go b/politeiad/backend/tstorebe/plugins/usermd/log.go similarity index 100% rename from politeiad/backend/tlogbe/plugins/usermd/log.go rename to politeiad/backend/tstorebe/plugins/usermd/log.go diff --git a/politeiad/backend/tlogbe/plugins/usermd/usermd.go b/politeiad/backend/tstorebe/plugins/usermd/usermd.go similarity index 92% rename from politeiad/backend/tlogbe/plugins/usermd/usermd.go rename to politeiad/backend/tstorebe/plugins/usermd/usermd.go index a5f0e0814..d033826fe 100644 --- a/politeiad/backend/tlogbe/plugins/usermd/usermd.go +++ b/politeiad/backend/tstorebe/plugins/usermd/usermd.go @@ -10,7 +10,7 @@ import ( "sync" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" + "github.com/decred/politeia/politeiad/backend/tstorebe/plugins" "github.com/decred/politeia/politeiad/plugins/usermd" ) @@ -20,7 +20,7 @@ var ( type userPlugin struct { sync.Mutex - tlog plugins.TlogClient + tstore plugins.TstoreClient // dataDir is the pi plugin data directory. The only data that is // stored here is cached data that can be re-created at any time @@ -96,7 +96,7 @@ func (p *userPlugin) Settings() []backend.PluginSetting { } // New returns a new userPlugin. -func New(tlog plugins.TlogClient, settings []backend.PluginSetting, dataDir string) (*userPlugin, error) { +func New(tstore plugins.TstoreClient, settings []backend.PluginSetting, dataDir string) (*userPlugin, error) { // Create plugin data directory dataDir = filepath.Join(dataDir, usermd.PluginID) err := os.MkdirAll(dataDir, 0700) @@ -105,7 +105,7 @@ func New(tlog plugins.TlogClient, settings []backend.PluginSetting, dataDir stri } return &userPlugin{ - tlog: tlog, + tstore: tstore, dataDir: dataDir, }, nil } diff --git a/politeiad/backend/tlogbe/store/fs/fs.go b/politeiad/backend/tstorebe/store/fs/fs.go similarity index 98% rename from politeiad/backend/tlogbe/store/fs/fs.go rename to politeiad/backend/tstorebe/store/fs/fs.go index 42624ed21..f1f728ff5 100644 --- a/politeiad/backend/tlogbe/store/fs/fs.go +++ b/politeiad/backend/tstorebe/store/fs/fs.go @@ -12,7 +12,7 @@ import ( "path/filepath" "sync" - "github.com/decred/politeia/politeiad/backend/tlogbe/store" + "github.com/decred/politeia/politeiad/backend/tstorebe/store" "github.com/google/uuid" ) diff --git a/politeiad/backend/tlogbe/store/fs/log.go b/politeiad/backend/tstorebe/store/fs/log.go similarity index 100% rename from politeiad/backend/tlogbe/store/fs/log.go rename to politeiad/backend/tstorebe/store/fs/log.go diff --git a/politeiad/backend/tlogbe/store/mysql/log.go b/politeiad/backend/tstorebe/store/mysql/log.go similarity index 100% rename from politeiad/backend/tlogbe/store/mysql/log.go rename to politeiad/backend/tstorebe/store/mysql/log.go diff --git a/politeiad/backend/tlogbe/store/mysql/mysql.go b/politeiad/backend/tstorebe/store/mysql/mysql.go similarity index 98% rename from politeiad/backend/tlogbe/store/mysql/mysql.go rename to politeiad/backend/tstorebe/store/mysql/mysql.go index 8215f51a2..f8fd0d436 100644 --- a/politeiad/backend/tlogbe/store/mysql/mysql.go +++ b/politeiad/backend/tstorebe/store/mysql/mysql.go @@ -11,7 +11,7 @@ import ( "net/url" "time" - "github.com/decred/politeia/politeiad/backend/tlogbe/store" + "github.com/decred/politeia/politeiad/backend/tstorebe/store" "github.com/google/uuid" ) diff --git a/politeiad/backend/tlogbe/store/store.go b/politeiad/backend/tstorebe/store/store.go similarity index 100% rename from politeiad/backend/tlogbe/store/store.go rename to politeiad/backend/tstorebe/store/store.go diff --git a/politeiad/backend/tlogbe/testing.go b/politeiad/backend/tstorebe/testing.go similarity index 55% rename from politeiad/backend/tlogbe/testing.go rename to politeiad/backend/tstorebe/testing.go index 3d177f3fb..7ae5743d7 100644 --- a/politeiad/backend/tlogbe/testing.go +++ b/politeiad/backend/tstorebe/testing.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlogbe +package tstorebe import ( "io/ioutil" @@ -10,31 +10,31 @@ import ( "path/filepath" "testing" - "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" + "github.com/decred/politeia/politeiad/backend/tstorebe/tstore" ) -// NewTestTlogBackend returns a tlogBackend that is setup for testing and a +// NewTestTstoreBackend returns a tstoreBackend that is setup for testing and a // closure that cleans up all test data when invoked. -func NewTestTlogBackend(t *testing.T) (*tlogBackend, func()) { +func NewTestTstoreBackend(t *testing.T) (*tstoreBackend, func()) { t.Helper() // Setup home dir and data dir - homeDir, err := ioutil.TempDir("", "tlogbackend.test") + homeDir, err := ioutil.TempDir("", "tstorebackend.test") if err != nil { t.Fatal(err) } dataDir := filepath.Join(homeDir, "data") - tlogBackend := tlogBackend{ + tstoreBackend := tstoreBackend{ homeDir: homeDir, dataDir: dataDir, - unvetted: tlog.NewTestTlogUnencrypted(t, dataDir, "unvetted"), - vetted: tlog.NewTestTlogEncrypted(t, dataDir, "vetted"), + unvetted: tstore.NewTestTstoreUnencrypted(t, dataDir, "unvetted"), + vetted: tstore.NewTestTstoreEncrypted(t, dataDir, "vetted"), prefixes: make(map[string][]byte), vettedTreeIDs: make(map[string]int64), } - return &tlogBackend, func() { + return &tstoreBackend, func() { err = os.RemoveAll(homeDir) if err != nil { t.Fatal(err) diff --git a/politeiad/backend/tlogbe/tlog/anchor.go b/politeiad/backend/tstorebe/tstore/anchor.go similarity index 96% rename from politeiad/backend/tlogbe/tlog/anchor.go rename to politeiad/backend/tstorebe/tstore/anchor.go index 2643f1206..042dc0731 100644 --- a/politeiad/backend/tlogbe/tlog/anchor.go +++ b/politeiad/backend/tstorebe/tstore/anchor.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlog +package tstore import ( "bytes" @@ -14,7 +14,7 @@ import ( dcrtime "github.com/decred/dcrtime/api/v2" "github.com/decred/dcrtime/merkle" - "github.com/decred/politeia/politeiad/backend/tlogbe/store" + "github.com/decred/politeia/politeiad/backend/tstorebe/store" "github.com/google/trillian" "github.com/google/trillian/types" "google.golang.org/grpc/codes" @@ -34,7 +34,7 @@ const ( // anchorID is included in the timestamp and verify requests as a // unique identifier. - anchorID = "tlogbe" + anchorID = "tstorebe" ) // anchor represents an anchor, i.e. timestamp, of a trillian tree at a @@ -50,14 +50,14 @@ type anchor struct { VerifyDigest *dcrtime.VerifyDigest `json:"verifydigest"` } -func (t *Tlog) droppingAnchorGet() bool { +func (t *Tstore) droppingAnchorGet() bool { t.Lock() defer t.Unlock() return t.droppingAnchor } -func (t *Tlog) droppingAnchorSet(b bool) { +func (t *Tstore) droppingAnchorSet(b bool) { t.Lock() defer t.Unlock() @@ -69,7 +69,7 @@ var ( ) // anchorForLeaf returns the anchor for a specific merkle leaf hash. -func (t *Tlog) anchorForLeaf(treeID int64, merkleLeafHash []byte, leaves []*trillian.LogLeaf) (*anchor, error) { +func (t *Tstore) anchorForLeaf(treeID int64, merkleLeafHash []byte, leaves []*trillian.LogLeaf) (*anchor, error) { // Find the leaf for the provided merkle leaf hash var l *trillian.LogLeaf for i, v := range leaves { @@ -151,7 +151,7 @@ func (t *Tlog) anchorForLeaf(treeID int64, merkleLeafHash []byte, leaves []*tril // anchorLatest returns the most recent anchor for the provided tree. A // errAnchorNotFound is returned if no anchor is found for the provided tree. -func (t *Tlog) anchorLatest(treeID int64) (*anchor, error) { +func (t *Tstore) anchorLatest(treeID int64) (*anchor, error) { // Get tree leaves leavesAll, err := t.trillian.leavesAll(treeID) if err != nil { @@ -201,7 +201,7 @@ func (t *Tlog) anchorLatest(treeID int64) (*anchor, error) { // anchorSave saves an anchor to the key-value store and appends a log leaf // to the trillian tree for the anchor. -func (t *Tlog) anchorSave(a anchor) error { +func (t *Tstore) anchorSave(a anchor) error { // Sanity checks switch { case a.TreeID == 0: @@ -273,7 +273,7 @@ func (t *Tlog) anchorSave(a anchor) error { // confirmations. Once the timestamp has been dropped, the anchor record is // saved to the key-value store and the record histories of the corresponding // timestamped trees are updated. -func (t *Tlog) anchorWait(anchors []anchor, digests []string) { +func (t *Tstore) anchorWait(anchors []anchor, digests []string) { // Verify we are not reentrant if t.droppingAnchorGet() { log.Errorf("waitForAchor: called reentrantly") @@ -437,7 +437,7 @@ func (t *Tlog) anchorWait(anchors []anchor, digests []string) { // time of function invocation. A SHA256 digest of the tree's log root at its // current height is timestamped onto the decred blockchain using the dcrtime // service. The anchor data is saved to the key-value store. -func (t *Tlog) anchorTrees() error { +func (t *Tstore) anchorTrees() error { log.Debugf("Start %v anchor process", t.id) // Ensure we are not reentrant diff --git a/politeiad/backend/tlogbe/tlog/tlogclient.go b/politeiad/backend/tstorebe/tstore/client.go similarity index 88% rename from politeiad/backend/tlogbe/tlog/tlogclient.go rename to politeiad/backend/tstorebe/tstore/client.go index bea3b8a75..d837ec138 100644 --- a/politeiad/backend/tlogbe/tlog/tlogclient.go +++ b/politeiad/backend/tstorebe/tstore/client.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlog +package tstore import ( "encoding/base64" @@ -11,18 +11,18 @@ import ( "fmt" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/store" + "github.com/decred/politeia/politeiad/backend/tstorebe/store" "github.com/google/trillian" "google.golang.org/grpc/codes" ) -// BlobSave saves a BlobEntry to the tlog instance. The BlobEntry will be -// encrypted prior to being written to disk if the tlog instance has an +// BlobSave saves a BlobEntry to the tstore instance. The BlobEntry will be +// encrypted prior to being written to disk if the tstore instance has an // encryption key set. The digest of the data, i.e. BlobEntry.Digest, can be -// thought of as the blob ID and can be used to get/del the blob from tlog. +// thought of as the blob ID and can be used to get/del the blob from tstore. // -// This function satisfies the plugins TlogClient interface. -func (t *Tlog) BlobSave(treeID int64, be store.BlobEntry) error { +// This function satisfies the plugins TstoreClient interface. +func (t *Tstore) BlobSave(treeID int64, be store.BlobEntry) error { log.Tracef("%v BlobSave: %v %v", t.id, treeID) // Parse the data descriptor @@ -98,8 +98,8 @@ func (t *Tlog) BlobSave(treeID int64, be store.BlobEntry) error { // BlobsDel deletes the blobs that correspond to the provided digests. // -// This function satisfies the plugins TlogClient interface. -func (t *Tlog) BlobsDel(treeID int64, digests [][]byte) error { +// This function satisfies the plugins TstoreClient interface. +func (t *Tstore) BlobsDel(treeID int64, digests [][]byte) error { log.Tracef("%v BlobsDel: %v %x", t.id, treeID, digests) // Verify tree exists. We allow blobs to be deleted from both @@ -149,8 +149,8 @@ func (t *Tlog) BlobsDel(treeID int64, digests [][]byte) error { // Blobs returns the blobs that correspond to the provided digests. If a blob // does not exist it will not be included in the returned map. // -// This function satisfies the plugins TlogClient interface. -func (t *Tlog) Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEntry, error) { +// This function satisfies the plugins TstoreClient interface. +func (t *Tstore) Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEntry, error) { log.Tracef("%v Blobs: %v %x", t.id, treeID, digests) // Verify tree exists @@ -233,8 +233,8 @@ func (t *Tlog) Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEntry // BlobsByDataDesc returns all blobs that match the provided data descriptor. // The blobs will be ordered from oldest to newest. // -// This function satisfies the plugins TlogClient interface. -func (t *Tlog) BlobsByDataDesc(treeID int64, dataDesc string) ([]store.BlobEntry, error) { +// This function satisfies the plugins TstoreClient interface. +func (t *Tstore) BlobsByDataDesc(treeID int64, dataDesc string) ([]store.BlobEntry, error) { log.Tracef("%v BlobsByDataDesc: %v %v", t.id, treeID, dataDesc) // Verify tree exists @@ -299,8 +299,8 @@ func (t *Tlog) BlobsByDataDesc(treeID int64, dataDesc string) ([]store.BlobEntry // DigestsByDataDesc returns the digests of all blobs that match the provided // data descriptor. // -// This function satisfies the plugins TlogClient interface. -func (t *Tlog) DigestsByDataDesc(treeID int64, dataDesc string) ([][]byte, error) { +// This function satisfies the plugins TstoreClient interface. +func (t *Tstore) DigestsByDataDesc(treeID int64, dataDesc string) ([][]byte, error) { log.Tracef("%v DigestsByDataDesc: %v %v", t.id, treeID, dataDesc) // Verify tree exists @@ -333,8 +333,8 @@ func (t *Tlog) DigestsByDataDesc(treeID int64, dataDesc string) ([][]byte, error // Timestamp returns the timestamp for the data blob that corresponds to the // provided digest. // -// This function satisfies the plugins TlogClient interface. -func (t *Tlog) Timestamp(treeID int64, digest []byte) (*backend.Timestamp, error) { +// This function satisfies the plugins TstoreClient interface. +func (t *Tstore) Timestamp(treeID int64, digest []byte) (*backend.Timestamp, error) { log.Tracef("%v Timestamp: %v %x", t.id, treeID, digest) // Get tree leaves diff --git a/politeiad/backend/tlogbe/tlog/convert.go b/politeiad/backend/tstorebe/tstore/convert.go similarity index 98% rename from politeiad/backend/tlogbe/tlog/convert.go rename to politeiad/backend/tstorebe/tstore/convert.go index 1145b889c..5ed3d6bd2 100644 --- a/politeiad/backend/tlogbe/tlog/convert.go +++ b/politeiad/backend/tstorebe/tstore/convert.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlog +package tstore import ( "bytes" @@ -12,7 +12,7 @@ import ( "fmt" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/store" + "github.com/decred/politeia/politeiad/backend/tstorebe/store" "github.com/decred/politeia/util" ) diff --git a/politeiad/backend/tlogbe/tlog/dcrtime.go b/politeiad/backend/tstorebe/tstore/dcrtime.go similarity index 99% rename from politeiad/backend/tlogbe/tlog/dcrtime.go rename to politeiad/backend/tstorebe/tstore/dcrtime.go index 10e0f60af..30758eda4 100644 --- a/politeiad/backend/tlogbe/tlog/dcrtime.go +++ b/politeiad/backend/tstorebe/tstore/dcrtime.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlog +package tstore import ( "bytes" diff --git a/politeiad/backend/tlogbe/tlog/encryptionkey.go b/politeiad/backend/tstorebe/tstore/encryptionkey.go similarity index 99% rename from politeiad/backend/tlogbe/tlog/encryptionkey.go rename to politeiad/backend/tstorebe/tstore/encryptionkey.go index 034e3ac89..b6d316ec7 100644 --- a/politeiad/backend/tlogbe/tlog/encryptionkey.go +++ b/politeiad/backend/tstorebe/tstore/encryptionkey.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlog +package tstore import ( "sync" diff --git a/politeiad/backend/tlogbe/log.go b/politeiad/backend/tstorebe/tstore/log.go similarity index 98% rename from politeiad/backend/tlogbe/log.go rename to politeiad/backend/tstorebe/tstore/log.go index 8a615c0a2..6345dad17 100644 --- a/politeiad/backend/tlogbe/log.go +++ b/politeiad/backend/tstorebe/tstore/log.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlogbe +package tstore import "github.com/decred/slog" diff --git a/politeiad/backend/tlogbe/tlog/plugin.go b/politeiad/backend/tstorebe/tstore/plugin.go similarity index 74% rename from politeiad/backend/tlogbe/tlog/plugin.go rename to politeiad/backend/tstorebe/tstore/plugin.go index 33d22c351..cce46575f 100644 --- a/politeiad/backend/tlogbe/tlog/plugin.go +++ b/politeiad/backend/tstorebe/tstore/plugin.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlog +package tstore import ( "errors" @@ -11,12 +11,12 @@ import ( "sort" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/comments" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/dcrdata" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/pi" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/ticketvote" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/usermd" + "github.com/decred/politeia/politeiad/backend/tstorebe/plugins" + "github.com/decred/politeia/politeiad/backend/tstorebe/plugins/comments" + "github.com/decred/politeia/politeiad/backend/tstorebe/plugins/dcrdata" + "github.com/decred/politeia/politeiad/backend/tstorebe/plugins/pi" + "github.com/decred/politeia/politeiad/backend/tstorebe/plugins/ticketvote" + "github.com/decred/politeia/politeiad/backend/tstorebe/plugins/usermd" cmplugin "github.com/decred/politeia/politeiad/plugins/comments" ddplugin "github.com/decred/politeia/politeiad/plugins/dcrdata" piplugin "github.com/decred/politeia/politeiad/plugins/pi" @@ -26,18 +26,18 @@ import ( const ( // pluginDataDirname is the plugin data directory name. It is - // located in the tlog instance data directory and is provided to + // located in the tstore instance data directory and is provided to // the plugins for storing plugin data. pluginDataDirname = "plugins" ) -// plugin represents a tlog plugin. +// plugin represents a tstore plugin. type plugin struct { id string client plugins.PluginClient } -func (t *Tlog) plugin(pluginID string) (plugin, bool) { +func (t *Tstore) plugin(pluginID string) (plugin, bool) { t.Lock() defer t.Unlock() @@ -45,7 +45,7 @@ func (t *Tlog) plugin(pluginID string) (plugin, bool) { return plugin, ok } -func (t *Tlog) pluginIDs() []string { +func (t *Tstore) pluginIDs() []string { t.Lock() defer t.Unlock() @@ -62,7 +62,7 @@ func (t *Tlog) pluginIDs() []string { return ids } -func (t *Tlog) PluginRegister(b backend.Backend, p backend.Plugin) error { +func (t *Tstore) PluginRegister(b backend.Backend, p backend.Plugin) error { log.Tracef("%v PluginRegister: %v", t.id, p.ID) var ( @@ -110,7 +110,7 @@ func (t *Tlog) PluginRegister(b backend.Backend, p backend.Plugin) error { return nil } -func (t *Tlog) PluginSetup(pluginID string) error { +func (t *Tstore) PluginSetup(pluginID string) error { log.Tracef("%v PluginSetup: %v", t.id, pluginID) p, ok := t.plugin(pluginID) @@ -121,7 +121,7 @@ func (t *Tlog) PluginSetup(pluginID string) error { return p.client.Setup() } -func (t *Tlog) PluginHookPre(treeID int64, token []byte, h plugins.HookT, payload string) error { +func (t *Tstore) PluginHookPre(treeID int64, token []byte, h plugins.HookT, payload string) error { log.Tracef("%v PluginHookPre: %v %v", t.id, treeID, plugins.Hooks[h]) // Pass hook event and payload to each plugin @@ -140,7 +140,7 @@ func (t *Tlog) PluginHookPre(treeID int64, token []byte, h plugins.HookT, payloa return nil } -func (t *Tlog) PluginHookPost(treeID int64, token []byte, h plugins.HookT, payload string) { +func (t *Tstore) PluginHookPost(treeID int64, token []byte, h plugins.HookT, payload string) { log.Tracef("%v PluginHookPost: %v %v", t.id, treeID, plugins.Hooks[h]) // Pass hook event and payload to each plugin @@ -153,7 +153,7 @@ func (t *Tlog) PluginHookPost(treeID int64, token []byte, h plugins.HookT, paylo err := p.client.Hook(treeID, token, h, payload) if err != nil { // This is the post plugin hook so the data has already been - // saved to tlog. We do not have the ability to unwind. Log + // saved to tstore. We do not have the ability to unwind. Log // the error and continue. log.Criticalf("%v PluginHookPost %v %v %v %x %v: %v", t.id, v, treeID, token, h, err, payload) @@ -162,7 +162,7 @@ func (t *Tlog) PluginHookPost(treeID int64, token []byte, h plugins.HookT, paylo } } -func (t *Tlog) PluginCmd(treeID int64, token []byte, pluginID, cmd, payload string) (string, error) { +func (t *Tstore) PluginCmd(treeID int64, token []byte, pluginID, cmd, payload string) (string, error) { log.Tracef("%v PluginCmd: %v %x %v %v", t.id, treeID, token, pluginID, cmd) // Get plugin @@ -175,8 +175,8 @@ func (t *Tlog) PluginCmd(treeID int64, token []byte, pluginID, cmd, payload stri return p.client.Cmd(treeID, token, cmd, payload) } -// Plugins returns all registered plugins for the tlog instance. -func (t *Tlog) Plugins() []backend.Plugin { +// Plugins returns all registered plugins for the tstore instance. +func (t *Tstore) Plugins() []backend.Plugin { log.Tracef("%v Plugins", t.id) t.Lock() diff --git a/politeiad/backend/tlogbe/tlog/recordindex.go b/politeiad/backend/tstorebe/tstore/recordindex.go similarity index 94% rename from politeiad/backend/tlogbe/tlog/recordindex.go rename to politeiad/backend/tstorebe/tstore/recordindex.go index 6e07c34ab..85031b26d 100644 --- a/politeiad/backend/tlogbe/tlog/recordindex.go +++ b/politeiad/backend/tstorebe/tstore/recordindex.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlog +package tstore import ( "encoding/hex" @@ -55,7 +55,7 @@ type recordIndex struct { // // TODO maybe it should be the same. The original iteration field // was to track unvetted changes in gitbe since unvetted gitbe - // records are not versioned. tlogbe unvetted records are versioned + // records are not versioned. tstorebe unvetted records are versioned // so the original use for the iteration field isn't needed anymore. Iteration uint32 `json:"iteration"` @@ -71,7 +71,7 @@ type recordIndex struct { // been frozen. This happens as a result of certain record status // changes. The only thing that can be appended onto a frozen tree // is one additional anchor record. Once a frozen tree has been - // anchored, the tlog fsck function will update the status of the + // anchored, the tstore fsck function will update the status of the // tree to frozen in trillian, at which point trillian will not // allow any additional leaves to be appended onto the tree. Frozen bool `json:"frozen,omitempty"` @@ -135,8 +135,8 @@ func parseRecordIndex(indexes []recordIndex, version uint32) (*recordIndex, erro return ri, nil } -// recordIndexSave saves a record index to tlog. -func (t *Tlog) recordIndexSave(treeID int64, ri recordIndex) error { +// recordIndexSave saves a record index to tstore. +func (t *Tstore) recordIndexSave(treeID int64, ri recordIndex) error { // Save record index to the store be, err := convertBlobEntryFromRecordIndex(ri) if err != nil { @@ -191,7 +191,7 @@ func (t *Tlog) recordIndexSave(treeID int64, ri recordIndex) error { // recordIndexes returns all record indexes found in the provided trillian // leaves. -func (t *Tlog) recordIndexes(leaves []*trillian.LogLeaf) ([]recordIndex, error) { +func (t *Tstore) recordIndexes(leaves []*trillian.LogLeaf) ([]recordIndex, error) { // Walk the leaves and compile the keys for all record indexes. It // is possible for multiple indexes to exist for the same record // version (they will have different iterations due to metadata @@ -275,7 +275,7 @@ func (t *Tlog) recordIndexes(leaves []*trillian.LogLeaf) ([]recordIndex, error) // recordIndex returns the specified version of a record index for a slice of // trillian leaves. -func (t *Tlog) recordIndex(leaves []*trillian.LogLeaf, version uint32) (*recordIndex, error) { +func (t *Tstore) recordIndex(leaves []*trillian.LogLeaf, version uint32) (*recordIndex, error) { indexes, err := t.recordIndexes(leaves) if err != nil { return nil, err @@ -285,6 +285,6 @@ func (t *Tlog) recordIndex(leaves []*trillian.LogLeaf, version uint32) (*recordI // recordIndexLatest returns the most recent record index for a slice of // trillian leaves. -func (t *Tlog) recordIndexLatest(leaves []*trillian.LogLeaf) (*recordIndex, error) { +func (t *Tstore) recordIndexLatest(leaves []*trillian.LogLeaf) (*recordIndex, error) { return t.recordIndex(leaves, 0) } diff --git a/politeiad/backend/tlogbe/tlog/testing.go b/politeiad/backend/tstorebe/tstore/testing.go similarity index 52% rename from politeiad/backend/tlogbe/tlog/testing.go rename to politeiad/backend/tstorebe/tstore/testing.go index 132044a06..5abc7fa19 100644 --- a/politeiad/backend/tlogbe/tlog/testing.go +++ b/politeiad/backend/tstorebe/tstore/testing.go @@ -2,22 +2,22 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlog +package tstore import ( "io/ioutil" "testing" - "github.com/decred/politeia/politeiad/backend/tlogbe/store/fs" + "github.com/decred/politeia/politeiad/backend/tstorebe/store/fs" "github.com/marcopeereboom/sbox" ) -func newTestTlog(t *testing.T, tlogID, dataDir string, encrypt bool) *Tlog { +func newTestTstore(t *testing.T, tstoreID, dataDir string, encrypt bool) *Tstore { t.Helper() - // Setup datadir for this tlog instance + // Setup datadir for this tstore instance var err error - dataDir, err = ioutil.TempDir(dataDir, tlogID) + dataDir, err = ioutil.TempDir(dataDir, tstoreID) if err != nil { t.Fatal(err) } @@ -39,22 +39,22 @@ func newTestTlog(t *testing.T, tlogID, dataDir string, encrypt bool) *Tlog { ek = newEncryptionKey(key) } - return &Tlog{ - id: tlogID, + return &Tstore{ + id: tstoreID, encryptionKey: ek, trillian: newTestTClient(t), store: store, } } -// NewTestTlogEncrypted returns a tlog instance that encrypts all data blobs +// NewTestTstoreEncrypted returns a tstore instance that encrypts all data blobs // and that has been setup for testing. -func NewTestTlogEncrypted(t *testing.T, tlogID, dataDir string) *Tlog { - return newTestTlog(t, tlogID, dataDir, true) +func NewTestTstoreEncrypted(t *testing.T, tstoreID, dataDir string) *Tstore { + return newTestTstore(t, tstoreID, dataDir, true) } -// NewTestTlogUnencrypted returns a tlog instance that save all data blobs +// NewTestTstoreUnencrypted returns a tstore instance that save all data blobs // as plaintext and that has been setup for testing. -func NewTestTlogUnencrypted(t *testing.T, tlogID, dataDir string) *Tlog { - return newTestTlog(t, tlogID, dataDir, false) +func NewTestTstoreUnencrypted(t *testing.T, tstoreID, dataDir string) *Tstore { + return newTestTstore(t, tstoreID, dataDir, false) } diff --git a/politeiad/backend/tlogbe/tlog/trillianclient.go b/politeiad/backend/tstorebe/tstore/trillianclient.go similarity index 99% rename from politeiad/backend/tlogbe/tlog/trillianclient.go rename to politeiad/backend/tstorebe/tstore/trillianclient.go index aaec3ac4e..7f3263b77 100644 --- a/politeiad/backend/tlogbe/tlog/trillianclient.go +++ b/politeiad/backend/tstorebe/tstore/trillianclient.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlog +package tstore import ( "bytes" @@ -39,7 +39,6 @@ import ( // QueuedLeaf, and the inclusion proof for that leaf. If the leaf append // command fails the QueuedLeaf will contain an error code from the failure and // the Proof will not be present. -// TODO I don't use this for anything. Just return the QueuedLeafProof. type queuedLeafProof struct { QueuedLeaf *trillian.QueuedLogLeaf Proof *trillian.Proof diff --git a/politeiad/backend/tlogbe/tlog/tlog.go b/politeiad/backend/tstorebe/tstore/tstore.go similarity index 92% rename from politeiad/backend/tlogbe/tlog/tlog.go rename to politeiad/backend/tstorebe/tstore/tstore.go index cd10c9be3..cd2e65531 100644 --- a/politeiad/backend/tlogbe/tlog/tlog.go +++ b/politeiad/backend/tstorebe/tstore/tstore.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlog +package tstore import ( "bytes" @@ -18,10 +18,10 @@ import ( "github.com/decred/dcrd/chaincfg/v3" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" - "github.com/decred/politeia/politeiad/backend/tlogbe/store" - "github.com/decred/politeia/politeiad/backend/tlogbe/store/fs" - "github.com/decred/politeia/politeiad/backend/tlogbe/store/mysql" + "github.com/decred/politeia/politeiad/backend/tstorebe/plugins" + "github.com/decred/politeia/politeiad/backend/tstorebe/store" + "github.com/decred/politeia/politeiad/backend/tstorebe/store/fs" + "github.com/decred/politeia/politeiad/backend/tstorebe/store/mysql" "github.com/decred/politeia/util" "github.com/google/trillian" "github.com/robfig/cron" @@ -45,11 +45,11 @@ const ( ) var ( - _ plugins.TlogClient = (*Tlog)(nil) + _ plugins.TstoreClient = (*Tstore)(nil) ) // We do not unwind. -type Tlog struct { +type Tstore struct { sync.Mutex id string dataDir string @@ -65,7 +65,7 @@ type Tlog struct { // will not be encrypted if this is left as nil. encryptionKey *encryptionKey - // droppingAnchor indicates whether tlog is in the process of + // droppingAnchor indicates whether tstore is in the process of // dropping an anchor, i.e. timestamping unanchored trillian trees // using dcrtime. An anchor is dropped periodically using cron. droppingAnchor bool @@ -106,7 +106,7 @@ func extraDataDecode(b []byte) (*extraData, error) { return &ed, nil } -func (t *Tlog) blobify(be store.BlobEntry) ([]byte, error) { +func (t *Tstore) blobify(be store.BlobEntry) ([]byte, error) { b, err := store.Blobify(be) if err != nil { return nil, err @@ -120,7 +120,7 @@ func (t *Tlog) blobify(be store.BlobEntry) ([]byte, error) { return b, nil } -func (t *Tlog) deblob(b []byte) (*store.BlobEntry, error) { +func (t *Tstore) deblob(b []byte) (*store.BlobEntry, error) { var err error if t.encryptionKey != nil { if !blobIsEncrypted(b) { @@ -138,7 +138,7 @@ func (t *Tlog) deblob(b []byte) (*store.BlobEntry, error) { return be, nil } -func (t *Tlog) TreeNew() (int64, error) { +func (t *Tstore) TreeNew() (int64, error) { log.Tracef("%v treeNew", t.id) tree, _, err := t.trillian.treeNew() @@ -149,7 +149,7 @@ func (t *Tlog) TreeNew() (int64, error) { return tree.TreeId, nil } -func (t *Tlog) TreeExists(treeID int64) bool { +func (t *Tstore) TreeExists(treeID int64) bool { log.Tracef("%v TreeExists: %v", t.id, treeID) _, err := t.trillian.tree(treeID) @@ -165,10 +165,10 @@ func (t *Tlog) TreeExists(treeID int64) bool { // Once the record index has been saved with its frozen field set, the tree // is considered to be frozen. The only thing that can be appended onto a // frozen tree is one additional anchor record. Once a frozen tree has been -// anchored, the tlog fsck function will update the status of the tree to +// anchored, the tstore fsck function will update the status of the tree to // frozen in trillian, at which point trillian will not allow any additional // leaves to be appended onto the tree. -func (t *Tlog) TreeFreeze(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, treePointer int64) error { +func (t *Tstore) TreeFreeze(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, treePointer int64) error { log.Tracef("%v TreeFreeze: %v", t.id, treeID) // Save metadata @@ -236,7 +236,7 @@ func (t *Tlog) TreeFreeze(treeID int64, rm backend.RecordMetadata, metadata []ba // TreePointer returns the tree pointer for the provided tree if one exists. // The returned bool will indicate if a tree pointer was found. -func (t *Tlog) TreePointer(treeID int64) (int64, bool) { +func (t *Tstore) TreePointer(treeID int64) (int64, bool) { log.Tracef("%v treePointer: %v", t.id, treeID) // Verify tree exists @@ -276,8 +276,8 @@ printErr: return 0, false } -// TreesAll returns the IDs of all trees in the tlog instance. -func (t *Tlog) TreesAll() ([]int64, error) { +// TreesAll returns the IDs of all trees in the tstore instance. +func (t *Tstore) TreesAll() ([]int64, error) { trees, err := t.trillian.treesAll() if err != nil { return nil, err @@ -289,7 +289,7 @@ func (t *Tlog) TreesAll() ([]int64, error) { return treeIDs, nil } -func (t *Tlog) treeIsFrozen(leaves []*trillian.LogLeaf) bool { +func (t *Tstore) treeIsFrozen(leaves []*trillian.LogLeaf) bool { r, err := t.recordIndexLatest(leaves) if err != nil { panic(err) @@ -334,7 +334,7 @@ type recordBlobsPrepareReply struct { // recordBlobsPrepare prepares the provided record content to be saved to // the blob kv store and appended onto a trillian tree. -func (t *Tlog) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) (*recordBlobsPrepareReply, error) { +func (t *Tstore) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) (*recordBlobsPrepareReply, error) { // Verify there are no duplicate or empty mdstream IDs mdstreamIDs := make(map[uint64]struct{}, len(metadata)) for _, v := range metadata { @@ -522,7 +522,7 @@ func (t *Tlog) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backen // recordBlobsSave saves the provided blobs to the kv store, appends a leaf // to the trillian tree for each blob, then updates the record index with the // trillian leaf information and returns it. -func (t *Tlog) recordBlobsSave(treeID int64, rbpr recordBlobsPrepareReply) (*recordIndex, error) { +func (t *Tstore) recordBlobsSave(treeID int64, rbpr recordBlobsPrepareReply) (*recordIndex, error) { log.Tracef("recordBlobsSave: %v", t.id, treeID) var ( @@ -607,11 +607,12 @@ func (t *Tlog) recordBlobsSave(treeID int64, rbpr recordBlobsPrepareReply) (*rec return &index, nil } -// RecordSave saves the provided record to tlog, creating a new version of the -// record (the record iteration also gets incremented on new versions). Once -// the record contents have been successfully saved to tlog, a recordIndex is -// created for this version of the record and saved to tlog as well. -func (t *Tlog) RecordSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { +// RecordSave saves the provided record to tstore, creating a new version of +// the record (the record iteration also gets incremented on new versions). +// Once the record contents have been successfully saved to tstore, a +// recordIndex is created for this version of the record and saved to tstore as +// well. +func (t *Tstore) RecordSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { log.Tracef("%v RecordSave: %v", t.id, treeID) // Verify tree exists @@ -734,7 +735,7 @@ func (t *Tlog) RecordSave(treeID int64, rm backend.RecordMetadata, metadata []ba // trillian tree. This code has been pulled out so that it can be called during // normal metadata updates as well as when an update requires a freeze record // to be saved along with the record index, such as when a record is censored. -func (t *Tlog) metadataSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) (*recordIndex, error) { +func (t *Tstore) metadataSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) (*recordIndex, error) { // Verify tree exists if !t.TreeExists(treeID) { return nil, backend.ErrRecordNotFound @@ -806,11 +807,11 @@ func (t *Tlog) metadataSave(treeID int64, rm backend.RecordMetadata, metadata [] return idx, nil } -// RecordMetadataSave saves the provided metadata to tlog, creating a new +// RecordMetadataSave saves the provided metadata to tstore, creating a new // iteration of the record while keeping the record version the same. Once the -// metadata has been successfully saved to tlog, a recordIndex is created for -// this iteration of the record and saved to tlog as well. -func (t *Tlog) RecordMetadataSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { +// metadata has been successfully saved to tstore, a recordIndex is created for +// this iteration of the record and saved to tstore as well. +func (t *Tstore) RecordMetadataSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { log.Tracef("%v RecordMetadataSave: %v", t.id, treeID) // Save metadata @@ -832,7 +833,7 @@ func (t *Tlog) RecordMetadataSave(treeID int64, rm backend.RecordMetadata, metad // that correspond to record files. This is done for all versions and all // iterations of the record. Record metadata and metadata stream blobs are not // deleted. -func (t *Tlog) RecordDel(treeID int64) error { +func (t *Tstore) RecordDel(treeID int64) error { log.Tracef("%v RecordDel: %v", t.id, treeID) // Verify tree exists @@ -908,7 +909,7 @@ func (t *Tlog) RecordDel(treeID int64) error { // onto a vetted tree. // // The tree pointer is also returned if one is found. -func (t *Tlog) RecordExists(treeID int64) bool { +func (t *Tstore) RecordExists(treeID int64) bool { log.Tracef("%v RecordExists: %v", t.id, treeID) // Verify tree exists @@ -957,7 +958,7 @@ printErr: // // Filenames can be used to request specific files. When filenames is not // empty, the only files that are returned will be those that are specified. -func (t *Tlog) record(treeID int64, version uint32, omitFiles bool, filenames []string) (*backend.Record, error) { +func (t *Tstore) record(treeID int64, version uint32, omitFiles bool, filenames []string) (*backend.Record, error) { // Verify tree exists if !t.TreeExists(treeID) { return nil, backend.ErrRecordNotFound @@ -1143,14 +1144,14 @@ func (t *Tlog) record(treeID int64, version uint32, omitFiles bool, filenames [] } // Record returns the specified version of the record. -func (t *Tlog) Record(treeID int64, version uint32) (*backend.Record, error) { +func (t *Tstore) Record(treeID int64, version uint32) (*backend.Record, error) { log.Tracef("%v record: %v %v", t.id, treeID, version) return t.record(treeID, version, false, []string{}) } // RecordLatest returns the latest version of a record. -func (t *Tlog) RecordLatest(treeID int64) (*backend.Record, error) { +func (t *Tstore) RecordLatest(treeID int64) (*backend.Record, error) { log.Tracef("%v RecordLatest: %v", t.id, treeID) return t.record(treeID, 0, false, []string{}) @@ -1159,14 +1160,14 @@ func (t *Tlog) RecordLatest(treeID int64) (*backend.Record, error) { // RecordPartial returns a partial record. This method gives the caller fine // grained control over what version and what files are returned. The only // required field is the token. All other fields are optional. -func (t *Tlog) RecordPartial(treeID int64, version uint32, omitFiles bool, filenames []string) (*backend.Record, error) { +func (t *Tstore) RecordPartial(treeID int64, version uint32, omitFiles bool, filenames []string) (*backend.Record, error) { log.Tracef("%v RecordPartial: %v %v %v %v", t.id, treeID, version, omitFiles, filenames) return t.record(treeID, version, omitFiles, filenames) } -func (t *Tlog) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trillian.LogLeaf) (*backend.Timestamp, error) { +func (t *Tstore) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trillian.LogLeaf) (*backend.Timestamp, error) { // Find the leaf var l *trillian.LogLeaf for _, v := range leaves { @@ -1304,7 +1305,7 @@ func (t *Tlog) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trillian return &ts, nil } -func (t *Tlog) RecordTimestamps(treeID int64, version uint32, token []byte) (*backend.RecordTimestamps, error) { +func (t *Tstore) RecordTimestamps(treeID int64, version uint32, token []byte) (*backend.RecordTimestamps, error) { log.Tracef("%v RecordTimestamps: %v %v", t.id, treeID, version) // Verify tree exists @@ -1358,14 +1359,14 @@ func (t *Tlog) RecordTimestamps(treeID int64, version uint32, token []byte) (*ba } // TODO run fsck episodically -func (t *Tlog) Fsck() { +func (t *Tstore) Fsck() { // Set tree status to frozen for any trees that are frozen and have // been anchored one last time. // Failed censor. Ensure all blobs have been deleted from all // record versions of a censored record. } -func (t *Tlog) Close() { +func (t *Tstore) Close() { log.Tracef("%v Close", t.id) // Close connections @@ -1378,7 +1379,7 @@ func (t *Tlog) Close() { } } -func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianKeyFile, encryptionKeyFile, dbType, dbHost, dbRootCert, dbCert, dbKey, dcrtimeHost, dcrtimeCert string) (*Tlog, error) { +func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianKeyFile, encryptionKeyFile, dbType, dbHost, dbRootCert, dbCert, dbKey, dcrtimeHost, dcrtimeCert string) (*Tstore, error) { // Load encryption key if provided. An encryption key is optional. var ek *encryptionKey if encryptionKeyFile != "" { @@ -1400,7 +1401,7 @@ func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trilli log.Infof("Encryption key %v: %v", id, encryptionKeyFile) } - // Setup datadir for this tlog instance + // Setup datadir for this tstore instance dataDir = filepath.Join(dataDir, id) err := os.MkdirAll(dataDir, 0700) if err != nil { @@ -1453,8 +1454,8 @@ func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trilli return nil, err } - // Setup tlog - t := Tlog{ + // Setup tstore + t := Tstore{ id: id, dataDir: dataDir, activeNetParams: anp, diff --git a/politeiad/backend/tlogbe/tlog/verify.go b/politeiad/backend/tstorebe/tstore/verify.go similarity index 99% rename from politeiad/backend/tlogbe/tlog/verify.go rename to politeiad/backend/tstorebe/tstore/verify.go index 762ba25cf..6caea0322 100644 --- a/politeiad/backend/tlogbe/tlog/verify.go +++ b/politeiad/backend/tstorebe/tstore/verify.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlog +package tstore import ( "crypto/sha256" diff --git a/politeiad/backend/tlogbe/tlogbe_test.go b/politeiad/backend/tstorebe/tstore_test.go similarity index 90% rename from politeiad/backend/tlogbe/tlogbe_test.go rename to politeiad/backend/tstorebe/tstore_test.go index 3ca1beed5..c7c268199 100644 --- a/politeiad/backend/tlogbe/tlogbe_test.go +++ b/politeiad/backend/tstorebe/tstore_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlogbe +package tstorebe import ( "bytes" @@ -355,7 +355,7 @@ func setupRecordContentTests(t *testing.T) []recordContentTest { } func TestNewRecord(t *testing.T) { - tlogBackend, cleanup := NewTestTlogBackend(t) + tstoreBackend, cleanup := NewTestTstoreBackend(t) defer cleanup() // Test all record content verification error through the New endpoint @@ -363,7 +363,7 @@ func TestNewRecord(t *testing.T) { for _, test := range recordContentTests { t.Run(test.description, func(t *testing.T) { // Make backend call - _, err := tlogBackend.New(test.metadata, test.files) + _, err := tstoreBackend.New(test.metadata, test.files) // Parse error var contentError backend.ContentVerificationError @@ -384,7 +384,7 @@ func TestNewRecord(t *testing.T) { fs := []backend.File{ newBackendFile(t, "index.md"), } - _, err := tlogBackend.New(md, fs) + _, err := tstoreBackend.New(md, fs) if err != nil { t.Errorf("success case failed with %v", err) } @@ -392,7 +392,7 @@ func TestNewRecord(t *testing.T) { /* func TestUpdateUnvettedRecord(t *testing.T) { - tlogBackend, cleanup := NewTestTlogBackend(t) + tstoreBackend, cleanup := NewTestTstoreBackend(t) defer cleanup() // Create new record @@ -402,7 +402,7 @@ func TestUpdateUnvettedRecord(t *testing.T) { fs := []backend.File{ newBackendFile(t, "index.md"), } - rec, err := tlogBackend.New(md, fs) + rec, err := tstoreBackend.New(md, fs) if err != nil { t.Fatal(err) } @@ -423,7 +423,7 @@ func TestUpdateUnvettedRecord(t *testing.T) { } // Make backend call - _, err = tlogBackend.UpdateUnvettedRecord(token, test.metadata, + _, err = tstoreBackend.UpdateUnvettedRecord(token, test.metadata, []backend.MetadataStream{}, test.files, test.filesDel) // Parse error @@ -451,7 +451,7 @@ func TestUpdateUnvettedRecord(t *testing.T) { tokenRandom := tokenFromTreeID(123) // test case: Frozen tree - recFrozen, err := tlogBackend.New(md, fs) + recFrozen, err := tstoreBackend.New(md, fs) if err != nil { t.Fatal(err) } @@ -459,7 +459,7 @@ func TestUpdateUnvettedRecord(t *testing.T) { if err != nil { t.Fatal(err) } - err = tlogBackend.unvetted.treeFreeze(treeIDFromToken(tokenFrozen), + err = tstoreBackend.unvetted.treeFreeze(treeIDFromToken(tokenFrozen), backend.RecordMetadata{}, []backend.MetadataStream{}, 0) if err != nil { t.Fatal(err) @@ -532,7 +532,7 @@ func TestUpdateUnvettedRecord(t *testing.T) { for _, test := range tests { t.Run(test.description, func(t *testing.T) { // Make backend call - _, err = tlogBackend.UpdateUnvettedRecord(test.token, + _, err = tstoreBackend.UpdateUnvettedRecord(test.token, test.mdAppend, test.mdOverwrite, test.filesAdd, test.filesDel) // Parse error @@ -566,7 +566,7 @@ func TestUpdateUnvettedRecord(t *testing.T) { } func TestUpdateVettedRecord(t *testing.T) { - tlogBackend, cleanup := NewTestTlogBackend(t) + tstoreBackend, cleanup := NewTestTstoreBackend(t) defer cleanup() // Create new record @@ -576,7 +576,7 @@ func TestUpdateVettedRecord(t *testing.T) { fs := []backend.File{ newBackendFile(t, "index.md"), } - rec, err := tlogBackend.New(md, fs) + rec, err := tstoreBackend.New(md, fs) if err != nil { t.Fatal(err) } @@ -587,7 +587,7 @@ func TestUpdateVettedRecord(t *testing.T) { md = append(md, newBackendMetadataStream(t, 2, "")) // Publish the created record - err = tlogBackend.unvettedPublish(token, *rec, md, fs) + err = tstoreBackend.unvettedPublish(token, *rec, md, fs) if err != nil { t.Fatal(err) } @@ -604,7 +604,7 @@ func TestUpdateVettedRecord(t *testing.T) { } // Make backend call - _, err = tlogBackend.UpdateVettedRecord(token, test.metadata, + _, err = tstoreBackend.UpdateVettedRecord(token, test.metadata, []backend.MetadataStream{}, test.files, test.filesDel) // Parse error @@ -632,7 +632,7 @@ func TestUpdateVettedRecord(t *testing.T) { tokenRandom := tokenFromTreeID(123) // test case: Frozen tree - recFrozen, err := tlogBackend.New(md, fs) + recFrozen, err := tstoreBackend.New(md, fs) if err != nil { t.Fatal(err) } @@ -641,12 +641,12 @@ func TestUpdateVettedRecord(t *testing.T) { t.Fatal(err) } md = append(md, newBackendMetadataStream(t, 3, "")) - err = tlogBackend.unvettedPublish(tokenFrozen, *recFrozen, md, fs) + err = tstoreBackend.unvettedPublish(tokenFrozen, *recFrozen, md, fs) if err != nil { t.Fatal(err) } - treeIDFrozenVetted := tlogBackend.vettedTreeIDs[recFrozen.Token] - err = tlogBackend.vetted.treeFreeze(treeIDFrozenVetted, + treeIDFrozenVetted := tstoreBackend.vettedTreeIDs[recFrozen.Token] + err = tstoreBackend.vetted.treeFreeze(treeIDFrozenVetted, backend.RecordMetadata{}, []backend.MetadataStream{}, 0) if err != nil { t.Fatal(err) @@ -719,7 +719,7 @@ func TestUpdateVettedRecord(t *testing.T) { for _, test := range tests { t.Run(test.description, func(t *testing.T) { // Make backend call - _, err = tlogBackend.UpdateVettedRecord(test.token, + _, err = tstoreBackend.UpdateVettedRecord(test.token, test.mdAppend, test.mdOverwirte, test.filesAdd, test.filesDel) // Parse error @@ -753,7 +753,7 @@ func TestUpdateVettedRecord(t *testing.T) { } func TestUpdateUnvettedMetadata(t *testing.T) { - tlogBackend, cleanup := NewTestTlogBackend(t) + tstoreBackend, cleanup := NewTestTstoreBackend(t) defer cleanup() // Create new record @@ -763,7 +763,7 @@ func TestUpdateUnvettedMetadata(t *testing.T) { fs := []backend.File{ newBackendFile(t, "index.md"), } - rec, err := tlogBackend.New(md, fs) + rec, err := tstoreBackend.New(md, fs) if err != nil { t.Fatal(err) } @@ -778,7 +778,7 @@ func TestUpdateUnvettedMetadata(t *testing.T) { for _, test := range recordContentTests { t.Run(test.description, func(t *testing.T) { // Make backend call - err := tlogBackend.UpdateUnvettedMetadata(token, + err := tstoreBackend.UpdateUnvettedMetadata(token, test.metadata, []backend.MetadataStream{}) // Parse error @@ -803,7 +803,7 @@ func TestUpdateUnvettedMetadata(t *testing.T) { tokenRandom := tokenFromTreeID(123) // test case: Frozen tree - recFrozen, err := tlogBackend.New(md, fs) + recFrozen, err := tstoreBackend.New(md, fs) if err != nil { t.Fatal(err) } @@ -811,7 +811,7 @@ func TestUpdateUnvettedMetadata(t *testing.T) { if err != nil { t.Fatal(err) } - err = tlogBackend.unvetted.treeFreeze(treeIDFromToken(tokenFrozen), + err = tstoreBackend.unvetted.treeFreeze(treeIDFromToken(tokenFrozen), backend.RecordMetadata{}, []backend.MetadataStream{}, 0) if err != nil { t.Fatal(err) @@ -897,7 +897,7 @@ func TestUpdateUnvettedMetadata(t *testing.T) { for _, test := range tests { t.Run(test.description, func(t *testing.T) { // Make backend call - err = tlogBackend.UpdateUnvettedMetadata(test.token, + err = tstoreBackend.UpdateUnvettedMetadata(test.token, test.mdAppend, test.mdOverwrite) // Parse error @@ -931,7 +931,7 @@ func TestUpdateUnvettedMetadata(t *testing.T) { } func TestUpdateVettedMetadata(t *testing.T) { - tlogBackend, cleanup := NewTestTlogBackend(t) + tstoreBackend, cleanup := NewTestTstoreBackend(t) defer cleanup() // Create new record @@ -941,7 +941,7 @@ func TestUpdateVettedMetadata(t *testing.T) { fs := []backend.File{ newBackendFile(t, "index.md"), } - rec, err := tlogBackend.New(md, fs) + rec, err := tstoreBackend.New(md, fs) if err != nil { t.Fatal(err) } @@ -950,7 +950,7 @@ func TestUpdateVettedMetadata(t *testing.T) { t.Fatal(err) } md = append(md, newBackendMetadataStream(t, 2, "")) - err = tlogBackend.unvettedPublish(token, *rec, md, fs) + err = tstoreBackend.unvettedPublish(token, *rec, md, fs) if err != nil { t.Fatal(err) } @@ -961,7 +961,7 @@ func TestUpdateVettedMetadata(t *testing.T) { for _, test := range recordContentTests { t.Run(test.description, func(t *testing.T) { // Make backend call - err := tlogBackend.UpdateVettedMetadata(token, + err := tstoreBackend.UpdateVettedMetadata(token, test.metadata, []backend.MetadataStream{}) // Parse error @@ -986,7 +986,7 @@ func TestUpdateVettedMetadata(t *testing.T) { tokenRandom := tokenFromTreeID(123) // test case: Frozen tree - recFrozen, err := tlogBackend.New(md, fs) + recFrozen, err := tstoreBackend.New(md, fs) if err != nil { t.Fatal(err) } @@ -995,12 +995,12 @@ func TestUpdateVettedMetadata(t *testing.T) { t.Fatal(err) } md = append(md, newBackendMetadataStream(t, 3, "")) - err = tlogBackend.unvettedPublish(tokenFrozen, *recFrozen, md, fs) + err = tstoreBackend.unvettedPublish(tokenFrozen, *recFrozen, md, fs) if err != nil { t.Fatal(err) } - treeIDFrozenVetted := tlogBackend.vettedTreeIDs[recFrozen.Token] - err = tlogBackend.vetted.treeFreeze(treeIDFrozenVetted, + treeIDFrozenVetted := tstoreBackend.vettedTreeIDs[recFrozen.Token] + err = tstoreBackend.vetted.treeFreeze(treeIDFrozenVetted, backend.RecordMetadata{}, []backend.MetadataStream{}, 0) if err != nil { t.Fatal(err) @@ -1081,7 +1081,7 @@ func TestUpdateVettedMetadata(t *testing.T) { for _, test := range tests { t.Run(test.description, func(t *testing.T) { // Make backend call - err = tlogBackend.UpdateVettedMetadata(test.token, + err = tstoreBackend.UpdateVettedMetadata(test.token, test.mdAppend, test.mdOverwrite) // Parse error @@ -1115,7 +1115,7 @@ func TestUpdateVettedMetadata(t *testing.T) { } func TestUnvettedExists(t *testing.T) { - tlogBackend, cleanup := NewTestTlogBackend(t) + tstoreBackend, cleanup := NewTestTstoreBackend(t) defer cleanup() // Create new record @@ -1125,7 +1125,7 @@ func TestUnvettedExists(t *testing.T) { fs := []backend.File{ newBackendFile(t, "index.md"), } - rec, err := tlogBackend.New(md, fs) + rec, err := tstoreBackend.New(md, fs) if err != nil { t.Fatal(err) } @@ -1140,19 +1140,19 @@ func TestUnvettedExists(t *testing.T) { // Run UnvettedExists test cases // // Record exists - result := tlogBackend.UnvettedExists(token) + result := tstoreBackend.UnvettedExists(token) if result == false { t.Errorf("got false, want true") } // Record does not exist - result = tlogBackend.UnvettedExists(tokenRandom) + result = tstoreBackend.UnvettedExists(tokenRandom) if result == true { t.Errorf("got true, want false") } } func TestVettedExists(t *testing.T) { - tlogBackend, cleanup := NewTestTlogBackend(t) + tstoreBackend, cleanup := NewTestTstoreBackend(t) defer cleanup() // Create unvetted record @@ -1162,7 +1162,7 @@ func TestVettedExists(t *testing.T) { fs := []backend.File{ newBackendFile(t, "index.md"), } - unvetted, err := tlogBackend.New(md, fs) + unvetted, err := tstoreBackend.New(md, fs) if err != nil { t.Fatal(err) } @@ -1172,7 +1172,7 @@ func TestVettedExists(t *testing.T) { } // Create vetted record - vetted, err := tlogBackend.New(md, fs) + vetted, err := tstoreBackend.New(md, fs) if err != nil { t.Fatal(err) } @@ -1181,7 +1181,7 @@ func TestVettedExists(t *testing.T) { t.Fatal(err) } md = append(md, newBackendMetadataStream(t, 2, "")) - err = tlogBackend.unvettedPublish(tokenVetted, *vetted, md, fs) + err = tstoreBackend.unvettedPublish(tokenVetted, *vetted, md, fs) if err != nil { t.Fatal(err) } @@ -1189,19 +1189,19 @@ func TestVettedExists(t *testing.T) { // Run VettedExists test cases // // Record exists - result := tlogBackend.VettedExists(tokenVetted) + result := tstoreBackend.VettedExists(tokenVetted) if result == false { t.Fatal("got false, want true") } // Record does not exist - result = tlogBackend.VettedExists(tokenUnvetted) + result = tstoreBackend.VettedExists(tokenUnvetted) if result == true { t.Fatal("got true, want false") } } func TestGetUnvetted(t *testing.T) { - tlogBackend, cleanup := NewTestTlogBackend(t) + tstoreBackend, cleanup := NewTestTstoreBackend(t) defer cleanup() // Create new record @@ -1211,7 +1211,7 @@ func TestGetUnvetted(t *testing.T) { fs := []backend.File{ newBackendFile(t, "index.md"), } - rec, err := tlogBackend.New(md, fs) + rec, err := tstoreBackend.New(md, fs) if err != nil { t.Fatal(err) } @@ -1224,26 +1224,26 @@ func TestGetUnvetted(t *testing.T) { tokenRandom := tokenFromTreeID(123) // Bad version error - _, err = tlogBackend.GetUnvetted(token, "badversion") + _, err = tstoreBackend.GetUnvetted(token, "badversion") if err != backend.ErrRecordNotFound { t.Errorf("got error %v, want %v", err, backend.ErrRecordNotFound) } // Bad token error - _, err = tlogBackend.GetUnvetted(tokenRandom, "") + _, err = tstoreBackend.GetUnvetted(tokenRandom, "") if err != backend.ErrRecordNotFound { t.Errorf("got error %v, want %v", err, backend.ErrRecordNotFound) } // Success - _, err = tlogBackend.GetUnvetted(token, "") + _, err = tstoreBackend.GetUnvetted(token, "") if err != nil { t.Errorf("got error %v, want nil", err) } } func TestGetVetted(t *testing.T) { - tlogBackend, cleanup := NewTestTlogBackend(t) + tstoreBackend, cleanup := NewTestTstoreBackend(t) defer cleanup() // Create new record @@ -1253,7 +1253,7 @@ func TestGetVetted(t *testing.T) { fs := []backend.File{ newBackendFile(t, "index.md"), } - rec, err := tlogBackend.New(md, fs) + rec, err := tstoreBackend.New(md, fs) if err != nil { t.Fatal(err) } @@ -1262,7 +1262,7 @@ func TestGetVetted(t *testing.T) { t.Fatal(err) } md = append(md, newBackendMetadataStream(t, 2, "")) - err = tlogBackend.unvettedPublish(token, *rec, md, fs) + err = tstoreBackend.unvettedPublish(token, *rec, md, fs) if err != nil { t.Fatal(err) } @@ -1271,26 +1271,26 @@ func TestGetVetted(t *testing.T) { tokenRandom := tokenFromTreeID(123) // Bad version error - _, err = tlogBackend.GetVetted(token, "badversion") + _, err = tstoreBackend.GetVetted(token, "badversion") if err != backend.ErrRecordNotFound { t.Errorf("got error %v, want %v", err, backend.ErrRecordNotFound) } // Bad token error - _, err = tlogBackend.GetVetted(tokenRandom, "") + _, err = tstoreBackend.GetVetted(tokenRandom, "") if err != backend.ErrRecordNotFound { t.Errorf("got error %v, want %v", err, backend.ErrRecordNotFound) } // Success - _, err = tlogBackend.GetVetted(token, "") + _, err = tstoreBackend.GetVetted(token, "") if err != nil { t.Errorf("got error %v, want nil", err) } } func TestSetUnvettedStatus(t *testing.T) { - tlogBackend, cleanup := NewTestTlogBackend(t) + tstoreBackend, cleanup := NewTestTstoreBackend(t) defer cleanup() // Helpers @@ -1304,7 +1304,7 @@ func TestSetUnvettedStatus(t *testing.T) { // Invalid status transitions // // test case: Unvetted to archived - recUnvetToArch, err := tlogBackend.New(md, fs) + recUnvetToArch, err := tstoreBackend.New(md, fs) if err != nil { t.Fatal(err) } @@ -1313,7 +1313,7 @@ func TestSetUnvettedStatus(t *testing.T) { t.Fatal(err) } // test case: Unvetted to unvetted - recUnvetToUnvet, err := tlogBackend.New(md, fs) + recUnvetToUnvet, err := tstoreBackend.New(md, fs) if err != nil { t.Fatal(err) } @@ -1325,7 +1325,7 @@ func TestSetUnvettedStatus(t *testing.T) { // Valid status transitions // // test case: Unvetted to vetted - recUnvetToVet, err := tlogBackend.New(md, fs) + recUnvetToVet, err := tstoreBackend.New(md, fs) if err != nil { t.Fatal(err) } @@ -1334,7 +1334,7 @@ func TestSetUnvettedStatus(t *testing.T) { t.Fatal(err) } // test case: Unvetted to censored - recUnvetToCensored, err := tlogBackend.New(md, fs) + recUnvetToCensored, err := tstoreBackend.New(md, fs) if err != nil { t.Fatal(err) } @@ -1429,7 +1429,7 @@ func TestSetUnvettedStatus(t *testing.T) { for _, test := range tests { t.Run(test.description, func(t *testing.T) { // Make backend call - _, err = tlogBackend.SetUnvettedStatus(test.token, test.status, + _, err = tstoreBackend.SetUnvettedStatus(test.token, test.status, test.mdAppend, test.mdOverwrite) // Parse error @@ -1463,7 +1463,7 @@ func TestSetUnvettedStatus(t *testing.T) { } func TestSetVettedStatus(t *testing.T) { - tlogBackend, cleanup := NewTestTlogBackend(t) + tstoreBackend, cleanup := NewTestTstoreBackend(t) defer cleanup() // Helpers @@ -1477,7 +1477,7 @@ func TestSetVettedStatus(t *testing.T) { // Invalid status transitions // // test case: Vetted to unvetted - recVetToUnvet, err := tlogBackend.New(md, fs) + recVetToUnvet, err := tstoreBackend.New(md, fs) if err != nil { t.Fatal(err) } @@ -1487,13 +1487,13 @@ func TestSetVettedStatus(t *testing.T) { } md = append(md, newBackendMetadataStream(t, 2, "")) - _, err = tlogBackend.SetUnvettedStatus(tokenVetToUnvet, + _, err = tstoreBackend.SetUnvettedStatus(tokenVetToUnvet, backend.MDStatusVetted, md, []backend.MetadataStream{}) if err != nil { t.Fatal(err) } // test case: Vetted to vetted - recVetToVet, err := tlogBackend.New(md, fs) + recVetToVet, err := tstoreBackend.New(md, fs) if err != nil { t.Fatal(err) } @@ -1502,7 +1502,7 @@ func TestSetVettedStatus(t *testing.T) { t.Fatal(err) } md = append(md, newBackendMetadataStream(t, 3, "")) - _, err = tlogBackend.SetUnvettedStatus(tokenVetToVet, + _, err = tstoreBackend.SetUnvettedStatus(tokenVetToVet, backend.MDStatusVetted, md, []backend.MetadataStream{}) if err != nil { t.Fatal(err) @@ -1511,7 +1511,7 @@ func TestSetVettedStatus(t *testing.T) { // Valid status transitions // // test case: Vetted to archived - recVetToArch, err := tlogBackend.New(md, fs) + recVetToArch, err := tstoreBackend.New(md, fs) if err != nil { t.Fatal(err) } @@ -1520,13 +1520,13 @@ func TestSetVettedStatus(t *testing.T) { t.Fatal(err) } md = append(md, newBackendMetadataStream(t, 4, "")) - _, err = tlogBackend.SetUnvettedStatus(tokenVetToArch, + _, err = tstoreBackend.SetUnvettedStatus(tokenVetToArch, backend.MDStatusVetted, md, []backend.MetadataStream{}) if err != nil { t.Fatal(err) } // test case: Vetted to censored - recVetToCensored, err := tlogBackend.New(md, fs) + recVetToCensored, err := tstoreBackend.New(md, fs) if err != nil { t.Fatal(err) } @@ -1535,7 +1535,7 @@ func TestSetVettedStatus(t *testing.T) { t.Fatal(err) } md = append(md, newBackendMetadataStream(t, 5, "")) - _, err = tlogBackend.SetUnvettedStatus(tokenVetToCensored, + _, err = tstoreBackend.SetUnvettedStatus(tokenVetToCensored, backend.MDStatusVetted, md, []backend.MetadataStream{}) if err != nil { t.Fatal(err) @@ -1627,7 +1627,7 @@ func TestSetVettedStatus(t *testing.T) { for _, test := range tests { t.Run(test.description, func(t *testing.T) { // Make backend call - _, err = tlogBackend.SetVettedStatus(test.token, test.status, + _, err = tstoreBackend.SetVettedStatus(test.token, test.status, test.mdAppend, test.mdOverwrite) // Parse error diff --git a/politeiad/backend/tlogbe/tlogbe.go b/politeiad/backend/tstorebe/tstorebe.go similarity index 90% rename from politeiad/backend/tlogbe/tlogbe.go rename to politeiad/backend/tstorebe/tstorebe.go index f29fe797d..4e7167bbd 100644 --- a/politeiad/backend/tlogbe/tlogbe.go +++ b/politeiad/backend/tstorebe/tstorebe.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package tlogbe +package tstorebe import ( "bytes" @@ -23,33 +23,33 @@ import ( v1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/mime" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins" - "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" + "github.com/decred/politeia/politeiad/backend/tstorebe/plugins" + "github.com/decred/politeia/politeiad/backend/tstorebe/tstore" "github.com/decred/politeia/util" "github.com/marcopeereboom/sbox" "github.com/subosito/gozaru" ) const ( - defaultEncryptionKeyFilename = "tlogbe.key" + defaultEncryptionKeyFilename = "tstorebe.key" - // Tlog instance IDs - tlogIDUnvetted = "unvetted" - tlogIDVetted = "vetted" + // Tstore instance IDs + tstoreIDUnvetted = "unvetted" + tstoreIDVetted = "vetted" ) var ( - _ backend.Backend = (*tlogBackend)(nil) + _ backend.Backend = (*tstoreBackend)(nil) ) -// tlogBackend implements the Backend interface. -type tlogBackend struct { +// tstoreBackend implements the Backend interface. +type tstoreBackend struct { sync.RWMutex homeDir string dataDir string shutdown bool - unvetted *tlog.Tlog - vetted *tlog.Tlog + unvetted *tstore.Tstore + vetted *tstore.Tstore // prefixes contains the prefix to full token mapping for unvetted // records. The prefix is the first n characters of the hex encoded @@ -82,7 +82,7 @@ func tokenFromTreeID(treeID int64) []byte { } func tokenIsFullLength(token []byte) bool { - return util.TokenIsFullLength(util.TokenTypeTlog, token) + return util.TokenIsFullLength(util.TokenTypeTstore, token) } func treeIDFromToken(token []byte) int64 { @@ -93,7 +93,7 @@ func treeIDFromToken(token []byte) int64 { } // recordMutex returns the mutex for a record. -func (t *tlogBackend) recordMutex(token []byte) *sync.Mutex { +func (t *tstoreBackend) recordMutex(token []byte) *sync.Mutex { t.Lock() defer t.Unlock() @@ -111,7 +111,7 @@ func (t *tlogBackend) recordMutex(token []byte) *sync.Mutex { // fullLengthToken returns the full length token given the token prefix. // // This function must be called WITHOUT the lock held. -func (t *tlogBackend) fullLengthToken(prefix []byte) ([]byte, bool) { +func (t *tstoreBackend) fullLengthToken(prefix []byte) ([]byte, bool) { t.Lock() defer t.Unlock() @@ -123,7 +123,7 @@ func (t *tlogBackend) fullLengthToken(prefix []byte) ([]byte, bool) { // This can be either the full length token or the token prefix. // // This function must be called WITHOUT the lock held. -func (t *tlogBackend) unvettedTreeIDFromToken(token []byte) int64 { +func (t *tstoreBackend) unvettedTreeIDFromToken(token []byte) int64 { if len(token) == util.TokenPrefixSize() { // This is a token prefix. Get the full token from the cache. var ok bool @@ -135,7 +135,7 @@ func (t *tlogBackend) unvettedTreeIDFromToken(token []byte) int64 { return treeIDFromToken(token) } -func (t *tlogBackend) prefixExists(fullToken []byte) bool { +func (t *tstoreBackend) prefixExists(fullToken []byte) bool { t.RLock() defer t.RUnlock() @@ -143,7 +143,7 @@ func (t *tlogBackend) prefixExists(fullToken []byte) bool { return ok } -func (t *tlogBackend) prefixAdd(fullToken []byte) { +func (t *tstoreBackend) prefixAdd(fullToken []byte) { t.Lock() defer t.Unlock() @@ -153,7 +153,7 @@ func (t *tlogBackend) prefixAdd(fullToken []byte) { log.Debugf("Add token prefix: %v", prefix) } -func (t *tlogBackend) vettedTreeID(token []byte) (int64, bool) { +func (t *tstoreBackend) vettedTreeID(token []byte) (int64, bool) { t.RLock() defer t.RUnlock() @@ -161,7 +161,7 @@ func (t *tlogBackend) vettedTreeID(token []byte) (int64, bool) { return treeID, ok } -func (t *tlogBackend) vettedTreeIDAdd(token string, treeID int64) { +func (t *tstoreBackend) vettedTreeIDAdd(token string, treeID int64) { t.Lock() defer t.Unlock() @@ -175,7 +175,7 @@ func (t *tlogBackend) vettedTreeIDAdd(token string, treeID int64) { // false. // // This function must be called WITHOUT the lock held. -func (t *tlogBackend) vettedTreeIDFromToken(token []byte) (int64, bool) { +func (t *tstoreBackend) vettedTreeIDFromToken(token []byte) (int64, bool) { // Check if the token is in the vetted cache. The vetted cache is // lazy loaded if the token is not present then we need to check // manually. @@ -379,10 +379,10 @@ var ( // statusChanges[currentStatus][newStatus] exists then the status // change is allowed. // - // Note, the tlog backend does not make use of the status + // Note, the tstore backend does not make use of the status // MDStatusIterationUnvetted. The original purpose of this status // was to show when an unvetted record had been altered since - // unvetted records were not versioned in the git backend. The tlog + // unvetted records were not versioned in the git backend. The tstore // backend versions unvetted records and thus does not need to use // this additional status. statusChanges = map[backend.MDStatusT]map[backend.MDStatusT]struct{}{ @@ -405,7 +405,7 @@ var ( ) // statusChangeIsAllowed returns whether the provided status change is allowed -// by tlogbe. +// by tstorebe. func statusChangeIsAllowed(from, to backend.MDStatusT) bool { allowed, ok := statusChanges[from] if !ok { @@ -503,7 +503,7 @@ func metadataStreamsUpdate(curr, mdAppend, mdOverwrite []backend.MetadataStream) return metadata } -func (t *tlogBackend) isShutdown() bool { +func (t *tstoreBackend) isShutdown() bool { t.RLock() defer t.RUnlock() @@ -514,7 +514,7 @@ func (t *tlogBackend) isShutdown() bool { // status is changed to a public status. // // This function satisfies the Backend interface. -func (t *tlogBackend) New(metadata []backend.MetadataStream, files []backend.File) (*backend.RecordMetadata, error) { +func (t *tstoreBackend) New(metadata []backend.MetadataStream, files []backend.File) (*backend.RecordMetadata, error) { log.Tracef("New") // Validate record contents @@ -554,7 +554,7 @@ func (t *tlogBackend) New(metadata []backend.MetadataStream, files []backend.Fil // Update the prefix cache. This must be done even if the // record creation fails since the tree will still exist in - // tlog. + // tstore. t.prefixAdd(token) break @@ -596,7 +596,7 @@ func (t *tlogBackend) New(metadata []backend.MetadataStream, files []backend.Fil } // This function satisfies the Backend interface. -func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { +func (t *tstoreBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { log.Tracef("UpdateUnvettedRecord: %x", token) // Validate record contents. Send in a single metadata array to @@ -699,7 +699,7 @@ func (t *tlogBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite [ } // This function satisfies the Backend interface. -func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { +func (t *tstoreBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { log.Tracef("UpdateVettedRecord: %x", token) // Validate record contents. Send in a single metadata array to @@ -802,7 +802,7 @@ func (t *tlogBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []b } // This function satisfies the Backend interface. -func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite []backend.MetadataStream) error { +func (t *tstoreBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite []backend.MetadataStream) error { // Validate record contents. Send in a single metadata array to // verify there are no dups. allMD := append(mdAppend, mdOverwrite...) @@ -892,7 +892,7 @@ func (t *tlogBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite } // This function satisfies the Backend interface. -func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite []backend.MetadataStream) error { +func (t *tstoreBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite []backend.MetadataStream) error { log.Tracef("UpdateVettedMetadata: %x", token) // Validate record contents. Send in a single metadata array to @@ -987,7 +987,7 @@ func (t *tlogBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite [ // record. // // This function satisfies the Backend interface. -func (t *tlogBackend) UnvettedExists(token []byte) bool { +func (t *tstoreBackend) UnvettedExists(token []byte) bool { log.Tracef("UnvettedExists %x", token) // Verify token is not in the vetted tree IDs cache. If it is then @@ -1004,7 +1004,7 @@ func (t *tlogBackend) UnvettedExists(token []byte) bool { } // This function satisfies the Backend interface. -func (t *tlogBackend) VettedExists(token []byte) bool { +func (t *tstoreBackend) VettedExists(token []byte) bool { log.Tracef("VettedExists %x", token) _, ok := t.vettedTreeIDFromToken(token) @@ -1012,14 +1012,14 @@ func (t *tlogBackend) VettedExists(token []byte) bool { } // This function must be called WITH the record lock held. -func (t *tlogBackend) unvettedPublish(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) (int64, error) { +func (t *tstoreBackend) unvettedPublish(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) (int64, error) { // Create a vetted tree vettedTreeID, err := t.vetted.TreeNew() if err != nil { return 0, err } - // Save the record to the vetted tlog + // Save the record to the vetted tstore err = t.vetted.RecordSave(vettedTreeID, rm, metadata, files) if err != nil { return 0, fmt.Errorf("vetted RecordSave: %v", err) @@ -1044,7 +1044,7 @@ func (t *tlogBackend) unvettedPublish(token []byte, rm backend.RecordMetadata, m } // This function must be called WITH the record lock held. -func (t *tlogBackend) unvettedCensor(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { +func (t *tstoreBackend) unvettedCensor(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { // Freeze the tree treeID := treeIDFromToken(token) err := t.unvetted.TreeFreeze(treeID, rm, metadata, 0) @@ -1065,7 +1065,7 @@ func (t *tlogBackend) unvettedCensor(token []byte, rm backend.RecordMetadata, me return nil } -func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { +func (t *tstoreBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { log.Tracef("SetUnvettedStatus: %x %v (%v)", token, status, backend.MDStatus[status]) @@ -1196,7 +1196,7 @@ func (t *tlogBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, } // This function must be called WITH the record lock held. -func (t *tlogBackend) vettedCensor(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { +func (t *tstoreBackend) vettedCensor(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { // Freeze the tree treeID, ok := t.vettedTreeID(token) if !ok { @@ -1221,7 +1221,7 @@ func (t *tlogBackend) vettedCensor(token []byte, rm backend.RecordMetadata, meta } // This function must be called WITH the record lock held. -func (t *tlogBackend) vettedArchive(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { +func (t *tstoreBackend) vettedArchive(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { // Freeze the tree. Nothing else needs to be done for an archived // record. treeID, ok := t.vettedTreeID(token) @@ -1239,7 +1239,7 @@ func (t *tlogBackend) vettedArchive(token []byte, rm backend.RecordMetadata, met } // This function satisfies the Backend interface. -func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { +func (t *tstoreBackend) SetVettedStatus(token []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { log.Tracef("SetVettedStatus: %x %v (%v)", token, status, backend.MDStatus[status]) @@ -1345,7 +1345,7 @@ func (t *tlogBackend) SetVettedStatus(token []byte, status backend.MDStatusT, md } // This function satisfies the Backend interface. -func (t *tlogBackend) GetUnvetted(token []byte, version string) (*backend.Record, error) { +func (t *tstoreBackend) GetUnvetted(token []byte, version string) (*backend.Record, error) { log.Tracef("GetUnvetted: %x %v", token, version) if t.isShutdown() { @@ -1375,7 +1375,7 @@ func (t *tlogBackend) GetUnvetted(token []byte, version string) (*backend.Record } // This function satisfies the Backend interface. -func (t *tlogBackend) GetVetted(token []byte, version string) (*backend.Record, error) { +func (t *tstoreBackend) GetVetted(token []byte, version string) (*backend.Record, error) { log.Tracef("GetVetted: %x %v", token, version) if t.isShutdown() { @@ -1409,7 +1409,7 @@ func (t *tlogBackend) GetVetted(token []byte, version string) (*backend.Record, // GetUnvettedBatch returns a batch of unvetted records. // // This function satisfies the Backend interface. -func (t *tlogBackend) GetUnvettedBatch(reqs []backend.RecordRequest) (map[string]backend.Record, error) { +func (t *tstoreBackend) GetUnvettedBatch(reqs []backend.RecordRequest) (map[string]backend.Record, error) { log.Tracef("GetUnvettedBatch: %v records ", len(reqs)) if t.isShutdown() { @@ -1457,7 +1457,7 @@ func (t *tlogBackend) GetUnvettedBatch(reqs []backend.RecordRequest) (map[string // GetVettedBatch returns a batch of unvetted records. // // This function satisfies the Backend interface. -func (t *tlogBackend) GetVettedBatch(reqs []backend.RecordRequest) (map[string]backend.Record, error) { +func (t *tstoreBackend) GetVettedBatch(reqs []backend.RecordRequest) (map[string]backend.Record, error) { log.Tracef("GetVettedBatch: %v records ", len(reqs)) if t.isShutdown() { @@ -1507,7 +1507,7 @@ func (t *tlogBackend) GetVettedBatch(reqs []backend.RecordRequest) (map[string]b } // This function satisfies the Backend interface. -func (t *tlogBackend) GetUnvettedTimestamps(token []byte, version string) (*backend.RecordTimestamps, error) { +func (t *tstoreBackend) GetUnvettedTimestamps(token []byte, version string) (*backend.RecordTimestamps, error) { log.Tracef("GetUnvettedTimestamps: %x %v", token, version) if t.isShutdown() { @@ -1534,7 +1534,7 @@ func (t *tlogBackend) GetUnvettedTimestamps(token []byte, version string) (*back } // This function satisfies the Backend interface. -func (t *tlogBackend) GetVettedTimestamps(token []byte, version string) (*backend.RecordTimestamps, error) { +func (t *tstoreBackend) GetVettedTimestamps(token []byte, version string) (*backend.RecordTimestamps, error) { log.Tracef("GetVettedTimestamps: %x %v", token, version) if t.isShutdown() { @@ -1565,7 +1565,7 @@ func (t *tlogBackend) GetVettedTimestamps(token []byte, version string) (*backen // categorized by MDStatusT. // // This function satisfies the Backend interface. -func (t *tlogBackend) InventoryByStatus(state string, status backend.MDStatusT, pageSize, page uint32) (*backend.InventoryByStatus, error) { +func (t *tstoreBackend) InventoryByStatus(state string, status backend.MDStatusT, pageSize, page uint32) (*backend.InventoryByStatus, error) { log.Tracef("InventoryByStatus: %v %v %v %v", state, status, pageSize, page) inv, err := t.invByStatus(state, status, pageSize, page) @@ -1579,10 +1579,10 @@ func (t *tlogBackend) InventoryByStatus(state string, status backend.MDStatusT, }, nil } -// RegisterUnvettedPlugin registers a plugin with the unvetted tlog instance. +// RegisterUnvettedPlugin registers a plugin with the unvetted tstore instance. // // This function satisfies the Backend interface. -func (t *tlogBackend) RegisterUnvettedPlugin(p backend.Plugin) error { +func (t *tstoreBackend) RegisterUnvettedPlugin(p backend.Plugin) error { log.Tracef("RegisterUnvettedPlugin: %v", p.ID) if t.isShutdown() { @@ -1595,7 +1595,7 @@ func (t *tlogBackend) RegisterUnvettedPlugin(p backend.Plugin) error { // RegisterVettedPlugin has not been implemented. // // This function satisfies the Backend interface. -func (t *tlogBackend) RegisterVettedPlugin(p backend.Plugin) error { +func (t *tstoreBackend) RegisterVettedPlugin(p backend.Plugin) error { log.Tracef("RegisterVettedPlugin: %v", p.ID) if t.isShutdown() { @@ -1609,7 +1609,7 @@ func (t *tlogBackend) RegisterVettedPlugin(p backend.Plugin) error { // unvetted plugin. // // This function satisfies the Backend interface. -func (t *tlogBackend) SetupUnvettedPlugin(pluginID string) error { +func (t *tstoreBackend) SetupUnvettedPlugin(pluginID string) error { log.Tracef("SetupUnvettedPlugin: %v", pluginID) if t.isShutdown() { @@ -1623,7 +1623,7 @@ func (t *tlogBackend) SetupUnvettedPlugin(pluginID string) error { // plugin. // // This function satisfies the Backend interface. -func (t *tlogBackend) SetupVettedPlugin(pluginID string) error { +func (t *tstoreBackend) SetupVettedPlugin(pluginID string) error { log.Tracef("SetupVettedPlugin: %v", pluginID) if t.isShutdown() { @@ -1633,7 +1633,7 @@ func (t *tlogBackend) SetupVettedPlugin(pluginID string) error { return t.vetted.PluginSetup(pluginID) } -func (t *tlogBackend) unvettedPluginRead(token []byte, pluginID, cmd, payload string) (string, error) { +func (t *tstoreBackend) unvettedPluginRead(token []byte, pluginID, cmd, payload string) (string, error) { // The token is optional. If a token is not provided then a tree ID // will not be provided to the plugin. var treeID int64 @@ -1658,7 +1658,7 @@ func (t *tlogBackend) unvettedPluginRead(token []byte, pluginID, cmd, payload st return t.unvetted.PluginCmd(treeID, token, pluginID, cmd, payload) } -func (t *tlogBackend) unvettedPluginWrite(token []byte, pluginID, cmd, payload string) (string, error) { +func (t *tstoreBackend) unvettedPluginWrite(token []byte, pluginID, cmd, payload string) (string, error) { // Get tree ID treeID := t.unvettedTreeIDFromToken(token) @@ -1721,7 +1721,7 @@ func (t *tlogBackend) unvettedPluginWrite(token []byte, pluginID, cmd, payload s // UnvettedPluginCmd executes a plugin command on an unvetted record. // // This function satisfies the Backend interface. -func (t *tlogBackend) UnvettedPluginCmd(action string, token []byte, pluginID, cmd, payload string) (string, error) { +func (t *tstoreBackend) UnvettedPluginCmd(action string, token []byte, pluginID, cmd, payload string) (string, error) { log.Tracef("UnvettedPluginCmd: %v %x %v %v", action, token, pluginID, cmd) if t.isShutdown() { @@ -1750,7 +1750,7 @@ func (t *tlogBackend) UnvettedPluginCmd(action string, token []byte, pluginID, c return reply, nil } -func (t *tlogBackend) vettedPluginRead(token []byte, pluginID, cmd, payload string) (string, error) { +func (t *tstoreBackend) vettedPluginRead(token []byte, pluginID, cmd, payload string) (string, error) { // The token is optional. If a token is not provided then a tree ID // will not be provided to the plugin. var treeID int64 @@ -1774,7 +1774,7 @@ func (t *tlogBackend) vettedPluginRead(token []byte, pluginID, cmd, payload stri return t.vetted.PluginCmd(treeID, token, pluginID, cmd, payload) } -func (t *tlogBackend) vettedPluginWrite(token []byte, pluginID, cmd, payload string) (string, error) { +func (t *tstoreBackend) vettedPluginWrite(token []byte, pluginID, cmd, payload string) (string, error) { // Get tree ID treeID, ok := t.vettedTreeIDFromToken(token) if !ok { @@ -1835,7 +1835,7 @@ func (t *tlogBackend) vettedPluginWrite(token []byte, pluginID, cmd, payload str // VettedPluginCmd executes a plugin command on an unvetted record. // // This function satisfies the Backend interface. -func (t *tlogBackend) VettedPluginCmd(action string, token []byte, pluginID, cmd, payload string) (string, error) { +func (t *tstoreBackend) VettedPluginCmd(action string, token []byte, pluginID, cmd, payload string) (string, error) { log.Tracef("VettedPluginCmd: %v %x %v %v", action, token, pluginID, cmd) if t.isShutdown() { @@ -1867,7 +1867,7 @@ func (t *tlogBackend) VettedPluginCmd(action string, token []byte, pluginID, cmd // GetUnvettedPlugins returns the unvetted plugins that have been registered. // // This function satisfies the Backend interface. -func (t *tlogBackend) GetUnvettedPlugins() []backend.Plugin { +func (t *tstoreBackend) GetUnvettedPlugins() []backend.Plugin { log.Tracef("GetUnvettedPlugins") return t.unvetted.Plugins() @@ -1876,7 +1876,7 @@ func (t *tlogBackend) GetUnvettedPlugins() []backend.Plugin { // GetVettedPlugins returns the vetted plugins that have been registered. // // This function satisfies the Backend interface. -func (t *tlogBackend) GetVettedPlugins() []backend.Plugin { +func (t *tstoreBackend) GetVettedPlugins() []backend.Plugin { log.Tracef("GetVettedPlugins") return t.vetted.Plugins() @@ -1885,28 +1885,28 @@ func (t *tlogBackend) GetVettedPlugins() []backend.Plugin { // Inventory has been DEPRECATED. // // This function satisfies the Backend interface. -func (t *tlogBackend) Inventory(vettedCount, vettedStart, unvettedCount uint, includeFiles, allVersions bool) ([]backend.Record, []backend.Record, error) { +func (t *tstoreBackend) Inventory(vettedCount, vettedStart, unvettedCount uint, includeFiles, allVersions bool) ([]backend.Record, []backend.Record, error) { return nil, nil, fmt.Errorf("not implemented") } // GetPlugins has been DEPRECATED. // // This function satisfies the Backend interface. -func (t *tlogBackend) GetPlugins() ([]backend.Plugin, error) { +func (t *tstoreBackend) GetPlugins() ([]backend.Plugin, error) { return nil, fmt.Errorf("not implemented") } // Plugin has been DEPRECATED. // // This function satisfies the Backend interface. -func (t *tlogBackend) Plugin(pluginID, cmd, cmdID, payload string) (string, error) { +func (t *tstoreBackend) Plugin(pluginID, cmd, cmdID, payload string) (string, error) { return "", fmt.Errorf("not implemented") } // Close shuts the backend down and performs cleanup. // // This function satisfies the Backend interface. -func (t *tlogBackend) Close() { +func (t *tstoreBackend) Close() { log.Tracef("Close") t.Lock() @@ -1915,13 +1915,13 @@ func (t *tlogBackend) Close() { // Shutdown backend t.shutdown = true - // Close tlog connections + // Close tstore connections t.unvetted.Close() t.vetted.Close() } -// setup creates the tlog backend caches. -func (t *tlogBackend) setup() error { +// setup creates the tstore backend caches. +func (t *tstoreBackend) setup() error { log.Tracef("setup") log.Infof("Building backend token prefix cache") @@ -1944,8 +1944,8 @@ func (t *tlogBackend) setup() error { return nil } -// New returns a new tlogBackend. -func New(anp *chaincfg.Params, homeDir, dataDir, unvettedTrillianHost, unvettedTrillianKeyFile, vettedTrillianHost, vettedTrillianKeyFile, encryptionKeyFile, dbType, dbHost, dbRootCert, dbCert, dbKey, dcrtimeHost, dcrtimeCert string) (*tlogBackend, error) { +// New returns a new tstoreBackend. +func New(anp *chaincfg.Params, homeDir, dataDir, unvettedTrillianHost, unvettedTrillianKeyFile, vettedTrillianHost, vettedTrillianKeyFile, encryptionKeyFile, dbType, dbHost, dbRootCert, dbCert, dbKey, dcrtimeHost, dcrtimeCert string) (*tstoreBackend, error) { // Setup encryption key file if encryptionKeyFile == "" { // No file path was given. Use the default path. @@ -1973,22 +1973,22 @@ func New(anp *chaincfg.Params, homeDir, dataDir, unvettedTrillianHost, unvettedT } log.Infof("Anchor host: %v", dcrtimeHost) - // Setup tlog instances - unvetted, err := tlog.New(tlogIDUnvetted, homeDir, dataDir, anp, + // Setup tstore instances + unvetted, err := tstore.New(tstoreIDUnvetted, homeDir, dataDir, anp, unvettedTrillianHost, unvettedTrillianKeyFile, encryptionKeyFile, dbType, dbHost, dbRootCert, dbCert, dbKey, dcrtimeHost, dcrtimeCert) if err != nil { - return nil, fmt.Errorf("new tlog unvetted: %v", err) + return nil, fmt.Errorf("new tstore unvetted: %v", err) } - vetted, err := tlog.New(tlogIDVetted, homeDir, dataDir, anp, + vetted, err := tstore.New(tstoreIDVetted, homeDir, dataDir, anp, vettedTrillianHost, vettedTrillianKeyFile, "", dbType, dbHost, dbRootCert, dbCert, dbKey, dcrtimeHost, dcrtimeCert) if err != nil { - return nil, fmt.Errorf("new tlog vetted: %v", err) + return nil, fmt.Errorf("new tstore vetted: %v", err) } - // Setup tlogbe - t := tlogBackend{ + // Setup tstore backend + t := tstoreBackend{ homeDir: homeDir, dataDir: dataDir, unvetted: unvetted, diff --git a/politeiad/config.go b/politeiad/config.go index 7ea1e7ad1..febb7898f 100644 --- a/politeiad/config.go +++ b/politeiad/config.go @@ -18,7 +18,7 @@ import ( "strings" v1 "github.com/decred/dcrtime/api/v1" - "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" + "github.com/decred/politeia/politeiad/backend/tstorebe/tstore" "github.com/decred/politeia/politeiad/sharedconfig" "github.com/decred/politeia/util" "github.com/decred/politeia/util/version" @@ -41,12 +41,12 @@ const ( // Backend options backendGit = "git" - backendTlog = "tlog" - defaultBackend = backendTlog + backendTstore = "tstore" + defaultBackend = backendTstore defaultTrillianHostUnvetted = "localhost:8090" defaultTrillianHostVetted = "localhost:8094" - defaultDBType = tlog.DBTypeFileSystem + defaultDBType = tstore.DBTypeFileSystem ) var ( diff --git a/politeiad/log.go b/politeiad/log.go index 41b4695a6..2a0b02d11 100644 --- a/politeiad/log.go +++ b/politeiad/log.go @@ -10,14 +10,14 @@ import ( "path/filepath" "github.com/decred/politeia/politeiad/backend/gitbe" - "github.com/decred/politeia/politeiad/backend/tlogbe" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/comments" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/dcrdata" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/ticketvote" - "github.com/decred/politeia/politeiad/backend/tlogbe/plugins/usermd" - "github.com/decred/politeia/politeiad/backend/tlogbe/store/fs" - "github.com/decred/politeia/politeiad/backend/tlogbe/store/mysql" - "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" + "github.com/decred/politeia/politeiad/backend/tstorebe" + "github.com/decred/politeia/politeiad/backend/tstorebe/plugins/comments" + "github.com/decred/politeia/politeiad/backend/tstorebe/plugins/dcrdata" + "github.com/decred/politeia/politeiad/backend/tstorebe/plugins/ticketvote" + "github.com/decred/politeia/politeiad/backend/tstorebe/plugins/usermd" + "github.com/decred/politeia/politeiad/backend/tstorebe/store/fs" + "github.com/decred/politeia/politeiad/backend/tstorebe/store/mysql" + "github.com/decred/politeia/politeiad/backend/tstorebe/tstore" "github.com/decred/politeia/wsdcrdata" "github.com/decred/slog" "github.com/jrick/logrotate/rotator" @@ -52,8 +52,8 @@ var ( log = backendLog.Logger("POLI") gitbeLog = backendLog.Logger("GITB") - tlogbeLog = backendLog.Logger("BACK") - tlogLog = backendLog.Logger("TLOG") + tstorebeLog = backendLog.Logger("BACK") + tstoreLog = backendLog.Logger("TSTR") wsdcrdataLog = backendLog.Logger("WSDD") pluginLog = backendLog.Logger("PLUG") ) @@ -62,12 +62,12 @@ var ( func init() { // Backend loggers gitbe.UseLogger(gitbeLog) - tlogbe.UseLogger(tlogbeLog) + tstorebe.UseLogger(tstorebeLog) - // Tlog loggers - tlog.UseLogger(tlogLog) - fs.UseLogger(tlogLog) - mysql.UseLogger(tlogLog) + // Tstore loggers + tstore.UseLogger(tstoreLog) + fs.UseLogger(tstoreLog) + mysql.UseLogger(tstoreLog) // Plugin loggers comments.UseLogger(pluginLog) @@ -83,8 +83,8 @@ func init() { var subsystemLoggers = map[string]slog.Logger{ "POLI": log, "GITB": gitbeLog, - "BACK": tlogbeLog, - "TLOG": tlogLog, + "BACK": tstorebeLog, + "TSTR": tstoreLog, "WSDD": wsdcrdataLog, "PLUG": pluginLog, } diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 69da3c3c3..d743dc28d 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -27,7 +27,7 @@ import ( "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/gitbe" - "github.com/decred/politeia/politeiad/backend/tlogbe" + "github.com/decred/politeia/politeiad/backend/tstorebe" "github.com/decred/politeia/politeiad/plugins/dcrdata" "github.com/decred/politeia/util" "github.com/decred/politeia/util/version" @@ -1607,14 +1607,14 @@ func _main() error { return fmt.Errorf("new gitbe: %v", err) } p.backend = b - case backendTlog: - b, err := tlogbe.New(activeNetParams.Params, cfg.HomeDir, cfg.DataDir, + case backendTstore: + b, err := tstorebe.New(activeNetParams.Params, cfg.HomeDir, cfg.DataDir, cfg.TrillianHostUnvetted, cfg.TrillianKeyUnvetted, cfg.TrillianHostVetted, cfg.TrillianKeyVetted, cfg.EncryptionKey, cfg.DBType, cfg.DBHost, cfg.DBRootCert, cfg.DBCert, cfg.DBKey, cfg.DcrtimeHost, cfg.DcrtimeCert) if err != nil { - return fmt.Errorf("new tlogbe: %v", err) + return fmt.Errorf("new tstorebe: %v", err) } p.backend = b default: diff --git a/politeiawww/cmd/pictl/cmdcommenttimestamps.go b/politeiawww/cmd/pictl/cmdcommenttimestamps.go index 47ecfa8d9..7e338a22a 100644 --- a/politeiawww/cmd/pictl/cmdcommenttimestamps.go +++ b/politeiawww/cmd/pictl/cmdcommenttimestamps.go @@ -8,7 +8,7 @@ import ( "fmt" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" + "github.com/decred/politeia/politeiad/backend/tstorebe/tstore" cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" pclient "github.com/decred/politeia/politeiawww/client" ) @@ -79,7 +79,7 @@ func (c *cmdCommentTimestamps) Execute(args []string) error { func verifyCommentTimestamp(t cmv1.Timestamp) error { ts := convertCommentTimestamp(t) - return tlog.VerifyTimestamp(ts) + return tstore.VerifyTimestamp(ts) } func convertCommentProof(p cmv1.Proof) backend.Proof { diff --git a/politeiawww/cmd/pictl/cmdproposaltimestamps.go b/politeiawww/cmd/pictl/cmdproposaltimestamps.go index b0ae4bf4b..742f7be74 100644 --- a/politeiawww/cmd/pictl/cmdproposaltimestamps.go +++ b/politeiawww/cmd/pictl/cmdproposaltimestamps.go @@ -8,7 +8,7 @@ import ( "fmt" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" + "github.com/decred/politeia/politeiad/backend/tstorebe/tstore" piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" pclient "github.com/decred/politeia/politeiawww/client" @@ -90,7 +90,7 @@ func (c *cmdProposalTimestamps) Execute(args []string) error { func verifyTimestamp(t rcv1.Timestamp) error { ts := convertTimestamp(t) - return tlog.VerifyTimestamp(ts) + return tstore.VerifyTimestamp(ts) } func convertProof(p rcv1.Proof) backend.Proof { diff --git a/politeiawww/cmd/pictl/cmdvotetimestamps.go b/politeiawww/cmd/pictl/cmdvotetimestamps.go index 7fef7b0f5..08d175a1d 100644 --- a/politeiawww/cmd/pictl/cmdvotetimestamps.go +++ b/politeiawww/cmd/pictl/cmdvotetimestamps.go @@ -8,7 +8,7 @@ import ( "fmt" "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tlogbe/tlog" + "github.com/decred/politeia/politeiad/backend/tstorebe/tstore" tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" pclient "github.com/decred/politeia/politeiawww/client" ) @@ -73,7 +73,7 @@ func (c *cmdVoteTimestamps) Execute(args []string) error { func verifyVoteTimestamp(t tkv1.Timestamp) error { ts := convertVoteTimestamp(t) - return tlog.VerifyTimestamp(ts) + return tstore.VerifyTimestamp(ts) } func convertVoteProof(p tkv1.Proof) backend.Proof { diff --git a/util/convert.go b/util/convert.go index 4aa81c609..09254bd04 100644 --- a/util/convert.go +++ b/util/convert.go @@ -35,8 +35,8 @@ func ConvertSignature(s string) ([identity.SignatureSize]byte, error) { // prefixes. func ConvertStringToken(token string) ([]byte, error) { switch { - case len(token) == pd.TokenSizeTlog*2: - // Tlog backend token; continue + case len(token) == pd.TokenSizeTstore*2: + // Tstore backend token; continue case len(token) != pd.TokenSizeGit*2: // Git backend token; continue case len(token) == pd.TokenPrefixLength: diff --git a/util/token.go b/util/token.go index 71311c6d3..562ae108a 100644 --- a/util/token.go +++ b/util/token.go @@ -12,16 +12,16 @@ import ( ) var ( - TokenTypeGit = "git" - TokenTypeTlog = "tlog" + TokenTypeGit = "git" + TokenTypeTstore = "tstore" ) // TokenIsFullLength returns whether a token is a valid, full length politeiad // censorship token. func TokenIsFullLength(tokenType string, token []byte) bool { switch tokenType { - case TokenTypeTlog: - return len(token) == pdv1.TokenSizeTlog + case TokenTypeTstore: + return len(token) == pdv1.TokenSizeTstore case TokenTypeGit: return len(token) == pdv1.TokenSizeGit default: @@ -89,8 +89,8 @@ func TokenDecodeAnyLength(tokenType, token string) ([]byte, error) { // regardless of token type. case tokenType == TokenTypeGit && TokenIsFullLength(TokenTypeGit, t): // Token is a valid git backend token - case tokenType == TokenTypeTlog && TokenIsFullLength(TokenTypeTlog, t): - // Token is a valid tlog backend token + case tokenType == TokenTypeTstore && TokenIsFullLength(TokenTypeTstore, t): + // Token is a valid tstore backend token default: return nil, fmt.Errorf("invalid token size") } From 51b99d5141dc84883a790779df7bf5634b6e907e Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 23 Feb 2021 17:37:46 -0600 Subject: [PATCH 331/449] Add setup scripts and fine tune perf. --- .../tstorebe/plugins/ticketvote/cmds.go | 16 +++- .../tstorebe/plugins/ticketvote/ticketvote.go | 80 ++++++++++++++++++- .../backend/tstorebe/store/mysql/mysql.go | 27 +++---- politeiad/backend/tstorebe/tstore/client.go | 3 + .../backend/tstorebe/tstore/trillianclient.go | 7 +- politeiad/backend/tstorebe/tstore/tstore.go | 13 ++- politeiad/backend/tstorebe/tstorebe.go | 6 +- politeiad/config.go | 13 +-- politeiad/plugins/ticketvote/ticketvote.go | 12 ++- politeiad/politeiad.go | 3 +- politeiad/scripts/mysql-tstore-reset.sh | 63 +++++++++++++++ politeiad/scripts/mysql-tstore-setup.sh | 53 ++++++++++++ 12 files changed, 250 insertions(+), 46 deletions(-) create mode 100755 politeiad/scripts/mysql-tstore-reset.sh create mode 100755 politeiad/scripts/mysql-tstore-setup.sh diff --git a/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go b/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go index 67d883378..b3cf85c43 100644 --- a/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go @@ -2237,7 +2237,7 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str tickets = append(tickets, v.Ticket) } addrs := p.activeVotes.CommitmentAddrs(token, tickets) - notInCache := make([]string, 0, len(cb.Ballot)) + notInCache := make([]string, 0, len(tickets)) for _, v := range tickets { _, ok := addrs[v] if !ok { @@ -2245,7 +2245,8 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str } } - log.Debugf("%v commitment addresses not found in cache", len(notInCache)) + log.Debugf("%v/%v commitment addresses found in cache", + len(tickets)-len(notInCache), len(tickets)) if len(notInCache) > 0 { // Get commitment addresses from dcrdata @@ -2321,14 +2322,21 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str // in the queue for all trees in the trillian instance. This value is // typically around the order of magnitude of 1000 queued leaves. // - // This is why a vote batch size of 10 was chosen. It is large enough + // The third variable that can cause errors is reaching the trillian + // datastore max connection limits. Each vote being cast creates a + // trillian connection. Overloading the trillian connections will + // cause max connection exceeded errors. The max allowed connections + // is a configurable trillian value, but should also be adjusted on + // the key-value store too. + // + // This is why a vote batch size of 5 was chosen. It is large enough // to alleviate performance bottlenecks from the log signer interval, // but small enough to still allow multiple records votes be held // concurrently without running into the queued leaf batch size limit. // Prepare work var ( - batchSize = 10 + batchSize = 5 batch = make([]ticketvote.CastVote, 0, batchSize) queue = make([][]ticketvote.CastVote, 0, len(votes)/batchSize) diff --git a/politeiad/backend/tstorebe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tstorebe/plugins/ticketvote/ticketvote.go index a6ac8d07f..0e97c1d69 100644 --- a/politeiad/backend/tstorebe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tstorebe/plugins/ticketvote/ticketvote.go @@ -79,11 +79,11 @@ func (p *ticketVotePlugin) Setup() error { // the inventory will cause it to update. log.Infof("Updating vote inventory") - bb, err := p.bestBlock() + bestBlock, err := p.bestBlock() if err != nil { return fmt.Errorf("bestBlock: %v", err) } - inv, err := p.inventory(bb) + inv, err := p.inventory(bestBlock) if err != nil { return fmt.Errorf("inventory: %v", err) } @@ -142,6 +142,42 @@ func (p *ticketVotePlugin) Setup() error { } } + // Verify votes + finished := make([]string, 0, len(inv.Entries)) + for _, v := range inv.Entries { + if v.Status == ticketvote.VoteStatusApproved || + v.Status == ticketvote.VoteStatusRejected { + finished = append(finished, v.Token) + } + } + for _, v := range finished { + // Get all cast votes + token, err := tokenDecode(v) + if err != nil { + return err + } + reply, err := p.backend.VettedPluginCmd(backend.PluginActionRead, + token, ticketvote.PluginID, ticketvote.CmdResults, "") + if err != nil { + return err + } + var rr ticketvote.ResultsReply + err = json.Unmarshal([]byte(reply), &rr) + if err != nil { + return err + } + + // Verify that there are no duplicates + tickets := make(map[string]struct{}, len(rr.Votes)) + for _, v := range rr.Votes { + _, ok := tickets[v.Ticket] + if ok { + return fmt.Errorf("duplicate ticket found %v %v", v.Token, v.Ticket) + } + tickets[v.Ticket] = struct{}{} + } + } + return nil } @@ -209,6 +245,46 @@ func (p *ticketVotePlugin) Fsck(treeIDs []int64) error { // Verify all caches + // Audit all finished votes + // - All votes that were cast were eligible + // - No duplicate votes + /* + finished := make([]string, 0, len(inv.Entries)) + for _, v := range inv.Entries { + if v.Status == ticketvote.VoteStatusApproved || + v.Status == ticketvote.VoteStatusRejected { + finished = append(finished, v.Token) + } + } + for _, v := range finished { + // Get all cast votes + token, err := tokenDecode(v) + if err != nil { + return err + } + reply, err := p.backend.VettedPluginCmd(backend.PluginActionRead, + token, ticketvote.PluginID, ticketvote.CmdResults, "") + if err != nil { + return err + } + var rr ticketvote.ResultsReply + err = json.Unmarshal([]byte(reply), &rr) + if err != nil { + return err + } + + // Verify that there are no duplicates + tickets := make(map[string]struct{}, len(rr.Votes)) + for _, v := range rr.Votes { + _, ok := tickets[v.Ticket] + if ok { + return fmt.Errorf("duplicate ticket found %v %v", v.Token, v.Ticket) + } + tickets[v.Ticket] = struct{}{} + } + } + */ + return nil } diff --git a/politeiad/backend/tstorebe/store/mysql/mysql.go b/politeiad/backend/tstorebe/store/mysql/mysql.go index f8fd0d436..4c301a37c 100644 --- a/politeiad/backend/tstorebe/store/mysql/mysql.go +++ b/politeiad/backend/tstorebe/store/mysql/mysql.go @@ -8,11 +8,12 @@ import ( "context" "database/sql" "fmt" - "net/url" "time" "github.com/decred/politeia/politeiad/backend/tstorebe/store" "github.com/google/uuid" + + _ "github.com/go-sql-driver/mysql" ) const ( @@ -20,7 +21,7 @@ const ( connTimeout = 1 * time.Minute connMaxLifetime = 1 * time.Minute maxOpenConns = 0 // 0 is unlimited - maxIdleConns = 10 + maxIdleConns = 100 ) // tableKeyValue defines the key-value table. The key is a uuid. @@ -155,6 +156,8 @@ func (s *mysql) Get(keys []string) (map[string][]byte, error) { } sql += ");" + log.Tracef("%v", sql) + // The keys must be converted to []interface{} for the query method // to accept them. args := make([]interface{}, len(keys)) @@ -192,17 +195,11 @@ func (s *mysql) Close() { s.db.Close() } -func New(host, user, dbname, sslRootCert, sslCert, sslKey string) (*mysql, error) { - // Setup database connection - v := url.Values{} - v.Add("sslmode", "require") - v.Add("sslca", sslRootCert) - v.Add("sslcert", sslCert) - v.Add("sslkey", sslKey) - - h := fmt.Sprintf("%v@tcp(%v)/%v?%v", user, host, dbname, v.Encode()) - log.Infof("Store host: %v", h) +func New(host, user, password, dbname string) (*mysql, error) { + // Connect to database + log.Infof("Host: %v:[password]@tcp(%v)/%v", user, host, dbname) + h := fmt.Sprintf("%v:%v@tcp(%v)/%v", user, password, host, dbname) db, err := sql.Open("mysql", h) if err != nil { return nil, err @@ -216,12 +213,12 @@ func New(host, user, dbname, sslRootCert, sslCert, sslKey string) (*mysql, error // Verify database connection err = db.Ping() if err != nil { - return nil, err + return nil, fmt.Errorf("db ping: %v", err) } // Setup key-value table - sql := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS kv (%s)`, tableKeyValue) - _, err = db.Exec(sql) + q := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS kv (%s)`, tableKeyValue) + _, err = db.Exec(q) if err != nil { return nil, fmt.Errorf("create table: %v", err) } diff --git a/politeiad/backend/tstorebe/tstore/client.go b/politeiad/backend/tstorebe/tstore/client.go index d837ec138..e08bc209d 100644 --- a/politeiad/backend/tstorebe/tstore/client.go +++ b/politeiad/backend/tstorebe/tstore/client.go @@ -260,6 +260,9 @@ func (t *Tstore) BlobsByDataDesc(treeID int64, dataDesc string) ([]store.BlobEnt keys = append(keys, ed.Key) } } + if len(keys) == 0 { + return []store.BlobEntry{}, nil + } // Pull the blobs from the store blobs, err := t.store.Get(keys) diff --git a/politeiad/backend/tstorebe/tstore/trillianclient.go b/politeiad/backend/tstorebe/tstore/trillianclient.go index 7f3263b77..e055c6fad 100644 --- a/politeiad/backend/tstorebe/tstore/trillianclient.go +++ b/politeiad/backend/tstorebe/tstore/trillianclient.go @@ -35,6 +35,10 @@ import ( "google.golang.org/grpc/status" ) +const ( + waitForInclusionTimeout = 60 * time.Second +) + // queuedLeafProof contains the results of a leaf append command, i.e. the // QueuedLeaf, and the inclusion proof for that leaf. If the leaf append // command fails the QueuedLeaf will contain an error code from the failure and @@ -379,7 +383,8 @@ func (t *tclient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queu return nil, nil, err } for _, v := range qlr.QueuedLeaves { - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), + waitForInclusionTimeout) defer cancel() err = c.WaitForInclusion(ctx, v.Leaf.LeafValue) if err != nil { diff --git a/politeiad/backend/tstorebe/tstore/tstore.go b/politeiad/backend/tstorebe/tstore/tstore.go index cd2e65531..37d1da5de 100644 --- a/politeiad/backend/tstorebe/tstore/tstore.go +++ b/politeiad/backend/tstorebe/tstore/tstore.go @@ -29,9 +29,9 @@ import ( ) const ( - DBTypeFileSystem = "filesystem" - DBTypeMySQL = "mysql" - dbUser = "politeiad" + DBTypeLevelDB = "leveldb" + DBTypeMySQL = "mysql" + dbUser = "politeiad" defaultTrillianKeyFilename = "trillian.key" defaultStoreDirname = "store" @@ -1379,7 +1379,7 @@ func (t *Tstore) Close() { } } -func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianKeyFile, encryptionKeyFile, dbType, dbHost, dbRootCert, dbCert, dbKey, dcrtimeHost, dcrtimeCert string) (*Tstore, error) { +func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianKeyFile, encryptionKeyFile, dbType, dbHost, dbPass, dcrtimeHost, dcrtimeCert string) (*Tstore, error) { // Load encryption key if provided. An encryption key is optional. var ek *encryptionKey if encryptionKeyFile != "" { @@ -1427,7 +1427,7 @@ func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trilli log.Infof("Database type %v: %v", id, dbType) var kvstore store.BlobKV switch dbType { - case DBTypeFileSystem: + case DBTypeLevelDB: fp := filepath.Join(dataDir, defaultStoreDirname) err = os.MkdirAll(fp, 0700) if err != nil { @@ -1438,8 +1438,7 @@ func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trilli case DBTypeMySQL: // Example db name: testnet3_unvetted_kv dbName := fmt.Sprintf("%v_%v_kv", anp.Name, id) - kvstore, err = mysql.New(dbHost, dbUser, dbName, - dbRootCert, dbCert, dbKey) + kvstore, err = mysql.New(dbHost, dbUser, dbPass, dbName) if err != nil { return nil, err } diff --git a/politeiad/backend/tstorebe/tstorebe.go b/politeiad/backend/tstorebe/tstorebe.go index 4e7167bbd..a54cbb48a 100644 --- a/politeiad/backend/tstorebe/tstorebe.go +++ b/politeiad/backend/tstorebe/tstorebe.go @@ -1945,7 +1945,7 @@ func (t *tstoreBackend) setup() error { } // New returns a new tstoreBackend. -func New(anp *chaincfg.Params, homeDir, dataDir, unvettedTrillianHost, unvettedTrillianKeyFile, vettedTrillianHost, vettedTrillianKeyFile, encryptionKeyFile, dbType, dbHost, dbRootCert, dbCert, dbKey, dcrtimeHost, dcrtimeCert string) (*tstoreBackend, error) { +func New(anp *chaincfg.Params, homeDir, dataDir, unvettedTrillianHost, unvettedTrillianKeyFile, vettedTrillianHost, vettedTrillianKeyFile, encryptionKeyFile, dbType, dbHost, dbPass, dcrtimeHost, dcrtimeCert string) (*tstoreBackend, error) { // Setup encryption key file if encryptionKeyFile == "" { // No file path was given. Use the default path. @@ -1976,13 +1976,13 @@ func New(anp *chaincfg.Params, homeDir, dataDir, unvettedTrillianHost, unvettedT // Setup tstore instances unvetted, err := tstore.New(tstoreIDUnvetted, homeDir, dataDir, anp, unvettedTrillianHost, unvettedTrillianKeyFile, encryptionKeyFile, - dbType, dbHost, dbRootCert, dbCert, dbKey, dcrtimeHost, dcrtimeCert) + dbType, dbHost, dbPass, dcrtimeHost, dcrtimeCert) if err != nil { return nil, fmt.Errorf("new tstore unvetted: %v", err) } vetted, err := tstore.New(tstoreIDVetted, homeDir, dataDir, anp, vettedTrillianHost, vettedTrillianKeyFile, "", - dbType, dbHost, dbRootCert, dbCert, dbKey, dcrtimeHost, dcrtimeCert) + dbType, dbHost, dbPass, dcrtimeHost, dcrtimeCert) if err != nil { return nil, fmt.Errorf("new tstore vetted: %v", err) } diff --git a/politeiad/config.go b/politeiad/config.go index febb7898f..56357b93a 100644 --- a/politeiad/config.go +++ b/politeiad/config.go @@ -46,7 +46,10 @@ const ( defaultTrillianHostUnvetted = "localhost:8090" defaultTrillianHostVetted = "localhost:8094" - defaultDBType = tstore.DBTypeFileSystem + + // Database defaults + defaultDBType = tstore.DBTypeLevelDB + defaultDBHost = "127.0.0.1:3306" // MySQL default host ) var ( @@ -92,17 +95,16 @@ type config struct { // TODO validate these config params and set defaults. Also consider // making them specific to tstore. Ex: tstore.TrillianHostUnvetted. + // TODO Verify that the trillian key is being used Backend string `long:"backend"` TrillianHostUnvetted string `long:"trillianhostunvetted"` TrillianHostVetted string `long:"trillianhostvetted"` TrillianKeyUnvetted string `long:"trilliankeyunvetted"` TrillianKeyVetted string `long:"trilliankeyvetted"` - EncryptionKey string `long:"encryptionkey"` DBType string `long:"dbtype"` DBHost string `long:"dbhost" description:"Database ip:port"` - DBRootCert string `long:"dbrootcert" description:"File containing the CA certificate for the database"` - DBCert string `long:"dbcert" description:"File containing the client certificate for the database"` - DBKey string `long:"dbkey" description:"File containing the client certificate key for the database"` + DBPass string `long:"dbpass" description:"Database password"` + EncryptionKey string `long:"encryptionkey" description:"Database encryption key"` // Plugin settings Plugins []string `long:"plugin"` @@ -259,6 +261,7 @@ func loadConfig() (*config, []string, error) { TrillianHostUnvetted: defaultTrillianHostUnvetted, TrillianHostVetted: defaultTrillianHostVetted, DBType: defaultDBType, + DBHost: defaultDBHost, } // Service options which are only added on Windows. diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index 05662a68f..e0580250d 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -80,12 +80,11 @@ const ( ErrorCodeStartDetailsInvalid ErrorCodeT = 8 ErrorCodeVoteParamsInvalid ErrorCodeT = 9 ErrorCodeVoteStatusInvalid ErrorCodeT = 10 - ErrorCodePageSizeExceeded ErrorCodeT = 11 - ErrorCodeVoteMetadataInvalid ErrorCodeT = 12 - ErrorCodeLinkByInvalid ErrorCodeT = 13 - ErrorCodeLinkToInvalid ErrorCodeT = 14 - ErrorCodeRunoffVoteParentInvalid ErrorCodeT = 15 - ErrorCodeLinkByNotExpired ErrorCodeT = 16 + ErrorCodeVoteMetadataInvalid ErrorCodeT = 11 + ErrorCodeLinkByInvalid ErrorCodeT = 12 + ErrorCodeLinkToInvalid ErrorCodeT = 13 + ErrorCodeRunoffVoteParentInvalid ErrorCodeT = 14 + ErrorCodeLinkByNotExpired ErrorCodeT = 15 ) var ( @@ -102,7 +101,6 @@ var ( ErrorCodeStartDetailsInvalid: "start details invalid", ErrorCodeVoteParamsInvalid: "vote params invalid", ErrorCodeVoteStatusInvalid: "vote status invalid", - ErrorCodePageSizeExceeded: "page size exceeded", ErrorCodeVoteMetadataInvalid: "vote metadata invalid", ErrorCodeLinkByInvalid: "linkby invalid", ErrorCodeLinkToInvalid: "linkto invalid", diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index d743dc28d..c5742aa31 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -1611,8 +1611,7 @@ func _main() error { b, err := tstorebe.New(activeNetParams.Params, cfg.HomeDir, cfg.DataDir, cfg.TrillianHostUnvetted, cfg.TrillianKeyUnvetted, cfg.TrillianHostVetted, cfg.TrillianKeyVetted, cfg.EncryptionKey, - cfg.DBType, cfg.DBHost, cfg.DBRootCert, cfg.DBCert, cfg.DBKey, - cfg.DcrtimeHost, cfg.DcrtimeCert) + cfg.DBType, cfg.DBHost, cfg.DBPass, cfg.DcrtimeHost, cfg.DcrtimeCert) if err != nil { return fmt.Errorf("new tstorebe: %v", err) } diff --git a/politeiad/scripts/mysql-tstore-reset.sh b/politeiad/scripts/mysql-tstore-reset.sh new file mode 100755 index 000000000..b5b38e7a0 --- /dev/null +++ b/politeiad/scripts/mysql-tstore-reset.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env sh + +# Accepts environment variables: +# - MYSQL_HOST: The hostname of the MySQL server (default: localhost). +# - MYSQL_PORT: The port the MySQL server is listening on (default: 3306). +# - MYSQL_ROOT_USER: A user with sufficient rights to create new users and +# create/drop the politeiad database (default: root). +# - MYSQL_ROOT_PASSWORD: The password for the user defined by MYSQL_ROOT_USER +# (requed, default: none). +# - MYSQL_USER_HOST: The host that the politeiad and trillian users will +# connect from; use '%' as a wildcard (default: localhost). +# - POLITEIAD_DIR: politeiad application directory. This will vary depending +# on you operating system (default: ${HOME}/.politeiad). + +# Set unset environment variables to defaults +[ -z ${MYSQL_HOST+x} ] && MYSQL_HOST="localhost" +[ -z ${MYSQL_PORT+x} ] && MYSQL_PORT="3306" +[ -z ${MYSQL_ROOT_USER+x} ] && MYSQL_ROOT_USER="root" +[ -z ${MYSQL_ROOT_PASSWORD+x} ] && MYSQL_ROOT_PASSWORD="" +[ -z ${MYSQL_USER_HOST+x} ] && MYSQL_USER_HOST="localhost" +[ -z ${POLITEIAD_DIR+x} ] && POLITEIAD_DIR="${HOME}/.politeiad" + +flags="-u "${MYSQL_ROOT_USER}" -p"${MYSQL_ROOT_PASSWORD}" --verbose \ + --host ${MYSQL_HOST} --port ${MYSQL_PORT}" + +# Database users +politeiad="politeiad" + +# Database names +testnet_unvetted_kv="testnet3_unvetted_kv" +testnet_vetted_kv="testnet3_vetted_kv" + +# Delete databases +mysql ${flags} -e "DROP DATABASE IF EXISTS ${testnet_unvetted_kv};" +mysql ${flags} -e "DROP DATABASE IF EXISTS ${testnet_vetted_kv};" + +# Setup kv databases. The trillian script creates the trillian databases. +mysql ${flags} -e "CREATE DATABASE ${testnet_unvetted_kv};" +mysql ${flags} -e "CREATE DATABASE ${testnet_vetted_kv};" + +mysql ${flags} -e \ + "GRANT ALL ON ${testnet_unvetted_kv}.* TO '${politeiad}'@'${MYSQL_USER_HOST}'" +mysql ${flags} -e \ + "GRANT ALL ON ${testnet_vetted_kv}.* TO '${politeiad}'@'${MYSQL_USER_HOST}'" + +# Delete cached politeiad data +politeiad_data_dir="${POLITEIAD_DIR}/data/testnet3/" +echo "Warning: about to delete the following directories: +${politeiad_data_dir}" +read -p "Continue? [Y/N] " answer +case $answer in + yes|Yes|y) + rm -rf ${politeiad_data_dir} + ;; + no|n) + echo "Delete aborted" + ;; +esac + +echo "politeiad testnet reset complete!" +echo "The trillian logs must be reset using the trillian script. See docs." +echo "Mainnet politeiad resets must be done manually." + diff --git a/politeiad/scripts/mysql-tstore-setup.sh b/politeiad/scripts/mysql-tstore-setup.sh new file mode 100755 index 000000000..578a485ae --- /dev/null +++ b/politeiad/scripts/mysql-tstore-setup.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env sh + +# Accepts environment variables: +# - MYSQL_HOST: The hostname of the MySQL server (default: localhost). +# - MYSQL_PORT: The port the MySQL server is listening on (default: 3306). +# - MYSQL_ROOT_USER: A user with sufficient rights to create new users and +# create/drop the politeiad database (default: root). +# - MYSQL_ROOT_PASSWORD: The password for the user defined by MYSQL_ROOT_USER +# (requed, default: none). +# - MYSQL_POLITEIAD_PASSWORD: The password for the politeiad user that will be +# created during this script (required, default: none). +# - MYSQL_TRILLIAN_PASSWORD: The password for the trillian user that will be +# created during this script (required, default: none). +# - MYSQL_USER_HOST: The host that the politeiad and trillian users will +# connect from; use '%' as a wildcard (default: localhost). + +# Set unset environment variables to defaults +[ -z ${MYSQL_HOST+x} ] && MYSQL_HOST="localhost" +[ -z ${MYSQL_PORT+x} ] && MYSQL_PORT="3306" +[ -z ${MYSQL_ROOT_USER+x} ] && MYSQL_ROOT_USER="root" +[ -z ${MYSQL_ROOT_PASSWORD+x} ] && MYSQL_ROOT_PASSWORD="" +[ -z ${MYSQL_POLITEIAD_PASSWORD+x} ] && MYSQL_POLITEIAD_PASSWORD="" +[ -z ${MYSQL_TRILLIAN_PASSWORD+x} ] && MYSQL_TRILLIAN_PASSWORD="" +[ -z ${MYSQL_USER_HOST+x} ] && MYSQL_USER_HOST="localhost" + +flags="-u "${MYSQL_ROOT_USER}" -p"${MYSQL_ROOT_PASSWORD}" --verbose \ + --host ${MYSQL_HOST} --port ${MYSQL_PORT}" + +# Database users +politeiad="politeiad" +trillian="trillian" + +# Database names +testnet_unvetted_kv="testnet3_unvetted_kv" +testnet_vetted_kv="testnet3_vetted_kv" + +# Setup database users +mysql ${flags} -e \ + "CREATE USER IF NOT EXISTS '${politeiad}'@'${MYSQL_USER_HOST}' \ + IDENTIFIED BY '${MYSQL_POLITEIAD_PASSWORD}'" + +mysql ${flags} -e \ + "CREATE USER IF NOT EXISTS '${trillian}'@'${MYSQL_USER_HOST}' \ + IDENTIFIED BY '${MYSQL_TRILLIAN_PASSWORD}'" + +# Setup kv databases. The trillian script creates the trillian databases. +mysql ${flags} -e "CREATE DATABASE IF NOT EXISTS ${testnet_unvetted_kv};" +mysql ${flags} -e "CREATE DATABASE IF NOT EXISTS ${testnet_vetted_kv};" + +mysql ${flags} -e \ + "GRANT ALL ON ${testnet_unvetted_kv}.* TO '${politeiad}'@'${MYSQL_USER_HOST}'" +mysql ${flags} -e \ + "GRANT ALL ON ${testnet_vetted_kv}.* TO '${politeiad}'@'${MYSQL_USER_HOST}'" From 982347405ca05ef871f591400bd8c14593d3c55e Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 24 Feb 2021 08:19:52 -0600 Subject: [PATCH 332/449] Bug fix. --- politeiad/backend/tstorebe/tstore/client.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/politeiad/backend/tstorebe/tstore/client.go b/politeiad/backend/tstorebe/tstore/client.go index e08bc209d..63349a069 100644 --- a/politeiad/backend/tstorebe/tstore/client.go +++ b/politeiad/backend/tstorebe/tstore/client.go @@ -153,6 +153,10 @@ func (t *Tstore) BlobsDel(treeID int64, digests [][]byte) error { func (t *Tstore) Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEntry, error) { log.Tracef("%v Blobs: %v %x", t.id, treeID, digests) + if len(digests) == 0 { + return map[string]store.BlobEntry{}, nil + } + // Verify tree exists if !t.TreeExists(treeID) { return nil, backend.ErrRecordNotFound From bb92a5bfab2428d7f7cab26a5ea3a8d7b5904bd8 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 24 Feb 2021 10:28:05 -0600 Subject: [PATCH 333/449] HTTP client append to system cert pool. --- util/net.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/util/net.go b/util/net.go index 7728db6f5..78516998c 100644 --- a/util/net.go +++ b/util/net.go @@ -41,7 +41,10 @@ func NewHTTPClient(skipVerify bool, certPath string) (*http.Client, error) { if err != nil { return nil, err } - certPool := x509.NewCertPool() + certPool, err := x509.SystemCertPool() + if err != nil { + certPool = x509.NewCertPool() + } certPool.AppendCertsFromPEM(cert) tlsConfig.RootCAs = certPool From 48c6e87acac4cc8580847f6ee197b25df7b6a443 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 24 Feb 2021 13:07:24 -0600 Subject: [PATCH 334/449] Update trillian to latest release. --- go.mod | 6 +- go.sum | 157 ++++++++++++++++++ .../backend/tstorebe/tstore/trillianclient.go | 4 +- politeiad/backend/tstorebe/tstore/verify.go | 4 +- 4 files changed, 164 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 4fcb84181..e038b5c6c 100644 --- a/go.mod +++ b/go.mod @@ -29,9 +29,9 @@ require ( github.com/decred/slog v1.1.0 github.com/go-sql-driver/mysql v1.5.0 github.com/go-test/deep v1.0.1 - github.com/golang/protobuf v1.4.2 + github.com/golang/protobuf v1.4.3 github.com/golang/snappy v0.0.1 // indirect - github.com/google/trillian v1.3.11 + github.com/google/trillian v1.3.13 github.com/google/uuid v1.1.1 github.com/gorilla/csrf v1.6.2 github.com/gorilla/mux v1.7.3 @@ -48,7 +48,7 @@ require ( github.com/otiai10/curr v0.0.0-20190513014714-f5a3d24e5776 // indirect github.com/pmezard/go-difflib v1.0.0 github.com/pquerna/otp v1.2.0 - github.com/prometheus/common v0.10.0 + github.com/prometheus/common v0.15.0 github.com/robfig/cron v1.2.0 github.com/subosito/gozaru v0.0.0-20190625071150-416082cce636 github.com/syndtr/goleveldb v1.0.0 diff --git a/go.sum b/go.sum index 64c6516a4..d59ee8f11 100644 --- a/go.sum +++ b/go.sum @@ -45,6 +45,7 @@ github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20O github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= @@ -52,7 +53,11 @@ github.com/Masterminds/sprig v2.15.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuN github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/Sereal/Sereal v0.0.0-20190618215532-0b8ac451a863 h1:BRrxwOZBolJN4gIwvZMJY1tzqBvQgpaZiQRuIDD40jM= github.com/Sereal/Sereal v0.0.0-20190618215532-0b8ac451a863/go.mod h1:D0JMgToj/WdxCgd30Kc1UcA9E+WdZoJqeVOuYW7iTBM= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/aead/siphash v0.0.0-20170329201724-e404fcfc8885/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7ISrnJIXKzwaspym5BTKGx93EI= github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412/go.mod h1:WPjqKcmVOxf0XSf3YxCJs6N6AOSrOx3obionmG7T0y0= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -61,13 +66,24 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= +github.com/apache/beam v2.27.0+incompatible/go.mod h1:/8NX3Qi8vGstDLLaeaU7+lzVEu/ACaQhYjeefzQ0y1o= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/asdine/storm/v3 v3.0.0-20191014171123-c370e07ad6d4 h1:92RJlxO6DcRon/jV6MxU6FYymYE02Ku1ZuRKpSOuTk4= github.com/asdine/storm/v3 v3.0.0-20191014171123-c370e07ad6d4/go.mod h1:wncSIXIbR3lvJQhBpnwAeNPQneL5Vx2KUox2jARUdmw= +github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -84,14 +100,18 @@ github.com/carterjones/go-cloudflare-scraper v0.1.2 h1:GNmlJEfhIVPVXaEItnPSDtwOp github.com/carterjones/go-cloudflare-scraper v0.1.2/go.mod h1:maO/ygX7QWbdh/TzHqr5uR42b2BW81g/05QRx7fpw38= github.com/carterjones/signalr v0.3.5 h1:kJSw+6a9XmsOb/+9HWTnY8SjTrVOdpzCSPV/9IVS2nI= github.com/carterjones/signalr v0.3.5/go.mod h1:SOGIwr/0/4GGNjHWSSginY66OVSaOeM85yWCNytdEwE= +github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -363,7 +383,12 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUn github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/goproxy v0.0.0-20181111060418-2ce16c963a8a/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -372,6 +397,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= +github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fullstorydev/grpcurl v1.6.0/go.mod h1:ZQ+ayqbKMJNhzLmbpCiurTVlaK2M/3nqZCxaQ2Ze/sM= @@ -381,16 +408,22 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-redis/redis v6.15.8+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.1 h1:UQhStjbkDClarlmv0am7OXXO4/GaPdCGiUiMTvi28sg= github.com/go-test/deep v1.0.1/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= @@ -425,6 +458,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -439,6 +474,7 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -451,16 +487,21 @@ github.com/google/pprof v0.0.0-20200507031123-427632fa3b1c/go.mod h1:ZgVRPoUq/hf github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/trillian v1.3.11 h1:pPzJPkK06mvXId1LHEAJxIegGgHzzp/FUnycPYfoCMI= github.com/google/trillian v1.3.11/go.mod h1:0tPraVHrSDkA3BO6vKX67zgLXs6SsOAbHEivX+9mPgw= +github.com/google/trillian v1.3.13 h1:V0avmojBPY7YAlcd/nUVvNRprU28tRsahPNxIedqekU= +github.com/google/trillian v1.3.13/go.mod h1:8y3zC8XuqFxsslWPkP0r3sprERfFf7hCWmicL0yHZNI= github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gordonklaus/ineffassign v0.0.0-20200309095847-7953dde2c7bf/go.mod h1:cuNKsD1zp2v6XfE/orVX2QE1LC+i254ceGcVeDT3pTU= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/csrf v1.6.2 h1:QqQ/OWwuFp4jMKgBFAzJVW3FMULdyUW7JoM4pEWuqKg= github.com/gorilla/csrf v1.6.2/go.mod h1:7tSf8kmjNYr7IWDCYhd3U8Ck34iQ/Yw5CJu7bAkHEGI= github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/schema v1.1.0 h1:CamqUDOFUBqzrvxuz2vEwo8+SUdwsluFh7IlzJh30LY= @@ -483,17 +524,37 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c h1:fEE5/5VNnYUoBOj2I9TP8Jc+a7lge3QWn9DKE7NCwfc= github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c/go.mod h1:ObS/W+h8RYb1Y7fYivughjxojTmIu5iAIjSrSLCLeqE= +github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= +github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= +github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.4/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.1-0.20200711081900-c17162fe8fd7 h1:Ug59miTxVKVg5Oi2S5uHlKOIV5jBx4Hb2u0jIxxDaSs= github.com/jessevdk/go-flags v1.4.1-0.20200711081900-c17162fe8fd7/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -506,6 +567,7 @@ github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jrick/bitset v1.0.0/go.mod h1:ZOYB5Uvkla7wIEY4FEssPVi3IQXa02arznRaYaAEPe4= github.com/jrick/logrotate v1.0.0 h1:lQ1bL/n9mBNeIXoTUoYRlK4dHuNJVofX9oWqBtPnSzI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= @@ -513,16 +575,20 @@ github.com/jrick/wsrpc/v2 v2.3.2/go.mod h1:XPYs8BnRWl99lCvXRM5SLpZmTPqWpSOPkDIqY github.com/jrick/wsrpc/v2 v2.3.4/go.mod h1:XPYs8BnRWl99lCvXRM5SLpZmTPqWpSOPkDIqYTwDPfU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -534,11 +600,17 @@ github.com/letsencrypt/pkcs11key/v4 v4.0.0/go.mod h1:EFUvBDay26dErnNb70Nd0/VW3tJ github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.9.0 h1:L8nSXQQzAYByakOFMTwpjRoHsMJklur4Gi59b6VivR8= +github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= +github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= +github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/marcopeereboom/sbox v1.0.0 h1:1xTRUzI5mVvsaPaBGVpXdyH2hZeZhCWazbGy6MVsRgg= github.com/marcopeereboom/sbox v1.0.0/go.mod h1:V9e7t7oKphNfXymk7Lqvbo9mZiVjmCt8vBHnROcpCSY= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= @@ -548,10 +620,17 @@ github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-sqlite3 v2.0.1+incompatible h1:xQ15muvnzGBHpIpdrNi1DA5x0+TcBZzsIDwmw9uTHzw= github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/pkcs11 v1.0.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= @@ -560,9 +639,19 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-proto-validators v0.0.0-20180403085117-0950a7990007/go.mod h1:m2XC9Qq0AlmmVksL6FktJCdTYyLk7V3fKyp0sl1yWQo= github.com/mwitkow/go-proto-validators v0.2.0/go.mod h1:ZfA1hW+UH/2ZHOWvQ3HnQaU0DtnpXu850MZiy+YUgcc= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nishanths/predeclared v0.0.0-20190419143655-18a43bb90ffc/go.mod h1:62PewwiQTlm/7Rj+cxVYqZvDIUc+JjZq6GHAC1fsObQ= +github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.2/go.mod h1:rSAaSIOAGT9odnlyGlUfAJaoc5w2fSBUmeGDbRWPxyQ= @@ -576,7 +665,15 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= +github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/otiai10/copy v1.0.1 h1:gtBjD8aq4nychvRZ2CyJvFWAw0aja+VHazDdruZKGZA= github.com/otiai10/copy v1.0.1/go.mod h1:8bMCJrAqOtN/d9oyh5HR7HhLQMvcGMpGdwRDYsfOCHc= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= @@ -585,30 +682,52 @@ github.com/otiai10/curr v0.0.0-20190513014714-f5a3d24e5776/go.mod h1:3HNVkVOU7vZ github.com/otiai10/mint v1.2.3/go.mod h1:YnfyPNhBvnY8bW4SGQHCs/aAFhkgySlMZbrF5U0bOVw= github.com/otiai10/mint v1.2.4 h1:DxYL0itZyPaR5Z9HILdxSoHx+gNs6Yx+neOGS3IVUk0= github.com/otiai10/mint v1.2.4/go.mod h1:d+b7n/0R3tdyUYYylALXpWQ/kTN+QobSq/4SRGBkR3M= +github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/otp v1.2.0 h1:/A3+Jn+cagqayeR3iHs/L62m5ue7710D35zl1zJ1kok= github.com/pquerna/otp v1.2.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/pseudomuto/protoc-gen-doc v1.3.2/go.mod h1:y5+P6n3iGrbKG+9O04V5ld71in3v/bX88wUwgt+U8EA= github.com/pseudomuto/protokit v0.2.0/go.mod h1:2PdH30hxVHsup8KpBTOXTBeMVhJZVio3Q8ViKSAXT0Q= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/robertkrimen/otto v0.0.0-20180617131154-15f95af6e78d h1:1VUlQbCfkoSGv7qP7Y+ro3ap1P1pPZxgdGVqiTVy5C4= github.com/robertkrimen/otto v0.0.0-20180617131154-15f95af6e78d/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= @@ -618,11 +737,18 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= @@ -632,6 +758,9 @@ github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v0.0.0-20170130113145-4d4bfba8f1d1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -660,7 +789,11 @@ go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v0.0.0-20200513171258-e048e166ab9c/go.mod h1:xCI7ZzBfRuGgBXyXO6yfWfDmlWd35khcWpUa4L0xI/k= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -679,12 +812,14 @@ golang.org/x/crypto v0.0.0-20180501155221-613d6eafa307/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20180718160520-a2144134853f/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180808211826-de0752318171/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -728,10 +863,13 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180808004115-f9ce57c11b24/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181207154023-610586996380/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -775,11 +913,14 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180810070207-f0d5e33068cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -798,6 +939,7 @@ golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -814,6 +956,9 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e h1:AyodaIpKjppX+cBfTASF2E1US3H2JFBj920Ot3rtDjs= +golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -833,6 +978,7 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -852,6 +998,7 @@ golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -873,6 +1020,7 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -890,6 +1038,7 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= @@ -906,6 +1055,7 @@ google.golang.org/genproto v0.0.0-20190415143225-d1146b9035b9/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= @@ -934,8 +1084,11 @@ google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEd google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= @@ -969,11 +1122,13 @@ gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI= gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -982,6 +1137,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.6 h1:97YCGUei5WVbkKfogoJQsLwUJ17cWvpLrgNvlcbxikE= gopkg.in/yaml.v2 v2.2.6/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -993,3 +1149,4 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= diff --git a/politeiad/backend/tstorebe/tstore/trillianclient.go b/politeiad/backend/tstorebe/tstore/trillianclient.go index e055c6fad..89c78cca6 100644 --- a/politeiad/backend/tstorebe/tstore/trillianclient.go +++ b/politeiad/backend/tstorebe/tstore/trillianclient.go @@ -25,7 +25,7 @@ import ( "github.com/google/trillian/crypto/keys/der" "github.com/google/trillian/crypto/keyspb" "github.com/google/trillian/crypto/sigpb" - "github.com/google/trillian/merkle/hashers" + "github.com/google/trillian/merkle/hashers/registry" "github.com/google/trillian/merkle/rfc6962" "github.com/google/trillian/types" "google.golang.org/genproto/protobuf/field_mask" @@ -274,7 +274,7 @@ func (t *tclient) inclusionProof(treeID int64, merkleLeafHash []byte, lrv1 *type proof := resp.Proof[0] // Verify inclusion proof - lh, err := hashers.NewLogHasher(trillian.HashStrategy_RFC6962_SHA256) + lh, err := registry.NewLogHasher(trillian.HashStrategy_RFC6962_SHA256) if err != nil { return nil, err } diff --git a/politeiad/backend/tstorebe/tstore/verify.go b/politeiad/backend/tstorebe/tstore/verify.go index 6caea0322..0caf0aa75 100644 --- a/politeiad/backend/tstorebe/tstore/verify.go +++ b/politeiad/backend/tstorebe/tstore/verify.go @@ -17,7 +17,7 @@ import ( "github.com/decred/politeia/util" "github.com/google/trillian" tmerkle "github.com/google/trillian/merkle" - "github.com/google/trillian/merkle/hashers" + "github.com/google/trillian/merkle/hashers/registry" ) const ( @@ -52,7 +52,7 @@ func verifyProofTrillian(p backend.Proof) error { // The digest of the data is stored in trillian as the leaf value. // The digest of the leaf value is the digest that is included in // the log merkle root. - h, err := hashers.NewLogHasher(trillian.HashStrategy_RFC6962_SHA256) + h, err := registry.NewLogHasher(trillian.HashStrategy_RFC6962_SHA256) if err != nil { return err } From 80808ee48583a371f75960a58a038037b7520f93 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 24 Feb 2021 13:26:42 -0600 Subject: [PATCH 335/449] Cleanup. --- .../tstorebe/plugins/ticketvote/ticketvote.go | 36 ------------------- 1 file changed, 36 deletions(-) diff --git a/politeiad/backend/tstorebe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tstorebe/plugins/ticketvote/ticketvote.go index 0e97c1d69..83851488e 100644 --- a/politeiad/backend/tstorebe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tstorebe/plugins/ticketvote/ticketvote.go @@ -142,42 +142,6 @@ func (p *ticketVotePlugin) Setup() error { } } - // Verify votes - finished := make([]string, 0, len(inv.Entries)) - for _, v := range inv.Entries { - if v.Status == ticketvote.VoteStatusApproved || - v.Status == ticketvote.VoteStatusRejected { - finished = append(finished, v.Token) - } - } - for _, v := range finished { - // Get all cast votes - token, err := tokenDecode(v) - if err != nil { - return err - } - reply, err := p.backend.VettedPluginCmd(backend.PluginActionRead, - token, ticketvote.PluginID, ticketvote.CmdResults, "") - if err != nil { - return err - } - var rr ticketvote.ResultsReply - err = json.Unmarshal([]byte(reply), &rr) - if err != nil { - return err - } - - // Verify that there are no duplicates - tickets := make(map[string]struct{}, len(rr.Votes)) - for _, v := range rr.Votes { - _, ok := tickets[v.Ticket] - if ok { - return fmt.Errorf("duplicate ticket found %v %v", v.Token, v.Ticket) - } - tickets[v.Ticket] = struct{}{} - } - } - return nil } From eb41ebc45afb987baa8ea506bc1a33e3467c52ec Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 24 Feb 2021 14:26:19 -0600 Subject: [PATCH 336/449] go mod tidy --- go.mod | 2 -- go.sum | 22 ++-------------------- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index e038b5c6c..fe0c4e49a 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,6 @@ require ( github.com/otiai10/curr v0.0.0-20190513014714-f5a3d24e5776 // indirect github.com/pmezard/go-difflib v1.0.0 github.com/pquerna/otp v1.2.0 - github.com/prometheus/common v0.15.0 github.com/robfig/cron v1.2.0 github.com/subosito/gozaru v0.0.0-20190625071150-416082cce636 github.com/syndtr/goleveldb v1.0.0 @@ -56,7 +55,6 @@ require ( golang.org/x/net v0.0.0-20200625001655-4c5254603344 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 - google.golang.org/appengine v1.6.6 google.golang.org/genproto v0.0.0-20200707001353-8e8330bf89df google.golang.org/grpc v1.29.1 ) diff --git a/go.sum b/go.sum index d59ee8f11..bc696a2b2 100644 --- a/go.sum +++ b/go.sum @@ -26,7 +26,6 @@ cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2k cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/pubsub v1.5.0/go.mod h1:ZEwJccE3z93Z2HWvstpri00jOg7oO4UZDtKhwDwqF0w= cloud.google.com/go/spanner v1.7.0/go.mod h1:sd3K2gZ9Fd0vMPLXzeCrF6fq4i63Q7aTLW/lBIfBkIk= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= @@ -67,7 +66,6 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= github.com/apache/beam v2.27.0+incompatible/go.mod h1:/8NX3Qi8vGstDLLaeaU7+lzVEu/ACaQhYjeefzQ0y1o= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -81,7 +79,6 @@ github.com/asdine/storm/v3 v3.0.0-20191014171123-c370e07ad6d4 h1:92RJlxO6DcRon/j github.com/asdine/storm/v3 v3.0.0-20191014171123-c370e07ad6d4/go.mod h1:wncSIXIbR3lvJQhBpnwAeNPQneL5Vx2KUox2jARUdmw= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -412,7 +409,6 @@ github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgO github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-redis/redis v6.15.8+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= @@ -472,8 +468,7 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -485,8 +480,6 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200507031123-427632fa3b1c/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/trillian v1.3.11 h1:pPzJPkK06mvXId1LHEAJxIegGgHzzp/FUnycPYfoCMI= -github.com/google/trillian v1.3.11/go.mod h1:0tPraVHrSDkA3BO6vKX67zgLXs6SsOAbHEivX+9mPgw= github.com/google/trillian v1.3.13 h1:V0avmojBPY7YAlcd/nUVvNRprU28tRsahPNxIedqekU= github.com/google/trillian v1.3.13/go.mod h1:8y3zC8XuqFxsslWPkP0r3sprERfFf7hCWmicL0yHZNI= github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -521,7 +514,6 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.2.2 h1:FlFbCRLd5Jr4iYXZufAvgWN6A github.com/grpc-ecosystem/go-grpc-middleware v1.2.2/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c h1:fEE5/5VNnYUoBOj2I9TP8Jc+a7lge3QWn9DKE7NCwfc= github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c/go.mod h1:ObS/W+h8RYb1Y7fYivughjxojTmIu5iAIjSrSLCLeqE= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= @@ -586,8 +578,6 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -598,8 +588,6 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/letsencrypt/pkcs11key/v4 v4.0.0/go.mod h1:EFUvBDay26dErnNb70Nd0/VW3tJiIbETBPTl9ATXQag= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= -github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.9.0 h1:L8nSXQQzAYByakOFMTwpjRoHsMJklur4Gi59b6VivR8= github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= @@ -733,7 +721,6 @@ github.com/robertkrimen/otto v0.0.0-20180617131154-15f95af6e78d/go.mod h1:xvqspo github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -790,7 +777,6 @@ go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= -go.etcd.io/etcd v0.0.0-20200513171258-e048e166ab9c/go.mod h1:xCI7ZzBfRuGgBXyXO6yfWfDmlWd35khcWpUa4L0xI/k= go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -881,7 +867,6 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1059,7 +1044,6 @@ google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dT google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= @@ -1132,11 +1116,9 @@ gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRN gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.6 h1:97YCGUei5WVbkKfogoJQsLwUJ17cWvpLrgNvlcbxikE= -gopkg.in/yaml.v2 v2.2.6/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 9a9e6fe5cbab153c5880ec7c0b97c9b593f36ee6 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 24 Feb 2021 16:51:21 -0600 Subject: [PATCH 337/449] Drop in leveldb store. --- politeiad/backend/tstorebe/store/fs/fs.go | 220 ------------------ .../backend/tstorebe/store/localdb/localdb.go | 126 ++++++++++ .../tstorebe/store/{fs => localdb}/log.go | 2 +- politeiad/backend/tstorebe/store/store.go | 2 +- politeiad/backend/tstorebe/tstore/testing.go | 7 +- politeiad/backend/tstorebe/tstore/tstore.go | 9 +- politeiad/log.go | 4 +- 7 files changed, 140 insertions(+), 230 deletions(-) delete mode 100644 politeiad/backend/tstorebe/store/fs/fs.go create mode 100644 politeiad/backend/tstorebe/store/localdb/localdb.go rename politeiad/backend/tstorebe/store/{fs => localdb}/log.go (97%) diff --git a/politeiad/backend/tstorebe/store/fs/fs.go b/politeiad/backend/tstorebe/store/fs/fs.go deleted file mode 100644 index f1f728ff5..000000000 --- a/politeiad/backend/tstorebe/store/fs/fs.go +++ /dev/null @@ -1,220 +0,0 @@ -// Copyright (c) 2020-2021 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package fs - -import ( - "errors" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "sync" - - "github.com/decred/politeia/politeiad/backend/tstorebe/store" - "github.com/google/uuid" -) - -var ( - _ store.BlobKV = (*fs)(nil) - - errNotFound = errors.New("not found") -) - -// fs implements the store.BlobKV interface using the file system. -// -// This implementation should be used for TESTING ONLY. -type fs struct { - sync.RWMutex - root string // Location of files -} - -// put saves a files to the file system. -// -// This function must be called WITH the lock held. -func (f *fs) put(key string, value []byte) error { - return ioutil.WriteFile(filepath.Join(f.root, key), value, 0600) -} - -// This function must be called WITH the lock held. -func (f *fs) del(key string) error { - err := os.Remove(filepath.Join(f.root, key)) - if err != nil { - if os.IsNotExist(err) { - return errNotFound - } - return err - } - return nil -} - -// get retrieves a file from the file system. -// -// This function must be called WITH the lock held. -func (f *fs) get(key string) ([]byte, error) { - b, err := ioutil.ReadFile(filepath.Join(f.root, key)) - if err != nil { - if os.IsNotExist(err) { - return nil, errNotFound - } - return nil, err - } - return b, nil -} - -// Put saves the provided blobs to the file system. The keys for the blobs -// are generated in this function and returned. -// -// This function satisfies the BlobKV interface. -func (f *fs) Put(blobs [][]byte) ([]string, error) { - log.Tracef("Put: %v", len(blobs)) - - f.Lock() - defer f.Unlock() - - // Save blobs to file system - keys := make([]string, 0, len(blobs)) - for _, v := range blobs { - key := uuid.New().String() - err := f.put(key, v) - if err != nil { - // Unwind blobs that have already been saved - for _, v := range keys { - err2 := f.del(v) - if err2 != nil { - // We're in trouble! - log.Criticalf("Failed to unwind put blob %v: %v", v, err2) - continue - } - } - return nil, err - } - keys = append(keys, key) - } - - log.Debugf("Saved blobs (%v) to store", len(blobs)) - - return keys, nil -} - -// Del deletes the files from the file system that correspond to the provided -// keys. -// -// This function satisfies the BlobKV interface. -func (f *fs) Del(keys []string) error { - log.Tracef("Del: %v", keys) - - f.Lock() - defer f.Unlock() - - // Temporarily store del files in case we need to unwind - dels := make(map[string][]byte, len(keys)) - for _, v := range keys { - b, err := f.get(v) - if err != nil { - return fmt.Errorf("get %v: %v", v, err) - } - dels[v] = b - } - - // Delete files - deleted := make([]string, 0, len(keys)) - for _, v := range keys { - err := f.del(v) - if err != nil { - if errors.Is(err, errNotFound) { - // File does not exist. This is ok. - continue - } - - // File does exist but del failed. Unwind deleted files. - for _, key := range deleted { - b := dels[key] - err2 := f.put(key, b) - if err2 != nil { - // We're in trouble! - log.Criticalf("Failed to unwind del blob %v: %v %x", key, err, b) - continue - } - } - return fmt.Errorf("del %v: %v", v, err) - } - - deleted = append(deleted, v) - } - - log.Debugf("Deleted blobs (%v) from store", len(keys)) - - return nil -} - -// Get returns blobs from the file system for the provided keys. An entry will -// not exist in the returned map if for any blobs that are not found. It is the -// responsibility of the caller to ensure a blob was returned for all provided -// keys. -// -// This function satisfies the BlobKV interface. -func (f *fs) Get(keys []string) (map[string][]byte, error) { - log.Tracef("Get: %v", keys) - - f.RLock() - defer f.RUnlock() - - blobs := make(map[string][]byte, len(keys)) - for _, v := range keys { - b, err := f.get(v) - if err != nil { - if errors.Is(err, errNotFound) { - // File does not exist. This is ok. - continue - } - return nil, fmt.Errorf("get %v: %v", v, err) - } - blobs[v] = b - } - - return blobs, nil -} - -// Enum enumerates over all blobs in the store, invoking the provided function -// for each blob. -func (f *fs) Enum(cb func(key string, blob []byte) error) error { - log.Tracef("Enum") - - f.RLock() - defer f.RUnlock() - - files, err := ioutil.ReadDir(f.root) - if err != nil { - return err - } - - for _, file := range files { - if file.Name() == ".." { - continue - } - blob, err := f.get(file.Name()) - if err != nil { - return err - } - err = cb(file.Name(), blob) - if err != nil { - return err - } - } - - return nil -} - -// Closes closes the blob store connection. -// -// This function satisfies the BlobKV interface. -func (f *fs) Close() {} - -// New returns a new fs. -func New(root string) *fs { - return &fs{ - root: root, - } -} diff --git a/politeiad/backend/tstorebe/store/localdb/localdb.go b/politeiad/backend/tstorebe/store/localdb/localdb.go new file mode 100644 index 000000000..e9d5522fc --- /dev/null +++ b/politeiad/backend/tstorebe/store/localdb/localdb.go @@ -0,0 +1,126 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package localdb + +import ( + "errors" + "fmt" + "sync" + + "github.com/decred/politeia/politeiad/backend/tstorebe/store" + "github.com/google/uuid" + "github.com/syndtr/goleveldb/leveldb" +) + +var ( + _ store.BlobKV = (*localdb)(nil) +) + +// localdb implements the store BlobKV interface using leveldb. +type localdb struct { + sync.Mutex + shutdown bool + root string // Location of database + db *leveldb.DB +} + +// Put saves the provided blobs to the store. The keys for the blobs are +// returned using the same odering that the blobs were provided in. This +// operation is performed atomically. +// +// This function satisfies the store BlobKV interface. +func (l *localdb) Put(blobs [][]byte) ([]string, error) { + log.Tracef("Put: %v", len(blobs)) + + // Setup batch + var ( + batch = new(leveldb.Batch) + keys = make([]string, 0, len(blobs)) + ) + for _, v := range blobs { + key := uuid.New().String() + batch.Put([]byte(key), v) + keys = append(keys, key) + } + + // Write batch + err := l.db.Write(batch, nil) + if err != nil { + return nil, err + } + + log.Debugf("Saved blobs (%v) to store", len(blobs)) + + return keys, nil +} + +// Del deletes the provided blobs from the store. This operation is performed +// atomically. +// +// This function satisfies the store BlobKV interface. +func (l *localdb) Del(keys []string) error { + log.Tracef("Del: %v", keys) + + batch := new(leveldb.Batch) + for _, v := range keys { + batch.Delete([]byte(v)) + } + err := l.db.Write(batch, nil) + if err != nil { + return err + } + + log.Debugf("Deleted blobs (%v) from store", len(keys)) + + return nil +} + +// Get returns blobs from the store for the provided keys. An entry will not +// exist in the returned map if for any blobs that are not found. It is the +// responsibility of the caller to ensure a blob was returned for all provided +// keys. +// +// This function satisfies the store BlobKV interface. +func (l *localdb) Get(keys []string) (map[string][]byte, error) { + log.Tracef("Get: %v", keys) + + blobs := make(map[string][]byte, len(keys)) + for _, v := range keys { + b, err := l.db.Get([]byte(v), nil) + if err != nil { + if errors.Is(err, leveldb.ErrNotFound) { + // File does not exist. This is ok. + continue + } + return nil, fmt.Errorf("get %v: %v", v, err) + } + blobs[v] = b + } + + return blobs, nil +} + +// Closes closes the store connection. +// +// This function satisfies the store BlobKV interface. +func (l *localdb) Close() { + l.Lock() + defer l.Unlock() + + l.db.Close() +} + +// New returns a new localdb. +func New(root string) (*localdb, error) { + db, err := leveldb.OpenFile(root, nil) + if err != nil { + return nil, err + } + + return &localdb{ + db: db, + root: root, + }, nil +} diff --git a/politeiad/backend/tstorebe/store/fs/log.go b/politeiad/backend/tstorebe/store/localdb/log.go similarity index 97% rename from politeiad/backend/tstorebe/store/fs/log.go rename to politeiad/backend/tstorebe/store/localdb/log.go index 07b628af4..3def5a168 100644 --- a/politeiad/backend/tstorebe/store/fs/log.go +++ b/politeiad/backend/tstorebe/store/localdb/log.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package fs +package localdb import "github.com/decred/slog" diff --git a/politeiad/backend/tstorebe/store/store.go b/politeiad/backend/tstorebe/store/store.go index 1bbef420c..0237c7e8a 100644 --- a/politeiad/backend/tstorebe/store/store.go +++ b/politeiad/backend/tstorebe/store/store.go @@ -92,6 +92,6 @@ type BlobKV interface { // was returned for all provided keys. Get(keys []string) (map[string][]byte, error) - // Closes closes the blob store connection. + // Closes closes the store connection. Close() } diff --git a/politeiad/backend/tstorebe/tstore/testing.go b/politeiad/backend/tstorebe/tstore/testing.go index 5abc7fa19..2c46f3c19 100644 --- a/politeiad/backend/tstorebe/tstore/testing.go +++ b/politeiad/backend/tstorebe/tstore/testing.go @@ -8,7 +8,7 @@ import ( "io/ioutil" "testing" - "github.com/decred/politeia/politeiad/backend/tstorebe/store/fs" + "github.com/decred/politeia/politeiad/backend/tstorebe/store/localdb" "github.com/marcopeereboom/sbox" ) @@ -27,7 +27,10 @@ func newTestTstore(t *testing.T, tstoreID, dataDir string, encrypt bool) *Tstore if err != nil { t.Fatal(err) } - store := fs.New(fp) + store, err := localdb.New(fp) + if err != nil { + t.Fatal(err) + } // Setup encryptin key if specified var ek *encryptionKey diff --git a/politeiad/backend/tstorebe/tstore/tstore.go b/politeiad/backend/tstorebe/tstore/tstore.go index 37d1da5de..85fd880d8 100644 --- a/politeiad/backend/tstorebe/tstore/tstore.go +++ b/politeiad/backend/tstorebe/tstore/tstore.go @@ -20,7 +20,7 @@ import ( "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/tstorebe/plugins" "github.com/decred/politeia/politeiad/backend/tstorebe/store" - "github.com/decred/politeia/politeiad/backend/tstorebe/store/fs" + "github.com/decred/politeia/politeiad/backend/tstorebe/store/localdb" "github.com/decred/politeia/politeiad/backend/tstorebe/store/mysql" "github.com/decred/politeia/util" "github.com/google/trillian" @@ -1433,8 +1433,10 @@ func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trilli if err != nil { return nil, err } - kvstore = fs.New(fp) - + kvstore, err = localdb.New(fp) + if err != nil { + return nil, err + } case DBTypeMySQL: // Example db name: testnet3_unvetted_kv dbName := fmt.Sprintf("%v_%v_kv", anp.Name, id) @@ -1442,7 +1444,6 @@ func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trilli if err != nil { return nil, err } - default: return nil, fmt.Errorf("invalid db type: %v", dbType) } diff --git a/politeiad/log.go b/politeiad/log.go index 2a0b02d11..2fa24492f 100644 --- a/politeiad/log.go +++ b/politeiad/log.go @@ -15,9 +15,9 @@ import ( "github.com/decred/politeia/politeiad/backend/tstorebe/plugins/dcrdata" "github.com/decred/politeia/politeiad/backend/tstorebe/plugins/ticketvote" "github.com/decred/politeia/politeiad/backend/tstorebe/plugins/usermd" - "github.com/decred/politeia/politeiad/backend/tstorebe/store/fs" "github.com/decred/politeia/politeiad/backend/tstorebe/store/mysql" "github.com/decred/politeia/politeiad/backend/tstorebe/tstore" + "github.com/decred/politeia/politeiawww/user/localdb" "github.com/decred/politeia/wsdcrdata" "github.com/decred/slog" "github.com/jrick/logrotate/rotator" @@ -66,7 +66,7 @@ func init() { // Tstore loggers tstore.UseLogger(tstoreLog) - fs.UseLogger(tstoreLog) + localdb.UseLogger(tstoreLog) mysql.UseLogger(tstoreLog) // Plugin loggers From bbecd7500ac44b25170f1b0ff9065660f710cde7 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 24 Feb 2021 17:04:32 -0600 Subject: [PATCH 338/449] ticketvote: Cast ballot bug fix. --- .../backend/tstorebe/plugins/ticketvote/activevotes.go | 4 ++++ politeiad/backend/tstorebe/plugins/ticketvote/cmds.go | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/politeiad/backend/tstorebe/plugins/ticketvote/activevotes.go b/politeiad/backend/tstorebe/plugins/ticketvote/activevotes.go index 74b30cc69..03208dfd6 100644 --- a/politeiad/backend/tstorebe/plugins/ticketvote/activevotes.go +++ b/politeiad/backend/tstorebe/plugins/ticketvote/activevotes.go @@ -143,6 +143,10 @@ func (a *activeVotes) VoteIsDuplicate(token, ticket string) bool { // returned if the provided token does not correspond to a record in the active // votes cache. func (a *activeVotes) CommitmentAddrs(token []byte, tickets []string) map[string]commitmentAddr { + if len(tickets) == 0 { + return map[string]commitmentAddr{} + } + a.RLock() defer a.RUnlock() diff --git a/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go b/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go index b3cf85c43..95203e61d 100644 --- a/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go @@ -2233,7 +2233,11 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str // commitment addresses have already been fetched. Any tickets // that are not found in the cache are fetched manually. tickets := make([]string, 0, len(cb.Ballot)) - for _, v := range cb.Ballot { + for k, v := range votes { + if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { + // Vote has an error. Skip it. + continue + } tickets = append(tickets, v.Ticket) } addrs := p.activeVotes.CommitmentAddrs(token, tickets) From 19655b414d20a7b6a44348c3b0419ad0cb013419 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 25 Feb 2021 10:01:20 -0600 Subject: [PATCH 339/449] tstorebe: Make record iteration consistent. --- .../backend/tstorebe/tstore/recordindex.go | 14 +---- politeiad/backend/tstorebe/tstorebe.go | 57 +++++++++++-------- 2 files changed, 35 insertions(+), 36 deletions(-) diff --git a/politeiad/backend/tstorebe/tstore/recordindex.go b/politeiad/backend/tstorebe/tstore/recordindex.go index 85031b26d..779746464 100644 --- a/politeiad/backend/tstorebe/tstore/recordindex.go +++ b/politeiad/backend/tstorebe/tstore/recordindex.go @@ -46,17 +46,8 @@ type recordIndex struct { // Iteration represents the iteration of the record. The iteration // is incremented anytime any record content changes. This includes - // file changes that bump the version as well metadata stream and - // record metadata changes that don't bump the version. - // - // Note, this field is not the same as the backend RecordMetadata - // iteration field, which does not get incremented on metadata - // updates. - // - // TODO maybe it should be the same. The original iteration field - // was to track unvetted changes in gitbe since unvetted gitbe - // records are not versioned. tstorebe unvetted records are versioned - // so the original use for the iteration field isn't needed anymore. + // file changes that bump the version, metadata stream only updates + // that don't bump the version, and status changes. Iteration uint32 `json:"iteration"` // The following fields contain the merkle leaf hashes of the @@ -76,7 +67,6 @@ type recordIndex struct { // allow any additional leaves to be appended onto the tree. Frozen bool `json:"frozen,omitempty"` - // TODO make this a generic ExtraData field // TreePointer is the tree ID of the tree that is the new location // of this record. A record can be copied to a new tree after // certain status changes, such as when a record is made public and diff --git a/politeiad/backend/tstorebe/tstorebe.go b/politeiad/backend/tstorebe/tstorebe.go index a54cbb48a..97d4127f4 100644 --- a/politeiad/backend/tstorebe/tstorebe.go +++ b/politeiad/backend/tstorebe/tstorebe.go @@ -648,10 +648,8 @@ func (t *tstoreBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite // Apply changes metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) files := filesUpdate(r.Files, filesAdd, filesDel) - - // Create record metadata recordMD, err := recordMetadataNew(token, files, - backend.MDStatusUnvetted, r.RecordMetadata.Iteration+1) + r.RecordMetadata.Status, r.RecordMetadata.Iteration+1) if err != nil { return nil, err } @@ -751,8 +749,6 @@ func (t *tstoreBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite [ // Apply changes metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) files := filesUpdate(r.Files, filesAdd, filesDel) - - // Create record metadata recordMD, err := recordMetadataNew(token, files, r.RecordMetadata.Status, r.RecordMetadata.Iteration+1) if err != nil { @@ -856,6 +852,11 @@ func (t *tstoreBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwri // Apply changes metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) + recordMD, err := recordMetadataNew(token, r.Files, + r.RecordMetadata.Status, r.RecordMetadata.Iteration+1) + if err != nil { + return err + } // Call pre plugin hooks hem := plugins.HookEditMetadata{ @@ -874,7 +875,7 @@ func (t *tstoreBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwri } // Update metadata - err = t.unvetted.RecordMetadataSave(treeID, r.RecordMetadata, metadata) + err = t.unvetted.RecordMetadataSave(treeID, *recordMD, metadata) if err != nil { switch err { case backend.ErrRecordLocked, backend.ErrNoChanges: @@ -948,6 +949,11 @@ func (t *tstoreBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite // Apply changes metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) + recordMD, err := recordMetadataNew(token, r.Files, + r.RecordMetadata.Status, r.RecordMetadata.Iteration+1) + if err != nil { + return err + } // Call pre plugin hooks hem := plugins.HookEditMetadata{ @@ -966,7 +972,7 @@ func (t *tstoreBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite } // Update metadata - err = t.vetted.RecordMetadataSave(treeID, r.RecordMetadata, metadata) + err = t.vetted.RecordMetadataSave(treeID, *recordMD, metadata) if err != nil { switch err { case backend.ErrRecordLocked, backend.ErrNoChanges: @@ -1097,8 +1103,7 @@ func (t *tstoreBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT if err != nil { return nil, fmt.Errorf("RecordLatest: %v", err) } - rm := r.RecordMetadata - currStatus := rm.Status + currStatus := r.RecordMetadata.Status // Validate status change if !statusChangeIsAllowed(currStatus, status) { @@ -1109,9 +1114,11 @@ func (t *tstoreBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT } // Apply status change - rm.Status = status - rm.Iteration += 1 - rm.Timestamp = time.Now().Unix() + recordMD, err := recordMetadataNew(token, r.Files, status, + r.RecordMetadata.Iteration+1) + if err != nil { + return nil, err + } // Apply metadata changes metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) @@ -1120,7 +1127,7 @@ func (t *tstoreBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT hsrs := plugins.HookSetRecordStatus{ State: plugins.RecordStateUnvetted, Current: *r, - RecordMetadata: rm, + RecordMetadata: *recordMD, Metadata: metadata, } b, err := json.Marshal(hsrs) @@ -1138,12 +1145,13 @@ func (t *tstoreBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT var vettedTreeID int64 switch status { case backend.MDStatusVetted: - vettedTreeID, err = t.unvettedPublish(token, rm, metadata, r.Files) + vettedTreeID, err = t.unvettedPublish(token, *recordMD, + metadata, r.Files) if err != nil { return nil, fmt.Errorf("unvettedPublish: %v", err) } case backend.MDStatusCensored: - err := t.unvettedCensor(token, rm, metadata) + err := t.unvettedCensor(token, *recordMD, metadata) if err != nil { return nil, fmt.Errorf("unvettedCensor: %v", err) } @@ -1271,11 +1279,10 @@ func (t *tstoreBackend) SetVettedStatus(token []byte, status backend.MDStatusT, if err != nil { return nil, fmt.Errorf("RecordLatest: %v", err) } - rm := r.RecordMetadata - currStatus := rm.Status + currStatus := r.RecordMetadata.Status // Validate status change - if !statusChangeIsAllowed(rm.Status, status) { + if !statusChangeIsAllowed(currStatus, status) { return nil, backend.StateTransitionError{ From: currStatus, To: status, @@ -1283,9 +1290,11 @@ func (t *tstoreBackend) SetVettedStatus(token []byte, status backend.MDStatusT, } // Apply status change - rm.Status = status - rm.Iteration += 1 - rm.Timestamp = time.Now().Unix() + recordMD, err := recordMetadataNew(token, r.Files, status, + r.RecordMetadata.Iteration+1) + if err != nil { + return nil, err + } // Apply metdata changes metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) @@ -1294,7 +1303,7 @@ func (t *tstoreBackend) SetVettedStatus(token []byte, status backend.MDStatusT, srs := plugins.HookSetRecordStatus{ State: plugins.RecordStateVetted, Current: *r, - RecordMetadata: rm, + RecordMetadata: *recordMD, Metadata: metadata, } b, err := json.Marshal(srs) @@ -1310,12 +1319,12 @@ func (t *tstoreBackend) SetVettedStatus(token []byte, status backend.MDStatusT, // Update record switch status { case backend.MDStatusCensored: - err := t.vettedCensor(token, rm, metadata) + err := t.vettedCensor(token, *recordMD, metadata) if err != nil { return nil, fmt.Errorf("vettedCensor: %v", err) } case backend.MDStatusArchived: - err := t.vettedArchive(token, rm, metadata) + err := t.vettedArchive(token, *recordMD, metadata) if err != nil { return nil, fmt.Errorf("vettedArchive: %v", err) } From 0bcfeb06c8846612d8d347e1a020c9ae23e756c2 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 25 Feb 2021 10:14:29 -0600 Subject: [PATCH 340/449] Include plugin ID in mdstream keys. --- politeiad/api/v1/v1.go | 4 ++-- politeiad/backend/backend.go | 2 +- politeiad/backend/tstorebe/tstore/recordindex.go | 2 +- politeiad/backend/tstorebe/tstore/tstore.go | 13 ++++++------- politeiad/politeiad.go | 2 +- politeiawww/api/records/v1/v1.go | 10 +++++----- politeiawww/records/process.go | 2 +- 7 files changed, 17 insertions(+), 18 deletions(-) diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index dbd4e7a7a..c79a71a4d 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -407,8 +407,8 @@ type RecordTimestamps struct { Version string `json:"version"` // Version of files RecordMetadata Timestamp `json:"recordmetadata"` - // map[metadataID]Timestamp - Metadata map[uint64]Timestamp `json:"metadata"` + // map[pluginID+metadataID]Timestamp + Metadata map[string]Timestamp `json:"metadata"` // map[filename]Timestamp Files map[string]Timestamp `json:"files"` diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index 7446f21b9..27955fb09 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -200,7 +200,7 @@ type RecordTimestamps struct { Token string // Censorship token Version string // Version of files RecordMetadata Timestamp - Metadata map[uint64]Timestamp // [metadataID]Timestamp + Metadata map[string]Timestamp // [metadataID]Timestamp Files map[string]Timestamp // [filename]Timestamp } diff --git a/politeiad/backend/tstorebe/tstore/recordindex.go b/politeiad/backend/tstorebe/tstore/recordindex.go index 779746464..ad32223cc 100644 --- a/politeiad/backend/tstorebe/tstore/recordindex.go +++ b/politeiad/backend/tstorebe/tstore/recordindex.go @@ -55,7 +55,7 @@ type recordIndex struct { // can be used to lookup the log leaf. The log leaf ExtraData field // contains the key for the record content in the key-value store. RecordMetadata []byte `json:"recordmetadata"` - Metadata map[uint64][]byte `json:"metadata"` // [metadataID]merkle + Metadata map[string][]byte `json:"metadata"` // [pluginID+ID]merkle Files map[string][]byte `json:"files"` // [filename]merkle // Frozen is used to indicate that the tree for this record has diff --git a/politeiad/backend/tstorebe/tstore/tstore.go b/politeiad/backend/tstorebe/tstore/tstore.go index 85fd880d8..f14c4e9a5 100644 --- a/politeiad/backend/tstorebe/tstore/tstore.go +++ b/politeiad/backend/tstorebe/tstore/tstore.go @@ -299,7 +299,7 @@ func (t *Tstore) treeIsFrozen(leaves []*trillian.LogLeaf) bool { type recordHashes struct { recordMetadata string // Record metadata hash - metadata map[string]uint64 // [hash]metadataID + metadata map[string]string // [hash]metadataID files map[string]string // [hash]filename } @@ -370,7 +370,7 @@ func (t *Tstore) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD back // Compute record content hashes rhashes := recordHashes{ - metadata: make(map[string]uint64, len(metadata)), // [hash]metadataID + metadata: make(map[string]string, len(metadata)), // [hash]metadataID files: make(map[string]string, len(files)), // [hash]filename } b, err := json.Marshal(recordMD) @@ -384,7 +384,7 @@ func (t *Tstore) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD back return nil, err } h := hex.EncodeToString(util.Digest(b)) - rhashes.metadata[h] = v.ID + rhashes.metadata[h] = v.PluginID + strconv.FormatUint(v.ID, 10) } for _, v := range files { b, err := json.Marshal(v) @@ -404,7 +404,7 @@ func (t *Tstore) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD back // Any duplicates that are found are added to the record index // since we already have the leaf data for them. index = recordIndex{ - Metadata: make(map[uint64][]byte, len(metadata)), + Metadata: make(map[string][]byte, len(metadata)), Files: make(map[string][]byte, len(files)), } ) @@ -631,7 +631,7 @@ func (t *Tstore) RecordSave(treeID int64, rm backend.RecordMetadata, metadata [] if errors.Is(err, backend.ErrRecordNotFound) { // No record versions exist yet. This is ok. currIdx = &recordIndex{ - Metadata: make(map[uint64][]byte), + Metadata: make(map[string][]byte), Files: make(map[string][]byte), } } else if err != nil { @@ -1330,7 +1330,7 @@ func (t *Tstore) RecordTimestamps(treeID int64, version uint32, token []byte) (* } // Get metadata timestamps - metadata := make(map[uint64]backend.Timestamp, len(idx.Metadata)) + metadata := make(map[string]backend.Timestamp, len(idx.Metadata)) for k, v := range idx.Metadata { ts, err := t.timestamp(treeID, v, leaves) if err != nil { @@ -1358,7 +1358,6 @@ func (t *Tstore) RecordTimestamps(treeID int64, version uint32, token []byte) (* }, nil } -// TODO run fsck episodically func (t *Tstore) Fsck() { // Set tree status to frozen for any trees that are frozen and have // been anchored one last time. diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index c5742aa31..adbc29aac 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -116,7 +116,7 @@ func convertBackendTimestamp(t backend.Timestamp) v1.Timestamp { } func convertBackendRecordTimestamps(rt backend.RecordTimestamps) v1.RecordTimestamps { - md := make(map[uint64]v1.Timestamp, len(rt.Metadata)) + md := make(map[string]v1.Timestamp, len(rt.Metadata)) for k, v := range rt.Metadata { md[k] = convertBackendTimestamp(v) } diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go index d6b11e45b..f0c9aecf6 100644 --- a/politeiawww/api/records/v1/v1.go +++ b/politeiawww/api/records/v1/v1.go @@ -167,9 +167,9 @@ type File struct { // MetadataStream describes a record metadata stream. type MetadataStream struct { - PluginID string `json:"pluginid,omitempty"` // Plugin ID - ID uint64 `json:"id"` // Metadata stream ID - Payload string `json:"payload"` // JSON encoded + PluginID string `json:"pluginid"` // Plugin ID + ID uint64 `json:"id"` // Metadata stream ID + Payload string `json:"payload"` // JSON encoded } // CensorshipRecord contains cryptographic proof that a record was accepted for @@ -386,8 +386,8 @@ type Timestamps struct { type TimestampsReply struct { RecordMetadata Timestamp `json:"recordmetadata"` - // map[metadataID]Timestamp - Metadata map[uint64]Timestamp `json:"metadata"` + // map[pluginID+metadataID]Timestamp + Metadata map[string]Timestamp `json:"metadata"` // map[filename]Timestamp Files map[string]Timestamp `json:"files"` diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index e00d1f57f..d640cb54b 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -497,7 +497,7 @@ func (r *Records) processTimestamps(ctx context.Context, t v1.Timestamps, isAdmi var ( recordMD = convertTimestampToV1(rt.RecordMetadata) - metadata = make(map[uint64]v1.Timestamp, len(rt.Files)) + metadata = make(map[string]v1.Timestamp, len(rt.Files)) files = make(map[string]v1.Timestamp, len(rt.Files)) ) for k, v := range rt.Metadata { From 2e8c64283c5eb86368dee8a03ac4373649afda7b Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 25 Feb 2021 16:51:39 -0600 Subject: [PATCH 341/449] usermd: Bug fix. --- politeiad/backend/tstorebe/plugins/usermd/hooks.go | 5 +++-- politeiad/plugins/usermd/usermd.go | 2 +- politeiawww/records/process.go | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/politeiad/backend/tstorebe/plugins/usermd/hooks.go b/politeiad/backend/tstorebe/plugins/usermd/hooks.go index bf89fed0c..c325da8c6 100644 --- a/politeiad/backend/tstorebe/plugins/usermd/hooks.go +++ b/politeiad/backend/tstorebe/plugins/usermd/hooks.go @@ -287,7 +287,7 @@ func statusChangeMetadataVerify(rm backend.RecordMetadata, metadata []backend.Me } // Verify status matches - if scm.Status != int(rm.Status) { + if scm.Status != uint32(rm.Status) { e := fmt.Sprintf("status from metadata does not match status from "+ "record metadata: got %v, want %v", scm.Status, rm.Status) return backend.PluginError{ @@ -308,7 +308,8 @@ func statusChangeMetadataVerify(rm backend.RecordMetadata, metadata []backend.Me } // Verify signature - msg := scm.Token + scm.Version + strconv.Itoa(scm.Status) + scm.Reason + s := strconv.FormatUint(uint64(scm.Status), 10) + msg := scm.Token + scm.Version + s + scm.Reason err = util.VerifySignature(scm.Signature, scm.PublicKey, msg) if err != nil { return convertSignatureError(err) diff --git a/politeiad/plugins/usermd/usermd.go b/politeiad/plugins/usermd/usermd.go index f5f2de07b..39f77f986 100644 --- a/politeiad/plugins/usermd/usermd.go +++ b/politeiad/plugins/usermd/usermd.go @@ -74,7 +74,7 @@ type UserMetadata struct { type StatusChangeMetadata struct { Token string `json:"token"` Version string `json:"version"` - Status int `json:"status"` + Status uint32 `json:"status"` Reason string `json:"message,omitempty"` PublicKey string `json:"publickey"` Signature string `json:"signature"` diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index d640cb54b..2e7876a09 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -228,7 +228,7 @@ func (r *Records) processSetStatus(ctx context.Context, ss v1.SetStatus, u user. scm := usermd.StatusChangeMetadata{ Token: ss.Token, Version: ss.Version, - Status: int(ss.Status), + Status: uint32(ss.Status), Reason: ss.Reason, PublicKey: ss.PublicKey, Signature: ss.Signature, From fe9c40ab7e8dfc68e4bdcf3d9f3d01aa9f21cf06 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 25 Feb 2021 18:59:30 -0600 Subject: [PATCH 342/449] politeiad: Add startup docs. --- politeiad/README.md | 220 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 209 insertions(+), 11 deletions(-) diff --git a/politeiad/README.md b/politeiad/README.md index fe6abf2b1..67f29e99e 100644 --- a/politeiad/README.md +++ b/politeiad/README.md @@ -40,32 +40,230 @@ politeiad ## Build from source -## Setup configuration file +1. Install Mariadb or MySQL. Make sure to setup a password for the root user. -[`sample-politeiad.conf`](https://github.com/decred/politeia/blob/master/politeiad/sample-politeiad.conf) +2. Update the MySQL max connections settings. -Copy the sample configuration file to the politeiad data directory for your OS. + Max connections defaults to 151 which is not enough for trillian. You will + be prompted for the MySQL root user's password when running these commands. -* **macOS** + ``` + # Update max connections + $ mysql -u root -p -e "SET GLOBAL max_connections = 2000;" - `/Users//Library/Application Support/Politeiad/politeiad.conf` + # Verify the setting + $ mysql -u root -p -e "SHOW VARIABLES LIKE 'max_connections';" + ``` -* **Windows** + You can also update the config file so you don't need to set it manually in + the future. Make sure to restart MySQL once you update the config file. - `C:\Users\\AppData\Local\Politeiad/politeiad.conf` + MariaDB config file: `/etc/mysql/mariadb.cnf` + MySQL config file: `/etc/mysql/my.cnf` + + ``` + [mysqld] + max_connections = 2000 + ``` -* **Ubuntu** +3. Install trillian v1.3.13. - `~/.politeiad/politeiad.conf` + ``` + $ mkdir -p $GOPATH/src/github.com/google/ + $ cd $GOPATH/src/github.com/google/ + $ git clone git@github.com:google/trillian.git + $ cd trillian + $ git checkout tags/v1.3.13 -b v1.3.13 + $ go install -v ./... + ``` -Use the following config settings to spin up a development politeiad instance. +4. Install politeia. -**politeiad.conf**: + ``` + $ mkdir -p $GOPATH/src/github.com/decred + $ cd $GOPATH/src/github.com/decred + $ git clone git@github.com:decred/politeia.git + $ cd politeia + $ go install -v ./... + ``` +5. Run the politeiad mysql setup scripts. + + This will create the politeiad and trillian users as well as creating the + politeiad databases. Trillian does not support SSL authentication to the + MySQL instance. Password authentication must be used. + + The setup script assumes MySQL is running on `localhost:3306` and the users + will be accessing the databse from `localhost`. See the setup script + comments for more complex setups. + + Run the following commands. You will need to replace "rootpass" with the + existing password of your root user. The "politeiadpass" and "trillianpass" + are the password that will be set for the politeiad and trillian users when + the script creates them. + + ``` + $ cd $GOPATH/src/github.com/decred/politeia/politeiad/scripts + $ env \ + MYSQL_ROOT_PASSWORD=rootpass \ + MYSQL_POLITEIAD_PASSWORD=politeiadpass \ + MYSQL_TRILLIAN_PASSWORD=trillianpass \ + ./mysql-tstore-setup.sh + ``` + +4. Run the trillian mysql setup scripts. + + These can only be run once the trillian MySQL user has been created in the + previous step. + + The "trillianpass" and "rootpass" will need to be updated to the passwords + for your trillian and root users. + + ``` + $ cd $GOPATH/src/github.com/google/trillian/scripts + + # Unvetted setup + $ env \ + MYSQL_USER=trillian \ + MYSQL_PASSWORD=trillianpass \ + MYSQL_DATABASE=testnet3_unvetted_trillian \ + MYSQL_URI="${MYSQL_USER}:${MYSQL_PASSWORD}@tcp(127.0.0.1:3306)/${MYSQL_DATABASE}" \ + MYSQL_ROOT_PASSWORD=rootpass \ + ./resetdb.sh + + # Vetted setup + $ env \ + MYSQL_USER=trillian \ + MYSQL_PASSWORD=trillianpass \ + MYSQL_DATABASE=testnet3_vetted_trillian \ + MYSQL_URI="${MYSQL_USER}:${MYSQL_PASSWORD}@tcp(127.0.0.1:3306)/${MYSQL_DATABASE}" \ + MYSQL_ROOT_PASSWORD=rootpass \ + ./resetdb.sh + + ``` + +5. Start up the trillian instances. + + Running trillian requires running a trillian log server and a trillian log + signer. These are seperate processes. Politeiad runs a trillian instance + for unvetted records and a trillian instance for vetted records. You will be + starting up 4 seperate processes in this step. + + You will need to replace the "trillianpass" with the trillian user's + password that you setup in previous steps. + + Startup unvetted log server + ``` + $ export MYSQL_USER=trillian && \ + export MYSQL_PASSWORD=trillianpass && \ + export MYSQL_DATABASE=testnet3_unvetted_trillian && \ + export MYSQL_URI="${MYSQL_USER}:${MYSQL_PASSWORD}@tcp(127.0.0.1:3306)/${MYSQL_DATABASE}" + + $ trillian_log_server \ + --mysql_uri=${MYSQL_URI} \ + --mysql_max_conns=2000 \ + --rpc_endpoint localhost:8090 \ + --http_endpoint localhost:8091 \ + --logtostderr ... + ``` + + Startup unvetted log signer + ``` + $ export MYSQL_USER=trillian && \ + export MYSQL_PASSWORD=trillianpass && \ + export MYSQL_DATABASE=testnet3_unvetted_trillian && \ + export MYSQL_URI="${MYSQL_USER}:${MYSQL_PASSWORD}@tcp(127.0.0.1:3306)/${MYSQL_DATABASE}" + + $ trillian_log_signer --logtostderr --force_master \ + --batch_size=1000 \ + --sequencer_guard_window=0 \ + --sequencer_interval=200ms \ + --mysql_uri=${MYSQL_URI} \ + --rpc_endpoint localhost:8092 \ + --http_endpoint=localhost:8093 + ``` + + Startup vetted log server + ``` + $ export MYSQL_USER=trillian && \ + export MYSQL_PASSWORD=trillianpass && \ + export MYSQL_DATABASE=testnet3_vetted_trillian && \ + export MYSQL_URI="${MYSQL_USER}:${MYSQL_PASSWORD}@tcp(127.0.0.1:3306)/${MYSQL_DATABASE}" + + $ trillian_log_server \ + --mysql_uri=${MYSQL_URI} \ + --mysql_max_conns=2000 \ + --rpc_endpoint localhost:8094 \ + --http_endpoint localhost:8095 \ + --logtostderr ... + ``` + + Startup vetted log signer + ``` + $ export MYSQL_USER=trillian && \ + export MYSQL_PASSWORD=trillianpass && \ + export MYSQL_DATABASE=testnet3_vetted_trillian && \ + export MYSQL_URI="${MYSQL_USER}:${MYSQL_PASSWORD}@tcp(127.0.0.1:3306)/${MYSQL_DATABASE}" + + $ trillian_log_signer --logtostderr --force_master \ + --batch_size=1000 \ + --sequencer_guard_window=0 \ + --sequencer_interval=200ms \ + --mysql_uri=${MYSQL_URI} \ + --rpc_endpoint localhost:8096 \ + --http_endpoint=localhost:8097 + ``` + + +5. Setup the politeiad configuration file. + + [`sample-politeiad.conf`](https://github.com/decred/politeia/blob/master/politeiad/sample-politeiad.conf) + + Copy the sample configuration file to the politeiad data directory. The data + directory will depend on your OS. + + * **macOS** + + `/Users//Library/Application Support/Politeiad/politeiad.conf` + + * **Windows** + + `C:\Users\\AppData\Local\Politeiad/politeiad.conf` + + * **Unix** + + `~/.politeiad/politeiad.conf` + + ``` + $ mkdir -p ${HOME}/.politeiad/ + $ cd $GOPATH/src/github.com/decred/politeia/politeiad + $ cp ./sample-politeiad.conf ${HOME}/.politeiad/politeiad.conf + ``` + + Use the following config settings to spin up a development politeiad + instance. + + **politeiad.conf** + + ``` rpcuser=user rpcpass=pass testnet=true + ; Pi plugin configuration + plugin=pi + plugin=comments + plugin=dcrdata + plugin=ticketvote + plugin=usermd + ``` + +6. Start up the politeiad instance. + + ``` + $ politeiad + ``` + # Tools and reference clients * [politeia](https://github.com/decred/politeia/tree/master/politeiad/cmd/politeia) - Reference client for politeiad. From 170313deaadbfba1da19ba14ba41f8087eb84e08 Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 26 Feb 2021 10:02:57 -0600 Subject: [PATCH 343/449] ticketvote: Add linkby testnet defaults. --- .../tstorebe/plugins/ticketvote/ticketvote.go | 15 +++++--- politeiad/plugins/ticketvote/ticketvote.go | 36 ++++++++++--------- 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/politeiad/backend/tstorebe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tstorebe/plugins/ticketvote/ticketvote.go index 83851488e..bc0c0a2b7 100644 --- a/politeiad/backend/tstorebe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tstorebe/plugins/ticketvote/ticketvote.go @@ -281,8 +281,8 @@ func (p *ticketVotePlugin) Settings() []backend.PluginSetting { func New(backend backend.Backend, tstore plugins.TstoreClient, settings []backend.PluginSetting, dataDir string, id *identity.FullIdentity, activeNetParams *chaincfg.Params) (*ticketVotePlugin, error) { // Plugin settings var ( - linkByPeriodMin = ticketvote.SettingLinkByPeriodMin - linkByPeriodMax = ticketvote.SettingLinkByPeriodMax + linkByPeriodMin int64 + linkByPeriodMax int64 voteDurationMin uint32 voteDurationMax uint32 ) @@ -291,14 +291,21 @@ func New(backend backend.Backend, tstore plugins.TstoreClient, settings []backen // the setting was specified by the user. switch activeNetParams.Name { case chaincfg.MainNetParams().Name: + linkByPeriodMin = ticketvote.SettingMainNetLinkByPeriodMin + linkByPeriodMax = ticketvote.SettingMainNetLinkByPeriodMax voteDurationMin = ticketvote.SettingMainNetVoteDurationMin voteDurationMax = ticketvote.SettingMainNetVoteDurationMax case chaincfg.TestNet3Params().Name: + linkByPeriodMin = ticketvote.SettingTestNetLinkByPeriodMin + linkByPeriodMax = ticketvote.SettingTestNetLinkByPeriodMax voteDurationMin = ticketvote.SettingTestNetVoteDurationMin voteDurationMax = ticketvote.SettingTestNetVoteDurationMax case chaincfg.SimNetParams().Name: - voteDurationMin = ticketvote.SettingSimNetVoteDurationMin - voteDurationMax = ticketvote.SettingSimNetVoteDurationMax + // Use testnet defaults for simnet + linkByPeriodMin = ticketvote.SettingTestNetLinkByPeriodMin + linkByPeriodMax = ticketvote.SettingTestNetLinkByPeriodMax + voteDurationMin = ticketvote.SettingTestNetVoteDurationMin + voteDurationMax = ticketvote.SettingTestNetVoteDurationMax default: return nil, fmt.Errorf("unknown active net: %v", activeNetParams.Name) } diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index e0580250d..0589f81ca 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -29,16 +29,28 @@ const ( SettingKeyVoteDurationMin = "votedurationmin" SettingKeyVoteDurationMax = "votedurationmax" - // SettingLinkByPeriodMin is the default minimum amount of time, - // in seconds, that the link by period can be set to. This value - // of 2 weeks was chosen assuming a 1 week voting period on + // SettingMainNetLinkByPeriodMin is the default minimum amount of + // time, in seconds, that the link by period can be set to. This + // value of 2 weeks was chosen assuming a 1 week voting period on // mainnet. - SettingLinkByPeriodMin int64 = 1209600 + SettingMainNetLinkByPeriodMin int64 = 1209600 - // SettingLinkByPeriodMax is the default maximum amount of time, - // in seconds, that the link by period can be set to. This value - // of 3 months was chosen arbitrarily. - SettingLinkByPeriodMax int64 = 7776000 + // SettingMainNetLinkByPeriodMax is the default maximum amount of + // time, in seconds, that the link by period can be set to. This + // value of 3 months was chosen arbitrarily. + SettingMainNetLinkByPeriodMax int64 = 7776000 + + // SettingTestNeLinkByPeriodMin is the default minimum amount of + // time, in seconds, that the link by period can be set to. This + // value of 1 second was chosen because this is the testnet + // default and a 1 second miniumum makes testing various scenerios + // easier. + SettingTestNetLinkByPeriodMin int64 = 1 + + // SettingTestNetLinkByPeriodMax is the default maximum amount of + // time, in seconds, that the link by period can be set to. This + // value of 3 months was chosen arbitrarily. + SettingTestNetLinkByPeriodMax int64 = 7776000 // SettingMainNetVoteDurationMin is the default minimum vote // duration on mainnet in blocks. @@ -55,14 +67,6 @@ const ( // SettingTestNetVoteDurationMax is the default maximum vote // duration on testnet in blocks. SettingTestNetVoteDurationMax uint32 = 4032 - - // SettingSimNetVoteDurationMin is the default minimum vote - // duration on simnet in blocks. - SettingSimNetVoteDurationMin uint32 = 1 - - // SettingSimNetVoteDurationMax is the default maximum vote - // duration on simnet in blocks. - SettingSimNetVoteDurationMax uint32 = 4032 ) // ErrorCodeT represents and error that is caused by the user. From 39c00828ec998793e56cee664e1a7f5b98790f8d Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 26 Feb 2021 16:51:40 -0600 Subject: [PATCH 344/449] politeiawww: Setup pi events. --- politeiawww/comments/pi.go | 9 +- politeiawww/config.go | 15 +- politeiawww/config/config.go | 6 +- politeiawww/email.go | 281 +--------- politeiawww/events.go | 471 ----------------- politeiawww/events/events.go | 4 + politeiawww/events/log.go | 25 + politeiawww/log.go | 34 +- politeiawww/mail/log.go | 25 + politeiawww/mail/mail.go | 108 ++++ politeiawww/middleware.go | 4 +- politeiawww/pi/events.go | 692 +++++++++++++++++++++++++ politeiawww/pi/pi.go | 15 +- politeiawww/piwww.go | 17 +- politeiawww/politeiad.go | 2 +- politeiawww/politeiawww.go | 7 +- politeiawww/records/pi.go | 9 +- politeiawww/sessions/sessions.go | 19 +- politeiawww/sessions/store.go | 22 +- politeiawww/smtp.go | 104 ---- politeiawww/testing.go | 13 +- politeiawww/user.go | 10 +- politeiawww/{pi/user.go => user/pi.go} | 6 +- politeiawww/www.go | 17 +- util/net.go | 2 +- 25 files changed, 989 insertions(+), 928 deletions(-) create mode 100644 politeiawww/events/log.go create mode 100644 politeiawww/mail/log.go create mode 100644 politeiawww/mail/mail.go create mode 100644 politeiawww/pi/events.go delete mode 100644 politeiawww/smtp.go rename politeiawww/{pi/user.go => user/pi.go} (90%) diff --git a/politeiawww/comments/pi.go b/politeiawww/comments/pi.go index 767c135bb..26a09a20a 100644 --- a/politeiawww/comments/pi.go +++ b/politeiawww/comments/pi.go @@ -6,7 +6,6 @@ package comments import ( v1 "github.com/decred/politeia/politeiawww/api/comments/v1" - "github.com/decred/politeia/politeiawww/pi" "github.com/decred/politeia/politeiawww/user" ) @@ -34,8 +33,8 @@ func (c *Comments) piHookNewPre(u user.User) error { // Verify user has paid registration paywall if !userHasPaid(u) { return v1.PluginErrorReply{ - PluginID: pi.UserPluginID, - ErrorCode: pi.ErrorCodeUserRegistrationNotPaid, + PluginID: user.PiUserPluginID, + ErrorCode: user.ErrorCodeUserRegistrationNotPaid, } } @@ -50,8 +49,8 @@ func (c *Comments) piHookVotePre(u user.User) error { // Verify user has paid registration paywall if !userHasPaid(u) { return v1.PluginErrorReply{ - PluginID: pi.UserPluginID, - ErrorCode: pi.ErrorCodeUserRegistrationNotPaid, + PluginID: user.PiUserPluginID, + ErrorCode: user.ErrorCodeUserRegistrationNotPaid, } } diff --git a/politeiawww/config.go b/politeiawww/config.go index 83f10e010..0097cb30c 100644 --- a/politeiawww/config.go +++ b/politeiawww/config.go @@ -619,17 +619,17 @@ func loadConfig() (*config.Config, []string, error) { cfg.WebServerAddress = u.String() // Validate smtp root cert. - if cfg.SMTPCert != "" { - cfg.SMTPCert = util.CleanAndExpandPath(cfg.SMTPCert) + if cfg.MailCert != "" { + cfg.MailCert = util.CleanAndExpandPath(cfg.MailCert) - b, err := ioutil.ReadFile(cfg.SMTPCert) + b, err := ioutil.ReadFile(cfg.MailCert) if err != nil { - return nil, nil, fmt.Errorf("read smtpcert: %v", err) + return nil, nil, fmt.Errorf("read mailcert: %v", err) } block, _ := pem.Decode(b) cert, err := x509.ParseCertificate(block.Bytes) if err != nil { - return nil, nil, fmt.Errorf("parse smtpcert: %v", err) + return nil, nil, fmt.Errorf("parse mailcert: %v", err) } systemCerts, err := x509.SystemCertPool() if err != nil { @@ -638,8 +638,9 @@ func loadConfig() (*config.Config, []string, error) { systemCerts.AddCert(cert) cfg.SystemCerts = systemCerts - if cfg.SMTPSkipVerify { - log.Warnf("SMTPCert has been set so SMTPSkipVerify is being disregarded") + if cfg.MailSkipVerify && cfg.MailCert != "" { + return nil, nil, fmt.Errorf("cannot set MailSkipVerify and provide " + + "a MailCert at the same time") } } diff --git a/politeiawww/config/config.go b/politeiawww/config/config.go index 78b2d140f..e1802de1e 100644 --- a/politeiawww/config/config.go +++ b/politeiawww/config/config.go @@ -83,9 +83,9 @@ type Config struct { MailUser string `long:"mailuser" description:"Email server username"` MailPass string `long:"mailpass" description:"Email server password"` MailAddress string `long:"mailaddress" description:"Email address for outgoing email in the format: name
"` - SMTPCert string `long:"smtpcert" description:"File containing the smtp certificate file"` - SMTPSkipVerify bool `long:"smtpskipverify" description:"Skip SMTP TLS cert verification. Will only skip if SMTPCert is empty"` - WebServerAddress string `long:"webserveraddress" description:"Address for the Politeia web server; it should have this format: ://[:]"` + MailCert string `long:"mailcert" description:"Email server certificate file"` + MailSkipVerify bool `long:"mailskipverify" description:"Skip TLS verification when connecting to the mail server"` + WebServerAddress string `long:"webserveraddress" description:"Web server address used to create email links (format: ://[:])"` // XXX These should all be plugin settings DcrdataHost string `long:"dcrdatahost" description:"Dcrdata ip:port"` diff --git a/politeiawww/email.go b/politeiawww/email.go index 97f19338c..f1f7a4345 100644 --- a/politeiawww/email.go +++ b/politeiawww/email.go @@ -6,13 +6,11 @@ package main import ( "bytes" - "fmt" "net/url" "strings" "text/template" "time" - rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" ) @@ -56,261 +54,6 @@ func (p *politeiawww) createEmailLink(path, email, token, username string) (stri return l.String(), nil } -// emailProposalSubmitted send a proposal submitted notification email to -// the provided list of emails. -func (p *politeiawww) emailProposalSubmitted(token, name, username string, emails []string) error { - route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) - l, err := url.Parse(p.cfg.WebServerAddress + route) - if err != nil { - return err - } - - tmplData := proposalSubmitted{ - Username: username, - Name: name, - Link: l.String(), - } - - subject := "New Proposal Submitted" - body, err := createBody(proposalSubmittedTmpl, tmplData) - if err != nil { - return err - } - - return p.smtp.sendEmailTo(subject, body, emails) -} - -// emailProposalEdited sends a proposal edited notification email to the -// provided list of emails. -func (p *politeiawww) emailProposalEdited(name, username, token, version string, emails []string) error { - route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) - l, err := url.Parse(p.cfg.WebServerAddress + route) - if err != nil { - return err - } - - tmplData := proposalEdited{ - Name: name, - Version: version, - Username: username, - Link: l.String(), - } - - subject := "Proposal Edited" - body, err := createBody(proposalEditedTmpl, tmplData) - if err != nil { - return err - } - - return p.smtp.sendEmailTo(subject, body, emails) -} - -// emailProposalStatusChange sends a proposal status change email to the -// provided email addresses. -func (p *politeiawww) emailProposalStatusChange(d dataProposalStatusChange, proposalName string, emails []string) error { - route := strings.Replace(guiRouteProposalDetails, "{token}", d.token, 1) - l, err := url.Parse(p.cfg.WebServerAddress + route) - if err != nil { - return err - } - - var ( - subject string - body string - ) - switch d.status { - case rcv1.RecordStatusPublic: - subject = "New Proposal Published" - tmplData := proposalVetted{ - Name: proposalName, - Link: l.String(), - } - body, err = createBody(tmplProposalVetted, tmplData) - if err != nil { - return err - } - - default: - log.Debugf("no user notification for prop status %v", d.status) - return nil - } - - return p.smtp.sendEmailTo(subject, body, emails) -} - -// emailProposalStatusChangeAuthor sends a proposal status change notification -// email to the provided email address. -func (p *politeiawww) emailProposalStatusChangeToAuthor(d dataProposalStatusChange, proposalName, authorEmail string) error { - route := strings.Replace(guiRouteProposalDetails, "{token}", d.token, 1) - l, err := url.Parse(p.cfg.WebServerAddress + route) - if err != nil { - return err - } - - var ( - subject string - body string - ) - switch d.status { - case rcv1.RecordStatusPublic: - subject = "Your Proposal Has Been Published" - tmplData := proposalVettedToAuthor{ - Name: proposalName, - Link: l.String(), - } - body, err = createBody(proposalVettedToAuthorTmpl, tmplData) - if err != nil { - return err - } - - case rcv1.RecordStatusCensored: - subject = "Your Proposal Has Been Censored" - tmplData := proposalCensoredToAuthor{ - Name: proposalName, - Reason: d.reason, - } - body, err = createBody(tmplProposalCensoredForAuthor, tmplData) - if err != nil { - return err - } - - default: - return fmt.Errorf("no author notification for prop status %v", d.status) - } - - return p.smtp.sendEmailTo(subject, body, []string{authorEmail}) -} - -// emailProposalCommentSubmitted sends a proposal comment submitted email to -// the provided email address. -func (p *politeiawww) emailProposalCommentSubmitted(token, commentID, commentUsername, proposalName, proposalAuthorEmail string) error { - // Setup comment URL - route := strings.Replace(guirouteProposalComments, "{token}", token, 1) - route = strings.Replace(route, "{id}", commentID, 1) - l, err := url.Parse(p.cfg.WebServerAddress + route) - if err != nil { - return err - } - - // Setup email - subject := "New Comment On Your Proposal" - tmplData := proposalCommentSubmitted{ - Username: commentUsername, - Name: proposalName, - Link: l.String(), - } - body, err := createBody(proposalCommentSubmittedTmpl, tmplData) - if err != nil { - return err - } - - // Send email - return p.smtp.sendEmailTo(subject, body, []string{proposalAuthorEmail}) -} - -// emailProposalCommentReply sends a proposal comment reply email to the -// provided email address. -func (p *politeiawww) emailProposalCommentReply(token, commentID, commentUsername, proposalName, parentCommentEmail string) error { - // Setup comment URL - route := strings.Replace(guirouteProposalComments, "{token}", token, 1) - route = strings.Replace(route, "{id}", commentID, 1) - l, err := url.Parse(p.cfg.WebServerAddress + route) - if err != nil { - return err - } - - // Setup email - subject := "New Reply To Your Comment" - tmplData := proposalCommentReply{ - Username: commentUsername, - Name: proposalName, - Link: l.String(), - } - body, err := createBody(proposalCommentReplyTmpl, tmplData) - if err != nil { - return err - } - - // Send email - return p.smtp.sendEmailTo(subject, body, []string{parentCommentEmail}) -} - -// emailProposalVoteAuthorized sends a proposal vote authorized email to the -// provided list of emails. -func (p *politeiawww) emailProposalVoteAuthorized(token, name, username string, emails []string) error { - // Setup URL - route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) - l, err := url.Parse(p.cfg.WebServerAddress + route) - if err != nil { - return err - } - - // Setup email - subject := "Proposal Authorized To Start Voting" - tplData := proposalVoteAuthorized{ - Username: username, - Name: name, - Link: l.String(), - } - body, err := createBody(proposalVoteAuthorizedTmpl, tplData) - if err != nil { - return err - } - - // Send email - return p.smtp.sendEmailTo(subject, body, emails) -} - -// emailProposalVoteStarted sends a proposal vote started email notification -// to the provided email addresses. -func (p *politeiawww) emailProposalVoteStarted(token, name string, emails []string) error { - // Setup URL - route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) - l, err := url.Parse(p.cfg.WebServerAddress + route) - if err != nil { - return err - } - - // Setup email - subject := "Voting Started for Proposal" - tplData := proposalVoteStarted{ - Name: name, - Link: l.String(), - } - body, err := createBody(proposalVoteStartedTmpl, tplData) - if err != nil { - return err - } - - // Send email - return p.smtp.sendEmailTo(subject, body, emails) -} - -// emailProposalVoteStartedToAuthor sends a proposal vote started email to -// the provided email address. -func (p *politeiawww) emailProposalVoteStartedToAuthor(token, name, username, email string) error { - // Setup URL - route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) - l, err := url.Parse(p.cfg.WebServerAddress + route) - if err != nil { - return err - } - - // Setup email - subject := "Your Proposal Has Started Voting" - tplData := proposalVoteStartedToAuthor{ - Name: name, - Link: l.String(), - } - body, err := createBody(proposalVoteStartedToAuthorTmpl, tplData) - if err != nil { - return err - } - - // Send email - return p.smtp.sendEmailTo(subject, body, []string{email}) -} - // emailUserEmailVerify sends a new user verification email to the // provided email address. func (p *politeiawww) emailUserEmailVerify(email, token, username string) error { @@ -332,7 +75,7 @@ func (p *politeiawww) emailUserEmailVerify(email, token, username string) error } recipients := []string{email} - return p.smtp.sendEmailTo(subject, body, recipients) + return p.mail.SendTo(subject, body, recipients) } // emailUserKeyUpdate emails the link with the verification token used for @@ -356,7 +99,7 @@ func (p *politeiawww) emailUserKeyUpdate(username, email, publicKey, token strin } recipients := []string{email} - return p.smtp.sendEmailTo(subject, body, recipients) + return p.mail.SendTo(subject, body, recipients) } // emailUserPasswordReset emails the link with the reset password verification @@ -383,7 +126,7 @@ func (p *politeiawww) emailUserPasswordReset(email, username, token string) erro } // Send email - return p.smtp.sendEmailTo(subject, body, []string{email}) + return p.mail.SendTo(subject, body, []string{email}) } // emailUserAccountLocked notifies the user its account has been locked and @@ -408,7 +151,7 @@ func (p *politeiawww) emailUserAccountLocked(username, email string) error { } recipients := []string{email} - return p.smtp.sendEmailTo(subject, body, recipients) + return p.mail.SendTo(subject, body, recipients) } // emailUserPasswordChanged notifies the user that his password was changed, @@ -425,7 +168,7 @@ func (p *politeiawww) emailUserPasswordChanged(username, email string) error { } recipients := []string{email} - return p.smtp.sendEmailTo(subject, body, recipients) + return p.mail.SendTo(subject, body, recipients) } // emailUserCMSInvite emails the invitation link for the Contractor Management @@ -448,7 +191,7 @@ func (p *politeiawww) emailUserCMSInvite(email, token string) error { } recipients := []string{email} - return p.smtp.sendEmailTo(subject, body, recipients) + return p.mail.SendTo(subject, body, recipients) } // emailUserDCCApproved emails the link to invite a user that has been approved @@ -465,7 +208,7 @@ func (p *politeiawww) emailUserDCCApproved(email string) error { } recipients := []string{email} - return p.smtp.sendEmailTo(subject, body, recipients) + return p.mail.SendTo(subject, body, recipients) } // emailDCCSubmitted sends email regarding the DCC New event. Sends email @@ -487,7 +230,7 @@ func (p *politeiawww) emailDCCSubmitted(token string, emails []string) error { return err } - return p.smtp.sendEmailTo(subject, body, emails) + return p.mail.SendTo(subject, body, emails) } // emailDCCSupportOppose sends emails regarding dcc support/oppose event. @@ -509,7 +252,7 @@ func (p *politeiawww) emailDCCSupportOppose(token string, emails []string) error return err } - return p.smtp.sendEmailTo(subject, body, emails) + return p.mail.SendTo(subject, body, emails) } // emailInvoiceStatusUpdate sends email for the invoice status update event. @@ -526,7 +269,7 @@ func (p *politeiawww) emailInvoiceStatusUpdate(invoiceToken, userEmail string) e } recipients := []string{userEmail} - return p.smtp.sendEmailTo(subject, body, recipients) + return p.mail.SendTo(subject, body, recipients) } // emailInvoiceNotSent sends a invoice not sent email notification to the @@ -547,7 +290,7 @@ func (p *politeiawww) emailInvoiceNotSent(email, username string) error { } recipients := []string{email} - return p.smtp.sendEmailTo(subject, body, recipients) + return p.mail.SendTo(subject, body, recipients) } // emailInvoiceNewComment sends email for the invoice new comment event. Send @@ -562,5 +305,5 @@ func (p *politeiawww) emailInvoiceNewComment(userEmail string) error { } recipients := []string{userEmail} - return p.smtp.sendEmailTo(subject, body, recipients) + return p.mail.SendTo(subject, body, recipients) } diff --git a/politeiawww/events.go b/politeiawww/events.go index cace30620..1c7830d8a 100644 --- a/politeiawww/events.go +++ b/politeiawww/events.go @@ -5,25 +5,11 @@ package main import ( - "fmt" - "strconv" - - "github.com/decred/politeia/politeiad/plugins/comments" - rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/user" - "github.com/google/uuid" ) const ( - // Pi events - eventProposalSubmitted = "eventProposalSubmitted" - eventProposalEdited = "eventProposalEdited" - eventProposalStatusChange = "eventProposalStatusChange" - eventProposalComment = "eventProposalComment" - eventProposalVoteAuthorized = "eventProposalVoteAuthorized" - eventProposalVoteStarted = "eventProposalVoteStarted" - // CMS events eventInvoiceComment = "eventInvoiceComment" eventInvoiceStatusUpdate = "eventInvoiceStatusUpdate" @@ -31,43 +17,6 @@ const ( eventDCCSupportOppose = "eventDCCSupportOppose" ) -func (p *politeiawww) setupEventListenersPi() { - // Setup process for each event: - // 1. Create a channel for the event - // 2. Register the channel with the event manager - // 3. Launch an event handler to listen for new events - - // Setup proposal submitted event - ch := make(chan interface{}) - p.events.Register(eventProposalSubmitted, ch) - go p.handleEventProposalSubmitted(ch) - - // Setup proposal edit event - ch = make(chan interface{}) - p.events.Register(eventProposalEdited, ch) - go p.handleEventProposalEdited(ch) - - // Setup proposal status change event - ch = make(chan interface{}) - p.events.Register(eventProposalStatusChange, ch) - go p.handleEventProposalStatusChange(ch) - - // Setup proposal comment event - ch = make(chan interface{}) - p.events.Register(eventProposalComment, ch) - go p.handleEventProposalComment(ch) - - // Setup proposal vote authorized event - ch = make(chan interface{}) - p.events.Register(eventProposalVoteAuthorized, ch) - go p.handleEventProposalVoteAuthorized(ch) - - // Setup proposal vote started event - ch = make(chan interface{}) - p.events.Register(eventProposalVoteStarted, ch) - go p.handleEventProposalVoteStarted(ch) -} - func (p *politeiawww) setupEventListenersCMS() { // Setup invoice comment event ch := make(chan interface{}) @@ -110,426 +59,6 @@ func userNotificationEnabled(u user.User, n www.EmailNotificationT) bool { return true } -type dataProposalSubmitted struct { - token string // Proposal token - name string // Proposal name - username string // Author username -} - -func (p *politeiawww) handleEventProposalSubmitted(ch chan interface{}) { - for msg := range ch { - d, ok := msg.(dataProposalSubmitted) - if !ok { - log.Errorf("handleEventProposalSubmitted invalid msg: %v", msg) - continue - } - - // Compile a list of users to send the notification to - emails := make([]string, 0, 256) - err := p.db.AllUsers(func(u *user.User) { - switch { - case !u.Admin: - // Only admins get this notification - return - case !userNotificationEnabled(*u, - www.NotificationEmailAdminProposalNew): - // Admin doesn't have notification bit set - return - } - - // Add user to notification list - emails = append(emails, u.Email) - }) - if err != nil { - log.Errorf("handleEventProposalSubmitted: AllUsers: %v", err) - return - } - - // Send email notification - err = p.emailProposalSubmitted(d.token, d.name, d.username, emails) - if err != nil { - log.Errorf("emailProposalSubmitted: %v", err) - } - - log.Debugf("Sent proposal submitted notification %v", d.token) - } -} - -type dataProposalEdited struct { - userID string // Author id - username string // Author username - token string // Proposal censorship token - name string // Proposal name - version string // Proposal version -} - -func (p *politeiawww) handleEventProposalEdited(ch chan interface{}) { - for msg := range ch { - d, ok := msg.(dataProposalEdited) - if !ok { - log.Errorf("handleEventProposalEdited invalid msg: %v", msg) - continue - } - - // Compile a list of users to send the notification to - emails := make([]string, 0, 256) - err := p.db.AllUsers(func(u *user.User) { - // Check circumstances where we don't notify - switch { - case u.ID.String() == d.userID: - // User is the author - return - case !userNotificationEnabled(*u, - www.NotificationEmailRegularProposalEdited): - // User doesn't have notification bit set - return - } - - // Add user to notification list - emails = append(emails, u.Email) - }) - if err != nil { - log.Errorf("handleEventProposalEdited: AllUsers: %v", err) - continue - } - - err = p.emailProposalEdited(d.name, d.username, - d.token, d.version, emails) - if err != nil { - log.Errorf("emailProposalEdited: %v", err) - continue - } - - log.Debugf("Sent proposal edited notifications %v", d.token) - } -} - -type dataProposalStatusChange struct { - token string // Proposal censorship token - state string // Updated proposal state - status rcv1.RecordStatusT // Updated proposal status - version string // Proposal version - reason string // Status change reason - adminID string // Admin uuid -} - -func (p *politeiawww) handleEventProposalStatusChange(ch chan interface{}) { - for msg := range ch { - d, ok := msg.(dataProposalStatusChange) - if !ok { - log.Errorf("handleProposalStatusChange invalid msg: %v", msg) - continue - } - - // Check if proposal is in correct status for notification - switch d.status { - case rcv1.RecordStatusPublic, rcv1.RecordStatusCensored: - // The status requires a notification be sent - default: - // The status does not require a notification be sent. Listen - // for next event. - continue - } - - // TODO - /* - // Get the proposal author - pr, err := p.proposalRecordLatest(context.Background(), d.state, d.token) - if err != nil { - log.Errorf("handleEventProposalStatusChange: proposalRecordLatest "+ - "%v %v: %v", d.state, d.token, err) - continue - } - author, err := p.db.UserGetByPubKey(pr.PublicKey) - if err != nil { - log.Errorf("handleEventProposalStatusChange: UserGetByPubKey %v: %v", - pr.PublicKey, err) - continue - } - - // Email author - proposalName := proposalName(*pr) - notification := www.NotificationEmailRegularProposalVetted - if userNotificationEnabled(*author, notification) { - err = p.emailProposalStatusChangeToAuthor(d, proposalName, author.Email) - if err != nil { - log.Errorf("emailProposalStatusChangeToAuthor: %v", err) - continue - } - } - - // Compile list of users to send the notification to - emails := make([]string, 0, 256) - err = p.db.AllUsers(func(u *user.User) { - switch { - case u.ID.String() == d.adminID: - // User is the admin that made the status change - return - case u.ID.String() == author.ID.String(): - // User is the author. The author is sent a different - // notification. Don't include them in the users list. - return - case !userNotificationEnabled(*u, notification): - // User does not have notification bit set - return - } - - // Add user to notification list - emails = append(emails, u.Email) - }) - if err != nil { - log.Errorf("handleEventProposalStatusChange: AllUsers: %v", err) - continue - } - - // Email users - err = p.emailProposalStatusChange(d, proposalName, emails) - if err != nil { - log.Errorf("emailProposalStatusChange: %v", err) - continue - } - */ - - log.Debugf("Sent proposal status change notifications %v", d.token) - } -} - -func (p *politeiawww) notifyProposalAuthorOnComment(d dataProposalComment, userID, proposalName string) error { - // Lookup proposal author to see if they should be sent a - // notification. - uuid, err := uuid.Parse(userID) - if err != nil { - return err - } - author, err := p.db.UserGetById(uuid) - if err != nil { - return fmt.Errorf("UserGetByID %v: %v", uuid.String(), err) - } - - // Check if notification should be sent to author - switch { - case d.username == author.Username: - // Author commented on their own proposal - return nil - case !userNotificationEnabled(*author, - www.NotificationEmailCommentOnMyProposal): - // Author does not have notification bit set on - return nil - } - - // Send notification eamil - commentID := strconv.FormatUint(uint64(d.commentID), 10) - return p.emailProposalCommentSubmitted(d.token, commentID, d.username, - proposalName, author.Email) -} - -func (p *politeiawww) notifyParentAuthorOnComment(d dataProposalComment, proposalName string) error { - // Verify this is a reply comment - if d.parentID == 0 { - return nil - } - - // Lookup the parent comment author to check if they should receive - // a reply notification. - g := comments.Get{ - CommentIDs: []uint32{d.parentID}, - } - // TODO - _ = g - var parentComment comments.GetReply - /* - parentComment, err := p.commentsGet(context.Background(), g) - if err != nil { - return err - } - */ - userID, err := uuid.Parse(parentComment.Comments[0].UserID) - if err != nil { - return err - } - author, err := p.db.UserGetById(userID) - if err != nil { - return err - } - - // Check if notification should be sent - switch { - case d.username == author.Username: - // Author replied to their own comment - return nil - case !userNotificationEnabled(*author, - www.NotificationEmailCommentOnMyComment): - // Author does not have notification bit set on - return nil - } - - // Send notification email to parent comment author - commentID := strconv.FormatUint(uint64(d.commentID), 10) - return p.emailProposalCommentReply(d.token, commentID, d.username, - proposalName, author.Email) -} - -type dataProposalComment struct { - state string - token string - commentID uint32 - parentID uint32 - username string // Comment author username -} - -func (p *politeiawww) handleEventProposalComment(ch chan interface{}) { - for msg := range ch { - d, ok := msg.(dataProposalComment) - if !ok { - log.Errorf("handleEventProposalComment invalid msg: %v", msg) - continue - } - - _ = d - /* TODO - // Fetch the proposal record here to avoid calling this two times - // on the notify functions below - pr, err := p.proposalRecordLatest(context.Background(), d.state, - d.token) - if err != nil { - err = fmt.Errorf("proposalRecordLatest %v %v: %v", - d.state, d.token, err) - goto next - } - - // Notify the proposal author - err = p.notifyProposalAuthorOnComment(d, pr.UserID, proposalName(*pr)) - if err != nil { - err = fmt.Errorf("notifyProposalAuthorOnComment: %v", err) - goto next - } - - // Notify the parent comment author - err = p.notifyParentAuthorOnComment(d, proposalName(*pr)) - if err != nil { - err = fmt.Errorf("notifyParentAuthorOnComment: %v", err) - goto next - } - - // Notifications successfully sent - log.Debugf("Sent proposal commment notification %v", d.token) - continue - - next: - // If we made it here then there was an error. Log the error - // before listening for the next event. - log.Errorf("handleEventProposalComment: %v", err) - continue - */ - } -} - -type dataProposalVoteAuthorized struct { - token string // Proposal censhorship token - name string // Proposal name - username string // Author username - email string // Author email -} - -func (p *politeiawww) handleEventProposalVoteAuthorized(ch chan interface{}) { - for msg := range ch { - d, ok := msg.(dataProposalVoteAuthorized) - if !ok { - log.Errorf("handleEventProposalVoteAuthorized invalid msg: %v", msg) - continue - } - - // Compile a list of emails to send the notification to. - emails := make([]string, 0, 256) - err := p.db.AllUsers(func(u *user.User) { - switch { - case !u.Admin: - // Only notify admin users - return - case !userNotificationEnabled(*u, - www.NotificationEmailAdminProposalVoteAuthorized): - // User does not have notification bit set - return - } - - // Add user to notification list - emails = append(emails, u.Email) - }) - if err != nil { - log.Errorf("handleEventProposalVoteAuthorized: AllUsers: %v", err) - continue - } - - // Send notification email - err = p.emailProposalVoteAuthorized(d.token, d.name, d.username, emails) - if err != nil { - log.Errorf("emailProposalVoteAuthorized: %v", err) - continue - } - - log.Debugf("Sent proposal vote authorized notifications %v", d.token) - } -} - -type dataProposalVoteStarted struct { - token string // Proposal censhorship token - name string // Proposal name - adminID string // Admin uuid - author user.User // Proposal author -} - -func (p *politeiawww) handleEventProposalVoteStarted(ch chan interface{}) { - for msg := range ch { - d, ok := msg.(dataProposalVoteStarted) - if !ok { - log.Errorf("handleEventProposalVoteStarted invalid msg: %v", msg) - continue - } - - // Email author - notification := www.NotificationEmailRegularProposalVoteStarted - if userNotificationEnabled(d.author, notification) { - err := p.emailProposalVoteStartedToAuthor(d.token, d.name, - d.author.Username, d.author.Email) - if err != nil { - log.Errorf("emailProposalVoteStartedToAuthor: %v", err) - continue - } - } - - // Compile a list of users to send the notification to. - emails := make([]string, 0, 256) - err := p.db.AllUsers(func(u *user.User) { - switch { - case u.ID.String() == d.adminID: - // Don't notify admin who started the vote - return - case u.ID.String() == d.author.ID.String(): - // Don't send this notification to the author - return - case !userNotificationEnabled(*u, notification): - // User does not have notification bit set - return - } - - // Add user to notification list - emails = append(emails, u.Email) - }) - if err != nil { - log.Errorf("handleEventProposalVoteStarted: AllUsers: %v", err) - } - - // Email users - err = p.emailProposalVoteStarted(d.token, d.name, emails) - if err != nil { - log.Errorf("emailProposalVoteStartedToUsers: %v", err) - continue - } - - log.Debugf("Sent proposal vote started notifications %v", d.token) - } -} - type dataInvoiceComment struct { token string // Comment token email string // User email diff --git a/politeiawww/events/events.go b/politeiawww/events/events.go index cf42a58e8..8845b766d 100644 --- a/politeiawww/events/events.go +++ b/politeiawww/events/events.go @@ -27,6 +27,8 @@ func (e *Manager) Register(event string, listener chan interface{}) { l = append(l, listener) e.listeners[event] = l + + log.Debugf("Register event %v", event) } // Emit emits an event by passing it to all channels that have been registered @@ -43,6 +45,8 @@ func (e *Manager) Emit(event string, data interface{}) { for _, ch := range listeners { ch <- data } + + log.Debugf("Emit event %v", event) } // NewManager returns a new Manager context. diff --git a/politeiawww/events/log.go b/politeiawww/events/log.go new file mode 100644 index 000000000..61ac35582 --- /dev/null +++ b/politeiawww/events/log.go @@ -0,0 +1,25 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package events + +import "github.com/decred/slog" + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log = slog.Disabled + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = slog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using slog. +func UseLogger(logger slog.Logger) { + log = logger +} diff --git a/politeiawww/log.go b/politeiawww/log.go index 4a7ac673b..190b5eb5a 100644 --- a/politeiawww/log.go +++ b/politeiawww/log.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2020 The Decred developers +// Copyright (c) 2017-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -13,6 +13,8 @@ import ( "github.com/decred/politeia/politeiawww/codetracker/github" ghdb "github.com/decred/politeia/politeiawww/codetracker/github/database/cockroachdb" "github.com/decred/politeia/politeiawww/comments" + "github.com/decred/politeia/politeiawww/events" + "github.com/decred/politeia/politeiawww/mail" "github.com/decred/politeia/politeiawww/pi" "github.com/decred/politeia/politeiawww/records" "github.com/decred/politeia/politeiawww/sessions" @@ -51,8 +53,13 @@ var ( // application shutdown. logRotator *rotator.Rotator - log = backendLog.Logger("PWWW") - userdbLog = backendLog.Logger("USER") + log = backendLog.Logger("PWWW") + userdbLog = backendLog.Logger("USER") + sessionsLog = backendLog.Logger("SESS") + eventsLog = backendLog.Logger("EVNT") + apiLog = backendLog.Logger("PAPI") + + // CMS loggers cmsdbLog = backendLog.Logger("CMDB") wsdcrdataLog = backendLog.Logger("WSDD") githubTrackerLog = backendLog.Logger("GHTR") @@ -61,23 +68,34 @@ var ( // Initialize package-global logger variables. func init() { + mail.UseLogger(log) + sessions.UseLogger(sessionsLog) + events.UseLogger(eventsLog) + + // UserDB loggers localdb.UseLogger(userdbLog) cockroachdb.UseLogger(userdbLog) + + // API loggers + records.UseLogger(apiLog) + comments.UseLogger(apiLog) + ticketvote.UseLogger(apiLog) + pi.UseLogger(apiLog) + + // CMS loggers cmsdb.UseLogger(cmsdbLog) wsdcrdata.UseLogger(wsdcrdataLog) github.UseLogger(githubTrackerLog) ghdb.UseLogger(githubdbLog) - sessions.UseLogger(log) - comments.UseLogger(log) - ticketvote.UseLogger(log) - records.UseLogger(log) - pi.UseLogger(log) } // subsystemLoggers maps each subsystem identifier to its associated logger. var subsystemLoggers = map[string]slog.Logger{ "PWWW": log, + "SESS": sessionsLog, + "EVNT": eventsLog, "USER": userdbLog, + "PAPI": apiLog, "CMDB": cmsdbLog, "WSDD": wsdcrdataLog, "GHTR": githubTrackerLog, diff --git a/politeiawww/mail/log.go b/politeiawww/mail/log.go new file mode 100644 index 000000000..729aa0e76 --- /dev/null +++ b/politeiawww/mail/log.go @@ -0,0 +1,25 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package mail + +import "github.com/decred/slog" + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log = slog.Disabled + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UseLogger or SetLogWriter are called. +func DisableLog() { + log = slog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using slog. +func UseLogger(logger slog.Logger) { + log = logger +} diff --git a/politeiawww/mail/mail.go b/politeiawww/mail/mail.go new file mode 100644 index 000000000..c99726278 --- /dev/null +++ b/politeiawww/mail/mail.go @@ -0,0 +1,108 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package mail + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" + "net/mail" + "net/url" + + "github.com/dajohi/goemail" +) + +// Client provides an SMTP client for sending emails from a preset email +// address. +type Client struct { + smtp *goemail.SMTP // SMTP server + mailName string // From name + mailAddress string // From email address + disabled bool // Has email been disabled +} + +// IsEnabled returns whether the mail server is enabled. +func (c *Client) IsEnabled() bool { + return !c.disabled +} + +// SendTo sends an email with the given subject and body to the provided list +// of email addresses. +func (c *Client) SendTo(subject, body string, recipients []string) error { + if c.disabled || len(recipients) == 0 { + return nil + } + + // Setup email + msg := goemail.NewMessage(c.mailAddress, subject, body) + msg.SetName(c.mailName) + + // Add all recipients to BCC + for _, v := range recipients { + msg.AddBCC(v) + } + + return c.smtp.Send(msg) +} + +// New returns a new mail Client. +func New(host, user, password, emailAddress, certPath string, skipVerify bool) (*Client, error) { + // Email is considered disabled if any of the required user + // credentials are mising. + if host == "" || user == "" || password == "" { + log.Infof("Email: DISABLED") + return &Client{ + disabled: true, + }, nil + } + + // Parse mail host + h := fmt.Sprintf("smtps://%v:%v@%v", user, password, host) + u, err := url.Parse(h) + if err != nil { + return nil, err + } + + log.Infof("Mail host: smtps://%v:[password]@%v", user, host) + + // Parse email address + a, err := mail.ParseAddress(emailAddress) + if err != nil { + return nil, err + } + + log.Infof("Mail address: %v", a.String()) + + // Setup tls config + tlsConfig := &tls.Config{ + InsecureSkipVerify: skipVerify, + } + if !skipVerify && certPath != "" { + cert, err := ioutil.ReadFile(certPath) + if err != nil { + return nil, err + } + certPool, err := x509.SystemCertPool() + if err != nil { + certPool = x509.NewCertPool() + } + certPool.AppendCertsFromPEM(cert) + tlsConfig.RootCAs = certPool + } + + // Setup smtp context + smtp, err := goemail.NewSMTP(u.String(), tlsConfig) + if err != nil { + return nil, err + } + + return &Client{ + smtp: smtp, + mailName: a.Name, + mailAddress: a.Address, + disabled: false, + }, nil +} diff --git a/politeiawww/middleware.go b/politeiawww/middleware.go index c4388a174..f8f91f72e 100644 --- a/politeiawww/middleware.go +++ b/politeiawww/middleware.go @@ -19,7 +19,7 @@ import ( // function. func (p *politeiawww) isLoggedIn(f http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - log.Debugf("isLoggedIn: %v %v %v %v", + log.Tracef("%v isLoggedIn: %v %v %v", util.RemoteAddr(r), r.Method, r.URL, r.Proto) id, err := p.sessions.GetSessionUserID(w, r) @@ -56,7 +56,7 @@ func (p *politeiawww) isAdmin(w http.ResponseWriter, r *http.Request) (bool, err // before calling the next function. func (p *politeiawww) isLoggedInAsAdmin(f http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - log.Debugf("isLoggedInAsAdmin: %v %v %v %v", + log.Tracef("%v isLoggedInAsAdmin: %v %v %v", util.RemoteAddr(r), r.Method, r.URL, r.Proto) // Check if user is admin diff --git a/politeiawww/pi/events.go b/politeiawww/pi/events.go new file mode 100644 index 000000000..5ccdfdaff --- /dev/null +++ b/politeiawww/pi/events.go @@ -0,0 +1,692 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package pi + +import ( + "github.com/decred/politeia/politeiawww/comments" + "github.com/decred/politeia/politeiawww/records" + "github.com/decred/politeia/politeiawww/ticketvote" +) + +func (p *Pi) setupEventListeners() { + // Setup process for each event: + // 1. Create a channel for the event. + // 2. Register the channel with the event manager. + // 3. Launch an event handler to listen for events emitted into the + // channel by the event manager. + + // Record new + ch := make(chan interface{}) + p.events.Register(records.EventTypeNew, ch) + go p.handleEventRecordNew(ch) + + // Record edit + ch = make(chan interface{}) + p.events.Register(records.EventTypeEdit, ch) + go p.handleEventRecordEdit(ch) + + // Record set status + ch = make(chan interface{}) + p.events.Register(records.EventTypeSetStatus, ch) + go p.handleEventRecordSetStatus(ch) + + // Comment new + ch = make(chan interface{}) + p.events.Register(comments.EventTypeNew, ch) + go p.handleEventCommentNew(ch) + + // Ticket vote authorized + ch = make(chan interface{}) + p.events.Register(ticketvote.EventTypeAuthorize, ch) + go p.handleEventVoteAuthorized(ch) + + // Ticket vote started + ch = make(chan interface{}) + p.events.Register(ticketvote.EventTypeStart, ch) + go p.handleEventVoteStart(ch) +} + +func (p *Pi) handleEventRecordNew(ch chan interface{}) { + for msg := range ch { + e, ok := msg.(records.EventNew) + if !ok { + log.Errorf("handleEventRecordNew invalid msg: %v", msg) + continue + } + + /* + // Compile a list of users to send the notification to + emails := make([]string, 0, 256) + err := p.db.AllUsers(func(u *user.User) { + switch { + case !u.Admin: + // Only admins get this notification + return + case !userNotificationEnabled(*u, + www.NotificationEmailAdminProposalNew): + // Admin doesn't have notification bit set + return + } + + // Add user to notification list + emails = append(emails, u.Email) + }) + if err != nil { + log.Errorf("handleEventRecordNew: AllUsers: %v", err) + return + } + + // Send email notification + err = p.emailRecordNew(d.token, d.name, d.username, emails) + if err != nil { + log.Errorf("emailRecordNew: %v", err) + } + + */ + log.Debugf("Record new event sent %v", e.Record.CensorshipRecord.Token) + } +} + +func (p *Pi) handleEventRecordEdit(ch chan interface{}) { + for msg := range ch { + e, ok := msg.(records.EventEdit) + if !ok { + log.Errorf("handleEventRecordEdit invalid msg: %v", msg) + continue + } + + /* + // Compile a list of users to send the notification to + emails := make([]string, 0, 256) + err := p.db.AllUsers(func(u *user.User) { + // Check circumstances where we don't notify + switch { + case u.ID.String() == d.userID: + // User is the author + return + case !userNotificationEnabled(*u, + www.NotificationEmailRegularRecordEdit): + // User doesn't have notification bit set + return + } + + // Add user to notification list + emails = append(emails, u.Email) + }) + if err != nil { + log.Errorf("handleEventRecordEdit: AllUsers: %v", err) + continue + } + + err = p.emailRecordEdit(d.name, d.username, + d.token, d.version, emails) + if err != nil { + log.Errorf("emailRecordEdit: %v", err) + continue + } + + */ + log.Debugf("Record edit event sent %v", e.Record.CensorshipRecord.Token) + } +} + +func (p *Pi) handleEventRecordSetStatus(ch chan interface{}) { + for msg := range ch { + e, ok := msg.(records.EventSetStatus) + if !ok { + log.Errorf("handleRecordSetStatus invalid msg: %v", msg) + continue + } + + /* + // Check if proposal is in correct status for notification + switch d.status { + case rcv1.RecordStatusPublic, rcv1.RecordStatusCensored: + // The status requires a notification be sent + default: + // The status does not require a notification be sent. Listen + // for next event. + continue + } + + // Get the proposal author + pr, err := p.proposalRecordLatest(context.Background(), d.state, d.token) + if err != nil { + log.Errorf("handleEventRecordSetStatus: proposalRecordLatest "+ + "%v %v: %v", d.state, d.token, err) + continue + } + author, err := p.db.UserGetByPubKey(pr.PublicKey) + if err != nil { + log.Errorf("handleEventRecordSetStatus: UserGetByPubKey %v: %v", + pr.PublicKey, err) + continue + } + + // Email author + proposalName := proposalName(*pr) + notification := www.NotificationEmailRegularProposalVetted + if userNotificationEnabled(*author, notification) { + err = p.emailRecordSetStatusToAuthor(d, proposalName, author.Email) + if err != nil { + log.Errorf("emailRecordSetStatusToAuthor: %v", err) + continue + } + } + + // Compile list of users to send the notification to + emails := make([]string, 0, 256) + err = p.db.AllUsers(func(u *user.User) { + switch { + case u.ID.String() == d.adminID: + // User is the admin that made the status change + return + case u.ID.String() == author.ID.String(): + // User is the author. The author is sent a different + // notification. Don't include them in the users list. + return + case !userNotificationEnabled(*u, notification): + // User does not have notification bit set + return + } + + // Add user to notification list + emails = append(emails, u.Email) + }) + if err != nil { + log.Errorf("handleEventRecordSetStatus: AllUsers: %v", err) + continue + } + + // Email users + err = p.emailRecordSetStatus(d, proposalName, emails) + if err != nil { + log.Errorf("emailRecordSetStatus: %v", err) + continue + } + */ + + log.Debugf("Record set status event sent %v", + e.Record.CensorshipRecord.Token) + } +} + +/* +func (p *Pi) notifyProposalAuthorOnComment(d dataCommentNew, userID, proposalName string) error { + // Lookup proposal author to see if they should be sent a + // notification. + uuid, err := uuid.Parse(userID) + if err != nil { + return err + } + author, err := p.db.UserGetById(uuid) + if err != nil { + return fmt.Errorf("UserGetByID %v: %v", uuid.String(), err) + } + + // Check if notification should be sent to author + switch { + case d.username == author.Username: + // Author commented on their own proposal + return nil + case !userNotificationEnabled(*author, + www.NotificationEmailCommentOnMyProposal): + // Author does not have notification bit set on + return nil + } + + // Send notification eamil + commentID := strconv.FormatUint(uint64(d.commentID), 10) + return p.emailCommentNewSubmitted(d.token, commentID, d.username, + proposalName, author.Email) +} + +func (p *Pi) notifyParentAuthorOnComment(d dataCommentNew, proposalName string) error { + // Verify this is a reply comment + if d.parentID == 0 { + return nil + } + + // Lookup the parent comment author to check if they should receive + // a reply notification. + g := comments.Get{ + CommentIDs: []uint32{d.parentID}, + } + _ = g + var parentComment comments.GetReply + parentComment, err := p.commentsGet(context.Background(), g) + if err != nil { + return err + } + userID, err := uuid.Parse(parentComment.Comments[0].UserID) + if err != nil { + return err + } + author, err := p.db.UserGetById(userID) + if err != nil { + return err + } + + // Check if notification should be sent + switch { + case d.username == author.Username: + // Author replied to their own comment + return nil + case !userNotificationEnabled(*author, + www.NotificationEmailCommentOnMyComment): + // Author does not have notification bit set on + return nil + } + + // Send notification email to parent comment author + commentID := strconv.FormatUint(uint64(d.commentID), 10) + return p.emailCommentNewReply(d.token, commentID, d.username, + proposalName, author.Email) +} +*/ + +func (p *Pi) handleEventCommentNew(ch chan interface{}) { + for msg := range ch { + d, ok := msg.(comments.EventNew) + if !ok { + log.Errorf("handleEventCommentNew invalid msg: %v", msg) + continue + } + + _ = d + /* + // Fetch the proposal record here to avoid calling this two times + // on the notify functions below + pr, err := p.proposalRecordLatest(context.Background(), d.state, + d.token) + if err != nil { + err = fmt.Errorf("proposalRecordLatest %v %v: %v", + d.state, d.token, err) + goto next + } + + // Notify the proposal author + err = p.notifyProposalAuthorOnComment(d, pr.UserID, proposalName(*pr)) + if err != nil { + err = fmt.Errorf("notifyProposalAuthorOnComment: %v", err) + goto next + } + + // Notify the parent comment author + err = p.notifyParentAuthorOnComment(d, proposalName(*pr)) + if err != nil { + err = fmt.Errorf("notifyParentAuthorOnComment: %v", err) + goto next + } + + // Notifications successfully sent + log.Debugf("Sent proposal commment notification %v", d.token) + continue + + next: + // If we made it here then there was an error. Log the error + // before listening for the next event. + log.Errorf("handleEventCommentNew: %v", err) + continue + */ + + log.Debugf("Comment new event sent %v", d.Comment.Token) + } +} + +func (p *Pi) handleEventVoteAuthorized(ch chan interface{}) { + for msg := range ch { + e, ok := msg.(ticketvote.EventAuthorize) + if !ok { + log.Errorf("handleEventVoteAuthorized invalid msg: %v", msg) + continue + } + + /* + // Compile a list of emails to send the notification to. + emails := make([]string, 0, 256) + err := p.db.AllUsers(func(u *user.User) { + switch { + case !u.Admin: + // Only notify admin users + return + case !userNotificationEnabled(*u, + www.NotificationEmailAdminVoteAuthorized): + // User does not have notification bit set + return + } + + // Add user to notification list + emails = append(emails, u.Email) + }) + if err != nil { + log.Errorf("handleEventVoteAuthorized: AllUsers: %v", err) + continue + } + + // Send notification email + err = p.emailVoteAuthorized(d.token, d.name, d.username, emails) + if err != nil { + log.Errorf("emailVoteAuthorized: %v", err) + continue + } + */ + + log.Debugf("Vote authorized event sent %v", e.Auth.Token) + } +} + +func (p *Pi) handleEventVoteStart(ch chan interface{}) { + for msg := range ch { + e, ok := msg.(ticketvote.EventStart) + if !ok { + log.Errorf("handleEventVoteStart invalid msg: %v", msg) + continue + } + + /* + // Email author + notification := www.NotificationEmailRegularVoteStart + if userNotificationEnabled(d.author, notification) { + err := p.emailVoteStartToAuthor(d.token, d.name, + d.author.Username, d.author.Email) + if err != nil { + log.Errorf("emailVoteStartToAuthor: %v", err) + continue + } + } + + // Compile a list of users to send the notification to. + emails := make([]string, 0, 256) + err := p.db.AllUsers(func(u *user.User) { + switch { + case u.ID.String() == d.adminID: + // Don't notify admin who started the vote + return + case u.ID.String() == d.author.ID.String(): + // Don't send this notification to the author + return + case !userNotificationEnabled(*u, notification): + // User does not have notification bit set + return + } + + // Add user to notification list + emails = append(emails, u.Email) + }) + if err != nil { + log.Errorf("handleEventVoteStart: AllUsers: %v", err) + } + + // Email users + err = p.emailVoteStart(d.token, d.name, emails) + if err != nil { + log.Errorf("emailVoteStartToUsers: %v", err) + continue + } + */ + + _ = e + token := "fix me" + log.Debugf("Vote start event sent %v", token) + } +} + +/* +// emailProposalSubmitted send a proposal submitted notification email to +// the provided list of emails. +func (p *politeiawww) emailProposalSubmitted(token, name, username string, emails []string) error { + route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) + l, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err + } + + tmplData := proposalSubmitted{ + Username: username, + Name: name, + Link: l.String(), + } + + subject := "New Proposal Submitted" + body, err := createBody(proposalSubmittedTmpl, tmplData) + if err != nil { + return err + } + + return p.smtp.sendEmailTo(subject, body, emails) +} + +// emailProposalEdited sends a proposal edited notification email to the +// provided list of emails. +func (p *politeiawww) emailProposalEdited(name, username, token, version string, emails []string) error { + route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) + l, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err + } + + tmplData := proposalEdited{ + Name: name, + Version: version, + Username: username, + Link: l.String(), + } + + subject := "Proposal Edited" + body, err := createBody(proposalEditedTmpl, tmplData) + if err != nil { + return err + } + + return p.smtp.sendEmailTo(subject, body, emails) +} + +// emailProposalStatusChange sends a proposal status change email to the +// provided email addresses. +func (p *politeiawww) emailProposalStatusChange(d dataProposalStatusChange, proposalName string, emails []string) error { + route := strings.Replace(guiRouteProposalDetails, "{token}", d.token, 1) + l, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err + } + + var ( + subject string + body string + ) + switch d.status { + case rcv1.RecordStatusPublic: + subject = "New Proposal Published" + tmplData := proposalVetted{ + Name: proposalName, + Link: l.String(), + } + body, err = createBody(tmplProposalVetted, tmplData) + if err != nil { + return err + } + + default: + log.Debugf("no user notification for prop status %v", d.status) + return nil + } + + return p.smtp.sendEmailTo(subject, body, emails) +} + +// emailProposalStatusChangeAuthor sends a proposal status change notification +// email to the provided email address. +func (p *politeiawww) emailProposalStatusChangeToAuthor(d dataProposalStatusChange, proposalName, authorEmail string) error { + route := strings.Replace(guiRouteProposalDetails, "{token}", d.token, 1) + l, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err + } + + var ( + subject string + body string + ) + switch d.status { + case rcv1.RecordStatusPublic: + subject = "Your Proposal Has Been Published" + tmplData := proposalVettedToAuthor{ + Name: proposalName, + Link: l.String(), + } + body, err = createBody(proposalVettedToAuthorTmpl, tmplData) + if err != nil { + return err + } + + case rcv1.RecordStatusCensored: + subject = "Your Proposal Has Been Censored" + tmplData := proposalCensoredToAuthor{ + Name: proposalName, + Reason: d.reason, + } + body, err = createBody(tmplProposalCensoredForAuthor, tmplData) + if err != nil { + return err + } + + default: + return fmt.Errorf("no author notification for prop status %v", d.status) + } + + return p.smtp.sendEmailTo(subject, body, []string{authorEmail}) +} + +// emailProposalCommentSubmitted sends a proposal comment submitted email to +// the provided email address. +func (p *politeiawww) emailProposalCommentSubmitted(token, commentID, commentUsername, proposalName, proposalAuthorEmail string) error { + // Setup comment URL + route := strings.Replace(guirouteProposalComments, "{token}", token, 1) + route = strings.Replace(route, "{id}", commentID, 1) + l, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err + } + + // Setup email + subject := "New Comment On Your Proposal" + tmplData := proposalCommentSubmitted{ + Username: commentUsername, + Name: proposalName, + Link: l.String(), + } + body, err := createBody(proposalCommentSubmittedTmpl, tmplData) + if err != nil { + return err + } + + // Send email + return p.smtp.sendEmailTo(subject, body, []string{proposalAuthorEmail}) +} + +// emailProposalCommentReply sends a proposal comment reply email to the +// provided email address. +func (p *politeiawww) emailProposalCommentReply(token, commentID, commentUsername, proposalName, parentCommentEmail string) error { + // Setup comment URL + route := strings.Replace(guirouteProposalComments, "{token}", token, 1) + route = strings.Replace(route, "{id}", commentID, 1) + l, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err + } + + // Setup email + subject := "New Reply To Your Comment" + tmplData := proposalCommentReply{ + Username: commentUsername, + Name: proposalName, + Link: l.String(), + } + body, err := createBody(proposalCommentReplyTmpl, tmplData) + if err != nil { + return err + } + + // Send email + return p.smtp.sendEmailTo(subject, body, []string{parentCommentEmail}) +} + +// emailProposalVoteAuthorized sends a proposal vote authorized email to the +// provided list of emails. +func (p *politeiawww) emailProposalVoteAuthorized(token, name, username string, emails []string) error { + // Setup URL + route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) + l, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err + } + + // Setup email + subject := "Proposal Authorized To Start Voting" + tplData := proposalVoteAuthorized{ + Username: username, + Name: name, + Link: l.String(), + } + body, err := createBody(proposalVoteAuthorizedTmpl, tplData) + if err != nil { + return err + } + + // Send email + return p.smtp.sendEmailTo(subject, body, emails) +} + +// emailProposalVoteStarted sends a proposal vote started email notification +// to the provided email addresses. +func (p *politeiawww) emailProposalVoteStarted(token, name string, emails []string) error { + // Setup URL + route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) + l, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err + } + + // Setup email + subject := "Voting Started for Proposal" + tplData := proposalVoteStarted{ + Name: name, + Link: l.String(), + } + body, err := createBody(proposalVoteStartedTmpl, tplData) + if err != nil { + return err + } + + // Send email + return p.smtp.sendEmailTo(subject, body, emails) +} + +// emailProposalVoteStartedToAuthor sends a proposal vote started email to +// the provided email address. +func (p *politeiawww) emailProposalVoteStartedToAuthor(token, name, username, email string) error { + // Setup URL + route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) + l, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err + } + + // Setup email + subject := "Your Proposal Has Started Voting" + tplData := proposalVoteStartedToAuthor{ + Name: name, + Link: l.String(), + } + body, err := createBody(proposalVoteStartedToAuthorTmpl, tplData) + if err != nil { + return err + } + + // Send email + return p.smtp.sendEmailTo(subject, body, []string{email}) +} +*/ diff --git a/politeiawww/pi/pi.go b/politeiawww/pi/pi.go index 5525e1e65..3999b24a5 100644 --- a/politeiawww/pi/pi.go +++ b/politeiawww/pi/pi.go @@ -15,6 +15,7 @@ import ( "github.com/decred/politeia/politeiad/plugins/pi" v1 "github.com/decred/politeia/politeiawww/api/pi/v1" "github.com/decred/politeia/politeiawww/config" + "github.com/decred/politeia/politeiawww/events" "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/politeiawww/user" "github.com/decred/politeia/util" @@ -26,6 +27,7 @@ type Pi struct { politeiad *pdclient.Client userdb user.Database sessions *sessions.Sessions + events *events.Manager policy *v1.PolicyReply } @@ -70,7 +72,7 @@ func (p *Pi) HandleProposals(w http.ResponseWriter, r *http.Request) { } // New returns a new Pi context. -func New(cfg *config.Config, pdc *pdclient.Client, udb user.Database, s *sessions.Sessions, plugins []pdv1.Plugin) (*Pi, error) { +func New(cfg *config.Config, pdc *pdclient.Client, udb user.Database, s *sessions.Sessions, e *events.Manager, plugins []pdv1.Plugin) (*Pi, error) { // Parse plugin settings var ( textFileSizeMax uint32 @@ -150,11 +152,13 @@ func New(cfg *config.Config, pdc *pdclient.Client, udb user.Database, s *session pi.SettingKeyProposalNameLengthMax) } - return &Pi{ + // Setup pi context + p := Pi{ cfg: cfg, politeiad: pdc, userdb: udb, sessions: s, + events: e, policy: &v1.PolicyReply{ TextFileSizeMax: textFileSizeMax, ImageFileCountMax: imageFileCountMax, @@ -163,5 +167,10 @@ func New(cfg *config.Config, pdc *pdclient.Client, udb user.Database, s *session NameLengthMax: nameLengthMax, NameSupportedChars: nameSupportedChars, }, - }, nil + } + + // Setup event listners + p.setupEventListeners() + + return &p, nil } diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index c9495b9e0..895ab4b0b 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -37,7 +37,8 @@ func (p *politeiawww) setupPiRoutes(r *records.Records, c *comments.Comments, t HandleFunc(www.PoliteiaWWWAPIRoute+www.RouteVersion, p.handleVersion). Methods(http.MethodGet) - // Legacy www routes. These routes have been DEPRECATED. + // Legacy www routes. These routes have been DEPRECATED. Support + // will be removed in a future release. p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, www.RoutePolicy, p.handlePolicy, permissionPublic) @@ -183,25 +184,26 @@ func (p *politeiawww) setupPi(plugins []pdv1.Plugin) error { } // Setup api contexts - r := records.New(p.cfg, p.politeiad, p.db, p.sessions, p.events) - c, err := comments.New(p.cfg, p.politeiad, p.db, + recordsCtx := records.New(p.cfg, p.politeiad, p.db, p.sessions, p.events) + commentsCtx, err := comments.New(p.cfg, p.politeiad, p.db, p.sessions, p.events, plugins) if err != nil { return fmt.Errorf("new comments api: %v", err) } - tv, err := ticketvote.New(p.cfg, p.politeiad, + voteCtx, err := ticketvote.New(p.cfg, p.politeiad, p.sessions, p.events, plugins) if err != nil { return fmt.Errorf("new ticketvote api: %v", err) } - pic, err := pi.New(p.cfg, p.politeiad, p.db, p.sessions, plugins) + piCtx, err := pi.New(p.cfg, p.politeiad, p.db, + p.sessions, p.events, plugins) if err != nil { return fmt.Errorf("new pi api: %v", err) } // Setup routes p.setUserWWWRoutes() - p.setupPiRoutes(r, c, tv, pic) + p.setupPiRoutes(recordsCtx, commentsCtx, voteCtx, piCtx) // Verify paywall settings switch { @@ -227,8 +229,5 @@ func (p *politeiawww) setupPi(plugins []pdv1.Plugin) error { return err } - // Setup event manager - p.setupEventListenersPi() - return nil } diff --git a/politeiawww/politeiad.go b/politeiawww/politeiad.go index 2010c2140..d8362f28a 100644 --- a/politeiawww/politeiad.go +++ b/politeiawww/politeiad.go @@ -63,7 +63,7 @@ func (p *politeiawww) makeRequest(ctx context.Context, method string, route stri return nil, err } req.SetBasicAuth(p.cfg.RPCUser, p.cfg.RPCPass) - r, err := p.client.Do(req) + r, err := p.http.Do(req) if err != nil { return nil, err } diff --git a/politeiawww/politeiawww.go b/politeiawww/politeiawww.go index 8e94d505d..30f371b55 100644 --- a/politeiawww/politeiawww.go +++ b/politeiawww/politeiawww.go @@ -24,6 +24,7 @@ import ( "github.com/decred/politeia/politeiawww/codetracker" "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/events" + "github.com/decred/politeia/politeiawww/mail" "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/politeiawww/user" utilwww "github.com/decred/politeia/politeiawww/util" @@ -71,8 +72,8 @@ type politeiawww struct { router *mux.Router auth *mux.Router // CSRF protected subrouter politeiad *pdclient.Client - client *http.Client - smtp *smtp + http *http.Client // Deprecated; use politeiad client + mail *mail.Client db user.Database sessions *sessions.Sessions events *events.Manager @@ -99,7 +100,7 @@ type politeiawww struct { wsDcrdata *wsdcrdata.Client tracker codetracker.CodeTracker - // The following fields are only used during testing. + // The following fields are only used during testing test bool } diff --git a/politeiawww/records/pi.go b/politeiawww/records/pi.go index 421b9417c..d49ceca8b 100644 --- a/politeiawww/records/pi.go +++ b/politeiawww/records/pi.go @@ -8,7 +8,6 @@ import ( "fmt" v1 "github.com/decred/politeia/politeiawww/api/records/v1" - "github.com/decred/politeia/politeiawww/pi" "github.com/decred/politeia/politeiawww/user" ) @@ -69,16 +68,16 @@ func (r *Records) piHookNewRecordPre(u user.User) error { // Verify user has paid registration paywall if !userHasPaid(u) { return v1.PluginErrorReply{ - PluginID: pi.UserPluginID, - ErrorCode: pi.ErrorCodeUserRegistrationNotPaid, + PluginID: user.PiUserPluginID, + ErrorCode: user.ErrorCodeUserRegistrationNotPaid, } } // Verify user has a proposal credit if !userHasProposalCredits(u) { return v1.PluginErrorReply{ - PluginID: pi.UserPluginID, - ErrorCode: pi.ErrorCodeUserBalanceInsufficient, + PluginID: user.PiUserPluginID, + ErrorCode: user.ErrorCodeUserBalanceInsufficient, } } return nil diff --git a/politeiawww/sessions/sessions.go b/politeiawww/sessions/sessions.go index 9e64589fa..820ea2cb0 100644 --- a/politeiawww/sessions/sessions.go +++ b/politeiawww/sessions/sessions.go @@ -66,12 +66,19 @@ func (s *Sessions) GetSessionUserID(w http.ResponseWriter, r *http.Request) (str return "", err } if session.IsNew { + // If the session is new it means the request did not contain a + // valid session. This could be because it was expired or it + // did not exist. + log.Debugf("Session not found for user") return "", ErrSessionNotFound } - // Delete the session if its expired. Setting the MaxAge - // to <= 0 and then saving it will trigger a deletion. + // Delete the session if its expired. Setting the MaxAge to <= 0 + // and saving the session will trigger a deletion. The previous + // GetSession call should already filter out expired sessions so + // this is really just a sanity check. if sessionIsExpired(session) { + log.Debug("Session is expired") session.Options.MaxAge = -1 s.store.Save(r, w, session) return "", ErrSessionNotFound @@ -101,6 +108,7 @@ func (s *Sessions) GetSessionUser(w http.ResponseWriter, r *http.Request) (*user } if user.Deactivated { + log.Debugf("User has been deactivated") err := s.DelSession(w, r) if err != nil { return nil, err @@ -108,6 +116,8 @@ func (s *Sessions) GetSessionUser(w http.ResponseWriter, r *http.Request) (*user return nil, ErrSessionNotFound } + log.Debugf("Session found for user %v", user.ID) + return user, nil } @@ -123,8 +133,7 @@ func (s *Sessions) DelSession(w http.ResponseWriter, r *http.Request) error { return ErrSessionNotFound } - log.Debugf("Deleting user session: %v %v", - session.ID, session.Values[sessionValueUserID]) + log.Debugf("Deleting user session %v", session.Values[sessionValueUserID]) // Saving the session with a negative MaxAge will cause it to be // deleted. @@ -149,6 +158,8 @@ func (s *Sessions) NewSession(w http.ResponseWriter, r *http.Request, userID str session.Values[sessionValueCreatedAt] = time.Now().Unix() session.Values[sessionValueUserID] = userID + log.Debugf("Session created for user %v", userID) + // Update session in the store and update the response cookie return s.store.Save(r, w, session) } diff --git a/politeiawww/sessions/store.go b/politeiawww/sessions/store.go index a80525560..49c02b4a2 100644 --- a/politeiawww/sessions/store.go +++ b/politeiawww/sessions/store.go @@ -49,7 +49,7 @@ func newSessionID() string { // // This function satisfies the sessions.Store interface. func (s *sessionStore) Get(r *http.Request, name string) (*sessions.Session, error) { - log.Tracef("Get: %v", name) + log.Tracef("SessionStore Get: %v", name) return sessions.GetRegistry(r).Get(s, name) } @@ -66,7 +66,7 @@ func (s *sessionStore) Get(r *http.Request, name string) (*sessions.Session, err // // This function satisfies the sessions.Store interface. func (s *sessionStore) New(r *http.Request, name string) (*sessions.Session, error) { - log.Tracef("New: %v", name) + log.Tracef("SessionStore New: %v", name) // Setup new session session := sessions.NewSession(s, name) @@ -78,7 +78,7 @@ func (s *sessionStore) New(r *http.Request, name string) (*sessions.Session, err // Check if the session cookie already exists c, err := r.Cookie(name) if errors.Is(err, http.ErrNoCookie) { - log.Debugf("Session cookie not found; returning new session") + log.Tracef("Session cookie not found; returning a new session") return session, nil } else if err != nil { return session, err @@ -91,11 +91,11 @@ func (s *sessionStore) New(r *http.Request, name string) (*sessions.Session, err // Decode session ID (overwrites existing session ID) err = securecookie.DecodeMulti(name, c.Value, &session.ID, s.Codecs...) if err != nil { - // If there are any issues decoding the session ID, the - // existing session is considered invalid and the newly - // created session is returned. - log.Debugf("securecookie.DecodeMulti: %v", err) - log.Debugf("Invalid session ID; returning new session") + // If there are any issues decoding the session ID, the existing + // session is considered invalid and the newly created session is + // returned. + log.Tracef("Decode session: %v", err) + log.Tracef("Session invalid; returning a new one") return session, nil } @@ -111,11 +111,11 @@ func (s *sessionStore) New(r *http.Request, name string) (*sessions.Session, err if err != nil { return session, err } - log.Debugf("Session found in store; returning existing session") + log.Tracef("Session found %v", session.ID) case user.ErrSessionNotFound: // Session not found in database; no action needed since the new // session will be returned. - log.Debugf("Session not found in store; returning new session") + log.Tracef("Session not found; returning new session") default: return session, err } @@ -133,7 +133,7 @@ func (s *sessionStore) New(r *http.Request, name string) (*sessions.Session, err // // This function satisfies the sessions.Store interface. func (s *sessionStore) Save(r *http.Request, w http.ResponseWriter, session *sessions.Session) error { - log.Tracef("Save: %v", session.ID) + log.Tracef("SessionStore Save: %v", session.ID) // Delete session if max-age is <= 0 if session.Options.MaxAge <= 0 { diff --git a/politeiawww/smtp.go b/politeiawww/smtp.go deleted file mode 100644 index 7ea111ef8..000000000 --- a/politeiawww/smtp.go +++ /dev/null @@ -1,104 +0,0 @@ -package main - -import ( - "crypto/tls" - "crypto/x509" - "fmt" - "net/mail" - "net/url" - - "github.com/dajohi/goemail" -) - -// smtp is a SMTP client for sending Politeia emails. -type smtp struct { - client *goemail.SMTP // SMTP client - mailName string // Email address name - mailAddress string // Email address - disabled bool // Has email been disabled -} - -// sendEmailTo sends an email with the given subject and body to the provided -// list of email addresses. -func (s *smtp) sendEmailTo(subject, body string, recipients []string) error { - if s.disabled { - return nil - } - if len(recipients) == 0 { - return nil - } - - // Setup email - msg := goemail.NewMessage(s.mailAddress, subject, body) - msg.SetName(s.mailName) - - // Add all recipients to BCC - for _, v := range recipients { - msg.AddBCC(v) - } - - return s.client.Send(msg) -} - -// sendEmail sends an email with the given subject and body, and the caller -// must supply a function which is used to add email addresses to send the -// email to. -func (s *smtp) sendEmail(subject, body string, addToAddressesFn func(*goemail.Message) error) error { - if s.disabled { - return nil - } - - msg := goemail.NewMessage(s.mailAddress, subject, body) - err := addToAddressesFn(msg) - if err != nil { - return err - } - - msg.SetName(s.mailName) - return s.client.Send(msg) -} - -// newSMTP returns a new smtp context. -func newSMTP(host, user, password, emailAddress string, systemCerts *x509.CertPool, skipVerify bool) (*smtp, error) { - // Check if email has been disabled - if host == "" || user == "" || password == "" { - log.Infof("Email: DISABLED") - return &smtp{ - disabled: true, - }, nil - } - - // Parse mail host - h := fmt.Sprintf("smtps://%v:%v@%v", user, password, host) - u, err := url.Parse(h) - if err != nil { - return nil, err - } - - // Parse email address - a, err := mail.ParseAddress(emailAddress) - if err != nil { - return nil, err - } - - // Config tlsConfig based on config settings - tlsConfig := &tls.Config{} - if systemCerts == nil && skipVerify { - tlsConfig.InsecureSkipVerify = true - } else if systemCerts != nil { - tlsConfig.RootCAs = systemCerts - } - - // Initialize SMTP client - client, err := goemail.NewSMTP(u.String(), tlsConfig) - if err != nil { - return nil, err - } - - return &smtp{ - client: client, - mailName: a.Name, - mailAddress: a.Address, - disabled: false, - }, nil -} diff --git a/politeiawww/testing.go b/politeiawww/testing.go index 8c17fe179..69eca5c47 100644 --- a/politeiawww/testing.go +++ b/politeiawww/testing.go @@ -28,6 +28,7 @@ import ( www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/comments" "github.com/decred/politeia/politeiawww/config" + "github.com/decred/politeia/politeiawww/mail" "github.com/decred/politeia/politeiawww/pi" "github.com/decred/politeia/politeiawww/records" "github.com/decred/politeia/politeiawww/sessions" @@ -337,10 +338,10 @@ func newTestPoliteiawww(t *testing.T) (*politeiawww, func()) { t.Fatalf("setup database: %v", err) } - // Setup smtp - smtp, err := newSMTP("", "", "", "", nil, false) + // Setup mail client + mailClient, err := mail.New("", "", "", "", "", false) if err != nil { - t.Fatalf("setup SMTP: %v", err) + t.Fatal(err) } // Setup sessions @@ -356,7 +357,7 @@ func newTestPoliteiawww(t *testing.T) (*politeiawww, func()) { router: mux.NewRouter(), auth: mux.NewRouter(), sessions: sessions.New(db, cookieKey), - smtp: smtp, + mail: mailClient, db: db, test: true, userEmails: make(map[string]uuid.UUID), @@ -443,7 +444,7 @@ func newTestCMSwww(t *testing.T) (*politeiawww, func()) { } // Setup smtp - smtp, err := newSMTP("", "", "", "", nil, false) + mailClient, err := mail.New("", "", "", "", "", false) if err != nil { t.Fatalf("setup SMTP: %v", err) } @@ -465,7 +466,7 @@ func newTestCMSwww(t *testing.T) (*politeiawww, func()) { router: mux.NewRouter(), auth: mux.NewRouter(), sessions: sessions.New(db, cookieKey), - smtp: smtp, + mail: mailClient, test: true, userEmails: make(map[string]uuid.UUID), userPaywallPool: make(map[uuid.UUID]paywallPoolMember), diff --git a/politeiawww/user.go b/politeiawww/user.go index 9136b2ab0..467abaed0 100644 --- a/politeiawww/user.go +++ b/politeiawww/user.go @@ -480,7 +480,7 @@ func (p *politeiawww) processNewUser(nu www.NewUser) (*www.NewUserReply, error) // Send reply. Only return the verification token in // the reply if the mail server has been disabled. var t string - if p.smtp.disabled { + if !p.mail.IsEnabled() { t = hex.EncodeToString(u.NewUserVerificationToken) } return &www.NewUserReply{ @@ -589,7 +589,7 @@ func (p *politeiawww) processNewUser(nu www.NewUser) (*www.NewUserReply, error) // Only return the verification token in the reply // if the mail server has been disabled. var t string - if p.smtp.disabled { + if !p.mail.IsEnabled() { t = hex.EncodeToString(u.NewUserVerificationToken) } return &www.NewUserReply{ @@ -862,7 +862,7 @@ func (p *politeiawww) processResendVerification(rv *www.ResendVerification) (*ww } // Only set the token if email verification is disabled. - if p.smtp.disabled { + if !p.mail.IsEnabled() { rvr.VerificationToken = hex.EncodeToString(token) } return &rvr, nil @@ -939,7 +939,7 @@ func (p *politeiawww) processUpdateUserKey(usr *user.User, uuk www.UpdateUserKey // Only set the token if email verification is disabled. var t string - if p.smtp.disabled { + if !p.mail.IsEnabled() { t = token } return &www.UpdateUserKeyReply{ @@ -1415,7 +1415,7 @@ func (p *politeiawww) resetPassword(rp www.ResetPassword) resetPasswordResult { // Only include the verification token in the reply if the // email server has been disabled. var reply www.ResetPasswordReply - if p.smtp.disabled { + if !p.mail.IsEnabled() { reply.VerificationToken = hex.EncodeToString(tokenb) } diff --git a/politeiawww/pi/user.go b/politeiawww/user/pi.go similarity index 90% rename from politeiawww/pi/user.go rename to politeiawww/user/pi.go index a58615030..e39c52f3f 100644 --- a/politeiawww/pi/user.go +++ b/politeiawww/user/pi.go @@ -2,16 +2,16 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package pi +package user // Everything defined in this file is a temporary measure until proper user // plugins have been added to politeiawww, at which point these errors will be // deprecated. const ( - // UserPluginID is a temporary plugin ID for user functionality + // PiUserPluginID is a temporary plugin ID for user functionality // that is specific to pi. - UserPluginID = "piuser" + PiUserPluginID = "piuser" // ErrorCodeInvalid is an invalid error code. ErrorCodeInvalid = 0 diff --git a/politeiawww/www.go b/politeiawww/www.go index 50c673adf..47b117b27 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -36,6 +36,7 @@ import ( ghtracker "github.com/decred/politeia/politeiawww/codetracker/github" "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/events" + "github.com/decred/politeia/politeiawww/mail" "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/politeiawww/user" "github.com/decred/politeia/politeiawww/user/cockroachdb" @@ -628,7 +629,7 @@ func _main() error { } // Setup user database - log.Infof("User db: %v", loadedCfg.UserDB) + log.Infof("User database: %v", loadedCfg.UserDB) var userDB user.Database switch loadedCfg.UserDB { @@ -688,15 +689,15 @@ func _main() error { } // Setup smtp client - smtp, err := newSMTP(loadedCfg.MailHost, loadedCfg.MailUser, - loadedCfg.MailPass, loadedCfg.MailAddress, loadedCfg.SystemCerts, - loadedCfg.SMTPSkipVerify) + mailClient, err := mail.New(loadedCfg.MailHost, loadedCfg.MailUser, + loadedCfg.MailPass, loadedCfg.MailAddress, loadedCfg.MailCert, + loadedCfg.MailSkipVerify) if err != nil { - return fmt.Errorf("newSMTP: %v", err) + return fmt.Errorf("new mail client: %v", err) } // Setup politeiad client - client, err := util.NewHTTPClient(false, loadedCfg.RPCCert) + httpClient, err := util.NewHTTPClient(false, loadedCfg.RPCCert) if err != nil { return err } @@ -708,8 +709,8 @@ func _main() error { router: router, auth: auth, politeiad: pdc, - client: client, - smtp: smtp, + http: httpClient, + mail: mailClient, db: userDB, sessions: sessions.New(userDB, cookieKey), events: events.NewManager(), diff --git a/util/net.go b/util/net.go index 78516998c..11b56e3b7 100644 --- a/util/net.go +++ b/util/net.go @@ -43,10 +43,10 @@ func NewHTTPClient(skipVerify bool, certPath string) (*http.Client, error) { } certPool, err := x509.SystemCertPool() if err != nil { + fmt.Printf("WARN: unable to get system cert pool: %v\n") certPool = x509.NewCertPool() } certPool.AppendCertsFromPEM(cert) - tlsConfig.RootCAs = certPool } From 767b69d3db307c563a4b5f0f13cb1f1f1a631c30 Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 26 Feb 2021 18:11:16 -0600 Subject: [PATCH 345/449] Cleanup. --- politeiad/README.md | 22 ++++++++++------ .../backend/tstorebe/tstore/trillianclient.go | 16 ++++++------ politeiad/backend/tstorebe/tstore/tstore.go | 25 ++++++++++--------- politeiad/backend/tstorebe/tstorebe.go | 22 ++++++++-------- politeiad/config.go | 16 ++++++------ politeiad/politeiad.go | 6 ++--- politeiawww/pi/events.go | 10 ++++---- politeiawww/records/process.go | 20 +++++++-------- 8 files changed, 71 insertions(+), 66 deletions(-) diff --git a/politeiad/README.md b/politeiad/README.md index 67f29e99e..5ce2cad38 100644 --- a/politeiad/README.md +++ b/politeiad/README.md @@ -97,8 +97,8 @@ politeiad will be accessing the databse from `localhost`. See the setup script comments for more complex setups. - Run the following commands. You will need to replace "rootpass" with the - existing password of your root user. The "politeiadpass" and "trillianpass" + Run the following commands. You will need to replace `rootpass` with the + existing password of your root user. The `politeiadpass` and `trillianpass` are the password that will be set for the politeiad and trillian users when the script creates them. @@ -116,7 +116,7 @@ politeiad These can only be run once the trillian MySQL user has been created in the previous step. - The "trillianpass" and "rootpass" will need to be updated to the passwords + The `trillianpass` and `rootpass` will need to be updated to the passwords for your trillian and root users. ``` @@ -149,7 +149,7 @@ politeiad for unvetted records and a trillian instance for vetted records. You will be starting up 4 seperate processes in this step. - You will need to replace the "trillianpass" with the trillian user's + You will need to replace the `trillianpass` with the trillian user's password that you setup in previous steps. Startup unvetted log server @@ -219,8 +219,8 @@ politeiad [`sample-politeiad.conf`](https://github.com/decred/politeia/blob/master/politeiad/sample-politeiad.conf) - Copy the sample configuration file to the politeiad data directory. The data - directory will depend on your OS. + Copy the sample configuration file to the politeiad app data directory. The + app data directory will depend on your OS. * **macOS** @@ -241,7 +241,8 @@ politeiad ``` Use the following config settings to spin up a development politeiad - instance. + instance. You'll need to replace the `politeiadpass` with the password + you created for your politeiad MySQL user. **politeiad.conf** @@ -250,6 +251,10 @@ politeiad rpcpass=pass testnet=true + ; Tstore database settings + dbtype=mysql + dbpass=politeiadpass + ; Pi plugin configuration plugin=pi plugin=comments @@ -264,6 +269,9 @@ politeiad $ politeiad ``` + A database encryption key and a trillian signing key will be created on + startup and saved to the politeiad app data directory. + # Tools and reference clients * [politeia](https://github.com/decred/politeia/tree/master/politeiad/cmd/politeia) - Reference client for politeiad. diff --git a/politeiad/backend/tstorebe/tstore/trillianclient.go b/politeiad/backend/tstorebe/tstore/trillianclient.go index 89c78cca6..8b639d934 100644 --- a/politeiad/backend/tstorebe/tstore/trillianclient.go +++ b/politeiad/backend/tstorebe/tstore/trillianclient.go @@ -137,12 +137,11 @@ func (t *tclient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { TreeType: trillian.TreeType_LOG, HashStrategy: trillian.HashStrategy_RFC6962_SHA256, HashAlgorithm: sigpb.DigitallySigned_SHA256, - SignatureAlgorithm: sigpb.DigitallySigned_ECDSA, - // TODO SignatureAlgorithm: sigpb.DigitallySigned_ED25519, - DisplayName: "", - Description: "", - MaxRootDuration: ptypes.DurationProto(0), - PrivateKey: pk, + SignatureAlgorithm: sigpb.DigitallySigned_ED25519, + DisplayName: "", + Description: "", + MaxRootDuration: ptypes.DurationProto(0), + PrivateKey: pk, }, }) if err != nil { @@ -499,8 +498,7 @@ func (t *tclient) close() { func newTrillianKey() (crypto.Signer, error) { return keys.NewFromSpec(&keyspb.Specification{ - // TODO Params: &keyspb.Specification_Ed25519Params{}, - Params: &keyspb.Specification_EcdsaParams{}, + Params: &keyspb.Specification_Ed25519Params{}, }) } @@ -605,7 +603,7 @@ func (t *testTClient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) TreeType: trillian.TreeType_LOG, HashStrategy: trillian.HashStrategy_RFC6962_SHA256, HashAlgorithm: sigpb.DigitallySigned_SHA256, - SignatureAlgorithm: sigpb.DigitallySigned_ECDSA, + SignatureAlgorithm: sigpb.DigitallySigned_ED25519, DisplayName: "", Description: "", MaxRootDuration: ptypes.DurationProto(0), diff --git a/politeiad/backend/tstorebe/tstore/tstore.go b/politeiad/backend/tstorebe/tstore/tstore.go index f14c4e9a5..831342813 100644 --- a/politeiad/backend/tstorebe/tstore/tstore.go +++ b/politeiad/backend/tstorebe/tstore/tstore.go @@ -33,8 +33,8 @@ const ( DBTypeMySQL = "mysql" dbUser = "politeiad" - defaultTrillianKeyFilename = "trillian.key" - defaultStoreDirname = "store" + defaultTrillianSigningKeyFilename = "trillian.key" + defaultStoreDirname = "store" // Blob entry data descriptors dataDescriptorFile = "file-v1" @@ -1378,11 +1378,12 @@ func (t *Tstore) Close() { } } -func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianKeyFile, encryptionKeyFile, dbType, dbHost, dbPass, dcrtimeHost, dcrtimeCert string) (*Tstore, error) { - // Load encryption key if provided. An encryption key is optional. +func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianSigningKeyFile, dbType, dbHost, dbPass, dbEncryptionKeyFile, dcrtimeHost, dcrtimeCert string) (*Tstore, error) { + // Load database encryption key if provided. An encryption key is + // optional. var ek *encryptionKey - if encryptionKeyFile != "" { - f, err := os.Open(encryptionKeyFile) + if dbEncryptionKeyFile != "" { + f, err := os.Open(dbEncryptionKeyFile) if err != nil { return nil, err } @@ -1397,7 +1398,7 @@ func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trilli f.Close() ek = newEncryptionKey(&key) - log.Infof("Encryption key %v: %v", id, encryptionKeyFile) + log.Infof("Encryption key %v: %v", id, dbEncryptionKeyFile) } // Setup datadir for this tstore instance @@ -1408,16 +1409,16 @@ func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trilli } // Setup trillian client - if trillianKeyFile == "" { + if trillianSigningKeyFile == "" { // No file path was given. Use the default path. - fn := fmt.Sprintf("%v-%v", id, defaultTrillianKeyFilename) - trillianKeyFile = filepath.Join(homeDir, fn) + fn := fmt.Sprintf("%v", defaultTrillianSigningKeyFilename) + trillianSigningKeyFile = filepath.Join(homeDir, fn) } - log.Infof("Trillian key %v: %v", id, trillianKeyFile) + log.Infof("Trillian key %v: %v", id, trillianSigningKeyFile) log.Infof("Trillian host %v: %v", id, trillianHost) - trillianClient, err := newTClient(trillianHost, trillianKeyFile) + trillianClient, err := newTClient(trillianHost, trillianSigningKeyFile) if err != nil { return nil, err } diff --git a/politeiad/backend/tstorebe/tstorebe.go b/politeiad/backend/tstorebe/tstorebe.go index 97d4127f4..67813503a 100644 --- a/politeiad/backend/tstorebe/tstorebe.go +++ b/politeiad/backend/tstorebe/tstorebe.go @@ -31,7 +31,7 @@ import ( ) const ( - defaultEncryptionKeyFilename = "tstorebe.key" + defaultEncryptionKeyFilename = "tstore-sbox.key" // Tstore instance IDs tstoreIDUnvetted = "unvetted" @@ -1954,25 +1954,25 @@ func (t *tstoreBackend) setup() error { } // New returns a new tstoreBackend. -func New(anp *chaincfg.Params, homeDir, dataDir, unvettedTrillianHost, unvettedTrillianKeyFile, vettedTrillianHost, vettedTrillianKeyFile, encryptionKeyFile, dbType, dbHost, dbPass, dcrtimeHost, dcrtimeCert string) (*tstoreBackend, error) { +func New(anp *chaincfg.Params, homeDir, dataDir, trillianHostUnvetted, trillianHostVetted, trillianSigningKey, dbType, dbHost, dbPass, dbEncryptionKeyFile, dcrtimeHost, dcrtimeCert string) (*tstoreBackend, error) { // Setup encryption key file - if encryptionKeyFile == "" { + if dbEncryptionKeyFile == "" { // No file path was given. Use the default path. - encryptionKeyFile = filepath.Join(homeDir, defaultEncryptionKeyFilename) + dbEncryptionKeyFile = filepath.Join(homeDir, defaultEncryptionKeyFilename) } - if !util.FileExists(encryptionKeyFile) { + if !util.FileExists(dbEncryptionKeyFile) { // Encryption key file does not exist. Create one. log.Infof("Generating encryption key") key, err := sbox.NewKey() if err != nil { return nil, err } - err = ioutil.WriteFile(encryptionKeyFile, key[:], 0400) + err = ioutil.WriteFile(dbEncryptionKeyFile, key[:], 0400) if err != nil { return nil, err } util.Zero(key[:]) - log.Infof("Encryption key created: %v", encryptionKeyFile) + log.Infof("Encryption key created: %v", dbEncryptionKeyFile) } // Verify dcrtime host @@ -1984,14 +1984,14 @@ func New(anp *chaincfg.Params, homeDir, dataDir, unvettedTrillianHost, unvettedT // Setup tstore instances unvetted, err := tstore.New(tstoreIDUnvetted, homeDir, dataDir, anp, - unvettedTrillianHost, unvettedTrillianKeyFile, encryptionKeyFile, - dbType, dbHost, dbPass, dcrtimeHost, dcrtimeCert) + trillianHostUnvetted, trillianSigningKey, dbType, dbHost, dbPass, + dbEncryptionKeyFile, dcrtimeHost, dcrtimeCert) if err != nil { return nil, fmt.Errorf("new tstore unvetted: %v", err) } vetted, err := tstore.New(tstoreIDVetted, homeDir, dataDir, anp, - vettedTrillianHost, vettedTrillianKeyFile, "", - dbType, dbHost, dbPass, dcrtimeHost, dcrtimeCert) + trillianHostVetted, trillianSigningKey, dbType, dbHost, dbPass, "", + dcrtimeHost, dcrtimeCert) if err != nil { return nil, fmt.Errorf("new tstore vetted: %v", err) } diff --git a/politeiad/config.go b/politeiad/config.go index 56357b93a..9d27d130b 100644 --- a/politeiad/config.go +++ b/politeiad/config.go @@ -93,18 +93,16 @@ type config struct { GitTrace bool `long:"gittrace" description:"Enable git tracing in logs"` DcrdataHost string `long:"dcrdatahost" description:"Dcrdata ip:port"` - // TODO validate these config params and set defaults. Also consider - // making them specific to tstore. Ex: tstore.TrillianHostUnvetted. - // TODO Verify that the trillian key is being used + // TODO validate these config params Backend string `long:"backend"` TrillianHostUnvetted string `long:"trillianhostunvetted"` TrillianHostVetted string `long:"trillianhostvetted"` - TrillianKeyUnvetted string `long:"trilliankeyunvetted"` - TrillianKeyVetted string `long:"trilliankeyvetted"` - DBType string `long:"dbtype"` - DBHost string `long:"dbhost" description:"Database ip:port"` - DBPass string `long:"dbpass" description:"Database password"` - EncryptionKey string `long:"encryptionkey" description:"Database encryption key"` + TrillianSigningKey string `long:"trilliansigningkey"` + + DBType string `long:"dbtype" description:"Database type` + DBHost string `long:"dbhost" description:"Database ip:port"` + DBPass string `long:"dbpass" description:"Database password"` + DBEncryptionKey string `long:"encryptionkey" description:"Database encryption key file"` // Plugin settings Plugins []string `long:"plugin"` diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index adbc29aac..48ad3f221 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -1609,9 +1609,9 @@ func _main() error { p.backend = b case backendTstore: b, err := tstorebe.New(activeNetParams.Params, cfg.HomeDir, cfg.DataDir, - cfg.TrillianHostUnvetted, cfg.TrillianKeyUnvetted, - cfg.TrillianHostVetted, cfg.TrillianKeyVetted, cfg.EncryptionKey, - cfg.DBType, cfg.DBHost, cfg.DBPass, cfg.DcrtimeHost, cfg.DcrtimeCert) + cfg.TrillianHostUnvetted, cfg.TrillianHostVetted, cfg.TrillianSigningKey, + cfg.DBType, cfg.DBHost, cfg.DBPass, cfg.DBEncryptionKey, + cfg.DcrtimeHost, cfg.DcrtimeCert) if err != nil { return fmt.Errorf("new tstorebe: %v", err) } diff --git a/politeiawww/pi/events.go b/politeiawww/pi/events.go index 5ccdfdaff..d82133448 100644 --- a/politeiawww/pi/events.go +++ b/politeiawww/pi/events.go @@ -85,7 +85,7 @@ func (p *Pi) handleEventRecordNew(ch chan interface{}) { } */ - log.Debugf("Record new event sent %v", e.Record.CensorshipRecord.Token) + log.Debugf("Proposal new ntfn sent %v", e.Record.CensorshipRecord.Token) } } @@ -128,7 +128,7 @@ func (p *Pi) handleEventRecordEdit(ch chan interface{}) { } */ - log.Debugf("Record edit event sent %v", e.Record.CensorshipRecord.Token) + log.Debugf("Proposal edit ntfn sent %v", e.Record.CensorshipRecord.Token) } } @@ -332,7 +332,7 @@ func (p *Pi) handleEventCommentNew(ch chan interface{}) { continue */ - log.Debugf("Comment new event sent %v", d.Comment.Token) + log.Debugf("Proposal comment ntfn sent %v", d.Comment.Token) } } @@ -374,7 +374,7 @@ func (p *Pi) handleEventVoteAuthorized(ch chan interface{}) { } */ - log.Debugf("Vote authorized event sent %v", e.Auth.Token) + log.Debugf("Proposal vote authorized ntfn sent %v", e.Auth.Token) } } @@ -430,7 +430,7 @@ func (p *Pi) handleEventVoteStart(ch chan interface{}) { _ = e token := "fix me" - log.Debugf("Vote start event sent %v", token) + log.Debugf("Proposal vote started ntfn sent %v", token) } } diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index 2e7876a09..996c4fc48 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -70,6 +70,11 @@ func (r *Records) processNew(ctx context.Context, n v1.New, u user.User) (*v1.Ne return nil, err } + log.Infof("Record submitted: %v", rc.CensorshipRecord.Token) + for k, f := range rc.Files { + log.Infof("%02v: %v", k, f.Name) + } + // Execute post plugin hooks. Checking the mode is a temporary // measure until user plugins have been properly implemented. switch r.cfg.Mode { @@ -87,11 +92,6 @@ func (r *Records) processNew(ctx context.Context, n v1.New, u user.User) (*v1.Ne Record: *rc, }) - log.Infof("Record submitted: %v", rc.CensorshipRecord.Token) - for k, f := range rc.Files { - log.Infof("%02v: %v %v", k, f.Name, f.Digest) - } - return &v1.NewReply{ Record: *rc, }, nil @@ -195,6 +195,11 @@ func (r *Records) processEdit(ctx context.Context, e v1.Edit, u user.User) (*v1. rc := convertRecordToV1(*pdr, e.State) recordPopulateUserData(&rc, u) + log.Infof("Record edited: %v %v", e.State, rc.CensorshipRecord.Token) + for k, f := range rc.Files { + log.Infof("%02v: %v", k, f.Name) + } + // Emit event r.events.Emit(EventTypeEdit, EventEdit{ @@ -203,11 +208,6 @@ func (r *Records) processEdit(ctx context.Context, e v1.Edit, u user.User) (*v1. Record: rc, }) - log.Infof("Record edited: %v %v", e.State, rc.CensorshipRecord.Token) - for k, f := range rc.Files { - log.Infof("%02v: %v %v", k, f.Name, f.Digest) - } - return &v1.EditReply{ Record: rc, }, nil From ecf06ba0d7f27d75b1bcd0d2c7edc0951448ae93 Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 26 Feb 2021 18:19:01 -0600 Subject: [PATCH 346/449] Fix typos. --- politeiad/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/politeiad/config.go b/politeiad/config.go index 9d27d130b..dd01ab37d 100644 --- a/politeiad/config.go +++ b/politeiad/config.go @@ -99,10 +99,10 @@ type config struct { TrillianHostVetted string `long:"trillianhostvetted"` TrillianSigningKey string `long:"trilliansigningkey"` - DBType string `long:"dbtype" description:"Database type` + DBType string `long:"dbtype" description:"Database type"` DBHost string `long:"dbhost" description:"Database ip:port"` DBPass string `long:"dbpass" description:"Database password"` - DBEncryptionKey string `long:"encryptionkey" description:"Database encryption key file"` + DBEncryptionKey string `long:"dbencryptionkey" description:"Database encryption key file"` // Plugin settings Plugins []string `long:"plugin"` From 2a77796ad0e06e7baddfd048c03f3cb8d0f25d67 Mon Sep 17 00:00:00 2001 From: luke Date: Sat, 27 Feb 2021 10:41:54 -0600 Subject: [PATCH 347/449] Add pi proposal new and edit ntfns. --- politeiawww/email.go | 6 +- politeiawww/events.go | 21 ---- politeiawww/pi/events.go | 186 ++++++++++++++------------------- politeiawww/pi/mail.go | 107 +++++++++++++++++++ politeiawww/pi/pi.go | 5 +- politeiawww/pi/process.go | 9 +- politeiawww/piwww.go | 2 +- politeiawww/records/process.go | 4 +- politeiawww/templates.go | 35 ------- politeiawww/user/user.go | 10 ++ 10 files changed, 209 insertions(+), 176 deletions(-) create mode 100644 politeiawww/pi/mail.go diff --git a/politeiawww/email.go b/politeiawww/email.go index f1f7a4345..fe8b92625 100644 --- a/politeiawww/email.go +++ b/politeiawww/email.go @@ -17,10 +17,8 @@ import ( const ( // GUI routes. These are used in notification emails to direct the // user to the correct GUI pages. - guiRouteProposalDetails = "/proposals/{token}" - guirouteProposalComments = "/proposals/{token}/comments/{id}" - guiRouteRegisterNewUser = "/register" - guiRouteDCCDetails = "/dcc/{token}" + guiRouteRegisterNewUser = "/register" + guiRouteDCCDetails = "/dcc/{token}" ) func createBody(tpl *template.Template, tplData interface{}) (string, error) { diff --git a/politeiawww/events.go b/politeiawww/events.go index 1c7830d8a..55c2e322b 100644 --- a/politeiawww/events.go +++ b/politeiawww/events.go @@ -5,7 +5,6 @@ package main import ( - www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/user" ) @@ -39,26 +38,6 @@ func (p *politeiawww) setupEventListenersCMS() { go p.handleEventDCCSupportOppose(ch) } -// notificationIsSet returns whether the provided user has the provided -// notification bit set. -func notificationIsSet(emailNotifications uint64, n www.EmailNotificationT) bool { - return emailNotifications&uint64(n) != 0 -} - -// userNotificationEnabled returns whether the user should receive the provided -// notification. -func userNotificationEnabled(u user.User, n www.EmailNotificationT) bool { - // Never send notification to deactivated users - if u.Deactivated { - return false - } - // Check if notification bit is set - if !notificationIsSet(u.EmailNotifications, n) { - return false - } - return true -} - type dataInvoiceComment struct { token string // Comment token email string // User email diff --git a/politeiawww/pi/events.go b/politeiawww/pi/events.go index d82133448..06b00c5ae 100644 --- a/politeiawww/pi/events.go +++ b/politeiawww/pi/events.go @@ -5,9 +5,12 @@ package pi import ( + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/comments" "github.com/decred/politeia/politeiawww/records" "github.com/decred/politeia/politeiawww/ticketvote" + "github.com/decred/politeia/politeiawww/user" ) func (p *Pi) setupEventListeners() { @@ -17,6 +20,8 @@ func (p *Pi) setupEventListeners() { // 3. Launch an event handler to listen for events emitted into the // channel by the event manager. + log.Debugf("Setting up pi event listeners") + // Record new ch := make(chan interface{}) p.events.Register(records.EventTypeNew, ch) @@ -56,36 +61,41 @@ func (p *Pi) handleEventRecordNew(ch chan interface{}) { continue } - /* - // Compile a list of users to send the notification to - emails := make([]string, 0, 256) - err := p.db.AllUsers(func(u *user.User) { - switch { - case !u.Admin: - // Only admins get this notification - return - case !userNotificationEnabled(*u, - www.NotificationEmailAdminProposalNew): - // Admin doesn't have notification bit set - return - } - - // Add user to notification list - emails = append(emails, u.Email) - }) - if err != nil { - log.Errorf("handleEventRecordNew: AllUsers: %v", err) + // Compile notification email list + var ( + emails = make([]string, 0, 256) + ntfnBit = uint64(www.NotificationEmailAdminProposalNew) + ) + err := p.userdb.AllUsers(func(u *user.User) { + switch { + case !u.Admin: + // Only admins get this notification return + case !u.NotificationIsEnabled(ntfnBit): + // Admin doesn't have notification bit set + return + default: + // User is an admin and has the notification bit set. Add + // them to the email list. + emails = append(emails, u.Email) } + }) + if err != nil { + log.Errorf("handleEventRecordNew: AllUsers: %v", err) + return + } - // Send email notification - err = p.emailRecordNew(d.token, d.name, d.username, emails) - if err != nil { - log.Errorf("emailRecordNew: %v", err) - } + // Send notfication email + var ( + token = e.Record.CensorshipRecord.Token + name = proposalName(e.Record) + ) + err = p.mailNtfnProposalNew(token, name, e.User.Username, emails) + if err != nil { + log.Errorf("mailNtfnProposalNew: %v", err) + } - */ - log.Debugf("Proposal new ntfn sent %v", e.Record.CensorshipRecord.Token) + log.Debugf("Proposal new ntfn sent %v", token) } } @@ -97,38 +107,53 @@ func (p *Pi) handleEventRecordEdit(ch chan interface{}) { continue } - /* - // Compile a list of users to send the notification to - emails := make([]string, 0, 256) - err := p.db.AllUsers(func(u *user.User) { - // Check circumstances where we don't notify - switch { - case u.ID.String() == d.userID: - // User is the author - return - case !userNotificationEnabled(*u, - www.NotificationEmailRegularRecordEdit): - // User doesn't have notification bit set - return - } + // Only send edit notifications for public proposals + if e.State == rcv1.RecordStateUnvetted { + log.Debugf("Proposal is unvetted no edit ntfn %v", + e.Record.CensorshipRecord.Token) + continue + } - // Add user to notification list + // Compile notification email list + var ( + emails = make([]string, 0, 256) + authorID = e.User.ID.String() + ntfnBit = uint64(www.NotificationEmailRegularProposalEdited) + ) + err := p.userdb.AllUsers(func(u *user.User) { + switch { + case u.ID.String() == authorID: + // User is the author. No need to send the notification to + // the author. + return + case u.NotificationIsEnabled(ntfnBit): + // User doesn't have notification bit set + return + default: + // User has the notification bit set. Add them to the email + // list. emails = append(emails, u.Email) - }) - if err != nil { - log.Errorf("handleEventRecordEdit: AllUsers: %v", err) - continue } + }) + if err != nil { + log.Errorf("handleEventRecordEdit: AllUsers: %v", err) + continue + } - err = p.emailRecordEdit(d.name, d.username, - d.token, d.version, emails) - if err != nil { - log.Errorf("emailRecordEdit: %v", err) - continue - } + // Send notification email + var ( + token = e.Record.CensorshipRecord.Token + version = e.Record.Version + name = proposalName(e.Record) + username = e.User.Username + ) + err = p.mailNtfnProposalEdit(token, version, name, username, emails) + if err != nil { + log.Errorf("mailNtfnProposaledit: %v", err) + continue + } - */ - log.Debugf("Proposal edit ntfn sent %v", e.Record.CensorshipRecord.Token) + log.Debugf("Proposal edit ntfn sent %v", token) } } @@ -435,57 +460,6 @@ func (p *Pi) handleEventVoteStart(ch chan interface{}) { } /* -// emailProposalSubmitted send a proposal submitted notification email to -// the provided list of emails. -func (p *politeiawww) emailProposalSubmitted(token, name, username string, emails []string) error { - route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) - l, err := url.Parse(p.cfg.WebServerAddress + route) - if err != nil { - return err - } - - tmplData := proposalSubmitted{ - Username: username, - Name: name, - Link: l.String(), - } - - subject := "New Proposal Submitted" - body, err := createBody(proposalSubmittedTmpl, tmplData) - if err != nil { - return err - } - - return p.smtp.sendEmailTo(subject, body, emails) -} - -// emailProposalEdited sends a proposal edited notification email to the -// provided list of emails. -func (p *politeiawww) emailProposalEdited(name, username, token, version string, emails []string) error { - route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) - l, err := url.Parse(p.cfg.WebServerAddress + route) - if err != nil { - return err - } - - tmplData := proposalEdited{ - Name: name, - Version: version, - Username: username, - Link: l.String(), - } - - subject := "Proposal Edited" - body, err := createBody(proposalEditedTmpl, tmplData) - if err != nil { - return err - } - - return p.smtp.sendEmailTo(subject, body, emails) -} - -// emailProposalStatusChange sends a proposal status change email to the -// provided email addresses. func (p *politeiawww) emailProposalStatusChange(d dataProposalStatusChange, proposalName string, emails []string) error { route := strings.Replace(guiRouteProposalDetails, "{token}", d.token, 1) l, err := url.Parse(p.cfg.WebServerAddress + route) @@ -517,8 +491,6 @@ func (p *politeiawww) emailProposalStatusChange(d dataProposalStatusChange, prop return p.smtp.sendEmailTo(subject, body, emails) } -// emailProposalStatusChangeAuthor sends a proposal status change notification -// email to the provided email address. func (p *politeiawww) emailProposalStatusChangeToAuthor(d dataProposalStatusChange, proposalName, authorEmail string) error { route := strings.Replace(guiRouteProposalDetails, "{token}", d.token, 1) l, err := url.Parse(p.cfg.WebServerAddress + route) @@ -560,8 +532,6 @@ func (p *politeiawww) emailProposalStatusChangeToAuthor(d dataProposalStatusChan return p.smtp.sendEmailTo(subject, body, []string{authorEmail}) } -// emailProposalCommentSubmitted sends a proposal comment submitted email to -// the provided email address. func (p *politeiawww) emailProposalCommentSubmitted(token, commentID, commentUsername, proposalName, proposalAuthorEmail string) error { // Setup comment URL route := strings.Replace(guirouteProposalComments, "{token}", token, 1) diff --git a/politeiawww/pi/mail.go b/politeiawww/pi/mail.go new file mode 100644 index 000000000..98d496446 --- /dev/null +++ b/politeiawww/pi/mail.go @@ -0,0 +1,107 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package pi + +import ( + "bytes" + "net/url" + "strings" + "text/template" +) + +const ( + // TODO GUI links needs to be updated + // The following routes are used in notification emails to direct + // the user to the correct GUI pages. + guiRouteRecordDetails = "/record/{token}" + guiRouteRecordComment = "/record/{token}/comments/{id}" +) + +type proposalNew struct { + Username string // Author username + Name string // Proposal name + Link string // GUI proposal details URL +} + +var proposalNewText = ` +A new proposal has been submitted on Politeia by {{.Username}}: + +{{.Name}} +{{.Link}} +` + +var proposalNewTmpl = template.Must( + template.New("proposalNew").Parse(proposalNewText)) + +func (p *Pi) mailNtfnProposalNew(token, name, username string, emails []string) error { + route := strings.Replace(guiRouteRecordDetails, "{token}", token, 1) + u, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err + } + + tmplData := proposalNew{ + Username: username, + Name: name, + Link: u.String(), + } + + subject := "New Proposal Submitted" + body, err := populateTemplate(proposalNewTmpl, tmplData) + if err != nil { + return err + } + + return p.mail.SendTo(subject, body, emails) +} + +type proposalEdit struct { + Name string // Proposal name + Version string // Proposal version + Username string // Author username + Link string // GUI proposal details URL +} + +var proposalEditText = ` +A proposal by {{.Username}} has just been edited: + +{{.Name}} (Version {{.Version}}) +{{.Link}} +` + +var proposalEditTmpl = template.Must( + template.New("proposalEdit").Parse(proposalEditText)) + +func (p *Pi) mailNtfnProposalEdit(token, version, name, username string, emails []string) error { + route := strings.Replace(guiRouteRecordDetails, "{token}", token, 1) + u, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err + } + + tmplData := proposalEdit{ + Name: name, + Version: version, + Username: username, + Link: u.String(), + } + + subject := "Proposal Edited" + body, err := populateTemplate(proposalEditTmpl, tmplData) + if err != nil { + return err + } + + return p.mail.SendTo(subject, body, emails) +} + +func populateTemplate(tpl *template.Template, tplData interface{}) (string, error) { + var buf bytes.Buffer + err := tpl.Execute(&buf, tplData) + if err != nil { + return "", err + } + return buf.String(), nil +} diff --git a/politeiawww/pi/pi.go b/politeiawww/pi/pi.go index 3999b24a5..ff47ecb27 100644 --- a/politeiawww/pi/pi.go +++ b/politeiawww/pi/pi.go @@ -16,6 +16,7 @@ import ( v1 "github.com/decred/politeia/politeiawww/api/pi/v1" "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/events" + "github.com/decred/politeia/politeiawww/mail" "github.com/decred/politeia/politeiawww/sessions" "github.com/decred/politeia/politeiawww/user" "github.com/decred/politeia/util" @@ -28,6 +29,7 @@ type Pi struct { userdb user.Database sessions *sessions.Sessions events *events.Manager + mail *mail.Client policy *v1.PolicyReply } @@ -72,7 +74,7 @@ func (p *Pi) HandleProposals(w http.ResponseWriter, r *http.Request) { } // New returns a new Pi context. -func New(cfg *config.Config, pdc *pdclient.Client, udb user.Database, s *sessions.Sessions, e *events.Manager, plugins []pdv1.Plugin) (*Pi, error) { +func New(cfg *config.Config, pdc *pdclient.Client, udb user.Database, s *sessions.Sessions, e *events.Manager, m *mail.Client, plugins []pdv1.Plugin) (*Pi, error) { // Parse plugin settings var ( textFileSizeMax uint32 @@ -159,6 +161,7 @@ func New(cfg *config.Config, pdc *pdclient.Client, udb user.Database, s *session userdb: udb, sessions: s, events: e, + mail: m, policy: &v1.PolicyReply{ TextFileSizeMax: textFileSizeMax, ImageFileCountMax: imageFileCountMax, diff --git a/politeiawww/pi/process.go b/politeiawww/pi/process.go index 6dc772f60..9a714882c 100644 --- a/politeiawww/pi/process.go +++ b/politeiawww/pi/process.go @@ -17,6 +17,7 @@ import ( "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/usermd" v1 "github.com/decred/politeia/politeiawww/api/pi/v1" + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" "github.com/decred/politeia/politeiawww/user" "github.com/google/uuid" ) @@ -126,10 +127,10 @@ func (p *Pi) processProposals(ctx context.Context, ps v1.Proposals, u *user.User }, nil } -// proposalName parses the proposal name from the ProposalMetadata and returns -// it. An empty string will be returned if any errors occur or if a name is not -// found. -func proposalName(r pdv1.Record) string { +// proposalName parses the proposal name from the ProposalMetadata file and +// returns it. An empty string will be returned if any errors occur or if a +// name is not found. +func proposalName(r rcv1.Record) string { var name string for _, v := range r.Files { if v.Name == pi.FileNameProposalMetadata { diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 895ab4b0b..846ee7cc0 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -196,7 +196,7 @@ func (p *politeiawww) setupPi(plugins []pdv1.Plugin) error { return fmt.Errorf("new ticketvote api: %v", err) } piCtx, err := pi.New(p.cfg, p.politeiad, p.db, - p.sessions, p.events, plugins) + p.sessions, p.events, p.mail, plugins) if err != nil { return fmt.Errorf("new pi api: %v", err) } diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index 996c4fc48..7a7dcd235 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -72,7 +72,7 @@ func (r *Records) processNew(ctx context.Context, n v1.New, u user.User) (*v1.Ne log.Infof("Record submitted: %v", rc.CensorshipRecord.Token) for k, f := range rc.Files { - log.Infof("%02v: %v", k, f.Name) + log.Debugf("%02v: %v", k, f.Name) } // Execute post plugin hooks. Checking the mode is a temporary @@ -197,7 +197,7 @@ func (r *Records) processEdit(ctx context.Context, e v1.Edit, u user.User) (*v1. log.Infof("Record edited: %v %v", e.State, rc.CensorshipRecord.Token) for k, f := range rc.Files { - log.Infof("%02v: %v", k, f.Name) + log.Debugf("%02v: %v", k, f.Name) } // Emit event diff --git a/politeiawww/templates.go b/politeiawww/templates.go index 2a64ca01d..5392e3381 100644 --- a/politeiawww/templates.go +++ b/politeiawww/templates.go @@ -6,41 +6,6 @@ package main import "text/template" -// Proposal submitted - Send to admins -type proposalSubmitted struct { - Username string // Author username - Name string // Proposal name - Link string // GUI proposal details URL -} - -const proposalSubmittedText = ` -A new proposal has been submitted on Politeia by {{.Username}}: - -{{.Name}} -{{.Link}} -` - -var proposalSubmittedTmpl = template.Must( - template.New("proposalSubmitted").Parse(proposalSubmittedText)) - -// Proposal edited - Send to users -type proposalEdited struct { - Name string // Proposal name - Version string // ProposalVersion - Username string // Author username - Link string // GUI proposal details URL -} - -const proposalEditedText = ` -A proposal by {{.Username}} has just been edited: - -{{.Name}} (Version: {{.Version}}) -{{.Link}} -` - -var proposalEditedTmpl = template.Must( - template.New("proposalEdited").Parse(proposalEditedText)) - // Proposal status change - Vetted - Send to users type proposalVetted struct { Name string // Proposal name diff --git a/politeiawww/user/user.go b/politeiawww/user/user.go index 91c95dbcc..cb6e864a1 100644 --- a/politeiawww/user/user.go +++ b/politeiawww/user/user.go @@ -352,6 +352,16 @@ func (u *User) ActivateIdentity(key []byte) error { return nil } +// NotificationIsEnabled returns whether the user has the provided notification +// bit enabled. This function will always return false if the user has been +// deactivated. +func (u *User) NotificationIsEnabled(ntfnBit uint64) bool { + if u.Deactivated { + return false + } + return u.EmailNotifications&ntfnBit != 0 +} + // EncodeUser encodes User into a JSON byte slice. func EncodeUser(u User) ([]byte, error) { b, err := json.Marshal(u) From 4302665c3a18519f2ffea9f4d1ac2f84a92b0034 Mon Sep 17 00:00:00 2001 From: luke Date: Sat, 27 Feb 2021 12:17:00 -0600 Subject: [PATCH 348/449] pi: Get status change ntfns working. --- politeiawww/pi/events.go | 332 ++++++++++++++++++++++---------------- politeiawww/pi/mail.go | 127 +++++++++++++++ politeiawww/pi/process.go | 55 +------ politeiawww/templates.go | 62 ------- 4 files changed, 330 insertions(+), 246 deletions(-) diff --git a/politeiawww/pi/events.go b/politeiawww/pi/events.go index 06b00c5ae..0d27d004f 100644 --- a/politeiawww/pi/events.go +++ b/politeiawww/pi/events.go @@ -5,12 +5,22 @@ package pi import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "strings" + + "github.com/decred/politeia/politeiad/plugins/pi" + "github.com/decred/politeia/politeiad/plugins/usermd" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/comments" "github.com/decred/politeia/politeiawww/records" "github.com/decred/politeia/politeiawww/ticketvote" "github.com/decred/politeia/politeiawww/user" + "github.com/google/uuid" ) func (p *Pi) setupEventListeners() { @@ -165,76 +175,113 @@ func (p *Pi) handleEventRecordSetStatus(ch chan interface{}) { continue } - /* - // Check if proposal is in correct status for notification - switch d.status { - case rcv1.RecordStatusPublic, rcv1.RecordStatusCensored: - // The status requires a notification be sent - default: - // The status does not require a notification be sent. Listen - // for next event. - continue + // Unpack args + var ( + token = e.Record.CensorshipRecord.Token + status = e.Record.Status + reason = "" // Populated below + name = proposalName(e.Record) + authorID = userIDFromMetadata(e.Record.Metadata) + + author *user.User + uid uuid.UUID + ntfnBit = uint64(www.NotificationEmailRegularProposalVetted) + emails = make([]string, 0, 256) + ) + + sc, err := statusChangesFromMetadata(e.Record.Metadata) + if err != nil { + err = fmt.Errorf("decode status changes: %v", err) + goto ntfnFailed + } + if len(sc) == 0 { + err = fmt.Errorf("not status changes found %v", token) + goto ntfnFailed + } + reason = sc[len(sc)-1].Reason + + // Verify a notification should be sent + switch status { + case rcv1.RecordStatusPublic, rcv1.RecordStatusCensored: + // The status requires a notification be sent + default: + // The status does not require a notification be sent. Listen + // for next event. + log.Debugf("Record set status ntfn not needed for %v %v", + token, rcv1.RecordStatuses[status]) + continue + } + + // Send author notification + uid, err = uuid.Parse(authorID) + if err != nil { + goto ntfnFailed + } + author, err = p.userdb.UserGetById(uid) + if err != nil { + err = fmt.Errorf("UserGetById %v: %v", uid, err) + goto ntfnFailed + } + switch { + case !author.NotificationIsEnabled(ntfnBit): + // Author does not have notification enabled + log.Debugf("Record set status ntfn to author not enabled %v", token) + + default: + // Author does have notification enabled + err = p.mailNtfnProposalSetStatusToAuthor(token, name, + status, reason, author.Email) + if err != nil { + // Log the error and continue. This error should not prevent + // the other notifications from being sent. + log.Errorf("mailNtfnProposalSetStatusToAuthor: %v", err) + break } - // Get the proposal author - pr, err := p.proposalRecordLatest(context.Background(), d.state, d.token) - if err != nil { - log.Errorf("handleEventRecordSetStatus: proposalRecordLatest "+ - "%v %v: %v", d.state, d.token, err) - continue - } - author, err := p.db.UserGetByPubKey(pr.PublicKey) - if err != nil { - log.Errorf("handleEventRecordSetStatus: UserGetByPubKey %v: %v", - pr.PublicKey, err) - continue - } + log.Debugf("Record set status ntfn sent to author %v", token) + } - // Email author - proposalName := proposalName(*pr) - notification := www.NotificationEmailRegularProposalVetted - if userNotificationEnabled(*author, notification) { - err = p.emailRecordSetStatusToAuthor(d, proposalName, author.Email) - if err != nil { - log.Errorf("emailRecordSetStatusToAuthor: %v", err) - continue - } - } + // Only send a notification to non-author users if the proposal + // is being made public. + if status != rcv1.RecordStatusPublic { + log.Debugf("Record set status ntfn to users not needed for %v %v", + token, rcv1.RecordStatuses[status]) + continue + } - // Compile list of users to send the notification to - emails := make([]string, 0, 256) - err = p.db.AllUsers(func(u *user.User) { - switch { - case u.ID.String() == d.adminID: - // User is the admin that made the status change - return - case u.ID.String() == author.ID.String(): - // User is the author. The author is sent a different - // notification. Don't include them in the users list. - return - case !userNotificationEnabled(*u, notification): - // User does not have notification bit set - return - } - - // Add user to notification list - emails = append(emails, u.Email) - }) - if err != nil { - log.Errorf("handleEventRecordSetStatus: AllUsers: %v", err) - continue - } + // Compile user notification email list + err = p.userdb.AllUsers(func(u *user.User) { + switch { + case u.ID.String() == author.ID.String(): + // User is the author. The author is sent a different + // notification. Don't include them in the users list. + return + case !u.NotificationIsEnabled(ntfnBit): + // User does not have notification bit set + return + default: + // Add user to notification list + emails = append(emails, u.Email) + } + }) + if err != nil { + err = fmt.Errorf("AllUsers: %v", err) + goto ntfnFailed + } - // Email users - err = p.emailRecordSetStatus(d, proposalName, emails) - if err != nil { - log.Errorf("emailRecordSetStatus: %v", err) - continue - } - */ + // Send user notifications + err = p.mailNtfnProposalSetStatus(token, name, status, emails) + if err != nil { + err = fmt.Errorf("mailNtfnProposalSetStatus: %v", err) + goto ntfnFailed + } + + log.Debugf("Record set status ntfn sent to users %v", token) - log.Debugf("Record set status event sent %v", - e.Record.CensorshipRecord.Token) + ntfnFailed: + log.Errorf("handleEventRecordSetStatus %v %v: %v", + token, rcv1.RecordStatuses[status], err) + continue } } @@ -460,78 +507,6 @@ func (p *Pi) handleEventVoteStart(ch chan interface{}) { } /* -func (p *politeiawww) emailProposalStatusChange(d dataProposalStatusChange, proposalName string, emails []string) error { - route := strings.Replace(guiRouteProposalDetails, "{token}", d.token, 1) - l, err := url.Parse(p.cfg.WebServerAddress + route) - if err != nil { - return err - } - - var ( - subject string - body string - ) - switch d.status { - case rcv1.RecordStatusPublic: - subject = "New Proposal Published" - tmplData := proposalVetted{ - Name: proposalName, - Link: l.String(), - } - body, err = createBody(tmplProposalVetted, tmplData) - if err != nil { - return err - } - - default: - log.Debugf("no user notification for prop status %v", d.status) - return nil - } - - return p.smtp.sendEmailTo(subject, body, emails) -} - -func (p *politeiawww) emailProposalStatusChangeToAuthor(d dataProposalStatusChange, proposalName, authorEmail string) error { - route := strings.Replace(guiRouteProposalDetails, "{token}", d.token, 1) - l, err := url.Parse(p.cfg.WebServerAddress + route) - if err != nil { - return err - } - - var ( - subject string - body string - ) - switch d.status { - case rcv1.RecordStatusPublic: - subject = "Your Proposal Has Been Published" - tmplData := proposalVettedToAuthor{ - Name: proposalName, - Link: l.String(), - } - body, err = createBody(proposalVettedToAuthorTmpl, tmplData) - if err != nil { - return err - } - - case rcv1.RecordStatusCensored: - subject = "Your Proposal Has Been Censored" - tmplData := proposalCensoredToAuthor{ - Name: proposalName, - Reason: d.reason, - } - body, err = createBody(tmplProposalCensoredForAuthor, tmplData) - if err != nil { - return err - } - - default: - return fmt.Errorf("no author notification for prop status %v", d.status) - } - - return p.smtp.sendEmailTo(subject, body, []string{authorEmail}) -} - func (p *politeiawww) emailProposalCommentSubmitted(token, commentID, commentUsername, proposalName, proposalAuthorEmail string) error { // Setup comment URL route := strings.Replace(guirouteProposalComments, "{token}", token, 1) @@ -660,3 +635,88 @@ func (p *politeiawww) emailProposalVoteStartedToAuthor(token, name, username, em return p.smtp.sendEmailTo(subject, body, []string{email}) } */ + +// userMetadataDecode decodes and returns the UserMetadata from the provided +// metadata streams. If a UserMetadata is not found, nil is returned. +func userMetadataDecode(ms []rcv1.MetadataStream) (*usermd.UserMetadata, error) { + var userMD *usermd.UserMetadata + for _, v := range ms { + if v.ID == usermd.MDStreamIDUserMetadata { + var um usermd.UserMetadata + err := json.Unmarshal([]byte(v.Payload), &um) + if err != nil { + return nil, err + } + userMD = &um + break + } + } + return userMD, nil +} + +// userIDFromMetadata searches for a UserMetadata and parses the user ID from +// it if found. An empty string is returned if no UserMetadata is found. +func userIDFromMetadata(ms []rcv1.MetadataStream) string { + um, err := userMetadataDecode(ms) + if err != nil { + return "" + } + if um == nil { + return "" + } + return um.UserID +} + +// proposalName parses the proposal name from the ProposalMetadata file and +// returns it. An empty string will be returned if any errors occur or if a +// name is not found. +func proposalName(r rcv1.Record) string { + var name string + for _, v := range r.Files { + if v.Name == pi.FileNameProposalMetadata { + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return "" + } + var pm pi.ProposalMetadata + err = json.Unmarshal(b, &pm) + if err != nil { + return "" + } + name = pm.Name + } + } + return name +} + +func statusChangesDecode(payload []byte) ([]usermd.StatusChangeMetadata, error) { + statuses := make([]usermd.StatusChangeMetadata, 0, 16) + d := json.NewDecoder(strings.NewReader(string(payload))) + for { + var sc usermd.StatusChangeMetadata + err := d.Decode(&sc) + if errors.Is(err, io.EOF) { + break + } else if err != nil { + return nil, err + } + statuses = append(statuses, sc) + } + return statuses, nil +} + +func statusChangesFromMetadata(metadata []rcv1.MetadataStream) ([]usermd.StatusChangeMetadata, error) { + var ( + sc []usermd.StatusChangeMetadata + err error + ) + for _, v := range metadata { + if v.ID == usermd.MDStreamIDStatusChanges { + sc, err = statusChangesDecode([]byte(v.Payload)) + if err != nil { + return nil, err + } + } + } + return sc, nil +} diff --git a/politeiawww/pi/mail.go b/politeiawww/pi/mail.go index 98d496446..614a81eef 100644 --- a/politeiawww/pi/mail.go +++ b/politeiawww/pi/mail.go @@ -6,9 +6,12 @@ package pi import ( "bytes" + "fmt" "net/url" "strings" "text/template" + + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" ) const ( @@ -97,6 +100,130 @@ func (p *Pi) mailNtfnProposalEdit(token, version, name, username string, emails return p.mail.SendTo(subject, body, emails) } +type proposalPublished struct { + Name string // Proposal name + Link string // GUI proposal details URL +} + +var proposalPublishedTmpl = template.Must( + template.New("proposalPublished").Parse(proposalPublishedText)) + +var proposalPublishedText = ` +A new proposal has just been published on Politeia. + +{{.Name}} +{{.Link}} +` + +func (p *Pi) mailNtfnProposalSetStatus(token, name string, status rcv1.RecordStatusT, emails []string) error { + route := strings.Replace(guiRouteRecordDetails, "{token}", token, 1) + u, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err + } + + var ( + subject string + body string + ) + switch status { + case rcv1.RecordStatusPublic: + subject = "New Proposal Published" + tmplData := proposalPublished{ + Name: name, + Link: u.String(), + } + body, err = populateTemplate(proposalPublishedTmpl, tmplData) + if err != nil { + return err + } + + default: + return fmt.Errorf("no mail ntfn for status %v", status) + } + + return p.mail.SendTo(subject, body, emails) +} + +type proposalPublishedToAuthor struct { + Name string // Proposal name + Link string // GUI proposal details URL +} + +var proposalPublishedToAuthorText = ` +Your proposal has just been made public on Politeia! + +Your proposal has now entered the discussion phase where the community can leave comments and provide feedback. Be sure to keep an eye out for new comments and to answer any questions that the community may have. You can edit your proposal at any point prior to the start of voting. + +Once you feel that enough time has been given for discussion you may authorize the vote to commence on your proposal. An admin is not able to start the voting process until you explicitly authorize it. You can authorize a proposal vote by opening the proposal page and clicking on the authorize vote button. + +{{.Name}} +{{.Link}} + +If you have any questions, drop by the proposals channel on matrix. +https://chat.decred.org/#/room/#proposals:decred.org +` +var proposalPublishedToAuthorTmpl = template.Must( + template.New("proposalPublishedToAuthor"). + Parse(proposalPublishedToAuthorText)) + +type proposalCensoredToAuthor struct { + Name string // Proposal name + Reason string // Reason for censoring +} + +var proposalCensoredToAuthorText = ` +Your proposal on Politeia has been censored. + +{{.Name}} +Reason: {{.Reason}} +` + +var proposalCensoredToAuthorTmpl = template.Must( + template.New("proposalCensoredToAuthor"). + Parse(proposalCensoredToAuthorText)) + +func (p *Pi) mailNtfnProposalSetStatusToAuthor(token, name string, status rcv1.RecordStatusT, reason, authorEmail string) error { + route := strings.Replace(guiRouteRecordDetails, "{token}", token, 1) + u, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err + } + + var ( + subject string + body string + ) + switch status { + case rcv1.RecordStatusPublic: + subject = "Your Proposal Has Been Published" + tmplData := proposalPublishedToAuthor{ + Name: name, + Link: u.String(), + } + body, err = populateTemplate(proposalPublishedToAuthorTmpl, tmplData) + if err != nil { + return err + } + + case rcv1.RecordStatusCensored: + subject = "Your Proposal Has Been Censored" + tmplData := proposalCensoredToAuthor{ + Name: name, + Reason: reason, + } + body, err = populateTemplate(proposalCensoredToAuthorTmpl, tmplData) + if err != nil { + return err + } + + default: + return fmt.Errorf("no author notification for prop status %v", status) + } + + return p.mail.SendTo(subject, body, []string{authorEmail}) +} + func populateTemplate(tpl *template.Template, tplData interface{}) (string, error) { var buf bytes.Buffer err := tpl.Execute(&buf, tplData) diff --git a/politeiawww/pi/process.go b/politeiawww/pi/process.go index 9a714882c..c986b8ac6 100644 --- a/politeiawww/pi/process.go +++ b/politeiawww/pi/process.go @@ -6,18 +6,12 @@ package pi import ( "context" - "encoding/base64" "encoding/json" - "errors" "fmt" - "io" - "strings" pdv1 "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/usermd" v1 "github.com/decred/politeia/politeiawww/api/pi/v1" - rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" "github.com/decred/politeia/politeiawww/user" "github.com/google/uuid" ) @@ -51,8 +45,11 @@ func (p *Pi) proposals(ctx context.Context, state string, reqs []pdv1.RecordRequ } // Fill in user data - userID := userIDFromMetadataStreams(v.Metadata) + userID := userIDFromMetadataStreamsPD(v.Metadata) uid, err := uuid.Parse(userID) + if err != nil { + return nil, err + } u, err := p.userdb.UserGetById(uid) if err != nil { return nil, err @@ -127,28 +124,6 @@ func (p *Pi) processProposals(ctx context.Context, ps v1.Proposals, u *user.User }, nil } -// proposalName parses the proposal name from the ProposalMetadata file and -// returns it. An empty string will be returned if any errors occur or if a -// name is not found. -func proposalName(r rcv1.Record) string { - var name string - for _, v := range r.Files { - if v.Name == pi.FileNameProposalMetadata { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return "" - } - var pm pi.ProposalMetadata - err = json.Unmarshal(b, &pm) - if err != nil { - return "" - } - name = pm.Name - } - } - return name -} - // proposalPopulateUserData populates a proposal with user data that is stored // in the user database and not in politeiad. func proposalPopulateUserData(pr *v1.Proposal, u user.User) { @@ -173,22 +148,6 @@ func convertStatus(s pdv1.RecordStatusT) v1.PropStatusT { return v1.PropStatusInvalid } -func statusChangesDecode(payload []byte) ([]usermd.StatusChangeMetadata, error) { - statuses := make([]usermd.StatusChangeMetadata, 0, 16) - d := json.NewDecoder(strings.NewReader(string(payload))) - for { - var sc usermd.StatusChangeMetadata - err := d.Decode(&sc) - if errors.Is(err, io.EOF) { - break - } else if err != nil { - return nil, err - } - statuses = append(statuses, sc) - } - return statuses, nil -} - func convertRecord(r pdv1.Record, state string) (*v1.Proposal, error) { // Decode metadata streams var ( @@ -260,7 +219,7 @@ func convertRecord(r pdv1.Record, state string) (*v1.Proposal, error) { // userMetadataDecode decodes and returns the UserMetadata from the provided // metadata streams. If a UserMetadata is not found, nil is returned. -func userMetadataDecode(ms []pdv1.MetadataStream) (*usermd.UserMetadata, error) { +func userMetadataDecodePD(ms []pdv1.MetadataStream) (*usermd.UserMetadata, error) { var userMD *usermd.UserMetadata for _, v := range ms { if v.ID == usermd.MDStreamIDUserMetadata { @@ -278,8 +237,8 @@ func userMetadataDecode(ms []pdv1.MetadataStream) (*usermd.UserMetadata, error) // userIDFromMetadataStreams searches for a UserMetadata and parses the user ID // from it if found. An empty string is returned if no UserMetadata is found. -func userIDFromMetadataStreams(ms []pdv1.MetadataStream) string { - um, err := userMetadataDecode(ms) +func userIDFromMetadataStreamsPD(ms []pdv1.MetadataStream) string { + um, err := userMetadataDecodePD(ms) if err != nil { return "" } diff --git a/politeiawww/templates.go b/politeiawww/templates.go index 5392e3381..fb91d6e92 100644 --- a/politeiawww/templates.go +++ b/politeiawww/templates.go @@ -6,68 +6,6 @@ package main import "text/template" -// Proposal status change - Vetted - Send to users -type proposalVetted struct { - Name string // Proposal name - Link string // GUI proposal details URL -} - -const proposalVettedText = ` -A new proposal has just been published on Politeia. - -{{.Name}} -{{.Link}} -` - -var tmplProposalVetted = template.Must( - template.New("proposalVetted").Parse(proposalVettedText)) - -// Proposal status change - Vetted - Send to author -type proposalVettedToAuthor struct { - Name string // Proposal name - Link string // GUI proposal details URL -} - -const proposalVettedToAuthorText = ` -Your proposal has just been made public on Politeia! - -Your proposal has now entered the discussion phase where the community can -leave comments and provide feedback. Be sure to keep an eye out for new -comments and to answer any questions that the community may have. You are -allowed to edit your proposal at any point prior to the start of voting. - -Once you feel that enough time has been given for discussion you may authorize -the vote to commence on your proposal. An admin is not able to start the -voting process until you explicitly authorize it. You can authorize a proposal -vote by opening the proposal page and clicking on the "Authorize Voting to -Start" button. - -{{.Name}} -{{.Link}} - -If you have any questions, drop by the proposals channel on matrix. -https://chat.decred.org/#/room/#proposals:decred.org -` - -var proposalVettedToAuthorTmpl = template.Must( - template.New("proposalVettedToAuthor").Parse(proposalVettedToAuthorText)) - -// Proposal status change - Censored - Send to author -type proposalCensoredToAuthor struct { - Name string // Proposal name - Reason string // Reason for censoring -} - -const proposalCensoredToAuthorText = ` -Your proposal on Politeia has been censored. - -{{.Name}} -Reason: {{.Reason}} -` - -var tmplProposalCensoredForAuthor = template.Must( - template.New("proposalCensoredToAuthor").Parse(proposalCensoredToAuthorText)) - // Proposal comment submitted - Send to proposal author type proposalCommentSubmitted struct { Username string // Comment author username From 7e1b4983e0d506a9c44b5b5a4a62e1f8fd0ffdba Mon Sep 17 00:00:00 2001 From: luke Date: Sat, 27 Feb 2021 18:01:09 -0600 Subject: [PATCH 349/449] pi: Get comment ntfns working. --- politeiad/client/comments.go | 46 ++++- politeiawww/comments/process.go | 2 +- politeiawww/pi/events.go | 343 +++++++++++++++++++------------- politeiawww/pi/mail.go | 80 ++++++++ politeiawww/records/process.go | 28 ++- politeiawww/templates.go | 34 ---- 6 files changed, 341 insertions(+), 192 deletions(-) diff --git a/politeiad/client/comments.go b/politeiad/client/comments.go index 98dedaf63..243b1011e 100644 --- a/politeiad/client/comments.go +++ b/politeiad/client/comments.go @@ -188,9 +188,51 @@ func (c *Client) CommentCount(ctx context.Context, state string, tokens []string return counts, nil } -// CommentGetAll sends the comments plugin GetAll command to the politeiad v1 +// CommentsGet sends the comments plugin Get command to the politeiad v1 API. +func (c *Client) CommentsGet(ctx context.Context, state, token string, g comments.Get) (map[uint32]comments.Comment, error) { + // Setup request + b, err := json.Marshal(g) + if err != nil { + return nil, err + } + cmds := []pdv1.PluginCommandV2{ + { + Action: pdv1.PluginActionRead, + State: state, + Token: token, + ID: comments.PluginID, + Command: comments.CmdGet, + Payload: string(b), + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var gr comments.GetReply + err = json.Unmarshal([]byte(pcr.Payload), &gr) + if err != nil { + return nil, err + } + + return gr.Comments, nil +} + +// CommentsGetAll sends the comments plugin GetAll command to the politeiad v1 // API. -func (c *Client) CommentGetAll(ctx context.Context, state, token string) ([]comments.Comment, error) { +func (c *Client) CommentsGetAll(ctx context.Context, state, token string) ([]comments.Comment, error) { // Setup request cmds := []pdv1.PluginCommandV2{ { diff --git a/politeiawww/comments/process.go b/politeiawww/comments/process.go index c57e9581c..0e082e421 100644 --- a/politeiawww/comments/process.go +++ b/politeiawww/comments/process.go @@ -211,7 +211,7 @@ func (c *Comments) processComments(ctx context.Context, cs v1.Comments, u *user. } // Send plugin command - pcomments, err := c.politeiad.CommentGetAll(ctx, cs.State, cs.Token) + pcomments, err := c.politeiad.CommentsGetAll(ctx, cs.State, cs.Token) if err != nil { return nil, err } diff --git a/politeiawww/pi/events.go b/politeiawww/pi/events.go index 0d27d004f..49e7e5efb 100644 --- a/politeiawww/pi/events.go +++ b/politeiawww/pi/events.go @@ -5,6 +5,7 @@ package pi import ( + "context" "encoding/base64" "encoding/json" "errors" @@ -12,8 +13,12 @@ import ( "io" "strings" + pdv1 "github.com/decred/politeia/politeiad/api/v1" + cmplugin "github.com/decred/politeia/politeiad/plugins/comments" "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/usermd" + cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" + v1 "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/comments" @@ -98,7 +103,7 @@ func (p *Pi) handleEventRecordNew(ch chan interface{}) { // Send notfication email var ( token = e.Record.CensorshipRecord.Token - name = proposalName(e.Record) + name = proposalNameFromRecord(e.Record) ) err = p.mailNtfnProposalNew(token, name, e.User.Username, emails) if err != nil { @@ -154,7 +159,7 @@ func (p *Pi) handleEventRecordEdit(ch chan interface{}) { var ( token = e.Record.CensorshipRecord.Token version = e.Record.Version - name = proposalName(e.Record) + name = proposalNameFromRecord(e.Record) username = e.User.Username ) err = p.mailNtfnProposalEdit(token, version, name, username, emails) @@ -180,7 +185,7 @@ func (p *Pi) handleEventRecordSetStatus(ch chan interface{}) { token = e.Record.CensorshipRecord.Token status = e.Record.Status reason = "" // Populated below - name = proposalName(e.Record) + name = proposalNameFromRecord(e.Record) authorID = userIDFromMetadata(e.Record.Metadata) author *user.User @@ -192,11 +197,11 @@ func (p *Pi) handleEventRecordSetStatus(ch chan interface{}) { sc, err := statusChangesFromMetadata(e.Record.Metadata) if err != nil { err = fmt.Errorf("decode status changes: %v", err) - goto ntfnFailed + goto failed } if len(sc) == 0 { err = fmt.Errorf("not status changes found %v", token) - goto ntfnFailed + goto failed } reason = sc[len(sc)-1].Reason @@ -215,12 +220,12 @@ func (p *Pi) handleEventRecordSetStatus(ch chan interface{}) { // Send author notification uid, err = uuid.Parse(authorID) if err != nil { - goto ntfnFailed + goto failed } author, err = p.userdb.UserGetById(uid) if err != nil { err = fmt.Errorf("UserGetById %v: %v", uid, err) - goto ntfnFailed + goto failed } switch { case !author.NotificationIsEnabled(ntfnBit): @@ -266,145 +271,175 @@ func (p *Pi) handleEventRecordSetStatus(ch chan interface{}) { }) if err != nil { err = fmt.Errorf("AllUsers: %v", err) - goto ntfnFailed + goto failed } // Send user notifications err = p.mailNtfnProposalSetStatus(token, name, status, emails) if err != nil { err = fmt.Errorf("mailNtfnProposalSetStatus: %v", err) - goto ntfnFailed + goto failed } log.Debugf("Record set status ntfn sent to users %v", token) + continue - ntfnFailed: + failed: log.Errorf("handleEventRecordSetStatus %v %v: %v", token, rcv1.RecordStatuses[status], err) continue } } -/* -func (p *Pi) notifyProposalAuthorOnComment(d dataCommentNew, userID, proposalName string) error { - // Lookup proposal author to see if they should be sent a - // notification. - uuid, err := uuid.Parse(userID) +func (p *Pi) ntfnCommentNewProposalAuthor(c cmv1.Comment, proposalAuthorID, proposalName string) error { + // Get the proposal author + uid, err := uuid.Parse(proposalAuthorID) if err != nil { return err } - author, err := p.db.UserGetById(uuid) + pauthor, err := p.userdb.UserGetById(uid) if err != nil { - return fmt.Errorf("UserGetByID %v: %v", uuid.String(), err) + return fmt.Errorf("UserGetByID %v: %v", uid.String(), err) } - // Check if notification should be sent to author + // Check if notification should be sent + ntfnBit := uint64(www.NotificationEmailCommentOnMyProposal) switch { - case d.username == author.Username: + case c.Username == pauthor.Username: // Author commented on their own proposal + log.Debugf("Comment ntfn to proposal author not needed %v", c.Token) return nil - case !userNotificationEnabled(*author, - www.NotificationEmailCommentOnMyProposal): + case !pauthor.NotificationIsEnabled(ntfnBit): // Author does not have notification bit set on + log.Debugf("Comment ntfn to proposal author not enabled %v", c.Token) return nil } - // Send notification eamil - commentID := strconv.FormatUint(uint64(d.commentID), 10) - return p.emailCommentNewSubmitted(d.token, commentID, d.username, - proposalName, author.Email) + // Send notification email + err = p.mailNtfnCommentNewToProposalAuthor(c.Token, c.CommentID, + c.Username, proposalName, pauthor.Email) + if err != nil { + return err + } + + log.Debugf("Comment new ntfn to proposal author sent %v", c.Token) + + return nil } -func (p *Pi) notifyParentAuthorOnComment(d dataCommentNew, proposalName string) error { - // Verify this is a reply comment - if d.parentID == 0 { +func (p *Pi) ntfnCommentReply(state string, c cmv1.Comment, proposalName string) error { + // Verify there is work to do. This notification only applies to + // reply comments. + if c.ParentID == 0 { + log.Debugf("Comment reply ntfn not needed %v", c.Token) return nil } - // Lookup the parent comment author to check if they should receive - // a reply notification. - g := comments.Get{ - CommentIDs: []uint32{d.parentID}, + // Get the parent comment author + g := cmplugin.Get{ + CommentIDs: []uint32{c.ParentID}, } - _ = g - var parentComment comments.GetReply - parentComment, err := p.commentsGet(context.Background(), g) - if err != nil { - return err - } - userID, err := uuid.Parse(parentComment.Comments[0].UserID) + cs, err := p.politeiad.CommentsGet(context.Background(), state, c.Token, g) if err != nil { return err } - author, err := p.db.UserGetById(userID) + parent, ok := cs[c.ParentID] + if !ok { + return fmt.Errorf("parent comment %v not found", c.ParentID) + } + userID, err := uuid.Parse(parent.UserID) + if err != nil { + return err + } + pauthor, err := p.userdb.UserGetById(userID) if err != nil { return err } // Check if notification should be sent + ntfnBit := uint64(www.NotificationEmailCommentOnMyComment) switch { - case d.username == author.Username: + case c.UserID == pauthor.ID.String(): // Author replied to their own comment + log.Debugf("Comment reply ntfn not needed %v", c.Token) return nil - case !userNotificationEnabled(*author, - www.NotificationEmailCommentOnMyComment): + case !pauthor.NotificationIsEnabled(ntfnBit): // Author does not have notification bit set on + log.Debugf("Comment reply ntfn not enabled %v", c.Token) return nil } - // Send notification email to parent comment author - commentID := strconv.FormatUint(uint64(d.commentID), 10) - return p.emailCommentNewReply(d.token, commentID, d.username, - proposalName, author.Email) + // Send notification email + err = p.mailNtfnCommentReply(c.Token, c.CommentID, + c.Username, proposalName, pauthor.Email) + if err != nil { + return err + } + + log.Debugf("Comment reply ntfn sent %v", c.Token) + + return nil } -*/ func (p *Pi) handleEventCommentNew(ch chan interface{}) { for msg := range ch { - d, ok := msg.(comments.EventNew) + e, ok := msg.(comments.EventNew) if !ok { log.Errorf("handleEventCommentNew invalid msg: %v", msg) continue } - _ = d - /* - // Fetch the proposal record here to avoid calling this two times - // on the notify functions below - pr, err := p.proposalRecordLatest(context.Background(), d.state, - d.token) - if err != nil { - err = fmt.Errorf("proposalRecordLatest %v %v: %v", - d.state, d.token, err) - goto next - } - - // Notify the proposal author - err = p.notifyProposalAuthorOnComment(d, pr.UserID, proposalName(*pr)) - if err != nil { - err = fmt.Errorf("notifyProposalAuthorOnComment: %v", err) - goto next - } + // Get the record author and record name + var ( + pdr pdv1.Record + r rcv1.Record + proposalAuthorID string + proposalName string + ) + reqs := []pdv1.RecordRequest{ + { + Token: e.Comment.Token, + Filenames: []string{ + v1.FileNameProposalMetadata, + }, + }, + } + rs, err := p.records(e.State, reqs) + if err != nil { + err = fmt.Errorf("politeiad records: %v", err) + goto failed + } + pdr, ok = rs[e.Comment.Token] + if !ok { + err = fmt.Errorf("record %v not found for comment %v", + e.Comment.Token, e.Comment.CommentID) + goto failed + } + r = convertRecordToV1(pdr, e.State) + proposalAuthorID = userIDFromMetadata(r.Metadata) + proposalName = proposalNameFromRecord(r) - // Notify the parent comment author - err = p.notifyParentAuthorOnComment(d, proposalName(*pr)) - if err != nil { - err = fmt.Errorf("notifyParentAuthorOnComment: %v", err) - goto next - } + // Notify the proposal author + err = p.ntfnCommentNewProposalAuthor(e.Comment, + proposalAuthorID, proposalName) + if err != nil { + // Log error and continue so the other notifications are still + // sent. + log.Errorf("ntfnCommentNewProposalAuthor: %v", err) + } - // Notifications successfully sent - log.Debugf("Sent proposal commment notification %v", d.token) - continue + // Notify the parent comment author + err = p.ntfnCommentReply(e.State, e.Comment, proposalName) + if err != nil { + log.Errorf("ntfnCommentReply: %v", err) + } - next: - // If we made it here then there was an error. Log the error - // before listening for the next event. - log.Errorf("handleEventCommentNew: %v", err) - continue - */ + // Notifications sent! + continue - log.Debugf("Proposal comment ntfn sent %v", d.Comment.Token) + failed: + log.Errorf("handleEventCommentNew: %v", err) + continue } } @@ -507,58 +542,6 @@ func (p *Pi) handleEventVoteStart(ch chan interface{}) { } /* -func (p *politeiawww) emailProposalCommentSubmitted(token, commentID, commentUsername, proposalName, proposalAuthorEmail string) error { - // Setup comment URL - route := strings.Replace(guirouteProposalComments, "{token}", token, 1) - route = strings.Replace(route, "{id}", commentID, 1) - l, err := url.Parse(p.cfg.WebServerAddress + route) - if err != nil { - return err - } - - // Setup email - subject := "New Comment On Your Proposal" - tmplData := proposalCommentSubmitted{ - Username: commentUsername, - Name: proposalName, - Link: l.String(), - } - body, err := createBody(proposalCommentSubmittedTmpl, tmplData) - if err != nil { - return err - } - - // Send email - return p.smtp.sendEmailTo(subject, body, []string{proposalAuthorEmail}) -} - -// emailProposalCommentReply sends a proposal comment reply email to the -// provided email address. -func (p *politeiawww) emailProposalCommentReply(token, commentID, commentUsername, proposalName, parentCommentEmail string) error { - // Setup comment URL - route := strings.Replace(guirouteProposalComments, "{token}", token, 1) - route = strings.Replace(route, "{id}", commentID, 1) - l, err := url.Parse(p.cfg.WebServerAddress + route) - if err != nil { - return err - } - - // Setup email - subject := "New Reply To Your Comment" - tmplData := proposalCommentReply{ - Username: commentUsername, - Name: proposalName, - Link: l.String(), - } - body, err := createBody(proposalCommentReplyTmpl, tmplData) - if err != nil { - return err - } - - // Send email - return p.smtp.sendEmailTo(subject, body, []string{parentCommentEmail}) -} - // emailProposalVoteAuthorized sends a proposal vote authorized email to the // provided list of emails. func (p *politeiawww) emailProposalVoteAuthorized(token, name, username string, emails []string) error { @@ -636,6 +619,29 @@ func (p *politeiawww) emailProposalVoteStartedToAuthor(token, name, username, em } */ +func (p *Pi) records(state string, reqs []pdv1.RecordRequest) (map[string]pdv1.Record, error) { + var ( + records map[string]pdv1.Record + err error + ) + switch state { + case rcv1.RecordStateUnvetted: + records, err = p.politeiad.GetUnvettedBatch(context.Background(), reqs) + if err != nil { + return nil, err + } + case rcv1.RecordStateVetted: + records, err = p.politeiad.GetVettedBatch(context.Background(), reqs) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("invalid state %v", state) + } + + return records, nil +} + // userMetadataDecode decodes and returns the UserMetadata from the provided // metadata streams. If a UserMetadata is not found, nil is returned. func userMetadataDecode(ms []rcv1.MetadataStream) (*usermd.UserMetadata, error) { @@ -667,10 +673,10 @@ func userIDFromMetadata(ms []rcv1.MetadataStream) string { return um.UserID } -// proposalName parses the proposal name from the ProposalMetadata file and -// returns it. An empty string will be returned if any errors occur or if a -// name is not found. -func proposalName(r rcv1.Record) string { +// proposalNameFromRecord parses the proposal name from the ProposalMetadata +// file and returns it. An empty string will be returned if any errors occur or +// if a name is not found. +func proposalNameFromRecord(r rcv1.Record) string { var name string for _, v := range r.Files { if v.Name == pi.FileNameProposalMetadata { @@ -720,3 +726,62 @@ func statusChangesFromMetadata(metadata []rcv1.MetadataStream) ([]usermd.StatusC } return sc, nil } + +func convertStatusToV1(s pdv1.RecordStatusT) rcv1.RecordStatusT { + switch s { + case pdv1.RecordStatusNotReviewed: + return rcv1.RecordStatusUnreviewed + case pdv1.RecordStatusPublic: + return rcv1.RecordStatusPublic + case pdv1.RecordStatusCensored: + return rcv1.RecordStatusCensored + case pdv1.RecordStatusArchived: + return rcv1.RecordStatusArchived + } + return rcv1.RecordStatusInvalid +} + +func convertFilesToV1(f []pdv1.File) []rcv1.File { + files := make([]rcv1.File, 0, len(f)) + for _, v := range f { + files = append(files, rcv1.File{ + Name: v.Name, + MIME: v.MIME, + Digest: v.Digest, + Payload: v.Payload, + }) + } + return files +} + +func convertMetadataStreamsToV1(ms []pdv1.MetadataStream) []rcv1.MetadataStream { + metadata := make([]rcv1.MetadataStream, 0, len(ms)) + for _, v := range ms { + metadata = append(metadata, rcv1.MetadataStream{ + PluginID: v.PluginID, + ID: v.ID, + Payload: v.Payload, + }) + } + return metadata +} + +func convertRecordToV1(r pdv1.Record, state string) rcv1.Record { + // User fields that are not part of the politeiad record have + // been intentionally left blank. These fields must be pulled + // from the user database. + return rcv1.Record{ + State: state, + Status: convertStatusToV1(r.Status), + Version: r.Version, + Timestamp: r.Timestamp, + Username: "", // Intentionally left blank + Metadata: convertMetadataStreamsToV1(r.Metadata), + Files: convertFilesToV1(r.Files), + CensorshipRecord: rcv1.CensorshipRecord{ + Token: r.CensorshipRecord.Token, + Merkle: r.CensorshipRecord.Merkle, + Signature: r.CensorshipRecord.Signature, + }, + } +} diff --git a/politeiawww/pi/mail.go b/politeiawww/pi/mail.go index 614a81eef..f22a3cc0c 100644 --- a/politeiawww/pi/mail.go +++ b/politeiawww/pi/mail.go @@ -8,6 +8,7 @@ import ( "bytes" "fmt" "net/url" + "strconv" "strings" "text/template" @@ -224,6 +225,85 @@ func (p *Pi) mailNtfnProposalSetStatusToAuthor(token, name string, status rcv1.R return p.mail.SendTo(subject, body, []string{authorEmail}) } +type commentNewToProposalAuthor struct { + Username string // Comment author username + Name string // Proposal name + Link string // Comment link +} + +const commentNewToProposalAuthorText = ` +{{.Username}} has commented on your proposal "{{.Name}}". + +{{.Link}} +` + +var commentNewToProposalAuthorTmpl = template.Must( + template.New("commentNewToProposalAuthor"). + Parse(commentNewToProposalAuthorText)) + +func (p *Pi) mailNtfnCommentNewToProposalAuthor(token string, commentID uint32, commentUsername, proposalName, proposalAuthorEmail string) error { + cid := strconv.FormatUint(uint64(commentID), 10) + route := strings.Replace(guiRouteRecordComment, "{token}", token, 1) + route = strings.Replace(route, "{id}", cid, 1) + + u, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err + } + + subject := "New Comment On Your Proposal" + tmplData := commentNewToProposalAuthor{ + Username: commentUsername, + Name: proposalName, + Link: u.String(), + } + body, err := populateTemplate(commentNewToProposalAuthorTmpl, tmplData) + if err != nil { + return err + } + + return p.mail.SendTo(subject, body, []string{proposalAuthorEmail}) +} + +type commentReply struct { + Username string // Comment author username + Name string // Proposal name + Link string // Comment link +} + +const commentReplyText = ` +{{.Username}} has replied to your comment on "{{.Name}}". + +{{.Link}} +` + +var commentReplyTmpl = template.Must( + template.New("commentReply").Parse(commentReplyText)) + +func (p *Pi) mailNtfnCommentReply(token string, commentID uint32, commentUsername, proposalName, parentAuthorEmail string) error { + cid := strconv.FormatUint(uint64(commentID), 10) + route := strings.Replace(guiRouteRecordComment, "{token}", token, 1) + route = strings.Replace(route, "{id}", cid, 1) + + u, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err + } + + subject := "New Reply To Your Comment" + tmplData := commentReply{ + Username: commentUsername, + Name: proposalName, + Link: u.String(), + } + body, err := populateTemplate(commentReplyTmpl, tmplData) + if err != nil { + return err + } + + return p.mail.SendTo(subject, body, []string{parentAuthorEmail}) +} + func populateTemplate(tpl *template.Template, tplData interface{}) (string, error) { var buf bytes.Buffer err := tpl.Execute(&buf, tplData) diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index 7a7dcd235..ce945fbf1 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -676,27 +676,23 @@ func convertMetadataStreamsToV1(ms []pdv1.MetadataStream) []v1.MetadataStream { return metadata } -func convertCensorshipRecordToV1(cr pdv1.CensorshipRecord) v1.CensorshipRecord { - return v1.CensorshipRecord{ - Token: cr.Token, - Merkle: cr.Merkle, - Signature: cr.Signature, - } -} - func convertRecordToV1(r pdv1.Record, state string) v1.Record { // User fields that are not part of the politeiad record have // been intentionally left blank. These fields must be pulled // from the user database. return v1.Record{ - State: state, - Status: convertStatusToV1(r.Status), - Version: r.Version, - Timestamp: r.Timestamp, - Username: "", // Intentionally left blank - Metadata: convertMetadataStreamsToV1(r.Metadata), - Files: convertFilesToV1(r.Files), - CensorshipRecord: convertCensorshipRecordToV1(r.CensorshipRecord), + State: state, + Status: convertStatusToV1(r.Status), + Version: r.Version, + Timestamp: r.Timestamp, + Username: "", // Intentionally left blank + Metadata: convertMetadataStreamsToV1(r.Metadata), + Files: convertFilesToV1(r.Files), + CensorshipRecord: v1.CensorshipRecord{ + Token: r.CensorshipRecord.Token, + Merkle: r.CensorshipRecord.Merkle, + Signature: r.CensorshipRecord.Signature, + }, } } diff --git a/politeiawww/templates.go b/politeiawww/templates.go index fb91d6e92..367856b45 100644 --- a/politeiawww/templates.go +++ b/politeiawww/templates.go @@ -6,40 +6,6 @@ package main import "text/template" -// Proposal comment submitted - Send to proposal author -type proposalCommentSubmitted struct { - Username string // Comment author username - Name string // Proposal name - Link string // Comment link -} - -const proposalCommentSubmittedText = ` -{{.Username}} has commented on your proposal! - -Proposal: {{.Name}} -Comment: {{.Link}} -` - -var proposalCommentSubmittedTmpl = template.Must( - template.New("proposalCommentSubmitted").Parse(proposalCommentSubmittedText)) - -// Proposal comment reply - Send to parent comment author -type proposalCommentReply struct { - Username string // Comment author username - Name string // Proposal name - Link string // Comment link -} - -const proposalCommentReplyText = ` -{{.Username}} has replied to your comment! - -Proposal: {{.Name}} -Comment: {{.Link}} -` - -var proposalCommentReplyTmpl = template.Must( - template.New("proposalCommentReply").Parse(proposalCommentReplyText)) - // Proposal vote authorized - Send to admins type proposalVoteAuthorized struct { Username string // Author username From 567e6ae47ec9861bf443a206ba88d7cc810eb0f6 Mon Sep 17 00:00:00 2001 From: luke Date: Sat, 27 Feb 2021 19:17:53 -0600 Subject: [PATCH 350/449] pi: Get vote ntfns working. --- politeiawww/pi/events.go | 184 +++++++++++++++++++++------------------ politeiawww/pi/mail.go | 47 ++++++++-- politeiawww/templates.go | 17 ---- 3 files changed, 138 insertions(+), 110 deletions(-) diff --git a/politeiawww/pi/events.go b/politeiawww/pi/events.go index 49e7e5efb..5becc4830 100644 --- a/politeiawww/pi/events.go +++ b/politeiawww/pi/events.go @@ -15,11 +15,11 @@ import ( pdv1 "github.com/decred/politeia/politeiad/api/v1" cmplugin "github.com/decred/politeia/politeiad/plugins/comments" - "github.com/decred/politeia/politeiad/plugins/pi" - "github.com/decred/politeia/politeiad/plugins/usermd" + usplugin "github.com/decred/politeia/politeiad/plugins/usermd" cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" - v1 "github.com/decred/politeia/politeiawww/api/pi/v1" + piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/comments" "github.com/decred/politeia/politeiawww/records" @@ -364,7 +364,7 @@ func (p *Pi) ntfnCommentReply(state string, c cmv1.Comment, proposalName string) log.Debugf("Comment reply ntfn not needed %v", c.Token) return nil case !pauthor.NotificationIsEnabled(ntfnBit): - // Author does not have notification bit set on + // Author does not have notification bit set log.Debugf("Comment reply ntfn not enabled %v", c.Token) return nil } @@ -391,31 +391,17 @@ func (p *Pi) handleEventCommentNew(ch chan interface{}) { // Get the record author and record name var ( - pdr pdv1.Record + pdr *pdv1.Record r rcv1.Record proposalAuthorID string proposalName string + err error ) - reqs := []pdv1.RecordRequest{ - { - Token: e.Comment.Token, - Filenames: []string{ - v1.FileNameProposalMetadata, - }, - }, - } - rs, err := p.records(e.State, reqs) + pdr, err = p.recordAbridged(e.State, e.Comment.Token) if err != nil { - err = fmt.Errorf("politeiad records: %v", err) - goto failed - } - pdr, ok = rs[e.Comment.Token] - if !ok { - err = fmt.Errorf("record %v not found for comment %v", - e.Comment.Token, e.Comment.CommentID) goto failed } - r = convertRecordToV1(pdr, e.State) + r = convertRecordToV1(*pdr, e.State) proposalAuthorID = userIDFromMetadata(r.Metadata) proposalName = proposalNameFromRecord(r) @@ -451,37 +437,64 @@ func (p *Pi) handleEventVoteAuthorized(ch chan interface{}) { continue } - /* - // Compile a list of emails to send the notification to. - emails := make([]string, 0, 256) - err := p.db.AllUsers(func(u *user.User) { - switch { - case !u.Admin: - // Only notify admin users - return - case !userNotificationEnabled(*u, - www.NotificationEmailAdminVoteAuthorized): - // User does not have notification bit set - return - } + // Verify there is work to do. We don't need to send a + // notification on revocations. + if e.Auth.Action != tkv1.AuthActionAuthorize { + log.Debugf("Vote authorize ntfn not needed %v", e.Auth.Token) + continue + } - // Add user to notification list + // Setup args to prevent goto errors + var ( + state = rcv1.RecordStateVetted + token = e.Auth.Token + proposalName string + r rcv1.Record + emails = make([]string, 0, 256) + ntfnBit = uint64(www.NotificationEmailAdminProposalVoteAuthorized) + err error + ) + + // Get record + pdr, err := p.recordAbridged(state, token) + if err != nil { + goto failed + } + r = convertRecordToV1(*pdr, state) + proposalName = proposalNameFromRecord(r) + + // Compile notification email list + err = p.userdb.AllUsers(func(u *user.User) { + switch { + case !u.Admin: + // Only notify admin users + return + case !u.NotificationIsEnabled(ntfnBit): + // Admin does not have notfication enabled + return + default: + // Admin has notification enabled emails = append(emails, u.Email) - }) - if err != nil { - log.Errorf("handleEventVoteAuthorized: AllUsers: %v", err) - continue } + }) + if err != nil { + err = fmt.Errorf("AllUsers: %v", err) + goto failed + } - // Send notification email - err = p.emailVoteAuthorized(d.token, d.name, d.username, emails) - if err != nil { - log.Errorf("emailVoteAuthorized: %v", err) - continue - } - */ + // Send notification email + err = p.mailNtfnVoteAuthorized(token, proposalName, emails) + if err != nil { + err = fmt.Errorf("mailNtfnVoteAuthorized: %v", err) + goto failed + } + + log.Debugf("Vote authorized ntfn sent %v", e.Auth.Token) + continue - log.Debugf("Proposal vote authorized ntfn sent %v", e.Auth.Token) + failed: + log.Debugf("handleEventVoteAuthorized: %v", err) + continue } } @@ -537,37 +550,11 @@ func (p *Pi) handleEventVoteStart(ch chan interface{}) { _ = e token := "fix me" - log.Debugf("Proposal vote started ntfn sent %v", token) + log.Debugf("Vote started ntfn sent %v", token) } } /* -// emailProposalVoteAuthorized sends a proposal vote authorized email to the -// provided list of emails. -func (p *politeiawww) emailProposalVoteAuthorized(token, name, username string, emails []string) error { - // Setup URL - route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) - l, err := url.Parse(p.cfg.WebServerAddress + route) - if err != nil { - return err - } - - // Setup email - subject := "Proposal Authorized To Start Voting" - tplData := proposalVoteAuthorized{ - Username: username, - Name: name, - Link: l.String(), - } - body, err := createBody(proposalVoteAuthorizedTmpl, tplData) - if err != nil { - return err - } - - // Send email - return p.smtp.sendEmailTo(subject, body, emails) -} - // emailProposalVoteStarted sends a proposal vote started email notification // to the provided email addresses. func (p *politeiawww) emailProposalVoteStarted(token, name string, emails []string) error { @@ -642,13 +629,36 @@ func (p *Pi) records(state string, reqs []pdv1.RecordRequest) (map[string]pdv1.R return records, nil } +// recordAbridged returns a proposal record without its index file or any +// attachment files. This allows the request to be light weight. +func (p *Pi) recordAbridged(state, token string) (*pdv1.Record, error) { + reqs := []pdv1.RecordRequest{ + { + Token: token, + Filenames: []string{ + piv1.FileNameProposalMetadata, + piv1.FileNameVoteMetadata, + }, + }, + } + rs, err := p.records(state, reqs) + if err != nil { + return nil, fmt.Errorf("politeiad records: %v", err) + } + r, ok := rs[token] + if !ok { + return nil, fmt.Errorf("record not found %v", token) + } + return &r, nil +} + // userMetadataDecode decodes and returns the UserMetadata from the provided // metadata streams. If a UserMetadata is not found, nil is returned. -func userMetadataDecode(ms []rcv1.MetadataStream) (*usermd.UserMetadata, error) { - var userMD *usermd.UserMetadata +func userMetadataDecode(ms []rcv1.MetadataStream) (*usplugin.UserMetadata, error) { + var userMD *usplugin.UserMetadata for _, v := range ms { - if v.ID == usermd.MDStreamIDUserMetadata { - var um usermd.UserMetadata + if v.ID == usplugin.MDStreamIDUserMetadata { + var um usplugin.UserMetadata err := json.Unmarshal([]byte(v.Payload), &um) if err != nil { return nil, err @@ -679,12 +689,12 @@ func userIDFromMetadata(ms []rcv1.MetadataStream) string { func proposalNameFromRecord(r rcv1.Record) string { var name string for _, v := range r.Files { - if v.Name == pi.FileNameProposalMetadata { + if v.Name == piv1.FileNameProposalMetadata { b, err := base64.StdEncoding.DecodeString(v.Payload) if err != nil { return "" } - var pm pi.ProposalMetadata + var pm piv1.ProposalMetadata err = json.Unmarshal(b, &pm) if err != nil { return "" @@ -695,11 +705,11 @@ func proposalNameFromRecord(r rcv1.Record) string { return name } -func statusChangesDecode(payload []byte) ([]usermd.StatusChangeMetadata, error) { - statuses := make([]usermd.StatusChangeMetadata, 0, 16) +func statusChangesDecode(payload []byte) ([]usplugin.StatusChangeMetadata, error) { + statuses := make([]usplugin.StatusChangeMetadata, 0, 16) d := json.NewDecoder(strings.NewReader(string(payload))) for { - var sc usermd.StatusChangeMetadata + var sc usplugin.StatusChangeMetadata err := d.Decode(&sc) if errors.Is(err, io.EOF) { break @@ -711,13 +721,13 @@ func statusChangesDecode(payload []byte) ([]usermd.StatusChangeMetadata, error) return statuses, nil } -func statusChangesFromMetadata(metadata []rcv1.MetadataStream) ([]usermd.StatusChangeMetadata, error) { +func statusChangesFromMetadata(metadata []rcv1.MetadataStream) ([]usplugin.StatusChangeMetadata, error) { var ( - sc []usermd.StatusChangeMetadata + sc []usplugin.StatusChangeMetadata err error ) for _, v := range metadata { - if v.ID == usermd.MDStreamIDStatusChanges { + if v.ID == usplugin.MDStreamIDStatusChanges { sc, err = statusChangesDecode([]byte(v.Payload)) if err != nil { return nil, err diff --git a/politeiawww/pi/mail.go b/politeiawww/pi/mail.go index f22a3cc0c..5e8d6b24a 100644 --- a/politeiawww/pi/mail.go +++ b/politeiawww/pi/mail.go @@ -231,7 +231,7 @@ type commentNewToProposalAuthor struct { Link string // Comment link } -const commentNewToProposalAuthorText = ` +var commentNewToProposalAuthorText = ` {{.Username}} has commented on your proposal "{{.Name}}". {{.Link}} @@ -271,7 +271,7 @@ type commentReply struct { Link string // Comment link } -const commentReplyText = ` +var commentReplyText = ` {{.Username}} has replied to your comment on "{{.Name}}". {{.Link}} @@ -304,11 +304,46 @@ func (p *Pi) mailNtfnCommentReply(token string, commentID uint32, commentUsernam return p.mail.SendTo(subject, body, []string{parentAuthorEmail}) } -func populateTemplate(tpl *template.Template, tplData interface{}) (string, error) { - var buf bytes.Buffer - err := tpl.Execute(&buf, tplData) +type voteAuthorized struct { + Name string // Proposal name + Link string // GUI proposal details url +} + +var voteAuthorizedText = ` +A proposal vote has been authorized. + +{{.Name}} +{{.Link}} +` + +var voteAuthorizedTmpl = template.Must( + template.New("voteAuthorized").Parse(voteAuthorizedText)) + +func (p *Pi) mailNtfnVoteAuthorized(token, name string, emails []string) error { + route := strings.Replace(guiRouteRecordDetails, "{token}", token, 1) + u, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err + } + + subject := "Proposal Vote Authorized" + tmplData := voteAuthorized{ + Name: name, + Link: u.String(), + } + body, err := populateTemplate(voteAuthorizedTmpl, tmplData) + if err != nil { + return err + } + + return p.mail.SendTo(subject, body, emails) +} + +func populateTemplate(tmpl *template.Template, tmplData interface{}) (string, error) { + var b bytes.Buffer + err := tmpl.Execute(&b, tmplData) if err != nil { return "", err } - return buf.String(), nil + return b.String(), nil } diff --git a/politeiawww/templates.go b/politeiawww/templates.go index 367856b45..dd2e960da 100644 --- a/politeiawww/templates.go +++ b/politeiawww/templates.go @@ -6,23 +6,6 @@ package main import "text/template" -// Proposal vote authorized - Send to admins -type proposalVoteAuthorized struct { - Username string // Author username - Name string // Proposal name - Link string // GUI proposal details url -} - -const proposalVoteAuthorizedText = ` -{{.Username}} has authorized a vote on their proposal. - -{{.Name}} -{{.Link}} -` - -var proposalVoteAuthorizedTmpl = template.Must( - template.New("proposalVoteAuthorized").Parse(proposalVoteAuthorizedText)) - // Proposal vote started - Send to users type proposalVoteStarted struct { Name string // Proposal name From 2f3f4d2eee7300bd4cf96422a115ded638475017 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 28 Feb 2021 11:01:38 -0600 Subject: [PATCH 351/449] pi: Get vote started ntfn working. --- politeiawww/pi/events.go | 229 ++++++++++++++++-------------- politeiawww/pi/mail.go | 70 +++++++++ politeiawww/templates.go | 33 ----- politeiawww/ticketvote/events.go | 4 +- politeiawww/ticketvote/process.go | 4 +- 5 files changed, 199 insertions(+), 141 deletions(-) diff --git a/politeiawww/pi/events.go b/politeiawww/pi/events.go index 5becc4830..eea5e72f7 100644 --- a/politeiawww/pi/events.go +++ b/politeiawww/pi/events.go @@ -65,7 +65,7 @@ func (p *Pi) setupEventListeners() { // Ticket vote started ch = make(chan interface{}) p.events.Register(ticketvote.EventTypeStart, ch) - go p.handleEventVoteStart(ch) + go p.handleEventVoteStarted(ch) } func (p *Pi) handleEventRecordNew(ch chan interface{}) { @@ -212,8 +212,8 @@ func (p *Pi) handleEventRecordSetStatus(ch chan interface{}) { default: // The status does not require a notification be sent. Listen // for next event. - log.Debugf("Record set status ntfn not needed for %v %v", - token, rcv1.RecordStatuses[status]) + log.Debugf("Record set status ntfn not needed for %v status %v", + rcv1.RecordStatuses[status], token) continue } @@ -243,14 +243,14 @@ func (p *Pi) handleEventRecordSetStatus(ch chan interface{}) { break } - log.Debugf("Record set status ntfn sent to author %v", token) + log.Debugf("Record set status ntfn to author sent %v", token) } // Only send a notification to non-author users if the proposal // is being made public. if status != rcv1.RecordStatusPublic { - log.Debugf("Record set status ntfn to users not needed for %v %v", - token, rcv1.RecordStatuses[status]) + log.Debugf("Record set status ntfn not needed for %v status %v", + rcv1.RecordStatuses[status], token) continue } @@ -281,7 +281,7 @@ func (p *Pi) handleEventRecordSetStatus(ch chan interface{}) { goto failed } - log.Debugf("Record set status ntfn sent to users %v", token) + log.Debugf("Record set status ntfn sent %v", token) continue failed: @@ -361,11 +361,11 @@ func (p *Pi) ntfnCommentReply(state string, c cmv1.Comment, proposalName string) switch { case c.UserID == pauthor.ID.String(): // Author replied to their own comment - log.Debugf("Comment reply ntfn not needed %v", c.Token) + log.Debugf("Comment reply ntfn to parent author not needed %v", c.Token) return nil case !pauthor.NotificationIsEnabled(ntfnBit): // Author does not have notification bit set - log.Debugf("Comment reply ntfn not enabled %v", c.Token) + log.Debugf("Comment reply ntfn to parent author not enabled %v", c.Token) return nil } @@ -376,7 +376,7 @@ func (p *Pi) ntfnCommentReply(state string, c cmv1.Comment, proposalName string) return err } - log.Debugf("Comment reply ntfn sent %v", c.Token) + log.Debugf("Comment reply ntfn to parent author sent %v", c.Token) return nil } @@ -409,15 +409,16 @@ func (p *Pi) handleEventCommentNew(ch chan interface{}) { err = p.ntfnCommentNewProposalAuthor(e.Comment, proposalAuthorID, proposalName) if err != nil { - // Log error and continue so the other notifications are still - // sent. + // Log error and continue. This error should not prevent the + // other notifications from attempting to be sent. log.Errorf("ntfnCommentNewProposalAuthor: %v", err) } // Notify the parent comment author err = p.ntfnCommentReply(e.State, e.Comment, proposalName) if err != nil { - log.Errorf("ntfnCommentReply: %v", err) + err = fmt.Errorf("ntfnCommentReply: %v", err) + goto failed } // Notifications sent! @@ -440,7 +441,7 @@ func (p *Pi) handleEventVoteAuthorized(ch chan interface{}) { // Verify there is work to do. We don't need to send a // notification on revocations. if e.Auth.Action != tkv1.AuthActionAuthorize { - log.Debugf("Vote authorize ntfn not needed %v", e.Auth.Token) + log.Debugf("Vote authorize ntfn to admin not needed %v", e.Auth.Token) continue } @@ -489,122 +490,142 @@ func (p *Pi) handleEventVoteAuthorized(ch chan interface{}) { goto failed } - log.Debugf("Vote authorized ntfn sent %v", e.Auth.Token) + log.Debugf("Vote authorized ntfn to admin sent %v", e.Auth.Token) continue failed: - log.Debugf("handleEventVoteAuthorized: %v", err) + log.Errorf("handleEventVoteAuthorized: %v", err) continue } } -func (p *Pi) handleEventVoteStart(ch chan interface{}) { - for msg := range ch { - e, ok := msg.(ticketvote.EventStart) - if !ok { - log.Errorf("handleEventVoteStart invalid msg: %v", msg) - continue - } - - /* - // Email author - notification := www.NotificationEmailRegularVoteStart - if userNotificationEnabled(d.author, notification) { - err := p.emailVoteStartToAuthor(d.token, d.name, - d.author.Username, d.author.Email) - if err != nil { - log.Errorf("emailVoteStartToAuthor: %v", err) - continue - } - } - - // Compile a list of users to send the notification to. - emails := make([]string, 0, 256) - err := p.db.AllUsers(func(u *user.User) { - switch { - case u.ID.String() == d.adminID: - // Don't notify admin who started the vote - return - case u.ID.String() == d.author.ID.String(): - // Don't send this notification to the author - return - case !userNotificationEnabled(*u, notification): - // User does not have notification bit set - return - } - - // Add user to notification list - emails = append(emails, u.Email) - }) - if err != nil { - log.Errorf("handleEventVoteStart: AllUsers: %v", err) - } - - // Email users - err = p.emailVoteStart(d.token, d.name, emails) - if err != nil { - log.Errorf("emailVoteStartToUsers: %v", err) - continue - } - */ - - _ = e - token := "fix me" - log.Debugf("Vote started ntfn sent %v", token) - } -} +func (p *Pi) ntfnVoteStartedToAuthor(sd tkv1.StartDetails, authorID, proposalName string) error { + var ( + token = sd.Params.Token + ntfnBit = uint64(www.NotificationEmailRegularProposalVoteStarted) + ) -/* -// emailProposalVoteStarted sends a proposal vote started email notification -// to the provided email addresses. -func (p *politeiawww) emailProposalVoteStarted(token, name string, emails []string) error { - // Setup URL - route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) - l, err := url.Parse(p.cfg.WebServerAddress + route) + // Get record author + uid, err := uuid.Parse(authorID) if err != nil { return err } + author, err := p.userdb.UserGetById(uid) + if err != nil { + return fmt.Errorf("UserGetByID %v: %v", authorID, err) + } - // Setup email - subject := "Voting Started for Proposal" - tplData := proposalVoteStarted{ - Name: name, - Link: l.String(), + // Verify author notification settings + if !author.NotificationIsEnabled(ntfnBit) { + log.Debugf("Vote started ntfn to author not enabled %v", token) + return nil } - body, err := createBody(proposalVoteStartedTmpl, tplData) + + // Send notification to author + err = p.mailNtfnVoteStartedToAuthor(token, proposalName, author.Email) if err != nil { return err } - // Send email - return p.smtp.sendEmailTo(subject, body, emails) + log.Debugf("Vote started ntfn to author sent %v", token) + + return nil } -// emailProposalVoteStartedToAuthor sends a proposal vote started email to -// the provided email address. -func (p *politeiawww) emailProposalVoteStartedToAuthor(token, name, username, email string) error { - // Setup URL - route := strings.Replace(guiRouteProposalDetails, "{token}", token, 1) - l, err := url.Parse(p.cfg.WebServerAddress + route) +func (p *Pi) ntfnVoteStarted(sd tkv1.StartDetails, eventUser user.User, authorID, proposalName string) error { + var ( + token = sd.Params.Token + ntfnBit = uint64(www.NotificationEmailRegularProposalVoteStarted) + ) + + // Compile user notification list + emails := make([]string, 0, 1024) + err := p.userdb.AllUsers(func(u *user.User) { + switch { + case u.ID.String() == eventUser.ID.String(): + // Don't send a notification to the user that sent the request + // to start the vote. + return + case u.ID.String() == authorID: + // Don't send the notification to the author. They are sent a + // seperate notification. + return + case !u.NotificationIsEnabled(ntfnBit): + // User does not have notification bit set + return + default: + // User has notification bit set + emails = append(emails, u.Email) + } + }) if err != nil { - return err + return fmt.Errorf("AllUsers: %v", err) } - // Setup email - subject := "Your Proposal Has Started Voting" - tplData := proposalVoteStartedToAuthor{ - Name: name, - Link: l.String(), - } - body, err := createBody(proposalVoteStartedToAuthorTmpl, tplData) + // Email users + err = p.mailNtfnVoteStarted(token, proposalName, emails) if err != nil { - return err + return fmt.Errorf("mailNtfnVoteStarted: %v", err) } - // Send email - return p.smtp.sendEmailTo(subject, body, []string{email}) + log.Debugf("Vote started ntfn sent %v", token) + + return nil +} + +func (p *Pi) handleEventVoteStarted(ch chan interface{}) { + for msg := range ch { + e, ok := msg.(ticketvote.EventStart) + if !ok { + log.Errorf("handleEventVoteStarted invalid msg: %v", msg) + continue + } + + for _, v := range e.Starts { + // Setup args to prevent goto errors + var ( + state = rcv1.RecordStateVetted + token = v.Params.Token + + pdr *pdv1.Record + r rcv1.Record + err error + + authorID string + proposalName string + ) + pdr, err = p.recordAbridged(state, token) + if err != nil { + goto failed + } + r = convertRecordToV1(*pdr, state) + authorID = userIDFromMetadata(r.Metadata) + proposalName = proposalNameFromRecord(r) + + // Send notification to record author + err = p.ntfnVoteStartedToAuthor(v, authorID, proposalName) + if err != nil { + // Log the error and continue. This error should not prevent + // the other notifications from attempting to be sent. + log.Errorf("ntfnVoteStartedToAuthor: %v", err) + } + + // Send notification to users + err = p.ntfnVoteStarted(v, e.User, authorID, proposalName) + if err != nil { + err = fmt.Errorf("ntfnVoteStarted: %v", err) + goto failed + } + + // Notifications sent! + continue + + failed: + log.Errorf("handleVoteStarted %v: %v", token, err) + continue + } + } } -*/ func (p *Pi) records(state string, reqs []pdv1.RecordRequest) (map[string]pdv1.Record, error) { var ( diff --git a/politeiawww/pi/mail.go b/politeiawww/pi/mail.go index 5e8d6b24a..bf664506b 100644 --- a/politeiawww/pi/mail.go +++ b/politeiawww/pi/mail.go @@ -339,6 +339,76 @@ func (p *Pi) mailNtfnVoteAuthorized(token, name string, emails []string) error { return p.mail.SendTo(subject, body, emails) } +type voteStarted struct { + Name string // Proposal name + Link string // GUI proposal details url +} + +const voteStartedText = ` +Voting has started on a Politeia proposal. + +{{.Name}} +{{.Link}} +` + +var voteStartedTmpl = template.Must( + template.New("voteStarted").Parse(voteStartedText)) + +func (p *Pi) mailNtfnVoteStarted(token, name string, emails []string) error { + route := strings.Replace(guiRouteRecordDetails, "{token}", token, 1) + u, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err + } + + subject := "Voting Started for Proposal" + tmplData := voteStarted{ + Name: name, + Link: u.String(), + } + body, err := populateTemplate(voteStartedTmpl, tmplData) + if err != nil { + return err + } + + return p.mail.SendTo(subject, body, emails) +} + +type voteStartedToAuthor struct { + Name string // Proposal name + Link string // GUI proposal details url +} + +const voteStartedToAuthorText = ` +Voting has just started on your Politeia proposal. + +{{.Name}} +{{.Link}} +` + +var voteStartedToAuthorTmpl = template.Must( + template.New("voteStartedToAuthor").Parse(voteStartedToAuthorText)) + +func (p *Pi) mailNtfnVoteStartedToAuthor(token, name, email string) error { + route := strings.Replace(guiRouteRecordDetails, "{token}", token, 1) + u, err := url.Parse(p.cfg.WebServerAddress + route) + if err != nil { + return err + } + + subject := "Voting Has Started On Your Proposal" + tmplData := voteStartedToAuthor{ + Name: name, + Link: u.String(), + } + body, err := populateTemplate(voteStartedToAuthorTmpl, tmplData) + if err != nil { + return err + } + + return p.mail.SendTo(subject, body, []string{email}) +} + func populateTemplate(tmpl *template.Template, tmplData interface{}) (string, error) { var b bytes.Buffer err := tmpl.Execute(&b, tmplData) diff --git a/politeiawww/templates.go b/politeiawww/templates.go index dd2e960da..2c760c25c 100644 --- a/politeiawww/templates.go +++ b/politeiawww/templates.go @@ -6,39 +6,6 @@ package main import "text/template" -// Proposal vote started - Send to users -type proposalVoteStarted struct { - Name string // Proposal name - Link string // GUI proposal details url -} - -const proposalVoteStartedText = ` -Voting has started on a Politeia proposal! - -{{.Name}} -{{.Link}} -` - -var proposalVoteStartedTmpl = template.Must( - template.New("proposalVoteStarted").Parse(proposalVoteStartedText)) - -// Proposal vote started - Send to author -type proposalVoteStartedToAuthor struct { - Name string // Proposal name - Link string // GUI proposal details url -} - -const proposalVoteStartedToAuthorText = ` -Voting has just started on your Politeia proposal! - -{{.Name}} -{{.Link}} -` - -var proposalVoteStartedToAuthorTmpl = template.Must( - template.New("proposalVoteStartedToAuthor"). - Parse(proposalVoteStartedToAuthorText)) - // User email verify - Send verification link to new user type userEmailVerify struct { Username string // User username diff --git a/politeiawww/ticketvote/events.go b/politeiawww/ticketvote/events.go index f1880edca..434755caa 100644 --- a/politeiawww/ticketvote/events.go +++ b/politeiawww/ticketvote/events.go @@ -25,6 +25,6 @@ type EventAuthorize struct { // EventStart is the event data for EventTypeStart. type EventStart struct { - Start v1.Start - User user.User + Starts []v1.StartDetails + User user.User } diff --git a/politeiawww/ticketvote/process.go b/politeiawww/ticketvote/process.go index 8e8b5db2d..92023ef0f 100644 --- a/politeiawww/ticketvote/process.go +++ b/politeiawww/ticketvote/process.go @@ -100,8 +100,8 @@ func (t *TicketVote) processStart(ctx context.Context, s v1.Start, u user.User) // Emit notification for each start t.events.Emit(EventTypeStart, EventStart{ - Start: s, - User: u, + Starts: s.Starts, + User: u, }) return &v1.StartReply{ From b3c8a56366ef692b885d277e25d32f9aa742e633 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 28 Feb 2021 11:46:46 -0600 Subject: [PATCH 352/449] pi: Cleanup ntfns. --- politeiawww/pi/events.go | 196 ++++++++++++++++++++++----------------- 1 file changed, 111 insertions(+), 85 deletions(-) diff --git a/politeiawww/pi/events.go b/politeiawww/pi/events.go index eea5e72f7..dd50cace3 100644 --- a/politeiawww/pi/events.go +++ b/politeiawww/pi/events.go @@ -78,7 +78,7 @@ func (p *Pi) handleEventRecordNew(ch chan interface{}) { // Compile notification email list var ( - emails = make([]string, 0, 256) + emails = make([]string, 0, 1024) ntfnBit = uint64(www.NotificationEmailAdminProposalNew) ) err := p.userdb.AllUsers(func(u *user.User) { @@ -131,7 +131,7 @@ func (p *Pi) handleEventRecordEdit(ch chan interface{}) { // Compile notification email list var ( - emails = make([]string, 0, 256) + emails = make([]string, 0, 1024) authorID = e.User.ID.String() ntfnBit = uint64(www.NotificationEmailRegularProposalEdited) ) @@ -172,6 +172,98 @@ func (p *Pi) handleEventRecordEdit(ch chan interface{}) { } } +func (p *Pi) ntfnRecordSetStatusToAuthor(r rcv1.Record) error { + // Unpack args + var ( + token = r.CensorshipRecord.Token + status = r.Status + name = proposalNameFromRecord(r) + authorID = userIDFromMetadata(r.Metadata) + ) + + // Parse the status change reason + sc, err := statusChangesFromMetadata(r.Metadata) + if err != nil { + return fmt.Errorf("decode status changes: %v", err) + } + if len(sc) == 0 { + return fmt.Errorf("not status changes found %v", token) + } + reason := sc[len(sc)-1].Reason + + // Get author + uid, err := uuid.Parse(authorID) + if err != nil { + return err + } + author, err := p.userdb.UserGetById(uid) + if err != nil { + return fmt.Errorf("UserGetById %v: %v", uid, err) + } + + // Send notification to author + ntfnBit := uint64(www.NotificationEmailRegularProposalVetted) + if !author.NotificationIsEnabled(ntfnBit) { + // Author does not have notification enabled + log.Debugf("Record set status ntfn to author not enabled %v", token) + return nil + } + + // Author has notification enabled + err = p.mailNtfnProposalSetStatusToAuthor(token, name, + status, reason, author.Email) + if err != nil { + return fmt.Errorf("mailNtfnProposalSetStatusToAuthor: %v", err) + } + + log.Debugf("Record set status ntfn to author sent %v", token) + + return nil +} + +func (p *Pi) ntfnRecordSetStatus(r rcv1.Record) error { + // Unpack args + var ( + token = r.CensorshipRecord.Token + status = r.Status + name = proposalNameFromRecord(r) + authorID = userIDFromMetadata(r.Metadata) + ) + + // Compile user notification email list + var ( + emails = make([]string, 0, 1024) + ntfnBit = uint64(www.NotificationEmailRegularProposalVetted) + ) + err := p.userdb.AllUsers(func(u *user.User) { + switch { + case u.ID.String() == authorID: + // User is the author. The author is sent a different + // notification. Don't include them in the users list. + return + case !u.NotificationIsEnabled(ntfnBit): + // User does not have notification bit set + return + default: + // Add user to notification list + emails = append(emails, u.Email) + } + }) + if err != nil { + return fmt.Errorf("AllUsers: %v", err) + } + + // Send user notifications + err = p.mailNtfnProposalSetStatus(token, name, status, emails) + if err != nil { + return fmt.Errorf("mailNtfnProposalSetStatus: %v", err) + } + + log.Debugf("Record set status ntfn to users sent %v", token) + + return nil +} + func (p *Pi) handleEventRecordSetStatus(ch chan interface{}) { for msg := range ch { e, ok := msg.(records.EventSetStatus) @@ -182,111 +274,45 @@ func (p *Pi) handleEventRecordSetStatus(ch chan interface{}) { // Unpack args var ( - token = e.Record.CensorshipRecord.Token - status = e.Record.Status - reason = "" // Populated below - name = proposalNameFromRecord(e.Record) - authorID = userIDFromMetadata(e.Record.Metadata) - - author *user.User - uid uuid.UUID - ntfnBit = uint64(www.NotificationEmailRegularProposalVetted) - emails = make([]string, 0, 256) + token = e.Record.CensorshipRecord.Token + status = e.Record.Status ) - sc, err := statusChangesFromMetadata(e.Record.Metadata) - if err != nil { - err = fmt.Errorf("decode status changes: %v", err) - goto failed - } - if len(sc) == 0 { - err = fmt.Errorf("not status changes found %v", token) - goto failed - } - reason = sc[len(sc)-1].Reason - // Verify a notification should be sent switch status { case rcv1.RecordStatusPublic, rcv1.RecordStatusCensored: - // The status requires a notification be sent + // Status requires a notification be sent default: - // The status does not require a notification be sent. Listen - // for next event. + // Status does not require a notification be sent log.Debugf("Record set status ntfn not needed for %v status %v", rcv1.RecordStatuses[status], token) continue } - // Send author notification - uid, err = uuid.Parse(authorID) - if err != nil { - goto failed - } - author, err = p.userdb.UserGetById(uid) + // Send notification to the author + err := p.ntfnRecordSetStatusToAuthor(e.Record) if err != nil { - err = fmt.Errorf("UserGetById %v: %v", uid, err) - goto failed - } - switch { - case !author.NotificationIsEnabled(ntfnBit): - // Author does not have notification enabled - log.Debugf("Record set status ntfn to author not enabled %v", token) - - default: - // Author does have notification enabled - err = p.mailNtfnProposalSetStatusToAuthor(token, name, - status, reason, author.Email) - if err != nil { - // Log the error and continue. This error should not prevent - // the other notifications from being sent. - log.Errorf("mailNtfnProposalSetStatusToAuthor: %v", err) - break - } - - log.Debugf("Record set status ntfn to author sent %v", token) + // Log the error and continue. This error should not prevent + // the other notifications from attempting to be sent. + log.Errorf("ntfnRecordSetStatusToAuthor: %v", err) } // Only send a notification to non-author users if the proposal // is being made public. if status != rcv1.RecordStatusPublic { - log.Debugf("Record set status ntfn not needed for %v status %v", + log.Debugf("Record set status ntfn to users not needed for %v status %v", rcv1.RecordStatuses[status], token) continue } - // Compile user notification email list - err = p.userdb.AllUsers(func(u *user.User) { - switch { - case u.ID.String() == author.ID.String(): - // User is the author. The author is sent a different - // notification. Don't include them in the users list. - return - case !u.NotificationIsEnabled(ntfnBit): - // User does not have notification bit set - return - default: - // Add user to notification list - emails = append(emails, u.Email) - } - }) - if err != nil { - err = fmt.Errorf("AllUsers: %v", err) - goto failed - } - - // Send user notifications - err = p.mailNtfnProposalSetStatus(token, name, status, emails) + // Send notification to the users + err = p.ntfnRecordSetStatus(e.Record) if err != nil { - err = fmt.Errorf("mailNtfnProposalSetStatus: %v", err) - goto failed + log.Errorf("ntfnRecordSetStatus: %v", err) + continue } - log.Debugf("Record set status ntfn sent %v", token) - continue - - failed: - log.Errorf("handleEventRecordSetStatus %v %v: %v", - token, rcv1.RecordStatuses[status], err) + // Notifications sent! continue } } @@ -451,7 +477,7 @@ func (p *Pi) handleEventVoteAuthorized(ch chan interface{}) { token = e.Auth.Token proposalName string r rcv1.Record - emails = make([]string, 0, 256) + emails = make([]string, 0, 1024) ntfnBit = uint64(www.NotificationEmailAdminProposalVoteAuthorized) err error ) @@ -568,7 +594,7 @@ func (p *Pi) ntfnVoteStarted(sd tkv1.StartDetails, eventUser user.User, authorID return fmt.Errorf("mailNtfnVoteStarted: %v", err) } - log.Debugf("Vote started ntfn sent %v", token) + log.Debugf("Vote started ntfn to users sent %v", token) return nil } From fe939059c235e39c90676f94a2877ad3decdf23f Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 28 Feb 2021 19:21:18 -0600 Subject: [PATCH 353/449] records: Add ability to specify filenames. --- politeiawww/api/records/v1/v1.go | 19 +++- politeiawww/client/pi.go | 20 ---- politeiawww/client/records.go | 20 ++++ politeiawww/cmd/pictl/cmdproposals.go | 29 +++--- politeiawww/cmd/pictl/proposal.go | 87 +++++++++++++++++ politeiawww/pi/events.go | 22 ++--- politeiawww/piwww.go | 3 + politeiawww/records/process.go | 130 +++++++++++++++++--------- 8 files changed, 240 insertions(+), 90 deletions(-) diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go index f0c9aecf6..7412fa104 100644 --- a/politeiawww/api/records/v1/v1.go +++ b/politeiawww/api/records/v1/v1.go @@ -292,9 +292,22 @@ type DetailsReply struct { const ( // RecordsPageSize is the maximum number of records that can be // requested in a Records request. - RecordsPageSize = 10 + RecordsPageSize = 5 ) +// RecordRequest requests a record and gives the client more granular control +// of parts of the record are returned. The only required field is the token. +// All other fields are optional. +// +// Only the record metadata, without any record files, will be returned by +// default. The client can request specific files be returned by specifying +// them in the Filenames field. +type RecordRequest struct { + Token string `json:"token"` + Version string `json:"version,omitempty"` + Filenames []string `json:"filenames,omitempty"` +} + // Records requests a batch of records. // // Only the record metadata is returned. The Details command must be used to @@ -302,8 +315,8 @@ const ( // files are not included in the reply, unvetted records are returned to all // users. type Records struct { - State string `json:"state"` - Tokens []string `json:"tokens"` + State string `json:"state"` + Requests []RecordRequest `json:"requests"` } // RecordsReply is the reply to the Records command. Any tokens that did not diff --git a/politeiawww/client/pi.go b/politeiawww/client/pi.go index 687b16fd6..feb7161be 100644 --- a/politeiawww/client/pi.go +++ b/politeiawww/client/pi.go @@ -32,23 +32,3 @@ func (c *Client) PiPolicy() (*piv1.PolicyReply, error) { return &pr, nil } - -// PiProposals sends a pi v1 Proposals request to politeiawww. -func (c *Client) PiProposals(p piv1.Proposals) (*piv1.ProposalsReply, error) { - resBody, err := c.makeReq(http.MethodPost, - piv1.APIRoute, piv1.RouteProposals, p) - if err != nil { - return nil, err - } - - var pr piv1.ProposalsReply - err = json.Unmarshal(resBody, &pr) - if err != nil { - return nil, err - } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(pr)) - } - - return &pr, nil -} diff --git a/politeiawww/client/records.go b/politeiawww/client/records.go index 098177f00..b25c3d1ce 100644 --- a/politeiawww/client/records.go +++ b/politeiawww/client/records.go @@ -96,6 +96,26 @@ func (c *Client) RecordDetails(d rcv1.Details) (*rcv1.Record, error) { return &dr.Record, nil } +// Records sends a records v1 Records request to politeiawww. +func (c *Client) Records(r rcv1.Records) (map[string]rcv1.Record, error) { + resBody, err := c.makeReq(http.MethodPost, + rcv1.APIRoute, rcv1.RouteRecords, r) + if err != nil { + return nil, err + } + + var rr rcv1.RecordsReply + err = json.Unmarshal(resBody, &rr) + if err != nil { + return nil, err + } + if c.verbose { + fmt.Printf("%v\n", util.FormatJSON(rr)) + } + + return rr.Records, nil +} + // RecordInventory sends a records v1 Inventory request to politeiawww. func (c *Client) RecordInventory(i rcv1.Inventory) (*rcv1.InventoryReply, error) { resBody, err := c.makeReq(http.MethodPost, diff --git a/politeiawww/cmd/pictl/cmdproposals.go b/politeiawww/cmd/pictl/cmdproposals.go index e33775832..95521b235 100644 --- a/politeiawww/cmd/pictl/cmdproposals.go +++ b/politeiawww/cmd/pictl/cmdproposals.go @@ -6,6 +6,7 @@ package main import ( piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" pclient "github.com/decred/politeia/politeiawww/client" ) @@ -47,23 +48,29 @@ func (c *cmdProposals) Execute(args []string) error { state = piv1.ProposalStateVetted } - // Get proposal details - p := piv1.Proposals{ - State: state, - Tokens: c.Args.Tokens, + // Get records + reqs := make([]rcv1.RecordRequest, 0, len(c.Args.Tokens)) + for _, v := range c.Args.Tokens { + reqs = append(reqs, rcv1.RecordRequest{ + Token: v, + Filenames: []string{ + piv1.FileNameProposalMetadata, + piv1.FileNameVoteMetadata, + }, + }) } - pr, err := pc.PiProposals(p) + r := rcv1.Records{ + State: state, + Requests: reqs, + } + records, err := pc.Records(r) if err != nil { return err } // Print proposals to stdout - for _, v := range pr.Proposals { - r, err := convertProposal(v) - if err != nil { - return err - } - err = printProposal(*r) + for _, v := range records { + err = printProposal(v) if err != nil { return err } diff --git a/politeiawww/cmd/pictl/proposal.go b/politeiawww/cmd/pictl/proposal.go index f6f41309d..7d1f1992d 100644 --- a/politeiawww/cmd/pictl/proposal.go +++ b/politeiawww/cmd/pictl/proposal.go @@ -9,10 +9,12 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" + "errors" "fmt" "image" "image/color" "image/png" + "io" "io/ioutil" "math/rand" "path/filepath" @@ -149,6 +151,91 @@ func convertProposal(p piv1.Proposal) (*rcv1.Record, error) { }, nil } +func convertRecord(r rcv1.Record) (*piv1.Proposal, error) { + // Decode metadata streams + var ( + um usermd.UserMetadata + sc = make([]usermd.StatusChangeMetadata, 0, 16) + err error + ) + for _, v := range r.Metadata { + switch v.ID { + case usermd.MDStreamIDUserMetadata: + err = json.Unmarshal([]byte(v.Payload), &um) + if err != nil { + return nil, err + } + case usermd.MDStreamIDStatusChanges: + sc, err = statusChangesDecode([]byte(v.Payload)) + if err != nil { + return nil, err + } + } + } + + // Convert files + files := make([]piv1.File, 0, len(r.Files)) + for _, v := range r.Files { + files = append(files, piv1.File{ + Name: v.Name, + MIME: v.MIME, + Digest: v.Digest, + Payload: v.Payload, + }) + } + + // Convert statuses + statuses := make([]piv1.StatusChange, 0, len(sc)) + for _, v := range sc { + statuses = append(statuses, piv1.StatusChange{ + Token: v.Token, + Version: v.Version, + Status: piv1.PropStatusT(v.Status), + Reason: v.Reason, + PublicKey: v.PublicKey, + Signature: v.Signature, + Timestamp: v.Timestamp, + }) + } + + // Some fields are intentionally omitted because they are user data + // that is not saved to politeiad and needs to be pulled from the + // user database. + return &piv1.Proposal{ + Version: r.Version, + Timestamp: r.Timestamp, + State: r.State, + Status: piv1.PropStatusT(r.Status), + UserID: um.UserID, + Username: r.Username, + PublicKey: um.PublicKey, + Signature: um.Signature, + Statuses: statuses, + Files: files, + CensorshipRecord: piv1.CensorshipRecord{ + Token: r.CensorshipRecord.Token, + Merkle: r.CensorshipRecord.Merkle, + Signature: r.CensorshipRecord.Signature, + }, + }, nil +} + +func statusChangesDecode(payload []byte) ([]usermd.StatusChangeMetadata, error) { + statuses := make([]usermd.StatusChangeMetadata, 0, 16) + d := json.NewDecoder(strings.NewReader(string(payload))) + for { + var sc usermd.StatusChangeMetadata + err := d.Decode(&sc) + if errors.Is(err, io.EOF) { + break + } else if err != nil { + return nil, err + } + statuses = append(statuses, sc) + } + return statuses, nil +} + // indexFileRandom returns a proposal index file filled with random data. func indexFileRandom(sizeInBytes int) (*rcv1.File, error) { // Create lines of text that are 80 characters long diff --git a/politeiawww/pi/events.go b/politeiawww/pi/events.go index dd50cace3..d5700e83a 100644 --- a/politeiawww/pi/events.go +++ b/politeiawww/pi/events.go @@ -15,7 +15,7 @@ import ( pdv1 "github.com/decred/politeia/politeiad/api/v1" cmplugin "github.com/decred/politeia/politeiad/plugins/comments" - usplugin "github.com/decred/politeia/politeiad/plugins/usermd" + "github.com/decred/politeia/politeiad/plugins/usermd" cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" @@ -701,11 +701,11 @@ func (p *Pi) recordAbridged(state, token string) (*pdv1.Record, error) { // userMetadataDecode decodes and returns the UserMetadata from the provided // metadata streams. If a UserMetadata is not found, nil is returned. -func userMetadataDecode(ms []rcv1.MetadataStream) (*usplugin.UserMetadata, error) { - var userMD *usplugin.UserMetadata +func userMetadataDecode(ms []rcv1.MetadataStream) (*usermd.UserMetadata, error) { + var userMD *usermd.UserMetadata for _, v := range ms { - if v.ID == usplugin.MDStreamIDUserMetadata { - var um usplugin.UserMetadata + if v.ID == usermd.MDStreamIDUserMetadata { + var um usermd.UserMetadata err := json.Unmarshal([]byte(v.Payload), &um) if err != nil { return nil, err @@ -752,11 +752,11 @@ func proposalNameFromRecord(r rcv1.Record) string { return name } -func statusChangesDecode(payload []byte) ([]usplugin.StatusChangeMetadata, error) { - statuses := make([]usplugin.StatusChangeMetadata, 0, 16) +func statusChangesDecode(payload []byte) ([]usermd.StatusChangeMetadata, error) { + statuses := make([]usermd.StatusChangeMetadata, 0, 16) d := json.NewDecoder(strings.NewReader(string(payload))) for { - var sc usplugin.StatusChangeMetadata + var sc usermd.StatusChangeMetadata err := d.Decode(&sc) if errors.Is(err, io.EOF) { break @@ -768,13 +768,13 @@ func statusChangesDecode(payload []byte) ([]usplugin.StatusChangeMetadata, error return statuses, nil } -func statusChangesFromMetadata(metadata []rcv1.MetadataStream) ([]usplugin.StatusChangeMetadata, error) { +func statusChangesFromMetadata(metadata []rcv1.MetadataStream) ([]usermd.StatusChangeMetadata, error) { var ( - sc []usplugin.StatusChangeMetadata + sc []usermd.StatusChangeMetadata err error ) for _, v := range metadata { - if v.ID == usplugin.MDStreamIDStatusChanges { + if v.ID == usermd.MDStreamIDStatusChanges { sc, err = statusChangesDecode([]byte(v.Payload)) if err != nil { return nil, err diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 846ee7cc0..60671017f 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -80,6 +80,9 @@ func (p *politeiawww) setupPiRoutes(r *records.Records, c *comments.Comments, t p.addRoute(http.MethodPost, rcv1.APIRoute, rcv1.RouteDetails, r.HandleDetails, permissionPublic) + p.addRoute(http.MethodPost, rcv1.APIRoute, + rcv1.RouteRecords, r.HandleRecords, + permissionPublic) p.addRoute(http.MethodPost, rcv1.APIRoute, rcv1.RouteInventory, r.HandleInventory, permissionPublic) diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index ce945fbf1..89d39d304 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -371,8 +371,47 @@ func (r *Records) processDetails(ctx context.Context, d v1.Details, u *user.User }, nil } +func (r *Records) records(ctx context.Context, state string, reqs []pdv1.RecordRequest) (map[string]v1.Record, error) { + var ( + pdr = make(map[string]pdv1.Record) + err error + ) + switch state { + case v1.RecordStateUnvetted: + pdr, err = r.politeiad.GetUnvettedBatch(ctx, reqs) + if err != nil { + return nil, err + } + case v1.RecordStateVetted: + pdr, err = r.politeiad.GetVettedBatch(ctx, reqs) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("invalid state %v", state) + } + + records := make(map[string]v1.Record, len(pdr)) + for k, v := range pdr { + rc := convertRecordToV1(v, state) + + // Fill in user data + userID := userIDFromMetadataStreams(rc.Metadata) + uid, err := uuid.Parse(userID) + u, err := r.userdb.UserGetById(uid) + if err != nil { + return nil, err + } + recordPopulateUserData(&rc, *u) + + records[k] = rc + } + + return records, nil +} + func (r *Records) processRecords(ctx context.Context, rs v1.Records, u *user.User) (*v1.RecordsReply, error) { - log.Tracef("processRecords: %v %v", rs.State, len(rs.Tokens)) + log.Tracef("processRecords: %v %v", rs.State, len(rs.Requests)) // Verify state switch rs.State { @@ -385,7 +424,7 @@ func (r *Records) processRecords(ctx context.Context, rs v1.Records, u *user.Use } // Verify page size - if len(rs.Tokens) > v1.RecordsPageSize { + if len(rs.Requests) > v1.RecordsPageSize { e := fmt.Sprintf("max page size is %v", v1.RecordsPageSize) return nil, v1.UserErrorReply{ ErrorCode: v1.ErrorCodePageSizeExceeded, @@ -393,22 +432,11 @@ func (r *Records) processRecords(ctx context.Context, rs v1.Records, u *user.Use } } - // Get all records in the batch. This should be a batched call to - // politeiad, but the politeiad API does not provided a batched - // records endpoint. - records := make(map[string]v1.Record, len(rs.Tokens)) - for _, v := range rs.Tokens { - rc, err := r.record(ctx, rs.State, v, "") - if err != nil { - // If any error occured simply skip this record. It will not - // be included in the reply. - continue - } - - // Record files are not returned in this call - rc.Files = []v1.File{} - - records[rc.CensorshipRecord.Token] = *rc + // Get records + reqs := convertRequestsToPD(rs.Requests) + records, err := r.records(ctx, rs.State, reqs) + if err != nil { + return nil, err } return &v1.RecordsReply{ @@ -610,33 +638,6 @@ func userIDFromMetadataStreams(ms []v1.MetadataStream) string { return um.UserID } -func convertFilesToPD(f []v1.File) []pdv1.File { - files := make([]pdv1.File, 0, len(f)) - for _, v := range f { - files = append(files, pdv1.File{ - Name: v.Name, - MIME: v.MIME, - Digest: v.Digest, - Payload: v.Payload, - }) - } - return files -} - -func convertStatusToPD(s v1.RecordStatusT) pdv1.RecordStatusT { - switch s { - case v1.RecordStatusUnreviewed: - return pdv1.RecordStatusNotReviewed - case v1.RecordStatusPublic: - return pdv1.RecordStatusPublic - case v1.RecordStatusCensored: - return pdv1.RecordStatusCensored - case v1.RecordStatusArchived: - return pdv1.RecordStatusArchived - } - return pdv1.RecordStatusInvalid -} - func convertStatusToV1(s pdv1.RecordStatusT) v1.RecordStatusT { switch s { case pdv1.RecordStatusNotReviewed: @@ -719,3 +720,42 @@ func convertTimestampToV1(t pdv1.Timestamp) v1.Timestamp { Proofs: proofs, } } + +func convertFilesToPD(f []v1.File) []pdv1.File { + files := make([]pdv1.File, 0, len(f)) + for _, v := range f { + files = append(files, pdv1.File{ + Name: v.Name, + MIME: v.MIME, + Digest: v.Digest, + Payload: v.Payload, + }) + } + return files +} + +func convertStatusToPD(s v1.RecordStatusT) pdv1.RecordStatusT { + switch s { + case v1.RecordStatusUnreviewed: + return pdv1.RecordStatusNotReviewed + case v1.RecordStatusPublic: + return pdv1.RecordStatusPublic + case v1.RecordStatusCensored: + return pdv1.RecordStatusCensored + case v1.RecordStatusArchived: + return pdv1.RecordStatusArchived + } + return pdv1.RecordStatusInvalid +} + +func convertRequestsToPD(reqs []v1.RecordRequest) []pdv1.RecordRequest { + r := make([]pdv1.RecordRequest, 0, len(reqs)) + for _, v := range reqs { + r = append(r, pdv1.RecordRequest{ + Token: v.Token, + Version: v.Version, + Filenames: v.Filenames, + }) + } + return r +} From 5a9de7bb308fd175d9ceba0b7b003b4fa7313f0d Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 28 Feb 2021 19:50:21 -0600 Subject: [PATCH 354/449] Add proposal name tests. --- .../backend/tstorebe/plugins/pi/hooks_test.go | 15 +++-- .../backend/tstorebe/plugins/pi/testing.go | 62 +++++++++++++++++++ politeiawww/pi/mail.go | 1 - politeiawww/www.go | 25 -------- 4 files changed, 71 insertions(+), 32 deletions(-) create mode 100644 politeiad/backend/tstorebe/plugins/pi/testing.go diff --git a/politeiad/backend/tstorebe/plugins/pi/hooks_test.go b/politeiad/backend/tstorebe/plugins/pi/hooks_test.go index 73bf7339e..f8418bc37 100644 --- a/politeiad/backend/tstorebe/plugins/pi/hooks_test.go +++ b/politeiad/backend/tstorebe/plugins/pi/hooks_test.go @@ -7,6 +7,10 @@ package pi import "testing" func TestProposalNameIsValid(t *testing.T) { + // Setup pi plugin + p, cleanup := newTestPiPlugin(t) + defer cleanup() + tests := []struct { name string want bool @@ -69,13 +73,12 @@ func TestProposalNameIsValid(t *testing.T) { true, }, } - // TODO - /* - for _, test := range tests { - isValid := proposalNameIsValid(test.name) + for _, test := range tests { + t.Run("", func(t *testing.T) { + isValid := p.proposalNameIsValid(test.name) if isValid != test.want { t.Errorf("got %v, want %v", isValid, test.want) } - } - */ + }) + } } diff --git a/politeiad/backend/tstorebe/plugins/pi/testing.go b/politeiad/backend/tstorebe/plugins/pi/testing.go new file mode 100644 index 000000000..073de8571 --- /dev/null +++ b/politeiad/backend/tstorebe/plugins/pi/testing.go @@ -0,0 +1,62 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package pi + +import ( + "encoding/json" + "io/ioutil" + "os" + "testing" + + "github.com/decred/politeia/politeiad/plugins/pi" + "github.com/decred/politeia/util" +) + +func newTestPiPlugin(t *testing.T) (*piPlugin, func()) { + // Create plugin data directory + dataDir, err := ioutil.TempDir("", pi.PluginID) + if err != nil { + t.Fatal(err) + } + + // Setup proposal name regex + var ( + nameSupportedChars = pi.SettingProposalNameSupportedChars + nameLengthMin = pi.SettingProposalNameLengthMin + nameLengthMax = pi.SettingProposalNameLengthMax + ) + rexp, err := util.Regexp(nameSupportedChars, uint64(nameLengthMin), + uint64(nameLengthMax)) + if err != nil { + t.Fatal(err) + } + + // Encode the supported chars so that they can be returned as a + // string plugin setting. + b, err := json.Marshal(nameSupportedChars) + if err != nil { + t.Fatal(err) + } + nameSupportedCharsString := string(b) + + // Setup plugin context + p := piPlugin{ + dataDir: dataDir, + textFileSizeMax: pi.SettingTextFileSizeMax, + imageFileCountMax: pi.SettingImageFileCountMax, + imageFileSizeMax: pi.SettingImageFileSizeMax, + proposalNameLengthMin: nameLengthMin, + proposalNameLengthMax: nameLengthMax, + proposalNameSupportedChars: nameSupportedCharsString, + proposalNameRegexp: rexp, + } + + return &p, func() { + err = os.RemoveAll(dataDir) + if err != nil { + t.Fatal(err) + } + } +} diff --git a/politeiawww/pi/mail.go b/politeiawww/pi/mail.go index bf664506b..383af3b53 100644 --- a/politeiawww/pi/mail.go +++ b/politeiawww/pi/mail.go @@ -16,7 +16,6 @@ import ( ) const ( - // TODO GUI links needs to be updated // The following routes are used in notification emails to direct // the user to the correct GUI pages. guiRouteRecordDetails = "/record/{token}" diff --git a/politeiawww/www.go b/politeiawww/www.go index 47b117b27..f9119c0f0 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -28,7 +28,6 @@ import ( pd "github.com/decred/politeia/politeiad/api/v1" pdv1 "github.com/decred/politeia/politeiad/api/v1" pdclient "github.com/decred/politeia/politeiad/client" - "github.com/decred/politeia/politeiad/plugins/ticketvote" cms "github.com/decred/politeia/politeiawww/api/cms/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" database "github.com/decred/politeia/politeiawww/cmsdatabase" @@ -86,36 +85,12 @@ func convertWWWErrorStatusFromPD(e pd.ErrorStatusT) www.ErrorStatusT { return www.ErrorStatusInvalid } -func convertWWWErrorStatusFromTicketVote(e ticketvote.ErrorCodeT) www.ErrorStatusT { - switch e { - case ticketvote.ErrorCodeTokenInvalid: - return www.ErrorStatusInvalidCensorshipToken - case ticketvote.ErrorCodePublicKeyInvalid: - return www.ErrorStatusInvalidPublicKey - case ticketvote.ErrorCodeSignatureInvalid: - return www.ErrorStatusInvalidSignature - case ticketvote.ErrorCodeRecordStatusInvalid: - return www.ErrorStatusWrongStatus - case ticketvote.ErrorCodeVoteParamsInvalid: - return www.ErrorStatusInvalidPropVoteParams - case ticketvote.ErrorCodeVoteStatusInvalid: - return www.ErrorStatusInvalidPropVoteStatus - } - return www.ErrorStatusInvalid -} - -// TODO make sure the legacy www routes have the plugin error conversions that -// they need. func convertWWWErrorStatus(pluginID string, errCode int) www.ErrorStatusT { switch pluginID { case "": // politeiad API e := pd.ErrorStatusT(errCode) return convertWWWErrorStatusFromPD(e) - case ticketvote.PluginID: - // Ticket vote plugin - e := ticketvote.ErrorCodeT(errCode) - return convertWWWErrorStatusFromTicketVote(e) } // No corresponding www error status found From bc2b49c6142b62ab0b0dcba4b93c358163fb02c5 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 28 Feb 2021 20:35:19 -0600 Subject: [PATCH 355/449] politeiawww/client: Fix verbose printing. --- .../backend/tstorebe/plugins/pi/testing.go | 4 +-- politeiawww/client/client.go | 10 ++++-- politeiawww/client/comments.go | 26 --------------- politeiawww/client/pi.go | 5 --- politeiawww/client/records.go | 24 -------------- politeiawww/client/ticketvote.go | 32 ------------------- 6 files changed, 9 insertions(+), 92 deletions(-) diff --git a/politeiad/backend/tstorebe/plugins/pi/testing.go b/politeiad/backend/tstorebe/plugins/pi/testing.go index 073de8571..176ba01e4 100644 --- a/politeiad/backend/tstorebe/plugins/pi/testing.go +++ b/politeiad/backend/tstorebe/plugins/pi/testing.go @@ -33,8 +33,8 @@ func newTestPiPlugin(t *testing.T) (*piPlugin, func()) { t.Fatal(err) } - // Encode the supported chars so that they can be returned as a - // string plugin setting. + // Encode the supported chars. This is done so that they can be + // returned as a plugin setting string. b, err := json.Marshal(nameSupportedChars) if err != nil { t.Fatal(err) diff --git a/politeiawww/client/client.go b/politeiawww/client/client.go index e5e03f5bd..659af3586 100644 --- a/politeiawww/client/client.go +++ b/politeiawww/client/client.go @@ -81,9 +81,13 @@ func (c *Client) makeReq(method string, api, route string, v interface{}) ([]byt // No JSON to print case c.verbose: fmt.Printf("Request: %v %v\n", method, fullRoute) - fmt.Printf("%v\n", util.FormatJSON(v)) + if len(reqBody) > 0 { + fmt.Printf("%s\n", reqBody) + } case c.rawJSON: - fmt.Printf("%s\n", reqBody) + if len(reqBody) > 0 { + fmt.Printf("%s\n", reqBody) + } } // Send request @@ -134,7 +138,7 @@ func (c *Client) makeReq(method string, api, route string, v interface{}) ([]byt // Print response body. Pretty printing the response body for the // verbose output must be handled by the calling function once it // has unmarshalled the body. - if c.rawJSON { + if c.verbose || c.rawJSON { fmt.Printf("%s\n", respBody) } diff --git a/politeiawww/client/comments.go b/politeiawww/client/comments.go index f19099bf7..d39070781 100644 --- a/politeiawww/client/comments.go +++ b/politeiawww/client/comments.go @@ -6,11 +6,9 @@ package client import ( "encoding/json" - "fmt" "net/http" cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" - "github.com/decred/politeia/util" ) // CommentPolicy sends a comments v1 Policy request to politeiawww. @@ -26,9 +24,6 @@ func (c *Client) CommentPolicy() (*cmv1.PolicyReply, error) { if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(pr)) - } return &pr, nil } @@ -46,9 +41,6 @@ func (c *Client) CommentNew(n cmv1.New) (*cmv1.NewReply, error) { if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(nr)) - } return &nr, nil } @@ -66,9 +58,6 @@ func (c *Client) CommentVote(v cmv1.Vote) (*cmv1.VoteReply, error) { if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(vr)) - } return &vr, nil } @@ -86,9 +75,6 @@ func (c *Client) CommentDel(d cmv1.Del) (*cmv1.DelReply, error) { if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(dr)) - } return &dr, nil } @@ -106,9 +92,6 @@ func (c *Client) CommentCount(cc cmv1.Count) (*cmv1.CountReply, error) { if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(cr)) - } return &cr, nil } @@ -126,9 +109,6 @@ func (c *Client) Comments(cm cmv1.Comments) (*cmv1.CommentsReply, error) { if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(cr)) - } return &cr, nil } @@ -146,9 +126,6 @@ func (c *Client) CommentVotes(v cmv1.Votes) (*cmv1.VotesReply, error) { if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(vr)) - } return &vr, nil } @@ -166,9 +143,6 @@ func (c *Client) CommentTimestamps(t cmv1.Timestamps) (*cmv1.TimestampsReply, er if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(tr)) - } return &tr, nil } diff --git a/politeiawww/client/pi.go b/politeiawww/client/pi.go index feb7161be..54184ea24 100644 --- a/politeiawww/client/pi.go +++ b/politeiawww/client/pi.go @@ -6,11 +6,9 @@ package client import ( "encoding/json" - "fmt" "net/http" piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" - "github.com/decred/politeia/util" ) // PiPolicy sends a pi v1 Policy request to politeiawww. @@ -26,9 +24,6 @@ func (c *Client) PiPolicy() (*piv1.PolicyReply, error) { if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(pr)) - } return &pr, nil } diff --git a/politeiawww/client/records.go b/politeiawww/client/records.go index b25c3d1ce..b0077b93f 100644 --- a/politeiawww/client/records.go +++ b/politeiawww/client/records.go @@ -29,9 +29,6 @@ func (c *Client) RecordNew(n rcv1.New) (*rcv1.NewReply, error) { if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(nr)) - } return &nr, nil } @@ -49,9 +46,6 @@ func (c *Client) RecordEdit(e rcv1.Edit) (*rcv1.EditReply, error) { if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(er)) - } return &er, nil } @@ -69,9 +63,6 @@ func (c *Client) RecordSetStatus(ss rcv1.SetStatus) (*rcv1.SetStatusReply, error if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(ssr)) - } return &ssr, nil } @@ -89,9 +80,6 @@ func (c *Client) RecordDetails(d rcv1.Details) (*rcv1.Record, error) { if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(dr)) - } return &dr.Record, nil } @@ -109,9 +97,6 @@ func (c *Client) Records(r rcv1.Records) (map[string]rcv1.Record, error) { if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(rr)) - } return rr.Records, nil } @@ -129,9 +114,6 @@ func (c *Client) RecordInventory(i rcv1.Inventory) (*rcv1.InventoryReply, error) if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(ir)) - } return &ir, nil } @@ -149,9 +131,6 @@ func (c *Client) RecordTimestamps(t rcv1.Timestamps) (*rcv1.TimestampsReply, err if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(tr)) - } return &tr, nil } @@ -169,9 +148,6 @@ func (c *Client) UserRecords(ur rcv1.UserRecords) (*rcv1.UserRecordsReply, error if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(urr)) - } return &urr, nil } diff --git a/politeiawww/client/ticketvote.go b/politeiawww/client/ticketvote.go index fed3d1c57..991dfb204 100644 --- a/politeiawww/client/ticketvote.go +++ b/politeiawww/client/ticketvote.go @@ -6,11 +6,9 @@ package client import ( "encoding/json" - "fmt" "net/http" tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" - "github.com/decred/politeia/util" ) // TicketVotePolicy sends a ticketvote v1 Policy request to politeiawww. @@ -26,9 +24,6 @@ func (c *Client) TicketVotePolicy() (*tkv1.PolicyReply, error) { if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(pr)) - } return &pr, nil } @@ -46,9 +41,6 @@ func (c *Client) TicketVoteAuthorize(a tkv1.Authorize) (*tkv1.AuthorizeReply, er if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(ar)) - } return &ar, nil } @@ -66,9 +58,6 @@ func (c *Client) TicketVoteStart(s tkv1.Start) (*tkv1.StartReply, error) { if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(sr)) - } return &sr, nil } @@ -87,9 +76,6 @@ func (c *Client) TicketVoteCastBallot(cb tkv1.CastBallot) (*tkv1.CastBallotReply if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(cbr)) - } return &cbr, nil } @@ -107,9 +93,6 @@ func (c *Client) TicketVoteDetails(d tkv1.Details) (*tkv1.DetailsReply, error) { if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(dr)) - } return &dr, nil } @@ -127,9 +110,6 @@ func (c *Client) TicketVoteResults(r tkv1.Results) (*tkv1.ResultsReply, error) { if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(rr)) - } return &rr, nil } @@ -147,9 +127,6 @@ func (c *Client) TicketVoteSummaries(s tkv1.Summaries) (*tkv1.SummariesReply, er if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(sr)) - } return &sr, nil } @@ -168,9 +145,6 @@ func (c *Client) TicketVoteSubmissions(s tkv1.Submissions) (*tkv1.SubmissionsRep if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(sr)) - } return &sr, nil } @@ -188,9 +162,6 @@ func (c *Client) TicketVoteInventory(i tkv1.Inventory) (*tkv1.InventoryReply, er if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(ir)) - } return &ir, nil } @@ -209,9 +180,6 @@ func (c *Client) TicketVoteTimestamps(t tkv1.Timestamps) (*tkv1.TimestampsReply, if err != nil { return nil, err } - if c.verbose { - fmt.Printf("%v\n", util.FormatJSON(tr)) - } return &tr, nil } From 617bd6397fa9658d2c862b1d652069fbea8e4c1c Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 1 Mar 2021 14:08:49 -0600 Subject: [PATCH 356/449] multi: Add ineligble vote status. --- .../tstorebe/plugins/ticketvote/cmds.go | 8 ++--- .../tstorebe/plugins/ticketvote/hooks.go | 6 +++- .../tstorebe/plugins/ticketvote/inventory.go | 29 ++++++++++++++----- .../tstorebe/plugins/ticketvote/ticketvote.go | 4 +-- politeiad/plugins/ticketvote/ticketvote.go | 5 ++++ politeiawww/api/ticketvote/v1/v1.go | 5 ++++ 6 files changed, 42 insertions(+), 15 deletions(-) diff --git a/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go b/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go index 95203e61d..94656ff81 100644 --- a/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go @@ -1080,7 +1080,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri // Should not happen return "", fmt.Errorf("invalid action %v", a.Action) } - p.inventoryUpdate(a.Token, status) + p.InventoryUpdate(a.Token, status) // Prepare reply ar := ticketvote.AuthorizeReply{ @@ -1383,7 +1383,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot } // Update inventory - p.inventoryUpdateToStarted(vd.Params.Token, ticketvote.VoteStatusStarted, + p.InventoryUpdateToStarted(vd.Params.Token, ticketvote.VoteStatusStarted, vd.EndBlockHeight) // Update active votes cache @@ -1568,7 +1568,7 @@ func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs sta } // Update inventory - p.inventoryUpdateToStarted(vd.Params.Token, ticketvote.VoteStatusStarted, + p.InventoryUpdateToStarted(vd.Params.Token, ticketvote.VoteStatusStarted, vd.EndBlockHeight) // Update active votes cache @@ -2582,7 +2582,7 @@ func (p *ticketVotePlugin) cmdInventory(payload string) (string, error) { } // Get the inventory - ibs, err := p.invByStatus(bb, i.Status, i.Page) + ibs, err := p.InventoryByStatus(bb, i.Status, i.Page) if err != nil { return "", fmt.Errorf("invByStatus: %v", err) } diff --git a/politeiad/backend/tstorebe/plugins/ticketvote/hooks.go b/politeiad/backend/tstorebe/plugins/ticketvote/hooks.go index 8eeeedc99..82ac24e39 100644 --- a/politeiad/backend/tstorebe/plugins/ticketvote/hooks.go +++ b/politeiad/backend/tstorebe/plugins/ticketvote/hooks.go @@ -327,8 +327,12 @@ func (p *ticketVotePlugin) hookSetRecordStatusPost(treeID int64, payload string) switch status { case backend.MDStatusVetted: // Add to inventory - p.inventoryAdd(srs.RecordMetadata.Token, + p.InventoryAdd(srs.RecordMetadata.Token, ticketvote.VoteStatusUnauthorized) + case backend.MDStatusCensored, backend.MDStatusArchived: + // These statuses do not allow for a vote. Mark as ineligible. + p.InventoryUpdate(srs.RecordMetadata.Token, + ticketvote.VoteStatusIneligible) } // Update the submissions cache if the linkto has been set. diff --git a/politeiad/backend/tstorebe/plugins/ticketvote/inventory.go b/politeiad/backend/tstorebe/plugins/ticketvote/inventory.go index 77574bf2e..93e64efa6 100644 --- a/politeiad/backend/tstorebe/plugins/ticketvote/inventory.go +++ b/politeiad/backend/tstorebe/plugins/ticketvote/inventory.go @@ -237,9 +237,9 @@ func (p *ticketVotePlugin) invUpdateForBlock(bestBlock uint32) (*inventory, erro return inv, nil } -// inventoryAdd is a wrapper around the invAdd method that allows us to decide +// InventoryAdd is a wrapper around the invAdd method that allows us to decide // how disk read/write errors should be handled. For now we just panic. -func (p *ticketVotePlugin) inventoryAdd(token string, s ticketvote.VoteStatusT) { +func (p *ticketVotePlugin) InventoryAdd(token string, s ticketvote.VoteStatusT) { err := p.invAdd(token, s) if err != nil { e := fmt.Sprintf("invAdd %v %v: %v", token, s, err) @@ -247,7 +247,9 @@ func (p *ticketVotePlugin) inventoryAdd(token string, s ticketvote.VoteStatusT) } } -func (p *ticketVotePlugin) inventoryUpdate(token string, s ticketvote.VoteStatusT) { +// InventoryUpdate is a wrapper around the invUpdate method that allows us to +// decide how disk read/write errors should be handled. For now we just panic. +func (p *ticketVotePlugin) InventoryUpdate(token string, s ticketvote.VoteStatusT) { err := p.invUpdate(token, s, 0) if err != nil { e := fmt.Sprintf("invUpdate %v %v: %v", token, s, err) @@ -255,7 +257,10 @@ func (p *ticketVotePlugin) inventoryUpdate(token string, s ticketvote.VoteStatus } } -func (p *ticketVotePlugin) inventoryUpdateToStarted(token string, s ticketvote.VoteStatusT, endHeight uint32) { +// InventoryUpdateToStarted is a wrapper around the invUpdate method that +// allows us to decide how disk read/write errors should be handled. For now we +// just panic. +func (p *ticketVotePlugin) InventoryUpdateToStarted(token string, s ticketvote.VoteStatusT, endHeight uint32) { err := p.invUpdate(token, s, endHeight) if err != nil { e := fmt.Sprintf("invUpdate %v %v: %v", token, s, err) @@ -263,7 +268,8 @@ func (p *ticketVotePlugin) inventoryUpdateToStarted(token string, s ticketvote.V } } -func (p *ticketVotePlugin) inventory(bestBlock uint32) (*inventory, error) { +// inventory returns the full ticketvote inventory. +func (p *ticketVotePlugin) Inventory(bestBlock uint32) (*inventory, error) { // Get inventory inv, err := p.invGet() if err != nil { @@ -289,7 +295,7 @@ type invByStatus struct { func (p *ticketVotePlugin) invByStatusAll(bestBlock, pageSize uint32) (*invByStatus, error) { // Get inventory - i, err := p.inventory(bestBlock) + i, err := p.Inventory(bestBlock) if err != nil { return nil, err } @@ -308,6 +314,8 @@ func (p *ticketVotePlugin) invByStatusAll(bestBlock, pageSize uint32) (*invBySta pageSize, 1) rejected = tokensParse(i.Entries, ticketvote.VoteStatusRejected, pageSize, 1) + ineligible = tokensParse(i.Entries, ticketvote.VoteStatusIneligible, + pageSize, 1) tokens = make(map[ticketvote.VoteStatusT][]string, 16) ) @@ -329,6 +337,9 @@ func (p *ticketVotePlugin) invByStatusAll(bestBlock, pageSize uint32) (*invBySta if len(rejected) != 0 { tokens[ticketvote.VoteStatusRejected] = rejected } + if len(ineligible) != 0 { + tokens[ticketvote.VoteStatusIneligible] = ineligible + } return &invByStatus{ Tokens: tokens, @@ -336,7 +347,9 @@ func (p *ticketVotePlugin) invByStatusAll(bestBlock, pageSize uint32) (*invBySta }, nil } -func (p *ticketVotePlugin) invByStatus(bestBlock uint32, s ticketvote.VoteStatusT, page uint32) (*invByStatus, error) { +// InventoryByStatus returns a page of tokens for the provided status. If no +// status is provided then a page for each status will be returned. +func (p *ticketVotePlugin) InventoryByStatus(bestBlock uint32, s ticketvote.VoteStatusT, page uint32) (*invByStatus, error) { pageSize := ticketvote.InventoryPageSize // If no status is provided a page of tokens for each status should @@ -346,7 +359,7 @@ func (p *ticketVotePlugin) invByStatus(bestBlock uint32, s ticketvote.VoteStatus } // A status was provided. Return a page of tokens for the status. - inv, err := p.inventory(bestBlock) + inv, err := p.Inventory(bestBlock) if err != nil { return nil, err } diff --git a/politeiad/backend/tstorebe/plugins/ticketvote/ticketvote.go b/politeiad/backend/tstorebe/plugins/ticketvote/ticketvote.go index bc0c0a2b7..eb8fc358f 100644 --- a/politeiad/backend/tstorebe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backend/tstorebe/plugins/ticketvote/ticketvote.go @@ -83,9 +83,9 @@ func (p *ticketVotePlugin) Setup() error { if err != nil { return fmt.Errorf("bestBlock: %v", err) } - inv, err := p.inventory(bestBlock) + inv, err := p.Inventory(bestBlock) if err != nil { - return fmt.Errorf("inventory: %v", err) + return fmt.Errorf("Inventory: %v", err) } // Build active votes cache diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index 0589f81ca..da43d020b 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -456,6 +456,10 @@ const ( // is only used when the vote type allows for a clear approved or // rejected outcome. VoteStatusRejected VoteStatusT = 6 + + // VoteStatusIneligible indicates that a record is not eligible to + // be voted on. This happens when a record is censored or archived. + VoteStatusIneligible VoteStatusT = 7 ) var ( @@ -468,6 +472,7 @@ var ( VoteStatusFinished: "finished", VoteStatusApproved: "approved", VoteStatusRejected: "rejected", + VoteStatusIneligible: "ineligible", } ) diff --git a/politeiawww/api/ticketvote/v1/v1.go b/politeiawww/api/ticketvote/v1/v1.go index a3adec6bf..3b5e50b16 100644 --- a/politeiawww/api/ticketvote/v1/v1.go +++ b/politeiawww/api/ticketvote/v1/v1.go @@ -210,6 +210,10 @@ const ( // is only used when the vote type allows for a clear approved or // rejected outcome. VoteStatusRejected VoteStatusT = 6 + + // VoteStatusIneligible indicates that a record is not eligible to + // be voted on. This happens when a record is censored or archived. + VoteStatusIneligible VoteStatusT = 7 ) var ( @@ -222,6 +226,7 @@ var ( VoteStatusFinished: "finished", VoteStatusApproved: "approved", VoteStatusRejected: "rejected", + VoteStatusIneligible: "ineligible", } ) From 08b76e96f4bf9a4756f2fdf5a8c9924955dc41a3 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 1 Mar 2021 15:59:56 -0600 Subject: [PATCH 357/449] ticketvote: Fix bug in set status hook. --- .../tstorebe/plugins/ticketvote/hooks.go | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/politeiad/backend/tstorebe/plugins/ticketvote/hooks.go b/politeiad/backend/tstorebe/plugins/ticketvote/hooks.go index 82ac24e39..54c874fd4 100644 --- a/politeiad/backend/tstorebe/plugins/ticketvote/hooks.go +++ b/politeiad/backend/tstorebe/plugins/ticketvote/hooks.go @@ -311,28 +311,34 @@ func (p *ticketVotePlugin) hookSetRecordStatusPost(treeID int64, payload string) if err != nil { return err } - status := srs.RecordMetadata.Status + var ( + oldStatus = srs.Current.RecordMetadata.Status + newStatus = srs.RecordMetadata.Status + ) // When a record is moved to vetted the plugin hooks are executed // on both the unvetted and vetted tstore instances. We only need // to update cached data if this is the vetted instance. We can // determine this by checking if the record exists. The unvetted // instance will return false. - if status == backend.MDStatusVetted && !p.tstore.RecordExists(treeID) { + if newStatus == backend.MDStatusVetted && !p.tstore.RecordExists(treeID) { // This is the unvetted instance. Nothing to do. return nil } // Update the inventory cache - switch status { + switch newStatus { case backend.MDStatusVetted: // Add to inventory p.InventoryAdd(srs.RecordMetadata.Token, ticketvote.VoteStatusUnauthorized) case backend.MDStatusCensored, backend.MDStatusArchived: // These statuses do not allow for a vote. Mark as ineligible. - p.InventoryUpdate(srs.RecordMetadata.Token, - ticketvote.VoteStatusIneligible) + // We only need to do this if the record is vetted. + if oldStatus == backend.MDStatusVetted { + p.InventoryUpdate(srs.RecordMetadata.Token, + ticketvote.VoteStatusIneligible) + } } // Update the submissions cache if the linkto has been set. @@ -347,7 +353,7 @@ func (p *ticketVotePlugin) hookSetRecordStatusPost(treeID int64, payload string) parentToken = vm.LinkTo childToken = srs.RecordMetadata.Token ) - switch status { + switch newStatus { case backend.MDStatusVetted: // Record has been made public. Add child token to parent's // submissions list. @@ -357,10 +363,13 @@ func (p *ticketVotePlugin) hookSetRecordStatusPost(treeID int64, payload string) } case backend.MDStatusCensored: // Record has been censored. Delete child token from parent's - // submissions list. - err := p.submissionsCacheDel(parentToken, childToken) - if err != nil { - return fmt.Errorf("submissionsCacheDel: %v", err) + // submissions list. We only need to do this if the record is + // vetted. + if oldStatus == backend.MDStatusVetted { + err := p.submissionsCacheDel(parentToken, childToken) + if err != nil { + return fmt.Errorf("submissionsCacheDel: %v", err) + } } } } From b171b6a7339025b6bb26a40caf4c0b4f1649facb Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 1 Mar 2021 16:45:46 -0600 Subject: [PATCH 358/449] tstore: Add RecordPartial to TstoreClient. --- politeiad/api/v1/v1.go | 18 +++++------ politeiad/backend/backend.go | 16 +++++----- politeiad/backend/tstorebe/plugins/plugins.go | 18 +++++++++++ politeiad/backend/tstorebe/tstore/tstore.go | 31 ++++++++++++------- politeiad/backend/tstorebe/tstorebe.go | 4 +-- politeiad/plugins/ticketvote/ticketvote.go | 2 +- politeiad/politeiad.go | 8 ++--- politeiawww/api/records/v1/v1.go | 24 ++++++++------ politeiawww/records/process.go | 7 +++-- 9 files changed, 81 insertions(+), 47 deletions(-) diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index c79a71a4d..19d9b1389 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Decred developers +// Copyright (c) 2017-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -644,16 +644,16 @@ type PluginCommandBatchReply struct { // Version is used to request a specific version of a record. If no version is // provided then the most recent version of the record will be returned. // -// OmitFiles can be used to retrieve a record without any of the record files -// being returned. +// Filenames can be used to request specific files. If filenames is not empty +// then the specified files will be the only files returned. // -// Filenames can be used to request specific files. When filenames is not -// empty, the only files that are returned will be those that are specified. +// OmitAllFiles can be used to retrieve a record without any of the record +// files. This supersedes the filenames argument. type RecordRequest struct { - Token string `json:"token"` - Version string `json:"version,omitempty"` - OmitFiles bool `json:"omitfiles,omitempty"` - Filenames []string `json:"filenames,omitempty"` + Token string `json:"token"` + Version string `json:"version,omitempty"` + Filenames []string `json:"filenames,omitempty"` + OmitAllFiles bool `json:"omitallfiles,omitempty"` } // GetUnvettedBatch requests a batch of unvetted records. diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index 27955fb09..cd2e363a7 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -153,16 +153,16 @@ type Record struct { // Version is used to request a specific version of a record. If no version is // provided then the most recent version of the record will be returned. // -// OmitFiles can be used to retrieve a record without any of the record files -// being returned. +// Filenames can be used to request specific files. If filenames is not empty +// then the specified files will be the only files returned. // -// Filenames can be used to request specific files. When filenames is not -// empty, the only files that are returned will be those that are specified. +// OmitAllFiles can be used to retrieve a record without any of the record +// files. This supersedes the filenames argument. type RecordRequest struct { - Token []byte `json:"token"` - Version string `json:"version,omitempty"` - OmitFiles bool `json:"omitfiles,omitempty"` - Filenames []string `json:"filenames,omitempty"` + Token []byte + Version string + Filenames []string + OmitAllFiles bool } // Proof contains an inclusion proof for the digest in the merkle root. All diff --git a/politeiad/backend/tstorebe/plugins/plugins.go b/politeiad/backend/tstorebe/plugins/plugins.go index abfeef3cc..1ac11c3c4 100644 --- a/politeiad/backend/tstorebe/plugins/plugins.go +++ b/politeiad/backend/tstorebe/plugins/plugins.go @@ -214,4 +214,22 @@ type TstoreClient interface { // RecordLatest returns the most recent version of a record. RecordLatest(treeID int64) (*backend.Record, error) + + // RecordPartial returns a partial record. This method gives the + // caller fine grained control over what version and what files are + // returned. The only required field is the token. All other fields + // are optional. + // + // Version is used to request a specific version of a record. If no + // version is provided then the most recent version of the record + // will be returned. + // + // Filenames can be used to request specific files. If filenames is + // not empty then the specified files will be the only files that + // are returned. + // + // OmitAllFiles can be used to retrieve a record without any of the + // record files. This supersedes the filenames argument. + RecordPartial(treeID int64, version uint32, filenames []string, + omitAllFiles bool) (*backend.Record, error) } diff --git a/politeiad/backend/tstorebe/tstore/tstore.go b/politeiad/backend/tstorebe/tstore/tstore.go index 831342813..2b5ff8866 100644 --- a/politeiad/backend/tstorebe/tstore/tstore.go +++ b/politeiad/backend/tstorebe/tstore/tstore.go @@ -953,12 +953,12 @@ printErr: // Version is used to request a specific version of a record. If no version is // provided then the most recent version of the record will be returned. // -// OmitFiles can be used to retrieve a record without any of the record files -// being returned. +// Filenames can be used to request specific files. If filenames is not empty +// then the specified files will be the only files returned. // -// Filenames can be used to request specific files. When filenames is not -// empty, the only files that are returned will be those that are specified. -func (t *Tstore) record(treeID int64, version uint32, omitFiles bool, filenames []string) (*backend.Record, error) { +// OmitAllFiles can be used to retrieve a record without any of the record +// files. This supersedes the filenames argument. +func (t *Tstore) record(treeID int64, version uint32, filenames []string, omitAllFiles bool) (*backend.Record, error) { // Verify tree exists if !t.TreeExists(treeID) { return nil, backend.ErrRecordNotFound @@ -1004,7 +1004,7 @@ func (t *Tstore) record(treeID int64, version uint32, omitFiles bool, filenames merkles[hex.EncodeToString(v)] = struct{}{} } switch { - case omitFiles: + case omitAllFiles: // Don't include any files case len(filenames) > 0: // Only included the specified files @@ -1147,24 +1147,33 @@ func (t *Tstore) record(treeID int64, version uint32, omitFiles bool, filenames func (t *Tstore) Record(treeID int64, version uint32) (*backend.Record, error) { log.Tracef("%v record: %v %v", t.id, treeID, version) - return t.record(treeID, version, false, []string{}) + return t.record(treeID, version, []string{}, false) } // RecordLatest returns the latest version of a record. func (t *Tstore) RecordLatest(treeID int64) (*backend.Record, error) { log.Tracef("%v RecordLatest: %v", t.id, treeID) - return t.record(treeID, 0, false, []string{}) + return t.record(treeID, 0, []string{}, false) } // RecordPartial returns a partial record. This method gives the caller fine // grained control over what version and what files are returned. The only // required field is the token. All other fields are optional. -func (t *Tstore) RecordPartial(treeID int64, version uint32, omitFiles bool, filenames []string) (*backend.Record, error) { +// +// Version is used to request a specific version of a record. If no version is +// provided then the most recent version of the record will be returned. +// +// Filenames can be used to request specific files. If filenames is not empty +// then the specified files will be the only files returned. +// +// OmitAllFiles can be used to retrieve a record without any of the record +// files. This supersedes the filenames argument. +func (t *Tstore) RecordPartial(treeID int64, version uint32, filenames []string, omitAllFiles bool) (*backend.Record, error) { log.Tracef("%v RecordPartial: %v %v %v %v", - t.id, treeID, version, omitFiles, filenames) + t.id, treeID, version, omitAllFiles, filenames) - return t.record(treeID, version, omitFiles, filenames) + return t.record(treeID, version, filenames, omitAllFiles) } func (t *Tstore) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trillian.LogLeaf) (*backend.Timestamp, error) { diff --git a/politeiad/backend/tstorebe/tstorebe.go b/politeiad/backend/tstorebe/tstorebe.go index 67813503a..a4da80bf6 100644 --- a/politeiad/backend/tstorebe/tstorebe.go +++ b/politeiad/backend/tstorebe/tstorebe.go @@ -1444,7 +1444,7 @@ func (t *tstoreBackend) GetUnvettedBatch(reqs []backend.RecordRequest) (map[stri // Get the record r, err := t.unvetted.RecordPartial(treeID, version, - v.OmitFiles, v.Filenames) + v.Filenames, v.OmitAllFiles) if err != nil { if err == backend.ErrRecordNotFound { // Record doesn't exist. This is ok. It will not be included @@ -1496,7 +1496,7 @@ func (t *tstoreBackend) GetVettedBatch(reqs []backend.RecordRequest) (map[string // Get the record r, err := t.vetted.RecordPartial(treeID, version, - v.OmitFiles, v.Filenames) + v.Filenames, v.OmitAllFiles) if err != nil { if err == backend.ErrRecordNotFound { // Record doesn't exist. This is ok. It will not be included diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index da43d020b..58777af18 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -43,7 +43,7 @@ const ( // SettingTestNeLinkByPeriodMin is the default minimum amount of // time, in seconds, that the link by period can be set to. This // value of 1 second was chosen because this is the testnet - // default and a 1 second miniumum makes testing various scenerios + // default and a 1 second miniumum makes testing various scenarios // easier. SettingTestNetLinkByPeriodMin int64 = 1 diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 48ad3f221..59a9ddf0c 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -626,10 +626,10 @@ func (p *politeia) getVetted(w http.ResponseWriter, r *http.Request) { func convertFrontendRecordRequest(r v1.RecordRequest) backend.RecordRequest { token, _ := util.ConvertStringToken(r.Token) return backend.RecordRequest{ - Token: token, - Version: r.Version, - OmitFiles: r.OmitFiles, - Filenames: r.Filenames, + Token: token, + Version: r.Version, + Filenames: r.Filenames, + OmitAllFiles: r.OmitAllFiles, } } diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go index 7412fa104..46b5dd7b2 100644 --- a/politeiawww/api/records/v1/v1.go +++ b/politeiawww/api/records/v1/v1.go @@ -295,17 +295,23 @@ const ( RecordsPageSize = 5 ) -// RecordRequest requests a record and gives the client more granular control -// of parts of the record are returned. The only required field is the token. -// All other fields are optional. +// RecordRequest is used to requests a record. It gives the client granular +// control over what is returned. The only required field is the token. All +// other fields are optional. // -// Only the record metadata, without any record files, will be returned by -// default. The client can request specific files be returned by specifying -// them in the Filenames field. +// Version is used to request a specific version of a record. If no version is +// provided then the most recent version of the record will be returned. +// +// Filenames can be used to request specific files. If filenames is not empty +// then the specified files will be the only files returned. +// +// OmitAllFiles can be used to retrieve a record without any of the record +// files. This supersedes the filenames argument. type RecordRequest struct { - Token string `json:"token"` - Version string `json:"version,omitempty"` - Filenames []string `json:"filenames,omitempty"` + Token string `json:"token"` + Version string `json:"version,omitempty"` + Filenames []string `json:"filenames,omitempty"` + OmitAllFiles bool `json:"omitallfiles,omitempty"` } // Records requests a batch of records. diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index 89d39d304..69300c80d 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -752,9 +752,10 @@ func convertRequestsToPD(reqs []v1.RecordRequest) []pdv1.RecordRequest { r := make([]pdv1.RecordRequest, 0, len(reqs)) for _, v := range reqs { r = append(r, pdv1.RecordRequest{ - Token: v.Token, - Version: v.Version, - Filenames: v.Filenames, + Token: v.Token, + Version: v.Version, + Filenames: v.Filenames, + OmitAllFiles: v.OmitAllFiles, }) } return r From 546bc1dd33d7f6229579168ff1a848de851b3c6a Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 1 Mar 2021 17:32:43 -0600 Subject: [PATCH 359/449] www/ticketvote: Add pd user errors. --- politeiawww/api/ticketvote/v1/v1.go | 6 ++ politeiawww/cmd/pictl/cmdvoteauthorize.go | 36 ++++--- politeiawww/records/error.go | 110 +++++++++++---------- politeiawww/ticketvote/error.go | 114 ++++++++++++++-------- 4 files changed, 160 insertions(+), 106 deletions(-) diff --git a/politeiawww/api/ticketvote/v1/v1.go b/politeiawww/api/ticketvote/v1/v1.go index 3b5e50b16..63eb8f082 100644 --- a/politeiawww/api/ticketvote/v1/v1.go +++ b/politeiawww/api/ticketvote/v1/v1.go @@ -31,6 +31,9 @@ const ( ErrorCodeInputInvalid ErrorCodeT = 1 ErrorCodePublicKeyInvalid ErrorCodeT = 2 ErrorCodeUnauthorized ErrorCodeT = 3 + ErrorCodeRecordNotFound ErrorCodeT = 4 + ErrorCodeRecordLocked ErrorCodeT = 5 + ErrorCodeTokenInvalid ErrorCodeT = 6 ) var ( @@ -40,6 +43,9 @@ var ( ErrorCodeInputInvalid: "input invalid", ErrorCodePublicKeyInvalid: "public key invalid", ErrorCodeUnauthorized: "unauthorized", + ErrorCodeRecordNotFound: "record not found", + ErrorCodeRecordLocked: "record locked", + ErrorCodeTokenInvalid: "token is invalid", } ) diff --git a/politeiawww/cmd/pictl/cmdvoteauthorize.go b/politeiawww/cmd/pictl/cmdvoteauthorize.go index a8b51d9f3..7fad6b86d 100644 --- a/politeiawww/cmd/pictl/cmdvoteauthorize.go +++ b/politeiawww/cmd/pictl/cmdvoteauthorize.go @@ -20,8 +20,9 @@ import ( // authorization. type cmdVoteAuthorize struct { Args struct { - Token string `positional-arg-name:"token" required:"true"` - Action string `positional-arg-name:"action"` + Token string `positional-arg-name:"token" required:"true"` + Action string `positional-arg-name:"action"` + Version uint32 `positional-arg-name:"version"` } `positional-args:"true"` } @@ -63,25 +64,30 @@ func (c *cmdVoteAuthorize) Execute(args []string) error { } // Get record version - d := rcv1.Details{ - State: rcv1.RecordStateVetted, - Token: c.Args.Token, - } - r, err := pc.RecordDetails(d) - if err != nil { - return err - } - version, err := strconv.ParseUint(r.Version, 10, 64) - if err != nil { - return err + version := c.Args.Version + if version == 0 { + d := rcv1.Details{ + State: rcv1.RecordStateVetted, + Token: c.Args.Token, + } + r, err := pc.RecordDetails(d) + if err != nil { + return err + } + u, err := strconv.ParseUint(r.Version, 10, 64) + if err != nil { + return err + } + version = uint32(u) } // Setup request - msg := c.Args.Token + r.Version + string(action) + msg := c.Args.Token + strconv.FormatUint(uint64(version), 10) + + string(action) sig := cfg.Identity.SignMessage([]byte(msg)) a := tkv1.Authorize{ Token: c.Args.Token, - Version: uint32(version), + Version: version, Action: action, PublicKey: cfg.Identity.Public.String(), Signature: hex.EncodeToString(sig[:]), diff --git a/politeiawww/records/error.go b/politeiawww/records/error.go index b2bb678b9..4464cee10 100644 --- a/politeiawww/records/error.go +++ b/politeiawww/records/error.go @@ -58,59 +58,7 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err case errors.As(err, &pde): // Politeiad error - var ( - pluginID = pde.ErrorReply.PluginID - errCode = pde.ErrorReply.ErrorCode - errContext = strings.Join(pde.ErrorReply.ErrorContext, ",") - ) - e := convertPDErrorCode(errCode) - switch { - case pluginID != "": - // politeiad plugin error. Log it and return a 400. - m := fmt.Sprintf("%v Plugin error: %v %v", - util.RemoteAddr(r), pluginID, errCode) - if errContext != "" { - m += fmt.Sprintf(": %v", errContext) - } - log.Infof(m) - util.RespondWithJSON(w, http.StatusBadRequest, - v1.PluginErrorReply{ - PluginID: pluginID, - ErrorCode: errCode, - ErrorContext: errContext, - }) - return - - case e == v1.ErrorCodeInvalid: - // politeiad error does not correspond to a user error. Log it - // and return a 500. - ts := time.Now().Unix() - log.Errorf("%v %v %v %v Internal error %v: error code "+ - "from politeiad: %v", util.RemoteAddr(r), r.Method, r.URL, - r.Proto, ts, errCode) - - util.RespondWithJSON(w, http.StatusInternalServerError, - v1.ServerErrorReply{ - ErrorCode: ts, - }) - return - - default: - // User error from politeiad that corresponds to a records user - // error. Log it and return a 400. - m := fmt.Sprintf("%v Records user error: %v %v", - util.RemoteAddr(r), e, v1.ErrorCodes[e]) - if errContext != "" { - m += fmt.Sprintf(": %v", errContext) - } - log.Infof(m) - util.RespondWithJSON(w, http.StatusBadRequest, - v1.UserErrorReply{ - ErrorCode: e, - ErrorContext: errContext, - }) - return - } + handlePoliteiadError(w, r, format, pde) default: // Internal server error. Log it and return a 500. @@ -128,6 +76,62 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err } } +func handlePoliteiadError(w http.ResponseWriter, r *http.Request, format string, pde pdclient.Error) { + var ( + pluginID = pde.ErrorReply.PluginID + errCode = pde.ErrorReply.ErrorCode + errContext = strings.Join(pde.ErrorReply.ErrorContext, ",") + ) + e := convertPDErrorCode(errCode) + switch { + case pluginID != "": + // politeiad plugin error. Log it and return a 400. + m := fmt.Sprintf("%v Plugin error: %v %v", + util.RemoteAddr(r), pluginID, errCode) + if errContext != "" { + m += fmt.Sprintf(": %v", errContext) + } + log.Infof(m) + util.RespondWithJSON(w, http.StatusBadRequest, + v1.PluginErrorReply{ + PluginID: pluginID, + ErrorCode: errCode, + ErrorContext: errContext, + }) + return + + case e != v1.ErrorCodeInvalid: + // User error from politeiad that corresponds to a records user + // error. Log it and return a 400. + m := fmt.Sprintf("%v Records user error: %v %v", + util.RemoteAddr(r), e, v1.ErrorCodes[e]) + if errContext != "" { + m += fmt.Sprintf(": %v", errContext) + } + log.Infof(m) + util.RespondWithJSON(w, http.StatusBadRequest, + v1.UserErrorReply{ + ErrorCode: e, + ErrorContext: errContext, + }) + return + + default: + // politeiad error does not correspond to a user error. Log it + // and return a 500. + ts := time.Now().Unix() + log.Errorf("%v %v %v %v Internal error %v: error code "+ + "from politeiad: %v", util.RemoteAddr(r), r.Method, r.URL, + r.Proto, ts, errCode) + + util.RespondWithJSON(w, http.StatusInternalServerError, + v1.ServerErrorReply{ + ErrorCode: ts, + }) + return + } +} + func convertPDErrorCode(errCode int) v1.ErrorCodeT { // Any error statuses that are intentionally omitted means that // politeiawww should 500. diff --git a/politeiawww/ticketvote/error.go b/politeiawww/ticketvote/error.go index ab46cfc24..68b94a8c0 100644 --- a/politeiawww/ticketvote/error.go +++ b/politeiawww/ticketvote/error.go @@ -12,6 +12,7 @@ import ( "strings" "time" + pdv1 "github.com/decred/politeia/politeiad/api/v1" pdclient "github.com/decred/politeia/politeiad/client" v1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" "github.com/decred/politeia/util" @@ -19,8 +20,8 @@ import ( func respondWithError(w http.ResponseWriter, r *http.Request, format string, err error) { var ( - ue v1.UserErrorReply - pe pdclient.Error + ue v1.UserErrorReply + pde pdclient.Error ) switch { case errors.As(err, &ue): @@ -38,43 +39,9 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err }) return - case errors.As(err, &pe): + case errors.As(err, &pde): // Politeiad error - var ( - pluginID = pe.ErrorReply.PluginID - errCode = pe.ErrorReply.ErrorCode - errContext = pe.ErrorReply.ErrorContext - ) - switch { - case pluginID != "": - // Politeiad plugin error. Log it and return a 400. - m := fmt.Sprintf("Plugin error: %v %v %v", - util.RemoteAddr(r), pluginID, errCode) - if len(errContext) > 0 { - m += fmt.Sprintf(": %v", strings.Join(errContext, ", ")) - } - log.Infof(m) - util.RespondWithJSON(w, http.StatusBadRequest, - v1.PluginErrorReply{ - PluginID: pluginID, - ErrorCode: errCode, - ErrorContext: strings.Join(errContext, ", "), - }) - return - - default: - // Unknown politeiad error. Log it and return a 500. - ts := time.Now().Unix() - log.Errorf("%v %v %v %v Internal error %v: error code "+ - "from politeiad: %v", util.RemoteAddr(r), r.Method, r.URL, - r.Proto, ts, errCode) - - util.RespondWithJSON(w, http.StatusInternalServerError, - v1.ServerErrorReply{ - ErrorCode: ts, - }) - return - } + handlePoliteiadError(w, r, format, pde) default: // Internal server error. Log it and return a 500. @@ -91,3 +58,74 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err return } } + +func handlePoliteiadError(w http.ResponseWriter, r *http.Request, format string, pde pdclient.Error) { + var ( + pluginID = pde.ErrorReply.PluginID + errCode = pde.ErrorReply.ErrorCode + errContext = strings.Join(pde.ErrorReply.ErrorContext, ",") + ) + e := convertPDErrorCode(errCode) + switch { + case pluginID != "": + // politeiad plugin error. Log it and return a 400. + m := fmt.Sprintf("%v Plugin error: %v %v", + util.RemoteAddr(r), pluginID, errCode) + if errContext != "" { + m += fmt.Sprintf(": %v", errContext) + } + log.Infof(m) + util.RespondWithJSON(w, http.StatusBadRequest, + v1.PluginErrorReply{ + PluginID: pluginID, + ErrorCode: errCode, + ErrorContext: errContext, + }) + return + + case e != v1.ErrorCodeInvalid: + // User error from politeiad that corresponds to a records user + // error. Log it and return a 400. + m := fmt.Sprintf("%v Ticketvote user error: %v %v", + util.RemoteAddr(r), e, v1.ErrorCodes[e]) + if errContext != "" { + m += fmt.Sprintf(": %v", errContext) + } + log.Infof(m) + util.RespondWithJSON(w, http.StatusBadRequest, + v1.UserErrorReply{ + ErrorCode: e, + ErrorContext: errContext, + }) + return + + default: + // politeiad error does not correspond to a user error. Log it + // and return a 500. + ts := time.Now().Unix() + log.Errorf("%v %v %v %v Internal error %v: error code "+ + "from politeiad: %v", util.RemoteAddr(r), r.Method, r.URL, + r.Proto, ts, errCode) + + util.RespondWithJSON(w, http.StatusInternalServerError, + v1.ServerErrorReply{ + ErrorCode: ts, + }) + return + } +} + +func convertPDErrorCode(errCode int) v1.ErrorCodeT { + // This list is only populated with politeiad errors that we expect + // for the ticketvote plugin commands. Any politeiad errors not + // included in this list will cause politeiawww to 500. + switch pdv1.ErrorStatusT(errCode) { + case pdv1.ErrorStatusRecordNotFound: + return v1.ErrorCodeRecordNotFound + case pdv1.ErrorStatusInvalidToken: + return v1.ErrorCodeTokenInvalid + case pdv1.ErrorStatusRecordLocked: + return v1.ErrorCodeRecordLocked + } + return v1.ErrorCodeInvalid +} From e41c9c44991040f05dc295aac7d04eecc81ce0cd Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 1 Mar 2021 19:12:55 -0600 Subject: [PATCH 360/449] Get build passing. --- politeiad/api/v1/v1.go | 6 +- .../tstorebe/plugins/comments/comments.go | 4 - .../plugins/comments/comments_test.go | 163 -- politeiad/backend/tstorebe/plugins/pi/pi.go | 2 - politeiad/backend/tstorebe/plugins/plugins.go | 4 +- .../plugins/ticketvote/activevotes.go | 2 +- .../tstorebe/plugins/ticketvote/cmds.go | 3 + politeiad/backend/tstorebe/tstore/plugin.go | 3 + politeiad/backend/tstorebe/tstore_test.go | 1662 ----------------- politeiad/backend/tstorebe/tstorebe.go | 4 +- politeiad/client/client.go | 6 +- politeiad/client/user.go | 2 +- politeiad/plugins/comments/comments.go | 2 +- politeiad/plugins/ticketvote/ticketvote.go | 2 +- politeiad/politeiad.go | 2 +- politeiawww/client/error.go | 2 +- politeiawww/cmd/pictl/README.md | 32 +- politeiawww/cmd/pictl/cmdcommentcensor.go | 2 +- politeiawww/cmd/pictl/cmdcommentvotes.go | 2 +- politeiawww/cmd/pictl/cmdproposaldetails.go | 2 +- politeiawww/cmd/pictl/cmdproposals.go | 2 +- politeiawww/cmd/pictl/cmdvotetest.go | 2 +- .../cmd/politeiaverify/politeiaverify.go | 90 +- politeiawww/cmd/shared/config.go | 1 - politeiawww/comments/process.go | 3 + politeiawww/pi/events.go | 2 +- politeiawww/pi/pi.go | 2 +- politeiawww/pi/process.go | 2 +- politeiawww/politeiawww_test.go | 3 + politeiawww/proposals.go | 11 +- politeiawww/records/error.go | 4 +- politeiawww/records/process.go | 8 +- politeiawww/testing.go | 11 - politeiawww/ticketvote/error.go | 4 +- politeiawww/userwww_test.go | 24 +- politeiawww/www.go | 4 +- util/net.go | 4 +- util/regexp.go | 2 +- util/token.go | 3 +- 39 files changed, 133 insertions(+), 1956 deletions(-) delete mode 100644 politeiad/backend/tstorebe/plugins/comments/comments_test.go delete mode 100644 politeiad/backend/tstorebe/tstore_test.go diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index 19d9b1389..3a38fc110 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -619,7 +619,7 @@ type PluginCommandReplyV2 struct { // before the plugin command could be executed. UserError *UserErrorReply `json:"usererror,omitempty"` - // PluginError will be populated if a plugin error occured during + // PluginError will be populated if a plugin error occurred during // plugin command execution. These errors will be specific to the // plugin. PluginError *PluginErrorReply `json:"pluginerror,omitempty"` @@ -663,7 +663,7 @@ type GetUnvettedBatch struct { } // GetUnvettedBatchReply is the reply to the GetUnvettedBatch command. If a -// record was not found or an error occured while retrieving it the token will +// record was not found or an error occurred while retrieving it the token will // not be included in the returned map. type GetUnvettedBatchReply struct { Response string `json:"response"` // Challenge response @@ -677,7 +677,7 @@ type GetVettedBatch struct { } // GetVettedBatchReply is the reply to the GetVettedBatch command. If a record -// was not found or an error occured while retrieving it the token will not be +// was not found or an error occurred while retrieving it the token will not be // included in the returned map. type GetVettedBatchReply struct { Response string `json:"response"` // Challenge response diff --git a/politeiad/backend/tstorebe/plugins/comments/comments.go b/politeiad/backend/tstorebe/plugins/comments/comments.go index 9ce6dc2ac..ca08a62ef 100644 --- a/politeiad/backend/tstorebe/plugins/comments/comments.go +++ b/politeiad/backend/tstorebe/plugins/comments/comments.go @@ -17,10 +17,6 @@ import ( "github.com/decred/politeia/politeiad/plugins/comments" ) -// TODO prevent duplicate comments -// TODO upvoting a comment twice in the same second causes a duplicate leaf -// error which causes a 500. Solution: add the timestamp to the vote index. - var ( _ plugins.PluginClient = (*commentsPlugin)(nil) ) diff --git a/politeiad/backend/tstorebe/plugins/comments/comments_test.go b/politeiad/backend/tstorebe/plugins/comments/comments_test.go deleted file mode 100644 index 67146c6d7..000000000 --- a/politeiad/backend/tstorebe/plugins/comments/comments_test.go +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright (c) 2020-2021 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package comments - -import ( - "encoding/hex" - "encoding/json" - "errors" - "io/ioutil" - "os" - "strconv" - "testing" - - "github.com/decred/politeia/politeiad/api/v1/identity" - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tstorebe/plugins" - "github.com/decred/politeia/politeiad/plugins/comments" - "github.com/google/uuid" -) - -func commentSignature(t *testing.T, fid *identity.FullIdentity, token []byte, parentID uint32, comment string) string { - t.Helper() - tk := hex.EncodeToString(token) - msg := tk + strconv.FormatInt(int64(parentID), 10) + comment - b := fid.SignMessage([]byte(msg)) - return hex.EncodeToString(b[:]) -} - -func commentDelSignature(t *testing.T, fid *identity.FullIdentity, token []byte, parentID uint32, reason string) string { - t.Helper() - tk := hex.EncodeToString(token) - msg := tk + strconv.FormatInt(int64(parentID), 10) + reason - b := fid.SignMessage([]byte(msg)) - return hex.EncodeToString(b[:]) -} - -// newTestCommentsPlugin returns a commentsPlugin that is setup for testing and -// a closure that cleans up the test data when invoked. -func newTestCommentsPlugin(t *testing.T) (*commentsPlugin, func()) { - t.Helper() - - // Setup data dir - dataDir, err := ioutil.TempDir("", "tstorebe.comments.test") - if err != nil { - t.Fatal(err) - } - - // TODO Implement a test clients.TstoreClient - // Setup tstore client - var tstore plugins.TstoreClient - - // Setup plugin identity - fid, err := identity.New() - if err != nil { - t.Fatal(err) - } - - // Setup comment plugins - c, err := New(tstore, []backend.PluginSetting{}, dataDir, fid) - if err != nil { - t.Fatal(err) - } - - return c, func() { - err = os.RemoveAll(dataDir) - if err != nil { - t.Fatal(err) - } - } -} - -func TestCmdNew(t *testing.T) { - p, cleanup := newTestCommentsPlugin(t) - defer cleanup() - - // Setup test data - fid, err := identity.New() - if err != nil { - t.Fatal(err) - } - var ( - treeID int64 - token []byte - - userID = uuid.New().String() - publicKey = fid.Public.String() - comment = "This is a comment." - - parentIDZero uint32 - // parerntIDInvalid uint32 = 99 - ) - - // Setup test cases - var tests = []struct { - description string - treeID int64 - token []byte - payload comments.New - wantErr error - wantReply string - }{ - { - "invalid token", - treeID, - token, - comments.New{ - UserID: userID, - Token: "invalid", - ParentID: parentIDZero, - Comment: comment, - PublicKey: publicKey, - Signature: commentSignature(t, fid, token, parentIDZero, comment), - }, - backend.PluginError{ - PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeTokenInvalid), - }, - "", - }, - } - - // Run test cases - for _, tc := range tests { - t.Run(tc.description, func(t *testing.T) { - // Encode payload - payload, err := json.Marshal(tc.payload) - if err != nil { - t.Fatal(err) - } - - // Execute plugin command - reply, err := p.cmdNew(tc.treeID, tc.token, string(payload)) - - if tc.wantErr != nil { - // We expect an error. Verify that the returned error is - // correct. - want := tc.wantErr.(backend.PluginError) - var ue backend.PluginError - switch { - case errors.As(err, &ue) && - want.PluginID == ue.PluginID && - want.ErrorCode == ue.ErrorCode: - // This is correct. Next test case. - return - default: - // Unexpected error - t.Errorf("got error %v, want error %v", err, tc.wantErr) - } - } - - // We expect a valid reply. Verify the reply. - var nr comments.NewReply - err = json.Unmarshal([]byte(reply), &nr) - if err != nil { - t.Errorf("invalid NewReply: %v", reply) - } - - // TODO Verify reply payload - }) - } -} diff --git a/politeiad/backend/tstorebe/plugins/pi/pi.go b/politeiad/backend/tstorebe/plugins/pi/pi.go index 355f3075e..5766fba38 100644 --- a/politeiad/backend/tstorebe/plugins/pi/pi.go +++ b/politeiad/backend/tstorebe/plugins/pi/pi.go @@ -48,8 +48,6 @@ type piPlugin struct { func (p *piPlugin) Setup() error { log.Tracef("pi Setup") - // TODO Verify vote and comment plugin dependency - return nil } diff --git a/politeiad/backend/tstorebe/plugins/plugins.go b/politeiad/backend/tstorebe/plugins/plugins.go index 1ac11c3c4..ffcde0962 100644 --- a/politeiad/backend/tstorebe/plugins/plugins.go +++ b/politeiad/backend/tstorebe/plugins/plugins.go @@ -86,7 +86,7 @@ const ( ) // HookNewRecordPre is the payload for the pre new record hook. The record -// state is not inlcuded since all new records will have a record state of +// state is not included since all new records will have a record state of // unvetted. type HookNewRecordPre struct { Metadata []backend.MetadataStream `json:"metadata"` @@ -94,7 +94,7 @@ type HookNewRecordPre struct { } // HookNewRecordPost is the payload for the post new record hook. The record -// state is not inlcuded since all new records will have a record state of +// state is not included since all new records will have a record state of // unvetted. RecordMetadata is only be present on the post new record hook // since the record metadata requires the creation of a trillian tree and the // pre new record hook should execute before any politeiad state is changed in diff --git a/politeiad/backend/tstorebe/plugins/ticketvote/activevotes.go b/politeiad/backend/tstorebe/plugins/ticketvote/activevotes.go index 03208dfd6..aa6d1ffea 100644 --- a/politeiad/backend/tstorebe/plugins/ticketvote/activevotes.go +++ b/politeiad/backend/tstorebe/plugins/ticketvote/activevotes.go @@ -49,7 +49,7 @@ type activeVote struct { // premise dcrdata instance with minimal latency. Any functions // that rely of this cache should fallback to fetching the // commitment addresses manually in the event the cache has not - // been fully populated yet or has experienced unforseen errors + // been fully populated yet or has experienced unforeseen errors // during creation (ex. network errors). If the initial job fails // to complete it will not be retried. Addrs map[string]string // [ticket]address diff --git a/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go b/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go index 94656ff81..99987ea26 100644 --- a/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go @@ -442,6 +442,9 @@ func (p *ticketVotePlugin) voteOptionResults(token []byte, options []ticketvote. // Votes are not in the cache. Pull them from the backend. reply, err := p.backend.VettedPluginCmd(backend.PluginActionRead, token, ticketvote.PluginID, ticketvote.CmdResults, "") + if err != nil { + return nil, err + } var rr ticketvote.ResultsReply err = json.Unmarshal([]byte(reply), &rr) if err != nil { diff --git a/politeiad/backend/tstorebe/tstore/plugin.go b/politeiad/backend/tstorebe/tstore/plugin.go index cce46575f..0cd7ceeed 100644 --- a/politeiad/backend/tstorebe/tstore/plugin.go +++ b/politeiad/backend/tstorebe/tstore/plugin.go @@ -95,6 +95,9 @@ func (t *Tstore) PluginRegister(b backend.Backend, p backend.Plugin) error { } case umplugin.PluginID: client, err = usermd.New(t, p.Settings, dataDir) + if err != nil { + return err + } default: return backend.ErrPluginInvalid } diff --git a/politeiad/backend/tstorebe/tstore_test.go b/politeiad/backend/tstorebe/tstore_test.go deleted file mode 100644 index c7c268199..000000000 --- a/politeiad/backend/tstorebe/tstore_test.go +++ /dev/null @@ -1,1662 +0,0 @@ -// Copyright (c) 2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package tstorebe - -import ( - "bytes" - "encoding/base64" - "encoding/hex" - "errors" - "image" - "image/jpeg" - "image/png" - "testing" - - v1 "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/politeiad/api/v1/mime" - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/util" -) - -func newBackendFile(t *testing.T, fileName string) backend.File { - t.Helper() - - r, err := util.Random(64) - if err != nil { - r = []byte{0, 0, 0} // random byte data - } - - payload := hex.EncodeToString(r) - digest := hex.EncodeToString(util.Digest([]byte(payload))) - b64 := base64.StdEncoding.EncodeToString([]byte(payload)) - - return backend.File{ - Name: fileName, - MIME: mime.DetectMimeType([]byte(payload)), - Digest: digest, - Payload: b64, - } -} - -func newBackendFileJPEG(t *testing.T) backend.File { - t.Helper() - - b := new(bytes.Buffer) - img := image.NewRGBA(image.Rect(0, 0, 1000, 500)) - - err := jpeg.Encode(b, img, &jpeg.Options{}) - if err != nil { - t.Fatal(err) - } - - // Generate a random name - r, err := util.Random(8) - if err != nil { - t.Fatal(err) - } - - return backend.File{ - Name: hex.EncodeToString(r) + ".jpeg", - MIME: mime.DetectMimeType(b.Bytes()), - Digest: hex.EncodeToString(util.Digest(b.Bytes())), - Payload: base64.StdEncoding.EncodeToString(b.Bytes()), - } -} - -func newBackendFilePNG(t *testing.T) backend.File { - t.Helper() - - b := new(bytes.Buffer) - img := image.NewRGBA(image.Rect(0, 0, 1000, 500)) - - err := png.Encode(b, img) - if err != nil { - t.Fatal(err) - } - - // Generate a random name - r, err := util.Random(8) - if err != nil { - t.Fatal(err) - } - - return backend.File{ - Name: hex.EncodeToString(r) + ".png", - MIME: mime.DetectMimeType(b.Bytes()), - Digest: hex.EncodeToString(util.Digest(b.Bytes())), - Payload: base64.StdEncoding.EncodeToString(b.Bytes()), - } -} - -func newBackendMetadataStream(t *testing.T, id uint64, payload string) backend.MetadataStream { - t.Helper() - - return backend.MetadataStream{ - ID: id, - Payload: payload, - } -} - -// recordContentTests defines the type used to describe the content -// verification error tests. -type recordContentTest struct { - description string - metadata []backend.MetadataStream - files []backend.File - filesDel []string - err backend.ContentVerificationError -} - -// setupRecordContentTests returns the list of tests for the verifyContent -// function. These tests are used on all backend api endpoints that verify -// content. -func setupRecordContentTests(t *testing.T) []recordContentTest { - t.Helper() - - var rct []recordContentTest - - // Invalid metadata ID error - md := []backend.MetadataStream{ - newBackendMetadataStream(t, v1.MetadataStreamsMax+1, ""), - } - fs := []backend.File{ - newBackendFile(t, "index.md"), - } - fsDel := []string{} - err := backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidMDID, - } - rct = append(rct, recordContentTest{ - description: "Invalid metadata ID error", - metadata: md, - files: fs, - filesDel: fsDel, - err: err, - }) - - // Duplicate metadata ID error - md = []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - newBackendMetadataStream(t, 1, ""), - } - fs = []backend.File{ - newBackendFile(t, "index.md"), - } - err = backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusDuplicateMDID, - } - rct = append(rct, recordContentTest{ - description: "Duplicate metadata ID error", - metadata: md, - files: fs, - filesDel: fsDel, - err: err, - }) - - // Invalid filename error - md = []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs = []backend.File{ - newBackendFile(t, "invalid/filename.md"), - } - err = backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidFilename, - } - rct = append(rct, recordContentTest{ - description: "Invalid filename error", - metadata: md, - files: fs, - filesDel: fsDel, - err: err, - }) - - // Invalid filename in filesDel error - md = []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs = []backend.File{ - newBackendFile(t, "index.md"), - } - fsDel = []string{"invalid/filename.md"} - err = backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidFilename, - } - rct = append(rct, recordContentTest{ - description: "Invalid filename in filesDel error", - metadata: md, - files: fs, - filesDel: fsDel, - err: err, - }) - - // Empty files error - md = []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs = []backend.File{} - fsDel = []string{} - err = backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusEmpty, - } - rct = append(rct, recordContentTest{ - description: "Empty files error", - metadata: md, - files: fs, - filesDel: fsDel, - err: err, - }) - - // Duplicate filename error - md = []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs = []backend.File{ - newBackendFile(t, "index.md"), - newBackendFile(t, "index.md"), - } - err = backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusDuplicateFilename, - } - rct = append(rct, recordContentTest{ - description: "Duplicate filename error", - metadata: md, - files: fs, - filesDel: fsDel, - err: err, - }) - - // Duplicate filename in filesDel error - md = []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs = []backend.File{ - newBackendFile(t, "index.md"), - } - fsDel = []string{ - "duplicate.md", - "duplicate.md", - } - err = backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusDuplicateFilename, - } - rct = append(rct, recordContentTest{ - description: "Duplicate filename in filesDel error", - metadata: md, - files: fs, - filesDel: fsDel, - err: err, - }) - - // Invalid file digest error - md = []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs = []backend.File{ - newBackendFile(t, "index.md"), - } - fsDel = []string{} - err = backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidFileDigest, - } - rct = append(rct, recordContentTest{ - description: "Invalid file digest error", - metadata: md, - files: fs, - filesDel: fsDel, - err: err, - }) - - // Invalid base64 error - md = []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - f := newBackendFile(t, "index.md") - f.Payload = "*" - fs = []backend.File{f} - err = backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidBase64, - } - rct = append(rct, recordContentTest{ - description: "Invalid file digest error", - metadata: md, - files: fs, - filesDel: fsDel, - err: err, - }) - - // Invalid payload digest error - md = []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - f = newBackendFile(t, "index.md") - f.Payload = "rand" - fs = []backend.File{f} - err = backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidFileDigest, - } - rct = append(rct, recordContentTest{ - description: "Invalid payload digest error", - metadata: md, - files: fs, - filesDel: fsDel, - err: err, - }) - - // Invalid MIME type from payload error - md = []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - jpeg := newBackendFileJPEG(t) - jpeg.Payload = "rand" - payload, er := base64.StdEncoding.DecodeString(jpeg.Payload) - if er != nil { - t.Fatalf(er.Error()) - } - jpeg.Digest = hex.EncodeToString(util.Digest(payload)) - fs = []backend.File{ - newBackendFile(t, "index.md"), - jpeg, - } - err = backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidMIMEType, - } - rct = append(rct, recordContentTest{ - description: "Invalid MIME type from payload error", - metadata: md, - files: fs, - filesDel: fsDel, - err: err, - }) - - // Unsupported MIME type error - md = []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - jpeg = newBackendFileJPEG(t) - fs = []backend.File{ - newBackendFile(t, "index.md"), - jpeg, - } - err = backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusUnsupportedMIMEType, - } - rct = append(rct, recordContentTest{ - description: "Unsupported MIME type error", - metadata: md, - files: fs, - filesDel: fsDel, - err: err, - }) - - return rct -} - -func TestNewRecord(t *testing.T) { - tstoreBackend, cleanup := NewTestTstoreBackend(t) - defer cleanup() - - // Test all record content verification error through the New endpoint - recordContentTests := setupRecordContentTests(t) - for _, test := range recordContentTests { - t.Run(test.description, func(t *testing.T) { - // Make backend call - _, err := tstoreBackend.New(test.metadata, test.files) - - // Parse error - var contentError backend.ContentVerificationError - if errors.As(err, &contentError) { - if contentError.ErrorCode != test.err.ErrorCode { - t.Errorf("got error %v, want %v", - v1.ErrorStatus[contentError.ErrorCode], - v1.ErrorStatus[test.err.ErrorCode]) - } - } - }) - } - - // Test success case - md := []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs := []backend.File{ - newBackendFile(t, "index.md"), - } - _, err := tstoreBackend.New(md, fs) - if err != nil { - t.Errorf("success case failed with %v", err) - } -} - -/* -func TestUpdateUnvettedRecord(t *testing.T) { - tstoreBackend, cleanup := NewTestTstoreBackend(t) - defer cleanup() - - // Create new record - md := []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs := []backend.File{ - newBackendFile(t, "index.md"), - } - rec, err := tstoreBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - token, err := tokenDecode(rec.Token) - if err != nil { - t.Fatal(err) - } - - // Test all record content verification error through the - // UpdateUnvettedRecord endpoint - recordContentTests := setupRecordContentTests(t) - for _, test := range recordContentTests { - t.Run(test.description, func(t *testing.T) { - // Convert token - token, err := tokenDecodeAnyLength(rec.Token) - if err != nil { - t.Error(err) - } - - // Make backend call - _, err = tstoreBackend.UpdateUnvettedRecord(token, test.metadata, - []backend.MetadataStream{}, test.files, test.filesDel) - - // Parse error - var contentError backend.ContentVerificationError - if errors.As(err, &contentError) { - if contentError.ErrorCode != test.err.ErrorCode { - t.Errorf("got error %v, want %v", - v1.ErrorStatus[contentError.ErrorCode], - v1.ErrorStatus[test.err.ErrorCode]) - } - } - }) - } - - // Random png image file to include in edit payload - imageRandom := newBackendFilePNG(t) - - // test case: Token not full length - tokenShort, err := tokenDecodeAnyLength(util.TokenToPrefix(rec.Token)) - if err != nil { - t.Fatal(err) - } - - // test case: Record not found - tokenRandom := tokenFromTreeID(123) - - // test case: Frozen tree - recFrozen, err := tstoreBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - tokenFrozen, err := tokenDecode(recFrozen.Token) - if err != nil { - t.Fatal(err) - } - err = tstoreBackend.unvetted.treeFreeze(treeIDFromToken(tokenFrozen), - backend.RecordMetadata{}, []backend.MetadataStream{}, 0) - if err != nil { - t.Fatal(err) - } - - // Setup UpdateUnvettedRecord tests - var tests = []struct { - description string - token []byte - mdAppend, mdOverwrite []backend.MetadataStream - filesAdd []backend.File - filesDel []string - wantContentErr error - wantErr error - }{ - { - "token not full length", - tokenShort, - []backend.MetadataStream{}, - []backend.MetadataStream{}, - []backend.File{imageRandom}, - []string{}, - backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidToken, - }, - nil, - }, - { - "record not found", - tokenRandom, - []backend.MetadataStream{}, - []backend.MetadataStream{}, - []backend.File{imageRandom}, - []string{}, - nil, - backend.ErrRecordNotFound, - }, - { - "tree frozen for changes", - tokenFrozen, - []backend.MetadataStream{}, - []backend.MetadataStream{}, - []backend.File{imageRandom}, - []string{}, - nil, - backend.ErrRecordLocked, - }, - { - "no changes to record", - token, - []backend.MetadataStream{}, - []backend.MetadataStream{}, - []backend.File{}, - []string{}, - nil, - backend.ErrNoChanges, - }, - { - "success", - token, - []backend.MetadataStream{}, - []backend.MetadataStream{}, - []backend.File{imageRandom}, - []string{}, - nil, - nil, - }, - } - - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - // Make backend call - _, err = tstoreBackend.UpdateUnvettedRecord(test.token, - test.mdAppend, test.mdOverwrite, test.filesAdd, test.filesDel) - - // Parse error - var contentError backend.ContentVerificationError - if errors.As(err, &contentError) { - if test.wantContentErr == nil { - t.Errorf("got error %v, want nil", err) - return - } - wantContentErr := - test.wantContentErr.(backend.ContentVerificationError) - if contentError.ErrorCode != wantContentErr.ErrorCode { - t.Errorf("got error %v, want %v", - v1.ErrorStatus[contentError.ErrorCode], - v1.ErrorStatus[wantContentErr.ErrorCode]) - } - return - } - - // Expecting content error, but got none - if test.wantContentErr != nil { - t.Errorf("got error %v, want %v", err, test.wantContentErr) - } - - // Expectations not met - if test.wantErr != err { - t.Errorf("got error %v, want %v", err, test.wantErr) - } - }) - } -} - -func TestUpdateVettedRecord(t *testing.T) { - tstoreBackend, cleanup := NewTestTstoreBackend(t) - defer cleanup() - - // Create new record - md := []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs := []backend.File{ - newBackendFile(t, "index.md"), - } - rec, err := tstoreBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - token, err := tokenDecode(rec.Token) - if err != nil { - t.Fatal(err) - } - md = append(md, newBackendMetadataStream(t, 2, "")) - - // Publish the created record - err = tstoreBackend.unvettedPublish(token, *rec, md, fs) - if err != nil { - t.Fatal(err) - } - - // Test all record content verification error through the - // UpdateVettedRecord endpoint - recordContentTests := setupRecordContentTests(t) - for _, test := range recordContentTests { - t.Run(test.description, func(t *testing.T) { - // Convert token - token, err := tokenDecodeAnyLength(rec.Token) - if err != nil { - t.Error(err) - } - - // Make backend call - _, err = tstoreBackend.UpdateVettedRecord(token, test.metadata, - []backend.MetadataStream{}, test.files, test.filesDel) - - // Parse error - var contentError backend.ContentVerificationError - if errors.As(err, &contentError) { - if contentError.ErrorCode != test.err.ErrorCode { - t.Errorf("got error %v, want %v", - v1.ErrorStatus[contentError.ErrorCode], - v1.ErrorStatus[test.err.ErrorCode]) - } - } - }) - } - - // Random png image file to include in edit payload - imageRandom := newBackendFilePNG(t) - - // test case: Token not full length - tokenShort, err := tokenDecodeAnyLength(util.TokenToPrefix(rec.Token)) - if err != nil { - t.Error(err) - } - - // test case: Record not found - tokenRandom := tokenFromTreeID(123) - - // test case: Frozen tree - recFrozen, err := tstoreBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - tokenFrozen, err := tokenDecode(recFrozen.Token) - if err != nil { - t.Fatal(err) - } - md = append(md, newBackendMetadataStream(t, 3, "")) - err = tstoreBackend.unvettedPublish(tokenFrozen, *recFrozen, md, fs) - if err != nil { - t.Fatal(err) - } - treeIDFrozenVetted := tstoreBackend.vettedTreeIDs[recFrozen.Token] - err = tstoreBackend.vetted.treeFreeze(treeIDFrozenVetted, - backend.RecordMetadata{}, []backend.MetadataStream{}, 0) - if err != nil { - t.Fatal(err) - } - - // Setup UpdateVettedRecord tests - var tests = []struct { - description string - token []byte - mdAppend, mdOverwirte []backend.MetadataStream - filesAdd []backend.File - filesDel []string - wantContentErr error - wantErr error - }{ - { - "token not full length", - tokenShort, - []backend.MetadataStream{}, - []backend.MetadataStream{}, - []backend.File{imageRandom}, - []string{}, - backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidToken, - }, - nil, - }, - { - "record not found", - tokenRandom, - []backend.MetadataStream{}, - []backend.MetadataStream{}, - []backend.File{imageRandom}, - []string{}, - nil, - backend.ErrRecordNotFound, - }, - { - "tree frozen for changes", - tokenFrozen, - []backend.MetadataStream{}, - []backend.MetadataStream{}, - []backend.File{imageRandom}, - []string{}, - nil, - backend.ErrRecordLocked, - }, - { - "no changes to record", - token, - []backend.MetadataStream{}, - []backend.MetadataStream{}, - []backend.File{}, - []string{}, - nil, - backend.ErrNoChanges, - }, - { - "success", - token, - []backend.MetadataStream{}, - []backend.MetadataStream{}, - []backend.File{imageRandom}, - []string{}, - nil, - nil, - }, - } - - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - // Make backend call - _, err = tstoreBackend.UpdateVettedRecord(test.token, - test.mdAppend, test.mdOverwirte, test.filesAdd, test.filesDel) - - // Parse error - var contentError backend.ContentVerificationError - if errors.As(err, &contentError) { - if test.wantContentErr == nil { - t.Errorf("got error %v, want nil", err) - return - } - wantContentErr := - test.wantContentErr.(backend.ContentVerificationError) - if contentError.ErrorCode != wantContentErr.ErrorCode { - t.Errorf("got error %v, want %v", - v1.ErrorStatus[contentError.ErrorCode], - v1.ErrorStatus[wantContentErr.ErrorCode]) - } - return - } - - // Expecting content error, but got none - if test.wantContentErr != nil { - t.Errorf("got error %v, want %v", err, test.wantContentErr) - } - - // Expectations not met - if test.wantErr != err { - t.Errorf("got error %v, want %v", err, test.wantErr) - } - }) - } -} - -func TestUpdateUnvettedMetadata(t *testing.T) { - tstoreBackend, cleanup := NewTestTstoreBackend(t) - defer cleanup() - - // Create new record - md := []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs := []backend.File{ - newBackendFile(t, "index.md"), - } - rec, err := tstoreBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - token, err := tokenDecode(rec.Token) - if err != nil { - t.Fatal(err) - } - - // Test all record content verification error through the - // UpdateUnvettedMetadata endpoint - recordContentTests := setupRecordContentTests(t) - for _, test := range recordContentTests { - t.Run(test.description, func(t *testing.T) { - // Make backend call - err := tstoreBackend.UpdateUnvettedMetadata(token, - test.metadata, []backend.MetadataStream{}) - - // Parse error - var contentError backend.ContentVerificationError - if errors.As(err, &contentError) { - if contentError.ErrorCode != test.err.ErrorCode { - t.Errorf("got error %v, want %v", - v1.ErrorStatus[contentError.ErrorCode], - v1.ErrorStatus[test.err.ErrorCode]) - } - } - }) - } - - // test case: Token not full length - tokenShort, err := tokenDecodeAnyLength(util.TokenToPrefix(rec.Token)) - if err != nil { - t.Fatal(err) - } - - // test case: Record not found - tokenRandom := tokenFromTreeID(123) - - // test case: Frozen tree - recFrozen, err := tstoreBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - tokenFrozen, err := tokenDecode(recFrozen.Token) - if err != nil { - t.Fatal(err) - } - err = tstoreBackend.unvetted.treeFreeze(treeIDFromToken(tokenFrozen), - backend.RecordMetadata{}, []backend.MetadataStream{}, 0) - if err != nil { - t.Fatal(err) - } - - // Setup UpdateUnvettedMetadata tests - var tests = []struct { - description string - token []byte - mdAppend, mdOverwrite []backend.MetadataStream - wantContentErr error - wantErr error - }{ - { - "no changes to record metadata, empty streams", - token, - []backend.MetadataStream{}, - []backend.MetadataStream{}, - backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusNoChanges, - }, - nil, - }, - { - "invalid token", - tokenShort, - []backend.MetadataStream{{ - ID: 2, - Payload: "random", - }}, - []backend.MetadataStream{}, - backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidToken, - }, - nil, - }, - { - "record not found", - tokenRandom, - []backend.MetadataStream{{ - ID: 2, - Payload: "random", - }}, - []backend.MetadataStream{}, - nil, - backend.ErrRecordNotFound, - }, - { - "tree frozen for changes", - tokenFrozen, - []backend.MetadataStream{{ - ID: 2, - Payload: "random", - }}, - []backend.MetadataStream{}, - nil, - backend.ErrRecordLocked, - }, - { - "no changes to record metadata, same payload", - token, - []backend.MetadataStream{}, - []backend.MetadataStream{{ - ID: 1, - Payload: "", - }}, - nil, - backend.ErrNoChanges, - }, - { - "success", - token, - []backend.MetadataStream{}, - []backend.MetadataStream{{ - ID: 1, - Payload: "newdata", - }}, - nil, - nil, - }, - } - - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - // Make backend call - err = tstoreBackend.UpdateUnvettedMetadata(test.token, - test.mdAppend, test.mdOverwrite) - - // Parse error - var contentError backend.ContentVerificationError - if errors.As(err, &contentError) { - if test.wantContentErr == nil { - t.Errorf("got error %v, want nil", err) - return - } - wantContentErr := - test.wantContentErr.(backend.ContentVerificationError) - if contentError.ErrorCode != wantContentErr.ErrorCode { - t.Errorf("got error %v, want %v", - v1.ErrorStatus[contentError.ErrorCode], - v1.ErrorStatus[wantContentErr.ErrorCode]) - } - return - } - - // Expecting content error, but got none - if test.wantContentErr != nil { - t.Errorf("got error %v, want %v", err, test.wantContentErr) - } - - // Expectations not met - if test.wantErr != err { - t.Errorf("got error %v, want %v", err, test.wantErr) - } - }) - } -} - -func TestUpdateVettedMetadata(t *testing.T) { - tstoreBackend, cleanup := NewTestTstoreBackend(t) - defer cleanup() - - // Create new record - md := []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs := []backend.File{ - newBackendFile(t, "index.md"), - } - rec, err := tstoreBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - token, err := tokenDecode(rec.Token) - if err != nil { - t.Fatal(err) - } - md = append(md, newBackendMetadataStream(t, 2, "")) - err = tstoreBackend.unvettedPublish(token, *rec, md, fs) - if err != nil { - t.Fatal(err) - } - - // Test all record content verification error through the - // UpdateVettedMetadata endpoint - recordContentTests := setupRecordContentTests(t) - for _, test := range recordContentTests { - t.Run(test.description, func(t *testing.T) { - // Make backend call - err := tstoreBackend.UpdateVettedMetadata(token, - test.metadata, []backend.MetadataStream{}) - - // Parse error - var contentError backend.ContentVerificationError - if errors.As(err, &contentError) { - if contentError.ErrorCode != test.err.ErrorCode { - t.Errorf("got error %v, want %v", - v1.ErrorStatus[contentError.ErrorCode], - v1.ErrorStatus[test.err.ErrorCode]) - } - } - }) - } - - // test case: Token not full length - tokenShort, err := tokenDecodeAnyLength(util.TokenToPrefix(rec.Token)) - if err != nil { - t.Fatal(err) - } - - // test case: Record not found - tokenRandom := tokenFromTreeID(123) - - // test case: Frozen tree - recFrozen, err := tstoreBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - tokenFrozen, err := tokenDecode(recFrozen.Token) - if err != nil { - t.Fatal(err) - } - md = append(md, newBackendMetadataStream(t, 3, "")) - err = tstoreBackend.unvettedPublish(tokenFrozen, *recFrozen, md, fs) - if err != nil { - t.Fatal(err) - } - treeIDFrozenVetted := tstoreBackend.vettedTreeIDs[recFrozen.Token] - err = tstoreBackend.vetted.treeFreeze(treeIDFrozenVetted, - backend.RecordMetadata{}, []backend.MetadataStream{}, 0) - if err != nil { - t.Fatal(err) - } - - // Setup UpdateVettedMetadata tests - var tests = []struct { - description string - token []byte - mdAppend, mdOverwrite []backend.MetadataStream - wantContentErr error - wantErr error - }{ - { - "no changes to record metadata, empty streams", - token, - []backend.MetadataStream{}, - []backend.MetadataStream{}, - backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusNoChanges, - }, - nil, - }, - { - "invalid token", - tokenShort, - []backend.MetadataStream{ - newBackendMetadataStream(t, 2, "random"), - }, - []backend.MetadataStream{}, - backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidToken, - }, - nil, - }, - { - "record not found", - tokenRandom, - []backend.MetadataStream{ - newBackendMetadataStream(t, 2, "random"), - }, - []backend.MetadataStream{}, - nil, - backend.ErrRecordNotFound, - }, - { - "tree frozen for changes", - tokenFrozen, - []backend.MetadataStream{ - newBackendMetadataStream(t, 2, "random"), - }, - []backend.MetadataStream{}, - nil, - backend.ErrRecordLocked, - }, - { - "no changes to record metadata, same payload", - token, - []backend.MetadataStream{}, - []backend.MetadataStream{ - newBackendMetadataStream(t, 2, ""), - }, - nil, - backend.ErrNoChanges, - }, - { - "success", - token, - []backend.MetadataStream{}, - []backend.MetadataStream{ - newBackendMetadataStream(t, 1, "newdata"), - }, - nil, - nil, - }, - } - - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - // Make backend call - err = tstoreBackend.UpdateVettedMetadata(test.token, - test.mdAppend, test.mdOverwrite) - - // Parse error - var contentError backend.ContentVerificationError - if errors.As(err, &contentError) { - if test.wantContentErr == nil { - t.Errorf("got error %v, want nil", err) - return - } - wantContentErr := - test.wantContentErr.(backend.ContentVerificationError) - if contentError.ErrorCode != wantContentErr.ErrorCode { - t.Errorf("got error %v, want %v", - v1.ErrorStatus[contentError.ErrorCode], - v1.ErrorStatus[wantContentErr.ErrorCode]) - } - return - } - - // Expecting content error, but got none - if test.wantContentErr != nil { - t.Errorf("got error %v, want %v", err, test.wantContentErr) - } - - // Expectations not met - if test.wantErr != err { - t.Errorf("got error %v, want %v", err, test.wantErr) - } - }) - } -} - -func TestUnvettedExists(t *testing.T) { - tstoreBackend, cleanup := NewTestTstoreBackend(t) - defer cleanup() - - // Create new record - md := []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs := []backend.File{ - newBackendFile(t, "index.md"), - } - rec, err := tstoreBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - token, err := tokenDecode(rec.Token) - if err != nil { - t.Fatal(err) - } - - // Random token - tokenRandom := tokenFromTreeID(123) - - // Run UnvettedExists test cases - // - // Record exists - result := tstoreBackend.UnvettedExists(token) - if result == false { - t.Errorf("got false, want true") - } - // Record does not exist - result = tstoreBackend.UnvettedExists(tokenRandom) - if result == true { - t.Errorf("got true, want false") - } -} - -func TestVettedExists(t *testing.T) { - tstoreBackend, cleanup := NewTestTstoreBackend(t) - defer cleanup() - - // Create unvetted record - md := []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs := []backend.File{ - newBackendFile(t, "index.md"), - } - unvetted, err := tstoreBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - tokenUnvetted, err := tokenDecode(unvetted.Token) - if err != nil { - t.Fatal(err) - } - - // Create vetted record - vetted, err := tstoreBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - tokenVetted, err := tokenDecode(vetted.Token) - if err != nil { - t.Fatal(err) - } - md = append(md, newBackendMetadataStream(t, 2, "")) - err = tstoreBackend.unvettedPublish(tokenVetted, *vetted, md, fs) - if err != nil { - t.Fatal(err) - } - - // Run VettedExists test cases - // - // Record exists - result := tstoreBackend.VettedExists(tokenVetted) - if result == false { - t.Fatal("got false, want true") - } - // Record does not exist - result = tstoreBackend.VettedExists(tokenUnvetted) - if result == true { - t.Fatal("got true, want false") - } -} - -func TestGetUnvetted(t *testing.T) { - tstoreBackend, cleanup := NewTestTstoreBackend(t) - defer cleanup() - - // Create new record - md := []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs := []backend.File{ - newBackendFile(t, "index.md"), - } - rec, err := tstoreBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - token, err := tokenDecode(rec.Token) - if err != nil { - t.Fatal(err) - } - - // Random token - tokenRandom := tokenFromTreeID(123) - - // Bad version error - _, err = tstoreBackend.GetUnvetted(token, "badversion") - if err != backend.ErrRecordNotFound { - t.Errorf("got error %v, want %v", err, backend.ErrRecordNotFound) - } - - // Bad token error - _, err = tstoreBackend.GetUnvetted(tokenRandom, "") - if err != backend.ErrRecordNotFound { - t.Errorf("got error %v, want %v", err, backend.ErrRecordNotFound) - } - - // Success - _, err = tstoreBackend.GetUnvetted(token, "") - if err != nil { - t.Errorf("got error %v, want nil", err) - } -} - -func TestGetVetted(t *testing.T) { - tstoreBackend, cleanup := NewTestTstoreBackend(t) - defer cleanup() - - // Create new record - md := []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs := []backend.File{ - newBackendFile(t, "index.md"), - } - rec, err := tstoreBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - token, err := tokenDecode(rec.Token) - if err != nil { - t.Fatal(err) - } - md = append(md, newBackendMetadataStream(t, 2, "")) - err = tstoreBackend.unvettedPublish(token, *rec, md, fs) - if err != nil { - t.Fatal(err) - } - - // Random token - tokenRandom := tokenFromTreeID(123) - - // Bad version error - _, err = tstoreBackend.GetVetted(token, "badversion") - if err != backend.ErrRecordNotFound { - t.Errorf("got error %v, want %v", err, backend.ErrRecordNotFound) - } - - // Bad token error - _, err = tstoreBackend.GetVetted(tokenRandom, "") - if err != backend.ErrRecordNotFound { - t.Errorf("got error %v, want %v", err, backend.ErrRecordNotFound) - } - - // Success - _, err = tstoreBackend.GetVetted(token, "") - if err != nil { - t.Errorf("got error %v, want nil", err) - } -} - -func TestSetUnvettedStatus(t *testing.T) { - tstoreBackend, cleanup := NewTestTstoreBackend(t) - defer cleanup() - - // Helpers - md := []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs := []backend.File{ - newBackendFile(t, "index.md"), - } - - // Invalid status transitions - // - // test case: Unvetted to archived - recUnvetToArch, err := tstoreBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - tokenUnvetToArch, err := tokenDecode(recUnvetToArch.Token) - if err != nil { - t.Fatal(err) - } - // test case: Unvetted to unvetted - recUnvetToUnvet, err := tstoreBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - tokenUnvetToUnvet, err := tokenDecode(recUnvetToUnvet.Token) - if err != nil { - t.Fatal(err) - } - - // Valid status transitions - // - // test case: Unvetted to vetted - recUnvetToVet, err := tstoreBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - tokenUnvetToVet, err := tokenDecode(recUnvetToVet.Token) - if err != nil { - t.Fatal(err) - } - // test case: Unvetted to censored - recUnvetToCensored, err := tstoreBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - tokenUnvetToCensored, err := tokenDecode(recUnvetToCensored.Token) - if err != nil { - t.Fatal(err) - } - - // test case: Token not full length - tokenShort, err := tokenDecodeAnyLength( - util.TokenToPrefix(recUnvetToVet.Token)) - if err != nil { - t.Fatal(err) - } - - // test case: Record not found - tokenRandom := tokenFromTreeID(123) - - // Setup SetUnvettedStatus tests - var tests = []struct { - description string - token []byte - status backend.MDStatusT - mdAppend, mdOverwrite []backend.MetadataStream - wantContentErr error - wantErr error - }{ - { - "invalid: unvetted to archived", - tokenUnvetToArch, - backend.MDStatusArchived, - []backend.MetadataStream{}, - []backend.MetadataStream{}, - nil, - backend.StateTransitionError{ - From: recUnvetToArch.Status, - To: backend.MDStatusArchived, - }, - }, - { - "invalid: unvetted to unvetted", - tokenUnvetToUnvet, - backend.MDStatusUnvetted, - []backend.MetadataStream{}, - []backend.MetadataStream{}, - nil, - backend.StateTransitionError{ - From: recUnvetToArch.Status, - To: backend.MDStatusUnvetted, - }, - }, - { - "valid: unvetted to vetted", - tokenUnvetToVet, - backend.MDStatusVetted, - []backend.MetadataStream{}, - []backend.MetadataStream{}, - nil, - nil, - }, - { - "valid: unvetted to censored", - tokenUnvetToCensored, - backend.MDStatusCensored, - []backend.MetadataStream{}, - []backend.MetadataStream{}, - nil, - nil, - }, - { - "invalid token", - tokenShort, - backend.MDStatusCensored, - []backend.MetadataStream{}, - []backend.MetadataStream{}, - backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidToken, - }, - nil, - }, - { - "record not found", - tokenRandom, - backend.MDStatusCensored, - []backend.MetadataStream{}, - []backend.MetadataStream{}, - nil, - backend.ErrRecordNotFound, - }, - } - - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - // Make backend call - _, err = tstoreBackend.SetUnvettedStatus(test.token, test.status, - test.mdAppend, test.mdOverwrite) - - // Parse error - var contentError backend.ContentVerificationError - if errors.As(err, &contentError) { - if test.wantContentErr == nil { - t.Errorf("got error %v, want nil", err) - return - } - wantContentErr := - test.wantContentErr.(backend.ContentVerificationError) - if contentError.ErrorCode != wantContentErr.ErrorCode { - t.Errorf("got error %v, want %v", - v1.ErrorStatus[contentError.ErrorCode], - v1.ErrorStatus[wantContentErr.ErrorCode]) - } - return - } - - // Expecting content error, but got none - if test.wantContentErr != nil { - t.Errorf("got error %v, want %v", err, test.wantContentErr) - } - - // Expectations not met - if test.wantErr != err { - t.Errorf("got error %v, want %v", err, test.wantErr) - } - }) - } -} - -func TestSetVettedStatus(t *testing.T) { - tstoreBackend, cleanup := NewTestTstoreBackend(t) - defer cleanup() - - // Helpers - md := []backend.MetadataStream{ - newBackendMetadataStream(t, 1, ""), - } - fs := []backend.File{ - newBackendFile(t, "index.md"), - } - - // Invalid status transitions - // - // test case: Vetted to unvetted - recVetToUnvet, err := tstoreBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - tokenVetToUnvet, err := tokenDecode(recVetToUnvet.Token) - if err != nil { - t.Fatal(err) - } - - md = append(md, newBackendMetadataStream(t, 2, "")) - _, err = tstoreBackend.SetUnvettedStatus(tokenVetToUnvet, - backend.MDStatusVetted, md, []backend.MetadataStream{}) - if err != nil { - t.Fatal(err) - } - // test case: Vetted to vetted - recVetToVet, err := tstoreBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - tokenVetToVet, err := tokenDecode(recVetToVet.Token) - if err != nil { - t.Fatal(err) - } - md = append(md, newBackendMetadataStream(t, 3, "")) - _, err = tstoreBackend.SetUnvettedStatus(tokenVetToVet, - backend.MDStatusVetted, md, []backend.MetadataStream{}) - if err != nil { - t.Fatal(err) - } - - // Valid status transitions - // - // test case: Vetted to archived - recVetToArch, err := tstoreBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - tokenVetToArch, err := tokenDecode(recVetToArch.Token) - if err != nil { - t.Fatal(err) - } - md = append(md, newBackendMetadataStream(t, 4, "")) - _, err = tstoreBackend.SetUnvettedStatus(tokenVetToArch, - backend.MDStatusVetted, md, []backend.MetadataStream{}) - if err != nil { - t.Fatal(err) - } - // test case: Vetted to censored - recVetToCensored, err := tstoreBackend.New(md, fs) - if err != nil { - t.Fatal(err) - } - tokenVetToCensored, err := tokenDecode(recVetToCensored.Token) - if err != nil { - t.Fatal(err) - } - md = append(md, newBackendMetadataStream(t, 5, "")) - _, err = tstoreBackend.SetUnvettedStatus(tokenVetToCensored, - backend.MDStatusVetted, md, []backend.MetadataStream{}) - if err != nil { - t.Fatal(err) - } - - // test case: Token not full length - tokenShort, err := tokenDecodeAnyLength( - util.TokenToPrefix(recVetToCensored.Token)) - if err != nil { - t.Fatal(err) - } - - // test case: Record not found - tokenRandom := tokenFromTreeID(123) - - // Setup SetVettedStatus tests - var tests = []struct { - description string - token []byte - status backend.MDStatusT - mdAppend, mdOverwrite []backend.MetadataStream - wantContentErr error - wantErr error - }{ - { - "invalid: vetted to unvetted", - tokenVetToUnvet, - backend.MDStatusUnvetted, - []backend.MetadataStream{}, - []backend.MetadataStream{}, - nil, - backend.StateTransitionError{ - From: backend.MDStatusVetted, - To: backend.MDStatusUnvetted, - }, - }, - { - "invalid: vetted to vetted", - tokenVetToVet, - backend.MDStatusVetted, - []backend.MetadataStream{}, - []backend.MetadataStream{}, - nil, - backend.StateTransitionError{ - From: backend.MDStatusVetted, - To: backend.MDStatusVetted, - }, - }, - { - "valid: vetted to archived", - tokenVetToArch, - backend.MDStatusArchived, - []backend.MetadataStream{}, - []backend.MetadataStream{}, - nil, - nil, - }, - { - "valid: vetted to censored", - tokenVetToCensored, - backend.MDStatusCensored, - []backend.MetadataStream{}, - []backend.MetadataStream{}, - nil, - nil, - }, - { - "invalid token", - tokenShort, - backend.MDStatusCensored, - []backend.MetadataStream{}, - []backend.MetadataStream{}, - backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidToken, - }, - nil, - }, - { - "record not found", - tokenRandom, - backend.MDStatusCensored, - []backend.MetadataStream{}, - []backend.MetadataStream{}, - nil, - backend.ErrRecordNotFound, - }, - } - - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - // Make backend call - _, err = tstoreBackend.SetVettedStatus(test.token, test.status, - test.mdAppend, test.mdOverwrite) - - // Parse error - var contentError backend.ContentVerificationError - if errors.As(err, &contentError) { - if test.wantContentErr == nil { - t.Errorf("got error %v, want nil", err) - return - } - wantContentErr := - test.wantContentErr.(backend.ContentVerificationError) - if contentError.ErrorCode != wantContentErr.ErrorCode { - t.Errorf("got error %v, want %v", - v1.ErrorStatus[contentError.ErrorCode], - v1.ErrorStatus[wantContentErr.ErrorCode]) - } - return - } - - // Expecting content error, but got none - if test.wantContentErr != nil { - t.Errorf("got error %v, want %v", err, test.wantContentErr) - } - - // Expectations not met - if test.wantErr != err { - t.Errorf("got error %v, want %v", err, test.wantErr) - } - }) - } -} -*/ diff --git a/politeiad/backend/tstorebe/tstorebe.go b/politeiad/backend/tstorebe/tstorebe.go index a4da80bf6..b52d27ca8 100644 --- a/politeiad/backend/tstorebe/tstorebe.go +++ b/politeiad/backend/tstorebe/tstorebe.go @@ -1451,7 +1451,7 @@ func (t *tstoreBackend) GetUnvettedBatch(reqs []backend.RecordRequest) (map[stri // in the reply. continue } - // An unexpected error occured. Log it and continue. + // An unexpected error occurred. Log it and continue. log.Debug("RecordPartial %v: %v", treeID, err) continue } @@ -1503,7 +1503,7 @@ func (t *tstoreBackend) GetVettedBatch(reqs []backend.RecordRequest) (map[string // in the reply. continue } - // An unexpected error occured. Log it and continue. + // An unexpected error occurred. Log it and continue. log.Debug("RecordPartial %v: %v", treeID, err) continue } diff --git a/politeiad/client/client.go b/politeiad/client/client.go index 41ed7ded4..98f520299 100644 --- a/politeiad/client/client.go +++ b/politeiad/client/client.go @@ -26,8 +26,8 @@ type Client struct { } // ErrorReply represents the request body that is returned from politeaid when -// an error occurs. PluginID will only be populated if the error occured during -// execution of a plugin command. +// an error occurs. PluginID will only be populated if the error occurred +// during execution of a plugin command. type ErrorReply struct { PluginID string `json:"pluginid"` ErrorCode int `json:"errorcode"` @@ -53,7 +53,7 @@ func (e Error) Error() string { // makeReq makes a politeiad http request to the method and route provided, // serializing the provided object as the request body, and returning a byte -// slice of the repsonse body. An Error is returned if politeiad responds with +// slice of the response body. An Error is returned if politeiad responds with // anything other than a 200 http status code. func (c *Client) makeReq(ctx context.Context, method string, route string, v interface{}) ([]byte, error) { // Serialize body diff --git a/politeiad/client/user.go b/politeiad/client/user.go index a95a67bb0..35f3afe8f 100644 --- a/politeiad/client/user.go +++ b/politeiad/client/user.go @@ -52,7 +52,7 @@ func (c *Client) Author(ctx context.Context, state, token string) (string, error } // UserRecords sends the user plugin UserRecords command to the politeiad v1 -// API. A seperate command is sent for the unvetted and vetted records. The +// API. A separate command is sent for the unvetted and vetted records. The // returned map is a map[recordState][]token. func (c *Client) UserRecords(ctx context.Context, userID string) (map[string][]string, error) { // Setup request diff --git a/politeiad/plugins/comments/comments.go b/politeiad/plugins/comments/comments.go index f6c72f2fb..ddf169d71 100644 --- a/politeiad/plugins/comments/comments.go +++ b/politeiad/plugins/comments/comments.go @@ -43,7 +43,7 @@ const ( // characters that are allowed in a comment. SettingCommentLengthMax uint32 = 8000 - // SettingVoteChangesMax is the defualt maximum number of times a + // SettingVoteChangesMax is the default maximum number of times a // user can change their vote on a comment. This prevents a // malicious user from being able to spam comment votes. SettingVoteChangesMax uint32 = 5 diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index 58777af18..d390dfb4e 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -591,7 +591,7 @@ const ( // that will be returned for any single request. A vote timestamp // is ~2000 bytes so a page of 100 votes will only be 0.2MB, but // the bottleneck on this call is performance, not size. Its - // expensive to retreive a large number of inclusion proofs from + // expensive to retrieve a large number of inclusion proofs from // trillian. A 100 timestamps will take ~1 second to compile. VoteTimestampsPageSize uint32 = 100 ) diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 59a9ddf0c..b6d788b6a 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -1434,7 +1434,7 @@ func (p *politeia) pluginCommandBatch(w http.ResponseWriter, r *http.Request) { }, } default: - // Unkown error. Log is as an internal server error and + // Unknown error. Log is as an internal server error and // respond with a server error. t := time.Now().Unix() log.Errorf("%v %v: batched plugin cmd failed: pluginID:%v "+ diff --git a/politeiawww/client/error.go b/politeiawww/client/error.go index a0b3062c1..174f3d908 100644 --- a/politeiawww/client/error.go +++ b/politeiawww/client/error.go @@ -20,7 +20,7 @@ import ( ) // ErrorReply represents the request body that is returned from politeiawww -// when an error occurs. PluginID will only be populated if the error occured +// when an error occurs. PluginID will only be populated if the error occurred // during execution of a plugin command. type ErrorReply struct { PluginID string diff --git a/politeiawww/cmd/pictl/README.md b/politeiawww/cmd/pictl/README.md index ff20a62d5..119e9228e 100644 --- a/politeiawww/cmd/pictl/README.md +++ b/politeiawww/cmd/pictl/README.md @@ -173,11 +173,35 @@ Voting on a proposal can be done using the `politeiavoter` tool. ### pictl -You can also vote on proposals using the `pictl voteballot` command. This casts -a ballot of votes. This will only work on testnet and if you are running your -dcrwallet locally using the default port. +You can vote on testnet proposals using `pictl` if you have the following +setup: +- dcrwallet is running locally on testnet and on the default port. +- A dcrwallet client cert has been setup for `pictl` using the instructions + in the `Dcrwallet Authentication` section of this README. - $ pictl voteballot [token] [voteID] +Cast a ballot of DCR ticket votes. + + $ pictl castballot [token] [voteID] + +# Dcrwallet Authentication + +Voting requires access to wallet GRPC. Therefore this tool needs the wallet's +server certificate to authenticate the server, as well as a local client +keypair to authenticate the client to `dcrwallet`. The server certificate by +default will be found in `~/.dcrwallet/rpc.cert`, and this can be modified to +another path using the `--walletgrpccert` flag. Client certs can be generated +using [`gencerts`](https://github.com/decred/dcrd/blob/master/cmd/gencerts/) +and `pictl` will read `client.pem` and `client-key.pem` from its +application directory by default. The certificate (`client.pem`) must be +appended to `~/.dcrwallet/clients.pem` in order for `dcrwallet` to trust the +client. + +For example: + +``` +$ gencerts ~/.pictl/client{,-key}.pem +$ cat ~/.pictl/client.pem >> ~/.dcrwallet/clients.pem +``` # Reference implementation diff --git a/politeiawww/cmd/pictl/cmdcommentcensor.go b/politeiawww/cmd/pictl/cmdcommentcensor.go index 45c56742e..208738d28 100644 --- a/politeiawww/cmd/pictl/cmdcommentcensor.go +++ b/politeiawww/cmd/pictl/cmdcommentcensor.go @@ -73,7 +73,7 @@ func (c *cmdCommentCensor) Execute(args []string) error { d := cmv1.Del{ State: state, Token: token, - CommentID: uint32(commentID), + CommentID: commentID, Reason: reason, Signature: hex.EncodeToString(sig[:]), PublicKey: cfg.Identity.Public.String(), diff --git a/politeiawww/cmd/pictl/cmdcommentvotes.go b/politeiawww/cmd/pictl/cmdcommentvotes.go index bfd2efee4..61ff76713 100644 --- a/politeiawww/cmd/pictl/cmdcommentvotes.go +++ b/politeiawww/cmd/pictl/cmdcommentvotes.go @@ -81,7 +81,7 @@ func (c *cmdCommentVotes) Execute(args []string) error { const commentVotesHelpMsg = `commentvotes "token" "userid" Get the provided user comment upvote/downvotes for a proposal. If no user ID -is provded then the command will assume the logged in user is requesting their +is provided then the command will assume the logged in user is requesting their own comment votes. Arguments: diff --git a/politeiawww/cmd/pictl/cmdproposaldetails.go b/politeiawww/cmd/pictl/cmdproposaldetails.go index b52f60642..420f8e970 100644 --- a/politeiawww/cmd/pictl/cmdproposaldetails.go +++ b/politeiawww/cmd/pictl/cmdproposaldetails.go @@ -71,7 +71,7 @@ func (c *cmdProposalDetails) Execute(args []string) error { // proposalDetailsHelpMsg is printed to stdout by the help command. const proposalDetailsHelpMsg = `proposaldetails [flags] "token" "version" -Retrive a full proposal record. +Retrieve a full proposal record. This command defaults to retrieving vetted proposals unless the --unvetted flag is used. This command accepts both the full tokens or the shortened token diff --git a/politeiawww/cmd/pictl/cmdproposals.go b/politeiawww/cmd/pictl/cmdproposals.go index 95521b235..1296b22ec 100644 --- a/politeiawww/cmd/pictl/cmdproposals.go +++ b/politeiawww/cmd/pictl/cmdproposals.go @@ -83,7 +83,7 @@ func (c *cmdProposals) Execute(args []string) error { // proposalsHelpMsg is printed to stdout by the help command. const proposalsHelpMsg = `proposals [flags] "tokens..." -Retrive the proposals for the provided tokens. The proposal index file and the +Retrieve the proposals for the provided tokens. The proposal index file and the proposal attachments are not returned from this command. Use the proposal details command if you are trying to retieve the full proposal. diff --git a/politeiawww/cmd/pictl/cmdvotetest.go b/politeiawww/cmd/pictl/cmdvotetest.go index fe7fff5df..46a7b3930 100644 --- a/politeiawww/cmd/pictl/cmdvotetest.go +++ b/politeiawww/cmd/pictl/cmdvotetest.go @@ -73,7 +73,7 @@ func (c *cmdVoteTest) Execute(args []string) error { start := time.Now() err := castBallot(token, voteOption, password) if err != nil { - fmt.Printf("castBallot %v: %v", token, err) + fmt.Printf("castBallot %v: %v\n", token, err) } end := time.Now() elapsed := end.Sub(start) diff --git a/politeiawww/cmd/politeiaverify/politeiaverify.go b/politeiawww/cmd/politeiaverify/politeiaverify.go index b1fdd2141..1aa0e6f81 100644 --- a/politeiawww/cmd/politeiaverify/politeiaverify.go +++ b/politeiawww/cmd/politeiaverify/politeiaverify.go @@ -9,24 +9,20 @@ import ( "path" "strings" - pi "github.com/decred/politeia/politeiawww/api/pi/v1" - wwwutil "github.com/decred/politeia/politeiawww/util" + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + "github.com/decred/politeia/politeiawww/client" "github.com/decred/politeia/util" ) -// proposal is used to unmarshal the data that is cointaned in the proposal -// JSON bundles downloded from the GUI. -type proposal struct { - PublicKey string `json:"publickey"` - Signature string `json:"signature"` - CensorshipRecord pi.CensorshipRecord `json:"censorshiprecord"` - Files []pi.File `json:"files"` - Metadata []pi.Metadata `json:"metadata"` - ServerPublicKey string `json:"serverpublickey"` +// record is used to unmarshal the data that is contained in the record JSON +// bundle downloded from the GUI. +type record struct { + Record rcv1.Record `json:"record"` + ServerPublicKey string `json:"serverpublickey"` } -// comments is used to unmarshal the data that is cointaned in the comments -// JSON bundles downloded from the GUI. +// comments is used to unmarshal the data that is contained in the comments +// JSON bundle downloded from the GUI. type comments []struct { CommentID string `json:"commentid"` Receipt string `json:"receipt"` @@ -35,7 +31,7 @@ type comments []struct { } var ( - flagVerifyProposal = flag.Bool("proposal", false, "Verify proposal bundle") + flagVerifyRecord = flag.Bool("record", false, "Verify record bundle") flagVerifyComments = flag.Bool("comments", false, "Verify comments bundle") ) @@ -48,59 +44,25 @@ func usage() { fmt.Fprintf(os.Stderr, "\n") } -func verifyProposal(payload []byte) error { - var prop proposal - err := json.Unmarshal(payload, &prop) +func verifyRecord(payload []byte) error { + var r record + err := json.Unmarshal(payload, &r) if err != nil { - return fmt.Errorf("Proposal bundle JSON in bad format, make sure to " + + return fmt.Errorf("Record bundle JSON in bad format, make sure to " + "download it from the GUI.") } - // Verify merkle root - merkle, err := wwwutil.MerkleRoot(prop.Files, prop.Metadata) + err = client.RecordVerify(r.Record, r.ServerPublicKey) if err != nil { - return err - } - if merkle != prop.CensorshipRecord.Merkle { - return fmt.Errorf("Merkle roots do not match: %v and %v", - prop.CensorshipRecord.Merkle, merkle) - } - - // Verify proposal signature - id, err := util.IdentityFromString(prop.PublicKey) - if err != nil { - return err - } - sig, err := util.ConvertSignature(prop.Signature) - if err != nil { - return err - } - if !id.VerifyMessage([]byte(merkle), sig) { - return fmt.Errorf("Invalid proposal signature %v", prop.Signature) - } - - // Verify censorship record signature - id, err = util.IdentityFromString(prop.ServerPublicKey) - if err != nil { - return err - } - sig, err = util.ConvertSignature(prop.CensorshipRecord.Signature) - if err != nil { - return err - } - if !id.VerifyMessage([]byte(merkle+prop.CensorshipRecord.Token), sig) { - return fmt.Errorf("Invalid censhorship record signature %v", - prop.CensorshipRecord.Signature) + return fmt.Errorf("Failed to verify record: %v", err) } - fmt.Println("Proposal signature:") - fmt.Printf(" Public key: %s\n", prop.PublicKey) - fmt.Printf(" Signature : %s\n", prop.Signature) - fmt.Println("Proposal censorship record signature:") - fmt.Printf(" Merkle root: %s\n", prop.CensorshipRecord.Merkle) - fmt.Printf(" Public key : %s\n", prop.ServerPublicKey) - fmt.Printf(" Signature : %s\n\n", prop.CensorshipRecord.Signature) - fmt.Println("Proposal successfully verified") + fmt.Println("Censorship record:") + fmt.Printf(" Token : %s\n", r.Record.CensorshipRecord.Token) + fmt.Printf(" Merkle root: %s\n", r.Record.CensorshipRecord.Merkle) + fmt.Printf(" Public key : %s\n", r.ServerPublicKey) + fmt.Printf(" Signature : %s\n\n", r.Record.CensorshipRecord.Signature) + fmt.Println("Record successfully verified") return nil } @@ -147,7 +109,7 @@ func _main() error { case len(args) != 1: usage() return fmt.Errorf("Must provide json bundle path as input") - case *flagVerifyProposal && *flagVerifyComments: + case *flagVerifyRecord && *flagVerifyComments: usage() return fmt.Errorf("Must choose only one verification type") } @@ -162,8 +124,8 @@ func _main() error { // Call verify method switch { - case *flagVerifyProposal: - return verifyProposal(payload) + case *flagVerifyRecord: + return verifyRecord(payload) case *flagVerifyComments: return verifyComments(payload) default: @@ -172,7 +134,7 @@ func _main() error { if strings.Contains(path.Base(file), "comments") { return verifyComments(payload) } - return verifyProposal(payload) + return verifyRecord(payload) } } diff --git a/politeiawww/cmd/shared/config.go b/politeiawww/cmd/shared/config.go index 57910e68c..08e448846 100644 --- a/politeiawww/cmd/shared/config.go +++ b/politeiawww/cmd/shared/config.go @@ -55,7 +55,6 @@ type Config struct { Verbose bool `short:"v" long:"verbose" description:"Print verbose output"` Silent bool `long:"silent" description:"Suppress all output"` - // TODO add docs for this to the piwww README ClientCert string `long:"clientcert" description:"Path to TLS certificate for client authentication"` ClientKey string `long:"clientkey" description:"Path to TLS client authentication key"` diff --git a/politeiawww/comments/process.go b/politeiawww/comments/process.go index 0e082e421..1de7743d7 100644 --- a/politeiawww/comments/process.go +++ b/politeiawww/comments/process.go @@ -258,6 +258,9 @@ func (c *Comments) processVotes(ctx context.Context, v v1.Votes) (*v1.VotesReply // Populate comment votes with user data uid, err := uuid.Parse(v.UserID) + if err != nil { + return nil, err + } u, err := c.userdb.UserGetById(uid) if err != nil { return nil, err diff --git a/politeiawww/pi/events.go b/politeiawww/pi/events.go index d5700e83a..4dee8ffd2 100644 --- a/politeiawww/pi/events.go +++ b/politeiawww/pi/events.go @@ -574,7 +574,7 @@ func (p *Pi) ntfnVoteStarted(sd tkv1.StartDetails, eventUser user.User, authorID return case u.ID.String() == authorID: // Don't send the notification to the author. They are sent a - // seperate notification. + // separate notification. return case !u.NotificationIsEnabled(ntfnBit): // User does not have notification bit set diff --git a/politeiawww/pi/pi.go b/politeiawww/pi/pi.go index ff47ecb27..fa1859f39 100644 --- a/politeiawww/pi/pi.go +++ b/politeiawww/pi/pi.go @@ -172,7 +172,7 @@ func New(cfg *config.Config, pdc *pdclient.Client, udb user.Database, s *session }, } - // Setup event listners + // Setup event listeners p.setupEventListeners() return &p, nil diff --git a/politeiawww/pi/process.go b/politeiawww/pi/process.go index c986b8ac6..b436a6662 100644 --- a/politeiawww/pi/process.go +++ b/politeiawww/pi/process.go @@ -84,7 +84,7 @@ func (p *Pi) processProposals(ctx context.Context, ps v1.Proposals, u *user.User } } - // Setup record requests. We don't retreive any index files or + // Setup record requests. We don't retrieve any index files or // attachment files in order to keep the payload size minimal. reqs := make([]pdv1.RecordRequest, 0, len(ps.Tokens)) for _, v := range ps.Tokens { diff --git a/politeiawww/politeiawww_test.go b/politeiawww/politeiawww_test.go index a97098ffe..c4917bab9 100644 --- a/politeiawww/politeiawww_test.go +++ b/politeiawww/politeiawww_test.go @@ -18,6 +18,9 @@ func TestHandleVersion(t *testing.T) { p, cleanup := newTestPoliteiawww(t) defer cleanup() + d := newTestPoliteiad(t, p) + defer d.Close() + expectedReply := www.VersionReply{ Version: www.PoliteiaWWWAPIVersion, BuildVersion: version.BuildMainVersion(), diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index c67262aa1..6232913ff 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -10,6 +10,7 @@ import ( "encoding/hex" "encoding/json" "errors" + "fmt" "io" "net/http" "strconv" @@ -54,6 +55,9 @@ func (p *politeiawww) proposal(ctx context.Context, token, version string) (*www // Fill in user data userID := userIDFromMetadataStreams(r.Metadata) uid, err := uuid.Parse(userID) + if err != nil { + return nil, err + } u, err := p.db.UserGetById(uid) if err != nil { return nil, err @@ -90,6 +94,9 @@ func (p *politeiawww) proposals(ctx context.Context, reqs []pdv1.RecordRequest) // Fill in user data userID := userIDFromMetadataStreams(v.Metadata) uid, err := uuid.Parse(userID) + if err != nil { + return nil, err + } u, err := p.db.UserGetById(uid) if err != nil { return nil, err @@ -186,9 +193,7 @@ func (p *politeiawww) processTokenInventory(ctx context.Context, isAdmin bool) ( func (p *politeiawww) processAllVetted(ctx context.Context, gav www.GetAllVetted) (*www.GetAllVettedReply, error) { log.Tracef("processAllVetted: %v %v", gav.Before, gav.After) - // TODO - - return nil, nil + return nil, fmt.Errorf("not implemented yet") } func (p *politeiawww) processProposalDetails(ctx context.Context, pd www.ProposalsDetails, u *user.User) (*www.ProposalDetailsReply, error) { diff --git a/politeiawww/records/error.go b/politeiawww/records/error.go index 4464cee10..fb9043087 100644 --- a/politeiawww/records/error.go +++ b/politeiawww/records/error.go @@ -58,7 +58,7 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err case errors.As(err, &pde): // Politeiad error - handlePoliteiadError(w, r, format, pde) + handlePDError(w, r, format, pde) default: // Internal server error. Log it and return a 500. @@ -76,7 +76,7 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err } } -func handlePoliteiadError(w http.ResponseWriter, r *http.Request, format string, pde pdclient.Error) { +func handlePDError(w http.ResponseWriter, r *http.Request, format string, pde pdclient.Error) { var ( pluginID = pde.ErrorReply.PluginID errCode = pde.ErrorReply.ErrorCode diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index 69300c80d..a2486b541 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -323,6 +323,9 @@ func (r *Records) record(ctx context.Context, state, token, version string) (*v1 // Fill in user data userID := userIDFromMetadataStreams(rc.Metadata) uid, err := uuid.Parse(userID) + if err != nil { + return nil, err + } u, err := r.userdb.UserGetById(uid) if err != nil { return nil, err @@ -373,7 +376,7 @@ func (r *Records) processDetails(ctx context.Context, d v1.Details, u *user.User func (r *Records) records(ctx context.Context, state string, reqs []pdv1.RecordRequest) (map[string]v1.Record, error) { var ( - pdr = make(map[string]pdv1.Record) + pdr map[string]pdv1.Record err error ) switch state { @@ -398,6 +401,9 @@ func (r *Records) records(ctx context.Context, state string, reqs []pdv1.RecordR // Fill in user data userID := userIDFromMetadataStreams(rc.Metadata) uid, err := uuid.Parse(userID) + if err != nil { + return nil, err + } u, err := r.userdb.UserGetById(uid) if err != nil { return nil, err diff --git a/politeiawww/testing.go b/politeiawww/testing.go index 69eca5c47..326359691 100644 --- a/politeiawww/testing.go +++ b/politeiawww/testing.go @@ -26,13 +26,9 @@ import ( cms "github.com/decred/politeia/politeiawww/api/cms/v1" piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" - "github.com/decred/politeia/politeiawww/comments" "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/mail" - "github.com/decred/politeia/politeiawww/pi" - "github.com/decred/politeia/politeiawww/records" "github.com/decred/politeia/politeiawww/sessions" - "github.com/decred/politeia/politeiawww/ticketvote" "github.com/decred/politeia/politeiawww/user" "github.com/decred/politeia/politeiawww/user/localdb" "github.com/decred/politeia/util" @@ -364,15 +360,8 @@ func newTestPoliteiawww(t *testing.T) (*politeiawww, func()) { userPaywallPool: make(map[uuid.UUID]paywallPoolMember), } - // TODO setup testing - var c *comments.Comments - var tv *ticketvote.TicketVote - var r *records.Records - var pic *pi.Pi - // Setup routes p.setUserWWWRoutes() - p.setupPiRoutes(r, c, tv, pic) // The cleanup is handled using a closure so that the temp dir // can be deleted using the local variable and not cfg.DataDir. diff --git a/politeiawww/ticketvote/error.go b/politeiawww/ticketvote/error.go index 68b94a8c0..63d98813f 100644 --- a/politeiawww/ticketvote/error.go +++ b/politeiawww/ticketvote/error.go @@ -41,7 +41,7 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err case errors.As(err, &pde): // Politeiad error - handlePoliteiadError(w, r, format, pde) + handlePDError(w, r, format, pde) default: // Internal server error. Log it and return a 500. @@ -59,7 +59,7 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err } } -func handlePoliteiadError(w http.ResponseWriter, r *http.Request, format string, pde pdclient.Error) { +func handlePDError(w http.ResponseWriter, r *http.Request, format string, pde pdclient.Error) { var ( pluginID = pde.ErrorReply.PluginID errCode = pde.ErrorReply.ErrorCode diff --git a/politeiawww/userwww_test.go b/politeiawww/userwww_test.go index 2e34e98ac..ba02ae375 100644 --- a/politeiawww/userwww_test.go +++ b/politeiawww/userwww_test.go @@ -22,6 +22,20 @@ import ( "github.com/gorilla/sessions" ) +const ( + testSessionMaxAge = 86400 // One day +) + +func newSessionOptions() *sessions.Options { + return &sessions.Options{ + Path: "/", + MaxAge: testSessionMaxAge, + Secure: true, + HttpOnly: true, + SameSite: http.SameSiteStrictMode, + } +} + // newPostReq returns an httptest post request that was created using the // passed in data. func newPostReq(t *testing.T, route string, body interface{}) *http.Request { @@ -44,7 +58,7 @@ func addSessionToReq(t *testing.T, p *politeiawww, req *http.Request, userID str // Init session adds a session cookie onto the http response. r := httptest.NewRequest(http.MethodGet, "/", bytes.NewReader([]byte{})) w := httptest.NewRecorder() - err := p.initSession(w, r, userID) + err := p.sessions.NewSession(w, r, userID) if err != nil { t.Fatal(err) } @@ -63,7 +77,7 @@ func addSessionToReq(t *testing.T, p *politeiawww, req *http.Request, userID str req.AddCookie(c) // Verify the session was added successfully. - s, err := p.getSession(req) + s, err := p.sessions.GetSession(req) if err != nil { t.Fatal(err) } @@ -412,7 +426,7 @@ func TestHandleLogin(t *testing.T) { if err != nil { t.Fatalf("%v", err) } - successReply.SessionMaxAge = sessionMaxAge + successReply.SessionMaxAge = testSessionMaxAge // Setup tests var tests = []struct { @@ -492,7 +506,7 @@ func TestHandleLogin(t *testing.T) { opts := newSessionOptions() c := sessions.NewCookie(www.CookieSession, sessionID, opts) req.AddCookie(c) - s, err := p.getSession(req) + s, err := p.sessions.GetSession(req) if err != nil { t.Error(err) } @@ -956,7 +970,7 @@ func TestHandleUserDetails(t *testing.T) { // Initialize the user session if v.loggedIn { - err := p.initSession(w, r, usr.ID.String()) + err := p.sessions.NewSession(w, r, usr.ID.String()) if err != nil { t.Fatalf("%v", err) } diff --git a/politeiawww/www.go b/politeiawww/www.go index f9119c0f0..bd80c27f9 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -311,9 +311,7 @@ func (p *politeiawww) getPluginInventory() ([]pdv1.Plugin, error) { time.Sleep(sleepInterval) continue } - for _, v := range pi { - plugins = append(plugins, v) - } + plugins = append(plugins, pi...) done = true } diff --git a/util/net.go b/util/net.go index 11b56e3b7..98387a6a9 100644 --- a/util/net.go +++ b/util/net.go @@ -43,7 +43,7 @@ func NewHTTPClient(skipVerify bool, certPath string) (*http.Client, error) { } certPool, err := x509.SystemCertPool() if err != nil { - fmt.Printf("WARN: unable to get system cert pool: %v\n") + fmt.Printf("WARN: unable to get system cert pool: %v\n", err) certPool = x509.NewCertPool() } certPool.AppendCertsFromPEM(cert) @@ -88,7 +88,7 @@ func ParseGetParams(r *http.Request, dst interface{}) error { return schema.NewDecoder().Decode(dst, r.Form) } -// RespBody returns the reponse body as a byte slice. +// RespBody returns the response body as a byte slice. func RespBody(r *http.Response) []byte { var mw io.Writer var body bytes.Buffer diff --git a/util/regexp.go b/util/regexp.go index fe2eb3711..68bbc9e83 100644 --- a/util/regexp.go +++ b/util/regexp.go @@ -13,7 +13,7 @@ import ( // Regexp returns a compiled Regexp for the provided parameters. func Regexp(supportedChars []string, lengthMin, lengthMax uint64) (*regexp.Regexp, error) { - // Match begining of string + // Match beginning of string var b bytes.Buffer b.WriteString("^") diff --git a/util/token.go b/util/token.go index 562ae108a..0831bab70 100644 --- a/util/token.go +++ b/util/token.go @@ -25,8 +25,7 @@ func TokenIsFullLength(tokenType string, token []byte) bool { case TokenTypeGit: return len(token) == pdv1.TokenSizeGit default: - e := fmt.Sprintf("invalid token type") - panic(e) + panic("invalid token type") } } From 971bf2b671e405f641593311865ae7220858a702 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 2 Mar 2021 16:24:45 -0600 Subject: [PATCH 361/449] pd/ticketvote: Improve start vote errors. --- .../tstorebe/plugins/ticketvote/cmds.go | 112 +++++++------ .../tstorebe/plugins/ticketvote/hooks.go | 4 +- .../tstorebe/plugins/ticketvote/inventory.go | 16 +- politeiad/plugins/ticketvote/ticketvote.go | 158 ++++++++++++++---- 4 files changed, 196 insertions(+), 94 deletions(-) diff --git a/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go b/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go index 99987ea26..011cffffd 100644 --- a/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go @@ -1016,6 +1016,21 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri } } + // Verify record version + r, err := p.tstore.RecordPartial(treeID, 0, nil, true) + if err != nil { + return "", fmt.Errorf("RecordPartial: %v", err) + } + if version != r.Version { + e := fmt.Sprintf("version is not latest: got %v, want %v", + a.Version, r.Version) + return "", backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: int(ticketvote.ErrorCodeRecordVersionInvalid), + ErrorContext: e, + } + } + // Get any previous authorizations to verify that the new action // is allowed based on the previous action. auths, err := p.auths(treeID) @@ -1083,7 +1098,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri // Should not happen return "", fmt.Errorf("invalid action %v", a.Action) } - p.InventoryUpdate(a.Token, status) + p.inventoryUpdate(a.Token, status) // Prepare reply ar := ticketvote.AuthorizeReply{ @@ -1130,11 +1145,9 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM case ticketvote.VoteTypeRunoff: // This is allowed default: - e := fmt.Sprintf("invalid type %v", vote.Type) return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), - ErrorContext: e, + PluginID: ticketvote.PluginID, + ErrorCode: int(ticketvote.ErrorCodeVoteTypeInvalid), } } @@ -1145,7 +1158,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM vote.Duration, voteDurationMax) return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteDurationInvalid), ErrorContext: e, } case vote.Duration < voteDurationMin: @@ -1153,7 +1166,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM vote.Duration, voteDurationMin) return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteDurationInvalid), ErrorContext: e, } case vote.QuorumPercentage > 100: @@ -1161,7 +1174,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM vote.QuorumPercentage) return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteQuorumInvalid), ErrorContext: e, } case vote.PassPercentage > 100: @@ -1169,7 +1182,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM vote.PassPercentage) return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVotePassRateInvalid), ErrorContext: e, } } @@ -1179,7 +1192,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM if len(vote.Options) == 0 { return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteOptionsInvalid), ErrorContext: "no vote options found", } } @@ -1193,7 +1206,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM len(vote.Options)) return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteOptionsInvalid), ErrorContext: e, } } @@ -1222,7 +1235,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM strings.Join(missing, ",")) return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteOptionsInvalid), ErrorContext: e, } } @@ -1234,7 +1247,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM if err != nil { return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteBitsInvalid), ErrorContext: err.Error(), } } @@ -1246,7 +1259,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM e := "parent token should not be provided for a standard vote" return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteParentInvalid), ErrorContext: e, } case vote.Type == ticketvote.VoteTypeRunoff: @@ -1255,7 +1268,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM e := fmt.Sprintf("invalid parent %v", vote.Parent) return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteParentInvalid), ErrorContext: e, } } @@ -1271,7 +1284,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot return nil, backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeStartDetailsInvalid), - ErrorContext: "more than one start details found", + ErrorContext: "more than one start details found for standard vote", } } sd := s.Starts[0] @@ -1285,7 +1298,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot } } if !bytes.Equal(t, token) { - e := fmt.Sprintf("plugin token does not match route token: "+ + e := fmt.Sprintf("plugin payload token does not match route token: "+ "got %x, want %x", t, token) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, @@ -1318,9 +1331,9 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot } // Verify record version - r, err := p.backend.GetVetted(token, "") + r, err := p.tstore.RecordPartial(treeID, 0, nil, true) if err != nil { - return nil, fmt.Errorf("GetVetted: %v", err) + return nil, fmt.Errorf("RecordPartial: %v", err) } version := strconv.FormatUint(uint64(sd.Params.Version), 10) if r.Version != version { @@ -1341,15 +1354,15 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot if len(auths) == 0 { return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), - ErrorContext: "authorization not found", + ErrorCode: int(ticketvote.ErrorCodeVoteStatusInvalid), + ErrorContext: "not authorized", } } action := ticketvote.AuthActionT(auths[len(auths)-1].Action) if action != ticketvote.AuthActionAuthorize { return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteStatusInvalid), ErrorContext: "not authorized", } } @@ -1386,7 +1399,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot } // Update inventory - p.InventoryUpdateToStarted(vd.Params.Token, ticketvote.VoteStatusStarted, + p.inventoryUpdateToStarted(vd.Params.Token, ticketvote.VoteStatusStarted, vd.EndBlockHeight) // Update active votes cache @@ -1538,9 +1551,9 @@ func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs sta } // Verify record version - r, err := p.backend.GetVetted(token, "") + r, err := p.tstore.RecordPartial(treeID, 0, nil, true) if err != nil { - return fmt.Errorf("GetVetted: %v", err) + return fmt.Errorf("RecordPartial: %v", err) } version := strconv.FormatUint(uint64(sd.Params.Version), 10) if r.Version != version { @@ -1571,7 +1584,7 @@ func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs sta } // Update inventory - p.InventoryUpdateToStarted(vd.Params.Token, ticketvote.VoteStatusStarted, + p.inventoryUpdateToStarted(vd.Params.Token, ticketvote.VoteStatusStarted, vd.EndBlockHeight) // Update active votes cache @@ -1607,25 +1620,30 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti } // Verify parent has a LinkBy and the LinkBy deadline is expired. - r, err := p.backend.GetVetted(token, "") + r, err := p.tstore.RecordPartial(treeID, 0, []string{ + ticketvote.FileNameVoteMetadata, + }, false) if err != nil { if errors.Is(err, backend.ErrRecordNotFound) { e := fmt.Sprintf("parent record not found %x", token) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteParentInvalid), ErrorContext: e, } } + return nil, fmt.Errorf("RecordPartial: %v", err) } vm, err := voteMetadataDecode(r.Files) if err != nil { return nil, err } if vm == nil || vm.LinkBy == 0 { + e := fmt.Sprintf("%x is not a runoff vote parent", token) return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeRunoffVoteParentInvalid), + PluginID: ticketvote.PluginID, + ErrorCode: int(ticketvote.ErrorCodeVoteParentInvalid), + ErrorContext: e, } } isExpired := vm.LinkBy < time.Now().Unix() @@ -1756,11 +1774,11 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. // Verify vote params are the same for all submissions switch { case v.Params.Type != ticketvote.VoteTypeRunoff: - e := fmt.Sprintf("%v vote type invalid: got %v, want %v", + e := fmt.Sprintf("got %v, want %v", v.Params.Token, v.Params.Type, ticketvote.VoteTypeRunoff) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteTypeInvalid), ErrorContext: e, } case v.Params.Mask != mask: @@ -1768,39 +1786,39 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. v.Params.Token) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteBitsInvalid), ErrorContext: e, } case v.Params.Duration != duration: - e := fmt.Sprintf("%v duration invalid: all must be the same", + e := fmt.Sprintf("%v duration does not match; all must be the same", v.Params.Token) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteDurationInvalid), ErrorContext: e, } case v.Params.QuorumPercentage != quorum: - e := fmt.Sprintf("%v quorum invalid: must be the same", + e := fmt.Sprintf("%v quorum does not match; all must be the same", v.Params.Token) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteQuorumInvalid), ErrorContext: e, } case v.Params.PassPercentage != pass: - e := fmt.Sprintf("%v pass rate invalid: all must be the same", + e := fmt.Sprintf("%v pass rate does not match; all must be the same", v.Params.Token) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVotePassRateInvalid), ErrorContext: e, } case v.Params.Parent != parent: - e := fmt.Sprintf("%v parent invalid: all must be the same", + e := fmt.Sprintf("%v parent does not match; all must be the same", v.Params.Token) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteParentInvalid), ErrorContext: e, } } @@ -1851,7 +1869,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. parent) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeRunoffVoteParentInvalid), + ErrorCode: int(ticketvote.ErrorCodeVoteParentInvalid), ErrorContext: e, } } @@ -1946,8 +1964,6 @@ func (p *ticketVotePlugin) cmdStart(treeID int64, token []byte, payload string) vtype := s.Starts[0].Params.Type // Start vote - // TODO these vote user errors need to become more granular. Update - // this when writing tests. var sr *ticketvote.StartReply switch vtype { case ticketvote.VoteTypeStandard: @@ -1961,11 +1977,9 @@ func (p *ticketVotePlugin) cmdStart(treeID int64, token []byte, payload string) return "", err } default: - e := fmt.Sprintf("invalid vote type %v", vtype) return "", backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteParamsInvalid), - ErrorContext: e, + PluginID: ticketvote.PluginID, + ErrorCode: int(ticketvote.ErrorCodeVoteTypeInvalid), } } @@ -2585,7 +2599,7 @@ func (p *ticketVotePlugin) cmdInventory(payload string) (string, error) { } // Get the inventory - ibs, err := p.InventoryByStatus(bb, i.Status, i.Page) + ibs, err := p.inventoryByStatus(bb, i.Status, i.Page) if err != nil { return "", fmt.Errorf("invByStatus: %v", err) } diff --git a/politeiad/backend/tstorebe/plugins/ticketvote/hooks.go b/politeiad/backend/tstorebe/plugins/ticketvote/hooks.go index 54c874fd4..9d30c0a35 100644 --- a/politeiad/backend/tstorebe/plugins/ticketvote/hooks.go +++ b/politeiad/backend/tstorebe/plugins/ticketvote/hooks.go @@ -330,13 +330,13 @@ func (p *ticketVotePlugin) hookSetRecordStatusPost(treeID int64, payload string) switch newStatus { case backend.MDStatusVetted: // Add to inventory - p.InventoryAdd(srs.RecordMetadata.Token, + p.inventoryAdd(srs.RecordMetadata.Token, ticketvote.VoteStatusUnauthorized) case backend.MDStatusCensored, backend.MDStatusArchived: // These statuses do not allow for a vote. Mark as ineligible. // We only need to do this if the record is vetted. if oldStatus == backend.MDStatusVetted { - p.InventoryUpdate(srs.RecordMetadata.Token, + p.inventoryUpdate(srs.RecordMetadata.Token, ticketvote.VoteStatusIneligible) } } diff --git a/politeiad/backend/tstorebe/plugins/ticketvote/inventory.go b/politeiad/backend/tstorebe/plugins/ticketvote/inventory.go index 93e64efa6..ee66676d9 100644 --- a/politeiad/backend/tstorebe/plugins/ticketvote/inventory.go +++ b/politeiad/backend/tstorebe/plugins/ticketvote/inventory.go @@ -237,9 +237,9 @@ func (p *ticketVotePlugin) invUpdateForBlock(bestBlock uint32) (*inventory, erro return inv, nil } -// InventoryAdd is a wrapper around the invAdd method that allows us to decide +// inventoryAdd is a wrapper around the invAdd method that allows us to decide // how disk read/write errors should be handled. For now we just panic. -func (p *ticketVotePlugin) InventoryAdd(token string, s ticketvote.VoteStatusT) { +func (p *ticketVotePlugin) inventoryAdd(token string, s ticketvote.VoteStatusT) { err := p.invAdd(token, s) if err != nil { e := fmt.Sprintf("invAdd %v %v: %v", token, s, err) @@ -247,9 +247,9 @@ func (p *ticketVotePlugin) InventoryAdd(token string, s ticketvote.VoteStatusT) } } -// InventoryUpdate is a wrapper around the invUpdate method that allows us to +// inventoryUpdate is a wrapper around the invUpdate method that allows us to // decide how disk read/write errors should be handled. For now we just panic. -func (p *ticketVotePlugin) InventoryUpdate(token string, s ticketvote.VoteStatusT) { +func (p *ticketVotePlugin) inventoryUpdate(token string, s ticketvote.VoteStatusT) { err := p.invUpdate(token, s, 0) if err != nil { e := fmt.Sprintf("invUpdate %v %v: %v", token, s, err) @@ -257,10 +257,10 @@ func (p *ticketVotePlugin) InventoryUpdate(token string, s ticketvote.VoteStatus } } -// InventoryUpdateToStarted is a wrapper around the invUpdate method that +// inventoryUpdateToStarted is a wrapper around the invUpdate method that // allows us to decide how disk read/write errors should be handled. For now we // just panic. -func (p *ticketVotePlugin) InventoryUpdateToStarted(token string, s ticketvote.VoteStatusT, endHeight uint32) { +func (p *ticketVotePlugin) inventoryUpdateToStarted(token string, s ticketvote.VoteStatusT, endHeight uint32) { err := p.invUpdate(token, s, endHeight) if err != nil { e := fmt.Sprintf("invUpdate %v %v: %v", token, s, err) @@ -347,9 +347,9 @@ func (p *ticketVotePlugin) invByStatusAll(bestBlock, pageSize uint32) (*invBySta }, nil } -// InventoryByStatus returns a page of tokens for the provided status. If no +// inventoryByStatus returns a page of tokens for the provided status. If no // status is provided then a page for each status will be returned. -func (p *ticketVotePlugin) InventoryByStatus(bestBlock uint32, s ticketvote.VoteStatusT, page uint32) (*invByStatus, error) { +func (p *ticketVotePlugin) inventoryByStatus(bestBlock uint32, s ticketvote.VoteStatusT, page uint32) (*invByStatus, error) { pageSize := ticketvote.InventoryPageSize // If no status is provided a page of tokens for each status should diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index d390dfb4e..8bf1ea795 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -20,15 +20,32 @@ const ( CmdSubmissions = "submissions" // Get runoff vote submissions CmdInventory = "inventory" // Get inventory by vote status CmdTimestamps = "timestamps" // Get vote data timestamps +) - // Setting keys are the plugin setting keys that can be used to - // override a default plugin setting. Defaults will be overridden - // if a plugin setting is provided to the plugin on startup. +// Plugin setting keys can be used to specify custom plugin settings. Default +// plugin setting values can be overridden by providing a plugin setting key +// and value to the plugin on startup. +const ( + // SettingKeyLinkByPeriodMin is the plugin setting key for the link + // by period min plugin setting. SettingKeyLinkByPeriodMin = "linkbyperiodmin" + + // SettingKeyLinkByPeriodMax is the plugin setting key for the link + // by period max plugin setting. SettingKeyLinkByPeriodMax = "linkbyperiodmax" + + // SettingKeyVoteDurationMin is the plugin setting key for the vote + // duration min plugin setting. SettingKeyVoteDurationMin = "votedurationmin" + + // SettingKeyVoteDurationMax is the plugin setting key for the vote + // duration max plugin setting. SettingKeyVoteDurationMax = "votedurationmax" +) +// Plugin setting default values. These can be overridden by providing a plugin +// setting key and value to the plugin on startup. +const ( // SettingMainNetLinkByPeriodMin is the default minimum amount of // time, in seconds, that the link by period can be set to. This // value of 2 weeks was chosen assuming a 1 week voting period on @@ -73,43 +90,114 @@ const ( type ErrorCodeT int const ( - ErrorCodeInvalid ErrorCodeT = 0 - ErrorCodeTokenInvalid ErrorCodeT = 1 - ErrorCodePublicKeyInvalid ErrorCodeT = 2 - ErrorCodeSignatureInvalid ErrorCodeT = 3 - ErrorCodeRecordVersionInvalid ErrorCodeT = 4 - ErrorCodeRecordStatusInvalid ErrorCodeT = 5 - ErrorCodeAuthorizationInvalid ErrorCodeT = 6 - ErrorCodeStartDetailsMissing ErrorCodeT = 7 - ErrorCodeStartDetailsInvalid ErrorCodeT = 8 - ErrorCodeVoteParamsInvalid ErrorCodeT = 9 - ErrorCodeVoteStatusInvalid ErrorCodeT = 10 - ErrorCodeVoteMetadataInvalid ErrorCodeT = 11 - ErrorCodeLinkByInvalid ErrorCodeT = 12 - ErrorCodeLinkToInvalid ErrorCodeT = 13 - ErrorCodeRunoffVoteParentInvalid ErrorCodeT = 14 - ErrorCodeLinkByNotExpired ErrorCodeT = 15 + // ErrorCodeInvalid is an invalid error code. + ErrorCodeInvalid ErrorCodeT = 0 + + // ErrorCodeTokenInvalid is returned when a record token is + // provided as part of a plugin command payload and is not a valid + // token or the payload token does not match the token that was + // used in the API request. + ErrorCodeTokenInvalid ErrorCodeT = 1 + + // ErrorCodePublicKeyInvalid is returned when a public key is not + // a valid hex encoded, Ed25519 public key. + ErrorCodePublicKeyInvalid ErrorCodeT = 2 + + // ErrorCodeSignatureInvalid is returned when a signature is not + // a valid hex encoded, Ed25519 signature or when the signature is + // wrong. + ErrorCodeSignatureInvalid ErrorCodeT = 3 + + // ErrorCodeRecordVersionInvalid is returned when the record + // version used in a plugin command is not the most recent record + // version. + ErrorCodeRecordVersionInvalid ErrorCodeT = 4 + + // ErrorCodeAuthorizationInvalid is returned when a vote + // authorization is invalid. + ErrorCodeAuthorizationInvalid ErrorCodeT = 5 + + // ErrorCodeStartDetailsMissing is returned when a start command + // is missing one or more of the start details that it expects to + // be present. + ErrorCodeStartDetailsMissing ErrorCodeT = 6 + + // ErrorCodeStartDetailsInvalid is returned when a start command + // contains a start details that is not suppose to be included. + ErrorCodeStartDetailsInvalid ErrorCodeT = 7 + + // ErrorCodeVoteTypeInvalid is returned when a start details vote + // type is invalid. + ErrorCodeVoteTypeInvalid ErrorCodeT = 8 + + // ErrorCodeVoteDurationInvalid is returned when a start details + // vote duration is invalid. + ErrorCodeVoteDurationInvalid ErrorCodeT = 9 + + // ErrorCodeVoteQuorumInvalid is returned when a start details + // quorum percentage is invalid. + ErrorCodeVoteQuorumInvalid ErrorCodeT = 10 + + // ErrorCodeVotePassRateInvalid is returned when a start details + // pass percentage is invalid. + ErrorCodeVotePassRateInvalid ErrorCodeT = 11 + + // ErrorCodeVoteOptionsInvalid is returned when a start details + // vote options are invalid. + ErrorCodeVoteOptionsInvalid ErrorCodeT = 12 + + // ErrorCodeVoteBitsInvalid is returned when a vote bit or the mask + // of a start details is invalid. + ErrorCodeVoteBitsInvalid ErrorCodeT = 13 + + // ErrorCodeVoteParentInvalid is returned when a parent record + // of a runoff submission's start details is invalid. + ErrorCodeVoteParentInvalid ErrorCodeT = 14 + + // ErrorCodeVoteStatusInvalid is returned when the record's vote + // status does not allow for the command to be executed. + ErrorCodeVoteStatusInvalid ErrorCodeT = 15 + + // ErrorCodeVoteMetadataInvalid is returned when vote metadata + // attached to a record is invalid. + ErrorCodeVoteMetadataInvalid ErrorCodeT = 16 + + // ErrorCodeLinkByInvalid is returned when a vote metadata link by + // is invalid. + ErrorCodeLinkByInvalid ErrorCodeT = 17 + + // ErrorCodeLinkToInvalid is returned when a vote metadata link to + // is invalid. + ErrorCodeLinkToInvalid ErrorCodeT = 18 + + // ErrorCodeLinkByNotExpired is returned when a runoff vote is + // attempted to be started before the link by deadline has expired. + ErrorCodeLinkByNotExpired ErrorCodeT = 19 ) var ( // ErrorCodes contains the human readable error messages. ErrorCodes = map[ErrorCodeT]string{ - ErrorCodeInvalid: "error code invalid", - ErrorCodeTokenInvalid: "token invalid", - ErrorCodePublicKeyInvalid: "public key invalid", - ErrorCodeSignatureInvalid: "signature invalid", - ErrorCodeRecordVersionInvalid: "record version invalid", - ErrorCodeRecordStatusInvalid: "record status invalid", - ErrorCodeAuthorizationInvalid: "authorization invalid", - ErrorCodeStartDetailsMissing: "start details missing", - ErrorCodeStartDetailsInvalid: "start details invalid", - ErrorCodeVoteParamsInvalid: "vote params invalid", - ErrorCodeVoteStatusInvalid: "vote status invalid", - ErrorCodeVoteMetadataInvalid: "vote metadata invalid", - ErrorCodeLinkByInvalid: "linkby invalid", - ErrorCodeLinkToInvalid: "linkto invalid", - ErrorCodeRunoffVoteParentInvalid: "runoff vote parent invalid", - ErrorCodeLinkByNotExpired: "linkby not exipred", + ErrorCodeInvalid: "error code invalid", + ErrorCodeTokenInvalid: "token invalid", + ErrorCodePublicKeyInvalid: "public key invalid", + ErrorCodeSignatureInvalid: "signature invalid", + ErrorCodeRecordVersionInvalid: "record version invalid", + ErrorCodeAuthorizationInvalid: "authorization invalid", + ErrorCodeStartDetailsMissing: "start details missing", + ErrorCodeStartDetailsInvalid: "start details invalid", + ErrorCodeVoteTypeInvalid: "vote type invalid", + ErrorCodeVoteDurationInvalid: "vote duration invalid", + ErrorCodeVoteQuorumInvalid: "quorum percentage invalid", + ErrorCodeVotePassRateInvalid: "pass rate invalid", + ErrorCodeVoteOptionsInvalid: "vote options invalid", + ErrorCodeVoteBitsInvalid: "vote bits invalid", + ErrorCodeVoteParentInvalid: "vote parent invalid", + ErrorCodeVoteStatusInvalid: "vote status invalid", + ErrorCodeVoteMetadataInvalid: "vote metadata invalid", + ErrorCodeLinkByInvalid: "linkby invalid", + ErrorCodeLinkToInvalid: "linkto invalid", + ErrorCodeLinkByNotExpired: "linkby not exipred", } ) From 6efe17df88beb491bfea1a823384d3cb26dd1c0e Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 4 Mar 2021 09:44:39 -0600 Subject: [PATCH 362/449] Linter fix and bug fix. --- politeiad/backend/tstorebe/plugins/ticketvote/cmds.go | 2 +- politeiad/client/user.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go b/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go index 011cffffd..05a6812c4 100644 --- a/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go +++ b/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go @@ -1774,7 +1774,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. // Verify vote params are the same for all submissions switch { case v.Params.Type != ticketvote.VoteTypeRunoff: - e := fmt.Sprintf("got %v, want %v", + e := fmt.Sprintf("%v got %v, want %v", v.Params.Token, v.Params.Type, ticketvote.VoteTypeRunoff) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, diff --git a/politeiad/client/user.go b/politeiad/client/user.go index 35f3afe8f..73d3f8176 100644 --- a/politeiad/client/user.go +++ b/politeiad/client/user.go @@ -72,6 +72,7 @@ func (c *Client) UserRecords(ctx context.Context, userID string) (map[string][]s Payload: string(b), }, { + Action: pdv1.PluginActionRead, State: pdv1.RecordStateVetted, ID: usermd.PluginID, Command: usermd.CmdUserRecords, From 385f781b6cfa3d82a7f8e09a06cb165e5dd8b80d Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 4 Mar 2021 08:47:11 -0600 Subject: [PATCH 363/449] politeiad: Add v2 api and backend. --- politeiad/api/v1/v1.go | 264 +-- politeiad/api/v2/v2.go | 511 +++++ politeiad/backend/backend.go | 166 +- politeiad/backend/gitbe/gitbe.go | 131 +- politeiad/backend/tstorebe/tstore/testing.go | 63 - politeiad/backend/tstorebe/tstorebe.go | 2016 ----------------- politeiad/backendv2/backendv2.go | 350 +++ .../tstorebe/inventory.go | 71 +- .../{backend => backendv2}/tstorebe/log.go | 0 .../tstorebe/plugins/comments/cmds.go | 4 +- .../tstorebe/plugins/comments/comments.go | 4 +- .../tstorebe/plugins/comments/log.go | 0 .../tstorebe/plugins/comments/recordindex.go | 0 .../tstorebe/plugins/dcrdata/dcrdata.go | 4 +- .../tstorebe/plugins/dcrdata/log.go | 0 .../tstorebe/plugins/pi/hooks.go | 23 +- .../tstorebe/plugins/pi/hooks_test.go | 0 .../tstorebe/plugins/pi/log.go | 0 .../tstorebe/plugins/pi/pi.go | 4 +- .../tstorebe/plugins/pi/testing.go | 0 .../tstorebe/plugins/plugins.go | 58 +- .../plugins/ticketvote/activevotes.go | 0 .../tstorebe/plugins/ticketvote/cmds.go | 100 +- .../tstorebe/plugins/ticketvote/hooks.go | 35 +- .../tstorebe/plugins/ticketvote/inventory.go | 0 .../tstorebe/plugins/ticketvote/log.go | 0 .../plugins/ticketvote/submissions.go | 0 .../tstorebe/plugins/ticketvote/summary.go | 0 .../tstorebe/plugins/ticketvote/ticketvote.go | 18 +- .../tstorebe/plugins/usermd/cache.go | 0 .../tstorebe/plugins/usermd/cmds.go | 0 .../tstorebe/plugins/usermd/hooks.go | 16 +- .../tstorebe/plugins/usermd/log.go | 0 .../tstorebe/plugins/usermd/usermd.go | 4 +- .../tstorebe/store/localdb/localdb.go | 2 +- .../tstorebe/store/localdb/log.go | 0 .../tstorebe/store/mysql/log.go | 0 .../tstorebe/store/mysql/mysql.go | 2 +- .../tstorebe/store/store.go | 0 .../tstorebe/testing.go | 20 +- .../tstorebe/tstore/anchor.go | 22 +- .../tstorebe/tstore/client.go | 16 +- .../tstorebe/tstore/convert.go | 4 +- .../tstorebe/tstore/dcrtime.go | 0 .../tstorebe/tstore/encryptionkey.go | 0 .../tstorebe/tstore/log.go | 0 .../tstorebe/tstore/plugin.go | 45 +- .../tstorebe/tstore/recordindex.go | 29 +- .../backendv2/tstorebe/tstore/testing.go | 46 + .../tstorebe/tstore/trillianclient.go | 2 +- .../tstorebe/tstore/tstore.go | 369 ++- .../tstorebe/tstore/verify.go | 2 +- politeiad/backendv2/tstorebe/tstorebe.go | 1156 ++++++++++ politeiad/config.go | 44 +- politeiad/log.go | 16 +- politeiad/plugins/usermd/usermd.go | 14 +- politeiad/politeiad.go | 862 ++----- politeiad/v2.go | 699 ++++++ politeiawww/api/records/v1/v1.go | 11 +- politeiawww/cmd/pictl/proposal.go | 3 - politeiawww/log.go | 4 +- util/convert.go | 13 +- util/token.go | 5 +- 63 files changed, 3375 insertions(+), 3853 deletions(-) create mode 100644 politeiad/api/v2/v2.go delete mode 100644 politeiad/backend/tstorebe/tstore/testing.go delete mode 100644 politeiad/backend/tstorebe/tstorebe.go create mode 100644 politeiad/backendv2/backendv2.go rename politeiad/{backend => backendv2}/tstorebe/inventory.go (79%) rename politeiad/{backend => backendv2}/tstorebe/log.go (100%) rename politeiad/{backend => backendv2}/tstorebe/plugins/comments/cmds.go (99%) rename politeiad/{backend => backendv2}/tstorebe/plugins/comments/comments.go (97%) rename politeiad/{backend => backendv2}/tstorebe/plugins/comments/log.go (100%) rename politeiad/{backend => backendv2}/tstorebe/plugins/comments/recordindex.go (100%) rename politeiad/{backend => backendv2}/tstorebe/plugins/dcrdata/dcrdata.go (99%) rename politeiad/{backend => backendv2}/tstorebe/plugins/dcrdata/log.go (100%) rename politeiad/{backend => backendv2}/tstorebe/plugins/pi/hooks.go (91%) rename politeiad/{backend => backendv2}/tstorebe/plugins/pi/hooks_test.go (100%) rename politeiad/{backend => backendv2}/tstorebe/plugins/pi/log.go (100%) rename politeiad/{backend => backendv2}/tstorebe/plugins/pi/pi.go (98%) rename politeiad/{backend => backendv2}/tstorebe/plugins/pi/testing.go (100%) rename politeiad/{backend => backendv2}/tstorebe/plugins/plugins.go (79%) rename politeiad/{backend => backendv2}/tstorebe/plugins/ticketvote/activevotes.go (100%) rename politeiad/{backend => backendv2}/tstorebe/plugins/ticketvote/cmds.go (96%) rename politeiad/{backend => backendv2}/tstorebe/plugins/ticketvote/hooks.go (92%) rename politeiad/{backend => backendv2}/tstorebe/plugins/ticketvote/inventory.go (100%) rename politeiad/{backend => backendv2}/tstorebe/plugins/ticketvote/log.go (100%) rename politeiad/{backend => backendv2}/tstorebe/plugins/ticketvote/submissions.go (100%) rename politeiad/{backend => backendv2}/tstorebe/plugins/ticketvote/summary.go (100%) rename politeiad/{backend => backendv2}/tstorebe/plugins/ticketvote/ticketvote.go (95%) rename politeiad/{backend => backendv2}/tstorebe/plugins/usermd/cache.go (100%) rename politeiad/{backend => backendv2}/tstorebe/plugins/usermd/cmds.go (100%) rename politeiad/{backend => backendv2}/tstorebe/plugins/usermd/hooks.go (96%) rename politeiad/{backend => backendv2}/tstorebe/plugins/usermd/log.go (100%) rename politeiad/{backend => backendv2}/tstorebe/plugins/usermd/usermd.go (95%) rename politeiad/{backend => backendv2}/tstorebe/store/localdb/localdb.go (97%) rename politeiad/{backend => backendv2}/tstorebe/store/localdb/log.go (100%) rename politeiad/{backend => backendv2}/tstorebe/store/mysql/log.go (100%) rename politeiad/{backend => backendv2}/tstorebe/store/mysql/mysql.go (98%) rename politeiad/{backend => backendv2}/tstorebe/store/store.go (100%) rename politeiad/{backend => backendv2}/tstorebe/testing.go (56%) rename politeiad/{backend => backendv2}/tstorebe/tstore/anchor.go (96%) rename politeiad/{backend => backendv2}/tstorebe/tstore/client.go (95%) rename politeiad/{backend => backendv2}/tstorebe/tstore/convert.go (97%) rename politeiad/{backend => backendv2}/tstorebe/tstore/dcrtime.go (100%) rename politeiad/{backend => backendv2}/tstorebe/tstore/encryptionkey.go (100%) rename politeiad/{backend => backendv2}/tstorebe/tstore/log.go (100%) rename politeiad/{backend => backendv2}/tstorebe/tstore/plugin.go (77%) rename politeiad/{backend => backendv2}/tstorebe/tstore/recordindex.go (89%) create mode 100644 politeiad/backendv2/tstorebe/tstore/testing.go rename politeiad/{backend => backendv2}/tstorebe/tstore/trillianclient.go (99%) rename politeiad/{backend => backendv2}/tstorebe/tstore/tstore.go (80%) rename politeiad/{backend => backendv2}/tstorebe/tstore/verify.go (98%) create mode 100644 politeiad/backendv2/tstorebe/tstorebe.go create mode 100644 politeiad/v2.go diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index 3a38fc110..3a547773c 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -30,26 +30,18 @@ const ( UpdateVettedMetadataRoute = "/v1/updatevettedmd/" // Update vetted metadata GetUnvettedRoute = "/v1/getunvetted/" // Get unvetted record GetVettedRoute = "/v1/getvetted/" // Get vetted record - GetUnvettedBatchRoute = "/v1/getunvettedbatch/" // Get unvetted records - GetVettedBatchRoute = "/v1/getvettedbatch/" // Get vetted records - GetUnvettedTimestampsRoute = "/v1/getunvettedts/" // Get unvetted timestamps - GetVettedTimestampsRoute = "/v1/getvettedts/" // Get vetted timestamps - InventoryByStatusRoute = "/v1/inventorybystatus/" // Auth required - InventoryRoute = "/v1/inventory/" // Inventory records - SetUnvettedStatusRoute = "/v1/setunvettedstatus/" // Set unvetted status - SetVettedStatusRoute = "/v1/setvettedstatus/" // Set vetted status - PluginCommandRoute = "/v1/plugin/" // Send a command to a plugin - PluginCommandBatchRoute = "/v1/plugin/batch" // Send commands to plugins - PluginInventoryRoute = "/v1/plugin/inventory/" // Inventory all plugins + InventoryRoute = "/v1/inventory/" // Inventory records + SetUnvettedStatusRoute = "/v1/setunvettedstatus/" // Set unvetted status + SetVettedStatusRoute = "/v1/setvettedstatus/" // Set vetted status + PluginCommandRoute = "/v1/plugin/" // Send a command to a plugin + PluginInventoryRoute = "/v1/plugin/inventory/" // Inventory all plugins ChallengeSize = 32 // Size of challenge token in bytes - // Token sizes. The size of the token depends on the politeiad - // backend configuration. - TokenSizeTstore = 8 - TokenSizeGit = 32 + // TokenSize is the size of a censorship record token in bytes. + TokenSize = 32 MetadataStreamsMax = uint64(16) // Maximum number of metadata streams @@ -71,11 +63,8 @@ const ( ErrorStatusNoChanges ErrorStatusT = 14 ErrorStatusRecordFound ErrorStatusT = 15 ErrorStatusInvalidRPCCredentials ErrorStatusT = 16 - ErrorStatusRecordNotFound ErrorStatusT = 17 - ErrorStatusInvalidToken ErrorStatusT = 18 - ErrorStatusRecordLocked ErrorStatusT = 19 - ErrorStatusInvalidRecordState ErrorStatusT = 20 - ErrorStatusInvalidPluginAction ErrorStatusT = 21 + ErrorStatusInvalidToken ErrorStatusT = 17 + ErrorStatusRecordNotFound ErrorStatusT = 18 // Record status codes (set and get) RecordStatusInvalid RecordStatusT = 0 // Invalid status @@ -113,12 +102,8 @@ var ( ErrorStatusDuplicateFilename: "duplicate filename", ErrorStatusFileNotFound: "file not found", ErrorStatusNoChanges: "no changes in record", - ErrorStatusRecordFound: "record found", - ErrorStatusInvalidRPCCredentials: "invalid RPC client credentials", ErrorStatusInvalidToken: "invalid token", - ErrorStatusRecordLocked: "record locked", - ErrorStatusInvalidRecordState: "invalid record state", - ErrorStatusInvalidPluginAction: "invalid plugin action", + ErrorStatusRecordNotFound: "record not found", } // RecordStatus converts record status codes to human readable text. @@ -229,9 +214,8 @@ type File struct { // MetadataStream identifies a metadata stream by its identity. type MetadataStream struct { - PluginID string `json:"pluginid,omitempty"` // Plugin identity - ID uint64 `json:"id"` // Stream identity - Payload string `json:"payload"` // String encoded metadata + ID uint64 `json:"id"` // Stream identity + Payload string `json:"payload"` // String encoded metadata } // Record is an entire record and it's content. @@ -258,8 +242,8 @@ type NewRecord struct { // NewRecordReply returns the CensorshipRecord that is associated with a valid // record. A valid record is not always going to be published. type NewRecordReply struct { - Response string `json:"response"` // Challenge response - CensorshipRecord CensorshipRecord `json:"censorshiprecord"` // Censorship record + Response string `json:"response"` // Challenge response + CensorshipRecord CensorshipRecord `json:"censorshiprecord"` } // GetUnvetted requests an unvetted record from the server. @@ -371,76 +355,6 @@ type UpdateUnvettedMetadataReply struct { Response string `json:"response"` // Challenge response } -// Proof contains an inclusion proof for the digest in the merkle root. All -// digests are hex encoded SHA256 digests. -// -// The ExtraData field is used by certain types of proofs to include additional -// data that is required to validate the proof. -type Proof struct { - Type string `json:"type"` - Digest string `json:"digest"` - MerkleRoot string `json:"merkleroot"` - MerklePath []string `json:"merklepath"` - ExtraData string `json:"extradata"` // JSON encoded -} - -// Timestamp contains all of the data required to verify that a piece of record -// content was timestamped onto the decred blockchain. -// -// All digests are hex encoded SHA256 digests. The merkle root can be found in -// the OP_RETURN of the specified DCR transaction. -// -// TxID, MerkleRoot, and Proofs will only be populated once the merkle root has -// been included in a DCR tx and the tx has 6 confirmations. The Data field -// will not be populated if the data has been censored. -type Timestamp struct { - Data string `json:"data"` // JSON encoded - Digest string `json:"digest"` - TxID string `json:"txid"` - MerkleRoot string `json:"merkleroot"` - Proofs []Proof `json:"proofs"` -} - -// RecordTimestamps contains a Timestamp for all record files -type RecordTimestamps struct { - Token string `json:"token"` // Censorship token - Version string `json:"version"` // Version of files - RecordMetadata Timestamp `json:"recordmetadata"` - - // map[pluginID+metadataID]Timestamp - Metadata map[string]Timestamp `json:"metadata"` - - // map[filename]Timestamp - Files map[string]Timestamp `json:"files"` -} - -// GetUnvettedTimestamps requests the timestamps for an unvetted record. -type GetUnvettedTimestamps struct { - Challenge string `json:"challenge"` // Random challenge - Token string `json:"token"` // Censorship token - Version string `json:"version"` // Record version -} - -// GetUnvettedTimestampsReply is the reply to the GetUnvettedTimestamps -// command. -type GetUnvettedTimestampsReply struct { - Response string `json:"response"` // Challenge response - RecordTimestamps RecordTimestamps `json:"timestamp"` -} - -// GetVettedTimestamps requests the timestamps for a vetted record. -type GetVettedTimestamps struct { - Challenge string `json:"challenge"` // Random challenge - Token string `json:"token"` // Censorship token - Version string `json:"version"` // Record version -} - -// GetVettedTimestampsReply is the reply to the GetVettedTimestamps command. -type GetVettedTimestampsReply struct { - Response string `json:"response"` // Challenge response - RecordTimestamps RecordTimestamps `json:"timestamp"` -} - // Inventory sends an (expensive and therefore authenticated) inventory request // for vetted records (master branch) and branches (censored, unpublished etc) // records. This is a very expensive call and should be only issued at start @@ -469,36 +383,6 @@ type InventoryReply struct { Branches []Record `json:"branches"` // Last N branches (censored, new etc) } -const ( - // InventoryPageSize is the maximum number of tokens that will be - // returned for any single status in an InventoryReply. - InventoryPageSize uint32 = 20 -) - -// Inventory requests the tokens of the records in the inventory, categorized -// by record state and record status. The tokens are ordered by the timestamp -// of their most recent status change, sorted from newest to oldest. -// -// The state, status, and page arguments can be provided to request a specific -// page of record tokens. -// -// If no status is provided then a page of tokens for all statuses will be -// returned. The page argument will be ignored. -type InventoryByStatus struct { - Challenge string `json:"challenge"` // Random challenge - State string `json:"state,omitempty"` - Status RecordStatusT `json:"status,omitempty"` - Page uint32 `json:"page,omitempty"` -} - -// InventoryByStatusReply returns all censorship record tokens categorized by -// record state and record status. -type InventoryByStatusReply struct { - Response string `json:"response"` // Challenge response - Unvetted map[RecordStatusT][]string `json:"unvetted"` - Vetted map[RecordStatusT][]string `json:"vetted"` -} - // UserErrorReply returns details about an error that occurred while trying to // execute a command due to bad input from the client. type UserErrorReply struct { @@ -511,19 +395,6 @@ func (e UserErrorReply) Error() string { return fmt.Sprintf("user error code: %v", e.ErrorCode) } -// PluginUserErrorReply returns details about a plugin error that occurred -// while trying to execute a command due to bad input from the client. -type PluginErrorReply struct { - PluginID string `json:"pluginid"` - ErrorCode int `json:"errorcode"` - ErrorContext []string `json:"errorcontext,omitempty"` -} - -// Error satisfies the error interface. -func (e PluginErrorReply) Error() string { - return fmt.Sprintf("plugin '%v' error code: %v", e.PluginID, e.ErrorCode) -} - // ServerErrorReply returns an error code that can be correlated with // server logs. type ServerErrorReply struct { @@ -576,110 +447,3 @@ type PluginCommandReply struct { CommandID string `json:"commandid"` // User setable command identifier Payload string `json:"payload"` // Actual command reply } - -const ( - // PluginActionRead is passed in as the Action of a PluginCommandV2 - // to indicate that the plugin command is a read only command. - PluginActionRead = "read" - - // PluginActionWrite is passed in as the Action of a PluginCommandV2 - // to indicate that the plugin command requires writing data. - PluginActionWrite = "write" - - // RecordStateUnvetted is passed in as the State field of a - // PluginCommandV2 to indicate that the plugin command is being - // executed on an unvetted record. - RecordStateUnvetted = "unvetted" - - // RecordStateVetted is passed in as the State field of a - // PluginCommandV2 to indicate that the plugin command is being - // executed on an vetted record. - RecordStateVetted = "vetted" -) - -// PluginCommandV2 sends a command to a plugin. -type PluginCommandV2 struct { - Action string `json:"action"` // Read or write - State string `json:"state"` // Unvetted or vetted - Token string `json:"token"` // Censorship token - ID string `json:"id"` // Plugin identifier - Command string `json:"command"` // Plugin command - Payload string `json:"payload"` // Command payload -} - -// PluginCommandReplyV2 is the reply to a PluginCommandV2. -type PluginCommandReplyV2 struct { - State string `json:"state"` // Unvetted or vetted - Token string `json:"token"` // Censorship token - ID string `json:"id"` // Plugin identifier - Command string `json:"command"` // Plugin command - Payload string `json:"payload"` // Response payload - - // UserError will be populated if a ErrorStatusT is encountered - // before the plugin command could be executed. - UserError *UserErrorReply `json:"usererror,omitempty"` - - // PluginError will be populated if a plugin error occurred during - // plugin command execution. These errors will be specific to the - // plugin. - PluginError *PluginErrorReply `json:"pluginerror,omitempty"` -} - -// PluginCommandBatch executes a batch of plugin commands. -type PluginCommandBatch struct { - Challenge string `json:"challenge"` // Random challenge - Commands []PluginCommandV2 `json:"commands"` -} - -// PluginCommandBatchReply is the reply to a PluginCommandBatch. -type PluginCommandBatchReply struct { - Response string `json:"response"` // Challenge response - Replies []PluginCommandReplyV2 `json:"replies"` -} - -// RecordRequest is used to requests a record. It gives the client granular -// control over what is returned. The only required field is the token. All -// other fields are optional. -// -// Version is used to request a specific version of a record. If no version is -// provided then the most recent version of the record will be returned. -// -// Filenames can be used to request specific files. If filenames is not empty -// then the specified files will be the only files returned. -// -// OmitAllFiles can be used to retrieve a record without any of the record -// files. This supersedes the filenames argument. -type RecordRequest struct { - Token string `json:"token"` - Version string `json:"version,omitempty"` - Filenames []string `json:"filenames,omitempty"` - OmitAllFiles bool `json:"omitallfiles,omitempty"` -} - -// GetUnvettedBatch requests a batch of unvetted records. -type GetUnvettedBatch struct { - Challenge string `json:"challenge"` // Random challenge - Requests []RecordRequest `json:"requests"` -} - -// GetUnvettedBatchReply is the reply to the GetUnvettedBatch command. If a -// record was not found or an error occurred while retrieving it the token will -// not be included in the returned map. -type GetUnvettedBatchReply struct { - Response string `json:"response"` // Challenge response - Records map[string]Record `json:"record"` -} - -// GetVettedBatch requests a batch of unvetted records. -type GetVettedBatch struct { - Challenge string `json:"challenge"` // Random challenge - Requests []RecordRequest `json:"requests"` -} - -// GetVettedBatchReply is the reply to the GetVettedBatch command. If a record -// was not found or an error occurred while retrieving it the token will not be -// included in the returned map. -type GetVettedBatchReply struct { - Response string `json:"response"` // Challenge response - Records map[string]Record `json:"record"` -} diff --git a/politeiad/api/v2/v2.go b/politeiad/api/v2/v2.go new file mode 100644 index 000000000..5b1f024c0 --- /dev/null +++ b/politeiad/api/v2/v2.go @@ -0,0 +1,511 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package v2 + +import "fmt" + +const ( + // RoutePrefix is prefixed onto all routes in this package. + RoutePrefix = "/v2" + + // Routes + RouteRecordNew = "/recordnew" + RouteRecordEdit = "/recordedit" + RouteRecordEditMetadata = "/recordeditmetadata" + RouteRecordSetStatus = "/recordsetstatus" + RouteRecordGet = "/recordget" + RouteRecordGetBatch = "/recordgetbatch" + RouteRecordGetTimestamps = "/recordgettimestamps" + RouteInventory = "/inventory" + RoutePluginWrite = "/pluginwrite" + RoutePluginReads = "/pluginreads" + RoutePluginInventory = "/plugininventory" + + // ChallengeSize is the size of a request challenge token in bytes. + ChallengeSize = 32 +) + +// ErrorCodeT represents a user error code. +type ErrorCodeT uint32 + +const ( + ErrorCodeInvalid ErrorCodeT = 0 + ErrorCodeRequestPayloadInvalid ErrorCodeT = 1 + ErrorCodeChallengeInvalid ErrorCodeT = 2 + ErrorCodeMetadataStreamInvalid ErrorCodeT = 3 + ErrorCodeMetadataStreamDuplicate ErrorCodeT = 4 + ErrorCodeFilesEmpty ErrorCodeT = 5 + ErrorCodeFileNameInvalid ErrorCodeT = 6 + ErrorCodeFileNameDuplicate ErrorCodeT = 7 + ErrorCodeFileDigestInvalid ErrorCodeT = 8 + ErrorCodeFilePayloadInvalid ErrorCodeT = 9 + ErrorCodeFileMIMETypeInvalid ErrorCodeT = 10 + ErrorCodeFileMIMETypeUnsupported ErrorCodeT = 11 + ErrorCodeTokenInvalid ErrorCodeT = 12 + ErrorCodeRecordNotFound ErrorCodeT = 13 + ErrorCodeRecordLocked ErrorCodeT = 14 + ErrorCodeNoRecordChanges ErrorCodeT = 15 + ErrorCodeStatusChangeInvalid ErrorCodeT = 16 + ErrorCodePluginIDInvalid ErrorCodeT = 17 + ErrorCodePluginCmdInvalid ErrorCodeT = 18 +) + +var ( + // ErrorCodes contains the human readable error codes. + ErrorCodes = map[ErrorCodeT]string{ + ErrorCodeInvalid: "invalid error", + ErrorCodeMetadataStreamInvalid: "metadata stream invalid", + ErrorCodeMetadataStreamDuplicate: "metadata stream duplicate", + ErrorCodeFilesEmpty: "files are empty", + ErrorCodeFileNameInvalid: "file name invalid", + ErrorCodeFileNameDuplicate: "file name is a duplicate", + ErrorCodeFileDigestInvalid: "file digest invalid", + ErrorCodeFilePayloadInvalid: "file payload invalid", + ErrorCodeFileMIMETypeInvalid: "file mime type invalid", + ErrorCodeFileMIMETypeUnsupported: "file mime type not supported", + ErrorCodeTokenInvalid: "token invalid", + ErrorCodeRecordNotFound: "record not found", + ErrorCodeRecordLocked: "record is locked", + ErrorCodeNoRecordChanges: "no record changes", + ErrorCodeStatusChangeInvalid: "status change invalid", + ErrorCodePluginIDInvalid: "pluguin id invalid", + ErrorCodePluginCmdInvalid: "plugin cmd invalid", + } +) + +// UserErrorReply is the reply that the server returns when it encounters an +// error that is caused by something that the user did (malformed input, bad +// timing, etc). The HTTP status code will be 400. +type UserErrorReply struct { + ErrorCode ErrorCodeT `json:"errorcode"` + ErrorContext string `json:"errorcontext"` +} + +// Error satisfies the error interface. +func (e UserErrorReply) Error() string { + return fmt.Sprintf("user error code: %v", e.ErrorCode) +} + +// PluginErrorReply is the reply that the server returns when it encounters +// a plugin error. The error code will be specific to the plugin. +type PluginErrorReply struct { + PluginID string `json:"pluginid"` + ErrorCode int `json:"errorcode"` + ErrorContext string `json:"errorcontext"` +} + +// Error satisfies the error interface. +func (e PluginErrorReply) Error() string { + return fmt.Sprintf("plugin %v error code: %v", e.PluginID, e.ErrorCode) +} + +// ServerErrorReply is the reply that the server returns when it encounters an +// unrecoverable error while executing a command. The HTTP status code will be +// 500 and the ErrorCode field will contain a UNIX timestamp that the user can +// provide to the server admin to track down the error details in the logs. +type ServerErrorReply struct { + ErrorCode int64 `json:"errorcode"` +} + +// Error satisfies the error interface. +func (e ServerErrorReply) Error() string { + return fmt.Sprintf("server error: %v", e.ErrorCode) +} + +// RecordStateT represents the state of a record. +type RecordStateT uint32 + +const ( + // RecordStateInvalid is an invalid record state. + RecordStateInvalid RecordStateT = 0 + + // RecordStateUnvetted indicates a record has not been made public. + RecordStateUnvetted RecordStateT = 1 + + // RecordStateVetted indicates a record has been made public. + RecordStateVetted RecordStateT = 2 +) + +var ( + // RecordStates contains the human readable record states. + RecordStates = map[RecordStateT]string{ + RecordStateInvalid: "invalid", + RecordStateUnvetted: "unvetted", + RecordStateVetted: "vetted", + } +) + +// RecordStatusT represents the status of a record. +type RecordStatusT uint32 + +const ( + // RecordStatusInvalid is an invalid status code. + RecordStatusInvalid RecordStatusT = 0 + + // RecordStatusUnreviewed indicates a record has not been made + // public yet. The state of an unreviewed record will always be + // unvetted. + RecordStatusUnreviewed RecordStatusT = 1 + + // RecordStatusPublic indicates a record has been made public. The + // state of a public record will always be vetted. + RecordStatusPublic RecordStatusT = 2 + + // RecordStatusCensored indicates a record has been censored. A + // censored record is locked from any further updates and all + // record content is permanently deleted. A censored record can + // have a state of either unvetted or vetted. + RecordStatusCensored RecordStatusT = 3 + + // RecordStatusArchived indicates a record has been archived. An + // archived record is locked from any further updates. An archived + // record have a state of either unvetted or vetted. + RecordStatusArchived RecordStatusT = 4 +) + +var ( + // RecordStatuses contains the human readable record statuses. + RecordStatuses = map[RecordStatusT]string{ + RecordStatusInvalid: "invalid", + RecordStatusUnreviewed: "unreviewed", + RecordStatusPublic: "public", + RecordStatusCensored: "censored", + RecordStatusArchived: "archived", + } +) + +// MetadataStream describes a single metada stream. +type MetadataStream struct { + PluginID string `json:"pluginid"` // Plugin identity + StreamID uint32 `json:"streamid"` // Stream identity + Payload string `json:"payload"` // JSON encoded metadata +} + +// File represents a record file. +type File struct { + Name string `json:"name"` // Basename of the file + MIME string `json:"mime"` // MIME type + Digest string `json:"digest"` // SHA256 of decoded Payload + Payload string `json:"payload"` // Base64 encoded file payload +} + +const ( + // TokenSize is the size of a censorship record token in bytes. + TokenSize = 8 + + // TokenSizeShort is the size, in characters, of a hex encoded + // token that has been shortened to improved UX. Short tokens can + // be used to retrieve record data but cannot be used on any routes + // that write record data. 7 characters was chosen to match the git + // abbreviated commitment hash size. + TokenSizeShort = 7 +) + +// CensorshipRecord contains cryptographic proof that a record was accepted for +// review by the server. The proof is verifiable by the client. +type CensorshipRecord struct { + // Token is a random censorship token that is generated by the + // server. It serves as a unique identifier for the record. + Token string `json:"token"` + + // Merkle is the ordered merkle root of all files in the record. + Merkle string `json:"merkle"` + + // Signature is the server signature of the Merkle+Token. + Signature string `json:"signature"` +} + +// Record represents a record and all of its contents. +type Record struct { + State RecordStateT `json:"state"` // Record state + Status RecordStatusT `json:"status"` // Record status + Version uint32 `json:"version"` // Version of this record + Timestamp int64 `json:"timestamp"` // Last update + Metadata []MetadataStream `json:"metadata"` + Files []File `json:"files"` + + CensorshipRecord CensorshipRecord `json:"censorshiprecord"` +} + +// RecordNew creates a new record. It must include all files that are part of +// the record and it may contain optional metadata. +type RecordNew struct { + Challenge string `json:"challenge"` // Random challenge + Metadata []MetadataStream `json:"metadata"` + Files []File `json:"files"` +} + +// RecordNewReply is the reply to the RecordNew command. +type RecordNewReply struct { + Response string `json:"response"` // Challenge response + Record Record `json:"record"` +} + +// RecordEdit edits and existing record. +// +// MDAppend appends metadata to a metadata stream. MDOverwrite overwrites a +// metadata stream. If the metadata stream does not exist yet for either of +// these arguments, a new metadata stream will be created. +// +// FilesAdd should include files that are being modified or added. FilesDel +// is the filenames of existing files that will be deleted. If a filename is +// provided in FilesDel that does not correspond to an actual record file, it +// will be ignored. +type RecordEdit struct { + Challenge string `json:"challenge"` // Random challenge + Token string `json:"token"` // Censorship token + MDAppend []MetadataStream `json:"mdappend"` + MDOverwrite []MetadataStream `json:"mdoverwrite"` + FilesAdd []File `json:"filesadd"` + FilesDel []string `json:"filesdel"` +} + +// RecordEditReply is the reply to the RecordEdit command. +type RecordEditReply struct { + Response string `json:"response"` // Challenge response + Record Record `json:"record"` +} + +// RecordEditMetadata edits the metadata of a record. +// +// MDAppend appends metadata to a metadata stream. MDOverwrite overwrites a +// metadata stream. If the metadata stream does not exist yet for either of +// these arguments, a new metadata stream will be created. +type RecordEditMetadata struct { + Challenge string `json:"challenge"` // Random challenge + Token string `json:"token"` // Censorship token + MDAppend []MetadataStream `json:"mdappend"` + MDOverwrite []MetadataStream `json:"mdoverwrite"` +} + +// RecordEditMetadataReply is the reply to the RecordEditMetadata command. +type RecordEditMetadataReply struct { + Response string `json:"response"` // Challenge response + Record Record `json:"record"` +} + +// RecordSetStatus sets the status of a record. +// +// MDAppend appends metadata to a metadata stream. MDOverwrite overwrites a +// metadata stream. If the metadata stream does not exist yet for either of +// these arguments, a new metadata stream will be created. +type RecordSetStatus struct { + Challenge string `json:"challenge"` // Random challenge + Token string `json:"token"` // Censorship token + Status RecordStatusT `json:"status"` + MDAppend []MetadataStream `json:"mdappend"` + MDOverwrite []MetadataStream `json:"mdoverwrite"` +} + +// RecordSetStatusReply is the reply to the RecordSetStatus command. +type RecordSetStatusReply struct { + Response string `json:"response"` // Challenge response + Record Record `json:"record"` +} + +// RecordGet retrieves a record. If no version is provided the most recent +// version will be returned. +type RecordGet struct { + Challenge string `json:"challenge"` // Random challenge + Token string `json:"token"` // Censorship token + Version uint32 `json:"version,omitempty"` // Record version +} + +// RecordGetReply is the reply to the RecordGet command. +type RecordGetReply struct { + Response string `json:"response"` // Challenge response + Record Record `json:"record"` +} + +// RecordRequest is used to request a record. It gives the caller granular +// control over what is returned. The only required field is the token. All +// other fields are optional. All record files are returned by default unless +// one of the file arguments is provided. +// +// Version is used to request a specific version of a record. If no version is +// provided then the most recent version of the record will be returned. +// +// Filenames can be used to request specific files. If filenames is not empty +// then the specified files will be the only files that are returned. +// +// OmitAllFiles can be used to retrieve a record without any of the record +// files. This supersedes the filenames argument. +type RecordRequest struct { + Token string `json:"token"` + Version uint32 `json:"version"` + Filenames []string `json:"filenames"` + OmitAllFiles bool `json:"omitallfiles"` +} + +// RecordGetBatch retrieves a record. If no version is provided the most recent +// version will be returned. +type RecordGetBatch struct { + Challenge string `json:"challenge"` // Random challenge + Requests []RecordRequest `json:"requests"` +} + +// RecordGetBatchReply is the reply to the RecordGetBatch command. If a record +// was not found or an error occurred while retrieving it the token will not be +// included in the returned map. +type RecordGetBatchReply struct { + Response string `json:"response"` // Challenge response + Records map[string]Record `json:"records"` +} + +// Proof contains an inclusion proof for the digest in the merkle root. All +// digests are hex encoded SHA256 digests. +// +// The ExtraData field is used by certain types of proofs to include additional +// data that is required to validate the proof. +type Proof struct { + Type string `json:"type"` + Digest string `json:"digest"` + MerkleRoot string `json:"merkleroot"` + MerklePath []string `json:"merklepath"` + ExtraData string `json:"extradata"` // JSON encoded +} + +// Timestamp contains all of the data required to verify that a piece of record +// content was timestamped onto the decred blockchain. +// +// All digests are hex encoded SHA256 digests. The merkle root can be found in +// the OP_RETURN of the specified DCR transaction. +// +// TxID, MerkleRoot, and Proofs will only be populated once the merkle root has +// been included in a DCR tx and the tx has 6 confirmations. The Data field +// will not be populated if the data has been censored. +type Timestamp struct { + Data string `json:"data"` // JSON encoded + Digest string `json:"digest"` + TxID string `json:"txid"` + MerkleRoot string `json:"merkleroot"` + Proofs []Proof `json:"proofs"` +} + +// RecordTimestamps contains the timestamps for a specific version of a record. +type RecordTimestamps struct { + RecordMetadata Timestamp `json:"recordmetadata"` + + // map[pluginID]map[streamID]Timestamp + Metadata map[string]map[uint32]Timestamp `json:"metadata"` + + // map[filename]Timestamp + Files map[string]Timestamp `json:"files"` +} + +type RecordGetTimestamps struct { + Challenge string `json:"challenge"` // Random challenge + Token string `json:"token"` // Censorship token + Version uint32 `json:"version"` // Record version +} + +// RecordTimestampsReply is the reply ot the RecordTimestamps command. +type RecordTimestampsReply struct { + Response string `json:"response"` // Challenge response + Timestamps RecordTimestamps `json:"timestamps"` +} + +const ( + // InventoryPageSize is the maximum number of tokens that will be + // returned for any single status in an InventoryReply. + InventoryPageSize uint32 = 20 +) + +// Inventory requests the tokens of the records in the inventory, categorized +// by record state and record status. The tokens are ordered by the timestamp +// of their most recent status change, sorted from newest to oldest. +// +// The state, status, and page arguments can be provided to request a specific +// page of record tokens. +// +// If no status is provided then a page of tokens for all statuses will be +// returned. All other arguments will be ignored. +type Inventory struct { + Challenge string `json:"challenge"` // Random challenge + State RecordStateT `json:"state,omitempty"` + Status RecordStatusT `json:"status,omitempty"` + Page uint32 `json:"page,omitempty"` +} + +// InventoryReply is the reply to the Inventory command. The map keys are the +// human readable record statuses defined by the RecordStatuses array. +type InventoryReply struct { + Response string `json:"response"` // Challenge response + Unvetted map[string][]string `json:"unvetted"` + Vetted map[string][]string `json:"vetted"` +} + +// PluginCmd represents plugin command and the command payload. A token is +// required for all plugin writes, but is optional for reads. +type PluginCmd struct { + Token string `json:"token,omitempty"` // Censorship token + ID string `json:"id"` // Plugin identifier + Command string `json:"command"` // Plugin command + Payload string `json:"payload"` // Command payload +} + +// PluginWrite executes a plugin command that writes data. +type PluginWrite struct { + Challenge string `json:"challenge"` // Random challenge + Cmd PluginCmd `json:"cmd"` +} + +// PluginWriteReply is the reply to the PluginWrite command. +type PluginWriteReply struct { + Response string `json:"response"` // Challenge response + Payload string `json:"payload"` // Response payload +} + +// PluginReads executes a batch of read only plugin commands. +type PluginReads struct { + Challenge string `json:"challenge"` // Random challenge + Cmds []PluginCmd `json:"cmds"` +} + +// PluginReadReply is the reply to an individual read only plugin command that +// is part of a batch of plugin commands. +type PluginReadReply struct { + Token string `json:"token"` // Censorship token + ID string `json:"id"` // Plugin identifier + Command string `json:"command"` // Plugin command + Payload string `json:"payload"` // Response payload + + // UserError will be populated if a ErrorCodeT is encountered + // before the plugin command could be executed. + UserError *UserErrorReply `json:"usererror,omitempty"` + + // PluginError will be populated if a plugin error occurred during + // plugin command execution. + PluginError *PluginErrorReply `json:"pluginerror,omitempty"` +} + +// PluginReadsReply is the reply to the PluginReads command. +type PluginReadsReply struct { + Response string `json:"response"` // Challenge response + Replies []PluginReadReply `json:"replies"` +} + +// PluginSetting is a structure that holds key/value pairs of a plugin setting. +type PluginSetting struct { + Key string `json:"key"` + Value string `json:"value"` +} + +// Plugin describes a plugin and its settings. +type Plugin struct { + ID string `json:"id"` + Settings []PluginSetting `json:"settings"` +} + +// PluginInventory retrieves all active plugins and their settings. +type PluginInventory struct { + Challenge string `json:"challenge"` // Random challenge +} + +// PluginInventoryReply returns all plugins and their settings. +type PluginInventoryReply struct { + Response string `json:"response"` // Challenge response + Plugins []Plugin `json:"plugins"` +} diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index cd2e363a7..e9bf22afb 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -50,11 +50,6 @@ var ( // used. ErrPluginCmdInvalid = errors.New("plugin command invalid") - // ErrPluginActionInvalid is emitted when an invalid plugin action - // is used. See PluginActionRead and PluginActionWrite for valid - // plugin actions. - ErrPluginActionInvalid = errors.New("plugin action invalid") - // Plugin names must be all lowercase letters and have a length of <20 PluginRE = regexp.MustCompile(`^[a-z]{1,20}$`) ) @@ -132,9 +127,8 @@ type RecordMetadata struct { // MetadataStream describes a single metada stream. type MetadataStream struct { - PluginID string `json:"pluginid,omitempty"` // Plugin identity - ID uint64 `json:"id"` // Stream identity - Payload string `json:"payload"` // String encoded metadata + ID uint64 `json:"id"` // Stream identity + Payload string `json:"payload"` // String encoded metadata } // Record is a permanent Record that includes the submitted files, metadata and @@ -146,79 +140,6 @@ type Record struct { Files []File // User provided files } -// RecordRequest is used to requests a record. It gives the client granular -// control over what is returned. The only required field is the token. All -// other fields are optional. -// -// Version is used to request a specific version of a record. If no version is -// provided then the most recent version of the record will be returned. -// -// Filenames can be used to request specific files. If filenames is not empty -// then the specified files will be the only files returned. -// -// OmitAllFiles can be used to retrieve a record without any of the record -// files. This supersedes the filenames argument. -type RecordRequest struct { - Token []byte - Version string - Filenames []string - OmitAllFiles bool -} - -// Proof contains an inclusion proof for the digest in the merkle root. All -// digests are hex encoded SHA256 digests. -// -// The ExtraData field is used by certain types of proofs to include additional -// data that is required to validate the proof. -type Proof struct { - Type string - Digest string - MerkleRoot string - MerklePath []string - ExtraData string // JSON encoded -} - -// Timestamp contains all of the data required to verify that a piece of record -// content was timestamped onto the decred blockchain. -// -// All digests are hex encoded SHA256 digests. The merkle root can be found in -// the OP_RETURN of the specified DCR transaction. -// -// TxID, MerkleRoot, and Proofs will only be populated once the merkle root has -// been included in a DCR tx and the tx has 6 confirmations. The Data field -// will not be populated if the data has been censored. -type Timestamp struct { - Data string // JSON encoded - Digest string - TxID string - MerkleRoot string - Proofs []Proof -} - -// RecordTimestamps contains a Timestamp for all record data. -type RecordTimestamps struct { - Token string // Censorship token - Version string // Version of files - RecordMetadata Timestamp - Metadata map[string]Timestamp // [metadataID]Timestamp - Files map[string]Timestamp // [filename]Timestamp -} - -const ( - // PluginActionRead is provided to the backend methods that execute - // plugin commands to indicate that the plugin command is a read - // only command. - PluginActionRead = "read" - - // PluginActionWrite is provided to the backend methods that execute - // plugin commands to indicate that the plugin command writes data - // to the backend. This allows the backend to prevent concurrent - // writes to the record so that individual plugin implementations - // do not need to worry about implementing logic to prevent race - // conditions. - PluginActionWrite = "write" -) - // PluginSettings are used to specify settings for a plugin at runtime. type PluginSetting struct { Key string // Name of setting @@ -237,37 +158,6 @@ type Plugin struct { Identity *identity.FullIdentity } -// PluginError represents a plugin error that is caused by the user. -type PluginError struct { - PluginID string - ErrorCode int - ErrorContext string -} - -// Error satisfies the error interface. -func (e PluginError) Error() string { - return fmt.Sprintf("plugin id '%v' error code %v", - e.PluginID, e.ErrorCode) -} - -const ( - // StateUnvetted is used to request the inventory of an unvetted - // status. - StateUnvetted = "unvetted" - - // StateVetted is used to request the inventory of a vetted status. - StateVetted = "vetted" -) - -// InventoryByStatus contains the tokens of the records in the inventory -// categorized by record state and record status. Each list contains a page of -// tokens that are sorted by the timestamp of the status change from newest to -// oldest. -type InventoryByStatus struct { - Unvetted map[MDStatusT][]string - Vetted map[MDStatusT][]string -} - // Backend provides an API for creating and editing records. When a record is // first submitted it is considered to be an unvetted, i.e. non-public, record. // Once the status of the record is updated to a public status, the record is @@ -312,63 +202,13 @@ type Backend interface { // Get vetted record GetVetted(token []byte, version string) (*Record, error) - // Get a batch of unvetted records - GetUnvettedBatch(reqs []RecordRequest) (map[string]Record, error) - - // Get a batch of vetted records - GetVettedBatch(reqs []RecordRequest) (map[string]Record, error) - - // Get unvetted record timestamps - GetUnvettedTimestamps(token []byte, - version string) (*RecordTimestamps, error) - - // Get vetted record timestamps - GetVettedTimestamps(token []byte, - version string) (*RecordTimestamps, error) - - // Get record tokens categorized by MDStatusT - InventoryByStatus(state string, s MDStatusT, - pageSize, page uint32) (*InventoryByStatus, error) - - // Register an unvetted plugin with the backend - RegisterUnvettedPlugin(Plugin) error - - // Register a vetted plugin with the backend - RegisterVettedPlugin(Plugin) error - - // Perform any plugin setup that is required - SetupUnvettedPlugin(pluginID string) error - - // Perform any plugin setup that is required - SetupVettedPlugin(pluginID string) error - - // Execute a unvetted plugin command - UnvettedPluginCmd(action string, token []byte, pluginID, - cmd, payload string) (string, error) - - // Execute a vetted plugin command - VettedPluginCmd(action string, token []byte, pluginID, - cmd, payload string) (string, error) - - // Get unvetted plugins - GetUnvettedPlugins() []Plugin - - // Get vetted plugins - GetVettedPlugins() []Plugin - // Inventory retrieves various record records - // - // This method has been DEPRECATED. Inventory(uint, uint, uint, bool, bool) ([]Record, []Record, error) // Plugin pass-through command - // - // This method has been DEPRECATED. - Plugin(pluginID, cmd, cmdID, payload string) (string, error) + Plugin(string, string) (string, string, error) // command type, payload, error // Obtain plugin settings - // - // This method has been DEPRECATED. GetPlugins() ([]Plugin, error) // Close performs cleanup of the backend diff --git a/politeiad/backend/gitbe/gitbe.go b/politeiad/backend/gitbe/gitbe.go index 8f5286358..030644f99 100644 --- a/politeiad/backend/gitbe/gitbe.go +++ b/politeiad/backend/gitbe/gitbe.go @@ -1281,7 +1281,7 @@ func (g *gitBackEnd) populateTokenPrefixCache() error { func (g *gitBackEnd) randomUniqueToken() ([]byte, error) { TRIES := 1000 for i := 0; i < TRIES; i++ { - token, err := util.Random(pd.TokenSizeGit) + token, err := util.Random(pd.TokenSize) if err != nil { return nil, err } @@ -2428,34 +2428,6 @@ func (g *gitBackEnd) GetVetted(token []byte, version string) (*backend.Record, e return g.getRecordLock(token, version, g.vetted, true) } -// GetUnvettedBatch is not implemented. -// -// This function satisfies the Backend interface. -func (g *gitBackEnd) GetUnvettedBatch(reqs []backend.RecordRequest) (map[string]backend.Record, error) { - return nil, fmt.Errorf("not implemented") -} - -// GetVettedBatch is not implemented. -// -// This function satisfies the Backend interface. -func (g *gitBackEnd) GetVettedBatch(reqs []backend.RecordRequest) (map[string]backend.Record, error) { - return nil, fmt.Errorf("not implemented") -} - -// GetUnvettedTimestamps is not implemented. -// -// This function satisfies the Backend interface. -func (g *gitBackEnd) GetUnvettedTimestamps(token []byte, version string) (*backend.RecordTimestamps, error) { - return nil, fmt.Errorf("not implemented") -} - -// GetVettedTimestamps is not implemented. -// -// This function satisfies the Backend interface. -func (g *gitBackEnd) GetVettedTimestamps(token []byte, version string) (*backend.RecordTimestamps, error) { - return nil, fmt.Errorf("not implemented") -} - // getVettedMetadataStream returns a byte slice of the given metadata stream. // // This function must be called with the read lock held. @@ -2823,98 +2795,41 @@ func (g *gitBackEnd) GetPlugins() ([]backend.Plugin, error) { // execute. // // Plugin satisfies the backend interface. -func (g *gitBackEnd) Plugin(pluginID, command, commandID, payload string) (string, error) { +func (g *gitBackEnd) Plugin(command, payload string) (string, string, error) { log.Tracef("Plugin: %v", command) switch command { - // Decred plugin case decredplugin.CmdBestBlock: - return g.pluginBestBlock() + payload, err := g.pluginBestBlock() + return decredplugin.CmdBestBlock, payload, err case decredplugin.CmdNewComment: - return g.pluginNewComment(payload) + payload, err := g.pluginNewComment(payload) + return decredplugin.CmdNewComment, payload, err case decredplugin.CmdCensorComment: - return g.pluginCensorComment(payload) + payload, err := g.pluginCensorComment(payload) + return decredplugin.CmdCensorComment, payload, err case decredplugin.CmdGetComments: - return g.pluginGetComments(payload) - - // CMS plugin + payload, err := g.pluginGetComments(payload) + return decredplugin.CmdGetComments, payload, err case cmsplugin.CmdInventory: - return g.pluginCMSInventory() + payload, err := g.pluginCMSInventory() + return cmsplugin.CmdInventory, payload, err case cmsplugin.CmdStartVote: - return g.pluginStartDCCVote(payload) + payload, err := g.pluginStartDCCVote(payload) + return cmsplugin.CmdStartVote, payload, err case cmsplugin.CmdCastVote: - return g.pluginCastVote(payload) + payload, err := g.pluginCastVote(payload) + return cmsplugin.CmdCastVote, payload, err case cmsplugin.CmdDCCVoteResults: - return g.pluginDCCVoteResults(payload) + payload, err := g.pluginDCCVoteResults(payload) + return cmsplugin.CmdDCCVoteResults, payload, err case cmsplugin.CmdVoteDetails: - return g.pluginDCCVoteDetails(payload) + payload, err := g.pluginDCCVoteDetails(payload) + return cmsplugin.CmdVoteDetails, payload, err case cmsplugin.CmdVoteSummary: - return g.pluginDCCVoteSummary(payload) + payload, err := g.pluginDCCVoteSummary(payload) + return cmsplugin.CmdVoteSummary, payload, err } - - return "", fmt.Errorf("invalid payload command") -} - -// InventoryByStatus has not been not implemented. -// -// This function satisfies the backend.Backend interface. -func (g *gitBackEnd) InventoryByStatus(state string, s backend.MDStatusT, pageSize, page uint32) (*backend.InventoryByStatus, error) { - return nil, fmt.Errorf("not implemented") -} - -// RegisterUnvettedPlugin has not been implemented. -// -// This function satisfies the backend.Backend interface. -func (g *gitBackEnd) RegisterUnvettedPlugin(p backend.Plugin) error { - return fmt.Errorf("not implemented") -} - -// RegisterVettedPlugin has not been implemented. -// -// This function satisfies the backend.Backend interface. -func (g *gitBackEnd) RegisterVettedPlugin(p backend.Plugin) error { - return fmt.Errorf("not implemented") -} - -// SetupUnvettedPlugin has not been implemented. -// -// This function satisfies the backend.Backend interface. -func (g *gitBackEnd) SetupUnvettedPlugin(pluginID string) error { - return fmt.Errorf("not implemented") -} - -// SetupVettedPlugin has not been implemented. -// -// This function satisfies the backend.Backend interface. -func (g *gitBackEnd) SetupVettedPlugin(pluginID string) error { - return fmt.Errorf("not implemented") -} - -// UnvettedPluginCmd has not been implemented. -// -// This function satisfies the backend.Backend interface. -func (g *gitBackEnd) UnvettedPluginCmd(action string, token []byte, pluginID, cmd, payload string) (string, error) { - return "", fmt.Errorf("not implemented") -} - -// VettedPluginCmd has not been implemented. -// -// This function satisfies the backend.Backend interface. -func (g *gitBackEnd) VettedPluginCmd(action string, token []byte, pluginID, cmd, payload string) (string, error) { - return "", fmt.Errorf("not implemented") -} - -// GetUnvettedPlugins has not been implemented. -// -// This function satisfies the backend.Backend interface. -func (g *gitBackEnd) GetUnvettedPlugins() []backend.Plugin { - return []backend.Plugin{} -} - -// GetVettedPlugins has not been implemented. -// -// This function satisfies the backend.Backend interface. -func (g *gitBackEnd) GetVettedPlugins() []backend.Plugin { - return []backend.Plugin{} + return "", "", fmt.Errorf("invalid payload command") // XXX this needs to become a type error } // Close shuts down the backend. It obtains the lock and sets the shutdown diff --git a/politeiad/backend/tstorebe/tstore/testing.go b/politeiad/backend/tstorebe/tstore/testing.go deleted file mode 100644 index 2c46f3c19..000000000 --- a/politeiad/backend/tstorebe/tstore/testing.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (c) 2020-2021 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package tstore - -import ( - "io/ioutil" - "testing" - - "github.com/decred/politeia/politeiad/backend/tstorebe/store/localdb" - "github.com/marcopeereboom/sbox" -) - -func newTestTstore(t *testing.T, tstoreID, dataDir string, encrypt bool) *Tstore { - t.Helper() - - // Setup datadir for this tstore instance - var err error - dataDir, err = ioutil.TempDir(dataDir, tstoreID) - if err != nil { - t.Fatal(err) - } - - // Setup key-value store - fp, err := ioutil.TempDir(dataDir, defaultStoreDirname) - if err != nil { - t.Fatal(err) - } - store, err := localdb.New(fp) - if err != nil { - t.Fatal(err) - } - - // Setup encryptin key if specified - var ek *encryptionKey - if encrypt { - key, err := sbox.NewKey() - if err != nil { - t.Fatal(err) - } - ek = newEncryptionKey(key) - } - - return &Tstore{ - id: tstoreID, - encryptionKey: ek, - trillian: newTestTClient(t), - store: store, - } -} - -// NewTestTstoreEncrypted returns a tstore instance that encrypts all data blobs -// and that has been setup for testing. -func NewTestTstoreEncrypted(t *testing.T, tstoreID, dataDir string) *Tstore { - return newTestTstore(t, tstoreID, dataDir, true) -} - -// NewTestTstoreUnencrypted returns a tstore instance that save all data blobs -// as plaintext and that has been setup for testing. -func NewTestTstoreUnencrypted(t *testing.T, tstoreID, dataDir string) *Tstore { - return newTestTstore(t, tstoreID, dataDir, false) -} diff --git a/politeiad/backend/tstorebe/tstorebe.go b/politeiad/backend/tstorebe/tstorebe.go deleted file mode 100644 index b52d27ca8..000000000 --- a/politeiad/backend/tstorebe/tstorebe.go +++ /dev/null @@ -1,2016 +0,0 @@ -// Copyright (c) 2020-2021 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package tstorebe - -import ( - "bytes" - "encoding/base64" - "encoding/binary" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "net/url" - "path/filepath" - "strconv" - "sync" - "time" - - "github.com/decred/dcrd/chaincfg/v3" - v1 "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/politeiad/api/v1/mime" - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tstorebe/plugins" - "github.com/decred/politeia/politeiad/backend/tstorebe/tstore" - "github.com/decred/politeia/util" - "github.com/marcopeereboom/sbox" - "github.com/subosito/gozaru" -) - -const ( - defaultEncryptionKeyFilename = "tstore-sbox.key" - - // Tstore instance IDs - tstoreIDUnvetted = "unvetted" - tstoreIDVetted = "vetted" -) - -var ( - _ backend.Backend = (*tstoreBackend)(nil) -) - -// tstoreBackend implements the Backend interface. -type tstoreBackend struct { - sync.RWMutex - homeDir string - dataDir string - shutdown bool - unvetted *tstore.Tstore - vetted *tstore.Tstore - - // prefixes contains the prefix to full token mapping for unvetted - // records. The prefix is the first n characters of the hex encoded - // record token, where n is defined by the TokenPrefixLength from - // the politeiad API. Record lookups by token prefix are allowed. - // This cache is used to prevent prefix collisions when creating - // new tokens and to facilitate lookups by token prefix. This cache - // is built on startup. - prefixes map[string][]byte // [tokenPrefix]token - - // vettedTreeIDs contains the token to tree ID mapping for vetted - // records. The token corresponds to the unvetted tree ID so - // unvetted lookups can be done directly, but vetted lookups - // required pulling the vetted tree pointer from the unvetted tree. - // This cache memoizes those results and is lazy loaded. - vettedTreeIDs map[string]int64 // [token]treeID - - // recordMtxs allows the backend to hold a lock on an individual - // record so that it can perform multiple read/write operations - // in a concurrent safe manner. These mutexes are lazy loaded. - recordMtxs map[string]*sync.Mutex -} - -func tokenFromTreeID(treeID int64) []byte { - b := make([]byte, 8) - // Converting between int64 and uint64 doesn't change - // the sign bit, only the way it's interpreted. - binary.LittleEndian.PutUint64(b, uint64(treeID)) - return b -} - -func tokenIsFullLength(token []byte) bool { - return util.TokenIsFullLength(util.TokenTypeTstore, token) -} - -func treeIDFromToken(token []byte) int64 { - if !tokenIsFullLength(token) { - return 0 - } - return int64(binary.LittleEndian.Uint64(token)) -} - -// recordMutex returns the mutex for a record. -func (t *tstoreBackend) recordMutex(token []byte) *sync.Mutex { - t.Lock() - defer t.Unlock() - - ts := hex.EncodeToString(token) - m, ok := t.recordMtxs[ts] - if !ok { - // recordMtxs is lazy loaded - m = &sync.Mutex{} - t.recordMtxs[ts] = m - } - - return m -} - -// fullLengthToken returns the full length token given the token prefix. -// -// This function must be called WITHOUT the lock held. -func (t *tstoreBackend) fullLengthToken(prefix []byte) ([]byte, bool) { - t.Lock() - defer t.Unlock() - - token, ok := t.prefixes[util.TokenPrefix(prefix)] - return token, ok -} - -// unvettedTreeIDFromToken returns the unvetted tree ID for the provided token. -// This can be either the full length token or the token prefix. -// -// This function must be called WITHOUT the lock held. -func (t *tstoreBackend) unvettedTreeIDFromToken(token []byte) int64 { - if len(token) == util.TokenPrefixSize() { - // This is a token prefix. Get the full token from the cache. - var ok bool - token, ok = t.fullLengthToken(token) - if !ok { - return 0 - } - } - return treeIDFromToken(token) -} - -func (t *tstoreBackend) prefixExists(fullToken []byte) bool { - t.RLock() - defer t.RUnlock() - - _, ok := t.prefixes[util.TokenPrefix(fullToken)] - return ok -} - -func (t *tstoreBackend) prefixAdd(fullToken []byte) { - t.Lock() - defer t.Unlock() - - prefix := util.TokenPrefix(fullToken) - t.prefixes[prefix] = fullToken - - log.Debugf("Add token prefix: %v", prefix) -} - -func (t *tstoreBackend) vettedTreeID(token []byte) (int64, bool) { - t.RLock() - defer t.RUnlock() - - treeID, ok := t.vettedTreeIDs[hex.EncodeToString(token)] - return treeID, ok -} - -func (t *tstoreBackend) vettedTreeIDAdd(token string, treeID int64) { - t.Lock() - defer t.Unlock() - - t.vettedTreeIDs[token] = treeID - - log.Debugf("Add vetted tree ID: %v %v", token, treeID) -} - -// vettedTreeIDFromToken returns the vetted tree ID that corresponds to the -// provided token. If a tree ID is not found then the returned bool will be -// false. -// -// This function must be called WITHOUT the lock held. -func (t *tstoreBackend) vettedTreeIDFromToken(token []byte) (int64, bool) { - // Check if the token is in the vetted cache. The vetted cache is - // lazy loaded if the token is not present then we need to check - // manually. - vettedTreeID, ok := t.vettedTreeID(token) - if ok { - return vettedTreeID, true - } - - // The token is derived from the unvetted tree ID. Check if the - // unvetted record has a tree pointer. The tree pointer will be - // the vetted tree ID. - unvettedTreeID := t.unvettedTreeIDFromToken(token) - vettedTreeID, ok = t.unvetted.TreePointer(unvettedTreeID) - if !ok { - // No tree pointer. This record either doesn't exist or is an - // unvetted record. - return 0, false - } - - // Verify the vetted tree exists. This should not fail. - if !t.vetted.TreeExists(vettedTreeID) { - // We're in trouble! - e := fmt.Sprintf("freeze record of unvetted tree %v points to "+ - "an invalid vetted tree %v", unvettedTreeID, vettedTreeID) - panic(e) - } - - // Update the vetted cache - t.vettedTreeIDAdd(hex.EncodeToString(token), vettedTreeID) - - return vettedTreeID, true -} - -// verifyContent verifies that all provided MetadataStream and File are sane. -func verifyContent(metadata []backend.MetadataStream, files []backend.File, filesDel []string) error { - // Make sure all metadata is within maxima. - for _, v := range metadata { - if v.ID > v1.MetadataStreamsMax-1 { - return backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidMDID, - ErrorContext: []string{ - strconv.FormatUint(v.ID, 10), - }, - } - } - } - for i := range metadata { - for j := range metadata { - // Skip self and non duplicates. - if i == j || metadata[i].ID != metadata[j].ID { - continue - } - return backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusDuplicateMDID, - ErrorContext: []string{ - strconv.FormatUint(metadata[i].ID, 10), - }, - } - } - } - - // Prevent paths - for i := range files { - if filepath.Base(files[i].Name) != files[i].Name { - return backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidFilename, - ErrorContext: []string{ - files[i].Name, - }, - } - } - } - for _, v := range filesDel { - if filepath.Base(v) != v { - return backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidFilename, - ErrorContext: []string{ - v, - }, - } - } - } - - // Now check files - if len(files) == 0 { - return backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusEmpty, - } - } - - // Prevent bad filenames and duplicate filenames - for i := range files { - for j := range files { - if i == j { - continue - } - if files[i].Name == files[j].Name { - return backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusDuplicateFilename, - ErrorContext: []string{ - files[i].Name, - }, - } - } - } - // Check against filesDel - for _, v := range filesDel { - if files[i].Name == v { - return backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusDuplicateFilename, - ErrorContext: []string{ - files[i].Name, - }, - } - } - } - } - - for i := range files { - if gozaru.Sanitize(files[i].Name) != files[i].Name { - return backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidFilename, - ErrorContext: []string{ - files[i].Name, - }, - } - } - - // Validate digest - d, ok := util.ConvertDigest(files[i].Digest) - if !ok { - return backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidFileDigest, - ErrorContext: []string{ - files[i].Name, - }, - } - } - - // Verify payload is not empty - if files[i].Payload == "" { - return backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidBase64, - ErrorContext: []string{ - files[i].Name, - }, - } - } - - // Decode base64 payload - var err error - payload, err := base64.StdEncoding.DecodeString(files[i].Payload) - if err != nil { - return backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidBase64, - ErrorContext: []string{ - files[i].Name, - }, - } - } - - // Calculate payload digest - dp := util.Digest(payload) - if !bytes.Equal(d[:], dp) { - return backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidFileDigest, - ErrorContext: []string{ - files[i].Name, - }, - } - } - - // Verify MIME - detectedMIMEType := mime.DetectMimeType(payload) - if detectedMIMEType != files[i].MIME { - return backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidMIMEType, - ErrorContext: []string{ - files[i].Name, - detectedMIMEType, - }, - } - } - - if !mime.MimeValid(files[i].MIME) { - return backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusUnsupportedMIMEType, - ErrorContext: []string{ - files[i].Name, - files[i].MIME, - }, - } - } - } - - return nil -} - -var ( - // statusChanges contains the allowed record status changes. If - // statusChanges[currentStatus][newStatus] exists then the status - // change is allowed. - // - // Note, the tstore backend does not make use of the status - // MDStatusIterationUnvetted. The original purpose of this status - // was to show when an unvetted record had been altered since - // unvetted records were not versioned in the git backend. The tstore - // backend versions unvetted records and thus does not need to use - // this additional status. - statusChanges = map[backend.MDStatusT]map[backend.MDStatusT]struct{}{ - // Unvetted status changes - backend.MDStatusUnvetted: { - backend.MDStatusVetted: {}, - backend.MDStatusCensored: {}, - }, - - // Vetted status changes - backend.MDStatusVetted: { - backend.MDStatusCensored: {}, - backend.MDStatusArchived: {}, - }, - - // Statuses that do not allow any further transitions - backend.MDStatusCensored: {}, - backend.MDStatusArchived: {}, - } -) - -// statusChangeIsAllowed returns whether the provided status change is allowed -// by tstorebe. -func statusChangeIsAllowed(from, to backend.MDStatusT) bool { - allowed, ok := statusChanges[from] - if !ok { - return false - } - _, ok = allowed[to] - return ok -} - -func recordMetadataNew(token []byte, files []backend.File, status backend.MDStatusT, iteration uint64) (*backend.RecordMetadata, error) { - digests := make([]string, 0, len(files)) - for _, v := range files { - digests = append(digests, v.Digest) - } - m, err := util.MerkleRoot(digests) - if err != nil { - return nil, err - } - return &backend.RecordMetadata{ - Version: backend.VersionRecordMD, - Iteration: iteration, - Status: status, - Merkle: hex.EncodeToString(m[:]), - Timestamp: time.Now().Unix(), - Token: hex.EncodeToString(token), - }, nil -} - -func filesUpdate(filesCurr, filesAdd []backend.File, filesDel []string) []backend.File { - // Put current files into a map - curr := make(map[string]backend.File, len(filesCurr)) // [filename]File - for _, v := range filesCurr { - curr[v.Name] = v - } - - // Apply deletes - for _, fn := range filesDel { - _, ok := curr[fn] - if ok { - delete(curr, fn) - } - } - - // Apply adds - for _, v := range filesAdd { - curr[v.Name] = v - } - - // Convert back to a slice - f := make([]backend.File, 0, len(curr)) - for _, v := range curr { - f = append(f, v) - } - - return f -} - -func metadataStreamsUpdate(curr, mdAppend, mdOverwrite []backend.MetadataStream) []backend.MetadataStream { - // Put current metadata into a map - md := make(map[uint64]backend.MetadataStream, len(curr)) - for _, v := range curr { - md[v.ID] = v - } - - // Apply overwrites - for _, v := range mdOverwrite { - md[v.ID] = v - } - - // Apply appends. Its ok if an append is specified but there is no - // existing metadata for that metadata stream. In this case the - // append data will become the full metadata stream. - for _, v := range mdAppend { - m, ok := md[v.ID] - if !ok { - // No existing metadata. Use append data as full metadata - // stream. - md[v.ID] = v - continue - } - - // Metadata exists. Append to it. - buf := bytes.NewBuffer([]byte(m.Payload)) - buf.WriteString(v.Payload) - m.Payload = buf.String() - md[v.ID] = m - } - - // Convert metadata back to a slice - metadata := make([]backend.MetadataStream, 0, len(md)) - for _, v := range md { - metadata = append(metadata, v) - } - - return metadata -} - -func (t *tstoreBackend) isShutdown() bool { - t.RLock() - defer t.RUnlock() - - return t.shutdown -} - -// New submites a new record. Records are considered unvetted until their -// status is changed to a public status. -// -// This function satisfies the Backend interface. -func (t *tstoreBackend) New(metadata []backend.MetadataStream, files []backend.File) (*backend.RecordMetadata, error) { - log.Tracef("New") - - // Validate record contents - err := verifyContent(metadata, files, []string{}) - if err != nil { - return nil, err - } - - // Call pre plugin hooks - pre := plugins.HookNewRecordPre{ - Metadata: metadata, - Files: files, - } - b, err := json.Marshal(pre) - if err != nil { - return nil, err - } - err = t.unvetted.PluginHookPre(0, []byte{}, - plugins.HookTypeNewRecordPre, string(b)) - if err != nil { - return nil, err - } - - // Create a new token - var token []byte - var treeID int64 - for retries := 0; retries < 10; retries++ { - treeID, err = t.unvetted.TreeNew() - if err != nil { - return nil, err - } - token = tokenFromTreeID(treeID) - - // Check for token prefix collisions - if !t.prefixExists(token) { - // Not a collision. Use this token. - - // Update the prefix cache. This must be done even if the - // record creation fails since the tree will still exist in - // tstore. - t.prefixAdd(token) - - break - } - - log.Infof("Token prefix collision %v, creating new token", - util.TokenPrefix(token)) - } - - // Create record metadata - rm, err := recordMetadataNew(token, files, backend.MDStatusUnvetted, 1) - if err != nil { - return nil, err - } - - // Save the record - err = t.unvetted.RecordSave(treeID, *rm, metadata, files) - if err != nil { - return nil, fmt.Errorf("RecordSave %x: %v", token, err) - } - - // Call post plugin hooks - post := plugins.HookNewRecordPost{ - Metadata: metadata, - Files: files, - RecordMetadata: rm, - } - b, err = json.Marshal(post) - if err != nil { - return nil, err - } - t.unvetted.PluginHookPost(treeID, token, - plugins.HookTypeNewRecordPost, string(b)) - - // Update the inventory cache - t.inventoryAdd(backend.StateUnvetted, token, backend.MDStatusUnvetted) - - return rm, nil -} - -// This function satisfies the Backend interface. -func (t *tstoreBackend) UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { - log.Tracef("UpdateUnvettedRecord: %x", token) - - // Validate record contents. Send in a single metadata array to - // verify there are no dups. - allMD := append(mdAppend, mdOverwrite...) - err := verifyContent(allMD, filesAdd, filesDel) - if err != nil { - var cve backend.ContentVerificationError - if !errors.As(err, &cve) { - return nil, err - } - // Allow ErrorStatusEmpty which indicates no new files are being - // added. This can happen when files are being deleted without - // any new files being added. - if cve.ErrorCode != v1.ErrorStatusEmpty { - return nil, err - } - } - - // Verify token is valid. The full length token must be used when - // writing data. - if !tokenIsFullLength(token) { - return nil, backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidToken, - } - } - - // Verify record exists and is unvetted - if !t.UnvettedExists(token) { - return nil, backend.ErrRecordNotFound - } - - // Apply the record changes and save the new version. The record - // lock needs to be held for the remainder of the function. - if t.isShutdown() { - return nil, backend.ErrShutdown - } - m := t.recordMutex(token) - m.Lock() - defer m.Unlock() - - // Get existing record - treeID := treeIDFromToken(token) - r, err := t.unvetted.RecordLatest(treeID) - if err != nil { - return nil, fmt.Errorf("RecordLatest: %v", err) - } - - // Apply changes - metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) - files := filesUpdate(r.Files, filesAdd, filesDel) - recordMD, err := recordMetadataNew(token, files, - r.RecordMetadata.Status, r.RecordMetadata.Iteration+1) - if err != nil { - return nil, err - } - - // Call pre plugin hooks - her := plugins.HookEditRecord{ - State: plugins.RecordStateUnvetted, - Current: *r, - RecordMetadata: *recordMD, - Metadata: metadata, - Files: files, - } - b, err := json.Marshal(her) - if err != nil { - return nil, err - } - err = t.unvetted.PluginHookPre(treeID, token, - plugins.HookTypeEditRecordPre, string(b)) - if err != nil { - return nil, err - } - - // Save record - err = t.unvetted.RecordSave(treeID, *recordMD, metadata, files) - if err != nil { - switch err { - case backend.ErrRecordLocked, backend.ErrNoChanges: - return nil, err - default: - return nil, fmt.Errorf("RecordSave: %v", err) - } - } - - // Call post plugin hooks - t.unvetted.PluginHookPost(treeID, token, - plugins.HookTypeEditRecordPost, string(b)) - - // Return updated record - r, err = t.unvetted.RecordLatest(treeID) - if err != nil { - return nil, fmt.Errorf("RecordLatest: %v", err) - } - - return r, nil -} - -// This function satisfies the Backend interface. -func (t *tstoreBackend) UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { - log.Tracef("UpdateVettedRecord: %x", token) - - // Validate record contents. Send in a single metadata array to - // verify there are no dups. - allMD := append(mdAppend, mdOverwrite...) - err := verifyContent(allMD, filesAdd, filesDel) - if err != nil { - var cve backend.ContentVerificationError - if !errors.As(err, &cve) { - return nil, err - } - // Allow ErrorStatusEmpty which indicates no new files are being - // added. This can happen when files are being deleted without - // any new files being added. - if cve.ErrorCode != v1.ErrorStatusEmpty { - return nil, err - } - } - - // Verify token is valid. The full length token must be used when - // writing data. - if !tokenIsFullLength(token) { - return nil, backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidToken, - } - } - - // Get vetted tree ID - treeID, ok := t.vettedTreeIDFromToken(token) - if !ok { - return nil, backend.ErrRecordNotFound - } - - // Apply the record changes and save the new version. The record - // lock needs to be held for the remainder of the function. - if t.isShutdown() { - return nil, backend.ErrShutdown - } - m := t.recordMutex(token) - m.Lock() - defer m.Unlock() - - // Get existing record - r, err := t.vetted.RecordLatest(treeID) - if err != nil { - return nil, fmt.Errorf("RecordLatest: %v", err) - } - - // Apply changes - metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) - files := filesUpdate(r.Files, filesAdd, filesDel) - recordMD, err := recordMetadataNew(token, files, - r.RecordMetadata.Status, r.RecordMetadata.Iteration+1) - if err != nil { - return nil, err - } - - // Call pre plugin hooks - her := plugins.HookEditRecord{ - State: plugins.RecordStateVetted, - Current: *r, - RecordMetadata: *recordMD, - Metadata: metadata, - Files: files, - } - b, err := json.Marshal(her) - if err != nil { - return nil, err - } - err = t.vetted.PluginHookPre(treeID, token, - plugins.HookTypeEditRecordPre, string(b)) - if err != nil { - return nil, err - } - - // Save record - err = t.vetted.RecordSave(treeID, *recordMD, metadata, files) - if err != nil { - switch err { - case backend.ErrRecordLocked, backend.ErrNoChanges: - return nil, err - default: - return nil, fmt.Errorf("RecordSave: %v", err) - } - } - - // Call post plugin hooks - t.vetted.PluginHookPost(treeID, token, - plugins.HookTypeEditRecordPost, string(b)) - - // Return updated record - r, err = t.vetted.RecordLatest(treeID) - if err != nil { - return nil, fmt.Errorf("RecordLatest: %v", err) - } - - return r, nil -} - -// This function satisfies the Backend interface. -func (t *tstoreBackend) UpdateUnvettedMetadata(token []byte, mdAppend, mdOverwrite []backend.MetadataStream) error { - // Validate record contents. Send in a single metadata array to - // verify there are no dups. - allMD := append(mdAppend, mdOverwrite...) - err := verifyContent(allMD, []backend.File{}, []string{}) - if err != nil { - var cve backend.ContentVerificationError - if !errors.As(err, &cve) { - return err - } - // Allow ErrorStatusEmpty which indicates no new files are being - // being added. This is expected since this is a metadata only - // update. - if cve.ErrorCode != v1.ErrorStatusEmpty { - return err - } - } - if len(mdAppend) == 0 && len(mdOverwrite) == 0 { - return backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusNoChanges, - } - } - - // Verify token is valid. The full length token must be used when - // writing data. - if !tokenIsFullLength(token) { - return backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidToken, - } - } - - // Verify record exists and is unvetted - if !t.UnvettedExists(token) { - return backend.ErrRecordNotFound - } - - // Pull the existing record and apply the metadata updates. The - // record lock must be held for the remainder of this function. - if t.isShutdown() { - return backend.ErrShutdown - } - m := t.recordMutex(token) - m.Lock() - defer m.Unlock() - - // Get existing record - treeID := treeIDFromToken(token) - r, err := t.unvetted.RecordLatest(treeID) - if err != nil { - return fmt.Errorf("RecordLatest: %v", err) - } - - // Apply changes - metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) - recordMD, err := recordMetadataNew(token, r.Files, - r.RecordMetadata.Status, r.RecordMetadata.Iteration+1) - if err != nil { - return err - } - - // Call pre plugin hooks - hem := plugins.HookEditMetadata{ - State: plugins.RecordStateUnvetted, - Current: *r, - Metadata: metadata, - } - b, err := json.Marshal(hem) - if err != nil { - return err - } - err = t.unvetted.PluginHookPre(treeID, token, - plugins.HookTypeEditMetadataPre, string(b)) - if err != nil { - return err - } - - // Update metadata - err = t.unvetted.RecordMetadataSave(treeID, *recordMD, metadata) - if err != nil { - switch err { - case backend.ErrRecordLocked, backend.ErrNoChanges: - return err - default: - return fmt.Errorf("RecordMetadataSave: %v", err) - } - } - - // Call post plugin hooks - t.unvetted.PluginHookPost(treeID, token, - plugins.HookTypeEditMetadataPost, string(b)) - - return nil -} - -// This function satisfies the Backend interface. -func (t *tstoreBackend) UpdateVettedMetadata(token []byte, mdAppend, mdOverwrite []backend.MetadataStream) error { - log.Tracef("UpdateVettedMetadata: %x", token) - - // Validate record contents. Send in a single metadata array to - // verify there are no dups. - allMD := append(mdAppend, mdOverwrite...) - err := verifyContent(allMD, []backend.File{}, []string{}) - if err != nil { - var cve backend.ContentVerificationError - if !errors.As(err, &cve) { - return err - } - // Allow ErrorStatusEmpty which indicates no new files are being - // being added. This is expected since this is a metadata only - // update. - if cve.ErrorCode != v1.ErrorStatusEmpty { - return err - } - } - if len(mdAppend) == 0 && len(mdOverwrite) == 0 { - return backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusNoChanges, - } - } - - // Verify token is valid. The full length token must be used when - // writing data. - if !tokenIsFullLength(token) { - return backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidToken, - } - } - - // Get vetted tree ID - treeID, ok := t.vettedTreeIDFromToken(token) - if !ok { - return backend.ErrRecordNotFound - } - - // Pull the existing record and apply the metadata updates. The - // record lock must be held for the remainder of this function. - if t.isShutdown() { - return backend.ErrShutdown - } - m := t.recordMutex(token) - m.Lock() - defer m.Unlock() - - // Get existing record - r, err := t.vetted.RecordLatest(treeID) - if err != nil { - return fmt.Errorf("RecordLatest: %v", err) - } - - // Apply changes - metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) - recordMD, err := recordMetadataNew(token, r.Files, - r.RecordMetadata.Status, r.RecordMetadata.Iteration+1) - if err != nil { - return err - } - - // Call pre plugin hooks - hem := plugins.HookEditMetadata{ - State: plugins.RecordStateVetted, - Current: *r, - Metadata: metadata, - } - b, err := json.Marshal(hem) - if err != nil { - return err - } - err = t.vetted.PluginHookPre(treeID, token, - plugins.HookTypeEditMetadataPre, string(b)) - if err != nil { - return err - } - - // Update metadata - err = t.vetted.RecordMetadataSave(treeID, *recordMD, metadata) - if err != nil { - switch err { - case backend.ErrRecordLocked, backend.ErrNoChanges: - return err - default: - return fmt.Errorf("RecordMetadataSave: %v", err) - } - } - - // Call post plugin hooks - t.vetted.PluginHookPost(treeID, token, - plugins.HookTypeEditMetadataPost, string(b)) - - return nil -} - -// UnvettedExists returns whether the provided token corresponds to an unvetted -// record. -// -// This function satisfies the Backend interface. -func (t *tstoreBackend) UnvettedExists(token []byte) bool { - log.Tracef("UnvettedExists %x", token) - - // Verify token is not in the vetted tree IDs cache. If it is then - // we can be sure that this is not a unvetted record without having - // to send any network requests. - _, ok := t.vettedTreeID(token) - if ok { - return false - } - - // Check for unvetted record - treeID := t.unvettedTreeIDFromToken(token) - return t.unvetted.RecordExists(treeID) -} - -// This function satisfies the Backend interface. -func (t *tstoreBackend) VettedExists(token []byte) bool { - log.Tracef("VettedExists %x", token) - - _, ok := t.vettedTreeIDFromToken(token) - return ok -} - -// This function must be called WITH the record lock held. -func (t *tstoreBackend) unvettedPublish(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) (int64, error) { - // Create a vetted tree - vettedTreeID, err := t.vetted.TreeNew() - if err != nil { - return 0, err - } - - // Save the record to the vetted tstore - err = t.vetted.RecordSave(vettedTreeID, rm, metadata, files) - if err != nil { - return 0, fmt.Errorf("vetted RecordSave: %v", err) - } - - log.Debugf("Unvetted record %x copied to vetted tree %v", - token, vettedTreeID) - - // Freeze the unvetted tree - treeID := treeIDFromToken(token) - err = t.unvetted.TreeFreeze(treeID, rm, metadata, vettedTreeID) - if err != nil { - return 0, fmt.Errorf("TreeFreeze %v: %v", treeID, err) - } - - log.Debugf("Unvetted record frozen %x", token) - - // Update the vetted cache - t.vettedTreeIDAdd(hex.EncodeToString(token), vettedTreeID) - - return vettedTreeID, nil -} - -// This function must be called WITH the record lock held. -func (t *tstoreBackend) unvettedCensor(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { - // Freeze the tree - treeID := treeIDFromToken(token) - err := t.unvetted.TreeFreeze(treeID, rm, metadata, 0) - if err != nil { - return fmt.Errorf("TreeFreeze %v: %v", treeID, err) - } - - log.Debugf("Unvetted record frozen %x", token) - - // Delete all record files - err = t.unvetted.RecordDel(treeID) - if err != nil { - return fmt.Errorf("RecordDel %v: %v", treeID, err) - } - - log.Debugf("Unvetted record files deleted %x", token) - - return nil -} - -func (t *tstoreBackend) SetUnvettedStatus(token []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { - log.Tracef("SetUnvettedStatus: %x %v (%v)", - token, status, backend.MDStatus[status]) - - // Verify token is valid. The full length token must be used when - // writing data. - if !tokenIsFullLength(token) { - return nil, backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidToken, - } - } - - // Verify record exists and is unvetted - if !t.UnvettedExists(token) { - return nil, backend.ErrRecordNotFound - } - - // The existing record must be pulled and updated. The record - // lock must be held for the rest of this function. - if t.isShutdown() { - return nil, backend.ErrShutdown - } - m := t.recordMutex(token) - m.Lock() - defer m.Unlock() - - // Get existing record - treeID := treeIDFromToken(token) - r, err := t.unvetted.RecordLatest(treeID) - if err != nil { - return nil, fmt.Errorf("RecordLatest: %v", err) - } - currStatus := r.RecordMetadata.Status - - // Validate status change - if !statusChangeIsAllowed(currStatus, status) { - return nil, backend.StateTransitionError{ - From: currStatus, - To: status, - } - } - - // Apply status change - recordMD, err := recordMetadataNew(token, r.Files, status, - r.RecordMetadata.Iteration+1) - if err != nil { - return nil, err - } - - // Apply metadata changes - metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) - - // Call pre plugin hooks - hsrs := plugins.HookSetRecordStatus{ - State: plugins.RecordStateUnvetted, - Current: *r, - RecordMetadata: *recordMD, - Metadata: metadata, - } - b, err := json.Marshal(hsrs) - if err != nil { - return nil, err - } - err = t.unvetted.PluginHookPre(treeID, token, - plugins.HookTypeSetRecordStatusPre, string(b)) - if err != nil { - return nil, err - } - - // Update record. When a record is made public it is moved from the - // unvetted instance to the vetted instance. - var vettedTreeID int64 - switch status { - case backend.MDStatusVetted: - vettedTreeID, err = t.unvettedPublish(token, *recordMD, - metadata, r.Files) - if err != nil { - return nil, fmt.Errorf("unvettedPublish: %v", err) - } - case backend.MDStatusCensored: - err := t.unvettedCensor(token, *recordMD, metadata) - if err != nil { - return nil, fmt.Errorf("unvettedCensor: %v", err) - } - default: - return nil, fmt.Errorf("unknown status: %v (%v)", - backend.MDStatus[status], status) - } - - log.Debugf("Status change %x from %v (%v) to %v (%v)", - token, backend.MDStatus[currStatus], currStatus, - backend.MDStatus[status], status) - - switch status { - case backend.MDStatusVetted: - // Record was made public. Call the plugin hook on both unvetted - // and vetted since both instances had state changes. - t.unvetted.PluginHookPost(treeID, token, - plugins.HookTypeSetRecordStatusPost, string(b)) - - t.vetted.PluginHookPost(vettedTreeID, token, - plugins.HookTypeSetRecordStatusPost, string(b)) - - // Update inventory cache - t.inventoryMoveToVetted(token, status) - - // Retrieve record - r, err = t.GetVetted(token, "") - if err != nil { - return nil, err - } - - default: - // All other status changes. The record is still unvetted. - - // Call post plugin hooks - t.unvetted.PluginHookPost(treeID, token, - plugins.HookTypeSetRecordStatusPost, string(b)) - - // Update inventory cache - t.inventoryUpdate(backend.StateUnvetted, token, status) - - // Retrieve record - r, err = t.GetUnvetted(token, "") - if err != nil { - return nil, err - } - } - - return r, nil -} - -// This function must be called WITH the record lock held. -func (t *tstoreBackend) vettedCensor(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { - // Freeze the tree - treeID, ok := t.vettedTreeID(token) - if !ok { - return fmt.Errorf("vetted record not found") - } - err := t.vetted.TreeFreeze(treeID, rm, metadata, 0) - if err != nil { - return fmt.Errorf("TreeFreeze %v: %v", treeID, err) - } - - log.Debugf("Vetted record frozen %x", token) - - // Delete all record files - err = t.vetted.RecordDel(treeID) - if err != nil { - return fmt.Errorf("RecordDel %v: %v", treeID, err) - } - - log.Debugf("Vetted record files deleted %x", token) - - return nil -} - -// This function must be called WITH the record lock held. -func (t *tstoreBackend) vettedArchive(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { - // Freeze the tree. Nothing else needs to be done for an archived - // record. - treeID, ok := t.vettedTreeID(token) - if !ok { - return fmt.Errorf("vetted record not found") - } - err := t.vetted.TreeFreeze(treeID, rm, metadata, 0) - if err != nil { - return fmt.Errorf("TreeFreeze %v: %v", treeID, err) - } - - log.Debugf("Vetted record frozen %x", token) - - return nil -} - -// This function satisfies the Backend interface. -func (t *tstoreBackend) SetVettedStatus(token []byte, status backend.MDStatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { - log.Tracef("SetVettedStatus: %x %v (%v)", - token, status, backend.MDStatus[status]) - - // Verify token is valid. The full length token must be used when - // writing data. - if !tokenIsFullLength(token) { - return nil, backend.ContentVerificationError{ - ErrorCode: v1.ErrorStatusInvalidToken, - } - } - - // Get vetted tree ID - treeID, ok := t.vettedTreeIDFromToken(token) - if !ok { - return nil, backend.ErrRecordNotFound - } - - // The existing record must be pulled and updated. The record lock - // must be held for the rest of this function. - if t.isShutdown() { - return nil, backend.ErrShutdown - } - m := t.recordMutex(token) - m.Lock() - defer m.Unlock() - - // Get existing record - r, err := t.vetted.RecordLatest(treeID) - if err != nil { - return nil, fmt.Errorf("RecordLatest: %v", err) - } - currStatus := r.RecordMetadata.Status - - // Validate status change - if !statusChangeIsAllowed(currStatus, status) { - return nil, backend.StateTransitionError{ - From: currStatus, - To: status, - } - } - - // Apply status change - recordMD, err := recordMetadataNew(token, r.Files, status, - r.RecordMetadata.Iteration+1) - if err != nil { - return nil, err - } - - // Apply metdata changes - metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) - - // Call pre plugin hooks - srs := plugins.HookSetRecordStatus{ - State: plugins.RecordStateVetted, - Current: *r, - RecordMetadata: *recordMD, - Metadata: metadata, - } - b, err := json.Marshal(srs) - if err != nil { - return nil, err - } - err = t.vetted.PluginHookPre(treeID, token, - plugins.HookTypeSetRecordStatusPre, string(b)) - if err != nil { - return nil, err - } - - // Update record - switch status { - case backend.MDStatusCensored: - err := t.vettedCensor(token, *recordMD, metadata) - if err != nil { - return nil, fmt.Errorf("vettedCensor: %v", err) - } - case backend.MDStatusArchived: - err := t.vettedArchive(token, *recordMD, metadata) - if err != nil { - return nil, fmt.Errorf("vettedArchive: %v", err) - } - default: - return nil, fmt.Errorf("unknown status: %v (%v)", - backend.MDStatus[status], status) - } - - // Call post plugin hooks - t.vetted.PluginHookPost(treeID, token, - plugins.HookTypeSetRecordStatusPost, string(b)) - - // Update inventory cache - t.inventoryUpdate(backend.StateVetted, token, status) - - log.Debugf("Status change %x from %v (%v) to %v (%v)", - token, backend.MDStatus[currStatus], currStatus, - backend.MDStatus[status], status) - - // Return the updated record - r, err = t.vetted.RecordLatest(treeID) - if err != nil { - return nil, fmt.Errorf("RecordLatest: %v", err) - } - - return r, nil -} - -// This function satisfies the Backend interface. -func (t *tstoreBackend) GetUnvetted(token []byte, version string) (*backend.Record, error) { - log.Tracef("GetUnvetted: %x %v", token, version) - - if t.isShutdown() { - return nil, backend.ErrShutdown - } - - treeID := t.unvettedTreeIDFromToken(token) - var v uint32 - if version != "" { - u, err := strconv.ParseUint(version, 10, 64) - if err != nil { - return nil, backend.ErrRecordNotFound - } - v = uint32(u) - } - - // Get unvetted record - r, err := t.unvetted.Record(treeID, v) - if err != nil { - if err == backend.ErrRecordNotFound { - return nil, err - } - return nil, fmt.Errorf("unvetted record: %v", err) - } - - return r, nil -} - -// This function satisfies the Backend interface. -func (t *tstoreBackend) GetVetted(token []byte, version string) (*backend.Record, error) { - log.Tracef("GetVetted: %x %v", token, version) - - if t.isShutdown() { - return nil, backend.ErrShutdown - } - - // Get tree ID - treeID, ok := t.vettedTreeIDFromToken(token) - if !ok { - return nil, backend.ErrRecordNotFound - } - - // Parse version - var v uint32 - if version != "" { - u, err := strconv.ParseUint(version, 10, 64) - if err != nil { - return nil, backend.ErrRecordNotFound - } - v = uint32(u) - } - - r, err := t.vetted.Record(treeID, v) - if err != nil { - return nil, err - } - - return r, nil -} - -// GetUnvettedBatch returns a batch of unvetted records. -// -// This function satisfies the Backend interface. -func (t *tstoreBackend) GetUnvettedBatch(reqs []backend.RecordRequest) (map[string]backend.Record, error) { - log.Tracef("GetUnvettedBatch: %v records ", len(reqs)) - - if t.isShutdown() { - return nil, backend.ErrShutdown - } - - records := make(map[string]backend.Record, len(reqs)) - for _, v := range reqs { - // Get tree ID - treeID := t.unvettedTreeIDFromToken(v.Token) - - // Parse version - var version uint32 - if v.Version != "" { - u, err := strconv.ParseUint(v.Version, 10, 64) - if err != nil { - // Not a valid version. Don't include this token in the - // reply. - continue - } - version = uint32(u) - } - - // Get the record - r, err := t.unvetted.RecordPartial(treeID, version, - v.Filenames, v.OmitAllFiles) - if err != nil { - if err == backend.ErrRecordNotFound { - // Record doesn't exist. This is ok. It will not be included - // in the reply. - continue - } - // An unexpected error occurred. Log it and continue. - log.Debug("RecordPartial %v: %v", treeID, err) - continue - } - - // Update reply - records[r.RecordMetadata.Token] = *r - } - - return records, nil -} - -// GetVettedBatch returns a batch of unvetted records. -// -// This function satisfies the Backend interface. -func (t *tstoreBackend) GetVettedBatch(reqs []backend.RecordRequest) (map[string]backend.Record, error) { - log.Tracef("GetVettedBatch: %v records ", len(reqs)) - - if t.isShutdown() { - return nil, backend.ErrShutdown - } - - records := make(map[string]backend.Record, len(reqs)) - for _, v := range reqs { - // Get tree ID - treeID, ok := t.vettedTreeIDFromToken(v.Token) - if !ok { - // Record doesn't exist - continue - } - - // Parse the version - var version uint32 - if v.Version != "" { - u, err := strconv.ParseUint(v.Version, 10, 64) - if err != nil { - // Not a valid version. Don't include this token in the - // reply. - continue - } - version = uint32(u) - } - - // Get the record - r, err := t.vetted.RecordPartial(treeID, version, - v.Filenames, v.OmitAllFiles) - if err != nil { - if err == backend.ErrRecordNotFound { - // Record doesn't exist. This is ok. It will not be included - // in the reply. - continue - } - // An unexpected error occurred. Log it and continue. - log.Debug("RecordPartial %v: %v", treeID, err) - continue - } - - // Update reply - records[r.RecordMetadata.Token] = *r - } - - return records, nil -} - -// This function satisfies the Backend interface. -func (t *tstoreBackend) GetUnvettedTimestamps(token []byte, version string) (*backend.RecordTimestamps, error) { - log.Tracef("GetUnvettedTimestamps: %x %v", token, version) - - if t.isShutdown() { - return nil, backend.ErrShutdown - } - - treeID := t.unvettedTreeIDFromToken(token) - var v uint32 - if version != "" { - u, err := strconv.ParseUint(version, 10, 64) - if err != nil { - return nil, backend.ErrRecordNotFound - } - v = uint32(u) - } - - // Verify record exists and is unvetted - if !t.UnvettedExists(token) { - return nil, backend.ErrRecordNotFound - } - - // Get timestamps - return t.unvetted.RecordTimestamps(treeID, v, token) -} - -// This function satisfies the Backend interface. -func (t *tstoreBackend) GetVettedTimestamps(token []byte, version string) (*backend.RecordTimestamps, error) { - log.Tracef("GetVettedTimestamps: %x %v", token, version) - - if t.isShutdown() { - return nil, backend.ErrShutdown - } - - // Get tree ID - treeID, ok := t.vettedTreeIDFromToken(token) - if !ok { - return nil, backend.ErrRecordNotFound - } - - // Parse version - var v uint32 - if version != "" { - u, err := strconv.ParseUint(version, 10, 64) - if err != nil { - return nil, backend.ErrRecordNotFound - } - v = uint32(u) - } - - // Get timestamps - return t.vetted.RecordTimestamps(treeID, v, token) -} - -// InventoryByStatus returns the record tokens of all records in the inventory -// categorized by MDStatusT. -// -// This function satisfies the Backend interface. -func (t *tstoreBackend) InventoryByStatus(state string, status backend.MDStatusT, pageSize, page uint32) (*backend.InventoryByStatus, error) { - log.Tracef("InventoryByStatus: %v %v %v %v", state, status, pageSize, page) - - inv, err := t.invByStatus(state, status, pageSize, page) - if err != nil { - return nil, err - } - - return &backend.InventoryByStatus{ - Unvetted: inv.Unvetted, - Vetted: inv.Vetted, - }, nil -} - -// RegisterUnvettedPlugin registers a plugin with the unvetted tstore instance. -// -// This function satisfies the Backend interface. -func (t *tstoreBackend) RegisterUnvettedPlugin(p backend.Plugin) error { - log.Tracef("RegisterUnvettedPlugin: %v", p.ID) - - if t.isShutdown() { - return backend.ErrShutdown - } - - return t.unvetted.PluginRegister(t, p) -} - -// RegisterVettedPlugin has not been implemented. -// -// This function satisfies the Backend interface. -func (t *tstoreBackend) RegisterVettedPlugin(p backend.Plugin) error { - log.Tracef("RegisterVettedPlugin: %v", p.ID) - - if t.isShutdown() { - return backend.ErrShutdown - } - - return t.vetted.PluginRegister(t, p) -} - -// SetupUnvettedPlugin performs plugin setup for a previously registered -// unvetted plugin. -// -// This function satisfies the Backend interface. -func (t *tstoreBackend) SetupUnvettedPlugin(pluginID string) error { - log.Tracef("SetupUnvettedPlugin: %v", pluginID) - - if t.isShutdown() { - return backend.ErrShutdown - } - - return t.unvetted.PluginSetup(pluginID) -} - -// SetupVettedPlugin performs plugin setup for a previously registered vetted -// plugin. -// -// This function satisfies the Backend interface. -func (t *tstoreBackend) SetupVettedPlugin(pluginID string) error { - log.Tracef("SetupVettedPlugin: %v", pluginID) - - if t.isShutdown() { - return backend.ErrShutdown - } - - return t.vetted.PluginSetup(pluginID) -} - -func (t *tstoreBackend) unvettedPluginRead(token []byte, pluginID, cmd, payload string) (string, error) { - // The token is optional. If a token is not provided then a tree ID - // will not be provided to the plugin. - var treeID int64 - if len(token) > 0 { - // Get tree ID - treeID = t.unvettedTreeIDFromToken(token) - - // Verify record exists and is unvetted - if !t.UnvettedExists(token) { - return "", backend.ErrRecordNotFound - } - } - - if len(token) > 0 { - log.Infof("Unvetted '%v' plugin read cmd '%v' on %x", - pluginID, cmd, token) - } else { - log.Infof("Unvetted '%v' plugin read cmd '%v'", - pluginID, cmd) - } - - return t.unvetted.PluginCmd(treeID, token, pluginID, cmd, payload) -} - -func (t *tstoreBackend) unvettedPluginWrite(token []byte, pluginID, cmd, payload string) (string, error) { - // Get tree ID - treeID := t.unvettedTreeIDFromToken(token) - - // Verify unvetted record exists - if !t.UnvettedExists(token) { - return "", backend.ErrRecordNotFound - } - - // Hold the record lock for the remainder of this function. We - // do this here in the backend so that the individual plugins - // implementations don't need to worry about race conditions. - m := t.recordMutex(token) - m.Lock() - defer m.Unlock() - - log.Infof("Unvetted '%v' plugin write cmd '%v' on %x", - pluginID, cmd, token) - - // Call pre plugin hooks - hp := plugins.HookPluginPre{ - State: plugins.RecordStateUnvetted, - PluginID: pluginID, - Cmd: cmd, - Payload: payload, - } - b, err := json.Marshal(hp) - if err != nil { - return "", err - } - err = t.unvetted.PluginHookPre(treeID, token, - plugins.HookTypePluginPre, string(b)) - if err != nil { - return "", err - } - - // Execute plugin command - reply, err := t.unvetted.PluginCmd(treeID, token, pluginID, cmd, payload) - if err != nil { - return "", err - } - - // Call post plugin hooks - hpp := plugins.HookPluginPost{ - State: plugins.RecordStateUnvetted, - PluginID: pluginID, - Cmd: cmd, - Payload: payload, - Reply: reply, - } - b, err = json.Marshal(hpp) - if err != nil { - return "", err - } - t.unvetted.PluginHookPost(treeID, token, - plugins.HookTypePluginPost, string(b)) - - return reply, nil -} - -// UnvettedPluginCmd executes a plugin command on an unvetted record. -// -// This function satisfies the Backend interface. -func (t *tstoreBackend) UnvettedPluginCmd(action string, token []byte, pluginID, cmd, payload string) (string, error) { - log.Tracef("UnvettedPluginCmd: %v %x %v %v", action, token, pluginID, cmd) - - if t.isShutdown() { - return "", backend.ErrShutdown - } - - var ( - reply string - err error - ) - switch action { - case backend.PluginActionRead: - reply, err = t.unvettedPluginRead(token, pluginID, cmd, payload) - if err != nil { - return "", err - } - case backend.PluginActionWrite: - reply, err = t.unvettedPluginWrite(token, pluginID, cmd, payload) - if err != nil { - return "", err - } - default: - return "", backend.ErrPluginActionInvalid - } - - return reply, nil -} - -func (t *tstoreBackend) vettedPluginRead(token []byte, pluginID, cmd, payload string) (string, error) { - // The token is optional. If a token is not provided then a tree ID - // will not be provided to the plugin. - var treeID int64 - var ok bool - if len(token) > 0 { - // Get tree ID - treeID, ok = t.vettedTreeIDFromToken(token) - if !ok { - return "", backend.ErrRecordNotFound - } - } - - if len(token) > 0 { - log.Infof("Vetted '%v' plugin read cmd '%v' on %x", - pluginID, cmd, token) - } else { - log.Infof("Vetted '%v' plugin read cmd '%v'", - pluginID, cmd) - } - - return t.vetted.PluginCmd(treeID, token, pluginID, cmd, payload) -} - -func (t *tstoreBackend) vettedPluginWrite(token []byte, pluginID, cmd, payload string) (string, error) { - // Get tree ID - treeID, ok := t.vettedTreeIDFromToken(token) - if !ok { - return "", backend.ErrRecordNotFound - } - - // Hold the record lock for the remainder of this function. We - // do this here in the backend so that the individual plugins - // implementations don't need to worry about race conditions. - m := t.recordMutex(token) - m.Lock() - defer m.Unlock() - - log.Infof("Vetted '%v' plugin write cmd '%v' on %x", - pluginID, cmd, token) - - // Call pre plugin hooks - hp := plugins.HookPluginPre{ - State: plugins.RecordStateVetted, - PluginID: pluginID, - Cmd: cmd, - Payload: payload, - } - b, err := json.Marshal(hp) - if err != nil { - return "", err - } - err = t.vetted.PluginHookPre(treeID, token, - plugins.HookTypePluginPre, string(b)) - if err != nil { - return "", err - } - - // Execute plugin command - reply, err := t.vetted.PluginCmd(treeID, token, pluginID, cmd, payload) - if err != nil { - return "", err - } - - // Call post plugin hooks - hpp := plugins.HookPluginPost{ - State: plugins.RecordStateVetted, - PluginID: pluginID, - Cmd: cmd, - Payload: payload, - Reply: reply, - } - b, err = json.Marshal(hpp) - if err != nil { - return "", err - } - t.vetted.PluginHookPost(treeID, token, - plugins.HookTypePluginPost, string(b)) - - return reply, nil -} - -// VettedPluginCmd executes a plugin command on an unvetted record. -// -// This function satisfies the Backend interface. -func (t *tstoreBackend) VettedPluginCmd(action string, token []byte, pluginID, cmd, payload string) (string, error) { - log.Tracef("VettedPluginCmd: %v %x %v %v", action, token, pluginID, cmd) - - if t.isShutdown() { - return "", backend.ErrShutdown - } - - var ( - reply string - err error - ) - switch action { - case backend.PluginActionRead: - reply, err = t.vettedPluginRead(token, pluginID, cmd, payload) - if err != nil { - return "", err - } - case backend.PluginActionWrite: - reply, err = t.vettedPluginWrite(token, pluginID, cmd, payload) - if err != nil { - return "", err - } - default: - return "", backend.ErrPluginActionInvalid - } - - return reply, nil -} - -// GetUnvettedPlugins returns the unvetted plugins that have been registered. -// -// This function satisfies the Backend interface. -func (t *tstoreBackend) GetUnvettedPlugins() []backend.Plugin { - log.Tracef("GetUnvettedPlugins") - - return t.unvetted.Plugins() -} - -// GetVettedPlugins returns the vetted plugins that have been registered. -// -// This function satisfies the Backend interface. -func (t *tstoreBackend) GetVettedPlugins() []backend.Plugin { - log.Tracef("GetVettedPlugins") - - return t.vetted.Plugins() -} - -// Inventory has been DEPRECATED. -// -// This function satisfies the Backend interface. -func (t *tstoreBackend) Inventory(vettedCount, vettedStart, unvettedCount uint, includeFiles, allVersions bool) ([]backend.Record, []backend.Record, error) { - return nil, nil, fmt.Errorf("not implemented") -} - -// GetPlugins has been DEPRECATED. -// -// This function satisfies the Backend interface. -func (t *tstoreBackend) GetPlugins() ([]backend.Plugin, error) { - return nil, fmt.Errorf("not implemented") -} - -// Plugin has been DEPRECATED. -// -// This function satisfies the Backend interface. -func (t *tstoreBackend) Plugin(pluginID, cmd, cmdID, payload string) (string, error) { - return "", fmt.Errorf("not implemented") -} - -// Close shuts the backend down and performs cleanup. -// -// This function satisfies the Backend interface. -func (t *tstoreBackend) Close() { - log.Tracef("Close") - - t.Lock() - defer t.Unlock() - - // Shutdown backend - t.shutdown = true - - // Close tstore connections - t.unvetted.Close() - t.vetted.Close() -} - -// setup creates the tstore backend caches. -func (t *tstoreBackend) setup() error { - log.Tracef("setup") - - log.Infof("Building backend token prefix cache") - - // A record token is created using the unvetted tree ID so we - // only need to retrieve the unvetted trees in order to build the - // token prefix cache. - treeIDs, err := t.unvetted.TreesAll() - if err != nil { - return fmt.Errorf("unvetted TreesAll: %v", err) - } - - log.Infof("%v records in the backend", len(treeIDs)) - - for _, v := range treeIDs { - token := tokenFromTreeID(v) - t.prefixAdd(token) - } - - return nil -} - -// New returns a new tstoreBackend. -func New(anp *chaincfg.Params, homeDir, dataDir, trillianHostUnvetted, trillianHostVetted, trillianSigningKey, dbType, dbHost, dbPass, dbEncryptionKeyFile, dcrtimeHost, dcrtimeCert string) (*tstoreBackend, error) { - // Setup encryption key file - if dbEncryptionKeyFile == "" { - // No file path was given. Use the default path. - dbEncryptionKeyFile = filepath.Join(homeDir, defaultEncryptionKeyFilename) - } - if !util.FileExists(dbEncryptionKeyFile) { - // Encryption key file does not exist. Create one. - log.Infof("Generating encryption key") - key, err := sbox.NewKey() - if err != nil { - return nil, err - } - err = ioutil.WriteFile(dbEncryptionKeyFile, key[:], 0400) - if err != nil { - return nil, err - } - util.Zero(key[:]) - log.Infof("Encryption key created: %v", dbEncryptionKeyFile) - } - - // Verify dcrtime host - _, err := url.Parse(dcrtimeHost) - if err != nil { - return nil, fmt.Errorf("parse dcrtime host '%v': %v", dcrtimeHost, err) - } - log.Infof("Anchor host: %v", dcrtimeHost) - - // Setup tstore instances - unvetted, err := tstore.New(tstoreIDUnvetted, homeDir, dataDir, anp, - trillianHostUnvetted, trillianSigningKey, dbType, dbHost, dbPass, - dbEncryptionKeyFile, dcrtimeHost, dcrtimeCert) - if err != nil { - return nil, fmt.Errorf("new tstore unvetted: %v", err) - } - vetted, err := tstore.New(tstoreIDVetted, homeDir, dataDir, anp, - trillianHostVetted, trillianSigningKey, dbType, dbHost, dbPass, "", - dcrtimeHost, dcrtimeCert) - if err != nil { - return nil, fmt.Errorf("new tstore vetted: %v", err) - } - - // Setup tstore backend - t := tstoreBackend{ - homeDir: homeDir, - dataDir: dataDir, - unvetted: unvetted, - vetted: vetted, - prefixes: make(map[string][]byte), - vettedTreeIDs: make(map[string]int64), - recordMtxs: make(map[string]*sync.Mutex), - } - - err = t.setup() - if err != nil { - return nil, fmt.Errorf("setup: %v", err) - } - - return &t, nil -} diff --git a/politeiad/backendv2/backendv2.go b/politeiad/backendv2/backendv2.go new file mode 100644 index 000000000..ab2573d33 --- /dev/null +++ b/politeiad/backendv2/backendv2.go @@ -0,0 +1,350 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package backendv2 + +import ( + "errors" + "fmt" + + "github.com/decred/politeia/politeiad/api/v1/identity" +) + +var ( + // ErrShutdown is returned when the backend is shutdown. + ErrShutdown = errors.New("backend is shutdown") + + // ErrTokenInvalid is returned when a token is invalid. + ErrTokenInvalid = errors.New("token is invalid") + + // ErrRecordNotFound is returned when a record is not found. + ErrRecordNotFound = errors.New("record not found") + + // ErrRecordLocked is returned when a record is attempted to be + // updated but the record status does not allow further updates. + ErrRecordLocked = errors.New("record is locked") + + // ErrNoRecordChanges is returned when a record update does not + // contain any changes. + ErrNoRecordChanges = errors.New("no record changes") + + // ErrPluginIDInvalid is returned when a invalid plugin ID is used. + ErrPluginIDInvalid = errors.New("plugin id invalid") + + // ErrPluginCmdInvalid is returned when a invalid plugin command is + // used. + ErrPluginCmdInvalid = errors.New("plugin command invalid") +) + +// StateT represents the state of a record. +type StateT uint32 + +const ( + // StateInvalid is an invalid record state. + StateInvalid StateT = 0 + + // StateUnvetted indicates a record has not been made public. + StateUnvetted StateT = 1 + + // StateVetted indicates a record has been made public. + StateVetted StateT = 2 +) + +var ( + // States contains the human readable record states. + States = map[StateT]string{ + StateInvalid: "invalid", + StateUnvetted: "unvetted", + StateVetted: "vetted", + } +) + +// StatusT represents the status of a record. +type StatusT uint32 + +const ( + // StatusInvalid is an invalid status code. + StatusInvalid StatusT = 0 + + // StatusUnreviewed indicates a record has not been made public + // yet. The state of an unreviewed record will always be unvetted. + StatusUnreviewed StatusT = 1 + + // StatusPublic indicates a record has been made public. The state + // of a public record will always be vetted. + StatusPublic StatusT = 2 + + // StatusCensored indicates a record has been censored. A censored + // record is locked from any further updates and all record content + // is permanently deleted. A censored record can have a state of + // either unvetted or vetted. + StatusCensored StatusT = 3 + + // StatusArchived indicates a record has been archived. An archived + // record is locked from any further updates. An archived record + // have a state of either unvetted or vetted. + StatusArchived StatusT = 4 +) + +var ( + // Statuses contains the human readable record statuses. + Statuses = map[StatusT]string{ + StatusInvalid: "invalid", + StatusUnreviewed: "unreviewed", + StatusPublic: "public", + StatusCensored: "censored", + StatusArchived: "archived", + } +) + +// StatusTransitionError indicates an invalid record status transition. +type StatusTransitionError struct { + From StatusT + To StatusT +} + +// Error satisfies the error interface. +func (s StatusTransitionError) Error() string { + return fmt.Sprintf("invalid record status transition %v (%v) -> %v (%v)", + Statuses[s.From], s.From, Statuses[s.To], s.To) +} + +// RecordMetadata represents metadata that is created by the backend on record +// submission and updates. +// +// The record version is incremented anytime the record files are updated. The +// record iteration is incremented anytime record files, metadata, or the +// record status are updated. +type RecordMetadata struct { + Token string `json:"token"` // Record identifier, hex encoded + Version uint32 `json:"version"` // Record version + Iteration uint32 `json:"iteration"` // Record iteration + State StateT `json:"state"` // Unvetted or vetted + Status StatusT `json:"status"` // Record status + Timestamp int64 `json:"timestamp"` // Last updated + Merkle string `json:"merkle"` // Merkle root of record files +} + +// MetadataStream describes a single metada stream. +type MetadataStream struct { + PluginID string `json:"pluginid"` // Plugin identity + StreamID uint32 `json:"streamid"` // Stream identity + Payload string `json:"payload"` // JSON encoded metadata +} + +// File represents a record file. +type File struct { + Name string `json:"name"` // Basename of the file + MIME string `json:"mime"` // MIME type + Digest string `json:"digest"` // SHA256 of decoded Payload + Payload string `json:"payload"` // Base64 encoded file payload +} + +// Record is a permanent record that includes the submitted files, metadata and +// internal metadata. +type Record struct { + RecordMetadata RecordMetadata `json:"recordmetadata"` + Metadata []MetadataStream `json:"metadata"` + Files []File `json:"files"` +} + +// ContentErrorCodeT represents a record content error. +type ContentErrorCodeT uint32 + +const ( + ContentErrorInvalid ContentErrorCodeT = 0 + ContentErrorMetadataStreamInvalid ContentErrorCodeT = 1 + ContentErrorMetadataStreamDuplicate ContentErrorCodeT = 2 + ContentErrorFilesEmpty ContentErrorCodeT = 3 + ContentErrorFileNameInvalid ContentErrorCodeT = 4 + ContentErrorFileNameDuplicate ContentErrorCodeT = 5 + ContentErrorFileDigestInvalid ContentErrorCodeT = 6 + ContentErrorFilePayloadInvalid ContentErrorCodeT = 7 + ContentErrorFileMIMETypeInvalid ContentErrorCodeT = 8 + ContentErrorFileMIMETypeUnsupported ContentErrorCodeT = 9 +) + +// ContentError is returned when the content of a record does not pass +// validation. +type ContentError struct { + ErrorCode ContentErrorCodeT `json:"errorcode"` + ErrorContext string `json:"errorcontext"` +} + +// Error satisfies the error interface. +func (e ContentError) Error() string { + return fmt.Sprintf("content error code: %v", e.ErrorCode) +} + +// RecordRequest is used to request a record. It gives the caller granular +// control over what is returned. The only required field is the token. All +// other fields are optional. All record files are returned by default unless +// one of the file arguments is provided. +// +// Version is used to request a specific version of a record. If no version is +// provided then the most recent version of the record will be returned. +// +// Filenames can be used to request specific files. If filenames is not empty +// then the specified files will be the only files that are returned. +// +// OmitAllFiles can be used to retrieve a record without any of the record +// files. This supersedes the filenames argument. +type RecordRequest struct { + Token []byte + Version uint32 + Filenames []string + OmitAllFiles bool +} + +// Proof contains an inclusion proof for the digest in the merkle root. All +// digests are hex encoded SHA256 digests. +// +// The ExtraData field is used by certain types of proofs to include additional +// data that is required to validate the proof. +type Proof struct { + Type string + Digest string + MerkleRoot string + MerklePath []string + ExtraData string // JSON encoded +} + +// Timestamp contains all of the data required to verify that a piece of record +// content was timestamped onto the decred blockchain. +// +// All digests are hex encoded SHA256 digests. The merkle root can be found in +// the OP_RETURN of the specified DCR transaction. +// +// TxID, MerkleRoot, and Proofs will only be populated once the merkle root has +// been included in a DCR tx and the tx has 6 confirmations. The Data field +// will not be populated if the data has been censored. +type Timestamp struct { + Data string // JSON encoded + Digest string + TxID string + MerkleRoot string + Proofs []Proof +} + +// RecordTimestamps contains a Timestamp for all record data. +type RecordTimestamps struct { + RecordMetadata Timestamp + + Metadata map[string]map[uint32]Timestamp // [pluginID][streamID]Timestamp + Files map[string]Timestamp // map[filename]Timestamp +} + +// Inventory contains the tokens of records in the inventory categorized by +// record state and record status. Tokens are sorted by the timestamp of the +// status change from newest to oldest. +type Inventory struct { + Unvetted map[StatusT][]string + Vetted map[StatusT][]string +} + +// PluginSetting represents a configurable plugin setting. +type PluginSetting struct { + Key string // Name of setting + Value string // Value of setting +} + +// Plugin describes a plugin and its settings. +type Plugin struct { + ID string + Settings []PluginSetting + + // Identity contains the full identity that the plugin uses to + // create receipts, i.e. signatures of user provided data that + // prove the backend received and processed a plugin command. + Identity *identity.FullIdentity +} + +// PluginError represents an error that occured during plugin execution that +// was caused by the user. +type PluginError struct { + PluginID string + ErrorCode int + ErrorContext string +} + +// Error satisfies the error interface. +func (e PluginError) Error() string { + return fmt.Sprintf("plugin '%v' error code %v", + e.PluginID, e.ErrorCode) +} + +// Backend provides an API for interacting with records in the backend. +type Backend interface { + // RecordNew creates a new record. + RecordNew([]MetadataStream, []File) (*Record, error) + + // RecordEdit edits an existing record. + RecordEdit(token []byte, mdAppend, mdOverwrite []MetadataStream, + filesAdd []File, filesDel []string) (*Record, error) + + // RecordEditMetadata edits the metadata of a record without + // editing any record files. + RecordEditMetadata(token []byte, mdAppend, + mdOverwrite []MetadataStream) (*Record, error) + + // RecordSetStatus sets the status of a record. + RecordSetStatus(token []byte, s StatusT, mdAppend, + mdOverwrite []MetadataStream) (*Record, error) + + // RecordExists returns whether a record exists. + RecordExists(token []byte) bool + + // RecordGet retrieves a record. If no version is provided then the + // most recent version will be returned. + RecordGet(token []byte, version uint32) (*Record, error) + + // RecordGetBatch retreives a batch of records. If a record is not + // found then it is simply not included in the returned map. An + // error is not returned. + RecordGetBatch(reqs []RecordRequest) (map[string]Record, error) + + // RecordTimestamps returns the timestamps for a record. If no + // version is provided then timestamps for the most recent version + // will be returned. + RecordTimestamps(token []byte, version uint32) (*RecordTimestamps, error) + + // Inventory returns the tokens of records in the inventory + // categorized by record state and record status. The tokens are + // ordered by the timestamp of their most recent status change, + // sorted from newest to oldest. + // + // The state, status, and page arguments can be provided to request + // a specific page of record tokens. + // + // If no status is provided then the most recent page of tokens for + // each statuses will be returned. All other arguments are ignored. + Inventory(state StateT, status StatusT, pageSize, + pageNumber uint32) (*Inventory, error) + + // InventoryTimeOrdered returns a page of record tokens sorted by + // timestamp of their most recent status change. The returned + // tokens are not sorted by status and will included all statuses. + InventoryTimeOrdered(state StateT, pageSize, + pageNumber uint32) ([]string, error) + + // PluginRegister registers a plugin. + PluginRegister(Plugin) error + + // PluginSetup performs any required plugin setup. + PluginSetup(pluginID string) error + + // PluginCmdRead executes a read plugin command. + PluginCmdRead(token []byte, pluginID, pluginCmd, + payload string) (string, error) + + // PluginCmdWrite executes a write plugin command. + PluginCmdWrite(token []byte, pluginID, pluginCmd, + payload string) (string, error) + + // PluginInventory returns all registered plugins. + PluginInventory() []Plugin + + // Close performs cleanup of the backend. + Close() +} diff --git a/politeiad/backend/tstorebe/inventory.go b/politeiad/backendv2/tstorebe/inventory.go similarity index 79% rename from politeiad/backend/tstorebe/inventory.go rename to politeiad/backendv2/tstorebe/inventory.go index 0f00c9215..85ddc85ea 100644 --- a/politeiad/backend/tstorebe/inventory.go +++ b/politeiad/backendv2/tstorebe/inventory.go @@ -13,7 +13,7 @@ import ( "os" "path/filepath" - "github.com/decred/politeia/politeiad/backend" + backend "github.com/decred/politeia/politeiad/backendv2" ) const ( @@ -22,8 +22,8 @@ const ( ) type entry struct { - Token string `json:"token"` - Status backend.MDStatusT `json:"status"` + Token string `json:"token"` + Status backend.StatusT `json:"status"` } type inventory struct { @@ -86,7 +86,7 @@ func (t *tstoreBackend) invSaveLocked(filePath string, inv inventory) error { return ioutil.WriteFile(filePath, b, 0664) } -func (t *tstoreBackend) invAdd(state string, token []byte, s backend.MDStatusT) error { +func (t *tstoreBackend) invAdd(state backend.StateT, token []byte, s backend.StatusT) error { t.Lock() defer t.Unlock() @@ -121,12 +121,12 @@ func (t *tstoreBackend) invAdd(state string, token []byte, s backend.MDStatusT) return err } - log.Debugf("Inv %v add %x %v", state, token, backend.MDStatus[s]) + log.Debugf("Inv %v add %x %v", state, token, backend.Statuses[s]) return nil } -func (t *tstoreBackend) invUpdate(state string, token []byte, s backend.MDStatusT) error { +func (t *tstoreBackend) invUpdate(state backend.StateT, token []byte, s backend.StatusT) error { t.Lock() defer t.Unlock() @@ -169,14 +169,14 @@ func (t *tstoreBackend) invUpdate(state string, token []byte, s backend.MDStatus return err } - log.Debugf("Inv %v update %x to %v", state, token, backend.MDStatus[s]) + log.Debugf("Inv %v update %x to %v", state, token, backend.Statuses[s]) return nil } // invMoveToVetted moves a token from the unvetted inventory to the vetted // inventory. -func (t *tstoreBackend) invMoveToVetted(token []byte, s backend.MDStatusT) error { +func (t *tstoreBackend) invMoveToVetted(token []byte, s backend.StatusT) error { t.Lock() defer t.Unlock() @@ -222,14 +222,14 @@ func (t *tstoreBackend) invMoveToVetted(token []byte, s backend.MDStatusT) error } log.Debugf("Inv move %x from unvetted to vetted status %v", - token, backend.MDStatus[s]) + token, backend.Statuses[s]) return nil } // inventoryAdd is a wrapper around the invAdd method that allows us to decide // how disk read/write errors should be handled. For now we just panic. -func (t *tstoreBackend) inventoryAdd(state string, token []byte, s backend.MDStatusT) { +func (t *tstoreBackend) inventoryAdd(state backend.StateT, token []byte, s backend.StatusT) { err := t.invAdd(state, token, s) if err != nil { e := fmt.Sprintf("invAdd %v %x %v: %v", state, token, s, err) @@ -239,7 +239,7 @@ func (t *tstoreBackend) inventoryAdd(state string, token []byte, s backend.MDSta // inventoryUpdate is a wrapper around the invUpdate method that allows us to // decide how disk read/write errors should be handled. For now we just panic. -func (t *tstoreBackend) inventoryUpdate(state string, token []byte, s backend.MDStatusT) { +func (t *tstoreBackend) inventoryUpdate(state backend.StateT, token []byte, s backend.StatusT) { err := t.invUpdate(state, token, s) if err != nil { e := fmt.Sprintf("invUpdate %v %x %v: %v", state, token, s, err) @@ -250,7 +250,8 @@ func (t *tstoreBackend) inventoryUpdate(state string, token []byte, s backend.MD // inventoryMoveToVetted is a wrapper around the invMoveToVetted method that // allows us to decide how disk read/write errors should be handled. For now we // just panic. -func (t *tstoreBackend) inventoryMoveToVetted(token []byte, s backend.MDStatusT) { +// TODO inventoryMoveToVetted should be automatic +func (t *tstoreBackend) inventoryMoveToVetted(token []byte, s backend.StatusT) { err := t.invMoveToVetted(token, s) if err != nil { e := fmt.Sprintf("invMoveToVetted %x %v: %v", token, s, err) @@ -262,8 +263,8 @@ func (t *tstoreBackend) inventoryMoveToVetted(token []byte, s backend.MDStatusT) // status. Each list contains a page of tokens that are sorted by the timestamp // of the status change from newest to oldest. type invByStatus struct { - Unvetted map[backend.MDStatusT][]string - Vetted map[backend.MDStatusT][]string + Unvetted map[backend.StatusT][]string + Vetted map[backend.StatusT][]string } func (t *tstoreBackend) invByStatusAll(pageSize uint32) (*invByStatus, error) { @@ -275,20 +276,20 @@ func (t *tstoreBackend) invByStatusAll(pageSize uint32) (*invByStatus, error) { // Prepare unvetted inventory reply var ( - unvetted = tokensParse(u.Entries, backend.MDStatusUnvetted, pageSize, 1) - censored = tokensParse(u.Entries, backend.MDStatusCensored, pageSize, 1) - archived = tokensParse(u.Entries, backend.MDStatusArchived, pageSize, 1) + unvetted = tokensParse(u.Entries, backend.StatusUnreviewed, pageSize, 1) + censored = tokensParse(u.Entries, backend.StatusCensored, pageSize, 1) + archived = tokensParse(u.Entries, backend.StatusArchived, pageSize, 1) - unvettedInv = make(map[backend.MDStatusT][]string, 16) + unvettedInv = make(map[backend.StatusT][]string, 16) ) if len(unvetted) != 0 { - unvettedInv[backend.MDStatusUnvetted] = unvetted + unvettedInv[backend.StatusUnreviewed] = unvetted } if len(censored) != 0 { - unvettedInv[backend.MDStatusCensored] = censored + unvettedInv[backend.StatusCensored] = censored } if len(archived) != 0 { - unvettedInv[backend.MDStatusArchived] = archived + unvettedInv[backend.StatusArchived] = archived } // Get vetted inventory @@ -299,20 +300,20 @@ func (t *tstoreBackend) invByStatusAll(pageSize uint32) (*invByStatus, error) { // Prepare vetted inventory reply var ( - vetted = tokensParse(v.Entries, backend.MDStatusVetted, pageSize, 1) - vcensored = tokensParse(v.Entries, backend.MDStatusCensored, pageSize, 1) - varchived = tokensParse(v.Entries, backend.MDStatusArchived, pageSize, 1) + vetted = tokensParse(v.Entries, backend.StatusPublic, pageSize, 1) + vcensored = tokensParse(v.Entries, backend.StatusCensored, pageSize, 1) + varchived = tokensParse(v.Entries, backend.StatusArchived, pageSize, 1) - vettedInv = make(map[backend.MDStatusT][]string, 16) + vettedInv = make(map[backend.StatusT][]string, 16) ) if len(vetted) != 0 { - vettedInv[backend.MDStatusVetted] = vetted + vettedInv[backend.StatusPublic] = vetted } if len(vcensored) != 0 { - vettedInv[backend.MDStatusCensored] = vcensored + vettedInv[backend.StatusCensored] = vcensored } if len(varchived) != 0 { - vettedInv[backend.MDStatusArchived] = varchived + vettedInv[backend.StatusArchived] = varchived } return &invByStatus{ @@ -321,10 +322,10 @@ func (t *tstoreBackend) invByStatusAll(pageSize uint32) (*invByStatus, error) { }, nil } -func (t *tstoreBackend) invByStatus(state string, s backend.MDStatusT, pageSize, page uint32) (*invByStatus, error) { +func (t *tstoreBackend) invByStatus(state backend.StateT, s backend.StatusT, pageSize, page uint32) (*invByStatus, error) { // If no status is provided a page of tokens for each status should // be returned. - if s == backend.MDStatusInvalid { + if s == backend.StatusInvalid { return t.invByStatusAll(pageSize) } @@ -354,15 +355,15 @@ func (t *tstoreBackend) invByStatus(state string, s backend.MDStatusT, pageSize, switch state { case backend.StateUnvetted: ibs = invByStatus{ - Unvetted: map[backend.MDStatusT][]string{ + Unvetted: map[backend.StatusT][]string{ s: tokens, }, - Vetted: map[backend.MDStatusT][]string{}, + Vetted: map[backend.StatusT][]string{}, } case backend.StateVetted: ibs = invByStatus{ - Unvetted: map[backend.MDStatusT][]string{}, - Vetted: map[backend.MDStatusT][]string{ + Unvetted: map[backend.StatusT][]string{}, + Vetted: map[backend.StatusT][]string{ s: tokens, }, } @@ -397,7 +398,7 @@ func entryDel(entries []entry, token []byte) ([]entry, error) { } // tokensParse parses a page of tokens from the provided entries. -func tokensParse(entries []entry, s backend.MDStatusT, countPerPage, page uint32) []string { +func tokensParse(entries []entry, s backend.StatusT, countPerPage, page uint32) []string { tokens := make([]string, 0, countPerPage) if countPerPage == 0 || page == 0 { return tokens diff --git a/politeiad/backend/tstorebe/log.go b/politeiad/backendv2/tstorebe/log.go similarity index 100% rename from politeiad/backend/tstorebe/log.go rename to politeiad/backendv2/tstorebe/log.go diff --git a/politeiad/backend/tstorebe/plugins/comments/cmds.go b/politeiad/backendv2/tstorebe/plugins/comments/cmds.go similarity index 99% rename from politeiad/backend/tstorebe/plugins/comments/cmds.go rename to politeiad/backendv2/tstorebe/plugins/comments/cmds.go index 1466713ce..0cb838dd7 100644 --- a/politeiad/backend/tstorebe/plugins/comments/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/comments/cmds.go @@ -15,8 +15,8 @@ import ( "strconv" "time" - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tstorebe/store" + backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" "github.com/decred/politeia/politeiad/plugins/comments" "github.com/decred/politeia/util" ) diff --git a/politeiad/backend/tstorebe/plugins/comments/comments.go b/politeiad/backendv2/tstorebe/plugins/comments/comments.go similarity index 97% rename from politeiad/backend/tstorebe/plugins/comments/comments.go rename to politeiad/backendv2/tstorebe/plugins/comments/comments.go index ca08a62ef..b5f530222 100644 --- a/politeiad/backend/tstorebe/plugins/comments/comments.go +++ b/politeiad/backendv2/tstorebe/plugins/comments/comments.go @@ -12,8 +12,8 @@ import ( "sync" "github.com/decred/politeia/politeiad/api/v1/identity" - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tstorebe/plugins" + backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins" "github.com/decred/politeia/politeiad/plugins/comments" ) diff --git a/politeiad/backend/tstorebe/plugins/comments/log.go b/politeiad/backendv2/tstorebe/plugins/comments/log.go similarity index 100% rename from politeiad/backend/tstorebe/plugins/comments/log.go rename to politeiad/backendv2/tstorebe/plugins/comments/log.go diff --git a/politeiad/backend/tstorebe/plugins/comments/recordindex.go b/politeiad/backendv2/tstorebe/plugins/comments/recordindex.go similarity index 100% rename from politeiad/backend/tstorebe/plugins/comments/recordindex.go rename to politeiad/backendv2/tstorebe/plugins/comments/recordindex.go diff --git a/politeiad/backend/tstorebe/plugins/dcrdata/dcrdata.go b/politeiad/backendv2/tstorebe/plugins/dcrdata/dcrdata.go similarity index 99% rename from politeiad/backend/tstorebe/plugins/dcrdata/dcrdata.go rename to politeiad/backendv2/tstorebe/plugins/dcrdata/dcrdata.go index 992fcdd76..0d0a8807d 100644 --- a/politeiad/backend/tstorebe/plugins/dcrdata/dcrdata.go +++ b/politeiad/backendv2/tstorebe/plugins/dcrdata/dcrdata.go @@ -19,8 +19,8 @@ import ( v5 "github.com/decred/dcrdata/api/types/v5" exptypes "github.com/decred/dcrdata/explorer/types/v2" pstypes "github.com/decred/dcrdata/pubsub/types/v3" - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tstorebe/plugins" + backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins" "github.com/decred/politeia/politeiad/plugins/dcrdata" "github.com/decred/politeia/util" "github.com/decred/politeia/wsdcrdata" diff --git a/politeiad/backend/tstorebe/plugins/dcrdata/log.go b/politeiad/backendv2/tstorebe/plugins/dcrdata/log.go similarity index 100% rename from politeiad/backend/tstorebe/plugins/dcrdata/log.go rename to politeiad/backendv2/tstorebe/plugins/dcrdata/log.go diff --git a/politeiad/backend/tstorebe/plugins/pi/hooks.go b/politeiad/backendv2/tstorebe/plugins/pi/hooks.go similarity index 91% rename from politeiad/backend/tstorebe/plugins/pi/hooks.go rename to politeiad/backendv2/tstorebe/plugins/pi/hooks.go index 771bed5f6..6006e7359 100644 --- a/politeiad/backend/tstorebe/plugins/pi/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/pi/hooks.go @@ -9,9 +9,8 @@ import ( "encoding/json" "fmt" - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tstorebe/plugins" - "github.com/decred/politeia/politeiad/plugins/comments" + backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins" "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/ticketvote" "github.com/decred/politeia/util" @@ -197,6 +196,7 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { } // Verify vote status allows proposal edits + /* TODO implement hook if er.RecordMetadata.Status == backend.MDStatusVetted { t, err := tokenDecode(er.RecordMetadata.Token) if err != nil { @@ -216,13 +216,14 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { } } } + */ return nil } func (p *piPlugin) voteSummary(token []byte) (*ticketvote.SummaryReply, error) { - reply, err := p.backend.VettedPluginCmd(backend.PluginActionRead, - token, ticketvote.PluginID, ticketvote.CmdSummary, "") + reply, err := p.backend.PluginCmdRead(token, ticketvote.PluginID, + ticketvote.CmdSummary, "") if err != nil { return nil, err } @@ -236,10 +237,10 @@ func (p *piPlugin) voteSummary(token []byte) (*ticketvote.SummaryReply, error) { // commentWritesVerify verifies that a record's vote status allows writes from // the comments plugin. -func (p *piPlugin) commentWritesVerify(s plugins.RecordStateT, token []byte) error { +func (p *piPlugin) commentWritesVerify(s backend.StateT, token []byte) error { // Verify that the vote status allows comment writes. This only // applies to vetted records. - if s == plugins.RecordStateUnvetted { + if s == backend.StateUnvetted { return nil } vs, err := p.voteSummary(token) @@ -260,15 +261,15 @@ func (p *piPlugin) commentWritesVerify(s plugins.RecordStateT, token []byte) err } } -func (p *piPlugin) hookCommentNew(s plugins.RecordStateT, token []byte) error { +func (p *piPlugin) hookCommentNew(s backend.StateT, token []byte) error { return p.commentWritesVerify(s, token) } -func (p *piPlugin) hookCommentDel(s plugins.RecordStateT, token []byte) error { +func (p *piPlugin) hookCommentDel(s backend.StateT, token []byte) error { return p.commentWritesVerify(s, token) } -func (p *piPlugin) hookCommentVote(s plugins.RecordStateT, token []byte) error { +func (p *piPlugin) hookCommentVote(s backend.StateT, token []byte) error { return p.commentWritesVerify(s, token) } @@ -280,6 +281,7 @@ func (p *piPlugin) hookPluginPre(treeID int64, token []byte, payload string) err return err } + /* TODO // Call plugin hook switch hpp.PluginID { case comments.PluginID: @@ -292,6 +294,7 @@ func (p *piPlugin) hookPluginPre(treeID int64, token []byte, payload string) err return p.hookCommentVote(hpp.State, token) } } + */ return nil } diff --git a/politeiad/backend/tstorebe/plugins/pi/hooks_test.go b/politeiad/backendv2/tstorebe/plugins/pi/hooks_test.go similarity index 100% rename from politeiad/backend/tstorebe/plugins/pi/hooks_test.go rename to politeiad/backendv2/tstorebe/plugins/pi/hooks_test.go diff --git a/politeiad/backend/tstorebe/plugins/pi/log.go b/politeiad/backendv2/tstorebe/plugins/pi/log.go similarity index 100% rename from politeiad/backend/tstorebe/plugins/pi/log.go rename to politeiad/backendv2/tstorebe/plugins/pi/log.go diff --git a/politeiad/backend/tstorebe/plugins/pi/pi.go b/politeiad/backendv2/tstorebe/plugins/pi/pi.go similarity index 98% rename from politeiad/backend/tstorebe/plugins/pi/pi.go rename to politeiad/backendv2/tstorebe/plugins/pi/pi.go index 5766fba38..98d4b7446 100644 --- a/politeiad/backend/tstorebe/plugins/pi/pi.go +++ b/politeiad/backendv2/tstorebe/plugins/pi/pi.go @@ -12,8 +12,8 @@ import ( "regexp" "strconv" - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tstorebe/plugins" + backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins" "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/util" ) diff --git a/politeiad/backend/tstorebe/plugins/pi/testing.go b/politeiad/backendv2/tstorebe/plugins/pi/testing.go similarity index 100% rename from politeiad/backend/tstorebe/plugins/pi/testing.go rename to politeiad/backendv2/tstorebe/plugins/pi/testing.go diff --git a/politeiad/backend/tstorebe/plugins/plugins.go b/politeiad/backendv2/tstorebe/plugins/plugins.go similarity index 79% rename from politeiad/backend/tstorebe/plugins/plugins.go rename to politeiad/backendv2/tstorebe/plugins/plugins.go index ffcde0962..d29e7f06b 100644 --- a/politeiad/backend/tstorebe/plugins/plugins.go +++ b/politeiad/backendv2/tstorebe/plugins/plugins.go @@ -5,8 +5,8 @@ package plugins import ( - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tstorebe/store" + backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" ) // HookT represents a plugin hook. @@ -71,44 +71,22 @@ var ( } ) -// RecordStateT represents a record state. -type RecordStateT int - -const ( - // RecordStateInvalid is an invalid record state. - RecordStateInvalid RecordStateT = 0 - - // RecordStateUnvetted represents an unvetted record. - RecordStateUnvetted RecordStateT = 1 - - // RecordStateVetted represents a vetted record. - RecordStateVetted RecordStateT = 2 -) - -// HookNewRecordPre is the payload for the pre new record hook. The record -// state is not included since all new records will have a record state of -// unvetted. +// HookNewRecordPre is the payload for the pre new record hook. type HookNewRecordPre struct { Metadata []backend.MetadataStream `json:"metadata"` Files []backend.File `json:"files"` } -// HookNewRecordPost is the payload for the post new record hook. The record -// state is not included since all new records will have a record state of -// unvetted. RecordMetadata is only be present on the post new record hook -// since the record metadata requires the creation of a trillian tree and the -// pre new record hook should execute before any politeiad state is changed in -// case of validation errors. +// HookNewRecordPost is the payload for the post new record hook. type HookNewRecordPost struct { Metadata []backend.MetadataStream `json:"metadata"` Files []backend.File `json:"files"` - RecordMetadata *backend.RecordMetadata `json:"recordmetadata"` + RecordMetadata backend.RecordMetadata `json:"recordmetadata"` } // HookEditRecord is the payload for the pre and post edit record hooks. type HookEditRecord struct { - State RecordStateT `json:"state"` - Current backend.Record `json:"record"` // Current record + Current backend.Record `json:"record"` // Record pre update // Updated fields RecordMetadata backend.RecordMetadata `json:"recordmetadata"` @@ -118,8 +96,7 @@ type HookEditRecord struct { // HookEditMetadata is the payload for the pre and post edit metadata hooks. type HookEditMetadata struct { - State RecordStateT `json:"state"` - Current backend.Record `json:"record"` // Current record + Current backend.Record `json:"record"` // Record pre update // Updated fields Metadata []backend.MetadataStream `json:"metadata"` @@ -128,8 +105,7 @@ type HookEditMetadata struct { // HookSetRecordStatus is the payload for the pre and post set record status // hooks. type HookSetRecordStatus struct { - State RecordStateT `json:"state"` - Current backend.Record `json:"record"` // Current record + Current backend.Record `json:"record"` // Record pre update // Updated fields RecordMetadata backend.RecordMetadata `json:"recordmetadata"` @@ -138,21 +114,19 @@ type HookSetRecordStatus struct { // HookPluginPre is the payload for the pre plugin hook. type HookPluginPre struct { - State RecordStateT `json:"state"` - Token string `json:"token"` - PluginID string `json:"pluginid"` - Cmd string `json:"cmd"` - Payload string `json:"payload"` + Token string `json:"token"` + PluginID string `json:"pluginid"` + Cmd string `json:"cmd"` + Payload string `json:"payload"` } // HookPluginPost is the payload for the post plugin hook. The post plugin hook // includes the plugin reply. type HookPluginPost struct { - State RecordStateT `json:"state"` - PluginID string `json:"pluginid"` - Cmd string `json:"cmd"` - Payload string `json:"payload"` - Reply string `json:"reply"` + PluginID string `json:"pluginid"` + Cmd string `json:"cmd"` + Payload string `json:"payload"` + Reply string `json:"reply"` } // PluginClient provides an API for a tstore instance to use when interacting diff --git a/politeiad/backend/tstorebe/plugins/ticketvote/activevotes.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/activevotes.go similarity index 100% rename from politeiad/backend/tstorebe/plugins/ticketvote/activevotes.go rename to politeiad/backendv2/tstorebe/plugins/ticketvote/activevotes.go diff --git a/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go similarity index 96% rename from politeiad/backend/tstorebe/plugins/ticketvote/cmds.go rename to politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go index 05a6812c4..0730e04d9 100644 --- a/politeiad/backend/tstorebe/plugins/ticketvote/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go @@ -22,8 +22,8 @@ import ( "github.com/decred/dcrd/dcrec/secp256k1/v3/ecdsa" "github.com/decred/dcrd/dcrutil/v3" "github.com/decred/dcrd/wire" - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tstorebe/store" + backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" "github.com/decred/politeia/politeiad/plugins/dcrdata" "github.com/decred/politeia/politeiad/plugins/ticketvote" "github.com/decred/politeia/util" @@ -374,8 +374,8 @@ func (p *ticketVotePlugin) voteDetails(treeID int64) (*ticketvote.VoteDetails, e } func (p *ticketVotePlugin) voteDetailsByToken(token []byte) (*ticketvote.VoteDetails, error) { - reply, err := p.backend.VettedPluginCmd(backend.PluginActionRead, - token, ticketvote.PluginID, ticketvote.CmdDetails, "") + reply, err := p.backend.PluginCmdRead(token, ticketvote.PluginID, + ticketvote.CmdDetails, "") if err != nil { return nil, err } @@ -440,8 +440,8 @@ func (p *ticketVotePlugin) voteOptionResults(token []byte, options []ticketvote. default: // Votes are not in the cache. Pull them from the backend. - reply, err := p.backend.VettedPluginCmd(backend.PluginActionRead, - token, ticketvote.PluginID, ticketvote.CmdResults, "") + reply, err := p.backend.PluginCmdRead(token, ticketvote.PluginID, + ticketvote.CmdResults, "") if err != nil { return nil, err } @@ -480,10 +480,10 @@ func (p *ticketVotePlugin) summariesForRunoff(parentToken string) (map[string]ti if err != nil { return nil, err } - reply, err := p.backend.VettedPluginCmd(backend.PluginActionRead, - parent, ticketvote.PluginID, cmdRunoffDetails, "") + reply, err := p.backend.PluginCmdRead(parent, ticketvote.PluginID, + cmdRunoffDetails, "") if err != nil { - return nil, fmt.Errorf("VettedPluginCmd %x %v %v: %v", + return nil, fmt.Errorf("PluginCmdRead %x %v %v: %v", parent, ticketvote.PluginID, cmdRunoffDetails, err) } var rdr runoffDetailsReply @@ -721,10 +721,10 @@ func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) } func (p *ticketVotePlugin) summaryByToken(token []byte) (*ticketvote.SummaryReply, error) { - reply, err := p.backend.VettedPluginCmd(backend.PluginActionRead, - token, ticketvote.PluginID, ticketvote.CmdSummary, "") + reply, err := p.backend.PluginCmdRead(token, ticketvote.PluginID, + ticketvote.CmdSummary, "") if err != nil { - return nil, fmt.Errorf("VettedPluginCmd %x %v %v: %v", + return nil, fmt.Errorf("PluginCmdRead %x %v %v: %v", token, ticketvote.PluginID, ticketvote.CmdSummary, err) } var sr ticketvote.SummaryReply @@ -761,6 +761,28 @@ func (p *ticketVotePlugin) timestamp(treeID int64, digest []byte) (*ticketvote.T }, nil } +// recordAbridged returns a record where the only record file returned is the +// vote metadata file if one exists. +func (p *ticketVotePlugin) recordAbridged(token []byte) (*backend.Record, error) { + reqs := []backend.RecordRequest{ + { + Token: token, + Filenames: []string{ + ticketvote.FileNameVoteMetadata, + }, + }, + } + rs, err := p.backend.RecordGetBatch(reqs) + if err != nil { + return nil, err + } + r, ok := rs[hex.EncodeToString(token)] + if !ok { + return nil, backend.ErrRecordNotFound + } + return &r, nil +} + // bestBlock fetches the best block from the dcrdata plugin and returns it. If // the dcrdata connection is not active, an error will be returned. func (p *ticketVotePlugin) bestBlock() (uint32, error) { @@ -769,10 +791,10 @@ func (p *ticketVotePlugin) bestBlock() (uint32, error) { if err != nil { return 0, err } - reply, err := p.backend.VettedPluginCmd(backend.PluginActionRead, - []byte{}, dcrdata.PluginID, dcrdata.CmdBestBlock, string(payload)) + reply, err := p.backend.PluginCmdRead(nil, dcrdata.PluginID, + dcrdata.CmdBestBlock, string(payload)) if err != nil { - return 0, fmt.Errorf("Plugin %v %v: %v", + return 0, fmt.Errorf("PluginCmdRead %v %v: %v", dcrdata.PluginID, dcrdata.CmdBestBlock, err) } @@ -805,10 +827,10 @@ func (p *ticketVotePlugin) bestBlockUnsafe() (uint32, error) { if err != nil { return 0, err } - reply, err := p.backend.VettedPluginCmd(backend.PluginActionRead, - []byte{}, dcrdata.PluginID, dcrdata.CmdBestBlock, string(payload)) + reply, err := p.backend.PluginCmdRead(nil, dcrdata.PluginID, + dcrdata.CmdBestBlock, string(payload)) if err != nil { - return 0, fmt.Errorf("Plugin %v %v: %v", + return 0, fmt.Errorf("PluginCmdRead %v %v: %v", dcrdata.PluginID, dcrdata.CmdBestBlock, err) } @@ -842,10 +864,10 @@ func (p *ticketVotePlugin) largestCommitmentAddrs(tickets []string) (map[string] if err != nil { return nil, err } - reply, err := p.backend.VettedPluginCmd(backend.PluginActionRead, - []byte{}, dcrdata.PluginID, dcrdata.CmdTxsTrimmed, string(payload)) + reply, err := p.backend.PluginCmdRead(nil, dcrdata.PluginID, + dcrdata.CmdTxsTrimmed, string(payload)) if err != nil { - return nil, fmt.Errorf("Plugin %v %v: %v", + return nil, fmt.Errorf("PluginCmdRead %v %v: %v", dcrdata.PluginID, dcrdata.CmdTxsTrimmed, err) } var ttr dcrdata.TxsTrimmedReply @@ -911,10 +933,10 @@ func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, if err != nil { return nil, err } - reply, err := p.backend.VettedPluginCmd(backend.PluginActionRead, - []byte{}, dcrdata.PluginID, dcrdata.CmdBlockDetails, string(payload)) + reply, err := p.backend.PluginCmdRead(nil, dcrdata.PluginID, + dcrdata.CmdBlockDetails, string(payload)) if err != nil { - return nil, fmt.Errorf("Plugin %v %v: %v", + return nil, fmt.Errorf("PluginCmdRead %v %v: %v", dcrdata.PluginID, dcrdata.CmdBlockDetails, err) } var bdr dcrdata.BlockDetailsReply @@ -935,10 +957,10 @@ func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, if err != nil { return nil, err } - reply, err = p.backend.VettedPluginCmd(backend.PluginActionRead, - []byte{}, dcrdata.PluginID, dcrdata.CmdTicketPool, string(payload)) + reply, err = p.backend.PluginCmdRead(nil, dcrdata.PluginID, + dcrdata.CmdTicketPool, string(payload)) if err != nil { - return nil, fmt.Errorf("Plugin %v %v: %v", + return nil, fmt.Errorf("PluginCmdRead %v %v: %v", dcrdata.PluginID, dcrdata.CmdTicketPool, err) } var tpr dcrdata.TicketPoolReply @@ -1021,9 +1043,9 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri if err != nil { return "", fmt.Errorf("RecordPartial: %v", err) } - if version != r.Version { + if a.Version != r.RecordMetadata.Version { e := fmt.Sprintf("version is not latest: got %v, want %v", - a.Version, r.Version) + a.Version, r.RecordMetadata.Version) return "", backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeRecordVersionInvalid), @@ -1335,10 +1357,9 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot if err != nil { return nil, fmt.Errorf("RecordPartial: %v", err) } - version := strconv.FormatUint(uint64(sd.Params.Version), 10) - if r.Version != version { + if sd.Params.Version != r.RecordMetadata.Version { e := fmt.Sprintf("version is not latest: got %v, want %v", - sd.Params.Version, r.Version) + sd.Params.Version, r.RecordMetadata.Version) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeRecordVersionInvalid), @@ -1555,10 +1576,9 @@ func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs sta if err != nil { return fmt.Errorf("RecordPartial: %v", err) } - version := strconv.FormatUint(uint64(sd.Params.Version), 10) - if r.Version != version { + if sd.Params.Version != r.RecordMetadata.Version { e := fmt.Sprintf("version is not latest %v: got %v, want %v", - sd.Params.Token, sd.Params.Version, r.Version) + sd.Params.Token, sd.Params.Version, r.RecordMetadata.Version) return backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeRecordVersionInvalid), @@ -1680,11 +1700,11 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti if err != nil { return nil, err } - r, err := p.backend.GetVetted(token, "") + r, err := p.recordAbridged(token) if err != nil { return nil, err } - if r.RecordMetadata.Status != backend.MDStatusVetted { + if r.RecordMetadata.Status != backend.StatusPublic { // This record is not public and should not be included // in the runoff vote. continue @@ -1897,14 +1917,14 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. if err != nil { return nil, err } - _, err = p.backend.VettedPluginCmd(backend.PluginActionWrite, - token, ticketvote.PluginID, cmdStartRunoffSubmission, string(b)) + _, err = p.backend.PluginCmdWrite(token, ticketvote.PluginID, + cmdStartRunoffSubmission, string(b)) if err != nil { var ue backend.PluginError if errors.As(err, &ue) { return nil, err } - return nil, fmt.Errorf("VettedPluginCmd %x %v %v: %v", + return nil, fmt.Errorf("PluginCmdWrite %x %v %v: %v", token, ticketvote.PluginID, cmdStartRunoffSubmission, err) } } diff --git a/politeiad/backend/tstorebe/plugins/ticketvote/hooks.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go similarity index 92% rename from politeiad/backend/tstorebe/plugins/ticketvote/hooks.go rename to politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go index 9d30c0a35..03b8c7135 100644 --- a/politeiad/backend/tstorebe/plugins/ticketvote/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go @@ -7,12 +7,11 @@ package ticketvote import ( "encoding/base64" "encoding/json" - "errors" "fmt" "time" - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tstorebe/plugins" + backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins" "github.com/decred/politeia/politeiad/plugins/ticketvote" ) @@ -76,9 +75,9 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { ErrorContext: "invalid hex", } } - r, err := p.backend.GetVetted(token, "") + r, err := p.recordAbridged(token) if err != nil { - if errors.Is(err, backend.ErrRecordNotFound) { + if err == backend.ErrRecordNotFound { return backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), @@ -87,10 +86,10 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { } return err } - if r.RecordMetadata.Status != backend.MDStatusVetted { + if r.RecordMetadata.Status != backend.StatusPublic { e := fmt.Sprintf("record status is invalid: got %v, want %v", - backend.MDStatus[r.RecordMetadata.Status], - backend.MDStatus[backend.MDStatusVetted]) + backend.Statuses[r.RecordMetadata.Status], + backend.Statuses[backend.StatusPublic]) return backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), @@ -203,7 +202,7 @@ func (p *ticketVotePlugin) hookEditRecordPre(payload string) error { // The LinkTo field is not allowed to change once the record has // become public. If this is a vetted record, verify that any // previously set LinkTo has not changed. - if er.State == plugins.RecordStateVetted { + if er.Current.RecordMetadata.State == backend.StateVetted { var ( oldLinkTo string newLinkTo string @@ -268,13 +267,13 @@ func (p *ticketVotePlugin) hookSetRecordStatusPre(payload string) error { // record is only updated for public records. This update occurs // in the set status post hook. switch srs.RecordMetadata.Status { - case backend.MDStatusVetted: + case backend.StatusPublic: // Get the parent record token, err := tokenDecode(vm.LinkTo) if err != nil { return err } - r, err := p.backend.GetVetted(token, "") + r, err := p.recordAbridged(token) if err != nil { return err } @@ -321,21 +320,21 @@ func (p *ticketVotePlugin) hookSetRecordStatusPost(treeID int64, payload string) // to update cached data if this is the vetted instance. We can // determine this by checking if the record exists. The unvetted // instance will return false. - if newStatus == backend.MDStatusVetted && !p.tstore.RecordExists(treeID) { + if newStatus == backend.StatusPublic && !p.tstore.RecordExists(treeID) { // This is the unvetted instance. Nothing to do. return nil } // Update the inventory cache switch newStatus { - case backend.MDStatusVetted: + case backend.StatusPublic: // Add to inventory p.inventoryAdd(srs.RecordMetadata.Token, ticketvote.VoteStatusUnauthorized) - case backend.MDStatusCensored, backend.MDStatusArchived: + case backend.StatusCensored, backend.StatusArchived: // These statuses do not allow for a vote. Mark as ineligible. // We only need to do this if the record is vetted. - if oldStatus == backend.MDStatusVetted { + if oldStatus == backend.StatusPublic { p.inventoryUpdate(srs.RecordMetadata.Token, ticketvote.VoteStatusIneligible) } @@ -354,18 +353,18 @@ func (p *ticketVotePlugin) hookSetRecordStatusPost(treeID int64, payload string) childToken = srs.RecordMetadata.Token ) switch newStatus { - case backend.MDStatusVetted: + case backend.StatusPublic: // Record has been made public. Add child token to parent's // submissions list. err := p.submissionsCacheAdd(parentToken, childToken) if err != nil { return fmt.Errorf("submissionsFromCacheAdd: %v", err) } - case backend.MDStatusCensored: + case backend.StatusCensored: // Record has been censored. Delete child token from parent's // submissions list. We only need to do this if the record is // vetted. - if oldStatus == backend.MDStatusVetted { + if oldStatus == backend.StatusPublic { err := p.submissionsCacheDel(parentToken, childToken) if err != nil { return fmt.Errorf("submissionsCacheDel: %v", err) diff --git a/politeiad/backend/tstorebe/plugins/ticketvote/inventory.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/inventory.go similarity index 100% rename from politeiad/backend/tstorebe/plugins/ticketvote/inventory.go rename to politeiad/backendv2/tstorebe/plugins/ticketvote/inventory.go diff --git a/politeiad/backend/tstorebe/plugins/ticketvote/log.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/log.go similarity index 100% rename from politeiad/backend/tstorebe/plugins/ticketvote/log.go rename to politeiad/backendv2/tstorebe/plugins/ticketvote/log.go diff --git a/politeiad/backend/tstorebe/plugins/ticketvote/submissions.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/submissions.go similarity index 100% rename from politeiad/backend/tstorebe/plugins/ticketvote/submissions.go rename to politeiad/backendv2/tstorebe/plugins/ticketvote/submissions.go diff --git a/politeiad/backend/tstorebe/plugins/ticketvote/summary.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/summary.go similarity index 100% rename from politeiad/backend/tstorebe/plugins/ticketvote/summary.go rename to politeiad/backendv2/tstorebe/plugins/ticketvote/summary.go diff --git a/politeiad/backend/tstorebe/plugins/ticketvote/ticketvote.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go similarity index 95% rename from politeiad/backend/tstorebe/plugins/ticketvote/ticketvote.go rename to politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go index eb8fc358f..e5fe31896 100644 --- a/politeiad/backend/tstorebe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go @@ -14,8 +14,8 @@ import ( "github.com/decred/dcrd/chaincfg/v3" "github.com/decred/politeia/politeiad/api/v1/identity" - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tstorebe/plugins" + backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins" "github.com/decred/politeia/politeiad/plugins/dcrdata" "github.com/decred/politeia/politeiad/plugins/ticketvote" ) @@ -65,7 +65,7 @@ func (p *ticketVotePlugin) Setup() error { // Verify plugin dependencies var dcrdataFound bool - for _, v := range p.backend.GetVettedPlugins() { + for _, v := range p.backend.PluginInventory() { if v.ID == dcrdata.PluginID { dcrdataFound = true } @@ -104,10 +104,10 @@ func (p *ticketVotePlugin) Setup() error { return err } - reply, err := p.backend.VettedPluginCmd(backend.PluginActionRead, - token, ticketvote.PluginID, ticketvote.CmdDetails, "") + reply, err := p.backend.PluginCmdRead(token, ticketvote.PluginID, + ticketvote.CmdDetails, "") if err != nil { - return fmt.Errorf("VettedPluginCmd %x %v %v: %v", + return fmt.Errorf("PluginCmdRead %x %v %v: %v", token, ticketvote.PluginID, ticketvote.CmdDetails, err) } var dr ticketvote.DetailsReply @@ -125,10 +125,10 @@ func (p *ticketVotePlugin) Setup() error { p.activeVotesAdd(*dr.Vote) // Get cast votes - reply, err = p.backend.VettedPluginCmd(backend.PluginActionRead, - token, ticketvote.PluginID, ticketvote.CmdResults, "") + reply, err = p.backend.PluginCmdRead(token, ticketvote.PluginID, + ticketvote.CmdResults, "") if err != nil { - return fmt.Errorf("VettedPluginCmd %x %v %v: %v", + return fmt.Errorf("PluginCmdRead %x %v %v: %v", token, ticketvote.PluginID, ticketvote.CmdResults, err) } var rr ticketvote.ResultsReply diff --git a/politeiad/backend/tstorebe/plugins/usermd/cache.go b/politeiad/backendv2/tstorebe/plugins/usermd/cache.go similarity index 100% rename from politeiad/backend/tstorebe/plugins/usermd/cache.go rename to politeiad/backendv2/tstorebe/plugins/usermd/cache.go diff --git a/politeiad/backend/tstorebe/plugins/usermd/cmds.go b/politeiad/backendv2/tstorebe/plugins/usermd/cmds.go similarity index 100% rename from politeiad/backend/tstorebe/plugins/usermd/cmds.go rename to politeiad/backendv2/tstorebe/plugins/usermd/cmds.go diff --git a/politeiad/backend/tstorebe/plugins/usermd/hooks.go b/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go similarity index 96% rename from politeiad/backend/tstorebe/plugins/usermd/hooks.go rename to politeiad/backendv2/tstorebe/plugins/usermd/hooks.go index c325da8c6..7c7f28f82 100644 --- a/politeiad/backend/tstorebe/plugins/usermd/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go @@ -13,8 +13,8 @@ import ( "strconv" "strings" - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tstorebe/plugins" + backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins" "github.com/decred/politeia/politeiad/plugins/usermd" "github.com/decred/politeia/util" "github.com/google/uuid" @@ -44,7 +44,7 @@ func userMetadataDecode(metadata []backend.MetadataStream) (*usermd.UserMetadata var userMD *usermd.UserMetadata for _, v := range metadata { if v.PluginID != usermd.PluginID || - v.ID != usermd.MDStreamIDUserMetadata { + v.StreamID != usermd.StreamIDUserMetadata { // Not the mdstream we're looking for continue } @@ -228,7 +228,7 @@ func statusChangesDecode(metadata []backend.MetadataStream) ([]usermd.StatusChan statuses := make([]usermd.StatusChangeMetadata, 0, 16) for _, v := range metadata { if v.PluginID != usermd.PluginID || - v.ID != usermd.MDStreamIDStatusChanges { + v.StreamID != usermd.StreamIDStatusChanges { // Not the mdstream we're looking for continue } @@ -251,9 +251,9 @@ func statusChangesDecode(metadata []backend.MetadataStream) ([]usermd.StatusChan var ( // statusReasonRequired contains the list of record statuses that // require an accompanying reason to be given in the status change. - statusReasonRequired = map[backend.MDStatusT]struct{}{ - backend.MDStatusCensored: {}, - backend.MDStatusArchived: {}, + statusReasonRequired = map[backend.StatusT]struct{}{ + backend.StatusCensored: {}, + backend.StatusArchived: {}, } ) @@ -350,7 +350,7 @@ func (p *userPlugin) hookSetRecordStatusPost(treeID int64, payload string) error // When a record is made public it is moved from the unvetted to // the vetted tstore instance. The token must be removed from the // unvetted user cache and added to the vetted user cache. - if srs.RecordMetadata.Status == backend.MDStatusVetted { + if srs.RecordMetadata.Status == backend.StatusPublic { // Decode user metadata um, err := userMetadataDecode(srs.Metadata) if err != nil { diff --git a/politeiad/backend/tstorebe/plugins/usermd/log.go b/politeiad/backendv2/tstorebe/plugins/usermd/log.go similarity index 100% rename from politeiad/backend/tstorebe/plugins/usermd/log.go rename to politeiad/backendv2/tstorebe/plugins/usermd/log.go diff --git a/politeiad/backend/tstorebe/plugins/usermd/usermd.go b/politeiad/backendv2/tstorebe/plugins/usermd/usermd.go similarity index 95% rename from politeiad/backend/tstorebe/plugins/usermd/usermd.go rename to politeiad/backendv2/tstorebe/plugins/usermd/usermd.go index d033826fe..eeaaf5207 100644 --- a/politeiad/backend/tstorebe/plugins/usermd/usermd.go +++ b/politeiad/backendv2/tstorebe/plugins/usermd/usermd.go @@ -9,8 +9,8 @@ import ( "path/filepath" "sync" - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tstorebe/plugins" + backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins" "github.com/decred/politeia/politeiad/plugins/usermd" ) diff --git a/politeiad/backend/tstorebe/store/localdb/localdb.go b/politeiad/backendv2/tstorebe/store/localdb/localdb.go similarity index 97% rename from politeiad/backend/tstorebe/store/localdb/localdb.go rename to politeiad/backendv2/tstorebe/store/localdb/localdb.go index e9d5522fc..552da8375 100644 --- a/politeiad/backend/tstorebe/store/localdb/localdb.go +++ b/politeiad/backendv2/tstorebe/store/localdb/localdb.go @@ -9,7 +9,7 @@ import ( "fmt" "sync" - "github.com/decred/politeia/politeiad/backend/tstorebe/store" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" "github.com/google/uuid" "github.com/syndtr/goleveldb/leveldb" ) diff --git a/politeiad/backend/tstorebe/store/localdb/log.go b/politeiad/backendv2/tstorebe/store/localdb/log.go similarity index 100% rename from politeiad/backend/tstorebe/store/localdb/log.go rename to politeiad/backendv2/tstorebe/store/localdb/log.go diff --git a/politeiad/backend/tstorebe/store/mysql/log.go b/politeiad/backendv2/tstorebe/store/mysql/log.go similarity index 100% rename from politeiad/backend/tstorebe/store/mysql/log.go rename to politeiad/backendv2/tstorebe/store/mysql/log.go diff --git a/politeiad/backend/tstorebe/store/mysql/mysql.go b/politeiad/backendv2/tstorebe/store/mysql/mysql.go similarity index 98% rename from politeiad/backend/tstorebe/store/mysql/mysql.go rename to politeiad/backendv2/tstorebe/store/mysql/mysql.go index 4c301a37c..f440665de 100644 --- a/politeiad/backend/tstorebe/store/mysql/mysql.go +++ b/politeiad/backendv2/tstorebe/store/mysql/mysql.go @@ -10,7 +10,7 @@ import ( "fmt" "time" - "github.com/decred/politeia/politeiad/backend/tstorebe/store" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" "github.com/google/uuid" _ "github.com/go-sql-driver/mysql" diff --git a/politeiad/backend/tstorebe/store/store.go b/politeiad/backendv2/tstorebe/store/store.go similarity index 100% rename from politeiad/backend/tstorebe/store/store.go rename to politeiad/backendv2/tstorebe/store/store.go diff --git a/politeiad/backend/tstorebe/testing.go b/politeiad/backendv2/tstorebe/testing.go similarity index 56% rename from politeiad/backend/tstorebe/testing.go rename to politeiad/backendv2/tstorebe/testing.go index 7ae5743d7..0acfe031a 100644 --- a/politeiad/backend/tstorebe/testing.go +++ b/politeiad/backendv2/tstorebe/testing.go @@ -8,9 +8,10 @@ import ( "io/ioutil" "os" "path/filepath" + "sync" "testing" - "github.com/decred/politeia/politeiad/backend/tstorebe/tstore" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/tstore" ) // NewTestTstoreBackend returns a tstoreBackend that is setup for testing and a @@ -19,23 +20,22 @@ func NewTestTstoreBackend(t *testing.T) (*tstoreBackend, func()) { t.Helper() // Setup home dir and data dir - homeDir, err := ioutil.TempDir("", "tstorebackend.test") + appDir, err := ioutil.TempDir("", "tstorebackend.test") if err != nil { t.Fatal(err) } - dataDir := filepath.Join(homeDir, "data") + dataDir := filepath.Join(appDir, "data") tstoreBackend := tstoreBackend{ - homeDir: homeDir, - dataDir: dataDir, - unvetted: tstore.NewTestTstoreUnencrypted(t, dataDir, "unvetted"), - vetted: tstore.NewTestTstoreEncrypted(t, dataDir, "vetted"), - prefixes: make(map[string][]byte), - vettedTreeIDs: make(map[string]int64), + appDir: appDir, + dataDir: dataDir, + tstore: tstore.NewTestTstore(t, dataDir), + prefixes: make(map[string][]byte), + recordMtxs: make(map[string]*sync.Mutex), } return &tstoreBackend, func() { - err = os.RemoveAll(homeDir) + err = os.RemoveAll(appDir) if err != nil { t.Fatal(err) } diff --git a/politeiad/backend/tstorebe/tstore/anchor.go b/politeiad/backendv2/tstorebe/tstore/anchor.go similarity index 96% rename from politeiad/backend/tstorebe/tstore/anchor.go rename to politeiad/backendv2/tstorebe/tstore/anchor.go index 042dc0731..d99ed5cd0 100644 --- a/politeiad/backend/tstorebe/tstore/anchor.go +++ b/politeiad/backendv2/tstorebe/tstore/anchor.go @@ -14,7 +14,7 @@ import ( dcrtime "github.com/decred/dcrtime/api/v2" "github.com/decred/dcrtime/merkle" - "github.com/decred/politeia/politeiad/backend/tstorebe/store" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" "github.com/google/trillian" "github.com/google/trillian/types" "google.golang.org/grpc/codes" @@ -261,8 +261,8 @@ func (t *Tstore) anchorSave(a anchor) error { return fmt.Errorf("append leaves failed: %v", failed) } - log.Debugf("Saved %v anchor for tree %v at height %v", - t.id, a.TreeID, a.LogRoot.TreeSize) + log.Debugf("Saved anchor for tree %v at height %v", + a.TreeID, a.LogRoot.TreeSize) return nil } @@ -294,7 +294,7 @@ func (t *Tstore) anchorWait(anchors []anchor, digests []string) { }() // Wait for anchor to drop - log.Infof("Waiting for %v anchor to drop", t.id) + log.Infof("Waiting for anchor to drop") // Continually check with dcrtime if the anchor has been dropped. // The anchor is not considered dropped until the ChainTimestamp @@ -314,7 +314,7 @@ func (t *Tstore) anchorWait(anchors []anchor, digests []string) { for try := 0; try < retries; try++ { <-ticker.C - log.Debugf("Verify %v anchor attempt %v/%v", t.id, try+1, retries) + log.Debugf("Verify anchor attempt %v/%v", try+1, retries) vbr, err := t.dcrtime.verifyBatch(anchorID, digests) if err != nil { @@ -425,7 +425,7 @@ func (t *Tstore) anchorWait(anchors []anchor, digests []string) { } } - log.Infof("Anchor dropped for %v %v records", len(vbr.Digests), t.id) + log.Infof("Anchor dropped for %v records", len(vbr.Digests)) return } @@ -438,7 +438,7 @@ func (t *Tstore) anchorWait(anchors []anchor, digests []string) { // current height is timestamped onto the decred blockchain using the dcrtime // service. The anchor data is saved to the key-value store. func (t *Tstore) anchorTrees() error { - log.Debugf("Start %v anchor process", t.id) + log.Debugf("Start anchor process") // Ensure we are not reentrant if t.droppingAnchorGet() { @@ -527,16 +527,16 @@ func (t *Tstore) anchorTrees() error { // dcrtime. digests = append(digests, hex.EncodeToString(lr.RootHash)) - log.Debugf("Anchoring %v tree %v at height %v", - t.id, v.TreeId, lr.TreeSize) + log.Debugf("Anchoring tree %v at height %v", + v.TreeId, lr.TreeSize) } if len(anchors) == 0 { - log.Infof("No %v trees to to anchor", t.id) + log.Infof("No trees to to anchor") return nil } // Submit dcrtime anchor request - log.Infof("Anchoring %v %v trees", len(anchors), t.id) + log.Infof("Anchoring %v trees", len(anchors)) tbr, err := t.dcrtime.timestampBatch(anchorID, digests) if err != nil { diff --git a/politeiad/backend/tstorebe/tstore/client.go b/politeiad/backendv2/tstorebe/tstore/client.go similarity index 95% rename from politeiad/backend/tstorebe/tstore/client.go rename to politeiad/backendv2/tstorebe/tstore/client.go index 63349a069..1d4a459da 100644 --- a/politeiad/backend/tstorebe/tstore/client.go +++ b/politeiad/backendv2/tstorebe/tstore/client.go @@ -10,8 +10,8 @@ import ( "encoding/json" "fmt" - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tstorebe/store" + backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" "github.com/google/trillian" "google.golang.org/grpc/codes" ) @@ -23,7 +23,7 @@ import ( // // This function satisfies the plugins TstoreClient interface. func (t *Tstore) BlobSave(treeID int64, be store.BlobEntry) error { - log.Tracef("%v BlobSave: %v %v", t.id, treeID) + log.Tracef("BlobSave: %v", treeID) // Parse the data descriptor b, err := base64.StdEncoding.DecodeString(be.DataHint) @@ -100,7 +100,7 @@ func (t *Tstore) BlobSave(treeID int64, be store.BlobEntry) error { // // This function satisfies the plugins TstoreClient interface. func (t *Tstore) BlobsDel(treeID int64, digests [][]byte) error { - log.Tracef("%v BlobsDel: %v %x", t.id, treeID, digests) + log.Tracef("BlobsDel: %v %x", treeID, digests) // Verify tree exists. We allow blobs to be deleted from both // frozen and non frozen trees. @@ -151,7 +151,7 @@ func (t *Tstore) BlobsDel(treeID int64, digests [][]byte) error { // // This function satisfies the plugins TstoreClient interface. func (t *Tstore) Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEntry, error) { - log.Tracef("%v Blobs: %v %x", t.id, treeID, digests) + log.Tracef("Blobs: %v %x", treeID, digests) if len(digests) == 0 { return map[string]store.BlobEntry{}, nil @@ -239,7 +239,7 @@ func (t *Tstore) Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEnt // // This function satisfies the plugins TstoreClient interface. func (t *Tstore) BlobsByDataDesc(treeID int64, dataDesc string) ([]store.BlobEntry, error) { - log.Tracef("%v BlobsByDataDesc: %v %v", t.id, treeID, dataDesc) + log.Tracef("BlobsByDataDesc: %v %v", treeID, dataDesc) // Verify tree exists if !t.TreeExists(treeID) { @@ -308,7 +308,7 @@ func (t *Tstore) BlobsByDataDesc(treeID int64, dataDesc string) ([]store.BlobEnt // // This function satisfies the plugins TstoreClient interface. func (t *Tstore) DigestsByDataDesc(treeID int64, dataDesc string) ([][]byte, error) { - log.Tracef("%v DigestsByDataDesc: %v %v", t.id, treeID, dataDesc) + log.Tracef("DigestsByDataDesc: %v %v", treeID, dataDesc) // Verify tree exists if !t.TreeExists(treeID) { @@ -342,7 +342,7 @@ func (t *Tstore) DigestsByDataDesc(treeID int64, dataDesc string) ([][]byte, err // // This function satisfies the plugins TstoreClient interface. func (t *Tstore) Timestamp(treeID int64, digest []byte) (*backend.Timestamp, error) { - log.Tracef("%v Timestamp: %v %x", t.id, treeID, digest) + log.Tracef("Timestamp: %v %x", treeID, digest) // Get tree leaves leaves, err := t.trillian.leavesAll(treeID) diff --git a/politeiad/backend/tstorebe/tstore/convert.go b/politeiad/backendv2/tstorebe/tstore/convert.go similarity index 97% rename from politeiad/backend/tstorebe/tstore/convert.go rename to politeiad/backendv2/tstorebe/tstore/convert.go index 5ed3d6bd2..8b72b397e 100644 --- a/politeiad/backend/tstorebe/tstore/convert.go +++ b/politeiad/backendv2/tstorebe/tstore/convert.go @@ -11,8 +11,8 @@ import ( "encoding/json" "fmt" - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tstorebe/store" + backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" "github.com/decred/politeia/util" ) diff --git a/politeiad/backend/tstorebe/tstore/dcrtime.go b/politeiad/backendv2/tstorebe/tstore/dcrtime.go similarity index 100% rename from politeiad/backend/tstorebe/tstore/dcrtime.go rename to politeiad/backendv2/tstorebe/tstore/dcrtime.go diff --git a/politeiad/backend/tstorebe/tstore/encryptionkey.go b/politeiad/backendv2/tstorebe/tstore/encryptionkey.go similarity index 100% rename from politeiad/backend/tstorebe/tstore/encryptionkey.go rename to politeiad/backendv2/tstorebe/tstore/encryptionkey.go diff --git a/politeiad/backend/tstorebe/tstore/log.go b/politeiad/backendv2/tstorebe/tstore/log.go similarity index 100% rename from politeiad/backend/tstorebe/tstore/log.go rename to politeiad/backendv2/tstorebe/tstore/log.go diff --git a/politeiad/backend/tstorebe/tstore/plugin.go b/politeiad/backendv2/tstorebe/tstore/plugin.go similarity index 77% rename from politeiad/backend/tstorebe/tstore/plugin.go rename to politeiad/backendv2/tstorebe/tstore/plugin.go index 0cd7ceeed..28fe0f158 100644 --- a/politeiad/backend/tstorebe/tstore/plugin.go +++ b/politeiad/backendv2/tstorebe/tstore/plugin.go @@ -10,13 +10,12 @@ import ( "path/filepath" "sort" - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tstorebe/plugins" - "github.com/decred/politeia/politeiad/backend/tstorebe/plugins/comments" - "github.com/decred/politeia/politeiad/backend/tstorebe/plugins/dcrdata" - "github.com/decred/politeia/politeiad/backend/tstorebe/plugins/pi" - "github.com/decred/politeia/politeiad/backend/tstorebe/plugins/ticketvote" - "github.com/decred/politeia/politeiad/backend/tstorebe/plugins/usermd" + backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins/dcrdata" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins/pi" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins/ticketvote" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins/usermd" cmplugin "github.com/decred/politeia/politeiad/plugins/comments" ddplugin "github.com/decred/politeia/politeiad/plugins/dcrdata" piplugin "github.com/decred/politeia/politeiad/plugins/pi" @@ -63,7 +62,7 @@ func (t *Tstore) pluginIDs() []string { } func (t *Tstore) PluginRegister(b backend.Backend, p backend.Plugin) error { - log.Tracef("%v PluginRegister: %v", t.id, p.ID) + log.Tracef("PluginRegister: %v", p.ID) var ( client plugins.PluginClient @@ -73,10 +72,12 @@ func (t *Tstore) PluginRegister(b backend.Backend, p backend.Plugin) error { ) switch p.ID { case cmplugin.PluginID: - client, err = comments.New(t, p.Settings, dataDir, p.Identity) - if err != nil { - return err - } + /* + client, err = comments.New(t, p.Settings, dataDir, p.Identity) + if err != nil { + return err + } + */ case ddplugin.PluginID: client, err = dcrdata.New(p.Settings, t.activeNetParams) if err != nil { @@ -99,7 +100,7 @@ func (t *Tstore) PluginRegister(b backend.Backend, p backend.Plugin) error { return err } default: - return backend.ErrPluginInvalid + return backend.ErrPluginIDInvalid } t.Lock() @@ -114,18 +115,18 @@ func (t *Tstore) PluginRegister(b backend.Backend, p backend.Plugin) error { } func (t *Tstore) PluginSetup(pluginID string) error { - log.Tracef("%v PluginSetup: %v", t.id, pluginID) + log.Tracef("PluginSetup: %v", pluginID) p, ok := t.plugin(pluginID) if !ok { - return backend.ErrPluginInvalid + return backend.ErrPluginIDInvalid } return p.client.Setup() } func (t *Tstore) PluginHookPre(treeID int64, token []byte, h plugins.HookT, payload string) error { - log.Tracef("%v PluginHookPre: %v %v", t.id, treeID, plugins.Hooks[h]) + log.Tracef("PluginHookPre: %v %v", treeID, plugins.Hooks[h]) // Pass hook event and payload to each plugin for _, v := range t.pluginIDs() { @@ -144,13 +145,13 @@ func (t *Tstore) PluginHookPre(treeID int64, token []byte, h plugins.HookT, payl } func (t *Tstore) PluginHookPost(treeID int64, token []byte, h plugins.HookT, payload string) { - log.Tracef("%v PluginHookPost: %v %v", t.id, treeID, plugins.Hooks[h]) + log.Tracef("PluginHookPost: %v %v", treeID, plugins.Hooks[h]) // Pass hook event and payload to each plugin for _, v := range t.pluginIDs() { p, ok := t.plugin(v) if !ok { - log.Errorf("%v PluginHookPost: plugin not found %v", t.id, v) + log.Errorf("%v PluginHookPost: plugin not found %v", v) continue } err := p.client.Hook(treeID, token, h, payload) @@ -159,19 +160,19 @@ func (t *Tstore) PluginHookPost(treeID int64, token []byte, h plugins.HookT, pay // saved to tstore. We do not have the ability to unwind. Log // the error and continue. log.Criticalf("%v PluginHookPost %v %v %v %x %v: %v", - t.id, v, treeID, token, h, err, payload) + v, treeID, token, h, err, payload) continue } } } func (t *Tstore) PluginCmd(treeID int64, token []byte, pluginID, cmd, payload string) (string, error) { - log.Tracef("%v PluginCmd: %v %x %v %v", t.id, treeID, token, pluginID, cmd) + log.Tracef("PluginCmd: %v %x %v %v", treeID, token, pluginID, cmd) // Get plugin p, ok := t.plugin(pluginID) if !ok { - return "", backend.ErrPluginInvalid + return "", backend.ErrPluginIDInvalid } // Execute plugin command @@ -180,7 +181,7 @@ func (t *Tstore) PluginCmd(treeID int64, token []byte, pluginID, cmd, payload st // Plugins returns all registered plugins for the tstore instance. func (t *Tstore) Plugins() []backend.Plugin { - log.Tracef("%v Plugins", t.id) + log.Tracef("Plugins") t.Lock() defer t.Unlock() diff --git a/politeiad/backend/tstorebe/tstore/recordindex.go b/politeiad/backendv2/tstorebe/tstore/recordindex.go similarity index 89% rename from politeiad/backend/tstorebe/tstore/recordindex.go rename to politeiad/backendv2/tstorebe/tstore/recordindex.go index ad32223cc..fe8f67f9a 100644 --- a/politeiad/backend/tstorebe/tstore/recordindex.go +++ b/politeiad/backendv2/tstorebe/tstore/recordindex.go @@ -9,7 +9,7 @@ import ( "fmt" "sort" - "github.com/decred/politeia/politeiad/backend" + backend "github.com/decred/politeia/politeiad/backendv2" "github.com/google/trillian" "google.golang.org/grpc/codes" ) @@ -66,33 +66,6 @@ type recordIndex struct { // tree to frozen in trillian, at which point trillian will not // allow any additional leaves to be appended onto the tree. Frozen bool `json:"frozen,omitempty"` - - // TreePointer is the tree ID of the tree that is the new location - // of this record. A record can be copied to a new tree after - // certain status changes, such as when a record is made public and - // the record is copied from an unvetted tree to a vetted tree. - // TreePointer should only be set if the tree has been frozen. - TreePointer int64 `json:"treepointer,omitempty"` -} - -// treePointerExists returns whether the provided record index has a tree -// pointer set. -func treePointerExists(r recordIndex) bool { - // Sanity checks - switch { - case !r.Frozen && r.TreePointer > 0: - // Tree pointer should only be set if the record is frozen - e := fmt.Sprintf("tree pointer set without record being frozen %v", - r.TreePointer) - panic(e) - case r.TreePointer < 0: - // Tree pointer should never be negative. Trillian uses a int64 - // for the tree ID so we do too. - e := fmt.Sprintf("tree pointer is < 0: %v", r.TreePointer) - panic(e) - } - - return r.TreePointer > 0 } // parseRecordIndex takes a list of record indexes and returns the most recent diff --git a/politeiad/backendv2/tstorebe/tstore/testing.go b/politeiad/backendv2/tstorebe/tstore/testing.go new file mode 100644 index 000000000..d7c6d013e --- /dev/null +++ b/politeiad/backendv2/tstorebe/tstore/testing.go @@ -0,0 +1,46 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tstore + +import ( + "io/ioutil" + "testing" + + "github.com/decred/politeia/politeiad/backendv2/tstorebe/store/localdb" + "github.com/marcopeereboom/sbox" +) + +func NewTestTstore(t *testing.T, dataDir string) *Tstore { + t.Helper() + + // Setup datadir for this tstore instance + dataDir, err := ioutil.TempDir(dataDir, "tstore.test") + if err != nil { + t.Fatal(err) + } + + // Setup key-value store + fp, err := ioutil.TempDir(dataDir, defaultStoreDirname) + if err != nil { + t.Fatal(err) + } + store, err := localdb.New(fp) + if err != nil { + t.Fatal(err) + } + + // Setup encryptin key if specified + key, err := sbox.NewKey() + if err != nil { + t.Fatal(err) + } + ek := newEncryptionKey(key) + + return &Tstore{ + encryptionKey: ek, + trillian: newTestTClient(t), + store: store, + } +} diff --git a/politeiad/backend/tstorebe/tstore/trillianclient.go b/politeiad/backendv2/tstorebe/tstore/trillianclient.go similarity index 99% rename from politeiad/backend/tstorebe/tstore/trillianclient.go rename to politeiad/backendv2/tstorebe/tstore/trillianclient.go index 8b639d934..dc7c25fef 100644 --- a/politeiad/backend/tstorebe/tstore/trillianclient.go +++ b/politeiad/backendv2/tstorebe/tstore/trillianclient.go @@ -36,7 +36,7 @@ import ( ) const ( - waitForInclusionTimeout = 60 * time.Second + waitForInclusionTimeout = 120 * time.Second ) // queuedLeafProof contains the results of a leaf append command, i.e. the diff --git a/politeiad/backend/tstorebe/tstore/tstore.go b/politeiad/backendv2/tstorebe/tstore/tstore.go similarity index 80% rename from politeiad/backend/tstorebe/tstore/tstore.go rename to politeiad/backendv2/tstorebe/tstore/tstore.go index 2b5ff8866..b099836e9 100644 --- a/politeiad/backend/tstorebe/tstore/tstore.go +++ b/politeiad/backendv2/tstorebe/tstore/tstore.go @@ -11,19 +11,22 @@ import ( "encoding/json" "errors" "fmt" + "io/ioutil" + "net/url" "os" "path/filepath" "strconv" "sync" "github.com/decred/dcrd/chaincfg/v3" - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tstorebe/plugins" - "github.com/decred/politeia/politeiad/backend/tstorebe/store" - "github.com/decred/politeia/politeiad/backend/tstorebe/store/localdb" - "github.com/decred/politeia/politeiad/backend/tstorebe/store/mysql" + backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/store/localdb" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/store/mysql" "github.com/decred/politeia/util" "github.com/google/trillian" + "github.com/marcopeereboom/sbox" "github.com/robfig/cron" "google.golang.org/grpc/codes" ) @@ -33,15 +36,16 @@ const ( DBTypeMySQL = "mysql" dbUser = "politeiad" + defaultEncryptionKeyFilename = "tstore-sbox.key" defaultTrillianSigningKeyFilename = "trillian.key" defaultStoreDirname = "store" // Blob entry data descriptors - dataDescriptorFile = "file-v1" - dataDescriptorRecordMetadata = "recordmd-v1" - dataDescriptorMetadataStream = "mdstream-v1" - dataDescriptorRecordIndex = "rindex-v1" - dataDescriptorAnchor = "anchor-v1" + dataDescriptorFile = "pd-file-v1" + dataDescriptorRecordMetadata = "pd-recordmd-v1" + dataDescriptorMetadataStream = "pd-mdstream-v1" + dataDescriptorRecordIndex = "pd-rindex-v1" + dataDescriptorAnchor = "pd-anchor-v1" ) var ( @@ -51,7 +55,6 @@ var ( // We do not unwind. type Tstore struct { sync.Mutex - id string dataDir string activeNetParams *chaincfg.Params trillian trillianClient @@ -139,7 +142,7 @@ func (t *Tstore) deblob(b []byte) (*store.BlobEntry, error) { } func (t *Tstore) TreeNew() (int64, error) { - log.Tracef("%v treeNew", t.id) + log.Tracef("TreeNew") tree, _, err := t.trillian.treeNew() if err != nil { @@ -149,27 +152,17 @@ func (t *Tstore) TreeNew() (int64, error) { return tree.TreeId, nil } -func (t *Tstore) TreeExists(treeID int64) bool { - log.Tracef("%v TreeExists: %v", t.id, treeID) - - _, err := t.trillian.tree(treeID) - return err == nil -} - -// TreeFreeze updates the status of a record and freezes the trillian tree as a -// result of a record status change. The tree pointer is the tree ID of the new -// location of the record. This is provided on certain status changes such as -// when a unvetted record is made public and the unvetted record is moved to a -// vetted tree. A value of 0 indicates that no tree pointer exists. +// TreeFreeze updates the status of a record then freezes the trillian tree to +// prevent any additional updates. // -// Once the record index has been saved with its frozen field set, the tree -// is considered to be frozen. The only thing that can be appended onto a +// A tree is considered to be frozen once the record index has been saved with +// its Frozen field set to true. The only thing that can be appended onto a // frozen tree is one additional anchor record. Once a frozen tree has been // anchored, the tstore fsck function will update the status of the tree to -// frozen in trillian, at which point trillian will not allow any additional -// leaves to be appended onto the tree. -func (t *Tstore) TreeFreeze(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, treePointer int64) error { - log.Tracef("%v TreeFreeze: %v", t.id, treeID) +// frozen in trillian, at which point trillian will prevent any changes to the +// tree. +func (t *Tstore) TreeFreeze(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { + log.Tracef("TreeFreeze: %v", treeID) // Save metadata idx, err := t.metadataSave(treeID, rm, metadata) @@ -179,101 +172,9 @@ func (t *Tstore) TreeFreeze(treeID int64, rm backend.RecordMetadata, metadata [] // Update the record index idx.Frozen = true - idx.TreePointer = treePointer - - // Blobify the record index - be, err := convertBlobEntryFromRecordIndex(*idx) - if err != nil { - return err - } - idxDigest, err := hex.DecodeString(be.Digest) - if err != nil { - return err - } - b, err := t.blobify(*be) - if err != nil { - return err - } - - // Save record index blob to the kv store - keys, err := t.store.Put([][]byte{b}) - if err != nil { - return fmt.Errorf("store Put: %v", err) - } - if len(keys) != 1 { - return fmt.Errorf("wrong number of keys: got %v, want 1", len(keys)) - } - - // Append record index leaf to the trillian tree - extraData, err := extraDataEncode(keys[0], dataDescriptorRecordIndex) - if err != nil { - return err - } - leaves := []*trillian.LogLeaf{ - newLogLeaf(idxDigest, extraData), - } - queued, _, err := t.trillian.leavesAppend(treeID, leaves) - if err != nil { - return fmt.Errorf("leavesAppend: %v", err) - } - if len(queued) != 1 { - return fmt.Errorf("wrong number of queued leaves: got %v, want 1", - len(queued)) - } - failed := make([]string, 0, len(queued)) - for _, v := range queued { - c := codes.Code(v.QueuedLeaf.GetStatus().GetCode()) - if c != codes.OK { - failed = append(failed, fmt.Sprintf("%v", c)) - } - } - if len(failed) > 0 { - return fmt.Errorf("append leaves failed: %v", failed) - } - - return nil -} - -// TreePointer returns the tree pointer for the provided tree if one exists. -// The returned bool will indicate if a tree pointer was found. -func (t *Tstore) TreePointer(treeID int64) (int64, bool) { - log.Tracef("%v treePointer: %v", t.id, treeID) - - // Verify tree exists - if !t.TreeExists(treeID) { - return 0, false - } - - // Verify record index exists - var idx *recordIndex - leavesAll, err := t.trillian.leavesAll(treeID) - if err != nil { - err = fmt.Errorf("leavesAll: %v", err) - goto printErr - } - idx, err = t.recordIndexLatest(leavesAll) - if err != nil { - if err == backend.ErrRecordNotFound { - // This is an empty tree. This can happen sometimes if a error - // occurred during record creation. Return gracefully. - return 0, false - } - err = fmt.Errorf("recordIndexLatest: %v", err) - goto printErr - } - // Check if a tree pointer exists - if !treePointerExists(*idx) { - // Tree pointer not found - return 0, false - } - - // Tree pointer found! - return idx.TreePointer, true - -printErr: - log.Errorf("%v treePointer: %v", t.id, err) - return 0, false + // Save the record index + return t.recordIndexSave(treeID, *idx) } // TreesAll returns the IDs of all trees in the tstore instance. @@ -289,6 +190,15 @@ func (t *Tstore) TreesAll() ([]int64, error) { return treeIDs, nil } +// TreeExists returns whether a tree exists in the trillian log. A tree +// existing doesn't necessarily mean that a record exists. Its possible for a +// tree to have been created but experienced an unexpected error prior to the +// record being saved. +func (t *Tstore) TreeExists(treeID int64) bool { + _, err := t.trillian.tree(treeID) + return err == nil +} + func (t *Tstore) treeIsFrozen(leaves []*trillian.LogLeaf) bool { r, err := t.recordIndexLatest(leaves) if err != nil { @@ -336,16 +246,22 @@ type recordBlobsPrepareReply struct { // the blob kv store and appended onto a trillian tree. func (t *Tstore) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) (*recordBlobsPrepareReply, error) { // Verify there are no duplicate or empty mdstream IDs - mdstreamIDs := make(map[uint64]struct{}, len(metadata)) + md := make(map[string]map[uint32]struct{}, len(metadata)) for _, v := range metadata { - if v.ID == 0 { + if v.StreamID == 0 { return nil, fmt.Errorf("invalid metadata stream ID 0") } - _, ok := mdstreamIDs[v.ID] + pmd, ok := md[v.PluginID] + if !ok { + pmd = make(map[uint32]struct{}, len(metadata)) + } + _, ok = pmd[v.StreamID] if ok { - return nil, fmt.Errorf("duplicate metadata stream ID: %v", v.ID) + return nil, fmt.Errorf("duplicate metadata stream: %v %v", + v.PluginID, v.StreamID) } - mdstreamIDs[v.ID] = struct{}{} + pmd[v.StreamID] = struct{}{} + md[v.PluginID] = pmd } // Verify there are no duplicate or empty filenames @@ -384,7 +300,8 @@ func (t *Tstore) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD back return nil, err } h := hex.EncodeToString(util.Digest(b)) - rhashes.metadata[h] = v.PluginID + strconv.FormatUint(v.ID, 10) + streamID := strconv.FormatUint(uint64(v.StreamID), 10) + rhashes.metadata[h] = v.PluginID + streamID } for _, v := range files { b, err := json.Marshal(v) @@ -523,7 +440,7 @@ func (t *Tstore) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD back // to the trillian tree for each blob, then updates the record index with the // trillian leaf information and returns it. func (t *Tstore) recordBlobsSave(treeID int64, rbpr recordBlobsPrepareReply) (*recordIndex, error) { - log.Tracef("recordBlobsSave: %v", t.id, treeID) + log.Tracef("recordBlobsSave: %v", treeID) var ( index = rbpr.recordIndex @@ -607,13 +524,13 @@ func (t *Tstore) recordBlobsSave(treeID int64, rbpr recordBlobsPrepareReply) (*r return &index, nil } -// RecordSave saves the provided record to tstore, creating a new version of -// the record (the record iteration also gets incremented on new versions). -// Once the record contents have been successfully saved to tstore, a -// recordIndex is created for this version of the record and saved to tstore as -// well. +// RecordSave saves the provided record to tstore. Once the record contents +// have been successfully saved to tstore, a recordIndex is created for this +// version of the record and saved to tstore as well. This iteration of the +// record is not considered to be valid until the record index has been +// successfully saved. func (t *Tstore) RecordSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { - log.Tracef("%v RecordSave: %v", t.id, treeID) + log.Tracef("RecordSave: %v", treeID) // Verify tree exists if !t.TreeExists(treeID) { @@ -688,7 +605,7 @@ func (t *Tstore) RecordSave(treeID int64, rm backend.RecordMetadata, metadata [] } } if !fileChanges { - return backend.ErrNoChanges + return backend.ErrNoRecordChanges } // Save blobs @@ -698,8 +615,8 @@ func (t *Tstore) RecordSave(treeID int64, rm backend.RecordMetadata, metadata [] } // Bump the index version and iteration - idx.Version = currIdx.Version + 1 - idx.Iteration = currIdx.Iteration + 1 + idx.Version = rm.Version + idx.Iteration = rm.Iteration // Sanity checks switch { @@ -728,13 +645,13 @@ func (t *Tstore) RecordSave(treeID int64, rm backend.RecordMetadata, metadata [] return nil } -// metadataSave saves the provided metadata to the kv store and trillian tree. -// The record index for this iteration of the record is returned. This is step -// one of a two step process. The record update will not be considered -// successful until the returned record index is also saved to the kv store and -// trillian tree. This code has been pulled out so that it can be called during -// normal metadata updates as well as when an update requires a freeze record -// to be saved along with the record index, such as when a record is censored. +// metadataSave saves the provided metadata to the tstore. The record index +// for this iteration of the record is returned. This is step one of a two step +// process. The record update will not be considered successful until the +// returned record index is also saved to the kv store and trillian tree. This +// code has been pulled out so that it can be called during normal metadata +// updates as well as when an update requires the tree to be frozen, such as +// when a record is censored. func (t *Tstore) metadataSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) (*recordIndex, error) { // Verify tree exists if !t.TreeExists(treeID) { @@ -764,7 +681,7 @@ func (t *Tstore) metadataSave(treeID int64, rm backend.RecordMetadata, metadata // Verify at least one new blob is being saved to the kv store if len(bpr.blobs) == 0 { - return nil, backend.ErrNoChanges + return nil, backend.ErrNoRecordChanges } // Save the blobs @@ -774,17 +691,17 @@ func (t *Tstore) metadataSave(treeID int64, rm backend.RecordMetadata, metadata } // Get the existing record index and add the unchanged fields to - // the new record index. The version and files will remain the - // same. + // the new record index. oldIdx, err := t.recordIndexLatest(leavesAll) if err != nil { return nil, fmt.Errorf("recordIndexLatest: %v", err) } - idx.Version = oldIdx.Version + idx.Version = rm.Version idx.Files = oldIdx.Files - // Increment the iteration - idx.Iteration = oldIdx.Iteration + 1 + // Update the version and iteration + idx.Version = rm.Version + idx.Iteration = rm.Iteration // Sanity check switch { @@ -812,7 +729,7 @@ func (t *Tstore) metadataSave(treeID int64, rm backend.RecordMetadata, metadata // metadata has been successfully saved to tstore, a recordIndex is created for // this iteration of the record and saved to tstore as well. func (t *Tstore) RecordMetadataSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { - log.Tracef("%v RecordMetadataSave: %v", t.id, treeID) + log.Tracef("RecordMetadataSave: %v", treeID) // Save metadata idx, err := t.metadataSave(treeID, rm, metadata) @@ -834,7 +751,7 @@ func (t *Tstore) RecordMetadataSave(treeID int64, rm backend.RecordMetadata, met // iterations of the record. Record metadata and metadata stream blobs are not // deleted. func (t *Tstore) RecordDel(treeID int64) error { - log.Tracef("%v RecordDel: %v", t.id, treeID) + log.Tracef("RecordDel: %v", treeID) // Verify tree exists if !t.TreeExists(treeID) { @@ -893,24 +810,17 @@ func (t *Tstore) RecordDel(treeID int64) error { return nil } -// RecordExists returns whether a record exists on the provided tree ID. A +// RecordExists returns whether a record exists given a trillian tree ID. A // record is considered to not exist if any of the following conditions are // met: // // * A tree does not exist for the tree ID. // // * A tree exists but a record index does not exist. This can happen if a -// tree was created but there was a network error prior to the record index -// being appended to the tree. -// -// * The tree is frozen and points to another tree. The record is considered to -// exists on the tree being pointed to, but not on this one. This happens -// in some situations like when an unvetted record is made public and copied -// onto a vetted tree. -// -// The tree pointer is also returned if one is found. +// tree was created but there was an unexpected error prior to the record +// index being appended to the tree. func (t *Tstore) RecordExists(treeID int64) bool { - log.Tracef("%v RecordExists: %v", t.id, treeID) + log.Tracef("RecordExists: %v", treeID) // Verify tree exists if !t.TreeExists(treeID) { @@ -918,13 +828,12 @@ func (t *Tstore) RecordExists(treeID int64) bool { } // Verify record index exists - var idx *recordIndex leavesAll, err := t.trillian.leavesAll(treeID) if err != nil { err = fmt.Errorf("leavesAll: %v", err) goto printErr } - idx, err = t.recordIndexLatest(leavesAll) + _, err = t.recordIndexLatest(leavesAll) if err != nil { if err == backend.ErrRecordNotFound { // This is an empty tree. This can happen sometimes if a error @@ -935,16 +844,11 @@ func (t *Tstore) RecordExists(treeID int64) bool { goto printErr } - // Verify a tree pointer does not exist - if treePointerExists(*idx) { - return false - } - // Record exists! return true printErr: - log.Errorf("%v RecordExists: %v", t.id, err) + log.Errorf("RecordExists: %v", err) return false } @@ -970,28 +874,13 @@ func (t *Tstore) record(treeID int64, version uint32, filenames []string, omitAl return nil, fmt.Errorf("leavesAll %v: %v", treeID, err) } - // Verify the latest record index does not point to another tree. - // If it does have a tree pointer, the record is considered to - // exists on the tree being pointed to, but not on this one. This - // happens in situations such as when an unvetted record is made - // public and copied to a vetted tree. Querying the unvetted tree - // will result in a backend.ErrRecordNotFound error being returned - // and the vetted tree must be queried instead. + // Use the record index to pull the record content from the store. + // The keys for the record content first need to be extracted from + // their log leaf. indexes, err := t.recordIndexes(leaves) if err != nil { return nil, err } - idxLatest, err := parseRecordIndex(indexes, 0) - if err != nil { - return nil, err - } - if treePointerExists(*idxLatest) { - return nil, backend.ErrRecordNotFound - } - - // Use the record index to pull the record content from the store. - // The keys for the record content first need to be extracted from - // their associated log leaf. idx, err := parseRecordIndex(indexes, version) if err != nil { return nil, err @@ -1136,7 +1025,6 @@ func (t *Tstore) record(treeID int64, version uint32, filenames []string, omitAl } return &backend.Record{ - Version: strconv.FormatUint(uint64(idx.Version), 10), RecordMetadata: *recordMD, Metadata: metadata, Files: files, @@ -1145,14 +1033,14 @@ func (t *Tstore) record(treeID int64, version uint32, filenames []string, omitAl // Record returns the specified version of the record. func (t *Tstore) Record(treeID int64, version uint32) (*backend.Record, error) { - log.Tracef("%v record: %v %v", t.id, treeID, version) + log.Tracef("Record: %v %v", treeID, version) return t.record(treeID, version, []string{}, false) } // RecordLatest returns the latest version of a record. func (t *Tstore) RecordLatest(treeID int64) (*backend.Record, error) { - log.Tracef("%v RecordLatest: %v", t.id, treeID) + log.Tracef("RecordLatest: %v", treeID) return t.record(treeID, 0, []string{}, false) } @@ -1170,8 +1058,8 @@ func (t *Tstore) RecordLatest(treeID int64) (*backend.Record, error) { // OmitAllFiles can be used to retrieve a record without any of the record // files. This supersedes the filenames argument. func (t *Tstore) RecordPartial(treeID int64, version uint32, filenames []string, omitAllFiles bool) (*backend.Record, error) { - log.Tracef("%v RecordPartial: %v %v %v %v", - t.id, treeID, version, omitAllFiles, filenames) + log.Tracef("RecordPartial: %v %v %v %v", + treeID, version, omitAllFiles, filenames) return t.record(treeID, version, filenames, omitAllFiles) } @@ -1315,7 +1203,7 @@ func (t *Tstore) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trilli } func (t *Tstore) RecordTimestamps(treeID int64, version uint32, token []byte) (*backend.RecordTimestamps, error) { - log.Tracef("%v RecordTimestamps: %v %v", t.id, treeID, version) + log.Tracef("RecordTimestamps: %v %v", treeID, version) // Verify tree exists if !t.TreeExists(treeID) { @@ -1358,12 +1246,13 @@ func (t *Tstore) RecordTimestamps(treeID int64, version uint32, token []byte) (* files[k] = *ts } + // TODO fix metadata timestamps + _ = metadata + return &backend.RecordTimestamps{ - Token: hex.EncodeToString(token), - Version: strconv.FormatUint(uint64(version), 10), RecordMetadata: *rm, - Metadata: metadata, - Files: files, + // Metadata: metadata, + Files: files, }, nil } @@ -1375,7 +1264,7 @@ func (t *Tstore) Fsck() { } func (t *Tstore) Close() { - log.Tracef("%v Close", t.id) + log.Tracef("Close") // Close connections t.store.Close() @@ -1387,53 +1276,69 @@ func (t *Tstore) Close() { } } -func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianSigningKeyFile, dbType, dbHost, dbPass, dbEncryptionKeyFile, dcrtimeHost, dcrtimeCert string) (*Tstore, error) { - // Load database encryption key if provided. An encryption key is - // optional. - var ek *encryptionKey - if dbEncryptionKeyFile != "" { - f, err := os.Open(dbEncryptionKeyFile) +func New(appDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianSigningKeyFile, dbType, dbHost, dbPass, dbEncryptionKeyFile, dcrtimeHost, dcrtimeCert string) (*Tstore, error) { + // Setup encryption key file + if dbEncryptionKeyFile == "" { + // No file path was given. Use the default path. + dbEncryptionKeyFile = filepath.Join(appDir, defaultEncryptionKeyFilename) + } + if !util.FileExists(dbEncryptionKeyFile) { + // Encryption key file does not exist. Create one. + log.Infof("Generating encryption key") + key, err := sbox.NewKey() if err != nil { return nil, err } - var key [32]byte - n, err := f.Read(key[:]) - if n != len(key) { - return nil, fmt.Errorf("invalid encryption key length") - } + err = ioutil.WriteFile(dbEncryptionKeyFile, key[:], 0400) if err != nil { return nil, err } - f.Close() - ek = newEncryptionKey(&key) - - log.Infof("Encryption key %v: %v", id, dbEncryptionKeyFile) + util.Zero(key[:]) + log.Infof("Encryption key created: %v", dbEncryptionKeyFile) } - // Setup datadir for this tstore instance - dataDir = filepath.Join(dataDir, id) - err := os.MkdirAll(dataDir, 0700) + // Load encryption key + f, err := os.Open(dbEncryptionKeyFile) if err != nil { return nil, err } + var key [32]byte + n, err := f.Read(key[:]) + if n != len(key) { + return nil, fmt.Errorf("invalid encryption key length") + } + if err != nil { + return nil, err + } + f.Close() + ek := newEncryptionKey(&key) + + log.Infof("Encryption key: %v", dbEncryptionKeyFile) // Setup trillian client if trillianSigningKeyFile == "" { // No file path was given. Use the default path. fn := fmt.Sprintf("%v", defaultTrillianSigningKeyFilename) - trillianSigningKeyFile = filepath.Join(homeDir, fn) + trillianSigningKeyFile = filepath.Join(appDir, fn) } - log.Infof("Trillian key %v: %v", id, trillianSigningKeyFile) - log.Infof("Trillian host %v: %v", id, trillianHost) + log.Infof("Trillian key: %v", trillianSigningKeyFile) + log.Infof("Trillian host: %v", trillianHost) trillianClient, err := newTClient(trillianHost, trillianSigningKeyFile) if err != nil { return nil, err } + // Setup datadir for this tstore instance + dataDir = filepath.Join(dataDir) + err = os.MkdirAll(dataDir, 0700) + if err != nil { + return nil, err + } + // Setup key-value store - log.Infof("Database type %v: %v", id, dbType) + log.Infof("Database type: %v", dbType) var kvstore store.BlobKV switch dbType { case DBTypeLevelDB: @@ -1448,7 +1353,7 @@ func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trilli } case DBTypeMySQL: // Example db name: testnet3_unvetted_kv - dbName := fmt.Sprintf("%v_%v_kv", anp.Name, id) + dbName := fmt.Sprintf("%v_kv", anp.Name) kvstore, err = mysql.New(dbHost, dbUser, dbPass, dbName) if err != nil { return nil, err @@ -1457,6 +1362,13 @@ func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trilli return nil, fmt.Errorf("invalid db type: %v", dbType) } + // Verify dcrtime host + _, err = url.Parse(dcrtimeHost) + if err != nil { + return nil, fmt.Errorf("parse dcrtime host '%v': %v", dcrtimeHost, err) + } + log.Infof("Anchor host: %v", dcrtimeHost) + // Setup dcrtime client dcrtimeClient, err := newDcrtimeClient(dcrtimeHost, dcrtimeCert) if err != nil { @@ -1465,7 +1377,6 @@ func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trilli // Setup tstore t := Tstore{ - id: id, dataDir: dataDir, activeNetParams: anp, trillian: trillianClient, @@ -1477,11 +1388,11 @@ func New(id, homeDir, dataDir string, anp *chaincfg.Params, trillianHost, trilli } // Launch cron - log.Infof("Launch %v cron anchor job", id) + log.Infof("Launch cron anchor job") err = t.cron.AddFunc(anchorSchedule, func() { err := t.anchorTrees() if err != nil { - log.Errorf("%v anchorTrees: %v", id, err) + log.Errorf("anchorTrees: %v", err) } }) if err != nil { diff --git a/politeiad/backend/tstorebe/tstore/verify.go b/politeiad/backendv2/tstorebe/tstore/verify.go similarity index 98% rename from politeiad/backend/tstorebe/tstore/verify.go rename to politeiad/backendv2/tstorebe/tstore/verify.go index 0caf0aa75..2a82ec181 100644 --- a/politeiad/backend/tstorebe/tstore/verify.go +++ b/politeiad/backendv2/tstorebe/tstore/verify.go @@ -13,7 +13,7 @@ import ( "github.com/decred/dcrtime/merkle" dmerkle "github.com/decred/dcrtime/merkle" - "github.com/decred/politeia/politeiad/backend" + backend "github.com/decred/politeia/politeiad/backendv2" "github.com/decred/politeia/util" "github.com/google/trillian" tmerkle "github.com/google/trillian/merkle" diff --git a/politeiad/backendv2/tstorebe/tstorebe.go b/politeiad/backendv2/tstorebe/tstorebe.go new file mode 100644 index 000000000..2d67a72ee --- /dev/null +++ b/politeiad/backendv2/tstorebe/tstorebe.go @@ -0,0 +1,1156 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tstorebe + +import ( + "bytes" + "encoding/base64" + "encoding/binary" + "encoding/hex" + "encoding/json" + "fmt" + "path/filepath" + "strconv" + "sync" + "time" + + "github.com/decred/dcrd/chaincfg/v3" + "github.com/decred/politeia/politeiad/api/v1/mime" + backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/tstore" + "github.com/decred/politeia/util" + "github.com/subosito/gozaru" +) + +var ( + _ backend.Backend = (*tstoreBackend)(nil) +) + +// tstoreBackend implements the Backend interface. +type tstoreBackend struct { + sync.RWMutex + appDir string + dataDir string + shutdown bool + tstore *tstore.Tstore + + // prefixes contains the prefix to full token mapping for all + // records. The prefix is the first n characters of the hex encoded + // record token, where n is defined by the token prefix length + // politeiad setting. Record lookups by token prefix are allowed. + // This cache is used to prevent prefix collisions when creating + // new tokens and to facilitate lookups by token prefix. This cache + // is built on startup. + prefixes map[string][]byte // [tokenPrefix]token + + // recordMtxs allows the backend to hold a lock on an individual + // record so that it can perform multiple read/write operations + // in a concurrent safe manner. These mutexes are lazy loaded. + recordMtxs map[string]*sync.Mutex +} + +// isShutdown returns whether the backend is shutdown. +func (t *tstoreBackend) isShutdown() bool { + t.RLock() + defer t.RUnlock() + + return t.shutdown +} + +// recordMutex returns the mutex for a record. +func (t *tstoreBackend) recordMutex(token []byte) *sync.Mutex { + t.Lock() + defer t.Unlock() + + ts := hex.EncodeToString(token) + m, ok := t.recordMtxs[ts] + if !ok { + // recordMtxs is lazy loaded + m = &sync.Mutex{} + t.recordMtxs[ts] = m + } + + return m +} + +func (t *tstoreBackend) prefixExists(fullToken []byte) bool { + t.RLock() + defer t.RUnlock() + + _, ok := t.prefixes[util.TokenPrefix(fullToken)] + return ok +} + +func (t *tstoreBackend) prefixAdd(fullToken []byte) { + t.Lock() + defer t.Unlock() + + prefix := util.TokenPrefix(fullToken) + t.prefixes[prefix] = fullToken + + log.Debugf("Add token prefix: %v", prefix) +} + +func tokenFromTreeID(treeID int64) []byte { + b := make([]byte, 8) + // Converting between int64 and uint64 doesn't change + // the sign bit, only the way it's interpreted. + binary.LittleEndian.PutUint64(b, uint64(treeID)) + return b +} + +func tokenIsFullLength(token []byte) bool { + return util.TokenIsFullLength(util.TokenTypeTstore, token) +} + +func treeIDFromToken(token []byte) int64 { + if !tokenIsFullLength(token) { + return 0 + } + return int64(binary.LittleEndian.Uint64(token)) +} + +// metadataStreamsVerify verifies that all provided metadata streams are sane. +func metadataStreamsVerify(metadata []backend.MetadataStream) error { + // Verify metadata + md := make(map[string]map[uint32]struct{}, len(metadata)) + for i, v := range metadata { + // Verify all fields are provided + switch { + case v.PluginID == "": + e := fmt.Sprintf("plugin id missing at index %v", i) + return backend.ContentError{ + ErrorCode: backend.ContentErrorMetadataStreamInvalid, + ErrorContext: e, + } + case v.StreamID == 0: + e := fmt.Sprintf("stream id missing at index %v", i) + return backend.ContentError{ + ErrorCode: backend.ContentErrorMetadataStreamInvalid, + ErrorContext: e, + } + case v.Payload == "": + e := fmt.Sprintf("payload missing on %v %v", v.PluginID, v.StreamID) + return backend.ContentError{ + ErrorCode: backend.ContentErrorMetadataStreamInvalid, + ErrorContext: e, + } + } + + // Verify no duplicates + m, ok := md[v.PluginID] + if !ok { + m = make(map[uint32]struct{}, len(metadata)) + md[v.PluginID] = m + } + if _, ok := m[v.StreamID]; ok { + e := fmt.Sprintf("%v %v", v.PluginID, v.StreamID) + return backend.ContentError{ + ErrorCode: backend.ContentErrorMetadataStreamDuplicate, + ErrorContext: e, + } + } + + // Add to metadata list + m[v.StreamID] = struct{}{} + md[v.PluginID] = m + } + + return nil +} + +func metadataStreamsUpdate(curr, mdAppend, mdOverwrite []backend.MetadataStream) []backend.MetadataStream { + // Put current metadata into a map + md := make(map[string]backend.MetadataStream, len(curr)) + for _, v := range curr { + k := v.PluginID + strconv.FormatUint(uint64(v.StreamID), 10) + md[k] = v + } + + // Apply overwrites + for _, v := range mdOverwrite { + k := v.PluginID + strconv.FormatUint(uint64(v.StreamID), 10) + md[k] = v + } + + // Apply appends. Its ok if an append is specified but there is no + // existing metadata for that metadata stream. In this case the + // append data will become the full metadata stream. + for _, v := range mdAppend { + k := v.PluginID + strconv.FormatUint(uint64(v.StreamID), 10) + m, ok := md[k] + if !ok { + // No existing metadata. Use append data as full metadata + // stream. + md[k] = v + continue + } + + // Metadata exists. Append to it. + buf := bytes.NewBuffer([]byte(m.Payload)) + buf.WriteString(v.Payload) + m.Payload = buf.String() + md[k] = m + } + + // Convert metadata back to a slice + metadata := make([]backend.MetadataStream, 0, len(md)) + for _, v := range md { + metadata = append(metadata, v) + } + + return metadata +} + +// filesVerify verifies that all provided files are sane. +func filesVerify(files []backend.File, filesDel []string) error { + // Verify files are being updated + if len(files) == 0 && len(filesDel) == 0 { + return backend.ContentError{ + ErrorCode: backend.ContentErrorFilesEmpty, + } + } + + // Prevent paths + for i := range files { + if filepath.Base(files[i].Name) != files[i].Name { + e := fmt.Sprintf("%v contains a file path", files[i].Name) + return backend.ContentError{ + ErrorCode: backend.ContentErrorFileNameInvalid, + ErrorContext: e, + } + } + } + for _, v := range filesDel { + if filepath.Base(v) != v { + e := fmt.Sprintf("%v contains a file path", v) + return backend.ContentError{ + ErrorCode: backend.ContentErrorFileNameInvalid, + ErrorContext: e, + } + } + } + + // Prevent duplicate filenames + fn := make(map[string]struct{}, len(files)+len(filesDel)) + for i := range files { + if _, ok := fn[files[i].Name]; ok { + return backend.ContentError{ + ErrorCode: backend.ContentErrorFileNameDuplicate, + ErrorContext: files[i].Name, + } + } + fn[files[i].Name] = struct{}{} + } + for _, v := range filesDel { + if _, ok := fn[v]; ok { + return backend.ContentError{ + ErrorCode: backend.ContentErrorFileNameDuplicate, + ErrorContext: v, + } + } + fn[v] = struct{}{} + } + + // Prevent bad filenames + for i := range files { + if gozaru.Sanitize(files[i].Name) != files[i].Name { + e := fmt.Sprintf("%v is not sanitized", files[i].Name) + return backend.ContentError{ + ErrorCode: backend.ContentErrorFileNameInvalid, + ErrorContext: e, + } + } + + // Verify digest + d, ok := util.ConvertDigest(files[i].Digest) + if !ok { + return backend.ContentError{ + ErrorCode: backend.ContentErrorFileDigestInvalid, + ErrorContext: files[i].Name, + } + } + + // Verify payload is not empty + if files[i].Payload == "" { + e := fmt.Sprintf("%v payload empty", files[i].Name) + return backend.ContentError{ + ErrorCode: backend.ContentErrorFilePayloadInvalid, + ErrorContext: e, + } + } + + // Decode base64 payload + payload, err := base64.StdEncoding.DecodeString(files[i].Payload) + if err != nil { + e := fmt.Sprintf("%v invalid base64", files[i].Name) + return backend.ContentError{ + ErrorCode: backend.ContentErrorFilePayloadInvalid, + ErrorContext: e, + } + } + + // Calculate payload digest + dp := util.Digest(payload) + if !bytes.Equal(d[:], dp) { + e := fmt.Sprintf("%v digest got %x, want %x", + files[i].Name, d[:], dp) + return backend.ContentError{ + ErrorCode: backend.ContentErrorFileDigestInvalid, + ErrorContext: e, + } + } + + // Verify MIME + detectedMIMEType := mime.DetectMimeType(payload) + if detectedMIMEType != files[i].MIME { + e := fmt.Sprintf("%v mime got %v, want %v", + files[i].Name, files[i].MIME, detectedMIMEType) + return backend.ContentError{ + ErrorCode: backend.ContentErrorFileMIMETypeInvalid, + ErrorContext: e, + } + } + + if !mime.MimeValid(files[i].MIME) { + return backend.ContentError{ + ErrorCode: backend.ContentErrorFileMIMETypeUnsupported, + ErrorContext: files[i].Name, + } + } + } + + return nil +} + +func filesUpdate(filesCurr, filesAdd []backend.File, filesDel []string) []backend.File { + // Put current files into a map + curr := make(map[string]backend.File, len(filesCurr)) // [filename]File + for _, v := range filesCurr { + curr[v.Name] = v + } + + // Apply deletes + for _, fn := range filesDel { + _, ok := curr[fn] + if ok { + delete(curr, fn) + } + } + + // Apply adds + for _, v := range filesAdd { + curr[v.Name] = v + } + + // Convert back to a slice + f := make([]backend.File, 0, len(curr)) + for _, v := range curr { + f = append(f, v) + } + + return f +} + +func recordMetadataNew(token []byte, files []backend.File, status backend.StatusT, version, iteration uint32) (*backend.RecordMetadata, error) { + digests := make([]string, 0, len(files)) + for _, v := range files { + digests = append(digests, v.Digest) + } + m, err := util.MerkleRoot(digests) + if err != nil { + return nil, err + } + return &backend.RecordMetadata{ + Token: hex.EncodeToString(token), + Version: version, + Iteration: iteration, + Status: status, + Timestamp: time.Now().Unix(), + Merkle: hex.EncodeToString(m[:]), + }, nil +} + +// RecordNew creates a new record. +// +// This function satisfies the Backend interface. +func (t *tstoreBackend) RecordNew(metadata []backend.MetadataStream, files []backend.File) (*backend.Record, error) { + log.Tracef("RecordNew") + + // Verify record content + err := metadataStreamsVerify(metadata) + if err != nil { + return nil, err + } + err = filesVerify(files, nil) + if err != nil { + return nil, err + } + + // Call pre plugin hooks + pre := plugins.HookNewRecordPre{ + Metadata: metadata, + Files: files, + } + b, err := json.Marshal(pre) + if err != nil { + return nil, err + } + err = t.tstore.PluginHookPre(0, []byte{}, + plugins.HookTypeNewRecordPre, string(b)) + if err != nil { + return nil, err + } + + // Create a new token + var token []byte + var treeID int64 + for retries := 0; retries < 10; retries++ { + treeID, err = t.tstore.TreeNew() + if err != nil { + return nil, err + } + token = tokenFromTreeID(treeID) + + // Check for token prefix collisions + if !t.prefixExists(token) { + // Not a collision. Use this token. + + // Update the prefix cache. This must be done even if the + // record creation fails since the tree will still exist in + // tstore. + t.prefixAdd(token) + + break + } + + log.Infof("Token prefix collision %v, creating new token", + util.TokenPrefix(token)) + } + + // Create record metadata + rm, err := recordMetadataNew(token, files, backend.StatusPublic, 1, 1) + if err != nil { + return nil, err + } + + // Save the record + err = t.tstore.RecordSave(treeID, *rm, metadata, files) + if err != nil { + return nil, fmt.Errorf("RecordSave: %v", err) + } + + // Call post plugin hooks + post := plugins.HookNewRecordPost{ + Metadata: metadata, + Files: files, + RecordMetadata: *rm, + } + b, err = json.Marshal(post) + if err != nil { + return nil, err + } + t.tstore.PluginHookPost(treeID, token, + plugins.HookTypeNewRecordPost, string(b)) + + // Update the inventory cache + t.inventoryAdd(backend.StateUnvetted, token, backend.StatusUnreviewed) + + // Get the full record to return + r, err := t.RecordGet(token, 0) + if err != nil { + return nil, fmt.Errorf("RecordGet %x: %v", token, err) + } + + return r, nil +} + +// RecordEdit edits an existing record. This creates a new version of the +// record. +// +// This function satisfies the Backend interface. +func (t *tstoreBackend) RecordEdit(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { + log.Tracef("RecordEdit: %x", token) + + // Verify record contents. Send in a single metadata array to + // verify there are no dups. + allMD := append(mdAppend, mdOverwrite...) + err := metadataStreamsVerify(allMD) + if err != nil { + return nil, err + } + err = filesVerify(filesAdd, filesDel) + if err != nil { + return nil, err + } + + // Verify token is valid. The full length token must be used when + // writing data. + if !tokenIsFullLength(token) { + return nil, backend.ErrTokenInvalid + } + + // Verify record exists + if !t.RecordExists(token) { + return nil, backend.ErrRecordNotFound + } + + // Apply the record changes and save the new version. The record + // lock needs to be held for the remainder of the function. + if t.isShutdown() { + return nil, backend.ErrShutdown + } + m := t.recordMutex(token) + m.Lock() + defer m.Unlock() + + // Get existing record + treeID := treeIDFromToken(token) + r, err := t.tstore.RecordLatest(treeID) + if err != nil { + return nil, fmt.Errorf("RecordLatest %v: %v", treeID, err) + } + + // Apply changes + metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) + files := filesUpdate(r.Files, filesAdd, filesDel) + recordMD, err := recordMetadataNew(token, files, r.RecordMetadata.Status, + r.RecordMetadata.Version+1, r.RecordMetadata.Iteration+1) + if err != nil { + return nil, err + } + + // Call pre plugin hooks + her := plugins.HookEditRecord{ + Current: *r, + RecordMetadata: *recordMD, + Metadata: metadata, + Files: files, + } + b, err := json.Marshal(her) + if err != nil { + return nil, err + } + err = t.tstore.PluginHookPre(treeID, token, + plugins.HookTypeEditRecordPre, string(b)) + if err != nil { + return nil, err + } + + // Save record + err = t.tstore.RecordSave(treeID, *recordMD, metadata, files) + if err != nil { + switch err { + case backend.ErrRecordLocked, backend.ErrNoRecordChanges: + return nil, err + default: + return nil, fmt.Errorf("RecordSave: %v", err) + } + } + + // Call post plugin hooks + t.tstore.PluginHookPost(treeID, token, + plugins.HookTypeEditRecordPost, string(b)) + + // Return updated record + r, err = t.tstore.RecordLatest(treeID) + if err != nil { + return nil, fmt.Errorf("RecordLatest %v: %v", treeID, err) + } + + return r, nil +} + +// RecordEditMetadata edits the metadata of a record without changing any +// record files. This creates a new iteration of the record, but not a new +// version of the record. +// +// This function satisfies the Backend interface. +func (t *tstoreBackend) RecordEditMetadata(token []byte, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { + log.Tracef("RecordEditMetadata: %x", token) + + // Verify metadata. Send in a single metadata array to verify there + // are no dups. + allMD := append(mdAppend, mdOverwrite...) + err := metadataStreamsVerify(allMD) + if err != nil { + return nil, err + } + if len(mdAppend) == 0 && len(mdOverwrite) == 0 { + return nil, backend.ErrNoRecordChanges + } + + // Verify token is valid. The full length token must be used when + // writing data. + if !tokenIsFullLength(token) { + return nil, backend.ErrTokenInvalid + } + + // Verify record exists + if !t.RecordExists(token) { + return nil, backend.ErrRecordNotFound + } + + // Apply the record changes and save the new version. The record + // lock needs to be held for the remainder of the function. + if t.isShutdown() { + return nil, backend.ErrShutdown + } + m := t.recordMutex(token) + m.Lock() + defer m.Unlock() + + // Get existing record + treeID := treeIDFromToken(token) + r, err := t.tstore.RecordLatest(treeID) + if err != nil { + return nil, fmt.Errorf("RecordLatest %v: %v", treeID, err) + } + + // Apply changes. The version is not incremented for metadata only + // updates. The iteration is incremented. + metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) + recordMD, err := recordMetadataNew(token, r.Files, r.RecordMetadata.Status, + r.RecordMetadata.Version, r.RecordMetadata.Iteration+1) + if err != nil { + return nil, err + } + + // Call pre plugin hooks + hem := plugins.HookEditMetadata{ + Current: *r, + Metadata: metadata, + } + b, err := json.Marshal(hem) + if err != nil { + return nil, err + } + err = t.tstore.PluginHookPre(treeID, token, + plugins.HookTypeEditMetadataPre, string(b)) + if err != nil { + return nil, err + } + + // Update metadata + err = t.tstore.RecordMetadataSave(treeID, *recordMD, metadata) + if err != nil { + switch err { + case backend.ErrRecordLocked, backend.ErrNoRecordChanges: + return nil, err + default: + return nil, fmt.Errorf("RecordMetadataSave: %v", err) + } + } + + // Call post plugin hooks + t.tstore.PluginHookPost(treeID, token, + plugins.HookTypeEditMetadataPost, string(b)) + + // Return updated record + r, err = t.tstore.RecordLatest(treeID) + if err != nil { + return nil, fmt.Errorf("RecordLatest %v: %v", treeID, err) + } + + return r, nil +} + +var ( + // statusChanges contains the allowed record status changes. If + // statusChanges[currentStatus][newStatus] exists then the status + // change is allowed. + statusChanges = map[backend.StatusT]map[backend.StatusT]struct{}{ + // Unreviewed to... + backend.StatusUnreviewed: { + backend.StatusPublic: {}, + backend.StatusCensored: {}, + }, + // Public to... + backend.StatusPublic: { + backend.StatusCensored: {}, + backend.StatusArchived: {}, + }, + // Statuses that do not allow any further transitions + backend.StatusCensored: {}, + backend.StatusArchived: {}, + } +) + +// statusChangeIsAllowed returns whether the provided status change is allowed. +func statusChangeIsAllowed(from, to backend.StatusT) bool { + allowed, ok := statusChanges[from] + if !ok { + return false + } + _, ok = allowed[to] + return ok +} + +// setStatusPublic updates the status of a record to public. +// +// This function must be called WITH the record lock held. +func (t *tstoreBackend) setStatusPublic(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { + // TODO tstore needs a publish method + treeID := treeIDFromToken(token) + return t.tstore.RecordMetadataSave(treeID, rm, metadata) +} + +// setStatusArchived updates the status of a record to archived. +// +// This function must be called WITH the record lock held. +func (t *tstoreBackend) setStatusArchived(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { + // Freeze the tree + treeID := treeIDFromToken(token) + err := t.tstore.TreeFreeze(treeID, rm, metadata) + if err != nil { + return fmt.Errorf("TreeFreeze %v: %v", treeID, err) + } + + log.Debugf("Record frozen %x", token) + + // Nothing else needs to be done for a archived record + + return nil +} + +// setStatusCensored updates the status of a record to censored. +// +// This function must be called WITH the record lock held. +func (t *tstoreBackend) setStatusCensored(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { + // Freeze the tree + treeID := treeIDFromToken(token) + err := t.tstore.TreeFreeze(treeID, rm, metadata) + if err != nil { + return fmt.Errorf("TreeFreeze %v: %v", treeID, err) + } + + log.Debugf("Record frozen %x", token) + + // Delete all record files + err = t.tstore.RecordDel(treeID) + if err != nil { + return fmt.Errorf("RecordDel %v: %v", treeID, err) + } + + log.Debugf("Record contents deleted %x", token) + + return nil +} + +// RecordSetStatus sets the status of a record. +// +// This function satisfies the Backend interface. +func (t *tstoreBackend) RecordSetStatus(token []byte, status backend.StatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { + log.Tracef("RecordSetStatus: %x %v", token, status) + + // Verify token is valid. The full length token must be used when + // writing data. + if !tokenIsFullLength(token) { + return nil, backend.ErrTokenInvalid + } + + // Verify record exists + if !t.RecordExists(token) { + return nil, backend.ErrRecordNotFound + } + + // The existing record must be pulled and updated. The record + // lock must be held for the rest of this function. + if t.isShutdown() { + return nil, backend.ErrShutdown + } + m := t.recordMutex(token) + m.Lock() + defer m.Unlock() + + // Get existing record + treeID := treeIDFromToken(token) + r, err := t.tstore.RecordLatest(treeID) + if err != nil { + return nil, fmt.Errorf("RecordLatest: %v", err) + } + currStatus := r.RecordMetadata.Status + + // Validate status change + if !statusChangeIsAllowed(currStatus, status) { + return nil, backend.StatusTransitionError{ + From: currStatus, + To: status, + } + } + + // Apply changes + recordMD, err := recordMetadataNew(token, r.Files, status, + r.RecordMetadata.Version, r.RecordMetadata.Iteration+1) + if err != nil { + return nil, err + } + metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) + + // Call pre plugin hooks + hsrs := plugins.HookSetRecordStatus{ + Current: *r, + RecordMetadata: *recordMD, + Metadata: metadata, + } + b, err := json.Marshal(hsrs) + if err != nil { + return nil, err + } + err = t.tstore.PluginHookPre(treeID, token, + plugins.HookTypeSetRecordStatusPre, string(b)) + if err != nil { + return nil, err + } + + // Update record status + switch status { + case backend.StatusPublic: + err := t.setStatusPublic(token, *recordMD, metadata) + if err != nil { + return nil, err + } + case backend.StatusArchived: + err := t.setStatusArchived(token, *recordMD, metadata) + if err != nil { + return nil, err + } + case backend.StatusCensored: + err := t.setStatusCensored(token, *recordMD, metadata) + if err != nil { + return nil, err + } + default: + // Should not happen + return nil, fmt.Errorf("unknown status %v", status) + } + + log.Debugf("Status updated %x from %v (%v) to %v (%v)", + token, backend.Statuses[currStatus], currStatus, + backend.Statuses[status], status) + + // Call post plugin hooks + t.tstore.PluginHookPost(treeID, token, + plugins.HookTypeSetRecordStatusPost, string(b)) + + // Update inventory cache + switch status { + case backend.StatusPublic: + // The state is updated to vetted when a record is made public + t.inventoryUpdate(backend.StateVetted, token, status) + default: + t.inventoryUpdate(r.RecordMetadata.State, token, status) + } + + // Return updated record + r, err = t.tstore.RecordLatest(treeID) + if err != nil { + return nil, fmt.Errorf("RecordLatest %v: %v", treeID, err) + } + + return r, nil +} + +// RecordExists returns whether a record exists. +// +// This method relies on the the tstore tree exists call. It's possible for a +// tree to exist that does not correspond to a record in the rare case that a +// tree was created but an unexpected error, such as a network error, was +// encoutered prior to the record being saved to the tree. We ignore this edge +// case because: +// +// 1. A user has no way to obtain this token unless the trillian instance has +// been opened to the public. +// +// 2. Even if they have the token they cannot do anything with it. Any attempt +// to read from the tree or write to the tree will return a RecordNotFound +// error. +// +// Pulling the leaves from the tree to see if a record has been saved to the +// tree adds a large amount of overhead to this call that should be very light +// weight. Its for this reason that we rely on the tree exists call despite the +// edge case. +// +// This function satisfies the Backend interface. +func (t *tstoreBackend) RecordExists(token []byte) bool { + log.Tracef("RecordExists: %x", token) + + treeID := treeIDFromToken(token) + return t.tstore.TreeExists(treeID) +} + +// RecordGet retrieves a record. If no version is provided then the most recent +// version will be returned. +// +// This function satisfies the Backend interface. +func (t *tstoreBackend) RecordGet(token []byte, version uint32) (*backend.Record, error) { + log.Tracef("RecordGet: %x", token) + + treeID := treeIDFromToken(token) + return t.tstore.Record(treeID, version) +} + +// RecordGetBatch retreives a batch of records. Individual record errors are +// not returned. If the record was not found then it will not be included in +// the returned map. +// +// This function satisfies the Backend interface. +func (t *tstoreBackend) RecordGetBatch(reqs []backend.RecordRequest) (map[string]backend.Record, error) { + log.Tracef("RecordGetBatch") + + records := make(map[string]backend.Record, len(reqs)) + for _, v := range reqs { + treeID := treeIDFromToken(v.Token) + r, err := t.tstore.RecordPartial(treeID, v.Version, + v.Filenames, v.OmitAllFiles) + if err != nil { + if err == backend.ErrRecordNotFound { + // Record doesn't exist. This is ok. It will not be included + // in the reply. + continue + } + // An unexpected error occurred. Log it and continue. + log.Debug("RecordPartial %v: %v", treeID, err) + continue + } + + // Update reply + records[r.RecordMetadata.Token] = *r + } + + return records, nil +} + +// RecordTimestamps returns the timestamps for a record. If no version is provided +// then timestamps for the most recent version will be returned. +// +// This function satisfies the Backend interface. +func (t *tstoreBackend) RecordTimestamps(token []byte, version uint32) (*backend.RecordTimestamps, error) { + log.Tracef("RecordTimestamps: %x %v", token, version) + + treeID := treeIDFromToken(token) + return t.tstore.RecordTimestamps(treeID, version, token) +} + +// Inventory returns the tokens of records in the inventory categorized by +// record state and record status. The tokens are ordered by the timestamp of +// their most recent status change, sorted from newest to oldest. +// +// The state, status, and page arguments can be provided to request a specific +// page of record tokens. +// +// If no status is provided then the most recent page of tokens for each +// statuses will be returned. All other arguments are ignored. +// +// This function satisfies the Backend interface. +func (t *tstoreBackend) Inventory(state backend.StateT, status backend.StatusT, pageSize, pageNumber uint32) (*backend.Inventory, error) { + log.Tracef("InventoryByStatus: %v %v %v %v", + state, status, pageSize, pageNumber) + + inv, err := t.invByStatus(state, status, pageSize, pageNumber) + if err != nil { + return nil, err + } + + return &backend.Inventory{ + Unvetted: inv.Unvetted, + Vetted: inv.Vetted, + }, nil +} + +// InventoryTimeOrdered returns a page of record tokens sorted by timestamp of +// their most recent status change. The returned tokens are not sorted by +// status and will included all statuses. +// +// This function satisfies the Backend interface. +func (t *tstoreBackend) InventoryTimeOrdered(state backend.StateT, pageSize, pageNumber uint32) ([]string, error) { + log.Tracef("InventoryTimeOrdered: %v %v %v", state, pageSize, pageNumber) + + return nil, fmt.Errorf("not implemented") +} + +// PluginRegister registers a plugin. +// +// This function satisfies the Backend interface. +func (t *tstoreBackend) PluginRegister(p backend.Plugin) error { + return t.tstore.PluginRegister(t, p) +} + +// PluginSetup performs any required plugin setup. +// +// This function satisfies the Backend interface. +func (t *tstoreBackend) PluginSetup(pluginID string) error { + log.Tracef("PluginSetup: %v", pluginID) + + return t.tstore.PluginSetup(pluginID) +} + +// PluginCmdRead executes a read plugin command. +// +// This function satisfies the Backend interface. +func (t *tstoreBackend) PluginCmdRead(token []byte, pluginID, pluginCmd, payload string) (string, error) { + log.Tracef("PluginCmdRead: %x %v %v", token, pluginID, pluginCmd) + + // The token is optional. If a token is not provided then a tree ID + // will not be provided to the plugin. + var treeID int64 + if len(token) > 0 { + treeID = treeIDFromToken(token) + + // Verify record exists + if !t.RecordExists(token) { + return "", backend.ErrRecordNotFound + } + } + + if len(token) > 0 { + log.Infof("Plugin '%v' read cmd '%v' on %x", + pluginID, pluginCmd, token) + } else { + log.Infof("Plugin '%v' read cmd '%v'", + pluginID, pluginCmd) + } + + return t.tstore.PluginCmd(treeID, token, pluginID, pluginCmd, payload) +} + +// PluginCmdWrite executes a write plugin command. +// +// This function satisfies the Backend interface. +func (t *tstoreBackend) PluginCmdWrite(token []byte, pluginID, pluginCmd, payload string) (string, error) { + log.Tracef("PluginCmdWrite: %x %v %v", token, pluginID, pluginCmd) + + // Verify record exists + if !t.RecordExists(token) { + return "", backend.ErrRecordNotFound + } + + // Hold the record lock for the remainder of this function. We + // do this here in the backend so that the individual plugins + // implementations don't need to worry about race conditions. + m := t.recordMutex(token) + m.Lock() + defer m.Unlock() + + log.Infof("Plugin '%v' write cmd '%v' on %x", + pluginID, pluginCmd, token) + + // Call pre plugin hooks + treeID := treeIDFromToken(token) + hp := plugins.HookPluginPre{ + PluginID: pluginID, + Cmd: pluginCmd, + Payload: payload, + } + b, err := json.Marshal(hp) + if err != nil { + return "", err + } + err = t.tstore.PluginHookPre(treeID, token, + plugins.HookTypePluginPre, string(b)) + if err != nil { + return "", err + } + + // Execute plugin command + reply, err := t.tstore.PluginCmd(treeID, token, + pluginID, pluginCmd, payload) + if err != nil { + return "", err + } + + // Call post plugin hooks + hpp := plugins.HookPluginPost{ + PluginID: pluginID, + Cmd: pluginCmd, + Payload: payload, + Reply: reply, + } + b, err = json.Marshal(hpp) + if err != nil { + return "", err + } + t.tstore.PluginHookPost(treeID, token, + plugins.HookTypePluginPost, string(b)) + + return reply, nil +} + +// PluginInventory returns all registered plugins. +// +// This function satisfies the Backend interface. +func (t *tstoreBackend) PluginInventory() []backend.Plugin { + log.Tracef("Plugins") + + return t.tstore.Plugins() +} + +// Close performs cleanup of the backend. +// +// This function satisfies the Backend interface. +func (t *tstoreBackend) Close() { + log.Tracef("Close") + + t.Lock() + defer t.Unlock() + + // Shutdown backend + t.shutdown = true + + // Close tstore connections + t.tstore.Close() +} + +// setup builds the tstore backend caches. +func (t *tstoreBackend) setup() error { + log.Tracef("setup") + + log.Infof("Building backend token prefix cache") + + // A record token is created using the unvetted tree ID so we + // only need to retrieve the unvetted trees in order to build the + // token prefix cache. + treeIDs, err := t.tstore.TreesAll() + if err != nil { + return fmt.Errorf("TreesAll: %v", err) + } + + log.Infof("%v records in the backend", len(treeIDs)) + + for _, v := range treeIDs { + token := tokenFromTreeID(v) + t.prefixAdd(token) + } + + return nil +} + +// New returns a new tstoreBackend. +func New(appDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianSigningKey, dbType, dbHost, dbPass, dbEncryptionKeyFile, dcrtimeHost, dcrtimeCert string) (*tstoreBackend, error) { + // Setup tstore instances + ts, err := tstore.New(appDir, dataDir, anp, trillianHost, + trillianSigningKey, dbType, dbHost, dbPass, dbEncryptionKeyFile, + dcrtimeHost, dcrtimeCert) + if err != nil { + return nil, fmt.Errorf("new tstore: %v", err) + } + + // Setup backend + t := tstoreBackend{ + appDir: appDir, + dataDir: dataDir, + tstore: ts, + prefixes: make(map[string][]byte), + recordMtxs: make(map[string]*sync.Mutex), + } + + err = t.setup() + if err != nil { + return nil, fmt.Errorf("setup: %v", err) + } + + return &t, nil +} diff --git a/politeiad/config.go b/politeiad/config.go index dd01ab37d..748f4c11a 100644 --- a/politeiad/config.go +++ b/politeiad/config.go @@ -18,7 +18,7 @@ import ( "strings" v1 "github.com/decred/dcrtime/api/v1" - "github.com/decred/politeia/politeiad/backend/tstorebe/tstore" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/tstore" "github.com/decred/politeia/politeiad/sharedconfig" "github.com/decred/politeia/util" "github.com/decred/politeia/util/version" @@ -44,12 +44,10 @@ const ( backendTstore = "tstore" defaultBackend = backendTstore - defaultTrillianHostUnvetted = "localhost:8090" - defaultTrillianHostVetted = "localhost:8094" - - // Database defaults - defaultDBType = tstore.DBTypeLevelDB - defaultDBHost = "127.0.0.1:3306" // MySQL default host + // Tstore default settings + defaultTrillianHost = "localhost:8090" + defaultDBType = tstore.DBTypeLevelDB + defaultDBHost = "127.0.0.1:3306" // MySQL default host ) var ( @@ -94,10 +92,9 @@ type config struct { DcrdataHost string `long:"dcrdatahost" description:"Dcrdata ip:port"` // TODO validate these config params - Backend string `long:"backend"` - TrillianHostUnvetted string `long:"trillianhostunvetted"` - TrillianHostVetted string `long:"trillianhostvetted"` - TrillianSigningKey string `long:"trilliansigningkey"` + Backend string `long:"backend"` + TrillianHost string `long:"trillianhost"` + TrillianSigningKey string `long:"trilliansigningkey"` DBType string `long:"dbtype" description:"Database type"` DBHost string `long:"dbhost" description:"Database ip:port"` @@ -247,19 +244,18 @@ func newConfigParser(cfg *config, so *serviceOptions, options flags.Options) *fl func loadConfig() (*config, []string, error) { // Default config. cfg := config{ - HomeDir: defaultHomeDir, - ConfigFile: defaultConfigFile, - DebugLevel: defaultLogLevel, - DataDir: defaultDataDir, - LogDir: defaultLogDir, - HTTPSKey: defaultHTTPSKeyFile, - HTTPSCert: defaultHTTPSCertFile, - Version: version.String(), - Backend: defaultBackend, - TrillianHostUnvetted: defaultTrillianHostUnvetted, - TrillianHostVetted: defaultTrillianHostVetted, - DBType: defaultDBType, - DBHost: defaultDBHost, + HomeDir: defaultHomeDir, + ConfigFile: defaultConfigFile, + DebugLevel: defaultLogLevel, + DataDir: defaultDataDir, + LogDir: defaultLogDir, + HTTPSKey: defaultHTTPSKeyFile, + HTTPSCert: defaultHTTPSCertFile, + Version: version.String(), + Backend: defaultBackend, + TrillianHost: defaultTrillianHost, + DBType: defaultDBType, + DBHost: defaultDBHost, } // Service options which are only added on Windows. diff --git a/politeiad/log.go b/politeiad/log.go index 2fa24492f..498760680 100644 --- a/politeiad/log.go +++ b/politeiad/log.go @@ -10,14 +10,14 @@ import ( "path/filepath" "github.com/decred/politeia/politeiad/backend/gitbe" - "github.com/decred/politeia/politeiad/backend/tstorebe" - "github.com/decred/politeia/politeiad/backend/tstorebe/plugins/comments" - "github.com/decred/politeia/politeiad/backend/tstorebe/plugins/dcrdata" - "github.com/decred/politeia/politeiad/backend/tstorebe/plugins/ticketvote" - "github.com/decred/politeia/politeiad/backend/tstorebe/plugins/usermd" - "github.com/decred/politeia/politeiad/backend/tstorebe/store/mysql" - "github.com/decred/politeia/politeiad/backend/tstorebe/tstore" - "github.com/decred/politeia/politeiawww/user/localdb" + "github.com/decred/politeia/politeiad/backendv2/tstorebe" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins/comments" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins/dcrdata" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins/ticketvote" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins/usermd" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/store/localdb" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/store/mysql" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/tstore" "github.com/decred/politeia/wsdcrdata" "github.com/decred/slog" "github.com/jrick/logrotate/rotator" diff --git a/politeiad/plugins/usermd/usermd.go b/politeiad/plugins/usermd/usermd.go index 39f77f986..509a76898 100644 --- a/politeiad/plugins/usermd/usermd.go +++ b/politeiad/plugins/usermd/usermd.go @@ -14,14 +14,14 @@ const ( CmdAuthor = "author" // Get record author CmdUserRecords = "userrecords" // Get user submitted records - // MDStreamIDUserMetadata is the politeiad metadata stream ID for - // the UserMetadata structure. - MDStreamIDUserMetadata = 1 + // StreamIDUserMetadata is the politeiad metadata stream ID for the + // UserMetadata structure. + StreamIDUserMetadata = 1 - // MDStreamIDStatusChanges is the politeiad metadata stream ID for - // the status changes metadata. Status changes should be appended - // onto this metadata stream. - MDStreamIDStatusChanges = 2 + // StreamIDStatusChanges is the politeiad metadata stream ID for + // the status changes metadata. Status changes are appended onto + // this metadata stream. + StreamIDStatusChanges = 2 ) // ErrorCodeT represents a plugin error that was caused by the user. diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index b6d788b6a..5dedd0041 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -21,14 +21,13 @@ import ( "syscall" "time" - "github.com/decred/politeia/cmsplugin" - "github.com/decred/politeia/decredplugin" + "github.com/decred/dcrd/chaincfg/v3" v1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/gitbe" - "github.com/decred/politeia/politeiad/backend/tstorebe" - "github.com/decred/politeia/politeiad/plugins/dcrdata" + "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/politeiad/backendv2/tstorebe" "github.com/decred/politeia/util" "github.com/decred/politeia/util/version" "github.com/gorilla/mux" @@ -43,10 +42,11 @@ const ( // politeia application context. type politeia struct { - backend backend.Backend - cfg *config - router *mux.Router - identity *identity.FullIdentity + backend backend.Backend + backendv2 backendv2.Backend + cfg *config + router *mux.Router + identity *identity.FullIdentity } func remoteAddr(r *http.Request) string { @@ -85,51 +85,8 @@ func convertBackendPlugins(bplugins []backend.Plugin) []v1.Plugin { // metadata stream. func convertBackendMetadataStream(mds backend.MetadataStream) v1.MetadataStream { return v1.MetadataStream{ - PluginID: mds.PluginID, - ID: mds.ID, - Payload: mds.Payload, - } -} - -func convertBackendProof(p backend.Proof) v1.Proof { - return v1.Proof{ - Type: p.Type, - Digest: p.Digest, - MerkleRoot: p.MerkleRoot, - MerklePath: p.MerklePath, - ExtraData: p.ExtraData, - } -} - -func convertBackendTimestamp(t backend.Timestamp) v1.Timestamp { - proofs := make([]v1.Proof, 0, len(t.Proofs)) - for _, v := range t.Proofs { - proofs = append(proofs, convertBackendProof(v)) - } - return v1.Timestamp{ - Data: t.Data, - Digest: t.Digest, - TxID: t.TxID, - MerkleRoot: t.MerkleRoot, - Proofs: proofs, - } -} - -func convertBackendRecordTimestamps(rt backend.RecordTimestamps) v1.RecordTimestamps { - md := make(map[string]v1.Timestamp, len(rt.Metadata)) - for k, v := range rt.Metadata { - md[k] = convertBackendTimestamp(v) - } - files := make(map[string]v1.Timestamp, len(rt.Files)) - for k, v := range rt.Files { - files[k] = convertBackendTimestamp(v) - } - return v1.RecordTimestamps{ - Token: rt.Token, - Version: rt.Version, - RecordMetadata: convertBackendTimestamp(rt.RecordMetadata), - Metadata: md, - Files: files, + ID: mds.ID, + Payload: mds.Payload, } } @@ -188,9 +145,8 @@ func convertFrontendMetadataStream(mds []v1.MetadataStream) []backend.MetadataSt m := make([]backend.MetadataStream, 0, len(mds)) for _, v := range mds { m = append(m, backend.MetadataStream{ - PluginID: v.PluginID, - ID: v.ID, - Payload: v.Payload, + ID: v.ID, + Payload: v.Payload, }) } return m @@ -260,14 +216,6 @@ func (p *politeia) respondWithUserError(w http.ResponseWriter, errorCode v1.Erro }) } -func (p *politeia) respondWithPluginError(w http.ResponseWriter, pluginID string, errorCode int, errorContext string) { - util.RespondWithJSON(w, http.StatusBadRequest, v1.PluginErrorReply{ - PluginID: pluginID, - ErrorCode: errorCode, - ErrorContext: []string{errorContext}, - }) -} - func (p *politeia) respondWithServerError(w http.ResponseWriter, errorCode int64) { log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) util.RespondWithJSON(w, http.StatusInternalServerError, v1.ServerErrorReply{ @@ -327,16 +275,6 @@ func (p *politeia) newRecord(w http.ResponseWriter, r *http.Request) { return } - // Check for plugin error - var e backend.PluginError - if errors.As(err, &e) { - log.Debugf("%v plugin user error: %v %v", - remoteAddr(r), e.PluginID, e.ErrorCode) - p.respondWithPluginError(w, e.PluginID, e.ErrorCode, - e.ErrorContext) - return - } - // Generic internal error. errorCode := time.Now().Unix() log.Errorf("%v New record error code %v: %v", remoteAddr(r), @@ -437,15 +375,6 @@ func (p *politeia) updateRecord(w http.ResponseWriter, r *http.Request, vetted b contentErr.ErrorContext) return } - // Check for plugin error - var e backend.PluginError - if errors.As(err, &e) { - log.Debugf("%v plugin user error: %v %v", - remoteAddr(r), e.PluginID, e.ErrorCode) - p.respondWithPluginError(w, e.PluginID, e.ErrorCode, - e.ErrorContext) - return - } // Generic internal error. errorCode := time.Now().Unix() log.Errorf("%v Update %v record error code %v: %v", @@ -623,16 +552,6 @@ func (p *politeia) getVetted(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, reply) } -func convertFrontendRecordRequest(r v1.RecordRequest) backend.RecordRequest { - token, _ := util.ConvertStringToken(r.Token) - return backend.RecordRequest{ - Token: token, - Version: r.Version, - Filenames: r.Filenames, - OmitAllFiles: r.OmitAllFiles, - } -} - func (p *politeia) convertBackendRecords(br map[string]backend.Record) map[string]v1.Record { r := make(map[string]v1.Record, len(br)) for k, v := range br { @@ -641,210 +560,6 @@ func (p *politeia) convertBackendRecords(br map[string]backend.Record) map[strin return r } -func (p *politeia) getUnvettedBatch(w http.ResponseWriter, r *http.Request) { - var gub v1.GetUnvettedBatch - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&gub); err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) - return - } - - challenge, err := hex.DecodeString(gub.Challenge) - if err != nil || len(challenge) != v1.ChallengeSize { - p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) - return - } - - // Setup backend requests - reqs := make([]backend.RecordRequest, 0, len(gub.Requests)) - for _, v := range gub.Requests { - // Verify token - _, err := util.ConvertStringToken(v.Token) - if err != nil { - // Not a valid token. Do not include it in the reply. - continue - } - reqs = append(reqs, convertFrontendRecordRequest(v)) - } - - // Get records batch - records, err := p.backend.GetUnvettedBatch(reqs) - if err != nil { - // Generic internal error - errorCode := time.Now().Unix() - log.Errorf("%v Get unvetted batch error code %v: %v", - remoteAddr(r), errorCode, err) - p.respondWithServerError(w, errorCode) - return - } - - log.Infof("%v Get unvetted batch %v/%v found", - remoteAddr(r), len(records), len(gub.Requests)) - - // Prepare reply - response := p.identity.SignMessage(challenge) - reply := v1.GetUnvettedBatchReply{ - Response: hex.EncodeToString(response[:]), - Records: p.convertBackendRecords(records), - } - - util.RespondWithJSON(w, http.StatusOK, reply) -} - -func (p *politeia) getVettedBatch(w http.ResponseWriter, r *http.Request) { - var gvb v1.GetVettedBatch - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&gvb); err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) - return - } - - challenge, err := hex.DecodeString(gvb.Challenge) - if err != nil || len(challenge) != v1.ChallengeSize { - p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) - return - } - - // Setup backend requests - reqs := make([]backend.RecordRequest, 0, len(gvb.Requests)) - for _, v := range gvb.Requests { - // Verify token - _, err := util.ConvertStringToken(v.Token) - if err != nil { - // Not a valid token. Do not include it in the reply. - continue - } - reqs = append(reqs, convertFrontendRecordRequest(v)) - } - - // Get records batch - records, err := p.backend.GetVettedBatch(reqs) - if err != nil { - // Generic internal error - errorCode := time.Now().Unix() - log.Errorf("%v Get vetted batch error code %v: %v", - remoteAddr(r), errorCode, err) - p.respondWithServerError(w, errorCode) - return - } - - log.Infof("%v Get vetted batch %v/%v found", - remoteAddr(r), len(records), len(gvb.Requests)) - - // Prepare reply - response := p.identity.SignMessage(challenge) - reply := v1.GetVettedBatchReply{ - Response: hex.EncodeToString(response[:]), - Records: p.convertBackendRecords(records), - } - - util.RespondWithJSON(w, http.StatusOK, reply) -} - -func (p *politeia) getUnvettedTimestamps(w http.ResponseWriter, r *http.Request) { - var t v1.GetUnvettedTimestamps - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&t); err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) - return - } - - challenge, err := hex.DecodeString(t.Challenge) - if err != nil || len(challenge) != v1.ChallengeSize { - p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) - return - } - - // Validate token - token, err := util.ConvertStringToken(t.Token) - if err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) - return - } - - // Get timestamps - rt, err := p.backend.GetUnvettedTimestamps(token, t.Version) - switch { - case errors.Is(err, backend.ErrRecordNotFound): - // Record not found - log.Infof("Get unvetted timestamps %v: %v not found", - remoteAddr(r), t.Token) - p.respondWithUserError(w, v1.ErrorStatusRecordNotFound, nil) - return - - case err != nil: - // Generic internal error - errorCode := time.Now().Unix() - log.Errorf("%v Get unvetted timestamps error code %v: %v", - remoteAddr(r), errorCode, err) - p.respondWithServerError(w, errorCode) - return - } - - log.Infof("Get unvetted timestamps %v: %v", remoteAddr(r), t.Token) - - // Setup reply - response := p.identity.SignMessage(challenge) - reply := v1.GetUnvettedTimestampsReply{ - Response: hex.EncodeToString(response[:]), - RecordTimestamps: convertBackendRecordTimestamps(*rt), - } - - util.RespondWithJSON(w, http.StatusOK, reply) -} - -func (p *politeia) getVettedTimestamps(w http.ResponseWriter, r *http.Request) { - var t v1.GetVettedTimestamps - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&t); err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) - return - } - - challenge, err := hex.DecodeString(t.Challenge) - if err != nil || len(challenge) != v1.ChallengeSize { - p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) - return - } - - // Validate token - token, err := util.ConvertStringToken(t.Token) - if err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) - return - } - - // Get timestamps - rt, err := p.backend.GetVettedTimestamps(token, t.Version) - switch { - case errors.Is(err, backend.ErrRecordNotFound): - // Record not found - log.Infof("Get vetted timestamps %v: %v not found", - remoteAddr(r), t.Token) - p.respondWithUserError(w, v1.ErrorStatusRecordNotFound, nil) - return - - case err != nil: - // Generic internal error - errorCode := time.Now().Unix() - log.Errorf("%v Get vetted timestamps error code %v: %v", - remoteAddr(r), errorCode, err) - p.respondWithServerError(w, errorCode) - return - } - - log.Infof("Get vetted timestamps %v: %v", remoteAddr(r), t.Token) - - // Setup reply - response := p.identity.SignMessage(challenge) - reply := v1.GetVettedTimestampsReply{ - Response: hex.EncodeToString(response[:]), - RecordTimestamps: convertBackendRecordTimestamps(*rt), - } - - util.RespondWithJSON(w, http.StatusOK, reply) -} - func (p *politeia) inventory(w http.ResponseWriter, r *http.Request) { var i v1.Inventory decoder := json.NewDecoder(r.Body) @@ -894,54 +609,6 @@ func (p *politeia) inventory(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, reply) } -func (p *politeia) inventoryByStatus(w http.ResponseWriter, r *http.Request) { - var ibs v1.InventoryByStatus - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&ibs); err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) - return - } - - challenge, err := hex.DecodeString(ibs.Challenge) - if err != nil || len(challenge) != v1.ChallengeSize { - p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) - return - } - - s := convertFrontendStatus(ibs.Status) - inv, err := p.backend.InventoryByStatus(ibs.State, s, - v1.InventoryPageSize, ibs.Page) - if err != nil { - // Generic internal error. - errorCode := time.Now().Unix() - log.Errorf("%v InventoryByStatus error code %v: %v", remoteAddr(r), - errorCode, err) - - p.respondWithServerError(w, errorCode) - return - } - - // Prepare reply - response := p.identity.SignMessage(challenge) - var ( - unvetted = make(map[v1.RecordStatusT][]string) - vetted = make(map[v1.RecordStatusT][]string) - ) - for status, tokens := range inv.Unvetted { - unvetted[convertBackendStatus(status)] = tokens - } - for status, tokens := range inv.Vetted { - vetted[convertBackendStatus(status)] = tokens - } - reply := v1.InventoryByStatusReply{ - Response: hex.EncodeToString(response[:]), - Unvetted: unvetted, - Vetted: vetted, - } - - util.RespondWithJSON(w, http.StatusOK, reply) -} - func (p *politeia) check(user, pass string) bool { if user != p.cfg.RPCUser || pass != p.cfg.RPCPass { return false @@ -1008,15 +675,6 @@ func (p *politeia) setVettedStatus(w http.ResponseWriter, r *http.Request) { p.respondWithUserError(w, v1.ErrorStatusInvalidRecordStatusTransition, nil) return } - // Check for plugin error - var e backend.PluginError - if errors.As(err, &e) { - log.Debugf("%v plugin user error: %v %v", - remoteAddr(r), e.PluginID, e.ErrorCode) - p.respondWithPluginError(w, e.PluginID, e.ErrorCode, - e.ErrorContext) - return - } // Generic internal error. errorCode := time.Now().Unix() log.Errorf("%v Set status error code %v: %v", @@ -1080,15 +738,6 @@ func (p *politeia) setUnvettedStatus(w http.ResponseWriter, r *http.Request) { p.respondWithUserError(w, v1.ErrorStatusInvalidRecordStatusTransition, nil) return } - // Check for plugin error - var e backend.PluginError - if errors.As(err, &e) { - log.Debugf("%v plugin user error: %v %v", - remoteAddr(r), e.PluginID, e.ErrorCode) - p.respondWithPluginError(w, e.PluginID, e.ErrorCode, - e.ErrorContext) - return - } // Generic internal error. errorCode := time.Now().Unix() log.Errorf("%v Set unvetted status error code %v: %v", @@ -1155,15 +804,6 @@ func (p *politeia) updateVettedMetadata(w http.ResponseWriter, r *http.Request) contentErr.ErrorContext) return } - // Check for plugin error - var e backend.PluginError - if errors.As(err, &e) { - log.Debugf("%v plugin user error: %v %v", - remoteAddr(r), e.PluginID, e.ErrorCode) - p.respondWithPluginError(w, e.PluginID, e.ErrorCode, - e.ErrorContext) - return - } // Generic internal error. errorCode := time.Now().Unix() log.Errorf("%v Update vetted metadata error code %v: %v", @@ -1225,15 +865,6 @@ func (p *politeia) updateUnvettedMetadata(w http.ResponseWriter, r *http.Request cverr.ErrorContext) return } - // Check for plugin error - var e backend.PluginError - if errors.As(err, &e) { - log.Debugf("%v plugin user error: %v %v", - remoteAddr(r), e.PluginID, e.ErrorCode) - p.respondWithPluginError(w, e.PluginID, e.ErrorCode, - e.ErrorContext) - return - } // Generic internal error. errorCode := time.Now().Unix() log.Errorf("%v update unvetted metadata error code %v: %v", @@ -1270,31 +901,15 @@ func (p *politeia) pluginInventory(w http.ResponseWriter, r *http.Request) { response := p.identity.SignMessage(challenge) // Get plugins - unvetted := p.backend.GetUnvettedPlugins() - vetted := p.backend.GetVettedPlugins() - - // Aggregate unique plugins - pid := make(map[string]struct{}, len(unvetted)+len(vetted)) - plugins := make([]backend.Plugin, len(unvetted)+len(vetted)) - for _, v := range unvetted { - _, ok := pid[v.ID] - if ok { - // Already added - continue - } - plugins = append(plugins, v) - pid[v.ID] = struct{}{} - } - for _, v := range vetted { - _, ok := pid[v.ID] - if ok { - // Already added - continue - } - plugins = append(plugins, v) - pid[v.ID] = struct{}{} + plugins, err := p.backend.GetPlugins() + if err != nil { + errorCode := time.Now().Unix() + log.Errorf("%v get plugins: %v ", remoteAddr(r), err) + p.respondWithServerError(w, errorCode) + return } + // Prepare reply reply := v1.PluginInventoryReply{ Plugins: convertBackendPlugins(plugins), Response: hex.EncodeToString(response[:]), @@ -1318,163 +933,27 @@ func (p *politeia) pluginCommand(w http.ResponseWriter, r *http.Request) { return } - payload, err := p.backend.Plugin(pc.ID, pc.Command, - pc.CommandID, pc.Payload) + cid, payload, err := p.backend.Plugin(pc.Command, pc.Payload) if err != nil { - // Check for a user error - var e backend.PluginError - if errors.As(err, &e) { - log.Infof("%v plugin user error: %v %v", - remoteAddr(r), e.PluginID, e.ErrorCode) - p.respondWithPluginError(w, e.PluginID, e.ErrorCode, - e.ErrorContext) - return - } - // Generic internal error. errorCode := time.Now().Unix() - log.Errorf("%v %v: backend plugin failed: pluginID:%v command:%v "+ - "payload:%v err:%v", remoteAddr(r), errorCode, pc.ID, pc.Command, - pc.Payload, err) + log.Errorf("%v %v: backend plugin failed with "+ + "command:%v payload:%v err:%v", remoteAddr(r), + errorCode, pc.Command, pc.Payload, err) p.respondWithServerError(w, errorCode) return } + // Prepare reply response := p.identity.SignMessage(challenge) reply := v1.PluginCommandReply{ Response: hex.EncodeToString(response[:]), ID: pc.ID, - Command: pc.Command, + Command: cid, CommandID: pc.CommandID, Payload: payload, } - log.Infof("%v Plugin cmd executed: %v %v", remoteAddr(r), pc.ID, pc.Command) - - util.RespondWithJSON(w, http.StatusOK, reply) -} - -func (p *politeia) pluginCommandBatch(w http.ResponseWriter, r *http.Request) { - var pcb v1.PluginCommandBatch - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&pcb); err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, - nil) - return - } - - // Verify challenge - challenge, err := hex.DecodeString(pcb.Challenge) - if err != nil || len(challenge) != v1.ChallengeSize { - p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) - return - } - - // Execute plugin commands - replies := make([]v1.PluginCommandReplyV2, len(pcb.Commands)) - for k, pc := range pcb.Commands { - // Verify token - token, err := util.ConvertStringToken(pc.Token) - if err != nil { - replies[k] = v1.PluginCommandReplyV2{ - UserError: &v1.UserErrorReply{ - ErrorCode: v1.ErrorStatusInvalidToken, - }, - } - continue - } - - // Execute plugin command - var payload string - switch pc.State { - case v1.RecordStateUnvetted: - payload, err = p.backend.UnvettedPluginCmd(pc.Action, token, - pc.ID, pc.Command, pc.Payload) - case v1.RecordStateVetted: - payload, err = p.backend.VettedPluginCmd(pc.Action, token, - pc.ID, pc.Command, pc.Payload) - default: - replies[k] = v1.PluginCommandReplyV2{ - UserError: &v1.UserErrorReply{ - ErrorCode: v1.ErrorStatusInvalidRecordState, - }, - } - continue - } - if err != nil { - var e backend.PluginError - switch { - case errors.As(err, &e): - log.Infof("%v batched plugin cmd user error: %v %v", - remoteAddr(r), e.PluginID, e.ErrorCode) - - replies[k] = v1.PluginCommandReplyV2{ - PluginError: &v1.PluginErrorReply{ - PluginID: e.PluginID, - ErrorCode: e.ErrorCode, - ErrorContext: []string{e.ErrorContext}, - }, - } - case err == backend.ErrPluginActionInvalid: - replies[k] = v1.PluginCommandReplyV2{ - UserError: &v1.UserErrorReply{ - ErrorCode: v1.ErrorStatusInvalidPluginAction, - }, - } - case err == backend.ErrRecordNotFound: - replies[k] = v1.PluginCommandReplyV2{ - UserError: &v1.UserErrorReply{ - ErrorCode: v1.ErrorStatusRecordNotFound, - }, - } - case err == backend.ErrRecordLocked: - replies[k] = v1.PluginCommandReplyV2{ - UserError: &v1.UserErrorReply{ - ErrorCode: v1.ErrorStatusRecordLocked, - }, - } - default: - // Unknown error. Log is as an internal server error and - // respond with a server error. - t := time.Now().Unix() - log.Errorf("%v %v: batched plugin cmd failed: pluginID:%v "+ - "cmd:%v err:'%v' payload:%v", remoteAddr(r), t, pc.ID, - pc.Command, err, pc.Payload) - - p.respondWithServerError(w, t) - return - } - - continue - } - - // Update reply - replies[k] = v1.PluginCommandReplyV2{ - Payload: payload, - } - } - - // Fill in remaining data for the replies - for k, v := range replies { - replies[k] = v1.PluginCommandReplyV2{ - State: pcb.Commands[k].State, - Token: pcb.Commands[k].Token, - ID: pcb.Commands[k].ID, - Command: pcb.Commands[k].Command, - Payload: v.Payload, - UserError: v.UserError, - PluginError: v.PluginError, - } - } - - response := p.identity.SignMessage(challenge) - reply := v1.PluginCommandBatchReply{ - Response: hex.EncodeToString(response[:]), - Replies: replies, - } - - log.Infof("%v Plugin cmd batch executed", remoteAddr(r)) - util.RespondWithJSON(w, http.StatusOK, reply) } @@ -1513,6 +992,129 @@ func (p *politeia) addRoute(method string, route string, handler http.HandlerFun p.router.StrictSlash(true).HandleFunc(route, handler).Methods(method) } +func (p *politeia) setupBackendGit(anp *chaincfg.Params) error { + b, err := gitbe.New(activeNetParams.Params, p.cfg.DataDir, + p.cfg.DcrtimeHost, "", p.identity, p.cfg.GitTrace, p.cfg.DcrdataHost) + if err != nil { + return fmt.Errorf("new gitbe: %v", err) + } + p.backend = b + + // Setup mux + p.router = mux.NewRouter() + + // Not found + p.router.NotFoundHandler = closeBody(p.handleNotFound) + + // Unprivileged routes + p.addRoute(http.MethodPost, v1.IdentityRoute, p.getIdentity, + permissionPublic) + p.addRoute(http.MethodPost, v1.NewRecordRoute, p.newRecord, + permissionPublic) + p.addRoute(http.MethodPost, v1.UpdateUnvettedRoute, p.updateUnvetted, + permissionPublic) + p.addRoute(http.MethodPost, v1.UpdateVettedRoute, p.updateVetted, + permissionPublic) + p.addRoute(http.MethodPost, v1.GetUnvettedRoute, p.getUnvetted, + permissionPublic) + p.addRoute(http.MethodPost, v1.GetVettedRoute, p.getVetted, + permissionPublic) + + // Routes that require auth + p.addRoute(http.MethodPost, v1.InventoryRoute, p.inventory, + permissionAuth) + p.addRoute(http.MethodPost, v1.SetUnvettedStatusRoute, + p.setUnvettedStatus, permissionAuth) + p.addRoute(http.MethodPost, v1.SetVettedStatusRoute, + p.setVettedStatus, permissionAuth) + p.addRoute(http.MethodPost, v1.UpdateVettedMetadataRoute, + p.updateVettedMetadata, permissionAuth) + p.addRoute(http.MethodPost, v1.UpdateUnvettedMetadataRoute, + p.updateUnvettedMetadata, permissionAuth) + + // Set plugin routes. Requires auth. + p.addRoute(http.MethodPost, v1.PluginCommandRoute, p.pluginCommand, + permissionAuth) + p.addRoute(http.MethodPost, v1.PluginInventoryRoute, p.pluginInventory, + permissionAuth) + + return nil +} + +func (p *politeia) setupBackendTstore(anp *chaincfg.Params) error { + b, err := tstorebe.New(p.cfg.HomeDir, p.cfg.DataDir, anp, + p.cfg.TrillianHost, p.cfg.TrillianSigningKey, + p.cfg.DBType, p.cfg.DBHost, p.cfg.DBPass, p.cfg.DBEncryptionKey, + p.cfg.DcrtimeHost, p.cfg.DcrtimeCert) + if err != nil { + return fmt.Errorf("new tstorebe: %v", err) + } + p.backendv2 = b + + // Setup routes + + // Setup plugins + if len(p.cfg.Plugins) > 0 { + // Parse plugin settings + settings := make(map[string][]backendv2.PluginSetting) + for _, v := range p.cfg.PluginSettings { + // Plugin setting will be in format: pluginID,key,value + s := strings.Split(v, ",") + if len(s) != 3 { + return fmt.Errorf("failed to parse plugin setting '%v'; format "+ + "should be 'pluginID,key,value'", s) + } + var ( + pluginID = s[0] + key = s[1] + value = s[2] + ) + ps, ok := settings[pluginID] + if !ok { + ps = make([]backendv2.PluginSetting, 0, 16) + } + ps = append(ps, backendv2.PluginSetting{ + Key: key, + Value: value, + }) + + settings[pluginID] = ps + } + + // Register plugins + for _, v := range p.cfg.Plugins { + // Setup plugin + ps, ok := settings[v] + if !ok { + ps = make([]backendv2.PluginSetting, 0) + } + plugin := backendv2.Plugin{ + ID: v, + Settings: ps, + Identity: p.identity, + } + + // Register plugin + log.Infof("Register plugin: %v", v) + err = p.backendv2.PluginRegister(plugin) + if err != nil { + return fmt.Errorf("PluginRegister %v: %v", v, err) + } + } + + // Setup plugins + for _, v := range p.backendv2.PluginInventory() { + log.Infof("Setup plugin: %v", v.ID) + err = p.backendv2.PluginSetup(v.ID) + if err != nil { + return fmt.Errorf("plugin setup %v: %v", v.ID, err) + } + } + } + + return nil +} + func _main() error { // Load configuration and parse command line. This function also // initializes logging and configures it accordingly. @@ -1527,7 +1129,7 @@ func _main() error { }() log.Infof("Version : %v", version.String()) - log.Infof("Build Version: %v", version.BuildMainVersion()) + log.Infof("Build : %v", version.BuildMainVersion()) log.Infof("Network : %v", activeNetParams.Params.Name) log.Infof("Home dir: %v", cfg.HomeDir) @@ -1601,177 +1203,19 @@ func _main() error { log.Infof("Backend: %v", cfg.Backend) switch cfg.Backend { case backendGit: - b, err := gitbe.New(activeNetParams.Params, cfg.DataDir, cfg.DcrtimeHost, - "", p.identity, cfg.GitTrace, cfg.DcrdataHost) + err := p.setupBackendGit(activeNetParams.Params) if err != nil { - return fmt.Errorf("new gitbe: %v", err) + return err } - p.backend = b case backendTstore: - b, err := tstorebe.New(activeNetParams.Params, cfg.HomeDir, cfg.DataDir, - cfg.TrillianHostUnvetted, cfg.TrillianHostVetted, cfg.TrillianSigningKey, - cfg.DBType, cfg.DBHost, cfg.DBPass, cfg.DBEncryptionKey, - cfg.DcrtimeHost, cfg.DcrtimeCert) + err := p.setupBackendTstore(activeNetParams.Params) if err != nil { - return fmt.Errorf("new tstorebe: %v", err) + return err } - p.backend = b default: return fmt.Errorf("invalid backend selected: %v", cfg.Backend) } - // Setup mux - p.router = mux.NewRouter() - - // Not found - p.router.NotFoundHandler = closeBody(p.handleNotFound) - - // Unprivileged routes - p.addRoute(http.MethodPost, v1.IdentityRoute, p.getIdentity, - permissionPublic) - p.addRoute(http.MethodPost, v1.NewRecordRoute, p.newRecord, - permissionPublic) - p.addRoute(http.MethodPost, v1.UpdateUnvettedRoute, p.updateUnvetted, - permissionPublic) - p.addRoute(http.MethodPost, v1.UpdateVettedRoute, p.updateVetted, - permissionPublic) - p.addRoute(http.MethodPost, v1.GetUnvettedRoute, p.getUnvetted, - permissionPublic) - p.addRoute(http.MethodPost, v1.GetVettedRoute, p.getVetted, - permissionPublic) - p.addRoute(http.MethodPost, v1.GetUnvettedBatchRoute, - p.getUnvettedBatch, permissionPublic) - p.addRoute(http.MethodPost, v1.GetVettedBatchRoute, - p.getVettedBatch, permissionPublic) - p.addRoute(http.MethodPost, v1.GetUnvettedTimestampsRoute, - p.getUnvettedTimestamps, permissionPublic) - p.addRoute(http.MethodPost, v1.GetVettedTimestampsRoute, - p.getVettedTimestamps, permissionPublic) - p.addRoute(http.MethodPost, v1.InventoryByStatusRoute, - p.inventoryByStatus, permissionPublic) - - // Routes that require auth - p.addRoute(http.MethodPost, v1.InventoryRoute, p.inventory, - permissionAuth) - p.addRoute(http.MethodPost, v1.SetUnvettedStatusRoute, - p.setUnvettedStatus, permissionAuth) - p.addRoute(http.MethodPost, v1.SetVettedStatusRoute, - p.setVettedStatus, permissionAuth) - p.addRoute(http.MethodPost, v1.UpdateVettedMetadataRoute, - p.updateVettedMetadata, permissionAuth) - p.addRoute(http.MethodPost, v1.UpdateUnvettedMetadataRoute, - p.updateUnvettedMetadata, permissionAuth) - - // Setup plugins - if len(cfg.Plugins) > 0 { - // Set plugin routes. Requires auth. - p.addRoute(http.MethodPost, v1.PluginCommandRoute, p.pluginCommand, - permissionAuth) - p.addRoute(http.MethodPost, v1.PluginCommandBatchRoute, - p.pluginCommandBatch, permissionAuth) - p.addRoute(http.MethodPost, v1.PluginInventoryRoute, p.pluginInventory, - permissionAuth) - - // Parse plugin settings - // map[pluginID][]backend.PluginSetting - settings := make(map[string][]backend.PluginSetting) - for _, v := range cfg.PluginSettings { - // Plugin setting will be in format: pluginID,key,value - s := strings.Split(v, ",") - if len(s) != 3 { - return fmt.Errorf("failed to parse plugin setting '%v'; format "+ - "should be 'pluginID,key,value'", s) - } - var ( - pluginID = s[0] - key = s[1] - value = s[2] - ) - ps, ok := settings[pluginID] - if !ok { - ps = make([]backend.PluginSetting, 0, 16) - } - ps = append(ps, backend.PluginSetting{ - Key: key, - Value: value, - }) - - settings[pluginID] = ps - } - - // Register plugins - for _, v := range cfg.Plugins { - // Verify plugin ID format - if backend.PluginRE.FindString(v) != v { - return fmt.Errorf("invalid plugin id format: %v %v", - v, backend.PluginRE.String()) - } - - // Get plugin settings - ps, ok := settings[v] - if !ok { - ps = make([]backend.PluginSetting, 0) - } - - // Prepare plugin - var ( - unvetted = true // Register as unvetted plugin - vetted = true // Register as vetted plugin - plugin = backend.Plugin{ - ID: v, - Settings: ps, - Identity: p.identity, - } - ) - switch v { - case dcrdata.PluginID: - unvetted = false - case decredplugin.ID: - // decredplugin uses the deprecated plugin methods. This - // plugin is also deprecated will eventually be removed. - unvetted = false - vetted = false - case cmsplugin.ID: - // cmsplugin uses the deprecated plugin methods. This - // plugin is also deprecated will eventually be removed. - unvetted = false - vetted = false - } - - // Register plugin - if unvetted { - log.Infof("Register unvetted plugin: %v", v) - err = p.backend.RegisterUnvettedPlugin(plugin) - if err != nil { - return fmt.Errorf("register unvetted plugin %v: %v", v, err) - } - } - if vetted { - log.Infof("Register vetted plugin: %v", v) - err = p.backend.RegisterVettedPlugin(plugin) - if err != nil { - return fmt.Errorf("register vetted plugin %v: %v", v, err) - } - } - } - - // Setup plugins - for _, v := range p.backend.GetUnvettedPlugins() { - log.Infof("Setup unvetted plugin: %v", v.ID) - err = p.backend.SetupUnvettedPlugin(v.ID) - if err != nil { - return fmt.Errorf("setup unvetted plugin %v: %v", v, err) - } - } - for _, v := range p.backend.GetVettedPlugins() { - log.Infof("Setup vetted plugin: %v", v.ID) - err = p.backend.SetupVettedPlugin(v.ID) - if err != nil { - return fmt.Errorf("setup vetted plugin %v: %v", v.ID, err) - } - } - } - // Bind to a port and pass our router in listenC := make(chan error) for _, listener := range cfg.Listeners { diff --git a/politeiad/v2.go b/politeiad/v2.go new file mode 100644 index 000000000..d493878b4 --- /dev/null +++ b/politeiad/v2.go @@ -0,0 +1,699 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. +package main + +import ( + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/http" + "runtime/debug" + "time" + + v2 "github.com/decred/politeia/politeiad/api/v2" + "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/util" +) + +func (p *politeia) handleRecordNew(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleRecordNew") + + // Decode request + var rn v2.RecordNew + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&rn); err != nil { + respondWithErrorV2(w, r, "handleRecordNew: unmarshal", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeRequestPayloadInvalid, + }) + return + } + challenge, err := hex.DecodeString(rn.Challenge) + if err != nil || len(challenge) != v2.ChallengeSize { + respondWithErrorV2(w, r, "handleRecordNew: decode challenge", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeChallengeInvalid, + }) + return + } + + // Create new record + var ( + metadata = convertMetadataStreamsToBackend(rn.Metadata) + files = convertFilesToBackend(rn.Files) + ) + rc, err := p.backendv2.RecordNew(metadata, files) + if err != nil { + respondWithErrorV2(w, r, + "handleRecordNew: RecordNew: %v", err) + return + } + + // Prepare reply + response := p.identity.SignMessage(challenge) + rnr := v2.RecordNewReply{ + Response: hex.EncodeToString(response[:]), + Record: p.convertRecordToV2(*rc), + } + + util.RespondWithJSON(w, http.StatusOK, rnr) +} + +func (p *politeia) handleRecordEdit(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleRecordEdit") + + // Decode request + var re v2.RecordEdit + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&re); err != nil { + respondWithErrorV2(w, r, "handleRecordEdit: unmarshal", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeRequestPayloadInvalid, + }) + return + } + challenge, err := hex.DecodeString(re.Challenge) + if err != nil || len(challenge) != v2.ChallengeSize { + respondWithErrorV2(w, r, "handleRecordEdit: decode challenge", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeChallengeInvalid, + }) + return + } + token, err := decodeToken(re.Token) + if err != nil { + respondWithErrorV2(w, r, "handleRecordEdit: decode token", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeTokenInvalid, + }) + return + } + + // Edit record + var ( + mdAppend = convertMetadataStreamsToBackend(re.MDAppend) + mdOverwrite = convertMetadataStreamsToBackend(re.MDOverwrite) + filesAdd = convertFilesToBackend(re.FilesAdd) + ) + rc, err := p.backendv2.RecordEdit(token, mdAppend, + mdOverwrite, filesAdd, re.FilesDel) + if err != nil { + respondWithErrorV2(w, r, + "handleRecordEdit: RecordEdit: %v", err) + return + } + + // Prepare reply + response := p.identity.SignMessage(challenge) + rer := v2.RecordEditReply{ + Response: hex.EncodeToString(response[:]), + Record: p.convertRecordToV2(*rc), + } + + util.RespondWithJSON(w, http.StatusOK, rer) +} + +func (p *politeia) handleRecordEditMetadata(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleRecordEditMetadata") + + // Decode request + var re v2.RecordEditMetadata + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&re); err != nil { + respondWithErrorV2(w, r, "handleRecordEditMetadata: unmarshal", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeRequestPayloadInvalid, + }) + return + } + challenge, err := hex.DecodeString(re.Challenge) + if err != nil || len(challenge) != v2.ChallengeSize { + respondWithErrorV2(w, r, "handleRecordEditMetadata: decode challenge", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeChallengeInvalid, + }) + return + } + token, err := decodeToken(re.Token) + if err != nil { + respondWithErrorV2(w, r, "handleRecordEditMetadata: decode token", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeTokenInvalid, + }) + return + } + + // Edit record metadata + var ( + mdAppend = convertMetadataStreamsToBackend(re.MDAppend) + mdOverwrite = convertMetadataStreamsToBackend(re.MDOverwrite) + ) + rc, err := p.backendv2.RecordEditMetadata(token, mdAppend, mdOverwrite) + if err != nil { + respondWithErrorV2(w, r, + "handleRecordEditMetadata: RecordEditMetadata: %v", err) + return + } + + // Prepare reply + response := p.identity.SignMessage(challenge) + rer := v2.RecordEditMetadataReply{ + Response: hex.EncodeToString(response[:]), + Record: p.convertRecordToV2(*rc), + } + + util.RespondWithJSON(w, http.StatusOK, rer) +} + +func (p *politeia) handleRecordSetStatus(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleRecordSetStatus") + + // Decode request + var rss v2.RecordSetStatus + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&rss); err != nil { + respondWithErrorV2(w, r, "handleRecordSetStatus: unmarshal", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeRequestPayloadInvalid, + }) + return + } + challenge, err := hex.DecodeString(rss.Challenge) + if err != nil || len(challenge) != v2.ChallengeSize { + respondWithErrorV2(w, r, "handleRecordSetStatus: decode challenge", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeChallengeInvalid, + }) + return + } + token, err := decodeToken(rss.Token) + if err != nil { + respondWithErrorV2(w, r, "handleRecordSetStatus: decode token", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeTokenInvalid, + }) + return + } + + // Set record status + var ( + mdAppend = convertMetadataStreamsToBackend(rss.MDAppend) + mdOverwrite = convertMetadataStreamsToBackend(rss.MDOverwrite) + status = backendv2.StatusT(rss.Status) + ) + rc, err := p.backendv2.RecordSetStatus(token, status, + mdAppend, mdOverwrite) + if err != nil { + respondWithErrorV2(w, r, + "handleRecordSetStatus: RecordSetStatus: %v", err) + return + } + + // Prepare reply + response := p.identity.SignMessage(challenge) + rer := v2.RecordSetStatusReply{ + Response: hex.EncodeToString(response[:]), + Record: p.convertRecordToV2(*rc), + } + + util.RespondWithJSON(w, http.StatusOK, rer) +} + +func (p *politeia) handleRecordGet(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleRecordGet") + + // Decode request + var rg v2.RecordGet + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&rg); err != nil { + respondWithErrorV2(w, r, "handleRecordGet: unmarshal", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeRequestPayloadInvalid, + }) + return + } + challenge, err := hex.DecodeString(rg.Challenge) + if err != nil || len(challenge) != v2.ChallengeSize { + respondWithErrorV2(w, r, "handleRecordGet: decode challenge", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeChallengeInvalid, + }) + return + } + token, err := decodeTokenAnyLength(rg.Token) + if err != nil { + respondWithErrorV2(w, r, "handleRecordGet: decode token", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeTokenInvalid, + }) + return + } + + // Get record + rc, err := p.backendv2.RecordGet(token, rg.Version) + if err != nil { + respondWithErrorV2(w, r, + "handleRecordGet: RecordGet: %v", err) + return + } + + // Prepare reply + response := p.identity.SignMessage(challenge) + rgr := v2.RecordGetReply{ + Response: hex.EncodeToString(response[:]), + Record: p.convertRecordToV2(*rc), + } + + util.RespondWithJSON(w, http.StatusOK, rgr) +} + +func (p *politeia) handleRecordGetBatch(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleRecordGetBatch") + + // Decode request + var rgb v2.RecordGetBatch + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&rgb); err != nil { + respondWithErrorV2(w, r, "handleRecordGetBatch: unmarshal", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeRequestPayloadInvalid, + }) + return + } + challenge, err := hex.DecodeString(rgb.Challenge) + if err != nil || len(challenge) != v2.ChallengeSize { + respondWithErrorV2(w, r, "handleRecordGetBatch: decode challenge", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeChallengeInvalid, + }) + return + } + + // Get record batch + reqs := convertRecordRequestsToBackend(rgb.Requests) + brecords, err := p.backendv2.RecordGetBatch(reqs) + if err != nil { + respondWithErrorV2(w, r, + "handleRecordGet: RecordGetBatch: %v", err) + return + } + + // Prepare reply + records := make(map[string]v2.Record, len(brecords)) + for k, v := range brecords { + records[k] = p.convertRecordToV2(v) + } + response := p.identity.SignMessage(challenge) + reply := v2.RecordGetBatchReply{ + Response: hex.EncodeToString(response[:]), + Records: records, + } + + util.RespondWithJSON(w, http.StatusOK, reply) +} + +func (p *politeia) handleRecordTimestamps(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleRecordTimestamps") + + // Decode request + var rgt v2.RecordGetTimestamps + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&rgt); err != nil { + respondWithErrorV2(w, r, "handleRecordTimestamps: unmarshal", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeRequestPayloadInvalid, + }) + return + } + challenge, err := hex.DecodeString(rgt.Challenge) + if err != nil || len(challenge) != v2.ChallengeSize { + respondWithErrorV2(w, r, "handleRecordTimestamps: decode challenge", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeChallengeInvalid, + }) + return + } + token, err := decodeTokenAnyLength(rgt.Token) + if err != nil { + respondWithErrorV2(w, r, "handleRecordTimestamps: decode token", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeTokenInvalid, + }) + return + } + + // Get record timestamps + rt, err := p.backendv2.RecordTimestamps(token, rgt.Version) + if err != nil { + respondWithErrorV2(w, r, + "handleRecordTimestamps: RecordTimestamps: %v", err) + return + } + + // Prepare reply + response := p.identity.SignMessage(challenge) + rtr := v2.RecordTimestampsReply{ + Response: hex.EncodeToString(response[:]), + Timestamps: convertRecordTimestampsToV2(*rt), + } + + util.RespondWithJSON(w, http.StatusOK, rtr) +} + +func (p *politeia) handleInventory(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleInventory") + + // Decode request + var i v2.Inventory + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&i); err != nil { + respondWithErrorV2(w, r, "handleInventory: unmarshal", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeRequestPayloadInvalid, + }) + return + } + challenge, err := hex.DecodeString(i.Challenge) + if err != nil || len(challenge) != v2.ChallengeSize { + respondWithErrorV2(w, r, "handleInventory: decode challenge", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeChallengeInvalid, + }) + return + } + + // Get inventory + var ( + state = backendv2.StateT(i.State) + status = backendv2.StatusT(i.Status) + pageSize = v2.InventoryPageSize + pageNumber = i.Page + ) + inv, err := p.backendv2.Inventory(state, status, pageSize, pageNumber) + if err != nil { + respondWithErrorV2(w, r, + "handleInventory: Inventory: %v", err) + return + } + + // Prepare reply + unvetted := make(map[string][]string, len(inv.Unvetted)) + for k, v := range inv.Unvetted { + key := backendv2.Statuses[k] + unvetted[key] = v + } + vetted := make(map[string][]string, len(inv.Vetted)) + for k, v := range inv.Vetted { + key := backendv2.Statuses[k] + vetted[key] = v + } + response := p.identity.SignMessage(challenge) + ir := v2.InventoryReply{ + Response: hex.EncodeToString(response[:]), + Unvetted: unvetted, + Vetted: vetted, + } + + util.RespondWithJSON(w, http.StatusOK, ir) +} + +// decodeToken decodes a v2 token and errors if the token is not the full +// length token. +func decodeToken(token string) ([]byte, error) { + return util.TokenDecode(util.TokenTypeTstore, token) +} + +// decodeTokenAnyLength decodes a v2 token. It accepts both the full length +// token and the short token. +func decodeTokenAnyLength(token string) ([]byte, error) { + return util.TokenDecodeAnyLength(util.TokenTypeTstore, token) +} + +func (p *politeia) convertRecordToV2(r backendv2.Record) v2.Record { + var ( + metadata = convertMetadataStreamsToV2(r.Metadata) + files = convertFilesToV2(r.Files) + rm = r.RecordMetadata + sig = p.identity.SignMessage([]byte(rm.Merkle + rm.Token)) + ) + return v2.Record{ + State: v2.RecordStateT(rm.State), + Status: v2.RecordStatusT(rm.Status), + Version: rm.Version, + Timestamp: rm.Timestamp, + Metadata: metadata, + Files: files, + CensorshipRecord: v2.CensorshipRecord{ + Token: rm.Token, + Merkle: rm.Merkle, + Signature: hex.EncodeToString(sig[:]), + }, + } +} + +func convertMetadataStreamsToBackend(metadata []v2.MetadataStream) []backendv2.MetadataStream { + ms := make([]backendv2.MetadataStream, 0, len(metadata)) + for _, v := range ms { + ms = append(ms, backendv2.MetadataStream{ + PluginID: v.PluginID, + StreamID: v.StreamID, + Payload: v.Payload, + }) + } + return ms +} + +func convertMetadataStreamsToV2(metadata []backendv2.MetadataStream) []v2.MetadataStream { + ms := make([]v2.MetadataStream, 0, len(metadata)) + for _, v := range ms { + ms = append(ms, v2.MetadataStream{ + PluginID: v.PluginID, + StreamID: v.StreamID, + Payload: v.Payload, + }) + } + return ms +} + +func convertFilesToBackend(files []v2.File) []backendv2.File { + fs := make([]backendv2.File, 0, len(files)) + for _, v := range files { + fs = append(fs, backendv2.File{ + Name: v.Name, + MIME: v.MIME, + Digest: v.Digest, + Payload: v.Payload, + }) + } + return fs +} + +func convertFilesToV2(files []backendv2.File) []v2.File { + fs := make([]v2.File, 0, len(files)) + for _, v := range files { + fs = append(fs, v2.File{ + Name: v.Name, + MIME: v.MIME, + Digest: v.Digest, + Payload: v.Payload, + }) + } + return fs +} + +func convertRecordRequestsToBackend(reqs []v2.RecordRequest) []backendv2.RecordRequest { + r := make([]backendv2.RecordRequest, 0, len(reqs)) + for _, v := range reqs { + token, err := decodeTokenAnyLength(v.Token) + if err != nil { + // Records with errors will not be included in the reply + log.Debugf("convertRecordRequestsToBackend: decode token: %v", err) + continue + } + r = append(r, backendv2.RecordRequest{ + Token: token, + Version: v.Version, + Filenames: v.Filenames, + OmitAllFiles: v.OmitAllFiles, + }) + } + return r +} + +func convertProofToV2(p backendv2.Proof) v2.Proof { + return v2.Proof{ + Type: p.Type, + Digest: p.Digest, + MerkleRoot: p.MerkleRoot, + MerklePath: p.MerklePath, + ExtraData: p.ExtraData, + } +} + +func convertTimestampToV2(t backendv2.Timestamp) v2.Timestamp { + proofs := make([]v2.Proof, 0, len(t.Proofs)) + for _, v := range t.Proofs { + proofs = append(proofs, convertProofToV2(v)) + } + return v2.Timestamp{ + Data: t.Data, + Digest: t.Digest, + TxID: t.TxID, + MerkleRoot: t.MerkleRoot, + Proofs: proofs, + } +} + +func convertRecordTimestampsToV2(r backendv2.RecordTimestamps) v2.RecordTimestamps { + metadata := make(map[string]map[uint32]v2.Timestamp, 16) + for pluginID, v := range r.Metadata { + timestamps, ok := metadata[pluginID] + if !ok { + timestamps = make(map[uint32]v2.Timestamp, 16) + } + for streamID, ts := range v { + timestamps[streamID] = convertTimestampToV2(ts) + } + metadata[pluginID] = timestamps + } + files := make(map[string]v2.Timestamp, len(r.Files)) + for k, v := range r.Files { + files[k] = convertTimestampToV2(v) + } + return v2.RecordTimestamps{ + RecordMetadata: convertTimestampToV2(r.RecordMetadata), + Metadata: metadata, + Files: files, + } +} + +func respondWithErrorV2(w http.ResponseWriter, r *http.Request, format string, err error) { + var ( + errCode = convertErrorToV2(err) + ue v2.UserErrorReply + ce backendv2.ContentError + ste backendv2.StatusTransitionError + pe backendv2.PluginError + ) + switch { + case errCode != v2.ErrorCodeInvalid: + // Backend error + log.Infof("%v User error: %v %v", util.RemoteAddr(r), + errCode, v2.ErrorCodes[errCode]) + util.RespondWithJSON(w, http.StatusBadRequest, + v2.UserErrorReply{ + ErrorCode: errCode, + }) + return + + case errors.As(err, &ue): + // Politeiad user error + m := fmt.Sprintf("%v User error: %v %v", util.RemoteAddr(r), + ue.ErrorCode, v2.ErrorCodes[ue.ErrorCode]) + if ce.ErrorContext != "" { + m += fmt.Sprintf(": %v", ce.ErrorContext) + } + log.Infof(m) + util.RespondWithJSON(w, http.StatusBadRequest, ue) + return + + case errors.As(err, &ce): + // Backend content error + errCode := convertContentErrorToV2(ce.ErrorCode) + m := fmt.Sprintf("%v User error: %v %v", util.RemoteAddr(r), + errCode, v2.ErrorCodes[errCode]) + if ce.ErrorContext != "" { + m += fmt.Sprintf(": %v", ce.ErrorContext) + } + log.Infof(m) + util.RespondWithJSON(w, http.StatusBadRequest, + v2.UserErrorReply{ + ErrorCode: errCode, + ErrorContext: ce.ErrorContext, + }) + return + + case errors.As(err, &ste): + // Backend status transition error + log.Infof("%v User error: %v", util.RemoteAddr(r), ste.Error()) + util.RespondWithJSON(w, http.StatusBadRequest, + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeStatusChangeInvalid, + ErrorContext: ste.Error(), + }) + return + + case errors.As(err, &pe): + // Plugin user error + m := fmt.Sprintf("%v Plugin error: %v %v", + util.RemoteAddr(r), pe.PluginID, pe.ErrorCode) + if pe.ErrorContext != "" { + m += fmt.Sprintf(": %v", pe.ErrorContext) + } + log.Infof(m) + util.RespondWithJSON(w, http.StatusBadRequest, + v2.PluginErrorReply{ + PluginID: pe.PluginID, + ErrorCode: pe.ErrorCode, + ErrorContext: pe.ErrorContext, + }) + return + } + + // Internal server error. Log it and return a 500. + t := time.Now().Unix() + e := fmt.Sprintf(format, err) + log.Errorf("%v %v %v %v Internal error %v: %v", + util.RemoteAddr(r), r.Method, r.URL, r.Proto, t, e) + log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) + + util.RespondWithJSON(w, http.StatusInternalServerError, + v2.ServerErrorReply{ + ErrorCode: t, + }) + return +} + +func convertErrorToV2(e error) v2.ErrorCodeT { + switch e { + case backendv2.ErrTokenInvalid: + return v2.ErrorCodeTokenInvalid + case backendv2.ErrRecordNotFound: + return v2.ErrorCodeRecordNotFound + case backendv2.ErrRecordLocked: + return v2.ErrorCodeRecordLocked + case backendv2.ErrNoRecordChanges: + return v2.ErrorCodeNoRecordChanges + case backendv2.ErrPluginIDInvalid: + return v2.ErrorCodePluginIDInvalid + case backendv2.ErrPluginCmdInvalid: + return v2.ErrorCodePluginCmdInvalid + } + return v2.ErrorCodeInvalid +} + +func convertContentErrorToV2(e backendv2.ContentErrorCodeT) v2.ErrorCodeT { + switch e { + case backendv2.ContentErrorMetadataStreamInvalid: + return v2.ErrorCodeMetadataStreamInvalid + case backendv2.ContentErrorMetadataStreamDuplicate: + return v2.ErrorCodeMetadataStreamDuplicate + case backendv2.ContentErrorFilesEmpty: + return v2.ErrorCodeFilesEmpty + case backendv2.ContentErrorFileNameInvalid: + return v2.ErrorCodeFileNameInvalid + case backendv2.ContentErrorFileNameDuplicate: + return v2.ErrorCodeFileNameDuplicate + case backendv2.ContentErrorFileDigestInvalid: + return v2.ErrorCodeFileDigestInvalid + case backendv2.ContentErrorFilePayloadInvalid: + return v2.ErrorCodeFilePayloadInvalid + case backendv2.ContentErrorFileMIMETypeInvalid: + return v2.ErrorCodeFileMIMETypeInvalid + case backendv2.ContentErrorFileMIMETypeUnsupported: + return v2.ErrorCodeFileMIMETypeUnsupported + } + return v2.ErrorCodeInvalid +} diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go index 46b5dd7b2..9aaf0705e 100644 --- a/politeiawww/api/records/v1/v1.go +++ b/politeiawww/api/records/v1/v1.go @@ -167,9 +167,9 @@ type File struct { // MetadataStream describes a record metadata stream. type MetadataStream struct { - PluginID string `json:"pluginid"` // Plugin ID - ID uint64 `json:"id"` // Metadata stream ID - Payload string `json:"payload"` // JSON encoded + PluginID string `json:"pluginid"` + StreamID uint64 `json:"streamid"` + Payload string `json:"payload"` // JSON encoded } // CensorshipRecord contains cryptographic proof that a record was accepted for @@ -179,8 +179,7 @@ type CensorshipRecord struct { // server. It serves as a unique identifier for the record. Token string `json:"token"` - // Merkle is the ordered merkle root of all files and metadata in - // in the record. + // Merkle is the ordered merkle root of all files in the record. Merkle string `json:"merkle"` // Signature is the server signature of the Merkle+Token. @@ -405,7 +404,7 @@ type Timestamps struct { type TimestampsReply struct { RecordMetadata Timestamp `json:"recordmetadata"` - // map[pluginID+metadataID]Timestamp + // map[pluginID+streamID]Timestamp Metadata map[string]Timestamp `json:"metadata"` // map[filename]Timestamp diff --git a/politeiawww/cmd/pictl/proposal.go b/politeiawww/cmd/pictl/proposal.go index 7d1f1992d..244d3aa4d 100644 --- a/politeiawww/cmd/pictl/proposal.go +++ b/politeiawww/cmd/pictl/proposal.go @@ -198,9 +198,6 @@ func convertRecord(r rcv1.Record) (*piv1.Proposal, error) { }) } - // Some fields are intentionally omitted because they are user data - // that is not saved to politeiad and needs to be pulled from the - // user database. return &piv1.Proposal{ Version: r.Version, Timestamp: r.Timestamp, diff --git a/politeiawww/log.go b/politeiawww/log.go index 190b5eb5a..dbe7ae16c 100644 --- a/politeiawww/log.go +++ b/politeiawww/log.go @@ -57,7 +57,7 @@ var ( userdbLog = backendLog.Logger("USER") sessionsLog = backendLog.Logger("SESS") eventsLog = backendLog.Logger("EVNT") - apiLog = backendLog.Logger("PAPI") + apiLog = backendLog.Logger("APIS") // CMS loggers cmsdbLog = backendLog.Logger("CMDB") @@ -95,7 +95,7 @@ var subsystemLoggers = map[string]slog.Logger{ "SESS": sessionsLog, "EVNT": eventsLog, "USER": userdbLog, - "PAPI": apiLog, + "APIS": apiLog, "CMDB": cmsdbLog, "WSDD": wsdcrdataLog, "GHTR": githubTrackerLog, diff --git a/util/convert.go b/util/convert.go index 09254bd04..a5592e568 100644 --- a/util/convert.go +++ b/util/convert.go @@ -10,7 +10,8 @@ import ( "fmt" dcrtime "github.com/decred/dcrtime/api/v1" - pd "github.com/decred/politeia/politeiad/api/v1" + pdv1 "github.com/decred/politeia/politeiad/api/v1" + pdv2 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/identity" ) @@ -35,11 +36,11 @@ func ConvertSignature(s string) ([identity.SignatureSize]byte, error) { // prefixes. func ConvertStringToken(token string) ([]byte, error) { switch { - case len(token) == pd.TokenSizeTstore*2: + case len(token) == pdv2.TokenSize*2: // Tstore backend token; continue - case len(token) != pd.TokenSizeGit*2: + case len(token) != pdv1.TokenSize*2: // Git backend token; continue - case len(token) == pd.TokenPrefixLength: + case len(token) == pdv1.TokenPrefixLength: // Token prefix; continue default: return nil, fmt.Errorf("invalid token size") @@ -99,8 +100,8 @@ func Zero(in []byte) { // TokenToPrefix returns a substring a token of length pd.TokenPrefixLength, // or the token itself, whichever is shorter. func TokenToPrefix(token string) string { - if len(token) > pd.TokenPrefixLength { - return token[0:pd.TokenPrefixLength] + if len(token) > pdv1.TokenPrefixLength { + return token[0:pdv1.TokenPrefixLength] } else { return token } diff --git a/util/token.go b/util/token.go index 0831bab70..0dcc71bf0 100644 --- a/util/token.go +++ b/util/token.go @@ -9,6 +9,7 @@ import ( "fmt" pdv1 "github.com/decred/politeia/politeiad/api/v1" + pdv2 "github.com/decred/politeia/politeiad/api/v2" ) var ( @@ -21,9 +22,9 @@ var ( func TokenIsFullLength(tokenType string, token []byte) bool { switch tokenType { case TokenTypeTstore: - return len(token) == pdv1.TokenSizeTstore + return len(token) == pdv2.TokenSize case TokenTypeGit: - return len(token) == pdv1.TokenSizeGit + return len(token) == pdv1.TokenSize default: panic("invalid token type") } From 7d2385c544f76d07068c364973f69171aaf19672 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 7 Mar 2021 20:04:39 -0600 Subject: [PATCH 364/449] politeiad/client: Add v2 methods. --- politeiad/api/v2/v2.go | 8 +- politeiad/client/client.go | 4 +- politeiad/client/comments.go | 26 +- politeiad/client/{politeiad.go => pdv1.go} | 315 +---------------- politeiad/client/pdv2.go | 393 +++++++++++++++++++++ politeiad/client/ticketvote.go | 11 +- politeiad/client/user.go | 11 +- politeiad/politeiad.go | 34 +- politeiad/v2.go | 21 +- 9 files changed, 473 insertions(+), 350 deletions(-) rename politeiad/client/{politeiad.go => pdv1.go} (51%) create mode 100644 politeiad/client/pdv2.go diff --git a/politeiad/api/v2/v2.go b/politeiad/api/v2/v2.go index 5b1f024c0..5d2402931 100644 --- a/politeiad/api/v2/v2.go +++ b/politeiad/api/v2/v2.go @@ -7,8 +7,8 @@ package v2 import "fmt" const ( - // RoutePrefix is prefixed onto all routes in this package. - RoutePrefix = "/v2" + // APIRoute is prefixed onto all routes in this package. + APIRoute = "/v2" // Routes RouteRecordNew = "/recordnew" @@ -401,8 +401,8 @@ type RecordGetTimestamps struct { Version uint32 `json:"version"` // Record version } -// RecordTimestampsReply is the reply ot the RecordTimestamps command. -type RecordTimestampsReply struct { +// RecordGetTimestampsReply is the reply ot the RecordTimestamps command. +type RecordGetTimestampsReply struct { Response string `json:"response"` // Challenge response Timestamps RecordTimestamps `json:"timestamps"` } diff --git a/politeiad/client/client.go b/politeiad/client/client.go index 98f520299..ecfff9eb8 100644 --- a/politeiad/client/client.go +++ b/politeiad/client/client.go @@ -55,7 +55,7 @@ func (e Error) Error() string { // serializing the provided object as the request body, and returning a byte // slice of the response body. An Error is returned if politeiad responds with // anything other than a 200 http status code. -func (c *Client) makeReq(ctx context.Context, method string, route string, v interface{}) ([]byte, error) { +func (c *Client) makeReq(ctx context.Context, method, api, route string, v interface{}) ([]byte, error) { // Serialize body var ( reqBody []byte @@ -69,7 +69,7 @@ func (c *Client) makeReq(ctx context.Context, method string, route string, v int } // Send request - fullRoute := c.rpcHost + route + fullRoute := c.rpcHost + api + route req, err := http.NewRequestWithContext(ctx, method, fullRoute, bytes.NewReader(reqBody)) if err != nil { diff --git a/politeiad/client/comments.go b/politeiad/client/comments.go index 243b1011e..5e6e87711 100644 --- a/politeiad/client/comments.go +++ b/politeiad/client/comments.go @@ -4,16 +4,7 @@ package client -import ( - "context" - "encoding/json" - "fmt" - - "github.com/davecgh/go-spew/spew" - pdv1 "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/politeiad/plugins/comments" -) - +/* // CommentNew sends the comments plugin New command to the politeiad v1 API. func (c *Client) CommentNew(ctx context.Context, state string, n comments.New) (*comments.NewReply, error) { // Setup request @@ -21,15 +12,11 @@ func (c *Client) CommentNew(ctx context.Context, state string, n comments.New) ( if err != nil { return nil, err } - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionWrite, - State: state, - Token: n.Token, - ID: comments.PluginID, - Command: comments.CmdNew, - Payload: string(b), - }, + cmd := pdv2.PluginCmd{ + Token: n.Token, + ID: comments.PluginID, + Command: comments.CmdNew, + Payload: string(b), } // Send request @@ -353,3 +340,4 @@ func (c *Client) CommentTimestamps(ctx context.Context, state, token string, t c return &tr, nil } +*/ diff --git a/politeiad/client/politeiad.go b/politeiad/client/pdv1.go similarity index 51% rename from politeiad/client/politeiad.go rename to politeiad/client/pdv1.go index cf47ac7c6..dad364abd 100644 --- a/politeiad/client/politeiad.go +++ b/politeiad/client/pdv1.go @@ -28,7 +28,8 @@ func (c *Client) NewRecord(ctx context.Context, metadata []pdv1.MetadataStream, } // Send request - resBody, err := c.makeReq(ctx, http.MethodPost, pdv1.NewRecordRoute, nr) + resBody, err := c.makeReq(ctx, http.MethodPost, "", + pdv1.NewRecordRoute, nr) if err != nil { return nil, err } @@ -62,7 +63,7 @@ func (c *Client) updateRecord(ctx context.Context, route, token string, mdAppend } // Send request - resBody, err := c.makeReq(ctx, http.MethodPost, route, ur) + resBody, err := c.makeReq(ctx, http.MethodPost, "", route, ur) if err != nil { return nil, err } @@ -110,7 +111,7 @@ func (c *Client) UpdateUnvettedMetadata(ctx context.Context, token string, mdApp } // Send request - resBody, err := c.makeReq(ctx, http.MethodPost, + resBody, err := c.makeReq(ctx, http.MethodPost, "", pdv1.UpdateUnvettedMetadataRoute, uum) if err != nil { return nil @@ -146,7 +147,7 @@ func (c *Client) UpdateVettedMetadata(ctx context.Context, token string, mdAppen } // Send request - resBody, err := c.makeReq(ctx, http.MethodPost, + resBody, err := c.makeReq(ctx, http.MethodPost, "", pdv1.UpdateVettedMetadataRoute, uvm) if err != nil { return nil @@ -183,7 +184,7 @@ func (c *Client) SetUnvettedStatus(ctx context.Context, token string, status pdv } // Send request - resBody, err := c.makeReq(ctx, http.MethodPost, + resBody, err := c.makeReq(ctx, http.MethodPost, "", pdv1.SetUnvettedStatusRoute, sus) if err != nil { return nil, err @@ -219,7 +220,7 @@ func (c *Client) SetVettedStatus(ctx context.Context, token string, status pdv1. } // Send request - resBody, err := c.makeReq(ctx, http.MethodPost, + resBody, err := c.makeReq(ctx, http.MethodPost, "", pdv1.SetVettedStatusRoute, svs) if err != nil { return nil, err @@ -253,7 +254,8 @@ func (c *Client) GetUnvetted(ctx context.Context, token, version string) (*pdv1. } // Send request - resBody, err := c.makeReq(ctx, http.MethodPost, pdv1.GetUnvettedRoute, gu) + resBody, err := c.makeReq(ctx, http.MethodPost, "", + pdv1.GetUnvettedRoute, gu) if err != nil { return nil, err } @@ -286,7 +288,7 @@ func (c *Client) GetVetted(ctx context.Context, token, version string) (*pdv1.Re } // Send request - resBody, err := c.makeReq(ctx, http.MethodPost, + resBody, err := c.makeReq(ctx, http.MethodPost, "", pdv1.GetVettedRoute, gv) if err != nil { return nil, err @@ -306,209 +308,6 @@ func (c *Client) GetVetted(ctx context.Context, token, version string) (*pdv1.Re return &gvr.Record, nil } -// GetUnvettedBatch sends a GetUnvettedBatch request to the politeiad v1 API. -func (c *Client) GetUnvettedBatch(ctx context.Context, reqs []pdv1.RecordRequest) (map[string]pdv1.Record, error) { - // Setup request - challenge, err := util.Random(pdv1.ChallengeSize) - if err != nil { - return nil, err - } - gub := pdv1.GetUnvettedBatch{ - Challenge: hex.EncodeToString(challenge), - Requests: reqs, - } - - // Send request - resBody, err := c.makeReq(ctx, http.MethodPost, - pdv1.GetUnvettedBatchRoute, gub) - if err != nil { - return nil, err - } - - // Decode reply - var r pdv1.GetUnvettedBatchReply - err = json.Unmarshal(resBody, &r) - if err != nil { - return nil, err - } - err = util.VerifyChallenge(c.pid, challenge, r.Response) - if err != nil { - return nil, err - } - - return r.Records, nil -} - -// GetVettedBatch sends a GetVettedBatch request to the politeiad v1 API. -func (c *Client) GetVettedBatch(ctx context.Context, reqs []pdv1.RecordRequest) (map[string]pdv1.Record, error) { - // Setup request - challenge, err := util.Random(pdv1.ChallengeSize) - if err != nil { - return nil, err - } - gvb := pdv1.GetVettedBatch{ - Challenge: hex.EncodeToString(challenge), - Requests: reqs, - } - - // Send request - resBody, err := c.makeReq(ctx, http.MethodPost, - pdv1.GetVettedBatchRoute, gvb) - if err != nil { - return nil, err - } - - // Decode reply - var r pdv1.GetVettedBatchReply - err = json.Unmarshal(resBody, &r) - if err != nil { - return nil, err - } - err = util.VerifyChallenge(c.pid, challenge, r.Response) - if err != nil { - return nil, err - } - - return r.Records, nil -} - -// GetUnvettedTimestamps sends a GetUnvettedTimestamps request to the politeiad -// v1 API. -func (c *Client) GetUnvettedTimestamps(ctx context.Context, token, version string) (*pdv1.RecordTimestamps, error) { - // Setup request - challenge, err := util.Random(pdv1.ChallengeSize) - if err != nil { - return nil, err - } - gut := pdv1.GetUnvettedTimestamps{ - Challenge: hex.EncodeToString(challenge), - Token: token, - Version: version, - } - - // Send request - resBody, err := c.makeReq(ctx, http.MethodPost, - pdv1.GetUnvettedTimestampsRoute, gut) - if err != nil { - return nil, err - } - - // Decode reply - var reply pdv1.GetUnvettedTimestampsReply - err = json.Unmarshal(resBody, &reply) - if err != nil { - return nil, err - } - err = util.VerifyChallenge(c.pid, challenge, reply.Response) - if err != nil { - return nil, err - } - - return &reply.RecordTimestamps, nil -} - -// GetVettedTimestamps sends a GetVettedTimestamps request to the politeiad -// v1 API. -func (c *Client) GetVettedTimestamps(ctx context.Context, token, version string) (*pdv1.RecordTimestamps, error) { - // Setup request - challenge, err := util.Random(pdv1.ChallengeSize) - if err != nil { - return nil, err - } - gvt := pdv1.GetVettedTimestamps{ - Challenge: hex.EncodeToString(challenge), - Token: token, - Version: version, - } - - // Send request - resBody, err := c.makeReq(ctx, http.MethodPost, - pdv1.GetVettedTimestampsRoute, gvt) - if err != nil { - return nil, err - } - - // Decode reply - var reply pdv1.GetVettedTimestampsReply - err = json.Unmarshal(resBody, &reply) - if err != nil { - return nil, err - } - err = util.VerifyChallenge(c.pid, challenge, reply.Response) - if err != nil { - return nil, err - } - - return &reply.RecordTimestamps, nil -} - -// Inventory sends a Inventory request to the politeiad v1 API. -func (c *Client) Inventory(ctx context.Context) (*pdv1.InventoryReply, error) { - // Setup request - challenge, err := util.Random(pdv1.ChallengeSize) - if err != nil { - return nil, err - } - ibs := pdv1.Inventory{ - Challenge: hex.EncodeToString(challenge), - } - - // Send request - resBody, err := c.makeReq(ctx, http.MethodPost, - pdv1.InventoryRoute, ibs) - if err != nil { - return nil, err - } - - // Decode reply - var ir pdv1.InventoryReply - err = json.Unmarshal(resBody, &ir) - if err != nil { - return nil, err - } - err = util.VerifyChallenge(c.pid, challenge, ir.Response) - if err != nil { - return nil, err - } - - return &ir, nil -} - -// InventoryByStatus sends a InventoryByStatus request to the politeiad v1 API. -func (c *Client) InventoryByStatus(ctx context.Context, state string, status pdv1.RecordStatusT, page uint32) (*pdv1.InventoryByStatusReply, error) { - // Setup request - challenge, err := util.Random(pdv1.ChallengeSize) - if err != nil { - return nil, err - } - ibs := pdv1.InventoryByStatus{ - Challenge: hex.EncodeToString(challenge), - State: state, - Status: status, - Page: page, - } - - // Send request - resBody, err := c.makeReq(ctx, http.MethodPost, - pdv1.InventoryByStatusRoute, ibs) - if err != nil { - return nil, err - } - - // Decode reply - var ibsr pdv1.InventoryByStatusReply - err = json.Unmarshal(resBody, &ibsr) - if err != nil { - return nil, err - } - err = util.VerifyChallenge(c.pid, challenge, ibsr.Response) - if err != nil { - return nil, err - } - - return &ibsr, nil -} - // PluginCommand sends a PluginCommand request to the politeiad v1 API. func (c *Client) PluginCommand(ctx context.Context, pluginID, cmd, payload string) (string, error) { // Setup request @@ -520,12 +319,12 @@ func (c *Client) PluginCommand(ctx context.Context, pluginID, cmd, payload strin Challenge: hex.EncodeToString(challenge), ID: pluginID, Command: cmd, - CommandID: "", + CommandID: cmd, Payload: payload, } // Send request - resBody, err := c.makeReq(ctx, http.MethodPost, + resBody, err := c.makeReq(ctx, http.MethodPost, "", pdv1.PluginCommandRoute, pc) if err != nil { return "", err @@ -544,93 +343,3 @@ func (c *Client) PluginCommand(ctx context.Context, pluginID, cmd, payload strin return pcr.Payload, nil } - -// PluginCommandBatch sends a PluginCommandBatch request to the politeiad v1 API. -func (c *Client) PluginCommandBatch(ctx context.Context, cmds []pdv1.PluginCommandV2) ([]pdv1.PluginCommandReplyV2, error) { - // Setup request - challenge, err := util.Random(pdv1.ChallengeSize) - if err != nil { - return nil, err - } - pcb := pdv1.PluginCommandBatch{ - Challenge: hex.EncodeToString(challenge), - Commands: cmds, - } - - // Send request - resBody, err := c.makeReq(ctx, http.MethodPost, - pdv1.PluginCommandBatchRoute, pcb) - if err != nil { - return nil, err - } - - // Decode reply - var pcbr pdv1.PluginCommandBatchReply - err = json.Unmarshal(resBody, &pcbr) - if err != nil { - return nil, err - } - err = util.VerifyChallenge(c.pid, challenge, pcbr.Response) - if err != nil { - return nil, err - } - - return pcbr.Replies, nil -} - -// PluginInventory sends a PluginInventory request to the politeiad v1 API. -func (c *Client) PluginInventory(ctx context.Context) ([]pdv1.Plugin, error) { - // Setup request - challenge, err := util.Random(pdv1.ChallengeSize) - if err != nil { - return nil, err - } - pi := pdv1.PluginInventory{ - Challenge: hex.EncodeToString(challenge), - } - - // Send request - resBody, err := c.makeReq(ctx, http.MethodPost, - pdv1.PluginInventoryRoute, pi) - if err != nil { - return nil, err - } - - // Decode reply - var pir pdv1.PluginInventoryReply - err = json.Unmarshal(resBody, &pir) - if err != nil { - return nil, err - } - err = util.VerifyChallenge(c.pid, challenge, pir.Response) - if err != nil { - return nil, err - } - - return pir.Plugins, nil -} - -// extractPluginCommandError extracts the error from a plugin command reply if -// one exists and converts it to a politeiad client Error. -func extractPluginCommandError(pcr pdv1.PluginCommandReplyV2) error { - switch { - case pcr.UserError != nil: - return Error{ - HTTPCode: http.StatusBadRequest, - ErrorReply: ErrorReply{ - ErrorCode: int(pcr.UserError.ErrorCode), - ErrorContext: pcr.UserError.ErrorContext, - }, - } - case pcr.PluginError != nil: - return Error{ - HTTPCode: http.StatusBadRequest, - ErrorReply: ErrorReply{ - PluginID: pcr.PluginError.PluginID, - ErrorCode: pcr.PluginError.ErrorCode, - ErrorContext: pcr.PluginError.ErrorContext, - }, - } - } - return nil -} diff --git a/politeiad/client/pdv2.go b/politeiad/client/pdv2.go new file mode 100644 index 000000000..5582b2c2e --- /dev/null +++ b/politeiad/client/pdv2.go @@ -0,0 +1,393 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package client + +import ( + "context" + "encoding/hex" + "encoding/json" + "net/http" + + pdv2 "github.com/decred/politeia/politeiad/api/v2" + "github.com/decred/politeia/util" +) + +// RecordNew sends a RecordNew command to the politeiad v2 API. +func (c *Client) RecordNew(ctx context.Context, metadata []pdv2.MetadataStream, files []pdv2.File) (*pdv2.Record, error) { + // Setup request + challenge, err := util.Random(pdv2.ChallengeSize) + if err != nil { + return nil, err + } + rn := pdv2.RecordNew{ + Challenge: hex.EncodeToString(challenge), + Metadata: metadata, + Files: files, + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, + pdv2.APIRoute, pdv2.RouteRecordNew, rn) + if err != nil { + return nil, err + } + + // Decode reply + var rnr pdv2.RecordNewReply + err = json.Unmarshal(resBody, &rnr) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(c.pid, challenge, rnr.Response) + if err != nil { + return nil, err + } + + return &rnr.Record, nil +} + +// RecordEdit sends a RecordEdit command to the politeiad v2 API. +func (c *Client) RecordEdit(ctx context.Context, token string, mdAppend, mdOverwrite []pdv2.MetadataStream, filesAdd []pdv2.File, filesDel []string) (*pdv2.Record, error) { + // Setup request + challenge, err := util.Random(pdv2.ChallengeSize) + if err != nil { + return nil, err + } + re := pdv2.RecordEdit{ + Challenge: hex.EncodeToString(challenge), + Token: token, + MDAppend: mdAppend, + MDOverwrite: mdOverwrite, + FilesAdd: filesAdd, + FilesDel: filesDel, + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, + pdv2.APIRoute, pdv2.RouteRecordEdit, re) + if err != nil { + return nil, err + } + + // Decode reply + var rer pdv2.RecordEditReply + err = json.Unmarshal(resBody, &rer) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(c.pid, challenge, rer.Response) + if err != nil { + return nil, err + } + + return &rer.Record, nil +} + +// RecordEditMetadata sends a RecordEditMetadata command to the politeiad v2 +// API. +func (c *Client) RecordEditMetadata(ctx context.Context, token string, mdAppend, mdOverwrite []pdv2.MetadataStream) (*pdv2.Record, error) { + // Setup request + challenge, err := util.Random(pdv2.ChallengeSize) + if err != nil { + return nil, err + } + rem := pdv2.RecordEditMetadata{ + Challenge: hex.EncodeToString(challenge), + Token: token, + MDAppend: mdAppend, + MDOverwrite: mdOverwrite, + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, + pdv2.APIRoute, pdv2.RouteRecordEditMetadata, rem) + if err != nil { + return nil, err + } + + // Decode reply + var reply pdv2.RecordEditMetadataReply + err = json.Unmarshal(resBody, &reply) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(c.pid, challenge, reply.Response) + if err != nil { + return nil, err + } + + return &reply.Record, nil +} + +// RecordSetStatus sends a RecordSetStatus command to the politeiad v2 API. +func (c *Client) RecordSetStatus(ctx context.Context, token string, status pdv2.RecordStatusT, mdAppend, mdOverwrite []pdv2.MetadataStream) (*pdv2.Record, error) { + // Setup request + challenge, err := util.Random(pdv2.ChallengeSize) + if err != nil { + return nil, err + } + rss := pdv2.RecordSetStatus{ + Challenge: hex.EncodeToString(challenge), + Token: token, + Status: status, + MDAppend: mdAppend, + MDOverwrite: mdOverwrite, + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, + pdv2.APIRoute, pdv2.RouteRecordSetStatus, rss) + if err != nil { + return nil, err + } + + // Decode reply + var reply pdv2.RecordSetStatusReply + err = json.Unmarshal(resBody, &reply) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(c.pid, challenge, reply.Response) + if err != nil { + return nil, err + } + + return &reply.Record, nil +} + +// RecordGet sends a RecordGet command to the politeiad v2 API. +func (c *Client) RecordGet(ctx context.Context, token string, version uint32) (*pdv2.Record, error) { + // Setup request + challenge, err := util.Random(pdv2.ChallengeSize) + if err != nil { + return nil, err + } + rg := pdv2.RecordGet{ + Challenge: hex.EncodeToString(challenge), + Token: token, + Version: version, + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, + pdv2.APIRoute, pdv2.RouteRecordGet, rg) + if err != nil { + return nil, err + } + + // Decode reply + var rgr pdv2.RecordGetReply + err = json.Unmarshal(resBody, &rgr) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(c.pid, challenge, rgr.Response) + if err != nil { + return nil, err + } + + return &rgr.Record, nil +} + +// RecordGetBatch sends a RecordGetBatch command to the politeiad v2 API. +func (c *Client) RecordGetBatch(ctx context.Context, reqs []pdv2.RecordRequest) (map[string]pdv2.Record, error) { + // Setup request + challenge, err := util.Random(pdv2.ChallengeSize) + if err != nil { + return nil, err + } + rgb := pdv2.RecordGetBatch{ + Challenge: hex.EncodeToString(challenge), + Requests: reqs, + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, + pdv2.APIRoute, pdv2.RouteRecordGetBatch, rgb) + if err != nil { + return nil, err + } + + // Decode reply + var reply pdv2.RecordGetBatchReply + err = json.Unmarshal(resBody, &reply) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(c.pid, challenge, reply.Response) + if err != nil { + return nil, err + } + + return reply.Records, nil +} + +// RecordGetTimestamps sends a RecordGetTimestamps command to the politeiad v2 +// API. +func (c *Client) RecordGetTimestamps(ctx context.Context, token string, version uint32) (*pdv2.RecordTimestamps, error) { + // Setup request + challenge, err := util.Random(pdv2.ChallengeSize) + if err != nil { + return nil, err + } + rgt := pdv2.RecordGetTimestamps{ + Challenge: hex.EncodeToString(challenge), + Token: token, + Version: version, + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, + pdv2.APIRoute, pdv2.RouteRecordGetTimestamps, rgt) + if err != nil { + return nil, err + } + + // Decode reply + var reply pdv2.RecordGetTimestampsReply + err = json.Unmarshal(resBody, &reply) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(c.pid, challenge, reply.Response) + if err != nil { + return nil, err + } + + return &reply.Timestamps, nil +} + +// Inventory sends a Inventory command to the politeiad v2 API. +func (c *Client) Inventory(ctx context.Context, state pdv2.RecordStateT, status pdv2.RecordStatusT, page uint32) (*pdv2.InventoryReply, error) { + // Setup request + challenge, err := util.Random(pdv2.ChallengeSize) + if err != nil { + return nil, err + } + i := pdv2.Inventory{ + Challenge: hex.EncodeToString(challenge), + State: state, + Status: status, + Page: page, + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, + pdv2.APIRoute, pdv2.RouteInventory, i) + if err != nil { + return nil, err + } + + // Decode reply + var ir pdv2.InventoryReply + err = json.Unmarshal(resBody, &ir) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(c.pid, challenge, ir.Response) + if err != nil { + return nil, err + } + + return &ir, nil +} + +// PluginWrite sends a PluginWrite command to the politeiad v2 API. +func (c *Client) PluginWrite(ctx context.Context, cmd pdv2.PluginCmd) (string, error) { + // Setup request + challenge, err := util.Random(pdv2.ChallengeSize) + if err != nil { + return "", err + } + pw := pdv2.PluginWrite{ + Challenge: hex.EncodeToString(challenge), + Cmd: cmd, + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, + pdv2.APIRoute, pdv2.RoutePluginWrite, pw) + if err != nil { + return "", err + } + + // Decode reply + var pwr pdv2.PluginWriteReply + err = json.Unmarshal(resBody, &pwr) + if err != nil { + return "", err + } + err = util.VerifyChallenge(c.pid, challenge, pwr.Response) + if err != nil { + return "", err + } + + return pwr.Payload, nil +} + +// PluginReads sends a PluginReads command to the politeiad v2 API. +func (c *Client) PluginReads(ctx context.Context, cmds []pdv2.PluginCmd) ([]pdv2.PluginReadReply, error) { + // Setup request + challenge, err := util.Random(pdv2.ChallengeSize) + if err != nil { + return nil, err + } + pr := pdv2.PluginReads{ + Challenge: hex.EncodeToString(challenge), + Cmds: cmds, + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, + pdv2.APIRoute, pdv2.RoutePluginReads, pr) + if err != nil { + return nil, err + } + + // Decode reply + var prr pdv2.PluginReadsReply + err = json.Unmarshal(resBody, &prr) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(c.pid, challenge, prr.Response) + if err != nil { + return nil, err + } + + return prr.Replies, nil +} + +// PluginInventory sends a PluginInventory command to the politeiad v2 API. +func (c *Client) PluginInventory(ctx context.Context) ([]pdv2.Plugin, error) { + // Setup request + challenge, err := util.Random(pdv2.ChallengeSize) + if err != nil { + return nil, err + } + pi := pdv2.PluginInventory{ + Challenge: hex.EncodeToString(challenge), + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, + pdv2.APIRoute, pdv2.RoutePluginInventory, pi) + if err != nil { + return nil, err + } + + // Decode reply + var pir pdv2.PluginInventoryReply + err = json.Unmarshal(resBody, &pir) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(c.pid, challenge, pir.Response) + if err != nil { + return nil, err + } + + return pir.Plugins, nil +} diff --git a/politeiad/client/ticketvote.go b/politeiad/client/ticketvote.go index aafc02c73..0d43ef161 100644 --- a/politeiad/client/ticketvote.go +++ b/politeiad/client/ticketvote.go @@ -4,15 +4,7 @@ package client -import ( - "context" - "encoding/json" - "fmt" - - pdv1 "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/politeiad/plugins/ticketvote" -) - +/* // TicketVoteAuthorize sends the ticketvote plugin Authorize command to the // politeiad v1 API. func (c *Client) TicketVoteAuthorize(ctx context.Context, a ticketvote.Authorize) (*ticketvote.AuthorizeReply, error) { @@ -425,3 +417,4 @@ func (c *Client) TicketVoteTimestamps(ctx context.Context, t ticketvote.Timestam return &sr, nil } +*/ diff --git a/politeiad/client/user.go b/politeiad/client/user.go index 73d3f8176..20fa74f3e 100644 --- a/politeiad/client/user.go +++ b/politeiad/client/user.go @@ -4,15 +4,7 @@ package client -import ( - "context" - "encoding/json" - "fmt" - - pdv1 "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/politeiad/plugins/usermd" -) - +/* // Author sends the user plugin Author command to the politeiad v1 API. func (c *Client) Author(ctx context.Context, state, token string) (string, error) { // Setup request @@ -107,3 +99,4 @@ func (c *Client) UserRecords(ctx context.Context, userID string) (map[string][]s return reply, nil } +*/ diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 5dedd0041..6e9f14f85 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -24,6 +24,7 @@ import ( "github.com/decred/dcrd/chaincfg/v3" v1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/identity" + v2 "github.com/decred/politeia/politeiad/api/v2" "github.com/decred/politeia/politeiad/backend" "github.com/decred/politeia/politeiad/backend/gitbe" "github.com/decred/politeia/politeiad/backendv2" @@ -992,6 +993,11 @@ func (p *politeia) addRoute(method string, route string, handler http.HandlerFun p.router.StrictSlash(true).HandleFunc(route, handler).Methods(method) } +func (p *politeia) addRouteV2(method string, route string, handler http.HandlerFunc, perm permission) { + route = v2.APIRoute + route + p.addRoute(method, route, handler, perm) +} + func (p *politeia) setupBackendGit(anp *chaincfg.Params) error { b, err := gitbe.New(activeNetParams.Params, p.cfg.DataDir, p.cfg.DcrtimeHost, "", p.identity, p.cfg.GitTrace, p.cfg.DcrdataHost) @@ -1051,7 +1057,33 @@ func (p *politeia) setupBackendTstore(anp *chaincfg.Params) error { } p.backendv2 = b - // Setup routes + // Setup v1 routes + p.addRoute(http.MethodPost, v1.IdentityRoute, + p.getIdentity, permissionPublic) + + // Setup v2 routes + p.addRouteV2(http.MethodPost, v2.RouteRecordNew, + p.handleRecordNew, permissionPublic) + p.addRouteV2(http.MethodPost, v2.RouteRecordEdit, + p.handleRecordEdit, permissionPublic) + p.addRouteV2(http.MethodPost, v2.RouteRecordEditMetadata, + p.handleRecordEditMetadata, permissionPublic) + p.addRouteV2(http.MethodPost, v2.RouteRecordSetStatus, + p.handleRecordSetStatus, permissionPublic) + p.addRouteV2(http.MethodPost, v2.RouteRecordGet, + p.handleRecordGet, permissionPublic) + p.addRouteV2(http.MethodPost, v2.RouteRecordGetBatch, + p.handleRecordGetBatch, permissionPublic) + p.addRouteV2(http.MethodPost, v2.RouteRecordGetTimestamps, + p.handleRecordGetTimestamps, permissionPublic) + p.addRouteV2(http.MethodPost, v2.RouteInventory, + p.handleInventory, permissionPublic) + p.addRouteV2(http.MethodPost, v2.RoutePluginWrite, + p.handlePluginWrite, permissionPublic) + p.addRouteV2(http.MethodPost, v2.RoutePluginReads, + p.handlePluginReads, permissionPublic) + p.addRouteV2(http.MethodPost, v2.RoutePluginInventory, + p.handlePluginInventory, permissionPublic) // Setup plugins if len(p.cfg.Plugins) > 0 { diff --git a/politeiad/v2.go b/politeiad/v2.go index d493878b4..f39906c61 100644 --- a/politeiad/v2.go +++ b/politeiad/v2.go @@ -314,8 +314,8 @@ func (p *politeia) handleRecordGetBatch(w http.ResponseWriter, r *http.Request) util.RespondWithJSON(w, http.StatusOK, reply) } -func (p *politeia) handleRecordTimestamps(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleRecordTimestamps") +func (p *politeia) handleRecordGetTimestamps(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleRecordGetTimestamps") // Decode request var rgt v2.RecordGetTimestamps @@ -354,7 +354,7 @@ func (p *politeia) handleRecordTimestamps(w http.ResponseWriter, r *http.Request // Prepare reply response := p.identity.SignMessage(challenge) - rtr := v2.RecordTimestampsReply{ + rtr := v2.RecordGetTimestampsReply{ Response: hex.EncodeToString(response[:]), Timestamps: convertRecordTimestampsToV2(*rt), } @@ -419,6 +419,21 @@ func (p *politeia) handleInventory(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, ir) } +func (p *politeia) handlePluginWrite(w http.ResponseWriter, r *http.Request) { + log.Tracef("handlePluginWrite") + panic("not implemented") +} + +func (p *politeia) handlePluginReads(w http.ResponseWriter, r *http.Request) { + log.Tracef("handlePluginReads") + panic("not implemented") +} + +func (p *politeia) handlePluginInventory(w http.ResponseWriter, r *http.Request) { + log.Tracef("handlePluginInventory") + panic("not implemented") +} + // decodeToken decodes a v2 token and errors if the token is not the full // length token. func decodeToken(token string) ([]byte, error) { From baf12c2377c9cd9900653ef8784bdd83dff7c23c Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 8 Mar 2021 08:21:47 -0600 Subject: [PATCH 365/449] politeiawww: Update to pd v2 API. --- politeiad/api/v2/v2.go | 2 +- politeiad/client/comments.go | 618 ++++++++++--------- politeiad/client/ticketvote.go | 752 ++++++++++++----------- politeiad/client/user.go | 173 +++--- politeiad/plugins/usermd/usermd.go | 9 +- politeiad/testpoliteiad/testpoliteiad.go | 2 +- politeiawww/api/pi/v1/v1.go | 229 +------ politeiawww/api/records/v1/v1.go | 133 ++-- politeiawww/comments/comments.go | 4 +- politeiawww/comments/error.go | 12 +- politeiawww/comments/process.go | 4 +- politeiawww/dcc.go | 14 - politeiawww/invoices.go | 2 +- politeiawww/pi/error.go | 93 --- politeiawww/pi/events.go | 96 ++- politeiawww/pi/mail.go | 4 +- politeiawww/pi/pi.go | 37 +- politeiawww/pi/process.go | 249 -------- politeiawww/piwww.go | 7 +- politeiawww/politeiawww.go | 2 - politeiawww/proposals.go | 81 ++- politeiawww/records/error.go | 64 +- politeiawww/records/events.go | 2 - politeiawww/records/process.go | 455 +++++--------- politeiawww/testing.go | 5 +- politeiawww/ticketvote/error.go | 10 +- politeiawww/ticketvote/process.go | 3 +- politeiawww/ticketvote/ticketvote.go | 4 +- politeiawww/www.go | 19 +- 29 files changed, 1212 insertions(+), 1873 deletions(-) delete mode 100644 politeiawww/pi/error.go delete mode 100644 politeiawww/pi/process.go diff --git a/politeiad/api/v2/v2.go b/politeiad/api/v2/v2.go index 5d2402931..2181edb97 100644 --- a/politeiad/api/v2/v2.go +++ b/politeiad/api/v2/v2.go @@ -327,7 +327,7 @@ type RecordGetReply struct { // Version is used to request a specific version of a record. If no version is // provided then the most recent version of the record will be returned. // -// Filenames can be used to request specific files. If filenames is not empty +// Filenames can be used to request specific files. If filenames is not W // then the specified files will be the only files that are returned. // // OmitAllFiles can be used to retrieve a record without any of the record diff --git a/politeiad/client/comments.go b/politeiad/client/comments.go index 5e6e87711..f7e2b046d 100644 --- a/politeiad/client/comments.go +++ b/politeiad/client/comments.go @@ -4,127 +4,141 @@ package client -/* +import ( + "context" + + "github.com/decred/politeia/politeiad/plugins/comments" +) + // CommentNew sends the comments plugin New command to the politeiad v1 API. func (c *Client) CommentNew(ctx context.Context, state string, n comments.New) (*comments.NewReply, error) { - // Setup request - b, err := json.Marshal(n) - if err != nil { - return nil, err - } - cmd := pdv2.PluginCmd{ - Token: n.Token, - ID: comments.PluginID, - Command: comments.CmdNew, - Payload: string(b), - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var nr comments.NewReply - err = json.Unmarshal([]byte(pcr.Payload), &nr) - if err != nil { - return nil, err - } - - return &nr, nil + /* + // Setup request + b, err := json.Marshal(n) + if err != nil { + return nil, err + } + cmd := pdv2.PluginCmd{ + Token: n.Token, + ID: comments.PluginID, + Command: comments.CmdNew, + Payload: string(b), + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var nr comments.NewReply + err = json.Unmarshal([]byte(pcr.Payload), &nr) + if err != nil { + return nil, err + } + + return &nr, nil + */ + return nil, nil } // CommentVote sends the comments plugin Vote command to the politeiad v1 API. func (c *Client) CommentVote(ctx context.Context, state string, v comments.Vote) (*comments.VoteReply, error) { - // Setup request - b, err := json.Marshal(v) - if err != nil { - return nil, err - } - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionWrite, - State: state, - Token: v.Token, - ID: comments.PluginID, - Command: comments.CmdVote, - Payload: string(b), - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var vr comments.VoteReply - err = json.Unmarshal([]byte(pcr.Payload), &vr) - if err != nil { - return nil, err - } - - return &vr, nil + /* + // Setup request + b, err := json.Marshal(v) + if err != nil { + return nil, err + } + cmds := []pdv1.PluginCommandV2{ + { + Action: pdv1.PluginActionWrite, + State: state, + Token: v.Token, + ID: comments.PluginID, + Command: comments.CmdVote, + Payload: string(b), + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var vr comments.VoteReply + err = json.Unmarshal([]byte(pcr.Payload), &vr) + if err != nil { + return nil, err + } + + return &vr, nil + */ + return nil, nil } // CommentDel sends the comments plugin Del command to the politeiad v1 API. func (c *Client) CommentDel(ctx context.Context, state string, d comments.Del) (*comments.DelReply, error) { - // Setup request - b, err := json.Marshal(d) - if err != nil { - return nil, err - } - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionWrite, - State: state, - Token: d.Token, - ID: comments.PluginID, - Command: comments.CmdDel, - Payload: string(b), - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var dr comments.DelReply - err = json.Unmarshal([]byte(pcr.Payload), &dr) - if err != nil { - return nil, err - } - - return &dr, nil + /* + // Setup request + b, err := json.Marshal(d) + if err != nil { + return nil, err + } + cmds := []pdv1.PluginCommandV2{ + { + Action: pdv1.PluginActionWrite, + State: state, + Token: d.Token, + ID: comments.PluginID, + Command: comments.CmdDel, + Payload: string(b), + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var dr comments.DelReply + err = json.Unmarshal([]byte(pcr.Payload), &dr) + if err != nil { + return nil, err + } + + return &dr, nil + */ + return nil, nil } // CommentCount sends a batch of comment plugin Count commands to the @@ -132,212 +146,226 @@ func (c *Client) CommentDel(ctx context.Context, state string, d comments.Del) ( // is not found for a token or any other error occurs, that token will not be // included in the reply. func (c *Client) CommentCount(ctx context.Context, state string, tokens []string) (map[string]uint32, error) { - // Setup request - cmds := make([]pdv1.PluginCommandV2, 0, len(tokens)) - for _, v := range tokens { - cmds = append(cmds, pdv1.PluginCommandV2{ - Action: pdv1.PluginActionRead, - State: state, - Token: v, - ID: comments.PluginID, - Command: comments.CmdCount, - }) - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) != len(cmds) { - return nil, fmt.Errorf("replies missing") - } - - // Decode replies - counts := make(map[string]uint32, len(replies)) - for _, v := range replies { - // This command swallows individual errors. The token of the - // command that errored will not be included in the reply. - err = extractPluginCommandError(v) - if err != nil { - spew.Dump(err) - continue + /* + // Setup request + cmds := make([]pdv1.PluginCommandV2, 0, len(tokens)) + for _, v := range tokens { + cmds = append(cmds, pdv1.PluginCommandV2{ + Action: pdv1.PluginActionRead, + State: state, + Token: v, + ID: comments.PluginID, + Command: comments.CmdCount, + }) } - var cr comments.CountReply - err = json.Unmarshal([]byte(v.Payload), &cr) + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) if err != nil { - continue + return nil, err + } + if len(replies) != len(cmds) { + return nil, fmt.Errorf("replies missing") + } + + // Decode replies + counts := make(map[string]uint32, len(replies)) + for _, v := range replies { + // This command swallows individual errors. The token of the + // command that errored will not be included in the reply. + err = extractPluginCommandError(v) + if err != nil { + spew.Dump(err) + continue + } + + var cr comments.CountReply + err = json.Unmarshal([]byte(v.Payload), &cr) + if err != nil { + continue + } + counts[v.Token] = cr.Count } - counts[v.Token] = cr.Count - } - return counts, nil + return counts, nil + */ + return nil, nil } // CommentsGet sends the comments plugin Get command to the politeiad v1 API. func (c *Client) CommentsGet(ctx context.Context, state, token string, g comments.Get) (map[uint32]comments.Comment, error) { - // Setup request - b, err := json.Marshal(g) - if err != nil { - return nil, err - } - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionRead, - State: state, - Token: token, - ID: comments.PluginID, - Command: comments.CmdGet, - Payload: string(b), - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var gr comments.GetReply - err = json.Unmarshal([]byte(pcr.Payload), &gr) - if err != nil { - return nil, err - } - - return gr.Comments, nil + /* + // Setup request + b, err := json.Marshal(g) + if err != nil { + return nil, err + } + cmds := []pdv1.PluginCommandV2{ + { + Action: pdv1.PluginActionRead, + State: state, + Token: token, + ID: comments.PluginID, + Command: comments.CmdGet, + Payload: string(b), + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var gr comments.GetReply + err = json.Unmarshal([]byte(pcr.Payload), &gr) + if err != nil { + return nil, err + } + + return gr.Comments, nil + */ + return nil, nil } // CommentsGetAll sends the comments plugin GetAll command to the politeiad v1 // API. func (c *Client) CommentsGetAll(ctx context.Context, state, token string) ([]comments.Comment, error) { - // Setup request - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionRead, - State: state, - Token: token, - ID: comments.PluginID, - Command: comments.CmdGetAll, - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var gar comments.GetAllReply - err = json.Unmarshal([]byte(pcr.Payload), &gar) - if err != nil { - return nil, err - } - - return gar.Comments, nil + /* + // Setup request + cmds := []pdv1.PluginCommandV2{ + { + Action: pdv1.PluginActionRead, + State: state, + Token: token, + ID: comments.PluginID, + Command: comments.CmdGetAll, + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var gar comments.GetAllReply + err = json.Unmarshal([]byte(pcr.Payload), &gar) + if err != nil { + return nil, err + } + + return gar.Comments, nil + */ + return nil, nil } // CommentVotes sends the comments plugin Votes command to the politeiad v1 // API. func (c *Client) CommentVotes(ctx context.Context, state, token string, v comments.Votes) ([]comments.CommentVote, error) { - // Setup request - b, err := json.Marshal(v) - if err != nil { - return nil, err - } - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionRead, - State: state, - Token: token, - ID: comments.PluginID, - Command: comments.CmdVotes, - Payload: string(b), - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var vr comments.VotesReply - err = json.Unmarshal([]byte(pcr.Payload), &vr) - if err != nil { - return nil, err - } - - return vr.Votes, nil + /* + // Setup request + b, err := json.Marshal(v) + if err != nil { + return nil, err + } + cmds := []pdv1.PluginCommandV2{ + { + Action: pdv1.PluginActionRead, + State: state, + Token: token, + ID: comments.PluginID, + Command: comments.CmdVotes, + Payload: string(b), + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var vr comments.VotesReply + err = json.Unmarshal([]byte(pcr.Payload), &vr) + if err != nil { + return nil, err + } + + return vr.Votes, nil + */ + return nil, nil } // CommentTimestamps sends the comments plugin Timestamps command to the // politeiad v1 API. func (c *Client) CommentTimestamps(ctx context.Context, state, token string, t comments.Timestamps) (*comments.TimestampsReply, error) { - // Setup request - b, err := json.Marshal(t) - if err != nil { - return nil, err - } - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionRead, - State: state, - Token: token, - ID: comments.PluginID, - Command: comments.CmdTimestamps, - Payload: string(b), - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var tr comments.TimestampsReply - err = json.Unmarshal([]byte(pcr.Payload), &tr) - if err != nil { - return nil, err - } - - return &tr, nil + /* + // Setup request + b, err := json.Marshal(t) + if err != nil { + return nil, err + } + cmds := []pdv1.PluginCommandV2{ + { + Action: pdv1.PluginActionRead, + State: state, + Token: token, + ID: comments.PluginID, + Command: comments.CmdTimestamps, + Payload: string(b), + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var tr comments.TimestampsReply + err = json.Unmarshal([]byte(pcr.Payload), &tr) + if err != nil { + return nil, err + } + + return &tr, nil + */ + return nil, nil } -*/ diff --git a/politeiad/client/ticketvote.go b/politeiad/client/ticketvote.go index 0d43ef161..9b7e843d3 100644 --- a/politeiad/client/ticketvote.go +++ b/politeiad/client/ticketvote.go @@ -4,417 +4,451 @@ package client -/* +import ( + "context" + + "github.com/decred/politeia/politeiad/plugins/ticketvote" +) + // TicketVoteAuthorize sends the ticketvote plugin Authorize command to the // politeiad v1 API. func (c *Client) TicketVoteAuthorize(ctx context.Context, a ticketvote.Authorize) (*ticketvote.AuthorizeReply, error) { - // Setup request - b, err := json.Marshal(a) - if err != nil { - return nil, err - } - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionWrite, - State: pdv1.RecordStateVetted, - Token: a.Token, - ID: ticketvote.PluginID, - Command: ticketvote.CmdAuthorize, - Payload: string(b), - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var ar ticketvote.AuthorizeReply - err = json.Unmarshal([]byte(pcr.Payload), &ar) - if err != nil { - return nil, err - } - - return &ar, nil + /* + // Setup request + b, err := json.Marshal(a) + if err != nil { + return nil, err + } + cmds := []pdv1.PluginCommandV2{ + { + Action: pdv1.PluginActionWrite, + State: pdv1.RecordStateVetted, + Token: a.Token, + ID: ticketvote.PluginID, + Command: ticketvote.CmdAuthorize, + Payload: string(b), + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var ar ticketvote.AuthorizeReply + err = json.Unmarshal([]byte(pcr.Payload), &ar) + if err != nil { + return nil, err + } + + return &ar, nil + */ + return nil, nil } // TicketVoteStart sends the ticketvote plugin Start command to the politeiad // v1 API. func (c *Client) TicketVoteStart(ctx context.Context, token string, s ticketvote.Start) (*ticketvote.StartReply, error) { - // Setup request - b, err := json.Marshal(s) - if err != nil { - return nil, err - } - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionWrite, - State: pdv1.RecordStateVetted, - Token: token, - ID: ticketvote.PluginID, - Command: ticketvote.CmdStart, - Payload: string(b), - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var sr ticketvote.StartReply - err = json.Unmarshal([]byte(pcr.Payload), &sr) - if err != nil { - return nil, err - } - - return &sr, nil + /* + // Setup request + b, err := json.Marshal(s) + if err != nil { + return nil, err + } + cmds := []pdv1.PluginCommandV2{ + { + Action: pdv1.PluginActionWrite, + State: pdv1.RecordStateVetted, + Token: token, + ID: ticketvote.PluginID, + Command: ticketvote.CmdStart, + Payload: string(b), + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var sr ticketvote.StartReply + err = json.Unmarshal([]byte(pcr.Payload), &sr) + if err != nil { + return nil, err + } + + return &sr, nil + */ + return nil, nil } // TicketVoteCastBallot sends the ticketvote plugin CastBallot command to the // politeiad v1 API. func (c *Client) TicketVoteCastBallot(ctx context.Context, token string, cb ticketvote.CastBallot) (*ticketvote.CastBallotReply, error) { - // Setup request - b, err := json.Marshal(cb) - if err != nil { - return nil, err - } - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionWrite, - State: pdv1.RecordStateVetted, - Token: token, - ID: ticketvote.PluginID, - Command: ticketvote.CmdCastBallot, - Payload: string(b), - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var cbr ticketvote.CastBallotReply - err = json.Unmarshal([]byte(pcr.Payload), &cbr) - if err != nil { - return nil, err - } - - return &cbr, nil + /* + // Setup request + b, err := json.Marshal(cb) + if err != nil { + return nil, err + } + cmds := []pdv1.PluginCommandV2{ + { + Action: pdv1.PluginActionWrite, + State: pdv1.RecordStateVetted, + Token: token, + ID: ticketvote.PluginID, + Command: ticketvote.CmdCastBallot, + Payload: string(b), + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var cbr ticketvote.CastBallotReply + err = json.Unmarshal([]byte(pcr.Payload), &cbr) + if err != nil { + return nil, err + } + + return &cbr, nil + */ + return nil, nil } // TicketVoteDetails sends the ticketvote plugin Details command to the // politeiad v1 API. func (c *Client) TicketVoteDetails(ctx context.Context, token string) (*ticketvote.DetailsReply, error) { - // Setup request - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionRead, - State: pdv1.RecordStateVetted, - Token: token, - ID: ticketvote.PluginID, - Command: ticketvote.CmdDetails, - Payload: "", - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var dr ticketvote.DetailsReply - err = json.Unmarshal([]byte(pcr.Payload), &dr) - if err != nil { - return nil, err - } - - return &dr, nil + /* + // Setup request + cmds := []pdv1.PluginCommandV2{ + { + Action: pdv1.PluginActionRead, + State: pdv1.RecordStateVetted, + Token: token, + ID: ticketvote.PluginID, + Command: ticketvote.CmdDetails, + Payload: "", + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var dr ticketvote.DetailsReply + err = json.Unmarshal([]byte(pcr.Payload), &dr) + if err != nil { + return nil, err + } + + return &dr, nil + */ + return nil, nil } // TicketVoteResults sends the ticketvote plugin Results command to the // politeiad v1 API. func (c *Client) TicketVoteResults(ctx context.Context, token string) (*ticketvote.ResultsReply, error) { - // Setup request - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionRead, - State: pdv1.RecordStateVetted, - Token: token, - ID: ticketvote.PluginID, - Command: ticketvote.CmdResults, - Payload: "", - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var rr ticketvote.ResultsReply - err = json.Unmarshal([]byte(pcr.Payload), &rr) - if err != nil { - return nil, err - } - - return &rr, nil + /* + // Setup request + cmds := []pdv1.PluginCommandV2{ + { + Action: pdv1.PluginActionRead, + State: pdv1.RecordStateVetted, + Token: token, + ID: ticketvote.PluginID, + Command: ticketvote.CmdResults, + Payload: "", + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var rr ticketvote.ResultsReply + err = json.Unmarshal([]byte(pcr.Payload), &rr) + if err != nil { + return nil, err + } + + return &rr, nil + */ + return nil, nil } // TicketVoteSummary sends the ticketvote plugin Summary command to the // politeiad v1 API. func (c *Client) TicketVoteSummary(ctx context.Context, token string) (*ticketvote.SummaryReply, error) { - // Setup request - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionRead, - State: pdv1.RecordStateVetted, - ID: ticketvote.PluginID, - Command: ticketvote.CmdSummary, - Token: token, - Payload: "", - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var sr ticketvote.SummaryReply - err = json.Unmarshal([]byte(pcr.Payload), &sr) - if err != nil { - return nil, err - } - - return &sr, nil + /* + // Setup request + cmds := []pdv1.PluginCommandV2{ + { + Action: pdv1.PluginActionRead, + State: pdv1.RecordStateVetted, + ID: ticketvote.PluginID, + Command: ticketvote.CmdSummary, + Token: token, + Payload: "", + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var sr ticketvote.SummaryReply + err = json.Unmarshal([]byte(pcr.Payload), &sr) + if err != nil { + return nil, err + } + + return &sr, nil + */ + return nil, nil } // TicketVoteSummaries sends a batch of ticketvote plugin Summary commands to // the politeiad v1 API. Individual summary errors are not returned, the token // will simply be left out of the returned map. func (c *Client) TicketVoteSummaries(ctx context.Context, tokens []string) (map[string]ticketvote.SummaryReply, error) { - // Setup request - cmds := make([]pdv1.PluginCommandV2, 0, len(tokens)) - for _, v := range tokens { - cmds = append(cmds, pdv1.PluginCommandV2{ - Action: pdv1.PluginActionRead, - State: pdv1.RecordStateVetted, - Token: v, - ID: ticketvote.PluginID, - Command: ticketvote.CmdSummary, - Payload: "", - }) - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - - // Prepare reply - summaries := make(map[string]ticketvote.SummaryReply, len(replies)) - for _, v := range replies { - err = extractPluginCommandError(v) - if err != nil { - // Individual summary errors are ignored. The token will not - // be included in the returned summaries map. - continue + /* + // Setup request + cmds := make([]pdv1.PluginCommandV2, 0, len(tokens)) + for _, v := range tokens { + cmds = append(cmds, pdv1.PluginCommandV2{ + Action: pdv1.PluginActionRead, + State: pdv1.RecordStateVetted, + Token: v, + ID: ticketvote.PluginID, + Command: ticketvote.CmdSummary, + Payload: "", + }) } - var sr ticketvote.SummaryReply - err = json.Unmarshal([]byte(v.Payload), &sr) + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) if err != nil { return nil, err } - summaries[v.Token] = sr - } - return summaries, nil + // Prepare reply + summaries := make(map[string]ticketvote.SummaryReply, len(replies)) + for _, v := range replies { + err = extractPluginCommandError(v) + if err != nil { + // Individual summary errors are ignored. The token will not + // be included in the returned summaries map. + continue + } + var sr ticketvote.SummaryReply + err = json.Unmarshal([]byte(v.Payload), &sr) + if err != nil { + return nil, err + } + summaries[v.Token] = sr + } + + return summaries, nil + */ + return nil, nil } // TicketVoteSubmissions sends the ticketvote plugin Submissions command to the // politeiad v1 API. func (c *Client) TicketVoteSubmissions(ctx context.Context, token string) ([]string, error) { - // Setup request - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionRead, - State: pdv1.RecordStateVetted, - Token: token, - ID: ticketvote.PluginID, - Command: ticketvote.CmdSubmissions, - Payload: "", - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var sr ticketvote.SubmissionsReply - err = json.Unmarshal([]byte(pcr.Payload), &sr) - if err != nil { - return nil, err - } - - return sr.Submissions, nil + /* + // Setup request + cmds := []pdv1.PluginCommandV2{ + { + Action: pdv1.PluginActionRead, + State: pdv1.RecordStateVetted, + Token: token, + ID: ticketvote.PluginID, + Command: ticketvote.CmdSubmissions, + Payload: "", + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var sr ticketvote.SubmissionsReply + err = json.Unmarshal([]byte(pcr.Payload), &sr) + if err != nil { + return nil, err + } + + return sr.Submissions, nil + */ + return nil, nil } // TicketVoteInventory sends the ticketvote plugin Inventory command to the // politeiad v1 API. func (c *Client) TicketVoteInventory(ctx context.Context, i ticketvote.Inventory) (*ticketvote.InventoryReply, error) { - // Setup request - b, err := json.Marshal(i) - if err != nil { - return nil, err - } - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionRead, - State: pdv1.RecordStateVetted, - ID: ticketvote.PluginID, - Command: ticketvote.CmdInventory, - Payload: string(b), - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var ir ticketvote.InventoryReply - err = json.Unmarshal([]byte(pcr.Payload), &ir) - if err != nil { - return nil, err - } - - return &ir, nil + /* + // Setup request + b, err := json.Marshal(i) + if err != nil { + return nil, err + } + cmds := []pdv1.PluginCommandV2{ + { + Action: pdv1.PluginActionRead, + State: pdv1.RecordStateVetted, + ID: ticketvote.PluginID, + Command: ticketvote.CmdInventory, + Payload: string(b), + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var ir ticketvote.InventoryReply + err = json.Unmarshal([]byte(pcr.Payload), &ir) + if err != nil { + return nil, err + } + + return &ir, nil + */ + return nil, nil } // TicketVoteTimestamps sends the ticketvote plugin Timestamps command to the // politeiad v1 API. func (c *Client) TicketVoteTimestamps(ctx context.Context, t ticketvote.Timestamps) (*ticketvote.TimestampsReply, error) { - // Setup request - b, err := json.Marshal(t) - if err != nil { - return nil, err - } - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionRead, - State: pdv1.RecordStateVetted, - ID: ticketvote.PluginID, - Command: ticketvote.CmdTimestamps, - Token: t.Token, - Payload: string(b), - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var sr ticketvote.TimestampsReply - err = json.Unmarshal([]byte(pcr.Payload), &sr) - if err != nil { - return nil, err - } - - return &sr, nil + /* + // Setup request + b, err := json.Marshal(t) + if err != nil { + return nil, err + } + cmds := []pdv1.PluginCommandV2{ + { + Action: pdv1.PluginActionRead, + State: pdv1.RecordStateVetted, + ID: ticketvote.PluginID, + Command: ticketvote.CmdTimestamps, + Token: t.Token, + Payload: string(b), + }, + } + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCommandError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var sr ticketvote.TimestampsReply + err = json.Unmarshal([]byte(pcr.Payload), &sr) + if err != nil { + return nil, err + } + + return &sr, nil + */ + return nil, nil } -*/ diff --git a/politeiad/client/user.go b/politeiad/client/user.go index 20fa74f3e..1c7892ec6 100644 --- a/politeiad/client/user.go +++ b/politeiad/client/user.go @@ -4,99 +4,108 @@ package client -/* +import ( + "context" + + "github.com/decred/politeia/politeiad/plugins/usermd" +) + // Author sends the user plugin Author command to the politeiad v1 API. -func (c *Client) Author(ctx context.Context, state, token string) (string, error) { - // Setup request - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionRead, - State: state, - Token: token, - ID: usermd.PluginID, - Command: usermd.CmdAuthor, - Payload: "", - }, - } +func (c *Client) Author(ctx context.Context, token string) (string, error) { + /* + // Setup request + cmds := []pdv1.PluginCommandV2{ + { + Action: pdv1.PluginActionRead, + State: state, + Token: token, + ID: usermd.PluginID, + Command: usermd.CmdAuthor, + Payload: "", + }, + } - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return "", err - } - if len(replies) == 0 { - return "", fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return "", err - } + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) + if err != nil { + return "", err + } + if len(replies) == 0 { + return "", fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCommandError(pcr) + if err != nil { + return "", err + } - // Decode reply - var ar usermd.AuthorReply - err = json.Unmarshal([]byte(pcr.Payload), &ar) - if err != nil { - return "", err - } + // Decode reply + var ar usermd.AuthorReply + err = json.Unmarshal([]byte(pcr.Payload), &ar) + if err != nil { + return "", err + } - return ar.UserID, nil + return ar.UserID, nil + */ + return "", nil } // UserRecords sends the user plugin UserRecords command to the politeiad v1 -// API. A separate command is sent for the unvetted and vetted records. The -// returned map is a map[recordState][]token. -func (c *Client) UserRecords(ctx context.Context, userID string) (map[string][]string, error) { - // Setup request - ur := usermd.UserRecords{ - UserID: userID, - } - b, err := json.Marshal(ur) - if err != nil { - return nil, err - } - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionRead, - State: pdv1.RecordStateUnvetted, - ID: usermd.PluginID, - Command: usermd.CmdUserRecords, - Payload: string(b), - }, - { - Action: pdv1.PluginActionRead, - State: pdv1.RecordStateVetted, - ID: usermd.PluginID, - Command: usermd.CmdUserRecords, - Payload: string(b), - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - - // Decode replies - reply := make(map[string][]string, 2) // [recordState][]token - for _, v := range replies { - err = extractPluginCommandError(v) +// API. A separate command is sent for the unvetted and vetted records. +func (c *Client) UserRecords(ctx context.Context, userID string) (*usermd.UserRecordsReply, error) { + /* + // Setup request + ur := usermd.UserRecords{ + UserID: userID, + } + b, err := json.Marshal(ur) if err != nil { - // Swallow individual errors - continue + return nil, err + } + cmds := []pdv1.PluginCommandV2{ + { + Action: pdv1.PluginActionRead, + State: pdv1.RecordStateUnvetted, + ID: usermd.PluginID, + Command: usermd.CmdUserRecords, + Payload: string(b), + }, + { + Action: pdv1.PluginActionRead, + State: pdv1.RecordStateVetted, + ID: usermd.PluginID, + Command: usermd.CmdUserRecords, + Payload: string(b), + }, } - var urr usermd.UserRecordsReply - err = json.Unmarshal([]byte(v.Payload), &urr) + + // Send request + replies, err := c.PluginCommandBatch(ctx, cmds) if err != nil { return nil, err } - reply[v.State] = urr.Records - } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + + // Decode replies + reply := make(map[string][]string, 2) // [recordState][]token + for _, v := range replies { + err = extractPluginCommandError(v) + if err != nil { + // Swallow individual errors + continue + } + var urr usermd.UserRecordsReply + err = json.Unmarshal([]byte(v.Payload), &urr) + if err != nil { + return nil, err + } + reply[v.State] = urr.Records + } - return reply, nil + return reply, nil + */ + return nil, nil } -*/ diff --git a/politeiad/plugins/usermd/usermd.go b/politeiad/plugins/usermd/usermd.go index 509a76898..46af3dab5 100644 --- a/politeiad/plugins/usermd/usermd.go +++ b/politeiad/plugins/usermd/usermd.go @@ -16,12 +16,12 @@ const ( // StreamIDUserMetadata is the politeiad metadata stream ID for the // UserMetadata structure. - StreamIDUserMetadata = 1 + StreamIDUserMetadata uint32 = 1 // StreamIDStatusChanges is the politeiad metadata stream ID for // the status changes metadata. Status changes are appended onto // this metadata stream. - StreamIDStatusChanges = 2 + StreamIDStatusChanges uint32 = 2 ) // ErrorCodeT represents a plugin error that was caused by the user. @@ -73,7 +73,7 @@ type UserMetadata struct { // Signature is the client signature of the Token+Version+Status+Reason. type StatusChangeMetadata struct { Token string `json:"token"` - Version string `json:"version"` + Version uint32 `json:"version"` Status uint32 `json:"status"` Reason string `json:"message,omitempty"` PublicKey string `json:"publickey"` @@ -98,5 +98,6 @@ type UserRecords struct { // UserRecordsReply is the reply to the UserInv command. type UserRecordsReply struct { - Records []string `json:"records"` + Unvetted []string `json:"unvetted"` + Vetted []string `json:"vetted"` } diff --git a/politeiad/testpoliteiad/testpoliteiad.go b/politeiad/testpoliteiad/testpoliteiad.go index a765b7953..ea91458ed 100644 --- a/politeiad/testpoliteiad/testpoliteiad.go +++ b/politeiad/testpoliteiad/testpoliteiad.go @@ -159,7 +159,7 @@ func (p *TestPoliteiad) handleNewRecord(w http.ResponseWriter, r *http.Request) } // Prepare response - tokenb, err := util.Random(v1.TokenSizeGit) + tokenb, err := util.Random(v1.TokenSize) if err != nil { util.RespondWithJSON(w, http.StatusInternalServerError, err) return diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 05319e599..43067e3f4 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -4,82 +4,14 @@ package v1 -import ( - "fmt" -) - const ( // APIRoute is prefixed onto all routes defined in this package. APIRoute = "/pi/v1" - // Routes - RoutePolicy = "/policy" - RouteProposals = "/proposals" - - // Proposal states - ProposalStateUnvetted = "unvetted" - ProposalStateVetted = "vetted" -) - -// ErrorCodeT represents a user error code. -type ErrorCodeT int - -const ( - // Error status codes - ErrorCodeInvalid ErrorCodeT = 0 - ErrorCodeInputInvalid ErrorCodeT = 1 - ErrorCodePageSizeExceeded ErrorCodeT = 2 - ErrorCodeProposalStateInvalid ErrorCodeT = 3 + // RoutePolicy returns the policy for the pi API. + RoutePolicy = "/policy" ) -var ( - // ErrorCodes contains human readable error messages. - ErrorCodes = map[ErrorCodeT]string{ - ErrorCodeInvalid: "error status invalid", - ErrorCodeInputInvalid: "input invalid", - ErrorCodePageSizeExceeded: "page size exceeded", - } -) - -// UserErrorReply is the reply that the server returns when it encounters an -// error that is caused by something that the user did (malformed input, bad -// timing, etc). The HTTP status code will be 400. -type UserErrorReply struct { - ErrorCode ErrorCodeT `json:"errorcode"` - ErrorContext string `json:"errorcontext"` -} - -// Error satisfies the error interface. -func (e UserErrorReply) Error() string { - return fmt.Sprintf("user error code: %v", e.ErrorCode) -} - -// PluginErrorReply is the reply that the server returns when it encounters -// a plugin error. -type PluginErrorReply struct { - PluginID string `json:"pluginid"` - ErrorCode int `json:"errorcode"` - ErrorContext string `json:"errorcontext"` -} - -// Error satisfies the error interface. -func (e PluginErrorReply) Error() string { - return fmt.Sprintf("plugin %v error code: %v", e.PluginID, e.ErrorCode) -} - -// ServerErrorReply is the reply that the server returns when it encounters an -// unrecoverable error while executing a command. The HTTP status code will be -// 500 and the ErrorCode field will contain a UNIX timestamp that the user can -// provide to the server admin to track down the error details in the logs. -type ServerErrorReply struct { - ErrorCode int64 `json:"errorcode"` -} - -// Error satisfies the error interface. -func (e ServerErrorReply) Error() string { - return fmt.Sprintf("server error: %v", e.ErrorCode) -} - // Policy requests the policy settings for the pi API. It includes the policy // guidlines for the contents of a proposal record. type Policy struct{} @@ -93,160 +25,3 @@ type PolicyReply struct { NameLengthMax uint32 `json:"namelengthmax"` // In characters NameSupportedChars []string `json:"namesupportedchars"` } - -// PropStatusT represents a proposal status. The proposal status codes map -// directly to the record status codes. Some have been renamed to give a more -// accurate representation of their use in pi. -type PropStatusT int - -const ( - // PropStatusInvalid indicates the proposal status is invalid. - PropStatusInvalid PropStatusT = 0 - - // PropStatusUnreviewed indicates the proposal has been submitted, - // but has not yet been reviewed and made public by an admin. A - // proposal with this status will have a proposal state of - // PropStateUnvetted. - PropStatusUnreviewed PropStatusT = 1 - - // PropStatusPublic indicates that a proposal has been reviewed and - // made public by an admin. A proposal with this status will have - // a proposal state of PropStateVetted. - PropStatusPublic PropStatusT = 2 - - // PropStatusCensored indicates that a proposal has been censored - // by an admin for violating the proposal guidlines. Both unvetted - // and vetted proposals can be censored so a proposal with this - // status can have a state of either PropStateUnvetted or - // PropStateVetted depending on whether the proposal was censored - // before or after it was made public. - PropStatusCensored PropStatusT = 3 - - // PropStatusUnreviewedChanges is deprecated. It is only here so - // the proposal status numbering maps directly to the record status - // numbering. - PropStatusUnreviewedChanges PropStatusT = 4 - - // PropStatusAbandoned indicates that a proposal has been marked - // as abandoned by an admin due to the author being inactive. - PropStatusAbandoned PropStatusT = 5 -) - -const ( - // FileNameIndexFile is the file name of the proposal markdown - // file that contains the main proposal contents. All proposal - // submissions must contain an index file. - FileNameIndexFile = "index.md" - - // FileNameProposalMetadata is the file name of the user submitted - // ProposalMetadata. All proposal submissions must contain a - // proposal metadata file. - FileNameProposalMetadata = "proposalmetadata.json" - - // FileNameVoteMetadata is the file name of the user submitted - // VoteMetadata. This file will only be present when proposals - // are hosting or participating in certain types of votes. - FileNameVoteMetadata = "votemetadata.json" -) - -// File describes an individual file that is part of the proposal. The -// directory structure must be flattened. -type File struct { - Name string `json:"name"` // Filename - MIME string `json:"mime"` // Mime type - Digest string `json:"digest"` // SHA256 digest of unencoded payload - Payload string `json:"payload"` // File content, base64 encoded -} - -// ProposalMetadata contains metadata that is specified by the user on proposal -// submission. -type ProposalMetadata struct { - Name string `json:"name"` // Proposal name -} - -// VoteMetadata is metadata that is specified by the user on proposal -// submission in order to host or participate in a runoff vote. -type VoteMetadata struct { - // LinkBy is set when the user intends for the proposal to be the - // parent proposal in a runoff vote. It is a UNIX timestamp that - // serves as the deadline for other proposals to declare their - // intent to participate in the runoff vote. - LinkBy int64 `json:"linkby,omitempty"` - - // LinkTo is the censorship token of a runoff vote parent proposal. - // It is set when a proposal is being submitted as a vote options - // in the runoff vote. - LinkTo string `json:"linkto,omitempty"` -} - -// CensorshipRecord contains cryptographic proof that a proposal was accepted -// for review by the server. The proof is verifiable by the client. -type CensorshipRecord struct { - // Token is a random censorship token that is generated by the - // server. It serves as a unique identifier for the proposal. - Token string `json:"token"` - - // Merkle is the ordered merkle root of all files and metadata in - // in the proposal. - Merkle string `json:"merkle"` - - // Signature is the server signature of the Merkle+Token. - Signature string `json:"signature"` -} - -// StatusChange represents a proposal status change. -// -// Signature is the client signature of the Token+Version+Status+Reason. -type StatusChange struct { - Token string `json:"token"` - Version string `json:"version"` - Status PropStatusT `json:"status"` - Reason string `json:"message,omitempty"` - PublicKey string `json:"publickey"` - Signature string `json:"signature"` - Timestamp int64 `json:"timestamp"` -} - -// Proposal represents a proposal submission and its metadata. -// -// Signature is the client signature of the proposal merkle root. The merkle -// root is the ordered merkle root of all proposal files. -type Proposal struct { - Version string `json:"version"` // Proposal version - Timestamp int64 `json:"timestamp"` // Submission UNIX timestamp - State string `json:"state"` // Proposal state - Status PropStatusT `json:"status"` // Proposal status - UserID string `json:"userid"` // Author ID - Username string `json:"username"` // Author username - PublicKey string `json:"publickey"` // Key used in signature - Signature string `json:"signature"` // Signature of merkle root - Files []File `json:"files"` // Proposal files - Statuses []StatusChange `json:"statuses"` // Status change history - - // CensorshipRecord contains cryptographic proof that the proposal - // was received and processed by the server. - CensorshipRecord CensorshipRecord `json:"censorshiprecord"` -} - -const ( - // ProposalsPageSize is the maximum number of proposals that can be - // requested in a Proposals request. - ProposalsPageSize = 10 -) - -// Proposals retrieves the Proposal for each of the provided tokens. -// -// This command does not return the proposal index file or any attachment -// files. It will return the ProposalMetadata file and the VoteMetadata file if -// one is present. Unvetted proposals are stripped of all user submitted data -// when being returned to non-admins. -type Proposals struct { - State string `json:"state"` - Tokens []string `json:"tokens"` -} - -// ProposalsReply is the reply to the Proposals command. Any tokens that did -// not correspond to a Proposal will not be included in the reply. -type ProposalsReply struct { - Proposals map[string]Proposal `json:"proposals"` // [token]Proposal -} diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go index 9aaf0705e..2a77e32b5 100644 --- a/politeiawww/api/records/v1/v1.go +++ b/politeiawww/api/records/v1/v1.go @@ -21,10 +21,6 @@ const ( // Metadata routes RouteUserRecords = "/userrecords" - - // Record states - RecordStateUnvetted = "unvetted" - RecordStateVetted = "vetted" ) // ErrorCodeT represents a user error code. @@ -34,21 +30,25 @@ const ( // Error codes ErrorCodeInvalid ErrorCodeT = 0 ErrorCodeInputInvalid ErrorCodeT = 1 - ErrorCodeFileNameInvalid ErrorCodeT = 2 - ErrorCodeFileMIMEInvalid ErrorCodeT = 3 - ErrorCodeFileDigestInvalid ErrorCodeT = 4 - ErrorCodeFilePayloadInvalid ErrorCodeT = 5 - ErrorCodeMetadataStreamIDInvalid ErrorCodeT = 6 - ErrorCodePublicKeyInvalid ErrorCodeT = 7 - ErrorCodeSignatureInvalid ErrorCodeT = 8 - ErrorCodeRecordTokenInvalid ErrorCodeT = 9 - ErrorCodeRecordStateInvalid ErrorCodeT = 10 - ErrorCodeRecordNotFound ErrorCodeT = 11 - ErrorCodeRecordLocked ErrorCodeT = 12 - ErrorCodeNoRecordChanges ErrorCodeT = 13 - ErrorCodeRecordStatusInvalid ErrorCodeT = 14 - ErrorCodeStatusReasonNotFound ErrorCodeT = 15 - ErrorCodePageSizeExceeded ErrorCodeT = 16 + ErrorCodeFilesEmpty ErrorCodeT = 2 + ErrorCodeFileNameInvalid ErrorCodeT = 3 + ErrorCodeFileNameDuplicate ErrorCodeT = 4 + ErrorCodeFileMIMETypeInvalid ErrorCodeT = 5 + ErrorCodeFileMIMETypeUnsupported ErrorCodeT = 6 + ErrorCodeFileDigestInvalid ErrorCodeT = 7 + ErrorCodeFilePayloadInvalid ErrorCodeT = 8 + ErrorCodeMetadataStreamIDInvalid ErrorCodeT = 9 + ErrorCodePublicKeyInvalid ErrorCodeT = 10 + ErrorCodeSignatureInvalid ErrorCodeT = 11 + ErrorCodeRecordTokenInvalid ErrorCodeT = 12 + ErrorCodeRecordNotFound ErrorCodeT = 13 + ErrorCodeRecordLocked ErrorCodeT = 14 + ErrorCodeNoRecordChanges ErrorCodeT = 15 + ErrorCodeRecordStateInvalid ErrorCodeT = 16 + ErrorCodeRecordStatusInvalid ErrorCodeT = 17 + ErrorCodeStatusChangeInvalid ErrorCodeT = 18 + ErrorCodeStatusReasonNotFound ErrorCodeT = 19 + ErrorCodePageSizeExceeded ErrorCodeT = 20 ) var ( @@ -56,19 +56,23 @@ var ( ErrorCodes = map[ErrorCodeT]string{ ErrorCodeInvalid: "error invalid", ErrorCodeInputInvalid: "input invalid", + ErrorCodeFilesEmpty: "files are empty", ErrorCodeFileNameInvalid: "file name invalid", - ErrorCodeFileMIMEInvalid: "file mime invalid", + ErrorCodeFileNameDuplicate: "file name duplicate", + ErrorCodeFileMIMETypeInvalid: "file mime type invalid", + ErrorCodeFileMIMETypeUnsupported: "file mime type unsupported", ErrorCodeFileDigestInvalid: "file digest invalid", ErrorCodeFilePayloadInvalid: "file payload invalid", ErrorCodeMetadataStreamIDInvalid: "metadata stream id invalid", ErrorCodePublicKeyInvalid: "public key invalid", ErrorCodeSignatureInvalid: "signature invalid", ErrorCodeRecordTokenInvalid: "record token invalid", - ErrorCodeRecordStateInvalid: "record state invalid", ErrorCodeRecordNotFound: "record not found", ErrorCodeRecordLocked: "record locked", ErrorCodeNoRecordChanges: "no record changes", + ErrorCodeRecordStateInvalid: "record state invalid", ErrorCodeRecordStatusInvalid: "record status invalid", + ErrorCodeStatusChangeInvalid: "status change invalid", ErrorCodeStatusReasonNotFound: "status reason not found", ErrorCodePageSizeExceeded: "page size exceeded", } @@ -113,37 +117,55 @@ func (e ServerErrorReply) Error() string { return fmt.Sprintf("server error: %v", e.ErrorCode) } -// RecordStatusT represents a record status. -type RecordStatusT int +// RecordStateT represents the state of a record. +type RecordStateT uint32 + +const ( + // RecordStateInvalid is an invalid record state. + RecordStateInvalid RecordStateT = 0 + + // RecordStateUnvetted indicates a record has not been made public. + RecordStateUnvetted RecordStateT = 1 + + // RecordStateVetted indicates a record has been made public. + RecordStateVetted RecordStateT = 2 +) + +var ( + // RecordStates contains the human readable record states. + RecordStates = map[RecordStateT]string{ + RecordStateInvalid: "invalid", + RecordStateUnvetted: "unvetted", + RecordStateVetted: "vetted", + } +) + +// RecordStatusT represents the status of a record. +type RecordStatusT uint32 const ( - // RecordStatusInvalid is an invalid record status. + // RecordStatusInvalid is an invalid status code. RecordStatusInvalid RecordStatusT = 0 - // RecordStatusUnreviewed indicates that a record has been - // submitted but has not been made public yet. A record with - // this status will have a state of unvetted. + // RecordStatusUnreviewed indicates a record has not been made + // public yet. The state of an unreviewed record will always be + // unvetted. RecordStatusUnreviewed RecordStatusT = 1 - // RecordStatusPublic indicates that a record has been made public. - // A record with this status will have a state of vetted. + // RecordStatusPublic indicates a record has been made public. The + // state of a public record will always be vetted. RecordStatusPublic RecordStatusT = 2 - // RecordStatusCensored indicates that a record has been censored. - // The record state can be either unvetted or vetted depending on - // whether the record was censored before or after it was made - // public. All user submitted content of a censored record will - // have been permanently deleted. + // RecordStatusCensored indicates a record has been censored. A + // censored record is locked from any further updates and all + // record content is permanently deleted. A censored record can + // have a state of either unvetted or vetted. RecordStatusCensored RecordStatusT = 3 - // RecordStatusUnreviewedChanges has been deprecated. - RecordStatusUnreviewedChanges RecordStatusT = 4 - - // RecordStatusArchived represents a record that has been archived. - // Both unvetted and vetted records can be marked as archived. - // Unlike with censored records, the user submitted content of an - // archived record is not deleted. - RecordStatusArchived RecordStatusT = 5 + // RecordStatusArchived indicates a record has been archived. An + // archived record is locked from any further updates. An archived + // record have a state of either unvetted or vetted. + RecordStatusArchived RecordStatusT = 4 ) var ( @@ -168,7 +190,7 @@ type File struct { // MetadataStream describes a record metadata stream. type MetadataStream struct { PluginID string `json:"pluginid"` - StreamID uint64 `json:"streamid"` + StreamID uint32 `json:"streamid"` Payload string `json:"payload"` // JSON encoded } @@ -188,9 +210,9 @@ type CensorshipRecord struct { // Record represents a record and all of its content. type Record struct { - State string `json:"state"` // Record state + State RecordStateT `json:"state"` // Record state Status RecordStatusT `json:"status"` // Record status - Version string `json:"version"` // Version of this record + Version uint32 `json:"version"` // Version of this record Timestamp int64 `json:"timestamp"` // Last update Username string `json:"username"` // Author username Metadata []MetadataStream `json:"metadata"` // Metadata streams @@ -216,7 +238,7 @@ type UserMetadata struct { // Signature is the client signature of the Token+Version+Status+Reason. type StatusChange struct { Token string `json:"token"` - Version string `json:"version"` + Version uint32 `json:"version"` Status RecordStatusT `json:"status"` Reason string `json:"message,omitempty"` PublicKey string `json:"publickey"` @@ -244,7 +266,6 @@ type NewReply struct { // Signature is the client signature of the record merkle root. The merkle root // is the ordered merkle root of all record Files. type Edit struct { - State string `json:"state"` Token string `json:"token"` Files []File `json:"files"` PublicKey string `json:"publickey"` @@ -261,9 +282,8 @@ type EditReply struct { // // Signature is the client signature of the Token+Version+Status+Reason. type SetStatus struct { - State string `json:"state"` Token string `json:"token"` - Version string `json:"version"` + Version uint32 `json:"version"` Status RecordStatusT `json:"status"` Reason string `json:"reason,omitempty"` PublicKey string `json:"publickey"` @@ -279,8 +299,7 @@ type SetStatusReply struct { // If no version is specified then the most recent version will be returned. type Details struct { Token string `json:"token"` - State string `json:"state"` - Version string `json:"version,omitempty"` + Version uint32 `json:"version,omitempty"` } // DetailsReply is the reply to the Details command. @@ -308,7 +327,7 @@ const ( // files. This supersedes the filenames argument. type RecordRequest struct { Token string `json:"token"` - Version string `json:"version,omitempty"` + Version uint32 `json:"version,omitempty"` Filenames []string `json:"filenames,omitempty"` OmitAllFiles bool `json:"omitallfiles,omitempty"` } @@ -320,7 +339,6 @@ type RecordRequest struct { // files are not included in the reply, unvetted records are returned to all // users. type Records struct { - State string `json:"state"` Requests []RecordRequest `json:"requests"` } @@ -348,7 +366,7 @@ const ( // // Unvetted record tokens will only be returned to admins. type Inventory struct { - State string `json:"state,omitempty"` + State RecordStateT `json:"state,omitempty"` Status RecordStatusT `json:"status,omitempty"` Page uint32 `json:"page,omitempty"` } @@ -395,17 +413,16 @@ type Timestamp struct { // version is omitted, the timestamps for the most recent version will be // returned. type Timestamps struct { - State string `json:"state"` Token string `json:"token"` - Version string `json:"version,omitempty"` + Version uint32 `json:"version,omitempty"` } // TimestampsReply is the reply to the Timestamps command. type TimestampsReply struct { RecordMetadata Timestamp `json:"recordmetadata"` - // map[pluginID+streamID]Timestamp - Metadata map[string]Timestamp `json:"metadata"` + // map[pluginID]map[streamID]Timestamp + Metadata map[string]map[uint32]Timestamp `json:"metadata"` // map[filename]Timestamp Files map[string]Timestamp `json:"files"` diff --git a/politeiawww/comments/comments.go b/politeiawww/comments/comments.go index 1d48d082f..b2eca92c8 100644 --- a/politeiawww/comments/comments.go +++ b/politeiawww/comments/comments.go @@ -10,7 +10,7 @@ import ( "net/http" "strconv" - pdv1 "github.com/decred/politeia/politeiad/api/v1" + pdv2 "github.com/decred/politeia/politeiad/api/v2" pdclient "github.com/decred/politeia/politeiad/client" "github.com/decred/politeia/politeiad/plugins/comments" v1 "github.com/decred/politeia/politeiawww/api/comments/v1" @@ -248,7 +248,7 @@ func (c *Comments) HandleTimestamps(w http.ResponseWriter, r *http.Request) { } // New returns a new Comments context. -func New(cfg *config.Config, pdc *pdclient.Client, udb user.Database, s *sessions.Sessions, e *events.Manager, plugins []pdv1.Plugin) (*Comments, error) { +func New(cfg *config.Config, pdc *pdclient.Client, udb user.Database, s *sessions.Sessions, e *events.Manager, plugins []pdv2.Plugin) (*Comments, error) { // Parse plugin settings var ( lengthMax uint32 diff --git a/politeiawww/comments/error.go b/politeiawww/comments/error.go index f9a7e7713..5fd51fd11 100644 --- a/politeiawww/comments/error.go +++ b/politeiawww/comments/error.go @@ -12,7 +12,7 @@ import ( "strings" "time" - pdv1 "github.com/decred/politeia/politeiad/api/v1" + pdv2 "github.com/decred/politeia/politeiad/api/v2" pdclient "github.com/decred/politeia/politeiad/client" v1 "github.com/decred/politeia/politeiawww/api/comments/v1" "github.com/decred/politeia/util" @@ -131,14 +131,12 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err func convertPDErrorCode(errCode int) v1.ErrorCodeT { // These are the only politeiad user errors that the comments // API expects to encounter. - switch pdv1.ErrorStatusT(errCode) { - case pdv1.ErrorStatusInvalidToken: + switch pdv2.ErrorCodeT(errCode) { + case pdv2.ErrorCodeTokenInvalid: return v1.ErrorCodeTokenInvalid - case pdv1.ErrorStatusInvalidRecordState: - return v1.ErrorCodeRecordStateInvalid - case pdv1.ErrorStatusRecordNotFound: + case pdv2.ErrorCodeRecordNotFound: return v1.ErrorCodeRecordNotFound - case pdv1.ErrorStatusRecordLocked: + case pdv2.ErrorCodeRecordLocked: return v1.ErrorCodeRecordLocked } return v1.ErrorCodeInvalid diff --git a/politeiawww/comments/process.go b/politeiawww/comments/process.go index 1de7743d7..e5ca396fa 100644 --- a/politeiawww/comments/process.go +++ b/politeiawww/comments/process.go @@ -29,7 +29,7 @@ func (c *Comments) processNew(ctx context.Context, n v1.New, u user.User) (*v1.N // unvetted records. if n.State == v1.RecordStateUnvetted && !u.Admin { // User is not an admin. Get the record author. - authorID, err := c.politeiad.Author(ctx, n.State, n.Token) + authorID, err := c.politeiad.Author(ctx, n.Token) if err != nil { return nil, err } @@ -193,7 +193,7 @@ func (c *Comments) processComments(ctx context.Context, cs v1.Comments, u *user. isAllowed = true default: // User is not an admin. Get the record author. - authorID, err := c.politeiad.Author(ctx, cs.State, cs.Token) + authorID, err := c.politeiad.Author(ctx, cs.Token) if err != nil { return nil, err } diff --git a/politeiawww/dcc.go b/politeiawww/dcc.go index 83bd11ce2..e24f43b6b 100644 --- a/politeiawww/dcc.go +++ b/politeiawww/dcc.go @@ -24,7 +24,6 @@ import ( pd "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/identity" cms "github.com/decred/politeia/politeiawww/api/cms/v1" - pi "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/cmsdatabase" "github.com/decred/politeia/politeiawww/user" @@ -402,19 +401,6 @@ func convertStartVoteToCMS(sv cms.StartVote) cmsplugin.StartVote { } -func convertPiFilesFromWWW(files []www.File) []pi.File { - f := make([]pi.File, 0, len(files)) - for _, v := range files { - f = append(f, pi.File{ - Name: v.Name, - MIME: v.MIME, - Digest: v.Digest, - Payload: v.Payload, - }) - } - return f -} - func (p *politeiawww) processNewDCC(ctx context.Context, nd cms.NewDCC, u *user.User) (*cms.NewDCCReply, error) { reply := &cms.NewDCCReply{} diff --git a/politeiawww/invoices.go b/politeiawww/invoices.go index 441418ae3..47334ed18 100644 --- a/politeiawww/invoices.go +++ b/politeiawww/invoices.go @@ -680,7 +680,7 @@ func (p *politeiawww) processNewInvoice(ctx context.Context, ni cms.NewInvoice, // Handle test case if p.test { - tokenBytes, err := util.Random(pd.TokenSizeGit) + tokenBytes, err := util.Random(pd.TokenSize) if err != nil { return nil, err } diff --git a/politeiawww/pi/error.go b/politeiawww/pi/error.go deleted file mode 100644 index 482a28eba..000000000 --- a/politeiawww/pi/error.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright (c) 2020-2021 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package pi - -import ( - "errors" - "fmt" - "net/http" - "runtime/debug" - "strings" - "time" - - pdclient "github.com/decred/politeia/politeiad/client" - v1 "github.com/decred/politeia/politeiawww/api/pi/v1" - "github.com/decred/politeia/util" -) - -func respondWithError(w http.ResponseWriter, r *http.Request, format string, err error) { - var ( - ue v1.UserErrorReply - pe pdclient.Error - ) - switch { - case errors.As(err, &ue): - // Pi user error - m := fmt.Sprintf("Pi user error: %v %v %v", - util.RemoteAddr(r), ue.ErrorCode, v1.ErrorCodes[ue.ErrorCode]) - if ue.ErrorContext != "" { - m += fmt.Sprintf(": %v", ue.ErrorContext) - } - log.Infof(m) - util.RespondWithJSON(w, http.StatusBadRequest, - v1.UserErrorReply{ - ErrorCode: ue.ErrorCode, - ErrorContext: ue.ErrorContext, - }) - return - - case errors.As(err, &pe): - // Politeiad error - var ( - pluginID = pe.ErrorReply.PluginID - errCode = pe.ErrorReply.ErrorCode - errContext = pe.ErrorReply.ErrorContext - ) - switch { - case pluginID != "": - // Politeiad plugin error. Log it and return a 400. - m := fmt.Sprintf("Plugin error: %v %v %v", - util.RemoteAddr(r), pluginID, errCode) - if len(errContext) > 0 { - m += fmt.Sprintf(": %v", strings.Join(errContext, ", ")) - } - log.Infof(m) - util.RespondWithJSON(w, http.StatusBadRequest, - v1.PluginErrorReply{ - PluginID: pluginID, - ErrorCode: errCode, - ErrorContext: strings.Join(errContext, ", "), - }) - return - - default: - // Unknown politeiad error. Log it and return a 500. - ts := time.Now().Unix() - log.Errorf("%v %v %v %v Internal error %v: error code "+ - "from politeiad: %v", util.RemoteAddr(r), r.Method, r.URL, - r.Proto, ts, errCode) - - util.RespondWithJSON(w, http.StatusInternalServerError, - v1.ServerErrorReply{ - ErrorCode: ts, - }) - return - } - - default: - // Internal server error. Log it and return a 500. - t := time.Now().Unix() - e := fmt.Sprintf(format, err) - log.Errorf("%v %v %v %v Internal error %v: %v", - util.RemoteAddr(r), r.Method, r.URL, r.Proto, t, e) - log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) - - util.RespondWithJSON(w, http.StatusInternalServerError, - v1.ServerErrorReply{ - ErrorCode: t, - }) - return - } -} diff --git a/politeiawww/pi/events.go b/politeiawww/pi/events.go index 4dee8ffd2..e46c137d8 100644 --- a/politeiawww/pi/events.go +++ b/politeiawww/pi/events.go @@ -13,11 +13,12 @@ import ( "io" "strings" - pdv1 "github.com/decred/politeia/politeiad/api/v1" + pdv2 "github.com/decred/politeia/politeiad/api/v2" cmplugin "github.com/decred/politeia/politeiad/plugins/comments" + piplugin "github.com/decred/politeia/politeiad/plugins/pi" + tkplugin "github.com/decred/politeia/politeiad/plugins/ticketvote" "github.com/decred/politeia/politeiad/plugins/usermd" cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" - piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" @@ -123,7 +124,7 @@ func (p *Pi) handleEventRecordEdit(ch chan interface{}) { } // Only send edit notifications for public proposals - if e.State == rcv1.RecordStateUnvetted { + if e.Record.State == rcv1.RecordStateUnvetted { log.Debugf("Proposal is unvetted no edit ntfn %v", e.Record.CensorshipRecord.Token) continue @@ -417,17 +418,17 @@ func (p *Pi) handleEventCommentNew(ch chan interface{}) { // Get the record author and record name var ( - pdr *pdv1.Record + pdr *pdv2.Record r rcv1.Record proposalAuthorID string proposalName string err error ) - pdr, err = p.recordAbridged(e.State, e.Comment.Token) + pdr, err = p.recordAbridged(e.Comment.Token) if err != nil { goto failed } - r = convertRecordToV1(*pdr, e.State) + r = convertRecordToV1(*pdr) proposalAuthorID = userIDFromMetadata(r.Metadata) proposalName = proposalNameFromRecord(r) @@ -473,7 +474,6 @@ func (p *Pi) handleEventVoteAuthorized(ch chan interface{}) { // Setup args to prevent goto errors var ( - state = rcv1.RecordStateVetted token = e.Auth.Token proposalName string r rcv1.Record @@ -483,11 +483,11 @@ func (p *Pi) handleEventVoteAuthorized(ch chan interface{}) { ) // Get record - pdr, err := p.recordAbridged(state, token) + pdr, err := p.recordAbridged(token) if err != nil { goto failed } - r = convertRecordToV1(*pdr, state) + r = convertRecordToV1(*pdr) proposalName = proposalNameFromRecord(r) // Compile notification email list @@ -610,21 +610,20 @@ func (p *Pi) handleEventVoteStarted(ch chan interface{}) { for _, v := range e.Starts { // Setup args to prevent goto errors var ( - state = rcv1.RecordStateVetted token = v.Params.Token - pdr *pdv1.Record + pdr *pdv2.Record r rcv1.Record err error authorID string proposalName string ) - pdr, err = p.recordAbridged(state, token) + pdr, err = p.recordAbridged(token) if err != nil { goto failed } - r = convertRecordToV1(*pdr, state) + r = convertRecordToV1(*pdr) authorID = userIDFromMetadata(r.Metadata) proposalName = proposalNameFromRecord(r) @@ -653,42 +652,19 @@ func (p *Pi) handleEventVoteStarted(ch chan interface{}) { } } -func (p *Pi) records(state string, reqs []pdv1.RecordRequest) (map[string]pdv1.Record, error) { - var ( - records map[string]pdv1.Record - err error - ) - switch state { - case rcv1.RecordStateUnvetted: - records, err = p.politeiad.GetUnvettedBatch(context.Background(), reqs) - if err != nil { - return nil, err - } - case rcv1.RecordStateVetted: - records, err = p.politeiad.GetVettedBatch(context.Background(), reqs) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("invalid state %v", state) - } - - return records, nil -} - // recordAbridged returns a proposal record without its index file or any // attachment files. This allows the request to be light weight. -func (p *Pi) recordAbridged(state, token string) (*pdv1.Record, error) { - reqs := []pdv1.RecordRequest{ +func (p *Pi) recordAbridged(token string) (*pdv2.Record, error) { + reqs := []pdv2.RecordRequest{ { Token: token, Filenames: []string{ - piv1.FileNameProposalMetadata, - piv1.FileNameVoteMetadata, + piplugin.FileNameProposalMetadata, + tkplugin.FileNameVoteMetadata, }, }, } - rs, err := p.records(state, reqs) + rs, err := p.politeiad.RecordGetBatch(context.Background(), reqs) if err != nil { return nil, fmt.Errorf("politeiad records: %v", err) } @@ -704,7 +680,7 @@ func (p *Pi) recordAbridged(state, token string) (*pdv1.Record, error) { func userMetadataDecode(ms []rcv1.MetadataStream) (*usermd.UserMetadata, error) { var userMD *usermd.UserMetadata for _, v := range ms { - if v.ID == usermd.MDStreamIDUserMetadata { + if v.StreamID == usermd.StreamIDUserMetadata { var um usermd.UserMetadata err := json.Unmarshal([]byte(v.Payload), &um) if err != nil { @@ -736,12 +712,12 @@ func userIDFromMetadata(ms []rcv1.MetadataStream) string { func proposalNameFromRecord(r rcv1.Record) string { var name string for _, v := range r.Files { - if v.Name == piv1.FileNameProposalMetadata { + if v.Name == piplugin.FileNameProposalMetadata { b, err := base64.StdEncoding.DecodeString(v.Payload) if err != nil { return "" } - var pm piv1.ProposalMetadata + var pm piplugin.ProposalMetadata err = json.Unmarshal(b, &pm) if err != nil { return "" @@ -774,7 +750,7 @@ func statusChangesFromMetadata(metadata []rcv1.MetadataStream) ([]usermd.StatusC err error ) for _, v := range metadata { - if v.ID == usermd.MDStreamIDStatusChanges { + if v.StreamID == usermd.StreamIDStatusChanges { sc, err = statusChangesDecode([]byte(v.Payload)) if err != nil { return nil, err @@ -784,21 +760,31 @@ func statusChangesFromMetadata(metadata []rcv1.MetadataStream) ([]usermd.StatusC return sc, nil } -func convertStatusToV1(s pdv1.RecordStatusT) rcv1.RecordStatusT { +func convertStateToV1(s pdv2.RecordStateT) rcv1.RecordStateT { + switch s { + case pdv2.RecordStateUnvetted: + return rcv1.RecordStateUnvetted + case pdv2.RecordStateVetted: + return rcv1.RecordStateVetted + } + return rcv1.RecordStateInvalid +} + +func convertStatusToV1(s pdv2.RecordStatusT) rcv1.RecordStatusT { switch s { - case pdv1.RecordStatusNotReviewed: + case pdv2.RecordStatusUnreviewed: return rcv1.RecordStatusUnreviewed - case pdv1.RecordStatusPublic: + case pdv2.RecordStatusPublic: return rcv1.RecordStatusPublic - case pdv1.RecordStatusCensored: + case pdv2.RecordStatusCensored: return rcv1.RecordStatusCensored - case pdv1.RecordStatusArchived: + case pdv2.RecordStatusArchived: return rcv1.RecordStatusArchived } return rcv1.RecordStatusInvalid } -func convertFilesToV1(f []pdv1.File) []rcv1.File { +func convertFilesToV1(f []pdv2.File) []rcv1.File { files := make([]rcv1.File, 0, len(f)) for _, v := range f { files = append(files, rcv1.File{ @@ -811,24 +797,24 @@ func convertFilesToV1(f []pdv1.File) []rcv1.File { return files } -func convertMetadataStreamsToV1(ms []pdv1.MetadataStream) []rcv1.MetadataStream { +func convertMetadataStreamsToV1(ms []pdv2.MetadataStream) []rcv1.MetadataStream { metadata := make([]rcv1.MetadataStream, 0, len(ms)) for _, v := range ms { metadata = append(metadata, rcv1.MetadataStream{ PluginID: v.PluginID, - ID: v.ID, + StreamID: v.StreamID, Payload: v.Payload, }) } return metadata } -func convertRecordToV1(r pdv1.Record, state string) rcv1.Record { +func convertRecordToV1(r pdv2.Record) rcv1.Record { // User fields that are not part of the politeiad record have // been intentionally left blank. These fields must be pulled // from the user database. return rcv1.Record{ - State: state, + State: convertStateToV1(r.State), Status: convertStatusToV1(r.Status), Version: r.Version, Timestamp: r.Timestamp, diff --git a/politeiawww/pi/mail.go b/politeiawww/pi/mail.go index 383af3b53..d6d5e2bbd 100644 --- a/politeiawww/pi/mail.go +++ b/politeiawww/pi/mail.go @@ -62,7 +62,7 @@ func (p *Pi) mailNtfnProposalNew(token, name, username string, emails []string) type proposalEdit struct { Name string // Proposal name - Version string // Proposal version + Version uint32 // Proposal version Username string // Author username Link string // GUI proposal details URL } @@ -77,7 +77,7 @@ A proposal by {{.Username}} has just been edited: var proposalEditTmpl = template.Must( template.New("proposalEdit").Parse(proposalEditText)) -func (p *Pi) mailNtfnProposalEdit(token, version, name, username string, emails []string) error { +func (p *Pi) mailNtfnProposalEdit(token string, version uint32, name, username string, emails []string) error { route := strings.Replace(guiRouteRecordDetails, "{token}", token, 1) u, err := url.Parse(p.cfg.WebServerAddress + route) if err != nil { diff --git a/politeiawww/pi/pi.go b/politeiawww/pi/pi.go index fa1859f39..470b23b00 100644 --- a/politeiawww/pi/pi.go +++ b/politeiawww/pi/pi.go @@ -10,7 +10,7 @@ import ( "net/http" "strconv" - pdv1 "github.com/decred/politeia/politeiad/api/v1" + pdv2 "github.com/decred/politeia/politeiad/api/v2" pdclient "github.com/decred/politeia/politeiad/client" "github.com/decred/politeia/politeiad/plugins/pi" v1 "github.com/decred/politeia/politeiawww/api/pi/v1" @@ -40,41 +40,8 @@ func (p *Pi) HandlePolicy(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, p.policy) } -// HandleProposals is the request handler for the pi v1 Proposals route. -func (p *Pi) HandleProposals(w http.ResponseWriter, r *http.Request) { - log.Tracef("HandleProposals") - - var ps v1.Proposals - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&ps); err != nil { - respondWithError(w, r, "HandleProposals: unmarshal", - v1.UserErrorReply{ - ErrorCode: v1.ErrorCodeInputInvalid, - }) - return - } - - // Lookup session user. This is a public route so a session may not - // exist. Ignore any session not found errors. - u, err := p.sessions.GetSessionUser(w, r) - if err != nil && err != sessions.ErrSessionNotFound { - respondWithError(w, r, - "HandleDetails: GetSessionUser: %v", err) - return - } - - psr, err := p.processProposals(r.Context(), ps, u) - if err != nil { - respondWithError(w, r, - "HandleProposals: processProposals: %v", err) - return - } - - util.RespondWithJSON(w, http.StatusOK, psr) -} - // New returns a new Pi context. -func New(cfg *config.Config, pdc *pdclient.Client, udb user.Database, s *sessions.Sessions, e *events.Manager, m *mail.Client, plugins []pdv1.Plugin) (*Pi, error) { +func New(cfg *config.Config, pdc *pdclient.Client, udb user.Database, s *sessions.Sessions, e *events.Manager, m *mail.Client, plugins []pdv2.Plugin) (*Pi, error) { // Parse plugin settings var ( textFileSizeMax uint32 diff --git a/politeiawww/pi/process.go b/politeiawww/pi/process.go deleted file mode 100644 index b436a6662..000000000 --- a/politeiawww/pi/process.go +++ /dev/null @@ -1,249 +0,0 @@ -// Copyright (c) 2020-2021 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package pi - -import ( - "context" - "encoding/json" - "fmt" - - pdv1 "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/politeiad/plugins/usermd" - v1 "github.com/decred/politeia/politeiawww/api/pi/v1" - "github.com/decred/politeia/politeiawww/user" - "github.com/google/uuid" -) - -func (p *Pi) proposals(ctx context.Context, state string, reqs []pdv1.RecordRequest) (map[string]v1.Proposal, error) { - var ( - records map[string]pdv1.Record - err error - ) - switch state { - case v1.ProposalStateUnvetted: - records, err = p.politeiad.GetUnvettedBatch(ctx, reqs) - if err != nil { - return nil, err - } - case v1.ProposalStateVetted: - records, err = p.politeiad.GetVettedBatch(ctx, reqs) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("invalid state %v", state) - } - - proposals := make(map[string]v1.Proposal, len(records)) - for k, v := range records { - // Convert to a proposal - pr, err := convertRecord(v, state) - if err != nil { - return nil, err - } - - // Fill in user data - userID := userIDFromMetadataStreamsPD(v.Metadata) - uid, err := uuid.Parse(userID) - if err != nil { - return nil, err - } - u, err := p.userdb.UserGetById(uid) - if err != nil { - return nil, err - } - proposalPopulateUserData(pr, *u) - - proposals[k] = *pr - } - - return proposals, nil -} - -func (p *Pi) processProposals(ctx context.Context, ps v1.Proposals, u *user.User) (*v1.ProposalsReply, error) { - log.Tracef("processProposals: %v %v", ps.State, ps.Tokens) - - // Verify state - switch ps.State { - case v1.ProposalStateUnvetted, v1.ProposalStateVetted: - // Allowed; continue - default: - return nil, v1.UserErrorReply{ - ErrorCode: v1.ErrorCodeProposalStateInvalid, - } - } - - // Verify page size - if len(ps.Tokens) > v1.ProposalsPageSize { - e := fmt.Sprintf("max page size is %v", v1.ProposalsPageSize) - return nil, v1.UserErrorReply{ - ErrorCode: v1.ErrorCodePageSizeExceeded, - ErrorContext: e, - } - } - - // Setup record requests. We don't retrieve any index files or - // attachment files in order to keep the payload size minimal. - reqs := make([]pdv1.RecordRequest, 0, len(ps.Tokens)) - for _, v := range ps.Tokens { - reqs = append(reqs, pdv1.RecordRequest{ - Token: v, - Filenames: []string{ - v1.FileNameProposalMetadata, - v1.FileNameVoteMetadata, - }, - }) - } - - // Get proposals - proposals, err := p.proposals(ctx, ps.State, reqs) - if err != nil { - return nil, err - } - - // Only admins and the proposal author are allowed to retrieve - // unvetted files. Remove files if the user is not an admin or the - // author. This is a public route so a user may not be present. - if ps.State == v1.ProposalStateUnvetted { - for k, v := range proposals { - var ( - isAuthor = u != nil && u.ID.String() == v.UserID - isAdmin = u != nil && u.Admin - ) - if !isAuthor && !isAdmin { - v.Files = []v1.File{} - proposals[k] = v - } - } - } - - return &v1.ProposalsReply{ - Proposals: proposals, - }, nil -} - -// proposalPopulateUserData populates a proposal with user data that is stored -// in the user database and not in politeiad. -func proposalPopulateUserData(pr *v1.Proposal, u user.User) { - pr.Username = u.Username -} - -func convertStatus(s pdv1.RecordStatusT) v1.PropStatusT { - switch s { - case pdv1.RecordStatusNotFound: - // Intentionally omitted. No corresponding PropStatusT. - case pdv1.RecordStatusNotReviewed: - return v1.PropStatusUnreviewed - case pdv1.RecordStatusCensored: - return v1.PropStatusCensored - case pdv1.RecordStatusPublic: - return v1.PropStatusPublic - case pdv1.RecordStatusUnreviewedChanges: - return v1.PropStatusUnreviewed - case pdv1.RecordStatusArchived: - return v1.PropStatusAbandoned - } - return v1.PropStatusInvalid -} - -func convertRecord(r pdv1.Record, state string) (*v1.Proposal, error) { - // Decode metadata streams - var ( - um usermd.UserMetadata - sc = make([]usermd.StatusChangeMetadata, 0, 16) - err error - ) - for _, v := range r.Metadata { - switch v.ID { - case usermd.MDStreamIDUserMetadata: - err = json.Unmarshal([]byte(v.Payload), &um) - if err != nil { - return nil, err - } - case usermd.MDStreamIDStatusChanges: - sc, err = statusChangesDecode([]byte(v.Payload)) - if err != nil { - return nil, err - } - } - } - - // Convert files - files := make([]v1.File, 0, len(r.Files)) - for _, v := range r.Files { - files = append(files, v1.File{ - Name: v.Name, - MIME: v.MIME, - Digest: v.Digest, - Payload: v.Payload, - }) - } - - // Convert statuses - statuses := make([]v1.StatusChange, 0, len(sc)) - for _, v := range sc { - statuses = append(statuses, v1.StatusChange{ - Token: v.Token, - Version: v.Version, - Status: v1.PropStatusT(v.Status), - Reason: v.Reason, - PublicKey: v.PublicKey, - Signature: v.Signature, - Timestamp: v.Timestamp, - }) - } - - // Some fields are intentionally omitted because they are user data - // that is not saved to politeiad and needs to be pulled from the - // user database. - return &v1.Proposal{ - Version: r.Version, - Timestamp: r.Timestamp, - State: state, - Status: convertStatus(r.Status), - UserID: um.UserID, - Username: "", // Intentionally omitted - PublicKey: um.PublicKey, - Signature: um.Signature, - Statuses: statuses, - Files: files, - CensorshipRecord: v1.CensorshipRecord{ - Token: r.CensorshipRecord.Token, - Merkle: r.CensorshipRecord.Merkle, - Signature: r.CensorshipRecord.Signature, - }, - }, nil -} - -// userMetadataDecode decodes and returns the UserMetadata from the provided -// metadata streams. If a UserMetadata is not found, nil is returned. -func userMetadataDecodePD(ms []pdv1.MetadataStream) (*usermd.UserMetadata, error) { - var userMD *usermd.UserMetadata - for _, v := range ms { - if v.ID == usermd.MDStreamIDUserMetadata { - var um usermd.UserMetadata - err := json.Unmarshal([]byte(v.Payload), &um) - if err != nil { - return nil, err - } - userMD = &um - break - } - } - return userMD, nil -} - -// userIDFromMetadataStreams searches for a UserMetadata and parses the user ID -// from it if found. An empty string is returned if no UserMetadata is found. -func userIDFromMetadataStreamsPD(ms []pdv1.MetadataStream) string { - um, err := userMetadataDecodePD(ms) - if err != nil { - return "" - } - if um == nil { - return "" - } - return um.UserID -} diff --git a/politeiawww/piwww.go b/politeiawww/piwww.go index 60671017f..fa1820e49 100644 --- a/politeiawww/piwww.go +++ b/politeiawww/piwww.go @@ -8,7 +8,7 @@ import ( "fmt" "net/http" - pdv1 "github.com/decred/politeia/politeiad/api/v1" + pdv2 "github.com/decred/politeia/politeiad/api/v2" cmplugin "github.com/decred/politeia/politeiad/plugins/comments" piplugin "github.com/decred/politeia/politeiad/plugins/pi" tkplugin "github.com/decred/politeia/politeiad/plugins/ticketvote" @@ -155,12 +155,9 @@ func (p *politeiawww) setupPiRoutes(r *records.Records, c *comments.Comments, t p.addRoute(http.MethodPost, piv1.APIRoute, piv1.RoutePolicy, pic.HandlePolicy, permissionPublic) - p.addRoute(http.MethodPost, piv1.APIRoute, - piv1.RouteProposals, pic.HandleProposals, - permissionPublic) } -func (p *politeiawww) setupPi(plugins []pdv1.Plugin) error { +func (p *politeiawww) setupPi(plugins []pdv2.Plugin) error { // Verify all required politeiad plugins have been registered required := map[string]bool{ piplugin.PluginID: false, diff --git a/politeiawww/politeiawww.go b/politeiawww/politeiawww.go index 30f371b55..cdf70cfdc 100644 --- a/politeiawww/politeiawww.go +++ b/politeiawww/politeiawww.go @@ -16,7 +16,6 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/decred/dcrd/chaincfg/v3" - pdv1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/mime" pdclient "github.com/decred/politeia/politeiad/client" www "github.com/decred/politeia/politeiawww/api/www/v1" @@ -77,7 +76,6 @@ type politeiawww struct { db user.Database sessions *sessions.Sessions events *events.Manager - plugins []pdv1.Plugin // Client websocket connections ws map[string]map[string]*wsContext // [uuid][]*context diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 6232913ff..4e1867422 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -17,8 +17,7 @@ import ( "strings" "github.com/decred/politeia/decredplugin" - pdv1 "github.com/decred/politeia/politeiad/api/v1" - v1 "github.com/decred/politeia/politeiad/api/v1" + pdv2 "github.com/decred/politeia/politeiad/api/v2" piplugin "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/ticketvote" tkplugin "github.com/decred/politeia/politeiad/plugins/ticketvote" @@ -33,11 +32,28 @@ import ( ) func (p *politeiawww) proposal(ctx context.Context, token, version string) (*www.ProposalRecord, error) { + // Parse version + v, err := strconv.ParseUint(version, 10, 64) + if err != nil { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + } + } + // Get record - r, err := p.politeiad.GetVetted(ctx, token, version) + r, err := p.politeiad.RecordGet(ctx, token, uint32(v)) if err != nil { return nil, err } + + // Legacy www routes are only for vetted records + if r.State == pdv2.RecordStateUnvetted { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + } + } + + // Convert to a proposal pr, err := convertRecordToProposal(*r) if err != nil { return nil, err @@ -67,14 +83,19 @@ func (p *politeiawww) proposal(ctx context.Context, token, version string) (*www return pr, nil } -func (p *politeiawww) proposals(ctx context.Context, reqs []pdv1.RecordRequest) (map[string]www.ProposalRecord, error) { - records, err := p.politeiad.GetVettedBatch(ctx, reqs) +func (p *politeiawww) proposals(ctx context.Context, reqs []pdv2.RecordRequest) (map[string]www.ProposalRecord, error) { + records, err := p.politeiad.RecordGetBatch(ctx, reqs) if err != nil { return nil, err } proposals := make(map[string]www.ProposalRecord, len(records)) for k, v := range records { + // Legacy www routes are only for vetted records + if v.State == pdv2.RecordStateUnvetted { + continue + } + // Convert to a proposal pr, err := convertRecordToProposal(v) if err != nil { @@ -113,8 +134,8 @@ func (p *politeiawww) processTokenInventory(ctx context.Context, isAdmin bool) ( log.Tracef("processTokenInventory") // Get record inventory - ir, err := p.politeiad.InventoryByStatus(ctx, "", - pdv1.RecordStatusInvalid, 0) + ir, err := p.politeiad.Inventory(ctx, pdv2.RecordStateInvalid, + pdv2.RecordStatusInvalid, 0) if err != nil { return nil, err } @@ -128,10 +149,12 @@ func (p *politeiawww) processTokenInventory(ctx context.Context, isAdmin bool) ( var ( // Unvetted - unvetted = ir.Unvetted[pdv1.RecordStatusNotReviewed] - unvettedChanges = ir.Unvetted[pdv1.RecordStatusUnreviewedChanges] - unreviewed = append(unvetted, unvettedChanges...) - censored = ir.Unvetted[pdv1.RecordStatusCensored] + statusUnreviewed = pdv2.RecordStatuses[pdv2.RecordStatusUnreviewed] + statusCensored = pdv2.RecordStatuses[pdv2.RecordStatusCensored] + statusArchived = pdv2.RecordStatuses[pdv2.RecordStatusArchived] + + unreviewed = ir.Unvetted[statusUnreviewed] + censored = ir.Unvetted[statusCensored] // Human readable vote statuses statusUnauth = tkplugin.VoteStatuses[tkplugin.VoteStatusUnauthorized] @@ -147,7 +170,7 @@ func (p *politeiawww) processTokenInventory(ctx context.Context, isAdmin bool) ( active = vir.Tokens[statusStarted] approved = vir.Tokens[statusApproved] rejected = vir.Tokens[statusRejected] - abandoned = ir.Vetted[pdv1.RecordStatusArchived] + abandoned = ir.Vetted[statusArchived] ) // Only return unvetted tokens to admins @@ -221,9 +244,9 @@ func (p *politeiawww) processBatchProposals(ctx context.Context, bp www.BatchPro } } - reqs := make([]pdv1.RecordRequest, 0, len(bp.Tokens)) + reqs := make([]pdv2.RecordRequest, 0, len(bp.Tokens)) for _, v := range bp.Tokens { - reqs = append(reqs, pdv1.RecordRequest{ + reqs = append(reqs, pdv2.RecordRequest{ Token: v, Filenames: []string{ piplugin.FileNameProposalMetadata, @@ -348,9 +371,9 @@ func (p *politeiawww) processActiveVote(ctx context.Context) (*www.ActiveVoteRep } // Get proposals - reqs := make([]pdv1.RecordRequest, 0, len(started)) + reqs := make([]pdv2.RecordRequest, 0, len(started)) for _, v := range started { - reqs = append(reqs, pdv1.RecordRequest{ + reqs = append(reqs, pdv2.RecordRequest{ Token: v, Filenames: []string{ piplugin.FileNameProposalMetadata, @@ -681,10 +704,10 @@ func (p *politeiawww) handleVoteResults(w http.ResponseWriter, r *http.Request) // userMetadataDecode decodes and returns the UserMetadata from the provided // metadata streams. If a UserMetadata is not found, nil is returned. -func userMetadataDecode(ms []v1.MetadataStream) (*umplugin.UserMetadata, error) { +func userMetadataDecode(ms []pdv2.MetadataStream) (*umplugin.UserMetadata, error) { var userMD *umplugin.UserMetadata for _, v := range ms { - if v.ID == umplugin.MDStreamIDUserMetadata { + if v.StreamID == umplugin.StreamIDUserMetadata { var um umplugin.UserMetadata err := json.Unmarshal([]byte(v.Payload), &um) if err != nil { @@ -699,7 +722,7 @@ func userMetadataDecode(ms []v1.MetadataStream) (*umplugin.UserMetadata, error) // userIDFromMetadataStreams searches for a UserMetadata and parses the user ID // from it if found. An empty string is returned if no UserMetadata is found. -func userIDFromMetadataStreams(ms []pdv1.MetadataStream) string { +func userIDFromMetadataStreams(ms []pdv2.MetadataStream) string { um, err := userMetadataDecode(ms) if err != nil { return "" @@ -710,22 +733,22 @@ func userIDFromMetadataStreams(ms []pdv1.MetadataStream) string { return um.UserID } -func convertStatusToWWW(status pdv1.RecordStatusT) www.PropStatusT { +func convertStatusToWWW(status pdv2.RecordStatusT) www.PropStatusT { switch status { - case pdv1.RecordStatusInvalid: + case pdv2.RecordStatusInvalid: return www.PropStatusInvalid - case pdv1.RecordStatusPublic: + case pdv2.RecordStatusPublic: return www.PropStatusPublic - case pdv1.RecordStatusCensored: + case pdv2.RecordStatusCensored: return www.PropStatusCensored - case pdv1.RecordStatusArchived: + case pdv2.RecordStatusArchived: return www.PropStatusAbandoned default: return www.PropStatusInvalid } } -func convertRecordToProposal(r pdv1.Record) (*www.ProposalRecord, error) { +func convertRecordToProposal(r pdv2.Record) (*www.ProposalRecord, error) { // Decode metadata var ( um *umplugin.UserMetadata @@ -737,15 +760,15 @@ func convertRecordToProposal(r pdv1.Record) (*www.ProposalRecord, error) { } // This is a usermd plugin metadata stream - switch v.ID { - case umplugin.MDStreamIDUserMetadata: + switch v.StreamID { + case umplugin.StreamIDUserMetadata: var m umplugin.UserMetadata err := json.Unmarshal([]byte(v.Payload), &m) if err != nil { return nil, err } um = &m - case umplugin.MDStreamIDStatusChanges: + case umplugin.StreamIDStatusChanges: d := json.NewDecoder(strings.NewReader(v.Payload)) for { var sc umplugin.StatusChangeMetadata @@ -850,7 +873,7 @@ func convertRecordToProposal(r pdv1.Record) (*www.ProposalRecord, error) { Username: "", // Intentionally omitted PublicKey: um.PublicKey, Signature: um.Signature, - Version: r.Version, + Version: strconv.FormatUint(uint64(r.Version), 10), StatusChangeMessage: changeMsg, PublishedAt: publishedAt, CensoredAt: censoredAt, diff --git a/politeiawww/records/error.go b/politeiawww/records/error.go index fb9043087..3636db786 100644 --- a/politeiawww/records/error.go +++ b/politeiawww/records/error.go @@ -12,7 +12,7 @@ import ( "strings" "time" - pdv1 "github.com/decred/politeia/politeiad/api/v1" + pdv2 "github.com/decred/politeia/politeiad/api/v2" pdclient "github.com/decred/politeia/politeiad/client" v1 "github.com/decred/politeia/politeiawww/api/records/v1" "github.com/decred/politeia/util" @@ -135,47 +135,43 @@ func handlePDError(w http.ResponseWriter, r *http.Request, format string, pde pd func convertPDErrorCode(errCode int) v1.ErrorCodeT { // Any error statuses that are intentionally omitted means that // politeiawww should 500. - switch pdv1.ErrorStatusT(errCode) { - case pdv1.ErrorStatusInvalidRequestPayload: + switch pdv2.ErrorCodeT(errCode) { + case pdv2.ErrorCodeRequestPayloadInvalid: // Intentionally omitted - case pdv1.ErrorStatusInvalidChallenge: + case pdv2.ErrorCodeChallengeInvalid: // Intentionally omitted - case pdv1.ErrorStatusInvalidFilename: + case pdv2.ErrorCodeMetadataStreamInvalid: + // Intentionally omitted + case pdv2.ErrorCodeMetadataStreamDuplicate: + // Intentionally omitted + case pdv2.ErrorCodeFilesEmpty: + return v1.ErrorCodeFilesEmpty + case pdv2.ErrorCodeFileNameInvalid: return v1.ErrorCodeFileNameInvalid - case pdv1.ErrorStatusInvalidFileDigest: + case pdv2.ErrorCodeFileNameDuplicate: + return v1.ErrorCodeFileNameDuplicate + case pdv2.ErrorCodeFileDigestInvalid: return v1.ErrorCodeFileDigestInvalid - case pdv1.ErrorStatusInvalidBase64: + case pdv2.ErrorCodeFilePayloadInvalid: return v1.ErrorCodeFilePayloadInvalid - case pdv1.ErrorStatusInvalidMIMEType: - return v1.ErrorCodeFileMIMEInvalid - case pdv1.ErrorStatusUnsupportedMIMEType: - return v1.ErrorCodeFileMIMEInvalid - case pdv1.ErrorStatusInvalidRecordStatusTransition: - return v1.ErrorCodeRecordStatusInvalid - case pdv1.ErrorStatusEmpty: - return v1.ErrorCodeRecordStatusInvalid - case pdv1.ErrorStatusInvalidMDID: - return v1.ErrorCodeMetadataStreamIDInvalid - case pdv1.ErrorStatusDuplicateMDID: - return v1.ErrorCodeMetadataStreamIDInvalid - case pdv1.ErrorStatusDuplicateFilename: - return v1.ErrorCodeFileNameInvalid - case pdv1.ErrorStatusFileNotFound: - // Intentionally omitted - case pdv1.ErrorStatusNoChanges: + case pdv2.ErrorCodeFileMIMETypeInvalid: + return v1.ErrorCodeFileMIMETypeInvalid + case pdv2.ErrorCodeFileMIMETypeUnsupported: + return v1.ErrorCodeFileMIMETypeUnsupported + case pdv2.ErrorCodeTokenInvalid: + return v1.ErrorCodeRecordTokenInvalid + case pdv2.ErrorCodeRecordNotFound: + return v1.ErrorCodeRecordNotFound + case pdv2.ErrorCodeRecordLocked: + return v1.ErrorCodeRecordLocked + case pdv2.ErrorCodeNoRecordChanges: return v1.ErrorCodeNoRecordChanges - case pdv1.ErrorStatusRecordFound: + case pdv2.ErrorCodeStatusChangeInvalid: + return v1.ErrorCodeStatusChangeInvalid + case pdv2.ErrorCodePluginIDInvalid: // Intentionally omitted - case pdv1.ErrorStatusInvalidRPCCredentials: + case pdv2.ErrorCodePluginCmdInvalid: // Intentionally omitted - case pdv1.ErrorStatusRecordNotFound: - return v1.ErrorCodeRecordNotFound - case pdv1.ErrorStatusInvalidToken: - return v1.ErrorCodeRecordTokenInvalid - case pdv1.ErrorStatusRecordLocked: - return v1.ErrorCodeRecordLocked - case pdv1.ErrorStatusInvalidRecordState: - return v1.ErrorCodeRecordStateInvalid } return v1.ErrorCodeInvalid } diff --git a/politeiawww/records/events.go b/politeiawww/records/events.go index 469bc0977..2099ccb03 100644 --- a/politeiawww/records/events.go +++ b/politeiawww/records/events.go @@ -29,12 +29,10 @@ type EventNew struct { // EventEdit is the event data for the EventTypeEdit. type EventEdit struct { User user.User - State string Record v1.Record } // EventSetStatus is the event data for the EventTypeSetStatus. type EventSetStatus struct { - State string Record v1.Record } diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index a2486b541..a64495628 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -10,7 +10,7 @@ import ( "fmt" "time" - pdv1 "github.com/decred/politeia/politeiad/api/v1" + pdv2 "github.com/decred/politeia/politeiad/api/v2" "github.com/decred/politeia/politeiad/plugins/usermd" v1 "github.com/decred/politeia/politeiawww/api/records/v1" "github.com/decred/politeia/politeiawww/config" @@ -49,23 +49,21 @@ func (r *Records) processNew(ctx context.Context, n v1.New, u user.User) (*v1.Ne if err != nil { return nil, err } - metadata := []pdv1.MetadataStream{ + metadata := []pdv2.MetadataStream{ { PluginID: usermd.PluginID, - ID: usermd.MDStreamIDUserMetadata, + StreamID: usermd.StreamIDUserMetadata, Payload: string(b), }, } // Save record to politeiad f := convertFilesToPD(n.Files) - cr, err := r.politeiad.NewRecord(ctx, metadata, f) + pdr, err := r.politeiad.RecordNew(ctx, metadata, f) if err != nil { return nil, err } - - // Get full record - rc, err := r.record(ctx, v1.RecordStateUnvetted, cr.Token, "") + rc, err := r.convertRecordToV1(*pdr) if err != nil { return nil, err } @@ -100,7 +98,7 @@ func (r *Records) processNew(ctx context.Context, n v1.New, u user.User) (*v1.Ne // filesToDel returns the names of the files that are included in the current // files but are not included in updated files. These are the files that need // to be deleted from a record on update. -func filesToDel(current []pdv1.File, updated []pdv1.File) []string { +func filesToDel(current []pdv2.File, updated []pdv2.File) []string { curr := make(map[string]struct{}, len(current)) // [name]struct for _, v := range updated { curr[v.Name] = struct{}{} @@ -129,25 +127,9 @@ func (r *Records) processEdit(ctx context.Context, e v1.Edit, u user.User) (*v1. } // Get current record - var ( - curr *pdv1.Record - err error - ) - switch e.State { - case v1.RecordStateUnvetted: - curr, err = r.politeiad.GetUnvetted(ctx, e.Token, "") - if err != nil { - return nil, err - } - case v1.RecordStateVetted: - curr, err = r.politeiad.GetVetted(ctx, e.Token, "") - if err != nil { - return nil, err - } - default: - return nil, v1.UserErrorReply{ - ErrorCode: v1.ErrorCodeRecordStateInvalid, - } + curr, err := r.politeiad.RecordGet(ctx, e.Token, 0) + if err != nil { + return nil, err } // Setup files @@ -164,38 +146,25 @@ func (r *Records) processEdit(ctx context.Context, e v1.Edit, u user.User) (*v1. if err != nil { return nil, err } - mdOverwrite := []pdv1.MetadataStream{ + mdOverwrite := []pdv2.MetadataStream{ { PluginID: usermd.PluginID, - ID: usermd.MDStreamIDUserMetadata, + StreamID: usermd.StreamIDUserMetadata, Payload: string(b), }, } - mdAppend := []pdv1.MetadataStream{} + mdAppend := []pdv2.MetadataStream{} // Save update to politeiad - var pdr *pdv1.Record - switch e.State { - case v1.RecordStateUnvetted: - pdr, err = r.politeiad.UpdateUnvetted(ctx, e.Token, mdAppend, - mdOverwrite, filesAdd, filesDel) - if err != nil { - return nil, err - } - case v1.RecordStateVetted: - pdr, err = r.politeiad.UpdateVetted(ctx, e.Token, mdAppend, - mdOverwrite, filesAdd, filesDel) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("invalid state %v", e.State) + pdr, err := r.politeiad.RecordEdit(ctx, e.Token, mdAppend, + mdOverwrite, filesAdd, filesDel) + if err != nil { + return nil, err } - - rc := convertRecordToV1(*pdr, e.State) + rc := convertRecordToV1(*pdr) recordPopulateUserData(&rc, u) - log.Infof("Record edited: %v %v", e.State, rc.CensorshipRecord.Token) + log.Infof("Record edited: %v", rc.CensorshipRecord.Token) for k, f := range rc.Files { log.Debugf("%02v: %v", k, f.Name) } @@ -204,7 +173,6 @@ func (r *Records) processEdit(ctx context.Context, e v1.Edit, u user.User) (*v1. r.events.Emit(EventTypeEdit, EventEdit{ User: u, - State: e.State, Record: rc, }) @@ -238,56 +206,28 @@ func (r *Records) processSetStatus(ctx context.Context, ss v1.SetStatus, u user. if err != nil { return nil, err } - mdAppend := []pdv1.MetadataStream{ + mdAppend := []pdv2.MetadataStream{ { PluginID: usermd.PluginID, - ID: usermd.MDStreamIDStatusChanges, + StreamID: usermd.StreamIDStatusChanges, Payload: string(b), }, } - mdOverwrite := []pdv1.MetadataStream{} + mdOverwrite := []pdv2.MetadataStream{} // Send politeiad request - var ( - s = convertStatusToPD(ss.Status) - pdr *pdv1.Record - ) - switch ss.State { - case v1.RecordStateUnvetted: - pdr, err = r.politeiad.SetUnvettedStatus(ctx, ss.Token, - s, mdAppend, mdOverwrite) - if err != nil { - return nil, err - } - case v1.RecordStateVetted: - pdr, err = r.politeiad.SetVettedStatus(ctx, ss.Token, - s, mdAppend, mdOverwrite) - if err != nil { - return nil, err - } - default: - return nil, v1.UserErrorReply{ - ErrorCode: v1.ErrorCodeRecordStateInvalid, - } - } - - // Convert the record. The state may need to be updated if the - // record was made public. - var state string - switch ss.Status { - case v1.RecordStatusPublic: - // Flip state from unvetted to vetted - state = pdv1.RecordStateVetted - default: - state = ss.State + s := convertStatusToPD(ss.Status) + pdr, err := r.politeiad.RecordSetStatus(ctx, ss.Token, s, + mdAppend, mdOverwrite) + if err != nil { + return nil, err } - rc := convertRecordToV1(*pdr, state) + rc := convertRecordToV1(*pdr) recordPopulateUserData(&rc, u) // Emit event r.events.Emit(EventTypeSetStatus, EventSetStatus{ - State: state, Record: rc, }) @@ -296,60 +236,11 @@ func (r *Records) processSetStatus(ctx context.Context, ss v1.SetStatus, u user. }, nil } -// record returns a version of a record from politeiad. If version is an empty -// string then the most recent version will be returned. -func (r *Records) record(ctx context.Context, state, token, version string) (*v1.Record, error) { - var ( - pdr *pdv1.Record - err error - ) - switch state { - case v1.RecordStateUnvetted: - pdr, err = r.politeiad.GetUnvetted(ctx, token, version) - if err != nil { - return nil, err - } - case v1.RecordStateVetted: - pdr, err = r.politeiad.GetVetted(ctx, token, version) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("invalid state %v", state) - } - - rc := convertRecordToV1(*pdr, state) - - // Fill in user data - userID := userIDFromMetadataStreams(rc.Metadata) - uid, err := uuid.Parse(userID) - if err != nil { - return nil, err - } - u, err := r.userdb.UserGetById(uid) - if err != nil { - return nil, err - } - recordPopulateUserData(&rc, *u) - - return &rc, nil -} - func (r *Records) processDetails(ctx context.Context, d v1.Details, u *user.User) (*v1.DetailsReply, error) { - log.Tracef("processDetails: %v %v %v", d.State, d.Token, d.Version) - - // Verify state - switch d.State { - case v1.RecordStateUnvetted, v1.RecordStateVetted: - // Allowed; continue - default: - return nil, v1.UserErrorReply{ - ErrorCode: v1.ErrorCodeRecordStateInvalid, - } - } + log.Tracef("processDetails: %v %v", d.Token, d.Version) // Get record - rc, err := r.record(ctx, d.State, d.Token, d.Version) + rc, err := r.record(ctx, d.Token, d.Version) if err != nil { return nil, err } @@ -358,7 +249,7 @@ func (r *Records) processDetails(ctx context.Context, d v1.Details, u *user.User // unvetted record files. Remove files if the user is not an admin // or the author. This is a public route so a user may not be // present. - if d.State == v1.RecordStateUnvetted { + if rc.State == v1.RecordStateUnvetted { var ( authorID = userIDFromMetadataStreams(rc.Metadata) isAuthor = u != nil && u.ID.String() == authorID @@ -374,60 +265,8 @@ func (r *Records) processDetails(ctx context.Context, d v1.Details, u *user.User }, nil } -func (r *Records) records(ctx context.Context, state string, reqs []pdv1.RecordRequest) (map[string]v1.Record, error) { - var ( - pdr map[string]pdv1.Record - err error - ) - switch state { - case v1.RecordStateUnvetted: - pdr, err = r.politeiad.GetUnvettedBatch(ctx, reqs) - if err != nil { - return nil, err - } - case v1.RecordStateVetted: - pdr, err = r.politeiad.GetVettedBatch(ctx, reqs) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("invalid state %v", state) - } - - records := make(map[string]v1.Record, len(pdr)) - for k, v := range pdr { - rc := convertRecordToV1(v, state) - - // Fill in user data - userID := userIDFromMetadataStreams(rc.Metadata) - uid, err := uuid.Parse(userID) - if err != nil { - return nil, err - } - u, err := r.userdb.UserGetById(uid) - if err != nil { - return nil, err - } - recordPopulateUserData(&rc, *u) - - records[k] = rc - } - - return records, nil -} - func (r *Records) processRecords(ctx context.Context, rs v1.Records, u *user.User) (*v1.RecordsReply, error) { - log.Tracef("processRecords: %v %v", rs.State, len(rs.Requests)) - - // Verify state - switch rs.State { - case v1.RecordStateUnvetted, v1.RecordStateVetted: - // Allowed; continue - default: - return nil, v1.UserErrorReply{ - ErrorCode: v1.ErrorCodeRecordStateInvalid, - } - } + log.Tracef("processRecords: %v reqs", len(rs.Requests)) // Verify page size if len(rs.Requests) > v1.RecordsPageSize { @@ -440,7 +279,7 @@ func (r *Records) processRecords(ctx context.Context, rs v1.Records, u *user.Use // Get records reqs := convertRequestsToPD(rs.Requests) - records, err := r.records(ctx, rs.State, reqs) + records, err := r.records(ctx, reqs) if err != nil { return nil, err } @@ -455,21 +294,22 @@ func (r *Records) processInventory(ctx context.Context, i v1.Inventory, u *user. // The inventory arguments are optional. If a status is provided // then they all arguments must be provided. - var s pdv1.RecordStatusT + var ( + state pdv2.RecordStateT + status pdv2.RecordStatusT + ) if i.Status != v1.RecordStatusInvalid { // Verify state - switch i.State { - case v1.RecordStateUnvetted, v1.RecordStateVetted: - // Allowed; continue - default: + state = convertStateToPD(i.State) + if state == pdv2.RecordStateInvalid { return nil, v1.UserErrorReply{ ErrorCode: v1.ErrorCodeRecordStateInvalid, } } // Verify status - s = convertStatusToPD(i.Status) - if s == pdv1.RecordStatusInvalid { + status = convertStatusToPD(i.Status) + if status == pdv2.RecordStatusInvalid { return nil, v1.UserErrorReply{ ErrorCode: v1.ErrorCodeRecordStatusInvalid, } @@ -477,81 +317,70 @@ func (r *Records) processInventory(ctx context.Context, i v1.Inventory, u *user. } // Get inventory - ir, err := r.politeiad.InventoryByStatus(ctx, i.State, s, i.Page) + ir, err := r.politeiad.Inventory(ctx, state, status, i.Page) if err != nil { return nil, err } - unvetted := make(map[string][]string, len(ir.Unvetted)) - vetted := make(map[string][]string, len(ir.Vetted)) - for k, v := range ir.Vetted { - ks := v1.RecordStatuses[convertStatusToV1(k)] - vetted[ks] = v - } - // Only admins are allowed to retrieve unvetted tokens. A user may // or may not exist. - if u != nil && u.Admin { - for k, v := range ir.Unvetted { - ks := v1.RecordStatuses[convertStatusToV1(k)] - unvetted[ks] = v - } + if u == nil || !u.Admin { + ir.Unvetted = map[string][]string{} } return &v1.InventoryReply{ - Unvetted: unvetted, - Vetted: vetted, + Unvetted: ir.Unvetted, + Vetted: ir.Vetted, }, nil } func (r *Records) processTimestamps(ctx context.Context, t v1.Timestamps, isAdmin bool) (*v1.TimestampsReply, error) { - log.Tracef("processTimestamps: %v %v %v", t.State, t.Token, t.Version) + log.Tracef("processTimestamps: %v %v", t.Token, t.Version) // Get record timestamps - var ( - rt *pdv1.RecordTimestamps - err error - ) - switch t.State { - case v1.RecordStateUnvetted: - rt, err = r.politeiad.GetUnvettedTimestamps(ctx, t.Token, t.Version) - if err != nil { - return nil, err - } - case v1.RecordStateVetted: - rt, err = r.politeiad.GetVettedTimestamps(ctx, t.Token, t.Version) - if err != nil { - return nil, err - } - default: - return nil, v1.UserErrorReply{ - ErrorCode: v1.ErrorCodeRecordStateInvalid, - } + rt, err := r.politeiad.RecordGetTimestamps(ctx, t.Token, t.Version) + if err != nil { + return nil, err } var ( recordMD = convertTimestampToV1(rt.RecordMetadata) - metadata = make(map[string]v1.Timestamp, len(rt.Files)) + metadata = make(map[string]map[uint32]v1.Timestamp, len(rt.Files)) files = make(map[string]v1.Timestamp, len(rt.Files)) ) - for k, v := range rt.Metadata { - metadata[k] = convertTimestampToV1(v) + for pluginID, v := range rt.Metadata { + streams, ok := metadata[pluginID] + if !ok { + streams = make(map[uint32]v1.Timestamp, 16) + } + for streamID, ts := range v { + streams[streamID] = convertTimestampToV1(ts) + } + metadata[pluginID] = streams } for k, v := range rt.Files { files[k] = convertTimestampToV1(v) } + // Get the record. We need to know the record state. + rc, err := r.record(ctx, t.Token, t.Version) + if err != nil { + return nil, err + } + // Unvetted data blobs are stripped if the user is not an admin. // The rest of the timestamp is still returned. - if t.State == v1.RecordStateUnvetted && !isAdmin { + if rc.State == v1.RecordStateUnvetted && !isAdmin { recordMD.Data = "" for k, v := range files { v.Data = "" files[k] = v } - for k, v := range metadata { - v.Data = "" - metadata[k] = v + for _, streams := range metadata { + for streamID, ts := range streams { + ts.Data = "" + streams[streamID] = ts + } } } @@ -565,45 +394,79 @@ func (r *Records) processTimestamps(ctx context.Context, t v1.Timestamps, isAdmi func (r *Records) processUserRecords(ctx context.Context, ur v1.UserRecords, u *user.User) (*v1.UserRecordsReply, error) { log.Tracef("processUserRecords: %v", ur.UserID) - reply, err := r.politeiad.UserRecords(ctx, ur.UserID) + urr, err := r.politeiad.UserRecords(ctx, ur.UserID) if err != nil { return nil, err } - // Unpack reply - var ( - unvetted = make([]string, 0) - vetted = make([]string, 0) - ) - tokens, ok := reply[v1.RecordStateUnvetted] - if ok { - unvetted = tokens - } - tokens, ok = reply[v1.RecordStateVetted] - if ok { - vetted = tokens - } - // Determine if unvetted tokens should be returned switch { case u == nil: // No user session. Remove unvetted. - unvetted = []string{} + urr.Unvetted = []string{} case u.Admin: // User is an admin. Return unvetted. case ur.UserID == u.ID.String(): // User is requesting their own records. Return unvetted. default: // Remove unvetted for all other cases - unvetted = []string{} + urr.Unvetted = []string{} } return &v1.UserRecordsReply{ - Unvetted: unvetted, - Vetted: vetted, + Unvetted: urr.Unvetted, + Vetted: urr.Vetted, }, nil } +// record returns a version of a record from politeiad. If version is an empty +// string then the most recent version will be returned. +func (r *Records) record(ctx context.Context, token string, version uint32) (*v1.Record, error) { + pdr, err := r.politeiad.RecordGet(ctx, token, version) + if err != nil { + return nil, err + } + return r.convertRecordToV1(*pdr) +} + +func (r *Records) records(ctx context.Context, reqs []pdv2.RecordRequest) (map[string]v1.Record, error) { + // Get records + pdr, err := r.politeiad.RecordGetBatch(ctx, reqs) + if err != nil { + return nil, err + } + + // Convert records + records := make(map[string]v1.Record, len(pdr)) + for k, v := range pdr { + rc, err := r.convertRecordToV1(v) + if err != nil { + return nil, err + } + records[k] = *rc + } + + return records, nil +} + +func (r *Records) convertRecordToV1(pdr pdv2.Record) (*v1.Record, error) { + rc := convertRecordToV1(pdr) + + // Fill in user data + userID := userIDFromMetadataStreams(rc.Metadata) + uid, err := uuid.Parse(userID) + if err != nil { + return nil, err + } + u, err := r.userdb.UserGetById(uid) + if err != nil { + return nil, err + } + recordPopulateUserData(&rc, *u) + + return &rc, nil +} + // recordPopulateUserData populates the record with user data that is not // stored in politeiad. func recordPopulateUserData(r *v1.Record, u user.User) { @@ -616,7 +479,7 @@ func userMetadataDecode(ms []v1.MetadataStream) (*usermd.UserMetadata, error) { var userMD *usermd.UserMetadata for _, v := range ms { if v.PluginID != usermd.PluginID || - v.ID != usermd.MDStreamIDUserMetadata { + v.StreamID != usermd.StreamIDUserMetadata { // Not the mdstream we're looking for continue } @@ -644,21 +507,31 @@ func userIDFromMetadataStreams(ms []v1.MetadataStream) string { return um.UserID } -func convertStatusToV1(s pdv1.RecordStatusT) v1.RecordStatusT { +func convertStateToV1(s pdv2.RecordStateT) v1.RecordStateT { + switch s { + case pdv2.RecordStateUnvetted: + return v1.RecordStateUnvetted + case pdv2.RecordStateVetted: + return v1.RecordStateVetted + } + return v1.RecordStateInvalid +} + +func convertStatusToV1(s pdv2.RecordStatusT) v1.RecordStatusT { switch s { - case pdv1.RecordStatusNotReviewed: + case pdv2.RecordStatusUnreviewed: return v1.RecordStatusUnreviewed - case pdv1.RecordStatusPublic: + case pdv2.RecordStatusPublic: return v1.RecordStatusPublic - case pdv1.RecordStatusCensored: + case pdv2.RecordStatusCensored: return v1.RecordStatusCensored - case pdv1.RecordStatusArchived: + case pdv2.RecordStatusArchived: return v1.RecordStatusArchived } return v1.RecordStatusInvalid } -func convertFilesToV1(f []pdv1.File) []v1.File { +func convertFilesToV1(f []pdv2.File) []v1.File { files := make([]v1.File, 0, len(f)) for _, v := range f { files = append(files, v1.File{ @@ -671,24 +544,24 @@ func convertFilesToV1(f []pdv1.File) []v1.File { return files } -func convertMetadataStreamsToV1(ms []pdv1.MetadataStream) []v1.MetadataStream { +func convertMetadataStreamsToV1(ms []pdv2.MetadataStream) []v1.MetadataStream { metadata := make([]v1.MetadataStream, 0, len(ms)) for _, v := range ms { metadata = append(metadata, v1.MetadataStream{ PluginID: v.PluginID, - ID: v.ID, + StreamID: v.StreamID, Payload: v.Payload, }) } return metadata } -func convertRecordToV1(r pdv1.Record, state string) v1.Record { +func convertRecordToV1(r pdv2.Record) v1.Record { // User fields that are not part of the politeiad record have // been intentionally left blank. These fields must be pulled // from the user database. return v1.Record{ - State: state, + State: convertStateToV1(r.State), Status: convertStatusToV1(r.Status), Version: r.Version, Timestamp: r.Timestamp, @@ -703,7 +576,7 @@ func convertRecordToV1(r pdv1.Record, state string) v1.Record { } } -func convertProofToV1(p pdv1.Proof) v1.Proof { +func convertProofToV1(p pdv2.Proof) v1.Proof { return v1.Proof{ Type: p.Type, Digest: p.Digest, @@ -713,7 +586,7 @@ func convertProofToV1(p pdv1.Proof) v1.Proof { } } -func convertTimestampToV1(t pdv1.Timestamp) v1.Timestamp { +func convertTimestampToV1(t pdv2.Timestamp) v1.Timestamp { proofs := make([]v1.Proof, 0, len(t.Proofs)) for _, v := range t.Proofs { proofs = append(proofs, convertProofToV1(v)) @@ -727,10 +600,10 @@ func convertTimestampToV1(t pdv1.Timestamp) v1.Timestamp { } } -func convertFilesToPD(f []v1.File) []pdv1.File { - files := make([]pdv1.File, 0, len(f)) +func convertFilesToPD(f []v1.File) []pdv2.File { + files := make([]pdv2.File, 0, len(f)) for _, v := range f { - files = append(files, pdv1.File{ + files = append(files, pdv2.File{ Name: v.Name, MIME: v.MIME, Digest: v.Digest, @@ -740,24 +613,34 @@ func convertFilesToPD(f []v1.File) []pdv1.File { return files } -func convertStatusToPD(s v1.RecordStatusT) pdv1.RecordStatusT { +func convertStateToPD(s v1.RecordStateT) pdv2.RecordStateT { + switch s { + case v1.RecordStateUnvetted: + return pdv2.RecordStateUnvetted + case v1.RecordStateVetted: + return pdv2.RecordStateVetted + } + return pdv2.RecordStateInvalid +} + +func convertStatusToPD(s v1.RecordStatusT) pdv2.RecordStatusT { switch s { case v1.RecordStatusUnreviewed: - return pdv1.RecordStatusNotReviewed + return pdv2.RecordStatusUnreviewed case v1.RecordStatusPublic: - return pdv1.RecordStatusPublic + return pdv2.RecordStatusPublic case v1.RecordStatusCensored: - return pdv1.RecordStatusCensored + return pdv2.RecordStatusCensored case v1.RecordStatusArchived: - return pdv1.RecordStatusArchived + return pdv2.RecordStatusArchived } - return pdv1.RecordStatusInvalid + return pdv2.RecordStatusInvalid } -func convertRequestsToPD(reqs []v1.RecordRequest) []pdv1.RecordRequest { - r := make([]pdv1.RecordRequest, 0, len(reqs)) +func convertRequestsToPD(reqs []v1.RecordRequest) []pdv2.RecordRequest { + r := make([]pdv2.RecordRequest, 0, len(reqs)) for _, v := range reqs { - r = append(r, pdv1.RecordRequest{ + r = append(r, pdv2.RecordRequest{ Token: v.Token, Version: v.Version, Filenames: v.Filenames, diff --git a/politeiawww/testing.go b/politeiawww/testing.go index 326359691..55e87d4c7 100644 --- a/politeiawww/testing.go +++ b/politeiawww/testing.go @@ -24,7 +24,6 @@ import ( "github.com/decred/politeia/politeiad/api/v1/mime" "github.com/decred/politeia/politeiad/testpoliteiad" cms "github.com/decred/politeia/politeiawww/api/cms/v1" - piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/mail" @@ -56,7 +55,7 @@ func errToStr(e error) string { // by default but can be filled in with random rgb colors by setting the // addColor parameter to true. The png without color will be ~3kB. The png with // color will be ~2MB. -func newFilePNG(t *testing.T, addColor bool) *piv1.File { +func newFilePNG(t *testing.T, addColor bool) *www.File { t.Helper() b := new(bytes.Buffer) @@ -87,7 +86,7 @@ func newFilePNG(t *testing.T, addColor bool) *piv1.File { t.Fatalf("%v", err) } - return &piv1.File{ + return &www.File{ Name: hex.EncodeToString(r) + ".png", MIME: mime.DetectMimeType(b.Bytes()), Digest: hex.EncodeToString(util.Digest(b.Bytes())), diff --git a/politeiawww/ticketvote/error.go b/politeiawww/ticketvote/error.go index 63d98813f..2165400ae 100644 --- a/politeiawww/ticketvote/error.go +++ b/politeiawww/ticketvote/error.go @@ -12,7 +12,7 @@ import ( "strings" "time" - pdv1 "github.com/decred/politeia/politeiad/api/v1" + pdv2 "github.com/decred/politeia/politeiad/api/v2" pdclient "github.com/decred/politeia/politeiad/client" v1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" "github.com/decred/politeia/util" @@ -119,12 +119,12 @@ func convertPDErrorCode(errCode int) v1.ErrorCodeT { // This list is only populated with politeiad errors that we expect // for the ticketvote plugin commands. Any politeiad errors not // included in this list will cause politeiawww to 500. - switch pdv1.ErrorStatusT(errCode) { - case pdv1.ErrorStatusRecordNotFound: + switch pdv2.ErrorCodeT(errCode) { + case pdv2.ErrorCodeRecordNotFound: return v1.ErrorCodeRecordNotFound - case pdv1.ErrorStatusInvalidToken: + case pdv2.ErrorCodeTokenInvalid: return v1.ErrorCodeTokenInvalid - case pdv1.ErrorStatusRecordLocked: + case pdv2.ErrorCodeRecordLocked: return v1.ErrorCodeRecordLocked } return v1.ErrorCodeInvalid diff --git a/politeiawww/ticketvote/process.go b/politeiawww/ticketvote/process.go index 92023ef0f..aded5a937 100644 --- a/politeiawww/ticketvote/process.go +++ b/politeiawww/ticketvote/process.go @@ -7,7 +7,6 @@ package ticketvote import ( "context" - pdv1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/plugins/ticketvote" v1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" "github.com/decred/politeia/politeiawww/user" @@ -25,7 +24,7 @@ func (t *TicketVote) processAuthorize(ctx context.Context, a v1.Authorize, u use } // Verify user is the record author - authorID, err := t.politeiad.Author(ctx, pdv1.RecordStateVetted, a.Token) + authorID, err := t.politeiad.Author(ctx, a.Token) if err != nil { return nil, err } diff --git a/politeiawww/ticketvote/ticketvote.go b/politeiawww/ticketvote/ticketvote.go index 2118d57a2..a77855fca 100644 --- a/politeiawww/ticketvote/ticketvote.go +++ b/politeiawww/ticketvote/ticketvote.go @@ -10,7 +10,7 @@ import ( "net/http" "strconv" - pdv1 "github.com/decred/politeia/politeiad/api/v1" + pdv2 "github.com/decred/politeia/politeiad/api/v2" pdclient "github.com/decred/politeia/politeiad/client" "github.com/decred/politeia/politeiad/plugins/ticketvote" v1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" @@ -273,7 +273,7 @@ func (t *TicketVote) HandleTimestamps(w http.ResponseWriter, r *http.Request) { } // New returns a new TicketVote context. -func New(cfg *config.Config, pdc *pdclient.Client, s *sessions.Sessions, e *events.Manager, plugins []pdv1.Plugin) (*TicketVote, error) { +func New(cfg *config.Config, pdc *pdclient.Client, s *sessions.Sessions, e *events.Manager, plugins []pdv2.Plugin) (*TicketVote, error) { // Parse plugin settings var ( linkByPeriodMin int64 diff --git a/politeiawww/www.go b/politeiawww/www.go index bd80c27f9..09d435c9c 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -26,7 +26,7 @@ import ( "github.com/decred/politeia/mdstream" pd "github.com/decred/politeia/politeiad/api/v1" - pdv1 "github.com/decred/politeia/politeiad/api/v1" + pdv2 "github.com/decred/politeia/politeiad/api/v2" pdclient "github.com/decred/politeia/politeiad/client" cms "github.com/decred/politeia/politeiawww/api/cms/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" @@ -286,14 +286,14 @@ func (p *politeiawww) addRoute(method string, routeVersion string, route string, // getPluginInventory returns the politeiad plugin inventory. If a politeiad // connection cannot be made, the call will be retried every 5 seconds for up // to 1000 tries. -func (p *politeiawww) getPluginInventory() ([]pdv1.Plugin, error) { +func (p *politeiawww) getPluginInventory() ([]pdv2.Plugin, error) { // Attempt to fetch the plugin inventory from politeiad until // either it is successful or the maxRetries has been exceeded. var ( done bool maxRetries = 1000 sleepInterval = 5 * time.Second - plugins = make([]pdv1.Plugin, 0, 32) + plugins = make([]pdv2.Plugin, 0, 32) ctx = context.Background() ) for retries := 0; !done; retries++ { @@ -337,18 +337,6 @@ func (p *politeiawww) setupCMS() error { } p.wsDcrdata = ws - // Verify politeiad plugins - pluginFound := false - for _, plugin := range p.plugins { - if plugin.ID == "cms" { - pluginFound = true - break - } - } - if !pluginFound { - return fmt.Errorf("politeiad plugin 'cms' not found") - } - // Setup cmsdb net := filepath.Base(p.cfg.DataDir) p.cmsDB, err = cmsdb.New(p.cfg.DBHost, net, p.cfg.DBRootCert, @@ -696,7 +684,6 @@ func _main() error { if err != nil { return fmt.Errorf("getPluginInventory: %v", err) } - p.plugins = plugins // Setup email-userID cache err = p.initUserEmailsCache() From 96bd0b2f9cadf1964b833398c5316f324ee1456c Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 8 Mar 2021 08:49:19 -0600 Subject: [PATCH 366/449] pictl: Get working. --- .../backendv2/tstorebe/plugins/usermd/cmds.go | 7 +- .../tstorebe/plugins/usermd/hooks.go | 5 +- politeiawww/api/pi/v1/v1.go | 38 +++++ politeiawww/client/error.go | 3 - politeiawww/cmd/pictl/cmdcommenttimestamps.go | 4 +- politeiawww/cmd/pictl/cmdproposaldetails.go | 24 +--- politeiawww/cmd/pictl/cmdproposaledit.go | 20 +-- politeiawww/cmd/pictl/cmdproposalinv.go | 30 ++-- politeiawww/cmd/pictl/cmdproposals.go | 18 --- politeiawww/cmd/pictl/cmdproposalstatusset.go | 55 ++++--- .../cmd/pictl/cmdproposaltimestamps.go | 38 ++--- politeiawww/cmd/pictl/cmdseedproposals.go | 29 +--- politeiawww/cmd/pictl/cmdvoteauthorize.go | 7 +- politeiawww/cmd/pictl/cmdvotestart.go | 15 +- politeiawww/cmd/pictl/cmdvotetimestamps.go | 4 +- politeiawww/cmd/pictl/proposal.go | 134 +----------------- 16 files changed, 117 insertions(+), 314 deletions(-) diff --git a/politeiad/backendv2/tstorebe/plugins/usermd/cmds.go b/politeiad/backendv2/tstorebe/plugins/usermd/cmds.go index c847f2514..55233fd8b 100644 --- a/politeiad/backendv2/tstorebe/plugins/usermd/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/usermd/cmds.go @@ -51,10 +51,11 @@ func (p *userPlugin) cmdUserRecords(payload string) (string, error) { return "", err } + // TODO fix the user records reply + _ = uc + // Prepare reply - urr := usermd.UserRecordsReply{ - Records: uc.Tokens, - } + urr := usermd.UserRecordsReply{} reply, err := json.Marshal(urr) if err != nil { return "", err diff --git a/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go b/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go index 7c7f28f82..20b78f59f 100644 --- a/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go @@ -308,8 +308,9 @@ func statusChangeMetadataVerify(rm backend.RecordMetadata, metadata []backend.Me } // Verify signature - s := strconv.FormatUint(uint64(scm.Status), 10) - msg := scm.Token + scm.Version + s + scm.Reason + status := strconv.FormatUint(uint64(scm.Status), 10) + version := strconv.FormatUint(uint64(scm.Version), 10) + msg := scm.Token + version + status + scm.Reason err = util.VerifySignature(scm.Signature, scm.PublicKey, msg) if err != nil { return convertSignatureError(err) diff --git a/politeiawww/api/pi/v1/v1.go b/politeiawww/api/pi/v1/v1.go index 43067e3f4..bff751ec8 100644 --- a/politeiawww/api/pi/v1/v1.go +++ b/politeiawww/api/pi/v1/v1.go @@ -25,3 +25,41 @@ type PolicyReply struct { NameLengthMax uint32 `json:"namelengthmax"` // In characters NameSupportedChars []string `json:"namesupportedchars"` } + +const ( + // FileNameIndexFile is the file name of the proposal markdown + // file that contains the main proposal contents. All proposal + // submissions must contain an index file. + FileNameIndexFile = "index.md" + + // FileNameProposalMetadata is the file name of the user submitted + // ProposalMetadata. All proposal submissions must contain a + // proposal metadata file. + FileNameProposalMetadata = "proposalmetadata.json" + + // FileNameVoteMetadata is the file name of the user submitted + // VoteMetadata. This file will only be present when proposals + // are hosting or participating in certain types of votes. + FileNameVoteMetadata = "votemetadata.json" +) + +// ProposalMetadata contains metadata that is specified by the user on proposal +// submission. +type ProposalMetadata struct { + Name string `json:"name"` // Proposal name +} + +// VoteMetadata is metadata that is specified by the user on proposal +// submission in order to host or participate in a runoff vote. +type VoteMetadata struct { + // LinkBy is set when the user intends for the proposal to be the + // parent proposal in a runoff vote. It is a UNIX timestamp that + // serves as the deadline for other proposals to declare their + // intent to participate in the runoff vote. + LinkBy int64 `json:"linkby,omitempty"` + + // LinkTo is the censorship token of a runoff vote parent proposal. + // It is set when a proposal is being submitted as a vote options + // in the runoff vote. + LinkTo string `json:"linkto,omitempty"` +} diff --git a/politeiawww/client/error.go b/politeiawww/client/error.go index 174f3d908..fe87d46f5 100644 --- a/politeiawww/client/error.go +++ b/politeiawww/client/error.go @@ -14,7 +14,6 @@ import ( tkplugin "github.com/decred/politeia/politeiad/plugins/ticketvote" umplugin "github.com/decred/politeia/politeiad/plugins/usermd" cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" - piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" ) @@ -62,8 +61,6 @@ func apiUserErr(api string, e ErrorReply) string { switch api { case cmv1.APIRoute: errMsg = cmv1.ErrorCodes[cmv1.ErrorCodeT(e.ErrorCode)] - case piv1.APIRoute: - errMsg = piv1.ErrorCodes[piv1.ErrorCodeT(e.ErrorCode)] case rcv1.APIRoute: errMsg = rcv1.ErrorCodes[rcv1.ErrorCodeT(e.ErrorCode)] case tkv1.APIRoute: diff --git a/politeiawww/cmd/pictl/cmdcommenttimestamps.go b/politeiawww/cmd/pictl/cmdcommenttimestamps.go index 7e338a22a..c58a47905 100644 --- a/politeiawww/cmd/pictl/cmdcommenttimestamps.go +++ b/politeiawww/cmd/pictl/cmdcommenttimestamps.go @@ -7,8 +7,8 @@ package main import ( "fmt" - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tstorebe/tstore" + backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/tstore" cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" pclient "github.com/decred/politeia/politeiawww/client" ) diff --git a/politeiawww/cmd/pictl/cmdproposaldetails.go b/politeiawww/cmd/pictl/cmdproposaldetails.go index 420f8e970..ca58625b4 100644 --- a/politeiawww/cmd/pictl/cmdproposaldetails.go +++ b/politeiawww/cmd/pictl/cmdproposaldetails.go @@ -13,13 +13,8 @@ import ( type cmdProposalDetails struct { Args struct { Token string `positional-arg-name:"token"` - Version string `postional-arg-name:"version" optional:"true"` + Version uint32 `postional-arg-name:"version" optional:"true"` } `positional-args:"true"` - - // Unvetted is used to indicate that the state of the requested - // proposal is unvetted. If this flag is not used it will be - // assumed that a vetted proposal is being requested. - Unvetted bool `long:"unvetted" optional:"true"` } // Execute executes the cmdProposalDetails command. @@ -39,18 +34,8 @@ func (c *cmdProposalDetails) Execute(args []string) error { return err } - // Setup state - var state string - switch { - case c.Unvetted: - state = rcv1.RecordStateUnvetted - default: - state = rcv1.RecordStateVetted - } - // Get proposal details d := rcv1.Details{ - State: state, Token: c.Args.Token, Version: c.Args.Version, } @@ -73,13 +58,8 @@ const proposalDetailsHelpMsg = `proposaldetails [flags] "token" "version" Retrieve a full proposal record. -This command defaults to retrieving vetted proposals unless the --unvetted flag -is used. This command accepts both the full tokens or the shortened token -prefixes. +This command accepts both the full tokens or the shortened token prefixes. Arguments: 1. token (string, required) Proposal token. - -Flags: - --unvetted (bool, optional) Retrieve an unvetted proposal. ` diff --git a/politeiawww/cmd/pictl/cmdproposaledit.go b/politeiawww/cmd/pictl/cmdproposaledit.go index a9a621e00..909036969 100644 --- a/politeiawww/cmd/pictl/cmdproposaledit.go +++ b/politeiawww/cmd/pictl/cmdproposaledit.go @@ -27,10 +27,6 @@ type cmdProposalEdit struct { Attachments []string `positional-arg-name:"attachmets"` } `positional-args:"true" optional:"true"` - // Unvetted is used to edit an unvetted proposal. If this flag is - // not used the command assumes the proposal is vetted. - Unvetted bool `long:"unvetted" optional:"true"` - // UseMD is a flag that is intended to make editing proposal // metadata easier by using exisiting proposal metadata values // instead of having to pass in specific values. @@ -115,15 +111,6 @@ func proposalEdit(c *cmdProposalEdit) (*rcv1.Record, error) { return nil, err } - // Setup state - var state string - switch { - case c.Unvetted: - state = rcv1.RecordStateUnvetted - default: - state = rcv1.RecordStateVetted - } - // Setup proposal files indexFileSize := 10000 // In bytes var files []rcv1.File @@ -153,7 +140,6 @@ func proposalEdit(c *cmdProposalEdit) (*rcv1.Record, error) { var curr *rcv1.Record if c.UseMD { d := rcv1.Details{ - State: state, Token: token, } curr, err = pc.RecordDetails(d) @@ -238,7 +224,6 @@ func proposalEdit(c *cmdProposalEdit) (*rcv1.Record, error) { return nil, err } e := rcv1.Edit{ - State: state, Token: token, Files: files, PublicKey: cfg.Identity.Public.String(), @@ -272,8 +257,7 @@ func proposalEdit(c *cmdProposalEdit) (*rcv1.Record, error) { // proposalEditHelpMsg is the printed to stdout by the help command. const proposalEditHelpMsg = `editproposal [flags] "token" "indexfile" "attachments" -Edit a proposal. This command assumes the proposal is a vetted record. If the -proposal is unvetted, the --unvetted flag must be used. +Edit an existing proposal. A proposal can be submitted as an RFP (Request for Proposals) by using either the --rfp flag or by manually specifying a link by deadline using the --linkby @@ -288,8 +272,6 @@ Arguments: 3. attachments (string, optional) Attachment files. Flags: - --unvetted (bool) Edit an unvetted record. - --usemd (bool) Use the existing proposal metadata. --name (string) Name of the proposal. diff --git a/politeiawww/cmd/pictl/cmdproposalinv.go b/politeiawww/cmd/pictl/cmdproposalinv.go index 9304ca28c..abcb8fe24 100644 --- a/politeiawww/cmd/pictl/cmdproposalinv.go +++ b/politeiawww/cmd/pictl/cmdproposalinv.go @@ -13,16 +13,10 @@ import ( // the inventory, categorized by status. type cmdProposalInv struct { Args struct { + State string `positional-arg-name:"state"` Status string `positional-arg-name:"status"` Page uint32 `positional-arg-name:"page"` } `positional-args:"true" optional:"true"` - - // Unvetted is used to indicate the state that should be sent in - // the inventory request. This flag is only required when - // requesting the inventory for a specific status. If a status - // argument is provided and this flag is not, it will be assumed - // that the state being requested is vetted. - Unvetted bool `long:"unvetted" optional:"true"` } // Execute executes the cmdProposalInv command. @@ -53,12 +47,14 @@ func proposalInv(c *cmdProposalInv) (*rcv1.InventoryReply, error) { } // Setup state - var state string - switch { - case c.Unvetted: - state = rcv1.RecordStateUnvetted - default: - state = rcv1.RecordStateVetted + var state rcv1.RecordStateT + if c.Args.State != "" { + // Parse state. This can be either the numeric state code or the + // human readable equivalent. + state, err = parseRecordState(c.Args.State) + if err != nil { + return nil, err + } } // Setup status and page number @@ -114,9 +110,7 @@ Valid statuses: abandoned Arguments: -1. status (string, optional) Status of tokens being requested. -2. page (uint32, optional) Page number. - -Flags: - --unvetted (bool, optional) Set status of an unvetted record. +1. state (string, optional) State of tokens being requested. +2. status (string, optional) Status of tokens being requested. +3. page (uint32, optional) Page number. ` diff --git a/politeiawww/cmd/pictl/cmdproposals.go b/politeiawww/cmd/pictl/cmdproposals.go index 1296b22ec..ffe824d76 100644 --- a/politeiawww/cmd/pictl/cmdproposals.go +++ b/politeiawww/cmd/pictl/cmdproposals.go @@ -15,11 +15,6 @@ type cmdProposals struct { Args struct { Tokens []string `positional-arg-name:"proposals" required:"true"` } `positional-args:"true"` - - // Unvetted is used to indicate the state of the proposals are - // unvetted. If this flag is not used it will be assumed that the - // proposals are vetted. - Unvetted bool `long:"unvetted" optional:"true"` } // Execute executes the cmdProposals command. @@ -39,15 +34,6 @@ func (c *cmdProposals) Execute(args []string) error { return err } - // Setup state - var state string - switch { - case c.Unvetted: - state = piv1.ProposalStateUnvetted - default: - state = piv1.ProposalStateVetted - } - // Get records reqs := make([]rcv1.RecordRequest, 0, len(c.Args.Tokens)) for _, v := range c.Args.Tokens { @@ -60,7 +46,6 @@ func (c *cmdProposals) Execute(args []string) error { }) } r := rcv1.Records{ - State: state, Requests: reqs, } records, err := pc.Records(r) @@ -93,9 +78,6 @@ is used. This command accepts both the full tokens or the token prefixes. Arguments: 1. tokens ([]string, required) Proposal tokens. -Flags: - --unvetted (bool, optional) Retrieve unvetted proposals. - Example: $ pictl proposals f6458c2d8d9ef41c 9f9af91cf609d839 917c6fde9bcc2118 $ pictl proposals f6458c2 9f9af91 917c6fd` diff --git a/politeiawww/cmd/pictl/cmdproposalstatusset.go b/politeiawww/cmd/pictl/cmdproposalstatusset.go index 41f6d2d25..09a103af6 100644 --- a/politeiawww/cmd/pictl/cmdproposalstatusset.go +++ b/politeiawww/cmd/pictl/cmdproposalstatusset.go @@ -20,13 +20,8 @@ type cmdProposalSetStatus struct { Token string `positional-arg-name:"token" required:"true"` Status string `positional-arg-name:"status" required:"true"` Reason string `positional-arg-name:"reason"` - Version string `positional-arg-name:"version"` + Version uint32 `positional-arg-name:"version"` } `positional-args:"true"` - - // Unvetted is used to indicate the state of the proposal is - // unvetted. If this flag is not used it will be assumed that - // the proposal is vetted. - Unvetted bool `long:"unvetted" optional:"true"` } // Execute executes the cmdProposalSetStatus command. @@ -63,15 +58,6 @@ func proposalSetStatus(c *cmdProposalSetStatus) (*rcv1.Record, error) { return nil, err } - // Setup state - var state string - switch { - case c.Unvetted: - state = rcv1.RecordStateUnvetted - default: - state = rcv1.RecordStateVetted - } - // Parse status. This can be either the numeric status code or the // human readable equivalent. status, err := parseRecordStatus(c.Args.Status) @@ -80,13 +66,12 @@ func proposalSetStatus(c *cmdProposalSetStatus) (*rcv1.Record, error) { } // Setup version - var version string - if c.Args.Version != "" { + var version uint32 + if c.Args.Version != 0 { version = c.Args.Version } else { // Get the version manually d := rcv1.Details{ - State: state, Token: c.Args.Token, } r, err := pc.RecordDetails(d) @@ -97,11 +82,11 @@ func proposalSetStatus(c *cmdProposalSetStatus) (*rcv1.Record, error) { } // Setup request - msg := c.Args.Token + version + strconv.Itoa(int(status)) + c.Args.Reason + msg := c.Args.Token + strconv.FormatUint(uint64(version), 10) + + strconv.Itoa(int(status)) + c.Args.Reason sig := cfg.Identity.SignMessage([]byte(msg)) ss := rcv1.SetStatus{ Token: c.Args.Token, - State: state, Version: version, Status: status, Reason: c.Args.Reason, @@ -134,6 +119,33 @@ func proposalSetStatus(c *cmdProposalSetStatus) (*rcv1.Record, error) { return &ssr.Record, nil } +func parseRecordState(state string) (rcv1.RecordStateT, error) { + // Parse status. This can be either the numeric status code or the + // human readable equivalent. + var ( + rc rcv1.RecordStateT + + states = map[string]rcv1.RecordStateT{ + "unvetted": rcv1.RecordStateUnvetted, + "vetted": rcv1.RecordStateVetted, + "1": rcv1.RecordStateUnvetted, + "2": rcv1.RecordStateVetted, + } + ) + u, err := strconv.ParseUint(state, 10, 32) + if err == nil { + // Numeric state code found + rc = rcv1.RecordStateT(u) + } else if s, ok := states[state]; ok { + // Human readable state code found + rc = s + } else { + return rc, fmt.Errorf("invalid state '%v'", state) + } + + return rc, nil +} + func parseRecordStatus(status string) (rcv1.RecordStatusT, error) { // Parse status. This can be either the numeric status code or the // human readable equivalent. @@ -189,7 +201,4 @@ Arguments: 3. message (string, optional) Status change message 4. version (string, optional) Proposal version. This will be retrieved from the backend if one is not provided. - -Flags: - --unvetted (bool, optional) Set status of an unvetted record. ` diff --git a/politeiawww/cmd/pictl/cmdproposaltimestamps.go b/politeiawww/cmd/pictl/cmdproposaltimestamps.go index 742f7be74..d9b0dd8d7 100644 --- a/politeiawww/cmd/pictl/cmdproposaltimestamps.go +++ b/politeiawww/cmd/pictl/cmdproposaltimestamps.go @@ -7,9 +7,8 @@ package main import ( "fmt" - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tstorebe/tstore" - piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" + backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/tstore" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" pclient "github.com/decred/politeia/politeiawww/client" ) @@ -18,13 +17,8 @@ import ( type cmdProposalTimestamps struct { Args struct { Token string `positional-arg-name:"token" required:"true"` - Version string `positional-arg-name:"version" optional:"true"` + Version uint32 `positional-arg-name:"version" optional:"true"` } `positional-args:"true"` - - // Unvetted is used to request the timestamps of an unvetted - // proposal. If this flag is not used it will be assume that the - // proposal is vetted. - Unvetted bool `long:"unvetted" optional:"true"` } // Execute executes the cmdProposalTimestamps command. @@ -44,18 +38,8 @@ func (c *cmdProposalTimestamps) Execute(args []string) error { return err } - // Setup state - var state string - switch { - case c.Unvetted: - state = piv1.ProposalStateUnvetted - default: - state = piv1.ProposalStateVetted - } - // Get timestamps t := rcv1.Timestamps{ - State: state, Token: c.Args.Token, Version: c.Args.Version, } @@ -69,10 +53,13 @@ func (c *cmdProposalTimestamps) Execute(args []string) error { if err != nil { return fmt.Errorf("verify proposal metadata timestamp: %v", err) } - for k, v := range tr.Metadata { - err = verifyTimestamp(v) - if err != nil { - return fmt.Errorf("verify metadata %v timestamp: %v", k, err) + for pluginID, v := range tr.Metadata { + for streamID, ts := range v { + err = verifyTimestamp(ts) + if err != nil { + return fmt.Errorf("verify metadata %v %v timestamp: %v", + pluginID, streamID, err) + } } } for k, v := range tr.Files { @@ -129,8 +116,5 @@ is used. Arguments: 1. token (string, required) Record token -2. version (string, optional) Record version - -Flags: - --unvetted (bool, optional) Request is for unvetted proposals. +2. version (uint32, optional) Record version ` diff --git a/politeiawww/cmd/pictl/cmdseedproposals.go b/politeiawww/cmd/pictl/cmdseedproposals.go index 6d97670db..9057dc27e 100644 --- a/politeiawww/cmd/pictl/cmdseedproposals.go +++ b/politeiawww/cmd/pictl/cmdseedproposals.go @@ -522,7 +522,6 @@ func proposalUnreviewed(u user, includeImages bool) (*rcv1.Record, error) { // Edit the proposal ce := cmdProposalEdit{ - Unvetted: true, Random: true, RandomImages: includeImages, } @@ -558,9 +557,7 @@ func proposalUnvettedCensored(author, admin user, includeImages bool) (*rcv1.Rec } // Censor the proposal - cs := cmdProposalSetStatus{ - Unvetted: true, - } + cs := cmdProposalSetStatus{} cs.Args.Token = r.CensorshipRecord.Token cs.Args.Status = strconv.Itoa(int(rcv1.RecordStatusCensored)) cs.Args.Reason = "Violates proposal rules." @@ -596,9 +593,7 @@ func proposalPublic(author, admin user, includeImages bool) (*rcv1.Record, error } // Make the proposal public - cs := cmdProposalSetStatus{ - Unvetted: true, - } + cs := cmdProposalSetStatus{} cs.Args.Token = r.CensorshipRecord.Token cs.Args.Status = strconv.Itoa(int(rcv1.RecordStatusPublic)) cs.Args.Version = r.Version @@ -621,7 +616,6 @@ func proposalPublic(author, admin user, includeImages bool) (*rcv1.Record, error // Edit the proposal ce := cmdProposalEdit{ - Unvetted: false, Random: true, RandomImages: includeImages, } @@ -658,9 +652,7 @@ func proposalVettedCensored(author, admin user, includeImages bool) (*rcv1.Recor } // Censor the proposal - cs := cmdProposalSetStatus{ - Unvetted: false, - } + cs := cmdProposalSetStatus{} cs.Args.Token = r.CensorshipRecord.Token cs.Args.Status = strconv.Itoa(int(rcv1.RecordStatusCensored)) cs.Args.Reason = "Violates proposal rules." @@ -697,9 +689,7 @@ func proposalAbandoned(author, admin user, includeImages bool) (*rcv1.Record, er } // Abandone the proposal - cs := cmdProposalSetStatus{ - Unvetted: false, - } + cs := cmdProposalSetStatus{} cs.Args.Token = r.CensorshipRecord.Token cs.Args.Status = strconv.Itoa(int(rcv1.RecordStatusArchived)) cs.Args.Reason = "No activity from author in 3 weeks." @@ -719,17 +709,10 @@ func proposalAbandoned(author, admin user, includeImages bool) (*rcv1.Record, er } // inv returns a page of tokens for a record status. -func inv(state string, status rcv1.RecordStatusT, page uint32) ([]string, error) { +func inv(state rcv1.RecordStateT, status rcv1.RecordStatusT, page uint32) ([]string, error) { // Setup command c := cmdProposalInv{} - switch state { - case rcv1.RecordStateUnvetted: - c.Unvetted = true - case rcv1.RecordStateVetted: - // Do nothing - default: - return nil, fmt.Errorf("invalid state") - } + c.Args.State = strconv.Itoa(int(state)) c.Args.Status = strconv.Itoa(int(status)) c.Args.Page = page diff --git a/politeiawww/cmd/pictl/cmdvoteauthorize.go b/politeiawww/cmd/pictl/cmdvoteauthorize.go index 7fad6b86d..132b19607 100644 --- a/politeiawww/cmd/pictl/cmdvoteauthorize.go +++ b/politeiawww/cmd/pictl/cmdvoteauthorize.go @@ -67,18 +67,13 @@ func (c *cmdVoteAuthorize) Execute(args []string) error { version := c.Args.Version if version == 0 { d := rcv1.Details{ - State: rcv1.RecordStateVetted, Token: c.Args.Token, } r, err := pc.RecordDetails(d) if err != nil { return err } - u, err := strconv.ParseUint(r.Version, 10, 64) - if err != nil { - return err - } - version = uint32(u) + version = r.Version } // Setup request diff --git a/politeiawww/cmd/pictl/cmdvotestart.go b/politeiawww/cmd/pictl/cmdvotestart.go index 04bfd6c16..c8e22bb16 100644 --- a/politeiawww/cmd/pictl/cmdvotestart.go +++ b/politeiawww/cmd/pictl/cmdvotestart.go @@ -8,7 +8,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "strconv" "github.com/decred/politeia/politeiad/plugins/ticketvote" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" @@ -39,22 +38,17 @@ type cmdVoteStart struct { func voteStartStandard(token string, duration, quorum, pass uint32, pc *pclient.Client) (*tkv1.StartReply, error) { // Get record version d := rcv1.Details{ - State: rcv1.RecordStateVetted, Token: token, } r, err := pc.RecordDetails(d) if err != nil { return nil, err } - version, err := strconv.ParseUint(r.Version, 10, 64) - if err != nil { - return nil, err - } // Setup request vp := tkv1.VoteParams{ Token: token, - Version: uint32(version), + Version: r.Version, Type: tkv1.VoteTypeStandard, Mask: 0x03, Duration: duration, @@ -109,17 +103,12 @@ func voteStartRunoff(parentToken string, duration, quorum, pass uint32, pc *pcli for _, v := range sr.Submissions { // Get record d := rcv1.Details{ - State: rcv1.RecordStateVetted, Token: v, } r, err := pc.RecordDetails(d) if err != nil { return nil, fmt.Errorf("RecordDetails %v: %v", v, err) } - version, err := strconv.ParseUint(r.Version, 10, 64) - if err != nil { - return nil, err - } // Don't include the record if it has been abandoned. if r.Status == rcv1.RecordStatusArchived { @@ -129,7 +118,7 @@ func voteStartRunoff(parentToken string, duration, quorum, pass uint32, pc *pcli // Setup vote params vp := tkv1.VoteParams{ Token: r.CensorshipRecord.Token, - Version: uint32(version), + Version: r.Version, Type: tkv1.VoteTypeRunoff, Mask: 0x03, // bit 0 no, bit 1 yes Duration: duration, diff --git a/politeiawww/cmd/pictl/cmdvotetimestamps.go b/politeiawww/cmd/pictl/cmdvotetimestamps.go index 08d175a1d..b6c63aa32 100644 --- a/politeiawww/cmd/pictl/cmdvotetimestamps.go +++ b/politeiawww/cmd/pictl/cmdvotetimestamps.go @@ -7,8 +7,8 @@ package main import ( "fmt" - "github.com/decred/politeia/politeiad/backend" - "github.com/decred/politeia/politeiad/backend/tstorebe/tstore" + backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/tstore" tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" pclient "github.com/decred/politeia/politeiawww/client" ) diff --git a/politeiawww/cmd/pictl/proposal.go b/politeiawww/cmd/pictl/proposal.go index 244d3aa4d..72d8f717f 100644 --- a/politeiawww/cmd/pictl/proposal.go +++ b/politeiawww/cmd/pictl/proposal.go @@ -79,144 +79,12 @@ func printProposal(r rcv1.Record) error { printf("Metadata\n") for _, v := range r.Metadata { size := byteCountSI(int64(len([]byte(v.Payload)))) - printf(" %-8v %-2v %v\n", v.PluginID, v.ID, size) + printf(" %-8v %-2v %v\n", v.PluginID, v.StreamID, size) } printf("Files\n") return printProposalFiles(r.Files) } -func convertProposal(p piv1.Proposal) (*rcv1.Record, error) { - // Setup files - files := make([]rcv1.File, 0, len(p.Files)) - for _, v := range p.Files { - files = append(files, rcv1.File{ - Name: v.Name, - MIME: v.MIME, - Digest: v.Digest, - Payload: v.Payload, - }) - } - - // Setup metadata - um := rcv1.UserMetadata{ - UserID: p.UserID, - PublicKey: p.PublicKey, - Signature: p.Signature, - } - umb, err := json.Marshal(um) - if err != nil { - return nil, err - } - var buf bytes.Buffer - for _, v := range p.Statuses { - sc := rcv1.StatusChange{ - Token: v.Token, - Version: v.Version, - Status: rcv1.RecordStatusT(v.Status), - Reason: v.Reason, - PublicKey: v.PublicKey, - Signature: v.Signature, - Timestamp: v.Timestamp, - } - b, err := json.Marshal(sc) - if err != nil { - return nil, err - } - buf.Write(b) - } - metadata := []rcv1.MetadataStream{ - { - ID: usermd.MDStreamIDUserMetadata, - Payload: string(umb), - }, - { - ID: usermd.MDStreamIDStatusChanges, - Payload: buf.String(), - }, - } - - return &rcv1.Record{ - State: p.State, - Status: rcv1.RecordStatusT(p.Status), - Version: p.Version, - Timestamp: p.Timestamp, - Username: p.Username, - Metadata: metadata, - Files: files, - CensorshipRecord: rcv1.CensorshipRecord{ - Token: p.CensorshipRecord.Token, - Merkle: p.CensorshipRecord.Merkle, - Signature: p.CensorshipRecord.Signature, - }, - }, nil -} - -func convertRecord(r rcv1.Record) (*piv1.Proposal, error) { - // Decode metadata streams - var ( - um usermd.UserMetadata - sc = make([]usermd.StatusChangeMetadata, 0, 16) - err error - ) - for _, v := range r.Metadata { - switch v.ID { - case usermd.MDStreamIDUserMetadata: - err = json.Unmarshal([]byte(v.Payload), &um) - if err != nil { - return nil, err - } - case usermd.MDStreamIDStatusChanges: - sc, err = statusChangesDecode([]byte(v.Payload)) - if err != nil { - return nil, err - } - } - } - - // Convert files - files := make([]piv1.File, 0, len(r.Files)) - for _, v := range r.Files { - files = append(files, piv1.File{ - Name: v.Name, - MIME: v.MIME, - Digest: v.Digest, - Payload: v.Payload, - }) - } - - // Convert statuses - statuses := make([]piv1.StatusChange, 0, len(sc)) - for _, v := range sc { - statuses = append(statuses, piv1.StatusChange{ - Token: v.Token, - Version: v.Version, - Status: piv1.PropStatusT(v.Status), - Reason: v.Reason, - PublicKey: v.PublicKey, - Signature: v.Signature, - Timestamp: v.Timestamp, - }) - } - - return &piv1.Proposal{ - Version: r.Version, - Timestamp: r.Timestamp, - State: r.State, - Status: piv1.PropStatusT(r.Status), - UserID: um.UserID, - Username: r.Username, - PublicKey: um.PublicKey, - Signature: um.Signature, - Statuses: statuses, - Files: files, - CensorshipRecord: piv1.CensorshipRecord{ - Token: r.CensorshipRecord.Token, - Merkle: r.CensorshipRecord.Merkle, - Signature: r.CensorshipRecord.Signature, - }, - }, nil -} - func statusChangesDecode(payload []byte) ([]usermd.StatusChangeMetadata, error) { statuses := make([]usermd.StatusChangeMetadata, 0, 16) d := json.NewDecoder(strings.NewReader(string(payload))) From 5c8658e0bc6ae13d0fc41a40c3d383ccaf627096 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 8 Mar 2021 09:51:22 -0600 Subject: [PATCH 367/449] Cleanup and bug fixes. --- politeiad/backendv2/tstorebe/inventory.go | 10 +- .../tstorebe/plugins/usermd/cache.go | 112 ++++++++++++++---- .../tstorebe/plugins/usermd/hooks.go | 34 ++---- politeiad/backendv2/tstorebe/tstore/plugin.go | 11 +- politeiad/backendv2/tstorebe/tstorebe.go | 39 ++++-- politeiad/client/client.go | 6 +- politeiad/{v2.go => handlersv2.go} | 59 ++++++++- politeiad/politeiad.go | 13 +- politeiad/scripts/mysql-tstore-reset.sh | 13 +- politeiad/scripts/mysql-tstore-setup.sh | 12 +- politeiawww/cmd/pictl/proposal.go | 1 + politeiawww/comments/error.go | 3 +- politeiawww/records/error.go | 3 +- politeiawww/ticketvote/error.go | 3 +- 14 files changed, 220 insertions(+), 99 deletions(-) rename politeiad/{v2.go => handlersv2.go} (93%) diff --git a/politeiad/backendv2/tstorebe/inventory.go b/politeiad/backendv2/tstorebe/inventory.go index 85ddc85ea..65055036e 100644 --- a/politeiad/backendv2/tstorebe/inventory.go +++ b/politeiad/backendv2/tstorebe/inventory.go @@ -121,7 +121,8 @@ func (t *tstoreBackend) invAdd(state backend.StateT, token []byte, s backend.Sta return err } - log.Debugf("Inv %v add %x %v", state, token, backend.Statuses[s]) + log.Debugf("Inv add %v %x %v", + backend.States[state], token, backend.Statuses[s]) return nil } @@ -169,7 +170,8 @@ func (t *tstoreBackend) invUpdate(state backend.StateT, token []byte, s backend. return err } - log.Debugf("Inv %v update %x to %v", state, token, backend.Statuses[s]) + log.Debugf("Inv update %v %x to %v", + backend.States[state], token, backend.Statuses[s]) return nil } @@ -221,8 +223,7 @@ func (t *tstoreBackend) invMoveToVetted(token []byte, s backend.StatusT) error { return err } - log.Debugf("Inv move %x from unvetted to vetted status %v", - token, backend.Statuses[s]) + log.Debugf("Inv move to vetted %x %v", token, backend.Statuses[s]) return nil } @@ -250,7 +251,6 @@ func (t *tstoreBackend) inventoryUpdate(state backend.StateT, token []byte, s ba // inventoryMoveToVetted is a wrapper around the invMoveToVetted method that // allows us to decide how disk read/write errors should be handled. For now we // just panic. -// TODO inventoryMoveToVetted should be automatic func (t *tstoreBackend) inventoryMoveToVetted(token []byte, s backend.StatusT) { err := t.invMoveToVetted(token, s) if err != nil { diff --git a/politeiad/backendv2/tstorebe/plugins/usermd/cache.go b/politeiad/backendv2/tstorebe/plugins/usermd/cache.go index 48c6bdec7..bf78bde82 100644 --- a/politeiad/backendv2/tstorebe/plugins/usermd/cache.go +++ b/politeiad/backendv2/tstorebe/plugins/usermd/cache.go @@ -12,6 +12,8 @@ import ( "os" "path/filepath" "strings" + + backend "github.com/decred/politeia/politeiad/backendv2" ) const ( @@ -22,10 +24,12 @@ const ( // userCache contains cached user metadata. The userCache JSON is saved to disk // in the user plugin data dir. The user ID is included in the filename. +// +// All record tokens are sorted by the timestamp of their most recent status +// change from newest to oldest. type userCache struct { - // Tokens contains a list of all record tokens that have been - // submitted by this user, ordered newest to oldest. - Tokens []string `json:"tokens"` + Unvetted []string `json:"unvetted"` + Vetted []string `json:"vetted"` } // userCachePath returns the filepath to the cached userCache struct for the @@ -47,7 +51,8 @@ func (p *userPlugin) userCacheWithLock(userID string) (*userCache, error) { if errors.As(err, &e) && !os.IsExist(err) { // File does't exist. Return an empty userCache. return &userCache{ - Tokens: []string{}, + Unvetted: []string{}, + Vetted: []string{}, }, nil } } @@ -87,7 +92,7 @@ func (p *userPlugin) userCacheSaveWithLock(userID string, uc userCache) error { // userCacheAddToken adds a token to a user cache. // // This function must be called WITHOUT the lock held. -func (p *userPlugin) userCacheAddToken(userID string, token string) error { +func (p *userPlugin) userCacheAddToken(userID string, state backend.StateT, token string) error { p.Lock() defer p.Unlock() @@ -98,7 +103,14 @@ func (p *userPlugin) userCacheAddToken(userID string, token string) error { } // Add token - uc.Tokens = append(uc.Tokens, token) + switch state { + case backend.StateUnvetted: + uc.Unvetted = append(uc.Unvetted, token) + case backend.StateVetted: + uc.Vetted = append(uc.Vetted, token) + default: + return fmt.Errorf("invalid state %v", state) + } // Save changes err = p.userCacheSaveWithLock(userID, *uc) @@ -106,7 +118,7 @@ func (p *userPlugin) userCacheAddToken(userID string, token string) error { return err } - log.Debugf("User cache add %v %v", userID, token) + log.Debugf("User cache add %v %v %v", backend.States[state], userID, token) return nil } @@ -114,7 +126,7 @@ func (p *userPlugin) userCacheAddToken(userID string, token string) error { // userCacheDelToken deletes a token from a user cache. // // This function must be called WITHOUT the lock held. -func (p *userPlugin) userCacheDelToken(userID string, token string) error { +func (p *userPlugin) userCacheDelToken(userID string, state backend.StateT, token string) error { p.Lock() defer p.Unlock() @@ -124,25 +136,54 @@ func (p *userPlugin) userCacheDelToken(userID string, token string) error { return err } - // Find token index - var i int - var found bool - for k, v := range uc.Tokens { - if v == token { - i = k - found = true - break + switch state { + case backend.StateUnvetted: + tokens, err := delToken(uc.Vetted, token) + if err != nil { + return fmt.Errorf("delToken %v %v: %v", + userID, state, err) } + uc.Unvetted = tokens + case backend.StateVetted: + tokens, err := delToken(uc.Vetted, token) + if err != nil { + return fmt.Errorf("delToken %v %v: %v", + userID, state, err) + } + uc.Vetted = tokens + default: + return fmt.Errorf("invalid state %v", state) } - if !found { - return fmt.Errorf("user token not found %v %v", userID, token) + + // Save changes + err = p.userCacheSaveWithLock(userID, *uc) + if err != nil { + return err } - // Del token (linear time) - t := uc.Tokens - copy(t[i:], t[i+1:]) // Shift t[i+1:] left one index - t[len(t)-1] = "" // Erase last element - uc.Tokens = t[:len(t)-1] // Truncate slice + log.Debugf("User cache del %v %v %v", backend.States[state], userID, token) + + return nil +} + +func (p *userPlugin) userCacheMoveTokenToVetted(userID string, token string) error { + p.Lock() + defer p.Unlock() + + // Get current user data + uc, err := p.userCacheWithLock(userID) + if err != nil { + return err + } + + // Del token from unvetted + uc.Unvetted, err = delToken(uc.Unvetted, token) + if err != nil { + return fmt.Errorf("delToken %v: %v", userID, err) + } + + // Add token to vetted + uc.Vetted = append(uc.Vetted, token) // Save changes err = p.userCacheSaveWithLock(userID, *uc) @@ -150,7 +191,30 @@ func (p *userPlugin) userCacheDelToken(userID string, token string) error { return err } - log.Debugf("User cache del %v %v", userID, token) + log.Debugf("User cache move to vetted %v %v", userID, token) return nil } + +func delToken(tokens []string, tokenToDel string) ([]string, error) { + // Find token index + var i int + var found bool + for k, v := range tokens { + if v == tokenToDel { + i = k + found = true + break + } + } + if !found { + return nil, fmt.Errorf("user token not found %v", tokenToDel) + } + + // Del token (linear time) + copy(tokens[i:], tokens[i+1:]) // Shift t[i+1:] left one index + tokens[len(tokens)-1] = "" // Erase last element + tokens = tokens[:len(tokens)-1] // Truncate slice + + return tokens, nil +} diff --git a/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go b/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go index 20b78f59f..53eace038 100644 --- a/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go @@ -170,7 +170,8 @@ func (p *userPlugin) hookNewRecordPost(payload string) error { } // Add token to the user cache - err = p.userCacheAddToken(um.UserID, nr.RecordMetadata.Token) + err = p.userCacheAddToken(um.UserID, nr.RecordMetadata.State, + nr.RecordMetadata.Token) if err != nil { return err } @@ -347,35 +348,18 @@ func (p *userPlugin) hookSetRecordStatusPost(treeID int64, payload string) error if err != nil { return err } + rm := srs.RecordMetadata - // When a record is made public it is moved from the unvetted to - // the vetted tstore instance. The token must be removed from the - // unvetted user cache and added to the vetted user cache. - if srs.RecordMetadata.Status == backend.StatusPublic { - // Decode user metadata + // When a record is made public the token must be moved from the + // unvetted to the vetted category in the user cache. + if rm.Status == backend.StatusPublic { um, err := userMetadataDecode(srs.Metadata) if err != nil { return err } - - // When a record is moved to vetted the plugin hooks are executed - // on both the unvetted and vetted tstore instances. The token - // needs to be removed from the unvetted tstore user cache and - // added to the vetted tstore user cache. We can determine this - // by checking if the record exists. The unvetted instance will - // return false. - if p.tstore.RecordExists(treeID) { - // This is the vetted tstore. Add token to the user cache. - err = p.userCacheAddToken(um.UserID, srs.RecordMetadata.Token) - if err != nil { - return err - } - } else { - // This is the unvetted tstore. Del token from user cache. - err = p.userCacheDelToken(um.UserID, srs.RecordMetadata.Token) - if err != nil { - return err - } + err = p.userCacheMoveTokenToVetted(um.UserID, rm.Token) + if err != nil { + return err } } diff --git a/politeiad/backendv2/tstorebe/tstore/plugin.go b/politeiad/backendv2/tstorebe/tstore/plugin.go index 28fe0f158..a47909408 100644 --- a/politeiad/backendv2/tstorebe/tstore/plugin.go +++ b/politeiad/backendv2/tstorebe/tstore/plugin.go @@ -12,6 +12,7 @@ import ( backend "github.com/decred/politeia/politeiad/backendv2" "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins/comments" "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins/dcrdata" "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins/pi" "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins/ticketvote" @@ -72,12 +73,10 @@ func (t *Tstore) PluginRegister(b backend.Backend, p backend.Plugin) error { ) switch p.ID { case cmplugin.PluginID: - /* - client, err = comments.New(t, p.Settings, dataDir, p.Identity) - if err != nil { - return err - } - */ + client, err = comments.New(t, p.Settings, dataDir, p.Identity) + if err != nil { + return err + } case ddplugin.PluginID: client, err = dcrdata.New(p.Settings, t.activeNetParams) if err != nil { diff --git a/politeiad/backendv2/tstorebe/tstorebe.go b/politeiad/backendv2/tstorebe/tstorebe.go index 2d67a72ee..524360482 100644 --- a/politeiad/backendv2/tstorebe/tstorebe.go +++ b/politeiad/backendv2/tstorebe/tstorebe.go @@ -355,7 +355,7 @@ func filesUpdate(filesCurr, filesAdd []backend.File, filesDel []string) []backen return f } -func recordMetadataNew(token []byte, files []backend.File, status backend.StatusT, version, iteration uint32) (*backend.RecordMetadata, error) { +func recordMetadataNew(token []byte, files []backend.File, state backend.StateT, status backend.StatusT, version, iteration uint32) (*backend.RecordMetadata, error) { digests := make([]string, 0, len(files)) for _, v := range files { digests = append(digests, v.Digest) @@ -368,6 +368,7 @@ func recordMetadataNew(token []byte, files []backend.File, status backend.Status Token: hex.EncodeToString(token), Version: version, Iteration: iteration, + State: state, Status: status, Timestamp: time.Now().Unix(), Merkle: hex.EncodeToString(m[:]), @@ -432,7 +433,8 @@ func (t *tstoreBackend) RecordNew(metadata []backend.MetadataStream, files []bac } // Create record metadata - rm, err := recordMetadataNew(token, files, backend.StatusPublic, 1, 1) + rm, err := recordMetadataNew(token, files, backend.StateUnvetted, + backend.StatusUnreviewed, 1, 1) if err != nil { return nil, err } @@ -515,10 +517,13 @@ func (t *tstoreBackend) RecordEdit(token []byte, mdAppend, mdOverwrite []backend } // Apply changes - metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) - files := filesUpdate(r.Files, filesAdd, filesDel) - recordMD, err := recordMetadataNew(token, files, r.RecordMetadata.Status, - r.RecordMetadata.Version+1, r.RecordMetadata.Iteration+1) + var ( + rm = r.RecordMetadata + metadata = metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) + files = filesUpdate(r.Files, filesAdd, filesDel) + ) + recordMD, err := recordMetadataNew(token, files, rm.State, rm.Status, + rm.Version+1, rm.Iteration+1) if err != nil { return nil, err } @@ -612,9 +617,12 @@ func (t *tstoreBackend) RecordEditMetadata(token []byte, mdAppend, mdOverwrite [ // Apply changes. The version is not incremented for metadata only // updates. The iteration is incremented. - metadata := metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) - recordMD, err := recordMetadataNew(token, r.Files, r.RecordMetadata.Status, - r.RecordMetadata.Version, r.RecordMetadata.Iteration+1) + var ( + rm = r.RecordMetadata + metadata = metadataStreamsUpdate(r.Metadata, mdAppend, mdOverwrite) + ) + recordMD, err := recordMetadataNew(token, r.Files, rm.State, rm.Status, + rm.Version, rm.Iteration+1) if err != nil { return nil, err } @@ -782,8 +790,17 @@ func (t *tstoreBackend) RecordSetStatus(token []byte, status backend.StatusT, md } } + // Determine the state. Making a record public will also trigger + // a state update to vetted. + var state backend.StateT + if status == backend.StatusPublic { + state = backend.StateVetted + } else { + state = r.RecordMetadata.State + } + // Apply changes - recordMD, err := recordMetadataNew(token, r.Files, status, + recordMD, err := recordMetadataNew(token, r.Files, state, status, r.RecordMetadata.Version, r.RecordMetadata.Iteration+1) if err != nil { return nil, err @@ -840,7 +857,7 @@ func (t *tstoreBackend) RecordSetStatus(token []byte, status backend.StatusT, md switch status { case backend.StatusPublic: // The state is updated to vetted when a record is made public - t.inventoryUpdate(backend.StateVetted, token, status) + t.inventoryMoveToVetted(token, status) default: t.inventoryUpdate(r.RecordMetadata.State, token, status) } diff --git a/politeiad/client/client.go b/politeiad/client/client.go index ecfff9eb8..a6f097d04 100644 --- a/politeiad/client/client.go +++ b/politeiad/client/client.go @@ -29,9 +29,9 @@ type Client struct { // an error occurs. PluginID will only be populated if the error occurred // during execution of a plugin command. type ErrorReply struct { - PluginID string `json:"pluginid"` - ErrorCode int `json:"errorcode"` - ErrorContext []string `json:"errorcontext"` + PluginID string `json:"pluginid"` + ErrorCode int `json:"errorcode"` + ErrorContext string `json:"errorcontext"` } // Error represents a politeiad error. Error is returned anytime the politeiad diff --git a/politeiad/v2.go b/politeiad/handlersv2.go similarity index 93% rename from politeiad/v2.go rename to politeiad/handlersv2.go index f39906c61..c77753a94 100644 --- a/politeiad/v2.go +++ b/politeiad/handlersv2.go @@ -431,7 +431,38 @@ func (p *politeia) handlePluginReads(w http.ResponseWriter, r *http.Request) { func (p *politeia) handlePluginInventory(w http.ResponseWriter, r *http.Request) { log.Tracef("handlePluginInventory") - panic("not implemented") + + // Decode request + var pi v2.PluginInventory + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&pi); err != nil { + respondWithErrorV2(w, r, "handlePluginInventory: unmarshal", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeRequestPayloadInvalid, + }) + return + } + challenge, err := hex.DecodeString(pi.Challenge) + if err != nil || len(challenge) != v2.ChallengeSize { + respondWithErrorV2(w, r, "handlePluginInventory: decode challenge", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeChallengeInvalid, + }) + return + } + + // Get plugin inventory + plugins := p.backendv2.PluginInventory() + + // Prepare reply + response := p.identity.SignMessage(challenge) + ir := v2.PluginInventoryReply{ + Response: hex.EncodeToString(response[:]), + Plugins: convertPluginsToV2(plugins), + } + + util.RespondWithJSON(w, http.StatusOK, ir) + } // decodeToken decodes a v2 token and errors if the token is not the full @@ -470,7 +501,7 @@ func (p *politeia) convertRecordToV2(r backendv2.Record) v2.Record { func convertMetadataStreamsToBackend(metadata []v2.MetadataStream) []backendv2.MetadataStream { ms := make([]backendv2.MetadataStream, 0, len(metadata)) - for _, v := range ms { + for _, v := range metadata { ms = append(ms, backendv2.MetadataStream{ PluginID: v.PluginID, StreamID: v.StreamID, @@ -482,7 +513,7 @@ func convertMetadataStreamsToBackend(metadata []v2.MetadataStream) []backendv2.M func convertMetadataStreamsToV2(metadata []backendv2.MetadataStream) []v2.MetadataStream { ms := make([]v2.MetadataStream, 0, len(metadata)) - for _, v := range ms { + for _, v := range metadata { ms = append(ms, v2.MetadataStream{ PluginID: v.PluginID, StreamID: v.StreamID, @@ -584,6 +615,28 @@ func convertRecordTimestampsToV2(r backendv2.RecordTimestamps) v2.RecordTimestam } } +func convertPluginSettingToV2(p backendv2.PluginSetting) v2.PluginSetting { + return v2.PluginSetting{ + Key: p.Key, + Value: p.Value, + } +} + +func convertPluginsToV2(bplugins []backendv2.Plugin) []v2.Plugin { + plugins := make([]v2.Plugin, 0, len(bplugins)) + for _, v := range bplugins { + settings := make([]v2.PluginSetting, 0, len(v.Settings)) + for _, v := range v.Settings { + settings = append(settings, convertPluginSettingToV2(v)) + } + plugins = append(plugins, v2.Plugin{ + ID: v.ID, + Settings: settings, + }) + } + return plugins +} + func respondWithErrorV2(w http.ResponseWriter, r *http.Request, format string, err error) { var ( errCode = convertErrorToV2(err) diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 6e9f14f85..60a8a99b0 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -1057,6 +1057,12 @@ func (p *politeia) setupBackendTstore(anp *chaincfg.Params) error { } p.backendv2 = b + // Setup mux + p.router = mux.NewRouter() + + // Setup not found handler + p.router.NotFoundHandler = closeBody(p.handleNotFound) + // Setup v1 routes p.addRoute(http.MethodPost, v1.IdentityRoute, p.getIdentity, permissionPublic) @@ -1277,7 +1283,12 @@ func _main() error { } } done: - p.backend.Close() + switch p.cfg.Backend { + case backendGit: + p.backend.Close() + case backendTstore: + p.backendv2.Close() + } log.Infof("Exiting") diff --git a/politeiad/scripts/mysql-tstore-reset.sh b/politeiad/scripts/mysql-tstore-reset.sh index b5b38e7a0..e001a342e 100755 --- a/politeiad/scripts/mysql-tstore-reset.sh +++ b/politeiad/scripts/mysql-tstore-reset.sh @@ -27,21 +27,16 @@ flags="-u "${MYSQL_ROOT_USER}" -p"${MYSQL_ROOT_PASSWORD}" --verbose \ politeiad="politeiad" # Database names -testnet_unvetted_kv="testnet3_unvetted_kv" -testnet_vetted_kv="testnet3_vetted_kv" +testnet_kv="testnet3_kv" # Delete databases -mysql ${flags} -e "DROP DATABASE IF EXISTS ${testnet_unvetted_kv};" -mysql ${flags} -e "DROP DATABASE IF EXISTS ${testnet_vetted_kv};" +mysql ${flags} -e "DROP DATABASE IF EXISTS ${testnet_kv};" # Setup kv databases. The trillian script creates the trillian databases. -mysql ${flags} -e "CREATE DATABASE ${testnet_unvetted_kv};" -mysql ${flags} -e "CREATE DATABASE ${testnet_vetted_kv};" +mysql ${flags} -e "CREATE DATABASE ${testnet_kv};" mysql ${flags} -e \ - "GRANT ALL ON ${testnet_unvetted_kv}.* TO '${politeiad}'@'${MYSQL_USER_HOST}'" -mysql ${flags} -e \ - "GRANT ALL ON ${testnet_vetted_kv}.* TO '${politeiad}'@'${MYSQL_USER_HOST}'" + "GRANT ALL ON ${testnet_kv}.* TO '${politeiad}'@'${MYSQL_USER_HOST}'" # Delete cached politeiad data politeiad_data_dir="${POLITEIAD_DIR}/data/testnet3/" diff --git a/politeiad/scripts/mysql-tstore-setup.sh b/politeiad/scripts/mysql-tstore-setup.sh index 578a485ae..9a761e80e 100755 --- a/politeiad/scripts/mysql-tstore-setup.sh +++ b/politeiad/scripts/mysql-tstore-setup.sh @@ -31,8 +31,8 @@ politeiad="politeiad" trillian="trillian" # Database names -testnet_unvetted_kv="testnet3_unvetted_kv" -testnet_vetted_kv="testnet3_vetted_kv" +testnet_kv="testnet3_kv" +mainnet_kv="mainnet_kv" # Setup database users mysql ${flags} -e \ @@ -44,10 +44,10 @@ mysql ${flags} -e \ IDENTIFIED BY '${MYSQL_TRILLIAN_PASSWORD}'" # Setup kv databases. The trillian script creates the trillian databases. -mysql ${flags} -e "CREATE DATABASE IF NOT EXISTS ${testnet_unvetted_kv};" -mysql ${flags} -e "CREATE DATABASE IF NOT EXISTS ${testnet_vetted_kv};" +mysql ${flags} -e "CREATE DATABASE IF NOT EXISTS ${testnet_kv};" +mysql ${flags} -e "CREATE DATABASE IF NOT EXISTS ${mainnet_kv};" mysql ${flags} -e \ - "GRANT ALL ON ${testnet_unvetted_kv}.* TO '${politeiad}'@'${MYSQL_USER_HOST}'" + "GRANT ALL ON ${testnet_kv}.* TO '${politeiad}'@'${MYSQL_USER_HOST}'" mysql ${flags} -e \ - "GRANT ALL ON ${testnet_vetted_kv}.* TO '${politeiad}'@'${MYSQL_USER_HOST}'" + "GRANT ALL ON ${mainnet_kv}.* TO '${politeiad}'@'${MYSQL_USER_HOST}'" diff --git a/politeiawww/cmd/pictl/proposal.go b/politeiawww/cmd/pictl/proposal.go index 72d8f717f..70557d70c 100644 --- a/politeiawww/cmd/pictl/proposal.go +++ b/politeiawww/cmd/pictl/proposal.go @@ -71,6 +71,7 @@ func printProposalFiles(files []rcv1.File) error { func printProposal(r rcv1.Record) error { printf("Token : %v\n", r.CensorshipRecord.Token) printf("Version : %v\n", r.Version) + printf("State : %v\n", rcv1.RecordStates[r.State]) printf("Status : %v\n", rcv1.RecordStatuses[r.Status]) printf("Timestamp: %v\n", timestampFromUnix(r.Timestamp)) printf("Username : %v\n", r.Username) diff --git a/politeiawww/comments/error.go b/politeiawww/comments/error.go index 5fd51fd11..f47e8db9e 100644 --- a/politeiawww/comments/error.go +++ b/politeiawww/comments/error.go @@ -9,7 +9,6 @@ import ( "fmt" "net/http" "runtime/debug" - "strings" "time" pdv2 "github.com/decred/politeia/politeiad/api/v2" @@ -61,7 +60,7 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err var ( pluginID = pde.ErrorReply.PluginID errCode = pde.ErrorReply.ErrorCode - errContext = strings.Join(pde.ErrorReply.ErrorContext, ",") + errContext = pde.ErrorReply.ErrorContext ) e := convertPDErrorCode(errCode) switch { diff --git a/politeiawww/records/error.go b/politeiawww/records/error.go index 3636db786..826519eea 100644 --- a/politeiawww/records/error.go +++ b/politeiawww/records/error.go @@ -9,7 +9,6 @@ import ( "fmt" "net/http" "runtime/debug" - "strings" "time" pdv2 "github.com/decred/politeia/politeiad/api/v2" @@ -80,7 +79,7 @@ func handlePDError(w http.ResponseWriter, r *http.Request, format string, pde pd var ( pluginID = pde.ErrorReply.PluginID errCode = pde.ErrorReply.ErrorCode - errContext = strings.Join(pde.ErrorReply.ErrorContext, ",") + errContext = pde.ErrorReply.ErrorContext ) e := convertPDErrorCode(errCode) switch { diff --git a/politeiawww/ticketvote/error.go b/politeiawww/ticketvote/error.go index 2165400ae..8f99306b8 100644 --- a/politeiawww/ticketvote/error.go +++ b/politeiawww/ticketvote/error.go @@ -9,7 +9,6 @@ import ( "fmt" "net/http" "runtime/debug" - "strings" "time" pdv2 "github.com/decred/politeia/politeiad/api/v2" @@ -63,7 +62,7 @@ func handlePDError(w http.ResponseWriter, r *http.Request, format string, pde pd var ( pluginID = pde.ErrorReply.PluginID errCode = pde.ErrorReply.ErrorCode - errContext = strings.Join(pde.ErrorReply.ErrorContext, ",") + errContext = pde.ErrorReply.ErrorContext ) e := convertPDErrorCode(errCode) switch { From 78e283d3a9683b63fd3eab3118781bdc1586e30d Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 8 Mar 2021 12:55:23 -0600 Subject: [PATCH 368/449] politeiad: Organize v1 routes. --- politeiad/politeiad.go | 848 +--------------------------- politeiad/v1.go | 849 +++++++++++++++++++++++++++++ politeiad/{handlersv2.go => v2.go} | 0 3 files changed, 851 insertions(+), 846 deletions(-) create mode 100644 politeiad/v1.go rename politeiad/{handlersv2.go => v2.go} (100%) diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index 60a8a99b0..bf49cc6c1 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -7,9 +7,6 @@ package main import ( "crypto/elliptic" "crypto/x509" - "encoding/hex" - "encoding/json" - "errors" "fmt" "io/ioutil" "net/http" @@ -19,7 +16,6 @@ import ( "runtime/debug" "strings" "syscall" - "time" "github.com/decred/dcrd/chaincfg/v3" v1 "github.com/decred/politeia/politeiad/api/v1" @@ -59,138 +55,6 @@ func remoteAddr(r *http.Request) string { return via } -func convertBackendPluginSetting(bpi backend.PluginSetting) v1.PluginSetting { - return v1.PluginSetting{ - Key: bpi.Key, - Value: bpi.Value, - } -} - -func convertBackendPlugins(bplugins []backend.Plugin) []v1.Plugin { - plugins := make([]v1.Plugin, 0, len(bplugins)) - for _, v := range bplugins { - p := v1.Plugin{ - ID: v.ID, - Version: v.Version, - Settings: make([]v1.PluginSetting, 0, len(v.Settings)), - } - for _, v := range v.Settings { - p.Settings = append(p.Settings, convertBackendPluginSetting(v)) - } - plugins = append(plugins, p) - } - return plugins -} - -// convertBackendMetadataStream converts a backend metadata stream to an API -// metadata stream. -func convertBackendMetadataStream(mds backend.MetadataStream) v1.MetadataStream { - return v1.MetadataStream{ - ID: mds.ID, - Payload: mds.Payload, - } -} - -// convertBackendStatus converts a backend MDStatus to an API status. -func convertBackendStatus(status backend.MDStatusT) v1.RecordStatusT { - s := v1.RecordStatusInvalid - switch status { - case backend.MDStatusInvalid: - s = v1.RecordStatusInvalid - case backend.MDStatusUnvetted: - s = v1.RecordStatusNotReviewed - case backend.MDStatusVetted: - s = v1.RecordStatusPublic - case backend.MDStatusCensored: - s = v1.RecordStatusCensored - case backend.MDStatusIterationUnvetted: - s = v1.RecordStatusUnreviewedChanges - case backend.MDStatusArchived: - s = v1.RecordStatusArchived - } - return s -} - -// convertFrontendStatus convert an API status to a backend MDStatus. -func convertFrontendStatus(status v1.RecordStatusT) backend.MDStatusT { - s := backend.MDStatusInvalid - switch status { - case v1.RecordStatusInvalid: - s = backend.MDStatusInvalid - case v1.RecordStatusNotReviewed: - s = backend.MDStatusUnvetted - case v1.RecordStatusPublic: - s = backend.MDStatusVetted - case v1.RecordStatusCensored: - s = backend.MDStatusCensored - case v1.RecordStatusArchived: - s = backend.MDStatusArchived - } - return s -} - -func convertFrontendFiles(f []v1.File) []backend.File { - files := make([]backend.File, 0, len(f)) - for _, v := range f { - files = append(files, backend.File{ - Name: v.Name, - MIME: v.MIME, - Digest: v.Digest, - Payload: v.Payload, - }) - } - return files -} - -func convertFrontendMetadataStream(mds []v1.MetadataStream) []backend.MetadataStream { - m := make([]backend.MetadataStream, 0, len(mds)) - for _, v := range mds { - m = append(m, backend.MetadataStream{ - ID: v.ID, - Payload: v.Payload, - }) - } - return m -} - -func (p *politeia) convertBackendRecord(br backend.Record) v1.Record { - rm := br.RecordMetadata - - // Calculate signature - signature := p.identity.SignMessage([]byte(rm.Merkle + rm.Token)) - - // Convert MetadataStream - md := make([]v1.MetadataStream, 0, len(br.Metadata)) - for k := range br.Metadata { - md = append(md, convertBackendMetadataStream(br.Metadata[k])) - } - - // Convert record - pr := v1.Record{ - Status: convertBackendStatus(rm.Status), - Timestamp: rm.Timestamp, - CensorshipRecord: v1.CensorshipRecord{ - Merkle: rm.Merkle, - Token: rm.Token, - Signature: hex.EncodeToString(signature[:]), - }, - Version: br.Version, - Metadata: md, - } - pr.Files = make([]v1.File, 0, len(br.Files)) - for _, v := range br.Files { - pr.Files = append(pr.Files, - v1.File{ - Name: v.Name, - MIME: v.MIME, - Digest: v.Digest, - Payload: v.Payload, - }) - } - - return pr -} - // handleNotFound is a generic handler for an invalid route. func (p *politeia) handleNotFound(w http.ResponseWriter, r *http.Request) { // Log incoming connection @@ -224,392 +88,6 @@ func (p *politeia) respondWithServerError(w http.ResponseWriter, errorCode int64 }) } -func (p *politeia) getIdentity(w http.ResponseWriter, r *http.Request) { - var t v1.Identity - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&t); err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) - return - } - - challenge, err := hex.DecodeString(t.Challenge) - if err != nil || len(challenge) != v1.ChallengeSize { - p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) - return - } - response := p.identity.SignMessage(challenge) - - reply := v1.IdentityReply{ - PublicKey: hex.EncodeToString(p.identity.Public.Key[:]), - Response: hex.EncodeToString(response[:]), - } - - util.RespondWithJSON(w, http.StatusOK, reply) -} - -func (p *politeia) newRecord(w http.ResponseWriter, r *http.Request) { - var t v1.NewRecord - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&t); err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) - return - } - - challenge, err := hex.DecodeString(t.Challenge) - if err != nil || len(challenge) != v1.ChallengeSize { - log.Infof("%v newRecord: invalid challenge", remoteAddr(r)) - p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) - return - } - - md := convertFrontendMetadataStream(t.Metadata) - files := convertFrontendFiles(t.Files) - rm, err := p.backend.New(md, files) - if err != nil { - // Check for content error. - var contentErr backend.ContentVerificationError - if errors.As(err, &contentErr) { - log.Infof("%v New record content error: %v", - remoteAddr(r), contentErr) - p.respondWithUserError(w, contentErr.ErrorCode, - contentErr.ErrorContext) - return - } - - // Generic internal error. - errorCode := time.Now().Unix() - log.Errorf("%v New record error code %v: %v", remoteAddr(r), - errorCode, err) - p.respondWithServerError(w, errorCode) - return - } - - // Prepare reply. - signature := p.identity.SignMessage([]byte(rm.Merkle + rm.Token)) - - response := p.identity.SignMessage(challenge) - reply := v1.NewRecordReply{ - Response: hex.EncodeToString(response[:]), - CensorshipRecord: v1.CensorshipRecord{ - Merkle: rm.Merkle, - Token: rm.Token, - Signature: hex.EncodeToString(signature[:]), - }, - } - - log.Infof("New record accepted %v: token %v", remoteAddr(r), - reply.CensorshipRecord.Token) - - util.RespondWithJSON(w, http.StatusOK, reply) -} - -func (p *politeia) updateRecord(w http.ResponseWriter, r *http.Request, vetted bool) { - cmd := "unvetted" - if vetted { - cmd = "vetted" - } - - var t v1.UpdateRecord - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&t); err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, - nil) - return - } - - challenge, err := hex.DecodeString(t.Challenge) - if err != nil || len(challenge) != v1.ChallengeSize { - log.Infof("%v update %v record: invalid challenge", - remoteAddr(r), cmd) - p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) - return - } - - // Validate token - token, err := util.ConvertStringToken(t.Token) - if err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) - return - } - - log.Infof("Update %v record submitted %v: %x", cmd, remoteAddr(r), - token) - - var record *backend.Record - if vetted { - record, err = p.backend.UpdateVettedRecord(token, - convertFrontendMetadataStream(t.MDAppend), - convertFrontendMetadataStream(t.MDOverwrite), - convertFrontendFiles(t.FilesAdd), t.FilesDel) - } else { - record, err = p.backend.UpdateUnvettedRecord(token, - convertFrontendMetadataStream(t.MDAppend), - convertFrontendMetadataStream(t.MDOverwrite), - convertFrontendFiles(t.FilesAdd), t.FilesDel) - } - if err != nil { - if errors.Is(err, backend.ErrRecordFound) { - log.Infof("%v update %v record found: %x", - remoteAddr(r), cmd, token) - p.respondWithUserError(w, v1.ErrorStatusRecordFound, - nil) - return - } - if errors.Is(err, backend.ErrRecordNotFound) { - log.Infof("%v update %v record not found: %x", - remoteAddr(r), cmd, token) - p.respondWithUserError(w, v1.ErrorStatusRecordNotFound, nil) - return - } - if errors.Is(err, backend.ErrNoChanges) { - log.Infof("%v update %v record no changes: %x", - remoteAddr(r), cmd, token) - p.respondWithUserError(w, v1.ErrorStatusNoChanges, nil) - return - } - // Check for content error. - var contentErr backend.ContentVerificationError - if errors.As(err, &contentErr) { - log.Infof("%v update %v record content error: %v", - remoteAddr(r), cmd, contentErr) - p.respondWithUserError(w, contentErr.ErrorCode, - contentErr.ErrorContext) - return - } - // Generic internal error. - errorCode := time.Now().Unix() - log.Errorf("%v Update %v record error code %v: %v", - remoteAddr(r), cmd, errorCode, err) - p.respondWithServerError(w, errorCode) - return - } - - // Prepare reply. - response := p.identity.SignMessage(challenge) - reply := v1.UpdateRecordReply{ - Response: hex.EncodeToString(response[:]), - Record: p.convertBackendRecord(*record), - } - - log.Infof("Update %v record %v: token %v", cmd, remoteAddr(r), - record.RecordMetadata.Token) - - util.RespondWithJSON(w, http.StatusOK, reply) -} - -func (p *politeia) updateUnvetted(w http.ResponseWriter, r *http.Request) { - p.updateRecord(w, r, false) -} - -func (p *politeia) updateVetted(w http.ResponseWriter, r *http.Request) { - p.updateRecord(w, r, true) -} - -func (p *politeia) getUnvetted(w http.ResponseWriter, r *http.Request) { - var t v1.GetUnvetted - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&t); err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) - return - } - - challenge, err := hex.DecodeString(t.Challenge) - if err != nil || len(challenge) != v1.ChallengeSize { - p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) - return - } - response := p.identity.SignMessage(challenge) - - reply := v1.GetUnvettedReply{ - Response: hex.EncodeToString(response[:]), - } - - // Validate token - token, err := util.ConvertStringToken(t.Token) - if err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) - return - } - - // Ask backend about the censorship token. - bpr, err := p.backend.GetUnvetted(token, t.Version) - switch { - case errors.Is(err, backend.ErrRecordNotFound): - // Record not found - log.Infof("Get unvetted record %v: token %v not found", - remoteAddr(r), t.Token) - p.respondWithUserError(w, v1.ErrorStatusRecordNotFound, nil) - return - - case err != nil: - // Generic internal error - errorCode := time.Now().Unix() - log.Errorf("%v Get unvetted record error code %v: %v", - remoteAddr(r), errorCode, err) - p.respondWithServerError(w, errorCode) - return - - case bpr.RecordMetadata.Status == backend.MDStatusCensored: - // Record has been censored. The default case will verify the - // record before sending it off. This will fail for censored - // records since the files will not exist, they've been deleted, - // so skip the verification step. - reply.Record = p.convertBackendRecord(*bpr) - log.Infof("Get unvetted record %v: token %v", remoteAddr(r), t.Token) - - default: - reply.Record = p.convertBackendRecord(*bpr) - - // Double check record bits before sending them off - err := v1.Verify(p.identity.Public, - reply.Record.CensorshipRecord, reply.Record.Files) - if err != nil { - // Generic internal error. - errorCode := time.Now().Unix() - log.Errorf("%v Get unvetted record CORRUPTION "+ - "error code %v: %v", remoteAddr(r), errorCode, - err) - p.respondWithServerError(w, errorCode) - return - } - - log.Infof("Get unvetted record %v: token %v", remoteAddr(r), t.Token) - } - - util.RespondWithJSON(w, http.StatusOK, reply) -} - -func (p *politeia) getVetted(w http.ResponseWriter, r *http.Request) { - var t v1.GetVetted - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&t); err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) - return - } - - challenge, err := hex.DecodeString(t.Challenge) - if err != nil || len(challenge) != v1.ChallengeSize { - p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) - return - } - response := p.identity.SignMessage(challenge) - - reply := v1.GetVettedReply{ - Response: hex.EncodeToString(response[:]), - } - - // Validate token - token, err := util.ConvertStringToken(t.Token) - if err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) - return - } - - // Ask backend about the censorship token. - bpr, err := p.backend.GetVetted(token, t.Version) - switch { - case errors.Is(err, backend.ErrRecordNotFound): - // Record not found - log.Infof("Get vetted record %v: token %v not found", - remoteAddr(r), t.Token) - p.respondWithUserError(w, v1.ErrorStatusRecordNotFound, nil) - return - - case err != nil: - // Generic internal error - errorCode := time.Now().Unix() - log.Errorf("%v Get vetted record error code %v: %v", - remoteAddr(r), errorCode, err) - p.respondWithServerError(w, errorCode) - return - - case bpr.RecordMetadata.Status == backend.MDStatusCensored: - // Record has been censored. The default case will verify the - // record before sending it off. This will fail for censored - // records since the files will not exist, they've been deleted, - // so skip the verification step. - reply.Record = p.convertBackendRecord(*bpr) - log.Infof("Get vetted record %v: token %v", remoteAddr(r), t.Token) - - default: - reply.Record = p.convertBackendRecord(*bpr) - - // Double check record bits before sending them off - err := v1.Verify(p.identity.Public, - reply.Record.CensorshipRecord, reply.Record.Files) - if err != nil { - // Generic internal error. - errorCode := time.Now().Unix() - log.Errorf("%v Get vetted record CORRUPTION "+ - "error code %v: %v", remoteAddr(r), errorCode, - err) - p.respondWithServerError(w, errorCode) - return - } - - log.Infof("Get vetted record %v: token %v", remoteAddr(r), t.Token) - } - - util.RespondWithJSON(w, http.StatusOK, reply) -} - -func (p *politeia) convertBackendRecords(br map[string]backend.Record) map[string]v1.Record { - r := make(map[string]v1.Record, len(br)) - for k, v := range br { - r[k] = p.convertBackendRecord(v) - } - return r -} - -func (p *politeia) inventory(w http.ResponseWriter, r *http.Request) { - var i v1.Inventory - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&i); err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) - return - } - - challenge, err := hex.DecodeString(i.Challenge) - if err != nil || len(challenge) != v1.ChallengeSize { - p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) - return - } - response := p.identity.SignMessage(challenge) - - reply := v1.InventoryReply{ - Response: hex.EncodeToString(response[:]), - } - - // Ask backend for inventory - prs, brs, err := p.backend.Inventory(i.VettedCount, i.VettedStart, i.BranchesCount, - i.IncludeFiles, i.AllVersions) - if err != nil { - // Generic internal error. - errorCode := time.Now().Unix() - log.Errorf("%v Inventory error code %v: %v", remoteAddr(r), - errorCode, err) - - p.respondWithServerError(w, errorCode) - return - } - - // Convert backend records - vetted := make([]v1.Record, 0, len(prs)) - for _, v := range prs { - vetted = append(vetted, p.convertBackendRecord(v)) - } - reply.Vetted = vetted - - // Convert branches - unvetted := make([]v1.Record, 0, len(brs)) - for _, v := range brs { - unvetted = append(unvetted, p.convertBackendRecord(v)) - } - reply.Branches = unvetted - - util.RespondWithJSON(w, http.StatusOK, reply) -} - func (p *politeia) check(user, pass string) bool { if user != p.cfg.RPCUser || pass != p.cfg.RPCPass { return false @@ -635,329 +113,6 @@ func (p *politeia) auth(fn http.HandlerFunc) http.HandlerFunc { } } -func (p *politeia) setVettedStatus(w http.ResponseWriter, r *http.Request) { - var t v1.SetVettedStatus - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&t); err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) - return - } - - challenge, err := hex.DecodeString(t.Challenge) - if err != nil || len(challenge) != v1.ChallengeSize { - p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) - return - } - response := p.identity.SignMessage(challenge) - - // Validate token - token, err := util.ConvertStringToken(t.Token) - if err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) - return - } - - // Ask backend to update status - record, err := p.backend.SetVettedStatus(token, - convertFrontendStatus(t.Status), - convertFrontendMetadataStream(t.MDAppend), - convertFrontendMetadataStream(t.MDOverwrite)) - if err != nil { - // Check for specific errors - if errors.Is(err, backend.ErrRecordNotFound) { - log.Infof("%v updateStatus record not "+ - "found: %x", remoteAddr(r), token) - p.respondWithUserError(w, v1.ErrorStatusRecordNotFound, nil) - return - } - var serr backend.StateTransitionError - if errors.As(err, &serr) { - log.Infof("%v %v %v", remoteAddr(r), t.Token, err) - p.respondWithUserError(w, v1.ErrorStatusInvalidRecordStatusTransition, nil) - return - } - // Generic internal error. - errorCode := time.Now().Unix() - log.Errorf("%v Set status error code %v: %v", - remoteAddr(r), errorCode, err) - - p.respondWithServerError(w, errorCode) - return - } - - // Prepare reply. - reply := v1.SetVettedStatusReply{ - Response: hex.EncodeToString(response[:]), - Record: p.convertBackendRecord(*record), - } - - s := convertBackendStatus(record.RecordMetadata.Status) - log.Infof("Set vetted record status %v: token %v status %v", - remoteAddr(r), t.Token, v1.RecordStatus[s]) - - util.RespondWithJSON(w, http.StatusOK, reply) -} - -func (p *politeia) setUnvettedStatus(w http.ResponseWriter, r *http.Request) { - var t v1.SetUnvettedStatus - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&t); err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) - return - } - - challenge, err := hex.DecodeString(t.Challenge) - if err != nil || len(challenge) != v1.ChallengeSize { - p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) - return - } - response := p.identity.SignMessage(challenge) - - // Validate token - token, err := util.ConvertStringToken(t.Token) - if err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) - return - } - - // Ask backend to update unvetted status - record, err := p.backend.SetUnvettedStatus(token, - convertFrontendStatus(t.Status), - convertFrontendMetadataStream(t.MDAppend), - convertFrontendMetadataStream(t.MDOverwrite)) - if err != nil { - // Check for specific errors - if errors.Is(err, backend.ErrRecordNotFound) { - log.Infof("%v updateUnvettedStatus record not "+ - "found: %x", remoteAddr(r), token) - p.respondWithUserError(w, v1.ErrorStatusRecordNotFound, nil) - return - } - var serr backend.StateTransitionError - if errors.As(err, &serr) { - log.Infof("%v %v %v", remoteAddr(r), t.Token, err) - p.respondWithUserError(w, v1.ErrorStatusInvalidRecordStatusTransition, nil) - return - } - // Generic internal error. - errorCode := time.Now().Unix() - log.Errorf("%v Set unvetted status error code %v: %v", - remoteAddr(r), errorCode, err) - - p.respondWithServerError(w, errorCode) - return - } - - // Prepare reply. - reply := v1.SetUnvettedStatusReply{ - Response: hex.EncodeToString(response[:]), - Record: p.convertBackendRecord(*record), - } - - s := convertBackendStatus(record.RecordMetadata.Status) - log.Infof("Set unvetted record status %v: token %v status %v", - remoteAddr(r), t.Token, v1.RecordStatus[s]) - - util.RespondWithJSON(w, http.StatusOK, reply) -} - -func (p *politeia) updateVettedMetadata(w http.ResponseWriter, r *http.Request) { - var t v1.UpdateVettedMetadata - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&t); err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) - return - } - - challenge, err := hex.DecodeString(t.Challenge) - if err != nil || len(challenge) != v1.ChallengeSize { - p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) - return - } - response := p.identity.SignMessage(challenge) - - // Validate token - token, err := util.ConvertStringToken(t.Token) - if err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) - return - } - - log.Infof("Update vetted metadata submitted %v: %x", remoteAddr(r), - token) - - err = p.backend.UpdateVettedMetadata(token, - convertFrontendMetadataStream(t.MDAppend), - convertFrontendMetadataStream(t.MDOverwrite)) - if err != nil { - if errors.Is(err, backend.ErrNoChanges) { - log.Infof("%v update vetted metadata no changes: %x", - remoteAddr(r), token) - p.respondWithUserError(w, v1.ErrorStatusNoChanges, nil) - return - } - // Check for content error. - var contentErr backend.ContentVerificationError - if errors.As(err, &contentErr) { - log.Infof("%v update vetted metadata content error: %v", - remoteAddr(r), contentErr) - p.respondWithUserError(w, contentErr.ErrorCode, - contentErr.ErrorContext) - return - } - // Generic internal error. - errorCode := time.Now().Unix() - log.Errorf("%v Update vetted metadata error code %v: %v", - remoteAddr(r), errorCode, err) - p.respondWithServerError(w, errorCode) - return - } - - // Reply - reply := v1.UpdateVettedMetadataReply{ - Response: hex.EncodeToString(response[:]), - } - - log.Infof("Update vetted metadata %v: token %x", remoteAddr(r), token) - - util.RespondWithJSON(w, http.StatusOK, reply) -} - -func (p *politeia) updateUnvettedMetadata(w http.ResponseWriter, r *http.Request) { - var t v1.UpdateUnvettedMetadata - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&t); err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) - return - } - - challenge, err := hex.DecodeString(t.Challenge) - if err != nil || len(challenge) != v1.ChallengeSize { - p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) - return - } - - token, err := util.ConvertStringToken(t.Token) - if err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) - return - } - - log.Infof("Update unvetted metadata submitted %v: %x", remoteAddr(r), - token) - - err = p.backend.UpdateUnvettedMetadata(token, - convertFrontendMetadataStream(t.MDAppend), - convertFrontendMetadataStream(t.MDOverwrite)) - if err != nil { - // Reply with error if there were no changes - if errors.Is(err, backend.ErrNoChanges) { - log.Infof("%v update unvetted metadata no changes: %x", - remoteAddr(r), token) - p.respondWithUserError(w, v1.ErrorStatusNoChanges, nil) - return - } - // Check for content error. - var cverr backend.ContentVerificationError - if errors.As(err, &cverr) { - log.Infof("%v update unvetted metadata content error: %v", - remoteAddr(r), cverr) - p.respondWithUserError(w, cverr.ErrorCode, - cverr.ErrorContext) - return - } - // Generic internal error. - errorCode := time.Now().Unix() - log.Errorf("%v update unvetted metadata error code %v: %v", - remoteAddr(r), errorCode, err) - p.respondWithServerError(w, errorCode) - return - } - - // Prepare reply - response := p.identity.SignMessage(challenge) - reply := v1.UpdateUnvettedMetadataReply{ - Response: hex.EncodeToString(response[:]), - } - - log.Infof("Update unvetted metadata %v: token %x", remoteAddr(r), token) - - util.RespondWithJSON(w, http.StatusOK, reply) -} - -func (p *politeia) pluginInventory(w http.ResponseWriter, r *http.Request) { - var pi v1.PluginInventory - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&pi); err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, - nil) - return - } - - challenge, err := hex.DecodeString(pi.Challenge) - if err != nil || len(challenge) != v1.ChallengeSize { - p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) - return - } - response := p.identity.SignMessage(challenge) - - // Get plugins - plugins, err := p.backend.GetPlugins() - if err != nil { - errorCode := time.Now().Unix() - log.Errorf("%v get plugins: %v ", remoteAddr(r), err) - p.respondWithServerError(w, errorCode) - return - } - - // Prepare reply - reply := v1.PluginInventoryReply{ - Plugins: convertBackendPlugins(plugins), - Response: hex.EncodeToString(response[:]), - } - - util.RespondWithJSON(w, http.StatusOK, reply) -} - -func (p *politeia) pluginCommand(w http.ResponseWriter, r *http.Request) { - var pc v1.PluginCommand - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&pc); err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, - nil) - return - } - - challenge, err := hex.DecodeString(pc.Challenge) - if err != nil || len(challenge) != v1.ChallengeSize { - p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) - return - } - - cid, payload, err := p.backend.Plugin(pc.Command, pc.Payload) - if err != nil { - // Generic internal error. - errorCode := time.Now().Unix() - log.Errorf("%v %v: backend plugin failed with "+ - "command:%v payload:%v err:%v", remoteAddr(r), - errorCode, pc.Command, pc.Payload, err) - p.respondWithServerError(w, errorCode) - return - } - - // Prepare reply - response := p.identity.SignMessage(challenge) - reply := v1.PluginCommandReply{ - Response: hex.EncodeToString(response[:]), - ID: pc.ID, - Command: cid, - CommandID: pc.CommandID, - Payload: payload, - } - - util.RespondWithJSON(w, http.StatusOK, reply) -} - func logging(f http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { // Trace incoming request @@ -1237,7 +392,8 @@ func _main() error { return fmt.Errorf("unable to load cert") } } - // Setup backend. + + // Setup backend log.Infof("Backend: %v", cfg.Backend) switch cfg.Backend { case backendGit: diff --git a/politeiad/v1.go b/politeiad/v1.go new file mode 100644 index 000000000..b18d5f421 --- /dev/null +++ b/politeiad/v1.go @@ -0,0 +1,849 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. +package main + +import ( + "encoding/hex" + "encoding/json" + "errors" + "net/http" + "time" + + v1 "github.com/decred/politeia/politeiad/api/v1" + "github.com/decred/politeia/politeiad/backend" + "github.com/decred/politeia/util" +) + +func (p *politeia) getIdentity(w http.ResponseWriter, r *http.Request) { + var t v1.Identity + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&t); err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) + return + } + + challenge, err := hex.DecodeString(t.Challenge) + if err != nil || len(challenge) != v1.ChallengeSize { + p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) + return + } + response := p.identity.SignMessage(challenge) + + reply := v1.IdentityReply{ + PublicKey: hex.EncodeToString(p.identity.Public.Key[:]), + Response: hex.EncodeToString(response[:]), + } + + util.RespondWithJSON(w, http.StatusOK, reply) +} + +func (p *politeia) newRecord(w http.ResponseWriter, r *http.Request) { + var t v1.NewRecord + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&t); err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) + return + } + + challenge, err := hex.DecodeString(t.Challenge) + if err != nil || len(challenge) != v1.ChallengeSize { + log.Infof("%v newRecord: invalid challenge", remoteAddr(r)) + p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) + return + } + + md := convertFrontendMetadataStream(t.Metadata) + files := convertFrontendFiles(t.Files) + rm, err := p.backend.New(md, files) + if err != nil { + // Check for content error. + var contentErr backend.ContentVerificationError + if errors.As(err, &contentErr) { + log.Infof("%v New record content error: %v", + remoteAddr(r), contentErr) + p.respondWithUserError(w, contentErr.ErrorCode, + contentErr.ErrorContext) + return + } + + // Generic internal error. + errorCode := time.Now().Unix() + log.Errorf("%v New record error code %v: %v", remoteAddr(r), + errorCode, err) + p.respondWithServerError(w, errorCode) + return + } + + // Prepare reply. + signature := p.identity.SignMessage([]byte(rm.Merkle + rm.Token)) + + response := p.identity.SignMessage(challenge) + reply := v1.NewRecordReply{ + Response: hex.EncodeToString(response[:]), + CensorshipRecord: v1.CensorshipRecord{ + Merkle: rm.Merkle, + Token: rm.Token, + Signature: hex.EncodeToString(signature[:]), + }, + } + + log.Infof("New record accepted %v: token %v", remoteAddr(r), + reply.CensorshipRecord.Token) + + util.RespondWithJSON(w, http.StatusOK, reply) +} + +func (p *politeia) updateRecord(w http.ResponseWriter, r *http.Request, vetted bool) { + cmd := "unvetted" + if vetted { + cmd = "vetted" + } + + var t v1.UpdateRecord + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&t); err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, + nil) + return + } + + challenge, err := hex.DecodeString(t.Challenge) + if err != nil || len(challenge) != v1.ChallengeSize { + log.Infof("%v update %v record: invalid challenge", + remoteAddr(r), cmd) + p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) + return + } + + // Validate token + token, err := util.ConvertStringToken(t.Token) + if err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) + return + } + + log.Infof("Update %v record submitted %v: %x", cmd, remoteAddr(r), + token) + + var record *backend.Record + if vetted { + record, err = p.backend.UpdateVettedRecord(token, + convertFrontendMetadataStream(t.MDAppend), + convertFrontendMetadataStream(t.MDOverwrite), + convertFrontendFiles(t.FilesAdd), t.FilesDel) + } else { + record, err = p.backend.UpdateUnvettedRecord(token, + convertFrontendMetadataStream(t.MDAppend), + convertFrontendMetadataStream(t.MDOverwrite), + convertFrontendFiles(t.FilesAdd), t.FilesDel) + } + if err != nil { + if errors.Is(err, backend.ErrRecordFound) { + log.Infof("%v update %v record found: %x", + remoteAddr(r), cmd, token) + p.respondWithUserError(w, v1.ErrorStatusRecordFound, + nil) + return + } + if errors.Is(err, backend.ErrRecordNotFound) { + log.Infof("%v update %v record not found: %x", + remoteAddr(r), cmd, token) + p.respondWithUserError(w, v1.ErrorStatusRecordNotFound, nil) + return + } + if errors.Is(err, backend.ErrNoChanges) { + log.Infof("%v update %v record no changes: %x", + remoteAddr(r), cmd, token) + p.respondWithUserError(w, v1.ErrorStatusNoChanges, nil) + return + } + // Check for content error. + var contentErr backend.ContentVerificationError + if errors.As(err, &contentErr) { + log.Infof("%v update %v record content error: %v", + remoteAddr(r), cmd, contentErr) + p.respondWithUserError(w, contentErr.ErrorCode, + contentErr.ErrorContext) + return + } + // Generic internal error. + errorCode := time.Now().Unix() + log.Errorf("%v Update %v record error code %v: %v", + remoteAddr(r), cmd, errorCode, err) + p.respondWithServerError(w, errorCode) + return + } + + // Prepare reply. + response := p.identity.SignMessage(challenge) + reply := v1.UpdateRecordReply{ + Response: hex.EncodeToString(response[:]), + Record: p.convertBackendRecord(*record), + } + + log.Infof("Update %v record %v: token %v", cmd, remoteAddr(r), + record.RecordMetadata.Token) + + util.RespondWithJSON(w, http.StatusOK, reply) +} + +func (p *politeia) updateUnvetted(w http.ResponseWriter, r *http.Request) { + p.updateRecord(w, r, false) +} + +func (p *politeia) updateVetted(w http.ResponseWriter, r *http.Request) { + p.updateRecord(w, r, true) +} + +func (p *politeia) getUnvetted(w http.ResponseWriter, r *http.Request) { + var t v1.GetUnvetted + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&t); err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) + return + } + + challenge, err := hex.DecodeString(t.Challenge) + if err != nil || len(challenge) != v1.ChallengeSize { + p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) + return + } + response := p.identity.SignMessage(challenge) + + reply := v1.GetUnvettedReply{ + Response: hex.EncodeToString(response[:]), + } + + // Validate token + token, err := util.ConvertStringToken(t.Token) + if err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) + return + } + + // Ask backend about the censorship token. + bpr, err := p.backend.GetUnvetted(token, t.Version) + switch { + case errors.Is(err, backend.ErrRecordNotFound): + // Record not found + log.Infof("Get unvetted record %v: token %v not found", + remoteAddr(r), t.Token) + p.respondWithUserError(w, v1.ErrorStatusRecordNotFound, nil) + return + + case err != nil: + // Generic internal error + errorCode := time.Now().Unix() + log.Errorf("%v Get unvetted record error code %v: %v", + remoteAddr(r), errorCode, err) + p.respondWithServerError(w, errorCode) + return + + case bpr.RecordMetadata.Status == backend.MDStatusCensored: + // Record has been censored. The default case will verify the + // record before sending it off. This will fail for censored + // records since the files will not exist, they've been deleted, + // so skip the verification step. + reply.Record = p.convertBackendRecord(*bpr) + log.Infof("Get unvetted record %v: token %v", remoteAddr(r), t.Token) + + default: + reply.Record = p.convertBackendRecord(*bpr) + + // Double check record bits before sending them off + err := v1.Verify(p.identity.Public, + reply.Record.CensorshipRecord, reply.Record.Files) + if err != nil { + // Generic internal error. + errorCode := time.Now().Unix() + log.Errorf("%v Get unvetted record CORRUPTION "+ + "error code %v: %v", remoteAddr(r), errorCode, + err) + p.respondWithServerError(w, errorCode) + return + } + + log.Infof("Get unvetted record %v: token %v", remoteAddr(r), t.Token) + } + + util.RespondWithJSON(w, http.StatusOK, reply) +} + +func (p *politeia) getVetted(w http.ResponseWriter, r *http.Request) { + var t v1.GetVetted + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&t); err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) + return + } + + challenge, err := hex.DecodeString(t.Challenge) + if err != nil || len(challenge) != v1.ChallengeSize { + p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) + return + } + response := p.identity.SignMessage(challenge) + + reply := v1.GetVettedReply{ + Response: hex.EncodeToString(response[:]), + } + + // Validate token + token, err := util.ConvertStringToken(t.Token) + if err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) + return + } + + // Ask backend about the censorship token. + bpr, err := p.backend.GetVetted(token, t.Version) + switch { + case errors.Is(err, backend.ErrRecordNotFound): + // Record not found + log.Infof("Get vetted record %v: token %v not found", + remoteAddr(r), t.Token) + p.respondWithUserError(w, v1.ErrorStatusRecordNotFound, nil) + return + + case err != nil: + // Generic internal error + errorCode := time.Now().Unix() + log.Errorf("%v Get vetted record error code %v: %v", + remoteAddr(r), errorCode, err) + p.respondWithServerError(w, errorCode) + return + + case bpr.RecordMetadata.Status == backend.MDStatusCensored: + // Record has been censored. The default case will verify the + // record before sending it off. This will fail for censored + // records since the files will not exist, they've been deleted, + // so skip the verification step. + reply.Record = p.convertBackendRecord(*bpr) + log.Infof("Get vetted record %v: token %v", remoteAddr(r), t.Token) + + default: + reply.Record = p.convertBackendRecord(*bpr) + + // Double check record bits before sending them off + err := v1.Verify(p.identity.Public, + reply.Record.CensorshipRecord, reply.Record.Files) + if err != nil { + // Generic internal error. + errorCode := time.Now().Unix() + log.Errorf("%v Get vetted record CORRUPTION "+ + "error code %v: %v", remoteAddr(r), errorCode, + err) + p.respondWithServerError(w, errorCode) + return + } + + log.Infof("Get vetted record %v: token %v", remoteAddr(r), t.Token) + } + + util.RespondWithJSON(w, http.StatusOK, reply) +} + +func (p *politeia) inventory(w http.ResponseWriter, r *http.Request) { + var i v1.Inventory + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&i); err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) + return + } + + challenge, err := hex.DecodeString(i.Challenge) + if err != nil || len(challenge) != v1.ChallengeSize { + p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) + return + } + response := p.identity.SignMessage(challenge) + + reply := v1.InventoryReply{ + Response: hex.EncodeToString(response[:]), + } + + // Ask backend for inventory + prs, brs, err := p.backend.Inventory(i.VettedCount, i.VettedStart, i.BranchesCount, + i.IncludeFiles, i.AllVersions) + if err != nil { + // Generic internal error. + errorCode := time.Now().Unix() + log.Errorf("%v Inventory error code %v: %v", remoteAddr(r), + errorCode, err) + + p.respondWithServerError(w, errorCode) + return + } + + // Convert backend records + vetted := make([]v1.Record, 0, len(prs)) + for _, v := range prs { + vetted = append(vetted, p.convertBackendRecord(v)) + } + reply.Vetted = vetted + + // Convert branches + unvetted := make([]v1.Record, 0, len(brs)) + for _, v := range brs { + unvetted = append(unvetted, p.convertBackendRecord(v)) + } + reply.Branches = unvetted + + util.RespondWithJSON(w, http.StatusOK, reply) +} + +func (p *politeia) setVettedStatus(w http.ResponseWriter, r *http.Request) { + var t v1.SetVettedStatus + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&t); err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) + return + } + + challenge, err := hex.DecodeString(t.Challenge) + if err != nil || len(challenge) != v1.ChallengeSize { + p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) + return + } + response := p.identity.SignMessage(challenge) + + // Validate token + token, err := util.ConvertStringToken(t.Token) + if err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) + return + } + + // Ask backend to update status + record, err := p.backend.SetVettedStatus(token, + convertFrontendStatus(t.Status), + convertFrontendMetadataStream(t.MDAppend), + convertFrontendMetadataStream(t.MDOverwrite)) + if err != nil { + // Check for specific errors + if errors.Is(err, backend.ErrRecordNotFound) { + log.Infof("%v updateStatus record not "+ + "found: %x", remoteAddr(r), token) + p.respondWithUserError(w, v1.ErrorStatusRecordNotFound, nil) + return + } + var serr backend.StateTransitionError + if errors.As(err, &serr) { + log.Infof("%v %v %v", remoteAddr(r), t.Token, err) + p.respondWithUserError(w, v1.ErrorStatusInvalidRecordStatusTransition, nil) + return + } + // Generic internal error. + errorCode := time.Now().Unix() + log.Errorf("%v Set status error code %v: %v", + remoteAddr(r), errorCode, err) + + p.respondWithServerError(w, errorCode) + return + } + + // Prepare reply. + reply := v1.SetVettedStatusReply{ + Response: hex.EncodeToString(response[:]), + Record: p.convertBackendRecord(*record), + } + + s := convertBackendStatus(record.RecordMetadata.Status) + log.Infof("Set vetted record status %v: token %v status %v", + remoteAddr(r), t.Token, v1.RecordStatus[s]) + + util.RespondWithJSON(w, http.StatusOK, reply) +} + +func (p *politeia) setUnvettedStatus(w http.ResponseWriter, r *http.Request) { + var t v1.SetUnvettedStatus + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&t); err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) + return + } + + challenge, err := hex.DecodeString(t.Challenge) + if err != nil || len(challenge) != v1.ChallengeSize { + p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) + return + } + response := p.identity.SignMessage(challenge) + + // Validate token + token, err := util.ConvertStringToken(t.Token) + if err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) + return + } + + // Ask backend to update unvetted status + record, err := p.backend.SetUnvettedStatus(token, + convertFrontendStatus(t.Status), + convertFrontendMetadataStream(t.MDAppend), + convertFrontendMetadataStream(t.MDOverwrite)) + if err != nil { + // Check for specific errors + if errors.Is(err, backend.ErrRecordNotFound) { + log.Infof("%v updateUnvettedStatus record not "+ + "found: %x", remoteAddr(r), token) + p.respondWithUserError(w, v1.ErrorStatusRecordNotFound, nil) + return + } + var serr backend.StateTransitionError + if errors.As(err, &serr) { + log.Infof("%v %v %v", remoteAddr(r), t.Token, err) + p.respondWithUserError(w, v1.ErrorStatusInvalidRecordStatusTransition, nil) + return + } + // Generic internal error. + errorCode := time.Now().Unix() + log.Errorf("%v Set unvetted status error code %v: %v", + remoteAddr(r), errorCode, err) + + p.respondWithServerError(w, errorCode) + return + } + + // Prepare reply. + reply := v1.SetUnvettedStatusReply{ + Response: hex.EncodeToString(response[:]), + Record: p.convertBackendRecord(*record), + } + + s := convertBackendStatus(record.RecordMetadata.Status) + log.Infof("Set unvetted record status %v: token %v status %v", + remoteAddr(r), t.Token, v1.RecordStatus[s]) + + util.RespondWithJSON(w, http.StatusOK, reply) +} + +func (p *politeia) updateVettedMetadata(w http.ResponseWriter, r *http.Request) { + var t v1.UpdateVettedMetadata + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&t); err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) + return + } + + challenge, err := hex.DecodeString(t.Challenge) + if err != nil || len(challenge) != v1.ChallengeSize { + p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) + return + } + response := p.identity.SignMessage(challenge) + + // Validate token + token, err := util.ConvertStringToken(t.Token) + if err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) + return + } + + log.Infof("Update vetted metadata submitted %v: %x", remoteAddr(r), + token) + + err = p.backend.UpdateVettedMetadata(token, + convertFrontendMetadataStream(t.MDAppend), + convertFrontendMetadataStream(t.MDOverwrite)) + if err != nil { + if errors.Is(err, backend.ErrNoChanges) { + log.Infof("%v update vetted metadata no changes: %x", + remoteAddr(r), token) + p.respondWithUserError(w, v1.ErrorStatusNoChanges, nil) + return + } + // Check for content error. + var contentErr backend.ContentVerificationError + if errors.As(err, &contentErr) { + log.Infof("%v update vetted metadata content error: %v", + remoteAddr(r), contentErr) + p.respondWithUserError(w, contentErr.ErrorCode, + contentErr.ErrorContext) + return + } + // Generic internal error. + errorCode := time.Now().Unix() + log.Errorf("%v Update vetted metadata error code %v: %v", + remoteAddr(r), errorCode, err) + p.respondWithServerError(w, errorCode) + return + } + + // Reply + reply := v1.UpdateVettedMetadataReply{ + Response: hex.EncodeToString(response[:]), + } + + log.Infof("Update vetted metadata %v: token %x", remoteAddr(r), token) + + util.RespondWithJSON(w, http.StatusOK, reply) +} + +func (p *politeia) updateUnvettedMetadata(w http.ResponseWriter, r *http.Request) { + var t v1.UpdateUnvettedMetadata + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&t); err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) + return + } + + challenge, err := hex.DecodeString(t.Challenge) + if err != nil || len(challenge) != v1.ChallengeSize { + p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) + return + } + + token, err := util.ConvertStringToken(t.Token) + if err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) + return + } + + log.Infof("Update unvetted metadata submitted %v: %x", remoteAddr(r), + token) + + err = p.backend.UpdateUnvettedMetadata(token, + convertFrontendMetadataStream(t.MDAppend), + convertFrontendMetadataStream(t.MDOverwrite)) + if err != nil { + // Reply with error if there were no changes + if errors.Is(err, backend.ErrNoChanges) { + log.Infof("%v update unvetted metadata no changes: %x", + remoteAddr(r), token) + p.respondWithUserError(w, v1.ErrorStatusNoChanges, nil) + return + } + // Check for content error. + var cverr backend.ContentVerificationError + if errors.As(err, &cverr) { + log.Infof("%v update unvetted metadata content error: %v", + remoteAddr(r), cverr) + p.respondWithUserError(w, cverr.ErrorCode, + cverr.ErrorContext) + return + } + // Generic internal error. + errorCode := time.Now().Unix() + log.Errorf("%v update unvetted metadata error code %v: %v", + remoteAddr(r), errorCode, err) + p.respondWithServerError(w, errorCode) + return + } + + // Prepare reply + response := p.identity.SignMessage(challenge) + reply := v1.UpdateUnvettedMetadataReply{ + Response: hex.EncodeToString(response[:]), + } + + log.Infof("Update unvetted metadata %v: token %x", remoteAddr(r), token) + + util.RespondWithJSON(w, http.StatusOK, reply) +} + +func (p *politeia) pluginInventory(w http.ResponseWriter, r *http.Request) { + var pi v1.PluginInventory + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&pi); err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, + nil) + return + } + + challenge, err := hex.DecodeString(pi.Challenge) + if err != nil || len(challenge) != v1.ChallengeSize { + p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) + return + } + response := p.identity.SignMessage(challenge) + + // Get plugins + plugins, err := p.backend.GetPlugins() + if err != nil { + errorCode := time.Now().Unix() + log.Errorf("%v get plugins: %v ", remoteAddr(r), err) + p.respondWithServerError(w, errorCode) + return + } + + // Prepare reply + reply := v1.PluginInventoryReply{ + Plugins: convertBackendPlugins(plugins), + Response: hex.EncodeToString(response[:]), + } + + util.RespondWithJSON(w, http.StatusOK, reply) +} + +func (p *politeia) pluginCommand(w http.ResponseWriter, r *http.Request) { + var pc v1.PluginCommand + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&pc); err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, + nil) + return + } + + challenge, err := hex.DecodeString(pc.Challenge) + if err != nil || len(challenge) != v1.ChallengeSize { + p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) + return + } + + cid, payload, err := p.backend.Plugin(pc.Command, pc.Payload) + if err != nil { + // Generic internal error. + errorCode := time.Now().Unix() + log.Errorf("%v %v: backend plugin failed with "+ + "command:%v payload:%v err:%v", remoteAddr(r), + errorCode, pc.Command, pc.Payload, err) + p.respondWithServerError(w, errorCode) + return + } + + // Prepare reply + response := p.identity.SignMessage(challenge) + reply := v1.PluginCommandReply{ + Response: hex.EncodeToString(response[:]), + ID: pc.ID, + Command: cid, + CommandID: pc.CommandID, + Payload: payload, + } + + util.RespondWithJSON(w, http.StatusOK, reply) +} + +func convertBackendPluginSetting(bpi backend.PluginSetting) v1.PluginSetting { + return v1.PluginSetting{ + Key: bpi.Key, + Value: bpi.Value, + } +} + +func convertBackendPlugins(bplugins []backend.Plugin) []v1.Plugin { + plugins := make([]v1.Plugin, 0, len(bplugins)) + for _, v := range bplugins { + p := v1.Plugin{ + ID: v.ID, + Version: v.Version, + Settings: make([]v1.PluginSetting, 0, len(v.Settings)), + } + for _, v := range v.Settings { + p.Settings = append(p.Settings, convertBackendPluginSetting(v)) + } + plugins = append(plugins, p) + } + return plugins +} + +// convertBackendMetadataStream converts a backend metadata stream to an API +// metadata stream. +func convertBackendMetadataStream(mds backend.MetadataStream) v1.MetadataStream { + return v1.MetadataStream{ + ID: mds.ID, + Payload: mds.Payload, + } +} + +// convertBackendStatus converts a backend MDStatus to an API status. +func convertBackendStatus(status backend.MDStatusT) v1.RecordStatusT { + s := v1.RecordStatusInvalid + switch status { + case backend.MDStatusInvalid: + s = v1.RecordStatusInvalid + case backend.MDStatusUnvetted: + s = v1.RecordStatusNotReviewed + case backend.MDStatusVetted: + s = v1.RecordStatusPublic + case backend.MDStatusCensored: + s = v1.RecordStatusCensored + case backend.MDStatusIterationUnvetted: + s = v1.RecordStatusUnreviewedChanges + case backend.MDStatusArchived: + s = v1.RecordStatusArchived + } + return s +} + +// convertFrontendStatus convert an API status to a backend MDStatus. +func convertFrontendStatus(status v1.RecordStatusT) backend.MDStatusT { + s := backend.MDStatusInvalid + switch status { + case v1.RecordStatusInvalid: + s = backend.MDStatusInvalid + case v1.RecordStatusNotReviewed: + s = backend.MDStatusUnvetted + case v1.RecordStatusPublic: + s = backend.MDStatusVetted + case v1.RecordStatusCensored: + s = backend.MDStatusCensored + case v1.RecordStatusArchived: + s = backend.MDStatusArchived + } + return s +} + +func convertFrontendFiles(f []v1.File) []backend.File { + files := make([]backend.File, 0, len(f)) + for _, v := range f { + files = append(files, backend.File{ + Name: v.Name, + MIME: v.MIME, + Digest: v.Digest, + Payload: v.Payload, + }) + } + return files +} + +func convertFrontendMetadataStream(mds []v1.MetadataStream) []backend.MetadataStream { + m := make([]backend.MetadataStream, 0, len(mds)) + for _, v := range mds { + m = append(m, backend.MetadataStream{ + ID: v.ID, + Payload: v.Payload, + }) + } + return m +} + +func (p *politeia) convertBackendRecord(br backend.Record) v1.Record { + rm := br.RecordMetadata + + // Calculate signature + signature := p.identity.SignMessage([]byte(rm.Merkle + rm.Token)) + + // Convert MetadataStream + md := make([]v1.MetadataStream, 0, len(br.Metadata)) + for k := range br.Metadata { + md = append(md, convertBackendMetadataStream(br.Metadata[k])) + } + + // Convert record + pr := v1.Record{ + Status: convertBackendStatus(rm.Status), + Timestamp: rm.Timestamp, + CensorshipRecord: v1.CensorshipRecord{ + Merkle: rm.Merkle, + Token: rm.Token, + Signature: hex.EncodeToString(signature[:]), + }, + Version: br.Version, + Metadata: md, + } + pr.Files = make([]v1.File, 0, len(br.Files)) + for _, v := range br.Files { + pr.Files = append(pr.Files, + v1.File{ + Name: v.Name, + MIME: v.MIME, + Digest: v.Digest, + Payload: v.Payload, + }) + } + + return pr +} diff --git a/politeiad/handlersv2.go b/politeiad/v2.go similarity index 100% rename from politeiad/handlersv2.go rename to politeiad/v2.go From e2937518829ee78d667ceddb51fe7d78c130608d Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 8 Mar 2021 13:31:04 -0600 Subject: [PATCH 369/449] politeiad: Implement plugin routes. --- politeiad/api/v2/v2.go | 15 +- politeiad/backendv2/backendv2.go | 8 +- .../backendv2/tstorebe/plugins/pi/hooks.go | 2 +- .../tstorebe/plugins/ticketvote/cmds.go | 36 ++--- .../tstorebe/plugins/ticketvote/ticketvote.go | 8 +- politeiad/backendv2/tstorebe/tstorebe.go | 12 +- politeiad/v2.go | 145 +++++++++++++++++- 7 files changed, 184 insertions(+), 42 deletions(-) diff --git a/politeiad/api/v2/v2.go b/politeiad/api/v2/v2.go index 2181edb97..76c8fd5a5 100644 --- a/politeiad/api/v2/v2.go +++ b/politeiad/api/v2/v2.go @@ -464,16 +464,17 @@ type PluginReads struct { Cmds []PluginCmd `json:"cmds"` } -// PluginReadReply is the reply to an individual read only plugin command that -// is part of a batch of plugin commands. -type PluginReadReply struct { +// PluginCmdReply is the reply to an individual plugin command that is part of +// a batch of plugin commands. The error will be included in the reply if one +// was encountered. +type PluginCmdReply struct { Token string `json:"token"` // Censorship token ID string `json:"id"` // Plugin identifier Command string `json:"command"` // Plugin command Payload string `json:"payload"` // Response payload - // UserError will be populated if a ErrorCodeT is encountered - // before the plugin command could be executed. + // UserError will be populated if a user error is encountered prior + // to plugin command execution. UserError *UserErrorReply `json:"usererror,omitempty"` // PluginError will be populated if a plugin error occurred during @@ -483,8 +484,8 @@ type PluginReadReply struct { // PluginReadsReply is the reply to the PluginReads command. type PluginReadsReply struct { - Response string `json:"response"` // Challenge response - Replies []PluginReadReply `json:"replies"` + Response string `json:"response"` // Challenge response + Replies []PluginCmdReply `json:"replies"` } // PluginSetting is a structure that holds key/value pairs of a plugin setting. diff --git a/politeiad/backendv2/backendv2.go b/politeiad/backendv2/backendv2.go index ab2573d33..e80950d0a 100644 --- a/politeiad/backendv2/backendv2.go +++ b/politeiad/backendv2/backendv2.go @@ -334,12 +334,12 @@ type Backend interface { // PluginSetup performs any required plugin setup. PluginSetup(pluginID string) error - // PluginCmdRead executes a read plugin command. - PluginCmdRead(token []byte, pluginID, pluginCmd, + // PluginRead executes a read-only plugin command. + PluginRead(token []byte, pluginID, pluginCmd, payload string) (string, error) - // PluginCmdWrite executes a write plugin command. - PluginCmdWrite(token []byte, pluginID, pluginCmd, + // PluginWrite executes a plugin command that writes data. + PluginWrite(token []byte, pluginID, pluginCmd, payload string) (string, error) // PluginInventory returns all registered plugins. diff --git a/politeiad/backendv2/tstorebe/plugins/pi/hooks.go b/politeiad/backendv2/tstorebe/plugins/pi/hooks.go index 6006e7359..7d8eb36a9 100644 --- a/politeiad/backendv2/tstorebe/plugins/pi/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/pi/hooks.go @@ -222,7 +222,7 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { } func (p *piPlugin) voteSummary(token []byte) (*ticketvote.SummaryReply, error) { - reply, err := p.backend.PluginCmdRead(token, ticketvote.PluginID, + reply, err := p.backend.PluginRead(token, ticketvote.PluginID, ticketvote.CmdSummary, "") if err != nil { return nil, err diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go index 0730e04d9..5c9d2d9bc 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go @@ -374,7 +374,7 @@ func (p *ticketVotePlugin) voteDetails(treeID int64) (*ticketvote.VoteDetails, e } func (p *ticketVotePlugin) voteDetailsByToken(token []byte) (*ticketvote.VoteDetails, error) { - reply, err := p.backend.PluginCmdRead(token, ticketvote.PluginID, + reply, err := p.backend.PluginRead(token, ticketvote.PluginID, ticketvote.CmdDetails, "") if err != nil { return nil, err @@ -440,7 +440,7 @@ func (p *ticketVotePlugin) voteOptionResults(token []byte, options []ticketvote. default: // Votes are not in the cache. Pull them from the backend. - reply, err := p.backend.PluginCmdRead(token, ticketvote.PluginID, + reply, err := p.backend.PluginRead(token, ticketvote.PluginID, ticketvote.CmdResults, "") if err != nil { return nil, err @@ -480,10 +480,10 @@ func (p *ticketVotePlugin) summariesForRunoff(parentToken string) (map[string]ti if err != nil { return nil, err } - reply, err := p.backend.PluginCmdRead(parent, ticketvote.PluginID, + reply, err := p.backend.PluginRead(parent, ticketvote.PluginID, cmdRunoffDetails, "") if err != nil { - return nil, fmt.Errorf("PluginCmdRead %x %v %v: %v", + return nil, fmt.Errorf("PluginRead %x %v %v: %v", parent, ticketvote.PluginID, cmdRunoffDetails, err) } var rdr runoffDetailsReply @@ -721,10 +721,10 @@ func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) } func (p *ticketVotePlugin) summaryByToken(token []byte) (*ticketvote.SummaryReply, error) { - reply, err := p.backend.PluginCmdRead(token, ticketvote.PluginID, + reply, err := p.backend.PluginRead(token, ticketvote.PluginID, ticketvote.CmdSummary, "") if err != nil { - return nil, fmt.Errorf("PluginCmdRead %x %v %v: %v", + return nil, fmt.Errorf("PluginRead %x %v %v: %v", token, ticketvote.PluginID, ticketvote.CmdSummary, err) } var sr ticketvote.SummaryReply @@ -791,10 +791,10 @@ func (p *ticketVotePlugin) bestBlock() (uint32, error) { if err != nil { return 0, err } - reply, err := p.backend.PluginCmdRead(nil, dcrdata.PluginID, + reply, err := p.backend.PluginRead(nil, dcrdata.PluginID, dcrdata.CmdBestBlock, string(payload)) if err != nil { - return 0, fmt.Errorf("PluginCmdRead %v %v: %v", + return 0, fmt.Errorf("PluginRead %v %v: %v", dcrdata.PluginID, dcrdata.CmdBestBlock, err) } @@ -827,10 +827,10 @@ func (p *ticketVotePlugin) bestBlockUnsafe() (uint32, error) { if err != nil { return 0, err } - reply, err := p.backend.PluginCmdRead(nil, dcrdata.PluginID, + reply, err := p.backend.PluginRead(nil, dcrdata.PluginID, dcrdata.CmdBestBlock, string(payload)) if err != nil { - return 0, fmt.Errorf("PluginCmdRead %v %v: %v", + return 0, fmt.Errorf("PluginRead %v %v: %v", dcrdata.PluginID, dcrdata.CmdBestBlock, err) } @@ -864,10 +864,10 @@ func (p *ticketVotePlugin) largestCommitmentAddrs(tickets []string) (map[string] if err != nil { return nil, err } - reply, err := p.backend.PluginCmdRead(nil, dcrdata.PluginID, + reply, err := p.backend.PluginRead(nil, dcrdata.PluginID, dcrdata.CmdTxsTrimmed, string(payload)) if err != nil { - return nil, fmt.Errorf("PluginCmdRead %v %v: %v", + return nil, fmt.Errorf("PluginRead %v %v: %v", dcrdata.PluginID, dcrdata.CmdTxsTrimmed, err) } var ttr dcrdata.TxsTrimmedReply @@ -933,10 +933,10 @@ func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, if err != nil { return nil, err } - reply, err := p.backend.PluginCmdRead(nil, dcrdata.PluginID, + reply, err := p.backend.PluginRead(nil, dcrdata.PluginID, dcrdata.CmdBlockDetails, string(payload)) if err != nil { - return nil, fmt.Errorf("PluginCmdRead %v %v: %v", + return nil, fmt.Errorf("PluginRead %v %v: %v", dcrdata.PluginID, dcrdata.CmdBlockDetails, err) } var bdr dcrdata.BlockDetailsReply @@ -957,10 +957,10 @@ func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, if err != nil { return nil, err } - reply, err = p.backend.PluginCmdRead(nil, dcrdata.PluginID, + reply, err = p.backend.PluginRead(nil, dcrdata.PluginID, dcrdata.CmdTicketPool, string(payload)) if err != nil { - return nil, fmt.Errorf("PluginCmdRead %v %v: %v", + return nil, fmt.Errorf("PluginRead %v %v: %v", dcrdata.PluginID, dcrdata.CmdTicketPool, err) } var tpr dcrdata.TicketPoolReply @@ -1917,14 +1917,14 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. if err != nil { return nil, err } - _, err = p.backend.PluginCmdWrite(token, ticketvote.PluginID, + _, err = p.backend.PluginWrite(token, ticketvote.PluginID, cmdStartRunoffSubmission, string(b)) if err != nil { var ue backend.PluginError if errors.As(err, &ue) { return nil, err } - return nil, fmt.Errorf("PluginCmdWrite %x %v %v: %v", + return nil, fmt.Errorf("PluginWrite %x %v %v: %v", token, ticketvote.PluginID, cmdStartRunoffSubmission, err) } } diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go index e5fe31896..47f28f168 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go @@ -104,10 +104,10 @@ func (p *ticketVotePlugin) Setup() error { return err } - reply, err := p.backend.PluginCmdRead(token, ticketvote.PluginID, + reply, err := p.backend.PluginRead(token, ticketvote.PluginID, ticketvote.CmdDetails, "") if err != nil { - return fmt.Errorf("PluginCmdRead %x %v %v: %v", + return fmt.Errorf("PluginRead %x %v %v: %v", token, ticketvote.PluginID, ticketvote.CmdDetails, err) } var dr ticketvote.DetailsReply @@ -125,10 +125,10 @@ func (p *ticketVotePlugin) Setup() error { p.activeVotesAdd(*dr.Vote) // Get cast votes - reply, err = p.backend.PluginCmdRead(token, ticketvote.PluginID, + reply, err = p.backend.PluginRead(token, ticketvote.PluginID, ticketvote.CmdResults, "") if err != nil { - return fmt.Errorf("PluginCmdRead %x %v %v: %v", + return fmt.Errorf("PluginRead %x %v %v: %v", token, ticketvote.PluginID, ticketvote.CmdResults, err) } var rr ticketvote.ResultsReply diff --git a/politeiad/backendv2/tstorebe/tstorebe.go b/politeiad/backendv2/tstorebe/tstorebe.go index 524360482..f9704fee9 100644 --- a/politeiad/backendv2/tstorebe/tstorebe.go +++ b/politeiad/backendv2/tstorebe/tstorebe.go @@ -1005,11 +1005,11 @@ func (t *tstoreBackend) PluginSetup(pluginID string) error { return t.tstore.PluginSetup(pluginID) } -// PluginCmdRead executes a read plugin command. +// PluginRead executes a read-only plugin command. // // This function satisfies the Backend interface. -func (t *tstoreBackend) PluginCmdRead(token []byte, pluginID, pluginCmd, payload string) (string, error) { - log.Tracef("PluginCmdRead: %x %v %v", token, pluginID, pluginCmd) +func (t *tstoreBackend) PluginRead(token []byte, pluginID, pluginCmd, payload string) (string, error) { + log.Tracef("PluginRead: %x %v %v", token, pluginID, pluginCmd) // The token is optional. If a token is not provided then a tree ID // will not be provided to the plugin. @@ -1034,11 +1034,11 @@ func (t *tstoreBackend) PluginCmdRead(token []byte, pluginID, pluginCmd, payload return t.tstore.PluginCmd(treeID, token, pluginID, pluginCmd, payload) } -// PluginCmdWrite executes a write plugin command. +// PluginWrite executes a plugin command that writes data. // // This function satisfies the Backend interface. -func (t *tstoreBackend) PluginCmdWrite(token []byte, pluginID, pluginCmd, payload string) (string, error) { - log.Tracef("PluginCmdWrite: %x %v %v", token, pluginID, pluginCmd) +func (t *tstoreBackend) PluginWrite(token []byte, pluginID, pluginCmd, payload string) (string, error) { + log.Tracef("PluginWrite: %x %v %v", token, pluginID, pluginCmd) // Verify record exists if !t.RecordExists(token) { diff --git a/politeiad/v2.go b/politeiad/v2.go index c77753a94..a49ccb6aa 100644 --- a/politeiad/v2.go +++ b/politeiad/v2.go @@ -421,12 +421,153 @@ func (p *politeia) handleInventory(w http.ResponseWriter, r *http.Request) { func (p *politeia) handlePluginWrite(w http.ResponseWriter, r *http.Request) { log.Tracef("handlePluginWrite") - panic("not implemented") + + // Decode request + var pw v2.PluginWrite + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&pw); err != nil { + respondWithErrorV2(w, r, "handlePluginWrite: unmarshal", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeRequestPayloadInvalid, + }) + return + } + challenge, err := hex.DecodeString(pw.Challenge) + if err != nil || len(challenge) != v2.ChallengeSize { + respondWithErrorV2(w, r, "handlePluginWrite: decode challenge", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeChallengeInvalid, + }) + return + } + token, err := decodeToken(pw.Cmd.Token) + if err != nil { + respondWithErrorV2(w, r, "handlePluginWrite: decode token", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeTokenInvalid, + }) + return + } + + // Execute plugin cmd + payload, err := p.backendv2.PluginWrite(token, pw.Cmd.ID, + pw.Cmd.Command, pw.Cmd.Payload) + if err != nil { + respondWithErrorV2(w, r, + "handlePluginWrite: PluginWrite: %v", err) + return + } + + // Prepare reply + response := p.identity.SignMessage(challenge) + pwr := v2.PluginWriteReply{ + Response: hex.EncodeToString(response[:]), + Payload: payload, + } + + util.RespondWithJSON(w, http.StatusOK, pwr) } func (p *politeia) handlePluginReads(w http.ResponseWriter, r *http.Request) { log.Tracef("handlePluginReads") - panic("not implemented") + + // Decode request + var pr v2.PluginReads + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&pr); err != nil { + respondWithErrorV2(w, r, "handlePluginReads: unmarshal", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeRequestPayloadInvalid, + }) + return + } + challenge, err := hex.DecodeString(pr.Challenge) + if err != nil || len(challenge) != v2.ChallengeSize { + respondWithErrorV2(w, r, "handlePluginReads: decode challenge", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeChallengeInvalid, + }) + return + } + + replies := make([]v2.PluginCmdReply, len(pr.Cmds)) + for k, v := range pr.Cmds { + // Decode token + token, err := decodeToken(v.Token) + if err != nil { + // Invalid token. Save the reply and continue to next cmd. + replies[k] = v2.PluginCmdReply{ + UserError: &v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeTokenInvalid, + }, + } + continue + } + + // Execute plugin cmd + replyPayload, err := p.backendv2.PluginRead(token, v.ID, + v.Command, v.Payload) + if err != nil { + var ( + errCode = convertErrorToV2(err) + pe backendv2.PluginError + ) + switch { + case errCode != v2.ErrorCodeInvalid: + // User error. Save the reply and continue to next cmd. + replies[k] = v2.PluginCmdReply{ + UserError: &v2.UserErrorReply{ + ErrorCode: errCode, + }, + } + continue + + case errors.As(err, &pe): + // Plugin error. Save the reply and continue to next cmd. + replies[k] = v2.PluginCmdReply{ + PluginError: &v2.PluginErrorReply{ + PluginID: pe.PluginID, + ErrorCode: pe.ErrorCode, + ErrorContext: pe.ErrorContext, + }, + } + + default: + // Internal server error. Log it and return a 500. + t := time.Now().Unix() + e := fmt.Sprintf("PluginRead %v %v %v: %v", + v.ID, v.Command, v.Payload, err) + log.Errorf("%v %v %v %v Internal error %v: %v", + util.RemoteAddr(r), r.Method, r.URL, r.Proto, t, e) + log.Errorf("Stacktrace (NOT A REAL CRASH): %s", debug.Stack()) + + util.RespondWithJSON(w, http.StatusInternalServerError, + v2.ServerErrorReply{ + ErrorCode: t, + }) + return + } + + // Successful cmd execution. Save the reply and continue to + // the next cmd. + replies[k] = v2.PluginCmdReply{ + Token: v.Token, + ID: v.ID, + Command: v.Command, + Payload: replyPayload, + } + } + } + + // Prepare reply + response := p.identity.SignMessage(challenge) + prr := v2.PluginReadsReply{ + Response: hex.EncodeToString(response[:]), + Replies: replies, + } + + util.RespondWithJSON(w, http.StatusOK, prr) + } func (p *politeia) handlePluginInventory(w http.ResponseWriter, r *http.Request) { From fd244596d768d509f30d2ea4411f30e23bc7c4f6 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 8 Mar 2021 13:57:29 -0600 Subject: [PATCH 370/449] politeiad: Plugin usermd ok. --- .../backendv2/tstorebe/plugins/usermd/cmds.go | 10 +- politeiad/client/pdv2.go | 25 ++- politeiad/client/user.go | 158 ++++++++---------- politeiad/v2.go | 37 ++-- 4 files changed, 119 insertions(+), 111 deletions(-) diff --git a/politeiad/backendv2/tstorebe/plugins/usermd/cmds.go b/politeiad/backendv2/tstorebe/plugins/usermd/cmds.go index 55233fd8b..550a66a3f 100644 --- a/politeiad/backendv2/tstorebe/plugins/usermd/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/usermd/cmds.go @@ -14,7 +14,7 @@ func (p *userPlugin) cmdAuthor(treeID int64) (string, error) { log.Tracef("cmdAuthor: %v", treeID) // Get user metadata - r, err := p.tstore.RecordLatest(treeID) + r, err := p.tstore.RecordPartial(treeID, 0, nil, true) if err != nil { return "", err } @@ -51,11 +51,11 @@ func (p *userPlugin) cmdUserRecords(payload string) (string, error) { return "", err } - // TODO fix the user records reply - _ = uc - // Prepare reply - urr := usermd.UserRecordsReply{} + urr := usermd.UserRecordsReply{ + Unvetted: uc.Unvetted, + Vetted: uc.Vetted, + } reply, err := json.Marshal(urr) if err != nil { return "", err diff --git a/politeiad/client/pdv2.go b/politeiad/client/pdv2.go index 5582b2c2e..402ffdd02 100644 --- a/politeiad/client/pdv2.go +++ b/politeiad/client/pdv2.go @@ -328,7 +328,7 @@ func (c *Client) PluginWrite(ctx context.Context, cmd pdv2.PluginCmd) (string, e } // PluginReads sends a PluginReads command to the politeiad v2 API. -func (c *Client) PluginReads(ctx context.Context, cmds []pdv2.PluginCmd) ([]pdv2.PluginReadReply, error) { +func (c *Client) PluginReads(ctx context.Context, cmds []pdv2.PluginCmd) ([]pdv2.PluginCmdReply, error) { // Setup request challenge, err := util.Random(pdv2.ChallengeSize) if err != nil { @@ -391,3 +391,26 @@ func (c *Client) PluginInventory(ctx context.Context) ([]pdv2.Plugin, error) { return pir.Plugins, nil } + +func extractPluginCmdError(pcr pdv2.PluginCmdReply) error { + switch { + case pcr.UserError != nil: + return Error{ + HTTPCode: http.StatusBadRequest, + ErrorReply: ErrorReply{ + ErrorCode: int(pcr.UserError.ErrorCode), + ErrorContext: pcr.UserError.ErrorContext, + }, + } + case pcr.PluginError != nil: + return Error{ + HTTPCode: http.StatusBadRequest, + ErrorReply: ErrorReply{ + PluginID: pcr.PluginError.PluginID, + ErrorCode: pcr.PluginError.ErrorCode, + ErrorContext: pcr.PluginError.ErrorContext, + }, + } + } + return nil +} diff --git a/politeiad/client/user.go b/politeiad/client/user.go index 1c7892ec6..1caf06b23 100644 --- a/politeiad/client/user.go +++ b/politeiad/client/user.go @@ -6,106 +6,88 @@ package client import ( "context" + "encoding/json" + "fmt" + pdv2 "github.com/decred/politeia/politeiad/api/v2" "github.com/decred/politeia/politeiad/plugins/usermd" ) -// Author sends the user plugin Author command to the politeiad v1 API. +// Author sends the user plugin Author command to the politeiad v2 API. func (c *Client) Author(ctx context.Context, token string) (string, error) { - /* - // Setup request - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionRead, - State: state, - Token: token, - ID: usermd.PluginID, - Command: usermd.CmdAuthor, - Payload: "", - }, - } + // Setup request + cmds := []pdv2.PluginCmd{ + { + Token: token, + ID: usermd.PluginID, + Command: usermd.CmdAuthor, + Payload: "", + }, + } - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return "", err - } - if len(replies) == 0 { - return "", fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return "", err - } + // Send request + replies, err := c.PluginReads(ctx, cmds) + if err != nil { + return "", err + } + if len(replies) == 0 { + return "", fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCmdError(pcr) + if err != nil { + return "", err + } - // Decode reply - var ar usermd.AuthorReply - err = json.Unmarshal([]byte(pcr.Payload), &ar) - if err != nil { - return "", err - } + // Decode reply + var ar usermd.AuthorReply + err = json.Unmarshal([]byte(pcr.Payload), &ar) + if err != nil { + return "", err + } - return ar.UserID, nil - */ - return "", nil + return ar.UserID, nil } -// UserRecords sends the user plugin UserRecords command to the politeiad v1 -// API. A separate command is sent for the unvetted and vetted records. +// UserRecords sends the user plugin UserRecords command to the politeiad v2 +// API. func (c *Client) UserRecords(ctx context.Context, userID string) (*usermd.UserRecordsReply, error) { - /* - // Setup request - ur := usermd.UserRecords{ - UserID: userID, - } - b, err := json.Marshal(ur) - if err != nil { - return nil, err - } - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionRead, - State: pdv1.RecordStateUnvetted, - ID: usermd.PluginID, - Command: usermd.CmdUserRecords, - Payload: string(b), - }, - { - Action: pdv1.PluginActionRead, - State: pdv1.RecordStateVetted, - ID: usermd.PluginID, - Command: usermd.CmdUserRecords, - Payload: string(b), - }, - } + // Setup request + ur := usermd.UserRecords{ + UserID: userID, + } + b, err := json.Marshal(ur) + if err != nil { + return nil, err + } + cmds := []pdv2.PluginCmd{ + { + ID: usermd.PluginID, + Command: usermd.CmdUserRecords, + Payload: string(b), + }, + } - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } + // Send request + replies, err := c.PluginReads(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCmdError(pcr) + if err != nil { + return nil, err + } - // Decode replies - reply := make(map[string][]string, 2) // [recordState][]token - for _, v := range replies { - err = extractPluginCommandError(v) - if err != nil { - // Swallow individual errors - continue - } - var urr usermd.UserRecordsReply - err = json.Unmarshal([]byte(v.Payload), &urr) - if err != nil { - return nil, err - } - reply[v.State] = urr.Records - } + // Decode reply + var urr usermd.UserRecordsReply + err = json.Unmarshal([]byte(pcr.Payload), &urr) + if err != nil { + return nil, err + } - return reply, nil - */ - return nil, nil + return &urr, nil } diff --git a/politeiad/v2.go b/politeiad/v2.go index a49ccb6aa..2480dddc7 100644 --- a/politeiad/v2.go +++ b/politeiad/v2.go @@ -492,16 +492,19 @@ func (p *politeia) handlePluginReads(w http.ResponseWriter, r *http.Request) { replies := make([]v2.PluginCmdReply, len(pr.Cmds)) for k, v := range pr.Cmds { - // Decode token - token, err := decodeToken(v.Token) - if err != nil { - // Invalid token. Save the reply and continue to next cmd. - replies[k] = v2.PluginCmdReply{ - UserError: &v2.UserErrorReply{ - ErrorCode: v2.ErrorCodeTokenInvalid, - }, + // Decode token. The token is optional on plugin reads. + var token []byte + if v.Token != "" { + token, err = decodeToken(v.Token) + if err != nil { + // Invalid token. Save the reply and continue to next cmd. + replies[k] = v2.PluginCmdReply{ + UserError: &v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeTokenInvalid, + }, + } + continue } - continue } // Execute plugin cmd @@ -547,15 +550,15 @@ func (p *politeia) handlePluginReads(w http.ResponseWriter, r *http.Request) { }) return } + } - // Successful cmd execution. Save the reply and continue to - // the next cmd. - replies[k] = v2.PluginCmdReply{ - Token: v.Token, - ID: v.ID, - Command: v.Command, - Payload: replyPayload, - } + // Successful cmd execution. Save the reply and continue to + // the next cmd. + replies[k] = v2.PluginCmdReply{ + Token: v.Token, + ID: v.ID, + Command: v.Command, + Payload: replyPayload, } } From 0f2345bb9c919e5e3904c1547869285d71298202 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 8 Mar 2021 15:52:19 -0600 Subject: [PATCH 371/449] multi: Comments plugin and routes ok. --- politeiad/client/comments.go | 602 +++++++++++++----------------- politeiad/v2.go | 2 +- politeiawww/api/comments/v1/v1.go | 9 - politeiawww/comments/events.go | 7 +- politeiawww/comments/process.go | 132 +++++-- politeiawww/pi/events.go | 6 +- 6 files changed, 386 insertions(+), 372 deletions(-) diff --git a/politeiad/client/comments.go b/politeiad/client/comments.go index f7e2b046d..af3159fbe 100644 --- a/politeiad/client/comments.go +++ b/politeiad/client/comments.go @@ -6,366 +6,302 @@ package client import ( "context" + "encoding/json" + "fmt" + pdv2 "github.com/decred/politeia/politeiad/api/v2" "github.com/decred/politeia/politeiad/plugins/comments" ) -// CommentNew sends the comments plugin New command to the politeiad v1 API. -func (c *Client) CommentNew(ctx context.Context, state string, n comments.New) (*comments.NewReply, error) { - /* - // Setup request - b, err := json.Marshal(n) - if err != nil { - return nil, err - } - cmd := pdv2.PluginCmd{ - Token: n.Token, - ID: comments.PluginID, - Command: comments.CmdNew, - Payload: string(b), - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var nr comments.NewReply - err = json.Unmarshal([]byte(pcr.Payload), &nr) - if err != nil { - return nil, err - } - - return &nr, nil - */ - return nil, nil +// CommentNew sends the comments plugin New command to the politeiad v2 API. +func (c *Client) CommentNew(ctx context.Context, n comments.New) (*comments.Comment, error) { + // Setup request + b, err := json.Marshal(n) + if err != nil { + return nil, err + } + cmd := pdv2.PluginCmd{ + Token: n.Token, + ID: comments.PluginID, + Command: comments.CmdNew, + Payload: string(b), + } + + // Send request + reply, err := c.PluginWrite(ctx, cmd) + if err != nil { + return nil, err + } + + // Decode reply + var nr comments.NewReply + err = json.Unmarshal([]byte(reply), &nr) + if err != nil { + return nil, err + } + + return &nr.Comment, nil } -// CommentVote sends the comments plugin Vote command to the politeiad v1 API. -func (c *Client) CommentVote(ctx context.Context, state string, v comments.Vote) (*comments.VoteReply, error) { - /* - // Setup request - b, err := json.Marshal(v) - if err != nil { - return nil, err - } - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionWrite, - State: state, - Token: v.Token, - ID: comments.PluginID, - Command: comments.CmdVote, - Payload: string(b), - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var vr comments.VoteReply - err = json.Unmarshal([]byte(pcr.Payload), &vr) - if err != nil { - return nil, err - } - - return &vr, nil - */ - return nil, nil +// CommentVote sends the comments plugin Vote command to the politeiad v2 API. +func (c *Client) CommentVote(ctx context.Context, v comments.Vote) (*comments.VoteReply, error) { + // Setup request + b, err := json.Marshal(v) + if err != nil { + return nil, err + } + cmd := pdv2.PluginCmd{ + Token: v.Token, + ID: comments.PluginID, + Command: comments.CmdVote, + Payload: string(b), + } + + // Send request + reply, err := c.PluginWrite(ctx, cmd) + if err != nil { + return nil, err + } + + // Decode reply + var vr comments.VoteReply + err = json.Unmarshal([]byte(reply), &vr) + if err != nil { + return nil, err + } + + return &vr, nil } -// CommentDel sends the comments plugin Del command to the politeiad v1 API. -func (c *Client) CommentDel(ctx context.Context, state string, d comments.Del) (*comments.DelReply, error) { - /* - // Setup request - b, err := json.Marshal(d) - if err != nil { - return nil, err - } - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionWrite, - State: state, - Token: d.Token, - ID: comments.PluginID, - Command: comments.CmdDel, - Payload: string(b), - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var dr comments.DelReply - err = json.Unmarshal([]byte(pcr.Payload), &dr) - if err != nil { - return nil, err - } - - return &dr, nil - */ - return nil, nil +// CommentDel sends the comments plugin Del command to the politeiad v2 API. +func (c *Client) CommentDel(ctx context.Context, d comments.Del) (*comments.DelReply, error) { + // Setup request + b, err := json.Marshal(d) + if err != nil { + return nil, err + } + cmd := pdv2.PluginCmd{ + Token: d.Token, + ID: comments.PluginID, + Command: comments.CmdDel, + Payload: string(b), + } + + // Send request + reply, err := c.PluginWrite(ctx, cmd) + if err != nil { + return nil, err + } + + // Decode reply + var dr comments.DelReply + err = json.Unmarshal([]byte(reply), &dr) + if err != nil { + return nil, err + } + + return &dr, nil } // CommentCount sends a batch of comment plugin Count commands to the -// politeiad v1 API and returns a map[token]count with the results. If a record +// politeiad v2 API and returns a map[token]count with the results. If a record // is not found for a token or any other error occurs, that token will not be // included in the reply. -func (c *Client) CommentCount(ctx context.Context, state string, tokens []string) (map[string]uint32, error) { - /* - // Setup request - cmds := make([]pdv1.PluginCommandV2, 0, len(tokens)) - for _, v := range tokens { - cmds = append(cmds, pdv1.PluginCommandV2{ - Action: pdv1.PluginActionRead, - State: state, - Token: v, - ID: comments.PluginID, - Command: comments.CmdCount, - }) - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) +func (c *Client) CommentCount(ctx context.Context, tokens []string) (map[string]uint32, error) { + // Setup request + cmds := make([]pdv2.PluginCmd, 0, len(tokens)) + for _, v := range tokens { + cmds = append(cmds, pdv2.PluginCmd{ + Token: v, + ID: comments.PluginID, + Command: comments.CmdCount, + }) + } + + // Send request + replies, err := c.PluginReads(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) != len(cmds) { + return nil, fmt.Errorf("replies missing") + } + + // Decode replies + counts := make(map[string]uint32, len(replies)) + for _, v := range replies { + // This command swallows individual errors. The token of the + // command that errored will not be included in the reply. + err = extractPluginCmdError(v) if err != nil { - return nil, err - } - if len(replies) != len(cmds) { - return nil, fmt.Errorf("replies missing") + continue } - // Decode replies - counts := make(map[string]uint32, len(replies)) - for _, v := range replies { - // This command swallows individual errors. The token of the - // command that errored will not be included in the reply. - err = extractPluginCommandError(v) - if err != nil { - spew.Dump(err) - continue - } - - var cr comments.CountReply - err = json.Unmarshal([]byte(v.Payload), &cr) - if err != nil { - continue - } - counts[v.Token] = cr.Count - } - - return counts, nil - */ - return nil, nil -} - -// CommentsGet sends the comments plugin Get command to the politeiad v1 API. -func (c *Client) CommentsGet(ctx context.Context, state, token string, g comments.Get) (map[uint32]comments.Comment, error) { - /* - // Setup request - b, err := json.Marshal(g) + var cr comments.CountReply + err = json.Unmarshal([]byte(v.Payload), &cr) if err != nil { - return nil, err - } - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionRead, - State: state, - Token: token, - ID: comments.PluginID, - Command: comments.CmdGet, - Payload: string(b), - }, + continue } + counts[v.Token] = cr.Count + } - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var gr comments.GetReply - err = json.Unmarshal([]byte(pcr.Payload), &gr) - if err != nil { - return nil, err - } + return counts, nil +} - return gr.Comments, nil - */ - return nil, nil +// CommentsGet sends the comments plugin Get command to the politeiad v2 API. +func (c *Client) CommentsGet(ctx context.Context, token string, g comments.Get) (map[uint32]comments.Comment, error) { + // Setup request + b, err := json.Marshal(g) + if err != nil { + return nil, err + } + cmds := []pdv2.PluginCmd{ + { + Token: token, + ID: comments.PluginID, + Command: comments.CmdGet, + Payload: string(b), + }, + } + + // Send request + replies, err := c.PluginReads(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCmdError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var gr comments.GetReply + err = json.Unmarshal([]byte(pcr.Payload), &gr) + if err != nil { + return nil, err + } + + return gr.Comments, nil } -// CommentsGetAll sends the comments plugin GetAll command to the politeiad v1 +// CommentsGetAll sends the comments plugin GetAll command to the politeiad v2 // API. -func (c *Client) CommentsGetAll(ctx context.Context, state, token string) ([]comments.Comment, error) { - /* - // Setup request - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionRead, - State: state, - Token: token, - ID: comments.PluginID, - Command: comments.CmdGetAll, - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var gar comments.GetAllReply - err = json.Unmarshal([]byte(pcr.Payload), &gar) - if err != nil { - return nil, err - } - - return gar.Comments, nil - */ - return nil, nil +func (c *Client) CommentsGetAll(ctx context.Context, token string) ([]comments.Comment, error) { + // Setup request + cmds := []pdv2.PluginCmd{ + { + Token: token, + ID: comments.PluginID, + Command: comments.CmdGetAll, + }, + } + + // Send request + replies, err := c.PluginReads(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCmdError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var gar comments.GetAllReply + err = json.Unmarshal([]byte(pcr.Payload), &gar) + if err != nil { + return nil, err + } + + return gar.Comments, nil } -// CommentVotes sends the comments plugin Votes command to the politeiad v1 +// CommentVotes sends the comments plugin Votes command to the politeiad v2 // API. -func (c *Client) CommentVotes(ctx context.Context, state, token string, v comments.Votes) ([]comments.CommentVote, error) { - /* - // Setup request - b, err := json.Marshal(v) - if err != nil { - return nil, err - } - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionRead, - State: state, - Token: token, - ID: comments.PluginID, - Command: comments.CmdVotes, - Payload: string(b), - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var vr comments.VotesReply - err = json.Unmarshal([]byte(pcr.Payload), &vr) - if err != nil { - return nil, err - } - - return vr.Votes, nil - */ - return nil, nil +func (c *Client) CommentVotes(ctx context.Context, token string, v comments.Votes) ([]comments.CommentVote, error) { + // Setup request + b, err := json.Marshal(v) + if err != nil { + return nil, err + } + cmds := []pdv2.PluginCmd{ + { + Token: token, + ID: comments.PluginID, + Command: comments.CmdVotes, + Payload: string(b), + }, + } + + // Send request + replies, err := c.PluginReads(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCmdError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var vr comments.VotesReply + err = json.Unmarshal([]byte(pcr.Payload), &vr) + if err != nil { + return nil, err + } + + return vr.Votes, nil } // CommentTimestamps sends the comments plugin Timestamps command to the -// politeiad v1 API. -func (c *Client) CommentTimestamps(ctx context.Context, state, token string, t comments.Timestamps) (*comments.TimestampsReply, error) { - /* - // Setup request - b, err := json.Marshal(t) - if err != nil { - return nil, err - } - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionRead, - State: state, - Token: token, - ID: comments.PluginID, - Command: comments.CmdTimestamps, - Payload: string(b), - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var tr comments.TimestampsReply - err = json.Unmarshal([]byte(pcr.Payload), &tr) - if err != nil { - return nil, err - } - - return &tr, nil - */ - return nil, nil +// politeiad v2 API. +func (c *Client) CommentTimestamps(ctx context.Context, token string, t comments.Timestamps) (*comments.TimestampsReply, error) { + // Setup request + b, err := json.Marshal(t) + if err != nil { + return nil, err + } + cmds := []pdv2.PluginCmd{ + { + Token: token, + ID: comments.PluginID, + Command: comments.CmdTimestamps, + Payload: string(b), + }, + } + + // Send request + replies, err := c.PluginReads(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCmdError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var tr comments.TimestampsReply + err = json.Unmarshal([]byte(pcr.Payload), &tr) + if err != nil { + return nil, err + } + + return &tr, nil } diff --git a/politeiad/v2.go b/politeiad/v2.go index 2480dddc7..d70466959 100644 --- a/politeiad/v2.go +++ b/politeiad/v2.go @@ -495,7 +495,7 @@ func (p *politeia) handlePluginReads(w http.ResponseWriter, r *http.Request) { // Decode token. The token is optional on plugin reads. var token []byte if v.Token != "" { - token, err = decodeToken(v.Token) + token, err = decodeTokenAnyLength(v.Token) if err != nil { // Invalid token. Save the reply and continue to next cmd. replies[k] = v2.PluginCmdReply{ diff --git a/politeiawww/api/comments/v1/v1.go b/politeiawww/api/comments/v1/v1.go index 596da6de4..aa0445c76 100644 --- a/politeiawww/api/comments/v1/v1.go +++ b/politeiawww/api/comments/v1/v1.go @@ -19,10 +19,6 @@ const ( RouteComments = "/comments" RouteVotes = "/votes" RouteTimestamps = "/timestamps" - - // Record states - RecordStateUnvetted = "unvetted" - RecordStateVetted = "vetted" ) // ErrorCodeT represents a user error code. @@ -152,7 +148,6 @@ type CommentVote struct { // // Signature is the client signature of Token+ParentID+Comment. type New struct { - State string `json:"state"` Token string `json:"token"` ParentID uint32 `json:"parentid"` Comment string `json:"comment"` @@ -213,7 +208,6 @@ type VoteReply struct { // // Signature is the client signature of the Token+CommentID+Reason type Del struct { - State string `json:"state"` Token string `json:"token"` CommentID uint32 `json:"commentid"` Reason string `json:"reason"` @@ -230,7 +224,6 @@ type DelReply struct { // records. If a record is not found for a token then it will not be included // in the returned map. type Count struct { - State string `json:"state"` Tokens []string `json:"tokens"` } @@ -241,7 +234,6 @@ type CountReply struct { // Comments requests a record's comments. type Comments struct { - State string `json:"state"` Token string `json:"token"` } @@ -295,7 +287,6 @@ type Timestamp struct { // comment IDs are provided then the timestamps for all comments will be // returned. type Timestamps struct { - State string `json:"state"` Token string `json:"token"` CommentIDs []uint32 `json:"commentids,omitempty"` } diff --git a/politeiawww/comments/events.go b/politeiawww/comments/events.go index 1ba453fd0..bcd46e2c6 100644 --- a/politeiawww/comments/events.go +++ b/politeiawww/comments/events.go @@ -4,7 +4,10 @@ package comments -import v1 "github.com/decred/politeia/politeiawww/api/comments/v1" +import ( + pdv2 "github.com/decred/politeia/politeiad/api/v2" + v1 "github.com/decred/politeia/politeiawww/api/comments/v1" +) const ( // EventTypeNew is emitted when a new comment is made. @@ -13,6 +16,6 @@ const ( // EventNew is the event data for the EventTypeNew. type EventNew struct { - State string + State pdv2.RecordStateT Comment v1.Comment } diff --git a/politeiawww/comments/process.go b/politeiawww/comments/process.go index e5ca396fa..1b275e0aa 100644 --- a/politeiawww/comments/process.go +++ b/politeiawww/comments/process.go @@ -6,7 +6,9 @@ package comments import ( "context" + "errors" + pdv2 "github.com/decred/politeia/politeiad/api/v2" "github.com/decred/politeia/politeiad/plugins/comments" v1 "github.com/decred/politeia/politeiawww/api/comments/v1" "github.com/decred/politeia/politeiawww/config" @@ -14,6 +16,17 @@ import ( "github.com/google/uuid" ) +// NOTE: the comment commands enforce different user permissions depending on +// the state of the record (ex. only admins and the author are allowed to +// comment on unvetted records). We currently pull the record without any files +// in order to determine the record state. This is the quick and dirty way and +// was implemented like this due to development time constraints. We could +// eliminate this network request by providing the plugin command with the +// record assumptions that are being made and allow the plugin to verify these +// assumptions during its validation and return an error if they do not hold. +// This would require the politeiawww client provide the record state along +// with all comment requests. + func (c *Comments) processNew(ctx context.Context, n v1.New, u user.User) (*v1.NewReply, error) { log.Tracef("processNew: %v %v %v", n.Token, u.Username) @@ -25,10 +38,29 @@ func (c *Comments) processNew(ctx context.Context, n v1.New, u user.User) (*v1.N } } + // Execute pre plugin hooks. Checking the mode is a temporary + // measure until user plugins have been properly implemented. + switch c.cfg.Mode { + case config.PoliteiaWWWMode: + err := c.piHookNewPre(u) + if err != nil { + return nil, err + } + } + // Only admins and the record author are allowed to comment on // unvetted records. - if n.State == v1.RecordStateUnvetted && !u.Admin { - // User is not an admin. Get the record author. + r, err := c.recordNoFiles(ctx, n.Token) + if err != nil { + if err == errRecordNotFound { + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeRecordNotFound, + } + } + return nil, err + } + if r.State == pdv2.RecordStateUnvetted && !u.Admin { + // User is not an admin. Check if the user is the author. authorID, err := c.politeiad.Author(ctx, n.Token) if err != nil { return nil, err @@ -41,16 +73,6 @@ func (c *Comments) processNew(ctx context.Context, n v1.New, u user.User) (*v1.N } } - // Execute pre plugin hooks. Checking the mode is a temporary - // measure until user plugins have been properly implemented. - switch c.cfg.Mode { - case config.PoliteiaWWWMode: - err := c.piHookNewPre(u) - if err != nil { - return nil, err - } - } - // Send plugin command cn := comments.New{ UserID: u.ID.String(), @@ -60,19 +82,19 @@ func (c *Comments) processNew(ctx context.Context, n v1.New, u user.User) (*v1.N PublicKey: n.PublicKey, Signature: n.Signature, } - cnr, err := c.politeiad.CommentNew(ctx, n.State, cn) + pdc, err := c.politeiad.CommentNew(ctx, cn) if err != nil { return nil, err } // Prepare reply - cm := convertComment(cnr.Comment) + cm := convertComment(*pdc) commentPopulateUserData(&cm, u) // Emit event c.events.Emit(EventTypeNew, EventNew{ - State: n.State, + State: r.State, Comment: cm, }) @@ -102,6 +124,23 @@ func (c *Comments) processVote(ctx context.Context, v v1.Vote, u user.User) (*v1 } } + // Votes are only allowed on vetted records + r, err := c.recordNoFiles(ctx, v.Token) + if err != nil { + if err == errRecordNotFound { + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeRecordNotFound, + } + } + return nil, err + } + if r.State != pdv2.RecordStateVetted { + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeRecordStateInvalid, + ErrorContext: "comment voting is only allowed on vetted records", + } + } + // Send plugin command cv := comments.Vote{ UserID: u.ID.String(), @@ -111,7 +150,7 @@ func (c *Comments) processVote(ctx context.Context, v v1.Vote, u user.User) (*v1 PublicKey: v.PublicKey, Signature: v.Signature, } - vr, err := c.politeiad.CommentVote(ctx, v1.RecordStateVetted, cv) + vr, err := c.politeiad.CommentVote(ctx, cv) if err != nil { return nil, err } @@ -143,7 +182,7 @@ func (c *Comments) processDel(ctx context.Context, d v1.Del, u user.User) (*v1.D PublicKey: d.PublicKey, Signature: d.Signature, } - cdr, err := c.politeiad.CommentDel(ctx, d.State, cd) + cdr, err := c.politeiad.CommentDel(ctx, cd) if err != nil { return nil, err } @@ -166,7 +205,7 @@ func (c *Comments) processCount(ctx context.Context, ct v1.Count) (*v1.CountRepl } } - counts, err := c.politeiad.CommentCount(ctx, ct.State, ct.Tokens) + counts, err := c.politeiad.CommentCount(ctx, ct.Tokens) if err != nil { return nil, err } @@ -182,7 +221,16 @@ func (c *Comments) processComments(ctx context.Context, cs v1.Comments, u *user. // Only admins and the record author are allowed to retrieve // unvetted comments. This is a public route so a user might // not exist. - if cs.State == v1.RecordStateUnvetted { + r, err := c.recordNoFiles(ctx, cs.Token) + if err != nil { + if err == errRecordNotFound { + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeRecordNotFound, + } + } + return nil, err + } + if r.State == pdv2.RecordStateUnvetted { var isAllowed bool switch { case u == nil: @@ -211,7 +259,7 @@ func (c *Comments) processComments(ctx context.Context, cs v1.Comments, u *user. } // Send plugin command - pcomments, err := c.politeiad.CommentsGetAll(ctx, cs.State, cs.Token) + pcomments, err := c.politeiad.CommentsGetAll(ctx, cs.Token) if err != nil { return nil, err } @@ -249,8 +297,7 @@ func (c *Comments) processVotes(ctx context.Context, v v1.Votes) (*v1.VotesReply cm := comments.Votes{ UserID: v.UserID, } - votes, err := c.politeiad.CommentVotes(ctx, - v1.RecordStateVetted, v.Token, cm) + votes, err := c.politeiad.CommentVotes(ctx, v.Token, cm) if err != nil { return nil, err } @@ -275,11 +322,22 @@ func (c *Comments) processVotes(ctx context.Context, v v1.Votes) (*v1.VotesReply func (c *Comments) processTimestamps(ctx context.Context, t v1.Timestamps, isAdmin bool) (*v1.TimestampsReply, error) { log.Tracef("processTimestamps: %v %v", t.Token, t.CommentIDs) + // Get record state + r, err := c.recordNoFiles(ctx, t.Token) + if err != nil { + if err == errRecordNotFound { + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeRecordNotFound, + } + } + return nil, err + } + // Get timestamps ct := comments.Timestamps{ CommentIDs: t.CommentIDs, } - ctr, err := c.politeiad.CommentTimestamps(ctx, t.State, t.Token, ct) + ctr, err := c.politeiad.CommentTimestamps(ctx, t.Token, ct) if err != nil { return nil, err } @@ -290,7 +348,7 @@ func (c *Comments) processTimestamps(ctx context.Context, t v1.Timestamps, isAdm ts := make([]v1.Timestamp, 0, len(timestamps)) for _, v := range timestamps { // Strip unvetted data blobs if the user is not an admin - if t.State == v1.RecordStateUnvetted && !isAdmin { + if r.State == pdv2.RecordStateUnvetted && !isAdmin { v.Data = "" } ts = append(ts, convertTimestamp(v)) @@ -303,6 +361,32 @@ func (c *Comments) processTimestamps(ctx context.Context, t v1.Timestamps, isAdm }, nil } +var ( + errRecordNotFound = errors.New("record not found") +) + +// recordNoFiles returns a politeiad record without any of its files. This +// allows the call to be light weight but still return metadata about the +// record such as state and status. +func (c *Comments) recordNoFiles(ctx context.Context, token string) (*pdv2.Record, error) { + req := []pdv2.RecordRequest{ + { + Token: token, + OmitAllFiles: true, + }, + } + records, err := c.politeiad.RecordGetBatch(ctx, req) + if err != nil { + return nil, err + } + r, ok := records[token] + if !ok { + return nil, errRecordNotFound + } + + return &r, nil +} + // commentPopulateUserData populates the comment with user data that is not // stored in politeiad. func commentPopulateUserData(c *v1.Comment, u user.User) { diff --git a/politeiawww/pi/events.go b/politeiawww/pi/events.go index e46c137d8..59f8ddd9c 100644 --- a/politeiawww/pi/events.go +++ b/politeiawww/pi/events.go @@ -354,7 +354,7 @@ func (p *Pi) ntfnCommentNewProposalAuthor(c cmv1.Comment, proposalAuthorID, prop return nil } -func (p *Pi) ntfnCommentReply(state string, c cmv1.Comment, proposalName string) error { +func (p *Pi) ntfnCommentReply(c cmv1.Comment, proposalName string) error { // Verify there is work to do. This notification only applies to // reply comments. if c.ParentID == 0 { @@ -366,7 +366,7 @@ func (p *Pi) ntfnCommentReply(state string, c cmv1.Comment, proposalName string) g := cmplugin.Get{ CommentIDs: []uint32{c.ParentID}, } - cs, err := p.politeiad.CommentsGet(context.Background(), state, c.Token, g) + cs, err := p.politeiad.CommentsGet(context.Background(), c.Token, g) if err != nil { return err } @@ -442,7 +442,7 @@ func (p *Pi) handleEventCommentNew(ch chan interface{}) { } // Notify the parent comment author - err = p.ntfnCommentReply(e.State, e.Comment, proposalName) + err = p.ntfnCommentReply(e.Comment, proposalName) if err != nil { err = fmt.Errorf("ntfnCommentReply: %v", err) goto failed From e45572325ffb453597656e2490840efe0531e56f Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 8 Mar 2021 16:06:20 -0600 Subject: [PATCH 372/449] multi: Ticketvote plugin and routes ok. --- .../tstorebe/plugins/ticketvote/hooks.go | 24 +- politeiad/client/ticketvote.go | 717 ++++++++---------- 2 files changed, 328 insertions(+), 413 deletions(-) diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go index 03b8c7135..f66030247 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go @@ -310,22 +310,17 @@ func (p *ticketVotePlugin) hookSetRecordStatusPost(treeID int64, payload string) if err != nil { return err } - var ( - oldStatus = srs.Current.RecordMetadata.Status - newStatus = srs.RecordMetadata.Status - ) - // When a record is moved to vetted the plugin hooks are executed - // on both the unvetted and vetted tstore instances. We only need - // to update cached data if this is the vetted instance. We can - // determine this by checking if the record exists. The unvetted - // instance will return false. - if newStatus == backend.StatusPublic && !p.tstore.RecordExists(treeID) { - // This is the unvetted instance. Nothing to do. + // Ticketvote caches only need to be updated for vetted records + if srs.RecordMetadata.State == backend.StateUnvetted { return nil } // Update the inventory cache + var ( + oldStatus = srs.Current.RecordMetadata.Status + newStatus = srs.RecordMetadata.Status + ) switch newStatus { case backend.StatusPublic: // Add to inventory @@ -333,11 +328,8 @@ func (p *ticketVotePlugin) hookSetRecordStatusPost(treeID int64, payload string) ticketvote.VoteStatusUnauthorized) case backend.StatusCensored, backend.StatusArchived: // These statuses do not allow for a vote. Mark as ineligible. - // We only need to do this if the record is vetted. - if oldStatus == backend.StatusPublic { - p.inventoryUpdate(srs.RecordMetadata.Token, - ticketvote.VoteStatusIneligible) - } + p.inventoryUpdate(srs.RecordMetadata.Token, + ticketvote.VoteStatusIneligible) } // Update the submissions cache if the linkto has been set. diff --git a/politeiad/client/ticketvote.go b/politeiad/client/ticketvote.go index 9b7e843d3..1d575d7ce 100644 --- a/politeiad/client/ticketvote.go +++ b/politeiad/client/ticketvote.go @@ -6,449 +6,372 @@ package client import ( "context" + "encoding/json" + "fmt" + pdv2 "github.com/decred/politeia/politeiad/api/v2" "github.com/decred/politeia/politeiad/plugins/ticketvote" ) // TicketVoteAuthorize sends the ticketvote plugin Authorize command to the -// politeiad v1 API. +// politeiad v2 API. func (c *Client) TicketVoteAuthorize(ctx context.Context, a ticketvote.Authorize) (*ticketvote.AuthorizeReply, error) { - /* - // Setup request - b, err := json.Marshal(a) - if err != nil { - return nil, err - } - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionWrite, - State: pdv1.RecordStateVetted, - Token: a.Token, - ID: ticketvote.PluginID, - Command: ticketvote.CmdAuthorize, - Payload: string(b), - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var ar ticketvote.AuthorizeReply - err = json.Unmarshal([]byte(pcr.Payload), &ar) - if err != nil { - return nil, err - } - - return &ar, nil - */ - return nil, nil + // Setup request + b, err := json.Marshal(a) + if err != nil { + return nil, err + } + cmd := pdv2.PluginCmd{ + Token: a.Token, + ID: ticketvote.PluginID, + Command: ticketvote.CmdAuthorize, + Payload: string(b), + } + + // Send request + reply, err := c.PluginWrite(ctx, cmd) + if err != nil { + return nil, err + } + + // Decode reply + var ar ticketvote.AuthorizeReply + err = json.Unmarshal([]byte(reply), &ar) + if err != nil { + return nil, err + } + + return &ar, nil } // TicketVoteStart sends the ticketvote plugin Start command to the politeiad -// v1 API. +// v2 API. func (c *Client) TicketVoteStart(ctx context.Context, token string, s ticketvote.Start) (*ticketvote.StartReply, error) { - /* - // Setup request - b, err := json.Marshal(s) - if err != nil { - return nil, err - } - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionWrite, - State: pdv1.RecordStateVetted, - Token: token, - ID: ticketvote.PluginID, - Command: ticketvote.CmdStart, - Payload: string(b), - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var sr ticketvote.StartReply - err = json.Unmarshal([]byte(pcr.Payload), &sr) - if err != nil { - return nil, err - } - - return &sr, nil - */ - return nil, nil + // Setup request + b, err := json.Marshal(s) + if err != nil { + return nil, err + } + cmd := pdv2.PluginCmd{ + Token: token, + ID: ticketvote.PluginID, + Command: ticketvote.CmdStart, + Payload: string(b), + } + + // Send request + reply, err := c.PluginWrite(ctx, cmd) + if err != nil { + return nil, err + } + + // Decode reply + var sr ticketvote.StartReply + err = json.Unmarshal([]byte(reply), &sr) + if err != nil { + return nil, err + } + + return &sr, nil } // TicketVoteCastBallot sends the ticketvote plugin CastBallot command to the -// politeiad v1 API. +// politeiad v2 API. func (c *Client) TicketVoteCastBallot(ctx context.Context, token string, cb ticketvote.CastBallot) (*ticketvote.CastBallotReply, error) { - /* - // Setup request - b, err := json.Marshal(cb) - if err != nil { - return nil, err - } - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionWrite, - State: pdv1.RecordStateVetted, - Token: token, - ID: ticketvote.PluginID, - Command: ticketvote.CmdCastBallot, - Payload: string(b), - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var cbr ticketvote.CastBallotReply - err = json.Unmarshal([]byte(pcr.Payload), &cbr) - if err != nil { - return nil, err - } - - return &cbr, nil - */ - return nil, nil + // Setup request + b, err := json.Marshal(cb) + if err != nil { + return nil, err + } + cmd := pdv2.PluginCmd{ + Token: token, + ID: ticketvote.PluginID, + Command: ticketvote.CmdCastBallot, + Payload: string(b), + } + + // Send request + reply, err := c.PluginWrite(ctx, cmd) + if err != nil { + return nil, err + } + + // Decode reply + var cbr ticketvote.CastBallotReply + err = json.Unmarshal([]byte(reply), &cbr) + if err != nil { + return nil, err + } + + return &cbr, nil } // TicketVoteDetails sends the ticketvote plugin Details command to the -// politeiad v1 API. +// politeiad v2 API. func (c *Client) TicketVoteDetails(ctx context.Context, token string) (*ticketvote.DetailsReply, error) { - /* - // Setup request - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionRead, - State: pdv1.RecordStateVetted, - Token: token, - ID: ticketvote.PluginID, - Command: ticketvote.CmdDetails, - Payload: "", - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var dr ticketvote.DetailsReply - err = json.Unmarshal([]byte(pcr.Payload), &dr) - if err != nil { - return nil, err - } - - return &dr, nil - */ - return nil, nil + // Setup request + cmds := []pdv2.PluginCmd{ + { + Token: token, + ID: ticketvote.PluginID, + Command: ticketvote.CmdDetails, + Payload: "", + }, + } + + // Send request + replies, err := c.PluginReads(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCmdError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var dr ticketvote.DetailsReply + err = json.Unmarshal([]byte(pcr.Payload), &dr) + if err != nil { + return nil, err + } + + return &dr, nil } // TicketVoteResults sends the ticketvote plugin Results command to the -// politeiad v1 API. +// politeiad v2 API. func (c *Client) TicketVoteResults(ctx context.Context, token string) (*ticketvote.ResultsReply, error) { - /* - // Setup request - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionRead, - State: pdv1.RecordStateVetted, - Token: token, - ID: ticketvote.PluginID, - Command: ticketvote.CmdResults, - Payload: "", - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var rr ticketvote.ResultsReply - err = json.Unmarshal([]byte(pcr.Payload), &rr) - if err != nil { - return nil, err - } - - return &rr, nil - */ - return nil, nil + // Setup request + cmds := []pdv2.PluginCmd{ + { + Token: token, + ID: ticketvote.PluginID, + Command: ticketvote.CmdResults, + Payload: "", + }, + } + + // Send request + replies, err := c.PluginReads(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCmdError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var rr ticketvote.ResultsReply + err = json.Unmarshal([]byte(pcr.Payload), &rr) + if err != nil { + return nil, err + } + + return &rr, nil } // TicketVoteSummary sends the ticketvote plugin Summary command to the -// politeiad v1 API. +// politeiad v2 API. func (c *Client) TicketVoteSummary(ctx context.Context, token string) (*ticketvote.SummaryReply, error) { - /* - // Setup request - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionRead, - State: pdv1.RecordStateVetted, - ID: ticketvote.PluginID, - Command: ticketvote.CmdSummary, - Token: token, - Payload: "", - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var sr ticketvote.SummaryReply - err = json.Unmarshal([]byte(pcr.Payload), &sr) - if err != nil { - return nil, err - } - - return &sr, nil - */ - return nil, nil + // Setup request + cmds := []pdv2.PluginCmd{ + { + ID: ticketvote.PluginID, + Command: ticketvote.CmdSummary, + Token: token, + Payload: "", + }, + } + + // Send request + replies, err := c.PluginReads(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCmdError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var sr ticketvote.SummaryReply + err = json.Unmarshal([]byte(pcr.Payload), &sr) + if err != nil { + return nil, err + } + + return &sr, nil } // TicketVoteSummaries sends a batch of ticketvote plugin Summary commands to -// the politeiad v1 API. Individual summary errors are not returned, the token +// the politeiad v2 API. Individual summary errors are not returned, the token // will simply be left out of the returned map. func (c *Client) TicketVoteSummaries(ctx context.Context, tokens []string) (map[string]ticketvote.SummaryReply, error) { - /* - // Setup request - cmds := make([]pdv1.PluginCommandV2, 0, len(tokens)) - for _, v := range tokens { - cmds = append(cmds, pdv1.PluginCommandV2{ - Action: pdv1.PluginActionRead, - State: pdv1.RecordStateVetted, - Token: v, - ID: ticketvote.PluginID, - Command: ticketvote.CmdSummary, - Payload: "", - }) + // Setup request + cmds := make([]pdv2.PluginCmd, 0, len(tokens)) + for _, v := range tokens { + cmds = append(cmds, pdv2.PluginCmd{ + Token: v, + ID: ticketvote.PluginID, + Command: ticketvote.CmdSummary, + Payload: "", + }) + } + + // Send request + replies, err := c.PluginReads(ctx, cmds) + if err != nil { + return nil, err + } + + // Prepare reply + summaries := make(map[string]ticketvote.SummaryReply, len(replies)) + for _, v := range replies { + err = extractPluginCmdError(v) + if err != nil { + // Individual summary errors are ignored. The token will not + // be included in the returned summaries map. + continue } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) + var sr ticketvote.SummaryReply + err = json.Unmarshal([]byte(v.Payload), &sr) if err != nil { return nil, err } + summaries[v.Token] = sr + } - // Prepare reply - summaries := make(map[string]ticketvote.SummaryReply, len(replies)) - for _, v := range replies { - err = extractPluginCommandError(v) - if err != nil { - // Individual summary errors are ignored. The token will not - // be included in the returned summaries map. - continue - } - var sr ticketvote.SummaryReply - err = json.Unmarshal([]byte(v.Payload), &sr) - if err != nil { - return nil, err - } - summaries[v.Token] = sr - } - - return summaries, nil - */ - return nil, nil + return summaries, nil } // TicketVoteSubmissions sends the ticketvote plugin Submissions command to the -// politeiad v1 API. +// politeiad v2 API. func (c *Client) TicketVoteSubmissions(ctx context.Context, token string) ([]string, error) { - /* - // Setup request - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionRead, - State: pdv1.RecordStateVetted, - Token: token, - ID: ticketvote.PluginID, - Command: ticketvote.CmdSubmissions, - Payload: "", - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var sr ticketvote.SubmissionsReply - err = json.Unmarshal([]byte(pcr.Payload), &sr) - if err != nil { - return nil, err - } - - return sr.Submissions, nil - */ - return nil, nil + // Setup request + cmds := []pdv2.PluginCmd{ + { + Token: token, + ID: ticketvote.PluginID, + Command: ticketvote.CmdSubmissions, + Payload: "", + }, + } + + // Send request + replies, err := c.PluginReads(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCmdError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var sr ticketvote.SubmissionsReply + err = json.Unmarshal([]byte(pcr.Payload), &sr) + if err != nil { + return nil, err + } + + return sr.Submissions, nil } // TicketVoteInventory sends the ticketvote plugin Inventory command to the -// politeiad v1 API. +// politeiad v2 API. func (c *Client) TicketVoteInventory(ctx context.Context, i ticketvote.Inventory) (*ticketvote.InventoryReply, error) { - /* - // Setup request - b, err := json.Marshal(i) - if err != nil { - return nil, err - } - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionRead, - State: pdv1.RecordStateVetted, - ID: ticketvote.PluginID, - Command: ticketvote.CmdInventory, - Payload: string(b), - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var ir ticketvote.InventoryReply - err = json.Unmarshal([]byte(pcr.Payload), &ir) - if err != nil { - return nil, err - } - - return &ir, nil - */ - return nil, nil + // Setup request + b, err := json.Marshal(i) + if err != nil { + return nil, err + } + cmds := []pdv2.PluginCmd{ + { + ID: ticketvote.PluginID, + Command: ticketvote.CmdInventory, + Payload: string(b), + }, + } + + // Send request + replies, err := c.PluginReads(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCmdError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var ir ticketvote.InventoryReply + err = json.Unmarshal([]byte(pcr.Payload), &ir) + if err != nil { + return nil, err + } + + return &ir, nil } // TicketVoteTimestamps sends the ticketvote plugin Timestamps command to the -// politeiad v1 API. +// politeiad v2 API. func (c *Client) TicketVoteTimestamps(ctx context.Context, t ticketvote.Timestamps) (*ticketvote.TimestampsReply, error) { - /* - // Setup request - b, err := json.Marshal(t) - if err != nil { - return nil, err - } - cmds := []pdv1.PluginCommandV2{ - { - Action: pdv1.PluginActionRead, - State: pdv1.RecordStateVetted, - ID: ticketvote.PluginID, - Command: ticketvote.CmdTimestamps, - Token: t.Token, - Payload: string(b), - }, - } - - // Send request - replies, err := c.PluginCommandBatch(ctx, cmds) - if err != nil { - return nil, err - } - if len(replies) == 0 { - return nil, fmt.Errorf("no replies found") - } - pcr := replies[0] - err = extractPluginCommandError(pcr) - if err != nil { - return nil, err - } - - // Decode reply - var sr ticketvote.TimestampsReply - err = json.Unmarshal([]byte(pcr.Payload), &sr) - if err != nil { - return nil, err - } - - return &sr, nil - */ - return nil, nil + // Setup request + b, err := json.Marshal(t) + if err != nil { + return nil, err + } + cmds := []pdv2.PluginCmd{ + { + ID: ticketvote.PluginID, + Command: ticketvote.CmdTimestamps, + Token: t.Token, + Payload: string(b), + }, + } + + // Send request + replies, err := c.PluginReads(ctx, cmds) + if err != nil { + return nil, err + } + if len(replies) == 0 { + return nil, fmt.Errorf("no replies found") + } + pcr := replies[0] + err = extractPluginCmdError(pcr) + if err != nil { + return nil, err + } + + // Decode reply + var sr ticketvote.TimestampsReply + err = json.Unmarshal([]byte(pcr.Payload), &sr) + if err != nil { + return nil, err + } + + return &sr, nil } From d4ee7ec8f57e05ec4559700fce940351b2e8a8da Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 8 Mar 2021 16:14:36 -0600 Subject: [PATCH 373/449] politeiad/pi: Pi plugin ok. --- .../backendv2/tstorebe/plugins/pi/hooks.go | 32 +++++++------------ 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/politeiad/backendv2/tstorebe/plugins/pi/hooks.go b/politeiad/backendv2/tstorebe/plugins/pi/hooks.go index 7d8eb36a9..16123981d 100644 --- a/politeiad/backendv2/tstorebe/plugins/pi/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/pi/hooks.go @@ -11,6 +11,7 @@ import ( backend "github.com/decred/politeia/politeiad/backendv2" "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins" + "github.com/decred/politeia/politeiad/plugins/comments" "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/ticketvote" "github.com/decred/politeia/util" @@ -196,8 +197,7 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { } // Verify vote status allows proposal edits - /* TODO implement hook - if er.RecordMetadata.Status == backend.MDStatusVetted { + if er.RecordMetadata.State == backend.StateVetted { t, err := tokenDecode(er.RecordMetadata.Token) if err != nil { return err @@ -216,7 +216,6 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { } } } - */ return nil } @@ -237,12 +236,7 @@ func (p *piPlugin) voteSummary(token []byte) (*ticketvote.SummaryReply, error) { // commentWritesVerify verifies that a record's vote status allows writes from // the comments plugin. -func (p *piPlugin) commentWritesVerify(s backend.StateT, token []byte) error { - // Verify that the vote status allows comment writes. This only - // applies to vetted records. - if s == backend.StateUnvetted { - return nil - } +func (p *piPlugin) commentWritesVerify(token []byte) error { vs, err := p.voteSummary(token) if err != nil { return err @@ -261,16 +255,16 @@ func (p *piPlugin) commentWritesVerify(s backend.StateT, token []byte) error { } } -func (p *piPlugin) hookCommentNew(s backend.StateT, token []byte) error { - return p.commentWritesVerify(s, token) +func (p *piPlugin) hookCommentNew(token []byte) error { + return p.commentWritesVerify(token) } -func (p *piPlugin) hookCommentDel(s backend.StateT, token []byte) error { - return p.commentWritesVerify(s, token) +func (p *piPlugin) hookCommentDel(token []byte) error { + return p.commentWritesVerify(token) } -func (p *piPlugin) hookCommentVote(s backend.StateT, token []byte) error { - return p.commentWritesVerify(s, token) +func (p *piPlugin) hookCommentVote(token []byte) error { + return p.commentWritesVerify(token) } func (p *piPlugin) hookPluginPre(treeID int64, token []byte, payload string) error { @@ -281,20 +275,18 @@ func (p *piPlugin) hookPluginPre(treeID int64, token []byte, payload string) err return err } - /* TODO // Call plugin hook switch hpp.PluginID { case comments.PluginID: switch hpp.Cmd { case comments.CmdNew: - return p.hookCommentNew(hpp.State, token) + return p.hookCommentNew(token) case comments.CmdDel: - return p.hookCommentDel(hpp.State, token) + return p.hookCommentDel(token) case comments.CmdVote: - return p.hookCommentVote(hpp.State, token) + return p.hookCommentVote(token) } } - */ return nil } From 8737729618f67ceaee607d822993d0ade75ced27 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 8 Mar 2021 16:25:12 -0600 Subject: [PATCH 374/449] politeiad: Update tstore setup docs. --- politeiad/README.md | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/politeiad/README.md b/politeiad/README.md index 5ce2cad38..af3500a2d 100644 --- a/politeiad/README.md +++ b/politeiad/README.md @@ -40,7 +40,7 @@ politeiad ## Build from source -1. Install Mariadb or MySQL. Make sure to setup a password for the root user. +1. Install MariaDB or MySQL. Make sure to setup a password for the root user. 2. Update the MySQL max connections settings. @@ -122,20 +122,20 @@ politeiad ``` $ cd $GOPATH/src/github.com/google/trillian/scripts - # Unvetted setup + # Testnet setup $ env \ MYSQL_USER=trillian \ MYSQL_PASSWORD=trillianpass \ - MYSQL_DATABASE=testnet3_unvetted_trillian \ + MYSQL_DATABASE=testnet3_trillian \ MYSQL_URI="${MYSQL_USER}:${MYSQL_PASSWORD}@tcp(127.0.0.1:3306)/${MYSQL_DATABASE}" \ MYSQL_ROOT_PASSWORD=rootpass \ ./resetdb.sh - # Vetted setup + # Mainnet setup $ env \ MYSQL_USER=trillian \ MYSQL_PASSWORD=trillianpass \ - MYSQL_DATABASE=testnet3_vetted_trillian \ + MYSQL_DATABASE=mainnet_trillian \ MYSQL_URI="${MYSQL_USER}:${MYSQL_PASSWORD}@tcp(127.0.0.1:3306)/${MYSQL_DATABASE}" \ MYSQL_ROOT_PASSWORD=rootpass \ ./resetdb.sh @@ -145,18 +145,19 @@ politeiad 5. Start up the trillian instances. Running trillian requires running a trillian log server and a trillian log - signer. These are seperate processes. Politeiad runs a trillian instance - for unvetted records and a trillian instance for vetted records. You will be - starting up 4 seperate processes in this step. + signer. These are seperate processes that will be started in this step. You will need to replace the `trillianpass` with the trillian user's - password that you setup in previous steps. + password that you setup in previous steps. The commands below for testnet + and mainnet run the trillian instances on the same ports so you can only + run one set of commands, testnet or mainnet. Run the testnet commands if + you're setting up a development environment. - Startup unvetted log server + Startup testnet log server ``` $ export MYSQL_USER=trillian && \ export MYSQL_PASSWORD=trillianpass && \ - export MYSQL_DATABASE=testnet3_unvetted_trillian && \ + export MYSQL_DATABASE=testnet3_trillian && \ export MYSQL_URI="${MYSQL_USER}:${MYSQL_PASSWORD}@tcp(127.0.0.1:3306)/${MYSQL_DATABASE}" $ trillian_log_server \ @@ -167,11 +168,11 @@ politeiad --logtostderr ... ``` - Startup unvetted log signer + Startup testnet log signer ``` $ export MYSQL_USER=trillian && \ export MYSQL_PASSWORD=trillianpass && \ - export MYSQL_DATABASE=testnet3_unvetted_trillian && \ + export MYSQL_DATABASE=testnet3_trillian && \ export MYSQL_URI="${MYSQL_USER}:${MYSQL_PASSWORD}@tcp(127.0.0.1:3306)/${MYSQL_DATABASE}" $ trillian_log_signer --logtostderr --force_master \ @@ -183,26 +184,26 @@ politeiad --http_endpoint=localhost:8093 ``` - Startup vetted log server + Startup mainnet log server ``` $ export MYSQL_USER=trillian && \ export MYSQL_PASSWORD=trillianpass && \ - export MYSQL_DATABASE=testnet3_vetted_trillian && \ + export MYSQL_DATABASE=mainnet_trillian && \ export MYSQL_URI="${MYSQL_USER}:${MYSQL_PASSWORD}@tcp(127.0.0.1:3306)/${MYSQL_DATABASE}" $ trillian_log_server \ --mysql_uri=${MYSQL_URI} \ --mysql_max_conns=2000 \ - --rpc_endpoint localhost:8094 \ - --http_endpoint localhost:8095 \ + --rpc_endpoint localhost:8090 \ + --http_endpoint localhost:8091 \ --logtostderr ... ``` - Startup vetted log signer + Startup mainnet log signer ``` $ export MYSQL_USER=trillian && \ export MYSQL_PASSWORD=trillianpass && \ - export MYSQL_DATABASE=testnet3_vetted_trillian && \ + export MYSQL_DATABASE=mainnet_trillian && \ export MYSQL_URI="${MYSQL_USER}:${MYSQL_PASSWORD}@tcp(127.0.0.1:3306)/${MYSQL_DATABASE}" $ trillian_log_signer --logtostderr --force_master \ @@ -210,8 +211,8 @@ politeiad --sequencer_guard_window=0 \ --sequencer_interval=200ms \ --mysql_uri=${MYSQL_URI} \ - --rpc_endpoint localhost:8096 \ - --http_endpoint=localhost:8097 + --rpc_endpoint localhost:8092 \ + --http_endpoint=localhost:8093 ``` From 9e0391d7f788145481f7074962f2c02c4d715730 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 9 Mar 2021 10:09:52 -0600 Subject: [PATCH 375/449] tstore: Fix metadata idx and timestamp. --- .../backendv2/tstorebe/tstore/recordindex.go | 20 +++-- politeiad/backendv2/tstorebe/tstore/tstore.go | 82 +++++++++++-------- 2 files changed, 61 insertions(+), 41 deletions(-) diff --git a/politeiad/backendv2/tstorebe/tstore/recordindex.go b/politeiad/backendv2/tstorebe/tstore/recordindex.go index fe8f67f9a..e5ee11e17 100644 --- a/politeiad/backendv2/tstorebe/tstore/recordindex.go +++ b/politeiad/backendv2/tstorebe/tstore/recordindex.go @@ -21,18 +21,18 @@ import ( // // A record is updated in three steps: // -// 1. Record content blobs are saved to the kv store. +// 1. Record content is saved to the kv store. // -// 2. A trillian leaf is created for each record content blob. The kv store -// key for the blob is stuffed into the LogLeaf.ExtraData field. All leaves -// are appended onto the trillian tree. +// 2. A trillian leaf is created for each piece of record content. The kv store +// key for each piece of content is stuffed into the LogLeaf.ExtraData +// field. The leaves are appended onto the trillian tree. // // 3. If there are failures in steps 1 or 2 for any of the blobs then the // update will exit without completing. No unwinding is performed. Blobs // will be left in the kv store as orphaned blobs. The trillian tree is -// append only so once a leaf is appended, it's there permanently. If steps -// 1 and 2 are successful then a recordIndex will be created, saved to the -// kv store, and appended onto the trillian tree. +// append-only so once a leaf is appended, it's there permanently. If steps +// 1 and 2 are successful then a recordIndex is created, saved to the kv +// store, and appended onto the trillian tree. // // Appending a recordIndex onto the trillian tree is the last operation that // occurs during a record update. If a recordIndex exists in the tree then the @@ -55,8 +55,10 @@ type recordIndex struct { // can be used to lookup the log leaf. The log leaf ExtraData field // contains the key for the record content in the key-value store. RecordMetadata []byte `json:"recordmetadata"` - Metadata map[string][]byte `json:"metadata"` // [pluginID+ID]merkle - Files map[string][]byte `json:"files"` // [filename]merkle + Files map[string][]byte `json:"files"` // [filename]merkle + + // [pluginID][streamID]merkle + Metadata map[string]map[uint32][]byte `json:"metadata"` // Frozen is used to indicate that the tree for this record has // been frozen. This happens as a result of certain record status diff --git a/politeiad/backendv2/tstorebe/tstore/tstore.go b/politeiad/backendv2/tstorebe/tstore/tstore.go index b099836e9..1c21dc960 100644 --- a/politeiad/backendv2/tstorebe/tstore/tstore.go +++ b/politeiad/backendv2/tstorebe/tstore/tstore.go @@ -15,7 +15,6 @@ import ( "net/url" "os" "path/filepath" - "strconv" "sync" "github.com/decred/dcrd/chaincfg/v3" @@ -208,9 +207,9 @@ func (t *Tstore) treeIsFrozen(leaves []*trillian.LogLeaf) bool { } type recordHashes struct { - recordMetadata string // Record metadata hash - metadata map[string]string // [hash]metadataID - files map[string]string // [hash]filename + recordMetadata string // Record metadata hash + metadata map[string]backend.MetadataStream // [hash]MetadataStream + files map[string]backend.File // [hash]File } type recordBlobsPrepareReply struct { @@ -286,8 +285,11 @@ func (t *Tstore) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD back // Compute record content hashes rhashes := recordHashes{ - metadata: make(map[string]string, len(metadata)), // [hash]metadataID - files: make(map[string]string, len(files)), // [hash]filename + // [hash]MetadataStream + metadata: make(map[string]backend.MetadataStream, len(metadata)), + + // [hash]File + files: make(map[string]backend.File, len(files)), } b, err := json.Marshal(recordMD) if err != nil { @@ -300,8 +302,7 @@ func (t *Tstore) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD back return nil, err } h := hex.EncodeToString(util.Digest(b)) - streamID := strconv.FormatUint(uint64(v.StreamID), 10) - rhashes.metadata[h] = v.PluginID + streamID + rhashes.metadata[h] = v } for _, v := range files { b, err := json.Marshal(v) @@ -309,7 +310,7 @@ func (t *Tstore) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD back return nil, err } h := hex.EncodeToString(util.Digest(b)) - rhashes.files[h] = v.Name + rhashes.files[h] = v } // Compare leaf data to record content hashes to find duplicates @@ -321,7 +322,7 @@ func (t *Tstore) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD back // Any duplicates that are found are added to the record index // since we already have the leaf data for them. index = recordIndex{ - Metadata: make(map[string][]byte, len(metadata)), + Metadata: make(map[string]map[uint32][]byte, len(metadata)), Files: make(map[string][]byte, len(files)), } ) @@ -336,18 +337,23 @@ func (t *Tstore) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD back } // Check metadata streams - id, ok := rhashes.metadata[h] + ms, ok := rhashes.metadata[h] if ok { dups[h] = struct{}{} - index.Metadata[id] = v.MerkleLeafHash + streams, ok := index.Metadata[ms.PluginID] + if !ok { + streams = make(map[uint32][]byte, 64) + } + streams[ms.StreamID] = v.MerkleLeafHash + index.Metadata[ms.PluginID] = streams continue } // Check files - fn, ok := rhashes.files[h] + f, ok := rhashes.files[h] if ok { dups[h] = struct{}{} - index.Files[fn] = v.MerkleLeafHash + index.Files[f.Name] = v.MerkleLeafHash continue } } @@ -502,16 +508,21 @@ func (t *Tstore) recordBlobsSave(treeID int64, rbpr recordBlobsPrepareReply) (*r } // Check metadata streams - id, ok := rhashes.metadata[h] + ms, ok := rhashes.metadata[h] if ok { - index.Metadata[id] = v.QueuedLeaf.Leaf.MerkleLeafHash + streams, ok := index.Metadata[ms.PluginID] + if !ok { + streams = make(map[uint32][]byte, 64) + } + streams[ms.StreamID] = v.QueuedLeaf.Leaf.MerkleLeafHash + index.Metadata[ms.PluginID] = streams continue } // Check files - fn, ok := rhashes.files[h] + f, ok := rhashes.files[h] if ok { - index.Files[fn] = v.QueuedLeaf.Leaf.MerkleLeafHash + index.Files[f.Name] = v.QueuedLeaf.Leaf.MerkleLeafHash continue } @@ -548,7 +559,7 @@ func (t *Tstore) RecordSave(treeID int64, rm backend.RecordMetadata, metadata [] if errors.Is(err, backend.ErrRecordNotFound) { // No record versions exist yet. This is ok. currIdx = &recordIndex{ - Metadata: make(map[string][]byte), + Metadata: make(map[string]map[uint32][]byte), Files: make(map[string][]byte), } } else if err != nil { @@ -889,8 +900,10 @@ func (t *Tstore) record(treeID int64, version uint32, filenames []string, omitAl // Compile merkle root hashes of record content merkles := make(map[string]struct{}, 64) merkles[hex.EncodeToString(idx.RecordMetadata)] = struct{}{} - for _, v := range idx.Metadata { - merkles[hex.EncodeToString(v)] = struct{}{} + for _, streams := range idx.Metadata { + for _, v := range streams { + merkles[hex.EncodeToString(v)] = struct{}{} + } } switch { case omitAllFiles: @@ -1227,13 +1240,21 @@ func (t *Tstore) RecordTimestamps(treeID int64, version uint32, token []byte) (* } // Get metadata timestamps - metadata := make(map[string]backend.Timestamp, len(idx.Metadata)) - for k, v := range idx.Metadata { - ts, err := t.timestamp(treeID, v, leaves) - if err != nil { - return nil, fmt.Errorf("metadata %v timestamp: %v", k, err) + metadata := make(map[string]map[uint32]backend.Timestamp, len(idx.Metadata)) + for pluginID, streams := range idx.Metadata { + for streamID, merkle := range streams { + ts, err := t.timestamp(treeID, merkle, leaves) + if err != nil { + return nil, fmt.Errorf("metadata %v %v timestamp: %v", + pluginID, streamID, err) + } + sts, ok := metadata[pluginID] + if !ok { + sts = make(map[uint32]backend.Timestamp, 64) + } + sts[streamID] = *ts + metadata[pluginID] = sts } - metadata[k] = *ts } // Get file timestamps @@ -1246,13 +1267,10 @@ func (t *Tstore) RecordTimestamps(treeID int64, version uint32, token []byte) (* files[k] = *ts } - // TODO fix metadata timestamps - _ = metadata - return &backend.RecordTimestamps{ RecordMetadata: *rm, - // Metadata: metadata, - Files: files, + Metadata: metadata, + Files: files, }, nil } From 4ac42b96b6c1e5e4e9f9d44adeb53ffd346e337f Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 9 Mar 2021 12:56:17 -0600 Subject: [PATCH 376/449] politeiad/store: Add PutKV method. --- .../tstorebe/store/localdb/localdb.go | 49 +++++++++++--- .../backendv2/tstorebe/store/mysql/mysql.go | 64 ++++++++++++++----- politeiad/backendv2/tstorebe/store/store.go | 5 ++ 3 files changed, 93 insertions(+), 25 deletions(-) diff --git a/politeiad/backendv2/tstorebe/store/localdb/localdb.go b/politeiad/backendv2/tstorebe/store/localdb/localdb.go index 552da8375..b4fcc7807 100644 --- a/politeiad/backendv2/tstorebe/store/localdb/localdb.go +++ b/politeiad/backendv2/tstorebe/store/localdb/localdb.go @@ -26,6 +26,24 @@ type localdb struct { db *leveldb.DB } +func (l *localdb) put(blobs map[string][]byte) error { + // Setup batch + batch := new(leveldb.Batch) + for k, v := range blobs { + batch.Put([]byte(k), v) + } + + // Write batch + err := l.db.Write(batch, nil) + if err != nil { + return fmt.Errorf("write batch: %v", err) + } + + log.Debugf("Saved blobs (%v) to store", len(blobs)) + + return nil +} + // Put saves the provided blobs to the store. The keys for the blobs are // returned using the same odering that the blobs were provided in. This // operation is performed atomically. @@ -34,28 +52,39 @@ type localdb struct { func (l *localdb) Put(blobs [][]byte) ([]string, error) { log.Tracef("Put: %v", len(blobs)) - // Setup batch + // Setup the keys. The keys are returned in the same order that + // the blobs are in. var ( - batch = new(leveldb.Batch) - keys = make([]string, 0, len(blobs)) + keys = make([]string, 0, len(blobs)) + blobkv = make(map[string][]byte, len(blobs)) ) for _, v := range blobs { - key := uuid.New().String() - batch.Put([]byte(key), v) - keys = append(keys, key) + k := uuid.New().String() + keys = append(keys, k) + blobkv[k] = v } - // Write batch - err := l.db.Write(batch, nil) + // Save blobs + err := l.put(blobkv) if err != nil { return nil, err } - log.Debugf("Saved blobs (%v) to store", len(blobs)) - + // Return the keys return keys, nil } +// PutKV saves the provided blobs to the store. This method allows the caller +// to specify the key instead of having the store create one. This operation is +// performed atomically. +// +// This function satisfies the store BlobKV interface. +func (l *localdb) PutKV(blobs map[string][]byte) error { + log.Tracef("PutKV: %v blobs", len(blobs)) + + return l.put(blobs) +} + // Del deletes the provided blobs from the store. This operation is performed // atomically. // diff --git a/politeiad/backendv2/tstorebe/store/mysql/mysql.go b/politeiad/backendv2/tstorebe/store/mysql/mysql.go index f440665de..09caaef5e 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/mysql.go +++ b/politeiad/backendv2/tstorebe/store/mysql/mysql.go @@ -43,14 +43,7 @@ func ctxWithTimeout() (context.Context, func()) { return context.WithTimeout(context.Background(), connTimeout) } -// Put saves the provided blobs to the store The keys for the blobs are -// returned using the same odering that the blobs were provided in. This -// operation is performed atomically. -// -// This function satisfies the store BlobKV interface. -func (s *mysql) Put(blobs [][]byte) ([]string, error) { - log.Tracef("Put: %v blobs", len(blobs)) - +func (s *mysql) put(blobs map[string][]byte) error { ctx, cancel := ctxWithTimeout() defer cancel() @@ -60,13 +53,11 @@ func (s *mysql) Put(blobs [][]byte) ([]string, error) { } tx, err := s.db.BeginTx(ctx, opts) if err != nil { - return nil, err + return fmt.Errorf("begin tx: %v", err) } // Save blobs - keys := make([]string, 0, len(blobs)) - for _, v := range blobs { - k := uuid.New().String() + for k, v := range blobs { _, err = tx.ExecContext(ctx, "INSERT INTO kv (k, v) VALUES (?, ?);", k, v) if err != nil { // Attempt to roll back the transaction @@ -76,19 +67,60 @@ func (s *mysql) Put(blobs [][]byte) ([]string, error) { panic(e) } } - - keys = append(keys, k) } // Commit transaction err = tx.Commit() if err != nil { - return nil, fmt.Errorf("commit: %v", err) + return fmt.Errorf("commit tx: %v", err) } + log.Debugf("Saved blobs (%v) to store", len(blobs)) + + return nil +} + +// Put saves the provided blobs to the store The keys for the blobs are +// returned using the same odering that the blobs were provided in. This +// operation is performed atomically. +// +// This function satisfies the store BlobKV interface. +func (s *mysql) Put(blobs [][]byte) ([]string, error) { + log.Tracef("Put: %v blobs", len(blobs)) + + // Setup the keys. The keys are returned in the same order that + // the blobs are in. + var ( + keys = make([]string, 0, len(blobs)) + blobkv = make(map[string][]byte, len(blobs)) + ) + for _, v := range blobs { + k := uuid.New().String() + keys = append(keys, k) + blobkv[k] = v + } + + // Save blobs + err := s.put(blobkv) + if err != nil { + return nil, err + } + + // Return the keys return keys, nil } +// PutKV saves the provided blobs to the store. This method allows the caller +// to specify the key instead of having the store create one. This operation is +// performed atomically. +// +// This function satisfies the store BlobKV interface. +func (s *mysql) PutKV(blobs map[string][]byte) error { + log.Tracef("PutKV: %v blobs", len(blobs)) + + return s.put(blobs) +} + // Del deletes the provided blobs from the store This operation is performed // atomically. // @@ -127,6 +159,8 @@ func (s *mysql) Del(keys []string) error { return fmt.Errorf("commit: %v", err) } + log.Debugf("Deleted blobs (%v) from store", len(keys)) + return nil } diff --git a/politeiad/backendv2/tstorebe/store/store.go b/politeiad/backendv2/tstorebe/store/store.go index 0237c7e8a..6a566762d 100644 --- a/politeiad/backendv2/tstorebe/store/store.go +++ b/politeiad/backendv2/tstorebe/store/store.go @@ -82,6 +82,11 @@ type BlobKV interface { // provided in. This operation is performed atomically. Put(blobs [][]byte) ([]string, error) + // PutKV saves the provided blobs to the store. This method allows + // the caller to specify the key instead of having the store create + // one. This operation is performed atomically. + PutKV(blobs map[string][]byte) error + // Del deletes the provided blobs from the store. This operation // is performed atomically. Del(keys []string) error From 20b723f1970db9ee4359d555b80f061f0a2de25a Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 9 Mar 2021 13:02:47 -0600 Subject: [PATCH 377/449] tstore: Bug fixes. --- politeiad/backendv2/tstorebe/tstore/tstore.go | 45 ------------------- 1 file changed, 45 deletions(-) diff --git a/politeiad/backendv2/tstorebe/tstore/tstore.go b/politeiad/backendv2/tstorebe/tstore/tstore.go index 1c21dc960..c29067356 100644 --- a/politeiad/backendv2/tstorebe/tstore/tstore.go +++ b/politeiad/backendv2/tstorebe/tstore/tstore.go @@ -629,24 +629,6 @@ func (t *Tstore) RecordSave(treeID int64, rm backend.RecordMetadata, metadata [] idx.Version = rm.Version idx.Iteration = rm.Iteration - // Sanity checks - switch { - case idx.Version != currIdx.Version+1: - return fmt.Errorf("invalid index version: got %v, want %v", - idx.Version, currIdx.Version+1) - case idx.Iteration != currIdx.Iteration+1: - return fmt.Errorf("invalid index iteration: got %v, want %v", - idx.Iteration, currIdx.Iteration+1) - case idx.RecordMetadata == nil: - return fmt.Errorf("invalid index record metadata") - case len(idx.Metadata) != len(metadata): - return fmt.Errorf("invalid index metadata: got %v, want %v", - len(idx.Metadata), len(metadata)) - case len(idx.Files) != len(files): - return fmt.Errorf("invalid index files: got %v, want %v", - len(idx.Files), len(files)) - } - // Save record index err = t.recordIndexSave(treeID, *idx) if err != nil { @@ -714,24 +696,6 @@ func (t *Tstore) metadataSave(treeID int64, rm backend.RecordMetadata, metadata idx.Version = rm.Version idx.Iteration = rm.Iteration - // Sanity check - switch { - case idx.Version != oldIdx.Version: - return nil, fmt.Errorf("invalid index version: got %v, want %v", - idx.Version, oldIdx.Version) - case idx.Iteration != oldIdx.Iteration+1: - return nil, fmt.Errorf("invalid index iteration: got %v, want %v", - idx.Iteration, oldIdx.Iteration+1) - case idx.RecordMetadata == nil: - return nil, fmt.Errorf("invalid index record metadata") - case len(idx.Metadata) != len(metadata): - return nil, fmt.Errorf("invalid index metadata: got %v, want %v", - len(idx.Metadata), len(metadata)) - case len(idx.Files) != len(oldIdx.Files): - return nil, fmt.Errorf("invalid index files: got %v, want %v", - len(idx.Files), len(oldIdx.Files)) - } - return idx, nil } @@ -1028,15 +992,6 @@ func (t *Tstore) record(treeID int64, version uint32, filenames []string, omitAl } } - // Sanity checks - switch { - case recordMD == nil: - return nil, fmt.Errorf("record metadata not found") - case len(metadata) != len(idx.Metadata): - return nil, fmt.Errorf("invalid number of metadata; got %v, want %v", - len(metadata), len(idx.Metadata)) - } - return &backend.Record{ RecordMetadata: *recordMD, Metadata: metadata, From 6634ece7ab03de934c98b24f719b6e889e3c3af5 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 9 Mar 2021 16:40:13 -0600 Subject: [PATCH 378/449] Naming. --- politeiad/backendv2/tstorebe/tstore/anchor.go | 12 ++--- politeiad/backendv2/tstorebe/tstore/client.go | 14 +++--- .../backendv2/tstorebe/tstore/recordindex.go | 2 +- .../backendv2/tstorebe/tstore/testing.go | 2 +- .../{trillianclient.go => tlogclient.go} | 50 +++++++++---------- politeiad/backendv2/tstorebe/tstore/tstore.go | 30 +++++------ 6 files changed, 55 insertions(+), 55 deletions(-) rename politeiad/backendv2/tstorebe/tstore/{trillianclient.go => tlogclient.go} (93%) diff --git a/politeiad/backendv2/tstorebe/tstore/anchor.go b/politeiad/backendv2/tstorebe/tstore/anchor.go index d99ed5cd0..c2f3b5561 100644 --- a/politeiad/backendv2/tstorebe/tstore/anchor.go +++ b/politeiad/backendv2/tstorebe/tstore/anchor.go @@ -153,7 +153,7 @@ func (t *Tstore) anchorForLeaf(treeID int64, merkleLeafHash []byte, leaves []*tr // errAnchorNotFound is returned if no anchor is found for the provided tree. func (t *Tstore) anchorLatest(treeID int64) (*anchor, error) { // Get tree leaves - leavesAll, err := t.trillian.leavesAll(treeID) + leavesAll, err := t.tlog.leavesAll(treeID) if err != nil { return nil, fmt.Errorf("leavesAll: %v", err) } @@ -242,7 +242,7 @@ func (t *Tstore) anchorSave(a anchor) error { leaves := []*trillian.LogLeaf{ newLogLeaf(d, extraData), } - queued, _, err := t.trillian.leavesAppend(a.TreeID, leaves) + queued, _, err := t.tlog.leavesAppend(a.TreeID, leaves) if err != nil { return fmt.Errorf("leavesAppend: %v", err) } @@ -455,7 +455,7 @@ func (t *Tstore) anchorTrees() error { return nil } - trees, err := t.trillian.treesAll() + trees, err := t.tlog.treesAll() if err != nil { return fmt.Errorf("treesAll: %v", err) } @@ -483,7 +483,7 @@ func (t *Tstore) anchorTrees() error { case errors.Is(err, errAnchorNotFound): // Tree has not been anchored yet. Verify that the tree has // leaves. A tree with no leaves does not need to be anchored. - leavesAll, err := t.trillian.leavesAll(v.TreeId) + leavesAll, err := t.tlog.leavesAll(v.TreeId) if err != nil { return fmt.Errorf("leavesAll: %v", err) } @@ -499,7 +499,7 @@ func (t *Tstore) anchorTrees() error { default: // Anchor record found. If the anchor height differs from the // current height then the tree needs to be anchored. - _, lr, err := t.trillian.signedLogRoot(v) + _, lr, err := t.tlog.signedLogRoot(v) if err != nil { return fmt.Errorf("signedLogRoot %v: %v", v.TreeId, err) } @@ -514,7 +514,7 @@ func (t *Tstore) anchorTrees() error { // Tree has not been anchored at current height. Add it to the // list of anchors. - _, lr, err := t.trillian.signedLogRoot(v) + _, lr, err := t.tlog.signedLogRoot(v) if err != nil { return fmt.Errorf("signedLogRoot %v: %v", v.TreeId, err) } diff --git a/politeiad/backendv2/tstorebe/tstore/client.go b/politeiad/backendv2/tstorebe/tstore/client.go index 1d4a459da..09d02fb37 100644 --- a/politeiad/backendv2/tstorebe/tstore/client.go +++ b/politeiad/backendv2/tstorebe/tstore/client.go @@ -52,7 +52,7 @@ func (t *Tstore) BlobSave(treeID int64, be store.BlobEntry) error { } // Verify tree is not frozen - leaves, err := t.trillian.leavesAll(treeID) + leaves, err := t.tlog.leavesAll(treeID) if err != nil { return fmt.Errorf("leavesAll: %v", err) } @@ -80,7 +80,7 @@ func (t *Tstore) BlobSave(treeID int64, be store.BlobEntry) error { } // Append log leaf to trillian tree - queued, _, err := t.trillian.leavesAppend(treeID, leaves) + queued, _, err := t.tlog.leavesAppend(treeID, leaves) if err != nil { return fmt.Errorf("leavesAppend: %v", err) } @@ -109,7 +109,7 @@ func (t *Tstore) BlobsDel(treeID int64, digests [][]byte) error { } // Get all tree leaves - leaves, err := t.trillian.leavesAll(treeID) + leaves, err := t.tlog.leavesAll(treeID) if err != nil { return fmt.Errorf("leavesAll: %v", err) } @@ -163,7 +163,7 @@ func (t *Tstore) Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEnt } // Get leaves - leavesAll, err := t.trillian.leavesAll(treeID) + leavesAll, err := t.tlog.leavesAll(treeID) if err != nil { return nil, fmt.Errorf("leavesAll: %v", err) } @@ -247,7 +247,7 @@ func (t *Tstore) BlobsByDataDesc(treeID int64, dataDesc string) ([]store.BlobEnt } // Get leaves - leaves, err := t.trillian.leavesAll(treeID) + leaves, err := t.tlog.leavesAll(treeID) if err != nil { return nil, fmt.Errorf("leavesAll: %v", err) } @@ -316,7 +316,7 @@ func (t *Tstore) DigestsByDataDesc(treeID int64, dataDesc string) ([][]byte, err } // Get leaves - leaves, err := t.trillian.leavesAll(treeID) + leaves, err := t.tlog.leavesAll(treeID) if err != nil { return nil, fmt.Errorf("leavesAll: %v", err) } @@ -345,7 +345,7 @@ func (t *Tstore) Timestamp(treeID int64, digest []byte) (*backend.Timestamp, err log.Tracef("Timestamp: %v %x", treeID, digest) // Get tree leaves - leaves, err := t.trillian.leavesAll(treeID) + leaves, err := t.tlog.leavesAll(treeID) if err != nil { return nil, fmt.Errorf("leavesAll: %v", err) } diff --git a/politeiad/backendv2/tstorebe/tstore/recordindex.go b/politeiad/backendv2/tstorebe/tstore/recordindex.go index e5ee11e17..ff707d963 100644 --- a/politeiad/backendv2/tstorebe/tstore/recordindex.go +++ b/politeiad/backendv2/tstorebe/tstore/recordindex.go @@ -132,7 +132,7 @@ func (t *Tstore) recordIndexSave(treeID int64, ri recordIndex) error { leaves := []*trillian.LogLeaf{ newLogLeaf(d, extraData), } - queued, _, err := t.trillian.leavesAppend(treeID, leaves) + queued, _, err := t.tlog.leavesAppend(treeID, leaves) if err != nil { return fmt.Errorf("leavesAppend: %v", err) } diff --git a/politeiad/backendv2/tstorebe/tstore/testing.go b/politeiad/backendv2/tstorebe/tstore/testing.go index d7c6d013e..e06d944eb 100644 --- a/politeiad/backendv2/tstorebe/tstore/testing.go +++ b/politeiad/backendv2/tstorebe/tstore/testing.go @@ -40,7 +40,7 @@ func NewTestTstore(t *testing.T, dataDir string) *Tstore { return &Tstore{ encryptionKey: ek, - trillian: newTestTClient(t), + tlog: newTestTClient(t), store: store, } } diff --git a/politeiad/backendv2/tstorebe/tstore/trillianclient.go b/politeiad/backendv2/tstorebe/tstore/tlogclient.go similarity index 93% rename from politeiad/backendv2/tstorebe/tstore/trillianclient.go rename to politeiad/backendv2/tstorebe/tstore/tlogclient.go index dc7c25fef..0a74167ed 100644 --- a/politeiad/backendv2/tstorebe/tstore/trillianclient.go +++ b/politeiad/backendv2/tstorebe/tstore/tlogclient.go @@ -48,11 +48,11 @@ type queuedLeafProof struct { Proof *trillian.Proof } -// trillianClient provides an interface for interacting with a trillian log. It +// tlogClient provides an interface for interacting with a trillian log. It // creates an abstraction over the trillian provided TrillianLogClient and // TrillianAdminClient, creating a simplified API for the backend to use and // allowing us to create a implementation that can be used for testing. -type trillianClient interface { +type tlogClient interface { // treeNew creates a new tree. treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) @@ -86,10 +86,10 @@ type trillianClient interface { } var ( - _ trillianClient = (*tclient)(nil) + _ tlogClient = (*tclient)(nil) ) -// tclient implements the trillianClient interface. +// tclient implements the tlogClient interface. type tclient struct { host string grpc *grpc.ClientConn @@ -121,7 +121,7 @@ func newLogLeaf(leafValue []byte, extraData []byte) *trillian.LogLeaf { // correct. It returns the tree and the signed log root which can be externally // verified. // -// This function satisfies the trillianClient interface. +// This function satisfies the tlogClient interface. func (t *tclient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { log.Tracef("trillian treeNew") @@ -188,7 +188,7 @@ func (t *tclient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { // treeFreeze updates the state of a tree to frozen. // -// This function satisfies the trillianClient interface. +// This function satisfies the tlogClient interface. func (t *tclient) treeFreeze(treeID int64) (*trillian.Tree, error) { log.Tracef("trillian treeFreeze: %v", treeID) @@ -217,7 +217,7 @@ func (t *tclient) treeFreeze(treeID int64) (*trillian.Tree, error) { // tree returns a trillian tree. // -// This function satisfies the trillianClient interface. +// This function satisfies the tlogClient interface. func (t *tclient) tree(treeID int64) (*trillian.Tree, error) { log.Tracef("trillian tree: %v", treeID) @@ -238,7 +238,7 @@ func (t *tclient) tree(treeID int64) (*trillian.Tree, error) { // treesAll returns all trees in the trillian instance. // -// This function satisfies the trillianClient interface +// This function satisfies the tlogClient interface func (t *tclient) treesAll() ([]*trillian.Tree, error) { log.Tracef("trillian treesAll") @@ -253,7 +253,7 @@ func (t *tclient) treesAll() ([]*trillian.Tree, error) { // inclusionProof returns a proof for the inclusion of a merkle leaf hash in a // log root. // -// This function satisfies the trillianClient interface +// This function satisfies the tlogClient interface func (t *tclient) inclusionProof(treeID int64, merkleLeafHash []byte, lrv1 *types.LogRootV1) (*trillian.Proof, error) { log.Tracef("tillian inclusionProof: %v %x", treeID, merkleLeafHash) @@ -288,7 +288,7 @@ func (t *tclient) inclusionProof(treeID int64, merkleLeafHash []byte, lrv1 *type // signedLogRoot returns the signed log root of a trillian tree. // -// This function satisfies the trillianClient interface. +// This function satisfies the tlogClient interface. func (t *tclient) signedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoot, *types.LogRootV1, error) { // Get the signed log root for the current tree height resp, err := t.log.GetLatestSignedLogRoot(t.ctx, @@ -325,7 +325,7 @@ func (t *tclient) signedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoot, * // methods that can be used. DO NOT rely on the leaves being in a specific // order. // -// This function satisfies the trillianClient interface. +// This function satisfies the tlogClient interface. func (t *tclient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, *types.LogRootV1, error) { log.Tracef("trillian leavesAppend: %v %v", treeID, len(leaves)) @@ -440,7 +440,7 @@ func (t *tclient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queu // leavesByRange returns the log leaves of a trillian tree by the range provided // by the user. // -// This function satisfies the trillianClient interface. +// This function satisfies the tlogClient interface. func (t *tclient) leavesByRange(treeID int64, startIndex, count int64) ([]*trillian.LogLeaf, error) { log.Tracef("trillian leavesByRange: %v %v %v", treeID, startIndex, count) @@ -459,7 +459,7 @@ func (t *tclient) leavesByRange(treeID int64, startIndex, count int64) ([]*trill // leavesAll returns all of the leaves for the provided treeID. // -// This function satisfies the trillianClient interface. +// This function satisfies the tlogClient interface. func (t *tclient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { log.Tracef("trillian leavesAll: %v", treeID) @@ -489,7 +489,7 @@ func (t *tclient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { // close closes the trillian grpc connection. // -// This function satisfies the trillianClient interface. +// This function satisfies the tlogClient interface. func (t *tclient) close() { log.Tracef("trillian close %v", t.host) @@ -569,10 +569,10 @@ func newTClient(host, keyFile string) (*tclient, error) { } var ( - _ trillianClient = (*testTClient)(nil) + _ tlogClient = (*testTClient)(nil) ) -// testTClient implements the trillianClient interface and is used for testing. +// testTClient implements the tlogClient interface and is used for testing. type testTClient struct { sync.Mutex @@ -585,7 +585,7 @@ type testTClient struct { // treeNew ceates a new trillian tree in memory. // -// This function satisfies the trillianClient interface. +// This function satisfies the tlogClient interface. func (t *testTClient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { t.Lock() defer t.Unlock() @@ -619,7 +619,7 @@ func (t *testTClient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) // treeFreeze sets the state of a tree to frozen. // -// This function satisfies the trillianClient interface. +// This function satisfies the tlogClient interface. func (t *testTClient) treeFreeze(treeID int64) (*trillian.Tree, error) { t.Lock() defer t.Unlock() @@ -631,7 +631,7 @@ func (t *testTClient) treeFreeze(treeID int64) (*trillian.Tree, error) { // tree returns trillian tree from passed in ID. // -// This function satisfies the trillianClient interface. +// This function satisfies the tlogClient interface. func (t *testTClient) tree(treeID int64) (*trillian.Tree, error) { t.Lock() defer t.Unlock() @@ -646,7 +646,7 @@ func (t *testTClient) tree(treeID int64) (*trillian.Tree, error) { // treesAll signed log roots are not used for testing up until now, so we // return a nil value for it. // -// This function satisfies the trillianClient interface. +// This function satisfies the tlogClient interface. func (t *testTClient) treesAll() ([]*trillian.Tree, error) { t.Lock() defer t.Unlock() @@ -662,7 +662,7 @@ func (t *testTClient) treesAll() ([]*trillian.Tree, error) { // leavesAppend satisfies the TClient interface. It appends leaves to the // corresponding trillian tree in memory. // -// This function satisfies the trillianClient interface. +// This function satisfies the tlogClient interface. func (t *testTClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, *types.LogRootV1, error) { t.Lock() defer t.Unlock() @@ -698,7 +698,7 @@ func (t *testTClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([] // leavesAll returns all leaves from a trillian tree. // -// This function satisfies the trillianClient interface. +// This function satisfies the tlogClient interface. func (t *testTClient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { t.Lock() defer t.Unlock() @@ -714,14 +714,14 @@ func (t *testTClient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { // signedLogRoot has not been implemented yet for the test client. // -// This function satisfies the trillianClient interface. +// This function satisfies the tlogClient interface. func (t *testTClient) signedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoot, *types.LogRootV1, error) { return nil, nil, nil } // inclusionProof has not been implement yet for the test client. // -// This function satisfies the trillianClient interface. +// This function satisfies the tlogClient interface. func (t *testTClient) inclusionProof(treeID int64, merkleLeafHash []byte, lr *types.LogRootV1) (*trillian.Proof, error) { return nil, nil } @@ -729,7 +729,7 @@ func (t *testTClient) inclusionProof(treeID int64, merkleLeafHash []byte, lr *ty // close closes the trillian client connection. There is nothing to do for the // test implementation. // -// This function satisfies the trillianClient interface. +// This function satisfies the tlogClient interface. func (t *testTClient) close() {} // newTestTClient returns a new testTClient. diff --git a/politeiad/backendv2/tstorebe/tstore/tstore.go b/politeiad/backendv2/tstorebe/tstore/tstore.go index c29067356..f00e41a19 100644 --- a/politeiad/backendv2/tstorebe/tstore/tstore.go +++ b/politeiad/backendv2/tstorebe/tstore/tstore.go @@ -56,7 +56,7 @@ type Tstore struct { sync.Mutex dataDir string activeNetParams *chaincfg.Params - trillian trillianClient + tlog tlogClient store store.BlobKV dcrtime *dcrtimeClient cron *cron.Cron @@ -143,7 +143,7 @@ func (t *Tstore) deblob(b []byte) (*store.BlobEntry, error) { func (t *Tstore) TreeNew() (int64, error) { log.Tracef("TreeNew") - tree, _, err := t.trillian.treeNew() + tree, _, err := t.tlog.treeNew() if err != nil { return 0, err } @@ -178,7 +178,7 @@ func (t *Tstore) TreeFreeze(treeID int64, rm backend.RecordMetadata, metadata [] // TreesAll returns the IDs of all trees in the tstore instance. func (t *Tstore) TreesAll() ([]int64, error) { - trees, err := t.trillian.treesAll() + trees, err := t.tlog.treesAll() if err != nil { return nil, err } @@ -194,7 +194,7 @@ func (t *Tstore) TreesAll() ([]int64, error) { // tree to have been created but experienced an unexpected error prior to the // record being saved. func (t *Tstore) TreeExists(treeID int64) bool { - _, err := t.trillian.tree(treeID) + _, err := t.tlog.tree(treeID) return err == nil } @@ -477,7 +477,7 @@ func (t *Tstore) recordBlobsSave(treeID int64, rbpr recordBlobsPrepareReply) (*r } // Append leaves to trillian tree - queued, _, err := t.trillian.leavesAppend(treeID, leaves) + queued, _, err := t.tlog.leavesAppend(treeID, leaves) if err != nil { return nil, fmt.Errorf("leavesAppend: %v", err) } @@ -549,7 +549,7 @@ func (t *Tstore) RecordSave(treeID int64, rm backend.RecordMetadata, metadata [] } // Get tree leaves - leavesAll, err := t.trillian.leavesAll(treeID) + leavesAll, err := t.tlog.leavesAll(treeID) if err != nil { return fmt.Errorf("leavesAll %v: %v", treeID, err) } @@ -652,7 +652,7 @@ func (t *Tstore) metadataSave(treeID int64, rm backend.RecordMetadata, metadata } // Get tree leaves - leavesAll, err := t.trillian.leavesAll(treeID) + leavesAll, err := t.tlog.leavesAll(treeID) if err != nil { return nil, fmt.Errorf("leavesAll: %v", err) } @@ -734,7 +734,7 @@ func (t *Tstore) RecordDel(treeID int64) error { } // Get all tree leaves - leavesAll, err := t.trillian.leavesAll(treeID) + leavesAll, err := t.tlog.leavesAll(treeID) if err != nil { return err } @@ -803,7 +803,7 @@ func (t *Tstore) RecordExists(treeID int64) bool { } // Verify record index exists - leavesAll, err := t.trillian.leavesAll(treeID) + leavesAll, err := t.tlog.leavesAll(treeID) if err != nil { err = fmt.Errorf("leavesAll: %v", err) goto printErr @@ -844,7 +844,7 @@ func (t *Tstore) record(treeID int64, version uint32, filenames []string, omitAl } // Get tree leaves - leaves, err := t.trillian.leavesAll(treeID) + leaves, err := t.tlog.leavesAll(treeID) if err != nil { return nil, fmt.Errorf("leavesAll %v: %v", treeID, err) } @@ -1096,7 +1096,7 @@ func (t *Tstore) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trilli } // Get trillian inclusion proof - p, err := t.trillian.inclusionProof(treeID, l.MerkleLeafHash, a.LogRoot) + p, err := t.tlog.inclusionProof(treeID, l.MerkleLeafHash, a.LogRoot) if err != nil { return nil, fmt.Errorf("inclusionProof %v %x: %v", treeID, l.MerkleLeafHash, err) @@ -1179,7 +1179,7 @@ func (t *Tstore) RecordTimestamps(treeID int64, version uint32, token []byte) (* } // Get record index - leaves, err := t.trillian.leavesAll(treeID) + leaves, err := t.tlog.leavesAll(treeID) if err != nil { return nil, fmt.Errorf("leavesAll %v: %v", treeID, err) } @@ -1241,7 +1241,7 @@ func (t *Tstore) Close() { // Close connections t.store.Close() - t.trillian.close() + t.tlog.close() // Zero out encryption key. An encryption key is optional. if t.encryptionKey != nil { @@ -1298,7 +1298,7 @@ func New(appDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianSig log.Infof("Trillian key: %v", trillianSigningKeyFile) log.Infof("Trillian host: %v", trillianHost) - trillianClient, err := newTClient(trillianHost, trillianSigningKeyFile) + tlogClient, err := newTClient(trillianHost, trillianSigningKeyFile) if err != nil { return nil, err } @@ -1352,7 +1352,7 @@ func New(appDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianSig t := Tstore{ dataDir: dataDir, activeNetParams: anp, - trillian: trillianClient, + tlog: tlogClient, store: kvstore, dcrtime: dcrtimeClient, cron: cron.New(), From 839d9740518f1254043a2169e7aae5eb805030d7 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 10 Mar 2021 09:03:36 -0600 Subject: [PATCH 379/449] tstore: Save vetted blobs as plain text. --- .../tstorebe/plugins/comments/comments.go | 2 +- .../tstorebe/plugins/comments/recordindex.go | 1 + .../tstorebe/plugins/dcrdata/dcrdata.go | 2 +- politeiad/backendv2/tstorebe/plugins/pi/pi.go | 2 +- .../tstorebe/plugins/ticketvote/ticketvote.go | 2 +- .../tstorebe/plugins/usermd/usermd.go | 2 +- .../backendv2/tstorebe/store/mysql/mysql.go | 2 +- politeiad/backendv2/tstorebe/tstore/anchor.go | 4 +- politeiad/backendv2/tstorebe/tstore/client.go | 34 +- .../backendv2/tstorebe/tstore/recordindex.go | 30 +- .../backendv2/tstorebe/tstore/tlogclient.go | 8 +- politeiad/backendv2/tstorebe/tstore/tstore.go | 722 ++++++++---------- politeiad/backendv2/tstorebe/tstorebe.go | 53 +- 13 files changed, 429 insertions(+), 435 deletions(-) diff --git a/politeiad/backendv2/tstorebe/plugins/comments/comments.go b/politeiad/backendv2/tstorebe/plugins/comments/comments.go index b5f530222..0c3baa5d7 100644 --- a/politeiad/backendv2/tstorebe/plugins/comments/comments.go +++ b/politeiad/backendv2/tstorebe/plugins/comments/comments.go @@ -88,7 +88,7 @@ func (p *commentsPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (s // // This function satisfies the plugins.PluginClient interface. func (p *commentsPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { - log.Tracef("comments Hook: %v %x %v", treeID, token, plugins.Hooks[h]) + log.Tracef("comments Hook: %v %x %v", plugins.Hooks[h], token, treeID) return nil } diff --git a/politeiad/backendv2/tstorebe/plugins/comments/recordindex.go b/politeiad/backendv2/tstorebe/plugins/comments/recordindex.go index 7dc101591..be48dfef6 100644 --- a/politeiad/backendv2/tstorebe/plugins/comments/recordindex.go +++ b/politeiad/backendv2/tstorebe/plugins/comments/recordindex.go @@ -45,6 +45,7 @@ type commentIndex struct { // recordIndex contains the indexes for all comments made on a record. type recordIndex struct { + // TODO make unvetted and vetted indexes Comments map[uint32]commentIndex `json:"comments"` // [commentID]comment } diff --git a/politeiad/backendv2/tstorebe/plugins/dcrdata/dcrdata.go b/politeiad/backendv2/tstorebe/plugins/dcrdata/dcrdata.go index 0d0a8807d..60c39f02b 100644 --- a/politeiad/backendv2/tstorebe/plugins/dcrdata/dcrdata.go +++ b/politeiad/backendv2/tstorebe/plugins/dcrdata/dcrdata.go @@ -608,7 +608,7 @@ func (p *dcrdataPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (st // // This function satisfies the plugins.PluginClient interface. func (p *dcrdataPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { - log.Tracef("dcrdata Hook: %v %x %v", treeID, token, plugins.Hooks[h]) + log.Tracef("dcrdata Hook: %v %x %v", plugins.Hooks[h], token, treeID) return nil } diff --git a/politeiad/backendv2/tstorebe/plugins/pi/pi.go b/politeiad/backendv2/tstorebe/plugins/pi/pi.go index 98d4b7446..bd2097551 100644 --- a/politeiad/backendv2/tstorebe/plugins/pi/pi.go +++ b/politeiad/backendv2/tstorebe/plugins/pi/pi.go @@ -64,7 +64,7 @@ func (p *piPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, // // This function satisfies the plugins.PluginClient interface. func (p *piPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { - log.Tracef("pi Hook: %v %x %v", treeID, token, plugins.Hooks[h]) + log.Tracef("pi Hook: %v %x %v", plugins.Hooks[h], token, treeID) switch h { case plugins.HookTypeNewRecordPre: diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go index 47f28f168..f52806d67 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go @@ -185,7 +185,7 @@ func (p *ticketVotePlugin) Cmd(treeID int64, token []byte, cmd, payload string) // // This function satisfies the plugins.PluginClient interface. func (p *ticketVotePlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { - log.Tracef("ticketvote Hook: %v %x %v", treeID, token, plugins.Hooks[h]) + log.Tracef("ticketvote Hook: %v %x %v", plugins.Hooks[h], token, treeID) switch h { case plugins.HookTypeNewRecordPre: diff --git a/politeiad/backendv2/tstorebe/plugins/usermd/usermd.go b/politeiad/backendv2/tstorebe/plugins/usermd/usermd.go index eeaaf5207..7a5e3e0a0 100644 --- a/politeiad/backendv2/tstorebe/plugins/usermd/usermd.go +++ b/politeiad/backendv2/tstorebe/plugins/usermd/usermd.go @@ -57,7 +57,7 @@ func (p *userPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (strin // // This function satisfies the plugins.PluginClient interface. func (p *userPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { - log.Tracef("usermd Hook: %v %x %v", treeID, token, plugins.Hooks[h]) + log.Tracef("usermd Hook: %v %x %v", plugins.Hooks[h], token, treeID) switch h { case plugins.HookTypeNewRecordPre: diff --git a/politeiad/backendv2/tstorebe/store/mysql/mysql.go b/politeiad/backendv2/tstorebe/store/mysql/mysql.go index 09caaef5e..cb02131ad 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/mysql.go +++ b/politeiad/backendv2/tstorebe/store/mysql/mysql.go @@ -26,7 +26,7 @@ const ( // tableKeyValue defines the key-value table. The key is a uuid. const tableKeyValue = ` - k CHAR(36) NOT NULL PRIMARY KEY, + k VARCHAR(38) NOT NULL PRIMARY KEY, v LONGBLOB NOT NULL ` diff --git a/politeiad/backendv2/tstorebe/tstore/anchor.go b/politeiad/backendv2/tstorebe/tstore/anchor.go index c2f3b5561..3eacd3a13 100644 --- a/politeiad/backendv2/tstorebe/tstore/anchor.go +++ b/politeiad/backendv2/tstorebe/tstore/anchor.go @@ -217,7 +217,7 @@ func (t *Tstore) anchorSave(a anchor) error { if err != nil { return err } - b, err := store.Blobify(*be) + b, err := t.blobify(*be, false) if err != nil { return err } @@ -235,7 +235,7 @@ func (t *Tstore) anchorSave(a anchor) error { if err != nil { return err } - extraData, err := extraDataEncode(keys[0], dataDescriptorAnchor) + extraData, err := extraDataEncode(keys[0], dataDescriptorAnchor, false) if err != nil { return err } diff --git a/politeiad/backendv2/tstorebe/tstore/client.go b/politeiad/backendv2/tstorebe/tstore/client.go index 09d02fb37..47fc13d5b 100644 --- a/politeiad/backendv2/tstorebe/tstore/client.go +++ b/politeiad/backendv2/tstorebe/tstore/client.go @@ -36,16 +36,6 @@ func (t *Tstore) BlobSave(treeID int64, be store.BlobEntry) error { return err } - // Prepare blob and digest - digest, err := hex.DecodeString(be.Digest) - if err != nil { - return err - } - blob, err := t.blobify(be) - if err != nil { - return err - } - // Verify tree exists if !t.TreeExists(treeID) { return backend.ErrRecordNotFound @@ -56,10 +46,30 @@ func (t *Tstore) BlobSave(treeID int64, be store.BlobEntry) error { if err != nil { return fmt.Errorf("leavesAll: %v", err) } - if t.treeIsFrozen(leaves) { + idx, err := t.recordIndexLatest(leaves) + if err != nil { + return fmt.Errorf("recordIndexLatest: %v", err) + } + if idx.Frozen { + // The tree is frozen. The record is locked. return backend.ErrRecordLocked } + // Prepare blob and digest + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return err + } + encrypt := true + if idx.State == backend.StateVetted { + // Vetted data is not encrypted + encrypt = false + } + blob, err := t.blobify(be, encrypt) + if err != nil { + return err + } + // Save blobs to store keys, err := t.store.Put([][]byte{blob}) if err != nil { @@ -71,7 +81,7 @@ func (t *Tstore) BlobSave(treeID int64, be store.BlobEntry) error { } // Prepare log leaf - extraData, err := extraDataEncode(keys[0], dd.Descriptor) + extraData, err := extraDataEncode(keys[0], dd.Descriptor, encrypt) if err != nil { return err } diff --git a/politeiad/backendv2/tstorebe/tstore/recordindex.go b/politeiad/backendv2/tstorebe/tstore/recordindex.go index ff707d963..45a175302 100644 --- a/politeiad/backendv2/tstorebe/tstore/recordindex.go +++ b/politeiad/backendv2/tstorebe/tstore/recordindex.go @@ -39,6 +39,8 @@ import ( // update is considered successful. Any record content leaves that are not part // of a recordIndex are considered to be orphaned and can be disregarded. type recordIndex struct { + State backend.StateT `json:"state"` + // Version represents the version of the record. The version is // only incremented when the record files are updated. Metadata // only updates do no increment the version. @@ -102,12 +104,15 @@ func parseRecordIndex(indexes []recordIndex, version uint32) (*recordIndex, erro // recordIndexSave saves a record index to tstore. func (t *Tstore) recordIndexSave(treeID int64, ri recordIndex) error { + // Only vetted data should be saved encrypted + encrypt := (ri.State != backend.StateVetted) + // Save record index to the store be, err := convertBlobEntryFromRecordIndex(ri) if err != nil { return err } - b, err := t.blobify(*be) + b, err := t.blobify(*be, encrypt) if err != nil { return err } @@ -125,7 +130,8 @@ func (t *Tstore) recordIndexSave(treeID int64, ri recordIndex) error { if err != nil { return err } - extraData, err := extraDataEncode(keys[0], dataDescriptorRecordIndex) + extraData, err := extraDataEncode(keys[0], + dataDescriptorRecordIndex, encrypt) if err != nil { return err } @@ -195,7 +201,10 @@ func (t *Tstore) recordIndexes(leaves []*trillian.LogLeaf) ([]recordIndex, error return nil, fmt.Errorf("record index not found: %v", missing) } - indexes := make([]recordIndex, 0, len(blobs)) + var ( + unvetted = make([]recordIndex, 0, len(blobs)) + vetted = make([]recordIndex, 0, len(blobs)) + ) for _, v := range blobs { be, err := t.deblob(v) if err != nil { @@ -205,7 +214,20 @@ func (t *Tstore) recordIndexes(leaves []*trillian.LogLeaf) ([]recordIndex, error if err != nil { return nil, err } - indexes = append(indexes, *ri) + switch ri.State { + case backend.StateUnvetted: + unvetted = append(unvetted, *ri) + case backend.StateVetted: + vetted = append(vetted, *ri) + } + } + + // Once a record is made vetted the record history is considered + // to restart. If any vetted indexes exist, ignore all unvetted + // indexes. + indexes := unvetted + if len(vetted) > 0 { + indexes = vetted } // Sort indexes by iteration, smallest to largets. The leaves diff --git a/politeiad/backendv2/tstorebe/tstore/tlogclient.go b/politeiad/backendv2/tstorebe/tstore/tlogclient.go index 0a74167ed..5ab62bbae 100644 --- a/politeiad/backendv2/tstorebe/tstore/tlogclient.go +++ b/politeiad/backendv2/tstorebe/tstore/tlogclient.go @@ -369,7 +369,7 @@ func (t *tclient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queu } } - log.Trace("Queued/Ignored leaves: %v/%v", len(leaves)-n, n) + log.Tracef("Queued/Ignored leaves: %v/%v", len(leaves)-n, n) log.Tracef("Waiting for inclusion of queued leaves...") var logRoot types.LogRootV1 @@ -434,6 +434,12 @@ func (t *tclient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queu proofs = append(proofs, qlp) } + // Sanity check + if len(proofs) != len(leaves) { + return nil, nil, fmt.Errorf("got %v queued leaves, want %v", + len(proofs), len(leaves)) + } + return proofs, lr, nil } diff --git a/politeiad/backendv2/tstorebe/tstore/tstore.go b/politeiad/backendv2/tstorebe/tstore/tstore.go index f00e41a19..0f0500520 100644 --- a/politeiad/backendv2/tstorebe/tstore/tstore.go +++ b/politeiad/backendv2/tstorebe/tstore/tstore.go @@ -25,6 +25,7 @@ import ( "github.com/decred/politeia/politeiad/backendv2/tstorebe/store/mysql" "github.com/decred/politeia/util" "github.com/google/trillian" + "github.com/google/uuid" "github.com/marcopeereboom/sbox" "github.com/robfig/cron" "google.golang.org/grpc/codes" @@ -40,11 +41,19 @@ const ( defaultStoreDirname = "store" // Blob entry data descriptors - dataDescriptorFile = "pd-file-v1" dataDescriptorRecordMetadata = "pd-recordmd-v1" dataDescriptorMetadataStream = "pd-mdstream-v1" + dataDescriptorFile = "pd-file-v1" dataDescriptorRecordIndex = "pd-rindex-v1" dataDescriptorAnchor = "pd-anchor-v1" + + // keyPrefixEncrypted is prefixed onto key-value store keys if the + // data is encrypted. We do this so that when a record is made + // public we can save the plain text record content blobs using the + // same keys, but without the prefix. Using a new key for the plain + // text blobs would not work since we cannot append a new leaf onto + // the tlog without getting a duplicate leaf error. + keyPrefixEncrypted = "e_" ) var ( @@ -76,7 +85,9 @@ type Tstore struct { // blobIsEncrypted returns whether the provided blob has been prefixed with an // sbox header, indicating that it is an encrypted blob. func blobIsEncrypted(b []byte) bool { - return bytes.HasPrefix(b, []byte("sbox")) + isEncrypted := bytes.HasPrefix(b, []byte("sbox")) + log.Tracef("Blob is encrypted: %v", isEncrypted) + return isEncrypted } // extraData is the data that is stored in the log leaf ExtraData field. It is @@ -85,12 +96,25 @@ func blobIsEncrypted(b []byte) bool { type extraData struct { Key string `json:"k"` // Key-value store key Desc string `json:"d"` // Blob entry data descriptor + + // Encrypted indicates if the key-value store blob is encrypted. + // The key for encrypted blobs must be prefixed. + Encrypted bool `json:"e,omitempty"` } -func extraDataEncode(key, desc string) ([]byte, error) { +func (e *extraData) storeKey() string { + if e.Encrypted == true { + return keyPrefixEncrypted + e.Key + } + return e.Key +} + +func extraDataEncode(key, desc string, encrypted bool) ([]byte, error) { + // The encryption prefix is stripped from the key if one exists. ed := extraData{ - Key: key, - Desc: desc, + Key: storeKeyClean(key), + Desc: desc, + Encrypted: encrypted, } b, err := json.Marshal(ed) if err != nil { @@ -108,12 +132,38 @@ func extraDataDecode(b []byte) (*extraData, error) { return &ed, nil } -func (t *Tstore) blobify(be store.BlobEntry) ([]byte, error) { +// storeKeyNew returns a new key for the key-value store. If the data is +// encrypted the key is prefixed. +func storeKeyNew(encrypt bool) string { + k := uuid.New().String() + if encrypt { + k = keyPrefixEncrypted + k + } + return k +} + +// storeKeyClean strips the key-value store key of the encryption prefix if +// one is present. +func storeKeyClean(key string) string { + // A uuid string is 36 bytes. Return the last 36 bytes of the + // string. This will strip the prefix if it exists. + return key[len(key)-36:] +} + +func merkleLeafHashForBlobEntry(be store.BlobEntry) ([]byte, error) { + leafValue, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, err + } + return merkleLeafHash(leafValue), nil +} + +func (t *Tstore) blobify(be store.BlobEntry, encrypt bool) ([]byte, error) { b, err := store.Blobify(be) if err != nil { return nil, err } - if t.encryptionKey != nil { + if encrypt { b, err = t.encryptionKey.encrypt(0, b) if err != nil { return nil, err @@ -124,10 +174,7 @@ func (t *Tstore) blobify(be store.BlobEntry) ([]byte, error) { func (t *Tstore) deblob(b []byte) (*store.BlobEntry, error) { var err error - if t.encryptionKey != nil { - if !blobIsEncrypted(b) { - return nil, fmt.Errorf("attempted to decrypt an unecrypted blob") - } + if blobIsEncrypted(b) { b, _, err = t.encryptionKey.decrypt(b) if err != nil { return nil, err @@ -160,16 +207,16 @@ func (t *Tstore) TreeNew() (int64, error) { // anchored, the tstore fsck function will update the status of the tree to // frozen in trillian, at which point trillian will prevent any changes to the // tree. -func (t *Tstore) TreeFreeze(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { +func (t *Tstore) TreeFreeze(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { log.Tracef("TreeFreeze: %v", treeID) - // Save metadata - idx, err := t.metadataSave(treeID, rm, metadata) + // Save updated record + idx, err := t.recordSave(treeID, rm, metadata, files) if err != nil { return err } - // Update the record index + // Mark the record as frozen idx.Frozen = true // Save the record index @@ -206,49 +253,18 @@ func (t *Tstore) treeIsFrozen(leaves []*trillian.LogLeaf) bool { return r.Frozen } -type recordHashes struct { - recordMetadata string // Record metadata hash - metadata map[string]backend.MetadataStream // [hash]MetadataStream - files map[string]backend.File // [hash]File -} - -type recordBlobsPrepareReply struct { - // recordIndex is the index for the record content. It is created - // during the blobs prepare step so that it can be populated with - // the merkle leaf hashes of duplicate data, i.e. data that remains - // unchanged between two versions of a record. It will be fully - // populated once the unique blobs haves been saved to the kv store - // and appended onto the trillian tree. - recordIndex recordIndex - - // recordHashes contains a mapping of the record content hashes to - // the record content type. This is used to populate the record - // index once the leaves have been appended onto the trillian tree. - recordHashes recordHashes - - // blobs contains the blobified record content that needs to be - // saved to the kv store. - // - // Hashes contains the hashes of the record content prior to being - // blobified. These hashes are saved to trilian log leaves. The - // hashes are SHA256 hashes of the JSON encoded data. - // - // hints contains the data hints of the blob entries. - // - // blobs, hashes, and descriptors share the same ordering. - blobs [][]byte - hashes [][]byte - hints []string -} +// recordBlobsSave saves the provided blobs to the kv store, appends a leaf +// to the trillian tree for each blob, and returns the record index for the +// blobs. +func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, recordMD backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) (*recordIndex, error) { + log.Tracef("recordBlobsSave: %v", treeID) -// recordBlobsPrepare prepares the provided record content to be saved to -// the blob kv store and appended onto a trillian tree. -func (t *Tstore) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) (*recordBlobsPrepareReply, error) { - // Verify there are no duplicate or empty mdstream IDs + // Verify there are no duplicate metadata streams md := make(map[string]map[uint32]struct{}, len(metadata)) for _, v := range metadata { - if v.StreamID == 0 { - return nil, fmt.Errorf("invalid metadata stream ID 0") + if v.PluginID == "" || v.StreamID == 0 { + return nil, fmt.Errorf("invalid metadata stream: '%v' %v", + v.PluginID, v.StreamID) } pmd, ok := md[v.PluginID] if !ok { @@ -263,228 +279,244 @@ func (t *Tstore) recordBlobsPrepare(leavesAll []*trillian.LogLeaf, recordMD back md[v.PluginID] = pmd } - // Verify there are no duplicate or empty filenames - filenames := make(map[string]struct{}, len(files)) + // Verify there are no duplicate files + fn := make(map[string]struct{}, len(files)) for _, v := range files { if v.Name == "" { return nil, fmt.Errorf("empty filename") } - _, ok := filenames[v.Name] + _, ok := fn[v.Name] if ok { return nil, fmt.Errorf("duplicate filename found: %v", v.Name) } - filenames[v.Name] = struct{}{} + fn[v.Name] = struct{}{} } - // Check if any of the content already exists. Different record - // versions that reference the same data is fine, but this data - // should not be saved to the store again. We can find duplicates - // by walking the trillian tree and comparing the hash of the - // provided record content to the log leaf data, which will be the - // same for duplicates. + // Prepare the blob entries. The record index can also be created + // during this step. + var ( + // [pluginID][streamID]BlobEntry + beMetadata = make(map[string]map[uint32]store.BlobEntry, len(metadata)) + + // [filename]BlobEntry + beFiles = make(map[string]store.BlobEntry, len(files)) + + idx = recordIndex{ + State: recordMD.State, + Version: recordMD.Version, + Iteration: recordMD.Iteration, + Metadata: make(map[string]map[uint32][]byte, len(metadata)), + Files: make(map[string][]byte, len(files)), + } - // Compute record content hashes - rhashes := recordHashes{ - // [hash]MetadataStream - metadata: make(map[string]backend.MetadataStream, len(metadata)), + // digests is used to aggregate the digests from all record + // content. This is used later on to see if any of the content + // already exists in the tstore. + digests = make(map[string]struct{}, 256) + ) - // [hash]File - files: make(map[string]backend.File, len(files)), + // Setup record metadata + beRecordMD, err := convertBlobEntryFromRecordMetadata(recordMD) + if err != nil { + return nil, err } - b, err := json.Marshal(recordMD) + m, err := merkleLeafHashForBlobEntry(*beRecordMD) if err != nil { return nil, err } - rhashes.recordMetadata = hex.EncodeToString(util.Digest(b)) + idx.RecordMetadata = m + digests[beRecordMD.Digest] = struct{}{} + + // Setup metdata streams for _, v := range metadata { - b, err := json.Marshal(v) + // Blob entry + be, err := convertBlobEntryFromMetadataStream(v) if err != nil { return nil, err } - h := hex.EncodeToString(util.Digest(b)) - rhashes.metadata[h] = v - } - for _, v := range files { - b, err := json.Marshal(v) + streams, ok := beMetadata[v.PluginID] + if !ok { + streams = make(map[uint32]store.BlobEntry, len(metadata)) + } + streams[v.StreamID] = *be + beMetadata[v.PluginID] = streams + + // Record index + m, err := merkleLeafHashForBlobEntry(*be) if err != nil { return nil, err } - h := hex.EncodeToString(util.Digest(b)) - rhashes.files[h] = v + streamsIdx, ok := idx.Metadata[v.PluginID] + if !ok { + streamsIdx = make(map[uint32][]byte, len(metadata)) + } + streamsIdx[v.StreamID] = m + idx.Metadata[v.PluginID] = streamsIdx + + // Aggregate digest + digests[be.Digest] = struct{}{} } - // Compare leaf data to record content hashes to find duplicates - var ( - // Dups tracks duplicates so we know which blobs should be - // skipped when blobifying record content. - dups = make(map[string]struct{}, 64) - - // Any duplicates that are found are added to the record index - // since we already have the leaf data for them. - index = recordIndex{ - Metadata: make(map[string]map[uint32][]byte, len(metadata)), - Files: make(map[string][]byte, len(files)), + // Setup files + for _, v := range files { + // Blob entry + be, err := convertBlobEntryFromFile(v) + if err != nil { + return nil, err } - ) - for _, v := range leavesAll { - h := hex.EncodeToString(v.LeafValue) + beFiles[v.Name] = *be - // Check record metadata - if h == rhashes.recordMetadata { - dups[h] = struct{}{} - index.RecordMetadata = v.MerkleLeafHash - continue + // Record Index + m, err := merkleLeafHashForBlobEntry(*be) + if err != nil { + return nil, err } + idx.Files[v.Name] = m - // Check metadata streams - ms, ok := rhashes.metadata[h] - if ok { - dups[h] = struct{}{} - streams, ok := index.Metadata[ms.PluginID] - if !ok { - streams = make(map[uint32][]byte, 64) - } - streams[ms.StreamID] = v.MerkleLeafHash - index.Metadata[ms.PluginID] = streams - continue - } + // Aggregate digest + digests[be.Digest] = struct{}{} + } - // Check files - f, ok := rhashes.files[h] + // Check if any of the content already exists. Different record + // versions that reference the same data is fine, but this data + // should not be saved to the store again. We can find duplicates + // by comparing the blob entry digest to the log leaf value. They + // will be the same if the record content is the same. + dups := make(map[string]struct{}, len(digests)) + for _, v := range leavesAll { + d := hex.EncodeToString(v.LeafValue) + _, ok := digests[d] if ok { - dups[h] = struct{}{} - index.Files[f.Name] = v.MerkleLeafHash - continue + // A piece of the new record content already exsits in the + // tstore. Save the digest as a duplcate. + dups[d] = struct{}{} } } - // Prepare kv store blobs. The hashes of the record content are - // also aggregated and will be used to create the log leaves that - // are appended to the trillian tree. - l := len(metadata) + len(files) + 1 - hashes := make([][]byte, 0, l) - blobs := make([][]byte, 0, l) - hints := make([]string, 0, l) + // Prepare blobs for the kv store + var ( + blobs = make(map[string][]byte, len(digests)) + leaves = make([]*trillian.LogLeaf, 0, len(blobs)) - // Prepare record metadata blob - be, err := convertBlobEntryFromRecordMetadata(recordMD) - if err != nil { - return nil, err - } - h, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, err - } - b, err = t.blobify(*be) - if err != nil { - return nil, err - } - _, ok := dups[be.Digest] - if !ok { - // Not a duplicate. Save blob to the store. - hashes = append(hashes, h) - blobs = append(blobs, b) - hints = append(hints, be.DataHint) - } + // dupBlobs contains the blob entries for record content that + // already exists. We may need these blob entries later on if + // the duplicate content is encrypted and it needs to be saved + // plain text. + dupBlobs = make(map[string]store.BlobEntry, len(digests)) - // Prepare metadata blobs - for _, v := range metadata { - be, err := convertBlobEntryFromMetadataStream(v) + // Only vetted data should be saved unencrypted + encrypt = (recordMD.State != backend.StateVetted) + ) + + // Prepare record metadata blobs and leaves + _, ok := dups[beRecordMD.Digest] + if !ok { + // Not a duplicate. Prepare kv store blob. + b, err := t.blobify(*beRecordMD, encrypt) if err != nil { return nil, err } - h, err := hex.DecodeString(be.Digest) + k := storeKeyNew(encrypt) + blobs[k] = b + + // Prepare tlog leaf + extraData, err := extraDataEncode(k, + dataDescriptorRecordMetadata, encrypt) if err != nil { return nil, err } - b, err := t.blobify(*be) + digest, err := hex.DecodeString(beRecordMD.Digest) if err != nil { return nil, err } - _, ok := dups[be.Digest] - if !ok { - // Not a duplicate. Save blob to the store. - hashes = append(hashes, h) - blobs = append(blobs, b) - hints = append(hints, be.DataHint) - } + leaves = append(leaves, newLogLeaf(digest, extraData)) + } else { + // This is a duplicate. Stash is for now. We may need to save + // it as plain text later. + dupBlobs[beRecordMD.Digest] = *beRecordMD } - // Prepare file blobs - for _, v := range files { - be, err := convertBlobEntryFromFile(v) - if err != nil { - return nil, err - } - h, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, err - } - b, err := t.blobify(*be) - if err != nil { - return nil, err - } - _, ok := dups[be.Digest] - if !ok { - // Not a duplicate. Save blob to the store. - hashes = append(hashes, h) - blobs = append(blobs, b) - hints = append(hints, be.DataHint) + // Prepare metadata stream blobs and leaves + for _, v := range beMetadata { + for _, be := range v { + _, ok := dups[be.Digest] + if !ok { + // Not a duplicate. Prepare kv store blob. + b, err := t.blobify(be, encrypt) + if err != nil { + return nil, err + } + k := storeKeyNew(encrypt) + blobs[k] = b + + // Prepare tlog leaf + extraData, err := extraDataEncode(k, + dataDescriptorMetadataStream, encrypt) + if err != nil { + return nil, err + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, err + } + leaves = append(leaves, newLogLeaf(digest, extraData)) + + continue + } + + // This is a duplicate. Stash is for now. We may need to save + // it as plain text later. + dupBlobs[be.Digest] = be } } - return &recordBlobsPrepareReply{ - recordIndex: index, - recordHashes: rhashes, - blobs: blobs, - hashes: hashes, - hints: hints, - }, nil -} + // Prepare file blobs and leaves + for _, be := range beFiles { + _, ok := dups[be.Digest] + if !ok { + // Not a duplicate. Prepare kv store blob. + b, err := t.blobify(be, encrypt) + if err != nil { + return nil, err + } + k := storeKeyNew(encrypt) + blobs[k] = b -// recordBlobsSave saves the provided blobs to the kv store, appends a leaf -// to the trillian tree for each blob, then updates the record index with the -// trillian leaf information and returns it. -func (t *Tstore) recordBlobsSave(treeID int64, rbpr recordBlobsPrepareReply) (*recordIndex, error) { - log.Tracef("recordBlobsSave: %v", treeID) + // Prepare tlog leaf + extraData, err := extraDataEncode(k, dataDescriptorFile, encrypt) + if err != nil { + return nil, err + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, err + } + leaves = append(leaves, newLogLeaf(digest, extraData)) - var ( - index = rbpr.recordIndex - rhashes = rbpr.recordHashes - blobs = rbpr.blobs - hashes = rbpr.hashes - hints = rbpr.hints - ) + continue + } - // Save blobs to store - keys, err := t.store.Put(blobs) - if err != nil { - return nil, fmt.Errorf("store Put: %v", err) + // This is a duplicate. Stash is for now. We may need to save + // it as plain text later. + dupBlobs[be.Digest] = be } - if len(keys) != len(blobs) { - return nil, fmt.Errorf("wrong number of keys: got %v, want %v", - len(keys), len(blobs)) + + // Verify at least one new blob is being saved to the kv store + if len(blobs) == 0 { + return nil, backend.ErrNoRecordChanges } - // Prepare log leaves. hashes and keys share the same ordering. - leaves := make([]*trillian.LogLeaf, 0, len(blobs)) - for k := range blobs { - extraData, err := extraDataEncode(keys[k], hints[k]) - if err != nil { - return nil, err - } - leaves = append(leaves, newLogLeaf(hashes[k], extraData)) + // Save blobs to the kv store + err = t.store.PutKV(blobs) + if err != nil { + return nil, fmt.Errorf("store PutKV: %v", err) } - // Append leaves to trillian tree + // Append leaves onto the trillian tree queued, _, err := t.tlog.leavesAppend(treeID, leaves) if err != nil { return nil, fmt.Errorf("leavesAppend: %v", err) } - if len(queued) != len(leaves) { - return nil, fmt.Errorf("wrong number of queued leaves: got %v, want %v", - len(queued), len(leaves)) - } failed := make([]string, 0, len(queued)) for _, v := range queued { c := codes.Code(v.QueuedLeaf.GetStatus().GetCode()) @@ -496,62 +528,75 @@ func (t *Tstore) recordBlobsSave(treeID int64, rbpr recordBlobsPrepareReply) (*r return nil, fmt.Errorf("append leaves failed: %v", failed) } - // Update the new record index with the log leaves - for _, v := range queued { - // Figure out what piece of record content this leaf represents - h := hex.EncodeToString(v.QueuedLeaf.Leaf.LeafValue) + // Check if any of the duplicates were saved as encrypted but now + // need to be resaved as plain text. This happens when a record is + // made public and the files need to be saved plain text. + if encrypt || len(dups) == 0 { + // Nothing that needs to be saved plain text. We're done. + log.Tracef("No blobs need to be resaved plain text") - // Check record metadata - if h == rhashes.recordMetadata { - index.RecordMetadata = v.QueuedLeaf.Leaf.MerkleLeafHash + return &idx, nil + } + + blobs = make(map[string][]byte, len(dupBlobs)) + for _, v := range leavesAll { + d := hex.EncodeToString(v.LeafValue) + _, ok := dups[d] + if !ok { + // Not a duplicate continue } - // Check metadata streams - ms, ok := rhashes.metadata[h] - if ok { - streams, ok := index.Metadata[ms.PluginID] - if !ok { - streams = make(map[uint32][]byte, 64) - } - streams[ms.StreamID] = v.QueuedLeaf.Leaf.MerkleLeafHash - index.Metadata[ms.PluginID] = streams + // This is a duplicate. If its encrypted it will need to be + // resaved as plain text. + ed, err := extraDataDecode(v.ExtraData) + if err != nil { + return nil, err + } + if !ed.Encrypted { + // Not encrypted continue } - // Check files - f, ok := rhashes.files[h] - if ok { - index.Files[f.Name] = v.QueuedLeaf.Leaf.MerkleLeafHash - continue + // Prepare plain text blob + be, ok := dupBlobs[d] + if !ok { + // Should not happen + return nil, fmt.Errorf("blob entry not found %v", d) } + b, err := t.blobify(be, false) + if err != nil { + return nil, err + } + blobs[ed.Key] = b + } + if len(blobs) == 0 { + // Nothing that needs to be saved plain text. We're done. + log.Tracef("No duplicates need to be resaved plain text") - // Something went wrong. None of the record content matches the - // leaf. - return nil, fmt.Errorf("record content does not match leaf: %x", - v.QueuedLeaf.Leaf.MerkleLeafHash) + return &idx, nil } - return &index, nil -} + err = t.store.PutKV(blobs) + if err != nil { + return nil, fmt.Errorf("store PutKV: %v", err) + } -// RecordSave saves the provided record to tstore. Once the record contents -// have been successfully saved to tstore, a recordIndex is created for this -// version of the record and saved to tstore as well. This iteration of the -// record is not considered to be valid until the record index has been -// successfully saved. -func (t *Tstore) RecordSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { - log.Tracef("RecordSave: %v", treeID) + log.Debugf("Resaved %v encrypted blobs as plain text", len(blobs)) + + return &idx, nil +} +func (t *Tstore) recordSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) (*recordIndex, error) { // Verify tree exists if !t.TreeExists(treeID) { - return backend.ErrRecordNotFound + return nil, backend.ErrRecordNotFound } // Get tree leaves leavesAll, err := t.tlog.leavesAll(treeID) if err != nil { - return fmt.Errorf("leavesAll %v: %v", treeID, err) + return nil, fmt.Errorf("leavesAll %v: %v", treeID, err) } // Get the existing record index @@ -563,156 +608,41 @@ func (t *Tstore) RecordSave(treeID int64, rm backend.RecordMetadata, metadata [] Files: make(map[string][]byte), } } else if err != nil { - return fmt.Errorf("recordIndexLatest: %v", err) - } - - // Verify tree state - if currIdx.Frozen { - return backend.ErrRecordLocked - } - - // Prepare kv store blobs - bpr, err := t.recordBlobsPrepare(leavesAll, rm, metadata, files) - if err != nil { - return err - } - - // Verify file changes are being made. - var fileChanges bool - for _, v := range files { - // Duplicate blobs have already been added to the new record - // index by the recordBlobsPrepare function. If a file is in the - // new record index it means that the file has existed in one of - // the previous versions of the record. - newMerkle, ok := bpr.recordIndex.Files[v.Name] - if !ok { - // File does not exist in the index. It is new. - fileChanges = true - break - } - - // We now know the file has existed in a previous version of the - // record, but it may not have be the most recent version. If the - // file is not part of the current record index then it means - // there are file changes between the current version and new - // version. - currMerkle, ok := currIdx.Files[v.Name] - if !ok { - // File is not part of the current version. - fileChanges = true - break - } - - // We now know that the new file has existed in some previous - // version of the record and the there is a file in the current - // version of the record that has the same filename as the new - // file. Check if the merkles match. If the merkles are different - // then it means the files are different, they just use the same - // filename. - if !bytes.Equal(newMerkle, currMerkle) { - // Files share the same name but have different content. - fileChanges = true - break - } - } - if !fileChanges { - return backend.ErrNoRecordChanges - } - - // Save blobs - idx, err := t.recordBlobsSave(treeID, *bpr) - if err != nil { - return fmt.Errorf("blobsSave: %v", err) - } - - // Bump the index version and iteration - idx.Version = rm.Version - idx.Iteration = rm.Iteration - - // Save record index - err = t.recordIndexSave(treeID, *idx) - if err != nil { - return fmt.Errorf("recordIndexSave: %v", err) - } - - return nil -} - -// metadataSave saves the provided metadata to the tstore. The record index -// for this iteration of the record is returned. This is step one of a two step -// process. The record update will not be considered successful until the -// returned record index is also saved to the kv store and trillian tree. This -// code has been pulled out so that it can be called during normal metadata -// updates as well as when an update requires the tree to be frozen, such as -// when a record is censored. -func (t *Tstore) metadataSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) (*recordIndex, error) { - // Verify tree exists - if !t.TreeExists(treeID) { - return nil, backend.ErrRecordNotFound - } - - // Get tree leaves - leavesAll, err := t.tlog.leavesAll(treeID) - if err != nil { - return nil, fmt.Errorf("leavesAll: %v", err) + return nil, fmt.Errorf("recordIndexLatest: %v", err) } - // Verify tree state - currIdx, err := t.recordIndexLatest(leavesAll) - if err != nil { - return nil, err - } + // Verify tree is not frozen if currIdx.Frozen { return nil, backend.ErrRecordLocked } - // Prepare kv store blobs - bpr, err := t.recordBlobsPrepare(leavesAll, rm, metadata, []backend.File{}) + // Save the record + idx, err := t.recordBlobsSave(treeID, leavesAll, rm, metadata, files) if err != nil { - return nil, err - } - - // Verify at least one new blob is being saved to the kv store - if len(bpr.blobs) == 0 { - return nil, backend.ErrNoRecordChanges - } - - // Save the blobs - idx, err := t.recordBlobsSave(treeID, *bpr) - if err != nil { - return nil, fmt.Errorf("blobsSave: %v", err) - } - - // Get the existing record index and add the unchanged fields to - // the new record index. - oldIdx, err := t.recordIndexLatest(leavesAll) - if err != nil { - return nil, fmt.Errorf("recordIndexLatest: %v", err) + if err == backend.ErrNoRecordChanges { + return nil, err + } + return nil, fmt.Errorf("recordBlobsSave: %v", err) } - idx.Version = rm.Version - idx.Files = oldIdx.Files - - // Update the version and iteration - idx.Version = rm.Version - idx.Iteration = rm.Iteration return idx, nil } -// RecordMetadataSave saves the provided metadata to tstore, creating a new -// iteration of the record while keeping the record version the same. Once the -// metadata has been successfully saved to tstore, a recordIndex is created for -// this iteration of the record and saved to tstore as well. -func (t *Tstore) RecordMetadataSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { - log.Tracef("RecordMetadataSave: %v", treeID) +// RecordSave saves the provided record to tstore. Once the record contents +// have been successfully saved to tstore, a recordIndex is created for this +// version of the record and saved to tstore as well. This iteration of the +// record is not considered to be valid until the record index has been +// successfully saved. +func (t *Tstore) RecordSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { + log.Tracef("RecordSave: %v", treeID) - // Save metadata - idx, err := t.metadataSave(treeID, rm, metadata) + // Save the record + idx, err := t.recordSave(treeID, rm, metadata, files) if err != nil { return err } - // Save record index + // Save the record index err = t.recordIndexSave(treeID, *idx) if err != nil { return fmt.Errorf("recordIndexSave: %v", err) @@ -772,7 +702,7 @@ func (t *Tstore) RecordDel(treeID int64) error { if err != nil { return err } - keys = append(keys, ed.Key) + keys = append(keys, ed.storeKey()) } } @@ -904,7 +834,19 @@ func (t *Tstore) record(treeID int64, version uint32, filenames []string, omitAl if err != nil { return nil, err } - keys = append(keys, ed.Key) + + var key string + switch idx.State { + case backend.StateVetted: + // If the record is vetted the content may exist in the store + // as both an encrypted blob and a plain text blob. Always pull + // the plaintext blob. + key = ed.Key + default: + // Pull the encrypted blob + key = ed.storeKey() + } + keys = append(keys, key) } // Get record content from store @@ -1050,7 +992,7 @@ func (t *Tstore) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trilli if err != nil { return nil, err } - blobs, err := t.store.Get([]string{ed.Key}) + blobs, err := t.store.Get([]string{ed.storeKey()}) if err != nil { return nil, fmt.Errorf("store get: %v", err) } @@ -1060,9 +1002,9 @@ func (t *Tstore) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trilli // the rest of the timestamp. var data []byte if len(blobs) == 1 { - b, ok := blobs[ed.Key] + b, ok := blobs[ed.storeKey()] if !ok { - return nil, fmt.Errorf("blob not found %v", ed.Key) + return nil, fmt.Errorf("blob not found %v", ed.storeKey()) } be, err := t.deblob(b) if err != nil { diff --git a/politeiad/backendv2/tstorebe/tstorebe.go b/politeiad/backendv2/tstorebe/tstorebe.go index f9704fee9..33bfec39c 100644 --- a/politeiad/backendv2/tstorebe/tstorebe.go +++ b/politeiad/backendv2/tstorebe/tstorebe.go @@ -528,6 +528,14 @@ func (t *tstoreBackend) RecordEdit(token []byte, mdAppend, mdOverwrite []backend return nil, err } + // Verify that file changes are being made. The merkle root is the + // merkle root of the files. It will be the same if no file changes + // are being made. + if r.RecordMetadata.Merkle == recordMD.Merkle { + // No file changes found + return nil, backend.ErrNoRecordChanges + } + // Call pre plugin hooks her := plugins.HookEditRecord{ Current: *r, @@ -549,7 +557,7 @@ func (t *tstoreBackend) RecordEdit(token []byte, mdAppend, mdOverwrite []backend err = t.tstore.RecordSave(treeID, *recordMD, metadata, files) if err != nil { switch err { - case backend.ErrRecordLocked, backend.ErrNoRecordChanges: + case backend.ErrRecordLocked: return nil, err default: return nil, fmt.Errorf("RecordSave: %v", err) @@ -643,13 +651,13 @@ func (t *tstoreBackend) RecordEditMetadata(token []byte, mdAppend, mdOverwrite [ } // Update metadata - err = t.tstore.RecordMetadataSave(treeID, *recordMD, metadata) + err = t.tstore.RecordSave(treeID, *recordMD, metadata, r.Files) if err != nil { switch err { case backend.ErrRecordLocked, backend.ErrNoRecordChanges: return nil, err default: - return nil, fmt.Errorf("RecordMetadataSave: %v", err) + return nil, fmt.Errorf("RecordSave: %v", err) } } @@ -700,19 +708,18 @@ func statusChangeIsAllowed(from, to backend.StatusT) bool { // setStatusPublic updates the status of a record to public. // // This function must be called WITH the record lock held. -func (t *tstoreBackend) setStatusPublic(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { - // TODO tstore needs a publish method +func (t *tstoreBackend) setStatusPublic(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { treeID := treeIDFromToken(token) - return t.tstore.RecordMetadataSave(treeID, rm, metadata) + return t.tstore.RecordSave(treeID, rm, metadata, files) } // setStatusArchived updates the status of a record to archived. // // This function must be called WITH the record lock held. -func (t *tstoreBackend) setStatusArchived(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { +func (t *tstoreBackend) setStatusArchived(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { // Freeze the tree treeID := treeIDFromToken(token) - err := t.tstore.TreeFreeze(treeID, rm, metadata) + err := t.tstore.TreeFreeze(treeID, rm, metadata, files) if err != nil { return fmt.Errorf("TreeFreeze %v: %v", treeID, err) } @@ -727,10 +734,10 @@ func (t *tstoreBackend) setStatusArchived(token []byte, rm backend.RecordMetadat // setStatusCensored updates the status of a record to censored. // // This function must be called WITH the record lock held. -func (t *tstoreBackend) setStatusCensored(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream) error { +func (t *tstoreBackend) setStatusCensored(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { // Freeze the tree treeID := treeIDFromToken(token) - err := t.tstore.TreeFreeze(treeID, rm, metadata) + err := t.tstore.TreeFreeze(treeID, rm, metadata, files) if err != nil { return fmt.Errorf("TreeFreeze %v: %v", treeID, err) } @@ -790,18 +797,24 @@ func (t *tstoreBackend) RecordSetStatus(token []byte, status backend.StatusT, md } } - // Determine the state. Making a record public will also trigger - // a state update to vetted. - var state backend.StateT + // If the record is being made public the record state gets updated + // to vetted and the version and iteration are reset. Otherwise, + // the state and version remain the same while the iteration gets + // incremented to reflect the status change. + var ( + state = r.RecordMetadata.State + version = r.RecordMetadata.Version + iter = r.RecordMetadata.Iteration + 1 // Increment for status change + ) if status == backend.StatusPublic { state = backend.StateVetted - } else { - state = r.RecordMetadata.State + version = 1 + iter = 1 } // Apply changes - recordMD, err := recordMetadataNew(token, r.Files, state, status, - r.RecordMetadata.Version, r.RecordMetadata.Iteration+1) + recordMD, err := recordMetadataNew(token, r.Files, + state, status, version, iter) if err != nil { return nil, err } @@ -826,17 +839,17 @@ func (t *tstoreBackend) RecordSetStatus(token []byte, status backend.StatusT, md // Update record status switch status { case backend.StatusPublic: - err := t.setStatusPublic(token, *recordMD, metadata) + err := t.setStatusPublic(token, *recordMD, metadata, r.Files) if err != nil { return nil, err } case backend.StatusArchived: - err := t.setStatusArchived(token, *recordMD, metadata) + err := t.setStatusArchived(token, *recordMD, metadata, r.Files) if err != nil { return nil, err } case backend.StatusCensored: - err := t.setStatusCensored(token, *recordMD, metadata) + err := t.setStatusCensored(token, *recordMD, metadata, r.Files) if err != nil { return nil, err } From 99f65d9ce75199e2636a698995d99ab0f729a51e Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 10 Mar 2021 15:03:07 -0600 Subject: [PATCH 380/449] tstore: Update TstoreClient methods. --- .../backendv2/tstorebe/plugins/plugins.go | 22 +- .../tstorebe/store/localdb/localdb.go | 50 +--- .../backendv2/tstorebe/store/mysql/mysql.go | 50 +--- politeiad/backendv2/tstorebe/store/store.go | 12 +- politeiad/backendv2/tstorebe/tstore/anchor.go | 14 +- politeiad/backendv2/tstorebe/tstore/client.go | 276 ++++++++++++------ .../backendv2/tstorebe/tstore/recordindex.go | 149 ++++++---- politeiad/backendv2/tstorebe/tstore/tstore.go | 133 ++++----- 8 files changed, 369 insertions(+), 337 deletions(-) diff --git a/politeiad/backendv2/tstorebe/plugins/plugins.go b/politeiad/backendv2/tstorebe/plugins/plugins.go index d29e7f06b..e46d30d00 100644 --- a/politeiad/backendv2/tstorebe/plugins/plugins.go +++ b/politeiad/backendv2/tstorebe/plugins/plugins.go @@ -153,10 +153,10 @@ type PluginClient interface { // backend. Editing plugin data is not allowed. type TstoreClient interface { // BlobSave saves a BlobEntry to the tstore instance. The BlobEntry - // will be encrypted prior to being written to disk if the tstore - // instance has an encryption key set. The digest of the data, - // i.e. BlobEntry.Digest, can be thought of as the blob ID and can - // be used to get/del the blob from tstore. + // will be encrypted prior to being written to disk if the record + // is unvetted. The digest of the data, i.e. BlobEntry.Digest, can + // be thought of as the blob ID that can be used to get/del the + // blob from tstore. BlobSave(treeID int64, be store.BlobEntry) error // BlobsDel deletes the blobs that correspond to the provided @@ -165,24 +165,24 @@ type TstoreClient interface { // Blobs returns the blobs that correspond to the provided digests. // If a blob does not exist it will not be included in the returned - // map. + // map. If a record is vetted, only vetted blobs will be returned. Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEntry, error) // BlobsByDataDesc returns all blobs that match the provided data - // descriptor. The blobs will be ordered from oldest to newest. + // descriptor. The blobs will be ordered from oldest to newest. If + // a record is vetted then only vetted blobs will be returned. BlobsByDataDesc(treeID int64, dataDesc string) ([]store.BlobEntry, error) // DigestsByDataDesc returns the digests of all blobs that match - // the provided data descriptor. + // the provided data descriptor. If a record is vetted, only + // vetted blobs will be returned. DigestsByDataDesc(treeID int64, dataDesc string) ([][]byte, error) // Timestamp returns the timestamp for the blob that correpsonds - // to the digest. + // to the digest. If a record is vetted, only vetted timestamps + // will be returned. Timestamp(treeID int64, digest []byte) (*backend.Timestamp, error) - // RecordExists returns whether a record exists. - RecordExists(treeID int64) bool - // Record returns a version of a record. Record(treeID int64, version uint32) (*backend.Record, error) diff --git a/politeiad/backendv2/tstorebe/store/localdb/localdb.go b/politeiad/backendv2/tstorebe/store/localdb/localdb.go index b4fcc7807..0f8efc7bf 100644 --- a/politeiad/backendv2/tstorebe/store/localdb/localdb.go +++ b/politeiad/backendv2/tstorebe/store/localdb/localdb.go @@ -10,7 +10,6 @@ import ( "sync" "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" - "github.com/google/uuid" "github.com/syndtr/goleveldb/leveldb" ) @@ -26,7 +25,13 @@ type localdb struct { db *leveldb.DB } -func (l *localdb) put(blobs map[string][]byte) error { +// Put saves the provided key-value pairs to the store. This operation is +// performed atomically. +// +// This function satisfies the store BlobKV interface. +func (l *localdb) Put(blobs map[string][]byte) error { + log.Tracef("Put: %v blobs", len(blobs)) + // Setup batch batch := new(leveldb.Batch) for k, v := range blobs { @@ -44,47 +49,6 @@ func (l *localdb) put(blobs map[string][]byte) error { return nil } -// Put saves the provided blobs to the store. The keys for the blobs are -// returned using the same odering that the blobs were provided in. This -// operation is performed atomically. -// -// This function satisfies the store BlobKV interface. -func (l *localdb) Put(blobs [][]byte) ([]string, error) { - log.Tracef("Put: %v", len(blobs)) - - // Setup the keys. The keys are returned in the same order that - // the blobs are in. - var ( - keys = make([]string, 0, len(blobs)) - blobkv = make(map[string][]byte, len(blobs)) - ) - for _, v := range blobs { - k := uuid.New().String() - keys = append(keys, k) - blobkv[k] = v - } - - // Save blobs - err := l.put(blobkv) - if err != nil { - return nil, err - } - - // Return the keys - return keys, nil -} - -// PutKV saves the provided blobs to the store. This method allows the caller -// to specify the key instead of having the store create one. This operation is -// performed atomically. -// -// This function satisfies the store BlobKV interface. -func (l *localdb) PutKV(blobs map[string][]byte) error { - log.Tracef("PutKV: %v blobs", len(blobs)) - - return l.put(blobs) -} - // Del deletes the provided blobs from the store. This operation is performed // atomically. // diff --git a/politeiad/backendv2/tstorebe/store/mysql/mysql.go b/politeiad/backendv2/tstorebe/store/mysql/mysql.go index cb02131ad..6619cf7be 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/mysql.go +++ b/politeiad/backendv2/tstorebe/store/mysql/mysql.go @@ -11,7 +11,6 @@ import ( "time" "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" - "github.com/google/uuid" _ "github.com/go-sql-driver/mysql" ) @@ -43,7 +42,13 @@ func ctxWithTimeout() (context.Context, func()) { return context.WithTimeout(context.Background(), connTimeout) } -func (s *mysql) put(blobs map[string][]byte) error { +// Put saves the provided key-value pairs to the store. This operation is +// performed atomically. +// +// This function satisfies the store BlobKV interface. +func (s *mysql) Put(blobs map[string][]byte) error { + log.Tracef("Put: %v blobs", len(blobs)) + ctx, cancel := ctxWithTimeout() defer cancel() @@ -80,47 +85,6 @@ func (s *mysql) put(blobs map[string][]byte) error { return nil } -// Put saves the provided blobs to the store The keys for the blobs are -// returned using the same odering that the blobs were provided in. This -// operation is performed atomically. -// -// This function satisfies the store BlobKV interface. -func (s *mysql) Put(blobs [][]byte) ([]string, error) { - log.Tracef("Put: %v blobs", len(blobs)) - - // Setup the keys. The keys are returned in the same order that - // the blobs are in. - var ( - keys = make([]string, 0, len(blobs)) - blobkv = make(map[string][]byte, len(blobs)) - ) - for _, v := range blobs { - k := uuid.New().String() - keys = append(keys, k) - blobkv[k] = v - } - - // Save blobs - err := s.put(blobkv) - if err != nil { - return nil, err - } - - // Return the keys - return keys, nil -} - -// PutKV saves the provided blobs to the store. This method allows the caller -// to specify the key instead of having the store create one. This operation is -// performed atomically. -// -// This function satisfies the store BlobKV interface. -func (s *mysql) PutKV(blobs map[string][]byte) error { - log.Tracef("PutKV: %v blobs", len(blobs)) - - return s.put(blobs) -} - // Del deletes the provided blobs from the store This operation is performed // atomically. // diff --git a/politeiad/backendv2/tstorebe/store/store.go b/politeiad/backendv2/tstorebe/store/store.go index 6a566762d..8c2156aaa 100644 --- a/politeiad/backendv2/tstorebe/store/store.go +++ b/politeiad/backendv2/tstorebe/store/store.go @@ -77,15 +77,9 @@ func Deblob(blob []byte) (*BlobEntry, error) { // BlobKV represents a blob key-value store. type BlobKV interface { - // Put saves the provided blobs to the store. The keys for the - // blobs are returned using the same odering that the blobs were - // provided in. This operation is performed atomically. - Put(blobs [][]byte) ([]string, error) - - // PutKV saves the provided blobs to the store. This method allows - // the caller to specify the key instead of having the store create - // one. This operation is performed atomically. - PutKV(blobs map[string][]byte) error + // Put saves the provided key-value pairs to the store. This + // operation is performed atomically. + Put(blobs map[string][]byte) error // Del deletes the provided blobs from the store. This operation // is performed atomically. diff --git a/politeiad/backendv2/tstorebe/tstore/anchor.go b/politeiad/backendv2/tstorebe/tstore/anchor.go index 3eacd3a13..7ffe88c50 100644 --- a/politeiad/backendv2/tstorebe/tstore/anchor.go +++ b/politeiad/backendv2/tstorebe/tstore/anchor.go @@ -99,7 +99,7 @@ func (t *Tstore) anchorForLeaf(treeID int64, merkleLeafHash []byte, leaves []*tr return nil, err } if ed.Desc == dataDescriptorAnchor { - keys = append(keys, ed.Key) + keys = append(keys, ed.storeKey()) if len(keys) == 2 { break } @@ -166,7 +166,7 @@ func (t *Tstore) anchorLatest(treeID int64) (*anchor, error) { return nil, err } if ed.Desc == dataDescriptorAnchor { - key = ed.Key + key = ed.storeKey() break } } @@ -221,21 +221,19 @@ func (t *Tstore) anchorSave(a anchor) error { if err != nil { return err } - keys, err := t.store.Put([][]byte{b}) + key := storeKeyNew(false) + kv := map[string][]byte{key: b} + err = t.store.Put(kv) if err != nil { return fmt.Errorf("store Put: %v", err) } - if len(keys) != 1 { - return fmt.Errorf("wrong number of keys: got %v, want 1", - len(keys)) - } // Append anchor leaf to trillian tree d, err := hex.DecodeString(be.Digest) if err != nil { return err } - extraData, err := extraDataEncode(keys[0], dataDescriptorAnchor, false) + extraData, err := extraDataEncode(key, dataDescriptorAnchor, 0) if err != nil { return err } diff --git a/politeiad/backendv2/tstorebe/tstore/client.go b/politeiad/backendv2/tstorebe/tstore/client.go index 47fc13d5b..f782167e4 100644 --- a/politeiad/backendv2/tstorebe/tstore/client.go +++ b/politeiad/backendv2/tstorebe/tstore/client.go @@ -5,6 +5,7 @@ package tstore import ( + "bytes" "encoding/base64" "encoding/hex" "encoding/json" @@ -17,25 +18,14 @@ import ( ) // BlobSave saves a BlobEntry to the tstore instance. The BlobEntry will be -// encrypted prior to being written to disk if the tstore instance has an -// encryption key set. The digest of the data, i.e. BlobEntry.Digest, can be -// thought of as the blob ID and can be used to get/del the blob from tstore. +// encrypted prior to being written to disk if the record is unvetted. The +// digest of the data, i.e. BlobEntry.Digest, can be thought of as the blob ID +// and can be used to get/del the blob from tstore. // // This function satisfies the plugins TstoreClient interface. func (t *Tstore) BlobSave(treeID int64, be store.BlobEntry) error { log.Tracef("BlobSave: %v", treeID) - // Parse the data descriptor - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return err - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return err - } - // Verify tree exists if !t.TreeExists(treeID) { return backend.ErrRecordNotFound @@ -55,33 +45,53 @@ func (t *Tstore) BlobSave(treeID int64, be store.BlobEntry) error { return backend.ErrRecordLocked } - // Prepare blob and digest - digest, err := hex.DecodeString(be.Digest) + // Parse the data descriptor + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return err + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) if err != nil { return err } - encrypt := true - if idx.State == backend.StateVetted { - // Vetted data is not encrypted + + // Only vetted data should be saved plain text + var encrypt bool + switch idx.State { + case backend.StateUnvetted: + encrypt = true + case backend.StateVetted: + // Save plain text encrypt = false + default: + // Something is wrong + e := fmt.Sprintf("invalid record state %v %v", treeID, idx.State) + panic(e) + } + + // Prepare blob and digest + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return err } blob, err := t.blobify(be, encrypt) if err != nil { return err } + key := storeKeyNew(encrypt) + kv := map[string][]byte{key: blob} - // Save blobs to store - keys, err := t.store.Put([][]byte{blob}) + log.Debugf("Saving plugin data blob") + + // Save blob to store + err = t.store.Put(kv) if err != nil { return fmt.Errorf("store Put: %v", err) } - if len(keys) != 1 { - return fmt.Errorf("wrong number of keys: got %v, want 1", - len(keys)) - } // Prepare log leaf - extraData, err := extraDataEncode(keys[0], dd.Descriptor, encrypt) + extraData, err := extraDataEncode(key, dd.Descriptor, idx.State) if err != nil { return err } @@ -95,8 +105,8 @@ func (t *Tstore) BlobSave(treeID int64, be store.BlobEntry) error { return fmt.Errorf("leavesAppend: %v", err) } if len(queued) != 1 { - return fmt.Errorf("wrong number of queued leaves: "+ - "got %v, want 1", len(queued)) + return fmt.Errorf("wrong queued leaves count: got %v, want 1", + len(queued)) } c := codes.Code(queued[0].QueuedLeaf.GetStatus().GetCode()) if c != codes.OK { @@ -143,7 +153,7 @@ func (t *Tstore) BlobsDel(treeID int64, digests [][]byte) error { if err != nil { return err } - keys = append(keys, ed.Key) + keys = append(keys, ed.storeKey()) } } @@ -157,7 +167,8 @@ func (t *Tstore) BlobsDel(treeID int64, digests [][]byte) error { } // Blobs returns the blobs that correspond to the provided digests. If a blob -// does not exist it will not be included in the returned map. +// does not exist it will not be included in the returned map. If a record +// is vetted, only vetted blobs will be returned. // // This function satisfies the plugins TstoreClient interface. func (t *Tstore) Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEntry, error) { @@ -173,71 +184,70 @@ func (t *Tstore) Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEnt } // Get leaves - leavesAll, err := t.tlog.leavesAll(treeID) + leaves, err := t.tlog.leavesAll(treeID) if err != nil { return nil, fmt.Errorf("leavesAll: %v", err) } - // Aggregate the leaves that correspond to the provided merkle - // hashes. - // map[merkleLeafHash]*trillian.LogLeaf - leaves := make(map[string]*trillian.LogLeaf, len(digests)) - for _, v := range digests { - m := hex.EncodeToString(merkleLeafHash(v)) - leaves[m] = nil - } - for _, v := range leavesAll { - m := hex.EncodeToString(v.MerkleLeafHash) - if _, ok := leaves[m]; ok { - leaves[m] = v - } - } + // Determine if the record is vetted. If the record is vetted, only + // vetted blobs will be returned. + isVetted := recordIsVetted(leaves) - // Ensure a leaf was found for all provided merkle hashes - for k, v := range leaves { - if v == nil { - return nil, fmt.Errorf("leaf not found: %v", k) - } + // Put digests into a map + ds := make(map[string]struct{}, len(digests)) + for _, v := range digests { + ds[hex.EncodeToString(v)] = struct{}{} } - // Extract the key-value store keys. These keys MUST be put in the - // same order that the digests were provided in. - keys := make([]string, 0, len(leaves)) - for _, v := range digests { - m := hex.EncodeToString(merkleLeafHash(v)) - l, ok := leaves[m] - if !ok { - return nil, fmt.Errorf("leaf not found: %x", v) - } - ed, err := extraDataDecode(l.ExtraData) + // Find the log leaves for the provided digests. matchedLeaves and + // matchedKeys MUST share the same ordering. + var ( + matchedLeaves = make([]*trillian.LogLeaf, 0, len(digests)) + matchedKeys = make([]string, 0, len(digests)) + ) + for _, v := range leaves { + ed, err := extraDataDecode(v.ExtraData) if err != nil { return nil, err } - keys = append(keys, ed.Key) + if isVetted && ed.State == backend.StateUnvetted { + // We don't return unvetted blobs if the record is vetted + continue + } + + // Check if this is one of the target digests + if _, ok := ds[hex.EncodeToString(v.LeafValue)]; ok { + // Its a match! + matchedLeaves = append(matchedLeaves, v) + matchedKeys = append(matchedKeys, ed.storeKey()) + } + } + if len(matchedKeys) == 0 { + return map[string]store.BlobEntry{}, nil } - // Pull the blobs from the store. It's ok if one or more blobs is - // not found. It is the responsibility of the caller to decide how - // this should be handled. - blobs, err := t.store.Get(keys) + // Pull the blobs from the store + blobs, err := t.store.Get(matchedKeys) if err != nil { return nil, fmt.Errorf("store Get: %v", err) } - // Deblob the blobs and put them in a map so the caller can - // determine if any blob entries are missing. - entries := make(map[string]store.BlobEntry, len(blobs)) // [digest]BlobEntry - for k, v := range keys { - // The digests slice and the keys slice share the same order - digest := hex.EncodeToString(digests[k]) + // Prepare reply + entries := make(map[string]store.BlobEntry, len(matchedKeys)) + for i, v := range matchedKeys { b, ok := blobs[v] if !ok { - return nil, fmt.Errorf("blob not found: %v", v) + // Blob wasn't found in the store. Skip it. + continue } be, err := t.deblob(b) if err != nil { - return nil, fmt.Errorf("deblob %v: %v", digest, err) + return nil, err } + + // Get the corresponding digest + l := matchedLeaves[i] + digest := hex.EncodeToString(l.LeafValue) entries[digest] = *be } @@ -245,7 +255,8 @@ func (t *Tstore) Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEnt } // BlobsByDataDesc returns all blobs that match the provided data descriptor. -// The blobs will be ordered from oldest to newest. +// The blobs will be ordered from oldest to newest. If a record is vetted then +// only vetted blobs will be returned. // // This function satisfies the plugins TstoreClient interface. func (t *Tstore) BlobsByDataDesc(treeID int64, dataDesc string) ([]store.BlobEntry, error) { @@ -262,20 +273,20 @@ func (t *Tstore) BlobsByDataDesc(treeID int64, dataDesc string) ([]store.BlobEnt return nil, fmt.Errorf("leavesAll: %v", err) } - // Walk leaves and aggregate the key-value store keys for all - // leaves with a matching key prefix. - keys := make([]string, 0, len(leaves)) - for _, v := range leaves { + // Find all matching leaves + matches := leavesForDescriptor(leaves, dataDesc) + if len(matches) == 0 { + return []store.BlobEntry{}, nil + } + + // Aggregate the keys of all the matches + keys := make([]string, 0, len(matches)) + for _, v := range matches { ed, err := extraDataDecode(v.ExtraData) if err != nil { return nil, err } - if ed.Desc == dataDesc { - keys = append(keys, ed.Key) - } - } - if len(keys) == 0 { - return []store.BlobEntry{}, nil + keys = append(keys, ed.storeKey()) } // Pull the blobs from the store @@ -314,7 +325,8 @@ func (t *Tstore) BlobsByDataDesc(treeID int64, dataDesc string) ([]store.BlobEnt } // DigestsByDataDesc returns the digests of all blobs that match the provided -// data descriptor. +// data descriptor. If a record is vetted then only vetted digests will be +// returned. // // This function satisfies the plugins TstoreClient interface. func (t *Tstore) DigestsByDataDesc(treeID int64, dataDesc string) ([][]byte, error) { @@ -331,24 +343,21 @@ func (t *Tstore) DigestsByDataDesc(treeID int64, dataDesc string) ([][]byte, err return nil, fmt.Errorf("leavesAll: %v", err) } - // Walk leaves and aggregate the digests, i.e. the leaf value, of - // all leaves that match the provided data type. - digests := make([][]byte, 0, len(leaves)) - for _, v := range leaves { - ed, err := extraDataDecode(v.ExtraData) - if err != nil { - return nil, err - } - if ed.Desc == dataDesc { - digests = append(digests, v.LeafValue) - } + // Find all matching leaves + matches := leavesForDescriptor(leaves, dataDesc) + + // Aggregate the digests, i.e. the leaf value, for all the matches + digests := make([][]byte, 0, len(matches)) + for _, v := range matches { + digests = append(digests, v.LeafValue) } return digests, nil } // Timestamp returns the timestamp for the data blob that corresponds to the -// provided digest. +// provided digest. If a record is vetted, only vetted timestamps will be +// returned. // // This function satisfies the plugins TstoreClient interface. func (t *Tstore) Timestamp(treeID int64, digest []byte) (*backend.Timestamp, error) { @@ -360,9 +369,82 @@ func (t *Tstore) Timestamp(treeID int64, digest []byte) (*backend.Timestamp, err return nil, fmt.Errorf("leavesAll: %v", err) } + // Determine if the record is vetted + isVetted := recordIsVetted(leaves) + + // If the record is vetted we cannot return an unvetted timestamp. + // Find the leaf for the digest and verify that its not unvetted. + if isVetted { + for _, v := range leaves { + if !bytes.Equal(v.LeafValue, digest) { + // Not the target leaf + continue + } + + // This is the target leaf. Verify that its vetted. + ed, err := extraDataDecode(v.ExtraData) + if err != nil { + return nil, err + } + if ed.State != backend.StateVetted { + log.Debugf("Caller is requesting an unvetted timestamp " + + "for a vetted record; not allowed") + return &backend.Timestamp{ + Proofs: []backend.Proof{}, + }, nil + } + } + } + // Get merkle leaf hash m := merkleLeafHash(digest) // Get timestamp return t.timestamp(treeID, m, leaves) } + +// recordIsVetted returns whether the provided leaves contain any vetted record +// indexes, which indicates whether the record is vetted. +func recordIsVetted(leaves []*trillian.LogLeaf) bool { + for _, v := range leaves { + ed, err := extraDataDecode(v.ExtraData) + if err != nil { + panic(err) + } + if ed.Desc == dataDescriptorRecordIndex && + ed.State == backend.StateVetted { + // Vetted record index found + return true + } + } + return false +} + +func leavesForDescriptor(leaves []*trillian.LogLeaf, desc string) []*trillian.LogLeaf { + // Determine if the record is vetted. If the record is vetted then + // only vetted leaves will be returned. + isVetted := recordIsVetted(leaves) + + // Walk leaves and aggregate all leaves that match the provided + // data descriptor. + matches := make([]*trillian.LogLeaf, 0, len(leaves)) + for _, v := range leaves { + ed, err := extraDataDecode(v.ExtraData) + if err != nil { + panic(err) + } + switch { + case ed.Desc != desc: + // Not the data descriptor we're looking for + continue + case isVetted && ed.State != backend.StateVetted: + // Unvetted leaf on a vetted record. Don't use it. + continue + default: + // We have a match! + matches = append(matches, v) + } + } + + return matches +} diff --git a/politeiad/backendv2/tstorebe/tstore/recordindex.go b/politeiad/backendv2/tstorebe/tstore/recordindex.go index 45a175302..fdbed441d 100644 --- a/politeiad/backendv2/tstorebe/tstore/recordindex.go +++ b/politeiad/backendv2/tstorebe/tstore/recordindex.go @@ -72,43 +72,26 @@ type recordIndex struct { Frozen bool `json:"frozen,omitempty"` } -// parseRecordIndex takes a list of record indexes and returns the most recent -// iteration of the specified version. A version of 0 indicates that the latest -// version should be returned. A backend.ErrRecordNotFound is returned if the -// provided version does not exist. -func parseRecordIndex(indexes []recordIndex, version uint32) (*recordIndex, error) { - // Return the record index for the specified version - var ri *recordIndex - if version == 0 { - // A version of 0 indicates that the most recent version should - // be returned. - ri = &indexes[len(indexes)-1] - } else { - // Walk the indexes backwards so the most recent iteration of the - // specified version is selected. - for i := len(indexes) - 1; i >= 0; i-- { - r := indexes[i] - if r.Version == version { - ri = &r - break - } - } - } - if ri == nil { - // The specified version does not exist - return nil, backend.ErrRecordNotFound +// recordIndexSave saves a record index to tstore. +func (t *Tstore) recordIndexSave(treeID int64, idx recordIndex) error { + // Only vetted data should be saved plain text + var encrypt bool + switch idx.State { + case backend.StateUnvetted: + encrypt = true + case backend.StateVetted: + // Save plain text + encrypt = false + default: + // Something is wrong + e := fmt.Sprintf("invalid record state %v %v", treeID, idx.State) + panic(e) } - return ri, nil -} - -// recordIndexSave saves a record index to tstore. -func (t *Tstore) recordIndexSave(treeID int64, ri recordIndex) error { - // Only vetted data should be saved encrypted - encrypt := (ri.State != backend.StateVetted) + log.Debugf("Saving record index") // Save record index to the store - be, err := convertBlobEntryFromRecordIndex(ri) + be, err := convertBlobEntryFromRecordIndex(idx) if err != nil { return err } @@ -116,22 +99,20 @@ func (t *Tstore) recordIndexSave(treeID int64, ri recordIndex) error { if err != nil { return err } - keys, err := t.store.Put([][]byte{b}) + key := storeKeyNew(encrypt) + kv := map[string][]byte{key: b} + err = t.store.Put(kv) if err != nil { return fmt.Errorf("store Put: %v", err) } - if len(keys) != 1 { - return fmt.Errorf("wrong number of keys: got %v, want 1", - len(keys)) - } // Append record index leaf to trillian tree d, err := hex.DecodeString(be.Digest) if err != nil { return err } - extraData, err := extraDataEncode(keys[0], - dataDescriptorRecordIndex, encrypt) + extraData, err := extraDataEncode(key, + dataDescriptorRecordIndex, idx.State) if err != nil { return err } @@ -163,24 +144,37 @@ func (t *Tstore) recordIndexSave(treeID int64, ri recordIndex) error { // recordIndexes returns all record indexes found in the provided trillian // leaves. func (t *Tstore) recordIndexes(leaves []*trillian.LogLeaf) ([]recordIndex, error) { - // Walk the leaves and compile the keys for all record indexes. It - // is possible for multiple indexes to exist for the same record - // version (they will have different iterations due to metadata - // only updates) so we have to pull the index blobs from the store - // in order to find the most recent iteration for the specified - // version. - keys := make([]string, 0, 64) + // Walk the leaves and compile the keys for all record indexes. + // Once a record is made vetted the record history is considered + // to restart. If any vetted indexes exist, ignore all unvetted + // indexes. + var ( + keysUnvetted = make([]string, 0, 256) + keysVetted = make([]string, 0, 256) + ) for _, v := range leaves { ed, err := extraDataDecode(v.ExtraData) if err != nil { return nil, err } if ed.Desc == dataDescriptorRecordIndex { - // This is a record index leaf. Save the kv store key. - keys = append(keys, ed.Key) + // This is a record index leaf + switch ed.State { + case backend.StateUnvetted: + keysUnvetted = append(keysUnvetted, ed.storeKey()) + case backend.StateVetted: + keysVetted = append(keysVetted, ed.storeKey()) + default: + // Should not happen + return nil, fmt.Errorf("invalid extra data state: %v %v", + v.LeafIndex, ed.State) + } } } - + keys := keysUnvetted + if len(keysVetted) > 0 { + keys = keysVetted + } if len(keys) == 0 { // No records have been added to this tree yet return nil, backend.ErrRecordNotFound @@ -219,12 +213,12 @@ func (t *Tstore) recordIndexes(leaves []*trillian.LogLeaf) ([]recordIndex, error unvetted = append(unvetted, *ri) case backend.StateVetted: vetted = append(vetted, *ri) + default: + return nil, fmt.Errorf("invalid record index state: %v", + ri.State) } } - // Once a record is made vetted the record history is considered - // to restart. If any vetted indexes exist, ignore all unvetted - // indexes. indexes := unvetted if len(vetted) > 0 { indexes = vetted @@ -275,3 +269,52 @@ func (t *Tstore) recordIndex(leaves []*trillian.LogLeaf, version uint32) (*recor func (t *Tstore) recordIndexLatest(leaves []*trillian.LogLeaf) (*recordIndex, error) { return t.recordIndex(leaves, 0) } + +// parseRecordIndex takes a list of record indexes and returns the most recent +// iteration of the specified version. A version of 0 indicates that the latest +// version should be returned. A backend.ErrRecordNotFound is returned if the +// provided version does not exist. +func parseRecordIndex(indexes []recordIndex, version uint32) (*recordIndex, error) { + if len(indexes) == 0 { + return nil, backend.ErrRecordNotFound + } + + // This function should only be used on record indexes that share + // the same record state. We would not want to accidentally return + // an unvetted index if the record is vetted. It is the + // responsibility of the caller to only provide a single state. + state := indexes[0].State + if state == backend.StateInvalid { + return nil, fmt.Errorf("invalid record index state: %v", state) + } + for _, v := range indexes { + if v.State != state { + return nil, fmt.Errorf("multiple record index states found: %v %v", + v.State, state) + } + } + + // Return the record index for the specified version + var ri *recordIndex + if version == 0 { + // A version of 0 indicates that the most recent version should + // be returned. + ri = &indexes[len(indexes)-1] + } else { + // Walk the indexes backwards so the most recent iteration of the + // specified version is selected. + for i := len(indexes) - 1; i >= 0; i-- { + r := indexes[i] + if r.Version == version { + ri = &r + break + } + } + } + if ri == nil { + // The specified version does not exist + return nil, backend.ErrRecordNotFound + } + + return ri, nil +} diff --git a/politeiad/backendv2/tstorebe/tstore/tstore.go b/politeiad/backendv2/tstorebe/tstore/tstore.go index 0f0500520..f22e04d1f 100644 --- a/politeiad/backendv2/tstorebe/tstore/tstore.go +++ b/politeiad/backendv2/tstorebe/tstore/tstore.go @@ -94,27 +94,46 @@ func blobIsEncrypted(b []byte) bool { // saved as a JSON encoded byte slice. The JSON keys have been abbreviated to // minimize the size of a trillian log leaf. type extraData struct { - Key string `json:"k"` // Key-value store key - Desc string `json:"d"` // Blob entry data descriptor - - // Encrypted indicates if the key-value store blob is encrypted. - // The key for encrypted blobs must be prefixed. - Encrypted bool `json:"e,omitempty"` + // Key contains the key-value store key. If this blob is part of an + // unvetted record the key will need to be prefixed with the + // keyPrefixEncrypted in order to retrieve the blob from the kv + // store. Use the extraData.storeKey() method to retrieve the key. + // Do NOT reference this key directly. + Key string `json:"k"` + + // Desc contains the blob entry data descriptor. + Desc string `json:"d"` + + // State indicates the record state of the blob that this leaf + // corresponds to. Unvetted blobs encrypted prior to being saved + // to the store. When retrieving unvetted blobs from the kv store + // the keyPrefixEncrypted prefix must be added to the Key field. + // State will not be populated for anchor records. + State backend.StateT `json:"s,omitempty"` } +// storeKey returns the kv store key for the blob. If the blob is part of an +// unvetted record it will be saved as an encrypted blob in the kv store and +// the key is prefixed with keyPrefixEncrypted. func (e *extraData) storeKey() string { - if e.Encrypted == true { + if e.State == backend.StateUnvetted { return keyPrefixEncrypted + e.Key } return e.Key } -func extraDataEncode(key, desc string, encrypted bool) ([]byte, error) { +// storeKeyNoPrefix returns the kv store key without any encryption prefix, +// even if the leaf corresponds to a unvetted blob. +func (e *extraData) storeKeyNoPrefix() string { + return e.Key +} + +func extraDataEncode(key, desc string, state backend.StateT) ([]byte, error) { // The encryption prefix is stripped from the key if one exists. ed := extraData{ - Key: storeKeyClean(key), - Desc: desc, - Encrypted: encrypted, + Key: storeKeyClean(key), + Desc: desc, + State: state, } b, err := json.Marshal(ed) if err != nil { @@ -404,10 +423,22 @@ func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, re // plain text. dupBlobs = make(map[string]store.BlobEntry, len(digests)) - // Only vetted data should be saved unencrypted - encrypt = (recordMD.State != backend.StateVetted) + encrypt bool ) + // Only vetted data should be saved plain text + switch idx.State { + case backend.StateUnvetted: + encrypt = true + case backend.StateVetted: + // Save plain text + encrypt = false + default: + // Something is wrong + e := fmt.Sprintf("invalid record state %v %v", treeID, idx.State) + panic(e) + } + // Prepare record metadata blobs and leaves _, ok := dups[beRecordMD.Digest] if !ok { @@ -421,7 +452,7 @@ func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, re // Prepare tlog leaf extraData, err := extraDataEncode(k, - dataDescriptorRecordMetadata, encrypt) + dataDescriptorRecordMetadata, idx.State) if err != nil { return nil, err } @@ -451,7 +482,7 @@ func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, re // Prepare tlog leaf extraData, err := extraDataEncode(k, - dataDescriptorMetadataStream, encrypt) + dataDescriptorMetadataStream, idx.State) if err != nil { return nil, err } @@ -483,7 +514,7 @@ func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, re blobs[k] = b // Prepare tlog leaf - extraData, err := extraDataEncode(k, dataDescriptorFile, encrypt) + extraData, err := extraDataEncode(k, dataDescriptorFile, idx.State) if err != nil { return nil, err } @@ -506,8 +537,10 @@ func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, re return nil, backend.ErrNoRecordChanges } + log.Debugf("Saving %v record content blobs", len(blobs)) + // Save blobs to the kv store - err = t.store.PutKV(blobs) + err = t.store.Put(blobs) if err != nil { return nil, fmt.Errorf("store PutKV: %v", err) } @@ -531,7 +564,7 @@ func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, re // Check if any of the duplicates were saved as encrypted but now // need to be resaved as plain text. This happens when a record is // made public and the files need to be saved plain text. - if encrypt || len(dups) == 0 { + if idx.State == backend.StateUnvetted || len(dups) == 0 { // Nothing that needs to be saved plain text. We're done. log.Tracef("No blobs need to be resaved plain text") @@ -547,14 +580,14 @@ func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, re continue } - // This is a duplicate. If its encrypted it will need to be + // This is a duplicate. If its unvetted it will need to be // resaved as plain text. ed, err := extraDataDecode(v.ExtraData) if err != nil { return nil, err } - if !ed.Encrypted { - // Not encrypted + if ed.State == backend.StateVetted { + // Not unvetted. No need to resave it. continue } @@ -568,7 +601,7 @@ func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, re if err != nil { return nil, err } - blobs[ed.Key] = b + blobs[ed.storeKeyNoPrefix()] = b } if len(blobs) == 0 { // Nothing that needs to be saved plain text. We're done. @@ -577,13 +610,13 @@ func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, re return &idx, nil } - err = t.store.PutKV(blobs) + log.Debugf("Resaving %v encrypted blobs as plain text", len(blobs)) + + err = t.store.Put(blobs) if err != nil { return nil, fmt.Errorf("store PutKV: %v", err) } - log.Debugf("Resaved %v encrypted blobs as plain text", len(blobs)) - return &idx, nil } @@ -715,48 +748,6 @@ func (t *Tstore) RecordDel(treeID int64) error { return nil } -// RecordExists returns whether a record exists given a trillian tree ID. A -// record is considered to not exist if any of the following conditions are -// met: -// -// * A tree does not exist for the tree ID. -// -// * A tree exists but a record index does not exist. This can happen if a -// tree was created but there was an unexpected error prior to the record -// index being appended to the tree. -func (t *Tstore) RecordExists(treeID int64) bool { - log.Tracef("RecordExists: %v", treeID) - - // Verify tree exists - if !t.TreeExists(treeID) { - return false - } - - // Verify record index exists - leavesAll, err := t.tlog.leavesAll(treeID) - if err != nil { - err = fmt.Errorf("leavesAll: %v", err) - goto printErr - } - _, err = t.recordIndexLatest(leavesAll) - if err != nil { - if err == backend.ErrRecordNotFound { - // This is an empty tree. This can happen sometimes if a error - // occurred during record creation. Return gracefully. - return false - } - err = fmt.Errorf("recordIndexLatest: %v", err) - goto printErr - } - - // Record exists! - return true - -printErr: - log.Errorf("RecordExists: %v", err) - return false -} - // record returns the specified record. // // Version is used to request a specific version of a record. If no version is @@ -782,11 +773,7 @@ func (t *Tstore) record(treeID int64, version uint32, filenames []string, omitAl // Use the record index to pull the record content from the store. // The keys for the record content first need to be extracted from // their log leaf. - indexes, err := t.recordIndexes(leaves) - if err != nil { - return nil, err - } - idx, err := parseRecordIndex(indexes, version) + idx, err := t.recordIndex(leaves, version) if err != nil { return nil, err } @@ -841,7 +828,7 @@ func (t *Tstore) record(treeID int64, version uint32, filenames []string, omitAl // If the record is vetted the content may exist in the store // as both an encrypted blob and a plain text blob. Always pull // the plaintext blob. - key = ed.Key + key = ed.storeKeyNoPrefix() default: // Pull the encrypted blob key = ed.storeKey() From 2221acbbc3100366aaf0683ba08f84fe2fdf7b23 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 10 Mar 2021 19:50:25 -0600 Subject: [PATCH 381/449] tstore: Add RecordState method. --- .../backendv2/tstorebe/plugins/plugins.go | 3 ++ politeiad/backendv2/tstorebe/tstore/client.go | 17 ---------- politeiad/backendv2/tstorebe/tstore/tstore.go | 32 +++++++++++++++++++ 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/politeiad/backendv2/tstorebe/plugins/plugins.go b/politeiad/backendv2/tstorebe/plugins/plugins.go index e46d30d00..2902ea0ff 100644 --- a/politeiad/backendv2/tstorebe/plugins/plugins.go +++ b/politeiad/backendv2/tstorebe/plugins/plugins.go @@ -206,4 +206,7 @@ type TstoreClient interface { // record files. This supersedes the filenames argument. RecordPartial(treeID int64, version uint32, filenames []string, omitAllFiles bool) (*backend.Record, error) + + // RecordState returns whether the record is unvetted or vetted. + RecordState(treeID int64) (backend.StateT, error) } diff --git a/politeiad/backendv2/tstorebe/tstore/client.go b/politeiad/backendv2/tstorebe/tstore/client.go index f782167e4..3a196544d 100644 --- a/politeiad/backendv2/tstorebe/tstore/client.go +++ b/politeiad/backendv2/tstorebe/tstore/client.go @@ -403,23 +403,6 @@ func (t *Tstore) Timestamp(treeID int64, digest []byte) (*backend.Timestamp, err return t.timestamp(treeID, m, leaves) } -// recordIsVetted returns whether the provided leaves contain any vetted record -// indexes, which indicates whether the record is vetted. -func recordIsVetted(leaves []*trillian.LogLeaf) bool { - for _, v := range leaves { - ed, err := extraDataDecode(v.ExtraData) - if err != nil { - panic(err) - } - if ed.Desc == dataDescriptorRecordIndex && - ed.State == backend.StateVetted { - // Vetted record index found - return true - } - } - return false -} - func leavesForDescriptor(leaves []*trillian.LogLeaf, desc string) []*trillian.LogLeaf { // Determine if the record is vetted. If the record is vetted then // only vetted leaves will be returned. diff --git a/politeiad/backendv2/tstorebe/tstore/tstore.go b/politeiad/backendv2/tstorebe/tstore/tstore.go index f22e04d1f..0e149033d 100644 --- a/politeiad/backendv2/tstorebe/tstore/tstore.go +++ b/politeiad/backendv2/tstorebe/tstore/tstore.go @@ -961,6 +961,38 @@ func (t *Tstore) RecordPartial(treeID int64, version uint32, filenames []string, return t.record(treeID, version, filenames, omitAllFiles) } +// recordIsVetted returns whether the provided leaves contain any vetted record +// indexes. The presence of a vetted record index means the record is vetted. +func recordIsVetted(leaves []*trillian.LogLeaf) bool { + for _, v := range leaves { + ed, err := extraDataDecode(v.ExtraData) + if err != nil { + panic(err) + } + if ed.Desc == dataDescriptorRecordIndex && + ed.State == backend.StateVetted { + // Vetted record index found + return true + } + } + return false +} + +func (t *Tstore) RecordState(treeID int64) (backend.StateT, error) { + log.Tracef("RecordState: %v", treeID) + + leaves, err := t.tlog.leavesAll(treeID) + if err != nil { + return 0, err + } + + if recordIsVetted(leaves) { + return backend.StateVetted, nil + } + + return backend.StateUnvetted, nil +} + func (t *Tstore) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trillian.LogLeaf) (*backend.Timestamp, error) { // Find the leaf var l *trillian.LogLeaf From 12bb9c2628c8623e1f1edebe1317f4e4a07f7db8 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 10 Mar 2021 20:23:47 -0600 Subject: [PATCH 382/449] politeiad/comments: Support unvetted/vetted. --- .../tstorebe/plugins/comments/cmds.go | 136 +++++++++++++-- .../tstorebe/plugins/comments/recordindex.go | 33 ++-- politeiad/plugins/comments/comments.go | 155 +++++++++++------- 3 files changed, 235 insertions(+), 89 deletions(-) diff --git a/politeiad/backendv2/tstorebe/plugins/comments/cmds.go b/politeiad/backendv2/tstorebe/plugins/comments/cmds.go index 0cb838dd7..74f6f7f09 100644 --- a/politeiad/backendv2/tstorebe/plugins/comments/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/comments/cmds.go @@ -620,7 +620,8 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str } // Verify signature - msg := n.Token + strconv.FormatUint(uint64(n.ParentID), 10) + n.Comment + msg := strconv.FormatUint(uint64(n.State), 10) + n.Token + + strconv.FormatUint(uint64(n.ParentID), 10) + n.Comment err = util.VerifySignature(n.Signature, n.PublicKey, msg) if err != nil { return "", convertSignatureError(err) @@ -636,8 +637,22 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str } } + // Verify record state + state, err := p.tstore.RecordState(treeID) + if err != nil { + return "", err + } + if uint32(n.State) != uint32(state) { + e := fmt.Sprintf("got %v, want %v", n.State, state) + return "", backend.PluginError{ + PluginID: comments.PluginID, + ErrorCode: int(comments.ErrorCodeStateInvalid), + ErrorContext: e, + } + } + // Get record index - ridx, err := p.recordIndex(token) + ridx, err := p.recordIndex(token, state) if err != nil { return "", err } @@ -656,6 +671,7 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str receipt := p.identity.SignMessage([]byte(n.Signature)) ca := comments.CommentAdd{ UserID: n.UserID, + State: n.State, Token: n.Token, ParentID: n.ParentID, Comment: n.Comment, @@ -685,7 +701,7 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str } // Save index - err = p.recordIndexSave(token, *ridx) + err = p.recordIndexSave(token, state, *ridx) if err != nil { return "", err } @@ -741,7 +757,8 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st } // Verify signature - msg := e.Token + strconv.FormatUint(uint64(e.ParentID), 10) + e.Comment + msg := strconv.FormatUint(uint64(e.State), 10) + e.Token + + strconv.FormatUint(uint64(e.ParentID), 10) + e.Comment err = util.VerifySignature(e.Signature, e.PublicKey, msg) if err != nil { return "", convertSignatureError(err) @@ -757,8 +774,22 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st } } + // Verify record state + state, err := p.tstore.RecordState(treeID) + if err != nil { + return "", err + } + if uint32(e.State) != uint32(state) { + e := fmt.Sprintf("got %v, want %v", e.State, state) + return "", backend.PluginError{ + PluginID: comments.PluginID, + ErrorCode: int(comments.ErrorCodeStateInvalid), + ErrorContext: e, + } + } + // Get record index - ridx, err := p.recordIndex(token) + ridx, err := p.recordIndex(token, state) if err != nil { return "", err } @@ -807,6 +838,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st receipt := p.identity.SignMessage([]byte(e.Signature)) ca := comments.CommentAdd{ UserID: e.UserID, + State: e.State, Token: e.Token, ParentID: e.ParentID, Comment: e.Comment, @@ -830,7 +862,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st ridx.Comments[ca.CommentID].Adds[ca.Version] = digest // Save index - err = p.recordIndexSave(token, *ridx) + err = p.recordIndexSave(token, state, *ridx) if err != nil { return "", err } @@ -886,14 +918,29 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str } // Verify signature - msg := d.Token + strconv.FormatUint(uint64(d.CommentID), 10) + d.Reason + msg := strconv.FormatUint(uint64(d.State), 10) + d.Token + + strconv.FormatUint(uint64(d.CommentID), 10) + d.Reason err = util.VerifySignature(d.Signature, d.PublicKey, msg) if err != nil { return "", convertSignatureError(err) } + // Verify record state + state, err := p.tstore.RecordState(treeID) + if err != nil { + return "", err + } + if uint32(d.State) != uint32(state) { + e := fmt.Sprintf("got %v, want %v", d.State, state) + return "", backend.PluginError{ + PluginID: comments.PluginID, + ErrorCode: int(comments.ErrorCodeStateInvalid), + ErrorContext: e, + } + } + // Get record index - ridx, err := p.recordIndex(token) + ridx, err := p.recordIndex(token, state) if err != nil { return "", err } @@ -915,6 +962,7 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str receipt := p.identity.SignMessage([]byte(d.Signature)) cd := comments.CommentDel{ Token: d.Token, + State: d.State, CommentID: d.CommentID, Reason: d.Reason, PublicKey: d.PublicKey, @@ -942,7 +990,7 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str ridx.Comments[d.CommentID] = cidx // Save index - err = p.recordIndexSave(token, *ridx) + err = p.recordIndexSave(token, state, *ridx) if err != nil { return "", err } @@ -1016,15 +1064,30 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st } // Verify signature - msg := v.Token + strconv.FormatUint(uint64(v.CommentID), 10) + + msg := strconv.FormatUint(uint64(v.State), 10) + v.Token + + strconv.FormatUint(uint64(v.CommentID), 10) + strconv.FormatInt(int64(v.Vote), 10) err = util.VerifySignature(v.Signature, v.PublicKey, msg) if err != nil { return "", convertSignatureError(err) } + // Verify record state + state, err := p.tstore.RecordState(treeID) + if err != nil { + return "", err + } + if uint32(v.State) != uint32(state) { + e := fmt.Sprintf("got %v, want %v", v.State, state) + return "", backend.PluginError{ + PluginID: comments.PluginID, + ErrorCode: int(comments.ErrorCodeStateInvalid), + ErrorContext: e, + } + } + // Get record index - ridx, err := p.recordIndex(token) + ridx, err := p.recordIndex(token, state) if err != nil { return "", err } @@ -1071,6 +1134,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st receipt := p.identity.SignMessage([]byte(v.Signature)) cv := comments.CommentVote{ UserID: v.UserID, + State: v.State, Token: v.Token, CommentID: v.CommentID, Vote: v.Vote, @@ -1101,7 +1165,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st ridx.Comments[cv.CommentID] = cidx // Save index - err = p.recordIndexSave(token, *ridx) + err = p.recordIndexSave(token, state, *ridx) if err != nil { return "", err } @@ -1134,8 +1198,14 @@ func (p *commentsPlugin) cmdGet(treeID int64, token []byte, payload string) (str return "", err } + // Get record state + state, err := p.tstore.RecordState(treeID) + if err != nil { + return "", err + } + // Get record index - ridx, err := p.recordIndex(token) + ridx, err := p.recordIndex(token, state) if err != nil { return "", err } @@ -1161,8 +1231,14 @@ func (p *commentsPlugin) cmdGet(treeID int64, token []byte, payload string) (str func (p *commentsPlugin) cmdGetAll(treeID int64, token []byte) (string, error) { log.Tracef("cmdGetAll: %v %x", treeID, token) + // Get record state + state, err := p.tstore.RecordState(treeID) + if err != nil { + return "", err + } + // Compile comment IDs - ridx, err := p.recordIndex(token) + ridx, err := p.recordIndex(token, state) if err != nil { return "", err } @@ -1210,8 +1286,14 @@ func (p *commentsPlugin) cmdGetVersion(treeID int64, token []byte, payload strin return "", err } + // Get record state + state, err := p.tstore.RecordState(treeID) + if err != nil { + return "", err + } + // Get record index - ridx, err := p.recordIndex(token) + ridx, err := p.recordIndex(token, state) if err != nil { return "", err } @@ -1271,8 +1353,14 @@ func (p *commentsPlugin) cmdGetVersion(treeID int64, token []byte, payload strin func (p *commentsPlugin) cmdCount(treeID int64, token []byte) (string, error) { log.Tracef("cmdCount: %v %x", treeID, token) + // Get record state + state, err := p.tstore.RecordState(treeID) + if err != nil { + return "", err + } + // Get record index - ridx, err := p.recordIndex(token) + ridx, err := p.recordIndex(token, state) if err != nil { return "", err } @@ -1299,8 +1387,14 @@ func (p *commentsPlugin) cmdVotes(treeID int64, token []byte, payload string) (s return "", err } + // Get record state + state, err := p.tstore.RecordState(treeID) + if err != nil { + return "", err + } + // Get record index - ridx, err := p.recordIndex(token) + ridx, err := p.recordIndex(token, state) if err != nil { return "", err } @@ -1349,8 +1443,14 @@ func (p *commentsPlugin) cmdTimestamps(treeID int64, token []byte, payload strin return "", err } + // Get record state + state, err := p.tstore.RecordState(treeID) + if err != nil { + return "", err + } + // Get record index - ridx, err := p.recordIndex(token) + ridx, err := p.recordIndex(token, state) if err != nil { return "", err } diff --git a/politeiad/backendv2/tstorebe/plugins/comments/recordindex.go b/politeiad/backendv2/tstorebe/plugins/comments/recordindex.go index be48dfef6..f8d8d18bb 100644 --- a/politeiad/backendv2/tstorebe/plugins/comments/recordindex.go +++ b/politeiad/backendv2/tstorebe/plugins/comments/recordindex.go @@ -7,19 +7,22 @@ package comments import ( "encoding/json" "errors" + "fmt" "io/ioutil" "os" "path/filepath" "strings" + backend "github.com/decred/politeia/politeiad/backendv2" "github.com/decred/politeia/politeiad/plugins/comments" "github.com/decred/politeia/util" ) const ( - // filenameRecordIndex is the file name of the record index that - // is saved to the comments plugin data dir. - filenameRecordIndex = "{tokenPrefix}-index.json" + // Filenames of the record indexes that are saved to the comments + // plugin data dir. + fnRecordIndexUnvetted = "{tokenPrefix}-index-unvetted.json" + fnRecordIndexVetted = "{tokenPrefix}-index-vetted.json" ) // voteIndex contains the comment vote and the digest of the vote record. @@ -45,15 +48,25 @@ type commentIndex struct { // recordIndex contains the indexes for all comments made on a record. type recordIndex struct { - // TODO make unvetted and vetted indexes Comments map[uint32]commentIndex `json:"comments"` // [commentID]comment } // recordIndexPath accepts full length token or token prefixes, but always uses // prefix when generating the comments index path string. -func (p *commentsPlugin) recordIndexPath(token []byte) string { +func (p *commentsPlugin) recordIndexPath(token []byte, s backend.StateT) string { + var fn string + switch s { + case backend.StateUnvetted: + fn = fnRecordIndexUnvetted + case backend.StateVetted: + fn = fnRecordIndexVetted + default: + e := fmt.Sprintf("invalid state %x %v", token, s) + panic(e) + } + tp := util.TokenPrefix(token) - fn := strings.Replace(filenameRecordIndex, "{tokenPrefix}", tp, 1) + fn = strings.Replace(fn, "{tokenPrefix}", tp, 1) return filepath.Join(p.dataDir, fn) } @@ -61,11 +74,11 @@ func (p *commentsPlugin) recordIndexPath(token []byte) string { // cached recordIndex does not exist, a new one will be returned. // // This function must be called WITHOUT the read lock held. -func (p *commentsPlugin) recordIndex(token []byte) (*recordIndex, error) { +func (p *commentsPlugin) recordIndex(token []byte, s backend.StateT) (*recordIndex, error) { p.RLock() defer p.RUnlock() - fp := p.recordIndexPath(token) + fp := p.recordIndexPath(token, s) b, err := ioutil.ReadFile(fp) if err != nil { var e *os.PathError @@ -91,7 +104,7 @@ func (p *commentsPlugin) recordIndex(token []byte) (*recordIndex, error) { // dir. // // This function must be called WITHOUT the read/write lock held. -func (p *commentsPlugin) recordIndexSave(token []byte, ridx recordIndex) error { +func (p *commentsPlugin) recordIndexSave(token []byte, s backend.StateT, ridx recordIndex) error { p.Lock() defer p.Unlock() @@ -99,7 +112,7 @@ func (p *commentsPlugin) recordIndexSave(token []byte, ridx recordIndex) error { if err != nil { return err } - fp := p.recordIndexPath(token) + fp := p.recordIndexPath(token, s) err = ioutil.WriteFile(fp, b, 0664) if err != nil { return err diff --git a/politeiad/plugins/comments/comments.go b/politeiad/plugins/comments/comments.go index ddf169d71..379bd9679 100644 --- a/politeiad/plugins/comments/comments.go +++ b/politeiad/plugins/comments/comments.go @@ -94,6 +94,10 @@ const ( // times the user has changed their vote has exceeded the vote // changes max plugin setting. ErrorCodeVoteChangesMaxExceeded ErrorCodeT = 10 + + // ErrorCodeStateInvalid is returned when the provided state does + // not match the record state. + ErrorCodeStateInvalid ErrorCodeT = 11 ) var ( @@ -113,22 +117,44 @@ var ( } ) +// RecordStateT represents the state of a record. +type RecordStateT uint32 + +const ( + // RecordStateInvalid is an invalid record state. + RecordStateInvalid RecordStateT = 0 + + // RecordStateUnvetted indicates a record has not been made public. + RecordStateUnvetted RecordStateT = 1 + + // RecordStateVetted indicates a record has been made public. + RecordStateVetted RecordStateT = 2 +) + // Comment represent a record comment. // -// Signature is the client signature of Token+ParentID+Comment. +// A parent ID of 0 indicates that the comment is a base level comment and not +// a reply commment. +// +// Comments made on a record when it is unvetted and when it is vetted are +// treated as two distinct groups of comments. When a record becomes vetted the +// comment ID starts back at 1. +// +// Signature is the client signature of State+Token+ParentID+Comment. type Comment struct { - UserID string `json:"userid"` // Unique user ID - Token string `json:"token"` // Record token - ParentID uint32 `json:"parentid"` // Parent comment ID if reply - Comment string `json:"comment"` // Comment text - PublicKey string `json:"publickey"` // Public key used for Signature - Signature string `json:"signature"` // Client signature - CommentID uint32 `json:"commentid"` // Comment ID - Version uint32 `json:"version"` // Comment version - Timestamp int64 `json:"timestamp"` // UNIX timestamp of last edit - Receipt string `json:"receipt"` // Server signature of client signature - Downvotes uint64 `json:"downvotes"` // Tolal downvotes on comment - Upvotes uint64 `json:"upvotes"` // Total upvotes on comment + UserID string `json:"userid"` // Unique user ID + State RecordStateT `json:"state"` // Record state + Token string `json:"token"` // Record token + ParentID uint32 `json:"parentid"` // Parent comment ID if reply + Comment string `json:"comment"` // Comment text + PublicKey string `json:"publickey"` // Public key used for Signature + Signature string `json:"signature"` // Client signature + CommentID uint32 `json:"commentid"` // Comment ID + Version uint32 `json:"version"` // Comment version + Timestamp int64 `json:"timestamp"` // UNIX timestamp of last edit + Receipt string `json:"receipt"` // Server sig of client sig + Downvotes uint64 `json:"downvotes"` // Tolal downvotes on comment + Upvotes uint64 `json:"upvotes"` // Total upvotes on comment Deleted bool `json:"deleted,omitempty"` // Comment has been deleted Reason string `json:"reason,omitempty"` // Reason for deletion @@ -141,15 +167,16 @@ type Comment struct { // CommentAdd is the structure that is saved to disk when a comment is created // or edited. // -// Signature is the client signature of Token+ParentID+Comment. +// Signature is the client signature of State+Token+ParentID+Comment. type CommentAdd struct { // Data generated by client - UserID string `json:"userid"` // Unique user ID - Token string `json:"token"` // Record token - ParentID uint32 `json:"parentid"` // Parent comment ID - Comment string `json:"comment"` // Comment - PublicKey string `json:"publickey"` // Pubkey used for Signature - Signature string `json:"signature"` // Client signature + UserID string `json:"userid"` // Unique user ID + State RecordStateT `json:"state"` // Record state + Token string `json:"token"` // Record token + ParentID uint32 `json:"parentid"` // Parent comment ID + Comment string `json:"comment"` // Comment + PublicKey string `json:"publickey"` // Pubkey used for Signature + Signature string `json:"signature"` // Client signature // Metadata generated by server CommentID uint32 `json:"commentid"` // Comment ID @@ -168,14 +195,15 @@ type CommentAdd struct { // additional fields to properly display the deleted comment in the comment // hierarchy. // -// Signature is the client signature of the Token+CommentID+Reason +// Signature is the client signature of the State+Token+CommentID+Reason type CommentDel struct { // Data generated by client - Token string `json:"token"` // Record token - CommentID uint32 `json:"commentid"` // Comment ID - Reason string `json:"reason"` // Reason for deleting - PublicKey string `json:"publickey"` // Pubkey used for Signature - Signature string `json:"signature"` // Client signature + Token string `json:"token"` // Record token + State RecordStateT `json:"state"` // Record state + CommentID uint32 `json:"commentid"` // Comment ID + Reason string `json:"reason"` // Reason for deleting + PublicKey string `json:"publickey"` // Pubkey used for Signature + Signature string `json:"signature"` // Client signature // Metadata generated by server ParentID uint32 `json:"parentid"` // Parent comment ID @@ -201,15 +229,16 @@ const ( // CommentVote is the structure that is saved to disk when a comment is voted // on. // -// Signature is the client signature of the Token+CommentID+Vote. +// Signature is the client signature of the State+Token+CommentID+Vote. type CommentVote struct { // Data generated by client - UserID string `json:"userid"` // Unique user ID - Token string `json:"token"` // Record token - CommentID uint32 `json:"commentid"` // Comment ID - Vote VoteT `json:"vote"` // Upvote or downvote - PublicKey string `json:"publickey"` // Public key used for signature - Signature string `json:"signature"` // Client signature + UserID string `json:"userid"` // Unique user ID + State RecordStateT `json:"state"` // Record state + Token string `json:"token"` // Record token + CommentID uint32 `json:"commentid"` // Comment ID + Vote VoteT `json:"vote"` // Upvote or downvote + PublicKey string `json:"publickey"` // Public key used for signature + Signature string `json:"signature"` // Client signature // Metadata generated by server Timestamp int64 `json:"timestamp"` // Received UNIX timestamp @@ -221,14 +250,15 @@ type CommentVote struct { // The parent ID is used to reply to an existing comment. A parent ID of 0 // indicates that the comment is a base level comment and not a reply commment. // -// Signature is the client signature of Token+ParentID+Comment. +// Signature is the client signature of State+Token+ParentID+Comment. type New struct { - UserID string `json:"userid"` // Unique user ID - Token string `json:"token"` // Record token - ParentID uint32 `json:"parentid"` // Parent comment ID - Comment string `json:"comment"` // Comment text - PublicKey string `json:"publickey"` // Pubkey used for Signature - Signature string `json:"signature"` // Client signature + UserID string `json:"userid"` // Unique user ID + State RecordStateT `json:"state"` // Record state + Token string `json:"token"` // Record token + ParentID uint32 `json:"parentid"` // Parent comment ID + Comment string `json:"comment"` // Comment text + PublicKey string `json:"publickey"` // Pubkey used for Signature + Signature string `json:"signature"` // Client signature // Optional fields to be used freely ExtraData string `json:"extradata,omitempty"` @@ -242,15 +272,16 @@ type NewReply struct { // Edit edits an existing comment. // -// Signature is the client signature of Token+ParentID+Comment. +// Signature is the client signature of State+Token+ParentID+Comment. type Edit struct { - UserID string `json:"userid"` // Unique user ID - Token string `json:"token"` // Record token - ParentID uint32 `json:"parentid"` // Parent comment ID - CommentID uint32 `json:"commentid"` // Comment ID - Comment string `json:"comment"` // Comment text - PublicKey string `json:"publickey"` // Pubkey used for Signature - Signature string `json:"signature"` // Client signature + UserID string `json:"userid"` // Unique user ID + State RecordStateT `json:"state"` // Record state + Token string `json:"token"` // Record token + ParentID uint32 `json:"parentid"` // Parent comment ID + CommentID uint32 `json:"commentid"` // Comment ID + Comment string `json:"comment"` // Comment text + PublicKey string `json:"publickey"` // Pubkey used for Signature + Signature string `json:"signature"` // Client signature // Optional fields to be used freely ExtraData string `json:"extradata,omitempty"` @@ -264,13 +295,14 @@ type EditReply struct { // Del permanently deletes all versions of the provided comment. // -// Signature is the client signature of the Token+CommentID+Reason +// Signature is the client signature of the State+Token+CommentID+Reason type Del struct { - Token string `json:"token"` // Record token - CommentID uint32 `json:"commentid"` // Comment ID - Reason string `json:"reason"` // Reason for deletion - PublicKey string `json:"publickey"` // Public key used for signature - Signature string `json:"signature"` // Client signature + State RecordStateT `json:"state"` // Record state + Token string `json:"token"` // Record token + CommentID uint32 `json:"commentid"` // Comment ID + Reason string `json:"reason"` // Reason for deletion + PublicKey string `json:"publickey"` // Public key used for signature + Signature string `json:"signature"` // Client signature } // DelReply is the reply to the Del command. @@ -286,14 +318,15 @@ type DelReply struct { // original upvote. The public key cannot be relied on to remain the same for // each user so a user ID must be included. // -// Signature is the client signature of the Token+CommentID+Vote. +// Signature is the client signature of the State+Token+CommentID+Vote. type Vote struct { - UserID string `json:"userid"` // Unique user ID - Token string `json:"token"` // Record token - CommentID uint32 `json:"commentid"` // Comment ID - Vote VoteT `json:"vote"` // Upvote or downvote - PublicKey string `json:"publickey"` // Public key used for signature - Signature string `json:"signature"` // Client signature + UserID string `json:"userid"` // Unique user ID + State RecordStateT `json:"state"` // Record state + Token string `json:"token"` // Record token + CommentID uint32 `json:"commentid"` // Comment ID + Vote VoteT `json:"vote"` // Upvote or downvote + PublicKey string `json:"publickey"` // Public key used for signature + Signature string `json:"signature"` // Client signature } // VoteReply is the reply to the Vote command. From cb95179416d76fef8465f09093a59e2d0e0062a5 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 11 Mar 2021 07:13:42 -0600 Subject: [PATCH 383/449] politeiawww: Apply comment changes. --- politeiad/plugins/comments/comments.go | 1 + politeiawww/api/comments/v1/v1.go | 110 +++++++++++------- politeiawww/cmd/pictl/cmdcommentcensor.go | 5 +- politeiawww/cmd/pictl/cmdcommentcount.go | 21 ---- politeiawww/cmd/pictl/cmdcommentnew.go | 7 +- politeiawww/cmd/pictl/cmdcomments.go | 19 +-- politeiawww/cmd/pictl/cmdcommenttimestamps.go | 18 --- politeiawww/cmd/pictl/cmdcommentvote.go | 5 +- politeiawww/comments/events.go | 3 +- politeiawww/comments/process.go | 104 +++++++++-------- 10 files changed, 137 insertions(+), 156 deletions(-) diff --git a/politeiad/plugins/comments/comments.go b/politeiad/plugins/comments/comments.go index 379bd9679..7774e01e1 100644 --- a/politeiad/plugins/comments/comments.go +++ b/politeiad/plugins/comments/comments.go @@ -114,6 +114,7 @@ var ( ErrorCodeParentIDInvalid: "parent id invalid", ErrorCodeVoteInvalid: "vote invalid", ErrorCodeVoteChangesMaxExceeded: "vote changes max exceeded", + ErrorCodeStateInvalid: "record state invalid", } ) diff --git a/politeiawww/api/comments/v1/v1.go b/politeiawww/api/comments/v1/v1.go index aa0445c76..7cdcdbe13 100644 --- a/politeiawww/api/comments/v1/v1.go +++ b/politeiawww/api/comments/v1/v1.go @@ -101,22 +101,44 @@ type PolicyReply struct { VoteChangesMax uint32 `json:"votechangesmax"` } +// RecordStateT represents the state of a record. +type RecordStateT uint32 + +const ( + // RecordStateInvalid is an invalid record state. + RecordStateInvalid RecordStateT = 0 + + // RecordStateUnvetted indicates a record has not been made public. + RecordStateUnvetted RecordStateT = 1 + + // RecordStateVetted indicates a record has been made public. + RecordStateVetted RecordStateT = 2 +) + // Comment represent a record comment. // -// Signature is the client signature of Token+ParentID+Comment. +// A parent ID of 0 indicates that the comment is a base level comment and not +// a reply commment. +// +// Comments made on a record when it is unvetted and when it is vetted are +// treated as two distinct groups of comments. When a record becomes vetted the +// comment ID starts back at 1. +// +// Signature is the client signature of State+Token+ParentID+Comment. type Comment struct { - UserID string `json:"userid"` // Unique user ID - Username string `json:"username"` // Username - Token string `json:"token"` // Record token - ParentID uint32 `json:"parentid"` // Parent comment ID if reply - Comment string `json:"comment"` // Comment text - PublicKey string `json:"publickey"` // Public key used for Signature - Signature string `json:"signature"` // Client signature - CommentID uint32 `json:"commentid"` // Comment ID - Timestamp int64 `json:"timestamp"` // UNIX timestamp of last edit - Receipt string `json:"receipt"` // Server signature of client signature - Downvotes uint64 `json:"downvotes"` // Tolal downvotes on comment - Upvotes uint64 `json:"upvotes"` // Total upvotes on comment + UserID string `json:"userid"` // Unique user ID + Username string `json:"username"` // Username + State RecordStateT `json:"state"` // Record state + Token string `json:"token"` // Record token + ParentID uint32 `json:"parentid"` // Parent comment ID if reply + Comment string `json:"comment"` // Comment text + PublicKey string `json:"publickey"` // Public key used for Signature + Signature string `json:"signature"` // Client signature + CommentID uint32 `json:"commentid"` // Comment ID + Timestamp int64 `json:"timestamp"` // UNIX timestamp of last edit + Receipt string `json:"receipt"` // Server sig of client sig + Downvotes uint64 `json:"downvotes"` // Tolal downvotes on comment + Upvotes uint64 `json:"upvotes"` // Total upvotes on comment Deleted bool `json:"deleted,omitempty"` // Comment has been deleted Reason string `json:"reason,omitempty"` // Reason for deletion @@ -128,17 +150,18 @@ type Comment struct { // CommentVote represents a comment vote (upvote/downvote). // -// Signature is the client signature of the Token+CommentID+Vote. +// Signature is the client signature of the State+Token+CommentID+Vote. type CommentVote struct { - UserID string `json:"userid"` // Unique user ID - Username string `json:"username"` // Username - Token string `json:"token"` // Record token - CommentID uint32 `json:"commentid"` // Comment ID - Vote VoteT `json:"vote"` // Upvote or downvote - PublicKey string `json:"publickey"` // Public key used for signature - Signature string `json:"signature"` // Client signature - Timestamp int64 `json:"timestamp"` // Received UNIX timestamp - Receipt string `json:"receipt"` // Server signature of client signature + UserID string `json:"userid"` // Unique user ID + Username string `json:"username"` // Username + State RecordStateT `json:"state"` // Record state + Token string `json:"token"` // Record token + CommentID uint32 `json:"commentid"` // Comment ID + Vote VoteT `json:"vote"` // Upvote or downvote + PublicKey string `json:"publickey"` // Public key used for signature + Signature string `json:"signature"` // Client signature + Timestamp int64 `json:"timestamp"` // Received UNIX timestamp + Receipt string `json:"receipt"` // Server sig of client sig } // New creates a new comment. @@ -146,13 +169,14 @@ type CommentVote struct { // The parent ID is used to reply to an existing comment. A parent ID of 0 // indicates that the comment is a base level comment and not a reply commment. // -// Signature is the client signature of Token+ParentID+Comment. +// Signature is the client signature of State+Token+ParentID+Comment. type New struct { - Token string `json:"token"` - ParentID uint32 `json:"parentid"` - Comment string `json:"comment"` - PublicKey string `json:"publickey"` - Signature string `json:"signature"` + State RecordStateT `json:"state"` + Token string `json:"token"` + ParentID uint32 `json:"parentid"` + Comment string `json:"comment"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` // Optional fields to be used freely ExtraData string `json:"extradata,omitempty"` @@ -186,13 +210,14 @@ const ( // upvoted, the resulting vote score is 0 due to the second upvote removing the // original upvote. // -// Signature is the client signature of the Token+CommentID+Vote. +// Signature is the client signature of the State+Token+CommentID+Vote. type Vote struct { - Token string `json:"token"` - CommentID uint32 `json:"commentid"` - Vote VoteT `json:"vote"` - PublicKey string `json:"publickey"` - Signature string `json:"signature"` + State RecordStateT `json:"state"` + Token string `json:"token"` + CommentID uint32 `json:"commentid"` + Vote VoteT `json:"vote"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` } // VoteReply is the reply to the Vote command. @@ -200,19 +225,20 @@ type VoteReply struct { Downvotes uint64 `json:"downvotes"` // Tolal downvotes on comment Upvotes uint64 `json:"upvotes"` // Total upvotes on comment Timestamp int64 `json:"timestamp"` // Received UNIX timestamp - Receipt string `json:"receipt"` // Server signature of client signature + Receipt string `json:"receipt"` // Server sig of client sig } // Del permanently deletes the provided comment. Only admins can delete // comments. A reason must be given for the deletion. // -// Signature is the client signature of the Token+CommentID+Reason +// Signature is the client signature of the State+Token+CommentID+Reason type Del struct { - Token string `json:"token"` - CommentID uint32 `json:"commentid"` - Reason string `json:"reason"` - PublicKey string `json:"publickey"` - Signature string `json:"signature"` + State RecordStateT `json:"state"` + Token string `json:"token"` + CommentID uint32 `json:"commentid"` + Reason string `json:"reason"` + PublicKey string `json:"publickey"` + Signature string `json:"signature"` } // DelReply is the reply to the Del command. diff --git a/politeiawww/cmd/pictl/cmdcommentcensor.go b/politeiawww/cmd/pictl/cmdcommentcensor.go index 208738d28..408db554d 100644 --- a/politeiawww/cmd/pictl/cmdcommentcensor.go +++ b/politeiawww/cmd/pictl/cmdcommentcensor.go @@ -46,7 +46,7 @@ func (c *cmdCommentCensor) Execute(args []string) error { } // Setup state - var state string + var state cmv1.RecordStateT switch { case c.Unvetted: state = cmv1.RecordStateUnvetted @@ -68,7 +68,8 @@ func (c *cmdCommentCensor) Execute(args []string) error { } // Setup request - msg := token + strconv.FormatUint(uint64(commentID), 10) + reason + msg := strconv.FormatUint(uint64(state), 10) + token + + strconv.FormatUint(uint64(commentID), 10) + reason sig := cfg.Identity.SignMessage([]byte(msg)) d := cmv1.Del{ State: state, diff --git a/politeiawww/cmd/pictl/cmdcommentcount.go b/politeiawww/cmd/pictl/cmdcommentcount.go index b1c4ae5fe..11b4a5a44 100644 --- a/politeiawww/cmd/pictl/cmdcommentcount.go +++ b/politeiawww/cmd/pictl/cmdcommentcount.go @@ -14,11 +14,6 @@ type cmdCommentCount struct { Args struct { Tokens []string `positional-arg-name:"tokens"` } `positional-args:"true" required:"true"` - - // Unvetted is used to request the comment counts of unvetted - // records. If this flag is not used the command assumes the - // records are vetted. - Unvetted bool `long:"unvetted" optional:"true"` } // Execute executes the cmdCommentCount command. @@ -49,18 +44,8 @@ func commentCount(c *cmdCommentCount) (map[string]uint32, error) { return nil, err } - // Setup state - var state string - switch { - case c.Unvetted: - state = cmv1.RecordStateUnvetted - default: - state = cmv1.RecordStateVetted - } - // Get comments cc := cmv1.Count{ - State: state, Tokens: c.Args.Tokens, } cr, err := pc.CommentCount(cc) @@ -82,15 +67,9 @@ const commentCountHelpMsg = `commentcount "tokens..." Get the number of comments that have been made on each of the provided records. -If the record is unvetted, the --unvetted flag must be used. This command -accepts both full length tokens or the token prefixes. - Arguments: 1. token (string, required) Proposal censorship token -Flags: - --unvetted (bool, optional) Record is unvetted. - Examples: $ pictl commentcount f6458c2d8d9ef41c 9f9af91cf609d839 917c6fde9bcc2118 $ pictl commentcount f6458c2 9f9af91 917c6fd` diff --git a/politeiawww/cmd/pictl/cmdcommentnew.go b/politeiawww/cmd/pictl/cmdcommentnew.go index 14bdef4a4..83a3fcb34 100644 --- a/politeiawww/cmd/pictl/cmdcommentnew.go +++ b/politeiawww/cmd/pictl/cmdcommentnew.go @@ -59,7 +59,7 @@ func (c *cmdCommentNew) Execute(args []string) error { } // Setup state - var state string + var state cmv1.RecordStateT switch { case c.Unvetted: state = cmv1.RecordStateUnvetted @@ -68,11 +68,12 @@ func (c *cmdCommentNew) Execute(args []string) error { } // Setup request - msg := token + strconv.FormatUint(uint64(parentID), 10) + comment + msg := strconv.FormatUint(uint64(state), 10) + token + + strconv.FormatUint(uint64(parentID), 10) + comment sig := cfg.Identity.SignMessage([]byte(msg)) n := cmv1.New{ - Token: token, State: state, + Token: token, ParentID: parentID, Comment: comment, Signature: hex.EncodeToString(sig[:]), diff --git a/politeiawww/cmd/pictl/cmdcomments.go b/politeiawww/cmd/pictl/cmdcomments.go index 026128a53..a2c1b618c 100644 --- a/politeiawww/cmd/pictl/cmdcomments.go +++ b/politeiawww/cmd/pictl/cmdcomments.go @@ -16,11 +16,6 @@ type cmdComments struct { Args struct { Token string `positional-arg-name:"token"` // Censorship token } `positional-args:"true" required:"true"` - - // Unvetted is used to request the comments of an unvetted record. - // If this flag is not used the command assumes the record is - // vetted. - Unvetted bool `long:"unvetted" optional:"true"` } // Execute executes the cmdComments command. @@ -40,18 +35,8 @@ func (c *cmdComments) Execute(args []string) error { return err } - // Setup state - var state string - switch { - case c.Unvetted: - state = cmv1.RecordStateUnvetted - default: - state = cmv1.RecordStateVetted - } - // Get comments cm := cmv1.Comments{ - State: state, Token: c.Args.Token, } cr, err := pc.Comments(cm) @@ -79,6 +64,4 @@ record author. Arguments: 1. token (string, required) Proposal censorship token - -Flags: - --unvetted (bool, optional) Record is unvetted.` +` diff --git a/politeiawww/cmd/pictl/cmdcommenttimestamps.go b/politeiawww/cmd/pictl/cmdcommenttimestamps.go index c58a47905..f5671b573 100644 --- a/politeiawww/cmd/pictl/cmdcommenttimestamps.go +++ b/politeiawww/cmd/pictl/cmdcommenttimestamps.go @@ -19,11 +19,6 @@ type cmdCommentTimestamps struct { Token string `positional-arg-name:"token" required:"true"` CommentIDs []uint32 `positional-arg-name:"commentids" optional:"true"` } `positional-args:"true"` - - // Unvetted is used to request the timestamps of an unvetted - // record. If this flag is not used the command assumes the record - // is vetted. - Unvetted bool `long:"unvetted" optional:"true"` } // Execute executes the cmdCommentTimestamps command. @@ -43,18 +38,8 @@ func (c *cmdCommentTimestamps) Execute(args []string) error { return err } - // Setup state - var state string - switch { - case c.Unvetted: - state = cmv1.RecordStateUnvetted - default: - state = cmv1.RecordStateVetted - } - // Get timestamps t := cmv1.Timestamps{ - State: state, Token: c.Args.Token, CommentIDs: c.Args.CommentIDs, } @@ -120,9 +105,6 @@ Arguments: 1. token (string, required) Proposal token 2. commentIDs ([]uint32, optional) Proposal version -Flags: - --unvetted (bool, optional) Record is unvetted. - Example: Fetch all record comment timestamps $ pictl commenttimestamps 0a265dd93e9bae6d diff --git a/politeiawww/cmd/pictl/cmdcommentvote.go b/politeiawww/cmd/pictl/cmdcommentvote.go index b2de528e1..39dcd6787 100644 --- a/politeiawww/cmd/pictl/cmdcommentvote.go +++ b/politeiawww/cmd/pictl/cmdcommentvote.go @@ -62,10 +62,13 @@ func (c *cmdCommentVote) Execute(args []string) error { } // Setup request - msg := c.Args.Token + strconv.FormatUint(uint64(c.Args.CommentID), 10) + + state := cmv1.RecordStateVetted + msg := strconv.FormatUint(uint64(state), 10) + c.Args.Token + + strconv.FormatUint(uint64(c.Args.CommentID), 10) + strconv.FormatInt(int64(vote), 10) sig := cfg.Identity.SignMessage([]byte(msg)) v := cmv1.Vote{ + State: state, Token: c.Args.Token, CommentID: c.Args.CommentID, Vote: vote, diff --git a/politeiawww/comments/events.go b/politeiawww/comments/events.go index bcd46e2c6..401fae1dc 100644 --- a/politeiawww/comments/events.go +++ b/politeiawww/comments/events.go @@ -5,7 +5,6 @@ package comments import ( - pdv2 "github.com/decred/politeia/politeiad/api/v2" v1 "github.com/decred/politeia/politeiawww/api/comments/v1" ) @@ -16,6 +15,6 @@ const ( // EventNew is the event data for the EventTypeNew. type EventNew struct { - State pdv2.RecordStateT + State v1.RecordStateT Comment v1.Comment } diff --git a/politeiawww/comments/process.go b/politeiawww/comments/process.go index 1b275e0aa..c9be44eef 100644 --- a/politeiawww/comments/process.go +++ b/politeiawww/comments/process.go @@ -16,20 +16,17 @@ import ( "github.com/google/uuid" ) -// NOTE: the comment commands enforce different user permissions depending on -// the state of the record (ex. only admins and the author are allowed to -// comment on unvetted records). We currently pull the record without any files -// in order to determine the record state. This is the quick and dirty way and -// was implemented like this due to development time constraints. We could -// eliminate this network request by providing the plugin command with the -// record assumptions that are being made and allow the plugin to verify these -// assumptions during its validation and return an error if they do not hold. -// This would require the politeiawww client provide the record state along -// with all comment requests. - func (c *Comments) processNew(ctx context.Context, n v1.New, u user.User) (*v1.NewReply, error) { log.Tracef("processNew: %v %v %v", n.Token, u.Username) + // Verify state + state := convertStateToPlugin(n.State) + if state == comments.RecordStateInvalid { + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeRecordStateInvalid, + } + } + // Verify user signed using active identity if u.PublicKey() != n.PublicKey { return nil, v1.UserErrorReply{ @@ -50,16 +47,7 @@ func (c *Comments) processNew(ctx context.Context, n v1.New, u user.User) (*v1.N // Only admins and the record author are allowed to comment on // unvetted records. - r, err := c.recordNoFiles(ctx, n.Token) - if err != nil { - if err == errRecordNotFound { - return nil, v1.UserErrorReply{ - ErrorCode: v1.ErrorCodeRecordNotFound, - } - } - return nil, err - } - if r.State == pdv2.RecordStateUnvetted && !u.Admin { + if n.State == v1.RecordStateUnvetted && !u.Admin { // User is not an admin. Check if the user is the author. authorID, err := c.politeiad.Author(ctx, n.Token) if err != nil { @@ -76,6 +64,7 @@ func (c *Comments) processNew(ctx context.Context, n v1.New, u user.User) (*v1.N // Send plugin command cn := comments.New{ UserID: u.ID.String(), + State: state, Token: n.Token, ParentID: n.ParentID, Comment: n.Comment, @@ -94,7 +83,7 @@ func (c *Comments) processNew(ctx context.Context, n v1.New, u user.User) (*v1.N // Emit event c.events.Emit(EventTypeNew, EventNew{ - State: r.State, + State: n.State, Comment: cm, }) @@ -106,6 +95,14 @@ func (c *Comments) processNew(ctx context.Context, n v1.New, u user.User) (*v1.N func (c *Comments) processVote(ctx context.Context, v v1.Vote, u user.User) (*v1.VoteReply, error) { log.Tracef("processVote: %v %v %v", v.Token, v.CommentID, v.Vote) + // Verify state + state := convertStateToPlugin(v.State) + if state == comments.RecordStateInvalid { + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeRecordStateInvalid, + } + } + // Verify user signed using active identity if u.PublicKey() != v.PublicKey { return nil, v1.UserErrorReply{ @@ -125,16 +122,7 @@ func (c *Comments) processVote(ctx context.Context, v v1.Vote, u user.User) (*v1 } // Votes are only allowed on vetted records - r, err := c.recordNoFiles(ctx, v.Token) - if err != nil { - if err == errRecordNotFound { - return nil, v1.UserErrorReply{ - ErrorCode: v1.ErrorCodeRecordNotFound, - } - } - return nil, err - } - if r.State != pdv2.RecordStateVetted { + if v.State != v1.RecordStateVetted { return nil, v1.UserErrorReply{ ErrorCode: v1.ErrorCodeRecordStateInvalid, ErrorContext: "comment voting is only allowed on vetted records", @@ -144,6 +132,7 @@ func (c *Comments) processVote(ctx context.Context, v v1.Vote, u user.User) (*v1 // Send plugin command cv := comments.Vote{ UserID: u.ID.String(), + State: state, Token: v.Token, CommentID: v.CommentID, Vote: comments.VoteT(v.Vote), @@ -166,6 +155,14 @@ func (c *Comments) processVote(ctx context.Context, v v1.Vote, u user.User) (*v1 func (c *Comments) processDel(ctx context.Context, d v1.Del, u user.User) (*v1.DelReply, error) { log.Tracef("processDel: %v %v %v", d.Token, d.CommentID, d.Reason) + // Verify state + state := convertStateToPlugin(d.State) + if state == comments.RecordStateInvalid { + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeRecordStateInvalid, + } + } + // Verify user signed with their active identity if u.PublicKey() != d.PublicKey { return nil, v1.UserErrorReply{ @@ -176,6 +173,7 @@ func (c *Comments) processDel(ctx context.Context, d v1.Del, u user.User) (*v1.D // Send plugin command cd := comments.Del{ + State: state, Token: d.Token, CommentID: d.CommentID, Reason: d.Reason, @@ -218,19 +216,21 @@ func (c *Comments) processCount(ctx context.Context, ct v1.Count) (*v1.CountRepl func (c *Comments) processComments(ctx context.Context, cs v1.Comments, u *user.User) (*v1.CommentsReply, error) { log.Tracef("processComments: %v", cs.Token) - // Only admins and the record author are allowed to retrieve - // unvetted comments. This is a public route so a user might - // not exist. - r, err := c.recordNoFiles(ctx, cs.Token) + // Send plugin command + pcomments, err := c.politeiad.CommentsGetAll(ctx, cs.Token) if err != nil { - if err == errRecordNotFound { - return nil, v1.UserErrorReply{ - ErrorCode: v1.ErrorCodeRecordNotFound, - } - } return nil, err } - if r.State == pdv2.RecordStateUnvetted { + if len(pcomments) == 0 { + return &v1.CommentsReply{ + Comments: []v1.Comment{}, + }, nil + } + + // Only admins and the record author are allowed to retrieve + // unvetted comments. This is a public route so a user might + // not exist. + if pcomments[0].State == comments.RecordStateUnvetted { var isAllowed bool switch { case u == nil: @@ -258,12 +258,6 @@ func (c *Comments) processComments(ctx context.Context, cs v1.Comments, u *user. } } - // Send plugin command - pcomments, err := c.politeiad.CommentsGetAll(ctx, cs.Token) - if err != nil { - return nil, err - } - // Prepare reply. Comment user data must be pulled from the // userdb. comments := make([]v1.Comment, 0, len(pcomments)) @@ -293,7 +287,9 @@ func (c *Comments) processComments(ctx context.Context, cs v1.Comments, u *user. func (c *Comments) processVotes(ctx context.Context, v v1.Votes) (*v1.VotesReply, error) { log.Tracef("processVotes: %v %v", v.Token, v.UserID) - // Get comment votes + // Get comment votes. Votes are only allowed on vetted comments so + // there is no need to check the user permissions since all vetted + // comments are public. cm := comments.Votes{ UserID: v.UserID, } @@ -401,6 +397,16 @@ func commentVotePopulateUserData(votes []v1.CommentVote, u user.User) { } } +func convertStateToPlugin(s v1.RecordStateT) comments.RecordStateT { + switch s { + case v1.RecordStateUnvetted: + return comments.RecordStateUnvetted + case v1.RecordStateVetted: + return comments.RecordStateVetted + } + return comments.RecordStateInvalid +} + func convertComment(c comments.Comment) v1.Comment { // Fields that are intentionally omitted are not stored in // politeiad. They need to be pulled from the userdb. From 751401be85c9ecdd717a38a6c4c0cc4700344efc Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 11 Mar 2021 11:04:24 -0600 Subject: [PATCH 384/449] tstorebe: Allow token prefixes on reads. --- politeiad/backendv2/tstorebe/tstorebe.go | 29 +++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/politeiad/backendv2/tstorebe/tstorebe.go b/politeiad/backendv2/tstorebe/tstorebe.go index 33bfec39c..eb4874b68 100644 --- a/politeiad/backendv2/tstorebe/tstorebe.go +++ b/politeiad/backendv2/tstorebe/tstorebe.go @@ -94,6 +94,18 @@ func (t *tstoreBackend) prefixAdd(fullToken []byte) { log.Debugf("Add token prefix: %v", prefix) } +func (t *tstoreBackend) fullLengthToken(token []byte) []byte { + t.RLock() + defer t.RUnlock() + + fullToken, ok := t.prefixes[util.TokenPrefix(token)] + if !ok { + return token + } + + return fullToken +} + func tokenFromTreeID(treeID int64) []byte { b := make([]byte, 8) // Converting between int64 and uint64 doesn't change @@ -908,6 +920,9 @@ func (t *tstoreBackend) RecordSetStatus(token []byte, status backend.StatusT, md func (t *tstoreBackend) RecordExists(token []byte) bool { log.Tracef("RecordExists: %x", token) + // Read methods are allowed to use token prefixes + token = t.fullLengthToken(token) + treeID := treeIDFromToken(token) return t.tstore.TreeExists(treeID) } @@ -919,6 +934,9 @@ func (t *tstoreBackend) RecordExists(token []byte) bool { func (t *tstoreBackend) RecordGet(token []byte, version uint32) (*backend.Record, error) { log.Tracef("RecordGet: %x", token) + // Read methods are allowed to use token prefixes + token = t.fullLengthToken(token) + treeID := treeIDFromToken(token) return t.tstore.Record(treeID, version) } @@ -933,6 +951,9 @@ func (t *tstoreBackend) RecordGetBatch(reqs []backend.RecordRequest) (map[string records := make(map[string]backend.Record, len(reqs)) for _, v := range reqs { + // Read methods are allowed to use token prefixes + v.Token = t.fullLengthToken(v.Token) + treeID := treeIDFromToken(v.Token) r, err := t.tstore.RecordPartial(treeID, v.Version, v.Filenames, v.OmitAllFiles) @@ -961,6 +982,9 @@ func (t *tstoreBackend) RecordGetBatch(reqs []backend.RecordRequest) (map[string func (t *tstoreBackend) RecordTimestamps(token []byte, version uint32) (*backend.RecordTimestamps, error) { log.Tracef("RecordTimestamps: %x %v", token, version) + // Read methods are allowed to use token prefixes + token = t.fullLengthToken(token) + treeID := treeIDFromToken(token) return t.tstore.RecordTimestamps(treeID, version, token) } @@ -1028,12 +1052,15 @@ func (t *tstoreBackend) PluginRead(token []byte, pluginID, pluginCmd, payload st // will not be provided to the plugin. var treeID int64 if len(token) > 0 { - treeID = treeIDFromToken(token) + // Read methods are allowed to use token prefixes + token = t.fullLengthToken(token) // Verify record exists if !t.RecordExists(token) { return "", backend.ErrRecordNotFound } + + treeID = treeIDFromToken(token) } if len(token) > 0 { From c47543853f7058d9311ae0455e2abcc0fb8b5433 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 11 Mar 2021 14:11:34 -0600 Subject: [PATCH 385/449] tstore: Move encryption to store. --- go.mod | 4 +- go.sum | 8 ++ .../tstorebe/store/localdb/encrypt.go | 41 ++++++ .../tstorebe/store/localdb/localdb.go | 57 ++++++++- .../backendv2/tstorebe/store/mysql/encrypt.go | 88 +++++++++++++ .../backendv2/tstorebe/store/mysql/mysql.go | 103 ++++++++++++--- politeiad/backendv2/tstorebe/store/store.go | 12 +- politeiad/backendv2/tstorebe/tstore/anchor.go | 4 +- politeiad/backendv2/tstorebe/tstore/client.go | 8 +- .../tstorebe/tstore/encryptionkey.go | 56 --------- .../backendv2/tstorebe/tstore/recordindex.go | 7 +- .../backendv2/tstorebe/tstore/testing.go | 15 +-- politeiad/backendv2/tstorebe/tstore/tstore.go | 117 +++--------------- util/convert.go | 11 -- util/encrypt.go | 69 +++++++++++ 15 files changed, 384 insertions(+), 216 deletions(-) create mode 100644 politeiad/backendv2/tstorebe/store/localdb/encrypt.go create mode 100644 politeiad/backendv2/tstorebe/store/mysql/encrypt.go delete mode 100644 politeiad/backendv2/tstorebe/tstore/encryptionkey.go create mode 100644 util/encrypt.go diff --git a/go.mod b/go.mod index fe0c4e49a..40d648dcf 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/jessevdk/go-flags v1.4.1-0.20200711081900-c17162fe8fd7 github.com/jinzhu/gorm v1.9.12 github.com/jrick/logrotate v1.0.0 - github.com/marcopeereboom/sbox v1.0.0 + github.com/marcopeereboom/sbox v1.1.0 github.com/otiai10/copy v1.0.1 github.com/otiai10/curr v0.0.0-20190513014714-f5a3d24e5776 // indirect github.com/pmezard/go-difflib v1.0.0 @@ -51,7 +51,7 @@ require ( github.com/robfig/cron v1.2.0 github.com/subosito/gozaru v0.0.0-20190625071150-416082cce636 github.com/syndtr/goleveldb v1.0.0 - golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 + golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 golang.org/x/net v0.0.0-20200625001655-4c5254603344 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 diff --git a/go.sum b/go.sum index bc696a2b2..e6c36b91e 100644 --- a/go.sum +++ b/go.sum @@ -596,6 +596,8 @@ github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0Q github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/marcopeereboom/sbox v1.0.0 h1:1xTRUzI5mVvsaPaBGVpXdyH2hZeZhCWazbGy6MVsRgg= github.com/marcopeereboom/sbox v1.0.0/go.mod h1:V9e7t7oKphNfXymk7Lqvbo9mZiVjmCt8vBHnROcpCSY= +github.com/marcopeereboom/sbox v1.1.0 h1:IiVHCi5f+nGRiMX551wnDk5ce+IEd3dWVH7ycf2uU2M= +github.com/marcopeereboom/sbox v1.1.0/go.mod h1:u2fh4EbQDXQXXzGypWkf2nMn2TnsqA23t224mii7oog= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -813,6 +815,8 @@ golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200214034016-1d94cc7ab1c6/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -923,6 +927,7 @@ golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -944,6 +949,9 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e h1:AyodaIpKjppX+cBfTASF2E1US3H2JFBj920Ot3rtDjs= golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/politeiad/backendv2/tstorebe/store/localdb/encrypt.go b/politeiad/backendv2/tstorebe/store/localdb/encrypt.go new file mode 100644 index 000000000..cc2a73905 --- /dev/null +++ b/politeiad/backendv2/tstorebe/store/localdb/encrypt.go @@ -0,0 +1,41 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package localdb + +import ( + "bytes" + + "github.com/decred/politeia/util" + "github.com/marcopeereboom/sbox" +) + +func (l *localdb) encrypt(data []byte) ([]byte, error) { + l.keyMtx.RLock() + defer l.keyMtx.RUnlock() + + return sbox.Encrypt(0, l.key, data) +} + +func (l *localdb) decrypt(data []byte) ([]byte, uint32, error) { + l.keyMtx.RLock() + defer l.keyMtx.RUnlock() + + return sbox.Decrypt(l.key, data) +} + +func (l *localdb) zeroKey() { + l.keyMtx.Lock() + defer l.keyMtx.Unlock() + + util.Zero(l.key[:]) + l.key = nil +} + +// isEncrypted returns whether the provided blob has been prefixed with an sbox +// header, indicating that it is an encrypted blob. +func isEncrypted(b []byte) bool { + isEncrypted := bytes.HasPrefix(b, []byte("sbox")) + return isEncrypted +} diff --git a/politeiad/backendv2/tstorebe/store/localdb/localdb.go b/politeiad/backendv2/tstorebe/store/localdb/localdb.go index 0f8efc7bf..6696940d6 100644 --- a/politeiad/backendv2/tstorebe/store/localdb/localdb.go +++ b/politeiad/backendv2/tstorebe/store/localdb/localdb.go @@ -7,9 +7,11 @@ package localdb import ( "errors" "fmt" + "path/filepath" "sync" "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" + "github.com/decred/politeia/util" "github.com/syndtr/goleveldb/leveldb" ) @@ -21,17 +23,33 @@ var ( type localdb struct { sync.Mutex shutdown bool - root string // Location of database db *leveldb.DB + + // Encryption key and mutex. The key is zero'd out on application + // exit so the read lock must be held during concurrent access to + // prevent the golang race detector from complaining. + key *[32]byte + keyMtx sync.RWMutex } // Put saves the provided key-value pairs to the store. This operation is // performed atomically. // // This function satisfies the store BlobKV interface. -func (l *localdb) Put(blobs map[string][]byte) error { +func (l *localdb) Put(blobs map[string][]byte, encrypt bool) error { log.Tracef("Put: %v blobs", len(blobs)) + // Encrypt blobs + if encrypt { + for k, v := range blobs { + e, err := l.encrypt(v) + if err != nil { + return fmt.Errorf("encrypt: %v", err) + } + blobs[k] = e + } + } + // Setup batch batch := new(leveldb.Batch) for k, v := range blobs { @@ -79,6 +97,7 @@ func (l *localdb) Del(keys []string) error { func (l *localdb) Get(keys []string) (map[string][]byte, error) { log.Tracef("Get: %v", keys) + // Lookup blobs blobs := make(map[string][]byte, len(keys)) for _, v := range keys { b, err := l.db.Get([]byte(v), nil) @@ -92,6 +111,20 @@ func (l *localdb) Get(keys []string) (map[string][]byte, error) { blobs[v] = b } + // Decrypt blobs + for k, v := range blobs { + encrypted := isEncrypted(v) + log.Tracef("Blob is encrypted: %v", encrypted) + if !encrypted { + continue + } + b, _, err := l.decrypt(v) + if err != nil { + return nil, fmt.Errorf("decrypt: %v", err) + } + blobs[k] = b + } + return blobs, nil } @@ -102,18 +135,30 @@ func (l *localdb) Close() { l.Lock() defer l.Unlock() + l.zeroKey() l.db.Close() } // New returns a new localdb. -func New(root string) (*localdb, error) { - db, err := leveldb.OpenFile(root, nil) +func New(appDir, dataDir, keyFile string) (*localdb, error) { + // Load encryption key + if keyFile == "" { + // No file path was given. Use the default path. + keyFile = filepath.Join(appDir, store.DefaultEncryptionKeyFilename) + } + key, err := util.LoadEncryptionKey(log, keyFile) + if err != nil { + return nil, err + } + + // Open database + db, err := leveldb.OpenFile(dataDir, nil) if err != nil { return nil, err } return &localdb{ - db: db, - root: root, + db: db, + key: key, }, nil } diff --git a/politeiad/backendv2/tstorebe/store/mysql/encrypt.go b/politeiad/backendv2/tstorebe/store/mysql/encrypt.go new file mode 100644 index 000000000..bcd49daee --- /dev/null +++ b/politeiad/backendv2/tstorebe/store/mysql/encrypt.go @@ -0,0 +1,88 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package mysql + +import ( + "bytes" + "context" + "database/sql" + "encoding/binary" + "fmt" + + "github.com/decred/politeia/util" + "github.com/marcopeereboom/sbox" +) + +func (s *mysql) encrypt(ctx context.Context, tx *sql.Tx, data []byte) ([]byte, error) { + // Create a new nonce value + _, err := tx.ExecContext(ctx, "INSERT INTO nonce () VALUES ();") + if err != nil { + return nil, err + } + + // Get the nonce value that was just created + rows, err := tx.QueryContext(ctx, "SELECT LAST_INSERT_ID();") + if err != nil { + return nil, fmt.Errorf("query: %v", err) + } + defer rows.Close() + + var i int64 + for rows.Next() { + err = rows.Scan(&i) + if err != nil { + return nil, fmt.Errorf("scan: %v", err) + } + + // There should only ever be one value to scan + break + } + err = rows.Err() + if err != nil { + return nil, fmt.Errorf("next: %v", err) + } + if i == 0 { + return nil, fmt.Errorf("invalid 0 nonce") + } + + log.Tracef("Encrypting with nonce: %v", i) + + // Prepare nonce + b := make([]byte, 8) + binary.LittleEndian.PutUint64(b, uint64(i)) + n, err := sbox.NewNonceFromBytes(b) + if err != nil { + return nil, err + } + nonce := n.Current() + + // Encrypt blob + s.keyMtx.RLock() + defer s.keyMtx.RUnlock() + + return sbox.EncryptN(0, s.key, nonce, data) +} + +func (s *mysql) decrypt(data []byte) ([]byte, uint32, error) { + s.keyMtx.RLock() + defer s.keyMtx.RUnlock() + + return sbox.Decrypt(s.key, data) +} + +func (s *mysql) zeroKey() { + s.keyMtx.Lock() + defer s.keyMtx.Unlock() + + util.Zero(s.key[:]) + s.key = nil +} + +// isEncrypted returns whether the provided blob has been prefixed with an sbox +// header, indicating that it is an encrypted blob. +func isEncrypted(b []byte) bool { + isEncrypted := bytes.HasPrefix(b, []byte("sbox")) + return isEncrypted +} diff --git a/politeiad/backendv2/tstorebe/store/mysql/mysql.go b/politeiad/backendv2/tstorebe/store/mysql/mysql.go index 6619cf7be..4d047aef7 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/mysql.go +++ b/politeiad/backendv2/tstorebe/store/mysql/mysql.go @@ -8,9 +8,12 @@ import ( "context" "database/sql" "fmt" + "path/filepath" + "sync" "time" "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" + "github.com/decred/politeia/util" _ "github.com/go-sql-driver/mysql" ) @@ -21,6 +24,10 @@ const ( connMaxLifetime = 1 * time.Minute maxOpenConns = 0 // 0 is unlimited maxIdleConns = 100 + + // Database table names + tableNameKeyValue = "kv" + tableNameNonce = "nonce" ) // tableKeyValue defines the key-value table. The key is a uuid. @@ -29,6 +36,11 @@ const tableKeyValue = ` v LONGBLOB NOT NULL ` +// tableNonce defines the table used to track the encryption nonce. +const tableNonce = ` + n BIGINT PRIMARY KEY AUTO_INCREMENT +` + var ( _ store.BlobKV = (*mysql)(nil) ) @@ -36,17 +48,47 @@ var ( // mysql implements the store BlobKV interface using a mysql driver. type mysql struct { db *sql.DB + + // Encryption key and mutex. The key is zero'd out on application + // exit so the read lock must be held during concurrent access to + // prevent the golang race detector from complaining. + key *[32]byte + keyMtx sync.RWMutex } func ctxWithTimeout() (context.Context, func()) { return context.WithTimeout(context.Background(), connTimeout) } +func (s *mysql) put(blobs map[string][]byte, encrypt bool, ctx context.Context, tx *sql.Tx) error { + // Encrypt blobs + if encrypt { + for k, v := range blobs { + e, err := s.encrypt(ctx, tx, v) + if err != nil { + return fmt.Errorf("encrypt: %v", err) + } + blobs[k] = e + } + } + + // Save blobs + for k, v := range blobs { + _, err := tx.ExecContext(ctx, + "INSERT INTO kv (k, v) VALUES (?, ?);", k, v) + if err != nil { + return fmt.Errorf("exec put: %v", err) + } + } + + return nil +} + // Put saves the provided key-value pairs to the store. This operation is // performed atomically. // // This function satisfies the store BlobKV interface. -func (s *mysql) Put(blobs map[string][]byte) error { +func (s *mysql) Put(blobs map[string][]byte, encrypt bool) error { log.Tracef("Put: %v blobs", len(blobs)) ctx, cancel := ctxWithTimeout() @@ -62,15 +104,13 @@ func (s *mysql) Put(blobs map[string][]byte) error { } // Save blobs - for k, v := range blobs { - _, err = tx.ExecContext(ctx, "INSERT INTO kv (k, v) VALUES (?, ?);", k, v) - if err != nil { - // Attempt to roll back the transaction - if err2 := tx.Rollback(); err2 != nil { - // We're in trouble! - e := fmt.Sprintf("put: %v, unable to rollback: %v", err, err2) - panic(e) - } + err = s.put(blobs, encrypt, ctx, tx) + if err != nil { + // Attempt to roll back the transaction + if err2 := tx.Rollback(); err2 != nil { + // We're in trouble! + e := fmt.Sprintf("put: %v, unable to rollback: %v", err, err2) + panic(e) } } @@ -185,15 +225,40 @@ func (s *mysql) Get(keys []string) (map[string][]byte, error) { return nil, fmt.Errorf("next: %v", err) } + // Decrypt data blobs + for k, v := range reply { + encrypted := isEncrypted(v) + log.Tracef("Blob is encrypted: %v", encrypted) + if !encrypted { + continue + } + b, _, err := s.decrypt(v) + if err != nil { + return nil, fmt.Errorf("decrypt: %v", err) + } + reply[k] = b + } + return reply, nil } // Closes closes the blob store connection. func (s *mysql) Close() { + s.zeroKey() s.db.Close() } -func New(host, user, password, dbname string) (*mysql, error) { +func New(appDir, host, user, password, dbname, keyFile string) (*mysql, error) { + // Load encryption key + if keyFile == "" { + // No file path was given. Use the default path. + keyFile = filepath.Join(appDir, store.DefaultEncryptionKeyFilename) + } + key, err := util.LoadEncryptionKey(log, keyFile) + if err != nil { + return nil, err + } + // Connect to database log.Infof("Host: %v:[password]@tcp(%v)/%v", user, host, dbname) @@ -215,13 +280,23 @@ func New(host, user, password, dbname string) (*mysql, error) { } // Setup key-value table - q := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS kv (%s)`, tableKeyValue) + q := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %v (%v)`, + tableNameKeyValue, tableKeyValue) + _, err = db.Exec(q) + if err != nil { + return nil, fmt.Errorf("create kv table: %v", err) + } + + // Setup nonce table + q = fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %v (%v)`, + tableNameNonce, tableNonce) _, err = db.Exec(q) if err != nil { - return nil, fmt.Errorf("create table: %v", err) + return nil, fmt.Errorf("create nonce table: %v", err) } return &mysql{ - db: db, + db: db, + key: key, }, nil } diff --git a/politeiad/backendv2/tstorebe/store/store.go b/politeiad/backendv2/tstorebe/store/store.go index 8c2156aaa..beec9a351 100644 --- a/politeiad/backendv2/tstorebe/store/store.go +++ b/politeiad/backendv2/tstorebe/store/store.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Decred developers +// Copyright (c) 2020-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -15,7 +15,13 @@ import ( ) const ( - // Data descriptor types + // DefaultEncryptionKeyFilename is the filename of the encryption + // key that is created in the store data directory if an encryption + // key file is not provided on startup. + DefaultEncryptionKeyFilename = "sbox.key" + + // DataTypeStructure is used as the data descriptor type when the + // blob entry contains a structure. DataTypeStructure = "struct" ) @@ -79,7 +85,7 @@ func Deblob(blob []byte) (*BlobEntry, error) { type BlobKV interface { // Put saves the provided key-value pairs to the store. This // operation is performed atomically. - Put(blobs map[string][]byte) error + Put(blobs map[string][]byte, encrypt bool) error // Del deletes the provided blobs from the store. This operation // is performed atomically. diff --git a/politeiad/backendv2/tstorebe/tstore/anchor.go b/politeiad/backendv2/tstorebe/tstore/anchor.go index 7ffe88c50..7cb604c50 100644 --- a/politeiad/backendv2/tstorebe/tstore/anchor.go +++ b/politeiad/backendv2/tstorebe/tstore/anchor.go @@ -217,13 +217,13 @@ func (t *Tstore) anchorSave(a anchor) error { if err != nil { return err } - b, err := t.blobify(*be, false) + b, err := store.Blobify(*be) if err != nil { return err } key := storeKeyNew(false) kv := map[string][]byte{key: b} - err = t.store.Put(kv) + err = t.store.Put(kv, false) if err != nil { return fmt.Errorf("store Put: %v", err) } diff --git a/politeiad/backendv2/tstorebe/tstore/client.go b/politeiad/backendv2/tstorebe/tstore/client.go index 3a196544d..93a0f37e2 100644 --- a/politeiad/backendv2/tstorebe/tstore/client.go +++ b/politeiad/backendv2/tstorebe/tstore/client.go @@ -75,7 +75,7 @@ func (t *Tstore) BlobSave(treeID int64, be store.BlobEntry) error { if err != nil { return err } - blob, err := t.blobify(be, encrypt) + blob, err := store.Blobify(be) if err != nil { return err } @@ -85,7 +85,7 @@ func (t *Tstore) BlobSave(treeID int64, be store.BlobEntry) error { log.Debugf("Saving plugin data blob") // Save blob to store - err = t.store.Put(kv) + err = t.store.Put(kv, encrypt) if err != nil { return fmt.Errorf("store Put: %v", err) } @@ -240,7 +240,7 @@ func (t *Tstore) Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEnt // Blob wasn't found in the store. Skip it. continue } - be, err := t.deblob(b) + be, err := store.Deblob(b) if err != nil { return nil, err } @@ -314,7 +314,7 @@ func (t *Tstore) BlobsByDataDesc(treeID int64, dataDesc string) ([]store.BlobEnt if !ok { return nil, fmt.Errorf("blob not found: %v", v) } - be, err := t.deblob(b) + be, err := store.Deblob(b) if err != nil { return nil, err } diff --git a/politeiad/backendv2/tstorebe/tstore/encryptionkey.go b/politeiad/backendv2/tstorebe/tstore/encryptionkey.go deleted file mode 100644 index b6d316ec7..000000000 --- a/politeiad/backendv2/tstorebe/tstore/encryptionkey.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright (c) 2020 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package tstore - -import ( - "sync" - - "github.com/decred/politeia/util" - "github.com/marcopeereboom/sbox" -) - -// encryptionKey provides an API for encrypting and decrypting data. The -// encryption key is zero'd out on application exit so the lock must be held -// anytime the key is accessed in order to prevent the golang race detector -// from complaining. -type encryptionKey struct { - sync.RWMutex - key *[32]byte -} - -// encrypt encrypts the provided data. It prefixes the encrypted blob with an -// sbox header which encodes the provided version. The version is user provided -// and can be used as a hint to identify or version the packed blob. Version is -// not inspected or used by Encrypt and Decrypt. -func (e *encryptionKey) encrypt(version uint32, blob []byte) ([]byte, error) { - e.RLock() - defer e.RUnlock() - - return sbox.Encrypt(version, e.key, blob) -} - -// decrypt decrypts the provided packed blob. The decrypted blob and the -// version that was used to encrypt the blob are returned. -func (e *encryptionKey) decrypt(blob []byte) ([]byte, uint32, error) { - e.RLock() - defer e.RUnlock() - - return sbox.Decrypt(e.key, blob) -} - -// Zero zeroes out the encryption key. -func (e *encryptionKey) zero() { - e.Lock() - defer e.Unlock() - - util.Zero(e.key[:]) - e.key = nil -} - -func newEncryptionKey(key *[32]byte) *encryptionKey { - return &encryptionKey{ - key: key, - } -} diff --git a/politeiad/backendv2/tstorebe/tstore/recordindex.go b/politeiad/backendv2/tstorebe/tstore/recordindex.go index fdbed441d..b0663d412 100644 --- a/politeiad/backendv2/tstorebe/tstore/recordindex.go +++ b/politeiad/backendv2/tstorebe/tstore/recordindex.go @@ -10,6 +10,7 @@ import ( "sort" backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" "github.com/google/trillian" "google.golang.org/grpc/codes" ) @@ -95,13 +96,13 @@ func (t *Tstore) recordIndexSave(treeID int64, idx recordIndex) error { if err != nil { return err } - b, err := t.blobify(*be, encrypt) + b, err := store.Blobify(*be) if err != nil { return err } key := storeKeyNew(encrypt) kv := map[string][]byte{key: b} - err = t.store.Put(kv) + err = t.store.Put(kv, encrypt) if err != nil { return fmt.Errorf("store Put: %v", err) } @@ -200,7 +201,7 @@ func (t *Tstore) recordIndexes(leaves []*trillian.LogLeaf) ([]recordIndex, error vetted = make([]recordIndex, 0, len(blobs)) ) for _, v := range blobs { - be, err := t.deblob(v) + be, err := store.Deblob(v) if err != nil { return nil, err } diff --git a/politeiad/backendv2/tstorebe/tstore/testing.go b/politeiad/backendv2/tstorebe/tstore/testing.go index e06d944eb..67fb5ae60 100644 --- a/politeiad/backendv2/tstorebe/tstore/testing.go +++ b/politeiad/backendv2/tstorebe/tstore/testing.go @@ -9,7 +9,6 @@ import ( "testing" "github.com/decred/politeia/politeiad/backendv2/tstorebe/store/localdb" - "github.com/marcopeereboom/sbox" ) func NewTestTstore(t *testing.T, dataDir string) *Tstore { @@ -26,21 +25,13 @@ func NewTestTstore(t *testing.T, dataDir string) *Tstore { if err != nil { t.Fatal(err) } - store, err := localdb.New(fp) + store, err := localdb.New(dataDir, fp, "") if err != nil { t.Fatal(err) } - // Setup encryptin key if specified - key, err := sbox.NewKey() - if err != nil { - t.Fatal(err) - } - ek := newEncryptionKey(key) - return &Tstore{ - encryptionKey: ek, - tlog: newTestTClient(t), - store: store, + tlog: newTestTClient(t), + store: store, } } diff --git a/politeiad/backendv2/tstorebe/tstore/tstore.go b/politeiad/backendv2/tstorebe/tstore/tstore.go index 0e149033d..036fd5553 100644 --- a/politeiad/backendv2/tstorebe/tstore/tstore.go +++ b/politeiad/backendv2/tstorebe/tstore/tstore.go @@ -11,7 +11,6 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" "net/url" "os" "path/filepath" @@ -26,7 +25,6 @@ import ( "github.com/decred/politeia/util" "github.com/google/trillian" "github.com/google/uuid" - "github.com/marcopeereboom/sbox" "github.com/robfig/cron" "google.golang.org/grpc/codes" ) @@ -36,7 +34,6 @@ const ( DBTypeMySQL = "mysql" dbUser = "politeiad" - defaultEncryptionKeyFilename = "tstore-sbox.key" defaultTrillianSigningKeyFilename = "trillian.key" defaultStoreDirname = "store" @@ -71,25 +68,12 @@ type Tstore struct { cron *cron.Cron plugins map[string]plugin // [pluginID]plugin - // encryptionKey is used to encrypt record blobs before saving them - // to the key-value store. This is an optional param. Record blobs - // will not be encrypted if this is left as nil. - encryptionKey *encryptionKey - // droppingAnchor indicates whether tstore is in the process of // dropping an anchor, i.e. timestamping unanchored trillian trees // using dcrtime. An anchor is dropped periodically using cron. droppingAnchor bool } -// blobIsEncrypted returns whether the provided blob has been prefixed with an -// sbox header, indicating that it is an encrypted blob. -func blobIsEncrypted(b []byte) bool { - isEncrypted := bytes.HasPrefix(b, []byte("sbox")) - log.Tracef("Blob is encrypted: %v", isEncrypted) - return isEncrypted -} - // extraData is the data that is stored in the log leaf ExtraData field. It is // saved as a JSON encoded byte slice. The JSON keys have been abbreviated to // minimize the size of a trillian log leaf. @@ -177,35 +161,6 @@ func merkleLeafHashForBlobEntry(be store.BlobEntry) ([]byte, error) { return merkleLeafHash(leafValue), nil } -func (t *Tstore) blobify(be store.BlobEntry, encrypt bool) ([]byte, error) { - b, err := store.Blobify(be) - if err != nil { - return nil, err - } - if encrypt { - b, err = t.encryptionKey.encrypt(0, b) - if err != nil { - return nil, err - } - } - return b, nil -} - -func (t *Tstore) deblob(b []byte) (*store.BlobEntry, error) { - var err error - if blobIsEncrypted(b) { - b, _, err = t.encryptionKey.decrypt(b) - if err != nil { - return nil, err - } - } - be, err := store.Deblob(b) - if err != nil { - return nil, err - } - return be, nil -} - func (t *Tstore) TreeNew() (int64, error) { log.Tracef("TreeNew") @@ -422,11 +377,10 @@ func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, re // the duplicate content is encrypted and it needs to be saved // plain text. dupBlobs = make(map[string]store.BlobEntry, len(digests)) - - encrypt bool ) // Only vetted data should be saved plain text + var encrypt bool switch idx.State { case backend.StateUnvetted: encrypt = true @@ -443,7 +397,7 @@ func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, re _, ok := dups[beRecordMD.Digest] if !ok { // Not a duplicate. Prepare kv store blob. - b, err := t.blobify(*beRecordMD, encrypt) + b, err := store.Blobify(*beRecordMD) if err != nil { return nil, err } @@ -473,7 +427,7 @@ func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, re _, ok := dups[be.Digest] if !ok { // Not a duplicate. Prepare kv store blob. - b, err := t.blobify(be, encrypt) + b, err := store.Blobify(be) if err != nil { return nil, err } @@ -506,7 +460,7 @@ func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, re _, ok := dups[be.Digest] if !ok { // Not a duplicate. Prepare kv store blob. - b, err := t.blobify(be, encrypt) + b, err := store.Blobify(be) if err != nil { return nil, err } @@ -540,9 +494,9 @@ func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, re log.Debugf("Saving %v record content blobs", len(blobs)) // Save blobs to the kv store - err = t.store.Put(blobs) + err = t.store.Put(blobs, encrypt) if err != nil { - return nil, fmt.Errorf("store PutKV: %v", err) + return nil, fmt.Errorf("store Put: %v", err) } // Append leaves onto the trillian tree @@ -597,7 +551,7 @@ func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, re // Should not happen return nil, fmt.Errorf("blob entry not found %v", d) } - b, err := t.blobify(be, false) + b, err := store.Blobify(be) if err != nil { return nil, err } @@ -612,9 +566,9 @@ func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, re log.Debugf("Resaving %v encrypted blobs as plain text", len(blobs)) - err = t.store.Put(blobs) + err = t.store.Put(blobs, false) if err != nil { - return nil, fmt.Errorf("store PutKV: %v", err) + return nil, fmt.Errorf("store Put: %v", err) } return &idx, nil @@ -852,7 +806,7 @@ func (t *Tstore) record(treeID int64, version uint32, filenames []string, omitAl // Decode blobs entries := make([]store.BlobEntry, 0, len(keys)) for _, v := range blobs { - be, err := t.deblob(v) + be, err := store.Deblob(v) if err != nil { return nil, err } @@ -1025,7 +979,7 @@ func (t *Tstore) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trilli if !ok { return nil, fmt.Errorf("blob not found %v", ed.storeKey()) } - be, err := t.deblob(b) + be, err := store.Deblob(b) if err != nil { return nil, err } @@ -1203,52 +1157,9 @@ func (t *Tstore) Close() { // Close connections t.store.Close() t.tlog.close() - - // Zero out encryption key. An encryption key is optional. - if t.encryptionKey != nil { - t.encryptionKey.zero() - } } func New(appDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianSigningKeyFile, dbType, dbHost, dbPass, dbEncryptionKeyFile, dcrtimeHost, dcrtimeCert string) (*Tstore, error) { - // Setup encryption key file - if dbEncryptionKeyFile == "" { - // No file path was given. Use the default path. - dbEncryptionKeyFile = filepath.Join(appDir, defaultEncryptionKeyFilename) - } - if !util.FileExists(dbEncryptionKeyFile) { - // Encryption key file does not exist. Create one. - log.Infof("Generating encryption key") - key, err := sbox.NewKey() - if err != nil { - return nil, err - } - err = ioutil.WriteFile(dbEncryptionKeyFile, key[:], 0400) - if err != nil { - return nil, err - } - util.Zero(key[:]) - log.Infof("Encryption key created: %v", dbEncryptionKeyFile) - } - - // Load encryption key - f, err := os.Open(dbEncryptionKeyFile) - if err != nil { - return nil, err - } - var key [32]byte - n, err := f.Read(key[:]) - if n != len(key) { - return nil, fmt.Errorf("invalid encryption key length") - } - if err != nil { - return nil, err - } - f.Close() - ek := newEncryptionKey(&key) - - log.Infof("Encryption key: %v", dbEncryptionKeyFile) - // Setup trillian client if trillianSigningKeyFile == "" { // No file path was given. Use the default path. @@ -1281,14 +1192,15 @@ func New(appDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianSig if err != nil { return nil, err } - kvstore, err = localdb.New(fp) + kvstore, err = localdb.New(appDir, fp, dbEncryptionKeyFile) if err != nil { return nil, err } case DBTypeMySQL: // Example db name: testnet3_unvetted_kv dbName := fmt.Sprintf("%v_kv", anp.Name) - kvstore, err = mysql.New(dbHost, dbUser, dbPass, dbName) + kvstore, err = mysql.New(appDir, dbHost, dbUser, dbPass, + dbName, dbEncryptionKeyFile) if err != nil { return nil, err } @@ -1318,7 +1230,6 @@ func New(appDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianSig dcrtime: dcrtimeClient, cron: cron.New(), plugins: make(map[string]plugin), - encryptionKey: ek, } // Launch cron diff --git a/util/convert.go b/util/convert.go index a5592e568..fd123a86f 100644 --- a/util/convert.go +++ b/util/convert.go @@ -86,17 +86,6 @@ func ConvertDigest(d string) ([sha256.Size]byte, bool) { return digest, true } -// Zero out a byte slice. -func Zero(in []byte) { - if in == nil { - return - } - inlen := len(in) - for i := 0; i < inlen; i++ { - in[i] ^= in[i] - } -} - // TokenToPrefix returns a substring a token of length pd.TokenPrefixLength, // or the token itself, whichever is shorter. func TokenToPrefix(token string) string { diff --git a/util/encrypt.go b/util/encrypt.go new file mode 100644 index 000000000..f21dd9534 --- /dev/null +++ b/util/encrypt.go @@ -0,0 +1,69 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package util + +import ( + "fmt" + "io/ioutil" + "os" + + "github.com/decred/slog" + "github.com/marcopeereboom/sbox" +) + +// Zero zeros out a byte slice. +func Zero(in []byte) { + if in == nil { + return + } + inlen := len(in) + for i := 0; i < inlen; i++ { + in[i] ^= in[i] + } +} + +// LoadEncryptionKey loads the encryption key at the provided file path. If a +// key does not exists at the file path then a new secretbox key is created +// and saved to the file path before returning the key. +func LoadEncryptionKey(log slog.Logger, keyFile string) (*[32]byte, error) { + if keyFile == "" { + return nil, fmt.Errorf("no key file provided") + } + + // Setup encryption key file + if !FileExists(keyFile) { + // Encryption key file does not exist. Create one. + log.Infof("Generating encryption key") + key, err := sbox.NewKey() + if err != nil { + return nil, err + } + err = ioutil.WriteFile(keyFile, key[:], 0400) + if err != nil { + return nil, err + } + Zero(key[:]) + log.Infof("Encryption key created: %v", keyFile) + } + + // Load encryption key + f, err := os.Open(keyFile) + if err != nil { + return nil, err + } + var key [32]byte + n, err := f.Read(key[:]) + if n != len(key) { + return nil, fmt.Errorf("invalid encryption key length") + } + if err != nil { + return nil, err + } + f.Close() + + log.Infof("Encryption key: %v", keyFile) + + return &key, nil +} From 43d74abfd89952da3049c21aeffd9dac42f96107 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 11 Mar 2021 14:18:31 -0600 Subject: [PATCH 386/449] typo --- politeiad/backendv2/tstorebe/store/mysql/mysql.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/politeiad/backendv2/tstorebe/store/mysql/mysql.go b/politeiad/backendv2/tstorebe/store/mysql/mysql.go index 4d047aef7..c6c2e800f 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/mysql.go +++ b/politeiad/backendv2/tstorebe/store/mysql/mysql.go @@ -38,7 +38,7 @@ const tableKeyValue = ` // tableNonce defines the table used to track the encryption nonce. const tableNonce = ` - n BIGINT PRIMARY KEY AUTO_INCREMENT + n BIGINT PRIMARY KEY AUTO_INCREMENT ` var ( From cd0dc5ca7ceef772b6dd56b44333179142a3efa3 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 11 Mar 2021 14:23:50 -0600 Subject: [PATCH 387/449] cleanup. --- go.sum | 1 + politeiad/backendv2/tstorebe/store/localdb/encrypt.go | 3 +-- politeiad/backendv2/tstorebe/store/mysql/encrypt.go | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/go.sum b/go.sum index e6c36b91e..2fa6772e5 100644 --- a/go.sum +++ b/go.sum @@ -951,6 +951,7 @@ golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e h1:AyodaIpKjppX+cBfTASF2E1US golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/politeiad/backendv2/tstorebe/store/localdb/encrypt.go b/politeiad/backendv2/tstorebe/store/localdb/encrypt.go index cc2a73905..d3a7927b1 100644 --- a/politeiad/backendv2/tstorebe/store/localdb/encrypt.go +++ b/politeiad/backendv2/tstorebe/store/localdb/encrypt.go @@ -36,6 +36,5 @@ func (l *localdb) zeroKey() { // isEncrypted returns whether the provided blob has been prefixed with an sbox // header, indicating that it is an encrypted blob. func isEncrypted(b []byte) bool { - isEncrypted := bytes.HasPrefix(b, []byte("sbox")) - return isEncrypted + return bytes.HasPrefix(b, []byte("sbox")) } diff --git a/politeiad/backendv2/tstorebe/store/mysql/encrypt.go b/politeiad/backendv2/tstorebe/store/mysql/encrypt.go index bcd49daee..ddbb7133d 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/encrypt.go +++ b/politeiad/backendv2/tstorebe/store/mysql/encrypt.go @@ -83,6 +83,5 @@ func (s *mysql) zeroKey() { // isEncrypted returns whether the provided blob has been prefixed with an sbox // header, indicating that it is an encrypted blob. func isEncrypted(b []byte) bool { - isEncrypted := bytes.HasPrefix(b, []byte("sbox")) - return isEncrypted + return bytes.HasPrefix(b, []byte("sbox")) } From ce6ef4d2130d04e5b142544b1c713a869149bea3 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 11 Mar 2021 16:00:48 -0600 Subject: [PATCH 388/449] Enable TOTP routes. --- politeiad/README.md | 16 ++++++++-------- .../backendv2/tstorebe/store/mysql/mysql.go | 2 +- politeiawww/userwww.go | 6 ++++++ 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/politeiad/README.md b/politeiad/README.md index af3500a2d..ef38ca2fd 100644 --- a/politeiad/README.md +++ b/politeiad/README.md @@ -111,7 +111,7 @@ politeiad ./mysql-tstore-setup.sh ``` -4. Run the trillian mysql setup scripts. +6. Run the trillian mysql setup scripts. These can only be run once the trillian MySQL user has been created in the previous step. @@ -142,7 +142,7 @@ politeiad ``` -5. Start up the trillian instances. +7. Start up the trillian instances. Running trillian requires running a trillian log server and a trillian log signer. These are seperate processes that will be started in this step. @@ -153,7 +153,7 @@ politeiad run one set of commands, testnet or mainnet. Run the testnet commands if you're setting up a development environment. - Startup testnet log server + Start testnet log server ``` $ export MYSQL_USER=trillian && \ export MYSQL_PASSWORD=trillianpass && \ @@ -168,7 +168,7 @@ politeiad --logtostderr ... ``` - Startup testnet log signer + Start testnet log signer ``` $ export MYSQL_USER=trillian && \ export MYSQL_PASSWORD=trillianpass && \ @@ -184,7 +184,7 @@ politeiad --http_endpoint=localhost:8093 ``` - Startup mainnet log server + Start mainnet log server ``` $ export MYSQL_USER=trillian && \ export MYSQL_PASSWORD=trillianpass && \ @@ -199,7 +199,7 @@ politeiad --logtostderr ... ``` - Startup mainnet log signer + Start mainnet log signer ``` $ export MYSQL_USER=trillian && \ export MYSQL_PASSWORD=trillianpass && \ @@ -216,7 +216,7 @@ politeiad ``` -5. Setup the politeiad configuration file. +8. Setup the politeiad configuration file. [`sample-politeiad.conf`](https://github.com/decred/politeia/blob/master/politeiad/sample-politeiad.conf) @@ -264,7 +264,7 @@ politeiad plugin=usermd ``` -6. Start up the politeiad instance. +9. Start up the politeiad instance. ``` $ politeiad diff --git a/politeiad/backendv2/tstorebe/store/mysql/mysql.go b/politeiad/backendv2/tstorebe/store/mysql/mysql.go index c6c2e800f..be1aedf7e 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/mysql.go +++ b/politeiad/backendv2/tstorebe/store/mysql/mysql.go @@ -30,7 +30,7 @@ const ( tableNameNonce = "nonce" ) -// tableKeyValue defines the key-value table. The key is a uuid. +// tableKeyValue defines the key-value table. const tableKeyValue = ` k VARCHAR(38) NOT NULL PRIMARY KEY, v LONGBLOB NOT NULL diff --git a/politeiawww/userwww.go b/politeiawww/userwww.go index 5c22023c0..f8eb13e4c 100644 --- a/politeiawww/userwww.go +++ b/politeiawww/userwww.go @@ -850,6 +850,12 @@ func (p *politeiawww) setUserWWWRoutes() { p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, www.RouteUserProposalCredits, p.handleUserProposalCredits, permissionLogin) + p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, + www.RouteSetTOTP, p.handleSetTOTP, + permissionLogin) + p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, + www.RouteVerifyTOTP, p.handleVerifyTOTP, + permissionLogin) // Routes that require being logged in as an admin user. p.addRoute(http.MethodPut, www.PoliteiaWWWAPIRoute, From 51df7b69df38bc403c80091f094b429cc36551f7 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 11 Mar 2021 18:59:29 -0600 Subject: [PATCH 389/449] tstore: Status change bug fix. --- politeiad/backendv2/tstorebe/tstore/tstore.go | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/politeiad/backendv2/tstorebe/tstore/tstore.go b/politeiad/backendv2/tstorebe/tstore/tstore.go index 036fd5553..acd2b06a4 100644 --- a/politeiad/backendv2/tstorebe/tstore/tstore.go +++ b/politeiad/backendv2/tstorebe/tstore/tstore.go @@ -515,16 +515,16 @@ func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, re return nil, fmt.Errorf("append leaves failed: %v", failed) } - // Check if any of the duplicates were saved as encrypted but now - // need to be resaved as plain text. This happens when a record is - // made public and the files need to be saved plain text. - if idx.State == backend.StateUnvetted || len(dups) == 0 { - // Nothing that needs to be saved plain text. We're done. - log.Tracef("No blobs need to be resaved plain text") - + // When a record is made public the record content needs to be + // resaved to the key-value store as unencrypted. + if recordMD.Status != backend.StatusPublic { + // Record is not being made public. Nothing else to do. return &idx, nil } + // Resave all of the duplicate blobs as plain text. A duplicate + // blob means the record content existed prior to the status + // change. blobs = make(map[string][]byte, len(dupBlobs)) for _, v := range leavesAll { d := hex.EncodeToString(v.LeafValue) @@ -558,10 +558,8 @@ func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, re blobs[ed.storeKeyNoPrefix()] = b } if len(blobs) == 0 { - // Nothing that needs to be saved plain text. We're done. - log.Tracef("No duplicates need to be resaved plain text") - - return &idx, nil + // This should not happen + return nil, fmt.Errorf("no blobs found to resave as plain text") } log.Debugf("Resaving %v encrypted blobs as plain text", len(blobs)) From a54a2bd03e4d12ba502f8369d2818a20e37408e0 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 11 Mar 2021 19:48:03 -0600 Subject: [PATCH 390/449] tstore: Bug fix. --- politeiad/backendv2/tstorebe/tstore/tstore.go | 9 ++++++++- politeiad/backendv2/tstorebe/tstorebe.go | 2 +- politeiad/v2.go | 9 +++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/politeiad/backendv2/tstorebe/tstore/tstore.go b/politeiad/backendv2/tstorebe/tstore/tstore.go index acd2b06a4..6c86433f4 100644 --- a/politeiad/backendv2/tstorebe/tstore/tstore.go +++ b/politeiad/backendv2/tstorebe/tstore/tstore.go @@ -517,7 +517,14 @@ func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, re // When a record is made public the record content needs to be // resaved to the key-value store as unencrypted. - if recordMD.Status != backend.StatusPublic { + var ( + isPublic = recordMD.Status == backend.StatusPublic + + // Iteration and version are reset back to 1 when a record is + // made public. + iterIsReset = recordMD.Iteration == 1 + ) + if !isPublic || !iterIsReset { // Record is not being made public. Nothing else to do. return &idx, nil } diff --git a/politeiad/backendv2/tstorebe/tstorebe.go b/politeiad/backendv2/tstorebe/tstorebe.go index eb4874b68..4c21f5f38 100644 --- a/politeiad/backendv2/tstorebe/tstorebe.go +++ b/politeiad/backendv2/tstorebe/tstorebe.go @@ -91,7 +91,7 @@ func (t *tstoreBackend) prefixAdd(fullToken []byte) { prefix := util.TokenPrefix(fullToken) t.prefixes[prefix] = fullToken - log.Debugf("Add token prefix: %v", prefix) + log.Tracef("Add token prefix: %v", prefix) } func (t *tstoreBackend) fullLengthToken(token []byte) []byte { diff --git a/politeiad/v2.go b/politeiad/v2.go index d70466959..aa9e51df7 100644 --- a/politeiad/v2.go +++ b/politeiad/v2.go @@ -58,6 +58,9 @@ func (p *politeia) handleRecordNew(w http.ResponseWriter, r *http.Request) { Record: p.convertRecordToV2(*rc), } + log.Infof("%v Record created %v", + util.RemoteAddr(r), rc.RecordMetadata.Token) + util.RespondWithJSON(w, http.StatusOK, rnr) } @@ -112,6 +115,9 @@ func (p *politeia) handleRecordEdit(w http.ResponseWriter, r *http.Request) { Record: p.convertRecordToV2(*rc), } + log.Infof("%v Record edited %v", + util.RemoteAddr(r), rc.RecordMetadata.Token) + util.RespondWithJSON(w, http.StatusOK, rer) } @@ -218,6 +224,9 @@ func (p *politeia) handleRecordSetStatus(w http.ResponseWriter, r *http.Request) Record: p.convertRecordToV2(*rc), } + log.Infof("%v Record status set %v %v", util.RemoteAddr(r), + rc.RecordMetadata.Token, backendv2.Statuses[rc.RecordMetadata.Status]) + util.RespondWithJSON(w, http.StatusOK, rer) } From 64a942847db9d38fad760f392db5110fa45f4b72 Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 12 Mar 2021 07:20:26 -0600 Subject: [PATCH 391/449] Bug fixes. --- politeiad/backendv2/tstorebe/store/mysql/encrypt.go | 9 ++++++--- politeiad/backendv2/tstorebe/store/mysql/mysql.go | 2 ++ util/encrypt.go | 3 ++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/politeiad/backendv2/tstorebe/store/mysql/encrypt.go b/politeiad/backendv2/tstorebe/store/mysql/encrypt.go index ddbb7133d..db3b081cf 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/encrypt.go +++ b/politeiad/backendv2/tstorebe/store/mysql/encrypt.go @@ -31,13 +31,16 @@ func (s *mysql) encrypt(ctx context.Context, tx *sql.Tx, data []byte) ([]byte, e var i int64 for rows.Next() { + if i > 0 { + // There should only ever be one row returned. Something is + // wrong if we've already scanned the nonce and its still + // scanning rows. + return nil, fmt.Errorf("multiple rows returned for nonce") + } err = rows.Scan(&i) if err != nil { return nil, fmt.Errorf("scan: %v", err) } - - // There should only ever be one value to scan - break } err = rows.Err() if err != nil { diff --git a/politeiad/backendv2/tstorebe/store/mysql/mysql.go b/politeiad/backendv2/tstorebe/store/mysql/mysql.go index be1aedf7e..60e6d1247 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/mysql.go +++ b/politeiad/backendv2/tstorebe/store/mysql/mysql.go @@ -112,6 +112,7 @@ func (s *mysql) Put(blobs map[string][]byte, encrypt bool) error { e := fmt.Sprintf("put: %v, unable to rollback: %v", err, err2) panic(e) } + return err } // Commit transaction @@ -154,6 +155,7 @@ func (s *mysql) Del(keys []string) error { e := fmt.Sprintf("del: %v, unable to rollback: %v", err, err2) panic(e) } + return err } } diff --git a/util/encrypt.go b/util/encrypt.go index f21dd9534..d7c04c816 100644 --- a/util/encrypt.go +++ b/util/encrypt.go @@ -53,6 +53,8 @@ func LoadEncryptionKey(log slog.Logger, keyFile string) (*[32]byte, error) { if err != nil { return nil, err } + defer f.Close() + var key [32]byte n, err := f.Read(key[:]) if n != len(key) { @@ -61,7 +63,6 @@ func LoadEncryptionKey(log slog.Logger, keyFile string) (*[32]byte, error) { if err != nil { return nil, err } - f.Close() log.Infof("Encryption key: %v", keyFile) From fcb69b2fe52962018464806e577dcfc2f04739eb Mon Sep 17 00:00:00 2001 From: luke Date: Sat, 13 Mar 2021 15:01:18 -0600 Subject: [PATCH 392/449] tstore: Cleanup databases. --- politeiad/api/v2/v2.go | 2 +- .../tstorebe/store/localdb/encrypt.go | 12 ++++---- .../tstorebe/store/localdb/localdb.go | 23 +++++++++----- .../backendv2/tstorebe/store/mysql/encrypt.go | 12 ++++---- .../backendv2/tstorebe/store/mysql/mysql.go | 30 ++++++------------- politeiad/backendv2/tstorebe/store/store.go | 5 ---- .../backendv2/tstorebe/tstore/testing.go | 2 +- politeiad/backendv2/tstorebe/tstore/tstore.go | 7 ++--- politeiad/backendv2/tstorebe/tstorebe.go | 5 ++-- politeiad/config.go | 12 ++++---- politeiad/politeiad.go | 2 +- 11 files changed, 50 insertions(+), 62 deletions(-) diff --git a/politeiad/api/v2/v2.go b/politeiad/api/v2/v2.go index 76c8fd5a5..5c7336116 100644 --- a/politeiad/api/v2/v2.go +++ b/politeiad/api/v2/v2.go @@ -327,7 +327,7 @@ type RecordGetReply struct { // Version is used to request a specific version of a record. If no version is // provided then the most recent version of the record will be returned. // -// Filenames can be used to request specific files. If filenames is not W +// Filenames can be used to request specific files. If filenames is provided // then the specified files will be the only files that are returned. // // OmitAllFiles can be used to retrieve a record without any of the record diff --git a/politeiad/backendv2/tstorebe/store/localdb/encrypt.go b/politeiad/backendv2/tstorebe/store/localdb/encrypt.go index d3a7927b1..1eac4e5d8 100644 --- a/politeiad/backendv2/tstorebe/store/localdb/encrypt.go +++ b/politeiad/backendv2/tstorebe/store/localdb/encrypt.go @@ -12,22 +12,22 @@ import ( ) func (l *localdb) encrypt(data []byte) ([]byte, error) { - l.keyMtx.RLock() - defer l.keyMtx.RUnlock() + l.RLock() + defer l.RUnlock() return sbox.Encrypt(0, l.key, data) } func (l *localdb) decrypt(data []byte) ([]byte, uint32, error) { - l.keyMtx.RLock() - defer l.keyMtx.RUnlock() + l.RLock() + defer l.RUnlock() return sbox.Decrypt(l.key, data) } func (l *localdb) zeroKey() { - l.keyMtx.Lock() - defer l.keyMtx.Unlock() + l.Lock() + defer l.Unlock() util.Zero(l.key[:]) l.key = nil diff --git a/politeiad/backendv2/tstorebe/store/localdb/localdb.go b/politeiad/backendv2/tstorebe/store/localdb/localdb.go index 6696940d6..fcdccc266 100644 --- a/politeiad/backendv2/tstorebe/store/localdb/localdb.go +++ b/politeiad/backendv2/tstorebe/store/localdb/localdb.go @@ -15,21 +15,31 @@ import ( "github.com/syndtr/goleveldb/leveldb" ) +const ( + // encryptionKeyFilename is the filename of the encryption key that + // is created in the store data directory. + encryptionKeyFilename = "sbox.key" +) + var ( _ store.BlobKV = (*localdb)(nil) ) // localdb implements the store BlobKV interface using leveldb. +// +// NOTE: this implementation was created for testing. The encryption techniques +// used may not be suitable for a production environment. A random secretbox +// encryption key is created on startup and saved to the politeiad application +// dir. Blobs are encrypted using random 24 byte nonces. type localdb struct { - sync.Mutex + sync.RWMutex shutdown bool db *leveldb.DB // Encryption key and mutex. The key is zero'd out on application // exit so the read lock must be held during concurrent access to // prevent the golang race detector from complaining. - key *[32]byte - keyMtx sync.RWMutex + key *[32]byte } // Put saves the provided key-value pairs to the store. This operation is @@ -140,12 +150,9 @@ func (l *localdb) Close() { } // New returns a new localdb. -func New(appDir, dataDir, keyFile string) (*localdb, error) { +func New(appDir, dataDir string) (*localdb, error) { // Load encryption key - if keyFile == "" { - // No file path was given. Use the default path. - keyFile = filepath.Join(appDir, store.DefaultEncryptionKeyFilename) - } + keyFile := filepath.Join(appDir, encryptionKeyFilename) key, err := util.LoadEncryptionKey(log, keyFile) if err != nil { return nil, err diff --git a/politeiad/backendv2/tstorebe/store/mysql/encrypt.go b/politeiad/backendv2/tstorebe/store/mysql/encrypt.go index db3b081cf..a2d7716a7 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/encrypt.go +++ b/politeiad/backendv2/tstorebe/store/mysql/encrypt.go @@ -62,22 +62,22 @@ func (s *mysql) encrypt(ctx context.Context, tx *sql.Tx, data []byte) ([]byte, e nonce := n.Current() // Encrypt blob - s.keyMtx.RLock() - defer s.keyMtx.RUnlock() + s.RLock() + defer s.RUnlock() return sbox.EncryptN(0, s.key, nonce, data) } func (s *mysql) decrypt(data []byte) ([]byte, uint32, error) { - s.keyMtx.RLock() - defer s.keyMtx.RUnlock() + s.RLock() + defer s.RUnlock() return sbox.Decrypt(s.key, data) } func (s *mysql) zeroKey() { - s.keyMtx.Lock() - defer s.keyMtx.Unlock() + s.Lock() + defer s.Unlock() util.Zero(s.key[:]) s.key = nil diff --git a/politeiad/backendv2/tstorebe/store/mysql/mysql.go b/politeiad/backendv2/tstorebe/store/mysql/mysql.go index 60e6d1247..133373c83 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/mysql.go +++ b/politeiad/backendv2/tstorebe/store/mysql/mysql.go @@ -8,12 +8,10 @@ import ( "context" "database/sql" "fmt" - "path/filepath" "sync" "time" "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" - "github.com/decred/politeia/util" _ "github.com/go-sql-driver/mysql" ) @@ -32,7 +30,7 @@ const ( // tableKeyValue defines the key-value table. const tableKeyValue = ` - k VARCHAR(38) NOT NULL PRIMARY KEY, + k VARCHAR(255) NOT NULL PRIMARY KEY, v LONGBLOB NOT NULL ` @@ -47,13 +45,13 @@ var ( // mysql implements the store BlobKV interface using a mysql driver. type mysql struct { + sync.RWMutex db *sql.DB - // Encryption key and mutex. The key is zero'd out on application - // exit so the read lock must be held during concurrent access to - // prevent the golang race detector from complaining. - key *[32]byte - keyMtx sync.RWMutex + // Encryption key. The key is zero'd out on application exit so the + // read lock must be held during concurrent access to prevent the + // golang race detector from complaining. + key *[32]byte } func ctxWithTimeout() (context.Context, func()) { @@ -250,17 +248,7 @@ func (s *mysql) Close() { s.db.Close() } -func New(appDir, host, user, password, dbname, keyFile string) (*mysql, error) { - // Load encryption key - if keyFile == "" { - // No file path was given. Use the default path. - keyFile = filepath.Join(appDir, store.DefaultEncryptionKeyFilename) - } - key, err := util.LoadEncryptionKey(log, keyFile) - if err != nil { - return nil, err - } - +func New(appDir, host, user, password, dbname string) (*mysql, error) { // Connect to database log.Infof("Host: %v:[password]@tcp(%v)/%v", user, host, dbname) @@ -298,7 +286,7 @@ func New(appDir, host, user, password, dbname, keyFile string) (*mysql, error) { } return &mysql{ - db: db, - key: key, + db: db, + // key: key, }, nil } diff --git a/politeiad/backendv2/tstorebe/store/store.go b/politeiad/backendv2/tstorebe/store/store.go index beec9a351..cf2cfcded 100644 --- a/politeiad/backendv2/tstorebe/store/store.go +++ b/politeiad/backendv2/tstorebe/store/store.go @@ -15,11 +15,6 @@ import ( ) const ( - // DefaultEncryptionKeyFilename is the filename of the encryption - // key that is created in the store data directory if an encryption - // key file is not provided on startup. - DefaultEncryptionKeyFilename = "sbox.key" - // DataTypeStructure is used as the data descriptor type when the // blob entry contains a structure. DataTypeStructure = "struct" diff --git a/politeiad/backendv2/tstorebe/tstore/testing.go b/politeiad/backendv2/tstorebe/tstore/testing.go index 67fb5ae60..6a2b2c3ea 100644 --- a/politeiad/backendv2/tstorebe/tstore/testing.go +++ b/politeiad/backendv2/tstorebe/tstore/testing.go @@ -25,7 +25,7 @@ func NewTestTstore(t *testing.T, dataDir string) *Tstore { if err != nil { t.Fatal(err) } - store, err := localdb.New(dataDir, fp, "") + store, err := localdb.New(dataDir, fp) if err != nil { t.Fatal(err) } diff --git a/politeiad/backendv2/tstorebe/tstore/tstore.go b/politeiad/backendv2/tstorebe/tstore/tstore.go index 6c86433f4..e1c32e83e 100644 --- a/politeiad/backendv2/tstorebe/tstore/tstore.go +++ b/politeiad/backendv2/tstorebe/tstore/tstore.go @@ -1164,7 +1164,7 @@ func (t *Tstore) Close() { t.tlog.close() } -func New(appDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianSigningKeyFile, dbType, dbHost, dbPass, dbEncryptionKeyFile, dcrtimeHost, dcrtimeCert string) (*Tstore, error) { +func New(appDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianSigningKeyFile, dbType, dbHost, dbPass, dcrtimeHost, dcrtimeCert string) (*Tstore, error) { // Setup trillian client if trillianSigningKeyFile == "" { // No file path was given. Use the default path. @@ -1197,15 +1197,14 @@ func New(appDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianSig if err != nil { return nil, err } - kvstore, err = localdb.New(appDir, fp, dbEncryptionKeyFile) + kvstore, err = localdb.New(appDir, fp) if err != nil { return nil, err } case DBTypeMySQL: // Example db name: testnet3_unvetted_kv dbName := fmt.Sprintf("%v_kv", anp.Name) - kvstore, err = mysql.New(appDir, dbHost, dbUser, dbPass, - dbName, dbEncryptionKeyFile) + kvstore, err = mysql.New(appDir, dbHost, dbUser, dbPass, dbName) if err != nil { return nil, err } diff --git a/politeiad/backendv2/tstorebe/tstorebe.go b/politeiad/backendv2/tstorebe/tstorebe.go index 4c21f5f38..a19a75346 100644 --- a/politeiad/backendv2/tstorebe/tstorebe.go +++ b/politeiad/backendv2/tstorebe/tstorebe.go @@ -1186,11 +1186,10 @@ func (t *tstoreBackend) setup() error { } // New returns a new tstoreBackend. -func New(appDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianSigningKey, dbType, dbHost, dbPass, dbEncryptionKeyFile, dcrtimeHost, dcrtimeCert string) (*tstoreBackend, error) { +func New(appDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianSigningKey, dbType, dbHost, dbPass, dcrtimeHost, dcrtimeCert string) (*tstoreBackend, error) { // Setup tstore instances ts, err := tstore.New(appDir, dataDir, anp, trillianHost, - trillianSigningKey, dbType, dbHost, dbPass, dbEncryptionKeyFile, - dcrtimeHost, dcrtimeCert) + trillianSigningKey, dbType, dbHost, dbPass, dcrtimeHost, dcrtimeCert) if err != nil { return nil, fmt.Errorf("new tstore: %v", err) } diff --git a/politeiad/config.go b/politeiad/config.go index 748f4c11a..c7eb52505 100644 --- a/politeiad/config.go +++ b/politeiad/config.go @@ -92,14 +92,14 @@ type config struct { DcrdataHost string `long:"dcrdatahost" description:"Dcrdata ip:port"` // TODO validate these config params - Backend string `long:"backend"` + Backend string `long:"backend"` + + // Tstore backend config TrillianHost string `long:"trillianhost"` TrillianSigningKey string `long:"trilliansigningkey"` - - DBType string `long:"dbtype" description:"Database type"` - DBHost string `long:"dbhost" description:"Database ip:port"` - DBPass string `long:"dbpass" description:"Database password"` - DBEncryptionKey string `long:"dbencryptionkey" description:"Database encryption key file"` + DBType string `long:"dbtype" description:"Database type"` + DBHost string `long:"dbhost" description:"Database ip:port"` + DBPass string `long:"dbpass" description:"Database password"` // Plugin settings Plugins []string `long:"plugin"` diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index bf49cc6c1..c86acdf1f 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -205,7 +205,7 @@ func (p *politeia) setupBackendGit(anp *chaincfg.Params) error { func (p *politeia) setupBackendTstore(anp *chaincfg.Params) error { b, err := tstorebe.New(p.cfg.HomeDir, p.cfg.DataDir, anp, p.cfg.TrillianHost, p.cfg.TrillianSigningKey, - p.cfg.DBType, p.cfg.DBHost, p.cfg.DBPass, p.cfg.DBEncryptionKey, + p.cfg.DBType, p.cfg.DBHost, p.cfg.DBPass, p.cfg.DcrtimeHost, p.cfg.DcrtimeCert) if err != nil { return fmt.Errorf("new tstorebe: %v", err) From e80adf59ba18dc9718f131a1beb3db613100efd7 Mon Sep 17 00:00:00 2001 From: luke Date: Sat, 13 Mar 2021 17:05:54 -0600 Subject: [PATCH 393/449] tstorebe/store: Derive key from password. --- .../backendv2/tstorebe/store/mysql/mysql.go | 63 ++++++++++++++++++- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/politeiad/backendv2/tstorebe/store/mysql/mysql.go b/politeiad/backendv2/tstorebe/store/mysql/mysql.go index 133373c83..06a180504 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/mysql.go +++ b/politeiad/backendv2/tstorebe/store/mysql/mysql.go @@ -12,6 +12,8 @@ import ( "time" "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" + "github.com/decred/politeia/util" + "golang.org/x/crypto/argon2" _ "github.com/go-sql-driver/mysql" ) @@ -248,6 +250,39 @@ func (s *mysql) Close() { s.db.Close() } +func (s *mysql) salt(size int) ([]byte, error) { + saltKey := "salt" + + // Check if a salt already exists in the database + blobs, err := s.Get([]string{saltKey}) + if err != nil { + return nil, fmt.Errorf("get: %v", err) + } + salt, ok := blobs[saltKey] + if ok { + // Salt already exists + log.Debugf("Salt found in kv store") + return salt, nil + } + + // Salt doesn't exist yet. Create one and save it. + salt, err = util.Random(size) + if err != nil { + return nil, err + } + kv := map[string][]byte{ + saltKey: salt, + } + err = s.Put(kv, false) + if err != nil { + return nil, fmt.Errorf("put: %v", err) + } + + log.Debugf("Salt created and saved to kv store") + + return salt, nil +} + func New(appDir, host, user, password, dbname string) (*mysql, error) { // Connect to database log.Infof("Host: %v:[password]@tcp(%v)/%v", user, host, dbname) @@ -285,8 +320,30 @@ func New(appDir, host, user, password, dbname string) (*mysql, error) { return nil, fmt.Errorf("create nonce table: %v", err) } - return &mysql{ + // Setup kv store context + s := mysql{ db: db, - // key: key, - }, nil + } + + // Derive encryption key from password + var ( + pass = []byte(password) + saltLen int = 16 // In bytes + time uint32 = 1 + memory uint32 = 64 * 1024 // 64 MB + threads uint8 = 4 // Number of available CPUs + keyLen uint32 = 32 // In bytes + ) + salt, err := s.salt(saltLen) + if err != nil { + return nil, fmt.Errorf("salt: %v", err) + } + k := argon2.IDKey(pass, salt, time, memory, threads, keyLen) + var key [32]byte + copy(key[:], k) + util.Zero(k) + + s.key = &key + + return &s, nil } From 20ee11fe3a374099344e75f142fca99a82df3df2 Mon Sep 17 00:00:00 2001 From: luke Date: Sat, 13 Mar 2021 17:33:11 -0600 Subject: [PATCH 394/449] politeiad: Verify config params. --- politeiad/config.go | 42 ++++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/politeiad/config.go b/politeiad/config.go index c7eb52505..25b7953fe 100644 --- a/politeiad/config.go +++ b/politeiad/config.go @@ -47,7 +47,7 @@ const ( // Tstore default settings defaultTrillianHost = "localhost:8090" defaultDBType = tstore.DBTypeLevelDB - defaultDBHost = "127.0.0.1:3306" // MySQL default host + defaultDBHost = "localhost:3306" // MySQL default host ) var ( @@ -88,22 +88,22 @@ type config struct { DcrtimeHost string `long:"dcrtimehost" description:"Dcrtime ip:port"` DcrtimeCert string `long:"dcrtimecert" description:"File containing the https certificate file for dcrtimehost"` Identity string `long:"identity" description:"File containing the politeiad identity file"` + Backend string `long:"backend" description:"Backend type"` + + // Git backend options GitTrace bool `long:"gittrace" description:"Enable git tracing in logs"` DcrdataHost string `long:"dcrdatahost" description:"Dcrdata ip:port"` - // TODO validate these config params - Backend string `long:"backend"` - - // Tstore backend config - TrillianHost string `long:"trillianhost"` - TrillianSigningKey string `long:"trilliansigningkey"` + // Tstore backend options + TrillianHost string `long:"trillianhost" description:"Trillian ip:port"` + TrillianSigningKey string `long:"trilliansigningkey" description:"Trillian signing key"` DBType string `long:"dbtype" description:"Database type"` DBHost string `long:"dbhost" description:"Database ip:port"` - DBPass string `long:"dbpass" description:"Database password"` + DBPass string // Provided in env variable "DBPASS" // Plugin settings - Plugins []string `long:"plugin"` - PluginSettings []string `long:"pluginsetting"` + Plugins []string `long:"plugin" description:"Plugins"` + PluginSettings []string `long:"pluginsetting" description:"Plugin settings"` } // serviceOptions defines the configuration options for the daemon as a service @@ -532,6 +532,28 @@ func loadConfig() (*config, []string, error) { log.Warnf("RPC password not set, using random value") } + // Verify backend type + switch cfg.Backend { + case backendGit, backendTstore: + // Allowed; continue + default: + return nil, nil, fmt.Errorf("invalid backend type '%v'", cfg.Backend) + } + + // Verify tstore backend settings + switch cfg.DBType { + case tstore.DBTypeLevelDB: + // Allowed; continue + case tstore.DBTypeMySQL: + // The database password is provided in the env variable "DBPASS" + cfg.DBPass = os.Getenv("DBPASS") + if cfg.DBPass == "" { + return nil, nil, fmt.Errorf("dbpass not found; you must provide " + + "the database password for the politeiad user in the env " + + "variable DBPASS") + } + } + // Warn about missing config file only after all other configuration is // done. This prevents the warning on help messages and invalid // options. Note this should go directly before the return. From 410cafa929d421dae37e6414d41b5746270f0ea5 Mon Sep 17 00:00:00 2001 From: luke Date: Sat, 13 Mar 2021 17:45:42 -0600 Subject: [PATCH 395/449] politeiad: Update README. --- politeiad/README.md | 60 +++++++++------------------------------------ 1 file changed, 11 insertions(+), 49 deletions(-) diff --git a/politeiad/README.md b/politeiad/README.md index ef38ca2fd..8d9fe58d6 100644 --- a/politeiad/README.md +++ b/politeiad/README.md @@ -119,6 +119,9 @@ politeiad The `trillianpass` and `rootpass` will need to be updated to the passwords for your trillian and root users. + If setting up a mainnet instance, change the `MYSQL_DATABASE` env variable + to `mainnet_trillian`. + ``` $ cd $GOPATH/src/github.com/google/trillian/scripts @@ -130,16 +133,6 @@ politeiad MYSQL_URI="${MYSQL_USER}:${MYSQL_PASSWORD}@tcp(127.0.0.1:3306)/${MYSQL_DATABASE}" \ MYSQL_ROOT_PASSWORD=rootpass \ ./resetdb.sh - - # Mainnet setup - $ env \ - MYSQL_USER=trillian \ - MYSQL_PASSWORD=trillianpass \ - MYSQL_DATABASE=mainnet_trillian \ - MYSQL_URI="${MYSQL_USER}:${MYSQL_PASSWORD}@tcp(127.0.0.1:3306)/${MYSQL_DATABASE}" \ - MYSQL_ROOT_PASSWORD=rootpass \ - ./resetdb.sh - ``` 7. Start up the trillian instances. @@ -153,6 +146,10 @@ politeiad run one set of commands, testnet or mainnet. Run the testnet commands if you're setting up a development environment. + If setting up a mainnet instance, change the `MYSQL_DATABASE` env variable + to `mainnet_trillian` for both the log server and log signer. + + Start testnet log server ``` $ export MYSQL_USER=trillian && \ @@ -183,38 +180,6 @@ politeiad --rpc_endpoint localhost:8092 \ --http_endpoint=localhost:8093 ``` - - Start mainnet log server - ``` - $ export MYSQL_USER=trillian && \ - export MYSQL_PASSWORD=trillianpass && \ - export MYSQL_DATABASE=mainnet_trillian && \ - export MYSQL_URI="${MYSQL_USER}:${MYSQL_PASSWORD}@tcp(127.0.0.1:3306)/${MYSQL_DATABASE}" - - $ trillian_log_server \ - --mysql_uri=${MYSQL_URI} \ - --mysql_max_conns=2000 \ - --rpc_endpoint localhost:8090 \ - --http_endpoint localhost:8091 \ - --logtostderr ... - ``` - - Start mainnet log signer - ``` - $ export MYSQL_USER=trillian && \ - export MYSQL_PASSWORD=trillianpass && \ - export MYSQL_DATABASE=mainnet_trillian && \ - export MYSQL_URI="${MYSQL_USER}:${MYSQL_PASSWORD}@tcp(127.0.0.1:3306)/${MYSQL_DATABASE}" - - $ trillian_log_signer --logtostderr --force_master \ - --batch_size=1000 \ - --sequencer_guard_window=0 \ - --sequencer_interval=200ms \ - --mysql_uri=${MYSQL_URI} \ - --rpc_endpoint localhost:8092 \ - --http_endpoint=localhost:8093 - ``` - 8. Setup the politeiad configuration file. @@ -252,9 +217,8 @@ politeiad rpcpass=pass testnet=true - ; Tstore database settings + ; Tstore settings dbtype=mysql - dbpass=politeiadpass ; Pi plugin configuration plugin=pi @@ -264,15 +228,13 @@ politeiad plugin=usermd ``` -9. Start up the politeiad instance. +9. Start up the politeiad instance. The password for the politeiad user must + be provided in the `DBPASS` env variable. ``` - $ politeiad + $ env DBPASS=politeiadpass politeiad ``` - A database encryption key and a trillian signing key will be created on - startup and saved to the politeiad app data directory. - # Tools and reference clients * [politeia](https://github.com/decred/politeia/tree/master/politeiad/cmd/politeia) - Reference client for politeiad. From dd95df595214039e46bc1bc4f5a389a0f910b184 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 14 Mar 2021 11:16:10 -0500 Subject: [PATCH 396/449] Cleanup. --- .../tstorebe/store/localdb/localdb.go | 2 +- .../backendv2/tstorebe/store/mysql/encrypt.go | 60 +++++++++++++++++++ .../backendv2/tstorebe/store/mysql/mysql.go | 56 ++--------------- 3 files changed, 67 insertions(+), 51 deletions(-) diff --git a/politeiad/backendv2/tstorebe/store/localdb/localdb.go b/politeiad/backendv2/tstorebe/store/localdb/localdb.go index fcdccc266..f1110e68f 100644 --- a/politeiad/backendv2/tstorebe/store/localdb/localdb.go +++ b/politeiad/backendv2/tstorebe/store/localdb/localdb.go @@ -18,7 +18,7 @@ import ( const ( // encryptionKeyFilename is the filename of the encryption key that // is created in the store data directory. - encryptionKeyFilename = "sbox.key" + encryptionKeyFilename = "leveldb-sbox.key" ) var ( diff --git a/politeiad/backendv2/tstorebe/store/mysql/encrypt.go b/politeiad/backendv2/tstorebe/store/mysql/encrypt.go index a2d7716a7..f87e2f372 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/encrypt.go +++ b/politeiad/backendv2/tstorebe/store/mysql/encrypt.go @@ -13,8 +13,68 @@ import ( "github.com/decred/politeia/util" "github.com/marcopeereboom/sbox" + "golang.org/x/crypto/argon2" ) +// salt creates a random salt and saves it to the kv store. Subsequent calls to +// this function will return the existing salt. +func (s *mysql) salt(size int) ([]byte, error) { + saltKey := "salt" + + // Check if a salt already exists in the database + blobs, err := s.Get([]string{saltKey}) + if err != nil { + return nil, fmt.Errorf("get: %v", err) + } + salt, ok := blobs[saltKey] + if ok { + // Salt already exists + log.Debugf("Salt found in kv store") + return salt, nil + } + + // Salt doesn't exist yet. Create one and save it. + salt, err = util.Random(size) + if err != nil { + return nil, err + } + kv := map[string][]byte{ + saltKey: salt, + } + err = s.Put(kv, false) + if err != nil { + return nil, fmt.Errorf("put: %v", err) + } + + log.Debugf("Salt created and saved to kv store") + + return salt, nil +} + +// aragon2idKey derives a 32 byte aragon2id key from the provided password. +// The salt is generated the first time the key is derived and saved to the kv +// store. Subsequent calls to this fuction will use the existing salt. +func (s *mysql) argon2idKey(password string) (*[32]byte, error) { + var ( + pass = []byte(password) + saltLen int = 16 // In bytes + time uint32 = 1 + memory uint32 = 64 * 1024 // 64 MB + threads uint8 = 4 // Number of available CPUs + keyLen uint32 = 32 // In bytes + ) + salt, err := s.salt(saltLen) + if err != nil { + return nil, fmt.Errorf("salt: %v", err) + } + k := argon2.IDKey(pass, salt, time, memory, threads, keyLen) + var key [32]byte + copy(key[:], k) + util.Zero(k) + + return &key, nil +} + func (s *mysql) encrypt(ctx context.Context, tx *sql.Tx, data []byte) ([]byte, error) { // Create a new nonce value _, err := tx.ExecContext(ctx, "INSERT INTO nonce () VALUES ();") diff --git a/politeiad/backendv2/tstorebe/store/mysql/mysql.go b/politeiad/backendv2/tstorebe/store/mysql/mysql.go index 06a180504..e9ff73e09 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/mysql.go +++ b/politeiad/backendv2/tstorebe/store/mysql/mysql.go @@ -12,8 +12,6 @@ import ( "time" "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" - "github.com/decred/politeia/util" - "golang.org/x/crypto/argon2" _ "github.com/go-sql-driver/mysql" ) @@ -250,40 +248,12 @@ func (s *mysql) Close() { s.db.Close() } -func (s *mysql) salt(size int) ([]byte, error) { - saltKey := "salt" - - // Check if a salt already exists in the database - blobs, err := s.Get([]string{saltKey}) - if err != nil { - return nil, fmt.Errorf("get: %v", err) - } - salt, ok := blobs[saltKey] - if ok { - // Salt already exists - log.Debugf("Salt found in kv store") - return salt, nil - } - - // Salt doesn't exist yet. Create one and save it. - salt, err = util.Random(size) - if err != nil { - return nil, err - } - kv := map[string][]byte{ - saltKey: salt, - } - err = s.Put(kv, false) - if err != nil { - return nil, fmt.Errorf("put: %v", err) +func New(appDir, host, user, password, dbname string) (*mysql, error) { + // The password is required to derive the encryption key + if password == "" { + return nil, fmt.Errorf("password not provided") } - log.Debugf("Salt created and saved to kv store") - - return salt, nil -} - -func New(appDir, host, user, password, dbname string) (*mysql, error) { // Connect to database log.Infof("Host: %v:[password]@tcp(%v)/%v", user, host, dbname) @@ -326,24 +296,10 @@ func New(appDir, host, user, password, dbname string) (*mysql, error) { } // Derive encryption key from password - var ( - pass = []byte(password) - saltLen int = 16 // In bytes - time uint32 = 1 - memory uint32 = 64 * 1024 // 64 MB - threads uint8 = 4 // Number of available CPUs - keyLen uint32 = 32 // In bytes - ) - salt, err := s.salt(saltLen) + s.key, err = s.argon2idKey(password) if err != nil { - return nil, fmt.Errorf("salt: %v", err) + return nil, fmt.Errorf("argon2idKey: %v", err) } - k := argon2.IDKey(pass, salt, time, memory, threads, keyLen) - var key [32]byte - copy(key[:], k) - util.Zero(k) - - s.key = &key return &s, nil } From ec23cc02c155c4d70a944394203d3e0ba1d80811 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 14 Mar 2021 11:43:08 -0500 Subject: [PATCH 397/449] ticketvote: Verify record status. --- .../tstorebe/plugins/ticketvote/cmds.go | 26 ++++++++++++++++--- politeiad/plugins/ticketvote/ticketvote.go | 5 ++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go index 5c9d2d9bc..ba23ea7c5 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go @@ -1038,11 +1038,18 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri } } - // Verify record version + // Verify record status and version r, err := p.tstore.RecordPartial(treeID, 0, nil, true) if err != nil { return "", fmt.Errorf("RecordPartial: %v", err) } + if r.RecordMetadata.Status != backend.StatusPublic { + return "", backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: int(ticketvote.ErrorCodeRecordStatusInvalid), + ErrorContext: "record is not public", + } + } if a.Version != r.RecordMetadata.Version { e := fmt.Sprintf("version is not latest: got %v, want %v", a.Version, r.RecordMetadata.Version) @@ -1357,6 +1364,10 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot if err != nil { return nil, fmt.Errorf("RecordPartial: %v", err) } + if r.RecordMetadata.State != backend.StateVetted { + // This should not be possible + return nil, fmt.Errorf("record is unvetted") + } if sd.Params.Version != r.RecordMetadata.Version { e := fmt.Sprintf("version is not latest: got %v, want %v", sd.Params.Version, r.RecordMetadata.Version) @@ -1576,6 +1587,10 @@ func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs sta if err != nil { return fmt.Errorf("RecordPartial: %v", err) } + if r.RecordMetadata.State != backend.StateVetted { + // This should not be possible + return fmt.Errorf("record is unvetted") + } if sd.Params.Version != r.RecordMetadata.Version { e := fmt.Sprintf("version is not latest %v: got %v, want %v", sd.Params.Token, sd.Params.Version, r.RecordMetadata.Version) @@ -1640,9 +1655,10 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti } // Verify parent has a LinkBy and the LinkBy deadline is expired. - r, err := p.tstore.RecordPartial(treeID, 0, []string{ + files := []string{ ticketvote.FileNameVoteMetadata, - }, false) + } + r, err := p.tstore.RecordPartial(treeID, 0, files, false) if err != nil { if errors.Is(err, backend.ErrRecordNotFound) { e := fmt.Sprintf("parent record not found %x", token) @@ -1654,6 +1670,10 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti } return nil, fmt.Errorf("RecordPartial: %v", err) } + if r.RecordMetadata.State != backend.StateVetted { + // This should not be possible + return nil, fmt.Errorf("record is unvetted") + } vm, err := voteMetadataDecode(r.Files) if err != nil { return nil, err diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index 8bf1ea795..cbc51ea07 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -173,6 +173,10 @@ const ( // ErrorCodeLinkByNotExpired is returned when a runoff vote is // attempted to be started before the link by deadline has expired. ErrorCodeLinkByNotExpired ErrorCodeT = 19 + + // ErrorCodeRecordStateInvalid is returned when a ticketvote write + // command is executed on a record that is not public. + ErrorCodeRecordStatusInvalid ErrorCodeT = 20 ) var ( @@ -198,6 +202,7 @@ var ( ErrorCodeLinkByInvalid: "linkby invalid", ErrorCodeLinkToInvalid: "linkto invalid", ErrorCodeLinkByNotExpired: "linkby not exipred", + ErrorCodeRecordStatusInvalid: "record status invalid", } ) From bd96aefc179e36ca238e6c0496a92f759c80519a Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 14 Mar 2021 17:32:41 -0500 Subject: [PATCH 398/449] ticketvote: Use a vote collider leaf. --- politeiad/README.md | 9 +- .../backendv2/tstorebe/plugins/plugins.go | 9 +- .../tstorebe/plugins/ticketvote/cmds.go | 776 +++++++++++------- .../tstorebe/plugins/ticketvote/ticketvote.go | 2 +- politeiad/backendv2/tstorebe/tstore/client.go | 30 +- politeiad/plugins/ticketvote/ticketvote.go | 3 +- politeiad/v2.go | 3 + 7 files changed, 532 insertions(+), 300 deletions(-) diff --git a/politeiad/README.md b/politeiad/README.md index 8d9fe58d6..54598f4c5 100644 --- a/politeiad/README.md +++ b/politeiad/README.md @@ -90,8 +90,13 @@ politeiad 5. Run the politeiad mysql setup scripts. This will create the politeiad and trillian users as well as creating the - politeiad databases. Trillian does not support SSL authentication to the - MySQL instance. Password authentication must be used. + politeiad databases. Password authentication is used for all database + connections. + + **The password that you set for the politeiad user will be used to derive an + encryption key that is used to encrypt non-public politeiad data. Make sure + to setup a strong password when running in production. Once set, the + politeiad user password cannot change.** The setup script assumes MySQL is running on `localhost:3306` and the users will be accessing the databse from `localhost`. See the setup script diff --git a/politeiad/backendv2/tstorebe/plugins/plugins.go b/politeiad/backendv2/tstorebe/plugins/plugins.go index 2902ea0ff..537f5fd30 100644 --- a/politeiad/backendv2/tstorebe/plugins/plugins.go +++ b/politeiad/backendv2/tstorebe/plugins/plugins.go @@ -171,12 +171,13 @@ type TstoreClient interface { // BlobsByDataDesc returns all blobs that match the provided data // descriptor. The blobs will be ordered from oldest to newest. If // a record is vetted then only vetted blobs will be returned. - BlobsByDataDesc(treeID int64, dataDesc string) ([]store.BlobEntry, error) + BlobsByDataDesc(treeID int64, dataDesc []string) ([]store.BlobEntry, error) // DigestsByDataDesc returns the digests of all blobs that match - // the provided data descriptor. If a record is vetted, only - // vetted blobs will be returned. - DigestsByDataDesc(treeID int64, dataDesc string) ([][]byte, error) + // the provided data descriptor. The digests will be ordered from + // oldest to newest. If a record is vetted, only vetted blobs will + // be returned. + DigestsByDataDesc(treeID int64, dataDesc []string) ([][]byte, error) // Timestamp returns the timestamp for the blob that correpsonds // to the digest. If a record is vetted, only vetted timestamps diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go index ba23ea7c5..3a923dd61 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go @@ -35,7 +35,8 @@ const ( // Blob entry data descriptors dataDescriptorAuthDetails = pluginID + "-auth-v1" dataDescriptorVoteDetails = pluginID + "-vote-v1" - dataDescriptorCastVoteDetails = pluginID + "-cvote-v1" + dataDescriptorCastVoteDetails = pluginID + "-castvote-v1" + dataDescriptorVoteCollider = pluginID + "-vcollider-v1" dataDescriptorStartRunoff = pluginID + "-startrunoff-v1" // Internal plugin commands @@ -77,226 +78,6 @@ func convertSignatureError(err error) backend.PluginError { } } -func convertAuthDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.AuthDetails, error) { - // Decode and validate data hint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorAuthDetails { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorAuthDetails) - } - - // Decode data - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - digest, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, fmt.Errorf("decode digest: %v", err) - } - if !bytes.Equal(util.Digest(b), digest) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), digest) - } - var ad ticketvote.AuthDetails - err = json.Unmarshal(b, &ad) - if err != nil { - return nil, fmt.Errorf("unmarshal AuthDetails: %v", err) - } - - return &ad, nil -} - -func convertVoteDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.VoteDetails, error) { - // Decode and validate data hint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorVoteDetails { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorVoteDetails) - } - - // Decode data - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - digest, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, fmt.Errorf("decode digest: %v", err) - } - if !bytes.Equal(util.Digest(b), digest) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), digest) - } - var vd ticketvote.VoteDetails - err = json.Unmarshal(b, &vd) - if err != nil { - return nil, fmt.Errorf("unmarshal VoteDetails: %v", err) - } - - return &vd, nil -} - -func convertCastVoteDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.CastVoteDetails, error) { - // Decode and validate data hint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorCastVoteDetails { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorCastVoteDetails) - } - - // Decode data - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - digest, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, fmt.Errorf("decode digest: %v", err) - } - if !bytes.Equal(util.Digest(b), digest) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), digest) - } - var cv ticketvote.CastVoteDetails - err = json.Unmarshal(b, &cv) - if err != nil { - return nil, fmt.Errorf("unmarshal CastVoteDetails: %v", err) - } - - return &cv, nil -} - -func convertStartRunoffFromBlobEntry(be store.BlobEntry) (*startRunoffRecord, error) { - // Decode and validate data hint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorStartRunoff { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorStartRunoff) - } - - // Decode data - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - digest, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, fmt.Errorf("decode digest: %v", err) - } - if !bytes.Equal(util.Digest(b), digest) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), digest) - } - var srr startRunoffRecord - err = json.Unmarshal(b, &srr) - if err != nil { - return nil, fmt.Errorf("unmarshal StartRunoffRecord: %v", err) - } - - return &srr, nil -} - -func convertBlobEntryFromAuthDetails(ad ticketvote.AuthDetails) (*store.BlobEntry, error) { - data, err := json.Marshal(ad) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorAuthDetails, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func convertBlobEntryFromVoteDetails(vd ticketvote.VoteDetails) (*store.BlobEntry, error) { - data, err := json.Marshal(vd) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorVoteDetails, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func convertBlobEntryFromCastVoteDetails(cv ticketvote.CastVoteDetails) (*store.BlobEntry, error) { - data, err := json.Marshal(cv) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorCastVoteDetails, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func convertBlobEntryFromStartRunoff(srr startRunoffRecord) (*store.BlobEntry, error) { - data, err := json.Marshal(srr) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorStartRunoff, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - func (p *ticketVotePlugin) authSave(treeID int64, ad ticketvote.AuthDetails) error { // Prepare blob be, err := convertBlobEntryFromAuthDetails(ad) @@ -310,7 +91,8 @@ func (p *ticketVotePlugin) authSave(treeID int64, ad ticketvote.AuthDetails) err func (p *ticketVotePlugin) auths(treeID int64) ([]ticketvote.AuthDetails, error) { // Retrieve blobs - blobs, err := p.tstore.BlobsByDataDesc(treeID, dataDescriptorAuthDetails) + blobs, err := p.tstore.BlobsByDataDesc(treeID, + []string{dataDescriptorAuthDetails}) if err != nil { return nil, err } @@ -347,7 +129,8 @@ func (p *ticketVotePlugin) voteDetailsSave(treeID int64, vd ticketvote.VoteDetai func (p *ticketVotePlugin) voteDetails(treeID int64) (*ticketvote.VoteDetails, error) { // Retrieve blobs - blobs, err := p.tstore.BlobsByDataDesc(treeID, dataDescriptorVoteDetails) + blobs, err := p.tstore.BlobsByDataDesc(treeID, + []string{dataDescriptorVoteDetails}) if err != nil { return nil, err } @@ -398,59 +181,166 @@ func (p *ticketVotePlugin) castVoteSave(treeID int64, cv ticketvote.CastVoteDeta return p.tstore.BlobSave(treeID, *be) } -func (p *ticketVotePlugin) castVotes(treeID int64) ([]ticketvote.CastVoteDetails, error) { +func (p *ticketVotePlugin) voteResults(treeID int64) ([]ticketvote.CastVoteDetails, error) { // Retrieve blobs - blobs, err := p.tstore.BlobsByDataDesc(treeID, - dataDescriptorCastVoteDetails) + desc := []string{ + dataDescriptorCastVoteDetails, + dataDescriptorVoteCollider, + } + blobs, err := p.tstore.BlobsByDataDesc(treeID, desc) if err != nil { return nil, err } - // Decode blobs - votes := make([]ticketvote.CastVoteDetails, 0, len(blobs)) - for _, v := range blobs { - cv, err := convertCastVoteDetailsFromBlobEntry(v) - if err != nil { - return nil, err - } - votes = append(votes, *cv) - } - - // Sort by ticket hash - sort.SliceStable(votes, func(i, j int) bool { - return votes[i].Ticket < votes[j].Ticket - }) - - return votes, nil -} - -func (p *ticketVotePlugin) voteOptionResults(token []byte, options []ticketvote.VoteOption) ([]ticketvote.VoteOptionResult, error) { - // Ongoing votes will have the cast votes cached. Calculate the - // results using the cached votes if we can since it will be much - // faster. + // Decode blobs. A cast vote is considered valid only if the vote + // collider exists for it. If there are multiple votes using the + // same ticket, the valid vote is the one that immediately precedes + // the vote collider blob entry. var ( - tally = make(map[string]uint32, len(options)) - t = hex.EncodeToString(token) - ctally = p.activeVotes.Tally(t) - ) - switch { - case len(ctally) > 0: - // Vote are in the cache. Use the cached results. - tally = ctally + // map[ticket]CastVoteDetails + votes = make(map[string]ticketvote.CastVoteDetails, len(blobs)) - default: - // Votes are not in the cache. Pull them from the backend. - reply, err := p.backend.PluginRead(token, ticketvote.PluginID, - ticketvote.CmdResults, "") + voteIndexes = make(map[string][]int, len(blobs)) // map[ticket][]index + colliderIndexes = make(map[string]int, len(blobs)) // map[ticket]index + ) + for i, v := range blobs { + // Decode data hint + b, err := base64.StdEncoding.DecodeString(v.DataHint) if err != nil { return nil, err } - var rr ticketvote.ResultsReply - err = json.Unmarshal([]byte(reply), &rr) + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) if err != nil { return nil, err } - + switch dd.Descriptor { + case dataDescriptorCastVoteDetails: + // Decode cast vote + cv, err := convertCastVoteDetailsFromBlobEntry(v) + if err != nil { + return nil, err + } + + // Save index of the cast vote + idx, ok := voteIndexes[cv.Ticket] + if !ok { + idx = make([]int, 0, 32) + } + idx = append(idx, i) + voteIndexes[cv.Ticket] = idx + + // Save the cast vote + votes[cv.Ticket] = *cv + + case dataDescriptorVoteCollider: + // Decode vote collider + vc, err := convertVoteColliderFromBlobEntry(v) + if err != nil { + return nil, err + } + + // Sanity check + _, ok := colliderIndexes[vc.Ticket] + if ok { + return nil, fmt.Errorf("duplicate vote colliders found %v", vc.Ticket) + } + + // Save the ticket and index for the collider + colliderIndexes[vc.Ticket] = i + + default: + return nil, fmt.Errorf("invalid data descriptor: %v", dd.Descriptor) + } + } + + for ticket, indexes := range voteIndexes { + // Remove any votes that do not have a collider blob + colliderIndex, ok := colliderIndexes[ticket] + if !ok { + // This is not a valid vote + delete(votes, ticket) + continue + } + + // If multiple votes have been cast using the same ticket then + // we must manually determine which vote is valid. + if len(indexes) == 1 { + // Only one cast vote exists for this ticket. This is good. + continue + } + + // Sanity check + if len(indexes) == 0 { + return nil, fmt.Errorf("no cast vote index found %v", ticket) + } + + log.Tracef("Multiple votes found for a single vote collider %v", ticket) + + // Multiple votes exist for this ticket. The vote that is valid + // is the one that immediately precedes the vote collider. Start + // at the end of the vote indexes and find the first vote index + // that precedes the collider index. + var validVoteIndex int + for i := len(indexes) - 1; i >= 0; i-- { + voteIndex := indexes[i] + if voteIndex < colliderIndex { + // This is the valid vote + validVoteIndex = voteIndex + break + } + } + + // Save the valid vote + b := blobs[validVoteIndex] + cv, err := convertCastVoteDetailsFromBlobEntry(b) + if err != nil { + return nil, err + } + votes[cv.Ticket] = *cv + } + + // Put votes into an array + cvotes := make([]ticketvote.CastVoteDetails, 0, len(blobs)) + for _, v := range votes { + cvotes = append(cvotes, v) + } + + // Sort by ticket hash + sort.SliceStable(cvotes, func(i, j int) bool { + return cvotes[i].Ticket < cvotes[j].Ticket + }) + + return cvotes, nil +} + +func (p *ticketVotePlugin) voteOptionResults(token []byte, options []ticketvote.VoteOption) ([]ticketvote.VoteOptionResult, error) { + // Ongoing votes will have the cast votes cached. Calculate the + // results using the cached votes if we can since it will be much + // faster. + var ( + tally = make(map[string]uint32, len(options)) + t = hex.EncodeToString(token) + ctally = p.activeVotes.Tally(t) + ) + switch { + case len(ctally) > 0: + // Vote are in the cache. Use the cached results. + tally = ctally + + default: + // Votes are not in the cache. Pull them from the backend. + reply, err := p.backend.PluginRead(token, ticketvote.PluginID, + ticketvote.CmdResults, "") + if err != nil { + return nil, err + } + var rr ticketvote.ResultsReply + err = json.Unmarshal([]byte(reply), &rr) + if err != nil { + return nil, err + } + // Tally the results for _, v := range rr.Votes { tally[v.VoteBit]++ @@ -1480,7 +1370,8 @@ func (p *ticketVotePlugin) startRunoffRecordSave(treeID int64, srr startRunoffRe // startRunoffRecord returns the startRunoff record if one exists on a tree. // nil will be returned if a startRunoff record is not found. func (p *ticketVotePlugin) startRunoffRecord(treeID int64) (*startRunoffRecord, error) { - blobs, err := p.tstore.BlobsByDataDesc(treeID, dataDescriptorStartRunoff) + blobs, err := p.tstore.BlobsByDataDesc(treeID, + []string{dataDescriptorStartRunoff}) if err != nil { return nil, fmt.Errorf("BlobsByDataDesc %v %v: %v", treeID, dataDescriptorStartRunoff, err) @@ -2110,11 +2001,33 @@ func (p *ticketVotePlugin) castVoteSignatureVerify(cv ticketvote.CastVote, addr return nil } +// voteCollider is used to prevent duplicate votes at the tlog level. The +// backend saves a digest of the data to the trillian log (tlog). Tlog does not +// allow leaves with duplicate values, so once a vote colider is saved to the +// backend for a ticket it should be impossible for another vote collider to be +// saved to the backend that is voting with the same ticket on the same record, +// regardless of what the vote bits are. The vote collider and the full cast +// vote are saved to the backend at the same time. A cast vote is not +// considered valid unless a corresponding vote collider is present. +type voteCollider struct { + Token string `json:"token"` // Record token + Ticket string `json:"ticket"` // Ticket hash +} + +func (p *ticketVotePlugin) voteColliderSave(treeID int64, vc voteCollider) error { + // Prepare blob + be, err := convertBlobEntryFromVoteCollider(vc) + if err != nil { + return err + } + + // Save blob + return p.tstore.BlobSave(treeID, *be) +} + // ballot casts the provided votes concurrently. The vote results are passed // back through the results channel to the calling function. This function // waits until all provided votes have been cast before returning. -// -// This function must be called WITH the record lock held. func (p *ticketVotePlugin) ballot(treeID int64, votes []ticketvote.CastVote, results chan ticketvote.CastVoteReply) { // Cast the votes concurrently var wg sync.WaitGroup @@ -2126,7 +2039,11 @@ func (p *ticketVotePlugin) ballot(treeID int64, votes []ticketvote.CastVote, res // Decrement wait group counter once vote is cast defer wg.Done() - // Setup cast vote details + // Setup cast vote details. The timestamp is included in a cast + // vote so that the vote can be retried if the vote collider + // is not saved succesfully. Without the timestamp, attempting + // to save the cast vote details multiple times would result in + // a duplicate leaf error from tlog. receipt := p.identity.SignMessage([]byte(v.Signature)) cv := ticketvote.CastVoteDetails{ Token: v.Token, @@ -2134,10 +2051,16 @@ func (p *ticketVotePlugin) ballot(treeID int64, votes []ticketvote.CastVote, res VoteBit: v.VoteBit, Signature: v.Signature, Receipt: hex.EncodeToString(receipt[:]), + Timestamp: time.Now().Unix(), } + // Declare variables here to prevent goto errors + var ( + cvr ticketvote.CastVoteReply + vc voteCollider + ) + // Save cast vote - var cvr ticketvote.CastVoteReply err := p.castVoteSave(treeID, cv) if err != nil { t := time.Now().Unix() @@ -2150,6 +2073,23 @@ func (p *ticketVotePlugin) ballot(treeID int64, votes []ticketvote.CastVote, res goto sendResult } + // Save vote collider + vc = voteCollider{ + Token: v.Token, + Ticket: v.Ticket, + } + err = p.voteColliderSave(treeID, vc) + if err != nil { + t := time.Now().Unix() + log.Errorf("cmdCastBallot: voteColliderSave %v: %v", t, err) + e := ticketvote.VoteErrorInternalError + cvr.Ticket = v.Ticket + cvr.ErrorCode = e + cvr.ErrorContext = fmt.Sprintf("%v: %v", + ticketvote.VoteErrors[e], t) + goto sendResult + } + // Update receipt cvr.Ticket = v.Ticket cvr.Receipt = cv.Receipt @@ -2375,29 +2315,30 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str // to be fully appended before considering the trillian call // successful. A person casting hundreds of votes in a single ballot // would cause UX issues for all the voting clients since the backend - // lock for this record is held during these calls. + // locks the record during any plugin write calls. Only one ballot + // can be cast at a time. // // The second variable that we must watch out for is the max trillian // queued leaf batch size. This is also a configurable trillian value // that represents the maximum number of leaves that can be waiting // in the queue for all trees in the trillian instance. This value is - // typically around the order of magnitude of 1000 queued leaves. + // typically around the order of magnitude of 1000s of queued leaves. // // The third variable that can cause errors is reaching the trillian // datastore max connection limits. Each vote being cast creates a - // trillian connection. Overloading the trillian connections will + // trillian connection. Overloading the trillian connections can // cause max connection exceeded errors. The max allowed connections // is a configurable trillian value, but should also be adjusted on - // the key-value store too. + // the key-value store database itself as well. // - // This is why a vote batch size of 5 was chosen. It is large enough + // This is why a vote batch size of 10 was chosen. It is large enough // to alleviate performance bottlenecks from the log signer interval, - // but small enough to still allow multiple records votes be held + // but small enough to still allow multiple records votes to be held // concurrently without running into the queued leaf batch size limit. // Prepare work var ( - batchSize = 5 + batchSize = 10 batch = make([]ticketvote.CastVote, 0, batchSize) queue = make([][]ticketvote.CastVote, 0, len(votes)/batchSize) @@ -2514,8 +2455,8 @@ func (p *ticketVotePlugin) cmdDetails(treeID int64, token []byte) (string, error func (p *ticketVotePlugin) cmdResults(treeID int64, token []byte) (string, error) { log.Tracef("cmdResults: %v %x", treeID, token) - // Get cast votes - votes, err := p.castVotes(treeID) + // Get vote results + votes, err := p.voteResults(treeID) if err != nil { return "", err } @@ -2683,7 +2624,7 @@ func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload str case t.VotesPage > 0: // Return a page of vote timestamps digests, err := p.tstore.DigestsByDataDesc(treeID, - dataDescriptorCastVoteDetails) + []string{dataDescriptorCastVoteDetails}) if err != nil { return "", fmt.Errorf("digestsByKeyPrefix %v %v: %v", treeID, dataDescriptorVoteDetails, err) @@ -2710,7 +2651,7 @@ func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload str // Auth timestamps digests, err := p.tstore.DigestsByDataDesc(treeID, - dataDescriptorAuthDetails) + []string{dataDescriptorAuthDetails}) if err != nil { return "", fmt.Errorf("DigestByDataDesc %v %v: %v", treeID, dataDescriptorAuthDetails, err) @@ -2726,7 +2667,7 @@ func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload str // Vote details timestamp digests, err = p.tstore.DigestsByDataDesc(treeID, - dataDescriptorVoteDetails) + []string{dataDescriptorVoteDetails}) if err != nil { return "", fmt.Errorf("DigestsByDataDesc %v %v: %v", treeID, dataDescriptorVoteDetails, err) @@ -2783,3 +2724,278 @@ func (p *ticketVotePlugin) cmdSubmissions(token []byte) (string, error) { return string(reply), nil } + +func convertAuthDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.AuthDetails, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorAuthDetails { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorAuthDetails) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, fmt.Errorf("decode digest: %v", err) + } + if !bytes.Equal(util.Digest(b), digest) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), digest) + } + var ad ticketvote.AuthDetails + err = json.Unmarshal(b, &ad) + if err != nil { + return nil, fmt.Errorf("unmarshal AuthDetails: %v", err) + } + + return &ad, nil +} + +func convertVoteDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.VoteDetails, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorVoteDetails { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorVoteDetails) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, fmt.Errorf("decode digest: %v", err) + } + if !bytes.Equal(util.Digest(b), digest) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), digest) + } + var vd ticketvote.VoteDetails + err = json.Unmarshal(b, &vd) + if err != nil { + return nil, fmt.Errorf("unmarshal VoteDetails: %v", err) + } + + return &vd, nil +} + +func convertCastVoteDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.CastVoteDetails, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorCastVoteDetails { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorCastVoteDetails) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, fmt.Errorf("decode digest: %v", err) + } + if !bytes.Equal(util.Digest(b), digest) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), digest) + } + var cv ticketvote.CastVoteDetails + err = json.Unmarshal(b, &cv) + if err != nil { + return nil, fmt.Errorf("unmarshal CastVoteDetails: %v", err) + } + + return &cv, nil +} + +func convertVoteColliderFromBlobEntry(be store.BlobEntry) (*voteCollider, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorVoteCollider { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorVoteCollider) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, fmt.Errorf("decode digest: %v", err) + } + if !bytes.Equal(util.Digest(b), digest) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), digest) + } + var vc voteCollider + err = json.Unmarshal(b, &vc) + if err != nil { + return nil, fmt.Errorf("unmarshal vote collider: %v", err) + } + + return &vc, nil +} + +func convertStartRunoffFromBlobEntry(be store.BlobEntry) (*startRunoffRecord, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorStartRunoff { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorStartRunoff) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, fmt.Errorf("decode digest: %v", err) + } + if !bytes.Equal(util.Digest(b), digest) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), digest) + } + var srr startRunoffRecord + err = json.Unmarshal(b, &srr) + if err != nil { + return nil, fmt.Errorf("unmarshal StartRunoffRecord: %v", err) + } + + return &srr, nil +} + +func convertBlobEntryFromAuthDetails(ad ticketvote.AuthDetails) (*store.BlobEntry, error) { + data, err := json.Marshal(ad) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorAuthDetails, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertBlobEntryFromVoteDetails(vd ticketvote.VoteDetails) (*store.BlobEntry, error) { + data, err := json.Marshal(vd) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorVoteDetails, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertBlobEntryFromCastVoteDetails(cv ticketvote.CastVoteDetails) (*store.BlobEntry, error) { + data, err := json.Marshal(cv) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorCastVoteDetails, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertBlobEntryFromVoteCollider(vc voteCollider) (*store.BlobEntry, error) { + data, err := json.Marshal(vc) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorVoteCollider, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertBlobEntryFromStartRunoff(srr startRunoffRecord) (*store.BlobEntry, error) { + data, err := json.Marshal(srr) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorStartRunoff, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go index f52806d67..7cd0abafc 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go @@ -89,7 +89,7 @@ func (p *ticketVotePlugin) Setup() error { } // Build active votes cache - log.Infof("Building votes cache") + log.Infof("Building active votes cache") started := make([]string, 0, len(inv.Entries)) for _, v := range inv.Entries { diff --git a/politeiad/backendv2/tstorebe/tstore/client.go b/politeiad/backendv2/tstorebe/tstore/client.go index 93a0f37e2..8e43269df 100644 --- a/politeiad/backendv2/tstorebe/tstore/client.go +++ b/politeiad/backendv2/tstorebe/tstore/client.go @@ -82,7 +82,7 @@ func (t *Tstore) BlobSave(treeID int64, be store.BlobEntry) error { key := storeKeyNew(encrypt) kv := map[string][]byte{key: blob} - log.Debugf("Saving plugin data blob") + log.Debugf("Saving plugin data blob %v", dd.Descriptor) // Save blob to store err = t.store.Put(kv, encrypt) @@ -254,12 +254,12 @@ func (t *Tstore) Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEnt return entries, nil } -// BlobsByDataDesc returns all blobs that match the provided data descriptor. +// BlobsByDataDesc returns all blobs that match the provided data descriptors. // The blobs will be ordered from oldest to newest. If a record is vetted then // only vetted blobs will be returned. // // This function satisfies the plugins TstoreClient interface. -func (t *Tstore) BlobsByDataDesc(treeID int64, dataDesc string) ([]store.BlobEntry, error) { +func (t *Tstore) BlobsByDataDesc(treeID int64, dataDesc []string) ([]store.BlobEntry, error) { log.Tracef("BlobsByDataDesc: %v %v", treeID, dataDesc) // Verify tree exists @@ -329,7 +329,7 @@ func (t *Tstore) BlobsByDataDesc(treeID int64, dataDesc string) ([]store.BlobEnt // returned. // // This function satisfies the plugins TstoreClient interface. -func (t *Tstore) DigestsByDataDesc(treeID int64, dataDesc string) ([][]byte, error) { +func (t *Tstore) DigestsByDataDesc(treeID int64, dataDesc []string) ([][]byte, error) { log.Tracef("DigestsByDataDesc: %v %v", treeID, dataDesc) // Verify tree exists @@ -403,7 +403,13 @@ func (t *Tstore) Timestamp(treeID int64, digest []byte) (*backend.Timestamp, err return t.timestamp(treeID, m, leaves) } -func leavesForDescriptor(leaves []*trillian.LogLeaf, desc string) []*trillian.LogLeaf { +func leavesForDescriptor(leaves []*trillian.LogLeaf, descriptors []string) []*trillian.LogLeaf { + // Put descriptors into a map for 0(n) lookups + desc := make(map[string]struct{}, len(descriptors)) + for _, v := range descriptors { + desc[v] = struct{}{} + } + // Determine if the record is vetted. If the record is vetted then // only vetted leaves will be returned. isVetted := recordIsVetted(leaves) @@ -416,17 +422,17 @@ func leavesForDescriptor(leaves []*trillian.LogLeaf, desc string) []*trillian.Lo if err != nil { panic(err) } - switch { - case ed.Desc != desc: - // Not the data descriptor we're looking for + if _, ok := desc[ed.Desc]; !ok { + // Not one of the data descriptor we're looking for continue - case isVetted && ed.State != backend.StateVetted: + } + if isVetted && ed.State != backend.StateVetted { // Unvetted leaf on a vetted record. Don't use it. continue - default: - // We have a match! - matches = append(matches, v) } + + // We have a match! + matches = append(matches, v) } return matches diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index cbc51ea07..40e9b0fd1 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -349,7 +349,8 @@ type CastVoteDetails struct { Signature string `json:"signature"` // Client signature // Metdata generated by server - Receipt string `json:"receipt"` // Server signature + Timestamp int64 `json:"timestamp"` + Receipt string `json:"receipt"` // Server signature } // AuthActionT represents the ticket vote authorization actions. diff --git a/politeiad/v2.go b/politeiad/v2.go index aa9e51df7..03a85939e 100644 --- a/politeiad/v2.go +++ b/politeiad/v2.go @@ -474,6 +474,9 @@ func (p *politeia) handlePluginWrite(w http.ResponseWriter, r *http.Request) { Payload: payload, } + log.Infof("%v Plugin '%v' write cmd '%v' executed", + util.RemoteAddr(r), pw.Cmd.ID, pw.Cmd.Command) + util.RespondWithJSON(w, http.StatusOK, pwr) } From 4e7eaf801d95775eaef8e8949a42ab477cd025ed Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 15 Mar 2021 10:09:40 -0500 Subject: [PATCH 399/449] tstorebe/store: Verify unique nonces. --- .../backendv2/tstorebe/store/mysql/encrypt.go | 52 +++++---- .../backendv2/tstorebe/store/mysql/nonce.go | 106 ++++++++++++++++++ 2 files changed, 139 insertions(+), 19 deletions(-) create mode 100644 politeiad/backendv2/tstorebe/store/mysql/nonce.go diff --git a/politeiad/backendv2/tstorebe/store/mysql/encrypt.go b/politeiad/backendv2/tstorebe/store/mysql/encrypt.go index f87e2f372..410937309 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/encrypt.go +++ b/politeiad/backendv2/tstorebe/store/mysql/encrypt.go @@ -75,57 +75,71 @@ func (s *mysql) argon2idKey(password string) (*[32]byte, error) { return &key, nil } -func (s *mysql) encrypt(ctx context.Context, tx *sql.Tx, data []byte) ([]byte, error) { - // Create a new nonce value +func (s *mysql) insertNonce(ctx context.Context, tx *sql.Tx) error { _, err := tx.ExecContext(ctx, "INSERT INTO nonce () VALUES ();") - if err != nil { - return nil, err - } + return err +} - // Get the nonce value that was just created +func (s *mysql) queryNonce(ctx context.Context, tx *sql.Tx) (int64, error) { rows, err := tx.QueryContext(ctx, "SELECT LAST_INSERT_ID();") if err != nil { - return nil, fmt.Errorf("query: %v", err) + return 0, fmt.Errorf("query: %v", err) } defer rows.Close() - var i int64 + var nonce int64 for rows.Next() { - if i > 0 { + if nonce > 0 { // There should only ever be one row returned. Something is // wrong if we've already scanned the nonce and its still // scanning rows. - return nil, fmt.Errorf("multiple rows returned for nonce") + return 0, fmt.Errorf("multiple rows returned for nonce") } - err = rows.Scan(&i) + err = rows.Scan(&nonce) if err != nil { - return nil, fmt.Errorf("scan: %v", err) + return 0, fmt.Errorf("scan: %v", err) } } err = rows.Err() if err != nil { - return nil, fmt.Errorf("next: %v", err) + return 0, fmt.Errorf("next: %v", err) + } + if nonce == 0 { + return 0, fmt.Errorf("invalid 0 nonce") + } + + return nonce, nil +} + +func (s *mysql) encrypt(ctx context.Context, tx *sql.Tx, data []byte) ([]byte, error) { + // Create a new nonce value + err := s.insertNonce(ctx, tx) + if err != nil { + return nil, fmt.Errorf("insert nonce: %v", err) } - if i == 0 { - return nil, fmt.Errorf("invalid 0 nonce") + + // Get the nonce value that was just created + nonce, err := s.queryNonce(ctx, tx) + if err != nil { + return nil, fmt.Errorf("query nonce: %v", err) } - log.Tracef("Encrypting with nonce: %v", i) + log.Tracef("Encrypting with nonce: %v", nonce) // Prepare nonce b := make([]byte, 8) - binary.LittleEndian.PutUint64(b, uint64(i)) + binary.LittleEndian.PutUint64(b, uint64(nonce)) n, err := sbox.NewNonceFromBytes(b) if err != nil { return nil, err } - nonce := n.Current() + nonceb := n.Current() // Encrypt blob s.RLock() defer s.RUnlock() - return sbox.EncryptN(0, s.key, nonce, data) + return sbox.EncryptN(0, s.key, nonceb, data) } func (s *mysql) decrypt(data []byte) ([]byte, uint32, error) { diff --git a/politeiad/backendv2/tstorebe/store/mysql/nonce.go b/politeiad/backendv2/tstorebe/store/mysql/nonce.go new file mode 100644 index 000000000..b43a685b9 --- /dev/null +++ b/politeiad/backendv2/tstorebe/store/mysql/nonce.go @@ -0,0 +1,106 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package mysql + +import ( + "context" + "database/sql" + "fmt" + "strconv" + "sync" +) + +// testNonce is used to verify that nonce races do not occur. This function +// is meant to be run against an actual MySQL/MariaDB instance, not as a unit +// test. +func (s *mysql) testNonce(ctx context.Context, tx *sql.Tx) error { + // Create a new nonce value + err := s.insertNonce(ctx, tx) + if err != nil { + return fmt.Errorf("insert nonce: %v", err) + } + + // Get the nonce value that was just created + nonce, err := s.queryNonce(ctx, tx) + if err != nil { + return fmt.Errorf("query nonce: %v", err) + } + + // Save a empty blob to the kv store using the nonce as the key. + // If a nonce is reused it will cause an error since the key must + // be unique. + k := strconv.FormatInt(nonce, 10) + _, err = tx.ExecContext(ctx, + "INSERT INTO kv (k, v) VALUES (?, ?);", k, []byte{}) + if err != nil { + return fmt.Errorf("exec put: %v", err) + } + + return nil +} + +// testNonceIsUnique verifies that nonce races do not occur. This function +// is meant to be run against an actual MySQL/MariaDB instance, not as a unit +// test. +func (s *mysql) testNonceIsUnique() error { + log.Infof("Starting nonce concurrency test") + + // Run test + var ( + wg sync.WaitGroup + threads = 1000 + ) + for i := 0; i < threads; i++ { + // Increment the wait group counter + wg.Add(1) + + go func() { + // Decrement wait group counter on exit + defer wg.Done() + + ctx, cancel := ctxWithTimeout() + defer cancel() + + // Start transaction + opts := &sql.TxOptions{ + Isolation: sql.LevelDefault, + } + tx, err := s.db.BeginTx(ctx, opts) + if err != nil { + log.Errorf("begin tx: %v", err) + return + } + + // Save blobs + err = s.testNonce(ctx, tx) + if err != nil { + // Attempt to roll back the transaction + if err2 := tx.Rollback(); err2 != nil { + // We're in trouble! + log.Errorf("testNonce: %v, unable to rollback: %v", err, err2) + return + } + log.Errorf("testNonce: %v", err) + return + } + + // Commit transaction + err = tx.Commit() + if err != nil { + log.Errorf("commit tx: %v", err) + return + } + }() + } + + log.Infof("Waiting for nonce concurrency test to complete...") + + // Wait for all tests to complete + wg.Wait() + + log.Infof("Nonce concurrency test success!") + + return nil +} From 48491246bf888f7ab50bd80bbbe709d41a5789c2 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 15 Mar 2021 12:51:10 -0500 Subject: [PATCH 400/449] tstorebe/mysql: Cleanup. --- .../backendv2/tstorebe/store/mysql/encrypt.go | 25 +++++++++---------- .../backendv2/tstorebe/store/mysql/nonce.go | 25 +++++++------------ 2 files changed, 21 insertions(+), 29 deletions(-) diff --git a/politeiad/backendv2/tstorebe/store/mysql/encrypt.go b/politeiad/backendv2/tstorebe/store/mysql/encrypt.go index 410937309..aaf7dc27b 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/encrypt.go +++ b/politeiad/backendv2/tstorebe/store/mysql/encrypt.go @@ -75,12 +75,17 @@ func (s *mysql) argon2idKey(password string) (*[32]byte, error) { return &key, nil } -func (s *mysql) insertNonce(ctx context.Context, tx *sql.Tx) error { +// nonce returns a new nonce value. This function guarantees that the returned +// nonce will be unique for every invocation. +// +// This function must be called using a transaction. +func (s *mysql) nonce(ctx context.Context, tx *sql.Tx) (int64, error) { _, err := tx.ExecContext(ctx, "INSERT INTO nonce () VALUES ();") - return err -} + if err != nil { + return 0, fmt.Errorf("insert: %v", err) + } -func (s *mysql) queryNonce(ctx context.Context, tx *sql.Tx) (int64, error) { + // Get the nonce value that was just created rows, err := tx.QueryContext(ctx, "SELECT LAST_INSERT_ID();") if err != nil { return 0, fmt.Errorf("query: %v", err) @@ -112,16 +117,10 @@ func (s *mysql) queryNonce(ctx context.Context, tx *sql.Tx) (int64, error) { } func (s *mysql) encrypt(ctx context.Context, tx *sql.Tx, data []byte) ([]byte, error) { - // Create a new nonce value - err := s.insertNonce(ctx, tx) + // Get nonce value + nonce, err := s.nonce(ctx, tx) if err != nil { - return nil, fmt.Errorf("insert nonce: %v", err) - } - - // Get the nonce value that was just created - nonce, err := s.queryNonce(ctx, tx) - if err != nil { - return nil, fmt.Errorf("query nonce: %v", err) + return nil, err } log.Tracef("Encrypting with nonce: %v", nonce) diff --git a/politeiad/backendv2/tstorebe/store/mysql/nonce.go b/politeiad/backendv2/tstorebe/store/mysql/nonce.go index b43a685b9..1ec161587 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/nonce.go +++ b/politeiad/backendv2/tstorebe/store/mysql/nonce.go @@ -12,23 +12,17 @@ import ( "sync" ) -// testNonce is used to verify that nonce races do not occur. This function -// is meant to be run against an actual MySQL/MariaDB instance, not as a unit +// testNonce is used to verify that nonce races do not occur. This function is +// meant to be run against an actual MySQL/MariaDB instance, not as a unit // test. func (s *mysql) testNonce(ctx context.Context, tx *sql.Tx) error { - // Create a new nonce value - err := s.insertNonce(ctx, tx) + // Get nonce + nonce, err := s.nonce(ctx, tx) if err != nil { - return fmt.Errorf("insert nonce: %v", err) + return fmt.Errorf("nonce: %v", err) } - // Get the nonce value that was just created - nonce, err := s.queryNonce(ctx, tx) - if err != nil { - return fmt.Errorf("query nonce: %v", err) - } - - // Save a empty blob to the kv store using the nonce as the key. + // Save an empty blob to the kv store using the nonce as the key. // If a nonce is reused it will cause an error since the key must // be unique. k := strconv.FormatInt(nonce, 10) @@ -44,7 +38,7 @@ func (s *mysql) testNonce(ctx context.Context, tx *sql.Tx) error { // testNonceIsUnique verifies that nonce races do not occur. This function // is meant to be run against an actual MySQL/MariaDB instance, not as a unit // test. -func (s *mysql) testNonceIsUnique() error { +func (s *mysql) testNonceIsUnique() { log.Infof("Starting nonce concurrency test") // Run test @@ -73,7 +67,7 @@ func (s *mysql) testNonceIsUnique() error { return } - // Save blobs + // Run nonce test err = s.testNonce(ctx, tx) if err != nil { // Attempt to roll back the transaction @@ -100,7 +94,6 @@ func (s *mysql) testNonceIsUnique() error { // Wait for all tests to complete wg.Wait() - log.Infof("Nonce concurrency test success!") + log.Infof("Nonce concurrency test complete") - return nil } From a03c4c2d277ee5570a8983a7b1c9b54a10eb19fa Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 15 Mar 2021 13:35:01 -0500 Subject: [PATCH 401/449] multi: Cleanup types. --- politeiad/api/v2/v2.go | 6 +- politeiad/backendv2/backendv2.go | 2 +- .../tstorebe/plugins/comments/cmds.go | 56 ++++++------ .../backendv2/tstorebe/plugins/pi/hooks.go | 18 ++-- .../tstorebe/plugins/ticketvote/cmds.go | 88 +++++++++---------- .../tstorebe/plugins/ticketvote/hooks.go | 24 ++--- .../tstorebe/plugins/usermd/hooks.go | 22 ++--- politeiad/client/client.go | 2 +- politeiad/client/pdv2.go | 2 +- politeiad/plugins/comments/comments.go | 4 +- politeiad/plugins/dcrdata/dcrdata.go | 2 +- politeiad/plugins/pi/pi.go | 2 +- politeiad/plugins/ticketvote/ticketvote.go | 8 +- politeiad/plugins/usermd/usermd.go | 2 +- politeiawww/api/comments/v1/v1.go | 10 +-- politeiawww/api/records/v1/v1.go | 8 +- politeiawww/api/ticketvote/v1/v1.go | 12 +-- politeiawww/comments/error.go | 2 +- politeiawww/{piwww.go => pi.go} | 0 politeiawww/records/error.go | 2 +- politeiawww/ticketvote/error.go | 2 +- 21 files changed, 137 insertions(+), 137 deletions(-) rename politeiawww/{piwww.go => pi.go} (100%) diff --git a/politeiad/api/v2/v2.go b/politeiad/api/v2/v2.go index 5c7336116..78eaeaf6e 100644 --- a/politeiad/api/v2/v2.go +++ b/politeiad/api/v2/v2.go @@ -80,7 +80,7 @@ var ( // timing, etc). The HTTP status code will be 400. type UserErrorReply struct { ErrorCode ErrorCodeT `json:"errorcode"` - ErrorContext string `json:"errorcontext"` + ErrorContext string `json:"errorcontext,omitempty"` } // Error satisfies the error interface. @@ -92,8 +92,8 @@ func (e UserErrorReply) Error() string { // a plugin error. The error code will be specific to the plugin. type PluginErrorReply struct { PluginID string `json:"pluginid"` - ErrorCode int `json:"errorcode"` - ErrorContext string `json:"errorcontext"` + ErrorCode uint32 `json:"errorcode"` + ErrorContext string `json:"errorcontext,omitempty"` } // Error satisfies the error interface. diff --git a/politeiad/backendv2/backendv2.go b/politeiad/backendv2/backendv2.go index e80950d0a..0504a05b7 100644 --- a/politeiad/backendv2/backendv2.go +++ b/politeiad/backendv2/backendv2.go @@ -264,7 +264,7 @@ type Plugin struct { // was caused by the user. type PluginError struct { PluginID string - ErrorCode int + ErrorCode uint32 ErrorContext string } diff --git a/politeiad/backendv2/tstorebe/plugins/comments/cmds.go b/politeiad/backendv2/tstorebe/plugins/comments/cmds.go index 74f6f7f09..4847c9476 100644 --- a/politeiad/backendv2/tstorebe/plugins/comments/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/comments/cmds.go @@ -47,7 +47,7 @@ func convertSignatureError(err error) backend.PluginError { } return backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(s), + ErrorCode: uint32(s), ErrorContext: e.ErrorContext, } } @@ -605,7 +605,7 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str if err != nil { return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeTokenInvalid), + ErrorCode: uint32(comments.ErrorCodeTokenInvalid), ErrorContext: err.Error(), } } @@ -614,7 +614,7 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str "got %x, want %x", t, token) return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeTokenInvalid), + ErrorCode: uint32(comments.ErrorCodeTokenInvalid), ErrorContext: e, } } @@ -632,7 +632,7 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str e := fmt.Sprintf("max length is %v characters", p.commentLengthMax) return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeMaxLengthExceeded), + ErrorCode: uint32(comments.ErrorCodeMaxLengthExceeded), ErrorContext: e, } } @@ -646,7 +646,7 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str e := fmt.Sprintf("got %v, want %v", n.State, state) return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeStateInvalid), + ErrorCode: uint32(comments.ErrorCodeStateInvalid), ErrorContext: e, } } @@ -662,7 +662,7 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str if n.ParentID > 0 && !commentExists(*ridx, n.ParentID) { return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeParentIDInvalid), + ErrorCode: uint32(comments.ErrorCodeParentIDInvalid), ErrorContext: "parent ID comment not found", } } @@ -742,7 +742,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st if err != nil { return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeTokenInvalid), + ErrorCode: uint32(comments.ErrorCodeTokenInvalid), ErrorContext: err.Error(), } } @@ -751,7 +751,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st "got %x, want %x", t, token) return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeTokenInvalid), + ErrorCode: uint32(comments.ErrorCodeTokenInvalid), ErrorContext: e, } } @@ -769,7 +769,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st e := fmt.Sprintf("max length is %v characters", p.commentLengthMax) return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeMaxLengthExceeded), + ErrorCode: uint32(comments.ErrorCodeMaxLengthExceeded), ErrorContext: e, } } @@ -783,7 +783,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st e := fmt.Sprintf("got %v, want %v", e.State, state) return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeStateInvalid), + ErrorCode: uint32(comments.ErrorCodeStateInvalid), ErrorContext: e, } } @@ -803,7 +803,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st if !ok { return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeCommentNotFound), + ErrorCode: uint32(comments.ErrorCodeCommentNotFound), } } @@ -811,7 +811,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st if e.UserID != existing.UserID { return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeUserUnauthorized), + ErrorCode: uint32(comments.ErrorCodeUserUnauthorized), } } @@ -821,7 +821,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st e.ParentID, existing.ParentID) return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeParentIDInvalid), + ErrorCode: uint32(comments.ErrorCodeParentIDInvalid), ErrorContext: e, } } @@ -830,7 +830,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st if e.Comment == existing.Comment { return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeNoChanges), + ErrorCode: uint32(comments.ErrorCodeNoChanges), } } @@ -903,7 +903,7 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str if err != nil { return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeTokenInvalid), + ErrorCode: uint32(comments.ErrorCodeTokenInvalid), ErrorContext: err.Error(), } } @@ -912,7 +912,7 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str "got %x, want %x", t, token) return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeTokenInvalid), + ErrorCode: uint32(comments.ErrorCodeTokenInvalid), ErrorContext: e, } } @@ -934,7 +934,7 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str e := fmt.Sprintf("got %v, want %v", d.State, state) return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeStateInvalid), + ErrorCode: uint32(comments.ErrorCodeStateInvalid), ErrorContext: e, } } @@ -954,7 +954,7 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str if !ok { return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeCommentNotFound), + ErrorCode: uint32(comments.ErrorCodeCommentNotFound), } } @@ -1038,7 +1038,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st if err != nil { return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeTokenInvalid), + ErrorCode: uint32(comments.ErrorCodeTokenInvalid), ErrorContext: err.Error(), } } @@ -1047,7 +1047,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st "got %x, want %x", t, token) return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeTokenInvalid), + ErrorCode: uint32(comments.ErrorCodeTokenInvalid), ErrorContext: e, } } @@ -1059,7 +1059,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st default: return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeVoteInvalid), + ErrorCode: uint32(comments.ErrorCodeVoteInvalid), } } @@ -1081,7 +1081,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st e := fmt.Sprintf("got %v, want %v", v.State, state) return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeStateInvalid), + ErrorCode: uint32(comments.ErrorCodeStateInvalid), ErrorContext: e, } } @@ -1097,7 +1097,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st if !ok { return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeCommentNotFound), + ErrorCode: uint32(comments.ErrorCodeCommentNotFound), } } @@ -1109,7 +1109,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st if len(uvotes) > int(p.voteChangesMax) { return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeVoteChangesMaxExceeded), + ErrorCode: uint32(comments.ErrorCodeVoteChangesMaxExceeded), } } @@ -1125,7 +1125,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st if v.UserID == c.UserID { return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeVoteInvalid), + ErrorCode: uint32(comments.ErrorCodeVoteInvalid), ErrorContext: "user cannot vote on their own comment", } } @@ -1303,13 +1303,13 @@ func (p *commentsPlugin) cmdGetVersion(treeID int64, token []byte, payload strin if !ok { return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeCommentNotFound), + ErrorCode: uint32(comments.ErrorCodeCommentNotFound), } } if cidx.Del != nil { return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeCommentNotFound), + ErrorCode: uint32(comments.ErrorCodeCommentNotFound), ErrorContext: "comment has been deleted", } } @@ -1319,7 +1319,7 @@ func (p *commentsPlugin) cmdGetVersion(treeID int64, token []byte, payload strin gv.CommentID, gv.Version) return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: int(comments.ErrorCodeCommentNotFound), + ErrorCode: uint32(comments.ErrorCodeCommentNotFound), ErrorContext: e, } } diff --git a/politeiad/backendv2/tstorebe/plugins/pi/hooks.go b/politeiad/backendv2/tstorebe/plugins/pi/hooks.go index 16123981d..832add12b 100644 --- a/politeiad/backendv2/tstorebe/plugins/pi/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/pi/hooks.go @@ -86,7 +86,7 @@ func (p *piPlugin) proposalFilesVerify(files []backend.File) error { if !ok { return backend.PluginError{ PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeTextFileNameInvalid), + ErrorCode: uint32(pi.ErrorCodeTextFileNameInvalid), ErrorContext: v.Name, } } @@ -97,7 +97,7 @@ func (p *piPlugin) proposalFilesVerify(files []backend.File) error { v.Name, len(payload), p.textFileSizeMax) return backend.PluginError{ PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeTextFileSizeInvalid), + ErrorCode: uint32(pi.ErrorCodeTextFileSizeInvalid), ErrorContext: e, } } @@ -111,7 +111,7 @@ func (p *piPlugin) proposalFilesVerify(files []backend.File) error { v.Name, len(payload), p.imageFileSizeMax) return backend.PluginError{ PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeImageFileSizeInvalid), + ErrorCode: uint32(pi.ErrorCodeImageFileSizeInvalid), ErrorContext: e, } } @@ -132,7 +132,7 @@ func (p *piPlugin) proposalFilesVerify(files []backend.File) error { if !found { return backend.PluginError{ PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeTextFileMissing), + ErrorCode: uint32(pi.ErrorCodeTextFileMissing), ErrorContext: pi.FileNameIndexFile, } } @@ -143,7 +143,7 @@ func (p *piPlugin) proposalFilesVerify(files []backend.File) error { imagesCount, p.imageFileCountMax) return backend.PluginError{ PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeImageFileCountInvalid), + ErrorCode: uint32(pi.ErrorCodeImageFileCountInvalid), ErrorContext: e, } } @@ -156,7 +156,7 @@ func (p *piPlugin) proposalFilesVerify(files []backend.File) error { if pm == nil { return backend.PluginError{ PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeTextFileMissing), + ErrorCode: uint32(pi.ErrorCodeTextFileMissing), ErrorContext: pi.FileNameProposalMetadata, } } @@ -165,7 +165,7 @@ func (p *piPlugin) proposalFilesVerify(files []backend.File) error { if !p.proposalNameIsValid(pm.Name) { return backend.PluginError{ PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeProposalNameInvalid), + ErrorCode: uint32(pi.ErrorCodeProposalNameInvalid), ErrorContext: p.proposalNameRegexp.String(), } } @@ -211,7 +211,7 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { ticketvote.VoteStatuses[s.Status]) return backend.PluginError{ PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeVoteStatusInvalid), + ErrorCode: uint32(pi.ErrorCodeVoteStatusInvalid), ErrorContext: e, } } @@ -249,7 +249,7 @@ func (p *piPlugin) commentWritesVerify(token []byte) error { default: return backend.PluginError{ PluginID: pi.PluginID, - ErrorCode: int(pi.ErrorCodeVoteStatusInvalid), + ErrorCode: uint32(pi.ErrorCodeVoteStatusInvalid), ErrorContext: "vote has ended; proposal is locked", } } diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go index 3a923dd61..6c9217670 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go @@ -73,7 +73,7 @@ func convertSignatureError(err error) backend.PluginError { } return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(s), + ErrorCode: uint32(s), ErrorContext: e.ErrorContext, } } @@ -892,7 +892,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri if err != nil { return "", backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeTokenInvalid), } } if !bytes.Equal(t, token) { @@ -900,7 +900,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri "got %x, want %x", t, token) return "", backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeTokenInvalid), ErrorContext: e, } } @@ -923,7 +923,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri e := fmt.Sprintf("%v not a valid action", a.Action) return "", backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeAuthorizationInvalid), ErrorContext: e, } } @@ -936,7 +936,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri if r.RecordMetadata.Status != backend.StatusPublic { return "", backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeRecordStatusInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeRecordStatusInvalid), ErrorContext: "record is not public", } } @@ -945,7 +945,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri a.Version, r.RecordMetadata.Version) return "", backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeRecordVersionInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeRecordVersionInvalid), ErrorContext: e, } } @@ -966,7 +966,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri if a.Action != ticketvote.AuthActionAuthorize { return "", backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeAuthorizationInvalid), ErrorContext: "no prev action; action must be authorize", } } @@ -975,7 +975,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri // Previous action was a authorize. This action must be revoke. return "", backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeAuthorizationInvalid), ErrorContext: "prev action was authorize", } case prevAction == ticketvote.AuthActionRevoke && @@ -983,7 +983,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri // Previous action was a revoke. This action must be authorize. return "", backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeAuthorizationInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeAuthorizationInvalid), ErrorContext: "prev action was revoke", } } @@ -1066,7 +1066,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM default: return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteTypeInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeVoteTypeInvalid), } } @@ -1077,7 +1077,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM vote.Duration, voteDurationMax) return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteDurationInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeVoteDurationInvalid), ErrorContext: e, } case vote.Duration < voteDurationMin: @@ -1085,7 +1085,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM vote.Duration, voteDurationMin) return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteDurationInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeVoteDurationInvalid), ErrorContext: e, } case vote.QuorumPercentage > 100: @@ -1093,7 +1093,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM vote.QuorumPercentage) return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteQuorumInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeVoteQuorumInvalid), ErrorContext: e, } case vote.PassPercentage > 100: @@ -1101,7 +1101,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM vote.PassPercentage) return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVotePassRateInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeVotePassRateInvalid), ErrorContext: e, } } @@ -1111,7 +1111,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM if len(vote.Options) == 0 { return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteOptionsInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeVoteOptionsInvalid), ErrorContext: "no vote options found", } } @@ -1125,7 +1125,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM len(vote.Options)) return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteOptionsInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeVoteOptionsInvalid), ErrorContext: e, } } @@ -1154,7 +1154,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM strings.Join(missing, ",")) return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteOptionsInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeVoteOptionsInvalid), ErrorContext: e, } } @@ -1166,7 +1166,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM if err != nil { return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteBitsInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeVoteBitsInvalid), ErrorContext: err.Error(), } } @@ -1178,7 +1178,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM e := "parent token should not be provided for a standard vote" return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteParentInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), ErrorContext: e, } case vote.Type == ticketvote.VoteTypeRunoff: @@ -1187,7 +1187,7 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM e := fmt.Sprintf("invalid parent %v", vote.Parent) return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteParentInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), ErrorContext: e, } } @@ -1202,7 +1202,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot if len(s.Starts) != 1 { return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeStartDetailsInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeStartDetailsInvalid), ErrorContext: "more than one start details found for standard vote", } } @@ -1213,7 +1213,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot if err != nil { return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeTokenInvalid), } } if !bytes.Equal(t, token) { @@ -1221,7 +1221,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot "got %x, want %x", t, token) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeTokenInvalid), ErrorContext: e, } } @@ -1263,7 +1263,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot sd.Params.Version, r.RecordMetadata.Version) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeRecordVersionInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeRecordVersionInvalid), ErrorContext: e, } } @@ -1276,7 +1276,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot if len(auths) == 0 { return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteStatusInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeVoteStatusInvalid), ErrorContext: "not authorized", } } @@ -1284,7 +1284,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot if action != ticketvote.AuthActionAuthorize { return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteStatusInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeVoteStatusInvalid), ErrorContext: "not authorized", } } @@ -1298,7 +1298,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot // Vote has already been started return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteStatusInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeVoteStatusInvalid), ErrorContext: "vote already started", } } @@ -1487,7 +1487,7 @@ func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs sta sd.Params.Token, sd.Params.Version, r.RecordMetadata.Version) return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeRecordVersionInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeRecordVersionInvalid), ErrorContext: e, } } @@ -1555,7 +1555,7 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti e := fmt.Sprintf("parent record not found %x", token) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteParentInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), ErrorContext: e, } } @@ -1573,7 +1573,7 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti e := fmt.Sprintf("%x is not a runoff vote parent", token) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteParentInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), ErrorContext: e, } } @@ -1585,7 +1585,7 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti token, vm.LinkBy) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeLinkByNotExpired), + ErrorCode: uint32(ticketvote.ErrorCodeLinkByNotExpired), ErrorContext: e, } case !isExpired: @@ -1635,7 +1635,7 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti v.Params.Token) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeStartDetailsInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeStartDetailsInvalid), ErrorContext: e, } } @@ -1652,7 +1652,7 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti // This records is missing from the runoff vote return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeStartDetailsMissing), + ErrorCode: uint32(ticketvote.ErrorCodeStartDetailsMissing), ErrorContext: k, } } @@ -1709,7 +1709,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. v.Params.Token, v.Params.Type, ticketvote.VoteTypeRunoff) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteTypeInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeVoteTypeInvalid), ErrorContext: e, } case v.Params.Mask != mask: @@ -1717,7 +1717,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. v.Params.Token) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteBitsInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeVoteBitsInvalid), ErrorContext: e, } case v.Params.Duration != duration: @@ -1725,7 +1725,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. v.Params.Token) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteDurationInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeVoteDurationInvalid), ErrorContext: e, } case v.Params.QuorumPercentage != quorum: @@ -1733,7 +1733,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. v.Params.Token) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteQuorumInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeVoteQuorumInvalid), ErrorContext: e, } case v.Params.PassPercentage != pass: @@ -1741,7 +1741,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. v.Params.Token) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVotePassRateInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeVotePassRateInvalid), ErrorContext: e, } case v.Params.Parent != parent: @@ -1749,7 +1749,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. v.Params.Token) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteParentInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), ErrorContext: e, } } @@ -1759,7 +1759,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. if err != nil { return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeTokenInvalid), ErrorContext: v.Params.Token, } } @@ -1770,7 +1770,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. e := fmt.Sprintf("parent token %v", v.Params.Parent) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeTokenInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeTokenInvalid), ErrorContext: e, } } @@ -1800,7 +1800,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. parent) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteParentInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), ErrorContext: e, } } @@ -1888,7 +1888,7 @@ func (p *ticketVotePlugin) cmdStart(treeID int64, token []byte, payload string) if len(s.Starts) == 0 { return "", backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeStartDetailsMissing), + ErrorCode: uint32(ticketvote.ErrorCodeStartDetailsMissing), ErrorContext: "no start details found", } } @@ -1910,7 +1910,7 @@ func (p *ticketVotePlugin) cmdStart(treeID int64, token []byte, payload string) default: return "", backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteTypeInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeVoteTypeInvalid), } } diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go index f66030247..c78a33de1 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go @@ -50,7 +50,7 @@ func (p *ticketVotePlugin) linkByVerify(linkBy int64) error { linkBy, min) return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeLinkByInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeLinkByInvalid), ErrorContext: e, } case linkBy > max: @@ -58,7 +58,7 @@ func (p *ticketVotePlugin) linkByVerify(linkBy int64) error { linkBy, max) return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeLinkByInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeLinkByInvalid), ErrorContext: e, } } @@ -71,7 +71,7 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { if err != nil { return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeLinkToInvalid), ErrorContext: "invalid hex", } } @@ -80,7 +80,7 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { if err == backend.ErrRecordNotFound { return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeLinkToInvalid), ErrorContext: "record not found", } } @@ -92,7 +92,7 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { backend.Statuses[backend.StatusPublic]) return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeLinkToInvalid), ErrorContext: e, } } @@ -106,7 +106,7 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { if parentVM == nil || parentVM.LinkBy == 0 { return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeLinkToInvalid), ErrorContext: "record not a runoff vote parent", } } @@ -115,7 +115,7 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { if time.Now().Unix() > parentVM.LinkBy { return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeLinkToInvalid), ErrorContext: "parent record linkby deadline has expired", } } @@ -128,7 +128,7 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { if vs.Status != ticketvote.VoteStatusApproved { return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeLinkToInvalid), ErrorContext: "parent record vote is not approved", } } @@ -142,7 +142,7 @@ func (p *ticketVotePlugin) voteMetadataVerify(vm ticketvote.VoteMetadata) error // Vote metadata is empty return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteMetadataInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeVoteMetadataInvalid), ErrorContext: "metadata is empty", } @@ -150,7 +150,7 @@ func (p *ticketVotePlugin) voteMetadataVerify(vm ticketvote.VoteMetadata) error // LinkBy and LinkTo cannot both be set return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeVoteMetadataInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeVoteMetadataInvalid), ErrorContext: "cannot set both linkby and linkto", } @@ -226,7 +226,7 @@ func (p *ticketVotePlugin) hookEditRecordPre(payload string) error { "got '%v', want '%v'", newLinkTo, oldLinkTo) return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeLinkToInvalid), ErrorContext: e, } } @@ -291,7 +291,7 @@ func (p *ticketVotePlugin) hookSetRecordStatusPre(payload string) error { if time.Now().Unix() > vmParent.LinkBy { return backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: int(ticketvote.ErrorCodeLinkToInvalid), + ErrorCode: uint32(ticketvote.ErrorCodeLinkToInvalid), ErrorContext: "parent record linkby has expired", } } diff --git a/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go b/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go index 53eace038..8abb5f9f4 100644 --- a/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go @@ -33,7 +33,7 @@ func convertSignatureError(err error) backend.PluginError { } return backend.PluginError{ PluginID: usermd.PluginID, - ErrorCode: int(s), + ErrorCode: uint32(s), ErrorContext: e.ErrorContext, } } @@ -70,7 +70,7 @@ func userMetadataVerify(metadata []backend.MetadataStream, files []backend.File) if um == nil { return backend.PluginError{ PluginID: usermd.PluginID, - ErrorCode: int(usermd.ErrorCodeUserMetadataNotFound), + ErrorCode: uint32(usermd.ErrorCodeUserMetadataNotFound), } } @@ -79,7 +79,7 @@ func userMetadataVerify(metadata []backend.MetadataStream, files []backend.File) if err != nil { return backend.PluginError{ PluginID: usermd.PluginID, - ErrorCode: int(usermd.ErrorCodeUserIDInvalid), + ErrorCode: uint32(usermd.ErrorCodeUserIDInvalid), } } @@ -120,7 +120,7 @@ func userMetadataPreventUpdates(current, update []backend.MetadataStream) error u.UserID, c.UserID) return backend.PluginError{ PluginID: usermd.PluginID, - ErrorCode: int(usermd.ErrorCodeUserIDInvalid), + ErrorCode: uint32(usermd.ErrorCodeUserIDInvalid), ErrorContext: e, } @@ -129,7 +129,7 @@ func userMetadataPreventUpdates(current, update []backend.MetadataStream) error u.PublicKey, c.PublicKey) return backend.PluginError{ PluginID: usermd.PluginID, - ErrorCode: int(usermd.ErrorCodePublicKeyInvalid), + ErrorCode: uint32(usermd.ErrorCodePublicKeyInvalid), ErrorContext: e, } @@ -138,7 +138,7 @@ func userMetadataPreventUpdates(current, update []backend.MetadataStream) error u.Signature, c.Signature) return backend.PluginError{ PluginID: usermd.PluginID, - ErrorCode: int(usermd.ErrorCodeSignatureInvalid), + ErrorCode: uint32(usermd.ErrorCodeSignatureInvalid), ErrorContext: e, } } @@ -206,7 +206,7 @@ func (p *userPlugin) hookEditRecordPre(payload string) error { um.UserID, umCurr.UserID) return backend.PluginError{ PluginID: usermd.PluginID, - ErrorCode: int(usermd.ErrorCodeUserIDInvalid), + ErrorCode: uint32(usermd.ErrorCodeUserIDInvalid), ErrorContext: e, } } @@ -271,7 +271,7 @@ func statusChangeMetadataVerify(rm backend.RecordMetadata, metadata []backend.Me if len(statusChanges) == 0 { return backend.PluginError{ PluginID: usermd.PluginID, - ErrorCode: int(usermd.ErrorCodeStatusChangeMetadataNotFound), + ErrorCode: uint32(usermd.ErrorCodeStatusChangeMetadataNotFound), } } scm := statusChanges[len(statusChanges)-1] @@ -282,7 +282,7 @@ func statusChangeMetadataVerify(rm backend.RecordMetadata, metadata []backend.Me "metadata token: got %v, want %v", scm.Token, rm.Token) return backend.PluginError{ PluginID: usermd.PluginID, - ErrorCode: int(usermd.ErrorCodeTokenInvalid), + ErrorCode: uint32(usermd.ErrorCodeTokenInvalid), ErrorContext: e, } } @@ -293,7 +293,7 @@ func statusChangeMetadataVerify(rm backend.RecordMetadata, metadata []backend.Me "record metadata: got %v, want %v", scm.Status, rm.Status) return backend.PluginError{ PluginID: usermd.PluginID, - ErrorCode: int(usermd.ErrorCodeStatusInvalid), + ErrorCode: uint32(usermd.ErrorCodeStatusInvalid), ErrorContext: e, } } @@ -303,7 +303,7 @@ func statusChangeMetadataVerify(rm backend.RecordMetadata, metadata []backend.Me if ok && scm.Reason == "" { return backend.PluginError{ PluginID: usermd.PluginID, - ErrorCode: int(usermd.ErrorCodeReasonInvalid), + ErrorCode: uint32(usermd.ErrorCodeReasonInvalid), ErrorContext: "a reason must be given for this status change", } } diff --git a/politeiad/client/client.go b/politeiad/client/client.go index a6f097d04..ca0b45433 100644 --- a/politeiad/client/client.go +++ b/politeiad/client/client.go @@ -30,7 +30,7 @@ type Client struct { // during execution of a plugin command. type ErrorReply struct { PluginID string `json:"pluginid"` - ErrorCode int `json:"errorcode"` + ErrorCode uint32 `json:"errorcode"` ErrorContext string `json:"errorcontext"` } diff --git a/politeiad/client/pdv2.go b/politeiad/client/pdv2.go index 402ffdd02..6a8a407b9 100644 --- a/politeiad/client/pdv2.go +++ b/politeiad/client/pdv2.go @@ -398,7 +398,7 @@ func extractPluginCmdError(pcr pdv2.PluginCmdReply) error { return Error{ HTTPCode: http.StatusBadRequest, ErrorReply: ErrorReply{ - ErrorCode: int(pcr.UserError.ErrorCode), + ErrorCode: uint32(pcr.UserError.ErrorCode), ErrorContext: pcr.UserError.ErrorContext, }, } diff --git a/politeiad/plugins/comments/comments.go b/politeiad/plugins/comments/comments.go index 7774e01e1..121e4495f 100644 --- a/politeiad/plugins/comments/comments.go +++ b/politeiad/plugins/comments/comments.go @@ -50,7 +50,7 @@ const ( ) // ErrorCodeT represents a error that was caused by the user. -type ErrorCodeT int +type ErrorCodeT uint32 const ( // ErrorCodeInvalid is an invalid error code. @@ -214,7 +214,7 @@ type CommentDel struct { } // VoteT represents a comment upvote/downvote. -type VoteT int +type VoteT int32 const ( // VoteInvalid is an invalid comment vote. diff --git a/politeiad/plugins/dcrdata/dcrdata.go b/politeiad/plugins/dcrdata/dcrdata.go index 09fafafa6..944c0e227 100644 --- a/politeiad/plugins/dcrdata/dcrdata.go +++ b/politeiad/plugins/dcrdata/dcrdata.go @@ -41,7 +41,7 @@ const ( // cached results and the connection status to let the caller know that the // cached data may be stale. It is the callers responsibility to determine the // correct course of action when dcrdata cannot be reached. -type StatusT int +type StatusT uint32 const ( // StatusInvalid is an invalid connection status. diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index fca74d7b3..5630367e9 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -51,7 +51,7 @@ var ( ) // ErrorCodeT represents a plugin error that was caused by the user. -type ErrorCodeT int +type ErrorCodeT uint32 const ( // ErrorCodeInvalid represents and invalid error code. diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index 40e9b0fd1..bfcf61479 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -87,7 +87,7 @@ const ( ) // ErrorCodeT represents and error that is caused by the user. -type ErrorCodeT int +type ErrorCodeT uint32 const ( // ErrorCodeInvalid is an invalid error code. @@ -248,7 +248,7 @@ type AuthDetails struct { } // VoteT represents the different types of ticket votes that are available. -type VoteT int +type VoteT uint32 const ( // VoteTypeInvalid is an invalid vote type. @@ -407,7 +407,7 @@ type StartReply struct { // VoteErrorT represents errors that can occur while attempting to cast ticket // votes. -type VoteErrorT int +type VoteErrorT uint32 const ( // VoteErrorInvalid is an invalid vote error. @@ -517,7 +517,7 @@ type ResultsReply struct { } // VoteStatusT represents the status of a ticket vote. -type VoteStatusT int +type VoteStatusT uint32 const ( // VoteStatusInvalid is an invalid vote status. diff --git a/politeiad/plugins/usermd/usermd.go b/politeiad/plugins/usermd/usermd.go index 46af3dab5..8c4e959fc 100644 --- a/politeiad/plugins/usermd/usermd.go +++ b/politeiad/plugins/usermd/usermd.go @@ -25,7 +25,7 @@ const ( ) // ErrorCodeT represents a plugin error that was caused by the user. -type ErrorCodeT int +type ErrorCodeT uint32 const ( // User error codes diff --git a/politeiawww/api/comments/v1/v1.go b/politeiawww/api/comments/v1/v1.go index 7cdcdbe13..9c7ab6722 100644 --- a/politeiawww/api/comments/v1/v1.go +++ b/politeiawww/api/comments/v1/v1.go @@ -22,7 +22,7 @@ const ( ) // ErrorCodeT represents a user error code. -type ErrorCodeT int +type ErrorCodeT uint32 const ( ErrorCodeInvalid ErrorCodeT = 0 @@ -58,7 +58,7 @@ var ( // timing, etc). The HTTP status code will be 400. type UserErrorReply struct { ErrorCode ErrorCodeT `json:"errorcode"` - ErrorContext string `json:"errorcontext"` + ErrorContext string `json:"errorcontext,omitempty"` } // Error satisfies the error interface. @@ -70,8 +70,8 @@ func (e UserErrorReply) Error() string { // a plugin error. type PluginErrorReply struct { PluginID string `json:"pluginid"` - ErrorCode int `json:"errorcode"` - ErrorContext string `json:"errorcontext"` + ErrorCode uint32 `json:"errorcode"` + ErrorContext string `json:"errorcontext,omitempty"` } // Error satisfies the error interface. @@ -189,7 +189,7 @@ type NewReply struct { } // VoteT represents a comment upvote/downvote. -type VoteT int +type VoteT int32 const ( // VoteInvalid is an invalid comment vote. diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go index 2a77e32b5..b8d9e0ad6 100644 --- a/politeiawww/api/records/v1/v1.go +++ b/politeiawww/api/records/v1/v1.go @@ -24,7 +24,7 @@ const ( ) // ErrorCodeT represents a user error code. -type ErrorCodeT int +type ErrorCodeT uint32 const ( // Error codes @@ -83,7 +83,7 @@ var ( // timing, etc). The HTTP status code will be 400. type UserErrorReply struct { ErrorCode ErrorCodeT `json:"errorcode"` - ErrorContext string `json:"errorcontext"` + ErrorContext string `json:"errorcontext,omitempty"` } // Error satisfies the error interface. @@ -95,8 +95,8 @@ func (e UserErrorReply) Error() string { // a plugin error. type PluginErrorReply struct { PluginID string `json:"pluginid"` - ErrorCode int `json:"errorcode"` - ErrorContext string `json:"errorcontext"` + ErrorCode uint32 `json:"errorcode"` + ErrorContext string `json:"errorcontext,omitempty"` } // Error satisfies the error interface. diff --git a/politeiawww/api/ticketvote/v1/v1.go b/politeiawww/api/ticketvote/v1/v1.go index 63eb8f082..2c09b24a7 100644 --- a/politeiawww/api/ticketvote/v1/v1.go +++ b/politeiawww/api/ticketvote/v1/v1.go @@ -23,7 +23,7 @@ const ( ) // ErrorCodeT represents a user error code. -type ErrorCodeT int +type ErrorCodeT uint32 const ( // Error codes @@ -54,7 +54,7 @@ var ( // timing, etc). The HTTP status code will be 400. type UserErrorReply struct { ErrorCode ErrorCodeT `json:"errorcode"` - ErrorContext string `json:"errorcontext"` + ErrorContext string `json:"errorcontext,omitempty"` } // Error satisfies the error interface. @@ -66,8 +66,8 @@ func (e UserErrorReply) Error() string { // a plugin error. type PluginErrorReply struct { PluginID string `json:"pluginid"` - ErrorCode int `json:"errorcode"` - ErrorContext string `json:"errorcontext"` + ErrorCode uint32 `json:"errorcode"` + ErrorContext string `json:"errorcontext,omitempty"` } // Error satisfies the error interface. @@ -132,7 +132,7 @@ type AuthorizeReply struct { } // VoteT represents a vote type. -type VoteT int +type VoteT uint32 const ( // VoteTypeInvalid represents and invalid vote type. @@ -180,7 +180,7 @@ var ( ) // VoteStatusT represents a vote status. -type VoteStatusT int +type VoteStatusT uint32 const ( // VoteStatusInvalid represents an invalid vote status. diff --git a/politeiawww/comments/error.go b/politeiawww/comments/error.go index f47e8db9e..9f205d3a6 100644 --- a/politeiawww/comments/error.go +++ b/politeiawww/comments/error.go @@ -127,7 +127,7 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err } } -func convertPDErrorCode(errCode int) v1.ErrorCodeT { +func convertPDErrorCode(errCode uint32) v1.ErrorCodeT { // These are the only politeiad user errors that the comments // API expects to encounter. switch pdv2.ErrorCodeT(errCode) { diff --git a/politeiawww/piwww.go b/politeiawww/pi.go similarity index 100% rename from politeiawww/piwww.go rename to politeiawww/pi.go diff --git a/politeiawww/records/error.go b/politeiawww/records/error.go index 826519eea..af3c5a778 100644 --- a/politeiawww/records/error.go +++ b/politeiawww/records/error.go @@ -131,7 +131,7 @@ func handlePDError(w http.ResponseWriter, r *http.Request, format string, pde pd } } -func convertPDErrorCode(errCode int) v1.ErrorCodeT { +func convertPDErrorCode(errCode uint32) v1.ErrorCodeT { // Any error statuses that are intentionally omitted means that // politeiawww should 500. switch pdv2.ErrorCodeT(errCode) { diff --git a/politeiawww/ticketvote/error.go b/politeiawww/ticketvote/error.go index 8f99306b8..c86c3a95b 100644 --- a/politeiawww/ticketvote/error.go +++ b/politeiawww/ticketvote/error.go @@ -114,7 +114,7 @@ func handlePDError(w http.ResponseWriter, r *http.Request, format string, pde pd } } -func convertPDErrorCode(errCode int) v1.ErrorCodeT { +func convertPDErrorCode(errCode uint32) v1.ErrorCodeT { // This list is only populated with politeiad errors that we expect // for the ticketvote plugin commands. Any politeiad errors not // included in this list will cause politeiawww to 500. From b352ce9fe079f11bbca79a708e3c407d2a4a3c1d Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 15 Mar 2021 14:32:57 -0500 Subject: [PATCH 402/449] politeiad: Cleanup API formatting. --- politeiad/api/v2/v2.go | 154 ++++++++---------- politeiad/backendv2/backendv2.go | 16 +- .../tstorebe/plugins/ticketvote/cmds.go | 2 +- politeiad/backendv2/tstorebe/tstorebe.go | 57 ++----- politeiad/client/pdv2.go | 61 ++----- politeiad/politeiad.go | 10 +- politeiad/v2.go | 107 ++++-------- politeiawww/api/ticketvote/v1/v1.go | 20 ++- politeiawww/comments/process.go | 2 +- politeiawww/pi/events.go | 2 +- politeiawww/proposals.go | 82 +++------- politeiawww/records/process.go | 56 +++++-- 12 files changed, 225 insertions(+), 344 deletions(-) diff --git a/politeiad/api/v2/v2.go b/politeiad/api/v2/v2.go index 78eaeaf6e..1dde91deb 100644 --- a/politeiad/api/v2/v2.go +++ b/politeiad/api/v2/v2.go @@ -11,17 +11,16 @@ const ( APIRoute = "/v2" // Routes - RouteRecordNew = "/recordnew" - RouteRecordEdit = "/recordedit" - RouteRecordEditMetadata = "/recordeditmetadata" - RouteRecordSetStatus = "/recordsetstatus" - RouteRecordGet = "/recordget" - RouteRecordGetBatch = "/recordgetbatch" - RouteRecordGetTimestamps = "/recordgettimestamps" - RouteInventory = "/inventory" - RoutePluginWrite = "/pluginwrite" - RoutePluginReads = "/pluginreads" - RoutePluginInventory = "/plugininventory" + RouteRecordNew = "/recordnew" + RouteRecordEdit = "/recordedit" + RouteRecordEditMetadata = "/recordeditmetadata" + RouteRecordSetStatus = "/recordsetstatus" + RouteRecordTimestamps = "/recordtimestamps" + RouteRecords = "/records" + RouteInventory = "/inventory" + RoutePluginWrite = "/pluginwrite" + RoutePluginReads = "/pluginreads" + RoutePluginInventory = "/plugininventory" // ChallengeSize is the size of a request challenge token in bytes. ChallengeSize = 32 @@ -50,6 +49,7 @@ const ( ErrorCodeStatusChangeInvalid ErrorCodeT = 16 ErrorCodePluginIDInvalid ErrorCodeT = 17 ErrorCodePluginCmdInvalid ErrorCodeT = 18 + ErrorCodePageSizeExceeded ErrorCodeT = 19 ) var ( @@ -72,6 +72,7 @@ var ( ErrorCodeStatusChangeInvalid: "status change invalid", ErrorCodePluginIDInvalid: "pluguin id invalid", ErrorCodePluginCmdInvalid: "plugin cmd invalid", + ErrorCodePageSizeExceeded: "page size exceeded", } ) @@ -233,7 +234,7 @@ type Record struct { // the record and it may contain optional metadata. type RecordNew struct { Challenge string `json:"challenge"` // Random challenge - Metadata []MetadataStream `json:"metadata"` + Metadata []MetadataStream `json:"metadata,omitempty"` Files []File `json:"files"` } @@ -256,10 +257,10 @@ type RecordNewReply struct { type RecordEdit struct { Challenge string `json:"challenge"` // Random challenge Token string `json:"token"` // Censorship token - MDAppend []MetadataStream `json:"mdappend"` - MDOverwrite []MetadataStream `json:"mdoverwrite"` - FilesAdd []File `json:"filesadd"` - FilesDel []string `json:"filesdel"` + MDAppend []MetadataStream `json:"mdappend,omitempty"` + MDOverwrite []MetadataStream `json:"mdoverwrite,omitempty"` + FilesAdd []File `json:"filesadd,omitempty"` + FilesDel []string `json:"filesdel,omitempty"` } // RecordEditReply is the reply to the RecordEdit command. @@ -276,8 +277,8 @@ type RecordEditReply struct { type RecordEditMetadata struct { Challenge string `json:"challenge"` // Random challenge Token string `json:"token"` // Censorship token - MDAppend []MetadataStream `json:"mdappend"` - MDOverwrite []MetadataStream `json:"mdoverwrite"` + MDAppend []MetadataStream `json:"mdappend,omitempty"` + MDOverwrite []MetadataStream `json:"mdoverwrite,omitempty"` } // RecordEditMetadataReply is the reply to the RecordEditMetadata command. @@ -295,8 +296,8 @@ type RecordSetStatus struct { Challenge string `json:"challenge"` // Random challenge Token string `json:"token"` // Censorship token Status RecordStatusT `json:"status"` - MDAppend []MetadataStream `json:"mdappend"` - MDOverwrite []MetadataStream `json:"mdoverwrite"` + MDAppend []MetadataStream `json:"mdappend,omitempty"` + MDOverwrite []MetadataStream `json:"mdoverwrite,omitempty"` } // RecordSetStatusReply is the reply to the RecordSetStatus command. @@ -305,55 +306,6 @@ type RecordSetStatusReply struct { Record Record `json:"record"` } -// RecordGet retrieves a record. If no version is provided the most recent -// version will be returned. -type RecordGet struct { - Challenge string `json:"challenge"` // Random challenge - Token string `json:"token"` // Censorship token - Version uint32 `json:"version,omitempty"` // Record version -} - -// RecordGetReply is the reply to the RecordGet command. -type RecordGetReply struct { - Response string `json:"response"` // Challenge response - Record Record `json:"record"` -} - -// RecordRequest is used to request a record. It gives the caller granular -// control over what is returned. The only required field is the token. All -// other fields are optional. All record files are returned by default unless -// one of the file arguments is provided. -// -// Version is used to request a specific version of a record. If no version is -// provided then the most recent version of the record will be returned. -// -// Filenames can be used to request specific files. If filenames is provided -// then the specified files will be the only files that are returned. -// -// OmitAllFiles can be used to retrieve a record without any of the record -// files. This supersedes the filenames argument. -type RecordRequest struct { - Token string `json:"token"` - Version uint32 `json:"version"` - Filenames []string `json:"filenames"` - OmitAllFiles bool `json:"omitallfiles"` -} - -// RecordGetBatch retrieves a record. If no version is provided the most recent -// version will be returned. -type RecordGetBatch struct { - Challenge string `json:"challenge"` // Random challenge - Requests []RecordRequest `json:"requests"` -} - -// RecordGetBatchReply is the reply to the RecordGetBatch command. If a record -// was not found or an error occurred while retrieving it the token will not be -// included in the returned map. -type RecordGetBatchReply struct { - Response string `json:"response"` // Challenge response - Records map[string]Record `json:"records"` -} - // Proof contains an inclusion proof for the digest in the merkle root. All // digests are hex encoded SHA256 digests. // @@ -384,8 +336,17 @@ type Timestamp struct { Proofs []Proof `json:"proofs"` } -// RecordTimestamps contains the timestamps for a specific version of a record. +// RecordTimestamps requests the timestamps for a record. If a version is not +// included the most recent version will be returned. type RecordTimestamps struct { + Challenge string `json:"challenge"` // Random challenge + Token string `json:"token"` // Censorship token + Version uint32 `json:"version,omitempty"` // Record version +} + +// RecordGetTimestampsReply is the reply ot the RecordTimestamps command. +type RecordTimestampsReply struct { + Response string `json:"response"` // Challenge response RecordMetadata Timestamp `json:"recordmetadata"` // map[pluginID]map[streamID]Timestamp @@ -395,16 +356,45 @@ type RecordTimestamps struct { Files map[string]Timestamp `json:"files"` } -type RecordGetTimestamps struct { - Challenge string `json:"challenge"` // Random challenge - Token string `json:"token"` // Censorship token - Version uint32 `json:"version"` // Record version +const ( + // RecordsPageSize is the maximum number of records that can be + // requested using the Records commands. + RecordsPageSize uint32 = 5 +) + +// RecordRequest is used to request a record. It gives the caller granular +// control over what is returned. The only required field is the token. All +// other fields are optional. All record files are returned by default unless +// one of the file arguments is provided. +// +// Version is used to request a specific version of a record. If no version is +// provided then the most recent version of the record will be returned. +// +// Filenames can be used to request specific files. If filenames is provided +// then the specified files will be the only files that are returned. +// +// OmitAllFiles can be used to retrieve a record without any of the record +// files. This supersedes the filenames argument. +type RecordRequest struct { + Token string `json:"token"` + Version uint32 `json:"version,omitempty"` + Filenames []string `json:"filenames,omitempty"` + OmitAllFiles bool `json:"omitallfiles,omitempty"` } -// RecordGetTimestampsReply is the reply ot the RecordTimestamps command. -type RecordGetTimestampsReply struct { - Response string `json:"response"` // Challenge response - Timestamps RecordTimestamps `json:"timestamps"` +// Records retrieves a record. If no version is provided the most recent +// version will be returned. +type Records struct { + Challenge string `json:"challenge"` // Random challenge + Requests []RecordRequest `json:"requests"` +} + +// RecordsReply is the reply to the Records command. If a record was not found +// or an error occurred while retrieving it the token will not be included in +// the returned map. +type RecordsReply struct { + Response string `json:"response"` // Challenge response + Records map[string]Record `json:"records"` // [token]Record } const ( @@ -440,10 +430,10 @@ type InventoryReply struct { // PluginCmd represents plugin command and the command payload. A token is // required for all plugin writes, but is optional for reads. type PluginCmd struct { - Token string `json:"token,omitempty"` // Censorship token - ID string `json:"id"` // Plugin identifier - Command string `json:"command"` // Plugin command - Payload string `json:"payload"` // Command payload + Token string `json:"token,omitempty"` // Censorship token + ID string `json:"id"` // Plugin identifier + Command string `json:"command"` // Plugin command + Payload string `json:"payload,omitempty"` // Command payload } // PluginWrite executes a plugin command that writes data. diff --git a/politeiad/backendv2/backendv2.go b/politeiad/backendv2/backendv2.go index 0504a05b7..5d8c95424 100644 --- a/politeiad/backendv2/backendv2.go +++ b/politeiad/backendv2/backendv2.go @@ -295,20 +295,16 @@ type Backend interface { // RecordExists returns whether a record exists. RecordExists(token []byte) bool - // RecordGet retrieves a record. If no version is provided then the - // most recent version will be returned. - RecordGet(token []byte, version uint32) (*Record, error) - - // RecordGetBatch retreives a batch of records. If a record is not - // found then it is simply not included in the returned map. An - // error is not returned. - RecordGetBatch(reqs []RecordRequest) (map[string]Record, error) - // RecordTimestamps returns the timestamps for a record. If no // version is provided then timestamps for the most recent version // will be returned. RecordTimestamps(token []byte, version uint32) (*RecordTimestamps, error) + // Records retreives a batch of records. If a record is not found + // then it is simply not included in the returned map. An error is + // not returned. + Records(reqs []RecordRequest) (map[string]Record, error) + // Inventory returns the tokens of records in the inventory // categorized by record state and record status. The tokens are // ordered by the timestamp of their most recent status change, @@ -318,7 +314,7 @@ type Backend interface { // a specific page of record tokens. // // If no status is provided then the most recent page of tokens for - // each statuses will be returned. All other arguments are ignored. + // all statuses will be returned. All other arguments are ignored. Inventory(state StateT, status StatusT, pageSize, pageNumber uint32) (*Inventory, error) diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go index 6c9217670..cd92a57d6 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go @@ -662,7 +662,7 @@ func (p *ticketVotePlugin) recordAbridged(token []byte) (*backend.Record, error) }, }, } - rs, err := p.backend.RecordGetBatch(reqs) + rs, err := p.backend.Records(reqs) if err != nil { return nil, err } diff --git a/politeiad/backendv2/tstorebe/tstorebe.go b/politeiad/backendv2/tstorebe/tstorebe.go index a19a75346..367186aa5 100644 --- a/politeiad/backendv2/tstorebe/tstorebe.go +++ b/politeiad/backendv2/tstorebe/tstorebe.go @@ -474,9 +474,9 @@ func (t *tstoreBackend) RecordNew(metadata []backend.MetadataStream, files []bac t.inventoryAdd(backend.StateUnvetted, token, backend.StatusUnreviewed) // Get the full record to return - r, err := t.RecordGet(token, 0) + r, err := t.tstore.RecordLatest(treeID) if err != nil { - return nil, fmt.Errorf("RecordGet %x: %v", token, err) + return nil, fmt.Errorf("RecordLatest %v: %v", treeID, err) } return r, nil @@ -927,27 +927,27 @@ func (t *tstoreBackend) RecordExists(token []byte) bool { return t.tstore.TreeExists(treeID) } -// RecordGet retrieves a record. If no version is provided then the most recent -// version will be returned. +// RecordTimestamps returns the timestamps for a record. If no version is provided +// then timestamps for the most recent version will be returned. // // This function satisfies the Backend interface. -func (t *tstoreBackend) RecordGet(token []byte, version uint32) (*backend.Record, error) { - log.Tracef("RecordGet: %x", token) +func (t *tstoreBackend) RecordTimestamps(token []byte, version uint32) (*backend.RecordTimestamps, error) { + log.Tracef("RecordTimestamps: %x %v", token, version) // Read methods are allowed to use token prefixes token = t.fullLengthToken(token) treeID := treeIDFromToken(token) - return t.tstore.Record(treeID, version) + return t.tstore.RecordTimestamps(treeID, version, token) } -// RecordGetBatch retreives a batch of records. Individual record errors are -// not returned. If the record was not found then it will not be included in -// the returned map. +// Records retreives a batch of records. Individual record errors are not +// returned. If the record was not found then it will not be included in the +// returned map. // // This function satisfies the Backend interface. -func (t *tstoreBackend) RecordGetBatch(reqs []backend.RecordRequest) (map[string]backend.Record, error) { - log.Tracef("RecordGetBatch") +func (t *tstoreBackend) Records(reqs []backend.RecordRequest) (map[string]backend.Record, error) { + log.Tracef("Records") records := make(map[string]backend.Record, len(reqs)) for _, v := range reqs { @@ -975,20 +975,6 @@ func (t *tstoreBackend) RecordGetBatch(reqs []backend.RecordRequest) (map[string return records, nil } -// RecordTimestamps returns the timestamps for a record. If no version is provided -// then timestamps for the most recent version will be returned. -// -// This function satisfies the Backend interface. -func (t *tstoreBackend) RecordTimestamps(token []byte, version uint32) (*backend.RecordTimestamps, error) { - log.Tracef("RecordTimestamps: %x %v", token, version) - - // Read methods are allowed to use token prefixes - token = t.fullLengthToken(token) - - treeID := treeIDFromToken(token) - return t.tstore.RecordTimestamps(treeID, version, token) -} - // Inventory returns the tokens of records in the inventory categorized by // record state and record status. The tokens are ordered by the timestamp of // their most recent status change, sorted from newest to oldest. @@ -996,13 +982,12 @@ func (t *tstoreBackend) RecordTimestamps(token []byte, version uint32) (*backend // The state, status, and page arguments can be provided to request a specific // page of record tokens. // -// If no status is provided then the most recent page of tokens for each +// If no status is provided then the most recent page of tokens for all // statuses will be returned. All other arguments are ignored. // // This function satisfies the Backend interface. func (t *tstoreBackend) Inventory(state backend.StateT, status backend.StatusT, pageSize, pageNumber uint32) (*backend.Inventory, error) { - log.Tracef("InventoryByStatus: %v %v %v %v", - state, status, pageSize, pageNumber) + log.Tracef("Inventory: %v %v %v %v", state, status, pageSize, pageNumber) inv, err := t.invByStatus(state, status, pageSize, pageNumber) if err != nil { @@ -1063,14 +1048,6 @@ func (t *tstoreBackend) PluginRead(token []byte, pluginID, pluginCmd, payload st treeID = treeIDFromToken(token) } - if len(token) > 0 { - log.Infof("Plugin '%v' read cmd '%v' on %x", - pluginID, pluginCmd, token) - } else { - log.Infof("Plugin '%v' read cmd '%v'", - pluginID, pluginCmd) - } - return t.tstore.PluginCmd(treeID, token, pluginID, pluginCmd, payload) } @@ -1085,6 +1062,9 @@ func (t *tstoreBackend) PluginWrite(token []byte, pluginID, pluginCmd, payload s return "", backend.ErrRecordNotFound } + log.Infof("Plugin '%v' write cmd '%v' on %x", + pluginID, pluginCmd, token) + // Hold the record lock for the remainder of this function. We // do this here in the backend so that the individual plugins // implementations don't need to worry about race conditions. @@ -1092,9 +1072,6 @@ func (t *tstoreBackend) PluginWrite(token []byte, pluginID, pluginCmd, payload s m.Lock() defer m.Unlock() - log.Infof("Plugin '%v' write cmd '%v' on %x", - pluginID, pluginCmd, token) - // Call pre plugin hooks treeID := treeIDFromToken(token) hp := plugins.HookPluginPre{ diff --git a/politeiad/client/pdv2.go b/politeiad/client/pdv2.go index 6a8a407b9..39f5efe1e 100644 --- a/politeiad/client/pdv2.go +++ b/politeiad/client/pdv2.go @@ -157,14 +157,14 @@ func (c *Client) RecordSetStatus(ctx context.Context, token string, status pdv2. return &reply.Record, nil } -// RecordGet sends a RecordGet command to the politeiad v2 API. -func (c *Client) RecordGet(ctx context.Context, token string, version uint32) (*pdv2.Record, error) { +// RecordTimestamps sends a RecordTimestamps command to the politeiad v2 API. +func (c *Client) RecordTimestamps(ctx context.Context, token string, version uint32) (*pdv2.RecordTimestampsReply, error) { // Setup request challenge, err := util.Random(pdv2.ChallengeSize) if err != nil { return nil, err } - rg := pdv2.RecordGet{ + rgt := pdv2.RecordTimestamps{ Challenge: hex.EncodeToString(challenge), Token: token, Version: version, @@ -172,46 +172,13 @@ func (c *Client) RecordGet(ctx context.Context, token string, version uint32) (* // Send request resBody, err := c.makeReq(ctx, http.MethodPost, - pdv2.APIRoute, pdv2.RouteRecordGet, rg) + pdv2.APIRoute, pdv2.RouteRecordTimestamps, rgt) if err != nil { return nil, err } // Decode reply - var rgr pdv2.RecordGetReply - err = json.Unmarshal(resBody, &rgr) - if err != nil { - return nil, err - } - err = util.VerifyChallenge(c.pid, challenge, rgr.Response) - if err != nil { - return nil, err - } - - return &rgr.Record, nil -} - -// RecordGetBatch sends a RecordGetBatch command to the politeiad v2 API. -func (c *Client) RecordGetBatch(ctx context.Context, reqs []pdv2.RecordRequest) (map[string]pdv2.Record, error) { - // Setup request - challenge, err := util.Random(pdv2.ChallengeSize) - if err != nil { - return nil, err - } - rgb := pdv2.RecordGetBatch{ - Challenge: hex.EncodeToString(challenge), - Requests: reqs, - } - - // Send request - resBody, err := c.makeReq(ctx, http.MethodPost, - pdv2.APIRoute, pdv2.RouteRecordGetBatch, rgb) - if err != nil { - return nil, err - } - - // Decode reply - var reply pdv2.RecordGetBatchReply + var reply pdv2.RecordTimestampsReply err = json.Unmarshal(resBody, &reply) if err != nil { return nil, err @@ -221,32 +188,30 @@ func (c *Client) RecordGetBatch(ctx context.Context, reqs []pdv2.RecordRequest) return nil, err } - return reply.Records, nil + return &reply, nil } -// RecordGetTimestamps sends a RecordGetTimestamps command to the politeiad v2 -// API. -func (c *Client) RecordGetTimestamps(ctx context.Context, token string, version uint32) (*pdv2.RecordTimestamps, error) { +// Records sends a Records command to the politeiad v2 API. +func (c *Client) Records(ctx context.Context, reqs []pdv2.RecordRequest) (map[string]pdv2.Record, error) { // Setup request challenge, err := util.Random(pdv2.ChallengeSize) if err != nil { return nil, err } - rgt := pdv2.RecordGetTimestamps{ + rgb := pdv2.Records{ Challenge: hex.EncodeToString(challenge), - Token: token, - Version: version, + Requests: reqs, } // Send request resBody, err := c.makeReq(ctx, http.MethodPost, - pdv2.APIRoute, pdv2.RouteRecordGetTimestamps, rgt) + pdv2.APIRoute, pdv2.RouteRecords, rgb) if err != nil { return nil, err } // Decode reply - var reply pdv2.RecordGetTimestampsReply + var reply pdv2.RecordsReply err = json.Unmarshal(resBody, &reply) if err != nil { return nil, err @@ -256,7 +221,7 @@ func (c *Client) RecordGetTimestamps(ctx context.Context, token string, version return nil, err } - return &reply.Timestamps, nil + return reply.Records, nil } // Inventory sends a Inventory command to the politeiad v2 API. diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index c86acdf1f..a639522fa 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -231,12 +231,10 @@ func (p *politeia) setupBackendTstore(anp *chaincfg.Params) error { p.handleRecordEditMetadata, permissionPublic) p.addRouteV2(http.MethodPost, v2.RouteRecordSetStatus, p.handleRecordSetStatus, permissionPublic) - p.addRouteV2(http.MethodPost, v2.RouteRecordGet, - p.handleRecordGet, permissionPublic) - p.addRouteV2(http.MethodPost, v2.RouteRecordGetBatch, - p.handleRecordGetBatch, permissionPublic) - p.addRouteV2(http.MethodPost, v2.RouteRecordGetTimestamps, - p.handleRecordGetTimestamps, permissionPublic) + p.addRouteV2(http.MethodPost, v2.RouteRecords, + p.handleRecords, permissionPublic) + p.addRouteV2(http.MethodPost, v2.RouteRecordTimestamps, + p.handleRecordTimestamps, permissionPublic) p.addRouteV2(http.MethodPost, v2.RouteInventory, p.handleInventory, permissionPublic) p.addRouteV2(http.MethodPost, v2.RoutePluginWrite, diff --git a/politeiad/v2.go b/politeiad/v2.go index 03a85939e..979a8dd45 100644 --- a/politeiad/v2.go +++ b/politeiad/v2.go @@ -230,82 +230,43 @@ func (p *politeia) handleRecordSetStatus(w http.ResponseWriter, r *http.Request) util.RespondWithJSON(w, http.StatusOK, rer) } -func (p *politeia) handleRecordGet(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleRecordGet") +func (p *politeia) handleRecords(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleRecords") // Decode request - var rg v2.RecordGet + var rgb v2.Records decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&rg); err != nil { - respondWithErrorV2(w, r, "handleRecordGet: unmarshal", + if err := decoder.Decode(&rgb); err != nil { + respondWithErrorV2(w, r, "handleRecords: unmarshal", v2.UserErrorReply{ ErrorCode: v2.ErrorCodeRequestPayloadInvalid, }) return } - challenge, err := hex.DecodeString(rg.Challenge) + challenge, err := hex.DecodeString(rgb.Challenge) if err != nil || len(challenge) != v2.ChallengeSize { - respondWithErrorV2(w, r, "handleRecordGet: decode challenge", + respondWithErrorV2(w, r, "handleRecords: decode challenge", v2.UserErrorReply{ ErrorCode: v2.ErrorCodeChallengeInvalid, }) return } - token, err := decodeTokenAnyLength(rg.Token) - if err != nil { - respondWithErrorV2(w, r, "handleRecordGet: decode token", - v2.UserErrorReply{ - ErrorCode: v2.ErrorCodeTokenInvalid, - }) - return - } - - // Get record - rc, err := p.backendv2.RecordGet(token, rg.Version) - if err != nil { - respondWithErrorV2(w, r, - "handleRecordGet: RecordGet: %v", err) - return - } - - // Prepare reply - response := p.identity.SignMessage(challenge) - rgr := v2.RecordGetReply{ - Response: hex.EncodeToString(response[:]), - Record: p.convertRecordToV2(*rc), - } - util.RespondWithJSON(w, http.StatusOK, rgr) -} - -func (p *politeia) handleRecordGetBatch(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleRecordGetBatch") - - // Decode request - var rgb v2.RecordGetBatch - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&rgb); err != nil { - respondWithErrorV2(w, r, "handleRecordGetBatch: unmarshal", - v2.UserErrorReply{ - ErrorCode: v2.ErrorCodeRequestPayloadInvalid, - }) - return - } - challenge, err := hex.DecodeString(rgb.Challenge) - if err != nil || len(challenge) != v2.ChallengeSize { - respondWithErrorV2(w, r, "handleRecordGetBatch: decode challenge", + // Verify page size + if len(rgb.Requests) > int(v2.RecordsPageSize) { + respondWithErrorV2(w, r, "handleRecords: unmarshal", v2.UserErrorReply{ - ErrorCode: v2.ErrorCodeChallengeInvalid, + ErrorCode: v2.ErrorCodePageSizeExceeded, }) return } // Get record batch reqs := convertRecordRequestsToBackend(rgb.Requests) - brecords, err := p.backendv2.RecordGetBatch(reqs) + brecords, err := p.backendv2.Records(reqs) if err != nil { respondWithErrorV2(w, r, - "handleRecordGet: RecordGetBatch: %v", err) + "handleRecordGet: Records: %v", err) return } @@ -315,7 +276,7 @@ func (p *politeia) handleRecordGetBatch(w http.ResponseWriter, r *http.Request) records[k] = p.convertRecordToV2(v) } response := p.identity.SignMessage(challenge) - reply := v2.RecordGetBatchReply{ + reply := v2.RecordsReply{ Response: hex.EncodeToString(response[:]), Records: records, } @@ -323,11 +284,11 @@ func (p *politeia) handleRecordGetBatch(w http.ResponseWriter, r *http.Request) util.RespondWithJSON(w, http.StatusOK, reply) } -func (p *politeia) handleRecordGetTimestamps(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleRecordGetTimestamps") +func (p *politeia) handleRecordTimestamps(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleRecordTimestamps") // Decode request - var rgt v2.RecordGetTimestamps + var rgt v2.RecordTimestamps decoder := json.NewDecoder(r.Body) if err := decoder.Decode(&rgt); err != nil { respondWithErrorV2(w, r, "handleRecordTimestamps: unmarshal", @@ -363,9 +324,11 @@ func (p *politeia) handleRecordGetTimestamps(w http.ResponseWriter, r *http.Requ // Prepare reply response := p.identity.SignMessage(challenge) - rtr := v2.RecordGetTimestampsReply{ - Response: hex.EncodeToString(response[:]), - Timestamps: convertRecordTimestampsToV2(*rt), + rtr := v2.RecordTimestampsReply{ + Response: hex.EncodeToString(response[:]), + RecordMetadata: convertTimestampToV2(rt.RecordMetadata), + Metadata: convertMetadataTimestampsToV2(rt.Metadata), + Files: convertFileTimestampsToV2(rt.Files), } util.RespondWithJSON(w, http.StatusOK, rtr) @@ -748,27 +711,27 @@ func convertTimestampToV2(t backendv2.Timestamp) v2.Timestamp { } } -func convertRecordTimestampsToV2(r backendv2.RecordTimestamps) v2.RecordTimestamps { - metadata := make(map[string]map[uint32]v2.Timestamp, 16) - for pluginID, v := range r.Metadata { - timestamps, ok := metadata[pluginID] +func convertMetadataTimestampsToV2(metadata map[string]map[uint32]backendv2.Timestamp) map[string]map[uint32]v2.Timestamp { + md := make(map[string]map[uint32]v2.Timestamp, 16) + for pluginID, v := range metadata { + timestamps, ok := md[pluginID] if !ok { timestamps = make(map[uint32]v2.Timestamp, 16) } for streamID, ts := range v { timestamps[streamID] = convertTimestampToV2(ts) } - metadata[pluginID] = timestamps + md[pluginID] = timestamps } - files := make(map[string]v2.Timestamp, len(r.Files)) - for k, v := range r.Files { - files[k] = convertTimestampToV2(v) - } - return v2.RecordTimestamps{ - RecordMetadata: convertTimestampToV2(r.RecordMetadata), - Metadata: metadata, - Files: files, + return md +} + +func convertFileTimestampsToV2(files map[string]backendv2.Timestamp) map[string]v2.Timestamp { + fs := make(map[string]v2.Timestamp, len(files)) + for k, v := range files { + fs[k] = convertTimestampToV2(v) } + return fs } func convertPluginSettingToV2(p backendv2.PluginSetting) v2.PluginSetting { diff --git a/politeiawww/api/ticketvote/v1/v1.go b/politeiawww/api/ticketvote/v1/v1.go index 2c09b24a7..b01be075e 100644 --- a/politeiawww/api/ticketvote/v1/v1.go +++ b/politeiawww/api/ticketvote/v1/v1.go @@ -158,7 +158,18 @@ const ( // requirements but still be rejected if it does not have the most // net yes votes. VoteTypeRunoff VoteT = 2 +) + +var ( + // VoteType contains the human readable vote types. + VoteTypes = map[VoteT]string{ + VoteTypeInvalid: "invalid vote type", + VoteTypeStandard: "standard", + VoteTypeRunoff: "runoff", + } +) +const ( // VoteOptionIDApprove is the vote option ID that indicates the // record should be approved. Standard votes and runoff vote // submissions are required to use this vote option ID. @@ -170,15 +181,6 @@ const ( VoteOptionIDReject = "no" ) -var ( - // VoteType contains the human readable vote types. - VoteTypes = map[VoteT]string{ - VoteTypeInvalid: "invalid vote type", - VoteTypeStandard: "standard", - VoteTypeRunoff: "runoff", - } -) - // VoteStatusT represents a vote status. type VoteStatusT uint32 diff --git a/politeiawww/comments/process.go b/politeiawww/comments/process.go index c9be44eef..6900b179b 100644 --- a/politeiawww/comments/process.go +++ b/politeiawww/comments/process.go @@ -371,7 +371,7 @@ func (c *Comments) recordNoFiles(ctx context.Context, token string) (*pdv2.Recor OmitAllFiles: true, }, } - records, err := c.politeiad.RecordGetBatch(ctx, req) + records, err := c.politeiad.Records(ctx, req) if err != nil { return nil, err } diff --git a/politeiawww/pi/events.go b/politeiawww/pi/events.go index 59f8ddd9c..49e7da9f2 100644 --- a/politeiawww/pi/events.go +++ b/politeiawww/pi/events.go @@ -664,7 +664,7 @@ func (p *Pi) recordAbridged(token string) (*pdv2.Record, error) { }, }, } - rs, err := p.politeiad.RecordGetBatch(context.Background(), reqs) + rs, err := p.politeiad.Records(context.Background(), reqs) if err != nil { return nil, fmt.Errorf("politeiad records: %v", err) } diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 4e1867422..648d549b4 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -31,60 +31,8 @@ import ( "github.com/gorilla/mux" ) -func (p *politeiawww) proposal(ctx context.Context, token, version string) (*www.ProposalRecord, error) { - // Parse version - v, err := strconv.ParseUint(version, 10, 64) - if err != nil { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - } - } - - // Get record - r, err := p.politeiad.RecordGet(ctx, token, uint32(v)) - if err != nil { - return nil, err - } - - // Legacy www routes are only for vetted records - if r.State == pdv2.RecordStateUnvetted { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, - } - } - - // Convert to a proposal - pr, err := convertRecordToProposal(*r) - if err != nil { - return nil, err - } - - // Get submissions list if this is an RFP - if pr.LinkBy != 0 { - subs, err := p.politeiad.TicketVoteSubmissions(ctx, token) - if err != nil { - return nil, err - } - pr.LinkedFrom = subs - } - - // Fill in user data - userID := userIDFromMetadataStreams(r.Metadata) - uid, err := uuid.Parse(userID) - if err != nil { - return nil, err - } - u, err := p.db.UserGetById(uid) - if err != nil { - return nil, err - } - pr.Username = u.Username - - return pr, nil -} - func (p *politeiawww) proposals(ctx context.Context, reqs []pdv2.RecordRequest) (map[string]www.ProposalRecord, error) { - records, err := p.politeiad.RecordGetBatch(ctx, reqs) + records, err := p.politeiad.Records(ctx, reqs) if err != nil { return nil, err } @@ -222,16 +170,34 @@ func (p *politeiawww) processAllVetted(ctx context.Context, gav www.GetAllVetted func (p *politeiawww) processProposalDetails(ctx context.Context, pd www.ProposalsDetails, u *user.User) (*www.ProposalDetailsReply, error) { log.Tracef("processProposalDetails: %v", pd.Token) - // This route will now only return vetted proposal. This is fine - // since API consumers of this legacy route will only need public - // proposals. - pr, err := p.proposal(ctx, pd.Token, pd.Version) + // Parse version + v, err := strconv.ParseUint(pd.Version, 10, 64) + if err != nil { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + } + } + + // Get proposal + reqs := []pdv2.RecordRequest{ + { + Token: pd.Token, + Version: uint32(v), + }, + } + prs, err := p.proposals(ctx, reqs) if err != nil { return nil, err } + pr, ok := prs[pd.Token] + if !ok { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + } + } return &www.ProposalDetailsReply{ - Proposal: *pr, + Proposal: pr, }, nil } diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index a64495628..1153791bb 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -7,6 +7,7 @@ package records import ( "context" "encoding/json" + "errors" "fmt" "time" @@ -98,7 +99,7 @@ func (r *Records) processNew(ctx context.Context, n v1.New, u user.User) (*v1.Ne // filesToDel returns the names of the files that are included in the current // files but are not included in updated files. These are the files that need // to be deleted from a record on update. -func filesToDel(current []pdv2.File, updated []pdv2.File) []string { +func filesToDel(current []v1.File, updated []v1.File) []string { curr := make(map[string]struct{}, len(current)) // [name]struct for _, v := range updated { curr[v.Name] = struct{}{} @@ -127,14 +128,19 @@ func (r *Records) processEdit(ctx context.Context, e v1.Edit, u user.User) (*v1. } // Get current record - curr, err := r.politeiad.RecordGet(ctx, e.Token, 0) + curr, err := r.record(ctx, e.Token, 0) if err != nil { + if err == errRecordNotFound { + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeRecordNotFound, + } + } return nil, err } // Setup files filesAdd := convertFilesToPD(e.Files) - filesDel := filesToDel(curr.Files, filesAdd) + filesDel := filesToDel(curr.Files, e.Files) // Setup metadata um := usermd.UserMetadata{ @@ -242,7 +248,11 @@ func (r *Records) processDetails(ctx context.Context, d v1.Details, u *user.User // Get record rc, err := r.record(ctx, d.Token, d.Version) if err != nil { - return nil, err + if err == errRecordNotFound { + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeRecordNotFound, + } + } } // Only admins and the record author are allowed to retrieve @@ -338,7 +348,7 @@ func (r *Records) processTimestamps(ctx context.Context, t v1.Timestamps, isAdmi log.Tracef("processTimestamps: %v %v", t.Token, t.Version) // Get record timestamps - rt, err := r.politeiad.RecordGetTimestamps(ctx, t.Token, t.Version) + rt, err := r.politeiad.RecordTimestamps(ctx, t.Token, t.Version) if err != nil { return nil, err } @@ -419,19 +429,9 @@ func (r *Records) processUserRecords(ctx context.Context, ur v1.UserRecords, u * }, nil } -// record returns a version of a record from politeiad. If version is an empty -// string then the most recent version will be returned. -func (r *Records) record(ctx context.Context, token string, version uint32) (*v1.Record, error) { - pdr, err := r.politeiad.RecordGet(ctx, token, version) - if err != nil { - return nil, err - } - return r.convertRecordToV1(*pdr) -} - func (r *Records) records(ctx context.Context, reqs []pdv2.RecordRequest) (map[string]v1.Record, error) { // Get records - pdr, err := r.politeiad.RecordGetBatch(ctx, reqs) + pdr, err := r.politeiad.Records(ctx, reqs) if err != nil { return nil, err } @@ -449,6 +449,30 @@ func (r *Records) records(ctx context.Context, reqs []pdv2.RecordRequest) (map[s return records, nil } +var ( + errRecordNotFound = errors.New("record not found") +) + +// record returns a version of a record from politeiad. If version is an empty +// string then the most recent version will be returned. +func (r *Records) record(ctx context.Context, token string, version uint32) (*v1.Record, error) { + reqs := []pdv2.RecordRequest{ + { + Token: token, + Version: version, + }, + } + rcs, err := r.records(ctx, reqs) + if err != nil { + return nil, err + } + rc, ok := rcs[token] + if !ok { + return nil, errRecordNotFound + } + return &rc, nil +} + func (r *Records) convertRecordToV1(pdr pdv2.Record) (*v1.Record, error) { rc := convertRecordToV1(pdr) From aa1cec37ee1a0be06a91908c540b639620f5e356 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 15 Mar 2021 16:56:57 -0500 Subject: [PATCH 403/449] tstore: Cleanup tlog client. --- .../backendv2/tstorebe/store/mysql/mysql.go | 2 +- politeiad/backendv2/tstorebe/tstore/anchor.go | 24 +-- politeiad/backendv2/tstorebe/tstore/client.go | 28 +-- .../backendv2/tstorebe/tstore/recordindex.go | 4 +- .../backendv2/tstorebe/tstore/tlogclient.go | 199 +++++++++--------- politeiad/backendv2/tstorebe/tstore/tstore.go | 32 +-- 6 files changed, 144 insertions(+), 145 deletions(-) diff --git a/politeiad/backendv2/tstorebe/store/mysql/mysql.go b/politeiad/backendv2/tstorebe/store/mysql/mysql.go index e9ff73e09..9b1e16bac 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/mysql.go +++ b/politeiad/backendv2/tstorebe/store/mysql/mysql.go @@ -255,7 +255,7 @@ func New(appDir, host, user, password, dbname string) (*mysql, error) { } // Connect to database - log.Infof("Host: %v:[password]@tcp(%v)/%v", user, host, dbname) + log.Infof("MySQL: %v:[password]@tcp(%v)/%v", user, host, dbname) h := fmt.Sprintf("%v:%v@tcp(%v)/%v", user, password, host, dbname) db, err := sql.Open("mysql", h) diff --git a/politeiad/backendv2/tstorebe/tstore/anchor.go b/politeiad/backendv2/tstorebe/tstore/anchor.go index 7cb604c50..b9062bda6 100644 --- a/politeiad/backendv2/tstorebe/tstore/anchor.go +++ b/politeiad/backendv2/tstorebe/tstore/anchor.go @@ -153,9 +153,9 @@ func (t *Tstore) anchorForLeaf(treeID int64, merkleLeafHash []byte, leaves []*tr // errAnchorNotFound is returned if no anchor is found for the provided tree. func (t *Tstore) anchorLatest(treeID int64) (*anchor, error) { // Get tree leaves - leavesAll, err := t.tlog.leavesAll(treeID) + leavesAll, err := t.tlog.LeavesAll(treeID) if err != nil { - return nil, fmt.Errorf("leavesAll: %v", err) + return nil, fmt.Errorf("LeavesAll: %v", err) } // Find the most recent anchor leaf @@ -240,9 +240,9 @@ func (t *Tstore) anchorSave(a anchor) error { leaves := []*trillian.LogLeaf{ newLogLeaf(d, extraData), } - queued, _, err := t.tlog.leavesAppend(a.TreeID, leaves) + queued, _, err := t.tlog.LeavesAppend(a.TreeID, leaves) if err != nil { - return fmt.Errorf("leavesAppend: %v", err) + return fmt.Errorf("LeavesAppend: %v", err) } if len(queued) != 1 { return fmt.Errorf("wrong number of queud leaves: got %v, want 1", @@ -453,9 +453,9 @@ func (t *Tstore) anchorTrees() error { return nil } - trees, err := t.tlog.treesAll() + trees, err := t.tlog.TreesAll() if err != nil { - return fmt.Errorf("treesAll: %v", err) + return fmt.Errorf("TreesAll: %v", err) } // digests contains the SHA256 digests of the LogRootV1.RootHash @@ -481,9 +481,9 @@ func (t *Tstore) anchorTrees() error { case errors.Is(err, errAnchorNotFound): // Tree has not been anchored yet. Verify that the tree has // leaves. A tree with no leaves does not need to be anchored. - leavesAll, err := t.tlog.leavesAll(v.TreeId) + leavesAll, err := t.tlog.LeavesAll(v.TreeId) if err != nil { - return fmt.Errorf("leavesAll: %v", err) + return fmt.Errorf("LeavesAll: %v", err) } if len(leavesAll) == 0 { // Tree does not have any leaves. Nothing to do. @@ -497,9 +497,9 @@ func (t *Tstore) anchorTrees() error { default: // Anchor record found. If the anchor height differs from the // current height then the tree needs to be anchored. - _, lr, err := t.tlog.signedLogRoot(v) + _, lr, err := t.tlog.SignedLogRoot(v) if err != nil { - return fmt.Errorf("signedLogRoot %v: %v", v.TreeId, err) + return fmt.Errorf("SignedLogRoot %v: %v", v.TreeId, err) } // Subtract one from the current height to account for the // anchor leaf. @@ -512,9 +512,9 @@ func (t *Tstore) anchorTrees() error { // Tree has not been anchored at current height. Add it to the // list of anchors. - _, lr, err := t.tlog.signedLogRoot(v) + _, lr, err := t.tlog.SignedLogRoot(v) if err != nil { - return fmt.Errorf("signedLogRoot %v: %v", v.TreeId, err) + return fmt.Errorf("SignedLogRoot %v: %v", v.TreeId, err) } anchors = append(anchors, anchor{ TreeID: v.TreeId, diff --git a/politeiad/backendv2/tstorebe/tstore/client.go b/politeiad/backendv2/tstorebe/tstore/client.go index 8e43269df..9c0b0a2d0 100644 --- a/politeiad/backendv2/tstorebe/tstore/client.go +++ b/politeiad/backendv2/tstorebe/tstore/client.go @@ -32,9 +32,9 @@ func (t *Tstore) BlobSave(treeID int64, be store.BlobEntry) error { } // Verify tree is not frozen - leaves, err := t.tlog.leavesAll(treeID) + leaves, err := t.tlog.LeavesAll(treeID) if err != nil { - return fmt.Errorf("leavesAll: %v", err) + return fmt.Errorf("LeavesAll: %v", err) } idx, err := t.recordIndexLatest(leaves) if err != nil { @@ -100,9 +100,9 @@ func (t *Tstore) BlobSave(treeID int64, be store.BlobEntry) error { } // Append log leaf to trillian tree - queued, _, err := t.tlog.leavesAppend(treeID, leaves) + queued, _, err := t.tlog.LeavesAppend(treeID, leaves) if err != nil { - return fmt.Errorf("leavesAppend: %v", err) + return fmt.Errorf("LeavesAppend: %v", err) } if len(queued) != 1 { return fmt.Errorf("wrong queued leaves count: got %v, want 1", @@ -129,9 +129,9 @@ func (t *Tstore) BlobsDel(treeID int64, digests [][]byte) error { } // Get all tree leaves - leaves, err := t.tlog.leavesAll(treeID) + leaves, err := t.tlog.LeavesAll(treeID) if err != nil { - return fmt.Errorf("leavesAll: %v", err) + return fmt.Errorf("LeavesAll: %v", err) } // Put merkle leaf hashes into a map so that we can tell if a leaf @@ -184,9 +184,9 @@ func (t *Tstore) Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEnt } // Get leaves - leaves, err := t.tlog.leavesAll(treeID) + leaves, err := t.tlog.LeavesAll(treeID) if err != nil { - return nil, fmt.Errorf("leavesAll: %v", err) + return nil, fmt.Errorf("LeavesAll: %v", err) } // Determine if the record is vetted. If the record is vetted, only @@ -268,9 +268,9 @@ func (t *Tstore) BlobsByDataDesc(treeID int64, dataDesc []string) ([]store.BlobE } // Get leaves - leaves, err := t.tlog.leavesAll(treeID) + leaves, err := t.tlog.LeavesAll(treeID) if err != nil { - return nil, fmt.Errorf("leavesAll: %v", err) + return nil, fmt.Errorf("LeavesAll: %v", err) } // Find all matching leaves @@ -338,9 +338,9 @@ func (t *Tstore) DigestsByDataDesc(treeID int64, dataDesc []string) ([][]byte, e } // Get leaves - leaves, err := t.tlog.leavesAll(treeID) + leaves, err := t.tlog.LeavesAll(treeID) if err != nil { - return nil, fmt.Errorf("leavesAll: %v", err) + return nil, fmt.Errorf("LeavesAll: %v", err) } // Find all matching leaves @@ -364,9 +364,9 @@ func (t *Tstore) Timestamp(treeID int64, digest []byte) (*backend.Timestamp, err log.Tracef("Timestamp: %v %x", treeID, digest) // Get tree leaves - leaves, err := t.tlog.leavesAll(treeID) + leaves, err := t.tlog.LeavesAll(treeID) if err != nil { - return nil, fmt.Errorf("leavesAll: %v", err) + return nil, fmt.Errorf("LeavesAll: %v", err) } // Determine if the record is vetted diff --git a/politeiad/backendv2/tstorebe/tstore/recordindex.go b/politeiad/backendv2/tstorebe/tstore/recordindex.go index b0663d412..c16fc1ccb 100644 --- a/politeiad/backendv2/tstorebe/tstore/recordindex.go +++ b/politeiad/backendv2/tstorebe/tstore/recordindex.go @@ -120,9 +120,9 @@ func (t *Tstore) recordIndexSave(treeID int64, idx recordIndex) error { leaves := []*trillian.LogLeaf{ newLogLeaf(d, extraData), } - queued, _, err := t.tlog.leavesAppend(treeID, leaves) + queued, _, err := t.tlog.LeavesAppend(treeID, leaves) if err != nil { - return fmt.Errorf("leavesAppend: %v", err) + return fmt.Errorf("LeavesAppend: %v", err) } if len(queued) != 1 { return fmt.Errorf("wrong number of queud leaves: got %v, want 1", diff --git a/politeiad/backendv2/tstorebe/tstore/tlogclient.go b/politeiad/backendv2/tstorebe/tstore/tlogclient.go index 5ab62bbae..458ba79e5 100644 --- a/politeiad/backendv2/tstorebe/tstore/tlogclient.go +++ b/politeiad/backendv2/tstorebe/tstore/tlogclient.go @@ -8,7 +8,6 @@ import ( "bytes" "context" "crypto" - "crypto/sha256" "fmt" "io/ioutil" "math/rand" @@ -35,6 +34,13 @@ import ( "google.golang.org/grpc/status" ) +var ( + // hasher contains the log hasher that trillian uses to compute the + // merkle leaf hash for a log leaf. It can be used by tstore to + // compute the merkle leaf hash for a given leaf value. + hasher = rfc6962.New(crypto.SHA256) +) + const ( waitForInclusionTimeout = 120 * time.Second ) @@ -53,36 +59,36 @@ type queuedLeafProof struct { // TrillianAdminClient, creating a simplified API for the backend to use and // allowing us to create a implementation that can be used for testing. type tlogClient interface { - // treeNew creates a new tree. - treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) + // TreeNew creates a new tree. + TreeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) - // treeFreeze sets the status of a tree to frozen. - treeFreeze(treeID int64) (*trillian.Tree, error) + // TreeFreeze sets the status of a tree to frozen. + TreeFreeze(treeID int64) (*trillian.Tree, error) - // tree returns a tree. - tree(treeID int64) (*trillian.Tree, error) + // Tree returns a tree. + Tree(treeID int64) (*trillian.Tree, error) - // treesAll returns all trees in the trillian instance. - treesAll() ([]*trillian.Tree, error) + // TreesAll returns all trees in the trillian instance. + TreesAll() ([]*trillian.Tree, error) - // leavesAppend appends leaves to a tree. - leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, + // LeavesAppend appends leaves to a tree. + LeavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, *types.LogRootV1, error) - // leavesAll returns all leaves of a tree. - leavesAll(treeID int64) ([]*trillian.LogLeaf, error) + // LeavesAll returns all leaves of a tree. + LeavesAll(treeID int64) ([]*trillian.LogLeaf, error) - // signedLogRoot returns the signed log root for a tree. - signedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoot, + // SignedLogRoot returns the signed log root for a tree. + SignedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoot, *types.LogRootV1, error) - // inclusionProof returns a proof for the inclusion of a merkle + // InclusionProof returns a proof for the inclusion of a merkle // leaf hash in a log root. - inclusionProof(treeID int64, merkleLeafHashe []byte, + InclusionProof(treeID int64, merkleLeafHashe []byte, lrv1 *types.LogRootV1) (*trillian.Proof, error) - // close closes the client connection. - close() + // Close closes the client connection. + Close() } var ( @@ -100,30 +106,13 @@ type tclient struct { publicKey crypto.PublicKey // Trillian public key } -// merkleLeafHash returns the merkle leaf hash for the provided leaf value. -// This is the same merkle leaf hash that is calculated by trillian. -func merkleLeafHash(leafValue []byte) []byte { - h := sha256.New() - h.Write([]byte{rfc6962.RFC6962LeafHashPrefix}) - h.Write(leafValue) - return h.Sum(nil) -} - -// newLogLeaf returns a trillian LogLeaf. -func newLogLeaf(leafValue []byte, extraData []byte) *trillian.LogLeaf { - return &trillian.LogLeaf{ - LeafValue: leafValue, - ExtraData: extraData, - } -} - -// treeNew returns a new trillian tree and verifies that the signatures are +// TreeNew returns a new trillian tree and verifies that the signatures are // correct. It returns the tree and the signed log root which can be externally // verified. // // This function satisfies the tlogClient interface. -func (t *tclient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { - log.Tracef("trillian treeNew") +func (t *tclient) TreeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { + log.Tracef("trillian TreeNew") pk, err := ptypes.MarshalAny(t.privateKey) if err != nil { @@ -186,14 +175,14 @@ func (t *tclient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { return tree, ilr.Created, nil } -// treeFreeze updates the state of a tree to frozen. +// TreeFreeze updates the state of a tree to frozen. // // This function satisfies the tlogClient interface. -func (t *tclient) treeFreeze(treeID int64) (*trillian.Tree, error) { - log.Tracef("trillian treeFreeze: %v", treeID) +func (t *tclient) TreeFreeze(treeID int64) (*trillian.Tree, error) { + log.Tracef("trillian TreeFreeze: %v", treeID) // Get the current tree - tree, err := t.tree(treeID) + tree, err := t.Tree(treeID) if err != nil { return nil, fmt.Errorf("tree: %v", err) } @@ -215,10 +204,10 @@ func (t *tclient) treeFreeze(treeID int64) (*trillian.Tree, error) { return updated, nil } -// tree returns a trillian tree. +// Tree returns a trillian tree. // // This function satisfies the tlogClient interface. -func (t *tclient) tree(treeID int64) (*trillian.Tree, error) { +func (t *tclient) Tree(treeID int64) (*trillian.Tree, error) { log.Tracef("trillian tree: %v", treeID) tree, err := t.admin.GetTree(t.ctx, &trillian.GetTreeRequest{ @@ -236,11 +225,11 @@ func (t *tclient) tree(treeID int64) (*trillian.Tree, error) { return tree, nil } -// treesAll returns all trees in the trillian instance. +// TreesAll returns all trees in the trillian instance. // // This function satisfies the tlogClient interface -func (t *tclient) treesAll() ([]*trillian.Tree, error) { - log.Tracef("trillian treesAll") +func (t *tclient) TreesAll() ([]*trillian.Tree, error) { + log.Tracef("trillian TreesAll") ltr, err := t.admin.ListTrees(t.ctx, &trillian.ListTreesRequest{}) if err != nil { @@ -250,12 +239,12 @@ func (t *tclient) treesAll() ([]*trillian.Tree, error) { return ltr.Tree, nil } -// inclusionProof returns a proof for the inclusion of a merkle leaf hash in a +// InclusionProof returns a proof for the inclusion of a merkle leaf hash in a // log root. // // This function satisfies the tlogClient interface -func (t *tclient) inclusionProof(treeID int64, merkleLeafHash []byte, lrv1 *types.LogRootV1) (*trillian.Proof, error) { - log.Tracef("tillian inclusionProof: %v %x", treeID, merkleLeafHash) +func (t *tclient) InclusionProof(treeID int64, merkleLeafHash []byte, lrv1 *types.LogRootV1) (*trillian.Proof, error) { + log.Tracef("tillian InclusionProof: %v %x", treeID, merkleLeafHash) resp, err := t.log.GetInclusionProofByHash(t.ctx, &trillian.GetInclusionProofByHashRequest{ @@ -286,10 +275,10 @@ func (t *tclient) inclusionProof(treeID int64, merkleLeafHash []byte, lrv1 *type return proof, nil } -// signedLogRoot returns the signed log root of a trillian tree. +// SignedLogRoot returns the signed log root of a trillian tree. // // This function satisfies the tlogClient interface. -func (t *tclient) signedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoot, *types.LogRootV1, error) { +func (t *tclient) SignedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoot, *types.LogRootV1, error) { // Get the signed log root for the current tree height resp, err := t.log.GetLatestSignedLogRoot(t.ctx, &trillian.GetLatestSignedLogRootRequest{LogId: tree.TreeId}) @@ -311,12 +300,12 @@ func (t *tclient) signedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoot, * return resp.SignedLogRoot, lrv1, nil } -// leavesAppend appends the provided leaves onto the provided tree. The queued -// leaf and the leaf inclusion proof are returned. If a leaf was not -// successfully appended, the queued leaf will still be returned and the error -// will be in the queued leaf. Inclusion proofs will not exist for leaves that -// fail to be appended. Note leaves that are duplicates will fail and it is the -// callers responsibility to determine how they should be handled. +// LeavesAppend appends leaves onto a tlog tree. The queued leaf and the leaf +// inclusion proof are returned. If a leaf was not successfully appended, the +// queued leaf will still be returned and the error will be in the queued leaf. +// Inclusion proofs will not exist for leaves that fail to be appended. Note +// leaves that are duplicates will fail and it is the callers responsibility to +// determine how they should be handled. // // Trillian DOES NOT guarantee that the leaves of a queued leaves batch are // appended in the order in which they were received. Trillian is also not @@ -326,25 +315,23 @@ func (t *tclient) signedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoot, * // order. // // This function satisfies the tlogClient interface. -func (t *tclient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, *types.LogRootV1, error) { - log.Tracef("trillian leavesAppend: %v %v", treeID, len(leaves)) +func (t *tclient) LeavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, *types.LogRootV1, error) { + log.Tracef("trillian LeavesAppend: %v %v", treeID, len(leaves)) // Get the latest signed log root - tree, err := t.tree(treeID) + tree, err := t.Tree(treeID) if err != nil { return nil, nil, err } - slr, _, err := t.signedLogRoot(tree) + slr, _, err := t.SignedLogRoot(tree) if err != nil { - return nil, nil, fmt.Errorf("signedLogRoot pre update: %v", err) + return nil, nil, fmt.Errorf("SignedLogRoot pre update: %v", err) } - - // Ensure the tree is not frozen if tree.TreeState == trillian.TreeState_FROZEN { return nil, nil, fmt.Errorf("tree is frozen") } - // Append leaves to log + // Append leaves qlr, err := t.log.QueueLeaves(t.ctx, &trillian.QueueLeavesRequest{ LogId: treeID, Leaves: leaves, @@ -394,9 +381,9 @@ func (t *tclient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queu log.Debugf("Appended leaves (%v) to tree %v", len(leaves), treeID) // Get the latest signed log root - _, lr, err := t.signedLogRoot(tree) + _, lr, err := t.SignedLogRoot(tree) if err != nil { - return nil, nil, fmt.Errorf("signedLogRoot post update: %v", err) + return nil, nil, fmt.Errorf("SignedLogRoot post update: %v", err) } // Get inclusion proofs @@ -424,9 +411,9 @@ func (t *tclient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queu // The LeafIndex of a QueuedLogLeaf will not be set. Get the // inclusion proof by MerkleLeafHash. - qlp.Proof, err = t.inclusionProof(treeID, v.Leaf.MerkleLeafHash, lr) + qlp.Proof, err = t.InclusionProof(treeID, v.Leaf.MerkleLeafHash, lr) if err != nil { - return nil, nil, fmt.Errorf("inclusionProof %v %x: %v", + return nil, nil, fmt.Errorf("InclusionProof %v %x: %v", treeID, v.Leaf.MerkleLeafHash, err) } } @@ -463,22 +450,22 @@ func (t *tclient) leavesByRange(treeID int64, startIndex, count int64) ([]*trill return glbrr.Leaves, nil } -// leavesAll returns all of the leaves for the provided treeID. +// LeavesAll returns all of the leaves for the provided treeID. // // This function satisfies the tlogClient interface. -func (t *tclient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { - log.Tracef("trillian leavesAll: %v", treeID) +func (t *tclient) LeavesAll(treeID int64) ([]*trillian.LogLeaf, error) { + log.Tracef("trillian LeavesAll: %v", treeID) // Get tree - tree, err := t.tree(treeID) + tree, err := t.Tree(treeID) if err != nil { return nil, fmt.Errorf("tree: %v", err) } // Get signed log root - _, lr, err := t.signedLogRoot(tree) + _, lr, err := t.SignedLogRoot(tree) if err != nil { - return nil, fmt.Errorf("signedLogRoot: %v", err) + return nil, fmt.Errorf("SignedLogRoot: %v", err) } if lr.TreeSize == 0 { return []*trillian.LogLeaf{}, nil @@ -493,15 +480,28 @@ func (t *tclient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { return leaves, nil } -// close closes the trillian grpc connection. +// Close closes the trillian grpc connection. // // This function satisfies the tlogClient interface. -func (t *tclient) close() { - log.Tracef("trillian close %v", t.host) +func (t *tclient) Close() { + log.Tracef("trillian Close %v", t.host) t.grpc.Close() } +// merkleLeafHash returns the merkle leaf hash for the provided leaf value. +// This is the same merkle leaf hash that is calculated by trillian. +func merkleLeafHash(leafValue []byte) []byte { + return hasher.HashLeaf(leafValue) +} + +func newLogLeaf(leafValue []byte, extraData []byte) *trillian.LogLeaf { + return &trillian.LogLeaf{ + LeafValue: leafValue, + ExtraData: extraData, + } +} + func newTrillianKey() (crypto.Signer, error) { return keys.NewFromSpec(&keyspb.Specification{ Params: &keyspb.Specification_Ed25519Params{}, @@ -532,10 +532,9 @@ func newTClient(host, keyFile string) (*tclient, error) { // Default gprc max message size is ~4MB (4194304 bytes). This is // not large enough for trees with tens of thousands of leaves. // Increase it to 20MB. - maxMsgSize := grpc.WithMaxMsgSize(20000000) + maxMsgSize := grpc.WithMaxMsgSize(20 * 1024 * 1024) // Setup trillian connection - // TODO should this be WithInsecure? g, err := grpc.Dial(host, grpc.WithInsecure(), maxMsgSize) if err != nil { return nil, fmt.Errorf("grpc dial: %v", err) @@ -589,10 +588,10 @@ type testTClient struct { publicKey crypto.PublicKey // Trillian public key } -// treeNew ceates a new trillian tree in memory. +// TreeNew ceates a new trillian tree in memory. // // This function satisfies the tlogClient interface. -func (t *testTClient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { +func (t *testTClient) TreeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { t.Lock() defer t.Unlock() @@ -623,10 +622,10 @@ func (t *testTClient) treeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) return &tree, nil, nil } -// treeFreeze sets the state of a tree to frozen. +// TreeFreeze sets the state of a tree to frozen. // // This function satisfies the tlogClient interface. -func (t *testTClient) treeFreeze(treeID int64) (*trillian.Tree, error) { +func (t *testTClient) TreeFreeze(treeID int64) (*trillian.Tree, error) { t.Lock() defer t.Unlock() @@ -635,10 +634,10 @@ func (t *testTClient) treeFreeze(treeID int64) (*trillian.Tree, error) { return t.trees[treeID], nil } -// tree returns trillian tree from passed in ID. +// Tree returns trillian tree from passed in ID. // // This function satisfies the tlogClient interface. -func (t *testTClient) tree(treeID int64) (*trillian.Tree, error) { +func (t *testTClient) Tree(treeID int64) (*trillian.Tree, error) { t.Lock() defer t.Unlock() @@ -649,11 +648,11 @@ func (t *testTClient) tree(treeID int64) (*trillian.Tree, error) { return nil, fmt.Errorf("tree ID not found") } -// treesAll signed log roots are not used for testing up until now, so we +// TreesAll signed log roots are not used for testing up until now, so we // return a nil value for it. // // This function satisfies the tlogClient interface. -func (t *testTClient) treesAll() ([]*trillian.Tree, error) { +func (t *testTClient) TreesAll() ([]*trillian.Tree, error) { t.Lock() defer t.Unlock() @@ -665,11 +664,11 @@ func (t *testTClient) treesAll() ([]*trillian.Tree, error) { return trees, nil } -// leavesAppend satisfies the TClient interface. It appends leaves to the +// LeavesAppend satisfies the TClient interface. It appends leaves to the // corresponding trillian tree in memory. // // This function satisfies the tlogClient interface. -func (t *testTClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, *types.LogRootV1, error) { +func (t *testTClient) LeavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, *types.LogRootV1, error) { t.Lock() defer t.Unlock() @@ -702,10 +701,10 @@ func (t *testTClient) leavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([] return queued, nil, nil } -// leavesAll returns all leaves from a trillian tree. +// LeavesAll returns all leaves from a trillian tree. // // This function satisfies the tlogClient interface. -func (t *testTClient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { +func (t *testTClient) LeavesAll(treeID int64) ([]*trillian.LogLeaf, error) { t.Lock() defer t.Unlock() @@ -718,25 +717,25 @@ func (t *testTClient) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { return t.leaves[treeID], nil } -// signedLogRoot has not been implemented yet for the test client. +// SignedLogRoot has not been implemented yet for the test client. // // This function satisfies the tlogClient interface. -func (t *testTClient) signedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoot, *types.LogRootV1, error) { +func (t *testTClient) SignedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoot, *types.LogRootV1, error) { return nil, nil, nil } -// inclusionProof has not been implement yet for the test client. +// InclusionProof has not been implement yet for the test client. // // This function satisfies the tlogClient interface. -func (t *testTClient) inclusionProof(treeID int64, merkleLeafHash []byte, lr *types.LogRootV1) (*trillian.Proof, error) { +func (t *testTClient) InclusionProof(treeID int64, merkleLeafHash []byte, lr *types.LogRootV1) (*trillian.Proof, error) { return nil, nil } -// close closes the trillian client connection. There is nothing to do for the +// Close closes the trillian client connection. There is nothing to do for the // test implementation. // // This function satisfies the tlogClient interface. -func (t *testTClient) close() {} +func (t *testTClient) Close() {} // newTestTClient returns a new testTClient. func newTestTClient(t *testing.T) *testTClient { diff --git a/politeiad/backendv2/tstorebe/tstore/tstore.go b/politeiad/backendv2/tstorebe/tstore/tstore.go index e1c32e83e..522d5d3e6 100644 --- a/politeiad/backendv2/tstorebe/tstore/tstore.go +++ b/politeiad/backendv2/tstorebe/tstore/tstore.go @@ -164,7 +164,7 @@ func merkleLeafHashForBlobEntry(be store.BlobEntry) ([]byte, error) { func (t *Tstore) TreeNew() (int64, error) { log.Tracef("TreeNew") - tree, _, err := t.tlog.treeNew() + tree, _, err := t.tlog.TreeNew() if err != nil { return 0, err } @@ -199,7 +199,7 @@ func (t *Tstore) TreeFreeze(treeID int64, rm backend.RecordMetadata, metadata [] // TreesAll returns the IDs of all trees in the tstore instance. func (t *Tstore) TreesAll() ([]int64, error) { - trees, err := t.tlog.treesAll() + trees, err := t.tlog.TreesAll() if err != nil { return nil, err } @@ -215,7 +215,7 @@ func (t *Tstore) TreesAll() ([]int64, error) { // tree to have been created but experienced an unexpected error prior to the // record being saved. func (t *Tstore) TreeExists(treeID int64) bool { - _, err := t.tlog.tree(treeID) + _, err := t.tlog.Tree(treeID) return err == nil } @@ -500,9 +500,9 @@ func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, re } // Append leaves onto the trillian tree - queued, _, err := t.tlog.leavesAppend(treeID, leaves) + queued, _, err := t.tlog.LeavesAppend(treeID, leaves) if err != nil { - return nil, fmt.Errorf("leavesAppend: %v", err) + return nil, fmt.Errorf("LeavesAppend: %v", err) } failed := make([]string, 0, len(queued)) for _, v := range queued { @@ -586,9 +586,9 @@ func (t *Tstore) recordSave(treeID int64, rm backend.RecordMetadata, metadata [] } // Get tree leaves - leavesAll, err := t.tlog.leavesAll(treeID) + leavesAll, err := t.tlog.LeavesAll(treeID) if err != nil { - return nil, fmt.Errorf("leavesAll %v: %v", treeID, err) + return nil, fmt.Errorf("LeavesAll %v: %v", treeID, err) } // Get the existing record index @@ -656,7 +656,7 @@ func (t *Tstore) RecordDel(treeID int64) error { } // Get all tree leaves - leavesAll, err := t.tlog.leavesAll(treeID) + leavesAll, err := t.tlog.LeavesAll(treeID) if err != nil { return err } @@ -724,9 +724,9 @@ func (t *Tstore) record(treeID int64, version uint32, filenames []string, omitAl } // Get tree leaves - leaves, err := t.tlog.leavesAll(treeID) + leaves, err := t.tlog.LeavesAll(treeID) if err != nil { - return nil, fmt.Errorf("leavesAll %v: %v", treeID, err) + return nil, fmt.Errorf("LeavesAll %v: %v", treeID, err) } // Use the record index to pull the record content from the store. @@ -940,7 +940,7 @@ func recordIsVetted(leaves []*trillian.LogLeaf) bool { func (t *Tstore) RecordState(treeID int64) (backend.StateT, error) { log.Tracef("RecordState: %v", treeID) - leaves, err := t.tlog.leavesAll(treeID) + leaves, err := t.tlog.LeavesAll(treeID) if err != nil { return 0, err } @@ -1016,9 +1016,9 @@ func (t *Tstore) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trilli } // Get trillian inclusion proof - p, err := t.tlog.inclusionProof(treeID, l.MerkleLeafHash, a.LogRoot) + p, err := t.tlog.InclusionProof(treeID, l.MerkleLeafHash, a.LogRoot) if err != nil { - return nil, fmt.Errorf("inclusionProof %v %x: %v", + return nil, fmt.Errorf("InclusionProof %v %x: %v", treeID, l.MerkleLeafHash, err) } @@ -1099,9 +1099,9 @@ func (t *Tstore) RecordTimestamps(treeID int64, version uint32, token []byte) (* } // Get record index - leaves, err := t.tlog.leavesAll(treeID) + leaves, err := t.tlog.LeavesAll(treeID) if err != nil { - return nil, fmt.Errorf("leavesAll %v: %v", treeID, err) + return nil, fmt.Errorf("LeavesAll %v: %v", treeID, err) } idx, err := t.recordIndex(leaves, version) if err != nil { @@ -1161,7 +1161,7 @@ func (t *Tstore) Close() { // Close connections t.store.Close() - t.tlog.close() + t.tlog.Close() } func New(appDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianSigningKeyFile, dbType, dbHost, dbPass, dcrtimeHost, dcrtimeCert string) (*Tstore, error) { From 610fc96f5f08a2ffb3387ee1d4025d585ea07bf0 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 15 Mar 2021 19:44:48 -0500 Subject: [PATCH 404/449] tstorebe: Retry vote collider on dup error. --- .../backendv2/tstorebe/plugins/plugins.go | 8 ++++++++ .../tstorebe/plugins/ticketvote/cmds.go | 19 ++++++++++--------- politeiad/backendv2/tstorebe/tstore/client.go | 8 +++++++- .../backendv2/tstorebe/tstore/tlogclient.go | 9 +++++++-- politeiad/plugins/ticketvote/ticketvote.go | 3 +-- 5 files changed, 33 insertions(+), 14 deletions(-) diff --git a/politeiad/backendv2/tstorebe/plugins/plugins.go b/politeiad/backendv2/tstorebe/plugins/plugins.go index 537f5fd30..fb7d5ac1a 100644 --- a/politeiad/backendv2/tstorebe/plugins/plugins.go +++ b/politeiad/backendv2/tstorebe/plugins/plugins.go @@ -5,10 +5,18 @@ package plugins import ( + "errors" + backend "github.com/decred/politeia/politeiad/backendv2" "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" ) +var ( + // ErrDuplicateBlob is returned when a plugin attempts to save a + // blob whose contents are an exact duplicate of an existing blob. + ErrDuplicateBlob = errors.New("duplicate blob") +) + // HookT represents a plugin hook. type HookT int diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go index cd92a57d6..e92e517a1 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go @@ -23,6 +23,7 @@ import ( "github.com/decred/dcrd/dcrutil/v3" "github.com/decred/dcrd/wire" backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins" "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" "github.com/decred/politeia/politeiad/plugins/dcrdata" "github.com/decred/politeia/politeiad/plugins/ticketvote" @@ -2039,11 +2040,7 @@ func (p *ticketVotePlugin) ballot(treeID int64, votes []ticketvote.CastVote, res // Decrement wait group counter once vote is cast defer wg.Done() - // Setup cast vote details. The timestamp is included in a cast - // vote so that the vote can be retried if the vote collider - // is not saved succesfully. Without the timestamp, attempting - // to save the cast vote details multiple times would result in - // a duplicate leaf error from tlog. + // Setup cast vote details receipt := p.identity.SignMessage([]byte(v.Signature)) cv := ticketvote.CastVoteDetails{ Token: v.Token, @@ -2051,10 +2048,9 @@ func (p *ticketVotePlugin) ballot(treeID int64, votes []ticketvote.CastVote, res VoteBit: v.VoteBit, Signature: v.Signature, Receipt: hex.EncodeToString(receipt[:]), - Timestamp: time.Now().Unix(), } - // Declare variables here to prevent goto errors + // Declare here to prevent goto errors var ( cvr ticketvote.CastVoteReply vc voteCollider @@ -2062,7 +2058,12 @@ func (p *ticketVotePlugin) ballot(treeID int64, votes []ticketvote.CastVote, res // Save cast vote err := p.castVoteSave(treeID, cv) - if err != nil { + if err == plugins.ErrDuplicateBlob { + // This cast vote has already been saved. Its possible that + // a previous attempt to vote with this ticket failed before + // the vote collider could be saved. Continue execution so + // that we re-attempt to save the vote collider. + } else if err != nil { t := time.Now().Unix() log.Errorf("cmdCastBallot: castVoteSave %v: %v", t, err) e := ticketvote.VoteErrorInternalError @@ -2214,7 +2215,7 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str continue } - // Verify ticket has not already vote + // Verify ticket has not already voted if p.activeVotes.VoteIsDuplicate(v.Token, v.Ticket) { e := ticketvote.VoteErrorTicketAlreadyVoted receipts[k].Ticket = v.Ticket diff --git a/politeiad/backendv2/tstorebe/tstore/client.go b/politeiad/backendv2/tstorebe/tstore/client.go index 9c0b0a2d0..764b1d8ae 100644 --- a/politeiad/backendv2/tstorebe/tstore/client.go +++ b/politeiad/backendv2/tstorebe/tstore/client.go @@ -12,6 +12,7 @@ import ( "fmt" backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins" "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" "github.com/google/trillian" "google.golang.org/grpc/codes" @@ -109,7 +110,12 @@ func (t *Tstore) BlobSave(treeID int64, be store.BlobEntry) error { len(queued)) } c := codes.Code(queued[0].QueuedLeaf.GetStatus().GetCode()) - if c != codes.OK { + switch c { + case codes.OK: + // This is ok; continue + case codes.AlreadyExists: + return plugins.ErrDuplicateBlob + default: return fmt.Errorf("queued leaf error: %v", c) } diff --git a/politeiad/backendv2/tstorebe/tstore/tlogclient.go b/politeiad/backendv2/tstorebe/tstore/tlogclient.go index 458ba79e5..df88f360d 100644 --- a/politeiad/backendv2/tstorebe/tstore/tlogclient.go +++ b/politeiad/backendv2/tstorebe/tstore/tlogclient.go @@ -378,8 +378,6 @@ func (t *tclient) LeavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queu } } - log.Debugf("Appended leaves (%v) to tree %v", len(leaves), treeID) - // Get the latest signed log root _, lr, err := t.SignedLogRoot(tree) if err != nil { @@ -388,6 +386,7 @@ func (t *tclient) LeavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queu // Get inclusion proofs proofs := make([]queuedLeafProof, 0, len(qlr.QueuedLeaves)) + var failed int for _, v := range qlr.QueuedLeaves { qlp := queuedLeafProof{ QueuedLeaf: v, @@ -416,6 +415,9 @@ func (t *tclient) LeavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queu return nil, nil, fmt.Errorf("InclusionProof %v %x: %v", treeID, v.Leaf.MerkleLeafHash, err) } + } else { + // Leaf contains an error + failed++ } proofs = append(proofs, qlp) @@ -427,6 +429,9 @@ func (t *tclient) LeavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queu len(proofs), len(leaves)) } + log.Debugf("Appended leaves (%v/%v) to tree %v", + len(leaves)-failed, len(leaves), treeID) + return proofs, lr, nil } diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index bfcf61479..f4e7b30c4 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -349,8 +349,7 @@ type CastVoteDetails struct { Signature string `json:"signature"` // Client signature // Metdata generated by server - Timestamp int64 `json:"timestamp"` - Receipt string `json:"receipt"` // Server signature + Receipt string `json:"receipt"` // Server signature } // AuthActionT represents the ticket vote authorization actions. From b28644a4d276804f0cd9674b411e28bfbd81477b Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 16 Mar 2021 13:11:32 -0500 Subject: [PATCH 405/449] politeiad: Revert all v1 API changes. --- mdstream/mdstream.go | 1 + politeiad/api/v1/v1.go | 85 +++---- politeiad/backend/backend.go | 117 ++++----- politeiad/backend/gitbe/gitbe.go | 18 +- politeiad/backend/gitbe/gitbe_test.go | 2 +- politeiad/backendv2/tstorebe/store/store.go | 3 +- politeiad/client/pdv1.go | 79 ++----- politeiad/politeiad.go | 2 - politeiad/v1.go | 249 ++++++++------------ politeiawww/www.go | 2 - 10 files changed, 209 insertions(+), 349 deletions(-) diff --git a/mdstream/mdstream.go b/mdstream/mdstream.go index abf01b4f3..6da8866ac 100644 --- a/mdstream/mdstream.go +++ b/mdstream/mdstream.go @@ -21,6 +21,7 @@ import ( const ( // mdstream IDs + IDInvalid = 0 IDRecordStatusChange = 2 IDInvoiceGeneral = 3 IDInvoiceStatusChange = 4 diff --git a/politeiad/api/v1/v1.go b/politeiad/api/v1/v1.go index 3a547773c..7ac75b063 100644 --- a/politeiad/api/v1/v1.go +++ b/politeiad/api/v1/v1.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2021 The Decred developers +// Copyright (c) 2017-2019 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -9,7 +9,6 @@ import ( "encoding/base64" "encoding/hex" "errors" - "fmt" "regexp" "github.com/decred/dcrtime/merkle" @@ -22,27 +21,24 @@ type RecordStatusT int const ( // Routes - IdentityRoute = "/v1/identity/" // Retrieve identity - NewRecordRoute = "/v1/newrecord/" // New record - UpdateUnvettedRoute = "/v1/updateunvetted/" // Update unvetted record - UpdateUnvettedMetadataRoute = "/v1/updateunvettedmd/" // Update unvetted metadata - UpdateVettedRoute = "/v1/updatevetted/" // Update vetted record - UpdateVettedMetadataRoute = "/v1/updatevettedmd/" // Update vetted metadata - GetUnvettedRoute = "/v1/getunvetted/" // Get unvetted record - GetVettedRoute = "/v1/getvetted/" // Get vetted record + IdentityRoute = "/v1/identity/" // Retrieve identity + NewRecordRoute = "/v1/newrecord/" // New record + UpdateUnvettedRoute = "/v1/updateunvetted/" // Update unvetted record + UpdateVettedRoute = "/v1/updatevetted/" // Update vetted record + UpdateVettedMetadataRoute = "/v1/updatevettedmd/" // Update vetted metadata + GetUnvettedRoute = "/v1/getunvetted/" // Retrieve unvetted record + GetVettedRoute = "/v1/getvetted/" // Retrieve vetted record // Auth required - InventoryRoute = "/v1/inventory/" // Inventory records - SetUnvettedStatusRoute = "/v1/setunvettedstatus/" // Set unvetted status - SetVettedStatusRoute = "/v1/setvettedstatus/" // Set vetted status - PluginCommandRoute = "/v1/plugin/" // Send a command to a plugin - PluginInventoryRoute = "/v1/plugin/inventory/" // Inventory all plugins - - ChallengeSize = 32 // Size of challenge token in bytes - - // TokenSize is the size of a censorship record token in bytes. - TokenSize = 32 - + InventoryRoute = "/v1/inventory/" // Inventory records + SetUnvettedStatusRoute = "/v1/setunvettedstatus/" // Set unvetted status + SetVettedStatusRoute = "/v1/setvettedstatus/" // Set vetted status + PluginCommandRoute = "/v1/plugin/" // Send a command to a plugin + PluginInventoryRoute = PluginCommandRoute + "inventory/" // Inventory all plugins + UpdateReadmeRoute = "/v1/updatereadme/" // Update README + + ChallengeSize = 32 // Size of challenge token in bytes + TokenSize = 32 // Size of token MetadataStreamsMax = uint64(16) // Maximum number of metadata streams // Error status codes @@ -63,12 +59,10 @@ const ( ErrorStatusNoChanges ErrorStatusT = 14 ErrorStatusRecordFound ErrorStatusT = 15 ErrorStatusInvalidRPCCredentials ErrorStatusT = 16 - ErrorStatusInvalidToken ErrorStatusT = 17 - ErrorStatusRecordNotFound ErrorStatusT = 18 // Record status codes (set and get) RecordStatusInvalid RecordStatusT = 0 // Invalid status - RecordStatusNotFound RecordStatusT = 1 // Record not found (deprecated) + RecordStatusNotFound RecordStatusT = 1 // Record not found RecordStatusNotReviewed RecordStatusT = 2 // Record has not been reviewed RecordStatusCensored RecordStatusT = 3 // Record has been censored RecordStatusPublic RecordStatusT = 4 // Record is publicly visible @@ -102,8 +96,8 @@ var ( ErrorStatusDuplicateFilename: "duplicate filename", ErrorStatusFileNotFound: "file not found", ErrorStatusNoChanges: "no changes in record", - ErrorStatusInvalidToken: "invalid token", - ErrorStatusRecordNotFound: "record not found", + ErrorStatusRecordFound: "record found", + ErrorStatusInvalidRPCCredentials: "invalid RPC client credentials", } // RecordStatus converts record status codes to human readable text. @@ -127,7 +121,7 @@ var ( ErrCorrupt = errors.New("signature verification failed") // Length of prefix of token used for lookups. The length 7 was selected to - // match github's abbreviated hash length. This is a var so that it can be + // match github's abbreviated hash length This is a var so that it can be // updated during testing. TokenPrefixLength = 7 ) @@ -232,7 +226,8 @@ type Record struct { } // NewRecord creates a new record. It must include all files that are part of -// the record and it may contain an optional metatda record. +// the record and it may contain an optional metatda record. Thet optional +// metadatarecord must be string encoded. type NewRecord struct { Challenge string `json:"challenge"` // Random challenge Metadata []MetadataStream `json:"metadata"` // Metadata streams @@ -250,7 +245,6 @@ type NewRecordReply struct { type GetUnvetted struct { Challenge string `json:"challenge"` // Random challenge Token string `json:"token"` // Censorship token - Version string `json:"version"` // Record version } // GetUnvettedReply returns an unvetted record. It retrieves the censorship @@ -285,11 +279,10 @@ type SetUnvettedStatus struct { MDOverwrite []MetadataStream `json:"mdoverwrite"` // Metadata streams to overwrite } -// SetUnvettedStatusReply is a response to a SetUnvettedStatus. It returns the +// SetUnvettedStatus is a response to a SetUnvettedStatus. It returns the // potentially modified record without the Files. type SetUnvettedStatusReply struct { Response string `json:"response"` // Challenge response - Record Record `json:"record"` // Record } // SetVettedStatus updates the status of a vetted record. This is used to @@ -306,11 +299,9 @@ type SetVettedStatus struct { // potentially modified record without the Files. type SetVettedStatusReply struct { Response string `json:"response"` // Challenge response - Record Record `json:"record"` // Record } -// UpdateRecord updates a record. This is used for both unvetted and vetted -// records. +// UpdateRecord update an unvetted record. type UpdateRecord struct { Challenge string `json:"challenge"` // Random challenge Token string `json:"token"` // Censorship token @@ -324,7 +315,6 @@ type UpdateRecord struct { // changed. Metadata only updates do not create a new CensorshipRecord. type UpdateRecordReply struct { Response string `json:"response"` // Challenge response - Record Record `json:"record"` // Record } // UpdateVettedMetadata update a vetted metadata. This is allowed for @@ -342,16 +332,15 @@ type UpdateVettedMetadataReply struct { Response string `json:"response"` // Challenge response } -// UpdateUnvettedMetadata update a unvetted metadata. -type UpdateUnvettedMetadata struct { - Challenge string `json:"challenge"` // Random challenge - Token string `json:"token"` // Censorship token - MDAppend []MetadataStream `json:"mdappend"` // Metadata streams to append - MDOverwrite []MetadataStream `json:"mdoverwrite"` // Metadata streams to overwrite +// UpdateReadme updated the README.md file in the vetted and unvetted repos. +type UpdateReadme struct { + Challenge string `json:"challenge"` // Random challenge + Content string `json:"content"` // New content of README.md } -// UpdateUnvettedMetadataReply returns a response challenge. -type UpdateUnvettedMetadataReply struct { +// UpdateReadmeReply returns a response challenge to an +// UpdateReadme command. +type UpdateReadmeReply struct { Response string `json:"response"` // Challenge response } @@ -390,22 +379,12 @@ type UserErrorReply struct { ErrorContext []string `json:"errorcontext,omitempty"` // Additional error information } -// Error satisfies the error interface. -func (e UserErrorReply) Error() string { - return fmt.Sprintf("user error code: %v", e.ErrorCode) -} - // ServerErrorReply returns an error code that can be correlated with // server logs. type ServerErrorReply struct { ErrorCode int64 `json:"code"` // Server error code } -// Error satisfies the error interface. -func (e ServerErrorReply) Error() string { - return fmt.Sprintf("server error: %v", e.ErrorCode) -} - // PluginSetting is a structure that holds key/value pairs of a plugin setting. type PluginSetting struct { Key string `json:"key"` // Name of setting diff --git a/politeiad/backend/backend.go b/politeiad/backend/backend.go index e9bf22afb..dd66db20d 100644 --- a/politeiad/backend/backend.go +++ b/politeiad/backend/backend.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2021 The Decred developers +// Copyright (c) 2017-2019 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -10,7 +10,6 @@ import ( "regexp" v1 "github.com/decred/politeia/politeiad/api/v1" - "github.com/decred/politeia/politeiad/api/v1/identity" ) var ( @@ -35,21 +34,14 @@ var ( // expected. ErrChangesRecord = errors.New("changes record") - // ErrRecordLocked is returned when a record status is one that - // does not allow any further changes. - ErrRecordLocked = errors.New("record is locked") + // ErrRecordArchived is returned when an update was attempted on a + // archived record. + ErrRecordArchived = errors.New("record is archived") - // ErrJournalsNotReplayed is returned when the journals have not - // been replayed and the subsequent code expect it to be replayed. + // ErrJournalsNotReplayed is returned when the journals have not been replayed + // and the subsequent code expect it to be replayed ErrJournalsNotReplayed = errors.New("journals have not been replayed") - // ErrPluginInvalid is emitted when an invalid plugin ID is used. - ErrPluginInvalid = errors.New("plugin invalid") - - // ErrPluginCmdInvalid is emitted when an invalid plugin command is - // used. - ErrPluginCmdInvalid = errors.New("plugin command invalid") - // Plugin names must be all lowercase letters and have a length of <20 PluginRE = regexp.MustCompile(`^[a-z]{1,20}$`) ) @@ -61,20 +53,17 @@ type ContentVerificationError struct { ErrorContext []string } -// Error satisfies the error interface. func (c ContentVerificationError) Error() string { return fmt.Sprintf("%v: %v", v1.ErrorStatus[c.ErrorCode], c.ErrorContext) } -// File represents a record file. type File struct { - Name string `json:"name"` // Basename of the file - MIME string `json:"mime"` // MIME type - Digest string `json:"digest"` // SHA256 of decoded Payload - Payload string `json:"payload"` // base64 encoded file + Name string // Basename of the file + MIME string // MIME type + Digest string // SHA256 of decoded Payload + Payload string // base64 encoded file } -// MDStatusT represents the status of a backend record. type MDStatusT int const ( @@ -105,7 +94,6 @@ type StateTransitionError struct { To MDStatusT } -// Error satisfies the error interface. func (s StateTransitionError) Error() string { return fmt.Sprintf("invalid record status transition %v (%v) -> %v (%v)", s.From, MDStatus[s.From], s.To, MDStatus[s.To]) @@ -114,21 +102,20 @@ func (s StateTransitionError) Error() string { // RecordMetadata is the metadata of a record. const VersionRecordMD = 1 -// RecordMetadata represents metadata that is created by the backend on record -// submission and updates. type RecordMetadata struct { Version uint64 `json:"version"` // Version of the scruture Iteration uint64 `json:"iteration"` // Iteration count of record Status MDStatusT `json:"status"` // Current status of the record Merkle string `json:"merkle"` // Merkle root of all files in record Timestamp int64 `json:"timestamp"` // Last updated - Token string `json:"token"` // Record authentication token, hex encoded + Token string `json:"token"` // Record authentication token } -// MetadataStream describes a single metada stream. +// MetadataStream describes a single metada stream. The ID determines how and +// where it is stored. type MetadataStream struct { - ID uint64 `json:"id"` // Stream identity - Payload string `json:"payload"` // String encoded metadata + ID uint64 // Stream identity + Payload string // String encoded metadata } // Record is a permanent Record that includes the submitted files, metadata and @@ -140,7 +127,7 @@ type Record struct { Files []File // User provided files } -// PluginSettings are used to specify settings for a plugin at runtime. +// PluginSettings type PluginSetting struct { Key string // Name of setting Value string // Value of setting @@ -151,66 +138,62 @@ type Plugin struct { ID string // Identifier Version string // Version Settings []PluginSetting // Settings - - // Identity contains the full identity that the plugin uses to - // create receipts, i.e. signatures of user provided data that - // prove the backend received and processed a plugin command. - Identity *identity.FullIdentity } -// Backend provides an API for creating and editing records. When a record is -// first submitted it is considered to be an unvetted, i.e. non-public, record. -// Once the status of the record is updated to a public status, the record is -// considered to be vetted. type Backend interface { // Create new record New([]MetadataStream, []File) (*RecordMetadata, error) - // Update unvetted record - UpdateUnvettedRecord(token []byte, mdAppend, mdOverwrite []MetadataStream, - filesAdd []File, filesDel []string) (*Record, error) - - // Update vetted record - UpdateVettedRecord(token []byte, mdAppend, mdOverwrite []MetadataStream, - filesAdd []File, filesDel []string) (*Record, error) - - // Update unvetted metadata - UpdateUnvettedMetadata(token []byte, mdAppend, - mdOverwrite []MetadataStream) error + // Update unvetted record (token, mdAppend, mdOverwrite, fAdd, fDelete) + UpdateUnvettedRecord([]byte, []MetadataStream, []MetadataStream, []File, + []string) (*Record, error) - // Update vetted metadata - UpdateVettedMetadata(token []byte, mdAppend, - mdOverwrite []MetadataStream) error + // Update vetted record (token, mdAppend, mdOverwrite, fAdd, fDelete) + UpdateVettedRecord([]byte, []MetadataStream, []MetadataStream, []File, + []string) (*Record, error) - // Set unvetted record status - SetUnvettedStatus(token []byte, s MDStatusT, mdAppend, - mdOverwrite []MetadataStream) (*Record, error) + // Update vetted metadata (token, mdAppend, mdOverwrite) + UpdateVettedMetadata([]byte, []MetadataStream, + []MetadataStream) error - // Set vetted record status - SetVettedStatus(token []byte, s MDStatusT, mdAppend, - mdOverwrite []MetadataStream) (*Record, error) + // Update README.md file at the root of git repo + UpdateReadme(string) error // Check if an unvetted record exists - UnvettedExists(token []byte) bool + UnvettedExists([]byte) bool // Check if a vetted record exists - VettedExists(token []byte) bool + VettedExists([]byte) bool + + // Get all unvetted record tokens + UnvettedTokens() ([][]byte, error) + + // Get all vetted record tokens + VettedTokens() ([][]byte, error) // Get unvetted record - GetUnvetted(token []byte, version string) (*Record, error) + GetUnvetted([]byte) (*Record, error) // Get vetted record - GetVetted(token []byte, version string) (*Record, error) + GetVetted([]byte, string) (*Record, error) - // Inventory retrieves various record records - Inventory(uint, uint, uint, bool, bool) ([]Record, []Record, error) + // Set unvetted record status + SetUnvettedStatus([]byte, MDStatusT, []MetadataStream, + []MetadataStream) (*Record, error) - // Plugin pass-through command - Plugin(string, string) (string, string, error) // command type, payload, error + // Set vetted record status + SetVettedStatus([]byte, MDStatusT, []MetadataStream, + []MetadataStream) (*Record, error) + + // Inventory retrieves various record records. + Inventory(uint, uint, uint, bool, bool) ([]Record, []Record, error) // Obtain plugin settings GetPlugins() ([]Plugin, error) - // Close performs cleanup of the backend + // Plugin pass-through command + Plugin(string, string) (string, string, error) // command type, payload, error + + // Close performs cleanup of the backend. Close() } diff --git a/politeiad/backend/gitbe/gitbe.go b/politeiad/backend/gitbe/gitbe.go index 030644f99..804ce3f66 100644 --- a/politeiad/backend/gitbe/gitbe.go +++ b/politeiad/backend/gitbe/gitbe.go @@ -1791,13 +1791,6 @@ func (g *gitBackEnd) UpdateUnvettedRecord(token []byte, mdAppend []backend.Metad false) } -// UpdateUnvettedMetadata is not implemented. -// -// This function satisfies the Backend interface. -func (g *gitBackEnd) UpdateUnvettedMetadata(token []byte, mdAppend []backend.MetadataStream, mdOverwrite []backend.MetadataStream) error { - return fmt.Errorf("not implemented") -} - // updateVettedMetadata updates metadata in the unvetted repo and pushes it // upstream followed by a rebase. Record is not updated. // This function must be called with the lock held. @@ -1869,7 +1862,7 @@ func (g *gitBackEnd) _updateVettedMetadata(token []byte, mdAppend []backend.Meta return err } if md.Status == backend.MDStatusArchived { - return backend.ErrRecordLocked + return backend.ErrRecordArchived } log.Debugf("updating vetted metadata %x", token) @@ -2003,7 +1996,7 @@ func (g *gitBackEnd) _updateVettedMetadataMulti(um []updateMetadata, idTmp strin return err } if md.Status == backend.MDStatusArchived { - return backend.ErrRecordLocked + return backend.ErrRecordArchived } } @@ -2411,12 +2404,9 @@ func (g *gitBackEnd) vettedMetadataStreamExists(token []byte, mdstreamID int) bo // unvetted/token directory. // // GetUnvetted satisfies the backend interface. -func (g *gitBackEnd) GetUnvetted(token []byte, version string) (*backend.Record, error) { +func (g *gitBackEnd) GetUnvetted(token []byte) (*backend.Record, error) { log.Tracef("GetUnvetted %x", token) - // The version argument is not used because gitbe does not version - // unvetted records. - return g.getRecordLock(token, "", g.unvetted, true) } @@ -2597,7 +2587,7 @@ func (g *gitBackEnd) _setVettedStatus(token []byte, status backend.MDStatusT, md return nil, err } if md.Status == backend.MDStatusArchived { - return nil, backend.ErrRecordLocked + return nil, backend.ErrRecordArchived } // Load record diff --git a/politeiad/backend/gitbe/gitbe_test.go b/politeiad/backend/gitbe/gitbe_test.go index a602f89d1..21c10a085 100644 --- a/politeiad/backend/gitbe/gitbe_test.go +++ b/politeiad/backend/gitbe/gitbe_test.go @@ -153,7 +153,7 @@ func TestAnchorWithCommits(t *testing.T) { if err != nil { t.Fatal(err) } - pru, err := g.GetUnvetted(token, "") + pru, err := g.GetUnvetted(token) if err != nil { t.Fatalf("%v", err) } diff --git a/politeiad/backendv2/tstorebe/store/store.go b/politeiad/backendv2/tstorebe/store/store.go index cf2cfcded..3a5b51ad0 100644 --- a/politeiad/backendv2/tstorebe/store/store.go +++ b/politeiad/backendv2/tstorebe/store/store.go @@ -28,8 +28,7 @@ type DataDescriptor struct { ExtraData string `json:"extradata,omitempty"` // Value to be freely used } -// BlobEntry is the structure used to store data in the Blob key-value store. -// All data in the Blob key-value store will be encoded as a BlobEntry. +// BlobEntry is the structure used to store data in the key-value store. type BlobEntry struct { Digest string `json:"digest"` // SHA256 digest of data, hex encoded DataHint string `json:"datahint"` // Hint that describes data, base64 encoded diff --git a/politeiad/client/pdv1.go b/politeiad/client/pdv1.go index dad364abd..dd8082d0d 100644 --- a/politeiad/client/pdv1.go +++ b/politeiad/client/pdv1.go @@ -48,11 +48,11 @@ func (c *Client) NewRecord(ctx context.Context, metadata []pdv1.MetadataStream, return &nrr.CensorshipRecord, nil } -func (c *Client) updateRecord(ctx context.Context, route, token string, mdAppend, mdOverwrite []pdv1.MetadataStream, filesAdd []pdv1.File, filesDel []string) (*pdv1.Record, error) { +func (c *Client) updateRecord(ctx context.Context, route, token string, mdAppend, mdOverwrite []pdv1.MetadataStream, filesAdd []pdv1.File, filesDel []string) error { // Setup request challenge, err := util.Random(pdv1.ChallengeSize) if err != nil { - return nil, err + return err } ur := pdv1.UpdateRecord{ Token: token, @@ -65,72 +65,36 @@ func (c *Client) updateRecord(ctx context.Context, route, token string, mdAppend // Send request resBody, err := c.makeReq(ctx, http.MethodPost, "", route, ur) if err != nil { - return nil, err + return err } // Decode reply var urr pdv1.UpdateRecordReply err = json.Unmarshal(resBody, &urr) if err != nil { - return nil, err + return err } err = util.VerifyChallenge(c.pid, challenge, urr.Response) if err != nil { - return nil, err + return err } - return &urr.Record, nil + return nil } // UpdateUnvetted sends a UpdateRecord request to the unvetted politeiad v1 // API. -func (c *Client) UpdateUnvetted(ctx context.Context, token string, mdAppend, mdOverwrite []pdv1.MetadataStream, filesAdd []pdv1.File, filesDel []string) (*pdv1.Record, error) { +func (c *Client) UpdateUnvetted(ctx context.Context, token string, mdAppend, mdOverwrite []pdv1.MetadataStream, filesAdd []pdv1.File, filesDel []string) error { return c.updateRecord(ctx, pdv1.UpdateUnvettedRoute, token, mdAppend, mdOverwrite, filesAdd, filesDel) } // UpdateVetted sends a UpdateRecord request to the vetted politeiad v1 API. -func (c *Client) UpdateVetted(ctx context.Context, token string, mdAppend, mdOverwrite []pdv1.MetadataStream, filesAdd []pdv1.File, filesDel []string) (*pdv1.Record, error) { +func (c *Client) UpdateVetted(ctx context.Context, token string, mdAppend, mdOverwrite []pdv1.MetadataStream, filesAdd []pdv1.File, filesDel []string) error { return c.updateRecord(ctx, pdv1.UpdateVettedRoute, token, mdAppend, mdOverwrite, filesAdd, filesDel) } -// UpdateUnvettedMetadata sends a UpdateVettedMetadata request to the politeiad -// v1 API. -func (c *Client) UpdateUnvettedMetadata(ctx context.Context, token string, mdAppend, mdOverwrite []pdv1.MetadataStream) error { - // Setup request - challenge, err := util.Random(pdv1.ChallengeSize) - if err != nil { - return err - } - uum := pdv1.UpdateUnvettedMetadata{ - Challenge: hex.EncodeToString(challenge), - Token: token, - MDAppend: mdAppend, - MDOverwrite: mdOverwrite, - } - - // Send request - resBody, err := c.makeReq(ctx, http.MethodPost, "", - pdv1.UpdateUnvettedMetadataRoute, uum) - if err != nil { - return nil - } - - // Decode reply - var uumr pdv1.UpdateUnvettedMetadataReply - err = json.Unmarshal(resBody, &uumr) - if err != nil { - return err - } - err = util.VerifyChallenge(c.pid, challenge, uumr.Response) - if err != nil { - return err - } - - return nil -} - // UpdateVettedMetadata sends a UpdateVettedMetadata request to the politeiad // v1 API. func (c *Client) UpdateVettedMetadata(ctx context.Context, token string, mdAppend, mdOverwrite []pdv1.MetadataStream) error { @@ -169,11 +133,11 @@ func (c *Client) UpdateVettedMetadata(ctx context.Context, token string, mdAppen // SetUnvettedStatus sends a SetUnvettedStatus request to the politeiad v1 // API. -func (c *Client) SetUnvettedStatus(ctx context.Context, token string, status pdv1.RecordStatusT, mdAppend, mdOverwrite []pdv1.MetadataStream) (*pdv1.Record, error) { +func (c *Client) SetUnvettedStatus(ctx context.Context, token string, status pdv1.RecordStatusT, mdAppend, mdOverwrite []pdv1.MetadataStream) error { // Setup request challenge, err := util.Random(pdv1.ChallengeSize) if err != nil { - return nil, err + return err } sus := pdv1.SetUnvettedStatus{ Challenge: hex.EncodeToString(challenge), @@ -187,29 +151,29 @@ func (c *Client) SetUnvettedStatus(ctx context.Context, token string, status pdv resBody, err := c.makeReq(ctx, http.MethodPost, "", pdv1.SetUnvettedStatusRoute, sus) if err != nil { - return nil, err + return err } // Decode reply var susr pdv1.SetUnvettedStatusReply err = json.Unmarshal(resBody, &susr) if err != nil { - return nil, err + return err } err = util.VerifyChallenge(c.pid, challenge, susr.Response) if err != nil { - return nil, err + return err } - return &susr.Record, nil + return nil } // SetVettedStatus sends a SetVettedStatus request to the politeiad v1 API. -func (c *Client) SetVettedStatus(ctx context.Context, token string, status pdv1.RecordStatusT, mdAppend, mdOverwrite []pdv1.MetadataStream) (*pdv1.Record, error) { +func (c *Client) SetVettedStatus(ctx context.Context, token string, status pdv1.RecordStatusT, mdAppend, mdOverwrite []pdv1.MetadataStream) error { // Setup request challenge, err := util.Random(pdv1.ChallengeSize) if err != nil { - return nil, err + return err } svs := pdv1.SetVettedStatus{ Challenge: hex.EncodeToString(challenge), @@ -223,25 +187,25 @@ func (c *Client) SetVettedStatus(ctx context.Context, token string, status pdv1. resBody, err := c.makeReq(ctx, http.MethodPost, "", pdv1.SetVettedStatusRoute, svs) if err != nil { - return nil, err + return err } // Decode reply var svsr pdv1.SetVettedStatusReply err = json.Unmarshal(resBody, &svsr) if err != nil { - return nil, err + return err } err = util.VerifyChallenge(c.pid, challenge, svsr.Response) if err != nil { - return nil, err + return err } - return &svsr.Record, nil + return nil } // GetUnvetted sends a GetUnvetted request to the politeiad v1 API. -func (c *Client) GetUnvetted(ctx context.Context, token, version string) (*pdv1.Record, error) { +func (c *Client) GetUnvetted(ctx context.Context, token string) (*pdv1.Record, error) { // Setup request challenge, err := util.Random(pdv1.ChallengeSize) if err != nil { @@ -250,7 +214,6 @@ func (c *Client) GetUnvetted(ctx context.Context, token, version string) (*pdv1. gu := pdv1.GetUnvetted{ Challenge: hex.EncodeToString(challenge), Token: token, - Version: version, } // Send request diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index a639522fa..c6b7bef80 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -190,8 +190,6 @@ func (p *politeia) setupBackendGit(anp *chaincfg.Params) error { p.setVettedStatus, permissionAuth) p.addRoute(http.MethodPost, v1.UpdateVettedMetadataRoute, p.updateVettedMetadata, permissionAuth) - p.addRoute(http.MethodPost, v1.UpdateUnvettedMetadataRoute, - p.updateUnvettedMetadata, permissionAuth) // Set plugin routes. Requires auth. p.addRoute(http.MethodPost, v1.PluginCommandRoute, p.pluginCommand, diff --git a/politeiad/v1.go b/politeiad/v1.go index b18d5f421..1b2029f6b 100644 --- a/politeiad/v1.go +++ b/politeiad/v1.go @@ -48,11 +48,13 @@ func (p *politeia) newRecord(w http.ResponseWriter, r *http.Request) { challenge, err := hex.DecodeString(t.Challenge) if err != nil || len(challenge) != v1.ChallengeSize { - log.Infof("%v newRecord: invalid challenge", remoteAddr(r)) + log.Errorf("%v newRecord: invalid challenge", remoteAddr(r)) p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) return } + log.Infof("New record submitted %v", remoteAddr(r)) + md := convertFrontendMetadataStream(t.Metadata) files := convertFrontendFiles(t.Files) rm, err := p.backend.New(md, files) @@ -60,7 +62,7 @@ func (p *politeia) newRecord(w http.ResponseWriter, r *http.Request) { // Check for content error. var contentErr backend.ContentVerificationError if errors.As(err, &contentErr) { - log.Infof("%v New record content error: %v", + log.Errorf("%v New record content error: %v", remoteAddr(r), contentErr) p.respondWithUserError(w, contentErr.ErrorCode, contentErr.ErrorContext) @@ -110,7 +112,7 @@ func (p *politeia) updateRecord(w http.ResponseWriter, r *http.Request, vetted b challenge, err := hex.DecodeString(t.Challenge) if err != nil || len(challenge) != v1.ChallengeSize { - log.Infof("%v update %v record: invalid challenge", + log.Errorf("%v update %v record: invalid challenge", remoteAddr(r), cmd) p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) return @@ -119,7 +121,8 @@ func (p *politeia) updateRecord(w http.ResponseWriter, r *http.Request, vetted b // Validate token token, err := util.ConvertStringToken(t.Token) if err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, + nil) return } @@ -140,20 +143,21 @@ func (p *politeia) updateRecord(w http.ResponseWriter, r *http.Request, vetted b } if err != nil { if errors.Is(err, backend.ErrRecordFound) { - log.Infof("%v update %v record found: %x", + log.Errorf("%v update %v record found: %x", remoteAddr(r), cmd, token) p.respondWithUserError(w, v1.ErrorStatusRecordFound, nil) return } if errors.Is(err, backend.ErrRecordNotFound) { - log.Infof("%v update %v record not found: %x", + log.Errorf("%v update %v record not found: %x", remoteAddr(r), cmd, token) - p.respondWithUserError(w, v1.ErrorStatusRecordNotFound, nil) + p.respondWithUserError(w, v1.ErrorStatusRecordFound, + nil) return } if errors.Is(err, backend.ErrNoChanges) { - log.Infof("%v update %v record no changes: %x", + log.Errorf("%v update %v record no changes: %x", remoteAddr(r), cmd, token) p.respondWithUserError(w, v1.ErrorStatusNoChanges, nil) return @@ -161,12 +165,13 @@ func (p *politeia) updateRecord(w http.ResponseWriter, r *http.Request, vetted b // Check for content error. var contentErr backend.ContentVerificationError if errors.As(err, &contentErr) { - log.Infof("%v update %v record content error: %v", + log.Errorf("%v update %v record content error: %v", remoteAddr(r), cmd, contentErr) p.respondWithUserError(w, contentErr.ErrorCode, contentErr.ErrorContext) return } + // Generic internal error. errorCode := time.Now().Unix() log.Errorf("%v Update %v record error code %v: %v", @@ -179,7 +184,6 @@ func (p *politeia) updateRecord(w http.ResponseWriter, r *http.Request, vetted b response := p.identity.SignMessage(challenge) reply := v1.UpdateRecordReply{ Response: hex.EncodeToString(response[:]), - Record: p.convertBackendRecord(*record), } log.Infof("Update %v record %v: token %v", cmd, remoteAddr(r), @@ -196,6 +200,37 @@ func (p *politeia) updateVetted(w http.ResponseWriter, r *http.Request) { p.updateRecord(w, r, true) } +func (p *politeia) updateReadme(w http.ResponseWriter, r *http.Request) { + var t v1.UpdateReadme + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&t); err != nil { + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) + return + } + + challenge, err := hex.DecodeString(t.Challenge) + if err != nil || len(challenge) != v1.ChallengeSize { + p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) + return + } + + response := p.identity.SignMessage(challenge) + + reply := v1.UpdateReadmeReply{ + Response: hex.EncodeToString(response[:]), + } + + err = p.backend.UpdateReadme(t.Content) + if err != nil { + errorCode := time.Now().Unix() + log.Errorf("Error updating readme: %v", err) + p.respondWithServerError(w, errorCode) + return + } + + util.RespondWithJSON(w, http.StatusOK, reply) +} + func (p *politeia) getUnvetted(w http.ResponseWriter, r *http.Request) { var t v1.GetUnvetted decoder := json.NewDecoder(r.Body) @@ -218,37 +253,25 @@ func (p *politeia) getUnvetted(w http.ResponseWriter, r *http.Request) { // Validate token token, err := util.ConvertStringToken(t.Token) if err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) return } // Ask backend about the censorship token. - bpr, err := p.backend.GetUnvetted(token, t.Version) - switch { - case errors.Is(err, backend.ErrRecordNotFound): - // Record not found - log.Infof("Get unvetted record %v: token %v not found", + bpr, err := p.backend.GetUnvetted(token) + if errors.Is(err, backend.ErrRecordNotFound) { + reply.Record.Status = v1.RecordStatusNotFound + log.Errorf("Get unvetted record %v: token %v not found", remoteAddr(r), t.Token) - p.respondWithUserError(w, v1.ErrorStatusRecordNotFound, nil) - return - - case err != nil: - // Generic internal error + } else if err != nil { + // Generic internal error. errorCode := time.Now().Unix() log.Errorf("%v Get unvetted record error code %v: %v", remoteAddr(r), errorCode, err) + p.respondWithServerError(w, errorCode) return - - case bpr.RecordMetadata.Status == backend.MDStatusCensored: - // Record has been censored. The default case will verify the - // record before sending it off. This will fail for censored - // records since the files will not exist, they've been deleted, - // so skip the verification step. - reply.Record = p.convertBackendRecord(*bpr) - log.Infof("Get unvetted record %v: token %v", remoteAddr(r), t.Token) - - default: + } else { reply.Record = p.convertBackendRecord(*bpr) // Double check record bits before sending them off @@ -260,11 +283,13 @@ func (p *politeia) getUnvetted(w http.ResponseWriter, r *http.Request) { log.Errorf("%v Get unvetted record CORRUPTION "+ "error code %v: %v", remoteAddr(r), errorCode, err) + p.respondWithServerError(w, errorCode) return } - log.Infof("Get unvetted record %v: token %v", remoteAddr(r), t.Token) + log.Infof("Get unvetted record %v: token %v", remoteAddr(r), + t.Token) } util.RespondWithJSON(w, http.StatusOK, reply) @@ -292,37 +317,25 @@ func (p *politeia) getVetted(w http.ResponseWriter, r *http.Request) { // Validate token token, err := util.ConvertStringToken(t.Token) if err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) return } // Ask backend about the censorship token. bpr, err := p.backend.GetVetted(token, t.Version) - switch { - case errors.Is(err, backend.ErrRecordNotFound): - // Record not found - log.Infof("Get vetted record %v: token %v not found", + if errors.Is(err, backend.ErrRecordNotFound) { + reply.Record.Status = v1.RecordStatusNotFound + log.Errorf("Get vetted record %v: token %v not found", remoteAddr(r), t.Token) - p.respondWithUserError(w, v1.ErrorStatusRecordNotFound, nil) - return - - case err != nil: - // Generic internal error + } else if err != nil { + // Generic internal error. errorCode := time.Now().Unix() log.Errorf("%v Get vetted record error code %v: %v", remoteAddr(r), errorCode, err) + p.respondWithServerError(w, errorCode) return - - case bpr.RecordMetadata.Status == backend.MDStatusCensored: - // Record has been censored. The default case will verify the - // record before sending it off. This will fail for censored - // records since the files will not exist, they've been deleted, - // so skip the verification step. - reply.Record = p.convertBackendRecord(*bpr) - log.Infof("Get vetted record %v: token %v", remoteAddr(r), t.Token) - - default: + } else { reply.Record = p.convertBackendRecord(*bpr) // Double check record bits before sending them off @@ -334,11 +347,12 @@ func (p *politeia) getVetted(w http.ResponseWriter, r *http.Request) { log.Errorf("%v Get vetted record CORRUPTION "+ "error code %v: %v", remoteAddr(r), errorCode, err) + p.respondWithServerError(w, errorCode) return } - - log.Infof("Get vetted record %v: token %v", remoteAddr(r), t.Token) + log.Infof("Get vetted record %v: token %v", remoteAddr(r), + t.Token) } util.RespondWithJSON(w, http.StatusOK, reply) @@ -411,7 +425,7 @@ func (p *politeia) setVettedStatus(w http.ResponseWriter, r *http.Request) { // Validate token token, err := util.ConvertStringToken(t.Token) if err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) return } @@ -423,14 +437,15 @@ func (p *politeia) setVettedStatus(w http.ResponseWriter, r *http.Request) { if err != nil { // Check for specific errors if errors.Is(err, backend.ErrRecordNotFound) { - log.Infof("%v updateStatus record not "+ + log.Errorf("%v updateStatus record not "+ "found: %x", remoteAddr(r), token) - p.respondWithUserError(w, v1.ErrorStatusRecordNotFound, nil) + p.respondWithUserError(w, v1.ErrorStatusRecordFound, + nil) return } var serr backend.StateTransitionError if errors.As(err, &serr) { - log.Infof("%v %v %v", remoteAddr(r), t.Token, err) + log.Errorf("%v %v %v", remoteAddr(r), t.Token, err) p.respondWithUserError(w, v1.ErrorStatusInvalidRecordStatusTransition, nil) return } @@ -446,7 +461,6 @@ func (p *politeia) setVettedStatus(w http.ResponseWriter, r *http.Request) { // Prepare reply. reply := v1.SetVettedStatusReply{ Response: hex.EncodeToString(response[:]), - Record: p.convertBackendRecord(*record), } s := convertBackendStatus(record.RecordMetadata.Status) @@ -474,7 +488,7 @@ func (p *politeia) setUnvettedStatus(w http.ResponseWriter, r *http.Request) { // Validate token token, err := util.ConvertStringToken(t.Token) if err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) return } @@ -486,14 +500,15 @@ func (p *politeia) setUnvettedStatus(w http.ResponseWriter, r *http.Request) { if err != nil { // Check for specific errors if errors.Is(err, backend.ErrRecordNotFound) { - log.Infof("%v updateUnvettedStatus record not "+ + log.Errorf("%v updateUnvettedStatus record not "+ "found: %x", remoteAddr(r), token) - p.respondWithUserError(w, v1.ErrorStatusRecordNotFound, nil) + p.respondWithUserError(w, v1.ErrorStatusRecordFound, + nil) return } var serr backend.StateTransitionError if errors.As(err, &serr) { - log.Infof("%v %v %v", remoteAddr(r), t.Token, err) + log.Errorf("%v %v %v", remoteAddr(r), t.Token, err) p.respondWithUserError(w, v1.ErrorStatusInvalidRecordStatusTransition, nil) return } @@ -509,7 +524,6 @@ func (p *politeia) setUnvettedStatus(w http.ResponseWriter, r *http.Request) { // Prepare reply. reply := v1.SetUnvettedStatusReply{ Response: hex.EncodeToString(response[:]), - Record: p.convertBackendRecord(*record), } s := convertBackendStatus(record.RecordMetadata.Status) @@ -537,7 +551,7 @@ func (p *politeia) updateVettedMetadata(w http.ResponseWriter, r *http.Request) // Validate token token, err := util.ConvertStringToken(t.Token) if err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) + p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) return } @@ -549,7 +563,7 @@ func (p *politeia) updateVettedMetadata(w http.ResponseWriter, r *http.Request) convertFrontendMetadataStream(t.MDOverwrite)) if err != nil { if errors.Is(err, backend.ErrNoChanges) { - log.Infof("%v update vetted metadata no changes: %x", + log.Errorf("%v update vetted metadata no changes: %x", remoteAddr(r), token) p.respondWithUserError(w, v1.ErrorStatusNoChanges, nil) return @@ -557,12 +571,13 @@ func (p *politeia) updateVettedMetadata(w http.ResponseWriter, r *http.Request) // Check for content error. var contentErr backend.ContentVerificationError if errors.As(err, &contentErr) { - log.Infof("%v update vetted metadata content error: %v", + log.Errorf("%v update vetted metadata content error: %v", remoteAddr(r), contentErr) p.respondWithUserError(w, contentErr.ErrorCode, contentErr.ErrorContext) return } + // Generic internal error. errorCode := time.Now().Unix() log.Errorf("%v Update vetted metadata error code %v: %v", @@ -581,68 +596,6 @@ func (p *politeia) updateVettedMetadata(w http.ResponseWriter, r *http.Request) util.RespondWithJSON(w, http.StatusOK, reply) } -func (p *politeia) updateUnvettedMetadata(w http.ResponseWriter, r *http.Request) { - var t v1.UpdateUnvettedMetadata - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&t); err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidRequestPayload, nil) - return - } - - challenge, err := hex.DecodeString(t.Challenge) - if err != nil || len(challenge) != v1.ChallengeSize { - p.respondWithUserError(w, v1.ErrorStatusInvalidChallenge, nil) - return - } - - token, err := util.ConvertStringToken(t.Token) - if err != nil { - p.respondWithUserError(w, v1.ErrorStatusInvalidToken, nil) - return - } - - log.Infof("Update unvetted metadata submitted %v: %x", remoteAddr(r), - token) - - err = p.backend.UpdateUnvettedMetadata(token, - convertFrontendMetadataStream(t.MDAppend), - convertFrontendMetadataStream(t.MDOverwrite)) - if err != nil { - // Reply with error if there were no changes - if errors.Is(err, backend.ErrNoChanges) { - log.Infof("%v update unvetted metadata no changes: %x", - remoteAddr(r), token) - p.respondWithUserError(w, v1.ErrorStatusNoChanges, nil) - return - } - // Check for content error. - var cverr backend.ContentVerificationError - if errors.As(err, &cverr) { - log.Infof("%v update unvetted metadata content error: %v", - remoteAddr(r), cverr) - p.respondWithUserError(w, cverr.ErrorCode, - cverr.ErrorContext) - return - } - // Generic internal error. - errorCode := time.Now().Unix() - log.Errorf("%v update unvetted metadata error code %v: %v", - remoteAddr(r), errorCode, err) - p.respondWithServerError(w, errorCode) - return - } - - // Prepare reply - response := p.identity.SignMessage(challenge) - reply := v1.UpdateUnvettedMetadataReply{ - Response: hex.EncodeToString(response[:]), - } - - log.Infof("Update unvetted metadata %v: token %x", remoteAddr(r), token) - - util.RespondWithJSON(w, http.StatusOK, reply) -} - func (p *politeia) pluginInventory(w http.ResponseWriter, r *http.Request) { var pi v1.PluginInventory decoder := json.NewDecoder(r.Body) @@ -659,19 +612,21 @@ func (p *politeia) pluginInventory(w http.ResponseWriter, r *http.Request) { } response := p.identity.SignMessage(challenge) - // Get plugins + reply := v1.PluginInventoryReply{ + Response: hex.EncodeToString(response[:]), + } + plugins, err := p.backend.GetPlugins() if err != nil { + // Generic internal error. errorCode := time.Now().Unix() - log.Errorf("%v get plugins: %v ", remoteAddr(r), err) + log.Errorf("%v Get plugins error code %v: %v", + remoteAddr(r), errorCode, err) p.respondWithServerError(w, errorCode) return } - - // Prepare reply - reply := v1.PluginInventoryReply{ - Plugins: convertBackendPlugins(plugins), - Response: hex.EncodeToString(response[:]), + for _, v := range plugins { + reply.Plugins = append(reply.Plugins, convertBackendPlugin(v)) } util.RespondWithJSON(w, http.StatusOK, reply) @@ -703,7 +658,6 @@ func (p *politeia) pluginCommand(w http.ResponseWriter, r *http.Request) { return } - // Prepare reply response := p.identity.SignMessage(challenge) reply := v1.PluginCommandReply{ Response: hex.EncodeToString(response[:]), @@ -723,20 +677,15 @@ func convertBackendPluginSetting(bpi backend.PluginSetting) v1.PluginSetting { } } -func convertBackendPlugins(bplugins []backend.Plugin) []v1.Plugin { - plugins := make([]v1.Plugin, 0, len(bplugins)) - for _, v := range bplugins { - p := v1.Plugin{ - ID: v.ID, - Version: v.Version, - Settings: make([]v1.PluginSetting, 0, len(v.Settings)), - } - for _, v := range v.Settings { - p.Settings = append(p.Settings, convertBackendPluginSetting(v)) - } - plugins = append(plugins, p) +func convertBackendPlugin(bpi backend.Plugin) v1.Plugin { + p := v1.Plugin{ + ID: bpi.ID, + } + for _, v := range bpi.Settings { + p.Settings = append(p.Settings, convertBackendPluginSetting(v)) } - return plugins + + return p } // convertBackendMetadataStream converts a backend metadata stream to an API diff --git a/politeiawww/www.go b/politeiawww/www.go index 09d435c9c..b09a38a1a 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -79,8 +79,6 @@ func convertWWWErrorStatusFromPD(e pd.ErrorStatusT) www.ErrorStatusT { return www.ErrorStatusUnsupportedMIMEType case pd.ErrorStatusInvalidRecordStatusTransition: return www.ErrorStatusInvalidPropStatusTransition - case pd.ErrorStatusRecordNotFound: - return www.ErrorStatusProposalNotFound } return www.ErrorStatusInvalid } From fb15325e6436db8ae5c270dafc452299ae259f27 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 16 Mar 2021 13:14:30 -0500 Subject: [PATCH 406/449] politeiad: Fix linter errors. --- politeiad/backendv2/backendv2.go | 2 +- politeiad/v2.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/politeiad/backendv2/backendv2.go b/politeiad/backendv2/backendv2.go index 5d8c95424..4caa771ef 100644 --- a/politeiad/backendv2/backendv2.go +++ b/politeiad/backendv2/backendv2.go @@ -260,7 +260,7 @@ type Plugin struct { Identity *identity.FullIdentity } -// PluginError represents an error that occured during plugin execution that +// PluginError represents an error that occurred during plugin execution that // was caused by the user. type PluginError struct { PluginID string diff --git a/politeiad/v2.go b/politeiad/v2.go index 979a8dd45..5e7d48901 100644 --- a/politeiad/v2.go +++ b/politeiad/v2.go @@ -840,7 +840,6 @@ func respondWithErrorV2(w http.ResponseWriter, r *http.Request, format string, e v2.ServerErrorReply{ ErrorCode: t, }) - return } func convertErrorToV2(e error) v2.ErrorCodeT { From dabffd4ac9e5c66a6f2d1be588d93fa14ef4fc9e Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 16 Mar 2021 13:35:42 -0500 Subject: [PATCH 407/449] multi: Verify all API omitempties. --- .../tstorebe/plugins/ticketvote/cmds.go | 2 +- politeiad/client/ticketvote.go | 4 +-- politeiad/plugins/dcrdata/dcrdata.go | 10 ++++---- politeiad/plugins/ticketvote/ticketvote.go | 25 +++++++++---------- politeiawww/ticketvote/process.go | 3 +-- 5 files changed, 21 insertions(+), 23 deletions(-) diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go index e92e517a1..c52b3af0a 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go @@ -2615,7 +2615,7 @@ func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload str } var ( - auths []ticketvote.Timestamp + auths = make([]ticketvote.Timestamp, 0, 32) details *ticketvote.Timestamp pageSize = ticketvote.VoteTimestampsPageSize diff --git a/politeiad/client/ticketvote.go b/politeiad/client/ticketvote.go index 1d575d7ce..82d267dd0 100644 --- a/politeiad/client/ticketvote.go +++ b/politeiad/client/ticketvote.go @@ -337,7 +337,7 @@ func (c *Client) TicketVoteInventory(ctx context.Context, i ticketvote.Inventory // TicketVoteTimestamps sends the ticketvote plugin Timestamps command to the // politeiad v2 API. -func (c *Client) TicketVoteTimestamps(ctx context.Context, t ticketvote.Timestamps) (*ticketvote.TimestampsReply, error) { +func (c *Client) TicketVoteTimestamps(ctx context.Context, token string, t ticketvote.Timestamps) (*ticketvote.TimestampsReply, error) { // Setup request b, err := json.Marshal(t) if err != nil { @@ -347,7 +347,7 @@ func (c *Client) TicketVoteTimestamps(ctx context.Context, t ticketvote.Timestam { ID: ticketvote.PluginID, Command: ticketvote.CmdTimestamps, - Token: t.Token, + Token: token, Payload: string(b), }, } diff --git a/politeiad/plugins/dcrdata/dcrdata.go b/politeiad/plugins/dcrdata/dcrdata.go index 944c0e227..765d0f7b9 100644 --- a/politeiad/plugins/dcrdata/dcrdata.go +++ b/politeiad/plugins/dcrdata/dcrdata.go @@ -85,9 +85,9 @@ type BlockDataBasic struct { Time int64 `json:"time"` // UNIX timestamp NumTx uint32 `json:"txlength"` MiningFee *int64 `json:"fees,omitempty"` - TotalSent *int64 `json:"total_sent,omitempty"` + TotalSent *int64 `json:"totalsent,omitempty"` // TicketPoolInfo may be nil for side chain blocks. - PoolInfo *TicketPoolInfo `json:"ticket_pool,omitempty"` + PoolInfo *TicketPoolInfo `json:"ticketpool,omitempty"` } // BlockDetails fetched the block details for the provided block height. @@ -132,7 +132,7 @@ type Vin struct { AmountIn float64 `json:"amountin"` BlockHeight uint32 `json:"blockheight"` BlockIndex uint32 `json:"blockindex"` - ScriptSig *ScriptSig `json:"scriptSig"` + ScriptSig *ScriptSig `json:"scriptsig"` } // ScriptPubKey is the script public key data. @@ -148,7 +148,7 @@ type ScriptPubKey struct { // TxInputID specifies a transaction input as hash:vin_index. type TxInputID struct { Hash string `json:"hash"` - Index uint32 `json:"vin_index"` + Index uint32 `json:"index"` } // Vout defines a transaction output. @@ -156,7 +156,7 @@ type Vout struct { Value float64 `json:"value"` N uint32 `json:"n"` Version uint16 `json:"version"` - ScriptPubKeyDecoded ScriptPubKey `json:"scriptPubKey"` + ScriptPubKeyDecoded ScriptPubKey `json:"scriptpubkeydecoded"` Spend *TxInputID `json:"spend,omitempty"` } diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index f4e7b30c4..fd60159f2 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -504,7 +504,7 @@ type Details struct{} // DetailsReply is the reply to the Details command. type DetailsReply struct { Auths []AuthDetails `json:"auths"` - Vote *VoteDetails `json:"vote"` + Vote *VoteDetails `json:"vote,omitempty"` } // Results requests the results of a vote. @@ -583,16 +583,16 @@ type Summary struct{} // SummaryReply is the reply to the Summary command. type SummaryReply struct { - Type VoteT `json:"type"` Status VoteStatusT `json:"status"` - Duration uint32 `json:"duration"` - StartBlockHeight uint32 `json:"startblockheight"` - StartBlockHash string `json:"startblockhash"` - EndBlockHeight uint32 `json:"endblockheight"` - EligibleTickets uint32 `json:"eligibletickets"` - QuorumPercentage uint32 `json:"quorumpercentage"` - PassPercentage uint32 `json:"passpercentage"` - Results []VoteOptionResult `json:"results"` + Type VoteT `json:"type,omitempty"` + Duration uint32 `json:"duration,omitempty"` + StartBlockHeight uint32 `json:"startblockheight,omitempty"` + StartBlockHash string `json:"startblockhash,omitempty"` + EndBlockHeight uint32 `json:"endblockheight,omitempty"` + EligibleTickets uint32 `json:"eligibletickets,omitempty"` + QuorumPercentage uint32 `json:"quorumpercentage,omitempty"` + PassPercentage uint32 `json:"passpercentage,omitempty"` + Results []VoteOptionResult `json:"results,omitempty"` // BestBlock is the best block value that was used to prepare this // summary. @@ -695,13 +695,12 @@ const ( // details timestamps will be returned. If a votes page number is provided then // the specified page of votes will be returned. type Timestamps struct { - Token string `json:"token"` VotesPage uint32 `json:"votespage,omitempty"` } // TimestampsReply is the reply to the Timestamps command. type TimestampsReply struct { - Auths []Timestamp `json:"auths,omitempty"` + Auths []Timestamp `json:"auths"` Details *Timestamp `json:"details,omitempty"` - Votes []Timestamp `json:"votes,omitempty"` + Votes []Timestamp `json:"votes"` } diff --git a/politeiawww/ticketvote/process.go b/politeiawww/ticketvote/process.go index aded5a937..572527c9d 100644 --- a/politeiawww/ticketvote/process.go +++ b/politeiawww/ticketvote/process.go @@ -218,10 +218,9 @@ func (t *TicketVote) processTimestamps(ctx context.Context, ts v1.Timestamps) (* // Send plugin command tt := ticketvote.Timestamps{ - Token: ts.Token, VotesPage: ts.VotesPage, } - tsr, err := t.politeiad.TicketVoteTimestamps(ctx, tt) + tsr, err := t.politeiad.TicketVoteTimestamps(ctx, ts.Token, tt) if err != nil { return nil, err } From 47522944a525edb51231c0e80e57512d86fb1818 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 16 Mar 2021 17:19:28 -0500 Subject: [PATCH 408/449] politeiawww: Fix records API. --- politeiawww/api/records/v1/v1.go | 32 +++++++++++--------------------- politeiawww/records/process.go | 11 +++++++++-- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go index b8d9e0ad6..bd7ad6fd3 100644 --- a/politeiawww/api/records/v1/v1.go +++ b/politeiawww/api/records/v1/v1.go @@ -313,31 +313,21 @@ const ( RecordsPageSize = 5 ) -// RecordRequest is used to requests a record. It gives the client granular -// control over what is returned. The only required field is the token. All -// other fields are optional. +// RecordRequest is used to requests select content from a record. The latest +// version of the record is returned. By default, all record files will be +// stripped from the record before being returned. // -// Version is used to request a specific version of a record. If no version is -// provided then the most recent version of the record will be returned. -// -// Filenames can be used to request specific files. If filenames is not empty -// then the specified files will be the only files returned. -// -// OmitAllFiles can be used to retrieve a record without any of the record -// files. This supersedes the filenames argument. +// Filenames can be used to request specific files. If filenames is empty than +// no record files will be returned. type RecordRequest struct { - Token string `json:"token"` - Version uint32 `json:"version,omitempty"` - Filenames []string `json:"filenames,omitempty"` - OmitAllFiles bool `json:"omitallfiles,omitempty"` + Token string `json:"token"` + Filenames []string `json:"filenames,omitempty"` } -// Records requests a batch of records. -// -// Only the record metadata is returned. The Details command must be used to -// retrieve the record files or a specific version of the record. Since record -// files are not included in the reply, unvetted records are returned to all -// users. +// Records requests a batch of records. This route should be used when the +// client only requires select content from the record. The Details command +// should be used when the full record content is required. Unvetted record +// files are only returned to admins and the author. type Records struct { Requests []RecordRequest `json:"requests"` } diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index 1153791bb..baa698673 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -664,11 +664,18 @@ func convertStatusToPD(s v1.RecordStatusT) pdv2.RecordStatusT { func convertRequestsToPD(reqs []v1.RecordRequest) []pdv2.RecordRequest { r := make([]pdv2.RecordRequest, 0, len(reqs)) for _, v := range reqs { + // The records API returns the record without any files by + // default. Files are only returned if the filenames are + // provided. This behavior differs from the politeiad API + // behavior, which returns all files by default. + var omitAllFiles bool + if len(v.Filenames) == 0 { + omitAllFiles = true + } r = append(r, pdv2.RecordRequest{ Token: v.Token, - Version: v.Version, Filenames: v.Filenames, - OmitAllFiles: v.OmitAllFiles, + OmitAllFiles: omitAllFiles, }) } return r From d05601d5a1472f7dfcd24639826cdb77015ebbee Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 17 Mar 2021 08:53:40 -0500 Subject: [PATCH 409/449] tstorebe/mysql: Fix review issues. --- .../backendv2/tstorebe/store/mysql/encrypt.go | 158 ++++++++---------- .../backendv2/tstorebe/store/mysql/mysql.go | 48 +++++- .../backendv2/tstorebe/store/mysql/nonce.go | 41 +++++ 3 files changed, 157 insertions(+), 90 deletions(-) diff --git a/politeiad/backendv2/tstorebe/store/mysql/encrypt.go b/politeiad/backendv2/tstorebe/store/mysql/encrypt.go index aaf7dc27b..1c5ed9eb2 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/encrypt.go +++ b/politeiad/backendv2/tstorebe/store/mysql/encrypt.go @@ -9,6 +9,7 @@ import ( "context" "database/sql" "encoding/binary" + "encoding/json" "fmt" "github.com/decred/politeia/util" @@ -16,107 +17,97 @@ import ( "golang.org/x/crypto/argon2" ) -// salt creates a random salt and saves it to the kv store. Subsequent calls to -// this function will return the existing salt. -func (s *mysql) salt(size int) ([]byte, error) { - saltKey := "salt" +const ( + // argon2idKey is the kv store key for the argon2idParams structure + // that is saved on initial key derivation. + argon2idKey = "argon2id" +) + +// argon2idParams is saved to the kv store the first time the key is derived. +type argon2idParams struct { + Time uint32 `json:"time"` + Memory uint32 `json:"memory"` + Threads uint8 `json:"threads"` + KeyLen uint32 `json:"keylen"` + Salt []byte `json:"salt"` +} - // Check if a salt already exists in the database - blobs, err := s.Get([]string{saltKey}) +// argon2idKey derives a 32 byte key from the provided password using the +// Aragon2id key derivation function. A random 16 byte salt is created the +// first time the key is derived. The salt and the other argon2id params are +// saved to the kv store. Subsequent calls to this fuction will pull the +// existing salt and params from the kv store and use them to derive the key. +func (s *mysql) argon2idKey(password string) (*[32]byte, error) { + // Check if a key already exists + blobs, err := s.Get([]string{argon2idKey}) if err != nil { return nil, fmt.Errorf("get: %v", err) } - salt, ok := blobs[saltKey] + var salt []byte + var wasFound bool + b, ok := blobs[argon2idKey] if ok { - // Salt already exists - log.Debugf("Salt found in kv store") - return salt, nil - } + // Key already exists. Use the existing salt. + log.Debugf("Existing salt found") - // Salt doesn't exist yet. Create one and save it. - salt, err = util.Random(size) - if err != nil { - return nil, err - } - kv := map[string][]byte{ - saltKey: salt, - } - err = s.Put(kv, false) - if err != nil { - return nil, fmt.Errorf("put: %v", err) - } + var ap argon2idParams + err = json.Unmarshal(b, &ap) + if err != nil { + return nil, err + } - log.Debugf("Salt created and saved to kv store") + salt = ap.Salt + wasFound = true + } else { + // Key does not exist. Create a random 16 byte salt. + log.Debugf("Salt not found; creating a new one") - return salt, nil -} + salt, err = util.Random(16) + if err != nil { + return nil, err + } + } -// aragon2idKey derives a 32 byte aragon2id key from the provided password. -// The salt is generated the first time the key is derived and saved to the kv -// store. Subsequent calls to this fuction will use the existing salt. -func (s *mysql) argon2idKey(password string) (*[32]byte, error) { + // Derive key var ( pass = []byte(password) - saltLen int = 16 // In bytes time uint32 = 1 memory uint32 = 64 * 1024 // 64 MB threads uint8 = 4 // Number of available CPUs keyLen uint32 = 32 // In bytes ) - salt, err := s.salt(saltLen) - if err != nil { - return nil, fmt.Errorf("salt: %v", err) - } k := argon2.IDKey(pass, salt, time, memory, threads, keyLen) var key [32]byte copy(key[:], k) util.Zero(k) - return &key, nil -} - -// nonce returns a new nonce value. This function guarantees that the returned -// nonce will be unique for every invocation. -// -// This function must be called using a transaction. -func (s *mysql) nonce(ctx context.Context, tx *sql.Tx) (int64, error) { - _, err := tx.ExecContext(ctx, "INSERT INTO nonce () VALUES ();") - if err != nil { - return 0, fmt.Errorf("insert: %v", err) - } - - // Get the nonce value that was just created - rows, err := tx.QueryContext(ctx, "SELECT LAST_INSERT_ID();") - if err != nil { - return 0, fmt.Errorf("query: %v", err) - } - defer rows.Close() - - var nonce int64 - for rows.Next() { - if nonce > 0 { - // There should only ever be one row returned. Something is - // wrong if we've already scanned the nonce and its still - // scanning rows. - return 0, fmt.Errorf("multiple rows returned for nonce") + // Save params to the kv store if this is the first time the key + // was derived. + if !wasFound { + ap := argon2idParams{ + Time: time, + Memory: memory, + Threads: threads, + KeyLen: keyLen, + Salt: salt, } - err = rows.Scan(&nonce) + b, err := json.Marshal(ap) if err != nil { - return 0, fmt.Errorf("scan: %v", err) + return nil, err + } + kv := map[string][]byte{ + argon2idKey: b, + } + err = s.Put(kv, false) + if err != nil { + return nil, fmt.Errorf("put: %v", err) } - } - err = rows.Err() - if err != nil { - return 0, fmt.Errorf("next: %v", err) - } - if nonce == 0 { - return 0, fmt.Errorf("invalid 0 nonce") } - return nonce, nil + return &key, nil } -func (s *mysql) encrypt(ctx context.Context, tx *sql.Tx, data []byte) ([]byte, error) { +func (s *mysql) encrypt(ctx context.Context, tx *sql.Tx, key *[32]byte, data []byte) ([]byte, error) { // Get nonce value nonce, err := s.nonce(ctx, tx) if err != nil { @@ -134,26 +125,23 @@ func (s *mysql) encrypt(ctx context.Context, tx *sql.Tx, data []byte) ([]byte, e } nonceb := n.Current() - // Encrypt blob + // The encryption key is zero'd out on application exit so the read + // lock must be held during concurrent access to prevent the golang + // race detector from complaining. s.RLock() defer s.RUnlock() - return sbox.EncryptN(0, s.key, nonceb, data) + return sbox.EncryptN(0, key, nonceb, data) } -func (s *mysql) decrypt(data []byte) ([]byte, uint32, error) { +func (s *mysql) decrypt(key *[32]byte, data []byte) ([]byte, uint32, error) { + // The encryption key is zero'd out on application exit so the read + // lock must be held during concurrent access to prevent the golang + // race detector from complaining. s.RLock() defer s.RUnlock() - return sbox.Decrypt(s.key, data) -} - -func (s *mysql) zeroKey() { - s.Lock() - defer s.Unlock() - - util.Zero(s.key[:]) - s.key = nil + return sbox.Decrypt(key, data) } // isEncrypted returns whether the provided blob has been prefixed with an sbox diff --git a/politeiad/backendv2/tstorebe/store/mysql/mysql.go b/politeiad/backendv2/tstorebe/store/mysql/mysql.go index 9b1e16bac..8db58bef6 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/mysql.go +++ b/politeiad/backendv2/tstorebe/store/mysql/mysql.go @@ -12,6 +12,7 @@ import ( "time" "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" + "github.com/decred/politeia/util" _ "github.com/go-sql-driver/mysql" ) @@ -58,11 +59,43 @@ func ctxWithTimeout() (context.Context, func()) { return context.WithTimeout(context.Background(), connTimeout) } +func (s *mysql) getKey() (*[32]byte, error) { + s.RLock() + defer s.RUnlock() + + if s.key == nil { + return nil, fmt.Errorf("encryption key not found") + } + + return s.key, nil +} + +func (s *mysql) setKey(key *[32]byte) { + s.Lock() + defer s.Unlock() + + s.key = key +} + +func (s *mysql) zeroKey() { + s.Lock() + defer s.Unlock() + + if s.key != nil { + util.Zero(s.key[:]) + s.key = nil + } +} + func (s *mysql) put(blobs map[string][]byte, encrypt bool, ctx context.Context, tx *sql.Tx) error { // Encrypt blobs if encrypt { + key, err := s.getKey() + if err != nil { + return err + } for k, v := range blobs { - e, err := s.encrypt(ctx, tx, v) + e, err := s.encrypt(ctx, tx, key, v) if err != nil { return fmt.Errorf("encrypt: %v", err) } @@ -232,7 +265,11 @@ func (s *mysql) Get(keys []string) (map[string][]byte, error) { if !encrypted { continue } - b, _, err := s.decrypt(v) + key, err := s.getKey() + if err != nil { + return nil, err + } + b, _, err := s.decrypt(key, v) if err != nil { return nil, fmt.Errorf("decrypt: %v", err) } @@ -255,7 +292,7 @@ func New(appDir, host, user, password, dbname string) (*mysql, error) { } // Connect to database - log.Infof("MySQL: %v:[password]@tcp(%v)/%v", user, host, dbname) + log.Infof("MySQL host: %v:[password]@tcp(%v)/%v", user, host, dbname) h := fmt.Sprintf("%v:%v@tcp(%v)/%v", user, password, host, dbname) db, err := sql.Open("mysql", h) @@ -290,16 +327,17 @@ func New(appDir, host, user, password, dbname string) (*mysql, error) { return nil, fmt.Errorf("create nonce table: %v", err) } - // Setup kv store context + // Setup mysql context s := mysql{ db: db, } // Derive encryption key from password - s.key, err = s.argon2idKey(password) + key, err := s.argon2idKey(password) if err != nil { return nil, fmt.Errorf("argon2idKey: %v", err) } + s.setKey(key) return &s, nil } diff --git a/politeiad/backendv2/tstorebe/store/mysql/nonce.go b/politeiad/backendv2/tstorebe/store/mysql/nonce.go index 1ec161587..cb06547b7 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/nonce.go +++ b/politeiad/backendv2/tstorebe/store/mysql/nonce.go @@ -12,6 +12,47 @@ import ( "sync" ) +// nonce returns a new nonce value. This function guarantees that the returned +// nonce will be unique for every invocation. +// +// This function must be called using a transaction. +func (s *mysql) nonce(ctx context.Context, tx *sql.Tx) (int64, error) { + // Create and retrieve new nonce value in an atomic database + // transaction. + _, err := tx.ExecContext(ctx, "INSERT INTO nonce () VALUES ();") + if err != nil { + return 0, fmt.Errorf("insert: %v", err) + } + rows, err := tx.QueryContext(ctx, "SELECT LAST_INSERT_ID();") + if err != nil { + return 0, fmt.Errorf("query: %v", err) + } + defer rows.Close() + + var nonce int64 + for rows.Next() { + if nonce > 0 { + // There should only ever be one row returned. Something is + // wrong if we've already scanned the nonce and its still + // scanning rows. + return 0, fmt.Errorf("multiple rows returned for nonce") + } + err = rows.Scan(&nonce) + if err != nil { + return 0, fmt.Errorf("scan: %v", err) + } + } + err = rows.Err() + if err != nil { + return 0, fmt.Errorf("next: %v", err) + } + if nonce == 0 { + return 0, fmt.Errorf("invalid 0 nonce") + } + + return nonce, nil +} + // testNonce is used to verify that nonce races do not occur. This function is // meant to be run against an actual MySQL/MariaDB instance, not as a unit // test. From 3cd70e87fda5f088d53a7159fdadc89065ac5e61 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 17 Mar 2021 09:31:18 -0500 Subject: [PATCH 410/449] politeiawww/records: Fix unvetted bug. --- politeiawww/records/process.go | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index baa698673..a5719b1eb 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -257,9 +257,8 @@ func (r *Records) processDetails(ctx context.Context, d v1.Details, u *user.User // Only admins and the record author are allowed to retrieve // unvetted record files. Remove files if the user is not an admin - // or the author. This is a public route so a user may not be - // present. - if rc.State == v1.RecordStateUnvetted { + // or the author. This is a public route so a user may not exist. + if rc.State != v1.RecordStateVetted { var ( authorID = userIDFromMetadataStreams(rc.Metadata) isAuthor = u != nil && u.ID.String() == authorID @@ -294,6 +293,23 @@ func (r *Records) processRecords(ctx context.Context, rs v1.Records, u *user.Use return nil, err } + // Only admins and the record author are allowed to retrieve + // unvetted record files. Remove files if the user is not an admin + // or the author. This is a public route so a user may not exist. + for k, v := range records { + if v.State != v1.RecordStateVetted { + var ( + authorID = userIDFromMetadataStreams(v.Metadata) + isAuthor = u != nil && u.ID.String() == authorID + isAdmin = u != nil && u.Admin + ) + if !isAuthor && !isAdmin { + v.Files = []v1.File{} + records[k] = v + } + } + } + return &v1.RecordsReply{ Records: records, }, nil @@ -380,7 +396,7 @@ func (r *Records) processTimestamps(ctx context.Context, t v1.Timestamps, isAdmi // Unvetted data blobs are stripped if the user is not an admin. // The rest of the timestamp is still returned. - if rc.State == v1.RecordStateUnvetted && !isAdmin { + if rc.State != v1.RecordStateVetted && !isAdmin { recordMD.Data = "" for k, v := range files { v.Data = "" From 7a4196e7284dfafb264e7d011e820d32aa2fd5c9 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 17 Mar 2021 09:36:59 -0500 Subject: [PATCH 411/449] politeiad: Fix setup scripts naming. --- politeiad/README.md | 2 +- .../scripts/{mysql-tstore-reset.sh => tstore-mysql-reset.sh} | 2 +- .../scripts/{mysql-tstore-setup.sh => tstore-mysql-setup.sh} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename politeiad/scripts/{mysql-tstore-reset.sh => tstore-mysql-reset.sh} (98%) rename politeiad/scripts/{mysql-tstore-setup.sh => tstore-mysql-setup.sh} (100%) diff --git a/politeiad/README.md b/politeiad/README.md index 54598f4c5..b3fbbf8d7 100644 --- a/politeiad/README.md +++ b/politeiad/README.md @@ -113,7 +113,7 @@ politeiad MYSQL_ROOT_PASSWORD=rootpass \ MYSQL_POLITEIAD_PASSWORD=politeiadpass \ MYSQL_TRILLIAN_PASSWORD=trillianpass \ - ./mysql-tstore-setup.sh + ./tstore-mysql-setup.sh ``` 6. Run the trillian mysql setup scripts. diff --git a/politeiad/scripts/mysql-tstore-reset.sh b/politeiad/scripts/tstore-mysql-reset.sh similarity index 98% rename from politeiad/scripts/mysql-tstore-reset.sh rename to politeiad/scripts/tstore-mysql-reset.sh index e001a342e..4f96f5d04 100755 --- a/politeiad/scripts/mysql-tstore-reset.sh +++ b/politeiad/scripts/tstore-mysql-reset.sh @@ -52,7 +52,7 @@ case $answer in ;; esac -echo "politeiad testnet reset complete!" +echo "Testnet politeiad reset complete!" echo "The trillian logs must be reset using the trillian script. See docs." echo "Mainnet politeiad resets must be done manually." diff --git a/politeiad/scripts/mysql-tstore-setup.sh b/politeiad/scripts/tstore-mysql-setup.sh similarity index 100% rename from politeiad/scripts/mysql-tstore-setup.sh rename to politeiad/scripts/tstore-mysql-setup.sh From 7d18e9d122956707a0e2550a0dd0f000d4610f8b Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 18 Mar 2021 10:10:19 -0500 Subject: [PATCH 412/449] politeiawww/client: General cleanup. --- politeiawww/client/client.go | 11 +++++++---- politeiawww/client/error.go | 12 ++++++++---- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/politeiawww/client/client.go b/politeiawww/client/client.go index 659af3586..cb860815c 100644 --- a/politeiawww/client/client.go +++ b/politeiawww/client/client.go @@ -124,7 +124,7 @@ func (c *Client) makeReq(method string, api, route string, v interface{}) ([]byt if err := decoder.Decode(&e); err != nil { return nil, fmt.Errorf("status code %v: %v", r.StatusCode, err) } - return nil, Error{ + return nil, RespErr{ HTTPCode: r.StatusCode, API: api, ErrorReply: e, @@ -147,9 +147,12 @@ func (c *Client) makeReq(method string, api, route string, v interface{}) ([]byt // Opts contains the politeiawww client options. All values are optional. // -// Any provided HTTPSCert will be added to the http client's trust cert pool. -// This allows you to interact with a politeiawww instance that uses a self -// signed cert. +// Any provided HTTPSCert will be added to the http client's trust cert pool, +// allowing you to interact with a politeiawww instance that uses a self signed +// cert. +// +// Authenticated routes require a CSRF cookie as well as the corresponding CSRF +// header. type Opts struct { HTTPSCert string Cookies []*http.Cookie diff --git a/politeiawww/client/error.go b/politeiawww/client/error.go index fe87d46f5..329227dff 100644 --- a/politeiawww/client/error.go +++ b/politeiawww/client/error.go @@ -27,16 +27,20 @@ type ErrorReply struct { ErrorContext string } -// Error represents a politeiawww response error. An Error is returned anytime -// the politeiawww response is not a 200. -type Error struct { +// RespErr represents a politeiawww response error. An Error is returned +// anytime the politeiawww response is not a 200. +// +// The various politeiawww APIs can have overlapping error codes. The API is +// included to allow the Error() method to return the correct human readable +// error message. +type RespErr struct { HTTPCode int API string ErrorReply ErrorReply } // Error satisfies the error interface. -func (e Error) Error() string { +func (e RespErr) Error() string { switch e.HTTPCode { case http.StatusInternalServerError: return fmt.Sprintf("500 internal server error: %v", From 7c4908818530a70924c7f247c6bc356694cacdf9 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 18 Mar 2021 14:55:45 -0500 Subject: [PATCH 413/449] tstorebe/comments: Address review issues. --- .../tstorebe/plugins/comments/cmds.go | 632 +++++++++--------- .../tstorebe/plugins/comments/comments.go | 17 +- .../tstorebe/plugins/comments/recordindex.go | 31 +- politeiad/plugins/comments/comments.go | 30 +- 4 files changed, 371 insertions(+), 339 deletions(-) diff --git a/politeiad/backendv2/tstorebe/plugins/comments/cmds.go b/politeiad/backendv2/tstorebe/plugins/comments/cmds.go index 4847c9476..ea2d223d6 100644 --- a/politeiad/backendv2/tstorebe/plugins/comments/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/comments/cmds.go @@ -30,261 +30,7 @@ const ( dataDescriptorCommentVote = pluginID + "-vote-v1" ) -func tokenDecode(token string) ([]byte, error) { - return util.TokenDecode(util.TokenTypeTstore, token) -} - -func convertSignatureError(err error) backend.PluginError { - var e util.SignatureError - var s comments.ErrorCodeT - if errors.As(err, &e) { - switch e.ErrorCode { - case util.ErrorStatusPublicKeyInvalid: - s = comments.ErrorCodePublicKeyInvalid - case util.ErrorStatusSignatureInvalid: - s = comments.ErrorCodeSignatureInvalid - } - } - return backend.PluginError{ - PluginID: comments.PluginID, - ErrorCode: uint32(s), - ErrorContext: e.ErrorContext, - } -} - -func convertBlobEntryFromCommentAdd(c comments.CommentAdd) (*store.BlobEntry, error) { - data, err := json.Marshal(c) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorCommentAdd, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func convertBlobEntryFromCommentDel(c comments.CommentDel) (*store.BlobEntry, error) { - data, err := json.Marshal(c) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorCommentDel, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func convertBlobEntryFromCommentVote(c comments.CommentVote) (*store.BlobEntry, error) { - data, err := json.Marshal(c) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorCommentVote, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func convertCommentAddFromBlobEntry(be store.BlobEntry) (*comments.CommentAdd, error) { - // Decode and validate data hint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorCommentAdd { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorCommentAdd) - } - - // Decode data - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - digest, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, fmt.Errorf("decode digest: %v", err) - } - if !bytes.Equal(util.Digest(b), digest) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), digest) - } - var c comments.CommentAdd - err = json.Unmarshal(b, &c) - if err != nil { - return nil, fmt.Errorf("unmarshal CommentAdd: %v", err) - } - - return &c, nil -} - -func convertCommentDelFromBlobEntry(be store.BlobEntry) (*comments.CommentDel, error) { - // Decode and validate data hint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorCommentDel { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorCommentDel) - } - - // Decode data - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - digest, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, fmt.Errorf("decode digest: %v", err) - } - if !bytes.Equal(util.Digest(b), digest) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), digest) - } - var c comments.CommentDel - err = json.Unmarshal(b, &c) - if err != nil { - return nil, fmt.Errorf("unmarshal CommentDel: %v", err) - } - - return &c, nil -} - -func convertCommentVoteFromBlobEntry(be store.BlobEntry) (*comments.CommentVote, error) { - // Decode and validate data hint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorCommentVote { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorCommentVote) - } - - // Decode data - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - digest, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, fmt.Errorf("decode digest: %v", err) - } - if !bytes.Equal(util.Digest(b), digest) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), digest) - } - var cv comments.CommentVote - err = json.Unmarshal(b, &cv) - if err != nil { - return nil, fmt.Errorf("unmarshal CommentVote: %v", err) - } - - return &cv, nil -} - -func convertCommentFromCommentAdd(ca comments.CommentAdd) comments.Comment { - return comments.Comment{ - UserID: ca.UserID, - Token: ca.Token, - ParentID: ca.ParentID, - Comment: ca.Comment, - PublicKey: ca.PublicKey, - Signature: ca.Signature, - CommentID: ca.CommentID, - Version: ca.Version, - Timestamp: ca.Timestamp, - Receipt: ca.Receipt, - Downvotes: 0, // Not part of commentAdd data - Upvotes: 0, // Not part of commentAdd data - Deleted: false, - Reason: "", - ExtraData: ca.ExtraData, - ExtraDataHint: ca.ExtraDataHint, - } -} - -func convertCommentFromCommentDel(cd comments.CommentDel) comments.Comment { - // Score needs to be filled in separately - return comments.Comment{ - UserID: cd.UserID, - Token: cd.Token, - ParentID: cd.ParentID, - Comment: "", - Signature: "", - CommentID: cd.CommentID, - Version: 0, - Timestamp: cd.Timestamp, - Receipt: cd.Receipt, - Downvotes: 0, - Upvotes: 0, - Deleted: true, - Reason: cd.Reason, - } -} - -// commentVersionLatest returns the latest comment version. -func commentVersionLatest(cidx commentIndex) uint32 { - var maxVersion uint32 - for version := range cidx.Adds { - if version > maxVersion { - maxVersion = version - } - } - return maxVersion -} - -// commentExists returns whether the provided comment ID exists. -func commentExists(ridx recordIndex, commentID uint32) bool { - _, ok := ridx.Comments[commentID] - return ok -} - -// commentIDLatest returns the latest comment ID. -func commentIDLatest(idx recordIndex) uint32 { - var maxID uint32 - for id := range idx.Comments { - if id > maxID { - maxID = id - } - } - return maxID -} - +// commentAddSave saves a CommentAdd to the backend. func (p *commentsPlugin) commentAddSave(treeID int64, ca comments.CommentAdd) ([]byte, error) { be, err := convertBlobEntryFromCommentAdd(ca) if err != nil { @@ -301,7 +47,9 @@ func (p *commentsPlugin) commentAddSave(treeID int64, ca comments.CommentAdd) ([ return d, nil } -// commentAdds returns the commentAdd for all specified digests. +// commentAdds returns a commentAdd for each of the provided digests. A digest +// refers to the blob entry digest, which can be used to retrieve the blob +// entry from the backend. func (p *commentsPlugin) commentAdds(treeID int64, digests [][]byte) ([]comments.CommentAdd, error) { // Retrieve blobs blobs, err := p.tstore.Blobs(treeID, digests) @@ -333,6 +81,7 @@ func (p *commentsPlugin) commentAdds(treeID int64, digests [][]byte) ([]comments return adds, nil } +// commentDelSave saves a CommentDel to the backend. func (p *commentsPlugin) commentDelSave(treeID int64, cd comments.CommentDel) ([]byte, error) { be, err := convertBlobEntryFromCommentDel(cd) if err != nil { @@ -349,6 +98,9 @@ func (p *commentsPlugin) commentDelSave(treeID int64, cd comments.CommentDel) ([ return d, nil } +// commentDels returns a commentDel for each of the provided digests. A digest +// refers to the blob entry digest, which can be used to retrieve the blob +// entry from the backend. func (p *commentsPlugin) commentDels(treeID int64, digests [][]byte) ([]comments.CommentDel, error) { // Retrieve blobs blobs, err := p.tstore.Blobs(treeID, digests) @@ -380,6 +132,7 @@ func (p *commentsPlugin) commentDels(treeID int64, digests [][]byte) ([]comments return dels, nil } +// commentVoteSave saves a CommentVote to the backend. func (p *commentsPlugin) commentVoteSave(treeID int64, cv comments.CommentVote) ([]byte, error) { be, err := convertBlobEntryFromCommentVote(cv) if err != nil { @@ -396,6 +149,9 @@ func (p *commentsPlugin) commentVoteSave(treeID int64, cv comments.CommentVote) return d, nil } +// commentVotes returns a CommentVote for each of the provided digests. A +// digest refers to the blob entry digest, which can be used to retrieve the +// blob entry from the backend. func (p *commentsPlugin) commentVotes(treeID int64, digests [][]byte) ([]comments.CommentVote, error) { // Retrieve blobs blobs, err := p.tstore.Blobs(treeID, digests) @@ -428,10 +184,10 @@ func (p *commentsPlugin) commentVotes(treeID int64, digests [][]byte) ([]comment } // comments returns the most recent version of the specified comments. Deleted -// comments are returned with limited data. Comment IDs that do not correspond -// to an actual comment are not included in the returned map. It is the -// responsibility of the caller to ensure a comment is returned for each of the -// provided comment IDs. +// comments are returned with limited data. If a comment is not found for a +// provided comment IDs, the comment ID is excluded from the returned map. An +// error will not be returned. It is the responsibility of the caller to ensure +// a comment is returned for each of the provided comment IDs. func (p *commentsPlugin) comments(treeID int64, ridx recordIndex, commentIDs []uint32) (map[uint32]comments.Comment, error) { // Aggregate the digests for all records that need to be looked up. // If a comment has been deleted then the only record that will @@ -488,7 +244,7 @@ func (p *commentsPlugin) comments(treeID int64, ridx recordIndex, commentIDs []u if !ok { return nil, fmt.Errorf("comment index not found %v", c.CommentID) } - c.Downvotes, c.Upvotes = calcVoteScore(cidx) + c.Downvotes, c.Upvotes = voteScore(cidx) cs[v.CommentID] = c } for _, v := range dels { @@ -499,7 +255,7 @@ func (p *commentsPlugin) comments(treeID int64, ridx recordIndex, commentIDs []u return cs, nil } -// comment returns the latest version of the provided comment. +// comment returns the latest version of a comment. func (p *commentsPlugin) comment(treeID int64, ridx recordIndex, commentID uint32) (*comments.Comment, error) { cs, err := p.comments(treeID, ridx, []uint32{commentID}) if err != nil { @@ -512,6 +268,7 @@ func (p *commentsPlugin) comment(treeID int64, ridx recordIndex, commentID uint3 return &c, nil } +// timestamp returns the timestamp for a blob entry digest. func (p *commentsPlugin) timestamp(treeID int64, digest []byte) (*comments.Timestamp, error) { // Get timestamp t, err := p.tstore.Timestamp(treeID, digest) @@ -539,9 +296,9 @@ func (p *commentsPlugin) timestamp(treeID int64, digest []byte) (*comments.Times }, nil } -// calcVoteScore returns the vote score for the provided comment index. The -// returned values are the downvotes and upvotes, respectively. -func calcVoteScore(cidx commentIndex) (uint64, uint64) { +// voteScore returns the total number of downvotes and upvotes, respectively, +// for a comment. +func voteScore(cidx commentIndex) (uint64, uint64) { // Find the vote score by replaying all existing votes from all // users. The net effect of a new vote on a comment score depends // on the previous vote from that uuid. Example, a user upvotes a @@ -590,6 +347,7 @@ func calcVoteScore(cidx commentIndex) (uint64, uint64) { return downvotes, upvotes } +// cmdNew creates a new comment. func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (string, error) { log.Tracef("cmdNew: %v %x %v", treeID, token, payload) @@ -643,11 +401,10 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str return "", err } if uint32(n.State) != uint32(state) { - e := fmt.Sprintf("got %v, want %v", n.State, state) return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: uint32(comments.ErrorCodeStateInvalid), - ErrorContext: e, + ErrorCode: uint32(comments.ErrorCodeRecordStateInvalid), + ErrorContext: fmt.Sprintf("got %v, want %v", n.State, state), } } @@ -691,7 +448,7 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str return "", fmt.Errorf("commentAddSave: %v", err) } - // Update index + // Update the index ridx.Comments[ca.CommentID] = commentIndex{ Adds: map[uint32][]byte{ 1: digest, @@ -700,11 +457,8 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str Votes: make(map[string][]voteIndex), } - // Save index - err = p.recordIndexSave(token, state, *ridx) - if err != nil { - return "", err - } + // Save the updated index + p.recordIndexSave(token, state, *ridx) log.Debugf("Comment saved to record %v comment ID %v", ca.Token, ca.CommentID) @@ -727,6 +481,7 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str return string(reply), nil } +// cmdEdit edits an existing comment. func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (string, error) { log.Tracef("cmdEdit: %v %x %v", treeID, token, payload) @@ -780,11 +535,10 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st return "", err } if uint32(e.State) != uint32(state) { - e := fmt.Sprintf("got %v, want %v", e.State, state) return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: uint32(comments.ErrorCodeStateInvalid), - ErrorContext: e, + ErrorCode: uint32(comments.ErrorCodeRecordStateInvalid), + ErrorContext: fmt.Sprintf("got %v, want %v", e.State, state), } } @@ -858,14 +612,11 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st return "", fmt.Errorf("commentAddSave: %v", err) } - // Update index + // Update the index ridx.Comments[ca.CommentID].Adds[ca.Version] = digest - // Save index - err = p.recordIndexSave(token, state, *ridx) - if err != nil { - return "", err - } + // Save the updated index + p.recordIndexSave(token, state, *ridx) log.Debugf("Comment edited on record %v comment ID %v", ca.Token, ca.CommentID) @@ -888,6 +639,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st return string(reply), nil } +// cmdDel deletes a comment. func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (string, error) { log.Tracef("cmdDel: %v %x %v", treeID, token, payload) @@ -931,11 +683,10 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str return "", err } if uint32(d.State) != uint32(state) { - e := fmt.Sprintf("got %v, want %v", d.State, state) return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: uint32(comments.ErrorCodeStateInvalid), - ErrorContext: e, + ErrorCode: uint32(comments.ErrorCodeRecordStateInvalid), + ErrorContext: fmt.Sprintf("got %v, want %v", d.State, state), } } @@ -979,7 +730,7 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str return "", fmt.Errorf("commentDelSave: %v", err) } - // Update index + // Update the index cidx, ok := ridx.Comments[d.CommentID] if !ok { // This should not be possible @@ -989,20 +740,22 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str cidx.Del = digest ridx.Comments[d.CommentID] = cidx - // Save index - err = p.recordIndexSave(token, state, *ridx) - if err != nil { - return "", err - } + // Svae the updated index + p.recordIndexSave(token, state, *ridx) - // Delete all comment versions + // Delete all comment versions. A comment is considered deleted + // once the CommenDel record has been saved. If attempts to + // actually delete the blobs fails, simply log the error and + // continue command execution. The period fsck will clean this up + // next time it is run. digests := make([][]byte, 0, len(cidx.Adds)) for _, v := range cidx.Adds { digests = append(digests, v) } err = p.tstore.BlobsDel(treeID, digests) if err != nil { - return "", fmt.Errorf("del: %v", err) + log.Errorf("comments cmdDel %x: BlobsDel %x: %v ", + token, digests, err) } // Return updated comment @@ -1023,6 +776,7 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str return string(reply), nil } +// cmdVote casts a upvote/downvote for a comment. func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (string, error) { log.Tracef("cmdVote: %v %x %v", treeID, token, payload) @@ -1078,11 +832,10 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st return "", err } if uint32(v.State) != uint32(state) { - e := fmt.Sprintf("got %v, want %v", v.State, state) return "", backend.PluginError{ PluginID: comments.PluginID, - ErrorCode: uint32(comments.ErrorCodeStateInvalid), - ErrorContext: e, + ErrorCode: uint32(comments.ErrorCodeRecordStateInvalid), + ErrorContext: fmt.Sprintf("got %v, want %v", v.State, state), } } @@ -1102,11 +855,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st } // Verify user has not exceeded max allowed vote changes - uvotes, ok := cidx.Votes[v.UserID] - if !ok { - uvotes = make([]voteIndex, 0) - } - if len(uvotes) > int(p.voteChangesMax) { + if len(cidx.Votes[v.UserID]) > int(p.voteChangesMax) { return "", backend.PluginError{ PluginID: comments.PluginID, ErrorCode: uint32(comments.ErrorCodeVoteChangesMaxExceeded), @@ -1160,18 +909,13 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st Digest: digest, }) cidx.Votes[cv.UserID] = votes - - // Update the record index ridx.Comments[cv.CommentID] = cidx - // Save index - err = p.recordIndexSave(token, state, *ridx) - if err != nil { - return "", err - } + // Save the updated index + p.recordIndexSave(token, state, *ridx) // Calculate the new vote scores - downvotes, upvotes := calcVoteScore(cidx) + downvotes, upvotes := voteScore(cidx) // Prepare reply vr := comments.VoteReply{ @@ -1188,6 +932,8 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st return string(reply), nil } +// cmdGet retrieves a batch of specified comments. The most recent version of +// each comment is returned. func (p *commentsPlugin) cmdGet(treeID int64, token []byte, payload string) (string, error) { log.Tracef("cmdGet: %v %x %v", treeID, token, payload) @@ -1228,6 +974,8 @@ func (p *commentsPlugin) cmdGet(treeID int64, token []byte, payload string) (str return string(reply), nil } +// cmdGetAll retrieves all comments for a record. The latest version of each +// comment is returned. func (p *commentsPlugin) cmdGetAll(treeID int64, token []byte) (string, error) { log.Tracef("cmdGetAll: %v %x", treeID, token) @@ -1276,6 +1024,7 @@ func (p *commentsPlugin) cmdGetAll(treeID int64, token []byte) (string, error) { return string(reply), nil } +// cmdGetVersion retrieves the specified version of a comment. func (p *commentsPlugin) cmdGetVersion(treeID int64, token []byte, payload string) (string, error) { log.Tracef("cmdGetVersion: %v %x %v", treeID, token, payload) @@ -1336,7 +1085,7 @@ func (p *commentsPlugin) cmdGetVersion(treeID int64, token []byte, payload strin // Convert to a comment c := convertCommentFromCommentAdd(adds[0]) - c.Downvotes, c.Upvotes = calcVoteScore(cidx) + c.Downvotes, c.Upvotes = voteScore(cidx) // Prepare reply gvr := comments.GetVersionReply{ @@ -1350,6 +1099,8 @@ func (p *commentsPlugin) cmdGetVersion(treeID int64, token []byte, payload strin return string(reply), nil } +// cmdCount retrieves the comments count for a record. The comments count is +// the number of comments that have been made on a record. func (p *commentsPlugin) cmdCount(treeID int64, token []byte) (string, error) { log.Tracef("cmdCount: %v %x", treeID, token) @@ -1377,6 +1128,8 @@ func (p *commentsPlugin) cmdCount(treeID int64, token []byte) (string, error) { return string(reply), nil } +// cmdVotes retrieves the comment votes that meet the provided filtering +// criteria. func (p *commentsPlugin) cmdVotes(treeID int64, token []byte, payload string) (string, error) { log.Tracef("cmdVotes: %v %x %v", treeID, token, payload) @@ -1433,6 +1186,7 @@ func (p *commentsPlugin) cmdVotes(treeID int64, token []byte, payload string) (s return string(reply), nil } +// cmdTimestamps retrieves the timestamps for the comments of a record. func (p *commentsPlugin) cmdTimestamps(treeID int64, token []byte, payload string) (string, error) { log.Tracef("cmdTimestamps: %v %x %v", treeID, token, payload) @@ -1530,3 +1284,259 @@ func (p *commentsPlugin) cmdTimestamps(treeID int64, token []byte, payload strin return string(reply), nil } + +// tokenDecode decodes a tstore token. It only accepts full length tokens. +func tokenDecode(token string) ([]byte, error) { + return util.TokenDecode(util.TokenTypeTstore, token) +} + +func convertCommentFromCommentAdd(ca comments.CommentAdd) comments.Comment { + return comments.Comment{ + UserID: ca.UserID, + Token: ca.Token, + ParentID: ca.ParentID, + Comment: ca.Comment, + PublicKey: ca.PublicKey, + Signature: ca.Signature, + CommentID: ca.CommentID, + Version: ca.Version, + Timestamp: ca.Timestamp, + Receipt: ca.Receipt, + Downvotes: 0, // Not part of commentAdd data + Upvotes: 0, // Not part of commentAdd data + Deleted: false, + Reason: "", + ExtraData: ca.ExtraData, + ExtraDataHint: ca.ExtraDataHint, + } +} + +func convertCommentFromCommentDel(cd comments.CommentDel) comments.Comment { + // Score needs to be filled in separately + return comments.Comment{ + UserID: cd.UserID, + Token: cd.Token, + ParentID: cd.ParentID, + Comment: "", + Signature: "", + CommentID: cd.CommentID, + Version: 0, + Timestamp: cd.Timestamp, + Receipt: cd.Receipt, + Downvotes: 0, + Upvotes: 0, + Deleted: true, + Reason: cd.Reason, + } +} + +// commentVersionLatest returns the latest comment version. +func commentVersionLatest(cidx commentIndex) uint32 { + var maxVersion uint32 + for version := range cidx.Adds { + if version > maxVersion { + maxVersion = version + } + } + return maxVersion +} + +// commentExists returns whether the provided comment ID exists. +func commentExists(ridx recordIndex, commentID uint32) bool { + _, ok := ridx.Comments[commentID] + return ok +} + +// commentIDLatest returns the latest comment ID. +func commentIDLatest(idx recordIndex) uint32 { + var maxID uint32 + for id := range idx.Comments { + if id > maxID { + maxID = id + } + } + return maxID +} + +func convertSignatureError(err error) backend.PluginError { + var e util.SignatureError + var s comments.ErrorCodeT + if errors.As(err, &e) { + switch e.ErrorCode { + case util.ErrorStatusPublicKeyInvalid: + s = comments.ErrorCodePublicKeyInvalid + case util.ErrorStatusSignatureInvalid: + s = comments.ErrorCodeSignatureInvalid + } + } + return backend.PluginError{ + PluginID: comments.PluginID, + ErrorCode: uint32(s), + ErrorContext: e.ErrorContext, + } +} + +func convertBlobEntryFromCommentAdd(c comments.CommentAdd) (*store.BlobEntry, error) { + data, err := json.Marshal(c) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorCommentAdd, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertBlobEntryFromCommentDel(c comments.CommentDel) (*store.BlobEntry, error) { + data, err := json.Marshal(c) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorCommentDel, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertBlobEntryFromCommentVote(c comments.CommentVote) (*store.BlobEntry, error) { + data, err := json.Marshal(c) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorCommentVote, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertCommentAddFromBlobEntry(be store.BlobEntry) (*comments.CommentAdd, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorCommentAdd { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorCommentAdd) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, fmt.Errorf("decode digest: %v", err) + } + if !bytes.Equal(util.Digest(b), digest) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), digest) + } + var c comments.CommentAdd + err = json.Unmarshal(b, &c) + if err != nil { + return nil, fmt.Errorf("unmarshal CommentAdd: %v", err) + } + + return &c, nil +} + +func convertCommentDelFromBlobEntry(be store.BlobEntry) (*comments.CommentDel, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorCommentDel { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorCommentDel) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, fmt.Errorf("decode digest: %v", err) + } + if !bytes.Equal(util.Digest(b), digest) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), digest) + } + var c comments.CommentDel + err = json.Unmarshal(b, &c) + if err != nil { + return nil, fmt.Errorf("unmarshal CommentDel: %v", err) + } + + return &c, nil +} + +func convertCommentVoteFromBlobEntry(be store.BlobEntry) (*comments.CommentVote, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorCommentVote { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorCommentVote) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, fmt.Errorf("decode digest: %v", err) + } + if !bytes.Equal(util.Digest(b), digest) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), digest) + } + var cv comments.CommentVote + err = json.Unmarshal(b, &cv) + if err != nil { + return nil, fmt.Errorf("unmarshal CommentVote: %v", err) + } + + return &cv, nil +} diff --git a/politeiad/backendv2/tstorebe/plugins/comments/comments.go b/politeiad/backendv2/tstorebe/plugins/comments/comments.go index 0c3baa5d7..a03a99f4a 100644 --- a/politeiad/backendv2/tstorebe/plugins/comments/comments.go +++ b/politeiad/backendv2/tstorebe/plugins/comments/comments.go @@ -23,7 +23,7 @@ var ( // commentsPlugin is the tstore backend implementation of the comments plugin. // -// commentsPlugin satisfies the plugins.PluginClient interface. +// commentsPlugin satisfies the plugins PluginClient interface. type commentsPlugin struct { sync.RWMutex tstore plugins.TstoreClient @@ -45,7 +45,7 @@ type commentsPlugin struct { // Setup performs any plugin setup that is required. // -// This function satisfies the plugins.PluginClient interface. +// This function satisfies the plugins PluginClient interface. func (p *commentsPlugin) Setup() error { log.Tracef("comments Setup") @@ -54,7 +54,7 @@ func (p *commentsPlugin) Setup() error { // Cmd executes a plugin command. // -// This function satisfies the plugins.PluginClient interface. +// This function satisfies the plugins PluginClient interface. func (p *commentsPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { log.Tracef("comments Cmd: %v %x %v", treeID, token, cmd) @@ -86,7 +86,7 @@ func (p *commentsPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (s // Hook executes a plugin hook. // -// This function satisfies the plugins.PluginClient interface. +// This function satisfies the plugins PluginClient interface. func (p *commentsPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { log.Tracef("comments Hook: %v %x %v", plugins.Hooks[h], token, treeID) @@ -95,18 +95,19 @@ func (p *commentsPlugin) Hook(treeID int64, token []byte, h plugins.HookT, paylo // Fsck performs a plugin filesystem check. // -// This function satisfies the plugins.PluginClient interface. +// This function satisfies the plugins PluginClient interface. func (p *commentsPlugin) Fsck(treeIDs []int64) error { log.Tracef("comments Fsck") + // Verify record index coherency // Verify CommentDel blobs were actually deleted return nil } -// Settings returns the plugin's settings. +// Settings returns the plugin settings. // -// This function satisfies the plugins.PluginClient interface. +// This function satisfies the plugins PluginClient interface. func (p *commentsPlugin) Settings() []backend.PluginSetting { log.Tracef("comments Settings") @@ -154,6 +155,8 @@ func New(tstore plugins.TstoreClient, settings []backend.PluginSetting, dataDir v.Key, v.Value, err) } voteChangesMax = uint32(u) + default: + return nil, fmt.Errorf("invalid comments plugin setting '%v'", v.Key) } } diff --git a/politeiad/backendv2/tstorebe/plugins/comments/recordindex.go b/politeiad/backendv2/tstorebe/plugins/comments/recordindex.go index f8d8d18bb..3cda339b4 100644 --- a/politeiad/backendv2/tstorebe/plugins/comments/recordindex.go +++ b/politeiad/backendv2/tstorebe/plugins/comments/recordindex.go @@ -26,6 +26,9 @@ const ( ) // voteIndex contains the comment vote and the digest of the vote record. +// Caching the vote allows us to tally the votes for a comment without needing +// to pull the vote blobs from the backend. The digest allows us to retrieve +// the vote blob if we need to. type voteIndex struct { Vote comments.VoteT `json:"vote"` Digest []byte `json:"digest"` @@ -51,8 +54,9 @@ type recordIndex struct { Comments map[uint32]commentIndex `json:"comments"` // [commentID]comment } -// recordIndexPath accepts full length token or token prefixes, but always uses -// prefix when generating the comments index path string. +// recordIndexPath returns the file path for a cached record index. It accepts +// the full length token or the token prefix, but always uses prefix when +// generating the comments index path string. func (p *commentsPlugin) recordIndexPath(token []byte, s backend.StateT) string { var fn string switch s { @@ -100,22 +104,33 @@ func (p *commentsPlugin) recordIndex(token []byte, s backend.StateT) (*recordInd return &ridx, nil } -// recordIndexSave saves the provided recordIndex to the comments plugin data -// dir. +// _recordIndexSave saves the provided recordIndex to the comments plugin data dir. // // This function must be called WITHOUT the read/write lock held. -func (p *commentsPlugin) recordIndexSave(token []byte, s backend.StateT, ridx recordIndex) error { - p.Lock() - defer p.Unlock() - +func (p *commentsPlugin) _recordIndexSave(token []byte, s backend.StateT, ridx recordIndex) error { b, err := json.Marshal(ridx) if err != nil { return err } fp := p.recordIndexPath(token, s) + + p.Lock() + defer p.Unlock() + err = ioutil.WriteFile(fp, b, 0664) if err != nil { return err } return nil } + +// recordIndexSave is a wrapper around the _recordIndexSave method that allows +// us to decide how update errors should be handled. For now, we simply panic. +// If an error occurs the cache is no longer coherent and the only way to fix +// it is to rebuild it. +func (p *commentsPlugin) recordIndexSave(token []byte, s backend.StateT, ridx recordIndex) { + err := p._recordIndexSave(token, s, ridx) + if err != nil { + panic(err) + } +} diff --git a/politeiad/plugins/comments/comments.go b/politeiad/plugins/comments/comments.go index 121e4495f..e23aa4707 100644 --- a/politeiad/plugins/comments/comments.go +++ b/politeiad/plugins/comments/comments.go @@ -95,9 +95,9 @@ const ( // changes max plugin setting. ErrorCodeVoteChangesMaxExceeded ErrorCodeT = 10 - // ErrorCodeStateInvalid is returned when the provided state does - // not match the record state. - ErrorCodeStateInvalid ErrorCodeT = 11 + // ErrorCodeRecordStateInvalid is returned when the provided state + // does not match the record state. + ErrorCodeRecordStateInvalid ErrorCodeT = 11 ) var ( @@ -114,7 +114,7 @@ var ( ErrorCodeParentIDInvalid: "parent id invalid", ErrorCodeVoteInvalid: "vote invalid", ErrorCodeVoteChangesMaxExceeded: "vote changes max exceeded", - ErrorCodeStateInvalid: "record state invalid", + ErrorCodeRecordStateInvalid: "record state invalid", } ) @@ -338,9 +338,10 @@ type VoteReply struct { Receipt string `json:"receipt"` // Server signature of client signature } -// Get returns the latest version of the comments for the provided comment IDs. -// An error is not returned if a comment is not found for one or more of the -// comment IDs. Those entries will simply not be included in the reply. +// Get retrieves a batch of specified comments. The most recent version of each +// comment is returned. An error is not returned if a comment is not found for +// one or more of the comment IDs. Those entries will simply not be included in +// the reply. type Get struct { CommentIDs []uint32 `json:"commentids"` } @@ -353,15 +354,17 @@ type GetReply struct { Comments map[uint32]Comment `json:"comments"` // [commentID]Comment } -// GetAll returns the latest version off all comments for a record. +// GetAll retrieves all comments for a record. The latest version of each +// comment is returned. type GetAll struct{} -// GetAllReply is the reply to the GetAll command. +// GetAllReply is the reply to the GetAll command. The returned comments array +// is ordered by comment ID from smallest to largest. type GetAllReply struct { Comments []Comment `json:"comments"` } -// GetVersion returns a specific version of a comment. +// GetVersion retrieves the specified version of a comment. type GetVersion struct { CommentID uint32 `json:"commentid"` Version uint32 `json:"version"` @@ -372,7 +375,8 @@ type GetVersionReply struct { Comment Comment `json:"comment"` } -// Count returns the comments count for a record. +// Count retrieves the comments count for a record. The comments count is the +// number of comments that have been made on a record. type Count struct{} // CountReply is the reply to the Count command. @@ -380,7 +384,7 @@ type CountReply struct { Count uint32 `json:"count"` } -// Votes returns the comment votes that meet the provided filtering criteria. +// Votes retrieves the comment votes that meet the provided filtering criteria. type Votes struct { UserID string `json:"userid"` } @@ -418,7 +422,7 @@ type Timestamp struct { Proofs []Proof `json:"proofs"` } -// Timestamps requests the timestamps for a record's comments. If no comment +// Timestamps retrieves the timestamps for a record's comments. If no comment // IDs are provided then timestamps for all comments made on the record will // be returned. If IncludeVotes is set to true then the timestamps for the // comment votes will also be returned. If a provided comment ID does not From 4f0e0d2f0aac1e546a40ea342884968f8f3025a6 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 18 Mar 2021 19:38:33 -0500 Subject: [PATCH 414/449] multi: Fixing review issues. --- politeiad/backendv2/tstorebe/inventory.go | 42 ++-- .../tstorebe/plugins/comments/cmds.go | 80 +++--- .../tstorebe/plugins/comments/recordindex.go | 24 +- .../backendv2/tstorebe/plugins/pi/hooks.go | 46 ++-- .../tstorebe/plugins/ticketvote/cmds.go | 234 ++++++++---------- .../tstorebe/plugins/ticketvote/hooks.go | 25 +- 6 files changed, 209 insertions(+), 242 deletions(-) diff --git a/politeiad/backendv2/tstorebe/inventory.go b/politeiad/backendv2/tstorebe/inventory.go index 65055036e..f16d146ec 100644 --- a/politeiad/backendv2/tstorebe/inventory.go +++ b/politeiad/backendv2/tstorebe/inventory.go @@ -98,8 +98,7 @@ func (t *tstoreBackend) invAdd(state backend.StateT, token []byte, s backend.Sta case backend.StateVetted: fp = t.invPathVetted() default: - e := fmt.Sprintf("unknown state '%v'", state) - panic(e) + return fmt.Errorf("invalid state %v", state) } // Get inventory @@ -139,8 +138,7 @@ func (t *tstoreBackend) invUpdate(state backend.StateT, token []byte, s backend. case backend.StateVetted: fp = t.invPathVetted() default: - e := fmt.Sprintf("unknown state '%v'", state) - panic(e) + return fmt.Errorf("invalid state %v", state) } // Get inventory @@ -152,9 +150,7 @@ func (t *tstoreBackend) invUpdate(state backend.StateT, token []byte, s backend. // Del entry entries, err := entryDel(inv.Entries, token) if err != nil { - // This should not happen. Panic if it does. - e := fmt.Sprintf("%v entry del: %v", state, err) - panic(e) + return fmt.Errorf("%v entry del: %v", state, err) } // Prepend new entry to inventory @@ -186,28 +182,26 @@ func (t *tstoreBackend) invMoveToVetted(token []byte, s backend.StatusT) error { upath := t.invPathUnvetted() u, err := t.invGetLocked(upath) if err != nil { - return err + return fmt.Errorf("unvetted invGetLocked: %v", err) } // Del entry u.Entries, err = entryDel(u.Entries, token) if err != nil { - // This should not happen. Panic if it does. - e := fmt.Sprintf("unvetted entry del: %v", err) - panic(e) + return fmt.Errorf("entryDel: %v", err) } // Save unvetted inventory err = t.invSaveLocked(upath, *u) if err != nil { - return err + return fmt.Errorf("unvetted invSaveLocked: %v", err) } // Get vetted inventory vpath := t.invPathVetted() v, err := t.invGetLocked(vpath) if err != nil { - return err + return fmt.Errorf("vetted invGetLocked: %v", err) } // Prepend new entry to inventory @@ -220,7 +214,7 @@ func (t *tstoreBackend) invMoveToVetted(token []byte, s backend.StatusT) error { // Save vetted inventory err = t.invSaveLocked(vpath, *v) if err != nil { - return err + return fmt.Errorf("vetted invSaveLocked: %v", err) } log.Debugf("Inv move to vetted %x %v", token, backend.Statuses[s]) @@ -229,33 +223,34 @@ func (t *tstoreBackend) invMoveToVetted(token []byte, s backend.StatusT) error { } // inventoryAdd is a wrapper around the invAdd method that allows us to decide -// how disk read/write errors should be handled. For now we just panic. +// how errors should be handled. For now we just panic. If an error occurs the +// cache is no longer coherent and the only way to fix it is to rebuild it. func (t *tstoreBackend) inventoryAdd(state backend.StateT, token []byte, s backend.StatusT) { err := t.invAdd(state, token, s) if err != nil { - e := fmt.Sprintf("invAdd %v %x %v: %v", state, token, s, err) - panic(e) + panic(fmt.Sprintf("invAdd %v %x %v: %v", state, token, s, err)) } } // inventoryUpdate is a wrapper around the invUpdate method that allows us to // decide how disk read/write errors should be handled. For now we just panic. +// If an error occurs the cache is no longer coherent and the only way to fix +// it is to rebuild it. func (t *tstoreBackend) inventoryUpdate(state backend.StateT, token []byte, s backend.StatusT) { err := t.invUpdate(state, token, s) if err != nil { - e := fmt.Sprintf("invUpdate %v %x %v: %v", state, token, s, err) - panic(e) + panic(fmt.Sprintf("invUpdate %v %x %v: %v", state, token, s, err)) } } // inventoryMoveToVetted is a wrapper around the invMoveToVetted method that // allows us to decide how disk read/write errors should be handled. For now we -// just panic. +// just panic. If an error occurs the cache is no longer coherent and the only +// way to fix it is to rebuild it. func (t *tstoreBackend) inventoryMoveToVetted(token []byte, s backend.StatusT) { err := t.invMoveToVetted(token, s) if err != nil { - e := fmt.Sprintf("invMoveToVetted %x %v: %v", token, s, err) - panic(e) + panic(fmt.Sprintf("invMoveToVetted %x %v: %v", token, s, err)) } } @@ -337,8 +332,7 @@ func (t *tstoreBackend) invByStatus(state backend.StateT, s backend.StatusT, pag case backend.StateVetted: fp = t.invPathVetted() default: - e := fmt.Sprintf("unknown state '%v'", state) - panic(e) + return nil, fmt.Errorf("unknown state '%v'", state) } // Get inventory diff --git a/politeiad/backendv2/tstorebe/plugins/comments/cmds.go b/politeiad/backendv2/tstorebe/plugins/comments/cmds.go index ea2d223d6..beaae495f 100644 --- a/politeiad/backendv2/tstorebe/plugins/comments/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/comments/cmds.go @@ -338,9 +338,8 @@ func voteScore(cidx commentIndex) (uint64, uint64) { case 1: upvotes++ default: - // Something went wrong - e := fmt.Errorf("unexpected vote score %v", score) - panic(e) + // Should not be possible + panic(fmt.Errorf("unexpected vote score %v", score)) } } @@ -368,12 +367,11 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str } } if !bytes.Equal(t, token) { - e := fmt.Sprintf("comment token does not match route token: "+ - "got %x, want %x", t, token) return "", backend.PluginError{ - PluginID: comments.PluginID, - ErrorCode: uint32(comments.ErrorCodeTokenInvalid), - ErrorContext: e, + PluginID: comments.PluginID, + ErrorCode: uint32(comments.ErrorCodeTokenInvalid), + ErrorContext: fmt.Sprintf("comment token does not match "+ + "route token: got %x, want %x", t, token), } } @@ -387,11 +385,11 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str // Verify comment if len(n.Comment) > int(p.commentLengthMax) { - e := fmt.Sprintf("max length is %v characters", p.commentLengthMax) return "", backend.PluginError{ - PluginID: comments.PluginID, - ErrorCode: uint32(comments.ErrorCodeMaxLengthExceeded), - ErrorContext: e, + PluginID: comments.PluginID, + ErrorCode: uint32(comments.ErrorCodeMaxLengthExceeded), + ErrorContext: fmt.Sprintf("max length is %v characters", + p.commentLengthMax), } } @@ -502,12 +500,11 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st } } if !bytes.Equal(t, token) { - e := fmt.Sprintf("comment token does not match route token: "+ - "got %x, want %x", t, token) return "", backend.PluginError{ - PluginID: comments.PluginID, - ErrorCode: uint32(comments.ErrorCodeTokenInvalid), - ErrorContext: e, + PluginID: comments.PluginID, + ErrorCode: uint32(comments.ErrorCodeTokenInvalid), + ErrorContext: fmt.Sprintf("comment token does not match "+ + "route token: got %x, want %x", t, token), } } @@ -521,11 +518,11 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st // Verify comment if len(e.Comment) > int(p.commentLengthMax) { - e := fmt.Sprintf("max length is %v characters", p.commentLengthMax) return "", backend.PluginError{ - PluginID: comments.PluginID, - ErrorCode: uint32(comments.ErrorCodeMaxLengthExceeded), - ErrorContext: e, + PluginID: comments.PluginID, + ErrorCode: uint32(comments.ErrorCodeMaxLengthExceeded), + ErrorContext: fmt.Sprintf("max length is %v characters", + p.commentLengthMax), } } @@ -571,12 +568,11 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st // Verify the parent ID if e.ParentID != existing.ParentID { - e := fmt.Sprintf("parent id cannot change; got %v, want %v", - e.ParentID, existing.ParentID) return "", backend.PluginError{ - PluginID: comments.PluginID, - ErrorCode: uint32(comments.ErrorCodeParentIDInvalid), - ErrorContext: e, + PluginID: comments.PluginID, + ErrorCode: uint32(comments.ErrorCodeParentIDInvalid), + ErrorContext: fmt.Sprintf("parent id cannot change; got %v, want %v", + e.ParentID, existing.ParentID), } } @@ -660,12 +656,11 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str } } if !bytes.Equal(t, token) { - e := fmt.Sprintf("comment token does not match route token: "+ - "got %x, want %x", t, token) return "", backend.PluginError{ - PluginID: comments.PluginID, - ErrorCode: uint32(comments.ErrorCodeTokenInvalid), - ErrorContext: e, + PluginID: comments.PluginID, + ErrorCode: uint32(comments.ErrorCodeTokenInvalid), + ErrorContext: fmt.Sprintf("comment token does not match "+ + "route token: got %x, want %x", t, token), } } @@ -733,9 +728,8 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str // Update the index cidx, ok := ridx.Comments[d.CommentID] if !ok { - // This should not be possible - e := fmt.Sprintf("comment not found in index: %v", d.CommentID) - panic(e) + // Should not be possible. The cache is not coherent. + panic(fmt.Sprintf("comment not found in index: %v", d.CommentID)) } cidx.Del = digest ridx.Comments[d.CommentID] = cidx @@ -797,12 +791,11 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st } } if !bytes.Equal(t, token) { - e := fmt.Sprintf("comment token does not match route token: "+ - "got %x, want %x", t, token) return "", backend.PluginError{ - PluginID: comments.PluginID, - ErrorCode: uint32(comments.ErrorCodeTokenInvalid), - ErrorContext: e, + PluginID: comments.PluginID, + ErrorCode: uint32(comments.ErrorCodeTokenInvalid), + ErrorContext: fmt.Sprintf("comment token does not match "+ + "route token: got %x, want %x", t, token), } } @@ -1064,12 +1057,11 @@ func (p *commentsPlugin) cmdGetVersion(treeID int64, token []byte, payload strin } digest, ok := cidx.Adds[gv.Version] if !ok { - e := fmt.Sprintf("comment %v does not have version %v", - gv.CommentID, gv.Version) return "", backend.PluginError{ - PluginID: comments.PluginID, - ErrorCode: uint32(comments.ErrorCodeCommentNotFound), - ErrorContext: e, + PluginID: comments.PluginID, + ErrorCode: uint32(comments.ErrorCodeCommentNotFound), + ErrorContext: fmt.Sprintf("comment %v does not have version %v", + gv.CommentID, gv.Version), } } diff --git a/politeiad/backendv2/tstorebe/plugins/comments/recordindex.go b/politeiad/backendv2/tstorebe/plugins/comments/recordindex.go index 3cda339b4..0aff76b6f 100644 --- a/politeiad/backendv2/tstorebe/plugins/comments/recordindex.go +++ b/politeiad/backendv2/tstorebe/plugins/comments/recordindex.go @@ -55,9 +55,9 @@ type recordIndex struct { } // recordIndexPath returns the file path for a cached record index. It accepts -// the full length token or the token prefix, but always uses prefix when -// generating the comments index path string. -func (p *commentsPlugin) recordIndexPath(token []byte, s backend.StateT) string { +// the full length token or the token prefix, but the token prefix is always +// used in the file path string. +func (p *commentsPlugin) recordIndexPath(token []byte, s backend.StateT) (string, error) { var fn string switch s { case backend.StateUnvetted: @@ -65,13 +65,12 @@ func (p *commentsPlugin) recordIndexPath(token []byte, s backend.StateT) string case backend.StateVetted: fn = fnRecordIndexVetted default: - e := fmt.Sprintf("invalid state %x %v", token, s) - panic(e) + return "", fmt.Errorf("invalid state") } tp := util.TokenPrefix(token) fn = strings.Replace(fn, "{tokenPrefix}", tp, 1) - return filepath.Join(p.dataDir, fn) + return filepath.Join(p.dataDir, fn), nil } // recordIndex returns the cached recordIndex for the provided record. If a @@ -79,10 +78,14 @@ func (p *commentsPlugin) recordIndexPath(token []byte, s backend.StateT) string // // This function must be called WITHOUT the read lock held. func (p *commentsPlugin) recordIndex(token []byte, s backend.StateT) (*recordIndex, error) { + fp, err := p.recordIndexPath(token, s) + if err != nil { + return nil, err + } + p.RLock() defer p.RUnlock() - fp := p.recordIndexPath(token, s) b, err := ioutil.ReadFile(fp) if err != nil { var e *os.PathError @@ -112,7 +115,10 @@ func (p *commentsPlugin) _recordIndexSave(token []byte, s backend.StateT, ridx r if err != nil { return err } - fp := p.recordIndexPath(token, s) + fp, err := p.recordIndexPath(token, s) + if err != nil { + return err + } p.Lock() defer p.Unlock() @@ -125,7 +131,7 @@ func (p *commentsPlugin) _recordIndexSave(token []byte, s backend.StateT, ridx r } // recordIndexSave is a wrapper around the _recordIndexSave method that allows -// us to decide how update errors should be handled. For now, we simply panic. +// us to decide how update errors should be handled. For now we just panic. // If an error occurs the cache is no longer coherent and the only way to fix // it is to rebuild it. func (p *commentsPlugin) recordIndexSave(token []byte, s backend.StateT, ridx recordIndex) { diff --git a/politeiad/backendv2/tstorebe/plugins/pi/hooks.go b/politeiad/backendv2/tstorebe/plugins/pi/hooks.go index 832add12b..7658e178d 100644 --- a/politeiad/backendv2/tstorebe/plugins/pi/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/pi/hooks.go @@ -43,6 +43,9 @@ func tokenDecode(token string) ([]byte, error) { func proposalMetadataDecode(files []backend.File) (*pi.ProposalMetadata, error) { var propMD *pi.ProposalMetadata for _, v := range files { + if v.Name != pi.FileNameProposalMetadata { + continue + } if v.Name == pi.FileNameProposalMetadata { b, err := base64.StdEncoding.DecodeString(v.Payload) if err != nil { @@ -66,10 +69,11 @@ func (p *piPlugin) proposalNameIsValid(name string) bool { return p.proposalNameRegexp.MatchString(name) } -// proposalFilesVerify verifies the files adhere to all plugin setting +// proposalFilesVerify verifies the files adhere to all pi plugin setting // requirements. If this hook is being executed then the files have already -// passed politeia validation so we can assume that the file has a unique name, -// a valid base64 payload, and that the file digest and MIME type are correct. +// passed politeiad validation so we can assume that the file has a unique +// name, a valid base64 payload, and that the file digest and MIME type are +// correct. func (p *piPlugin) proposalFilesVerify(files []backend.File) error { var imagesCount uint32 for _, v := range files { @@ -93,12 +97,11 @@ func (p *piPlugin) proposalFilesVerify(files []backend.File) error { // Verify text file size if len(payload) > int(p.textFileSizeMax) { - e := fmt.Sprintf("file %v size %v exceeds max size %v", - v.Name, len(payload), p.textFileSizeMax) return backend.PluginError{ - PluginID: pi.PluginID, - ErrorCode: uint32(pi.ErrorCodeTextFileSizeInvalid), - ErrorContext: e, + PluginID: pi.PluginID, + ErrorCode: uint32(pi.ErrorCodeTextFileSizeInvalid), + ErrorContext: fmt.Sprintf("file %v size %v exceeds max size %v", + v.Name, len(payload), p.textFileSizeMax), } } @@ -107,12 +110,11 @@ func (p *piPlugin) proposalFilesVerify(files []backend.File) error { // Verify image file size if len(payload) > int(p.imageFileSizeMax) { - e := fmt.Sprintf("image %v size %v exceeds max size %v", - v.Name, len(payload), p.imageFileSizeMax) return backend.PluginError{ - PluginID: pi.PluginID, - ErrorCode: uint32(pi.ErrorCodeImageFileSizeInvalid), - ErrorContext: e, + PluginID: pi.PluginID, + ErrorCode: uint32(pi.ErrorCodeImageFileSizeInvalid), + ErrorContext: fmt.Sprintf("image %v size %v exceeds max size %v", + v.Name, len(payload), p.imageFileSizeMax), } } @@ -139,12 +141,11 @@ func (p *piPlugin) proposalFilesVerify(files []backend.File) error { // Verify image file count is acceptable if imagesCount > p.imageFileCountMax { - e := fmt.Sprintf("got %v image files, max is %v", - imagesCount, p.imageFileCountMax) return backend.PluginError{ - PluginID: pi.PluginID, - ErrorCode: uint32(pi.ErrorCodeImageFileCountInvalid), - ErrorContext: e, + PluginID: pi.PluginID, + ErrorCode: uint32(pi.ErrorCodeImageFileCountInvalid), + ErrorContext: fmt.Sprintf("got %v image files, max is %v", + imagesCount, p.imageFileCountMax), } } @@ -207,12 +208,11 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { return err } if s.Status != ticketvote.VoteStatusUnauthorized { - e := fmt.Sprintf("vote status '%v' does not allow for proposal edits", - ticketvote.VoteStatuses[s.Status]) return backend.PluginError{ - PluginID: pi.PluginID, - ErrorCode: uint32(pi.ErrorCodeVoteStatusInvalid), - ErrorContext: e, + PluginID: pi.PluginID, + ErrorCode: uint32(pi.ErrorCodeVoteStatusInvalid), + ErrorContext: fmt.Sprintf("vote status '%v' does not allow "+ + "for proposal edits", ticketvote.VoteStatuses[s.Status]), } } } diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go index c52b3af0a..26540550b 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go @@ -61,24 +61,6 @@ func tokenDecodeAnyLength(token string) ([]byte, error) { return util.TokenDecodeAnyLength(util.TokenTypeTstore, token) } -func convertSignatureError(err error) backend.PluginError { - var e util.SignatureError - var s ticketvote.ErrorCodeT - if errors.As(err, &e) { - switch e.ErrorCode { - case util.ErrorStatusPublicKeyInvalid: - s = ticketvote.ErrorCodePublicKeyInvalid - case util.ErrorStatusSignatureInvalid: - s = ticketvote.ErrorCodeSignatureInvalid - } - } - return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(s), - ErrorContext: e.ErrorContext, - } -} - func (p *ticketVotePlugin) authSave(treeID int64, ad ticketvote.AuthDetails) error { // Prepare blob be, err := convertBlobEntryFromAuthDetails(ad) @@ -897,12 +879,11 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri } } if !bytes.Equal(t, token) { - e := fmt.Sprintf("plugin token does not match route token: "+ - "got %x, want %x", t, token) return "", backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeTokenInvalid), - ErrorContext: e, + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeTokenInvalid), + ErrorContext: fmt.Sprintf("plugin token does not match "+ + "route token: got %x, want %x", t, token), } } @@ -921,11 +902,10 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri case ticketvote.AuthActionRevoke: // This is allowed default: - e := fmt.Sprintf("%v not a valid action", a.Action) return "", backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeAuthorizationInvalid), - ErrorContext: e, + ErrorContext: fmt.Sprintf("%v not a valid action", a.Action), } } @@ -942,12 +922,11 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri } } if a.Version != r.RecordMetadata.Version { - e := fmt.Sprintf("version is not latest: got %v, want %v", - a.Version, r.RecordMetadata.Version) return "", backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeRecordVersionInvalid), - ErrorContext: e, + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeRecordVersionInvalid), + ErrorContext: fmt.Sprintf("version is not latest: got %v, want %v", + a.Version, r.RecordMetadata.Version), } } @@ -1074,36 +1053,32 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM // Verify vote params switch { case vote.Duration > voteDurationMax: - e := fmt.Sprintf("duration %v exceeds max duration %v", - vote.Duration, voteDurationMax) return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteDurationInvalid), - ErrorContext: e, + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteDurationInvalid), + ErrorContext: fmt.Sprintf("duration %v exceeds max duration %v", + vote.Duration, voteDurationMax), } case vote.Duration < voteDurationMin: - e := fmt.Sprintf("duration %v under min duration %v", - vote.Duration, voteDurationMin) return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteDurationInvalid), - ErrorContext: e, + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteDurationInvalid), + ErrorContext: fmt.Sprintf("duration %v under min duration %v", + vote.Duration, voteDurationMin), } case vote.QuorumPercentage > 100: - e := fmt.Sprintf("quorum percent %v exceeds 100 percent", - vote.QuorumPercentage) return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteQuorumInvalid), - ErrorContext: e, + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteQuorumInvalid), + ErrorContext: fmt.Sprintf("quorum percent %v exceeds 100 percent", + vote.QuorumPercentage), } case vote.PassPercentage > 100: - e := fmt.Sprintf("pass percent %v exceeds 100 percent", - vote.PassPercentage) return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVotePassRateInvalid), - ErrorContext: e, + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVotePassRateInvalid), + ErrorContext: fmt.Sprintf("pass percent %v exceeds 100 percent", + vote.PassPercentage), } } @@ -1122,12 +1097,11 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM // that the only options present are approve/reject and that they // use the vote option IDs specified by the ticketvote API. if len(vote.Options) != 2 { - e := fmt.Sprintf("vote options count got %v, want 2", - len(vote.Options)) return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteOptionsInvalid), - ErrorContext: e, + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteOptionsInvalid), + ErrorContext: fmt.Sprintf("vote options count got %v, want 2", + len(vote.Options)), } } // map[optionID]found @@ -1151,12 +1125,11 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM } } if len(missing) > 0 { - e := fmt.Sprintf("vote option IDs not found: %v", - strings.Join(missing, ",")) return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteOptionsInvalid), - ErrorContext: e, + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteOptionsInvalid), + ErrorContext: fmt.Sprintf("vote option IDs not found: %v", + strings.Join(missing, ",")), } } } @@ -1176,20 +1149,18 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM // Verify parent token switch { case vote.Type == ticketvote.VoteTypeStandard && vote.Parent != "": - e := "parent token should not be provided for a standard vote" return backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), - ErrorContext: e, + ErrorContext: "parent token should not be provided for a standard vote", } case vote.Type == ticketvote.VoteTypeRunoff: _, err := tokenDecode(vote.Parent) if err != nil { - e := fmt.Sprintf("invalid parent %v", vote.Parent) return backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), - ErrorContext: e, + ErrorContext: fmt.Sprintf("invalid parent %v", vote.Parent), } } } @@ -1218,12 +1189,11 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot } } if !bytes.Equal(t, token) { - e := fmt.Sprintf("plugin payload token does not match route token: "+ - "got %x, want %x", t, token) return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeTokenInvalid), - ErrorContext: e, + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeTokenInvalid), + ErrorContext: fmt.Sprintf("plugin payload token does not match "+ + "route token: got %x, want %x", t, token), } } @@ -1260,12 +1230,11 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot return nil, fmt.Errorf("record is unvetted") } if sd.Params.Version != r.RecordMetadata.Version { - e := fmt.Sprintf("version is not latest: got %v, want %v", - sd.Params.Version, r.RecordMetadata.Version) return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeRecordVersionInvalid), - ErrorContext: e, + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeRecordVersionInvalid), + ErrorContext: fmt.Sprintf("version is not latest: got %v, want %v", + sd.Params.Version, r.RecordMetadata.Version), } } @@ -1484,12 +1453,11 @@ func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs sta return fmt.Errorf("record is unvetted") } if sd.Params.Version != r.RecordMetadata.Version { - e := fmt.Sprintf("version is not latest %v: got %v, want %v", - sd.Params.Token, sd.Params.Version, r.RecordMetadata.Version) return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeRecordVersionInvalid), - ErrorContext: e, + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeRecordVersionInvalid), + ErrorContext: fmt.Sprintf("version is not latest %v: got %v, want %v", + sd.Params.Token, sd.Params.Version, r.RecordMetadata.Version), } } @@ -1553,11 +1521,10 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti r, err := p.tstore.RecordPartial(treeID, 0, files, false) if err != nil { if errors.Is(err, backend.ErrRecordNotFound) { - e := fmt.Sprintf("parent record not found %x", token) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), - ErrorContext: e, + ErrorContext: fmt.Sprintf("parent record not found %x", token), } } return nil, fmt.Errorf("RecordPartial: %v", err) @@ -1571,23 +1538,21 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti return nil, err } if vm == nil || vm.LinkBy == 0 { - e := fmt.Sprintf("%x is not a runoff vote parent", token) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), - ErrorContext: e, + ErrorContext: fmt.Sprintf("%x is not a runoff vote parent", token), } } isExpired := vm.LinkBy < time.Now().Unix() isMainNet := p.activeNetParams.Name == chaincfg.MainNetParams().Name switch { case !isExpired && isMainNet: - e := fmt.Sprintf("parent record %x linkby deadline not met %v", - token, vm.LinkBy) return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeLinkByNotExpired), - ErrorContext: e, + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeLinkByNotExpired), + ErrorContext: fmt.Sprintf("parent record %x linkby deadline not met %v", + token, vm.LinkBy), } case !isExpired: // Allow the vote to be started before the link by deadline @@ -1632,12 +1597,11 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti _, ok := expected[v.Params.Token] if !ok { // This submission should not be here - e := fmt.Sprintf("record %v should not be included", - v.Params.Token) return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeStartDetailsInvalid), - ErrorContext: e, + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeStartDetailsInvalid), + ErrorContext: fmt.Sprintf("record %v should not be included", + v.Params.Token), } } } @@ -1706,52 +1670,46 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. // Verify vote params are the same for all submissions switch { case v.Params.Type != ticketvote.VoteTypeRunoff: - e := fmt.Sprintf("%v got %v, want %v", - v.Params.Token, v.Params.Type, ticketvote.VoteTypeRunoff) return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteTypeInvalid), - ErrorContext: e, + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteTypeInvalid), + ErrorContext: fmt.Sprintf("%v got %v, want %v", + v.Params.Token, v.Params.Type, ticketvote.VoteTypeRunoff), } case v.Params.Mask != mask: - e := fmt.Sprintf("%v mask invalid: all must be the same", - v.Params.Token) return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteBitsInvalid), - ErrorContext: e, + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteBitsInvalid), + ErrorContext: fmt.Sprintf("%v mask invalid: all must be the same", + v.Params.Token), } case v.Params.Duration != duration: - e := fmt.Sprintf("%v duration does not match; all must be the same", - v.Params.Token) return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteDurationInvalid), - ErrorContext: e, + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteDurationInvalid), + ErrorContext: fmt.Sprintf("%v duration does not match; "+ + "all must be the same", v.Params.Token), } case v.Params.QuorumPercentage != quorum: - e := fmt.Sprintf("%v quorum does not match; all must be the same", - v.Params.Token) return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteQuorumInvalid), - ErrorContext: e, + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteQuorumInvalid), + ErrorContext: fmt.Sprintf("%v quorum does not match; "+ + "all must be the same", v.Params.Token), } case v.Params.PassPercentage != pass: - e := fmt.Sprintf("%v pass rate does not match; all must be the same", - v.Params.Token) return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVotePassRateInvalid), - ErrorContext: e, + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVotePassRateInvalid), + ErrorContext: fmt.Sprintf("%v pass rate does not match; "+ + "all must be the same", v.Params.Token), } case v.Params.Parent != parent: - e := fmt.Sprintf("%v parent does not match; all must be the same", - v.Params.Token) return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), - ErrorContext: e, + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), + ErrorContext: fmt.Sprintf("%v parent does not match; "+ + "all must be the same", v.Params.Token), } } @@ -1768,11 +1726,10 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. // Verify parent token _, err = tokenDecode(v.Params.Parent) if err != nil { - e := fmt.Sprintf("parent token %v", v.Params.Parent) return nil, backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeTokenInvalid), - ErrorContext: e, + ErrorContext: fmt.Sprintf("parent token %v", v.Params.Parent), } } @@ -1797,12 +1754,11 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. // Verify plugin command is being executed on the parent record if hex.EncodeToString(token) != parent { - e := fmt.Sprintf("runoff vote must be started on parent record %v", - parent) return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), - ErrorContext: e, + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), + ErrorContext: fmt.Sprintf("runoff vote must be started on "+ + "the parent record %v", parent), } } @@ -2726,6 +2682,24 @@ func (p *ticketVotePlugin) cmdSubmissions(token []byte) (string, error) { return string(reply), nil } +func convertSignatureError(err error) backend.PluginError { + var e util.SignatureError + var s ticketvote.ErrorCodeT + if errors.As(err, &e) { + switch e.ErrorCode { + case util.ErrorStatusPublicKeyInvalid: + s = ticketvote.ErrorCodePublicKeyInvalid + case util.ErrorStatusSignatureInvalid: + s = ticketvote.ErrorCodeSignatureInvalid + } + } + return backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(s), + ErrorContext: e.ErrorContext, + } +} + func convertAuthDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.AuthDetails, error) { // Decode and validate data hint b, err := base64.StdEncoding.DecodeString(be.DataHint) diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go index c78a33de1..b272e7349 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go @@ -20,19 +20,20 @@ import ( func voteMetadataDecode(files []backend.File) (*ticketvote.VoteMetadata, error) { var voteMD *ticketvote.VoteMetadata for _, v := range files { - if v.Name == ticketvote.FileNameVoteMetadata { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return nil, err - } - var m ticketvote.VoteMetadata - err = json.Unmarshal(b, &m) - if err != nil { - return nil, err - } - voteMD = &m - break + if v.Name != ticketvote.FileNameVoteMetadata { + continue + } + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return nil, err + } + var m ticketvote.VoteMetadata + err = json.Unmarshal(b, &m) + if err != nil { + return nil, err } + voteMD = &m + break } return voteMD, nil } From a41e17155e5c2854dbc5ec88094dc8d795fe8d3f Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 19 Mar 2021 09:33:35 -0500 Subject: [PATCH 415/449] tstorbe/store: Fix leveldb deadlock. --- .../tstorebe/store/localdb/encrypt.go | 40 --------------- .../tstorebe/store/localdb/localdb.go | 51 ++++++++++++++++++- .../backendv2/tstorebe/store/mysql/encrypt.go | 8 ++- .../backendv2/tstorebe/store/mysql/mysql.go | 47 ++++++++++++----- politeiad/backendv2/tstorebe/store/store.go | 5 ++ politeiad/backendv2/tstorebe/tstore/tstore.go | 2 +- politeiad/log.go | 12 +++-- 7 files changed, 104 insertions(+), 61 deletions(-) delete mode 100644 politeiad/backendv2/tstorebe/store/localdb/encrypt.go diff --git a/politeiad/backendv2/tstorebe/store/localdb/encrypt.go b/politeiad/backendv2/tstorebe/store/localdb/encrypt.go deleted file mode 100644 index 1eac4e5d8..000000000 --- a/politeiad/backendv2/tstorebe/store/localdb/encrypt.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) 2020-2021 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package localdb - -import ( - "bytes" - - "github.com/decred/politeia/util" - "github.com/marcopeereboom/sbox" -) - -func (l *localdb) encrypt(data []byte) ([]byte, error) { - l.RLock() - defer l.RUnlock() - - return sbox.Encrypt(0, l.key, data) -} - -func (l *localdb) decrypt(data []byte) ([]byte, uint32, error) { - l.RLock() - defer l.RUnlock() - - return sbox.Decrypt(l.key, data) -} - -func (l *localdb) zeroKey() { - l.Lock() - defer l.Unlock() - - util.Zero(l.key[:]) - l.key = nil -} - -// isEncrypted returns whether the provided blob has been prefixed with an sbox -// header, indicating that it is an encrypted blob. -func isEncrypted(b []byte) bool { - return bytes.HasPrefix(b, []byte("sbox")) -} diff --git a/politeiad/backendv2/tstorebe/store/localdb/localdb.go b/politeiad/backendv2/tstorebe/store/localdb/localdb.go index f1110e68f..53429df8f 100644 --- a/politeiad/backendv2/tstorebe/store/localdb/localdb.go +++ b/politeiad/backendv2/tstorebe/store/localdb/localdb.go @@ -5,6 +5,7 @@ package localdb import ( + "bytes" "errors" "fmt" "path/filepath" @@ -12,6 +13,7 @@ import ( "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" "github.com/decred/politeia/util" + "github.com/marcopeereboom/sbox" "github.com/syndtr/goleveldb/leveldb" ) @@ -42,6 +44,27 @@ type localdb struct { key *[32]byte } +func (l *localdb) isShutdown() bool { + l.RLock() + defer l.RUnlock() + + return l.shutdown +} + +func (l *localdb) encrypt(data []byte) ([]byte, error) { + l.RLock() + defer l.RUnlock() + + return sbox.Encrypt(0, l.key, data) +} + +func (l *localdb) decrypt(data []byte) ([]byte, uint32, error) { + l.RLock() + defer l.RUnlock() + + return sbox.Decrypt(l.key, data) +} + // Put saves the provided key-value pairs to the store. This operation is // performed atomically. // @@ -49,6 +72,10 @@ type localdb struct { func (l *localdb) Put(blobs map[string][]byte, encrypt bool) error { log.Tracef("Put: %v blobs", len(blobs)) + if l.isShutdown() { + return store.ErrShutdown + } + // Encrypt blobs if encrypt { for k, v := range blobs { @@ -84,6 +111,10 @@ func (l *localdb) Put(blobs map[string][]byte, encrypt bool) error { func (l *localdb) Del(keys []string) error { log.Tracef("Del: %v", keys) + if l.isShutdown() { + return store.ErrShutdown + } + batch := new(leveldb.Batch) for _, v := range keys { batch.Delete([]byte(v)) @@ -98,6 +129,12 @@ func (l *localdb) Del(keys []string) error { return nil } +// isEncrypted returns whether the provided blob has been prefixed with an sbox +// header, indicating that it is an encrypted blob. +func isEncrypted(b []byte) bool { + return bytes.HasPrefix(b, []byte("sbox")) +} + // Get returns blobs from the store for the provided keys. An entry will not // exist in the returned map if for any blobs that are not found. It is the // responsibility of the caller to ensure a blob was returned for all provided @@ -107,6 +144,10 @@ func (l *localdb) Del(keys []string) error { func (l *localdb) Get(keys []string) (map[string][]byte, error) { log.Tracef("Get: %v", keys) + if l.isShutdown() { + return nil, store.ErrShutdown + } + // Lookup blobs blobs := make(map[string][]byte, len(keys)) for _, v := range keys { @@ -142,10 +183,18 @@ func (l *localdb) Get(keys []string) (map[string][]byte, error) { // // This function satisfies the store BlobKV interface. func (l *localdb) Close() { + log.Tracef("Close") + l.Lock() defer l.Unlock() - l.zeroKey() + l.shutdown = true + + // Zero the encryption key + util.Zero(l.key[:]) + l.key = nil + + // Close database l.db.Close() } diff --git a/politeiad/backendv2/tstorebe/store/mysql/encrypt.go b/politeiad/backendv2/tstorebe/store/mysql/encrypt.go index 1c5ed9eb2..b0a5e45f9 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/encrypt.go +++ b/politeiad/backendv2/tstorebe/store/mysql/encrypt.go @@ -38,6 +38,8 @@ type argon2idParams struct { // saved to the kv store. Subsequent calls to this fuction will pull the // existing salt and params from the kv store and use them to derive the key. func (s *mysql) argon2idKey(password string) (*[32]byte, error) { + log.Infof("Deriving encryption key from password") + // Check if a key already exists blobs, err := s.Get([]string{argon2idKey}) if err != nil { @@ -48,7 +50,7 @@ func (s *mysql) argon2idKey(password string) (*[32]byte, error) { b, ok := blobs[argon2idKey] if ok { // Key already exists. Use the existing salt. - log.Debugf("Existing salt found") + log.Infof("Encryption key salt already exists") var ap argon2idParams err = json.Unmarshal(b, &ap) @@ -60,7 +62,7 @@ func (s *mysql) argon2idKey(password string) (*[32]byte, error) { wasFound = true } else { // Key does not exist. Create a random 16 byte salt. - log.Debugf("Salt not found; creating a new one") + log.Infof("Encryption key salt not found; creating a new one") salt, err = util.Random(16) if err != nil { @@ -102,6 +104,8 @@ func (s *mysql) argon2idKey(password string) (*[32]byte, error) { if err != nil { return nil, fmt.Errorf("put: %v", err) } + + log.Infof("Encryption key derivation params saved to kv store") } return &key, nil diff --git a/politeiad/backendv2/tstorebe/store/mysql/mysql.go b/politeiad/backendv2/tstorebe/store/mysql/mysql.go index 8db58bef6..0f84b5abc 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/mysql.go +++ b/politeiad/backendv2/tstorebe/store/mysql/mysql.go @@ -47,7 +47,8 @@ var ( // mysql implements the store BlobKV interface using a mysql driver. type mysql struct { sync.RWMutex - db *sql.DB + db *sql.DB + shutdown bool // Encryption key. The key is zero'd out on application exit so the // read lock must be held during concurrent access to prevent the @@ -59,6 +60,13 @@ func ctxWithTimeout() (context.Context, func()) { return context.WithTimeout(context.Background(), connTimeout) } +func (s *mysql) isShutdown() bool { + s.RLock() + defer s.RUnlock() + + return s.shutdown +} + func (s *mysql) getKey() (*[32]byte, error) { s.RLock() defer s.RUnlock() @@ -77,16 +85,6 @@ func (s *mysql) setKey(key *[32]byte) { s.key = key } -func (s *mysql) zeroKey() { - s.Lock() - defer s.Unlock() - - if s.key != nil { - util.Zero(s.key[:]) - s.key = nil - } -} - func (s *mysql) put(blobs map[string][]byte, encrypt bool, ctx context.Context, tx *sql.Tx) error { // Encrypt blobs if encrypt { @@ -122,6 +120,10 @@ func (s *mysql) put(blobs map[string][]byte, encrypt bool, ctx context.Context, func (s *mysql) Put(blobs map[string][]byte, encrypt bool) error { log.Tracef("Put: %v blobs", len(blobs)) + if s.isShutdown() { + return store.ErrShutdown + } + ctx, cancel := ctxWithTimeout() defer cancel() @@ -164,6 +166,10 @@ func (s *mysql) Put(blobs map[string][]byte, encrypt bool) error { func (s *mysql) Del(keys []string) error { log.Tracef("Del: %v", keys) + if s.isShutdown() { + return store.ErrShutdown + } + ctx, cancel := ctxWithTimeout() defer cancel() @@ -210,6 +216,10 @@ func (s *mysql) Del(keys []string) error { func (s *mysql) Get(keys []string) (map[string][]byte, error) { log.Tracef("Get: %v", keys) + if s.isShutdown() { + return nil, store.ErrShutdown + } + ctx, cancel := ctxWithTimeout() defer cancel() @@ -281,7 +291,20 @@ func (s *mysql) Get(keys []string) (map[string][]byte, error) { // Closes closes the blob store connection. func (s *mysql) Close() { - s.zeroKey() + log.Tracef("Close") + + s.Lock() + defer s.Unlock() + + s.shutdown = true + + // Zero the encryption key + if s.key != nil { + util.Zero(s.key[:]) + s.key = nil + } + + // Close mysql connection s.db.Close() } diff --git a/politeiad/backendv2/tstorebe/store/store.go b/politeiad/backendv2/tstorebe/store/store.go index 3a5b51ad0..1a530c78a 100644 --- a/politeiad/backendv2/tstorebe/store/store.go +++ b/politeiad/backendv2/tstorebe/store/store.go @@ -10,10 +10,15 @@ import ( "encoding/base64" "encoding/gob" "encoding/hex" + "errors" "github.com/decred/politeia/util" ) +var ( + ErrShutdown = errors.New("store is shutdown") +) + const ( // DataTypeStructure is used as the data descriptor type when the // blob entry contains a structure. diff --git a/politeiad/backendv2/tstorebe/tstore/tstore.go b/politeiad/backendv2/tstorebe/tstore/tstore.go index 522d5d3e6..ce95419e9 100644 --- a/politeiad/backendv2/tstorebe/tstore/tstore.go +++ b/politeiad/backendv2/tstorebe/tstore/tstore.go @@ -1160,8 +1160,8 @@ func (t *Tstore) Close() { log.Tracef("Close") // Close connections - t.store.Close() t.tlog.Close() + t.store.Close() } func New(appDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianSigningKeyFile, dbType, dbHost, dbPass, dcrtimeHost, dcrtimeCert string) (*Tstore, error) { diff --git a/politeiad/log.go b/politeiad/log.go index 498760680..90ae4cf0e 100644 --- a/politeiad/log.go +++ b/politeiad/log.go @@ -54,20 +54,21 @@ var ( gitbeLog = backendLog.Logger("GITB") tstorebeLog = backendLog.Logger("BACK") tstoreLog = backendLog.Logger("TSTR") + kvstoreLog = backendLog.Logger("STOR") wsdcrdataLog = backendLog.Logger("WSDD") pluginLog = backendLog.Logger("PLUG") ) // Initialize package-global logger variables. func init() { - // Backend loggers + // Git backend loggers gitbe.UseLogger(gitbeLog) - tstorebe.UseLogger(tstorebeLog) - // Tstore loggers + // Tstore backend loggers + tstorebe.UseLogger(tstorebeLog) tstore.UseLogger(tstoreLog) - localdb.UseLogger(tstoreLog) - mysql.UseLogger(tstoreLog) + localdb.UseLogger(kvstoreLog) + mysql.UseLogger(kvstoreLog) // Plugin loggers comments.UseLogger(pluginLog) @@ -85,6 +86,7 @@ var subsystemLoggers = map[string]slog.Logger{ "GITB": gitbeLog, "BACK": tstorebeLog, "TSTR": tstoreLog, + "STOR": kvstoreLog, "WSDD": wsdcrdataLog, "PLUG": pluginLog, } From d303fb7784ff76d2af8df1ecb30851909781b5dd Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 19 Mar 2021 15:02:16 -0500 Subject: [PATCH 416/449] multi: Refactor token API. --- politeiad/api/v2/v2.go | 4 +- .../tstorebe/plugins/comments/recordindex.go | 13 +- .../plugins/ticketvote/submissions.go | 29 ++-- .../tstorebe/plugins/ticketvote/summary.go | 7 +- politeiad/backendv2/tstorebe/testing.go | 2 +- politeiad/backendv2/tstorebe/tstorebe.go | 158 ++++++++++++------ util/token.go | 96 +++++++---- 7 files changed, 206 insertions(+), 103 deletions(-) diff --git a/politeiad/api/v2/v2.go b/politeiad/api/v2/v2.go index 1dde91deb..cff7fe15d 100644 --- a/politeiad/api/v2/v2.go +++ b/politeiad/api/v2/v2.go @@ -196,12 +196,12 @@ const ( // TokenSize is the size of a censorship record token in bytes. TokenSize = 8 - // TokenSizeShort is the size, in characters, of a hex encoded + // ShortTokenLength is the length, in characters, of a hex encoded // token that has been shortened to improved UX. Short tokens can // be used to retrieve record data but cannot be used on any routes // that write record data. 7 characters was chosen to match the git // abbreviated commitment hash size. - TokenSizeShort = 7 + ShortTokenLength = 7 ) // CensorshipRecord contains cryptographic proof that a record was accepted for diff --git a/politeiad/backendv2/tstorebe/plugins/comments/recordindex.go b/politeiad/backendv2/tstorebe/plugins/comments/recordindex.go index 0aff76b6f..92a21e6de 100644 --- a/politeiad/backendv2/tstorebe/plugins/comments/recordindex.go +++ b/politeiad/backendv2/tstorebe/plugins/comments/recordindex.go @@ -21,8 +21,8 @@ import ( const ( // Filenames of the record indexes that are saved to the comments // plugin data dir. - fnRecordIndexUnvetted = "{tokenPrefix}-index-unvetted.json" - fnRecordIndexVetted = "{tokenPrefix}-index-vetted.json" + fnRecordIndexUnvetted = "{shorttoken}-index-unvetted.json" + fnRecordIndexVetted = "{shorttoken}-index-vetted.json" ) // voteIndex contains the comment vote and the digest of the vote record. @@ -55,7 +55,7 @@ type recordIndex struct { } // recordIndexPath returns the file path for a cached record index. It accepts -// the full length token or the token prefix, but the token prefix is always +// both the full length token or the short token, but the short token is always // used in the file path string. func (p *commentsPlugin) recordIndexPath(token []byte, s backend.StateT) (string, error) { var fn string @@ -68,8 +68,11 @@ func (p *commentsPlugin) recordIndexPath(token []byte, s backend.StateT) (string return "", fmt.Errorf("invalid state") } - tp := util.TokenPrefix(token) - fn = strings.Replace(fn, "{tokenPrefix}", tp, 1) + t, err := util.ShortTokenEncode(token) + if err != nil { + return "", err + } + fn = strings.Replace(fn, "{shorttoken}", t, 1) return filepath.Join(p.dataDir, fn), nil } diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/submissions.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/submissions.go index b20152d4a..c2d2ffebf 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/submissions.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/submissions.go @@ -18,7 +18,7 @@ import ( const ( // fnSubmissions is the filename for the cached submissions data // that is saved to the plugin data dir. - fnSubmissions = "{tokenprefix}-submissions.json" + fnSubmissions = "{shorttoken}-submissions.json" ) // submissions is the the structure that is updated and cached for record A @@ -34,13 +34,16 @@ type submissions struct { } // submissionsCachePath returns the path to the submissions list for the -// provided record token. The token prefix is used in the file path so that the -// submissions list can be retrieved using either the full token or the token -// prefix. -func (p *ticketVotePlugin) submissionsCachePath(token []byte) string { - t := util.TokenPrefix(token) - fn := strings.Replace(fnSubmissions, "{tokenprefix}", t, 1) - return filepath.Join(p.dataDir, fn) +// provided record token. The short token is used in the file path so that the +// submissions list can be retrieved using either the full token or the short +// token. +func (p *ticketVotePlugin) submissionsCachePath(token []byte) (string, error) { + t, err := util.ShortTokenEncode(token) + if err != nil { + return "", err + } + fn := strings.Replace(fnSubmissions, "{shorttoken}", t, 1) + return filepath.Join(p.dataDir, fn), nil } // submissionsCacheWithLock return the submissions list for a record token. If @@ -49,7 +52,10 @@ func (p *ticketVotePlugin) submissionsCachePath(token []byte) string { // // This function must be called WITH the lock held. func (p *ticketVotePlugin) submissionsCacheWithLock(token []byte) (*submissions, error) { - fp := p.submissionsCachePath(token) + fp, err := p.submissionsCachePath(token) + if err != nil { + return nil, err + } b, err := ioutil.ReadFile(fp) if err != nil { var e *os.PathError @@ -89,7 +95,10 @@ func (p *ticketVotePlugin) submissionsCacheSaveWithLock(token []byte, s submissi if err != nil { return err } - fp := p.submissionsCachePath(token) + fp, err := p.submissionsCachePath(token) + if err != nil { + return err + } return ioutil.WriteFile(fp, b, 0664) } diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/summary.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/summary.go index 7aa675cf6..56929a6d6 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/summary.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/summary.go @@ -19,19 +19,18 @@ import ( const ( // filenameSummary is the file name of the vote summary for a // record. These summaries are cached in the plugin data dir. - filenameSummary = "{tokenprefix}-summary.json" + filenameSummary = "{shorttoken}-summary.json" ) // summaryCachePath accepts both full tokens and token prefixes, however it // always uses the token prefix when generatig the path. func (p *ticketVotePlugin) summaryCachePath(token string) (string, error) { // Use token prefix - t, err := tokenDecodeAnyLength(token) + stoken, err := util.ShortTokenString(token) if err != nil { return "", err } - token = util.TokenPrefix(t) - fn := strings.Replace(filenameSummary, "{tokenprefix}", token, 1) + fn := strings.Replace(filenameSummary, "{shorttoken}", stoken, 1) return filepath.Join(p.dataDir, fn), nil } diff --git a/politeiad/backendv2/tstorebe/testing.go b/politeiad/backendv2/tstorebe/testing.go index 0acfe031a..a4c092d6f 100644 --- a/politeiad/backendv2/tstorebe/testing.go +++ b/politeiad/backendv2/tstorebe/testing.go @@ -30,7 +30,7 @@ func NewTestTstoreBackend(t *testing.T) (*tstoreBackend, func()) { appDir: appDir, dataDir: dataDir, tstore: tstore.NewTestTstore(t, dataDir), - prefixes: make(map[string][]byte), + tokens: make(map[string][]byte), recordMtxs: make(map[string]*sync.Mutex), } diff --git a/politeiad/backendv2/tstorebe/tstorebe.go b/politeiad/backendv2/tstorebe/tstorebe.go index 367186aa5..3fdd75f23 100644 --- a/politeiad/backendv2/tstorebe/tstorebe.go +++ b/politeiad/backendv2/tstorebe/tstorebe.go @@ -37,14 +37,14 @@ type tstoreBackend struct { shutdown bool tstore *tstore.Tstore - // prefixes contains the prefix to full token mapping for all - // records. The prefix is the first n characters of the hex encoded - // record token, where n is defined by the token prefix length - // politeiad setting. Record lookups by token prefix are allowed. - // This cache is used to prevent prefix collisions when creating - // new tokens and to facilitate lookups by token prefix. This cache + // tokens contains the short token to full token mappings. The + // short token is the first n characters of the hex encoded record + // token, where n is defined by the short token length politeiad + // setting. Record lookups using short tokens are allowed. This + // cache is used to prevent collisions when creating new tokens + // and to facilitate lookups using only the short token. This cache // is built on startup. - prefixes map[string][]byte // [tokenPrefix]token + tokens map[string][]byte // [shortToken]fullToken // recordMtxs allows the backend to hold a lock on an individual // record so that it can perform multiple read/write operations @@ -76,34 +76,72 @@ func (t *tstoreBackend) recordMutex(token []byte) *sync.Mutex { return m } -func (t *tstoreBackend) prefixExists(fullToken []byte) bool { +// tokenCollision returns whether the short version of the provided token +// already exists. This can be used to prevent collisions when creating new +// tokens. +func (t *tstoreBackend) tokenCollision(fullToken []byte) bool { + shortToken, err := util.ShortTokenEncode(fullToken) + if err != nil { + return false + } + t.RLock() defer t.RUnlock() - _, ok := t.prefixes[util.TokenPrefix(fullToken)] + _, ok := t.tokens[shortToken] return ok } -func (t *tstoreBackend) prefixAdd(fullToken []byte) { +// tokenAdd adds a entry to the tokens cache. +func (t *tstoreBackend) tokenAdd(fullToken []byte) error { + if !tokenIsFullLength(fullToken) { + return fmt.Errorf("token is not full length") + } + + shortToken, err := util.ShortTokenEncode(fullToken) + if err != nil { + return err + } + t.Lock() - defer t.Unlock() + t.tokens[shortToken] = fullToken + t.Unlock() - prefix := util.TokenPrefix(fullToken) - t.prefixes[prefix] = fullToken + log.Tracef("Token cache add: %v", shortToken) - log.Tracef("Add token prefix: %v", prefix) + return nil } -func (t *tstoreBackend) fullLengthToken(token []byte) []byte { +// fullLengthToken returns the full length token given the short token. A +// ErrRecordNotFound error is returned if a record does not exist for the +// provided token. +func (t *tstoreBackend) fullLengthToken(token []byte) ([]byte, error) { + if tokenIsFullLength(token) { + // Token is already full length. Nothing else to do. + return token, nil + } + + shortToken, err := util.ShortTokenEncode(token) + if err != nil { + // Token was not large enough to be a short token. This cannot + // be used to lookup a record. + return nil, backend.ErrRecordNotFound + } + t.RLock() defer t.RUnlock() - fullToken, ok := t.prefixes[util.TokenPrefix(token)] + fullToken, ok := t.tokens[shortToken] if !ok { - return token + // Short token does not correspond to a record token + return nil, backend.ErrRecordNotFound } - return fullToken + return fullToken, nil +} + +func tokenIsFullLength(token []byte) bool { + return util.TokenIsFullLength(util.TokenTypeTstore, token) } func tokenFromTreeID(treeID int64) []byte { @@ -114,10 +152,6 @@ func tokenFromTreeID(treeID int64) []byte { return b } -func tokenIsFullLength(token []byte) bool { - return util.TokenIsFullLength(util.TokenTypeTstore, token) -} - func treeIDFromToken(token []byte) int64 { if !tokenIsFullLength(token) { return 0 @@ -428,20 +462,18 @@ func (t *tstoreBackend) RecordNew(metadata []backend.MetadataStream, files []bac } token = tokenFromTreeID(treeID) - // Check for token prefix collisions - if !t.prefixExists(token) { - // Not a collision. Use this token. - - // Update the prefix cache. This must be done even if the - // record creation fails since the tree will still exist in - // tstore. - t.prefixAdd(token) - - break + // Check for shortened token collisions + if t.tokenCollision(token) { + // This is a collision. We cannot use this tree. Try again. + log.Infof("Token collision %x, creating new token", token) + continue } - log.Infof("Token prefix collision %v, creating new token", - util.TokenPrefix(token)) + // We've found a valid token. Update the tokens cache. This must + // be done even if the record creation fails since the tree will + // still exist in tstore. + t.tokenAdd(token) + break } // Create record metadata @@ -920,8 +952,13 @@ func (t *tstoreBackend) RecordSetStatus(token []byte, status backend.StatusT, md func (t *tstoreBackend) RecordExists(token []byte) bool { log.Tracef("RecordExists: %x", token) - // Read methods are allowed to use token prefixes - token = t.fullLengthToken(token) + // Read methods are allowed to use short tokens. Lookup the full + // length token. + var err error + token, err = t.fullLengthToken(token) + if err != nil { + return false + } treeID := treeIDFromToken(token) return t.tstore.TreeExists(treeID) @@ -934,8 +971,13 @@ func (t *tstoreBackend) RecordExists(token []byte) bool { func (t *tstoreBackend) RecordTimestamps(token []byte, version uint32) (*backend.RecordTimestamps, error) { log.Tracef("RecordTimestamps: %x %v", token, version) - // Read methods are allowed to use token prefixes - token = t.fullLengthToken(token) + // Read methods are allowed to use short tokens. Lookup the full + // length token. + var err error + token, err = t.fullLengthToken(token) + if err != nil { + return nil, err + } treeID := treeIDFromToken(token) return t.tstore.RecordTimestamps(treeID, version, token) @@ -947,29 +989,39 @@ func (t *tstoreBackend) RecordTimestamps(token []byte, version uint32) (*backend // // This function satisfies the Backend interface. func (t *tstoreBackend) Records(reqs []backend.RecordRequest) (map[string]backend.Record, error) { - log.Tracef("Records") + log.Tracef("Records: %v reqs", len(reqs)) - records := make(map[string]backend.Record, len(reqs)) + records := make(map[string]backend.Record, len(reqs)) // [token]Record for _, v := range reqs { - // Read methods are allowed to use token prefixes - v.Token = t.fullLengthToken(v.Token) + // Read methods are allowed to use short tokens. Lookup the full + // length token. + token, err := t.fullLengthToken(v.Token) + if err != nil { + return nil, err + } - treeID := treeIDFromToken(v.Token) + // Lookup the record + treeID := treeIDFromToken(token) r, err := t.tstore.RecordPartial(treeID, v.Version, v.Filenames, v.OmitAllFiles) if err != nil { if err == backend.ErrRecordNotFound { + log.Debugf("Record not found %x", token) + // Record doesn't exist. This is ok. It will not be included // in the reply. continue } // An unexpected error occurred. Log it and continue. - log.Debug("RecordPartial %v: %v", treeID, err) + log.Errorf("RecordPartial %x: %v", token, err) continue } - // Update reply - records[r.RecordMetadata.Token] = *r + // Update reply. Use whatever token was provided as the key so + // that the client can validate the reply using the same token + // that they provided, regardless of whether its a short token + // or full length token. + records[util.TokenEncode(v.Token)] = *r } return records, nil @@ -1037,8 +1089,13 @@ func (t *tstoreBackend) PluginRead(token []byte, pluginID, pluginCmd, payload st // will not be provided to the plugin. var treeID int64 if len(token) > 0 { - // Read methods are allowed to use token prefixes - token = t.fullLengthToken(token) + // Read methods are allowed to use short tokens. Lookup the full + // length token. + var err error + token, err = t.fullLengthToken(token) + if err != nil { + return "", err + } // Verify record exists if !t.RecordExists(token) { @@ -1155,8 +1212,7 @@ func (t *tstoreBackend) setup() error { log.Infof("%v records in the backend", len(treeIDs)) for _, v := range treeIDs { - token := tokenFromTreeID(v) - t.prefixAdd(token) + t.tokenAdd(tokenFromTreeID(v)) } return nil @@ -1176,7 +1232,7 @@ func New(appDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianSig appDir: appDir, dataDir: dataDir, tstore: ts, - prefixes: make(map[string][]byte), + tokens: make(map[string][]byte), recordMtxs: make(map[string]*sync.Mutex), } diff --git a/util/token.go b/util/token.go index 0dcc71bf0..972f24959 100644 --- a/util/token.go +++ b/util/token.go @@ -17,43 +17,65 @@ var ( TokenTypeTstore = "tstore" ) -// TokenIsFullLength returns whether a token is a valid, full length politeiad -// censorship token. -func TokenIsFullLength(tokenType string, token []byte) bool { - switch tokenType { - case TokenTypeTstore: - return len(token) == pdv2.TokenSize - case TokenTypeGit: - return len(token) == pdv1.TokenSize - default: - panic("invalid token type") - } -} - -// TokenPrefixSize returns the size (in bytes) of a politeiad token prefix. -func TokenPrefixSize() int { - // If the token prefix length is an odd number of characters then - // padding would have needed to be added to it prior to decoding it +// ShortTokenSize returns the size, in bytes, of a politeiad short token. +func ShortTokenSize() int { + // If the short token length is an odd number of characters then + // padding would have needed to be added to it prior to encoding it // to hex to prevent a hex.ErrLenth (odd length hex string) error. - // Account for this padding in the prefix size. + // This function accounts for this padding in the returned size. var size int if pdv1.TokenPrefixLength%2 == 1 { // Add 1 to the length to account for padding - size = (pdv1.TokenPrefixLength + 1) / 2 + size = (pdv2.ShortTokenLength + 1) / 2 } else { // No padding was required - size = pdv1.TokenPrefixLength / 2 + size = pdv2.ShortTokenLength / 2 } return size } -// TokenPrefix returns the token prefix given the token. -func TokenPrefix(token []byte) string { - return hex.EncodeToString(token)[:pdv1.TokenPrefixLength] +// ShortToken returns the short version of a token. +func ShortToken(token []byte) ([]byte, error) { + s := ShortTokenSize() + if len(token) < s { + return nil, fmt.Errorf("token is not large enough") + } + return token[:s], nil } -// TokenDecode decodes full length tokens. An error is returned if the token -// is not a valid full length, hex token. +// ShortTokenString takes a hex encoded token and returns the shortened token +// for it. +func ShortTokenString(token string) (string, error) { + if len(token) < pdv2.ShortTokenLength { + return "", fmt.Errorf("token is not large enough") + } + return token[:pdv2.ShortTokenLength], nil +} + +// ShortTokenEncode returns the hex encoded shortened token. +func ShortTokenEncode(token []byte) (string, error) { + t, err := ShortToken(token) + if err != nil { + return "", err + } + return TokenEncode(t), nil +} + +// TokenIsFullLength returns whether a token is a valid, full length politeiad +// censorship token. +func TokenIsFullLength(tokenType string, token []byte) bool { + switch tokenType { + case TokenTypeGit: + return len(token) == pdv1.TokenSize + case TokenTypeTstore: + return len(token) == pdv2.TokenSize + default: + panic("invalid token type") + } +} + +// TokenDecode decodes a full length token. An error is returned if the token +// is not a full length, hex token. func TokenDecode(tokenType, token string) ([]byte, error) { // Decode token t, err := hex.DecodeString(token) @@ -69,7 +91,7 @@ func TokenDecode(tokenType, token string) ([]byte, error) { return t, nil } -// TokenDecodeAnyLength decodes both token prefixes and full length tokens. +// TokenDecodeAnyLength decodes both short tokens and full length tokens. func TokenDecodeAnyLength(tokenType, token string) ([]byte, error) { // Decode token. If provided token has odd length, add padding // to prevent a hex.ErrLength (odd length hex string) error. @@ -81,11 +103,11 @@ func TokenDecodeAnyLength(tokenType, token string) ([]byte, error) { return nil, fmt.Errorf("invalid hex") } - // Verify token byte slice is either a token prefix or a valid - // full length token. + // Verify token byte slice is either a short token or a full length + // token. switch { - case len(t) == TokenPrefixSize(): - // This is a token prefix. Token prefixes are the same size + case len(t) == ShortTokenSize(): + // This is a short token. Short tokens are the same size // regardless of token type. case tokenType == TokenTypeGit && TokenIsFullLength(TokenTypeGit, t): // Token is a valid git backend token @@ -97,3 +119,17 @@ func TokenDecodeAnyLength(tokenType, token string) ([]byte, error) { return t, nil } + +// TokenEncode returns the hex encoded token. Its possible that padding has +// been added to the token when it was orginally decode in order to make it +// valid hex. This function checks for padding and removes it before encoding +// the token. +func TokenEncode(token []byte) string { + t := hex.EncodeToString(token) + if len(t) == pdv2.ShortTokenLength+1 { + // This is a short token that has had padding added to it. Remove + // the padding. + t = t[:pdv2.ShortTokenLength] + } + return t +} From 0484f0d0fe7895bde03d862e9df49bd6662e0f75 Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 19 Mar 2021 15:32:38 -0500 Subject: [PATCH 417/449] tstorebe: Fix hook json error. --- politeiad/backendv2/tstorebe/plugins/plugins.go | 6 +++--- .../backendv2/tstorebe/plugins/ticketvote/hooks.go | 12 ++++++------ politeiad/backendv2/tstorebe/plugins/usermd/hooks.go | 6 +++--- politeiad/backendv2/tstorebe/tstorebe.go | 6 +++--- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/politeiad/backendv2/tstorebe/plugins/plugins.go b/politeiad/backendv2/tstorebe/plugins/plugins.go index fb7d5ac1a..57b50c690 100644 --- a/politeiad/backendv2/tstorebe/plugins/plugins.go +++ b/politeiad/backendv2/tstorebe/plugins/plugins.go @@ -94,7 +94,7 @@ type HookNewRecordPost struct { // HookEditRecord is the payload for the pre and post edit record hooks. type HookEditRecord struct { - Current backend.Record `json:"record"` // Record pre update + Record backend.Record `json:"record"` // Record pre update // Updated fields RecordMetadata backend.RecordMetadata `json:"recordmetadata"` @@ -104,7 +104,7 @@ type HookEditRecord struct { // HookEditMetadata is the payload for the pre and post edit metadata hooks. type HookEditMetadata struct { - Current backend.Record `json:"record"` // Record pre update + Record backend.Record `json:"record"` // Record pre update // Updated fields Metadata []backend.MetadataStream `json:"metadata"` @@ -113,7 +113,7 @@ type HookEditMetadata struct { // HookSetRecordStatus is the payload for the pre and post set record status // hooks. type HookSetRecordStatus struct { - Current backend.Record `json:"record"` // Record pre update + Record backend.Record `json:"record"` // Record pre update // Updated fields RecordMetadata backend.RecordMetadata `json:"recordmetadata"` diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go index b272e7349..08dc48b4b 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go @@ -73,7 +73,7 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { return backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeLinkToInvalid), - ErrorContext: "invalid hex", + ErrorContext: err, } } r, err := p.recordAbridged(token) @@ -203,12 +203,12 @@ func (p *ticketVotePlugin) hookEditRecordPre(payload string) error { // The LinkTo field is not allowed to change once the record has // become public. If this is a vetted record, verify that any // previously set LinkTo has not changed. - if er.Current.RecordMetadata.State == backend.StateVetted { + if er.Record.RecordMetadata.State == backend.StateVetted { var ( oldLinkTo string newLinkTo string ) - vm, err := voteMetadataDecode(er.Current.Files) + vm, err := voteMetadataDecode(er.Record.Files) if err != nil { return err } @@ -257,7 +257,7 @@ func (p *ticketVotePlugin) hookSetRecordStatusPre(payload string) error { } // Check if the LinkTo has been set - vm, err := voteMetadataDecode(srs.Current.Files) + vm, err := voteMetadataDecode(srs.Record.Files) if err != nil { return err } @@ -319,7 +319,7 @@ func (p *ticketVotePlugin) hookSetRecordStatusPost(treeID int64, payload string) // Update the inventory cache var ( - oldStatus = srs.Current.RecordMetadata.Status + oldStatus = srs.Record.RecordMetadata.Status newStatus = srs.RecordMetadata.Status ) switch newStatus { @@ -334,7 +334,7 @@ func (p *ticketVotePlugin) hookSetRecordStatusPost(treeID int64, payload string) } // Update the submissions cache if the linkto has been set. - vm, err := voteMetadataDecode(srs.Current.Files) + vm, err := voteMetadataDecode(srs.Record.Files) if err != nil { return err } diff --git a/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go b/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go index 8abb5f9f4..4fc8c2240 100644 --- a/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go @@ -197,7 +197,7 @@ func (p *userPlugin) hookEditRecordPre(payload string) error { if err != nil { return err } - umCurr, err := userMetadataDecode(er.Current.Metadata) + umCurr, err := userMetadataDecode(er.Record.Metadata) if err != nil { return err } @@ -222,7 +222,7 @@ func (p *userPlugin) hookEditMetadataPre(payload string) error { } // User metadata should not change on metadata updates - return userMetadataPreventUpdates(em.Current.Metadata, em.Metadata) + return userMetadataPreventUpdates(em.Record.Metadata, em.Metadata) } func statusChangesDecode(metadata []backend.MetadataStream) ([]usermd.StatusChangeMetadata, error) { @@ -328,7 +328,7 @@ func (p *userPlugin) hookSetRecordStatusPre(payload string) error { } // User metadata should not change on status changes - err = userMetadataPreventUpdates(srs.Current.Metadata, srs.Metadata) + err = userMetadataPreventUpdates(srs.Record.Metadata, srs.Metadata) if err != nil { return err } diff --git a/politeiad/backendv2/tstorebe/tstorebe.go b/politeiad/backendv2/tstorebe/tstorebe.go index 3fdd75f23..7c69f0b9c 100644 --- a/politeiad/backendv2/tstorebe/tstorebe.go +++ b/politeiad/backendv2/tstorebe/tstorebe.go @@ -582,7 +582,7 @@ func (t *tstoreBackend) RecordEdit(token []byte, mdAppend, mdOverwrite []backend // Call pre plugin hooks her := plugins.HookEditRecord{ - Current: *r, + Record: *r, RecordMetadata: *recordMD, Metadata: metadata, Files: files, @@ -681,7 +681,7 @@ func (t *tstoreBackend) RecordEditMetadata(token []byte, mdAppend, mdOverwrite [ // Call pre plugin hooks hem := plugins.HookEditMetadata{ - Current: *r, + Record: *r, Metadata: metadata, } b, err := json.Marshal(hem) @@ -866,7 +866,7 @@ func (t *tstoreBackend) RecordSetStatus(token []byte, status backend.StatusT, md // Call pre plugin hooks hsrs := plugins.HookSetRecordStatus{ - Current: *r, + Record: *r, RecordMetadata: *recordMD, Metadata: metadata, } From 2ff99c36cfef472b619492316a25bce8b7da267f Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 19 Mar 2021 17:25:04 -0500 Subject: [PATCH 418/449] tstore: Docs and cleanup. --- .../tstorebe/plugins/ticketvote/hooks.go | 2 +- politeiad/backendv2/tstorebe/tstore/anchor.go | 28 +-- politeiad/backendv2/tstorebe/tstore/client.go | 3 + politeiad/backendv2/tstorebe/tstore/plugin.go | 18 +- .../backendv2/tstorebe/tstore/testing.go | 3 +- .../backendv2/tstorebe/tstore/tlogclient.go | 170 ++++++++++-------- politeiad/backendv2/tstorebe/tstore/tstore.go | 86 ++++++--- politeiad/backendv2/tstorebe/tstore/verify.go | 21 ++- 8 files changed, 205 insertions(+), 126 deletions(-) diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go index 08dc48b4b..8d7fb839c 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go @@ -73,7 +73,7 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { return backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeLinkToInvalid), - ErrorContext: err, + ErrorContext: err.Error(), } } r, err := p.recordAbridged(token) diff --git a/politeiad/backendv2/tstorebe/tstore/anchor.go b/politeiad/backendv2/tstorebe/tstore/anchor.go index b9062bda6..d95a982e4 100644 --- a/politeiad/backendv2/tstorebe/tstore/anchor.go +++ b/politeiad/backendv2/tstorebe/tstore/anchor.go @@ -20,11 +20,6 @@ import ( "google.golang.org/grpc/codes" ) -// TODO handle reorgs. An anchor record may become invalid in the case of a -// reorg. We don't create the anchor record until the anchor tx has 6 -// confirmations so the probability of this occurring on mainnet is low, but it -// still needs to be handled. - const ( // anchorSchedule determines how often we anchor records. dcrtime // currently drops an anchor on the hour mark so we submit new @@ -50,6 +45,8 @@ type anchor struct { VerifyDigest *dcrtime.VerifyDigest `json:"verifydigest"` } +// droppingAnchorGet returns the dropping anchor boolean, which is used to +// prevent reentrant anchor drops. func (t *Tstore) droppingAnchorGet() bool { t.Lock() defer t.Unlock() @@ -57,6 +54,8 @@ func (t *Tstore) droppingAnchorGet() bool { return t.droppingAnchor } +// droppingAnchorSet sets the dropping anchor boolean, which is used to prevent +// reentrant anchor drops. func (t *Tstore) droppingAnchorSet(b bool) { t.Lock() defer t.Unlock() @@ -150,7 +149,7 @@ func (t *Tstore) anchorForLeaf(treeID int64, merkleLeafHash []byte, leaves []*tr } // anchorLatest returns the most recent anchor for the provided tree. A -// errAnchorNotFound is returned if no anchor is found for the provided tree. +// errAnchorNotFound is returned if no anchor is found. func (t *Tstore) anchorLatest(treeID int64) (*anchor, error) { // Get tree leaves leavesAll, err := t.tlog.LeavesAll(treeID) @@ -212,7 +211,7 @@ func (t *Tstore) anchorSave(a anchor) error { return fmt.Errorf("verify digest not found") } - // Save the anchor record to store + // Save anchor record to the kv store be, err := convertBlobEntryFromAnchor(a) if err != nil { return err @@ -228,7 +227,7 @@ func (t *Tstore) anchorSave(a anchor) error { return fmt.Errorf("store Put: %v", err) } - // Append anchor leaf to trillian tree + // Append anchor leaf to tlog d, err := hex.DecodeString(be.Digest) if err != nil { return err @@ -259,7 +258,7 @@ func (t *Tstore) anchorSave(a anchor) error { return fmt.Errorf("append leaves failed: %v", failed) } - log.Debugf("Saved anchor for tree %v at height %v", + log.Debugf("Anchor saved for tree %v at height %v", a.TreeID, a.LogRoot.TreeSize) return nil @@ -269,8 +268,8 @@ func (t *Tstore) anchorSave(a anchor) error { // dropped until dcrtime returns the ChainTimestamp in the reply. dcrtime does // not return the ChainTimestamp until the timestamp transaction has 6 // confirmations. Once the timestamp has been dropped, the anchor record is -// saved to the key-value store and the record histories of the corresponding -// timestamped trees are updated. +// saved to the tstore, which means that an anchor leaf will be appended onto +// all trees that were anchored and the anchor records saved to the kv store. func (t *Tstore) anchorWait(anchors []anchor, digests []string) { // Verify we are not reentrant if t.droppingAnchorGet() { @@ -432,9 +431,10 @@ func (t *Tstore) anchorWait(anchors []anchor, digests []string) { } // anchorTrees drops an anchor for any trees that have unanchored leaves at the -// time of function invocation. A SHA256 digest of the tree's log root at its -// current height is timestamped onto the decred blockchain using the dcrtime -// service. The anchor data is saved to the key-value store. +// time of invocation. A SHA256 digest of the tree's log root at its current +// height is timestamped onto the decred blockchain using the dcrtime service. +// The anchor data is saved to the key-value store and the tlog tree is updated +// with an anchor leaf. func (t *Tstore) anchorTrees() error { log.Debugf("Start anchor process") diff --git a/politeiad/backendv2/tstorebe/tstore/client.go b/politeiad/backendv2/tstorebe/tstore/client.go index 764b1d8ae..17db0c4dc 100644 --- a/politeiad/backendv2/tstorebe/tstore/client.go +++ b/politeiad/backendv2/tstorebe/tstore/client.go @@ -409,6 +409,9 @@ func (t *Tstore) Timestamp(treeID int64, digest []byte) (*backend.Timestamp, err return t.timestamp(treeID, m, leaves) } +// leavesForDescriptor returns all leaves that have and extra data descriptor +// that matches the provided descriptor. If a record is vetted, only vetted +// leaves will be returned. func leavesForDescriptor(leaves []*trillian.LogLeaf, descriptors []string) []*trillian.LogLeaf { // Put descriptors into a map for 0(n) lookups desc := make(map[string]struct{}, len(descriptors)) diff --git a/politeiad/backendv2/tstorebe/tstore/plugin.go b/politeiad/backendv2/tstorebe/tstore/plugin.go index a47909408..4b1734e61 100644 --- a/politeiad/backendv2/tstorebe/tstore/plugin.go +++ b/politeiad/backendv2/tstorebe/tstore/plugin.go @@ -26,8 +26,8 @@ import ( const ( // pluginDataDirname is the plugin data directory name. It is - // located in the tstore instance data directory and is provided to - // the plugins for storing plugin data. + // located in the tstore backend data directory and is provided + // to the plugins for storing plugin data. pluginDataDirname = "plugins" ) @@ -37,6 +37,8 @@ type plugin struct { client plugins.PluginClient } +// plugin returns the specified plugin. Only plugins that have been registered +// will be returned. func (t *Tstore) plugin(pluginID string) (plugin, bool) { t.Lock() defer t.Unlock() @@ -45,6 +47,7 @@ func (t *Tstore) plugin(pluginID string) (plugin, bool) { return plugin, ok } +// pluginIDs returns the plugin ID of all registered plugins. func (t *Tstore) pluginIDs() []string { t.Lock() defer t.Unlock() @@ -62,6 +65,8 @@ func (t *Tstore) pluginIDs() []string { return ids } +// PluginRegister registers a plugin. Plugin commands and hooks can be executed +// on the plugin once registered. func (t *Tstore) PluginRegister(b backend.Backend, p backend.Plugin) error { log.Tracef("PluginRegister: %v", p.ID) @@ -113,6 +118,7 @@ func (t *Tstore) PluginRegister(b backend.Backend, p backend.Plugin) error { return nil } +// PluginSetup performs any required setup for the specified plugin. func (t *Tstore) PluginSetup(pluginID string) error { log.Tracef("PluginSetup: %v", pluginID) @@ -124,6 +130,10 @@ func (t *Tstore) PluginSetup(pluginID string) error { return p.client.Setup() } +// PluginHookPre executes a tstore backend pre hook. Pre hooks are hooks that +// are executed prior to the tstore backend writing data to disk. These hooks +// give plugins the opportunity to add plugin specific validation to record +// methods or plugin commands that write data. func (t *Tstore) PluginHookPre(treeID int64, token []byte, h plugins.HookT, payload string) error { log.Tracef("PluginHookPre: %v %v", treeID, plugins.Hooks[h]) @@ -143,6 +153,9 @@ func (t *Tstore) PluginHookPre(treeID int64, token []byte, h plugins.HookT, payl return nil } +// PluginHookPre executes a tstore backend post hook. Post hooks are hooks that +// are executed after the tstore backend successfully writes data to disk. +// These hooks give plugins the opportunity to cache data from the write. func (t *Tstore) PluginHookPost(treeID int64, token []byte, h plugins.HookT, payload string) { log.Tracef("PluginHookPost: %v %v", treeID, plugins.Hooks[h]) @@ -165,6 +178,7 @@ func (t *Tstore) PluginHookPost(treeID int64, token []byte, h plugins.HookT, pay } } +// PluginCmd executes a plugin command. func (t *Tstore) PluginCmd(treeID int64, token []byte, pluginID, cmd, payload string) (string, error) { log.Tracef("PluginCmd: %v %x %v %v", treeID, token, pluginID, cmd) diff --git a/politeiad/backendv2/tstorebe/tstore/testing.go b/politeiad/backendv2/tstorebe/tstore/testing.go index 6a2b2c3ea..fd002c7bd 100644 --- a/politeiad/backendv2/tstorebe/tstore/testing.go +++ b/politeiad/backendv2/tstorebe/tstore/testing.go @@ -11,6 +11,7 @@ import ( "github.com/decred/politeia/politeiad/backendv2/tstorebe/store/localdb" ) +// NewTestTstore returns a tstore instance that is setup for testing. func NewTestTstore(t *testing.T, dataDir string) *Tstore { t.Helper() @@ -21,7 +22,7 @@ func NewTestTstore(t *testing.T, dataDir string) *Tstore { } // Setup key-value store - fp, err := ioutil.TempDir(dataDir, defaultStoreDirname) + fp, err := ioutil.TempDir(dataDir, storeDirname) if err != nil { t.Fatal(err) } diff --git a/politeiad/backendv2/tstorebe/tstore/tlogclient.go b/politeiad/backendv2/tstorebe/tstore/tlogclient.go index df88f360d..7eca30c16 100644 --- a/politeiad/backendv2/tstorebe/tstore/tlogclient.go +++ b/politeiad/backendv2/tstorebe/tstore/tlogclient.go @@ -27,6 +27,7 @@ import ( "github.com/google/trillian/merkle/hashers/registry" "github.com/google/trillian/merkle/rfc6962" "github.com/google/trillian/types" + rstatus "google.golang.org/genproto/googleapis/rpc/status" "google.golang.org/genproto/protobuf/field_mask" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -42,6 +43,8 @@ var ( ) const ( + // waitForInclusionTimeout is the amount of time that we wait for + // a queued leaf to be appended onto a tlog tree before timing out. waitForInclusionTimeout = 120 * time.Second ) @@ -62,7 +65,8 @@ type tlogClient interface { // TreeNew creates a new tree. TreeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) - // TreeFreeze sets the status of a tree to frozen. + // TreeFreeze sets the status of a tree to frozen and returns the + // updated tree. TreeFreeze(treeID int64) (*trillian.Tree, error) // Tree returns a tree. @@ -71,7 +75,7 @@ type tlogClient interface { // TreesAll returns all trees in the trillian instance. TreesAll() ([]*trillian.Tree, error) - // LeavesAppend appends leaves to a tree. + // LeavesAppend appends leaves onto a tree. LeavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, *types.LogRootV1, error) @@ -175,7 +179,7 @@ func (t *tclient) TreeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { return tree, ilr.Created, nil } -// TreeFreeze updates the state of a tree to frozen. +// TreeFreeze sets the status of a tree to frozen and returns the updated tree. // // This function satisfies the tlogClient interface. func (t *tclient) TreeFreeze(treeID int64) (*trillian.Tree, error) { @@ -588,24 +592,15 @@ type testTClient struct { trees map[int64]*trillian.Tree // [treeID]Tree leaves map[int64][]*trillian.LogLeaf // [treeID][]LogLeaf - - privateKey *keyspb.PrivateKey // Trillian signing key - publicKey crypto.PublicKey // Trillian public key } -// TreeNew ceates a new trillian tree in memory. +// TreeNew creates a new tree. // // This function satisfies the tlogClient interface. func (t *testTClient) TreeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) { t.Lock() defer t.Unlock() - // Retrieve private key - pk, err := ptypes.MarshalAny(t.privateKey) - if err != nil { - return nil, nil, err - } - // Create trillian tree tree := trillian.Tree{ TreeId: rand.Int63(), @@ -616,45 +611,48 @@ func (t *testTClient) TreeNew() (*trillian.Tree, *trillian.SignedLogRoot, error) SignatureAlgorithm: sigpb.DigitallySigned_ED25519, DisplayName: "", Description: "", - MaxRootDuration: ptypes.DurationProto(0), - PrivateKey: pk, } t.trees[tree.TreeId] = &tree - // Initialize leaves map for that tree + // Initialize leaves t.leaves[tree.TreeId] = []*trillian.LogLeaf{} return &tree, nil, nil } -// TreeFreeze sets the state of a tree to frozen. +// TreeFreeze sets the status of a tree to frozen and returns the updated tree. // // This function satisfies the tlogClient interface. func (t *testTClient) TreeFreeze(treeID int64) (*trillian.Tree, error) { t.Lock() defer t.Unlock() - t.trees[treeID].TreeState = trillian.TreeState_FROZEN + tree, ok := t.trees[treeID] + if !ok { + return nil, fmt.Errorf("tree not found") + } + tree.TreeState = trillian.TreeState_FROZEN + t.trees[treeID] = tree - return t.trees[treeID], nil + return tree, nil } -// Tree returns trillian tree from passed in ID. +// Tree returns a tree. // // This function satisfies the tlogClient interface. func (t *testTClient) Tree(treeID int64) (*trillian.Tree, error) { t.Lock() defer t.Unlock() - if tree, ok := t.trees[treeID]; ok { - return tree, nil + tree, ok := t.trees[treeID] + if !ok { + return nil, fmt.Errorf("tree not found") } - return nil, fmt.Errorf("tree ID not found") + return tree, nil } -// TreesAll signed log roots are not used for testing up until now, so we -// return a nil value for it. +// TreesAll returns all trees in the trillian instance. // // This function satisfies the tlogClient interface. func (t *testTClient) TreesAll() ([]*trillian.Tree, error) { @@ -662,106 +660,130 @@ func (t *testTClient) TreesAll() ([]*trillian.Tree, error) { defer t.Unlock() trees := make([]*trillian.Tree, len(t.trees)) - for _, t := range t.trees { - trees = append(trees, t) + for _, v := range t.trees { + trees = append(trees, &trillian.Tree{ + TreeId: v.TreeId, + TreeState: v.TreeState, + TreeType: v.TreeType, + HashStrategy: v.HashStrategy, + HashAlgorithm: v.HashAlgorithm, + SignatureAlgorithm: v.SignatureAlgorithm, + DisplayName: v.DisplayName, + Description: v.Description, + }) } return trees, nil } -// LeavesAppend satisfies the TClient interface. It appends leaves to the -// corresponding trillian tree in memory. +// LeavesAppend appends leaves onto a tree. // // This function satisfies the tlogClient interface. -func (t *testTClient) LeavesAppend(treeID int64, leaves []*trillian.LogLeaf) ([]queuedLeafProof, *types.LogRootV1, error) { +func (t *testTClient) LeavesAppend(treeID int64, leavesAppend []*trillian.LogLeaf) ([]queuedLeafProof, *types.LogRootV1, error) { t.Lock() defer t.Unlock() + leaves, ok := t.leaves[treeID] + if !ok { + leaves = make([]*trillian.LogLeaf, 0, len(leavesAppend)) + } + // Get last leaf index var index int64 - if len(t.leaves[treeID]) > 0 { - l := len(t.leaves[treeID]) - index = t.leaves[treeID][l-1].LeafIndex + 1 - } else { - index = 0 + if len(leaves) > 0 { + index = int64(len(leaves)) - 1 } - // Set merkle hash for each leaf and append to memory. Also append the - // queued value for the leaves to be returned by the function. - var queued []queuedLeafProof - for _, l := range leaves { - l.LeafIndex = index - l.MerkleLeafHash = merkleLeafHash(l.LeafValue) - t.leaves[treeID] = append(t.leaves[treeID], l) - + // Append leaves + queued := make([]queuedLeafProof, 0, len(leavesAppend)) + for _, v := range leavesAppend { + // Append to leaves + v.MerkleLeafHash = merkleLeafHash(v.LeafValue) + v.ExtraData = v.ExtraData + v.LeafIndex = index + 1 + leaves = append(leaves, v) + index++ + + // Append to reply queued = append(queued, queuedLeafProof{ QueuedLeaf: &trillian.QueuedLogLeaf{ - Leaf: l, - Status: nil, + Leaf: v, + Status: &rstatus.Status{ + Code: 0, // 0 indicates OK + }, }, - Proof: nil, }) } + // Save updated leaves + t.leaves[treeID] = leaves + return queued, nil, nil } -// LeavesAll returns all leaves from a trillian tree. +// LeavesAll returns all leaves of a tree. // // This function satisfies the tlogClient interface. func (t *testTClient) LeavesAll(treeID int64) ([]*trillian.LogLeaf, error) { t.Lock() defer t.Unlock() - // Check if treeID entry exists - if _, ok := t.leaves[treeID]; !ok { - return nil, fmt.Errorf("tree ID %d does not contain any leaf data", - treeID) + // Verify tree exists + _, ok := t.trees[treeID] + if !ok { + return nil, fmt.Errorf("tree not found") + } + + // Get leaves + leaves, ok := t.leaves[treeID] + if !ok { + leaves = make([]*trillian.LogLeaf, 0) + } + + // Copy leaves + leavesCopy := make([]*trillian.LogLeaf, 0, len(leaves)) + for _, v := range leaves { + var ( + leafValue []byte + extraData []byte + ) + copy(leafValue, v.LeafValue) + copy(extraData, v.ExtraData) + leavesCopy = append(leavesCopy, &trillian.LogLeaf{ + MerkleLeafHash: merkleLeafHash(leafValue), + LeafValue: leafValue, + ExtraData: extraData, + LeafIndex: v.LeafIndex, + }) } - return t.leaves[treeID], nil + return leavesCopy, nil } -// SignedLogRoot has not been implemented yet for the test client. +// SignedLogRoot has not been implemented yet. // // This function satisfies the tlogClient interface. func (t *testTClient) SignedLogRoot(tree *trillian.Tree) (*trillian.SignedLogRoot, *types.LogRootV1, error) { - return nil, nil, nil + return nil, nil, fmt.Errorf("not implemented") } -// InclusionProof has not been implement yet for the test client. +// InclusionProof has not been implement yet. // // This function satisfies the tlogClient interface. func (t *testTClient) InclusionProof(treeID int64, merkleLeafHash []byte, lr *types.LogRootV1) (*trillian.Proof, error) { - return nil, nil + return nil, fmt.Errorf("not implemented") } -// Close closes the trillian client connection. There is nothing to do for the -// test implementation. +// Close closes the client connection. There is nothing to do for the test tlog +// client. // // This function satisfies the tlogClient interface. func (t *testTClient) Close() {} // newTestTClient returns a new testTClient. func newTestTClient(t *testing.T) *testTClient { - key, err := newTrillianKey() - if err != nil { - t.Fatal(err) - } - b, err := der.MarshalPrivateKey(key) - if err != nil { - t.Fatal(err) - } - signer, err := der.UnmarshalPrivateKey(b) - if err != nil { - t.Fatal(err) - } return &testTClient{ trees: make(map[int64]*trillian.Tree), leaves: make(map[int64][]*trillian.LogLeaf), - privateKey: &keyspb.PrivateKey{ - Der: b, - }, - publicKey: signer.Public(), } } diff --git a/politeiad/backendv2/tstorebe/tstore/tstore.go b/politeiad/backendv2/tstorebe/tstore/tstore.go index ce95419e9..cc18ffa2c 100644 --- a/politeiad/backendv2/tstorebe/tstore/tstore.go +++ b/politeiad/backendv2/tstorebe/tstore/tstore.go @@ -30,12 +30,22 @@ import ( ) const ( + // DBTypeLevelDB is a config option that sets the backing key-value + // store to a leveldb instance. DBTypeLevelDB = "leveldb" - DBTypeMySQL = "mysql" - dbUser = "politeiad" + // DBTypeLevelDB is a config option that sets the backing key-value + // store to a MySQL instance. + DBTypeMySQL = "mysql" + + // LevelDB settings + storeDirname = "store" + + // MySQL settings + dbUser = "politeiad" + + // Config option defaults defaultTrillianSigningKeyFilename = "trillian.key" - defaultStoreDirname = "store" // Blob entry data descriptors dataDescriptorRecordMetadata = "pd-recordmd-v1" @@ -57,7 +67,26 @@ var ( _ plugins.TstoreClient = (*Tstore)(nil) ) -// We do not unwind. +// Tstore represents a trillian log (tlog) backed by a key-value store. When +// data is saved to a tstore instance it is first saved to the key-value store +// then a digest of the data is appended onto the tlog tree. Tlog trees are +// episodically timestamped onto the decred blockchain. An inlcusion proof, +// i.e. the cryptographic proof that the data was included in the decred +// timestamp, can be retrieved for any individual piece of data saved to the +// tstore instance. +// +// Saving only the digest of the data to tlog means that we separate the +// timestamp from the data itself. This allows us to remove content that is +// deemed undesirable from the key-value store without impacting the ability +// to retrieve inclusion proofs for any other pieces of data saved to tstore. +// +// The key-value store is write once. Edits to data in the key-value store are +// not allowed. Deletes, however, are allowed. +// +// The tlog tree is append only and is treated as the source of truth. If any +// blobs make it into the key-value store but do not make it into the tlog tree +// they are considered to be orphaned and are simply ignored. We do not unwind +// failed calls. type Tstore struct { sync.Mutex dataDir string @@ -115,7 +144,7 @@ func (e *extraData) storeKeyNoPrefix() string { func extraDataEncode(key, desc string, state backend.StateT) ([]byte, error) { // The encryption prefix is stripped from the key if one exists. ed := extraData{ - Key: storeKeyClean(key), + Key: storeKeyCleaned(key), Desc: desc, State: state, } @@ -145,14 +174,16 @@ func storeKeyNew(encrypt bool) string { return k } -// storeKeyClean strips the key-value store key of the encryption prefix if +// storeKeyCleaned strips the key-value store key of the encryption prefix if // one is present. -func storeKeyClean(key string) string { +func storeKeyCleaned(key string) string { // A uuid string is 36 bytes. Return the last 36 bytes of the // string. This will strip the prefix if it exists. return key[len(key)-36:] } +// merkleLeafHashForBlobEntry returns the merkle leaf hash for a blob entry. +// The merkle leaf hash can be used to retrieve a leaf from its tlog tree. func merkleLeafHashForBlobEntry(be store.BlobEntry) ([]byte, error) { leafValue, err := hex.DecodeString(be.Digest) if err != nil { @@ -161,6 +192,7 @@ func merkleLeafHashForBlobEntry(be store.BlobEntry) ([]byte, error) { return merkleLeafHash(leafValue), nil } +// TreeNew creates a new tlog tree and returns the tree ID. func (t *Tstore) TreeNew() (int64, error) { log.Tracef("TreeNew") @@ -219,14 +251,6 @@ func (t *Tstore) TreeExists(treeID int64) bool { return err == nil } -func (t *Tstore) treeIsFrozen(leaves []*trillian.LogLeaf) bool { - r, err := t.recordIndexLatest(leaves) - if err != nil { - panic(err) - } - return r.Frozen -} - // recordBlobsSave saves the provided blobs to the kv store, appends a leaf // to the trillian tree for each blob, and returns the record index for the // blobs. @@ -622,9 +646,10 @@ func (t *Tstore) recordSave(treeID int64, rm backend.RecordMetadata, metadata [] // RecordSave saves the provided record to tstore. Once the record contents // have been successfully saved to tstore, a recordIndex is created for this -// version of the record and saved to tstore as well. This iteration of the -// record is not considered to be valid until the record index has been -// successfully saved. +// version of the record and saved to tstore as well. The record update is not +// considered to be valid until the record index has been successfully saved. +// If the record content makes it in but the record index does not, the record +// content blobs are orphaned and ignored. func (t *Tstore) RecordSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { log.Tracef("RecordSave: %v", treeID) @@ -643,10 +668,9 @@ func (t *Tstore) RecordSave(treeID int64, rm backend.RecordMetadata, metadata [] return nil } -// RecordDel walks the provided tree and deletes all file blobs in the store -// that correspond to record files. This is done for all versions and all -// iterations of the record. Record metadata and metadata stream blobs are not -// deleted. +// RecordDel walks the provided tree and deletes all blobs in the store that +// correspond to record files. This is done for all versions and all iterations +// of the record. Record metadata and metadata stream blobs are not deleted. func (t *Tstore) RecordDel(treeID int64) error { log.Tracef("RecordDel: %v", treeID) @@ -922,6 +946,8 @@ func (t *Tstore) RecordPartial(treeID int64, version uint32, filenames []string, // recordIsVetted returns whether the provided leaves contain any vetted record // indexes. The presence of a vetted record index means the record is vetted. +// The state of a record index is saved to the leaf extra data, which is how we +// determine if a record index is vetted. func recordIsVetted(leaves []*trillian.LogLeaf) bool { for _, v := range leaves { ed, err := extraDataDecode(v.ExtraData) @@ -937,6 +963,9 @@ func recordIsVetted(leaves []*trillian.LogLeaf) bool { return false } +// RecordState returns the state of a record. This call does not require +// retrieving any blobs from the kv store. The record state can be derived from +// only the tlog leaves. func (t *Tstore) RecordState(treeID int64) (backend.StateT, error) { log.Tracef("RecordState: %v", treeID) @@ -952,6 +981,7 @@ func (t *Tstore) RecordState(treeID int64) (backend.StateT, error) { return backend.StateUnvetted, nil } +// timestamp returns the timestamp given a tlog merkle leaf hash. func (t *Tstore) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trillian.LogLeaf) (*backend.Timestamp, error) { // Find the leaf var l *trillian.LogLeaf @@ -1090,6 +1120,9 @@ func (t *Tstore) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trilli return &ts, nil } +// RecordTimestamps returns the timestamps for the contents of a record. +// Timestamps for the record metadata, metadata streams, and files are all +// returned. func (t *Tstore) RecordTimestamps(treeID int64, version uint32, token []byte) (*backend.RecordTimestamps, error) { log.Tracef("RecordTimestamps: %v %v", treeID, version) @@ -1149,13 +1182,15 @@ func (t *Tstore) RecordTimestamps(treeID int64, version uint32, token []byte) (* }, nil } +// Fsck performs a filesystem check on the tstore. func (t *Tstore) Fsck() { // Set tree status to frozen for any trees that are frozen and have // been anchored one last time. - // Failed censor. Ensure all blobs have been deleted from all - // record versions of a censored record. + + // Verify all file blobs have been deleted for censored records. } +// Close performs cleanup of the tstore. func (t *Tstore) Close() { log.Tracef("Close") @@ -1164,6 +1199,7 @@ func (t *Tstore) Close() { t.store.Close() } +// New returns a new tstore instance. func New(appDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianSigningKeyFile, dbType, dbHost, dbPass, dcrtimeHost, dcrtimeCert string) (*Tstore, error) { // Setup trillian client if trillianSigningKeyFile == "" { @@ -1192,7 +1228,7 @@ func New(appDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianSig var kvstore store.BlobKV switch dbType { case DBTypeLevelDB: - fp := filepath.Join(dataDir, defaultStoreDirname) + fp := filepath.Join(dataDir, storeDirname) err = os.MkdirAll(fp, 0700) if err != nil { return nil, err diff --git a/politeiad/backendv2/tstorebe/tstore/verify.go b/politeiad/backendv2/tstorebe/tstore/verify.go index 2a82ec181..ccfa1bac2 100644 --- a/politeiad/backendv2/tstorebe/tstore/verify.go +++ b/politeiad/backendv2/tstorebe/tstore/verify.go @@ -21,11 +21,11 @@ import ( ) const ( - // ProofTypeTrillianRFC6962 indicates a trillian proof that uses + // ProofTypeTrillianRFC6962 represents a trillian proof that uses // the trillian hashing strategy HashStrategy_RFC6962_SHA256. ProofTypeTrillianRFC6962 = "trillian-rfc6962" - // ProofTypeDcrtime indicates a dcrtime proof. + // ProofTypeDcrtime represents a dcrtime proof. ProofTypeDcrtime = "dcrtime" ) @@ -36,13 +36,7 @@ type ExtraDataTrillianRFC6962 struct { TreeSize int64 `json:"treesize"` } -// ExtraDataDcrtime requires the extra data required to verify a dcrtime -// inclusion proof. -type ExtraDataDcrtime struct { - NumLeaves uint32 // Nuber of leaves - Flags string // Bitmap of merkle tree, base64 encoded -} - +// verifyProofTrillian verifies a proof with the type ProofTypeTrillianRFC6962. func verifyProofTrillian(p backend.Proof) error { // Verify type if p.Type != ProofTypeTrillianRFC6962 { @@ -87,6 +81,14 @@ func verifyProofTrillian(p backend.Proof) error { merklePath, merkleRoot, leafHash) } +// ExtraDataDcrtime contains the extra data required to verify a dcrtime +// inclusion proof. +type ExtraDataDcrtime struct { + NumLeaves uint32 // Nuber of leaves + Flags string // Bitmap of merkle tree, base64 encoded +} + +// verifyProofDcrtime verifies a proof with the type ProofTypeDcrtime. func verifyProofDcrtime(p backend.Proof) error { if p.Type != ProofTypeDcrtime { return fmt.Errorf("invalid proof type") @@ -147,6 +149,7 @@ func verifyProofDcrtime(p backend.Proof) error { return nil } +// verifyProof verifies a backend proof. func verifyProof(p backend.Proof) error { switch p.Type { case ProofTypeTrillianRFC6962: From 338750c8794d6c1e941387c45bae3a48a265400c Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 21 Mar 2021 13:28:57 -0500 Subject: [PATCH 419/449] multi: Add InventoryOrdered. --- politeiad/api/v2/v2.go | 26 +++- politeiad/backendv2/backendv2.go | 9 +- politeiad/backendv2/tstorebe/inventory.go | 42 +++++- politeiad/backendv2/tstorebe/store/store.go | 6 +- politeiad/backendv2/tstorebe/tstore/tstore.go | 25 ++-- politeiad/backendv2/tstorebe/tstorebe.go | 20 ++- politeiad/client/pdv2.go | 34 +++++ politeiad/politeiad.go | 5 + politeiad/v2.go | 106 ++++++++++++++- politeiawww/api/records/v1/v1.go | 123 +++++++++-------- politeiawww/client/records.go | 30 ++++- politeiawww/cmd/pictl/cmdhelp.go | 78 +++++------ politeiawww/cmd/pictl/cmdproposalinv.go | 10 +- .../cmd/pictl/cmdproposalinvordered.go | 104 +++++++++++++++ politeiawww/cmd/pictl/pictl.go | 6 +- politeiawww/pi.go | 5 +- politeiawww/records/process.go | 125 +++++++++++------- politeiawww/records/records.go | 56 ++++++-- 18 files changed, 615 insertions(+), 195 deletions(-) create mode 100644 politeiawww/cmd/pictl/cmdproposalinvordered.go diff --git a/politeiad/api/v2/v2.go b/politeiad/api/v2/v2.go index cff7fe15d..5e5b24152 100644 --- a/politeiad/api/v2/v2.go +++ b/politeiad/api/v2/v2.go @@ -18,6 +18,7 @@ const ( RouteRecordTimestamps = "/recordtimestamps" RouteRecords = "/records" RouteInventory = "/inventory" + RouteInventoryOrdered = "/inventoryordered" RoutePluginWrite = "/pluginwrite" RoutePluginReads = "/pluginreads" RoutePluginInventory = "/plugininventory" @@ -50,6 +51,8 @@ const ( ErrorCodePluginIDInvalid ErrorCodeT = 17 ErrorCodePluginCmdInvalid ErrorCodeT = 18 ErrorCodePageSizeExceeded ErrorCodeT = 19 + ErrorCodeRecordStateInvalid ErrorCodeT = 20 + ErrorCodeRecordStatusInvalid ErrorCodeT = 21 ) var ( @@ -73,6 +76,8 @@ var ( ErrorCodePluginIDInvalid: "pluguin id invalid", ErrorCodePluginCmdInvalid: "plugin cmd invalid", ErrorCodePageSizeExceeded: "page size exceeded", + ErrorCodeRecordStateInvalid: "record state invalid", + ErrorCodeRecordStatusInvalid: "record status invalid", } ) @@ -162,7 +167,7 @@ const ( // RecordStatusArchived indicates a record has been archived. An // archived record is locked from any further updates. An archived - // record have a state of either unvetted or vetted. + // record can have a state of either unvetted or vetted. RecordStatusArchived RecordStatusT = 4 ) @@ -398,8 +403,8 @@ type RecordsReply struct { } const ( - // InventoryPageSize is the maximum number of tokens that will be - // returned for any single status in an InventoryReply. + // InventoryPageSize is the number of tokens that will be returned + // per page for all inventory commands. InventoryPageSize uint32 = 20 ) @@ -427,6 +432,21 @@ type InventoryReply struct { Vetted map[string][]string `json:"vetted"` } +// InventoryOrdered requests a page of record tokens ordered by the timestamp +// of their most recent status change from newest to oldest. The reply will +// include tokens for all record statuses. +type InventoryOrdered struct { + Challenge string `json:"challenge"` // Random challenge + State RecordStateT `json:"state"` + Page uint32 `json:"page"` +} + +// InventoryOrderedReply is the reply to the InventoryOrdered command. +type InventoryOrderedReply struct { + Response string `json:"response"` // Challenge response + Tokens []string `json:"tokens"` +} + // PluginCmd represents plugin command and the command payload. A token is // required for all plugin writes, but is optional for reads. type PluginCmd struct { diff --git a/politeiad/backendv2/backendv2.go b/politeiad/backendv2/backendv2.go index 4caa771ef..86cf0ab64 100644 --- a/politeiad/backendv2/backendv2.go +++ b/politeiad/backendv2/backendv2.go @@ -318,11 +318,10 @@ type Backend interface { Inventory(state StateT, status StatusT, pageSize, pageNumber uint32) (*Inventory, error) - // InventoryTimeOrdered returns a page of record tokens sorted by - // timestamp of their most recent status change. The returned - // tokens are not sorted by status and will included all statuses. - InventoryTimeOrdered(state StateT, pageSize, - pageNumber uint32) ([]string, error) + // InventoryOrdered returns a page of record tokens ordered by the + // timestamp of their most recent status change from newest to + // oldest. The returned tokens will include all record statuses. + InventoryOrdered(s StateT, pageSize, pageNumber uint32) ([]string, error) // PluginRegister registers a plugin. PluginRegister(Plugin) error diff --git a/politeiad/backendv2/tstorebe/inventory.go b/politeiad/backendv2/tstorebe/inventory.go index f16d146ec..647076590 100644 --- a/politeiad/backendv2/tstorebe/inventory.go +++ b/politeiad/backendv2/tstorebe/inventory.go @@ -366,6 +366,45 @@ func (t *tstoreBackend) invByStatus(state backend.StateT, s backend.StatusT, pag return &ibs, nil } +// invOrdered returns a page of record tokens ordered by the timestamp of their +// most recent status change. The returned tokens will include tokens for all +// record statuses. +func (t *tstoreBackend) invOrdered(state backend.StateT, pageSize, pageNumber uint32) ([]string, error) { + // Get inventory file path + var fp string + switch state { + case backend.StateUnvetted: + fp = t.invPathUnvetted() + case backend.StateVetted: + fp = t.invPathVetted() + default: + return nil, fmt.Errorf("unknown state '%v'", state) + } + + // Get inventory + inv, err := t.invGet(fp) + if err != nil { + return nil, err + } + + // Return specified page of tokens + var ( + startIdx = int((pageNumber - 1) * pageSize) + endIdx = startIdx + int(pageSize) + tokens = make([]string, 0, pageSize) + ) + for i := startIdx; i < endIdx; i++ { + if i >= len(inv.Entries) { + // We've reached the end of the inventory. We're done. + break + } + + tokens = append(tokens, inv.Entries[i].Token) + } + + return tokens, nil +} + // entryDel removes the entry for the token and returns the updated slice. func entryDel(entries []entry, token []byte) ([]entry, error) { // Find token in entries @@ -391,7 +430,8 @@ func entryDel(entries []entry, token []byte) ([]entry, error) { return entries, nil } -// tokensParse parses a page of tokens from the provided entries. +// tokensParse parses a page of tokens from the provided entries that meet the +// provided criteria. func tokensParse(entries []entry, s backend.StatusT, countPerPage, page uint32) []string { tokens := make([]string, 0, countPerPage) if countPerPage == 0 || page == 0 { diff --git a/politeiad/backendv2/tstorebe/store/store.go b/politeiad/backendv2/tstorebe/store/store.go index 1a530c78a..fef0db571 100644 --- a/politeiad/backendv2/tstorebe/store/store.go +++ b/politeiad/backendv2/tstorebe/store/store.go @@ -16,12 +16,14 @@ import ( ) var ( + // ErrShutdown is returned when a action is attempted against a + // store that is shutdown. ErrShutdown = errors.New("store is shutdown") ) const ( - // DataTypeStructure is used as the data descriptor type when the - // blob entry contains a structure. + // DataTypeStructure describes a blob entry that contains a + // structure. DataTypeStructure = "struct" ) diff --git a/politeiad/backendv2/tstorebe/tstore/tstore.go b/politeiad/backendv2/tstorebe/tstore/tstore.go index cc18ffa2c..7fd370dd1 100644 --- a/politeiad/backendv2/tstorebe/tstore/tstore.go +++ b/politeiad/backendv2/tstorebe/tstore/tstore.go @@ -67,21 +67,20 @@ var ( _ plugins.TstoreClient = (*Tstore)(nil) ) -// Tstore represents a trillian log (tlog) backed by a key-value store. When -// data is saved to a tstore instance it is first saved to the key-value store -// then a digest of the data is appended onto the tlog tree. Tlog trees are -// episodically timestamped onto the decred blockchain. An inlcusion proof, -// i.e. the cryptographic proof that the data was included in the decred -// timestamp, can be retrieved for any individual piece of data saved to the -// tstore instance. +// Tstore is a data store that automatically timestamps all data saved to it +// onto the decred blockchain, making it possible to cryptographically prove +// that a piece of data existed at a specific block height. It combines a +// trillian log (tlog) and a key-value store. When data is saved to a tstore +// instance it is first saved to the key-value store then a digest of the data +// is appended onto the tlog tree. Tlog trees are episodically timestamped onto +// the decred blockchain. An inlcusion proof, i.e. the cryptographic proof that +// the data was included in the decred timestamp, can be retrieved for any +// individual piece of data saved to the tstore. // // Saving only the digest of the data to tlog means that we separate the // timestamp from the data itself. This allows us to remove content that is -// deemed undesirable from the key-value store without impacting the ability -// to retrieve inclusion proofs for any other pieces of data saved to tstore. -// -// The key-value store is write once. Edits to data in the key-value store are -// not allowed. Deletes, however, are allowed. +// deemed undesirable from the key-value store without impacting the ability to +// retrieve inclusion proofs for any other pieces of data saved to tstore. // // The tlog tree is append only and is treated as the source of truth. If any // blobs make it into the key-value store but do not make it into the tlog tree @@ -98,7 +97,7 @@ type Tstore struct { plugins map[string]plugin // [pluginID]plugin // droppingAnchor indicates whether tstore is in the process of - // dropping an anchor, i.e. timestamping unanchored trillian trees + // dropping an anchor, i.e. timestamping unanchored tlog trees // using dcrtime. An anchor is dropped periodically using cron. droppingAnchor bool } diff --git a/politeiad/backendv2/tstorebe/tstorebe.go b/politeiad/backendv2/tstorebe/tstorebe.go index 7c69f0b9c..f2c04df58 100644 --- a/politeiad/backendv2/tstorebe/tstorebe.go +++ b/politeiad/backendv2/tstorebe/tstorebe.go @@ -29,7 +29,8 @@ var ( _ backend.Backend = (*tstoreBackend)(nil) ) -// tstoreBackend implements the Backend interface. +// tstoreBackend implements the Backend interface using a tstore as the +// data store. type tstoreBackend struct { sync.RWMutex appDir string @@ -1052,15 +1053,20 @@ func (t *tstoreBackend) Inventory(state backend.StateT, status backend.StatusT, }, nil } -// InventoryTimeOrdered returns a page of record tokens sorted by timestamp of -// their most recent status change. The returned tokens are not sorted by -// status and will included all statuses. +// InventoryOrdered returns a page of record tokens ordered by the timestamp of +// their most recent status change. The returned tokens will include all record +// statuses. // // This function satisfies the Backend interface. -func (t *tstoreBackend) InventoryTimeOrdered(state backend.StateT, pageSize, pageNumber uint32) ([]string, error) { - log.Tracef("InventoryTimeOrdered: %v %v %v", state, pageSize, pageNumber) +func (t *tstoreBackend) InventoryOrdered(state backend.StateT, pageSize, pageNumber uint32) ([]string, error) { + log.Tracef("InventoryOrdered: %v %v %v", state, pageSize, pageNumber) - return nil, fmt.Errorf("not implemented") + tokens, err := t.invOrdered(state, pageSize, pageNumber) + if err != nil { + return nil, err + } + + return tokens, nil } // PluginRegister registers a plugin. diff --git a/politeiad/client/pdv2.go b/politeiad/client/pdv2.go index 39f5efe1e..26bd7c9c2 100644 --- a/politeiad/client/pdv2.go +++ b/politeiad/client/pdv2.go @@ -259,6 +259,40 @@ func (c *Client) Inventory(ctx context.Context, state pdv2.RecordStateT, status return &ir, nil } +// InventoryOrdered sends a InventoryOrdered command to the politeiad v2 API. +func (c *Client) InventoryOrdered(ctx context.Context, state pdv2.RecordStateT, page uint32) ([]string, error) { + // Setup request + challenge, err := util.Random(pdv2.ChallengeSize) + if err != nil { + return nil, err + } + i := pdv2.InventoryOrdered{ + Challenge: hex.EncodeToString(challenge), + State: state, + Page: page, + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, + pdv2.APIRoute, pdv2.RouteInventoryOrdered, i) + if err != nil { + return nil, err + } + + // Decode reply + var ir pdv2.InventoryOrderedReply + err = json.Unmarshal(resBody, &ir) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(c.pid, challenge, ir.Response) + if err != nil { + return nil, err + } + + return ir.Tokens, nil +} + // PluginWrite sends a PluginWrite command to the politeiad v2 API. func (c *Client) PluginWrite(ctx context.Context, cmd pdv2.PluginCmd) (string, error) { // Setup request diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index c6b7bef80..e28b01403 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -235,6 +235,8 @@ func (p *politeia) setupBackendTstore(anp *chaincfg.Params) error { p.handleRecordTimestamps, permissionPublic) p.addRouteV2(http.MethodPost, v2.RouteInventory, p.handleInventory, permissionPublic) + p.addRouteV2(http.MethodPost, v2.RouteInventoryOrdered, + p.handleInventoryOrdered, permissionPublic) p.addRouteV2(http.MethodPost, v2.RoutePluginWrite, p.handlePluginWrite, permissionPublic) p.addRouteV2(http.MethodPost, v2.RoutePluginReads, @@ -242,6 +244,9 @@ func (p *politeia) setupBackendTstore(anp *chaincfg.Params) error { p.addRouteV2(http.MethodPost, v2.RoutePluginInventory, p.handlePluginInventory, permissionPublic) + p.addRouteV2(http.MethodPost, v2.RoutePluginInventory, + p.handlePluginInventory, permissionPublic) + // Setup plugins if len(p.cfg.Plugins) > 0 { // Parse plugin settings diff --git a/politeiad/v2.go b/politeiad/v2.go index 5e7d48901..dcdea9420 100644 --- a/politeiad/v2.go +++ b/politeiad/v2.go @@ -356,13 +356,36 @@ func (p *politeia) handleInventory(w http.ResponseWriter, r *http.Request) { return } - // Get inventory + // Verify inventory arguments. These arguments are optional. Only + // return an error if the arguments have been provided. var ( - state = backendv2.StateT(i.State) - status = backendv2.StatusT(i.Status) + state backendv2.StateT + status backendv2.StatusT pageSize = v2.InventoryPageSize pageNumber = i.Page ) + if i.State != v2.RecordStateInvalid { + state = convertRecordStateToBackend(i.State) + if state == backendv2.StateInvalid { + respondWithErrorV2(w, r, "", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeRecordStateInvalid, + }) + return + } + } + if i.Status != v2.RecordStatusInvalid { + status = convertRecordStatusToBackend(i.Status) + if status == backendv2.StatusInvalid { + respondWithErrorV2(w, r, "", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeRecordStatusInvalid, + }) + return + } + } + + // Get inventory inv, err := p.backendv2.Inventory(state, status, pageSize, pageNumber) if err != nil { respondWithErrorV2(w, r, @@ -391,6 +414,59 @@ func (p *politeia) handleInventory(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, ir) } +func (p *politeia) handleInventoryOrdered(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleInventoryOrdered") + + // Decode request + var i v2.InventoryOrdered + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&i); err != nil { + respondWithErrorV2(w, r, "handleInventoryOrdered: unmarshal", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeRequestPayloadInvalid, + }) + return + } + challenge, err := hex.DecodeString(i.Challenge) + if err != nil || len(challenge) != v2.ChallengeSize { + respondWithErrorV2(w, r, "handleInventoryOrdered: decode challenge", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeChallengeInvalid, + }) + return + } + + // Verify record state + var state backendv2.StateT + if i.State != v2.RecordStateInvalid { + state = convertRecordStateToBackend(i.State) + if state == backendv2.StateInvalid { + respondWithErrorV2(w, r, "", + v2.UserErrorReply{ + ErrorCode: v2.ErrorCodeRecordStateInvalid, + }) + return + } + } + + // Get inventory + tokens, err := p.backendv2.InventoryOrdered(state, + v2.InventoryPageSize, i.Page) + if err != nil { + respondWithErrorV2(w, r, + "handleInventoryOrdered: InventoryOrdered: %v", err) + return + } + + response := p.identity.SignMessage(challenge) + ir := v2.InventoryOrderedReply{ + Response: hex.EncodeToString(response[:]), + Tokens: tokens, + } + + util.RespondWithJSON(w, http.StatusOK, ir) +} + func (p *politeia) handlePluginWrite(w http.ResponseWriter, r *http.Request) { log.Tracef("handlePluginWrite") @@ -734,6 +810,30 @@ func convertFileTimestampsToV2(files map[string]backendv2.Timestamp) map[string] return fs } +func convertRecordStateToBackend(s v2.RecordStateT) backendv2.StateT { + switch s { + case v2.RecordStateUnvetted: + return backendv2.StateUnvetted + case v2.RecordStateVetted: + return backendv2.StateVetted + } + return backendv2.StateInvalid +} + +func convertRecordStatusToBackend(s v2.RecordStatusT) backendv2.StatusT { + switch s { + case v2.RecordStatusUnreviewed: + return backendv2.StatusUnreviewed + case v2.RecordStatusPublic: + return backendv2.StatusPublic + case v2.RecordStatusCensored: + return backendv2.StatusCensored + case v2.RecordStatusArchived: + return backendv2.StatusArchived + } + return backendv2.StatusInvalid +} + func convertPluginSettingToV2(p backendv2.PluginSetting) v2.PluginSetting { return v2.PluginSetting{ Key: p.Key, diff --git a/politeiawww/api/records/v1/v1.go b/politeiawww/api/records/v1/v1.go index bd7ad6fd3..d6b772d3c 100644 --- a/politeiawww/api/records/v1/v1.go +++ b/politeiawww/api/records/v1/v1.go @@ -11,13 +11,14 @@ const ( APIRoute = "/records/v1" // Record routes - RouteNew = "/new" - RouteEdit = "/edit" - RouteSetStatus = "/setstatus" - RouteDetails = "/details" - RouteRecords = "/records" - RouteInventory = "/inventory" - RouteTimestamps = "/timestamps" + RouteNew = "/new" + RouteEdit = "/edit" + RouteSetStatus = "/setstatus" + RouteDetails = "/details" + RouteTimestamps = "/timestamps" + RouteRecords = "/records" + RouteInventory = "/inventory" + RouteInventoryOrdered = "/inventoryordered" // Metadata routes RouteUserRecords = "/userrecords" @@ -307,6 +308,55 @@ type DetailsReply struct { Record Record `json:"record"` } +// Proof contains an inclusion proof for the digest in the merkle root. All +// digests are hex encoded SHA256 digests. +// +// The ExtraData field is used by certain types of proofs to include additional +// data that is required to validate the proof. +type Proof struct { + Type string `json:"type"` + Digest string `json:"digest"` + MerkleRoot string `json:"merkleroot"` + MerklePath []string `json:"merklepath"` + ExtraData string `json:"extradata"` // JSON encoded +} + +// Timestamp contains all of the data required to verify that a piece of record +// data was timestamped onto the decred blockchain. +// +// All digests are hex encoded SHA256 digests. The merkle root can be found in +// the OP_RETURN of the specified DCR transaction. +// +// TxID, MerkleRoot, and Proofs will only be populated once the merkle root has +// been included in a DCR tx and the tx has 6 confirmations. The Data field +// will not be populated if the data has been censored. +type Timestamp struct { + Data string `json:"data"` // JSON encoded + Digest string `json:"digest"` + TxID string `json:"txid"` + MerkleRoot string `json:"merkleroot"` + Proofs []Proof `json:"proofs"` +} + +// Timestamps requests the timestamps for a specific record version. If the +// version is omitted, the timestamps for the most recent version will be +// returned. +type Timestamps struct { + Token string `json:"token"` + Version uint32 `json:"version,omitempty"` +} + +// TimestampsReply is the reply to the Timestamps command. +type TimestampsReply struct { + RecordMetadata Timestamp `json:"recordmetadata"` + + // map[pluginID]map[streamID]Timestamp + Metadata map[string]map[uint32]Timestamp `json:"metadata"` + + // map[filename]Timestamp + Files map[string]Timestamp `json:"files"` +} + const ( // RecordsPageSize is the maximum number of records that can be // requested in a Records request. @@ -339,8 +389,8 @@ type RecordsReply struct { } const ( - // InventoryPageSize is the maximum number of tokens that will be - // returned for any single status in an inventory reply. + // InventoryPageSize is the number of tokens that will be returned + // per page for all inventory commands. InventoryPageSize uint32 = 20 ) @@ -369,53 +419,18 @@ type InventoryReply struct { Vetted map[string][]string `json:"vetted"` } -// Proof contains an inclusion proof for the digest in the merkle root. All -// digests are hex encoded SHA256 digests. -// -// The ExtraData field is used by certain types of proofs to include additional -// data that is required to validate the proof. -type Proof struct { - Type string `json:"type"` - Digest string `json:"digest"` - MerkleRoot string `json:"merkleroot"` - MerklePath []string `json:"merklepath"` - ExtraData string `json:"extradata"` // JSON encoded +// InventoryOrdered requests a page of record tokens ordered by the timestamp +// of their most recent status change from newest to oldest. The reply will +// include tokens for all record statuses. Unvetted tokens will only be +// returned to admins. +type InventoryOrdered struct { + State RecordStateT `json:"state"` + Page uint32 `json:"page"` } -// Timestamp contains all of the data required to verify that a piece of record -// data was timestamped onto the decred blockchain. -// -// All digests are hex encoded SHA256 digests. The merkle root can be found in -// the OP_RETURN of the specified DCR transaction. -// -// TxID, MerkleRoot, and Proofs will only be populated once the merkle root has -// been included in a DCR tx and the tx has 6 confirmations. The Data field -// will not be populated if the data has been censored. -type Timestamp struct { - Data string `json:"data"` // JSON encoded - Digest string `json:"digest"` - TxID string `json:"txid"` - MerkleRoot string `json:"merkleroot"` - Proofs []Proof `json:"proofs"` -} - -// Timestamps requests the timestamps for a specific record version. If the -// version is omitted, the timestamps for the most recent version will be -// returned. -type Timestamps struct { - Token string `json:"token"` - Version uint32 `json:"version,omitempty"` -} - -// TimestampsReply is the reply to the Timestamps command. -type TimestampsReply struct { - RecordMetadata Timestamp `json:"recordmetadata"` - - // map[pluginID]map[streamID]Timestamp - Metadata map[string]map[uint32]Timestamp `json:"metadata"` - - // map[filename]Timestamp - Files map[string]Timestamp `json:"files"` +// InventoryOrderedReply is the reply to the InventoryOrdered command. +type InventoryOrderedReply struct { + Tokens []string `json:"tokens"` } // UserRecords requests the tokens of all records submitted by a user. Unvetted diff --git a/politeiawww/client/records.go b/politeiawww/client/records.go index b0077b93f..0fea96f54 100644 --- a/politeiawww/client/records.go +++ b/politeiawww/client/records.go @@ -84,6 +84,23 @@ func (c *Client) RecordDetails(d rcv1.Details) (*rcv1.Record, error) { return &dr.Record, nil } +// RecordTimestamps sends a records v1 Timestamps request to politeiawww. +func (c *Client) RecordTimestamps(t rcv1.Timestamps) (*rcv1.TimestampsReply, error) { + resBody, err := c.makeReq(http.MethodPost, + rcv1.APIRoute, rcv1.RouteTimestamps, t) + if err != nil { + return nil, err + } + + var tr rcv1.TimestampsReply + err = json.Unmarshal(resBody, &tr) + if err != nil { + return nil, err + } + + return &tr, nil +} + // Records sends a records v1 Records request to politeiawww. func (c *Client) Records(r rcv1.Records) (map[string]rcv1.Record, error) { resBody, err := c.makeReq(http.MethodPost, @@ -118,21 +135,22 @@ func (c *Client) RecordInventory(i rcv1.Inventory) (*rcv1.InventoryReply, error) return &ir, nil } -// RecordTimestamps sends a records v1 Timestamps request to politeiawww. -func (c *Client) RecordTimestamps(t rcv1.Timestamps) (*rcv1.TimestampsReply, error) { +// RecordInventoryOrdered sends a records v1 InventoryOrdered request to +// politeiawww. +func (c *Client) RecordInventoryOrdered(i rcv1.InventoryOrdered) (*rcv1.InventoryOrderedReply, error) { resBody, err := c.makeReq(http.MethodPost, - rcv1.APIRoute, rcv1.RouteTimestamps, t) + rcv1.APIRoute, rcv1.RouteInventoryOrdered, i) if err != nil { return nil, err } - var tr rcv1.TimestampsReply - err = json.Unmarshal(resBody, &tr) + var ir rcv1.InventoryOrderedReply + err = json.Unmarshal(resBody, &ir) if err != nil { return nil, err } - return &tr, nil + return &ir, nil } // UserRecords sends a records v1 UserRecords request to politeiawww. diff --git a/politeiawww/cmd/pictl/cmdhelp.go b/politeiawww/cmd/pictl/cmdhelp.go index ac4c87a42..b4175961d 100644 --- a/politeiawww/cmd/pictl/cmdhelp.go +++ b/politeiawww/cmd/pictl/cmdhelp.go @@ -22,7 +22,7 @@ type cmdHelp struct { // This function satisfies the go-flags Commander interface. func (c *cmdHelp) Execute(args []string) error { switch c.Args.Command { - // Server commands + // Basic commands case "version": fmt.Printf("%s\n", shared.VersionHelpMsg) case "policy": @@ -34,6 +34,40 @@ func (c *cmdHelp) Execute(args []string) error { case "me": fmt.Printf("%s\n", shared.MeHelpMsg) + // User commands + case "usernew": + fmt.Printf("%s\n", userNewHelpMsg) + case "useredit": + fmt.Printf("%s\n", userEditHelpMsg) + case "userdetails": + fmt.Printf("%s\n", userDetailsHelpMsg) + case "useremailverify": + fmt.Printf("%s\n", userEmailVerifyHelpMsg) + case "userregistrationpayment": + fmt.Printf("%s\n", userRegistrationPaymentHelpMsg) + case "userproposalpaywall": + fmt.Printf("%s\n", userProposalPaywallHelpMsg) + case "userproposalpaywalltx": + fmt.Printf("%s\n", userProposalPaywallTxHelpMsg) + case "userproposalcredits": + fmt.Printf("%s\n", userProposalCreditsHelpMsg) + case "userpaymentsrescan": + fmt.Printf("%s\n", userPaymentsRescanHelpMsg) + case "usermanage": + fmt.Printf("%s\n", shared.UserManageHelpMsg) + case "userkeyupdate": + fmt.Printf("%s\n", shared.UserKeyUpdateHelpMsg) + case "userverificationresend": + fmt.Printf("%s\n", userVerificationResendHelpMsg) + case "userusernamechange": + fmt.Printf("%s\n", shared.UserUsernameChangeHelpMsg) + case "userpasswordchange": + fmt.Printf("%s\n", shared.UserPasswordChangeHelpMsg) + case "userpasswordreset": + fmt.Printf("%s\n", shared.UserPasswordResetHelpMsg) + case "users": + fmt.Printf("%s\n", shared.UsersHelpMsg) + // Proposal commands case "proposalpolicy": fmt.Printf("%s\n", proposalPolicyHelpMsg) @@ -43,12 +77,16 @@ func (c *cmdHelp) Execute(args []string) error { fmt.Printf("%s\n", proposalEditHelpMsg) case "proposalsetstatus": fmt.Printf("%s\n", proposalSetStatusHelpMsg) + case "proposaldetails": + fmt.Printf("%s\n", proposalDetailsHelpMsg) + case "proposaltimestamps": + fmt.Printf("%s\n", proposalTimestampsHelpMsg) case "proposals": fmt.Printf("%s\n", proposalsHelpMsg) case "proposalinv": fmt.Printf("%s\n", proposalInvHelpMsg) - case "proposaltimestamps": - fmt.Printf("%s\n", proposalTimestampsHelpMsg) + case "proposalinvordered": + fmt.Printf("%s\n", proposalInvOrderedHelpMsg) // Comment commands case "commentpolicy": @@ -90,40 +128,6 @@ func (c *cmdHelp) Execute(args []string) error { case "votetimestamps": fmt.Printf("%s\n", voteTimestampsHelpMsg) - // User commands - case "usernew": - fmt.Printf("%s\n", userNewHelpMsg) - case "useredit": - fmt.Printf("%s\n", userEditHelpMsg) - case "userdetails": - fmt.Printf("%s\n", userDetailsHelpMsg) - case "useremailverify": - fmt.Printf("%s\n", userEmailVerifyHelpMsg) - case "userregistrationpayment": - fmt.Printf("%s\n", userRegistrationPaymentHelpMsg) - case "userproposalpaywall": - fmt.Printf("%s\n", userProposalPaywallHelpMsg) - case "userproposalpaywalltx": - fmt.Printf("%s\n", userProposalPaywallTxHelpMsg) - case "userproposalcredits": - fmt.Printf("%s\n", userProposalCreditsHelpMsg) - case "userpaymentsrescan": - fmt.Printf("%s\n", userPaymentsRescanHelpMsg) - case "usermanage": - fmt.Printf("%s\n", shared.UserManageHelpMsg) - case "userkeyupdate": - fmt.Printf("%s\n", shared.UserKeyUpdateHelpMsg) - case "userverificationresend": - fmt.Printf("%s\n", userVerificationResendHelpMsg) - case "userusernamechange": - fmt.Printf("%s\n", shared.UserUsernameChangeHelpMsg) - case "userpasswordchange": - fmt.Printf("%s\n", shared.UserPasswordChangeHelpMsg) - case "userpasswordreset": - fmt.Printf("%s\n", shared.UserPasswordResetHelpMsg) - case "users": - fmt.Printf("%s\n", shared.UsersHelpMsg) - // Websocket commands case "subscribe": fmt.Printf("%s\n", subscribeHelpMsg) diff --git a/politeiawww/cmd/pictl/cmdproposalinv.go b/politeiawww/cmd/pictl/cmdproposalinv.go index abcb8fe24..b2323649e 100644 --- a/politeiawww/cmd/pictl/cmdproposalinv.go +++ b/politeiawww/cmd/pictl/cmdproposalinv.go @@ -104,10 +104,14 @@ record tokens. If no status is specified then a page of tokens for each status are returned. The state and page arguments will be ignored. +Valid states: + (1) unvetted + (2) vetted + Valid statuses: - public - censored - abandoned + (2) public + (3) censored + (4) abandoned Arguments: 1. state (string, optional) State of tokens being requested. diff --git a/politeiawww/cmd/pictl/cmdproposalinvordered.go b/politeiawww/cmd/pictl/cmdproposalinvordered.go new file mode 100644 index 000000000..28b0d1fc5 --- /dev/null +++ b/politeiawww/cmd/pictl/cmdproposalinvordered.go @@ -0,0 +1,104 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + pclient "github.com/decred/politeia/politeiawww/client" +) + +// cmdProposalInvOrdered retrieves a page of chronologically ordered censorship +// record tokens. The tokens will include records of all statuses. +type cmdProposalInvOrdered struct { + Args struct { + State string `positional-arg-name:"state"` + Page uint32 `positional-arg-name:"page"` + } `positional-args:"true" optional:"true"` +} + +// Execute executes the cmdProposalInvOrdered command. +// +// This function satisfies the go-flags Commander interface. +func (c *cmdProposalInvOrdered) Execute(args []string) error { + _, err := proposalInvOrdered(c) + if err != nil { + return err + } + return nil +} + +// proposalInvOrdered retrieves a page of chronologically ordered proposal +// tokens. This function has been pulled out of the Execute method so that it +// can be used in test commands. +func proposalInvOrdered(c *cmdProposalInvOrdered) (*rcv1.InventoryOrderedReply, error) { + // Setup client + opts := pclient.Opts{ + HTTPSCert: cfg.HTTPSCert, + Cookies: cfg.Cookies, + HeaderCSRF: cfg.CSRF, + Verbose: cfg.Verbose, + RawJSON: cfg.RawJSON, + } + pc, err := pclient.New(cfg.Host, opts) + if err != nil { + return nil, err + } + + // Setup state + var state rcv1.RecordStateT + if c.Args.State != "" { + // A state was provided. This can be either the numeric state + // code or the human readable equivalent. + state, err = parseRecordState(c.Args.State) + if err != nil { + return nil, err + } + } else { + // No state was provided. Default to vetted. + state = rcv1.RecordStateVetted + } + + // If a status was given but no page number was given, default to + // page 1. + if c.Args.Page == 0 { + c.Args.Page = 1 + } + + // Get inventory + i := rcv1.InventoryOrdered{ + State: state, + Page: c.Args.Page, + } + ir, err := pc.RecordInventoryOrdered(i) + if err != nil { + return nil, err + } + + // Print inventory + printJSON(ir) + + return ir, nil +} + +// proposalInvOrderedHelpMsg is printed to stdout by the help command. +const proposalInvOrderedHelpMsg = `proposalinvordered + +Inventory ordered returns a page of record tokens ordered by the timestamp of +their most recent status change from newest to oldest. The reply will include +tokens for all record statuses. Unvetted tokens will only be returned to +admins. + +If no state is provided this command defaults to requesting vetted tokens. + +If no page number is provided this command defaults to requesting page 1. + +Valid states: + (1) unvetted + (2) vetted + +Arguments: +1. state (string, optional) State of tokens being requested. +2. page (uint32, optional) Page number. +` diff --git a/politeiawww/cmd/pictl/pictl.go b/politeiawww/cmd/pictl/pictl.go index b53f67bb6..491f775aa 100644 --- a/politeiawww/cmd/pictl/pictl.go +++ b/politeiawww/cmd/pictl/pictl.go @@ -69,9 +69,10 @@ type pictl struct { ProposalEdit cmdProposalEdit `command:"proposaledit"` ProposalSetStatus cmdProposalSetStatus `command:"proposalsetstatus"` ProposalDetails cmdProposalDetails `command:"proposaldetails"` + ProposalTimestamps cmdProposalTimestamps `command:"proposaltimestamps"` Proposals cmdProposals `command:"proposals"` ProposalInv cmdProposalInv `command:"proposalinv"` - ProposalTimestamps cmdProposalTimestamps `command:"proposaltimestamps"` + ProposalInvOrdered cmdProposalInvOrdered `command:"proposalinvordered"` UserProposals cmdUserProposals `command:"userproposals"` // Comments commands @@ -155,9 +156,10 @@ Proposal commands proposaledit (user) Edit an existing proposal proposalstatusset (admin) Set the status of a proposal proposaldetails (public) Get a full proposal record + proposaltimestamps (public) Get timestamps for a proposal proposals (public) Get proposals without their files proposalinv (public) Get inventory by proposal status - proposaltimestamps (public) Get timestamps for a proposal + proposalinvordered (public) Get inventory ordered chronologically userproposals (public) Get proposals submitted by a user Comment commands diff --git a/politeiawww/pi.go b/politeiawww/pi.go index fa1820e49..929b713f3 100644 --- a/politeiawww/pi.go +++ b/politeiawww/pi.go @@ -80,6 +80,9 @@ func (p *politeiawww) setupPiRoutes(r *records.Records, c *comments.Comments, t p.addRoute(http.MethodPost, rcv1.APIRoute, rcv1.RouteDetails, r.HandleDetails, permissionPublic) + p.addRoute(http.MethodPost, rcv1.APIRoute, + rcv1.RouteTimestamps, r.HandleTimestamps, + permissionPublic) p.addRoute(http.MethodPost, rcv1.APIRoute, rcv1.RouteRecords, r.HandleRecords, permissionPublic) @@ -87,7 +90,7 @@ func (p *politeiawww) setupPiRoutes(r *records.Records, c *comments.Comments, t rcv1.RouteInventory, r.HandleInventory, permissionPublic) p.addRoute(http.MethodPost, rcv1.APIRoute, - rcv1.RouteTimestamps, r.HandleTimestamps, + rcv1.RouteInventoryOrdered, r.HandleInventoryOrdered, permissionPublic) p.addRoute(http.MethodPost, rcv1.APIRoute, rcv1.RouteUserRecords, r.HandleUserRecords, diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index a5719b1eb..b52a2d84b 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -274,6 +274,63 @@ func (r *Records) processDetails(ctx context.Context, d v1.Details, u *user.User }, nil } +func (r *Records) processTimestamps(ctx context.Context, t v1.Timestamps, isAdmin bool) (*v1.TimestampsReply, error) { + log.Tracef("processTimestamps: %v %v", t.Token, t.Version) + + // Get record timestamps + rt, err := r.politeiad.RecordTimestamps(ctx, t.Token, t.Version) + if err != nil { + return nil, err + } + + var ( + recordMD = convertTimestampToV1(rt.RecordMetadata) + metadata = make(map[string]map[uint32]v1.Timestamp, len(rt.Files)) + files = make(map[string]v1.Timestamp, len(rt.Files)) + ) + for pluginID, v := range rt.Metadata { + streams, ok := metadata[pluginID] + if !ok { + streams = make(map[uint32]v1.Timestamp, 16) + } + for streamID, ts := range v { + streams[streamID] = convertTimestampToV1(ts) + } + metadata[pluginID] = streams + } + for k, v := range rt.Files { + files[k] = convertTimestampToV1(v) + } + + // Get the record. We need to know the record state. + rc, err := r.record(ctx, t.Token, t.Version) + if err != nil { + return nil, err + } + + // Unvetted data blobs are stripped if the user is not an admin. + // The rest of the timestamp is still returned. + if rc.State != v1.RecordStateVetted && !isAdmin { + recordMD.Data = "" + for k, v := range files { + v.Data = "" + files[k] = v + } + for _, streams := range metadata { + for streamID, ts := range streams { + ts.Data = "" + streams[streamID] = ts + } + } + } + + return &v1.TimestampsReply{ + RecordMetadata: recordMD, + Files: files, + Metadata: metadata, + }, nil +} + func (r *Records) processRecords(ctx context.Context, rs v1.Records, u *user.User) (*v1.RecordsReply, error) { log.Tracef("processRecords: %v reqs", len(rs.Requests)) @@ -348,8 +405,8 @@ func (r *Records) processInventory(ctx context.Context, i v1.Inventory, u *user. return nil, err } - // Only admins are allowed to retrieve unvetted tokens. A user may - // or may not exist. + // Only admins are allowed to retrieve unvetted tokens. This is a + // public route so a user may or may not exist. if u == nil || !u.Admin { ir.Unvetted = map[string][]string{} } @@ -360,60 +417,34 @@ func (r *Records) processInventory(ctx context.Context, i v1.Inventory, u *user. }, nil } -func (r *Records) processTimestamps(ctx context.Context, t v1.Timestamps, isAdmin bool) (*v1.TimestampsReply, error) { - log.Tracef("processTimestamps: %v %v", t.Token, t.Version) +func (r *Records) processInventoryOrdered(ctx context.Context, i v1.InventoryOrdered, u *user.User) (*v1.InventoryOrderedReply, error) { + log.Tracef("processInventoryOrdered: %v %v", i.State, i.Page) - // Get record timestamps - rt, err := r.politeiad.RecordTimestamps(ctx, t.Token, t.Version) - if err != nil { - return nil, err - } - - var ( - recordMD = convertTimestampToV1(rt.RecordMetadata) - metadata = make(map[string]map[uint32]v1.Timestamp, len(rt.Files)) - files = make(map[string]v1.Timestamp, len(rt.Files)) - ) - for pluginID, v := range rt.Metadata { - streams, ok := metadata[pluginID] - if !ok { - streams = make(map[uint32]v1.Timestamp, 16) - } - for streamID, ts := range v { - streams[streamID] = convertTimestampToV1(ts) + // Verify state + state := convertStateToPD(i.State) + if state == pdv2.RecordStateInvalid { + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeRecordStateInvalid, } - metadata[pluginID] = streams } - for k, v := range rt.Files { - files[k] = convertTimestampToV1(v) + + // Only admins are allowed to retrieve unvetted tokens. This is a + // public route so a user may or may not exist. + isAdmin := u != nil && u.Admin + if state == pdv2.RecordStateUnvetted && !isAdmin { + return &v1.InventoryOrderedReply{ + Tokens: []string{}, + }, nil } - // Get the record. We need to know the record state. - rc, err := r.record(ctx, t.Token, t.Version) + // Get inventory + tokens, err := r.politeiad.InventoryOrdered(ctx, state, i.Page) if err != nil { return nil, err } - // Unvetted data blobs are stripped if the user is not an admin. - // The rest of the timestamp is still returned. - if rc.State != v1.RecordStateVetted && !isAdmin { - recordMD.Data = "" - for k, v := range files { - v.Data = "" - files[k] = v - } - for _, streams := range metadata { - for streamID, ts := range streams { - ts.Data = "" - streams[streamID] = ts - } - } - } - - return &v1.TimestampsReply{ - RecordMetadata: recordMD, - Files: files, - Metadata: metadata, + return &v1.InventoryOrderedReply{ + Tokens: tokens, }, nil } diff --git a/politeiawww/records/records.go b/politeiawww/records/records.go index 2223408c8..e27d02938 100644 --- a/politeiawww/records/records.go +++ b/politeiawww/records/records.go @@ -152,6 +152,40 @@ func (c *Records) HandleDetails(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, dr) } +// HandleTimestamps is the request handler for the records v1 Timestamps route. +func (c *Records) HandleTimestamps(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleTimestamps") + + var t v1.Timestamps + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&t); err != nil { + respondWithError(w, r, "HandleTimestamps: unmarshal", + v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeInputInvalid, + }) + return + } + + // Lookup session user. This is a public route so a session may not + // exist. Ignore any session not found errors. + u, err := c.sessions.GetSessionUser(w, r) + if err != nil && err != sessions.ErrSessionNotFound { + respondWithError(w, r, + "HandleTimestamps: getSessionUser: %v", err) + return + } + + isAdmin := u != nil && u.Admin + tr, err := c.processTimestamps(r.Context(), t, isAdmin) + if err != nil { + respondWithError(w, r, + "HandleTimestamps: processTimestamps: %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, tr) +} + // HandleRecords is the request handler for the records v1 Records route. func (c *Records) HandleRecords(w http.ResponseWriter, r *http.Request) { log.Tracef("HandleRecords") @@ -218,14 +252,15 @@ func (c *Records) HandleInventory(w http.ResponseWriter, r *http.Request) { util.RespondWithJSON(w, http.StatusOK, ir) } -// HandleTimestamps is the request handler for the records v1 Timestamps route. -func (c *Records) HandleTimestamps(w http.ResponseWriter, r *http.Request) { - log.Tracef("HandleTimestamps") +// HandleInventoryOrdered is the request handler for the records v1 +// InventoryOrdered route. +func (c *Records) HandleInventoryOrdered(w http.ResponseWriter, r *http.Request) { + log.Tracef("HandleInventoryOrdered") - var t v1.Timestamps + var i v1.InventoryOrdered decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&t); err != nil { - respondWithError(w, r, "HandleTimestamps: unmarshal", + if err := decoder.Decode(&i); err != nil { + respondWithError(w, r, "HandleInventoryOrdered: unmarshal", v1.UserErrorReply{ ErrorCode: v1.ErrorCodeInputInvalid, }) @@ -237,19 +272,18 @@ func (c *Records) HandleTimestamps(w http.ResponseWriter, r *http.Request) { u, err := c.sessions.GetSessionUser(w, r) if err != nil && err != sessions.ErrSessionNotFound { respondWithError(w, r, - "HandleTimestamps: getSessionUser: %v", err) + "HandleInventoryOrdered: GetSessionUser: %v", err) return } - isAdmin := u != nil && u.Admin - tr, err := c.processTimestamps(r.Context(), t, isAdmin) + ir, err := c.processInventoryOrdered(r.Context(), i, u) if err != nil { respondWithError(w, r, - "HandleTimestamps: processTimestamps: %v", err) + "HandleInventoryOrdered: processInventoryOrdered: %v", err) return } - util.RespondWithJSON(w, http.StatusOK, tr) + util.RespondWithJSON(w, http.StatusOK, ir) } // HandleUserRecords is the request handler for the records v1 UserRecords From 7436e239e533d85c58a241cdf05e46c04ad689f6 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 21 Mar 2021 14:28:39 -0500 Subject: [PATCH 420/449] politeiawww: Add legacy www support. --- politeiawww/api/www/v1/v1.go | 10 +-- politeiawww/pi.go | 3 + politeiawww/proposals.go | 124 ++++++++++++++++++++++++++++++++++- 3 files changed, 131 insertions(+), 6 deletions(-) diff --git a/politeiawww/api/www/v1/v1.go b/politeiawww/api/www/v1/v1.go index 16c3aa94a..b3d79f7e1 100644 --- a/politeiawww/api/www/v1/v1.go +++ b/politeiawww/api/www/v1/v1.go @@ -61,6 +61,7 @@ const ( RouteBatchProposals = "/proposals/batch" RouteActiveVote = "/proposals/activevote" RouteVoteResults = "/proposals/{token:[A-Fa-f0-9]{7,64}}/votes" + RouteAllVoteStatus = "/proposals/votestatus" RouteCastVotes = "/proposals/castvotes" RouteBatchVoteSummary = "/proposals/batchvotesummary" @@ -69,7 +70,6 @@ const ( RouteEditProposal = "/proposals/edit" RouteAuthorizeVote = "/proposals/authorizevote" RouteStartVote = "/proposals/startvote" - RouteAllVoteStatus = "/proposals/votestatus" RouteSetProposalStatus = "/proposals/{token:[A-Fa-f0-9]{7,64}}/status" RouteCommentsGet = "/proposals/{token:[A-Fa-f0-9]{7,64}}/comments" RouteVoteStatus = "/proposals/{token:[A-Fa-f0-9]{7,64}}/votestatus" @@ -955,7 +955,9 @@ type SetProposalStatusReply struct { // If Before is specified, the "page" returned starts before the provided // proposal censorship token, when sorted in reverse chronological order. // -// This request is DEPRECATED. +// This request is DEPRECATED. The Before and After arguments are NO LONGER +// SUPPORTED. This route will only return a single page of vetted tokens. The +// records API InventoryOrdered command should be used instead. type GetAllVetted struct { Before string `schema:"before"` After string `schema:"after"` @@ -1293,12 +1295,12 @@ type VoteStatusReply struct { // GetAllVoteStatus attempts to fetch the vote status of all public propsals // -// This request is NO LONGER SUPPORTED. +// This request is DEPRECATED. type GetAllVoteStatus struct{} // GetAllVoteStatusReply returns the vote status of all public proposals // -// This request is NO LONGER SUPPORTED. +// This request is DEPRECATED. type GetAllVoteStatusReply struct { VotesStatus []VoteStatusReply `json:"votesstatus"` // Vote status of all public proposals } diff --git a/politeiawww/pi.go b/politeiawww/pi.go index 929b713f3..aceef8296 100644 --- a/politeiawww/pi.go +++ b/politeiawww/pi.go @@ -54,6 +54,9 @@ func (p *politeiawww) setupPiRoutes(r *records.Records, c *comments.Comments, t p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, www.RouteBatchProposals, p.handleBatchProposals, permissionPublic) + p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, + www.RouteAllVoteStatus, p.handleAllVoteStatus, + permissionPublic) p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, www.RouteActiveVote, p.handleActiveVote, permissionPublic) diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 648d549b4..194bcd038 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -10,7 +10,6 @@ import ( "encoding/hex" "encoding/json" "errors" - "fmt" "io" "net/http" "strconv" @@ -164,7 +163,50 @@ func (p *politeiawww) processTokenInventory(ctx context.Context, isAdmin bool) ( func (p *politeiawww) processAllVetted(ctx context.Context, gav www.GetAllVetted) (*www.GetAllVettedReply, error) { log.Tracef("processAllVetted: %v %v", gav.Before, gav.After) - return nil, fmt.Errorf("not implemented yet") + // NOTE: this route is not scalable and needs to be removed ASAP. + // It only needs to be supported to give dcrdata a change to switch + // to the records API. + + // The Before and After arguments are NO LONGER SUPPORTED. This + // route will only return a single page of vetted tokens. The + // records API InventoryOrdered command should be used instead. + tokens, err := p.politeiad.InventoryOrdered(ctx, pdv2.RecordStateVetted, 1) + if err != nil { + return nil, err + } + + // Get the records without any files + reqs := make([]pdv2.RecordRequest, 0, pdv2.RecordsPageSize) + proposals := make([]www.ProposalRecord, 0, len(tokens)) + for _, v := range tokens { + reqs = append(reqs, pdv2.RecordRequest{ + Token: v, + OmitAllFiles: true, + }) + + // The records request must be broken up because the records + // page size is much smaller than the inventory page size. + if len(reqs) == int(pdv2.RecordsPageSize) { + // Get this batch of proposals + props, err := p.proposals(ctx, reqs) + if err != nil { + return nil, err + } + for _, v := range reqs { + pr, ok := props[v.Token] + if !ok { + proposals = append(proposals, pr) + } + } + + // Reset requesets + reqs = make([]pdv2.RecordRequest, 0, pdv2.RecordsPageSize) + } + } + + return &www.GetAllVettedReply{ + Proposals: proposals, + }, nil } func (p *politeiawww) processProposalDetails(ctx context.Context, pd www.ProposalsDetails, u *user.User) (*www.ProposalDetailsReply, error) { @@ -284,6 +326,61 @@ func (p *politeiawww) processBatchVoteSummary(ctx context.Context, bvs www.Batch }, nil } +func (p *politeiawww) processAllVoteStatus(ctx context.Context, avs www.GetAllVoteStatus) (*www.GetAllVoteStatusReply, error) { + log.Tracef("processAllVoteStatus") + + // NOTE: This route is suppose to return the vote status of all + // public proposals. This is horrendously unscalable. We are only + // supporting this route until dcrdata has a chance to update and + // use the ticketvote API. Until then, we only return a single page + // of vote statuses. + + // Get a page of vetted records + tokens, err := p.politeiad.InventoryOrdered(ctx, pdv2.RecordStateVetted, 1) + if err != nil { + return nil, err + } + + // Get vote summaries + vs, err := p.politeiad.TicketVoteSummaries(ctx, tokens) + if err != nil { + return nil, err + } + + // Prepare reply + statuses := make([]www.VoteStatusReply, 0, len(vs)) + var totalVotes uint64 + for token, v := range vs { + results := make([]www.VoteOptionResult, len(v.Results)) + for k, r := range v.Results { + results[k] = www.VoteOptionResult{ + VotesReceived: r.Votes, + Option: www.VoteOption{ + Id: r.ID, + Description: r.Description, + Bits: r.VoteBit, + }, + } + totalVotes += r.Votes + } + statuses = append(statuses, www.VoteStatusReply{ + Token: token, + Status: convertVoteStatusToWWW(v.Status), + TotalVotes: totalVotes, + OptionsResult: results, + EndHeight: strconv.FormatUint(uint64(v.EndBlockHeight), 10), + BestBlock: strconv.FormatUint(uint64(v.BestBlock), 10), + NumOfEligibleVotes: int(v.EligibleTickets), + QuorumPercentage: v.QuorumPercentage, + PassPercentage: v.PassPercentage, + }) + } + + return &www.GetAllVoteStatusReply{ + VotesStatus: statuses, + }, nil +} + func convertVoteDetails(vd tkplugin.VoteDetails) (www.StartVote, www.StartVoteReply) { options := make([]www.VoteOption, 0, len(vd.Params.Options)) for _, v := range vd.Params.Options { @@ -616,6 +713,29 @@ func (p *politeiawww) handleBatchVoteSummary(w http.ResponseWriter, r *http.Requ util.RespondWithJSON(w, http.StatusOK, reply) } +func (p *politeiawww) handleAllVoteStatus(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleAllVoteStatus") + + var avs www.GetAllVoteStatus + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&avs); err != nil { + RespondWithError(w, r, 0, "handleAllVoteStatus: unmarshal", + www.UserError{ + ErrorCode: www.ErrorStatusInvalidInput, + }) + return + } + + reply, err := p.processAllVoteStatus(r.Context(), avs) + if err != nil { + RespondWithError(w, r, 0, + "handleAllVoteStatus: processAllVoteStatus %v", err) + return + } + + util.RespondWithJSON(w, http.StatusOK, reply) +} + func (p *politeiawww) handleActiveVote(w http.ResponseWriter, r *http.Request) { log.Tracef("handleActiveVote") From 05c1dae065ef160397037a46a20a813f4d7a5bf9 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 21 Mar 2021 16:27:03 -0500 Subject: [PATCH 421/449] politeiawww: Add and test remaining legacy routes. --- politeiawww/api/www/v1/v1.go | 12 +-- politeiawww/cmd/pictl/cmdlegacytest.go | 94 +++++++++++++++++ politeiawww/cmd/pictl/pictl.go | 2 + politeiawww/cmd/shared/client.go | 62 +++++------ politeiawww/pi.go | 3 + politeiawww/proposals.go | 140 ++++++++++++++++--------- 6 files changed, 222 insertions(+), 91 deletions(-) create mode 100644 politeiawww/cmd/pictl/cmdlegacytest.go diff --git a/politeiawww/api/www/v1/v1.go b/politeiawww/api/www/v1/v1.go index b3d79f7e1..51c0005e3 100644 --- a/politeiawww/api/www/v1/v1.go +++ b/politeiawww/api/www/v1/v1.go @@ -59,11 +59,12 @@ const ( RouteProposalDetails = "/proposals/{token:[A-Fa-f0-9]{7,64}}" RouteAllVetted = "/proposals/vetted" RouteBatchProposals = "/proposals/batch" - RouteActiveVote = "/proposals/activevote" - RouteVoteResults = "/proposals/{token:[A-Fa-f0-9]{7,64}}/votes" + RouteVoteStatus = "/proposals/{token:[A-Fa-f0-9]{7,64}}/votestatus" RouteAllVoteStatus = "/proposals/votestatus" - RouteCastVotes = "/proposals/castvotes" RouteBatchVoteSummary = "/proposals/batchvotesummary" + RouteActiveVote = "/proposals/activevote" + RouteCastVotes = "/proposals/castvotes" + RouteVoteResults = "/proposals/{token:[A-Fa-f0-9]{7,64}}/votes" // The following routes are NO LONGER SUPPORTED. RouteNewProposal = "/proposals/new" @@ -72,7 +73,6 @@ const ( RouteStartVote = "/proposals/startvote" RouteSetProposalStatus = "/proposals/{token:[A-Fa-f0-9]{7,64}}/status" RouteCommentsGet = "/proposals/{token:[A-Fa-f0-9]{7,64}}/comments" - RouteVoteStatus = "/proposals/{token:[A-Fa-f0-9]{7,64}}/votestatus" RouteNewComment = "/comments/new" RouteLikeComment = "/comments/like" RouteCensorComment = "/comments/censor" @@ -1275,12 +1275,12 @@ type VoteOptionResult struct { // VoteStatus is a command to fetch the the current vote status for a single // public proposal // -// This request is NO LONGER SUPPORTED. +// This request is DEPRECATED. type VoteStatus struct{} // VoteStatusReply describes the vote status for a given proposal // -// This request is NO LONGER SUPPORTED. +// This request is DEPRECATED. type VoteStatusReply struct { Token string `json:"token"` // Censorship token Status PropVoteStatusT `json:"status"` // Vote status (finished, started, etc) diff --git a/politeiawww/cmd/pictl/cmdlegacytest.go b/politeiawww/cmd/pictl/cmdlegacytest.go new file mode 100644 index 000000000..2f12d6b46 --- /dev/null +++ b/politeiawww/cmd/pictl/cmdlegacytest.go @@ -0,0 +1,94 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + + www "github.com/decred/politeia/politeiawww/api/www/v1" +) + +type cmdLegacyTest struct { +} + +// Execute executes the cmdLegacyTest command. +// +// This function satisfies the go-flags Commander interface. +func (c *cmdLegacyTest) Execute(args []string) error { + printf("Policy\n") + pr, err := client.Policy() + if err != nil { + return err + } + printJSON(pr) + + printf("Token inventory\n") + tir, err := client.TokenInventory() + if err != nil { + return err + } + printJSON(tir) + + printf("All vetted\n") + avr, err := client.GetAllVetted(&www.GetAllVetted{}) + if err != nil { + return err + } + printJSON(avr) + + if len(tir.Pre) == 0 { + return fmt.Errorf("no proposals found; cannot get proposal details") + } + token := tir.Pre[0] + + printf("Proposal details %v\n", token) + pdr, err := client.ProposalDetails(token, &www.ProposalsDetails{}) + if err != nil { + return err + } + printJSON(pdr) + + printf("Batch proposals %v\n", token) + bp := www.BatchProposals{ + Tokens: []string{token}, + } + bpr, err := client.BatchProposals(&bp) + if err != nil { + return err + } + printJSON(bpr) + + printf("All vote status\n") + avsr, err := client.GetAllVoteStatus() + if err != nil { + return err + } + printJSON(avsr) + + if len(tir.Approved) == 0 { + return fmt.Errorf("no vote approvals found; cannot get vote status") + } + token = tir.Approved[0] + + printf("Vote status %v\n", token) + vsr, err := client.VoteStatus(token) + if err != nil { + return err + } + printJSON(vsr) + + printf("Vote results %v\n", token) + vrr, err := client.VoteResults(token) + if err != nil { + return err + } + vrr.StartVoteReply.EligibleTickets = []string{ + fmt.Sprintf("%v ticket hashes removed for readability", + len(vrr.StartVoteReply.EligibleTickets)), + } + printJSON(vrr) + + return nil +} diff --git a/politeiawww/cmd/pictl/pictl.go b/politeiawww/cmd/pictl/pictl.go index 491f775aa..5232a7d04 100644 --- a/politeiawww/cmd/pictl/pictl.go +++ b/politeiawww/cmd/pictl/pictl.go @@ -106,6 +106,7 @@ type pictl struct { SeedProposals cmdSeedProposals `command:"seedproposals"` VoteTestSetup cmdVoteTestSetup `command:"votetestsetup"` VoteTest cmdVoteTest `command:"votetest"` + LegacyTest cmdLegacyTest `command:"legacytest"` // Legacy www routes (deprecated) TokenInventory shared.TokenInventoryCmd `command:"tokeninventory"` @@ -193,6 +194,7 @@ Dev commands seedproposals Seed the backend with proposals votetestsetup Setup a vote test votetest Execute a vote test + legacytest Test legacy routes that do not have a command ` func _main() error { diff --git a/politeiawww/cmd/shared/client.go b/politeiawww/cmd/shared/client.go index bb9340a9f..a379b87c1 100644 --- a/politeiawww/cmd/shared/client.go +++ b/politeiawww/cmd/shared/client.go @@ -19,7 +19,6 @@ import ( "decred.org/dcrwallet/rpc/walletrpc" cms "github.com/decred/politeia/politeiawww/api/cms/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" - www2 "github.com/decred/politeia/politeiawww/api/www/v2" "github.com/decred/politeia/util" "github.com/gorilla/schema" "golang.org/x/net/publicsuffix" @@ -1235,39 +1234,6 @@ func (c *Client) UserRegistrationPayment() (*www.UserRegistrationPaymentReply, e return &urpr, nil } -// VoteDetailsV2 returns the proposal vote details for the given token using -// the www v2 VoteDetails route. -func (c *Client) VoteDetailsV2(token string) (*www2.VoteDetailsReply, error) { - route := "/vote/" + token - statusCode, respBody, err := c.makeRequest(http.MethodGet, www2.APIRoute, - route, nil) - if err != nil { - return nil, err - } - - if statusCode != http.StatusOK { - return nil, wwwError(respBody, statusCode) - } - - var vdr www2.VoteDetailsReply - err = json.Unmarshal(respBody, &vdr) - if err != nil { - return nil, err - } - - if c.cfg.Verbose { - vdr.EligibleTickets = []string{ - "removed by piwww for readability", - } - err = prettyPrintJSON(vdr) - if err != nil { - return nil, err - } - } - - return &vdr, nil -} - // UserDetails retrieves the user details for the specified user. func (c *Client) UserDetails(userID string) (*www.UserDetailsReply, error) { route := "/user/" + userID @@ -1440,6 +1406,34 @@ func (c *Client) VoteStatus(token string) (*www.VoteStatusReply, error) { return &vsr, nil } +// VoteResults retrieves the vote results for a proposal. +func (c *Client) VoteResults(token string) (*www.VoteResultsReply, error) { + route := "/proposals/" + token + "/votes" + statusCode, respBody, err := c.makeRequest(http.MethodGet, + www.PoliteiaWWWAPIRoute, route, nil) + if err != nil { + return nil, err + } + if statusCode != http.StatusOK { + return nil, wwwError(respBody, statusCode) + } + + var vsr www.VoteResultsReply + err = json.Unmarshal(respBody, &vsr) + if err != nil { + return nil, fmt.Errorf("unmarshal VoteResultsReply: %v", err) + } + + if c.cfg.Verbose { + err := prettyPrintJSON(vsr) + if err != nil { + return nil, err + } + } + + return &vsr, nil +} + // GetAllVoteStatus retreives the vote status of all public proposals. func (c *Client) GetAllVoteStatus() (*www.GetAllVoteStatusReply, error) { statusCode, respBody, err := c.makeRequest(http.MethodGet, diff --git a/politeiawww/pi.go b/politeiawww/pi.go index aceef8296..3e26e1ef1 100644 --- a/politeiawww/pi.go +++ b/politeiawww/pi.go @@ -54,6 +54,9 @@ func (p *politeiawww) setupPiRoutes(r *records.Records, c *comments.Comments, t p.addRoute(http.MethodPost, www.PoliteiaWWWAPIRoute, www.RouteBatchProposals, p.handleBatchProposals, permissionPublic) + p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, + www.RouteVoteStatus, p.handleVoteStatus, + permissionPublic) p.addRoute(http.MethodGet, www.PoliteiaWWWAPIRoute, www.RouteAllVoteStatus, p.handleAllVoteStatus, permissionPublic) diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 194bcd038..0333643c2 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -180,8 +180,11 @@ func (p *politeiawww) processAllVetted(ctx context.Context, gav www.GetAllVetted proposals := make([]www.ProposalRecord, 0, len(tokens)) for _, v := range tokens { reqs = append(reqs, pdv2.RecordRequest{ - Token: v, - OmitAllFiles: true, + Token: v, + Filenames: []string{ + piplugin.FileNameProposalMetadata, + tkplugin.FileNameVoteMetadata, + }, }) // The records request must be broken up because the records @@ -195,8 +198,9 @@ func (p *politeiawww) processAllVetted(ctx context.Context, gav www.GetAllVetted for _, v := range reqs { pr, ok := props[v.Token] if !ok { - proposals = append(proposals, pr) + continue } + proposals = append(proposals, pr) } // Reset requesets @@ -213,10 +217,14 @@ func (p *politeiawww) processProposalDetails(ctx context.Context, pd www.Proposa log.Tracef("processProposalDetails: %v", pd.Token) // Parse version - v, err := strconv.ParseUint(pd.Version, 10, 64) - if err != nil { - return nil, www.UserError{ - ErrorCode: www.ErrorStatusProposalNotFound, + var version uint64 + var err error + if pd.Version != "" { + version, err = strconv.ParseUint(pd.Version, 10, 64) + if err != nil { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + } } } @@ -224,7 +232,7 @@ func (p *politeiawww) processProposalDetails(ctx context.Context, pd www.Proposa reqs := []pdv2.RecordRequest{ { Token: pd.Token, - Version: uint32(v), + Version: uint32(version), }, } prs, err := p.proposals(ctx, reqs) @@ -326,7 +334,26 @@ func (p *politeiawww) processBatchVoteSummary(ctx context.Context, bvs www.Batch }, nil } -func (p *politeiawww) processAllVoteStatus(ctx context.Context, avs www.GetAllVoteStatus) (*www.GetAllVoteStatusReply, error) { +func (p *politeiawww) processVoteStatus(ctx context.Context, token string) (*www.VoteStatusReply, error) { + log.Tracef("processVoteStatus") + + // Get vote summaries + summaries, err := p.politeiad.TicketVoteSummaries(ctx, []string{token}) + if err != nil { + return nil, err + } + s, ok := summaries[token] + if !ok { + return nil, www.UserError{ + ErrorCode: www.ErrorStatusProposalNotFound, + } + } + vsr := convertVoteStatusReply(token, s) + + return &vsr, nil +} + +func (p *politeiawww) processAllVoteStatus(ctx context.Context) (*www.GetAllVoteStatusReply, error) { log.Tracef("processAllVoteStatus") // NOTE: This route is suppose to return the vote status of all @@ -349,31 +376,8 @@ func (p *politeiawww) processAllVoteStatus(ctx context.Context, avs www.GetAllVo // Prepare reply statuses := make([]www.VoteStatusReply, 0, len(vs)) - var totalVotes uint64 for token, v := range vs { - results := make([]www.VoteOptionResult, len(v.Results)) - for k, r := range v.Results { - results[k] = www.VoteOptionResult{ - VotesReceived: r.Votes, - Option: www.VoteOption{ - Id: r.ID, - Description: r.Description, - Bits: r.VoteBit, - }, - } - totalVotes += r.Votes - } - statuses = append(statuses, www.VoteStatusReply{ - Token: token, - Status: convertVoteStatusToWWW(v.Status), - TotalVotes: totalVotes, - OptionsResult: results, - EndHeight: strconv.FormatUint(uint64(v.EndBlockHeight), 10), - BestBlock: strconv.FormatUint(uint64(v.BestBlock), 10), - NumOfEligibleVotes: int(v.EligibleTickets), - QuorumPercentage: v.QuorumPercentage, - PassPercentage: v.PassPercentage, - }) + statuses = append(statuses, convertVoteStatusReply(token, v)) } return &www.GetAllVoteStatusReply{ @@ -713,20 +717,26 @@ func (p *politeiawww) handleBatchVoteSummary(w http.ResponseWriter, r *http.Requ util.RespondWithJSON(w, http.StatusOK, reply) } -func (p *politeiawww) handleAllVoteStatus(w http.ResponseWriter, r *http.Request) { - log.Tracef("handleAllVoteStatus") +func (p *politeiawww) handleVoteStatus(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleVoteStatus") - var avs www.GetAllVoteStatus - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&avs); err != nil { - RespondWithError(w, r, 0, "handleAllVoteStatus: unmarshal", - www.UserError{ - ErrorCode: www.ErrorStatusInvalidInput, - }) + pathParams := mux.Vars(r) + token := pathParams["token"] + + reply, err := p.processVoteStatus(r.Context(), token) + if err != nil { + RespondWithError(w, r, 0, + "handleVoteStatus: processVoteStatus %v", err) return } - reply, err := p.processAllVoteStatus(r.Context(), avs) + util.RespondWithJSON(w, http.StatusOK, reply) +} + +func (p *politeiawww) handleAllVoteStatus(w http.ResponseWriter, r *http.Request) { + log.Tracef("handleAllVoteStatus") + + reply, err := p.processAllVoteStatus(r.Context()) if err != nil { RespondWithError(w, r, 0, "handleAllVoteStatus: processAllVoteStatus %v", err) @@ -793,15 +803,16 @@ func (p *politeiawww) handleVoteResults(w http.ResponseWriter, r *http.Request) func userMetadataDecode(ms []pdv2.MetadataStream) (*umplugin.UserMetadata, error) { var userMD *umplugin.UserMetadata for _, v := range ms { - if v.StreamID == umplugin.StreamIDUserMetadata { - var um umplugin.UserMetadata - err := json.Unmarshal([]byte(v.Payload), &um) - if err != nil { - return nil, err - } - userMD = &um - break + if v.StreamID != umplugin.StreamIDUserMetadata { + continue + } + var um umplugin.UserMetadata + err := json.Unmarshal([]byte(v.Payload), &um) + if err != nil { + return nil, err } + userMD = &um + break } return userMD, nil } @@ -1035,3 +1046,30 @@ func convertVoteErrorCodeToWWW(e tkplugin.VoteErrorT) decredplugin.ErrorStatusT } return decredplugin.ErrorStatusInternalError } + +func convertVoteStatusReply(token string, s tkplugin.SummaryReply) www.VoteStatusReply { + results := make([]www.VoteOptionResult, 0, len(s.Results)) + var totalVotes uint64 + for _, v := range s.Results { + totalVotes += v.Votes + results = append(results, www.VoteOptionResult{ + VotesReceived: v.Votes, + Option: www.VoteOption{ + Id: v.ID, + Description: v.Description, + Bits: v.VoteBit, + }, + }) + } + return www.VoteStatusReply{ + Token: token, + Status: convertVoteStatusToWWW(s.Status), + TotalVotes: totalVotes, + OptionsResult: results, + EndHeight: strconv.FormatUint(uint64(s.EndBlockHeight), 10), + BestBlock: strconv.FormatUint(uint64(s.BestBlock), 10), + NumOfEligibleVotes: int(s.EligibleTickets), + QuorumPercentage: s.QuorumPercentage, + PassPercentage: s.PassPercentage, + } +} From b0d5d070dd9c48e980d0f8e4509cb34cd725abde Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 21 Mar 2021 17:27:44 -0500 Subject: [PATCH 422/449] www/ticketvote: Add summaries page size. --- politeiawww/api/ticketvote/v1/v1.go | 8 ++++++++ politeiawww/ticketvote/process.go | 11 +++++++++++ 2 files changed, 19 insertions(+) diff --git a/politeiawww/api/ticketvote/v1/v1.go b/politeiawww/api/ticketvote/v1/v1.go index b01be075e..b8dfac829 100644 --- a/politeiawww/api/ticketvote/v1/v1.go +++ b/politeiawww/api/ticketvote/v1/v1.go @@ -34,6 +34,7 @@ const ( ErrorCodeRecordNotFound ErrorCodeT = 4 ErrorCodeRecordLocked ErrorCodeT = 5 ErrorCodeTokenInvalid ErrorCodeT = 6 + ErrorCodePageSizeExceeded ErrorCodeT = 7 ) var ( @@ -46,6 +47,7 @@ var ( ErrorCodeRecordNotFound: "record not found", ErrorCodeRecordLocked: "record locked", ErrorCodeTokenInvalid: "token is invalid", + ErrorCodePageSizeExceeded: "page size exceeded", } ) @@ -494,6 +496,12 @@ type Summary struct { BestBlock uint32 `json:"bestblock"` } +const ( + // SummariesPageSize is the maximum number of vote summaries that + // can be requested at any one time. + SummariesPageSize uint32 = 5 +) + // Summaries requests the vote summaries for the provided record tokens. type Summaries struct { Tokens []string `json:"tokens"` diff --git a/politeiawww/ticketvote/process.go b/politeiawww/ticketvote/process.go index 572527c9d..01f964bbf 100644 --- a/politeiawww/ticketvote/process.go +++ b/politeiawww/ticketvote/process.go @@ -6,6 +6,7 @@ package ticketvote import ( "context" + "fmt" "github.com/decred/politeia/politeiad/plugins/ticketvote" v1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" @@ -171,6 +172,16 @@ func (t *TicketVote) processResults(ctx context.Context, r v1.Results) (*v1.Resu func (t *TicketVote) processSummaries(ctx context.Context, s v1.Summaries) (*v1.SummariesReply, error) { log.Tracef("processSummaries: %v", s.Tokens) + // Verify request size + if len(s.Tokens) > int(v1.SummariesPageSize) { + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodePageSizeExceeded, + ErrorContext: fmt.Sprintf("max page size is %v", + v1.SummariesPageSize), + } + } + + // Get vote summaries ts, err := t.politeiad.TicketVoteSummaries(ctx, s.Tokens) if err != nil { return nil, err From a895a878f3269e46a72ace17436fa3b4ca323549 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 21 Mar 2021 17:48:52 -0500 Subject: [PATCH 423/449] www/comments: Add pagination to timestamps. --- politeiawww/api/comments/v1/v1.go | 16 ++++++++++++++-- politeiawww/comments/process.go | 31 +++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/politeiawww/api/comments/v1/v1.go b/politeiawww/api/comments/v1/v1.go index 9c7ab6722..66262cab7 100644 --- a/politeiawww/api/comments/v1/v1.go +++ b/politeiawww/api/comments/v1/v1.go @@ -34,7 +34,7 @@ const ( ErrorCodeTokenInvalid ErrorCodeT = 6 ErrorCodeRecordNotFound ErrorCodeT = 7 ErrorCodeRecordLocked ErrorCodeT = 8 - ErrorCodeNoTokensFound ErrorCodeT = 9 + ErrorCodePageSizeExceeded ErrorCodeT = 9 ) var ( @@ -49,7 +49,7 @@ var ( ErrorCodeTokenInvalid: "token invalid", ErrorCodeRecordNotFound: "record not found", ErrorCodeRecordLocked: "record is locked", - ErrorCodeNoTokensFound: "no tokens found", + ErrorCodePageSizeExceeded: "page size exceeded", } ) @@ -246,6 +246,12 @@ type DelReply struct { Comment Comment `json:"comment"` } +const ( + // CountPageSize is the maximum number of tokens that can be + // included in the Count command. + CountPageSize uint32 = 10 +) + // Count requests the number of comments on that have been made on the given // records. If a record is not found for a token then it will not be included // in the returned map. @@ -309,6 +315,12 @@ type Timestamp struct { Proofs []Proof `json:"proofs"` } +const ( + // TimestampsPageSize is the maximum number of comment timestamps + // that can be requests at any one time. + TimestampsPageSize uint32 = 100 +) + // Timestamps requests the timestamps for the comments of a record. If no // comment IDs are provided then the timestamps for all comments will be // returned. diff --git a/politeiawww/comments/process.go b/politeiawww/comments/process.go index 6900b179b..17596c68c 100644 --- a/politeiawww/comments/process.go +++ b/politeiawww/comments/process.go @@ -7,6 +7,7 @@ package comments import ( "context" "errors" + "fmt" pdv2 "github.com/decred/politeia/politeiad/api/v2" "github.com/decred/politeia/politeiad/plugins/comments" @@ -197,12 +198,22 @@ func (c *Comments) processDel(ctx context.Context, d v1.Del, u user.User) (*v1.D func (c *Comments) processCount(ctx context.Context, ct v1.Count) (*v1.CountReply, error) { log.Tracef("processCount: %v", ct.Tokens) - if len(ct.Tokens) == 0 { + // Verify size of request + switch { + case len(ct.Tokens) == 0: + // Nothing to do + return &v1.CountReply{ + Counts: map[string]uint32{}, + }, nil + + case len(ct.Tokens) > int(v1.CountPageSize): return nil, v1.UserErrorReply{ - ErrorCode: v1.ErrorCodeNoTokensFound, + ErrorCode: v1.ErrorCodePageSizeExceeded, + ErrorContext: fmt.Sprintf("max page size is %v", v1.CountPageSize), } } + // Get comment counts counts, err := c.politeiad.CommentCount(ctx, ct.Tokens) if err != nil { return nil, err @@ -329,6 +340,22 @@ func (c *Comments) processTimestamps(ctx context.Context, t v1.Timestamps, isAdm return nil, err } + // Verify size of request + switch { + case len(t.CommentIDs) == 0: + // Nothing to do + return &v1.TimestampsReply{ + Comments: map[uint32][]v1.Timestamp{}, + }, nil + + case len(t.CommentIDs) > int(v1.TimestampsPageSize): + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodePageSizeExceeded, + ErrorContext: fmt.Sprintf("max page size is %v", + v1.TimestampsPageSize), + } + } + // Get timestamps ct := comments.Timestamps{ CommentIDs: t.CommentIDs, From 930bff69682b50bbfe1522da74e890f8741de94f Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 22 Mar 2021 10:09:56 -0500 Subject: [PATCH 424/449] tstorbe/dcrdata: Cleanup plugin and add docs. --- .../tstorebe/plugins/comments/cmds.go | 20 - .../tstorebe/plugins/comments/comments.go | 3 +- .../tstorebe/plugins/dcrdata/cmds.go | 415 +++++++++++++++++ .../tstorebe/plugins/dcrdata/dcrdata.go | 426 +----------------- .../tstorebe/plugins/ticketvote/cmds.go | 22 - .../tstorebe/plugins/ticketvote/ticketvote.go | 2 +- .../backendv2/tstorebe/plugins/usermd/cmds.go | 4 - politeiad/client/client.go | 14 +- politeiad/client/pdv2.go | 4 +- politeiad/plugins/dcrdata/dcrdata.go | 4 +- politeiawww/client/error.go | 2 +- politeiawww/comments/error.go | 2 +- politeiawww/comments/process.go | 22 +- politeiawww/records/error.go | 4 +- politeiawww/ticketvote/error.go | 4 +- 15 files changed, 462 insertions(+), 486 deletions(-) create mode 100644 politeiad/backendv2/tstorebe/plugins/dcrdata/cmds.go diff --git a/politeiad/backendv2/tstorebe/plugins/comments/cmds.go b/politeiad/backendv2/tstorebe/plugins/comments/cmds.go index beaae495f..14bbbb4d3 100644 --- a/politeiad/backendv2/tstorebe/plugins/comments/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/comments/cmds.go @@ -348,8 +348,6 @@ func voteScore(cidx commentIndex) (uint64, uint64) { // cmdNew creates a new comment. func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdNew: %v %x %v", treeID, token, payload) - // Decode payload var n comments.New err := json.Unmarshal([]byte(payload), &n) @@ -481,8 +479,6 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str // cmdEdit edits an existing comment. func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdEdit: %v %x %v", treeID, token, payload) - // Decode payload var e comments.Edit err := json.Unmarshal([]byte(payload), &e) @@ -637,8 +633,6 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st // cmdDel deletes a comment. func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdDel: %v %x %v", treeID, token, payload) - // Decode payload var d comments.Del err := json.Unmarshal([]byte(payload), &d) @@ -772,8 +766,6 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str // cmdVote casts a upvote/downvote for a comment. func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdVote: %v %x %v", treeID, token, payload) - // Decode payload var v comments.Vote err := json.Unmarshal([]byte(payload), &v) @@ -928,8 +920,6 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st // cmdGet retrieves a batch of specified comments. The most recent version of // each comment is returned. func (p *commentsPlugin) cmdGet(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdGet: %v %x %v", treeID, token, payload) - // Decode payload var g comments.Get err := json.Unmarshal([]byte(payload), &g) @@ -970,8 +960,6 @@ func (p *commentsPlugin) cmdGet(treeID int64, token []byte, payload string) (str // cmdGetAll retrieves all comments for a record. The latest version of each // comment is returned. func (p *commentsPlugin) cmdGetAll(treeID int64, token []byte) (string, error) { - log.Tracef("cmdGetAll: %v %x", treeID, token) - // Get record state state, err := p.tstore.RecordState(treeID) if err != nil { @@ -1019,8 +1007,6 @@ func (p *commentsPlugin) cmdGetAll(treeID int64, token []byte) (string, error) { // cmdGetVersion retrieves the specified version of a comment. func (p *commentsPlugin) cmdGetVersion(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdGetVersion: %v %x %v", treeID, token, payload) - // Decode payload var gv comments.GetVersion err := json.Unmarshal([]byte(payload), &gv) @@ -1094,8 +1080,6 @@ func (p *commentsPlugin) cmdGetVersion(treeID int64, token []byte, payload strin // cmdCount retrieves the comments count for a record. The comments count is // the number of comments that have been made on a record. func (p *commentsPlugin) cmdCount(treeID int64, token []byte) (string, error) { - log.Tracef("cmdCount: %v %x", treeID, token) - // Get record state state, err := p.tstore.RecordState(treeID) if err != nil { @@ -1123,8 +1107,6 @@ func (p *commentsPlugin) cmdCount(treeID int64, token []byte) (string, error) { // cmdVotes retrieves the comment votes that meet the provided filtering // criteria. func (p *commentsPlugin) cmdVotes(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdVotes: %v %x %v", treeID, token, payload) - // Decode payload var v comments.Votes err := json.Unmarshal([]byte(payload), &v) @@ -1180,8 +1162,6 @@ func (p *commentsPlugin) cmdVotes(treeID int64, token []byte, payload string) (s // cmdTimestamps retrieves the timestamps for the comments of a record. func (p *commentsPlugin) cmdTimestamps(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdTimestamps: %v %x %v", treeID, token, payload) - // Decode payload var t comments.Timestamps err := json.Unmarshal([]byte(payload), &t) diff --git a/politeiad/backendv2/tstorebe/plugins/comments/comments.go b/politeiad/backendv2/tstorebe/plugins/comments/comments.go index a03a99f4a..640854c88 100644 --- a/politeiad/backendv2/tstorebe/plugins/comments/comments.go +++ b/politeiad/backendv2/tstorebe/plugins/comments/comments.go @@ -22,6 +22,7 @@ var ( ) // commentsPlugin is the tstore backend implementation of the comments plugin. +// It provides an API for adding comment functionality onto a record. // // commentsPlugin satisfies the plugins PluginClient interface. type commentsPlugin struct { @@ -56,7 +57,7 @@ func (p *commentsPlugin) Setup() error { // // This function satisfies the plugins PluginClient interface. func (p *commentsPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { - log.Tracef("comments Cmd: %v %x %v", treeID, token, cmd) + log.Tracef("comments Cmd: %v %x %v %v", treeID, token, cmd, payload) switch cmd { case comments.CmdNew: diff --git a/politeiad/backendv2/tstorebe/plugins/dcrdata/cmds.go b/politeiad/backendv2/tstorebe/plugins/dcrdata/cmds.go new file mode 100644 index 000000000..9c16c2902 --- /dev/null +++ b/politeiad/backendv2/tstorebe/plugins/dcrdata/cmds.go @@ -0,0 +1,415 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package dcrdata + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strconv" + "strings" + + jsonrpc "github.com/decred/dcrd/rpc/jsonrpc/types/v2" + v5 "github.com/decred/dcrdata/api/types/v5" + "github.com/decred/politeia/politeiad/plugins/dcrdata" + "github.com/decred/politeia/util" +) + +// cmdBestBlock returns the best block. If the dcrdata websocket has been +// disconnected the best block will be fetched from the dcrdata HTTP API. If +// dcrdata cannot be reached then the most recent cached best block will be +// returned along with a status of StatusDisconnected. It is the callers +// responsibility to determine if the stale best block should be used. +func (p *dcrdataPlugin) cmdBestBlock(payload string) (string, error) { + // Payload is empty. Nothing to decode. + + // Get the cached best block + bb := p.bestBlockGet() + var ( + fetch bool + stale uint32 + status = dcrdata.StatusConnected + ) + switch { + case bb == 0: + // No cached best block means that the best block has not been + // populated by the websocket yet. Fetch is manually. + fetch = true + case p.bestBlockIsStale(): + // The cached best block has been populated by the websocket, but + // the websocket is currently disconnected and the cached value + // is stale. Try to fetch the best block manually and only use + // the stale value if manually fetching it fails. + fetch = true + stale = bb + } + + // Fetch the best block manually if required + if fetch { + block, err := p.bestBlockHTTP() + switch { + case err == nil: + // We got the best block. Use it. + bb = block.Height + case stale != 0: + // Unable to fetch the best block manually. Use the stale + // value and mark the connection status as disconnected. + bb = stale + status = dcrdata.StatusDisconnected + default: + // Unable to fetch the best block manually and there is no + // stale cached value to return. + return "", fmt.Errorf("bestBlockHTTP: %v", err) + } + } + + // Prepare reply + bbr := dcrdata.BestBlockReply{ + Status: status, + Height: bb, + } + reply, err := json.Marshal(bbr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +// cmdBlockDetails retrieves the block details for the provided block height. +func (p *dcrdataPlugin) cmdBlockDetails(payload string) (string, error) { + // Decode payload + var bd dcrdata.BlockDetails + err := json.Unmarshal([]byte(payload), &bd) + if err != nil { + return "", err + } + + // Fetch block details + bdb, err := p.blockDetails(bd.Height) + if err != nil { + return "", fmt.Errorf("blockDetails: %v", err) + } + + // Prepare reply + bdr := dcrdata.BlockDetailsReply{ + Block: convertBlockDataBasicFromV5(*bdb), + } + reply, err := json.Marshal(bdr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +// cmdTicketPool requests the lists of tickets in the ticket pool at a +// specified block hash. +func (p *dcrdataPlugin) cmdTicketPool(payload string) (string, error) { + // Decode payload + var tp dcrdata.TicketPool + err := json.Unmarshal([]byte(payload), &tp) + if err != nil { + return "", err + } + + // Get the ticket pool + tickets, err := p.ticketPool(tp.BlockHash) + if err != nil { + return "", fmt.Errorf("ticketPool: %v", err) + } + + // Prepare reply + tpr := dcrdata.TicketPoolReply{ + Tickets: tickets, + } + reply, err := json.Marshal(tpr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +// TxsTrimmed requests the trimmed transaction information for the provided +// transaction IDs. +func (p *dcrdataPlugin) cmdTxsTrimmed(payload string) (string, error) { + // Decode payload + var tt dcrdata.TxsTrimmed + err := json.Unmarshal([]byte(payload), &tt) + if err != nil { + return "", err + } + + // Get trimmed txs + txs, err := p.txsTrimmed(tt.TxIDs) + if err != nil { + return "", fmt.Errorf("txsTrimmed: %v", err) + } + + // Prepare reply + ttr := dcrdata.TxsTrimmedReply{ + Txs: convertTrimmedTxsFromV5(txs), + } + reply, err := json.Marshal(ttr) + if err != nil { + return "", err + } + + return string(reply), nil +} + +// makeReq makes a dcrdata http request to the method and route provided, +// serializing the provided object as the request body, and returning a byte +// slice of the response body. An error is returned if dcrdata responds with +// anything other than a 200 http status code. +func (p *dcrdataPlugin) makeReq(method string, route string, headers map[string]string, v interface{}) ([]byte, error) { + var ( + url = p.hostHTTP + route + reqBody []byte + err error + ) + + log.Tracef("%v %v", method, url) + + // Setup request + if v != nil { + reqBody, err = json.Marshal(v) + if err != nil { + return nil, err + } + } + req, err := http.NewRequest(method, url, bytes.NewReader(reqBody)) + if err != nil { + return nil, err + } + for k, v := range headers { + req.Header.Add(k, v) + } + + // Send request + r, err := p.client.Do(req) + if err != nil { + return nil, err + } + defer r.Body.Close() + + // Handle response + if r.StatusCode != http.StatusOK { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + return nil, fmt.Errorf("%v %v %v %v", + r.StatusCode, method, url, err) + } + return nil, fmt.Errorf("%v %v %v %s", + r.StatusCode, method, url, body) + } + + return util.RespBody(r), nil +} + +// bestBlockHTTP fetches and returns the best block from the dcrdata http API. +func (p *dcrdataPlugin) bestBlockHTTP() (*v5.BlockDataBasic, error) { + resBody, err := p.makeReq(http.MethodGet, routeBestBlock, nil, nil) + if err != nil { + return nil, err + } + + var bdb v5.BlockDataBasic + err = json.Unmarshal(resBody, &bdb) + if err != nil { + return nil, err + } + + return &bdb, nil +} + +// blockDetails returns the block details for the block at the specified block +// height. +func (p *dcrdataPlugin) blockDetails(height uint32) (*v5.BlockDataBasic, error) { + h := strconv.FormatUint(uint64(height), 10) + + route := strings.Replace(routeBlockDetails, "{height}", h, 1) + resBody, err := p.makeReq(http.MethodGet, route, nil, nil) + if err != nil { + return nil, err + } + + var bdb v5.BlockDataBasic + err = json.Unmarshal(resBody, &bdb) + if err != nil { + return nil, err + } + + return &bdb, nil +} + +// ticketPool returns the list of tickets in the ticket pool at the specified +// block hash. +func (p *dcrdataPlugin) ticketPool(blockHash string) ([]string, error) { + route := strings.Replace(routeTicketPool, "{hash}", blockHash, 1) + route += "?sort=true" + resBody, err := p.makeReq(http.MethodGet, route, nil, nil) + if err != nil { + return nil, err + } + + var tickets []string + err = json.Unmarshal(resBody, &tickets) + if err != nil { + return nil, err + } + + return tickets, nil +} + +// txsTrimmed returns the TrimmedTx for the specified tx IDs. +func (p *dcrdataPlugin) txsTrimmed(txIDs []string) ([]v5.TrimmedTx, error) { + t := v5.Txns{ + Transactions: txIDs, + } + headers := map[string]string{ + headerContentType: contentTypeJSON, + } + resBody, err := p.makeReq(http.MethodPost, routeTxsTrimmed, headers, t) + if err != nil { + return nil, err + } + + var txs []v5.TrimmedTx + err = json.Unmarshal(resBody, &txs) + if err != nil { + return nil, err + } + + return txs, nil +} + +func convertTicketPoolInfoFromV5(t v5.TicketPoolInfo) dcrdata.TicketPoolInfo { + return dcrdata.TicketPoolInfo{ + Height: t.Height, + Size: t.Size, + Value: t.Value, + ValAvg: t.ValAvg, + Winners: t.Winners, + } +} + +func convertBlockDataBasicFromV5(b v5.BlockDataBasic) dcrdata.BlockDataBasic { + var poolInfo *dcrdata.TicketPoolInfo + if b.PoolInfo != nil { + p := convertTicketPoolInfoFromV5(*b.PoolInfo) + poolInfo = &p + } + return dcrdata.BlockDataBasic{ + Height: b.Height, + Size: b.Size, + Hash: b.Hash, + Difficulty: b.Difficulty, + StakeDiff: b.StakeDiff, + Time: b.Time.UNIX(), + NumTx: b.NumTx, + MiningFee: b.MiningFee, + TotalSent: b.TotalSent, + PoolInfo: poolInfo, + } +} + +func convertScriptSigFromJSONRPC(s jsonrpc.ScriptSig) dcrdata.ScriptSig { + return dcrdata.ScriptSig{ + Asm: s.Asm, + Hex: s.Hex, + } +} + +func convertVinFromJSONRPC(v jsonrpc.Vin) dcrdata.Vin { + var scriptSig *dcrdata.ScriptSig + if v.ScriptSig != nil { + s := convertScriptSigFromJSONRPC(*v.ScriptSig) + scriptSig = &s + } + return dcrdata.Vin{ + Coinbase: v.Coinbase, + Stakebase: v.Stakebase, + Txid: v.Txid, + Vout: v.Vout, + Tree: v.Tree, + Sequence: v.Sequence, + AmountIn: v.AmountIn, + BlockHeight: v.BlockHeight, + BlockIndex: v.BlockIndex, + ScriptSig: scriptSig, + } +} + +func convertVinsFromV5(ins []jsonrpc.Vin) []dcrdata.Vin { + i := make([]dcrdata.Vin, 0, len(ins)) + for _, v := range ins { + i = append(i, convertVinFromJSONRPC(v)) + } + return i +} + +func convertScriptPubKeyFromV5(s v5.ScriptPubKey) dcrdata.ScriptPubKey { + return dcrdata.ScriptPubKey{ + Asm: s.Asm, + Hex: s.Hex, + ReqSigs: s.ReqSigs, + Type: s.Type, + Addresses: s.Addresses, + CommitAmt: s.CommitAmt, + } +} + +func convertTxInputIDFromV5(t v5.TxInputID) dcrdata.TxInputID { + return dcrdata.TxInputID{ + Hash: t.Hash, + Index: t.Index, + } +} + +func convertVoutFromV5(v v5.Vout) dcrdata.Vout { + var spend *dcrdata.TxInputID + if v.Spend != nil { + s := convertTxInputIDFromV5(*v.Spend) + spend = &s + } + return dcrdata.Vout{ + Value: v.Value, + N: v.N, + Version: v.Version, + ScriptPubKeyDecoded: convertScriptPubKeyFromV5(v.ScriptPubKeyDecoded), + Spend: spend, + } +} + +func convertVoutsFromV5(outs []v5.Vout) []dcrdata.Vout { + o := make([]dcrdata.Vout, 0, len(outs)) + for _, v := range outs { + o = append(o, convertVoutFromV5(v)) + } + return o +} + +func convertTrimmedTxFromV5(t v5.TrimmedTx) dcrdata.TrimmedTx { + return dcrdata.TrimmedTx{ + TxID: t.TxID, + Version: t.Version, + Locktime: t.Locktime, + Expiry: t.Expiry, + Vin: convertVinsFromV5(t.Vin), + Vout: convertVoutsFromV5(t.Vout), + } +} + +func convertTrimmedTxsFromV5(txs []v5.TrimmedTx) []dcrdata.TrimmedTx { + t := make([]dcrdata.TrimmedTx, 0, len(txs)) + for _, v := range txs { + t = append(t, convertTrimmedTxFromV5(v)) + } + return t +} diff --git a/politeiad/backendv2/tstorebe/plugins/dcrdata/dcrdata.go b/politeiad/backendv2/tstorebe/plugins/dcrdata/dcrdata.go index 60c39f02b..a24315d02 100644 --- a/politeiad/backendv2/tstorebe/plugins/dcrdata/dcrdata.go +++ b/politeiad/backendv2/tstorebe/plugins/dcrdata/dcrdata.go @@ -5,18 +5,11 @@ package dcrdata import ( - "bytes" - "encoding/json" "fmt" - "io/ioutil" "net/http" - "strconv" - "strings" "sync" "github.com/decred/dcrd/chaincfg/v3" - jsonrpc "github.com/decred/dcrd/rpc/jsonrpc/types/v2" - v5 "github.com/decred/dcrdata/api/types/v5" exptypes "github.com/decred/dcrdata/explorer/types/v2" pstypes "github.com/decred/dcrdata/pubsub/types/v3" backend "github.com/decred/politeia/politeiad/backendv2" @@ -27,7 +20,7 @@ import ( ) const ( - // Dcrdata routes + // Dcrdata http routes routeBestBlock = "/api/block/best" routeBlockDetails = "/api/block/{height}" routeTicketPool = "/api/stake/pool/b/{hash}/full" @@ -44,7 +37,10 @@ var ( _ plugins.PluginClient = (*dcrdataPlugin)(nil) ) -// dcrdataplugin satisfies the plugins.PluginClient interface. +// dcrdataPlugin is the tstore backend implementation of the dcrdata plugin. It +// provides and API for interacting with the dcrdata http and websocket APIs. +// +// dcrdataPlugin satisfies the plugins PluginClient interface. type dcrdataPlugin struct { sync.Mutex activeNetParams *chaincfg.Params @@ -64,6 +60,7 @@ type dcrdataPlugin struct { bestBlockStale bool } +// bestBlockGet returns the cached best block. func (p *dcrdataPlugin) bestBlockGet() uint32 { p.Lock() defer p.Unlock() @@ -71,6 +68,7 @@ func (p *dcrdataPlugin) bestBlockGet() uint32 { return p.bestBlock } +// bestBlockSet sets the cached best block to a new value. func (p *dcrdataPlugin) bestBlockSet(bb uint32) { p.Lock() defer p.Unlock() @@ -79,6 +77,7 @@ func (p *dcrdataPlugin) bestBlockSet(bb uint32) { p.bestBlockStale = false } +// bestBlockSetStale marks the cached best block as stale. func (p *dcrdataPlugin) bestBlockSetStale() { p.Lock() defer p.Unlock() @@ -86,6 +85,8 @@ func (p *dcrdataPlugin) bestBlockSetStale() { p.bestBlockStale = true } +// bestBlockIsStale returns whether the cached best block has been marked as +// being stale. func (p *dcrdataPlugin) bestBlockIsStale() bool { p.Lock() defer p.Unlock() @@ -93,401 +94,6 @@ func (p *dcrdataPlugin) bestBlockIsStale() bool { return p.bestBlockStale } -func (p *dcrdataPlugin) makeReq(method string, route string, headers map[string]string, v interface{}) ([]byte, error) { - var ( - url = p.hostHTTP + route - reqBody []byte - err error - ) - - log.Tracef("%v %v", method, url) - - // Setup request - if v != nil { - reqBody, err = json.Marshal(v) - if err != nil { - return nil, err - } - } - req, err := http.NewRequest(method, url, bytes.NewReader(reqBody)) - if err != nil { - return nil, err - } - for k, v := range headers { - req.Header.Add(k, v) - } - - // Send request - r, err := p.client.Do(req) - if err != nil { - return nil, err - } - defer r.Body.Close() - - // Handle response - if r.StatusCode != http.StatusOK { - body, err := ioutil.ReadAll(r.Body) - if err != nil { - return nil, fmt.Errorf("%v %v %v %v", - r.StatusCode, method, url, err) - } - return nil, fmt.Errorf("%v %v %v %s", - r.StatusCode, method, url, body) - } - - return util.RespBody(r), nil -} - -// bestBlockHTTP fetches and returns the best block from the dcrdata http API. -func (p *dcrdataPlugin) bestBlockHTTP() (*v5.BlockDataBasic, error) { - resBody, err := p.makeReq(http.MethodGet, routeBestBlock, nil, nil) - if err != nil { - return nil, err - } - - var bdb v5.BlockDataBasic - err = json.Unmarshal(resBody, &bdb) - if err != nil { - return nil, err - } - - return &bdb, nil -} - -// blockDetailsHTTP fetches and returns the block details from the dcrdata API -// for the provided block height. -func (p *dcrdataPlugin) blockDetailsHTTP(height uint32) (*v5.BlockDataBasic, error) { - h := strconv.FormatUint(uint64(height), 10) - - route := strings.Replace(routeBlockDetails, "{height}", h, 1) - resBody, err := p.makeReq(http.MethodGet, route, nil, nil) - if err != nil { - return nil, err - } - - var bdb v5.BlockDataBasic - err = json.Unmarshal(resBody, &bdb) - if err != nil { - return nil, err - } - - return &bdb, nil -} - -// ticketPoolHTTP fetches and returns the list of tickets in the ticket pool -// from the dcrdata API at the provided block hash. -func (p *dcrdataPlugin) ticketPoolHTTP(blockHash string) ([]string, error) { - route := strings.Replace(routeTicketPool, "{hash}", blockHash, 1) - route += "?sort=true" - resBody, err := p.makeReq(http.MethodGet, route, nil, nil) - if err != nil { - return nil, err - } - - var tickets []string - err = json.Unmarshal(resBody, &tickets) - if err != nil { - return nil, err - } - - return tickets, nil -} - -// txsTrimmedHTTP fetches and returns the TrimmedTx from the dcrdata API for -// the provided tx IDs. -func (p *dcrdataPlugin) txsTrimmedHTTP(txIDs []string) ([]v5.TrimmedTx, error) { - t := v5.Txns{ - Transactions: txIDs, - } - headers := map[string]string{ - headerContentType: contentTypeJSON, - } - resBody, err := p.makeReq(http.MethodPost, routeTxsTrimmed, headers, t) - if err != nil { - return nil, err - } - - var txs []v5.TrimmedTx - err = json.Unmarshal(resBody, &txs) - if err != nil { - return nil, err - } - - return txs, nil -} - -// cmdBestBlock returns the best block. If the dcrdata websocket has been -// disconnected the best block will be fetched from the dcrdata API. If dcrdata -// cannot be reached then the most recent cached best block will be returned -// along with a status of StatusDisconnected. It is the callers responsibility -// to determine if the stale best block should be used. -func (p *dcrdataPlugin) cmdBestBlock(payload string) (string, error) { - log.Tracef("cmdBestBlock: %v", payload) - - // Payload is empty. Nothing to decode. - - // Get the cached best block - bb := p.bestBlockGet() - var ( - fetch bool - stale uint32 - status = dcrdata.StatusConnected - ) - switch { - case bb == 0: - // No cached best block means that the best block has not been - // populated by the websocket yet. Fetch is manually. - fetch = true - case p.bestBlockIsStale(): - // The cached best block has been populated by the websocket, but - // the websocket is currently disconnected and the cached value - // is stale. Try to fetch the best block manually and only use - // the stale value if manually fetching it fails. - fetch = true - stale = bb - } - - // Fetch the best block manually if required - if fetch { - block, err := p.bestBlockHTTP() - switch { - case err == nil: - // We got the best block. Use it. - bb = block.Height - case stale != 0: - // Unable to fetch the best block manually. Use the stale - // value and mark the connection status as disconnected. - bb = stale - status = dcrdata.StatusDisconnected - default: - // Unable to fetch the best block manually and there is no - // stale cached value to return. - return "", fmt.Errorf("bestBlockHTTP: %v", err) - } - } - - // Prepare reply - bbr := dcrdata.BestBlockReply{ - Status: status, - Height: bb, - } - reply, err := json.Marshal(bbr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func convertTicketPoolInfoFromV5(t v5.TicketPoolInfo) dcrdata.TicketPoolInfo { - return dcrdata.TicketPoolInfo{ - Height: t.Height, - Size: t.Size, - Value: t.Value, - ValAvg: t.ValAvg, - Winners: t.Winners, - } -} - -func convertBlockDataBasicFromV5(b v5.BlockDataBasic) dcrdata.BlockDataBasic { - var poolInfo *dcrdata.TicketPoolInfo - if b.PoolInfo != nil { - p := convertTicketPoolInfoFromV5(*b.PoolInfo) - poolInfo = &p - } - return dcrdata.BlockDataBasic{ - Height: b.Height, - Size: b.Size, - Hash: b.Hash, - Difficulty: b.Difficulty, - StakeDiff: b.StakeDiff, - Time: b.Time.UNIX(), - NumTx: b.NumTx, - MiningFee: b.MiningFee, - TotalSent: b.TotalSent, - PoolInfo: poolInfo, - } -} - -func (p *dcrdataPlugin) cmdBlockDetails(payload string) (string, error) { - log.Tracef("cmdBlockDetails: %v", payload) - - // Decode payload - var bd dcrdata.BlockDetails - err := json.Unmarshal([]byte(payload), &bd) - if err != nil { - return "", err - } - - // Fetch block details - bdb, err := p.blockDetailsHTTP(bd.Height) - if err != nil { - return "", fmt.Errorf("blockDetailsHTTP: %v", err) - } - - // Prepare reply - bdr := dcrdata.BlockDetailsReply{ - Block: convertBlockDataBasicFromV5(*bdb), - } - reply, err := json.Marshal(bdr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *dcrdataPlugin) cmdTicketPool(payload string) (string, error) { - log.Tracef("cmdTicketPool: %v", payload) - - // Decode payload - var tp dcrdata.TicketPool - err := json.Unmarshal([]byte(payload), &tp) - if err != nil { - return "", err - } - - // Get the ticket pool - tickets, err := p.ticketPoolHTTP(tp.BlockHash) - if err != nil { - return "", fmt.Errorf("ticketPoolHTTP: %v", err) - } - - // Prepare reply - tpr := dcrdata.TicketPoolReply{ - Tickets: tickets, - } - reply, err := json.Marshal(tpr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func convertScriptSigFromJSONRPC(s jsonrpc.ScriptSig) dcrdata.ScriptSig { - return dcrdata.ScriptSig{ - Asm: s.Asm, - Hex: s.Hex, - } -} - -func convertVinFromJSONRPC(v jsonrpc.Vin) dcrdata.Vin { - var scriptSig *dcrdata.ScriptSig - if v.ScriptSig != nil { - s := convertScriptSigFromJSONRPC(*v.ScriptSig) - scriptSig = &s - } - return dcrdata.Vin{ - Coinbase: v.Coinbase, - Stakebase: v.Stakebase, - Txid: v.Txid, - Vout: v.Vout, - Tree: v.Tree, - Sequence: v.Sequence, - AmountIn: v.AmountIn, - BlockHeight: v.BlockHeight, - BlockIndex: v.BlockIndex, - ScriptSig: scriptSig, - } -} - -func convertVinsFromV5(ins []jsonrpc.Vin) []dcrdata.Vin { - i := make([]dcrdata.Vin, 0, len(ins)) - for _, v := range ins { - i = append(i, convertVinFromJSONRPC(v)) - } - return i -} - -func convertScriptPubKeyFromV5(s v5.ScriptPubKey) dcrdata.ScriptPubKey { - return dcrdata.ScriptPubKey{ - Asm: s.Asm, - Hex: s.Hex, - ReqSigs: s.ReqSigs, - Type: s.Type, - Addresses: s.Addresses, - CommitAmt: s.CommitAmt, - } -} - -func convertTxInputIDFromV5(t v5.TxInputID) dcrdata.TxInputID { - return dcrdata.TxInputID{ - Hash: t.Hash, - Index: t.Index, - } -} - -func convertVoutFromV5(v v5.Vout) dcrdata.Vout { - var spend *dcrdata.TxInputID - if v.Spend != nil { - s := convertTxInputIDFromV5(*v.Spend) - spend = &s - } - return dcrdata.Vout{ - Value: v.Value, - N: v.N, - Version: v.Version, - ScriptPubKeyDecoded: convertScriptPubKeyFromV5(v.ScriptPubKeyDecoded), - Spend: spend, - } -} - -func convertVoutsFromV5(outs []v5.Vout) []dcrdata.Vout { - o := make([]dcrdata.Vout, 0, len(outs)) - for _, v := range outs { - o = append(o, convertVoutFromV5(v)) - } - return o -} - -func convertTrimmedTxFromV5(t v5.TrimmedTx) dcrdata.TrimmedTx { - return dcrdata.TrimmedTx{ - TxID: t.TxID, - Version: t.Version, - Locktime: t.Locktime, - Expiry: t.Expiry, - Vin: convertVinsFromV5(t.Vin), - Vout: convertVoutsFromV5(t.Vout), - } -} - -func convertTrimmedTxsFromV5(txs []v5.TrimmedTx) []dcrdata.TrimmedTx { - t := make([]dcrdata.TrimmedTx, 0, len(txs)) - for _, v := range txs { - t = append(t, convertTrimmedTxFromV5(v)) - } - return t -} - -func (p *dcrdataPlugin) cmdTxsTrimmed(payload string) (string, error) { - log.Tracef("cmdTxsTrimmed: %v", payload) - - // Decode payload - var tt dcrdata.TxsTrimmed - err := json.Unmarshal([]byte(payload), &tt) - if err != nil { - return "", err - } - - // Get trimmed txs - txs, err := p.txsTrimmedHTTP(tt.TxIDs) - if err != nil { - return "", fmt.Errorf("txsTrimmedHTTP: %v", err) - } - - // Prepare reply - ttr := dcrdata.TxsTrimmedReply{ - Txs: convertTrimmedTxsFromV5(txs), - } - reply, err := json.Marshal(ttr) - if err != nil { - return "", err - } - - return string(reply), nil -} - func (p *dcrdataPlugin) websocketMonitor() { defer func() { log.Infof("Dcrdata websocket closed") @@ -571,7 +177,7 @@ func (p *dcrdataPlugin) websocketSetup() { // Setup performs any plugin setup that is required. // -// This function satisfies the plugins.PluginClient interface. +// This function satisfies the plugins PluginClient interface. func (p *dcrdataPlugin) Setup() error { log.Tracef("dcrdata Setup") @@ -586,9 +192,9 @@ func (p *dcrdataPlugin) Setup() error { // Cmd executes a plugin command. // -// This function satisfies the plugins.PluginClient interface. +// This function satisfies the plugins PluginClient interface. func (p *dcrdataPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { - log.Tracef("dcrdata Cmd: %v %x %v", treeID, token, cmd) + log.Tracef("dcrdata Cmd: %v %x %v %v", treeID, token, cmd, payload) switch cmd { case dcrdata.CmdBestBlock: @@ -606,7 +212,7 @@ func (p *dcrdataPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (st // Hook executes a plugin hook. // -// This function satisfies the plugins.PluginClient interface. +// This function satisfies the plugins PluginClient interface. func (p *dcrdataPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { log.Tracef("dcrdata Hook: %v %x %v", plugins.Hooks[h], token, treeID) @@ -615,7 +221,7 @@ func (p *dcrdataPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payloa // Fsck performs a plugin filesystem check. // -// This function satisfies the plugins.PluginClient interface. +// This function satisfies the plugins PluginClient interface. func (p *dcrdataPlugin) Fsck(treeIDs []int64) error { log.Tracef("dcrdata Fsck") @@ -624,7 +230,7 @@ func (p *dcrdataPlugin) Fsck(treeIDs []int64) error { // Settings returns the plugin's settings. // -// This function satisfies the plugins.PluginClient interface. +// This function satisfies the plugins PluginClient interface. func (p *dcrdataPlugin) Settings() []backend.PluginSetting { log.Tracef("dcrdata Settings") diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go index 26540550b..949292c69 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go @@ -861,8 +861,6 @@ func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, } func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdAuthorize: %v %x %v", treeID, token, payload) - // Decode payload var a ticketvote.Authorize err := json.Unmarshal([]byte(payload), &a) @@ -1377,8 +1375,6 @@ type runoffDetailsReply struct { } func (p *ticketVotePlugin) cmdRunoffDetails(treeID int64) (string, error) { - log.Tracef("cmdRunoffDetails: %x", treeID) - // Get start runoff record srs, err := p.startRunoffRecord(treeID) if err != nil { @@ -1813,8 +1809,6 @@ type startRunoffSubmission struct { } func (p *ticketVotePlugin) cmdStartRunoffSubmission(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdStartRunoffSubmission: %v %x %v", treeID, token, payload) - // Decode payload var srs startRunoffSubmission err := json.Unmarshal([]byte(payload), &srs) @@ -1832,8 +1826,6 @@ func (p *ticketVotePlugin) cmdStartRunoffSubmission(treeID int64, token []byte, } func (p *ticketVotePlugin) cmdStart(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdStart: %v %x %v", treeID, token, payload) - // Decode payload var s ticketvote.Start err := json.Unmarshal([]byte(payload), &s) @@ -2068,8 +2060,6 @@ func (p *ticketVotePlugin) ballot(treeID int64, votes []ticketvote.CastVote, res // error if one occurs. It will instead return the ballot reply with the error // included in the individual cast vote reply that it applies to. func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdCastBallot: %v %x %v", treeID, token, payload) - // Decode payload var cb ticketvote.CastBallot err := json.Unmarshal([]byte(payload), &cb) @@ -2382,8 +2372,6 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str } func (p *ticketVotePlugin) cmdDetails(treeID int64, token []byte) (string, error) { - log.Tracef("cmdDetails: %v %x", treeID, token) - // Get vote authorizations auths, err := p.auths(treeID) if err != nil { @@ -2410,8 +2398,6 @@ func (p *ticketVotePlugin) cmdDetails(treeID int64, token []byte) (string, error } func (p *ticketVotePlugin) cmdResults(treeID int64, token []byte) (string, error) { - log.Tracef("cmdResults: %v %x", treeID, token) - // Get vote results votes, err := p.voteResults(treeID) if err != nil { @@ -2496,8 +2482,6 @@ func voteIsApproved(vd ticketvote.VoteDetails, results []ticketvote.VoteOptionRe } func (p *ticketVotePlugin) cmdSummary(treeID int64, token []byte) (string, error) { - log.Tracef("cmdSummaries: %v %x %v", treeID, token) - // Get best block. This cmd does not write any data so we do not // have to use the safe best block. bb, err := p.bestBlockUnsafe() @@ -2521,8 +2505,6 @@ func (p *ticketVotePlugin) cmdSummary(treeID int64, token []byte) (string, error } func (p *ticketVotePlugin) cmdInventory(payload string) (string, error) { - log.Tracef("cmdInventory") - var i ticketvote.Inventory err := json.Unmarshal([]byte(payload), &i) if err != nil { @@ -2561,8 +2543,6 @@ func (p *ticketVotePlugin) cmdInventory(payload string) (string, error) { } func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload string) (string, error) { - log.Tracef("cmdTimestamps: %v %x %v", treeID, token, payload) - // Decode payload var t ticketvote.Timestamps err := json.Unmarshal([]byte(payload), &t) @@ -2658,8 +2638,6 @@ func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload str } func (p *ticketVotePlugin) cmdSubmissions(token []byte) (string, error) { - log.Tracef("cmdSubmissions: %x", token) - // Get submissions list lf, err := p.submissionsCache(token) if err != nil { diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go index 7cd0abafc..6428867a9 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go @@ -149,7 +149,7 @@ func (p *ticketVotePlugin) Setup() error { // // This function satisfies the plugins.PluginClient interface. func (p *ticketVotePlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { - log.Tracef("ticketvote Cmd: %v %x %v", treeID, token, cmd) + log.Tracef("ticketvote Cmd: %v %x %v %v", treeID, token, cmd, payload) switch cmd { case ticketvote.CmdAuthorize: diff --git a/politeiad/backendv2/tstorebe/plugins/usermd/cmds.go b/politeiad/backendv2/tstorebe/plugins/usermd/cmds.go index 550a66a3f..9d11ecf9c 100644 --- a/politeiad/backendv2/tstorebe/plugins/usermd/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/usermd/cmds.go @@ -11,8 +11,6 @@ import ( ) func (p *userPlugin) cmdAuthor(treeID int64) (string, error) { - log.Tracef("cmdAuthor: %v", treeID) - // Get user metadata r, err := p.tstore.RecordPartial(treeID, 0, nil, true) if err != nil { @@ -36,8 +34,6 @@ func (p *userPlugin) cmdAuthor(treeID int64) (string, error) { } func (p *userPlugin) cmdUserRecords(payload string) (string, error) { - log.Tracef("cmdUserRecords: %v", payload) - // Decode payload var ur usermd.UserRecords err := json.Unmarshal([]byte(payload), &ur) diff --git a/politeiad/client/client.go b/politeiad/client/client.go index ca0b45433..4d7287c85 100644 --- a/politeiad/client/client.go +++ b/politeiad/client/client.go @@ -34,15 +34,15 @@ type ErrorReply struct { ErrorContext string `json:"errorcontext"` } -// Error represents a politeiad error. Error is returned anytime the politeiad -// response is not a 200. -type Error struct { +// RespErr represents a politeiad response error. A RespError is returned +// anytime the politeiad response is not a 200. +type RespError struct { HTTPCode int ErrorReply ErrorReply } // Error satisfies the error interface. -func (e Error) Error() string { +func (e RespError) Error() string { if e.ErrorReply.PluginID != "" { return fmt.Sprintf("politeiad plugin error: %v %v %v", e.HTTPCode, e.ErrorReply.PluginID, e.ErrorReply.ErrorCode) @@ -53,8 +53,8 @@ func (e Error) Error() string { // makeReq makes a politeiad http request to the method and route provided, // serializing the provided object as the request body, and returning a byte -// slice of the response body. An Error is returned if politeiad responds with -// anything other than a 200 http status code. +// slice of the response body. A RespError is returned if politeiad responds +// with anything other than a 200 http status code. func (c *Client) makeReq(ctx context.Context, method, api, route string, v interface{}) ([]byte, error) { // Serialize body var ( @@ -89,7 +89,7 @@ func (c *Client) makeReq(ctx context.Context, method, api, route string, v inter if err := decoder.Decode(&e); err != nil { return nil, fmt.Errorf("status code %v: %v", r.StatusCode, err) } - return nil, Error{ + return nil, RespError{ HTTPCode: r.StatusCode, ErrorReply: e, } diff --git a/politeiad/client/pdv2.go b/politeiad/client/pdv2.go index 26bd7c9c2..d910dd500 100644 --- a/politeiad/client/pdv2.go +++ b/politeiad/client/pdv2.go @@ -394,7 +394,7 @@ func (c *Client) PluginInventory(ctx context.Context) ([]pdv2.Plugin, error) { func extractPluginCmdError(pcr pdv2.PluginCmdReply) error { switch { case pcr.UserError != nil: - return Error{ + return RespError{ HTTPCode: http.StatusBadRequest, ErrorReply: ErrorReply{ ErrorCode: uint32(pcr.UserError.ErrorCode), @@ -402,7 +402,7 @@ func extractPluginCmdError(pcr pdv2.PluginCmdReply) error { }, } case pcr.PluginError != nil: - return Error{ + return RespError{ HTTPCode: http.StatusBadRequest, ErrorReply: ErrorReply{ PluginID: pcr.PluginError.PluginID, diff --git a/politeiad/plugins/dcrdata/dcrdata.go b/politeiad/plugins/dcrdata/dcrdata.go index 765d0f7b9..5e82e59e5 100644 --- a/politeiad/plugins/dcrdata/dcrdata.go +++ b/politeiad/plugins/dcrdata/dcrdata.go @@ -90,7 +90,7 @@ type BlockDataBasic struct { PoolInfo *TicketPoolInfo `json:"ticketpool,omitempty"` } -// BlockDetails fetched the block details for the provided block height. +// BlockDetails retrieves the block details for the provided block height. type BlockDetails struct { Height uint32 `json:"height"` } @@ -100,7 +100,7 @@ type BlockDetailsReply struct { Block BlockDataBasic `json:"block"` } -// TicketPool requests the lists of tickets in the ticket for at the provided +// TicketPool requests the lists of tickets in the ticket pool at a specified // block hash. type TicketPool struct { BlockHash string `json:"blockhash"` diff --git a/politeiawww/client/error.go b/politeiawww/client/error.go index 329227dff..55f6f27f9 100644 --- a/politeiawww/client/error.go +++ b/politeiawww/client/error.go @@ -27,7 +27,7 @@ type ErrorReply struct { ErrorContext string } -// RespErr represents a politeiawww response error. An Error is returned +// RespErr represents a politeiawww response error. A RespErr is returned // anytime the politeiawww response is not a 200. // // The various politeiawww APIs can have overlapping error codes. The API is diff --git a/politeiawww/comments/error.go b/politeiawww/comments/error.go index 9f205d3a6..8dfbc2f0a 100644 --- a/politeiawww/comments/error.go +++ b/politeiawww/comments/error.go @@ -21,7 +21,7 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err var ( ue v1.UserErrorReply pe v1.PluginErrorReply - pde pdclient.Error + pde pdclient.RespError ) switch { case errors.As(err, &ue): diff --git a/politeiawww/comments/process.go b/politeiawww/comments/process.go index 17596c68c..13bf0fed4 100644 --- a/politeiawww/comments/process.go +++ b/politeiawww/comments/process.go @@ -329,17 +329,6 @@ func (c *Comments) processVotes(ctx context.Context, v v1.Votes) (*v1.VotesReply func (c *Comments) processTimestamps(ctx context.Context, t v1.Timestamps, isAdmin bool) (*v1.TimestampsReply, error) { log.Tracef("processTimestamps: %v %v", t.Token, t.CommentIDs) - // Get record state - r, err := c.recordNoFiles(ctx, t.Token) - if err != nil { - if err == errRecordNotFound { - return nil, v1.UserErrorReply{ - ErrorCode: v1.ErrorCodeRecordNotFound, - } - } - return nil, err - } - // Verify size of request switch { case len(t.CommentIDs) == 0: @@ -356,6 +345,17 @@ func (c *Comments) processTimestamps(ctx context.Context, t v1.Timestamps, isAdm } } + // Get record state + r, err := c.recordNoFiles(ctx, t.Token) + if err != nil { + if err == errRecordNotFound { + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeRecordNotFound, + } + } + return nil, err + } + // Get timestamps ct := comments.Timestamps{ CommentIDs: t.CommentIDs, diff --git a/politeiawww/records/error.go b/politeiawww/records/error.go index af3c5a778..6d02177d1 100644 --- a/politeiawww/records/error.go +++ b/politeiawww/records/error.go @@ -21,7 +21,7 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err var ( ue v1.UserErrorReply pe v1.PluginErrorReply - pde pdclient.Error + pde pdclient.RespError ) switch { case errors.As(err, &ue): @@ -75,7 +75,7 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err } } -func handlePDError(w http.ResponseWriter, r *http.Request, format string, pde pdclient.Error) { +func handlePDError(w http.ResponseWriter, r *http.Request, format string, pde pdclient.RespError) { var ( pluginID = pde.ErrorReply.PluginID errCode = pde.ErrorReply.ErrorCode diff --git a/politeiawww/ticketvote/error.go b/politeiawww/ticketvote/error.go index c86c3a95b..2f87ff68c 100644 --- a/politeiawww/ticketvote/error.go +++ b/politeiawww/ticketvote/error.go @@ -20,7 +20,7 @@ import ( func respondWithError(w http.ResponseWriter, r *http.Request, format string, err error) { var ( ue v1.UserErrorReply - pde pdclient.Error + pde pdclient.RespError ) switch { case errors.As(err, &ue): @@ -58,7 +58,7 @@ func respondWithError(w http.ResponseWriter, r *http.Request, format string, err } } -func handlePDError(w http.ResponseWriter, r *http.Request, format string, pde pdclient.Error) { +func handlePDError(w http.ResponseWriter, r *http.Request, format string, pde pdclient.RespError) { var ( pluginID = pde.ErrorReply.PluginID errCode = pde.ErrorReply.ErrorCode From f0240673f998ac3a1ae13a8063cfebcade7d92d6 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 22 Mar 2021 13:35:57 -0500 Subject: [PATCH 425/449] tstore/pi: Cleanup and add docs. --- .../backendv2/tstorebe/plugins/pi/hooks.go | 228 ++++++++++-------- politeiad/backendv2/tstorebe/plugins/pi/pi.go | 14 +- .../backendv2/tstorebe/plugins/pi/testing.go | 1 + 3 files changed, 133 insertions(+), 110 deletions(-) diff --git a/politeiad/backendv2/tstorebe/plugins/pi/hooks.go b/politeiad/backendv2/tstorebe/plugins/pi/hooks.go index 7658e178d..f719495cd 100644 --- a/politeiad/backendv2/tstorebe/plugins/pi/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/pi/hooks.go @@ -25,8 +25,8 @@ const ( ) var ( - // allowedTextFiles contains the only text files that are allowed - // to be submitted as part of a proposal. + // allowedTextFiles contains the filenames of the only text files + // that are allowed to be submitted as part of a proposal. allowedTextFiles = map[string]struct{}{ pi.FileNameIndexFile: {}, pi.FileNameProposalMetadata: {}, @@ -34,33 +34,101 @@ var ( } ) -func tokenDecode(token string) ([]byte, error) { - return util.TokenDecode(util.TokenTypeTstore, token) +// hookNewRecordPre adds pi specific validation to the RecordNew tstore backend +// method. +func (p *piPlugin) hookNewRecordPre(payload string) error { + var nr plugins.HookNewRecordPre + err := json.Unmarshal([]byte(payload), &nr) + if err != nil { + return err + } + + return p.proposalFilesVerify(nr.Files) } -// proposalMetadataDecode decodes and returns the ProposalMetadata from the -// provided backend files. If a ProposalMetadata is not found, nil is returned. -func proposalMetadataDecode(files []backend.File) (*pi.ProposalMetadata, error) { - var propMD *pi.ProposalMetadata - for _, v := range files { - if v.Name != pi.FileNameProposalMetadata { - continue +// hookEditRecordPre adds pi specific validation to the RecordEdit tstore +// backend method. +func (p *piPlugin) hookEditRecordPre(payload string) error { + var er plugins.HookEditRecord + err := json.Unmarshal([]byte(payload), &er) + if err != nil { + return err + } + + // Verify proposal files + err = p.proposalFilesVerify(er.Files) + if err != nil { + return err + } + + // Verify vote status. Edits are not allowed to be made once a vote + // has been authorized. This only needs to be checked for vetted + // records since you cannot authorize or start a ticket vote on an + // unvetted record. + if er.RecordMetadata.State == backend.StateVetted { + t, err := tokenDecode(er.RecordMetadata.Token) + if err != nil { + return err } - if v.Name == pi.FileNameProposalMetadata { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return nil, err - } - var m pi.ProposalMetadata - err = json.Unmarshal(b, &m) - if err != nil { - return nil, err + s, err := p.voteSummary(t) + if err != nil { + return err + } + if s.Status != ticketvote.VoteStatusUnauthorized { + return backend.PluginError{ + PluginID: pi.PluginID, + ErrorCode: uint32(pi.ErrorCodeVoteStatusInvalid), + ErrorContext: fmt.Sprintf("vote status '%v' does not allow "+ + "for proposal edits", ticketvote.VoteStatuses[s.Status]), } - propMD = &m - break } } - return propMD, nil + + return nil +} + +// hookCommentNew extends the comments plugin New command with pi specific +// validation. +func (p *piPlugin) hookCommentNew(token []byte) error { + return p.commentWritesAllowed(token) +} + +// hookCommentNew extends the comments plugin Del command with pi specific +// validation. +func (p *piPlugin) hookCommentDel(token []byte) error { + return p.commentWritesAllowed(token) +} + +// hookCommentNew extends the comments plugin Vote command with pi specific +// validation. +func (p *piPlugin) hookCommentVote(token []byte) error { + return p.commentWritesAllowed(token) +} + +// hookPluginPre extends write commands from other plugins with pi specific +// validation. +func (p *piPlugin) hookPluginPre(treeID int64, token []byte, payload string) error { + // Decode payload + var hpp plugins.HookPluginPre + err := json.Unmarshal([]byte(payload), &hpp) + if err != nil { + return err + } + + // Call plugin hook + switch hpp.PluginID { + case comments.PluginID: + switch hpp.Cmd { + case comments.CmdNew: + return p.hookCommentNew(token) + case comments.CmdDel: + return p.hookCommentDel(token) + case comments.CmdVote: + return p.hookCommentVote(token) + } + } + + return nil } // proposalNameIsValid returns whether the provided name is a valid proposal @@ -174,52 +242,8 @@ func (p *piPlugin) proposalFilesVerify(files []backend.File) error { return nil } -func (p *piPlugin) hookNewRecordPre(payload string) error { - var nr plugins.HookNewRecordPre - err := json.Unmarshal([]byte(payload), &nr) - if err != nil { - return err - } - - return p.proposalFilesVerify(nr.Files) -} - -func (p *piPlugin) hookEditRecordPre(payload string) error { - var er plugins.HookEditRecord - err := json.Unmarshal([]byte(payload), &er) - if err != nil { - return err - } - - // Verify proposal files - err = p.proposalFilesVerify(er.Files) - if err != nil { - return err - } - - // Verify vote status allows proposal edits - if er.RecordMetadata.State == backend.StateVetted { - t, err := tokenDecode(er.RecordMetadata.Token) - if err != nil { - return err - } - s, err := p.voteSummary(t) - if err != nil { - return err - } - if s.Status != ticketvote.VoteStatusUnauthorized { - return backend.PluginError{ - PluginID: pi.PluginID, - ErrorCode: uint32(pi.ErrorCodeVoteStatusInvalid), - ErrorContext: fmt.Sprintf("vote status '%v' does not allow "+ - "for proposal edits", ticketvote.VoteStatuses[s.Status]), - } - } - } - - return nil -} - +// voteSummary requests the vote summary from the ticketvote plugin for a +// record. func (p *piPlugin) voteSummary(token []byte) (*ticketvote.SummaryReply, error) { reply, err := p.backend.PluginRead(token, ticketvote.PluginID, ticketvote.CmdSummary, "") @@ -234,9 +258,11 @@ func (p *piPlugin) voteSummary(token []byte) (*ticketvote.SummaryReply, error) { return &sr, nil } -// commentWritesVerify verifies that a record's vote status allows writes from -// the comments plugin. -func (p *piPlugin) commentWritesVerify(token []byte) error { +// commentWritesAllowed verifies that a proposal has a vote status that allows +// comment writes to be made to the proposal. This includes both comments and +// comment votes. Comment writes are allowed up until the proposal has finished +// voting. +func (p *piPlugin) commentWritesAllowed(token []byte) error { vs, err := p.voteSummary(token) if err != nil { return err @@ -244,9 +270,10 @@ func (p *piPlugin) commentWritesVerify(token []byte) error { switch vs.Status { case ticketvote.VoteStatusUnauthorized, ticketvote.VoteStatusAuthorized, ticketvote.VoteStatusStarted: - // Writes are allowed on these vote statuses + // Comment writes are allowed on these vote statuses return nil default: + // Vote status does not allow writes return backend.PluginError{ PluginID: pi.PluginID, ErrorCode: uint32(pi.ErrorCodeVoteStatusInvalid), @@ -255,38 +282,33 @@ func (p *piPlugin) commentWritesVerify(token []byte) error { } } -func (p *piPlugin) hookCommentNew(token []byte) error { - return p.commentWritesVerify(token) -} - -func (p *piPlugin) hookCommentDel(token []byte) error { - return p.commentWritesVerify(token) -} - -func (p *piPlugin) hookCommentVote(token []byte) error { - return p.commentWritesVerify(token) +// tokenDecode returns the decoded censorship token. An error will be returned +// if the token is not a full length token. +func tokenDecode(token string) ([]byte, error) { + return util.TokenDecode(util.TokenTypeTstore, token) } -func (p *piPlugin) hookPluginPre(treeID int64, token []byte, payload string) error { - // Decode payload - var hpp plugins.HookPluginPre - err := json.Unmarshal([]byte(payload), &hpp) - if err != nil { - return err - } - - // Call plugin hook - switch hpp.PluginID { - case comments.PluginID: - switch hpp.Cmd { - case comments.CmdNew: - return p.hookCommentNew(token) - case comments.CmdDel: - return p.hookCommentDel(token) - case comments.CmdVote: - return p.hookCommentVote(token) +// proposalMetadataDecode decodes and returns the ProposalMetadata from the +// provided backend files. If a ProposalMetadata is not found, nil is returned. +func proposalMetadataDecode(files []backend.File) (*pi.ProposalMetadata, error) { + var propMD *pi.ProposalMetadata + for _, v := range files { + if v.Name != pi.FileNameProposalMetadata { + continue + } + if v.Name == pi.FileNameProposalMetadata { + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return nil, err + } + var m pi.ProposalMetadata + err = json.Unmarshal(b, &m) + if err != nil { + return nil, err + } + propMD = &m + break } } - - return nil + return propMD, nil } diff --git a/politeiad/backendv2/tstorebe/plugins/pi/pi.go b/politeiad/backendv2/tstorebe/plugins/pi/pi.go index bd2097551..a07d782d0 100644 --- a/politeiad/backendv2/tstorebe/plugins/pi/pi.go +++ b/politeiad/backendv2/tstorebe/plugins/pi/pi.go @@ -22,7 +22,7 @@ var ( _ plugins.PluginClient = (*piPlugin)(nil) ) -// piPlugin satisfies the plugins.PluginClient interface. +// piPlugin satisfies the plugins PluginClient interface. type piPlugin struct { backend backend.Backend @@ -44,7 +44,7 @@ type piPlugin struct { // Setup performs any plugin setup that is required. // -// This function satisfies the plugins.PluginClient interface. +// This function satisfies the plugins PluginClient interface. func (p *piPlugin) Setup() error { log.Tracef("pi Setup") @@ -53,7 +53,7 @@ func (p *piPlugin) Setup() error { // Cmd executes a plugin command. // -// This function satisfies the plugins.PluginClient interface. +// This function satisfies the plugins PluginClient interface. func (p *piPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { log.Tracef("pi Cmd: %v %x %v %v", treeID, token, cmd, payload) @@ -62,7 +62,7 @@ func (p *piPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, // Hook executes a plugin hook. // -// This function satisfies the plugins.PluginClient interface. +// This function satisfies the plugins PluginClient interface. func (p *piPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { log.Tracef("pi Hook: %v %x %v", plugins.Hooks[h], token, treeID) @@ -80,7 +80,7 @@ func (p *piPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload str // Fsck performs a plugin filesystem check. // -// This function satisfies the plugins.PluginClient interface. +// This function satisfies the plugins PluginClient interface. func (p *piPlugin) Fsck(treeIDs []int64) error { log.Tracef("pi Fsck") @@ -89,7 +89,7 @@ func (p *piPlugin) Fsck(treeIDs []int64) error { // Settings returns the plugin's settings. // -// This function satisfies the plugins.PluginClient interface. +// This function satisfies the plugins PluginClient interface. func (p *piPlugin) Settings() []backend.PluginSetting { log.Tracef("pi Settings") @@ -203,7 +203,7 @@ func New(backend backend.Backend, settings []backend.PluginSetting, dataDir stri } // Encode the supported chars so that they can be returned as a - // string plugin setting. + // plugin setting string. b, err := json.Marshal(nameSupportedChars) if err != nil { return nil, err diff --git a/politeiad/backendv2/tstorebe/plugins/pi/testing.go b/politeiad/backendv2/tstorebe/plugins/pi/testing.go index 176ba01e4..d8a55e008 100644 --- a/politeiad/backendv2/tstorebe/plugins/pi/testing.go +++ b/politeiad/backendv2/tstorebe/plugins/pi/testing.go @@ -14,6 +14,7 @@ import ( "github.com/decred/politeia/util" ) +// newTestPiPlugin returns a piPlugin that has been setup for testing. func newTestPiPlugin(t *testing.T) (*piPlugin, func()) { // Create plugin data directory dataDir, err := ioutil.TempDir("", pi.PluginID) From b354f8bd1b0f777af3b05dc126c8492e9328ba88 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 22 Mar 2021 14:06:11 -0500 Subject: [PATCH 426/449] tstore: Cleanup. --- politeiad/backendv2/tstorebe/tstore/anchor.go | 58 + .../tstore/{client.go => blobclient.go} | 0 .../backendv2/tstorebe/tstore/convert.go | 178 --- .../backendv2/tstorebe/tstore/extradata.go | 104 ++ politeiad/backendv2/tstorebe/tstore/record.go | 907 ++++++++++++++ .../backendv2/tstorebe/tstore/recordindex.go | 59 + politeiad/backendv2/tstorebe/tstore/trees.go | 191 +++ politeiad/backendv2/tstorebe/tstore/tstore.go | 1105 ----------------- politeiad/backendv2/tstorebe/tstorebe.go | 14 +- 9 files changed, 1326 insertions(+), 1290 deletions(-) rename politeiad/backendv2/tstorebe/tstore/{client.go => blobclient.go} (100%) delete mode 100644 politeiad/backendv2/tstorebe/tstore/convert.go create mode 100644 politeiad/backendv2/tstorebe/tstore/extradata.go create mode 100644 politeiad/backendv2/tstorebe/tstore/record.go create mode 100644 politeiad/backendv2/tstorebe/tstore/trees.go diff --git a/politeiad/backendv2/tstorebe/tstore/anchor.go b/politeiad/backendv2/tstorebe/tstore/anchor.go index d95a982e4..a54f554a7 100644 --- a/politeiad/backendv2/tstorebe/tstore/anchor.go +++ b/politeiad/backendv2/tstorebe/tstore/anchor.go @@ -7,7 +7,9 @@ package tstore import ( "bytes" "crypto/sha256" + "encoding/base64" "encoding/hex" + "encoding/json" "errors" "fmt" "time" @@ -15,6 +17,7 @@ import ( dcrtime "github.com/decred/dcrtime/api/v2" "github.com/decred/dcrtime/merkle" "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" + "github.com/decred/politeia/util" "github.com/google/trillian" "github.com/google/trillian/types" "google.golang.org/grpc/codes" @@ -567,3 +570,58 @@ func (t *Tstore) anchorTrees() error { return nil } + +func convertBlobEntryFromAnchor(a anchor) (*store.BlobEntry, error) { + data, err := json.Marshal(a) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorAnchor, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertAnchorFromBlobEntry(be store.BlobEntry) (*anchor, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorAnchor { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorAnchor) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, fmt.Errorf("decode digest: %v", err) + } + if !bytes.Equal(util.Digest(b), digest) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), digest) + } + var a anchor + err = json.Unmarshal(b, &a) + if err != nil { + return nil, fmt.Errorf("unmarshal anchor: %v", err) + } + + return &a, nil +} diff --git a/politeiad/backendv2/tstorebe/tstore/client.go b/politeiad/backendv2/tstorebe/tstore/blobclient.go similarity index 100% rename from politeiad/backendv2/tstorebe/tstore/client.go rename to politeiad/backendv2/tstorebe/tstore/blobclient.go diff --git a/politeiad/backendv2/tstorebe/tstore/convert.go b/politeiad/backendv2/tstorebe/tstore/convert.go deleted file mode 100644 index 8b72b397e..000000000 --- a/politeiad/backendv2/tstorebe/tstore/convert.go +++ /dev/null @@ -1,178 +0,0 @@ -// Copyright (c) 2020-2021 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package tstore - -import ( - "bytes" - "encoding/base64" - "encoding/hex" - "encoding/json" - "fmt" - - backend "github.com/decred/politeia/politeiad/backendv2" - "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" - "github.com/decred/politeia/util" -) - -func convertBlobEntryFromFile(f backend.File) (*store.BlobEntry, error) { - data, err := json.Marshal(f) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorFile, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func convertBlobEntryFromMetadataStream(ms backend.MetadataStream) (*store.BlobEntry, error) { - data, err := json.Marshal(ms) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorMetadataStream, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func convertBlobEntryFromRecordMetadata(rm backend.RecordMetadata) (*store.BlobEntry, error) { - data, err := json.Marshal(rm) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorRecordMetadata, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func convertBlobEntryFromRecordIndex(ri recordIndex) (*store.BlobEntry, error) { - data, err := json.Marshal(ri) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorRecordIndex, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func convertBlobEntryFromAnchor(a anchor) (*store.BlobEntry, error) { - data, err := json.Marshal(a) - if err != nil { - return nil, err - } - hint, err := json.Marshal( - store.DataDescriptor{ - Type: store.DataTypeStructure, - Descriptor: dataDescriptorAnchor, - }) - if err != nil { - return nil, err - } - be := store.NewBlobEntry(hint, data) - return &be, nil -} - -func convertRecordIndexFromBlobEntry(be store.BlobEntry) (*recordIndex, error) { - // Decode and validate data hint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorRecordIndex { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorRecordIndex) - } - - // Decode data - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - digest, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, fmt.Errorf("decode digest: %v", err) - } - if !bytes.Equal(util.Digest(b), digest) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), digest) - } - var ri recordIndex - err = json.Unmarshal(b, &ri) - if err != nil { - return nil, fmt.Errorf("unmarshal recordIndex: %v", err) - } - - return &ri, nil -} - -func convertAnchorFromBlobEntry(be store.BlobEntry) (*anchor, error) { - // Decode and validate data hint - b, err := base64.StdEncoding.DecodeString(be.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Descriptor != dataDescriptorAnchor { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorAnchor) - } - - // Decode data - b, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - digest, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, fmt.Errorf("decode digest: %v", err) - } - if !bytes.Equal(util.Digest(b), digest) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), digest) - } - var a anchor - err = json.Unmarshal(b, &a) - if err != nil { - return nil, fmt.Errorf("unmarshal anchor: %v", err) - } - - return &a, nil -} diff --git a/politeiad/backendv2/tstorebe/tstore/extradata.go b/politeiad/backendv2/tstorebe/tstore/extradata.go new file mode 100644 index 000000000..e66004a16 --- /dev/null +++ b/politeiad/backendv2/tstorebe/tstore/extradata.go @@ -0,0 +1,104 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tstore + +import ( + "encoding/json" + + backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/google/uuid" +) + +const ( + // keyPrefixEncrypted is prefixed onto key-value store keys if the + // data is encrypted. We do this so that when a record is made + // public we can save the plain text record content blobs using the + // same keys, but without the prefix. Using a new key for the plain + // text blobs would not work since we cannot append a new leaf onto + // the tlog without getting a duplicate leaf error. + keyPrefixEncrypted = "e_" +) + +// extraData is the data that is stored in the log leaf ExtraData field. It is +// saved as a JSON encoded byte slice. The JSON keys have been abbreviated to +// minimize the size of a trillian log leaf. +type extraData struct { + // Key contains the key-value store key. If this blob is part of an + // unvetted record the key will need to be prefixed with the + // keyPrefixEncrypted in order to retrieve the blob from the kv + // store. Use the extraData.storeKey() method to retrieve the key. + // Do NOT reference this key directly. + Key string `json:"k"` + + // Desc contains the blob entry data descriptor. + Desc string `json:"d"` + + // State indicates the record state of the blob that this leaf + // corresponds to. Unvetted blobs encrypted prior to being saved + // to the store. When retrieving unvetted blobs from the kv store + // the keyPrefixEncrypted prefix must be added to the Key field. + // State will not be populated for anchor records. + State backend.StateT `json:"s,omitempty"` +} + +// storeKey returns the kv store key for the blob. If the blob is part of an +// unvetted record it will be saved as an encrypted blob in the kv store and +// the key is prefixed with keyPrefixEncrypted. +func (e *extraData) storeKey() string { + if e.State == backend.StateUnvetted { + return keyPrefixEncrypted + e.Key + } + return e.Key +} + +// storeKeyNoPrefix returns the kv store key without any encryption prefix, +// even if the leaf corresponds to a unvetted blob. +func (e *extraData) storeKeyNoPrefix() string { + return e.Key +} + +// extraDataEncode encodes prepares an extraData using the provided arguments +// then returns the JSON encoded byte slice. +func extraDataEncode(key, desc string, state backend.StateT) ([]byte, error) { + // The encryption prefix is stripped from the key if one exists. + ed := extraData{ + Key: storeKeyCleaned(key), + Desc: desc, + State: state, + } + b, err := json.Marshal(ed) + if err != nil { + return nil, err + } + return b, nil +} + +// extraDataDecode decodes a JSON byte slice into a extraData. +func extraDataDecode(b []byte) (*extraData, error) { + var ed extraData + err := json.Unmarshal(b, &ed) + if err != nil { + return nil, err + } + return &ed, nil +} + +// storeKeyNew returns a new key for the key-value store. If the data is +// encrypted the key is prefixed. +func storeKeyNew(encrypt bool) string { + k := uuid.New().String() + if encrypt { + k = keyPrefixEncrypted + k + } + return k +} + +// storeKeyCleaned strips the key-value store key of the encryption prefix if +// one is present. +func storeKeyCleaned(key string) string { + // A uuid string is 36 bytes. Return the last 36 bytes of the + // string. This will strip the prefix if it exists. + return key[len(key)-36:] +} diff --git a/politeiad/backendv2/tstorebe/tstore/record.go b/politeiad/backendv2/tstorebe/tstore/record.go new file mode 100644 index 000000000..bc3ab9ac9 --- /dev/null +++ b/politeiad/backendv2/tstorebe/tstore/record.go @@ -0,0 +1,907 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tstore + +import ( + "bytes" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + + backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" + "github.com/decred/politeia/util" + "github.com/google/trillian" + "google.golang.org/grpc/codes" +) + +const ( + // Blob entry data descriptors + dataDescriptorRecordMetadata = "pd-recordmd-v1" + dataDescriptorMetadataStream = "pd-mdstream-v1" + dataDescriptorFile = "pd-file-v1" + dataDescriptorRecordIndex = "pd-rindex-v1" + dataDescriptorAnchor = "pd-anchor-v1" +) + +// recordBlobsSave saves the provided blobs to the kv store, appends a leaf +// to the trillian tree for each blob, and returns the record index for the +// blobs. +func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, recordMD backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) (*recordIndex, error) { + log.Tracef("recordBlobsSave: %v", treeID) + + // Verify there are no duplicate metadata streams + md := make(map[string]map[uint32]struct{}, len(metadata)) + for _, v := range metadata { + if v.PluginID == "" || v.StreamID == 0 { + return nil, fmt.Errorf("invalid metadata stream: '%v' %v", + v.PluginID, v.StreamID) + } + pmd, ok := md[v.PluginID] + if !ok { + pmd = make(map[uint32]struct{}, len(metadata)) + } + _, ok = pmd[v.StreamID] + if ok { + return nil, fmt.Errorf("duplicate metadata stream: %v %v", + v.PluginID, v.StreamID) + } + pmd[v.StreamID] = struct{}{} + md[v.PluginID] = pmd + } + + // Verify there are no duplicate files + fn := make(map[string]struct{}, len(files)) + for _, v := range files { + if v.Name == "" { + return nil, fmt.Errorf("empty filename") + } + _, ok := fn[v.Name] + if ok { + return nil, fmt.Errorf("duplicate filename found: %v", v.Name) + } + fn[v.Name] = struct{}{} + } + + // Prepare the blob entries. The record index can also be created + // during this step. + var ( + // [pluginID][streamID]BlobEntry + beMetadata = make(map[string]map[uint32]store.BlobEntry, len(metadata)) + + // [filename]BlobEntry + beFiles = make(map[string]store.BlobEntry, len(files)) + + idx = recordIndex{ + State: recordMD.State, + Version: recordMD.Version, + Iteration: recordMD.Iteration, + Metadata: make(map[string]map[uint32][]byte, len(metadata)), + Files: make(map[string][]byte, len(files)), + } + + // digests is used to aggregate the digests from all record + // content. This is used later on to see if any of the content + // already exists in the tstore. + digests = make(map[string]struct{}, 256) + ) + + // Setup record metadata + beRecordMD, err := convertBlobEntryFromRecordMetadata(recordMD) + if err != nil { + return nil, err + } + m, err := merkleLeafHashForBlobEntry(*beRecordMD) + if err != nil { + return nil, err + } + idx.RecordMetadata = m + digests[beRecordMD.Digest] = struct{}{} + + // Setup metdata streams + for _, v := range metadata { + // Blob entry + be, err := convertBlobEntryFromMetadataStream(v) + if err != nil { + return nil, err + } + streams, ok := beMetadata[v.PluginID] + if !ok { + streams = make(map[uint32]store.BlobEntry, len(metadata)) + } + streams[v.StreamID] = *be + beMetadata[v.PluginID] = streams + + // Record index + m, err := merkleLeafHashForBlobEntry(*be) + if err != nil { + return nil, err + } + streamsIdx, ok := idx.Metadata[v.PluginID] + if !ok { + streamsIdx = make(map[uint32][]byte, len(metadata)) + } + streamsIdx[v.StreamID] = m + idx.Metadata[v.PluginID] = streamsIdx + + // Aggregate digest + digests[be.Digest] = struct{}{} + } + + // Setup files + for _, v := range files { + // Blob entry + be, err := convertBlobEntryFromFile(v) + if err != nil { + return nil, err + } + beFiles[v.Name] = *be + + // Record Index + m, err := merkleLeafHashForBlobEntry(*be) + if err != nil { + return nil, err + } + idx.Files[v.Name] = m + + // Aggregate digest + digests[be.Digest] = struct{}{} + } + + // Check if any of the content already exists. Different record + // versions that reference the same data is fine, but this data + // should not be saved to the store again. We can find duplicates + // by comparing the blob entry digest to the log leaf value. They + // will be the same if the record content is the same. + dups := make(map[string]struct{}, len(digests)) + for _, v := range leavesAll { + d := hex.EncodeToString(v.LeafValue) + _, ok := digests[d] + if ok { + // A piece of the new record content already exsits in the + // tstore. Save the digest as a duplcate. + dups[d] = struct{}{} + } + } + + // Prepare blobs for the kv store + var ( + blobs = make(map[string][]byte, len(digests)) + leaves = make([]*trillian.LogLeaf, 0, len(blobs)) + + // dupBlobs contains the blob entries for record content that + // already exists. We may need these blob entries later on if + // the duplicate content is encrypted and it needs to be saved + // plain text. + dupBlobs = make(map[string]store.BlobEntry, len(digests)) + ) + + // Only vetted data should be saved plain text + var encrypt bool + switch idx.State { + case backend.StateUnvetted: + encrypt = true + case backend.StateVetted: + // Save plain text + encrypt = false + default: + // Something is wrong + e := fmt.Sprintf("invalid record state %v %v", treeID, idx.State) + panic(e) + } + + // Prepare record metadata blobs and leaves + _, ok := dups[beRecordMD.Digest] + if !ok { + // Not a duplicate. Prepare kv store blob. + b, err := store.Blobify(*beRecordMD) + if err != nil { + return nil, err + } + k := storeKeyNew(encrypt) + blobs[k] = b + + // Prepare tlog leaf + extraData, err := extraDataEncode(k, + dataDescriptorRecordMetadata, idx.State) + if err != nil { + return nil, err + } + digest, err := hex.DecodeString(beRecordMD.Digest) + if err != nil { + return nil, err + } + leaves = append(leaves, newLogLeaf(digest, extraData)) + } else { + // This is a duplicate. Stash is for now. We may need to save + // it as plain text later. + dupBlobs[beRecordMD.Digest] = *beRecordMD + } + + // Prepare metadata stream blobs and leaves + for _, v := range beMetadata { + for _, be := range v { + _, ok := dups[be.Digest] + if !ok { + // Not a duplicate. Prepare kv store blob. + b, err := store.Blobify(be) + if err != nil { + return nil, err + } + k := storeKeyNew(encrypt) + blobs[k] = b + + // Prepare tlog leaf + extraData, err := extraDataEncode(k, + dataDescriptorMetadataStream, idx.State) + if err != nil { + return nil, err + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, err + } + leaves = append(leaves, newLogLeaf(digest, extraData)) + + continue + } + + // This is a duplicate. Stash is for now. We may need to save + // it as plain text later. + dupBlobs[be.Digest] = be + } + } + + // Prepare file blobs and leaves + for _, be := range beFiles { + _, ok := dups[be.Digest] + if !ok { + // Not a duplicate. Prepare kv store blob. + b, err := store.Blobify(be) + if err != nil { + return nil, err + } + k := storeKeyNew(encrypt) + blobs[k] = b + + // Prepare tlog leaf + extraData, err := extraDataEncode(k, dataDescriptorFile, idx.State) + if err != nil { + return nil, err + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, err + } + leaves = append(leaves, newLogLeaf(digest, extraData)) + + continue + } + + // This is a duplicate. Stash is for now. We may need to save + // it as plain text later. + dupBlobs[be.Digest] = be + } + + // Verify at least one new blob is being saved to the kv store + if len(blobs) == 0 { + return nil, backend.ErrNoRecordChanges + } + + log.Debugf("Saving %v record content blobs", len(blobs)) + + // Save blobs to the kv store + err = t.store.Put(blobs, encrypt) + if err != nil { + return nil, fmt.Errorf("store Put: %v", err) + } + + // Append leaves onto the trillian tree + queued, _, err := t.tlog.LeavesAppend(treeID, leaves) + if err != nil { + return nil, fmt.Errorf("LeavesAppend: %v", err) + } + failed := make([]string, 0, len(queued)) + for _, v := range queued { + c := codes.Code(v.QueuedLeaf.GetStatus().GetCode()) + if c != codes.OK { + failed = append(failed, fmt.Sprintf("%v", c)) + } + } + if len(failed) > 0 { + return nil, fmt.Errorf("append leaves failed: %v", failed) + } + + // When a record is made public the record content needs to be + // resaved to the key-value store as unencrypted. + var ( + isPublic = recordMD.Status == backend.StatusPublic + + // Iteration and version are reset back to 1 when a record is + // made public. + iterIsReset = recordMD.Iteration == 1 + ) + if !isPublic || !iterIsReset { + // Record is not being made public. Nothing else to do. + return &idx, nil + } + + // Resave all of the duplicate blobs as plain text. A duplicate + // blob means the record content existed prior to the status + // change. + blobs = make(map[string][]byte, len(dupBlobs)) + for _, v := range leavesAll { + d := hex.EncodeToString(v.LeafValue) + _, ok := dups[d] + if !ok { + // Not a duplicate + continue + } + + // This is a duplicate. If its unvetted it will need to be + // resaved as plain text. + ed, err := extraDataDecode(v.ExtraData) + if err != nil { + return nil, err + } + if ed.State == backend.StateVetted { + // Not unvetted. No need to resave it. + continue + } + + // Prepare plain text blob + be, ok := dupBlobs[d] + if !ok { + // Should not happen + return nil, fmt.Errorf("blob entry not found %v", d) + } + b, err := store.Blobify(be) + if err != nil { + return nil, err + } + blobs[ed.storeKeyNoPrefix()] = b + } + if len(blobs) == 0 { + // This should not happen + return nil, fmt.Errorf("no blobs found to resave as plain text") + } + + log.Debugf("Resaving %v encrypted blobs as plain text", len(blobs)) + + err = t.store.Put(blobs, false) + if err != nil { + return nil, fmt.Errorf("store Put: %v", err) + } + + return &idx, nil +} + +func (t *Tstore) recordSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) (*recordIndex, error) { + // Verify tree exists + if !t.TreeExists(treeID) { + return nil, backend.ErrRecordNotFound + } + + // Get tree leaves + leavesAll, err := t.tlog.LeavesAll(treeID) + if err != nil { + return nil, fmt.Errorf("LeavesAll %v: %v", treeID, err) + } + + // Get the existing record index + currIdx, err := t.recordIndexLatest(leavesAll) + if errors.Is(err, backend.ErrRecordNotFound) { + // No record versions exist yet. This is ok. + currIdx = &recordIndex{ + Metadata: make(map[string]map[uint32][]byte), + Files: make(map[string][]byte), + } + } else if err != nil { + return nil, fmt.Errorf("recordIndexLatest: %v", err) + } + + // Verify tree is not frozen + if currIdx.Frozen { + return nil, backend.ErrRecordLocked + } + + // Save the record + idx, err := t.recordBlobsSave(treeID, leavesAll, rm, metadata, files) + if err != nil { + if err == backend.ErrNoRecordChanges { + return nil, err + } + return nil, fmt.Errorf("recordBlobsSave: %v", err) + } + + return idx, nil +} + +// RecordSave saves the provided record to tstore. Once the record contents +// have been successfully saved to tstore, a recordIndex is created for this +// version of the record and saved to tstore as well. The record update is not +// considered to be valid until the record index has been successfully saved. +// If the record content makes it in but the record index does not, the record +// content blobs are orphaned and ignored. +func (t *Tstore) RecordSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { + log.Tracef("RecordSave: %v", treeID) + + // Save the record + idx, err := t.recordSave(treeID, rm, metadata, files) + if err != nil { + return err + } + + // Save the record index + err = t.recordIndexSave(treeID, *idx) + if err != nil { + return fmt.Errorf("recordIndexSave: %v", err) + } + + return nil +} + +// RecordDel walks the provided tree and deletes all blobs in the store that +// correspond to record files. This is done for all versions and all iterations +// of the record. Record metadata and metadata stream blobs are not deleted. +func (t *Tstore) RecordDel(treeID int64) error { + log.Tracef("RecordDel: %v", treeID) + + // Verify tree exists + if !t.TreeExists(treeID) { + return backend.ErrRecordNotFound + } + + // Get all tree leaves + leavesAll, err := t.tlog.LeavesAll(treeID) + if err != nil { + return err + } + + // Ensure tree is frozen. Deleting files from the store is only + // allowed on frozen trees. + currIdx, err := t.recordIndexLatest(leavesAll) + if err != nil { + return err + } + if !currIdx.Frozen { + return fmt.Errorf("tree is not frozen") + } + + // Retrieve all record indexes + indexes, err := t.recordIndexes(leavesAll) + if err != nil { + return err + } + + // Aggregate the keys for all file blobs of all versions. The + // record index points to the log leaf merkle leaf hash. The log + // leaf contains the kv store key. + merkles := make(map[string]struct{}, len(leavesAll)) + for _, v := range indexes { + for _, merkle := range v.Files { + merkles[hex.EncodeToString(merkle)] = struct{}{} + } + } + keys := make([]string, 0, len(merkles)) + for _, v := range leavesAll { + _, ok := merkles[hex.EncodeToString(v.MerkleLeafHash)] + if ok { + ed, err := extraDataDecode(v.ExtraData) + if err != nil { + return err + } + keys = append(keys, ed.storeKey()) + } + } + + // Delete file blobs from the store + err = t.store.Del(keys) + if err != nil { + return fmt.Errorf("store Del: %v", err) + } + + return nil +} + +// RecordFreeze updates the status of a record then freezes the trillian tree +// to prevent any additional updates. +// +// A tree is considered to be frozen once the record index has been saved with +// its Frozen field set to true. The only thing that can be appended onto a +// frozen tree is one additional anchor record. Once a frozen tree has been +// anchored, the tstore fsck function will update the status of the tree to +// frozen in trillian, at which point trillian will prevent any changes to the +// tree. +func (t *Tstore) RecordFreeze(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { + log.Tracef("RecordFreeze: %v", treeID) + + // Save updated record + idx, err := t.recordSave(treeID, rm, metadata, files) + if err != nil { + return err + } + + // Mark the record as frozen + idx.Frozen = true + + // Save the record index + return t.recordIndexSave(treeID, *idx) +} + +// record returns the specified record. +// +// Version is used to request a specific version of a record. If no version is +// provided then the most recent version of the record will be returned. +// +// Filenames can be used to request specific files. If filenames is not empty +// then the specified files will be the only files returned. +// +// OmitAllFiles can be used to retrieve a record without any of the record +// files. This supersedes the filenames argument. +func (t *Tstore) record(treeID int64, version uint32, filenames []string, omitAllFiles bool) (*backend.Record, error) { + // Verify tree exists + if !t.TreeExists(treeID) { + return nil, backend.ErrRecordNotFound + } + + // Get tree leaves + leaves, err := t.tlog.LeavesAll(treeID) + if err != nil { + return nil, fmt.Errorf("LeavesAll %v: %v", treeID, err) + } + + // Use the record index to pull the record content from the store. + // The keys for the record content first need to be extracted from + // their log leaf. + idx, err := t.recordIndex(leaves, version) + if err != nil { + return nil, err + } + + // Compile merkle root hashes of record content + merkles := make(map[string]struct{}, 64) + merkles[hex.EncodeToString(idx.RecordMetadata)] = struct{}{} + for _, streams := range idx.Metadata { + for _, v := range streams { + merkles[hex.EncodeToString(v)] = struct{}{} + } + } + switch { + case omitAllFiles: + // Don't include any files + case len(filenames) > 0: + // Only included the specified files + filesToInclude := make(map[string]struct{}, len(filenames)) + for _, v := range filenames { + filesToInclude[v] = struct{}{} + } + for fn, v := range idx.Files { + if _, ok := filesToInclude[fn]; ok { + merkles[hex.EncodeToString(v)] = struct{}{} + } + } + default: + // Include all files + for _, v := range idx.Files { + merkles[hex.EncodeToString(v)] = struct{}{} + } + } + + // Walk the tree and extract the record content keys + keys := make([]string, 0, len(idx.Metadata)+len(idx.Files)+1) + for _, v := range leaves { + _, ok := merkles[hex.EncodeToString(v.MerkleLeafHash)] + if !ok { + // Not part of the record content + continue + } + + // Leaf is part of record content. Save the kv store key. + ed, err := extraDataDecode(v.ExtraData) + if err != nil { + return nil, err + } + + var key string + switch idx.State { + case backend.StateVetted: + // If the record is vetted the content may exist in the store + // as both an encrypted blob and a plain text blob. Always pull + // the plaintext blob. + key = ed.storeKeyNoPrefix() + default: + // Pull the encrypted blob + key = ed.storeKey() + } + keys = append(keys, key) + } + + // Get record content from store + blobs, err := t.store.Get(keys) + if err != nil { + return nil, fmt.Errorf("store Get: %v", err) + } + if len(keys) != len(blobs) { + // One or more blobs were not found. This is allowed since the + // blobs for a censored record will not exist, but the record + // metadata and metadata streams should still be returned. + log.Tracef("Blobs not found %v: want %v, got %v", + treeID, len(keys), len(blobs)) + } + + // Decode blobs + entries := make([]store.BlobEntry, 0, len(keys)) + for _, v := range blobs { + be, err := store.Deblob(v) + if err != nil { + return nil, err + } + entries = append(entries, *be) + } + + // Decode blob entries + var ( + recordMD *backend.RecordMetadata + metadata = make([]backend.MetadataStream, 0, len(idx.Metadata)) + files = make([]backend.File, 0, len(idx.Files)) + ) + for _, v := range entries { + // Decode the data hint + b, err := base64.StdEncoding.DecodeString(v.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Type != store.DataTypeStructure { + return nil, fmt.Errorf("invalid data type; got %v, want %v", + dd.Type, store.DataTypeStructure) + } + + // Decode the data + b, err = base64.StdEncoding.DecodeString(v.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + digest, err := hex.DecodeString(v.Digest) + if err != nil { + return nil, fmt.Errorf("decode Hash: %v", err) + } + if !bytes.Equal(util.Digest(b), digest) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), digest) + } + switch dd.Descriptor { + case dataDescriptorRecordMetadata: + var rm backend.RecordMetadata + err = json.Unmarshal(b, &rm) + if err != nil { + return nil, fmt.Errorf("unmarshal RecordMetadata: %v", err) + } + recordMD = &rm + case dataDescriptorMetadataStream: + var ms backend.MetadataStream + err = json.Unmarshal(b, &ms) + if err != nil { + return nil, fmt.Errorf("unmarshal MetadataStream: %v", err) + } + metadata = append(metadata, ms) + case dataDescriptorFile: + var f backend.File + err = json.Unmarshal(b, &f) + if err != nil { + return nil, fmt.Errorf("unmarshal File: %v", err) + } + files = append(files, f) + default: + return nil, fmt.Errorf("invalid descriptor %v", dd.Descriptor) + } + } + + return &backend.Record{ + RecordMetadata: *recordMD, + Metadata: metadata, + Files: files, + }, nil +} + +// Record returns the specified version of the record. +func (t *Tstore) Record(treeID int64, version uint32) (*backend.Record, error) { + log.Tracef("Record: %v %v", treeID, version) + + return t.record(treeID, version, []string{}, false) +} + +// RecordLatest returns the latest version of a record. +func (t *Tstore) RecordLatest(treeID int64) (*backend.Record, error) { + log.Tracef("RecordLatest: %v", treeID) + + return t.record(treeID, 0, []string{}, false) +} + +// RecordPartial returns a partial record. This method gives the caller fine +// grained control over what version and what files are returned. The only +// required field is the token. All other fields are optional. +// +// Version is used to request a specific version of a record. If no version is +// provided then the most recent version of the record will be returned. +// +// Filenames can be used to request specific files. If filenames is not empty +// then the specified files will be the only files returned. +// +// OmitAllFiles can be used to retrieve a record without any of the record +// files. This supersedes the filenames argument. +func (t *Tstore) RecordPartial(treeID int64, version uint32, filenames []string, omitAllFiles bool) (*backend.Record, error) { + log.Tracef("RecordPartial: %v %v %v %v", + treeID, version, omitAllFiles, filenames) + + return t.record(treeID, version, filenames, omitAllFiles) +} + +// RecordState returns the state of a record. This call does not require +// retrieving any blobs from the kv store. The record state can be derived from +// only the tlog leaves. +func (t *Tstore) RecordState(treeID int64) (backend.StateT, error) { + log.Tracef("RecordState: %v", treeID) + + leaves, err := t.tlog.LeavesAll(treeID) + if err != nil { + return 0, err + } + + if recordIsVetted(leaves) { + return backend.StateVetted, nil + } + + return backend.StateUnvetted, nil +} + +// RecordTimestamps returns the timestamps for the contents of a record. +// Timestamps for the record metadata, metadata streams, and files are all +// returned. +func (t *Tstore) RecordTimestamps(treeID int64, version uint32, token []byte) (*backend.RecordTimestamps, error) { + log.Tracef("RecordTimestamps: %v %v", treeID, version) + + // Verify tree exists + if !t.TreeExists(treeID) { + return nil, backend.ErrRecordNotFound + } + + // Get record index + leaves, err := t.tlog.LeavesAll(treeID) + if err != nil { + return nil, fmt.Errorf("LeavesAll %v: %v", treeID, err) + } + idx, err := t.recordIndex(leaves, version) + if err != nil { + return nil, err + } + + // Get record metadata timestamp + rm, err := t.timestamp(treeID, idx.RecordMetadata, leaves) + if err != nil { + return nil, fmt.Errorf("record metadata timestamp: %v", err) + } + + // Get metadata timestamps + metadata := make(map[string]map[uint32]backend.Timestamp, len(idx.Metadata)) + for pluginID, streams := range idx.Metadata { + for streamID, merkle := range streams { + ts, err := t.timestamp(treeID, merkle, leaves) + if err != nil { + return nil, fmt.Errorf("metadata %v %v timestamp: %v", + pluginID, streamID, err) + } + sts, ok := metadata[pluginID] + if !ok { + sts = make(map[uint32]backend.Timestamp, 64) + } + sts[streamID] = *ts + metadata[pluginID] = sts + } + } + + // Get file timestamps + files := make(map[string]backend.Timestamp, len(idx.Files)) + for k, v := range idx.Files { + ts, err := t.timestamp(treeID, v, leaves) + if err != nil { + return nil, fmt.Errorf("file %v timestamp: %v", k, err) + } + files[k] = *ts + } + + return &backend.RecordTimestamps{ + RecordMetadata: *rm, + Metadata: metadata, + Files: files, + }, nil +} + +// recordIsVetted returns whether the provided leaves contain any vetted record +// indexes. The presence of a vetted record index means the record is vetted. +// The state of a record index is saved to the leaf extra data, which is how we +// determine if a record index is vetted. +func recordIsVetted(leaves []*trillian.LogLeaf) bool { + for _, v := range leaves { + ed, err := extraDataDecode(v.ExtraData) + if err != nil { + panic(err) + } + if ed.Desc == dataDescriptorRecordIndex && + ed.State == backend.StateVetted { + // Vetted record index found + return true + } + } + return false +} + +// merkleLeafHashForBlobEntry returns the merkle leaf hash for a blob entry. +// The merkle leaf hash can be used to retrieve a leaf from its tlog tree. +func merkleLeafHashForBlobEntry(be store.BlobEntry) ([]byte, error) { + leafValue, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, err + } + return merkleLeafHash(leafValue), nil +} + +func convertBlobEntryFromFile(f backend.File) (*store.BlobEntry, error) { + data, err := json.Marshal(f) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorFile, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertBlobEntryFromMetadataStream(ms backend.MetadataStream) (*store.BlobEntry, error) { + data, err := json.Marshal(ms) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorMetadataStream, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertBlobEntryFromRecordMetadata(rm backend.RecordMetadata) (*store.BlobEntry, error) { + data, err := json.Marshal(rm) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorRecordMetadata, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} diff --git a/politeiad/backendv2/tstorebe/tstore/recordindex.go b/politeiad/backendv2/tstorebe/tstore/recordindex.go index c16fc1ccb..c1d9286e9 100644 --- a/politeiad/backendv2/tstorebe/tstore/recordindex.go +++ b/politeiad/backendv2/tstorebe/tstore/recordindex.go @@ -5,12 +5,16 @@ package tstore import ( + "bytes" + "encoding/base64" "encoding/hex" + "encoding/json" "fmt" "sort" backend "github.com/decred/politeia/politeiad/backendv2" "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" + "github.com/decred/politeia/util" "github.com/google/trillian" "google.golang.org/grpc/codes" ) @@ -319,3 +323,58 @@ func parseRecordIndex(indexes []recordIndex, version uint32) (*recordIndex, erro return ri, nil } + +func convertBlobEntryFromRecordIndex(ri recordIndex) (*store.BlobEntry, error) { + data, err := json.Marshal(ri) + if err != nil { + return nil, err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: dataDescriptorRecordIndex, + }) + if err != nil { + return nil, err + } + be := store.NewBlobEntry(hint, data) + return &be, nil +} + +func convertRecordIndexFromBlobEntry(be store.BlobEntry) (*recordIndex, error) { + // Decode and validate data hint + b, err := base64.StdEncoding.DecodeString(be.DataHint) + if err != nil { + return nil, fmt.Errorf("decode DataHint: %v", err) + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, fmt.Errorf("unmarshal DataHint: %v", err) + } + if dd.Descriptor != dataDescriptorRecordIndex { + return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", + dd.Descriptor, dataDescriptorRecordIndex) + } + + // Decode data + b, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, fmt.Errorf("decode Data: %v", err) + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, fmt.Errorf("decode digest: %v", err) + } + if !bytes.Equal(util.Digest(b), digest) { + return nil, fmt.Errorf("data is not coherent; got %x, want %x", + util.Digest(b), digest) + } + var ri recordIndex + err = json.Unmarshal(b, &ri) + if err != nil { + return nil, fmt.Errorf("unmarshal recordIndex: %v", err) + } + + return &ri, nil +} diff --git a/politeiad/backendv2/tstorebe/tstore/trees.go b/politeiad/backendv2/tstorebe/tstore/trees.go new file mode 100644 index 000000000..a6c5db943 --- /dev/null +++ b/politeiad/backendv2/tstorebe/tstore/trees.go @@ -0,0 +1,191 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tstore + +import ( + "bytes" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + + backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" + "github.com/decred/politeia/util" + "github.com/google/trillian" +) + +// TreeNew creates a new tlog tree and returns the tree ID. +func (t *Tstore) TreeNew() (int64, error) { + log.Tracef("TreeNew") + + tree, _, err := t.tlog.TreeNew() + if err != nil { + return 0, err + } + + return tree.TreeId, nil +} + +// TreesAll returns the IDs of all trees in the tstore instance. +func (t *Tstore) TreesAll() ([]int64, error) { + trees, err := t.tlog.TreesAll() + if err != nil { + return nil, err + } + treeIDs := make([]int64, 0, len(trees)) + for _, v := range trees { + treeIDs = append(treeIDs, v.TreeId) + } + return treeIDs, nil +} + +// TreeExists returns whether a tree exists in the trillian log. A tree +// existing doesn't necessarily mean that a record exists. Its possible for a +// tree to have been created but experienced an unexpected error prior to the +// record being saved. +func (t *Tstore) TreeExists(treeID int64) bool { + _, err := t.tlog.Tree(treeID) + return err == nil +} + +// timestamp returns the timestamp given a tlog tree merkle leaf hash. +func (t *Tstore) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trillian.LogLeaf) (*backend.Timestamp, error) { + // Find the leaf + var l *trillian.LogLeaf + for _, v := range leaves { + if bytes.Equal(merkleLeafHash, v.MerkleLeafHash) { + l = v + break + } + } + if l == nil { + return nil, fmt.Errorf("leaf not found") + } + + // Get blob entry from the kv store + ed, err := extraDataDecode(l.ExtraData) + if err != nil { + return nil, err + } + blobs, err := t.store.Get([]string{ed.storeKey()}) + if err != nil { + return nil, fmt.Errorf("store get: %v", err) + } + + // Extract the data blob. Its possible for the data blob to not + // exist if it has been censored. This is ok. We'll still return + // the rest of the timestamp. + var data []byte + if len(blobs) == 1 { + b, ok := blobs[ed.storeKey()] + if !ok { + return nil, fmt.Errorf("blob not found %v", ed.storeKey()) + } + be, err := store.Deblob(b) + if err != nil { + return nil, err + } + data, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, err + } + // Sanity check + if !bytes.Equal(l.LeafValue, util.Digest(data)) { + return nil, fmt.Errorf("data digest does not match leaf value") + } + } + + // Setup timestamp + ts := backend.Timestamp{ + Data: string(data), + Digest: hex.EncodeToString(l.LeafValue), + Proofs: []backend.Proof{}, + } + + // Get the anchor record for this leaf + a, err := t.anchorForLeaf(treeID, merkleLeafHash, leaves) + if err != nil { + if err == errAnchorNotFound { + // This data has not been anchored yet + return &ts, nil + } + return nil, fmt.Errorf("anchor: %v", err) + } + + // Get trillian inclusion proof + p, err := t.tlog.InclusionProof(treeID, l.MerkleLeafHash, a.LogRoot) + if err != nil { + return nil, fmt.Errorf("InclusionProof %v %x: %v", + treeID, l.MerkleLeafHash, err) + } + + // Setup proof for data digest inclusion in the log merkle root + edt := ExtraDataTrillianRFC6962{ + LeafIndex: p.LeafIndex, + TreeSize: int64(a.LogRoot.TreeSize), + } + extraData, err := json.Marshal(edt) + if err != nil { + return nil, err + } + merklePath := make([]string, 0, len(p.Hashes)) + for _, v := range p.Hashes { + merklePath = append(merklePath, hex.EncodeToString(v)) + } + trillianProof := backend.Proof{ + Type: ProofTypeTrillianRFC6962, + Digest: ts.Digest, + MerkleRoot: hex.EncodeToString(a.LogRoot.RootHash), + MerklePath: merklePath, + ExtraData: string(extraData), + } + + // Setup proof for log merkle root inclusion in the dcrtime merkle + // root + if a.VerifyDigest.Digest != trillianProof.MerkleRoot { + return nil, fmt.Errorf("trillian merkle root not anchored") + } + var ( + numLeaves = a.VerifyDigest.ChainInformation.MerklePath.NumLeaves + hashes = a.VerifyDigest.ChainInformation.MerklePath.Hashes + flags = a.VerifyDigest.ChainInformation.MerklePath.Flags + ) + edd := ExtraDataDcrtime{ + NumLeaves: numLeaves, + Flags: base64.StdEncoding.EncodeToString(flags), + } + extraData, err = json.Marshal(edd) + if err != nil { + return nil, err + } + merklePath = make([]string, 0, len(hashes)) + for _, v := range hashes { + merklePath = append(merklePath, hex.EncodeToString(v[:])) + } + dcrtimeProof := backend.Proof{ + Type: ProofTypeDcrtime, + Digest: a.VerifyDigest.Digest, + MerkleRoot: a.VerifyDigest.ChainInformation.MerkleRoot, + MerklePath: merklePath, + ExtraData: string(extraData), + } + + // Update timestamp + ts.TxID = a.VerifyDigest.ChainInformation.Transaction + ts.MerkleRoot = a.VerifyDigest.ChainInformation.MerkleRoot + ts.Proofs = []backend.Proof{ + trillianProof, + dcrtimeProof, + } + + // Verify timestamp + err = VerifyTimestamp(ts) + if err != nil { + return nil, fmt.Errorf("VerifyTimestamp: %v", err) + } + + return &ts, nil +} diff --git a/politeiad/backendv2/tstorebe/tstore/tstore.go b/politeiad/backendv2/tstorebe/tstore/tstore.go index 7fd370dd1..8461597b9 100644 --- a/politeiad/backendv2/tstorebe/tstore/tstore.go +++ b/politeiad/backendv2/tstorebe/tstore/tstore.go @@ -5,11 +5,6 @@ package tstore import ( - "bytes" - "encoding/base64" - "encoding/hex" - "encoding/json" - "errors" "fmt" "net/url" "os" @@ -17,16 +12,11 @@ import ( "sync" "github.com/decred/dcrd/chaincfg/v3" - backend "github.com/decred/politeia/politeiad/backendv2" "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins" "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" "github.com/decred/politeia/politeiad/backendv2/tstorebe/store/localdb" "github.com/decred/politeia/politeiad/backendv2/tstorebe/store/mysql" - "github.com/decred/politeia/util" - "github.com/google/trillian" - "github.com/google/uuid" "github.com/robfig/cron" - "google.golang.org/grpc/codes" ) const ( @@ -46,21 +36,6 @@ const ( // Config option defaults defaultTrillianSigningKeyFilename = "trillian.key" - - // Blob entry data descriptors - dataDescriptorRecordMetadata = "pd-recordmd-v1" - dataDescriptorMetadataStream = "pd-mdstream-v1" - dataDescriptorFile = "pd-file-v1" - dataDescriptorRecordIndex = "pd-rindex-v1" - dataDescriptorAnchor = "pd-anchor-v1" - - // keyPrefixEncrypted is prefixed onto key-value store keys if the - // data is encrypted. We do this so that when a record is made - // public we can save the plain text record content blobs using the - // same keys, but without the prefix. Using a new key for the plain - // text blobs would not work since we cannot append a new leaf onto - // the tlog without getting a duplicate leaf error. - keyPrefixEncrypted = "e_" ) var ( @@ -102,1090 +77,10 @@ type Tstore struct { droppingAnchor bool } -// extraData is the data that is stored in the log leaf ExtraData field. It is -// saved as a JSON encoded byte slice. The JSON keys have been abbreviated to -// minimize the size of a trillian log leaf. -type extraData struct { - // Key contains the key-value store key. If this blob is part of an - // unvetted record the key will need to be prefixed with the - // keyPrefixEncrypted in order to retrieve the blob from the kv - // store. Use the extraData.storeKey() method to retrieve the key. - // Do NOT reference this key directly. - Key string `json:"k"` - - // Desc contains the blob entry data descriptor. - Desc string `json:"d"` - - // State indicates the record state of the blob that this leaf - // corresponds to. Unvetted blobs encrypted prior to being saved - // to the store. When retrieving unvetted blobs from the kv store - // the keyPrefixEncrypted prefix must be added to the Key field. - // State will not be populated for anchor records. - State backend.StateT `json:"s,omitempty"` -} - -// storeKey returns the kv store key for the blob. If the blob is part of an -// unvetted record it will be saved as an encrypted blob in the kv store and -// the key is prefixed with keyPrefixEncrypted. -func (e *extraData) storeKey() string { - if e.State == backend.StateUnvetted { - return keyPrefixEncrypted + e.Key - } - return e.Key -} - -// storeKeyNoPrefix returns the kv store key without any encryption prefix, -// even if the leaf corresponds to a unvetted blob. -func (e *extraData) storeKeyNoPrefix() string { - return e.Key -} - -func extraDataEncode(key, desc string, state backend.StateT) ([]byte, error) { - // The encryption prefix is stripped from the key if one exists. - ed := extraData{ - Key: storeKeyCleaned(key), - Desc: desc, - State: state, - } - b, err := json.Marshal(ed) - if err != nil { - return nil, err - } - return b, nil -} - -func extraDataDecode(b []byte) (*extraData, error) { - var ed extraData - err := json.Unmarshal(b, &ed) - if err != nil { - return nil, err - } - return &ed, nil -} - -// storeKeyNew returns a new key for the key-value store. If the data is -// encrypted the key is prefixed. -func storeKeyNew(encrypt bool) string { - k := uuid.New().String() - if encrypt { - k = keyPrefixEncrypted + k - } - return k -} - -// storeKeyCleaned strips the key-value store key of the encryption prefix if -// one is present. -func storeKeyCleaned(key string) string { - // A uuid string is 36 bytes. Return the last 36 bytes of the - // string. This will strip the prefix if it exists. - return key[len(key)-36:] -} - -// merkleLeafHashForBlobEntry returns the merkle leaf hash for a blob entry. -// The merkle leaf hash can be used to retrieve a leaf from its tlog tree. -func merkleLeafHashForBlobEntry(be store.BlobEntry) ([]byte, error) { - leafValue, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, err - } - return merkleLeafHash(leafValue), nil -} - -// TreeNew creates a new tlog tree and returns the tree ID. -func (t *Tstore) TreeNew() (int64, error) { - log.Tracef("TreeNew") - - tree, _, err := t.tlog.TreeNew() - if err != nil { - return 0, err - } - - return tree.TreeId, nil -} - -// TreeFreeze updates the status of a record then freezes the trillian tree to -// prevent any additional updates. -// -// A tree is considered to be frozen once the record index has been saved with -// its Frozen field set to true. The only thing that can be appended onto a -// frozen tree is one additional anchor record. Once a frozen tree has been -// anchored, the tstore fsck function will update the status of the tree to -// frozen in trillian, at which point trillian will prevent any changes to the -// tree. -func (t *Tstore) TreeFreeze(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { - log.Tracef("TreeFreeze: %v", treeID) - - // Save updated record - idx, err := t.recordSave(treeID, rm, metadata, files) - if err != nil { - return err - } - - // Mark the record as frozen - idx.Frozen = true - - // Save the record index - return t.recordIndexSave(treeID, *idx) -} - -// TreesAll returns the IDs of all trees in the tstore instance. -func (t *Tstore) TreesAll() ([]int64, error) { - trees, err := t.tlog.TreesAll() - if err != nil { - return nil, err - } - treeIDs := make([]int64, 0, len(trees)) - for _, v := range trees { - treeIDs = append(treeIDs, v.TreeId) - } - return treeIDs, nil -} - -// TreeExists returns whether a tree exists in the trillian log. A tree -// existing doesn't necessarily mean that a record exists. Its possible for a -// tree to have been created but experienced an unexpected error prior to the -// record being saved. -func (t *Tstore) TreeExists(treeID int64) bool { - _, err := t.tlog.Tree(treeID) - return err == nil -} - -// recordBlobsSave saves the provided blobs to the kv store, appends a leaf -// to the trillian tree for each blob, and returns the record index for the -// blobs. -func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, recordMD backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) (*recordIndex, error) { - log.Tracef("recordBlobsSave: %v", treeID) - - // Verify there are no duplicate metadata streams - md := make(map[string]map[uint32]struct{}, len(metadata)) - for _, v := range metadata { - if v.PluginID == "" || v.StreamID == 0 { - return nil, fmt.Errorf("invalid metadata stream: '%v' %v", - v.PluginID, v.StreamID) - } - pmd, ok := md[v.PluginID] - if !ok { - pmd = make(map[uint32]struct{}, len(metadata)) - } - _, ok = pmd[v.StreamID] - if ok { - return nil, fmt.Errorf("duplicate metadata stream: %v %v", - v.PluginID, v.StreamID) - } - pmd[v.StreamID] = struct{}{} - md[v.PluginID] = pmd - } - - // Verify there are no duplicate files - fn := make(map[string]struct{}, len(files)) - for _, v := range files { - if v.Name == "" { - return nil, fmt.Errorf("empty filename") - } - _, ok := fn[v.Name] - if ok { - return nil, fmt.Errorf("duplicate filename found: %v", v.Name) - } - fn[v.Name] = struct{}{} - } - - // Prepare the blob entries. The record index can also be created - // during this step. - var ( - // [pluginID][streamID]BlobEntry - beMetadata = make(map[string]map[uint32]store.BlobEntry, len(metadata)) - - // [filename]BlobEntry - beFiles = make(map[string]store.BlobEntry, len(files)) - - idx = recordIndex{ - State: recordMD.State, - Version: recordMD.Version, - Iteration: recordMD.Iteration, - Metadata: make(map[string]map[uint32][]byte, len(metadata)), - Files: make(map[string][]byte, len(files)), - } - - // digests is used to aggregate the digests from all record - // content. This is used later on to see if any of the content - // already exists in the tstore. - digests = make(map[string]struct{}, 256) - ) - - // Setup record metadata - beRecordMD, err := convertBlobEntryFromRecordMetadata(recordMD) - if err != nil { - return nil, err - } - m, err := merkleLeafHashForBlobEntry(*beRecordMD) - if err != nil { - return nil, err - } - idx.RecordMetadata = m - digests[beRecordMD.Digest] = struct{}{} - - // Setup metdata streams - for _, v := range metadata { - // Blob entry - be, err := convertBlobEntryFromMetadataStream(v) - if err != nil { - return nil, err - } - streams, ok := beMetadata[v.PluginID] - if !ok { - streams = make(map[uint32]store.BlobEntry, len(metadata)) - } - streams[v.StreamID] = *be - beMetadata[v.PluginID] = streams - - // Record index - m, err := merkleLeafHashForBlobEntry(*be) - if err != nil { - return nil, err - } - streamsIdx, ok := idx.Metadata[v.PluginID] - if !ok { - streamsIdx = make(map[uint32][]byte, len(metadata)) - } - streamsIdx[v.StreamID] = m - idx.Metadata[v.PluginID] = streamsIdx - - // Aggregate digest - digests[be.Digest] = struct{}{} - } - - // Setup files - for _, v := range files { - // Blob entry - be, err := convertBlobEntryFromFile(v) - if err != nil { - return nil, err - } - beFiles[v.Name] = *be - - // Record Index - m, err := merkleLeafHashForBlobEntry(*be) - if err != nil { - return nil, err - } - idx.Files[v.Name] = m - - // Aggregate digest - digests[be.Digest] = struct{}{} - } - - // Check if any of the content already exists. Different record - // versions that reference the same data is fine, but this data - // should not be saved to the store again. We can find duplicates - // by comparing the blob entry digest to the log leaf value. They - // will be the same if the record content is the same. - dups := make(map[string]struct{}, len(digests)) - for _, v := range leavesAll { - d := hex.EncodeToString(v.LeafValue) - _, ok := digests[d] - if ok { - // A piece of the new record content already exsits in the - // tstore. Save the digest as a duplcate. - dups[d] = struct{}{} - } - } - - // Prepare blobs for the kv store - var ( - blobs = make(map[string][]byte, len(digests)) - leaves = make([]*trillian.LogLeaf, 0, len(blobs)) - - // dupBlobs contains the blob entries for record content that - // already exists. We may need these blob entries later on if - // the duplicate content is encrypted and it needs to be saved - // plain text. - dupBlobs = make(map[string]store.BlobEntry, len(digests)) - ) - - // Only vetted data should be saved plain text - var encrypt bool - switch idx.State { - case backend.StateUnvetted: - encrypt = true - case backend.StateVetted: - // Save plain text - encrypt = false - default: - // Something is wrong - e := fmt.Sprintf("invalid record state %v %v", treeID, idx.State) - panic(e) - } - - // Prepare record metadata blobs and leaves - _, ok := dups[beRecordMD.Digest] - if !ok { - // Not a duplicate. Prepare kv store blob. - b, err := store.Blobify(*beRecordMD) - if err != nil { - return nil, err - } - k := storeKeyNew(encrypt) - blobs[k] = b - - // Prepare tlog leaf - extraData, err := extraDataEncode(k, - dataDescriptorRecordMetadata, idx.State) - if err != nil { - return nil, err - } - digest, err := hex.DecodeString(beRecordMD.Digest) - if err != nil { - return nil, err - } - leaves = append(leaves, newLogLeaf(digest, extraData)) - } else { - // This is a duplicate. Stash is for now. We may need to save - // it as plain text later. - dupBlobs[beRecordMD.Digest] = *beRecordMD - } - - // Prepare metadata stream blobs and leaves - for _, v := range beMetadata { - for _, be := range v { - _, ok := dups[be.Digest] - if !ok { - // Not a duplicate. Prepare kv store blob. - b, err := store.Blobify(be) - if err != nil { - return nil, err - } - k := storeKeyNew(encrypt) - blobs[k] = b - - // Prepare tlog leaf - extraData, err := extraDataEncode(k, - dataDescriptorMetadataStream, idx.State) - if err != nil { - return nil, err - } - digest, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, err - } - leaves = append(leaves, newLogLeaf(digest, extraData)) - - continue - } - - // This is a duplicate. Stash is for now. We may need to save - // it as plain text later. - dupBlobs[be.Digest] = be - } - } - - // Prepare file blobs and leaves - for _, be := range beFiles { - _, ok := dups[be.Digest] - if !ok { - // Not a duplicate. Prepare kv store blob. - b, err := store.Blobify(be) - if err != nil { - return nil, err - } - k := storeKeyNew(encrypt) - blobs[k] = b - - // Prepare tlog leaf - extraData, err := extraDataEncode(k, dataDescriptorFile, idx.State) - if err != nil { - return nil, err - } - digest, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, err - } - leaves = append(leaves, newLogLeaf(digest, extraData)) - - continue - } - - // This is a duplicate. Stash is for now. We may need to save - // it as plain text later. - dupBlobs[be.Digest] = be - } - - // Verify at least one new blob is being saved to the kv store - if len(blobs) == 0 { - return nil, backend.ErrNoRecordChanges - } - - log.Debugf("Saving %v record content blobs", len(blobs)) - - // Save blobs to the kv store - err = t.store.Put(blobs, encrypt) - if err != nil { - return nil, fmt.Errorf("store Put: %v", err) - } - - // Append leaves onto the trillian tree - queued, _, err := t.tlog.LeavesAppend(treeID, leaves) - if err != nil { - return nil, fmt.Errorf("LeavesAppend: %v", err) - } - failed := make([]string, 0, len(queued)) - for _, v := range queued { - c := codes.Code(v.QueuedLeaf.GetStatus().GetCode()) - if c != codes.OK { - failed = append(failed, fmt.Sprintf("%v", c)) - } - } - if len(failed) > 0 { - return nil, fmt.Errorf("append leaves failed: %v", failed) - } - - // When a record is made public the record content needs to be - // resaved to the key-value store as unencrypted. - var ( - isPublic = recordMD.Status == backend.StatusPublic - - // Iteration and version are reset back to 1 when a record is - // made public. - iterIsReset = recordMD.Iteration == 1 - ) - if !isPublic || !iterIsReset { - // Record is not being made public. Nothing else to do. - return &idx, nil - } - - // Resave all of the duplicate blobs as plain text. A duplicate - // blob means the record content existed prior to the status - // change. - blobs = make(map[string][]byte, len(dupBlobs)) - for _, v := range leavesAll { - d := hex.EncodeToString(v.LeafValue) - _, ok := dups[d] - if !ok { - // Not a duplicate - continue - } - - // This is a duplicate. If its unvetted it will need to be - // resaved as plain text. - ed, err := extraDataDecode(v.ExtraData) - if err != nil { - return nil, err - } - if ed.State == backend.StateVetted { - // Not unvetted. No need to resave it. - continue - } - - // Prepare plain text blob - be, ok := dupBlobs[d] - if !ok { - // Should not happen - return nil, fmt.Errorf("blob entry not found %v", d) - } - b, err := store.Blobify(be) - if err != nil { - return nil, err - } - blobs[ed.storeKeyNoPrefix()] = b - } - if len(blobs) == 0 { - // This should not happen - return nil, fmt.Errorf("no blobs found to resave as plain text") - } - - log.Debugf("Resaving %v encrypted blobs as plain text", len(blobs)) - - err = t.store.Put(blobs, false) - if err != nil { - return nil, fmt.Errorf("store Put: %v", err) - } - - return &idx, nil -} - -func (t *Tstore) recordSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) (*recordIndex, error) { - // Verify tree exists - if !t.TreeExists(treeID) { - return nil, backend.ErrRecordNotFound - } - - // Get tree leaves - leavesAll, err := t.tlog.LeavesAll(treeID) - if err != nil { - return nil, fmt.Errorf("LeavesAll %v: %v", treeID, err) - } - - // Get the existing record index - currIdx, err := t.recordIndexLatest(leavesAll) - if errors.Is(err, backend.ErrRecordNotFound) { - // No record versions exist yet. This is ok. - currIdx = &recordIndex{ - Metadata: make(map[string]map[uint32][]byte), - Files: make(map[string][]byte), - } - } else if err != nil { - return nil, fmt.Errorf("recordIndexLatest: %v", err) - } - - // Verify tree is not frozen - if currIdx.Frozen { - return nil, backend.ErrRecordLocked - } - - // Save the record - idx, err := t.recordBlobsSave(treeID, leavesAll, rm, metadata, files) - if err != nil { - if err == backend.ErrNoRecordChanges { - return nil, err - } - return nil, fmt.Errorf("recordBlobsSave: %v", err) - } - - return idx, nil -} - -// RecordSave saves the provided record to tstore. Once the record contents -// have been successfully saved to tstore, a recordIndex is created for this -// version of the record and saved to tstore as well. The record update is not -// considered to be valid until the record index has been successfully saved. -// If the record content makes it in but the record index does not, the record -// content blobs are orphaned and ignored. -func (t *Tstore) RecordSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { - log.Tracef("RecordSave: %v", treeID) - - // Save the record - idx, err := t.recordSave(treeID, rm, metadata, files) - if err != nil { - return err - } - - // Save the record index - err = t.recordIndexSave(treeID, *idx) - if err != nil { - return fmt.Errorf("recordIndexSave: %v", err) - } - - return nil -} - -// RecordDel walks the provided tree and deletes all blobs in the store that -// correspond to record files. This is done for all versions and all iterations -// of the record. Record metadata and metadata stream blobs are not deleted. -func (t *Tstore) RecordDel(treeID int64) error { - log.Tracef("RecordDel: %v", treeID) - - // Verify tree exists - if !t.TreeExists(treeID) { - return backend.ErrRecordNotFound - } - - // Get all tree leaves - leavesAll, err := t.tlog.LeavesAll(treeID) - if err != nil { - return err - } - - // Ensure tree is frozen. Deleting files from the store is only - // allowed on frozen trees. - currIdx, err := t.recordIndexLatest(leavesAll) - if err != nil { - return err - } - if !currIdx.Frozen { - return fmt.Errorf("tree is not frozen") - } - - // Retrieve all record indexes - indexes, err := t.recordIndexes(leavesAll) - if err != nil { - return err - } - - // Aggregate the keys for all file blobs of all versions. The - // record index points to the log leaf merkle leaf hash. The log - // leaf contains the kv store key. - merkles := make(map[string]struct{}, len(leavesAll)) - for _, v := range indexes { - for _, merkle := range v.Files { - merkles[hex.EncodeToString(merkle)] = struct{}{} - } - } - keys := make([]string, 0, len(merkles)) - for _, v := range leavesAll { - _, ok := merkles[hex.EncodeToString(v.MerkleLeafHash)] - if ok { - ed, err := extraDataDecode(v.ExtraData) - if err != nil { - return err - } - keys = append(keys, ed.storeKey()) - } - } - - // Delete file blobs from the store - err = t.store.Del(keys) - if err != nil { - return fmt.Errorf("store Del: %v", err) - } - - return nil -} - -// record returns the specified record. -// -// Version is used to request a specific version of a record. If no version is -// provided then the most recent version of the record will be returned. -// -// Filenames can be used to request specific files. If filenames is not empty -// then the specified files will be the only files returned. -// -// OmitAllFiles can be used to retrieve a record without any of the record -// files. This supersedes the filenames argument. -func (t *Tstore) record(treeID int64, version uint32, filenames []string, omitAllFiles bool) (*backend.Record, error) { - // Verify tree exists - if !t.TreeExists(treeID) { - return nil, backend.ErrRecordNotFound - } - - // Get tree leaves - leaves, err := t.tlog.LeavesAll(treeID) - if err != nil { - return nil, fmt.Errorf("LeavesAll %v: %v", treeID, err) - } - - // Use the record index to pull the record content from the store. - // The keys for the record content first need to be extracted from - // their log leaf. - idx, err := t.recordIndex(leaves, version) - if err != nil { - return nil, err - } - - // Compile merkle root hashes of record content - merkles := make(map[string]struct{}, 64) - merkles[hex.EncodeToString(idx.RecordMetadata)] = struct{}{} - for _, streams := range idx.Metadata { - for _, v := range streams { - merkles[hex.EncodeToString(v)] = struct{}{} - } - } - switch { - case omitAllFiles: - // Don't include any files - case len(filenames) > 0: - // Only included the specified files - filesToInclude := make(map[string]struct{}, len(filenames)) - for _, v := range filenames { - filesToInclude[v] = struct{}{} - } - for fn, v := range idx.Files { - if _, ok := filesToInclude[fn]; ok { - merkles[hex.EncodeToString(v)] = struct{}{} - } - } - default: - // Include all files - for _, v := range idx.Files { - merkles[hex.EncodeToString(v)] = struct{}{} - } - } - - // Walk the tree and extract the record content keys - keys := make([]string, 0, len(idx.Metadata)+len(idx.Files)+1) - for _, v := range leaves { - _, ok := merkles[hex.EncodeToString(v.MerkleLeafHash)] - if !ok { - // Not part of the record content - continue - } - - // Leaf is part of record content. Save the kv store key. - ed, err := extraDataDecode(v.ExtraData) - if err != nil { - return nil, err - } - - var key string - switch idx.State { - case backend.StateVetted: - // If the record is vetted the content may exist in the store - // as both an encrypted blob and a plain text blob. Always pull - // the plaintext blob. - key = ed.storeKeyNoPrefix() - default: - // Pull the encrypted blob - key = ed.storeKey() - } - keys = append(keys, key) - } - - // Get record content from store - blobs, err := t.store.Get(keys) - if err != nil { - return nil, fmt.Errorf("store Get: %v", err) - } - if len(keys) != len(blobs) { - // One or more blobs were not found. This is allowed since the - // blobs for a censored record will not exist, but the record - // metadata and metadata streams should still be returned. - log.Tracef("Blobs not found %v: want %v, got %v", - treeID, len(keys), len(blobs)) - } - - // Decode blobs - entries := make([]store.BlobEntry, 0, len(keys)) - for _, v := range blobs { - be, err := store.Deblob(v) - if err != nil { - return nil, err - } - entries = append(entries, *be) - } - - // Decode blob entries - var ( - recordMD *backend.RecordMetadata - metadata = make([]backend.MetadataStream, 0, len(idx.Metadata)) - files = make([]backend.File, 0, len(idx.Files)) - ) - for _, v := range entries { - // Decode the data hint - b, err := base64.StdEncoding.DecodeString(v.DataHint) - if err != nil { - return nil, fmt.Errorf("decode DataHint: %v", err) - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, fmt.Errorf("unmarshal DataHint: %v", err) - } - if dd.Type != store.DataTypeStructure { - return nil, fmt.Errorf("invalid data type; got %v, want %v", - dd.Type, store.DataTypeStructure) - } - - // Decode the data - b, err = base64.StdEncoding.DecodeString(v.Data) - if err != nil { - return nil, fmt.Errorf("decode Data: %v", err) - } - digest, err := hex.DecodeString(v.Digest) - if err != nil { - return nil, fmt.Errorf("decode Hash: %v", err) - } - if !bytes.Equal(util.Digest(b), digest) { - return nil, fmt.Errorf("data is not coherent; got %x, want %x", - util.Digest(b), digest) - } - switch dd.Descriptor { - case dataDescriptorRecordMetadata: - var rm backend.RecordMetadata - err = json.Unmarshal(b, &rm) - if err != nil { - return nil, fmt.Errorf("unmarshal RecordMetadata: %v", err) - } - recordMD = &rm - case dataDescriptorMetadataStream: - var ms backend.MetadataStream - err = json.Unmarshal(b, &ms) - if err != nil { - return nil, fmt.Errorf("unmarshal MetadataStream: %v", err) - } - metadata = append(metadata, ms) - case dataDescriptorFile: - var f backend.File - err = json.Unmarshal(b, &f) - if err != nil { - return nil, fmt.Errorf("unmarshal File: %v", err) - } - files = append(files, f) - default: - return nil, fmt.Errorf("invalid descriptor %v", dd.Descriptor) - } - } - - return &backend.Record{ - RecordMetadata: *recordMD, - Metadata: metadata, - Files: files, - }, nil -} - -// Record returns the specified version of the record. -func (t *Tstore) Record(treeID int64, version uint32) (*backend.Record, error) { - log.Tracef("Record: %v %v", treeID, version) - - return t.record(treeID, version, []string{}, false) -} - -// RecordLatest returns the latest version of a record. -func (t *Tstore) RecordLatest(treeID int64) (*backend.Record, error) { - log.Tracef("RecordLatest: %v", treeID) - - return t.record(treeID, 0, []string{}, false) -} - -// RecordPartial returns a partial record. This method gives the caller fine -// grained control over what version and what files are returned. The only -// required field is the token. All other fields are optional. -// -// Version is used to request a specific version of a record. If no version is -// provided then the most recent version of the record will be returned. -// -// Filenames can be used to request specific files. If filenames is not empty -// then the specified files will be the only files returned. -// -// OmitAllFiles can be used to retrieve a record without any of the record -// files. This supersedes the filenames argument. -func (t *Tstore) RecordPartial(treeID int64, version uint32, filenames []string, omitAllFiles bool) (*backend.Record, error) { - log.Tracef("RecordPartial: %v %v %v %v", - treeID, version, omitAllFiles, filenames) - - return t.record(treeID, version, filenames, omitAllFiles) -} - -// recordIsVetted returns whether the provided leaves contain any vetted record -// indexes. The presence of a vetted record index means the record is vetted. -// The state of a record index is saved to the leaf extra data, which is how we -// determine if a record index is vetted. -func recordIsVetted(leaves []*trillian.LogLeaf) bool { - for _, v := range leaves { - ed, err := extraDataDecode(v.ExtraData) - if err != nil { - panic(err) - } - if ed.Desc == dataDescriptorRecordIndex && - ed.State == backend.StateVetted { - // Vetted record index found - return true - } - } - return false -} - -// RecordState returns the state of a record. This call does not require -// retrieving any blobs from the kv store. The record state can be derived from -// only the tlog leaves. -func (t *Tstore) RecordState(treeID int64) (backend.StateT, error) { - log.Tracef("RecordState: %v", treeID) - - leaves, err := t.tlog.LeavesAll(treeID) - if err != nil { - return 0, err - } - - if recordIsVetted(leaves) { - return backend.StateVetted, nil - } - - return backend.StateUnvetted, nil -} - -// timestamp returns the timestamp given a tlog merkle leaf hash. -func (t *Tstore) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trillian.LogLeaf) (*backend.Timestamp, error) { - // Find the leaf - var l *trillian.LogLeaf - for _, v := range leaves { - if bytes.Equal(merkleLeafHash, v.MerkleLeafHash) { - l = v - break - } - } - if l == nil { - return nil, fmt.Errorf("leaf not found") - } - - // Get blob entry from the kv store - ed, err := extraDataDecode(l.ExtraData) - if err != nil { - return nil, err - } - blobs, err := t.store.Get([]string{ed.storeKey()}) - if err != nil { - return nil, fmt.Errorf("store get: %v", err) - } - - // Extract the data blob. Its possible for the data blob to not - // exist if it has been censored. This is ok. We'll still return - // the rest of the timestamp. - var data []byte - if len(blobs) == 1 { - b, ok := blobs[ed.storeKey()] - if !ok { - return nil, fmt.Errorf("blob not found %v", ed.storeKey()) - } - be, err := store.Deblob(b) - if err != nil { - return nil, err - } - data, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, err - } - // Sanity check - if !bytes.Equal(l.LeafValue, util.Digest(data)) { - return nil, fmt.Errorf("data digest does not match leaf value") - } - } - - // Setup timestamp - ts := backend.Timestamp{ - Data: string(data), - Digest: hex.EncodeToString(l.LeafValue), - Proofs: []backend.Proof{}, - } - - // Get the anchor record for this leaf - a, err := t.anchorForLeaf(treeID, merkleLeafHash, leaves) - if err != nil { - if err == errAnchorNotFound { - // This data has not been anchored yet - return &ts, nil - } - return nil, fmt.Errorf("anchor: %v", err) - } - - // Get trillian inclusion proof - p, err := t.tlog.InclusionProof(treeID, l.MerkleLeafHash, a.LogRoot) - if err != nil { - return nil, fmt.Errorf("InclusionProof %v %x: %v", - treeID, l.MerkleLeafHash, err) - } - - // Setup proof for data digest inclusion in the log merkle root - edt := ExtraDataTrillianRFC6962{ - LeafIndex: p.LeafIndex, - TreeSize: int64(a.LogRoot.TreeSize), - } - extraData, err := json.Marshal(edt) - if err != nil { - return nil, err - } - merklePath := make([]string, 0, len(p.Hashes)) - for _, v := range p.Hashes { - merklePath = append(merklePath, hex.EncodeToString(v)) - } - trillianProof := backend.Proof{ - Type: ProofTypeTrillianRFC6962, - Digest: ts.Digest, - MerkleRoot: hex.EncodeToString(a.LogRoot.RootHash), - MerklePath: merklePath, - ExtraData: string(extraData), - } - - // Setup proof for log merkle root inclusion in the dcrtime merkle - // root - if a.VerifyDigest.Digest != trillianProof.MerkleRoot { - return nil, fmt.Errorf("trillian merkle root not anchored") - } - var ( - numLeaves = a.VerifyDigest.ChainInformation.MerklePath.NumLeaves - hashes = a.VerifyDigest.ChainInformation.MerklePath.Hashes - flags = a.VerifyDigest.ChainInformation.MerklePath.Flags - ) - edd := ExtraDataDcrtime{ - NumLeaves: numLeaves, - Flags: base64.StdEncoding.EncodeToString(flags), - } - extraData, err = json.Marshal(edd) - if err != nil { - return nil, err - } - merklePath = make([]string, 0, len(hashes)) - for _, v := range hashes { - merklePath = append(merklePath, hex.EncodeToString(v[:])) - } - dcrtimeProof := backend.Proof{ - Type: ProofTypeDcrtime, - Digest: a.VerifyDigest.Digest, - MerkleRoot: a.VerifyDigest.ChainInformation.MerkleRoot, - MerklePath: merklePath, - ExtraData: string(extraData), - } - - // Update timestamp - ts.TxID = a.VerifyDigest.ChainInformation.Transaction - ts.MerkleRoot = a.VerifyDigest.ChainInformation.MerkleRoot - ts.Proofs = []backend.Proof{ - trillianProof, - dcrtimeProof, - } - - // Verify timestamp - err = VerifyTimestamp(ts) - if err != nil { - return nil, fmt.Errorf("VerifyTimestamp: %v", err) - } - - return &ts, nil -} - -// RecordTimestamps returns the timestamps for the contents of a record. -// Timestamps for the record metadata, metadata streams, and files are all -// returned. -func (t *Tstore) RecordTimestamps(treeID int64, version uint32, token []byte) (*backend.RecordTimestamps, error) { - log.Tracef("RecordTimestamps: %v %v", treeID, version) - - // Verify tree exists - if !t.TreeExists(treeID) { - return nil, backend.ErrRecordNotFound - } - - // Get record index - leaves, err := t.tlog.LeavesAll(treeID) - if err != nil { - return nil, fmt.Errorf("LeavesAll %v: %v", treeID, err) - } - idx, err := t.recordIndex(leaves, version) - if err != nil { - return nil, err - } - - // Get record metadata timestamp - rm, err := t.timestamp(treeID, idx.RecordMetadata, leaves) - if err != nil { - return nil, fmt.Errorf("record metadata timestamp: %v", err) - } - - // Get metadata timestamps - metadata := make(map[string]map[uint32]backend.Timestamp, len(idx.Metadata)) - for pluginID, streams := range idx.Metadata { - for streamID, merkle := range streams { - ts, err := t.timestamp(treeID, merkle, leaves) - if err != nil { - return nil, fmt.Errorf("metadata %v %v timestamp: %v", - pluginID, streamID, err) - } - sts, ok := metadata[pluginID] - if !ok { - sts = make(map[uint32]backend.Timestamp, 64) - } - sts[streamID] = *ts - metadata[pluginID] = sts - } - } - - // Get file timestamps - files := make(map[string]backend.Timestamp, len(idx.Files)) - for k, v := range idx.Files { - ts, err := t.timestamp(treeID, v, leaves) - if err != nil { - return nil, fmt.Errorf("file %v timestamp: %v", k, err) - } - files[k] = *ts - } - - return &backend.RecordTimestamps{ - RecordMetadata: *rm, - Metadata: metadata, - Files: files, - }, nil -} - // Fsck performs a filesystem check on the tstore. func (t *Tstore) Fsck() { // Set tree status to frozen for any trees that are frozen and have // been anchored one last time. - // Verify all file blobs have been deleted for censored records. } diff --git a/politeiad/backendv2/tstorebe/tstorebe.go b/politeiad/backendv2/tstorebe/tstorebe.go index f2c04df58..ecd9538d7 100644 --- a/politeiad/backendv2/tstorebe/tstorebe.go +++ b/politeiad/backendv2/tstorebe/tstorebe.go @@ -764,9 +764,9 @@ func (t *tstoreBackend) setStatusPublic(token []byte, rm backend.RecordMetadata, func (t *tstoreBackend) setStatusArchived(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { // Freeze the tree treeID := treeIDFromToken(token) - err := t.tstore.TreeFreeze(treeID, rm, metadata, files) + err := t.tstore.RecordFreeze(treeID, rm, metadata, files) if err != nil { - return fmt.Errorf("TreeFreeze %v: %v", treeID, err) + return fmt.Errorf("RecordFreeze %v: %v", treeID, err) } log.Debugf("Record frozen %x", token) @@ -782,9 +782,9 @@ func (t *tstoreBackend) setStatusArchived(token []byte, rm backend.RecordMetadat func (t *tstoreBackend) setStatusCensored(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { // Freeze the tree treeID := treeIDFromToken(token) - err := t.tstore.TreeFreeze(treeID, rm, metadata, files) + err := t.tstore.RecordFreeze(treeID, rm, metadata, files) if err != nil { - return fmt.Errorf("TreeFreeze %v: %v", treeID, err) + return fmt.Errorf("RecordFreeze %v: %v", treeID, err) } log.Debugf("Record frozen %x", token) @@ -945,9 +945,9 @@ func (t *tstoreBackend) RecordSetStatus(token []byte, status backend.StatusT, md // error. // // Pulling the leaves from the tree to see if a record has been saved to the -// tree adds a large amount of overhead to this call that should be very light -// weight. Its for this reason that we rely on the tree exists call despite the -// edge case. +// tree adds a large amount of overhead to this call, which should be a very +// light weight. Its for this reason that we rely on the tree exists call +// despite the edge case. // // This function satisfies the Backend interface. func (t *tstoreBackend) RecordExists(token []byte) bool { From 5ebf7d9102acb0606bcfdbdbc0b45e3bcb1a98f2 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 22 Mar 2021 18:39:06 -0500 Subject: [PATCH 427/449] tstorebe/usermd: Cleanup and add docs. --- politeiad/backendv2/tstorebe/inventory.go | 47 ++- .../tstorebe/plugins/comments/comments.go | 2 +- .../tstorebe/plugins/dcrdata/dcrdata.go | 5 +- .../backendv2/tstorebe/plugins/pi/hooks.go | 24 +- politeiad/backendv2/tstorebe/plugins/pi/pi.go | 4 + .../tstorebe/plugins/ticketvote/ticketvote.go | 16 +- .../tstorebe/plugins/usermd/cache.go | 47 +-- .../backendv2/tstorebe/plugins/usermd/cmds.go | 7 +- .../tstorebe/plugins/usermd/hooks.go | 282 +++++++++--------- .../tstorebe/plugins/usermd/usermd.go | 34 ++- politeiad/backendv2/tstorebe/tstorebe.go | 36 +-- politeiad/plugins/usermd/usermd.go | 3 +- 12 files changed, 283 insertions(+), 224 deletions(-) diff --git a/politeiad/backendv2/tstorebe/inventory.go b/politeiad/backendv2/tstorebe/inventory.go index 647076590..a4c287248 100644 --- a/politeiad/backendv2/tstorebe/inventory.go +++ b/politeiad/backendv2/tstorebe/inventory.go @@ -17,23 +17,28 @@ import ( ) const ( + // Filenames of the inventory caches. filenameInvUnvetted = "inv-unvetted.json" filenameInvVetted = "inv-vetted.json" ) +// entry represents a record entry in the inventory. type entry struct { Token string `json:"token"` Status backend.StatusT `json:"status"` } +// inventory represents the record inventory. type inventory struct { Entries []entry `json:"entries"` } +// invPathUnvetted returns the file path for the unvetted inventory. func (t *tstoreBackend) invPathUnvetted() string { return filepath.Join(t.dataDir, filenameInvUnvetted) } +// invPathVetted returns the file path for the vetted inventory. func (t *tstoreBackend) invPathVetted() string { return filepath.Join(t.dataDir, filenameInvVetted) } @@ -86,10 +91,10 @@ func (t *tstoreBackend) invSaveLocked(filePath string, inv inventory) error { return ioutil.WriteFile(filePath, b, 0664) } +// invAdd adds a new record to the inventory. +// +// This function must be called WITHOUT the read/write lock held. func (t *tstoreBackend) invAdd(state backend.StateT, token []byte, s backend.StatusT) error { - t.Lock() - defer t.Unlock() - // Get inventory file path var fp string switch state { @@ -101,6 +106,9 @@ func (t *tstoreBackend) invAdd(state backend.StateT, token []byte, s backend.Sta return fmt.Errorf("invalid state %v", state) } + t.Lock() + defer t.Unlock() + // Get inventory inv, err := t.invGetLocked(fp) if err != nil { @@ -126,10 +134,11 @@ func (t *tstoreBackend) invAdd(state backend.StateT, token []byte, s backend.Sta return nil } +// invUpdate updates the status of a record in the inventory. The record state +// must remain the same. +// +// This function must be called WITHOUT the read/write lock held. func (t *tstoreBackend) invUpdate(state backend.StateT, token []byte, s backend.StatusT) error { - t.Lock() - defer t.Unlock() - // Get inventory file path var fp string switch state { @@ -141,6 +150,9 @@ func (t *tstoreBackend) invUpdate(state backend.StateT, token []byte, s backend. return fmt.Errorf("invalid state %v", state) } + t.Lock() + defer t.Unlock() + // Get inventory inv, err := t.invGetLocked(fp) if err != nil { @@ -172,14 +184,20 @@ func (t *tstoreBackend) invUpdate(state backend.StateT, token []byte, s backend. return nil } -// invMoveToVetted moves a token from the unvetted inventory to the vetted -// inventory. +// invMoveToVetted deletes a record from the unvetted inventory then adds it +// to the vetted inventory. +// +// This function must be called WITHOUT the read/write lock held. func (t *tstoreBackend) invMoveToVetted(token []byte, s backend.StatusT) error { + var ( + upath = t.invPathUnvetted() + vpath = t.invPathVetted() + ) + t.Lock() defer t.Unlock() // Get unvetted inventory - upath := t.invPathUnvetted() u, err := t.invGetLocked(upath) if err != nil { return fmt.Errorf("unvetted invGetLocked: %v", err) @@ -198,7 +216,6 @@ func (t *tstoreBackend) invMoveToVetted(token []byte, s backend.StatusT) error { } // Get vetted inventory - vpath := t.invPathVetted() v, err := t.invGetLocked(vpath) if err != nil { return fmt.Errorf("vetted invGetLocked: %v", err) @@ -262,6 +279,7 @@ type invByStatus struct { Vetted map[backend.StatusT][]string } +// invByStatusAll returns a page of tokens for all record states and statuses. func (t *tstoreBackend) invByStatusAll(pageSize uint32) (*invByStatus, error) { // Get unvetted inventory u, err := t.invGet(t.invPathUnvetted()) @@ -317,6 +335,15 @@ func (t *tstoreBackend) invByStatusAll(pageSize uint32) (*invByStatus, error) { }, nil } +// invByStatus returns the tokens of records in the inventory categorized by +// record state and record status. The tokens are ordered by the timestamp of +// their most recent status change, sorted from newest to oldest. +// +// The state, status, and page arguments can be provided to request a specific +// page of record tokens. +// +// If no status is provided then the most recent page of tokens for all +// statuses will be returned. All other arguments are ignored. func (t *tstoreBackend) invByStatus(state backend.StateT, s backend.StatusT, pageSize, page uint32) (*invByStatus, error) { // If no status is provided a page of tokens for each status should // be returned. diff --git a/politeiad/backendv2/tstorebe/plugins/comments/comments.go b/politeiad/backendv2/tstorebe/plugins/comments/comments.go index 640854c88..2d176e165 100644 --- a/politeiad/backendv2/tstorebe/plugins/comments/comments.go +++ b/politeiad/backendv2/tstorebe/plugins/comments/comments.go @@ -22,7 +22,7 @@ var ( ) // commentsPlugin is the tstore backend implementation of the comments plugin. -// It provides an API for adding comment functionality onto a record. +// The comments plugin extends a record with comment functionality. // // commentsPlugin satisfies the plugins PluginClient interface. type commentsPlugin struct { diff --git a/politeiad/backendv2/tstorebe/plugins/dcrdata/dcrdata.go b/politeiad/backendv2/tstorebe/plugins/dcrdata/dcrdata.go index a24315d02..3a2f63986 100644 --- a/politeiad/backendv2/tstorebe/plugins/dcrdata/dcrdata.go +++ b/politeiad/backendv2/tstorebe/plugins/dcrdata/dcrdata.go @@ -37,8 +37,9 @@ var ( _ plugins.PluginClient = (*dcrdataPlugin)(nil) ) -// dcrdataPlugin is the tstore backend implementation of the dcrdata plugin. It -// provides and API for interacting with the dcrdata http and websocket APIs. +// dcrdataPlugin is the tstore backend implementation of the dcrdata plugin. +// The dcrdata plugin provides and API for interacting with the dcrdata http +// and websocket APIs. // // dcrdataPlugin satisfies the plugins PluginClient interface. type dcrdataPlugin struct { diff --git a/politeiad/backendv2/tstorebe/plugins/pi/hooks.go b/politeiad/backendv2/tstorebe/plugins/pi/hooks.go index f719495cd..9675515ab 100644 --- a/politeiad/backendv2/tstorebe/plugins/pi/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/pi/hooks.go @@ -34,8 +34,8 @@ var ( } ) -// hookNewRecordPre adds pi specific validation to the RecordNew tstore backend -// method. +// hookNewRecordPre adds plugin specific validation onto the tstore backend +// RecordNew method. func (p *piPlugin) hookNewRecordPre(payload string) error { var nr plugins.HookNewRecordPre err := json.Unmarshal([]byte(payload), &nr) @@ -46,8 +46,8 @@ func (p *piPlugin) hookNewRecordPre(payload string) error { return p.proposalFilesVerify(nr.Files) } -// hookEditRecordPre adds pi specific validation to the RecordEdit tstore -// backend method. +// hookEditRecordPre adds plugin specific validation onto the tstore backend +// RecordEdit method. func (p *piPlugin) hookEditRecordPre(payload string) error { var er plugins.HookEditRecord err := json.Unmarshal([]byte(payload), &er) @@ -87,26 +87,26 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { return nil } -// hookCommentNew extends the comments plugin New command with pi specific -// validation. +// hookCommentNew adds pi specific validation onto the comments plugin New +// command. func (p *piPlugin) hookCommentNew(token []byte) error { return p.commentWritesAllowed(token) } -// hookCommentNew extends the comments plugin Del command with pi specific -// validation. +// hookCommentDel adds pi specific validation onto the comments plugin Del +// command. func (p *piPlugin) hookCommentDel(token []byte) error { return p.commentWritesAllowed(token) } -// hookCommentNew extends the comments plugin Vote command with pi specific -// validation. +// hookCommentVote adds pi specific validation onto the comments plugin Vote +// command. func (p *piPlugin) hookCommentVote(token []byte) error { return p.commentWritesAllowed(token) } -// hookPluginPre extends write commands from other plugins with pi specific -// validation. +// hookPluginPre extends plugin write commands from other plugins with pi +// specific validation. func (p *piPlugin) hookPluginPre(treeID int64, token []byte, payload string) error { // Decode payload var hpp plugins.HookPluginPre diff --git a/politeiad/backendv2/tstorebe/plugins/pi/pi.go b/politeiad/backendv2/tstorebe/plugins/pi/pi.go index a07d782d0..99822f929 100644 --- a/politeiad/backendv2/tstorebe/plugins/pi/pi.go +++ b/politeiad/backendv2/tstorebe/plugins/pi/pi.go @@ -22,6 +22,10 @@ var ( _ plugins.PluginClient = (*piPlugin)(nil) ) +// piPlugin is the tstore backend implementation of the pi plugin. The pi +// plugin extends a record with functionality specific to the decred proposal +// system. +// // piPlugin satisfies the plugins PluginClient interface. type piPlugin struct { backend backend.Backend diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go index 6428867a9..0924d0bdb 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go @@ -24,7 +24,11 @@ var ( _ plugins.PluginClient = (*ticketVotePlugin)(nil) ) -// ticketVotePlugin satisfies the plugins.PluginClient interface. +// ticketVotePlugin is the tstore backend implementation of the ticketvote +// plugin. The ticketvote plugin extends a record with dcr ticket voting +// functionality. +// +// ticketVotePlugin satisfies the plugins PluginClient interface. type ticketVotePlugin struct { backend backend.Backend tstore plugins.TstoreClient @@ -59,7 +63,7 @@ type ticketVotePlugin struct { // Setup performs any plugin setup that is required. // -// This function satisfies the plugins.PluginClient interface. +// This function satisfies the plugins PluginClient interface. func (p *ticketVotePlugin) Setup() error { log.Tracef("ticketvote Setup") @@ -147,7 +151,7 @@ func (p *ticketVotePlugin) Setup() error { // Cmd executes a plugin command. // -// This function satisfies the plugins.PluginClient interface. +// This function satisfies the plugins PluginClient interface. func (p *ticketVotePlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { log.Tracef("ticketvote Cmd: %v %x %v %v", treeID, token, cmd, payload) @@ -183,7 +187,7 @@ func (p *ticketVotePlugin) Cmd(treeID int64, token []byte, cmd, payload string) // Hook executes a plugin hook. // -// This function satisfies the plugins.PluginClient interface. +// This function satisfies the plugins PluginClient interface. func (p *ticketVotePlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { log.Tracef("ticketvote Hook: %v %x %v", plugins.Hooks[h], token, treeID) @@ -203,7 +207,7 @@ func (p *ticketVotePlugin) Hook(treeID int64, token []byte, h plugins.HookT, pay // Fsck performs a plugin filesystem check. // -// This function satisfies the plugins.PluginClient interface. +// This function satisfies the plugins PluginClient interface. func (p *ticketVotePlugin) Fsck(treeIDs []int64) error { log.Tracef("ticketvote Fsck") @@ -254,7 +258,7 @@ func (p *ticketVotePlugin) Fsck(treeIDs []int64) error { // Settings returns the plugin's settings. // -// This function satisfies the plugins.PluginClient interface. +// This function satisfies the plugins PluginClient interface. func (p *ticketVotePlugin) Settings() []backend.PluginSetting { log.Tracef("ticketvote Settings") diff --git a/politeiad/backendv2/tstorebe/plugins/usermd/cache.go b/politeiad/backendv2/tstorebe/plugins/usermd/cache.go index bf78bde82..1bfddd54a 100644 --- a/politeiad/backendv2/tstorebe/plugins/usermd/cache.go +++ b/politeiad/backendv2/tstorebe/plugins/usermd/cache.go @@ -25,25 +25,24 @@ const ( // userCache contains cached user metadata. The userCache JSON is saved to disk // in the user plugin data dir. The user ID is included in the filename. // -// All record tokens are sorted by the timestamp of their most recent status -// change from newest to oldest. +// The Unvetted and Vetted fields contain the records that have been submitted +// by the user. All record tokens are sorted by the timestamp of their most +// recent status change from newest to oldest. type userCache struct { Unvetted []string `json:"unvetted"` Vetted []string `json:"vetted"` } -// userCachePath returns the filepath to the cached userCache struct for the -// specified user. -func (p *userPlugin) userCachePath(userID string) string { +// userCachePath returns the filepath to the userCache for the specified user. +func (p *usermdPlugin) userCachePath(userID string) string { fn := strings.Replace(fnUserCache, "{userid}", userID, 1) return filepath.Join(p.dataDir, fn) } -// userCacheWithLock returns the cached userCache struct for the specified -// user. +// userCacheLocked returns the userCache for the specified user. // // This function must be called WITH the lock held. -func (p *userPlugin) userCacheWithLock(userID string) (*userCache, error) { +func (p *usermdPlugin) userCacheLocked(userID string) (*userCache, error) { fp := p.userCachePath(userID) b, err := ioutil.ReadFile(fp) if err != nil { @@ -66,20 +65,20 @@ func (p *userPlugin) userCacheWithLock(userID string) (*userCache, error) { return &uc, nil } -// userCache returns the cached userCache struct for the specified user. +// userCacheLocked returns the userCache for the specified user. // // This function must be called WITHOUT the lock held. -func (p *userPlugin) userCache(userID string) (*userCache, error) { +func (p *usermdPlugin) userCache(userID string) (*userCache, error) { p.Lock() defer p.Unlock() - return p.userCacheWithLock(userID) + return p.userCacheLocked(userID) } -// userCacheSaveWithLock saves the provided userCache to the pi plugin data dir. +// userCacheSaveLocked saves the provided userCache to the plugin data dir. // // This function must be called WITH the lock held. -func (p *userPlugin) userCacheSaveWithLock(userID string, uc userCache) error { +func (p *usermdPlugin) userCacheSaveLocked(userID string, uc userCache) error { b, err := json.Marshal(uc) if err != nil { return err @@ -92,12 +91,12 @@ func (p *userPlugin) userCacheSaveWithLock(userID string, uc userCache) error { // userCacheAddToken adds a token to a user cache. // // This function must be called WITHOUT the lock held. -func (p *userPlugin) userCacheAddToken(userID string, state backend.StateT, token string) error { +func (p *usermdPlugin) userCacheAddToken(userID string, state backend.StateT, token string) error { p.Lock() defer p.Unlock() // Get current user data - uc, err := p.userCacheWithLock(userID) + uc, err := p.userCacheLocked(userID) if err != nil { return err } @@ -113,7 +112,7 @@ func (p *userPlugin) userCacheAddToken(userID string, state backend.StateT, toke } // Save changes - err = p.userCacheSaveWithLock(userID, *uc) + err = p.userCacheSaveLocked(userID, *uc) if err != nil { return err } @@ -126,12 +125,12 @@ func (p *userPlugin) userCacheAddToken(userID string, state backend.StateT, toke // userCacheDelToken deletes a token from a user cache. // // This function must be called WITHOUT the lock held. -func (p *userPlugin) userCacheDelToken(userID string, state backend.StateT, token string) error { +func (p *usermdPlugin) userCacheDelToken(userID string, state backend.StateT, token string) error { p.Lock() defer p.Unlock() // Get current user data - uc, err := p.userCacheWithLock(userID) + uc, err := p.userCacheLocked(userID) if err != nil { return err } @@ -156,7 +155,7 @@ func (p *userPlugin) userCacheDelToken(userID string, state backend.StateT, toke } // Save changes - err = p.userCacheSaveWithLock(userID, *uc) + err = p.userCacheSaveLocked(userID, *uc) if err != nil { return err } @@ -166,12 +165,14 @@ func (p *userPlugin) userCacheDelToken(userID string, state backend.StateT, toke return nil } -func (p *userPlugin) userCacheMoveTokenToVetted(userID string, token string) error { +// userCacheMoveTokenToVetted moves a record token from the unvetted to vetted +// list in the userCache. +func (p *usermdPlugin) userCacheMoveTokenToVetted(userID string, token string) error { p.Lock() defer p.Unlock() // Get current user data - uc, err := p.userCacheWithLock(userID) + uc, err := p.userCacheLocked(userID) if err != nil { return err } @@ -186,7 +187,7 @@ func (p *userPlugin) userCacheMoveTokenToVetted(userID string, token string) err uc.Vetted = append(uc.Vetted, token) // Save changes - err = p.userCacheSaveWithLock(userID, *uc) + err = p.userCacheSaveLocked(userID, *uc) if err != nil { return err } @@ -196,6 +197,8 @@ func (p *userPlugin) userCacheMoveTokenToVetted(userID string, token string) err return nil } +// delToken deletes the tokenToDel from the tokens list. An error is returned +// if the token is not found. func delToken(tokens []string, tokenToDel string) ([]string, error) { // Find token index var i int diff --git a/politeiad/backendv2/tstorebe/plugins/usermd/cmds.go b/politeiad/backendv2/tstorebe/plugins/usermd/cmds.go index 9d11ecf9c..11a2555f4 100644 --- a/politeiad/backendv2/tstorebe/plugins/usermd/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/usermd/cmds.go @@ -10,7 +10,8 @@ import ( "github.com/decred/politeia/politeiad/plugins/usermd" ) -func (p *userPlugin) cmdAuthor(treeID int64) (string, error) { +// cmdAuthor returns the user ID of a record's author. +func (p *usermdPlugin) cmdAuthor(treeID int64) (string, error) { // Get user metadata r, err := p.tstore.RecordPartial(treeID, 0, nil, true) if err != nil { @@ -33,7 +34,9 @@ func (p *userPlugin) cmdAuthor(treeID int64) (string, error) { return string(reply), nil } -func (p *userPlugin) cmdUserRecords(payload string) (string, error) { +// cmdUserRecords retrieves the tokens of all records that were submitted by +// the provided user ID. The returned tokens are sorted from newest to oldest. +func (p *usermdPlugin) cmdUserRecords(payload string) (string, error) { // Decode payload var ur usermd.UserRecords err := json.Unmarshal([]byte(payload), &ur) diff --git a/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go b/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go index 4fc8c2240..947cf349f 100644 --- a/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go @@ -20,22 +20,141 @@ import ( "github.com/google/uuid" ) -func convertSignatureError(err error) backend.PluginError { - var e util.SignatureError - var s usermd.ErrorCodeT - if errors.As(err, &e) { - switch e.ErrorCode { - case util.ErrorStatusPublicKeyInvalid: - s = usermd.ErrorCodePublicKeyInvalid - case util.ErrorStatusSignatureInvalid: - s = usermd.ErrorCodeSignatureInvalid +// hookNewRecordPre adds plugin specific validation onto the tstore backend +// RecordNew method. +func (p *usermdPlugin) hookNewRecordPre(payload string) error { + var nr plugins.HookNewRecordPre + err := json.Unmarshal([]byte(payload), &nr) + if err != nil { + return err + } + + return userMetadataVerify(nr.Metadata, nr.Files) +} + +// hookNewRecordPre caches plugin data from the tstore backend RecordNew +// method. +func (p *usermdPlugin) hookNewRecordPost(payload string) error { + var nr plugins.HookNewRecordPost + err := json.Unmarshal([]byte(payload), &nr) + if err != nil { + return err + } + + // Decode user metadata + um, err := userMetadataDecode(nr.Metadata) + if err != nil { + return err + } + + // Add token to the user cache + err = p.userCacheAddToken(um.UserID, nr.RecordMetadata.State, + nr.RecordMetadata.Token) + if err != nil { + return err + } + + return nil +} + +// hookEditRecordPre adds plugin specific validation onto the tstore backend +// RecordEdit method. +func (p *usermdPlugin) hookEditRecordPre(payload string) error { + var er plugins.HookEditRecord + err := json.Unmarshal([]byte(payload), &er) + if err != nil { + return err + } + + // Verify user metadata + err = userMetadataVerify(er.Metadata, er.Files) + if err != nil { + return err + } + + // Verify user ID has not changed + um, err := userMetadataDecode(er.Metadata) + if err != nil { + return err + } + umCurr, err := userMetadataDecode(er.Record.Metadata) + if err != nil { + return err + } + if um.UserID != umCurr.UserID { + e := fmt.Sprintf("user id cannot change: got %v, want %v", + um.UserID, umCurr.UserID) + return backend.PluginError{ + PluginID: usermd.PluginID, + ErrorCode: uint32(usermd.ErrorCodeUserIDInvalid), + ErrorContext: e, } } - return backend.PluginError{ - PluginID: usermd.PluginID, - ErrorCode: uint32(s), - ErrorContext: e.ErrorContext, + + return nil +} + +// hookEditRecordPre adds plugin specific validation onto the tstore backend +// RecordEdit method. +func (p *usermdPlugin) hookEditMetadataPre(payload string) error { + var em plugins.HookEditMetadata + err := json.Unmarshal([]byte(payload), &em) + if err != nil { + return err } + + // User metadata should not change on metadata updates + return userMetadataPreventUpdates(em.Record.Metadata, em.Metadata) +} + +// hookSetStatusRecordPre adds plugin specific validation onto the tstore +// backend RecordSetStatus method. +func (p *usermdPlugin) hookSetRecordStatusPre(payload string) error { + var srs plugins.HookSetRecordStatus + err := json.Unmarshal([]byte(payload), &srs) + if err != nil { + return err + } + + // User metadata should not change on status changes + err = userMetadataPreventUpdates(srs.Record.Metadata, srs.Metadata) + if err != nil { + return err + } + + // Verify status change metadata + err = statusChangeMetadataVerify(srs.RecordMetadata, srs.Metadata) + if err != nil { + return err + } + + return nil +} + +// hookNewRecordPre caches plugin data from the tstore backend RecordSetStatus +// method. +func (p *usermdPlugin) hookSetRecordStatusPost(treeID int64, payload string) error { + var srs plugins.HookSetRecordStatus + err := json.Unmarshal([]byte(payload), &srs) + if err != nil { + return err + } + rm := srs.RecordMetadata + + // When a record is made public the token must be moved from the + // unvetted list to the vetted list in the user cache. + if rm.Status == backend.StatusPublic { + um, err := userMetadataDecode(srs.Metadata) + if err != nil { + return err + } + err = p.userCacheMoveTokenToVetted(um.UserID, rm.Token) + if err != nil { + return err + } + } + + return nil } // userMetadataDecode decodes and returns the UserMetadata from the provided @@ -146,85 +265,8 @@ func userMetadataPreventUpdates(current, update []backend.MetadataStream) error return nil } -func (p *userPlugin) hookNewRecordPre(payload string) error { - var nr plugins.HookNewRecordPre - err := json.Unmarshal([]byte(payload), &nr) - if err != nil { - return err - } - - return userMetadataVerify(nr.Metadata, nr.Files) -} - -func (p *userPlugin) hookNewRecordPost(payload string) error { - var nr plugins.HookNewRecordPost - err := json.Unmarshal([]byte(payload), &nr) - if err != nil { - return err - } - - // Decode user metadata - um, err := userMetadataDecode(nr.Metadata) - if err != nil { - return err - } - - // Add token to the user cache - err = p.userCacheAddToken(um.UserID, nr.RecordMetadata.State, - nr.RecordMetadata.Token) - if err != nil { - return err - } - - return nil -} - -func (p *userPlugin) hookEditRecordPre(payload string) error { - var er plugins.HookEditRecord - err := json.Unmarshal([]byte(payload), &er) - if err != nil { - return err - } - - // Verify user metadata - err = userMetadataVerify(er.Metadata, er.Files) - if err != nil { - return err - } - - // Verify user ID has not changed - um, err := userMetadataDecode(er.Metadata) - if err != nil { - return err - } - umCurr, err := userMetadataDecode(er.Record.Metadata) - if err != nil { - return err - } - if um.UserID != umCurr.UserID { - e := fmt.Sprintf("user id cannot change: got %v, want %v", - um.UserID, umCurr.UserID) - return backend.PluginError{ - PluginID: usermd.PluginID, - ErrorCode: uint32(usermd.ErrorCodeUserIDInvalid), - ErrorContext: e, - } - } - - return nil -} - -func (p *userPlugin) hookEditMetadataPre(payload string) error { - var em plugins.HookEditMetadata - err := json.Unmarshal([]byte(payload), &em) - if err != nil { - return err - } - - // User metadata should not change on metadata updates - return userMetadataPreventUpdates(em.Record.Metadata, em.Metadata) -} - +// statusChangesDecode decodes and returns the StatusChangeMetadata from the +// metadata streams if one is present. func statusChangesDecode(metadata []backend.MetadataStream) ([]usermd.StatusChangeMetadata, error) { statuses := make([]usermd.StatusChangeMetadata, 0, 16) for _, v := range metadata { @@ -320,48 +362,20 @@ func statusChangeMetadataVerify(rm backend.RecordMetadata, metadata []backend.Me return nil } -func (p *userPlugin) hookSetRecordStatusPre(payload string) error { - var srs plugins.HookSetRecordStatus - err := json.Unmarshal([]byte(payload), &srs) - if err != nil { - return err - } - - // User metadata should not change on status changes - err = userMetadataPreventUpdates(srs.Record.Metadata, srs.Metadata) - if err != nil { - return err - } - - // Verify status change metadata - err = statusChangeMetadataVerify(srs.RecordMetadata, srs.Metadata) - if err != nil { - return err - } - - return nil -} - -func (p *userPlugin) hookSetRecordStatusPost(treeID int64, payload string) error { - var srs plugins.HookSetRecordStatus - err := json.Unmarshal([]byte(payload), &srs) - if err != nil { - return err - } - rm := srs.RecordMetadata - - // When a record is made public the token must be moved from the - // unvetted to the vetted category in the user cache. - if rm.Status == backend.StatusPublic { - um, err := userMetadataDecode(srs.Metadata) - if err != nil { - return err - } - err = p.userCacheMoveTokenToVetted(um.UserID, rm.Token) - if err != nil { - return err +func convertSignatureError(err error) backend.PluginError { + var e util.SignatureError + var s usermd.ErrorCodeT + if errors.As(err, &e) { + switch e.ErrorCode { + case util.ErrorStatusPublicKeyInvalid: + s = usermd.ErrorCodePublicKeyInvalid + case util.ErrorStatusSignatureInvalid: + s = usermd.ErrorCodeSignatureInvalid } } - - return nil + return backend.PluginError{ + PluginID: usermd.PluginID, + ErrorCode: uint32(s), + ErrorContext: e.ErrorContext, + } } diff --git a/politeiad/backendv2/tstorebe/plugins/usermd/usermd.go b/politeiad/backendv2/tstorebe/plugins/usermd/usermd.go index 7a5e3e0a0..28f3ee0d9 100644 --- a/politeiad/backendv2/tstorebe/plugins/usermd/usermd.go +++ b/politeiad/backendv2/tstorebe/plugins/usermd/usermd.go @@ -15,10 +15,14 @@ import ( ) var ( - _ plugins.PluginClient = (*userPlugin)(nil) + _ plugins.PluginClient = (*usermdPlugin)(nil) ) -type userPlugin struct { +// usermdPlugin is the tstore backend implementation of the usermd plugin. The +// usermd plugin extends a record with user metadata. +// +// usermdPlugin satisfies the plugins PluginClient interface. +type usermdPlugin struct { sync.Mutex tstore plugins.TstoreClient @@ -30,8 +34,8 @@ type userPlugin struct { // Setup performs any plugin setup that is required. // -// This function satisfies the plugins.PluginClient interface. -func (p *userPlugin) Setup() error { +// This function satisfies the plugins PluginClient interface. +func (p *usermdPlugin) Setup() error { log.Tracef("usermd Setup") return nil @@ -39,8 +43,8 @@ func (p *userPlugin) Setup() error { // Cmd executes a plugin command. // -// This function satisfies the plugins.PluginClient interface. -func (p *userPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { +// This function satisfies the plugins PluginClient interface. +func (p *usermdPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { log.Tracef("usermd Cmd: %v %x %v %v", treeID, token, cmd, payload) switch cmd { @@ -55,8 +59,8 @@ func (p *userPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (strin // Hook executes a plugin hook. // -// This function satisfies the plugins.PluginClient interface. -func (p *userPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { +// This function satisfies the plugins PluginClient interface. +func (p *usermdPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { log.Tracef("usermd Hook: %v %x %v", plugins.Hooks[h], token, treeID) switch h { @@ -79,8 +83,8 @@ func (p *userPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload s // Fsck performs a plugin filesystem check. // -// This function satisfies the plugins.PluginClient interface. -func (p *userPlugin) Fsck(treeIDs []int64) error { +// This function satisfies the plugins PluginClient interface. +func (p *usermdPlugin) Fsck(treeIDs []int64) error { log.Tracef("usermd Fsck") return nil @@ -88,15 +92,15 @@ func (p *userPlugin) Fsck(treeIDs []int64) error { // Settings returns the plugin's settings. // -// This function satisfies the plugins.PluginClient interface. -func (p *userPlugin) Settings() []backend.PluginSetting { +// This function satisfies the plugins PluginClient interface. +func (p *usermdPlugin) Settings() []backend.PluginSetting { log.Tracef("usermd Settings") return nil } -// New returns a new userPlugin. -func New(tstore plugins.TstoreClient, settings []backend.PluginSetting, dataDir string) (*userPlugin, error) { +// New returns a new usermdPlugin. +func New(tstore plugins.TstoreClient, settings []backend.PluginSetting, dataDir string) (*usermdPlugin, error) { // Create plugin data directory dataDir = filepath.Join(dataDir, usermd.PluginID) err := os.MkdirAll(dataDir, 0700) @@ -104,7 +108,7 @@ func New(tstore plugins.TstoreClient, settings []backend.PluginSetting, dataDir return nil, err } - return &userPlugin{ + return &usermdPlugin{ tstore: tstore, dataDir: dataDir, }, nil diff --git a/politeiad/backendv2/tstorebe/tstorebe.go b/politeiad/backendv2/tstorebe/tstorebe.go index ecd9538d7..550b4536e 100644 --- a/politeiad/backendv2/tstorebe/tstorebe.go +++ b/politeiad/backendv2/tstorebe/tstorebe.go @@ -29,8 +29,8 @@ var ( _ backend.Backend = (*tstoreBackend)(nil) ) -// tstoreBackend implements the Backend interface using a tstore as the -// data store. +// tstoreBackend implements the backendv2 Backend interface using a tstore as +// the backing data store. type tstoreBackend struct { sync.RWMutex appDir string @@ -424,7 +424,7 @@ func recordMetadataNew(token []byte, files []backend.File, state backend.StateT, // RecordNew creates a new record. // -// This function satisfies the Backend interface. +// This function satisfies the backendv2 Backend interface. func (t *tstoreBackend) RecordNew(metadata []backend.MetadataStream, files []backend.File) (*backend.Record, error) { log.Tracef("RecordNew") @@ -518,7 +518,7 @@ func (t *tstoreBackend) RecordNew(metadata []backend.MetadataStream, files []bac // RecordEdit edits an existing record. This creates a new version of the // record. // -// This function satisfies the Backend interface. +// This function satisfies the backendv2 Backend interface. func (t *tstoreBackend) RecordEdit(token []byte, mdAppend, mdOverwrite []backend.MetadataStream, filesAdd []backend.File, filesDel []string) (*backend.Record, error) { log.Tracef("RecordEdit: %x", token) @@ -626,7 +626,7 @@ func (t *tstoreBackend) RecordEdit(token []byte, mdAppend, mdOverwrite []backend // record files. This creates a new iteration of the record, but not a new // version of the record. // -// This function satisfies the Backend interface. +// This function satisfies the backendv2 Backend interface. func (t *tstoreBackend) RecordEditMetadata(token []byte, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { log.Tracef("RecordEditMetadata: %x", token) @@ -802,7 +802,7 @@ func (t *tstoreBackend) setStatusCensored(token []byte, rm backend.RecordMetadat // RecordSetStatus sets the status of a record. // -// This function satisfies the Backend interface. +// This function satisfies the backendv2 Backend interface. func (t *tstoreBackend) RecordSetStatus(token []byte, status backend.StatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { log.Tracef("RecordSetStatus: %x %v", token, status) @@ -949,7 +949,7 @@ func (t *tstoreBackend) RecordSetStatus(token []byte, status backend.StatusT, md // light weight. Its for this reason that we rely on the tree exists call // despite the edge case. // -// This function satisfies the Backend interface. +// This function satisfies the backendv2 Backend interface. func (t *tstoreBackend) RecordExists(token []byte) bool { log.Tracef("RecordExists: %x", token) @@ -968,7 +968,7 @@ func (t *tstoreBackend) RecordExists(token []byte) bool { // RecordTimestamps returns the timestamps for a record. If no version is provided // then timestamps for the most recent version will be returned. // -// This function satisfies the Backend interface. +// This function satisfies the backendv2 Backend interface. func (t *tstoreBackend) RecordTimestamps(token []byte, version uint32) (*backend.RecordTimestamps, error) { log.Tracef("RecordTimestamps: %x %v", token, version) @@ -988,7 +988,7 @@ func (t *tstoreBackend) RecordTimestamps(token []byte, version uint32) (*backend // returned. If the record was not found then it will not be included in the // returned map. // -// This function satisfies the Backend interface. +// This function satisfies the backendv2 Backend interface. func (t *tstoreBackend) Records(reqs []backend.RecordRequest) (map[string]backend.Record, error) { log.Tracef("Records: %v reqs", len(reqs)) @@ -1038,7 +1038,7 @@ func (t *tstoreBackend) Records(reqs []backend.RecordRequest) (map[string]backen // If no status is provided then the most recent page of tokens for all // statuses will be returned. All other arguments are ignored. // -// This function satisfies the Backend interface. +// This function satisfies the backendv2 Backend interface. func (t *tstoreBackend) Inventory(state backend.StateT, status backend.StatusT, pageSize, pageNumber uint32) (*backend.Inventory, error) { log.Tracef("Inventory: %v %v %v %v", state, status, pageSize, pageNumber) @@ -1057,7 +1057,7 @@ func (t *tstoreBackend) Inventory(state backend.StateT, status backend.StatusT, // their most recent status change. The returned tokens will include all record // statuses. // -// This function satisfies the Backend interface. +// This function satisfies the backendv2 Backend interface. func (t *tstoreBackend) InventoryOrdered(state backend.StateT, pageSize, pageNumber uint32) ([]string, error) { log.Tracef("InventoryOrdered: %v %v %v", state, pageSize, pageNumber) @@ -1071,14 +1071,14 @@ func (t *tstoreBackend) InventoryOrdered(state backend.StateT, pageSize, pageNum // PluginRegister registers a plugin. // -// This function satisfies the Backend interface. +// This function satisfies the backendv2 Backend interface. func (t *tstoreBackend) PluginRegister(p backend.Plugin) error { return t.tstore.PluginRegister(t, p) } // PluginSetup performs any required plugin setup. // -// This function satisfies the Backend interface. +// This function satisfies the backendv2 Backend interface. func (t *tstoreBackend) PluginSetup(pluginID string) error { log.Tracef("PluginSetup: %v", pluginID) @@ -1087,7 +1087,7 @@ func (t *tstoreBackend) PluginSetup(pluginID string) error { // PluginRead executes a read-only plugin command. // -// This function satisfies the Backend interface. +// This function satisfies the backendv2 Backend interface. func (t *tstoreBackend) PluginRead(token []byte, pluginID, pluginCmd, payload string) (string, error) { log.Tracef("PluginRead: %x %v %v", token, pluginID, pluginCmd) @@ -1116,7 +1116,7 @@ func (t *tstoreBackend) PluginRead(token []byte, pluginID, pluginCmd, payload st // PluginWrite executes a plugin command that writes data. // -// This function satisfies the Backend interface. +// This function satisfies the backendv2 Backend interface. func (t *tstoreBackend) PluginWrite(token []byte, pluginID, pluginCmd, payload string) (string, error) { log.Tracef("PluginWrite: %x %v %v", token, pluginID, pluginCmd) @@ -1178,7 +1178,7 @@ func (t *tstoreBackend) PluginWrite(token []byte, pluginID, pluginCmd, payload s // PluginInventory returns all registered plugins. // -// This function satisfies the Backend interface. +// This function satisfies the backendv2 Backend interface. func (t *tstoreBackend) PluginInventory() []backend.Plugin { log.Tracef("Plugins") @@ -1187,7 +1187,7 @@ func (t *tstoreBackend) PluginInventory() []backend.Plugin { // Close performs cleanup of the backend. // -// This function satisfies the Backend interface. +// This function satisfies the backendv2 Backend interface. func (t *tstoreBackend) Close() { log.Tracef("Close") @@ -1201,7 +1201,7 @@ func (t *tstoreBackend) Close() { t.tstore.Close() } -// setup builds the tstore backend caches. +// setup builds the tstore backend memory caches. func (t *tstoreBackend) setup() error { log.Tracef("setup") diff --git a/politeiad/plugins/usermd/usermd.go b/politeiad/plugins/usermd/usermd.go index 8c4e959fc..6a0ab29bd 100644 --- a/politeiad/plugins/usermd/usermd.go +++ b/politeiad/plugins/usermd/usermd.go @@ -81,8 +81,7 @@ type StatusChangeMetadata struct { Timestamp int64 `json:"timestamp"` } -// Author returns the user ID of a record's author. If no UserMetadata is -// present for the record then an empty string will be returned. +// Author returns the user ID of a record's author. type Author struct{} // AuthorReply is the reply to the Author command. From 7d3b3fe95a7aa31a34291e63291279cc502c832f Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 23 Mar 2021 13:41:59 -0500 Subject: [PATCH 428/449] politeiad: Cleanup plugin APIs. --- LICENSE | 2 +- .../tstorebe/plugins/usermd/hooks.go | 57 +++++++++---------- politeiad/plugins/comments/comments.go | 10 ++-- politeiad/plugins/dcrdata/dcrdata.go | 22 +++++-- politeiad/plugins/pi/pi.go | 43 ++++++++++---- politeiad/plugins/ticketvote/ticketvote.go | 23 ++++---- politeiad/plugins/usermd/usermd.go | 49 ++++++++++++---- politeiawww/client/client.go | 9 ++- util/net.go | 2 +- 9 files changed, 137 insertions(+), 80 deletions(-) diff --git a/LICENSE b/LICENSE index 113ba9f69..4c9457ebb 100644 --- a/LICENSE +++ b/LICENSE @@ -1,7 +1,7 @@ ISC License Copyright (c) 2013-2019 The btcsuite developers -Copyright (c) 2015-2020 The Decred developers +Copyright (c) 2015-2021 The Decred developers Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above diff --git a/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go b/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go index 947cf349f..1fa512a8d 100644 --- a/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go @@ -82,12 +82,11 @@ func (p *usermdPlugin) hookEditRecordPre(payload string) error { return err } if um.UserID != umCurr.UserID { - e := fmt.Sprintf("user id cannot change: got %v, want %v", - um.UserID, umCurr.UserID) return backend.PluginError{ - PluginID: usermd.PluginID, - ErrorCode: uint32(usermd.ErrorCodeUserIDInvalid), - ErrorContext: e, + PluginID: usermd.PluginID, + ErrorCode: uint32(usermd.ErrorCodeUserIDInvalid), + ErrorContext: fmt.Sprintf("user id cannot change: got %v, want %v", + um.UserID, umCurr.UserID), } } @@ -235,30 +234,27 @@ func userMetadataPreventUpdates(current, update []backend.MetadataStream) error // Verify user metadata has not changed switch { case u.UserID != c.UserID: - e := fmt.Sprintf("user id cannot change: got %v, want %v", - u.UserID, c.UserID) return backend.PluginError{ - PluginID: usermd.PluginID, - ErrorCode: uint32(usermd.ErrorCodeUserIDInvalid), - ErrorContext: e, + PluginID: usermd.PluginID, + ErrorCode: uint32(usermd.ErrorCodeUserIDInvalid), + ErrorContext: fmt.Sprintf("user id cannot change: got %v, want %v", + u.UserID, c.UserID), } case u.PublicKey != c.PublicKey: - e := fmt.Sprintf("public key cannot change: got %v, want %v", - u.PublicKey, c.PublicKey) return backend.PluginError{ - PluginID: usermd.PluginID, - ErrorCode: uint32(usermd.ErrorCodePublicKeyInvalid), - ErrorContext: e, + PluginID: usermd.PluginID, + ErrorCode: uint32(usermd.ErrorCodePublicKeyInvalid), + ErrorContext: fmt.Sprintf("public key cannot change: got %v, want %v", + u.PublicKey, c.PublicKey), } case c.Signature != c.Signature: - e := fmt.Sprintf("signature cannot change: got %v, want %v", - u.Signature, c.Signature) return backend.PluginError{ - PluginID: usermd.PluginID, - ErrorCode: uint32(usermd.ErrorCodeSignatureInvalid), - ErrorContext: e, + PluginID: usermd.PluginID, + ErrorCode: uint32(usermd.ErrorCodeSignatureInvalid), + ErrorContext: fmt.Sprintf("signature cannot change: got %v, want %v", + u.Signature, c.Signature), } } @@ -320,23 +316,22 @@ func statusChangeMetadataVerify(rm backend.RecordMetadata, metadata []backend.Me // Verify token matches if scm.Token != rm.Token { - e := fmt.Sprintf("status change token does not match record "+ - "metadata token: got %v, want %v", scm.Token, rm.Token) return backend.PluginError{ - PluginID: usermd.PluginID, - ErrorCode: uint32(usermd.ErrorCodeTokenInvalid), - ErrorContext: e, + PluginID: usermd.PluginID, + ErrorCode: uint32(usermd.ErrorCodeTokenInvalid), + ErrorContext: fmt.Sprintf("status change token does not match "+ + "record metadata token: got %v, want %v", scm.Token, rm.Token), } } // Verify status matches if scm.Status != uint32(rm.Status) { - e := fmt.Sprintf("status from metadata does not match status from "+ - "record metadata: got %v, want %v", scm.Status, rm.Status) return backend.PluginError{ - PluginID: usermd.PluginID, - ErrorCode: uint32(usermd.ErrorCodeStatusInvalid), - ErrorContext: e, + PluginID: usermd.PluginID, + ErrorCode: uint32(usermd.ErrorCodeStatusInvalid), + ErrorContext: fmt.Sprintf("status from metadata does not "+ + "match status from record metadata: got %v, want %v", + scm.Status, rm.Status), } } @@ -345,7 +340,7 @@ func statusChangeMetadataVerify(rm backend.RecordMetadata, metadata []backend.Me if ok && scm.Reason == "" { return backend.PluginError{ PluginID: usermd.PluginID, - ErrorCode: uint32(usermd.ErrorCodeReasonInvalid), + ErrorCode: uint32(usermd.ErrorCodeReasonMissing), ErrorContext: "a reason must be given for this status change", } } diff --git a/politeiad/plugins/comments/comments.go b/politeiad/plugins/comments/comments.go index e23aa4707..5184c1f3c 100644 --- a/politeiad/plugins/comments/comments.go +++ b/politeiad/plugins/comments/comments.go @@ -2,8 +2,8 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -// Package comments provides a plugin for adding comments and comment votes to -// records. +// Package comments provides a plugin for extending a record with comment +// functionality. package comments const ( @@ -28,11 +28,11 @@ const ( // and value to the plugin on startup. const ( // SettingKeyCommentLengthMax is the plugin setting key for the - // comment length max plugin setting. + // SettingCommentLengthMax plugin setting. SettingKeyCommentLengthMax = "commentlengthmax" - // SettingKeyVoteChangesMax is the plugin setting key for the vote - // changes max plugin setting. + // SettingKeyVoteChangesMax is the plugin setting key for the + // SettingVoteChangesMax plugin setting. SettingKeyVoteChangesMax = "votechangesmax" ) diff --git a/politeiad/plugins/dcrdata/dcrdata.go b/politeiad/plugins/dcrdata/dcrdata.go index 5e82e59e5..62ed43fa1 100644 --- a/politeiad/plugins/dcrdata/dcrdata.go +++ b/politeiad/plugins/dcrdata/dcrdata.go @@ -2,8 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -// Package dcrdata provides a plugin for retrieving data from the dcrdata block -// explorer. +// Package dcrdata provides a plugin for querying the dcrdata block explorer. package dcrdata const ( @@ -15,13 +14,24 @@ const ( CmdBlockDetails = "blockdetails" // Get details of a block CmdTicketPool = "ticketpool" // Get ticket pool CmdTxsTrimmed = "txstrimmed" // Get trimmed transactions +) - // Setting keys are the plugin setting keys that can be used to - // override a default plugin setting. Defaults will be overridden - // if a plugin setting is provided to the plugin on startup. +// Plugin setting keys can be used to specify custom plugin settings. Default +// plugin setting values can be overridden by providing a plugin setting key +// and value to the plugin on startup. +const ( + // SettingKeyHostHTTP is the plugin setting key for the plugin + // setting SettingHostHTTP. SettingKeyHostHTTP = "hosthttp" - SettingKeyHostWS = "hostws" + // SettingKeyHostWS is the plugin setting key for the plugin + // setting SettingHostWS. + SettingKeyHostWS = "hostws" +) + +// Plugin setting default values. These can be overridden by providing a plugin +// setting key and value to the plugin on startup. +const ( // SettingHostHTTPMainNet is the default dcrdata mainnet http host. SettingHostHTTPMainNet = "https://dcrdata.decred.org" diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index 5630367e9..7732a325d 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -2,24 +2,47 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -// Package pi provides a plugin for functionality that is specific to decred's -// proposal system. +// Pacakge pi provides a plugin that extends records with functionality for +// decred's proposal system. package pi const ( // PluginID is the unique identifier for this plugin. PluginID = "pi" +) + +// Plugin setting keys can be used to specify custom plugin settings. Default +// plugin setting values can be overridden by providing a plugin setting key +// and value to the plugin on startup. +const ( + // SettingKeyTextFileSizeMax is the plugin setting key for the + // SettingTextFileSizeMax plugin setting. + SettingKeyTextFileSizeMax = "textfilesizemax" + + // SettingKeyImageFileCountMax is the plugin setting key for the + // SettingImageFileCountMax plugin setting. + SettingKeyImageFileCountMax = "imagefilecountmax" + + // SettingKeyImageFileSizeMax is the plugin setting key for the + // SettingImageFileSizeMax plugin setting. + SettingKeyImageFileSizeMax = "imagefilesizemax" - // Setting keys are the plugin setting keys that can be used to - // override a default plugin setting. Defaults will be overridden - // if a plugin setting is provided to the plugin on startup. - SettingKeyTextFileSizeMax = "textfilesizemax" - SettingKeyImageFileCountMax = "imagefilecountmax" - SettingKeyImageFileSizeMax = "imagefilesizemax" - SettingKeyProposalNameLengthMin = "proposalnamelengthmin" - SettingKeyProposalNameLengthMax = "proposalnamelengthmax" + // SettingKeyProposalNameLengthMin is the plugin setting key for + // the SettingProposalNameLengthMin plugin setting. + SettingKeyProposalNameLengthMin = "proposalnamelengthmin" + + // SettingKeyProposalNameLengthMax is the plugin setting key for + // the SettingProposalNameLengthMax plugin setting. + SettingKeyProposalNameLengthMax = "proposalnamelengthmax" + + // SettingKeyProposalNameSupportedChars is the plugin setting key + // for the SettingProposalNameSupportedChars plugin setting. SettingKeyProposalNameSupportedChars = "proposalnamesupportedchars" +) +// Plugin setting default values. These can be overridden by providing a plugin +// setting key and value to the plugin on startup. +const ( // SettingTextFileSizeMax is the default maximum allowed size of a // text file in bytes. SettingTextFileSizeMax uint32 = 512 * 1024 diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index fd60159f2..1c758905b 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -19,27 +19,27 @@ const ( CmdSummary = "summary" // Get vote summary CmdSubmissions = "submissions" // Get runoff vote submissions CmdInventory = "inventory" // Get inventory by vote status - CmdTimestamps = "timestamps" // Get vote data timestamps + CmdTimestamps = "timestamps" // Get vote timestamps ) // Plugin setting keys can be used to specify custom plugin settings. Default // plugin setting values can be overridden by providing a plugin setting key // and value to the plugin on startup. const ( - // SettingKeyLinkByPeriodMin is the plugin setting key for the link - // by period min plugin setting. + // SettingKeyLinkByPeriodMin is the plugin setting key for the + // SettingLinkByPeriodMin plugin setting. SettingKeyLinkByPeriodMin = "linkbyperiodmin" - // SettingKeyLinkByPeriodMax is the plugin setting key for the link - // by period max plugin setting. + // SettingKeyLinkByPeriodMax is the plugin setting key for the + // SettingLinkByPeriodMax plugin setting. SettingKeyLinkByPeriodMax = "linkbyperiodmax" - // SettingKeyVoteDurationMin is the plugin setting key for the vote - // duration min plugin setting. + // SettingKeyVoteDurationMin is the plugin setting key for the + // SettingVoteDurationMin plugin setting. SettingKeyVoteDurationMin = "votedurationmin" - // SettingKeyVoteDurationMax is the plugin setting key for the vote - // duration max plugin setting. + // SettingKeyVoteDurationMax is the plugin setting key for the + // SettingVoteDurationMax plugin setting. SettingKeyVoteDurationMax = "votedurationmax" ) @@ -337,7 +337,7 @@ type VoteDetails struct { } // CastVoteDetails contains the details of a cast vote. A JSON encoded cast -// vote details is 405 bytes (could vary slightly depending on the votebit). +// vote details is 405 bytes. // // Signature is the client signature of the Token+Ticket+VoteBit. The receipt // is the server signature of the client signature. @@ -685,7 +685,8 @@ const ( // is ~2000 bytes so a page of 100 votes will only be 0.2MB, but // the bottleneck on this call is performance, not size. Its // expensive to retrieve a large number of inclusion proofs from - // trillian. A 100 timestamps will take ~1 second to compile. + // trillian. A 100 timestamps request will take ~1 second to + // complete. VoteTimestampsPageSize uint32 = 100 ) diff --git a/politeiad/plugins/usermd/usermd.go b/politeiad/plugins/usermd/usermd.go index 6a0ab29bd..71aa5e041 100644 --- a/politeiad/plugins/usermd/usermd.go +++ b/politeiad/plugins/usermd/usermd.go @@ -13,7 +13,10 @@ const ( // Plugin commands CmdAuthor = "author" // Get record author CmdUserRecords = "userrecords" // Get user submitted records +) +// Stream IDs are the metadata stream IDs for metadata defined in this package. +const ( // StreamIDUserMetadata is the politeiad metadata stream ID for the // UserMetadata structure. StreamIDUserMetadata uint32 = 1 @@ -28,16 +31,42 @@ const ( type ErrorCodeT uint32 const ( - // User error codes - ErrorCodeInvalid ErrorCodeT = 0 - ErrorCodeUserMetadataNotFound ErrorCodeT = 1 - ErrorCodeUserIDInvalid ErrorCodeT = 2 - ErrorCodePublicKeyInvalid ErrorCodeT = 3 - ErrorCodeSignatureInvalid ErrorCodeT = 4 + // ErrorCodeInvalid is an invalid error code. + ErrorCodeInvalid ErrorCodeT = 0 + + // ErrorCodeuserMetadataNotFound is returned when a record does + // not contain a metdata stream for user metadata. + ErrorCodeUserMetadataNotFound ErrorCodeT = 1 + + // ErrorCodeUserIDInvalid is returned when a user ID is changed + // between versions of a record. + ErrorCodeUserIDInvalid ErrorCodeT = 2 + + // ErrorCodePlublicKeyInvalid is returned when a public key used + // in a signature is not valid. + ErrorCodePublicKeyInvalid ErrorCodeT = 3 + + // ErrorCodeSignatureInvalid is returned when the signature does + // not match the expected signature. + ErrorCodeSignatureInvalid ErrorCodeT = 4 + + // ErrorCodeStatusChangeMetadataNotFound is returned when a record + // is having its status updated but is missing the status change + // metadata. ErrorCodeStatusChangeMetadataNotFound ErrorCodeT = 5 - ErrorCodeTokenInvalid ErrorCodeT = 6 - ErrorCodeStatusInvalid ErrorCodeT = 7 - ErrorCodeReasonInvalid ErrorCodeT = 8 + + // ErrorCodeTokenInvalid is returned when a token that is included + // in the metadata does not match the token of the record that the + // command is being executed on. + ErrorCodeTokenInvalid ErrorCodeT = 6 + + // ErrorCodeStatusInvalid is returned when the status defined in + // the status change metadata does not match the record status. + ErrorCodeStatusInvalid ErrorCodeT = 7 + + // ErrorCodeReasonMissing is returned when the status change reason + // is required but is not included. + ErrorCodeReasonMissing ErrorCodeT = 8 ) var ( @@ -51,7 +80,7 @@ var ( ErrorCodeStatusChangeMetadataNotFound: "status change metadata not found", ErrorCodeTokenInvalid: "token invalid", ErrorCodeStatusInvalid: "status invalid", - ErrorCodeReasonInvalid: "status reason invalid", + ErrorCodeReasonMissing: "status change reason is missing", } ) diff --git a/politeiawww/client/client.go b/politeiawww/client/client.go index cb860815c..287bbf9d4 100644 --- a/politeiawww/client/client.go +++ b/politeiawww/client/client.go @@ -19,6 +19,7 @@ import ( ) var ( + // HTTP headers headerCSRF = "X-CSRF-Token" ) @@ -33,7 +34,7 @@ type Client struct { // makeReq makes a politeiawww http request to the method and route provided, // serializing the provided object as the request body, and returning a byte -// slice of the response body. An Error is returned if politeiawww responds +// slice of the response body. An ReqError is returned if politeiawww responds // with anything other than a 200 http status code. func (c *Client) makeReq(method string, api, route string, v interface{}) ([]byte, error) { // Serialize body @@ -135,9 +136,7 @@ func (c *Client) makeReq(method string, api, route string, v interface{}) ([]byt // Decode response body respBody := util.RespBody(r) - // Print response body. Pretty printing the response body for the - // verbose output must be handled by the calling function once it - // has unmarshalled the body. + // Print response body if c.verbose || c.rawJSON { fmt.Printf("%s\n", respBody) } @@ -147,7 +146,7 @@ func (c *Client) makeReq(method string, api, route string, v interface{}) ([]byt // Opts contains the politeiawww client options. All values are optional. // -// Any provided HTTPSCert will be added to the http client's trust cert pool, +// Any provided HTTPSCert will be added to the http client's trusted cert pool, // allowing you to interact with a politeiawww instance that uses a self signed // cert. // diff --git a/util/net.go b/util/net.go index 98387a6a9..cb4fb8070 100644 --- a/util/net.go +++ b/util/net.go @@ -30,7 +30,7 @@ func NormalizeAddress(addr, defaultPort string) string { return addr } -// NewHTTPClient returns a new http.Client instance +// NewHTTPClient returns a new http Client. func NewHTTPClient(skipVerify bool, certPath string) (*http.Client, error) { tlsConfig := &tls.Config{ InsecureSkipVerify: skipVerify, From 6d769440dc050999fbc9cc2c5e7b8bcf098fc0d2 Mon Sep 17 00:00:00 2001 From: luke Date: Tue, 23 Mar 2021 18:36:26 -0500 Subject: [PATCH 429/449] tstorebe/ticketvote: Cleanup and add docs. --- .../tstorebe/plugins/comments/cmds.go | 90 +- .../plugins/ticketvote/activevotes.go | 36 +- .../tstorebe/plugins/ticketvote/cmds.go | 4152 +++++++++-------- .../tstorebe/plugins/ticketvote/hooks.go | 466 +- .../plugins/ticketvote/internalcmds.go | 59 + .../tstorebe/plugins/ticketvote/ticketvote.go | 36 - politeiad/plugins/ticketvote/ticketvote.go | 4 +- 7 files changed, 2463 insertions(+), 2380 deletions(-) create mode 100644 politeiad/backendv2/tstorebe/plugins/ticketvote/internalcmds.go diff --git a/politeiad/backendv2/tstorebe/plugins/comments/cmds.go b/politeiad/backendv2/tstorebe/plugins/comments/cmds.go index 14bbbb4d3..1907b6c6e 100644 --- a/politeiad/backendv2/tstorebe/plugins/comments/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/comments/cmds.go @@ -356,21 +356,9 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str } // Verify token - t, err := tokenDecode(n.Token) + err = tokenVerify(token, n.Token) if err != nil { - return "", backend.PluginError{ - PluginID: comments.PluginID, - ErrorCode: uint32(comments.ErrorCodeTokenInvalid), - ErrorContext: err.Error(), - } - } - if !bytes.Equal(t, token) { - return "", backend.PluginError{ - PluginID: comments.PluginID, - ErrorCode: uint32(comments.ErrorCodeTokenInvalid), - ErrorContext: fmt.Sprintf("comment token does not match "+ - "route token: got %x, want %x", t, token), - } + return "", err } // Verify signature @@ -487,21 +475,9 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st } // Verify token - t, err := tokenDecode(e.Token) + err = tokenVerify(token, e.Token) if err != nil { - return "", backend.PluginError{ - PluginID: comments.PluginID, - ErrorCode: uint32(comments.ErrorCodeTokenInvalid), - ErrorContext: err.Error(), - } - } - if !bytes.Equal(t, token) { - return "", backend.PluginError{ - PluginID: comments.PluginID, - ErrorCode: uint32(comments.ErrorCodeTokenInvalid), - ErrorContext: fmt.Sprintf("comment token does not match "+ - "route token: got %x, want %x", t, token), - } + return "", err } // Verify signature @@ -641,21 +617,9 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str } // Verify token - t, err := tokenDecode(d.Token) + err = tokenVerify(token, d.Token) if err != nil { - return "", backend.PluginError{ - PluginID: comments.PluginID, - ErrorCode: uint32(comments.ErrorCodeTokenInvalid), - ErrorContext: err.Error(), - } - } - if !bytes.Equal(t, token) { - return "", backend.PluginError{ - PluginID: comments.PluginID, - ErrorCode: uint32(comments.ErrorCodeTokenInvalid), - ErrorContext: fmt.Sprintf("comment token does not match "+ - "route token: got %x, want %x", t, token), - } + return "", err } // Verify signature @@ -774,21 +738,9 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st } // Verify token - t, err := tokenDecode(v.Token) + err = tokenVerify(token, v.Token) if err != nil { - return "", backend.PluginError{ - PluginID: comments.PluginID, - ErrorCode: uint32(comments.ErrorCodeTokenInvalid), - ErrorContext: err.Error(), - } - } - if !bytes.Equal(t, token) { - return "", backend.PluginError{ - PluginID: comments.PluginID, - ErrorCode: uint32(comments.ErrorCodeTokenInvalid), - ErrorContext: fmt.Sprintf("comment token does not match "+ - "route token: got %x, want %x", t, token), - } + return "", err } // Verify vote @@ -1262,6 +1214,32 @@ func tokenDecode(token string) ([]byte, error) { return util.TokenDecode(util.TokenTypeTstore, token) } +// tokenVerify verifies that a token that is part of a plugin command payload +// is valid. This is applicable when a plugin command payload contains a +// signature that includes the record token. The token included in payload must +// be a valid, full length record token and it must match the token that was +// passed into the politeiad API for this plugin command, i.e. the token for +// the record that this plugin command is being executed on. +func tokenVerify(cmdToken []byte, payloadToken string) error { + pt, err := tokenDecode(payloadToken) + if err != nil { + return backend.PluginError{ + PluginID: comments.PluginID, + ErrorCode: uint32(comments.ErrorCodeTokenInvalid), + ErrorContext: err.Error(), + } + } + if !bytes.Equal(cmdToken, pt) { + return backend.PluginError{ + PluginID: comments.PluginID, + ErrorCode: uint32(comments.ErrorCodeTokenInvalid), + ErrorContext: fmt.Sprintf("payload token does not match "+ + "command token: got %x, want %x", pt, cmdToken), + } + } + return nil +} + func convertCommentFromCommentAdd(ca comments.CommentAdd) comments.Comment { return comments.Comment{ UserID: ca.UserID, diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/activevotes.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/activevotes.go index aa6d1ffea..53acc9783 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/activevotes.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/activevotes.go @@ -62,8 +62,7 @@ func (a *activeVotes) VoteDetails(token []byte) *ticketvote.VoteDetails { a.RLock() defer a.RUnlock() - t := hex.EncodeToString(token) - av, ok := a.activeVotes[t] + av, ok := a.activeVotes[hex.EncodeToString(token)] if !ok { return nil } @@ -105,8 +104,7 @@ func (a *activeVotes) EligibleTickets(token []byte) map[string]struct{} { a.RLock() defer a.RUnlock() - t := hex.EncodeToString(token) - av, ok := a.activeVotes[t] + av, ok := a.activeVotes[hex.EncodeToString(token)] if !ok { return nil } @@ -130,8 +128,7 @@ func (a *activeVotes) VoteIsDuplicate(token, ticket string) bool { av, ok := a.activeVotes[token] if !ok { // This should not happen - e := fmt.Errorf("active vote not found %v", token) - panic(e) + panic(fmt.Sprintf("active vote not found %v", token)) } _, isDup := av.CastVotes[ticket] @@ -197,8 +194,7 @@ func (a *activeVotes) AddCastVote(token, ticket, votebit string) { av, ok := a.activeVotes[token] if !ok { // This should not happen - e := fmt.Sprintf("active vote not found %v", token) - panic(e) + panic(fmt.Sprintf("active vote not found %v", token)) } av.CastVotes[ticket] = votebit @@ -214,8 +210,7 @@ func (a *activeVotes) AddCommitmentAddrs(token string, addrs map[string]commitme av, ok := a.activeVotes[token] if !ok { // This should not happen - e := fmt.Sprintf("active vote not found %v", token) - panic(e) + panic(fmt.Sprintf("active vote not found %v", token)) } for ticket, v := range addrs { @@ -231,38 +226,40 @@ func (a *activeVotes) AddCommitmentAddrs(token string, addrs map[string]commitme // Del deletes an active vote from the active votes cache. func (a *activeVotes) Del(token string) { a.Lock() - defer a.Unlock() - delete(a.activeVotes, token) + a.Unlock() log.Debugf("Active votes del %v", token) } // Add adds a active vote to the active votes cache. // -// This function should NOT be used directly. The activeVotesAdd function, -// which also kicks of an async job to fetch the commitment addresses for this -// active votes entry, should be used instead. +// This function should NOT be called directly. The ticketvote method +// activeVotesAdd(), which also kicks of an async job to fetch the commitment +// addresses for this active votes entry, should be used instead. func (a *activeVotes) Add(vd ticketvote.VoteDetails) { - a.Lock() - defer a.Unlock() - token := vd.Params.Token + + a.Lock() a.activeVotes[token] = activeVote{ Details: &vd, CastVotes: make(map[string]string, 40960), // Ticket pool size Addrs: make(map[string]string, 40960), // Ticket pool size } + a.Unlock() log.Debugf("Active votes add %v", token) } +// newActiveVotes returns a new activeVotes. func newActiveVotes() *activeVotes { return &activeVotes{ activeVotes: make(map[string]activeVote, 256), } } +// activeVotePopulateAddrs fetches the largest commitment address for each +// ticket in a vote from dcrdata and caches the results. func (p *ticketVotePlugin) activeVotePopulateAddrs(vd ticketvote.VoteDetails) { // Get largest commitment address for each eligible ticket. A // TrimmedTxs response for 500 tickets is ~1MB. It takes ~1.5 @@ -300,6 +297,9 @@ func (p *ticketVotePlugin) activeVotePopulateAddrs(vd ticketvote.VoteDetails) { } } +// activeVotesAdd creates a active votes cache entry for the provided vote +// details and kicks off an async job that fetches and caches the largest +// commitment address for each eligible ticket. func (p *ticketVotePlugin) activeVotesAdd(vd ticketvote.VoteDetails) { // Add the vote to the active votes cache p.activeVotes.Add(vd) diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go index 949292c69..4ead1ba00 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go @@ -39,2381 +39,2531 @@ const ( dataDescriptorCastVoteDetails = pluginID + "-castvote-v1" dataDescriptorVoteCollider = pluginID + "-vcollider-v1" dataDescriptorStartRunoff = pluginID + "-startrunoff-v1" - - // Internal plugin commands - cmdStartRunoffSubmission = "startrunoffsub" - cmdRunoffDetails = "runoffdetails" ) -// voteHasEnded returns whether the vote has ended. -func voteHasEnded(bestBlock, endHeight uint32) bool { - return bestBlock >= endHeight -} - -// tokenDecode decodes a record token and only accepts full length tokens. -func tokenDecode(token string) ([]byte, error) { - return util.TokenDecode(util.TokenTypeTstore, token) -} - -// tokenDecode decodes a record token and accepts both full length tokens and -// token prefixes. -func tokenDecodeAnyLength(token string) ([]byte, error) { - return util.TokenDecodeAnyLength(util.TokenTypeTstore, token) -} - -func (p *ticketVotePlugin) authSave(treeID int64, ad ticketvote.AuthDetails) error { - // Prepare blob - be, err := convertBlobEntryFromAuthDetails(ad) +// cmdAuthorize authorizes a ticket vote or revokes a previous authorization. +func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload string) (string, error) { + // Decode payload + var a ticketvote.Authorize + err := json.Unmarshal([]byte(payload), &a) if err != nil { - return err + return "", err } - // Save blob - return p.tstore.BlobSave(treeID, *be) -} + // Verify token + err = tokenVerify(token, a.Token) + if err != nil { + return "", err + } -func (p *ticketVotePlugin) auths(treeID int64) ([]ticketvote.AuthDetails, error) { - // Retrieve blobs - blobs, err := p.tstore.BlobsByDataDesc(treeID, - []string{dataDescriptorAuthDetails}) + // Verify signature + version := strconv.FormatUint(uint64(a.Version), 10) + msg := a.Token + version + string(a.Action) + err = util.VerifySignature(a.Signature, a.PublicKey, msg) if err != nil { - return nil, err + return "", convertSignatureError(err) } - // Decode blobs - auths := make([]ticketvote.AuthDetails, 0, len(blobs)) - for _, v := range blobs { - a, err := convertAuthDetailsFromBlobEntry(v) - if err != nil { - return nil, err + // Verify action + switch a.Action { + case ticketvote.AuthActionAuthorize: + // This is allowed + case ticketvote.AuthActionRevoke: + // This is allowed + default: + return "", backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeAuthorizationInvalid), + ErrorContext: fmt.Sprintf("%v not a valid action", a.Action), } - auths = append(auths, *a) } - // Sanity check. They should already be sorted from oldest to - // newest. - sort.SliceStable(auths, func(i, j int) bool { - return auths[i].Timestamp < auths[j].Timestamp - }) - - return auths, nil -} - -func (p *ticketVotePlugin) voteDetailsSave(treeID int64, vd ticketvote.VoteDetails) error { - // Prepare blob - be, err := convertBlobEntryFromVoteDetails(vd) + // Verify record status and version + r, err := p.tstore.RecordPartial(treeID, 0, nil, true) if err != nil { - return err + return "", fmt.Errorf("RecordPartial: %v", err) } - - // Save blob - return p.tstore.BlobSave(treeID, *be) -} - -func (p *ticketVotePlugin) voteDetails(treeID int64) (*ticketvote.VoteDetails, error) { - // Retrieve blobs - blobs, err := p.tstore.BlobsByDataDesc(treeID, - []string{dataDescriptorVoteDetails}) - if err != nil { - return nil, err + if r.RecordMetadata.Status != backend.StatusPublic { + return "", backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeRecordStatusInvalid), + ErrorContext: "record is not public", + } } - switch len(blobs) { - case 0: - // A vote details does not exist - return nil, nil - case 1: - // A vote details exists; continue - default: - // This should not happen. There should only ever be a max of - // one vote details. - return nil, fmt.Errorf("multiple vote details found (%v) on %x", - len(blobs), treeID) + if a.Version != r.RecordMetadata.Version { + return "", backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeRecordVersionInvalid), + ErrorContext: fmt.Sprintf("version is not latest: got %v, want %v", + a.Version, r.RecordMetadata.Version), + } } - // Decode blob - vd, err := convertVoteDetailsFromBlobEntry(blobs[0]) + // Get any previous authorizations to verify that the new action + // is allowed based on the previous action. + auths, err := p.auths(treeID) if err != nil { - return nil, err + return "", err + } + var prevAction ticketvote.AuthActionT + if len(auths) > 0 { + prevAction = ticketvote.AuthActionT(auths[len(auths)-1].Action) + } + switch { + case len(auths) == 0: + // No previous actions. New action must be an authorize. + if a.Action != ticketvote.AuthActionAuthorize { + return "", backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeAuthorizationInvalid), + ErrorContext: "no prev action; action must be authorize", + } + } + case prevAction == ticketvote.AuthActionAuthorize && + a.Action != ticketvote.AuthActionRevoke: + // Previous action was a authorize. This action must be revoke. + return "", backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeAuthorizationInvalid), + ErrorContext: "prev action was authorize", + } + case prevAction == ticketvote.AuthActionRevoke && + a.Action != ticketvote.AuthActionAuthorize: + // Previous action was a revoke. This action must be authorize. + return "", backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeAuthorizationInvalid), + ErrorContext: "prev action was revoke", + } } - return vd, nil -} + // Prepare authorize vote + receipt := p.identity.SignMessage([]byte(a.Signature)) + auth := ticketvote.AuthDetails{ + Token: a.Token, + Version: a.Version, + Action: string(a.Action), + PublicKey: a.PublicKey, + Signature: a.Signature, + Timestamp: time.Now().Unix(), + Receipt: hex.EncodeToString(receipt[:]), + } -func (p *ticketVotePlugin) voteDetailsByToken(token []byte) (*ticketvote.VoteDetails, error) { - reply, err := p.backend.PluginRead(token, ticketvote.PluginID, - ticketvote.CmdDetails, "") + // Save authorize vote + err = p.authSave(treeID, auth) if err != nil { - return nil, err + return "", err } - var dr ticketvote.DetailsReply - err = json.Unmarshal([]byte(reply), &dr) - if err != nil { - return nil, err + + // Update inventory + var status ticketvote.VoteStatusT + switch a.Action { + case ticketvote.AuthActionAuthorize: + status = ticketvote.VoteStatusAuthorized + case ticketvote.AuthActionRevoke: + status = ticketvote.VoteStatusUnauthorized + default: + // Action has already been validated. This should not happen. + return "", fmt.Errorf("invalid action %v", a.Action) } - return dr.Vote, nil -} + p.inventoryUpdate(a.Token, status) -func (p *ticketVotePlugin) castVoteSave(treeID int64, cv ticketvote.CastVoteDetails) error { - // Prepare blob - be, err := convertBlobEntryFromCastVoteDetails(cv) + // Prepare reply + ar := ticketvote.AuthorizeReply{ + Timestamp: auth.Timestamp, + Receipt: auth.Receipt, + } + reply, err := json.Marshal(ar) if err != nil { - return err + return "", err } - // Save blob - return p.tstore.BlobSave(treeID, *be) + return string(reply), nil } -func (p *ticketVotePlugin) voteResults(treeID int64) ([]ticketvote.CastVoteDetails, error) { - // Retrieve blobs - desc := []string{ - dataDescriptorCastVoteDetails, - dataDescriptorVoteCollider, +// voteBitVerify verifies that the vote bit corresponds to a valid vote option. +func voteBitVerify(options []ticketvote.VoteOption, mask, bit uint64) error { + if len(options) == 0 { + return fmt.Errorf("no vote options found") } - blobs, err := p.tstore.BlobsByDataDesc(treeID, desc) - if err != nil { - return nil, err + if bit == 0 { + return fmt.Errorf("invalid bit 0x%x", bit) } - // Decode blobs. A cast vote is considered valid only if the vote - // collider exists for it. If there are multiple votes using the - // same ticket, the valid vote is the one that immediately precedes - // the vote collider blob entry. - var ( - // map[ticket]CastVoteDetails - votes = make(map[string]ticketvote.CastVoteDetails, len(blobs)) + // Verify bit is included in mask + if mask&bit != bit { + return fmt.Errorf("invalid mask 0x%x bit 0x%x", mask, bit) + } - voteIndexes = make(map[string][]int, len(blobs)) // map[ticket][]index - colliderIndexes = make(map[string]int, len(blobs)) // map[ticket]index - ) - for i, v := range blobs { - // Decode data hint - b, err := base64.StdEncoding.DecodeString(v.DataHint) - if err != nil { - return nil, err - } - var dd store.DataDescriptor - err = json.Unmarshal(b, &dd) - if err != nil { - return nil, err + // Verify bit is included in vote options + for _, v := range options { + if v.Bit == bit { + // Bit matches one of the options. We're done. + return nil } - switch dd.Descriptor { - case dataDescriptorCastVoteDetails: - // Decode cast vote - cv, err := convertCastVoteDetailsFromBlobEntry(v) - if err != nil { - return nil, err - } + } - // Save index of the cast vote - idx, ok := voteIndexes[cv.Ticket] - if !ok { - idx = make([]int, 0, 32) - } - idx = append(idx, i) - voteIndexes[cv.Ticket] = idx + return fmt.Errorf("bit 0x%x not found in vote options", bit) +} - // Save the cast vote - votes[cv.Ticket] = *cv - - case dataDescriptorVoteCollider: - // Decode vote collider - vc, err := convertVoteColliderFromBlobEntry(v) - if err != nil { - return nil, err - } - - // Sanity check - _, ok := colliderIndexes[vc.Ticket] - if ok { - return nil, fmt.Errorf("duplicate vote colliders found %v", vc.Ticket) - } - - // Save the ticket and index for the collider - colliderIndexes[vc.Ticket] = i - - default: - return nil, fmt.Errorf("invalid data descriptor: %v", dd.Descriptor) +// voteParamsVerify verifies that the params of a ticket vote are within +// acceptable values. +func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationMax uint32) error { + // Verify vote type + switch vote.Type { + case ticketvote.VoteTypeStandard: + // This is allowed + case ticketvote.VoteTypeRunoff: + // This is allowed + default: + return backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteTypeInvalid), } } - for ticket, indexes := range voteIndexes { - // Remove any votes that do not have a collider blob - colliderIndex, ok := colliderIndexes[ticket] - if !ok { - // This is not a valid vote - delete(votes, ticket) - continue + // Verify vote params + switch { + case vote.Duration > voteDurationMax: + return backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteDurationInvalid), + ErrorContext: fmt.Sprintf("duration %v exceeds max duration %v", + vote.Duration, voteDurationMax), } - - // If multiple votes have been cast using the same ticket then - // we must manually determine which vote is valid. - if len(indexes) == 1 { - // Only one cast vote exists for this ticket. This is good. - continue + case vote.Duration < voteDurationMin: + return backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteDurationInvalid), + ErrorContext: fmt.Sprintf("duration %v under min duration %v", + vote.Duration, voteDurationMin), } - - // Sanity check - if len(indexes) == 0 { - return nil, fmt.Errorf("no cast vote index found %v", ticket) + case vote.QuorumPercentage > 100: + return backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteQuorumInvalid), + ErrorContext: fmt.Sprintf("quorum percent %v exceeds 100 percent", + vote.QuorumPercentage), } + case vote.PassPercentage > 100: + return backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVotePassRateInvalid), + ErrorContext: fmt.Sprintf("pass percent %v exceeds 100 percent", + vote.PassPercentage), + } + } - log.Tracef("Multiple votes found for a single vote collider %v", ticket) - - // Multiple votes exist for this ticket. The vote that is valid - // is the one that immediately precedes the vote collider. Start - // at the end of the vote indexes and find the first vote index - // that precedes the collider index. - var validVoteIndex int - for i := len(indexes) - 1; i >= 0; i-- { - voteIndex := indexes[i] - if voteIndex < colliderIndex { - // This is the valid vote - validVoteIndex = voteIndex - break + // Verify vote options. Different vote types have different + // requirements. + if len(vote.Options) == 0 { + return backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteOptionsInvalid), + ErrorContext: "no vote options found", + } + } + switch vote.Type { + case ticketvote.VoteTypeStandard, ticketvote.VoteTypeRunoff: + // These vote types only allow for approve/reject votes. Ensure + // that the only options present are approve/reject and that they + // use the vote option IDs specified by the ticketvote API. + if len(vote.Options) != 2 { + return backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteOptionsInvalid), + ErrorContext: fmt.Sprintf("vote options count got %v, want 2", + len(vote.Options)), } } - - // Save the valid vote - b := blobs[validVoteIndex] - cv, err := convertCastVoteDetailsFromBlobEntry(b) - if err != nil { - return nil, err + // map[optionID]found + options := map[string]bool{ + ticketvote.VoteOptionIDApprove: false, + ticketvote.VoteOptionIDReject: false, + } + for _, v := range vote.Options { + switch v.ID { + case ticketvote.VoteOptionIDApprove: + options[v.ID] = true + case ticketvote.VoteOptionIDReject: + options[v.ID] = true + } + } + missing := make([]string, 0, 2) + for k, v := range options { + if !v { + // Option ID was not found + missing = append(missing, k) + } + } + if len(missing) > 0 { + return backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteOptionsInvalid), + ErrorContext: fmt.Sprintf("vote option IDs not found: %v", + strings.Join(missing, ",")), + } } - votes[cv.Ticket] = *cv } - // Put votes into an array - cvotes := make([]ticketvote.CastVoteDetails, 0, len(blobs)) - for _, v := range votes { - cvotes = append(cvotes, v) + // Verify vote bits are somewhat sane + for _, v := range vote.Options { + err := voteBitVerify(vote.Options, vote.Mask, v.Bit) + if err != nil { + return backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteBitsInvalid), + ErrorContext: err.Error(), + } + } } - // Sort by ticket hash - sort.SliceStable(cvotes, func(i, j int) bool { - return cvotes[i].Ticket < cvotes[j].Ticket - }) - - return cvotes, nil -} - -func (p *ticketVotePlugin) voteOptionResults(token []byte, options []ticketvote.VoteOption) ([]ticketvote.VoteOptionResult, error) { - // Ongoing votes will have the cast votes cached. Calculate the - // results using the cached votes if we can since it will be much - // faster. - var ( - tally = make(map[string]uint32, len(options)) - t = hex.EncodeToString(token) - ctally = p.activeVotes.Tally(t) - ) + // Verify parent token switch { - case len(ctally) > 0: - // Vote are in the cache. Use the cached results. - tally = ctally - - default: - // Votes are not in the cache. Pull them from the backend. - reply, err := p.backend.PluginRead(token, ticketvote.PluginID, - ticketvote.CmdResults, "") - if err != nil { - return nil, err + case vote.Type == ticketvote.VoteTypeStandard && vote.Parent != "": + return backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), + ErrorContext: "parent token should not be provided for a standard vote", } - var rr ticketvote.ResultsReply - err = json.Unmarshal([]byte(reply), &rr) + case vote.Type == ticketvote.VoteTypeRunoff: + _, err := tokenDecode(vote.Parent) if err != nil { - return nil, err - } - - // Tally the results - for _, v := range rr.Votes { - tally[v.VoteBit]++ + return backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), + ErrorContext: fmt.Sprintf("invalid parent %v", vote.Parent), + } } } - // Prepare reply - results := make([]ticketvote.VoteOptionResult, 0, len(options)) - for _, v := range options { - bit := strconv.FormatUint(v.Bit, 16) - results = append(results, ticketvote.VoteOptionResult{ - ID: v.ID, - Description: v.Description, - VoteBit: v.Bit, - Votes: uint64(tally[bit]), - }) + return nil +} + +// startReply fetches all date required to populate a StartReply then returns +// the newly created StartReply. +func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, error) { + // Get the best block height + bb, err := p.bestBlock() + if err != nil { + return nil, fmt.Errorf("bestBlock: %v", err) } - return results, nil -} + // Find the snapshot height. Subtract the ticket maturity from the + // block height to get into unforkable territory. + ticketMaturity := uint32(p.activeNetParams.TicketMaturity) + snapshotHeight := bb - ticketMaturity -// voteSummariesForRunoff returns the vote summaries of all submissions in a -// runoff vote. This should only be called once the vote has finished. -func (p *ticketVotePlugin) summariesForRunoff(parentToken string) (map[string]ticketvote.SummaryReply, error) { - // Get runoff vote details - parent, err := tokenDecode(parentToken) + // Fetch the block details for the snapshot height. We need the + // block hash in order to fetch the ticket pool snapshot. + bd := dcrdata.BlockDetails{ + Height: snapshotHeight, + } + payload, err := json.Marshal(bd) if err != nil { return nil, err } - reply, err := p.backend.PluginRead(parent, ticketvote.PluginID, - cmdRunoffDetails, "") + reply, err := p.backend.PluginRead(nil, dcrdata.PluginID, + dcrdata.CmdBlockDetails, string(payload)) if err != nil { - return nil, fmt.Errorf("PluginRead %x %v %v: %v", - parent, ticketvote.PluginID, cmdRunoffDetails, err) + return nil, fmt.Errorf("PluginRead %v %v: %v", + dcrdata.PluginID, dcrdata.CmdBlockDetails, err) } - var rdr runoffDetailsReply - err = json.Unmarshal([]byte(reply), &rdr) + var bdr dcrdata.BlockDetailsReply + err = json.Unmarshal([]byte(reply), &bdr) if err != nil { return nil, err } + if bdr.Block.Hash == "" { + return nil, fmt.Errorf("invalid block hash for height %v", snapshotHeight) + } + snapshotHash := bdr.Block.Hash - // Verify submissions exist - subs := rdr.Runoff.Submissions - if len(subs) == 0 { - return map[string]ticketvote.SummaryReply{}, nil + // Fetch the ticket pool snapshot + tp := dcrdata.TicketPool{ + BlockHash: snapshotHash, + } + payload, err = json.Marshal(tp) + if err != nil { + return nil, err + } + reply, err = p.backend.PluginRead(nil, dcrdata.PluginID, + dcrdata.CmdTicketPool, string(payload)) + if err != nil { + return nil, fmt.Errorf("PluginRead %v %v: %v", + dcrdata.PluginID, dcrdata.CmdTicketPool, err) + } + var tpr dcrdata.TicketPoolReply + err = json.Unmarshal([]byte(reply), &tpr) + if err != nil { + return nil, err + } + if len(tpr.Tickets) == 0 { + return nil, fmt.Errorf("no tickets found for block %v %v", + snapshotHeight, snapshotHash) } - // Compile summaries for all submissions - var ( - summaries = make(map[string]ticketvote.SummaryReply, len(subs)) - winnerNetApprove int // Net number of approve votes of the winner - winnerToken string // Token of the winner - ) - for _, v := range subs { - token, err := tokenDecode(v) - if err != nil { - return nil, err - } - - // Get vote details - vd, err := p.voteDetailsByToken(token) - if err != nil { - return nil, err - } - - // Get vote options results - results, err := p.voteOptionResults(token, vd.Params.Options) - if err != nil { - return nil, err - } - - // Save summary - s := ticketvote.SummaryReply{ - Type: vd.Params.Type, - Status: ticketvote.VoteStatusRejected, - Duration: vd.Params.Duration, - StartBlockHeight: vd.StartBlockHeight, - StartBlockHash: vd.StartBlockHash, - EndBlockHeight: vd.EndBlockHeight, - EligibleTickets: uint32(len(vd.EligibleTickets)), - QuorumPercentage: vd.Params.QuorumPercentage, - PassPercentage: vd.Params.PassPercentage, - Results: results, - } - summaries[v] = s + // The start block height has the ticket maturity subtracted from + // it to prevent forking issues. This means we the vote starts in + // the past. The ticket maturity needs to be added to the end block + // height to correct for this. + endBlockHeight := snapshotHeight + duration + ticketMaturity - // We now check if this record has the most net yes votes. + return &ticketvote.StartReply{ + StartBlockHeight: snapshotHeight, + StartBlockHash: snapshotHash, + EndBlockHeight: endBlockHeight, + EligibleTickets: tpr.Tickets, + }, nil +} - // Verify the vote met quorum and pass requirements - approved := voteIsApproved(*vd, results) - if !approved { - // Vote did not meet quorum and pass requirements. Nothing - // else to do. Record vote is not approved. - continue +// startStandard starts a standard vote. +func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvote.Start) (*ticketvote.StartReply, error) { + // Verify there is only one start details + if len(s.Starts) != 1 { + return nil, backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeStartDetailsInvalid), + ErrorContext: "more than one start details found for standard vote", } + } + sd := s.Starts[0] - // Check if this record has more net approved votes then current - // highest. - var ( - votesApprove uint64 // Number of approve votes - votesReject uint64 // Number of reject votes - ) - for _, vor := range s.Results { - switch vor.ID { - case ticketvote.VoteOptionIDApprove: - votesApprove = vor.Votes - case ticketvote.VoteOptionIDReject: - votesReject = vor.Votes - default: - // Runoff vote options can only be approve/reject - return nil, fmt.Errorf("unknown runoff vote option %v", vor.ID) - } - - netApprove := int(votesApprove) - int(votesReject) - if netApprove > winnerNetApprove { - // New winner! - winnerToken = v - winnerNetApprove = netApprove - } + // Verify token + err := tokenVerify(token, sd.Params.Token) + if err != nil { + return nil, err + } - // This function doesn't handle the unlikely case that the - // runoff vote results in a tie. - } + // Verify signature + vb, err := json.Marshal(sd.Params) + if err != nil { + return nil, err } - if winnerToken != "" { - // A winner was found. Mark their summary as approved. - s := summaries[winnerToken] - s.Status = ticketvote.VoteStatusApproved - summaries[winnerToken] = s + msg := hex.EncodeToString(util.Digest(vb)) + err = util.VerifySignature(sd.Signature, sd.PublicKey, msg) + if err != nil { + return nil, convertSignatureError(err) } - return summaries, nil -} - -// summary returns the vote summary for a record. -func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) (*ticketvote.SummaryReply, error) { - // Check if the summary has been cached - s, err := p.summaryCache(hex.EncodeToString(token)) - switch { - case errors.Is(err, errSummaryNotFound): - // Cached summary not found. Continue. - case err != nil: - // Some other error - return nil, fmt.Errorf("summaryCache: %v", err) - default: - // Caches summary was found. Return it. - return s, nil + // Verify vote options and params + err = voteParamsVerify(sd.Params, p.voteDurationMin, p.voteDurationMax) + if err != nil { + return nil, err } - // Summary has not been cached. Get it manually. + // Get vote blockchain data + sr, err := p.startReply(sd.Params.Duration) + if err != nil { + return nil, err + } - // Assume vote is unauthorized. Only update the status when the - // appropriate record has been found that proves otherwise. - status := ticketvote.VoteStatusUnauthorized + // Verify record version + r, err := p.tstore.RecordPartial(treeID, 0, nil, true) + if err != nil { + return nil, fmt.Errorf("RecordPartial: %v", err) + } + if r.RecordMetadata.State != backend.StateVetted { + // This should not be possible + return nil, fmt.Errorf("record is unvetted") + } + if sd.Params.Version != r.RecordMetadata.Version { + return nil, backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeRecordVersionInvalid), + ErrorContext: fmt.Sprintf("version is not latest: got %v, want %v", + sd.Params.Version, r.RecordMetadata.Version), + } + } - // Check if the vote has been authorized. Not all vote types - // require an authorization. + // Verify vote authorization auths, err := p.auths(treeID) if err != nil { - return nil, fmt.Errorf("auths: %v", err) + return nil, err } - if len(auths) > 0 { - lastAuth := auths[len(auths)-1] - switch ticketvote.AuthActionT(lastAuth.Action) { - case ticketvote.AuthActionAuthorize: - // Vote has been authorized; continue - status = ticketvote.VoteStatusAuthorized - case ticketvote.AuthActionRevoke: - // Vote authorization has been revoked. Its not possible for - // the vote to have been started. We can stop looking. - return &ticketvote.SummaryReply{ - Status: status, - Results: []ticketvote.VoteOptionResult{}, - BestBlock: bestBlock, - }, nil + if len(auths) == 0 { + return nil, backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteStatusInvalid), + ErrorContext: "not authorized", + } + } + action := ticketvote.AuthActionT(auths[len(auths)-1].Action) + if action != ticketvote.AuthActionAuthorize { + return nil, backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteStatusInvalid), + ErrorContext: "not authorized", } } - // Check if the vote has been started - vd, err := p.voteDetails(treeID) + // Verify vote has not already been started + svp, err := p.voteDetails(treeID) if err != nil { - return nil, fmt.Errorf("startDetails: %v", err) + return nil, err } - if vd == nil { - // Vote has not been started yet - return &ticketvote.SummaryReply{ - Status: status, - Results: []ticketvote.VoteOptionResult{}, - BestBlock: bestBlock, - }, nil + if svp != nil { + // Vote has already been started + return nil, backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteStatusInvalid), + ErrorContext: "vote already started", + } } - // Vote has been started. We need to check if the vote has ended - // yet and if it can be considered approved or rejected. - status = ticketvote.VoteStatusStarted + // Prepare vote details + vd := ticketvote.VoteDetails{ + Params: sd.Params, + PublicKey: sd.PublicKey, + Signature: sd.Signature, + StartBlockHeight: sr.StartBlockHeight, + StartBlockHash: sr.StartBlockHash, + EndBlockHeight: sr.EndBlockHeight, + EligibleTickets: sr.EligibleTickets, + } - // Tally vote results - results, err := p.voteOptionResults(token, vd.Params.Options) + // Save vote details + err = p.voteDetailsSave(treeID, vd) if err != nil { - return nil, err + return nil, fmt.Errorf("voteDetailsSave: %v", err) } - // Prepare summary - summary := ticketvote.SummaryReply{ - Type: vd.Params.Type, - Status: status, - Duration: vd.Params.Duration, - StartBlockHeight: vd.StartBlockHeight, - StartBlockHash: vd.StartBlockHash, - EndBlockHeight: vd.EndBlockHeight, - EligibleTickets: uint32(len(vd.EligibleTickets)), - QuorumPercentage: vd.Params.QuorumPercentage, - PassPercentage: vd.Params.PassPercentage, - Results: results, - BestBlock: bestBlock, - } + // Update inventory + p.inventoryUpdateToStarted(vd.Params.Token, ticketvote.VoteStatusStarted, + vd.EndBlockHeight) - // If the vote has not finished yet then we are done for now. - if !voteHasEnded(bestBlock, vd.EndBlockHeight) { - return &summary, nil - } + // Update active votes cache + p.activeVotesAdd(vd) - // The vote has finished. Find whether the vote was approved and - // cache the vote summary. - switch vd.Params.Type { - case ticketvote.VoteTypeStandard: - // Standard vote uses a simple approve/reject result - if voteIsApproved(*vd, results) { - summary.Status = ticketvote.VoteStatusApproved - } else { - summary.Status = ticketvote.VoteStatusRejected - } + return &ticketvote.StartReply{ + StartBlockHeight: sr.StartBlockHeight, + StartBlockHash: sr.StartBlockHash, + EndBlockHeight: sr.EndBlockHeight, + EligibleTickets: sr.EligibleTickets, + }, nil +} - // Cache summary - err = p.summaryCacheSave(vd.Params.Token, summary) - if err != nil { - return nil, err - } +// startRunoffRecordSave saves a startRunoffRecord to the backend. +func (p *ticketVotePlugin) startRunoffRecordSave(treeID int64, srr startRunoffRecord) error { + be, err := convertBlobEntryFromStartRunoff(srr) + if err != nil { + return err + } + err = p.tstore.BlobSave(treeID, *be) + if err != nil { + return fmt.Errorf("BlobSave %v %v: %v", + treeID, dataDescriptorStartRunoff, err) + } + return nil +} - // Remove record from the active votes cache - p.activeVotes.Del(vd.Params.Token) +// startRunoffRecord returns the startRunoff record if one exists. Nil is +// returned if a startRunoff record is not found. +func (p *ticketVotePlugin) startRunoffRecord(treeID int64) (*startRunoffRecord, error) { + blobs, err := p.tstore.BlobsByDataDesc(treeID, + []string{dataDescriptorStartRunoff}) + if err != nil { + return nil, fmt.Errorf("BlobsByDataDesc %v %v: %v", + treeID, dataDescriptorStartRunoff, err) + } - case ticketvote.VoteTypeRunoff: - // A runoff vote requires that we pull all other runoff vote - // submissions to determine if the vote actually passed. - summaries, err := p.summariesForRunoff(vd.Params.Parent) + var srr *startRunoffRecord + switch len(blobs) { + case 0: + // Nothing found + return nil, nil + case 1: + // A start runoff record was found + srr, err = convertStartRunoffFromBlobEntry(blobs[0]) if err != nil { return nil, err } - for k, v := range summaries { - // Cache summary - err = p.summaryCacheSave(k, v) - if err != nil { - return nil, err - } - - // Remove record from active votes cache - p.activeVotes.Del(k) - } - - summary = summaries[vd.Params.Token] - default: - return nil, fmt.Errorf("unknown vote type") + // This should not be possible + e := fmt.Sprintf("%v start runoff blobs found", len(blobs)) + panic(e) } - return &summary, nil + return srr, nil } -func (p *ticketVotePlugin) summaryByToken(token []byte) (*ticketvote.SummaryReply, error) { - reply, err := p.backend.PluginRead(token, ticketvote.PluginID, - ticketvote.CmdSummary, "") +// startRunoffForSub starts the voting period for a runoff vote submission. +func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs startRunoffSubmission) error { + // Sanity check + sd := srs.StartDetails + t, err := tokenDecode(sd.Params.Token) if err != nil { - return nil, fmt.Errorf("PluginRead %x %v %v: %v", - token, ticketvote.PluginID, ticketvote.CmdSummary, err) + return err } - var sr ticketvote.SummaryReply - err = json.Unmarshal([]byte(reply), &sr) - if err != nil { - return nil, err + if !bytes.Equal(token, t) { + return fmt.Errorf("invalid token") } - return &sr, nil -} -func (p *ticketVotePlugin) timestamp(treeID int64, digest []byte) (*ticketvote.Timestamp, error) { - t, err := p.tstore.Timestamp(treeID, digest) + // Get the start runoff record from the parent tree + srr, err := p.startRunoffRecord(srs.ParentTreeID) if err != nil { - return nil, fmt.Errorf("timestamp %v %x: %v", treeID, digest, err) - } - - // Convert response - proofs := make([]ticketvote.Proof, 0, len(t.Proofs)) - for _, v := range t.Proofs { - proofs = append(proofs, ticketvote.Proof{ - Type: v.Type, - Digest: v.Digest, - MerkleRoot: v.MerkleRoot, - MerklePath: v.MerklePath, - ExtraData: v.ExtraData, - }) + return err } - return &ticketvote.Timestamp{ - Data: t.Data, - Digest: t.Digest, - TxID: t.TxID, - MerkleRoot: t.MerkleRoot, - Proofs: proofs, - }, nil -} -// recordAbridged returns a record where the only record file returned is the -// vote metadata file if one exists. -func (p *ticketVotePlugin) recordAbridged(token []byte) (*backend.Record, error) { - reqs := []backend.RecordRequest{ - { - Token: token, - Filenames: []string{ - ticketvote.FileNameVoteMetadata, - }, - }, - } - rs, err := p.backend.Records(reqs) - if err != nil { - return nil, err + // Sanity check. Verify token is part of the start runoff record + // submissions. + var found bool + for _, v := range srr.Submissions { + if hex.EncodeToString(token) == v { + found = true + break + } } - r, ok := rs[hex.EncodeToString(token)] - if !ok { - return nil, backend.ErrRecordNotFound + if !found { + // This submission should not be here + return fmt.Errorf("record not in submission list") } - return &r, nil -} -// bestBlock fetches the best block from the dcrdata plugin and returns it. If -// the dcrdata connection is not active, an error will be returned. -func (p *ticketVotePlugin) bestBlock() (uint32, error) { - // Get best block - payload, err := json.Marshal(dcrdata.BestBlock{}) + // If the vote has already been started, exit gracefully. This + // allows us to recover from unexpected errors to the start runoff + // vote call as it updates the state of multiple records. If the + // call were to fail before completing, we can simply call the + // command again with the same arguments and it will pick up where + // it left off. + svp, err := p.voteDetails(treeID) if err != nil { - return 0, err + return err } - reply, err := p.backend.PluginRead(nil, dcrdata.PluginID, - dcrdata.CmdBestBlock, string(payload)) - if err != nil { - return 0, fmt.Errorf("PluginRead %v %v: %v", - dcrdata.PluginID, dcrdata.CmdBestBlock, err) + if svp != nil { + // Vote has already been started. Exit gracefully. + return nil } - // Handle response - var bbr dcrdata.BestBlockReply - err = json.Unmarshal([]byte(reply), &bbr) + // Verify record version + r, err := p.tstore.RecordPartial(treeID, 0, nil, true) if err != nil { - return 0, err + return fmt.Errorf("RecordPartial: %v", err) } - if bbr.Status != dcrdata.StatusConnected { - // The dcrdata connection is down. The best block cannot be - // trusted as being accurate. - return 0, fmt.Errorf("dcrdata connection is down") + if r.RecordMetadata.State != backend.StateVetted { + // This should not be possible + return fmt.Errorf("record is unvetted") } - if bbr.Height == 0 { - return 0, fmt.Errorf("invalid best block height 0") + if sd.Params.Version != r.RecordMetadata.Version { + return backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeRecordVersionInvalid), + ErrorContext: fmt.Sprintf("version is not latest %v: got %v, want %v", + sd.Params.Token, sd.Params.Version, r.RecordMetadata.Version), + } } - return bbr.Height, nil -} - -// bestBlockUnsafe fetches the best block from the dcrdata plugin and returns -// it. If the dcrdata connection is not active, an error WILL NOT be returned. -// The dcrdata cached best block height will be returned even though it may be -// stale. Use bestBlock() if the caller requires a guarantee that the best -// block is not stale. -func (p *ticketVotePlugin) bestBlockUnsafe() (uint32, error) { - // Get best block - payload, err := json.Marshal(dcrdata.BestBlock{}) - if err != nil { - return 0, err - } - reply, err := p.backend.PluginRead(nil, dcrdata.PluginID, - dcrdata.CmdBestBlock, string(payload)) - if err != nil { - return 0, fmt.Errorf("PluginRead %v %v: %v", - dcrdata.PluginID, dcrdata.CmdBestBlock, err) + // Prepare vote details + vd := ticketvote.VoteDetails{ + Params: sd.Params, + PublicKey: sd.PublicKey, + Signature: sd.Signature, + StartBlockHeight: srr.StartBlockHeight, + StartBlockHash: srr.StartBlockHash, + EndBlockHeight: srr.EndBlockHeight, + EligibleTickets: srr.EligibleTickets, } - // Handle response - var bbr dcrdata.BestBlockReply - err = json.Unmarshal([]byte(reply), &bbr) + // Save vote details + err = p.voteDetailsSave(treeID, vd) if err != nil { - return 0, err - } - if bbr.Height == 0 { - return 0, fmt.Errorf("invalid best block height 0") + return fmt.Errorf("voteDetailsSave: %v", err) } - return bbr.Height, nil -} + // Update inventory + p.inventoryUpdateToStarted(vd.Params.Token, ticketvote.VoteStatusStarted, + vd.EndBlockHeight) -type commitmentAddr struct { - addr string // Commitment address - err error // Error if one occurred + // Update active votes cache + p.activeVotesAdd(vd) + + return nil } -// largestCommitmentAddrs retrieves the largest commitment addresses for each -// of the provided tickets from dcrdata. A map[ticket]commitmentAddr is -// returned. -func (p *ticketVotePlugin) largestCommitmentAddrs(tickets []string) (map[string]commitmentAddr, error) { - // Get tx details - tt := dcrdata.TxsTrimmed{ - TxIDs: tickets, - } - payload, err := json.Marshal(tt) +// startRunoffForParent saves a startRunoffRecord to the parent record. Once +// this has been saved the runoff vote is considered to be started and the +// voting period on individual runoff vote submissions can be started. +func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ticketvote.Start) (*startRunoffRecord, error) { + // Check if the runoff vote data already exists on the parent tree. + srr, err := p.startRunoffRecord(treeID) if err != nil { return nil, err } - reply, err := p.backend.PluginRead(nil, dcrdata.PluginID, - dcrdata.CmdTxsTrimmed, string(payload)) - if err != nil { - return nil, fmt.Errorf("PluginRead %v %v: %v", - dcrdata.PluginID, dcrdata.CmdTxsTrimmed, err) + if srr != nil { + // We already have a start runoff record for this runoff vote. + // This can happen if the previous call failed due to an + // unexpected error such as a network error. Return the start + // runoff record so we can pick up where we left off. + return srr, nil } - var ttr dcrdata.TxsTrimmedReply - err = json.Unmarshal([]byte(reply), &ttr) + + // Get blockchain data + var ( + mask = s.Starts[0].Params.Mask + duration = s.Starts[0].Params.Duration + quorum = s.Starts[0].Params.QuorumPercentage + pass = s.Starts[0].Params.PassPercentage + ) + sr, err := p.startReply(duration) if err != nil { return nil, err } - // Find the largest commitment address for each tx - addrs := make(map[string]commitmentAddr, len(ttr.Txs)) - for _, tx := range ttr.Txs { - var ( - bestAddr string // Addr with largest commitment amount - bestAmt float64 // Largest commitment amount - addrErr error // Error if one is encountered - ) - for _, vout := range tx.Vout { - scriptPubKey := vout.ScriptPubKeyDecoded - switch { - case scriptPubKey.CommitAmt == nil: - // No commitment amount; continue - case len(scriptPubKey.Addresses) == 0: - // No commitment address; continue - case *scriptPubKey.CommitAmt > bestAmt: - // New largest commitment address found - bestAddr = scriptPubKey.Addresses[0] - bestAmt = *scriptPubKey.CommitAmt - } - } - if bestAddr == "" || bestAmt == 0.0 { - addrErr = fmt.Errorf("no largest commitment address found") - } - - // Store result - addrs[tx.TxID] = commitmentAddr{ - addr: bestAddr, - err: addrErr, - } + // Verify parent has a LinkBy and the LinkBy deadline is expired. + files := []string{ + ticketvote.FileNameVoteMetadata, } - - return addrs, nil -} - -// startReply fetches all required data and returns a StartReply. -func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, error) { - // Get the best block height - bb, err := p.bestBlock() + r, err := p.tstore.RecordPartial(treeID, 0, files, false) if err != nil { - return nil, fmt.Errorf("bestBlock: %v", err) + if errors.Is(err, backend.ErrRecordNotFound) { + return nil, backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), + ErrorContext: fmt.Sprintf("parent record not found %x", token), + } + } + return nil, fmt.Errorf("RecordPartial: %v", err) } - - // Find the snapshot height. Subtract the ticket maturity from the - // block height to get into unforkable territory. - ticketMaturity := uint32(p.activeNetParams.TicketMaturity) - snapshotHeight := bb - ticketMaturity - - // Fetch the block details for the snapshot height. We need the - // block hash in order to fetch the ticket pool snapshot. - bd := dcrdata.BlockDetails{ - Height: snapshotHeight, + if r.RecordMetadata.State != backend.StateVetted { + // This should not be possible + return nil, fmt.Errorf("record is unvetted") } - payload, err := json.Marshal(bd) + vm, err := voteMetadataDecode(r.Files) if err != nil { return nil, err } - reply, err := p.backend.PluginRead(nil, dcrdata.PluginID, - dcrdata.CmdBlockDetails, string(payload)) - if err != nil { - return nil, fmt.Errorf("PluginRead %v %v: %v", - dcrdata.PluginID, dcrdata.CmdBlockDetails, err) + if vm == nil || vm.LinkBy == 0 { + return nil, backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), + ErrorContext: fmt.Sprintf("%x is not a runoff vote parent", token), + } } - var bdr dcrdata.BlockDetailsReply - err = json.Unmarshal([]byte(reply), &bdr) + isExpired := vm.LinkBy < time.Now().Unix() + isMainNet := p.activeNetParams.Name == chaincfg.MainNetParams().Name + switch { + case !isExpired && isMainNet: + return nil, backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeLinkByNotExpired), + ErrorContext: fmt.Sprintf("parent record %x linkby deadline not met %v", + token, vm.LinkBy), + } + case !isExpired: + // Allow the vote to be started before the link by deadline + // expires on testnet and simnet only. This makes testing the + // runoff vote process easier. + log.Warnf("Parent record linkby deadline has not been met; " + + "disregarding deadline since this is not mainnet") + } + + // Compile a list of the expected submissions that should be in the + // runoff vote. This will be all of the public records that have + // linked to the parent record. The parent record's submissions + // list will include abandoned proposals that need to be filtered + // out. + lf, err := p.submissionsCache(token) if err != nil { return nil, err } - if bdr.Block.Hash == "" { - return nil, fmt.Errorf("invalid block hash for height %v", snapshotHeight) + expected := make(map[string]struct{}, len(lf.Tokens)) // [token]struct{} + for k := range lf.Tokens { + token, err := tokenDecode(k) + if err != nil { + return nil, err + } + r, err := p.recordAbridged(token) + if err != nil { + return nil, err + } + if r.RecordMetadata.Status != backend.StatusPublic { + // This record is not public and should not be included + // in the runoff vote. + continue + } + + // This is a public record that is part of the parent record's + // submissions list. It is required to be in the runoff vote. + expected[k] = struct{}{} } - snapshotHash := bdr.Block.Hash - // Fetch the ticket pool snapshot - tp := dcrdata.TicketPool{ - BlockHash: snapshotHash, + // Verify that there are no extra submissions in the runoff vote + for _, v := range s.Starts { + _, ok := expected[v.Params.Token] + if !ok { + // This submission should not be here + return nil, backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeStartDetailsInvalid), + ErrorContext: fmt.Sprintf("record %v should not be included", + v.Params.Token), + } + } } - payload, err = json.Marshal(tp) - if err != nil { - return nil, err + + // Verify that the runoff vote is not missing any submissions + subs := make(map[string]struct{}, len(s.Starts)) + for _, v := range s.Starts { + subs[v.Params.Token] = struct{}{} } - reply, err = p.backend.PluginRead(nil, dcrdata.PluginID, - dcrdata.CmdTicketPool, string(payload)) - if err != nil { - return nil, fmt.Errorf("PluginRead %v %v: %v", - dcrdata.PluginID, dcrdata.CmdTicketPool, err) + for k := range expected { + _, ok := subs[k] + if !ok { + // This records is missing from the runoff vote + return nil, backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeStartDetailsMissing), + ErrorContext: k, + } + } } - var tpr dcrdata.TicketPoolReply - err = json.Unmarshal([]byte(reply), &tpr) - if err != nil { - return nil, err + + // Prepare start runoff record + submissions := make([]string, 0, len(subs)) + for k := range subs { + submissions = append(submissions, k) } - if len(tpr.Tickets) == 0 { - return nil, fmt.Errorf("no tickets found for block %v %v", - snapshotHeight, snapshotHash) + srr = &startRunoffRecord{ + Submissions: submissions, + Mask: mask, + Duration: duration, + QuorumPercentage: quorum, + PassPercentage: pass, + StartBlockHeight: sr.StartBlockHeight, + StartBlockHash: sr.StartBlockHash, + EndBlockHeight: sr.EndBlockHeight, + EligibleTickets: sr.EligibleTickets, } - // The start block height has the ticket maturity subtracted from - // it to prevent forking issues. This means we the vote starts in - // the past. The ticket maturity needs to be added to the end block - // height to correct for this. - endBlockHeight := snapshotHeight + duration + ticketMaturity + // Save start runoff record + err = p.startRunoffRecordSave(treeID, *srr) + if err != nil { + return nil, fmt.Errorf("startRunoffRecordSave %v: %v", + treeID, err) + } - return &ticketvote.StartReply{ - StartBlockHeight: snapshotHeight, - StartBlockHash: snapshotHash, - EndBlockHeight: endBlockHeight, - EligibleTickets: tpr.Tickets, - }, nil + return srr, nil } -func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload string) (string, error) { - // Decode payload - var a ticketvote.Authorize - err := json.Unmarshal([]byte(payload), &a) - if err != nil { - return "", err +// startRunoff starts the voting period for all submissions in a runoff vote. +func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote.Start) (*ticketvote.StartReply, error) { + // Sanity check + if len(s.Starts) == 0 { + return nil, fmt.Errorf("no start details found") } - // Verify token - t, err := tokenDecode(a.Token) - if err != nil { - return "", backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeTokenInvalid), + // Perform validation that can be done without fetching any records + // from the backend. + var ( + mask = s.Starts[0].Params.Mask + duration = s.Starts[0].Params.Duration + quorum = s.Starts[0].Params.QuorumPercentage + pass = s.Starts[0].Params.PassPercentage + parent = s.Starts[0].Params.Parent + ) + for _, v := range s.Starts { + // Verify vote params are the same for all submissions + switch { + case v.Params.Type != ticketvote.VoteTypeRunoff: + return nil, backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteTypeInvalid), + ErrorContext: fmt.Sprintf("%v got %v, want %v", + v.Params.Token, v.Params.Type, ticketvote.VoteTypeRunoff), + } + case v.Params.Mask != mask: + return nil, backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteBitsInvalid), + ErrorContext: fmt.Sprintf("%v mask invalid: all must be the same", + v.Params.Token), + } + case v.Params.Duration != duration: + return nil, backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteDurationInvalid), + ErrorContext: fmt.Sprintf("%v duration does not match; "+ + "all must be the same", v.Params.Token), + } + case v.Params.QuorumPercentage != quorum: + return nil, backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteQuorumInvalid), + ErrorContext: fmt.Sprintf("%v quorum does not match; "+ + "all must be the same", v.Params.Token), + } + case v.Params.PassPercentage != pass: + return nil, backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVotePassRateInvalid), + ErrorContext: fmt.Sprintf("%v pass rate does not match; "+ + "all must be the same", v.Params.Token), + } + case v.Params.Parent != parent: + return nil, backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), + ErrorContext: fmt.Sprintf("%v parent does not match; "+ + "all must be the same", v.Params.Token), + } } - } - if !bytes.Equal(t, token) { - return "", backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeTokenInvalid), - ErrorContext: fmt.Sprintf("plugin token does not match "+ - "route token: got %x, want %x", t, token), + + // Verify token + _, err := tokenDecode(v.Params.Token) + if err != nil { + return nil, backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeTokenInvalid), + ErrorContext: v.Params.Token, + } } - } - // Verify signature - version := strconv.FormatUint(uint64(a.Version), 10) - msg := a.Token + version + string(a.Action) - err = util.VerifySignature(a.Signature, a.PublicKey, msg) - if err != nil { - return "", convertSignatureError(err) - } + // Verify parent token + _, err = tokenDecode(v.Params.Parent) + if err != nil { + return nil, backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeTokenInvalid), + ErrorContext: fmt.Sprintf("parent token %v", v.Params.Parent), + } + } - // Verify action - switch a.Action { - case ticketvote.AuthActionAuthorize: - // This is allowed - case ticketvote.AuthActionRevoke: - // This is allowed - default: - return "", backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeAuthorizationInvalid), - ErrorContext: fmt.Sprintf("%v not a valid action", a.Action), + // Verify signature + vb, err := json.Marshal(v.Params) + if err != nil { + return nil, err + } + msg := hex.EncodeToString(util.Digest(vb)) + err = util.VerifySignature(v.Signature, v.PublicKey, msg) + if err != nil { + return nil, convertSignatureError(err) } - } - // Verify record status and version - r, err := p.tstore.RecordPartial(treeID, 0, nil, true) - if err != nil { - return "", fmt.Errorf("RecordPartial: %v", err) - } - if r.RecordMetadata.Status != backend.StatusPublic { - return "", backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeRecordStatusInvalid), - ErrorContext: "record is not public", + // Verify vote options and params. Vote optoins are required to + // be approve and reject. + err = voteParamsVerify(v.Params, p.voteDurationMin, p.voteDurationMax) + if err != nil { + return nil, err } } - if a.Version != r.RecordMetadata.Version { - return "", backend.PluginError{ + + // Verify plugin command is being executed on the parent record + if hex.EncodeToString(token) != parent { + return nil, backend.PluginError{ PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeRecordVersionInvalid), - ErrorContext: fmt.Sprintf("version is not latest: got %v, want %v", - a.Version, r.RecordMetadata.Version), + ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), + ErrorContext: fmt.Sprintf("runoff vote must be started on "+ + "the parent record %v", parent), } } - // Get any previous authorizations to verify that the new action - // is allowed based on the previous action. - auths, err := p.auths(treeID) + // This function is being invoked on the runoff vote parent record. + // Create and save a start runoff record onto the parent record's + // tree. + srr, err := p.startRunoffForParent(treeID, token, s) if err != nil { - return "", err - } - var prevAction ticketvote.AuthActionT - if len(auths) > 0 { - prevAction = ticketvote.AuthActionT(auths[len(auths)-1].Action) + return nil, err } - switch { - case len(auths) == 0: - // No previous actions. New action must be an authorize. - if a.Action != ticketvote.AuthActionAuthorize { - return "", backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeAuthorizationInvalid), - ErrorContext: "no prev action; action must be authorize", - } + + // Start the voting period of each runoff vote submissions by + // using the internal plugin command startRunoffSubmission. + for _, v := range s.Starts { + token, err = tokenDecode(v.Params.Token) + if err != nil { + return nil, err } - case prevAction == ticketvote.AuthActionAuthorize && - a.Action != ticketvote.AuthActionRevoke: - // Previous action was a authorize. This action must be revoke. - return "", backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeAuthorizationInvalid), - ErrorContext: "prev action was authorize", + srs := startRunoffSubmission{ + ParentTreeID: treeID, + StartDetails: v, } - case prevAction == ticketvote.AuthActionRevoke && - a.Action != ticketvote.AuthActionAuthorize: - // Previous action was a revoke. This action must be authorize. - return "", backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeAuthorizationInvalid), - ErrorContext: "prev action was revoke", + b, err := json.Marshal(srs) + if err != nil { + return nil, err + } + _, err = p.backend.PluginWrite(token, ticketvote.PluginID, + cmdStartRunoffSubmission, string(b)) + if err != nil { + var ue backend.PluginError + if errors.As(err, &ue) { + return nil, err + } + return nil, fmt.Errorf("PluginWrite %x %v %v: %v", + token, ticketvote.PluginID, cmdStartRunoffSubmission, err) } } - // Prepare authorize vote - receipt := p.identity.SignMessage([]byte(a.Signature)) - auth := ticketvote.AuthDetails{ - Token: a.Token, - Version: a.Version, - Action: string(a.Action), - PublicKey: a.PublicKey, - Signature: a.Signature, - Timestamp: time.Now().Unix(), - Receipt: hex.EncodeToString(receipt[:]), - } + return &ticketvote.StartReply{ + StartBlockHeight: srr.StartBlockHeight, + StartBlockHash: srr.StartBlockHash, + EndBlockHeight: srr.EndBlockHeight, + EligibleTickets: srr.EligibleTickets, + }, nil +} - // Save authorize vote - err = p.authSave(treeID, auth) +// cmdStartRunoffSubmission is an internal plugin command that is used to start +// the voting period on a runoff vote submission. +func (p *ticketVotePlugin) cmdStartRunoffSubmission(treeID int64, token []byte, payload string) (string, error) { + // Decode payload + var srs startRunoffSubmission + err := json.Unmarshal([]byte(payload), &srs) if err != nil { return "", err } - // Update inventory - var status ticketvote.VoteStatusT - switch a.Action { - case ticketvote.AuthActionAuthorize: - status = ticketvote.VoteStatusAuthorized - case ticketvote.AuthActionRevoke: - status = ticketvote.VoteStatusUnauthorized - default: - // Should not happen - return "", fmt.Errorf("invalid action %v", a.Action) - } - p.inventoryUpdate(a.Token, status) - - // Prepare reply - ar := ticketvote.AuthorizeReply{ - Timestamp: auth.Timestamp, - Receipt: auth.Receipt, - } - reply, err := json.Marshal(ar) + // Start voting period on runoff vote submission + err = p.startRunoffForSub(treeID, token, srs) if err != nil { return "", err } - return string(reply), nil + return "", nil } -func voteBitVerify(options []ticketvote.VoteOption, mask, bit uint64) error { - if len(options) == 0 { - return fmt.Errorf("no vote options found") - } - if bit == 0 { - return fmt.Errorf("invalid bit 0x%x", bit) - } - - // Verify bit is included in mask - if mask&bit != bit { - return fmt.Errorf("invalid mask 0x%x bit 0x%x", mask, bit) +// cmdStart starts a ticket vote. +func (p *ticketVotePlugin) cmdStart(treeID int64, token []byte, payload string) (string, error) { + // Decode payload + var s ticketvote.Start + err := json.Unmarshal([]byte(payload), &s) + if err != nil { + return "", err } - // Verify bit is included in vote options - for _, v := range options { - if v.Bit == bit { - // Bit matches one of the options. We're done. - return nil + // Parse vote type + if len(s.Starts) == 0 { + return "", backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeStartDetailsMissing), + ErrorContext: "no start details found", } } + vtype := s.Starts[0].Params.Type - return fmt.Errorf("bit 0x%x not found in vote options", bit) -} - -func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationMax uint32) error { - // Verify vote type - switch vote.Type { + // Start vote + var sr *ticketvote.StartReply + switch vtype { case ticketvote.VoteTypeStandard: - // This is allowed + sr, err = p.startStandard(treeID, token, s) + if err != nil { + return "", err + } case ticketvote.VoteTypeRunoff: - // This is allowed + sr, err = p.startRunoff(treeID, token, s) + if err != nil { + return "", err + } default: - return backend.PluginError{ + return "", backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeVoteTypeInvalid), } } - // Verify vote params - switch { - case vote.Duration > voteDurationMax: - return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteDurationInvalid), - ErrorContext: fmt.Sprintf("duration %v exceeds max duration %v", - vote.Duration, voteDurationMax), - } - case vote.Duration < voteDurationMin: - return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteDurationInvalid), - ErrorContext: fmt.Sprintf("duration %v under min duration %v", - vote.Duration, voteDurationMin), - } - case vote.QuorumPercentage > 100: - return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteQuorumInvalid), - ErrorContext: fmt.Sprintf("quorum percent %v exceeds 100 percent", - vote.QuorumPercentage), - } - case vote.PassPercentage > 100: - return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVotePassRateInvalid), - ErrorContext: fmt.Sprintf("pass percent %v exceeds 100 percent", - vote.PassPercentage), - } + // Prepare reply + reply, err := json.Marshal(*sr) + if err != nil { + return "", err } - // Verify vote options. Different vote types have different - // requirements. - if len(vote.Options) == 0 { - return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteOptionsInvalid), - ErrorContext: "no vote options found", - } + return string(reply), nil +} + +// voteMessageVerify verifies a cast vote message is properly signed. Copied +// from: +// github.com/decred/dcrd/blob/0fc55252f912756c23e641839b1001c21442c38a/rpcserver.go#L5605 +func (p *ticketVotePlugin) voteMessageVerify(address, message, signature string) (bool, error) { + // Decode the provided address. + addr, err := dcrutil.DecodeAddress(address, p.activeNetParams) + if err != nil { + return false, fmt.Errorf("Could not decode address: %v", + err) } - switch vote.Type { - case ticketvote.VoteTypeStandard, ticketvote.VoteTypeRunoff: - // These vote types only allow for approve/reject votes. Ensure - // that the only options present are approve/reject and that they - // use the vote option IDs specified by the ticketvote API. - if len(vote.Options) != 2 { - return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteOptionsInvalid), - ErrorContext: fmt.Sprintf("vote options count got %v, want 2", - len(vote.Options)), - } - } - // map[optionID]found - options := map[string]bool{ - ticketvote.VoteOptionIDApprove: false, - ticketvote.VoteOptionIDReject: false, - } - for _, v := range vote.Options { - switch v.ID { - case ticketvote.VoteOptionIDApprove: - options[v.ID] = true - case ticketvote.VoteOptionIDReject: - options[v.ID] = true - } - } - missing := make([]string, 0, 2) - for k, v := range options { - if !v { - // Option ID was not found - missing = append(missing, k) - } - } - if len(missing) > 0 { - return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteOptionsInvalid), - ErrorContext: fmt.Sprintf("vote option IDs not found: %v", - strings.Join(missing, ",")), - } - } + + // Only P2PKH addresses are valid for signing. + if _, ok := addr.(*dcrutil.AddressPubKeyHash); !ok { + return false, fmt.Errorf("Address is not a pay-to-pubkey-hash "+ + "address: %v", address) } - // Verify vote bits are somewhat sane - for _, v := range vote.Options { - err := voteBitVerify(vote.Options, vote.Mask, v.Bit) - if err != nil { - return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteBitsInvalid), - ErrorContext: err.Error(), - } - } + // Decode base64 signature. + sig, err := base64.StdEncoding.DecodeString(signature) + if err != nil { + return false, fmt.Errorf("Malformed base64 encoding: %v", err) } - // Verify parent token - switch { - case vote.Type == ticketvote.VoteTypeStandard && vote.Parent != "": - return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), - ErrorContext: "parent token should not be provided for a standard vote", - } - case vote.Type == ticketvote.VoteTypeRunoff: - _, err := tokenDecode(vote.Parent) - if err != nil { - return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), - ErrorContext: fmt.Sprintf("invalid parent %v", vote.Parent), - } - } + // Validate the signature - this just shows that it was valid at all. + // we will compare it with the key next. + var buf bytes.Buffer + wire.WriteVarString(&buf, 0, "Decred Signed Message:\n") + wire.WriteVarString(&buf, 0, message) + expectedMessageHash := chainhash.HashB(buf.Bytes()) + pk, wasCompressed, err := ecdsa.RecoverCompact(sig, + expectedMessageHash) + if err != nil { + // Mirror Bitcoin Core behavior, which treats error in + // RecoverCompact as invalid signature. + return false, nil } - return nil + // Reconstruct the pubkey hash. + dcrPK := pk + var serializedPK []byte + if wasCompressed { + serializedPK = dcrPK.SerializeCompressed() + } else { + serializedPK = dcrPK.SerializeUncompressed() + } + a, err := dcrutil.NewAddressSecpPubKey(serializedPK, p.activeNetParams) + if err != nil { + // Again mirror Bitcoin Core behavior, which treats error in + // public key reconstruction as invalid signature. + return false, nil + } + + // Return boolean if addresses match. + return a.Address() == address, nil } -// startStandard starts a standard vote. -func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvote.Start) (*ticketvote.StartReply, error) { - // Verify there is only one start details - if len(s.Starts) != 1 { - return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeStartDetailsInvalid), - ErrorContext: "more than one start details found for standard vote", - } +func (p *ticketVotePlugin) castVoteSignatureVerify(cv ticketvote.CastVote, addr string) error { + msg := cv.Token + cv.Ticket + cv.VoteBit + + // Convert hex signature to base64. The voteMessageVerify function + // expects base64. + b, err := hex.DecodeString(cv.Signature) + if err != nil { + return fmt.Errorf("invalid hex") } - sd := s.Starts[0] + sig := base64.StdEncoding.EncodeToString(b) - // Verify token - t, err := tokenDecode(sd.Params.Token) + // Verify message + validated, err := p.voteMessageVerify(addr, msg, sig) if err != nil { - return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeTokenInvalid), - } + return err } - if !bytes.Equal(t, token) { - return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeTokenInvalid), - ErrorContext: fmt.Sprintf("plugin payload token does not match "+ - "route token: got %x, want %x", t, token), - } + if !validated { + return fmt.Errorf("could not verify message") } - // Verify signature - vb, err := json.Marshal(sd.Params) + return nil +} + +// commitmentAddr represents the largest commitment address for a dcr ticket. +type commitmentAddr struct { + addr string // Commitment address + err error // Error if one occurred +} + +// largestCommitmentAddrs retrieves the largest commitment addresses for each +// of the provided tickets from dcrdata. A map[ticket]commitmentAddr is +// returned. +func (p *ticketVotePlugin) largestCommitmentAddrs(tickets []string) (map[string]commitmentAddr, error) { + // Get tx details + tt := dcrdata.TxsTrimmed{ + TxIDs: tickets, + } + payload, err := json.Marshal(tt) if err != nil { return nil, err } - msg := hex.EncodeToString(util.Digest(vb)) - err = util.VerifySignature(sd.Signature, sd.PublicKey, msg) + reply, err := p.backend.PluginRead(nil, dcrdata.PluginID, + dcrdata.CmdTxsTrimmed, string(payload)) if err != nil { - return nil, convertSignatureError(err) + return nil, fmt.Errorf("PluginRead %v %v: %v", + dcrdata.PluginID, dcrdata.CmdTxsTrimmed, err) } - - // Verify vote options and params - err = voteParamsVerify(sd.Params, p.voteDurationMin, p.voteDurationMax) + var ttr dcrdata.TxsTrimmedReply + err = json.Unmarshal([]byte(reply), &ttr) if err != nil { return nil, err } - // Get vote blockchain data - sr, err := p.startReply(sd.Params.Duration) - if err != nil { - return nil, err - } - - // Verify record version - r, err := p.tstore.RecordPartial(treeID, 0, nil, true) - if err != nil { - return nil, fmt.Errorf("RecordPartial: %v", err) - } - if r.RecordMetadata.State != backend.StateVetted { - // This should not be possible - return nil, fmt.Errorf("record is unvetted") - } - if sd.Params.Version != r.RecordMetadata.Version { - return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeRecordVersionInvalid), - ErrorContext: fmt.Sprintf("version is not latest: got %v, want %v", - sd.Params.Version, r.RecordMetadata.Version), - } - } - - // Verify vote authorization - auths, err := p.auths(treeID) - if err != nil { - return nil, err - } - if len(auths) == 0 { - return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteStatusInvalid), - ErrorContext: "not authorized", + // Find the largest commitment address for each tx + addrs := make(map[string]commitmentAddr, len(ttr.Txs)) + for _, tx := range ttr.Txs { + var ( + bestAddr string // Addr with largest commitment amount + bestAmt float64 // Largest commitment amount + addrErr error // Error if one is encountered + ) + for _, vout := range tx.Vout { + scriptPubKey := vout.ScriptPubKeyDecoded + switch { + case scriptPubKey.CommitAmt == nil: + // No commitment amount; continue + case len(scriptPubKey.Addresses) == 0: + // No commitment address; continue + case *scriptPubKey.CommitAmt > bestAmt: + // New largest commitment address found + bestAddr = scriptPubKey.Addresses[0] + bestAmt = *scriptPubKey.CommitAmt + } } - } - action := ticketvote.AuthActionT(auths[len(auths)-1].Action) - if action != ticketvote.AuthActionAuthorize { - return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteStatusInvalid), - ErrorContext: "not authorized", + if bestAddr == "" || bestAmt == 0.0 { + addrErr = fmt.Errorf("no largest commitment address found") } - } - // Verify vote has not already been started - svp, err := p.voteDetails(treeID) - if err != nil { - return nil, err - } - if svp != nil { - // Vote has already been started - return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteStatusInvalid), - ErrorContext: "vote already started", + // Store result + addrs[tx.TxID] = commitmentAddr{ + addr: bestAddr, + err: addrErr, } } - // Prepare vote details - vd := ticketvote.VoteDetails{ - Params: sd.Params, - PublicKey: sd.PublicKey, - Signature: sd.Signature, - StartBlockHeight: sr.StartBlockHeight, - StartBlockHash: sr.StartBlockHash, - EndBlockHeight: sr.EndBlockHeight, - EligibleTickets: sr.EligibleTickets, - } - - // Save vote details - err = p.voteDetailsSave(treeID, vd) - if err != nil { - return nil, fmt.Errorf("voteDetailsSave: %v", err) - } - - // Update inventory - p.inventoryUpdateToStarted(vd.Params.Token, ticketvote.VoteStatusStarted, - vd.EndBlockHeight) - - // Update active votes cache - p.activeVotesAdd(vd) - - return &ticketvote.StartReply{ - StartBlockHeight: sr.StartBlockHeight, - StartBlockHash: sr.StartBlockHash, - EndBlockHeight: sr.EndBlockHeight, - EligibleTickets: sr.EligibleTickets, - }, nil + return addrs, nil } -// startRunoffRecord is the record that is saved to the runoff vote's parent -// tree as the first step in starting a runoff vote. Plugins are not able to -// update multiple records atomically, so if this call gets interrupted before -// if can start the voting period on all runoff vote submissions, subsequent -// calls will use this record to pick up where the previous call left off. This -// allows us to recover from unexpected errors, such as network errors, and not -// leave a runoff vote in a weird state. -type startRunoffRecord struct { - Submissions []string `json:"submissions"` - Mask uint64 `json:"mask"` - Duration uint32 `json:"duration"` - QuorumPercentage uint32 `json:"quorumpercentage"` - PassPercentage uint32 `json:"passpercentage"` - StartBlockHeight uint32 `json:"startblockheight"` - StartBlockHash string `json:"startblockhash"` - EndBlockHeight uint32 `json:"endblockheight"` - EligibleTickets []string `json:"eligibletickets"` +// voteCollider is used to prevent duplicate votes at the tlog level. The +// backend saves a digest of the data to the trillian log (tlog). Tlog does not +// allow leaves with duplicate values, so once a vote colider is saved to the +// backend for a ticket it should be impossible for another vote collider to be +// saved to the backend that is voting with the same ticket on the same record, +// regardless of what the vote bits are. The vote collider and the full cast +// vote are saved to the backend at the same time. A cast vote is not +// considered valid unless a corresponding vote collider is present. +type voteCollider struct { + Token string `json:"token"` // Record token + Ticket string `json:"ticket"` // Ticket hash } -func (p *ticketVotePlugin) startRunoffRecordSave(treeID int64, srr startRunoffRecord) error { - be, err := convertBlobEntryFromStartRunoff(srr) +// voteColliderSave saves a voteCollider to the backend. +func (p *ticketVotePlugin) voteColliderSave(treeID int64, vc voteCollider) error { + // Prepare blob + be, err := convertBlobEntryFromVoteCollider(vc) if err != nil { return err } - err = p.tstore.BlobSave(treeID, *be) - if err != nil { - return fmt.Errorf("BlobSave %v %v: %v", - treeID, dataDescriptorStartRunoff, err) - } - return nil + + // Save blob + return p.tstore.BlobSave(treeID, *be) } -// startRunoffRecord returns the startRunoff record if one exists on a tree. -// nil will be returned if a startRunoff record is not found. -func (p *ticketVotePlugin) startRunoffRecord(treeID int64) (*startRunoffRecord, error) { - blobs, err := p.tstore.BlobsByDataDesc(treeID, - []string{dataDescriptorStartRunoff}) +// castVoteSave saves a CastVoteDetails to the backend. +func (p *ticketVotePlugin) castVoteSave(treeID int64, cv ticketvote.CastVoteDetails) error { + // Prepare blob + be, err := convertBlobEntryFromCastVoteDetails(cv) if err != nil { - return nil, fmt.Errorf("BlobsByDataDesc %v %v: %v", - treeID, dataDescriptorStartRunoff, err) - } - - var srr *startRunoffRecord - switch len(blobs) { - case 0: - // Nothing found - return nil, nil - case 1: - // A start runoff record was found - srr, err = convertStartRunoffFromBlobEntry(blobs[0]) - if err != nil { - return nil, err - } - default: - // This should not be possible - e := fmt.Sprintf("%v start runoff blobs found", len(blobs)) - panic(e) + return err } - return srr, nil + // Save blob + return p.tstore.BlobSave(treeID, *be) } -// runoffDetails is an internal plugin command that requests the details of a -// runoff vote. -type runoffDetails struct{} +// ballot casts the provided votes concurrently. The vote results are passed +// back through the results channel to the calling function. This function +// waits until all provided votes have been cast before returning. +func (p *ticketVotePlugin) ballot(treeID int64, votes []ticketvote.CastVote, results chan ticketvote.CastVoteReply) { + // Cast the votes concurrently + var wg sync.WaitGroup + for _, v := range votes { + // Increment the wait group counter + wg.Add(1) -// runoffDetailsReply is the reply to the runoffDetails command. -type runoffDetailsReply struct { - Runoff startRunoffRecord `json:"runoff"` -} + go func(v ticketvote.CastVote) { + // Decrement wait group counter once vote is cast + defer wg.Done() -func (p *ticketVotePlugin) cmdRunoffDetails(treeID int64) (string, error) { - // Get start runoff record - srs, err := p.startRunoffRecord(treeID) - if err != nil { - return "", err - } + // Setup cast vote details + receipt := p.identity.SignMessage([]byte(v.Signature)) + cv := ticketvote.CastVoteDetails{ + Token: v.Token, + Ticket: v.Ticket, + VoteBit: v.VoteBit, + Signature: v.Signature, + Receipt: hex.EncodeToString(receipt[:]), + } - // Prepare reply - r := runoffDetailsReply{ - Runoff: *srs, - } - reply, err := json.Marshal(r) - if err != nil { - return "", err + // Declare here to prevent goto errors + var ( + cvr ticketvote.CastVoteReply + vc voteCollider + ) + + // Save cast vote + err := p.castVoteSave(treeID, cv) + if err == plugins.ErrDuplicateBlob { + // This cast vote has already been saved. Its possible that + // a previous attempt to vote with this ticket failed before + // the vote collider could be saved. Continue execution so + // that we re-attempt to save the vote collider. + } else if err != nil { + t := time.Now().Unix() + log.Errorf("cmdCastBallot: castVoteSave %v: %v", t, err) + e := ticketvote.VoteErrorInternalError + cvr.Ticket = v.Ticket + cvr.ErrorCode = e + cvr.ErrorContext = fmt.Sprintf("%v: %v", + ticketvote.VoteErrors[e], t) + goto sendResult + } + + // Save vote collider + vc = voteCollider{ + Token: v.Token, + Ticket: v.Ticket, + } + err = p.voteColliderSave(treeID, vc) + if err != nil { + t := time.Now().Unix() + log.Errorf("cmdCastBallot: voteColliderSave %v: %v", t, err) + e := ticketvote.VoteErrorInternalError + cvr.Ticket = v.Ticket + cvr.ErrorCode = e + cvr.ErrorContext = fmt.Sprintf("%v: %v", + ticketvote.VoteErrors[e], t) + goto sendResult + } + + // Update receipt + cvr.Ticket = v.Ticket + cvr.Receipt = cv.Receipt + + // Update cast votes cache + p.activeVotes.AddCastVote(v.Token, v.Ticket, v.VoteBit) + + sendResult: + // Send result back to calling function + results <- cvr + }(v) } - return string(reply), nil + // Wait for the full ballot to be cast before returning. + wg.Wait() } -func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs startRunoffSubmission) error { - // Sanity check - sd := srs.StartDetails - t, err := tokenDecode(sd.Params.Token) +// cmdCastBallot casts a ballot of votes. This function will not return a user +// error if one occurs for an individual vote. It will instead return the +// ballot reply with the error included in the individual cast vote reply. +func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload string) (string, error) { + // Decode payload + var cb ticketvote.CastBallot + err := json.Unmarshal([]byte(payload), &cb) if err != nil { - return err + return "", err } - if !bytes.Equal(token, t) { - return fmt.Errorf("invalid token") + votes := cb.Ballot + + // Verify there is work to do + if len(votes) == 0 { + // Nothing to do + cbr := ticketvote.CastBallotReply{ + Receipts: []ticketvote.CastVoteReply{}, + } + reply, err := json.Marshal(cbr) + if err != nil { + return "", err + } + return string(reply), nil } - // Get the start runoff record from the parent tree - srr, err := p.startRunoffRecord(srs.ParentTreeID) + // Get the data that we need to validate the votes + voteDetails := p.activeVotes.VoteDetails(token) + eligible := p.activeVotes.EligibleTickets(token) + bestBlock, err := p.bestBlock() if err != nil { - return err + return "", err } - // Sanity check. Verify token is part of the start runoff record - // submissions. - var found bool - for _, v := range srr.Submissions { - if hex.EncodeToString(token) == v { - found = true - break + // Perform all validation that does not require fetching the + // commitment addresses. + receipts := make([]ticketvote.CastVoteReply, len(votes)) + for k, v := range votes { + // Verify token is a valid token + t, err := tokenDecode(v.Token) + if err != nil { + e := ticketvote.VoteErrorTokenInvalid + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = fmt.Sprintf("%v: not hex", + ticketvote.VoteErrors[e]) + continue + } + + // Verify vote token and command token are the same + if !bytes.Equal(t, token) { + e := ticketvote.VoteErrorMultipleRecordVotes + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = ticketvote.VoteErrors[e] + continue + } + + // Verify vote is still active + if voteDetails == nil { + e := ticketvote.VoteErrorVoteStatusInvalid + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = fmt.Sprintf("%v: vote is not active", + ticketvote.VoteErrors[e]) + continue + } + if voteHasEnded(bestBlock, voteDetails.EndBlockHeight) { + e := ticketvote.VoteErrorVoteStatusInvalid + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = fmt.Sprintf("%v: vote has ended", + ticketvote.VoteErrors[e]) + continue + } + + // Verify vote bit + bit, err := strconv.ParseUint(v.VoteBit, 16, 64) + if err != nil { + e := ticketvote.VoteErrorVoteBitInvalid + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = ticketvote.VoteErrors[e] + continue + } + err = voteBitVerify(voteDetails.Params.Options, + voteDetails.Params.Mask, bit) + if err != nil { + e := ticketvote.VoteErrorVoteBitInvalid + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = fmt.Sprintf("%v: %v", + ticketvote.VoteErrors[e], err) + continue + } + + // Verify ticket is eligible to vote + _, ok := eligible[v.Ticket] + if !ok { + e := ticketvote.VoteErrorTicketNotEligible + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = ticketvote.VoteErrors[e] + continue + } + + // Verify ticket has not already voted + if p.activeVotes.VoteIsDuplicate(v.Token, v.Ticket) { + e := ticketvote.VoteErrorTicketAlreadyVoted + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = ticketvote.VoteErrors[e] + continue } - } - if !found { - // This submission should not be here - return fmt.Errorf("record not in submission list") } - // If the vote has already been started, exit gracefully. This - // allows us to recover from unexpected errors to the start runoff - // vote call as it updates the state of multiple records. If the - // call were to fail before completing, we can simply call the - // command again with the same arguments and it will pick up where - // it left off. - svp, err := p.voteDetails(treeID) - if err != nil { - return err + // Get the largest commitment address for each ticket and verify + // that the vote was signed using the private key from this + // address. We first check the active votes cache to see if the + // commitment addresses have already been fetched. Any tickets + // that are not found in the cache are fetched manually. + tickets := make([]string, 0, len(cb.Ballot)) + for k, v := range votes { + if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { + // Vote has an error. Skip it. + continue + } + tickets = append(tickets, v.Ticket) } - if svp != nil { - // Vote has already been started. Exit gracefully. - return nil + addrs := p.activeVotes.CommitmentAddrs(token, tickets) + notInCache := make([]string, 0, len(tickets)) + for _, v := range tickets { + _, ok := addrs[v] + if !ok { + notInCache = append(notInCache, v) + } } - // Verify record version - r, err := p.tstore.RecordPartial(treeID, 0, nil, true) - if err != nil { - return fmt.Errorf("RecordPartial: %v", err) + log.Debugf("%v/%v commitment addresses found in cache", + len(tickets)-len(notInCache), len(tickets)) + + if len(notInCache) > 0 { + // Get commitment addresses from dcrdata + caddrs, err := p.largestCommitmentAddrs(tickets) + if err != nil { + return "", fmt.Errorf("largestCommitmentAddrs: %v", err) + } + + // Add addresses to the existing map + for k, v := range caddrs { + addrs[k] = v + } } - if r.RecordMetadata.State != backend.StateVetted { - // This should not be possible - return fmt.Errorf("record is unvetted") + + // Verify the signatures + for k, v := range votes { + if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { + // Vote has an error. Skip it. + continue + } + + // Verify vote signature + commitmentAddr, ok := addrs[v.Ticket] + if !ok { + t := time.Now().Unix() + log.Errorf("cmdCastBallot: commitment addr not found %v: %v", + t, v.Ticket) + e := ticketvote.VoteErrorInternalError + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = fmt.Sprintf("%v: %v", + ticketvote.VoteErrors[e], t) + continue + } + if commitmentAddr.err != nil { + t := time.Now().Unix() + log.Errorf("cmdCastBallot: commitment addr error %v: %v %v", + t, v.Ticket, commitmentAddr.err) + e := ticketvote.VoteErrorInternalError + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = fmt.Sprintf("%v: %v", + ticketvote.VoteErrors[e], t) + continue + } + err = p.castVoteSignatureVerify(v, commitmentAddr.addr) + if err != nil { + e := ticketvote.VoteErrorSignatureInvalid + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = fmt.Sprintf("%v: %v", + ticketvote.VoteErrors[e], err) + continue + } } - if sd.Params.Version != r.RecordMetadata.Version { - return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeRecordVersionInvalid), - ErrorContext: fmt.Sprintf("version is not latest %v: got %v, want %v", - sd.Params.Token, sd.Params.Version, r.RecordMetadata.Version), + + // The votes that have passed validation will be cast in batches of + // size batchSize. Each batch of votes is cast concurrently in order + // to accommodate the trillian log signer bottleneck. The log signer + // picks up queued leaves and appends them onto the trillian tree + // every xxx ms, where xxx is a configurable value on the log signer, + // but is typically a few hundred milliseconds. Lets use 200ms as an + // example. If we don't cast the votes in batches then every vote in + // the ballot will take 200 milliseconds since we wait for the leaf + // to be fully appended before considering the trillian call + // successful. A person casting hundreds of votes in a single ballot + // would cause UX issues for all the voting clients since the backend + // locks the record during any plugin write calls. Only one ballot + // can be cast at a time. + // + // The second variable that we must watch out for is the max trillian + // queued leaf batch size. This is also a configurable trillian value + // that represents the maximum number of leaves that can be waiting + // in the queue for all trees in the trillian instance. This value is + // typically around the order of magnitude of 1000s of queued leaves. + // + // The third variable that can cause errors is reaching the trillian + // datastore max connection limits. Each vote being cast creates a + // trillian connection. Overloading the trillian connections can + // cause max connection exceeded errors. The max allowed connections + // is a configurable trillian value, but should also be adjusted on + // the key-value store database itself as well. + // + // This is why a vote batch size of 10 was chosen. It is large enough + // to alleviate performance bottlenecks from the log signer interval, + // but small enough to still allow multiple records votes to be held + // concurrently without running into the queued leaf batch size limit. + + // Prepare work + var ( + batchSize = 10 + batch = make([]ticketvote.CastVote, 0, batchSize) + queue = make([][]ticketvote.CastVote, 0, len(votes)/batchSize) + + // ballotCount is the number of votes that have passed validation + // and are being cast in this ballot. + ballotCount int + ) + for k, v := range votes { + if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { + // Vote has an error. Skip it. + continue + } + + // Add vote to the current batch + batch = append(batch, v) + ballotCount++ + + if len(batch) == batchSize { + // This batch is full. Add the batch to the queue and start + // a new batch. + queue = append(queue, batch) + batch = make([]ticketvote.CastVote, 0, batchSize) } } + if len(batch) != 0 { + // Add leftover batch to the queue + queue = append(queue, batch) + } - // Prepare vote details - vd := ticketvote.VoteDetails{ - Params: sd.Params, - PublicKey: sd.PublicKey, - Signature: sd.Signature, - StartBlockHeight: srr.StartBlockHeight, - StartBlockHash: srr.StartBlockHash, - EndBlockHeight: srr.EndBlockHeight, - EligibleTickets: srr.EligibleTickets, + log.Debugf("Casting %v votes in %v batches of size %v", + ballotCount, len(queue), batchSize) + + // Cast ballot in batches + results := make(chan ticketvote.CastVoteReply, ballotCount) + for i, batch := range queue { + log.Debugf("Casting %v votes in batch %v/%v", len(batch), i+1, len(queue)) + + p.ballot(treeID, batch, results) } - // Save vote details - err = p.voteDetailsSave(treeID, vd) - if err != nil { - return fmt.Errorf("voteDetailsSave: %v", err) + // Empty out the results channel + r := make(map[string]ticketvote.CastVoteReply, ballotCount) + close(results) + for v := range results { + r[v.Ticket] = v } - // Update inventory - p.inventoryUpdateToStarted(vd.Params.Token, ticketvote.VoteStatusStarted, - vd.EndBlockHeight) + if len(r) != ballotCount { + log.Errorf("Missing results: got %v, want %v", len(r), ballotCount) + } - // Update active votes cache - p.activeVotesAdd(vd) + // Fill in the receipts + for k, v := range votes { + if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { + // Vote has an error. Skip it. + continue + } + cvr, ok := r[v.Ticket] + if !ok { + t := time.Now().Unix() + log.Errorf("cmdCastBallot: vote result not found %v: %v", t, v.Ticket) + e := ticketvote.VoteErrorInternalError + receipts[k].Ticket = v.Ticket + receipts[k].ErrorCode = e + receipts[k].ErrorContext = fmt.Sprintf("%v: %v", + ticketvote.VoteErrors[e], t) + continue + } - return nil -} + // Fill in receipt + receipts[k] = cvr + } -func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ticketvote.Start) (*startRunoffRecord, error) { - // Check if the runoff vote data already exists on the parent tree. - srr, err := p.startRunoffRecord(treeID) + // Prepare reply + cbr := ticketvote.CastBallotReply{ + Receipts: receipts, + } + reply, err := json.Marshal(cbr) if err != nil { - return nil, err + return "", err } - if srr != nil { - // We already have a start runoff record for this runoff vote. - // This can happen if the previous call failed due to an - // unexpected error such as a network error. Return the start - // runoff record so we can pick up where we left off. - return srr, nil + + return string(reply), nil +} + +// cmdDetails returns the vote details for a record. +func (p *ticketVotePlugin) cmdDetails(treeID int64, token []byte) (string, error) { + // Get vote authorizations + auths, err := p.auths(treeID) + if err != nil { + return "", fmt.Errorf("auths: %v", err) } - // Get blockchain data - var ( - mask = s.Starts[0].Params.Mask - duration = s.Starts[0].Params.Duration - quorum = s.Starts[0].Params.QuorumPercentage - pass = s.Starts[0].Params.PassPercentage - ) - sr, err := p.startReply(duration) + // Get vote details + vd, err := p.voteDetails(treeID) if err != nil { - return nil, err + return "", fmt.Errorf("voteDetails: %v", err) } - // Verify parent has a LinkBy and the LinkBy deadline is expired. - files := []string{ - ticketvote.FileNameVoteMetadata, + // Prepare rely + dr := ticketvote.DetailsReply{ + Auths: auths, + Vote: vd, } - r, err := p.tstore.RecordPartial(treeID, 0, files, false) + reply, err := json.Marshal(dr) if err != nil { - if errors.Is(err, backend.ErrRecordNotFound) { - return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), - ErrorContext: fmt.Sprintf("parent record not found %x", token), - } - } - return nil, fmt.Errorf("RecordPartial: %v", err) - } - if r.RecordMetadata.State != backend.StateVetted { - // This should not be possible - return nil, fmt.Errorf("record is unvetted") + return "", err } - vm, err := voteMetadataDecode(r.Files) + + return string(reply), nil +} + +// cmdRunoffDetails is an internal plugin command that requests the details of +// a runoff vote. +func (p *ticketVotePlugin) cmdRunoffDetails(treeID int64) (string, error) { + // Get start runoff record + srs, err := p.startRunoffRecord(treeID) if err != nil { - return nil, err + return "", err } - if vm == nil || vm.LinkBy == 0 { - return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), - ErrorContext: fmt.Sprintf("%x is not a runoff vote parent", token), - } + + // Prepare reply + r := runoffDetailsReply{ + Runoff: *srs, } - isExpired := vm.LinkBy < time.Now().Unix() - isMainNet := p.activeNetParams.Name == chaincfg.MainNetParams().Name - switch { - case !isExpired && isMainNet: - return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeLinkByNotExpired), - ErrorContext: fmt.Sprintf("parent record %x linkby deadline not met %v", - token, vm.LinkBy), - } - case !isExpired: - // Allow the vote to be started before the link by deadline - // expires on testnet and simnet only. This makes testing the - // runoff vote process easier. - log.Warnf("Parent record linkby deadline has not been met; " + - "disregarding deadline since this is not mainnet") + reply, err := json.Marshal(r) + if err != nil { + return "", err + } + + return string(reply), nil +} + +// cmdResults requests the vote objects of all votes that were cast in a ticket +// vote. +func (p *ticketVotePlugin) cmdResults(treeID int64, token []byte) (string, error) { + // Get vote results + votes, err := p.voteResults(treeID) + if err != nil { + return "", err } - // Compile a list of the expected submissions that should be in the - // runoff vote. This will be all of the public records that have - // linked to the parent record. The parent record's submissions - // list will include abandoned proposals that need to be filtered - // out. - lf, err := p.submissionsCache(token) + // Prepare reply + rr := ticketvote.ResultsReply{ + Votes: votes, + } + reply, err := json.Marshal(rr) if err != nil { - return nil, err + return "", err } - expected := make(map[string]struct{}, len(lf.Tokens)) // [token]struct{} - for k := range lf.Tokens { - token, err := tokenDecode(k) - if err != nil { - return nil, err - } - r, err := p.recordAbridged(token) - if err != nil { - return nil, err - } - if r.RecordMetadata.Status != backend.StatusPublic { - // This record is not public and should not be included - // in the runoff vote. - continue - } - // This is a public record that is part of the parent record's - // submissions list. It is required to be in the runoff vote. - expected[k] = struct{}{} + return string(reply), nil +} + +// cmdSummary requests the vote summary for a record. +func (p *ticketVotePlugin) cmdSummary(treeID int64, token []byte) (string, error) { + // Get best block. This cmd does not write any data so we do not + // have to use the safe best block. + bb, err := p.bestBlockUnsafe() + if err != nil { + return "", fmt.Errorf("bestBlockUnsafe: %v", err) } - // Verify that there are no extra submissions in the runoff vote - for _, v := range s.Starts { - _, ok := expected[v.Params.Token] - if !ok { - // This submission should not be here - return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeStartDetailsInvalid), - ErrorContext: fmt.Sprintf("record %v should not be included", - v.Params.Token), - } - } + // Get summary + sr, err := p.summary(treeID, token, bb) + if err != nil { + return "", fmt.Errorf("summary: %v", err) } - // Verify that the runoff vote is not missing any submissions - subs := make(map[string]struct{}, len(s.Starts)) - for _, v := range s.Starts { - subs[v.Params.Token] = struct{}{} + // Prepare reply + reply, err := json.Marshal(sr) + if err != nil { + return "", err } - for k := range expected { - _, ok := subs[k] - if !ok { - // This records is missing from the runoff vote - return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeStartDetailsMissing), - ErrorContext: k, - } - } + + return string(reply), nil +} + +// cmdInventory requests a page of tokens for the provided status. If no status +// is provided then a page for each status will be returned. +func (p *ticketVotePlugin) cmdInventory(payload string) (string, error) { + var i ticketvote.Inventory + err := json.Unmarshal([]byte(payload), &i) + if err != nil { + return "", err } - // Prepare start runoff record - submissions := make([]string, 0, len(subs)) - for k := range subs { - submissions = append(submissions, k) + // Get best block. This command does not write any data so we can + // use the unsafe best block. + bb, err := p.bestBlockUnsafe() + if err != nil { + return "", fmt.Errorf("bestBlockUnsafe: %v", err) } - srr = &startRunoffRecord{ - Submissions: submissions, - Mask: mask, - Duration: duration, - QuorumPercentage: quorum, - PassPercentage: pass, - StartBlockHeight: sr.StartBlockHeight, - StartBlockHash: sr.StartBlockHash, - EndBlockHeight: sr.EndBlockHeight, - EligibleTickets: sr.EligibleTickets, + + // Get the inventory + ibs, err := p.inventoryByStatus(bb, i.Status, i.Page) + if err != nil { + return "", fmt.Errorf("invByStatus: %v", err) } - // Save start runoff record - err = p.startRunoffRecordSave(treeID, *srr) + // Prepare reply + tokens := make(map[string][]string, len(ibs.Tokens)) + for k, v := range ibs.Tokens { + vs := ticketvote.VoteStatuses[k] + tokens[vs] = v + } + ir := ticketvote.InventoryReply{ + Tokens: tokens, + BestBlock: ibs.BestBlock, + } + reply, err := json.Marshal(ir) if err != nil { - return nil, fmt.Errorf("startRunoffRecordSave %v: %v", - treeID, err) + return "", err } - return srr, nil + return string(reply), nil } -// startRunoff starts a runoff vote. -func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote.Start) (*ticketvote.StartReply, error) { - // Sanity check - if len(s.Starts) == 0 { - return nil, fmt.Errorf("no start details found") +// cmdTimestamps requests the timestamps for a ticket vote. +func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload string) (string, error) { + // Decode payload + var t ticketvote.Timestamps + err := json.Unmarshal([]byte(payload), &t) + if err != nil { + return "", err } - // Perform validation that can be done without fetching any records - // from the backend. var ( - mask = s.Starts[0].Params.Mask - duration = s.Starts[0].Params.Duration - quorum = s.Starts[0].Params.QuorumPercentage - pass = s.Starts[0].Params.PassPercentage - parent = s.Starts[0].Params.Parent - ) - for _, v := range s.Starts { - // Verify vote params are the same for all submissions - switch { - case v.Params.Type != ticketvote.VoteTypeRunoff: - return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteTypeInvalid), - ErrorContext: fmt.Sprintf("%v got %v, want %v", - v.Params.Token, v.Params.Type, ticketvote.VoteTypeRunoff), - } - case v.Params.Mask != mask: - return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteBitsInvalid), - ErrorContext: fmt.Sprintf("%v mask invalid: all must be the same", - v.Params.Token), - } - case v.Params.Duration != duration: - return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteDurationInvalid), - ErrorContext: fmt.Sprintf("%v duration does not match; "+ - "all must be the same", v.Params.Token), - } - case v.Params.QuorumPercentage != quorum: - return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteQuorumInvalid), - ErrorContext: fmt.Sprintf("%v quorum does not match; "+ - "all must be the same", v.Params.Token), - } - case v.Params.PassPercentage != pass: - return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVotePassRateInvalid), - ErrorContext: fmt.Sprintf("%v pass rate does not match; "+ - "all must be the same", v.Params.Token), - } - case v.Params.Parent != parent: - return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), - ErrorContext: fmt.Sprintf("%v parent does not match; "+ - "all must be the same", v.Params.Token), - } - } + auths = make([]ticketvote.Timestamp, 0, 32) + details *ticketvote.Timestamp - // Verify token - _, err := tokenDecode(v.Params.Token) + pageSize = ticketvote.VoteTimestampsPageSize + votes = make([]ticketvote.Timestamp, 0, pageSize) + ) + switch { + case t.VotesPage > 0: + // Return a page of vote timestamps + digests, err := p.tstore.DigestsByDataDesc(treeID, + []string{dataDescriptorCastVoteDetails}) if err != nil { - return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeTokenInvalid), - ErrorContext: v.Params.Token, - } + return "", fmt.Errorf("digestsByKeyPrefix %v %v: %v", + treeID, dataDescriptorVoteDetails, err) } - // Verify parent token - _, err = tokenDecode(v.Params.Parent) - if err != nil { - return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeTokenInvalid), - ErrorContext: fmt.Sprintf("parent token %v", v.Params.Parent), + startAt := (t.VotesPage - 1) * pageSize + for i, v := range digests { + if i < int(startAt) { + continue + } + ts, err := p.timestamp(treeID, v) + if err != nil { + return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) + } + votes = append(votes, *ts) + if len(votes) == int(pageSize) { + // We have a full page. We're done. + break } } - // Verify signature - vb, err := json.Marshal(v.Params) - if err != nil { - return nil, err - } - msg := hex.EncodeToString(util.Digest(vb)) - err = util.VerifySignature(v.Signature, v.PublicKey, msg) - if err != nil { - return nil, convertSignatureError(err) - } - - // Verify vote options and params. Vote optoins are required to - // be approve and reject. - err = voteParamsVerify(v.Params, p.voteDurationMin, p.voteDurationMax) - if err != nil { - return nil, err - } - } - - // Verify plugin command is being executed on the parent record - if hex.EncodeToString(token) != parent { - return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), - ErrorContext: fmt.Sprintf("runoff vote must be started on "+ - "the parent record %v", parent), - } - } - - // This function is being invoked on the runoff vote parent record. - // Create and save a start runoff record onto the parent record's - // tree. - srr, err := p.startRunoffForParent(treeID, token, s) - if err != nil { - return nil, err - } + default: + // Return authorization timestamps and the vote details timestamp. - // Start the voting period of each runoff vote submissions by - // using the internal plugin command startRunoffSubmission. - for _, v := range s.Starts { - token, err = tokenDecode(v.Params.Token) + // Auth timestamps + digests, err := p.tstore.DigestsByDataDesc(treeID, + []string{dataDescriptorAuthDetails}) if err != nil { - return nil, err + return "", fmt.Errorf("DigestByDataDesc %v %v: %v", + treeID, dataDescriptorAuthDetails, err) } - srs := startRunoffSubmission{ - ParentTreeID: treeID, - StartDetails: v, + auths = make([]ticketvote.Timestamp, 0, len(digests)) + for _, v := range digests { + ts, err := p.timestamp(treeID, v) + if err != nil { + return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) + } + auths = append(auths, *ts) } - b, err := json.Marshal(srs) + + // Vote details timestamp + digests, err = p.tstore.DigestsByDataDesc(treeID, + []string{dataDescriptorVoteDetails}) if err != nil { - return nil, err + return "", fmt.Errorf("DigestsByDataDesc %v %v: %v", + treeID, dataDescriptorVoteDetails, err) } - _, err = p.backend.PluginWrite(token, ticketvote.PluginID, - cmdStartRunoffSubmission, string(b)) - if err != nil { - var ue backend.PluginError - if errors.As(err, &ue) { - return nil, err + // There should never be more than a one vote details + if len(digests) > 1 { + return "", fmt.Errorf("invalid vote details count: got %v, want 1", + len(digests)) + } + for _, v := range digests { + ts, err := p.timestamp(treeID, v) + if err != nil { + return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) } - return nil, fmt.Errorf("PluginWrite %x %v %v: %v", - token, ticketvote.PluginID, cmdStartRunoffSubmission, err) + details = ts } } - return &ticketvote.StartReply{ - StartBlockHeight: srr.StartBlockHeight, - StartBlockHash: srr.StartBlockHash, - EndBlockHeight: srr.EndBlockHeight, - EligibleTickets: srr.EligibleTickets, - }, nil -} + // Prepare reply + tr := ticketvote.TimestampsReply{ + Auths: auths, + Details: details, + Votes: votes, + } + reply, err := json.Marshal(tr) + if err != nil { + return "", err + } -// startRunoffSubmission is an internal plugin command that is used to start -// the voting period on a runoff vote submission. -type startRunoffSubmission struct { - ParentTreeID int64 `json:"parenttreeid"` - StartDetails ticketvote.StartDetails `json:"startdetails"` + return string(reply), nil } -func (p *ticketVotePlugin) cmdStartRunoffSubmission(treeID int64, token []byte, payload string) (string, error) { - // Decode payload - var srs startRunoffSubmission - err := json.Unmarshal([]byte(payload), &srs) +// Submissions requests the submissions of a runoff vote. The only records that +// will have a submissions list are the parent records in a runoff vote. The +// list will contain all public runoff vote submissions, i.e. records that have +// linked to the parent record using the VoteMetadata.LinkTo field. +func (p *ticketVotePlugin) cmdSubmissions(token []byte) (string, error) { + // Get submissions list + lf, err := p.submissionsCache(token) if err != nil { return "", err } - // Start voting period on runoff vote submission - err = p.startRunoffForSub(treeID, token, srs) + // Prepare reply + tokens := make([]string, 0, len(lf.Tokens)) + for k := range lf.Tokens { + tokens = append(tokens, k) + } + lfr := ticketvote.SubmissionsReply{ + Submissions: tokens, + } + reply, err := json.Marshal(lfr) if err != nil { return "", err } - return "", nil + return string(reply), nil } -func (p *ticketVotePlugin) cmdStart(treeID int64, token []byte, payload string) (string, error) { - // Decode payload - var s ticketvote.Start - err := json.Unmarshal([]byte(payload), &s) +// authSave saves a AuthDetails to the backend. +func (p *ticketVotePlugin) authSave(treeID int64, ad ticketvote.AuthDetails) error { + // Prepare blob + be, err := convertBlobEntryFromAuthDetails(ad) if err != nil { - return "", err + return err } - // Parse vote type - if len(s.Starts) == 0 { - return "", backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeStartDetailsMissing), - ErrorContext: "no start details found", - } + // Save blob + return p.tstore.BlobSave(treeID, *be) +} + +// auths returns all AuthDetails for a record. +func (p *ticketVotePlugin) auths(treeID int64) ([]ticketvote.AuthDetails, error) { + // Retrieve blobs + blobs, err := p.tstore.BlobsByDataDesc(treeID, + []string{dataDescriptorAuthDetails}) + if err != nil { + return nil, err } - vtype := s.Starts[0].Params.Type - // Start vote - var sr *ticketvote.StartReply - switch vtype { - case ticketvote.VoteTypeStandard: - sr, err = p.startStandard(treeID, token, s) - if err != nil { - return "", err - } - case ticketvote.VoteTypeRunoff: - sr, err = p.startRunoff(treeID, token, s) + // Decode blobs + auths := make([]ticketvote.AuthDetails, 0, len(blobs)) + for _, v := range blobs { + a, err := convertAuthDetailsFromBlobEntry(v) if err != nil { - return "", err - } - default: - return "", backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteTypeInvalid), + return nil, err } + auths = append(auths, *a) } - // Prepare reply - reply, err := json.Marshal(*sr) + // Sanity check. They should already be sorted from oldest to + // newest. + sort.SliceStable(auths, func(i, j int) bool { + return auths[i].Timestamp < auths[j].Timestamp + }) + + return auths, nil +} + +// voteDetailsSave saves a VoteDetails to the backend. +func (p *ticketVotePlugin) voteDetailsSave(treeID int64, vd ticketvote.VoteDetails) error { + // Prepare blob + be, err := convertBlobEntryFromVoteDetails(vd) if err != nil { - return "", err + return err } - return string(reply), nil + // Save blob + return p.tstore.BlobSave(treeID, *be) } -// voteMessageVerify verifies a cast vote message is properly signed. Copied -// from: github.com/decred/dcrd/blob/0fc55252f912756c23e641839b1001c21442c38a/rpcserver.go#L5605 -func (p *ticketVotePlugin) voteMessageVerify(address, message, signature string) (bool, error) { - // Decode the provided address. - addr, err := dcrutil.DecodeAddress(address, p.activeNetParams) +// voteDetails returns the VoteDetails for a record. Nil is returned if a vote +// details is not found. +func (p *ticketVotePlugin) voteDetails(treeID int64) (*ticketvote.VoteDetails, error) { + // Retrieve blobs + blobs, err := p.tstore.BlobsByDataDesc(treeID, + []string{dataDescriptorVoteDetails}) if err != nil { - return false, fmt.Errorf("Could not decode address: %v", - err) + return nil, err + } + switch len(blobs) { + case 0: + // A vote details does not exist + return nil, nil + case 1: + // A vote details exists; continue + default: + // This should not happen. There should only ever be a max of + // one vote details. + return nil, fmt.Errorf("multiple vote details found (%v) on %x", + len(blobs), treeID) } - // Only P2PKH addresses are valid for signing. - if _, ok := addr.(*dcrutil.AddressPubKeyHash); !ok { - return false, fmt.Errorf("Address is not a pay-to-pubkey-hash "+ - "address: %v", address) + // Decode blob + vd, err := convertVoteDetailsFromBlobEntry(blobs[0]) + if err != nil { + return nil, err } - // Decode base64 signature. - sig, err := base64.StdEncoding.DecodeString(signature) + return vd, nil +} + +// voteDetailsByToken returns the VoteDetails for a record. Nil is returned +// if the vote details are not found. +func (p *ticketVotePlugin) voteDetailsByToken(token []byte) (*ticketvote.VoteDetails, error) { + reply, err := p.backend.PluginRead(token, ticketvote.PluginID, + ticketvote.CmdDetails, "") if err != nil { - return false, fmt.Errorf("Malformed base64 encoding: %v", err) + return nil, err + } + var dr ticketvote.DetailsReply + err = json.Unmarshal([]byte(reply), &dr) + if err != nil { + return nil, err } + return dr.Vote, nil +} - // Validate the signature - this just shows that it was valid at all. - // we will compare it with the key next. - var buf bytes.Buffer - wire.WriteVarString(&buf, 0, "Decred Signed Message:\n") - wire.WriteVarString(&buf, 0, message) - expectedMessageHash := chainhash.HashB(buf.Bytes()) - pk, wasCompressed, err := ecdsa.RecoverCompact(sig, - expectedMessageHash) +// voteResults returns all votes that were cast in a ticket vote. +func (p *ticketVotePlugin) voteResults(treeID int64) ([]ticketvote.CastVoteDetails, error) { + // Retrieve blobs + desc := []string{ + dataDescriptorCastVoteDetails, + dataDescriptorVoteCollider, + } + blobs, err := p.tstore.BlobsByDataDesc(treeID, desc) if err != nil { - // Mirror Bitcoin Core behavior, which treats error in - // RecoverCompact as invalid signature. - return false, nil + return nil, err } - // Reconstruct the pubkey hash. - dcrPK := pk - var serializedPK []byte - if wasCompressed { - serializedPK = dcrPK.SerializeCompressed() - } else { - serializedPK = dcrPK.SerializeUncompressed() - } - a, err := dcrutil.NewAddressSecpPubKey(serializedPK, p.activeNetParams) - if err != nil { - // Again mirror Bitcoin Core behavior, which treats error in - // public key reconstruction as invalid signature. - return false, nil - } + // Decode blobs. A cast vote is considered valid only if the vote + // collider exists for it. If there are multiple votes using the + // same ticket, the valid vote is the one that immediately precedes + // the vote collider blob entry. + var ( + // map[ticket]CastVoteDetails + votes = make(map[string]ticketvote.CastVoteDetails, len(blobs)) + + voteIndexes = make(map[string][]int, len(blobs)) // map[ticket][]index + colliderIndexes = make(map[string]int, len(blobs)) // map[ticket]index + ) + for i, v := range blobs { + // Decode data hint + b, err := base64.StdEncoding.DecodeString(v.DataHint) + if err != nil { + return nil, err + } + var dd store.DataDescriptor + err = json.Unmarshal(b, &dd) + if err != nil { + return nil, err + } + switch dd.Descriptor { + case dataDescriptorCastVoteDetails: + // Decode cast vote + cv, err := convertCastVoteDetailsFromBlobEntry(v) + if err != nil { + return nil, err + } + + // Save index of the cast vote + idx, ok := voteIndexes[cv.Ticket] + if !ok { + idx = make([]int, 0, 32) + } + idx = append(idx, i) + voteIndexes[cv.Ticket] = idx - // Return boolean if addresses match. - return a.Address() == address, nil -} + // Save the cast vote + votes[cv.Ticket] = *cv -func (p *ticketVotePlugin) castVoteSignatureVerify(cv ticketvote.CastVote, addr string) error { - msg := cv.Token + cv.Ticket + cv.VoteBit + case dataDescriptorVoteCollider: + // Decode vote collider + vc, err := convertVoteColliderFromBlobEntry(v) + if err != nil { + return nil, err + } - // Convert hex signature to base64. The voteMessageVerify function - // expects base64. - b, err := hex.DecodeString(cv.Signature) - if err != nil { - return fmt.Errorf("invalid hex") - } - sig := base64.StdEncoding.EncodeToString(b) + // Sanity check + _, ok := colliderIndexes[vc.Ticket] + if ok { + return nil, fmt.Errorf("duplicate vote colliders found %v", vc.Ticket) + } - // Verify message - validated, err := p.voteMessageVerify(addr, msg, sig) - if err != nil { - return err - } - if !validated { - return fmt.Errorf("could not verify message") + // Save the ticket and index for the collider + colliderIndexes[vc.Ticket] = i + + default: + return nil, fmt.Errorf("invalid data descriptor: %v", dd.Descriptor) + } } - return nil -} + for ticket, indexes := range voteIndexes { + // Remove any votes that do not have a collider blob + colliderIndex, ok := colliderIndexes[ticket] + if !ok { + // This is not a valid vote + delete(votes, ticket) + continue + } -// voteCollider is used to prevent duplicate votes at the tlog level. The -// backend saves a digest of the data to the trillian log (tlog). Tlog does not -// allow leaves with duplicate values, so once a vote colider is saved to the -// backend for a ticket it should be impossible for another vote collider to be -// saved to the backend that is voting with the same ticket on the same record, -// regardless of what the vote bits are. The vote collider and the full cast -// vote are saved to the backend at the same time. A cast vote is not -// considered valid unless a corresponding vote collider is present. -type voteCollider struct { - Token string `json:"token"` // Record token - Ticket string `json:"ticket"` // Ticket hash -} + // If multiple votes have been cast using the same ticket then + // we must manually determine which vote is valid. + if len(indexes) == 1 { + // Only one cast vote exists for this ticket. This is good. + continue + } -func (p *ticketVotePlugin) voteColliderSave(treeID int64, vc voteCollider) error { - // Prepare blob - be, err := convertBlobEntryFromVoteCollider(vc) - if err != nil { - return err - } + // Sanity check + if len(indexes) == 0 { + return nil, fmt.Errorf("no cast vote index found %v", ticket) + } - // Save blob - return p.tstore.BlobSave(treeID, *be) -} + log.Tracef("Multiple votes found for a single vote collider %v", ticket) -// ballot casts the provided votes concurrently. The vote results are passed -// back through the results channel to the calling function. This function -// waits until all provided votes have been cast before returning. -func (p *ticketVotePlugin) ballot(treeID int64, votes []ticketvote.CastVote, results chan ticketvote.CastVoteReply) { - // Cast the votes concurrently - var wg sync.WaitGroup - for _, v := range votes { - // Increment the wait group counter - wg.Add(1) + // Multiple votes exist for this ticket. The vote that is valid + // is the one that immediately precedes the vote collider. Start + // at the end of the vote indexes and find the first vote index + // that precedes the collider index. + var validVoteIndex int + for i := len(indexes) - 1; i >= 0; i-- { + voteIndex := indexes[i] + if voteIndex < colliderIndex { + // This is the valid vote + validVoteIndex = voteIndex + break + } + } - go func(v ticketvote.CastVote) { - // Decrement wait group counter once vote is cast - defer wg.Done() + // Save the valid vote + b := blobs[validVoteIndex] + cv, err := convertCastVoteDetailsFromBlobEntry(b) + if err != nil { + return nil, err + } + votes[cv.Ticket] = *cv + } - // Setup cast vote details - receipt := p.identity.SignMessage([]byte(v.Signature)) - cv := ticketvote.CastVoteDetails{ - Token: v.Token, - Ticket: v.Ticket, - VoteBit: v.VoteBit, - Signature: v.Signature, - Receipt: hex.EncodeToString(receipt[:]), - } + // Put votes into an array + cvotes := make([]ticketvote.CastVoteDetails, 0, len(blobs)) + for _, v := range votes { + cvotes = append(cvotes, v) + } - // Declare here to prevent goto errors - var ( - cvr ticketvote.CastVoteReply - vc voteCollider - ) + // Sort by ticket hash + sort.SliceStable(cvotes, func(i, j int) bool { + return cvotes[i].Ticket < cvotes[j].Ticket + }) - // Save cast vote - err := p.castVoteSave(treeID, cv) - if err == plugins.ErrDuplicateBlob { - // This cast vote has already been saved. Its possible that - // a previous attempt to vote with this ticket failed before - // the vote collider could be saved. Continue execution so - // that we re-attempt to save the vote collider. - } else if err != nil { - t := time.Now().Unix() - log.Errorf("cmdCastBallot: castVoteSave %v: %v", t, err) - e := ticketvote.VoteErrorInternalError - cvr.Ticket = v.Ticket - cvr.ErrorCode = e - cvr.ErrorContext = fmt.Sprintf("%v: %v", - ticketvote.VoteErrors[e], t) - goto sendResult - } + return cvotes, nil +} - // Save vote collider - vc = voteCollider{ - Token: v.Token, - Ticket: v.Ticket, - } - err = p.voteColliderSave(treeID, vc) - if err != nil { - t := time.Now().Unix() - log.Errorf("cmdCastBallot: voteColliderSave %v: %v", t, err) - e := ticketvote.VoteErrorInternalError - cvr.Ticket = v.Ticket - cvr.ErrorCode = e - cvr.ErrorContext = fmt.Sprintf("%v: %v", - ticketvote.VoteErrors[e], t) - goto sendResult - } +// voteOptionResults tallies the results of a ticket vote and returns a +// VoteOptionResult for each vote option in the ticket vote. +func (p *ticketVotePlugin) voteOptionResults(token []byte, options []ticketvote.VoteOption) ([]ticketvote.VoteOptionResult, error) { + // Ongoing votes will have the cast votes cached. Calculate the + // results using the cached votes if we can since it will be much + // faster. + var ( + tally = make(map[string]uint32, len(options)) + t = hex.EncodeToString(token) + ctally = p.activeVotes.Tally(t) + ) + switch { + case len(ctally) > 0: + // Vote are in the cache. Use the cached results. + tally = ctally - // Update receipt - cvr.Ticket = v.Ticket - cvr.Receipt = cv.Receipt + default: + // Votes are not in the cache. Pull them from the backend. + reply, err := p.backend.PluginRead(token, ticketvote.PluginID, + ticketvote.CmdResults, "") + if err != nil { + return nil, err + } + var rr ticketvote.ResultsReply + err = json.Unmarshal([]byte(reply), &rr) + if err != nil { + return nil, err + } - // Update cast votes cache - p.activeVotes.AddCastVote(v.Token, v.Ticket, v.VoteBit) + // Tally the results + for _, v := range rr.Votes { + tally[v.VoteBit]++ + } + } - sendResult: - // Send result back to calling function - results <- cvr - }(v) + // Prepare reply + results := make([]ticketvote.VoteOptionResult, 0, len(options)) + for _, v := range options { + bit := strconv.FormatUint(v.Bit, 16) + results = append(results, ticketvote.VoteOptionResult{ + ID: v.ID, + Description: v.Description, + VoteBit: v.Bit, + Votes: uint64(tally[bit]), + }) } - // Wait for the full ballot to be cast before returning. - wg.Wait() + return results, nil } -// cmdCastBallot casts a ballot of votes. This function will not return a user -// error if one occurs. It will instead return the ballot reply with the error -// included in the individual cast vote reply that it applies to. -func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload string) (string, error) { - // Decode payload - var cb ticketvote.CastBallot - err := json.Unmarshal([]byte(payload), &cb) +// voteSummariesForRunoff calculates and returns the vote summaries of all +// submissions in a runoff vote. This should only be called once the vote has +// finished. +func (p *ticketVotePlugin) summariesForRunoff(parentToken string) (map[string]ticketvote.SummaryReply, error) { + // Get runoff vote details + parent, err := tokenDecode(parentToken) if err != nil { - return "", err + return nil, err } - votes := cb.Ballot - - // Verify there is work to do - if len(votes) == 0 { - // Nothing to do - cbr := ticketvote.CastBallotReply{ - Receipts: []ticketvote.CastVoteReply{}, - } - reply, err := json.Marshal(cbr) - if err != nil { - return "", err - } - return string(reply), nil + reply, err := p.backend.PluginRead(parent, ticketvote.PluginID, + cmdRunoffDetails, "") + if err != nil { + return nil, fmt.Errorf("PluginRead %x %v %v: %v", + parent, ticketvote.PluginID, cmdRunoffDetails, err) } - - // Get the data that we need to validate the votes - voteDetails := p.activeVotes.VoteDetails(token) - eligible := p.activeVotes.EligibleTickets(token) - bestBlock, err := p.bestBlock() + var rdr runoffDetailsReply + err = json.Unmarshal([]byte(reply), &rdr) if err != nil { - return "", err + return nil, err } - // Perform all validation that does not require fetching the - // commitment addresses. - receipts := make([]ticketvote.CastVoteReply, len(votes)) - for k, v := range votes { - // Verify token is a valid token - t, err := tokenDecode(v.Token) - if err != nil { - e := ticketvote.VoteErrorTokenInvalid - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = fmt.Sprintf("%v: not hex", - ticketvote.VoteErrors[e]) - continue - } + // Verify submissions exist + subs := rdr.Runoff.Submissions + if len(subs) == 0 { + return map[string]ticketvote.SummaryReply{}, nil + } - // Verify vote token and command token are the same - if !bytes.Equal(t, token) { - e := ticketvote.VoteErrorMultipleRecordVotes - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = ticketvote.VoteErrors[e] - continue + // Compile summaries for all submissions + var ( + summaries = make(map[string]ticketvote.SummaryReply, len(subs)) + winnerNetApprove int // Net number of approve votes of the winner + winnerToken string // Token of the winner + ) + for _, v := range subs { + token, err := tokenDecode(v) + if err != nil { + return nil, err } - // Verify vote is still active - if voteDetails == nil { - e := ticketvote.VoteErrorVoteStatusInvalid - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = fmt.Sprintf("%v: vote is not active", - ticketvote.VoteErrors[e]) - continue - } - if voteHasEnded(bestBlock, voteDetails.EndBlockHeight) { - e := ticketvote.VoteErrorVoteStatusInvalid - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = fmt.Sprintf("%v: vote has ended", - ticketvote.VoteErrors[e]) - continue + // Get vote details + vd, err := p.voteDetailsByToken(token) + if err != nil { + return nil, err } - // Verify vote bit - bit, err := strconv.ParseUint(v.VoteBit, 16, 64) + // Get vote options results + results, err := p.voteOptionResults(token, vd.Params.Options) if err != nil { - e := ticketvote.VoteErrorVoteBitInvalid - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = ticketvote.VoteErrors[e] - continue + return nil, err } - err = voteBitVerify(voteDetails.Params.Options, - voteDetails.Params.Mask, bit) - if err != nil { - e := ticketvote.VoteErrorVoteBitInvalid - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = fmt.Sprintf("%v: %v", - ticketvote.VoteErrors[e], err) - continue + + // Add summary to the reply + s := ticketvote.SummaryReply{ + Type: vd.Params.Type, + Status: ticketvote.VoteStatusRejected, + Duration: vd.Params.Duration, + StartBlockHeight: vd.StartBlockHeight, + StartBlockHash: vd.StartBlockHash, + EndBlockHeight: vd.EndBlockHeight, + EligibleTickets: uint32(len(vd.EligibleTickets)), + QuorumPercentage: vd.Params.QuorumPercentage, + PassPercentage: vd.Params.PassPercentage, + Results: results, } + summaries[v] = s - // Verify ticket is eligible to vote - _, ok := eligible[v.Ticket] - if !ok { - e := ticketvote.VoteErrorTicketNotEligible - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = ticketvote.VoteErrors[e] + // We now check if this record has the most net yes votes. + + // Verify the vote met quorum and pass requirements + approved := voteIsApproved(*vd, results) + if !approved { + // Vote did not meet quorum and pass requirements. Nothing + // else to do. Record vote is not approved. continue } - // Verify ticket has not already voted - if p.activeVotes.VoteIsDuplicate(v.Token, v.Ticket) { - e := ticketvote.VoteErrorTicketAlreadyVoted - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = ticketvote.VoteErrors[e] - continue + // Check if this record has more net approved votes then current + // highest. + var ( + votesApprove uint64 // Number of approve votes + votesReject uint64 // Number of reject votes + ) + for _, vor := range s.Results { + switch vor.ID { + case ticketvote.VoteOptionIDApprove: + votesApprove = vor.Votes + case ticketvote.VoteOptionIDReject: + votesReject = vor.Votes + default: + // Runoff vote options can only be approve/reject + return nil, fmt.Errorf("unknown runoff vote option %v", vor.ID) + } + + netApprove := int(votesApprove) - int(votesReject) + if netApprove > winnerNetApprove { + // New winner! + winnerToken = v + winnerNetApprove = netApprove + } + + // This function doesn't handle the unlikely case that the + // runoff vote results in a tie. } } + if winnerToken != "" { + // A winner was found. Mark their summary as approved. + s := summaries[winnerToken] + s.Status = ticketvote.VoteStatusApproved + summaries[winnerToken] = s + } - // Get the largest commitment address for each ticket and verify - // that the vote was signed using the private key from this - // address. We first check the active votes cache to see if the - // commitment addresses have already been fetched. Any tickets - // that are not found in the cache are fetched manually. - tickets := make([]string, 0, len(cb.Ballot)) - for k, v := range votes { - if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { - // Vote has an error. Skip it. - continue - } - tickets = append(tickets, v.Ticket) + return summaries, nil +} + +// summary returns the vote summary for a record. +func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) (*ticketvote.SummaryReply, error) { + // Check if the summary has been cached + s, err := p.summaryCache(hex.EncodeToString(token)) + switch { + case errors.Is(err, errSummaryNotFound): + // Cached summary not found. Continue. + case err != nil: + // Some other error + return nil, fmt.Errorf("summaryCache: %v", err) + default: + // Caches summary was found. Return it. + return s, nil } - addrs := p.activeVotes.CommitmentAddrs(token, tickets) - notInCache := make([]string, 0, len(tickets)) - for _, v := range tickets { - _, ok := addrs[v] - if !ok { - notInCache = append(notInCache, v) + + // Summary has not been cached. Get it manually. + + // Assume vote is unauthorized. Only update the status when the + // appropriate record has been found that proves otherwise. + status := ticketvote.VoteStatusUnauthorized + + // Check if the vote has been authorized. Not all vote types + // require an authorization. + auths, err := p.auths(treeID) + if err != nil { + return nil, fmt.Errorf("auths: %v", err) + } + if len(auths) > 0 { + lastAuth := auths[len(auths)-1] + switch ticketvote.AuthActionT(lastAuth.Action) { + case ticketvote.AuthActionAuthorize: + // Vote has been authorized; continue + status = ticketvote.VoteStatusAuthorized + case ticketvote.AuthActionRevoke: + // Vote authorization has been revoked. Its not possible for + // the vote to have been started. We can stop looking. + return &ticketvote.SummaryReply{ + Status: status, + Results: []ticketvote.VoteOptionResult{}, + BestBlock: bestBlock, + }, nil } } - log.Debugf("%v/%v commitment addresses found in cache", - len(tickets)-len(notInCache), len(tickets)) + // Check if the vote has been started + vd, err := p.voteDetails(treeID) + if err != nil { + return nil, fmt.Errorf("startDetails: %v", err) + } + if vd == nil { + // Vote has not been started yet + return &ticketvote.SummaryReply{ + Status: status, + Results: []ticketvote.VoteOptionResult{}, + BestBlock: bestBlock, + }, nil + } + + // Vote has been started. We need to check if the vote has ended + // yet and if it can be considered approved or rejected. + status = ticketvote.VoteStatusStarted + + // Tally vote results + results, err := p.voteOptionResults(token, vd.Params.Options) + if err != nil { + return nil, err + } - if len(notInCache) > 0 { - // Get commitment addresses from dcrdata - caddrs, err := p.largestCommitmentAddrs(tickets) - if err != nil { - return "", fmt.Errorf("largestCommitmentAddrs: %v", err) - } + // Prepare summary + summary := ticketvote.SummaryReply{ + Type: vd.Params.Type, + Status: status, + Duration: vd.Params.Duration, + StartBlockHeight: vd.StartBlockHeight, + StartBlockHash: vd.StartBlockHash, + EndBlockHeight: vd.EndBlockHeight, + EligibleTickets: uint32(len(vd.EligibleTickets)), + QuorumPercentage: vd.Params.QuorumPercentage, + PassPercentage: vd.Params.PassPercentage, + Results: results, + BestBlock: bestBlock, + } - // Add addresses to the existing map - for k, v := range caddrs { - addrs[k] = v - } + // If the vote has not finished yet then we are done for now. + if !voteHasEnded(bestBlock, vd.EndBlockHeight) { + return &summary, nil } - // Verify the signatures - for k, v := range votes { - if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { - // Vote has an error. Skip it. - continue + // The vote has finished. Find whether the vote was approved and + // cache the vote summary. + switch vd.Params.Type { + case ticketvote.VoteTypeStandard: + // Standard vote uses a simple approve/reject result + if voteIsApproved(*vd, results) { + summary.Status = ticketvote.VoteStatusApproved + } else { + summary.Status = ticketvote.VoteStatusRejected } - // Verify vote signature - commitmentAddr, ok := addrs[v.Ticket] - if !ok { - t := time.Now().Unix() - log.Errorf("cmdCastBallot: commitment addr not found %v: %v", - t, v.Ticket) - e := ticketvote.VoteErrorInternalError - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = fmt.Sprintf("%v: %v", - ticketvote.VoteErrors[e], t) - continue - } - if commitmentAddr.err != nil { - t := time.Now().Unix() - log.Errorf("cmdCastBallot: commitment addr error %v: %v %v", - t, v.Ticket, commitmentAddr.err) - e := ticketvote.VoteErrorInternalError - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = fmt.Sprintf("%v: %v", - ticketvote.VoteErrors[e], t) - continue - } - err = p.castVoteSignatureVerify(v, commitmentAddr.addr) + // Cache summary + err = p.summaryCacheSave(vd.Params.Token, summary) if err != nil { - e := ticketvote.VoteErrorSignatureInvalid - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = fmt.Sprintf("%v: %v", - ticketvote.VoteErrors[e], err) - continue + return nil, err } - } - // The votes that have passed validation will be cast in batches of - // size batchSize. Each batch of votes is cast concurrently in order - // to accommodate the trillian log signer bottleneck. The log signer - // picks up queued leaves and appends them onto the trillian tree - // every xxx ms, where xxx is a configurable value on the log signer, - // but is typically a few hundred milliseconds. Lets use 200ms as an - // example. If we don't cast the votes in batches then every vote in - // the ballot will take 200 milliseconds since we wait for the leaf - // to be fully appended before considering the trillian call - // successful. A person casting hundreds of votes in a single ballot - // would cause UX issues for all the voting clients since the backend - // locks the record during any plugin write calls. Only one ballot - // can be cast at a time. - // - // The second variable that we must watch out for is the max trillian - // queued leaf batch size. This is also a configurable trillian value - // that represents the maximum number of leaves that can be waiting - // in the queue for all trees in the trillian instance. This value is - // typically around the order of magnitude of 1000s of queued leaves. - // - // The third variable that can cause errors is reaching the trillian - // datastore max connection limits. Each vote being cast creates a - // trillian connection. Overloading the trillian connections can - // cause max connection exceeded errors. The max allowed connections - // is a configurable trillian value, but should also be adjusted on - // the key-value store database itself as well. - // - // This is why a vote batch size of 10 was chosen. It is large enough - // to alleviate performance bottlenecks from the log signer interval, - // but small enough to still allow multiple records votes to be held - // concurrently without running into the queued leaf batch size limit. + // Remove record from the active votes cache + p.activeVotes.Del(vd.Params.Token) - // Prepare work - var ( - batchSize = 10 - batch = make([]ticketvote.CastVote, 0, batchSize) - queue = make([][]ticketvote.CastVote, 0, len(votes)/batchSize) + case ticketvote.VoteTypeRunoff: + // A runoff vote requires that we pull all other runoff vote + // submissions to determine if the vote actually passed. + summaries, err := p.summariesForRunoff(vd.Params.Parent) + if err != nil { + return nil, err + } + for k, v := range summaries { + // Cache summary + err = p.summaryCacheSave(k, v) + if err != nil { + return nil, err + } - // ballotCount is the number of votes that have passed validation - // and are being cast in this ballot. - ballotCount int - ) - for k, v := range votes { - if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { - // Vote has an error. Skip it. - continue + // Remove record from active votes cache + p.activeVotes.Del(k) } - // Add vote to the current batch - batch = append(batch, v) - ballotCount++ + summary = summaries[vd.Params.Token] - if len(batch) == batchSize { - // This batch is full. Add the batch to the queue and start - // a new batch. - queue = append(queue, batch) - batch = make([]ticketvote.CastVote, 0, batchSize) - } - } - if len(batch) != 0 { - // Add leftover batch to the queue - queue = append(queue, batch) + default: + return nil, fmt.Errorf("unknown vote type") } - log.Debugf("Casting %v votes in %v batches of size %v", - ballotCount, len(queue), batchSize) - - // Cast ballot in batches - results := make(chan ticketvote.CastVoteReply, ballotCount) - for i, batch := range queue { - log.Debugf("Casting %v votes in batch %v/%v", len(batch), i+1, len(queue)) + return &summary, nil +} - p.ballot(treeID, batch, results) +// summaryByToken returns the vote summary for a record. +func (p *ticketVotePlugin) summaryByToken(token []byte) (*ticketvote.SummaryReply, error) { + reply, err := p.backend.PluginRead(token, ticketvote.PluginID, + ticketvote.CmdSummary, "") + if err != nil { + return nil, fmt.Errorf("PluginRead %x %v %v: %v", + token, ticketvote.PluginID, ticketvote.CmdSummary, err) } - - // Empty out the results channel - r := make(map[string]ticketvote.CastVoteReply, ballotCount) - close(results) - for v := range results { - r[v.Ticket] = v + var sr ticketvote.SummaryReply + err = json.Unmarshal([]byte(reply), &sr) + if err != nil { + return nil, err } + return &sr, nil +} - if len(r) != ballotCount { - log.Errorf("Missing results: got %v, want %v", len(r), ballotCount) +// timestamp returns the timestamp for a specific piece of data. +func (p *ticketVotePlugin) timestamp(treeID int64, digest []byte) (*ticketvote.Timestamp, error) { + t, err := p.tstore.Timestamp(treeID, digest) + if err != nil { + return nil, fmt.Errorf("timestamp %v %x: %v", treeID, digest, err) } - // Fill in the receipts - for k, v := range votes { - if receipts[k].ErrorCode != ticketvote.VoteErrorInvalid { - // Vote has an error. Skip it. - continue - } - cvr, ok := r[v.Ticket] - if !ok { - t := time.Now().Unix() - log.Errorf("cmdCastBallot: vote result not found %v: %v", t, v.Ticket) - e := ticketvote.VoteErrorInternalError - receipts[k].Ticket = v.Ticket - receipts[k].ErrorCode = e - receipts[k].ErrorContext = fmt.Sprintf("%v: %v", - ticketvote.VoteErrors[e], t) - continue - } - - // Fill in receipt - receipts[k] = cvr + // Convert response + proofs := make([]ticketvote.Proof, 0, len(t.Proofs)) + for _, v := range t.Proofs { + proofs = append(proofs, ticketvote.Proof{ + Type: v.Type, + Digest: v.Digest, + MerkleRoot: v.MerkleRoot, + MerklePath: v.MerklePath, + ExtraData: v.ExtraData, + }) } + return &ticketvote.Timestamp{ + Data: t.Data, + Digest: t.Digest, + TxID: t.TxID, + MerkleRoot: t.MerkleRoot, + Proofs: proofs, + }, nil +} - // Prepare reply - cbr := ticketvote.CastBallotReply{ - Receipts: receipts, +// recordAbridged returns a record where the only record file returned is the +// vote metadata file if one exists. +func (p *ticketVotePlugin) recordAbridged(token []byte) (*backend.Record, error) { + reqs := []backend.RecordRequest{ + { + Token: token, + Filenames: []string{ + ticketvote.FileNameVoteMetadata, + }, + }, } - reply, err := json.Marshal(cbr) + rs, err := p.backend.Records(reqs) if err != nil { - return "", err + return nil, err + } + r, ok := rs[hex.EncodeToString(token)] + if !ok { + return nil, backend.ErrRecordNotFound + } + return &r, nil +} + +// bestBlock fetches the best block from the dcrdata plugin and returns it. If +// the dcrdata connection is not active, an error will be returned. +func (p *ticketVotePlugin) bestBlock() (uint32, error) { + // Get best block + payload, err := json.Marshal(dcrdata.BestBlock{}) + if err != nil { + return 0, err } - - return string(reply), nil -} - -func (p *ticketVotePlugin) cmdDetails(treeID int64, token []byte) (string, error) { - // Get vote authorizations - auths, err := p.auths(treeID) + reply, err := p.backend.PluginRead(nil, dcrdata.PluginID, + dcrdata.CmdBestBlock, string(payload)) if err != nil { - return "", fmt.Errorf("auths: %v", err) + return 0, fmt.Errorf("PluginRead %v %v: %v", + dcrdata.PluginID, dcrdata.CmdBestBlock, err) } - // Get vote details - vd, err := p.voteDetails(treeID) + // Handle response + var bbr dcrdata.BestBlockReply + err = json.Unmarshal([]byte(reply), &bbr) if err != nil { - return "", fmt.Errorf("voteDetails: %v", err) + return 0, err } - - // Prepare rely - dr := ticketvote.DetailsReply{ - Auths: auths, - Vote: vd, + if bbr.Status != dcrdata.StatusConnected { + // The dcrdata connection is down. The best block cannot be + // trusted as being accurate. + return 0, fmt.Errorf("dcrdata connection is down") } - reply, err := json.Marshal(dr) - if err != nil { - return "", err + if bbr.Height == 0 { + return 0, fmt.Errorf("invalid best block height 0") } - return string(reply), nil + return bbr.Height, nil } -func (p *ticketVotePlugin) cmdResults(treeID int64, token []byte) (string, error) { - // Get vote results - votes, err := p.voteResults(treeID) +// bestBlockUnsafe fetches the best block from the dcrdata plugin and returns +// it. If the dcrdata connection is not active, an error WILL NOT be returned. +// The dcrdata cached best block height will be returned even though it may be +// stale. Use bestBlock() if the caller requires a guarantee that the best +// block is not stale. +func (p *ticketVotePlugin) bestBlockUnsafe() (uint32, error) { + // Get best block + payload, err := json.Marshal(dcrdata.BestBlock{}) if err != nil { - return "", err + return 0, err } - - // Prepare reply - rr := ticketvote.ResultsReply{ - Votes: votes, + reply, err := p.backend.PluginRead(nil, dcrdata.PluginID, + dcrdata.CmdBestBlock, string(payload)) + if err != nil { + return 0, fmt.Errorf("PluginRead %v %v: %v", + dcrdata.PluginID, dcrdata.CmdBestBlock, err) } - reply, err := json.Marshal(rr) + + // Handle response + var bbr dcrdata.BestBlockReply + err = json.Unmarshal([]byte(reply), &bbr) if err != nil { - return "", err + return 0, err + } + if bbr.Height == 0 { + return 0, fmt.Errorf("invalid best block height 0") } - return string(reply), nil + return bbr.Height, nil +} + +// voteHasEnded returns whether the vote has ended. +func voteHasEnded(bestBlock, endHeight uint32) bool { + return bestBlock >= endHeight } // voteIsApproved returns whether the provided vote option results met the @@ -2481,183 +2631,35 @@ func voteIsApproved(vd ticketvote.VoteDetails, results []ticketvote.VoteOptionRe return approved } -func (p *ticketVotePlugin) cmdSummary(treeID int64, token []byte) (string, error) { - // Get best block. This cmd does not write any data so we do not - // have to use the safe best block. - bb, err := p.bestBlockUnsafe() - if err != nil { - return "", fmt.Errorf("bestBlockUnsafe: %v", err) - } - - // Get summary - sr, err := p.summary(treeID, token, bb) - if err != nil { - return "", fmt.Errorf("summary: %v", err) - } - - // Prepare reply - reply, err := json.Marshal(sr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *ticketVotePlugin) cmdInventory(payload string) (string, error) { - var i ticketvote.Inventory - err := json.Unmarshal([]byte(payload), &i) - if err != nil { - return "", err - } - - // Get best block. This command does not write any data so we can - // use the unsafe best block. - bb, err := p.bestBlockUnsafe() - if err != nil { - return "", fmt.Errorf("bestBlockUnsafe: %v", err) - } - - // Get the inventory - ibs, err := p.inventoryByStatus(bb, i.Status, i.Page) - if err != nil { - return "", fmt.Errorf("invByStatus: %v", err) - } - - // Prepare reply - tokens := make(map[string][]string, len(ibs.Tokens)) - for k, v := range ibs.Tokens { - vs := ticketvote.VoteStatuses[k] - tokens[vs] = v - } - ir := ticketvote.InventoryReply{ - Tokens: tokens, - BestBlock: ibs.BestBlock, - } - reply, err := json.Marshal(ir) - if err != nil { - return "", err - } - - return string(reply), nil +// tokenDecode decodes a record token and only accepts full length tokens. +func tokenDecode(token string) ([]byte, error) { + return util.TokenDecode(util.TokenTypeTstore, token) } -func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload string) (string, error) { - // Decode payload - var t ticketvote.Timestamps - err := json.Unmarshal([]byte(payload), &t) +// tokenVerify verifies that a token that is part of a plugin command payload +// is valid. This is applicable when a plugin command payload contains a +// signature that includes the record token. The token included in payload must +// be a valid, full length record token and it must match the token that was +// passed into the politeiad API for this plugin command, i.e. the token for +// the record that this plugin command is being executed on. +func tokenVerify(cmdToken []byte, payloadToken string) error { + pt, err := tokenDecode(payloadToken) if err != nil { - return "", err - } - - var ( - auths = make([]ticketvote.Timestamp, 0, 32) - details *ticketvote.Timestamp - - pageSize = ticketvote.VoteTimestampsPageSize - votes = make([]ticketvote.Timestamp, 0, pageSize) - ) - switch { - case t.VotesPage > 0: - // Return a page of vote timestamps - digests, err := p.tstore.DigestsByDataDesc(treeID, - []string{dataDescriptorCastVoteDetails}) - if err != nil { - return "", fmt.Errorf("digestsByKeyPrefix %v %v: %v", - treeID, dataDescriptorVoteDetails, err) - } - - startAt := (t.VotesPage - 1) * pageSize - for i, v := range digests { - if i < int(startAt) { - continue - } - ts, err := p.timestamp(treeID, v) - if err != nil { - return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) - } - votes = append(votes, *ts) - if len(votes) == int(pageSize) { - // We have a full page. We're done. - break - } - } - - default: - // Return authorization timestamps and the vote details timestamp. - - // Auth timestamps - digests, err := p.tstore.DigestsByDataDesc(treeID, - []string{dataDescriptorAuthDetails}) - if err != nil { - return "", fmt.Errorf("DigestByDataDesc %v %v: %v", - treeID, dataDescriptorAuthDetails, err) - } - auths = make([]ticketvote.Timestamp, 0, len(digests)) - for _, v := range digests { - ts, err := p.timestamp(treeID, v) - if err != nil { - return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) - } - auths = append(auths, *ts) - } - - // Vote details timestamp - digests, err = p.tstore.DigestsByDataDesc(treeID, - []string{dataDescriptorVoteDetails}) - if err != nil { - return "", fmt.Errorf("DigestsByDataDesc %v %v: %v", - treeID, dataDescriptorVoteDetails, err) - } - // There should never be more than a one vote details - if len(digests) > 1 { - return "", fmt.Errorf("invalid vote details count: got %v, want 1", - len(digests)) - } - for _, v := range digests { - ts, err := p.timestamp(treeID, v) - if err != nil { - return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) - } - details = ts + return backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeTokenInvalid), + ErrorContext: err.Error(), } } - - // Prepare reply - tr := ticketvote.TimestampsReply{ - Auths: auths, - Details: details, - Votes: votes, - } - reply, err := json.Marshal(tr) - if err != nil { - return "", err - } - - return string(reply), nil -} - -func (p *ticketVotePlugin) cmdSubmissions(token []byte) (string, error) { - // Get submissions list - lf, err := p.submissionsCache(token) - if err != nil { - return "", err - } - - // Prepare reply - tokens := make([]string, 0, len(lf.Tokens)) - for k := range lf.Tokens { - tokens = append(tokens, k) - } - lfr := ticketvote.SubmissionsReply{ - Submissions: tokens, - } - reply, err := json.Marshal(lfr) - if err != nil { - return "", err + if !bytes.Equal(cmdToken, pt) { + return backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeTokenInvalid), + ErrorContext: fmt.Sprintf("payload token does not match "+ + "command token: got %x, want %x", pt, cmdToken), + } } - - return string(reply), nil + return nil } func convertSignatureError(err error) backend.PluginError { diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go index 8d7fb839c..a97c7b7b4 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go @@ -15,29 +15,78 @@ import ( "github.com/decred/politeia/politeiad/plugins/ticketvote" ) -// voteMetadataDecode decodes and returns the VoteMetadata from the -// provided backend files. If a VoteMetadata is not found, nil is returned. -func voteMetadataDecode(files []backend.File) (*ticketvote.VoteMetadata, error) { - var voteMD *ticketvote.VoteMetadata - for _, v := range files { - if v.Name != ticketvote.FileNameVoteMetadata { - continue - } - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return nil, err - } - var m ticketvote.VoteMetadata - err = json.Unmarshal(b, &m) - if err != nil { - return nil, err - } - voteMD = &m - break +// hookNewRecordPre adds plugin specific validation onto the tstore backend +// RecordNew method. +func (p *ticketVotePlugin) hookNewRecordPre(payload string) error { + var nr plugins.HookNewRecordPre + err := json.Unmarshal([]byte(payload), &nr) + if err != nil { + return err } - return voteMD, nil + + // Verify vote metadata + return p.voteMetadataVerify(nr.Files) +} + +// hookEditRecordPre adds plugin specific validation onto the tstore backend +// RecordEdit method. +func (p *ticketVotePlugin) hookEditRecordPre(payload string) error { + var er plugins.HookEditRecord + err := json.Unmarshal([]byte(payload), &er) + if err != nil { + return err + } + + // Verify vote metadata + return p.voteMetadataVerifyOnEdits(er.Record, er.Files) +} + +// hookSetStatusRecordPre adds plugin specific validation onto the tstore +// backend RecordSetStatus method. +func (p *ticketVotePlugin) hookSetRecordStatusPre(payload string) error { + var srs plugins.HookSetRecordStatus + err := json.Unmarshal([]byte(payload), &srs) + if err != nil { + return err + } + + return p.voteMetadataVerifyOnStatusChange(srs.RecordMetadata.Status, + srs.Record.Files) } +// hookNewRecordPre caches plugin data from the tstore backend RecordSetStatus +// method. +func (p *ticketVotePlugin) hookSetRecordStatusPost(treeID int64, payload string) error { + var srs plugins.HookSetRecordStatus + err := json.Unmarshal([]byte(payload), &srs) + if err != nil { + return err + } + + // Ticketvote caches only need to be updated for vetted records + if srs.RecordMetadata.State == backend.StateUnvetted { + return nil + } + + // Update the inventory cache + switch srs.RecordMetadata.Status { + case backend.StatusPublic: + // Add to inventory + p.inventoryAdd(srs.RecordMetadata.Token, + ticketvote.VoteStatusUnauthorized) + case backend.StatusCensored, backend.StatusArchived: + // These statuses do not allow for a vote. Mark as ineligible. + p.inventoryUpdate(srs.RecordMetadata.Token, + ticketvote.VoteStatusIneligible) + } + + // Update cached vote metadata + return p.voteMetadataCacheOnStatusChange(srs.RecordMetadata.Token, + srs.RecordMetadata.State, srs.RecordMetadata.Status, srs.Record.Files) +} + +// linkByVerify verifies that the provided link by timestamp meets all +// ticketvote plugin requirements. func (p *ticketVotePlugin) linkByVerify(linkBy int64) error { if linkBy == 0 { // LinkBy as not been set @@ -47,25 +96,25 @@ func (p *ticketVotePlugin) linkByVerify(linkBy int64) error { max := time.Now().Unix() + p.linkByPeriodMax switch { case linkBy < min: - e := fmt.Sprintf("linkby %v is less than min required of %v", - linkBy, min) return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeLinkByInvalid), - ErrorContext: e, + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeLinkByInvalid), + ErrorContext: fmt.Sprintf("linkby %v is less than min required of %v", + linkBy, min), } case linkBy > max: - e := fmt.Sprintf("linkby %v is more than max allowed of %v", - linkBy, max) return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeLinkByInvalid), - ErrorContext: e, + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeLinkByInvalid), + ErrorContext: fmt.Sprintf("linkby %v is more than max allowed of %v", + linkBy, max), } } return nil } +// linkToVerify verifies that the provided link to meets all ticketvote plugin +// requirements. func (p *ticketVotePlugin) linkToVerify(linkTo string) error { // LinkTo must be a public record token, err := tokenDecode(linkTo) @@ -88,13 +137,12 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { return err } if r.RecordMetadata.Status != backend.StatusPublic { - e := fmt.Sprintf("record status is invalid: got %v, want %v", - backend.Statuses[r.RecordMetadata.Status], - backend.Statuses[backend.StatusPublic]) return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeLinkToInvalid), - ErrorContext: e, + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeLinkToInvalid), + ErrorContext: fmt.Sprintf("record status is invalid: got %v, want %v", + backend.Statuses[r.RecordMetadata.Status], + backend.Statuses[backend.StatusPublic]), } } @@ -137,7 +185,107 @@ func (p *ticketVotePlugin) linkToVerify(linkTo string) error { return nil } -func (p *ticketVotePlugin) voteMetadataVerify(vm ticketvote.VoteMetadata) error { +// linkToVerifyOnEdits runs LinkTo validation that is specific to record edits. +func (p *ticketVotePlugin) linkToVerifyOnEdits(r backend.Record, newFiles []backend.File) error { + // The LinkTo field is not allowed to change once the record has + // become public. + if r.RecordMetadata.State != backend.StateVetted { + // Not vetted. Nothing to do. + return nil + } + var ( + oldLinkTo string + newLinkTo string + ) + vm, err := voteMetadataDecode(r.Files) + if err != nil { + return err + } + // Vote metadata is optional so one may not exist + if vm != nil { + oldLinkTo = vm.LinkTo + } + vm, err = voteMetadataDecode(newFiles) + if err != nil { + return err + } + if vm != nil { + newLinkTo = vm.LinkTo + } + if newLinkTo != oldLinkTo { + return backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeLinkToInvalid), + ErrorContext: fmt.Sprintf("linkto cannot change on vetted record: "+ + "got '%v', want '%v'", newLinkTo, oldLinkTo), + } + } + return nil +} + +// linkToVerifyOnStatusChange runs LinkTo validation that is specific to record +// status changes. +func (p *ticketVotePlugin) linkToVerifyOnStatusChange(status backend.StatusT, vm ticketvote.VoteMetadata) error { + if vm.LinkTo == "" { + // Link to not set. Nothing to do. + return nil + } + + // Verify that the deadline to link to this record has not expired. + // We only need to do this when a record is being made public since + // the submissions list of the parent record is only updated for + // public records. + if status != backend.StatusPublic { + // Not being made public. Nothing to do. + return nil + } + + // Get the parent record + token, err := tokenDecode(vm.LinkTo) + if err != nil { + return err + } + r, err := p.recordAbridged(token) + if err != nil { + return err + } + + // Verify linkby has not expired + vmParent, err := voteMetadataDecode(r.Files) + if err != nil { + return err + } + if vmParent == nil { + return fmt.Errorf("vote metadata does not exist on parent record %v", + vm.LinkTo) + } + if time.Now().Unix() > vmParent.LinkBy { + return backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeLinkToInvalid), + ErrorContext: "parent record linkby has expired", + } + } + + return nil +} + +// voteMetadataVerify decodes the VoteMetadata from the provided files and +// verifies that it meets the ticketvote plugin requirements. Vote metadata is +// optional so one may not exist. +func (p *ticketVotePlugin) voteMetadataVerify(files []backend.File) error { + // Decode vote metadata. The vote metadata is optional so one may + // not exist. + vm, err := voteMetadataDecode(files) + if err != nil { + return err + } + if vm == nil { + // Vote metadata not found. Nothing to do. + return nil + } + + // Verify vote metadata fields are sane switch { case vm.LinkBy == 0 && vm.LinkTo == "": // Vote metadata is empty @@ -171,200 +319,132 @@ func (p *ticketVotePlugin) voteMetadataVerify(vm ticketvote.VoteMetadata) error return nil } -func (p *ticketVotePlugin) hookNewRecordPre(payload string) error { - var nr plugins.HookNewRecordPre - err := json.Unmarshal([]byte(payload), &nr) +func (p *ticketVotePlugin) voteMetadataVerifyOnEdits(r backend.Record, newFiles []backend.File) error { + // Verify LinkTo has not changed. This must be run even if a vote + // metadata is not present. + err := p.linkToVerifyOnEdits(r, newFiles) if err != nil { return err } - // Verify the vote metadata if the record contains one - vm, err := voteMetadataDecode(nr.Files) + // Decode vote metadata. The vote metadata is optional so one may not + // exist. + vm, err := voteMetadataDecode(newFiles) if err != nil { return err } - if vm != nil { - err = p.voteMetadataVerify(*vm) - if err != nil { - return err - } + if vm == nil { + // Vote metadata not found. Nothing to do. + return nil } - return nil -} - -func (p *ticketVotePlugin) hookEditRecordPre(payload string) error { - var er plugins.HookEditRecord - err := json.Unmarshal([]byte(payload), &er) + // Verify LinkBy + err = p.linkByVerify(vm.LinkBy) if err != nil { return err } - // The LinkTo field is not allowed to change once the record has - // become public. If this is a vetted record, verify that any - // previously set LinkTo has not changed. - if er.Record.RecordMetadata.State == backend.StateVetted { - var ( - oldLinkTo string - newLinkTo string - ) - vm, err := voteMetadataDecode(er.Record.Files) - if err != nil { - return err - } - if vm != nil { - oldLinkTo = vm.LinkTo - } - vm, err = voteMetadataDecode(er.Files) - if err != nil { - return err - } - if vm != nil { - newLinkTo = vm.LinkTo - } - if newLinkTo != oldLinkTo { - e := fmt.Sprintf("linkto cannot change on vetted record: "+ - "got '%v', want '%v'", newLinkTo, oldLinkTo) - return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeLinkToInvalid), - ErrorContext: e, - } - } - } - - // Verify LinkBy if one was included. The VoteMetadata is optional - // so the record may not contain one. - vm, err := voteMetadataDecode(er.Files) - if err != nil { - return err - } - if vm != nil { - err = p.linkByVerify(vm.LinkBy) - if err != nil { - return err - } - } + // The LinkTo does not need to be validated since we have already + // confirmed that it has not changed from the previous record + // version and it would have already been validated when the record + // was originally submitted. It should not be possible for it to be + // invalid at this point. return nil } -func (p *ticketVotePlugin) hookSetRecordStatusPre(payload string) error { - var srs plugins.HookSetRecordStatus - err := json.Unmarshal([]byte(payload), &srs) +// voteMetadataVerifyOnStatusChange runs vote metadata validation that is +// specific to record status changes. +func (p *ticketVotePlugin) voteMetadataVerifyOnStatusChange(status backend.StatusT, files []backend.File) error { + // Decode vote metadata. Vote metadata is optional so one may not + // exist. + vm, err := voteMetadataDecode(files) if err != nil { return err } + if vm == nil { + // Vote metadata not found. Nothing to do. + return nil + } - // Check if the LinkTo has been set - vm, err := voteMetadataDecode(srs.Record.Files) + // Verify LinkTo + err = p.linkToVerifyOnStatusChange(status, *vm) if err != nil { return err } - if vm != nil && vm.LinkTo != "" { - // LinkTo has been set. Verify that the deadline to link to this - // record has not expired. We only need to do this when a record - // is being made public since the submissions list of the parent - // record is only updated for public records. This update occurs - // in the set status post hook. - switch srs.RecordMetadata.Status { - case backend.StatusPublic: - // Get the parent record - token, err := tokenDecode(vm.LinkTo) - if err != nil { - return err - } - r, err := p.recordAbridged(token) - if err != nil { - return err - } - // Verify linkby has not expired - vmParent, err := voteMetadataDecode(r.Files) - if err != nil { - return err - } - if vmParent == nil { - e := fmt.Sprintf("vote metadata does not exist on parent record %v", - srs.RecordMetadata.Token) - panic(e) - } - if time.Now().Unix() > vmParent.LinkBy { - return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeLinkToInvalid), - ErrorContext: "parent record linkby has expired", - } - } - - default: - // Nothing to do - } - } - - return nil + // Verify LinkBy + return p.linkByVerify(vm.LinkBy) } -func (p *ticketVotePlugin) hookSetRecordStatusPost(treeID int64, payload string) error { - var srs plugins.HookSetRecordStatus - err := json.Unmarshal([]byte(payload), &srs) +// voteMetadataCacheOnStatusChange performs vote metadata cache updates after +// a record status change. +func (p *ticketVotePlugin) voteMetadataCacheOnStatusChange(token string, state backend.StateT, status backend.StatusT, files []backend.File) error { + // Decode vote metadata. Vote metadata is optional so one may not + // exist. + vm, err := voteMetadataDecode(files) if err != nil { return err } - - // Ticketvote caches only need to be updated for vetted records - if srs.RecordMetadata.State == backend.StateUnvetted { + if vm == nil { + // Vote metadata doesn't exist. Nothing to do. + return nil + } + if vm.LinkTo == "" { + // LinkTo not set. Nothing to do. return nil } - // Update the inventory cache + // LinkTo has been set. Check if the status change requires the + // submissions list of the linked record to be updated. var ( - oldStatus = srs.Record.RecordMetadata.Status - newStatus = srs.RecordMetadata.Status + parentToken = vm.LinkTo + childToken = token ) - switch newStatus { - case backend.StatusPublic: - // Add to inventory - p.inventoryAdd(srs.RecordMetadata.Token, - ticketvote.VoteStatusUnauthorized) - case backend.StatusCensored, backend.StatusArchived: - // These statuses do not allow for a vote. Mark as ineligible. - p.inventoryUpdate(srs.RecordMetadata.Token, - ticketvote.VoteStatusIneligible) - } + switch { + case state == backend.StateUnvetted: + // We do not update the submissions cache for unvetted records. + // Do nothing. + + case status == backend.StatusPublic: + // Record has been made public. Add child token to parent's + // submissions list. + err := p.submissionsCacheAdd(parentToken, childToken) + if err != nil { + return fmt.Errorf("submissionsFromCacheAdd: %v", err) + } - // Update the submissions cache if the linkto has been set. - vm, err := voteMetadataDecode(srs.Record.Files) - if err != nil { - return err - } - if vm != nil && vm.LinkTo != "" { - // LinkTo has been set. Check if the status change requires the - // submissions list of the linked record to be updated. - var ( - parentToken = vm.LinkTo - childToken = srs.RecordMetadata.Token - ) - switch newStatus { - case backend.StatusPublic: - // Record has been made public. Add child token to parent's - // submissions list. - err := p.submissionsCacheAdd(parentToken, childToken) - if err != nil { - return fmt.Errorf("submissionsFromCacheAdd: %v", err) - } - case backend.StatusCensored: - // Record has been censored. Delete child token from parent's - // submissions list. We only need to do this if the record is - // vetted. - if oldStatus == backend.StatusPublic { - err := p.submissionsCacheDel(parentToken, childToken) - if err != nil { - return fmt.Errorf("submissionsCacheDel: %v", err) - } - } + case status == backend.StatusCensored: + // Record has been censored. Delete child token from parent's + // submissions list. + err := p.submissionsCacheDel(parentToken, childToken) + if err != nil { + return fmt.Errorf("submissionsCacheDel: %v", err) } } return nil } + +// voteMetadataDecode decodes and returns the VoteMetadata from the +// provided backend files. If a VoteMetadata is not found, nil is returned. +func voteMetadataDecode(files []backend.File) (*ticketvote.VoteMetadata, error) { + var voteMD *ticketvote.VoteMetadata + for _, v := range files { + if v.Name != ticketvote.FileNameVoteMetadata { + continue + } + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return nil, err + } + var m ticketvote.VoteMetadata + err = json.Unmarshal(b, &m) + if err != nil { + return nil, err + } + voteMD = &m + break + } + return voteMD, nil +} diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/internalcmds.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/internalcmds.go new file mode 100644 index 000000000..83983436d --- /dev/null +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/internalcmds.go @@ -0,0 +1,59 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ticketvote + +import "github.com/decred/politeia/politeiad/plugins/ticketvote" + +const ( + // Internal plugin commands. These commands are not part of the + // public plugin API. They are for internal use only. + // + // These are are necessary because the command to start a runoff + // vote is executed on the parent record, but state is updated in + // all of the runoff vote submissions as well. Plugin commands + // should not be doing this. This is an exception and we use these + // internal plugin commands as a workaround. + cmdStartRunoffSubmission = "startrunoffsub" + cmdRunoffDetails = "runoffdetails" +) + +// startRunoffRecord is the record that is saved to the runoff vote's parent +// tree as the first step in starting a runoff vote. Plugins are not able to +// update multiple records atomically, so if this call gets interrupted before +// if can start the voting period on all runoff vote submissions, subsequent +// calls will use this record to pick up where the previous call left off. This +// allows us to recover from unexpected errors, such as network errors, and not +// leave a runoff vote in a weird state. +type startRunoffRecord struct { + Submissions []string `json:"submissions"` + Mask uint64 `json:"mask"` + Duration uint32 `json:"duration"` + QuorumPercentage uint32 `json:"quorumpercentage"` + PassPercentage uint32 `json:"passpercentage"` + StartBlockHeight uint32 `json:"startblockheight"` + StartBlockHash string `json:"startblockhash"` + EndBlockHeight uint32 `json:"endblockheight"` + EligibleTickets []string `json:"eligibletickets"` +} + +// startRunoffSubmission is an internal plugin command that is used to start +// the voting period on a runoff vote submission. +type startRunoffSubmission struct { + ParentTreeID int64 `json:"parenttreeid"` + StartDetails ticketvote.StartDetails `json:"startdetails"` +} + +// startRunoffSubmissionReply is the reply to the startRunoffSubmission +// command. +type startRunoffSubmissionReply struct{} + +// runoffDetails is an internal plugin command that requests the details of a +// runoff vote. +type runoffDetails struct{} + +// runoffDetailsReply is the reply to the runoffDetails command. +type runoffDetailsReply struct { + Runoff startRunoffRecord `json:"runoff"` +} diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go index 0924d0bdb..e5e3e765d 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go @@ -216,42 +216,6 @@ func (p *ticketVotePlugin) Fsck(treeIDs []int64) error { // Audit all finished votes // - All votes that were cast were eligible // - No duplicate votes - /* - finished := make([]string, 0, len(inv.Entries)) - for _, v := range inv.Entries { - if v.Status == ticketvote.VoteStatusApproved || - v.Status == ticketvote.VoteStatusRejected { - finished = append(finished, v.Token) - } - } - for _, v := range finished { - // Get all cast votes - token, err := tokenDecode(v) - if err != nil { - return err - } - reply, err := p.backend.VettedPluginCmd(backend.PluginActionRead, - token, ticketvote.PluginID, ticketvote.CmdResults, "") - if err != nil { - return err - } - var rr ticketvote.ResultsReply - err = json.Unmarshal([]byte(reply), &rr) - if err != nil { - return err - } - - // Verify that there are no duplicates - tickets := make(map[string]struct{}, len(rr.Votes)) - for _, v := range rr.Votes { - _, ok := tickets[v.Ticket] - if ok { - return fmt.Errorf("duplicate ticket found %v %v", v.Token, v.Ticket) - } - tickets[v.Ticket] = struct{}{} - } - } - */ return nil } diff --git a/politeiad/plugins/ticketvote/ticketvote.go b/politeiad/plugins/ticketvote/ticketvote.go index 1c758905b..067959bb9 100644 --- a/politeiad/plugins/ticketvote/ticketvote.go +++ b/politeiad/plugins/ticketvote/ticketvote.go @@ -578,7 +578,7 @@ type VoteOptionResult struct { Votes uint64 `json:"votes"` // Votes cast for this option } -// Summary requests the vote summaries for a record. +// Summary requests the vote summary for a record. type Summary struct{} // SummaryReply is the reply to the Summary command. @@ -690,7 +690,7 @@ const ( VoteTimestampsPageSize uint32 = 100 ) -// Timestamps requests the timestamps for ticket vote data. +// Timestamps requests the timestamps for a ticket vote. // // If no votes page number is provided then the vote authorization and vote // details timestamps will be returned. If a votes page number is provided then From fcee39abf7079c3c28f98f8ec3fcf8ced36039e5 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 24 Mar 2021 10:29:54 -0500 Subject: [PATCH 430/449] tstore/ticketvote: Finish cleanup. --- .../tstorebe/plugins/ticketvote/inventory.go | 68 +++++++++++-------- .../plugins/ticketvote/submissions.go | 10 +-- .../tstorebe/plugins/ticketvote/summary.go | 8 +++ 3 files changed, 53 insertions(+), 33 deletions(-) diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/inventory.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/inventory.go index ee66676d9..17faa179e 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/inventory.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/inventory.go @@ -47,7 +47,7 @@ func (p *ticketVotePlugin) invPath() string { // invGetLocked retrieves the inventory from disk. A new inventory is returned // if one does not exist yet. // -// This function must be called WITH the read lock held. +// This function must be called WITH the mtxInv read lock held. func (p *ticketVotePlugin) invGetLocked() (*inventory, error) { b, err := ioutil.ReadFile(p.invPath()) if err != nil { @@ -74,7 +74,7 @@ func (p *ticketVotePlugin) invGetLocked() (*inventory, error) { // invGetLocked retrieves the inventory from disk. A new inventory is returned // if one does not exist yet. // -// This function must be called WITHOUT the read lock held. +// This function must be called WITHOUT the mtxInv write lock held. func (p *ticketVotePlugin) invGet() (*inventory, error) { p.mtxInv.RLock() defer p.mtxInv.RUnlock() @@ -84,7 +84,7 @@ func (p *ticketVotePlugin) invGet() (*inventory, error) { // invSaveLocked writes the inventory to disk. // -// This function must be called WITH the read/write lock held. +// This function must be called WITH the mtxInv write lock held. func (p *ticketVotePlugin) invSaveLocked(inv inventory) error { b, err := json.Marshal(inv) if err != nil { @@ -93,6 +93,9 @@ func (p *ticketVotePlugin) invSaveLocked(inv inventory) error { return ioutil.WriteFile(p.invPath(), b, 0664) } +// invAdd adds a token to the ticketvote inventory. +// +// This function must be called WITHOUT the mtxInv write lock held. func (p *ticketVotePlugin) invAdd(token string, s ticketvote.VoteStatusT) error { p.mtxInv.Lock() defer p.mtxInv.Unlock() @@ -121,7 +124,19 @@ func (p *ticketVotePlugin) invAdd(token string, s ticketvote.VoteStatusT) error return nil } -// This function must be called WITH the read/write lock held. +// inventoryAdd is a wrapper around the invAdd method that allows us to decide +// how disk read/write errors should be handled. For now we just panic. +func (p *ticketVotePlugin) inventoryAdd(token string, s ticketvote.VoteStatusT) { + err := p.invAdd(token, s) + if err != nil { + panic(fmt.Sprintf("invAdd %v %v: %v", token, s, err)) + } +} + +// invUpdateLocked updates a pre existing token in the inventory to a new +// vote status. +// +// This function must be called WITH the mtxInv write lock held. func (p *ticketVotePlugin) invUpdateLocked(token string, s ticketvote.VoteStatusT, endHeight uint32) error { // Get inventory inv, err := p.invGetLocked() @@ -133,8 +148,7 @@ func (p *ticketVotePlugin) invUpdateLocked(token string, s ticketvote.VoteStatus entries, err := entryDel(inv.Entries, token) if err != nil { // This should not happen. Panic if it does. - e := fmt.Sprintf("entry del: %v", err) - panic(e) + panic(fmt.Sprintf("entry del: %v", err)) } // Prepend new entry to inventory @@ -156,6 +170,10 @@ func (p *ticketVotePlugin) invUpdateLocked(token string, s ticketvote.VoteStatus return nil } +// invUpdate updates a pre existing token in the inventory to a new vote +// status. +// +// This function must be called WITHOUT the mtxInv write lock held. func (p *ticketVotePlugin) invUpdate(token string, s ticketvote.VoteStatusT, endHeight uint32) error { p.mtxInv.Lock() defer p.mtxInv.Unlock() @@ -163,6 +181,20 @@ func (p *ticketVotePlugin) invUpdate(token string, s ticketvote.VoteStatusT, end return p.invUpdateLocked(token, s, endHeight) } +// inventoryUpdate is a wrapper around the invUpdate method that allows us to +// decide how disk read/write errors should be handled. For now we just panic. +func (p *ticketVotePlugin) inventoryUpdate(token string, s ticketvote.VoteStatusT) { + err := p.invUpdate(token, s, 0) + if err != nil { + panic(fmt.Sprintf("invUpdate %v %v: %v", token, s, err)) + } +} + +// invUpdateForBlock updates the inventory for a new best block value. This +// means checking if ongoing ticket votes have finished and updating their +// status if they have. +// +// This function must be called WITHOUT the mtxInv write lock held. func (p *ticketVotePlugin) invUpdateForBlock(bestBlock uint32) (*inventory, error) { p.mtxInv.Lock() defer p.mtxInv.Unlock() @@ -237,34 +269,13 @@ func (p *ticketVotePlugin) invUpdateForBlock(bestBlock uint32) (*inventory, erro return inv, nil } -// inventoryAdd is a wrapper around the invAdd method that allows us to decide -// how disk read/write errors should be handled. For now we just panic. -func (p *ticketVotePlugin) inventoryAdd(token string, s ticketvote.VoteStatusT) { - err := p.invAdd(token, s) - if err != nil { - e := fmt.Sprintf("invAdd %v %v: %v", token, s, err) - panic(e) - } -} - -// inventoryUpdate is a wrapper around the invUpdate method that allows us to -// decide how disk read/write errors should be handled. For now we just panic. -func (p *ticketVotePlugin) inventoryUpdate(token string, s ticketvote.VoteStatusT) { - err := p.invUpdate(token, s, 0) - if err != nil { - e := fmt.Sprintf("invUpdate %v %v: %v", token, s, err) - panic(e) - } -} - // inventoryUpdateToStarted is a wrapper around the invUpdate method that // allows us to decide how disk read/write errors should be handled. For now we // just panic. func (p *ticketVotePlugin) inventoryUpdateToStarted(token string, s ticketvote.VoteStatusT, endHeight uint32) { err := p.invUpdate(token, s, endHeight) if err != nil { - e := fmt.Sprintf("invUpdate %v %v: %v", token, s, err) - panic(e) + panic(fmt.Sprintf("invUpdate %v %v: %v", token, s, err)) } } @@ -293,6 +304,7 @@ type invByStatus struct { BestBlock uint32 } +// invByStatusAll returns a page of token for all vote statuses. func (p *ticketVotePlugin) invByStatusAll(bestBlock, pageSize uint32) (*invByStatus, error) { // Get inventory i, err := p.Inventory(bestBlock) diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/submissions.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/submissions.go index c2d2ffebf..7bbcd055a 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/submissions.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/submissions.go @@ -50,7 +50,7 @@ func (p *ticketVotePlugin) submissionsCachePath(token []byte) (string, error) { // a submissions list does not exist for the token then an empty list will be // returned. // -// This function must be called WITH the lock held. +// This function must be called WITH the mtxSubs lock held. func (p *ticketVotePlugin) submissionsCacheWithLock(token []byte) (*submissions, error) { fp, err := p.submissionsCachePath(token) if err != nil { @@ -79,7 +79,7 @@ func (p *ticketVotePlugin) submissionsCacheWithLock(token []byte) (*submissions, // submissionsCache return the submissions list for a record token. If a linked // from list does not exist for the token then an empty list will be returned. // -// This function must be called WITHOUT the lock held. +// This function must be called WITHOUT the mtxSubs lock held. func (p *ticketVotePlugin) submissionsCache(token []byte) (*submissions, error) { p.mtxSubs.Lock() defer p.mtxSubs.Unlock() @@ -89,7 +89,7 @@ func (p *ticketVotePlugin) submissionsCache(token []byte) (*submissions, error) // submissionsCacheSaveWithLock saves a submissions to the plugin data dir. // -// This function must be called WITH the lock held. +// This function must be called WITH the mtxSubs lock held. func (p *ticketVotePlugin) submissionsCacheSaveWithLock(token []byte, s submissions) error { b, err := json.Marshal(s) if err != nil { @@ -105,7 +105,7 @@ func (p *ticketVotePlugin) submissionsCacheSaveWithLock(token []byte, s submissi // submissionsCacheAdd updates the cached submissions list for the parentToken, // adding the childToken to the list. The full length token MUST be used. // -// This function must be called WITHOUT the lock held. +// This function must be called WITHOUT the mtxSubs lock held. func (p *ticketVotePlugin) submissionsCacheAdd(parentToken, childToken string) error { p.mtxSubs.Lock() defer p.mtxSubs.Unlock() @@ -144,7 +144,7 @@ func (p *ticketVotePlugin) submissionsCacheAdd(parentToken, childToken string) e // submissionsCacheDel updates the cached submissions list for the parentToken, // deleting the childToken from the list. // -// This function must be called WITHOUT the lock held. +// This function must be called WITHOUT the mtxSubs lock held. func (p *ticketVotePlugin) submissionsCacheDel(parentToken, childToken string) error { p.mtxSubs.Lock() defer p.mtxSubs.Unlock() diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/summary.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/summary.go index 56929a6d6..865c8af25 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/summary.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/summary.go @@ -35,9 +35,14 @@ func (p *ticketVotePlugin) summaryCachePath(token string) (string, error) { } var ( + // errSummaryNotFound is returned when a cached summary is not + // found for a record. errSummaryNotFound = errors.New("summary not found") ) +// summaryCache returns the cached vote summary for a record. +// +// This function must be called WITHOUT the mtxSummary lock held. func (p *ticketVotePlugin) summaryCache(token string) (*ticketvote.SummaryReply, error) { p.mtxSummary.Lock() defer p.mtxSummary.Unlock() @@ -65,6 +70,9 @@ func (p *ticketVotePlugin) summaryCache(token string) (*ticketvote.SummaryReply, return &sr, nil } +// summaryCacheSave saves a vote summary to the cache for a record. +// +// This function must be called WITHOUT the mtxSummary lock held. func (p *ticketVotePlugin) summaryCacheSave(token string, sr ticketvote.SummaryReply) error { b, err := json.Marshal(sr) if err != nil { From 765727d3adcf3f52098a10f2b042ff5ddecdf5cc Mon Sep 17 00:00:00 2001 From: Marco Peereboom Date: Wed, 24 Mar 2021 16:01:26 +0000 Subject: [PATCH 431/449] tstore: Key cleanup. --- .../tstorebe/store/localdb/localdb.go | 46 +++---- .../backendv2/tstorebe/store/mysql/encrypt.go | 130 ++++++++++-------- .../tstorebe/store/mysql/encrypt_test.go | 43 ++++++ .../backendv2/tstorebe/store/mysql/mysql.go | 67 ++------- 4 files changed, 142 insertions(+), 144 deletions(-) create mode 100644 politeiad/backendv2/tstorebe/store/mysql/encrypt_test.go diff --git a/politeiad/backendv2/tstorebe/store/localdb/localdb.go b/politeiad/backendv2/tstorebe/store/localdb/localdb.go index 53429df8f..de8bdcf78 100644 --- a/politeiad/backendv2/tstorebe/store/localdb/localdb.go +++ b/politeiad/backendv2/tstorebe/store/localdb/localdb.go @@ -9,7 +9,7 @@ import ( "errors" "fmt" "path/filepath" - "sync" + "sync/atomic" "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" "github.com/decred/politeia/util" @@ -34,35 +34,21 @@ var ( // encryption key is created on startup and saved to the politeiad application // dir. Blobs are encrypted using random 24 byte nonces. type localdb struct { - sync.RWMutex - shutdown bool + shutdown uint64 db *leveldb.DB - - // Encryption key and mutex. The key is zero'd out on application - // exit so the read lock must be held during concurrent access to - // prevent the golang race detector from complaining. - key *[32]byte + key [32]byte } func (l *localdb) isShutdown() bool { - l.RLock() - defer l.RUnlock() - - return l.shutdown + return atomic.LoadUint64(&l.shutdown) != 0 } func (l *localdb) encrypt(data []byte) ([]byte, error) { - l.RLock() - defer l.RUnlock() - - return sbox.Encrypt(0, l.key, data) + return sbox.Encrypt(0, &l.key, data) } func (l *localdb) decrypt(data []byte) ([]byte, uint32, error) { - l.RLock() - defer l.RUnlock() - - return sbox.Decrypt(l.key, data) + return sbox.Decrypt(&l.key, data) } // Put saves the provided key-value pairs to the store. This operation is @@ -185,14 +171,10 @@ func (l *localdb) Get(keys []string) (map[string][]byte, error) { func (l *localdb) Close() { log.Tracef("Close") - l.Lock() - defer l.Unlock() - - l.shutdown = true + atomic.AddUint64(&l.shutdown, 1) // Zero the encryption key util.Zero(l.key[:]) - l.key = nil // Close database l.db.Close() @@ -200,7 +182,7 @@ func (l *localdb) Close() { // New returns a new localdb. func New(appDir, dataDir string) (*localdb, error) { - // Load encryption key + // Load encryption key. keyFile := filepath.Join(appDir, encryptionKeyFilename) key, err := util.LoadEncryptionKey(log, keyFile) if err != nil { @@ -213,8 +195,12 @@ func New(appDir, dataDir string) (*localdb, error) { return nil, err } - return &localdb{ - db: db, - key: key, - }, nil + // Create context + ldb := localdb{ + db: db, + } + copy(ldb.key[:], key[:]) + util.Zero(key[:]) + + return &ldb, nil } diff --git a/politeiad/backendv2/tstorebe/store/mysql/encrypt.go b/politeiad/backendv2/tstorebe/store/mysql/encrypt.go index b0a5e45f9..a8c3904da 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/encrypt.go +++ b/politeiad/backendv2/tstorebe/store/mysql/encrypt.go @@ -32,90 +32,95 @@ type argon2idParams struct { Salt []byte `json:"salt"` } +func newArgon2Params() argon2idParams { + salt, err := util.Random(16) + if err != nil { + panic(err) + } + return argon2idParams{ + Time: 1, + Memory: 64 * 1024, // In KiB + Threads: 4, + KeyLen: 32, + Salt: salt, + } +} + // argon2idKey derives a 32 byte key from the provided password using the // Aragon2id key derivation function. A random 16 byte salt is created the // first time the key is derived. The salt and the other argon2id params are // saved to the kv store. Subsequent calls to this fuction will pull the // existing salt and params from the kv store and use them to derive the key. -func (s *mysql) argon2idKey(password string) (*[32]byte, error) { +func (s *mysql) argon2idKey(password string) error { log.Infof("Deriving encryption key from password") - // Check if a key already exists - blobs, err := s.Get([]string{argon2idKey}) - if err != nil { - return nil, fmt.Errorf("get: %v", err) + // Check if a key already exists. If db is nil then we are running unit + // tests. + var ( + blobs map[string][]byte + err error + ) + if s.db == nil { + // Testing mode + blobs = make(map[string][]byte) + } else { + blobs, err = s.Get([]string{argon2idKey}) + if err != nil { + return fmt.Errorf("get: %v", err) + } } - var salt []byte - var wasFound bool + + var ( + save bool + ap argon2idParams + ) b, ok := blobs[argon2idKey] if ok { - // Key already exists. Use the existing salt. - log.Infof("Encryption key salt already exists") - - var ap argon2idParams + log.Debugf("Encryption key salt already exists") err = json.Unmarshal(b, &ap) if err != nil { - return nil, err + return err } - - salt = ap.Salt - wasFound = true } else { - // Key does not exist. Create a random 16 byte salt. - log.Infof("Encryption key salt not found; creating a new one") - - salt, err = util.Random(16) - if err != nil { - return nil, err - } + log.Infof("Encryption key not found; creating a new one") + ap = newArgon2Params() + save = true } // Derive key - var ( - pass = []byte(password) - time uint32 = 1 - memory uint32 = 64 * 1024 // 64 MB - threads uint8 = 4 // Number of available CPUs - keyLen uint32 = 32 // In bytes - ) - k := argon2.IDKey(pass, salt, time, memory, threads, keyLen) - var key [32]byte - copy(key[:], k) + k := argon2.IDKey([]byte(password), ap.Salt, ap.Time, ap.Memory, + ap.Threads, ap.KeyLen) + copy(s.key[:], k) util.Zero(k) // Save params to the kv store if this is the first time the key // was derived. - if !wasFound { - ap := argon2idParams{ - Time: time, - Memory: memory, - Threads: threads, - KeyLen: keyLen, - Salt: salt, - } + if save && s.db != nil { b, err := json.Marshal(ap) if err != nil { - return nil, err + return err } kv := map[string][]byte{ argon2idKey: b, } err = s.Put(kv, false) if err != nil { - return nil, fmt.Errorf("put: %v", err) + return fmt.Errorf("put: %v", err) } log.Infof("Encryption key derivation params saved to kv store") } - return &key, nil + return nil } -func (s *mysql) encrypt(ctx context.Context, tx *sql.Tx, key *[32]byte, data []byte) ([]byte, error) { +var emptyNonce = [24]byte{} + +func (s *mysql) getDbNonce(ctx context.Context, tx *sql.Tx) ([24]byte, error) { // Get nonce value nonce, err := s.nonce(ctx, tx) if err != nil { - return nil, err + return emptyNonce, err } log.Tracef("Encrypting with nonce: %v", nonce) @@ -125,27 +130,30 @@ func (s *mysql) encrypt(ctx context.Context, tx *sql.Tx, key *[32]byte, data []b binary.LittleEndian.PutUint64(b, uint64(nonce)) n, err := sbox.NewNonceFromBytes(b) if err != nil { - return nil, err + return emptyNonce, err } - nonceb := n.Current() - - // The encryption key is zero'd out on application exit so the read - // lock must be held during concurrent access to prevent the golang - // race detector from complaining. - s.RLock() - defer s.RUnlock() + return n.Current(), nil +} - return sbox.EncryptN(0, key, nonceb, data) +func (s *mysql) getTestNonce(ctx context.Context, tx *sql.Tx) ([24]byte, error) { + nonce, err := util.Random(8) + n, err := sbox.NewNonceFromBytes(nonce) + if err != nil { + return emptyNonce, err + } + return n.Current(), nil } -func (s *mysql) decrypt(key *[32]byte, data []byte) ([]byte, uint32, error) { - // The encryption key is zero'd out on application exit so the read - // lock must be held during concurrent access to prevent the golang - // race detector from complaining. - s.RLock() - defer s.RUnlock() +func (s *mysql) encrypt(ctx context.Context, tx *sql.Tx, data []byte) ([]byte, error) { + nonce, err := s.getNonce(ctx, tx) + if err != nil { + return nil, err + } + return sbox.EncryptN(0, &s.key, nonce, data) +} - return sbox.Decrypt(key, data) +func (s *mysql) decrypt(data []byte) ([]byte, uint32, error) { + return sbox.Decrypt(&s.key, data) } // isEncrypted returns whether the provided blob has been prefixed with an sbox diff --git a/politeiad/backendv2/tstorebe/store/mysql/encrypt_test.go b/politeiad/backendv2/tstorebe/store/mysql/encrypt_test.go new file mode 100644 index 000000000..86019531a --- /dev/null +++ b/politeiad/backendv2/tstorebe/store/mysql/encrypt_test.go @@ -0,0 +1,43 @@ +package mysql + +import ( + "bytes" + "testing" +) + +func TestEncryptDecrypt(t *testing.T) { + password := "passwordsosikrit" + blob := []byte("encryptmeyo") + + // setup fake context + s := &mysql{} + s.getNonce = s.getTestNonce + err := s.argon2idKey(password) + if err != nil { + t.Fatal(err) + } + + // Encrypt and make sure cleartext isn't the same as the encypted blob. + eb, err := s.encrypt(nil, nil, blob) + if err != nil { + t.Fatal(err) + } + if bytes.Equal(eb, blob) { + t.Fatal("equal") + } + + // Decrypt and make sure cleartext is the same as the initial blob. + db, _, err := s.decrypt(eb) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(db, blob) { + t.Fatal("not equal") + } + + // Try to decrypt invalid blob. + _, _, err = s.decrypt(blob) + if err == nil { + t.Fatal("expected invalid sbox header") + } +} diff --git a/politeiad/backendv2/tstorebe/store/mysql/mysql.go b/politeiad/backendv2/tstorebe/store/mysql/mysql.go index 0f84b5abc..fff36fce2 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/mysql.go +++ b/politeiad/backendv2/tstorebe/store/mysql/mysql.go @@ -8,7 +8,7 @@ import ( "context" "database/sql" "fmt" - "sync" + "sync/atomic" "time" "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" @@ -46,14 +46,10 @@ var ( // mysql implements the store BlobKV interface using a mysql driver. type mysql struct { - sync.RWMutex + shutdown uint64 db *sql.DB - shutdown bool - - // Encryption key. The key is zero'd out on application exit so the - // read lock must be held during concurrent access to prevent the - // golang race detector from complaining. - key *[32]byte + getNonce func(context.Context, *sql.Tx) ([24]byte, error) + key [32]byte } func ctxWithTimeout() (context.Context, func()) { @@ -61,39 +57,14 @@ func ctxWithTimeout() (context.Context, func()) { } func (s *mysql) isShutdown() bool { - s.RLock() - defer s.RUnlock() - - return s.shutdown -} - -func (s *mysql) getKey() (*[32]byte, error) { - s.RLock() - defer s.RUnlock() - - if s.key == nil { - return nil, fmt.Errorf("encryption key not found") - } - - return s.key, nil -} - -func (s *mysql) setKey(key *[32]byte) { - s.Lock() - defer s.Unlock() - - s.key = key + return atomic.LoadUint64(&s.shutdown) != 0 } func (s *mysql) put(blobs map[string][]byte, encrypt bool, ctx context.Context, tx *sql.Tx) error { // Encrypt blobs if encrypt { - key, err := s.getKey() - if err != nil { - return err - } for k, v := range blobs { - e, err := s.encrypt(ctx, tx, key, v) + e, err := s.encrypt(ctx, tx, v) if err != nil { return fmt.Errorf("encrypt: %v", err) } @@ -275,11 +246,7 @@ func (s *mysql) Get(keys []string) (map[string][]byte, error) { if !encrypted { continue } - key, err := s.getKey() - if err != nil { - return nil, err - } - b, _, err := s.decrypt(key, v) + b, _, err := s.decrypt(v) if err != nil { return nil, fmt.Errorf("decrypt: %v", err) } @@ -293,16 +260,10 @@ func (s *mysql) Get(keys []string) (map[string][]byte, error) { func (s *mysql) Close() { log.Tracef("Close") - s.Lock() - defer s.Unlock() - - s.shutdown = true + atomic.AddUint64(&s.shutdown, 1) // Zero the encryption key - if s.key != nil { - util.Zero(s.key[:]) - s.key = nil - } + util.Zero(s.key[:]) // Close mysql connection s.db.Close() @@ -351,16 +312,16 @@ func New(appDir, host, user, password, dbname string) (*mysql, error) { } // Setup mysql context - s := mysql{ + s := &mysql{ db: db, } + s.getNonce = s.getDbNonce - // Derive encryption key from password - key, err := s.argon2idKey(password) + // Derive encryption key from password. Key is set in argon2idKey + err = s.argon2idKey(password) if err != nil { return nil, fmt.Errorf("argon2idKey: %v", err) } - s.setKey(key) - return &s, nil + return s, nil } From 349ef3a4664bd1381deafa2a7602742f8969824e Mon Sep 17 00:00:00 2001 From: Marco Peereboom Date: Wed, 24 Mar 2021 17:55:45 +0000 Subject: [PATCH 432/449] multi: Clean up several PR change requests. --- .../backendv2/tstorebe/plugins/pi/hooks.go | 43 +- .../plugins/ticketvote/activevotes.go | 16 +- .../tstorebe/plugins/ticketvote/cmds.go | 378 ++++++++++-------- .../backendv2/tstorebe/tstore/recordindex.go | 100 ++--- 4 files changed, 299 insertions(+), 238 deletions(-) diff --git a/politeiad/backendv2/tstorebe/plugins/pi/hooks.go b/politeiad/backendv2/tstorebe/plugins/pi/hooks.go index 9675515ab..4b1db4faf 100644 --- a/politeiad/backendv2/tstorebe/plugins/pi/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/pi/hooks.go @@ -78,8 +78,9 @@ func (p *piPlugin) hookEditRecordPre(payload string) error { return backend.PluginError{ PluginID: pi.PluginID, ErrorCode: uint32(pi.ErrorCodeVoteStatusInvalid), - ErrorContext: fmt.Sprintf("vote status '%v' does not allow "+ - "for proposal edits", ticketvote.VoteStatuses[s.Status]), + ErrorContext: fmt.Sprintf("vote status '%v' "+ + "does not allow for proposal edits", + ticketvote.VoteStatuses[s.Status]), } } } @@ -168,8 +169,10 @@ func (p *piPlugin) proposalFilesVerify(files []backend.File) error { return backend.PluginError{ PluginID: pi.PluginID, ErrorCode: uint32(pi.ErrorCodeTextFileSizeInvalid), - ErrorContext: fmt.Sprintf("file %v size %v exceeds max size %v", - v.Name, len(payload), p.textFileSizeMax), + ErrorContext: fmt.Sprintf("file %v "+ + "size %v exceeds max size %v", + v.Name, len(payload), + p.textFileSizeMax), } } @@ -181,8 +184,10 @@ func (p *piPlugin) proposalFilesVerify(files []backend.File) error { return backend.PluginError{ PluginID: pi.PluginID, ErrorCode: uint32(pi.ErrorCodeImageFileSizeInvalid), - ErrorContext: fmt.Sprintf("image %v size %v exceeds max size %v", - v.Name, len(payload), p.imageFileSizeMax), + ErrorContext: fmt.Sprintf("image %v "+ + "size %v exceeds max size %v", + v.Name, len(payload), + p.imageFileSizeMax), } } @@ -212,8 +217,8 @@ func (p *piPlugin) proposalFilesVerify(files []backend.File) error { return backend.PluginError{ PluginID: pi.PluginID, ErrorCode: uint32(pi.ErrorCodeImageFileCountInvalid), - ErrorContext: fmt.Sprintf("got %v image files, max is %v", - imagesCount, p.imageFileCountMax), + ErrorContext: fmt.Sprintf("got %v image files, max "+ + "is %v", imagesCount, p.imageFileCountMax), } } @@ -296,19 +301,17 @@ func proposalMetadataDecode(files []backend.File) (*pi.ProposalMetadata, error) if v.Name != pi.FileNameProposalMetadata { continue } - if v.Name == pi.FileNameProposalMetadata { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return nil, err - } - var m pi.ProposalMetadata - err = json.Unmarshal(b, &m) - if err != nil { - return nil, err - } - propMD = &m - break + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return nil, err + } + var m pi.ProposalMetadata + err = json.Unmarshal(b, &m) + if err != nil { + return nil, err } + propMD = &m + break } return propMD, nil } diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/activevotes.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/activevotes.go index 53acc9783..b1228e89e 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/activevotes.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/activevotes.go @@ -59,10 +59,12 @@ type activeVote struct { // provided token. If the token does not correspond to an active vote then nil // is returned. func (a *activeVotes) VoteDetails(token []byte) *ticketvote.VoteDetails { + t := hex.EncodeToString(token) + a.RLock() defer a.RUnlock() - av, ok := a.activeVotes[hex.EncodeToString(token)] + av, ok := a.activeVotes[t] if !ok { return nil } @@ -101,10 +103,12 @@ func (a *activeVotes) VoteDetails(token []byte) *ticketvote.VoteDetails { // the provided token. If the token does not correspond to an active vote then // nil is returned. func (a *activeVotes) EligibleTickets(token []byte) map[string]struct{} { + t := hex.EncodeToString(token) + a.RLock() defer a.RUnlock() - av, ok := a.activeVotes[hex.EncodeToString(token)] + av, ok := a.activeVotes[t] if !ok { return nil } @@ -144,16 +148,17 @@ func (a *activeVotes) CommitmentAddrs(token []byte, tickets []string) map[string return map[string]commitmentAddr{} } + t := hex.EncodeToString(token) + ca := make(map[string]commitmentAddr, len(tickets)) + a.RLock() defer a.RUnlock() - t := hex.EncodeToString(token) av, ok := a.activeVotes[t] if !ok { return nil } - ca := make(map[string]commitmentAddr, len(tickets)) for _, v := range tickets { addr, ok := av.Addrs[v] if ok { @@ -170,10 +175,11 @@ func (a *activeVotes) CommitmentAddrs(token []byte, tickets []string) map[string // vote. The returned map is a map[votebit]tally. An empty map is returned if // the requested token is not in the active votes cache. func (a *activeVotes) Tally(token string) map[string]uint32 { + tally := make(map[string]uint32, 16) + a.RLock() defer a.RUnlock() - tally := make(map[string]uint32, 16) av, ok := a.activeVotes[token] if !ok { return tally diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go index 4ead1ba00..ac1ae6515 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go @@ -72,9 +72,10 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri // This is allowed default: return "", backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeAuthorizationInvalid), - ErrorContext: fmt.Sprintf("%v not a valid action", a.Action), + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeAuthorizationInvalid), + ErrorContext: fmt.Sprintf("%v not a valid action", + a.Action), } } @@ -94,8 +95,9 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri return "", backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeRecordVersionInvalid), - ErrorContext: fmt.Sprintf("version is not latest: got %v, want %v", - a.Version, r.RecordMetadata.Version), + ErrorContext: fmt.Sprintf("version is not latest: "+ + "got %v, want %v", a.Version, + r.RecordMetadata.Version), } } @@ -114,9 +116,10 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri // No previous actions. New action must be an authorize. if a.Action != ticketvote.AuthActionAuthorize { return "", backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeAuthorizationInvalid), - ErrorContext: "no prev action; action must be authorize", + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeAuthorizationInvalid), + ErrorContext: "no prev action; action must " + + "be authorize", } } case prevAction == ticketvote.AuthActionAuthorize && @@ -228,29 +231,29 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM return backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeVoteDurationInvalid), - ErrorContext: fmt.Sprintf("duration %v exceeds max duration %v", - vote.Duration, voteDurationMax), + ErrorContext: fmt.Sprintf("duration %v exceeds max "+ + "duration %v", vote.Duration, voteDurationMax), } case vote.Duration < voteDurationMin: return backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeVoteDurationInvalid), - ErrorContext: fmt.Sprintf("duration %v under min duration %v", - vote.Duration, voteDurationMin), + ErrorContext: fmt.Sprintf("duration %v under min "+ + "duration %v", vote.Duration, voteDurationMin), } case vote.QuorumPercentage > 100: return backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeVoteQuorumInvalid), - ErrorContext: fmt.Sprintf("quorum percent %v exceeds 100 percent", - vote.QuorumPercentage), + ErrorContext: fmt.Sprintf("quorum percent %v exceeds "+ + "100 percent", vote.QuorumPercentage), } case vote.PassPercentage > 100: return backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeVotePassRateInvalid), - ErrorContext: fmt.Sprintf("pass percent %v exceeds 100 percent", - vote.PassPercentage), + ErrorContext: fmt.Sprintf("pass percent %v exceeds "+ + "100 percent", vote.PassPercentage), } } @@ -272,7 +275,8 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM return backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeVoteOptionsInvalid), - ErrorContext: fmt.Sprintf("vote options count got %v, want 2", + ErrorContext: fmt.Sprintf("vote options "+ + "count got %v, want 2", len(vote.Options)), } } @@ -300,7 +304,8 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM return backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeVoteOptionsInvalid), - ErrorContext: fmt.Sprintf("vote option IDs not found: %v", + ErrorContext: fmt.Sprintf("vote option IDs "+ + "not found: %v", strings.Join(missing, ",")), } } @@ -322,17 +327,19 @@ func voteParamsVerify(vote ticketvote.VoteParams, voteDurationMin, voteDurationM switch { case vote.Type == ticketvote.VoteTypeStandard && vote.Parent != "": return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), - ErrorContext: "parent token should not be provided for a standard vote", + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), + ErrorContext: "parent token should not be provided " + + "for a standard vote", } case vote.Type == ticketvote.VoteTypeRunoff: _, err := tokenDecode(vote.Parent) if err != nil { return backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), - ErrorContext: fmt.Sprintf("invalid parent %v", vote.Parent), + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), + ErrorContext: fmt.Sprintf("invalid parent %v", + vote.Parent), } } } @@ -375,7 +382,8 @@ func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, return nil, err } if bdr.Block.Hash == "" { - return nil, fmt.Errorf("invalid block hash for height %v", snapshotHeight) + return nil, fmt.Errorf("invalid block hash for height %v", + snapshotHeight) } snapshotHash := bdr.Block.Hash @@ -422,9 +430,10 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot // Verify there is only one start details if len(s.Starts) != 1 { return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeStartDetailsInvalid), - ErrorContext: "more than one start details found for standard vote", + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeStartDetailsInvalid), + ErrorContext: "more than one start details found for " + + "standard vote", } } sd := s.Starts[0] @@ -471,8 +480,9 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot return nil, backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeRecordVersionInvalid), - ErrorContext: fmt.Sprintf("version is not latest: got %v, want %v", - sd.Params.Version, r.RecordMetadata.Version), + ErrorContext: fmt.Sprintf("version is not latest: "+ + "got %v, want %v", sd.Params.Version, + r.RecordMetadata.Version), } } @@ -647,8 +657,9 @@ func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs sta return backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeRecordVersionInvalid), - ErrorContext: fmt.Sprintf("version is not latest %v: got %v, want %v", - sd.Params.Token, sd.Params.Version, r.RecordMetadata.Version), + ErrorContext: fmt.Sprintf("version is not latest %v: "+ + "got %v, want %v", sd.Params.Token, + sd.Params.Version, r.RecordMetadata.Version), } } @@ -670,8 +681,8 @@ func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs sta } // Update inventory - p.inventoryUpdateToStarted(vd.Params.Token, ticketvote.VoteStatusStarted, - vd.EndBlockHeight) + p.inventoryUpdateToStarted(vd.Params.Token, + ticketvote.VoteStatusStarted, vd.EndBlockHeight) // Update active votes cache p.activeVotesAdd(vd) @@ -716,9 +727,10 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti if err != nil { if errors.Is(err, backend.ErrRecordNotFound) { return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), - ErrorContext: fmt.Sprintf("parent record not found %x", token), + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), + ErrorContext: fmt.Sprintf("parent record not "+ + "found %x", token), } } return nil, fmt.Errorf("RecordPartial: %v", err) @@ -733,9 +745,10 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti } if vm == nil || vm.LinkBy == 0 { return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), - ErrorContext: fmt.Sprintf("%x is not a runoff vote parent", token), + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), + ErrorContext: fmt.Sprintf("%x is not a runoff vote "+ + "parent", token), } } isExpired := vm.LinkBy < time.Now().Unix() @@ -745,8 +758,8 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti return nil, backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeLinkByNotExpired), - ErrorContext: fmt.Sprintf("parent record %x linkby deadline not met %v", - token, vm.LinkBy), + ErrorContext: fmt.Sprintf("parent record %x linkby "+ + "deadline not met %v", token, vm.LinkBy), } case !isExpired: // Allow the vote to be started before the link by deadline @@ -794,8 +807,8 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti return nil, backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeStartDetailsInvalid), - ErrorContext: fmt.Sprintf("record %v should not be included", - v.Params.Token), + ErrorContext: fmt.Sprintf("record %v should "+ + "not be included", v.Params.Token), } } } @@ -868,42 +881,47 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeVoteTypeInvalid), ErrorContext: fmt.Sprintf("%v got %v, want %v", - v.Params.Token, v.Params.Type, ticketvote.VoteTypeRunoff), + v.Params.Token, v.Params.Type, + ticketvote.VoteTypeRunoff), } case v.Params.Mask != mask: return nil, backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeVoteBitsInvalid), - ErrorContext: fmt.Sprintf("%v mask invalid: all must be the same", - v.Params.Token), + ErrorContext: fmt.Sprintf("%v mask invalid: "+ + "all must be the same", v.Params.Token), } case v.Params.Duration != duration: return nil, backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeVoteDurationInvalid), - ErrorContext: fmt.Sprintf("%v duration does not match; "+ - "all must be the same", v.Params.Token), + ErrorContext: fmt.Sprintf("%v duration does "+ + "not match; all must be the same", + v.Params.Token), } case v.Params.QuorumPercentage != quorum: return nil, backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeVoteQuorumInvalid), - ErrorContext: fmt.Sprintf("%v quorum does not match; "+ - "all must be the same", v.Params.Token), + ErrorContext: fmt.Sprintf("%v quorum does "+ + "not match; all must be the same", + v.Params.Token), } case v.Params.PassPercentage != pass: return nil, backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeVotePassRateInvalid), - ErrorContext: fmt.Sprintf("%v pass rate does not match; "+ - "all must be the same", v.Params.Token), + ErrorContext: fmt.Sprintf("%v pass rate does "+ + "not match; all must be the same", + v.Params.Token), } case v.Params.Parent != parent: return nil, backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), - ErrorContext: fmt.Sprintf("%v parent does not match; "+ - "all must be the same", v.Params.Token), + ErrorContext: fmt.Sprintf("%v parent does "+ + "not match; all must be the same", + v.Params.Token), } } @@ -921,9 +939,10 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. _, err = tokenDecode(v.Params.Parent) if err != nil { return nil, backend.PluginError{ - PluginID: ticketvote.PluginID, - ErrorCode: uint32(ticketvote.ErrorCodeTokenInvalid), - ErrorContext: fmt.Sprintf("parent token %v", v.Params.Parent), + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeTokenInvalid), + ErrorContext: fmt.Sprintf("parent token %v", + v.Params.Parent), } } @@ -940,7 +959,8 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. // Verify vote options and params. Vote optoins are required to // be approve and reject. - err = voteParamsVerify(v.Params, p.voteDurationMin, p.voteDurationMax) + err = voteParamsVerify(v.Params, p.voteDurationMin, + p.voteDurationMax) if err != nil { return nil, err } @@ -951,21 +971,20 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. return nil, backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeVoteParentInvalid), - ErrorContext: fmt.Sprintf("runoff vote must be started on "+ - "the parent record %v", parent), + ErrorContext: fmt.Sprintf("runoff vote must be "+ + "started on the parent record %v", parent), } } // This function is being invoked on the runoff vote parent record. - // Create and save a start runoff record onto the parent record's - // tree. + // Create and save a start runoff record onto the parent record's tree. srr, err := p.startRunoffForParent(treeID, token, s) if err != nil { return nil, err } - // Start the voting period of each runoff vote submissions by - // using the internal plugin command startRunoffSubmission. + // Start the voting period of each runoff vote submissions by using the + // internal plugin command startRunoffSubmission. for _, v := range s.Starts { token, err = tokenDecode(v.Params.Token) if err != nil { @@ -987,7 +1006,8 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. return nil, err } return nil, fmt.Errorf("PluginWrite %x %v %v: %v", - token, ticketvote.PluginID, cmdStartRunoffSubmission, err) + token, ticketvote.PluginID, + cmdStartRunoffSubmission, err) } } @@ -1197,7 +1217,8 @@ func (p *ticketVotePlugin) largestCommitmentAddrs(tickets []string) (map[string] } } if bestAddr == "" || bestAmt == 0.0 { - addrErr = fmt.Errorf("no largest commitment address found") + addrErr = fmt.Errorf("no largest commitment address " + + "found") } // Store result @@ -1280,13 +1301,16 @@ func (p *ticketVotePlugin) ballot(treeID int64, votes []ticketvote.CastVote, res // Save cast vote err := p.castVoteSave(treeID, cv) if err == plugins.ErrDuplicateBlob { - // This cast vote has already been saved. Its possible that - // a previous attempt to vote with this ticket failed before - // the vote collider could be saved. Continue execution so - // that we re-attempt to save the vote collider. + // This cast vote has already been saved. Its + // possible that a previous attempt to vote + // with this ticket failed before the vote + // collider could be saved. Continue execution + // so that we re-attempt to save the vote + // collider. } else if err != nil { t := time.Now().Unix() - log.Errorf("cmdCastBallot: castVoteSave %v: %v", t, err) + log.Errorf("cmdCastBallot: castVoteSave %v: "+ + "%v", t, err) e := ticketvote.VoteErrorInternalError cvr.Ticket = v.Ticket cvr.ErrorCode = e @@ -1391,16 +1415,16 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str e := ticketvote.VoteErrorVoteStatusInvalid receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e - receipts[k].ErrorContext = fmt.Sprintf("%v: vote is not active", - ticketvote.VoteErrors[e]) + receipts[k].ErrorContext = fmt.Sprintf("%v: vote is "+ + "not active", ticketvote.VoteErrors[e]) continue } if voteHasEnded(bestBlock, voteDetails.EndBlockHeight) { e := ticketvote.VoteErrorVoteStatusInvalid receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e - receipts[k].ErrorContext = fmt.Sprintf("%v: vote has ended", - ticketvote.VoteErrors[e]) + receipts[k].ErrorContext = fmt.Sprintf("%v: vote has "+ + "ended", ticketvote.VoteErrors[e]) continue } @@ -1493,8 +1517,8 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str commitmentAddr, ok := addrs[v.Ticket] if !ok { t := time.Now().Unix() - log.Errorf("cmdCastBallot: commitment addr not found %v: %v", - t, v.Ticket) + log.Errorf("cmdCastBallot: commitment addr not found "+ + "%v: %v", t, v.Ticket) e := ticketvote.VoteErrorInternalError receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e @@ -1504,8 +1528,8 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str } if commitmentAddr.err != nil { t := time.Now().Unix() - log.Errorf("cmdCastBallot: commitment addr error %v: %v %v", - t, v.Ticket, commitmentAddr.err) + log.Errorf("cmdCastBallot: commitment addr error %v: "+ + "%v %v", t, v.Ticket, commitmentAddr.err) e := ticketvote.VoteErrorInternalError receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e @@ -1525,31 +1549,30 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str } // The votes that have passed validation will be cast in batches of - // size batchSize. Each batch of votes is cast concurrently in order - // to accommodate the trillian log signer bottleneck. The log signer - // picks up queued leaves and appends them onto the trillian tree - // every xxx ms, where xxx is a configurable value on the log signer, - // but is typically a few hundred milliseconds. Lets use 200ms as an - // example. If we don't cast the votes in batches then every vote in - // the ballot will take 200 milliseconds since we wait for the leaf - // to be fully appended before considering the trillian call - // successful. A person casting hundreds of votes in a single ballot - // would cause UX issues for all the voting clients since the backend - // locks the record during any plugin write calls. Only one ballot - // can be cast at a time. + // size batchSize. Each batch of votes is cast concurrently in order to + // accommodate the trillian log signer bottleneck. The log signer picks + // up queued leaves and appends them onto the trillian tree every xxx + // ms, where xxx is a configurable value on the log signer, but is + // typically a few hundred milliseconds. Lets use 200ms as an example. + // If we don't cast the votes in batches then every vote in the ballot + // will take 200 milliseconds since we wait for the leaf to be fully + // appended before considering the trillian call successful. A person + // casting hundreds of votes in a single ballot would cause UX issues + // for all the voting clients since the backend locks the record during + // any plugin write calls. Only one ballot can be cast at a time. // // The second variable that we must watch out for is the max trillian // queued leaf batch size. This is also a configurable trillian value - // that represents the maximum number of leaves that can be waiting - // in the queue for all trees in the trillian instance. This value is + // that represents the maximum number of leaves that can be waiting in + // the queue for all trees in the trillian instance. This value is // typically around the order of magnitude of 1000s of queued leaves. // // The third variable that can cause errors is reaching the trillian // datastore max connection limits. Each vote being cast creates a - // trillian connection. Overloading the trillian connections can - // cause max connection exceeded errors. The max allowed connections - // is a configurable trillian value, but should also be adjusted on - // the key-value store database itself as well. + // trillian connection. Overloading the trillian connections can cause + // max connection exceeded errors. The max allowed connections is a + // configurable trillian value, but should also be adjusted on the + // key-value store database itself as well. // // This is why a vote batch size of 10 was chosen. It is large enough // to alleviate performance bottlenecks from the log signer interval, @@ -1560,10 +1583,11 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str var ( batchSize = 10 batch = make([]ticketvote.CastVote, 0, batchSize) - queue = make([][]ticketvote.CastVote, 0, len(votes)/batchSize) + queue = make([][]ticketvote.CastVote, 0, + len(votes)/batchSize) - // ballotCount is the number of votes that have passed validation - // and are being cast in this ballot. + // ballotCount is the number of votes that have passed + // validation and are being cast in this ballot. ballotCount int ) for k, v := range votes { @@ -1577,8 +1601,8 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str ballotCount++ if len(batch) == batchSize { - // This batch is full. Add the batch to the queue and start - // a new batch. + // This batch is full. Add the batch to the queue and + // start a new batch. queue = append(queue, batch) batch = make([]ticketvote.CastVote, 0, batchSize) } @@ -1594,7 +1618,8 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str // Cast ballot in batches results := make(chan ticketvote.CastVoteReply, ballotCount) for i, batch := range queue { - log.Debugf("Casting %v votes in batch %v/%v", len(batch), i+1, len(queue)) + log.Debugf("Casting %v votes in batch %v/%v", len(batch), i+1, + len(queue)) p.ballot(treeID, batch, results) } @@ -1607,7 +1632,8 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str } if len(r) != ballotCount { - log.Errorf("Missing results: got %v, want %v", len(r), ballotCount) + log.Errorf("Missing results: got %v, want %v", len(r), + ballotCount) } // Fill in the receipts @@ -1619,7 +1645,8 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str cvr, ok := r[v.Ticket] if !ok { t := time.Now().Unix() - log.Errorf("cmdCastBallot: vote result not found %v: %v", t, v.Ticket) + log.Errorf("cmdCastBallot: vote result not found %v: "+ + "%v", t, v.Ticket) e := ticketvote.VoteErrorInternalError receipts[k].Ticket = v.Ticket receipts[k].ErrorCode = e @@ -1810,7 +1837,8 @@ func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload str } ts, err := p.timestamp(treeID, v) if err != nil { - return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) + return "", fmt.Errorf("timestamp %x %x: %v", + token, v, err) } votes = append(votes, *ts) if len(votes) == int(pageSize) { @@ -1820,7 +1848,8 @@ func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload str } default: - // Return authorization timestamps and the vote details timestamp. + // Return authorization timestamps and the vote details + // timestamp. // Auth timestamps digests, err := p.tstore.DigestsByDataDesc(treeID, @@ -1833,7 +1862,8 @@ func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload str for _, v := range digests { ts, err := p.timestamp(treeID, v) if err != nil { - return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) + return "", fmt.Errorf("timestamp %x %x: %v", + token, v, err) } auths = append(auths, *ts) } @@ -1847,13 +1877,14 @@ func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload str } // There should never be more than a one vote details if len(digests) > 1 { - return "", fmt.Errorf("invalid vote details count: got %v, want 1", - len(digests)) + return "", fmt.Errorf("invalid vote details count: "+ + "got %v, want 1", len(digests)) } for _, v := range digests { ts, err := p.timestamp(treeID, v) if err != nil { - return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) + return "", fmt.Errorf("timestamp %x %x: %v", + token, v, err) } details = ts } @@ -2012,15 +2043,18 @@ func (p *ticketVotePlugin) voteResults(treeID int64) ([]ticketvote.CastVoteDetai } // Decode blobs. A cast vote is considered valid only if the vote - // collider exists for it. If there are multiple votes using the - // same ticket, the valid vote is the one that immediately precedes - // the vote collider blob entry. + // collider exists for it. If there are multiple votes using the same + // ticket, the valid vote is the one that immediately precedes the vote + // collider blob entry. var ( // map[ticket]CastVoteDetails votes = make(map[string]ticketvote.CastVoteDetails, len(blobs)) - voteIndexes = make(map[string][]int, len(blobs)) // map[ticket][]index - colliderIndexes = make(map[string]int, len(blobs)) // map[ticket]index + // map[ticket][]index + voteIndexes = make(map[string][]int, len(blobs)) + + // map[ticket]index + colliderIndexes = make(map[string]int, len(blobs)) ) for i, v := range blobs { // Decode data hint @@ -2062,14 +2096,16 @@ func (p *ticketVotePlugin) voteResults(treeID int64) ([]ticketvote.CastVoteDetai // Sanity check _, ok := colliderIndexes[vc.Ticket] if ok { - return nil, fmt.Errorf("duplicate vote colliders found %v", vc.Ticket) + return nil, fmt.Errorf("duplicate vote "+ + "colliders found %v", vc.Ticket) } // Save the ticket and index for the collider colliderIndexes[vc.Ticket] = i default: - return nil, fmt.Errorf("invalid data descriptor: %v", dd.Descriptor) + return nil, fmt.Errorf("invalid data descriptor: %v", + dd.Descriptor) } } @@ -2085,21 +2121,24 @@ func (p *ticketVotePlugin) voteResults(treeID int64) ([]ticketvote.CastVoteDetai // If multiple votes have been cast using the same ticket then // we must manually determine which vote is valid. if len(indexes) == 1 { - // Only one cast vote exists for this ticket. This is good. + // Only one cast vote exists for this ticket. This is + // good. continue } // Sanity check if len(indexes) == 0 { - return nil, fmt.Errorf("no cast vote index found %v", ticket) + return nil, fmt.Errorf("no cast vote index found %v", + ticket) } - log.Tracef("Multiple votes found for a single vote collider %v", ticket) + log.Tracef("Multiple votes found for a single vote collider %v", + ticket) // Multiple votes exist for this ticket. The vote that is valid - // is the one that immediately precedes the vote collider. Start - // at the end of the vote indexes and find the first vote index - // that precedes the collider index. + // is the one that immediately precedes the vote collider. + // Start at the end of the vote indexes and find the first vote + // index that precedes the collider index. var validVoteIndex int for i := len(indexes) - 1; i >= 0; i-- { voteIndex := indexes[i] @@ -2136,9 +2175,8 @@ func (p *ticketVotePlugin) voteResults(treeID int64) ([]ticketvote.CastVoteDetai // voteOptionResults tallies the results of a ticket vote and returns a // VoteOptionResult for each vote option in the ticket vote. func (p *ticketVotePlugin) voteOptionResults(token []byte, options []ticketvote.VoteOption) ([]ticketvote.VoteOptionResult, error) { - // Ongoing votes will have the cast votes cached. Calculate the - // results using the cached votes if we can since it will be much - // faster. + // Ongoing votes will have the cast votes cached. Calculate the results + // using the cached votes if we can since it will be much faster. var ( tally = make(map[string]uint32, len(options)) t = hex.EncodeToString(token) @@ -2146,7 +2184,7 @@ func (p *ticketVotePlugin) voteOptionResults(token []byte, options []ticketvote. ) switch { case len(ctally) > 0: - // Vote are in the cache. Use the cached results. + // Votes are in the cache. Use the cached results. tally = ctally default: @@ -2212,9 +2250,14 @@ func (p *ticketVotePlugin) summariesForRunoff(parentToken string) (map[string]ti // Compile summaries for all submissions var ( - summaries = make(map[string]ticketvote.SummaryReply, len(subs)) - winnerNetApprove int // Net number of approve votes of the winner - winnerToken string // Token of the winner + summaries = make(map[string]ticketvote.SummaryReply, + len(subs)) + + // Net number of approve votes of the winner + winnerNetApprove int + + // Token of the winner + winnerToken string ) for _, v := range subs { token, err := tokenDecode(v) @@ -2254,13 +2297,13 @@ func (p *ticketVotePlugin) summariesForRunoff(parentToken string) (map[string]ti // Verify the vote met quorum and pass requirements approved := voteIsApproved(*vd, results) if !approved { - // Vote did not meet quorum and pass requirements. Nothing - // else to do. Record vote is not approved. + // Vote did not meet quorum and pass requirements. + // Nothing else to do. Record vote is not approved. continue } - // Check if this record has more net approved votes then current - // highest. + // Check if this record has more net approved votes then + // current highest. var ( votesApprove uint64 // Number of approve votes votesReject uint64 // Number of reject votes @@ -2272,8 +2315,10 @@ func (p *ticketVotePlugin) summariesForRunoff(parentToken string) (map[string]ti case ticketvote.VoteOptionIDReject: votesReject = vor.Votes default: - // Runoff vote options can only be approve/reject - return nil, fmt.Errorf("unknown runoff vote option %v", vor.ID) + // Runoff vote options can only be + // approve/reject + return nil, fmt.Errorf("unknown runoff vote "+ + "option %v", vor.ID) } netApprove := int(votesApprove) - int(votesReject) @@ -2283,8 +2328,8 @@ func (p *ticketVotePlugin) summariesForRunoff(parentToken string) (map[string]ti winnerNetApprove = netApprove } - // This function doesn't handle the unlikely case that the - // runoff vote results in a tie. + // This function doesn't handle the unlikely case that + // the runoff vote results in a tie. } } if winnerToken != "" { @@ -2331,8 +2376,9 @@ func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) // Vote has been authorized; continue status = ticketvote.VoteStatusAuthorized case ticketvote.AuthActionRevoke: - // Vote authorization has been revoked. Its not possible for - // the vote to have been started. We can stop looking. + // Vote authorization has been revoked. Its not + // possible for the vote to have been started. We can + // stop looking. return &ticketvote.SummaryReply{ Status: status, Results: []ticketvote.VoteOptionResult{}, @@ -2355,8 +2401,8 @@ func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) }, nil } - // Vote has been started. We need to check if the vote has ended - // yet and if it can be considered approved or rejected. + // Vote has been started. We need to check if the vote has ended yet + // and if it can be considered approved or rejected. status = ticketvote.VoteStatusStarted // Tally vote results @@ -2385,8 +2431,8 @@ func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) return &summary, nil } - // The vote has finished. Find whether the vote was approved and - // cache the vote summary. + // The vote has finished. Find whether the vote was approved and cache + // the vote summary. switch vd.Params.Type { case ticketvote.VoteTypeStandard: // Standard vote uses a simple approve/reject result @@ -2452,7 +2498,8 @@ func (p *ticketVotePlugin) summaryByToken(token []byte) (*ticketvote.SummaryRepl func (p *ticketVotePlugin) timestamp(treeID int64, digest []byte) (*ticketvote.Timestamp, error) { t, err := p.tstore.Timestamp(treeID, digest) if err != nil { - return nil, fmt.Errorf("timestamp %v %x: %v", treeID, digest, err) + return nil, fmt.Errorf("timestamp %v %x: %v", + treeID, digest, err) } // Convert response @@ -2598,7 +2645,8 @@ func voteIsApproved(vd ticketvote.VoteDetails, results []ticketvote.VoteOptionRe // Valid vote option default: // Invalid vote option - e := fmt.Sprintf("invalid vote option id found: %v", v.ID) + e := fmt.Sprintf("invalid vote option id found: %v", + v.ID) panic(e) } } @@ -2617,15 +2665,16 @@ func voteIsApproved(vd ticketvote.VoteDetails, results []ticketvote.VoteOptionRe // Pass percentage not met approved = false - log.Debugf("Pass threshold not met on %v: approved %v, required %v", - vd.Params.Token, total, quorum) + log.Debugf("Pass threshold not met on %v: approved %v, "+ + "required %v", vd.Params.Token, total, quorum) default: // Vote was approved approved = true - log.Debugf("Vote %v approved: quorum %v, pass %v, total %v, approved %v", - vd.Params.Token, quorum, pass, total, approvedVotes) + log.Debugf("Vote %v approved: quorum %v, pass %v, total %v, "+ + "approved %v", vd.Params.Token, quorum, pass, total, + approvedVotes) } return approved @@ -2655,8 +2704,9 @@ func tokenVerify(cmdToken []byte, payloadToken string) error { return backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeTokenInvalid), - ErrorContext: fmt.Sprintf("payload token does not match "+ - "command token: got %x, want %x", pt, cmdToken), + ErrorContext: fmt.Sprintf("payload token does not "+ + "match command token: got %x, want %x", pt, + cmdToken), } } return nil @@ -2692,8 +2742,8 @@ func convertAuthDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.AuthDetail return nil, fmt.Errorf("unmarshal DataHint: %v", err) } if dd.Descriptor != dataDescriptorAuthDetails { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorAuthDetails) + return nil, fmt.Errorf("unexpected data descriptor: got %v, "+ + "want %v", dd.Descriptor, dataDescriptorAuthDetails) } // Decode data @@ -2730,8 +2780,8 @@ func convertVoteDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.VoteDetail return nil, fmt.Errorf("unmarshal DataHint: %v", err) } if dd.Descriptor != dataDescriptorVoteDetails { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorVoteDetails) + return nil, fmt.Errorf("unexpected data descriptor: got %v, "+ + "want %v", dd.Descriptor, dataDescriptorVoteDetails) } // Decode data @@ -2768,8 +2818,8 @@ func convertCastVoteDetailsFromBlobEntry(be store.BlobEntry) (*ticketvote.CastVo return nil, fmt.Errorf("unmarshal DataHint: %v", err) } if dd.Descriptor != dataDescriptorCastVoteDetails { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorCastVoteDetails) + return nil, fmt.Errorf("unexpected data descriptor: got %v, "+ + "want %v", dd.Descriptor, dataDescriptorCastVoteDetails) } // Decode data @@ -2806,8 +2856,8 @@ func convertVoteColliderFromBlobEntry(be store.BlobEntry) (*voteCollider, error) return nil, fmt.Errorf("unmarshal DataHint: %v", err) } if dd.Descriptor != dataDescriptorVoteCollider { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorVoteCollider) + return nil, fmt.Errorf("unexpected data descriptor: got %v, "+ + "want %v", dd.Descriptor, dataDescriptorVoteCollider) } // Decode data @@ -2844,8 +2894,8 @@ func convertStartRunoffFromBlobEntry(be store.BlobEntry) (*startRunoffRecord, er return nil, fmt.Errorf("unmarshal DataHint: %v", err) } if dd.Descriptor != dataDescriptorStartRunoff { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorStartRunoff) + return nil, fmt.Errorf("unexpected data descriptor: got %v, "+ + "want %v", dd.Descriptor, dataDescriptorStartRunoff) } // Decode data diff --git a/politeiad/backendv2/tstorebe/tstore/recordindex.go b/politeiad/backendv2/tstorebe/tstore/recordindex.go index c1d9286e9..453887b31 100644 --- a/politeiad/backendv2/tstorebe/tstore/recordindex.go +++ b/politeiad/backendv2/tstorebe/tstore/recordindex.go @@ -51,29 +51,29 @@ type recordIndex struct { // only updates do no increment the version. Version uint32 `json:"version"` - // Iteration represents the iteration of the record. The iteration - // is incremented anytime any record content changes. This includes - // file changes that bump the version, metadata stream only updates - // that don't bump the version, and status changes. + // Iteration represents the iteration of the record. The iteration is + // incremented anytime any record content changes. This includes file + // changes that bump the version, metadata stream only updates that + // don't bump the version, and status changes. Iteration uint32 `json:"iteration"` - // The following fields contain the merkle leaf hashes of the - // trillian log leaves for the record content. The merkle leaf hash - // can be used to lookup the log leaf. The log leaf ExtraData field - // contains the key for the record content in the key-value store. + // The following fields contain the merkle leaf hashes of the trillian + // log leaves for the record content. The merkle leaf hash can be used + // to lookup the log leaf. The log leaf ExtraData field contains the + // key for the record content in the key-value store. RecordMetadata []byte `json:"recordmetadata"` Files map[string][]byte `json:"files"` // [filename]merkle // [pluginID][streamID]merkle Metadata map[string]map[uint32][]byte `json:"metadata"` - // Frozen is used to indicate that the tree for this record has - // been frozen. This happens as a result of certain record status - // changes. The only thing that can be appended onto a frozen tree - // is one additional anchor record. Once a frozen tree has been - // anchored, the tstore fsck function will update the status of the - // tree to frozen in trillian, at which point trillian will not - // allow any additional leaves to be appended onto the tree. + // Frozen is used to indicate that the tree for this record has been + // frozen. This happens as a result of certain record status changes. + // The only thing that can be appended onto a frozen tree is one + // additional anchor record. Once a frozen tree has been anchored, the + // tstore fsck function will update the status of the tree to frozen in + // trillian, at which point trillian will not allow any additional + // leaves to be appended onto the tree. Frozen bool `json:"frozen,omitempty"` } @@ -89,7 +89,8 @@ func (t *Tstore) recordIndexSave(treeID int64, idx recordIndex) error { encrypt = false default: // Something is wrong - e := fmt.Sprintf("invalid record state %v %v", treeID, idx.State) + e := fmt.Sprintf("invalid record state %v %v", + treeID, idx.State) panic(e) } @@ -129,8 +130,8 @@ func (t *Tstore) recordIndexSave(treeID int64, idx recordIndex) error { return fmt.Errorf("LeavesAppend: %v", err) } if len(queued) != 1 { - return fmt.Errorf("wrong number of queud leaves: got %v, want 1", - len(queued)) + return fmt.Errorf("wrong number of queud leaves: got %v, "+ + "want 1", len(queued)) } failed := make([]string, 0, len(queued)) for _, v := range queued { @@ -149,10 +150,9 @@ func (t *Tstore) recordIndexSave(treeID int64, idx recordIndex) error { // recordIndexes returns all record indexes found in the provided trillian // leaves. func (t *Tstore) recordIndexes(leaves []*trillian.LogLeaf) ([]recordIndex, error) { - // Walk the leaves and compile the keys for all record indexes. - // Once a record is made vetted the record history is considered - // to restart. If any vetted indexes exist, ignore all unvetted - // indexes. + // Walk the leaves and compile the keys for all record indexes. Once a + // record is made vetted the record history is considered to restart. + // If any vetted indexes exist, ignore all unvetted indexes. var ( keysUnvetted = make([]string, 0, 256) keysVetted = make([]string, 0, 256) @@ -162,18 +162,19 @@ func (t *Tstore) recordIndexes(leaves []*trillian.LogLeaf) ([]recordIndex, error if err != nil { return nil, err } - if ed.Desc == dataDescriptorRecordIndex { - // This is a record index leaf - switch ed.State { - case backend.StateUnvetted: - keysUnvetted = append(keysUnvetted, ed.storeKey()) - case backend.StateVetted: - keysVetted = append(keysVetted, ed.storeKey()) - default: - // Should not happen - return nil, fmt.Errorf("invalid extra data state: %v %v", - v.LeafIndex, ed.State) - } + if ed.Desc != dataDescriptorRecordIndex { + continue + } + // This is a record index leaf + switch ed.State { + case backend.StateUnvetted: + keysUnvetted = append(keysUnvetted, ed.storeKey()) + case backend.StateVetted: + keysVetted = append(keysVetted, ed.storeKey()) + default: + // Should not happen + return nil, fmt.Errorf("invalid extra data state: "+ + "%v %v", v.LeafIndex, ed.State) } } keys := keysUnvetted @@ -229,8 +230,8 @@ func (t *Tstore) recordIndexes(leaves []*trillian.LogLeaf) ([]recordIndex, error indexes = vetted } - // Sort indexes by iteration, smallest to largets. The leaves - // ordering was not preserved in the returned blobs map. + // Sort indexes by iteration, smallest to largets. The leaves ordering + // was not preserved in the returned blobs map. sort.SliceStable(indexes, func(i, j int) bool { return indexes[i].Iteration < indexes[j].Iteration }) @@ -243,13 +244,14 @@ func (t *Tstore) recordIndexes(leaves []*trillian.LogLeaf) ([]recordIndex, error var i uint32 = 1 for _, v := range indexes { if v.Iteration != i { - return nil, fmt.Errorf("invalid record index iteration: "+ - "got %v, want %v", v.Iteration, i) + return nil, fmt.Errorf("invalid record index "+ + "iteration: got %v, want %v", v.Iteration, i) } diff := v.Version - versionPrev if diff != 0 && diff != 1 { return nil, fmt.Errorf("invalid record index version: "+ - "curr version %v, prev version %v", v.Version, versionPrev) + "curr version %v, prev version %v", + v.Version, versionPrev) } i++ @@ -284,18 +286,18 @@ func parseRecordIndex(indexes []recordIndex, version uint32) (*recordIndex, erro return nil, backend.ErrRecordNotFound } - // This function should only be used on record indexes that share - // the same record state. We would not want to accidentally return - // an unvetted index if the record is vetted. It is the - // responsibility of the caller to only provide a single state. + // This function should only be used on record indexes that share the + // same record state. We would not want to accidentally return an + // unvetted index if the record is vetted. It is the responsibility of + // the caller to only provide a single state. state := indexes[0].State if state == backend.StateInvalid { return nil, fmt.Errorf("invalid record index state: %v", state) } for _, v := range indexes { if v.State != state { - return nil, fmt.Errorf("multiple record index states found: %v %v", - v.State, state) + return nil, fmt.Errorf("multiple record index states "+ + "found: %v %v", v.State, state) } } @@ -306,8 +308,8 @@ func parseRecordIndex(indexes []recordIndex, version uint32) (*recordIndex, erro // be returned. ri = &indexes[len(indexes)-1] } else { - // Walk the indexes backwards so the most recent iteration of the - // specified version is selected. + // Walk the indexes backwards so the most recent iteration of + // the specified version is selected. for i := len(indexes) - 1; i >= 0; i-- { r := indexes[i] if r.Version == version { @@ -353,8 +355,8 @@ func convertRecordIndexFromBlobEntry(be store.BlobEntry) (*recordIndex, error) { return nil, fmt.Errorf("unmarshal DataHint: %v", err) } if dd.Descriptor != dataDescriptorRecordIndex { - return nil, fmt.Errorf("unexpected data descriptor: got %v, want %v", - dd.Descriptor, dataDescriptorRecordIndex) + return nil, fmt.Errorf("unexpected data descriptor: got %v, "+ + "want %v", dd.Descriptor, dataDescriptorRecordIndex) } // Decode data From e390de05f1019b24dc4b4fe815c679a06f77b6f9 Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 24 Mar 2021 13:15:15 -0500 Subject: [PATCH 433/449] multi: More cleanup. --- .../tstorebe/plugins/ticketvote/hooks.go | 1 + .../tstorebe/plugins/ticketvote/submissions.go | 12 ++++++------ .../tstorebe/plugins/ticketvote/summary.go | 15 ++++++++------- politeiad/backendv2/tstorebe/tstore/anchor.go | 2 ++ politeiad/backendv2/tstorebe/tstore/blobclient.go | 3 +-- 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go index a97c7b7b4..9c98aa968 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go @@ -50,6 +50,7 @@ func (p *ticketVotePlugin) hookSetRecordStatusPre(payload string) error { return err } + // Verify vote metadata return p.voteMetadataVerifyOnStatusChange(srs.RecordMetadata.Status, srs.Record.Files) } diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/submissions.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/submissions.go index 7bbcd055a..69147bc25 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/submissions.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/submissions.go @@ -107,9 +107,6 @@ func (p *ticketVotePlugin) submissionsCacheSaveWithLock(token []byte, s submissi // // This function must be called WITHOUT the mtxSubs lock held. func (p *ticketVotePlugin) submissionsCacheAdd(parentToken, childToken string) error { - p.mtxSubs.Lock() - defer p.mtxSubs.Unlock() - // Verify tokens parent, err := tokenDecode(parentToken) if err != nil { @@ -120,6 +117,9 @@ func (p *ticketVotePlugin) submissionsCacheAdd(parentToken, childToken string) e return err } + p.mtxSubs.Lock() + defer p.mtxSubs.Unlock() + // Get existing submissions list s, err := p.submissionsCacheWithLock(parent) if err != nil { @@ -146,9 +146,6 @@ func (p *ticketVotePlugin) submissionsCacheAdd(parentToken, childToken string) e // // This function must be called WITHOUT the mtxSubs lock held. func (p *ticketVotePlugin) submissionsCacheDel(parentToken, childToken string) error { - p.mtxSubs.Lock() - defer p.mtxSubs.Unlock() - // Verify tokens parent, err := tokenDecode(parentToken) if err != nil { @@ -159,6 +156,9 @@ func (p *ticketVotePlugin) submissionsCacheDel(parentToken, childToken string) e return err } + p.mtxSubs.Lock() + defer p.mtxSubs.Unlock() + // Get existing submissions list s, err := p.submissionsCacheWithLock(parent) if err != nil { diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/summary.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/summary.go index 865c8af25..c6be9bcfc 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/summary.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/summary.go @@ -44,13 +44,14 @@ var ( // // This function must be called WITHOUT the mtxSummary lock held. func (p *ticketVotePlugin) summaryCache(token string) (*ticketvote.SummaryReply, error) { - p.mtxSummary.Lock() - defer p.mtxSummary.Unlock() - fp, err := p.summaryCachePath(token) if err != nil { return nil, err } + + p.mtxSummary.Lock() + defer p.mtxSummary.Unlock() + b, err := ioutil.ReadFile(fp) if err != nil { var e *os.PathError @@ -78,14 +79,14 @@ func (p *ticketVotePlugin) summaryCacheSave(token string, sr ticketvote.SummaryR if err != nil { return err } - - p.mtxSummary.Lock() - defer p.mtxSummary.Unlock() - fp, err := p.summaryCachePath(token) if err != nil { return err } + + p.mtxSummary.Lock() + defer p.mtxSummary.Unlock() + err = ioutil.WriteFile(fp, b, 0664) if err != nil { return err diff --git a/politeiad/backendv2/tstorebe/tstore/anchor.go b/politeiad/backendv2/tstorebe/tstore/anchor.go index a54f554a7..ac7f2d945 100644 --- a/politeiad/backendv2/tstorebe/tstore/anchor.go +++ b/politeiad/backendv2/tstorebe/tstore/anchor.go @@ -67,6 +67,8 @@ func (t *Tstore) droppingAnchorSet(b bool) { } var ( + // errAnchorNotFound is returned when a anchor record does not + // exist for a leaf yet. errAnchorNotFound = errors.New("anchor not found") ) diff --git a/politeiad/backendv2/tstorebe/tstore/blobclient.go b/politeiad/backendv2/tstorebe/tstore/blobclient.go index 17db0c4dc..70d4bb608 100644 --- a/politeiad/backendv2/tstorebe/tstore/blobclient.go +++ b/politeiad/backendv2/tstorebe/tstore/blobclient.go @@ -67,8 +67,7 @@ func (t *Tstore) BlobSave(treeID int64, be store.BlobEntry) error { encrypt = false default: // Something is wrong - e := fmt.Sprintf("invalid record state %v %v", treeID, idx.State) - panic(e) + panic(fmt.Sprintf("invalid record state %v %v", treeID, idx.State)) } // Prepare blob and digest From 302d679f0ce87e300c379d8e2b711e731abb978f Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 24 Mar 2021 13:48:53 -0500 Subject: [PATCH 434/449] multi: Fixing more review issues. --- .../tstorebe/plugins/ticketvote/cmds.go | 6 +- .../tstorebe/plugins/ticketvote/hooks.go | 13 +++- politeiad/backendv2/tstorebe/tstore/record.go | 70 +++++++++---------- 3 files changed, 50 insertions(+), 39 deletions(-) diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go index ac1ae6515..b3b8260fa 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go @@ -2329,7 +2329,11 @@ func (p *ticketVotePlugin) summariesForRunoff(parentToken string) (map[string]ti } // This function doesn't handle the unlikely case that - // the runoff vote results in a tie. + // the runoff vote results in a tie. If this happens + // then we need to have a debate about how this should + // be handled before implementing anything. The cached + // vote summary would need to be removed and recreated + // using whatever methodology is decided upon. } } if winnerToken != "" { diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go index 9c98aa968..e5f3958a8 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go @@ -87,12 +87,15 @@ func (p *ticketVotePlugin) hookSetRecordStatusPost(treeID int64, payload string) } // linkByVerify verifies that the provided link by timestamp meets all -// ticketvote plugin requirements. +// ticketvote plugin requirements. See the ticketvote VoteMetadata for details +// on the link by timestamp. func (p *ticketVotePlugin) linkByVerify(linkBy int64) error { if linkBy == 0 { // LinkBy as not been set return nil } + + // Min and max link by periods are a ticketvote plugin setting min := time.Now().Unix() + p.linkByPeriodMin max := time.Now().Unix() + p.linkByPeriodMax switch { @@ -111,11 +114,13 @@ func (p *ticketVotePlugin) linkByVerify(linkBy int64) error { linkBy, max), } } + return nil } // linkToVerify verifies that the provided link to meets all ticketvote plugin -// requirements. +// requirements. See the ticketvote VoteMetadata for details on the link to +// field. func (p *ticketVotePlugin) linkToVerify(linkTo string) error { // LinkTo must be a public record token, err := tokenDecode(linkTo) @@ -289,7 +294,7 @@ func (p *ticketVotePlugin) voteMetadataVerify(files []backend.File) error { // Verify vote metadata fields are sane switch { case vm.LinkBy == 0 && vm.LinkTo == "": - // Vote metadata is empty + // Vote metadata is empty. This is not allowed. return backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeVoteMetadataInvalid), @@ -305,12 +310,14 @@ func (p *ticketVotePlugin) voteMetadataVerify(files []backend.File) error { } case vm.LinkBy != 0: + // LinkBy has been set. Verify that is meets plugin requirements. err := p.linkByVerify(vm.LinkBy) if err != nil { return err } case vm.LinkTo != "": + // LinkTo has been set. Verify that is meets plugin requirements. err := p.linkToVerify(vm.LinkTo) if err != nil { return err diff --git a/politeiad/backendv2/tstorebe/tstore/record.go b/politeiad/backendv2/tstorebe/tstore/record.go index bc3ab9ac9..332f7bdf6 100644 --- a/politeiad/backendv2/tstorebe/tstore/record.go +++ b/politeiad/backendv2/tstorebe/tstore/record.go @@ -226,40 +226,13 @@ func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, re for _, v := range beMetadata { for _, be := range v { _, ok := dups[be.Digest] - if !ok { - // Not a duplicate. Prepare kv store blob. - b, err := store.Blobify(be) - if err != nil { - return nil, err - } - k := storeKeyNew(encrypt) - blobs[k] = b - - // Prepare tlog leaf - extraData, err := extraDataEncode(k, - dataDescriptorMetadataStream, idx.State) - if err != nil { - return nil, err - } - digest, err := hex.DecodeString(be.Digest) - if err != nil { - return nil, err - } - leaves = append(leaves, newLogLeaf(digest, extraData)) - + if ok { + // This is a duplicate. Stash is for now. We may need to save + // it as plain text later. + dupBlobs[be.Digest] = be continue } - // This is a duplicate. Stash is for now. We may need to save - // it as plain text later. - dupBlobs[be.Digest] = be - } - } - - // Prepare file blobs and leaves - for _, be := range beFiles { - _, ok := dups[be.Digest] - if !ok { // Not a duplicate. Prepare kv store blob. b, err := store.Blobify(be) if err != nil { @@ -269,7 +242,8 @@ func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, re blobs[k] = b // Prepare tlog leaf - extraData, err := extraDataEncode(k, dataDescriptorFile, idx.State) + extraData, err := extraDataEncode(k, + dataDescriptorMetadataStream, idx.State) if err != nil { return nil, err } @@ -277,14 +251,40 @@ func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, re if err != nil { return nil, err } + leaves = append(leaves, newLogLeaf(digest, extraData)) + } + } + // Prepare file blobs and leaves + for _, be := range beFiles { + _, ok := dups[be.Digest] + if ok { + // This is a duplicate. Stash is for now. We may need to save + // it as plain text later. + dupBlobs[be.Digest] = be continue } - // This is a duplicate. Stash is for now. We may need to save - // it as plain text later. - dupBlobs[be.Digest] = be + // Not a duplicate. Prepare kv store blob. + b, err := store.Blobify(be) + if err != nil { + return nil, err + } + k := storeKeyNew(encrypt) + blobs[k] = b + + // Prepare tlog leaf + extraData, err := extraDataEncode(k, dataDescriptorFile, idx.State) + if err != nil { + return nil, err + } + digest, err := hex.DecodeString(be.Digest) + if err != nil { + return nil, err + } + + leaves = append(leaves, newLogLeaf(digest, extraData)) } // Verify at least one new blob is being saved to the kv store From b68952b26d47d22ce93fe8116f7c31ac928071de Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 24 Mar 2021 13:53:33 -0500 Subject: [PATCH 435/449] One more time... --- .../backendv2/tstorebe/plugins/ticketvote/hooks.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go index e5f3958a8..2467a4b6b 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go @@ -87,8 +87,8 @@ func (p *ticketVotePlugin) hookSetRecordStatusPost(treeID int64, payload string) } // linkByVerify verifies that the provided link by timestamp meets all -// ticketvote plugin requirements. See the ticketvote VoteMetadata for details -// on the link by timestamp. +// ticketvote plugin requirements. See the ticketvote VoteMetadata structure +// for more details on the link by timestamp. func (p *ticketVotePlugin) linkByVerify(linkBy int64) error { if linkBy == 0 { // LinkBy as not been set @@ -119,8 +119,8 @@ func (p *ticketVotePlugin) linkByVerify(linkBy int64) error { } // linkToVerify verifies that the provided link to meets all ticketvote plugin -// requirements. See the ticketvote VoteMetadata for details on the link to -// field. +// requirements. See the ticketvote VoteMetadata structure for more details on +// the link to field. func (p *ticketVotePlugin) linkToVerify(linkTo string) error { // LinkTo must be a public record token, err := tokenDecode(linkTo) @@ -327,6 +327,8 @@ func (p *ticketVotePlugin) voteMetadataVerify(files []backend.File) error { return nil } +// voteMetadataVerifyOnEdits runs vote metadata validation that is specific to +// record edits. func (p *ticketVotePlugin) voteMetadataVerifyOnEdits(r backend.Record, newFiles []backend.File) error { // Verify LinkTo has not changed. This must be run even if a vote // metadata is not present. From 99e7d26bb5e7f6965c9f090ed9b0ab5500d66ccb Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 25 Mar 2021 11:47:00 -0500 Subject: [PATCH 436/449] pd/politeia: Rewrite cmd to use v2 API. --- politeiad/api/v2/v2.go | 4 +- politeiad/backendv2/tstorebe/tstore/record.go | 3 +- politeiad/client/pdv1.go | 38 + politeiad/client/pdv2.go | 68 + politeiad/cmd/politeia/README.md | 373 ++-- politeiad/cmd/politeia/politeia.go | 1760 +++++------------ 6 files changed, 828 insertions(+), 1418 deletions(-) diff --git a/politeiad/api/v2/v2.go b/politeiad/api/v2/v2.go index 5e5b24152..661d4bf82 100644 --- a/politeiad/api/v2/v2.go +++ b/politeiad/api/v2/v2.go @@ -428,8 +428,8 @@ type Inventory struct { // human readable record statuses defined by the RecordStatuses array. type InventoryReply struct { Response string `json:"response"` // Challenge response - Unvetted map[string][]string `json:"unvetted"` - Vetted map[string][]string `json:"vetted"` + Unvetted map[string][]string `json:"unvetted"` // [status][]token + Vetted map[string][]string `json:"vetted"` // [status][]token } // InventoryOrdered requests a page of record tokens ordered by the timestamp diff --git a/politeiad/backendv2/tstorebe/tstore/record.go b/politeiad/backendv2/tstorebe/tstore/record.go index 332f7bdf6..9e8928bfc 100644 --- a/politeiad/backendv2/tstorebe/tstore/record.go +++ b/politeiad/backendv2/tstorebe/tstore/record.go @@ -190,8 +190,7 @@ func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, re encrypt = false default: // Something is wrong - e := fmt.Sprintf("invalid record state %v %v", treeID, idx.State) - panic(e) + panic(fmt.Sprintf("invalid record state %v %v", treeID, idx.State)) } // Prepare record metadata blobs and leaves diff --git a/politeiad/client/pdv1.go b/politeiad/client/pdv1.go index dd8082d0d..f5c3869b7 100644 --- a/politeiad/client/pdv1.go +++ b/politeiad/client/pdv1.go @@ -11,9 +11,47 @@ import ( "net/http" pdv1 "github.com/decred/politeia/politeiad/api/v1" + v1 "github.com/decred/politeia/politeiad/api/v1" + "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/util" ) +// Identity sends a Identity request to the politeiad v1 API. +func (c *Client) Identity(ctx context.Context) (*identity.PublicIdentity, error) { + // Setup request + challenge, err := util.Random(pdv1.ChallengeSize) + if err != nil { + return nil, err + } + i := v1.Identity{ + Challenge: hex.EncodeToString(challenge), + } + + // Send request + resBody, err := c.makeReq(ctx, http.MethodPost, "", + pdv1.IdentityRoute, i) + if err != nil { + return nil, err + } + + // Decode reply + var ir v1.IdentityReply + err = json.Unmarshal(resBody, &ir) + if err != nil { + return nil, err + } + pid, err := util.IdentityFromString(ir.PublicKey) + if err != nil { + return nil, err + } + err = util.VerifyChallenge(pid, challenge, ir.Response) + if err != nil { + return nil, err + } + + return pid, nil +} + // NewRecord sends a NewRecord request to the politeiad v1 API. func (c *Client) NewRecord(ctx context.Context, metadata []pdv1.MetadataStream, files []pdv1.File) (*pdv1.CensorshipRecord, error) { // Setup request diff --git a/politeiad/client/pdv2.go b/politeiad/client/pdv2.go index d910dd500..034764546 100644 --- a/politeiad/client/pdv2.go +++ b/politeiad/client/pdv2.go @@ -5,12 +5,16 @@ package client import ( + "bytes" "context" + "encoding/base64" "encoding/hex" "encoding/json" + "fmt" "net/http" pdv2 "github.com/decred/politeia/politeiad/api/v2" + v2 "github.com/decred/politeia/politeiad/api/v2" "github.com/decred/politeia/util" ) @@ -391,6 +395,70 @@ func (c *Client) PluginInventory(ctx context.Context) ([]pdv2.Plugin, error) { return pir.Plugins, nil } +// RecordVerify verifies the censorship record of a v2 Record. +func RecordVerify(r pdv2.Record, serverPubKey string) error { + // Verify censorship record merkle root + if len(r.Files) > 0 { + // Verify digests + err := digestsVerify(r.Files) + if err != nil { + return err + } + + // Verify merkle root + digests := make([]string, 0, len(r.Files)) + for _, v := range r.Files { + digests = append(digests, v.Digest) + } + mr, err := util.MerkleRoot(digests) + if err != nil { + return err + } + if hex.EncodeToString(mr[:]) != r.CensorshipRecord.Merkle { + return fmt.Errorf("merkle roots do not match") + } + } + + // Verify censorship record signature + id, err := util.IdentityFromString(serverPubKey) + if err != nil { + return err + } + s, err := util.ConvertSignature(r.CensorshipRecord.Signature) + if err != nil { + return err + } + msg := []byte(r.CensorshipRecord.Merkle + r.CensorshipRecord.Token) + if !id.VerifyMessage(msg, s) { + return fmt.Errorf("invalid censorship record signature") + } + + return nil +} + +// digestsVerify verifies that all file digests match the calculated SHA256 +// digests of the file payloads. +func digestsVerify(files []v2.File) error { + for _, f := range files { + b, err := base64.StdEncoding.DecodeString(f.Payload) + if err != nil { + return fmt.Errorf("file: %v decode payload err %v", + f.Name, err) + } + digest := util.Digest(b) + d, ok := util.ConvertDigest(f.Digest) + if !ok { + return fmt.Errorf("file: %v invalid digest %v", + f.Name, f.Digest) + } + if !bytes.Equal(digest, d[:]) { + return fmt.Errorf("file: %v digests do not match", + f.Name) + } + } + return nil +} + func extractPluginCmdError(pcr pdv2.PluginCmdReply) error { switch { case pcr.UserError != nil: diff --git a/politeiad/cmd/politeia/README.md b/politeiad/cmd/politeia/README.md index fcf3393ea..765e60648 100644 --- a/politeiad/cmd/politeia/README.md +++ b/politeiad/cmd/politeia/README.md @@ -1,19 +1,24 @@ # politeia refclient examples -Available commands: -`identity` -`verify` -`new` -`updateunvetted` -`updateunvettedmd` -`setunvettedstatus` -`getunvetted` -`updatevetted` -`updatevettedmd` -`getvetted` -`plugin` -`plugininventory` -`inventory` +``` +Available commands: + identity Get server identity + new Submit new record + Args: [metadata::metadataJSON]... ... + verify Verify record was accepted + Args: ... + edit Edit record + Args: [actionMetadata::metadataJSON]... + ... token: + editmetadata Edit record metdata + Args: [actionMetadata::metadataJSON]... token: + setstatus Set record status + Args: + record Get a record + Args: + inventory Get the record inventory + Args (optional): +``` ## Obtain politeiad identity @@ -22,75 +27,78 @@ The retrieved identity is used to verify replies from politeiad. ``` $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass identity -Key : 8f627e9da14322626d7e81d789f7fcafd25f62235a95377f39cbc7293c4944ad -Fingerprint: j2J+naFDImJtfoHXiff8r9JfYiNalTd/OcvHKTxJRK0= +Key : e88df79a4b02699e6c051adbae05f21f2a2f24942e0f27cade165548ec3d6387 +Fingerprint: 6I33mksCaZ5sBRrbrgXyHyovJJQuDyfK3hZVSOw9Y4c= Save to /home/user/.politeia/identity.json or ctrl-c to abort Identity saved to: /home/user/.politeia/identity.json ``` -## Add a new record +## Submit a new record + +Args: `[metadata::metadataJSON]... ...` At least one file must be submitted. This example uses an `index.md` file. -Arguments are matched against the regex `^metadata[\d]{1,2}:` to determine if -the string is record metadata. Arguments that are not classified as metadata -are assumed to be file paths. +Metadata is submitted as JSON and must be identified by a `pluginID` string and +a `streamID` uint32. Metadata is passed in as an argument by prefixing the JSON +with `metadata:[pluginID][streamID]:`. Below is an example metadata argument +where the plugin ID is `testid` and the stream ID is `1`. +`metadata:testid1:{"foo":"bar"}` ``` $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass new \ - 'metadata12:{"moo":"lala"}' 'metadata2:{"foo":"bar"}' index.md +'metadata:testid1:{"moo":"lala"}' 'metadata:testid12:{"foo":"bar"}' index.md -00: 4bde9f923b61e26147c79500e6d6dfa27291559a74cd878c29a7f96984dd48bb index.md text/plain; charset=utf-8 -Record submitted +Record submitted: + Status : unreviewed + Timestamp : 2021-03-25 16:36:40 +0000 UTC + Version : 1 Censorship record: - Merkle : 4bde9f923b61e26147c79500e6d6dfa27291559a74cd878c29a7f96984dd48bb - Token : 9dfe084fccb7f27c0000 - Signature: e69a38b6e6c21021db2fe37c6b38886ef987c7347bb881e2358feb766974577a742e535d34cd4d7a140b2555b3771a194fea4be942cbd99247c143d07419bc06 -$ + Merkle : 2d00aaa0768701fd011943fbe8ae92f84ee268ca134d6b14f877c3153072bb3c + Token : 39868e5e91c78255 + Signature: b2a69823f85b62941d845c439726a2504026a0d29fd50ecabe5648b0128328c2fade0ddb354594d48a209dff24e73795ec9cb175d028155cbfa1901114f4b608 + File (00) : + Name : index.md + MIME : text/plain; charset=utf-8 + Digest : 2d00aaa0768701fd011943fbe8ae92f84ee268ca134d6b14f877c3153072bb3c + Metadata stream testid 01: + {"moo":"lala"} + Metadata stream testid 12: + {"foo":"bar"} +Server public key: e88df79a4b02699e6c051adbae05f21f2a2f24942e0f27cade165548ec3d6387 ``` -## Verify a record +## Verify record -Verifies the censorship signature of a record to make sure it was received by -the server. It receives as input the server's public key, the record token, -the record merkle root and the signature. +Args: ` ...` -``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass verify \ - df0f5cff8d9a3c6d55429c7e9e66b13ec175768b24f20db1b91188f00f7ea4b7 \ - c800ff5195ddb2360000 \ - 36a2e53b25183dcfd192224fdff1074472ad7fdf5989abf9dfb8e7972caceae1 \ - f9495f8100fc3d2d2bcd3d57705adc1f859d29f5a0ed1d999593a57de5af8c047615efb65e08e93935d071b94ee3883f15661defbfa83b503e6e84be4c18aa0b +Verify a record was recieved by the server by verifying the censorship +record signature. - Record successfully verified ``` - -## Get unvetted record - +$ politeia verify \ +e88df79a4b02699e6c051adbae05f21f2a2f24942e0f27cade165548ec3d6387 \ +39868e5e91c78255 \ +b2a69823f85b62941d845c439726a2504026a0d29fd50ecabe5648b0128328c2fade0ddb354594d48a209dff24e73795ec9cb175d028155cbfa1901114f4b608 \ +index.md + +Server key : e88df79a4b02699e6c051adbae05f21f2a2f24942e0f27cade165548ec3d6387 +Token : 39868e5e91c78255 +Merkle root: 2d00aaa0768701fd011943fbe8ae92f84ee268ca134d6b14f877c3153072bb3c +Signature : b2a69823f85b62941d845c439726a2504026a0d29fd50ecabe5648b0128328c2fade0ddb354594d48a209dff24e73795ec9cb175d028155cbfa1901114f4b608 + +Record successfully verified ``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass getunvetted 9dfe084fccb7f27c0000 -Unvetted record: - Status : not reviewed - Timestamp : 2020-10-01 14:36:11 +0000 UTC - Censorship record: - Merkle : 4bde9f923b61e26147c79500e6d6dfa27291559a74cd878c29a7f96984dd48bb - Token : 9dfe084fccb7f27c0000 - Signature: e69a38b6e6c21021db2fe37c6b38886ef987c7347bb881e2358feb766974577a742e535d34cd4d7a140b2555b3771a194fea4be942cbd99247c143d07419bc06 - Metadata : [{2 {"foo":"bar"}} {12 {"moo":"lala"}}] - Version : 1 - File (00) : - Name : index.md - MIME : text/plain; charset=utf-8 - Digest : 4bde9f923b61e26147c79500e6d6dfa27291559a74cd878c29a7f96984dd48bb -``` +## Edit record -## Update an unvetted record +Args: `[actionMetadata::metadataJSON]... ... +token:` Metadata can be updated using the arguments: -`'appendmetadata[ID]:[metadataJSON]'` -`'overwritemetadata[ID]:[metadataJSON]'` +`'appendmetadata:[pluginID][streamID]:[metadataJSON]'` +`'overwritemetadata:[pluginID][streamID]:[metadataJSON]'` Files can be updated using the arguments: `add:[filepath]` @@ -103,22 +111,41 @@ Metadata provided using the `overwritemetadata` argument does not have to already exist. ``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass updateunvetted \ - 'appendmetadata12:{"foo":"bar"}' 'overwritemetadata2:{"12foo":"12bar"}' \ - del:index.md add:updated.md token:9dfe084fccb7f27c0000 - -Update record: 9dfe084fccb7f27c0000 - Files add : 00: 22036b8b67a7c54f2bae29e1f9a11551cf62a33a038788b8f2e8f8d6e7f60425 updated.md text/plain; charset=utf-8 - Files delete : index.md - Metadata overwrite: 2 - Metadata append : 12 +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass edit \ +'appendmetadata:testid1:{"foo":"bar"}' \ +'overwritemetadata:testid12:{"12foo":"12bar"}' \ +del:index.md add:updated.md token:39868e5e91c78255 + +Record updated: + Status : unreviewed + Timestamp : 2021-03-25 16:38:59 +0000 UTC + Version : 2 + Censorship record: + Merkle : db09a4371b32086241999b7db196c4bda04bd93194cdb90940a88741d5bbf166 + Token : 39868e5e91c78255 + Signature: 7f26ab5d5fc4a67cfe6320fa1a1c2cbb5d6dadbfcd74d255d0c048c32e9da413cfb8cdcc9440c53300ce0907c7d274435d4e98c36c189dfcc81dbecc44e79003 + File (00) : + Name : updated.md + MIME : text/plain; charset=utf-8 + Digest : db09a4371b32086241999b7db196c4bda04bd93194cdb90940a88741d5bbf166 + Metadata stream testid 12: + {"12foo":"12bar"} + Metadata stream testid 01: + {"moo":"lala"}{"foo":"bar"} +Server public key: e88df79a4b02699e6c051adbae05f21f2a2f24942e0f27cade165548ec3d6387 ``` -## Update unvetted metadata only +## Edit record metadata + +Args: `[actionMetadata::metadataJSON]... token:` + +Metadata can be updated when updating the record files or the client can +perform a metadata only update using this command. Updating only the metadata +will not change the censorship record signature. Metadata can be updated using the arguments: -`'appendmetadata[ID]:[metadataJSON]'` -`'overwritemetadata[ID]:[metadataJSON]'` +`'appendmetadata:[pluginID][streamID]:[metadataJSON]'` +`'overwritemetadata:[pluginID][streamID]:[metadataJSON]'` The token is specified using the argument: `token:[token]` @@ -127,132 +154,136 @@ Metadata provided using the `overwritemetadata` argument does not have to already exist. ``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass updateunvettedmd \ - 'appendmetadata12:{"foo":"bar"}' 'overwritemetadata2:{"12foo":"12bar"}' \ - token:0e4a82a370228b710000 - -Update record: 0e4a82a370228b710000 - Metadata overwrite: 2 - Metadata append : 12 +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass editmetadata \ +'appendmetadata:testid1:{"foo":"bar"}' \ +'overwritemetadata:testid12:{"123foo":"123bar"}' \ +token:39868e5e91c78255 + +Record metadata updated: + Status : unreviewed + Timestamp : 2021-03-25 16:39:35 +0000 UTC + Version : 2 + Censorship record: + Merkle : db09a4371b32086241999b7db196c4bda04bd93194cdb90940a88741d5bbf166 + Token : 39868e5e91c78255 + Signature: 7f26ab5d5fc4a67cfe6320fa1a1c2cbb5d6dadbfcd74d255d0c048c32e9da413cfb8cdcc9440c53300ce0907c7d274435d4e98c36c189dfcc81dbecc44e79003 + File (00) : + Name : updated.md + MIME : text/plain; charset=utf-8 + Digest : db09a4371b32086241999b7db196c4bda04bd93194cdb90940a88741d5bbf166 + Metadata stream testid 12: + {"123foo":"123bar"} + Metadata stream testid 01: + {"moo":"lala"}{"foo":"bar"}{"foo":"bar"} +Server public key: e88df79a4b02699e6c051adbae05f21f2a2f24942e0f27cade165548ec3d6387 ``` -## Set unvetted status - -You can update the status of an unvetted record using one of the following -statuses: -- `censored` - keep the record unvetted and mark as censored. -- `public` - make the record a public, vetted record. -- `archived` - archive the record. +## Set record status -Note `token:` is not prefixed to the token in this command. Status change -validation is done in the backend. - -``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass setunvettedstatus public 0e4a82a370228b710000 +Args: -Set record status: - Status : public -``` +You can update the status of a record using one of the following statuses: +- `public` - make the record a public +- `archived` - lock the record from further edits +- `censored` - lock the record from further edits and delete all files -## Get vetted record +Note `token:` is not prefixed to the token in this command. ``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass getvetted 9dfe084fccb7f27c0000 +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass setstatus 39868e5e91c78255 public -Vetted record: +Record status updated: Status : public - Timestamp : 2020-10-01 14:38:43 +0000 UTC - Censorship record: - Merkle : 22036b8b67a7c54f2bae29e1f9a11551cf62a33a038788b8f2e8f8d6e7f60425 - Token : 9dfe084fccb7f27c0000 - Signature: 531e5103e9f8905d52d7bf3c6fdb40070cca4f88e69f3b6c647baf8bd84148471e378b5c137014a1f3f46a2cb9a40cdc302dea4bf828fb6dd09a858fa2748c0e - Metadata : [{2 {"12foo":"12bar"}} {12 {"moo":"lala"}{"foo":"bar"}}] + Timestamp : 2021-03-25 16:40:40 +0000 UTC Version : 1 + Censorship record: + Merkle : db09a4371b32086241999b7db196c4bda04bd93194cdb90940a88741d5bbf166 + Token : 39868e5e91c78255 + Signature: 7f26ab5d5fc4a67cfe6320fa1a1c2cbb5d6dadbfcd74d255d0c048c32e9da413cfb8cdcc9440c53300ce0907c7d274435d4e98c36c189dfcc81dbecc44e79003 File (00) : Name : updated.md MIME : text/plain; charset=utf-8 - Digest : 22036b8b67a7c54f2bae29e1f9a11551cf62a33a038788b8f2e8f8d6e7f60425 -``` - -## Update a vetted record - -Metadata can be updated using the arguments: -`'appendmetadata[ID]:[metadataJSON]'` -`'overwritemetadata[ID]:[metadataJSON]'` - -Files can be updated using the arguments: -`add:[filepath]` -`del:[filename]` - -The token is specified using the argument: -`token:[token]` - -Metadata provided using the `overwritemetadata` argument does not have to -already exist. - -``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass updatevetted \ - 'appendmetadata12:{"foo":"bar"}' 'overwritemetadata2:{"12foo":"12bar"}' \ - del:updated add:newfile.md token:9dfe084fccb7f27c0000 - -Update record: 9dfe084fccb7f27c0000 - Files add : 00: 22036b8b67a7c54f2bae29e1f9a11551cf62a33a038788b8f2e8f8d6e7f60425 newfile.md text/plain; charset=utf-8 - Files delete : updated - Metadata overwrite: 2 - Metadata append : 12 + Digest : db09a4371b32086241999b7db196c4bda04bd93194cdb90940a88741d5bbf166 + Metadata stream testid 01: + {"moo":"lala"}{"foo":"bar"}{"foo":"bar"} + Metadata stream testid 12: + {"123foo":"123bar"} +Server public key: e88df79a4b02699e6c051adbae05f21f2a2f24942e0f27cade165548ec3d6387 ``` -## Update vetted metadata only - -Metadata can be updated using the arguments: -`'appendmetadata[ID]:[metadataJSON]'` -`'overwritemetadata[ID]:[metadataJSON]'` - -The token is specified using the argument: -`token:[token]` +## Get record -Metadata provided using the `overwritemetadata` argument does not have to -already exist. +Retrieve a record. ``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass updatevettedmd \ - 'appendmetadata12:{"foo":"bar"}' 'overwritemetadata2:{"12foo":"12bar"}' \ - token:0e4a82a370228b710000 +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass record 39868e5e91c78255 -Update record: 0e4a82a370228b710000 - Metadata overwrite: 2 - Metadata append : 12 +Record: + Status : public + Timestamp : 2021-03-25 16:40:40 +0000 UTC + Version : 1 + Censorship record: + Merkle : db09a4371b32086241999b7db196c4bda04bd93194cdb90940a88741d5bbf166 + Token : 39868e5e91c78255 + Signature: 7f26ab5d5fc4a67cfe6320fa1a1c2cbb5d6dadbfcd74d255d0c048c32e9da413cfb8cdcc9440c53300ce0907c7d274435d4e98c36c189dfcc81dbecc44e79003 + File (00) : + Name : updated.md + MIME : text/plain; charset=utf-8 + Digest : db09a4371b32086241999b7db196c4bda04bd93194cdb90940a88741d5bbf166 + Metadata stream testid 12: + {"123foo":"123bar"} + Metadata stream testid 01: + {"moo":"lala"}{"foo":"bar"}{"foo":"bar"} +Server public key: e88df79a4b02699e6c051adbae05f21f2a2f24942e0f27cade165548ec3d6387 ``` -## Set vetted status - -You can update the status of a vetted record using one of the following -statuses: -- `censored` - keep the record unvetted and mark as censored. -- `archived` - archive the record. - -Note `token:` is not prefixed to the token in this command. Status change -validation is done in the backend. +## Inventory -``` -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass setvettedstatus censored 0e4a82a370228b710000 'overwritemetadata12:"zap"' -Set record status: - Status: censored -``` +Retrieve the censorship record tokens of the records in the inventory, +categorized by their record state and record status. -## Inventory +The user can request a page of tokens from a specific record state and record +status by providing the arguments. -The `inventory` command retrieves the censorship record tokens from all records, -categorized by their record status. +If not arguments are provided then a page of tokens for every state and status +will be returned. ``` $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass inventory -Inventory: - Unvetted - not reviewed : 344044686e9ba76f0000, 43df32e2405065250000 - censored : 9f104b878a83242d0000 - Vetted - public : e2228e7a7e7c80030000, a6c064874351f4120000 - archived : 6b099750984e05490000 +Unvetted +{ + "censored": [ + "de127f2cb24c702b", + ], + "unreviewed": [ + "d0545038224c5054", + "ea260a4ab9170d70" + ] +} +Vetted +{ + "archived": [ + "77396eccc387b07e" + ], + "censored": [ + "0439c5355ef94e36" + ], + "public": [ + "39868e5e91c78255", + "2f5d6bbb15b63e76", + "f1f7337397a79b51" + ] +} + + +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass inventory unvetted unreviewed 1 + +Unvetted +{ + "unreviewed": [ + "d0545038224c5054", + "ea260a4ab9170d70" + ] +} ``` diff --git a/politeiad/cmd/politeia/politeia.go b/politeiad/cmd/politeia/politeia.go index a8a86f9b9..e6a17b86a 100644 --- a/politeiad/cmd/politeia/politeia.go +++ b/politeiad/cmd/politeia/politeia.go @@ -6,36 +6,36 @@ package main import ( "bufio" - "bytes" + "context" "crypto/sha256" "encoding/hex" - "encoding/json" "flag" "fmt" - "net/http" "net/url" "os" "path/filepath" "regexp" "strconv" - "strings" "time" "github.com/decred/dcrd/dcrutil/v3" - "github.com/decred/dcrtime/merkle" v1 "github.com/decred/politeia/politeiad/api/v1" "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/api/v1/mime" + v2 "github.com/decred/politeia/politeiad/api/v2" + pdclient "github.com/decred/politeia/politeiad/client" "github.com/decred/politeia/util" ) const allowInteractive = "i-know-this-is-a-bad-idea" var ( - regexMD = regexp.MustCompile(`^metadata[\d]{1,2}:`) - regexMDID = regexp.MustCompile(`[\d]{1,2}`) - regexAppendMD = regexp.MustCompile(`^appendmetadata[\d]{1,2}:`) - regexOverwriteMD = regexp.MustCompile(`^overwritemetadata[\d]{1,2}:`) + regexMD = regexp.MustCompile(`^metadata:`) + regexMDID = regexp.MustCompile(`[a-z]{1,16}[\d]{1,2}:`) + regexMDPluginID = regexp.MustCompile(`[a-z]{1,16}`) + regexMDStreamID = regexp.MustCompile(`[\d]{1,2}`) + regexAppendMD = regexp.MustCompile(`^appendmetadata:`) + regexOverwriteMD = regexp.MustCompile(`^overwritemetadata:`) regexFileAdd = regexp.MustCompile(`^add:`) regexFileDel = regexp.MustCompile(`^del:`) regexToken = regexp.MustCompile(`^token:`) @@ -43,10 +43,12 @@ var ( defaultHomeDir = dcrutil.AppDataDir("politeia", false) defaultIdentityFilename = "identity.json" - identityFilename = flag.String("-id", filepath.Join(defaultHomeDir, + defaultPDAppDir = dcrutil.AppDataDir("politeiad", false) + defaultRPCCertFile = filepath.Join(defaultPDAppDir, "https.cert") + + identityFilename = flag.String("id", filepath.Join(defaultHomeDir, defaultIdentityFilename), "remote server identity file") testnet = flag.Bool("testnet", false, "Use testnet port") - printJson = flag.Bool("json", false, "Print JSON") verbose = flag.Bool("v", false, "Verbose") rpcuser = flag.String("rpcuser", "", "RPC user name for privileged calls") rpcpass = flag.String("rpcpass", "", "RPC password for privileged calls") @@ -55,1218 +57,600 @@ var ( interactive = flag.String("interactive", "", "Set to "+ allowInteractive+" to to turn off interactive mode during "+ "identity fetch") - - verify = false // Validate server TLS certificate ) +const availableCmds = ` +Available commands: + identity Get server identity + new Submit new record + Args: [metadata::metadataJSON]... ... + verify Verify record was accepted + Args: ... + edit Edit record + Args: [actionMetadata::metadataJSON]... + ... token: + editmetadata Edit record metdata + Args: [actionMetadata::metadataJSON]... token: + setstatus Set record status + Args: + record Get a record + Args: + inventory Get the record inventory + Args (optional): + +Metadata actions: appendmetadata, overwritemetadata +File actions: add, del +Record statuses: public, censored, or archived + +A metadata consists of the . Plugin IDs are strings +and stream IDs are uint32. Below are example metadata arguments where the +plugin ID is 'testid' and the stream ID is '1'. + +Submit new metadata: 'metadata:testid1:{"foo":"bar"}' +Append metadata : 'appendmetadata:testid1:{"foo":"bar"}' +Overwrite metadata : 'overwritemetadata:testid1:{"foo":"bar"}' + +` + func usage() { fmt.Fprintf(os.Stderr, "usage: politeia [flags] [arguments]\n") fmt.Fprintf(os.Stderr, " flags:\n") flag.PrintDefaults() - fmt.Fprintf(os.Stderr, "\n actions:\n") - fmt.Fprintf(os.Stderr, " identity - Retrieve server "+ - "identity\n") - fmt.Fprintf(os.Stderr, " plugins - Retrieve plugin "+ - "inventory\n") - fmt.Fprintf(os.Stderr, " inventory - Inventory records by "+ - "status\n") - fmt.Fprintf(os.Stderr, " new - Create new record "+ - "[metadata]... ...\n") - fmt.Fprintf(os.Stderr, " verify - Verify a record "+ - " \n") - fmt.Fprintf(os.Stderr, " getunvetted - Retrieve record "+ - "\n") - fmt.Fprintf(os.Stderr, " setunvettedstatus - Set unvetted record "+ - "status "+ - "[actionmdid:metadata]...\n") - fmt.Fprintf(os.Stderr, " updateunvetted - Update unvetted record "+ - "[actionmdid:metadata]... ... "+ - "token:\n") - fmt.Fprintf(os.Stderr, " updateunvettedmd - Update unvetted record "+ - "metadata [actionmdid:metadata]... token:\n") - fmt.Fprintf(os.Stderr, " updatevetted - Update vetted record "+ - "[actionmdid:metadata]... ... "+ - "token:\n") - fmt.Fprintf(os.Stderr, " updatevettedmd - Update vetted record "+ - "metadata [actionmdid:metadata]... token:\n") - fmt.Fprintf(os.Stderr, " setvettedstatus - Set vetted record "+ - "status "+ - "[actionmdid:metadata]...\n") - fmt.Fprintf(os.Stderr, "\n") - fmt.Fprintf(os.Stderr, " metadata is the word metadata followed "+ - "by digits. Example with 2 metadata records "+ - "metadata0:{\"moo\":\"12\",\"blah\":\"baz\"} "+ - "metadata1:{\"lala\":42}\n") - fmt.Fprintf(os.Stderr, " actionmdid is an action + metadatastream id "+ - "E.g. appendmetadata0:{\"foo\":\"bar\"} or "+ - "overwritemetadata12:{\"bleh\":\"truff\"}\n") - - fmt.Fprintf(os.Stderr, "\n") -} - -// getErrorFromResponse extracts a user-readable string from the response from -// politeiad, which will contain a JSON error. -func getErrorFromResponse(r *http.Response) (string, error) { - var errMsg string - decoder := json.NewDecoder(r.Body) - if r.StatusCode == http.StatusInternalServerError { - var e v1.ServerErrorReply - if err := decoder.Decode(&e); err != nil { - return "", err - } - errMsg = fmt.Sprintf("%v", e.ErrorCode) - } else { - var e v1.UserErrorReply - if err := decoder.Decode(&e); err != nil { - return "", err - } - errMsg = v1.ErrorStatus[e.ErrorCode] + " " - if e.ErrorContext != nil && len(e.ErrorContext) > 0 { - errMsg += strings.Join(e.ErrorContext, ", ") - } - } - - return errMsg, nil + fmt.Fprintf(os.Stderr, availableCmds) } -func getIdentity() error { - // Fetch remote identity - id, err := util.RemoteIdentity(verify, *rpchost, *rpccert) - if err != nil { - return err - } - - rf := filepath.Join(defaultHomeDir, defaultIdentityFilename) - - // Pretty print identity. - fmt.Printf("Key : %x\n", id.Key) - fmt.Printf("Fingerprint: %v\n", id.Fingerprint()) - - // Ask user if we like this identity - if *interactive != allowInteractive { - fmt.Printf("\nSave to %v or ctrl-c to abort ", rf) - scanner := bufio.NewScanner(os.Stdin) - scanner.Scan() - if err = scanner.Err(); err != nil { - return err - } - if len(scanner.Text()) != 0 { - rf = scanner.Text() - } - } else { - fmt.Printf("Saving identity to %v\n", rf) - } - rf = util.CleanAndExpandPath(rf) - - // Save identity - err = os.MkdirAll(filepath.Dir(rf), 0700) - if err != nil { - return err - } - err = id.SavePublicIdentity(rf) - if err != nil { - return err - } - fmt.Printf("Identity saved to: %v\n", rf) - - return nil -} - -func printCensorshipRecord(c v1.CensorshipRecord) { - fmt.Printf(" Censorship record:\n") - fmt.Printf(" Merkle : %v\n", c.Merkle) - fmt.Printf(" Token : %v\n", c.Token) - fmt.Printf(" Signature: %v\n", c.Signature) -} - -func printRecord(header string, pr v1.Record) { +func printRecord(header string, r v2.Record) { // Pretty print record - status, ok := v1.RecordStatus[pr.Status] + status, ok := v2.RecordStatuses[r.Status] if !ok { - status = v1.RecordStatus[v1.RecordStatusInvalid] + status = v2.RecordStatuses[v2.RecordStatusInvalid] } fmt.Printf("%v:\n", header) fmt.Printf(" Status : %v\n", status) - fmt.Printf(" Timestamp : %v\n", time.Unix(pr.Timestamp, 0).UTC()) - printCensorshipRecord(pr.CensorshipRecord) - fmt.Printf(" Metadata : %v\n", pr.Metadata) - fmt.Printf(" Version : %v\n", pr.Version) - for k, v := range pr.Files { + fmt.Printf(" Timestamp : %v\n", time.Unix(r.Timestamp, 0).UTC()) + fmt.Printf(" Version : %v\n", r.Version) + fmt.Printf(" Censorship record:\n") + fmt.Printf(" Merkle : %v\n", r.CensorshipRecord.Merkle) + fmt.Printf(" Token : %v\n", r.CensorshipRecord.Token) + fmt.Printf(" Signature: %v\n", r.CensorshipRecord.Signature) + for k, v := range r.Files { fmt.Printf(" File (%02v) :\n", k) fmt.Printf(" Name : %v\n", v.Name) fmt.Printf(" MIME : %v\n", v.MIME) fmt.Printf(" Digest : %v\n", v.Digest) } -} - -func pluginInventory() (*v1.PluginInventoryReply, error) { - challenge, err := util.Random(v1.ChallengeSize) - if err != nil { - return nil, err - } - b, err := json.Marshal(v1.PluginInventory{ - Challenge: hex.EncodeToString(challenge), - }) - if err != nil { - return nil, err - } - - if *printJson { - fmt.Println(string(b)) - } - - c, err := util.NewHTTPClient(verify, *rpccert) - if err != nil { - return nil, err - } - req, err := http.NewRequest("POST", *rpchost+v1.PluginInventoryRoute, - bytes.NewReader(b)) - if err != nil { - return nil, err - } - req.SetBasicAuth(*rpcuser, *rpcpass) - r, err := c.Do(req) - if err != nil { - return nil, err - } - defer r.Body.Close() - if r.StatusCode != http.StatusOK { - e, err := getErrorFromResponse(r) - if err != nil { - return nil, fmt.Errorf("%v", r.Status) - } - return nil, fmt.Errorf("%v: %v", r.Status, e) - } - - bodyBytes := util.ConvertBodyToByteArray(r.Body, *printJson) - - var ir v1.PluginInventoryReply - err = json.Unmarshal(bodyBytes, &ir) - if err != nil { - return nil, fmt.Errorf("Could node unmarshal "+ - "PluginInventoryReply: %v", err) - } - - // Fetch remote identity - id, err := identity.LoadPublicIdentity(*identityFilename) - if err != nil { - return nil, err + for _, v := range r.Metadata { + fmt.Printf(" Metadata stream %v %02v:\n", v.PluginID, v.StreamID) + fmt.Printf(" %v\n", v.Payload) } - - err = util.VerifyChallenge(id, challenge, ir.Response) - if err != nil { - return nil, err - } - - return &ir, nil } -func plugin() error { - flags := flag.Args()[1:] // Chop off action. +// parseMetadataIDs parses and returns the plugin ID and stream ID from a full +// metadata ID string. See the example below. +// +// Metadata ID string: "pluginid12:" +// Plugin ID: "plugindid" +// Stream ID: 12 +func parseMetadataIDs(mdID string) (string, uint32, error) { + // Parse the plugin ID. This is the "pluginid" part of the + // "pluginid12:" metadata ID. + pluginID := regexMDPluginID.FindString(mdID) - if len(flags) != 4 { - return fmt.Errorf("not enough parameters") - } - - challenge, err := util.Random(v1.ChallengeSize) + // Parse the stream ID. This is the "12" part of the + // "pluginid12:" metadata ID. + streamID, err := strconv.ParseUint(regexMDStreamID.FindString(mdID), + 10, 64) if err != nil { - return err - } - b, err := json.Marshal(v1.PluginCommand{ - Challenge: hex.EncodeToString(challenge), - ID: flags[0], - Command: flags[1], - CommandID: flags[2], - Payload: flags[3], - }) - if err != nil { - return err + return "", 0, err } - if *printJson { - fmt.Println(string(b)) - } - - c, err := util.NewHTTPClient(verify, *rpccert) - if err != nil { - return err - } - req, err := http.NewRequest("POST", *rpchost+v1.PluginCommandRoute, - bytes.NewReader(b)) - if err != nil { - return err - } - req.SetBasicAuth(*rpcuser, *rpcpass) - r, err := c.Do(req) - if err != nil { - return err - } - defer r.Body.Close() - if r.StatusCode != http.StatusOK { - e, err := getErrorFromResponse(r) - if err != nil { - return fmt.Errorf("%v", r.Status) - } - return fmt.Errorf("%v: %v", r.Status, e) - } - - bodyBytes := util.ConvertBodyToByteArray(r.Body, *printJson) - - var pcr v1.PluginCommandReply - err = json.Unmarshal(bodyBytes, &pcr) - if err != nil { - return fmt.Errorf("Could node unmarshal "+ - "PluginCommandReply: %v", err) - } - - // Fetch remote identity - id, err := identity.LoadPublicIdentity(*identityFilename) - if err != nil { - return err - } - - return util.VerifyChallenge(id, challenge, pcr.Response) + return pluginID, uint32(streamID), nil } -func getPluginInventory() error { - pr, err := pluginInventory() - if err != nil { - return err - } +// parseMetadata returns the metadata streams for all metadata flags. +func parseMetadata(flags []string) ([]v2.MetadataStream, error) { + md := make([]v2.MetadataStream, 0, len(flags)) + for _, v := range flags { + // Example metadata: 'metadata:pluginid12:{"moo":"lala"}' - for _, v := range pr.Plugins { - fmt.Printf("Plugin ID : %v\n", v.ID) - if len(v.Settings) > 0 { - fmt.Printf("Plugin settings: %v = %v\n", - v.Settings[0].Key, - v.Settings[0].Value) - } - for _, vv := range v.Settings[1:] { - fmt.Printf(" %v = %v\n", vv.Key, - vv.Value) + // Parse metadata tag. This is the 'metadata:' part of the + // example metadata. + mdTag := regexMD.FindString(v) + if mdTag == "" { + // This is not metadata + continue } - } - - return nil -} - -func getFile(filename string) (*v1.File, *[sha256.Size]byte, error) { - var err error - - filename = util.CleanAndExpandPath(filename) - file := &v1.File{ - Name: filepath.Base(filename), - } - file.MIME, file.Digest, file.Payload, err = util.LoadFile(filename) - if err != nil { - return nil, nil, err - } - if !mime.MimeValid(file.MIME) { - return nil, nil, fmt.Errorf("unsupported mime type '%v' "+ - "for file '%v'", file.MIME, filename) - } - - // Get digest - digest, err := hex.DecodeString(file.Digest) - if err != nil { - return nil, nil, err - } - - // Store for merkle root verification later - var digest32 [sha256.Size]byte - copy(digest32[:], digest) - - return file, &digest32, nil -} - -func recordInventory() error { - // Prepare request - challenge, err := util.Random(v1.ChallengeSize) - if err != nil { - return err - } - - ibs, err := json.Marshal(v1.InventoryByStatus{ - Challenge: hex.EncodeToString(challenge), - }) - if err != nil { - return err - } - - if *printJson { - fmt.Println(string(ibs)) - } - // Make request - c, err := util.NewHTTPClient(verify, *rpccert) - if err != nil { - return err - } - req, err := http.NewRequest("POST", *rpchost+v1.InventoryByStatusRoute, - bytes.NewReader(ibs)) - if err != nil { - return err - } - req.SetBasicAuth(*rpcuser, *rpcpass) - r, err := c.Do(req) - if err != nil { - return err - } - defer r.Body.Close() + // Parse the full metatdata ID string. This is the "pluginid12:" + // part of the example metadata. + mdID := regexMDID.FindString(v) - // Verify status code response - if r.StatusCode != http.StatusOK { - e, err := getErrorFromResponse(r) + // Parse the plugin ID and stream ID + pluginID, streamID, err := parseMetadataIDs(mdID) if err != nil { - return fmt.Errorf("%v", r.Status) + return nil, err } - return fmt.Errorf("%v: %v", r.Status, e) - } - - bodyBytes := util.ConvertBodyToByteArray(r.Body, *printJson) - - var ibsr v1.InventoryByStatusReply - err = json.Unmarshal(bodyBytes, &ibsr) - if err != nil { - return fmt.Errorf("Could node unmarshal "+ - "InventoryByStatusReply: %v", err) - } - // Fetch remote identity - id, err := identity.LoadPublicIdentity(*identityFilename) - if err != nil { - return err - } - - // Verify challenge - err = util.VerifyChallenge(id, challenge, ibsr.Response) - if err != nil { - return err - } - - // Print response to user - fmt.Printf("Inventory:\n") - fmt.Printf(" Unvetted\n") - for status, tokens := range ibsr.Unvetted { - fmt.Printf(" %-15v: %v\n", - v1.RecordStatus[status], strings.Join(tokens, ", ")) - } - fmt.Printf(" Vetted\n") - for status, tokens := range ibsr.Vetted { - fmt.Printf(" %-15v: %v\n", - v1.RecordStatus[status], strings.Join(tokens, ", ")) + md = append(md, v2.MetadataStream{ + PluginID: pluginID, + StreamID: streamID, + Payload: v[len(mdTag)+len(mdID):], + }) } - return nil + return md, nil } -func newRecord() error { - flags := flag.Args()[1:] // Chop off action. - - // Fish out metadata records and filenames - md := make([]v1.MetadataStream, 0, len(flags)) - filenames := make([]string, 0, len(flags)) +// parseMetadata returns the metadata streams for all appendmetadata flags. +func parseMetadataAppend(flags []string) ([]v2.MetadataStream, error) { + md := make([]v2.MetadataStream, 0, len(flags)) for _, v := range flags { - mdRecord := regexMD.FindString(v) - if mdRecord == "" { - // Filename - filenames = append(filenames, v) - continue - } + // Example metadata: 'appendmetadata:pluginid12:{"moo":"lala"}' - id, err := strconv.ParseUint(regexMDID.FindString(mdRecord), - 10, 64) - if err != nil { - return err + // Parse append metadata tag. This is the 'appendmetadata:' part + // of the example metadata. + appendTag := regexAppendMD.FindString(v) + if appendTag == "" { + // This is not a metadata append + continue } - md = append(md, v1.MetadataStream{ - ID: id, - Payload: v[len(mdRecord):], - }) - } - - if len(filenames) == 0 { - return fmt.Errorf("no filenames provided") - } - - // Fetch remote identity - id, err := identity.LoadPublicIdentity(*identityFilename) - if err != nil { - return err - } - // Create New command - challenge, err := util.Random(v1.ChallengeSize) - if err != nil { - return err - } - n := v1.NewRecord{ - Challenge: hex.EncodeToString(challenge), - Metadata: md, - Files: make([]v1.File, 0, len(flags[1:])), - } + // Parse the full metatdata ID string. This is the "pluginid12:" + // part of the example metadata. + mdID := regexMDID.FindString(v) - // Open all files, validate MIME type and digest them. - hashes := make([]*[sha256.Size]byte, 0, len(flags[1:])) - for i, a := range filenames { - file, digest, err := getFile(a) + // Parse the plugin ID and stream ID + pluginID, streamID, err := parseMetadataIDs(mdID) if err != nil { - return err + return nil, err } - n.Files = append(n.Files, *file) - hashes = append(hashes, digest) - if !*printJson { - fmt.Printf("%02v: %v %v %v\n", - i, file.Digest, file.Name, file.MIME) - } + md = append(md, v2.MetadataStream{ + PluginID: pluginID, + StreamID: streamID, + Payload: v[len(appendTag)+len(mdID):], + }) } - if !*printJson { - fmt.Printf("Record submitted\n") - } + return md, nil +} - // Convert Verify to JSON - b, err := json.Marshal(n) - if err != nil { - return err - } +// parseMetadata returns the metadata streams for all overwritemetadata flags. +func parseMetadataOverwrite(flags []string) ([]v2.MetadataStream, error) { + md := make([]v2.MetadataStream, 0, len(flags)) + for _, v := range flags { + // Example metadata: 'overwritemetadata:pluginid12:{"moo":"lala"}' - if *printJson { - fmt.Println(string(b)) - } + // Parse overwrite metadata tag. This is the 'overwritemetadata:' + // part of the example metadata. + overwriteTag := regexOverwriteMD.FindString(v) + if overwriteTag == "" { + // This is not a metadata overwrite + continue + } - c, err := util.NewHTTPClient(verify, *rpccert) - if err != nil { - return err - } - r, err := c.Post(*rpchost+v1.NewRecordRoute, "application/json", - bytes.NewReader(b)) - if err != nil { - return err - } - defer r.Body.Close() + // Parse the full metatdata ID string. This is the "pluginid12:" + // part of the example metadata. + mdID := regexMDID.FindString(v) - if r.StatusCode != http.StatusOK { - e, err := getErrorFromResponse(r) + // Parse the plugin ID and stream ID + pluginID, streamID, err := parseMetadataIDs(mdID) if err != nil { - return fmt.Errorf("%v", r.Status) + return nil, err } - return fmt.Errorf("%v: %v", r.Status, e) - } - - bodyBytes := util.ConvertBodyToByteArray(r.Body, *printJson) - - var reply v1.NewRecordReply - err = json.Unmarshal(bodyBytes, &reply) - if err != nil { - return fmt.Errorf("Could node unmarshal NewReply: %v", err) - } - - // Verify challenge. - err = util.VerifyChallenge(id, challenge, reply.Response) - if err != nil { - return err - } - - // Convert merkle, token and signature to verify reply. - root, err := hex.DecodeString(reply.CensorshipRecord.Merkle) - if err != nil { - return err - } - sig, err := hex.DecodeString(reply.CensorshipRecord.Signature) - if err != nil { - return err - } - var signature [identity.SignatureSize]byte - copy(signature[:], sig) - - // Verify merkle root. - m := merkle.Root(hashes) - if !bytes.Equal(m[:], root) { - return fmt.Errorf("invalid merkle root; got %x, want %x", - root, m[:]) - } - - // Verify record token signature. - merkleToken := reply.CensorshipRecord.Merkle + reply.CensorshipRecord.Token - if !id.VerifyMessage([]byte(merkleToken), signature) { - return fmt.Errorf("verification failed") - } - - if !*printJson { - fmt.Printf(" Server public key: %v\n", id.String()) - printCensorshipRecord(reply.CensorshipRecord) - } - - return nil -} - -func verifyRecord() error { - flags := flag.Args()[1:] // Chop off action. - - // Action arguments - pk := flags[0] - token := flags[1] - merkleRoot := flags[2] - signature := flags[3] - - if len(flags) < 4 { - return fmt.Errorf("Must pass all input parameters") - } - id, err := util.IdentityFromString(pk) - if err != nil { - return err - } - sig, err := util.ConvertSignature(signature) - if err != nil { - return err - } - - // Verify merkle+token msg against signature - if !id.VerifyMessage([]byte(merkleRoot+token), sig) { - return fmt.Errorf("Invalid censorship record signature") + md = append(md, v2.MetadataStream{ + PluginID: pluginID, + StreamID: streamID, + Payload: v[len(overwriteTag)+len(mdID):], + }) } - fmt.Printf("Public key : %s\n", pk) - fmt.Printf("Token : %s\n", token) - fmt.Printf("Merkle root: %s\n", merkleRoot) - fmt.Printf("Signature : %s\n\n", signature) - fmt.Println("Record successfully verified") - - return nil + return md, nil } -func validateMetadataFlags(flags []string) ([]v1.MetadataStream, []v1.MetadataStream, string, error) { - var mdAppend []v1.MetadataStream - var mdOverwrite []v1.MetadataStream - var token string - var tokenCount uint +// parseFiles returns the files for all filename flags. +func parseFiles(flags []string) ([]v2.File, error) { + // Parse file names from flags + filenames := make([]string, 0, len(flags)) for _, v := range flags { - switch { - case regexAppendMD.MatchString(v): - s := regexAppendMD.FindString(v) - i, err := strconv.ParseUint(regexMDID.FindString(s), - 10, 64) - if err != nil { - return nil, nil, "", err - } - mdAppend = append(mdAppend, v1.MetadataStream{ - ID: i, - Payload: v[len(s):], - }) - - case regexOverwriteMD.MatchString(v): - s := regexOverwriteMD.FindString(v) - i, err := strconv.ParseUint(regexMDID.FindString(s), - 10, 64) - if err != nil { - return nil, nil, "", err - } - mdOverwrite = append(mdOverwrite, v1.MetadataStream{ - ID: i, - Payload: v[len(s):], - }) - - case regexToken.MatchString(v): - if tokenCount != 0 { - return nil, nil, "", fmt.Errorf("only 1 token allowed") - } - s := regexToken.FindString(v) - token = v[len(s):] - tokenCount++ - - default: - return nil, nil, "", fmt.Errorf("invalid action %v", v) - } - } - - if tokenCount != 1 { - return nil, nil, "", fmt.Errorf("must provide token") - } - - return mdAppend, mdOverwrite, token, nil -} - -func updateVettedMetadata() error { - flags := flag.Args()[1:] // Chop off action. - - // Create New command - challenge, err := util.Random(v1.ChallengeSize) - if err != nil { - return err - } - n := v1.UpdateVettedMetadata{ - Challenge: hex.EncodeToString(challenge), - } - - // Fish out metadata records and filenames - mdAppend, mdOverwrite, token, err := validateMetadataFlags(flags) - if err != nil { - return err - } - - // Set request fields - n.MDAppend = mdAppend - n.MDOverwrite = mdOverwrite - n.Token = token - - // Fetch remote identity - id, err := identity.LoadPublicIdentity(*identityFilename) - if err != nil { - return err - } - - // Prety print - if *verbose { - fmt.Printf("Update vetted metadata: %v\n", n.Token) - if len(n.MDOverwrite) > 0 { - s := "Metadata overwrite: " - for _, v := range n.MDOverwrite { - fmt.Printf("%s%v", s, v.ID) - s = ", " - } - fmt.Printf("\n") - } - if len(n.MDAppend) > 0 { - s := "Metadata append: " - for _, v := range n.MDAppend { - fmt.Printf("%s%v", s, v.ID) - s = ", " - } - fmt.Printf("\n") + if regexMD.FindString(v) != "" { + // This is metadata, not a filename + continue } - } - // Convert Verify to JSON - b, err := json.Marshal(n) - if err != nil { - return err + // This is a filename + filenames = append(filenames, v) } - - if *printJson { - fmt.Println(string(b)) - } - - c, err := util.NewHTTPClient(verify, *rpccert) - if err != nil { - return err - } - req, err := http.NewRequest("POST", *rpchost+v1.UpdateVettedMetadataRoute, - bytes.NewReader(b)) - if err != nil { - return err - } - req.SetBasicAuth(*rpcuser, *rpcpass) - r, err := c.Do(req) - if err != nil { - return err + if len(filenames) == 0 { + return nil, fmt.Errorf("no filenames provided") } - defer r.Body.Close() - if r.StatusCode != http.StatusOK { - e, err := getErrorFromResponse(r) + // Read files from disk + files := make([]v2.File, 0, len(filenames)) + for _, v := range filenames { + f, _, err := getFile(v) if err != nil { - return fmt.Errorf("%v", r.Status) + return nil, err } - return fmt.Errorf("%v: %v", r.Status, e) + files = append(files, *f) } - bodyBytes := util.ConvertBodyToByteArray(r.Body, *printJson) - - var reply v1.UpdateVettedMetadataReply - err = json.Unmarshal(bodyBytes, &reply) - if err != nil { - return fmt.Errorf("Could node unmarshal UpdateReply: %v", err) - } + return files, nil - // Verify challenge. - return util.VerifyChallenge(id, challenge, reply.Response) } -func updateUnvettedMetadata() error { - flags := flag.Args()[1:] - - // Create new command - challenge, err := util.Random(v1.ChallengeSize) - if err != nil { - return err - } - uum := v1.UpdateUnvettedMetadata{ - Challenge: hex.EncodeToString(challenge), - } - - // Fish out metadata records and filenames from flags - mdAppend, mdOverwrite, token, err := validateMetadataFlags(flags) - if err != nil { - return err - } - - // Set request fields - uum.MDAppend = mdAppend - uum.MDOverwrite = mdOverwrite - uum.Token = token - - // Prety print - if *verbose { - fmt.Printf("Update unvetted metadata: %v\n", uum.Token) - if len(uum.MDOverwrite) > 0 { - s := "Metadata overwrite: " - for _, v := range uum.MDOverwrite { - fmt.Printf("%s%v", s, v.ID) - s = ", " - } - fmt.Printf("\n") - } - if len(uum.MDAppend) > 0 { - s := "Metadata append: " - for _, v := range uum.MDAppend { - fmt.Printf("%s%v", s, v.ID) - s = ", " - } - fmt.Printf("\n") - } - } - - // Convert request object to JSON - b, err := json.Marshal(uum) - if err != nil { - return err - } - if *printJson { - fmt.Println(string(b)) - } - - // Make request - c, err := util.NewHTTPClient(verify, *rpccert) - if err != nil { - return err - } - req, err := http.NewRequest("POST", *rpchost+v1.UpdateUnvettedMetadataRoute, - bytes.NewReader(b)) - if err != nil { - return err - } - req.SetBasicAuth(*rpcuser, *rpcpass) - r, err := c.Do(req) - if err != nil { - return err - } - defer r.Body.Close() - - // Verify status code response - if r.StatusCode != http.StatusOK { - e, err := getErrorFromResponse(r) - if err != nil { - return fmt.Errorf("%v", r.Status) +// parseFileAdds returns the files for all file add flags. +func parseFileAdds(flags []string) ([]v2.File, error) { + // Parse file names from flags + filenames := make([]string, 0, len(flags)) + for _, v := range flags { + fileAddTag := regexFileAdd.FindString(v) + if fileAddTag == "" { + // This is not a file add flag + continue } - return fmt.Errorf("%v: %v", r.Status, e) - } - // Fetch remote identity - id, err := identity.LoadPublicIdentity(*identityFilename) - if err != nil { - return err + // This is a filename + filenames = append(filenames, v[len(fileAddTag):]) } - // Prepare reply - bodyBytes := util.ConvertBodyToByteArray(r.Body, *printJson) - var reply v1.UpdateUnvettedMetadataReply - err = json.Unmarshal(bodyBytes, &reply) - if err != nil { - return fmt.Errorf("Could node unmarshal UpdateReply: %v", err) + // Read files from disk + files := make([]v2.File, 0, len(filenames)) + for _, v := range filenames { + f, _, err := getFile(v) + if err != nil { + return nil, err + } + files = append(files, *f) } - // Verify challenge - return util.VerifyChallenge(id, challenge, reply.Response) + return files, nil } -func updateRecord(vetted bool) error { - flags := flag.Args()[1:] // Chop off action. +// parseFileDels returns the filenames for all file del flags. +func parseFileDels(flags []string) []string { + // Parse file names from flags + filenames := make([]string, 0, len(flags)) + for _, v := range flags { + fileDelTag := regexFileDel.FindString(v) + if fileDelTag == "" { + // This is not a file del flag + continue + } - // Create New command - challenge, err := util.Random(v1.ChallengeSize) - if err != nil { - return err - } - n := v1.UpdateRecord{ - Challenge: hex.EncodeToString(challenge), + // This is a filename + filenames = append(filenames, v[len(fileDelTag):]) } + return filenames +} - // Fish out metadata records and filenames - var tokenCount uint +// parseToken returns the token from the flags. +func parseToken(flags []string) string { + var token string for _, v := range flags { - switch { - case regexAppendMD.MatchString(v): - s := regexAppendMD.FindString(v) - i, err := strconv.ParseUint(regexMDID.FindString(s), - 10, 64) - if err != nil { - return err - } - n.MDAppend = append(n.MDAppend, v1.MetadataStream{ - ID: i, - Payload: v[len(s):], - }) - - case regexOverwriteMD.MatchString(v): - s := regexOverwriteMD.FindString(v) - i, err := strconv.ParseUint(regexMDID.FindString(s), - 10, 64) - if err != nil { - return err - } - n.MDOverwrite = append(n.MDOverwrite, v1.MetadataStream{ - ID: i, - Payload: v[len(s):], - }) - - case regexFileAdd.MatchString(v): - s := regexFileAdd.FindString(v) - f, _, err := getFile(v[len(s):]) - if err != nil { - return err - } - n.FilesAdd = append(n.FilesAdd, *f) - - case regexFileDel.MatchString(v): - s := regexFileDel.FindString(v) - n.FilesDel = append(n.FilesDel, v[len(s):]) + tokenTag := regexToken.FindString(v) + if tokenTag == "" { + // This is not the token + continue + } + token = v[len(tokenTag):] + } + return token +} - case regexToken.MatchString(v): - if tokenCount != 0 { - return fmt.Errorf("only 1 token allowed") - } - s := regexToken.FindString(v) - n.Token = v[len(s):] - tokenCount++ +// decodeToken decodes the provided token string into a byte slice. The token +// must be a full length politeiad v2 token. +func decodeToken(t string) ([]byte, error) { + return util.TokenDecode(util.TokenTypeTstore, t) +} - default: - return fmt.Errorf("invalid action %v", v) - } +func convertStatus(s string) v2.RecordStatusT { + switch s { + case "unreviewed": + return v2.RecordStatusUnreviewed + case "public": + return v2.RecordStatusPublic + case "censored": + return v2.RecordStatusCensored + case "archived": + return v2.RecordStatusArchived } + return v2.RecordStatusInvalid +} - if tokenCount != 1 { - return fmt.Errorf("must provide token") +func convertState(s string) v2.RecordStateT { + switch s { + case "unvetted": + return v2.RecordStateUnvetted + case "vetted": + return v2.RecordStateVetted } + return v2.RecordStateInvalid +} - // Fetch remote identity - id, err := identity.LoadPublicIdentity(*identityFilename) +func getFile(filename string) (*v2.File, *[sha256.Size]byte, error) { + var err error + + filename = util.CleanAndExpandPath(filename) + file := &v2.File{ + Name: filepath.Base(filename), + } + file.MIME, file.Digest, file.Payload, err = util.LoadFile(filename) if err != nil { - return err + return nil, nil, err } - - // Prety print - if *verbose { - fmt.Printf("Update record: %v\n", n.Token) - if len(n.FilesAdd) > 0 { - s := " Files add : " - ss := strings.Repeat(" ", len(s)) - for i, v := range n.FilesAdd { - fmt.Printf("%s%02v: %v %v %v\n", - s, i, v.Digest, v.Name, v.MIME) - s = ss - } - } - if len(n.FilesDel) > 0 { - s := " Files delete : " - ss := strings.Repeat(" ", len(s)) - for _, v := range n.FilesDel { - fmt.Printf("%s%v\n", s, v) - s = ss - } - } - if len(n.MDOverwrite) > 0 { - s := " Metadata overwrite: " - for _, v := range n.MDOverwrite { - fmt.Printf("%s%v", s, v.ID) - s = ", " - } - fmt.Printf("\n") - } - if len(n.MDAppend) > 0 { - s := " Metadata append : " - for _, v := range n.MDAppend { - fmt.Printf("%s%v", s, v.ID) - s = ", " - } - fmt.Printf("\n") - } + if !mime.MimeValid(file.MIME) { + return nil, nil, fmt.Errorf("unsupported mime type '%v' "+ + "for file '%v'", file.MIME, filename) } - // Convert Verify to JSON - b, err := json.Marshal(n) + // Get digest + digest, err := hex.DecodeString(file.Digest) if err != nil { - return err + return nil, nil, err } - if *printJson { - fmt.Println(string(b)) - } + // Store for merkle root verification later + var digest32 [sha256.Size]byte + copy(digest32[:], digest) + + return file, &digest32, nil +} - c, err := util.NewHTTPClient(verify, *rpccert) +// getIdentity retrieves the politeiad server identity, i.e. public key. +func getIdentity() error { + // Fetch remote identity + c, err := pdclient.New(*rpchost, *rpccert, *rpcuser, *rpcpass, nil) if err != nil { return err } - route := *rpchost + v1.UpdateUnvettedRoute - if vetted { - route = *rpchost + v1.UpdateVettedRoute - } - r, err := c.Post(route, "application/json", bytes.NewReader(b)) + id, err := c.Identity(context.Background()) if err != nil { return err } - defer r.Body.Close() - if r.StatusCode != http.StatusOK { - e, err := getErrorFromResponse(r) - if err != nil { - return fmt.Errorf("%v", r.Status) + rf := filepath.Join(defaultHomeDir, defaultIdentityFilename) + + // Pretty print identity. + fmt.Printf("Key : %x\n", id.Key) + fmt.Printf("Fingerprint: %v\n", id.Fingerprint()) + + // Ask user if we like this identity + if *interactive != allowInteractive { + fmt.Printf("\nSave to %v or ctrl-c to abort ", rf) + scanner := bufio.NewScanner(os.Stdin) + scanner.Scan() + if err = scanner.Err(); err != nil { + return err + } + if len(scanner.Text()) != 0 { + rf = scanner.Text() } - return fmt.Errorf("%v: %v", r.Status, e) + } else { + fmt.Printf("Saving identity to %v\n", rf) } + rf = util.CleanAndExpandPath(rf) - bodyBytes := util.ConvertBodyToByteArray(r.Body, *printJson) - - var reply v1.UpdateRecordReply - err = json.Unmarshal(bodyBytes, &reply) + // Save identity + err = os.MkdirAll(filepath.Dir(rf), 0700) if err != nil { - return fmt.Errorf("Could node unmarshal UpdateReply: %v", err) + return err } - - // Verify challenge. - err = util.VerifyChallenge(id, challenge, reply.Response) + err = id.SavePublicIdentity(rf) if err != nil { return err } + fmt.Printf("Identity saved to: %v\n", rf) return nil } -func getUnvetted() error { +// recordNew submits a new record to the politeiad v2 API. +func recordNew() error { flags := flag.Args()[1:] // Chop off action. - // Make sure we have the censorship token - if len(flags) != 1 { - return fmt.Errorf("must provide one and only one censorship " + - "token") + // Parse metadata and files + metadata, err := parseMetadata(flags) + if err != nil { + return err } - - // Validate censorship token - _, err := util.ConvertStringToken(flags[0]) + files, err := parseFiles(flags) if err != nil { return err } - // Fetch remote identity - id, err := identity.LoadPublicIdentity(*identityFilename) + // Load server identity + pid, err := identity.LoadPublicIdentity(*identityFilename) if err != nil { return err } - // Create New command - challenge, err := util.Random(v1.ChallengeSize) + // Setup client + c, err := pdclient.New(*rpchost, *rpccert, *rpcuser, *rpcpass, pid) if err != nil { return err } - n := v1.GetUnvetted{ - Challenge: hex.EncodeToString(challenge), - Token: flags[0], - } - // Convert to JSON - b, err := json.Marshal(n) + // Submit record + r, err := c.RecordNew(context.Background(), metadata, files) if err != nil { return err } - if *printJson { - fmt.Println(string(b)) + if *verbose { + printRecord("Record submitted", *r) + fmt.Printf("Server public key: %v\n", pid.String()) } - c, err := util.NewHTTPClient(verify, *rpccert) - if err != nil { - return err + // Verify record + return pdclient.RecordVerify(*r, pid.String()) +} + +// recordVerify verifies that a record was submitted by verifying the +// censorship record signature. +func recordVerify() error { + flags := flag.Args()[1:] // Chop off action. + if len(flags) < 3 { + return fmt.Errorf("arguments are missing") } - r, err := c.Post(*rpchost+v1.GetUnvettedRoute, "application/json", - bytes.NewReader(b)) + + // Unpack args + var ( + serverKey = flags[0] + token = flags[1] + signature = flags[2] + ) + + // Parse files + files, err := parseFiles(flags[3:]) if err != nil { return err } - defer r.Body.Close() - - if r.StatusCode != http.StatusOK { - e, err := getErrorFromResponse(r) - if err != nil { - return fmt.Errorf("%v", r.Status) - } - return fmt.Errorf("%v: %v", r.Status, e) + if len(files) == 0 { + return fmt.Errorf("no files found") } - bodyBytes := util.ConvertBodyToByteArray(r.Body, *printJson) - - var reply v1.GetUnvettedReply - err = json.Unmarshal(bodyBytes, &reply) + // Calc merkle root of files + digests := make([]string, 0, len(files)) + for _, v := range files { + digests = append(digests, v.Digest) + } + mr, err := util.MerkleRoot(digests) if err != nil { - return fmt.Errorf("Could not unmarshal GetUnvettedReply: %v", - err) + return err } + merkle := hex.EncodeToString(mr[:]) - // Verify challenge. - err = util.VerifyChallenge(id, challenge, reply.Response) + // Load identity + pid, err := util.IdentityFromString(serverKey) if err != nil { return err } - switch reply.Record.Status { - case v1.RecordStatusInvalid, v1.RecordStatusNotFound: - status, ok := v1.RecordStatus[reply.Record.Status] - if !ok { - status = v1.RecordStatus[v1.RecordStatusInvalid] - } - fmt.Printf("Record : %v\n", flags[0]) - fmt.Printf(" Status : %v\n", status) - case v1.RecordStatusCensored: - // Censored records will not contain any file so the verification - // is skipped. - if !*printJson { - printRecord("Unvetted record", reply.Record) - } - default: - // Verify content - err = v1.Verify(*id, reply.Record.CensorshipRecord, - reply.Record.Files) - if err != nil { - return err - } - if !*printJson { - printRecord("Unvetted record", reply.Record) - } + // Verify record + r := v2.Record{ + Files: files, + CensorshipRecord: v2.CensorshipRecord{ + Token: token, + Merkle: merkle, + Signature: signature, + }, + } + err = pdclient.RecordVerify(r, pid.String()) + if err != nil { + return err } + fmt.Printf("Server key : %s\n", serverKey) + fmt.Printf("Token : %s\n", token) + fmt.Printf("Merkle root: %s\n", merkle) + fmt.Printf("Signature : %s\n\n", signature) + fmt.Println("Record successfully verified") + return nil } -func getVetted() error { +// recordEdit edits an existing record. +func recordEdit() error { flags := flag.Args()[1:] // Chop off action. - // Make sure we have the censorship token - if len(flags) != 1 { - return fmt.Errorf("must provide one and only one censorship " + - "token") + // Parse args + mdAppend, err := parseMetadataAppend(flags) + if err != nil { + return err } - - // Validate censorship token - _, err := util.ConvertStringToken(flags[0]) + mdOverwrite, err := parseMetadataOverwrite(flags) if err != nil { return err } - - // Fetch remote identity - id, err := identity.LoadPublicIdentity(*identityFilename) + fileAdds, err := parseFileAdds(flags) if err != nil { return err } + fileDels := parseFileDels(flags) + token := parseToken(flags) + if token == "" { + return fmt.Errorf("must provide token") + } - // Create New command - challenge, err := util.Random(v1.ChallengeSize) + // Load server identity + pid, err := identity.LoadPublicIdentity(*identityFilename) if err != nil { return err } - n := v1.GetVetted{ - Challenge: hex.EncodeToString(challenge), - Token: flags[0], + + // Setup client + c, err := pdclient.New(*rpchost, *rpccert, *rpcuser, *rpcpass, pid) + if err != nil { + return err } - // Convert to JSON - b, err := json.Marshal(n) + // Edit record + r, err := c.RecordEdit(context.Background(), token, + mdAppend, mdOverwrite, fileAdds, fileDels) if err != nil { return err } - if *printJson { - fmt.Println(string(b)) + if *verbose { + printRecord("Record updated", *r) + fmt.Printf("Server public key: %v\n", pid.String()) } - c, err := util.NewHTTPClient(verify, *rpccert) + // Verify record + return pdclient.RecordVerify(*r, pid.String()) +} + +// recordEditMetadata edits the metadata of a record. +func recordEditMetadata() error { + flags := flag.Args()[1:] // Chop off action. + + // Parse args + mdAppend, err := parseMetadataAppend(flags) if err != nil { return err } - r, err := c.Post(*rpchost+v1.GetVettedRoute, "application/json", - bytes.NewReader(b)) + mdOverwrite, err := parseMetadataOverwrite(flags) if err != nil { return err } - defer r.Body.Close() - - if r.StatusCode != http.StatusOK { - e, err := getErrorFromResponse(r) - if err != nil { - return fmt.Errorf("%v", r.Status) - } - return fmt.Errorf("%v: %v", r.Status, e) + token := parseToken(flags) + if token == "" { + return fmt.Errorf("must provide token") } - bodyBytes := util.ConvertBodyToByteArray(r.Body, *printJson) - - var reply v1.GetVettedReply - err = json.Unmarshal(bodyBytes, &reply) + // Load server identity + pid, err := identity.LoadPublicIdentity(*identityFilename) if err != nil { - return fmt.Errorf("Could not unmarshal GetVettedReply: %v", - err) + return err } - // Verify challenge. - err = util.VerifyChallenge(id, challenge, reply.Response) + // Setup client + c, err := pdclient.New(*rpchost, *rpccert, *rpcuser, *rpcpass, pid) if err != nil { return err } - switch reply.Record.Status { - case v1.RecordStatusInvalid, v1.RecordStatusNotFound: - status, ok := v1.RecordStatus[reply.Record.Status] - if !ok { - status = v1.RecordStatus[v1.RecordStatusInvalid] - } - fmt.Printf("Record : %v\n", flags[0]) - fmt.Printf(" Status : %v\n", status) - case v1.RecordStatusCensored: - // Censored records will not contain any file so the verification - // is skipped. - if !*printJson { - printRecord("Vetted record", reply.Record) - } - default: - // Verify content - err = v1.Verify(*id, reply.Record.CensorshipRecord, - reply.Record.Files) - if err != nil { - return err - } - if !*printJson { - printRecord("Vetted record", reply.Record) - } + // Edit record metadata + r, err := c.RecordEditMetadata(context.Background(), + token, mdAppend, mdOverwrite) + if err != nil { + return err } - return nil -} - -func convertStatus(s string) (v1.RecordStatusT, error) { - switch s { - case "censored": - return v1.RecordStatusCensored, nil - case "public": - return v1.RecordStatusPublic, nil - case "archived": - return v1.RecordStatusArchived, nil + if *verbose { + printRecord("Record metadata updated", *r) + fmt.Printf("Server public key: %v\n", pid.String()) } - return v1.RecordStatusInvalid, fmt.Errorf("invalid status") + // Verify record + return pdclient.RecordVerify(*r, pid.String()) } -func setUnvettedStatus() error { - flags := flag.Args()[1:] // Chop off action. +// recordSetStatus sets the status of a record. +func recordSetStatus() error { + flags := flag.Args()[1:] // Make sure we have the status and the censorship token if len(flags) < 2 { @@ -1274,292 +658,194 @@ func setUnvettedStatus() error { "censorship token") } - // Verify we got a valid status - status, err := convertStatus(flags[0]) - if err != nil { - return err - } - // Validate censorship token - _, err = util.ConvertStringToken(flags[1]) - if err != nil { - return err - } - - // Fetch remote identity - id, err := identity.LoadPublicIdentity(*identityFilename) + token := flags[0] + _, err := decodeToken(token) if err != nil { return err } - // Create New command - challenge, err := util.Random(v1.ChallengeSize) - if err != nil { - return err - } - n := v1.SetUnvettedStatus{ - Challenge: hex.EncodeToString(challenge), - Status: status, - Token: flags[1], - } - - // Optional metadata updates - for _, v := range flags[2:] { - switch { - case regexAppendMD.MatchString(v): - s := regexAppendMD.FindString(v) - i, err := strconv.ParseUint(regexMDID.FindString(s), - 10, 64) - if err != nil { - return err - } - n.MDAppend = append(n.MDAppend, v1.MetadataStream{ - ID: i, - Payload: v[len(s):], - }) - - case regexOverwriteMD.MatchString(v): - s := regexOverwriteMD.FindString(v) - i, err := strconv.ParseUint(regexMDID.FindString(s), - 10, 64) - if err != nil { - return err - } - n.MDOverwrite = append(n.MDOverwrite, v1.MetadataStream{ - ID: i, - Payload: v[len(s):], - }) - default: - return fmt.Errorf("invalid metadata action %v", v) - } + // Validate status + status := convertStatus(flags[1]) + if status == v2.RecordStatusInvalid { + return fmt.Errorf("invalid status") } - // Convert to JSON - b, err := json.Marshal(n) + // Load server identity + pid, err := identity.LoadPublicIdentity(*identityFilename) if err != nil { return err } - if *printJson { - fmt.Println(string(b)) - } - - c, err := util.NewHTTPClient(verify, *rpccert) - if err != nil { - return err - } - req, err := http.NewRequest("POST", *rpchost+v1.SetUnvettedStatusRoute, - bytes.NewReader(b)) - if err != nil { - return err - } - req.SetBasicAuth(*rpcuser, *rpcpass) - r, err := c.Do(req) + // Setup client + c, err := pdclient.New(*rpchost, *rpccert, *rpcuser, *rpcpass, pid) if err != nil { return err } - defer r.Body.Close() - - if r.StatusCode != http.StatusOK { - e, err := getErrorFromResponse(r) - if err != nil { - return fmt.Errorf("%v", r.Status) - } - return fmt.Errorf("%v: %v", r.Status, e) - } - - bodyBytes := util.ConvertBodyToByteArray(r.Body, *printJson) - - var reply v1.SetUnvettedStatusReply - err = json.Unmarshal(bodyBytes, &reply) - if err != nil { - return fmt.Errorf("Could not unmarshal "+ - "SetUnvettedStatusReply: %v", err) - } - // Verify challenge. - err = util.VerifyChallenge(id, challenge, reply.Response) + // Set record status + r, err := c.RecordSetStatus(context.Background(), + token, status, nil, nil) if err != nil { return err } - if !*printJson { - // Pretty print record - status, ok := v1.RecordStatus[n.Status] - if !ok { - status = v1.RecordStatus[v1.RecordStatusInvalid] - } - fmt.Printf("Set unvetted record status:\n") - fmt.Printf(" Status : %v\n", status) + if *verbose { + printRecord("Record status updated", *r) + fmt.Printf("Server public key: %v\n", pid.String()) } - return nil + // Verify record + return pdclient.RecordVerify(*r, pid.String()) } -func setVettedStatus() error { - flags := flag.Args()[1:] - - // Make sure we have the status and the censorship token - if len(flags) < 2 { - return fmt.Errorf("must at least provide status and " + - "censorship token") - } +// record retreives a record. +func record() error { + flags := flag.Args()[1:] // Chop off action. - // Validate status - status, err := convertStatus(flags[0]) - if err != nil { - return err + // Make sure we have the censorship token + if len(flags) != 1 { + return fmt.Errorf("must provide one and only one censorship " + + "token") } // Validate censorship token - _, err = util.ConvertStringToken(flags[1]) + token := flags[0] + _, err := decodeToken(token) if err != nil { return err } - // Create command - challenge, err := util.Random(v1.ChallengeSize) + // Load server identity + pid, err := identity.LoadPublicIdentity(*identityFilename) if err != nil { return err } - sus := v1.SetVettedStatus{ - Challenge: hex.EncodeToString(challenge), - Status: status, - Token: flags[1], - } - - // Optional metadata updates - for _, md := range flags[2:] { - switch { - case regexAppendMD.MatchString(md): - s := regexAppendMD.FindString(md) - i, err := strconv.ParseUint(regexMDID.FindString(s), - 10, 64) - if err != nil { - return err - } - sus.MDAppend = append(sus.MDAppend, v1.MetadataStream{ - ID: i, - Payload: md[len(s):], - }) - - case regexOverwriteMD.MatchString(md): - s := regexOverwriteMD.FindString(md) - i, err := strconv.ParseUint(regexMDID.FindString(s), - 10, 64) - if err != nil { - return err - } - sus.MDOverwrite = append(sus.MDOverwrite, v1.MetadataStream{ - ID: i, - Payload: md[len(s):], - }) - default: - return fmt.Errorf("invalid metadata action %v", md) - } - } - // Convert command object to JSON - b, err := json.Marshal(sus) + // Setup client + c, err := pdclient.New(*rpchost, *rpccert, *rpcuser, *rpcpass, pid) if err != nil { return err } - if *printJson { - fmt.Println(string(b)) - } - // Make request - c, err := util.NewHTTPClient(verify, *rpccert) - if err != nil { - return err + // Set record status + reqs := []v2.RecordRequest{ + { + Token: token, + }, } - req, err := http.NewRequest("POST", *rpchost+v1.SetVettedStatusRoute, - bytes.NewReader(b)) + records, err := c.Records(context.Background(), reqs) if err != nil { return err } - req.SetBasicAuth(*rpcuser, *rpcpass) - r, err := c.Do(req) - if err != nil { - return err + r, ok := records[token] + if !ok { + return fmt.Errorf("record not found") } - defer r.Body.Close() - // Verify status code response - if r.StatusCode != http.StatusOK { - e, err := getErrorFromResponse(r) + if *verbose { + printRecord("Record", r) + fmt.Printf("Server public key: %v\n", pid.String()) + } + + // Verify record + return pdclient.RecordVerify(r, pid.String()) +} + +// recordInventory retrieves the censorship record tokens of the records in +// the inventory, categorized by their record state and record status. +func recordInventory() error { + flags := flag.Args()[1:] // Chop off action. + + // Either the state, status and page number must all be given or + // none should be given at all. + if len(flags) > 0 && len(flags) != 3 { + return fmt.Errorf("invalid number of arguments (%v); you can " + + "either provide a state, status, and page number or you can " + + "provide no arguments at all") + } + + // Unpack args + var ( + state v2.RecordStateT + status v2.RecordStatusT + pageNumber uint32 + ) + if len(flags) == 3 { + state = convertState(flags[0]) + status = convertStatus(flags[1]) + u, err := strconv.ParseUint(flags[2], 10, 64) if err != nil { - return fmt.Errorf("%v", r.Status) + return fmt.Errorf("unable to parse page number '%v': %v", + flags[2], err) } - return fmt.Errorf("%v: %v", r.Status, e) + pageNumber = uint32(u) } - // Prepare reply - bodyBytes := util.ConvertBodyToByteArray(r.Body, *printJson) - var reply v1.SetVettedStatusReply - err = json.Unmarshal(bodyBytes, &reply) + // Load server identity + pid, err := identity.LoadPublicIdentity(*identityFilename) if err != nil { - return fmt.Errorf("Could not unmarshal "+ - "SetVettedStatusReply: %v", err) + return err } - // Fetch remote identity - id, err := identity.LoadPublicIdentity(*identityFilename) + // Setup client + c, err := pdclient.New(*rpchost, *rpccert, *rpcuser, *rpcpass, pid) if err != nil { return err } - // Verify challenge. - err = util.VerifyChallenge(id, challenge, reply.Response) + // Get inventory + ir, err := c.Inventory(context.Background(), state, status, pageNumber) if err != nil { return err } - if !*printJson { - // Pretty print record - status, ok := v1.RecordStatus[sus.Status] - if !ok { - status = v1.RecordStatus[v1.RecordStatusInvalid] + if *verbose { + if len(ir.Unvetted) > 0 { + fmt.Printf("Unvetted\n") + fmt.Printf("%v\n", util.FormatJSON(ir.Unvetted)) + } + if len(ir.Vetted) > 0 { + fmt.Printf("Vetted\n") + fmt.Printf("%v\n", util.FormatJSON(ir.Vetted)) } - fmt.Printf("Set vetted record status:\n") - fmt.Printf(" Status : %v\n", status) } return nil } func _main() error { + flag.Usage = usage flag.Parse() if len(flag.Args()) == 0 { usage() return fmt.Errorf("must provide action") } + // Setup RPC host if *rpchost == "" { if *testnet { *rpchost = v1.DefaultTestnetHost } else { *rpchost = v1.DefaultMainnetHost } - } else { - // For now assume we can't verify server TLS certificate - verify = true } - port := v1.DefaultMainnetPort if *testnet { port = v1.DefaultTestnetPort } - *rpchost = util.NormalizeAddress(*rpchost, port) - - // Set port if not specified. u, err := url.Parse("https://" + *rpchost) if err != nil { return err } *rpchost = u.String() + // Setup RPC cert + if *rpccert == "" { + *rpccert = defaultRPCCertFile + } + // Scan through command line arguments. for i, a := range flag.Args() { // Select action @@ -1567,30 +853,18 @@ func _main() error { switch a { case "identity": return getIdentity() - case "verify": - return verifyRecord() case "new": - return newRecord() - case "updateunvetted": - return updateRecord(false) - case "updateunvettedmd": - return updateUnvettedMetadata() - case "setunvettedstatus": - return setUnvettedStatus() - case "getunvetted": - return getUnvetted() - case "updatevetted": - return updateRecord(true) - case "updatevettedmd": - return updateVettedMetadata() - case "setvettedstatus": - return setVettedStatus() - case "getvetted": - return getVetted() - case "plugin": - return plugin() - case "plugininventory": - return getPluginInventory() + return recordNew() + case "verify": + return recordVerify() + case "edit": + return recordEdit() + case "editmetadata": + return recordEditMetadata() + case "setstatus": + return recordSetStatus() + case "record": + return record() case "inventory": return recordInventory() default: From ecd91d2c7949f5582e86e87cec93fcf5a22cefc2 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 25 Mar 2021 11:53:31 -0500 Subject: [PATCH 437/449] Fix linter errors. --- politeiad/backendv2/tstorebe/store/mysql/encrypt.go | 3 +++ politeiad/backendv2/tstorebe/tstore/tlogclient.go | 1 - politeiad/cmd/politeia/politeia.go | 6 +++--- politeiad/plugins/pi/pi.go | 2 +- util/token.go | 2 +- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/politeiad/backendv2/tstorebe/store/mysql/encrypt.go b/politeiad/backendv2/tstorebe/store/mysql/encrypt.go index a8c3904da..6c8088c1d 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/encrypt.go +++ b/politeiad/backendv2/tstorebe/store/mysql/encrypt.go @@ -137,6 +137,9 @@ func (s *mysql) getDbNonce(ctx context.Context, tx *sql.Tx) ([24]byte, error) { func (s *mysql) getTestNonce(ctx context.Context, tx *sql.Tx) ([24]byte, error) { nonce, err := util.Random(8) + if err != nil { + return emptyNonce, err + } n, err := sbox.NewNonceFromBytes(nonce) if err != nil { return emptyNonce, err diff --git a/politeiad/backendv2/tstorebe/tstore/tlogclient.go b/politeiad/backendv2/tstorebe/tstore/tlogclient.go index 7eca30c16..f8b22ab06 100644 --- a/politeiad/backendv2/tstorebe/tstore/tlogclient.go +++ b/politeiad/backendv2/tstorebe/tstore/tlogclient.go @@ -699,7 +699,6 @@ func (t *testTClient) LeavesAppend(treeID int64, leavesAppend []*trillian.LogLea for _, v := range leavesAppend { // Append to leaves v.MerkleLeafHash = merkleLeafHash(v.LeafValue) - v.ExtraData = v.ExtraData v.LeafIndex = index + 1 leaves = append(leaves, v) index++ diff --git a/politeiad/cmd/politeia/politeia.go b/politeiad/cmd/politeia/politeia.go index e6a17b86a..fa567338c 100644 --- a/politeiad/cmd/politeia/politeia.go +++ b/politeiad/cmd/politeia/politeia.go @@ -760,9 +760,9 @@ func recordInventory() error { // Either the state, status and page number must all be given or // none should be given at all. if len(flags) > 0 && len(flags) != 3 { - return fmt.Errorf("invalid number of arguments (%v); you can " + - "either provide a state, status, and page number or you can " + - "provide no arguments at all") + return fmt.Errorf("invalid number of arguments (%v); you can "+ + "either provide a state, status, and page number or you can "+ + "provide no arguments at all", len(flags)) } // Unpack args diff --git a/politeiad/plugins/pi/pi.go b/politeiad/plugins/pi/pi.go index 7732a325d..f3dce557c 100644 --- a/politeiad/plugins/pi/pi.go +++ b/politeiad/plugins/pi/pi.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -// Pacakge pi provides a plugin that extends records with functionality for +// Package pi provides a plugin that extends records with functionality for // decred's proposal system. package pi diff --git a/util/token.go b/util/token.go index 972f24959..4dbeb3d07 100644 --- a/util/token.go +++ b/util/token.go @@ -121,7 +121,7 @@ func TokenDecodeAnyLength(tokenType, token string) ([]byte, error) { } // TokenEncode returns the hex encoded token. Its possible that padding has -// been added to the token when it was orginally decode in order to make it +// been added to the token when it was originally decode in order to make it // valid hex. This function checks for padding and removes it before encoding // the token. func TokenEncode(token []byte) string { From ba5af4713bcd62b22ce07577e86cc76d9dcd733a Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 25 Mar 2021 20:59:26 -0500 Subject: [PATCH 438/449] www/politeiaverify: Refactor. --- politeiad/cmd/politeia/README.md | 34 +-- politeiawww/api/ticketvote/v1/v1.go | 16 +- politeiawww/client/client.go | 2 +- politeiawww/client/pi.go | 52 +++++ politeiawww/client/records.go | 176 ++++++++++++++ politeiawww/cmd/pictl/cmdproposaledit.go | 4 +- .../cmd/pictl/cmdproposaltimestamps.go | 52 +---- politeiawww/cmd/pictl/proposal.go | 69 +----- politeiawww/cmd/politeiaverify/README.md | 79 ++++--- .../cmd/politeiaverify/politeiaverify.go | 218 ++++++++++-------- politeiawww/cmd/politeiaverify/record.go | 100 ++++++++ politeiawww/cmd/politeiaverify/ticketvote.go | 18 ++ politeiawww/pi/events.go | 104 ++------- politeiawww/proposals.go | 5 +- politeiawww/records/process.go | 24 +- 15 files changed, 580 insertions(+), 373 deletions(-) create mode 100644 politeiawww/cmd/politeiaverify/record.go create mode 100644 politeiawww/cmd/politeiaverify/ticketvote.go diff --git a/politeiad/cmd/politeia/README.md b/politeiad/cmd/politeia/README.md index 765e60648..183741171 100644 --- a/politeiad/cmd/politeia/README.md +++ b/politeiad/cmd/politeia/README.md @@ -22,7 +22,8 @@ Available commands: ## Obtain politeiad identity -The retrieved identity is used to verify replies from politeiad. +The politeiad identity is the contains the public key that is sued to verify +replies from politeiad. ``` $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass identity @@ -44,6 +45,7 @@ Metadata is submitted as JSON and must be identified by a `pluginID` string and a `streamID` uint32. Metadata is passed in as an argument by prefixing the JSON with `metadata:[pluginID][streamID]:`. Below is an example metadata argument where the plugin ID is `testid` and the stream ID is `1`. + `metadata:testid1:{"foo":"bar"}` ``` @@ -180,14 +182,14 @@ Server public key: e88df79a4b02699e6c051adbae05f21f2a2f24942e0f27cade165548ec3d6 ## Set record status -Args: +Args: ` ` You can update the status of a record using one of the following statuses: - `public` - make the record a public - `archived` - lock the record from further edits - `censored` - lock the record from further edits and delete all files -Note `token:` is not prefixed to the token in this command. +Note, token is not prefixed with `token:` in this command. ``` $ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass setstatus 39868e5e91c78255 public @@ -245,6 +247,21 @@ categorized by their record state and record status. The user can request a page of tokens from a specific record state and record status by providing the arguments. +States: `unvetted`, `vetted` +Statuses: `unreviewed`, `public`, `censored`, `abandoned` + +``` +$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass inventory unvetted unreviewed 1 + +Unvetted +{ + "unreviewed": [ + "d0545038224c5054", + "ea260a4ab9170d70" + ] +} +``` + If not arguments are provided then a page of tokens for every state and status will be returned. @@ -275,15 +292,4 @@ Vetted "f1f7337397a79b51" ] } - - -$ politeia -v -testnet -rpchost 127.0.0.1 -rpcuser=user -rpcpass=pass inventory unvetted unreviewed 1 - -Unvetted -{ - "unreviewed": [ - "d0545038224c5054", - "ea260a4ab9170d70" - ] -} ``` diff --git a/politeiawww/api/ticketvote/v1/v1.go b/politeiawww/api/ticketvote/v1/v1.go index b8dfac829..7bfff7abf 100644 --- a/politeiawww/api/ticketvote/v1/v1.go +++ b/politeiawww/api/ticketvote/v1/v1.go @@ -609,7 +609,7 @@ const ( // // If no votes page number is provided then the vote authorization and vote // details timestamps will be returned. If a votes page number is provided then -// the specified page of votes will be returned. +// the specified page of cast vote timestamps will be returned. type Timestamps struct { Token string `json:"token"` VotesPage uint32 `json:"votespage,omitempty"` @@ -617,7 +617,15 @@ type Timestamps struct { // TimestampsReply is the reply to the Timestamps command. type TimestampsReply struct { - Auths []Timestamp `json:"auths,omitempty"` - Details *Timestamp `json:"details,omitempty"` - Votes []Timestamp `json:"votes,omitempty"` + // Auths contains the timestamps for vote authorizations. The data + // payloads will contain AuthDetails structures. + Auths []Timestamp `json:"auths,omitempty"` + + // Details contains the timestamps for the vote details. The data + // payload will contain a VoteDetails structure. + Details *Timestamp `json:"details,omitempty"` + + // Votes contains the timestamps for the cast votes. The data + // payloads will contain CastVoteDetails strucutures. + Votes []Timestamp `json:"votes,omitempty"` } diff --git a/politeiawww/client/client.go b/politeiawww/client/client.go index 287bbf9d4..8af0f6068 100644 --- a/politeiawww/client/client.go +++ b/politeiawww/client/client.go @@ -156,7 +156,7 @@ type Opts struct { HTTPSCert string Cookies []*http.Cookie HeaderCSRF string - Verbose bool // Pretty print details + Verbose bool // Print verbose output RawJSON bool // Print raw json } diff --git a/politeiawww/client/pi.go b/politeiawww/client/pi.go index 54184ea24..0cf0cc5b2 100644 --- a/politeiawww/client/pi.go +++ b/politeiawww/client/pi.go @@ -5,10 +5,13 @@ package client import ( + "encoding/base64" "encoding/json" + "fmt" "net/http" piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" ) // PiPolicy sends a pi v1 Policy request to politeiawww. @@ -27,3 +30,52 @@ func (c *Client) PiPolicy() (*piv1.PolicyReply, error) { return &pr, nil } + +// ProposalMetadataDecode decodes and returns the ProposalMetadata from the +// Provided record files. An error returned if a ProposalMetadata is not found. +func ProposalMetadataDecode(files []rcv1.File) (*piv1.ProposalMetadata, error) { + var pmp *piv1.ProposalMetadata + for _, v := range files { + if v.Name != piv1.FileNameProposalMetadata { + continue + } + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return nil, err + } + var pm piv1.ProposalMetadata + err = json.Unmarshal(b, &pm) + if err != nil { + return nil, err + } + pmp = &pm + break + } + if pmp == nil { + return nil, fmt.Errorf("proposal metadata not found") + } + return pmp, nil +} + +// VoteMetadataDecode decodes and returns the VoteMetadata from the provided +// backend files. Nil is returned if a VoteMetadata is not found. +func VoteMetadataDecode(files []rcv1.File) (*piv1.VoteMetadata, error) { + var vmp *piv1.VoteMetadata + for _, v := range files { + if v.Name != piv1.FileNameVoteMetadata { + continue + } + b, err := base64.StdEncoding.DecodeString(v.Payload) + if err != nil { + return nil, err + } + var vm piv1.VoteMetadata + err = json.Unmarshal(b, &vm) + if err != nil { + return nil, err + } + vmp = &vm + break + } + return vmp, nil +} diff --git a/politeiawww/client/records.go b/politeiawww/client/records.go index 0fea96f54..f4d0ee9b1 100644 --- a/politeiawww/client/records.go +++ b/politeiawww/client/records.go @@ -9,11 +9,20 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" + "errors" "fmt" + "io" "net/http" + "strconv" + "strings" + backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/tstore" + "github.com/decred/politeia/politeiad/plugins/usermd" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + v1 "github.com/decred/politeia/politeiawww/api/records/v1" "github.com/decred/politeia/util" + "github.com/google/uuid" ) // RecordNew sends a records v1 New request to politeiawww. @@ -232,3 +241,170 @@ func RecordVerify(r rcv1.Record, serverPubKey string) error { return nil } + +// RecordTimestampVerify verifies a records v1 API timestamp. This proves +// inclusion of the data in the merkle root that was timestamped onto the dcr +// blockchain. +func RecordTimestampVerify(t rcv1.Timestamp) error { + return tstore.VerifyTimestamp(convertRecordTimestamp(t)) +} + +// RecordTimestampsVerify verifies all timestamps in a records v1 API +// timestamps reply. This proves the inclusion of the data in the merkle root +// that was timestamped onto the dcr blockchain. +func RecordTimestampsVerify(tr rcv1.TimestampsReply) error { + err := RecordTimestampVerify(tr.RecordMetadata) + if err != nil { + return fmt.Errorf("could not verify record metadata timestamp: %v", err) + } + for pluginID, v := range tr.Metadata { + for streamID, ts := range v { + err = RecordTimestampVerify(ts) + if err != nil { + return fmt.Errorf("could not verify metadata %v %v timestamp: %v", + pluginID, streamID, err) + } + } + } + for k, v := range tr.Files { + err = RecordTimestampVerify(v) + if err != nil { + return fmt.Errorf("could not verify file %v timestamp: %v", k, err) + } + } + return nil +} + +// UserMetadataDecode decodes and returns the UserMetadata from the provided +// metadata streams. An error is returned if a UserMetadata is not found. +func UserMetadataDecode(ms []v1.MetadataStream) (*usermd.UserMetadata, error) { + var ump *usermd.UserMetadata + for _, v := range ms { + if v.PluginID != usermd.PluginID || + v.StreamID != usermd.StreamIDUserMetadata { + // Not user metadata + continue + } + var um usermd.UserMetadata + err := json.Unmarshal([]byte(v.Payload), &um) + if err != nil { + return nil, err + } + ump = &um + break + } + if ump == nil { + return nil, fmt.Errorf("user metadata not found") + } + return ump, nil +} + +// UserMetadataVerify verifies that the UserMetadata contains a valid user ID, +// a valid public key, and that this signature is a valid signature of the +// record merkle root. An error is returned if a UserMetadata is not found. +func UserMetadataVerify(metadata []v1.MetadataStream, files []v1.File) error { + // Decode user metadata + um, err := UserMetadataDecode(metadata) + if err != nil { + return err + } + + // Verify user ID + _, err = uuid.Parse(um.UserID) + if err != nil { + return fmt.Errorf("invalid user id: %v", err) + } + + // Verify signature + digests := make([]string, 0, len(files)) + for _, v := range files { + digests = append(digests, v.Digest) + } + m, err := util.MerkleRoot(digests) + if err != nil { + return err + } + mr := hex.EncodeToString(m[:]) + err = util.VerifySignature(um.Signature, um.PublicKey, mr) + if err != nil { + return fmt.Errorf("invalid user metadata: %v", err) + } + + return nil +} + +// StatusChangesDecode decodes and returns the status changes metadata stream +// from the provided metadata. An error IS NOT returned is status change +// metadata is not found. +func StatusChangesDecode(metadata []v1.MetadataStream) ([]v1.StatusChange, error) { + statuses := make([]v1.StatusChange, 0, 16) + for _, v := range metadata { + if v.PluginID != usermd.PluginID || + v.StreamID != usermd.StreamIDStatusChanges { + // Not status change metadata + continue + } + d := json.NewDecoder(strings.NewReader(v.Payload)) + for { + var sc v1.StatusChange + err := d.Decode(&sc) + if errors.Is(err, io.EOF) { + break + } else if err != nil { + return nil, err + } + statuses = append(statuses, sc) + } + break + } + return statuses, nil +} + +// StatusChanges verifies the signatures on all status change metadata. A +// record might not have any status changes yet so an error IS NOT returned if +// status change metadata is not found. +func StatusChangesVerify(metadata []v1.MetadataStream) error { + // Decode status changes + sc, err := StatusChangesDecode(metadata) + if err != nil { + return err + } + + // Verify signatures + for _, v := range sc { + status := strconv.FormatUint(uint64(v.Status), 10) + version := strconv.FormatUint(uint64(v.Version), 10) + msg := v.Token + version + status + v.Reason + err = util.VerifySignature(v.Signature, v.PublicKey, msg) + if err != nil { + return fmt.Errorf("invalid status change signature %v %v: %v", + v.Token, v1.RecordStatuses[v.Status], err) + } + } + + return nil +} + +func convertRecordProof(p rcv1.Proof) backend.Proof { + return backend.Proof{ + Type: p.Type, + Digest: p.Digest, + MerkleRoot: p.MerkleRoot, + MerklePath: p.MerklePath, + ExtraData: p.ExtraData, + } +} + +func convertRecordTimestamp(t rcv1.Timestamp) backend.Timestamp { + proofs := make([]backend.Proof, 0, len(t.Proofs)) + for _, v := range t.Proofs { + proofs = append(proofs, convertRecordProof(v)) + } + return backend.Timestamp{ + Data: t.Data, + Digest: t.Digest, + TxID: t.TxID, + MerkleRoot: t.MerkleRoot, + Proofs: proofs, + } +} diff --git a/politeiawww/cmd/pictl/cmdproposaledit.go b/politeiawww/cmd/pictl/cmdproposaledit.go index 909036969..35fd98eab 100644 --- a/politeiawww/cmd/pictl/cmdproposaledit.go +++ b/politeiawww/cmd/pictl/cmdproposaledit.go @@ -152,7 +152,7 @@ func proposalEdit(c *cmdProposalEdit) (*rcv1.Record, error) { switch { case c.UseMD: // Use the existing proposal name - pm, err := proposalMetadataDecode(curr.Files) + pm, err := pclient.ProposalMetadataDecode(curr.Files) if err != nil { return nil, err } @@ -184,7 +184,7 @@ func proposalEdit(c *cmdProposalEdit) (*rcv1.Record, error) { switch { case c.UseMD: // Use existing vote metadata values - vm, err := voteMetadataDecode(curr.Files) + vm, err := pclient.VoteMetadataDecode(curr.Files) if err != nil { return nil, err } diff --git a/politeiawww/cmd/pictl/cmdproposaltimestamps.go b/politeiawww/cmd/pictl/cmdproposaltimestamps.go index d9b0dd8d7..923af086e 100644 --- a/politeiawww/cmd/pictl/cmdproposaltimestamps.go +++ b/politeiawww/cmd/pictl/cmdproposaltimestamps.go @@ -5,10 +5,6 @@ package main import ( - "fmt" - - backend "github.com/decred/politeia/politeiad/backendv2" - "github.com/decred/politeia/politeiad/backendv2/tstorebe/tstore" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" pclient "github.com/decred/politeia/politeiawww/client" ) @@ -49,24 +45,9 @@ func (c *cmdProposalTimestamps) Execute(args []string) error { } // Verify timestamps - err = verifyTimestamp(tr.RecordMetadata) + err = pclient.RecordTimestampsVerify(*tr) if err != nil { - return fmt.Errorf("verify proposal metadata timestamp: %v", err) - } - for pluginID, v := range tr.Metadata { - for streamID, ts := range v { - err = verifyTimestamp(ts) - if err != nil { - return fmt.Errorf("verify metadata %v %v timestamp: %v", - pluginID, streamID, err) - } - } - } - for k, v := range tr.Files { - err = verifyTimestamp(v) - if err != nil { - return fmt.Errorf("verify file %v timestamp: %v", k, err) - } + return err } // Print timestamps @@ -75,35 +56,6 @@ func (c *cmdProposalTimestamps) Execute(args []string) error { return nil } -func verifyTimestamp(t rcv1.Timestamp) error { - ts := convertTimestamp(t) - return tstore.VerifyTimestamp(ts) -} - -func convertProof(p rcv1.Proof) backend.Proof { - return backend.Proof{ - Type: p.Type, - Digest: p.Digest, - MerkleRoot: p.MerkleRoot, - MerklePath: p.MerklePath, - ExtraData: p.ExtraData, - } -} - -func convertTimestamp(t rcv1.Timestamp) backend.Timestamp { - proofs := make([]backend.Proof, 0, len(t.Proofs)) - for _, v := range t.Proofs { - proofs = append(proofs, convertProof(v)) - } - return backend.Timestamp{ - Data: t.Data, - Digest: t.Digest, - TxID: t.TxID, - MerkleRoot: t.MerkleRoot, - Proofs: proofs, - } -} - // proposalTimestampsHelpMsg is printed to stdout by the help command. const proposalTimestampsHelpMsg = `proposaltimestamps [flags] "token" "version" diff --git a/politeiawww/cmd/pictl/proposal.go b/politeiawww/cmd/pictl/proposal.go index 70557d70c..0872a6b98 100644 --- a/politeiawww/cmd/pictl/proposal.go +++ b/politeiawww/cmd/pictl/proposal.go @@ -8,13 +8,10 @@ import ( "bytes" "encoding/base64" "encoding/hex" - "encoding/json" - "errors" "fmt" "image" "image/color" "image/png" - "io" "io/ioutil" "math/rand" "path/filepath" @@ -23,9 +20,9 @@ import ( "github.com/decred/politeia/politeiad/api/v1/identity" "github.com/decred/politeia/politeiad/api/v1/mime" piplugin "github.com/decred/politeia/politeiad/plugins/pi" - "github.com/decred/politeia/politeiad/plugins/usermd" piv1 "github.com/decred/politeia/politeiawww/api/pi/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + pclient "github.com/decred/politeia/politeiawww/client" "github.com/decred/politeia/util" ) @@ -41,7 +38,7 @@ func printProposalFiles(files []rcv1.File) error { // Its possible for a proposal metadata to not exist if the // proposal has been censored. - pm, err := proposalMetadataDecode(files) + pm, err := pclient.ProposalMetadataDecode(files) if err != nil { return err } @@ -51,7 +48,7 @@ func printProposalFiles(files []rcv1.File) error { } // A vote metadata file is optional - vm, err := voteMetadataDecode(files) + vm, err := pclient.VoteMetadataDecode(files) if err != nil { return err } @@ -86,22 +83,6 @@ func printProposal(r rcv1.Record) error { return printProposalFiles(r.Files) } -func statusChangesDecode(payload []byte) ([]usermd.StatusChangeMetadata, error) { - statuses := make([]usermd.StatusChangeMetadata, 0, 16) - d := json.NewDecoder(strings.NewReader(string(payload))) - for { - var sc usermd.StatusChangeMetadata - err := d.Decode(&sc) - if errors.Is(err, io.EOF) { - break - } else if err != nil { - return nil, err - } - statuses = append(statuses, sc) - } - return statuses, nil -} - // indexFileRandom returns a proposal index file filled with random data. func indexFileRandom(sizeInBytes int) (*rcv1.File, error) { // Create lines of text that are 80 characters long @@ -240,47 +221,3 @@ func signedMerkleRoot(files []rcv1.File, fid *identity.FullIdentity) (string, er sig := fid.SignMessage([]byte(mr)) return hex.EncodeToString(sig[:]), nil } - -// proposalMetadataDecode decodes and returns the ProposalMetadata from the -// provided record files. nil is returned if a ProposalMetadata is not found. -func proposalMetadataDecode(files []rcv1.File) (*piv1.ProposalMetadata, error) { - var propMD *piv1.ProposalMetadata - for _, v := range files { - if v.Name == piv1.FileNameProposalMetadata { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return nil, err - } - var m piv1.ProposalMetadata - err = json.Unmarshal(b, &m) - if err != nil { - return nil, err - } - propMD = &m - break - } - } - return propMD, nil -} - -// voteMetadataDecode decodes and returns the VoteMetadata from the provided -// backend files. If a VoteMetadata is not found, nil will be returned. -func voteMetadataDecode(files []rcv1.File) (*piv1.VoteMetadata, error) { - var voteMD *piv1.VoteMetadata - for _, v := range files { - if v.Name == piv1.FileNameVoteMetadata { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return nil, err - } - var vm piv1.VoteMetadata - err = json.Unmarshal(b, &vm) - if err != nil { - return nil, err - } - voteMD = &vm - break - } - } - return voteMD, nil -} diff --git a/politeiawww/cmd/politeiaverify/README.md b/politeiawww/cmd/politeiaverify/README.md index 45a2682e0..2201141f2 100644 --- a/politeiawww/cmd/politeiaverify/README.md +++ b/politeiawww/cmd/politeiaverify/README.md @@ -1,51 +1,64 @@ -# Politeia Verify +# politeiaverify -`politeiaverify` is a simple tool that allows anyone to independently verify -that Politeia has received your proposal/comment and that it is sound. The -input received in this command is the json bundle downloaded from the GUI. -Files from the gui are downloaded with filename `.json` for proposal -bundles and `-comments.json` for proposal comments bundle. If no flag -is passed in, the tool will try to read the filename and call the corresponding -verify method. +`politeiaverify` is a tool that allows anyone to independently verify the +validity of data submitted to Politeia. This includes: +- Verifying the censorship record of a record submission. A censorship record + provides cryptographic proof that a record was received by Politeia. +- Verifying the receipts for non-record data (ex. comments). The receipts + provide cryptographic proof the non-record data was received by Politeia. +- Verifying user signatures. Anytime a user submits data to Politeia they must + sign the data using a key pair that is specific to the user. The public key + and signature is saved along with the data, providing cryptographic proof + that the data was submitted by the user. +- Verifying timestamps. All data submitted to Politeia is timestamped onto + the Decred blockchain. A timestamp provides cryptographic proof that data + existed at block height x and has not been altered since then. ## Usage -`politeiaverify [flags] ` +``` +politeiaverify [flags] ...` -Flags: - `-proposal` - verify proposal bundle - `-comments` - verify comments bundle +Options: + -k Politiea's public server key + -t Record censorship token + -s Record censorship signature +``` -Examples: +## Verifying politeiagui bundles -To verify a proposal bundle +Any of the file bundles that are available for download in politeiagui can +be passed into `politeiaverify` directly. These files contain all the data +needed to verify the contents. ``` -politeiaverify -proposal c093b8a808ef68665709995a5a741bd02502b9c6c48a99a4b179fef742ca6b2a.json - -Proposal signature: - Public key: 49912d8dd296ce00a4b6afce4f300481ed5403142740e8b510276dccd1cbaccd - Signature : a0c1e9d887bd77ddf3b4fd650082ae8bc0f7c09631de7dce1b3147140d1163347768abbf2cecd0196ad5019c40dd2a9a16db482955f5cc30a1b79771ccffa90b -Proposal censorship record signature: - Merkle root: e905baa3391e446ab89270153f45581640e7cef6e162152fa6e469737699c6bd - Public key : a70134196c3cdf3f85f8af6abaa38c15feb7bccf5e6d3db6212358363465e502 - Signature : 0bf4c685102db88c06df12f0980f87bda5e285fcc37be4bef118b138bc5dcdfa3dceaa234eaff2e47f5f16224743d9cf7fe31e3244e67e50b1b7685910362e01 +$ politeiaverify 39868e5e91c78255-v2.json -Proposal successfully verified +Server key : e88df79a4b02699e6c051adbae05f21f2a2f24942e0f27cade165548ec3d6387 +Token : 39868e5e91c78255 +Merkle root: 2d00aaa0768701fd011943fbe8ae92f84ee268ca134d6b14f877c3153072bb3c +Signature : b2a69823f85b62941d845c439726a2504026a0d29fd50ecabe5648b0128328c2fade0ddb354594d48a209dff24e73795ec9cb175d028155cbfa1901114f4b608 +Record successfully verified ``` -To verify a proposal comments bundle +## Manual verification + +When verifying manually the user must provide the server public key (`-k`), +the censorship token (`-t`), the censorship record signature (`-s`), and the +file paths for all files that were part of the record. ``` -politeiaverify -comments c093b8a808ef68665709995a5a741bd02502b9c6c48a99a4b179fef742ca6b2a-comments.json +$ politeiaverify \ +-k e88df79a4b02699e6c051adbae05f21f2a2f24942e0f27cade165548ec3d6387 \ +-t 39868e5e91c78255 \ +-s b2a69823f85b62941d845c439726a2504026a0d29fd50ecabe5648b0128328c2fade0ddb354594d48a209dff24e73795ec9cb175d028155cbfa1901114f4b608 \ +index.md -Comment ID: 1 - Public key: a70134196c3cdf3f85f8af6abaa38c15feb7bccf5e6d3db6212358363465e502 - Receipt : fdf466b5511a0ad7304bdb45cb387b7c4ebe720c9d5c3271ec144fee92775de5e6f30482d42375eda99c3b704f37a7f70108d4f201f7c91d6024b9ec5aaa110b - Signature : c48d643784b5c3645afce7965e7d7d9b44978c22829da30174c51e22eda2849e87d6cd304f7fc2e5bdc5d17ab4b515bc89605b7a814355a44cdfa86d8dc4030e +Server key : e88df79a4b02699e6c051adbae05f21f2a2f24942e0f27cade165548ec3d6387 +Token : 39868e5e91c78255 +Merkle root: 2d00aaa0768701fd011943fbe8ae92f84ee268ca134d6b14f877c3153072bb3c +Signature : b2a69823f85b62941d845c439726a2504026a0d29fd50ecabe5648b0128328c2fade0ddb354594d48a209dff24e73795ec9cb175d028155cbfa1901114f4b608 -Comments successfully verified +Record successfully verified ``` - -If the bundle is in bad format or if it fails to verify, it will return an error. diff --git a/politeiawww/cmd/politeiaverify/politeiaverify.go b/politeiawww/cmd/politeiaverify/politeiaverify.go index 1aa0e6f81..c19b3e70f 100644 --- a/politeiawww/cmd/politeiaverify/politeiaverify.go +++ b/politeiawww/cmd/politeiaverify/politeiaverify.go @@ -1,141 +1,169 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + package main import ( - "encoding/json" + "encoding/hex" "flag" "fmt" - "io/ioutil" "os" - "path" - "strings" + "path/filepath" + "regexp" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" "github.com/decred/politeia/politeiawww/client" "github.com/decred/politeia/util" ) -// record is used to unmarshal the data that is contained in the record JSON -// bundle downloded from the GUI. -type record struct { - Record rcv1.Record `json:"record"` - ServerPublicKey string `json:"serverpublickey"` -} - -// comments is used to unmarshal the data that is contained in the comments -// JSON bundle downloded from the GUI. -type comments []struct { - CommentID string `json:"commentid"` - Receipt string `json:"receipt"` - Signature string `json:"signature"` - ServerPublicKey string `json:"serverpublickey"` -} - var ( - flagVerifyRecord = flag.Bool("record", false, "Verify record bundle") - flagVerifyComments = flag.Bool("comments", false, "Verify comments bundle") + // CLI flags + publicKey = flag.String("k", "", "server public key") + token = flag.String("t", "", "record censorship token") + signature = flag.String("s", "", "record censorship signature") + + // Regexp for matching politeiagui bundles + expJSONFile = `.json$` + expRecord = `^[0-9a-f]{16}-v[\d]{1,2}.json$` + expRecordTimestamps = `^[0-9a-f]{16}-v[\d]{1,2}-timestamps.json$` + + regexJSONFile = regexp.MustCompile(expJSONFile) + regexRecord = regexp.MustCompile(expRecord) + regexRecordTimestamps = regexp.MustCompile(expRecordTimestamps) ) -func usage() { - fmt.Fprintf(os.Stderr, "usage: politeiaverify [flags] \n") - fmt.Fprintf(os.Stderr, " flags:\n") - flag.PrintDefaults() - fmt.Fprintf(os.Stderr, " - Path to the JSON bundle "+ - "downloaded from the GUI\n") - fmt.Fprintf(os.Stderr, "\n") +// loadFiles loads and returns a politeiawww records v1 File for each provided +// file path. +func loadFiles(paths []string) ([]rcv1.File, error) { + files := make([]rcv1.File, 0, len(paths)) + for _, fp := range paths { + fp = util.CleanAndExpandPath(fp) + mime, digest, payload, err := util.LoadFile(fp) + if err != nil { + return nil, err + } + files = append(files, rcv1.File{ + Name: filepath.Base(fp), + MIME: mime, + Digest: digest, + Payload: payload, + }) + } + return files, nil } -func verifyRecord(payload []byte) error { - var r record - err := json.Unmarshal(payload, &r) +// verifyCensorshipRecord verifies a censorship record signature for a politeia +// record submission. This requires passing in the server public key, the +// censorship token, the censorship record signature, and the filepaths of all +// files that are part of the record. +func verifyCensorshipRecord(serverPubKey, token, signature string, filepaths []string) error { + // Verify all args are present + switch { + case serverPubKey == "": + return fmt.Errorf("server public key not provided") + case token == "": + return fmt.Errorf("censorship token not provided") + case signature == "": + return fmt.Errorf("censorship record signature not provided") + case len(filepaths) == 0: + return fmt.Errorf("record files not provided") + } + + // Load record files + files, err := loadFiles(filepaths) + if err != nil { + return err + } + + // Calc merkle root of files + digests := make([]string, 0, len(files)) + for _, v := range files { + digests = append(digests, v.Digest) + } + mr, err := util.MerkleRoot(digests) + if err != nil { + return err + } + merkle := hex.EncodeToString(mr[:]) + + // Load identity + pid, err := util.IdentityFromString(serverPubKey) if err != nil { - return fmt.Errorf("Record bundle JSON in bad format, make sure to " + - "download it from the GUI.") + return err } - err = client.RecordVerify(r.Record, r.ServerPublicKey) + // Verify record + r := rcv1.Record{ + Files: files, + CensorshipRecord: rcv1.CensorshipRecord{ + Token: token, + Merkle: merkle, + Signature: signature, + }, + } + err = client.RecordVerify(r, pid.String()) if err != nil { - return fmt.Errorf("Failed to verify record: %v", err) + return err } - fmt.Println("Censorship record:") - fmt.Printf(" Token : %s\n", r.Record.CensorshipRecord.Token) - fmt.Printf(" Merkle root: %s\n", r.Record.CensorshipRecord.Merkle) - fmt.Printf(" Public key : %s\n", r.ServerPublicKey) - fmt.Printf(" Signature : %s\n\n", r.Record.CensorshipRecord.Signature) + fmt.Printf("Server key : %s\n", serverPubKey) + fmt.Printf("Token : %s\n", token) + fmt.Printf("Merkle root: %s\n", merkle) + fmt.Printf("Signature : %s\n\n", signature) fmt.Println("Record successfully verified") return nil } -func verifyComments(payload []byte) error { - var comments comments - err := json.Unmarshal(payload, &comments) - if err != nil { - return fmt.Errorf("Comments bundle JSON in bad format, make sure to " + - "download it from the GUI.") - } +// verifyFile verifies a data file downloaded from politeiagui. This can be +// one of the data bundles or one of the timestamp files. The file name MUST +// be the same file name that was downloaded from politeiagui. +func verifyFile(fp string) error { + fp = util.CleanAndExpandPath(fp) + filename := filepath.Base(fp) - for _, c := range comments { - // Verify receipt - id, err := util.IdentityFromString(c.ServerPublicKey) - if err != nil { - return err - } - receipt, err := util.ConvertSignature(c.Receipt) - if err != nil { - return err - } - if !id.VerifyMessage([]byte(c.Signature), receipt) { - return fmt.Errorf("Could not verify receipt %v of comment id %v", - c.Receipt, c.CommentID) - } - fmt.Printf("Comment ID: %s\n", c.CommentID) - fmt.Printf(" Public key: %s\n", c.ServerPublicKey) - fmt.Printf(" Receipt : %s\n", c.Receipt) - fmt.Printf(" Signature : %s\n", c.Signature) + // Match file type + switch { + case regexRecord.FindString(filename) != "": + return verifyRecordBundleFile(fp) + case regexRecordTimestamps.FindString(filename) != "": + return verifyRecordTimestampsFile(fp) } - fmt.Println("\nComments successfully verified") - - return nil + return fmt.Errorf("file not recognized") } func _main() error { + // Parse CLI arguments flag.Parse() args := flag.Args() + if len(args) == 0 { + return fmt.Errorf("no arguments provided") + } - // Validate flags and arguments - switch { - case len(args) != 1: - usage() - return fmt.Errorf("Must provide json bundle path as input") - case *flagVerifyRecord && *flagVerifyComments: - usage() - return fmt.Errorf("Must choose only one verification type") + // Check if the user is trying to verify a record submission + // manually. This requires passing in the server public key, the + // censorship token, the censorship record signature, and all of + // the record filepaths. + manual := (*publicKey != "") || (*token != "") || (*signature != "") + if manual { + // The user is trying to verify manually + return verifyCensorshipRecord(*publicKey, *token, *signature, args) } - // Read bundle payload - file := args[0] - var payload []byte - payload, err := ioutil.ReadFile(file) + // The user is trying to verify a bundle file that was downloaded + // from politeiagui. + fp := args[0] + if regexJSONFile.FindString(fp) == "" { + return fmt.Errorf("'%v' is not a json file", fp) + } + err := verifyFile(fp) if err != nil { return err } - // Call verify method - switch { - case *flagVerifyRecord: - return verifyRecord(payload) - case *flagVerifyComments: - return verifyComments(payload) - default: - // No flags used, read filename and try to call corresponding - // verify method - if strings.Contains(path.Base(file), "comments") { - return verifyComments(payload) - } - return verifyRecord(payload) - } + return nil } func main() { diff --git a/politeiawww/cmd/politeiaverify/record.go b/politeiawww/cmd/politeiaverify/record.go new file mode 100644 index 000000000..2c87039d4 --- /dev/null +++ b/politeiawww/cmd/politeiaverify/record.go @@ -0,0 +1,100 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + + rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + "github.com/decred/politeia/politeiawww/client" +) + +// recordBundle represents the record bundle that is available for download +// in politeiagui. +type recordBundle struct { + Record rcv1.Record `json:"record"` + ServerPublicKey string `json:"serverpublickey"` +} + +// verifyRecordBundle verifies that the validity of a record bundle. This +// includes verifying the censorship record and the user metadata signatures. +func verifyRecordBundle(rb recordBundle) error { + // Verify censorship record + err := client.RecordVerify(rb.Record, rb.ServerPublicKey) + if err != nil { + return fmt.Errorf("could not verify record: %v", err) + } + + fmt.Printf("Server public key: %v\n", rb.ServerPublicKey) + fmt.Printf("Censorship record:\n") + fmt.Printf(" Token : %v\n", rb.Record.CensorshipRecord.Token) + fmt.Printf(" Merkle root: %v\n", rb.Record.CensorshipRecord.Merkle) + fmt.Printf(" Signature : %v\n", rb.Record.CensorshipRecord.Signature) + fmt.Printf("\n") + fmt.Printf("Censorship record verified\n") + + // Verify user metadata + err = client.UserMetadataVerify(rb.Record.Metadata, rb.Record.Files) + if err != nil { + return err + } + + fmt.Printf("User signature verified\n") + + // Verify status change signatures + err = client.StatusChangesVerify(rb.Record.Metadata) + if err != nil { + return err + } + + fmt.Printf("Status change signatures verified\n") + + return nil +} + +// verifyRecordBundleFile takes the filepath of a record bundle and verifies +// the contents of the file. +func verifyRecordBundleFile(fp string) error { + // Decode record bundle + b, err := ioutil.ReadFile(fp) + if err != nil { + return err + } + var rb recordBundle + err = json.Unmarshal(b, &rb) + if err != nil { + return fmt.Errorf("could not unmarshal record bundle: %v", err) + } + + // Verify record bundle + return verifyRecordBundle(rb) +} + +// verifyRecordTimestampsFile takes the filepath of record timestamps and +// verifies the contents of the file. +func verifyRecordTimestampsFile(fp string) error { + // Decode timestamps reply + b, err := ioutil.ReadFile(fp) + if err != nil { + return err + } + var tr rcv1.TimestampsReply + err = json.Unmarshal(b, &tr) + if err != nil { + return fmt.Errorf("could not unmarshal record timestamps: %v", err) + } + + // Verify timestamps + err = client.RecordTimestampsVerify(tr) + if err != nil { + return err + } + + fmt.Printf("Record timestamps verified\n") + + return nil +} diff --git a/politeiawww/cmd/politeiaverify/ticketvote.go b/politeiawww/cmd/politeiaverify/ticketvote.go new file mode 100644 index 000000000..eb917236e --- /dev/null +++ b/politeiawww/cmd/politeiaverify/ticketvote.go @@ -0,0 +1,18 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package main + +import ( + tkv1 "github.com/decred/politeia/politeiad/plugins/ticketvote" +) + +// voteBundle represents the bundle that is downloaded from politeiagui for +// DCR ticket votes. +type voteBundle struct { + Auths []tkv1.AuthDetails `json:"auths,omitempty"` + Details *tkv1.VoteDetails `json:"details,omitempty"` + Votes []tkv1.CastVoteDetails `json:"votes,omitempty"` + ServerPublicKey string `json:"serverpublickey"` +} diff --git a/politeiawww/pi/events.go b/politeiawww/pi/events.go index 49e7da9f2..9f29a53c8 100644 --- a/politeiawww/pi/events.go +++ b/politeiawww/pi/events.go @@ -6,22 +6,18 @@ package pi import ( "context" - "encoding/base64" - "encoding/json" - "errors" "fmt" - "io" - "strings" pdv2 "github.com/decred/politeia/politeiad/api/v2" cmplugin "github.com/decred/politeia/politeiad/plugins/comments" piplugin "github.com/decred/politeia/politeiad/plugins/pi" tkplugin "github.com/decred/politeia/politeiad/plugins/ticketvote" - "github.com/decred/politeia/politeiad/plugins/usermd" cmv1 "github.com/decred/politeia/politeiawww/api/comments/v1" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" + v1 "github.com/decred/politeia/politeiawww/api/records/v1" tkv1 "github.com/decred/politeia/politeiawww/api/ticketvote/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" + "github.com/decred/politeia/politeiawww/client" "github.com/decred/politeia/politeiawww/comments" "github.com/decred/politeia/politeiawww/records" "github.com/decred/politeia/politeiawww/ticketvote" @@ -104,7 +100,7 @@ func (p *Pi) handleEventRecordNew(ch chan interface{}) { // Send notfication email var ( token = e.Record.CensorshipRecord.Token - name = proposalNameFromRecord(e.Record) + name = proposalNameFromFiles(e.Record.Files) ) err = p.mailNtfnProposalNew(token, name, e.User.Username, emails) if err != nil { @@ -160,7 +156,7 @@ func (p *Pi) handleEventRecordEdit(ch chan interface{}) { var ( token = e.Record.CensorshipRecord.Token version = e.Record.Version - name = proposalNameFromRecord(e.Record) + name = proposalNameFromFiles(e.Record.Files) username = e.User.Username ) err = p.mailNtfnProposalEdit(token, version, name, username, emails) @@ -178,12 +174,12 @@ func (p *Pi) ntfnRecordSetStatusToAuthor(r rcv1.Record) error { var ( token = r.CensorshipRecord.Token status = r.Status - name = proposalNameFromRecord(r) + name = proposalNameFromFiles(r.Files) authorID = userIDFromMetadata(r.Metadata) ) // Parse the status change reason - sc, err := statusChangesFromMetadata(r.Metadata) + sc, err := client.StatusChangesDecode(r.Metadata) if err != nil { return fmt.Errorf("decode status changes: %v", err) } @@ -227,7 +223,7 @@ func (p *Pi) ntfnRecordSetStatus(r rcv1.Record) error { var ( token = r.CensorshipRecord.Token status = r.Status - name = proposalNameFromRecord(r) + name = proposalNameFromFiles(r.Files) authorID = userIDFromMetadata(r.Metadata) ) @@ -430,7 +426,7 @@ func (p *Pi) handleEventCommentNew(ch chan interface{}) { } r = convertRecordToV1(*pdr) proposalAuthorID = userIDFromMetadata(r.Metadata) - proposalName = proposalNameFromRecord(r) + proposalName = proposalNameFromFiles(r.Files) // Notify the proposal author err = p.ntfnCommentNewProposalAuthor(e.Comment, @@ -488,7 +484,7 @@ func (p *Pi) handleEventVoteAuthorized(ch chan interface{}) { goto failed } r = convertRecordToV1(*pdr) - proposalName = proposalNameFromRecord(r) + proposalName = proposalNameFromFiles(r.Files) // Compile notification email list err = p.userdb.AllUsers(func(u *user.User) { @@ -625,7 +621,7 @@ func (p *Pi) handleEventVoteStarted(ch chan interface{}) { } r = convertRecordToV1(*pdr) authorID = userIDFromMetadata(r.Metadata) - proposalName = proposalNameFromRecord(r) + proposalName = proposalNameFromFiles(r.Files) // Send notification to record author err = p.ntfnVoteStartedToAuthor(v, authorID, proposalName) @@ -675,28 +671,20 @@ func (p *Pi) recordAbridged(token string) (*pdv2.Record, error) { return &r, nil } -// userMetadataDecode decodes and returns the UserMetadata from the provided -// metadata streams. If a UserMetadata is not found, nil is returned. -func userMetadataDecode(ms []rcv1.MetadataStream) (*usermd.UserMetadata, error) { - var userMD *usermd.UserMetadata - for _, v := range ms { - if v.StreamID == usermd.StreamIDUserMetadata { - var um usermd.UserMetadata - err := json.Unmarshal([]byte(v.Payload), &um) - if err != nil { - return nil, err - } - userMD = &um - break - } +// proposalNameFromFiles parses the proposal name from the ProposalMetadata file and +// returns it. An empty string is returned if a proposal name is not found. +func proposalNameFromFiles(files []rcv1.File) string { + pm, err := client.ProposalMetadataDecode(files) + if err != nil { + return "" } - return userMD, nil + return pm.Name } // userIDFromMetadata searches for a UserMetadata and parses the user ID from // it if found. An empty string is returned if no UserMetadata is found. -func userIDFromMetadata(ms []rcv1.MetadataStream) string { - um, err := userMetadataDecode(ms) +func userIDFromMetadata(ms []v1.MetadataStream) string { + um, err := client.UserMetadataDecode(ms) if err != nil { return "" } @@ -706,60 +694,6 @@ func userIDFromMetadata(ms []rcv1.MetadataStream) string { return um.UserID } -// proposalNameFromRecord parses the proposal name from the ProposalMetadata -// file and returns it. An empty string will be returned if any errors occur or -// if a name is not found. -func proposalNameFromRecord(r rcv1.Record) string { - var name string - for _, v := range r.Files { - if v.Name == piplugin.FileNameProposalMetadata { - b, err := base64.StdEncoding.DecodeString(v.Payload) - if err != nil { - return "" - } - var pm piplugin.ProposalMetadata - err = json.Unmarshal(b, &pm) - if err != nil { - return "" - } - name = pm.Name - } - } - return name -} - -func statusChangesDecode(payload []byte) ([]usermd.StatusChangeMetadata, error) { - statuses := make([]usermd.StatusChangeMetadata, 0, 16) - d := json.NewDecoder(strings.NewReader(string(payload))) - for { - var sc usermd.StatusChangeMetadata - err := d.Decode(&sc) - if errors.Is(err, io.EOF) { - break - } else if err != nil { - return nil, err - } - statuses = append(statuses, sc) - } - return statuses, nil -} - -func statusChangesFromMetadata(metadata []rcv1.MetadataStream) ([]usermd.StatusChangeMetadata, error) { - var ( - sc []usermd.StatusChangeMetadata - err error - ) - for _, v := range metadata { - if v.StreamID == usermd.StreamIDStatusChanges { - sc, err = statusChangesDecode([]byte(v.Payload)) - if err != nil { - return nil, err - } - } - } - return sc, nil -} - func convertStateToV1(s pdv2.RecordStateT) rcv1.RecordStateT { switch s { case pdv2.RecordStateUnvetted: diff --git a/politeiawww/proposals.go b/politeiawww/proposals.go index 0333643c2..211b8530d 100644 --- a/politeiawww/proposals.go +++ b/politeiawww/proposals.go @@ -20,6 +20,7 @@ import ( piplugin "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/ticketvote" tkplugin "github.com/decred/politeia/politeiad/plugins/ticketvote" + "github.com/decred/politeia/politeiad/plugins/usermd" umplugin "github.com/decred/politeia/politeiad/plugins/usermd" rcv1 "github.com/decred/politeia/politeiawww/api/records/v1" www "github.com/decred/politeia/politeiawww/api/www/v1" @@ -803,7 +804,9 @@ func (p *politeiawww) handleVoteResults(w http.ResponseWriter, r *http.Request) func userMetadataDecode(ms []pdv2.MetadataStream) (*umplugin.UserMetadata, error) { var userMD *umplugin.UserMetadata for _, v := range ms { - if v.StreamID != umplugin.StreamIDUserMetadata { + if v.PluginID != usermd.PluginID || + v.StreamID != umplugin.StreamIDUserMetadata { + // This is not user metadata continue } var um umplugin.UserMetadata diff --git a/politeiawww/records/process.go b/politeiawww/records/process.go index b52a2d84b..208ca4af8 100644 --- a/politeiawww/records/process.go +++ b/politeiawww/records/process.go @@ -14,6 +14,7 @@ import ( pdv2 "github.com/decred/politeia/politeiad/api/v2" "github.com/decred/politeia/politeiad/plugins/usermd" v1 "github.com/decred/politeia/politeiawww/api/records/v1" + "github.com/decred/politeia/politeiawww/client" "github.com/decred/politeia/politeiawww/config" "github.com/decred/politeia/politeiawww/user" "github.com/google/uuid" @@ -544,31 +545,10 @@ func recordPopulateUserData(r *v1.Record, u user.User) { r.Username = u.Username } -// userMetadataDecode decodes and returns the UserMetadata from the provided -// metadata streams. If a UserMetadata is not found, nil is returned. -func userMetadataDecode(ms []v1.MetadataStream) (*usermd.UserMetadata, error) { - var userMD *usermd.UserMetadata - for _, v := range ms { - if v.PluginID != usermd.PluginID || - v.StreamID != usermd.StreamIDUserMetadata { - // Not the mdstream we're looking for - continue - } - var um usermd.UserMetadata - err := json.Unmarshal([]byte(v.Payload), &um) - if err != nil { - return nil, err - } - userMD = &um - break - } - return userMD, nil -} - // userIDFromMetadataStreams searches for a UserMetadata and parses the user ID // from it if found. An empty string is returned if no UserMetadata is found. func userIDFromMetadataStreams(ms []v1.MetadataStream) string { - um, err := userMetadataDecode(ms) + um, err := client.UserMetadataDecode(ms) if err != nil { return "" } From 4579f572d24b64bc3962a2c85ad954043b01c9c1 Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 26 Mar 2021 08:56:31 -0500 Subject: [PATCH 439/449] Bug fixes. --- politeiawww/cmd/politeiaverify/record.go | 5 +++-- politeiawww/pi.go | 9 +++++++-- politeiawww/www.go | 8 +------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/politeiawww/cmd/politeiaverify/record.go b/politeiawww/cmd/politeiaverify/record.go index 2c87039d4..772e80b1a 100644 --- a/politeiawww/cmd/politeiaverify/record.go +++ b/politeiawww/cmd/politeiaverify/record.go @@ -20,8 +20,9 @@ type recordBundle struct { ServerPublicKey string `json:"serverpublickey"` } -// verifyRecordBundle verifies that the validity of a record bundle. This -// includes verifying the censorship record and the user metadata signatures. +// verifyRecordBundle verifies that a record bundle has been accepted by +// politeia and that all user signatures are correct. A record bundle is the +// JSON data file that is downloaded from politeiagui for a record. func verifyRecordBundle(rb recordBundle) error { // Verify censorship record err := client.RecordVerify(rb.Record, rb.ServerPublicKey) diff --git a/politeiawww/pi.go b/politeiawww/pi.go index 3e26e1ef1..07f66c400 100644 --- a/politeiawww/pi.go +++ b/politeiawww/pi.go @@ -8,7 +8,6 @@ import ( "fmt" "net/http" - pdv2 "github.com/decred/politeia/politeiad/api/v2" cmplugin "github.com/decred/politeia/politeiad/plugins/comments" piplugin "github.com/decred/politeia/politeiad/plugins/pi" tkplugin "github.com/decred/politeia/politeiad/plugins/ticketvote" @@ -166,7 +165,13 @@ func (p *politeiawww) setupPiRoutes(r *records.Records, c *comments.Comments, t permissionPublic) } -func (p *politeiawww) setupPi(plugins []pdv2.Plugin) error { +func (p *politeiawww) setupPi() error { + // Get politeiad plugins + plugins, err := p.getPluginInventory() + if err != nil { + return fmt.Errorf("getPluginInventory: %v", err) + } + // Verify all required politeiad plugins have been registered required := map[string]bool{ piplugin.PluginID: false, diff --git a/politeiawww/www.go b/politeiawww/www.go index b09a38a1a..50f8be476 100644 --- a/politeiawww/www.go +++ b/politeiawww/www.go @@ -677,12 +677,6 @@ func _main() error { userEmails: make(map[string]uuid.UUID), } - // Setup politeiad plugins - plugins, err := p.getPluginInventory() - if err != nil { - return fmt.Errorf("getPluginInventory: %v", err) - } - // Setup email-userID cache err = p.initUserEmailsCache() if err != nil { @@ -692,7 +686,7 @@ func _main() error { // Perform application specific setup switch p.cfg.Mode { case config.PoliteiaWWWMode: - err = p.setupPi(plugins) + err = p.setupPi() if err != nil { return fmt.Errorf("setupPi: %v", err) } From 10426f7ff3b6e6fb3ac9c89f927333e3e4d0a446 Mon Sep 17 00:00:00 2001 From: lukebp Date: Sat, 27 Mar 2021 09:07:32 -0500 Subject: [PATCH 440/449] politeiad: Derive tlog key from passphrase. --- .../backendv2/tstorebe/store/mysql/encrypt.go | 117 +++++++++--------- .../tstorebe/store/mysql/encrypt_test.go | 7 +- .../backendv2/tstorebe/store/mysql/mysql.go | 4 +- .../backendv2/tstorebe/store/mysql/nonce.go | 1 - .../backendv2/tstorebe/tstore/tlogclient.go | 116 ++++++++++++++--- .../tstorebe/tstore/tlogclient_test.go | 54 ++++++++ politeiad/backendv2/tstorebe/tstore/tstore.go | 33 ++--- politeiad/backendv2/tstorebe/tstorebe.go | 6 +- politeiad/config.go | 69 +++++++---- politeiad/politeiad.go | 5 +- util/argon2.go | 30 +++++ 11 files changed, 306 insertions(+), 136 deletions(-) create mode 100644 politeiad/backendv2/tstorebe/tstore/tlogclient_test.go create mode 100644 util/argon2.go diff --git a/politeiad/backendv2/tstorebe/store/mysql/encrypt.go b/politeiad/backendv2/tstorebe/store/mysql/encrypt.go index 6c8088c1d..a83688843 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/encrypt.go +++ b/politeiad/backendv2/tstorebe/store/mysql/encrypt.go @@ -18,97 +18,96 @@ import ( ) const ( - // argon2idKey is the kv store key for the argon2idParams structure - // that is saved on initial key derivation. - argon2idKey = "argon2id" + // encryptionKeyParamsKey is the kv store key for the encryption + // key params that are saved on initial key derivation. + encryptionKeyParamsKey = "store-mysql-encryptionkeyparams" ) -// argon2idParams is saved to the kv store the first time the key is derived. -type argon2idParams struct { - Time uint32 `json:"time"` - Memory uint32 `json:"memory"` - Threads uint8 `json:"threads"` - KeyLen uint32 `json:"keylen"` - Salt []byte `json:"salt"` +// encryptionKeyParams is saved to the kv store on initial derivation of the +// encryption key. It contains the params that were used to derive the key and +// a SHA256 digest of the key. Subsequent derivations will use the existing +// params to derive the key and will use the digest to verify that the +// encryption key has not changed. +type encryptionKeyParams struct { + Digest []byte `json:"digest"` // SHA256 digest + Params util.Argon2Params `json:"params"` } -func newArgon2Params() argon2idParams { - salt, err := util.Random(16) - if err != nil { - panic(err) - } - return argon2idParams{ - Time: 1, - Memory: 64 * 1024, // In KiB - Threads: 4, - KeyLen: 32, - Salt: salt, - } +// argon2idKey derives an encryption key using the provided parameters and the +// Argon2id key derivation function. The derived key is set to be the +// encryption key on the mysql context. +func (s *mysql) argon2idKey(password string, ap util.Argon2Params) { + k := argon2.IDKey([]byte(password), ap.Salt, ap.Time, ap.Memory, + ap.Threads, ap.KeyLen) + copy(s.key[:], k) + util.Zero(k) } -// argon2idKey derives a 32 byte key from the provided password using the +// deriveEncryption derives a 32 byte key from the provided password using the // Aragon2id key derivation function. A random 16 byte salt is created the // first time the key is derived. The salt and the other argon2id params are // saved to the kv store. Subsequent calls to this fuction will pull the -// existing salt and params from the kv store and use them to derive the key. -func (s *mysql) argon2idKey(password string) error { - log.Infof("Deriving encryption key from password") - - // Check if a key already exists. If db is nil then we are running unit - // tests. - var ( - blobs map[string][]byte - err error - ) - if s.db == nil { - // Testing mode - blobs = make(map[string][]byte) - } else { - blobs, err = s.Get([]string{argon2idKey}) - if err != nil { - return fmt.Errorf("get: %v", err) - } +// existing salt and params from the kv store and use them to derive the key, +// then will use the saved encryption key digest to verify that the key has +// not changed. +func (s *mysql) deriveEncryptionKey(password string) error { + log.Infof("Deriving encryption key") + + // Check if the key params already exist in the kv store. Existing + // params means that the key has been derived previously. These + // params will be used if found. If no params exist then new ones + // will be created and saved to the kv store for future use. + blobs, err := s.Get([]string{encryptionKeyParamsKey}) + if err != nil { + return fmt.Errorf("get: %v", err) } - var ( save bool - ap argon2idParams + ekp encryptionKeyParams ) - b, ok := blobs[argon2idKey] + b, ok := blobs[encryptionKeyParamsKey] if ok { - log.Debugf("Encryption key salt already exists") - err = json.Unmarshal(b, &ap) + log.Debugf("Encryption key params found in kv store") + err = json.Unmarshal(b, &ekp) if err != nil { return err } } else { - log.Infof("Encryption key not found; creating a new one") - ap = newArgon2Params() + log.Infof("Encryption key params not found; creating new ones") + ekp = encryptionKeyParams{ + Params: util.NewArgon2Params(), + } save = true } // Derive key - k := argon2.IDKey([]byte(password), ap.Salt, ap.Time, ap.Memory, - ap.Threads, ap.KeyLen) - copy(s.key[:], k) - util.Zero(k) - - // Save params to the kv store if this is the first time the key - // was derived. - if save && s.db != nil { - b, err := json.Marshal(ap) + s.argon2idKey(password, ekp.Params) + + // Check if the params need to be saved + keyDigest := util.Digest(s.key[:]) + if save { + // This was the first time the key was derived. Save the params + // to the kv store. + ekp.Digest = keyDigest + b, err := json.Marshal(ekp) if err != nil { return err } kv := map[string][]byte{ - argon2idKey: b, + encryptionKeyParamsKey: b, } err = s.Put(kv, false) if err != nil { return fmt.Errorf("put: %v", err) } - log.Infof("Encryption key derivation params saved to kv store") + log.Infof("Encryption key params saved to kv store") + } else { + // This was not the first time the key was derived. Verify that + // the key has not changed. + if !bytes.Equal(ekp.Digest, keyDigest) { + return fmt.Errorf("attempting to use different encryption key") + } } return nil diff --git a/politeiad/backendv2/tstorebe/store/mysql/encrypt_test.go b/politeiad/backendv2/tstorebe/store/mysql/encrypt_test.go index 86019531a..a198d2f13 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/encrypt_test.go +++ b/politeiad/backendv2/tstorebe/store/mysql/encrypt_test.go @@ -3,6 +3,8 @@ package mysql import ( "bytes" "testing" + + "github.com/decred/politeia/util" ) func TestEncryptDecrypt(t *testing.T) { @@ -12,10 +14,7 @@ func TestEncryptDecrypt(t *testing.T) { // setup fake context s := &mysql{} s.getNonce = s.getTestNonce - err := s.argon2idKey(password) - if err != nil { - t.Fatal(err) - } + s.argon2idKey(password, util.NewArgon2Params()) // Encrypt and make sure cleartext isn't the same as the encypted blob. eb, err := s.encrypt(nil, nil, blob) diff --git a/politeiad/backendv2/tstorebe/store/mysql/mysql.go b/politeiad/backendv2/tstorebe/store/mysql/mysql.go index fff36fce2..0256d7970 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/mysql.go +++ b/politeiad/backendv2/tstorebe/store/mysql/mysql.go @@ -318,9 +318,9 @@ func New(appDir, host, user, password, dbname string) (*mysql, error) { s.getNonce = s.getDbNonce // Derive encryption key from password. Key is set in argon2idKey - err = s.argon2idKey(password) + err = s.deriveEncryptionKey(password) if err != nil { - return nil, fmt.Errorf("argon2idKey: %v", err) + return nil, fmt.Errorf("deriveEncryptionKey: %v", err) } return s, nil diff --git a/politeiad/backendv2/tstorebe/store/mysql/nonce.go b/politeiad/backendv2/tstorebe/store/mysql/nonce.go index cb06547b7..e27ddb276 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/nonce.go +++ b/politeiad/backendv2/tstorebe/store/mysql/nonce.go @@ -136,5 +136,4 @@ func (s *mysql) testNonceIsUnique() { wg.Wait() log.Infof("Nonce concurrency test complete") - } diff --git a/politeiad/backendv2/tstorebe/tstore/tlogclient.go b/politeiad/backendv2/tstorebe/tstore/tlogclient.go index f8b22ab06..c510082f6 100644 --- a/politeiad/backendv2/tstorebe/tstore/tlogclient.go +++ b/politeiad/backendv2/tstorebe/tstore/tlogclient.go @@ -8,13 +8,15 @@ import ( "bytes" "context" "crypto" + "crypto/ed25519" + "encoding/json" "fmt" - "io/ioutil" "math/rand" "sync" "testing" "time" + "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" "github.com/decred/politeia/util" "github.com/golang/protobuf/ptypes" "github.com/google/trillian" @@ -27,6 +29,7 @@ import ( "github.com/google/trillian/merkle/hashers/registry" "github.com/google/trillian/merkle/rfc6962" "github.com/google/trillian/types" + "golang.org/x/crypto/argon2" rstatus "google.golang.org/genproto/googleapis/rpc/status" "google.golang.org/genproto/protobuf/field_mask" "google.golang.org/grpc" @@ -99,7 +102,8 @@ var ( _ tlogClient = (*tclient)(nil) ) -// tclient implements the tlogClient interface. +// tclient implements the tlogClient interface using the trillian provided +// TrillianLogClient and TrillianAdminClient. type tclient struct { host string grpc *grpc.ClientConn @@ -517,27 +521,106 @@ func newTrillianKey() (crypto.Signer, error) { }) } -// newTClient returns a new tclient. -func newTClient(host, keyFile string) (*tclient, error) { - // Setup trillian key file - if !util.FileExists(keyFile) { - // Trillian key file does not exist. Create one. - log.Infof("Generating trillian private key") - key, err := newTrillianKey() +// tlogKeyParams is saved to the kv store on initial derivation of the tlog +// private key. It contains the params that were used to derive the key and a +// SHA256 digest of the key. Subsequent derivations, i.e. anytime politeiad is +// restarted, will use the existing params to derive the key and will use the +// digest to verify that the tlog key has not changed. +type tlogKeyParams struct { + Digest []byte `json:"digest"` // SHA256 digest + Params util.Argon2Params `json:"params"` +} + +const ( + // tlogKeyParamsKey is the kv store key for the tlogKeyParams + // structure that is saved to the kv store on initial tlog key + // derivation. + tlogKeyParamsKey = "tlogkeyparams" +) + +// deriveTlogKey derives a ed25519 tlog private signing key using the provided +// passphrase and the Aragon2id key derivation function. A random 16 byte salt +// is created the first time the key is derived. The salt and the other argon2 +// params are saved to the kv store. Subsequent calls to this fuction will pull +// the existing salt and params from the kv store and use them to derive the +// key, then will use the saved private key digest to verify that the key has +// not changed. +func deriveTlogKey(kvstore store.BlobKV, passphrase string) (*keyspb.PrivateKey, error) { + log.Infof("Deriving tlog signing key") + + // Check if argon2 params already exist in the kv store for the + // tlog key. Existing params means that the key has been derived + // previously. These params will be used if found. If no params + // exist then new ones will be created and saved to the kv store + // for future use. + blobs, err := kvstore.Get([]string{tlogKeyParamsKey}) + if err != nil { + return nil, fmt.Errorf("get: %v", err) + } + var ( + save bool + tkp tlogKeyParams + ) + b, ok := blobs[tlogKeyParamsKey] + if ok { + log.Debugf("Tlog private key params found in kv store") + err = json.Unmarshal(b, &tkp) if err != nil { return nil, err } - b, err := der.MarshalPrivateKey(key) + } else { + log.Infof("Tlog private key params not found; creating new ones") + tkp = tlogKeyParams{ + Params: util.NewArgon2Params(), + } + save = true + } + + // Derive key + seed := argon2.IDKey([]byte(passphrase), tkp.Params.Salt, + tkp.Params.Time, tkp.Params.Memory, tkp.Params.Threads, + tkp.Params.KeyLen) + pk := ed25519.NewKeyFromSeed(seed) + util.Zero(seed) + + derKey, err := der.MarshalPrivateKey(pk) + if err != nil { + return nil, err + } + + keyDigest := util.Digest(derKey) + if save { + // This was the first time the key was derived. Save the params + // to the kv store. + tkp.Digest = keyDigest + b, err := json.Marshal(tkp) if err != nil { return nil, err } - err = ioutil.WriteFile(keyFile, b, 0400) + kv := map[string][]byte{ + tlogKeyParamsKey: b, + } + err = kvstore.Put(kv, false) if err != nil { - return nil, err + return nil, fmt.Errorf("put: %v", err) + } + + log.Infof("Tlog private key params saved to kv store") + } else { + // This was not the first time the key was derived. Verify that + // the key has not changed. + if !bytes.Equal(tkp.Digest, keyDigest) { + return nil, fmt.Errorf("attempting to use different tlog signing key") } - log.Infof("Trillian private key created: %v", keyFile) } + return &keyspb.PrivateKey{ + Der: derKey, + }, nil +} + +// newTClient returns a new tclient. +func newTClient(host string, privateKey *keyspb.PrivateKey) (*tclient, error) { // Default gprc max message size is ~4MB (4194304 bytes). This is // not large enough for trees with tens of thousands of leaves. // Increase it to 20MB. @@ -549,12 +632,7 @@ func newTClient(host, keyFile string) (*tclient, error) { return nil, fmt.Errorf("grpc dial: %v", err) } - // Load trillian key pair - var privateKey = &keyspb.PrivateKey{} - privateKey.Der, err = ioutil.ReadFile(keyFile) - if err != nil { - return nil, err - } + // Setup signing key signer, err := der.UnmarshalPrivateKey(privateKey.Der) if err != nil { return nil, err diff --git a/politeiad/backendv2/tstorebe/tstore/tlogclient_test.go b/politeiad/backendv2/tstorebe/tstore/tlogclient_test.go new file mode 100644 index 000000000..b70aab637 --- /dev/null +++ b/politeiad/backendv2/tstorebe/tstore/tlogclient_test.go @@ -0,0 +1,54 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tstore + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/decred/politeia/politeiad/backendv2/tstorebe/store/localdb" +) + +func TestDeriveTlogKey(t *testing.T) { + // Setup a localdb kv store + appDir, err := ioutil.TempDir("", "tstore.test") + if err != nil { + t.Fatal(err) + } + defer func() { + err = os.RemoveAll(appDir) + if err != nil { + t.Fatal(err) + } + }() + storeDir := filepath.Join(appDir, "store") + kvstore, err := localdb.New(appDir, storeDir) + if err != nil { + t.Fatal(err) + } + + // Key derivation params should be created and saved to the kv + // store the first time the key is derived. + pass := "testpasshrase" + key1, err := deriveTlogKey(kvstore, pass) + if err != nil { + t.Fatal(err) + } + + // Subsequent calls should use the existing derivation params and + // return the same key. This function will error if the derived + // keys are not the same. + key2, err := deriveTlogKey(kvstore, pass) + if err != nil { + t.Fatal(err) + } + + // Sanity check + if key1.String() != key2.String() { + t.Fatalf("different key was returned without any errors") + } +} diff --git a/politeiad/backendv2/tstorebe/tstore/tstore.go b/politeiad/backendv2/tstorebe/tstore/tstore.go index 8461597b9..e8fe94650 100644 --- a/politeiad/backendv2/tstorebe/tstore/tstore.go +++ b/politeiad/backendv2/tstorebe/tstore/tstore.go @@ -33,9 +33,6 @@ const ( // MySQL settings dbUser = "politeiad" - - // Config option defaults - defaultTrillianSigningKeyFilename = "trillian.key" ) var ( @@ -94,25 +91,10 @@ func (t *Tstore) Close() { } // New returns a new tstore instance. -func New(appDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianSigningKeyFile, dbType, dbHost, dbPass, dcrtimeHost, dcrtimeCert string) (*Tstore, error) { - // Setup trillian client - if trillianSigningKeyFile == "" { - // No file path was given. Use the default path. - fn := fmt.Sprintf("%v", defaultTrillianSigningKeyFilename) - trillianSigningKeyFile = filepath.Join(appDir, fn) - } - - log.Infof("Trillian key: %v", trillianSigningKeyFile) - log.Infof("Trillian host: %v", trillianHost) - - tlogClient, err := newTClient(trillianHost, trillianSigningKeyFile) - if err != nil { - return nil, err - } - +func New(appDir, dataDir string, anp *chaincfg.Params, tlogHost, tlogPass, dbType, dbHost, dbPass, dcrtimeHost, dcrtimeCert string) (*Tstore, error) { // Setup datadir for this tstore instance dataDir = filepath.Join(dataDir) - err = os.MkdirAll(dataDir, 0700) + err := os.MkdirAll(dataDir, 0700) if err != nil { return nil, err } @@ -142,6 +124,17 @@ func New(appDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianSig return nil, fmt.Errorf("invalid db type: %v", dbType) } + // Setup trillian client + log.Infof("Tlog host: %v", tlogHost) + tlogKey, err := deriveTlogKey(kvstore, tlogPass) + if err != nil { + return nil, err + } + tlogClient, err := newTClient(tlogHost, tlogKey) + if err != nil { + return nil, err + } + // Verify dcrtime host _, err = url.Parse(dcrtimeHost) if err != nil { diff --git a/politeiad/backendv2/tstorebe/tstorebe.go b/politeiad/backendv2/tstorebe/tstorebe.go index 550b4536e..08d8dc208 100644 --- a/politeiad/backendv2/tstorebe/tstorebe.go +++ b/politeiad/backendv2/tstorebe/tstorebe.go @@ -1225,10 +1225,10 @@ func (t *tstoreBackend) setup() error { } // New returns a new tstoreBackend. -func New(appDir, dataDir string, anp *chaincfg.Params, trillianHost, trillianSigningKey, dbType, dbHost, dbPass, dcrtimeHost, dcrtimeCert string) (*tstoreBackend, error) { +func New(appDir, dataDir string, anp *chaincfg.Params, tlogHost, tlogPass, dbType, dbHost, dbPass, dcrtimeHost, dcrtimeCert string) (*tstoreBackend, error) { // Setup tstore instances - ts, err := tstore.New(appDir, dataDir, anp, trillianHost, - trillianSigningKey, dbType, dbHost, dbPass, dcrtimeHost, dcrtimeCert) + ts, err := tstore.New(appDir, dataDir, anp, tlogHost, + tlogPass, dbType, dbHost, dbPass, dcrtimeHost, dcrtimeCert) if err != nil { return nil, fmt.Errorf("new tstore: %v", err) } diff --git a/politeiad/config.go b/politeiad/config.go index 25b7953fe..1afdfdb39 100644 --- a/politeiad/config.go +++ b/politeiad/config.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "net" + "net/url" "os" "path/filepath" "runtime" @@ -45,9 +46,14 @@ const ( defaultBackend = backendTstore // Tstore default settings - defaultTrillianHost = "localhost:8090" - defaultDBType = tstore.DBTypeLevelDB - defaultDBHost = "localhost:3306" // MySQL default host + defaultDBType = tstore.DBTypeLevelDB + defaultDBHost = "localhost:3306" // MySQL default host + defaultTlogHost = "localhost:8090" + + // Environment variables + envDcrtimeCert = "DCRTIMECERT" + envDBPass = "DBPASS" + envTlogPass = "TLOGPASS" ) var ( @@ -86,7 +92,7 @@ type config struct { RPCUser string `long:"rpcuser" description:"RPC user name for privileged commands"` RPCPass string `long:"rpcpass" description:"RPC password for privileged commands"` DcrtimeHost string `long:"dcrtimehost" description:"Dcrtime ip:port"` - DcrtimeCert string `long:"dcrtimecert" description:"File containing the https certificate file for dcrtimehost"` + DcrtimeCert string // Provided in env variable "DCRTIMECERT" Identity string `long:"identity" description:"File containing the politeiad identity file"` Backend string `long:"backend" description:"Backend type"` @@ -95,13 +101,13 @@ type config struct { DcrdataHost string `long:"dcrdatahost" description:"Dcrdata ip:port"` // Tstore backend options - TrillianHost string `long:"trillianhost" description:"Trillian ip:port"` - TrillianSigningKey string `long:"trilliansigningkey" description:"Trillian signing key"` - DBType string `long:"dbtype" description:"Database type"` - DBHost string `long:"dbhost" description:"Database ip:port"` - DBPass string // Provided in env variable "DBPASS" + DBType string `long:"dbtype" description:"Database type"` + DBHost string `long:"dbhost" description:"Database ip:port"` + DBPass string // Provided in env variable "DBPASS" + TlogHost string `long:"tloghost" description:"Trillian log ip:port"` + TlogPass string // Provided in env variable "TLOGPASS" - // Plugin settings + // Plugin options Plugins []string `long:"plugin" description:"Plugins"` PluginSettings []string `long:"pluginsetting" description:"Plugin settings"` } @@ -244,18 +250,18 @@ func newConfigParser(cfg *config, so *serviceOptions, options flags.Options) *fl func loadConfig() (*config, []string, error) { // Default config. cfg := config{ - HomeDir: defaultHomeDir, - ConfigFile: defaultConfigFile, - DebugLevel: defaultLogLevel, - DataDir: defaultDataDir, - LogDir: defaultLogDir, - HTTPSKey: defaultHTTPSKeyFile, - HTTPSCert: defaultHTTPSCertFile, - Version: version.String(), - Backend: defaultBackend, - TrillianHost: defaultTrillianHost, - DBType: defaultDBType, - DBHost: defaultDBHost, + HomeDir: defaultHomeDir, + ConfigFile: defaultConfigFile, + DebugLevel: defaultLogLevel, + DataDir: defaultDataDir, + LogDir: defaultLogDir, + HTTPSKey: defaultHTTPSKeyFile, + HTTPSCert: defaultHTTPSCertFile, + Version: version.String(), + Backend: defaultBackend, + DBType: defaultDBType, + DBHost: defaultDBHost, + TlogHost: defaultTlogHost, } // Service options which are only added on Windows. @@ -540,13 +546,13 @@ func loadConfig() (*config, []string, error) { return nil, nil, fmt.Errorf("invalid backend type '%v'", cfg.Backend) } - // Verify tstore backend settings + // Verify tstore backend database choice switch cfg.DBType { case tstore.DBTypeLevelDB: // Allowed; continue case tstore.DBTypeMySQL: - // The database password is provided in the env variable "DBPASS" - cfg.DBPass = os.Getenv("DBPASS") + // The database password is provided in an env variable + cfg.DBPass = os.Getenv(envDBPass) if cfg.DBPass == "" { return nil, nil, fmt.Errorf("dbpass not found; you must provide " + "the database password for the politeiad user in the env " + @@ -554,6 +560,19 @@ func loadConfig() (*config, []string, error) { } } + // Verify tlog options + _, err = url.Parse(cfg.TlogHost) + if err != nil { + return nil, nil, fmt.Errorf("invalid tlog host '%v': %v", + cfg.TlogHost, err) + } + cfg.TlogPass = os.Getenv(envTlogPass) + if cfg.TlogPass == "" { + return nil, nil, fmt.Errorf("tlogpass not found: a tlog "+ + "password that will be used to derive the tlog signing key "+ + "must be provided in the env variable %v", envTlogPass) + } + // Warn about missing config file only after all other configuration is // done. This prevents the warning on help messages and invalid // options. Note this should go directly before the return. diff --git a/politeiad/politeiad.go b/politeiad/politeiad.go index e28b01403..294f97f12 100644 --- a/politeiad/politeiad.go +++ b/politeiad/politeiad.go @@ -202,9 +202,8 @@ func (p *politeia) setupBackendGit(anp *chaincfg.Params) error { func (p *politeia) setupBackendTstore(anp *chaincfg.Params) error { b, err := tstorebe.New(p.cfg.HomeDir, p.cfg.DataDir, anp, - p.cfg.TrillianHost, p.cfg.TrillianSigningKey, - p.cfg.DBType, p.cfg.DBHost, p.cfg.DBPass, - p.cfg.DcrtimeHost, p.cfg.DcrtimeCert) + p.cfg.TlogHost, p.cfg.TlogPass, p.cfg.DBType, p.cfg.DBHost, + p.cfg.DBPass, p.cfg.DcrtimeHost, p.cfg.DcrtimeCert) if err != nil { return fmt.Errorf("new tstorebe: %v", err) } diff --git a/util/argon2.go b/util/argon2.go new file mode 100644 index 000000000..4e561c2bf --- /dev/null +++ b/util/argon2.go @@ -0,0 +1,30 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package util + +// Argon2Params represent the argon2 key derivation parameters that are used +// to derive various keys in politeia. +type Argon2Params struct { + Time uint32 `json:"time"` + Memory uint32 `json:"memory"` + Threads uint8 `json:"threads"` + KeyLen uint32 `json:"keylen"` + Salt []byte `json:"salt"` +} + +// NewArgon2Params returns a new Argon2Params with default values. +func NewArgon2Params() Argon2Params { + salt, err := Random(16) + if err != nil { + panic(err) + } + return Argon2Params{ + Time: 1, + Memory: 64 * 1024, // In KiB + Threads: 4, + KeyLen: 32, + Salt: salt, + } +} From ce3ee979bc76db9a8f698a2327b7984bedbe936f Mon Sep 17 00:00:00 2001 From: luke Date: Sat, 27 Mar 2021 14:11:17 -0500 Subject: [PATCH 441/449] multi: Validate token with regexp. --- .../tstorebe/plugins/comments/cmds.go | 2 +- .../tstorebe/plugins/ticketvote/cmds.go | 2 +- politeiad/v2.go | 18 +++++++---- util/token.go | 31 +++++++++++++++++-- 4 files changed, 42 insertions(+), 11 deletions(-) diff --git a/politeiad/backendv2/tstorebe/plugins/comments/cmds.go b/politeiad/backendv2/tstorebe/plugins/comments/cmds.go index 1907b6c6e..bb9b2152a 100644 --- a/politeiad/backendv2/tstorebe/plugins/comments/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/comments/cmds.go @@ -1226,7 +1226,7 @@ func tokenVerify(cmdToken []byte, payloadToken string) error { return backend.PluginError{ PluginID: comments.PluginID, ErrorCode: uint32(comments.ErrorCodeTokenInvalid), - ErrorContext: err.Error(), + ErrorContext: util.TokenRegexp(), } } if !bytes.Equal(cmdToken, pt) { diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go index b3b8260fa..760f6447c 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go @@ -2701,7 +2701,7 @@ func tokenVerify(cmdToken []byte, payloadToken string) error { return backend.PluginError{ PluginID: ticketvote.PluginID, ErrorCode: uint32(ticketvote.ErrorCodeTokenInvalid), - ErrorContext: err.Error(), + ErrorContext: util.TokenRegexp(), } } if !bytes.Equal(cmdToken, pt) { diff --git a/politeiad/v2.go b/politeiad/v2.go index dcdea9420..0c9ec37d4 100644 --- a/politeiad/v2.go +++ b/politeiad/v2.go @@ -89,7 +89,8 @@ func (p *politeia) handleRecordEdit(w http.ResponseWriter, r *http.Request) { if err != nil { respondWithErrorV2(w, r, "handleRecordEdit: decode token", v2.UserErrorReply{ - ErrorCode: v2.ErrorCodeTokenInvalid, + ErrorCode: v2.ErrorCodeTokenInvalid, + ErrorContext: util.TokenRegexp(), }) return } @@ -146,7 +147,8 @@ func (p *politeia) handleRecordEditMetadata(w http.ResponseWriter, r *http.Reque if err != nil { respondWithErrorV2(w, r, "handleRecordEditMetadata: decode token", v2.UserErrorReply{ - ErrorCode: v2.ErrorCodeTokenInvalid, + ErrorCode: v2.ErrorCodeTokenInvalid, + ErrorContext: util.TokenRegexp(), }) return } @@ -198,7 +200,8 @@ func (p *politeia) handleRecordSetStatus(w http.ResponseWriter, r *http.Request) if err != nil { respondWithErrorV2(w, r, "handleRecordSetStatus: decode token", v2.UserErrorReply{ - ErrorCode: v2.ErrorCodeTokenInvalid, + ErrorCode: v2.ErrorCodeTokenInvalid, + ErrorContext: util.TokenRegexp(), }) return } @@ -309,7 +312,8 @@ func (p *politeia) handleRecordTimestamps(w http.ResponseWriter, r *http.Request if err != nil { respondWithErrorV2(w, r, "handleRecordTimestamps: decode token", v2.UserErrorReply{ - ErrorCode: v2.ErrorCodeTokenInvalid, + ErrorCode: v2.ErrorCodeTokenInvalid, + ErrorContext: util.TokenRegexp(), }) return } @@ -492,7 +496,8 @@ func (p *politeia) handlePluginWrite(w http.ResponseWriter, r *http.Request) { if err != nil { respondWithErrorV2(w, r, "handlePluginWrite: decode token", v2.UserErrorReply{ - ErrorCode: v2.ErrorCodeTokenInvalid, + ErrorCode: v2.ErrorCodeTokenInvalid, + ErrorContext: util.TokenRegexp(), }) return } @@ -551,7 +556,8 @@ func (p *politeia) handlePluginReads(w http.ResponseWriter, r *http.Request) { // Invalid token. Save the reply and continue to next cmd. replies[k] = v2.PluginCmdReply{ UserError: &v2.UserErrorReply{ - ErrorCode: v2.ErrorCodeTokenInvalid, + ErrorCode: v2.ErrorCodeTokenInvalid, + ErrorContext: util.TokenRegexp(), }, } continue diff --git a/util/token.go b/util/token.go index 4dbeb3d07..830075f10 100644 --- a/util/token.go +++ b/util/token.go @@ -7,14 +7,24 @@ package util import ( "encoding/hex" "fmt" + "regexp" pdv1 "github.com/decred/politeia/politeiad/api/v1" pdv2 "github.com/decred/politeia/politeiad/api/v2" ) var ( - TokenTypeGit = "git" + // TokenTypeGit represents a token from the politeiad git backend. + TokenTypeGit = "git" + + // TokenTypeTstore represents a token from the politeiad tstore + // backend. TokenTypeTstore = "tstore" + + // tokenRegexp is a regexp that matches short tokens and full + // length tokens. + tokenRegexp = regexp.MustCompile(fmt.Sprintf("^[0-9a-f]{%v,%v}$", + pdv2.ShortTokenLength, pdv2.TokenSize*2)) ) // ShortTokenSize returns the size, in bytes, of a politeiad short token. @@ -46,8 +56,8 @@ func ShortToken(token []byte) ([]byte, error) { // ShortTokenString takes a hex encoded token and returns the shortened token // for it. func ShortTokenString(token string) (string, error) { - if len(token) < pdv2.ShortTokenLength { - return "", fmt.Errorf("token is not large enough") + if tokenRegexp.FindString(token) == "" { + return "", fmt.Errorf("invalid token") } return token[:pdv2.ShortTokenLength], nil } @@ -77,6 +87,11 @@ func TokenIsFullLength(tokenType string, token []byte) bool { // TokenDecode decodes a full length token. An error is returned if the token // is not a full length, hex token. func TokenDecode(tokenType, token string) ([]byte, error) { + // Verify token is valid + if tokenRegexp.FindString(token) == "" { + return nil, fmt.Errorf("invalid token") + } + // Decode token t, err := hex.DecodeString(token) if err != nil { @@ -93,6 +108,11 @@ func TokenDecode(tokenType, token string) ([]byte, error) { // TokenDecodeAnyLength decodes both short tokens and full length tokens. func TokenDecodeAnyLength(tokenType, token string) ([]byte, error) { + // Verify token is valid + if tokenRegexp.FindString(token) == "" { + return nil, fmt.Errorf("invalid token") + } + // Decode token. If provided token has odd length, add padding // to prevent a hex.ErrLength (odd length hex string) error. if len(token)%2 == 1 { @@ -133,3 +153,8 @@ func TokenEncode(token []byte) string { } return t } + +// TokenRegexp returns the string regexp that is used to match tokens. +func TokenRegexp() string { + return tokenRegexp.String() +} From 10b50862dc40c24bc8c1121acc09fbd0511df7ef Mon Sep 17 00:00:00 2001 From: luke Date: Sat, 27 Mar 2021 18:59:17 -0500 Subject: [PATCH 442/449] tstorebe: Move tree ID down a layer. --- .../tstorebe/plugins/comments/cmds.go | 114 +++--- .../tstorebe/plugins/comments/comments.go | 30 +- .../tstorebe/plugins/dcrdata/dcrdata.go | 10 +- .../backendv2/tstorebe/plugins/pi/hooks.go | 8 +- politeiad/backendv2/tstorebe/plugins/pi/pi.go | 12 +- .../backendv2/tstorebe/plugins/plugins.go | 28 +- .../tstorebe/plugins/ticketvote/cmds.go | 172 +++++---- .../tstorebe/plugins/ticketvote/hooks.go | 2 +- .../plugins/ticketvote/internalcmds.go | 2 +- .../tstorebe/plugins/ticketvote/ticketvote.go | 30 +- .../backendv2/tstorebe/plugins/usermd/cmds.go | 4 +- .../tstorebe/plugins/usermd/hooks.go | 2 +- .../tstorebe/plugins/usermd/usermd.go | 14 +- politeiad/backendv2/tstorebe/tstore/anchor.go | 2 +- .../backendv2/tstorebe/tstore/blobclient.go | 85 ++--- politeiad/backendv2/tstorebe/tstore/plugin.go | 22 +- politeiad/backendv2/tstorebe/tstore/record.go | 342 +++++++++++++----- politeiad/backendv2/tstorebe/tstore/tlog.go | 28 ++ .../backendv2/tstorebe/tstore/tlogclient.go | 4 +- politeiad/backendv2/tstorebe/tstore/trees.go | 191 ---------- politeiad/backendv2/tstorebe/tstore/tstore.go | 11 + politeiad/backendv2/tstorebe/tstorebe.go | 150 +++----- 22 files changed, 607 insertions(+), 656 deletions(-) create mode 100644 politeiad/backendv2/tstorebe/tstore/tlog.go delete mode 100644 politeiad/backendv2/tstorebe/tstore/trees.go diff --git a/politeiad/backendv2/tstorebe/plugins/comments/cmds.go b/politeiad/backendv2/tstorebe/plugins/comments/cmds.go index bb9b2152a..c492252f0 100644 --- a/politeiad/backendv2/tstorebe/plugins/comments/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/comments/cmds.go @@ -31,7 +31,7 @@ const ( ) // commentAddSave saves a CommentAdd to the backend. -func (p *commentsPlugin) commentAddSave(treeID int64, ca comments.CommentAdd) ([]byte, error) { +func (p *commentsPlugin) commentAddSave(token []byte, ca comments.CommentAdd) ([]byte, error) { be, err := convertBlobEntryFromCommentAdd(ca) if err != nil { return nil, err @@ -40,7 +40,7 @@ func (p *commentsPlugin) commentAddSave(treeID int64, ca comments.CommentAdd) ([ if err != nil { return nil, err } - err = p.tstore.BlobSave(treeID, *be) + err = p.tstore.BlobSave(token, *be) if err != nil { return nil, err } @@ -50,9 +50,9 @@ func (p *commentsPlugin) commentAddSave(treeID int64, ca comments.CommentAdd) ([ // commentAdds returns a commentAdd for each of the provided digests. A digest // refers to the blob entry digest, which can be used to retrieve the blob // entry from the backend. -func (p *commentsPlugin) commentAdds(treeID int64, digests [][]byte) ([]comments.CommentAdd, error) { +func (p *commentsPlugin) commentAdds(token []byte, digests [][]byte) ([]comments.CommentAdd, error) { // Retrieve blobs - blobs, err := p.tstore.Blobs(treeID, digests) + blobs, err := p.tstore.Blobs(token, digests) if err != nil { return nil, err } @@ -82,7 +82,7 @@ func (p *commentsPlugin) commentAdds(treeID int64, digests [][]byte) ([]comments } // commentDelSave saves a CommentDel to the backend. -func (p *commentsPlugin) commentDelSave(treeID int64, cd comments.CommentDel) ([]byte, error) { +func (p *commentsPlugin) commentDelSave(token []byte, cd comments.CommentDel) ([]byte, error) { be, err := convertBlobEntryFromCommentDel(cd) if err != nil { return nil, err @@ -91,7 +91,7 @@ func (p *commentsPlugin) commentDelSave(treeID int64, cd comments.CommentDel) ([ if err != nil { return nil, err } - err = p.tstore.BlobSave(treeID, *be) + err = p.tstore.BlobSave(token, *be) if err != nil { return nil, err } @@ -101,9 +101,9 @@ func (p *commentsPlugin) commentDelSave(treeID int64, cd comments.CommentDel) ([ // commentDels returns a commentDel for each of the provided digests. A digest // refers to the blob entry digest, which can be used to retrieve the blob // entry from the backend. -func (p *commentsPlugin) commentDels(treeID int64, digests [][]byte) ([]comments.CommentDel, error) { +func (p *commentsPlugin) commentDels(token []byte, digests [][]byte) ([]comments.CommentDel, error) { // Retrieve blobs - blobs, err := p.tstore.Blobs(treeID, digests) + blobs, err := p.tstore.Blobs(token, digests) if err != nil { return nil, err } @@ -133,7 +133,7 @@ func (p *commentsPlugin) commentDels(treeID int64, digests [][]byte) ([]comments } // commentVoteSave saves a CommentVote to the backend. -func (p *commentsPlugin) commentVoteSave(treeID int64, cv comments.CommentVote) ([]byte, error) { +func (p *commentsPlugin) commentVoteSave(token []byte, cv comments.CommentVote) ([]byte, error) { be, err := convertBlobEntryFromCommentVote(cv) if err != nil { return nil, err @@ -142,7 +142,7 @@ func (p *commentsPlugin) commentVoteSave(treeID int64, cv comments.CommentVote) if err != nil { return nil, err } - err = p.tstore.BlobSave(treeID, *be) + err = p.tstore.BlobSave(token, *be) if err != nil { return nil, err } @@ -152,9 +152,9 @@ func (p *commentsPlugin) commentVoteSave(treeID int64, cv comments.CommentVote) // commentVotes returns a CommentVote for each of the provided digests. A // digest refers to the blob entry digest, which can be used to retrieve the // blob entry from the backend. -func (p *commentsPlugin) commentVotes(treeID int64, digests [][]byte) ([]comments.CommentVote, error) { +func (p *commentsPlugin) commentVotes(token []byte, digests [][]byte) ([]comments.CommentVote, error) { // Retrieve blobs - blobs, err := p.tstore.Blobs(treeID, digests) + blobs, err := p.tstore.Blobs(token, digests) if err != nil { return nil, err } @@ -188,7 +188,7 @@ func (p *commentsPlugin) commentVotes(treeID int64, digests [][]byte) ([]comment // provided comment IDs, the comment ID is excluded from the returned map. An // error will not be returned. It is the responsibility of the caller to ensure // a comment is returned for each of the provided comment IDs. -func (p *commentsPlugin) comments(treeID int64, ridx recordIndex, commentIDs []uint32) (map[uint32]comments.Comment, error) { +func (p *commentsPlugin) comments(token []byte, ridx recordIndex, commentIDs []uint32) (map[uint32]comments.Comment, error) { // Aggregate the digests for all records that need to be looked up. // If a comment has been deleted then the only record that will // still exist is the comment del record. If the comment has not @@ -217,7 +217,7 @@ func (p *commentsPlugin) comments(treeID int64, ridx recordIndex, commentIDs []u } // Get comment add records - adds, err := p.commentAdds(treeID, digestAdds) + adds, err := p.commentAdds(token, digestAdds) if err != nil { return nil, fmt.Errorf("commentAdds: %v", err) } @@ -227,7 +227,7 @@ func (p *commentsPlugin) comments(treeID int64, ridx recordIndex, commentIDs []u } // Get comment del records - dels, err := p.commentDels(treeID, digestDels) + dels, err := p.commentDels(token, digestDels) if err != nil { return nil, fmt.Errorf("commentDels: %v", err) } @@ -256,8 +256,8 @@ func (p *commentsPlugin) comments(treeID int64, ridx recordIndex, commentIDs []u } // comment returns the latest version of a comment. -func (p *commentsPlugin) comment(treeID int64, ridx recordIndex, commentID uint32) (*comments.Comment, error) { - cs, err := p.comments(treeID, ridx, []uint32{commentID}) +func (p *commentsPlugin) comment(token []byte, ridx recordIndex, commentID uint32) (*comments.Comment, error) { + cs, err := p.comments(token, ridx, []uint32{commentID}) if err != nil { return nil, fmt.Errorf("comments: %v", err) } @@ -269,9 +269,9 @@ func (p *commentsPlugin) comment(treeID int64, ridx recordIndex, commentID uint3 } // timestamp returns the timestamp for a blob entry digest. -func (p *commentsPlugin) timestamp(treeID int64, digest []byte) (*comments.Timestamp, error) { +func (p *commentsPlugin) timestamp(token []byte, digest []byte) (*comments.Timestamp, error) { // Get timestamp - t, err := p.tstore.Timestamp(treeID, digest) + t, err := p.tstore.Timestamp(token, digest) if err != nil { return nil, err } @@ -347,7 +347,7 @@ func voteScore(cidx commentIndex) (uint64, uint64) { } // cmdNew creates a new comment. -func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (string, error) { +func (p *commentsPlugin) cmdNew(token []byte, payload string) (string, error) { // Decode payload var n comments.New err := json.Unmarshal([]byte(payload), &n) @@ -380,7 +380,7 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str } // Verify record state - state, err := p.tstore.RecordState(treeID) + state, err := p.tstore.RecordState(token) if err != nil { return "", err } @@ -427,7 +427,7 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str } // Save comment - digest, err := p.commentAddSave(treeID, ca) + digest, err := p.commentAddSave(token, ca) if err != nil { return "", fmt.Errorf("commentAddSave: %v", err) } @@ -448,7 +448,7 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str ca.Token, ca.CommentID) // Return new comment - c, err := p.comment(treeID, *ridx, ca.CommentID) + c, err := p.comment(token, *ridx, ca.CommentID) if err != nil { return "", fmt.Errorf("comment %x %v: %v", token, ca.CommentID, err) } @@ -466,7 +466,7 @@ func (p *commentsPlugin) cmdNew(treeID int64, token []byte, payload string) (str } // cmdEdit edits an existing comment. -func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (string, error) { +func (p *commentsPlugin) cmdEdit(token []byte, payload string) (string, error) { // Decode payload var e comments.Edit err := json.Unmarshal([]byte(payload), &e) @@ -499,7 +499,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st } // Verify record state - state, err := p.tstore.RecordState(treeID) + state, err := p.tstore.RecordState(token) if err != nil { return "", err } @@ -518,7 +518,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st } // Get the existing comment - cs, err := p.comments(treeID, *ridx, []uint32{e.CommentID}) + cs, err := p.comments(token, *ridx, []uint32{e.CommentID}) if err != nil { return "", fmt.Errorf("comments %v: %v", e.CommentID, err) } @@ -575,7 +575,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st } // Save comment - digest, err := p.commentAddSave(treeID, ca) + digest, err := p.commentAddSave(token, ca) if err != nil { return "", fmt.Errorf("commentAddSave: %v", err) } @@ -590,7 +590,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st ca.Token, ca.CommentID) // Return updated comment - c, err := p.comment(treeID, *ridx, e.CommentID) + c, err := p.comment(token, *ridx, e.CommentID) if err != nil { return "", fmt.Errorf("comment %x %v: %v", token, e.CommentID, err) } @@ -608,7 +608,7 @@ func (p *commentsPlugin) cmdEdit(treeID int64, token []byte, payload string) (st } // cmdDel deletes a comment. -func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (string, error) { +func (p *commentsPlugin) cmdDel(token []byte, payload string) (string, error) { // Decode payload var d comments.Del err := json.Unmarshal([]byte(payload), &d) @@ -631,7 +631,7 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str } // Verify record state - state, err := p.tstore.RecordState(treeID) + state, err := p.tstore.RecordState(token) if err != nil { return "", err } @@ -650,7 +650,7 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str } // Get the existing comment - cs, err := p.comments(treeID, *ridx, []uint32{d.CommentID}) + cs, err := p.comments(token, *ridx, []uint32{d.CommentID}) if err != nil { return "", fmt.Errorf("comments %v: %v", d.CommentID, err) } @@ -678,7 +678,7 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str } // Save comment del - digest, err := p.commentDelSave(treeID, cd) + digest, err := p.commentDelSave(token, cd) if err != nil { return "", fmt.Errorf("commentDelSave: %v", err) } @@ -704,14 +704,14 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str for _, v := range cidx.Adds { digests = append(digests, v) } - err = p.tstore.BlobsDel(treeID, digests) + err = p.tstore.BlobsDel(token, digests) if err != nil { log.Errorf("comments cmdDel %x: BlobsDel %x: %v ", token, digests, err) } // Return updated comment - c, err := p.comment(treeID, *ridx, d.CommentID) + c, err := p.comment(token, *ridx, d.CommentID) if err != nil { return "", fmt.Errorf("comment %v: %v", d.CommentID, err) } @@ -729,7 +729,7 @@ func (p *commentsPlugin) cmdDel(treeID int64, token []byte, payload string) (str } // cmdVote casts a upvote/downvote for a comment. -func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (string, error) { +func (p *commentsPlugin) cmdVote(token []byte, payload string) (string, error) { // Decode payload var v comments.Vote err := json.Unmarshal([]byte(payload), &v) @@ -764,7 +764,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st } // Verify record state - state, err := p.tstore.RecordState(treeID) + state, err := p.tstore.RecordState(token) if err != nil { return "", err } @@ -800,7 +800,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st } // Verify user is not voting on their own comment - cs, err := p.comments(treeID, *ridx, []uint32{v.CommentID}) + cs, err := p.comments(token, *ridx, []uint32{v.CommentID}) if err != nil { return "", fmt.Errorf("comments %v: %v", v.CommentID, err) } @@ -831,7 +831,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st } // Save comment vote - digest, err := p.commentVoteSave(treeID, cv) + digest, err := p.commentVoteSave(token, cv) if err != nil { return "", fmt.Errorf("commentVoteSave: %v", err) } @@ -871,7 +871,7 @@ func (p *commentsPlugin) cmdVote(treeID int64, token []byte, payload string) (st // cmdGet retrieves a batch of specified comments. The most recent version of // each comment is returned. -func (p *commentsPlugin) cmdGet(treeID int64, token []byte, payload string) (string, error) { +func (p *commentsPlugin) cmdGet(token []byte, payload string) (string, error) { // Decode payload var g comments.Get err := json.Unmarshal([]byte(payload), &g) @@ -880,7 +880,7 @@ func (p *commentsPlugin) cmdGet(treeID int64, token []byte, payload string) (str } // Get record state - state, err := p.tstore.RecordState(treeID) + state, err := p.tstore.RecordState(token) if err != nil { return "", err } @@ -892,7 +892,7 @@ func (p *commentsPlugin) cmdGet(treeID int64, token []byte, payload string) (str } // Get comments - cs, err := p.comments(treeID, *ridx, g.CommentIDs) + cs, err := p.comments(token, *ridx, g.CommentIDs) if err != nil { return "", fmt.Errorf("comments: %v", err) } @@ -911,9 +911,9 @@ func (p *commentsPlugin) cmdGet(treeID int64, token []byte, payload string) (str // cmdGetAll retrieves all comments for a record. The latest version of each // comment is returned. -func (p *commentsPlugin) cmdGetAll(treeID int64, token []byte) (string, error) { +func (p *commentsPlugin) cmdGetAll(token []byte) (string, error) { // Get record state - state, err := p.tstore.RecordState(treeID) + state, err := p.tstore.RecordState(token) if err != nil { return "", err } @@ -929,7 +929,7 @@ func (p *commentsPlugin) cmdGetAll(treeID int64, token []byte) (string, error) { } // Get comments - c, err := p.comments(treeID, *ridx, commentIDs) + c, err := p.comments(token, *ridx, commentIDs) if err != nil { return "", fmt.Errorf("comments: %v", err) } @@ -958,7 +958,7 @@ func (p *commentsPlugin) cmdGetAll(treeID int64, token []byte) (string, error) { } // cmdGetVersion retrieves the specified version of a comment. -func (p *commentsPlugin) cmdGetVersion(treeID int64, token []byte, payload string) (string, error) { +func (p *commentsPlugin) cmdGetVersion(token []byte, payload string) (string, error) { // Decode payload var gv comments.GetVersion err := json.Unmarshal([]byte(payload), &gv) @@ -967,7 +967,7 @@ func (p *commentsPlugin) cmdGetVersion(treeID int64, token []byte, payload strin } // Get record state - state, err := p.tstore.RecordState(treeID) + state, err := p.tstore.RecordState(token) if err != nil { return "", err } @@ -1004,7 +1004,7 @@ func (p *commentsPlugin) cmdGetVersion(treeID int64, token []byte, payload strin } // Get comment add record - adds, err := p.commentAdds(treeID, [][]byte{digest}) + adds, err := p.commentAdds(token, [][]byte{digest}) if err != nil { return "", fmt.Errorf("commentAdds: %v", err) } @@ -1031,9 +1031,9 @@ func (p *commentsPlugin) cmdGetVersion(treeID int64, token []byte, payload strin // cmdCount retrieves the comments count for a record. The comments count is // the number of comments that have been made on a record. -func (p *commentsPlugin) cmdCount(treeID int64, token []byte) (string, error) { +func (p *commentsPlugin) cmdCount(token []byte) (string, error) { // Get record state - state, err := p.tstore.RecordState(treeID) + state, err := p.tstore.RecordState(token) if err != nil { return "", err } @@ -1058,7 +1058,7 @@ func (p *commentsPlugin) cmdCount(treeID int64, token []byte) (string, error) { // cmdVotes retrieves the comment votes that meet the provided filtering // criteria. -func (p *commentsPlugin) cmdVotes(treeID int64, token []byte, payload string) (string, error) { +func (p *commentsPlugin) cmdVotes(token []byte, payload string) (string, error) { // Decode payload var v comments.Votes err := json.Unmarshal([]byte(payload), &v) @@ -1067,7 +1067,7 @@ func (p *commentsPlugin) cmdVotes(treeID int64, token []byte, payload string) (s } // Get record state - state, err := p.tstore.RecordState(treeID) + state, err := p.tstore.RecordState(token) if err != nil { return "", err } @@ -1095,7 +1095,7 @@ func (p *commentsPlugin) cmdVotes(treeID int64, token []byte, payload string) (s } // Lookup votes - votes, err := p.commentVotes(treeID, digests) + votes, err := p.commentVotes(token, digests) if err != nil { return "", fmt.Errorf("commentVotes: %v", err) } @@ -1113,7 +1113,7 @@ func (p *commentsPlugin) cmdVotes(treeID int64, token []byte, payload string) (s } // cmdTimestamps retrieves the timestamps for the comments of a record. -func (p *commentsPlugin) cmdTimestamps(treeID int64, token []byte, payload string) (string, error) { +func (p *commentsPlugin) cmdTimestamps(token []byte, payload string) (string, error) { // Decode payload var t comments.Timestamps err := json.Unmarshal([]byte(payload), &t) @@ -1122,7 +1122,7 @@ func (p *commentsPlugin) cmdTimestamps(treeID int64, token []byte, payload strin } // Get record state - state, err := p.tstore.RecordState(treeID) + state, err := p.tstore.RecordState(token) if err != nil { return "", err } @@ -1156,7 +1156,7 @@ func (p *commentsPlugin) cmdTimestamps(treeID int64, token []byte, payload strin // Get timestamps for adds ts := make([]comments.Timestamp, 0, len(cidx.Adds)+1) for _, v := range cidx.Adds { - t, err := p.timestamp(treeID, v) + t, err := p.timestamp(token, v) if err != nil { return "", err } @@ -1165,7 +1165,7 @@ func (p *commentsPlugin) cmdTimestamps(treeID int64, token []byte, payload strin // Get timestamp for del if cidx.Del != nil { - t, err := p.timestamp(treeID, cidx.Del) + t, err := p.timestamp(token, cidx.Del) if err != nil { return "", err } @@ -1184,7 +1184,7 @@ func (p *commentsPlugin) cmdTimestamps(treeID int64, token []byte, payload strin ts = make([]comments.Timestamp, 0, len(cidx.Votes)) for _, votes := range cidx.Votes { for _, v := range votes { - t, err := p.timestamp(treeID, v.Digest) + t, err := p.timestamp(token, v.Digest) if err != nil { return "", err } diff --git a/politeiad/backendv2/tstorebe/plugins/comments/comments.go b/politeiad/backendv2/tstorebe/plugins/comments/comments.go index 2d176e165..fdd674b27 100644 --- a/politeiad/backendv2/tstorebe/plugins/comments/comments.go +++ b/politeiad/backendv2/tstorebe/plugins/comments/comments.go @@ -56,30 +56,30 @@ func (p *commentsPlugin) Setup() error { // Cmd executes a plugin command. // // This function satisfies the plugins PluginClient interface. -func (p *commentsPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { - log.Tracef("comments Cmd: %v %x %v %v", treeID, token, cmd, payload) +func (p *commentsPlugin) Cmd(token []byte, cmd, payload string) (string, error) { + log.Tracef("comments Cmd: %x %v %v", token, cmd, payload) switch cmd { case comments.CmdNew: - return p.cmdNew(treeID, token, payload) + return p.cmdNew(token, payload) case comments.CmdEdit: - return p.cmdEdit(treeID, token, payload) + return p.cmdEdit(token, payload) case comments.CmdDel: - return p.cmdDel(treeID, token, payload) + return p.cmdDel(token, payload) case comments.CmdVote: - return p.cmdVote(treeID, token, payload) + return p.cmdVote(token, payload) case comments.CmdGet: - return p.cmdGet(treeID, token, payload) + return p.cmdGet(token, payload) case comments.CmdGetAll: - return p.cmdGetAll(treeID, token) + return p.cmdGetAll(token) case comments.CmdGetVersion: - return p.cmdGetVersion(treeID, token, payload) + return p.cmdGetVersion(token, payload) case comments.CmdCount: - return p.cmdCount(treeID, token) + return p.cmdCount(token) case comments.CmdVotes: - return p.cmdVotes(treeID, token, payload) + return p.cmdVotes(token, payload) case comments.CmdTimestamps: - return p.cmdTimestamps(treeID, token, payload) + return p.cmdTimestamps(token, payload) } return "", backend.ErrPluginCmdInvalid @@ -88,8 +88,8 @@ func (p *commentsPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (s // Hook executes a plugin hook. // // This function satisfies the plugins PluginClient interface. -func (p *commentsPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { - log.Tracef("comments Hook: %v %x %v", plugins.Hooks[h], token, treeID) +func (p *commentsPlugin) Hook(h plugins.HookT, payload string) error { + log.Tracef("comments Hook: %x %v", plugins.Hooks[h]) return nil } @@ -97,7 +97,7 @@ func (p *commentsPlugin) Hook(treeID int64, token []byte, h plugins.HookT, paylo // Fsck performs a plugin filesystem check. // // This function satisfies the plugins PluginClient interface. -func (p *commentsPlugin) Fsck(treeIDs []int64) error { +func (p *commentsPlugin) Fsck() error { log.Tracef("comments Fsck") // Verify record index coherency diff --git a/politeiad/backendv2/tstorebe/plugins/dcrdata/dcrdata.go b/politeiad/backendv2/tstorebe/plugins/dcrdata/dcrdata.go index 3a2f63986..b87915347 100644 --- a/politeiad/backendv2/tstorebe/plugins/dcrdata/dcrdata.go +++ b/politeiad/backendv2/tstorebe/plugins/dcrdata/dcrdata.go @@ -194,8 +194,8 @@ func (p *dcrdataPlugin) Setup() error { // Cmd executes a plugin command. // // This function satisfies the plugins PluginClient interface. -func (p *dcrdataPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { - log.Tracef("dcrdata Cmd: %v %x %v %v", treeID, token, cmd, payload) +func (p *dcrdataPlugin) Cmd(token []byte, cmd, payload string) (string, error) { + log.Tracef("dcrdata Cmd: %x %v %v", token, cmd, payload) switch cmd { case dcrdata.CmdBestBlock: @@ -214,8 +214,8 @@ func (p *dcrdataPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (st // Hook executes a plugin hook. // // This function satisfies the plugins PluginClient interface. -func (p *dcrdataPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { - log.Tracef("dcrdata Hook: %v %x %v", plugins.Hooks[h], token, treeID) +func (p *dcrdataPlugin) Hook(h plugins.HookT, payload string) error { + log.Tracef("dcrdata Hook: %v", plugins.Hooks[h]) return nil } @@ -223,7 +223,7 @@ func (p *dcrdataPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payloa // Fsck performs a plugin filesystem check. // // This function satisfies the plugins PluginClient interface. -func (p *dcrdataPlugin) Fsck(treeIDs []int64) error { +func (p *dcrdataPlugin) Fsck() error { log.Tracef("dcrdata Fsck") return nil diff --git a/politeiad/backendv2/tstorebe/plugins/pi/hooks.go b/politeiad/backendv2/tstorebe/plugins/pi/hooks.go index 4b1db4faf..9b454b347 100644 --- a/politeiad/backendv2/tstorebe/plugins/pi/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/pi/hooks.go @@ -108,7 +108,7 @@ func (p *piPlugin) hookCommentVote(token []byte) error { // hookPluginPre extends plugin write commands from other plugins with pi // specific validation. -func (p *piPlugin) hookPluginPre(treeID int64, token []byte, payload string) error { +func (p *piPlugin) hookPluginPre(payload string) error { // Decode payload var hpp plugins.HookPluginPre err := json.Unmarshal([]byte(payload), &hpp) @@ -121,11 +121,11 @@ func (p *piPlugin) hookPluginPre(treeID int64, token []byte, payload string) err case comments.PluginID: switch hpp.Cmd { case comments.CmdNew: - return p.hookCommentNew(token) + return p.hookCommentNew(hpp.Token) case comments.CmdDel: - return p.hookCommentDel(token) + return p.hookCommentDel(hpp.Token) case comments.CmdVote: - return p.hookCommentVote(token) + return p.hookCommentVote(hpp.Token) } } diff --git a/politeiad/backendv2/tstorebe/plugins/pi/pi.go b/politeiad/backendv2/tstorebe/plugins/pi/pi.go index 99822f929..f9d861efe 100644 --- a/politeiad/backendv2/tstorebe/plugins/pi/pi.go +++ b/politeiad/backendv2/tstorebe/plugins/pi/pi.go @@ -58,8 +58,8 @@ func (p *piPlugin) Setup() error { // Cmd executes a plugin command. // // This function satisfies the plugins PluginClient interface. -func (p *piPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { - log.Tracef("pi Cmd: %v %x %v %v", treeID, token, cmd, payload) +func (p *piPlugin) Cmd(token []byte, cmd, payload string) (string, error) { + log.Tracef("pi Cmd: %x %v %v", token, cmd, payload) return "", backend.ErrPluginCmdInvalid } @@ -67,8 +67,8 @@ func (p *piPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, // Hook executes a plugin hook. // // This function satisfies the plugins PluginClient interface. -func (p *piPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { - log.Tracef("pi Hook: %v %x %v", plugins.Hooks[h], token, treeID) +func (p *piPlugin) Hook(h plugins.HookT, payload string) error { + log.Tracef("pi Hook: %v", plugins.Hooks[h]) switch h { case plugins.HookTypeNewRecordPre: @@ -76,7 +76,7 @@ func (p *piPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload str case plugins.HookTypeEditRecordPre: return p.hookEditRecordPre(payload) case plugins.HookTypePluginPre: - return p.hookPluginPre(treeID, token, payload) + return p.hookPluginPre(payload) } return nil @@ -85,7 +85,7 @@ func (p *piPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload str // Fsck performs a plugin filesystem check. // // This function satisfies the plugins PluginClient interface. -func (p *piPlugin) Fsck(treeIDs []int64) error { +func (p *piPlugin) Fsck() error { log.Tracef("pi Fsck") return nil diff --git a/politeiad/backendv2/tstorebe/plugins/plugins.go b/politeiad/backendv2/tstorebe/plugins/plugins.go index 57b50c690..0a3d94f89 100644 --- a/politeiad/backendv2/tstorebe/plugins/plugins.go +++ b/politeiad/backendv2/tstorebe/plugins/plugins.go @@ -122,7 +122,7 @@ type HookSetRecordStatus struct { // HookPluginPre is the payload for the pre plugin hook. type HookPluginPre struct { - Token string `json:"token"` + Token []byte `json:"token"` PluginID string `json:"pluginid"` Cmd string `json:"cmd"` Payload string `json:"payload"` @@ -144,13 +144,13 @@ type PluginClient interface { Setup() error // Cmd executes a plugin command. - Cmd(treeID int64, token []byte, cmd, payload string) (string, error) + Cmd(token []byte, cmd, payload string) (string, error) // Hook executes a plugin hook. - Hook(treeID int64, token []byte, h HookT, payload string) error + Hook(h HookT, payload string) error // Fsck performs a plugin file system check. - Fsck(treeIDs []int64) error + Fsck() error // Settings returns the plugin settings. Settings() []backend.PluginSetting @@ -165,38 +165,38 @@ type TstoreClient interface { // is unvetted. The digest of the data, i.e. BlobEntry.Digest, can // be thought of as the blob ID that can be used to get/del the // blob from tstore. - BlobSave(treeID int64, be store.BlobEntry) error + BlobSave(token []byte, be store.BlobEntry) error // BlobsDel deletes the blobs that correspond to the provided // digests. - BlobsDel(treeID int64, digests [][]byte) error + BlobsDel(token []byte, digests [][]byte) error // Blobs returns the blobs that correspond to the provided digests. // If a blob does not exist it will not be included in the returned // map. If a record is vetted, only vetted blobs will be returned. - Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEntry, error) + Blobs(token []byte, digests [][]byte) (map[string]store.BlobEntry, error) // BlobsByDataDesc returns all blobs that match the provided data // descriptor. The blobs will be ordered from oldest to newest. If // a record is vetted then only vetted blobs will be returned. - BlobsByDataDesc(treeID int64, dataDesc []string) ([]store.BlobEntry, error) + BlobsByDataDesc(token []byte, dataDesc []string) ([]store.BlobEntry, error) // DigestsByDataDesc returns the digests of all blobs that match // the provided data descriptor. The digests will be ordered from // oldest to newest. If a record is vetted, only vetted blobs will // be returned. - DigestsByDataDesc(treeID int64, dataDesc []string) ([][]byte, error) + DigestsByDataDesc(token []byte, dataDesc []string) ([][]byte, error) // Timestamp returns the timestamp for the blob that correpsonds // to the digest. If a record is vetted, only vetted timestamps // will be returned. - Timestamp(treeID int64, digest []byte) (*backend.Timestamp, error) + Timestamp(token []byte, digest []byte) (*backend.Timestamp, error) // Record returns a version of a record. - Record(treeID int64, version uint32) (*backend.Record, error) + Record(token []byte, version uint32) (*backend.Record, error) // RecordLatest returns the most recent version of a record. - RecordLatest(treeID int64) (*backend.Record, error) + RecordLatest(token []byte) (*backend.Record, error) // RecordPartial returns a partial record. This method gives the // caller fine grained control over what version and what files are @@ -213,9 +213,9 @@ type TstoreClient interface { // // OmitAllFiles can be used to retrieve a record without any of the // record files. This supersedes the filenames argument. - RecordPartial(treeID int64, version uint32, filenames []string, + RecordPartial(token []byte, version uint32, filenames []string, omitAllFiles bool) (*backend.Record, error) // RecordState returns whether the record is unvetted or vetted. - RecordState(treeID int64) (backend.StateT, error) + RecordState(token []byte) (backend.StateT, error) } diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go index 760f6447c..5f5af5ba9 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go @@ -42,7 +42,7 @@ const ( ) // cmdAuthorize authorizes a ticket vote or revokes a previous authorization. -func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload string) (string, error) { +func (p *ticketVotePlugin) cmdAuthorize(token []byte, payload string) (string, error) { // Decode payload var a ticketvote.Authorize err := json.Unmarshal([]byte(payload), &a) @@ -80,7 +80,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri } // Verify record status and version - r, err := p.tstore.RecordPartial(treeID, 0, nil, true) + r, err := p.tstore.RecordPartial(token, 0, nil, true) if err != nil { return "", fmt.Errorf("RecordPartial: %v", err) } @@ -103,7 +103,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri // Get any previous authorizations to verify that the new action // is allowed based on the previous action. - auths, err := p.auths(treeID) + auths, err := p.auths(token) if err != nil { return "", err } @@ -153,7 +153,7 @@ func (p *ticketVotePlugin) cmdAuthorize(treeID int64, token []byte, payload stri } // Save authorize vote - err = p.authSave(treeID, auth) + err = p.authSave(token, auth) if err != nil { return "", err } @@ -426,7 +426,7 @@ func (p *ticketVotePlugin) startReply(duration uint32) (*ticketvote.StartReply, } // startStandard starts a standard vote. -func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvote.Start) (*ticketvote.StartReply, error) { +func (p *ticketVotePlugin) startStandard(token []byte, s ticketvote.Start) (*ticketvote.StartReply, error) { // Verify there is only one start details if len(s.Starts) != 1 { return nil, backend.PluginError{ @@ -468,7 +468,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot } // Verify record version - r, err := p.tstore.RecordPartial(treeID, 0, nil, true) + r, err := p.tstore.RecordPartial(token, 0, nil, true) if err != nil { return nil, fmt.Errorf("RecordPartial: %v", err) } @@ -487,7 +487,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot } // Verify vote authorization - auths, err := p.auths(treeID) + auths, err := p.auths(token) if err != nil { return nil, err } @@ -508,7 +508,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot } // Verify vote has not already been started - svp, err := p.voteDetails(treeID) + svp, err := p.voteDetails(token) if err != nil { return nil, err } @@ -533,7 +533,7 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot } // Save vote details - err = p.voteDetailsSave(treeID, vd) + err = p.voteDetailsSave(token, vd) if err != nil { return nil, fmt.Errorf("voteDetailsSave: %v", err) } @@ -554,27 +554,25 @@ func (p *ticketVotePlugin) startStandard(treeID int64, token []byte, s ticketvot } // startRunoffRecordSave saves a startRunoffRecord to the backend. -func (p *ticketVotePlugin) startRunoffRecordSave(treeID int64, srr startRunoffRecord) error { +func (p *ticketVotePlugin) startRunoffRecordSave(token []byte, srr startRunoffRecord) error { be, err := convertBlobEntryFromStartRunoff(srr) if err != nil { return err } - err = p.tstore.BlobSave(treeID, *be) + err = p.tstore.BlobSave(token, *be) if err != nil { - return fmt.Errorf("BlobSave %v %v: %v", - treeID, dataDescriptorStartRunoff, err) + return err } return nil } // startRunoffRecord returns the startRunoff record if one exists. Nil is // returned if a startRunoff record is not found. -func (p *ticketVotePlugin) startRunoffRecord(treeID int64) (*startRunoffRecord, error) { - blobs, err := p.tstore.BlobsByDataDesc(treeID, +func (p *ticketVotePlugin) startRunoffRecord(token []byte) (*startRunoffRecord, error) { + blobs, err := p.tstore.BlobsByDataDesc(token, []string{dataDescriptorStartRunoff}) if err != nil { - return nil, fmt.Errorf("BlobsByDataDesc %v %v: %v", - treeID, dataDescriptorStartRunoff, err) + return nil, err } var srr *startRunoffRecord @@ -598,7 +596,7 @@ func (p *ticketVotePlugin) startRunoffRecord(treeID int64) (*startRunoffRecord, } // startRunoffForSub starts the voting period for a runoff vote submission. -func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs startRunoffSubmission) error { +func (p *ticketVotePlugin) startRunoffForSub(token []byte, srs startRunoffSubmission) error { // Sanity check sd := srs.StartDetails t, err := tokenDecode(sd.Params.Token) @@ -609,8 +607,8 @@ func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs sta return fmt.Errorf("invalid token") } - // Get the start runoff record from the parent tree - srr, err := p.startRunoffRecord(srs.ParentTreeID) + // Get the start runoff record from the parent record + srr, err := p.startRunoffRecord(srs.ParentToken) if err != nil { return err } @@ -635,7 +633,7 @@ func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs sta // call were to fail before completing, we can simply call the // command again with the same arguments and it will pick up where // it left off. - svp, err := p.voteDetails(treeID) + svp, err := p.voteDetails(token) if err != nil { return err } @@ -645,7 +643,7 @@ func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs sta } // Verify record version - r, err := p.tstore.RecordPartial(treeID, 0, nil, true) + r, err := p.tstore.RecordPartial(token, 0, nil, true) if err != nil { return fmt.Errorf("RecordPartial: %v", err) } @@ -675,9 +673,9 @@ func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs sta } // Save vote details - err = p.voteDetailsSave(treeID, vd) + err = p.voteDetailsSave(token, vd) if err != nil { - return fmt.Errorf("voteDetailsSave: %v", err) + return fmt.Errorf("voteDetailsSave %x: %v", token, err) } // Update inventory @@ -693,9 +691,9 @@ func (p *ticketVotePlugin) startRunoffForSub(treeID int64, token []byte, srs sta // startRunoffForParent saves a startRunoffRecord to the parent record. Once // this has been saved the runoff vote is considered to be started and the // voting period on individual runoff vote submissions can be started. -func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ticketvote.Start) (*startRunoffRecord, error) { +func (p *ticketVotePlugin) startRunoffForParent(token []byte, s ticketvote.Start) (*startRunoffRecord, error) { // Check if the runoff vote data already exists on the parent tree. - srr, err := p.startRunoffRecord(treeID) + srr, err := p.startRunoffRecord(token) if err != nil { return nil, err } @@ -723,7 +721,7 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti files := []string{ ticketvote.FileNameVoteMetadata, } - r, err := p.tstore.RecordPartial(treeID, 0, files, false) + r, err := p.tstore.RecordPartial(token, 0, files, false) if err != nil { if errors.Is(err, backend.ErrRecordNotFound) { return nil, backend.PluginError{ @@ -848,17 +846,17 @@ func (p *ticketVotePlugin) startRunoffForParent(treeID int64, token []byte, s ti } // Save start runoff record - err = p.startRunoffRecordSave(treeID, *srr) + err = p.startRunoffRecordSave(token, *srr) if err != nil { - return nil, fmt.Errorf("startRunoffRecordSave %v: %v", - treeID, err) + return nil, fmt.Errorf("startRunoffRecordSave %x: %v", + token, err) } return srr, nil } // startRunoff starts the voting period for all submissions in a runoff vote. -func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote.Start) (*ticketvote.StartReply, error) { +func (p *ticketVotePlugin) startRunoff(token []byte, s ticketvote.Start) (*ticketvote.StartReply, error) { // Sanity check if len(s.Starts) == 0 { return nil, fmt.Errorf("no start details found") @@ -978,7 +976,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. // This function is being invoked on the runoff vote parent record. // Create and save a start runoff record onto the parent record's tree. - srr, err := p.startRunoffForParent(treeID, token, s) + srr, err := p.startRunoffForParent(token, s) if err != nil { return nil, err } @@ -991,7 +989,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. return nil, err } srs := startRunoffSubmission{ - ParentTreeID: treeID, + ParentToken: token, StartDetails: v, } b, err := json.Marshal(srs) @@ -1021,7 +1019,7 @@ func (p *ticketVotePlugin) startRunoff(treeID int64, token []byte, s ticketvote. // cmdStartRunoffSubmission is an internal plugin command that is used to start // the voting period on a runoff vote submission. -func (p *ticketVotePlugin) cmdStartRunoffSubmission(treeID int64, token []byte, payload string) (string, error) { +func (p *ticketVotePlugin) cmdStartRunoffSubmission(token []byte, payload string) (string, error) { // Decode payload var srs startRunoffSubmission err := json.Unmarshal([]byte(payload), &srs) @@ -1030,7 +1028,7 @@ func (p *ticketVotePlugin) cmdStartRunoffSubmission(treeID int64, token []byte, } // Start voting period on runoff vote submission - err = p.startRunoffForSub(treeID, token, srs) + err = p.startRunoffForSub(token, srs) if err != nil { return "", err } @@ -1039,7 +1037,7 @@ func (p *ticketVotePlugin) cmdStartRunoffSubmission(treeID int64, token []byte, } // cmdStart starts a ticket vote. -func (p *ticketVotePlugin) cmdStart(treeID int64, token []byte, payload string) (string, error) { +func (p *ticketVotePlugin) cmdStart(token []byte, payload string) (string, error) { // Decode payload var s ticketvote.Start err := json.Unmarshal([]byte(payload), &s) @@ -1061,12 +1059,12 @@ func (p *ticketVotePlugin) cmdStart(treeID int64, token []byte, payload string) var sr *ticketvote.StartReply switch vtype { case ticketvote.VoteTypeStandard: - sr, err = p.startStandard(treeID, token, s) + sr, err = p.startStandard(token, s) if err != nil { return "", err } case ticketvote.VoteTypeRunoff: - sr, err = p.startRunoff(treeID, token, s) + sr, err = p.startRunoff(token, s) if err != nil { return "", err } @@ -1245,7 +1243,7 @@ type voteCollider struct { } // voteColliderSave saves a voteCollider to the backend. -func (p *ticketVotePlugin) voteColliderSave(treeID int64, vc voteCollider) error { +func (p *ticketVotePlugin) voteColliderSave(token []byte, vc voteCollider) error { // Prepare blob be, err := convertBlobEntryFromVoteCollider(vc) if err != nil { @@ -1253,11 +1251,11 @@ func (p *ticketVotePlugin) voteColliderSave(treeID int64, vc voteCollider) error } // Save blob - return p.tstore.BlobSave(treeID, *be) + return p.tstore.BlobSave(token, *be) } // castVoteSave saves a CastVoteDetails to the backend. -func (p *ticketVotePlugin) castVoteSave(treeID int64, cv ticketvote.CastVoteDetails) error { +func (p *ticketVotePlugin) castVoteSave(token []byte, cv ticketvote.CastVoteDetails) error { // Prepare blob be, err := convertBlobEntryFromCastVoteDetails(cv) if err != nil { @@ -1265,13 +1263,13 @@ func (p *ticketVotePlugin) castVoteSave(treeID int64, cv ticketvote.CastVoteDeta } // Save blob - return p.tstore.BlobSave(treeID, *be) + return p.tstore.BlobSave(token, *be) } // ballot casts the provided votes concurrently. The vote results are passed // back through the results channel to the calling function. This function // waits until all provided votes have been cast before returning. -func (p *ticketVotePlugin) ballot(treeID int64, votes []ticketvote.CastVote, results chan ticketvote.CastVoteReply) { +func (p *ticketVotePlugin) ballot(token []byte, votes []ticketvote.CastVote, results chan ticketvote.CastVoteReply) { // Cast the votes concurrently var wg sync.WaitGroup for _, v := range votes { @@ -1299,7 +1297,7 @@ func (p *ticketVotePlugin) ballot(treeID int64, votes []ticketvote.CastVote, res ) // Save cast vote - err := p.castVoteSave(treeID, cv) + err := p.castVoteSave(token, cv) if err == plugins.ErrDuplicateBlob { // This cast vote has already been saved. Its // possible that a previous attempt to vote @@ -1324,7 +1322,7 @@ func (p *ticketVotePlugin) ballot(treeID int64, votes []ticketvote.CastVote, res Token: v.Token, Ticket: v.Ticket, } - err = p.voteColliderSave(treeID, vc) + err = p.voteColliderSave(token, vc) if err != nil { t := time.Now().Unix() log.Errorf("cmdCastBallot: voteColliderSave %v: %v", t, err) @@ -1356,7 +1354,7 @@ func (p *ticketVotePlugin) ballot(treeID int64, votes []ticketvote.CastVote, res // cmdCastBallot casts a ballot of votes. This function will not return a user // error if one occurs for an individual vote. It will instead return the // ballot reply with the error included in the individual cast vote reply. -func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload string) (string, error) { +func (p *ticketVotePlugin) cmdCastBallot(token []byte, payload string) (string, error) { // Decode payload var cb ticketvote.CastBallot err := json.Unmarshal([]byte(payload), &cb) @@ -1621,7 +1619,7 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str log.Debugf("Casting %v votes in batch %v/%v", len(batch), i+1, len(queue)) - p.ballot(treeID, batch, results) + p.ballot(token, batch, results) } // Empty out the results channel @@ -1672,15 +1670,15 @@ func (p *ticketVotePlugin) cmdCastBallot(treeID int64, token []byte, payload str } // cmdDetails returns the vote details for a record. -func (p *ticketVotePlugin) cmdDetails(treeID int64, token []byte) (string, error) { +func (p *ticketVotePlugin) cmdDetails(token []byte) (string, error) { // Get vote authorizations - auths, err := p.auths(treeID) + auths, err := p.auths(token) if err != nil { return "", fmt.Errorf("auths: %v", err) } // Get vote details - vd, err := p.voteDetails(treeID) + vd, err := p.voteDetails(token) if err != nil { return "", fmt.Errorf("voteDetails: %v", err) } @@ -1700,9 +1698,9 @@ func (p *ticketVotePlugin) cmdDetails(treeID int64, token []byte) (string, error // cmdRunoffDetails is an internal plugin command that requests the details of // a runoff vote. -func (p *ticketVotePlugin) cmdRunoffDetails(treeID int64) (string, error) { +func (p *ticketVotePlugin) cmdRunoffDetails(token []byte) (string, error) { // Get start runoff record - srs, err := p.startRunoffRecord(treeID) + srs, err := p.startRunoffRecord(token) if err != nil { return "", err } @@ -1721,9 +1719,9 @@ func (p *ticketVotePlugin) cmdRunoffDetails(treeID int64) (string, error) { // cmdResults requests the vote objects of all votes that were cast in a ticket // vote. -func (p *ticketVotePlugin) cmdResults(treeID int64, token []byte) (string, error) { +func (p *ticketVotePlugin) cmdResults(token []byte) (string, error) { // Get vote results - votes, err := p.voteResults(treeID) + votes, err := p.voteResults(token) if err != nil { return "", err } @@ -1741,7 +1739,7 @@ func (p *ticketVotePlugin) cmdResults(treeID int64, token []byte) (string, error } // cmdSummary requests the vote summary for a record. -func (p *ticketVotePlugin) cmdSummary(treeID int64, token []byte) (string, error) { +func (p *ticketVotePlugin) cmdSummary(token []byte) (string, error) { // Get best block. This cmd does not write any data so we do not // have to use the safe best block. bb, err := p.bestBlockUnsafe() @@ -1750,7 +1748,7 @@ func (p *ticketVotePlugin) cmdSummary(treeID int64, token []byte) (string, error } // Get summary - sr, err := p.summary(treeID, token, bb) + sr, err := p.summary(token, bb) if err != nil { return "", fmt.Errorf("summary: %v", err) } @@ -1805,7 +1803,7 @@ func (p *ticketVotePlugin) cmdInventory(payload string) (string, error) { } // cmdTimestamps requests the timestamps for a ticket vote. -func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload string) (string, error) { +func (p *ticketVotePlugin) cmdTimestamps(token []byte, payload string) (string, error) { // Decode payload var t ticketvote.Timestamps err := json.Unmarshal([]byte(payload), &t) @@ -1823,11 +1821,11 @@ func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload str switch { case t.VotesPage > 0: // Return a page of vote timestamps - digests, err := p.tstore.DigestsByDataDesc(treeID, + digests, err := p.tstore.DigestsByDataDesc(token, []string{dataDescriptorCastVoteDetails}) if err != nil { - return "", fmt.Errorf("digestsByKeyPrefix %v %v: %v", - treeID, dataDescriptorVoteDetails, err) + return "", fmt.Errorf("digestsByKeyPrefix %x %v: %v", + token, dataDescriptorVoteDetails, err) } startAt := (t.VotesPage - 1) * pageSize @@ -1835,7 +1833,7 @@ func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload str if i < int(startAt) { continue } - ts, err := p.timestamp(treeID, v) + ts, err := p.timestamp(token, v) if err != nil { return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) @@ -1852,15 +1850,15 @@ func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload str // timestamp. // Auth timestamps - digests, err := p.tstore.DigestsByDataDesc(treeID, + digests, err := p.tstore.DigestsByDataDesc(token, []string{dataDescriptorAuthDetails}) if err != nil { - return "", fmt.Errorf("DigestByDataDesc %v %v: %v", - treeID, dataDescriptorAuthDetails, err) + return "", fmt.Errorf("DigestByDataDesc %x %v: %v", + token, dataDescriptorAuthDetails, err) } auths = make([]ticketvote.Timestamp, 0, len(digests)) for _, v := range digests { - ts, err := p.timestamp(treeID, v) + ts, err := p.timestamp(token, v) if err != nil { return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) @@ -1869,11 +1867,11 @@ func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload str } // Vote details timestamp - digests, err = p.tstore.DigestsByDataDesc(treeID, + digests, err = p.tstore.DigestsByDataDesc(token, []string{dataDescriptorVoteDetails}) if err != nil { - return "", fmt.Errorf("DigestsByDataDesc %v %v: %v", - treeID, dataDescriptorVoteDetails, err) + return "", fmt.Errorf("DigestsByDataDesc %x %v: %v", + token, dataDescriptorVoteDetails, err) } // There should never be more than a one vote details if len(digests) > 1 { @@ -1881,7 +1879,7 @@ func (p *ticketVotePlugin) cmdTimestamps(treeID int64, token []byte, payload str "got %v, want 1", len(digests)) } for _, v := range digests { - ts, err := p.timestamp(treeID, v) + ts, err := p.timestamp(token, v) if err != nil { return "", fmt.Errorf("timestamp %x %x: %v", token, v, err) @@ -1932,7 +1930,7 @@ func (p *ticketVotePlugin) cmdSubmissions(token []byte) (string, error) { } // authSave saves a AuthDetails to the backend. -func (p *ticketVotePlugin) authSave(treeID int64, ad ticketvote.AuthDetails) error { +func (p *ticketVotePlugin) authSave(token []byte, ad ticketvote.AuthDetails) error { // Prepare blob be, err := convertBlobEntryFromAuthDetails(ad) if err != nil { @@ -1940,13 +1938,13 @@ func (p *ticketVotePlugin) authSave(treeID int64, ad ticketvote.AuthDetails) err } // Save blob - return p.tstore.BlobSave(treeID, *be) + return p.tstore.BlobSave(token, *be) } // auths returns all AuthDetails for a record. -func (p *ticketVotePlugin) auths(treeID int64) ([]ticketvote.AuthDetails, error) { +func (p *ticketVotePlugin) auths(token []byte) ([]ticketvote.AuthDetails, error) { // Retrieve blobs - blobs, err := p.tstore.BlobsByDataDesc(treeID, + blobs, err := p.tstore.BlobsByDataDesc(token, []string{dataDescriptorAuthDetails}) if err != nil { return nil, err @@ -1972,7 +1970,7 @@ func (p *ticketVotePlugin) auths(treeID int64) ([]ticketvote.AuthDetails, error) } // voteDetailsSave saves a VoteDetails to the backend. -func (p *ticketVotePlugin) voteDetailsSave(treeID int64, vd ticketvote.VoteDetails) error { +func (p *ticketVotePlugin) voteDetailsSave(token []byte, vd ticketvote.VoteDetails) error { // Prepare blob be, err := convertBlobEntryFromVoteDetails(vd) if err != nil { @@ -1980,14 +1978,14 @@ func (p *ticketVotePlugin) voteDetailsSave(treeID int64, vd ticketvote.VoteDetai } // Save blob - return p.tstore.BlobSave(treeID, *be) + return p.tstore.BlobSave(token, *be) } // voteDetails returns the VoteDetails for a record. Nil is returned if a vote // details is not found. -func (p *ticketVotePlugin) voteDetails(treeID int64) (*ticketvote.VoteDetails, error) { +func (p *ticketVotePlugin) voteDetails(token []byte) (*ticketvote.VoteDetails, error) { // Retrieve blobs - blobs, err := p.tstore.BlobsByDataDesc(treeID, + blobs, err := p.tstore.BlobsByDataDesc(token, []string{dataDescriptorVoteDetails}) if err != nil { return nil, err @@ -2002,7 +2000,7 @@ func (p *ticketVotePlugin) voteDetails(treeID int64) (*ticketvote.VoteDetails, e // This should not happen. There should only ever be a max of // one vote details. return nil, fmt.Errorf("multiple vote details found (%v) on %x", - len(blobs), treeID) + len(blobs), token) } // Decode blob @@ -2031,13 +2029,13 @@ func (p *ticketVotePlugin) voteDetailsByToken(token []byte) (*ticketvote.VoteDet } // voteResults returns all votes that were cast in a ticket vote. -func (p *ticketVotePlugin) voteResults(treeID int64) ([]ticketvote.CastVoteDetails, error) { +func (p *ticketVotePlugin) voteResults(token []byte) ([]ticketvote.CastVoteDetails, error) { // Retrieve blobs desc := []string{ dataDescriptorCastVoteDetails, dataDescriptorVoteCollider, } - blobs, err := p.tstore.BlobsByDataDesc(treeID, desc) + blobs, err := p.tstore.BlobsByDataDesc(token, desc) if err != nil { return nil, err } @@ -2347,7 +2345,7 @@ func (p *ticketVotePlugin) summariesForRunoff(parentToken string) (map[string]ti } // summary returns the vote summary for a record. -func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) (*ticketvote.SummaryReply, error) { +func (p *ticketVotePlugin) summary(token []byte, bestBlock uint32) (*ticketvote.SummaryReply, error) { // Check if the summary has been cached s, err := p.summaryCache(hex.EncodeToString(token)) switch { @@ -2369,7 +2367,7 @@ func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) // Check if the vote has been authorized. Not all vote types // require an authorization. - auths, err := p.auths(treeID) + auths, err := p.auths(token) if err != nil { return nil, fmt.Errorf("auths: %v", err) } @@ -2392,7 +2390,7 @@ func (p *ticketVotePlugin) summary(treeID int64, token []byte, bestBlock uint32) } // Check if the vote has been started - vd, err := p.voteDetails(treeID) + vd, err := p.voteDetails(token) if err != nil { return nil, fmt.Errorf("startDetails: %v", err) } @@ -2499,11 +2497,11 @@ func (p *ticketVotePlugin) summaryByToken(token []byte) (*ticketvote.SummaryRepl } // timestamp returns the timestamp for a specific piece of data. -func (p *ticketVotePlugin) timestamp(treeID int64, digest []byte) (*ticketvote.Timestamp, error) { - t, err := p.tstore.Timestamp(treeID, digest) +func (p *ticketVotePlugin) timestamp(token []byte, digest []byte) (*ticketvote.Timestamp, error) { + t, err := p.tstore.Timestamp(token, digest) if err != nil { - return nil, fmt.Errorf("timestamp %v %x: %v", - treeID, digest, err) + return nil, fmt.Errorf("timestamp %x %x: %v", + token, digest, err) } // Convert response diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go index 2467a4b6b..40b7a48ad 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/hooks.go @@ -57,7 +57,7 @@ func (p *ticketVotePlugin) hookSetRecordStatusPre(payload string) error { // hookNewRecordPre caches plugin data from the tstore backend RecordSetStatus // method. -func (p *ticketVotePlugin) hookSetRecordStatusPost(treeID int64, payload string) error { +func (p *ticketVotePlugin) hookSetRecordStatusPost(payload string) error { var srs plugins.HookSetRecordStatus err := json.Unmarshal([]byte(payload), &srs) if err != nil { diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/internalcmds.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/internalcmds.go index 83983436d..c924bd834 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/internalcmds.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/internalcmds.go @@ -41,7 +41,7 @@ type startRunoffRecord struct { // startRunoffSubmission is an internal plugin command that is used to start // the voting period on a runoff vote submission. type startRunoffSubmission struct { - ParentTreeID int64 `json:"parenttreeid"` + ParentToken []byte `json:"parenttoken"` StartDetails ticketvote.StartDetails `json:"startdetails"` } diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go index e5e3e765d..422a79aad 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/ticketvote.go @@ -152,34 +152,34 @@ func (p *ticketVotePlugin) Setup() error { // Cmd executes a plugin command. // // This function satisfies the plugins PluginClient interface. -func (p *ticketVotePlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { - log.Tracef("ticketvote Cmd: %v %x %v %v", treeID, token, cmd, payload) +func (p *ticketVotePlugin) Cmd(token []byte, cmd, payload string) (string, error) { + log.Tracef("ticketvote Cmd: %x %v %v", token, cmd, payload) switch cmd { case ticketvote.CmdAuthorize: - return p.cmdAuthorize(treeID, token, payload) + return p.cmdAuthorize(token, payload) case ticketvote.CmdStart: - return p.cmdStart(treeID, token, payload) + return p.cmdStart(token, payload) case ticketvote.CmdCastBallot: - return p.cmdCastBallot(treeID, token, payload) + return p.cmdCastBallot(token, payload) case ticketvote.CmdDetails: - return p.cmdDetails(treeID, token) + return p.cmdDetails(token) case ticketvote.CmdResults: - return p.cmdResults(treeID, token) + return p.cmdResults(token) case ticketvote.CmdSummary: - return p.cmdSummary(treeID, token) + return p.cmdSummary(token) case ticketvote.CmdSubmissions: return p.cmdSubmissions(token) case ticketvote.CmdInventory: return p.cmdInventory(payload) case ticketvote.CmdTimestamps: - return p.cmdTimestamps(treeID, token, payload) + return p.cmdTimestamps(token, payload) // Internal plugin commands case cmdStartRunoffSubmission: - return p.cmdStartRunoffSubmission(treeID, token, payload) + return p.cmdStartRunoffSubmission(token, payload) case cmdRunoffDetails: - return p.cmdRunoffDetails(treeID) + return p.cmdRunoffDetails(token) } return "", backend.ErrPluginCmdInvalid @@ -188,8 +188,8 @@ func (p *ticketVotePlugin) Cmd(treeID int64, token []byte, cmd, payload string) // Hook executes a plugin hook. // // This function satisfies the plugins PluginClient interface. -func (p *ticketVotePlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { - log.Tracef("ticketvote Hook: %v %x %v", plugins.Hooks[h], token, treeID) +func (p *ticketVotePlugin) Hook(h plugins.HookT, payload string) error { + log.Tracef("ticketvote Hook: %v", plugins.Hooks[h]) switch h { case plugins.HookTypeNewRecordPre: @@ -199,7 +199,7 @@ func (p *ticketVotePlugin) Hook(treeID int64, token []byte, h plugins.HookT, pay case plugins.HookTypeSetRecordStatusPre: return p.hookSetRecordStatusPre(payload) case plugins.HookTypeSetRecordStatusPost: - return p.hookSetRecordStatusPost(treeID, payload) + return p.hookSetRecordStatusPost(payload) } return nil @@ -208,7 +208,7 @@ func (p *ticketVotePlugin) Hook(treeID int64, token []byte, h plugins.HookT, pay // Fsck performs a plugin filesystem check. // // This function satisfies the plugins PluginClient interface. -func (p *ticketVotePlugin) Fsck(treeIDs []int64) error { +func (p *ticketVotePlugin) Fsck() error { log.Tracef("ticketvote Fsck") // Verify all caches diff --git a/politeiad/backendv2/tstorebe/plugins/usermd/cmds.go b/politeiad/backendv2/tstorebe/plugins/usermd/cmds.go index 11a2555f4..ad8f3e16e 100644 --- a/politeiad/backendv2/tstorebe/plugins/usermd/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/usermd/cmds.go @@ -11,9 +11,9 @@ import ( ) // cmdAuthor returns the user ID of a record's author. -func (p *usermdPlugin) cmdAuthor(treeID int64) (string, error) { +func (p *usermdPlugin) cmdAuthor(token []byte) (string, error) { // Get user metadata - r, err := p.tstore.RecordPartial(treeID, 0, nil, true) + r, err := p.tstore.RecordPartial(token, 0, nil, true) if err != nil { return "", err } diff --git a/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go b/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go index 1fa512a8d..2328ec1eb 100644 --- a/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go +++ b/politeiad/backendv2/tstorebe/plugins/usermd/hooks.go @@ -132,7 +132,7 @@ func (p *usermdPlugin) hookSetRecordStatusPre(payload string) error { // hookNewRecordPre caches plugin data from the tstore backend RecordSetStatus // method. -func (p *usermdPlugin) hookSetRecordStatusPost(treeID int64, payload string) error { +func (p *usermdPlugin) hookSetRecordStatusPost(payload string) error { var srs plugins.HookSetRecordStatus err := json.Unmarshal([]byte(payload), &srs) if err != nil { diff --git a/politeiad/backendv2/tstorebe/plugins/usermd/usermd.go b/politeiad/backendv2/tstorebe/plugins/usermd/usermd.go index 28f3ee0d9..bb658ff23 100644 --- a/politeiad/backendv2/tstorebe/plugins/usermd/usermd.go +++ b/politeiad/backendv2/tstorebe/plugins/usermd/usermd.go @@ -44,12 +44,12 @@ func (p *usermdPlugin) Setup() error { // Cmd executes a plugin command. // // This function satisfies the plugins PluginClient interface. -func (p *usermdPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (string, error) { - log.Tracef("usermd Cmd: %v %x %v %v", treeID, token, cmd, payload) +func (p *usermdPlugin) Cmd(token []byte, cmd, payload string) (string, error) { + log.Tracef("usermd Cmd: %x %v %v", token, cmd, payload) switch cmd { case usermd.CmdAuthor: - return p.cmdAuthor(treeID) + return p.cmdAuthor(token) case usermd.CmdUserRecords: return p.cmdUserRecords(payload) } @@ -60,8 +60,8 @@ func (p *usermdPlugin) Cmd(treeID int64, token []byte, cmd, payload string) (str // Hook executes a plugin hook. // // This function satisfies the plugins PluginClient interface. -func (p *usermdPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload string) error { - log.Tracef("usermd Hook: %v %x %v", plugins.Hooks[h], token, treeID) +func (p *usermdPlugin) Hook(h plugins.HookT, payload string) error { + log.Tracef("usermd Hook: %v", plugins.Hooks[h]) switch h { case plugins.HookTypeNewRecordPre: @@ -75,7 +75,7 @@ func (p *usermdPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload case plugins.HookTypeSetRecordStatusPre: return p.hookSetRecordStatusPre(payload) case plugins.HookTypeSetRecordStatusPost: - return p.hookSetRecordStatusPost(treeID, payload) + return p.hookSetRecordStatusPost(payload) } return nil @@ -84,7 +84,7 @@ func (p *usermdPlugin) Hook(treeID int64, token []byte, h plugins.HookT, payload // Fsck performs a plugin filesystem check. // // This function satisfies the plugins PluginClient interface. -func (p *usermdPlugin) Fsck(treeIDs []int64) error { +func (p *usermdPlugin) Fsck() error { log.Tracef("usermd Fsck") return nil diff --git a/politeiad/backendv2/tstorebe/tstore/anchor.go b/politeiad/backendv2/tstorebe/tstore/anchor.go index ac7f2d945..aaefa695b 100644 --- a/politeiad/backendv2/tstorebe/tstore/anchor.go +++ b/politeiad/backendv2/tstorebe/tstore/anchor.go @@ -159,7 +159,7 @@ func (t *Tstore) anchorLatest(treeID int64) (*anchor, error) { // Get tree leaves leavesAll, err := t.tlog.LeavesAll(treeID) if err != nil { - return nil, fmt.Errorf("LeavesAll: %v", err) + return nil, err } // Find the most recent anchor leaf diff --git a/politeiad/backendv2/tstorebe/tstore/blobclient.go b/politeiad/backendv2/tstorebe/tstore/blobclient.go index 70d4bb608..12f9acc17 100644 --- a/politeiad/backendv2/tstorebe/tstore/blobclient.go +++ b/politeiad/backendv2/tstorebe/tstore/blobclient.go @@ -24,22 +24,18 @@ import ( // and can be used to get/del the blob from tstore. // // This function satisfies the plugins TstoreClient interface. -func (t *Tstore) BlobSave(treeID int64, be store.BlobEntry) error { - log.Tracef("BlobSave: %v", treeID) - - // Verify tree exists - if !t.TreeExists(treeID) { - return backend.ErrRecordNotFound - } +func (t *Tstore) BlobSave(token []byte, be store.BlobEntry) error { + log.Tracef("BlobSave: %x", token) // Verify tree is not frozen - leaves, err := t.tlog.LeavesAll(treeID) + treeID := treeIDFromToken(token) + leaves, err := t.leavesAll(treeID) if err != nil { - return fmt.Errorf("LeavesAll: %v", err) + return err } idx, err := t.recordIndexLatest(leaves) if err != nil { - return fmt.Errorf("recordIndexLatest: %v", err) + return err } if idx.Frozen { // The tree is frozen. The record is locked. @@ -121,22 +117,18 @@ func (t *Tstore) BlobSave(treeID int64, be store.BlobEntry) error { return nil } -// BlobsDel deletes the blobs that correspond to the provided digests. +// BlobsDel deletes the blobs that correspond to the provided digests. Blobs +// can be deleted from both frozen and non-frozen records. // // This function satisfies the plugins TstoreClient interface. -func (t *Tstore) BlobsDel(treeID int64, digests [][]byte) error { - log.Tracef("BlobsDel: %v %x", treeID, digests) - - // Verify tree exists. We allow blobs to be deleted from both - // frozen and non frozen trees. - if !t.TreeExists(treeID) { - return backend.ErrRecordNotFound - } +func (t *Tstore) BlobsDel(token []byte, digests [][]byte) error { + log.Tracef("BlobsDel: %x %x", token, digests) // Get all tree leaves - leaves, err := t.tlog.LeavesAll(treeID) + treeID := treeIDFromToken(token) + leaves, err := t.leavesAll(treeID) if err != nil { - return fmt.Errorf("LeavesAll: %v", err) + return err } // Put merkle leaf hashes into a map so that we can tell if a leaf @@ -176,22 +168,18 @@ func (t *Tstore) BlobsDel(treeID int64, digests [][]byte) error { // is vetted, only vetted blobs will be returned. // // This function satisfies the plugins TstoreClient interface. -func (t *Tstore) Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEntry, error) { - log.Tracef("Blobs: %v %x", treeID, digests) +func (t *Tstore) Blobs(token []byte, digests [][]byte) (map[string]store.BlobEntry, error) { + log.Tracef("Blobs: %x %x", token, digests) if len(digests) == 0 { return map[string]store.BlobEntry{}, nil } - // Verify tree exists - if !t.TreeExists(treeID) { - return nil, backend.ErrRecordNotFound - } - // Get leaves - leaves, err := t.tlog.LeavesAll(treeID) + treeID := treeIDFromToken(token) + leaves, err := t.leavesAll(treeID) if err != nil { - return nil, fmt.Errorf("LeavesAll: %v", err) + return nil, err } // Determine if the record is vetted. If the record is vetted, only @@ -264,18 +252,14 @@ func (t *Tstore) Blobs(treeID int64, digests [][]byte) (map[string]store.BlobEnt // only vetted blobs will be returned. // // This function satisfies the plugins TstoreClient interface. -func (t *Tstore) BlobsByDataDesc(treeID int64, dataDesc []string) ([]store.BlobEntry, error) { - log.Tracef("BlobsByDataDesc: %v %v", treeID, dataDesc) - - // Verify tree exists - if !t.TreeExists(treeID) { - return nil, backend.ErrRecordNotFound - } +func (t *Tstore) BlobsByDataDesc(token []byte, dataDesc []string) ([]store.BlobEntry, error) { + log.Tracef("BlobsByDataDesc: %x %v", token, dataDesc) // Get leaves - leaves, err := t.tlog.LeavesAll(treeID) + treeID := treeIDFromToken(token) + leaves, err := t.leavesAll(treeID) if err != nil { - return nil, fmt.Errorf("LeavesAll: %v", err) + return nil, err } // Find all matching leaves @@ -334,18 +318,14 @@ func (t *Tstore) BlobsByDataDesc(treeID int64, dataDesc []string) ([]store.BlobE // returned. // // This function satisfies the plugins TstoreClient interface. -func (t *Tstore) DigestsByDataDesc(treeID int64, dataDesc []string) ([][]byte, error) { - log.Tracef("DigestsByDataDesc: %v %v", treeID, dataDesc) - - // Verify tree exists - if !t.TreeExists(treeID) { - return nil, backend.ErrRecordNotFound - } +func (t *Tstore) DigestsByDataDesc(token []byte, dataDesc []string) ([][]byte, error) { + log.Tracef("DigestsByDataDesc: %x %v", token, dataDesc) // Get leaves - leaves, err := t.tlog.LeavesAll(treeID) + treeID := treeIDFromToken(token) + leaves, err := t.leavesAll(treeID) if err != nil { - return nil, fmt.Errorf("LeavesAll: %v", err) + return nil, err } // Find all matching leaves @@ -365,13 +345,14 @@ func (t *Tstore) DigestsByDataDesc(treeID int64, dataDesc []string) ([][]byte, e // returned. // // This function satisfies the plugins TstoreClient interface. -func (t *Tstore) Timestamp(treeID int64, digest []byte) (*backend.Timestamp, error) { - log.Tracef("Timestamp: %v %x", treeID, digest) +func (t *Tstore) Timestamp(token []byte, digest []byte) (*backend.Timestamp, error) { + log.Tracef("Timestamp: %x %x", token, digest) // Get tree leaves - leaves, err := t.tlog.LeavesAll(treeID) + treeID := treeIDFromToken(token) + leaves, err := t.leavesAll(treeID) if err != nil { - return nil, fmt.Errorf("LeavesAll: %v", err) + return nil, err } // Determine if the record is vetted diff --git a/politeiad/backendv2/tstorebe/tstore/plugin.go b/politeiad/backendv2/tstorebe/tstore/plugin.go index 4b1734e61..762c07186 100644 --- a/politeiad/backendv2/tstorebe/tstore/plugin.go +++ b/politeiad/backendv2/tstorebe/tstore/plugin.go @@ -134,13 +134,13 @@ func (t *Tstore) PluginSetup(pluginID string) error { // are executed prior to the tstore backend writing data to disk. These hooks // give plugins the opportunity to add plugin specific validation to record // methods or plugin commands that write data. -func (t *Tstore) PluginHookPre(treeID int64, token []byte, h plugins.HookT, payload string) error { - log.Tracef("PluginHookPre: %v %v", treeID, plugins.Hooks[h]) +func (t *Tstore) PluginHookPre(h plugins.HookT, payload string) error { + log.Tracef("PluginHookPre: %v", plugins.Hooks[h]) // Pass hook event and payload to each plugin for _, v := range t.pluginIDs() { p, _ := t.plugin(v) - err := p.client.Hook(treeID, token, h, payload) + err := p.client.Hook(h, payload) if err != nil { var e backend.PluginError if errors.As(err, &e) { @@ -156,8 +156,8 @@ func (t *Tstore) PluginHookPre(treeID int64, token []byte, h plugins.HookT, payl // PluginHookPre executes a tstore backend post hook. Post hooks are hooks that // are executed after the tstore backend successfully writes data to disk. // These hooks give plugins the opportunity to cache data from the write. -func (t *Tstore) PluginHookPost(treeID int64, token []byte, h plugins.HookT, payload string) { - log.Tracef("PluginHookPost: %v %v", treeID, plugins.Hooks[h]) +func (t *Tstore) PluginHookPost(h plugins.HookT, payload string) { + log.Tracef("PluginHookPost: %v", plugins.Hooks[h]) // Pass hook event and payload to each plugin for _, v := range t.pluginIDs() { @@ -166,21 +166,21 @@ func (t *Tstore) PluginHookPost(treeID int64, token []byte, h plugins.HookT, pay log.Errorf("%v PluginHookPost: plugin not found %v", v) continue } - err := p.client.Hook(treeID, token, h, payload) + err := p.client.Hook(h, payload) if err != nil { // This is the post plugin hook so the data has already been // saved to tstore. We do not have the ability to unwind. Log // the error and continue. - log.Criticalf("%v PluginHookPost %v %v %v %x %v: %v", - v, treeID, token, h, err, payload) + log.Criticalf("%v PluginHookPost %v %v: %v: %v", + v, h, err, payload) continue } } } // PluginCmd executes a plugin command. -func (t *Tstore) PluginCmd(treeID int64, token []byte, pluginID, cmd, payload string) (string, error) { - log.Tracef("PluginCmd: %v %x %v %v", treeID, token, pluginID, cmd) +func (t *Tstore) PluginCmd(token []byte, pluginID, cmd, payload string) (string, error) { + log.Tracef("PluginCmd: %x %v %v", token, pluginID, cmd) // Get plugin p, ok := t.plugin(pluginID) @@ -189,7 +189,7 @@ func (t *Tstore) PluginCmd(treeID int64, token []byte, pluginID, cmd, payload st } // Execute plugin command - return p.client.Cmd(treeID, token, cmd, payload) + return p.client.Cmd(token, cmd, payload) } // Plugins returns all registered plugins for the tstore instance. diff --git a/politeiad/backendv2/tstorebe/tstore/record.go b/politeiad/backendv2/tstorebe/tstore/record.go index 9e8928bfc..a4b7a482b 100644 --- a/politeiad/backendv2/tstorebe/tstore/record.go +++ b/politeiad/backendv2/tstorebe/tstore/record.go @@ -9,7 +9,6 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" - "errors" "fmt" backend "github.com/decred/politeia/politeiad/backendv2" @@ -28,11 +27,49 @@ const ( dataDescriptorAnchor = "pd-anchor-v1" ) -// recordBlobsSave saves the provided blobs to the kv store, appends a leaf -// to the trillian tree for each blob, and returns the record index for the -// blobs. -func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, recordMD backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) (*recordIndex, error) { - log.Tracef("recordBlobsSave: %v", treeID) +// RecordNew creates a new record in the tstore and returns the record token +// that serves as the unique identifier for the record. Creating a new record +// means creating a tlog tree for the record. Nothing is saved to the tree yet. +func (t *Tstore) RecordNew() ([]byte, error) { + tree, _, err := t.tlog.TreeNew() + if err != nil { + return nil, err + } + return tokenFromTreeID(tree.TreeId), nil +} + +// recordSave saves the provided record content to the kv store, appends a leaf +// to the trillian tree for each piece of content, then returns a record +// index for the newly saved record. If the record state is unvetted the record +// content will be saved to the key-value store encrypted. +// +// If the record is being made public the record version and iteration are both +// reset back to 1. This function detects when a record is being made public +// and re-saves any encrypted content that is part of the public record as +// clear text in the key-value store. +func (t *Tstore) recordSave(treeID int64, recordMD backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) (*recordIndex, error) { + // Get tree leaves + leavesAll, err := t.leavesAll(treeID) + if err != nil { + return nil, err + } + + // Get the existing record index + currIdx, err := t.recordIndexLatest(leavesAll) + if err == backend.ErrRecordNotFound { + // No record versions exist yet. This is ok. + currIdx = &recordIndex{ + Metadata: make(map[string]map[uint32][]byte), + Files: make(map[string][]byte), + } + } else if err != nil { + return nil, fmt.Errorf("recordIndexLatest: %v", err) + } + + // Verify tree is not frozen + if currIdx.Frozen { + return nil, backend.ErrRecordLocked + } // Verify there are no duplicate metadata streams md := make(map[string]map[uint32]struct{}, len(metadata)) @@ -379,57 +416,17 @@ func (t *Tstore) recordBlobsSave(treeID int64, leavesAll []*trillian.LogLeaf, re return &idx, nil } -func (t *Tstore) recordSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) (*recordIndex, error) { - // Verify tree exists - if !t.TreeExists(treeID) { - return nil, backend.ErrRecordNotFound - } - - // Get tree leaves - leavesAll, err := t.tlog.LeavesAll(treeID) - if err != nil { - return nil, fmt.Errorf("LeavesAll %v: %v", treeID, err) - } - - // Get the existing record index - currIdx, err := t.recordIndexLatest(leavesAll) - if errors.Is(err, backend.ErrRecordNotFound) { - // No record versions exist yet. This is ok. - currIdx = &recordIndex{ - Metadata: make(map[string]map[uint32][]byte), - Files: make(map[string][]byte), - } - } else if err != nil { - return nil, fmt.Errorf("recordIndexLatest: %v", err) - } - - // Verify tree is not frozen - if currIdx.Frozen { - return nil, backend.ErrRecordLocked - } - - // Save the record - idx, err := t.recordBlobsSave(treeID, leavesAll, rm, metadata, files) - if err != nil { - if err == backend.ErrNoRecordChanges { - return nil, err - } - return nil, fmt.Errorf("recordBlobsSave: %v", err) - } - - return idx, nil -} - // RecordSave saves the provided record to tstore. Once the record contents // have been successfully saved to tstore, a recordIndex is created for this // version of the record and saved to tstore as well. The record update is not // considered to be valid until the record index has been successfully saved. // If the record content makes it in but the record index does not, the record // content blobs are orphaned and ignored. -func (t *Tstore) RecordSave(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { - log.Tracef("RecordSave: %v", treeID) +func (t *Tstore) RecordSave(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { + log.Tracef("RecordSave: %x", token) // Save the record + treeID := treeIDFromToken(token) idx, err := t.recordSave(treeID, rm, metadata, files) if err != nil { return err @@ -447,16 +444,12 @@ func (t *Tstore) RecordSave(treeID int64, rm backend.RecordMetadata, metadata [] // RecordDel walks the provided tree and deletes all blobs in the store that // correspond to record files. This is done for all versions and all iterations // of the record. Record metadata and metadata stream blobs are not deleted. -func (t *Tstore) RecordDel(treeID int64) error { - log.Tracef("RecordDel: %v", treeID) - - // Verify tree exists - if !t.TreeExists(treeID) { - return backend.ErrRecordNotFound - } +func (t *Tstore) RecordDel(token []byte) error { + log.Tracef("RecordDel: %x", token) // Get all tree leaves - leavesAll, err := t.tlog.LeavesAll(treeID) + treeID := treeIDFromToken(token) + leavesAll, err := t.leavesAll(treeID) if err != nil { return err } @@ -516,10 +509,11 @@ func (t *Tstore) RecordDel(treeID int64) error { // anchored, the tstore fsck function will update the status of the tree to // frozen in trillian, at which point trillian will prevent any changes to the // tree. -func (t *Tstore) RecordFreeze(treeID int64, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { - log.Tracef("RecordFreeze: %v", treeID) +func (t *Tstore) RecordFreeze(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { + log.Tracef("RecordFreeze: %x", token) // Save updated record + treeID := treeIDFromToken(token) idx, err := t.recordSave(treeID, rm, metadata, files) if err != nil { return err @@ -532,6 +526,30 @@ func (t *Tstore) RecordFreeze(treeID int64, rm backend.RecordMetadata, metadata return t.recordIndexSave(treeID, *idx) } +// RecordExists returns whether a record exists. +// +// This method only returns whether a tree exists for the provided token. It's +// possible for a tree to exist that does not correspond to a record in the +// rare case that a tree was created but an unexpected error, such as a network +// error, was encoutered prior to the record being saved to the tree. We ignore +// this edge case because: +// +// 1. A user has no way to obtain this token unless the trillian instance has +// been opened to the public. +// +// 2. Even if they have the token they cannot do anything with it. Any attempt +// to read from the tree or write to the tree will return a RecordNotFound +// error. +// +// Pulling the leaves from the tree to see if a record has been saved to the +// tree adds a large amount of overhead to this call, which should be a very +// light weight. Its for this reason that we rely on the tree exists call +// despite the edge case. +func (t *Tstore) RecordExists(token []byte) bool { + _, err := t.tlog.Tree(treeIDFromToken(token)) + return err == nil +} + // record returns the specified record. // // Version is used to request a specific version of a record. If no version is @@ -543,15 +561,10 @@ func (t *Tstore) RecordFreeze(treeID int64, rm backend.RecordMetadata, metadata // OmitAllFiles can be used to retrieve a record without any of the record // files. This supersedes the filenames argument. func (t *Tstore) record(treeID int64, version uint32, filenames []string, omitAllFiles bool) (*backend.Record, error) { - // Verify tree exists - if !t.TreeExists(treeID) { - return nil, backend.ErrRecordNotFound - } - // Get tree leaves - leaves, err := t.tlog.LeavesAll(treeID) + leaves, err := t.leavesAll(treeID) if err != nil { - return nil, fmt.Errorf("LeavesAll %v: %v", treeID, err) + return nil, err } // Use the record index to pull the record content from the store. @@ -713,16 +726,18 @@ func (t *Tstore) record(treeID int64, version uint32, filenames []string, omitAl } // Record returns the specified version of the record. -func (t *Tstore) Record(treeID int64, version uint32) (*backend.Record, error) { - log.Tracef("Record: %v %v", treeID, version) +func (t *Tstore) Record(token []byte, version uint32) (*backend.Record, error) { + log.Tracef("Record: %x %v", token, version) + treeID := treeIDFromToken(token) return t.record(treeID, version, []string{}, false) } // RecordLatest returns the latest version of a record. -func (t *Tstore) RecordLatest(treeID int64) (*backend.Record, error) { - log.Tracef("RecordLatest: %v", treeID) +func (t *Tstore) RecordLatest(token []byte) (*backend.Record, error) { + log.Tracef("RecordLatest: %x", token) + treeID := treeIDFromToken(token) return t.record(treeID, 0, []string{}, false) } @@ -738,24 +753,25 @@ func (t *Tstore) RecordLatest(treeID int64) (*backend.Record, error) { // // OmitAllFiles can be used to retrieve a record without any of the record // files. This supersedes the filenames argument. -func (t *Tstore) RecordPartial(treeID int64, version uint32, filenames []string, omitAllFiles bool) (*backend.Record, error) { - log.Tracef("RecordPartial: %v %v %v %v", - treeID, version, omitAllFiles, filenames) +func (t *Tstore) RecordPartial(token []byte, version uint32, filenames []string, omitAllFiles bool) (*backend.Record, error) { + log.Tracef("RecordPartial: %x %v %v %v", + token, version, omitAllFiles, filenames) + treeID := treeIDFromToken(token) return t.record(treeID, version, filenames, omitAllFiles) } // RecordState returns the state of a record. This call does not require // retrieving any blobs from the kv store. The record state can be derived from // only the tlog leaves. -func (t *Tstore) RecordState(treeID int64) (backend.StateT, error) { - log.Tracef("RecordState: %v", treeID) +func (t *Tstore) RecordState(token []byte) (backend.StateT, error) { + log.Tracef("RecordState: %x", token) - leaves, err := t.tlog.LeavesAll(treeID) + treeID := treeIDFromToken(token) + leaves, err := t.leavesAll(treeID) if err != nil { - return 0, err + return backend.StateInvalid, err } - if recordIsVetted(leaves) { return backend.StateVetted, nil } @@ -763,21 +779,156 @@ func (t *Tstore) RecordState(treeID int64) (backend.StateT, error) { return backend.StateUnvetted, nil } +// timestamp returns the timestamp given a tlog tree merkle leaf hash. +func (t *Tstore) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trillian.LogLeaf) (*backend.Timestamp, error) { + // Find the leaf + var l *trillian.LogLeaf + for _, v := range leaves { + if bytes.Equal(merkleLeafHash, v.MerkleLeafHash) { + l = v + break + } + } + if l == nil { + return nil, fmt.Errorf("leaf not found") + } + + // Get blob entry from the kv store + ed, err := extraDataDecode(l.ExtraData) + if err != nil { + return nil, err + } + blobs, err := t.store.Get([]string{ed.storeKey()}) + if err != nil { + return nil, fmt.Errorf("store get: %v", err) + } + + // Extract the data blob. Its possible for the data blob to not + // exist if it has been censored. This is ok. We'll still return + // the rest of the timestamp. + var data []byte + if len(blobs) == 1 { + b, ok := blobs[ed.storeKey()] + if !ok { + return nil, fmt.Errorf("blob not found %v", ed.storeKey()) + } + be, err := store.Deblob(b) + if err != nil { + return nil, err + } + data, err = base64.StdEncoding.DecodeString(be.Data) + if err != nil { + return nil, err + } + // Sanity check + if !bytes.Equal(l.LeafValue, util.Digest(data)) { + return nil, fmt.Errorf("data digest does not match leaf value") + } + } + + // Setup timestamp + ts := backend.Timestamp{ + Data: string(data), + Digest: hex.EncodeToString(l.LeafValue), + Proofs: []backend.Proof{}, + } + + // Get the anchor record for this leaf + a, err := t.anchorForLeaf(treeID, merkleLeafHash, leaves) + if err != nil { + if err == errAnchorNotFound { + // This data has not been anchored yet + return &ts, nil + } + return nil, fmt.Errorf("anchor: %v", err) + } + + // Get trillian inclusion proof + p, err := t.tlog.InclusionProof(treeID, l.MerkleLeafHash, a.LogRoot) + if err != nil { + return nil, fmt.Errorf("InclusionProof %v %x: %v", + treeID, l.MerkleLeafHash, err) + } + + // Setup proof for data digest inclusion in the log merkle root + edt := ExtraDataTrillianRFC6962{ + LeafIndex: p.LeafIndex, + TreeSize: int64(a.LogRoot.TreeSize), + } + extraData, err := json.Marshal(edt) + if err != nil { + return nil, err + } + merklePath := make([]string, 0, len(p.Hashes)) + for _, v := range p.Hashes { + merklePath = append(merklePath, hex.EncodeToString(v)) + } + trillianProof := backend.Proof{ + Type: ProofTypeTrillianRFC6962, + Digest: ts.Digest, + MerkleRoot: hex.EncodeToString(a.LogRoot.RootHash), + MerklePath: merklePath, + ExtraData: string(extraData), + } + + // Setup proof for log merkle root inclusion in the dcrtime merkle + // root + if a.VerifyDigest.Digest != trillianProof.MerkleRoot { + return nil, fmt.Errorf("trillian merkle root not anchored") + } + var ( + numLeaves = a.VerifyDigest.ChainInformation.MerklePath.NumLeaves + hashes = a.VerifyDigest.ChainInformation.MerklePath.Hashes + flags = a.VerifyDigest.ChainInformation.MerklePath.Flags + ) + edd := ExtraDataDcrtime{ + NumLeaves: numLeaves, + Flags: base64.StdEncoding.EncodeToString(flags), + } + extraData, err = json.Marshal(edd) + if err != nil { + return nil, err + } + merklePath = make([]string, 0, len(hashes)) + for _, v := range hashes { + merklePath = append(merklePath, hex.EncodeToString(v[:])) + } + dcrtimeProof := backend.Proof{ + Type: ProofTypeDcrtime, + Digest: a.VerifyDigest.Digest, + MerkleRoot: a.VerifyDigest.ChainInformation.MerkleRoot, + MerklePath: merklePath, + ExtraData: string(extraData), + } + + // Update timestamp + ts.TxID = a.VerifyDigest.ChainInformation.Transaction + ts.MerkleRoot = a.VerifyDigest.ChainInformation.MerkleRoot + ts.Proofs = []backend.Proof{ + trillianProof, + dcrtimeProof, + } + + // Verify timestamp + err = VerifyTimestamp(ts) + if err != nil { + return nil, fmt.Errorf("VerifyTimestamp: %v", err) + } + + return &ts, nil +} + // RecordTimestamps returns the timestamps for the contents of a record. // Timestamps for the record metadata, metadata streams, and files are all // returned. -func (t *Tstore) RecordTimestamps(treeID int64, version uint32, token []byte) (*backend.RecordTimestamps, error) { - log.Tracef("RecordTimestamps: %v %v", treeID, version) - - // Verify tree exists - if !t.TreeExists(treeID) { - return nil, backend.ErrRecordNotFound - } +func (t *Tstore) RecordTimestamps(token []byte, version uint32) (*backend.RecordTimestamps, error) { + log.Tracef("RecordTimestamps: %x %v", token, version) // Get record index - leaves, err := t.tlog.LeavesAll(treeID) + treeID := treeIDFromToken(token) + leaves, err := t.leavesAll(treeID) if err != nil { - return nil, fmt.Errorf("LeavesAll %v: %v", treeID, err) + return nil, err } idx, err := t.recordIndex(leaves, version) if err != nil { @@ -825,6 +976,23 @@ func (t *Tstore) RecordTimestamps(treeID int64, version uint32, token []byte) (* }, nil } +// Inventory returns all record tokens that are in the tstore. Its possible for +// a token to be returned that does not correspond to an actual record. For +// example, if the tlog tree was created but saving the record to the tree +// failed due to an unexpected error then a empty tree with exist. This +// function does not filter those tokens out. +func (t *Tstore) Inventory() ([][]byte, error) { + trees, err := t.tlog.TreesAll() + if err != nil { + return nil, err + } + tokens := make([][]byte, 0, len(trees)) + for _, v := range trees { + tokens = append(tokens, tokenFromTreeID(v.TreeId)) + } + return tokens, nil +} + // recordIsVetted returns whether the provided leaves contain any vetted record // indexes. The presence of a vetted record index means the record is vetted. // The state of a record index is saved to the leaf extra data, which is how we diff --git a/politeiad/backendv2/tstorebe/tstore/tlog.go b/politeiad/backendv2/tstorebe/tstore/tlog.go new file mode 100644 index 000000000..6b14231d2 --- /dev/null +++ b/politeiad/backendv2/tstorebe/tstore/tlog.go @@ -0,0 +1,28 @@ +// Copyright (c) 2020-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package tstore + +import ( + "fmt" + + backend "github.com/decred/politeia/politeiad/backendv2" + "github.com/google/trillian" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// leavesAll provides a wrapper around the tlog LeavesAll method that unpacks +// any tree not found errors and instead returns a backend ErrRecordNotFound +// error. +func (t *Tstore) leavesAll(treeID int64) ([]*trillian.LogLeaf, error) { + leaves, err := t.tlog.LeavesAll(treeID) + if err != nil { + if c := status.Code(err); c == codes.NotFound { + return nil, backend.ErrRecordNotFound + } + return nil, fmt.Errorf("LeavesAll: %v", err) + } + return leaves, nil +} diff --git a/politeiad/backendv2/tstorebe/tstore/tlogclient.go b/politeiad/backendv2/tstorebe/tstore/tlogclient.go index c510082f6..03c024971 100644 --- a/politeiad/backendv2/tstorebe/tstore/tlogclient.go +++ b/politeiad/backendv2/tstorebe/tstore/tlogclient.go @@ -222,7 +222,7 @@ func (t *tclient) Tree(treeID int64) (*trillian.Tree, error) { TreeId: treeID, }) if err != nil { - return nil, fmt.Errorf("GetTree: %v", err) + return nil, err } if tree.TreeId != treeID { // Sanity check @@ -472,7 +472,7 @@ func (t *tclient) LeavesAll(treeID int64) ([]*trillian.LogLeaf, error) { // Get tree tree, err := t.Tree(treeID) if err != nil { - return nil, fmt.Errorf("tree: %v", err) + return nil, err } // Get signed log root diff --git a/politeiad/backendv2/tstorebe/tstore/trees.go b/politeiad/backendv2/tstorebe/tstore/trees.go deleted file mode 100644 index a6c5db943..000000000 --- a/politeiad/backendv2/tstorebe/tstore/trees.go +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright (c) 2020-2021 The Decred developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package tstore - -import ( - "bytes" - "encoding/base64" - "encoding/hex" - "encoding/json" - "fmt" - - backend "github.com/decred/politeia/politeiad/backendv2" - "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" - "github.com/decred/politeia/util" - "github.com/google/trillian" -) - -// TreeNew creates a new tlog tree and returns the tree ID. -func (t *Tstore) TreeNew() (int64, error) { - log.Tracef("TreeNew") - - tree, _, err := t.tlog.TreeNew() - if err != nil { - return 0, err - } - - return tree.TreeId, nil -} - -// TreesAll returns the IDs of all trees in the tstore instance. -func (t *Tstore) TreesAll() ([]int64, error) { - trees, err := t.tlog.TreesAll() - if err != nil { - return nil, err - } - treeIDs := make([]int64, 0, len(trees)) - for _, v := range trees { - treeIDs = append(treeIDs, v.TreeId) - } - return treeIDs, nil -} - -// TreeExists returns whether a tree exists in the trillian log. A tree -// existing doesn't necessarily mean that a record exists. Its possible for a -// tree to have been created but experienced an unexpected error prior to the -// record being saved. -func (t *Tstore) TreeExists(treeID int64) bool { - _, err := t.tlog.Tree(treeID) - return err == nil -} - -// timestamp returns the timestamp given a tlog tree merkle leaf hash. -func (t *Tstore) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trillian.LogLeaf) (*backend.Timestamp, error) { - // Find the leaf - var l *trillian.LogLeaf - for _, v := range leaves { - if bytes.Equal(merkleLeafHash, v.MerkleLeafHash) { - l = v - break - } - } - if l == nil { - return nil, fmt.Errorf("leaf not found") - } - - // Get blob entry from the kv store - ed, err := extraDataDecode(l.ExtraData) - if err != nil { - return nil, err - } - blobs, err := t.store.Get([]string{ed.storeKey()}) - if err != nil { - return nil, fmt.Errorf("store get: %v", err) - } - - // Extract the data blob. Its possible for the data blob to not - // exist if it has been censored. This is ok. We'll still return - // the rest of the timestamp. - var data []byte - if len(blobs) == 1 { - b, ok := blobs[ed.storeKey()] - if !ok { - return nil, fmt.Errorf("blob not found %v", ed.storeKey()) - } - be, err := store.Deblob(b) - if err != nil { - return nil, err - } - data, err = base64.StdEncoding.DecodeString(be.Data) - if err != nil { - return nil, err - } - // Sanity check - if !bytes.Equal(l.LeafValue, util.Digest(data)) { - return nil, fmt.Errorf("data digest does not match leaf value") - } - } - - // Setup timestamp - ts := backend.Timestamp{ - Data: string(data), - Digest: hex.EncodeToString(l.LeafValue), - Proofs: []backend.Proof{}, - } - - // Get the anchor record for this leaf - a, err := t.anchorForLeaf(treeID, merkleLeafHash, leaves) - if err != nil { - if err == errAnchorNotFound { - // This data has not been anchored yet - return &ts, nil - } - return nil, fmt.Errorf("anchor: %v", err) - } - - // Get trillian inclusion proof - p, err := t.tlog.InclusionProof(treeID, l.MerkleLeafHash, a.LogRoot) - if err != nil { - return nil, fmt.Errorf("InclusionProof %v %x: %v", - treeID, l.MerkleLeafHash, err) - } - - // Setup proof for data digest inclusion in the log merkle root - edt := ExtraDataTrillianRFC6962{ - LeafIndex: p.LeafIndex, - TreeSize: int64(a.LogRoot.TreeSize), - } - extraData, err := json.Marshal(edt) - if err != nil { - return nil, err - } - merklePath := make([]string, 0, len(p.Hashes)) - for _, v := range p.Hashes { - merklePath = append(merklePath, hex.EncodeToString(v)) - } - trillianProof := backend.Proof{ - Type: ProofTypeTrillianRFC6962, - Digest: ts.Digest, - MerkleRoot: hex.EncodeToString(a.LogRoot.RootHash), - MerklePath: merklePath, - ExtraData: string(extraData), - } - - // Setup proof for log merkle root inclusion in the dcrtime merkle - // root - if a.VerifyDigest.Digest != trillianProof.MerkleRoot { - return nil, fmt.Errorf("trillian merkle root not anchored") - } - var ( - numLeaves = a.VerifyDigest.ChainInformation.MerklePath.NumLeaves - hashes = a.VerifyDigest.ChainInformation.MerklePath.Hashes - flags = a.VerifyDigest.ChainInformation.MerklePath.Flags - ) - edd := ExtraDataDcrtime{ - NumLeaves: numLeaves, - Flags: base64.StdEncoding.EncodeToString(flags), - } - extraData, err = json.Marshal(edd) - if err != nil { - return nil, err - } - merklePath = make([]string, 0, len(hashes)) - for _, v := range hashes { - merklePath = append(merklePath, hex.EncodeToString(v[:])) - } - dcrtimeProof := backend.Proof{ - Type: ProofTypeDcrtime, - Digest: a.VerifyDigest.Digest, - MerkleRoot: a.VerifyDigest.ChainInformation.MerkleRoot, - MerklePath: merklePath, - ExtraData: string(extraData), - } - - // Update timestamp - ts.TxID = a.VerifyDigest.ChainInformation.Transaction - ts.MerkleRoot = a.VerifyDigest.ChainInformation.MerkleRoot - ts.Proofs = []backend.Proof{ - trillianProof, - dcrtimeProof, - } - - // Verify timestamp - err = VerifyTimestamp(ts) - if err != nil { - return nil, fmt.Errorf("VerifyTimestamp: %v", err) - } - - return &ts, nil -} diff --git a/politeiad/backendv2/tstorebe/tstore/tstore.go b/politeiad/backendv2/tstorebe/tstore/tstore.go index e8fe94650..859b8d4a3 100644 --- a/politeiad/backendv2/tstorebe/tstore/tstore.go +++ b/politeiad/backendv2/tstorebe/tstore/tstore.go @@ -5,6 +5,7 @@ package tstore import ( + "encoding/binary" "fmt" "net/url" "os" @@ -74,6 +75,16 @@ type Tstore struct { droppingAnchor bool } +func tokenFromTreeID(treeID int64) []byte { + b := make([]byte, 8) + binary.LittleEndian.PutUint64(b, uint64(treeID)) + return b +} + +func treeIDFromToken(token []byte) int64 { + return int64(binary.LittleEndian.Uint64(token)) +} + // Fsck performs a filesystem check on the tstore. func (t *Tstore) Fsck() { // Set tree status to frozen for any trees that are frozen and have diff --git a/politeiad/backendv2/tstorebe/tstorebe.go b/politeiad/backendv2/tstorebe/tstorebe.go index 08d8dc208..ce080699e 100644 --- a/politeiad/backendv2/tstorebe/tstorebe.go +++ b/politeiad/backendv2/tstorebe/tstorebe.go @@ -7,7 +7,6 @@ package tstorebe import ( "bytes" "encoding/base64" - "encoding/binary" "encoding/hex" "encoding/json" "fmt" @@ -145,21 +144,6 @@ func tokenIsFullLength(token []byte) bool { return util.TokenIsFullLength(util.TokenTypeTstore, token) } -func tokenFromTreeID(treeID int64) []byte { - b := make([]byte, 8) - // Converting between int64 and uint64 doesn't change - // the sign bit, only the way it's interpreted. - binary.LittleEndian.PutUint64(b, uint64(treeID)) - return b -} - -func treeIDFromToken(token []byte) int64 { - if !tokenIsFullLength(token) { - return 0 - } - return int64(binary.LittleEndian.Uint64(token)) -} - // metadataStreamsVerify verifies that all provided metadata streams are sane. func metadataStreamsVerify(metadata []backend.MetadataStream) error { // Verify metadata @@ -447,21 +431,18 @@ func (t *tstoreBackend) RecordNew(metadata []backend.MetadataStream, files []bac if err != nil { return nil, err } - err = t.tstore.PluginHookPre(0, []byte{}, - plugins.HookTypeNewRecordPre, string(b)) + err = t.tstore.PluginHookPre(plugins.HookTypeNewRecordPre, string(b)) if err != nil { return nil, err } // Create a new token var token []byte - var treeID int64 for retries := 0; retries < 10; retries++ { - treeID, err = t.tstore.TreeNew() + token, err = t.tstore.RecordNew() if err != nil { return nil, err } - token = tokenFromTreeID(treeID) // Check for shortened token collisions if t.tokenCollision(token) { @@ -485,7 +466,7 @@ func (t *tstoreBackend) RecordNew(metadata []backend.MetadataStream, files []bac } // Save the record - err = t.tstore.RecordSave(treeID, *rm, metadata, files) + err = t.tstore.RecordSave(token, *rm, metadata, files) if err != nil { return nil, fmt.Errorf("RecordSave: %v", err) } @@ -500,16 +481,15 @@ func (t *tstoreBackend) RecordNew(metadata []backend.MetadataStream, files []bac if err != nil { return nil, err } - t.tstore.PluginHookPost(treeID, token, - plugins.HookTypeNewRecordPost, string(b)) + t.tstore.PluginHookPost(plugins.HookTypeNewRecordPost, string(b)) // Update the inventory cache t.inventoryAdd(backend.StateUnvetted, token, backend.StatusUnreviewed) // Get the full record to return - r, err := t.tstore.RecordLatest(treeID) + r, err := t.tstore.RecordLatest(token) if err != nil { - return nil, fmt.Errorf("RecordLatest %v: %v", treeID, err) + return nil, fmt.Errorf("RecordLatest %x: %v", token, err) } return r, nil @@ -555,10 +535,9 @@ func (t *tstoreBackend) RecordEdit(token []byte, mdAppend, mdOverwrite []backend defer m.Unlock() // Get existing record - treeID := treeIDFromToken(token) - r, err := t.tstore.RecordLatest(treeID) + r, err := t.tstore.RecordLatest(token) if err != nil { - return nil, fmt.Errorf("RecordLatest %v: %v", treeID, err) + return nil, fmt.Errorf("RecordLatest: %v", err) } // Apply changes @@ -592,14 +571,13 @@ func (t *tstoreBackend) RecordEdit(token []byte, mdAppend, mdOverwrite []backend if err != nil { return nil, err } - err = t.tstore.PluginHookPre(treeID, token, - plugins.HookTypeEditRecordPre, string(b)) + err = t.tstore.PluginHookPre(plugins.HookTypeEditRecordPre, string(b)) if err != nil { return nil, err } // Save record - err = t.tstore.RecordSave(treeID, *recordMD, metadata, files) + err = t.tstore.RecordSave(token, *recordMD, metadata, files) if err != nil { switch err { case backend.ErrRecordLocked: @@ -610,13 +588,12 @@ func (t *tstoreBackend) RecordEdit(token []byte, mdAppend, mdOverwrite []backend } // Call post plugin hooks - t.tstore.PluginHookPost(treeID, token, - plugins.HookTypeEditRecordPost, string(b)) + t.tstore.PluginHookPost(plugins.HookTypeEditRecordPost, string(b)) // Return updated record - r, err = t.tstore.RecordLatest(treeID) + r, err = t.tstore.RecordLatest(token) if err != nil { - return nil, fmt.Errorf("RecordLatest %v: %v", treeID, err) + return nil, fmt.Errorf("RecordLatest: %v", err) } return r, nil @@ -662,10 +639,9 @@ func (t *tstoreBackend) RecordEditMetadata(token []byte, mdAppend, mdOverwrite [ defer m.Unlock() // Get existing record - treeID := treeIDFromToken(token) - r, err := t.tstore.RecordLatest(treeID) + r, err := t.tstore.RecordLatest(token) if err != nil { - return nil, fmt.Errorf("RecordLatest %v: %v", treeID, err) + return nil, fmt.Errorf("RecordLatest: %v", err) } // Apply changes. The version is not incremented for metadata only @@ -689,14 +665,13 @@ func (t *tstoreBackend) RecordEditMetadata(token []byte, mdAppend, mdOverwrite [ if err != nil { return nil, err } - err = t.tstore.PluginHookPre(treeID, token, - plugins.HookTypeEditMetadataPre, string(b)) + err = t.tstore.PluginHookPre(plugins.HookTypeEditMetadataPre, string(b)) if err != nil { return nil, err } // Update metadata - err = t.tstore.RecordSave(treeID, *recordMD, metadata, r.Files) + err = t.tstore.RecordSave(token, *recordMD, metadata, r.Files) if err != nil { switch err { case backend.ErrRecordLocked, backend.ErrNoRecordChanges: @@ -707,13 +682,12 @@ func (t *tstoreBackend) RecordEditMetadata(token []byte, mdAppend, mdOverwrite [ } // Call post plugin hooks - t.tstore.PluginHookPost(treeID, token, - plugins.HookTypeEditMetadataPost, string(b)) + t.tstore.PluginHookPost(plugins.HookTypeEditMetadataPost, string(b)) // Return updated record - r, err = t.tstore.RecordLatest(treeID) + r, err = t.tstore.RecordLatest(token) if err != nil { - return nil, fmt.Errorf("RecordLatest %v: %v", treeID, err) + return nil, fmt.Errorf("RecordLatest: %v", err) } return r, nil @@ -754,19 +728,17 @@ func statusChangeIsAllowed(from, to backend.StatusT) bool { // // This function must be called WITH the record lock held. func (t *tstoreBackend) setStatusPublic(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { - treeID := treeIDFromToken(token) - return t.tstore.RecordSave(treeID, rm, metadata, files) + return t.tstore.RecordSave(token, rm, metadata, files) } // setStatusArchived updates the status of a record to archived. // // This function must be called WITH the record lock held. func (t *tstoreBackend) setStatusArchived(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { - // Freeze the tree - treeID := treeIDFromToken(token) - err := t.tstore.RecordFreeze(treeID, rm, metadata, files) + // Freeze record + err := t.tstore.RecordFreeze(token, rm, metadata, files) if err != nil { - return fmt.Errorf("RecordFreeze %v: %v", treeID, err) + return fmt.Errorf("RecordFreeze: %v", err) } log.Debugf("Record frozen %x", token) @@ -781,18 +753,17 @@ func (t *tstoreBackend) setStatusArchived(token []byte, rm backend.RecordMetadat // This function must be called WITH the record lock held. func (t *tstoreBackend) setStatusCensored(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { // Freeze the tree - treeID := treeIDFromToken(token) - err := t.tstore.RecordFreeze(treeID, rm, metadata, files) + err := t.tstore.RecordFreeze(token, rm, metadata, files) if err != nil { - return fmt.Errorf("RecordFreeze %v: %v", treeID, err) + return fmt.Errorf("RecordFreeze: %v", err) } log.Debugf("Record frozen %x", token) // Delete all record files - err = t.tstore.RecordDel(treeID) + err = t.tstore.RecordDel(token) if err != nil { - return fmt.Errorf("RecordDel %v: %v", treeID, err) + return fmt.Errorf("RecordDel: %v", err) } log.Debugf("Record contents deleted %x", token) @@ -827,8 +798,7 @@ func (t *tstoreBackend) RecordSetStatus(token []byte, status backend.StatusT, md defer m.Unlock() // Get existing record - treeID := treeIDFromToken(token) - r, err := t.tstore.RecordLatest(treeID) + r, err := t.tstore.RecordLatest(token) if err != nil { return nil, fmt.Errorf("RecordLatest: %v", err) } @@ -875,8 +845,7 @@ func (t *tstoreBackend) RecordSetStatus(token []byte, status backend.StatusT, md if err != nil { return nil, err } - err = t.tstore.PluginHookPre(treeID, token, - plugins.HookTypeSetRecordStatusPre, string(b)) + err = t.tstore.PluginHookPre(plugins.HookTypeSetRecordStatusPre, string(b)) if err != nil { return nil, err } @@ -908,8 +877,7 @@ func (t *tstoreBackend) RecordSetStatus(token []byte, status backend.StatusT, md backend.Statuses[status], status) // Call post plugin hooks - t.tstore.PluginHookPost(treeID, token, - plugins.HookTypeSetRecordStatusPost, string(b)) + t.tstore.PluginHookPost(plugins.HookTypeSetRecordStatusPost, string(b)) // Update inventory cache switch status { @@ -921,9 +889,9 @@ func (t *tstoreBackend) RecordSetStatus(token []byte, status backend.StatusT, md } // Return updated record - r, err = t.tstore.RecordLatest(treeID) + r, err = t.tstore.RecordLatest(token) if err != nil { - return nil, fmt.Errorf("RecordLatest %v: %v", treeID, err) + return nil, fmt.Errorf("RecordLatest: %v", err) } return r, nil @@ -931,11 +899,11 @@ func (t *tstoreBackend) RecordSetStatus(token []byte, status backend.StatusT, md // RecordExists returns whether a record exists. // -// This method relies on the the tstore tree exists call. It's possible for a -// tree to exist that does not correspond to a record in the rare case that a -// tree was created but an unexpected error, such as a network error, was -// encoutered prior to the record being saved to the tree. We ignore this edge -// case because: +// This method only returns whether a tree exists for the provided token. It's +// possible for a tree to exist that does not correspond to a record in the +// rare case that a tree was created but an unexpected error, such as a network +// error, was encoutered prior to the record being saved to the tree. We ignore +// this edge case because: // // 1. A user has no way to obtain this token unless the trillian instance has // been opened to the public. @@ -961,12 +929,11 @@ func (t *tstoreBackend) RecordExists(token []byte) bool { return false } - treeID := treeIDFromToken(token) - return t.tstore.TreeExists(treeID) + return t.tstore.RecordExists(token) } -// RecordTimestamps returns the timestamps for a record. If no version is provided -// then timestamps for the most recent version will be returned. +// RecordTimestamps returns the timestamps for a record. If no version is +// provided then timestamps for the most recent version will be returned. // // This function satisfies the backendv2 Backend interface. func (t *tstoreBackend) RecordTimestamps(token []byte, version uint32) (*backend.RecordTimestamps, error) { @@ -980,8 +947,7 @@ func (t *tstoreBackend) RecordTimestamps(token []byte, version uint32) (*backend return nil, err } - treeID := treeIDFromToken(token) - return t.tstore.RecordTimestamps(treeID, version, token) + return t.tstore.RecordTimestamps(token, version) } // Records retreives a batch of records. Individual record errors are not @@ -1002,8 +968,7 @@ func (t *tstoreBackend) Records(reqs []backend.RecordRequest) (map[string]backen } // Lookup the record - treeID := treeIDFromToken(token) - r, err := t.tstore.RecordPartial(treeID, v.Version, + r, err := t.tstore.RecordPartial(token, v.Version, v.Filenames, v.OmitAllFiles) if err != nil { if err == backend.ErrRecordNotFound { @@ -1093,7 +1058,6 @@ func (t *tstoreBackend) PluginRead(token []byte, pluginID, pluginCmd, payload st // The token is optional. If a token is not provided then a tree ID // will not be provided to the plugin. - var treeID int64 if len(token) > 0 { // Read methods are allowed to use short tokens. Lookup the full // length token. @@ -1107,11 +1071,9 @@ func (t *tstoreBackend) PluginRead(token []byte, pluginID, pluginCmd, payload st if !t.RecordExists(token) { return "", backend.ErrRecordNotFound } - - treeID = treeIDFromToken(token) } - return t.tstore.PluginCmd(treeID, token, pluginID, pluginCmd, payload) + return t.tstore.PluginCmd(token, pluginID, pluginCmd, payload) } // PluginWrite executes a plugin command that writes data. @@ -1136,8 +1098,8 @@ func (t *tstoreBackend) PluginWrite(token []byte, pluginID, pluginCmd, payload s defer m.Unlock() // Call pre plugin hooks - treeID := treeIDFromToken(token) hp := plugins.HookPluginPre{ + Token: token, PluginID: pluginID, Cmd: pluginCmd, Payload: payload, @@ -1146,15 +1108,13 @@ func (t *tstoreBackend) PluginWrite(token []byte, pluginID, pluginCmd, payload s if err != nil { return "", err } - err = t.tstore.PluginHookPre(treeID, token, - plugins.HookTypePluginPre, string(b)) + err = t.tstore.PluginHookPre(plugins.HookTypePluginPre, string(b)) if err != nil { return "", err } // Execute plugin command - reply, err := t.tstore.PluginCmd(treeID, token, - pluginID, pluginCmd, payload) + reply, err := t.tstore.PluginCmd(token, pluginID, pluginCmd, payload) if err != nil { return "", err } @@ -1170,8 +1130,7 @@ func (t *tstoreBackend) PluginWrite(token []byte, pluginID, pluginCmd, payload s if err != nil { return "", err } - t.tstore.PluginHookPost(treeID, token, - plugins.HookTypePluginPost, string(b)) + t.tstore.PluginHookPost(plugins.HookTypePluginPost, string(b)) return reply, nil } @@ -1207,18 +1166,15 @@ func (t *tstoreBackend) setup() error { log.Infof("Building backend token prefix cache") - // A record token is created using the unvetted tree ID so we - // only need to retrieve the unvetted trees in order to build the - // token prefix cache. - treeIDs, err := t.tstore.TreesAll() + tokens, err := t.tstore.Inventory() if err != nil { - return fmt.Errorf("TreesAll: %v", err) + return fmt.Errorf("tstore Inventory: %v", err) } - log.Infof("%v records in the backend", len(treeIDs)) + log.Infof("%v records in the backend", len(tokens)) - for _, v := range treeIDs { - t.tokenAdd(tokenFromTreeID(v)) + for _, v := range tokens { + t.tokenAdd(v) } return nil From dd7af48ddd13ff7bf8a3ecec9a3d3b8e08a7493e Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 28 Mar 2021 11:36:32 -0500 Subject: [PATCH 443/449] tstorbe: Move short token cache to tstore layer. --- politeiad/backendv2/tstorebe/testing.go | 1 - politeiad/backendv2/tstorebe/tstore/plugin.go | 31 ++- politeiad/backendv2/tstorebe/tstore/record.go | 94 ++++++++- politeiad/backendv2/tstorebe/tstore/tstore.go | 104 ++++++++- politeiad/backendv2/tstorebe/tstorebe.go | 199 ++---------------- 5 files changed, 242 insertions(+), 187 deletions(-) diff --git a/politeiad/backendv2/tstorebe/testing.go b/politeiad/backendv2/tstorebe/testing.go index a4c092d6f..db714cddb 100644 --- a/politeiad/backendv2/tstorebe/testing.go +++ b/politeiad/backendv2/tstorebe/testing.go @@ -30,7 +30,6 @@ func NewTestTstoreBackend(t *testing.T) (*tstoreBackend, func()) { appDir: appDir, dataDir: dataDir, tstore: tstore.NewTestTstore(t, dataDir), - tokens: make(map[string][]byte), recordMtxs: make(map[string]*sync.Mutex), } diff --git a/politeiad/backendv2/tstorebe/tstore/plugin.go b/politeiad/backendv2/tstorebe/tstore/plugin.go index 762c07186..91030ec25 100644 --- a/politeiad/backendv2/tstorebe/tstore/plugin.go +++ b/politeiad/backendv2/tstorebe/tstore/plugin.go @@ -178,9 +178,34 @@ func (t *Tstore) PluginHookPost(h plugins.HookT, payload string) { } } -// PluginCmd executes a plugin command. -func (t *Tstore) PluginCmd(token []byte, pluginID, cmd, payload string) (string, error) { - log.Tracef("PluginCmd: %x %v %v", token, pluginID, cmd) +// PluginRead executes a read-only plugin command. +func (t *Tstore) PluginRead(token []byte, pluginID, cmd, payload string) (string, error) { + log.Tracef("PluginRead: %x %v %v", token, pluginID, cmd) + + // The token is optional + if len(token) > 0 { + // Read methods are allowed to use short tokens. Lookup the full + // length token. + var err error + token, err = t.fullLengthToken(token) + if err != nil { + return "", err + } + } + + // Get plugin + p, ok := t.plugin(pluginID) + if !ok { + return "", backend.ErrPluginIDInvalid + } + + // Execute plugin command + return p.client.Cmd(token, cmd, payload) +} + +// PluginWrite executes a plugin command that writes data. +func (t *Tstore) PluginWrite(token []byte, pluginID, cmd, payload string) (string, error) { + log.Tracef("PluginWrite: %x %v %v", token, pluginID, cmd) // Get plugin p, ok := t.plugin(pluginID) diff --git a/politeiad/backendv2/tstorebe/tstore/record.go b/politeiad/backendv2/tstorebe/tstore/record.go index a4b7a482b..4151d3c08 100644 --- a/politeiad/backendv2/tstorebe/tstore/record.go +++ b/politeiad/backendv2/tstorebe/tstore/record.go @@ -31,11 +31,29 @@ const ( // that serves as the unique identifier for the record. Creating a new record // means creating a tlog tree for the record. Nothing is saved to the tree yet. func (t *Tstore) RecordNew() ([]byte, error) { - tree, _, err := t.tlog.TreeNew() - if err != nil { - return nil, err + var token []byte + for retries := 0; retries < 10; retries++ { + tree, _, err := t.tlog.TreeNew() + if err != nil { + return nil, err + } + token = tokenFromTreeID(tree.TreeId) + + // Check for shortened token collisions + if t.tokenCollision(token) { + // This is a collision. We cannot use this tree. Try again. + log.Infof("Token collision %x, creating new token", token) + continue + } + + // We've found a valid token. Update the tokens cache. This must + // be done even if the record creation fails since the tree will + // still exist. + t.tokenAdd(token) + break } - return tokenFromTreeID(tree.TreeId), nil + + return token, nil } // recordSave saves the provided record content to the kv store, appends a leaf @@ -425,6 +443,12 @@ func (t *Tstore) recordSave(treeID int64, recordMD backend.RecordMetadata, metad func (t *Tstore) RecordSave(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { log.Tracef("RecordSave: %x", token) + // Verify token is valid. The full length token must be used when + // writing data. + if !tokenIsFullLength(token) { + return backend.ErrTokenInvalid + } + // Save the record treeID := treeIDFromToken(token) idx, err := t.recordSave(treeID, rm, metadata, files) @@ -447,6 +471,12 @@ func (t *Tstore) RecordSave(token []byte, rm backend.RecordMetadata, metadata [] func (t *Tstore) RecordDel(token []byte) error { log.Tracef("RecordDel: %x", token) + // Verify token is valid. The full length token must be used when + // writing data. + if !tokenIsFullLength(token) { + return backend.ErrTokenInvalid + } + // Get all tree leaves treeID := treeIDFromToken(token) leavesAll, err := t.leavesAll(treeID) @@ -512,6 +542,12 @@ func (t *Tstore) RecordDel(token []byte) error { func (t *Tstore) RecordFreeze(token []byte, rm backend.RecordMetadata, metadata []backend.MetadataStream, files []backend.File) error { log.Tracef("RecordFreeze: %x", token) + // Verify token is valid. The full length token must be used when + // writing data. + if !tokenIsFullLength(token) { + return backend.ErrTokenInvalid + } + // Save updated record treeID := treeIDFromToken(token) idx, err := t.recordSave(treeID, rm, metadata, files) @@ -546,7 +582,15 @@ func (t *Tstore) RecordFreeze(token []byte, rm backend.RecordMetadata, metadata // light weight. Its for this reason that we rely on the tree exists call // despite the edge case. func (t *Tstore) RecordExists(token []byte) bool { - _, err := t.tlog.Tree(treeIDFromToken(token)) + // Read methods are allowed to use short tokens. Lookup the full + // length token. + var err error + token, err = t.fullLengthToken(token) + if err != nil { + return false + } + + _, err = t.tlog.Tree(treeIDFromToken(token)) return err == nil } @@ -729,6 +773,14 @@ func (t *Tstore) record(treeID int64, version uint32, filenames []string, omitAl func (t *Tstore) Record(token []byte, version uint32) (*backend.Record, error) { log.Tracef("Record: %x %v", token, version) + // Read methods are allowed to use short tokens. Lookup the full + // length token. + var err error + token, err = t.fullLengthToken(token) + if err != nil { + return nil, err + } + treeID := treeIDFromToken(token) return t.record(treeID, version, []string{}, false) } @@ -737,6 +789,14 @@ func (t *Tstore) Record(token []byte, version uint32) (*backend.Record, error) { func (t *Tstore) RecordLatest(token []byte) (*backend.Record, error) { log.Tracef("RecordLatest: %x", token) + // Read methods are allowed to use short tokens. Lookup the full + // length token. + var err error + token, err = t.fullLengthToken(token) + if err != nil { + return nil, err + } + treeID := treeIDFromToken(token) return t.record(treeID, 0, []string{}, false) } @@ -757,6 +817,14 @@ func (t *Tstore) RecordPartial(token []byte, version uint32, filenames []string, log.Tracef("RecordPartial: %x %v %v %v", token, version, omitAllFiles, filenames) + // Read methods are allowed to use short tokens. Lookup the full + // length token. + var err error + token, err = t.fullLengthToken(token) + if err != nil { + return nil, err + } + treeID := treeIDFromToken(token) return t.record(treeID, version, filenames, omitAllFiles) } @@ -767,6 +835,14 @@ func (t *Tstore) RecordPartial(token []byte, version uint32, filenames []string, func (t *Tstore) RecordState(token []byte) (backend.StateT, error) { log.Tracef("RecordState: %x", token) + // Read methods are allowed to use short tokens. Lookup the full + // length token. + var err error + token, err = t.fullLengthToken(token) + if err != nil { + return backend.StateInvalid, err + } + treeID := treeIDFromToken(token) leaves, err := t.leavesAll(treeID) if err != nil { @@ -924,6 +1000,14 @@ func (t *Tstore) timestamp(treeID int64, merkleLeafHash []byte, leaves []*trilli func (t *Tstore) RecordTimestamps(token []byte, version uint32) (*backend.RecordTimestamps, error) { log.Tracef("RecordTimestamps: %x %v", token, version) + // Read methods are allowed to use short tokens. Lookup the full + // length token. + var err error + token, err = t.fullLengthToken(token) + if err != nil { + return nil, err + } + // Get record index treeID := treeIDFromToken(token) leaves, err := t.leavesAll(treeID) diff --git a/politeiad/backendv2/tstorebe/tstore/tstore.go b/politeiad/backendv2/tstorebe/tstore/tstore.go index 859b8d4a3..c91e9884e 100644 --- a/politeiad/backendv2/tstorebe/tstore/tstore.go +++ b/politeiad/backendv2/tstorebe/tstore/tstore.go @@ -13,10 +13,12 @@ import ( "sync" "github.com/decred/dcrd/chaincfg/v3" + backend "github.com/decred/politeia/politeiad/backendv2" "github.com/decred/politeia/politeiad/backendv2/tstorebe/plugins" "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" "github.com/decred/politeia/politeiad/backendv2/tstorebe/store/localdb" "github.com/decred/politeia/politeiad/backendv2/tstorebe/store/mysql" + "github.com/decred/politeia/util" "github.com/robfig/cron" ) @@ -60,7 +62,7 @@ var ( // they are considered to be orphaned and are simply ignored. We do not unwind // failed calls. type Tstore struct { - sync.Mutex + sync.RWMutex dataDir string activeNetParams *chaincfg.Params tlog tlogClient @@ -73,18 +75,99 @@ type Tstore struct { // dropping an anchor, i.e. timestamping unanchored tlog trees // using dcrtime. An anchor is dropped periodically using cron. droppingAnchor bool + + // tokens contains the short token to full token mappings. The + // short token is the first n characters of the hex encoded record + // token, where n is defined by the short token length politeiad + // setting. Record lookups using short tokens are allowed. This + // cache is used to prevent collisions when creating new tokens + // and to facilitate lookups using only the short token. This cache + // is built on startup. + tokens map[string][]byte // [shortToken]fullToken + } +// tokenFromTreeID returns the record token for a tlog tree. func tokenFromTreeID(treeID int64) []byte { b := make([]byte, 8) binary.LittleEndian.PutUint64(b, uint64(treeID)) return b } +// treeIDFromToken returns the tlog tree ID for the given record token. func treeIDFromToken(token []byte) int64 { return int64(binary.LittleEndian.Uint64(token)) } +// tokenIsFullLength returns whether the token is a full length token. +func tokenIsFullLength(token []byte) bool { + return util.TokenIsFullLength(util.TokenTypeTstore, token) +} + +// tokenCollision returns whether the short version of the provided token +// already exists. This can be used to prevent collisions when creating new +// tokens. +func (t *Tstore) tokenCollision(fullToken []byte) bool { + shortToken, err := util.ShortTokenEncode(fullToken) + if err != nil { + return false + } + + t.RLock() + defer t.RUnlock() + + _, ok := t.tokens[shortToken] + return ok +} + +// tokenAdd adds a entry to the tokens cache. +func (t *Tstore) tokenAdd(fullToken []byte) error { + if !tokenIsFullLength(fullToken) { + return fmt.Errorf("token is not full length") + } + + shortToken, err := util.ShortTokenEncode(fullToken) + if err != nil { + return err + } + + t.Lock() + t.tokens[shortToken] = fullToken + t.Unlock() + + log.Tracef("Token cache add: %v", shortToken) + + return nil +} + +// fullLengthToken returns the full length token given the short token. A +// ErrRecordNotFound error is returned if a record does not exist for the +// provided token. +func (t *Tstore) fullLengthToken(token []byte) ([]byte, error) { + if tokenIsFullLength(token) { + // Token is already full length. Nothing else to do. + return token, nil + } + + shortToken, err := util.ShortTokenEncode(token) + if err != nil { + // Token was not large enough to be a short token. This cannot + // be used to lookup a record. + return nil, backend.ErrRecordNotFound + } + + t.RLock() + defer t.RUnlock() + + fullToken, ok := t.tokens[shortToken] + if !ok { + // Short token does not correspond to a record token + return nil, backend.ErrRecordNotFound + } + + return fullToken, nil +} + // Fsck performs a filesystem check on the tstore. func (t *Tstore) Fsck() { // Set tree status to frozen for any trees that are frozen and have @@ -101,6 +184,24 @@ func (t *Tstore) Close() { t.store.Close() } +// Setup performs any required work to setup the tstore instance. +func (t *Tstore) Setup() error { + log.Infof("Building backend token prefix cache") + + tokens, err := t.Inventory() + if err != nil { + return fmt.Errorf("Inventory: %v", err) + } + + log.Infof("%v records in the tstore", len(tokens)) + + for _, v := range tokens { + t.tokenAdd(v) + } + + return nil +} + // New returns a new tstore instance. func New(appDir, dataDir string, anp *chaincfg.Params, tlogHost, tlogPass, dbType, dbHost, dbPass, dcrtimeHost, dcrtimeCert string) (*Tstore, error) { // Setup datadir for this tstore instance @@ -168,6 +269,7 @@ func New(appDir, dataDir string, anp *chaincfg.Params, tlogHost, tlogPass, dbTyp dcrtime: dcrtimeClient, cron: cron.New(), plugins: make(map[string]plugin), + tokens: make(map[string][]byte), } // Launch cron diff --git a/politeiad/backendv2/tstorebe/tstorebe.go b/politeiad/backendv2/tstorebe/tstorebe.go index ce080699e..aa01dcf5b 100644 --- a/politeiad/backendv2/tstorebe/tstorebe.go +++ b/politeiad/backendv2/tstorebe/tstorebe.go @@ -37,15 +37,6 @@ type tstoreBackend struct { shutdown bool tstore *tstore.Tstore - // tokens contains the short token to full token mappings. The - // short token is the first n characters of the hex encoded record - // token, where n is defined by the short token length politeiad - // setting. Record lookups using short tokens are allowed. This - // cache is used to prevent collisions when creating new tokens - // and to facilitate lookups using only the short token. This cache - // is built on startup. - tokens map[string][]byte // [shortToken]fullToken - // recordMtxs allows the backend to hold a lock on an individual // record so that it can perform multiple read/write operations // in a concurrent safe manner. These mutexes are lazy loaded. @@ -76,74 +67,6 @@ func (t *tstoreBackend) recordMutex(token []byte) *sync.Mutex { return m } -// tokenCollision returns whether the short version of the provided token -// already exists. This can be used to prevent collisions when creating new -// tokens. -func (t *tstoreBackend) tokenCollision(fullToken []byte) bool { - shortToken, err := util.ShortTokenEncode(fullToken) - if err != nil { - return false - } - - t.RLock() - defer t.RUnlock() - - _, ok := t.tokens[shortToken] - return ok -} - -// tokenAdd adds a entry to the tokens cache. -func (t *tstoreBackend) tokenAdd(fullToken []byte) error { - if !tokenIsFullLength(fullToken) { - return fmt.Errorf("token is not full length") - } - - shortToken, err := util.ShortTokenEncode(fullToken) - if err != nil { - return err - } - - t.Lock() - t.tokens[shortToken] = fullToken - t.Unlock() - - log.Tracef("Token cache add: %v", shortToken) - - return nil -} - -// fullLengthToken returns the full length token given the short token. A -// ErrRecordNotFound error is returned if a record does not exist for the -// provided token. -func (t *tstoreBackend) fullLengthToken(token []byte) ([]byte, error) { - if tokenIsFullLength(token) { - // Token is already full length. Nothing else to do. - return token, nil - } - - shortToken, err := util.ShortTokenEncode(token) - if err != nil { - // Token was not large enough to be a short token. This cannot - // be used to lookup a record. - return nil, backend.ErrRecordNotFound - } - - t.RLock() - defer t.RUnlock() - - fullToken, ok := t.tokens[shortToken] - if !ok { - // Short token does not correspond to a record token - return nil, backend.ErrRecordNotFound - } - - return fullToken, nil -} - -func tokenIsFullLength(token []byte) bool { - return util.TokenIsFullLength(util.TokenTypeTstore, token) -} - // metadataStreamsVerify verifies that all provided metadata streams are sane. func metadataStreamsVerify(metadata []backend.MetadataStream) error { // Verify metadata @@ -357,6 +280,7 @@ func filesVerify(files []backend.File, filesDel []string) error { return nil } +// filesUpdate updates the current files with new file adds and deletes. func filesUpdate(filesCurr, filesAdd []backend.File, filesDel []string) []backend.File { // Put current files into a map curr := make(map[string]backend.File, len(filesCurr)) // [filename]File @@ -386,6 +310,7 @@ func filesUpdate(filesCurr, filesAdd []backend.File, filesDel []string) []backen return f } +// recordMetadataNew returns a new record metadata. func recordMetadataNew(token []byte, files []backend.File, state backend.StateT, status backend.StatusT, version, iteration uint32) (*backend.RecordMetadata, error) { digests := make([]string, 0, len(files)) for _, v := range files { @@ -410,7 +335,7 @@ func recordMetadataNew(token []byte, files []backend.File, state backend.StateT, // // This function satisfies the backendv2 Backend interface. func (t *tstoreBackend) RecordNew(metadata []backend.MetadataStream, files []backend.File) (*backend.Record, error) { - log.Tracef("RecordNew") + log.Tracef("RecordNew: %v metadata, %v files", len(metadata), len(files)) // Verify record content err := metadataStreamsVerify(metadata) @@ -437,25 +362,9 @@ func (t *tstoreBackend) RecordNew(metadata []backend.MetadataStream, files []bac } // Create a new token - var token []byte - for retries := 0; retries < 10; retries++ { - token, err = t.tstore.RecordNew() - if err != nil { - return nil, err - } - - // Check for shortened token collisions - if t.tokenCollision(token) { - // This is a collision. We cannot use this tree. Try again. - log.Infof("Token collision %x, creating new token", token) - continue - } - - // We've found a valid token. Update the tokens cache. This must - // be done even if the record creation fails since the tree will - // still exist in tstore. - t.tokenAdd(token) - break + token, err := t.tstore.RecordNew() + if err != nil { + return nil, err } // Create record metadata @@ -514,12 +423,6 @@ func (t *tstoreBackend) RecordEdit(token []byte, mdAppend, mdOverwrite []backend return nil, err } - // Verify token is valid. The full length token must be used when - // writing data. - if !tokenIsFullLength(token) { - return nil, backend.ErrTokenInvalid - } - // Verify record exists if !t.RecordExists(token) { return nil, backend.ErrRecordNotFound @@ -618,12 +521,6 @@ func (t *tstoreBackend) RecordEditMetadata(token []byte, mdAppend, mdOverwrite [ return nil, backend.ErrNoRecordChanges } - // Verify token is valid. The full length token must be used when - // writing data. - if !tokenIsFullLength(token) { - return nil, backend.ErrTokenInvalid - } - // Verify record exists if !t.RecordExists(token) { return nil, backend.ErrRecordNotFound @@ -777,12 +674,6 @@ func (t *tstoreBackend) setStatusCensored(token []byte, rm backend.RecordMetadat func (t *tstoreBackend) RecordSetStatus(token []byte, status backend.StatusT, mdAppend, mdOverwrite []backend.MetadataStream) (*backend.Record, error) { log.Tracef("RecordSetStatus: %x %v", token, status) - // Verify token is valid. The full length token must be used when - // writing data. - if !tokenIsFullLength(token) { - return nil, backend.ErrTokenInvalid - } - // Verify record exists if !t.RecordExists(token) { return nil, backend.ErrRecordNotFound @@ -921,14 +812,6 @@ func (t *tstoreBackend) RecordSetStatus(token []byte, status backend.StatusT, md func (t *tstoreBackend) RecordExists(token []byte) bool { log.Tracef("RecordExists: %x", token) - // Read methods are allowed to use short tokens. Lookup the full - // length token. - var err error - token, err = t.fullLengthToken(token) - if err != nil { - return false - } - return t.tstore.RecordExists(token) } @@ -939,14 +822,6 @@ func (t *tstoreBackend) RecordExists(token []byte) bool { func (t *tstoreBackend) RecordTimestamps(token []byte, version uint32) (*backend.RecordTimestamps, error) { log.Tracef("RecordTimestamps: %x %v", token, version) - // Read methods are allowed to use short tokens. Lookup the full - // length token. - var err error - token, err = t.fullLengthToken(token) - if err != nil { - return nil, err - } - return t.tstore.RecordTimestamps(token, version) } @@ -960,26 +835,18 @@ func (t *tstoreBackend) Records(reqs []backend.RecordRequest) (map[string]backen records := make(map[string]backend.Record, len(reqs)) // [token]Record for _, v := range reqs { - // Read methods are allowed to use short tokens. Lookup the full - // length token. - token, err := t.fullLengthToken(v.Token) - if err != nil { - return nil, err - } - // Lookup the record - r, err := t.tstore.RecordPartial(token, v.Version, + r, err := t.tstore.RecordPartial(v.Token, v.Version, v.Filenames, v.OmitAllFiles) if err != nil { if err == backend.ErrRecordNotFound { - log.Debugf("Record not found %x", token) - // Record doesn't exist. This is ok. It will not be included // in the reply. + log.Debugf("Record not found %x", v.Token) continue } // An unexpected error occurred. Log it and continue. - log.Errorf("RecordPartial %x: %v", token, err) + log.Errorf("RecordPartial %x: %v", v.Token, err) continue } @@ -1056,24 +923,14 @@ func (t *tstoreBackend) PluginSetup(pluginID string) error { func (t *tstoreBackend) PluginRead(token []byte, pluginID, pluginCmd, payload string) (string, error) { log.Tracef("PluginRead: %x %v %v", token, pluginID, pluginCmd) - // The token is optional. If a token is not provided then a tree ID - // will not be provided to the plugin. - if len(token) > 0 { - // Read methods are allowed to use short tokens. Lookup the full - // length token. - var err error - token, err = t.fullLengthToken(token) - if err != nil { - return "", err - } - - // Verify record exists - if !t.RecordExists(token) { - return "", backend.ErrRecordNotFound - } + // Verify record exists if a token was provided. The token is + // optional on read commands so one may not exist. + if len(token) > 0 && !t.RecordExists(token) { + return "", backend.ErrRecordNotFound } - return t.tstore.PluginCmd(token, pluginID, pluginCmd, payload) + // Execute plugin command + return t.tstore.PluginRead(token, pluginID, pluginCmd, payload) } // PluginWrite executes a plugin command that writes data. @@ -1093,6 +950,9 @@ func (t *tstoreBackend) PluginWrite(token []byte, pluginID, pluginCmd, payload s // Hold the record lock for the remainder of this function. We // do this here in the backend so that the individual plugins // implementations don't need to worry about race conditions. + if t.isShutdown() { + return "", backend.ErrShutdown + } m := t.recordMutex(token) m.Lock() defer m.Unlock() @@ -1114,7 +974,7 @@ func (t *tstoreBackend) PluginWrite(token []byte, pluginID, pluginCmd, payload s } // Execute plugin command - reply, err := t.tstore.PluginCmd(token, pluginID, pluginCmd, payload) + reply, err := t.tstore.PluginWrite(token, pluginID, pluginCmd, payload) if err != nil { return "", err } @@ -1160,24 +1020,9 @@ func (t *tstoreBackend) Close() { t.tstore.Close() } -// setup builds the tstore backend memory caches. +// setup performs any required work to setup the tstore instance. func (t *tstoreBackend) setup() error { - log.Tracef("setup") - - log.Infof("Building backend token prefix cache") - - tokens, err := t.tstore.Inventory() - if err != nil { - return fmt.Errorf("tstore Inventory: %v", err) - } - - log.Infof("%v records in the backend", len(tokens)) - - for _, v := range tokens { - t.tokenAdd(v) - } - - return nil + return t.tstore.Setup() } // New returns a new tstoreBackend. @@ -1194,10 +1039,10 @@ func New(appDir, dataDir string, anp *chaincfg.Params, tlogHost, tlogPass, dbTyp appDir: appDir, dataDir: dataDir, tstore: ts, - tokens: make(map[string][]byte), recordMtxs: make(map[string]*sync.Mutex), } + // Perform any required setup err = t.setup() if err != nil { return nil, fmt.Errorf("setup: %v", err) From ec1b3aa8e06c4473b723a9c33364953b5778775d Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 28 Mar 2021 12:11:58 -0500 Subject: [PATCH 444/449] pd/ticketvote: Start vote bug fix. --- .../tstorebe/plugins/ticketvote/cmds.go | 28 ++++++++++++------- politeiad/backendv2/tstorebe/tstore/tstore.go | 1 - 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go index 5f5af5ba9..bcfaff89c 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go @@ -461,20 +461,17 @@ func (p *ticketVotePlugin) startStandard(token []byte, s ticketvote.Start) (*tic return nil, err } - // Get vote blockchain data - sr, err := p.startReply(sd.Params.Duration) - if err != nil { - return nil, err - } - - // Verify record version + // Verify record status and version r, err := p.tstore.RecordPartial(token, 0, nil, true) if err != nil { return nil, fmt.Errorf("RecordPartial: %v", err) } - if r.RecordMetadata.State != backend.StateVetted { - // This should not be possible - return nil, fmt.Errorf("record is unvetted") + if r.RecordMetadata.Status != backend.StatusPublic { + return nil, backend.PluginError{ + PluginID: ticketvote.PluginID, + ErrorCode: uint32(ticketvote.ErrorCodeRecordStatusInvalid), + ErrorContext: "record is not public", + } } if sd.Params.Version != r.RecordMetadata.Version { return nil, backend.PluginError{ @@ -486,6 +483,12 @@ func (p *ticketVotePlugin) startStandard(token []byte, s ticketvote.Start) (*tic } } + // Get vote blockchain data + sr, err := p.startReply(sd.Params.Duration) + if err != nil { + return nil, err + } + // Verify vote authorization auths, err := p.auths(token) if err != nil { @@ -856,6 +859,11 @@ func (p *ticketVotePlugin) startRunoffForParent(token []byte, s ticketvote.Start } // startRunoff starts the voting period for all submissions in a runoff vote. +// It does this by first adding a startRunoffRecord to the runoff vote parent +// record. Once this has been successfully added the runoff vote is considered +// to have started. The voting period must now be started on all of the runoff +// vote submissions individually. If any of these calls fail, they can be +// retried. This function will pick up where it left off. func (p *ticketVotePlugin) startRunoff(token []byte, s ticketvote.Start) (*ticketvote.StartReply, error) { // Sanity check if len(s.Starts) == 0 { diff --git a/politeiad/backendv2/tstorebe/tstore/tstore.go b/politeiad/backendv2/tstorebe/tstore/tstore.go index c91e9884e..831d3169f 100644 --- a/politeiad/backendv2/tstorebe/tstore/tstore.go +++ b/politeiad/backendv2/tstorebe/tstore/tstore.go @@ -84,7 +84,6 @@ type Tstore struct { // and to facilitate lookups using only the short token. This cache // is built on startup. tokens map[string][]byte // [shortToken]fullToken - } // tokenFromTreeID returns the record token for a tlog tree. From 1cc3557a7b21d535efe22c2c5e6c05ceb6f3fff3 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 28 Mar 2021 12:17:36 -0500 Subject: [PATCH 445/449] tstore/mysql: Encrypted blobs nit. --- politeiad/backendv2/tstorebe/store/mysql/mysql.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/politeiad/backendv2/tstorebe/store/mysql/mysql.go b/politeiad/backendv2/tstorebe/store/mysql/mysql.go index 0256d7970..fa12bd823 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/mysql.go +++ b/politeiad/backendv2/tstorebe/store/mysql/mysql.go @@ -60,16 +60,25 @@ func (s *mysql) isShutdown() bool { return atomic.LoadUint64(&s.shutdown) != 0 } +// put saves the provided blobs to the kv store using the provided transaction. func (s *mysql) put(blobs map[string][]byte, encrypt bool, ctx context.Context, tx *sql.Tx) error { // Encrypt blobs if encrypt { + encrypted := make(map[string][]byte, len(blobs)) for k, v := range blobs { e, err := s.encrypt(ctx, tx, v) if err != nil { return fmt.Errorf("encrypt: %v", err) } - blobs[k] = e + encrypted[k] = e } + + // Sanity check + if len(encrypted) != len(blobs) { + return fmt.Errorf("unexpected number of encrypted blobs") + } + + blobs = encrypted } // Save blobs From ec0be69a498c01db875a10bf7d479618584aa7b5 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 28 Mar 2021 12:33:27 -0500 Subject: [PATCH 446/449] tstorebe/mysql: Add testing field. --- politeiad/backendv2/tstorebe/store/mysql/encrypt.go | 9 ++++++++- politeiad/backendv2/tstorebe/store/mysql/encrypt_test.go | 5 +++-- politeiad/backendv2/tstorebe/store/mysql/mysql.go | 3 +-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/politeiad/backendv2/tstorebe/store/mysql/encrypt.go b/politeiad/backendv2/tstorebe/store/mysql/encrypt.go index a83688843..bf2b3914d 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/encrypt.go +++ b/politeiad/backendv2/tstorebe/store/mysql/encrypt.go @@ -115,7 +115,7 @@ func (s *mysql) deriveEncryptionKey(password string) error { var emptyNonce = [24]byte{} -func (s *mysql) getDbNonce(ctx context.Context, tx *sql.Tx) ([24]byte, error) { +func (s *mysql) getDBNonce(ctx context.Context, tx *sql.Tx) ([24]byte, error) { // Get nonce value nonce, err := s.nonce(ctx, tx) if err != nil { @@ -146,6 +146,13 @@ func (s *mysql) getTestNonce(ctx context.Context, tx *sql.Tx) ([24]byte, error) return n.Current(), nil } +func (s *mysql) getNonce(ctx context.Context, tx *sql.Tx) ([24]byte, error) { + if s.testing { + return s.getTestNonce(ctx, tx) + } + return s.getDBNonce(ctx, tx) +} + func (s *mysql) encrypt(ctx context.Context, tx *sql.Tx, data []byte) ([]byte, error) { nonce, err := s.getNonce(ctx, tx) if err != nil { diff --git a/politeiad/backendv2/tstorebe/store/mysql/encrypt_test.go b/politeiad/backendv2/tstorebe/store/mysql/encrypt_test.go index a198d2f13..c40cda6da 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/encrypt_test.go +++ b/politeiad/backendv2/tstorebe/store/mysql/encrypt_test.go @@ -12,8 +12,9 @@ func TestEncryptDecrypt(t *testing.T) { blob := []byte("encryptmeyo") // setup fake context - s := &mysql{} - s.getNonce = s.getTestNonce + s := &mysql{ + testing: true, + } s.argon2idKey(password, util.NewArgon2Params()) // Encrypt and make sure cleartext isn't the same as the encypted blob. diff --git a/politeiad/backendv2/tstorebe/store/mysql/mysql.go b/politeiad/backendv2/tstorebe/store/mysql/mysql.go index fa12bd823..769f8f35b 100644 --- a/politeiad/backendv2/tstorebe/store/mysql/mysql.go +++ b/politeiad/backendv2/tstorebe/store/mysql/mysql.go @@ -48,8 +48,8 @@ var ( type mysql struct { shutdown uint64 db *sql.DB - getNonce func(context.Context, *sql.Tx) ([24]byte, error) key [32]byte + testing bool // Only set during unit tests } func ctxWithTimeout() (context.Context, func()) { @@ -324,7 +324,6 @@ func New(appDir, host, user, password, dbname string) (*mysql, error) { s := &mysql{ db: db, } - s.getNonce = s.getDbNonce // Derive encryption key from password. Key is set in argon2idKey err = s.deriveEncryptionKey(password) From 6cc1c2e5d3ac01609e17de406726580071803565 Mon Sep 17 00:00:00 2001 From: luke Date: Sun, 28 Mar 2021 14:03:27 -0500 Subject: [PATCH 447/449] pd/ticketvote: Fix runoff vote bug. --- politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go | 8 ++++++-- .../backendv2/tstorebe/plugins/ticketvote/internalcmds.go | 2 +- politeiawww/ticketvote/process.go | 8 ++++++++ util/token.go | 6 +++--- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go index bcfaff89c..33ec7a768 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/cmds.go @@ -611,7 +611,11 @@ func (p *ticketVotePlugin) startRunoffForSub(token []byte, srs startRunoffSubmis } // Get the start runoff record from the parent record - srr, err := p.startRunoffRecord(srs.ParentToken) + parent, err := tokenDecode(srs.ParentToken) + if err != nil { + return err + } + srr, err := p.startRunoffRecord(parent) if err != nil { return err } @@ -997,7 +1001,7 @@ func (p *ticketVotePlugin) startRunoff(token []byte, s ticketvote.Start) (*ticke return nil, err } srs := startRunoffSubmission{ - ParentToken: token, + ParentToken: v.Params.Parent, StartDetails: v, } b, err := json.Marshal(srs) diff --git a/politeiad/backendv2/tstorebe/plugins/ticketvote/internalcmds.go b/politeiad/backendv2/tstorebe/plugins/ticketvote/internalcmds.go index c924bd834..d54c9bc91 100644 --- a/politeiad/backendv2/tstorebe/plugins/ticketvote/internalcmds.go +++ b/politeiad/backendv2/tstorebe/plugins/ticketvote/internalcmds.go @@ -41,7 +41,7 @@ type startRunoffRecord struct { // startRunoffSubmission is an internal plugin command that is used to start // the voting period on a runoff vote submission. type startRunoffSubmission struct { - ParentToken []byte `json:"parenttoken"` + ParentToken string `json:"parenttoken"` StartDetails ticketvote.StartDetails `json:"startdetails"` } diff --git a/politeiawww/ticketvote/process.go b/politeiawww/ticketvote/process.go index 01f964bbf..07c378544 100644 --- a/politeiawww/ticketvote/process.go +++ b/politeiawww/ticketvote/process.go @@ -65,6 +65,14 @@ func (t *TicketVote) processAuthorize(ctx context.Context, a v1.Authorize, u use func (t *TicketVote) processStart(ctx context.Context, s v1.Start, u user.User) (*v1.StartReply, error) { log.Tracef("processStart: %v", len(s.Starts)) + // Verify there is work to be done + if len(s.Starts) == 0 { + return nil, v1.UserErrorReply{ + ErrorCode: v1.ErrorCodeInputInvalid, + ErrorContext: "no start details found", + } + } + // Verify user signed with their active identity for _, v := range s.Starts { if u.PublicKey() != v.PublicKey { diff --git a/util/token.go b/util/token.go index 830075f10..27cc1760d 100644 --- a/util/token.go +++ b/util/token.go @@ -57,7 +57,7 @@ func ShortToken(token []byte) ([]byte, error) { // for it. func ShortTokenString(token string) (string, error) { if tokenRegexp.FindString(token) == "" { - return "", fmt.Errorf("invalid token") + return "", fmt.Errorf("invalid token %v", tokenRegexp.String()) } return token[:pdv2.ShortTokenLength], nil } @@ -89,7 +89,7 @@ func TokenIsFullLength(tokenType string, token []byte) bool { func TokenDecode(tokenType, token string) ([]byte, error) { // Verify token is valid if tokenRegexp.FindString(token) == "" { - return nil, fmt.Errorf("invalid token") + return nil, fmt.Errorf("invalid token %v", tokenRegexp.String()) } // Decode token @@ -110,7 +110,7 @@ func TokenDecode(tokenType, token string) ([]byte, error) { func TokenDecodeAnyLength(tokenType, token string) ([]byte, error) { // Verify token is valid if tokenRegexp.FindString(token) == "" { - return nil, fmt.Errorf("invalid token") + return nil, fmt.Errorf("invalid token %v", tokenRegexp.String()) } // Decode token. If provided token has odd length, add padding From f921b1bcf97c4115c192a94d0129ade2025edb36 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 29 Mar 2021 08:08:19 -0500 Subject: [PATCH 448/449] Clean READMEs. --- README.md | 424 +----------------------------------------- politeiad/README.md | 49 +++-- politeiawww/README.md | 39 ++++ 3 files changed, 73 insertions(+), 439 deletions(-) create mode 100644 politeiawww/README.md diff --git a/README.md b/README.md index 72a5a9103..51dc5a745 100644 --- a/README.md +++ b/README.md @@ -31,427 +31,7 @@ The politeia stack is as follows: +-------------------------+ ``` -## Components - Core software: -* politeiad - Reference server daemon. -* politeiawww - Web backend server; depends on politeiad. - -### Tools and reference clients - -* [politeiawww_dbutil](https://github.com/decred/politeia/tree/master/politeiawww/cmd/politeiawww_dbutil) - Tool for updating the politeiawww user database manually. -* [piwww](https://github.com/decred/politeia/tree/master/politeiawww/cmd/piwww) - Command-line tool for interacting with the politeiawww pi API. -* [cmswww](https://github.com/decred/politeia/tree/master/politeiawww/cmd/cmswww) - Command-line tool for interacting with politeiawww cms API. - -**Note:** politeiawww does not provide HTML output. It strictly handles the -JSON REST RPC commands only. The GUI for politeiawww can be found at: -https://github.com/decred/politeiagui - -## Development - -#### 1. Install [Go](https://golang.org/doc/install) version 1.14 or higher, and [Git](https://git-scm.com/downloads). - -Make sure each of these are in the PATH. - -#### 2. Clone this repository. - -#### 3. Setup configuration files: - -politeiad and politeiawww both have configuration files that you should -set up to make execution easier. You should create the configuration files -under the following paths: - -* **macOS** - - ``` - /Users//Library/Application Support/Politeiad/politeiad.conf - /Users//Library/Application Support/Politeiawww/politeiawww.conf - ``` - -* **Windows** - - ``` - C:\Users\\AppData\Local\Politeiad/politeiad.conf - C:\Users\\AppData\Local\Politeiawww/politeiawww.conf - ``` - -* **Ubuntu** - - ``` - ~/.politeiad/politeiad.conf - ~/.politeiawww/politeiawww.conf - ``` - -Copy and change the [`sample-politeiawww.conf`](https://github.com/decred/politeia/blob/master/politeiawww/sample-politeiawww.conf) -and [`sample-politeiad.conf`](https://github.com/decred/politeia/blob/master/politeiad/sample-politeiad.conf) files. - -You can also use the following default configurations: - -**politeiad.conf**: - - rpcuser=user - rpcpass=pass - testnet=true - -**politeiawww.conf**: - - mode=piwww - rpchost=127.0.0.1 - rpcuser=user - rpcpass=pass - rpccert="~/.politeiad/https.cert" - testnet=true - paywallxpub=tpubVobLtToNtTq6TZNw4raWQok35PRPZou53vegZqNubtBTJMMFmuMpWybFCfweJ52N8uZJPZZdHE5SRnBBuuRPfC5jdNstfKjiAs8JtbYG9jx - paywallamount=10000000 - dbhost=localhost:26257 - dbrootcert="~/.cockroachdb/certs/clients/politeiawww/ca.crt" - dbcert="~/.cockroachdb/certs/clients/politeiawww/client.politeiawww.crt" - dbkey="~/.cockroachdb/certs/clients/politeiawww/client.politeiawww.key" - -**Things to note:** - -* The `rpccert` path is referencing a Linux path. See above for -more OS paths. - -* politeiawww uses an email server to send verification codes for -things like new user registration, and those settings are also configured within - `politeiawww.conf`. The current code should work with most SSL-based SMTP servers -(but not TLS) using username and password as authentication. - -#### 4. Setup politeiad cache: - -politeiad stores proposal data in git repositories that are regularly backed up -to github and cryptographically timestamped onto the Decred blockchain. The -politeiad git repositories serve as the source of truth for proposal data. A -CockroachDB database is used as a cache for proposal data in order to increase -query performance. - -**The cache is not required if you're running just politeiad. politeiad has -the cache disable by default. If you're running the full politeia stack, -politeiad and politeiawww, running the cache is required.** - -politeiad has read and write access to the cache. politeiawww has only read -access to the cache. The flow of data is as follows: - -1. politeiawww receives a command from a user -2. politeiawww creates a politeiad request for the command and sends it -3. politeiad writes new data to the git repository then updates the cache -4. politeiad returns the status of the update to politeiawww -5. politeiawww reads the updated data from the cache -6. politeiawww returns a response to the user - -We use CockroachDB for the cache in the instructions below. CockroachDB is -built to be compatible with Postgres so you can use Postgres for the cache if -you so choose. Using Postgres for the cache has not been thoroughly tested and -bugs may exist. - -Install CockroachDB using the instructions found in the [CockroachDB -Documentation](https://www.cockroachlabs.com/docs/stable/install-cockroachdb-mac.html). - -Run the following commands to create the CockroachDB certificates required for -running CockroachDB with Politeia. - - cd $GOPATH/src/github.com/decred/politeia - ./scripts/cockroachcerts.sh - -The script creates following certificates and directories. - - ~/.cockroachdb - ├── ca.key - └── certs - ├── ca.crt - ├── clients -    │    ├── politeiad -    │    │   ├── ca.crt -    │    │   ├── client.politeiad.crt -    │    │   └── client.politeiad.key -    │    ├── politeiawww -    │    │   ├── ca.crt -    │    │   ├── client.politeiawww.crt -    │    │   └── client.politeiawww.key - │   └── root - │   ├── ca.crt - │   ├── client.root.crt - │   └── client.root.key - └── node - ├── ca.crt - ├── node.crt - └── node.key - -These are the certificates required to run a CockroachDB node locally. This -includes creating a CA certificate, a node certificate, and client certificates -for the root user, politeiad user, and politeiawww user. The root user is used -to setup the databases and can be used to open a sql shell. Each client -directory contains all of the certificates required to connect to the database -with that user. - -The node directory contains the certificates for running a CockroachDB instance -on localhost. Directions for generating node certificates when deploying a -CockroachDB cluster can be found in the [CockroachDB manual deployment -docs](https://www.cockroachlabs.com/docs/stable/manual-deployment.html). - -You can now start CockroachDB using the command below. The `cachesetup.sh` -script that is run next requires that a CockroachDB is running. - - cockroach start-single-node \ - --certs-dir=${HOME}/.cockroachdb/certs/node \ - --listen-addr=localhost \ - --store=${HOME}/.cockroachdb/data - -Once CockroachDB is running, you can setup the cache databases using the -commands below. - - cd $GOPATH/src/github.com/decred/politeia - ./scripts/cachesetup.sh - -The database setup is now complete. If you want to run database commands -manually you can do so by opening a sql shell. - - cockroach sql \ - --certs-dir=${HOME}/.cockroachdb/certs/clients/root \ - --host localhost - -#### 4a. Setup cms database: - -CMS uses both the cache database and its own database. Once the cache database -has been setup using the instructions above, you can setup the CMS database -using the script below. CockroachDB must be running when you execute this -script. - - cd $GOPATH/src/github.com/decred/politeia - ./scripts/cmssetup.sh - -#### 4b. Setup github codetracker: - -CMS uses the gitub API to figure out pull request, review and commit information -of current users that have their domain set to "Developer". - -To use the code tracker, set the github apitoken, the org and the repos that -you would like for the code tracker to crawl. - -#### 5. Build the programs: - -Go 1.11 introduced [modules](https://github.com/golang/go/wiki/Modules), a new -dependency management approach, that obviates the need for third party tooling -such as `dep`. - -Usage is simple and nothing is required except Go 1.11. If building in a folder -under `GOPATH`, it is necessary to explicitly build with modules enabled: - -``` -cd $GOPATH/src/github.com/decred/politeia -export GO111MODULE=on -go install -v ./... -``` - -If building outside of `GOPATH`, modules are automatically enabled, and `go -install` is sufficient. - -``` -go install -v ./... -``` - -The go tool will process the source code and automatically download -dependencies. If the dependencies are configured correctly, there will be no -modifications to the `go.mod` and `go.sum` files. - -#### 6. Start the politeiad server by running on your terminal: - - politeiad - -#### 7. Download politeiad's identity to politeiawww: - - politeiawww --fetchidentity - -Accept politeiad's identity by pressing Enter. - -The result should look something like this: - -``` -2018-08-01 22:48:48.468 [INF] PWWW: Identity fetched from politeiad -2018-08-01 22:48:48.468 [INF] PWWW: Key : 331819226de0270d0c997749ce9f2b56bc5aed110f57faef8d381129e7ee6d26 -2018-08-01 22:48:48.468 [INF] PWWW: Fingerprint: MxgZIm3gJw0MmXdJzp8rVrxa7REPV/rvjTgRKefubSY= -2018-08-01 22:48:48.468 [INF] PWWW: Save to /Users//Library/Application Support/Politeiawww/identity.json or ctrl-c to abort - -2018-08-01 22:49:53.929 [INF] PWWW: Identity saved to: /Users//Library/Application Support/Politeiawww/identity.json -``` - -#### 8. Start the politeiawww server by running on your terminal: - - politeiawww - -**Awesome!** Now you have your Politeia servers up and running! - -At this point, you can: - -* Follow the instructions at [decred/politeiagui](https://github.com/decred/politeiagui) -to setup Politeia and access it through the UI. -* Use the [politeiawww](https://github.com/decred/politeia/tree/master/politeiawww/cmd/) tools to interact with politeiawww. -* Use the [politeia](https://github.com/decred/politeia/tree/master/politeiad/cmd/politeia) tool to interact directly with politeiad. -* Use any other tools or clients that are listed above. - - -### Further information - - -#### politeiawww user database options - -Both Pi and CMS use the same politeiawww user database. The default user -database is LevelDB, a simple key-value store. This is fine if you're just -getting started, but LevelDB has some scalability limitations due to it being a -simple key-value store that doesn't allow concurrent connections. - -A more scalable option is setting up the user database to use CockroachDB. The -CockroachDB implementation makes public user fields queryable and encrypts -private user data at rest. You can setup the user database to use CockroachDB -with the following commands. Before running these commands, make sure that -you've followed the instructions above and have a CockroachDB instance running. - -Create a CockroachDB user database and assign user privileges: - - cd $GOPATH/src/github.com/decred/politeia - ./scripts/userdbsetup.sh - -Create an encryption key to be used to encrypt data at rest: - - $ politeiawww_dbutil -createkey - Encryption key saved to: ~/.politeiawww/sbox.key - -Add the following settings to your politeiawww config file. The encryption key -location may be different depending on your operating system. - - userdb=cockroachdb - encryptionkey=~/.politeiawww/sbox.key - -##### Rotating encryption key - -Encryption keys can be rotated using the `oldencryptionkey` politeiawww config -setting. To rotate keys, set `oldencryptionkey` to the existing key and set -`encryptionkey` to the new key. Starting politeiawww with both of these config -params set will kick off a key rotation. - -##### Migrating LevelDB to CockroachDB - -If you need to migrate a LevelDB user database to CockroachDB, instructions are -provided in the README of -[politeiawww_dbutil](https://github.com/decred/politeia/tree/master/politeiawww/cmd/politeiawww_dbutil). - -#### Paywall - -This politeiawww feature prevents users from submitting new proposals and -comments until a payment in DCR has been paid. By default, it needs a -transaction with 2 confirmations to accept the payment. - -Setting up the paywall requires a master public key for an account to -derive payment addresses. You may either use one of the pre-generated test -keys (see [`sample-politeiawww.conf`](https://github.com/decred/politeia/blob/master/politeiawww/sample-politeiawww.conf)) -or you may acquire one by creating accounts and retrieving the public keys -for those accounts: - -Put the result of the following command as `paywallxpub=tpub...` in -`politeiawww.conf`: - -``` -dcrctl --wallet --testnet createnewaccount politeiapayments -dcrctl --wallet --testnet getmasterpubkey politeiapayments -``` - -If running with paywall enabled on testnet, it's possible to change the -minimum blocks required for accept the payment by setting `minconfirmations` -flag for politeiawww: - - politeiawww --minconfirmations=1 - - -##### Paywall with politeiawww_refclient - -When using politeiawww_refclient, the `-use-paywall` flag is true by default. When running the refclient without the paywall, set `-use-paywall=false`, but note that it will not be possible to test new proposals and comments or the `admin` flag. - -* To test the admin flow: - * Run the refclient once with paywall enabled and make the payment. - * Stop politeiawww. - * Set the user created in the first refclient execution as admin with politeiawww_dbutil. - * Run refclient again with the `email` and `password` flags set to the user created in the first refclient execution. - -#### Building with repository version - -It is often useful to have version information from the repository where -politeia was fetched and built from, such as the commit hash it is using. -To accomplish this, politeia needs to be built with `go get` from outside -of your local files. If you build using your local checked out repository, -the build information will return `(devel)` instead of the actual version, -since it was built locally on your development environment. If built -properly, and suppose politeia has a release of the version 1.0.0, it will -return `v1.0.0--`. This build version is logged on -startup and returned from the version API call. Below are examples on how -to build politeia from outside of `GOPATH` and your local repository: - -`GO111MODULE=on go get github.com/decred/politeia/...@master` - -This will fetch and install politeia from gh master branch, and will include -the build version information. If you need to add build flags and/or -environment variables, do it normally as building from source: - -`env GOOS=linux GOARCH=amd64 CGO_ENABLED=0 GO111MODULE=on go get -trimpath --tags 'net,go' github.com/decred/politeia/...@master@master` - -## Integrated Projects / External APIs / Official URLs - -* https://faucet.decred.org - instance of [testnetfaucet](https://github.com/decred/testnetfaucet) - which is used by **politeiawww_refclient** to satisfy paywall requests in an - automated fashion. - -* https://test-proposals.decred.org/ - testing/development instance of Politeia. - -* https://pi-staging.decred.org/ - politeia staging environment. - -* https://proposals.decred.org/ - live production instance of Politeia. - -## Library and interfaces - -* `politeiad/api/v1` - JSON REST API for politeiad clients. -* `politeiawww/api/v1` - JSON REST API for politeiawww clients. -* `util` - common used miscellaneous utility functions. - -## Misc - -#### nginx reverse proxy sample (testnet) - -``` -# politeiawww -location /api/ { - # disable caching - expires off; - - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_cache_bypass $http_upgrade; - - proxy_http_version 1.1; - proxy_ssl_trusted_certificate /path/to/politeiawww.crt; - proxy_ssl_verify on; - proxy_pass https://test-politeia.domain.com:4443/; -} - -# politeiagui -location / { - # redirect not found - error_page 404 =200 /; - proxy_intercept_errors on; - - # disable caching - expires off; - - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_http_version 1.1; - - # backend - proxy_pass http://127.0.0.1:8000; -} -``` +* politeiad - Reference server daemon. Data layer. +* politeiawww - Web backend server; depends on politeiad. User layer. diff --git a/politeiad/README.md b/politeiad/README.md index b3fbbf8d7..5c6553d1d 100644 --- a/politeiad/README.md +++ b/politeiad/README.md @@ -3,7 +3,7 @@ politeiad # Installing and running -## Install Dependencies +## Install dependencies
Go 1.14 or 1.15 @@ -38,11 +38,19 @@ politeiad ```
-## Build from source +
MySQL/MariaDB + + Installation instructions can be found at the links below. + MySQL: https://www.mysql.com + MariaDB: https://mariadb.com + +
-1. Install MariaDB or MySQL. Make sure to setup a password for the root user. -2. Update the MySQL max connections settings. +## Build from source + +1. Set the password for the MySQL root user and update the MySQL max + connections settings. Max connections defaults to 151 which is not enough for trillian. You will be prompted for the MySQL root user's password when running these commands. @@ -66,7 +74,7 @@ politeiad max_connections = 2000 ``` -3. Install trillian v1.3.13. +2. Install trillian v1.3.13. ``` $ mkdir -p $GOPATH/src/github.com/google/ @@ -77,7 +85,7 @@ politeiad $ go install -v ./... ``` -4. Install politeia. +3. Install politeia. ``` $ mkdir -p $GOPATH/src/github.com/decred @@ -87,16 +95,16 @@ politeiad $ go install -v ./... ``` -5. Run the politeiad mysql setup scripts. +4. Run the politeiad mysql setup scripts. This will create the politeiad and trillian users as well as creating the politeiad databases. Password authentication is used for all database connections. - **The password that you set for the politeiad user will be used to derive an - encryption key that is used to encrypt non-public politeiad data. Make sure - to setup a strong password when running in production. Once set, the - politeiad user password cannot change.** + **The password that you set for the politeiad MySQL user will be used to + derive an encryption key that is used to encrypt non-public data at rest. + Make sure to setup a strong password when running in production. Once set, + the politeiad user password cannot change.** The setup script assumes MySQL is running on `localhost:3306` and the users will be accessing the databse from `localhost`. See the setup script @@ -116,7 +124,7 @@ politeiad ./tstore-mysql-setup.sh ``` -6. Run the trillian mysql setup scripts. +5. Run the trillian mysql setup scripts. These can only be run once the trillian MySQL user has been created in the previous step. @@ -140,7 +148,7 @@ politeiad ./resetdb.sh ``` -7. Start up the trillian instances. +6. Start up the trillian instances. Running trillian requires running a trillian log server and a trillian log signer. These are seperate processes that will be started in this step. @@ -186,7 +194,7 @@ politeiad --http_endpoint=localhost:8093 ``` -8. Setup the politeiad configuration file. +7. Setup the politeiad configuration file. [`sample-politeiad.conf`](https://github.com/decred/politeia/blob/master/politeiad/sample-politeiad.conf) @@ -233,11 +241,18 @@ politeiad plugin=usermd ``` -9. Start up the politeiad instance. The password for the politeiad user must - be provided in the `DBPASS` env variable. +8. Start up the politeiad instance. + + The password for the politeiad MySQL user must be provided in the `DBPASS` + env variable. The encryption key used to encrypt non-public data at rest + will be derived from the `DBPASS`. The `DBPASS` cannot change. + + A password to derive the trillian signing key must be provided in the + `TLOGPASS` env variable. This password has not been created yet. You can + set it to whatever you want, but it cannot change once set. ``` - $ env DBPASS=politeiadpass politeiad + $ env DBPASS=politeiadpass TLOGPASS=tlogpass politeiad ``` # Tools and reference clients diff --git a/politeiawww/README.md b/politeiawww/README.md new file mode 100644 index 000000000..99dc7d559 --- /dev/null +++ b/politeiawww/README.md @@ -0,0 +1,39 @@ +politeiawww +==== + +# Installing and running + +## Install dependencies + +
Go 1.14 or 1.15 + + Installation instructions can be found here: https://golang.org/doc/install. + Ensure Go was installed properly and is a supported version: + + ```sh + $ go version + $ go env GOROOT GOPATH + ``` + + NOTE: `GOROOT` and `GOPATH` must not be on the same path. Since Go 1.8 + (2016), `GOROOT` and `GOPATH` are set automatically, and you do not need to + change them. However, you still need to add `$GOPATH/bin` to your `PATH` in + order to run binaries installed by `go get` and `go install` (On Windows, + this happens automatically). + + Unix example -- add these lines to .profile: + + ``` + PATH="$PATH:/usr/local/go/bin" # main Go binaries ($GOROOT/bin) + PATH="$PATH:$HOME/go/bin" # installed Go projects ($GOPATH/bin) + ``` +
+ +
Git + + Installation instructions can be found at https://git-scm.com or + https://gitforwindows.org. + ```sh + $ git version + ``` +
From d7cf0f0a0418290f124ee07265e24d0a657d5d48 Mon Sep 17 00:00:00 2001 From: luke Date: Mon, 29 Mar 2021 08:19:45 -0500 Subject: [PATCH 449/449] Clean READMEs. --- README.md | 45 +++++++++++++++++++++++++++++++++++++++++++ politeiad/README.md | 35 --------------------------------- politeiawww/README.md | 33 ++----------------------------- 3 files changed, 47 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index 51dc5a745..7f72d0c6d 100644 --- a/README.md +++ b/README.md @@ -35,3 +35,48 @@ Core software: * politeiad - Reference server daemon. Data layer. * politeiawww - Web backend server; depends on politeiad. User layer. + +# Installing and running + +## Install dependencies + +
Go 1.14 or 1.15 + + Installation instructions can be found here: https://golang.org/doc/install. + Ensure Go was installed properly and is a supported version: + + ```sh + $ go version + $ go env GOROOT GOPATH + ``` + + NOTE: `GOROOT` and `GOPATH` must not be on the same path. Since Go 1.8 + (2016), `GOROOT` and `GOPATH` are set automatically, and you do not need to + change them. However, you still need to add `$GOPATH/bin` to your `PATH` in + order to run binaries installed by `go get` and `go install` (On Windows, + this happens automatically). + + Unix example -- add these lines to .profile: + + ``` + PATH="$PATH:/usr/local/go/bin" # main Go binaries ($GOROOT/bin) + PATH="$PATH:$HOME/go/bin" # installed Go projects ($GOPATH/bin) + ``` +
+ +
Git + + Installation instructions can be found at https://git-scm.com or + https://gitforwindows.org. + ```sh + $ git version + ``` +
+ +## Build from source + +See the politeiad instructions for [building from +source](https://github.com/decred/politeia/tree/master/politeiad#build-from-source). + +See the politeiawww instructions for [building from +source](https://github.com/decred/politeia/tree/master/politeiawww#build-from-source). diff --git a/politeiad/README.md b/politeiad/README.md index 5c6553d1d..8043d5bfc 100644 --- a/politeiad/README.md +++ b/politeiad/README.md @@ -5,39 +5,6 @@ politeiad ## Install dependencies -
Go 1.14 or 1.15 - - Installation instructions can be found here: https://golang.org/doc/install. - Ensure Go was installed properly and is a supported version: - - ```sh - $ go version - $ go env GOROOT GOPATH - ``` - - NOTE: `GOROOT` and `GOPATH` must not be on the same path. Since Go 1.8 - (2016), `GOROOT` and `GOPATH` are set automatically, and you do not need to - change them. However, you still need to add `$GOPATH/bin` to your `PATH` in - order to run binaries installed by `go get` and `go install` (On Windows, - this happens automatically). - - Unix example -- add these lines to .profile: - - ``` - PATH="$PATH:/usr/local/go/bin" # main Go binaries ($GOROOT/bin) - PATH="$PATH:$HOME/go/bin" # installed Go projects ($GOPATH/bin) - ``` -
- -
Git - - Installation instructions can be found at https://git-scm.com or - https://gitforwindows.org. - ```sh - $ git version - ``` -
-
MySQL/MariaDB Installation instructions can be found at the links below. @@ -258,5 +225,3 @@ politeiad # Tools and reference clients * [politeia](https://github.com/decred/politeia/tree/master/politeiad/cmd/politeia) - Reference client for politeiad. - - diff --git a/politeiawww/README.md b/politeiawww/README.md index 99dc7d559..0e96ffbce 100644 --- a/politeiawww/README.md +++ b/politeiawww/README.md @@ -5,35 +5,6 @@ politeiawww ## Install dependencies -
Go 1.14 or 1.15 +## Build from source - Installation instructions can be found here: https://golang.org/doc/install. - Ensure Go was installed properly and is a supported version: - - ```sh - $ go version - $ go env GOROOT GOPATH - ``` - - NOTE: `GOROOT` and `GOPATH` must not be on the same path. Since Go 1.8 - (2016), `GOROOT` and `GOPATH` are set automatically, and you do not need to - change them. However, you still need to add `$GOPATH/bin` to your `PATH` in - order to run binaries installed by `go get` and `go install` (On Windows, - this happens automatically). - - Unix example -- add these lines to .profile: - - ``` - PATH="$PATH:/usr/local/go/bin" # main Go binaries ($GOROOT/bin) - PATH="$PATH:$HOME/go/bin" # installed Go projects ($GOPATH/bin) - ``` -
- -
Git - - Installation instructions can be found at https://git-scm.com or - https://gitforwindows.org. - ```sh - $ git version - ``` -
+Coming soon...